import time
import pandas as pd
import numpy as np

# 生成百萬級測試數(shù)據(jù)
data = {
    'id': np.arange(1, 1000001),
    'value1': np.random.randn(1000000),
    'value2': np.random.randint(0, 100, size=1000000),
    'category': np.random.choice(['A', 'B', 'C', 'D'], size=1000000)
}
df = pd.DataFrame(data)

# 方法1：time 模塊統(tǒng)計(jì)耗時
start_time = time.time()
# 待測試操作（示例：按類別分組計(jì)算均值）
result = df.groupby('category')[['value1', 'value2']].mean()
end_time = time.time()
print(f"執(zhí)行耗時：{end_time - start_time:.2f} 秒")

# 方法2：timeit 模塊（適合多次執(zhí)行取平均，排除偶然因素）
import timeit
stmt = "df.groupby('category')[['value1', 'value2']].mean()"
setup = "import pandas as pd; import numpy as np; df = pd.DataFrame({'id': np.arange(1, 1000001), 'value1': np.random.randn(1000000), 'value2': np.random.randint(0, 100, size=1000000), 'category': np.random.choice(['A', 'B', 'C', 'D'], size=1000000)})"
time_cost = timeit.timeit(stmt, setup, number=5) / 5
print(f"平均執(zhí)行耗時：{time_cost:.2f} 秒")

import pandas as pd

# 查看整體內(nèi)存占用（info() 方法）
df.info(memory_usage='deep')  # memory_usage='deep' 精準(zhǔn)計(jì)算對象類型內(nèi)存

# 查看各列內(nèi)存占用（memory_usage() 方法）
memory_detail = df.memory_usage(deep=True)
print("\n各列內(nèi)存占用：")
print(memory_detail)
print(f"\n總內(nèi)存占用：{memory_detail.sum() / 1024 / 1024:.2f} MB")

# 計(jì)算內(nèi)存占用比例
memory_ratio = (memory_detail / memory_detail.sum() * 100).round(2)
print("\n各列內(nèi)存占用比例：")
for col, ratio in memory_ratio.items():
    print(f"{col}: {ratio}%")

import pandas as pd

# 生成測試數(shù)據(jù)
df = pd.DataFrame({
    'int_col': np.random.randint(0, 1000, size=1000000),  # 范圍0-999，無需int64
    'float_col': np.random.randn(1000000),  # 可根據(jù)精度需求降低float位數(shù)
    'cat_col': np.random.choice(['apple', 'banana', 'orange', 'grape'], size=1000000)  # 低基數(shù)字符串
})

# 查看優(yōu)化前內(nèi)存
print("優(yōu)化前內(nèi)存占用：")
print(f"總內(nèi)存：{df.memory_usage(deep=True).sum() / 1024 / 1024:.2f} MB")

# 1. 優(yōu)化整數(shù)類型（int64 → int16）
df['int_col'] = df['int_col'].astype('int16')

# 2. 優(yōu)化浮點(diǎn)類型（float64 → float32，精度可滿足多數(shù)場景）
df['float_col'] = df['float_col'].astype('float32')

# 3. 優(yōu)化字符串類型（object → category，低基數(shù)場景）
df['cat_col'] = df['cat_col'].astype('category')

# 查看優(yōu)化后內(nèi)存
print("\n優(yōu)化后內(nèi)存占用：")
print(f"總內(nèi)存：{df.memory_usage(deep=True).sum() / 1024 / 1024:.2f} MB")
print(f"內(nèi)存節(jié)省比例：{(1 - df.memory_usage(deep=True).sum() / df_original.memory_usage(deep=True).sum()) * 100:.1f}%")

import pandas as pd

# 讀取Excel文件，僅加載所需列（避免加載全部列）
df = pd.read_excel(
    'big_data.xlsx',
    usecols=['id', 'value1', 'category'],  # 僅加載3列，排除冗余列
    dtype={'id': 'int16', 'category': 'category'},  # 讀取時直接指定優(yōu)化后類型
    skiprows=[0]  # 跳過首行注釋行（若有）
)

# 讀取CSV文件時，同樣支持參數(shù)優(yōu)化
df = pd.read_csv(
    'big_data.csv',
    usecols=['id', 'value1', 'category'],
    dtype={'id': 'int16', 'category': 'category'},
    nrows=1000000  # 僅加載前100萬行（如需分批處理）
)

import pandas as pd
import numpy as np
import time

# 生成百萬級數(shù)據(jù)
df = pd.DataFrame({
    'a': np.random.randint(0, 100, size=1000000),
    'b': np.random.randint(0, 100, size=1000000)
})

# 方法1：原生for循環(huán)（低效）
start_time = time.time()
df['c'] = 0
for i in range(len(df)):
    if df.loc[i, 'a'] > 50:
        df.loc[i, 'c'] = df.loc[i, 'a'] + df.loc[i, 'b']
    else:
        df.loc[i, 'c'] = df.loc[i, 'a'] - df.loc[i, 'b']
print(f"for循環(huán)耗時：{time.time() - start_time:.2f} 秒")

# 方法2：矢量化運(yùn)算（高效）
start_time = time.time()
df['c'] = np.where(df['a'] > 50, df['a'] + df['b'], df['a'] - df['b'])
print(f"矢量化運(yùn)算耗時：{time.time() - start_time:.2f} 秒")

import pandas as pd
import time

# 復(fù)雜業(yè)務(wù)邏輯：根據(jù)多列值計(jì)算結(jié)果
def calculate_result(row):
    if row['a'] > 50 and row['b'] > 50:
        return row['a'] * row['b']
    elif row['a'] < 30 or row['b']< 30:
        return row['a'] / row['b'] if row['b'] != 0 else 0
    else:
        return row['a'] + row['b']

# 生成數(shù)據(jù)
df = pd.DataFrame({
    'a': np.random.randint(0, 100, size=1000000),
    'b': np.random.randint(1, 100, size=1000000)  # b從1開始，避免除零錯誤
})

# 用 apply() 執(zhí)行復(fù)雜邏輯
start_time = time.time()
df['result'] = df.apply(lambda x: calculate_result(x), axis=1)
print(f"apply() 耗時：{time.time() - start_time:.2f} 秒")

import pandas as pd
import time

df = pd.DataFrame({
    'a': np.random.randint(0, 100, size=1000000),
    'b': np.random.randint(0, 100, size=1000000),
    'c': np.random.randn(1000000)
})

# 方法1：布爾索引篩選
start_time = time.time()
filtered_df = df[(df['a'] > 30) & (df['b'] < 70) & (df['c'] > 0)]
print(f"布爾索引耗時：{time.time() - start_time:.2f} 秒")

# 方法2：query() 篩選
start_time = time.time()
filtered_df = df.query("a > 30 and b< 70 and c > 0")
print(f"query() 耗時：{time.time() - start_time:.2f} 秒")

import pandas as pd
import swifter
import time

# 安裝 Swifter：pip install swifter

# 生成數(shù)據(jù)
df = pd.DataFrame({
    'a': np.random.randint(0, 100, size=1000000),
    'b': np.random.randint(1, 100, size=1000000)
})

# 定義復(fù)雜邏輯函數(shù)
def complex_logic(row):
    return row['a'] * row['b'] if row['a'] > 50 else row['a'] / row['b']

# 方法1：原生 apply()
start_time = time.time()
df['result1'] = df.apply(complex_logic, axis=1)
print(f"原生 apply() 耗時：{time.time() - start_time:.2f} 秒")

# 方法2：Swifter 并行 apply()
start_time = time.time()
df['result2'] = df.swifter.apply(complex_logic, axis=1)
print(f"Swifter 并行耗時：{time.time() - start_time:.2f} 秒")

# 生成百萬級數(shù)據(jù)并保存為CSV（模擬超大規(guī)模數(shù)據(jù)）
df_pandas = pd.DataFrame({
    'a': np.random.randint(0, 100, size=1000000),
    'b': np.random.randn(1000000),
    'category': np.random.choice(['A', 'B', 'C'], size=1000000)
})
df_pandas.to_csv('big_data.csv', index=False)

# 方法1：Pandas 聚合計(jì)算
start_time = time.time()
df_pandas.groupby('category')[['a', 'b']].agg(['mean', 'sum'])
print(f"Pandas 聚合耗時：{time.time() - start_time:.2f} 秒")

# 方法2：Dask 并行聚合計(jì)算
start_time = time.time()
df_dask = dd.read_csv('big_data.csv', dtype={'a': 'int16', 'category': 'category'})
result_dask = df_dask.groupby('category')[['a', 'b']].agg(['mean', 'sum']).compute()  # compute() 觸發(fā)計(jì)算
print(f"Dask 聚合耗時：{time.time() - start_time:.2f} 秒")

import pandas as pd
import time

# 生成百萬級數(shù)據(jù)
df = pd.DataFrame({
    'id': np.arange(1, 1000001),
    'value1': np.random.randn(1000000),
    'value2': np.random.randint(0, 100, size=1000000),
    'category': np.random.choice(['A', 'B', 'C'], size=1000000)
})

# 1. CSV 讀寫耗時
start_time = time.time()
df.to_csv('data.csv', index=False)
df_csv = pd.read_csv('data.csv')
print(f"CSV 讀寫總耗時：{time.time() - start_time:.2f} 秒")
print(f"CSV 文件大?。簕os.path.getsize('data.csv') / 1024 / 1024:.2f} MB")

# 2. Parquet 讀寫耗時（需安裝 pyarrow 或 fastparquet）
# pip install pyarrow
start_time = time.time()
df.to_parquet('data.parquet', index=False, compression='snappy')  # snappy 壓縮格式
df_parquet = pd.read_parquet('data.parquet')
print(f"\nParquet 讀寫總耗時：{time.time() - start_time:.2f} 秒")
print(f"Parquet 文件大?。簕os.path.getsize('data.parquet') / 1024 / 1024:.2f} MB")

import pandas as pd
import time

# 分塊讀取 CSV 文件（每塊10萬行）
start_time = time.time()
chunk_list = []
for chunk in pd.read_csv('big_data.csv', chunksize=100000, dtype={'id': 'int16', 'category': 'category'}):
    # 逐塊處理數(shù)據(jù)（示例：篩選有效數(shù)據(jù)）
    processed_chunk = chunk[chunk['value1'] > 0]
    chunk_list.append(processed_chunk)
# 合并所有塊
df_total = pd.concat(chunk_list, ignore_index=True)
print(f"分塊讀取并處理耗時：{time.time() - start_time:.2f} 秒")

# 分塊寫入 Excel 文件（需借助 openpyxl）
with pd.ExcelWriter('result.xlsx', engine='openpyxl') as writer:
    for i, chunk in enumerate(chunk_list):
        chunk.to_excel(writer, sheet_name=f'Sheet_{i+1}', index=False)

import pandas as pd
import time
from datetime import datetime

# 記錄總耗時
start_total = time.time()

# 1. 讀取數(shù)據(jù)（默認(rèn)參數(shù)）
df = pd.read_csv('user_behavior.csv')
print(f"讀取后內(nèi)存占用：{df.memory_usage(deep=True).sum() / 1024 / 1024:.2f} MB")

# 2. 數(shù)據(jù)篩選（布爾索引）
start_filter = time.time()
df['access_time'] = pd.to_datetime(df['access_time'])
filtered_df = df[
    (df['consume_amount'] > 0) & 
    (df['access_time'] >= datetime(2024, 1, 1)) & 
    (df['access_time'] <= datetime(2024, 12, 31))
]
print(f"篩選耗時：{time.time() - start_filter:.2f} 秒")

# 3. 分組聚合（原生 groupby）
start_group = time.time()
result = filtered_df.groupby(['user_id', 'behavior_type']).agg({
    'consume_amount': ['sum', 'mean'],
    'access_time': 'count'
}).reset_index()
result.columns = ['user_id', 'behavior_type', 'total_consume', 'avg_consume', 'behavior_count']
print(f"分組聚合耗時：{time.time() - start_group:.2f} 秒")

# 4. 保存結(jié)果（CSV格式）
start_save = time.time()
result.to_csv('user_behavior_result.csv', index=False)
print(f"保存耗時：{time.time() - start_save:.2f} 秒")

# 總耗時
print(f"\n總處理耗時：{time.time() - start_total:.2f} 秒")

import pandas as pd
import time
import swifter
from datetime import datetime

# 記錄總耗時
start_total = time.time()

# 1. 讀取數(shù)據(jù)（優(yōu)化參數(shù)+數(shù)據(jù)類型）
start_read = time.time()
df = pd.read_csv(
    'user_behavior.csv',
    usecols=['user_id', 'behavior_type', 'access_time', 'consume_amount'],  # 僅加載所需列
    dtype={
        'user_id': 'int32',
        'behavior_type': 'category',
        'consume_amount': 'float32'
    },
    parse_dates=['access_time']  # 讀取時直接解析日期，避免二次轉(zhuǎn)換
)
print(f"讀取耗時：{time.time() - start_read:.2f} 秒")
print(f"優(yōu)化后內(nèi)存占用：{df.memory_usage(deep=True).sum() / 1024 / 1024:.2f} MB")

# 2. 數(shù)據(jù)篩選（query() 優(yōu)化）
start_filter = time.time()
filtered_df = df.query(
    "consume_amount > 0 and access_time >= '2024-01-01' and access_time <= '2024-12-31'"
)
print(f"篩選耗時：{time.time() - start_filter:.2f} 秒")

# 3. 分組聚合（Swifter 并行優(yōu)化，若數(shù)據(jù)量更大可改用 Dask）
start_group = time.time()
result = filtered_df.groupby(['user_id', 'behavior_type']).agg({
    'consume_amount': ['sum', 'mean'],
    'access_time': 'count'
}).reset_index()
result.columns = ['user_id', 'behavior_type', 'total_consume', 'avg_consume', 'behavior_count']
# 進(jìn)一步優(yōu)化：將結(jié)果數(shù)據(jù)類型壓縮
result['total_consume'] = result['total_consume'].astype('float32')
result['avg_consume'] = result['avg_consume'].astype('float32')
result['behavior_count'] = result['behavior_count'].astype('int16')
print(f"分組聚合耗時：{time.time() - start_group:.2f} 秒")

# 4. 保存結(jié)果（Parquet 格式）
start_save = time.time()
result.to_parquet('user_behavior_result.parquet', index=False, compression='snappy')
print(f"保存耗時：{time.time() - start_save:.2f} 秒")

# 總耗時
print(f"\n優(yōu)化后總處理耗時：{time.time() - start_total:.2f} 秒")