def reduce_mem_usage(df):
start_mem = df.memory_usage().sum() / (1024 ** 3)
print('Memory usage of dataframe is {:.2f} GB'.format(start_mem))
for col in df.columns:
col_type = df[col].dtype
if col_type != object:
min_val = df[col].min()
max_val = df[col].max()
if str(col_type).startswith('int'):
type_list = [np.int8, np.int16, np.int32, np.int64]
for i in type_list:
if min_val >= np.iinfo(i).min and max_val <= np.iinfo(i).max:
df[col] = df[col].astype(i)
break
else:
type_list = [np.float16, np.float32, np.float64]
for i in type_list:
if min_val >= np.iinfo(i).min and max_val <= np.iinfo(i).max:
df[col] = df[col].astype(i)
break
end_mem = df.memory_usage().sum() / (1024 ** 3)
print('Memory usage of dataframe is {:.2f} GB'.format(end_mem))
return df
使用pandas的read_csv或者excel读取大文件时,在读取过程中出现OOM(Out of memory,内存溢出),但是结合watch -n 0.1 free -hm和已读取的行数占比来查看的话,发现需要内存超出实际内存大约占10%左右,可通过设置chunksize进行分块读取(如总行数的1/10)。