内容
- 从/kaggle/working/的输出下载文件
- 执行代码后显存变化量CPU/GPU检测
- 使用python Garbage Collection清理运行内存垃圾
- 重置运行内存
- 刷新输出记录
一、下载
1. 从/kaggle/working/的输出下载文件
# 将下载目录添加到压缩文件
from pathlib import Path
import zipfile
img_root = Path('/kaggle/working/img')
with zipfile.ZipFile('img.zip', 'w') as z:
for img_name in img_root.iterdir():
z.write(img_name)
# 创建下载链接
import os
os.chdir('/kaggle/working/')
print(os.getcwd())
print(os.listdir("/kaggle/working/"))
from IPython.display import FileLink
FileLink('img.zip')
二、CPU运行内存RAM优化
1. 执行代码后内存变化量检测
import psutil
import os
import time
import sys
import math
from contextlib import contextmanager
@contextmanager
def trace(title):
t0 = time.time()
p = psutil.Process(os.getpid())
m0 = p.memory_info().rss / 2. ** 30
yield
m1 = p.memory_info().rss / 2. ** 30
delta = m1 - m0
sign = '+' if delta >= 0 else '-'
delta = math.fabs(delta)
print(f"[{m1:.1f}GB({sign}{delta:.1f}GB):{time.time() - t0:.1f}sec] {title} ", file=sys.stderr)
# 执行代码后输出变化量
with trace('execute'):
## 代码段落 ##
2. 使用python Garbage Collection清理运行内存垃圾
import gc
gc.collect()
3. 重置运行内存RAM
%reset -f
三、GPU显存变化量检测
from contextlib import contextmanager
import math
import os
import subprocess
import sys
import time
import numpy as np
import psutil
import torch
def get_gpu_memory(cmd_path="nvidia-smi",
target_properties=("memory.total", "memory.used")):
"""
ref: https://www.12-technology.com/2022/01/pythongpu.html
Returns
-------
gpu_total : ndarray, "memory.total"
gpu_used: ndarray, "memory.used"
"""
# format option
format_option = "--format=csv,noheader,nounits"
cmd = '%s --query-gpu=%s %s' % (cmd_path, ','.join(target_properties), format_option)
# Command execution in sub-processes
cmd_res = subprocess.check_output(cmd, shell=True)
gpu_lines = cmd_res.decode().split('\n')[0].split(', ')
gpu_total = int(gpu_lines[0]) / 1024
gpu_used = int(gpu_lines[1]) / 1024
gpu_total = np.round(gpu_used, 1)
gpu_used = np.round(gpu_used, 1)
return gpu_total, gpu_used
class Trace():
cuda = torch.cuda.is_available()
@contextmanager
def timer(self, title):
t0 = time.time()
p = psutil.Process(os.getpid())
cpu_m0 = p.memory_info().rss / 2. ** 30
if self.cuda: gpu_m0 = get_gpu_memory()[0]
yield
cpu_m1 = p.memory_info().rss / 2. ** 30
if self.cuda: gpu_m1 = get_gpu_memory()[0]
cpu_delta = cpu_m1 - cpu_m0
if self.cuda: gpu_delta = gpu_m1 - gpu_m0
cpu_sign = '+' if cpu_delta >= 0 else '-'
cpu_delta = math.fabs(cpu_delta)
if self.cuda: gpu_sign = '+' if gpu_delta >= 0 else '-'
if self.cuda: gpu_delta = math.fabs(gpu_delta)
cpu_message = f'{cpu_m1:.1f}GB({cpu_sign}{cpu_delta:.1f}GB)'
if self.cuda: gpu_message = f'{gpu_m1:.1f}GB({gpu_sign}{gpu_delta:.1f}GB)'
if self.cuda:
message = f"[cpu: {cpu_message}, gpu: {gpu_message}: {time.time() - t0:.1f}sec] {title} "
else:
message = f"[cpu: {cpu_message}: {time.time() - t0:.1f}sec] {title} "
print(message, file=sys.stderr)
trace = Trace()
with trace.timer('read train'):
data = cudf.read_csv('../input/foursquare-location-matching/train.csv')
with trace.timer('del train'):
del data
gc.collect()
四、刷新输出
1. tqdm刷新进度
2. sys刷新输出记录
import sys
sys.stdout.write('\r'+str(x))
sys.stdout.flush()