ubuntu 使用Python代码监控GPU和CPU的用量及温度
问题:
项目中,由于发现存在程序怪掉起不来,检查发现是输入命令nvidia-smi后,不打印NVIDIA显卡的驱动信息。
解决办法1:
reboot重启之后, 再次输入nvidia-smi -l 1后,正常打印。
解决办法2:
猜测原因可能是由于,Ubuntu内核升级导致NVIDIA驱动无法连接到。参考 https://zhuanlan.zhihu.com/p/611276995?utm_id=0 ,产生通过如下方法避免内核自动更新。
通过如下两条命令,并将里面的参数值均改成0
vi /etc/apt/apt.conf.d/10periodic
vi /etc/apt/apt.conf.d/20auto-apgrades
另,
猜测原因:
可能是由于散热问题,导致驱动会频繁崩掉
故,写了一段python代码,每隔5分钟,保存一次GPU和CPU的用量及温度到txt文件中。源码如下:
涉及需要安装的python库
# pip3 install nvidia-ml-py3 -i http://pypi.douban.com/simple --trusted-host pypi.douban.com
# pip3 install psutil
def print_gpu_info():
# ref: https://blog.csdn.net/zxc120389574/article/details/106220612
# 获取GPU的用量和温度
# 这里假设使用的是NVIDIA显卡,需要安装nvidia-ml-py库
print("---------GPU usage and T---------")
import pynvml
pynvml.nvmlInit()
# Use device index to get gpu_device
gpu_device = pynvml.nvmlDeviceGetHandleByIndex(0)
# Use gpu_device to get device stats
memory_info = pynvml.nvmlDeviceGetMemoryInfo(gpu_device)
memoInfo = "Memory: Used/Total = {}/{}".format(memory_info.used / 1024 / 1024, memory_info.total / 1024 / 1024)
powerUsage = pynvml.nvmlDeviceGetPowerUsage(gpu_device)
powerState = pynvml.nvmlDeviceGetPowerState(gpu_device)
# get GPU temperature
gpu_device = pynvml.nvmlDeviceGetHandleByIndex(0)
temperature = pynvml.nvmlDeviceGetTemperature(gpu_device, pynvml.NVML_TEMPERATURE_GPU)
tempInfo = "Temperature:{0}°C".format(temperature)
gpu_info ='gpu: ' + memoInfo + ' ' + tempInfo + ' '
return gpu_info
def print_cpu_info():
# 获取CPU的用量和温度 ()
print("---------CPU usage and T---------")
import psutil
cpu_percent = psutil.cpu_percent()
# psutil.sensors_temperatures() 官方文档这有说的只适用于 linux
# 此程序在ubuntu上验证pass,
cpu_temperature = psutil.sensors_temperatures()['coretemp'][0].current
memoInfo = f"CPU utilization: {cpu_percent}%"
tempInfo = f"CPU temperature: {cpu_temperature}°C"
cpu_info = ' |||| cpu: ' + memoInfo + ' ' + tempInfo + ' '
return cpu_info
def write2Info(info):
from datetime import datetime
t = datetime.now().strftime('%Y-%m-%d %H:%M:%S.%f')
print('info : ', info)
info = t + ' '+ info + '\n'
txtName = './GPUCPU_usage{}.txt'.format(1)
with open(txtName, 'a', encoding='utf-8') as wf:
wf.write(info)
if __name__ == '__main__':
import time
while True:
gpu_info = print_gpu_info()
cpu_info = print_cpu_info()
info = gpu_info + cpu_info
write2Info(info)
time.sleep(2)
或 (上下两端代码可能是一样的,只不过下面的代码试运行noproblem)
import psutil
import time
def print_gpu_info():
#print("---------GPU usage and T---------")
import pynvml
pynvml.nvmlInit()
# Use device index to get gpu_device
gpu_device = pynvml.nvmlDeviceGetHandleByIndex(0)
# Use gpu_device to get device stats
memory_info = pynvml.nvmlDeviceGetMemoryInfo(gpu_device)
memoInfo = "Memory: Used/Total = {}/{}".format(memory_info.used / 1024 / 1024, memory_info.total / 1024 / 1024)
gpu_device = pynvml.nvmlDeviceGetHandleByIndex(0)
temperature = pynvml.nvmlDeviceGetTemperature(gpu_device, pynvml.NVML_TEMPERATURE_GPU)
tempInfo = "Temperature:{0}°C".format(temperature)
# print(tempInfo)
gpu_info ='gpu: ' + memoInfo + ' ' + tempInfo + ' '
return gpu_info
def print_cpu_info():
#print("---------CPU usage and T---------")
import psutil
cpu_percent = psutil.cpu_percent()
# psutil.sensors_temperatures() 官方文档这有说的只适用于 linux
# 此程序在ubuntu上验证pass,
cpu_temperature = psutil.sensors_temperatures()['coretemp'][0].current
memoInfo = f"CPU utilization: {cpu_percent}%"
# print(memoInfo)
tempInfo = f"CPU temperature: {cpu_temperature}°C"
# print(tempInfo)
cpu_info = ' |||| cpu: ' + memoInfo + ' ' + tempInfo + ' '
return cpu_info
def write2Info(info):
from datetime import datetime
t = datetime.now().strftime('%Y-%m-%d %H:%M:%S.%f')
print('info : ', info)
info = t + ' '+ info + '\n'
txtName = './GPUCPU_usage{}.txt'.format(1)
with open(txtName, 'a', encoding='utf-8') as wf:
wf.write(info)
if __name__ == '__main__':
import time
while True:
gpu_info = print_gpu_info()
cpu_info = print_cpu_info()
info = gpu_info + cpu_info
write2Info(info)
time.sleep(300)
连接
链接:https://pan.baidu.com/s/1ZoPp9mSeydqNuRobkChWsw