监控GPU使用率的脚本如何写

本文目录导读：

1. 带告警的监控
2. 日志记录

我来介绍几种监控GPU使用率的脚本方法：

使用 nvidia-smi 的基础监控脚本

Linux Bash 脚本

#!/bin/bash
# gpu_monitor.sh
while true; do
    # 显示GPU使用率、温度、显存使用
    nvidia-smi --query-gpu=index,utilization.gpu,utilization.memory,temperature.gpu,memory.used,memory.total --format=csv,noheader
    # 或使用更简洁的输出
    nvidia-smi stats -d gpuUtil
    sleep 2
    clear
done

实时监控（每秒刷新）

#!/bin/bash
watch -n 1 nvidia-smi --query-gpu=index,utilization.gpu,utilization.memory,temperature.gpu --format=csv,noheader

Python 脚本

基础监控类

#!/usr/bin/env python3
# gpu_monitor.py
import subprocess
import json
import time
from datetime import datetime
def get_gpu_info():
    """获取GPU信息"""
    try:
        # 执行 nvidia-smi 命令
        result = subprocess.run([
            'nvidia-smi', 
            '--query-gpu=index,name,utilization.gpu,utilization.memory,memory.used,memory.total,temperature.gpu',
            '--format=csv,noheader,nounits'
        ], capture_output=True, text=True, check=True)
        gpu_info = []
        for line in result.stdout.strip().split('\n'):
            if line:
                parts = [x.strip() for x in line.split(',')]
                gpu_info.append({
                    'index': parts[0],
                    'name': parts[1],
                    'gpu_util': float(parts[2]),
                    'memory_util': float(parts[3]),
                    'memory_used': parts[4],
                    'memory_total': parts[5],
                    'temperature': parts[6]
                })
        return gpu_info
    except (subprocess.CalledProcessError, FileNotFoundError):
        print("Error: nvidia-smi not found or failed")
        return []
def monitor_gpu(interval=2):
    """持续监控GPU"""
    print(f"Starting GPU monitoring (interval: {interval}s)")
    print("-" * 80)
    while True:
        timestamp = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
        gpus = get_gpu_info()
        if gpus:
            print(f"\n[{timestamp}]")
            for gpu in gpus:
                print(f"GPU {gpu['index']} ({gpu['name']}):")
                print(f"  GPU使用率: {gpu['gpu_util']}%")
                print(f"  显存使用率: {gpu['memory_util']}%")
                print(f"  显存: {gpu['memory_used']}/{gpu['memory_total']} MB")
                print(f"  温度: {gpu['temperature']}°C")
        else:
            print(f"[{timestamp}] No GPU information available")
        time.sleep(interval)
if __name__ == "__main__":
    try:
        monitor_gpu()
    except KeyboardInterrupt:
        print("\nMonitoring stopped.")

带告警的监控脚本

#!/usr/bin/env python3
# gpu_monitor_alarm.py
import subprocess
import time
import smtplib
from email.mime.text import MIMEText
class GPUMonitor:
    def __init__(self, threshold=90, interval=5):
        self.threshold = threshold  # GPU使用率阈值
        self.interval = interval     # 检查间隔
        self.alerted = {}           # 记录已告警的GPU
    def get_gpu_usage(self):
        """获取GPU使用率"""
        try:
            result = subprocess.run([
                'nvidia-smi',
                '--query-gpu=index,utilization.gpu',
                '--format=csv,noheader,nounits'
            ], capture_output=True, text=True, check=True)
            usage = {}
            for line in result.stdout.strip().split('\n'):
                if line:
                    idx, util = line.strip().split(', ')
                    usage[int(idx)] = float(util)
            return usage
        except Exception as e:
            print(f"Error getting GPU info: {e}")
            return {}
    def check_and_alert(self, gpu_id, usage):
        """检查是否需要告警"""
        if usage > self.threshold and not self.alerted.get(gpu_id):
            print(f"ALERT: GPU {gpu_id} usage at {usage}% (threshold: {self.threshold}%)")
            # 这里可以添加发送邮件或消息的代码
            self.alerted[gpu_id] = True
        elif usage <= self.threshold and self.alerted.get(gpu_id):
            print(f"INFO: GPU {gpu_id} usage back to normal at {usage}%")
            self.alerted[gpu_id] = False
    def run(self):
        print(f"GPU Monitor Started")
        print(f"Threshold: {self.threshold}%")
        print(f"Check interval: {self.interval}s")
        print("-" * 50)
        while True:
            usage = self.get_gpu_usage()
            if usage:
                for gpu_id, util in usage.items():
                    status = "⚠️" if util > self.threshold else "✓"
                    print(f"GPU {gpu_id}: {util}% {status}")
                    self.check_and_alert(gpu_id, util)
            else:
                print("No GPU information available")
            time.sleep(self.interval)
if __name__ == "__main__":
    monitor = GPUMonitor(threshold=90, interval=5)
    try:
        monitor.run()
    except KeyboardInterrupt:
        print("\nMonitoring stopped.")

日志记录版本

#!/usr/bin/env python3
# gpu_monitor_log.py
import subprocess
import csv
import time
from datetime import datetime
def log_gpu_usage(log_file="gpu_usage.csv", interval=60):
    """记录GPU使用率到CSV文件"""
    # 创建CSV文件并写入表头
    with open(log_file, 'w', newline='') as f:
        writer = csv.writer(f)
        writer.writerow(['Timestamp', 'GPU_ID', 'GPU_Util%', 'Memory_Util%', 'Temperature°C'])
    print(f"Logging GPU usage to {log_file} (interval: {interval}s)")
    while True:
        timestamp = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
        try:
            result = subprocess.run([
                'nvidia-smi',
                '--query-gpu=index,utilization.gpu,utilization.memory,temperature.gpu',
                '--format=csv,noheader,nounits'
            ], capture_output=True, text=True, check=True)
            with open(log_file, 'a', newline='') as f:
                writer = csv.writer(f)
                for line in result.stdout.strip().split('\n'):
                    if line:
                        data = [timestamp] + [x.strip() for x in line.split(',')]
                        writer.writerow(data)
            print(f"[{timestamp}] Data logged")
        except (subprocess.CalledProcessError, FileNotFoundError) as e:
            print(f"[{timestamp}] Error: {e}")
        time.sleep(interval)
if __name__ == "__main__":
    try:
        log_gpu_usage("gpu_usage.csv", 60)  # 每分钟记录一次
    except KeyboardInterrupt:
        print("\nLogging stopped.")

Web监控（Flask版本）

from flask import Flask, jsonify
import subprocess
import json
app = Flask(__name__)
@app.route('/gpu')
def get_gpu_info():
    try:
        result = subprocess.run([
            'nvidia-smi',
            '--query-gpu=index,name,utilization.gpu,utilization.memory,memory.used,memory.total,temperature.gpu',
            '--format=csv,noheader,nounits'
        ], capture_output=True, text=True, check=True)
        gpu_data = []
        for line in result.stdout.strip().split('\n'):
            if line:
                parts = [x.strip() for x in line.split(',')]
                gpu_data.append({
                    'index': int(parts[0]),
                    'name': parts[1],
                    'gpu_util': float(parts[2]),
                    'memory_util': float(parts[3]),
                    'memory_used': parts[4],
                    'memory_total': parts[5],
                    'temperature': parts[6]
                })
        return jsonify({
            'status': 'success',
            'data': gpu_data,
            'count': len(gpu_data)
        })
    except Exception as e:
        return jsonify({
            'status': 'error',
            'message': str(e)
        }), 500
if __name__ == '__main__':
    app.run(host='0.0.0.0', port=5000, debug=True)

使用说明

1. 安装依赖：

   # Python脚本需要
   pip install flask  # Web监控需要

2. 运行脚本：

   # 基础监控
   python3 gpu_monitor.py

带告警的监控

python3 gpu_monitor_alarm.py

日志记录

python3 gpu_monitor_log.py

3. **设置定时任务**（cron）：
```bash
# 每5分钟执行一次监控脚本
*/5 * * * * /path/to/gpu_monitor.sh >> /var/log/gpu_monitor.log 2>&1

这些脚本可以根据你的具体需求进行修改和扩展。