如何用脚本生成日志分析图表

本文目录导读：

1. 使用Python脚本（推荐）
2. 使用Shell脚本 + gnuplot
3. 使用GoAccess (实时分析)
4. 使用Logstash + Kibana (企业级方案)
5. 快速使用脚本
6. 使用示例

我来介绍几种生成日志分析图表的方法,包括命令行工具、Python脚本和Shell脚本：

使用Python脚本（推荐）

安装依赖

pip install matplotlib pandas seaborn

完整的日志分析脚本

#!/usr/bin/env python3
import matplotlib.pyplot as plt
import pandas as pd
import seaborn as sns
import re
from collections import Counter
from datetime import datetime
def parse_apache_log(log_file):
    """解析Apache日志文件"""
    pattern = r'(\S+) (\S+) (\S+) \[([^\]]+)\] "(\S+) (\S+) (\S+)" (\d+) (\d+)'
    data = []
    with open(log_file, 'r') as f:
        for line in f:
            match = re.match(pattern, line)
            if match:
                ip, _, user, time_str, method, url, protocol, status, size = match.groups()
                data.append({
                    'ip': ip,
                    'time': datetime.strptime(time_str.split()[0], '%d/%b/%Y:%H:%M:%S'),
                    'method': method,
                    'url': url,
                    'status': int(status),
                    'size': int(size)
                })
    return pd.DataFrame(data)
def generate_charts(df):
    """生成各种分析图表"""
    # 1. 状态码分布饼图
    plt.figure(figsize=(15, 10))
    plt.subplot(2, 3, 1)
    status_counts = df['status'].value_counts()
    plt.pie(status_counts.values, labels=status_counts.index, autopct='%1.1f%%')
    plt.title('HTTP Status Code Distribution')
    # 2. 请求方法柱状图
    plt.subplot(2, 3, 2)
    method_counts = df['method'].value_counts()
    plt.bar(method_counts.index, method_counts.values)
    plt.title('Request Methods')
    plt.xlabel('Method')
    plt.ylabel('Count')
    # 3. 时间序列请求数
    plt.subplot(2, 3, 3)
    df['time_hour'] = df['time'].dt.floor('H')
    hourly_requests = df.groupby('time_hour').size()
    plt.plot(hourly_requests.index, hourly_requests.values, marker='o')
    plt.title('Hourly Request Count')
    plt.xlabel('Time')
    plt.ylabel('Requests')
    plt.xticks(rotation=45)
    # 4. Top 10 IPs
    plt.subplot(2, 3, 4)
    top_ips = df['ip'].value_counts().head(10)
    plt.barh(range(len(top_ips)), top_ips.values)
    plt.yticks(range(len(top_ips)), top_ips.index)
    plt.title('Top 10 IP Addresses')
    plt.xlabel('Request Count')
    # 5. 热门URL
    plt.subplot(2, 3, 5)
    top_urls = df['url'].value_counts().head(10)
    plt.barh(range(len(top_urls)), top_urls.values)
    plt.yticks(range(len(top_urls)), [url[:30]+'...' if len(url)>30 else url for url in top_urls.index])
    plt.title('Top 10 URLs')
    plt.xlabel('Count')
    # 6. 响应大小分布
    plt.subplot(2, 3, 6)
    df['size_mb'] = df['size'] / (1024 * 1024)
    plt.hist(df['size_mb'].clip(upper=10), bins=50)
    plt.title('Response Size Distribution')
    plt.xlabel('Size (MB)')
    plt.ylabel('Frequency')
    plt.tight_layout()
    plt.savefig('log_analysis.png', dpi=300, bbox_inches='tight')
    print("Chart saved as log_analysis.png")
    # 生成额外的时间序列图表
    plt.figure(figsize=(12, 6))
    # 每日请求量趋势
    df['time_date'] = df['time'].dt.date
    daily_requests = df.groupby('time_date').size()
    plt.plot(daily_requests.index, daily_requests.values, marker='s', linewidth=2)
    plt.title('Daily Request Trend')
    plt.xlabel('Date')
    plt.ylabel('Requests')
    plt.grid(True, alpha=0.3)
    plt.savefig('daily_trend.png', dpi=300, bbox_inches='tight')
    print("Chart saved as daily_trend.png")
def generate_html_report(df):
    """生成HTML格式的统计报告"""
    stats = {
        'total_requests': len(df),
        'unique_ips': df['ip'].nunique(),
        'unique_urls': df['url'].nunique(),
        'avg_response_size': df['size'].mean(),
        'status_200': len(df[df['status'] == 200]),
        'status_404': len(df[df['status'] == 404]),
        'status_500': len(df[df['status'] == 500]),
    }
    html = f"""
    <html>
    <head>
        <title>Log Analysis Report</title>
        <style>
            body {{ font-family: Arial, sans-serif; margin: 20px; }}
            .stats {{ display: grid; grid-template-columns: repeat(3, 1fr); gap: 20px; }}
            .stat-box {{ background: #f0f0f0; padding: 15px; border-radius: 8px; text-align: center; }}
            .stat-value {{ font-size: 24px; font-weight: bold; color: #2c3e50; }}
            .stat-label {{ color: #7f8c8d; margin-top: 5px; }}
            img {{ max-width: 100%; margin: 20px 0; }}
        </style>
    </head>
    <body>
        <h1>Log Analysis Report</h1>
        <div class="stats">
            <div class="stat-box">
                <div class="stat-value">{stats['total_requests']}</div>
                <div class="stat-label">Total Requests</div>
            </div>
            <div class="stat-box">
                <div class="stat-value">{stats['unique_ips']}</div>
                <div class="stat-label">Unique IPs</div>
            </div>
            <div class="stat-box">
                <div class="stat-value">{stats['unique_urls']}</div>
                <div class="stat-label">Unique URLs</div>
            </div>
        </div>
        <h2>Charts</h2>
        <img src="log_analysis.png" alt="Analysis Charts">
        <img src="daily_trend.png" alt="Daily Trend">
    </body>
    </html>
    """
    with open('report.html', 'w') as f:
        f.write(html)
    print("HTML report saved as report.html")
if __name__ == "__main__":
    # 使用示例
    log_file = "access.log"  # 替换为你的日志文件路径
    try:
        df = parse_apache_log(log_file)
        generate_charts(df)
        generate_html_report(df)
        print("Analysis complete!")
    except FileNotFoundError:
        print(f"Error: File '{log_file}' not found")
    except Exception as e:
        print(f"Error: {e}")

使用Shell脚本 + gnuplot

安装gnuplot

apt-get install gnuplot  # Ubuntu/Debian
brew install gnuplot      # macOS

Shell脚本

#!/bin/bash
# log_analyzer.sh
LOG_FILE="access.log"
# 提取状态码统计
echo "Analyzing log file: $LOG_FILE"
# 1. 状态码统计
echo "Status Code Distribution:"
awk '{print $9}' $LOG_FILE | sort | uniq -c | sort -rn
# 2. 输出数据文件用于绘图
# 每小时请求数
awk '{split($4, a, ":"); print substr(a[1],2) " " a[2]}' $LOG_FILE | \
    sort | uniq -c | awk '{print $2 " " $3 " " $1}' > hourly_data.txt
# 生成gnuplot图表
cat > plot_commands.gnu << 'EOF'
set terminal png size 1200,800
set output 'hourly_requests.png''Hourly Request Count'
set xlabel 'Hour'
set ylabel 'Requests'
set grid
plot 'hourly_data.txt' using 2:3 with linespoints title 'Requests'
# Top 10 IPs
set output 'top_ips.png''Top 10 IP Addresses'
set style data histogram
set style fill solid
set xtics rotate by -45
plot '< awk "{print \$1}" '$LOG_FILE' | sort | uniq -c | sort -rn | head -10' \
    using 1:xtic(2) title 'IPs'
EOF
gnuplot plot_commands.gnu
echo "Charts generated: hourly_requests.png, top_ips.png"

使用GoAccess (实时分析)

# 安装GoAccess
apt-get install goaccess
# 生成HTML报告
goaccess access.log -o report.html --log-format=COMBINED
# 实时分析
goaccess access.log --log-format=COMBINED

使用Logstash + Kibana (企业级方案)

# docker-compose.yml 示例
version: '3'
services:
  elasticsearch:
    image: docker.elastic.co/elasticsearch/elasticsearch:7.15.0
    environment:
      - discovery.type=single-node
    ports:
      - "9200:9200"
  logstash:
    image: docker.elastic.co/logstash/logstash:7.15.0
    volumes:
      - ./logstash.conf:/usr/share/logstash/pipeline/logstash.conf
      - ./logs:/var/log/nginx
    depends_on:
      - elasticsearch
  kibana:
    image: docker.elastic.co/kibana/kibana:7.15.0
    ports:
      - "5601:5601"
    depends_on:
      - elasticsearch

快速使用脚本

一键分析脚本

#!/bin/bash
# quick_log_analysis.sh
LOG_FILE="${1:-access.log}"  # 默认使用access.log
# 检查文件是否存在
if [ ! -f "$LOG_FILE" ]; then
    echo "Error: $LOG_FILE not found"
    exit 1
fi
echo "=== Quick Log Analysis ==="
echo "File: $LOG_FILE"
echo ""
# 基础统计
echo "Total lines: $(wc -l < $LOG_FILE)"
echo "Unique IPs: $(awk '{print $1}' $LOG_FILE | sort -u | wc -l)"
echo "404 errors: $(grep -c ' 404 ' $LOG_FILE)"
echo "500 errors: $(grep -c ' 500 ' $LOG_FILE)"
# 生成Python图表（如果有Python）
if command -v python3 &> /dev/null; then
    python3 -c "
import sys
sys.path.insert(0, '.')
from log_analyzer import *
df = parse_apache_log('$LOG_FILE')
generate_charts(df)
print('Charts generated successfully!')
" 2>/dev/null || echo "Python chart generation skipped"
fi
echo ""
echo "Analysis complete!"

使用示例

# 保存Python脚本为 log_analyzer.py
# 保存Shell脚本为 analyze.sh
# 给予执行权限
chmod +x log_analyzer.py analyze.sh
# 运行分析
python3 log_analyzer.py
# 或
./quick_log_analysis.sh /var/log/nginx/access.log

这些脚本会根据你的日志文件自动生成：

• Python版本：生成6个图表 + HTML报告
• Shell版本：生成基础统计 + gnuplot图表
• GoAccess：生成专业的HTML报告

选择适合你需求的方式,根据实际日志格式调整解析规则。