Shell脚本如何设置告警恢复通知

本文目录导读：

1. 基于状态文件的实现
2. 使用计数器防止抖动
3. 支持多指标的监控脚本
4. 通知发送脚本示例
5. 使用crontab定时执行
6. 关键点说明

在Shell脚本中设置告警恢复通知，通常需要跟踪告警状态并在状态从“异常”转为“正常”时触发通知,以下是几种常见的实现方式：

基于状态文件的实现

#!/bin/bash
# 配置
ALERT_STATUS_FILE="/tmp/alert_status.txt"
CHECK_COMMAND="ping -c 1 example.com"  # 替换为实际的检查命令
NOTIFY_COMMAND='echo "告警: $1" | mail -s "告警通知" admin@example.com'
# 获取当前状态
$CHECK_COMMAND > /dev/null 2>&1
CURRENT_STATUS=$?
# 读取上次状态
if [ -f "$ALERT_STATUS_FILE" ]; then
    LAST_STATUS=$(cat "$ALERT_STATUS_FILE")
else
    LAST_STATUS=0  # 默认为正常状态
fi
# 判断并处理
if [ $CURRENT_STATUS -ne 0 ]; then
    # 当前异常
    if [ "$LAST_STATUS" -eq 0 ]; then
        # 从正常变为异常，触发告警
        eval $NOTIFY_COMMAND "服务异常"
        echo 1 > "$ALERT_STATUS_FILE"
    fi
else
    # 当前正常
    if [ "$LAST_STATUS" -ne 0 ]; then
        # 从异常变为正常，触发恢复通知
        eval $NOTIFY_COMMAND "服务已恢复"
        echo 0 > "$ALERT_STATUS_FILE"
    fi
fi

使用计数器防止抖动

#!/bin/bash
# 配置
CHECK_INTERVAL=60  # 检查间隔（秒）
ALERT_THRESHOLD=3   # 连续异常次数触发告警
RECOVERY_THRESHOLD=2  # 连续正常次数触发恢复
COUNTER_FILE="/tmp/alert_counter.txt"
STATUS_FILE="/tmp/alert_status.txt"
check_service() {
    # 替换为实际检查逻辑
    curl -s -o /dev/null -w "%{http_code}" http://localhost:8080/health
}
init_files() {
    [ ! -f "$COUNTER_FILE" ] && echo "0" > "$COUNTER_FILE"
    [ ! -f "$STATUS_FILE" ] && echo "0" > "$STATUS_FILE"
}
send_notification() {
    local message="$1"
    echo "$message" | mail -s "服务状态通知" admin@example.com
    # 或使用企业微信、钉钉等
    # curl -X POST -H "Content-Type: application/json" -d "{\"msgtype\":\"text\",\"text\":{\"content\":\"$message\"}}" "https://qyapi.weixin.qq.com/cgi-bin/webhook/send?key=YOUR_KEY"
}
main() {
    init_files
    local http_code=$(check_service)
    local is_healthy=false
    # 判断健康状态
    if [ "$http_code" -eq 200 ] || [ "$http_code" -eq 302 ]; then
        is_healthy=true
    fi
    local counter=$(cat "$COUNTER_FILE")
    local status=$(cat "$STATUS_FILE")
    if $is_healthy; then
        # 服务正常
        if [ "$status" -eq 1 ]; then
            # 之前是告警状态
            counter=$((counter + 1))
            if [ "$counter" -ge "$RECOVERY_THRESHOLD" ]; then
                send_notification "服务已恢复正常"
                echo "0" > "$STATUS_FILE"
                echo "0" > "$COUNTER_FILE"
            else
                echo "$counter" > "$COUNTER_FILE"
            fi
        fi
    else
        # 服务异常
        if [ "$status" -eq 0 ]; then
            # 之前是正常状态
            counter=$((counter + 1))
            if [ "$counter" -ge "$ALERT_THRESHOLD" ]; then
                send_notification "服务异常，HTTP状态码: $http_code"
                echo "1" > "$STATUS_FILE"
                echo "0" > "$COUNTER_FILE"
            else
                echo "$counter" > "$COUNTER_FILE"
            fi
        fi
    fi
}
# 主循环
while true; do
    main
    sleep "$CHECK_INTERVAL"
done

支持多指标的监控脚本

#!/bin/bash
# 配置
ALERT_DIR="/tmp/alerts"
NOTIFY_SCRIPT="/usr/local/bin/send_notify.sh"
# 初始化告警目录
mkdir -p "$ALERT_DIR"
check_disk() {
    local usage=$(df / | awk 'NR==2 {print $5}' | sed 's/%//')
    echo "$usage"
}
check_memory() {
    local usage=$(free | awk '/Mem/ {printf "%.0f", $3/$2 * 100}')
    echo "$usage"
}
check_cpu() {
    local usage=$(top -bn1 | grep "Cpu(s)" | awk '{print $2}')
    echo "${usage%.*}"
}
send_alert() {
    local metric="$1"
    local value="$2"
    local threshold="$3"
    local type="$4"  # alert 或 recovery
    local alert_file="$ALERT_DIR/${metric}_alert"
    if [ "$type" = "alert" ]; then
        if [ ! -f "$alert_file" ]; then
            # 发送告警
            $NOTIFY_SCRIPT "告警: $metric 当前值: $value, 阈值: $threshold"
            touch "$alert_file"
        fi
    elif [ "$type" = "recovery" ]; then
        if [ -f "$alert_file" ]; then
            # 发送恢复通知
            $NOTIFY_SCRIPT "恢复: $metric 已恢复正常, 当前值: $value"
            rm "$alert_file"
        fi
    fi
}
# 主检查逻辑
DISK_THRESHOLD=80
MEMORY_THRESHOLD=90
CPU_THRESHOLD=80
# 磁盘检查
disk_usage=$(check_disk)
if [ "$disk_usage" -gt "$DISK_THRESHOLD" ]; then
    send_alert "disk" "$disk_usage" "$DISK_THRESHOLD" "alert"
else
    send_alert "disk" "$disk_usage" "$DISK_THRESHOLD" "recovery"
fi
# 内存检查
mem_usage=$(check_memory)
if [ "$mem_usage" -gt "$MEMORY_THRESHOLD" ]; then
    send_alert "memory" "$mem_usage" "$MEMORY_THRESHOLD" "alert"
else
    send_alert "memory" "$mem_usage" "$MEMORY_THRESHOLD" "recovery"
fi
# CPU检查
cpu_usage=$(check_cpu)
if [ "$cpu_usage" -gt "$CPU_THRESHOLD" ]; then
    send_alert "cpu" "$cpu_usage" "$CPU_THRESHOLD" "alert"
else
    send_alert "cpu" "$cpu_usage" "$CPU_THRESHOLD" "recovery"
fi

通知发送脚本示例

#!/bin/bash
# send_notify.sh
send_email() {
    local subject="$1"
    local body="$2"
    echo "$body" | mail -s "$subject" admin@example.com
}
send_wechat() {
    local webhook_url="https://qyapi.weixin.qq.com/cgi-bin/webhook/send?key=YOUR_KEY"
    local content="$1"
    curl -s -X POST "$webhook_url" \
        -H "Content-Type: application/json" \
        -d "{\"msgtype\":\"text\",\"text\":{\"content\":\"$content\"}}" > /dev/null
}
send_dingtalk() {
    local webhook_url="https://oapi.dingtalk.com/robot/send?access_token=YOUR_TOKEN"
    local content="$1"
    curl -s -X POST "$webhook_url" \
        -H "Content-Type: application/json" \
        -d "{\"msgtype\":\"text\",\"text\":{\"content\":\"$content\"}}" > /dev/null
}
# 主通知函数
message="$1"
send_email "系统监控通知" "$message"
send_wechat "$message"
send_dingtalk "$message"

使用crontab定时执行

# 编辑crontab
crontab -e
# 每分钟检查一次
* * * * * /path/to/monitor_script.sh
# 或每5分钟检查一次
*/5 * * * * /path/to/monitor_script.sh

关键点说明

1. 状态持久化：使用文件记录上次告警状态
2. 防抖动：使用计数器避免短时间内的状态波动
3. 多指标支持：为每个监控指标维护独立的状态
4. 通知渠道：支持邮件、企业微信、钉钉等多种通知方式
5. 告警升级：可以实现告警级别，长时间未恢复发送更紧急的通知

选择适合你场景的实现方式,并根据实际需求调整阈值和检查间隔。