Shell脚本如何实现熔断恢复机制

本文目录导读：

1. 基于失败计数的熔断器（推荐）
2. 基于时间窗口的熔断器
3. 功能增强版本（带监控输出）
4. 使用建议

在Shell脚本中实现熔断恢复机制,通常采用以下两种经典模式：

基于失败计数的熔断器（推荐）

#!/bin/bash
# 熔断器配置
CIRCUIT_BREAKER_FILE="/tmp/circuit_breaker.state"
FAILURE_THRESHOLD=3
SUCCESS_THRESHOLD=2
TIMEOUT_SECONDS=30
# 读取当前状态
read_state() {
    if [ -f "$CIRCUIT_BREAKER_FILE" ]; then
        source "$CIRCUIT_BREAKER_FILE"
    else
        STATE="CLOSED"
        FAILURE_COUNT=0
        SUCCESS_COUNT=0
        LAST_FAILURE_TIME=0
    fi
}
# 保存状态
save_state() {
    cat > "$CIRCUIT_BREAKER_FILE" << EOF
STATE=$STATE
FAILURE_COUNT=$FAILURE_COUNT
SUCCESS_COUNT=$SUCCESS_COUNT
LAST_FAILURE_TIME=$LAST_FAILURE_TIME
EOF
}
# 检查是否应该打开熔断器
should_break() {
    read_state
    case "$STATE" in
        "CLOSED")
            if [ $FAILURE_COUNT -ge $FAILURE_THRESHOLD ]; then
                STATE="OPEN"
                LAST_FAILURE_TIME=$(date +%s)
                save_state
                return 0
            fi
            return 1
            ;;
        "OPEN")
            current_time=$(date +%s)
            elapsed=$((current_time - LAST_FAILURE_TIME))
            if [ $elapsed -ge $TIMEOUT_SECONDS ]; then
                STATE="HALF_OPEN"
                SUCCESS_COUNT=0
                save_state
                return 1  # 允许尝试
            fi
            return 0  # 仍然熔断
            ;;
        "HALF_OPEN")
            return 1  # 允许尝试
            ;;
    esac
}
# 记录成功
record_success() {
    read_state
    if [ "$STATE" = "HALF_OPEN" ]; then
        SUCCESS_COUNT=$((SUCCESS_COUNT + 1))
        if [ $SUCCESS_COUNT -ge $SUCCESS_THRESHOLD ]; then
            STATE="CLOSED"
            FAILURE_COUNT=0
            echo "Circuit breaker reset to CLOSED state"
        fi
    else
        FAILURE_COUNT=0
    fi
    save_state
}
# 记录失败
record_failure() {
    read_state
    FAILURE_COUNT=$((FAILURE_COUNT + 1))
    LAST_FAILURE_TIME=$(date +%s)
    save_state
    echo "Failure recorded. Count: $FAILURE_COUNT"
}
# 执行受保护的操作
protected_operation() {
    if should_break; then
        echo "Circuit is OPEN. Request rejected."
        return 1
    fi
    # 执行实际操作（示例为ping命令）
    if ping -c 1 google.com &> /dev/null; then
        record_success
        echo "Operation succeeded"
        return 0
    else
        record_failure
        echo "Operation failed"
        return 1
    fi
}
# 主循环示例
while true; do
    protected_operation
    sleep 2
done

基于时间窗口的熔断器

#!/bin/bash
# 配置
METRICS_WINDOW=60  # 时间窗口（秒）
MAX_FAILURES=5     # 窗口内最大失败次数
RECOVERY_TIME=30   # 恢复等待时间（秒）
# 使用临时文件存储指标
METRICS_FILE="/tmp/circuit_metrics.$$"
mkdir -p /tmp
# 记录事件
record_event() {
    local result=$1
    local timestamp=$(date +%s)
    echo "$timestamp:$result" >> "$METRICS_FILE"
    # 清理过期记录
    local cutoff=$((timestamp - METRICS_WINDOW))
    sed -i "/^$cutoff:/d" "$METRICS_FILE" 2>/dev/null
}
# 计算窗口内失败率
get_failure_rate() {
    local total=0
    local failures=0
    local timestamp=$(date +%s)
    local cutoff=$((timestamp - METRICS_WINDOW))
    while IFS=: read -r time result; do
        if [ "$time" -ge "$cutoff" ]; then
            total=$((total + 1))
            [ "$result" = "failure" ] && failures=$((failures + 1))
        fi
    done < "$METRICS_FILE"
    [ $total -eq 0 ] && echo 0 || echo $((failures * 100 / total))
}
# 熔断器逻辑
circuit_breaker() {
    local state_file="/tmp/circuit_state.$$"
    local state="CLOSED"
    local recovery_time=0
    if [ -f "$state_file" ]; then
        read state recovery_time < "$state_file"
        recovery_time=${recovery_time:-0}
    fi
    case "$state" in
        "OPEN")
            if [ $(date +%s) -ge $recovery_time ]; then
                state="HALF_OPEN"
                echo "Circuit transitioning to HALF_OPEN"
            else
                echo "Circuit is OPEN, rejecting request"
                return 1
            fi
            ;;
        "HALF_OPEN")
            # 在HALF_OPEN状态只允许有限的重试
            ;;
        "CLOSED")
            local failure_rate=$(get_failure_rate)
            if [ "$failure_rate" -ge 50 ]; then  # 50%失败率触发
                state="OPEN"
                recovery_time=$(( $(date +%s) + RECOVERY_TIME ))
                echo "Circuit OPENED due to high failure rate: ${failure_rate}%"
            fi
            ;;
    esac
    # 保存状态
    echo "$state $recovery_time" > "$state_file"
    return 0
}
# 受保护的函数
protected_api_call() {
    if ! circuit_breaker; then
        return 1
    fi
    # 模拟API调用
    if [ $((RANDOM % 10)) -lt 3 ]; then  # 30%失败率
        record_event "failure"
        echo "API call FAILED"
        return 1
    else
        record_event "success"
        echo "API call SUCCESS"
        return 0
    fi
}
# 主程序
echo "Starting circuit breaker demo..."
for i in {1..20}; do
    echo -n "Request $i: "
    protected_api_call
    sleep 1
done
# 清理
rm -f "$METRICS_FILE" "$state_file"

功能增强版本（带监控输出）

#!/bin/bash
# 配置
THRESHOLD_FAILURES=3
RECOVERY_TIMEOUT=10
STATS_INTERVAL=5
# 状态机
STATE_CLOSED=0
STATE_OPEN=1
STATE_HALF_OPEN=2
# 全局变量
current_state=$STATE_CLOSED
failure_count=0
success_count=0
last_failure_time=0
total_requests=0
rejected_requests=0
# 重置熔断器
reset_circuit() {
    current_state=$STATE_CLOSED
    failure_count=0
    success_count=0
    echo "Circuit breaker RESET to CLOSED"
}
# 检查熔断器状态
check_circuit() {
    case $current_state in
        $STATE_CLOSED)
            if [ $failure_count -ge $THRESHOLD_FAILURES ]; then
                current_state=$STATE_OPEN
                last_failure_time=$(date +%s)
                echo "CIRCUIT OPENED at $(date)"
                return 1
            fi
            return 0
            ;;
        $STATE_OPEN)
            local current_time=$(date +%s)
            if [ $((current_time - last_failure_time)) -ge $RECOVERY_TIMEOUT ]; then
                current_state=$STATE_HALF_OPEN
                echo "CIRCUIT HALF-OPENED at $(date)"
                return 0
            fi
            return 1
            ;;
        $STATE_HALF_OPEN)
            return 0
            ;;
    esac
}
# 记录操作结果
record_result() {
    local result=$1
    total_requests=$((total_requests + 1))
    if [ "$result" = "success" ]; then
        case $current_state in
            $STATE_HALF_OPEN)
                success_count=$((success_count + 1))
                if [ $success_count -ge 2 ]; then
                    reset_circuit
                fi
                ;;
            $STATE_CLOSED)
                failure_count=0
                ;;
        esac
    else
        failure_count=$((failure_count + 1))
        success_count=0
    fi
}
# 执行受保护的调用
protected_call() {
    total_requests=$((total_requests + 1))
    if ! check_circuit; then
        rejected_requests=$((rejected_requests + 1))
        echo "Request REJECTED (circuit open)"
        return 1
    fi
    # 模拟实际调用
    local exit_code=$1
    if [ $exit_code -eq 0 ]; then
        record_result "success"
        echo "Request SUCCESS"
        return 0
    else
        record_result "failure"
        echo "Request FAILED"
        return 1
    fi
}
# 监控函数
monitor() {
    echo "=== Circuit Breaker Status ==="
    echo "State: $(state_to_string)"
    echo "Failures: $failure_count/$THRESHOLD_FAILURES"
    echo "Total Requests: $total_requests"
    echo "Rejected: $rejected_requests"
    echo "============================"
}
state_to_string() {
    case $current_state in
        $STATE_CLOSED) echo "CLOSED" ;;
        $STATE_OPEN) echo "OPEN" ;;
        $STATE_HALF_OPEN) echo "HALF_OPEN" ;;
    esac
}
# 主程序
echo "Starting enhanced circuit breaker..."
echo "Threshold: $THRESHOLD_FAILURES failures, Recovery: ${RECOVERY_TIMEOUT}s"
# 模拟测试序列
test_sequence() {
    # 模拟连续失败
    echo "--- Testing failure scenario ---"
    for i in 1 2 3 4; do
        protected_call 1  # 模拟失败
        monitor
        sleep 1
    done
    # 模拟恢复
    echo "--- Waiting for recovery ---"
    sleep $((RECOVERY_TIMEOUT + 1))
    monitor
    # 测试半开状态
    echo "--- Testing half-open recovery ---"
    for i in 1 2 3; do
        protected_call 0  # 模拟成功
        monitor
        sleep 1
    done
}
test_sequence
# 清理
rm -f /tmp/circuit_*.state 2>/dev/null

使用建议

1. 选择合适的策略：

   • 计数法：简单高效，适合明确的失败模式
   • 时间窗口法：适合需要统计分析的场景

2. 优化配置参数：

   • 调整失败阈值基于系统容错能力
   • 恢复时间基于服务恢复的平均时间

3. 添加日志和告警：

   • 记录熔断状态变化
   • 发送告警通知运维团队

4. 处理并发访问：

   • 使用文件锁避免竞态条件
   • 考虑使用系统级别的信号量

这些实现可以根据具体需求调整,适用于各种需要熔断保护的生产环境。