PHP项目如何实现告警恢复？

wen java案例 2026-07-23 131

本文目录导读：

PHP项目如何实现告警恢复？

基于状态机的告警恢复模型
时间窗口恢复策略
基于心跳的恢复机制
自动恢复执行器
告警恢复通知
完整的告警生命周期管理
最佳实践建议

在PHP项目中实现告警恢复,核心思路是记录告警状态、检测恢复条件、生成恢复事件，以下是几种常见的实现方案及代码示例：

基于状态机的告警恢复模型

数据库设计

-- 告警记录表
CREATE TABLE alerts (
    id INT AUTO_INCREMENT PRIMARY KEY,
    alert_key VARCHAR(100) NOT NULL,  -- 唯一标识（如：server_cpu_high）
    status ENUM('firing', 'resolved', 'acknowledged') DEFAULT 'firing',
    start_time DATETIME,
    end_time DATETIME,
    value DECIMAL(10,2),
    threshold DECIMAL(10,2),
    created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
    updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
    UNIQUE KEY `uk_alert_key_status` (`alert_key`, `status`(1))  -- 部分索引
) ENGINE=InnoDB;
-- 告警历史表
CREATE TABLE alert_history (
    id INT AUTO_INCREMENT PRIMARY KEY,
    alert_id INT,
    event_type ENUM('firing', 'resolved', 'escalated'),
    message TEXT,
    created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
    FOREIGN KEY (alert_id) REFERENCES alerts(id)
);

核心恢复逻辑

class AlertManager {
    private $db;
    private $recoveryWindow = 300; // 5分钟恢复窗口
    public function evaluateAlert($alertKey, $currentValue, $threshold) {
        // 1. 检查当前是否存在firing状态的告警
        $activeAlert = $this->getActiveAlert($alertKey);
        if ($activeAlert) {
            // 2. 如果存在活跃告警，检查是否满足恢复条件
            if ($this->isRecovered($currentValue, $threshold)) {
                $this->resolveAlert($activeAlert['id']);
                $this->logRecoveryEvent($activeAlert['id'], $currentValue);
                return 'resolved';
            }
            // 3. 告警持续中，更新值
            $this->updateActiveValue($activeAlert['id'], $currentValue);
            return 'still_firing';
        } else {
            // 4. 没有活跃告警，检查是否需要触发新告警
            if ($currentValue > $threshold) {
                return $this->createAlert($alertKey, $currentValue, $threshold);
            }
            return 'no_alert';
        }
    }
    private function isRecovered($currentValue, $threshold) {
        // 基于阈值判断恢复（支持滞后机制）
        $hysteresisFactor = 0.9; // 恢复阈值 = 告警阈值 * 0.9
        return $currentValue <= ($threshold * $hysteresisFactor);
    }
    private function getActiveAlert($alertKey) {
        $sql = "SELECT * FROM alerts 
                WHERE alert_key = ? AND status = 'firing' 
                ORDER BY created_at DESC LIMIT 1";
        // 执行查询...
    }
    private function resolveAlert($alertId) {
        $sql = "UPDATE alerts SET status = 'resolved', end_time = NOW() 
                WHERE id = ? AND status = 'firing'";
        // 执行更新...
    }
}

时间窗口恢复策略

实现连续采样恢复（防止抖动）

class TimeWindowRecovery {
    private $windowSize = 300; // 5分钟窗口
    private $sampleCount = 3;   // 需要连续3次正常
    public function checkRecovery($alertKey, $currentValue, $threshold) {
        // 获取最近N次采样数据
        $samples = $this->getRecentSamples($alertKey, $this->sampleCount);
        // 检查是否所有采样都低于阈值
        $allRecovered = true;
        foreach ($samples as $sample) {
            if ($sample['value'] > $threshold) {
                $allRecovered = false;
                break;
            }
        }
        if ($allRecovered) {
            // 更新恢复计数
            $this->incrementRecoveryCount($alertKey);
            // 检查是否达到确认恢复次数
            if ($this->getRecoveryCount($alertKey) >= $this->sampleCount) {
                return true;
            }
        } else {
            // 重置恢复计数
            $this->resetRecoveryCount($alertKey);
        }
        return false;
    }
    private function getRecentSamples($alertKey, $count) {
        // 从监控数据库获取最近采样数据
        $sql = "SELECT value FROM metrics 
                WHERE metric_key = ? 
                ORDER BY timestamp DESC LIMIT ?";
        // 执行查询...
    }
}

基于心跳的恢复机制

适用于服务可用性监控

class HeartbeatRecovery {
    private $missedThreshold = 3; // 允许丢失3次心跳后恢复
    public function processHeartbeat($serviceId) {
        // 记录心跳时间
        $this->recordHeartbeat($serviceId);
        // 检查服务当前告警状态
        $alertStatus = $this->getServiceAlertStatus($serviceId);
        if ($alertStatus === 'firing') {
            // 检查是否收到连续心跳
            $consecutiveHeartbeats = $this->getConsecutiveHeartbeats($serviceId);
            if ($consecutiveHeartbeats >= $this->missedThreshold) {
                $this->resolveAlert($serviceId);
                $this->notifyRecovery($serviceId, '服务已恢复');
            }
        }
    }
    private function getConsecutiveHeartbeats($serviceId) {
        $sql = "SELECT COUNT(*) as cnt FROM heartbeats 
                WHERE service_id = ? 
                AND received_at > DATE_SUB(NOW(), INTERVAL 5 MINUTE)
                AND is_missed = 0
                ORDER BY received_at DESC";
        // 执行查询，判断连续心跳数...
    }
}

自动恢复执行器

在恢复时执行自动修复操作

class AutoRecoveryExecutor {
    private $actions = [
        'server_cpu_high' => [
            'type' => 'command',
            'command' => 'systemctl restart nginx',
            'cooldown' => 600
        ],
        'disk_space_low' => [
            'type' => 'script',
            'path' => '/opt/scripts/clean_disk.sh',
            'params' => ['target_dir' => '/var/log']
        ],
        'service_down' => [
            'type' => 'api',
            'url' => 'http://orchestrator/restart-service',
            'method' => 'POST'
        ]
    ];
    public function executeAutoRecovery($alertKey, $alertId) {
        if (!isset($this->actions[$alertKey])) {
            return false;
        }
        $action = $this->actions[$alertKey];
        // 检查冷却时间
        if (!$this->checkCooldown($alertKey, $action['cooldown'])) {
            return false;
        }
        try {
            switch ($action['type']) {
                case 'command':
                    exec($action['command'], $output, $returnVar);
                    break;
                case 'script':
                    $this->executeScript($action);
                    break;
                case 'api':
                    $this->callApi($action);
                    break;
            }
            // 记录执行日志
            $this->logAutoRecovery($alertId, 'success', $action);
            return true;
        } catch (Exception $e) {
            $this->logAutoRecovery($alertId, 'failed', $action, $e->getMessage());
            return false;
        }
    }
}

告警恢复通知

通过多渠道发送恢复通知

class RecoveryNotifier {
    private $channels = ['email', 'slack', 'webhook'];
    public function notifyRecovery($alertData) {
        $message = $this->buildRecoveryMessage($alertData);
        foreach ($this->channels as $channel) {
            try {
                switch ($channel) {
                    case 'email':
                        $this->sendEmail($alertData['contacts'], $message);
                        break;
                    case 'slack':
                        $this->sendSlack($alertData['slack_webhook'], $message);
                        break;
                    case 'webhook':
                        $this->sendWebhook($alertData['webhook_url'], $message);
                        break;
                }
            } catch (Exception $e) {
                error_log("Recovery notification failed on $channel: " . $e->getMessage());
            }
        }
    }
    private function buildRecoveryMessage($alertData) {
        return sprintf(
            "[RECOVERED] 告警: %s\n状态: 已恢复\n恢复时间: %s\n持续时长: %s\n当前值: %.2f (阈值: %.2f)",
            $alertData['name'],
            date('Y-m-d H:i:s'),
            $this->formatDuration($alertData['duration']),
            $alertData['current_value'],
            $alertData['threshold']
        );
    }
}

完整的告警生命周期管理

class AlertLifecycle {
    public function processAlertEvaluation($alertKey, $currentValue) {
        $alertConfig = $this->getAlertConfig($alertKey);
        // 1. 检查是否满足触发条件
        if ($currentValue > $alertConfig['trigger_threshold']) {
            // 告警触发逻辑
            $alertId = $this->triggerAlert($alertKey, $currentValue, $alertConfig);
        }
        // 2. 检查是否有活跃告警需要恢复
        $this->checkRecovery($alertKey, $currentValue, $alertConfig);
        // 3. 评估告警严重性（可选升级）
        $this->evaluateSeverity($alertKey, $currentValue);
        // 4. 执行自动恢复（如果有配置）
        if ($alertConfig['auto_recovery']) {
            $this->executeAutoRecovery($alertKey);
        }
    }
    private function checkRecovery($alertKey, $currentValue, $config) {
        $activeAlert = $this->getActiveAlert($alertKey);
        if (!$activeAlert) {
            return;
        }
        // 恢复条件判断
        $recoveryThreshold = $config['recovery_threshold'] ?? ($config['trigger_threshold'] * 0.9);
        $recoveryDuration = $config['recovery_duration'] ?? 300; // 5分钟
        if ($currentValue <= $recoveryThreshold) {
            // 检查持续正常时间
            $normalDuration = $this->getConsecutiveNormalDuration($alertKey);
            if ($normalDuration >= $recoveryDuration) {
                $this->resolveAlert($activeAlert['id']);
                $this->notifyRecovery($activeAlert);
                $this->logRecoveryEvent($activeAlert);
                return true;
            }
        } else {
            // 值再次超标，重置正常时间计数
            $this->resetNormalDuration($alertKey);
        }
        return false;
    }
}