本文目录导读：
在PHP项目中实现告警恢复,核心思路是记录告警状态、检测恢复条件、生成恢复事件，以下是几种常见的实现方案及代码示例：
基于状态机的告警恢复模型
数据库设计
-- 告警记录表
CREATE TABLE alerts (
id INT AUTO_INCREMENT PRIMARY KEY,
alert_key VARCHAR(100) NOT NULL, -- 唯一标识（如：server_cpu_high）
status ENUM('firing', 'resolved', 'acknowledged') DEFAULT 'firing',
start_time DATETIME,
end_time DATETIME,
value DECIMAL(10,2),
threshold DECIMAL(10,2),
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
UNIQUE KEY `uk_alert_key_status` (`alert_key`, `status`(1)) -- 部分索引
) ENGINE=InnoDB;
-- 告警历史表
CREATE TABLE alert_history (
id INT AUTO_INCREMENT PRIMARY KEY,
alert_id INT,
event_type ENUM('firing', 'resolved', 'escalated'),
message TEXT,
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
FOREIGN KEY (alert_id) REFERENCES alerts(id)
);
核心恢复逻辑
class AlertManager {
private $db;
private $recoveryWindow = 300; // 5分钟恢复窗口
public function evaluateAlert($alertKey, $currentValue, $threshold) {
// 1. 检查当前是否存在firing状态的告警
$activeAlert = $this->getActiveAlert($alertKey);
if ($activeAlert) {
// 2. 如果存在活跃告警，检查是否满足恢复条件
if ($this->isRecovered($currentValue, $threshold)) {
$this->resolveAlert($activeAlert['id']);
$this->logRecoveryEvent($activeAlert['id'], $currentValue);
return 'resolved';
}
// 3. 告警持续中，更新值
$this->updateActiveValue($activeAlert['id'], $currentValue);
return 'still_firing';
} else {
// 4. 没有活跃告警，检查是否需要触发新告警
if ($currentValue > $threshold) {
return $this->createAlert($alertKey, $currentValue, $threshold);
}
return 'no_alert';
}
}
private function isRecovered($currentValue, $threshold) {
// 基于阈值判断恢复（支持滞后机制）
$hysteresisFactor = 0.9; // 恢复阈值 = 告警阈值 * 0.9
return $currentValue <= ($threshold * $hysteresisFactor);
}
private function getActiveAlert($alertKey) {
$sql = "SELECT * FROM alerts
WHERE alert_key = ? AND status = 'firing'
ORDER BY created_at DESC LIMIT 1";
// 执行查询...
}
private function resolveAlert($alertId) {
$sql = "UPDATE alerts SET status = 'resolved', end_time = NOW()
WHERE id = ? AND status = 'firing'";
// 执行更新...
}
}
时间窗口恢复策略
实现连续采样恢复（防止抖动）
class TimeWindowRecovery {
private $windowSize = 300; // 5分钟窗口
private $sampleCount = 3; // 需要连续3次正常
public function checkRecovery($alertKey, $currentValue, $threshold) {
// 获取最近N次采样数据
$samples = $this->getRecentSamples($alertKey, $this->sampleCount);
// 检查是否所有采样都低于阈值
$allRecovered = true;
foreach ($samples as $sample) {
if ($sample['value'] > $threshold) {
$allRecovered = false;
break;
}
}
if ($allRecovered) {
// 更新恢复计数
$this->incrementRecoveryCount($alertKey);
// 检查是否达到确认恢复次数
if ($this->getRecoveryCount($alertKey) >= $this->sampleCount) {
return true;
}
} else {
// 重置恢复计数
$this->resetRecoveryCount($alertKey);
}
return false;
}
private function getRecentSamples($alertKey, $count) {
// 从监控数据库获取最近采样数据
$sql = "SELECT value FROM metrics
WHERE metric_key = ?
ORDER BY timestamp DESC LIMIT ?";
// 执行查询...
}
}
基于心跳的恢复机制
适用于服务可用性监控
class HeartbeatRecovery {
private $missedThreshold = 3; // 允许丢失3次心跳后恢复
public function processHeartbeat($serviceId) {
// 记录心跳时间
$this->recordHeartbeat($serviceId);
// 检查服务当前告警状态
$alertStatus = $this->getServiceAlertStatus($serviceId);
if ($alertStatus === 'firing') {
// 检查是否收到连续心跳
$consecutiveHeartbeats = $this->getConsecutiveHeartbeats($serviceId);
if ($consecutiveHeartbeats >= $this->missedThreshold) {
$this->resolveAlert($serviceId);
$this->notifyRecovery($serviceId, '服务已恢复');
}
}
}
private function getConsecutiveHeartbeats($serviceId) {
$sql = "SELECT COUNT(*) as cnt FROM heartbeats
WHERE service_id = ?
AND received_at > DATE_SUB(NOW(), INTERVAL 5 MINUTE)
AND is_missed = 0
ORDER BY received_at DESC";
// 执行查询，判断连续心跳数...
}
}
自动恢复执行器
在恢复时执行自动修复操作
class AutoRecoveryExecutor {
private $actions = [
'server_cpu_high' => [
'type' => 'command',
'command' => 'systemctl restart nginx',
'cooldown' => 600
],
'disk_space_low' => [
'type' => 'script',
'path' => '/opt/scripts/clean_disk.sh',
'params' => ['target_dir' => '/var/log']
],
'service_down' => [
'type' => 'api',
'url' => 'http://orchestrator/restart-service',
'method' => 'POST'
]
];
public function executeAutoRecovery($alertKey, $alertId) {
if (!isset($this->actions[$alertKey])) {
return false;
}
$action = $this->actions[$alertKey];
// 检查冷却时间
if (!$this->checkCooldown($alertKey, $action['cooldown'])) {
return false;
}
try {
switch ($action['type']) {
case 'command':
exec($action['command'], $output, $returnVar);
break;
case 'script':
$this->executeScript($action);
break;
case 'api':
$this->callApi($action);
break;
}
// 记录执行日志
$this->logAutoRecovery($alertId, 'success', $action);
return true;
} catch (Exception $e) {
$this->logAutoRecovery($alertId, 'failed', $action, $e->getMessage());
return false;
}
}
}
告警恢复通知
通过多渠道发送恢复通知
class RecoveryNotifier {
private $channels = ['email', 'slack', 'webhook'];
public function notifyRecovery($alertData) {
$message = $this->buildRecoveryMessage($alertData);
foreach ($this->channels as $channel) {
try {
switch ($channel) {
case 'email':
$this->sendEmail($alertData['contacts'], $message);
break;
case 'slack':
$this->sendSlack($alertData['slack_webhook'], $message);
break;
case 'webhook':
$this->sendWebhook($alertData['webhook_url'], $message);
break;
}
} catch (Exception $e) {
error_log("Recovery notification failed on $channel: " . $e->getMessage());
}
}
}
private function buildRecoveryMessage($alertData) {
return sprintf(
"[RECOVERED] 告警: %s\n状态: 已恢复\n恢复时间: %s\n持续时长: %s\n当前值: %.2f (阈值: %.2f)",
$alertData['name'],
date('Y-m-d H:i:s'),
$this->formatDuration($alertData['duration']),
$alertData['current_value'],
$alertData['threshold']
);
}
}
完整的告警生命周期管理
class AlertLifecycle {
public function processAlertEvaluation($alertKey, $currentValue) {
$alertConfig = $this->getAlertConfig($alertKey);
// 1. 检查是否满足触发条件
if ($currentValue > $alertConfig['trigger_threshold']) {
// 告警触发逻辑
$alertId = $this->triggerAlert($alertKey, $currentValue, $alertConfig);
}
// 2. 检查是否有活跃告警需要恢复
$this->checkRecovery($alertKey, $currentValue, $alertConfig);
// 3. 评估告警严重性（可选升级）
$this->evaluateSeverity($alertKey, $currentValue);
// 4. 执行自动恢复（如果有配置）
if ($alertConfig['auto_recovery']) {
$this->executeAutoRecovery($alertKey);
}
}
private function checkRecovery($alertKey, $currentValue, $config) {
$activeAlert = $this->getActiveAlert($alertKey);
if (!$activeAlert) {
return;
}
// 恢复条件判断
$recoveryThreshold = $config['recovery_threshold'] ?? ($config['trigger_threshold'] * 0.9);
$recoveryDuration = $config['recovery_duration'] ?? 300; // 5分钟
if ($currentValue <= $recoveryThreshold) {
// 检查持续正常时间
$normalDuration = $this->getConsecutiveNormalDuration($alertKey);
if ($normalDuration >= $recoveryDuration) {
$this->resolveAlert($activeAlert['id']);
$this->notifyRecovery($activeAlert);
$this->logRecoveryEvent($activeAlert);
return true;
}
} else {
// 值再次超标，重置正常时间计数
$this->resetNormalDuration($alertKey);
}
return false;
}
}
最佳实践建议
- 避免抖动：使用滞后机制（hysteresis），恢复阈值低于触发阈值
- 确认机制：连续多个采样正常才确认恢复
- 时间窗口：设置合理的恢复确认时间窗口
- 幂等性：确保恢复操作可重复执行且安全
- 监控恢复：记录恢复操作的执行和结果
- 优先级处理：高优先级告警可能需要更快的恢复确认
通过合理设计状态机和恢复策略,可以实现可靠的告警恢复机制，避免误报和抖动。
