本文目录导读：
我将为您提供一个完整的Java故障演练案例,涵盖常见的故障场景和演练方法。
故障演练框架设计
package com.example.faultsimulator;
import java.util.concurrent.*;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicLong;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
public class FaultSimulator {
private static volatile FaultSimulator instance;
private final Map<String, FaultRule> faultRules = new ConcurrentHashMap<>();
private final AtomicBoolean enabled = new AtomicBoolean(false);
private FaultSimulator() {}
public static FaultSimulator getInstance() {
if (instance == null) {
synchronized (FaultSimulator.class) {
if (instance == null) {
instance = new FaultSimulator();
}
}
}
return instance;
}
/**
* 注册故障规则
*/
public void registerFaultRule(String name, FaultRule rule) {
faultRules.put(name, rule);
enabled.set(true);
}
/**
* 模拟故障触发
*/
public void triggerFault(String ruleName) {
FaultRule rule = faultRules.get(ruleName);
if (rule != null && enabled.get()) {
rule.execute();
}
}
public void disableAllFaults() {
enabled.set(false);
faultRules.clear();
}
/**
* 故障规则接口
*/
public interface FaultRule {
void execute();
}
}
常见故障场景实现
package com.example.faultsimulator.scenarios;
import com.example.faultsimulator.FaultSimulator;
import java.util.concurrent.*;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.Random;
public class CommonFaultScenarios {
/**
* 场景1: CPU飙升故障
*/
public static class CPUHighUsageFault implements FaultSimulator.FaultRule {
private volatile boolean running = true;
private ExecutorService executor;
@Override
public void execute() {
int cores = Runtime.getRuntime().availableProcessors();
executor = Executors.newFixedThreadPool(cores);
for (int i = 0; i < cores; i++) {
executor.submit(() -> {
while (running) {
// 死循环消耗CPU
Math.pow(Math.random(), Math.random());
}
});
}
// 自动恢复定时器
ScheduledExecutorService scheduler = Executors.newScheduledThreadPool(1);
scheduler.schedule(() -> stop(), 30, TimeUnit.SECONDS);
}
public void stop() {
running = false;
if (executor != null) {
executor.shutdownNow();
}
}
}
/**
* 场景2: 内存溢出故障
*/
public static class MemoryLeakFault implements FaultSimulator.FaultRule {
private final java.util.List<byte[]> memoryHog = new java.util.ArrayList<>();
@Override
public void execute() {
ScheduledExecutorService scheduler = Executors.newScheduledThreadPool(1);
scheduler.scheduleAtFixedRate(() -> {
// 每次分配10MB内存
byte[] data = new byte[10 * 1024 * 1024];
memoryHog.add(data);
// 打印当前内存使用情况
Runtime runtime = Runtime.getRuntime();
long usedMemory = (runtime.totalMemory() - runtime.freeMemory()) / (1024 * 1024);
System.out.println("已使用内存: " + usedMemory + "MB");
}, 0, 2, TimeUnit.SECONDS);
}
}
/**
* 场景3: 线程死锁故障
*/
public static class DeadlockFault implements FaultSimulator.FaultRule {
private final Object lock1 = new Object();
private final Object lock2 = new Object();
@Override
public void execute() {
// 线程1持有lock1，等待lock2
new Thread(() -> {
synchronized (lock1) {
System.out.println("线程1持有lock1");
try { Thread.sleep(500); } catch (InterruptedException e) {}
synchronized (lock2) {
System.out.println("线程1已获取lock2");
}
}
}).start();
// 线程2持有lock2，等待lock1
new Thread(() -> {
synchronized (lock2) {
System.out.println("线程2持有lock2");
try { Thread.sleep(500); } catch (InterruptedException e) {}
synchronized (lock1) {
System.out.println("线程2已获取lock1");
}
}
}).start();
}
}
/**
* 场景4: 响应延迟故障
*/
public static class LatencyFault implements FaultSimulator.FaultRule {
private final long delayMillis;
private final double delayProbability;
public LatencyFault(long delayMillis, double delayProbability) {
this.delayMillis = delayMillis;
this.delayProbability = delayProbability;
}
@Override
public void execute() {
Random random = new Random();
// 通过AOP或代理方式实现
if (random.nextDouble() < delayProbability) {
try {
Thread.sleep(delayMillis);
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
}
}
}
}
/**
* 场景5: 网络异常故障
*/
public static class NetworkExceptionFault implements FaultSimulator.FaultRule {
private final double exceptionProbability;
public NetworkExceptionFault(double exceptionProbability) {
this.exceptionProbability = exceptionProbability;
}
@Override
public void execute() {
Random random = new Random();
if (random.nextDouble() < exceptionProbability) {
throw new RuntimeException("模拟网络连接异常");
}
}
}
}
业务系统模拟
package com.example.faultsimulator.business;
import java.util.concurrent.*;
import java.util.Random;
public class BusinessService {
private final ExecutorService executorService = Executors.newFixedThreadPool(10);
private final Random random = new Random();
/**
* 模拟订单处理
*/
public String processOrder(String orderId) {
// 模拟业务处理逻辑
long startTime = System.currentTimeMillis();
try {
// 模拟数据库操作
Thread.sleep(100 + random.nextInt(200));
// 模拟外部接口调用
callExternalService();
// 模拟数据处理
processData();
long endTime = System.currentTimeMillis();
return "订单 " + orderId + " 处理成功，耗时: " + (endTime - startTime) + "ms";
} catch (Exception e) {
return "订单 " + orderId + " 处理失败: " + e.getMessage();
}
}
/**
* 模拟高并发请求
*/
public void simulateHighConcurrency(int concurrentClients, int requestsPerClient) {
CountDownLatch latch = new CountDownLatch(concurrentClients);
for (int i = 0; i < concurrentClients; i++) {
final int clientId = i;
executorService.submit(() -> {
try {
for (int j = 0; j < requestsPerClient; j++) {
String orderId = "Order-" + clientId + "-" + j;
processOrder(orderId);
}
} finally {
latch.countDown();
}
});
}
try {
latch.await();
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
}
}
private void callExternalService() throws InterruptedException {
// 模拟外部服务调用
Thread.sleep(random.nextInt(100));
}
private void processData() throws InterruptedException {
// 模拟数据处理
Thread.sleep(random.nextInt(100));
}
}
监控和告警系统
package com.example.faultsimulator.monitor;
import java.lang.management.*;
import java.util.concurrent.*;
import java.util.concurrent.atomic.AtomicLong;
public class SystemMonitor {
private final ScheduledExecutorService scheduler = Executors.newScheduledThreadPool(1);
private final MemoryMXBean memoryMXBean = ManagementFactory.getMemoryMXBean();
private final ThreadMXBean threadMXBean = ManagementFactory.getThreadMXBean();
private final AtomicLong totalRequests = new AtomicLong(0);
private final AtomicLong failedRequests = new AtomicLong(0);
public void startMonitoring() {
scheduler.scheduleAtFixedRate(this::monitor, 0, 5, TimeUnit.SECONDS);
}
private void monitor() {
// CPU使用率
double cpuUsage = getCpuUsage();
// 内存使用情况
MemoryUsage heapMemoryUsage = memoryMXBean.getHeapMemoryUsage();
long usedMemory = heapMemoryUsage.getUsed() / (1024 * 1024);
long maxMemory = heapMemoryUsage.getMax() / (1024 * 1024);
// 线程数量
int threadCount = threadMXBean.getThreadCount();
// 死锁检测
long[] deadlockedThreads = threadMXBean.findDeadlockedThreads();
// 请求统计
long total = totalRequests.get();
long failed = failedRequests.get();
double errorRate = total > 0 ? (double) failed / total * 100 : 0;
// 打印监控信息
System.out.println("========== 系统监控 ==========");
System.out.printf("CPU使用率: %.2f%%%n", cpuUsage);
System.out.printf("内存使用: %dMB / %dMB%n", usedMemory, maxMemory);
System.out.println("活动线程数: " + threadCount);
System.out.println("死锁线程: " + (deadlockedThreads != null ? deadlockedThreads.length : 0));
System.out.printf("请求总数: %d, 失败率: %.2f%%%n", total, errorRate);
System.out.println("================================");
}
private double getCpuUsage() {
OperatingSystemMXBean osBean = ManagementFactory.getOperatingSystemMXBean();
com.sun.management.OperatingSystemMXBean sunOsBean =
(com.sun.management.OperatingSystemMXBean) osBean;
return sunOsBean.getProcessCpuLoad() * 100;
}
public void incrementRequests() {
totalRequests.incrementAndGet();
}
public void incrementFailedRequests() {
failedRequests.incrementAndGet();
}
}
故障演练执行器
package com.example.faultsimulator;
import com.example.faultsimulator.scenarios.CommonFaultScenarios;
import com.example.faultsimulator.business.BusinessService;
import com.example.faultsimulator.monitor.SystemMonitor;
import java.util.Scanner;
import java.util.concurrent.*;
public class FaultDrillExecutor {
private final FaultSimulator faultSimulator = FaultSimulator.getInstance();
private final BusinessService businessService = new BusinessService();
private final SystemMonitor systemMonitor = new SystemMonitor();
/**
* 执行故障演练
*/
public void executeDrill() {
System.out.println("====== Java故障演练系统 ======");
System.out.println("请选择故障演练场景:");
System.out.println("1. CPU飙升故障");
System.out.println("2. 内存溢出故障");
System.out.println("3. 线程死锁故障");
System.out.println("4. 响应延迟故障");
System.out.println("5. 网络异常故障");
System.out.println("6. 高并发压力演练");
System.out.println("0. 退出");
Scanner scanner = new Scanner(System.in);
int choice = scanner.nextInt();
// 启动监控
systemMonitor.startMonitoring();
switch (choice) {
case 1:
cpuHignUsageDrill();
break;
case 2:
memoryLeakDrill();
break;
case 3:
deadlockDrill();
break;
case 4:
latencyDrill();
break;
case 5:
networkExceptionDrill();
break;
case 6:
highConcurrencyDrill();
break;
case 0:
System.out.println("退出系统");
System.exit(0);
break;
default:
System.out.println("无效选项");
}
}
/**
* CPU飙升演练
*/
private void cpuHignUsageDrill() {
System.out.println("开始CPU飙升故障演练...");
// 注册故障
CommonFaultScenarios.CPUHighUsageFault cpuFault =
new CommonFaultScenarios.CPUHighUsageFault();
faultSimulator.registerFaultRule("cpu-exhaustion", cpuFault);
// 触发故障
faultSimulator.triggerFault("cpu-exhaustion");
// 业务请求
ScheduledExecutorService requestExecutor = Executors.newScheduledThreadPool(1);
requestExecutor.scheduleAtFixedRate(() -> {
SystemMonitor monitor = getMonitor();
businessService.processOrder("CPU-Fault-Order");
}, 0, 1, TimeUnit.SECONDS);
// 30秒后自动恢复
ScheduledExecutorService recoveryExecutor = Executors.newScheduledThreadPool(1);
recoveryExecutor.schedule(() -> {
cpuFault.stop();
System.out.println("CPU故障演练完成，系统状态恢复");
}, 30, TimeUnit.SECONDS);
}
/**
* 内存溢出演练
*/
private void memoryLeakDrill() {
System.out.println("开始内存溢出故障演练...");
CommonFaultScenarios.MemoryLeakFault memoryFault =
new CommonFaultScenarios.MemoryLeakFault();
// 设置JVM参数
System.setProperty("JAVA_OPTS", "-Xmx256m");
// 触发故障
Thread faultThread = new Thread(() -> {
memoryFault.execute();
// 定时检查内存
while (true) {
Runtime runtime = Runtime.getRuntime();
long usedMemory = (runtime.totalMemory() - runtime.freeMemory()) / (1024 * 1024);
long maxMemory = runtime.maxMemory() / (1024 * 1024);
if (usedMemory > maxMemory * 0.8) {
System.out.println("警告: 内存使用超过80%，即将触发OOM");
throw new OutOfMemoryError("模拟内存溢出故障");
}
try {
Thread.sleep(5000);
} catch (InterruptedException e) {
break;
}
}
});
faultThread.start();
}
/**
* 死锁演练
*/
private void deadlockDrill() {
System.out.println("开始线程死锁故障演练...");
CommonFaultScenarios.DeadlockFault deadlockFault =
new CommonFaultScenarios.DeadlockFault();
faultSimulator.registerFaultRule("deadlock", deadlockFault);
faultSimulator.triggerFault("deadlock");
// 等待30秒后检查死锁并恢复
ScheduledExecutorService scheduler = Executors.newScheduledThreadPool(1);
scheduler.schedule(() -> {
ThreadMXBean threadMXBean = ManagementFactory.getThreadMXBean();
long[] deadlockedThreads = threadMXBean.findDeadlockedThreads();
if (deadlockedThreads != null) {
System.out.println("检测到 " + deadlockedThreads.length + " 个死锁线程");
// 中断死锁线程
for (long threadId : deadlockedThreads) {
ThreadInfo threadInfo = threadMXBean.getThreadInfo(threadId);
System.out.println("死锁线程: " + threadInfo.getThreadName());
Thread thread = ThreadUtils.findThread(threadId);
if (thread != null) {
thread.interrupt();
}
}
} else {
System.out.println("未检测到死锁");
}
}, 30, TimeUnit.SECONDS);
}
/**
* 延迟演练
*/
private void latencyDrill() {
System.out.println("开始响应延迟故障演练...");
long delay = 3000; // 3秒延迟
double probability = 0.5; // 50%概率
CommonFaultScenarios.LatencyFault latencyFault =
new CommonFaultScenarios.LatencyFault(delay, probability);
faultSimulator.registerFaultRule("latency", latencyFault);
// 发送10个测试请求
for (int i = 0; i < 10; i++) {
faultSimulator.triggerFault("latency");
long start = System.currentTimeMillis();
String result = businessService.processOrder("Latency-Order-" + i);
long end = System.currentTimeMillis();
System.out.println("请求" + i + ": " + result + ", 实际耗时: " + (end - start) + "ms");
}
}
/**
* 网络异常演练
*/
private void networkExceptionDrill() {
System.out.println("开始网络异常故障演练...");
double probability = 0.3; // 30%概率抛出异常
CommonFaultScenarios.NetworkExceptionFault networkFault =
new CommonFaultScenarios.NetworkExceptionFault(probability);
// 模拟网络异常
for (int i = 0; i < 10; i++) {
try {
networkFault.execute();
System.out.println("请求" + i + ": 网络正常");
} catch (Exception e) {
System.out.println("请求" + i + ": 网络异常 - " + e.getMessage());
systemMonitor.incrementFailedRequests();
}
try {
Thread.sleep(100);
} catch (InterruptedException e) {
break;
}
}
}
/**
* 高并发演练
*/
private void highConcurrencyDrill() {
System.out.println("开始高并发压力演练...");
int concurrentClients = 100;
int requestsPerClient = 50;
ScheduledExecutorService monitorExecutor = Executors.newScheduledThreadPool(1);
monitorExecutor.scheduleAtFixedRate(() -> {
System.out.println("持续高并发压力测试中...");
}, 0, 5, TimeUnit.SECONDS);
businessService.simulateHighConcurrency(concurrentClients, requestsPerClient);
System.out.println("高并发演练完成");
}
private SystemMonitor getMonitor() {
return systemMonitor;
}
public static void main(String[] args) throws InterruptedException {
FaultDrillExecutor executor = new FaultDrillExecutor();
// 添加钩子，确保程序退出时清理资源
Runtime.getRuntime().addShutdownHook(new Thread(() -> {
System.out.println("\n清理资源...");
FaultSimulator.getInstance().disableAllFaults();
}));
// 运行演练
executor.executeDrill();
// 等待演练完成
Thread.sleep(35000); // 等待35秒
System.out.println("\n演练结束，查看监控报告...");
}
}
// 线程工具类
class ThreadUtils {
public static Thread findThread(long threadId) {
for (Thread thread : Thread.getAllStackTraces().keySet()) {
if (thread.getId() == threadId) {
return thread;
}
}
return null;
}
}
故障演练结果分析
package com.example.faultsimulator.analysis;
import java.util.*;
import java.util.concurrent.*;
public class DrillAnalyzer {
private final List<DrillResult> results = new CopyOnWriteArrayList<>();
public void addResult(DrillResult result) {
results.add(result);
}
/**
* 生成分析报告
*/
public void generateReport() {
long totalDuration = 0;
int successCount = 0;
int failedCount = 0;
System.out.println("\n===== 故障演练分析报告 =====");
System.out.println("总演练次数: " + results.size());
for (DrillResult result : results) {
totalDuration += result.getDurationMs();
if (result.isSuccess()) {
successCount++;
} else {
failedCount++;
}
System.out.println("演练场景: " + result.getScenario());
System.out.println("执行时间: " + result.getDurationMs() + "ms");
System.out.println("成功率: " + result.getSuccessRate() + "%");
System.out.println("---");
}
System.out.printf("平均耗时: %.2fms%n", totalDuration / (double) results.size());
System.out.println("成功次数: " + successCount);
System.out.println("失败次数: " + failedCount);
System.out.printf("总体成功率: %.2f%%%n", (successCount / (double) results.size()) * 100);
}
public static class DrillResult {
private final String scenario;
private final long durationMs;
private final boolean success;
private final double successRate;
public DrillResult(String scenario, long durationMs, boolean success, double successRate) {
this.scenario = scenario;
this.durationMs = durationMs;
this.success = success;
this.successRate = successRate;
}
public long getDurationMs() { return durationMs; }
public boolean isSuccess() { return success; }
public double getSuccessRate() { return successRate; }
public String getScenario() { return scenario; }
}
}
使用说明
运行准备
<project>
<dependencies>
<!-- JVM监控依赖 -->
<dependency>
<groupId>com.sun</groupId>
<artifactId>tools</artifactId>
<version>1.8</version>
</dependency>
</dependencies>
</project>
启动命令
# 内存溢出演练时添加JVM参数 java -Xmx256m -Xms128m -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/logs/heap.hprof FaultDrillExecutor # 常规启动 java FaultDrillExecutor
安全建议
- 生产环境限制：故障演练应在测试环境或预生产环境执行
- 自动恢复机制：所有故障规则均应设置自动恢复时间
- 监控告警：演练过程中必须启用实时监控
- 演练审计：记录所有演练操作，便于回溯分析
- 优雅降级：确保系统具有降级和容错机制
这个故障演练框架能够帮助团队：
- 验证系统的容错性
- 测试故障恢复机制
- 评估系统的高可用性
- 培训运维人员的应急处理能力
