AI系统专属故障模式：
├── LLM API层
│   ├── 模型API超时（30s+）
│   ├── 模型API限流（429 Too Many Requests）
│   ├── 模型返回乱码/截断响应
│   ├── 模型版本静默升级（行为改变）
│   └── Token超出上下文窗口
├── 提示词层
│   ├── 提示词注入攻击
│   ├── 提示词过长导致截断
│   └── 模板变量缺失
├── 数据层
│   ├── 向量数据库不可用
│   ├── 检索返回空结果
│   └── 知识库数据陈旧
└── 模型行为层
    ├── 幻觉率突然升高
    ├── 输出格式不符合预期
    └── 响应长度异常（过短/过长）

1.2 混沌实验矩阵

// ChaosExperimentRegistry.java
@Component
public class ChaosExperimentRegistry {
    
    // 30个实验场景
    public static final List<ChaosExperiment> ALL_EXPERIMENTS = List.of(
        
        // === LLM API故障 ===
        ChaosExperiment.of("llm-api-timeout-5s", 
            "LLM API 5秒超时", Severity.HIGH,
            new TimeoutFaultInjector(5000)),
        
        ChaosExperiment.of("llm-api-timeout-30s",
            "LLM API 30秒超时", Severity.HIGH,
            new TimeoutFaultInjector(30000)),
        
        ChaosExperiment.of("llm-api-rate-limit",
            "LLM API 限流 (429)", Severity.HIGH,
            new HttpStatusFaultInjector(429)),
        
        ChaosExperiment.of("llm-api-server-error",
            "LLM API 服务器错误 (500)", Severity.HIGH,
            new HttpStatusFaultInjector(500)),
        
        ChaosExperiment.of("llm-api-partial-response",
            "LLM API 返回截断响应", Severity.MEDIUM,
            new TruncatedResponseFaultInjector(0.5)),  // 截断50%
        
        ChaosExperiment.of("llm-api-malformed-json",
            "LLM API 返回无效JSON", Severity.MEDIUM,
            new MalformedJsonFaultInjector()),
        
        // === 向量数据库故障 ===
        ChaosExperiment.of("vector-db-unavailable",
            "向量数据库完全不可用", Severity.HIGH,
            new ServiceUnavailableFaultInjector("vector-db")),
        
        ChaosExperiment.of("vector-db-slow",
            "向量数据库响应慢 (3s)", Severity.MEDIUM,
            new SlowResponseFaultInjector("vector-db", 3000)),
        
        ChaosExperiment.of("vector-db-empty-results",
            "向量检索返回空结果", Severity.MEDIUM,
            new EmptyResultsFaultInjector()),
        
        // === 上下文/Token故障 ===
        ChaosExperiment.of("context-token-exceeded",
            "超出上下文Token限制", Severity.MEDIUM,
            new TokenOverflowFaultInjector()),
        
        ChaosExperiment.of("context-history-lost",
            "对话历史丢失", Severity.LOW,
            new ContextLostFaultInjector())
        
        // ... 更多实验
    );
}

二、混沌实验框架：Spring AI拦截器实现

2.1 混沌注入拦截器

// ChaosInjectionAdvisor.java
@Component
@Order(Ordered.HIGHEST_PRECEDENCE)
public class ChaosInjectionAdvisor implements CallAroundAdvisor {
    
    private final ChaosExperimentController experimentController;
    private final Random random = new Random();
    
    @Value("${chaos.enabled:false}")
    private boolean chaosEnabled;
    
    @Override
    public String getName() {
        return "ChaosInjectionAdvisor";
    }
    
    @Override
    public int getOrder() {
        return 0;
    }
    
    @Override
    public AdvisedResponse aroundCall(AdvisedRequest advisedRequest, 
                                       CallAroundAdvisorChain chain) {
        if (!chaosEnabled) {
            return chain.nextAroundCall(advisedRequest);
        }
        
        ActiveChaosExperiment experiment = experimentController.getActiveExperiment();
        
        if (experiment == null) {
            return chain.nextAroundCall(advisedRequest);
        }
        
        // 按照实验的注入概率随机触发
        if (random.nextDouble() > experiment.getInjectionRate()) {
            return chain.nextAroundCall(advisedRequest);
        }
        
        log.info("[混沌注入] 触发实验: {}", experiment.getName());
        
        return experiment.getFaultInjector().inject(advisedRequest, chain);
    }
}

// TimeoutFaultInjector.java
public class TimeoutFaultInjector implements FaultInjector {
    
    private final long timeoutMs;
    
    @Override
    public AdvisedResponse inject(AdvisedRequest request, CallAroundAdvisorChain chain) {
        // 模拟超时
        try {
            Thread.sleep(timeoutMs);
        } catch (InterruptedException e) {
            Thread.currentThread().interrupt();
        }
        
        throw new RuntimeException("Chaos: Simulated timeout after " + timeoutMs + "ms");
    }
}

// MalformedJsonFaultInjector.java（模拟LLM返回不规则JSON）
public class MalformedJsonFaultInjector implements FaultInjector {
    
    private static final List<String> MALFORMED_RESPONSES = List.of(
        "{\"result\": \"incomplete response...",  // 截断JSON
        "Here is the JSON: {\"key\": }",           // 语法错误
        "{result: 'unquoted_keys'}",               // 非标准JSON
        "Sorry, I cannot complete this request"    // 完全非JSON
    );
    
    @Override
    public AdvisedResponse inject(AdvisedRequest request, CallAroundAdvisorChain chain) {
        // 返回一个带乱码内容的响应
        String malformed = MALFORMED_RESPONSES.get(
            (int)(Math.random() * MALFORMED_RESPONSES.size()));
        
        AssistantMessage message = new AssistantMessage(malformed);
        ChatResponse response = new ChatResponse(
            List.of(new Generation(message)));
        
        return new AdvisedResponse(response, request.adviseContext());
    }
}

2.2 混沌实验控制台

// ChaosExperimentController.java（REST API）
@RestController
@RequestMapping("/api/chaos")
@Slf4j
public class ChaosController {
    
    private final ChaosExperimentController experimentController;
    private final ChaosReportService reportService;
    
    // 开始实验
    @PostMapping("/experiments/{name}/start")
    @PreAuthorize("hasRole('CHAOS_ENGINEER')")
    public ResponseEntity<ExperimentStatus> startExperiment(
            @PathVariable String name,
            @RequestParam(defaultValue = "0.1") double injectionRate,
            @RequestParam(defaultValue = "300") int durationSeconds) {
        
        ChaosExperiment experiment = ChaosExperimentRegistry.find(name)
            .orElseThrow(() -> new NotFoundException("实验不存在: " + name));
        
        experimentController.start(experiment, injectionRate, durationSeconds);
        
        log.info("[混沌] 开始实验 [{}], 注入率={}%, 持续={}秒",
            name, injectionRate * 100, durationSeconds);
        
        return ResponseEntity.ok(ExperimentStatus.running(name));
    }
    
    // 停止实验
    @PostMapping("/experiments/stop")
    public ResponseEntity<String> stopAllExperiments() {
        experimentController.stopAll();
        return ResponseEntity.ok("所有混沌实验已停止");
    }
    
    // 获取实验报告
    @GetMapping("/reports/{experimentName}")
    public ResponseEntity<ChaosReport> getReport(@PathVariable String experimentName) {
        return ResponseEntity.ok(reportService.getReport(experimentName));
    }
    
    // 自动化混沌测试（在测试环境定时运行）
    @PostMapping("/auto-test")
    public ResponseEntity<String> runAutoTest() {
        CompletableFuture.runAsync(() -> {
            for (ChaosExperiment experiment : ChaosExperimentRegistry.SAFE_TO_AUTO_RUN) {
                log.info("自动运行混沌实验: {}", experiment.getName());
                experimentController.start(experiment, 0.2, 60);
                
                try {
                    Thread.sleep(90000);  // 等90秒
                } catch (InterruptedException e) {
                    break;
                }
                
                experimentController.stopAll();
                
                // 检查系统是否自愈
                boolean recovered = checkSystemHealth();
                reportService.recordAutoTestResult(experiment.getName(), recovered);
            }
        });
        
        return ResponseEntity.ok("自动化混沌测试已启动");
    }
}

三、自愈机制：三层架构

3.1 感知层：实时健康监测

// AiSystemHealthMonitor.java
@Component
@Slf4j
public class AiSystemHealthMonitor {
    
    private final MeterRegistry meterRegistry;
    private final ApplicationEventPublisher eventPublisher;
    
    // 滑动窗口统计（最近60秒）
    private final SlidingWindowCounter errorCounter;
    private final SlidingWindowCounter requestCounter;
    private final SlidingWindowLatency latencyTracker;
    
    // 连续错误计数
    private final AtomicInteger consecutiveErrors = new AtomicInteger(0);
    
    @Scheduled(fixedRate = 5000)  // 每5秒评估一次健康状态
    public void evaluateHealth() {
        HealthStatus status = calculateHealthStatus();
        
        if (status.isDegraded()) {
            log.warn("AI系统健康状态恶化: {}", status);
            eventPublisher.publishEvent(new HealthDegradedEvent(this, status));
        }
        
        // 更新健康指标
        meterRegistry.gauge("ai.system.health.score", 
            status.getHealthScore());
    }
    
    private HealthStatus calculateHealthStatus() {
        double errorRate = errorCounter.getRate(TimeUnit.MINUTES);
        double requestRate = requestCounter.getRate(TimeUnit.MINUTES);
        double p95Latency = latencyTracker.getPercentile(0.95);
        int consecutive = consecutiveErrors.get();
        
        // 综合健康评分（0-100）
        double score = 100.0;
        
        if (errorRate > 0.05) score -= 30;      // 错误率>5%扣30分
        if (errorRate > 0.20) score -= 30;       // 错误率>20%再扣30分
        if (p95Latency > 5000) score -= 20;      // P95延迟>5s扣20分
        if (p95Latency > 15000) score -= 20;     // P95延迟>15s再扣20分
        if (consecutive > 5) score -= 20;         // 连续5个错误扣20分
        if (requestRate < 1) score -= 10;         // 请求量异常低（可能宕机）
        
        return HealthStatus.builder()
            .score(Math.max(0, score))
            .errorRate(errorRate)
            .p95LatencyMs(p95Latency)
            .consecutiveErrors(consecutive)
            .timestamp(LocalDateTime.now())
            .build();
    }
    
    // 记录请求结果（供Processor调用）
    public void recordResult(boolean success, long latencyMs) {
        requestCounter.add(1);
        latencyTracker.record(latencyMs);
        
        if (success) {
            consecutiveErrors.set(0);
        } else {
            errorCounter.add(1);
            consecutiveErrors.incrementAndGet();
        }
    }
}

3.2 决策层：自愈决策引擎

// SelfHealingDecisionEngine.java
@Component
@Slf4j
public class SelfHealingDecisionEngine {
    
    @EventListener
    public void onHealthDegraded(HealthDegradedEvent event) {
        HealthStatus status = event.getStatus();
        
        log.warn("触发自愈决策，健康评分: {}", status.getHealthScore());
        
        // 根据健康状态决定自愈策略
        List<HealingAction> actions = decideActions(status);
        
        // 按优先级执行
        actions.sort(Comparator.comparing(HealingAction::getPriority).reversed());
        
        for (HealingAction action : actions) {
            try {
                action.execute();
                log.info("自愈动作执行成功: {}", action.getName());
            } catch (Exception e) {
                log.error("自愈动作执行失败 [{}]: {}", action.getName(), e.getMessage());
            }
        }
    }
    
    private List<HealingAction> decideActions(HealthStatus status) {
        List<HealingAction> actions = new ArrayList<>();
        
        // 规则1：连续错误超过3次 → 切换备用模型
        if (status.getConsecutiveErrors() > 3) {
            actions.add(new SwitchModelAction("gpt-4o-mini", Duration.ofMinutes(5)));
        }
        
        // 规则2：P95延迟超过10秒 → 降级（使用缓存答案）
        if (status.getP95LatencyMs() > 10000) {
            actions.add(new EnableCacheFallbackAction());
        }
        
        // 规则3：错误率超过30% → 限流
        if (status.getErrorRate() > 0.3) {
            actions.add(new ThrottleRequestsAction(0.5)); // 降至50%流量
        }
        
        // 规则4：错误率超过50% → 完全降级
        if (status.getErrorRate() > 0.5) {
            actions.add(new FullDegradationAction());
        }
        
        return actions;
    }
}

3.3 执行层：自愈动作实现

// SwitchModelAction.java
@Slf4j
public class SwitchModelAction implements HealingAction {
    
    private final ModelSwitchManager switchManager;
    private final String fallbackModel;
    private final Duration switchDuration;
    
    @Override
    public void execute() {
        log.warn("自愈：切换到备用模型 {}", fallbackModel);
        switchManager.switchTo(fallbackModel, switchDuration);
    }
    
    @Override
    public String getName() {
        return "switch-to-" + fallbackModel;
    }
    
    @Override
    public int getPriority() {
        return 90;  // 高优先级
    }
}

// ModelSwitchManager.java
@Component
@Slf4j
public class ModelSwitchManager {
    
    private final AtomicReference<String> currentModel = 
        new AtomicReference<>("gpt-4o");
    
    private final Map<String, ChatClient> modelClients;
    private final ScheduledExecutorService scheduler = 
        Executors.newSingleThreadScheduledExecutor();
    
    public ModelSwitchManager(OpenAiChatModel openaiModel, 
                               ChatClient fallbackClient) {
        this.modelClients = Map.of(
            "gpt-4o", ChatClient.builder(openaiModel).build(),
            "gpt-4o-mini", ChatClient.builder(openaiModel)
                .defaultOptions(OpenAiChatOptions.builder()
                    .withModel("gpt-4o-mini").build())
                .build(),
            "local-model", fallbackClient
        );
    }
    
    public void switchTo(String model, Duration duration) {
        String previousModel = currentModel.getAndSet(model);
        
        log.info("模型切换: {} → {}", previousModel, model);
        
        // 计划恢复
        scheduler.schedule(() -> {
            log.info("自愈：尝试恢复原始模型 {}", previousModel);
            currentModel.set(previousModel);
        }, duration.toSeconds(), TimeUnit.SECONDS);
    }
    
    public ChatClient getCurrentClient() {
        String model = currentModel.get();
        return modelClients.getOrDefault(model, 
            modelClients.get("gpt-4o-mini"));
    }
}

// EnableCacheFallbackAction.java
@Slf4j
public class EnableCacheFallbackAction implements HealingAction {
    
    private final CacheAnswerService cacheService;
    private final AtomicBoolean cacheEnabled;
    
    @Override
    public void execute() {
        log.warn("自愈：启用缓存降级，将优先使用缓存答案");
        cacheEnabled.set(true);
        
        // 10分钟后关闭缓存降级
        Executors.newSingleThreadScheduledExecutor().schedule(
            () -> {
                cacheEnabled.set(false);
                log.info("缓存降级已关闭，恢复正常AI调用");
            },
            10, TimeUnit.MINUTES
        );
    }
    
    @Override
    public String getName() {
        return "enable-cache-fallback";
    }
    
    @Override
    public int getPriority() {
        return 70;
    }
}

四、Spring AI Advisor集成自愈

4.1 自愈感知的ChatClient

// SelfHealingChatAdvisor.java
@Component
@Order(Ordered.HIGHEST_PRECEDENCE + 10)
public class SelfHealingChatAdvisor implements CallAroundAdvisor {
    
    private final AiSystemHealthMonitor healthMonitor;
    private final ModelSwitchManager modelSwitchManager;
    private final CacheAnswerService cacheService;
    private final AtomicBoolean cacheEnabled;
    
    @Override
    public String getName() {
        return "SelfHealingAdvisor";
    }
    
    @Override
    public int getOrder() {
        return Ordered.HIGHEST_PRECEDENCE + 10;
    }
    
    @Override
    public AdvisedResponse aroundCall(AdvisedRequest advisedRequest,
                                       CallAroundAdvisorChain chain) {
        
        // 如果缓存降级已启用，先尝试缓存
        if (cacheEnabled.get()) {
            String userMessage = extractUserMessage(advisedRequest);
            String cached = cacheService.getSimilar(userMessage);
            
            if (cached != null) {
                log.debug("命中缓存降级响应");
                return buildCachedResponse(cached, advisedRequest);
            }
        }
        
        long startTime = System.currentTimeMillis();
        
        try {
            AdvisedResponse response = chain.nextAroundCall(advisedRequest);
            
            // 记录成功
            long latency = System.currentTimeMillis() - startTime;
            healthMonitor.recordResult(true, latency);
            
            // 缓存成功响应（供后续降级使用）
            cacheService.cache(
                extractUserMessage(advisedRequest),
                extractContent(response));
            
            return response;
            
        } catch (Exception e) {
            long latency = System.currentTimeMillis() - startTime;
            healthMonitor.recordResult(false, latency);
            
            log.warn("AI调用失败，尝试自愈: {}", e.getMessage());
            
            // 尝试用备用模型重试一次
            try {
                ChatClient fallbackClient = modelSwitchManager.getCurrentClient();
                // 如果当前已经切换了模型，这里会用备用模型
                return chain.nextAroundCall(advisedRequest);
            } catch (Exception retryE) {
                // 两次都失败，返回降级响应
                return buildFallbackResponse(advisedRequest);
            }
        }
    }
    
    private AdvisedResponse buildFallbackResponse(AdvisedRequest request) {
        String fallbackMessage = "抱歉，AI服务暂时遇到问题，正在自动恢复。" +
            "您可以稍后重试，或联系客服获取帮助。";
        
        AssistantMessage message = new AssistantMessage(fallbackMessage);
        ChatResponse response = new ChatResponse(
            List.of(new Generation(message)));
        
        return new AdvisedResponse(response, request.adviseContext());
    }
}

五、混沌实验的运行流程

5.1 混沌实验SOP

// ChaosExperimentRunner.java
@Service
@Slf4j
public class ChaosExperimentRunner {
    
    private final ChaosExperimentController chaosController;
    private final AiSystemHealthMonitor healthMonitor;
    private final AlertService alertService;
    
    // 结构化的混沌实验执行流程
    public ExperimentReport runExperiment(ChaosExperiment experiment) {
        
        log.info("=== 开始混沌实验: {} ===", experiment.getName());
        
        ExperimentReport.Builder reportBuilder = ExperimentReport.builder()
            .experimentName(experiment.getName())
            .startTime(LocalDateTime.now());
        
        try {
            // 阶段1：基准线测量（5分钟）
            log.info("阶段1: 测量基准线...");
            HealthSnapshot baseline = measureHealthBaseline(Duration.ofMinutes(5));
            reportBuilder.baseline(baseline);
            
            // 阶段2：注入故障
            log.info("阶段2: 注入故障（{}%概率）...", 
                experiment.getDefaultInjectionRate() * 100);
            chaosController.start(experiment, 
                experiment.getDefaultInjectionRate(), 
                (int) experiment.getDuration().getSeconds());
            
            // 阶段3：观测系统响应
            log.info("阶段3: 观测系统响应...");
            HealthSnapshot underChaos = measureHealthDuringChaos(
                experiment.getDuration());
            reportBuilder.underChaosSnapshot(underChaos);
            
            // 阶段4：检验自愈
            log.info("阶段4: 检验自愈能力...");
            boolean selfHealed = checkSelfHealing(
                baseline, underChaos, Duration.ofMinutes(10));
            reportBuilder.selfHealed(selfHealed);
            
        } finally {
            // 确保停止混沌注入
            chaosController.stopAll();
        }
        
        // 阶段5：恢复后测量
        log.info("阶段5: 恢复后测量...");
        HealthSnapshot afterRecovery = measureHealthBaseline(Duration.ofMinutes(3));
        reportBuilder.afterRecoverySnapshot(afterRecovery);
        
        ExperimentReport report = reportBuilder
            .endTime(LocalDateTime.now())
            .build();
        
        // 如果系统未能自愈，发送告警
        if (!report.isSelfHealed()) {
            alertService.sendCriticalAlert(
                "混沌实验失败：系统未能自愈", 
                "实验: " + experiment.getName() + 
                "\n健康评分: " + afterRecovery.getHealthScore());
        }
        
        log.info("=== 混沌实验完成: {} | 自愈: {} ===", 
            experiment.getName(), report.isSelfHealed());
        
        return report;
    }
    
    private HealthSnapshot measureHealthBaseline(Duration duration) throws InterruptedException {
        List<Double> healthScores = new ArrayList<>();
        long endTime = System.currentTimeMillis() + duration.toMillis();
        
        while (System.currentTimeMillis() < endTime) {
            healthScores.add(healthMonitor.getCurrentHealthScore());
            Thread.sleep(10000);  // 每10秒采样
        }
        
        return HealthSnapshot.builder()
            .avgScore(healthScores.stream()
                .mapToDouble(Double::doubleValue).average().orElse(0))
            .minScore(healthScores.stream()
                .mapToDouble(Double::doubleValue).min().orElse(0))
            .build();
    }
    
    private boolean checkSelfHealing(HealthSnapshot baseline, 
                                      HealthSnapshot underChaos,
                                      Duration maxRecoveryTime) {
        // 只有在混沌期间确实造成了损害才检查自愈
        if (underChaos.getMinScore() > baseline.getAvgScore() * 0.9) {
            log.info("混沌实验对系统健康影响极小，无需自愈");
            return true;
        }
        
        long deadline = System.currentTimeMillis() + maxRecoveryTime.toMillis();
        
        while (System.currentTimeMillis() < deadline) {
            double currentScore = healthMonitor.getCurrentHealthScore();
            
            // 健康评分恢复到基准线的90%视为自愈成功
            if (currentScore >= baseline.getAvgScore() * 0.9) {
                log.info("系统自愈成功！当前评分: {}", currentScore);
                return true;
            }
            
            try {
                Thread.sleep(5000);
            } catch (InterruptedException e) {
                break;
            }
        }
        
        log.error("系统未能在{}内自愈", maxRecoveryTime);
        return false;
    }
}

六、混沌实验报告与改进

6.1 系统弱点分析

// WeaknessAnalyzer.java
@Service
public class WeaknessAnalyzer {
    
    private final ExperimentReportRepository reportRepository;
    private final ChatClient analysisClient;
    
    // 分析所有实验报告，找出系统弱点
    public SystemWeaknessReport analyzeWeaknesses() {
        List<ExperimentReport> reports = reportRepository.findAll();
        
        // 找出自愈失败的实验
        List<ExperimentReport> failures = reports.stream()
            .filter(r -> !r.isSelfHealed())
            .toList();
        
        // 找出自愈时间最长的实验
        List<ExperimentReport> slowRecovery = reports.stream()
            .filter(r -> r.getRecoveryTimeMs() > 60000)
            .sorted(Comparator.comparingLong(ExperimentReport::getRecoveryTimeMs).reversed())
            .limit(5)
            .toList();
        
        // 用AI生成改进建议
        String improvementSuggestions = generateImprovementSuggestions(
            failures, slowRecovery);
        
        return SystemWeaknessReport.builder()
            .totalExperiments(reports.size())
            .failedExperiments(failures.size())
            .selfHealingRate((double)(reports.size() - failures.size()) / reports.size())
            .topWeaknesses(identifyTopWeaknesses(failures))
            .slowRecoveryScenarios(slowRecovery)
            .improvementSuggestions(improvementSuggestions)
            .build();
    }
    
    private String generateImprovementSuggestions(
            List<ExperimentReport> failures,
            List<ExperimentReport> slowRecovery) {
        
        String failureDetails = failures.stream()
            .map(r -> "- " + r.getExperimentName() + ": " + r.getFailureReason())
            .collect(Collectors.joining("\n"));
        
        return analysisClient.prompt()
            .user(String.format("""
                AI系统混沌工程报告分析：
                
                未通过自愈的实验：
                %s
                
                请分析原因并给出具体的改进建议，包括：
                1. 每个失败场景的根本原因
                2. 具体的代码/配置改进建议
                3. 优先级排序（最紧迫先）
                """, failureDetails))
            .call()
            .content();
    }
}

七、常见问题 FAQ

Q1：混沌工程应该在什么环境中运行？

A：推荐策略：

开发/测试环境：所有实验，可以激进
预发布环境：核心实验，注入率20-50%
生产环境（初期）：只在低峰期运行极低注入率实验（5%），配合自动熔断保护
成熟阶段：生产环境持续混沌测试（Netflix的方式）

Q2：AI系统的自愈切换模型，用户会感知到质量下降吗？

A：

从GPT-4o切换到GPT-4o-mini：质量轻微下降，大多数用户不明显感知
切换到本地小模型：质量可能明显下降，建议限制在特定场景使用
缓存降级：质量基本维持，但可能返回"过期"答案
完全降级（固定回复）：明显感知，只用于极端故障

Q3：自愈动作本身失败了怎么办？

A：自愈动作也需要Fallback：

每个自愈动作设置超时（30秒）
自愈失败时升级到下一级降级策略
最终保底：返回预设的静态降级响应 + 触发人工告警
记录所有自愈动作的执行结果，供后续分析

Q4：如何防止混沌实验意外影响真实用户？

A：

生产环境混沌只在非业务高峰期（凌晨2-5点）运行
实验开始前必须确保告警系统正常（不能在告警系统故障时做混沌）
给混沌请求打标（x-chaos-injection: true），方便排查
实验具有自动安全停止条件：健康评分低于40分时自动停止

Q5：如何建立混沌工程文化？

A：

从"游戏日"开始：组织团队集中做一天混沌实验，像演习一样
可观测性先行：没有好的监控，混沌工程是盲目的
小步快走：先从最简单的网络延迟开始，不要一开始就随机杀进程
记录和分享：每次实验的发现要在全团队分享，建立学习文化

八、总结

AI系统的混沌工程与传统系统的不同：

维度	传统混沌工程	AI混沌工程
故障类型	基础设施故障	模型行为异常 + 基础设施故障
自愈策略	重启/扩容	模型切换/降级/缓存
影响评估	可用性	可用性 + 响应质量
工具	Chaos Monkey等	需要AI感知的自定义框架

吴杰的故事说明：混沌工程不是找麻烦，而是在自己可控的条件下找到系统的弱点，比生产故障先发现。每一次成功的混沌实验，都是一次避免真实故障的演练。

把混沌工程加入你的AI工程清单，让你的AI系统从"脆弱"走向"反脆弱"。