AI-Driven Remediation Testing

meta: id: scenario-pod-crash-001 title: "Pod Crash Recovery Test" owner: "platform-team" defaults: model: "gpt-4" timeout: 300 policies: max_retries: 3 bindings: namespace: "production" service: "api-gateway" detector_id: "detector-12345" prechecks: signalflow: - program: "data('cpu.utilization').mean().publish()" assert_rules: - "value < 80" fault: type: "pod_kill" params: namespace: "${namespace}" deployment: "${service}" kill_count: "1" stabilize: wait_for: detector: "${detector_id}" timeout: 120 assistant_rca: model: "gpt-4" system: "You are an expert SRE analyzing a production incident." user: "Analyze the pod crash in ${namespace}/${service}. Provide root cause analysis." expect: references: - "pod" - "crash" - "${service}" metrics: - "cpu" - "memory" - "restart_count" guards: - type: "regex" pattern: "(?i)(root cause|analysis|pod)" assistant_remedy: model: "gpt-4" system: "You are an expert SRE providing remediation steps." user: "Based on the RCA, provide specific remediation commands." expect: references: - "kubectl" - "restart" metrics: - "scale" - "rollout" guards: - type: "regex" pattern: "(?i)(kubectl|command|remediation)" execute_remedy: sandbox: service_account: "sre-automation" namespace: "${namespace}" policies: deny_patterns: - ".*rm -rf.*" - ".*delete.*production.*" commands: - name: "Scale deployment" cmd: "kubectl" args: - "scale" - "deployment/${service}" - "--replicas=3" - "-n" - "${namespace}" - name: "Check pod status" cmd: "kubectl" args: - "get" - "pods" - "-n" - "${namespace}" verify: signalflow: - program: "data('cpu.utilization').mean().publish()" assert_rules: - "value < 70" detector_clear: - detector_id: "${detector_id}" expected_state: "clear" cleanup: always: - name: "Reset scaling" when_failed: false cmd: "kubectl" args: - "scale" - "deployment/${service}" - "--replicas=2" - "-n" - "${namespace}" report: formats: - "json" - "xml" store: "s3://test-results/scenarios/"