AI-Driven Remediation Testing

meta:
  id: scenario-pod-crash-001
  title: "Pod Crash Recovery Test"
  owner: "platform-team"

defaults:
  model: "gpt-4"
  timeout: 300
  policies:
    max_retries: 3

bindings:
  namespace: "production"
  service: "api-gateway"
  detector_id: "detector-12345"

prechecks:
  signalflow:
    - program: "data('cpu.utilization').mean().publish()"
      assert_rules:
        - "value < 80"

fault:
  type: "pod_kill"
  params:
    namespace: "${namespace}"
    deployment: "${service}"
    kill_count: "1"

stabilize:
  wait_for:
    detector: "${detector_id}"
    timeout: 120

assistant_rca:
  model: "gpt-4"
  system: "You are an expert SRE analyzing a production incident."
  user: "Analyze the pod crash in ${namespace}/${service}. Provide root cause analysis."
  expect:
    references:
      - "pod"
      - "crash"
      - "${service}"
    metrics:
      - "cpu"
      - "memory"
      - "restart_count"
    guards:
      - type: "regex"
        pattern: "(?i)(root cause|analysis|pod)"

assistant_remedy:
  model: "gpt-4"
  system: "You are an expert SRE providing remediation steps."
  user: "Based on the RCA, provide specific remediation commands."
  expect:
    references:
      - "kubectl"
      - "restart"
    metrics:
      - "scale"
      - "rollout"
    guards:
      - type: "regex"
        pattern: "(?i)(kubectl|command|remediation)"

execute_remedy:
  sandbox:
    service_account: "sre-automation"
    namespace: "${namespace}"
    policies:
      deny_patterns:
        - ".*rm -rf.*"
        - ".*delete.*production.*"
  commands:
    - name: "Scale deployment"
      cmd: "kubectl"
      args:
        - "scale"
        - "deployment/${service}"
        - "--replicas=3"
        - "-n"
        - "${namespace}"
    - name: "Check pod status"
      cmd: "kubectl"
      args:
        - "get"
        - "pods"
        - "-n"
        - "${namespace}"

verify:
  signalflow:
    - program: "data('cpu.utilization').mean().publish()"
      assert_rules:
        - "value < 70"
  detector_clear:
    - detector_id: "${detector_id}"
      expected_state: "clear"

cleanup:
  always:
    - name: "Reset scaling"
      when_failed: false
      cmd: "kubectl"
      args:
        - "scale"
        - "deployment/${service}"
        - "--replicas=2"
        - "-n"
        - "${namespace}"

report:
  formats:
    - "json"
    - "xml"
  store: "s3://test-results/scenarios/"