[
{
"acknowledgements": [],
"alert_counts": {
"all": 1,
"resolved": 1,
"triggered": 0
},
"alert_grouping": null,
"assigned_via": "escalation_policy",
"assignments": [],
"basic_alert_grouping": null,
"created_at": "2025-02-17T23:19:05Z",
"description": "REDACTED",
"escalation_policy": {
"html_url": "https://pagerduty.com/123",
"id": "Escalation policy 123",
"self": "https://api.pagerduty.com/escalation_policies/123",
"summary": "REDACTED",
"type": "escalation_policy_reference"
},
"first_trigger_log_entry": {
"html_url": "https://pagerduty.com/123",
"id": "Log Entry 123",
"self": "https://api.pagerduty.com/log_entries/123",
"summary": "Triggered through the API.",
"type": "trigger_log_entry_reference"
},
"html_url": "https://pagerduty.com/123",
"id": "123",
"incident_key": null,
"incident_number": 123,
"incident_type": {
"name": "incident_default"
},
"incidents_responders": [],
"is_mergeable": true,
"last_status_change_at": "2025-02-17T23:27:00Z",
"last_status_change_by": {
"html_url": "https://pagerduty.com/123",
"id": "123",
"self": "https://api.pagerduty.com/services/123",
"summary": "Service 123",
"type": "service_reference"
},
"pending_actions": [],
"priority": null,
"resolve_reason": null,
"resolved_at": "2025-02-17T23:27:00Z",
"responder_requests": [],
"self": "https://api.pagerduty.com/incidents/123",
"service": {
"html_url": "https://pagerduty.com/123",
"id": "123",
"self": "https://api.pagerduty.com/services/123",
"summary": "Service 123",
"type": "service_reference"
},
"status": "resolved",
"subscriber_requests": [],
"summary": "REDACTED",
"teams": [
{
"html_url": "https://pagerduty.com/123",
"id": "PX4X5QR",
"self": "https://api.pagerduty.com/teams/123",
"summary": "Team 123",
"type": "team_reference"
}
],
"title": "Incident 123",
"type": "incident",
"updated_at": "2025-02-17T23:27:00Z",
"urgency": "high",
"body": {
"details": {
"__pd_cef_payload": {
"client": "Datadog",
"client_url": "https://example.datadoghq.com/monitors/123456?group=env%3Aproduction&from_ts=1745234973000&to_ts=1745236173000&event_id=1234567890123456789&link_source=monitor_notif",
"contexts": [
{
"alt": "Snapshot of metric",
"href": "https://example.datadoghq.com/monitors/123456?group=env%3Aproduction&from_ts=1745234973000&to_ts=1745236173000&event_id=1234567890123456789&link_source=monitor_notif",
"src": "https://p.datadoghq.com/snapshot/view/dd-snapshots-prod/org_123456/2025-04-21/8c49ccf7fbaf60d919d1cc7e4dca53f2361f83b8.png",
"type": "image"
},
{
"href": "https://example.datadoghq.com/monitors/123456?group=env%3Aproduction&from_ts=1745234973000&to_ts=1745236173000&event_id=1234567890123456789&link_source=monitor_notif",
"text": "Monitor Status",
"type": "link"
},
{
"href": "https://example.datadoghq.com/monitors/triggered?link_source=monitor_notif",
"text": "Triggered Monitors",
"type": "link"
}
],
"creation_time": "2025-04-21T11:44:33Z",
"dedup_key": "53ceed2d-dd0d-bdef-3fc2-487bb1c4cc34",
"description": "[ORG][team][service]: Anomaly: Number of Responses - Anomalous 200s Drop for ServiceName on env:production",
"details": {
"body": "The number of 200 responses is anomalous compared to the same 30-minute window a week ago for the ServiceName API.\nOncall engineer should first look into datadog: https://example.datadoghq.com/dashboard/abc-123/service-metrics?fromUser=false&fullscreen_end_ts=1720629828993&fullscreen_paused=false&fullscreen_refresh_mode=sliding&fullscreen_section=overview&fullscreen_start_ts=1720543428993&fullscreen_widget=5166521952505604&refresh_mode=sliding&tile_focus=5166521952505604&view=spans&from_ts=1720543383592&to_ts=1720629783592&live=true\nto check there isn't a spike occurring from something like an enumeration attack or a sudden drop due to an outage. As a potential follow-up, you can consult https://example.com/docs/troubleshooting for relevant logs \n\n\n @pagerduty-team-service-terraform\n \n\n\n\n\n\n\n\n\n\nPercent anomalous: 100.0%",
"event_id": "1234567890123456789",
"event_type": "query_alert_monitor",
"monitor_state": "Triggered",
"org": "Example",
"priority": "normal",
"query": "min(last_30m):\nanomalies(sum:framework.actions.service_api.clients.response_codes{service:service AND (env:production) AND region IN (*) AND api:service AND code:200s} by {env}, 'agile', 2, direction='below', alert_window='last_30m', interval=60, count_default_zero='true', seasonality='weekly')\n> 1",
"tags": "api:service, code:200s, env:production, integration:custom, monitor, org:example, service:service, source:https://github.com/example/tf-mod-datadog-monitors/tree/main/modules/anomaly, team:team",
"title": "[ORG][team][service]: Anomaly: Number of Responses - Anomalous 200s Drop for ServiceName on env:production",
"user": null
},
"event_action": "trigger",
"event_class": "[ORG][team][service]: Anomaly: Number of Responses - Anomalous 200s Drop for ServiceName",
"event_id": "78285a8a-bec9-45d7-9fcb-1534c7ba5d58",
"event_storage_id": "2b56b46e-19a8-30b3-8ea6-015fe36a5cdb",
"local_instance_id": null,
"message": "[ORG][team][service]: Anomaly: Number of Responses - Anomalous 200s Drop for ServiceName on env:production",
"mutations": [],
"priority": null,
"reporter_component": null,
"reporter_location": null,
"routing_key": "f7c45bae98a84a0dc054a8a92791b695",
"service_group": "api:service, code:200s, env:production, integration:custom, monitor, org:example, service:service, source:https://github.com/example/tf-mod-datadog-monitors/tree/main/modules/anomaly, team:team",
"severity": "error",
"source_component": "",
"source_location": null,
"source_origin": "https://github.com/example/tf-mod-datadog-monitors/tree/main/modules/anomaly",
"urgency": null,
"version": "1.0"
}
}
}
},
{
"acknowledgements": [],
"alert_counts": {
"all": 1,
"resolved": 0,
"triggered": 1
},
"alert_grouping": {
"alert_grouping_active": false,
"ended_at": null,
"grouping_type": "advanced",
"started_at": "2025-02-18T00:31:12Z"
},
"assigned_via": "escalation_policy",
"assignments": [
{
"assignee": {
"html_url": "https://square.pagerduty.com/users/456",
"id": "456",
"self": "https://api.pagerduty.com/users/456",
"summary": "User 456",
"type": "user_reference"
},
"at": "2025-02-18T00:31:12Z"
}
],
"basic_alert_grouping": null,
"created_at": "2025-02-18T00:31:12Z",
"description": "REDACTED",
"escalation_policy": {
"html_url": "https://square.pagerduty.com/escalation_policies/456",
"id": "456",
"self": "https://api.pagerduty.com/escalation_policies/456",
"summary": "Escalation Policy 456",
"type": "escalation_policy_reference"
},
"first_trigger_log_entry": {
"html_url": "https://square.pagerduty.com/incidents/456/log_entries/456",
"id": "456",
"self": "https://api.pagerduty.com/log_entries/456",
"summary": "Log entry 456",
"type": "trigger_log_entry_reference"
},
"html_url": "https://square.pagerduty.com/incidents/456",
"id": "456",
"incident_key": null,
"incident_number": 456,
"incident_type": {
"name": "incident_default"
},
"incidents_responders": [],
"is_mergeable": true,
"last_status_change_at": "2025-02-18T00:31:12Z",
"last_status_change_by": {
"html_url": "https://square.pagerduty.com/service-directory/456",
"id": "456",
"self": "https://api.pagerduty.com/services/456",
"summary": "Service 456",
"type": "service_reference"
},
"pending_actions": [],
"priority": null,
"resolved_at": null,
"responder_requests": [],
"self": "https://api.pagerduty.com/incidents/456",
"service": {
"html_url": "https://square.pagerduty.com/service-directory/456",
"id": "456",
"self": "https://api.pagerduty.com/services/456",
"summary": "Service 456",
"type": "service_reference"
},
"status": "triggered",
"subscriber_requests": [],
"summary": "REDACTED",
"teams": [
{
"html_url": "https://square.pagerduty.com/teams/456",
"id": "456",
"self": "https://api.pagerduty.com/teams/456",
"summary": "Team 456",
"type": "team_reference"
}
],
"title": "Incident 456",
"type": "incident",
"updated_at": "2025-02-18T00:31:12Z",
"urgency": "high"
},
{
"acknowledgements": [
{
"acknowledger": {
"html_url": "https://square.pagerduty.com/users/789",
"id": "789",
"self": "https://api.pagerduty.com/users/789",
"summary": "User 789",
"type": "user_reference"
},
"at": "2025-02-18T00:54:51Z"
}
],
"alert_counts": {
"all": 1,
"resolved": 0,
"triggered": 1
},
"alert_grouping": {
"alert_grouping_active": false,
"ended_at": null,
"grouping_type": "advanced",
"started_at": "2025-02-18T00:24:36Z"
},
"assigned_via": "escalation_policy",
"assignments": [
{
"assignee": {
"html_url": "https://square.pagerduty.com/users/789",
"id": "789",
"self": "https://api.pagerduty.com/users/789",
"summary": "User 789",
"type": "user_reference"
},
"at": "2025-02-18T00:39:37Z"
}
],
"basic_alert_grouping": null,
"created_at": "2025-02-18T00:24:36Z",
"description": "REDACTED",
"escalation_policy": {
"html_url": "https://square.pagerduty.com/escalation_policies/789",
"id": "789",
"self": "https://api.pagerduty.com/escalation_policies/789",
"summary": "Escalation Policy 789",
"type": "escalation_policy_reference"
},
"first_trigger_log_entry": {
"html_url": "https://square.pagerduty.com/incidents/789/log_entries/789",
"id": "789",
"self": "https://api.pagerduty.com/log_entries/789",
"summary": "Log entry 789",
"type": "trigger_log_entry_reference"
},
"html_url": "https://square.pagerduty.com/incidents/789",
"id": "789",
"incident_key": null,
"incident_number": 789,
"incident_type": {
"name": "incident_default"
},
"incidents_responders": [],
"is_mergeable": true,
"last_status_change_at": "2025-02-18T00:54:51Z",
"last_status_change_by": {
"html_url": "https://square.pagerduty.com/users/789",
"id": "789",
"self": "https://api.pagerduty.com/users/789",
"summary": "User 789",
"type": "user_reference"
},
"pending_actions": [],
"priority": null,
"resolved_at": null,
"responder_requests": [],
"self": "https://api.pagerduty.com/incidents/789",
"service": {
"html_url": "https://square.pagerduty.com/service-directory/789",
"id": "789",
"self": "https://api.pagerduty.com/services/789",
"summary": "Service 789",
"type": "service_reference"
},
"status": "acknowledged",
"subscriber_requests": [],
"summary": "REDACTED",
"teams": [
{
"html_url": "https://square.pagerduty.com/teams/789",
"id": "789",
"self": "https://api.pagerduty.com/teams/789",
"summary": "Team 789",
"type": "team_reference"
}
],
"title": "Incident 789",
"type": "incident",
"updated_at": "2025-02-18T00:54:52Z",
"urgency": "high"
}
]
