System Initiative

{ "id": null, "uid": "snapshot-eviction-metrics", "title": "Snapshot Eviction Metrics", "tags": [ "snapshot", "eviction", "garbage-collection", "workspace" ], "timezone": "browser", "schemaVersion": 16, "version": 0, "refresh": "5s", "annotations": { "list": [ { "builtIn": 1, "datasource": { "type": "grafana", "uid": "-- Grafana --" }, "enable": true, "hide": true, "iconColor": "rgba(0, 211, 255, 1)", "name": "Annotations & Alerts", "type": "dashboard" } ] }, "templating": { "list": [ { "name": "datasource", "type": "datasource", "label": "Data Source", "query": "prometheus", "refresh": 1, "hide": 0 }, { "name": "instance", "type": "query", "label": "Instance", "datasource": { "type": "prometheus", "uid": "${datasource}" }, "query": "label_values(node_exporter_build_info, instance)", "refresh": 2, "sort": 1, "multi": true, "includeAll": true, "allValue": ".*" } ] }, "panels": [ { "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 }, "id": 100, "panels": [], "title": "Health", "type": "row" }, { "id": 1, "gridPos": { "x": 0, "y": 1, "w": 24, "h": 4 }, "type": "stat", "title": "Success Rate", "datasource": { "type": "prometheus", "uid": "${datasource}" }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum(rate(snapshot_eviction_successes_total{instance=~\"$instance\"}[5m])) / (sum(rate(snapshot_eviction_successes_total{instance=~\"$instance\"}[5m])) + (sum(rate(snapshot_eviction_failures_total{instance=~\"$instance\"}[5m])) or vector(0))) or vector(1)", "refId": "A" } ], "fieldConfig": { "defaults": { "unit": "percentunit", "color": { "mode": "thresholds" }, "thresholds": { "mode": "absolute", "steps": [ { "value": null, "color": "red" }, { "value": 0.95, "color": "yellow" }, { "value": 0.99, "color": "green" } ] }, "min": 0, "max": 1 } }, "options": { "textMode": "value_and_name", "colorMode": "background", "graphMode": "area", "orientation": "auto", "reduceOptions": { "values": false, "calcs": [ "lastNotNull" ] } } }, { "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 5 }, "id": 101, "panels": [], "title": "Workload", "type": "row" }, { "id": 4, "gridPos": { "x": 0, "y": 6, "w": 12, "h": 8 }, "type": "timeseries", "title": "Candidates Found", "datasource": { "type": "prometheus", "uid": "${datasource}" }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "increase(snapshot_eviction_candidates_found_sum{instance=~\"$instance\"}[1m])", "legendFormat": "{{instance}} - candidates", "refId": "A" } ], "fieldConfig": { "defaults": { "unit": "short", "color": { "mode": "palette-classic" }, "custom": { "lineWidth": 2, "fillOpacity": 10, "showPoints": "never" } } }, "options": { "tooltip": { "mode": "multi", "sort": "desc" }, "legend": { "displayMode": "table", "placement": "right", "calcs": [ "lastNotNull", "max", "mean" ] } } }, { "id": 5, "gridPos": { "x": 12, "y": 6, "w": 12, "h": 8 }, "type": "timeseries", "title": "Candidate Queue Latency - P50, P95, P99", "datasource": { "type": "prometheus", "uid": "${datasource}" }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "histogram_quantile(0.50, sum(rate(snapshot_candidate_queue_latency_seconds_bucket{instance=~\"$instance\"}[1m])) by (le))", "legendFormat": "{{instance}} - P50", "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "histogram_quantile(0.95, sum(rate(snapshot_candidate_queue_latency_seconds_bucket{instance=~\"$instance\"}[1m])) by (le))", "legendFormat": "{{instance}} - P95", "refId": "B" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "histogram_quantile(0.99, sum(rate(snapshot_candidate_queue_latency_seconds_bucket{instance=~\"$instance\"}[1m])) by (le))", "legendFormat": "{{instance}} - P99", "refId": "C" } ], "fieldConfig": { "defaults": { "unit": "s", "color": { "mode": "palette-classic" }, "custom": { "lineWidth": 2, "fillOpacity": 10, "showPoints": "never" } } }, "options": { "tooltip": { "mode": "multi", "sort": "desc" }, "legend": { "displayMode": "table", "placement": "right", "calcs": [ "lastNotNull", "max", "mean", "min" ] } } }, { "id": 8, "gridPos": { "x": 0, "y": 14, "w": 12, "h": 8 }, "type": "timeseries", "title": "Cycle Duration - P50, P95, P99", "datasource": { "type": "prometheus", "uid": "${datasource}" }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "histogram_quantile(0.50, sum(rate(snapshot_eviction_cycle_duration_seconds_bucket{instance=~\"$instance\"}[1m])) by (le))", "legendFormat": "{{instance}} - P50", "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "histogram_quantile(0.95, sum(rate(snapshot_eviction_cycle_duration_seconds_bucket{instance=~\"$instance\"}[1m])) by (le))", "legendFormat": "{{instance}} - P95", "refId": "B" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "histogram_quantile(0.99, sum(rate(snapshot_eviction_cycle_duration_seconds_bucket{instance=~\"$instance\"}[1m])) by (le))", "legendFormat": "{{instance}} - P99", "refId": "C" } ], "fieldConfig": { "defaults": { "unit": "s", "color": { "mode": "palette-classic" }, "custom": { "lineWidth": 2, "fillOpacity": 10, "showPoints": "never" } } }, "options": { "tooltip": { "mode": "multi", "sort": "desc" }, "legend": { "displayMode": "table", "placement": "right", "calcs": [ "lastNotNull", "max", "mean", "min" ] } } }, { "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 22 }, "id": 102, "panels": [], "title": "Throughput", "type": "row" }, { "id": 2, "gridPos": { "x": 0, "y": 23, "w": 24, "h": 8 }, "type": "timeseries", "title": "Eviction Results", "datasource": { "type": "prometheus", "uid": "${datasource}" }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "increase(snapshot_eviction_failures_total{instance=~\"$instance\"}[1m])", "legendFormat": "{{instance}} - failures", "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "increase(snapshot_eviction_successes_total{instance=~\"$instance\"}[1m])", "legendFormat": "{{instance}} - successes", "refId": "B" } ], "fieldConfig": { "defaults": { "unit": "short", "color": { "mode": "palette-classic" }, "custom": { "lineWidth": 2, "fillOpacity": 50, "showPoints": "never", "stacking": { "mode": "normal" } } }, "overrides": [ { "matcher": { "id": "byRegexp", "options": ".*successes" }, "properties": [ { "id": "color", "value": { "mode": "fixed", "fixedColor": "green" } } ] }, { "matcher": { "id": "byRegexp", "options": ".*failures" }, "properties": [ { "id": "color", "value": { "mode": "fixed", "fixedColor": "red" } } ] } ] }, "options": { "tooltip": { "mode": "multi", "sort": "desc" }, "legend": { "displayMode": "table", "placement": "right", "calcs": [ "lastNotNull", "max", "mean", "sum" ] } } }, { "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 31 }, "id": 103, "panels": [], "title": "Reliability", "type": "row" }, { "id": 3, "gridPos": { "x": 0, "y": 32, "w": 24, "h": 8 }, "type": "timeseries", "title": "Failures by Error Type", "datasource": { "type": "prometheus", "uid": "${datasource}" }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "increase(snapshot_eviction_failures_total{instance=~\"$instance\"}[1m])", "legendFormat": "{{instance}} - {{error_type}}", "refId": "A" } ], "fieldConfig": { "defaults": { "unit": "short", "color": { "mode": "palette-classic" }, "custom": { "lineWidth": 2, "fillOpacity": 50, "showPoints": "never", "stacking": { "mode": "normal" } } } }, "options": { "tooltip": { "mode": "multi", "sort": "desc" }, "legend": { "displayMode": "table", "placement": "right", "calcs": [ "lastNotNull", "max", "sum" ] } } }, { "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 40 }, "id": 104, "panels": [], "title": "Performance", "type": "row" }, { "id": 6, "gridPos": { "x": 0, "y": 41, "w": 12, "h": 8 }, "type": "timeseries", "title": "Eviction Duration - P50, P95, P99 (Success)", "datasource": { "type": "prometheus", "uid": "${datasource}" }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "histogram_quantile(0.50, sum(rate(snapshot_eviction_duration_seconds_bucket{instance=~\"$instance\", status=\"success\"}[1m])) by (le))", "legendFormat": "{{instance}} - P50", "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "histogram_quantile(0.95, sum(rate(snapshot_eviction_duration_seconds_bucket{instance=~\"$instance\", status=\"success\"}[1m])) by (le))", "legendFormat": "{{instance}} - P95", "refId": "B" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "histogram_quantile(0.99, sum(rate(snapshot_eviction_duration_seconds_bucket{instance=~\"$instance\", status=\"success\"}[1m])) by (le))", "legendFormat": "{{instance}} - P99", "refId": "C" } ], "fieldConfig": { "defaults": { "unit": "s", "color": { "mode": "palette-classic" }, "custom": { "lineWidth": 2, "fillOpacity": 10, "showPoints": "never" } } }, "options": { "tooltip": { "mode": "multi", "sort": "desc" }, "legend": { "displayMode": "table", "placement": "right", "calcs": [ "lastNotNull", "max", "mean", "min" ] } } }, { "id": 7, "gridPos": { "x": 12, "y": 41, "w": 12, "h": 8 }, "type": "timeseries", "title": "Failure Duration by Error Type - P50, P95, P99", "datasource": { "type": "prometheus", "uid": "${datasource}" }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "histogram_quantile(0.50, sum(rate(snapshot_eviction_duration_seconds_bucket{instance=~\"$instance\", status=\"failure\"}[1m])) by (le))", "legendFormat": "{{instance}} - {{error_type}} - P50", "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "histogram_quantile(0.95, sum(rate(snapshot_eviction_duration_seconds_bucket{instance=~\"$instance\", status=\"failure\"}[1m])) by (le))", "legendFormat": "{{instance}} - {{error_type}} - P95", "refId": "B" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "histogram_quantile(0.99, sum(rate(snapshot_eviction_duration_seconds_bucket{instance=~\"$instance\", status=\"failure\"}[1m])) by (le))", "legendFormat": "{{instance}} - {{error_type}} - P99", "refId": "C" } ], "fieldConfig": { "defaults": { "unit": "s", "color": { "mode": "palette-classic" }, "custom": { "lineWidth": 2, "fillOpacity": 10, "showPoints": "never" } } }, "options": { "tooltip": { "mode": "multi", "sort": "desc" }, "legend": { "displayMode": "table", "placement": "right", "calcs": [ "lastNotNull", "max", "mean", "min" ] } } } ], "time": { "from": "now-15m", "to": "now" }, "timepicker": { "refresh_intervals": [ "5s", "10s", "30s", "1m", "5m", "15m", "30m", "1h", "2h", "1d" ] } }