{ "apiVersion": "dashboard.grafana.app/v2beta1", "kind": "Dashboard", "metadata": { "name": "codex_pooler_runtime_triage", "generation": 6, "creationTimestamp": "2026-06-01T13:36:25Z", "labels": {}, "annotations": {} }, "spec": { "annotations": [ { "kind": "AnnotationQuery", "spec": { "builtIn": true, "enable": true, "hide": true, "iconColor": "rgba(0, 211, 255, 1)", "name": "Annotations & Alerts", "query": { "group": "grafana", "kind": "DataQuery", "spec": { "limit": 100, "matchAny": false, "tags": [], "type": "dashboard" }, "version": "v0" } } } ], "cursorSync": "Crosshair", "description": "Production triage view for Codex Pooler runtime health, memory pressure, gateway admission, HTTP traffic, database query pressure, admin stats dashboard reloads, request-log reloads, and restart evidence.", "editable": true, "elements": { "panel-10": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${datasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "editorMode": "code", "expr": "vm_system_counts_process_count{namespace=\"$namespace\", job=\"codex-pooler-app\", pod=~\"${pod:regex}\"}", "interval": "$resolution", "legendFormat": "{{pod}} processes" }, "version": "v0" }, "refId": "A" } }, { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${datasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "editorMode": "code", "expr": "vm_system_counts_port_count{namespace=\"$namespace\", job=\"codex-pooler-app\", pod=~\"${pod:regex}\"}", "interval": "$resolution", "legendFormat": "{{pod}} ports" }, "version": "v0" }, "refId": "B" } } ], "queryOptions": {}, "transformations": [] } }, "description": "", "id": 10, "links": [], "title": "BEAM Process and Port Counts", "vizConfig": { "group": "timeseries", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 15, "gradientMode": "opacity", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 2, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "showValues": false, "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 } ] }, "unit": "short" }, "overrides": [] }, "options": { "legend": { "calcs": [ "lastNotNull", "max" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "multi", "sort": "desc" } } }, "version": "12.3.1" } } }, "panel-11": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${datasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "editorMode": "code", "expr": "vm_total_run_queue_lengths_total{namespace=\"$namespace\", job=\"codex-pooler-app\", pod=~\"${pod:regex}\"}", "interval": "$resolution", "legendFormat": "{{pod}} total" }, "version": "v0" }, "refId": "A" } }, { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${datasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "editorMode": "code", "expr": "vm_total_run_queue_lengths_cpu{namespace=\"$namespace\", job=\"codex-pooler-app\", pod=~\"${pod:regex}\"}", "interval": "$resolution", "legendFormat": "{{pod}} cpu" }, "version": "v0" }, "refId": "B" } }, { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${datasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "editorMode": "code", "expr": "vm_total_run_queue_lengths_io{namespace=\"$namespace\", job=\"codex-pooler-app\", pod=~\"${pod:regex}\"}", "interval": "$resolution", "legendFormat": "{{pod}} io" }, "version": "v0" }, "refId": "C" } } ], "queryOptions": {}, "transformations": [] } }, "description": "Total, CPU, and IO scheduler queues. Sudden growth usually means the VM is saturated or blocked on external work.", "id": 11, "links": [], "title": "BEAM Run Queue", "vizConfig": { "group": "timeseries", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 20, "gradientMode": "opacity", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 2, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "showValues": false, "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 } ] }, "unit": "none" }, "overrides": [] }, "options": { "legend": { "calcs": [ "lastNotNull", "max" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "multi", "sort": "desc" } } }, "version": "12.3.1" } } }, "panel-13": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${datasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "editorMode": "code", "expr": "sum by (pod) (rate(phoenix_endpoint_stop_count{namespace=\"$namespace\", job=\"codex-pooler-app\", pod=~\"${pod:regex}\"}[$__rate_interval]))", "interval": "$resolution", "legendFormat": "{{pod}}" }, "version": "v0" }, "refId": "A" } } ], "queryOptions": {}, "transformations": [] } }, "description": "", "id": 13, "links": [], "title": "Request Rate", "vizConfig": { "group": "timeseries", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 15, "gradientMode": "opacity", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 2, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "showValues": false, "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 } ] }, "unit": "reqps" }, "overrides": [] }, "options": { "legend": { "calcs": [ "lastNotNull", "max" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "multi", "sort": "desc" } } }, "version": "12.3.1" } } }, "panel-14": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${datasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "editorMode": "code", "expr": "histogram_quantile(0.95, sum by (le, pod) (rate(phoenix_endpoint_stop_duration_seconds_bucket{namespace=\"$namespace\", job=\"codex-pooler-app\", pod=~\"${pod:regex}\"}[$__rate_interval])))", "interval": "$resolution", "legendFormat": "{{pod}} p95" }, "version": "v0" }, "refId": "A" } } ], "queryOptions": {}, "transformations": [] } }, "description": "", "id": 14, "links": [], "title": "Endpoint Duration p95", "vizConfig": { "group": "timeseries", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "opacity", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 2, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "showValues": false, "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 } ] }, "unit": "s" }, "overrides": [] }, "options": { "legend": { "calcs": [ "lastNotNull", "max" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "multi", "sort": "desc" } } }, "version": "12.3.1" } } }, "panel-15": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${datasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "editorMode": "code", "expr": "topk(8, histogram_quantile(0.95, sum by (le, route) (rate(phoenix_router_dispatch_stop_duration_seconds_bucket{namespace=\"$namespace\", job=\"codex-pooler-app\", pod=~\"${pod:regex}\"}[$__rate_interval]))) and on (route) (sum by (route) (rate(phoenix_router_dispatch_stop_duration_seconds_count{namespace=\"$namespace\", job=\"codex-pooler-app\", pod=~\"${pod:regex}\"}[$__rate_interval])) > 0))", "interval": "$resolution", "legendFormat": "{{route}} p95" }, "version": "v0" }, "refId": "A" } } ], "queryOptions": {}, "transformations": [] } }, "description": "Top active routes by p95 latency. The active-route filter avoids stale NaN/empty legend rows when a route has no observations in the current interval.", "id": 15, "links": [], "title": "Slowest Routes p95", "vizConfig": { "group": "timeseries", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "opacity", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 2, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "showValues": false, "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 } ] }, "unit": "s" }, "overrides": [] }, "options": { "legend": { "calcs": [ "lastNotNull", "max" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "multi", "sort": "desc" } } }, "version": "12.3.1" } } }, "panel-16": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${datasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "editorMode": "code", "expr": "sum by (buffer, transport, route_class, endpoint) (rate(codex_pooler_gateway_stream_buffer_oversized_count_total{namespace=\"$namespace\", pod=~\"${pod:regex}\"}[$__rate_interval]) or rate(codex_pooler_gateway_stream_buffer_oversized_count{namespace=\"$namespace\", pod=~\"${pod:regex}\"}[$__rate_interval]))", "interval": "$resolution", "legendFormat": "oversized {{transport}} {{route_class}} {{endpoint}}" }, "version": "v0" }, "refId": "A" } }, { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${datasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "editorMode": "code", "expr": "sum by (buffer, transport, route_class, endpoint) (rate(codex_pooler_gateway_stream_buffer_truncated_count_total{namespace=\"$namespace\", pod=~\"${pod:regex}\"}[$__rate_interval]) or rate(codex_pooler_gateway_stream_buffer_truncated_count{namespace=\"$namespace\", pod=~\"${pod:regex}\"}[$__rate_interval]))", "interval": "$resolution", "legendFormat": "truncated {{transport}} {{route_class}} {{endpoint}}" }, "version": "v0" }, "refId": "B" } } ], "queryOptions": {}, "transformations": [] } }, "description": "", "id": 16, "links": [], "title": "Stream Buffer Release Events", "vizConfig": { "group": "timeseries", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 20, "gradientMode": "opacity", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 2, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "showValues": false, "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 } ] }, "unit": "ops" }, "overrides": [] }, "options": { "legend": { "calcs": [ "lastNotNull", "max" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "multi", "sort": "desc" } } }, "version": "12.3.1" } } }, "panel-17": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${datasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "editorMode": "code", "expr": "histogram_quantile(0.95, sum by (le, buffer, transport, route_class, endpoint) (rate(codex_pooler_gateway_stream_buffer_oversized_bytes_bucket{namespace=\"$namespace\", pod=~\"${pod:regex}\"}[$__rate_interval])))", "interval": "$resolution", "legendFormat": "oversized p95 {{transport}} {{route_class}} {{endpoint}}" }, "version": "v0" }, "refId": "A" } }, { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${datasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "editorMode": "code", "expr": "histogram_quantile(0.95, sum by (le, buffer, transport, route_class, endpoint) (rate(codex_pooler_gateway_stream_buffer_truncated_bytes_bucket{namespace=\"$namespace\", pod=~\"${pod:regex}\"}[$__rate_interval])))", "interval": "$resolution", "legendFormat": "truncated p95 {{transport}} {{route_class}} {{endpoint}}" }, "version": "v0" }, "refId": "B" } } ], "queryOptions": {}, "transformations": [] } }, "description": "", "id": 17, "links": [], "title": "Stream Buffer Size p95", "vizConfig": { "group": "timeseries", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 15, "gradientMode": "opacity", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 2, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "showValues": false, "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 } ] }, "unit": "bytes" }, "overrides": [] }, "options": { "legend": { "calcs": [ "lastNotNull", "max" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "multi", "sort": "desc" } } }, "version": "12.3.1" } } }, "panel-19": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${datasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "editorMode": "code", "expr": "sum by (pod) (increase(kube_pod_container_status_restarts_total{cluster=\"$cluster\", namespace=\"$namespace\", pod=~\"${pod:regex}\", exported_container=\"app\"}[$__rate_interval]))", "interval": "$resolution", "legendFormat": "{{pod}} restarts" }, "version": "v0" }, "refId": "A" } }, { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${datasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "editorMode": "code", "expr": "sum by (pod) (max by (pod, name) (increase(container_oom_events_total{namespace=\"$namespace\", pod=~\"${pod:regex}\", container=\"app\"}[$__rate_interval])))", "interval": "$resolution", "legendFormat": "{{pod}} oom events" }, "version": "v0" }, "refId": "B" } } ], "queryOptions": {}, "transformations": [] } }, "description": "", "id": 19, "links": [], "title": "Restarts and OOM Events", "vizConfig": { "group": "timeseries", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "bars", "fillOpacity": 60, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "showValues": false, "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 }, { "color": "red", "value": 1 } ] }, "unit": "short" }, "overrides": [] }, "options": { "legend": { "calcs": [ "lastNotNull", "max" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "multi", "sort": "desc" } } }, "version": "12.3.1" } } }, "panel-2": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${datasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "editorMode": "code", "expr": "sum(up{namespace=\"$namespace\", job=\"codex-pooler-app\", pod=~\"${pod:regex}\"})", "instant": true, "legendFormat": "scraped pods" }, "version": "v0" }, "refId": "A" } } ], "queryOptions": {}, "transformations": [] } }, "description": "", "id": 2, "links": [], "title": "Scraped App Pods", "vizConfig": { "group": "stat", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [ { "color": "red", "value": 0 }, { "color": "green", "value": 1 } ] }, "unit": "none" }, "overrides": [] }, "options": { "colorMode": "background", "graphMode": "none", "justifyMode": "center", "orientation": "auto", "percentChangeColorMode": "standard", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "showPercentChange": false, "textMode": "auto", "wideLayout": true } }, "version": "12.3.1" } } }, "panel-20": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${datasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "editorMode": "code", "expr": "kube_pod_container_status_last_terminated_reason{cluster=\"$cluster\", namespace=\"$namespace\", pod=~\"${pod:regex}\", exported_container=\"app\"} == 1", "format": "table", "instant": true, "legendFormat": "{{pod}} {{reason}}" }, "version": "v0" }, "refId": "A" } } ], "queryOptions": {}, "transformations": [ { "kind": "labelsToFields", "spec": { "id": "labelsToFields", "options": { "mode": "columns" } } }, { "kind": "organize", "spec": { "id": "organize", "options": { "excludeByName": { "Time": true, "Value": false, "__name__": true, "cluster": true, "container": true, "endpoint": true, "instance": true, "job": true, "namespace": false, "service": true, "uid": true }, "indexByName": { "Value": 4, "exported_container": 2, "namespace": 0, "pod": 1, "reason": 3 }, "renameByName": { "Value": "active", "exported_container": "container" } } } } ] } }, "description": "Current kube-state-metrics lastState.terminated reason for live pods only. Deleted pod history is covered by restart and OOM panels, not this table.", "id": 20, "links": [], "title": "Current Pods Last Terminated Reason", "vizConfig": { "group": "table", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "custom": { "align": "auto", "cellOptions": { "type": "auto" }, "footer": { "reducers": [] }, "inspect": false }, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 } ] }, "unit": "short" }, "overrides": [] }, "options": { "cellHeight": "sm", "showHeader": true } }, "version": "12.3.1" } } }, "panel-22": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${datasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "editorMode": "code", "expr": "sum by (pod) (rate(codex_pooler_repo_query_count{namespace=\"$namespace\", job=\"codex-pooler-app\", pod=~\"${pod:regex}\"}[$__rate_interval]))", "interval": "$resolution", "legendFormat": "{{pod}}" }, "version": "v0" }, "refId": "A" } } ], "queryOptions": {}, "transformations": [] } }, "description": "", "id": 22, "links": [], "title": "Repo Query Rate", "vizConfig": { "group": "timeseries", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 15, "gradientMode": "opacity", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 2, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "showValues": false, "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 } ] }, "unit": "ops" }, "overrides": [] }, "options": { "legend": { "calcs": [ "lastNotNull", "max" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "multi", "sort": "desc" } } }, "version": "12.3.1" } } }, "panel-23": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${datasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "editorMode": "code", "expr": "sum by (pod) (rate(codex_pooler_repo_query_count{namespace=\"$namespace\", job=\"codex-pooler-app\", pod=~\"${pod:regex}\"}[$__rate_interval])) / clamp_min(sum by (pod) (rate(phoenix_endpoint_stop_count{namespace=\"$namespace\", job=\"codex-pooler-app\", pod=~\"${pod:regex}\"}[$__rate_interval])), 0.001)", "interval": "$resolution", "legendFormat": "{{pod}}" }, "version": "v0" }, "refId": "A" } } ], "queryOptions": {}, "transformations": [] } }, "description": "", "id": 23, "links": [], "title": "Queries / Request", "vizConfig": { "group": "timeseries", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 15, "gradientMode": "opacity", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 2, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "showValues": false, "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 } ] }, "unit": "short" }, "overrides": [] }, "options": { "legend": { "calcs": [ "lastNotNull", "max" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "multi", "sort": "desc" } } }, "version": "12.3.1" } } }, "panel-24": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${datasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "editorMode": "code", "expr": "topk(10, sum by (source, command) (rate(codex_pooler_repo_query_count{namespace=\"$namespace\", job=\"codex-pooler-app\", pod=~\"${pod:regex}\"}[$__rate_interval])))", "interval": "$resolution", "legendFormat": "{{source}} {{command}}" }, "version": "v0" }, "refId": "A" } } ], "queryOptions": {}, "transformations": [] } }, "description": "Top Ecto query sources by safe source and SQL command. Source falls back to a sanitized table identifier for raw SQL when available.", "id": 24, "links": [], "title": "Hot Query Sources", "vizConfig": { "group": "timeseries", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 15, "gradientMode": "opacity", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 2, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "showValues": false, "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 } ] }, "unit": "ops" }, "overrides": [] }, "options": { "legend": { "calcs": [ "lastNotNull", "max" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "multi", "sort": "desc" } } }, "version": "12.3.1" } } }, "panel-25": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${datasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "editorMode": "code", "expr": "histogram_quantile(0.95, sum by (le, source, command) (rate(codex_pooler_repo_query_total_time_seconds_bucket{namespace=\"$namespace\", job=\"codex-pooler-app\", pod=~\"${pod:regex}\"}[$__rate_interval])))", "interval": "$resolution", "legendFormat": "{{source}} {{command}} p95" }, "version": "v0" }, "refId": "A" } } ], "queryOptions": {}, "transformations": [] } }, "description": "", "id": 25, "links": [], "title": "Repo Total Time p95", "vizConfig": { "group": "timeseries", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "opacity", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 2, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "showValues": false, "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 } ] }, "unit": "s" }, "overrides": [] }, "options": { "legend": { "calcs": [ "lastNotNull", "max" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "multi", "sort": "desc" } } }, "version": "12.3.1" } } }, "panel-26": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${datasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "editorMode": "code", "expr": "histogram_quantile(0.95, sum by (le, source, command) (rate(codex_pooler_repo_query_query_time_seconds_bucket{namespace=\"$namespace\", job=\"codex-pooler-app\", pod=~\"${pod:regex}\"}[$__rate_interval])))", "interval": "$resolution", "legendFormat": "{{source}} {{command}} p95" }, "version": "v0" }, "refId": "A" } } ], "queryOptions": {}, "transformations": [] } }, "description": "", "id": 26, "links": [], "title": "DB Execution Time p95", "vizConfig": { "group": "timeseries", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "opacity", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 2, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "showValues": false, "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 } ] }, "unit": "s" }, "overrides": [] }, "options": { "legend": { "calcs": [ "lastNotNull", "max" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "multi", "sort": "desc" } } }, "version": "12.3.1" } } }, "panel-27": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${datasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "editorMode": "code", "expr": "histogram_quantile(0.95, sum by (le, pod) (rate(codex_pooler_repo_query_queue_time_seconds_bucket{namespace=\"$namespace\", job=\"codex-pooler-app\", pod=~\"${pod:regex}\"}[$__rate_interval])))", "interval": "$resolution", "legendFormat": "{{pod}} p95" }, "version": "v0" }, "refId": "A" } } ], "queryOptions": {}, "transformations": [] } }, "description": "", "id": 27, "links": [], "title": "DB Queue Time p95", "vizConfig": { "group": "timeseries", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "opacity", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 2, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "showValues": false, "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 } ] }, "unit": "s" }, "overrides": [] }, "options": { "legend": { "calcs": [ "lastNotNull", "max" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "multi", "sort": "desc" } } }, "version": "12.3.1" } } }, "panel-28": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${datasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "editorMode": "code", "expr": "clamp_min(max by (pod) (container_memory_working_set_bytes{namespace=\"$namespace\", pod=~\"${pod:regex}\", container=\"app\", image!=\"\"}) - on (pod) max by (pod) (vm_memory_total_bytes{namespace=\"$namespace\", job=\"codex-pooler-app\", pod=~\"${pod:regex}\"}), 0)", "interval": "$resolution", "legendFormat": "{{pod}} cgroup - beam" }, "version": "v0" }, "refId": "A" } } ], "queryOptions": {}, "transformations": [] } }, "description": "Container working set not accounted for by BEAM total memory. Use this to separate BEAM heap/binary growth from native allocator, libc, NIF, or cgroup accounting pressure.", "id": 28, "links": [], "title": "Cgroup Minus BEAM", "vizConfig": { "group": "timeseries", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 20, "gradientMode": "opacity", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 2, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "showValues": false, "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 } ] }, "unit": "bytes" }, "overrides": [ { "matcher": { "id": "byRegexp", "options": ".* limit$" }, "properties": [ { "id": "color", "value": { "fixedColor": "red", "mode": "fixed" } }, { "id": "custom.fillOpacity", "value": 0 }, { "id": "custom.lineWidth", "value": 1 } ] } ] }, "options": { "legend": { "calcs": [ "lastNotNull", "max" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "multi", "sort": "desc" } } }, "version": "12.3.1" } } }, "panel-29": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${datasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "editorMode": "code", "expr": "max by (pod) (container_memory_rss{namespace=\"$namespace\", pod=~\"${pod:regex}\", container=\"app\", image!=\"\"})", "interval": "$resolution", "legendFormat": "{{pod}} rss" }, "version": "v0" }, "refId": "A" } }, { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${datasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "editorMode": "code", "expr": "max by (pod) (container_memory_cache{namespace=\"$namespace\", pod=~\"${pod:regex}\", container=\"app\", image!=\"\"})", "interval": "$resolution", "legendFormat": "{{pod}} cache" }, "version": "v0" }, "refId": "B" } }, { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${datasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "editorMode": "code", "expr": "max by (pod) (container_memory_mapped_file{namespace=\"$namespace\", pod=~\"${pod:regex}\", container=\"app\", image!=\"\"})", "interval": "$resolution", "legendFormat": "{{pod}} mapped file" }, "version": "v0" }, "refId": "C" } } ], "queryOptions": {}, "transformations": [] } }, "description": "Cgroup memory categories from cAdvisor. RSS rising with flat BEAM memory points outside normal BEAM heap attribution.", "id": 29, "links": [], "title": "Container RSS / Cache", "vizConfig": { "group": "timeseries", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 20, "gradientMode": "opacity", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 2, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "showValues": false, "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 } ] }, "unit": "bytes" }, "overrides": [ { "matcher": { "id": "byRegexp", "options": ".* limit$" }, "properties": [ { "id": "color", "value": { "fixedColor": "red", "mode": "fixed" } }, { "id": "custom.fillOpacity", "value": 0 }, { "id": "custom.lineWidth", "value": 1 } ] } ] }, "options": { "legend": { "calcs": [ "lastNotNull", "max" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "multi", "sort": "desc" } } }, "version": "12.3.1" } } }, "panel-3": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${datasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "editorMode": "code", "expr": "max((max by (pod) (container_memory_working_set_bytes{namespace=\"$namespace\", pod=~\"${pod:regex}\", container=\"app\", image!=\"\"}) / max by (pod) (kube_pod_container_resource_limits{cluster=\"$cluster\", namespace=\"$namespace\", pod=~\"${pod:regex}\", resource=\"memory\", exported_container=\"app\"})))", "instant": true, "legendFormat": "max pod usage / limit" }, "version": "v0" }, "refId": "A" } } ], "queryOptions": {}, "transformations": [] } }, "description": "", "id": 3, "links": [], "title": "Max Pod Memory / Limit", "vizConfig": { "group": "stat", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "decimals": 1, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 }, { "color": "yellow", "value": 0.7 }, { "color": "red", "value": 0.9 } ] }, "unit": "percentunit" }, "overrides": [] }, "options": { "colorMode": "background", "graphMode": "area", "justifyMode": "center", "orientation": "auto", "percentChangeColorMode": "standard", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "showPercentChange": false, "textMode": "auto", "wideLayout": true } }, "version": "12.3.1" } } }, "panel-30": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${datasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "editorMode": "code", "expr": "sum by (pod) (rate(container_cpu_cfs_throttled_periods_total{namespace=\"$namespace\", pod=~\"${pod:regex}\", container=\"app\", image!=\"\"}[$__rate_interval])) / clamp_min(sum by (pod) (rate(container_cpu_cfs_periods_total{namespace=\"$namespace\", pod=~\"${pod:regex}\", container=\"app\", image!=\"\"}[$__rate_interval])), 0.001)", "interval": "$resolution", "legendFormat": "{{pod}} throttled" }, "version": "v0" }, "refId": "A" } } ], "queryOptions": {}, "transformations": [] } }, "description": "CPU throttling can stretch request lifetimes and queues, increasing memory retained by in-flight requests.", "id": 30, "links": [], "title": "CPU Throttling Ratio", "vizConfig": { "group": "timeseries", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 15, "gradientMode": "opacity", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 2, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "showValues": false, "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 } ] }, "unit": "percentunit" }, "overrides": [] }, "options": { "legend": { "calcs": [ "lastNotNull", "max" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "multi", "sort": "desc" } } }, "version": "12.3.1" } } }, "panel-31": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${datasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "editorMode": "code", "expr": "max by (pod, container) (container_memory_working_set_bytes{namespace=\"$namespace\", pod=~\"codex-pooler-(app|oban-worker|oban-scheduler).*\", container=~\"app|oban-worker|oban-scheduler\", image!=\"\"})", "interval": "$resolution", "legendFormat": "{{pod}} {{container}}" }, "version": "v0" }, "refId": "A" } } ], "queryOptions": {}, "transformations": [] } }, "description": "App, worker, and scheduler cgroup memory in one view. App Prometheus metrics are scraped from the app service; worker/scheduler visibility here comes from Kubernetes cgroup metrics.", "id": 31, "links": [], "title": "Workload Memory by Component", "vizConfig": { "group": "timeseries", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 20, "gradientMode": "opacity", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 2, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "showValues": false, "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 } ] }, "unit": "bytes" }, "overrides": [ { "matcher": { "id": "byRegexp", "options": ".* limit$" }, "properties": [ { "id": "color", "value": { "fixedColor": "red", "mode": "fixed" } }, { "id": "custom.fillOpacity", "value": 0 }, { "id": "custom.lineWidth", "value": 1 } ] } ] }, "options": { "legend": { "calcs": [ "lastNotNull", "max" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "multi", "sort": "desc" } } }, "version": "12.3.1" } } }, "panel-32": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${datasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "editorMode": "code", "expr": "sum by (pod) (rate(container_cpu_usage_seconds_total{namespace=\"$namespace\", pod=~\"${pod:regex}\", container=\"app\", image!=\"\"}[$__rate_interval]))", "interval": "$resolution", "legendFormat": "{{pod}} cpu" }, "version": "v0" }, "refId": "A" } } ], "queryOptions": {}, "transformations": [] } }, "description": "App container CPU usage in cores. Correlate CPU spikes with request rate, DB query pressure, run queue, and admission queueing.", "id": 32, "links": [], "title": "CPU Usage", "vizConfig": { "group": "timeseries", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 15, "gradientMode": "opacity", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 2, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "showValues": false, "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 } ] }, "unit": "cores" }, "overrides": [] }, "options": { "legend": { "calcs": [ "lastNotNull", "max" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "multi", "sort": "desc" } } }, "version": "12.3.1" } } }, "panel-33": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${datasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "editorMode": "code", "expr": "sum by (method, status_class) (rate(codex_pooler_http_request_count{namespace=\"$namespace\", job=\"codex-pooler-app\", pod=~\"${pod:regex}\"}[$__rate_interval]))", "interval": "$resolution", "legendFormat": "{{method}} {{status_class}}" }, "version": "v0" }, "refId": "A" } } ], "queryOptions": {}, "transformations": [] } }, "description": "Low-cardinality request outcome rate. Use this to separate traffic growth from error storms.", "id": 33, "links": [], "title": "HTTP Status Class Rate", "vizConfig": { "group": "timeseries", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 15, "gradientMode": "opacity", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 2, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "showValues": false, "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 } ] }, "unit": "reqps" }, "overrides": [] }, "options": { "legend": { "calcs": [ "lastNotNull", "max" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "multi", "sort": "desc" } } }, "version": "12.3.1" } } }, "panel-34": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${datasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "editorMode": "code", "expr": "sum by (route_class, transport) (rate(codex_pooler_gateway_admission_accepted_count{namespace=\"$namespace\", job=\"codex-pooler-app\", pod=~\"${pod:regex}\"}[$__rate_interval]))", "interval": "$resolution", "legendFormat": "accepted {{route_class}} {{transport}}" }, "version": "v0" }, "refId": "A" } }, { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${datasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "editorMode": "code", "expr": "sum by (route_class, transport) (rate(codex_pooler_gateway_admission_enqueued_count{namespace=\"$namespace\", job=\"codex-pooler-app\", pod=~\"${pod:regex}\"}[$__rate_interval]))", "interval": "$resolution", "legendFormat": "enqueued {{route_class}} {{transport}}" }, "version": "v0" }, "refId": "B" } }, { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${datasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "editorMode": "code", "expr": "sum by (route_class, transport) (rate(codex_pooler_gateway_admission_dequeued_count{namespace=\"$namespace\", job=\"codex-pooler-app\", pod=~\"${pod:regex}\"}[$__rate_interval]))", "interval": "$resolution", "legendFormat": "dequeued {{route_class}} {{transport}}" }, "version": "v0" }, "refId": "C" } }, { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${datasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "editorMode": "code", "expr": "sum by (route_class, transport) (rate(codex_pooler_gateway_admission_rejected_count{namespace=\"$namespace\", job=\"codex-pooler-app\", pod=~\"${pod:regex}\"}[$__rate_interval]))", "interval": "$resolution", "legendFormat": "rejected {{route_class}} {{transport}}" }, "version": "v0" }, "refId": "D" } }, { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${datasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "editorMode": "code", "expr": "sum by (route_class, transport) (rate(codex_pooler_gateway_admission_timeout_count{namespace=\"$namespace\", job=\"codex-pooler-app\", pod=~\"${pod:regex}\"}[$__rate_interval]))", "interval": "$resolution", "legendFormat": "timeout {{route_class}} {{transport}}" }, "version": "v0" }, "refId": "E" } } ], "queryOptions": {}, "transformations": [] } }, "description": "Local gateway bulkhead pressure by route class and transport. Queue/reject/timeout spikes show admission backpressure before memory climbs.", "id": 34, "links": [], "title": "Admission Events", "vizConfig": { "group": "timeseries", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 15, "gradientMode": "opacity", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 2, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "showValues": false, "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 } ] }, "unit": "ops" }, "overrides": [] }, "options": { "legend": { "calcs": [ "lastNotNull", "max" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "multi", "sort": "desc" } } }, "version": "12.3.1" } } }, "panel-35": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${datasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "editorMode": "code", "expr": "histogram_quantile(0.95, sum by (le, route_class, transport) (rate(codex_pooler_gateway_admission_dequeued_time_seconds_bucket{namespace=\"$namespace\", job=\"codex-pooler-app\", pod=~\"${pod:regex}\"}[$__rate_interval])))", "interval": "$resolution", "legendFormat": "dequeued {{route_class}} {{transport}} p95" }, "version": "v0" }, "refId": "A" } }, { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${datasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "editorMode": "code", "expr": "histogram_quantile(0.95, sum by (le, route_class, transport) (rate(codex_pooler_gateway_admission_timeout_time_seconds_bucket{namespace=\"$namespace\", job=\"codex-pooler-app\", pod=~\"${pod:regex}\"}[$__rate_interval])))", "interval": "$resolution", "legendFormat": "timeout {{route_class}} {{transport}} p95" }, "version": "v0" }, "refId": "B" } } ], "queryOptions": {}, "transformations": [] } }, "description": "Time requests spent waiting in local admission queues before execution or timeout.", "id": 35, "links": [], "title": "Admission Queue Time p95", "vizConfig": { "group": "timeseries", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 15, "gradientMode": "opacity", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 2, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "showValues": false, "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 } ] }, "unit": "s" }, "overrides": [] }, "options": { "legend": { "calcs": [ "lastNotNull", "max" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "multi", "sort": "desc" } } }, "version": "12.3.1" } } }, "panel-36": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${datasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "editorMode": "code", "expr": "sum by (pod, command) (rate(codex_pooler_repo_query_count{namespace=\"$namespace\", job=\"codex-pooler-app\", pod=~\"${pod:regex}\", source=\"unknown\"}[$__rate_interval]))", "interval": "$resolution", "legendFormat": "{{pod}} {{command}}" }, "version": "v0" }, "refId": "A" } } ], "queryOptions": {}, "transformations": [] } }, "description": "Source-less queries that could not be mapped to a safe Ecto source or table identifier. Use this to spot unclassified raw SQL or telemetry source-attribution gaps.", "id": 36, "links": [], "title": "Unknown Query Rate", "vizConfig": { "group": "timeseries", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 15, "gradientMode": "opacity", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 2, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "showValues": false, "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 } ] }, "unit": "ops" }, "overrides": [] }, "options": { "legend": { "calcs": [ "lastNotNull", "max" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "multi", "sort": "desc" } } }, "version": "12.3.1" } } }, "panel-37": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${datasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "editorMode": "code", "expr": "sum by (command) (rate(codex_pooler_repo_query_count{namespace=\"$namespace\", job=\"codex-pooler-app\", pod=~\"${pod:regex}\"}[$__rate_interval]))", "interval": "$resolution", "legendFormat": "{{command}}" }, "version": "v0" }, "refId": "A" } } ], "queryOptions": {}, "transformations": [] } }, "description": "SQL command mix across repository queries. Useful for spotting unexpected write or transaction churn during incidents.", "id": 37, "links": [], "title": "Query Command Mix", "vizConfig": { "group": "timeseries", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 15, "gradientMode": "opacity", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 2, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "showValues": false, "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 } ] }, "unit": "ops" }, "overrides": [] }, "options": { "legend": { "calcs": [ "lastNotNull", "max" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "multi", "sort": "desc" } } }, "version": "12.3.1" } } }, "panel-38": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${datasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "editorMode": "code", "expr": "histogram_quantile(0.95, sum by (le, source, command) (rate(codex_pooler_repo_query_queue_time_seconds_bucket{namespace=\"$namespace\", job=\"codex-pooler-app\", pod=~\"${pod:regex}\"}[$__rate_interval])))", "interval": "$resolution", "legendFormat": "{{source}} {{command}} p95" }, "version": "v0" }, "refId": "A" } } ], "queryOptions": {}, "transformations": [] } }, "description": "Database connection checkout pressure by safe source and command.", "id": 38, "links": [], "title": "Repo Queue Time by Source p95", "vizConfig": { "group": "timeseries", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 15, "gradientMode": "opacity", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 2, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "showValues": false, "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 } ] }, "unit": "s" }, "overrides": [] }, "options": { "legend": { "calcs": [ "lastNotNull", "max" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "multi", "sort": "desc" } } }, "version": "12.3.1" } } }, "panel-4": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${datasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "editorMode": "code", "expr": "sum(increase(kube_pod_container_status_restarts_total{cluster=\"$cluster\", namespace=\"$namespace\", pod=~\"${pod:regex}\", exported_container=\"app\"}[30m]))", "instant": true, "legendFormat": "restarts" }, "version": "v0" }, "refId": "A" } } ], "queryOptions": {}, "transformations": [] } }, "description": "", "id": 4, "links": [], "title": "Restarts In 30m", "vizConfig": { "group": "stat", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 }, { "color": "red", "value": 1 } ] }, "unit": "short" }, "overrides": [] }, "options": { "colorMode": "background", "graphMode": "area", "justifyMode": "center", "orientation": "auto", "percentChangeColorMode": "standard", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "showPercentChange": false, "textMode": "auto", "wideLayout": true } }, "version": "12.3.1" } } }, "panel-5": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${datasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "editorMode": "code", "expr": "sum by () (max by (pod, name) (increase(container_oom_events_total{namespace=\"$namespace\", pod=~\"${pod:regex}\", container=\"app\"}[30m])))", "instant": true, "legendFormat": "oom events" }, "version": "v0" }, "refId": "A" } } ], "queryOptions": {}, "transformations": [] } }, "description": "", "id": 5, "links": [], "title": "OOM Events In 30m", "vizConfig": { "group": "stat", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 }, { "color": "red", "value": 1 } ] }, "unit": "short" }, "overrides": [] }, "options": { "colorMode": "background", "graphMode": "area", "justifyMode": "center", "orientation": "auto", "percentChangeColorMode": "standard", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "showPercentChange": false, "textMode": "auto", "wideLayout": true } }, "version": "12.3.1" } } }, "panel-6": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${datasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "editorMode": "code", "expr": "max(vm_memory_binary_bytes{namespace=\"$namespace\", job=\"codex-pooler-app\", pod=~\"${pod:regex}\"} / vm_memory_total_bytes{namespace=\"$namespace\", job=\"codex-pooler-app\", pod=~\"${pod:regex}\"})", "instant": true, "legendFormat": "binary / total" }, "version": "v0" }, "refId": "A" } } ], "queryOptions": {}, "transformations": [] } }, "description": "", "id": 6, "links": [], "title": "BEAM Binary Share", "vizConfig": { "group": "stat", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "decimals": 1, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 }, { "color": "yellow", "value": 0.05 }, { "color": "red", "value": 0.15 } ] }, "unit": "percentunit" }, "overrides": [] }, "options": { "colorMode": "background", "graphMode": "area", "justifyMode": "center", "orientation": "auto", "percentChangeColorMode": "standard", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "showPercentChange": false, "textMode": "auto", "wideLayout": true } }, "version": "12.3.1" } } }, "panel-8": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${datasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "editorMode": "code", "expr": "max by (pod) (container_memory_working_set_bytes{namespace=\"$namespace\", pod=~\"${pod:regex}\", container=\"app\", image!=\"\"})", "interval": "$resolution", "legendFormat": "{{pod}} working set" }, "version": "v0" }, "refId": "A" } }, { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${datasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "editorMode": "code", "expr": "max by (pod) (kube_pod_container_resource_limits{cluster=\"$cluster\", namespace=\"$namespace\", pod=~\"${pod:regex}\", resource=\"memory\", exported_container=\"app\"})", "interval": "$resolution", "legendFormat": "{{pod}} limit" }, "version": "v0" }, "refId": "B" } } ], "queryOptions": {}, "transformations": [] } }, "description": "", "id": 8, "links": [], "title": "Container Working Set vs Limit", "vizConfig": { "group": "timeseries", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 20, "gradientMode": "opacity", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 2, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "showValues": false, "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 } ] }, "unit": "bytes" }, "overrides": [ { "matcher": { "id": "byRegexp", "options": ".* limit$" }, "properties": [ { "id": "color", "value": { "fixedColor": "red", "mode": "fixed" } }, { "id": "custom.fillOpacity", "value": 0 }, { "id": "custom.lineWidth", "value": 1 } ] } ] }, "options": { "legend": { "calcs": [ "lastNotNull", "max" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "multi", "sort": "desc" } } }, "version": "12.3.1" } } }, "panel-9": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${datasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "editorMode": "code", "expr": "vm_memory_total_bytes{namespace=\"$namespace\", job=\"codex-pooler-app\", pod=~\"${pod:regex}\"}", "interval": "$resolution", "legendFormat": "{{pod}} total" }, "version": "v0" }, "refId": "A" } }, { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${datasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "editorMode": "code", "expr": "vm_memory_processes_bytes{namespace=\"$namespace\", job=\"codex-pooler-app\", pod=~\"${pod:regex}\"}", "interval": "$resolution", "legendFormat": "{{pod}} processes" }, "version": "v0" }, "refId": "B" } }, { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${datasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "editorMode": "code", "expr": "vm_memory_binary_bytes{namespace=\"$namespace\", job=\"codex-pooler-app\", pod=~\"${pod:regex}\"}", "interval": "$resolution", "legendFormat": "{{pod}} binary" }, "version": "v0" }, "refId": "C" } }, { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${datasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "editorMode": "code", "expr": "vm_memory_ets_bytes{namespace=\"$namespace\", job=\"codex-pooler-app\", pod=~\"${pod:regex}\"}", "interval": "$resolution", "legendFormat": "{{pod}} ets" }, "version": "v0" }, "refId": "D" } }, { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${datasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "editorMode": "code", "expr": "vm_memory_code_bytes{namespace=\"$namespace\", job=\"codex-pooler-app\", pod=~\"${pod:regex}\"}", "interval": "$resolution", "legendFormat": "{{pod}} code" }, "version": "v0" }, "refId": "E" } }, { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${datasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "editorMode": "code", "expr": "vm_memory_atom_bytes{namespace=\"$namespace\", job=\"codex-pooler-app\", pod=~\"${pod:regex}\"}", "interval": "$resolution", "legendFormat": "{{pod}} atom" }, "version": "v0" }, "refId": "F" } }, { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${datasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "editorMode": "code", "expr": "vm_memory_atom_used_bytes{namespace=\"$namespace\", job=\"codex-pooler-app\", pod=~\"${pod:regex}\"}", "interval": "$resolution", "legendFormat": "{{pod}} atom used" }, "version": "v0" }, "refId": "G" } }, { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${datasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "editorMode": "code", "expr": "vm_memory_system_bytes{namespace=\"$namespace\", job=\"codex-pooler-app\", pod=~\"${pod:regex}\"}", "interval": "$resolution", "legendFormat": "{{pod}} system" }, "version": "v0" }, "refId": "H" } } ], "queryOptions": {}, "transformations": [] } }, "description": "", "id": 9, "links": [], "title": "BEAM Memory Breakdown", "vizConfig": { "group": "timeseries", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 15, "gradientMode": "opacity", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 2, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "showValues": false, "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 } ] }, "unit": "bytes" }, "overrides": [] }, "options": { "legend": { "calcs": [ "lastNotNull", "max" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "multi", "sort": "desc" } } }, "version": "12.3.1" } } }, "panel-40": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${datasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "editorMode": "code", "expr": "sum by (stage, window, scope) (rate(codex_pooler_admin_stats_reload_count{namespace=\"$namespace\", job=\"codex-pooler-app\", pod=~\"${pod:regex}\"}[$__rate_interval]))", "interval": "$resolution", "legendFormat": "{{stage}} {{window}} {{scope}}" }, "version": "v0" }, "refId": "A" } } ], "queryOptions": {}, "transformations": [] } }, "description": "Admin stats LiveView reload scheduling, coalescing, cancellation, and execution rate by window and scope. Coalesced spikes mean a page is already waiting on one debounced reload.", "id": 40, "links": [], "title": "Admin Stats Reload Rate", "vizConfig": { "group": "timeseries", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 15, "gradientMode": "opacity", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 2, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "showValues": false, "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 } ] }, "unit": "ops" }, "overrides": [] }, "options": { "legend": { "calcs": [ "lastNotNull", "max" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "multi", "sort": "desc" } } }, "version": "12.3.1" } } }, "panel-41": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${datasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "editorMode": "code", "expr": "sum by (outcome, window, scope) (rate(codex_pooler_admin_stats_dashboard_build_count{namespace=\"$namespace\", job=\"codex-pooler-app\", pod=~\"${pod:regex}\"}[$__rate_interval]))", "interval": "$resolution", "legendFormat": "{{outcome}} {{window}} {{scope}}" }, "version": "v0" }, "refId": "A" } } ], "queryOptions": {}, "transformations": [] } }, "description": "Full admin stats dashboard builds by outcome, selected time window, and scope. This shows the reload work that can drive read-model DB pressure from open admin pages.", "id": 41, "links": [], "title": "Admin Stats Dashboard Build Rate", "vizConfig": { "group": "timeseries", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 15, "gradientMode": "opacity", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 2, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "showValues": false, "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 } ] }, "unit": "ops" }, "overrides": [] }, "options": { "legend": { "calcs": [ "lastNotNull", "max" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "multi", "sort": "desc" } } }, "version": "12.3.1" } } }, "panel-42": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${datasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "editorMode": "code", "expr": "histogram_quantile(0.95, sum by (le, outcome, window, scope) (rate(codex_pooler_admin_stats_dashboard_build_duration_seconds_bucket{namespace=\"$namespace\", job=\"codex-pooler-app\", pod=~\"${pod:regex}\"}[$__rate_interval])))", "interval": "$resolution", "legendFormat": "{{outcome}} {{window}} {{scope}} p95" }, "version": "v0" }, "refId": "A" } } ], "queryOptions": {}, "transformations": [] } }, "description": "p95 duration for full admin stats dashboard builds. Pair spikes here with hot query source and DB queue panels to identify expensive stats reload paths.", "id": 42, "links": [], "title": "Admin Stats Dashboard Build p95", "vizConfig": { "group": "timeseries", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 15, "gradientMode": "opacity", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 2, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "showValues": false, "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 } ] }, "unit": "s" }, "overrides": [] }, "options": { "legend": { "calcs": [ "lastNotNull", "max" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "multi", "sort": "desc" } } }, "version": "12.3.1" } } }, "panel-43": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${datasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "editorMode": "code", "expr": "sum by (stage, scope) (rate(codex_pooler_admin_request_logs_reload_count{namespace=\"$namespace\", job=\"codex-pooler-app\", pod=~\"${pod:regex}\"}[$__rate_interval]))", "interval": "$resolution", "legendFormat": "{{stage}} {{scope}}" }, "version": "v0" }, "refId": "A" } } ], "queryOptions": {}, "transformations": [] } }, "description": "Admin request-log LiveView reload rate by bounded stage and scope. Use this to separate first load, filter patch, and event-refresh pressure without tagging pool ids, models, request ids, users, paths, params, or API key prefixes.", "id": 43, "links": [], "title": "Admin Request Logs Reload Rate", "vizConfig": { "group": "timeseries", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 15, "gradientMode": "opacity", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 2, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "showValues": false, "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 } ] }, "unit": "ops" }, "overrides": [] }, "options": { "legend": { "calcs": [ "lastNotNull", "max" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "multi", "sort": "desc" } } }, "version": "12.3.1" } } }, "panel-44": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${datasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "editorMode": "code", "expr": "histogram_quantile(0.95, sum by (le, stage, scope) (rate(codex_pooler_admin_request_logs_reload_duration_seconds_bucket{namespace=\"$namespace\", job=\"codex-pooler-app\", pod=~\"${pod:regex}\"}[$__rate_interval])))", "interval": "$resolution", "legendFormat": "{{stage}} {{scope}} p95" }, "version": "v0" }, "refId": "A" } } ], "queryOptions": {}, "transformations": [] } }, "description": "p95 admin request-log reload duration by bounded stage and scope. Pair spikes with DB queue, hot safe query source, and pg_stat_statements evidence when validating request-log read-path remediation.", "id": 44, "links": [], "title": "Admin Request Logs Reload p95", "vizConfig": { "group": "timeseries", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 15, "gradientMode": "opacity", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 2, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "showValues": false, "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 } ] }, "unit": "s" }, "overrides": [] }, "options": { "legend": { "calcs": [ "lastNotNull", "max" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "multi", "sort": "desc" } } }, "version": "12.3.1" } } } }, "layout": { "kind": "GridLayout", "spec": { "items": [] } }, "links": [ { "asDropdown": false, "icon": "external link", "includeVars": true, "keepTime": true, "tags": [], "targetBlank": false, "title": "Kubernetes pod details", "tooltip": "Open the generic pod dashboard for the same namespace/pod selection", "type": "link", "url": "/d/k8s_views_pods/kubernetes-views-pods?var-datasource=${datasource}&var-cluster=${cluster}&var-namespace=${namespace}&var-pod=${pod}" } ], "liveNow": false, "preload": false, "tags": [ "codex-pooler", "kubernetes", "memory", "production" ], "timeSettings": { "autoRefresh": "auto", "autoRefreshIntervals": [ "5s", "10s", "30s", "1m", "5m", "15m", "30m", "1h", "2h", "1d" ], "fiscalYearStartMonth": 0, "from": "now-1h", "hideTimepicker": false, "timezone": "browser", "to": "now" }, "title": "Codex Pooler / Runtime Triage", "variables": [ { "kind": "DatasourceVariable", "spec": { "allowCustomValue": true, "current": { "text": "", "value": "" }, "hide": "dontHide", "includeAll": false, "label": "Datasource", "multi": false, "name": "datasource", "options": [], "pluginId": "prometheus", "refresh": "onDashboardLoad", "regex": "", "skipUrlSync": false } }, { "kind": "QueryVariable", "spec": { "allowCustomValue": true, "current": { "text": "", "value": "" }, "definition": "label_values(kube_node_info, cluster)", "hide": "dontHide", "includeAll": false, "label": "Cluster", "multi": false, "name": "cluster", "options": [], "query": { "datasource": { "name": "${datasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "qryType": 1, "query": "label_values(kube_node_info, cluster)", "refId": "PrometheusVariableQueryEditor-VariableQuery" }, "version": "v0" }, "refresh": "onDashboardLoad", "regex": "", "skipUrlSync": false, "sort": "alphabeticalAsc" } }, { "kind": "QueryVariable", "spec": { "allowCustomValue": true, "current": { "text": "", "value": "" }, "definition": "label_values(up{job=\"codex-pooler-app\"}, namespace)", "hide": "dontHide", "includeAll": false, "label": "Namespace", "multi": false, "name": "namespace", "options": [], "query": { "datasource": { "name": "${datasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "query": "label_values(up{job=\"codex-pooler-app\"}, namespace)", "refId": "Prometheus-namespace-Variable-Query" }, "version": "v0" }, "refresh": "onDashboardLoad", "regex": "", "skipUrlSync": false, "sort": "alphabeticalAsc" } }, { "kind": "QueryVariable", "spec": { "allValue": ".*", "allowCustomValue": true, "current": { "text": "", "value": "" }, "definition": "label_values(up{job=\"codex-pooler-app\", namespace=\"$namespace\"}, pod)", "hide": "dontHide", "includeAll": true, "label": "Pod", "multi": true, "name": "pod", "options": [], "query": { "datasource": { "name": "${datasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "query": "label_values(up{job=\"codex-pooler-app\", namespace=\"$namespace\"}, pod)", "refId": "Prometheus-pod-Variable-Query" }, "version": "v0" }, "refresh": "onTimeRangeChanged", "regex": "", "skipUrlSync": false, "sort": "alphabeticalAsc" } }, { "kind": "CustomVariable", "spec": { "allowCustomValue": true, "current": { "text": "10s", "value": "10s" }, "hide": "dontHide", "includeAll": false, "label": "Resolution", "multi": false, "name": "resolution", "options": [ { "selected": true, "text": "10s", "value": "10s" }, { "selected": false, "text": "15s", "value": "15s" }, { "selected": false, "text": "30s", "value": "30s" }, { "selected": false, "text": "1m", "value": "1m" } ], "query": "10s,15s,30s,1m", "skipUrlSync": false } } ] }, "status": {} }