{ "annotations": { "list": [ { "builtIn": 1, "datasource": { "type": "grafana", "uid": "-- Grafana --" }, "enable": true, "hide": true, "iconColor": "rgba(0, 211, 255, 1)", "name": "Annotations & Alerts", "type": "dashboard" } ] }, "editable": true, "fiscalYearStartMonth": 0, "graphTooltip": 1, "links": [], "panels": [ { "type": "row", "title": "Overview", "collapsed": false, "gridPos": { "x": 0, "y": 0, "w": 24, "h": 1 }, "id": 1, "panels": [] }, { "type": "stat", "title": "Request rate", "description": "Chat-completions dispatched to workers per second.", "datasource": { "type": "prometheus", "uid": "${datasource}" }, "gridPos": { "x": 0, "y": 1, "w": 6, "h": 4 }, "id": 2, "fieldConfig": { "defaults": { "unit": "reqps", "color": { "mode": "thresholds" }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] } }, "overrides": [] }, "options": { "colorMode": "value", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "textMode": "auto" }, "pluginVersion": "11.6.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "code", "expr": "sum(rate(sgl_router_worker_requests_total{model_id=~\"$model_id\",worker_url=~\"$worker_url\"}[$__rate_interval]))", "range": true, "refId": "A", "legendFormat": "__auto", "format": "time_series" } ] }, { "type": "stat", "title": "Error ratio", "description": "Share of dispatches that failed on the WORKER's side (outcome=error). Excludes client_error (4xx), backpressure (429/503) and cancelled, so a saturated fleet or a bad client does not read as a worker fault.", "datasource": { "type": "prometheus", "uid": "${datasource}" }, "gridPos": { "x": 6, "y": 1, "w": 6, "h": 4 }, "id": 3, "fieldConfig": { "defaults": { "unit": "percent", "color": { "mode": "thresholds" }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "yellow", "value": 1 }, { "color": "red", "value": 5 } ] } }, "overrides": [] }, "options": { "colorMode": "value", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "textMode": "auto" }, "pluginVersion": "11.6.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "code", "expr": "100 * sum(rate(sgl_router_worker_requests_total{outcome=\"error\",model_id=~\"$model_id\",worker_url=~\"$worker_url\"}[$__rate_interval])) / clamp_min(sum(rate(sgl_router_worker_requests_total{model_id=~\"$model_id\",worker_url=~\"$worker_url\"}[$__rate_interval])), 1e-9)", "range": true, "refId": "A", "legendFormat": "__auto", "format": "time_series" } ] }, { "type": "stat", "title": "P99 latency", "description": "99th percentile end-to-end request latency.", "datasource": { "type": "prometheus", "uid": "${datasource}" }, "gridPos": { "x": 12, "y": 1, "w": 6, "h": 4 }, "id": 4, "fieldConfig": { "defaults": { "unit": "s", "color": { "mode": "thresholds" }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "yellow", "value": 1 }, { "color": "red", "value": 5 } ] } }, "overrides": [] }, "options": { "colorMode": "value", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "textMode": "auto" }, "pluginVersion": "11.6.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "code", "expr": "histogram_quantile(0.99, sum by (le) (rate(sgl_router_request_duration_seconds_bucket{model_id=~\"$model_id\"}[$__rate_interval])))", "range": true, "refId": "A", "legendFormat": "__auto", "format": "time_series" } ] }, { "type": "stat", "title": "Healthy workers", "description": "Workers whose circuit breaker currently admits requests.", "datasource": { "type": "prometheus", "uid": "${datasource}" }, "gridPos": { "x": 18, "y": 1, "w": 6, "h": 4 }, "id": 5, "fieldConfig": { "defaults": { "unit": "short", "color": { "mode": "thresholds" }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "red", "value": null }, { "color": "green", "value": 1 } ] } }, "overrides": [] }, "options": { "colorMode": "value", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "textMode": "auto" }, "pluginVersion": "11.6.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "code", "expr": "sum(sgl_router_worker_health)", "range": true, "refId": "A", "legendFormat": "__auto", "format": "time_series" } ] }, { "type": "row", "title": "Traffic & Errors", "collapsed": false, "gridPos": { "x": 0, "y": 5, "w": 24, "h": 1 }, "id": 6, "panels": [] }, { "type": "timeseries", "title": "Request rate by outcome", "description": "success / error / cancelled.", "datasource": { "type": "prometheus", "uid": "${datasource}" }, "gridPos": { "x": 0, "y": 6, "w": 12, "h": 8 }, "id": 7, "fieldConfig": { "defaults": { "unit": "reqps", "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "normal" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] } }, "overrides": [] }, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "pluginVersion": "11.6.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "code", "expr": "sum by (outcome) (rate(sgl_router_worker_requests_total{model_id=~\"$model_id\",worker_url=~\"$worker_url\"}[$__rate_interval]))", "range": true, "refId": "A", "legendFormat": "{{outcome}}", "format": "time_series" } ] }, { "type": "timeseries", "title": "Request rate by mode", "description": "prefill / decode / plain dispatch split.", "datasource": { "type": "prometheus", "uid": "${datasource}" }, "gridPos": { "x": 12, "y": 6, "w": 12, "h": 8 }, "id": 8, "fieldConfig": { "defaults": { "unit": "reqps", "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "normal" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] } }, "overrides": [] }, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "pluginVersion": "11.6.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "code", "expr": "sum by (mode) (rate(sgl_router_worker_requests_total{model_id=~\"$model_id\",worker_url=~\"$worker_url\"}[$__rate_interval]))", "range": true, "refId": "A", "legendFormat": "{{mode}}", "format": "time_series" } ] }, { "type": "timeseries", "title": "Responses by HTTP status", "description": "Client-visible HTTP status codes. A 504 is any router-side give-up \u2014 the stale-request deadline or an upstream timeout \u2014 or a worker 504 forwarded unchanged; worker_requests_total{outcome} tells them apart.", "datasource": { "type": "prometheus", "uid": "${datasource}" }, "gridPos": { "x": 0, "y": 14, "w": 12, "h": 8 }, "id": 9, "fieldConfig": { "defaults": { "unit": "reqps", "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "normal" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] } }, "overrides": [] }, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "pluginVersion": "11.6.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "code", "expr": "sum by (status_code) (rate(sgl_router_responses_total[$__rate_interval]))", "range": true, "refId": "A", "legendFormat": "{{status_code}}", "format": "time_series" } ] }, { "type": "timeseries", "title": "Request rate by worker", "description": "Per-worker dispatch rate \u2014 reveals routing imbalance.", "datasource": { "type": "prometheus", "uid": "${datasource}" }, "gridPos": { "x": 12, "y": 14, "w": 12, "h": 8 }, "id": 10, "fieldConfig": { "defaults": { "unit": "reqps", "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] } }, "overrides": [] }, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "pluginVersion": "11.6.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "code", "expr": "sum by (worker_url) (rate(sgl_router_worker_requests_total{model_id=~\"$model_id\",worker_url=~\"$worker_url\"}[$__rate_interval]))", "range": true, "refId": "A", "legendFormat": "{{worker_url}}", "format": "time_series" } ] }, { "type": "row", "title": "Latency", "collapsed": false, "gridPos": { "x": 0, "y": 22, "w": 24, "h": 1 }, "id": 11, "panels": [] }, { "type": "timeseries", "title": "Request latency quantiles", "description": "End-to-end request latency percentiles.", "datasource": { "type": "prometheus", "uid": "${datasource}" }, "gridPos": { "x": 0, "y": 23, "w": 8, "h": 8 }, "id": 12, "fieldConfig": { "defaults": { "unit": "s", "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] } }, "overrides": [] }, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "pluginVersion": "11.6.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "code", "expr": "histogram_quantile(0.5, sum by (le) (rate(sgl_router_request_duration_seconds_bucket{model_id=~\"$model_id\"}[$__rate_interval])))", "range": true, "refId": "A", "legendFormat": "p50", "format": "time_series" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "code", "expr": "histogram_quantile(0.9, sum by (le) (rate(sgl_router_request_duration_seconds_bucket{model_id=~\"$model_id\"}[$__rate_interval])))", "range": true, "refId": "B", "legendFormat": "p90", "format": "time_series" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "code", "expr": "histogram_quantile(0.99, sum by (le) (rate(sgl_router_request_duration_seconds_bucket{model_id=~\"$model_id\"}[$__rate_interval])))", "range": true, "refId": "C", "legendFormat": "p99", "format": "time_series" } ] }, { "type": "timeseries", "title": "TTFT quantiles (streaming)", "description": "Time to first token (first upstream chunk) for streaming requests.", "datasource": { "type": "prometheus", "uid": "${datasource}" }, "gridPos": { "x": 8, "y": 23, "w": 8, "h": 8 }, "id": 13, "fieldConfig": { "defaults": { "unit": "s", "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] } }, "overrides": [] }, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "pluginVersion": "11.6.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "code", "expr": "histogram_quantile(0.5, sum by (le) (rate(sgl_router_ttft_seconds_bucket{model_id=~\"$model_id\"}[$__rate_interval])))", "range": true, "refId": "A", "legendFormat": "p50", "format": "time_series" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "code", "expr": "histogram_quantile(0.9, sum by (le) (rate(sgl_router_ttft_seconds_bucket{model_id=~\"$model_id\"}[$__rate_interval])))", "range": true, "refId": "B", "legendFormat": "p90", "format": "time_series" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "code", "expr": "histogram_quantile(0.99, sum by (le) (rate(sgl_router_ttft_seconds_bucket{model_id=~\"$model_id\"}[$__rate_interval])))", "range": true, "refId": "C", "legendFormat": "p99", "format": "time_series" } ] }, { "type": "heatmap", "title": "Request latency distribution", "description": "Latency histogram over time.", "datasource": { "type": "prometheus", "uid": "${datasource}" }, "gridPos": { "x": 16, "y": 23, "w": 8, "h": 8 }, "id": 14, "options": { "calculate": false, "cellGap": 1, "color": { "scheme": "Spectral", "mode": "scheme", "steps": 64, "reverse": false, "exponent": 0.5, "fill": "dark-orange" }, "yAxis": { "unit": "s", "axisPlacement": "left" }, "legend": { "show": true }, "tooltip": { "mode": "single", "yHistogram": true, "showColorScale": true }, "rowsFrame": { "layout": "auto" }, "cellValues": {}, "filterValues": { "le": 1e-09 } }, "pluginVersion": "11.6.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "code", "expr": "sum by (le) (rate(sgl_router_request_duration_seconds_bucket{model_id=~\"$model_id\"}[$__rate_interval]))", "format": "heatmap", "legendFormat": "{{le}}", "range": true, "refId": "A" } ] }, { "type": "row", "title": "Workers & Health", "collapsed": false, "gridPos": { "x": 0, "y": 31, "w": 24, "h": 1 }, "id": 15, "panels": [] }, { "type": "timeseries", "title": "Workers by mode", "description": "Registered worker count per mode.", "datasource": { "type": "prometheus", "uid": "${datasource}" }, "gridPos": { "x": 0, "y": 32, "w": 6, "h": 8 }, "id": 16, "fieldConfig": { "defaults": { "unit": "short", "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] } }, "overrides": [] }, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "pluginVersion": "11.6.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "code", "expr": "sgl_router_workers", "range": true, "refId": "A", "legendFormat": "{{mode}}", "format": "time_series" } ] }, { "type": "timeseries", "title": "Worker health", "description": "1 = breaker admits requests, 0 = open.", "datasource": { "type": "prometheus", "uid": "${datasource}" }, "gridPos": { "x": 6, "y": 32, "w": 6, "h": 8 }, "id": 17, "fieldConfig": { "defaults": { "unit": "short", "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] } }, "overrides": [] }, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "pluginVersion": "11.6.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "code", "expr": "sgl_router_worker_health{worker_url=~\"$worker_url\"}", "range": true, "refId": "A", "legendFormat": "{{worker_url}}", "format": "time_series" } ] }, { "type": "timeseries", "title": "Circuit breaker state", "description": "0=closed, 1=open, 2=half_open.", "datasource": { "type": "prometheus", "uid": "${datasource}" }, "gridPos": { "x": 12, "y": 32, "w": 6, "h": 8 }, "id": 18, "fieldConfig": { "defaults": { "unit": "short", "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [ { "type": "value", "options": { "0": { "text": "closed", "color": "green" }, "1": { "text": "open", "color": "red" }, "2": { "text": "half_open", "color": "yellow" } } } ], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] } }, "overrides": [] }, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "pluginVersion": "11.6.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "code", "expr": "sgl_router_worker_cb_state{worker_url=~\"$worker_url\"}", "range": true, "refId": "A", "legendFormat": "{{worker_url}}", "format": "time_series" } ] }, { "type": "timeseries", "title": "In-flight requests per worker", "description": "Requests currently dispatched to each worker.", "datasource": { "type": "prometheus", "uid": "${datasource}" }, "gridPos": { "x": 18, "y": 32, "w": 6, "h": 8 }, "id": 19, "fieldConfig": { "defaults": { "unit": "short", "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] } }, "overrides": [] }, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "pluginVersion": "11.6.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "code", "expr": "sgl_router_worker_inflight_requests{worker_url=~\"$worker_url\"}", "range": true, "refId": "A", "legendFormat": "{{worker_url}}", "format": "time_series" } ] }, { "type": "row", "title": "Cache & Load", "collapsed": false, "gridPos": { "x": 0, "y": 40, "w": 24, "h": 1 }, "id": 20, "panels": [] }, { "type": "timeseries", "title": "Active load \u2014 prefill tokens", "description": "Per-worker prefill-token load tracked by the active-load policy.", "datasource": { "type": "prometheus", "uid": "${datasource}" }, "gridPos": { "x": 0, "y": 41, "w": 8, "h": 8 }, "id": 21, "fieldConfig": { "defaults": { "unit": "short", "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] } }, "overrides": [] }, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "pluginVersion": "11.6.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "code", "expr": "sgl_router_active_load{kind=\"prefill_tokens\",worker_url=~\"$worker_url\"}", "range": true, "refId": "A", "legendFormat": "{{worker_url}}", "format": "time_series" } ] }, { "type": "timeseries", "title": "Active load \u2014 decode blocks", "description": "Per-worker decode-block load tracked by the active-load policy.", "datasource": { "type": "prometheus", "uid": "${datasource}" }, "gridPos": { "x": 8, "y": 41, "w": 8, "h": 8 }, "id": 22, "fieldConfig": { "defaults": { "unit": "short", "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] } }, "overrides": [] }, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "pluginVersion": "11.6.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "code", "expr": "sgl_router_active_load{kind=\"decode_blocks\",worker_url=~\"$worker_url\"}", "range": true, "refId": "A", "legendFormat": "{{worker_url}}", "format": "time_series" } ] }, { "type": "row", "title": "Routing Policy", "collapsed": false, "gridPos": { "x": 0, "y": 49, "w": 24, "h": 1 }, "id": 24, "panels": [] }, { "type": "timeseries", "title": "Sticky-session outcomes", "description": "hit / assigned / remap / no_routing_key.", "datasource": { "type": "prometheus", "uid": "${datasource}" }, "gridPos": { "x": 0, "y": 50, "w": 8, "h": 8 }, "id": 25, "fieldConfig": { "defaults": { "unit": "reqps", "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "normal" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] } }, "overrides": [] }, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "pluginVersion": "11.6.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "code", "expr": "sum by (outcome) (rate(sgl_router_sticky_total[$__rate_interval]))", "range": true, "refId": "A", "legendFormat": "{{outcome}}", "format": "time_series" } ] }, { "type": "timeseries", "title": "Decode-affinity outcomes", "description": "same_host_picked / fallback_breaker / fallback_load_imbalance.", "datasource": { "type": "prometheus", "uid": "${datasource}" }, "gridPos": { "x": 8, "y": 50, "w": 8, "h": 8 }, "id": 26, "fieldConfig": { "defaults": { "unit": "reqps", "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "normal" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] } }, "overrides": [] }, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "pluginVersion": "11.6.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "code", "expr": "sum by (outcome) (rate(sgl_router_decode_affinity_total[$__rate_interval]))", "range": true, "refId": "A", "legendFormat": "{{outcome}}", "format": "time_series" } ] }, { "type": "timeseries", "title": "Stale-request cancellations", "description": "Stale-request cancellations fired by the janitor.", "datasource": { "type": "prometheus", "uid": "${datasource}" }, "gridPos": { "x": 16, "y": 50, "w": 8, "h": 8 }, "id": 27, "fieldConfig": { "defaults": { "unit": "reqps", "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] } }, "overrides": [] }, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "pluginVersion": "11.6.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "code", "expr": "sum(rate(sgl_router_stale_requests_total[$__rate_interval]))", "range": true, "refId": "A", "legendFormat": "expired", "format": "time_series" } ] }, { "type": "row", "title": "KV tree vs engine (storage-tier coverage)", "collapsed": false, "gridPos": { "x": 0, "y": 58, "w": 24, "h": 1 }, "id": 28, "panels": [] }, { "type": "timeseries", "title": "KV event blocks by medium", "description": "Blocks/s the router tree consumes, by event kind and the storage-medium tag the engine put on the event, per deployment. On a hierarchical-cache fleet block_stored/CPU_PINNED runs at roughly the block_removed/GPU rate; they are not equal, because the engine also evicts device blocks it never backed up. A CPU_PINNED line at 0 while the engines report hicache backups means the tier stream is not reaching the router. Both the busiest and the quietest replica are plotted: every replica receives the full stream, so a quietest-replica line at 0 while the busiest is healthy means one replica has lost its subscription and is routing blind.", "datasource": { "type": "prometheus", "uid": "${datasource}" }, "gridPos": { "x": 0, "y": 59, "w": 12, "h": 8 }, "id": 29, "fieldConfig": { "defaults": { "unit": "short", "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] } }, "overrides": [] }, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "pluginVersion": "11.6.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "code", "expr": "max by (namespace, event, medium) (rate(sgl_router_kv_event_blocks_total[$__rate_interval]))", "range": true, "refId": "A", "legendFormat": "{{namespace}} {{event}}/{{medium}}", "format": "time_series" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "code", "expr": "min by (namespace, event, medium) (rate(sgl_router_kv_event_blocks_total[$__rate_interval]))", "range": true, "refId": "B", "legendFormat": "{{namespace}} {{event}}/{{medium}} (quietest replica)", "format": "time_series" } ] }, { "type": "timeseries", "title": "KV tree blocks by tier (fleet)", "description": "Blocks the router tree attributes to the fleet, by the tier the worker holds them on (a block held on device and host counts under both). Absolute counts, so summing across workers is valid. host should sit at roughly hicache-ratio times device on a fleet with a full host tier; host at 0 means routing cannot see the L2 tier at all. max per worker across replicas, then summed.", "datasource": { "type": "prometheus", "uid": "${datasource}" }, "gridPos": { "x": 12, "y": 59, "w": 12, "h": 8 }, "id": 30, "fieldConfig": { "defaults": { "unit": "short", "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] } }, "overrides": [] }, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "pluginVersion": "11.6.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "code", "expr": "sum by (tier) (max by (worker_url, dp_rank, tier) (sgl_router_kv_tree_blocks))", "range": true, "refId": "A", "legendFormat": "{{tier}}", "format": "time_series" } ] }, { "type": "timeseries", "title": "Tree coverage of engine host tier", "description": "Router-attributed host-tier tokens / engine hicache host tokens in use, per deployment. ~1: every block the engines hold on host is routable. ~0 with a full host tier: the L2 tier is invisible to routing and repeats of anything evicted from device land on random pods -- this is the ratio that separates a retention problem (the engines really evicted it) from a visibility problem (the engines still hold it, the router forgot). Above 1: the tree holds tiers a worker already released, so check sgl_router_kv_event_batches_lost_total -- a tagged removal clears only its own tier, and a batch lost in transit strands the rest. Both sides are grouped by namespace because the ratio is not additive across deployments.", "datasource": { "type": "prometheus", "uid": "${datasource}" }, "gridPos": { "x": 0, "y": 67, "w": 24, "h": 8 }, "id": 31, "fieldConfig": { "defaults": { "unit": "percentunit", "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "min": 0 }, "overrides": [] }, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "pluginVersion": "11.6.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "code", "expr": "(sum by (namespace) (max by (namespace, worker_url, dp_rank) (sgl_router_kv_tree_blocks{tier=\"host\"})) * on(namespace) group_left() max by (namespace) (sgl_router_kv_block_size)) / sum by (namespace) (sglang:hicache_host_used_tokens{tp_rank=\"0\"})", "range": true, "refId": "A", "legendFormat": "{{namespace}}", "format": "time_series" } ] } ], "refresh": "30s", "schemaVersion": 39, "tags": [ "sglang", "sgl-router", "experimental" ], "templating": { "list": [ { "current": {}, "hide": 0, "includeAll": false, "label": "Datasource", "multi": false, "name": "datasource", "options": [], "query": "prometheus", "refresh": 1, "regex": "", "skipUrlSync": false, "type": "datasource" }, { "current": { "text": "All", "value": "$__all" }, "datasource": { "type": "prometheus", "uid": "${datasource}" }, "definition": "label_values(sgl_router_worker_requests_total, model_id)", "hide": 0, "includeAll": true, "allValue": ".*", "label": "model_id", "multi": true, "name": "model_id", "options": [], "query": { "query": "label_values(sgl_router_worker_requests_total, model_id)", "refId": "StandardVariableQuery" }, "refresh": 2, "regex": "", "sort": 1, "type": "query" }, { "current": { "text": "All", "value": "$__all" }, "datasource": { "type": "prometheus", "uid": "${datasource}" }, "definition": "label_values(sgl_router_worker_requests_total, worker_url)", "hide": 0, "includeAll": true, "allValue": ".*", "label": "worker_url", "multi": true, "name": "worker_url", "options": [], "query": { "query": "label_values(sgl_router_worker_requests_total, worker_url)", "refId": "StandardVariableQuery" }, "refresh": 2, "regex": "", "sort": 1, "type": "query" } ] }, "time": { "from": "now-1h", "to": "now" }, "timepicker": {}, "timezone": "", "title": "SGLang Router (experimental)", "uid": "sgl-router-experimental", "version": 1, "weekStart": "" }