minke.yu/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 12 Packages Projects Releases Wiki Activity

[router] Count requests/responses at the HTTP edge for true intake (#29467)

Browse Source
This commit is contained in:
Liangsheng Yin
2026-06-26 21:21:41 -07:00
committed by GitHub
parent 09ca4fc96b
commit cd6dedf972
10 changed files with 225 additions and 65 deletions
Download Patch File Download Diff File Expand all files Collapse all files
experimental/sgl-router/monitoring/README.md
+3 -2
View File
@@ -20,10 +20,11 @@ The dashboard graphs every family the router emits:
| Metric | Type | What it shows |
|---|---|---|
| `sgl_router_requests_total` | Counter | Dispatches by `worker_url`, `model_id`, `mode`, `outcome` |
| `sgl_router_requests_total` | Counter | **Edge intake** — every request received at the router HTTP boundary, by `route`, `method`, counted before worker dispatch (true intake) |
| `sgl_router_responses_total` | Counter | **Edge responses** — every response returned, by `route`, `method`, `status_code` (incl. early-exit 400/413/503). `requests_total - responses_total` = received-but-not-answered |
| `sgl_router_worker_requests_total` | Counter | Per-worker **dispatches** by `worker_url`, `model_id`, `mode`, `outcome` (recorded after dispatch; blind to pre-dispatch drops) |
| `sgl_router_request_duration_seconds` | Histogram | End-to-end request latency by `model_id` |
| `sgl_router_ttft_seconds` | Histogram | Time to first token (streaming) by `model_id` |
| `sgl_router_responses_total` | Counter | Client-visible HTTP `status_code` |
| `sgl_router_overlap_blocks` | Histogram | Cache-aware-zmq overlap blocks by `model_id` |
| `sgl_router_active_load` | Gauge | Per-worker prefill-token / decode-block load |
| `sgl_router_workers` | Gauge | Registered worker count by `mode` |
experimental/sgl-router/monitoring/grafana-dashboard.json
+9 -9
View File
@@ -89,7 +89,7 @@
"uid": "${datasource}"
},
"editorMode": "code",
"expr": "sum(rate(sgl_router_requests_total{model_id=~\"$model_id\",worker_url=~\"$worker_url\"}[$__rate_interval]))",
"expr": "sum(rate(sgl_router_worker_requests_total{model_id=~\"$model_id\",worker_url=~\"$worker_url\"}[$__rate_interval]))",
"range": true,
"refId": "A",
"legendFormat": "__auto",
@@ -161,7 +161,7 @@
"uid": "${datasource}"
},
"editorMode": "code",
"expr": "100 * sum(rate(sgl_router_requests_total{outcome=\"error\",model_id=~\"$model_id\",worker_url=~\"$worker_url\"}[$__rate_interval])) / clamp_min(sum(rate(sgl_router_requests_total{model_id=~\"$model_id\",worker_url=~\"$worker_url\"}[$__rate_interval])), 1e-9)",
"expr": "100 * sum(rate(sgl_router_worker_requests_total{outcome=\"error\",model_id=~\"$model_id\",worker_url=~\"$worker_url\"}[$__rate_interval])) / clamp_min(sum(rate(sgl_router_worker_requests_total{model_id=~\"$model_id\",worker_url=~\"$worker_url\"}[$__rate_interval])), 1e-9)",
"range": true,
"refId": "A",
"legendFormat": "__auto",
@@ -407,7 +407,7 @@
"uid": "${datasource}"
},
"editorMode": "code",
"expr": "sum by (outcome) (rate(sgl_router_requests_total{model_id=~\"$model_id\",worker_url=~\"$worker_url\"}[$__rate_interval]))",
"expr": "sum by (outcome) (rate(sgl_router_worker_requests_total{model_id=~\"$model_id\",worker_url=~\"$worker_url\"}[$__rate_interval]))",
"range": true,
"refId": "A",
"legendFormat": "{{outcome}}",
@@ -500,7 +500,7 @@
"uid": "${datasource}"
},
"editorMode": "code",
"expr": "sum by (mode) (rate(sgl_router_requests_total{model_id=~\"$model_id\",worker_url=~\"$worker_url\"}[$__rate_interval]))",
"expr": "sum by (mode) (rate(sgl_router_worker_requests_total{model_id=~\"$model_id\",worker_url=~\"$worker_url\"}[$__rate_interval]))",
"range": true,
"refId": "A",
"legendFormat": "{{mode}}",
@@ -686,7 +686,7 @@
"uid": "${datasource}"
},
"editorMode": "code",
"expr": "sum by (worker_url) (rate(sgl_router_requests_total{model_id=~\"$model_id\",worker_url=~\"$worker_url\"}[$__rate_interval]))",
"expr": "sum by (worker_url) (rate(sgl_router_worker_requests_total{model_id=~\"$model_id\",worker_url=~\"$worker_url\"}[$__rate_interval]))",
"range": true,
"refId": "A",
"legendFormat": "{{worker_url}}",
@@ -2035,7 +2035,7 @@
"type": "prometheus",
"uid": "${datasource}"
},
"definition": "label_values(sgl_router_requests_total, model_id)",
"definition": "label_values(sgl_router_worker_requests_total, model_id)",
"hide": 0,
"includeAll": true,
"allValue": ".*",
@@ -2044,7 +2044,7 @@
"name": "model_id",
"options": [],
"query": {
"query": "label_values(sgl_router_requests_total, model_id)",
"query": "label_values(sgl_router_worker_requests_total, model_id)",
"refId": "StandardVariableQuery"
},
"refresh": 2,
@@ -2061,7 +2061,7 @@
"type": "prometheus",
"uid": "${datasource}"
},
"definition": "label_values(sgl_router_requests_total, worker_url)",
"definition": "label_values(sgl_router_worker_requests_total, worker_url)",
"hide": 0,
"includeAll": true,
"allValue": ".*",
@@ -2070,7 +2070,7 @@
"name": "worker_url",
"options": [],
"query": {
"query": "label_values(sgl_router_requests_total, worker_url)",
"query": "label_values(sgl_router_worker_requests_total, worker_url)",
"refId": "StandardVariableQuery"
},
"refresh": 2,