[router] Count requests/responses at the HTTP edge for true intake (#29467)

2026-06-26 21:21:41 -07:00
parent 09ca4fc96b
commit cd6dedf972
10 changed files with 225 additions and 65 deletions
@@ -19,7 +19,7 @@ _ACTIVE_RE = re.compile(
    r'^sgl_router_active_load\{worker_url="([^"]+)",kind="prefill_tokens"\}\s+(-?\d+)'
 )
 _REQ_TOTAL_RE = re.compile(
-    r"^sgl_router_requests_total\{([^}]*)\}\s+(\d+(?:\.\d+)?)\s*$"
+    r"^sgl_router_worker_requests_total\{([^}]*)\}\s+(\d+(?:\.\d+)?)\s*$"
 )
 _LABEL_RE = re.compile(r'(\w+)="([^"]*)"')

@@ -74,7 +74,7 @@ PREFIX_Y = (_PREFIX_Y_BODY * 8).strip()


 _REQ_TOTAL_RE = re.compile(
-    r"^sgl_router_requests_total\{([^}]*)\}\s+(\d+(?:\.\d+)?)\s*$"
+    r"^sgl_router_worker_requests_total\{([^}]*)\}\s+(\d+(?:\.\d+)?)\s*$"
 )
 _LABEL_RE = re.compile(r'(\w+)="([^"]*)"')

@@ -90,6 +90,48 @@ async fn non_streaming_returns_200() {
    assert_eq!(v["choices"][0]["message"]["content"], "ok");
 }

+/// Edge counters fire through the real middleware: `requests_total` at entry +
+/// `responses_total` on exit, with matched-route/method labels. The unit tests
+/// call record_* directly, so this is the only check that the middleware is
+/// actually wired (MatchedPath -> record_ingress / record_response).
+#[tokio::test]
+async fn edge_counters_recorded_through_middleware() {
+    let worker = crate::common::mock_worker::MockWorker::start(vec![]).await;
+    let ctx = build_ctx_with_worker(&worker.url);
+    let app = build_router(ctx.clone());
+
+    let req = Request::builder()
+        .method("POST")
+        .uri("/v1/chat/completions")
+        .header("content-type", "application/json")
+        .body(Body::from(
+            serde_json::to_vec(&serde_json::json!({
+                "model": "tiny",
+                "messages": [{"role": "user", "content": "hi"}],
+                "stream": false
+            }))
+            .unwrap(),
+        ))
+        .unwrap();
+    let res = app.oneshot(req).await.unwrap();
+    assert_eq!(res.status(), StatusCode::OK);
+    let _ = res.into_body().collect().await;
+
+    let m = ctx.metrics.render();
+    // intake — counted at entry by the middleware (the path unit tests miss)
+    assert!(
+        m.contains(r#"sgl_router_requests_total{route="/v1/chat/completions",method="POST"} 1"#),
+        "edge intake counter missing; got:\n{m}",
+    );
+    // response — counted on the way out by the middleware
+    assert!(
+        m.contains(
+            r#"sgl_router_responses_total{route="/v1/chat/completions",method="POST",status_code="200"} 1"#
+        ),
+        "edge response counter missing; got:\n{m}",
+    );
+}
+
 #[tokio::test]
 async fn non_streaming_upstream_unreachable_returns_502_unreachable() {
    // Bind a port, drop it — guarantees a closed/refused TCP destination.
@@ -288,7 +330,9 @@ async fn streaming_5xx_request_records_duration_and_status_but_not_ttft() {
        "TTFT must NOT be recorded for a non-2xx streaming response; got:\n{m}",
    );
    assert!(
-        m.contains(r#"sgl_router_responses_total{status_code="500"} 1"#),
+        m.contains(
+            r#"sgl_router_responses_total{route="/v1/chat/completions",method="POST",status_code="500"} 1"#
+        ),
        "the 500 status must be counted; got:\n{m}",
    );
    assert!(
@@ -93,12 +93,12 @@ fn chat_request(header: Option<(&str, &str)>) -> Request<Body> {
        .unwrap()
 }

-/// Parse `sgl_router_requests_total{...,outcome="success"} N` lines into a
+/// Parse `sgl_router_worker_requests_total{...,outcome="success"} N` lines into a
 /// map of worker_url -> success count.
 fn success_counts(metrics: &str) -> std::collections::HashMap<String, u64> {
    let mut counts = std::collections::HashMap::new();
    for line in metrics.lines() {
-        let Some(rest) = line.strip_prefix("sgl_router_requests_total{") else {
+        let Some(rest) = line.strip_prefix("sgl_router_worker_requests_total{") else {
            continue;
        };
        if !rest.contains(r#"outcome="success""#) {