Fix reasoning metrics and add TPOT to bench_multiturn (#35443)

benchmark/hicache/bench_multiturn.py

@@ -17,6 +17,7 @@ from sglang.benchmark.utils import get_tokenizer
 from sglang.test.kits.cache_hit_kit import (
     async_request_openai_chat_completions,
     async_request_sglang_generate,
+    calculate_tpot_statistics,
     gen_payload,
     gen_payload_openai,
 )
@@ -362,6 +363,7 @@ class WorkloadGenerator:
         self.pbar = tqdm(total=self.total_requests)
         self.performance_metrics = {
             "ttft": [],
+            "tpot": [],
             "itl": [],
             "latency": [],
             "prompt_len": [],
@@ -458,6 +460,8 @@ class WorkloadGenerator:
                 current_round = self.client_records[client_id]["round"]
                 self.client_records[client_id]["round"] += 1
                 self.performance_metrics["ttft"].append(response.ttft)
+                if response.tpot is not None:
+                    self.performance_metrics["tpot"].append(response.tpot)
                 self.performance_metrics["itl"].extend(response.itl)
                 self.performance_metrics["latency"].append(response.latency)
                 self.performance_metrics["prompt_len"].append(response.prompt_len)
@@ -582,6 +586,8 @@ class WorkloadGenerator:
         def max_or_zero(sorted_vals):
             return sorted_vals[-1] if sorted_vals else 0.0
 
+        tpot_statistics = calculate_tpot_statistics(self.performance_metrics["tpot"])
+
         performance_data = {
             "summary": {
                 "total_requests": len(self.performance_metrics["ttft"]),
@@ -608,6 +614,7 @@ class WorkloadGenerator:
                 "p99_ttft": percentile(sorted_ttft, 0.99),
                 "median_ttft": percentile(sorted_ttft, 0.5),
                 "max_ttft": max_or_zero(sorted_ttft),
+                **tpot_statistics,
                 "average_itl": (
                     sum(self.performance_metrics["itl"])
                     / len(self.performance_metrics["itl"])
@@ -686,6 +693,11 @@ class WorkloadGenerator:
         print(f"  P99 TTFT: {performance_data['summary']['p99_ttft']:.2f}")
         print(f"  Median TTFT: {performance_data['summary']['median_ttft']:.2f}")
         print(f"  Max TTFT: {performance_data['summary']['max_ttft']:.2f}")
+        print(f"  Average TPOT: {performance_data['summary']['average_tpot']:.4f}")
+        print(f"  P90 TPOT: {performance_data['summary']['p90_tpot']:.4f}")
+        print(f"  P99 TPOT: {performance_data['summary']['p99_tpot']:.4f}")
+        print(f"  Median TPOT: {performance_data['summary']['median_tpot']:.4f}")
+        print(f"  Max TPOT: {performance_data['summary']['max_tpot']:.4f}")
         print(f"  Average ITL: {performance_data['summary']['average_itl']:.4f}")
         print(f"  P90 ITL: {performance_data['summary']['p90_itl']:.4f}")
         print(f"  P99 ITL: {performance_data['summary']['p99_itl']:.4f}")

python/sglang/benchmark/serving.py

@@ -101,6 +101,7 @@ class RequestFuncOutput:
     success: bool = False
     latency: float = 0.0
     ttft: float = 0.0  # Time to first token
+    tpot: Optional[float] = None  # Time per output token
     itl: List[float] = field(default_factory=list)  # List of inter-token latencies
     text_chunks: List[str] = field(default_factory=list)
     prompt_len: int = 0

python/sglang/test/kits/cache_hit_kit.py

@@ -3,6 +3,7 @@ import json
 import time
 
 import aiohttp
+import numpy as np
 import requests
 
 from sglang.benchmark.datasets.random import sample_random_requests
@@ -12,6 +13,40 @@ from sglang.benchmark.utils import get_tokenizer, remove_prefix
 AIOHTTP_TIMEOUT = aiohttp.ClientTimeout(total=20 * 60 * 60)
 
 
+def get_openai_chat_output_delta(delta):
+    """Return text emitted by an OpenAI-compatible chat streaming delta."""
+    if not delta:
+        return ""
+    reasoning = delta.get("reasoning_content") or delta.get("reasoning") or ""
+    return reasoning + (delta.get("content") or "")
+
+
+def calculate_tpot(latency, ttft, completion_tokens):
+    """Calculate request-level time per output token when inputs are valid."""
+    if ttft <= 0 or completion_tokens <= 1 or latency < ttft:
+        return None
+    return (latency - ttft) / (completion_tokens - 1)
+
+
+def calculate_tpot_statistics(tpots):
+    """Aggregate TPOT samples using the same NumPy definitions as bench_serving."""
+    if not tpots:
+        return {
+            "average_tpot": 0.0,
+            "p90_tpot": 0.0,
+            "p99_tpot": 0.0,
+            "median_tpot": 0.0,
+            "max_tpot": 0.0,
+        }
+    return {
+        "average_tpot": float(np.mean(tpots)),
+        "p90_tpot": float(np.percentile(tpots, 90)),
+        "p99_tpot": float(np.percentile(tpots, 99)),
+        "median_tpot": float(np.median(tpots)),
+        "max_tpot": float(np.max(tpots)),
+    }
+
+
 async def async_request_sglang_generate(
     payload,
     url,
@@ -75,7 +110,10 @@ async def async_request_sglang_generate(
                     output.latency = latency
                     output.prompt_len = prompt_tokens
                     output.cached_tokens = cached_tokens
-                    output.generated_len = len(output.itl) + 1
+                    output.generated_len = len(all_output_ids)
+                    output.tpot = calculate_tpot(
+                        output.latency, output.ttft, output.generated_len
+                    )
                 else:
                     output.error = response.reason or ""
                     output.success = False
@@ -130,9 +168,10 @@ async def async_request_openai_chat_completions(
                             # Streaming token chunks
                             if data.get("choices"):
                                 raw_delta = data["choices"][0].get("delta")
-                                text = raw_delta.get("content", "") if raw_delta else ""
+                                output_delta = get_openai_chat_output_delta(raw_delta)
-                                if text:
+                                if output_delta:
-                                    generated_text += text
+                                    content = raw_delta.get("content") or ""
+                                    generated_text += content
                                     timestamp = time.perf_counter()
 
                                     if ttft == 0.0:
@@ -162,6 +201,9 @@ async def async_request_openai_chat_completions(
                     output.generated_len = (
                         completion_tokens if completion_tokens else len(output.itl) + 1
                     )
+                    output.tpot = calculate_tpot(
+                        output.latency, output.ttft, output.generated_len
+                    )
                 else:
                     output.error = response.reason or ""
                     output.success = False

test/registered/unit/test_cache_hit_kit_metrics.py

@@ -0,0 +1,120 @@
+import asyncio
+import json
+import unittest
+from unittest.mock import patch
+
+from sglang.test.ci.ci_register import register_cpu_ci
+from sglang.test.kits.cache_hit_kit import (
+    async_request_openai_chat_completions,
+    calculate_tpot,
+    calculate_tpot_statistics,
+    get_openai_chat_output_delta,
+)
+from sglang.test.test_utils import CustomTestCase
+
+register_cpu_ci(est_time=1, suite="base-a-test-cpu")
+
+
+class TestCacheHitKitMetrics(CustomTestCase):
+    def test_openai_chat_tpot_without_usage(self):
+        chunks = [
+            {"choices": [{"delta": {"reasoning_content": output}}]}
+            for output in ("a", "b", "c")
+        ]
+        content = [f"data: {json.dumps(chunk)}".encode("utf-8") for chunk in chunks] + [
+            b"data: [DONE]"
+        ]
+
+        class MockResponse:
+            status = 200
+            reason = ""
+
+            async def __aenter__(self):
+                return self
+
+            async def __aexit__(self, *args):
+                pass
+
+            @property
+            def content(self):
+                async def iterate():
+                    for chunk in content:
+                        yield chunk
+
+                return iterate()
+
+        class MockSession:
+            async def __aenter__(self):
+                return self
+
+            async def __aexit__(self, *args):
+                pass
+
+            def post(self, **kwargs):
+                return MockResponse()
+
+        timestamps = [0.0, 0.1, 0.1, 0.1, 0.2, 0.2, 0.3, 0.3, 0.4]
+        with (
+            patch(
+                "sglang.test.kits.cache_hit_kit.aiohttp.ClientSession",
+                return_value=MockSession(),
+            ),
+            patch(
+                "sglang.test.kits.cache_hit_kit.time.perf_counter",
+                side_effect=timestamps,
+            ),
+        ):
+            output = asyncio.run(
+                async_request_openai_chat_completions({}, "http://test")
+            )
+
+        self.assertTrue(output.success)
+        self.assertEqual(output.generated_len, 3)
+        self.assertAlmostEqual(output.tpot, 0.15)
+
+    def test_openai_chat_output_delta(self):
+        self.assertEqual(get_openai_chat_output_delta({"content": "answer"}), "answer")
+        self.assertEqual(
+            get_openai_chat_output_delta({"reasoning_content": "think"}), "think"
+        )
+        self.assertEqual(get_openai_chat_output_delta({"reasoning": "think"}), "think")
+        self.assertEqual(
+            get_openai_chat_output_delta(
+                {"reasoning_content": "think", "content": "answer"}
+            ),
+            "thinkanswer",
+        )
+        self.assertEqual(get_openai_chat_output_delta({"role": "assistant"}), "")
+        self.assertEqual(get_openai_chat_output_delta(None), "")
+
+    def test_calculate_tpot(self):
+        self.assertAlmostEqual(calculate_tpot(1.1, 0.1, 101), 0.01)
+        self.assertIsNone(calculate_tpot(1.1, 0.0, 101))
+        self.assertIsNone(calculate_tpot(1.1, 0.1, 0))
+        self.assertIsNone(calculate_tpot(1.1, 0.1, 1))
+        self.assertIsNone(calculate_tpot(0.05, 0.1, 101))
+
+    def test_calculate_tpot_statistics(self):
+        stats = calculate_tpot_statistics([0.0024, 0.0025, 0.0026, 0.0035])
+
+        self.assertAlmostEqual(stats["average_tpot"], 0.00275)
+        self.assertAlmostEqual(stats["p90_tpot"], 0.00323)
+        self.assertAlmostEqual(stats["p99_tpot"], 0.003473)
+        self.assertAlmostEqual(stats["median_tpot"], 0.00255)
+        self.assertAlmostEqual(stats["max_tpot"], 0.0035)
+
+    def test_calculate_tpot_statistics_empty(self):
+        self.assertEqual(
+            calculate_tpot_statistics([]),
+            {
+                "average_tpot": 0.0,
+                "p90_tpot": 0.0,
+                "p99_tpot": 0.0,
+                "median_tpot": 0.0,
+                "max_tpot": 0.0,
+            },
+        )
+
+
+if __name__ == "__main__":
+    unittest.main()