From 2d9c64394f5aa609f0d83b30e0a7f837722e91bf Mon Sep 17 00:00:00 2001 From: Wes Date: Wed, 2 Sep 2026 11:28:47 +0800 Subject: [PATCH] Fix reasoning metrics and add TPOT to bench_multiturn (#35443) --- benchmark/hicache/bench_multiturn.py | 12 ++ python/sglang/benchmark/serving.py | 1 + python/sglang/test/kits/cache_hit_kit.py | 50 +++++++- .../unit/test_cache_hit_kit_metrics.py | 120 ++++++++++++++++++ 4 files changed, 179 insertions(+), 4 deletions(-) create mode 100644 test/registered/unit/test_cache_hit_kit_metrics.py diff --git a/benchmark/hicache/bench_multiturn.py b/benchmark/hicache/bench_multiturn.py index 49483fe84..c1f1c3d03 100644 --- a/benchmark/hicache/bench_multiturn.py +++ b/benchmark/hicache/bench_multiturn.py @@ -17,6 +17,7 @@ from sglang.benchmark.utils import get_tokenizer from sglang.test.kits.cache_hit_kit import ( async_request_openai_chat_completions, async_request_sglang_generate, + calculate_tpot_statistics, gen_payload, gen_payload_openai, ) @@ -362,6 +363,7 @@ class WorkloadGenerator: self.pbar = tqdm(total=self.total_requests) self.performance_metrics = { "ttft": [], + "tpot": [], "itl": [], "latency": [], "prompt_len": [], @@ -458,6 +460,8 @@ class WorkloadGenerator: current_round = self.client_records[client_id]["round"] self.client_records[client_id]["round"] += 1 self.performance_metrics["ttft"].append(response.ttft) + if response.tpot is not None: + self.performance_metrics["tpot"].append(response.tpot) self.performance_metrics["itl"].extend(response.itl) self.performance_metrics["latency"].append(response.latency) self.performance_metrics["prompt_len"].append(response.prompt_len) @@ -582,6 +586,8 @@ class WorkloadGenerator: def max_or_zero(sorted_vals): return sorted_vals[-1] if sorted_vals else 0.0 + tpot_statistics = calculate_tpot_statistics(self.performance_metrics["tpot"]) + performance_data = { "summary": { "total_requests": len(self.performance_metrics["ttft"]), @@ -608,6 +614,7 @@ class WorkloadGenerator: "p99_ttft": percentile(sorted_ttft, 0.99), "median_ttft": percentile(sorted_ttft, 0.5), "max_ttft": max_or_zero(sorted_ttft), + **tpot_statistics, "average_itl": ( sum(self.performance_metrics["itl"]) / len(self.performance_metrics["itl"]) @@ -686,6 +693,11 @@ class WorkloadGenerator: print(f" P99 TTFT: {performance_data['summary']['p99_ttft']:.2f}") print(f" Median TTFT: {performance_data['summary']['median_ttft']:.2f}") print(f" Max TTFT: {performance_data['summary']['max_ttft']:.2f}") + print(f" Average TPOT: {performance_data['summary']['average_tpot']:.4f}") + print(f" P90 TPOT: {performance_data['summary']['p90_tpot']:.4f}") + print(f" P99 TPOT: {performance_data['summary']['p99_tpot']:.4f}") + print(f" Median TPOT: {performance_data['summary']['median_tpot']:.4f}") + print(f" Max TPOT: {performance_data['summary']['max_tpot']:.4f}") print(f" Average ITL: {performance_data['summary']['average_itl']:.4f}") print(f" P90 ITL: {performance_data['summary']['p90_itl']:.4f}") print(f" P99 ITL: {performance_data['summary']['p99_itl']:.4f}") diff --git a/python/sglang/benchmark/serving.py b/python/sglang/benchmark/serving.py index 39a716306..9ba76b1cb 100644 --- a/python/sglang/benchmark/serving.py +++ b/python/sglang/benchmark/serving.py @@ -101,6 +101,7 @@ class RequestFuncOutput: success: bool = False latency: float = 0.0 ttft: float = 0.0 # Time to first token + tpot: Optional[float] = None # Time per output token itl: List[float] = field(default_factory=list) # List of inter-token latencies text_chunks: List[str] = field(default_factory=list) prompt_len: int = 0 diff --git a/python/sglang/test/kits/cache_hit_kit.py b/python/sglang/test/kits/cache_hit_kit.py index c7202d833..280fb10fb 100644 --- a/python/sglang/test/kits/cache_hit_kit.py +++ b/python/sglang/test/kits/cache_hit_kit.py @@ -3,6 +3,7 @@ import json import time import aiohttp +import numpy as np import requests from sglang.benchmark.datasets.random import sample_random_requests @@ -12,6 +13,40 @@ from sglang.benchmark.utils import get_tokenizer, remove_prefix AIOHTTP_TIMEOUT = aiohttp.ClientTimeout(total=20 * 60 * 60) +def get_openai_chat_output_delta(delta): + """Return text emitted by an OpenAI-compatible chat streaming delta.""" + if not delta: + return "" + reasoning = delta.get("reasoning_content") or delta.get("reasoning") or "" + return reasoning + (delta.get("content") or "") + + +def calculate_tpot(latency, ttft, completion_tokens): + """Calculate request-level time per output token when inputs are valid.""" + if ttft <= 0 or completion_tokens <= 1 or latency < ttft: + return None + return (latency - ttft) / (completion_tokens - 1) + + +def calculate_tpot_statistics(tpots): + """Aggregate TPOT samples using the same NumPy definitions as bench_serving.""" + if not tpots: + return { + "average_tpot": 0.0, + "p90_tpot": 0.0, + "p99_tpot": 0.0, + "median_tpot": 0.0, + "max_tpot": 0.0, + } + return { + "average_tpot": float(np.mean(tpots)), + "p90_tpot": float(np.percentile(tpots, 90)), + "p99_tpot": float(np.percentile(tpots, 99)), + "median_tpot": float(np.median(tpots)), + "max_tpot": float(np.max(tpots)), + } + + async def async_request_sglang_generate( payload, url, @@ -75,7 +110,10 @@ async def async_request_sglang_generate( output.latency = latency output.prompt_len = prompt_tokens output.cached_tokens = cached_tokens - output.generated_len = len(output.itl) + 1 + output.generated_len = len(all_output_ids) + output.tpot = calculate_tpot( + output.latency, output.ttft, output.generated_len + ) else: output.error = response.reason or "" output.success = False @@ -130,9 +168,10 @@ async def async_request_openai_chat_completions( # Streaming token chunks if data.get("choices"): raw_delta = data["choices"][0].get("delta") - text = raw_delta.get("content", "") if raw_delta else "" - if text: - generated_text += text + output_delta = get_openai_chat_output_delta(raw_delta) + if output_delta: + content = raw_delta.get("content") or "" + generated_text += content timestamp = time.perf_counter() if ttft == 0.0: @@ -162,6 +201,9 @@ async def async_request_openai_chat_completions( output.generated_len = ( completion_tokens if completion_tokens else len(output.itl) + 1 ) + output.tpot = calculate_tpot( + output.latency, output.ttft, output.generated_len + ) else: output.error = response.reason or "" output.success = False diff --git a/test/registered/unit/test_cache_hit_kit_metrics.py b/test/registered/unit/test_cache_hit_kit_metrics.py new file mode 100644 index 000000000..00d336515 --- /dev/null +++ b/test/registered/unit/test_cache_hit_kit_metrics.py @@ -0,0 +1,120 @@ +import asyncio +import json +import unittest +from unittest.mock import patch + +from sglang.test.ci.ci_register import register_cpu_ci +from sglang.test.kits.cache_hit_kit import ( + async_request_openai_chat_completions, + calculate_tpot, + calculate_tpot_statistics, + get_openai_chat_output_delta, +) +from sglang.test.test_utils import CustomTestCase + +register_cpu_ci(est_time=1, suite="base-a-test-cpu") + + +class TestCacheHitKitMetrics(CustomTestCase): + def test_openai_chat_tpot_without_usage(self): + chunks = [ + {"choices": [{"delta": {"reasoning_content": output}}]} + for output in ("a", "b", "c") + ] + content = [f"data: {json.dumps(chunk)}".encode("utf-8") for chunk in chunks] + [ + b"data: [DONE]" + ] + + class MockResponse: + status = 200 + reason = "" + + async def __aenter__(self): + return self + + async def __aexit__(self, *args): + pass + + @property + def content(self): + async def iterate(): + for chunk in content: + yield chunk + + return iterate() + + class MockSession: + async def __aenter__(self): + return self + + async def __aexit__(self, *args): + pass + + def post(self, **kwargs): + return MockResponse() + + timestamps = [0.0, 0.1, 0.1, 0.1, 0.2, 0.2, 0.3, 0.3, 0.4] + with ( + patch( + "sglang.test.kits.cache_hit_kit.aiohttp.ClientSession", + return_value=MockSession(), + ), + patch( + "sglang.test.kits.cache_hit_kit.time.perf_counter", + side_effect=timestamps, + ), + ): + output = asyncio.run( + async_request_openai_chat_completions({}, "http://test") + ) + + self.assertTrue(output.success) + self.assertEqual(output.generated_len, 3) + self.assertAlmostEqual(output.tpot, 0.15) + + def test_openai_chat_output_delta(self): + self.assertEqual(get_openai_chat_output_delta({"content": "answer"}), "answer") + self.assertEqual( + get_openai_chat_output_delta({"reasoning_content": "think"}), "think" + ) + self.assertEqual(get_openai_chat_output_delta({"reasoning": "think"}), "think") + self.assertEqual( + get_openai_chat_output_delta( + {"reasoning_content": "think", "content": "answer"} + ), + "thinkanswer", + ) + self.assertEqual(get_openai_chat_output_delta({"role": "assistant"}), "") + self.assertEqual(get_openai_chat_output_delta(None), "") + + def test_calculate_tpot(self): + self.assertAlmostEqual(calculate_tpot(1.1, 0.1, 101), 0.01) + self.assertIsNone(calculate_tpot(1.1, 0.0, 101)) + self.assertIsNone(calculate_tpot(1.1, 0.1, 0)) + self.assertIsNone(calculate_tpot(1.1, 0.1, 1)) + self.assertIsNone(calculate_tpot(0.05, 0.1, 101)) + + def test_calculate_tpot_statistics(self): + stats = calculate_tpot_statistics([0.0024, 0.0025, 0.0026, 0.0035]) + + self.assertAlmostEqual(stats["average_tpot"], 0.00275) + self.assertAlmostEqual(stats["p90_tpot"], 0.00323) + self.assertAlmostEqual(stats["p99_tpot"], 0.003473) + self.assertAlmostEqual(stats["median_tpot"], 0.00255) + self.assertAlmostEqual(stats["max_tpot"], 0.0035) + + def test_calculate_tpot_statistics_empty(self): + self.assertEqual( + calculate_tpot_statistics([]), + { + "average_tpot": 0.0, + "p90_tpot": 0.0, + "p99_tpot": 0.0, + "median_tpot": 0.0, + "max_tpot": 0.0, + }, + ) + + +if __name__ == "__main__": + unittest.main()