minke.yu/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 13 Packages Projects Releases Wiki Activity

Fix reasoning metrics and add TPOT to bench_multiturn (#35443)

Browse Source
This commit is contained in:
Wes
2026-09-02 11:28:47 +08:00
committed by GitHub
parent 0157f1f552
commit 2d9c64394f
4 changed files with 179 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files
benchmark/hicache/bench_multiturn.py
+12
View File
@@ -17,6 +17,7 @@ from sglang.benchmark.utils import get_tokenizer
from sglang.test.kits.cache_hit_kit import (
async_request_openai_chat_completions,
async_request_sglang_generate,
calculate_tpot_statistics,
gen_payload,
gen_payload_openai,
)
@@ -362,6 +363,7 @@ class WorkloadGenerator:
self.pbar = tqdm(total=self.total_requests)
self.performance_metrics = {
"ttft": [],
"tpot": [],
"itl": [],
"latency": [],
"prompt_len": [],
@@ -458,6 +460,8 @@ class WorkloadGenerator:
current_round = self.client_records[client_id]["round"]
self.client_records[client_id]["round"] += 1
self.performance_metrics["ttft"].append(response.ttft)
if response.tpot is not None:
self.performance_metrics["tpot"].append(response.tpot)
self.performance_metrics["itl"].extend(response.itl)
self.performance_metrics["latency"].append(response.latency)
self.performance_metrics["prompt_len"].append(response.prompt_len)
@@ -582,6 +586,8 @@ class WorkloadGenerator:
def max_or_zero(sorted_vals):
return sorted_vals[-1] if sorted_vals else 0.0
tpot_statistics = calculate_tpot_statistics(self.performance_metrics["tpot"])
performance_data = {
"summary": {
"total_requests": len(self.performance_metrics["ttft"]),
@@ -608,6 +614,7 @@ class WorkloadGenerator:
"p99_ttft": percentile(sorted_ttft, 0.99),
"median_ttft": percentile(sorted_ttft, 0.5),
"max_ttft": max_or_zero(sorted_ttft),
**tpot_statistics,
"average_itl": (
sum(self.performance_metrics["itl"])
/ len(self.performance_metrics["itl"])
@@ -686,6 +693,11 @@ class WorkloadGenerator:
print(f" P99 TTFT: {performance_data['summary']['p99_ttft']:.2f}")
print(f" Median TTFT: {performance_data['summary']['median_ttft']:.2f}")
print(f" Max TTFT: {performance_data['summary']['max_ttft']:.2f}")
print(f" Average TPOT: {performance_data['summary']['average_tpot']:.4f}")
print(f" P90 TPOT: {performance_data['summary']['p90_tpot']:.4f}")
print(f" P99 TPOT: {performance_data['summary']['p99_tpot']:.4f}")
print(f" Median TPOT: {performance_data['summary']['median_tpot']:.4f}")
print(f" Max TPOT: {performance_data['summary']['max_tpot']:.4f}")
print(f" Average ITL: {performance_data['summary']['average_itl']:.4f}")
print(f" P90 ITL: {performance_data['summary']['p90_itl']:.4f}")
print(f" P99 ITL: {performance_data['summary']['p99_itl']:.4f}")
python/sglang/benchmark/serving.py
+1
View File
@@ -101,6 +101,7 @@ class RequestFuncOutput:
success: bool = False
latency: float = 0.0
ttft: float = 0.0 # Time to first token
tpot: Optional[float] = None # Time per output token
itl: List[float] = field(default_factory=list) # List of inter-token latencies
text_chunks: List[str] = field(default_factory=list)
prompt_len: int = 0
python/sglang/test/kits/cache_hit_kit.py
+46 -4
View File
@@ -3,6 +3,7 @@ import json
import time
import aiohttp
import numpy as np
import requests
from sglang.benchmark.datasets.random import sample_random_requests
@@ -12,6 +13,40 @@ from sglang.benchmark.utils import get_tokenizer, remove_prefix
AIOHTTP_TIMEOUT = aiohttp.ClientTimeout(total=20 * 60 * 60)
def get_openai_chat_output_delta(delta):
"""Return text emitted by an OpenAI-compatible chat streaming delta."""
if not delta:
return ""
reasoning = delta.get("reasoning_content") or delta.get("reasoning") or ""
return reasoning + (delta.get("content") or "")
def calculate_tpot(latency, ttft, completion_tokens):
"""Calculate request-level time per output token when inputs are valid."""
if ttft <= 0 or completion_tokens <= 1 or latency < ttft:
return None
return (latency - ttft) / (completion_tokens - 1)
def calculate_tpot_statistics(tpots):
"""Aggregate TPOT samples using the same NumPy definitions as bench_serving."""
if not tpots:
return {
"average_tpot": 0.0,
"p90_tpot": 0.0,
"p99_tpot": 0.0,
"median_tpot": 0.0,
"max_tpot": 0.0,
}
return {
"average_tpot": float(np.mean(tpots)),
"p90_tpot": float(np.percentile(tpots, 90)),
"p99_tpot": float(np.percentile(tpots, 99)),
"median_tpot": float(np.median(tpots)),
"max_tpot": float(np.max(tpots)),
}
async def async_request_sglang_generate(
payload,
url,
@@ -75,7 +110,10 @@ async def async_request_sglang_generate(
output.latency = latency
output.prompt_len = prompt_tokens
output.cached_tokens = cached_tokens
output.generated_len = len(output.itl) + 1
output.generated_len = len(all_output_ids)
output.tpot = calculate_tpot(
output.latency, output.ttft, output.generated_len
)
else:
output.error = response.reason or ""
output.success = False
@@ -130,9 +168,10 @@ async def async_request_openai_chat_completions(
# Streaming token chunks
if data.get("choices"):
raw_delta = data["choices"][0].get("delta")
text = raw_delta.get("content", "") if raw_delta else ""
if text:
generated_text += text
output_delta = get_openai_chat_output_delta(raw_delta)
if output_delta:
content = raw_delta.get("content") or ""
generated_text += content
timestamp = time.perf_counter()
if ttft == 0.0:
@@ -162,6 +201,9 @@ async def async_request_openai_chat_completions(
output.generated_len = (
completion_tokens if completion_tokens else len(output.itl) + 1
)
output.tpot = calculate_tpot(
output.latency, output.ttft, output.generated_len
)
else:
output.error = response.reason or ""
output.success = False
test/registered/unit/test_cache_hit_kit_metrics.py
+120
View File
@@ -0,0 +1,120 @@
import asyncio
import json
import unittest
from unittest.mock import patch
from sglang.test.ci.ci_register import register_cpu_ci
from sglang.test.kits.cache_hit_kit import (
async_request_openai_chat_completions,
calculate_tpot,
calculate_tpot_statistics,
get_openai_chat_output_delta,
)
from sglang.test.test_utils import CustomTestCase
register_cpu_ci(est_time=1, suite="base-a-test-cpu")
class TestCacheHitKitMetrics(CustomTestCase):
def test_openai_chat_tpot_without_usage(self):
chunks = [
{"choices": [{"delta": {"reasoning_content": output}}]}
for output in ("a", "b", "c")
]
content = [f"data: {json.dumps(chunk)}".encode("utf-8") for chunk in chunks] + [
b"data: [DONE]"
]
class MockResponse:
status = 200
reason = ""
async def __aenter__(self):
return self
async def __aexit__(self, *args):
pass
@property
def content(self):
async def iterate():
for chunk in content:
yield chunk
return iterate()
class MockSession:
async def __aenter__(self):
return self
async def __aexit__(self, *args):
pass
def post(self, **kwargs):
return MockResponse()
timestamps = [0.0, 0.1, 0.1, 0.1, 0.2, 0.2, 0.3, 0.3, 0.4]
with (
patch(
"sglang.test.kits.cache_hit_kit.aiohttp.ClientSession",
return_value=MockSession(),
),
patch(
"sglang.test.kits.cache_hit_kit.time.perf_counter",
side_effect=timestamps,
),
):
output = asyncio.run(
async_request_openai_chat_completions({}, "http://test")
)
self.assertTrue(output.success)
self.assertEqual(output.generated_len, 3)
self.assertAlmostEqual(output.tpot, 0.15)
def test_openai_chat_output_delta(self):
self.assertEqual(get_openai_chat_output_delta({"content": "answer"}), "answer")
self.assertEqual(
get_openai_chat_output_delta({"reasoning_content": "think"}), "think"
)
self.assertEqual(get_openai_chat_output_delta({"reasoning": "think"}), "think")
self.assertEqual(
get_openai_chat_output_delta(
{"reasoning_content": "think", "content": "answer"}
),
"thinkanswer",
)
self.assertEqual(get_openai_chat_output_delta({"role": "assistant"}), "")
self.assertEqual(get_openai_chat_output_delta(None), "")
def test_calculate_tpot(self):
self.assertAlmostEqual(calculate_tpot(1.1, 0.1, 101), 0.01)
self.assertIsNone(calculate_tpot(1.1, 0.0, 101))
self.assertIsNone(calculate_tpot(1.1, 0.1, 0))
self.assertIsNone(calculate_tpot(1.1, 0.1, 1))
self.assertIsNone(calculate_tpot(0.05, 0.1, 101))
def test_calculate_tpot_statistics(self):
stats = calculate_tpot_statistics([0.0024, 0.0025, 0.0026, 0.0035])
self.assertAlmostEqual(stats["average_tpot"], 0.00275)
self.assertAlmostEqual(stats["p90_tpot"], 0.00323)
self.assertAlmostEqual(stats["p99_tpot"], 0.003473)
self.assertAlmostEqual(stats["median_tpot"], 0.00255)
self.assertAlmostEqual(stats["max_tpot"], 0.0035)
def test_calculate_tpot_statistics_empty(self):
self.assertEqual(
calculate_tpot_statistics([]),
{
"average_tpot": 0.0,
"p90_tpot": 0.0,
"p99_tpot": 0.0,
"median_tpot": 0.0,
"max_tpot": 0.0,
},
)
if __name__ == "__main__":
unittest.main()