From 1ed68220394444d336a50e0401a3dcf3dd3d434a Mon Sep 17 00:00:00 2001 From: Liangsheng Yin Date: Mon, 21 Sep 2026 13:02:10 -0700 Subject: [PATCH] [Test] Anchor `basic_perf` thresholds to each metric's measured spread (#40617) --- test/registered/basic_perf/test_embeddings_api.py | 12 ++++++------ test/registered/basic_perf/test_lora_latency.py | 4 ++-- test/registered/basic_perf/test_moe_throughput.py | 2 +- test/registered/basic_perf/test_pp_throughput.py | 2 +- test/registered/basic_perf/test_score_api.py | 8 ++++---- test/registered/basic_perf/test_serving_latency.py | 4 ++-- .../registered/basic_perf/test_serving_throughput.py | 4 ++-- test/registered/basic_perf/test_vlm_serving_fa3.py | 4 ++-- .../basic_perf/test_vlm_serving_flashinfer.py | 8 ++++---- 9 files changed, 24 insertions(+), 24 deletions(-) diff --git a/test/registered/basic_perf/test_embeddings_api.py b/test/registered/basic_perf/test_embeddings_api.py index 22bd51871..4e81fbf06 100644 --- a/test/registered/basic_perf/test_embeddings_api.py +++ b/test/registered/basic_perf/test_embeddings_api.py @@ -34,9 +34,9 @@ class TestEmbeddingsAPI(CustomTestCase): self.assertEqual(res["successful_requests"], res["total_requests"]) check_perf( self, - at_most("avg_latency_ms", res["avg_latency_ms"], 21, amd=35, unit="ms"), - at_most("p95_latency_ms", res["p95_latency_ms"], 26, amd=40, unit="ms"), - at_least("throughput", res["throughput"], 48, amd=30, unit="req/s"), + at_most("avg_latency_ms", res["avg_latency_ms"], 23, amd=35, unit="ms"), + at_most("p95_latency_ms", res["p95_latency_ms"], 34, amd=40, unit="ms"), + at_least("throughput", res["throughput"], 45, amd=30, unit="req/s"), ) def test_embeddings_api_batch_scaling(self): @@ -50,9 +50,9 @@ class TestEmbeddingsAPI(CustomTestCase): ), # batch size, avg ms, p95 ms, then the same two relaxed for mi300x [ - (10, 43, 49, 80, 90), - (25, 70, 78, 140, 150), - (50, 122, 158, 230, 240), + (10, 44, 52, 80, 90), + (25, 72, 101, 140, 150), + (50, 126, 200, 230, 240), ], ) diff --git a/test/registered/basic_perf/test_lora_latency.py b/test/registered/basic_perf/test_lora_latency.py index 249e11046..02eec8cc3 100644 --- a/test/registered/basic_perf/test_lora_latency.py +++ b/test/registered/basic_perf/test_lora_latency.py @@ -32,7 +32,7 @@ class TestLoRALatency(CustomTestCase): unit="ms", ), # mi300x is about twice as slow as mi325 on LoRA TTFT. - at_most("median_ttft_ms", res["median_ttft_ms"], 51, amd=100, unit="ms"), + at_most("median_ttft_ms", res["median_ttft_ms"], 52, amd=100, unit="ms"), ) def test_online_lora_latency_with_concurrent_adapter_updates(self): @@ -43,7 +43,7 @@ class TestLoRALatency(CustomTestCase): at_most( "median_e2e_latency_ms", res["median_e2e_latency_ms"], - 3170, + 3420, amd=6000, unit="ms", ), diff --git a/test/registered/basic_perf/test_moe_throughput.py b/test/registered/basic_perf/test_moe_throughput.py index f4a881ba6..c86befa15 100644 --- a/test/registered/basic_perf/test_moe_throughput.py +++ b/test/registered/basic_perf/test_moe_throughput.py @@ -29,7 +29,7 @@ class TestMoEThroughput(CustomTestCase): at_least( "output_throughput", res["output_throughput"], - 2670, + 2660, amd=2100, unit="token/s", ), diff --git a/test/registered/basic_perf/test_pp_throughput.py b/test/registered/basic_perf/test_pp_throughput.py index abf8d1e79..80355b5e7 100644 --- a/test/registered/basic_perf/test_pp_throughput.py +++ b/test/registered/basic_perf/test_pp_throughput.py @@ -60,7 +60,7 @@ class TestPPThroughput(CustomTestCase): "input_throughput", res["input_throughput"], 4380, - amd=3000, + amd=2190, unit="token/s", ), ) diff --git a/test/registered/basic_perf/test_score_api.py b/test/registered/basic_perf/test_score_api.py index 3c9da5f1c..10011da22 100644 --- a/test/registered/basic_perf/test_score_api.py +++ b/test/registered/basic_perf/test_score_api.py @@ -33,9 +33,9 @@ class TestScoreAPI(CustomTestCase): self.assertEqual(res["successful_requests"], res["total_requests"]) check_perf( self, - at_most("avg_latency_ms", res["avg_latency_ms"], 30, amd=60, unit="ms"), - at_most("p95_latency_ms", res["p95_latency_ms"], 32, amd=65, unit="ms"), - at_least("throughput", res["throughput"], 34, amd=16, unit="req/s"), + at_most("avg_latency_ms", res["avg_latency_ms"], 31, amd=60, unit="ms"), + at_most("p95_latency_ms", res["p95_latency_ms"], 37, amd=65, unit="ms"), + at_least("throughput", res["throughput"], 32, amd=16, unit="req/s"), ) def test_score_api_batch_scaling(self): @@ -47,7 +47,7 @@ class TestScoreAPI(CustomTestCase): num_requests=500, ), # batch size, avg ms, p95 ms, then the same two relaxed for mi300x - [(10, 30, 34, 60, 65), (25, 35, 39, 70, 80), (50, 51, 59, 80, 90)], + [(10, 32, 40, 60, 65), (25, 37, 42, 70, 80), (50, 54, 64, 80, 90)], ) diff --git a/test/registered/basic_perf/test_serving_latency.py b/test/registered/basic_perf/test_serving_latency.py index d612ef3c3..b6136d99c 100644 --- a/test/registered/basic_perf/test_serving_latency.py +++ b/test/registered/basic_perf/test_serving_latency.py @@ -28,10 +28,10 @@ class TestServingLatency(CustomTestCase): at_most( "median_e2e_latency_ms", res["median_e2e_latency_ms"], - 9100, + 9140, unit="ms", ), - at_most("median_ttft_ms", res["median_ttft_ms"], 80, amd=115, unit="ms"), + at_most("median_ttft_ms", res["median_ttft_ms"], 84, amd=115, unit="ms"), at_most("median_itl_ms", res["median_itl_ms"], 9, unit="ms"), ) diff --git a/test/registered/basic_perf/test_serving_throughput.py b/test/registered/basic_perf/test_serving_throughput.py index e8c1ca571..af2d47b8d 100644 --- a/test/registered/basic_perf/test_serving_throughput.py +++ b/test/registered/basic_perf/test_serving_throughput.py @@ -53,7 +53,7 @@ class TestServingThroughput(CustomTestCase): at_least( "output_throughput", res["output_throughput"], - 1110, + 1100, amd=1000, unit="token/s", ), @@ -96,7 +96,7 @@ class TestServingThroughput(CustomTestCase): at_least( "output_throughput", res["output_throughput"], - 4870, + 4860, amd=3500, unit="token/s", ), diff --git a/test/registered/basic_perf/test_vlm_serving_fa3.py b/test/registered/basic_perf/test_vlm_serving_fa3.py index fc2fb7101..2ac8cce5f 100644 --- a/test/registered/basic_perf/test_vlm_serving_fa3.py +++ b/test/registered/basic_perf/test_vlm_serving_fa3.py @@ -15,9 +15,9 @@ class TestVLMServingFa3(CustomTestCase): self, "fa3", # No offline bound: never measured on this lane. - output_throughput=16700, + output_throughput=15640, e2e_ms=11000, - ttft_ms=84, + ttft_ms=100, itl_ms=5.2, ) diff --git a/test/registered/basic_perf/test_vlm_serving_flashinfer.py b/test/registered/basic_perf/test_vlm_serving_flashinfer.py index 6d5af9741..6ea0bb40b 100644 --- a/test/registered/basic_perf/test_vlm_serving_flashinfer.py +++ b/test/registered/basic_perf/test_vlm_serving_flashinfer.py @@ -14,10 +14,10 @@ class TestVLMServingFlashinfer(CustomTestCase): check_vlm_serving_perf( self, "flashinfer", - output_throughput=6900, - e2e_ms=17300, - ttft_ms=76, - itl_ms=8.3, + output_throughput=5940, + e2e_ms=17480, + ttft_ms=83, + itl_ms=8.4, )