[Test] Anchor basic_perf thresholds to each metric's measured spread (#40617)

test/registered/basic_perf/test_embeddings_api.py

@@ -34,9 +34,9 @@ class TestEmbeddingsAPI(CustomTestCase):
         self.assertEqual(res["successful_requests"], res["total_requests"])
         check_perf(
             self,
-            at_most("avg_latency_ms", res["avg_latency_ms"], 21, amd=35, unit="ms"),
+            at_most("avg_latency_ms", res["avg_latency_ms"], 23, amd=35, unit="ms"),
-            at_most("p95_latency_ms", res["p95_latency_ms"], 26, amd=40, unit="ms"),
+            at_most("p95_latency_ms", res["p95_latency_ms"], 34, amd=40, unit="ms"),
-            at_least("throughput", res["throughput"], 48, amd=30, unit="req/s"),
+            at_least("throughput", res["throughput"], 45, amd=30, unit="req/s"),
         )
 
     def test_embeddings_api_batch_scaling(self):
@@ -50,9 +50,9 @@ class TestEmbeddingsAPI(CustomTestCase):
             ),
             # batch size, avg ms, p95 ms, then the same two relaxed for mi300x
             [
-                (10, 43, 49, 80, 90),
+                (10, 44, 52, 80, 90),
-                (25, 70, 78, 140, 150),
+                (25, 72, 101, 140, 150),
-                (50, 122, 158, 230, 240),
+                (50, 126, 200, 230, 240),
             ],
         )
 

test/registered/basic_perf/test_lora_latency.py

@@ -32,7 +32,7 @@ class TestLoRALatency(CustomTestCase):
                 unit="ms",
             ),
             # mi300x is about twice as slow as mi325 on LoRA TTFT.
-            at_most("median_ttft_ms", res["median_ttft_ms"], 51, amd=100, unit="ms"),
+            at_most("median_ttft_ms", res["median_ttft_ms"], 52, amd=100, unit="ms"),
         )
 
     def test_online_lora_latency_with_concurrent_adapter_updates(self):
@@ -43,7 +43,7 @@ class TestLoRALatency(CustomTestCase):
             at_most(
                 "median_e2e_latency_ms",
                 res["median_e2e_latency_ms"],
-                3170,
+                3420,
                 amd=6000,
                 unit="ms",
             ),

test/registered/basic_perf/test_moe_throughput.py

@@ -29,7 +29,7 @@ class TestMoEThroughput(CustomTestCase):
             at_least(
                 "output_throughput",
                 res["output_throughput"],
-                2670,
+                2660,
                 amd=2100,
                 unit="token/s",
             ),

test/registered/basic_perf/test_pp_throughput.py

@@ -60,7 +60,7 @@ class TestPPThroughput(CustomTestCase):
                 "input_throughput",
                 res["input_throughput"],
                 4380,
-                amd=3000,
+                amd=2190,
                 unit="token/s",
             ),
         )

test/registered/basic_perf/test_score_api.py

@@ -33,9 +33,9 @@ class TestScoreAPI(CustomTestCase):
         self.assertEqual(res["successful_requests"], res["total_requests"])
         check_perf(
             self,
-            at_most("avg_latency_ms", res["avg_latency_ms"], 30, amd=60, unit="ms"),
+            at_most("avg_latency_ms", res["avg_latency_ms"], 31, amd=60, unit="ms"),
-            at_most("p95_latency_ms", res["p95_latency_ms"], 32, amd=65, unit="ms"),
+            at_most("p95_latency_ms", res["p95_latency_ms"], 37, amd=65, unit="ms"),
-            at_least("throughput", res["throughput"], 34, amd=16, unit="req/s"),
+            at_least("throughput", res["throughput"], 32, amd=16, unit="req/s"),
         )
 
     def test_score_api_batch_scaling(self):
@@ -47,7 +47,7 @@ class TestScoreAPI(CustomTestCase):
                 num_requests=500,
             ),
             # batch size, avg ms, p95 ms, then the same two relaxed for mi300x
-            [(10, 30, 34, 60, 65), (25, 35, 39, 70, 80), (50, 51, 59, 80, 90)],
+            [(10, 32, 40, 60, 65), (25, 37, 42, 70, 80), (50, 54, 64, 80, 90)],
         )
 
 

test/registered/basic_perf/test_serving_latency.py

@@ -28,10 +28,10 @@ class TestServingLatency(CustomTestCase):
             at_most(
                 "median_e2e_latency_ms",
                 res["median_e2e_latency_ms"],
-                9100,
+                9140,
                 unit="ms",
             ),
-            at_most("median_ttft_ms", res["median_ttft_ms"], 80, amd=115, unit="ms"),
+            at_most("median_ttft_ms", res["median_ttft_ms"], 84, amd=115, unit="ms"),
             at_most("median_itl_ms", res["median_itl_ms"], 9, unit="ms"),
         )
 

test/registered/basic_perf/test_serving_throughput.py

@@ -53,7 +53,7 @@ class TestServingThroughput(CustomTestCase):
             at_least(
                 "output_throughput",
                 res["output_throughput"],
-                1110,
+                1100,
                 amd=1000,
                 unit="token/s",
             ),
@@ -96,7 +96,7 @@ class TestServingThroughput(CustomTestCase):
             at_least(
                 "output_throughput",
                 res["output_throughput"],
-                4870,
+                4860,
                 amd=3500,
                 unit="token/s",
             ),

test/registered/basic_perf/test_vlm_serving_fa3.py

@@ -15,9 +15,9 @@ class TestVLMServingFa3(CustomTestCase):
             self,
             "fa3",
             # No offline bound: never measured on this lane.
-            output_throughput=16700,
+            output_throughput=15640,
             e2e_ms=11000,
-            ttft_ms=84,
+            ttft_ms=100,
             itl_ms=5.2,
         )
 

test/registered/basic_perf/test_vlm_serving_flashinfer.py

@@ -14,10 +14,10 @@ class TestVLMServingFlashinfer(CustomTestCase):
         check_vlm_serving_perf(
             self,
             "flashinfer",
-            output_throughput=6900,
+            output_throughput=5940,
-            e2e_ms=17300,
+            e2e_ms=17480,
-            ttft_ms=76,
+            ttft_ms=83,
-            itl_ms=8.3,
+            itl_ms=8.4,
         )