[Test] Split the serving perf tests by topic into basic_perf/ and route their thresholds through a kit (#40505)

2026-09-21 10:05:57 -07:00
parent 14e9c40a72
commit 800613a74b
21 changed files with 927 additions and 740 deletions
@@ -0,0 +1,61 @@
+"""Latency and throughput of the /v1/embeddings endpoint."""
+
+import unittest
+
+from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci
+from sglang.test.kits.perf_bench_kit import (
+    at_least,
+    at_most,
+    check_batch_scaling,
+    check_perf,
+)
+from sglang.test.test_utils import (
+    DEFAULT_SMALL_EMBEDDING_MODEL_NAME_FOR_TEST,
+    CustomTestCase,
+    run_embeddings_benchmark,
+    run_embeddings_benchmark_multi,
+)
+
+register_cuda_ci(est_time=245, stage="extra-a", runner_config="1-gpu-large")
+register_amd_ci(est_time=240, suite="stage-b-test-1-gpu-large-amd")
+
+
+class TestEmbeddingsAPI(CustomTestCase):
+    def test_embeddings_api_latency_throughput(self):
+        res = run_embeddings_benchmark(
+            model=DEFAULT_SMALL_EMBEDDING_MODEL_NAME_FOR_TEST,
+            num_requests=1000,
+            batch_size=1,
+            input_tokens=500,
+            other_server_args=[],
+            need_warmup=True,
+        )
+
+        self.assertEqual(res["successful_requests"], res["total_requests"])
+        check_perf(
+            self,
+            at_most("avg_latency_ms", res["avg_latency_ms"], 21, amd=35, unit="ms"),
+            at_most("p95_latency_ms", res["p95_latency_ms"], 26, amd=40, unit="ms"),
+            at_least("throughput", res["throughput"], 48, amd=30, unit="req/s"),
+        )
+
+    def test_embeddings_api_batch_scaling(self):
+        check_batch_scaling(
+            self,
+            lambda batch_sizes: run_embeddings_benchmark_multi(
+                DEFAULT_SMALL_EMBEDDING_MODEL_NAME_FOR_TEST,
+                batch_sizes,
+                num_requests=500,
+                input_tokens=500,
+            ),
+            # batch size, avg ms, p95 ms, then the same two relaxed for mi300x
+            [
+                (10, 43, 49, 80, 90),
+                (25, 70, 78, 140, 150),
+                (50, 122, 158, 230, 240),
+            ],
+        )
+
+
+if __name__ == "__main__":
+    unittest.main()