minke.yu/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 13 Packages Projects Releases Wiki Activity

[AMD] prepare for MI300x PR runner pool: registry mirror, runner routing, threshold tuning (#23156)

Browse Source
This commit is contained in:
Bingxu Chen
2026-04-21 00:58:23 -07:00
committed by GitHub
parent 74fdf9cd77
commit 09b1d10d59
14 changed files with 145 additions and 117 deletions
Download Patch File Download Diff File Expand all files Collapse all files
test/registered/amd/test_deepseek_v32_basic.py
+1 -1
View File
@@ -15,7 +15,7 @@ from sglang.test.test_utils import (
write_github_step_summary,
)
register_amd_ci(est_time=1800, suite="stage-c-test-large-8-gpu-amd")
register_amd_ci(est_time=3600, suite="stage-c-test-large-8-gpu-amd")
DEEPSEEK_V32_MODEL_PATH = "deepseek-ai/DeepSeek-V3.2"
test/registered/amd/test_deepseek_v3_mtp.py
+2 -1
View File
@@ -84,7 +84,8 @@ class TestDeepseekV3MTP(CustomTestCase):
f'{metrics["accuracy"]=:.3f}\n'
f"{avg_spec_accept_length=:.2f}\n"
)
self.assertGreater(metrics["accuracy"], 0.935)
# relax for mi300x
self.assertGreaterEqual(metrics["accuracy"], 0.93)
if is_in_amd_ci():
self.assertGreater(avg_spec_accept_length, 2.8)
else:
test/registered/amd/test_moriep_small.py
+1 -1
View File
@@ -60,7 +60,7 @@ common_args = [
"1",
"--enable-dp-lm-head",
"--mem-fraction-static",
"0.6",
"0.72", # relax for mi300x
"--chunked-prefill-size",
"32768",
"--max-running-requests",
test/registered/moe/test_torch_compile_moe.py
+4
View File
@@ -12,6 +12,7 @@ from sglang.test.test_utils import (
DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
DEFAULT_URL_FOR_TEST,
CustomTestCase,
is_in_amd_ci,
popen_launch_server,
)
@@ -73,6 +74,9 @@ class TestTorchCompileMoe(CustomTestCase):
throughput = max_tokens / (tok - tic)
if is_cuda():
self.assertGreaterEqual(throughput, 285)
elif is_in_amd_ci():
# relax for mi300x
self.assertGreaterEqual(throughput, 240)
else:
self.assertGreaterEqual(throughput, 270)
test/registered/perf/test_bench_serving_1gpu_part1.py
+10 -8
View File
@@ -142,9 +142,6 @@ class TestBenchServing1GPUPart1(CustomTestCase):
self.assertLess(res["median_itl_ms"], 10)
def test_online_lora_latency(self):
if is_in_amd_ci():
pass
res = self._run_lora_latency_test(enable_background_task=False)
if is_in_ci():
@@ -154,12 +151,13 @@ class TestBenchServing1GPUPart1(CustomTestCase):
f"median_ttft_ms: {res['median_ttft_ms']:.2f} ms\n"
)
self.assertLess(res["median_e2e_latency_ms"], 2400)
self.assertLess(res["median_ttft_ms"], 58)
# relax for mi300x (LoRA TTFT ~2x slower than mi325)
if is_in_amd_ci():
self.assertLess(res["median_ttft_ms"], 100)
else:
self.assertLess(res["median_ttft_ms"], 58)
def test_online_lora_latency_with_concurrent_adapter_updates(self):
if is_in_amd_ci():
pass
res = self._run_lora_latency_test(enable_background_task=True)
if is_in_ci():
@@ -169,7 +167,11 @@ class TestBenchServing1GPUPart1(CustomTestCase):
f"median_ttft_ms: {res['median_ttft_ms']:.2f} ms\n"
)
self.assertLess(res["median_e2e_latency_ms"], 4000)
self.assertLess(res["median_ttft_ms"], 80)
# relax for mi300x (LoRA TTFT ~2x slower than mi325)
if is_in_amd_ci():
self.assertLess(res["median_ttft_ms"], 130)
else:
self.assertLess(res["median_ttft_ms"], 80)
def _run_lora_latency_test(self, enable_background_task: bool):
"""
test/registered/perf/test_bench_serving_1gpu_part2.py
+31 -16
View File
@@ -41,8 +41,9 @@ class TestBenchServing1GPUPart2(CustomTestCase):
f"### test_vlm_offline_throughput\n"
f"Output throughput: {res['output_throughput']:.2f} token/s\n"
)
# relax for mi300x
if is_in_amd_ci():
self.assertGreater(res["output_throughput"], 2000)
self.assertGreater(res["output_throughput"], 900)
else:
self.assertGreater(res["output_throughput"], 2500)
@@ -116,12 +117,16 @@ class TestBenchServing1GPUPart2(CustomTestCase):
)
self.assertEqual(res["successful_requests"], res["total_requests"])
bounds = {
10: (45, 50),
25: (50, 60),
50: (60, 65),
}
avg_latency_bound, p95_latency_bound = bounds.get(batch_size, (60, 65))
# relax for mi300x
if is_in_amd_ci():
bounds = {10: (60, 65), 25: (70, 80), 50: (80, 90)}
default_bounds = (90, 90)
else:
bounds = {10: (45, 50), 25: (50, 60), 50: (60, 65)}
default_bounds = (60, 65)
avg_latency_bound, p95_latency_bound = bounds.get(
batch_size, default_bounds
)
self.assertLess(res["avg_latency_ms"], avg_latency_bound)
self.assertLess(res["p95_latency_ms"], p95_latency_bound)
@@ -146,9 +151,15 @@ class TestBenchServing1GPUPart2(CustomTestCase):
)
self.assertEqual(res["successful_requests"], res["total_requests"])
self.assertLess(res["avg_latency_ms"], 20)
self.assertLess(res["p95_latency_ms"], 25)
self.assertGreater(res["throughput"], 60)
# relax for mi300x
if is_in_amd_ci():
self.assertLess(res["avg_latency_ms"], 35)
self.assertLess(res["p95_latency_ms"], 40)
self.assertGreater(res["throughput"], 30)
else:
self.assertLess(res["avg_latency_ms"], 20)
self.assertLess(res["p95_latency_ms"], 25)
self.assertGreater(res["throughput"], 60)
def test_embeddings_api_batch_scaling(self):
"""Test embeddings API performance with different batch sizes"""
@@ -173,12 +184,16 @@ class TestBenchServing1GPUPart2(CustomTestCase):
)
self.assertEqual(res["successful_requests"], res["total_requests"])
bounds = {
10: (60, 65),
25: (115, 120),
50: (190, 195),
}
avg_latency_bound, p95_latency_bound = bounds.get(batch_size, (250, 250))
# relax for mi300x
if is_in_amd_ci():
bounds = {10: (80, 90), 25: (140, 150), 50: (230, 240)}
default_bounds = (300, 300)
else:
bounds = {10: (60, 65), 25: (115, 120), 50: (190, 195)}
default_bounds = (250, 250)
avg_latency_bound, p95_latency_bound = bounds.get(
batch_size, default_bounds
)
self.assertLess(res["avg_latency_ms"], avg_latency_bound)
self.assertLess(res["p95_latency_ms"], p95_latency_bound)
test/registered/sampling/test_pytorch_sampling_backend.py
+7
View File
@@ -11,6 +11,7 @@ from sglang.test.test_utils import (
DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
DEFAULT_URL_FOR_TEST,
CustomTestCase,
is_in_amd_ci,
popen_launch_server,
)
@@ -47,6 +48,12 @@ class TestPyTorchSamplingBackend(CustomTestCase):
metrics = run_eval(args)
self.assertGreaterEqual(metrics["score"], 0.65)
@unittest.skipIf(
is_in_amd_ci(),
"Skip on MI300x: greedy decode is not bit-exact across runs on MI300x "
"(kernel-level numerical jitter), so the assertEqual on identical "
"regenerated text is flaky on this runner pool.",
)
def test_greedy(self):
first_text = None
test/registered/tokenizer/test_multi_tokenizer.py
+3 -1
View File
@@ -10,6 +10,7 @@ from sglang.test.test_utils import (
CustomTestCase,
auto_config_device,
get_benchmark_args,
is_in_amd_ci,
is_in_ci,
popen_launch_server,
run_benchmark,
@@ -70,7 +71,8 @@ class TestMultiTokenizer(CustomTestCase, MMLUMixin):
f"median_e2e_latency_ms: {res['median_e2e_latency_ms']:.2f} ms\n"
)
self.assertLess(res["median_e2e_latency_ms"], 11000)
self.assertLess(res["median_ttft_ms"], 86)
# relax for mi300x
self.assertLess(res["median_ttft_ms"], 130 if is_in_amd_ci() else 86)
self.assertLess(res["median_itl_ms"], 10)