diff --git a/python/sglang/test/gb300_utils.py b/python/sglang/test/gb300_utils.py new file mode 100644 index 000000000..59d4b06ed --- /dev/null +++ b/python/sglang/test/gb300_utils.py @@ -0,0 +1,6 @@ +"""Shared configuration for tests on the dedicated GB300 runner.""" + +# The runner's ephemeral client-port range is 10240-65535. Keep the TCPStore +# rendezvous below it so a recently closed client connection cannot make the +# subsequent TCPStore bind fail with EADDRINUSE. +GB300_NCCL_PORT = "10000" diff --git a/test/registered/attention/test_glm4_moe_lite_deterministic.py b/test/registered/attention/test_glm4_moe_lite_deterministic.py index 05f2005a6..3327a72cd 100644 --- a/test/registered/attention/test_glm4_moe_lite_deterministic.py +++ b/test/registered/attention/test_glm4_moe_lite_deterministic.py @@ -20,7 +20,7 @@ from sglang.test.test_deterministic_utils import ( ) from sglang.test.test_utils import DEFAULT_URL_FOR_TEST -register_cuda_ci(est_time=900, stage="nightly", runner_config="4-gpu-gb300") +register_cuda_ci(est_time=360, stage="nightly", runner_config="4-gpu-b200") GLM_MODEL = "zai-org/GLM-4.7-Flash" diff --git a/test/registered/gb300/test_deepseek_v4_pro_fp4.py b/test/registered/gb300/test_deepseek_v4_pro_fp4.py deleted file mode 100644 index 6cb02bede..000000000 --- a/test/registered/gb300/test_deepseek_v4_pro_fp4.py +++ /dev/null @@ -1,150 +0,0 @@ -import unittest - -from sglang.test.accuracy_test_runner import AccuracyTestParams -from sglang.test.ci.ci_register import register_cuda_ci -from sglang.test.performance_test_runner import PerformanceTestParams -from sglang.test.run_combined_tests import run_combined_tests -from sglang.test.test_utils import ModelLaunchSettings - -register_cuda_ci(est_time=7200, stage="nightly", runner_config="4-gpu-gb300") - -MODEL_PATH = "deepseek-ai/DeepSeek-V4-Pro" -SERVER_LAUNCH_TIMEOUT = 3600 - -DEEPEP_CONFIG = '{"normal_dispatch":{"num_sms":96},"normal_combine":{"num_sms":96}}' - -LOW_LATENCY_ARGS = [ - "--trust-remote-code", - "--moe-runner-backend", - "flashinfer_mxfp4", - "--speculative-algorithm", - "EAGLE", - "--speculative-num-steps", - "3", - "--speculative-eagle-topk", - "1", - "--speculative-num-draft-tokens", - "4", - "--chunked-prefill-size", - "8192", - "--disable-flashinfer-autotune", - "--swa-full-tokens-ratio", - "0.1", - "--mem-fraction-static", - "0.85", -] - -BALANCED_ARGS = [ - "--trust-remote-code", - "--dp", - "4", - "--enable-dp-attention", - "--moe-a2a-backend", - "deepep", - "--speculative-algorithm", - "EAGLE", - "--speculative-num-steps", - "1", - "--speculative-eagle-topk", - "1", - "--speculative-num-draft-tokens", - "2", - "--mem-fraction-static", - "0.85", - "--cuda-graph-max-bs-decode", - "128", - "--max-running-requests", - "256", - "--deepep-config", - DEEPEP_CONFIG, -] - -HIGH_THROUGHPUT_ARGS = [ - "--trust-remote-code", - "--dp", - "4", - "--enable-dp-attention", - "--moe-a2a-backend", - "megamoe", - "--mem-fraction-static", - "0.9", - "--cuda-graph-max-bs-decode", - "128", - "--max-running-requests", - "256", -] - -BALANCED_ENV = { - "SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK": "256", -} - -HIGH_THROUGHPUT_ENV = { - "SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK": "8320", -} - -PERFORMANCE_BATCH_SIZES = { - "low-latency": [1, 4, 16], - "balanced": [64], - "high-throughput": [128], -} - - -class TestDeepSeekV4ProFp4(unittest.TestCase): - """DeepSeek-V4-Pro FP4 on GB300 (4x B200 NVL4, tp=4).""" - - def test_deepseek_v4_pro_fp4(self): - variants = [ - ModelLaunchSettings( - MODEL_PATH, - tp_size=4, - extra_args=LOW_LATENCY_ARGS, - variant="low-latency", - launch_timeout=SERVER_LAUNCH_TIMEOUT, - ), - ModelLaunchSettings( - MODEL_PATH, - tp_size=4, - extra_args=BALANCED_ARGS, - env=BALANCED_ENV, - variant="balanced", - launch_timeout=SERVER_LAUNCH_TIMEOUT, - ), - ModelLaunchSettings( - MODEL_PATH, - tp_size=4, - extra_args=HIGH_THROUGHPUT_ARGS, - env=HIGH_THROUGHPUT_ENV, - variant="high-throughput", - launch_timeout=SERVER_LAUNCH_TIMEOUT, - ), - ] - - failures = [] - accuracy_params = AccuracyTestParams( - dataset="gsm8k", - baseline_accuracy=0.935, - temperature=1.0, - top_p=1.0, - ) - for variant in variants: - try: - run_combined_tests( - models=[variant], - test_name=f"DeepSeek-V4-Pro-FP4 ({variant.variant})", - accuracy_params=accuracy_params, - performance_params=PerformanceTestParams( - batch_sizes=PERFORMANCE_BATCH_SIZES[variant.variant], - result_dir="performance_results_gb300", - ), - ) - except AssertionError as e: - failures.append(f"{variant.variant}: {e}") - - if failures: - raise AssertionError( - "DeepSeek-V4-Pro-FP4 failures:\n" + "\n".join(failures) - ) - - -if __name__ == "__main__": - unittest.main() diff --git a/test/registered/gb300/test_deepseek_v4_pro_fp4_balanced.py b/test/registered/gb300/test_deepseek_v4_pro_fp4_balanced.py new file mode 100644 index 000000000..873d8c225 --- /dev/null +++ b/test/registered/gb300/test_deepseek_v4_pro_fp4_balanced.py @@ -0,0 +1,78 @@ +import unittest + +from sglang.test.accuracy_test_runner import AccuracyTestParams +from sglang.test.ci.ci_register import register_cuda_ci +from sglang.test.gb300_utils import GB300_NCCL_PORT +from sglang.test.performance_test_runner import PerformanceTestParams +from sglang.test.run_combined_tests import run_combined_tests +from sglang.test.test_utils import CustomTestCase, ModelLaunchSettings + +register_cuda_ci(est_time=600, stage="nightly", runner_config="4-gpu-gb300") + +MODEL_PATH = "deepseek-ai/DeepSeek-V4-Pro" +SERVER_LAUNCH_TIMEOUT = 3600 +DEEPEP_CONFIG = '{"normal_dispatch":{"num_sms":96},"normal_combine":{"num_sms":96}}' + +BALANCED_ARGS = [ + "--trust-remote-code", + "--dp", + "4", + "--enable-dp-attention", + "--moe-a2a-backend", + "deepep", + "--speculative-algorithm", + "EAGLE", + "--speculative-num-steps", + "1", + "--speculative-eagle-topk", + "1", + "--speculative-num-draft-tokens", + "2", + "--mem-fraction-static", + "0.85", + "--cuda-graph-max-bs-decode", + "128", + "--max-running-requests", + "256", + "--deepep-config", + DEEPEP_CONFIG, + "--nccl-port", + GB300_NCCL_PORT, +] + +BALANCED_ENV = { + "SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK": "256", +} + + +class TestDeepSeekV4ProFp4Balanced(CustomTestCase): + """DeepSeek-V4-Pro FP4 balanced config on GB300.""" + + def test_deepseek_v4_pro_fp4_balanced(self): + run_combined_tests( + models=[ + ModelLaunchSettings( + MODEL_PATH, + tp_size=4, + extra_args=BALANCED_ARGS, + env=BALANCED_ENV, + variant="balanced", + launch_timeout=SERVER_LAUNCH_TIMEOUT, + ) + ], + test_name="DeepSeek-V4-Pro-FP4 (balanced)", + accuracy_params=AccuracyTestParams( + dataset="gsm8k", + baseline_accuracy=0.935, + temperature=1.0, + top_p=1.0, + ), + performance_params=PerformanceTestParams( + batch_sizes=[64], + result_dir="performance_results_gb300", + ), + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/gb300/test_deepseek_v4_pro_fp4_high_throughput.py b/test/registered/gb300/test_deepseek_v4_pro_fp4_high_throughput.py new file mode 100644 index 000000000..800f35cc6 --- /dev/null +++ b/test/registered/gb300/test_deepseek_v4_pro_fp4_high_throughput.py @@ -0,0 +1,67 @@ +import unittest + +from sglang.test.accuracy_test_runner import AccuracyTestParams +from sglang.test.ci.ci_register import register_cuda_ci +from sglang.test.gb300_utils import GB300_NCCL_PORT +from sglang.test.performance_test_runner import PerformanceTestParams +from sglang.test.run_combined_tests import run_combined_tests +from sglang.test.test_utils import CustomTestCase, ModelLaunchSettings + +register_cuda_ci(est_time=600, stage="nightly", runner_config="4-gpu-gb300") + +MODEL_PATH = "deepseek-ai/DeepSeek-V4-Pro" +SERVER_LAUNCH_TIMEOUT = 3600 + +HIGH_THROUGHPUT_ARGS = [ + "--trust-remote-code", + "--dp", + "4", + "--enable-dp-attention", + "--moe-a2a-backend", + "megamoe", + "--mem-fraction-static", + "0.9", + "--cuda-graph-max-bs-decode", + "128", + "--max-running-requests", + "256", + "--nccl-port", + GB300_NCCL_PORT, +] + +HIGH_THROUGHPUT_ENV = { + "SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK": "8320", +} + + +class TestDeepSeekV4ProFp4HighThroughput(CustomTestCase): + """DeepSeek-V4-Pro FP4 high-throughput config on GB300.""" + + def test_deepseek_v4_pro_fp4_high_throughput(self): + run_combined_tests( + models=[ + ModelLaunchSettings( + MODEL_PATH, + tp_size=4, + extra_args=HIGH_THROUGHPUT_ARGS, + env=HIGH_THROUGHPUT_ENV, + variant="high-throughput", + launch_timeout=SERVER_LAUNCH_TIMEOUT, + ) + ], + test_name="DeepSeek-V4-Pro-FP4 (high-throughput)", + accuracy_params=AccuracyTestParams( + dataset="gsm8k", + baseline_accuracy=0.935, + temperature=1.0, + top_p=1.0, + ), + performance_params=PerformanceTestParams( + batch_sizes=[128], + result_dir="performance_results_gb300", + ), + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/gb300/test_deepseek_v4_pro_fp4_low_latency.py b/test/registered/gb300/test_deepseek_v4_pro_fp4_low_latency.py new file mode 100644 index 000000000..65e010b7d --- /dev/null +++ b/test/registered/gb300/test_deepseek_v4_pro_fp4_low_latency.py @@ -0,0 +1,68 @@ +import unittest + +from sglang.test.accuracy_test_runner import AccuracyTestParams +from sglang.test.ci.ci_register import register_cuda_ci +from sglang.test.gb300_utils import GB300_NCCL_PORT +from sglang.test.performance_test_runner import PerformanceTestParams +from sglang.test.run_combined_tests import run_combined_tests +from sglang.test.test_utils import CustomTestCase, ModelLaunchSettings + +register_cuda_ci(est_time=720, stage="nightly", runner_config="4-gpu-gb300") + +MODEL_PATH = "deepseek-ai/DeepSeek-V4-Pro" +SERVER_LAUNCH_TIMEOUT = 3600 + +LOW_LATENCY_ARGS = [ + "--trust-remote-code", + "--moe-runner-backend", + "flashinfer_mxfp4", + "--speculative-algorithm", + "EAGLE", + "--speculative-num-steps", + "3", + "--speculative-eagle-topk", + "1", + "--speculative-num-draft-tokens", + "4", + "--chunked-prefill-size", + "8192", + "--disable-flashinfer-autotune", + "--swa-full-tokens-ratio", + "0.1", + "--mem-fraction-static", + "0.85", + "--nccl-port", + GB300_NCCL_PORT, +] + + +class TestDeepSeekV4ProFp4LowLatency(CustomTestCase): + """DeepSeek-V4-Pro FP4 low-latency config on GB300.""" + + def test_deepseek_v4_pro_fp4_low_latency(self): + run_combined_tests( + models=[ + ModelLaunchSettings( + MODEL_PATH, + tp_size=4, + extra_args=LOW_LATENCY_ARGS, + variant="low-latency", + launch_timeout=SERVER_LAUNCH_TIMEOUT, + ) + ], + test_name="DeepSeek-V4-Pro-FP4 (low-latency)", + accuracy_params=AccuracyTestParams( + dataset="gsm8k", + baseline_accuracy=0.935, + temperature=1.0, + top_p=1.0, + ), + performance_params=PerformanceTestParams( + batch_sizes=[1, 4, 16], + result_dir="performance_results_gb300", + ), + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/gb300/test_glm52_nvfp4.py b/test/registered/gb300/test_glm52_nvfp4.py index d193659e3..0be117022 100644 --- a/test/registered/gb300/test_glm52_nvfp4.py +++ b/test/registered/gb300/test_glm52_nvfp4.py @@ -2,9 +2,10 @@ import unittest from sglang.test.accuracy_test_runner import AccuracyTestParams from sglang.test.ci.ci_register import register_cuda_ci +from sglang.test.gb300_utils import GB300_NCCL_PORT from sglang.test.performance_test_runner import PerformanceTestParams from sglang.test.run_combined_tests import run_combined_tests -from sglang.test.test_utils import ModelLaunchSettings +from sglang.test.test_utils import CustomTestCase, ModelLaunchSettings register_cuda_ci(est_time=2280, stage="nightly", runner_config="4-gpu-gb300") @@ -18,6 +19,8 @@ COMMON_ARGS = [ "--moe-runner-backend=flashinfer_trtllm", "--mem-fraction-static=0.9", "--enable-metrics", + "--nccl-port", + GB300_NCCL_PORT, ] TP_MTP_ARGS = [ @@ -35,7 +38,7 @@ DP_MTP_ARGS = [ ] -class TestGlm52Nvfp4(unittest.TestCase): +class TestGlm52Nvfp4(CustomTestCase): """GLM-5.2 NVFP4 on GB300 (4x GB300 NVL4, tp=4).""" def test_glm52_nvfp4(self): diff --git a/test/registered/gb300/test_kimi_k25_nvfp4.py b/test/registered/gb300/test_kimi_k25_nvfp4.py deleted file mode 100644 index 95512d9a6..000000000 --- a/test/registered/gb300/test_kimi_k25_nvfp4.py +++ /dev/null @@ -1,100 +0,0 @@ -import unittest - -from sglang.test.accuracy_test_runner import AccuracyTestParams -from sglang.test.ci.ci_register import register_cuda_ci -from sglang.test.performance_test_runner import PerformanceTestParams -from sglang.test.run_combined_tests import run_combined_tests -from sglang.test.test_utils import ModelLaunchSettings - -register_cuda_ci(est_time=7200, stage="nightly", runner_config="4-gpu-gb300") - -MODEL_PATH = "nvidia/Kimi-K2.5-NVFP4" -DRAFT_MODEL_PATH = "lightseekorg/kimi-k2.5-eagle3-mla" - -COMMON_ARGS = [ - "--trust-remote-code", - "--reasoning-parser=kimi_k2", - "--tool-call-parser=kimi_k2", - "--quantization=modelopt_fp4", - "--attention-backend=tokenspeed_mla", - "--kv-cache-dtype=fp8_e4m3", - "--moe-runner-backend=flashinfer_trtllm", - "--mem-fraction-static=0.8", - "--enable-metrics", - "--speculative-algorithm=EAGLE3", - f"--speculative-draft-model-path={DRAFT_MODEL_PATH}", - "--speculative-draft-model-quantization=unquant", -] - -TP_EAGLE_ARGS = [ - "--speculative-num-steps=3", - "--speculative-eagle-topk=1", - "--speculative-num-draft-tokens=4", -] - -DP_EAGLE_ARGS = [ - "--speculative-num-steps=1", - "--speculative-eagle-topk=1", - "--speculative-num-draft-tokens=2", -] - -PERFORMANCE_BATCH_SIZES = { - "TP4+EAGLE3": [1, 8], - "TP4+DP4+DPA+EAGLE3": [16], -} - - -class TestKimiK25Nvfp4(unittest.TestCase): - """Kimi-K2.5 NVFP4 + EAGLE3 on GB300 (4x GB300 NVL4, tp=4).""" - - def test_kimi_k25_nvfp4(self): - variants = [ - ModelLaunchSettings( - MODEL_PATH, - tp_size=4, - extra_args=COMMON_ARGS + TP_EAGLE_ARGS, - variant="TP4+EAGLE3", - ), - ModelLaunchSettings( - MODEL_PATH, - tp_size=4, - extra_args=COMMON_ARGS - + ["--dp-size=4", "--enable-dp-attention"] - + DP_EAGLE_ARGS, - variant="TP4+DP4+DPA+EAGLE3", - ), - ] - - failures = [] - # Pinned to what `ns eval --benchmarks=mmmu-pro:1` sent implicitly -- - # its `:1` suffix means temperature 0.7, not greedy -- so the baseline - # carries over unchanged. Do not "simplify" these away. - accuracy_params = AccuracyTestParams( - dataset="mmmu_pro_vision", - baseline_accuracy=0.69, - repeat=1, - max_tokens=32768, - temperature=0.7, - seed=0, - sgl_eval_thinking=False, - ) - for variant in variants: - try: - run_combined_tests( - models=[variant], - test_name=f"Kimi-K2.5-NVFP4 ({variant.variant})", - accuracy_params=accuracy_params, - performance_params=PerformanceTestParams( - batch_sizes=PERFORMANCE_BATCH_SIZES[variant.variant], - result_dir="performance_results_gb300", - ), - ) - except AssertionError as e: - failures.append(f"{variant.variant}: {e}") - - if failures: - raise AssertionError("Kimi-K2.5-NVFP4 failures:\n" + "\n".join(failures)) - - -if __name__ == "__main__": - unittest.main() diff --git a/test/registered/gb300/test_kimi_k25_nvfp4_dp.py b/test/registered/gb300/test_kimi_k25_nvfp4_dp.py new file mode 100644 index 000000000..589ff2801 --- /dev/null +++ b/test/registered/gb300/test_kimi_k25_nvfp4_dp.py @@ -0,0 +1,75 @@ +import unittest + +from sglang.test.accuracy_test_runner import AccuracyTestParams +from sglang.test.ci.ci_register import register_cuda_ci +from sglang.test.gb300_utils import GB300_NCCL_PORT +from sglang.test.performance_test_runner import PerformanceTestParams +from sglang.test.run_combined_tests import run_combined_tests +from sglang.test.test_utils import CustomTestCase, ModelLaunchSettings + +register_cuda_ci(est_time=7200, stage="nightly", runner_config="4-gpu-gb300") + +MODEL_PATH = "nvidia/Kimi-K2.5-NVFP4" +DRAFT_MODEL_PATH = "lightseekorg/kimi-k2.5-eagle3-mla" + +COMMON_ARGS = [ + "--trust-remote-code", + "--reasoning-parser=kimi_k2", + "--tool-call-parser=kimi_k2", + "--quantization=modelopt_fp4", + "--attention-backend=tokenspeed_mla", + "--kv-cache-dtype=fp8_e4m3", + "--moe-runner-backend=flashinfer_trtllm", + "--mem-fraction-static=0.8", + "--enable-metrics", + "--speculative-algorithm=EAGLE3", + f"--speculative-draft-model-path={DRAFT_MODEL_PATH}", + "--speculative-draft-model-quantization=unquant", + "--nccl-port", + GB300_NCCL_PORT, +] + +DP_EAGLE_ARGS = [ + "--speculative-num-steps=1", + "--speculative-eagle-topk=1", + "--speculative-num-draft-tokens=2", +] + + +class TestKimiK25Nvfp4Dp(CustomTestCase): + """Kimi-K2.5 NVFP4 DP4+DPA+EAGLE3 on GB300 (4x GB300 NVL4).""" + + def test_kimi_k25_nvfp4_dp(self): + # Pinned to what `ns eval --benchmarks=mmmu-pro:1` sent implicitly -- + # its `:1` suffix means temperature 0.7, not greedy -- so the baseline + # carries over unchanged. Do not "simplify" these away. + run_combined_tests( + models=[ + ModelLaunchSettings( + MODEL_PATH, + tp_size=4, + extra_args=COMMON_ARGS + + ["--dp-size=4", "--enable-dp-attention"] + + DP_EAGLE_ARGS, + variant="TP4+DP4+DPA+EAGLE3", + ) + ], + test_name="Kimi-K2.5-NVFP4 (TP4+DP4+DPA+EAGLE3)", + accuracy_params=AccuracyTestParams( + dataset="mmmu_pro_vision", + baseline_accuracy=0.69, + repeat=1, + max_tokens=32768, + temperature=0.7, + seed=0, + sgl_eval_thinking=False, + ), + performance_params=PerformanceTestParams( + batch_sizes=[16], + result_dir="performance_results_gb300", + ), + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/gb300/test_kimi_k25_nvfp4_tp.py b/test/registered/gb300/test_kimi_k25_nvfp4_tp.py new file mode 100644 index 000000000..36c49c450 --- /dev/null +++ b/test/registered/gb300/test_kimi_k25_nvfp4_tp.py @@ -0,0 +1,73 @@ +import unittest + +from sglang.test.accuracy_test_runner import AccuracyTestParams +from sglang.test.ci.ci_register import register_cuda_ci +from sglang.test.gb300_utils import GB300_NCCL_PORT +from sglang.test.performance_test_runner import PerformanceTestParams +from sglang.test.run_combined_tests import run_combined_tests +from sglang.test.test_utils import CustomTestCase, ModelLaunchSettings + +register_cuda_ci(est_time=4800, stage="nightly", runner_config="4-gpu-gb300") + +MODEL_PATH = "nvidia/Kimi-K2.5-NVFP4" +DRAFT_MODEL_PATH = "lightseekorg/kimi-k2.5-eagle3-mla" + +COMMON_ARGS = [ + "--trust-remote-code", + "--reasoning-parser=kimi_k2", + "--tool-call-parser=kimi_k2", + "--quantization=modelopt_fp4", + "--attention-backend=tokenspeed_mla", + "--kv-cache-dtype=fp8_e4m3", + "--moe-runner-backend=flashinfer_trtllm", + "--mem-fraction-static=0.8", + "--enable-metrics", + "--speculative-algorithm=EAGLE3", + f"--speculative-draft-model-path={DRAFT_MODEL_PATH}", + "--speculative-draft-model-quantization=unquant", + "--nccl-port", + GB300_NCCL_PORT, +] + +TP_EAGLE_ARGS = [ + "--speculative-num-steps=3", + "--speculative-eagle-topk=1", + "--speculative-num-draft-tokens=4", +] + + +class TestKimiK25Nvfp4Tp(CustomTestCase): + """Kimi-K2.5 NVFP4 TP4+EAGLE3 on GB300 (4x GB300 NVL4).""" + + def test_kimi_k25_nvfp4_tp(self): + # Pinned to what `ns eval --benchmarks=mmmu-pro:1` sent implicitly -- + # its `:1` suffix means temperature 0.7, not greedy -- so the baseline + # carries over unchanged. Do not "simplify" these away. + run_combined_tests( + models=[ + ModelLaunchSettings( + MODEL_PATH, + tp_size=4, + extra_args=COMMON_ARGS + TP_EAGLE_ARGS, + variant="TP4+EAGLE3", + ) + ], + test_name="Kimi-K2.5-NVFP4 (TP4+EAGLE3)", + accuracy_params=AccuracyTestParams( + dataset="mmmu_pro_vision", + baseline_accuracy=0.69, + repeat=1, + max_tokens=32768, + temperature=0.7, + seed=0, + sgl_eval_thinking=False, + ), + performance_params=PerformanceTestParams( + batch_sizes=[1, 8], + result_dir="performance_results_gb300", + ), + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/gb300/test_qwen35_fp8.py b/test/registered/gb300/test_qwen35_fp8.py deleted file mode 100644 index 8a2d4b177..000000000 --- a/test/registered/gb300/test_qwen35_fp8.py +++ /dev/null @@ -1,98 +0,0 @@ -import unittest - -from sglang.test.accuracy_test_runner import AccuracyTestParams -from sglang.test.ci.ci_register import register_cuda_ci -from sglang.test.performance_test_runner import PerformanceTestParams -from sglang.test.run_combined_tests import run_combined_tests -from sglang.test.test_utils import ModelLaunchSettings - -register_cuda_ci(est_time=7200, stage="nightly", runner_config="4-gpu-gb300") - -MODEL_PATH = "Qwen/Qwen3.5-397B-A17B-FP8" - -COMMON_ARGS = [ - "--trust-remote-code", - "--reasoning-parser=qwen3", - "--tool-call-parser=qwen3_coder", - "--enable-flashinfer-allreduce-fusion", - "--attention-backend=trtllm_mha", - "--mem-fraction-static=0.8", - "--mamba-scheduler-strategy=extra_buffer", - "--enable-multimodal", - "--enable-metrics", -] - -TP_MTP_ARGS = [ - "--speculative-algorithm=EAGLE", - "--speculative-num-steps=3", - "--speculative-eagle-topk=1", - "--speculative-num-draft-tokens=4", -] - -DP_MTP_ARGS = [ - "--speculative-algorithm=EAGLE", - "--speculative-num-steps=1", - "--speculative-eagle-topk=1", - "--speculative-num-draft-tokens=2", -] - -PERFORMANCE_BATCH_SIZES = { - "TP4+MTP": [1, 4], - "TP4+DP4+DPA+MTP": [16], -} - - -class TestQwen35Fp8(unittest.TestCase): - """Qwen3.5-397B FP8 on GB300 (4x GB300 NVL4, tp=4).""" - - def test_qwen35_fp8(self): - variants = [ - ModelLaunchSettings( - MODEL_PATH, - tp_size=4, - extra_args=COMMON_ARGS + TP_MTP_ARGS, - variant="TP4+MTP", - ), - ModelLaunchSettings( - MODEL_PATH, - tp_size=4, - extra_args=COMMON_ARGS - + ["--dp-size=4", "--enable-dp-attention"] - + DP_MTP_ARGS, - variant="TP4+DP4+DPA+MTP", - ), - ] - - failures = [] - # Pinned to what `ns eval --benchmarks=mmmu-pro:1` sent implicitly -- - # its `:1` suffix means temperature 0.7, not greedy -- so the baseline - # carries over unchanged. Do not "simplify" these away. - accuracy_params = AccuracyTestParams( - dataset="mmmu_pro_vision", - baseline_accuracy=0.76, - repeat=1, - max_tokens=32768, - temperature=0.7, - seed=0, - sgl_eval_thinking=False, - ) - for variant in variants: - try: - run_combined_tests( - models=[variant], - test_name=f"Qwen3.5-397B-FP8 ({variant.variant})", - accuracy_params=accuracy_params, - performance_params=PerformanceTestParams( - batch_sizes=PERFORMANCE_BATCH_SIZES[variant.variant], - result_dir="performance_results_gb300", - ), - ) - except AssertionError as e: - failures.append(f"{variant.variant}: {e}") - - if failures: - raise AssertionError("Qwen3.5-397B-FP8 failures:\n" + "\n".join(failures)) - - -if __name__ == "__main__": - unittest.main() diff --git a/test/registered/gb300/test_qwen35_fp8_dp.py b/test/registered/gb300/test_qwen35_fp8_dp.py new file mode 100644 index 000000000..481e787bf --- /dev/null +++ b/test/registered/gb300/test_qwen35_fp8_dp.py @@ -0,0 +1,72 @@ +import unittest + +from sglang.test.accuracy_test_runner import AccuracyTestParams +from sglang.test.ci.ci_register import register_cuda_ci +from sglang.test.gb300_utils import GB300_NCCL_PORT +from sglang.test.performance_test_runner import PerformanceTestParams +from sglang.test.run_combined_tests import run_combined_tests +from sglang.test.test_utils import CustomTestCase, ModelLaunchSettings + +register_cuda_ci(est_time=7200, stage="nightly", runner_config="4-gpu-gb300") + +MODEL_PATH = "Qwen/Qwen3.5-397B-A17B-FP8" + +COMMON_ARGS = [ + "--trust-remote-code", + "--reasoning-parser=qwen3", + "--tool-call-parser=qwen3_coder", + "--enable-flashinfer-allreduce-fusion", + "--attention-backend=trtllm_mha", + "--mem-fraction-static=0.8", + "--mamba-scheduler-strategy=extra_buffer", + "--enable-multimodal", + "--enable-metrics", + "--nccl-port", + GB300_NCCL_PORT, +] + +DP_MTP_ARGS = [ + "--speculative-algorithm=EAGLE", + "--speculative-num-steps=1", + "--speculative-eagle-topk=1", + "--speculative-num-draft-tokens=2", +] + + +class TestQwen35Fp8Dp(CustomTestCase): + """Qwen3.5-397B FP8 DP4+DPA+MTP on GB300 (4x GB300 NVL4).""" + + def test_qwen35_fp8_dp(self): + # Pinned to what `ns eval --benchmarks=mmmu-pro:1` sent implicitly -- + # its `:1` suffix means temperature 0.7, not greedy -- so the baseline + # carries over unchanged. Do not "simplify" these away. + run_combined_tests( + models=[ + ModelLaunchSettings( + MODEL_PATH, + tp_size=4, + extra_args=COMMON_ARGS + + ["--dp-size=4", "--enable-dp-attention"] + + DP_MTP_ARGS, + variant="TP4+DP4+DPA+MTP", + ) + ], + test_name="Qwen3.5-397B-FP8 (TP4+DP4+DPA+MTP)", + accuracy_params=AccuracyTestParams( + dataset="mmmu_pro_vision", + baseline_accuracy=0.76, + repeat=1, + max_tokens=32768, + temperature=0.7, + seed=0, + sgl_eval_thinking=False, + ), + performance_params=PerformanceTestParams( + batch_sizes=[16], + result_dir="performance_results_gb300", + ), + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/gb300/test_qwen35_fp8_tp.py b/test/registered/gb300/test_qwen35_fp8_tp.py new file mode 100644 index 000000000..13b2acd74 --- /dev/null +++ b/test/registered/gb300/test_qwen35_fp8_tp.py @@ -0,0 +1,70 @@ +import unittest + +from sglang.test.accuracy_test_runner import AccuracyTestParams +from sglang.test.ci.ci_register import register_cuda_ci +from sglang.test.gb300_utils import GB300_NCCL_PORT +from sglang.test.performance_test_runner import PerformanceTestParams +from sglang.test.run_combined_tests import run_combined_tests +from sglang.test.test_utils import CustomTestCase, ModelLaunchSettings + +register_cuda_ci(est_time=4800, stage="nightly", runner_config="4-gpu-gb300") + +MODEL_PATH = "Qwen/Qwen3.5-397B-A17B-FP8" + +COMMON_ARGS = [ + "--trust-remote-code", + "--reasoning-parser=qwen3", + "--tool-call-parser=qwen3_coder", + "--enable-flashinfer-allreduce-fusion", + "--attention-backend=trtllm_mha", + "--mem-fraction-static=0.8", + "--mamba-scheduler-strategy=extra_buffer", + "--enable-multimodal", + "--enable-metrics", + "--nccl-port", + GB300_NCCL_PORT, +] + +TP_MTP_ARGS = [ + "--speculative-algorithm=EAGLE", + "--speculative-num-steps=3", + "--speculative-eagle-topk=1", + "--speculative-num-draft-tokens=4", +] + + +class TestQwen35Fp8Tp(CustomTestCase): + """Qwen3.5-397B FP8 TP4+MTP on GB300 (4x GB300 NVL4).""" + + def test_qwen35_fp8_tp(self): + # Pinned to what `ns eval --benchmarks=mmmu-pro:1` sent implicitly -- + # its `:1` suffix means temperature 0.7, not greedy -- so the baseline + # carries over unchanged. Do not "simplify" these away. + run_combined_tests( + models=[ + ModelLaunchSettings( + MODEL_PATH, + tp_size=4, + extra_args=COMMON_ARGS + TP_MTP_ARGS, + variant="TP4+MTP", + ) + ], + test_name="Qwen3.5-397B-FP8 (TP4+MTP)", + accuracy_params=AccuracyTestParams( + dataset="mmmu_pro_vision", + baseline_accuracy=0.76, + repeat=1, + max_tokens=32768, + temperature=0.7, + seed=0, + sgl_eval_thinking=False, + ), + performance_params=PerformanceTestParams( + batch_sizes=[1, 8], + result_dir="performance_results_gb300", + ), + ) + + +if __name__ == "__main__": + unittest.main()