diff --git a/.github/workflows/nightly-test-nvidia.yml b/.github/workflows/nightly-test-nvidia.yml index 06369abb9..5660007de 100644 --- a/.github/workflows/nightly-test-nvidia.yml +++ b/.github/workflows/nightly-test-nvidia.yml @@ -24,6 +24,7 @@ on: - 'nightly-test-perf-4-gpu-b200' - 'nightly-test-perf-8-gpu-b200' - 'nightly-test-specialized-8-gpu-b200' + - 'nightly-test-perf-4-gpu-gb300' - 'nightly-test-kernel-1-gpu-h100' - 'nightly-test-diffusion-comparison' - 'nightly-test-kernel-8-gpu-h200' @@ -512,6 +513,31 @@ jobs: - uses: ./.github/actions/upload-cuda-coredumps if: failure() + # GB300 (Grace-Blackwell NVL4) performance tests - 4 GPU (ARM64) + nightly-test-perf-4-gpu-gb300: + if: github.repository == 'sgl-project/sglang' && (inputs.job_filter == '' || inputs.job_filter == 'all' || inputs.job_filter == 'nightly-test-perf-4-gpu-gb300') + runs-on: 4-gpu-gb300 + steps: + - name: Checkout code + uses: actions/checkout@v4 + with: + ref: ${{ inputs.ref || github.ref }} + + - uses: ./.github/actions/check-maintenance + + - name: Install dependencies + run: | + bash scripts/ci/cuda/ci_install_dependency.sh + + - name: Run test + timeout-minutes: 600 + run: | + cd test + python3 run_suite.py --hw cuda --suite nightly-4-gpu-gb300 --nightly --continue-on-error --timeout-per-file 7200 + + - uses: ./.github/actions/upload-cuda-coredumps + if: failure() + # Specialized B200 tests - 8 GPU, for specific backends and configs nightly-test-specialized-8-gpu-b200: if: github.repository == 'sgl-project/sglang' && (inputs.job_filter == '' || inputs.job_filter == 'all' || inputs.job_filter == 'nightly-test-perf-8-gpu-b200' || inputs.job_filter == 'nightly-test-specialized-8-gpu-b200') @@ -695,6 +721,7 @@ jobs: - nightly-test-vlm-perf-2-gpu-h100 - nightly-test-perf-4-gpu-b200 - nightly-test-specialized-8-gpu-b200 + - nightly-test-perf-4-gpu-gb300 - nightly-test-diffusion-comparison - nightly-test-precision-8-gpu-h200 - consolidate-metrics diff --git a/python/sglang/test/performance_test_runner.py b/python/sglang/test/performance_test_runner.py index 5985e8692..398252148 100644 --- a/python/sglang/test/performance_test_runner.py +++ b/python/sglang/test/performance_test_runner.py @@ -10,8 +10,8 @@ from sglang.test.test_utils import DEFAULT_URL_FOR_TEST, ModelLaunchSettings class PerformanceTestParams: """Parameters for performance testing.""" - batch_sizes: List[int] = field(default_factory=lambda: [1, 8, 16, 64]) - input_lens: Tuple[int, ...] = (4096,) + batch_sizes: List[int] = field(default_factory=lambda: [1, 8, 16]) + input_lens: Tuple[int, ...] = (8192,) output_lens: Tuple[int, ...] = (512,) profile_dir: Optional[str] = None # None = auto-generate based on is_vlm dataset_name: str = "mmmu" # For VLM perf test diff --git a/test/registered/gb300/test_deepseek_v32.py b/test/registered/gb300/test_deepseek_v32.py deleted file mode 100644 index e370f5e46..000000000 --- a/test/registered/gb300/test_deepseek_v32.py +++ /dev/null @@ -1,78 +0,0 @@ -import unittest - -from sglang.test.accuracy_test_runner import AccuracyTestParams -from sglang.test.ci.ci_register import register_cuda_ci -from sglang.test.performance_test_runner import PerformanceTestParams -from sglang.test.run_combined_tests import run_combined_tests -from sglang.test.test_utils import ModelLaunchSettings - -register_cuda_ci(est_time=7200, suite="nightly-4-gpu-gb300", nightly=True) - -MODEL_PATH = "deepseek-ai/DeepSeek-V3.2" - -COMMON_ARGS = [ - "--trust-remote-code", - "--reasoning-parser=deepseek-v3", - "--tool-call-parser=deepseekv32", - "--mem-fraction-static=0.8", - "--enable-metrics", -] - -MTP_ARGS = [ - "--speculative-algorithm=EAGLE", - "--speculative-num-steps=3", - "--speculative-eagle-topk=1", - "--speculative-num-draft-tokens=4", -] - - -class TestDeepseekV32(unittest.TestCase): - """DeepSeek V3.2 on GB300 (4x B200 NVL4, tp=4).""" - - def test_deepseek_v32(self): - variants = [ - ModelLaunchSettings( - MODEL_PATH, - tp_size=4, - extra_args=COMMON_ARGS, - variant="TP4", - ), - ModelLaunchSettings( - MODEL_PATH, - tp_size=4, - extra_args=COMMON_ARGS - + [ - "--dp-size=4", - "--ep-size=4", - "--enable-dp-attention", - ], - variant="TP4+DP4+DPA", - ), - ModelLaunchSettings( - MODEL_PATH, - tp_size=4, - extra_args=COMMON_ARGS - + [ - "--dp-size=4", - "--ep-size=4", - "--enable-dp-attention", - ] - + MTP_ARGS, - variant="TP4+DP4+DPA+MTP", - ), - ] - - run_combined_tests( - models=variants, - test_name="DeepSeek-V3.2", - accuracy_params=AccuracyTestParams( - dataset="gsm8k", baseline_accuracy=0.935 - ), - performance_params=PerformanceTestParams( - profile_dir="performance_profiles_gb300", - ), - ) - - -if __name__ == "__main__": - unittest.main() diff --git a/test/registered/gb300/test_deepseek_v32_nvfp4.py b/test/registered/gb300/test_deepseek_v32_nvfp4.py deleted file mode 100644 index 4640f4fb1..000000000 --- a/test/registered/gb300/test_deepseek_v32_nvfp4.py +++ /dev/null @@ -1,81 +0,0 @@ -import unittest - -from sglang.test.accuracy_test_runner import AccuracyTestParams -from sglang.test.ci.ci_register import register_cuda_ci -from sglang.test.performance_test_runner import PerformanceTestParams -from sglang.test.run_combined_tests import run_combined_tests -from sglang.test.test_utils import ModelLaunchSettings - -register_cuda_ci(est_time=7200, suite="nightly-4-gpu-gb300", nightly=True) - -MODEL_PATH = "nvidia/DeepSeek-V3.2-NVFP4" - -COMMON_ARGS = [ - "--trust-remote-code", - "--reasoning-parser=deepseek-v3", - "--tool-call-parser=deepseekv32", - "--quantization=modelopt_fp4", - "--moe-runner-backend=flashinfer_trtllm", - "--kv-cache-dtype=bfloat16", - "--mem-fraction-static=0.8", - "--enable-metrics", -] - -MTP_ARGS = [ - "--speculative-algorithm=EAGLE", - "--speculative-num-steps=3", - "--speculative-eagle-topk=1", - "--speculative-num-draft-tokens=4", -] - - -class TestDeepseekV32Nvfp4(unittest.TestCase): - """DeepSeek V3.2 NVFP4 on GB300 (4x B200 NVL4, tp=4).""" - - def test_deepseek_v32_nvfp4(self): - variants = [ - ModelLaunchSettings( - MODEL_PATH, - tp_size=4, - extra_args=COMMON_ARGS, - variant="TP4", - ), - ModelLaunchSettings( - MODEL_PATH, - tp_size=4, - extra_args=COMMON_ARGS - + [ - "--dp-size=4", - "--ep-size=4", - "--enable-dp-attention", - ], - variant="TP4+DP4+DPA", - ), - ModelLaunchSettings( - MODEL_PATH, - tp_size=4, - extra_args=COMMON_ARGS - + [ - "--dp-size=4", - "--ep-size=4", - "--enable-dp-attention", - ] - + MTP_ARGS, - variant="TP4+DP4+DPA+MTP", - ), - ] - - run_combined_tests( - models=variants, - test_name="DeepSeek-V3.2-NVFP4", - accuracy_params=AccuracyTestParams( - dataset="gsm8k", baseline_accuracy=0.935 - ), - performance_params=PerformanceTestParams( - profile_dir="performance_profiles_gb300", - ), - ) - - -if __name__ == "__main__": - unittest.main() diff --git a/test/registered/gb300/test_glm5_fp8.py b/test/registered/gb300/test_glm5_fp8.py index c9d19bea7..a7dca9ce7 100644 --- a/test/registered/gb300/test_glm5_fp8.py +++ b/test/registered/gb300/test_glm5_fp8.py @@ -6,7 +6,9 @@ from sglang.test.performance_test_runner import PerformanceTestParams from sglang.test.run_combined_tests import run_combined_tests from sglang.test.test_utils import ModelLaunchSettings -register_cuda_ci(est_time=7200, suite="nightly-4-gpu-gb300", nightly=True) +register_cuda_ci( + est_time=7200, suite="nightly-4-gpu-gb300", nightly=True, disabled="not needed" +) MODEL_PATH = "zai-org/GLM-5.1-FP8" @@ -27,7 +29,7 @@ MTP_ARGS = [ class TestGlm5Fp8(unittest.TestCase): - """GLM-5.1 FP8 on GB300 (4x B200 NVL4, tp=4).""" + """GLM-5.1 FP8 on GB300 (4x GB300 NVL4, tp=4).""" def test_glm5_fp8(self): variants = [ diff --git a/test/registered/gb300/test_glm5_nvfp4.py b/test/registered/gb300/test_glm5_nvfp4.py index 816f05881..fe3aabd97 100644 --- a/test/registered/gb300/test_glm5_nvfp4.py +++ b/test/registered/gb300/test_glm5_nvfp4.py @@ -16,42 +16,42 @@ COMMON_ARGS = [ "--tool-call-parser=glm47", "--quantization=modelopt_fp4", "--moe-runner-backend=flashinfer_trtllm", - "--kv-cache-dtype=bfloat16", "--mem-fraction-static=0.9", "--enable-metrics", ] -MTP_ARGS = [ +TP_MTP_ARGS = [ "--speculative-algorithm=EAGLE", "--speculative-num-steps=3", "--speculative-eagle-topk=1", "--speculative-num-draft-tokens=4", ] +DP_MTP_ARGS = [ + "--speculative-algorithm=EAGLE", + "--speculative-num-steps=1", + "--speculative-eagle-topk=1", + "--speculative-num-draft-tokens=2", +] + class TestGlm5Nvfp4(unittest.TestCase): - """GLM-5 NVFP4 on GB300 (4x B200 NVL4, tp=4).""" + """GLM-5 NVFP4 on GB300 (4x GB300 NVL4, tp=4).""" def test_glm5_nvfp4(self): variants = [ ModelLaunchSettings( MODEL_PATH, tp_size=4, - extra_args=COMMON_ARGS, - variant="TP4", - ), - ModelLaunchSettings( - MODEL_PATH, - tp_size=4, - extra_args=COMMON_ARGS + ["--dp-size=4", "--enable-dp-attention"], - variant="TP4+DP4+DPA", + extra_args=COMMON_ARGS + TP_MTP_ARGS, + variant="TP4+MTP", ), ModelLaunchSettings( MODEL_PATH, tp_size=4, extra_args=COMMON_ARGS + ["--dp-size=4", "--enable-dp-attention"] - + MTP_ARGS, + + DP_MTP_ARGS, variant="TP4+DP4+DPA+MTP", ), ] diff --git a/test/registered/gb300/test_kimi_k25.py b/test/registered/gb300/test_kimi_k25.py index 47beb0b19..d1efa400f 100644 --- a/test/registered/gb300/test_kimi_k25.py +++ b/test/registered/gb300/test_kimi_k25.py @@ -6,7 +6,9 @@ from sglang.test.performance_test_runner import PerformanceTestParams from sglang.test.run_combined_tests import run_combined_tests from sglang.test.test_utils import ModelLaunchSettings -register_cuda_ci(est_time=7200, suite="nightly-4-gpu-gb300", nightly=True) +register_cuda_ci( + est_time=7200, suite="nightly-4-gpu-gb300", nightly=True, disabled="not needed" +) MODEL_PATH = "moonshotai/Kimi-K2.5" @@ -21,7 +23,7 @@ COMMON_ARGS = [ class TestKimiK25(unittest.TestCase): - """Kimi-K2.5 (native INT4) on GB300 (4x B200 NVL4, tp=4). + """Kimi-K2.5 (native INT4) on GB300 (4x GB300 NVL4, tp=4). No EAGLE/MTP support for Kimi-K2.5 — only TP and TP+DP+DPA variants. """ diff --git a/test/registered/gb300/test_kimi_k25_nvfp4.py b/test/registered/gb300/test_kimi_k25_nvfp4.py index 7faf6c92b..4268ef3fa 100644 --- a/test/registered/gb300/test_kimi_k25_nvfp4.py +++ b/test/registered/gb300/test_kimi_k25_nvfp4.py @@ -15,7 +15,8 @@ COMMON_ARGS = [ "--reasoning-parser=kimi_k2", "--tool-call-parser=kimi_k2", "--quantization=modelopt_fp4", - "--attention-backend=trtllm_mla", + "--attention-backend=tokenspeed_mla", + "--kv-cache-dtype=fp8_e4m3", "--moe-runner-backend=flashinfer_trtllm", "--mem-fraction-static=0.8", "--enable-multimodal", @@ -24,7 +25,7 @@ COMMON_ARGS = [ class TestKimiK25Nvfp4(unittest.TestCase): - """Kimi-K2.5 NVFP4 on GB300 (4x B200 NVL4, tp=4). + """Kimi-K2.5 NVFP4 on GB300 (4x GB300 NVL4, tp=4). No EAGLE/MTP support for Kimi-K2.5 — only TP and TP+DP+DPA variants. """ diff --git a/test/registered/gb300/test_qwen35_fp8.py b/test/registered/gb300/test_qwen35_fp8.py index 702bebdbc..df364761a 100644 --- a/test/registered/gb300/test_qwen35_fp8.py +++ b/test/registered/gb300/test_qwen35_fp8.py @@ -17,43 +17,43 @@ COMMON_ARGS = [ "--enable-flashinfer-allreduce-fusion", "--attention-backend=trtllm_mha", "--mem-fraction-static=0.8", + "--mamba-scheduler-strategy=extra_buffer", "--enable-multimodal", "--enable-metrics", ] -MTP_ARGS = [ +TP_MTP_ARGS = [ "--speculative-algorithm=EAGLE", "--speculative-num-steps=3", "--speculative-eagle-topk=1", "--speculative-num-draft-tokens=4", - "--mamba-scheduler-strategy=extra_buffer", - "--page-size=64", +] + +DP_MTP_ARGS = [ + "--speculative-algorithm=EAGLE", + "--speculative-num-steps=1", + "--speculative-eagle-topk=1", + "--speculative-num-draft-tokens=2", ] class TestQwen35Fp8(unittest.TestCase): - """Qwen3.5-397B FP8 on GB300 (4x B200 NVL4, tp=4).""" + """Qwen3.5-397B FP8 on GB300 (4x GB300 NVL4, tp=4).""" def test_qwen35_fp8(self): variants = [ ModelLaunchSettings( MODEL_PATH, tp_size=4, - extra_args=COMMON_ARGS, - variant="TP4", - ), - ModelLaunchSettings( - MODEL_PATH, - tp_size=4, - extra_args=COMMON_ARGS + ["--dp-size=4", "--enable-dp-attention"], - variant="TP4+DP4+DPA", + extra_args=COMMON_ARGS + TP_MTP_ARGS, + variant="TP4+MTP", ), ModelLaunchSettings( MODEL_PATH, tp_size=4, extra_args=COMMON_ARGS + ["--dp-size=4", "--enable-dp-attention"] - + MTP_ARGS, + + DP_MTP_ARGS, variant="TP4+DP4+DPA+MTP", ), ] @@ -62,7 +62,7 @@ class TestQwen35Fp8(unittest.TestCase): models=variants, test_name="Qwen3.5-397B-FP8", accuracy_params=AccuracyTestParams( - dataset="mmmu-pro", baseline_accuracy=0.78, repeat=1, max_tokens=32768 + dataset="mmmu-pro", baseline_accuracy=0.76, repeat=1, max_tokens=32768 ), performance_params=PerformanceTestParams( profile_dir="performance_profiles_gb300", diff --git a/test/registered/gb300/test_qwen35_nvfp4.py b/test/registered/gb300/test_qwen35_nvfp4.py index 7199ff90f..a2866b7cd 100644 --- a/test/registered/gb300/test_qwen35_nvfp4.py +++ b/test/registered/gb300/test_qwen35_nvfp4.py @@ -6,7 +6,9 @@ from sglang.test.performance_test_runner import PerformanceTestParams from sglang.test.run_combined_tests import run_combined_tests from sglang.test.test_utils import ModelLaunchSettings -register_cuda_ci(est_time=7200, suite="nightly-4-gpu-gb300", nightly=True) +register_cuda_ci( + est_time=7200, suite="nightly-4-gpu-gb300", nightly=True, disabled="not needed" +) MODEL_PATH = "nvidia/Qwen3.5-397B-A17B-NVFP4" @@ -36,7 +38,7 @@ MTP_ARGS = [ class TestQwen35Nvfp4(unittest.TestCase): - """Qwen3.5-397B NVFP4 on GB300 (4x B200 NVL4, tp=4).""" + """Qwen3.5-397B NVFP4 on GB300 (4x GB300 NVL4, tp=4).""" def test_qwen35_nvfp4(self): variants = [ @@ -66,7 +68,7 @@ class TestQwen35Nvfp4(unittest.TestCase): models=variants, test_name="Qwen3.5-397B-NVFP4", accuracy_params=AccuracyTestParams( - dataset="mmmu-pro", baseline_accuracy=0.78, repeat=1, max_tokens=32768 + dataset="mmmu-pro", baseline_accuracy=0.76, repeat=1, max_tokens=32768 ), performance_params=PerformanceTestParams( profile_dir="performance_profiles_gb300", diff --git a/test/run_suite.py b/test/run_suite.py index 42688cf6e..de7d3247f 100644 --- a/test/run_suite.py +++ b/test/run_suite.py @@ -121,7 +121,7 @@ NIGHTLY_SUITES = { "nightly-eval-vlm-2-gpu", "nightly-perf-text-2-gpu", "nightly-perf-vlm-2-gpu", - # GB300 (4x B200 NVL4) nightly suite + # GB300 (4x GB300 NVL4) nightly suite "nightly-4-gpu-gb300", # Nightly precision regression (per-layer hidden state comparison) "nightly-precision-8-gpu-h200",