diff --git a/.github/workflows/nightly-test-nvidia.yml b/.github/workflows/nightly-test-nvidia.yml index 1ec8e85f7..e9b3ed0dd 100644 --- a/.github/workflows/nightly-test-nvidia.yml +++ b/.github/workflows/nightly-test-nvidia.yml @@ -539,7 +539,20 @@ jobs: # GB300 (Grace-Blackwell NVL4) performance tests - 4 GPU (ARM64) nightly-test-perf-4-gpu-gb300: if: github.repository == 'sgl-project/sglang' && (inputs.job_filter == '' || inputs.job_filter == 'all' || inputs.job_filter == 'nightly-test-perf-4-gpu-gb300') - runs-on: 4-gpu-gb300 + name: nightly-test-perf-4-gpu-gb300 (${{ matrix.model }}) + runs-on: 4-gpu-gb300-nightly + strategy: + fail-fast: false + matrix: + include: + - model: glm5-nvfp4 + suite: nightly-4-gpu-gb300-glm5-nvfp4 + - model: qwen35-fp8 + suite: nightly-4-gpu-gb300-qwen35-fp8 + - model: deepseek-v4-pro-fp4 + suite: nightly-4-gpu-gb300-deepseek-v4-pro-fp4 + - model: kimi-k25-nvfp4 + suite: nightly-4-gpu-gb300-kimi-k25-nvfp4 steps: - name: Checkout code uses: actions/checkout@v4 @@ -549,8 +562,10 @@ jobs: - uses: ./.github/actions/check-maintenance - name: Install dependencies + env: + GRACE_BLACKWELL: "1" run: | - bash scripts/ci/cuda/ci_install_dependency.sh + bash scripts/ci/cuda/ci_install_deepep.sh - name: Run test timeout-minutes: 600 @@ -560,7 +575,7 @@ jobs: GPU_CONFIG: "4-gpu-gb300" run: | cd test - python3 run_suite.py --hw cuda --suite nightly-4-gpu-gb300 --nightly --continue-on-error --timeout-per-file 7200 + python3 run_suite.py --hw cuda --suite ${{ matrix.suite }} --nightly --continue-on-error --timeout-per-file 7200 - name: Publish traces to storage repo if: always() diff --git a/test/registered/gb300/test_deepseek_v4_pro_fp4.py b/test/registered/gb300/test_deepseek_v4_pro_fp4.py new file mode 100644 index 000000000..ace08c3e6 --- /dev/null +++ b/test/registered/gb300/test_deepseek_v4_pro_fp4.py @@ -0,0 +1,152 @@ +import unittest + +from sglang.test.accuracy_test_runner import AccuracyTestParams +from sglang.test.ci.ci_register import register_cuda_ci +from sglang.test.performance_test_runner import PerformanceTestParams +from sglang.test.run_combined_tests import run_combined_tests +from sglang.test.test_utils import ModelLaunchSettings + +register_cuda_ci( + est_time=7200, suite="nightly-4-gpu-gb300-deepseek-v4-pro-fp4", nightly=True +) + +MODEL_PATH = "deepseek-ai/DeepSeek-V4-Pro" +SERVER_LAUNCH_TIMEOUT = 3600 + +DEEPEP_CONFIG = '{"normal_dispatch":{"num_sms":96},"normal_combine":{"num_sms":96}}' + +LOW_LATENCY_ARGS = [ + "--trust-remote-code", + "--moe-runner-backend", + "flashinfer_mxfp4", + "--speculative-algorithm", + "EAGLE", + "--speculative-num-steps", + "3", + "--speculative-eagle-topk", + "1", + "--speculative-num-draft-tokens", + "4", + "--chunked-prefill-size", + "8192", + "--disable-flashinfer-autotune", + "--swa-full-tokens-ratio", + "0.1", + "--mem-fraction-static", + "0.85", +] + +BALANCED_ARGS = [ + "--trust-remote-code", + "--dp", + "4", + "--enable-dp-attention", + "--moe-a2a-backend", + "deepep", + "--speculative-algorithm", + "EAGLE", + "--speculative-num-steps", + "1", + "--speculative-eagle-topk", + "1", + "--speculative-num-draft-tokens", + "2", + "--mem-fraction-static", + "0.85", + "--cuda-graph-max-bs", + "128", + "--max-running-requests", + "256", + "--deepep-config", + DEEPEP_CONFIG, +] + +HIGH_THROUGHPUT_ARGS = [ + "--trust-remote-code", + "--dp", + "4", + "--enable-dp-attention", + "--moe-a2a-backend", + "megamoe", + "--mem-fraction-static", + "0.85", + "--cuda-graph-max-bs", + "128", + "--max-running-requests", + "256", +] + +BALANCED_ENV = { + "SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK": "256", +} + +HIGH_THROUGHPUT_ENV = { + "SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK": "8320", +} + +PERFORMANCE_BATCH_SIZES = { + "low-latency": [1, 4, 16], + "balanced": [64], + "high-throughput": [128], +} + + +class TestDeepSeekV4ProFp4(unittest.TestCase): + """DeepSeek-V4-Pro FP4 on GB300 (4x B200 NVL4, tp=4).""" + + def test_deepseek_v4_pro_fp4(self): + variants = [ + ModelLaunchSettings( + MODEL_PATH, + tp_size=4, + extra_args=LOW_LATENCY_ARGS, + variant="low-latency", + launch_timeout=SERVER_LAUNCH_TIMEOUT, + ), + ModelLaunchSettings( + MODEL_PATH, + tp_size=4, + extra_args=BALANCED_ARGS, + env=BALANCED_ENV, + variant="balanced", + launch_timeout=SERVER_LAUNCH_TIMEOUT, + ), + ModelLaunchSettings( + MODEL_PATH, + tp_size=4, + extra_args=HIGH_THROUGHPUT_ARGS, + env=HIGH_THROUGHPUT_ENV, + variant="high-throughput", + launch_timeout=SERVER_LAUNCH_TIMEOUT, + ), + ] + + failures = [] + accuracy_params = AccuracyTestParams( + dataset="gsm8k", + baseline_accuracy=0.935, + temperature=1.0, + top_p=1.0, + ) + for variant in variants: + try: + run_combined_tests( + models=[variant], + test_name=f"DeepSeek-V4-Pro-FP4 ({variant.variant})", + accuracy_params=accuracy_params, + performance_params=PerformanceTestParams( + batch_sizes=PERFORMANCE_BATCH_SIZES[variant.variant], + profile_dir="performance_profiles_gb300", + ), + ) + except AssertionError as e: + failures.append(f"{variant.variant}: {e}") + + if failures: + raise AssertionError( + "DeepSeek-V4-Pro-FP4 failures:\n" + "\n".join(failures) + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/gb300/test_glm5_fp8.py b/test/registered/gb300/test_glm5_fp8.py index a7dca9ce7..93ebcb303 100644 --- a/test/registered/gb300/test_glm5_fp8.py +++ b/test/registered/gb300/test_glm5_fp8.py @@ -7,7 +7,10 @@ from sglang.test.run_combined_tests import run_combined_tests from sglang.test.test_utils import ModelLaunchSettings register_cuda_ci( - est_time=7200, suite="nightly-4-gpu-gb300", nightly=True, disabled="not needed" + est_time=7200, + suite="nightly-4-gpu-gb300-glm5-fp8", + nightly=True, + disabled="not needed", ) MODEL_PATH = "zai-org/GLM-5.1-FP8" diff --git a/test/registered/gb300/test_glm5_nvfp4.py b/test/registered/gb300/test_glm5_nvfp4.py index fe3aabd97..83f7440e8 100644 --- a/test/registered/gb300/test_glm5_nvfp4.py +++ b/test/registered/gb300/test_glm5_nvfp4.py @@ -6,7 +6,7 @@ from sglang.test.performance_test_runner import PerformanceTestParams from sglang.test.run_combined_tests import run_combined_tests from sglang.test.test_utils import ModelLaunchSettings -register_cuda_ci(est_time=7200, suite="nightly-4-gpu-gb300", nightly=True) +register_cuda_ci(est_time=7200, suite="nightly-4-gpu-gb300-glm5-nvfp4", nightly=True) MODEL_PATH = "nvidia/GLM-5-NVFP4" diff --git a/test/registered/gb300/test_kimi_k25.py b/test/registered/gb300/test_kimi_k25.py index d1efa400f..b2a0ffe0f 100644 --- a/test/registered/gb300/test_kimi_k25.py +++ b/test/registered/gb300/test_kimi_k25.py @@ -7,7 +7,10 @@ from sglang.test.run_combined_tests import run_combined_tests from sglang.test.test_utils import ModelLaunchSettings register_cuda_ci( - est_time=7200, suite="nightly-4-gpu-gb300", nightly=True, disabled="not needed" + est_time=7200, + suite="nightly-4-gpu-gb300-kimi-k25", + nightly=True, + disabled="not needed", ) MODEL_PATH = "moonshotai/Kimi-K2.5" diff --git a/test/registered/gb300/test_kimi_k25_nvfp4.py b/test/registered/gb300/test_kimi_k25_nvfp4.py index 4268ef3fa..d6b073b0f 100644 --- a/test/registered/gb300/test_kimi_k25_nvfp4.py +++ b/test/registered/gb300/test_kimi_k25_nvfp4.py @@ -6,9 +6,12 @@ from sglang.test.performance_test_runner import PerformanceTestParams from sglang.test.run_combined_tests import run_combined_tests from sglang.test.test_utils import ModelLaunchSettings -register_cuda_ci(est_time=7200, suite="nightly-4-gpu-gb300", nightly=True) +register_cuda_ci( + est_time=7200, suite="nightly-4-gpu-gb300-kimi-k25-nvfp4", nightly=True +) MODEL_PATH = "nvidia/Kimi-K2.5-NVFP4" +DRAFT_MODEL_PATH = "lightseekorg/kimi-k2.5-eagle3-mla" COMMON_ARGS = [ "--trust-remote-code", @@ -19,30 +22,43 @@ COMMON_ARGS = [ "--kv-cache-dtype=fp8_e4m3", "--moe-runner-backend=flashinfer_trtllm", "--mem-fraction-static=0.8", - "--enable-multimodal", "--enable-metrics", + "--speculative-algorithm=EAGLE3", + f"--speculative-draft-model-path={DRAFT_MODEL_PATH}", + "--speculative-draft-model-quantization=unquant", +] + +TP_EAGLE_ARGS = [ + "--speculative-num-steps=3", + "--speculative-eagle-topk=1", + "--speculative-num-draft-tokens=4", +] + +DP_EAGLE_ARGS = [ + "--speculative-num-steps=1", + "--speculative-eagle-topk=1", + "--speculative-num-draft-tokens=2", ] class TestKimiK25Nvfp4(unittest.TestCase): - """Kimi-K2.5 NVFP4 on GB300 (4x GB300 NVL4, tp=4). - - No EAGLE/MTP support for Kimi-K2.5 — only TP and TP+DP+DPA variants. - """ + """Kimi-K2.5 NVFP4 + EAGLE3 on GB300 (4x GB300 NVL4, tp=4).""" def test_kimi_k25_nvfp4(self): variants = [ ModelLaunchSettings( MODEL_PATH, tp_size=4, - extra_args=COMMON_ARGS, - variant="TP4", + extra_args=COMMON_ARGS + TP_EAGLE_ARGS, + variant="TP4+EAGLE3", ), ModelLaunchSettings( MODEL_PATH, tp_size=4, - extra_args=COMMON_ARGS + ["--dp-size=4", "--enable-dp-attention"], - variant="TP4+DP4+DPA", + extra_args=COMMON_ARGS + + ["--dp-size=4", "--enable-dp-attention"] + + DP_EAGLE_ARGS, + variant="TP4+DP4+DPA+EAGLE3", ), ] diff --git a/test/registered/gb300/test_qwen35_fp8.py b/test/registered/gb300/test_qwen35_fp8.py index df364761a..9c8a21bae 100644 --- a/test/registered/gb300/test_qwen35_fp8.py +++ b/test/registered/gb300/test_qwen35_fp8.py @@ -6,7 +6,7 @@ from sglang.test.performance_test_runner import PerformanceTestParams from sglang.test.run_combined_tests import run_combined_tests from sglang.test.test_utils import ModelLaunchSettings -register_cuda_ci(est_time=7200, suite="nightly-4-gpu-gb300", nightly=True) +register_cuda_ci(est_time=7200, suite="nightly-4-gpu-gb300-qwen35-fp8", nightly=True) MODEL_PATH = "Qwen/Qwen3.5-397B-A17B-FP8" diff --git a/test/registered/gb300/test_qwen35_nvfp4.py b/test/registered/gb300/test_qwen35_nvfp4.py index a2866b7cd..abaf640f4 100644 --- a/test/registered/gb300/test_qwen35_nvfp4.py +++ b/test/registered/gb300/test_qwen35_nvfp4.py @@ -7,7 +7,10 @@ from sglang.test.run_combined_tests import run_combined_tests from sglang.test.test_utils import ModelLaunchSettings register_cuda_ci( - est_time=7200, suite="nightly-4-gpu-gb300", nightly=True, disabled="not needed" + est_time=7200, + suite="nightly-4-gpu-gb300-qwen35-nvfp4", + nightly=True, + disabled="not needed", ) MODEL_PATH = "nvidia/Qwen3.5-397B-A17B-NVFP4" diff --git a/test/run_suite.py b/test/run_suite.py index aa42c158f..e4c2350c4 100644 --- a/test/run_suite.py +++ b/test/run_suite.py @@ -128,8 +128,15 @@ NIGHTLY_SUITES = { "nightly-eval-vlm-2-gpu", "nightly-perf-text-2-gpu", "nightly-perf-vlm-2-gpu", - # GB300 (4x GB300 NVL4) nightly suite + # GB300 (4x GB300 NVL4) nightly suites "nightly-4-gpu-gb300", + "nightly-4-gpu-gb300-deepseek-v4-pro-fp4", + "nightly-4-gpu-gb300-glm5-fp8", + "nightly-4-gpu-gb300-glm5-nvfp4", + "nightly-4-gpu-gb300-kimi-k25", + "nightly-4-gpu-gb300-kimi-k25-nvfp4", + "nightly-4-gpu-gb300-qwen35-fp8", + "nightly-4-gpu-gb300-qwen35-nvfp4", # Nightly precision regression (per-layer hidden state comparison) "nightly-precision-8-gpu-h200", ],