Add DeepSeek V4 Pro GB300 nightly and expand Kimi K25 nightly test (#28103)

.github/workflows/nightly-test-nvidia.yml

@@ -539,7 +539,20 @@ jobs:
   # GB300 (Grace-Blackwell NVL4) performance tests - 4 GPU (ARM64)
   nightly-test-perf-4-gpu-gb300:
     if: github.repository == 'sgl-project/sglang' && (inputs.job_filter == '' || inputs.job_filter == 'all' || inputs.job_filter == 'nightly-test-perf-4-gpu-gb300')
-    runs-on: 4-gpu-gb300
+    name: nightly-test-perf-4-gpu-gb300 (${{ matrix.model }})
+    runs-on: 4-gpu-gb300-nightly
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - model: glm5-nvfp4
+            suite: nightly-4-gpu-gb300-glm5-nvfp4
+          - model: qwen35-fp8
+            suite: nightly-4-gpu-gb300-qwen35-fp8
+          - model: deepseek-v4-pro-fp4
+            suite: nightly-4-gpu-gb300-deepseek-v4-pro-fp4
+          - model: kimi-k25-nvfp4
+            suite: nightly-4-gpu-gb300-kimi-k25-nvfp4
     steps:
       - name: Checkout code
         uses: actions/checkout@v4
@@ -549,8 +562,10 @@ jobs:
       - uses: ./.github/actions/check-maintenance
 
       - name: Install dependencies
+        env:
+          GRACE_BLACKWELL: "1"
         run: |
-          bash scripts/ci/cuda/ci_install_dependency.sh
+          bash scripts/ci/cuda/ci_install_deepep.sh
 
       - name: Run test
         timeout-minutes: 600
@@ -560,7 +575,7 @@ jobs:
           GPU_CONFIG: "4-gpu-gb300"
         run: |
           cd test
-          python3 run_suite.py --hw cuda --suite nightly-4-gpu-gb300 --nightly --continue-on-error --timeout-per-file 7200
+          python3 run_suite.py --hw cuda --suite ${{ matrix.suite }} --nightly --continue-on-error --timeout-per-file 7200
 
       - name: Publish traces to storage repo
         if: always()

test/registered/gb300/test_deepseek_v4_pro_fp4.py

@@ -0,0 +1,152 @@
+import unittest
+
+from sglang.test.accuracy_test_runner import AccuracyTestParams
+from sglang.test.ci.ci_register import register_cuda_ci
+from sglang.test.performance_test_runner import PerformanceTestParams
+from sglang.test.run_combined_tests import run_combined_tests
+from sglang.test.test_utils import ModelLaunchSettings
+
+register_cuda_ci(
+    est_time=7200, suite="nightly-4-gpu-gb300-deepseek-v4-pro-fp4", nightly=True
+)
+
+MODEL_PATH = "deepseek-ai/DeepSeek-V4-Pro"
+SERVER_LAUNCH_TIMEOUT = 3600
+
+DEEPEP_CONFIG = '{"normal_dispatch":{"num_sms":96},"normal_combine":{"num_sms":96}}'
+
+LOW_LATENCY_ARGS = [
+    "--trust-remote-code",
+    "--moe-runner-backend",
+    "flashinfer_mxfp4",
+    "--speculative-algorithm",
+    "EAGLE",
+    "--speculative-num-steps",
+    "3",
+    "--speculative-eagle-topk",
+    "1",
+    "--speculative-num-draft-tokens",
+    "4",
+    "--chunked-prefill-size",
+    "8192",
+    "--disable-flashinfer-autotune",
+    "--swa-full-tokens-ratio",
+    "0.1",
+    "--mem-fraction-static",
+    "0.85",
+]
+
+BALANCED_ARGS = [
+    "--trust-remote-code",
+    "--dp",
+    "4",
+    "--enable-dp-attention",
+    "--moe-a2a-backend",
+    "deepep",
+    "--speculative-algorithm",
+    "EAGLE",
+    "--speculative-num-steps",
+    "1",
+    "--speculative-eagle-topk",
+    "1",
+    "--speculative-num-draft-tokens",
+    "2",
+    "--mem-fraction-static",
+    "0.85",
+    "--cuda-graph-max-bs",
+    "128",
+    "--max-running-requests",
+    "256",
+    "--deepep-config",
+    DEEPEP_CONFIG,
+]
+
+HIGH_THROUGHPUT_ARGS = [
+    "--trust-remote-code",
+    "--dp",
+    "4",
+    "--enable-dp-attention",
+    "--moe-a2a-backend",
+    "megamoe",
+    "--mem-fraction-static",
+    "0.85",
+    "--cuda-graph-max-bs",
+    "128",
+    "--max-running-requests",
+    "256",
+]
+
+BALANCED_ENV = {
+    "SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK": "256",
+}
+
+HIGH_THROUGHPUT_ENV = {
+    "SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK": "8320",
+}
+
+PERFORMANCE_BATCH_SIZES = {
+    "low-latency": [1, 4, 16],
+    "balanced": [64],
+    "high-throughput": [128],
+}
+
+
+class TestDeepSeekV4ProFp4(unittest.TestCase):
+    """DeepSeek-V4-Pro FP4 on GB300 (4x B200 NVL4, tp=4)."""
+
+    def test_deepseek_v4_pro_fp4(self):
+        variants = [
+            ModelLaunchSettings(
+                MODEL_PATH,
+                tp_size=4,
+                extra_args=LOW_LATENCY_ARGS,
+                variant="low-latency",
+                launch_timeout=SERVER_LAUNCH_TIMEOUT,
+            ),
+            ModelLaunchSettings(
+                MODEL_PATH,
+                tp_size=4,
+                extra_args=BALANCED_ARGS,
+                env=BALANCED_ENV,
+                variant="balanced",
+                launch_timeout=SERVER_LAUNCH_TIMEOUT,
+            ),
+            ModelLaunchSettings(
+                MODEL_PATH,
+                tp_size=4,
+                extra_args=HIGH_THROUGHPUT_ARGS,
+                env=HIGH_THROUGHPUT_ENV,
+                variant="high-throughput",
+                launch_timeout=SERVER_LAUNCH_TIMEOUT,
+            ),
+        ]
+
+        failures = []
+        accuracy_params = AccuracyTestParams(
+            dataset="gsm8k",
+            baseline_accuracy=0.935,
+            temperature=1.0,
+            top_p=1.0,
+        )
+        for variant in variants:
+            try:
+                run_combined_tests(
+                    models=[variant],
+                    test_name=f"DeepSeek-V4-Pro-FP4 ({variant.variant})",
+                    accuracy_params=accuracy_params,
+                    performance_params=PerformanceTestParams(
+                        batch_sizes=PERFORMANCE_BATCH_SIZES[variant.variant],
+                        profile_dir="performance_profiles_gb300",
+                    ),
+                )
+            except AssertionError as e:
+                failures.append(f"{variant.variant}: {e}")
+
+        if failures:
+            raise AssertionError(
+                "DeepSeek-V4-Pro-FP4 failures:\n" + "\n".join(failures)
+            )
+
+
+if __name__ == "__main__":
+    unittest.main()

test/registered/gb300/test_glm5_fp8.py

@@ -7,7 +7,10 @@ from sglang.test.run_combined_tests import run_combined_tests
 from sglang.test.test_utils import ModelLaunchSettings
 
 register_cuda_ci(
-    est_time=7200, suite="nightly-4-gpu-gb300", nightly=True, disabled="not needed"
+    est_time=7200,
+    suite="nightly-4-gpu-gb300-glm5-fp8",
+    nightly=True,
+    disabled="not needed",
 )
 
 MODEL_PATH = "zai-org/GLM-5.1-FP8"

test/registered/gb300/test_glm5_nvfp4.py

@@ -6,7 +6,7 @@ from sglang.test.performance_test_runner import PerformanceTestParams
 from sglang.test.run_combined_tests import run_combined_tests
 from sglang.test.test_utils import ModelLaunchSettings
 
-register_cuda_ci(est_time=7200, suite="nightly-4-gpu-gb300", nightly=True)
+register_cuda_ci(est_time=7200, suite="nightly-4-gpu-gb300-glm5-nvfp4", nightly=True)
 
 MODEL_PATH = "nvidia/GLM-5-NVFP4"
 

test/registered/gb300/test_kimi_k25.py

@@ -7,7 +7,10 @@ from sglang.test.run_combined_tests import run_combined_tests
 from sglang.test.test_utils import ModelLaunchSettings
 
 register_cuda_ci(
-    est_time=7200, suite="nightly-4-gpu-gb300", nightly=True, disabled="not needed"
+    est_time=7200,
+    suite="nightly-4-gpu-gb300-kimi-k25",
+    nightly=True,
+    disabled="not needed",
 )
 
 MODEL_PATH = "moonshotai/Kimi-K2.5"

test/registered/gb300/test_kimi_k25_nvfp4.py

@@ -6,9 +6,12 @@ from sglang.test.performance_test_runner import PerformanceTestParams
 from sglang.test.run_combined_tests import run_combined_tests
 from sglang.test.test_utils import ModelLaunchSettings
 
-register_cuda_ci(est_time=7200, suite="nightly-4-gpu-gb300", nightly=True)
+register_cuda_ci(
+    est_time=7200, suite="nightly-4-gpu-gb300-kimi-k25-nvfp4", nightly=True
+)
 
 MODEL_PATH = "nvidia/Kimi-K2.5-NVFP4"
+DRAFT_MODEL_PATH = "lightseekorg/kimi-k2.5-eagle3-mla"
 
 COMMON_ARGS = [
     "--trust-remote-code",
@@ -19,30 +22,43 @@ COMMON_ARGS = [
     "--kv-cache-dtype=fp8_e4m3",
     "--moe-runner-backend=flashinfer_trtllm",
     "--mem-fraction-static=0.8",
-    "--enable-multimodal",
     "--enable-metrics",
+    "--speculative-algorithm=EAGLE3",
+    f"--speculative-draft-model-path={DRAFT_MODEL_PATH}",
+    "--speculative-draft-model-quantization=unquant",
+]
+
+TP_EAGLE_ARGS = [
+    "--speculative-num-steps=3",
+    "--speculative-eagle-topk=1",
+    "--speculative-num-draft-tokens=4",
+]
+
+DP_EAGLE_ARGS = [
+    "--speculative-num-steps=1",
+    "--speculative-eagle-topk=1",
+    "--speculative-num-draft-tokens=2",
 ]
 
 
 class TestKimiK25Nvfp4(unittest.TestCase):
-    """Kimi-K2.5 NVFP4 on GB300 (4x GB300 NVL4, tp=4).
+    """Kimi-K2.5 NVFP4 + EAGLE3 on GB300 (4x GB300 NVL4, tp=4)."""
-
-    No EAGLE/MTP support for Kimi-K2.5 — only TP and TP+DP+DPA variants.
-    """
 
     def test_kimi_k25_nvfp4(self):
         variants = [
             ModelLaunchSettings(
                 MODEL_PATH,
                 tp_size=4,
-                extra_args=COMMON_ARGS,
+                extra_args=COMMON_ARGS + TP_EAGLE_ARGS,
-                variant="TP4",
+                variant="TP4+EAGLE3",
             ),
             ModelLaunchSettings(
                 MODEL_PATH,
                 tp_size=4,
-                extra_args=COMMON_ARGS + ["--dp-size=4", "--enable-dp-attention"],
+                extra_args=COMMON_ARGS
-                variant="TP4+DP4+DPA",
+                + ["--dp-size=4", "--enable-dp-attention"]
+                + DP_EAGLE_ARGS,
+                variant="TP4+DP4+DPA+EAGLE3",
             ),
         ]
 

test/registered/gb300/test_qwen35_fp8.py

@@ -6,7 +6,7 @@ from sglang.test.performance_test_runner import PerformanceTestParams
 from sglang.test.run_combined_tests import run_combined_tests
 from sglang.test.test_utils import ModelLaunchSettings
 
-register_cuda_ci(est_time=7200, suite="nightly-4-gpu-gb300", nightly=True)
+register_cuda_ci(est_time=7200, suite="nightly-4-gpu-gb300-qwen35-fp8", nightly=True)
 
 MODEL_PATH = "Qwen/Qwen3.5-397B-A17B-FP8"
 

test/registered/gb300/test_qwen35_nvfp4.py

@@ -7,7 +7,10 @@ from sglang.test.run_combined_tests import run_combined_tests
 from sglang.test.test_utils import ModelLaunchSettings
 
 register_cuda_ci(
-    est_time=7200, suite="nightly-4-gpu-gb300", nightly=True, disabled="not needed"
+    est_time=7200,
+    suite="nightly-4-gpu-gb300-qwen35-nvfp4",
+    nightly=True,
+    disabled="not needed",
 )
 
 MODEL_PATH = "nvidia/Qwen3.5-397B-A17B-NVFP4"

test/run_suite.py

@@ -128,8 +128,15 @@ NIGHTLY_SUITES = {
         "nightly-eval-vlm-2-gpu",
         "nightly-perf-text-2-gpu",
         "nightly-perf-vlm-2-gpu",
-        # GB300 (4x GB300 NVL4) nightly suite
+        # GB300 (4x GB300 NVL4) nightly suites
         "nightly-4-gpu-gb300",
+        "nightly-4-gpu-gb300-deepseek-v4-pro-fp4",
+        "nightly-4-gpu-gb300-glm5-fp8",
+        "nightly-4-gpu-gb300-glm5-nvfp4",
+        "nightly-4-gpu-gb300-kimi-k25",
+        "nightly-4-gpu-gb300-kimi-k25-nvfp4",
+        "nightly-4-gpu-gb300-qwen35-fp8",
+        "nightly-4-gpu-gb300-qwen35-nvfp4",
         # Nightly precision regression (per-layer hidden state comparison)
         "nightly-precision-8-gpu-h200",
     ],