feat(ci): add GB300 nightly benchmark test suites (#21487)

Co-authored-by: Claude Opus 4.6 (1M context) <noreply@anthropic.com>

python/sglang/test/accuracy_test_runner.py

@@ -150,6 +150,288 @@ def _run_simple_eval(
             kill_process_tree(process.pid)
 
 
+# Cached uv venv for NeMo Skills (persists across variants within a process).
+_nemo_venv_dir: Optional[str] = None
+_nemo_data_prepared: set = set()
+
+
+def _get_nemo_venv() -> Tuple[str, dict]:
+    """Get or create a uv venv with nemo_skills installed.
+
+    Returns (venv_python_path, env_dict) reusable across calls.
+    """
+    import os
+    import subprocess
+    import tempfile
+
+    global _nemo_venv_dir
+
+    if _nemo_venv_dir is not None:
+        venv_python = f"{_nemo_venv_dir}/venv/bin/python"
+        env = {
+            **dict(os.environ),
+            "NEMO_SKILLS_DISABLE_UNCOMMITTED_CHANGES_CHECK": "1",
+            "OPENAI_API_KEY": "dummy",
+            "VIRTUAL_ENV": f"{_nemo_venv_dir}/venv",
+            "PATH": f"{_nemo_venv_dir}/venv/bin:" + os.environ.get("PATH", ""),
+        }
+        return venv_python, env
+
+    _nemo_venv_dir = tempfile.mkdtemp(prefix="nemo_skills_")
+    print(f"Creating NeMo Skills venv in {_nemo_venv_dir}...")
+
+    # Create venv
+    result = subprocess.run(
+        ["uv", "venv", f"{_nemo_venv_dir}/venv", "--python", "3.12"],
+        capture_output=True,
+        text=True,
+    )
+    if result.returncode != 0:
+        subprocess.run(
+            ["uv", "venv", f"{_nemo_venv_dir}/venv"],
+            capture_output=True,
+            text=True,
+        )
+
+    # Install nemo_skills
+    print("Installing nemo_skills...")
+    pip_result = subprocess.run(
+        [
+            "uv",
+            "pip",
+            "install",
+            "--python",
+            f"{_nemo_venv_dir}/venv/bin/python",
+            "git+https://github.com/NVIDIA/NeMo-Skills.git",
+        ],
+        capture_output=True,
+        text=True,
+        timeout=300,
+    )
+    if pip_result.returncode != 0:
+        raise RuntimeError(f"Failed to install nemo_skills: {pip_result.stderr[-500:]}")
+
+    print("NeMo Skills installed successfully")
+    return _get_nemo_venv()
+
+
+def _ensure_nemo_data_prepared(
+    venv_python: str, env: dict, dataset: str
+) -> Tuple[bool, Optional[str]]:
+    """Prepare NeMo Skills dataset data if not already done.
+
+    Uses the venv python so data lands inside the venv's nemo_skills package.
+    """
+    import subprocess
+
+    if dataset in _nemo_data_prepared:
+        return True, None
+
+    print(f"Preparing {dataset} data (this may take a few minutes for VLM datasets)...")
+    result = subprocess.run(
+        [venv_python, "-m", "nemo_skills.dataset.prepare", dataset],
+        text=True,
+        timeout=600,
+        env=env,
+    )
+    if result.returncode != 0:
+        return False, f"Failed to prepare {dataset} data (exit {result.returncode})"
+
+    _nemo_data_prepared.add(dataset)
+    return True, None
+
+
+def _run_nemo_skills_eval(
+    model: ModelLaunchSettings,
+    base_url: str,
+    dataset: str,
+    max_tokens: Optional[int] = None,
+    repeat: Optional[int] = None,
+    temperature: Optional[float] = None,
+    top_p: Optional[float] = None,
+) -> Tuple[bool, Optional[str], Optional[dict]]:
+    """Run evaluation using NeMo Skills (ns eval) for benchmarks like mmmu-pro.
+
+    Uses an isolated uv venv (shared across variants) so nemo_skills dependencies
+    don't interfere with the system python / sglang server.
+
+    Returns:
+        Tuple of (success, error_message, metrics_dict)
+    """
+    import subprocess
+    import tempfile
+
+    process = None
+    try:
+        # Get or create the shared venv (once per process)
+        venv_python, env = _get_nemo_venv()
+
+        # Prepare dataset (once per process, cached)
+        ok, err = _ensure_nemo_data_prepared(venv_python, env, dataset)
+        if not ok:
+            return False, err, None
+
+        process = popen_launch_server(
+            model.model_path,
+            base_url,
+            other_args=model.extra_args,
+            timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
+            env=model.env,
+        )
+
+        port = int(base_url.split(":")[-1])
+        server_address = f"http://127.0.0.1:{port}/v1"
+        repeat_val = repeat or 1
+        max_tokens_val = max_tokens or 32768
+        benchmark_spec = f"{dataset}:{repeat_val}"
+
+        # Build ns eval command using venv python
+        # Note: nemo_skills.pipeline.eval requires the "eval" subcommand
+        output_dir = tempfile.mkdtemp(prefix="ns_eval_output_")
+        cmd = [
+            venv_python,
+            "-m",
+            "nemo_skills.pipeline.eval",
+            "eval",
+            f"--benchmarks={benchmark_spec}",
+            "--server_type=sglang",
+            f"--model={model.model_path}",
+            f"--server_address={server_address}",
+            f"--output_dir={output_dir}",
+            f"++inference.tokens_to_generate={max_tokens_val}",
+        ]
+
+        if temperature is not None:
+            cmd.append(f"++inference.temperature={temperature}")
+        if top_p is not None:
+            cmd.append(f"++inference.top_p={top_p}")
+
+        # Add VLM-specific config
+        if dataset in ("mmmu-pro", "mmmu_pro"):
+            cmd.append("++prompt_config=vlm/mmmu-pro")
+            cmd.append("++max_concurrent_requests=512")
+            cmd.append("++max_samples=500")
+
+        print(f"Running: {' '.join(cmd)}")
+        eval_result = subprocess.run(
+            cmd,
+            capture_output=True,
+            text=True,
+            timeout=7200,
+            env=env,
+        )
+
+        print(eval_result.stdout[-2000:] if eval_result.stdout else "(no stdout)")
+        if eval_result.stderr:
+            print(eval_result.stderr[-1000:])
+
+        if eval_result.returncode != 0:
+            return (
+                False,
+                f"ns eval failed (exit {eval_result.returncode}): {eval_result.stderr[-500:]}",
+                None,
+            )
+
+        # Parse results
+        summarize_result = subprocess.run(
+            [
+                venv_python,
+                "-m",
+                "nemo_skills.pipeline.summarize_results",
+                f"{output_dir}/eval-results",
+            ],
+            capture_output=True,
+            text=True,
+            timeout=60,
+            env=env,
+        )
+
+        output = summarize_result.stdout + "\n" + eval_result.stdout
+        print(f"Summary: {summarize_result.stdout[:1000]}")
+
+        # Parse accuracy from output (format varies, look for common patterns)
+        import re
+
+        score = None
+        for line in output.split("\n"):
+            match = re.search(r"(?:accuracy|score)[:\s]+([0-9.]+)", line, re.IGNORECASE)
+            if match:
+                score = float(match.group(1))
+
+        if score is None:
+            # Try to find it in eval-results directory
+            import glob
+            import json
+
+            for result_file in glob.glob(
+                f"{output_dir}/eval-results/**/*.json", recursive=True
+            ):
+                try:
+                    with open(result_file) as f:
+                        data = json.load(f)
+                    if isinstance(data, dict):
+                        score = (
+                            data.get("accuracy")
+                            or data.get("score")
+                            or data.get("mean_score")
+                        )
+                        if score is not None:
+                            break
+                except (json.JSONDecodeError, KeyError):
+                    continue
+
+        if score is None:
+            # Last resort: compute accuracy directly from JSONL output
+            import glob
+            import json
+
+            for jsonl_file in sorted(
+                glob.glob(f"{output_dir}/eval-results/**/*.jsonl*", recursive=True)
+            ):
+                correct = 0
+                total = 0
+                try:
+                    with open(jsonl_file) as f:
+                        for line in f:
+                            line = line.strip()
+                            if not line:
+                                continue
+                            entry = json.loads(line)
+                            expected = entry.get("expected_answer", "")
+                            generation = entry.get("generation", "")
+                            # Extract "Answer: X" from the end of generation
+                            answer_match = re.search(
+                                r"Answer:\s*([A-J])", generation, re.IGNORECASE
+                            )
+                            if answer_match:
+                                predicted = answer_match.group(1).upper()
+                                if predicted == expected.upper():
+                                    correct += 1
+                            total += 1
+                except (json.JSONDecodeError, KeyError, OSError):
+                    continue
+                if total > 0:
+                    score = correct / total
+                    print(
+                        f"Computed accuracy from {jsonl_file}: "
+                        f"{correct}/{total} = {score:.4f}"
+                    )
+                    break
+
+        if score is None:
+            return False, "Could not parse accuracy from ns eval output", None
+
+        return True, None, {"score": score}
+
+    except subprocess.TimeoutExpired:
+        return False, "NeMo Skills eval timed out", None
+    except Exception as e:
+        return False, f"NeMo Skills eval exception: {str(e)}", None
+    finally:
+        if process:
+            kill_process_tree(process.pid)
+
+
 def _run_few_shot_eval(
     model: ModelLaunchSettings,
     base_url: str,
@@ -224,13 +506,24 @@ def run_accuracy_test(
     print(f"{'='*60}\n")
 
     # Run evaluation based on dataset type
-    # Use few_shot_eval for gsm8k by default for backward compatibility.
+    # - NeMo Skills: mmmu-pro (and other VLM evals needing ns eval)
-    # Use simple_eval when any extended params are set that few_shot_eval doesn't support.
+    # - few_shot_eval: gsm8k (default, backward compatible)
+    # - simple_eval: everything else (gpqa, mmmu, etc.)
     has_extended_params = any(
         getattr(params, field) is not None
         for field in ("thinking_mode", "temperature", "top_p", "top_k", "repeat")
     )
-    if params.dataset == "gsm8k" and not has_extended_params:
+    if params.dataset in ("mmmu-pro", "mmmu_pro"):
+        success, error, metrics = _run_nemo_skills_eval(
+            model=model,
+            base_url=base_url,
+            dataset="mmmu-pro",
+            max_tokens=params.max_tokens,
+            repeat=params.repeat or 1,
+            temperature=params.temperature,
+            top_p=params.top_p,
+        )
+    elif params.dataset == "gsm8k" and not has_extended_params:
         success, error, metrics = _run_few_shot_eval(
             model=model,
             base_url=base_url,

python/sglang/test/run_combined_tests.py

@@ -104,6 +104,7 @@ def run_combined_tests(
 
         model_result = {
             "model": model.model_path,
+            "variant": model.variant,
             "perf_result": None,
             "accuracy_result": None,
             "tool_call_result": None,
@@ -243,8 +244,9 @@ def run_combined_tests(
 
                 failed_test_str = ", ".join(failed_tests) if failed_tests else "unknown"
                 error_str = "; ".join(str(e) for e in r["errors"])
+                variant_str = f" [{r['variant']}]" if r.get("variant") else ""
                 failure_lines.append(
-                    f"  Model {i + 1} ({r['model']}): {failed_test_str} - {error_str}"
+                    f"  Model {i + 1} ({r['model']}{variant_str}): {failed_test_str} - {error_str}"
                 )
 
         failure_summary = "\n".join(failure_lines)

test/registered/gb300/test_deepseek_v32.py

@@ -0,0 +1,79 @@
+import unittest
+
+from sglang.test.accuracy_test_runner import AccuracyTestParams
+from sglang.test.ci.ci_register import register_cuda_ci
+from sglang.test.performance_test_runner import PerformanceTestParams
+from sglang.test.run_combined_tests import run_combined_tests
+from sglang.test.test_utils import ModelLaunchSettings
+
+register_cuda_ci(est_time=7200, suite="nightly-4-gpu-gb300", nightly=True)
+
+MODEL_PATH = "deepseek-ai/DeepSeek-V3.2"
+
+COMMON_ARGS = [
+    "--trust-remote-code",
+    "--reasoning-parser=deepseek-v3",
+    "--tool-call-parser=deepseekv32",
+    "--mem-fraction-static=0.8",
+    "--enable-metrics",
+]
+
+MTP_ARGS = [
+    "--speculative-algorithm=EAGLE",
+    "--speculative-num-steps=3",
+    "--speculative-eagle-topk=1",
+    "--speculative-num-draft-tokens=4",
+]
+
+
+class TestDeepseekV32(unittest.TestCase):
+    """DeepSeek V3.2 on GB300 (4x B200 NVL4, tp=4)."""
+
+    def test_deepseek_v32(self):
+        variants = [
+            ModelLaunchSettings(
+                MODEL_PATH,
+                tp_size=4,
+                extra_args=COMMON_ARGS,
+                variant="TP4",
+            ),
+            ModelLaunchSettings(
+                MODEL_PATH,
+                tp_size=4,
+                extra_args=COMMON_ARGS
+                + [
+                    "--dp-size=4",
+                    "--ep-size=4",
+                    "--enable-dp-attention",
+                ],
+                variant="TP4+DP4+DPA",
+            ),
+            ModelLaunchSettings(
+                MODEL_PATH,
+                tp_size=4,
+                extra_args=COMMON_ARGS
+                + [
+                    "--dp-size=4",
+                    "--ep-size=4",
+                    "--enable-dp-attention",
+                ]
+                + MTP_ARGS,
+                variant="TP4+DP4+DPA+MTP",
+                env={"SGLANG_ENABLE_SPEC_V2": "1"},
+            ),
+        ]
+
+        run_combined_tests(
+            models=variants,
+            test_name="DeepSeek-V3.2",
+            accuracy_params=AccuracyTestParams(
+                dataset="gsm8k", baseline_accuracy=0.935
+            ),
+            performance_params=PerformanceTestParams(
+                profile_dir="performance_profiles_gb300",
+            ),
+        )
+
+
+if __name__ == "__main__":
+    unittest.main()

test/registered/gb300/test_deepseek_v32_nvfp4.py

@@ -0,0 +1,82 @@
+import unittest
+
+from sglang.test.accuracy_test_runner import AccuracyTestParams
+from sglang.test.ci.ci_register import register_cuda_ci
+from sglang.test.performance_test_runner import PerformanceTestParams
+from sglang.test.run_combined_tests import run_combined_tests
+from sglang.test.test_utils import ModelLaunchSettings
+
+register_cuda_ci(est_time=7200, suite="nightly-4-gpu-gb300", nightly=True)
+
+MODEL_PATH = "nvidia/DeepSeek-V3.2-NVFP4"
+
+COMMON_ARGS = [
+    "--trust-remote-code",
+    "--reasoning-parser=deepseek-v3",
+    "--tool-call-parser=deepseekv32",
+    "--quantization=modelopt_fp4",
+    "--moe-runner-backend=flashinfer_trtllm",
+    "--kv-cache-dtype=bfloat16",
+    "--mem-fraction-static=0.8",
+    "--enable-metrics",
+]
+
+MTP_ARGS = [
+    "--speculative-algorithm=EAGLE",
+    "--speculative-num-steps=3",
+    "--speculative-eagle-topk=1",
+    "--speculative-num-draft-tokens=4",
+]
+
+
+class TestDeepseekV32Nvfp4(unittest.TestCase):
+    """DeepSeek V3.2 NVFP4 on GB300 (4x B200 NVL4, tp=4)."""
+
+    def test_deepseek_v32_nvfp4(self):
+        variants = [
+            ModelLaunchSettings(
+                MODEL_PATH,
+                tp_size=4,
+                extra_args=COMMON_ARGS,
+                variant="TP4",
+            ),
+            ModelLaunchSettings(
+                MODEL_PATH,
+                tp_size=4,
+                extra_args=COMMON_ARGS
+                + [
+                    "--dp-size=4",
+                    "--ep-size=4",
+                    "--enable-dp-attention",
+                ],
+                variant="TP4+DP4+DPA",
+            ),
+            ModelLaunchSettings(
+                MODEL_PATH,
+                tp_size=4,
+                extra_args=COMMON_ARGS
+                + [
+                    "--dp-size=4",
+                    "--ep-size=4",
+                    "--enable-dp-attention",
+                ]
+                + MTP_ARGS,
+                variant="TP4+DP4+DPA+MTP",
+                env={"SGLANG_ENABLE_SPEC_V2": "1"},
+            ),
+        ]
+
+        run_combined_tests(
+            models=variants,
+            test_name="DeepSeek-V3.2-NVFP4",
+            accuracy_params=AccuracyTestParams(
+                dataset="gsm8k", baseline_accuracy=0.935
+            ),
+            performance_params=PerformanceTestParams(
+                profile_dir="performance_profiles_gb300",
+            ),
+        )
+
+
+if __name__ == "__main__":
+    unittest.main()

test/registered/gb300/test_glm5_fp8.py

@@ -0,0 +1,68 @@
+import unittest
+
+from sglang.test.accuracy_test_runner import AccuracyTestParams
+from sglang.test.ci.ci_register import register_cuda_ci
+from sglang.test.performance_test_runner import PerformanceTestParams
+from sglang.test.run_combined_tests import run_combined_tests
+from sglang.test.test_utils import ModelLaunchSettings
+
+register_cuda_ci(est_time=7200, suite="nightly-4-gpu-gb300", nightly=True)
+
+MODEL_PATH = "zai-org/GLM-5-FP8"
+
+COMMON_ARGS = [
+    "--trust-remote-code",
+    "--reasoning-parser=glm45",
+    "--tool-call-parser=glm47",
+    "--mem-fraction-static=0.9",
+    "--enable-metrics",
+]
+
+MTP_ARGS = [
+    "--speculative-algorithm=EAGLE",
+    "--speculative-num-steps=3",
+    "--speculative-eagle-topk=1",
+    "--speculative-num-draft-tokens=4",
+]
+
+
+class TestGlm5Fp8(unittest.TestCase):
+    """GLM-5 FP8 on GB300 (4x B200 NVL4, tp=4)."""
+
+    def test_glm5_fp8(self):
+        variants = [
+            ModelLaunchSettings(
+                MODEL_PATH,
+                tp_size=4,
+                extra_args=COMMON_ARGS,
+                variant="TP4",
+            ),
+            ModelLaunchSettings(
+                MODEL_PATH,
+                tp_size=4,
+                extra_args=COMMON_ARGS + ["--dp-size=4", "--enable-dp-attention"],
+                variant="TP4+DP4+DPA",
+            ),
+            ModelLaunchSettings(
+                MODEL_PATH,
+                tp_size=4,
+                extra_args=COMMON_ARGS
+                + ["--dp-size=4", "--enable-dp-attention"]
+                + MTP_ARGS,
+                variant="TP4+DP4+DPA+MTP",
+                env={"SGLANG_ENABLE_SPEC_V2": "1"},
+            ),
+        ]
+
+        run_combined_tests(
+            models=variants,
+            test_name="GLM-5-FP8",
+            accuracy_params=AccuracyTestParams(dataset="gsm8k", baseline_accuracy=0.92),
+            performance_params=PerformanceTestParams(
+                profile_dir="performance_profiles_gb300",
+            ),
+        )
+
+
+if __name__ == "__main__":
+    unittest.main()

test/registered/gb300/test_glm5_nvfp4.py

@@ -0,0 +1,71 @@
+import unittest
+
+from sglang.test.accuracy_test_runner import AccuracyTestParams
+from sglang.test.ci.ci_register import register_cuda_ci
+from sglang.test.performance_test_runner import PerformanceTestParams
+from sglang.test.run_combined_tests import run_combined_tests
+from sglang.test.test_utils import ModelLaunchSettings
+
+register_cuda_ci(est_time=7200, suite="nightly-4-gpu-gb300", nightly=True)
+
+MODEL_PATH = "nvidia/GLM-5-NVFP4"
+
+COMMON_ARGS = [
+    "--trust-remote-code",
+    "--reasoning-parser=glm45",
+    "--tool-call-parser=glm47",
+    "--quantization=modelopt_fp4",
+    "--moe-runner-backend=flashinfer_trtllm",
+    "--kv-cache-dtype=bfloat16",
+    "--mem-fraction-static=0.9",
+    "--enable-metrics",
+]
+
+MTP_ARGS = [
+    "--speculative-algorithm=EAGLE",
+    "--speculative-num-steps=3",
+    "--speculative-eagle-topk=1",
+    "--speculative-num-draft-tokens=4",
+]
+
+
+class TestGlm5Nvfp4(unittest.TestCase):
+    """GLM-5 NVFP4 on GB300 (4x B200 NVL4, tp=4)."""
+
+    def test_glm5_nvfp4(self):
+        variants = [
+            ModelLaunchSettings(
+                MODEL_PATH,
+                tp_size=4,
+                extra_args=COMMON_ARGS,
+                variant="TP4",
+            ),
+            ModelLaunchSettings(
+                MODEL_PATH,
+                tp_size=4,
+                extra_args=COMMON_ARGS + ["--dp-size=4", "--enable-dp-attention"],
+                variant="TP4+DP4+DPA",
+            ),
+            ModelLaunchSettings(
+                MODEL_PATH,
+                tp_size=4,
+                extra_args=COMMON_ARGS
+                + ["--dp-size=4", "--enable-dp-attention"]
+                + MTP_ARGS,
+                variant="TP4+DP4+DPA+MTP",
+                env={"SGLANG_ENABLE_SPEC_V2": "1"},
+            ),
+        ]
+
+        run_combined_tests(
+            models=variants,
+            test_name="GLM-5-NVFP4",
+            accuracy_params=AccuracyTestParams(dataset="gsm8k", baseline_accuracy=0.92),
+            performance_params=PerformanceTestParams(
+                profile_dir="performance_profiles_gb300",
+            ),
+        )
+
+
+if __name__ == "__main__":
+    unittest.main()

test/registered/gb300/test_kimi_k25.py

@@ -0,0 +1,58 @@
+import unittest
+
+from sglang.test.accuracy_test_runner import AccuracyTestParams
+from sglang.test.ci.ci_register import register_cuda_ci
+from sglang.test.performance_test_runner import PerformanceTestParams
+from sglang.test.run_combined_tests import run_combined_tests
+from sglang.test.test_utils import ModelLaunchSettings
+
+register_cuda_ci(est_time=7200, suite="nightly-4-gpu-gb300", nightly=True)
+
+MODEL_PATH = "moonshotai/Kimi-K2.5"
+
+COMMON_ARGS = [
+    "--trust-remote-code",
+    "--reasoning-parser=kimi_k2",
+    "--tool-call-parser=kimi_k2",
+    "--mem-fraction-static=0.8",
+    "--enable-multimodal",
+    "--enable-metrics",
+]
+
+
+class TestKimiK25(unittest.TestCase):
+    """Kimi-K2.5 (native INT4) on GB300 (4x B200 NVL4, tp=4).
+
+    No EAGLE/MTP support for Kimi-K2.5 — only TP and TP+DP+DPA variants.
+    """
+
+    def test_kimi_k25(self):
+        variants = [
+            ModelLaunchSettings(
+                MODEL_PATH,
+                tp_size=4,
+                extra_args=COMMON_ARGS,
+                variant="TP4",
+            ),
+            ModelLaunchSettings(
+                MODEL_PATH,
+                tp_size=4,
+                extra_args=COMMON_ARGS + ["--dp-size=4", "--enable-dp-attention"],
+                variant="TP4+DP4+DPA",
+            ),
+        ]
+
+        run_combined_tests(
+            models=variants,
+            test_name="Kimi-K2.5",
+            accuracy_params=AccuracyTestParams(
+                dataset="mmmu-pro", baseline_accuracy=0.69, repeat=1, max_tokens=32768
+            ),
+            performance_params=PerformanceTestParams(
+                profile_dir="performance_profiles_gb300",
+            ),
+        )
+
+
+if __name__ == "__main__":
+    unittest.main()

test/registered/gb300/test_kimi_k25_nvfp4.py

@@ -0,0 +1,61 @@
+import unittest
+
+from sglang.test.accuracy_test_runner import AccuracyTestParams
+from sglang.test.ci.ci_register import register_cuda_ci
+from sglang.test.performance_test_runner import PerformanceTestParams
+from sglang.test.run_combined_tests import run_combined_tests
+from sglang.test.test_utils import ModelLaunchSettings
+
+register_cuda_ci(est_time=7200, suite="nightly-4-gpu-gb300", nightly=True)
+
+MODEL_PATH = "nvidia/Kimi-K2.5-NVFP4"
+
+COMMON_ARGS = [
+    "--trust-remote-code",
+    "--reasoning-parser=kimi_k2",
+    "--tool-call-parser=kimi_k2",
+    "--quantization=modelopt_fp4",
+    "--attention-backend=trtllm_mla",
+    "--moe-runner-backend=flashinfer_trtllm",
+    "--mem-fraction-static=0.8",
+    "--enable-multimodal",
+    "--enable-metrics",
+]
+
+
+class TestKimiK25Nvfp4(unittest.TestCase):
+    """Kimi-K2.5 NVFP4 on GB300 (4x B200 NVL4, tp=4).
+
+    No EAGLE/MTP support for Kimi-K2.5 — only TP and TP+DP+DPA variants.
+    """
+
+    def test_kimi_k25_nvfp4(self):
+        variants = [
+            ModelLaunchSettings(
+                MODEL_PATH,
+                tp_size=4,
+                extra_args=COMMON_ARGS,
+                variant="TP4",
+            ),
+            ModelLaunchSettings(
+                MODEL_PATH,
+                tp_size=4,
+                extra_args=COMMON_ARGS + ["--dp-size=4", "--enable-dp-attention"],
+                variant="TP4+DP4+DPA",
+            ),
+        ]
+
+        run_combined_tests(
+            models=variants,
+            test_name="Kimi-K2.5-NVFP4",
+            accuracy_params=AccuracyTestParams(
+                dataset="mmmu-pro", baseline_accuracy=0.69, repeat=1, max_tokens=32768
+            ),
+            performance_params=PerformanceTestParams(
+                profile_dir="performance_profiles_gb300",
+            ),
+        )
+
+
+if __name__ == "__main__":
+    unittest.main()

test/registered/gb300/test_qwen35_fp8.py

@@ -0,0 +1,75 @@
+import unittest
+
+from sglang.test.accuracy_test_runner import AccuracyTestParams
+from sglang.test.ci.ci_register import register_cuda_ci
+from sglang.test.performance_test_runner import PerformanceTestParams
+from sglang.test.run_combined_tests import run_combined_tests
+from sglang.test.test_utils import ModelLaunchSettings
+
+register_cuda_ci(est_time=7200, suite="nightly-4-gpu-gb300", nightly=True)
+
+MODEL_PATH = "Qwen/Qwen3.5-397B-A17B-FP8"
+
+COMMON_ARGS = [
+    "--trust-remote-code",
+    "--reasoning-parser=qwen3",
+    "--tool-call-parser=qwen3_coder",
+    "--enable-flashinfer-allreduce-fusion",
+    "--attention-backend=trtllm_mha",
+    "--mem-fraction-static=0.8",
+    "--enable-multimodal",
+    "--enable-metrics",
+]
+
+MTP_ARGS = [
+    "--speculative-algorithm=EAGLE",
+    "--speculative-num-steps=3",
+    "--speculative-eagle-topk=1",
+    "--speculative-num-draft-tokens=4",
+    "--mamba-scheduler-strategy=extra_buffer",
+    "--page-size=64",
+]
+
+
+class TestQwen35Fp8(unittest.TestCase):
+    """Qwen3.5-397B FP8 on GB300 (4x B200 NVL4, tp=4)."""
+
+    def test_qwen35_fp8(self):
+        variants = [
+            ModelLaunchSettings(
+                MODEL_PATH,
+                tp_size=4,
+                extra_args=COMMON_ARGS,
+                variant="TP4",
+            ),
+            ModelLaunchSettings(
+                MODEL_PATH,
+                tp_size=4,
+                extra_args=COMMON_ARGS + ["--dp-size=4", "--enable-dp-attention"],
+                variant="TP4+DP4+DPA",
+            ),
+            ModelLaunchSettings(
+                MODEL_PATH,
+                tp_size=4,
+                extra_args=COMMON_ARGS
+                + ["--dp-size=4", "--enable-dp-attention"]
+                + MTP_ARGS,
+                variant="TP4+DP4+DPA+MTP",
+                env={"SGLANG_ENABLE_SPEC_V2": "1"},
+            ),
+        ]
+
+        run_combined_tests(
+            models=variants,
+            test_name="Qwen3.5-397B-FP8",
+            accuracy_params=AccuracyTestParams(
+                dataset="mmmu-pro", baseline_accuracy=0.78, repeat=1, max_tokens=32768
+            ),
+            performance_params=PerformanceTestParams(
+                profile_dir="performance_profiles_gb300",
+            ),
+        )
+
+
+if __name__ == "__main__":
+    unittest.main()

test/registered/gb300/test_qwen35_nvfp4.py

@@ -0,0 +1,79 @@
+import unittest
+
+from sglang.test.accuracy_test_runner import AccuracyTestParams
+from sglang.test.ci.ci_register import register_cuda_ci
+from sglang.test.performance_test_runner import PerformanceTestParams
+from sglang.test.run_combined_tests import run_combined_tests
+from sglang.test.test_utils import ModelLaunchSettings
+
+register_cuda_ci(est_time=7200, suite="nightly-4-gpu-gb300", nightly=True)
+
+MODEL_PATH = "nvidia/Qwen3.5-397B-A17B-NVFP4"
+
+COMMON_ARGS = [
+    "--trust-remote-code",
+    "--reasoning-parser=qwen3",
+    "--tool-call-parser=qwen3_coder",
+    "--quantization=modelopt_fp4",
+    "--fp4-gemm-backend=flashinfer_cutlass",
+    "--moe-runner-backend=flashinfer_trtllm",
+    "--kv-cache-dtype=fp8_e4m3",
+    "--enable-flashinfer-allreduce-fusion",
+    "--attention-backend=trtllm_mha",
+    "--mem-fraction-static=0.8",
+    "--enable-multimodal",
+    "--enable-metrics",
+]
+
+MTP_ARGS = [
+    "--speculative-algorithm=EAGLE",
+    "--speculative-num-steps=3",
+    "--speculative-eagle-topk=1",
+    "--speculative-num-draft-tokens=4",
+    "--mamba-scheduler-strategy=extra_buffer",
+    "--page-size=64",
+]
+
+
+class TestQwen35Nvfp4(unittest.TestCase):
+    """Qwen3.5-397B NVFP4 on GB300 (4x B200 NVL4, tp=4)."""
+
+    def test_qwen35_nvfp4(self):
+        variants = [
+            ModelLaunchSettings(
+                MODEL_PATH,
+                tp_size=4,
+                extra_args=COMMON_ARGS,
+                variant="TP4",
+            ),
+            ModelLaunchSettings(
+                MODEL_PATH,
+                tp_size=4,
+                extra_args=COMMON_ARGS + ["--dp-size=4", "--enable-dp-attention"],
+                variant="TP4+DP4+DPA",
+            ),
+            ModelLaunchSettings(
+                MODEL_PATH,
+                tp_size=4,
+                extra_args=COMMON_ARGS
+                + ["--dp-size=4", "--enable-dp-attention"]
+                + MTP_ARGS,
+                variant="TP4+DP4+DPA+MTP",
+                env={"SGLANG_ENABLE_SPEC_V2": "1"},
+            ),
+        ]
+
+        run_combined_tests(
+            models=variants,
+            test_name="Qwen3.5-397B-NVFP4",
+            accuracy_params=AccuracyTestParams(
+                dataset="mmmu-pro", baseline_accuracy=0.78, repeat=1, max_tokens=32768
+            ),
+            performance_params=PerformanceTestParams(
+                profile_dir="performance_profiles_gb300",
+            ),
+        )
+
+
+if __name__ == "__main__":
+    unittest.main()

test/run_suite.py

@@ -84,6 +84,8 @@ NIGHTLY_SUITES = {
         "nightly-eval-vlm-2-gpu",
         "nightly-perf-text-2-gpu",
         "nightly-perf-vlm-2-gpu",
+        # GB300 (4x B200 NVL4) nightly suite
+        "nightly-4-gpu-gb300",
     ],
     HWBackend.AMD: [
         "nightly-amd",