diff --git a/python/sglang/test/accuracy_test_runner.py b/python/sglang/test/accuracy_test_runner.py index 0cf007220..83bc6c524 100644 --- a/python/sglang/test/accuracy_test_runner.py +++ b/python/sglang/test/accuracy_test_runner.py @@ -150,6 +150,288 @@ def _run_simple_eval( kill_process_tree(process.pid) +# Cached uv venv for NeMo Skills (persists across variants within a process). +_nemo_venv_dir: Optional[str] = None +_nemo_data_prepared: set = set() + + +def _get_nemo_venv() -> Tuple[str, dict]: + """Get or create a uv venv with nemo_skills installed. + + Returns (venv_python_path, env_dict) reusable across calls. + """ + import os + import subprocess + import tempfile + + global _nemo_venv_dir + + if _nemo_venv_dir is not None: + venv_python = f"{_nemo_venv_dir}/venv/bin/python" + env = { + **dict(os.environ), + "NEMO_SKILLS_DISABLE_UNCOMMITTED_CHANGES_CHECK": "1", + "OPENAI_API_KEY": "dummy", + "VIRTUAL_ENV": f"{_nemo_venv_dir}/venv", + "PATH": f"{_nemo_venv_dir}/venv/bin:" + os.environ.get("PATH", ""), + } + return venv_python, env + + _nemo_venv_dir = tempfile.mkdtemp(prefix="nemo_skills_") + print(f"Creating NeMo Skills venv in {_nemo_venv_dir}...") + + # Create venv + result = subprocess.run( + ["uv", "venv", f"{_nemo_venv_dir}/venv", "--python", "3.12"], + capture_output=True, + text=True, + ) + if result.returncode != 0: + subprocess.run( + ["uv", "venv", f"{_nemo_venv_dir}/venv"], + capture_output=True, + text=True, + ) + + # Install nemo_skills + print("Installing nemo_skills...") + pip_result = subprocess.run( + [ + "uv", + "pip", + "install", + "--python", + f"{_nemo_venv_dir}/venv/bin/python", + "git+https://github.com/NVIDIA/NeMo-Skills.git", + ], + capture_output=True, + text=True, + timeout=300, + ) + if pip_result.returncode != 0: + raise RuntimeError(f"Failed to install nemo_skills: {pip_result.stderr[-500:]}") + + print("NeMo Skills installed successfully") + return _get_nemo_venv() + + +def _ensure_nemo_data_prepared( + venv_python: str, env: dict, dataset: str +) -> Tuple[bool, Optional[str]]: + """Prepare NeMo Skills dataset data if not already done. + + Uses the venv python so data lands inside the venv's nemo_skills package. + """ + import subprocess + + if dataset in _nemo_data_prepared: + return True, None + + print(f"Preparing {dataset} data (this may take a few minutes for VLM datasets)...") + result = subprocess.run( + [venv_python, "-m", "nemo_skills.dataset.prepare", dataset], + text=True, + timeout=600, + env=env, + ) + if result.returncode != 0: + return False, f"Failed to prepare {dataset} data (exit {result.returncode})" + + _nemo_data_prepared.add(dataset) + return True, None + + +def _run_nemo_skills_eval( + model: ModelLaunchSettings, + base_url: str, + dataset: str, + max_tokens: Optional[int] = None, + repeat: Optional[int] = None, + temperature: Optional[float] = None, + top_p: Optional[float] = None, +) -> Tuple[bool, Optional[str], Optional[dict]]: + """Run evaluation using NeMo Skills (ns eval) for benchmarks like mmmu-pro. + + Uses an isolated uv venv (shared across variants) so nemo_skills dependencies + don't interfere with the system python / sglang server. + + Returns: + Tuple of (success, error_message, metrics_dict) + """ + import subprocess + import tempfile + + process = None + try: + # Get or create the shared venv (once per process) + venv_python, env = _get_nemo_venv() + + # Prepare dataset (once per process, cached) + ok, err = _ensure_nemo_data_prepared(venv_python, env, dataset) + if not ok: + return False, err, None + + process = popen_launch_server( + model.model_path, + base_url, + other_args=model.extra_args, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + env=model.env, + ) + + port = int(base_url.split(":")[-1]) + server_address = f"http://127.0.0.1:{port}/v1" + repeat_val = repeat or 1 + max_tokens_val = max_tokens or 32768 + benchmark_spec = f"{dataset}:{repeat_val}" + + # Build ns eval command using venv python + # Note: nemo_skills.pipeline.eval requires the "eval" subcommand + output_dir = tempfile.mkdtemp(prefix="ns_eval_output_") + cmd = [ + venv_python, + "-m", + "nemo_skills.pipeline.eval", + "eval", + f"--benchmarks={benchmark_spec}", + "--server_type=sglang", + f"--model={model.model_path}", + f"--server_address={server_address}", + f"--output_dir={output_dir}", + f"++inference.tokens_to_generate={max_tokens_val}", + ] + + if temperature is not None: + cmd.append(f"++inference.temperature={temperature}") + if top_p is not None: + cmd.append(f"++inference.top_p={top_p}") + + # Add VLM-specific config + if dataset in ("mmmu-pro", "mmmu_pro"): + cmd.append("++prompt_config=vlm/mmmu-pro") + cmd.append("++max_concurrent_requests=512") + cmd.append("++max_samples=500") + + print(f"Running: {' '.join(cmd)}") + eval_result = subprocess.run( + cmd, + capture_output=True, + text=True, + timeout=7200, + env=env, + ) + + print(eval_result.stdout[-2000:] if eval_result.stdout else "(no stdout)") + if eval_result.stderr: + print(eval_result.stderr[-1000:]) + + if eval_result.returncode != 0: + return ( + False, + f"ns eval failed (exit {eval_result.returncode}): {eval_result.stderr[-500:]}", + None, + ) + + # Parse results + summarize_result = subprocess.run( + [ + venv_python, + "-m", + "nemo_skills.pipeline.summarize_results", + f"{output_dir}/eval-results", + ], + capture_output=True, + text=True, + timeout=60, + env=env, + ) + + output = summarize_result.stdout + "\n" + eval_result.stdout + print(f"Summary: {summarize_result.stdout[:1000]}") + + # Parse accuracy from output (format varies, look for common patterns) + import re + + score = None + for line in output.split("\n"): + match = re.search(r"(?:accuracy|score)[:\s]+([0-9.]+)", line, re.IGNORECASE) + if match: + score = float(match.group(1)) + + if score is None: + # Try to find it in eval-results directory + import glob + import json + + for result_file in glob.glob( + f"{output_dir}/eval-results/**/*.json", recursive=True + ): + try: + with open(result_file) as f: + data = json.load(f) + if isinstance(data, dict): + score = ( + data.get("accuracy") + or data.get("score") + or data.get("mean_score") + ) + if score is not None: + break + except (json.JSONDecodeError, KeyError): + continue + + if score is None: + # Last resort: compute accuracy directly from JSONL output + import glob + import json + + for jsonl_file in sorted( + glob.glob(f"{output_dir}/eval-results/**/*.jsonl*", recursive=True) + ): + correct = 0 + total = 0 + try: + with open(jsonl_file) as f: + for line in f: + line = line.strip() + if not line: + continue + entry = json.loads(line) + expected = entry.get("expected_answer", "") + generation = entry.get("generation", "") + # Extract "Answer: X" from the end of generation + answer_match = re.search( + r"Answer:\s*([A-J])", generation, re.IGNORECASE + ) + if answer_match: + predicted = answer_match.group(1).upper() + if predicted == expected.upper(): + correct += 1 + total += 1 + except (json.JSONDecodeError, KeyError, OSError): + continue + if total > 0: + score = correct / total + print( + f"Computed accuracy from {jsonl_file}: " + f"{correct}/{total} = {score:.4f}" + ) + break + + if score is None: + return False, "Could not parse accuracy from ns eval output", None + + return True, None, {"score": score} + + except subprocess.TimeoutExpired: + return False, "NeMo Skills eval timed out", None + except Exception as e: + return False, f"NeMo Skills eval exception: {str(e)}", None + finally: + if process: + kill_process_tree(process.pid) + + def _run_few_shot_eval( model: ModelLaunchSettings, base_url: str, @@ -224,13 +506,24 @@ def run_accuracy_test( print(f"{'='*60}\n") # Run evaluation based on dataset type - # Use few_shot_eval for gsm8k by default for backward compatibility. - # Use simple_eval when any extended params are set that few_shot_eval doesn't support. + # - NeMo Skills: mmmu-pro (and other VLM evals needing ns eval) + # - few_shot_eval: gsm8k (default, backward compatible) + # - simple_eval: everything else (gpqa, mmmu, etc.) has_extended_params = any( getattr(params, field) is not None for field in ("thinking_mode", "temperature", "top_p", "top_k", "repeat") ) - if params.dataset == "gsm8k" and not has_extended_params: + if params.dataset in ("mmmu-pro", "mmmu_pro"): + success, error, metrics = _run_nemo_skills_eval( + model=model, + base_url=base_url, + dataset="mmmu-pro", + max_tokens=params.max_tokens, + repeat=params.repeat or 1, + temperature=params.temperature, + top_p=params.top_p, + ) + elif params.dataset == "gsm8k" and not has_extended_params: success, error, metrics = _run_few_shot_eval( model=model, base_url=base_url, diff --git a/python/sglang/test/run_combined_tests.py b/python/sglang/test/run_combined_tests.py index c1ac82657..fa419b3ff 100644 --- a/python/sglang/test/run_combined_tests.py +++ b/python/sglang/test/run_combined_tests.py @@ -104,6 +104,7 @@ def run_combined_tests( model_result = { "model": model.model_path, + "variant": model.variant, "perf_result": None, "accuracy_result": None, "tool_call_result": None, @@ -243,8 +244,9 @@ def run_combined_tests( failed_test_str = ", ".join(failed_tests) if failed_tests else "unknown" error_str = "; ".join(str(e) for e in r["errors"]) + variant_str = f" [{r['variant']}]" if r.get("variant") else "" failure_lines.append( - f" Model {i + 1} ({r['model']}): {failed_test_str} - {error_str}" + f" Model {i + 1} ({r['model']}{variant_str}): {failed_test_str} - {error_str}" ) failure_summary = "\n".join(failure_lines) diff --git a/test/registered/gb300/test_deepseek_v32.py b/test/registered/gb300/test_deepseek_v32.py new file mode 100644 index 000000000..0f9ff25cd --- /dev/null +++ b/test/registered/gb300/test_deepseek_v32.py @@ -0,0 +1,79 @@ +import unittest + +from sglang.test.accuracy_test_runner import AccuracyTestParams +from sglang.test.ci.ci_register import register_cuda_ci +from sglang.test.performance_test_runner import PerformanceTestParams +from sglang.test.run_combined_tests import run_combined_tests +from sglang.test.test_utils import ModelLaunchSettings + +register_cuda_ci(est_time=7200, suite="nightly-4-gpu-gb300", nightly=True) + +MODEL_PATH = "deepseek-ai/DeepSeek-V3.2" + +COMMON_ARGS = [ + "--trust-remote-code", + "--reasoning-parser=deepseek-v3", + "--tool-call-parser=deepseekv32", + "--mem-fraction-static=0.8", + "--enable-metrics", +] + +MTP_ARGS = [ + "--speculative-algorithm=EAGLE", + "--speculative-num-steps=3", + "--speculative-eagle-topk=1", + "--speculative-num-draft-tokens=4", +] + + +class TestDeepseekV32(unittest.TestCase): + """DeepSeek V3.2 on GB300 (4x B200 NVL4, tp=4).""" + + def test_deepseek_v32(self): + variants = [ + ModelLaunchSettings( + MODEL_PATH, + tp_size=4, + extra_args=COMMON_ARGS, + variant="TP4", + ), + ModelLaunchSettings( + MODEL_PATH, + tp_size=4, + extra_args=COMMON_ARGS + + [ + "--dp-size=4", + "--ep-size=4", + "--enable-dp-attention", + ], + variant="TP4+DP4+DPA", + ), + ModelLaunchSettings( + MODEL_PATH, + tp_size=4, + extra_args=COMMON_ARGS + + [ + "--dp-size=4", + "--ep-size=4", + "--enable-dp-attention", + ] + + MTP_ARGS, + variant="TP4+DP4+DPA+MTP", + env={"SGLANG_ENABLE_SPEC_V2": "1"}, + ), + ] + + run_combined_tests( + models=variants, + test_name="DeepSeek-V3.2", + accuracy_params=AccuracyTestParams( + dataset="gsm8k", baseline_accuracy=0.935 + ), + performance_params=PerformanceTestParams( + profile_dir="performance_profiles_gb300", + ), + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/gb300/test_deepseek_v32_nvfp4.py b/test/registered/gb300/test_deepseek_v32_nvfp4.py new file mode 100644 index 000000000..f6be6f94a --- /dev/null +++ b/test/registered/gb300/test_deepseek_v32_nvfp4.py @@ -0,0 +1,82 @@ +import unittest + +from sglang.test.accuracy_test_runner import AccuracyTestParams +from sglang.test.ci.ci_register import register_cuda_ci +from sglang.test.performance_test_runner import PerformanceTestParams +from sglang.test.run_combined_tests import run_combined_tests +from sglang.test.test_utils import ModelLaunchSettings + +register_cuda_ci(est_time=7200, suite="nightly-4-gpu-gb300", nightly=True) + +MODEL_PATH = "nvidia/DeepSeek-V3.2-NVFP4" + +COMMON_ARGS = [ + "--trust-remote-code", + "--reasoning-parser=deepseek-v3", + "--tool-call-parser=deepseekv32", + "--quantization=modelopt_fp4", + "--moe-runner-backend=flashinfer_trtllm", + "--kv-cache-dtype=bfloat16", + "--mem-fraction-static=0.8", + "--enable-metrics", +] + +MTP_ARGS = [ + "--speculative-algorithm=EAGLE", + "--speculative-num-steps=3", + "--speculative-eagle-topk=1", + "--speculative-num-draft-tokens=4", +] + + +class TestDeepseekV32Nvfp4(unittest.TestCase): + """DeepSeek V3.2 NVFP4 on GB300 (4x B200 NVL4, tp=4).""" + + def test_deepseek_v32_nvfp4(self): + variants = [ + ModelLaunchSettings( + MODEL_PATH, + tp_size=4, + extra_args=COMMON_ARGS, + variant="TP4", + ), + ModelLaunchSettings( + MODEL_PATH, + tp_size=4, + extra_args=COMMON_ARGS + + [ + "--dp-size=4", + "--ep-size=4", + "--enable-dp-attention", + ], + variant="TP4+DP4+DPA", + ), + ModelLaunchSettings( + MODEL_PATH, + tp_size=4, + extra_args=COMMON_ARGS + + [ + "--dp-size=4", + "--ep-size=4", + "--enable-dp-attention", + ] + + MTP_ARGS, + variant="TP4+DP4+DPA+MTP", + env={"SGLANG_ENABLE_SPEC_V2": "1"}, + ), + ] + + run_combined_tests( + models=variants, + test_name="DeepSeek-V3.2-NVFP4", + accuracy_params=AccuracyTestParams( + dataset="gsm8k", baseline_accuracy=0.935 + ), + performance_params=PerformanceTestParams( + profile_dir="performance_profiles_gb300", + ), + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/gb300/test_glm5_fp8.py b/test/registered/gb300/test_glm5_fp8.py new file mode 100644 index 000000000..e429e5873 --- /dev/null +++ b/test/registered/gb300/test_glm5_fp8.py @@ -0,0 +1,68 @@ +import unittest + +from sglang.test.accuracy_test_runner import AccuracyTestParams +from sglang.test.ci.ci_register import register_cuda_ci +from sglang.test.performance_test_runner import PerformanceTestParams +from sglang.test.run_combined_tests import run_combined_tests +from sglang.test.test_utils import ModelLaunchSettings + +register_cuda_ci(est_time=7200, suite="nightly-4-gpu-gb300", nightly=True) + +MODEL_PATH = "zai-org/GLM-5-FP8" + +COMMON_ARGS = [ + "--trust-remote-code", + "--reasoning-parser=glm45", + "--tool-call-parser=glm47", + "--mem-fraction-static=0.9", + "--enable-metrics", +] + +MTP_ARGS = [ + "--speculative-algorithm=EAGLE", + "--speculative-num-steps=3", + "--speculative-eagle-topk=1", + "--speculative-num-draft-tokens=4", +] + + +class TestGlm5Fp8(unittest.TestCase): + """GLM-5 FP8 on GB300 (4x B200 NVL4, tp=4).""" + + def test_glm5_fp8(self): + variants = [ + ModelLaunchSettings( + MODEL_PATH, + tp_size=4, + extra_args=COMMON_ARGS, + variant="TP4", + ), + ModelLaunchSettings( + MODEL_PATH, + tp_size=4, + extra_args=COMMON_ARGS + ["--dp-size=4", "--enable-dp-attention"], + variant="TP4+DP4+DPA", + ), + ModelLaunchSettings( + MODEL_PATH, + tp_size=4, + extra_args=COMMON_ARGS + + ["--dp-size=4", "--enable-dp-attention"] + + MTP_ARGS, + variant="TP4+DP4+DPA+MTP", + env={"SGLANG_ENABLE_SPEC_V2": "1"}, + ), + ] + + run_combined_tests( + models=variants, + test_name="GLM-5-FP8", + accuracy_params=AccuracyTestParams(dataset="gsm8k", baseline_accuracy=0.92), + performance_params=PerformanceTestParams( + profile_dir="performance_profiles_gb300", + ), + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/gb300/test_glm5_nvfp4.py b/test/registered/gb300/test_glm5_nvfp4.py new file mode 100644 index 000000000..595276c68 --- /dev/null +++ b/test/registered/gb300/test_glm5_nvfp4.py @@ -0,0 +1,71 @@ +import unittest + +from sglang.test.accuracy_test_runner import AccuracyTestParams +from sglang.test.ci.ci_register import register_cuda_ci +from sglang.test.performance_test_runner import PerformanceTestParams +from sglang.test.run_combined_tests import run_combined_tests +from sglang.test.test_utils import ModelLaunchSettings + +register_cuda_ci(est_time=7200, suite="nightly-4-gpu-gb300", nightly=True) + +MODEL_PATH = "nvidia/GLM-5-NVFP4" + +COMMON_ARGS = [ + "--trust-remote-code", + "--reasoning-parser=glm45", + "--tool-call-parser=glm47", + "--quantization=modelopt_fp4", + "--moe-runner-backend=flashinfer_trtllm", + "--kv-cache-dtype=bfloat16", + "--mem-fraction-static=0.9", + "--enable-metrics", +] + +MTP_ARGS = [ + "--speculative-algorithm=EAGLE", + "--speculative-num-steps=3", + "--speculative-eagle-topk=1", + "--speculative-num-draft-tokens=4", +] + + +class TestGlm5Nvfp4(unittest.TestCase): + """GLM-5 NVFP4 on GB300 (4x B200 NVL4, tp=4).""" + + def test_glm5_nvfp4(self): + variants = [ + ModelLaunchSettings( + MODEL_PATH, + tp_size=4, + extra_args=COMMON_ARGS, + variant="TP4", + ), + ModelLaunchSettings( + MODEL_PATH, + tp_size=4, + extra_args=COMMON_ARGS + ["--dp-size=4", "--enable-dp-attention"], + variant="TP4+DP4+DPA", + ), + ModelLaunchSettings( + MODEL_PATH, + tp_size=4, + extra_args=COMMON_ARGS + + ["--dp-size=4", "--enable-dp-attention"] + + MTP_ARGS, + variant="TP4+DP4+DPA+MTP", + env={"SGLANG_ENABLE_SPEC_V2": "1"}, + ), + ] + + run_combined_tests( + models=variants, + test_name="GLM-5-NVFP4", + accuracy_params=AccuracyTestParams(dataset="gsm8k", baseline_accuracy=0.92), + performance_params=PerformanceTestParams( + profile_dir="performance_profiles_gb300", + ), + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/gb300/test_kimi_k25.py b/test/registered/gb300/test_kimi_k25.py new file mode 100644 index 000000000..47beb0b19 --- /dev/null +++ b/test/registered/gb300/test_kimi_k25.py @@ -0,0 +1,58 @@ +import unittest + +from sglang.test.accuracy_test_runner import AccuracyTestParams +from sglang.test.ci.ci_register import register_cuda_ci +from sglang.test.performance_test_runner import PerformanceTestParams +from sglang.test.run_combined_tests import run_combined_tests +from sglang.test.test_utils import ModelLaunchSettings + +register_cuda_ci(est_time=7200, suite="nightly-4-gpu-gb300", nightly=True) + +MODEL_PATH = "moonshotai/Kimi-K2.5" + +COMMON_ARGS = [ + "--trust-remote-code", + "--reasoning-parser=kimi_k2", + "--tool-call-parser=kimi_k2", + "--mem-fraction-static=0.8", + "--enable-multimodal", + "--enable-metrics", +] + + +class TestKimiK25(unittest.TestCase): + """Kimi-K2.5 (native INT4) on GB300 (4x B200 NVL4, tp=4). + + No EAGLE/MTP support for Kimi-K2.5 — only TP and TP+DP+DPA variants. + """ + + def test_kimi_k25(self): + variants = [ + ModelLaunchSettings( + MODEL_PATH, + tp_size=4, + extra_args=COMMON_ARGS, + variant="TP4", + ), + ModelLaunchSettings( + MODEL_PATH, + tp_size=4, + extra_args=COMMON_ARGS + ["--dp-size=4", "--enable-dp-attention"], + variant="TP4+DP4+DPA", + ), + ] + + run_combined_tests( + models=variants, + test_name="Kimi-K2.5", + accuracy_params=AccuracyTestParams( + dataset="mmmu-pro", baseline_accuracy=0.69, repeat=1, max_tokens=32768 + ), + performance_params=PerformanceTestParams( + profile_dir="performance_profiles_gb300", + ), + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/gb300/test_kimi_k25_nvfp4.py b/test/registered/gb300/test_kimi_k25_nvfp4.py new file mode 100644 index 000000000..7faf6c92b --- /dev/null +++ b/test/registered/gb300/test_kimi_k25_nvfp4.py @@ -0,0 +1,61 @@ +import unittest + +from sglang.test.accuracy_test_runner import AccuracyTestParams +from sglang.test.ci.ci_register import register_cuda_ci +from sglang.test.performance_test_runner import PerformanceTestParams +from sglang.test.run_combined_tests import run_combined_tests +from sglang.test.test_utils import ModelLaunchSettings + +register_cuda_ci(est_time=7200, suite="nightly-4-gpu-gb300", nightly=True) + +MODEL_PATH = "nvidia/Kimi-K2.5-NVFP4" + +COMMON_ARGS = [ + "--trust-remote-code", + "--reasoning-parser=kimi_k2", + "--tool-call-parser=kimi_k2", + "--quantization=modelopt_fp4", + "--attention-backend=trtllm_mla", + "--moe-runner-backend=flashinfer_trtllm", + "--mem-fraction-static=0.8", + "--enable-multimodal", + "--enable-metrics", +] + + +class TestKimiK25Nvfp4(unittest.TestCase): + """Kimi-K2.5 NVFP4 on GB300 (4x B200 NVL4, tp=4). + + No EAGLE/MTP support for Kimi-K2.5 — only TP and TP+DP+DPA variants. + """ + + def test_kimi_k25_nvfp4(self): + variants = [ + ModelLaunchSettings( + MODEL_PATH, + tp_size=4, + extra_args=COMMON_ARGS, + variant="TP4", + ), + ModelLaunchSettings( + MODEL_PATH, + tp_size=4, + extra_args=COMMON_ARGS + ["--dp-size=4", "--enable-dp-attention"], + variant="TP4+DP4+DPA", + ), + ] + + run_combined_tests( + models=variants, + test_name="Kimi-K2.5-NVFP4", + accuracy_params=AccuracyTestParams( + dataset="mmmu-pro", baseline_accuracy=0.69, repeat=1, max_tokens=32768 + ), + performance_params=PerformanceTestParams( + profile_dir="performance_profiles_gb300", + ), + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/gb300/test_qwen35_fp8.py b/test/registered/gb300/test_qwen35_fp8.py new file mode 100644 index 000000000..1121b1a81 --- /dev/null +++ b/test/registered/gb300/test_qwen35_fp8.py @@ -0,0 +1,75 @@ +import unittest + +from sglang.test.accuracy_test_runner import AccuracyTestParams +from sglang.test.ci.ci_register import register_cuda_ci +from sglang.test.performance_test_runner import PerformanceTestParams +from sglang.test.run_combined_tests import run_combined_tests +from sglang.test.test_utils import ModelLaunchSettings + +register_cuda_ci(est_time=7200, suite="nightly-4-gpu-gb300", nightly=True) + +MODEL_PATH = "Qwen/Qwen3.5-397B-A17B-FP8" + +COMMON_ARGS = [ + "--trust-remote-code", + "--reasoning-parser=qwen3", + "--tool-call-parser=qwen3_coder", + "--enable-flashinfer-allreduce-fusion", + "--attention-backend=trtllm_mha", + "--mem-fraction-static=0.8", + "--enable-multimodal", + "--enable-metrics", +] + +MTP_ARGS = [ + "--speculative-algorithm=EAGLE", + "--speculative-num-steps=3", + "--speculative-eagle-topk=1", + "--speculative-num-draft-tokens=4", + "--mamba-scheduler-strategy=extra_buffer", + "--page-size=64", +] + + +class TestQwen35Fp8(unittest.TestCase): + """Qwen3.5-397B FP8 on GB300 (4x B200 NVL4, tp=4).""" + + def test_qwen35_fp8(self): + variants = [ + ModelLaunchSettings( + MODEL_PATH, + tp_size=4, + extra_args=COMMON_ARGS, + variant="TP4", + ), + ModelLaunchSettings( + MODEL_PATH, + tp_size=4, + extra_args=COMMON_ARGS + ["--dp-size=4", "--enable-dp-attention"], + variant="TP4+DP4+DPA", + ), + ModelLaunchSettings( + MODEL_PATH, + tp_size=4, + extra_args=COMMON_ARGS + + ["--dp-size=4", "--enable-dp-attention"] + + MTP_ARGS, + variant="TP4+DP4+DPA+MTP", + env={"SGLANG_ENABLE_SPEC_V2": "1"}, + ), + ] + + run_combined_tests( + models=variants, + test_name="Qwen3.5-397B-FP8", + accuracy_params=AccuracyTestParams( + dataset="mmmu-pro", baseline_accuracy=0.78, repeat=1, max_tokens=32768 + ), + performance_params=PerformanceTestParams( + profile_dir="performance_profiles_gb300", + ), + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/gb300/test_qwen35_nvfp4.py b/test/registered/gb300/test_qwen35_nvfp4.py new file mode 100644 index 000000000..f48ad701c --- /dev/null +++ b/test/registered/gb300/test_qwen35_nvfp4.py @@ -0,0 +1,79 @@ +import unittest + +from sglang.test.accuracy_test_runner import AccuracyTestParams +from sglang.test.ci.ci_register import register_cuda_ci +from sglang.test.performance_test_runner import PerformanceTestParams +from sglang.test.run_combined_tests import run_combined_tests +from sglang.test.test_utils import ModelLaunchSettings + +register_cuda_ci(est_time=7200, suite="nightly-4-gpu-gb300", nightly=True) + +MODEL_PATH = "nvidia/Qwen3.5-397B-A17B-NVFP4" + +COMMON_ARGS = [ + "--trust-remote-code", + "--reasoning-parser=qwen3", + "--tool-call-parser=qwen3_coder", + "--quantization=modelopt_fp4", + "--fp4-gemm-backend=flashinfer_cutlass", + "--moe-runner-backend=flashinfer_trtllm", + "--kv-cache-dtype=fp8_e4m3", + "--enable-flashinfer-allreduce-fusion", + "--attention-backend=trtllm_mha", + "--mem-fraction-static=0.8", + "--enable-multimodal", + "--enable-metrics", +] + +MTP_ARGS = [ + "--speculative-algorithm=EAGLE", + "--speculative-num-steps=3", + "--speculative-eagle-topk=1", + "--speculative-num-draft-tokens=4", + "--mamba-scheduler-strategy=extra_buffer", + "--page-size=64", +] + + +class TestQwen35Nvfp4(unittest.TestCase): + """Qwen3.5-397B NVFP4 on GB300 (4x B200 NVL4, tp=4).""" + + def test_qwen35_nvfp4(self): + variants = [ + ModelLaunchSettings( + MODEL_PATH, + tp_size=4, + extra_args=COMMON_ARGS, + variant="TP4", + ), + ModelLaunchSettings( + MODEL_PATH, + tp_size=4, + extra_args=COMMON_ARGS + ["--dp-size=4", "--enable-dp-attention"], + variant="TP4+DP4+DPA", + ), + ModelLaunchSettings( + MODEL_PATH, + tp_size=4, + extra_args=COMMON_ARGS + + ["--dp-size=4", "--enable-dp-attention"] + + MTP_ARGS, + variant="TP4+DP4+DPA+MTP", + env={"SGLANG_ENABLE_SPEC_V2": "1"}, + ), + ] + + run_combined_tests( + models=variants, + test_name="Qwen3.5-397B-NVFP4", + accuracy_params=AccuracyTestParams( + dataset="mmmu-pro", baseline_accuracy=0.78, repeat=1, max_tokens=32768 + ), + performance_params=PerformanceTestParams( + profile_dir="performance_profiles_gb300", + ), + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/run_suite.py b/test/run_suite.py index 97b39ee2e..3762cb0ad 100644 --- a/test/run_suite.py +++ b/test/run_suite.py @@ -84,6 +84,8 @@ NIGHTLY_SUITES = { "nightly-eval-vlm-2-gpu", "nightly-perf-text-2-gpu", "nightly-perf-vlm-2-gpu", + # GB300 (4x B200 NVL4) nightly suite + "nightly-4-gpu-gb300", ], HWBackend.AMD: [ "nightly-amd",