minke.yu/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 12 Packages Projects Releases Wiki Activity

feat(ci): add GB300 nightly benchmark test suites (#21487)

Browse Source 
Co-authored-by: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
Kangyan-Zhou
2026-03-28 21:54:03 -07:00
committed by GitHub
co-authored by Claude Opus 4.6
parent 166e9090ee
commit 9d64a82173
11 changed files with 874 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files
python/sglang/test/accuracy_test_runner.py
+296 -3
View File
@@ -150,6 +150,288 @@ def _run_simple_eval(
kill_process_tree(process.pid)
# Cached uv venv for NeMo Skills (persists across variants within a process).
_nemo_venv_dir: Optional[str] = None
_nemo_data_prepared: set = set()
def _get_nemo_venv() -> Tuple[str, dict]:
"""Get or create a uv venv with nemo_skills installed.
Returns (venv_python_path, env_dict) reusable across calls.
"""
import os
import subprocess
import tempfile
global _nemo_venv_dir
if _nemo_venv_dir is not None:
venv_python = f"{_nemo_venv_dir}/venv/bin/python"
env = {
**dict(os.environ),
"NEMO_SKILLS_DISABLE_UNCOMMITTED_CHANGES_CHECK": "1",
"OPENAI_API_KEY": "dummy",
"VIRTUAL_ENV": f"{_nemo_venv_dir}/venv",
"PATH": f"{_nemo_venv_dir}/venv/bin:" + os.environ.get("PATH", ""),
}
return venv_python, env
_nemo_venv_dir = tempfile.mkdtemp(prefix="nemo_skills_")
print(f"Creating NeMo Skills venv in {_nemo_venv_dir}...")
# Create venv
result = subprocess.run(
["uv", "venv", f"{_nemo_venv_dir}/venv", "--python", "3.12"],
capture_output=True,
text=True,
)
if result.returncode != 0:
subprocess.run(
["uv", "venv", f"{_nemo_venv_dir}/venv"],
capture_output=True,
text=True,
)
# Install nemo_skills
print("Installing nemo_skills...")
pip_result = subprocess.run(
[
"uv",
"pip",
"install",
"--python",
f"{_nemo_venv_dir}/venv/bin/python",
"git+https://github.com/NVIDIA/NeMo-Skills.git",
],
capture_output=True,
text=True,
timeout=300,
)
if pip_result.returncode != 0:
raise RuntimeError(f"Failed to install nemo_skills: {pip_result.stderr[-500:]}")
print("NeMo Skills installed successfully")
return _get_nemo_venv()
def _ensure_nemo_data_prepared(
venv_python: str, env: dict, dataset: str
) -> Tuple[bool, Optional[str]]:
"""Prepare NeMo Skills dataset data if not already done.
Uses the venv python so data lands inside the venv's nemo_skills package.
"""
import subprocess
if dataset in _nemo_data_prepared:
return True, None
print(f"Preparing {dataset} data (this may take a few minutes for VLM datasets)...")
result = subprocess.run(
[venv_python, "-m", "nemo_skills.dataset.prepare", dataset],
text=True,
timeout=600,
env=env,
)
if result.returncode != 0:
return False, f"Failed to prepare {dataset} data (exit {result.returncode})"
_nemo_data_prepared.add(dataset)
return True, None
def _run_nemo_skills_eval(
model: ModelLaunchSettings,
base_url: str,
dataset: str,
max_tokens: Optional[int] = None,
repeat: Optional[int] = None,
temperature: Optional[float] = None,
top_p: Optional[float] = None,
) -> Tuple[bool, Optional[str], Optional[dict]]:
"""Run evaluation using NeMo Skills (ns eval) for benchmarks like mmmu-pro.
Uses an isolated uv venv (shared across variants) so nemo_skills dependencies
don't interfere with the system python / sglang server.
Returns:
Tuple of (success, error_message, metrics_dict)
"""
import subprocess
import tempfile
process = None
try:
# Get or create the shared venv (once per process)
venv_python, env = _get_nemo_venv()
# Prepare dataset (once per process, cached)
ok, err = _ensure_nemo_data_prepared(venv_python, env, dataset)
if not ok:
return False, err, None
process = popen_launch_server(
model.model_path,
base_url,
other_args=model.extra_args,
timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
env=model.env,
)
port = int(base_url.split(":")[-1])
server_address = f"http://127.0.0.1:{port}/v1"
repeat_val = repeat or 1
max_tokens_val = max_tokens or 32768
benchmark_spec = f"{dataset}:{repeat_val}"
# Build ns eval command using venv python
# Note: nemo_skills.pipeline.eval requires the "eval" subcommand
output_dir = tempfile.mkdtemp(prefix="ns_eval_output_")
cmd = [
venv_python,
"-m",
"nemo_skills.pipeline.eval",
"eval",
f"--benchmarks={benchmark_spec}",
"--server_type=sglang",
f"--model={model.model_path}",
f"--server_address={server_address}",
f"--output_dir={output_dir}",
f"++inference.tokens_to_generate={max_tokens_val}",
]
if temperature is not None:
cmd.append(f"++inference.temperature={temperature}")
if top_p is not None:
cmd.append(f"++inference.top_p={top_p}")
# Add VLM-specific config
if dataset in ("mmmu-pro", "mmmu_pro"):
cmd.append("++prompt_config=vlm/mmmu-pro")
cmd.append("++max_concurrent_requests=512")
cmd.append("++max_samples=500")
print(f"Running: {' '.join(cmd)}")
eval_result = subprocess.run(
cmd,
capture_output=True,
text=True,
timeout=7200,
env=env,
)
print(eval_result.stdout[-2000:] if eval_result.stdout else "(no stdout)")
if eval_result.stderr:
print(eval_result.stderr[-1000:])
if eval_result.returncode != 0:
return (
False,
f"ns eval failed (exit {eval_result.returncode}): {eval_result.stderr[-500:]}",
None,
)
# Parse results
summarize_result = subprocess.run(
[
venv_python,
"-m",
"nemo_skills.pipeline.summarize_results",
f"{output_dir}/eval-results",
],
capture_output=True,
text=True,
timeout=60,
env=env,
)
output = summarize_result.stdout + "\n" + eval_result.stdout
print(f"Summary: {summarize_result.stdout[:1000]}")
# Parse accuracy from output (format varies, look for common patterns)
import re
score = None
for line in output.split("\n"):
match = re.search(r"(?:accuracy|score)[:\s]+([0-9.]+)", line, re.IGNORECASE)
if match:
score = float(match.group(1))
if score is None:
# Try to find it in eval-results directory
import glob
import json
for result_file in glob.glob(
f"{output_dir}/eval-results/**/*.json", recursive=True
):
try:
with open(result_file) as f:
data = json.load(f)
if isinstance(data, dict):
score = (
data.get("accuracy")
or data.get("score")
or data.get("mean_score")
)
if score is not None:
break
except (json.JSONDecodeError, KeyError):
continue
if score is None:
# Last resort: compute accuracy directly from JSONL output
import glob
import json
for jsonl_file in sorted(
glob.glob(f"{output_dir}/eval-results/**/*.jsonl*", recursive=True)
):
correct = 0
total = 0
try:
with open(jsonl_file) as f:
for line in f:
line = line.strip()
if not line:
continue
entry = json.loads(line)
expected = entry.get("expected_answer", "")
generation = entry.get("generation", "")
# Extract "Answer: X" from the end of generation
answer_match = re.search(
r"Answer:\s*([A-J])", generation, re.IGNORECASE
)
if answer_match:
predicted = answer_match.group(1).upper()
if predicted == expected.upper():
correct += 1
total += 1
except (json.JSONDecodeError, KeyError, OSError):
continue
if total > 0:
score = correct / total
print(
f"Computed accuracy from {jsonl_file}: "
f"{correct}/{total} = {score:.4f}"
)
break
if score is None:
return False, "Could not parse accuracy from ns eval output", None
return True, None, {"score": score}
except subprocess.TimeoutExpired:
return False, "NeMo Skills eval timed out", None
except Exception as e:
return False, f"NeMo Skills eval exception: {str(e)}", None
finally:
if process:
kill_process_tree(process.pid)
def _run_few_shot_eval(
model: ModelLaunchSettings,
base_url: str,
@@ -224,13 +506,24 @@ def run_accuracy_test(
print(f"{'='*60}\n")
# Run evaluation based on dataset type
# Use few_shot_eval for gsm8k by default for backward compatibility.
# Use simple_eval when any extended params are set that few_shot_eval doesn't support.
# - NeMo Skills: mmmu-pro (and other VLM evals needing ns eval)
# - few_shot_eval: gsm8k (default, backward compatible)
# - simple_eval: everything else (gpqa, mmmu, etc.)
has_extended_params = any(
getattr(params, field) is not None
for field in ("thinking_mode", "temperature", "top_p", "top_k", "repeat")
)
if params.dataset == "gsm8k" and not has_extended_params:
if params.dataset in ("mmmu-pro", "mmmu_pro"):
success, error, metrics = _run_nemo_skills_eval(
model=model,
base_url=base_url,
dataset="mmmu-pro",
max_tokens=params.max_tokens,
repeat=params.repeat or 1,
temperature=params.temperature,
top_p=params.top_p,
)
elif params.dataset == "gsm8k" and not has_extended_params:
success, error, metrics = _run_few_shot_eval(
model=model,
base_url=base_url,
python/sglang/test/run_combined_tests.py
+3 -1
View File
@@ -104,6 +104,7 @@ def run_combined_tests(
model_result = {
"model": model.model_path,
"variant": model.variant,
"perf_result": None,
"accuracy_result": None,
"tool_call_result": None,
@@ -243,8 +244,9 @@ def run_combined_tests(
failed_test_str = ", ".join(failed_tests) if failed_tests else "unknown"
error_str = "; ".join(str(e) for e in r["errors"])
variant_str = f" [{r['variant']}]" if r.get("variant") else ""
failure_lines.append(
f" Model {i + 1} ({r['model']}): {failed_test_str} - {error_str}"
f" Model {i + 1} ({r['model']}{variant_str}): {failed_test_str} - {error_str}"
)
failure_summary = "\n".join(failure_lines)
test/registered/gb300/test_deepseek_v32.py
+79
View File
@@ -0,0 +1,79 @@
import unittest
from sglang.test.accuracy_test_runner import AccuracyTestParams
from sglang.test.ci.ci_register import register_cuda_ci
from sglang.test.performance_test_runner import PerformanceTestParams
from sglang.test.run_combined_tests import run_combined_tests
from sglang.test.test_utils import ModelLaunchSettings
register_cuda_ci(est_time=7200, suite="nightly-4-gpu-gb300", nightly=True)
MODEL_PATH = "deepseek-ai/DeepSeek-V3.2"
COMMON_ARGS = [
"--trust-remote-code",
"--reasoning-parser=deepseek-v3",
"--tool-call-parser=deepseekv32",
"--mem-fraction-static=0.8",
"--enable-metrics",
]
MTP_ARGS = [
"--speculative-algorithm=EAGLE",
"--speculative-num-steps=3",
"--speculative-eagle-topk=1",
"--speculative-num-draft-tokens=4",
]
class TestDeepseekV32(unittest.TestCase):
"""DeepSeek V3.2 on GB300 (4x B200 NVL4, tp=4)."""
def test_deepseek_v32(self):
variants = [
ModelLaunchSettings(
MODEL_PATH,
tp_size=4,
extra_args=COMMON_ARGS,
variant="TP4",
),
ModelLaunchSettings(
MODEL_PATH,
tp_size=4,
extra_args=COMMON_ARGS
+ [
"--dp-size=4",
"--ep-size=4",
"--enable-dp-attention",
],
variant="TP4+DP4+DPA",
),
ModelLaunchSettings(
MODEL_PATH,
tp_size=4,
extra_args=COMMON_ARGS
+ [
"--dp-size=4",
"--ep-size=4",
"--enable-dp-attention",
]
+ MTP_ARGS,
variant="TP4+DP4+DPA+MTP",
env={"SGLANG_ENABLE_SPEC_V2": "1"},
),
]
run_combined_tests(
models=variants,
test_name="DeepSeek-V3.2",
accuracy_params=AccuracyTestParams(
dataset="gsm8k", baseline_accuracy=0.935
),
performance_params=PerformanceTestParams(
profile_dir="performance_profiles_gb300",
),
)
if __name__ == "__main__":
unittest.main()
test/registered/gb300/test_deepseek_v32_nvfp4.py
+82
View File
@@ -0,0 +1,82 @@
import unittest
from sglang.test.accuracy_test_runner import AccuracyTestParams
from sglang.test.ci.ci_register import register_cuda_ci
from sglang.test.performance_test_runner import PerformanceTestParams
from sglang.test.run_combined_tests import run_combined_tests
from sglang.test.test_utils import ModelLaunchSettings
register_cuda_ci(est_time=7200, suite="nightly-4-gpu-gb300", nightly=True)
MODEL_PATH = "nvidia/DeepSeek-V3.2-NVFP4"
COMMON_ARGS = [
"--trust-remote-code",
"--reasoning-parser=deepseek-v3",
"--tool-call-parser=deepseekv32",
"--quantization=modelopt_fp4",
"--moe-runner-backend=flashinfer_trtllm",
"--kv-cache-dtype=bfloat16",
"--mem-fraction-static=0.8",
"--enable-metrics",
]
MTP_ARGS = [
"--speculative-algorithm=EAGLE",
"--speculative-num-steps=3",
"--speculative-eagle-topk=1",
"--speculative-num-draft-tokens=4",
]
class TestDeepseekV32Nvfp4(unittest.TestCase):
"""DeepSeek V3.2 NVFP4 on GB300 (4x B200 NVL4, tp=4)."""
def test_deepseek_v32_nvfp4(self):
variants = [
ModelLaunchSettings(
MODEL_PATH,
tp_size=4,
extra_args=COMMON_ARGS,
variant="TP4",
),
ModelLaunchSettings(
MODEL_PATH,
tp_size=4,
extra_args=COMMON_ARGS
+ [
"--dp-size=4",
"--ep-size=4",
"--enable-dp-attention",
],
variant="TP4+DP4+DPA",
),
ModelLaunchSettings(
MODEL_PATH,
tp_size=4,
extra_args=COMMON_ARGS
+ [
"--dp-size=4",
"--ep-size=4",
"--enable-dp-attention",
]
+ MTP_ARGS,
variant="TP4+DP4+DPA+MTP",
env={"SGLANG_ENABLE_SPEC_V2": "1"},
),
]
run_combined_tests(
models=variants,
test_name="DeepSeek-V3.2-NVFP4",
accuracy_params=AccuracyTestParams(
dataset="gsm8k", baseline_accuracy=0.935
),
performance_params=PerformanceTestParams(
profile_dir="performance_profiles_gb300",
),
)
if __name__ == "__main__":
unittest.main()
test/registered/gb300/test_glm5_fp8.py
+68
View File
@@ -0,0 +1,68 @@
import unittest
from sglang.test.accuracy_test_runner import AccuracyTestParams
from sglang.test.ci.ci_register import register_cuda_ci
from sglang.test.performance_test_runner import PerformanceTestParams
from sglang.test.run_combined_tests import run_combined_tests
from sglang.test.test_utils import ModelLaunchSettings
register_cuda_ci(est_time=7200, suite="nightly-4-gpu-gb300", nightly=True)
MODEL_PATH = "zai-org/GLM-5-FP8"
COMMON_ARGS = [
"--trust-remote-code",
"--reasoning-parser=glm45",
"--tool-call-parser=glm47",
"--mem-fraction-static=0.9",
"--enable-metrics",
]
MTP_ARGS = [
"--speculative-algorithm=EAGLE",
"--speculative-num-steps=3",
"--speculative-eagle-topk=1",
"--speculative-num-draft-tokens=4",
]
class TestGlm5Fp8(unittest.TestCase):
"""GLM-5 FP8 on GB300 (4x B200 NVL4, tp=4)."""
def test_glm5_fp8(self):
variants = [
ModelLaunchSettings(
MODEL_PATH,
tp_size=4,
extra_args=COMMON_ARGS,
variant="TP4",
),
ModelLaunchSettings(
MODEL_PATH,
tp_size=4,
extra_args=COMMON_ARGS + ["--dp-size=4", "--enable-dp-attention"],
variant="TP4+DP4+DPA",
),
ModelLaunchSettings(
MODEL_PATH,
tp_size=4,
extra_args=COMMON_ARGS
+ ["--dp-size=4", "--enable-dp-attention"]
+ MTP_ARGS,
variant="TP4+DP4+DPA+MTP",
env={"SGLANG_ENABLE_SPEC_V2": "1"},
),
]
run_combined_tests(
models=variants,
test_name="GLM-5-FP8",
accuracy_params=AccuracyTestParams(dataset="gsm8k", baseline_accuracy=0.92),
performance_params=PerformanceTestParams(
profile_dir="performance_profiles_gb300",
),
)
if __name__ == "__main__":
unittest.main()
test/registered/gb300/test_glm5_nvfp4.py
+71
View File
@@ -0,0 +1,71 @@
import unittest
from sglang.test.accuracy_test_runner import AccuracyTestParams
from sglang.test.ci.ci_register import register_cuda_ci
from sglang.test.performance_test_runner import PerformanceTestParams
from sglang.test.run_combined_tests import run_combined_tests
from sglang.test.test_utils import ModelLaunchSettings
register_cuda_ci(est_time=7200, suite="nightly-4-gpu-gb300", nightly=True)
MODEL_PATH = "nvidia/GLM-5-NVFP4"
COMMON_ARGS = [
"--trust-remote-code",
"--reasoning-parser=glm45",
"--tool-call-parser=glm47",
"--quantization=modelopt_fp4",
"--moe-runner-backend=flashinfer_trtllm",
"--kv-cache-dtype=bfloat16",
"--mem-fraction-static=0.9",
"--enable-metrics",
]
MTP_ARGS = [
"--speculative-algorithm=EAGLE",
"--speculative-num-steps=3",
"--speculative-eagle-topk=1",
"--speculative-num-draft-tokens=4",
]
class TestGlm5Nvfp4(unittest.TestCase):
"""GLM-5 NVFP4 on GB300 (4x B200 NVL4, tp=4)."""
def test_glm5_nvfp4(self):
variants = [
ModelLaunchSettings(
MODEL_PATH,
tp_size=4,
extra_args=COMMON_ARGS,
variant="TP4",
),
ModelLaunchSettings(
MODEL_PATH,
tp_size=4,
extra_args=COMMON_ARGS + ["--dp-size=4", "--enable-dp-attention"],
variant="TP4+DP4+DPA",
),
ModelLaunchSettings(
MODEL_PATH,
tp_size=4,
extra_args=COMMON_ARGS
+ ["--dp-size=4", "--enable-dp-attention"]
+ MTP_ARGS,
variant="TP4+DP4+DPA+MTP",
env={"SGLANG_ENABLE_SPEC_V2": "1"},
),
]
run_combined_tests(
models=variants,
test_name="GLM-5-NVFP4",
accuracy_params=AccuracyTestParams(dataset="gsm8k", baseline_accuracy=0.92),
performance_params=PerformanceTestParams(
profile_dir="performance_profiles_gb300",
),
)
if __name__ == "__main__":
unittest.main()
test/registered/gb300/test_kimi_k25.py
+58
View File
@@ -0,0 +1,58 @@
import unittest
from sglang.test.accuracy_test_runner import AccuracyTestParams
from sglang.test.ci.ci_register import register_cuda_ci
from sglang.test.performance_test_runner import PerformanceTestParams
from sglang.test.run_combined_tests import run_combined_tests
from sglang.test.test_utils import ModelLaunchSettings
register_cuda_ci(est_time=7200, suite="nightly-4-gpu-gb300", nightly=True)
MODEL_PATH = "moonshotai/Kimi-K2.5"
COMMON_ARGS = [
"--trust-remote-code",
"--reasoning-parser=kimi_k2",
"--tool-call-parser=kimi_k2",
"--mem-fraction-static=0.8",
"--enable-multimodal",
"--enable-metrics",
]
class TestKimiK25(unittest.TestCase):
"""Kimi-K2.5 (native INT4) on GB300 (4x B200 NVL4, tp=4).
No EAGLE/MTP support for Kimi-K2.5 — only TP and TP+DP+DPA variants.
"""
def test_kimi_k25(self):
variants = [
ModelLaunchSettings(
MODEL_PATH,
tp_size=4,
extra_args=COMMON_ARGS,
variant="TP4",
),
ModelLaunchSettings(
MODEL_PATH,
tp_size=4,
extra_args=COMMON_ARGS + ["--dp-size=4", "--enable-dp-attention"],
variant="TP4+DP4+DPA",
),
]
run_combined_tests(
models=variants,
test_name="Kimi-K2.5",
accuracy_params=AccuracyTestParams(
dataset="mmmu-pro", baseline_accuracy=0.69, repeat=1, max_tokens=32768
),
performance_params=PerformanceTestParams(
profile_dir="performance_profiles_gb300",
),
)
if __name__ == "__main__":
unittest.main()
test/registered/gb300/test_kimi_k25_nvfp4.py
+61
View File
@@ -0,0 +1,61 @@
import unittest
from sglang.test.accuracy_test_runner import AccuracyTestParams
from sglang.test.ci.ci_register import register_cuda_ci
from sglang.test.performance_test_runner import PerformanceTestParams
from sglang.test.run_combined_tests import run_combined_tests
from sglang.test.test_utils import ModelLaunchSettings
register_cuda_ci(est_time=7200, suite="nightly-4-gpu-gb300", nightly=True)
MODEL_PATH = "nvidia/Kimi-K2.5-NVFP4"
COMMON_ARGS = [
"--trust-remote-code",
"--reasoning-parser=kimi_k2",
"--tool-call-parser=kimi_k2",
"--quantization=modelopt_fp4",
"--attention-backend=trtllm_mla",
"--moe-runner-backend=flashinfer_trtllm",
"--mem-fraction-static=0.8",
"--enable-multimodal",
"--enable-metrics",
]
class TestKimiK25Nvfp4(unittest.TestCase):
"""Kimi-K2.5 NVFP4 on GB300 (4x B200 NVL4, tp=4).
No EAGLE/MTP support for Kimi-K2.5 — only TP and TP+DP+DPA variants.
"""
def test_kimi_k25_nvfp4(self):
variants = [
ModelLaunchSettings(
MODEL_PATH,
tp_size=4,
extra_args=COMMON_ARGS,
variant="TP4",
),
ModelLaunchSettings(
MODEL_PATH,
tp_size=4,
extra_args=COMMON_ARGS + ["--dp-size=4", "--enable-dp-attention"],
variant="TP4+DP4+DPA",
),
]
run_combined_tests(
models=variants,
test_name="Kimi-K2.5-NVFP4",
accuracy_params=AccuracyTestParams(
dataset="mmmu-pro", baseline_accuracy=0.69, repeat=1, max_tokens=32768
),
performance_params=PerformanceTestParams(
profile_dir="performance_profiles_gb300",
),
)
if __name__ == "__main__":
unittest.main()
test/registered/gb300/test_qwen35_fp8.py
+75
View File
@@ -0,0 +1,75 @@
import unittest
from sglang.test.accuracy_test_runner import AccuracyTestParams
from sglang.test.ci.ci_register import register_cuda_ci
from sglang.test.performance_test_runner import PerformanceTestParams
from sglang.test.run_combined_tests import run_combined_tests
from sglang.test.test_utils import ModelLaunchSettings
register_cuda_ci(est_time=7200, suite="nightly-4-gpu-gb300", nightly=True)
MODEL_PATH = "Qwen/Qwen3.5-397B-A17B-FP8"
COMMON_ARGS = [
"--trust-remote-code",
"--reasoning-parser=qwen3",
"--tool-call-parser=qwen3_coder",
"--enable-flashinfer-allreduce-fusion",
"--attention-backend=trtllm_mha",
"--mem-fraction-static=0.8",
"--enable-multimodal",
"--enable-metrics",
]
MTP_ARGS = [
"--speculative-algorithm=EAGLE",
"--speculative-num-steps=3",
"--speculative-eagle-topk=1",
"--speculative-num-draft-tokens=4",
"--mamba-scheduler-strategy=extra_buffer",
"--page-size=64",
]
class TestQwen35Fp8(unittest.TestCase):
"""Qwen3.5-397B FP8 on GB300 (4x B200 NVL4, tp=4)."""
def test_qwen35_fp8(self):
variants = [
ModelLaunchSettings(
MODEL_PATH,
tp_size=4,
extra_args=COMMON_ARGS,
variant="TP4",
),
ModelLaunchSettings(
MODEL_PATH,
tp_size=4,
extra_args=COMMON_ARGS + ["--dp-size=4", "--enable-dp-attention"],
variant="TP4+DP4+DPA",
),
ModelLaunchSettings(
MODEL_PATH,
tp_size=4,
extra_args=COMMON_ARGS
+ ["--dp-size=4", "--enable-dp-attention"]
+ MTP_ARGS,
variant="TP4+DP4+DPA+MTP",
env={"SGLANG_ENABLE_SPEC_V2": "1"},
),
]
run_combined_tests(
models=variants,
test_name="Qwen3.5-397B-FP8",
accuracy_params=AccuracyTestParams(
dataset="mmmu-pro", baseline_accuracy=0.78, repeat=1, max_tokens=32768
),
performance_params=PerformanceTestParams(
profile_dir="performance_profiles_gb300",
),
)
if __name__ == "__main__":
unittest.main()
test/registered/gb300/test_qwen35_nvfp4.py
+79
View File
@@ -0,0 +1,79 @@
import unittest
from sglang.test.accuracy_test_runner import AccuracyTestParams
from sglang.test.ci.ci_register import register_cuda_ci
from sglang.test.performance_test_runner import PerformanceTestParams
from sglang.test.run_combined_tests import run_combined_tests
from sglang.test.test_utils import ModelLaunchSettings
register_cuda_ci(est_time=7200, suite="nightly-4-gpu-gb300", nightly=True)
MODEL_PATH = "nvidia/Qwen3.5-397B-A17B-NVFP4"
COMMON_ARGS = [
"--trust-remote-code",
"--reasoning-parser=qwen3",
"--tool-call-parser=qwen3_coder",
"--quantization=modelopt_fp4",
"--fp4-gemm-backend=flashinfer_cutlass",
"--moe-runner-backend=flashinfer_trtllm",
"--kv-cache-dtype=fp8_e4m3",
"--enable-flashinfer-allreduce-fusion",
"--attention-backend=trtllm_mha",
"--mem-fraction-static=0.8",
"--enable-multimodal",
"--enable-metrics",
]
MTP_ARGS = [
"--speculative-algorithm=EAGLE",
"--speculative-num-steps=3",
"--speculative-eagle-topk=1",
"--speculative-num-draft-tokens=4",
"--mamba-scheduler-strategy=extra_buffer",
"--page-size=64",
]
class TestQwen35Nvfp4(unittest.TestCase):
"""Qwen3.5-397B NVFP4 on GB300 (4x B200 NVL4, tp=4)."""
def test_qwen35_nvfp4(self):
variants = [
ModelLaunchSettings(
MODEL_PATH,
tp_size=4,
extra_args=COMMON_ARGS,
variant="TP4",
),
ModelLaunchSettings(
MODEL_PATH,
tp_size=4,
extra_args=COMMON_ARGS + ["--dp-size=4", "--enable-dp-attention"],
variant="TP4+DP4+DPA",
),
ModelLaunchSettings(
MODEL_PATH,
tp_size=4,
extra_args=COMMON_ARGS
+ ["--dp-size=4", "--enable-dp-attention"]
+ MTP_ARGS,
variant="TP4+DP4+DPA+MTP",
env={"SGLANG_ENABLE_SPEC_V2": "1"},
),
]
run_combined_tests(
models=variants,
test_name="Qwen3.5-397B-NVFP4",
accuracy_params=AccuracyTestParams(
dataset="mmmu-pro", baseline_accuracy=0.78, repeat=1, max_tokens=32768
),
performance_params=PerformanceTestParams(
profile_dir="performance_profiles_gb300",
),
)
if __name__ == "__main__":
unittest.main()
test/run_suite.py
+2
View File
@@ -84,6 +84,8 @@ NIGHTLY_SUITES = {
"nightly-eval-vlm-2-gpu",
"nightly-perf-text-2-gpu",
"nightly-perf-vlm-2-gpu",
# GB300 (4x B200 NVL4) nightly suite
"nightly-4-gpu-gb300",
],
HWBackend.AMD: [
"nightly-amd",