minke.yu/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 12 Packages Projects Releases Wiki Activity

Stabilize GB300 nightly tests (#35044)

Browse Source
This commit is contained in:
Mohammad Miadh Angkad
2026-08-17 03:30:16 -07:00
committed by GitHub
parent 744740dbea
commit 82995a001b
13 changed files with 515 additions and 351 deletions
Download Patch File Download Diff File Expand all files Collapse all files
python/sglang/test/gb300_utils.py
+6
View File
@@ -0,0 +1,6 @@
"""Shared configuration for tests on the dedicated GB300 runner."""
# The runner's ephemeral client-port range is 10240-65535. Keep the TCPStore
# rendezvous below it so a recently closed client connection cannot make the
# subsequent TCPStore bind fail with EADDRINUSE.
GB300_NCCL_PORT = "10000"
test/registered/attention/test_glm4_moe_lite_deterministic.py
+1 -1
View File
@@ -20,7 +20,7 @@ from sglang.test.test_deterministic_utils import (
)
from sglang.test.test_utils import DEFAULT_URL_FOR_TEST
register_cuda_ci(est_time=900, stage="nightly", runner_config="4-gpu-gb300")
register_cuda_ci(est_time=360, stage="nightly", runner_config="4-gpu-b200")
GLM_MODEL = "zai-org/GLM-4.7-Flash"
test/registered/gb300/test_deepseek_v4_pro_fp4.py
-150
View File
@@ -1,150 +0,0 @@
import unittest
from sglang.test.accuracy_test_runner import AccuracyTestParams
from sglang.test.ci.ci_register import register_cuda_ci
from sglang.test.performance_test_runner import PerformanceTestParams
from sglang.test.run_combined_tests import run_combined_tests
from sglang.test.test_utils import ModelLaunchSettings
register_cuda_ci(est_time=7200, stage="nightly", runner_config="4-gpu-gb300")
MODEL_PATH = "deepseek-ai/DeepSeek-V4-Pro"
SERVER_LAUNCH_TIMEOUT = 3600
DEEPEP_CONFIG = '{"normal_dispatch":{"num_sms":96},"normal_combine":{"num_sms":96}}'
LOW_LATENCY_ARGS = [
"--trust-remote-code",
"--moe-runner-backend",
"flashinfer_mxfp4",
"--speculative-algorithm",
"EAGLE",
"--speculative-num-steps",
"3",
"--speculative-eagle-topk",
"1",
"--speculative-num-draft-tokens",
"4",
"--chunked-prefill-size",
"8192",
"--disable-flashinfer-autotune",
"--swa-full-tokens-ratio",
"0.1",
"--mem-fraction-static",
"0.85",
]
BALANCED_ARGS = [
"--trust-remote-code",
"--dp",
"4",
"--enable-dp-attention",
"--moe-a2a-backend",
"deepep",
"--speculative-algorithm",
"EAGLE",
"--speculative-num-steps",
"1",
"--speculative-eagle-topk",
"1",
"--speculative-num-draft-tokens",
"2",
"--mem-fraction-static",
"0.85",
"--cuda-graph-max-bs-decode",
"128",
"--max-running-requests",
"256",
"--deepep-config",
DEEPEP_CONFIG,
]
HIGH_THROUGHPUT_ARGS = [
"--trust-remote-code",
"--dp",
"4",
"--enable-dp-attention",
"--moe-a2a-backend",
"megamoe",
"--mem-fraction-static",
"0.9",
"--cuda-graph-max-bs-decode",
"128",
"--max-running-requests",
"256",
]
BALANCED_ENV = {
"SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK": "256",
}
HIGH_THROUGHPUT_ENV = {
"SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK": "8320",
}
PERFORMANCE_BATCH_SIZES = {
"low-latency": [1, 4, 16],
"balanced": [64],
"high-throughput": [128],
}
class TestDeepSeekV4ProFp4(unittest.TestCase):
"""DeepSeek-V4-Pro FP4 on GB300 (4x B200 NVL4, tp=4)."""
def test_deepseek_v4_pro_fp4(self):
variants = [
ModelLaunchSettings(
MODEL_PATH,
tp_size=4,
extra_args=LOW_LATENCY_ARGS,
variant="low-latency",
launch_timeout=SERVER_LAUNCH_TIMEOUT,
),
ModelLaunchSettings(
MODEL_PATH,
tp_size=4,
extra_args=BALANCED_ARGS,
env=BALANCED_ENV,
variant="balanced",
launch_timeout=SERVER_LAUNCH_TIMEOUT,
),
ModelLaunchSettings(
MODEL_PATH,
tp_size=4,
extra_args=HIGH_THROUGHPUT_ARGS,
env=HIGH_THROUGHPUT_ENV,
variant="high-throughput",
launch_timeout=SERVER_LAUNCH_TIMEOUT,
),
]
failures = []
accuracy_params = AccuracyTestParams(
dataset="gsm8k",
baseline_accuracy=0.935,
temperature=1.0,
top_p=1.0,
)
for variant in variants:
try:
run_combined_tests(
models=[variant],
test_name=f"DeepSeek-V4-Pro-FP4 ({variant.variant})",
accuracy_params=accuracy_params,
performance_params=PerformanceTestParams(
batch_sizes=PERFORMANCE_BATCH_SIZES[variant.variant],
result_dir="performance_results_gb300",
),
)
except AssertionError as e:
failures.append(f"{variant.variant}: {e}")
if failures:
raise AssertionError(
"DeepSeek-V4-Pro-FP4 failures:\n" + "\n".join(failures)
)
if __name__ == "__main__":
unittest.main()
test/registered/gb300/test_deepseek_v4_pro_fp4_balanced.py
+78
View File
@@ -0,0 +1,78 @@
import unittest
from sglang.test.accuracy_test_runner import AccuracyTestParams
from sglang.test.ci.ci_register import register_cuda_ci
from sglang.test.gb300_utils import GB300_NCCL_PORT
from sglang.test.performance_test_runner import PerformanceTestParams
from sglang.test.run_combined_tests import run_combined_tests
from sglang.test.test_utils import CustomTestCase, ModelLaunchSettings
register_cuda_ci(est_time=600, stage="nightly", runner_config="4-gpu-gb300")
MODEL_PATH = "deepseek-ai/DeepSeek-V4-Pro"
SERVER_LAUNCH_TIMEOUT = 3600
DEEPEP_CONFIG = '{"normal_dispatch":{"num_sms":96},"normal_combine":{"num_sms":96}}'
BALANCED_ARGS = [
"--trust-remote-code",
"--dp",
"4",
"--enable-dp-attention",
"--moe-a2a-backend",
"deepep",
"--speculative-algorithm",
"EAGLE",
"--speculative-num-steps",
"1",
"--speculative-eagle-topk",
"1",
"--speculative-num-draft-tokens",
"2",
"--mem-fraction-static",
"0.85",
"--cuda-graph-max-bs-decode",
"128",
"--max-running-requests",
"256",
"--deepep-config",
DEEPEP_CONFIG,
"--nccl-port",
GB300_NCCL_PORT,
]
BALANCED_ENV = {
"SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK": "256",
}
class TestDeepSeekV4ProFp4Balanced(CustomTestCase):
"""DeepSeek-V4-Pro FP4 balanced config on GB300."""
def test_deepseek_v4_pro_fp4_balanced(self):
run_combined_tests(
models=[
ModelLaunchSettings(
MODEL_PATH,
tp_size=4,
extra_args=BALANCED_ARGS,
env=BALANCED_ENV,
variant="balanced",
launch_timeout=SERVER_LAUNCH_TIMEOUT,
)
],
test_name="DeepSeek-V4-Pro-FP4 (balanced)",
accuracy_params=AccuracyTestParams(
dataset="gsm8k",
baseline_accuracy=0.935,
temperature=1.0,
top_p=1.0,
),
performance_params=PerformanceTestParams(
batch_sizes=[64],
result_dir="performance_results_gb300",
),
)
if __name__ == "__main__":
unittest.main()
test/registered/gb300/test_deepseek_v4_pro_fp4_high_throughput.py
+67
View File
@@ -0,0 +1,67 @@
import unittest
from sglang.test.accuracy_test_runner import AccuracyTestParams
from sglang.test.ci.ci_register import register_cuda_ci
from sglang.test.gb300_utils import GB300_NCCL_PORT
from sglang.test.performance_test_runner import PerformanceTestParams
from sglang.test.run_combined_tests import run_combined_tests
from sglang.test.test_utils import CustomTestCase, ModelLaunchSettings
register_cuda_ci(est_time=600, stage="nightly", runner_config="4-gpu-gb300")
MODEL_PATH = "deepseek-ai/DeepSeek-V4-Pro"
SERVER_LAUNCH_TIMEOUT = 3600
HIGH_THROUGHPUT_ARGS = [
"--trust-remote-code",
"--dp",
"4",
"--enable-dp-attention",
"--moe-a2a-backend",
"megamoe",
"--mem-fraction-static",
"0.9",
"--cuda-graph-max-bs-decode",
"128",
"--max-running-requests",
"256",
"--nccl-port",
GB300_NCCL_PORT,
]
HIGH_THROUGHPUT_ENV = {
"SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK": "8320",
}
class TestDeepSeekV4ProFp4HighThroughput(CustomTestCase):
"""DeepSeek-V4-Pro FP4 high-throughput config on GB300."""
def test_deepseek_v4_pro_fp4_high_throughput(self):
run_combined_tests(
models=[
ModelLaunchSettings(
MODEL_PATH,
tp_size=4,
extra_args=HIGH_THROUGHPUT_ARGS,
env=HIGH_THROUGHPUT_ENV,
variant="high-throughput",
launch_timeout=SERVER_LAUNCH_TIMEOUT,
)
],
test_name="DeepSeek-V4-Pro-FP4 (high-throughput)",
accuracy_params=AccuracyTestParams(
dataset="gsm8k",
baseline_accuracy=0.935,
temperature=1.0,
top_p=1.0,
),
performance_params=PerformanceTestParams(
batch_sizes=[128],
result_dir="performance_results_gb300",
),
)
if __name__ == "__main__":
unittest.main()
test/registered/gb300/test_deepseek_v4_pro_fp4_low_latency.py
+68
View File
@@ -0,0 +1,68 @@
import unittest
from sglang.test.accuracy_test_runner import AccuracyTestParams
from sglang.test.ci.ci_register import register_cuda_ci
from sglang.test.gb300_utils import GB300_NCCL_PORT
from sglang.test.performance_test_runner import PerformanceTestParams
from sglang.test.run_combined_tests import run_combined_tests
from sglang.test.test_utils import CustomTestCase, ModelLaunchSettings
register_cuda_ci(est_time=720, stage="nightly", runner_config="4-gpu-gb300")
MODEL_PATH = "deepseek-ai/DeepSeek-V4-Pro"
SERVER_LAUNCH_TIMEOUT = 3600
LOW_LATENCY_ARGS = [
"--trust-remote-code",
"--moe-runner-backend",
"flashinfer_mxfp4",
"--speculative-algorithm",
"EAGLE",
"--speculative-num-steps",
"3",
"--speculative-eagle-topk",
"1",
"--speculative-num-draft-tokens",
"4",
"--chunked-prefill-size",
"8192",
"--disable-flashinfer-autotune",
"--swa-full-tokens-ratio",
"0.1",
"--mem-fraction-static",
"0.85",
"--nccl-port",
GB300_NCCL_PORT,
]
class TestDeepSeekV4ProFp4LowLatency(CustomTestCase):
"""DeepSeek-V4-Pro FP4 low-latency config on GB300."""
def test_deepseek_v4_pro_fp4_low_latency(self):
run_combined_tests(
models=[
ModelLaunchSettings(
MODEL_PATH,
tp_size=4,
extra_args=LOW_LATENCY_ARGS,
variant="low-latency",
launch_timeout=SERVER_LAUNCH_TIMEOUT,
)
],
test_name="DeepSeek-V4-Pro-FP4 (low-latency)",
accuracy_params=AccuracyTestParams(
dataset="gsm8k",
baseline_accuracy=0.935,
temperature=1.0,
top_p=1.0,
),
performance_params=PerformanceTestParams(
batch_sizes=[1, 4, 16],
result_dir="performance_results_gb300",
),
)
if __name__ == "__main__":
unittest.main()
test/registered/gb300/test_glm52_nvfp4.py
+5 -2
View File
@@ -2,9 +2,10 @@ import unittest
from sglang.test.accuracy_test_runner import AccuracyTestParams
from sglang.test.ci.ci_register import register_cuda_ci
from sglang.test.gb300_utils import GB300_NCCL_PORT
from sglang.test.performance_test_runner import PerformanceTestParams
from sglang.test.run_combined_tests import run_combined_tests
from sglang.test.test_utils import ModelLaunchSettings
from sglang.test.test_utils import CustomTestCase, ModelLaunchSettings
register_cuda_ci(est_time=2280, stage="nightly", runner_config="4-gpu-gb300")
@@ -18,6 +19,8 @@ COMMON_ARGS = [
"--moe-runner-backend=flashinfer_trtllm",
"--mem-fraction-static=0.9",
"--enable-metrics",
"--nccl-port",
GB300_NCCL_PORT,
]
TP_MTP_ARGS = [
@@ -35,7 +38,7 @@ DP_MTP_ARGS = [
]
class TestGlm52Nvfp4(unittest.TestCase):
class TestGlm52Nvfp4(CustomTestCase):
"""GLM-5.2 NVFP4 on GB300 (4x GB300 NVL4, tp=4)."""
def test_glm52_nvfp4(self):
test/registered/gb300/test_kimi_k25_nvfp4.py
-100
View File
@@ -1,100 +0,0 @@
import unittest
from sglang.test.accuracy_test_runner import AccuracyTestParams
from sglang.test.ci.ci_register import register_cuda_ci
from sglang.test.performance_test_runner import PerformanceTestParams
from sglang.test.run_combined_tests import run_combined_tests
from sglang.test.test_utils import ModelLaunchSettings
register_cuda_ci(est_time=7200, stage="nightly", runner_config="4-gpu-gb300")
MODEL_PATH = "nvidia/Kimi-K2.5-NVFP4"
DRAFT_MODEL_PATH = "lightseekorg/kimi-k2.5-eagle3-mla"
COMMON_ARGS = [
"--trust-remote-code",
"--reasoning-parser=kimi_k2",
"--tool-call-parser=kimi_k2",
"--quantization=modelopt_fp4",
"--attention-backend=tokenspeed_mla",
"--kv-cache-dtype=fp8_e4m3",
"--moe-runner-backend=flashinfer_trtllm",
"--mem-fraction-static=0.8",
"--enable-metrics",
"--speculative-algorithm=EAGLE3",
f"--speculative-draft-model-path={DRAFT_MODEL_PATH}",
"--speculative-draft-model-quantization=unquant",
]
TP_EAGLE_ARGS = [
"--speculative-num-steps=3",
"--speculative-eagle-topk=1",
"--speculative-num-draft-tokens=4",
]
DP_EAGLE_ARGS = [
"--speculative-num-steps=1",
"--speculative-eagle-topk=1",
"--speculative-num-draft-tokens=2",
]
PERFORMANCE_BATCH_SIZES = {
"TP4+EAGLE3": [1, 8],
"TP4+DP4+DPA+EAGLE3": [16],
}
class TestKimiK25Nvfp4(unittest.TestCase):
"""Kimi-K2.5 NVFP4 + EAGLE3 on GB300 (4x GB300 NVL4, tp=4)."""
def test_kimi_k25_nvfp4(self):
variants = [
ModelLaunchSettings(
MODEL_PATH,
tp_size=4,
extra_args=COMMON_ARGS + TP_EAGLE_ARGS,
variant="TP4+EAGLE3",
),
ModelLaunchSettings(
MODEL_PATH,
tp_size=4,
extra_args=COMMON_ARGS
+ ["--dp-size=4", "--enable-dp-attention"]
+ DP_EAGLE_ARGS,
variant="TP4+DP4+DPA+EAGLE3",
),
]
failures = []
# Pinned to what `ns eval --benchmarks=mmmu-pro:1` sent implicitly --
# its `:1` suffix means temperature 0.7, not greedy -- so the baseline
# carries over unchanged. Do not "simplify" these away.
accuracy_params = AccuracyTestParams(
dataset="mmmu_pro_vision",
baseline_accuracy=0.69,
repeat=1,
max_tokens=32768,
temperature=0.7,
seed=0,
sgl_eval_thinking=False,
)
for variant in variants:
try:
run_combined_tests(
models=[variant],
test_name=f"Kimi-K2.5-NVFP4 ({variant.variant})",
accuracy_params=accuracy_params,
performance_params=PerformanceTestParams(
batch_sizes=PERFORMANCE_BATCH_SIZES[variant.variant],
result_dir="performance_results_gb300",
),
)
except AssertionError as e:
failures.append(f"{variant.variant}: {e}")
if failures:
raise AssertionError("Kimi-K2.5-NVFP4 failures:\n" + "\n".join(failures))
if __name__ == "__main__":
unittest.main()
test/registered/gb300/test_kimi_k25_nvfp4_dp.py
+75
View File
@@ -0,0 +1,75 @@
import unittest
from sglang.test.accuracy_test_runner import AccuracyTestParams
from sglang.test.ci.ci_register import register_cuda_ci
from sglang.test.gb300_utils import GB300_NCCL_PORT
from sglang.test.performance_test_runner import PerformanceTestParams
from sglang.test.run_combined_tests import run_combined_tests
from sglang.test.test_utils import CustomTestCase, ModelLaunchSettings
register_cuda_ci(est_time=7200, stage="nightly", runner_config="4-gpu-gb300")
MODEL_PATH = "nvidia/Kimi-K2.5-NVFP4"
DRAFT_MODEL_PATH = "lightseekorg/kimi-k2.5-eagle3-mla"
COMMON_ARGS = [
"--trust-remote-code",
"--reasoning-parser=kimi_k2",
"--tool-call-parser=kimi_k2",
"--quantization=modelopt_fp4",
"--attention-backend=tokenspeed_mla",
"--kv-cache-dtype=fp8_e4m3",
"--moe-runner-backend=flashinfer_trtllm",
"--mem-fraction-static=0.8",
"--enable-metrics",
"--speculative-algorithm=EAGLE3",
f"--speculative-draft-model-path={DRAFT_MODEL_PATH}",
"--speculative-draft-model-quantization=unquant",
"--nccl-port",
GB300_NCCL_PORT,
]
DP_EAGLE_ARGS = [
"--speculative-num-steps=1",
"--speculative-eagle-topk=1",
"--speculative-num-draft-tokens=2",
]
class TestKimiK25Nvfp4Dp(CustomTestCase):
"""Kimi-K2.5 NVFP4 DP4+DPA+EAGLE3 on GB300 (4x GB300 NVL4)."""
def test_kimi_k25_nvfp4_dp(self):
# Pinned to what `ns eval --benchmarks=mmmu-pro:1` sent implicitly --
# its `:1` suffix means temperature 0.7, not greedy -- so the baseline
# carries over unchanged. Do not "simplify" these away.
run_combined_tests(
models=[
ModelLaunchSettings(
MODEL_PATH,
tp_size=4,
extra_args=COMMON_ARGS
+ ["--dp-size=4", "--enable-dp-attention"]
+ DP_EAGLE_ARGS,
variant="TP4+DP4+DPA+EAGLE3",
)
],
test_name="Kimi-K2.5-NVFP4 (TP4+DP4+DPA+EAGLE3)",
accuracy_params=AccuracyTestParams(
dataset="mmmu_pro_vision",
baseline_accuracy=0.69,
repeat=1,
max_tokens=32768,
temperature=0.7,
seed=0,
sgl_eval_thinking=False,
),
performance_params=PerformanceTestParams(
batch_sizes=[16],
result_dir="performance_results_gb300",
),
)
if __name__ == "__main__":
unittest.main()
test/registered/gb300/test_kimi_k25_nvfp4_tp.py
+73
View File
@@ -0,0 +1,73 @@
import unittest
from sglang.test.accuracy_test_runner import AccuracyTestParams
from sglang.test.ci.ci_register import register_cuda_ci
from sglang.test.gb300_utils import GB300_NCCL_PORT
from sglang.test.performance_test_runner import PerformanceTestParams
from sglang.test.run_combined_tests import run_combined_tests
from sglang.test.test_utils import CustomTestCase, ModelLaunchSettings
register_cuda_ci(est_time=4800, stage="nightly", runner_config="4-gpu-gb300")
MODEL_PATH = "nvidia/Kimi-K2.5-NVFP4"
DRAFT_MODEL_PATH = "lightseekorg/kimi-k2.5-eagle3-mla"
COMMON_ARGS = [
"--trust-remote-code",
"--reasoning-parser=kimi_k2",
"--tool-call-parser=kimi_k2",
"--quantization=modelopt_fp4",
"--attention-backend=tokenspeed_mla",
"--kv-cache-dtype=fp8_e4m3",
"--moe-runner-backend=flashinfer_trtllm",
"--mem-fraction-static=0.8",
"--enable-metrics",
"--speculative-algorithm=EAGLE3",
f"--speculative-draft-model-path={DRAFT_MODEL_PATH}",
"--speculative-draft-model-quantization=unquant",
"--nccl-port",
GB300_NCCL_PORT,
]
TP_EAGLE_ARGS = [
"--speculative-num-steps=3",
"--speculative-eagle-topk=1",
"--speculative-num-draft-tokens=4",
]
class TestKimiK25Nvfp4Tp(CustomTestCase):
"""Kimi-K2.5 NVFP4 TP4+EAGLE3 on GB300 (4x GB300 NVL4)."""
def test_kimi_k25_nvfp4_tp(self):
# Pinned to what `ns eval --benchmarks=mmmu-pro:1` sent implicitly --
# its `:1` suffix means temperature 0.7, not greedy -- so the baseline
# carries over unchanged. Do not "simplify" these away.
run_combined_tests(
models=[
ModelLaunchSettings(
MODEL_PATH,
tp_size=4,
extra_args=COMMON_ARGS + TP_EAGLE_ARGS,
variant="TP4+EAGLE3",
)
],
test_name="Kimi-K2.5-NVFP4 (TP4+EAGLE3)",
accuracy_params=AccuracyTestParams(
dataset="mmmu_pro_vision",
baseline_accuracy=0.69,
repeat=1,
max_tokens=32768,
temperature=0.7,
seed=0,
sgl_eval_thinking=False,
),
performance_params=PerformanceTestParams(
batch_sizes=[1, 8],
result_dir="performance_results_gb300",
),
)
if __name__ == "__main__":
unittest.main()
test/registered/gb300/test_qwen35_fp8.py
-98
View File
@@ -1,98 +0,0 @@
import unittest
from sglang.test.accuracy_test_runner import AccuracyTestParams
from sglang.test.ci.ci_register import register_cuda_ci
from sglang.test.performance_test_runner import PerformanceTestParams
from sglang.test.run_combined_tests import run_combined_tests
from sglang.test.test_utils import ModelLaunchSettings
register_cuda_ci(est_time=7200, stage="nightly", runner_config="4-gpu-gb300")
MODEL_PATH = "Qwen/Qwen3.5-397B-A17B-FP8"
COMMON_ARGS = [
"--trust-remote-code",
"--reasoning-parser=qwen3",
"--tool-call-parser=qwen3_coder",
"--enable-flashinfer-allreduce-fusion",
"--attention-backend=trtllm_mha",
"--mem-fraction-static=0.8",
"--mamba-scheduler-strategy=extra_buffer",
"--enable-multimodal",
"--enable-metrics",
]
TP_MTP_ARGS = [
"--speculative-algorithm=EAGLE",
"--speculative-num-steps=3",
"--speculative-eagle-topk=1",
"--speculative-num-draft-tokens=4",
]
DP_MTP_ARGS = [
"--speculative-algorithm=EAGLE",
"--speculative-num-steps=1",
"--speculative-eagle-topk=1",
"--speculative-num-draft-tokens=2",
]
PERFORMANCE_BATCH_SIZES = {
"TP4+MTP": [1, 4],
"TP4+DP4+DPA+MTP": [16],
}
class TestQwen35Fp8(unittest.TestCase):
"""Qwen3.5-397B FP8 on GB300 (4x GB300 NVL4, tp=4)."""
def test_qwen35_fp8(self):
variants = [
ModelLaunchSettings(
MODEL_PATH,
tp_size=4,
extra_args=COMMON_ARGS + TP_MTP_ARGS,
variant="TP4+MTP",
),
ModelLaunchSettings(
MODEL_PATH,
tp_size=4,
extra_args=COMMON_ARGS
+ ["--dp-size=4", "--enable-dp-attention"]
+ DP_MTP_ARGS,
variant="TP4+DP4+DPA+MTP",
),
]
failures = []
# Pinned to what `ns eval --benchmarks=mmmu-pro:1` sent implicitly --
# its `:1` suffix means temperature 0.7, not greedy -- so the baseline
# carries over unchanged. Do not "simplify" these away.
accuracy_params = AccuracyTestParams(
dataset="mmmu_pro_vision",
baseline_accuracy=0.76,
repeat=1,
max_tokens=32768,
temperature=0.7,
seed=0,
sgl_eval_thinking=False,
)
for variant in variants:
try:
run_combined_tests(
models=[variant],
test_name=f"Qwen3.5-397B-FP8 ({variant.variant})",
accuracy_params=accuracy_params,
performance_params=PerformanceTestParams(
batch_sizes=PERFORMANCE_BATCH_SIZES[variant.variant],
result_dir="performance_results_gb300",
),
)
except AssertionError as e:
failures.append(f"{variant.variant}: {e}")
if failures:
raise AssertionError("Qwen3.5-397B-FP8 failures:\n" + "\n".join(failures))
if __name__ == "__main__":
unittest.main()
test/registered/gb300/test_qwen35_fp8_dp.py
+72
View File
@@ -0,0 +1,72 @@
import unittest
from sglang.test.accuracy_test_runner import AccuracyTestParams
from sglang.test.ci.ci_register import register_cuda_ci
from sglang.test.gb300_utils import GB300_NCCL_PORT
from sglang.test.performance_test_runner import PerformanceTestParams
from sglang.test.run_combined_tests import run_combined_tests
from sglang.test.test_utils import CustomTestCase, ModelLaunchSettings
register_cuda_ci(est_time=7200, stage="nightly", runner_config="4-gpu-gb300")
MODEL_PATH = "Qwen/Qwen3.5-397B-A17B-FP8"
COMMON_ARGS = [
"--trust-remote-code",
"--reasoning-parser=qwen3",
"--tool-call-parser=qwen3_coder",
"--enable-flashinfer-allreduce-fusion",
"--attention-backend=trtllm_mha",
"--mem-fraction-static=0.8",
"--mamba-scheduler-strategy=extra_buffer",
"--enable-multimodal",
"--enable-metrics",
"--nccl-port",
GB300_NCCL_PORT,
]
DP_MTP_ARGS = [
"--speculative-algorithm=EAGLE",
"--speculative-num-steps=1",
"--speculative-eagle-topk=1",
"--speculative-num-draft-tokens=2",
]
class TestQwen35Fp8Dp(CustomTestCase):
"""Qwen3.5-397B FP8 DP4+DPA+MTP on GB300 (4x GB300 NVL4)."""
def test_qwen35_fp8_dp(self):
# Pinned to what `ns eval --benchmarks=mmmu-pro:1` sent implicitly --
# its `:1` suffix means temperature 0.7, not greedy -- so the baseline
# carries over unchanged. Do not "simplify" these away.
run_combined_tests(
models=[
ModelLaunchSettings(
MODEL_PATH,
tp_size=4,
extra_args=COMMON_ARGS
+ ["--dp-size=4", "--enable-dp-attention"]
+ DP_MTP_ARGS,
variant="TP4+DP4+DPA+MTP",
)
],
test_name="Qwen3.5-397B-FP8 (TP4+DP4+DPA+MTP)",
accuracy_params=AccuracyTestParams(
dataset="mmmu_pro_vision",
baseline_accuracy=0.76,
repeat=1,
max_tokens=32768,
temperature=0.7,
seed=0,
sgl_eval_thinking=False,
),
performance_params=PerformanceTestParams(
batch_sizes=[16],
result_dir="performance_results_gb300",
),
)
if __name__ == "__main__":
unittest.main()
test/registered/gb300/test_qwen35_fp8_tp.py
+70
View File
@@ -0,0 +1,70 @@
import unittest
from sglang.test.accuracy_test_runner import AccuracyTestParams
from sglang.test.ci.ci_register import register_cuda_ci
from sglang.test.gb300_utils import GB300_NCCL_PORT
from sglang.test.performance_test_runner import PerformanceTestParams
from sglang.test.run_combined_tests import run_combined_tests
from sglang.test.test_utils import CustomTestCase, ModelLaunchSettings
register_cuda_ci(est_time=4800, stage="nightly", runner_config="4-gpu-gb300")
MODEL_PATH = "Qwen/Qwen3.5-397B-A17B-FP8"
COMMON_ARGS = [
"--trust-remote-code",
"--reasoning-parser=qwen3",
"--tool-call-parser=qwen3_coder",
"--enable-flashinfer-allreduce-fusion",
"--attention-backend=trtllm_mha",
"--mem-fraction-static=0.8",
"--mamba-scheduler-strategy=extra_buffer",
"--enable-multimodal",
"--enable-metrics",
"--nccl-port",
GB300_NCCL_PORT,
]
TP_MTP_ARGS = [
"--speculative-algorithm=EAGLE",
"--speculative-num-steps=3",
"--speculative-eagle-topk=1",
"--speculative-num-draft-tokens=4",
]
class TestQwen35Fp8Tp(CustomTestCase):
"""Qwen3.5-397B FP8 TP4+MTP on GB300 (4x GB300 NVL4)."""
def test_qwen35_fp8_tp(self):
# Pinned to what `ns eval --benchmarks=mmmu-pro:1` sent implicitly --
# its `:1` suffix means temperature 0.7, not greedy -- so the baseline
# carries over unchanged. Do not "simplify" these away.
run_combined_tests(
models=[
ModelLaunchSettings(
MODEL_PATH,
tp_size=4,
extra_args=COMMON_ARGS + TP_MTP_ARGS,
variant="TP4+MTP",
)
],
test_name="Qwen3.5-397B-FP8 (TP4+MTP)",
accuracy_params=AccuracyTestParams(
dataset="mmmu_pro_vision",
baseline_accuracy=0.76,
repeat=1,
max_tokens=32768,
temperature=0.7,
seed=0,
sgl_eval_thinking=False,
),
performance_params=PerformanceTestParams(
batch_sizes=[1, 8],
result_dir="performance_results_gb300",
),
)
if __name__ == "__main__":
unittest.main()