minke.yu/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 12 Packages Projects Releases Wiki Activity

ci: tag-gated nightly migration — foundation + 40 whole-file moves (#24725)

Browse Source 
Co-authored-by: hnyls2002 <lsyincs@gmail.com>
Co-authored-by: Liangsheng Yin <hnyls2002@gmail.com>
This commit is contained in:
Alison Shao
2026-05-14 16:11:15 -07:00
committed by GitHub
co-authored by hnyls2002 Liangsheng Yin
parent 67096f48bf
commit ba214ef3d3
78 changed files with 2254 additions and 2131 deletions
Download Patch File Download Diff File Expand all files Collapse all files
test/registered/openai_server/function_call/test_tool_choice.py → test/manual/openai_server/function_call/test_tool_choice.py
View File
test/manual/quant/test_deepseek_v3_fp4_4gpu_trtllm.py
+93
View File
@@ -0,0 +1,93 @@
"""DeepSeek-V3 FP4 4-GPU test, TRTLLM variant.
Backend: `--attention-backend trtllm_mla --moe-runner-backend flashinfer_trtllm`.
Not registered in any CI suite -- runnable manually only.
"""
import unittest
from types import SimpleNamespace
from sglang.srt.utils import kill_process_tree
from sglang.test.run_eval import run_eval
from sglang.test.send_one import BenchArgs, send_one_prompt
from sglang.test.test_utils import (
DEFAULT_URL_FOR_TEST,
CustomTestCase,
is_in_ci,
popen_launch_server,
write_github_step_summary,
)
FULL_DEEPSEEK_V3_FP4_MODEL_PATH = "nvidia/DeepSeek-V3-0324-FP4"
SERVER_LAUNCH_TIMEOUT = 1200
class TestDeepseekV3FP4(CustomTestCase):
@classmethod
def setUpClass(cls):
cls.model = FULL_DEEPSEEK_V3_FP4_MODEL_PATH
cls.base_url = DEFAULT_URL_FOR_TEST
other_args = [
"--tp",
"4",
"--attention-backend",
"trtllm_mla",
"--moe-runner-backend",
"flashinfer_trtllm",
"--quantization",
"modelopt_fp4",
"--kv-cache-dtype",
"fp8_e4m3",
"--model-loader-extra-config",
'{"enable_multithread_load": true,"num_threads": 64}',
]
cls.process = popen_launch_server(
cls.model,
cls.base_url,
timeout=SERVER_LAUNCH_TIMEOUT,
other_args=other_args,
)
@classmethod
def tearDownClass(cls):
kill_process_tree(cls.process.pid)
def test_a_gsm8k(
self,
): # Append an "a" to make this test run first (alphabetically) to warm up the server
args = SimpleNamespace(
base_url=self.base_url,
model=self.model,
eval_name="gsm8k",
api="completion",
max_tokens=512,
num_examples=1319,
num_threads=1319,
num_shots=8,
)
metrics = run_eval(args)
print(f"{metrics=}")
if is_in_ci():
write_github_step_summary(
f"### test_gsm8k (deepseek-v3-fp4)\n" f'{metrics["score"]=:.3f}\n'
)
self.assertGreater(metrics["score"], 0.93)
def test_bs_1_speed(self):
args = BenchArgs(port=int(self.base_url.split(":")[-1]), max_new_tokens=2048)
_, speed = send_one_prompt(args)
print(f"{speed=:.2f}")
if is_in_ci():
write_github_step_summary(
f"### test_bs_1_speed (deepseek-v3-fp4)\n" f"{speed=:.2f} token/s\n"
)
self.assertGreater(speed, 120)
if __name__ == "__main__":
unittest.main()
test/manual/spec/test_spec_ngram_fa3.py
+18
View File
@@ -0,0 +1,18 @@
"""NGRAM speculative-decoding test, FA3 attention-backend variant.
Backend: `--attention-backend fa3`.
Not registered in any CI suite -- runnable manually only.
"""
import unittest
from sglang.test.kits.eval_accuracy_kit import GSM8KMixin
from sglang.test.server_fixtures.ngram_fixture import NgramServerBase
class TestNgramSpeculativeDecodingBase(NgramServerBase, GSM8KMixin):
attention_backend = "fa3"
if __name__ == "__main__":
unittest.main()
test/registered/4-gpu-models/test_qwen35_hicache.py
+1 -1
View File
@@ -22,7 +22,7 @@ from sglang.test.test_utils import (
popen_launch_server,
)
register_cuda_ci(est_time=540, stage="stage-c", runner_config="4-gpu-h100")
register_cuda_ci(est_time=540, stage="extra-b", runner_config="4-gpu-h100")
QWEN35_27B_MODEL = "Qwen/Qwen3.5-27B"
ACC_THRESHOLDS = {QWEN35_27B_MODEL: {"gsm8k": 0.8}}
test/registered/8-gpu-models/test_deepseek_v32_indexcache.py
+1 -1
View File
@@ -13,7 +13,7 @@ from sglang.test.test_utils import (
write_github_step_summary,
)
register_cuda_ci(est_time=450, suite="nightly-8-gpu-h200", nightly=True)
register_cuda_ci(est_time=450, stage="extra-b", runner_config="8-gpu-h200")
DEEPSEEK_V32_MODEL_PATH = "deepseek-ai/DeepSeek-V3.2"
test/registered/8-gpu-models/test_nvidia_nemotron_3_super_bf16.py
+1 -1
View File
@@ -11,7 +11,7 @@ from sglang.test.test_utils import (
popen_launch_server,
)
register_cuda_ci(est_time=370, suite="nightly-8-gpu-h200", nightly=True)
register_cuda_ci(est_time=370, stage="extra-b", runner_config="8-gpu-h200")
NEMOTRON_3_SUPER_BF16_MODEL = "nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-BF16"
test/registered/8-gpu-models/test_return_indexer_topk.py
+1 -1
View File
@@ -17,7 +17,7 @@ from sglang.test.test_utils import (
popen_launch_server,
)
register_cuda_ci(est_time=270, suite="nightly-8-gpu-h200", nightly=True)
register_cuda_ci(est_time=270, stage="extra-b", runner_config="8-gpu-h200")
DEEPSEEK_V32_MODEL_PATH = "deepseek-ai/DeepSeek-V3.2"
test/registered/8-gpu-models/test_step3p5_flash_chain_mtp.py
+1 -1
View File
@@ -17,7 +17,7 @@ from sglang.test.test_utils import (
write_github_step_summary,
)
register_cuda_ci(est_time=480, suite="nightly-8-gpu-h200", nightly=True)
register_cuda_ci(est_time=480, stage="extra-b", runner_config="8-gpu-h200")
STEP3P5_FLASH_MODEL_PATH = "stepfun-ai/Step-3.5-Flash"
test/registered/attention/test_hybrid_attn_backend.py
+33 -132
View File
@@ -1,166 +1,67 @@
import unittest
from types import SimpleNamespace
import requests
from sglang.srt.environ import envs
from sglang.srt.utils import get_device_sm, kill_process_tree
from sglang.test.ci.ci_register import register_cuda_ci
from sglang.test.run_eval import run_eval
from sglang.test.server_fixtures.hybrid_attn_backend_fixture import (
TestHybridAttnBackendBase,
)
from sglang.test.test_utils import (
DEFAULT_DRAFT_MODEL_EAGLE,
DEFAULT_MODEL_NAME_FOR_TEST,
DEFAULT_MODEL_NAME_FOR_TEST_MLA,
DEFAULT_TARGET_MODEL_EAGLE,
DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
DEFAULT_URL_FOR_TEST,
CustomTestCase,
popen_launch_server,
)
# Hybrid attention backend tests (FA3 prefill + FlashInfer decode, requires SM 90+ / H100)
# Multiple test classes: base, MLA, TorchCompile, SpecDecode variants
register_cuda_ci(est_time=407, stage="stage-b", runner_config="1-gpu-large")
GSM_DATASET_PATH = None
# Default server arguments shared across all tests
DEFAULT_SERVER_ARGS = [
"--trust-remote-code",
"--cuda-graph-max-bs",
"8",
"--prefill-attention-backend",
"fa3",
"--decode-attention-backend",
"flashinfer",
]
@unittest.skipIf(get_device_sm() < 90, "Test requires CUDA SM 90 or higher")
class TestHybridAttnBackendBase(CustomTestCase):
model = DEFAULT_MODEL_NAME_FOR_TEST
base_url = DEFAULT_URL_FOR_TEST
accuracy_threshold = 0.65 # derived tests need to override this
speculative_decode = False
spec_decode_threshold = 2.2 # derived spec decoding tests need to override this
@classmethod
def get_server_args(cls):
"""Return the arguments for the server launch. Override in subclasses."""
return DEFAULT_SERVER_ARGS
@classmethod
def setUpClass(cls):
# disable deep gemm precompile to make launch server faster
# please don't do this if you want to make your inference workload faster
with (
envs.SGLANG_JIT_DEEPGEMM_PRECOMPILE.override(False),
envs.SGLANG_ENABLE_JIT_DEEPGEMM.override(False),
):
if cls.speculative_decode:
model = DEFAULT_TARGET_MODEL_EAGLE
else:
model = cls.model
cls.process = popen_launch_server(
model,
cls.base_url,
timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
other_args=cls.get_server_args(),
)
@classmethod
def tearDownClass(cls):
kill_process_tree(cls.process.pid)
def test_gsm8k(self):
requests.get(self.base_url + "/flush_cache")
model = DEFAULT_TARGET_MODEL_EAGLE if self.speculative_decode else self.model
args = SimpleNamespace(
base_url=self.base_url,
model=model,
eval_name="gsm8k",
api="completion",
max_tokens=512,
num_examples=100,
num_threads=128,
)
metrics = run_eval(args)
print(f"{metrics=}")
self.assertGreater(metrics["score"], self.accuracy_threshold)
if self.speculative_decode:
server_info = requests.get(self.base_url + "/server_info")
avg_spec_accept_length = server_info.json()["internal_states"][0][
"avg_spec_accept_length"
]
print(f"{avg_spec_accept_length=}")
self.assertGreater(avg_spec_accept_length, self.spec_decode_threshold)
register_cuda_ci(est_time=407, stage="extra-a", runner_config="1-gpu-large")
class TestHybridAttnBackendMLA(TestHybridAttnBackendBase):
accuracy_threshold = 0.60
model = DEFAULT_MODEL_NAME_FOR_TEST_MLA
@classmethod
def get_server_args(cls):
return DEFAULT_SERVER_ARGS
class TestHybridAttnBackendTorchCompile(TestHybridAttnBackendBase):
accuracy_threshold = 0.65
@classmethod
def get_server_args(cls):
return DEFAULT_SERVER_ARGS + ["--enable-torch-compile"]
extra_args = ["--enable-torch-compile"]
class TestHybridAttnBackendSpeculativeDecodingPrefillBackend(TestHybridAttnBackendBase):
speculative_decode = True
# This eagle test uses a very small model, so the accuracy is low.
accuracy_threshold = 0.2
@classmethod
def get_server_args(cls):
return DEFAULT_SERVER_ARGS + [
"--speculative-algorithm",
"EAGLE",
"--speculative-draft-model-path",
DEFAULT_DRAFT_MODEL_EAGLE,
"--speculative-num-steps",
"3",
"--speculative-eagle-topk",
"2",
"--speculative-num-draft-tokens",
"4",
"--speculative-attention-mode",
"prefill",
]
extra_args = [
"--speculative-algorithm",
"EAGLE",
"--speculative-draft-model-path",
DEFAULT_DRAFT_MODEL_EAGLE,
"--speculative-num-steps",
"3",
"--speculative-eagle-topk",
"2",
"--speculative-num-draft-tokens",
"4",
"--speculative-attention-mode",
"prefill",
]
class TestHybridAttnBackendSpeculativeDecodingDecodeBackend(TestHybridAttnBackendBase):
speculative_decode = True
# This eagle test uses a very small model, so the accuracy is low.
accuracy_threshold = 0.2
@classmethod
def get_server_args(cls):
return DEFAULT_SERVER_ARGS + [
"--speculative-algorithm",
"EAGLE",
"--speculative-draft-model-path",
DEFAULT_DRAFT_MODEL_EAGLE,
"--speculative-num-steps",
"3",
"--speculative-eagle-topk",
"2",
"--speculative-num-draft-tokens",
"4",
"--speculative-attention-mode",
"decode",
]
extra_args = [
"--speculative-algorithm",
"EAGLE",
"--speculative-draft-model-path",
DEFAULT_DRAFT_MODEL_EAGLE,
"--speculative-num-steps",
"3",
"--speculative-eagle-topk",
"2",
"--speculative-num-draft-tokens",
"4",
"--speculative-attention-mode",
"decode",
]
if __name__ == "__main__":
test/registered/attention/test_torch_native_attention_backend.py
+1 -1
View File
@@ -18,7 +18,7 @@ from sglang.test.test_utils import (
)
# Torch native attention backend integration test with MMLU eval
register_cuda_ci(est_time=140, stage="stage-b", runner_config="1-gpu-small")
register_cuda_ci(est_time=140, stage="extra-a", runner_config="1-gpu-small")
register_amd_ci(est_time=150, suite="stage-b-test-1-gpu-small-amd")
test/registered/attention/test_triton_sliding_window.py
+1 -1
View File
@@ -16,7 +16,7 @@ from sglang.test.test_utils import (
)
# Sliding window attention with Triton backend (Gemma-3 model)
register_cuda_ci(est_time=93, stage="stage-b", runner_config="1-gpu-large")
register_cuda_ci(est_time=93, stage="extra-a", runner_config="1-gpu-large")
register_amd_ci(est_time=200, suite="stage-b-test-1-gpu-small-amd")
test/registered/backends/test_torch_compile.py
+1 -1
View File
@@ -15,7 +15,7 @@ from sglang.test.test_utils import (
popen_launch_server,
)
register_cuda_ci(est_time=126, stage="stage-b", runner_config="1-gpu-large")
register_cuda_ci(est_time=126, stage="extra-a", runner_config="1-gpu-large")
register_amd_ci(est_time=1100, suite="stage-b-test-1-gpu-small-amd")
test/registered/core/test_gpt_oss_sm120.py
+1 -1
View File
@@ -5,7 +5,7 @@ import torch
from sglang.test.ci.ci_register import register_cuda_ci
from sglang.test.gpt_oss_common import BaseTestGptOss
register_cuda_ci(est_time=345, stage="stage-b", runner_config="1-gpu-small")
register_cuda_ci(est_time=345, stage="extra-a", runner_config="1-gpu-small")
@unittest.skipIf(not torch.cuda.is_available(), "CUDA is not available")
test/registered/cp/test_deepseek_v32_cp_single_node.py
+1 -5
View File
@@ -13,11 +13,7 @@ from sglang.test.test_utils import (
write_github_step_summary,
)
register_cuda_ci(
est_time=616,
stage="stage-c",
runner_config="deepep-8-gpu-h200",
)
register_cuda_ci(est_time=616, stage="extra-b", runner_config="deepep-8-gpu-h200")
DEEPSEEK_V32_MODEL_PATH = "deepseek-ai/DeepSeek-V3.2"
test/registered/distributed/test_disaggregation_hybrid_attention.py
+1 -1
View File
@@ -12,7 +12,7 @@ from sglang.test.test_utils import (
popen_launch_pd_server,
)
register_cuda_ci(est_time=310, suite="nightly-8-gpu-h200", nightly=True)
register_cuda_ci(est_time=310, stage="extra-b", runner_config="8-gpu-h200")
@unittest.skipIf(is_in_ci(), "Temporarily disable the flaky test.")
test/registered/distributed/test_load_weights_from_remote_instance.py
+1 -1
View File
@@ -38,7 +38,7 @@ from sglang.utils import terminate_process
mp.set_start_method("spawn", force=True)
register_cuda_ci(est_time=145, stage="stage-b", runner_config="2-gpu-large")
register_cuda_ci(est_time=145, stage="extra-a", runner_config="2-gpu-large")
register_amd_ci(est_time=72, suite="stage-b-test-2-gpu-large-amd")
test/registered/ep/test_deepep_large.py
+1 -1
View File
@@ -16,7 +16,7 @@ from sglang.test.test_utils import (
popen_launch_server,
)
register_cuda_ci(est_time=528, stage="stage-c", runner_config="deepep-8-gpu-h200")
register_cuda_ci(est_time=528, stage="extra-b", runner_config="deepep-8-gpu-h200")
DEEPSEEK_V32_MODEL_PATH = "deepseek-ai/DeepSeek-V3.2"
test/registered/hicache/test_hicache_spec_file_storage.py
+1 -1
View File
@@ -30,7 +30,7 @@ from sglang.test.test_utils import (
)
from sglang.utils import wait_for_http_ready
register_cuda_ci(est_time=200, stage="stage-b", runner_config="1-gpu-large")
register_cuda_ci(est_time=200, stage="extra-a", runner_config="1-gpu-large")
@unittest.skipIf(is_hip(), "HiCache + EAGLE3 file-storage loadback e2e is CUDA-only.")
test/registered/lora/test_lora_drainer.py
+1 -1
View File
@@ -12,7 +12,7 @@ from sglang.test.lora_utils import (
)
from sglang.test.test_utils import is_in_ci
register_cuda_ci(est_time=100, stage="stage-b", runner_config="1-gpu-small")
register_cuda_ci(est_time=100, stage="extra-a", runner_config="1-gpu-small")
register_amd_ci(est_time=100, suite="stage-b-test-1-gpu-small-amd")
MOCK_START_TIME = 1000.0
test/registered/lora/test_lora_eviction.py
+1 -1
View File
@@ -23,7 +23,7 @@ from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci
from sglang.test.runners import SRTRunner
from sglang.test.test_utils import CustomTestCase
register_cuda_ci(est_time=263, stage="stage-b", runner_config="1-gpu-small")
register_cuda_ci(est_time=263, stage="extra-a", runner_config="1-gpu-small")
register_amd_ci(est_time=224, suite="stage-b-test-1-gpu-small-amd")
PROMPTS = [
test/registered/lora/test_lora_gpt_oss_20b_logprob_diff.py
+1 -5
View File
@@ -34,11 +34,7 @@ import sglang as sgl
from sglang.test.ci.ci_register import register_cuda_ci
from sglang.test.test_utils import CustomTestCase
register_cuda_ci(
est_time=90,
suite="nightly-4-gpu-b200",
nightly=True,
)
register_cuda_ci(est_time=90, stage="extra-b", runner_config="4-gpu-b200")
BASE_MODEL = "lmsys/gpt-oss-20b-bf16"
LORA_HF_REPO = "yushengsu/lora-diff-gpt-oss-20b"
test/registered/lora/test_lora_moe_tp_logprob_diff.py
+1 -5
View File
@@ -32,11 +32,7 @@ from sglang.test.test_utils import (
is_in_ci,
)
register_cuda_ci(
est_time=200,
stage="stage-b",
runner_config="2-gpu-large",
)
register_cuda_ci(est_time=200, stage="extra-a", runner_config="2-gpu-large")
LOGPROB_THRESHOLD = 5e-04
MAX_NEW_TOKENS = 10
test/registered/lora/test_lora_nemotron_3_super_120b_a12b_logprob_diff.py
+1 -5
View File
@@ -34,11 +34,7 @@ import sglang as sgl
from sglang.test.ci.ci_register import register_cuda_ci
from sglang.test.test_utils import CustomTestCase
register_cuda_ci(
est_time=100,
suite="nightly-4-gpu-b200",
nightly=True,
)
register_cuda_ci(est_time=100, stage="extra-b", runner_config="4-gpu-b200")
BASE_MODEL = "nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-BF16"
LORA_HF_REPO = "opherlie/lora-test-case-NVIDIA-Nemotron-3-Super-120B-A12B-BF16"
test/registered/lora/test_lora_qwen3_30b_a3b_instruct_2507_logprob_diff.py
+1 -5
View File
@@ -34,11 +34,7 @@ import sglang as sgl
from sglang.test.ci.ci_register import register_cuda_ci
from sglang.test.test_utils import CustomTestCase
register_cuda_ci(
est_time=100,
suite="nightly-4-gpu-b200",
nightly=True,
)
register_cuda_ci(est_time=100, stage="extra-b", runner_config="4-gpu-b200")
BASE_MODEL = "Qwen/Qwen3-30B-A3B-Instruct-2507"
LORA_HF_REPO = "yushengsu/lora-diff-Qwen3-30B-A3B-Instruct-2507"
test/registered/lora/test_lora_qwen3_5_4b_logprob_diff.py
+1 -5
View File
@@ -34,11 +34,7 @@ import sglang as sgl
from sglang.test.ci.ci_register import register_cuda_ci
from sglang.test.test_utils import CustomTestCase
register_cuda_ci(
est_time=90,
stage="stage-b",
runner_config="1-gpu-large",
)
register_cuda_ci(est_time=90, stage="extra-a", runner_config="1-gpu-large")
BASE_MODEL = "Qwen/Qwen3.5-4B"
LORA_HF_REPO = "opherlie/lora-test-case-Qwen3.5-4B"
test/registered/lora/test_lora_qwen3_8b_logprob_diff.py
+1 -5
View File
@@ -37,11 +37,7 @@ from sglang.srt.lora.utils import auto_detect_lora_target_modules
from sglang.test.ci.ci_register import register_cuda_ci
from sglang.test.test_utils import CustomTestCase
register_cuda_ci(
est_time=40,
stage="stage-b",
runner_config="1-gpu-large",
)
register_cuda_ci(est_time=40, stage="extra-a", runner_config="1-gpu-large")
BASE_MODEL = "Qwen/Qwen3-8B"
LORA_HF_REPO = "yushengsu/lora-diff-Qwen3-8B"
test/registered/models/test_compressed_tensors_models.py
+1 -1
View File
@@ -13,7 +13,7 @@ from sglang.test.test_utils import (
popen_launch_server,
)
register_cuda_ci(est_time=65, stage="stage-b", runner_config="1-gpu-large")
register_cuda_ci(est_time=65, stage="extra-a", runner_config="1-gpu-large")
register_amd_ci(est_time=42, suite="stage-b-test-1-gpu-small-amd")
test/registered/models/test_generation_models.py
+1 -1
View File
@@ -1,7 +1,7 @@
from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci
# Generation model tests (CUDA only)
register_cuda_ci(est_time=150, stage="stage-b", runner_config="1-gpu-large")
register_cuda_ci(est_time=150, stage="extra-a", runner_config="1-gpu-large")
register_amd_ci(est_time=106, suite="stage-b-test-1-gpu-small-amd")
# Copyright 2023-2024 SGLang Team
test/registered/models/test_ministral4_models.py
+1 -5
View File
@@ -6,11 +6,7 @@ from sglang.test.kits.mmmu_vlm_kit import MMMUMixin
from sglang.test.server_fixtures.default_fixture import DefaultServerBase
from sglang.test.server_fixtures.mmmu_fixture import MMMUServerBase
register_cuda_ci(
est_time=200,
stage="stage-b",
runner_config="2-gpu-large",
)
register_cuda_ci(est_time=200, stage="extra-a", runner_config="2-gpu-large")
MODEL = "mistralai/Mistral-Small-4-119B-2603"
test/registered/models/test_vlm_models.py
+1 -1
View File
@@ -13,7 +13,7 @@ from sglang.test.test_utils import is_in_ci
# VLM (Vision Language Model) tests
register_cuda_ci(est_time=317, stage="stage-b", runner_config="1-gpu-large")
register_cuda_ci(est_time=317, stage="extra-a", runner_config="1-gpu-large")
register_amd_ci(est_time=850, suite="stage-b-test-1-gpu-small-amd-nondeterministic")
_is_hip = is_hip()
test/registered/moe/test_cutedsl_moe.py
+1 -1
View File
@@ -16,7 +16,7 @@ except ImportError:
CuteDslMoEWrapper = None
convert_sf_to_mma_layout = None
register_cuda_ci(est_time=24, suite="nightly-4-gpu-b200", nightly=True)
register_cuda_ci(est_time=24, stage="extra-b", runner_config="4-gpu-b200")
SKIP_TEST = torch.cuda.get_device_capability() < (10, 0)
SKIP_REASON = "Nvfp4 Requires compute capability of 10 or above."
test/registered/moe/test_moe_ep.py
+2 -41
View File
@@ -12,6 +12,8 @@ from sglang.test.test_utils import (
popen_launch_server,
)
# Per-commit: TP=2 EP=2 baseline.
# DeepGEMM/FP8 variant moved to test_moe_ep_nightly.py.
register_cuda_ci(est_time=279, stage="stage-b", runner_config="2-gpu-large")
@@ -52,46 +54,5 @@ class TestEp(CustomTestCase):
self.assertGreater(metrics["score"], 0.60)
class TestEpDeepGEMM(CustomTestCase):
@classmethod
def setUpClass(cls):
cls.model = DEFAULT_MODEL_NAME_FOR_TEST_MLA
cls.base_url = DEFAULT_URL_FOR_TEST
cls.process = popen_launch_server(
cls.model,
cls.base_url,
timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
other_args=[
"--trust-remote-code",
"--tp",
"2",
"--ep-size",
"2",
"--quantization",
"fp8",
"--moe-runner-backend",
"deep_gemm",
],
)
@classmethod
def tearDownClass(cls):
kill_process_tree(cls.process.pid)
def test_gsm8k(self):
args = SimpleNamespace(
base_url=self.base_url,
eval_name="gsm8k",
api="completion",
max_tokens=512,
num_examples=200,
num_threads=128,
)
metrics = run_eval(args)
print(metrics)
self.assertGreater(metrics["score"], 0.60)
if __name__ == "__main__":
unittest.main()
test/registered/moe/test_moe_ep_extra.py
+66
View File
@@ -0,0 +1,66 @@
"""Extra: TP=2 EP=2 with FP8 + DeepGEMM MoE backend.
Sibling per-commit file (test_moe_ep.py) keeps the baseline TP=2 EP=2
variant.
"""
import unittest
from types import SimpleNamespace
from sglang.srt.utils import kill_process_tree
from sglang.test.ci.ci_register import register_cuda_ci
from sglang.test.run_eval import run_eval
from sglang.test.test_utils import (
DEFAULT_MODEL_NAME_FOR_TEST_MLA,
DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
DEFAULT_URL_FOR_TEST,
CustomTestCase,
popen_launch_server,
)
register_cuda_ci(est_time=279, stage="extra-a", runner_config="2-gpu-large")
class TestEpDeepGEMM(CustomTestCase):
@classmethod
def setUpClass(cls):
cls.model = DEFAULT_MODEL_NAME_FOR_TEST_MLA
cls.base_url = DEFAULT_URL_FOR_TEST
cls.process = popen_launch_server(
cls.model,
cls.base_url,
timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
other_args=[
"--trust-remote-code",
"--tp",
"2",
"--ep-size",
"2",
"--quantization",
"fp8",
"--moe-runner-backend",
"deep_gemm",
],
)
@classmethod
def tearDownClass(cls):
kill_process_tree(cls.process.pid)
def test_gsm8k(self):
args = SimpleNamespace(
base_url=self.base_url,
eval_name="gsm8k",
api="completion",
max_tokens=512,
num_examples=200,
num_threads=128,
)
metrics = run_eval(args)
print(metrics)
self.assertGreater(metrics["score"], 0.60)
if __name__ == "__main__":
unittest.main()
test/registered/observability/test_tracing.py
+1 -1
View File
@@ -46,7 +46,7 @@ from sglang.test.test_utils import (
logger = logging.getLogger(__name__)
# CI registration
register_cuda_ci(est_time=113, stage="stage-b", runner_config="1-gpu-small")
register_cuda_ci(est_time=113, stage="extra-a", runner_config="1-gpu-small")
# ============================================================================
test/registered/perf/test_bench_one_batch_2gpu.py
+1 -1
View File
@@ -11,7 +11,7 @@ from sglang.test.test_utils import (
write_github_step_summary,
)
register_cuda_ci(est_time=209, stage="stage-b", runner_config="2-gpu-large")
register_cuda_ci(est_time=209, stage="extra-a", runner_config="2-gpu-large")
register_amd_ci(est_time=630, suite="stage-b-test-2-gpu-large-amd")
test/registered/perf/test_bench_serving_1gpu_large.py
+1 -1
View File
@@ -17,7 +17,7 @@ from sglang.test.test_utils import (
write_github_step_summary,
)
register_cuda_ci(est_time=286, stage="stage-b", runner_config="1-gpu-large")
register_cuda_ci(est_time=286, stage="extra-a", runner_config="1-gpu-large")
register_amd_ci(est_time=300, suite="stage-b-test-1-gpu-large-amd")
test/registered/perf/test_bench_serving_1gpu_part1.py
+1 -1
View File
@@ -19,7 +19,7 @@ from sglang.test.test_utils import (
write_github_step_summary,
)
register_cuda_ci(est_time=1210, stage="stage-b", runner_config="1-gpu-large")
register_cuda_ci(est_time=1210, stage="extra-a", runner_config="1-gpu-large")
register_amd_ci(est_time=1100, suite="stage-b-test-1-gpu-large-amd")
test/registered/perf/test_bench_serving_1gpu_part2.py
+1 -1
View File
@@ -19,7 +19,7 @@ from sglang.test.test_utils import (
write_github_step_summary,
)
register_cuda_ci(est_time=968, stage="stage-b", runner_config="1-gpu-large")
register_cuda_ci(est_time=968, stage="extra-a", runner_config="1-gpu-large")
register_amd_ci(est_time=900, suite="stage-b-test-1-gpu-large-amd")
test/registered/perf/test_bench_serving_2gpu.py
+1 -1
View File
@@ -14,7 +14,7 @@ from sglang.test.test_utils import (
write_github_step_summary,
)
register_cuda_ci(est_time=721, stage="stage-b", runner_config="2-gpu-large")
register_cuda_ci(est_time=721, stage="extra-a", runner_config="2-gpu-large")
register_amd_ci(est_time=1450, suite="stage-b-test-2-gpu-large-amd")
test/registered/piecewise_cuda_graph/test_pcg_with_speculative_decoding.py
+30 -228
View File
@@ -1,242 +1,44 @@
"""Test piecewise CUDA graph coexisting with speculative decoding.
"""Test piecewise CUDA graph coexisting with speculative decoding (EAGLE3).
PCG handles prefill/extend path while speculative decoding (MTP/EAGLE3/STANDALONE/NGRAM)
uses decode CUDA graphs. This test verifies they don't interfere with each other.
PCG handles prefill/extend path while speculative decoding (EAGLE3) uses
decode CUDA graphs. This test verifies they don't interfere with each
other. MTP / STANDALONE / NGRAM variants moved to the sibling file
test_pcg_with_speculative_decoding_extra.py.
"""
import unittest
from types import SimpleNamespace
import requests
from sglang.srt.utils import kill_process_tree
from sglang.test.ci.ci_register import register_cuda_ci
from sglang.test.run_eval import run_eval
from sglang.test.test_utils import (
DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
DEFAULT_URL_FOR_TEST,
popen_launch_server,
)
from sglang.test.server_fixtures.pcg_spec_fixture import PCGSpecBase
register_cuda_ci(est_time=531, stage="stage-b", runner_config="2-gpu-large")
class TestPCGWithMTP(unittest.TestCase):
"""Test PCG + MTP (NEXTN) on Qwen3.5-35B-A3B with FP8."""
class TestPCGWithEAGLE3(PCGSpecBase, unittest.TestCase):
"""PCG + EAGLE3 on Qwen3-30B-A3B-Instruct-2507."""
@classmethod
def setUpClass(cls):
cls.model = "Qwen/Qwen3.5-35B-A3B"
cls.base_url = DEFAULT_URL_FOR_TEST
other_args = [
"--tp",
"2",
"--trust-remote-code",
"--quantization",
"fp8",
"--mamba-scheduler-strategy",
"extra_buffer",
"--enable-piecewise-cuda-graph",
"--speculative-algorithm",
"NEXTN",
"--reasoning-parser",
"qwen3",
]
cls.process = popen_launch_server(
cls.model,
cls.base_url,
timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH * 3,
other_args=other_args,
)
@classmethod
def tearDownClass(cls):
kill_process_tree(cls.process.pid)
def test_gsm8k(self):
args = SimpleNamespace(
base_url=self.base_url,
model=self.model,
eval_name="gsm8k",
max_tokens=8192,
num_examples=200,
num_threads=200,
thinking_mode="qwen3",
)
metrics = run_eval(args)
print(metrics)
self.assertGreater(metrics["score"], 0.75)
server_info = requests.get(self.base_url + "/server_info").json()
avg_spec_accept_length = server_info["internal_states"][0][
"avg_spec_accept_length"
]
print(f"{avg_spec_accept_length=}")
self.assertGreater(avg_spec_accept_length, 1.5)
class TestPCGWithEAGLE3(unittest.TestCase):
"""Test PCG + EAGLE3 on Qwen3-30B-A3B-Instruct-2507."""
@classmethod
def setUpClass(cls):
cls.model = "Qwen/Qwen3-30B-A3B-Instruct-2507"
cls.base_url = DEFAULT_URL_FOR_TEST
other_args = [
"--tp",
"2",
"--trust-remote-code",
"--enforce-piecewise-cuda-graph",
"--mem-fraction-static",
"0.6",
"--speculative-algorithm",
"EAGLE3",
"--speculative-draft-model-path",
"lmsys/SGLang-EAGLE3-Qwen3-30B-A3B-Instruct-2507-SpecForge-Nex",
"--speculative-num-steps",
"5",
"--speculative-eagle-topk",
"4",
"--speculative-num-draft-tokens",
"8",
]
cls.process = popen_launch_server(
cls.model,
cls.base_url,
timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH * 3,
other_args=other_args,
env={"SGLANG_ALLOW_OVERWRITE_LONGER_CONTEXT_LEN": "1"},
)
@classmethod
def tearDownClass(cls):
kill_process_tree(cls.process.pid)
def test_gsm8k(self):
args = SimpleNamespace(
base_url=self.base_url,
model=self.model,
eval_name="gsm8k",
max_tokens=512,
num_examples=200,
num_threads=200,
)
metrics = run_eval(args)
print(metrics)
self.assertGreater(metrics["score"], 0.75)
server_info = requests.get(self.base_url + "/server_info").json()
avg_spec_accept_length = server_info["internal_states"][0][
"avg_spec_accept_length"
]
print(f"{avg_spec_accept_length=}")
self.assertGreater(avg_spec_accept_length, 1.5)
class TestPCGWithSTANDALONE(unittest.TestCase):
"""Test PCG + STANDALONE on Llama-3.1-8B-Instruct + Llama-3.2-1B-Instruct."""
@classmethod
def setUpClass(cls):
cls.model = "meta-llama/Llama-3.1-8B-Instruct"
cls.base_url = DEFAULT_URL_FOR_TEST
other_args = [
"--trust-remote-code",
"--enforce-piecewise-cuda-graph",
"--mem-fraction-static",
"0.5",
"--speculative-algorithm",
"STANDALONE",
"--speculative-draft-model-path",
"meta-llama/Llama-3.2-1B-Instruct",
"--speculative-num-steps",
"3",
"--speculative-eagle-topk",
"1",
"--speculative-num-draft-tokens",
"4",
]
cls.process = popen_launch_server(
cls.model,
cls.base_url,
timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH * 2,
other_args=other_args,
)
@classmethod
def tearDownClass(cls):
kill_process_tree(cls.process.pid)
def test_gsm8k(self):
args = SimpleNamespace(
base_url=self.base_url,
model=self.model,
eval_name="gsm8k",
max_tokens=512,
num_examples=200,
num_threads=200,
)
metrics = run_eval(args)
print(metrics)
self.assertGreater(metrics["score"], 0.50)
server_info = requests.get(self.base_url + "/server_info").json()
avg_spec_accept_length = server_info["internal_states"][0][
"avg_spec_accept_length"
]
print(f"{avg_spec_accept_length=}")
self.assertGreater(avg_spec_accept_length, 1.5)
class TestPCGWithNGRAM(unittest.TestCase):
"""Test PCG + NGRAM on Qwen2.5-Coder-7B-Instruct."""
@classmethod
def setUpClass(cls):
cls.model = "Qwen/Qwen2.5-Coder-7B-Instruct"
cls.base_url = DEFAULT_URL_FOR_TEST
other_args = [
"--trust-remote-code",
"--enforce-piecewise-cuda-graph",
"--speculative-algorithm",
"NGRAM",
"--speculative-num-draft-tokens",
"16",
"--cuda-graph-max-bs",
"8",
"--mem-fraction-static",
"0.8",
]
cls.process = popen_launch_server(
cls.model,
cls.base_url,
timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH * 2,
other_args=other_args,
)
@classmethod
def tearDownClass(cls):
kill_process_tree(cls.process.pid)
def test_gsm8k(self):
args = SimpleNamespace(
base_url=self.base_url,
model=self.model,
eval_name="gsm8k",
max_tokens=512,
num_examples=200,
num_threads=200,
)
metrics = run_eval(args)
print(metrics)
self.assertGreater(metrics["score"], 0.70)
server_info = requests.get(self.base_url + "/server_info").json()
avg_spec_accept_length = server_info["internal_states"][0][
"avg_spec_accept_length"
]
print(f"{avg_spec_accept_length=}")
self.assertGreater(avg_spec_accept_length, 1.5)
model = "Qwen/Qwen3-30B-A3B-Instruct-2507"
server_args = [
"--tp",
"2",
"--trust-remote-code",
"--enforce-piecewise-cuda-graph",
"--mem-fraction-static",
"0.6",
"--speculative-algorithm",
"EAGLE3",
"--speculative-draft-model-path",
"lmsys/SGLang-EAGLE3-Qwen3-30B-A3B-Instruct-2507-SpecForge-Nex",
"--speculative-num-steps",
"5",
"--speculative-eagle-topk",
"4",
"--speculative-num-draft-tokens",
"8",
]
timeout_mult = 3
server_env = {"SGLANG_ALLOW_OVERWRITE_LONGER_CONTEXT_LEN": "1"}
accuracy_threshold = 0.75
if __name__ == "__main__":
test/registered/piecewise_cuda_graph/test_pcg_with_speculative_decoding_extra.py
+81
View File
@@ -0,0 +1,81 @@
"""Extra: PCG coexistence with non-EAGLE3 speculative decoding variants.
EAGLE3 stays per-commit in the sibling file
test_pcg_with_speculative_decoding.py.
"""
import unittest
from sglang.test.ci.ci_register import register_cuda_ci
from sglang.test.server_fixtures.pcg_spec_fixture import PCGSpecBase
register_cuda_ci(est_time=531, stage="extra-a", runner_config="2-gpu-large")
class TestPCGWithMTP(PCGSpecBase, unittest.TestCase):
"""PCG + MTP (NEXTN) on Qwen3.5-35B-A3B with FP8."""
model = "Qwen/Qwen3.5-35B-A3B"
server_args = [
"--tp",
"2",
"--trust-remote-code",
"--quantization",
"fp8",
"--mamba-scheduler-strategy",
"extra_buffer",
"--enable-piecewise-cuda-graph",
"--speculative-algorithm",
"NEXTN",
"--reasoning-parser",
"qwen3",
]
timeout_mult = 3
max_tokens = 8192
thinking_mode = "qwen3"
accuracy_threshold = 0.75
class TestPCGWithSTANDALONE(PCGSpecBase, unittest.TestCase):
"""PCG + STANDALONE on Llama-3.1-8B-Instruct + Llama-3.2-1B-Instruct."""
model = "meta-llama/Llama-3.1-8B-Instruct"
server_args = [
"--trust-remote-code",
"--enforce-piecewise-cuda-graph",
"--mem-fraction-static",
"0.5",
"--speculative-algorithm",
"STANDALONE",
"--speculative-draft-model-path",
"meta-llama/Llama-3.2-1B-Instruct",
"--speculative-num-steps",
"3",
"--speculative-eagle-topk",
"1",
"--speculative-num-draft-tokens",
"4",
]
accuracy_threshold = 0.50
class TestPCGWithNGRAM(PCGSpecBase, unittest.TestCase):
"""PCG + NGRAM on Qwen2.5-Coder-7B-Instruct."""
model = "Qwen/Qwen2.5-Coder-7B-Instruct"
server_args = [
"--trust-remote-code",
"--enforce-piecewise-cuda-graph",
"--speculative-algorithm",
"NGRAM",
"--speculative-num-draft-tokens",
"16",
"--cuda-graph-max-bs",
"8",
"--mem-fraction-static",
"0.8",
]
if __name__ == "__main__":
unittest.main()
test/registered/quant/test_deepseek_v3_fp4_4gpu.py
+3 -127
View File
@@ -1,11 +1,9 @@
import os
import unittest
from types import SimpleNamespace
from sglang.srt.utils import kill_process_tree
from sglang.test.ci.ci_register import register_cuda_ci
from sglang.test.run_eval import run_eval
from sglang.test.send_one import BenchArgs, send_one_prompt
from sglang.test.test_utils import (
DEFAULT_URL_FOR_TEST,
CustomTestCase,
@@ -14,137 +12,15 @@ from sglang.test.test_utils import (
write_github_step_summary,
)
# Per-commit: SymmetricMemory variant only.
# - TestDeepseekV3FP4 (TRTLLM) archived to test/manual/quant/test_deepseek_v3_fp4_4gpu_trtllm.py
# - TestDeepseekV3FP4CutlassMoE moved to test_deepseek_v3_fp4_4gpu_extra.py
register_cuda_ci(est_time=960, stage="stage-c", runner_config="4-gpu-b200")
FULL_DEEPSEEK_V3_FP4_MODEL_PATH = "nvidia/DeepSeek-V3-0324-FP4"
SERVER_LAUNCH_TIMEOUT = 1200
class TestDeepseekV3FP4(CustomTestCase):
@classmethod
def setUpClass(cls):
cls.model = FULL_DEEPSEEK_V3_FP4_MODEL_PATH
cls.base_url = DEFAULT_URL_FOR_TEST
other_args = [
"--tp",
"4",
"--attention-backend",
"trtllm_mla",
"--moe-runner-backend",
"flashinfer_trtllm",
"--quantization",
"modelopt_fp4",
"--kv-cache-dtype",
"fp8_e4m3",
"--model-loader-extra-config",
'{"enable_multithread_load": true,"num_threads": 64}',
]
cls.process = popen_launch_server(
cls.model,
cls.base_url,
timeout=SERVER_LAUNCH_TIMEOUT,
other_args=other_args,
)
@classmethod
def tearDownClass(cls):
kill_process_tree(cls.process.pid)
def test_a_gsm8k(
self,
): # Append an "a" to make this test run first (alphabetically) to warm up the server
args = SimpleNamespace(
base_url=self.base_url,
model=self.model,
eval_name="gsm8k",
api="completion",
max_tokens=512,
num_examples=1319,
num_threads=1319,
num_shots=8,
)
metrics = run_eval(args)
print(f"{metrics=}")
if is_in_ci():
write_github_step_summary(
f"### test_gsm8k (deepseek-v3-fp4)\n" f'{metrics["score"]=:.3f}\n'
)
self.assertGreater(metrics["score"], 0.93)
def test_bs_1_speed(self):
args = BenchArgs(port=int(self.base_url.split(":")[-1]), max_new_tokens=2048)
_, speed = send_one_prompt(args)
print(f"{speed=:.2f}")
if is_in_ci():
write_github_step_summary(
f"### test_bs_1_speed (deepseek-v3-fp4)\n" f"{speed=:.2f} token/s\n"
)
self.assertGreater(speed, 120)
class TestDeepseekV3FP4CutlassMoE(CustomTestCase):
@classmethod
def setUpClass(cls):
cls.model = FULL_DEEPSEEK_V3_FP4_MODEL_PATH
cls.base_url = DEFAULT_URL_FOR_TEST
other_args = [
"--tp",
"4",
"--ep",
"4",
"--attention-backend",
"trtllm_mla",
"--moe-runner-backend",
"flashinfer_cutlass",
"--quantization",
"modelopt_fp4",
"--model-loader-extra-config",
'{"enable_multithread_load": true}',
]
cls.process = popen_launch_server(
cls.model,
cls.base_url,
timeout=SERVER_LAUNCH_TIMEOUT,
other_args=other_args,
env={
**os.environ,
"SGLANG_MOE_NVFP4_DISPATCH": "1", # Enable nvfp4 all gather
},
)
@classmethod
def tearDownClass(cls):
kill_process_tree(cls.process.pid)
def test_a_gsm8k(
self,
): # Append an "a" to make this test run first (alphabetically) to warm up the server
args = SimpleNamespace(
base_url=self.base_url,
model=self.model,
eval_name="gsm8k",
api="completion",
max_tokens=512,
num_examples=1319,
num_threads=1319,
num_shots=8,
)
metrics = run_eval(args)
print(f"{metrics=}")
if is_in_ci():
write_github_step_summary(
f"### test_gsm8k (deepseek-v3-fp4-cutlass-moe)\n"
f'{metrics["score"]=:.3f}\n'
)
self.assertGreater(metrics["score"], 0.93)
class TestDeepseekV3FP4SymmetricMemory(CustomTestCase):
@classmethod
def setUpClass(cls):
test/registered/quant/test_deepseek_v3_fp4_4gpu_extra.py
+87
View File
@@ -0,0 +1,87 @@
"""Extra: DeepSeek-V3 FP4 with FlashInfer Cutlass MoE backend.
Sibling per-commit file (test_deepseek_v3_fp4_4gpu.py) keeps the
SymmetricMemory variant.
"""
import os
import unittest
from types import SimpleNamespace
from sglang.srt.utils import kill_process_tree
from sglang.test.ci.ci_register import register_cuda_ci
from sglang.test.run_eval import run_eval
from sglang.test.test_utils import (
DEFAULT_URL_FOR_TEST,
CustomTestCase,
is_in_ci,
popen_launch_server,
write_github_step_summary,
)
register_cuda_ci(est_time=960, stage="extra-b", runner_config="4-gpu-b200")
FULL_DEEPSEEK_V3_FP4_MODEL_PATH = "nvidia/DeepSeek-V3-0324-FP4"
SERVER_LAUNCH_TIMEOUT = 1200
class TestDeepseekV3FP4CutlassMoE(CustomTestCase):
@classmethod
def setUpClass(cls):
cls.model = FULL_DEEPSEEK_V3_FP4_MODEL_PATH
cls.base_url = DEFAULT_URL_FOR_TEST
other_args = [
"--tp",
"4",
"--ep",
"4",
"--attention-backend",
"trtllm_mla",
"--moe-runner-backend",
"flashinfer_cutlass",
"--quantization",
"modelopt_fp4",
"--model-loader-extra-config",
'{"enable_multithread_load": true}',
]
cls.process = popen_launch_server(
cls.model,
cls.base_url,
timeout=SERVER_LAUNCH_TIMEOUT,
other_args=other_args,
env={
**os.environ,
"SGLANG_MOE_NVFP4_DISPATCH": "1", # Enable nvfp4 all gather
},
)
@classmethod
def tearDownClass(cls):
kill_process_tree(cls.process.pid)
def test_a_gsm8k(
self,
): # Append an "a" to make this test run first (alphabetically) to warm up the server
args = SimpleNamespace(
base_url=self.base_url,
model=self.model,
eval_name="gsm8k",
api="completion",
max_tokens=512,
num_examples=1319,
num_threads=1319,
num_shots=8,
)
metrics = run_eval(args)
print(f"{metrics=}")
if is_in_ci():
write_github_step_summary(
f"### test_gsm8k (deepseek-v3-fp4-cutlass-moe)\n"
f'{metrics["score"]=:.3f}\n'
)
self.assertGreater(metrics["score"], 0.93)
if __name__ == "__main__":
unittest.main()
test/registered/quant/test_fp8_blockwise_gemm.py
+1 -1
View File
@@ -12,7 +12,7 @@ from sglang.test.test_utils import (
try_cached_model,
)
register_cuda_ci(est_time=430, suite="nightly-4-gpu-b200", nightly=True)
register_cuda_ci(est_time=430, stage="extra-b", runner_config="4-gpu-b200")
MODEL_PATH = "Qwen/Qwen3-4B-Instruct-2507-FP8"
MXFP8_MODEL_PATH = "zianglih/Qwen3-4B-Instruct-2507-MXFP8"
test/registered/quant/test_fp8_gemm_sm120.py
+1 -1
View File
@@ -12,7 +12,7 @@ from sglang.test.test_utils import (
try_cached_model,
)
register_cuda_ci(est_time=146, stage="stage-b", runner_config="1-gpu-small")
register_cuda_ci(est_time=146, stage="extra-a", runner_config="1-gpu-small")
PERTENSOR_MODEL_PATH = "nvidia/Llama-3.1-8B-Instruct-FP8"
BLOCKWISE_MODEL_PATH = "Qwen/Qwen3-4B-Instruct-2507-FP8"
test/registered/quant/test_fp8kv_triton.py
+1 -1
View File
@@ -12,7 +12,7 @@ from sglang.test.test_utils import (
popen_launch_server,
)
register_cuda_ci(est_time=73, stage="stage-b", runner_config="1-gpu-large")
register_cuda_ci(est_time=73, stage="extra-a", runner_config="1-gpu-large")
class TestFP8KVCacheTritonBackend(CustomTestCase):
test/registered/quant/test_gptqmodel_dynamic.py
+1 -1
View File
@@ -14,7 +14,7 @@ from sglang.test.test_utils import (
popen_launch_server,
)
register_cuda_ci(est_time=100, stage="stage-b", runner_config="1-gpu-large")
register_cuda_ci(est_time=100, stage="extra-a", runner_config="1-gpu-large")
def check_quant_method(model_path: str, use_marlin_kernel: bool):
test/registered/quant/test_w8a8_quantization.py
+1 -1
View File
@@ -14,7 +14,7 @@ from sglang.test.test_utils import (
popen_launch_server,
)
register_cuda_ci(est_time=232, stage="stage-b", runner_config="1-gpu-large")
register_cuda_ci(est_time=232, stage="extra-a", runner_config="1-gpu-large")
class BaseW8A8Test(CustomTestCase):
test/registered/rl/test_lora_load_from_tensor.py
+1 -1
View File
@@ -10,7 +10,7 @@ import sglang as sgl
from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci
from sglang.test.test_utils import CustomTestCase
register_cuda_ci(est_time=102, stage="stage-b", runner_config="1-gpu-large")
register_cuda_ci(est_time=102, stage="extra-a", runner_config="1-gpu-large")
register_amd_ci(est_time=90, suite="stage-b-test-1-gpu-small-amd")
MODEL_PATH = "Qwen/Qwen3-0.6B"
test/registered/rl/test_multi_instance_release_memory_occupation.py
+1 -1
View File
@@ -20,7 +20,7 @@ from sglang.test.test_utils import (
find_available_port,
)
register_cuda_ci(est_time=57, stage="stage-c", runner_config="4-gpu-h100")
register_cuda_ci(est_time=57, stage="extra-b", runner_config="4-gpu-h100")
register_amd_ci(
est_time=64,
suite="stage-c-test-4-gpu-amd",
test/registered/rl/test_return_routed_experts.py
+1 -1
View File
@@ -24,7 +24,7 @@ from sglang.test.test_utils import (
popen_launch_server,
)
register_cuda_ci(est_time=400, stage="stage-c", runner_config="4-gpu-h100")
register_cuda_ci(est_time=400, stage="extra-b", runner_config="4-gpu-h100")
# FP8 variant of Qwen3-30B-A3B: required because DeepEP normal/LL fast paths in
# ep_moe/layer.py only run for {Fp8Config (via deep_gemm), W4AFp8Config, aiter,
test/registered/rl/test_update_weights_from_disk_blackwell.py
+1 -1
View File
@@ -1,6 +1,6 @@
from sglang.test.ci.ci_register import register_cuda_ci
register_cuda_ci(est_time=320, suite="nightly-4-gpu-b200", nightly=True)
register_cuda_ci(est_time=320, stage="extra-b", runner_config="4-gpu-b200")
import unittest
test/registered/rl/test_update_weights_from_distributed.py
+1 -1
View File
@@ -43,7 +43,7 @@ from sglang.test.test_utils import (
)
from sglang.utils import terminate_process
register_cuda_ci(est_time=137, stage="stage-b", runner_config="2-gpu-large")
register_cuda_ci(est_time=137, stage="extra-a", runner_config="2-gpu-large")
register_amd_ci(est_time=400, suite="stage-b-test-2-gpu-large-amd")
mp.set_start_method("spawn", force=True)
test/registered/rl/test_update_weights_from_tensor.py
+1 -1
View File
@@ -1,6 +1,6 @@
from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci
register_cuda_ci(est_time=147, stage="stage-b", runner_config="1-gpu-small")
register_cuda_ci(est_time=147, stage="extra-a", runner_config="1-gpu-small")
register_amd_ci(est_time=195, suite="stage-b-test-1-gpu-small-amd")
import gc
test/registered/scheduler/test_priority_scheduling.py
+1 -1
View File
@@ -17,7 +17,7 @@ from sglang.test.test_utils import (
send_concurrent_generate_requests_with_custom_params,
)
register_cuda_ci(est_time=149, stage="stage-b", runner_config="1-gpu-small")
register_cuda_ci(est_time=149, stage="extra-a", runner_config="1-gpu-small")
register_amd_ci(est_time=195, suite="stage-b-test-1-gpu-small-amd")
test/registered/sessions/test_session_control.py
+1 -1
View File
@@ -24,7 +24,7 @@ from sglang.test.test_utils import (
popen_launch_server,
)
register_cuda_ci(est_time=87, stage="stage-b", runner_config="1-gpu-large")
register_cuda_ci(est_time=87, stage="extra-a", runner_config="1-gpu-large")
def remove_prefix(text: str, prefix: str) -> str:
test/registered/sessions/test_session_latency.py
+1 -5
View File
@@ -31,11 +31,7 @@ from sglang.test.test_utils import (
popen_launch_server,
)
register_cuda_ci(
est_time=122,
stage="stage-b",
runner_config="1-gpu-large",
)
register_cuda_ci(est_time=122, stage="extra-a", runner_config="1-gpu-large")
NUM_TURNS = 150
INPUT_LEN = 16
test/registered/sessions/test_streaming_session.py
+60 -1070
View File
File diff suppressed because it is too large Load Diff
test/registered/sessions/test_streaming_session_extra.py
+105
View File
@@ -0,0 +1,105 @@
import unittest
from sglang.test.ci.ci_register import register_cuda_ci
from sglang.test.kits.streaming_session_kit import StreamingSessionKitMixin
from sglang.test.server_fixtures.streaming_session_fixture import (
StreamingSessionServerBase,
)
from sglang.test.test_utils import (
DEFAULT_DRAFT_MODEL_EAGLE3,
DEFAULT_TARGET_MODEL_EAGLE3,
)
register_cuda_ci(est_time=691, stage="extra-a", runner_config="1-gpu-large")
class TestStreamingSessionRetractMixedChunk(
StreamingSessionServerBase, StreamingSessionKitMixin
):
"""Retract + --enable-mixed-chunk."""
extra_args = ["--chunked-prefill-size", "128", "--enable-mixed-chunk"]
env_overrides = [("SGLANG_TEST_RETRACT", True)]
class TestStreamingSessionRetractLargePage(
StreamingSessionServerBase, StreamingSessionKitMixin
):
"""Retract + page=256: exercises page-aligned `_free_tail`. Partial-page
free would corrupt pages still holding committed tokens."""
extra_args = ["--chunked-prefill-size", "4096", "--page-size", "256"]
env_overrides = [("SGLANG_TEST_RETRACT", True)]
# Common EAGLE3 spec args; reused by Eagle/EagleV2/EagleRetractLargePage variants.
_EAGLE3_SPEC_ARGS = [
"--dtype=float16",
"--speculative-algorithm",
"EAGLE3",
"--speculative-draft-model",
DEFAULT_DRAFT_MODEL_EAGLE3,
"--speculative-num-steps",
"3",
"--speculative-eagle-topk",
"1",
"--speculative-num-draft-tokens",
"4",
"--mem-fraction-static",
"0.7",
]
class TestStreamingSessionEagle(StreamingSessionServerBase, StreamingSessionKitMixin):
"""EAGLE3 spec v1 (overlap disabled); offset=-1 — see kit's note."""
kv_inherit_offset = -1
model = DEFAULT_TARGET_MODEL_EAGLE3
extra_args = [
"--disable-overlap-schedule",
"--chunked-prefill-size",
"512",
*_EAGLE3_SPEC_ARGS,
]
env_overrides = [("SGLANG_ALLOW_OVERWRITE_LONGER_CONTEXT_LEN", True)]
class TestStreamingSessionEagleV2(StreamingSessionServerBase, StreamingSessionKitMixin):
"""EAGLE3 spec v2 (overlap on)."""
model = DEFAULT_TARGET_MODEL_EAGLE3
extra_args = [
"--chunked-prefill-size",
"512",
*_EAGLE3_SPEC_ARGS,
]
env_overrides = [
("SGLANG_ENABLE_SPEC_V2", True),
("SGLANG_ALLOW_OVERWRITE_LONGER_CONTEXT_LEN", True),
]
class TestStreamingSessionEagleRetractLargePage(
StreamingSessionServerBase, StreamingSessionKitMixin
):
"""EAGLE3 spec v1 + retract + page=256: max-pressure on `_free_tail`
(spec tail + retract alloc-commit gap + page alignment)."""
kv_inherit_offset = -1
model = DEFAULT_TARGET_MODEL_EAGLE3
extra_args = [
"--disable-overlap-schedule",
"--chunked-prefill-size",
"4096",
*_EAGLE3_SPEC_ARGS,
"--page-size",
"256",
]
env_overrides = [
("SGLANG_TEST_RETRACT", True),
("SGLANG_ALLOW_OVERWRITE_LONGER_CONTEXT_LEN", True),
]
if __name__ == "__main__":
unittest.main()
test/registered/sessions/test_streaming_session_swa.py
+48 -122
View File
@@ -1,27 +1,15 @@
import os
import sys
import unittest
from sglang.srt.environ import envs
from sglang.srt.utils import kill_process_tree
from sglang.srt.utils.hf_transformers_utils import get_tokenizer
from sglang.test.ci.ci_register import register_cuda_ci
from sglang.test.test_utils import (
DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
DEFAULT_URL_FOR_TEST,
popen_launch_server,
from sglang.test.kits.streaming_session_kit import (
AbortLeakReproKitMixin,
StreamingSessionKitMixin,
)
# test/ has no __init__.py; add sibling dir so sibling module is importable
# when this file is run as a script via `python3 <path>`.
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
from test_streaming_session import ( # noqa: E402
from sglang.test.server_fixtures.streaming_session_fixture import (
ABORT_REPRO_CHUNKED_PREFILL_SIZE,
ABORT_REPRO_CONTEXT_LEN,
ABORT_REPRO_PAGE_SIZE,
TestStreamingSession,
TestStreamingSessionAbortLeakRepro,
StreamingSessionServerBase,
)
register_cuda_ci(est_time=519, stage="stage-b", runner_config="1-gpu-large")
@@ -37,125 +25,63 @@ SWA_COMMON_ARGS = [
]
class TestStreamingSessionSWA(TestStreamingSession):
class TestStreamingSessionSWA(StreamingSessionServerBase, StreamingSessionKitMixin):
"""Baseline streaming session on a hybrid-SWA model."""
@classmethod
def setUpClass(cls):
cls.model = SWA_MODEL
cls.base_url = DEFAULT_URL_FOR_TEST
with envs.SGLANG_ENABLE_STRICT_MEM_CHECK_DURING_BUSY.override(2):
cls.process = popen_launch_server(
cls.model,
cls.base_url,
timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
other_args=[
"--enable-streaming-session",
"--chunked-prefill-size",
"512",
*SWA_COMMON_ARGS,
],
)
cls.tokenizer = get_tokenizer(cls.model)
@classmethod
def tearDownClass(cls):
kill_process_tree(cls.process.pid)
model = SWA_MODEL
extra_args = ["--chunked-prefill-size", "512", *SWA_COMMON_ARGS]
class TestStreamingSessionSWARetractLargePage(TestStreamingSession):
class TestStreamingSessionSWARetractLargePage(
StreamingSessionServerBase, StreamingSessionKitMixin
):
"""SWA under retract decode with page=256."""
@classmethod
def setUpClass(cls):
cls.model = SWA_MODEL
cls.base_url = DEFAULT_URL_FOR_TEST
with (
envs.SGLANG_TEST_RETRACT.override(True),
envs.SGLANG_ENABLE_STRICT_MEM_CHECK_DURING_BUSY.override(2),
):
cls.process = popen_launch_server(
cls.model,
cls.base_url,
timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
other_args=[
"--enable-streaming-session",
"--chunked-prefill-size",
"4096",
"--page-size",
"256",
*SWA_COMMON_ARGS,
],
)
cls.tokenizer = get_tokenizer(cls.model)
@classmethod
def tearDownClass(cls):
kill_process_tree(cls.process.pid)
model = SWA_MODEL
extra_args = [
"--chunked-prefill-size",
"4096",
"--page-size",
"256",
*SWA_COMMON_ARGS,
]
env_overrides = [("SGLANG_TEST_RETRACT", True)]
class TestStreamingSessionSWARetractMixedChunk(TestStreamingSession):
class TestStreamingSessionSWARetractMixedChunk(
StreamingSessionServerBase, StreamingSessionKitMixin
):
"""SWA under retract decode with --enable-mixed-chunk."""
@classmethod
def setUpClass(cls):
cls.model = SWA_MODEL
cls.base_url = DEFAULT_URL_FOR_TEST
with (
envs.SGLANG_TEST_RETRACT.override(True),
envs.SGLANG_ENABLE_STRICT_MEM_CHECK_DURING_BUSY.override(2),
):
cls.process = popen_launch_server(
cls.model,
cls.base_url,
timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
other_args=[
"--enable-streaming-session",
"--chunked-prefill-size",
"128",
"--enable-mixed-chunk",
*SWA_COMMON_ARGS,
],
)
cls.tokenizer = get_tokenizer(cls.model)
@classmethod
def tearDownClass(cls):
kill_process_tree(cls.process.pid)
model = SWA_MODEL
extra_args = [
"--chunked-prefill-size",
"128",
"--enable-mixed-chunk",
*SWA_COMMON_ARGS,
]
env_overrides = [("SGLANG_TEST_RETRACT", True)]
class TestStreamingSessionSWAAbortLeakRepro(TestStreamingSessionAbortLeakRepro):
class TestStreamingSessionSWAAbortLeakRepro(
StreamingSessionServerBase, AbortLeakReproKitMixin
):
"""SWA abort-heavy chunked prefill leak repro."""
@classmethod
def setUpClass(cls):
cls.model = SWA_MODEL
cls.base_url = DEFAULT_URL_FOR_TEST
with envs.SGLANG_ENABLE_STRICT_MEM_CHECK_DURING_BUSY.override(2):
cls.process = popen_launch_server(
cls.model,
cls.base_url,
timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
other_args=[
"--enable-streaming-session",
"--chunked-prefill-size",
str(ABORT_REPRO_CHUNKED_PREFILL_SIZE),
"--context-length",
str(ABORT_REPRO_CONTEXT_LEN),
"--page-size",
str(ABORT_REPRO_PAGE_SIZE),
"--max-running-requests",
"32",
"--log-level",
"info",
*SWA_COMMON_ARGS,
],
)
cls.tokenizer = get_tokenizer(cls.model)
@classmethod
def tearDownClass(cls):
kill_process_tree(cls.process.pid)
model = SWA_MODEL
extra_args = [
"--chunked-prefill-size",
str(ABORT_REPRO_CHUNKED_PREFILL_SIZE),
"--context-length",
str(ABORT_REPRO_CONTEXT_LEN),
"--page-size",
str(ABORT_REPRO_PAGE_SIZE),
"--max-running-requests",
"32",
"--log-level",
"info",
*SWA_COMMON_ARGS,
]
if __name__ == "__main__":
test/registered/spec/eagle/test_eagle_dp_attention.py
+5 -2
View File
@@ -20,8 +20,11 @@ from sglang.test.test_utils import (
write_github_step_summary,
)
# EAGLE3 with DP attention (tp=2, dp=2, requires 4 GPUs)
register_cuda_ci(est_time=99, stage="stage-c", runner_config="4-gpu-h100")
# EAGLE3 with DP attention (tp=2, dp=2, requires 4 GPUs).
# Per-commit EAGLE + DP-attn coverage on CUDA is provided by
# test_eagle_infer_beta_dp_attention.py (B200 4-gpu), so this H100 variant
# is gated to extra-b only.
register_cuda_ci(est_time=99, stage="extra-b", runner_config="4-gpu-h100")
register_amd_ci(est_time=200, suite="stage-c-test-4-gpu-amd")
test/registered/spec/eagle/test_eagle_infer_a.py
+1 -1
View File
@@ -13,7 +13,7 @@ from sglang.test.test_utils import (
CustomTestCase,
)
register_cuda_ci(est_time=357, stage="stage-b", runner_config="1-gpu-large")
register_cuda_ci(est_time=357, stage="extra-a", runner_config="1-gpu-large")
class TestEAGLEEngine(CustomTestCase):
test/registered/spec/test_spec_ngram.py
+19
View File
@@ -0,0 +1,19 @@
import unittest
from sglang.test.ci.ci_register import register_cuda_ci
from sglang.test.kits.eval_accuracy_kit import GSM8KMixin
from sglang.test.server_fixtures.ngram_fixture import NgramServerBase
# Per-commit: Paged backend only.
# - FA3 base test archived to test/manual/spec/test_spec_ngram_fa3.py
# - Triton + Flashinfer moved to test_spec_ngram_extra.py
register_cuda_ci(est_time=254, stage="stage-b", runner_config="1-gpu-large")
class TestNgramSpeculativeDecodingPaged(NgramServerBase, GSM8KMixin):
attention_backend = "flashinfer"
extra_args = ["--page-size", "64"]
if __name__ == "__main__":
unittest.main()
test/registered/spec/test_ngram_speculative_decoding.py → test/registered/spec/test_spec_ngram_extra.py
+9 -82
View File
@@ -2,83 +2,22 @@ import unittest
import requests
from sglang.srt.environ import envs
from sglang.srt.utils import kill_process_tree
from sglang.test.ci.ci_register import register_cuda_ci
from sglang.test.kits.eval_accuracy_kit import GSM8KMixin
from sglang.test.test_utils import (
DEFAULT_TARGET_MODEL_NGRAM,
DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
DEFAULT_URL_FOR_TEST,
CustomTestCase,
popen_launch_server,
)
from sglang.test.server_fixtures.ngram_fixture import NgramServerBase
register_cuda_ci(est_time=254, stage="stage-b", runner_config="1-gpu-large")
GSM_DATASET_PATH = None
# Extra: Triton + Flashinfer NGRAM backends. Sibling per-commit file
# (test_spec_ngram.py) keeps the Paged variant.
register_cuda_ci(est_time=254, stage="extra-a", runner_config="1-gpu-large")
# Default server arguments shared across all tests
DEFAULT_SERVER_ARGS = [
"--trust-remote-code",
"--cuda-graph-max-bs",
"8",
"--speculative-algorithm",
"NGRAM",
"--speculative-num-draft-tokens",
"16",
"--mem-fraction-static",
0.8,
]
class TestNgramSpeculativeDecodingTriton(NgramServerBase, GSM8KMixin):
attention_backend = "triton"
class TestNgramSpeculativeDecodingBase(GSM8KMixin, CustomTestCase):
model = DEFAULT_TARGET_MODEL_NGRAM
base_url = DEFAULT_URL_FOR_TEST
gsm8k_accuracy_thres = 0.79 # derived tests need to override this
gsm8k_accept_length_thres = 1.8 # derived spec decoding tests need to override this
@classmethod
def get_server_args(cls):
"""Return the arguments for the server launch. Override in subclasses."""
return DEFAULT_SERVER_ARGS + ["--attention-backend", "fa3"]
@classmethod
def setUpClass(cls):
# disable deep gemm precompile to make launch server faster
# please don't do this if you want to make your inference workload faster
envs.SGLANG_JIT_DEEPGEMM_PRECOMPILE.set(False)
envs.SGLANG_ENABLE_JIT_DEEPGEMM.set(False)
model = cls.model
cls.process = popen_launch_server(
model,
cls.base_url,
timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
other_args=cls.get_server_args(),
)
@classmethod
def tearDownClass(cls):
kill_process_tree(cls.process.pid)
class TestNgramSpeculativeDecodingTriton(TestNgramSpeculativeDecodingBase):
@classmethod
def get_server_args(cls):
return DEFAULT_SERVER_ARGS + ["--attention-backend", "triton"]
class TestNgramSpeculativeDecodingFlashinfer(TestNgramSpeculativeDecodingBase):
@classmethod
def get_server_args(cls):
return DEFAULT_SERVER_ARGS + [
"--attention-backend",
"flashinfer",
"--speculative-ngram-external-sam-budget",
"8",
]
class TestNgramSpeculativeDecodingFlashinfer(NgramServerBase, GSM8KMixin):
attention_backend = "flashinfer"
extra_args = ["--speculative-ngram-external-sam-budget", "8"]
def test_output_as_corpus_boosts_accept_length(self):
"""Baseline → HTTP add corpus → verify accept length boost."""
@@ -147,17 +86,5 @@ class TestNgramSpeculativeDecodingFlashinfer(TestNgramSpeculativeDecodingBase):
)
class TestNgramSpeculativeDecodingPaged(TestNgramSpeculativeDecodingBase):
@classmethod
def get_server_args(cls):
return DEFAULT_SERVER_ARGS + [
"--attention-backend",
"flashinfer",
"--page-size",
"64",
]
if __name__ == "__main__":
unittest.main()
test/registered/spec/test_spec_standalone.py
+27
View File
@@ -0,0 +1,27 @@
import unittest
from sglang.test.ci.ci_register import register_cuda_ci
from sglang.test.server_fixtures.standalone_fixture import StandaloneServerBase
from sglang.test.test_utils import CustomTestCase
# V2 standalone speculative decoding tests (FA3, Triton, FlashInfer backends).
# Non-V2 backends moved to test_spec_standalone_extra.py.
register_cuda_ci(est_time=406, stage="stage-b", runner_config="1-gpu-large")
class TestStandaloneV2SpeculativeDecodingBase(StandaloneServerBase, CustomTestCase):
attention_backend = "fa3"
class TestStandaloneV2SpeculativeDecodingTriton(StandaloneServerBase, CustomTestCase):
attention_backend = "triton"
class TestStandaloneV2SpeculativeDecodingFlashinfer(
StandaloneServerBase, CustomTestCase
):
attention_backend = "flashinfer"
if __name__ == "__main__":
unittest.main()
test/registered/spec/test_spec_standalone_extra.py
+34
View File
@@ -0,0 +1,34 @@
import unittest
from sglang.test.ci.ci_register import register_cuda_ci
from sglang.test.server_fixtures.standalone_fixture import StandaloneServerBase
from sglang.test.test_utils import CustomTestCase
# Non-V2 standalone speculative decoding tests (FA3, Triton, FlashInfer
# backends). Sibling V2 classes stay per-commit in test_spec_standalone.py.
register_cuda_ci(est_time=406, stage="extra-a", runner_config="1-gpu-large")
class TestStandaloneSpeculativeDecodingBase(StandaloneServerBase, CustomTestCase):
attention_backend = "fa3"
speculative_eagle_topk = 2
speculative_num_draft_tokens = 7
enable_spec_v2 = False
class TestStandaloneSpeculativeDecodingTriton(StandaloneServerBase, CustomTestCase):
attention_backend = "triton"
speculative_eagle_topk = 2
speculative_num_draft_tokens = 7
enable_spec_v2 = False
class TestStandaloneSpeculativeDecodingFlashinfer(StandaloneServerBase, CustomTestCase):
attention_backend = "flashinfer"
speculative_eagle_topk = 2
speculative_num_draft_tokens = 7
enable_spec_v2 = False
if __name__ == "__main__":
unittest.main()
test/registered/spec/test_standalone_speculative_decoding.py
-223
View File
@@ -1,223 +0,0 @@
import unittest
from types import SimpleNamespace
import requests
from sglang.srt.environ import envs
from sglang.srt.utils import kill_process_tree
from sglang.test.ci.ci_register import register_cuda_ci
from sglang.test.kits.radix_cache_server_kit import run_radix_attention_test
from sglang.test.run_eval import run_eval
from sglang.test.test_utils import (
DEFAULT_DRAFT_MODEL_STANDALONE,
DEFAULT_TARGET_MODEL_STANDALONE,
DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
DEFAULT_URL_FOR_TEST,
CustomTestCase,
popen_launch_server,
)
# Standalone speculative decoding tests (FA3, Triton, FlashInfer backends)
register_cuda_ci(est_time=406, stage="stage-b", runner_config="1-gpu-large")
GSM_DATASET_PATH = None
# Default server arguments shared across all tests
DEFAULT_SERVER_ARGS = [
"--trust-remote-code",
"--cuda-graph-max-bs",
"8",
"--speculative-algorithm",
"STANDALONE",
"--speculative-draft-model-path",
DEFAULT_DRAFT_MODEL_STANDALONE,
"--speculative-num-steps",
"4",
"--speculative-eagle-topk",
"2",
"--speculative-num-draft-tokens",
"7",
"--mem-fraction-static",
0.7,
]
# Default server arguments for V2 tests
DEFAULT_SERVER_ARGS_V2 = [
"--trust-remote-code",
"--cuda-graph-max-bs",
"8",
"--speculative-algorithm",
"STANDALONE",
"--speculative-draft-model-path",
DEFAULT_DRAFT_MODEL_STANDALONE,
"--speculative-num-steps",
"4",
"--speculative-eagle-topk",
"1",
"--speculative-num-draft-tokens",
"5",
"--mem-fraction-static",
0.7,
]
class TestStandaloneSpeculativeDecodingBase(CustomTestCase):
model = DEFAULT_TARGET_MODEL_STANDALONE
draft_model = DEFAULT_DRAFT_MODEL_STANDALONE
base_url = DEFAULT_URL_FOR_TEST
accuracy_threshold = 0.69 # derived tests need to override this
spec_decode_threshold = 3.6 # derived spec decoding tests need to override this
@classmethod
def get_server_args(cls):
"""Return the arguments for the server launch. Override in subclasses."""
return DEFAULT_SERVER_ARGS + ["--attention-backend", "fa3"]
@classmethod
def setUpClass(cls):
# disable deep gemm precompile to make launch server faster
# please don't do this if you want to make your inference workload faster
envs.SGLANG_JIT_DEEPGEMM_PRECOMPILE.set(False)
envs.SGLANG_ENABLE_JIT_DEEPGEMM.set(False)
envs.SGLANG_ENABLE_SPEC_V2.set(False)
model = cls.model
cls.process = popen_launch_server(
model,
cls.base_url,
timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
other_args=cls.get_server_args(),
)
@classmethod
def tearDownClass(cls):
kill_process_tree(cls.process.pid)
envs.SGLANG_ENABLE_SPEC_V2.clear()
def test_gsm8k(self):
requests.get(self.base_url + "/flush_cache")
args = SimpleNamespace(
base_url=self.base_url,
model=self.model,
eval_name="gsm8k",
api="completion",
max_tokens=512,
num_examples=100,
num_threads=128,
num_shots=4,
gsm8k_data_path=GSM_DATASET_PATH,
)
metrics = run_eval(args)
print(f"{metrics=}")
# Use the appropriate metric key based on the test class
metric_key = "score"
self.assertGreaterEqual(metrics[metric_key], self.accuracy_threshold)
server_info = requests.get(self.base_url + "/server_info")
avg_spec_accept_length = server_info.json()["internal_states"][0][
"avg_spec_accept_length"
]
print(f"{avg_spec_accept_length=}")
self.assertGreater(avg_spec_accept_length, self.spec_decode_threshold)
class TestStandaloneV2SpeculativeDecodingBase(CustomTestCase):
model = DEFAULT_TARGET_MODEL_STANDALONE
draft_model = DEFAULT_DRAFT_MODEL_STANDALONE
base_url = DEFAULT_URL_FOR_TEST
accuracy_threshold = 0.69 # derived tests need to override this
spec_decode_threshold = 3.6 # derived spec decoding tests need to override this
@classmethod
def get_server_args(cls):
"""Return the arguments for the server launch. Override in subclasses."""
return DEFAULT_SERVER_ARGS_V2 + ["--attention-backend", "fa3"]
@classmethod
def setUpClass(cls):
# disable deep gemm precompile to make launch server faster
# please don't do this if you want to make your inference workload faster
envs.SGLANG_JIT_DEEPGEMM_PRECOMPILE.set(False)
envs.SGLANG_ENABLE_JIT_DEEPGEMM.set(False)
model = cls.model
cls.process = popen_launch_server(
model,
cls.base_url,
timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
other_args=cls.get_server_args(),
)
@classmethod
def tearDownClass(cls):
kill_process_tree(cls.process.pid)
def test_gsm8k(self):
requests.get(self.base_url + "/flush_cache")
args = SimpleNamespace(
base_url=self.base_url,
model=self.model,
eval_name="gsm8k",
api="completion",
max_tokens=512,
num_examples=100,
num_threads=128,
num_shots=4,
gsm8k_data_path=GSM_DATASET_PATH,
)
metrics = run_eval(args)
print(f"{metrics=}")
# Use the appropriate metric key based on the test class
metric_key = "score"
self.assertGreaterEqual(metrics[metric_key], self.accuracy_threshold)
server_info = requests.get(self.base_url + "/server_info")
avg_spec_accept_length = server_info.json()["internal_states"][0][
"avg_spec_accept_length"
]
print(f"{avg_spec_accept_length=}")
self.assertGreater(avg_spec_accept_length, self.spec_decode_threshold)
class TestStandaloneSpeculativeDecodingTriton(TestStandaloneSpeculativeDecodingBase):
@classmethod
def get_server_args(cls):
return DEFAULT_SERVER_ARGS + ["--attention-backend", "triton"]
class TestStandaloneSpeculativeDecodingFlashinfer(
TestStandaloneSpeculativeDecodingBase
):
@classmethod
def get_server_args(cls):
return DEFAULT_SERVER_ARGS + ["--attention-backend", "flashinfer"]
class TestStandaloneV2SpeculativeDecodingTriton(
TestStandaloneV2SpeculativeDecodingBase
):
@classmethod
def get_server_args(cls):
return DEFAULT_SERVER_ARGS_V2 + ["--attention-backend", "triton"]
def test_radix_attention(self):
run_radix_attention_test(self.base_url)
assert self.process.poll() is None
class TestStandaloneV2SpeculativeDecodingFlashinfer(
TestStandaloneV2SpeculativeDecodingBase
):
@classmethod
def get_server_args(cls):
return DEFAULT_SERVER_ARGS_V2 + ["--attention-backend", "flashinfer"]
if __name__ == "__main__":
unittest.main()
test/run_suite.py
+9 -1
View File
@@ -55,9 +55,17 @@ PER_COMMIT_SUITES = {
"stage-c-test-8-gpu-h200",
"stage-c-test-8-gpu-b200",
"stage-c-test-deepep-4-gpu-h100",
"stage-c-test-deepep-8-gpu-h200",
"stage-c-test-dsv4-4-gpu-b200",
"stage-c-test-dsv4-8-gpu-h200",
# extra-a / extra-b: label-gated PR opt-in suites in pr-test-extra.yml
# (tests still tagged per-commit but skipped on default PR runs).
"extra-a-test-1-gpu-small",
"extra-a-test-1-gpu-large",
"extra-a-test-2-gpu-large",
"extra-b-test-4-gpu-h100",
"extra-b-test-4-gpu-b200",
"extra-b-test-8-gpu-h200",
"extra-b-test-deepep-8-gpu-h200",
],
HWBackend.NPU: [
"stage-a-test-1-gpu-small",