test/registered: cleanup pure model e2e tests (moves, splits, dedup, kit) (#26610)

This commit is contained in:
Liangsheng Yin
2026-05-28 15:41:46 -07:00
committed by GitHub
parent 435c4ffb30
commit ec78fa6518
26 changed files with 611 additions and 816 deletions
@@ -1,25 +1,12 @@
import random import random
import unittest
from types import SimpleNamespace from types import SimpleNamespace
from urllib.parse import urlparse from urllib.parse import urlparse
from sglang.srt.utils import kill_process_tree
from sglang.test.ci.ci_register import register_cuda_ci
from sglang.test.kl_multiturn_utils import ( from sglang.test.kl_multiturn_utils import (
get_input_ids,
make_mamba_decode_assert,
make_mamba_prefill_assert,
test_input_output_logprobs_match_decode_cache_hit_helper, test_input_output_logprobs_match_decode_cache_hit_helper,
test_input_output_logprobs_match_helper, test_input_output_logprobs_match_helper,
test_input_output_logprobs_match_prefill_cache_hit_helper, test_input_output_logprobs_match_prefill_cache_hit_helper,
) )
from sglang.test.test_utils import (
DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
DEFAULT_URL_FOR_TEST,
CustomTestCase,
is_in_ci,
popen_launch_server,
)
def _random_suffixes(n, length, seed): def _random_suffixes(n, length, seed):
@@ -28,18 +15,8 @@ def _random_suffixes(n, length, seed):
return [[rng.randint(1, 30000) for _ in range(length)] for _ in range(n)] return [[rng.randint(1, 30000) for _ in range(length)] for _ in range(n)]
MAMBA_MODEL = "Qwen/Qwen3-Next-80B-A3B-Instruct"
MAMBA_CHUNK_SIZE = 64
MAMBA_TRACK_INTERVAL = 128
SWA_MODEL = "openai/gpt-oss-20b"
FULL_MODEL = "Qwen/Qwen3-32B"
register_cuda_ci(est_time=760, stage="base-c", runner_config="4-gpu-h100")
class UnifiedRadixTreeTestMixin: class UnifiedRadixTreeTestMixin:
"""Mixin: gsm8k、mmlu and multi-turn KL tests with multi-branch interleaving.""" """Mixin: gsm8k, mmlu and multi-turn KL tests with multi-branch interleaving."""
kl_threshold: float = 0.003 kl_threshold: float = 0.003
max_new_tokens: int = 512 max_new_tokens: int = 512
@@ -168,112 +145,3 @@ class UnifiedRadixTreeTestMixin:
request_batch_size=self.decode_hit_request_batch_size, request_batch_size=self.decode_hit_request_batch_size,
inter_batch_delay_s=self.decode_hit_inter_batch_delay_s, inter_batch_delay_s=self.decode_hit_inter_batch_delay_s,
) )
class TestUnifiedFullRadixCache(UnifiedRadixTreeTestMixin, CustomTestCase):
"""Full attention."""
kl_threshold = 0.0025
@classmethod
def setUpClass(cls):
cls.model = FULL_MODEL
cls.base_url = DEFAULT_URL_FOR_TEST
cls.process = popen_launch_server(
cls.model,
cls.base_url,
timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
other_args=[
"--tp-size",
"4",
"--mem-fraction-static",
"0.80",
"--page-size",
"64",
],
env={"SGLANG_ENABLE_UNIFIED_RADIX_TREE": "1"},
)
cls.input_ids = get_input_ids(cls.model, num_samples=18)
@classmethod
def tearDownClass(cls):
kill_process_tree(cls.process.pid)
class TestUnifiedMambaRadixCache(UnifiedRadixTreeTestMixin, CustomTestCase):
"""Mamba hybrid + UnifiedRadixCache."""
kl_threshold = 0.003
prefill_cache_assert = staticmethod(
make_mamba_prefill_assert(chunk_size=MAMBA_CHUNK_SIZE)
)
decode_cache_assert = staticmethod(
make_mamba_decode_assert(track_interval=MAMBA_TRACK_INTERVAL)
)
@classmethod
def setUpClass(cls):
cls.model = MAMBA_MODEL
cls.base_url = DEFAULT_URL_FOR_TEST
cls.process = popen_launch_server(
cls.model,
cls.base_url,
timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
other_args=[
"--tp-size",
"4",
"--chunked-prefill-size",
"2048",
"--mem-fraction-static",
"0.85",
"--mamba-scheduler-strategy",
"extra_buffer",
"--mamba-track-interval",
str(MAMBA_TRACK_INTERVAL),
],
env={"SGLANG_ENABLE_UNIFIED_RADIX_TREE": "1"},
)
cls.input_ids = get_input_ids(cls.model, num_samples=18)
@classmethod
def tearDownClass(cls):
kill_process_tree(cls.process.pid)
class TestUnifiedSWARadixCache(UnifiedRadixTreeTestMixin, CustomTestCase):
"""SWA hybrid + UnifiedRadixCache."""
kl_threshold = 0.03
gsm8k_threshold = 0.7
mmlu_threshold = 0.7
@unittest.skipIf(is_in_ci(), "SWA model mmlu eval not stable enough")
def test_mmlu(self):
super().test_mmlu()
@classmethod
def setUpClass(cls):
cls.model = SWA_MODEL
cls.base_url = DEFAULT_URL_FOR_TEST
cls.process = popen_launch_server(
cls.model,
cls.base_url,
timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
other_args=[
"--tp-size",
"4",
"--mem-fraction-static",
"0.7",
"--disable-piecewise-cuda-graph",
],
env={"SGLANG_ENABLE_UNIFIED_RADIX_TREE": "1"},
)
cls.input_ids = get_input_ids(cls.model, num_samples=18)
@classmethod
def tearDownClass(cls):
kill_process_tree(cls.process.pid)
if __name__ == "__main__":
unittest.main()
@@ -23,7 +23,6 @@ Test strategy:
import unittest import unittest
from test_unified_radix_cache_kl import UnifiedRadixTreeTestMixin
from test_unified_radix_cache_kl_hicache import ( from test_unified_radix_cache_kl_hicache import (
DSV4_FLASH_LAUNCH_TIMEOUT, DSV4_FLASH_LAUNCH_TIMEOUT,
DSV4_FLASH_MODEL, DSV4_FLASH_MODEL,
@@ -31,6 +30,7 @@ from test_unified_radix_cache_kl_hicache import (
) )
from sglang.srt.utils import kill_process_tree from sglang.srt.utils import kill_process_tree
from sglang.test.kits.unified_radix_cache_kit import UnifiedRadixTreeTestMixin
from sglang.test.kl_multiturn_utils import get_input_ids from sglang.test.kl_multiturn_utils import get_input_ids
from sglang.test.test_utils import ( from sglang.test.test_utils import (
DEFAULT_URL_FOR_TEST, DEFAULT_URL_FOR_TEST,
@@ -1,105 +0,0 @@
import unittest
from types import SimpleNamespace
import requests
from sglang.srt.utils import kill_process_tree
from sglang.test.ci.ci_register import register_cuda_ci
from sglang.test.kits.reasoning_kit import ReasoningTokenUsageMixin
# This eval harness applies the chat_template, which is critical for qwen3.5
# to get good accuracy on gsm8k
from sglang.test.run_eval import run_eval
from sglang.test.test_utils import (
DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
DEFAULT_URL_FOR_TEST,
CustomTestCase,
popen_launch_server,
)
register_cuda_ci(est_time=370, stage="base-c", runner_config="4-gpu-b200")
QWEN35_FP4_MODEL = "nvidia/Qwen3.5-397B-A17B-NVFP4"
ACC_THRESHOLDS = {QWEN35_FP4_MODEL: {"gsm8k": 0.95}}
class TestQwen35FP4MTPV2(ReasoningTokenUsageMixin, CustomTestCase):
reasoning_parser_name = "qwen3"
@classmethod
def setUpClass(cls):
cls.model = QWEN35_FP4_MODEL
cls.base_url = DEFAULT_URL_FOR_TEST
cls.init_reasoning_token_verifier()
cls.process = popen_launch_server(
cls.model,
cls.base_url,
timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
other_args=[
"--tp-size",
"4",
"--chunked-prefill-size",
"2048",
"--mamba-scheduler-strategy",
"extra_buffer",
"--mamba-track-interval",
"128",
"--mamba-ssm-dtype",
"bfloat16",
"--max-running-requests",
"128",
"--reasoning-parser",
"qwen3",
"--attention-backend",
"trtllm_mha",
"--quantization",
"modelopt_fp4",
"--speculative-algorithm",
"NEXTN",
"--speculative-num-steps",
"3",
"--speculative-eagle-topk",
"1",
"--speculative-num-draft-tokens",
"4",
"--mem-fraction-static",
"0.8",
"--model-loader-extra-config",
'{"enable_multithread_load": true,"num_threads": 64}',
],
)
@classmethod
def tearDownClass(cls):
kill_process_tree(cls.process.pid)
def test_gsm8k(self):
args = SimpleNamespace(
model=self.model,
eval_name="gsm8k",
num_shots=5,
num_examples=200,
max_tokens=16000,
num_threads=128,
repeat=1,
temperature=0.6,
top_p=0.95,
top_k=20,
base_url=self.base_url,
host="http://127.0.0.1",
port=int(self.base_url.split(":")[-1]),
)
metrics = run_eval(args)
print(f"{metrics=}")
self.assertGreaterEqual(metrics["score"], ACC_THRESHOLDS[self.model]["gsm8k"])
server_info = requests.get(self.base_url + "/server_info")
avg_spec_accept_length = server_info.json()["internal_states"][0][
"avg_spec_accept_length"
]
print(f"{avg_spec_accept_length=}")
self.assertGreater(avg_spec_accept_length, 3.3)
if __name__ == "__main__":
unittest.main()
@@ -1,110 +0,0 @@
import unittest
from types import SimpleNamespace
import requests
from sglang.srt.utils import kill_process_tree
from sglang.test.ci.ci_register import register_cuda_ci
from sglang.test.run_eval import run_eval
from sglang.test.send_one import BenchArgs, send_one_prompt
from sglang.test.test_utils import (
DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
DEFAULT_URL_FOR_TEST,
CustomTestCase,
is_in_amd_ci,
is_in_ci,
popen_launch_server,
write_github_step_summary,
)
register_cuda_ci(est_time=300, stage="base-c", runner_config="8-gpu-h200")
FULL_DEEPSEEK_V3_MODEL_PATH = "deepseek-ai/DeepSeek-V3-0324"
class TestDeepseekV3MTP(CustomTestCase):
@classmethod
def setUpClass(cls):
cls.model = FULL_DEEPSEEK_V3_MODEL_PATH
cls.base_url = DEFAULT_URL_FOR_TEST
other_args = [
"--tp",
"8",
"--trust-remote-code",
"--speculative-algorithm",
"EAGLE",
"--speculative-num-steps",
"3",
"--speculative-eagle-topk",
"1",
"--speculative-num-draft-tokens",
"4",
"--model-loader-extra-config",
'{"enable_multithread_load": true, "num_threads": 64}',
]
if not is_in_amd_ci():
other_args += ["--mem-frac", "0.7"]
cls.process = popen_launch_server(
cls.model,
cls.base_url,
timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH * 5,
other_args=other_args,
)
@classmethod
def tearDownClass(cls):
kill_process_tree(cls.process.pid)
def test_a_gsm8k(
self,
): # Append an "a" to make this test run first (alphabetically) to warm up the server
requests.get(self.base_url + "/flush_cache")
args = SimpleNamespace(
base_url=self.base_url,
model=self.model,
eval_name="gsm8k",
api="completion",
max_tokens=512,
num_examples=200,
num_threads=128,
)
metrics = run_eval(args)
print(f"{metrics=}")
server_info = requests.get(self.base_url + "/server_info")
avg_spec_accept_length = server_info.json()["internal_states"][0][
"avg_spec_accept_length"
]
print(f"{avg_spec_accept_length=}")
if is_in_ci():
write_github_step_summary(
f"### test_gsm8k (deepseek-v3 mtp)\n"
f'{metrics["score"]=:.3f}\n'
f"{avg_spec_accept_length=:.2f}\n"
)
self.assertGreater(metrics["score"], 0.935)
self.assertGreater(avg_spec_accept_length, 2.8)
def test_bs_1_speed(self):
args = BenchArgs(port=int(self.base_url.split(":")[-1]), max_new_tokens=2048)
acc_length, speed = send_one_prompt(args)
print(f"{acc_length=:.2f} {speed=:.2f}")
if is_in_ci():
write_github_step_summary(
f"### test_bs_1_speed (deepseek-v3 mtp)\n"
f"{acc_length=:.2f}\n"
f"{speed=:.2f} token/s\n"
)
self.assertGreater(acc_length, 2.8)
if is_in_amd_ci():
self.assertGreater(speed, 15)
else:
self.assertGreater(speed, 130)
if __name__ == "__main__":
unittest.main()
@@ -0,0 +1,68 @@
import unittest
from sglang.test.ci.ci_register import register_cuda_ci
from sglang.test.kits.eval_accuracy_kit import GSM8KMixin
from sglang.test.send_one import BenchArgs, send_one_prompt
from sglang.test.server_fixtures.default_fixture import DefaultServerBase
from sglang.test.test_utils import is_in_ci, write_github_step_summary
register_cuda_ci(est_time=400, stage="base-c", runner_config="4-gpu-b200")
DSV32_FP4_MODEL = "nvidia/DeepSeek-V3.2-NVFP4"
class TestDeepseekV32FP4DPSpec(GSM8KMixin, DefaultServerBase):
model = DSV32_FP4_MODEL
timeout = 1200
other_args = [
"--tp",
"4",
"--dp",
"4",
"--enable-dp-attention",
"--attention-backend",
"dsa",
"--moe-runner-backend",
"flashinfer_trtllm",
"--quantization",
"modelopt_fp4",
"--tool-call-parser",
"deepseekv32",
"--reasoning-parser",
"deepseek-v3",
"--speculative-algorithm",
"EAGLE",
"--speculative-num-steps",
"3",
"--speculative-eagle-topk",
"1",
"--speculative-num-draft-tokens",
"4",
"--model-loader-extra-config",
'{"enable_multithread_load": true,"num_threads": 64}',
]
gsm8k_accuracy_thres = 0.93
gsm8k_num_questions = 500
gsm8k_num_threads = 500
gsm8k_num_shots = 20
gsm8k_accept_length_thres = 2.7
def test_z_bs_1_speed(self):
args = BenchArgs(port=int(self.base_url.split(":")[-1]), max_new_tokens=2048)
acc_length, speed = send_one_prompt(args)
print(f"{acc_length=:.2f} {speed=:.2f}")
if is_in_ci():
write_github_step_summary(
f"### test_bs_1_speed (deepseek-v32 mtp dp)\n"
f"{acc_length=:.2f}\n"
f"{speed=:.2f} token/s\n"
)
self.assertGreater(acc_length, 2.7)
self.assertGreater(speed, 90)
if __name__ == "__main__":
unittest.main()
@@ -0,0 +1,65 @@
import unittest
from sglang.test.ci.ci_register import register_cuda_ci
from sglang.test.kits.eval_accuracy_kit import GSM8KMixin
from sglang.test.send_one import BenchArgs, send_one_prompt
from sglang.test.server_fixtures.default_fixture import DefaultServerBase
from sglang.test.test_utils import is_in_ci, write_github_step_summary
register_cuda_ci(est_time=400, stage="base-c", runner_config="4-gpu-b200")
DSV32_FP4_MODEL = "nvidia/DeepSeek-V3.2-NVFP4"
class TestDeepseekV32FP4TPSpec(GSM8KMixin, DefaultServerBase):
model = DSV32_FP4_MODEL
timeout = 1200
other_args = [
"--tp",
"4",
"--attention-backend",
"dsa",
"--moe-runner-backend",
"flashinfer_trtllm",
"--quantization",
"modelopt_fp4",
"--tool-call-parser",
"deepseekv32",
"--reasoning-parser",
"deepseek-v3",
"--speculative-algorithm",
"EAGLE",
"--speculative-num-steps",
"3",
"--speculative-eagle-topk",
"1",
"--speculative-num-draft-tokens",
"4",
"--model-loader-extra-config",
'{"enable_multithread_load": true,"num_threads": 64}',
]
gsm8k_accuracy_thres = 0.93
gsm8k_num_questions = 500
gsm8k_num_threads = 500
gsm8k_num_shots = 20
gsm8k_accept_length_thres = 2.7
def test_z_bs_1_speed(self):
args = BenchArgs(port=int(self.base_url.split(":")[-1]), max_new_tokens=2048)
acc_length, speed = send_one_prompt(args)
print(f"{acc_length=:.2f} {speed=:.2f}")
if is_in_ci():
write_github_step_summary(
f"### test_bs_1_speed (deepseek-v32 mtp tp)\n"
f"{acc_length=:.2f}\n"
f"{speed=:.2f} token/s\n"
)
self.assertGreater(acc_length, 2.7)
self.assertGreater(speed, 150)
if __name__ == "__main__":
unittest.main()
@@ -0,0 +1,41 @@
import unittest
from sglang.test.ci.ci_register import register_cuda_ci
from sglang.test.kits.eval_accuracy_kit import GSM8KMixin
from sglang.test.server_fixtures.default_fixture import DefaultServerBase
# Per-commit: SymmetricMemory variant only.
# - TestDeepseekV3FP4 (TRTLLM) archived to test/manual/quant/test_deepseek_v3_fp4_4gpu_trtllm.py
# - TestDeepseekV3FP4CutlassMoE moved to test_deepseek_v3_fp4_4gpu_extra.py
register_cuda_ci(est_time=960, stage="base-c", runner_config="4-gpu-b200")
FULL_DEEPSEEK_V3_FP4_MODEL_PATH = "nvidia/DeepSeek-V3-0324-FP4"
class TestDeepseekV3FP4SymmetricMemory(GSM8KMixin, DefaultServerBase):
model = FULL_DEEPSEEK_V3_FP4_MODEL_PATH
timeout = 1200
other_args = [
"--tp",
"4",
"--attention-backend",
"trtllm_mla",
"--moe-runner-backend",
"flashinfer_trtllm",
"--quantization",
"modelopt_fp4",
"--kv-cache-dtype",
"fp8_e4m3",
"--model-loader-extra-config",
'{"enable_multithread_load": true,"num_threads": 64}',
"--enable-symm-mem",
]
gsm8k_accuracy_thres = 0.93
gsm8k_num_questions = 1319
gsm8k_num_threads = 1319
gsm8k_num_shots = 8
if __name__ == "__main__":
unittest.main()
@@ -0,0 +1,67 @@
import unittest
from sglang.test.ci.ci_register import register_cuda_ci
from sglang.test.kits.eval_accuracy_kit import GSM8KMixin
from sglang.test.send_one import BenchArgs, send_one_prompt
from sglang.test.server_fixtures.default_fixture import DefaultServerBase
from sglang.test.test_utils import (
DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
is_in_amd_ci,
is_in_ci,
write_github_step_summary,
)
register_cuda_ci(est_time=300, stage="base-c", runner_config="8-gpu-h200")
FULL_DEEPSEEK_V3_MODEL_PATH = "deepseek-ai/DeepSeek-V3-0324"
_OTHER_ARGS = [
"--tp",
"8",
"--trust-remote-code",
"--speculative-algorithm",
"EAGLE",
"--speculative-num-steps",
"3",
"--speculative-eagle-topk",
"1",
"--speculative-num-draft-tokens",
"4",
"--model-loader-extra-config",
'{"enable_multithread_load": true, "num_threads": 64}',
]
if not is_in_amd_ci():
_OTHER_ARGS += ["--mem-frac", "0.7"]
class TestDeepseekV3MTP(GSM8KMixin, DefaultServerBase):
model = FULL_DEEPSEEK_V3_MODEL_PATH
other_args = _OTHER_ARGS
timeout = DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH * 5
gsm8k_accuracy_thres = 0.935
gsm8k_accept_length_thres = 2.8
# `test_z_bs_1_speed` runs after `test_gsm8k` (alphabetical) so it
# measures steady-state speed on a warmed server.
def test_z_bs_1_speed(self):
args = BenchArgs(port=int(self.base_url.split(":")[-1]), max_new_tokens=2048)
acc_length, speed = send_one_prompt(args)
print(f"{acc_length=:.2f} {speed=:.2f}")
if is_in_ci():
write_github_step_summary(
f"### test_bs_1_speed (deepseek-v3 mtp)\n"
f"{acc_length=:.2f}\n"
f"{speed=:.2f} token/s\n"
)
self.assertGreater(acc_length, 2.8)
if is_in_amd_ci():
self.assertGreater(speed, 15)
else:
self.assertGreater(speed, 130)
if __name__ == "__main__":
unittest.main()
@@ -0,0 +1,23 @@
import unittest
from sglang.test.ci.ci_register import register_cuda_ci
from sglang.test.gpt_oss_common import BaseTestGptOss
register_cuda_ci(est_time=220, stage="base-c", runner_config="4-gpu-h100")
register_cuda_ci(est_time=220, stage="base-c", runner_config="4-gpu-b200")
class TestGptOss4GpuBf16(BaseTestGptOss):
def test_bf16_120b(self):
self.run_test(
model_variant="120b",
quantization="bf16",
expected_score_of_reasoning_effort={
"low": 0.58,
},
other_args=["--tp", "4", "--cuda-graph-max-bs", "200"],
)
if __name__ == "__main__":
unittest.main()
@@ -3,21 +3,11 @@ import unittest
from sglang.test.ci.ci_register import register_cuda_ci from sglang.test.ci.ci_register import register_cuda_ci
from sglang.test.gpt_oss_common import BaseTestGptOss from sglang.test.gpt_oss_common import BaseTestGptOss
register_cuda_ci(est_time=392, stage="base-c", runner_config="4-gpu-h100") register_cuda_ci(est_time=220, stage="base-c", runner_config="4-gpu-h100")
register_cuda_ci(est_time=350, stage="base-c", runner_config="4-gpu-b200") register_cuda_ci(est_time=220, stage="base-c", runner_config="4-gpu-b200")
class TestGptOss4Gpu(BaseTestGptOss): class TestGptOss4GpuMxfp4(BaseTestGptOss):
def test_bf16_120b(self):
self.run_test(
model_variant="120b",
quantization="bf16",
expected_score_of_reasoning_effort={
"low": 0.58,
},
other_args=["--tp", "4", "--cuda-graph-max-bs", "200"],
)
def test_mxfp4_120b(self): def test_mxfp4_120b(self):
self.run_test( self.run_test(
model_variant="120b", model_variant="120b",
@@ -0,0 +1,47 @@
import unittest
from sglang.test.ci.ci_register import register_cuda_ci
from sglang.test.kits.eval_accuracy_kit import GSM8KMixin
from sglang.test.server_fixtures.mmmu_fixture import MMMUServerBase
register_cuda_ci(est_time=400, stage="base-c", runner_config="8-gpu-h200")
MIMO_V2_MODEL = "XiaomiMiMo/MiMo-V2.5"
MIMO_V2_OTHER_ARGS = [
"--tp",
"8",
"--dp",
"2",
"--enable-dp-attention",
"--mm-enable-dp-encoder",
"--attention-backend",
"fa3",
"--mm-attention-backend",
"fa3",
"--reasoning-parser",
"mimo",
]
MIMO_V2_MTP_OTHER_ARGS = MIMO_V2_OTHER_ARGS + [
"--speculative-algorithm",
"EAGLE",
"--speculative-num-steps",
"3",
"--speculative-eagle-topk",
"1",
"--speculative-num-draft-tokens",
"4",
"--enable-multi-layer-eagle",
]
class TestMiMoV2(GSM8KMixin, MMMUServerBase):
gsm8k_accuracy_thres = 0.75
gsm8k_accept_length_thres = 2.5
model = MIMO_V2_MODEL
mem_fraction_static = 0.65
server_api_key = None
other_args = MIMO_V2_MTP_OTHER_ARGS
if __name__ == "__main__":
unittest.main()
@@ -4,9 +4,8 @@ from sglang.test.ci.ci_register import register_cuda_ci
from sglang.test.kits.eval_accuracy_kit import GSM8KMixin from sglang.test.kits.eval_accuracy_kit import GSM8KMixin
from sglang.test.kits.spec_decoding_kit import SpecDecodingMixin from sglang.test.kits.spec_decoding_kit import SpecDecodingMixin
from sglang.test.server_fixtures.default_fixture import DefaultServerBase from sglang.test.server_fixtures.default_fixture import DefaultServerBase
from sglang.test.server_fixtures.mmmu_fixture import MMMUServerBase
register_cuda_ci(est_time=500, stage="base-c", runner_config="8-gpu-h200") register_cuda_ci(est_time=350, stage="base-c", runner_config="8-gpu-h200")
class TestMiMoV2Flash(GSM8KMixin, SpecDecodingMixin, DefaultServerBase): class TestMiMoV2Flash(GSM8KMixin, SpecDecodingMixin, DefaultServerBase):
@@ -49,42 +48,5 @@ class TestMiMoV2Flash(GSM8KMixin, SpecDecodingMixin, DefaultServerBase):
accept_length_thres = 3.2 accept_length_thres = 3.2
MIMO_V2_MODEL = "XiaomiMiMo/MiMo-V2.5"
MIMO_V2_OTHER_ARGS = [
"--tp",
"8",
"--dp",
"2",
"--enable-dp-attention",
"--mm-enable-dp-encoder",
"--attention-backend",
"fa3",
"--mm-attention-backend",
"fa3",
"--reasoning-parser",
"mimo",
]
MIMO_V2_MTP_OTHER_ARGS = MIMO_V2_OTHER_ARGS + [
"--speculative-algorithm",
"EAGLE",
"--speculative-num-steps",
"3",
"--speculative-eagle-topk",
"1",
"--speculative-num-draft-tokens",
"4",
"--enable-multi-layer-eagle",
]
class TestMiMoV2(GSM8KMixin, MMMUServerBase):
gsm8k_accuracy_thres = 0.75
gsm8k_accept_length_thres = 2.5
model = MIMO_V2_MODEL
mem_fraction_static = 0.65
server_api_key = None
other_args = MIMO_V2_MTP_OTHER_ARGS
if __name__ == "__main__": if __name__ == "__main__":
unittest.main() unittest.main()
@@ -0,0 +1,69 @@
import unittest
from types import SimpleNamespace
from sglang.srt.utils import kill_process_tree
from sglang.test.ci.ci_register import register_cuda_ci
from sglang.test.run_eval import run_eval
from sglang.test.test_utils import (
DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
DEFAULT_URL_FOR_TEST,
CustomTestCase,
popen_launch_server,
)
register_cuda_ci(est_time=200, stage="extra-b", runner_config="8-gpu-h200")
NEMOTRON_3_SUPER_BF16_MODEL = "nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-BF16"
NEMOTRON_3_SUPER_BF16_ARGS = [
"--tp-size",
"8",
"--trust-remote-code",
"--reasoning-parser",
"nemotron_3",
"--tool-call-parser",
"qwen3_coder",
"--disable-radix-cache",
"--model-loader-extra-config",
'{"enable_multithread_load": true, "num_threads": 50}',
]
class TestNvidiaNemotron3SuperBF16(CustomTestCase):
@classmethod
def setUpClass(cls):
cls.model = NEMOTRON_3_SUPER_BF16_MODEL
cls.base_url = DEFAULT_URL_FOR_TEST
cls.process = popen_launch_server(
cls.model,
cls.base_url,
timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
other_args=NEMOTRON_3_SUPER_BF16_ARGS,
)
@classmethod
def tearDownClass(cls):
kill_process_tree(cls.process.pid)
def test_gsm8k(self):
args = SimpleNamespace(
model=self.model,
eval_name="gsm8k",
num_shots=5,
num_examples=200,
max_tokens=16000,
num_threads=200,
repeat=1,
temperature=1.0,
top_p=0.95,
base_url=self.base_url,
host="http://127.0.0.1",
port=int(self.base_url.split(":")[-1]),
)
metrics = run_eval(args)
print(f"{metrics=}")
self.assertGreaterEqual(metrics["score"], 0.96)
if __name__ == "__main__":
unittest.main()
@@ -11,7 +11,7 @@ from sglang.test.test_utils import (
popen_launch_server, popen_launch_server,
) )
register_cuda_ci(est_time=370, stage="extra-b", runner_config="8-gpu-h200") register_cuda_ci(est_time=200, stage="extra-b", runner_config="8-gpu-h200")
NEMOTRON_3_SUPER_BF16_MODEL = "nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-BF16" NEMOTRON_3_SUPER_BF16_MODEL = "nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-BF16"
@@ -44,46 +44,6 @@ MTP_ARGS = [
] ]
def _run_gsm8k(test_case):
args = SimpleNamespace(
model=test_case.model,
eval_name="gsm8k",
num_shots=5,
num_examples=200,
max_tokens=16000,
num_threads=200,
repeat=1,
temperature=1.0,
top_p=0.95,
base_url=test_case.base_url,
host="http://127.0.0.1",
port=int(test_case.base_url.split(":")[-1]),
)
metrics = run_eval(args)
print(f"{metrics=}")
test_case.assertGreaterEqual(metrics["score"], 0.96)
class TestNvidiaNemotron3SuperBF16(CustomTestCase):
@classmethod
def setUpClass(cls):
cls.model = NEMOTRON_3_SUPER_BF16_MODEL
cls.base_url = DEFAULT_URL_FOR_TEST
cls.process = popen_launch_server(
cls.model,
cls.base_url,
timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
other_args=NEMOTRON_3_SUPER_BF16_ARGS,
)
@classmethod
def tearDownClass(cls):
kill_process_tree(cls.process.pid)
def test_gsm8k(self):
_run_gsm8k(self)
class TestNvidiaNemotron3SuperBF16MTP(CustomTestCase): class TestNvidiaNemotron3SuperBF16MTP(CustomTestCase):
@classmethod @classmethod
def setUpClass(cls): def setUpClass(cls):
@@ -101,7 +61,23 @@ class TestNvidiaNemotron3SuperBF16MTP(CustomTestCase):
kill_process_tree(cls.process.pid) kill_process_tree(cls.process.pid)
def test_gsm8k(self): def test_gsm8k(self):
_run_gsm8k(self) args = SimpleNamespace(
model=self.model,
eval_name="gsm8k",
num_shots=5,
num_examples=200,
max_tokens=16000,
num_threads=200,
repeat=1,
temperature=1.0,
top_p=0.95,
base_url=self.base_url,
host="http://127.0.0.1",
port=int(self.base_url.split(":")[-1]),
)
metrics = run_eval(args)
print(f"{metrics=}")
self.assertGreaterEqual(metrics["score"], 0.96)
if __name__ == "__main__": if __name__ == "__main__":
@@ -21,7 +21,7 @@ QWEN35_FP4_MODEL = "nvidia/Qwen3.5-397B-A17B-NVFP4"
ACC_THRESHOLDS = {QWEN35_FP4_MODEL: {"gsm8k": 0.95}} ACC_THRESHOLDS = {QWEN35_FP4_MODEL: {"gsm8k": 0.95}}
class TestQwen35FP4MTPV2(ReasoningTokenUsageMixin, CustomTestCase): class TestQwen35FP4MTP(ReasoningTokenUsageMixin, CustomTestCase):
reasoning_parser_name = "qwen3" reasoning_parser_name = "qwen3"
@classmethod @classmethod
@@ -1,28 +1,20 @@
import unittest import unittest
from types import SimpleNamespace
import numpy as np import numpy as np
import requests import requests
from sglang.srt.environ import envs from sglang.srt.environ import envs
from sglang.srt.utils import kill_process_tree
from sglang.test.ci.ci_register import register_cuda_ci from sglang.test.ci.ci_register import register_cuda_ci
from sglang.test.run_eval import run_eval from sglang.test.kits.eval_accuracy_kit import GSM8KMixin
from sglang.test.test_utils import ( from sglang.test.server_fixtures.default_fixture import DefaultServerBase
DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, from sglang.test.test_utils import DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH
DEFAULT_URL_FOR_TEST,
CustomTestCase,
is_in_ci,
popen_launch_server,
write_github_step_summary,
)
register_cuda_ci(est_time=480, stage="extra-b", runner_config="8-gpu-h200") register_cuda_ci(est_time=480, stage="extra-b", runner_config="8-gpu-h200")
STEP3P5_FLASH_MODEL_PATH = "stepfun-ai/Step-3.5-Flash" STEP3P5_FLASH_MODEL_PATH = "stepfun-ai/Step-3.5-Flash"
class TestStep3p5FlashChainMTP(CustomTestCase): class TestStep3p5FlashChainMTP(GSM8KMixin, DefaultServerBase):
"""Chain-style multi-layer EAGLE speculative decoding on Step-3.5-Flash. """Chain-style multi-layer EAGLE speculative decoding on Step-3.5-Flash.
Step3p5ForCausalLM auto-enables multi-layer EAGLE and spec v2 when Step3p5ForCausalLM auto-enables multi-layer EAGLE and spec v2 when
@@ -31,75 +23,38 @@ class TestStep3p5FlashChainMTP(CustomTestCase):
automatically for the Step3p5MTP draft architecture. automatically for the Step3p5MTP draft architecture.
""" """
model = STEP3P5_FLASH_MODEL_PATH
timeout = DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH * 3
other_args = [
"--tp",
"8",
"--trust-remote-code",
"--speculative-algorithm",
"EAGLE",
"--speculative-num-steps",
"3",
"--speculative-eagle-topk",
"1",
"--speculative-num-draft-tokens",
"4",
"--attention-backend",
"fa3",
"--enable-multi-layer-eagle",
"--mem-fraction-static",
"0.75",
"--chunked-prefill-size",
"4096",
"--model-loader-extra-config",
'{"enable_multithread_load": true, "num_threads": 64}',
]
gsm8k_accuracy_thres = 0.83
gsm8k_accept_length_thres = 2.6
@classmethod @classmethod
def setUpClass(cls): def setUpClass(cls):
cls.model = STEP3P5_FLASH_MODEL_PATH
cls.base_url = DEFAULT_URL_FOR_TEST
other_args = [
"--tp",
"8",
"--trust-remote-code",
"--speculative-algorithm",
"EAGLE",
"--speculative-num-steps",
"3",
"--speculative-eagle-topk",
"1",
"--speculative-num-draft-tokens",
"4",
"--attention-backend",
"fa3",
"--enable-multi-layer-eagle",
"--mem-fraction-static",
"0.75",
"--chunked-prefill-size",
"4096",
"--model-loader-extra-config",
'{"enable_multithread_load": true, "num_threads": 64}',
]
with envs.SGLANG_ENABLE_SPEC_V2.override(True): with envs.SGLANG_ENABLE_SPEC_V2.override(True):
cls.process = popen_launch_server( super().setUpClass()
cls.model,
cls.base_url,
timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH * 3,
other_args=other_args,
)
@classmethod
def tearDownClass(cls):
if hasattr(cls, "process") and cls.process:
kill_process_tree(cls.process.pid)
def test_gsm8k(self):
requests.get(self.base_url + "/flush_cache")
args = SimpleNamespace(
base_url=self.base_url,
model=self.model,
eval_name="gsm8k",
api="completion",
max_tokens=512,
num_examples=200,
num_threads=128,
)
metrics = run_eval(args)
print(f"{metrics=}")
server_info = requests.get(self.base_url + "/server_info")
avg_spec_accept_length = server_info.json()["internal_states"][0][
"avg_spec_accept_length"
]
print(f"{avg_spec_accept_length=}")
print(f"{metrics=}")
if is_in_ci():
write_github_step_summary(
f"### test_gsm8k (step-3.5-flash chain mtp)\n"
f'{metrics["score"]=:.3f}\n'
f"{avg_spec_accept_length=:.2f}\n"
)
self.assertGreater(metrics["score"], 0.83)
self.assertGreater(avg_spec_accept_length, 2.6)
def test_logprob_spec_v2_match(self): def test_logprob_spec_v2_match(self):
"""Verify spec v2 decode logprobs match prefill scoring logprobs. """Verify spec v2 decode logprobs match prefill scoring logprobs.
@@ -1,212 +0,0 @@
import unittest
from types import SimpleNamespace
import requests
from sglang.srt.utils import kill_process_tree
from sglang.test.ci.ci_register import register_cuda_ci
from sglang.test.run_eval import run_eval
from sglang.test.send_one import BenchArgs, send_one_prompt
from sglang.test.test_utils import (
DEFAULT_URL_FOR_TEST,
CustomTestCase,
is_in_ci,
popen_launch_server,
write_github_step_summary,
)
register_cuda_ci(
est_time=690,
stage="base-c",
runner_config="4-gpu-b200",
)
FULL_DEEPSEEK_V3_FP4_MODEL_PATH = "nvidia/DeepSeek-V3.2-NVFP4"
SERVER_LAUNCH_TIMEOUT = 1200
class TestDeepseekV32FP4DPSpecV2(CustomTestCase):
@classmethod
def setUpClass(cls):
cls.model = FULL_DEEPSEEK_V3_FP4_MODEL_PATH
cls.base_url = DEFAULT_URL_FOR_TEST
other_args = [
"--tp",
"4",
"--dp",
"4",
"--enable-dp-attention",
"--attention-backend",
"dsa",
"--moe-runner-backend",
"flashinfer_trtllm",
"--quantization",
"modelopt_fp4",
"--tool-call-parser",
"deepseekv32",
"--reasoning-parser",
"deepseek-v3",
"--speculative-algorithm",
"EAGLE",
"--speculative-num-steps",
"3",
"--speculative-eagle-topk",
"1",
"--speculative-num-draft-tokens",
"4",
"--model-loader-extra-config",
'{"enable_multithread_load": true,"num_threads": 64}',
]
cls.process = popen_launch_server(
cls.model,
cls.base_url,
timeout=SERVER_LAUNCH_TIMEOUT,
other_args=other_args,
)
@classmethod
def tearDownClass(cls):
kill_process_tree(cls.process.pid)
def test_a_gsm8k(
self,
): # Append an "a" to make this test run first (alphabetically) to warm up the server
requests.get(self.base_url + "/flush_cache")
args = SimpleNamespace(
base_url=self.base_url,
model=self.model,
eval_name="gsm8k",
api="completion",
max_tokens=512,
num_examples=500,
num_threads=500,
num_shots=20,
)
metrics = run_eval(args)
print(f"{metrics=}")
server_info = requests.get(self.base_url + "/server_info")
avg_spec_accept_length = server_info.json()["internal_states"][0][
"avg_spec_accept_length"
]
print(f"{avg_spec_accept_length=}")
if is_in_ci():
write_github_step_summary(
f"### test_gsm8k (deepseek-v32 mtp)\n"
f'{metrics["score"]=:.3f}\n'
f"{avg_spec_accept_length=:.2f}\n"
)
self.assertGreater(metrics["score"], 0.93)
self.assertGreater(avg_spec_accept_length, 2.7)
def test_bs_1_speed(self):
args = BenchArgs(port=int(self.base_url.split(":")[-1]), max_new_tokens=2048)
acc_length, speed = send_one_prompt(args)
print(f"{acc_length=:.2f} {speed=:.2f}")
if is_in_ci():
write_github_step_summary(
f"### test_bs_1_speed (deepseek-v32 mtp)\n"
f"{acc_length=:.2f}\n"
f"{speed=:.2f} token/s\n"
)
self.assertGreater(acc_length, 2.7)
self.assertGreater(speed, 90)
class TestDeepseekV32FP4TPSpecV2(CustomTestCase):
@classmethod
def setUpClass(cls):
cls.model = FULL_DEEPSEEK_V3_FP4_MODEL_PATH
cls.base_url = DEFAULT_URL_FOR_TEST
other_args = [
"--tp",
"4",
"--attention-backend",
"dsa",
"--moe-runner-backend",
"flashinfer_trtllm",
"--quantization",
"modelopt_fp4",
"--tool-call-parser",
"deepseekv32",
"--reasoning-parser",
"deepseek-v3",
"--speculative-algorithm",
"EAGLE",
"--speculative-num-steps",
"3",
"--speculative-eagle-topk",
"1",
"--speculative-num-draft-tokens",
"4",
"--model-loader-extra-config",
'{"enable_multithread_load": true,"num_threads": 64}',
]
cls.process = popen_launch_server(
cls.model,
cls.base_url,
timeout=SERVER_LAUNCH_TIMEOUT,
other_args=other_args,
)
@classmethod
def tearDownClass(cls):
kill_process_tree(cls.process.pid)
def test_a_gsm8k(
self,
): # Append an "a" to make this test run first (alphabetically) to warm up the server
requests.get(self.base_url + "/flush_cache")
args = SimpleNamespace(
base_url=self.base_url,
model=self.model,
eval_name="gsm8k",
api="completion",
max_tokens=512,
num_examples=500,
num_threads=500,
num_shots=20,
)
metrics = run_eval(args)
print(f"{metrics=}")
server_info = requests.get(self.base_url + "/server_info")
avg_spec_accept_length = server_info.json()["internal_states"][0][
"avg_spec_accept_length"
]
print(f"{avg_spec_accept_length=}")
if is_in_ci():
write_github_step_summary(
f"### test_gsm8k (deepseek-v32 mtp)\n"
f'{metrics["score"]=:.3f}\n'
f"{avg_spec_accept_length=:.2f}\n"
)
self.assertGreater(metrics["score"], 0.93)
self.assertGreater(avg_spec_accept_length, 2.7)
def test_bs_1_speed(self):
args = BenchArgs(port=int(self.base_url.split(":")[-1]), max_new_tokens=2048)
acc_length, speed = send_one_prompt(args)
print(f"{acc_length=:.2f} {speed=:.2f}")
if is_in_ci():
write_github_step_summary(
f"### test_bs_1_speed (deepseek-v32 mtp)\n"
f"{acc_length=:.2f}\n"
f"{speed=:.2f} token/s\n"
)
self.assertGreater(acc_length, 2.7)
self.assertGreater(speed, 150)
if __name__ == "__main__":
unittest.main()
@@ -1,80 +0,0 @@
import unittest
from types import SimpleNamespace
from sglang.srt.utils import kill_process_tree
from sglang.test.ci.ci_register import register_cuda_ci
from sglang.test.run_eval import run_eval
from sglang.test.test_utils import (
DEFAULT_URL_FOR_TEST,
CustomTestCase,
is_in_ci,
popen_launch_server,
write_github_step_summary,
)
# Per-commit: SymmetricMemory variant only.
# - TestDeepseekV3FP4 (TRTLLM) archived to test/manual/quant/test_deepseek_v3_fp4_4gpu_trtllm.py
# - TestDeepseekV3FP4CutlassMoE moved to test_deepseek_v3_fp4_4gpu_extra.py
register_cuda_ci(est_time=960, stage="base-c", runner_config="4-gpu-b200")
FULL_DEEPSEEK_V3_FP4_MODEL_PATH = "nvidia/DeepSeek-V3-0324-FP4"
SERVER_LAUNCH_TIMEOUT = 1200
class TestDeepseekV3FP4SymmetricMemory(CustomTestCase):
@classmethod
def setUpClass(cls):
cls.model = FULL_DEEPSEEK_V3_FP4_MODEL_PATH
cls.base_url = DEFAULT_URL_FOR_TEST
other_args = [
"--tp",
"4",
"--attention-backend",
"trtllm_mla",
"--moe-runner-backend",
"flashinfer_trtllm",
"--quantization",
"modelopt_fp4",
"--kv-cache-dtype",
"fp8_e4m3",
"--model-loader-extra-config",
'{"enable_multithread_load": true,"num_threads": 64}',
"--enable-symm-mem",
]
cls.process = popen_launch_server(
cls.model,
cls.base_url,
timeout=SERVER_LAUNCH_TIMEOUT,
other_args=other_args,
)
@classmethod
def tearDownClass(cls):
kill_process_tree(cls.process.pid)
def test_a_gsm8k(
self,
): # Append an "a" to make this test run first (alphabetically) to warm up the server
args = SimpleNamespace(
base_url=self.base_url,
model=self.model,
eval_name="gsm8k",
api="completion",
max_tokens=512,
num_examples=1319,
num_threads=1319,
num_shots=8,
)
metrics = run_eval(args)
print(f"{metrics=}")
if is_in_ci():
write_github_step_summary(
f"### test_gsm8k (deepseek-v3-fp4)\n" f'{metrics["score"]=:.3f}\n'
)
self.assertGreater(metrics["score"], 0.93)
if __name__ == "__main__":
unittest.main()
@@ -0,0 +1,50 @@
import unittest
from sglang.srt.utils import kill_process_tree
from sglang.test.ci.ci_register import register_cuda_ci
from sglang.test.kits.unified_radix_cache_kit import UnifiedRadixTreeTestMixin
from sglang.test.kl_multiturn_utils import get_input_ids
from sglang.test.test_utils import (
DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
DEFAULT_URL_FOR_TEST,
CustomTestCase,
popen_launch_server,
)
register_cuda_ci(est_time=250, stage="base-c", runner_config="4-gpu-h100")
FULL_MODEL = "Qwen/Qwen3-32B"
class TestUnifiedFullRadixCache(UnifiedRadixTreeTestMixin, CustomTestCase):
"""Full attention."""
kl_threshold = 0.0025
@classmethod
def setUpClass(cls):
cls.model = FULL_MODEL
cls.base_url = DEFAULT_URL_FOR_TEST
cls.process = popen_launch_server(
cls.model,
cls.base_url,
timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
other_args=[
"--tp-size",
"4",
"--mem-fraction-static",
"0.80",
"--page-size",
"64",
],
env={"SGLANG_ENABLE_UNIFIED_RADIX_TREE": "1"},
)
cls.input_ids = get_input_ids(cls.model, num_samples=18)
@classmethod
def tearDownClass(cls):
kill_process_tree(cls.process.pid)
if __name__ == "__main__":
unittest.main()
@@ -1,9 +1,8 @@
import unittest import unittest
from test_unified_radix_cache_kl import UnifiedRadixTreeTestMixin
from sglang.srt.utils import kill_process_tree from sglang.srt.utils import kill_process_tree
from sglang.test.ci.ci_register import register_cuda_ci from sglang.test.ci.ci_register import register_cuda_ci
from sglang.test.kits.unified_radix_cache_kit import UnifiedRadixTreeTestMixin
from sglang.test.kl_multiturn_utils import ( from sglang.test.kl_multiturn_utils import (
get_input_ids, get_input_ids,
make_mamba_decode_assert, make_mamba_decode_assert,
@@ -0,0 +1,66 @@
import unittest
from sglang.srt.utils import kill_process_tree
from sglang.test.ci.ci_register import register_cuda_ci
from sglang.test.kits.unified_radix_cache_kit import UnifiedRadixTreeTestMixin
from sglang.test.kl_multiturn_utils import (
get_input_ids,
make_mamba_decode_assert,
make_mamba_prefill_assert,
)
from sglang.test.test_utils import (
DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
DEFAULT_URL_FOR_TEST,
CustomTestCase,
popen_launch_server,
)
register_cuda_ci(est_time=250, stage="base-c", runner_config="4-gpu-h100")
MAMBA_MODEL = "Qwen/Qwen3-Next-80B-A3B-Instruct"
MAMBA_CHUNK_SIZE = 64
MAMBA_TRACK_INTERVAL = 128
class TestUnifiedMambaRadixCache(UnifiedRadixTreeTestMixin, CustomTestCase):
"""Mamba hybrid + UnifiedRadixCache."""
kl_threshold = 0.003
prefill_cache_assert = staticmethod(
make_mamba_prefill_assert(chunk_size=MAMBA_CHUNK_SIZE)
)
decode_cache_assert = staticmethod(
make_mamba_decode_assert(track_interval=MAMBA_TRACK_INTERVAL)
)
@classmethod
def setUpClass(cls):
cls.model = MAMBA_MODEL
cls.base_url = DEFAULT_URL_FOR_TEST
cls.process = popen_launch_server(
cls.model,
cls.base_url,
timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
other_args=[
"--tp-size",
"4",
"--chunked-prefill-size",
"2048",
"--mem-fraction-static",
"0.85",
"--mamba-scheduler-strategy",
"extra_buffer",
"--mamba-track-interval",
str(MAMBA_TRACK_INTERVAL),
],
env={"SGLANG_ENABLE_UNIFIED_RADIX_TREE": "1"},
)
cls.input_ids = get_input_ids(cls.model, num_samples=18)
@classmethod
def tearDownClass(cls):
kill_process_tree(cls.process.pid)
if __name__ == "__main__":
unittest.main()
@@ -0,0 +1,56 @@
import unittest
from sglang.srt.utils import kill_process_tree
from sglang.test.ci.ci_register import register_cuda_ci
from sglang.test.kits.unified_radix_cache_kit import UnifiedRadixTreeTestMixin
from sglang.test.kl_multiturn_utils import get_input_ids
from sglang.test.test_utils import (
DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
DEFAULT_URL_FOR_TEST,
CustomTestCase,
is_in_ci,
popen_launch_server,
)
register_cuda_ci(est_time=250, stage="base-c", runner_config="4-gpu-h100")
SWA_MODEL = "openai/gpt-oss-20b"
class TestUnifiedSWARadixCache(UnifiedRadixTreeTestMixin, CustomTestCase):
"""SWA hybrid + UnifiedRadixCache."""
kl_threshold = 0.03
gsm8k_threshold = 0.7
mmlu_threshold = 0.7
@unittest.skipIf(is_in_ci(), "SWA model mmlu eval not stable enough")
def test_mmlu(self):
super().test_mmlu()
@classmethod
def setUpClass(cls):
cls.model = SWA_MODEL
cls.base_url = DEFAULT_URL_FOR_TEST
cls.process = popen_launch_server(
cls.model,
cls.base_url,
timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
other_args=[
"--tp-size",
"4",
"--mem-fraction-static",
"0.7",
"--disable-piecewise-cuda-graph",
],
env={"SGLANG_ENABLE_UNIFIED_RADIX_TREE": "1"},
)
cls.input_ids = get_input_ids(cls.model, num_samples=18)
@classmethod
def tearDownClass(cls):
kill_process_tree(cls.process.pid)
if __name__ == "__main__":
unittest.main()