minke.yu/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 12 Packages Projects Releases Wiki Activity

test/registered: cleanup pure model e2e tests (moves, splits, dedup, kit) (#26610)

Browse Source
This commit is contained in:
Liangsheng Yin
2026-05-28 15:41:46 -07:00
committed by GitHub
parent 435c4ffb30
commit ec78fa6518
26 changed files with 611 additions and 816 deletions
Download Patch File Download Diff File Expand all files Collapse all files
test/registered/radix_cache/test_unified_radix_cache_kl.py → python/sglang/test/kits/unified_radix_cache_kit.py
+1 -133
View File
@@ -1,25 +1,12 @@
import random
import unittest
from types import SimpleNamespace
from urllib.parse import urlparse
from sglang.srt.utils import kill_process_tree
from sglang.test.ci.ci_register import register_cuda_ci
from sglang.test.kl_multiturn_utils import (
get_input_ids,
make_mamba_decode_assert,
make_mamba_prefill_assert,
test_input_output_logprobs_match_decode_cache_hit_helper,
test_input_output_logprobs_match_helper,
test_input_output_logprobs_match_prefill_cache_hit_helper,
)
from sglang.test.test_utils import (
DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
DEFAULT_URL_FOR_TEST,
CustomTestCase,
is_in_ci,
popen_launch_server,
)
def _random_suffixes(n, length, seed):
@@ -28,18 +15,8 @@ def _random_suffixes(n, length, seed):
return [[rng.randint(1, 30000) for _ in range(length)] for _ in range(n)]
MAMBA_MODEL = "Qwen/Qwen3-Next-80B-A3B-Instruct"
MAMBA_CHUNK_SIZE = 64
MAMBA_TRACK_INTERVAL = 128
SWA_MODEL = "openai/gpt-oss-20b"
FULL_MODEL = "Qwen/Qwen3-32B"
register_cuda_ci(est_time=760, stage="base-c", runner_config="4-gpu-h100")
class UnifiedRadixTreeTestMixin:
"""Mixin: gsm8k、mmlu and multi-turn KL tests with multi-branch interleaving."""
"""Mixin: gsm8k, mmlu and multi-turn KL tests with multi-branch interleaving."""
kl_threshold: float = 0.003
max_new_tokens: int = 512
@@ -168,112 +145,3 @@ class UnifiedRadixTreeTestMixin:
request_batch_size=self.decode_hit_request_batch_size,
inter_batch_delay_s=self.decode_hit_inter_batch_delay_s,
)
class TestUnifiedFullRadixCache(UnifiedRadixTreeTestMixin, CustomTestCase):
"""Full attention."""
kl_threshold = 0.0025
@classmethod
def setUpClass(cls):
cls.model = FULL_MODEL
cls.base_url = DEFAULT_URL_FOR_TEST
cls.process = popen_launch_server(
cls.model,
cls.base_url,
timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
other_args=[
"--tp-size",
"4",
"--mem-fraction-static",
"0.80",
"--page-size",
"64",
],
env={"SGLANG_ENABLE_UNIFIED_RADIX_TREE": "1"},
)
cls.input_ids = get_input_ids(cls.model, num_samples=18)
@classmethod
def tearDownClass(cls):
kill_process_tree(cls.process.pid)
class TestUnifiedMambaRadixCache(UnifiedRadixTreeTestMixin, CustomTestCase):
"""Mamba hybrid + UnifiedRadixCache."""
kl_threshold = 0.003
prefill_cache_assert = staticmethod(
make_mamba_prefill_assert(chunk_size=MAMBA_CHUNK_SIZE)
)
decode_cache_assert = staticmethod(
make_mamba_decode_assert(track_interval=MAMBA_TRACK_INTERVAL)
)
@classmethod
def setUpClass(cls):
cls.model = MAMBA_MODEL
cls.base_url = DEFAULT_URL_FOR_TEST
cls.process = popen_launch_server(
cls.model,
cls.base_url,
timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
other_args=[
"--tp-size",
"4",
"--chunked-prefill-size",
"2048",
"--mem-fraction-static",
"0.85",
"--mamba-scheduler-strategy",
"extra_buffer",
"--mamba-track-interval",
str(MAMBA_TRACK_INTERVAL),
],
env={"SGLANG_ENABLE_UNIFIED_RADIX_TREE": "1"},
)
cls.input_ids = get_input_ids(cls.model, num_samples=18)
@classmethod
def tearDownClass(cls):
kill_process_tree(cls.process.pid)
class TestUnifiedSWARadixCache(UnifiedRadixTreeTestMixin, CustomTestCase):
"""SWA hybrid + UnifiedRadixCache."""
kl_threshold = 0.03
gsm8k_threshold = 0.7
mmlu_threshold = 0.7
@unittest.skipIf(is_in_ci(), "SWA model mmlu eval not stable enough")
def test_mmlu(self):
super().test_mmlu()
@classmethod
def setUpClass(cls):
cls.model = SWA_MODEL
cls.base_url = DEFAULT_URL_FOR_TEST
cls.process = popen_launch_server(
cls.model,
cls.base_url,
timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
other_args=[
"--tp-size",
"4",
"--mem-fraction-static",
"0.7",
"--disable-piecewise-cuda-graph",
],
env={"SGLANG_ENABLE_UNIFIED_RADIX_TREE": "1"},
)
cls.input_ids = get_input_ids(cls.model, num_samples=18)
@classmethod
def tearDownClass(cls):
kill_process_tree(cls.process.pid)
if __name__ == "__main__":
unittest.main()
test/manual/core/test_dsv4_hicache_swa_translation_cache.py
+1 -1
View File
@@ -23,7 +23,6 @@ Test strategy:
import unittest
from test_unified_radix_cache_kl import UnifiedRadixTreeTestMixin
from test_unified_radix_cache_kl_hicache import (
DSV4_FLASH_LAUNCH_TIMEOUT,
DSV4_FLASH_MODEL,
@@ -31,6 +30,7 @@ from test_unified_radix_cache_kl_hicache import (
)
from sglang.srt.utils import kill_process_tree
from sglang.test.kits.unified_radix_cache_kit import UnifiedRadixTreeTestMixin
from sglang.test.kl_multiturn_utils import get_input_ids
from sglang.test.test_utils import (
DEFAULT_URL_FOR_TEST,
test/registered/4-gpu-models/test_qwen35_models.py
-105
View File
@@ -1,105 +0,0 @@
import unittest
from types import SimpleNamespace
import requests
from sglang.srt.utils import kill_process_tree
from sglang.test.ci.ci_register import register_cuda_ci
from sglang.test.kits.reasoning_kit import ReasoningTokenUsageMixin
# This eval harness applies the chat_template, which is critical for qwen3.5
# to get good accuracy on gsm8k
from sglang.test.run_eval import run_eval
from sglang.test.test_utils import (
DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
DEFAULT_URL_FOR_TEST,
CustomTestCase,
popen_launch_server,
)
register_cuda_ci(est_time=370, stage="base-c", runner_config="4-gpu-b200")
QWEN35_FP4_MODEL = "nvidia/Qwen3.5-397B-A17B-NVFP4"
ACC_THRESHOLDS = {QWEN35_FP4_MODEL: {"gsm8k": 0.95}}
class TestQwen35FP4MTPV2(ReasoningTokenUsageMixin, CustomTestCase):
reasoning_parser_name = "qwen3"
@classmethod
def setUpClass(cls):
cls.model = QWEN35_FP4_MODEL
cls.base_url = DEFAULT_URL_FOR_TEST
cls.init_reasoning_token_verifier()
cls.process = popen_launch_server(
cls.model,
cls.base_url,
timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
other_args=[
"--tp-size",
"4",
"--chunked-prefill-size",
"2048",
"--mamba-scheduler-strategy",
"extra_buffer",
"--mamba-track-interval",
"128",
"--mamba-ssm-dtype",
"bfloat16",
"--max-running-requests",
"128",
"--reasoning-parser",
"qwen3",
"--attention-backend",
"trtllm_mha",
"--quantization",
"modelopt_fp4",
"--speculative-algorithm",
"NEXTN",
"--speculative-num-steps",
"3",
"--speculative-eagle-topk",
"1",
"--speculative-num-draft-tokens",
"4",
"--mem-fraction-static",
"0.8",
"--model-loader-extra-config",
'{"enable_multithread_load": true,"num_threads": 64}',
],
)
@classmethod
def tearDownClass(cls):
kill_process_tree(cls.process.pid)
def test_gsm8k(self):
args = SimpleNamespace(
model=self.model,
eval_name="gsm8k",
num_shots=5,
num_examples=200,
max_tokens=16000,
num_threads=128,
repeat=1,
temperature=0.6,
top_p=0.95,
top_k=20,
base_url=self.base_url,
host="http://127.0.0.1",
port=int(self.base_url.split(":")[-1]),
)
metrics = run_eval(args)
print(f"{metrics=}")
self.assertGreaterEqual(metrics["score"], ACC_THRESHOLDS[self.model]["gsm8k"])
server_info = requests.get(self.base_url + "/server_info")
avg_spec_accept_length = server_info.json()["internal_states"][0][
"avg_spec_accept_length"
]
print(f"{avg_spec_accept_length=}")
self.assertGreater(avg_spec_accept_length, 3.3)
if __name__ == "__main__":
unittest.main()
test/registered/8-gpu-models/test_deepseek_v3_mtp.py
-110
View File
@@ -1,110 +0,0 @@
import unittest
from types import SimpleNamespace
import requests
from sglang.srt.utils import kill_process_tree
from sglang.test.ci.ci_register import register_cuda_ci
from sglang.test.run_eval import run_eval
from sglang.test.send_one import BenchArgs, send_one_prompt
from sglang.test.test_utils import (
DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
DEFAULT_URL_FOR_TEST,
CustomTestCase,
is_in_amd_ci,
is_in_ci,
popen_launch_server,
write_github_step_summary,
)
register_cuda_ci(est_time=300, stage="base-c", runner_config="8-gpu-h200")
FULL_DEEPSEEK_V3_MODEL_PATH = "deepseek-ai/DeepSeek-V3-0324"
class TestDeepseekV3MTP(CustomTestCase):
@classmethod
def setUpClass(cls):
cls.model = FULL_DEEPSEEK_V3_MODEL_PATH
cls.base_url = DEFAULT_URL_FOR_TEST
other_args = [
"--tp",
"8",
"--trust-remote-code",
"--speculative-algorithm",
"EAGLE",
"--speculative-num-steps",
"3",
"--speculative-eagle-topk",
"1",
"--speculative-num-draft-tokens",
"4",
"--model-loader-extra-config",
'{"enable_multithread_load": true, "num_threads": 64}',
]
if not is_in_amd_ci():
other_args += ["--mem-frac", "0.7"]
cls.process = popen_launch_server(
cls.model,
cls.base_url,
timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH * 5,
other_args=other_args,
)
@classmethod
def tearDownClass(cls):
kill_process_tree(cls.process.pid)
def test_a_gsm8k(
self,
): # Append an "a" to make this test run first (alphabetically) to warm up the server
requests.get(self.base_url + "/flush_cache")
args = SimpleNamespace(
base_url=self.base_url,
model=self.model,
eval_name="gsm8k",
api="completion",
max_tokens=512,
num_examples=200,
num_threads=128,
)
metrics = run_eval(args)
print(f"{metrics=}")
server_info = requests.get(self.base_url + "/server_info")
avg_spec_accept_length = server_info.json()["internal_states"][0][
"avg_spec_accept_length"
]
print(f"{avg_spec_accept_length=}")
if is_in_ci():
write_github_step_summary(
f"### test_gsm8k (deepseek-v3 mtp)\n"
f'{metrics["score"]=:.3f}\n'
f"{avg_spec_accept_length=:.2f}\n"
)
self.assertGreater(metrics["score"], 0.935)
self.assertGreater(avg_spec_accept_length, 2.8)
def test_bs_1_speed(self):
args = BenchArgs(port=int(self.base_url.split(":")[-1]), max_new_tokens=2048)
acc_length, speed = send_one_prompt(args)
print(f"{acc_length=:.2f} {speed=:.2f}")
if is_in_ci():
write_github_step_summary(
f"### test_bs_1_speed (deepseek-v3 mtp)\n"
f"{acc_length=:.2f}\n"
f"{speed=:.2f} token/s\n"
)
self.assertGreater(acc_length, 2.8)
if is_in_amd_ci():
self.assertGreater(speed, 15)
else:
self.assertGreater(speed, 130)
if __name__ == "__main__":
unittest.main()
test/registered/4-gpu-models/test_qwen35_hicache.py → test/registered/hicache/test_qwen35_hicache.py
View File
test/registered/models_e2e/test_deepseek_v32_fp4_mtp_dp.py
+68
View File
@@ -0,0 +1,68 @@
import unittest
from sglang.test.ci.ci_register import register_cuda_ci
from sglang.test.kits.eval_accuracy_kit import GSM8KMixin
from sglang.test.send_one import BenchArgs, send_one_prompt
from sglang.test.server_fixtures.default_fixture import DefaultServerBase
from sglang.test.test_utils import is_in_ci, write_github_step_summary
register_cuda_ci(est_time=400, stage="base-c", runner_config="4-gpu-b200")
DSV32_FP4_MODEL = "nvidia/DeepSeek-V3.2-NVFP4"
class TestDeepseekV32FP4DPSpec(GSM8KMixin, DefaultServerBase):
model = DSV32_FP4_MODEL
timeout = 1200
other_args = [
"--tp",
"4",
"--dp",
"4",
"--enable-dp-attention",
"--attention-backend",
"dsa",
"--moe-runner-backend",
"flashinfer_trtllm",
"--quantization",
"modelopt_fp4",
"--tool-call-parser",
"deepseekv32",
"--reasoning-parser",
"deepseek-v3",
"--speculative-algorithm",
"EAGLE",
"--speculative-num-steps",
"3",
"--speculative-eagle-topk",
"1",
"--speculative-num-draft-tokens",
"4",
"--model-loader-extra-config",
'{"enable_multithread_load": true,"num_threads": 64}',
]
gsm8k_accuracy_thres = 0.93
gsm8k_num_questions = 500
gsm8k_num_threads = 500
gsm8k_num_shots = 20
gsm8k_accept_length_thres = 2.7
def test_z_bs_1_speed(self):
args = BenchArgs(port=int(self.base_url.split(":")[-1]), max_new_tokens=2048)
acc_length, speed = send_one_prompt(args)
print(f"{acc_length=:.2f} {speed=:.2f}")
if is_in_ci():
write_github_step_summary(
f"### test_bs_1_speed (deepseek-v32 mtp dp)\n"
f"{acc_length=:.2f}\n"
f"{speed=:.2f} token/s\n"
)
self.assertGreater(acc_length, 2.7)
self.assertGreater(speed, 90)
if __name__ == "__main__":
unittest.main()
test/registered/models_e2e/test_deepseek_v32_fp4_mtp_tp.py
+65
View File
@@ -0,0 +1,65 @@
import unittest
from sglang.test.ci.ci_register import register_cuda_ci
from sglang.test.kits.eval_accuracy_kit import GSM8KMixin
from sglang.test.send_one import BenchArgs, send_one_prompt
from sglang.test.server_fixtures.default_fixture import DefaultServerBase
from sglang.test.test_utils import is_in_ci, write_github_step_summary
register_cuda_ci(est_time=400, stage="base-c", runner_config="4-gpu-b200")
DSV32_FP4_MODEL = "nvidia/DeepSeek-V3.2-NVFP4"
class TestDeepseekV32FP4TPSpec(GSM8KMixin, DefaultServerBase):
model = DSV32_FP4_MODEL
timeout = 1200
other_args = [
"--tp",
"4",
"--attention-backend",
"dsa",
"--moe-runner-backend",
"flashinfer_trtllm",
"--quantization",
"modelopt_fp4",
"--tool-call-parser",
"deepseekv32",
"--reasoning-parser",
"deepseek-v3",
"--speculative-algorithm",
"EAGLE",
"--speculative-num-steps",
"3",
"--speculative-eagle-topk",
"1",
"--speculative-num-draft-tokens",
"4",
"--model-loader-extra-config",
'{"enable_multithread_load": true,"num_threads": 64}',
]
gsm8k_accuracy_thres = 0.93
gsm8k_num_questions = 500
gsm8k_num_threads = 500
gsm8k_num_shots = 20
gsm8k_accept_length_thres = 2.7
def test_z_bs_1_speed(self):
args = BenchArgs(port=int(self.base_url.split(":")[-1]), max_new_tokens=2048)
acc_length, speed = send_one_prompt(args)
print(f"{acc_length=:.2f} {speed=:.2f}")
if is_in_ci():
write_github_step_summary(
f"### test_bs_1_speed (deepseek-v32 mtp tp)\n"
f"{acc_length=:.2f}\n"
f"{speed=:.2f} token/s\n"
)
self.assertGreater(acc_length, 2.7)
self.assertGreater(speed, 150)
if __name__ == "__main__":
unittest.main()
test/registered/models_e2e/test_deepseek_v3_fp4.py
+41
View File
@@ -0,0 +1,41 @@
import unittest
from sglang.test.ci.ci_register import register_cuda_ci
from sglang.test.kits.eval_accuracy_kit import GSM8KMixin
from sglang.test.server_fixtures.default_fixture import DefaultServerBase
# Per-commit: SymmetricMemory variant only.
# - TestDeepseekV3FP4 (TRTLLM) archived to test/manual/quant/test_deepseek_v3_fp4_4gpu_trtllm.py
# - TestDeepseekV3FP4CutlassMoE moved to test_deepseek_v3_fp4_4gpu_extra.py
register_cuda_ci(est_time=960, stage="base-c", runner_config="4-gpu-b200")
FULL_DEEPSEEK_V3_FP4_MODEL_PATH = "nvidia/DeepSeek-V3-0324-FP4"
class TestDeepseekV3FP4SymmetricMemory(GSM8KMixin, DefaultServerBase):
model = FULL_DEEPSEEK_V3_FP4_MODEL_PATH
timeout = 1200
other_args = [
"--tp",
"4",
"--attention-backend",
"trtllm_mla",
"--moe-runner-backend",
"flashinfer_trtllm",
"--quantization",
"modelopt_fp4",
"--kv-cache-dtype",
"fp8_e4m3",
"--model-loader-extra-config",
'{"enable_multithread_load": true,"num_threads": 64}',
"--enable-symm-mem",
]
gsm8k_accuracy_thres = 0.93
gsm8k_num_questions = 1319
gsm8k_num_threads = 1319
gsm8k_num_shots = 8
if __name__ == "__main__":
unittest.main()
test/registered/models_e2e/test_deepseek_v3_mtp.py
+67
View File
@@ -0,0 +1,67 @@
import unittest
from sglang.test.ci.ci_register import register_cuda_ci
from sglang.test.kits.eval_accuracy_kit import GSM8KMixin
from sglang.test.send_one import BenchArgs, send_one_prompt
from sglang.test.server_fixtures.default_fixture import DefaultServerBase
from sglang.test.test_utils import (
DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
is_in_amd_ci,
is_in_ci,
write_github_step_summary,
)
register_cuda_ci(est_time=300, stage="base-c", runner_config="8-gpu-h200")
FULL_DEEPSEEK_V3_MODEL_PATH = "deepseek-ai/DeepSeek-V3-0324"
_OTHER_ARGS = [
"--tp",
"8",
"--trust-remote-code",
"--speculative-algorithm",
"EAGLE",
"--speculative-num-steps",
"3",
"--speculative-eagle-topk",
"1",
"--speculative-num-draft-tokens",
"4",
"--model-loader-extra-config",
'{"enable_multithread_load": true, "num_threads": 64}',
]
if not is_in_amd_ci():
_OTHER_ARGS += ["--mem-frac", "0.7"]
class TestDeepseekV3MTP(GSM8KMixin, DefaultServerBase):
model = FULL_DEEPSEEK_V3_MODEL_PATH
other_args = _OTHER_ARGS
timeout = DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH * 5
gsm8k_accuracy_thres = 0.935
gsm8k_accept_length_thres = 2.8
# `test_z_bs_1_speed` runs after `test_gsm8k` (alphabetical) so it
# measures steady-state speed on a warmed server.
def test_z_bs_1_speed(self):
args = BenchArgs(port=int(self.base_url.split(":")[-1]), max_new_tokens=2048)
acc_length, speed = send_one_prompt(args)
print(f"{acc_length=:.2f} {speed=:.2f}")
if is_in_ci():
write_github_step_summary(
f"### test_bs_1_speed (deepseek-v3 mtp)\n"
f"{acc_length=:.2f}\n"
f"{speed=:.2f} token/s\n"
)
self.assertGreater(acc_length, 2.8)
if is_in_amd_ci():
self.assertGreater(speed, 15)
else:
self.assertGreater(speed, 130)
if __name__ == "__main__":
unittest.main()
test/registered/models_e2e/test_gpt_oss_4gpu_bf16.py
+23
View File
@@ -0,0 +1,23 @@
import unittest
from sglang.test.ci.ci_register import register_cuda_ci
from sglang.test.gpt_oss_common import BaseTestGptOss
register_cuda_ci(est_time=220, stage="base-c", runner_config="4-gpu-h100")
register_cuda_ci(est_time=220, stage="base-c", runner_config="4-gpu-b200")
class TestGptOss4GpuBf16(BaseTestGptOss):
def test_bf16_120b(self):
self.run_test(
model_variant="120b",
quantization="bf16",
expected_score_of_reasoning_effort={
"low": 0.58,
},
other_args=["--tp", "4", "--cuda-graph-max-bs", "200"],
)
if __name__ == "__main__":
unittest.main()
test/registered/4-gpu-models/test_gpt_oss_4gpu.py → test/registered/models_e2e/test_gpt_oss_4gpu_mxfp4.py
+3 -13
View File
@@ -3,21 +3,11 @@ import unittest
from sglang.test.ci.ci_register import register_cuda_ci
from sglang.test.gpt_oss_common import BaseTestGptOss
register_cuda_ci(est_time=392, stage="base-c", runner_config="4-gpu-h100")
register_cuda_ci(est_time=350, stage="base-c", runner_config="4-gpu-b200")
register_cuda_ci(est_time=220, stage="base-c", runner_config="4-gpu-h100")
register_cuda_ci(est_time=220, stage="base-c", runner_config="4-gpu-b200")
class TestGptOss4Gpu(BaseTestGptOss):
def test_bf16_120b(self):
self.run_test(
model_variant="120b",
quantization="bf16",
expected_score_of_reasoning_effort={
"low": 0.58,
},
other_args=["--tp", "4", "--cuda-graph-max-bs", "200"],
)
class TestGptOss4GpuMxfp4(BaseTestGptOss):
def test_mxfp4_120b(self):
self.run_test(
model_variant="120b",
test/registered/models_e2e/test_mimo_v2.py
+47
View File
@@ -0,0 +1,47 @@
import unittest
from sglang.test.ci.ci_register import register_cuda_ci
from sglang.test.kits.eval_accuracy_kit import GSM8KMixin
from sglang.test.server_fixtures.mmmu_fixture import MMMUServerBase
register_cuda_ci(est_time=400, stage="base-c", runner_config="8-gpu-h200")
MIMO_V2_MODEL = "XiaomiMiMo/MiMo-V2.5"
MIMO_V2_OTHER_ARGS = [
"--tp",
"8",
"--dp",
"2",
"--enable-dp-attention",
"--mm-enable-dp-encoder",
"--attention-backend",
"fa3",
"--mm-attention-backend",
"fa3",
"--reasoning-parser",
"mimo",
]
MIMO_V2_MTP_OTHER_ARGS = MIMO_V2_OTHER_ARGS + [
"--speculative-algorithm",
"EAGLE",
"--speculative-num-steps",
"3",
"--speculative-eagle-topk",
"1",
"--speculative-num-draft-tokens",
"4",
"--enable-multi-layer-eagle",
]
class TestMiMoV2(GSM8KMixin, MMMUServerBase):
gsm8k_accuracy_thres = 0.75
gsm8k_accept_length_thres = 2.5
model = MIMO_V2_MODEL
mem_fraction_static = 0.65
server_api_key = None
other_args = MIMO_V2_MTP_OTHER_ARGS
if __name__ == "__main__":
unittest.main()
test/registered/8-gpu-models/test_mimo_models.py → test/registered/models_e2e/test_mimo_v2_flash.py
+1 -39
View File
@@ -4,9 +4,8 @@ from sglang.test.ci.ci_register import register_cuda_ci
from sglang.test.kits.eval_accuracy_kit import GSM8KMixin
from sglang.test.kits.spec_decoding_kit import SpecDecodingMixin
from sglang.test.server_fixtures.default_fixture import DefaultServerBase
from sglang.test.server_fixtures.mmmu_fixture import MMMUServerBase
register_cuda_ci(est_time=500, stage="base-c", runner_config="8-gpu-h200")
register_cuda_ci(est_time=350, stage="base-c", runner_config="8-gpu-h200")
class TestMiMoV2Flash(GSM8KMixin, SpecDecodingMixin, DefaultServerBase):
@@ -49,42 +48,5 @@ class TestMiMoV2Flash(GSM8KMixin, SpecDecodingMixin, DefaultServerBase):
accept_length_thres = 3.2
MIMO_V2_MODEL = "XiaomiMiMo/MiMo-V2.5"
MIMO_V2_OTHER_ARGS = [
"--tp",
"8",
"--dp",
"2",
"--enable-dp-attention",
"--mm-enable-dp-encoder",
"--attention-backend",
"fa3",
"--mm-attention-backend",
"fa3",
"--reasoning-parser",
"mimo",
]
MIMO_V2_MTP_OTHER_ARGS = MIMO_V2_OTHER_ARGS + [
"--speculative-algorithm",
"EAGLE",
"--speculative-num-steps",
"3",
"--speculative-eagle-topk",
"1",
"--speculative-num-draft-tokens",
"4",
"--enable-multi-layer-eagle",
]
class TestMiMoV2(GSM8KMixin, MMMUServerBase):
gsm8k_accuracy_thres = 0.75
gsm8k_accept_length_thres = 2.5
model = MIMO_V2_MODEL
mem_fraction_static = 0.65
server_api_key = None
other_args = MIMO_V2_MTP_OTHER_ARGS
if __name__ == "__main__":
unittest.main()
test/registered/8-gpu-models/test_minimax_m25_basic.py → test/registered/models_e2e/test_minimax_m25_basic.py
View File
test/registered/models_e2e/test_nvidia_nemotron_3_super_bf16.py
+69
View File
@@ -0,0 +1,69 @@
import unittest
from types import SimpleNamespace
from sglang.srt.utils import kill_process_tree
from sglang.test.ci.ci_register import register_cuda_ci
from sglang.test.run_eval import run_eval
from sglang.test.test_utils import (
DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
DEFAULT_URL_FOR_TEST,
CustomTestCase,
popen_launch_server,
)
register_cuda_ci(est_time=200, stage="extra-b", runner_config="8-gpu-h200")
NEMOTRON_3_SUPER_BF16_MODEL = "nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-BF16"
NEMOTRON_3_SUPER_BF16_ARGS = [
"--tp-size",
"8",
"--trust-remote-code",
"--reasoning-parser",
"nemotron_3",
"--tool-call-parser",
"qwen3_coder",
"--disable-radix-cache",
"--model-loader-extra-config",
'{"enable_multithread_load": true, "num_threads": 50}',
]
class TestNvidiaNemotron3SuperBF16(CustomTestCase):
@classmethod
def setUpClass(cls):
cls.model = NEMOTRON_3_SUPER_BF16_MODEL
cls.base_url = DEFAULT_URL_FOR_TEST
cls.process = popen_launch_server(
cls.model,
cls.base_url,
timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
other_args=NEMOTRON_3_SUPER_BF16_ARGS,
)
@classmethod
def tearDownClass(cls):
kill_process_tree(cls.process.pid)
def test_gsm8k(self):
args = SimpleNamespace(
model=self.model,
eval_name="gsm8k",
num_shots=5,
num_examples=200,
max_tokens=16000,
num_threads=200,
repeat=1,
temperature=1.0,
top_p=0.95,
base_url=self.base_url,
host="http://127.0.0.1",
port=int(self.base_url.split(":")[-1]),
)
metrics = run_eval(args)
print(f"{metrics=}")
self.assertGreaterEqual(metrics["score"], 0.96)
if __name__ == "__main__":
unittest.main()
test/registered/8-gpu-models/test_nvidia_nemotron_3_super_bf16.py → test/registered/models_e2e/test_nvidia_nemotron_3_super_bf16_mtp.py
+18 -42
View File
@@ -11,7 +11,7 @@ from sglang.test.test_utils import (
popen_launch_server,
)
register_cuda_ci(est_time=370, stage="extra-b", runner_config="8-gpu-h200")
register_cuda_ci(est_time=200, stage="extra-b", runner_config="8-gpu-h200")
NEMOTRON_3_SUPER_BF16_MODEL = "nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-BF16"
@@ -44,46 +44,6 @@ MTP_ARGS = [
]
def _run_gsm8k(test_case):
args = SimpleNamespace(
model=test_case.model,
eval_name="gsm8k",
num_shots=5,
num_examples=200,
max_tokens=16000,
num_threads=200,
repeat=1,
temperature=1.0,
top_p=0.95,
base_url=test_case.base_url,
host="http://127.0.0.1",
port=int(test_case.base_url.split(":")[-1]),
)
metrics = run_eval(args)
print(f"{metrics=}")
test_case.assertGreaterEqual(metrics["score"], 0.96)
class TestNvidiaNemotron3SuperBF16(CustomTestCase):
@classmethod
def setUpClass(cls):
cls.model = NEMOTRON_3_SUPER_BF16_MODEL
cls.base_url = DEFAULT_URL_FOR_TEST
cls.process = popen_launch_server(
cls.model,
cls.base_url,
timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
other_args=NEMOTRON_3_SUPER_BF16_ARGS,
)
@classmethod
def tearDownClass(cls):
kill_process_tree(cls.process.pid)
def test_gsm8k(self):
_run_gsm8k(self)
class TestNvidiaNemotron3SuperBF16MTP(CustomTestCase):
@classmethod
def setUpClass(cls):
@@ -101,7 +61,23 @@ class TestNvidiaNemotron3SuperBF16MTP(CustomTestCase):
kill_process_tree(cls.process.pid)
def test_gsm8k(self):
_run_gsm8k(self)
args = SimpleNamespace(
model=self.model,
eval_name="gsm8k",
num_shots=5,
num_examples=200,
max_tokens=16000,
num_threads=200,
repeat=1,
temperature=1.0,
top_p=0.95,
base_url=self.base_url,
host="http://127.0.0.1",
port=int(self.base_url.split(":")[-1]),
)
metrics = run_eval(args)
print(f"{metrics=}")
self.assertGreaterEqual(metrics["score"], 0.96)
if __name__ == "__main__":
test/registered/4-gpu-models/test_qwen35_fp4_flashinfer.py → test/registered/models_e2e/test_qwen35_fp4_flashinfer.py
View File
test/registered/4-gpu-models/test_qwen35_fp4_mtp_v2.py → test/registered/models_e2e/test_qwen35_fp4_mtp.py
+1 -1
View File
@@ -21,7 +21,7 @@ QWEN35_FP4_MODEL = "nvidia/Qwen3.5-397B-A17B-NVFP4"
ACC_THRESHOLDS = {QWEN35_FP4_MODEL: {"gsm8k": 0.95}}
class TestQwen35FP4MTPV2(ReasoningTokenUsageMixin, CustomTestCase):
class TestQwen35FP4MTP(ReasoningTokenUsageMixin, CustomTestCase):
reasoning_parser_name = "qwen3"
@classmethod
test/registered/4-gpu-models/test_qwen3_next_models_mtp.py → test/registered/models_e2e/test_qwen3_next_models_mtp.py
View File
test/registered/8-gpu-models/test_step3p5_flash_chain_mtp.py → test/registered/models_e2e/test_step3p5_flash_chain_mtp.py
+33 -78
View File
@@ -1,28 +1,20 @@
import unittest
from types import SimpleNamespace
import numpy as np
import requests
from sglang.srt.environ import envs
from sglang.srt.utils import kill_process_tree
from sglang.test.ci.ci_register import register_cuda_ci
from sglang.test.run_eval import run_eval
from sglang.test.test_utils import (
DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
DEFAULT_URL_FOR_TEST,
CustomTestCase,
is_in_ci,
popen_launch_server,
write_github_step_summary,
)
from sglang.test.kits.eval_accuracy_kit import GSM8KMixin
from sglang.test.server_fixtures.default_fixture import DefaultServerBase
from sglang.test.test_utils import DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH
register_cuda_ci(est_time=480, stage="extra-b", runner_config="8-gpu-h200")
STEP3P5_FLASH_MODEL_PATH = "stepfun-ai/Step-3.5-Flash"
class TestStep3p5FlashChainMTP(CustomTestCase):
class TestStep3p5FlashChainMTP(GSM8KMixin, DefaultServerBase):
"""Chain-style multi-layer EAGLE speculative decoding on Step-3.5-Flash.
Step3p5ForCausalLM auto-enables multi-layer EAGLE and spec v2 when
@@ -31,75 +23,38 @@ class TestStep3p5FlashChainMTP(CustomTestCase):
automatically for the Step3p5MTP draft architecture.
"""
model = STEP3P5_FLASH_MODEL_PATH
timeout = DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH * 3
other_args = [
"--tp",
"8",
"--trust-remote-code",
"--speculative-algorithm",
"EAGLE",
"--speculative-num-steps",
"3",
"--speculative-eagle-topk",
"1",
"--speculative-num-draft-tokens",
"4",
"--attention-backend",
"fa3",
"--enable-multi-layer-eagle",
"--mem-fraction-static",
"0.75",
"--chunked-prefill-size",
"4096",
"--model-loader-extra-config",
'{"enable_multithread_load": true, "num_threads": 64}',
]
gsm8k_accuracy_thres = 0.83
gsm8k_accept_length_thres = 2.6
@classmethod
def setUpClass(cls):
cls.model = STEP3P5_FLASH_MODEL_PATH
cls.base_url = DEFAULT_URL_FOR_TEST
other_args = [
"--tp",
"8",
"--trust-remote-code",
"--speculative-algorithm",
"EAGLE",
"--speculative-num-steps",
"3",
"--speculative-eagle-topk",
"1",
"--speculative-num-draft-tokens",
"4",
"--attention-backend",
"fa3",
"--enable-multi-layer-eagle",
"--mem-fraction-static",
"0.75",
"--chunked-prefill-size",
"4096",
"--model-loader-extra-config",
'{"enable_multithread_load": true, "num_threads": 64}',
]
with envs.SGLANG_ENABLE_SPEC_V2.override(True):
cls.process = popen_launch_server(
cls.model,
cls.base_url,
timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH * 3,
other_args=other_args,
)
@classmethod
def tearDownClass(cls):
if hasattr(cls, "process") and cls.process:
kill_process_tree(cls.process.pid)
def test_gsm8k(self):
requests.get(self.base_url + "/flush_cache")
args = SimpleNamespace(
base_url=self.base_url,
model=self.model,
eval_name="gsm8k",
api="completion",
max_tokens=512,
num_examples=200,
num_threads=128,
)
metrics = run_eval(args)
print(f"{metrics=}")
server_info = requests.get(self.base_url + "/server_info")
avg_spec_accept_length = server_info.json()["internal_states"][0][
"avg_spec_accept_length"
]
print(f"{avg_spec_accept_length=}")
print(f"{metrics=}")
if is_in_ci():
write_github_step_summary(
f"### test_gsm8k (step-3.5-flash chain mtp)\n"
f'{metrics["score"]=:.3f}\n'
f"{avg_spec_accept_length=:.2f}\n"
)
self.assertGreater(metrics["score"], 0.83)
self.assertGreater(avg_spec_accept_length, 2.6)
super().setUpClass()
def test_logprob_spec_v2_match(self):
"""Verify spec v2 decode logprobs match prefill scoring logprobs.
test/registered/quant/test_deepseek_v32_fp4_mtp_4gpu.py
-212
View File
@@ -1,212 +0,0 @@
import unittest
from types import SimpleNamespace
import requests
from sglang.srt.utils import kill_process_tree
from sglang.test.ci.ci_register import register_cuda_ci
from sglang.test.run_eval import run_eval
from sglang.test.send_one import BenchArgs, send_one_prompt
from sglang.test.test_utils import (
DEFAULT_URL_FOR_TEST,
CustomTestCase,
is_in_ci,
popen_launch_server,
write_github_step_summary,
)
register_cuda_ci(
est_time=690,
stage="base-c",
runner_config="4-gpu-b200",
)
FULL_DEEPSEEK_V3_FP4_MODEL_PATH = "nvidia/DeepSeek-V3.2-NVFP4"
SERVER_LAUNCH_TIMEOUT = 1200
class TestDeepseekV32FP4DPSpecV2(CustomTestCase):
@classmethod
def setUpClass(cls):
cls.model = FULL_DEEPSEEK_V3_FP4_MODEL_PATH
cls.base_url = DEFAULT_URL_FOR_TEST
other_args = [
"--tp",
"4",
"--dp",
"4",
"--enable-dp-attention",
"--attention-backend",
"dsa",
"--moe-runner-backend",
"flashinfer_trtllm",
"--quantization",
"modelopt_fp4",
"--tool-call-parser",
"deepseekv32",
"--reasoning-parser",
"deepseek-v3",
"--speculative-algorithm",
"EAGLE",
"--speculative-num-steps",
"3",
"--speculative-eagle-topk",
"1",
"--speculative-num-draft-tokens",
"4",
"--model-loader-extra-config",
'{"enable_multithread_load": true,"num_threads": 64}',
]
cls.process = popen_launch_server(
cls.model,
cls.base_url,
timeout=SERVER_LAUNCH_TIMEOUT,
other_args=other_args,
)
@classmethod
def tearDownClass(cls):
kill_process_tree(cls.process.pid)
def test_a_gsm8k(
self,
): # Append an "a" to make this test run first (alphabetically) to warm up the server
requests.get(self.base_url + "/flush_cache")
args = SimpleNamespace(
base_url=self.base_url,
model=self.model,
eval_name="gsm8k",
api="completion",
max_tokens=512,
num_examples=500,
num_threads=500,
num_shots=20,
)
metrics = run_eval(args)
print(f"{metrics=}")
server_info = requests.get(self.base_url + "/server_info")
avg_spec_accept_length = server_info.json()["internal_states"][0][
"avg_spec_accept_length"
]
print(f"{avg_spec_accept_length=}")
if is_in_ci():
write_github_step_summary(
f"### test_gsm8k (deepseek-v32 mtp)\n"
f'{metrics["score"]=:.3f}\n'
f"{avg_spec_accept_length=:.2f}\n"
)
self.assertGreater(metrics["score"], 0.93)
self.assertGreater(avg_spec_accept_length, 2.7)
def test_bs_1_speed(self):
args = BenchArgs(port=int(self.base_url.split(":")[-1]), max_new_tokens=2048)
acc_length, speed = send_one_prompt(args)
print(f"{acc_length=:.2f} {speed=:.2f}")
if is_in_ci():
write_github_step_summary(
f"### test_bs_1_speed (deepseek-v32 mtp)\n"
f"{acc_length=:.2f}\n"
f"{speed=:.2f} token/s\n"
)
self.assertGreater(acc_length, 2.7)
self.assertGreater(speed, 90)
class TestDeepseekV32FP4TPSpecV2(CustomTestCase):
@classmethod
def setUpClass(cls):
cls.model = FULL_DEEPSEEK_V3_FP4_MODEL_PATH
cls.base_url = DEFAULT_URL_FOR_TEST
other_args = [
"--tp",
"4",
"--attention-backend",
"dsa",
"--moe-runner-backend",
"flashinfer_trtllm",
"--quantization",
"modelopt_fp4",
"--tool-call-parser",
"deepseekv32",
"--reasoning-parser",
"deepseek-v3",
"--speculative-algorithm",
"EAGLE",
"--speculative-num-steps",
"3",
"--speculative-eagle-topk",
"1",
"--speculative-num-draft-tokens",
"4",
"--model-loader-extra-config",
'{"enable_multithread_load": true,"num_threads": 64}',
]
cls.process = popen_launch_server(
cls.model,
cls.base_url,
timeout=SERVER_LAUNCH_TIMEOUT,
other_args=other_args,
)
@classmethod
def tearDownClass(cls):
kill_process_tree(cls.process.pid)
def test_a_gsm8k(
self,
): # Append an "a" to make this test run first (alphabetically) to warm up the server
requests.get(self.base_url + "/flush_cache")
args = SimpleNamespace(
base_url=self.base_url,
model=self.model,
eval_name="gsm8k",
api="completion",
max_tokens=512,
num_examples=500,
num_threads=500,
num_shots=20,
)
metrics = run_eval(args)
print(f"{metrics=}")
server_info = requests.get(self.base_url + "/server_info")
avg_spec_accept_length = server_info.json()["internal_states"][0][
"avg_spec_accept_length"
]
print(f"{avg_spec_accept_length=}")
if is_in_ci():
write_github_step_summary(
f"### test_gsm8k (deepseek-v32 mtp)\n"
f'{metrics["score"]=:.3f}\n'
f"{avg_spec_accept_length=:.2f}\n"
)
self.assertGreater(metrics["score"], 0.93)
self.assertGreater(avg_spec_accept_length, 2.7)
def test_bs_1_speed(self):
args = BenchArgs(port=int(self.base_url.split(":")[-1]), max_new_tokens=2048)
acc_length, speed = send_one_prompt(args)
print(f"{acc_length=:.2f} {speed=:.2f}")
if is_in_ci():
write_github_step_summary(
f"### test_bs_1_speed (deepseek-v32 mtp)\n"
f"{acc_length=:.2f}\n"
f"{speed=:.2f} token/s\n"
)
self.assertGreater(acc_length, 2.7)
self.assertGreater(speed, 150)
if __name__ == "__main__":
unittest.main()
test/registered/quant/test_deepseek_v3_fp4_4gpu.py
-80
View File
@@ -1,80 +0,0 @@
import unittest
from types import SimpleNamespace
from sglang.srt.utils import kill_process_tree
from sglang.test.ci.ci_register import register_cuda_ci
from sglang.test.run_eval import run_eval
from sglang.test.test_utils import (
DEFAULT_URL_FOR_TEST,
CustomTestCase,
is_in_ci,
popen_launch_server,
write_github_step_summary,
)
# Per-commit: SymmetricMemory variant only.
# - TestDeepseekV3FP4 (TRTLLM) archived to test/manual/quant/test_deepseek_v3_fp4_4gpu_trtllm.py
# - TestDeepseekV3FP4CutlassMoE moved to test_deepseek_v3_fp4_4gpu_extra.py
register_cuda_ci(est_time=960, stage="base-c", runner_config="4-gpu-b200")
FULL_DEEPSEEK_V3_FP4_MODEL_PATH = "nvidia/DeepSeek-V3-0324-FP4"
SERVER_LAUNCH_TIMEOUT = 1200
class TestDeepseekV3FP4SymmetricMemory(CustomTestCase):
@classmethod
def setUpClass(cls):
cls.model = FULL_DEEPSEEK_V3_FP4_MODEL_PATH
cls.base_url = DEFAULT_URL_FOR_TEST
other_args = [
"--tp",
"4",
"--attention-backend",
"trtllm_mla",
"--moe-runner-backend",
"flashinfer_trtllm",
"--quantization",
"modelopt_fp4",
"--kv-cache-dtype",
"fp8_e4m3",
"--model-loader-extra-config",
'{"enable_multithread_load": true,"num_threads": 64}',
"--enable-symm-mem",
]
cls.process = popen_launch_server(
cls.model,
cls.base_url,
timeout=SERVER_LAUNCH_TIMEOUT,
other_args=other_args,
)
@classmethod
def tearDownClass(cls):
kill_process_tree(cls.process.pid)
def test_a_gsm8k(
self,
): # Append an "a" to make this test run first (alphabetically) to warm up the server
args = SimpleNamespace(
base_url=self.base_url,
model=self.model,
eval_name="gsm8k",
api="completion",
max_tokens=512,
num_examples=1319,
num_threads=1319,
num_shots=8,
)
metrics = run_eval(args)
print(f"{metrics=}")
if is_in_ci():
write_github_step_summary(
f"### test_gsm8k (deepseek-v3-fp4)\n" f'{metrics["score"]=:.3f}\n'
)
self.assertGreater(metrics["score"], 0.93)
if __name__ == "__main__":
unittest.main()
test/registered/radix_cache/test_unified_radix_cache_kl_full.py
+50
View File
@@ -0,0 +1,50 @@
import unittest
from sglang.srt.utils import kill_process_tree
from sglang.test.ci.ci_register import register_cuda_ci
from sglang.test.kits.unified_radix_cache_kit import UnifiedRadixTreeTestMixin
from sglang.test.kl_multiturn_utils import get_input_ids
from sglang.test.test_utils import (
DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
DEFAULT_URL_FOR_TEST,
CustomTestCase,
popen_launch_server,
)
register_cuda_ci(est_time=250, stage="base-c", runner_config="4-gpu-h100")
FULL_MODEL = "Qwen/Qwen3-32B"
class TestUnifiedFullRadixCache(UnifiedRadixTreeTestMixin, CustomTestCase):
"""Full attention."""
kl_threshold = 0.0025
@classmethod
def setUpClass(cls):
cls.model = FULL_MODEL
cls.base_url = DEFAULT_URL_FOR_TEST
cls.process = popen_launch_server(
cls.model,
cls.base_url,
timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
other_args=[
"--tp-size",
"4",
"--mem-fraction-static",
"0.80",
"--page-size",
"64",
],
env={"SGLANG_ENABLE_UNIFIED_RADIX_TREE": "1"},
)
cls.input_ids = get_input_ids(cls.model, num_samples=18)
@classmethod
def tearDownClass(cls):
kill_process_tree(cls.process.pid)
if __name__ == "__main__":
unittest.main()
test/registered/radix_cache/test_unified_radix_cache_kl_hicache.py
+1 -2
View File
@@ -1,9 +1,8 @@
import unittest
from test_unified_radix_cache_kl import UnifiedRadixTreeTestMixin
from sglang.srt.utils import kill_process_tree
from sglang.test.ci.ci_register import register_cuda_ci
from sglang.test.kits.unified_radix_cache_kit import UnifiedRadixTreeTestMixin
from sglang.test.kl_multiturn_utils import (
get_input_ids,
make_mamba_decode_assert,
test/registered/radix_cache/test_unified_radix_cache_kl_mamba.py
+66
View File
@@ -0,0 +1,66 @@
import unittest
from sglang.srt.utils import kill_process_tree
from sglang.test.ci.ci_register import register_cuda_ci
from sglang.test.kits.unified_radix_cache_kit import UnifiedRadixTreeTestMixin
from sglang.test.kl_multiturn_utils import (
get_input_ids,
make_mamba_decode_assert,
make_mamba_prefill_assert,
)
from sglang.test.test_utils import (
DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
DEFAULT_URL_FOR_TEST,
CustomTestCase,
popen_launch_server,
)
register_cuda_ci(est_time=250, stage="base-c", runner_config="4-gpu-h100")
MAMBA_MODEL = "Qwen/Qwen3-Next-80B-A3B-Instruct"
MAMBA_CHUNK_SIZE = 64
MAMBA_TRACK_INTERVAL = 128
class TestUnifiedMambaRadixCache(UnifiedRadixTreeTestMixin, CustomTestCase):
"""Mamba hybrid + UnifiedRadixCache."""
kl_threshold = 0.003
prefill_cache_assert = staticmethod(
make_mamba_prefill_assert(chunk_size=MAMBA_CHUNK_SIZE)
)
decode_cache_assert = staticmethod(
make_mamba_decode_assert(track_interval=MAMBA_TRACK_INTERVAL)
)
@classmethod
def setUpClass(cls):
cls.model = MAMBA_MODEL
cls.base_url = DEFAULT_URL_FOR_TEST
cls.process = popen_launch_server(
cls.model,
cls.base_url,
timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
other_args=[
"--tp-size",
"4",
"--chunked-prefill-size",
"2048",
"--mem-fraction-static",
"0.85",
"--mamba-scheduler-strategy",
"extra_buffer",
"--mamba-track-interval",
str(MAMBA_TRACK_INTERVAL),
],
env={"SGLANG_ENABLE_UNIFIED_RADIX_TREE": "1"},
)
cls.input_ids = get_input_ids(cls.model, num_samples=18)
@classmethod
def tearDownClass(cls):
kill_process_tree(cls.process.pid)
if __name__ == "__main__":
unittest.main()
test/registered/radix_cache/test_unified_radix_cache_kl_swa.py
+56
View File
@@ -0,0 +1,56 @@
import unittest
from sglang.srt.utils import kill_process_tree
from sglang.test.ci.ci_register import register_cuda_ci
from sglang.test.kits.unified_radix_cache_kit import UnifiedRadixTreeTestMixin
from sglang.test.kl_multiturn_utils import get_input_ids
from sglang.test.test_utils import (
DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
DEFAULT_URL_FOR_TEST,
CustomTestCase,
is_in_ci,
popen_launch_server,
)
register_cuda_ci(est_time=250, stage="base-c", runner_config="4-gpu-h100")
SWA_MODEL = "openai/gpt-oss-20b"
class TestUnifiedSWARadixCache(UnifiedRadixTreeTestMixin, CustomTestCase):
"""SWA hybrid + UnifiedRadixCache."""
kl_threshold = 0.03
gsm8k_threshold = 0.7
mmlu_threshold = 0.7
@unittest.skipIf(is_in_ci(), "SWA model mmlu eval not stable enough")
def test_mmlu(self):
super().test_mmlu()
@classmethod
def setUpClass(cls):
cls.model = SWA_MODEL
cls.base_url = DEFAULT_URL_FOR_TEST
cls.process = popen_launch_server(
cls.model,
cls.base_url,
timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
other_args=[
"--tp-size",
"4",
"--mem-fraction-static",
"0.7",
"--disable-piecewise-cuda-graph",
],
env={"SGLANG_ENABLE_UNIFIED_RADIX_TREE": "1"},
)
cls.input_ids = get_input_ids(cls.model, num_samples=18)
@classmethod
def tearDownClass(cls):
kill_process_tree(cls.process.pid)
if __name__ == "__main__":
unittest.main()