test/registered: cleanup pure model e2e tests (moves, splits, dedup, kit) (#26610)
This commit is contained in:
+1
-133
@@ -1,25 +1,12 @@
|
|||||||
import random
|
import random
|
||||||
import unittest
|
|
||||||
from types import SimpleNamespace
|
from types import SimpleNamespace
|
||||||
from urllib.parse import urlparse
|
from urllib.parse import urlparse
|
||||||
|
|
||||||
from sglang.srt.utils import kill_process_tree
|
|
||||||
from sglang.test.ci.ci_register import register_cuda_ci
|
|
||||||
from sglang.test.kl_multiturn_utils import (
|
from sglang.test.kl_multiturn_utils import (
|
||||||
get_input_ids,
|
|
||||||
make_mamba_decode_assert,
|
|
||||||
make_mamba_prefill_assert,
|
|
||||||
test_input_output_logprobs_match_decode_cache_hit_helper,
|
test_input_output_logprobs_match_decode_cache_hit_helper,
|
||||||
test_input_output_logprobs_match_helper,
|
test_input_output_logprobs_match_helper,
|
||||||
test_input_output_logprobs_match_prefill_cache_hit_helper,
|
test_input_output_logprobs_match_prefill_cache_hit_helper,
|
||||||
)
|
)
|
||||||
from sglang.test.test_utils import (
|
|
||||||
DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
|
|
||||||
DEFAULT_URL_FOR_TEST,
|
|
||||||
CustomTestCase,
|
|
||||||
is_in_ci,
|
|
||||||
popen_launch_server,
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def _random_suffixes(n, length, seed):
|
def _random_suffixes(n, length, seed):
|
||||||
@@ -28,18 +15,8 @@ def _random_suffixes(n, length, seed):
|
|||||||
return [[rng.randint(1, 30000) for _ in range(length)] for _ in range(n)]
|
return [[rng.randint(1, 30000) for _ in range(length)] for _ in range(n)]
|
||||||
|
|
||||||
|
|
||||||
MAMBA_MODEL = "Qwen/Qwen3-Next-80B-A3B-Instruct"
|
|
||||||
MAMBA_CHUNK_SIZE = 64
|
|
||||||
MAMBA_TRACK_INTERVAL = 128
|
|
||||||
|
|
||||||
SWA_MODEL = "openai/gpt-oss-20b"
|
|
||||||
FULL_MODEL = "Qwen/Qwen3-32B"
|
|
||||||
|
|
||||||
register_cuda_ci(est_time=760, stage="base-c", runner_config="4-gpu-h100")
|
|
||||||
|
|
||||||
|
|
||||||
class UnifiedRadixTreeTestMixin:
|
class UnifiedRadixTreeTestMixin:
|
||||||
"""Mixin: gsm8k、mmlu and multi-turn KL tests with multi-branch interleaving."""
|
"""Mixin: gsm8k, mmlu and multi-turn KL tests with multi-branch interleaving."""
|
||||||
|
|
||||||
kl_threshold: float = 0.003
|
kl_threshold: float = 0.003
|
||||||
max_new_tokens: int = 512
|
max_new_tokens: int = 512
|
||||||
@@ -168,112 +145,3 @@ class UnifiedRadixTreeTestMixin:
|
|||||||
request_batch_size=self.decode_hit_request_batch_size,
|
request_batch_size=self.decode_hit_request_batch_size,
|
||||||
inter_batch_delay_s=self.decode_hit_inter_batch_delay_s,
|
inter_batch_delay_s=self.decode_hit_inter_batch_delay_s,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
class TestUnifiedFullRadixCache(UnifiedRadixTreeTestMixin, CustomTestCase):
|
|
||||||
"""Full attention."""
|
|
||||||
|
|
||||||
kl_threshold = 0.0025
|
|
||||||
|
|
||||||
@classmethod
|
|
||||||
def setUpClass(cls):
|
|
||||||
cls.model = FULL_MODEL
|
|
||||||
cls.base_url = DEFAULT_URL_FOR_TEST
|
|
||||||
cls.process = popen_launch_server(
|
|
||||||
cls.model,
|
|
||||||
cls.base_url,
|
|
||||||
timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
|
|
||||||
other_args=[
|
|
||||||
"--tp-size",
|
|
||||||
"4",
|
|
||||||
"--mem-fraction-static",
|
|
||||||
"0.80",
|
|
||||||
"--page-size",
|
|
||||||
"64",
|
|
||||||
],
|
|
||||||
env={"SGLANG_ENABLE_UNIFIED_RADIX_TREE": "1"},
|
|
||||||
)
|
|
||||||
cls.input_ids = get_input_ids(cls.model, num_samples=18)
|
|
||||||
|
|
||||||
@classmethod
|
|
||||||
def tearDownClass(cls):
|
|
||||||
kill_process_tree(cls.process.pid)
|
|
||||||
|
|
||||||
|
|
||||||
class TestUnifiedMambaRadixCache(UnifiedRadixTreeTestMixin, CustomTestCase):
|
|
||||||
"""Mamba hybrid + UnifiedRadixCache."""
|
|
||||||
|
|
||||||
kl_threshold = 0.003
|
|
||||||
prefill_cache_assert = staticmethod(
|
|
||||||
make_mamba_prefill_assert(chunk_size=MAMBA_CHUNK_SIZE)
|
|
||||||
)
|
|
||||||
decode_cache_assert = staticmethod(
|
|
||||||
make_mamba_decode_assert(track_interval=MAMBA_TRACK_INTERVAL)
|
|
||||||
)
|
|
||||||
|
|
||||||
@classmethod
|
|
||||||
def setUpClass(cls):
|
|
||||||
cls.model = MAMBA_MODEL
|
|
||||||
cls.base_url = DEFAULT_URL_FOR_TEST
|
|
||||||
cls.process = popen_launch_server(
|
|
||||||
cls.model,
|
|
||||||
cls.base_url,
|
|
||||||
timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
|
|
||||||
other_args=[
|
|
||||||
"--tp-size",
|
|
||||||
"4",
|
|
||||||
"--chunked-prefill-size",
|
|
||||||
"2048",
|
|
||||||
"--mem-fraction-static",
|
|
||||||
"0.85",
|
|
||||||
"--mamba-scheduler-strategy",
|
|
||||||
"extra_buffer",
|
|
||||||
"--mamba-track-interval",
|
|
||||||
str(MAMBA_TRACK_INTERVAL),
|
|
||||||
],
|
|
||||||
env={"SGLANG_ENABLE_UNIFIED_RADIX_TREE": "1"},
|
|
||||||
)
|
|
||||||
cls.input_ids = get_input_ids(cls.model, num_samples=18)
|
|
||||||
|
|
||||||
@classmethod
|
|
||||||
def tearDownClass(cls):
|
|
||||||
kill_process_tree(cls.process.pid)
|
|
||||||
|
|
||||||
|
|
||||||
class TestUnifiedSWARadixCache(UnifiedRadixTreeTestMixin, CustomTestCase):
|
|
||||||
"""SWA hybrid + UnifiedRadixCache."""
|
|
||||||
|
|
||||||
kl_threshold = 0.03
|
|
||||||
gsm8k_threshold = 0.7
|
|
||||||
mmlu_threshold = 0.7
|
|
||||||
|
|
||||||
@unittest.skipIf(is_in_ci(), "SWA model mmlu eval not stable enough")
|
|
||||||
def test_mmlu(self):
|
|
||||||
super().test_mmlu()
|
|
||||||
|
|
||||||
@classmethod
|
|
||||||
def setUpClass(cls):
|
|
||||||
cls.model = SWA_MODEL
|
|
||||||
cls.base_url = DEFAULT_URL_FOR_TEST
|
|
||||||
cls.process = popen_launch_server(
|
|
||||||
cls.model,
|
|
||||||
cls.base_url,
|
|
||||||
timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
|
|
||||||
other_args=[
|
|
||||||
"--tp-size",
|
|
||||||
"4",
|
|
||||||
"--mem-fraction-static",
|
|
||||||
"0.7",
|
|
||||||
"--disable-piecewise-cuda-graph",
|
|
||||||
],
|
|
||||||
env={"SGLANG_ENABLE_UNIFIED_RADIX_TREE": "1"},
|
|
||||||
)
|
|
||||||
cls.input_ids = get_input_ids(cls.model, num_samples=18)
|
|
||||||
|
|
||||||
@classmethod
|
|
||||||
def tearDownClass(cls):
|
|
||||||
kill_process_tree(cls.process.pid)
|
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
|
||||||
unittest.main()
|
|
||||||
@@ -23,7 +23,6 @@ Test strategy:
|
|||||||
|
|
||||||
import unittest
|
import unittest
|
||||||
|
|
||||||
from test_unified_radix_cache_kl import UnifiedRadixTreeTestMixin
|
|
||||||
from test_unified_radix_cache_kl_hicache import (
|
from test_unified_radix_cache_kl_hicache import (
|
||||||
DSV4_FLASH_LAUNCH_TIMEOUT,
|
DSV4_FLASH_LAUNCH_TIMEOUT,
|
||||||
DSV4_FLASH_MODEL,
|
DSV4_FLASH_MODEL,
|
||||||
@@ -31,6 +30,7 @@ from test_unified_radix_cache_kl_hicache import (
|
|||||||
)
|
)
|
||||||
|
|
||||||
from sglang.srt.utils import kill_process_tree
|
from sglang.srt.utils import kill_process_tree
|
||||||
|
from sglang.test.kits.unified_radix_cache_kit import UnifiedRadixTreeTestMixin
|
||||||
from sglang.test.kl_multiturn_utils import get_input_ids
|
from sglang.test.kl_multiturn_utils import get_input_ids
|
||||||
from sglang.test.test_utils import (
|
from sglang.test.test_utils import (
|
||||||
DEFAULT_URL_FOR_TEST,
|
DEFAULT_URL_FOR_TEST,
|
||||||
|
|||||||
@@ -1,105 +0,0 @@
|
|||||||
import unittest
|
|
||||||
from types import SimpleNamespace
|
|
||||||
|
|
||||||
import requests
|
|
||||||
|
|
||||||
from sglang.srt.utils import kill_process_tree
|
|
||||||
from sglang.test.ci.ci_register import register_cuda_ci
|
|
||||||
from sglang.test.kits.reasoning_kit import ReasoningTokenUsageMixin
|
|
||||||
|
|
||||||
# This eval harness applies the chat_template, which is critical for qwen3.5
|
|
||||||
# to get good accuracy on gsm8k
|
|
||||||
from sglang.test.run_eval import run_eval
|
|
||||||
from sglang.test.test_utils import (
|
|
||||||
DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
|
|
||||||
DEFAULT_URL_FOR_TEST,
|
|
||||||
CustomTestCase,
|
|
||||||
popen_launch_server,
|
|
||||||
)
|
|
||||||
|
|
||||||
register_cuda_ci(est_time=370, stage="base-c", runner_config="4-gpu-b200")
|
|
||||||
|
|
||||||
QWEN35_FP4_MODEL = "nvidia/Qwen3.5-397B-A17B-NVFP4"
|
|
||||||
ACC_THRESHOLDS = {QWEN35_FP4_MODEL: {"gsm8k": 0.95}}
|
|
||||||
|
|
||||||
|
|
||||||
class TestQwen35FP4MTPV2(ReasoningTokenUsageMixin, CustomTestCase):
|
|
||||||
reasoning_parser_name = "qwen3"
|
|
||||||
|
|
||||||
@classmethod
|
|
||||||
def setUpClass(cls):
|
|
||||||
cls.model = QWEN35_FP4_MODEL
|
|
||||||
cls.base_url = DEFAULT_URL_FOR_TEST
|
|
||||||
cls.init_reasoning_token_verifier()
|
|
||||||
cls.process = popen_launch_server(
|
|
||||||
cls.model,
|
|
||||||
cls.base_url,
|
|
||||||
timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
|
|
||||||
other_args=[
|
|
||||||
"--tp-size",
|
|
||||||
"4",
|
|
||||||
"--chunked-prefill-size",
|
|
||||||
"2048",
|
|
||||||
"--mamba-scheduler-strategy",
|
|
||||||
"extra_buffer",
|
|
||||||
"--mamba-track-interval",
|
|
||||||
"128",
|
|
||||||
"--mamba-ssm-dtype",
|
|
||||||
"bfloat16",
|
|
||||||
"--max-running-requests",
|
|
||||||
"128",
|
|
||||||
"--reasoning-parser",
|
|
||||||
"qwen3",
|
|
||||||
"--attention-backend",
|
|
||||||
"trtllm_mha",
|
|
||||||
"--quantization",
|
|
||||||
"modelopt_fp4",
|
|
||||||
"--speculative-algorithm",
|
|
||||||
"NEXTN",
|
|
||||||
"--speculative-num-steps",
|
|
||||||
"3",
|
|
||||||
"--speculative-eagle-topk",
|
|
||||||
"1",
|
|
||||||
"--speculative-num-draft-tokens",
|
|
||||||
"4",
|
|
||||||
"--mem-fraction-static",
|
|
||||||
"0.8",
|
|
||||||
"--model-loader-extra-config",
|
|
||||||
'{"enable_multithread_load": true,"num_threads": 64}',
|
|
||||||
],
|
|
||||||
)
|
|
||||||
|
|
||||||
@classmethod
|
|
||||||
def tearDownClass(cls):
|
|
||||||
kill_process_tree(cls.process.pid)
|
|
||||||
|
|
||||||
def test_gsm8k(self):
|
|
||||||
args = SimpleNamespace(
|
|
||||||
model=self.model,
|
|
||||||
eval_name="gsm8k",
|
|
||||||
num_shots=5,
|
|
||||||
num_examples=200,
|
|
||||||
max_tokens=16000,
|
|
||||||
num_threads=128,
|
|
||||||
repeat=1,
|
|
||||||
temperature=0.6,
|
|
||||||
top_p=0.95,
|
|
||||||
top_k=20,
|
|
||||||
base_url=self.base_url,
|
|
||||||
host="http://127.0.0.1",
|
|
||||||
port=int(self.base_url.split(":")[-1]),
|
|
||||||
)
|
|
||||||
metrics = run_eval(args)
|
|
||||||
print(f"{metrics=}")
|
|
||||||
self.assertGreaterEqual(metrics["score"], ACC_THRESHOLDS[self.model]["gsm8k"])
|
|
||||||
|
|
||||||
server_info = requests.get(self.base_url + "/server_info")
|
|
||||||
avg_spec_accept_length = server_info.json()["internal_states"][0][
|
|
||||||
"avg_spec_accept_length"
|
|
||||||
]
|
|
||||||
print(f"{avg_spec_accept_length=}")
|
|
||||||
self.assertGreater(avg_spec_accept_length, 3.3)
|
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
|
||||||
unittest.main()
|
|
||||||
@@ -1,110 +0,0 @@
|
|||||||
import unittest
|
|
||||||
from types import SimpleNamespace
|
|
||||||
|
|
||||||
import requests
|
|
||||||
|
|
||||||
from sglang.srt.utils import kill_process_tree
|
|
||||||
from sglang.test.ci.ci_register import register_cuda_ci
|
|
||||||
from sglang.test.run_eval import run_eval
|
|
||||||
from sglang.test.send_one import BenchArgs, send_one_prompt
|
|
||||||
from sglang.test.test_utils import (
|
|
||||||
DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
|
|
||||||
DEFAULT_URL_FOR_TEST,
|
|
||||||
CustomTestCase,
|
|
||||||
is_in_amd_ci,
|
|
||||||
is_in_ci,
|
|
||||||
popen_launch_server,
|
|
||||||
write_github_step_summary,
|
|
||||||
)
|
|
||||||
|
|
||||||
register_cuda_ci(est_time=300, stage="base-c", runner_config="8-gpu-h200")
|
|
||||||
|
|
||||||
FULL_DEEPSEEK_V3_MODEL_PATH = "deepseek-ai/DeepSeek-V3-0324"
|
|
||||||
|
|
||||||
|
|
||||||
class TestDeepseekV3MTP(CustomTestCase):
|
|
||||||
@classmethod
|
|
||||||
def setUpClass(cls):
|
|
||||||
cls.model = FULL_DEEPSEEK_V3_MODEL_PATH
|
|
||||||
cls.base_url = DEFAULT_URL_FOR_TEST
|
|
||||||
other_args = [
|
|
||||||
"--tp",
|
|
||||||
"8",
|
|
||||||
"--trust-remote-code",
|
|
||||||
"--speculative-algorithm",
|
|
||||||
"EAGLE",
|
|
||||||
"--speculative-num-steps",
|
|
||||||
"3",
|
|
||||||
"--speculative-eagle-topk",
|
|
||||||
"1",
|
|
||||||
"--speculative-num-draft-tokens",
|
|
||||||
"4",
|
|
||||||
"--model-loader-extra-config",
|
|
||||||
'{"enable_multithread_load": true, "num_threads": 64}',
|
|
||||||
]
|
|
||||||
if not is_in_amd_ci():
|
|
||||||
other_args += ["--mem-frac", "0.7"]
|
|
||||||
cls.process = popen_launch_server(
|
|
||||||
cls.model,
|
|
||||||
cls.base_url,
|
|
||||||
timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH * 5,
|
|
||||||
other_args=other_args,
|
|
||||||
)
|
|
||||||
|
|
||||||
@classmethod
|
|
||||||
def tearDownClass(cls):
|
|
||||||
kill_process_tree(cls.process.pid)
|
|
||||||
|
|
||||||
def test_a_gsm8k(
|
|
||||||
self,
|
|
||||||
): # Append an "a" to make this test run first (alphabetically) to warm up the server
|
|
||||||
requests.get(self.base_url + "/flush_cache")
|
|
||||||
|
|
||||||
args = SimpleNamespace(
|
|
||||||
base_url=self.base_url,
|
|
||||||
model=self.model,
|
|
||||||
eval_name="gsm8k",
|
|
||||||
api="completion",
|
|
||||||
max_tokens=512,
|
|
||||||
num_examples=200,
|
|
||||||
num_threads=128,
|
|
||||||
)
|
|
||||||
metrics = run_eval(args)
|
|
||||||
print(f"{metrics=}")
|
|
||||||
|
|
||||||
server_info = requests.get(self.base_url + "/server_info")
|
|
||||||
avg_spec_accept_length = server_info.json()["internal_states"][0][
|
|
||||||
"avg_spec_accept_length"
|
|
||||||
]
|
|
||||||
print(f"{avg_spec_accept_length=}")
|
|
||||||
|
|
||||||
if is_in_ci():
|
|
||||||
write_github_step_summary(
|
|
||||||
f"### test_gsm8k (deepseek-v3 mtp)\n"
|
|
||||||
f'{metrics["score"]=:.3f}\n'
|
|
||||||
f"{avg_spec_accept_length=:.2f}\n"
|
|
||||||
)
|
|
||||||
self.assertGreater(metrics["score"], 0.935)
|
|
||||||
self.assertGreater(avg_spec_accept_length, 2.8)
|
|
||||||
|
|
||||||
def test_bs_1_speed(self):
|
|
||||||
args = BenchArgs(port=int(self.base_url.split(":")[-1]), max_new_tokens=2048)
|
|
||||||
acc_length, speed = send_one_prompt(args)
|
|
||||||
|
|
||||||
print(f"{acc_length=:.2f} {speed=:.2f}")
|
|
||||||
|
|
||||||
if is_in_ci():
|
|
||||||
write_github_step_summary(
|
|
||||||
f"### test_bs_1_speed (deepseek-v3 mtp)\n"
|
|
||||||
f"{acc_length=:.2f}\n"
|
|
||||||
f"{speed=:.2f} token/s\n"
|
|
||||||
)
|
|
||||||
self.assertGreater(acc_length, 2.8)
|
|
||||||
if is_in_amd_ci():
|
|
||||||
self.assertGreater(speed, 15)
|
|
||||||
else:
|
|
||||||
self.assertGreater(speed, 130)
|
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
|
||||||
unittest.main()
|
|
||||||
@@ -0,0 +1,68 @@
|
|||||||
|
import unittest
|
||||||
|
|
||||||
|
from sglang.test.ci.ci_register import register_cuda_ci
|
||||||
|
from sglang.test.kits.eval_accuracy_kit import GSM8KMixin
|
||||||
|
from sglang.test.send_one import BenchArgs, send_one_prompt
|
||||||
|
from sglang.test.server_fixtures.default_fixture import DefaultServerBase
|
||||||
|
from sglang.test.test_utils import is_in_ci, write_github_step_summary
|
||||||
|
|
||||||
|
register_cuda_ci(est_time=400, stage="base-c", runner_config="4-gpu-b200")
|
||||||
|
|
||||||
|
DSV32_FP4_MODEL = "nvidia/DeepSeek-V3.2-NVFP4"
|
||||||
|
|
||||||
|
|
||||||
|
class TestDeepseekV32FP4DPSpec(GSM8KMixin, DefaultServerBase):
|
||||||
|
model = DSV32_FP4_MODEL
|
||||||
|
timeout = 1200
|
||||||
|
other_args = [
|
||||||
|
"--tp",
|
||||||
|
"4",
|
||||||
|
"--dp",
|
||||||
|
"4",
|
||||||
|
"--enable-dp-attention",
|
||||||
|
"--attention-backend",
|
||||||
|
"dsa",
|
||||||
|
"--moe-runner-backend",
|
||||||
|
"flashinfer_trtllm",
|
||||||
|
"--quantization",
|
||||||
|
"modelopt_fp4",
|
||||||
|
"--tool-call-parser",
|
||||||
|
"deepseekv32",
|
||||||
|
"--reasoning-parser",
|
||||||
|
"deepseek-v3",
|
||||||
|
"--speculative-algorithm",
|
||||||
|
"EAGLE",
|
||||||
|
"--speculative-num-steps",
|
||||||
|
"3",
|
||||||
|
"--speculative-eagle-topk",
|
||||||
|
"1",
|
||||||
|
"--speculative-num-draft-tokens",
|
||||||
|
"4",
|
||||||
|
"--model-loader-extra-config",
|
||||||
|
'{"enable_multithread_load": true,"num_threads": 64}',
|
||||||
|
]
|
||||||
|
|
||||||
|
gsm8k_accuracy_thres = 0.93
|
||||||
|
gsm8k_num_questions = 500
|
||||||
|
gsm8k_num_threads = 500
|
||||||
|
gsm8k_num_shots = 20
|
||||||
|
gsm8k_accept_length_thres = 2.7
|
||||||
|
|
||||||
|
def test_z_bs_1_speed(self):
|
||||||
|
args = BenchArgs(port=int(self.base_url.split(":")[-1]), max_new_tokens=2048)
|
||||||
|
acc_length, speed = send_one_prompt(args)
|
||||||
|
|
||||||
|
print(f"{acc_length=:.2f} {speed=:.2f}")
|
||||||
|
|
||||||
|
if is_in_ci():
|
||||||
|
write_github_step_summary(
|
||||||
|
f"### test_bs_1_speed (deepseek-v32 mtp dp)\n"
|
||||||
|
f"{acc_length=:.2f}\n"
|
||||||
|
f"{speed=:.2f} token/s\n"
|
||||||
|
)
|
||||||
|
self.assertGreater(acc_length, 2.7)
|
||||||
|
self.assertGreater(speed, 90)
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
unittest.main()
|
||||||
@@ -0,0 +1,65 @@
|
|||||||
|
import unittest
|
||||||
|
|
||||||
|
from sglang.test.ci.ci_register import register_cuda_ci
|
||||||
|
from sglang.test.kits.eval_accuracy_kit import GSM8KMixin
|
||||||
|
from sglang.test.send_one import BenchArgs, send_one_prompt
|
||||||
|
from sglang.test.server_fixtures.default_fixture import DefaultServerBase
|
||||||
|
from sglang.test.test_utils import is_in_ci, write_github_step_summary
|
||||||
|
|
||||||
|
register_cuda_ci(est_time=400, stage="base-c", runner_config="4-gpu-b200")
|
||||||
|
|
||||||
|
DSV32_FP4_MODEL = "nvidia/DeepSeek-V3.2-NVFP4"
|
||||||
|
|
||||||
|
|
||||||
|
class TestDeepseekV32FP4TPSpec(GSM8KMixin, DefaultServerBase):
|
||||||
|
model = DSV32_FP4_MODEL
|
||||||
|
timeout = 1200
|
||||||
|
other_args = [
|
||||||
|
"--tp",
|
||||||
|
"4",
|
||||||
|
"--attention-backend",
|
||||||
|
"dsa",
|
||||||
|
"--moe-runner-backend",
|
||||||
|
"flashinfer_trtllm",
|
||||||
|
"--quantization",
|
||||||
|
"modelopt_fp4",
|
||||||
|
"--tool-call-parser",
|
||||||
|
"deepseekv32",
|
||||||
|
"--reasoning-parser",
|
||||||
|
"deepseek-v3",
|
||||||
|
"--speculative-algorithm",
|
||||||
|
"EAGLE",
|
||||||
|
"--speculative-num-steps",
|
||||||
|
"3",
|
||||||
|
"--speculative-eagle-topk",
|
||||||
|
"1",
|
||||||
|
"--speculative-num-draft-tokens",
|
||||||
|
"4",
|
||||||
|
"--model-loader-extra-config",
|
||||||
|
'{"enable_multithread_load": true,"num_threads": 64}',
|
||||||
|
]
|
||||||
|
|
||||||
|
gsm8k_accuracy_thres = 0.93
|
||||||
|
gsm8k_num_questions = 500
|
||||||
|
gsm8k_num_threads = 500
|
||||||
|
gsm8k_num_shots = 20
|
||||||
|
gsm8k_accept_length_thres = 2.7
|
||||||
|
|
||||||
|
def test_z_bs_1_speed(self):
|
||||||
|
args = BenchArgs(port=int(self.base_url.split(":")[-1]), max_new_tokens=2048)
|
||||||
|
acc_length, speed = send_one_prompt(args)
|
||||||
|
|
||||||
|
print(f"{acc_length=:.2f} {speed=:.2f}")
|
||||||
|
|
||||||
|
if is_in_ci():
|
||||||
|
write_github_step_summary(
|
||||||
|
f"### test_bs_1_speed (deepseek-v32 mtp tp)\n"
|
||||||
|
f"{acc_length=:.2f}\n"
|
||||||
|
f"{speed=:.2f} token/s\n"
|
||||||
|
)
|
||||||
|
self.assertGreater(acc_length, 2.7)
|
||||||
|
self.assertGreater(speed, 150)
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
unittest.main()
|
||||||
@@ -0,0 +1,41 @@
|
|||||||
|
import unittest
|
||||||
|
|
||||||
|
from sglang.test.ci.ci_register import register_cuda_ci
|
||||||
|
from sglang.test.kits.eval_accuracy_kit import GSM8KMixin
|
||||||
|
from sglang.test.server_fixtures.default_fixture import DefaultServerBase
|
||||||
|
|
||||||
|
# Per-commit: SymmetricMemory variant only.
|
||||||
|
# - TestDeepseekV3FP4 (TRTLLM) archived to test/manual/quant/test_deepseek_v3_fp4_4gpu_trtllm.py
|
||||||
|
# - TestDeepseekV3FP4CutlassMoE moved to test_deepseek_v3_fp4_4gpu_extra.py
|
||||||
|
register_cuda_ci(est_time=960, stage="base-c", runner_config="4-gpu-b200")
|
||||||
|
|
||||||
|
FULL_DEEPSEEK_V3_FP4_MODEL_PATH = "nvidia/DeepSeek-V3-0324-FP4"
|
||||||
|
|
||||||
|
|
||||||
|
class TestDeepseekV3FP4SymmetricMemory(GSM8KMixin, DefaultServerBase):
|
||||||
|
model = FULL_DEEPSEEK_V3_FP4_MODEL_PATH
|
||||||
|
timeout = 1200
|
||||||
|
other_args = [
|
||||||
|
"--tp",
|
||||||
|
"4",
|
||||||
|
"--attention-backend",
|
||||||
|
"trtllm_mla",
|
||||||
|
"--moe-runner-backend",
|
||||||
|
"flashinfer_trtllm",
|
||||||
|
"--quantization",
|
||||||
|
"modelopt_fp4",
|
||||||
|
"--kv-cache-dtype",
|
||||||
|
"fp8_e4m3",
|
||||||
|
"--model-loader-extra-config",
|
||||||
|
'{"enable_multithread_load": true,"num_threads": 64}',
|
||||||
|
"--enable-symm-mem",
|
||||||
|
]
|
||||||
|
|
||||||
|
gsm8k_accuracy_thres = 0.93
|
||||||
|
gsm8k_num_questions = 1319
|
||||||
|
gsm8k_num_threads = 1319
|
||||||
|
gsm8k_num_shots = 8
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
unittest.main()
|
||||||
@@ -0,0 +1,67 @@
|
|||||||
|
import unittest
|
||||||
|
|
||||||
|
from sglang.test.ci.ci_register import register_cuda_ci
|
||||||
|
from sglang.test.kits.eval_accuracy_kit import GSM8KMixin
|
||||||
|
from sglang.test.send_one import BenchArgs, send_one_prompt
|
||||||
|
from sglang.test.server_fixtures.default_fixture import DefaultServerBase
|
||||||
|
from sglang.test.test_utils import (
|
||||||
|
DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
|
||||||
|
is_in_amd_ci,
|
||||||
|
is_in_ci,
|
||||||
|
write_github_step_summary,
|
||||||
|
)
|
||||||
|
|
||||||
|
register_cuda_ci(est_time=300, stage="base-c", runner_config="8-gpu-h200")
|
||||||
|
|
||||||
|
FULL_DEEPSEEK_V3_MODEL_PATH = "deepseek-ai/DeepSeek-V3-0324"
|
||||||
|
|
||||||
|
_OTHER_ARGS = [
|
||||||
|
"--tp",
|
||||||
|
"8",
|
||||||
|
"--trust-remote-code",
|
||||||
|
"--speculative-algorithm",
|
||||||
|
"EAGLE",
|
||||||
|
"--speculative-num-steps",
|
||||||
|
"3",
|
||||||
|
"--speculative-eagle-topk",
|
||||||
|
"1",
|
||||||
|
"--speculative-num-draft-tokens",
|
||||||
|
"4",
|
||||||
|
"--model-loader-extra-config",
|
||||||
|
'{"enable_multithread_load": true, "num_threads": 64}',
|
||||||
|
]
|
||||||
|
if not is_in_amd_ci():
|
||||||
|
_OTHER_ARGS += ["--mem-frac", "0.7"]
|
||||||
|
|
||||||
|
|
||||||
|
class TestDeepseekV3MTP(GSM8KMixin, DefaultServerBase):
|
||||||
|
model = FULL_DEEPSEEK_V3_MODEL_PATH
|
||||||
|
other_args = _OTHER_ARGS
|
||||||
|
timeout = DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH * 5
|
||||||
|
|
||||||
|
gsm8k_accuracy_thres = 0.935
|
||||||
|
gsm8k_accept_length_thres = 2.8
|
||||||
|
|
||||||
|
# `test_z_bs_1_speed` runs after `test_gsm8k` (alphabetical) so it
|
||||||
|
# measures steady-state speed on a warmed server.
|
||||||
|
def test_z_bs_1_speed(self):
|
||||||
|
args = BenchArgs(port=int(self.base_url.split(":")[-1]), max_new_tokens=2048)
|
||||||
|
acc_length, speed = send_one_prompt(args)
|
||||||
|
|
||||||
|
print(f"{acc_length=:.2f} {speed=:.2f}")
|
||||||
|
|
||||||
|
if is_in_ci():
|
||||||
|
write_github_step_summary(
|
||||||
|
f"### test_bs_1_speed (deepseek-v3 mtp)\n"
|
||||||
|
f"{acc_length=:.2f}\n"
|
||||||
|
f"{speed=:.2f} token/s\n"
|
||||||
|
)
|
||||||
|
self.assertGreater(acc_length, 2.8)
|
||||||
|
if is_in_amd_ci():
|
||||||
|
self.assertGreater(speed, 15)
|
||||||
|
else:
|
||||||
|
self.assertGreater(speed, 130)
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
unittest.main()
|
||||||
@@ -0,0 +1,23 @@
|
|||||||
|
import unittest
|
||||||
|
|
||||||
|
from sglang.test.ci.ci_register import register_cuda_ci
|
||||||
|
from sglang.test.gpt_oss_common import BaseTestGptOss
|
||||||
|
|
||||||
|
register_cuda_ci(est_time=220, stage="base-c", runner_config="4-gpu-h100")
|
||||||
|
register_cuda_ci(est_time=220, stage="base-c", runner_config="4-gpu-b200")
|
||||||
|
|
||||||
|
|
||||||
|
class TestGptOss4GpuBf16(BaseTestGptOss):
|
||||||
|
def test_bf16_120b(self):
|
||||||
|
self.run_test(
|
||||||
|
model_variant="120b",
|
||||||
|
quantization="bf16",
|
||||||
|
expected_score_of_reasoning_effort={
|
||||||
|
"low": 0.58,
|
||||||
|
},
|
||||||
|
other_args=["--tp", "4", "--cuda-graph-max-bs", "200"],
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
unittest.main()
|
||||||
+3
-13
@@ -3,21 +3,11 @@ import unittest
|
|||||||
from sglang.test.ci.ci_register import register_cuda_ci
|
from sglang.test.ci.ci_register import register_cuda_ci
|
||||||
from sglang.test.gpt_oss_common import BaseTestGptOss
|
from sglang.test.gpt_oss_common import BaseTestGptOss
|
||||||
|
|
||||||
register_cuda_ci(est_time=392, stage="base-c", runner_config="4-gpu-h100")
|
register_cuda_ci(est_time=220, stage="base-c", runner_config="4-gpu-h100")
|
||||||
register_cuda_ci(est_time=350, stage="base-c", runner_config="4-gpu-b200")
|
register_cuda_ci(est_time=220, stage="base-c", runner_config="4-gpu-b200")
|
||||||
|
|
||||||
|
|
||||||
class TestGptOss4Gpu(BaseTestGptOss):
|
class TestGptOss4GpuMxfp4(BaseTestGptOss):
|
||||||
def test_bf16_120b(self):
|
|
||||||
self.run_test(
|
|
||||||
model_variant="120b",
|
|
||||||
quantization="bf16",
|
|
||||||
expected_score_of_reasoning_effort={
|
|
||||||
"low": 0.58,
|
|
||||||
},
|
|
||||||
other_args=["--tp", "4", "--cuda-graph-max-bs", "200"],
|
|
||||||
)
|
|
||||||
|
|
||||||
def test_mxfp4_120b(self):
|
def test_mxfp4_120b(self):
|
||||||
self.run_test(
|
self.run_test(
|
||||||
model_variant="120b",
|
model_variant="120b",
|
||||||
@@ -0,0 +1,47 @@
|
|||||||
|
import unittest
|
||||||
|
|
||||||
|
from sglang.test.ci.ci_register import register_cuda_ci
|
||||||
|
from sglang.test.kits.eval_accuracy_kit import GSM8KMixin
|
||||||
|
from sglang.test.server_fixtures.mmmu_fixture import MMMUServerBase
|
||||||
|
|
||||||
|
register_cuda_ci(est_time=400, stage="base-c", runner_config="8-gpu-h200")
|
||||||
|
|
||||||
|
MIMO_V2_MODEL = "XiaomiMiMo/MiMo-V2.5"
|
||||||
|
MIMO_V2_OTHER_ARGS = [
|
||||||
|
"--tp",
|
||||||
|
"8",
|
||||||
|
"--dp",
|
||||||
|
"2",
|
||||||
|
"--enable-dp-attention",
|
||||||
|
"--mm-enable-dp-encoder",
|
||||||
|
"--attention-backend",
|
||||||
|
"fa3",
|
||||||
|
"--mm-attention-backend",
|
||||||
|
"fa3",
|
||||||
|
"--reasoning-parser",
|
||||||
|
"mimo",
|
||||||
|
]
|
||||||
|
MIMO_V2_MTP_OTHER_ARGS = MIMO_V2_OTHER_ARGS + [
|
||||||
|
"--speculative-algorithm",
|
||||||
|
"EAGLE",
|
||||||
|
"--speculative-num-steps",
|
||||||
|
"3",
|
||||||
|
"--speculative-eagle-topk",
|
||||||
|
"1",
|
||||||
|
"--speculative-num-draft-tokens",
|
||||||
|
"4",
|
||||||
|
"--enable-multi-layer-eagle",
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
|
class TestMiMoV2(GSM8KMixin, MMMUServerBase):
|
||||||
|
gsm8k_accuracy_thres = 0.75
|
||||||
|
gsm8k_accept_length_thres = 2.5
|
||||||
|
model = MIMO_V2_MODEL
|
||||||
|
mem_fraction_static = 0.65
|
||||||
|
server_api_key = None
|
||||||
|
other_args = MIMO_V2_MTP_OTHER_ARGS
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
unittest.main()
|
||||||
+1
-39
@@ -4,9 +4,8 @@ from sglang.test.ci.ci_register import register_cuda_ci
|
|||||||
from sglang.test.kits.eval_accuracy_kit import GSM8KMixin
|
from sglang.test.kits.eval_accuracy_kit import GSM8KMixin
|
||||||
from sglang.test.kits.spec_decoding_kit import SpecDecodingMixin
|
from sglang.test.kits.spec_decoding_kit import SpecDecodingMixin
|
||||||
from sglang.test.server_fixtures.default_fixture import DefaultServerBase
|
from sglang.test.server_fixtures.default_fixture import DefaultServerBase
|
||||||
from sglang.test.server_fixtures.mmmu_fixture import MMMUServerBase
|
|
||||||
|
|
||||||
register_cuda_ci(est_time=500, stage="base-c", runner_config="8-gpu-h200")
|
register_cuda_ci(est_time=350, stage="base-c", runner_config="8-gpu-h200")
|
||||||
|
|
||||||
|
|
||||||
class TestMiMoV2Flash(GSM8KMixin, SpecDecodingMixin, DefaultServerBase):
|
class TestMiMoV2Flash(GSM8KMixin, SpecDecodingMixin, DefaultServerBase):
|
||||||
@@ -49,42 +48,5 @@ class TestMiMoV2Flash(GSM8KMixin, SpecDecodingMixin, DefaultServerBase):
|
|||||||
accept_length_thres = 3.2
|
accept_length_thres = 3.2
|
||||||
|
|
||||||
|
|
||||||
MIMO_V2_MODEL = "XiaomiMiMo/MiMo-V2.5"
|
|
||||||
MIMO_V2_OTHER_ARGS = [
|
|
||||||
"--tp",
|
|
||||||
"8",
|
|
||||||
"--dp",
|
|
||||||
"2",
|
|
||||||
"--enable-dp-attention",
|
|
||||||
"--mm-enable-dp-encoder",
|
|
||||||
"--attention-backend",
|
|
||||||
"fa3",
|
|
||||||
"--mm-attention-backend",
|
|
||||||
"fa3",
|
|
||||||
"--reasoning-parser",
|
|
||||||
"mimo",
|
|
||||||
]
|
|
||||||
MIMO_V2_MTP_OTHER_ARGS = MIMO_V2_OTHER_ARGS + [
|
|
||||||
"--speculative-algorithm",
|
|
||||||
"EAGLE",
|
|
||||||
"--speculative-num-steps",
|
|
||||||
"3",
|
|
||||||
"--speculative-eagle-topk",
|
|
||||||
"1",
|
|
||||||
"--speculative-num-draft-tokens",
|
|
||||||
"4",
|
|
||||||
"--enable-multi-layer-eagle",
|
|
||||||
]
|
|
||||||
|
|
||||||
|
|
||||||
class TestMiMoV2(GSM8KMixin, MMMUServerBase):
|
|
||||||
gsm8k_accuracy_thres = 0.75
|
|
||||||
gsm8k_accept_length_thres = 2.5
|
|
||||||
model = MIMO_V2_MODEL
|
|
||||||
mem_fraction_static = 0.65
|
|
||||||
server_api_key = None
|
|
||||||
other_args = MIMO_V2_MTP_OTHER_ARGS
|
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
unittest.main()
|
unittest.main()
|
||||||
@@ -0,0 +1,69 @@
|
|||||||
|
import unittest
|
||||||
|
from types import SimpleNamespace
|
||||||
|
|
||||||
|
from sglang.srt.utils import kill_process_tree
|
||||||
|
from sglang.test.ci.ci_register import register_cuda_ci
|
||||||
|
from sglang.test.run_eval import run_eval
|
||||||
|
from sglang.test.test_utils import (
|
||||||
|
DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
|
||||||
|
DEFAULT_URL_FOR_TEST,
|
||||||
|
CustomTestCase,
|
||||||
|
popen_launch_server,
|
||||||
|
)
|
||||||
|
|
||||||
|
register_cuda_ci(est_time=200, stage="extra-b", runner_config="8-gpu-h200")
|
||||||
|
|
||||||
|
NEMOTRON_3_SUPER_BF16_MODEL = "nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-BF16"
|
||||||
|
|
||||||
|
NEMOTRON_3_SUPER_BF16_ARGS = [
|
||||||
|
"--tp-size",
|
||||||
|
"8",
|
||||||
|
"--trust-remote-code",
|
||||||
|
"--reasoning-parser",
|
||||||
|
"nemotron_3",
|
||||||
|
"--tool-call-parser",
|
||||||
|
"qwen3_coder",
|
||||||
|
"--disable-radix-cache",
|
||||||
|
"--model-loader-extra-config",
|
||||||
|
'{"enable_multithread_load": true, "num_threads": 50}',
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
|
class TestNvidiaNemotron3SuperBF16(CustomTestCase):
|
||||||
|
@classmethod
|
||||||
|
def setUpClass(cls):
|
||||||
|
cls.model = NEMOTRON_3_SUPER_BF16_MODEL
|
||||||
|
cls.base_url = DEFAULT_URL_FOR_TEST
|
||||||
|
cls.process = popen_launch_server(
|
||||||
|
cls.model,
|
||||||
|
cls.base_url,
|
||||||
|
timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
|
||||||
|
other_args=NEMOTRON_3_SUPER_BF16_ARGS,
|
||||||
|
)
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def tearDownClass(cls):
|
||||||
|
kill_process_tree(cls.process.pid)
|
||||||
|
|
||||||
|
def test_gsm8k(self):
|
||||||
|
args = SimpleNamespace(
|
||||||
|
model=self.model,
|
||||||
|
eval_name="gsm8k",
|
||||||
|
num_shots=5,
|
||||||
|
num_examples=200,
|
||||||
|
max_tokens=16000,
|
||||||
|
num_threads=200,
|
||||||
|
repeat=1,
|
||||||
|
temperature=1.0,
|
||||||
|
top_p=0.95,
|
||||||
|
base_url=self.base_url,
|
||||||
|
host="http://127.0.0.1",
|
||||||
|
port=int(self.base_url.split(":")[-1]),
|
||||||
|
)
|
||||||
|
metrics = run_eval(args)
|
||||||
|
print(f"{metrics=}")
|
||||||
|
self.assertGreaterEqual(metrics["score"], 0.96)
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
unittest.main()
|
||||||
+18
-42
@@ -11,7 +11,7 @@ from sglang.test.test_utils import (
|
|||||||
popen_launch_server,
|
popen_launch_server,
|
||||||
)
|
)
|
||||||
|
|
||||||
register_cuda_ci(est_time=370, stage="extra-b", runner_config="8-gpu-h200")
|
register_cuda_ci(est_time=200, stage="extra-b", runner_config="8-gpu-h200")
|
||||||
|
|
||||||
NEMOTRON_3_SUPER_BF16_MODEL = "nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-BF16"
|
NEMOTRON_3_SUPER_BF16_MODEL = "nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-BF16"
|
||||||
|
|
||||||
@@ -44,46 +44,6 @@ MTP_ARGS = [
|
|||||||
]
|
]
|
||||||
|
|
||||||
|
|
||||||
def _run_gsm8k(test_case):
|
|
||||||
args = SimpleNamespace(
|
|
||||||
model=test_case.model,
|
|
||||||
eval_name="gsm8k",
|
|
||||||
num_shots=5,
|
|
||||||
num_examples=200,
|
|
||||||
max_tokens=16000,
|
|
||||||
num_threads=200,
|
|
||||||
repeat=1,
|
|
||||||
temperature=1.0,
|
|
||||||
top_p=0.95,
|
|
||||||
base_url=test_case.base_url,
|
|
||||||
host="http://127.0.0.1",
|
|
||||||
port=int(test_case.base_url.split(":")[-1]),
|
|
||||||
)
|
|
||||||
metrics = run_eval(args)
|
|
||||||
print(f"{metrics=}")
|
|
||||||
test_case.assertGreaterEqual(metrics["score"], 0.96)
|
|
||||||
|
|
||||||
|
|
||||||
class TestNvidiaNemotron3SuperBF16(CustomTestCase):
|
|
||||||
@classmethod
|
|
||||||
def setUpClass(cls):
|
|
||||||
cls.model = NEMOTRON_3_SUPER_BF16_MODEL
|
|
||||||
cls.base_url = DEFAULT_URL_FOR_TEST
|
|
||||||
cls.process = popen_launch_server(
|
|
||||||
cls.model,
|
|
||||||
cls.base_url,
|
|
||||||
timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
|
|
||||||
other_args=NEMOTRON_3_SUPER_BF16_ARGS,
|
|
||||||
)
|
|
||||||
|
|
||||||
@classmethod
|
|
||||||
def tearDownClass(cls):
|
|
||||||
kill_process_tree(cls.process.pid)
|
|
||||||
|
|
||||||
def test_gsm8k(self):
|
|
||||||
_run_gsm8k(self)
|
|
||||||
|
|
||||||
|
|
||||||
class TestNvidiaNemotron3SuperBF16MTP(CustomTestCase):
|
class TestNvidiaNemotron3SuperBF16MTP(CustomTestCase):
|
||||||
@classmethod
|
@classmethod
|
||||||
def setUpClass(cls):
|
def setUpClass(cls):
|
||||||
@@ -101,7 +61,23 @@ class TestNvidiaNemotron3SuperBF16MTP(CustomTestCase):
|
|||||||
kill_process_tree(cls.process.pid)
|
kill_process_tree(cls.process.pid)
|
||||||
|
|
||||||
def test_gsm8k(self):
|
def test_gsm8k(self):
|
||||||
_run_gsm8k(self)
|
args = SimpleNamespace(
|
||||||
|
model=self.model,
|
||||||
|
eval_name="gsm8k",
|
||||||
|
num_shots=5,
|
||||||
|
num_examples=200,
|
||||||
|
max_tokens=16000,
|
||||||
|
num_threads=200,
|
||||||
|
repeat=1,
|
||||||
|
temperature=1.0,
|
||||||
|
top_p=0.95,
|
||||||
|
base_url=self.base_url,
|
||||||
|
host="http://127.0.0.1",
|
||||||
|
port=int(self.base_url.split(":")[-1]),
|
||||||
|
)
|
||||||
|
metrics = run_eval(args)
|
||||||
|
print(f"{metrics=}")
|
||||||
|
self.assertGreaterEqual(metrics["score"], 0.96)
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
+1
-1
@@ -21,7 +21,7 @@ QWEN35_FP4_MODEL = "nvidia/Qwen3.5-397B-A17B-NVFP4"
|
|||||||
ACC_THRESHOLDS = {QWEN35_FP4_MODEL: {"gsm8k": 0.95}}
|
ACC_THRESHOLDS = {QWEN35_FP4_MODEL: {"gsm8k": 0.95}}
|
||||||
|
|
||||||
|
|
||||||
class TestQwen35FP4MTPV2(ReasoningTokenUsageMixin, CustomTestCase):
|
class TestQwen35FP4MTP(ReasoningTokenUsageMixin, CustomTestCase):
|
||||||
reasoning_parser_name = "qwen3"
|
reasoning_parser_name = "qwen3"
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
+33
-78
@@ -1,28 +1,20 @@
|
|||||||
import unittest
|
import unittest
|
||||||
from types import SimpleNamespace
|
|
||||||
|
|
||||||
import numpy as np
|
import numpy as np
|
||||||
import requests
|
import requests
|
||||||
|
|
||||||
from sglang.srt.environ import envs
|
from sglang.srt.environ import envs
|
||||||
from sglang.srt.utils import kill_process_tree
|
|
||||||
from sglang.test.ci.ci_register import register_cuda_ci
|
from sglang.test.ci.ci_register import register_cuda_ci
|
||||||
from sglang.test.run_eval import run_eval
|
from sglang.test.kits.eval_accuracy_kit import GSM8KMixin
|
||||||
from sglang.test.test_utils import (
|
from sglang.test.server_fixtures.default_fixture import DefaultServerBase
|
||||||
DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
|
from sglang.test.test_utils import DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH
|
||||||
DEFAULT_URL_FOR_TEST,
|
|
||||||
CustomTestCase,
|
|
||||||
is_in_ci,
|
|
||||||
popen_launch_server,
|
|
||||||
write_github_step_summary,
|
|
||||||
)
|
|
||||||
|
|
||||||
register_cuda_ci(est_time=480, stage="extra-b", runner_config="8-gpu-h200")
|
register_cuda_ci(est_time=480, stage="extra-b", runner_config="8-gpu-h200")
|
||||||
|
|
||||||
STEP3P5_FLASH_MODEL_PATH = "stepfun-ai/Step-3.5-Flash"
|
STEP3P5_FLASH_MODEL_PATH = "stepfun-ai/Step-3.5-Flash"
|
||||||
|
|
||||||
|
|
||||||
class TestStep3p5FlashChainMTP(CustomTestCase):
|
class TestStep3p5FlashChainMTP(GSM8KMixin, DefaultServerBase):
|
||||||
"""Chain-style multi-layer EAGLE speculative decoding on Step-3.5-Flash.
|
"""Chain-style multi-layer EAGLE speculative decoding on Step-3.5-Flash.
|
||||||
|
|
||||||
Step3p5ForCausalLM auto-enables multi-layer EAGLE and spec v2 when
|
Step3p5ForCausalLM auto-enables multi-layer EAGLE and spec v2 when
|
||||||
@@ -31,75 +23,38 @@ class TestStep3p5FlashChainMTP(CustomTestCase):
|
|||||||
automatically for the Step3p5MTP draft architecture.
|
automatically for the Step3p5MTP draft architecture.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
|
model = STEP3P5_FLASH_MODEL_PATH
|
||||||
|
timeout = DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH * 3
|
||||||
|
other_args = [
|
||||||
|
"--tp",
|
||||||
|
"8",
|
||||||
|
"--trust-remote-code",
|
||||||
|
"--speculative-algorithm",
|
||||||
|
"EAGLE",
|
||||||
|
"--speculative-num-steps",
|
||||||
|
"3",
|
||||||
|
"--speculative-eagle-topk",
|
||||||
|
"1",
|
||||||
|
"--speculative-num-draft-tokens",
|
||||||
|
"4",
|
||||||
|
"--attention-backend",
|
||||||
|
"fa3",
|
||||||
|
"--enable-multi-layer-eagle",
|
||||||
|
"--mem-fraction-static",
|
||||||
|
"0.75",
|
||||||
|
"--chunked-prefill-size",
|
||||||
|
"4096",
|
||||||
|
"--model-loader-extra-config",
|
||||||
|
'{"enable_multithread_load": true, "num_threads": 64}',
|
||||||
|
]
|
||||||
|
|
||||||
|
gsm8k_accuracy_thres = 0.83
|
||||||
|
gsm8k_accept_length_thres = 2.6
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
def setUpClass(cls):
|
def setUpClass(cls):
|
||||||
cls.model = STEP3P5_FLASH_MODEL_PATH
|
|
||||||
cls.base_url = DEFAULT_URL_FOR_TEST
|
|
||||||
other_args = [
|
|
||||||
"--tp",
|
|
||||||
"8",
|
|
||||||
"--trust-remote-code",
|
|
||||||
"--speculative-algorithm",
|
|
||||||
"EAGLE",
|
|
||||||
"--speculative-num-steps",
|
|
||||||
"3",
|
|
||||||
"--speculative-eagle-topk",
|
|
||||||
"1",
|
|
||||||
"--speculative-num-draft-tokens",
|
|
||||||
"4",
|
|
||||||
"--attention-backend",
|
|
||||||
"fa3",
|
|
||||||
"--enable-multi-layer-eagle",
|
|
||||||
"--mem-fraction-static",
|
|
||||||
"0.75",
|
|
||||||
"--chunked-prefill-size",
|
|
||||||
"4096",
|
|
||||||
"--model-loader-extra-config",
|
|
||||||
'{"enable_multithread_load": true, "num_threads": 64}',
|
|
||||||
]
|
|
||||||
with envs.SGLANG_ENABLE_SPEC_V2.override(True):
|
with envs.SGLANG_ENABLE_SPEC_V2.override(True):
|
||||||
cls.process = popen_launch_server(
|
super().setUpClass()
|
||||||
cls.model,
|
|
||||||
cls.base_url,
|
|
||||||
timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH * 3,
|
|
||||||
other_args=other_args,
|
|
||||||
)
|
|
||||||
|
|
||||||
@classmethod
|
|
||||||
def tearDownClass(cls):
|
|
||||||
if hasattr(cls, "process") and cls.process:
|
|
||||||
kill_process_tree(cls.process.pid)
|
|
||||||
|
|
||||||
def test_gsm8k(self):
|
|
||||||
requests.get(self.base_url + "/flush_cache")
|
|
||||||
|
|
||||||
args = SimpleNamespace(
|
|
||||||
base_url=self.base_url,
|
|
||||||
model=self.model,
|
|
||||||
eval_name="gsm8k",
|
|
||||||
api="completion",
|
|
||||||
max_tokens=512,
|
|
||||||
num_examples=200,
|
|
||||||
num_threads=128,
|
|
||||||
)
|
|
||||||
metrics = run_eval(args)
|
|
||||||
print(f"{metrics=}")
|
|
||||||
|
|
||||||
server_info = requests.get(self.base_url + "/server_info")
|
|
||||||
avg_spec_accept_length = server_info.json()["internal_states"][0][
|
|
||||||
"avg_spec_accept_length"
|
|
||||||
]
|
|
||||||
print(f"{avg_spec_accept_length=}")
|
|
||||||
print(f"{metrics=}")
|
|
||||||
|
|
||||||
if is_in_ci():
|
|
||||||
write_github_step_summary(
|
|
||||||
f"### test_gsm8k (step-3.5-flash chain mtp)\n"
|
|
||||||
f'{metrics["score"]=:.3f}\n'
|
|
||||||
f"{avg_spec_accept_length=:.2f}\n"
|
|
||||||
)
|
|
||||||
self.assertGreater(metrics["score"], 0.83)
|
|
||||||
self.assertGreater(avg_spec_accept_length, 2.6)
|
|
||||||
|
|
||||||
def test_logprob_spec_v2_match(self):
|
def test_logprob_spec_v2_match(self):
|
||||||
"""Verify spec v2 decode logprobs match prefill scoring logprobs.
|
"""Verify spec v2 decode logprobs match prefill scoring logprobs.
|
||||||
@@ -1,212 +0,0 @@
|
|||||||
import unittest
|
|
||||||
from types import SimpleNamespace
|
|
||||||
|
|
||||||
import requests
|
|
||||||
|
|
||||||
from sglang.srt.utils import kill_process_tree
|
|
||||||
from sglang.test.ci.ci_register import register_cuda_ci
|
|
||||||
from sglang.test.run_eval import run_eval
|
|
||||||
from sglang.test.send_one import BenchArgs, send_one_prompt
|
|
||||||
from sglang.test.test_utils import (
|
|
||||||
DEFAULT_URL_FOR_TEST,
|
|
||||||
CustomTestCase,
|
|
||||||
is_in_ci,
|
|
||||||
popen_launch_server,
|
|
||||||
write_github_step_summary,
|
|
||||||
)
|
|
||||||
|
|
||||||
register_cuda_ci(
|
|
||||||
est_time=690,
|
|
||||||
stage="base-c",
|
|
||||||
runner_config="4-gpu-b200",
|
|
||||||
)
|
|
||||||
|
|
||||||
FULL_DEEPSEEK_V3_FP4_MODEL_PATH = "nvidia/DeepSeek-V3.2-NVFP4"
|
|
||||||
SERVER_LAUNCH_TIMEOUT = 1200
|
|
||||||
|
|
||||||
|
|
||||||
class TestDeepseekV32FP4DPSpecV2(CustomTestCase):
|
|
||||||
@classmethod
|
|
||||||
def setUpClass(cls):
|
|
||||||
cls.model = FULL_DEEPSEEK_V3_FP4_MODEL_PATH
|
|
||||||
cls.base_url = DEFAULT_URL_FOR_TEST
|
|
||||||
other_args = [
|
|
||||||
"--tp",
|
|
||||||
"4",
|
|
||||||
"--dp",
|
|
||||||
"4",
|
|
||||||
"--enable-dp-attention",
|
|
||||||
"--attention-backend",
|
|
||||||
"dsa",
|
|
||||||
"--moe-runner-backend",
|
|
||||||
"flashinfer_trtllm",
|
|
||||||
"--quantization",
|
|
||||||
"modelopt_fp4",
|
|
||||||
"--tool-call-parser",
|
|
||||||
"deepseekv32",
|
|
||||||
"--reasoning-parser",
|
|
||||||
"deepseek-v3",
|
|
||||||
"--speculative-algorithm",
|
|
||||||
"EAGLE",
|
|
||||||
"--speculative-num-steps",
|
|
||||||
"3",
|
|
||||||
"--speculative-eagle-topk",
|
|
||||||
"1",
|
|
||||||
"--speculative-num-draft-tokens",
|
|
||||||
"4",
|
|
||||||
"--model-loader-extra-config",
|
|
||||||
'{"enable_multithread_load": true,"num_threads": 64}',
|
|
||||||
]
|
|
||||||
cls.process = popen_launch_server(
|
|
||||||
cls.model,
|
|
||||||
cls.base_url,
|
|
||||||
timeout=SERVER_LAUNCH_TIMEOUT,
|
|
||||||
other_args=other_args,
|
|
||||||
)
|
|
||||||
|
|
||||||
@classmethod
|
|
||||||
def tearDownClass(cls):
|
|
||||||
kill_process_tree(cls.process.pid)
|
|
||||||
|
|
||||||
def test_a_gsm8k(
|
|
||||||
self,
|
|
||||||
): # Append an "a" to make this test run first (alphabetically) to warm up the server
|
|
||||||
requests.get(self.base_url + "/flush_cache")
|
|
||||||
|
|
||||||
args = SimpleNamespace(
|
|
||||||
base_url=self.base_url,
|
|
||||||
model=self.model,
|
|
||||||
eval_name="gsm8k",
|
|
||||||
api="completion",
|
|
||||||
max_tokens=512,
|
|
||||||
num_examples=500,
|
|
||||||
num_threads=500,
|
|
||||||
num_shots=20,
|
|
||||||
)
|
|
||||||
metrics = run_eval(args)
|
|
||||||
print(f"{metrics=}")
|
|
||||||
|
|
||||||
server_info = requests.get(self.base_url + "/server_info")
|
|
||||||
avg_spec_accept_length = server_info.json()["internal_states"][0][
|
|
||||||
"avg_spec_accept_length"
|
|
||||||
]
|
|
||||||
print(f"{avg_spec_accept_length=}")
|
|
||||||
|
|
||||||
if is_in_ci():
|
|
||||||
write_github_step_summary(
|
|
||||||
f"### test_gsm8k (deepseek-v32 mtp)\n"
|
|
||||||
f'{metrics["score"]=:.3f}\n'
|
|
||||||
f"{avg_spec_accept_length=:.2f}\n"
|
|
||||||
)
|
|
||||||
self.assertGreater(metrics["score"], 0.93)
|
|
||||||
self.assertGreater(avg_spec_accept_length, 2.7)
|
|
||||||
|
|
||||||
def test_bs_1_speed(self):
|
|
||||||
args = BenchArgs(port=int(self.base_url.split(":")[-1]), max_new_tokens=2048)
|
|
||||||
acc_length, speed = send_one_prompt(args)
|
|
||||||
|
|
||||||
print(f"{acc_length=:.2f} {speed=:.2f}")
|
|
||||||
|
|
||||||
if is_in_ci():
|
|
||||||
write_github_step_summary(
|
|
||||||
f"### test_bs_1_speed (deepseek-v32 mtp)\n"
|
|
||||||
f"{acc_length=:.2f}\n"
|
|
||||||
f"{speed=:.2f} token/s\n"
|
|
||||||
)
|
|
||||||
|
|
||||||
self.assertGreater(acc_length, 2.7)
|
|
||||||
self.assertGreater(speed, 90)
|
|
||||||
|
|
||||||
|
|
||||||
class TestDeepseekV32FP4TPSpecV2(CustomTestCase):
|
|
||||||
@classmethod
|
|
||||||
def setUpClass(cls):
|
|
||||||
cls.model = FULL_DEEPSEEK_V3_FP4_MODEL_PATH
|
|
||||||
cls.base_url = DEFAULT_URL_FOR_TEST
|
|
||||||
other_args = [
|
|
||||||
"--tp",
|
|
||||||
"4",
|
|
||||||
"--attention-backend",
|
|
||||||
"dsa",
|
|
||||||
"--moe-runner-backend",
|
|
||||||
"flashinfer_trtllm",
|
|
||||||
"--quantization",
|
|
||||||
"modelopt_fp4",
|
|
||||||
"--tool-call-parser",
|
|
||||||
"deepseekv32",
|
|
||||||
"--reasoning-parser",
|
|
||||||
"deepseek-v3",
|
|
||||||
"--speculative-algorithm",
|
|
||||||
"EAGLE",
|
|
||||||
"--speculative-num-steps",
|
|
||||||
"3",
|
|
||||||
"--speculative-eagle-topk",
|
|
||||||
"1",
|
|
||||||
"--speculative-num-draft-tokens",
|
|
||||||
"4",
|
|
||||||
"--model-loader-extra-config",
|
|
||||||
'{"enable_multithread_load": true,"num_threads": 64}',
|
|
||||||
]
|
|
||||||
cls.process = popen_launch_server(
|
|
||||||
cls.model,
|
|
||||||
cls.base_url,
|
|
||||||
timeout=SERVER_LAUNCH_TIMEOUT,
|
|
||||||
other_args=other_args,
|
|
||||||
)
|
|
||||||
|
|
||||||
@classmethod
|
|
||||||
def tearDownClass(cls):
|
|
||||||
kill_process_tree(cls.process.pid)
|
|
||||||
|
|
||||||
def test_a_gsm8k(
|
|
||||||
self,
|
|
||||||
): # Append an "a" to make this test run first (alphabetically) to warm up the server
|
|
||||||
requests.get(self.base_url + "/flush_cache")
|
|
||||||
|
|
||||||
args = SimpleNamespace(
|
|
||||||
base_url=self.base_url,
|
|
||||||
model=self.model,
|
|
||||||
eval_name="gsm8k",
|
|
||||||
api="completion",
|
|
||||||
max_tokens=512,
|
|
||||||
num_examples=500,
|
|
||||||
num_threads=500,
|
|
||||||
num_shots=20,
|
|
||||||
)
|
|
||||||
metrics = run_eval(args)
|
|
||||||
print(f"{metrics=}")
|
|
||||||
|
|
||||||
server_info = requests.get(self.base_url + "/server_info")
|
|
||||||
avg_spec_accept_length = server_info.json()["internal_states"][0][
|
|
||||||
"avg_spec_accept_length"
|
|
||||||
]
|
|
||||||
print(f"{avg_spec_accept_length=}")
|
|
||||||
|
|
||||||
if is_in_ci():
|
|
||||||
write_github_step_summary(
|
|
||||||
f"### test_gsm8k (deepseek-v32 mtp)\n"
|
|
||||||
f'{metrics["score"]=:.3f}\n'
|
|
||||||
f"{avg_spec_accept_length=:.2f}\n"
|
|
||||||
)
|
|
||||||
self.assertGreater(metrics["score"], 0.93)
|
|
||||||
self.assertGreater(avg_spec_accept_length, 2.7)
|
|
||||||
|
|
||||||
def test_bs_1_speed(self):
|
|
||||||
args = BenchArgs(port=int(self.base_url.split(":")[-1]), max_new_tokens=2048)
|
|
||||||
acc_length, speed = send_one_prompt(args)
|
|
||||||
|
|
||||||
print(f"{acc_length=:.2f} {speed=:.2f}")
|
|
||||||
|
|
||||||
if is_in_ci():
|
|
||||||
write_github_step_summary(
|
|
||||||
f"### test_bs_1_speed (deepseek-v32 mtp)\n"
|
|
||||||
f"{acc_length=:.2f}\n"
|
|
||||||
f"{speed=:.2f} token/s\n"
|
|
||||||
)
|
|
||||||
|
|
||||||
self.assertGreater(acc_length, 2.7)
|
|
||||||
self.assertGreater(speed, 150)
|
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
|
||||||
unittest.main()
|
|
||||||
@@ -1,80 +0,0 @@
|
|||||||
import unittest
|
|
||||||
from types import SimpleNamespace
|
|
||||||
|
|
||||||
from sglang.srt.utils import kill_process_tree
|
|
||||||
from sglang.test.ci.ci_register import register_cuda_ci
|
|
||||||
from sglang.test.run_eval import run_eval
|
|
||||||
from sglang.test.test_utils import (
|
|
||||||
DEFAULT_URL_FOR_TEST,
|
|
||||||
CustomTestCase,
|
|
||||||
is_in_ci,
|
|
||||||
popen_launch_server,
|
|
||||||
write_github_step_summary,
|
|
||||||
)
|
|
||||||
|
|
||||||
# Per-commit: SymmetricMemory variant only.
|
|
||||||
# - TestDeepseekV3FP4 (TRTLLM) archived to test/manual/quant/test_deepseek_v3_fp4_4gpu_trtllm.py
|
|
||||||
# - TestDeepseekV3FP4CutlassMoE moved to test_deepseek_v3_fp4_4gpu_extra.py
|
|
||||||
register_cuda_ci(est_time=960, stage="base-c", runner_config="4-gpu-b200")
|
|
||||||
|
|
||||||
FULL_DEEPSEEK_V3_FP4_MODEL_PATH = "nvidia/DeepSeek-V3-0324-FP4"
|
|
||||||
SERVER_LAUNCH_TIMEOUT = 1200
|
|
||||||
|
|
||||||
|
|
||||||
class TestDeepseekV3FP4SymmetricMemory(CustomTestCase):
|
|
||||||
@classmethod
|
|
||||||
def setUpClass(cls):
|
|
||||||
cls.model = FULL_DEEPSEEK_V3_FP4_MODEL_PATH
|
|
||||||
cls.base_url = DEFAULT_URL_FOR_TEST
|
|
||||||
other_args = [
|
|
||||||
"--tp",
|
|
||||||
"4",
|
|
||||||
"--attention-backend",
|
|
||||||
"trtllm_mla",
|
|
||||||
"--moe-runner-backend",
|
|
||||||
"flashinfer_trtllm",
|
|
||||||
"--quantization",
|
|
||||||
"modelopt_fp4",
|
|
||||||
"--kv-cache-dtype",
|
|
||||||
"fp8_e4m3",
|
|
||||||
"--model-loader-extra-config",
|
|
||||||
'{"enable_multithread_load": true,"num_threads": 64}',
|
|
||||||
"--enable-symm-mem",
|
|
||||||
]
|
|
||||||
cls.process = popen_launch_server(
|
|
||||||
cls.model,
|
|
||||||
cls.base_url,
|
|
||||||
timeout=SERVER_LAUNCH_TIMEOUT,
|
|
||||||
other_args=other_args,
|
|
||||||
)
|
|
||||||
|
|
||||||
@classmethod
|
|
||||||
def tearDownClass(cls):
|
|
||||||
kill_process_tree(cls.process.pid)
|
|
||||||
|
|
||||||
def test_a_gsm8k(
|
|
||||||
self,
|
|
||||||
): # Append an "a" to make this test run first (alphabetically) to warm up the server
|
|
||||||
args = SimpleNamespace(
|
|
||||||
base_url=self.base_url,
|
|
||||||
model=self.model,
|
|
||||||
eval_name="gsm8k",
|
|
||||||
api="completion",
|
|
||||||
max_tokens=512,
|
|
||||||
num_examples=1319,
|
|
||||||
num_threads=1319,
|
|
||||||
num_shots=8,
|
|
||||||
)
|
|
||||||
metrics = run_eval(args)
|
|
||||||
print(f"{metrics=}")
|
|
||||||
|
|
||||||
if is_in_ci():
|
|
||||||
write_github_step_summary(
|
|
||||||
f"### test_gsm8k (deepseek-v3-fp4)\n" f'{metrics["score"]=:.3f}\n'
|
|
||||||
)
|
|
||||||
|
|
||||||
self.assertGreater(metrics["score"], 0.93)
|
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
|
||||||
unittest.main()
|
|
||||||
@@ -0,0 +1,50 @@
|
|||||||
|
import unittest
|
||||||
|
|
||||||
|
from sglang.srt.utils import kill_process_tree
|
||||||
|
from sglang.test.ci.ci_register import register_cuda_ci
|
||||||
|
from sglang.test.kits.unified_radix_cache_kit import UnifiedRadixTreeTestMixin
|
||||||
|
from sglang.test.kl_multiturn_utils import get_input_ids
|
||||||
|
from sglang.test.test_utils import (
|
||||||
|
DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
|
||||||
|
DEFAULT_URL_FOR_TEST,
|
||||||
|
CustomTestCase,
|
||||||
|
popen_launch_server,
|
||||||
|
)
|
||||||
|
|
||||||
|
register_cuda_ci(est_time=250, stage="base-c", runner_config="4-gpu-h100")
|
||||||
|
|
||||||
|
FULL_MODEL = "Qwen/Qwen3-32B"
|
||||||
|
|
||||||
|
|
||||||
|
class TestUnifiedFullRadixCache(UnifiedRadixTreeTestMixin, CustomTestCase):
|
||||||
|
"""Full attention."""
|
||||||
|
|
||||||
|
kl_threshold = 0.0025
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def setUpClass(cls):
|
||||||
|
cls.model = FULL_MODEL
|
||||||
|
cls.base_url = DEFAULT_URL_FOR_TEST
|
||||||
|
cls.process = popen_launch_server(
|
||||||
|
cls.model,
|
||||||
|
cls.base_url,
|
||||||
|
timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
|
||||||
|
other_args=[
|
||||||
|
"--tp-size",
|
||||||
|
"4",
|
||||||
|
"--mem-fraction-static",
|
||||||
|
"0.80",
|
||||||
|
"--page-size",
|
||||||
|
"64",
|
||||||
|
],
|
||||||
|
env={"SGLANG_ENABLE_UNIFIED_RADIX_TREE": "1"},
|
||||||
|
)
|
||||||
|
cls.input_ids = get_input_ids(cls.model, num_samples=18)
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def tearDownClass(cls):
|
||||||
|
kill_process_tree(cls.process.pid)
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
unittest.main()
|
||||||
@@ -1,9 +1,8 @@
|
|||||||
import unittest
|
import unittest
|
||||||
|
|
||||||
from test_unified_radix_cache_kl import UnifiedRadixTreeTestMixin
|
|
||||||
|
|
||||||
from sglang.srt.utils import kill_process_tree
|
from sglang.srt.utils import kill_process_tree
|
||||||
from sglang.test.ci.ci_register import register_cuda_ci
|
from sglang.test.ci.ci_register import register_cuda_ci
|
||||||
|
from sglang.test.kits.unified_radix_cache_kit import UnifiedRadixTreeTestMixin
|
||||||
from sglang.test.kl_multiturn_utils import (
|
from sglang.test.kl_multiturn_utils import (
|
||||||
get_input_ids,
|
get_input_ids,
|
||||||
make_mamba_decode_assert,
|
make_mamba_decode_assert,
|
||||||
|
|||||||
@@ -0,0 +1,66 @@
|
|||||||
|
import unittest
|
||||||
|
|
||||||
|
from sglang.srt.utils import kill_process_tree
|
||||||
|
from sglang.test.ci.ci_register import register_cuda_ci
|
||||||
|
from sglang.test.kits.unified_radix_cache_kit import UnifiedRadixTreeTestMixin
|
||||||
|
from sglang.test.kl_multiturn_utils import (
|
||||||
|
get_input_ids,
|
||||||
|
make_mamba_decode_assert,
|
||||||
|
make_mamba_prefill_assert,
|
||||||
|
)
|
||||||
|
from sglang.test.test_utils import (
|
||||||
|
DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
|
||||||
|
DEFAULT_URL_FOR_TEST,
|
||||||
|
CustomTestCase,
|
||||||
|
popen_launch_server,
|
||||||
|
)
|
||||||
|
|
||||||
|
register_cuda_ci(est_time=250, stage="base-c", runner_config="4-gpu-h100")
|
||||||
|
|
||||||
|
MAMBA_MODEL = "Qwen/Qwen3-Next-80B-A3B-Instruct"
|
||||||
|
MAMBA_CHUNK_SIZE = 64
|
||||||
|
MAMBA_TRACK_INTERVAL = 128
|
||||||
|
|
||||||
|
|
||||||
|
class TestUnifiedMambaRadixCache(UnifiedRadixTreeTestMixin, CustomTestCase):
|
||||||
|
"""Mamba hybrid + UnifiedRadixCache."""
|
||||||
|
|
||||||
|
kl_threshold = 0.003
|
||||||
|
prefill_cache_assert = staticmethod(
|
||||||
|
make_mamba_prefill_assert(chunk_size=MAMBA_CHUNK_SIZE)
|
||||||
|
)
|
||||||
|
decode_cache_assert = staticmethod(
|
||||||
|
make_mamba_decode_assert(track_interval=MAMBA_TRACK_INTERVAL)
|
||||||
|
)
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def setUpClass(cls):
|
||||||
|
cls.model = MAMBA_MODEL
|
||||||
|
cls.base_url = DEFAULT_URL_FOR_TEST
|
||||||
|
cls.process = popen_launch_server(
|
||||||
|
cls.model,
|
||||||
|
cls.base_url,
|
||||||
|
timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
|
||||||
|
other_args=[
|
||||||
|
"--tp-size",
|
||||||
|
"4",
|
||||||
|
"--chunked-prefill-size",
|
||||||
|
"2048",
|
||||||
|
"--mem-fraction-static",
|
||||||
|
"0.85",
|
||||||
|
"--mamba-scheduler-strategy",
|
||||||
|
"extra_buffer",
|
||||||
|
"--mamba-track-interval",
|
||||||
|
str(MAMBA_TRACK_INTERVAL),
|
||||||
|
],
|
||||||
|
env={"SGLANG_ENABLE_UNIFIED_RADIX_TREE": "1"},
|
||||||
|
)
|
||||||
|
cls.input_ids = get_input_ids(cls.model, num_samples=18)
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def tearDownClass(cls):
|
||||||
|
kill_process_tree(cls.process.pid)
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
unittest.main()
|
||||||
@@ -0,0 +1,56 @@
|
|||||||
|
import unittest
|
||||||
|
|
||||||
|
from sglang.srt.utils import kill_process_tree
|
||||||
|
from sglang.test.ci.ci_register import register_cuda_ci
|
||||||
|
from sglang.test.kits.unified_radix_cache_kit import UnifiedRadixTreeTestMixin
|
||||||
|
from sglang.test.kl_multiturn_utils import get_input_ids
|
||||||
|
from sglang.test.test_utils import (
|
||||||
|
DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
|
||||||
|
DEFAULT_URL_FOR_TEST,
|
||||||
|
CustomTestCase,
|
||||||
|
is_in_ci,
|
||||||
|
popen_launch_server,
|
||||||
|
)
|
||||||
|
|
||||||
|
register_cuda_ci(est_time=250, stage="base-c", runner_config="4-gpu-h100")
|
||||||
|
|
||||||
|
SWA_MODEL = "openai/gpt-oss-20b"
|
||||||
|
|
||||||
|
|
||||||
|
class TestUnifiedSWARadixCache(UnifiedRadixTreeTestMixin, CustomTestCase):
|
||||||
|
"""SWA hybrid + UnifiedRadixCache."""
|
||||||
|
|
||||||
|
kl_threshold = 0.03
|
||||||
|
gsm8k_threshold = 0.7
|
||||||
|
mmlu_threshold = 0.7
|
||||||
|
|
||||||
|
@unittest.skipIf(is_in_ci(), "SWA model mmlu eval not stable enough")
|
||||||
|
def test_mmlu(self):
|
||||||
|
super().test_mmlu()
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def setUpClass(cls):
|
||||||
|
cls.model = SWA_MODEL
|
||||||
|
cls.base_url = DEFAULT_URL_FOR_TEST
|
||||||
|
cls.process = popen_launch_server(
|
||||||
|
cls.model,
|
||||||
|
cls.base_url,
|
||||||
|
timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
|
||||||
|
other_args=[
|
||||||
|
"--tp-size",
|
||||||
|
"4",
|
||||||
|
"--mem-fraction-static",
|
||||||
|
"0.7",
|
||||||
|
"--disable-piecewise-cuda-graph",
|
||||||
|
],
|
||||||
|
env={"SGLANG_ENABLE_UNIFIED_RADIX_TREE": "1"},
|
||||||
|
)
|
||||||
|
cls.input_ids = get_input_ids(cls.model, num_samples=18)
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def tearDownClass(cls):
|
||||||
|
kill_process_tree(cls.process.pid)
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
unittest.main()
|
||||||
Reference in New Issue
Block a user