minke.yu/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 13 Packages Projects Releases Wiki Activity

[DSV4] Cherry pick missing commits from deepseek_v4 branch and enhance tests (#24793)

Browse Source 
Co-authored-by: Xinyuan Tong <115166877+JustinTong0323@users.noreply.github.com>
Co-authored-by: yueming-yuan <yym022502@gmail.com>
This commit is contained in:
Baizhou Zhang
2026-05-09 04:15:37 -07:00
committed by GitHub
co-authored by Xinyuan Tong yueming-yuan
parent 4b23f6bdc5
commit ef5e9f8aba
15 changed files with 481 additions and 87 deletions
Download Patch File Download Diff File Expand all files Collapse all files
test/registered/4-gpu-models/test_deepseek_v4_flash_fp4_b200_nightly.py → test/registered/dsv4/test_deepseek_v4_flash_fp4_b200.py
+65 -15
View File
@@ -1,14 +1,10 @@
"""B200 nightly CI: DeepSeek-V4-Flash FP4 (Balanced + MaxThroughput recipes).
"""B200 per-commit CI: DeepSeek-V4-Flash FP4 (LowLatency recipe).
Two server configurations exercise the DeepEP all-to-all + DP-attention path
that the per-commit LowLatency test does not cover.
Launches TP=4 with flashinfer_mxfp4 MoE runner + EAGLE speculative decoding.
Runs 12 ServerSanity probes (correctness, streaming, concurrency, determinism)
plus a GSM8K accuracy gate.
Balanced: TP=4, DP=4, DeepEP, EAGLE (1 step)
MaxThroughput: TP=4, DP=4, DeepEP, no speculation
Each class inherits 12 ServerSanity probes plus a GSM8K accuracy gate.
Registry: nightly-4-gpu-b200
Registry: stage-c-test-dsv4-4-gpu-b200 (per-commit, 4x B200)
"""
import unittest
@@ -25,7 +21,7 @@ from sglang.test.test_utils import (
try_cached_model,
)
register_cuda_ci(est_time=3600, suite="nightly-4-gpu-b200", nightly=True)
register_cuda_ci(est_time=1800, suite="stage-c-test-dsv4-4-gpu-b200")
MODEL = "deepseek-ai/DeepSeek-V4-Flash"
SERVER_LAUNCH_TIMEOUT = 3600
@@ -35,6 +31,14 @@ _DEEPEP_ENV = {
"SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK": "1024",
}
_MEGAMOE_ENV = {
"SGLANG_OPT_USE_DEEPGEMM_MEGA_MOE": "1",
"SGLANG_OPT_FIX_MEGA_MOE_MEMORY": "1",
"SGLANG_OPT_FIX_NEXTN_MEGA_MOE": "1",
"SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK": "4096",
"SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK": "0",
}
def _gsm8k_check(test_case):
args = SimpleNamespace(
@@ -51,6 +55,46 @@ def _gsm8k_check(test_case):
test_case.assertGreater(metrics["score"], 0.93)
class TestDSV4FlashFP4B200(ServerSanityMixin, CustomTestCase):
"""LowLatency recipe: TP=4, FP4 (mxfp4), EAGLE spec decoding."""
@classmethod
def setUpClass(cls):
cls.model = try_cached_model(MODEL)
cls.base_url = DEFAULT_URL_FOR_TEST
cls.process = popen_launch_server(
cls.model,
cls.base_url,
timeout=SERVER_LAUNCH_TIMEOUT,
other_args=[
"--trust-remote-code",
"--tp",
"4",
"--moe-runner-backend",
"flashinfer_mxfp4",
"--speculative-algorithm",
"EAGLE",
"--speculative-num-steps",
"3",
"--speculative-eagle-topk",
"1",
"--speculative-num-draft-tokens",
"4",
"--chunked-prefill-size",
"4096",
"--disable-flashinfer-autotune",
],
)
@classmethod
def tearDownClass(cls):
if hasattr(cls, "process") and cls.process:
kill_process_tree(cls.process.pid)
def test_gsm8k(self):
_gsm8k_check(self)
class TestDSV4FlashFP4B200Balanced(ServerSanityMixin, CustomTestCase):
"""Balanced recipe: TP=4, DP=4, DeepEP, EAGLE (1-step spec)."""
@@ -94,8 +138,8 @@ class TestDSV4FlashFP4B200Balanced(ServerSanityMixin, CustomTestCase):
_gsm8k_check(self)
class TestDSV4FlashFP4B200MaxThroughput(ServerSanityMixin, CustomTestCase):
"""MaxThroughput recipe: TP=4, DP=4, DeepEP, no speculation."""
class TestDSV4FlashFP4B200MegaMoE(ServerSanityMixin, CustomTestCase):
"""Balanced recipe: TP=4, DP=4, MegaMoE."""
@classmethod
def setUpClass(cls):
@@ -114,10 +158,16 @@ class TestDSV4FlashFP4B200MaxThroughput(ServerSanityMixin, CustomTestCase):
"--enable-dp-attention",
"--moe-a2a-backend",
"deepep",
"--deepep-config",
DEEPEP_CONFIG,
"--speculative-algorithm",
"EAGLE",
"--speculative-num-steps",
"1",
"--speculative-eagle-topk",
"1",
"--speculative-num-draft-tokens",
"2",
],
env=_DEEPEP_ENV,
env=_MEGAMOE_ENV,
)
@classmethod
test/registered/8-gpu-models/test_deepseek_v4_flash_fp4_h200.py → test/registered/dsv4/test_deepseek_v4_flash_fp4_h200.py
+2
View File
@@ -24,7 +24,9 @@ from sglang.test.test_utils import (
register_cuda_ci(est_time=900, suite="stage-c-test-dsv4-8-gpu-h200")
MODEL = "deepseek-ai/DeepSeek-V4-Flash"
MODEL_FP8 = "sgl-project/DeepSeek-V4-Flash-FP8"
SERVER_LAUNCH_TIMEOUT = 3600
DEEPEP_CONFIG = '{"normal_dispatch":{"num_sms":96},"normal_combine":{"num_sms":96}}'
class TestDSV4FlashFP4H200(ServerSanityMixin, CustomTestCase):
test/registered/4-gpu-models/test_deepseek_v4_flash_fp4_b200.py → test/registered/dsv4/test_deepseek_v4_flash_fp8_h200.py
+28 -16
View File
@@ -1,10 +1,11 @@
"""B200 per-commit CI: DeepSeek-V4-Flash FP4 (LowLatency recipe).
"""H200 per-commit CI: DeepSeek-V4-Flash FP8 (LowLatency recipe).
Launches TP=4 with flashinfer_mxfp4 MoE runner + EAGLE speculative decoding.
Launches TP=4 with DeepEP a2a backend + EAGLE speculative decoding,
with FP4 experts disabled via SGLANG_DSV4_FP4_EXPERTS=0.
Runs 12 ServerSanity probes (correctness, streaming, concurrency, determinism)
plus a GSM8K accuracy gate.
Registry: stage-c-test-dsv4-4-gpu-b200 (per-commit, 4x B200)
Registry: stage-c-test-dsv4-8-gpu-h200 (per-commit, 8x H200 — only 4 used by TP=4)
"""
import unittest
@@ -21,18 +22,19 @@ from sglang.test.test_utils import (
try_cached_model,
)
register_cuda_ci(est_time=900, suite="stage-c-test-dsv4-4-gpu-b200")
register_cuda_ci(est_time=900, suite="stage-c-test-dsv4-8-gpu-h200")
MODEL = "deepseek-ai/DeepSeek-V4-Flash"
MODEL_FP8 = "sgl-project/DeepSeek-V4-Flash-FP8"
SERVER_LAUNCH_TIMEOUT = 3600
DEEPEP_CONFIG = '{"normal_dispatch":{"num_sms":96},"normal_combine":{"num_sms":96}}'
class TestDSV4FlashFP4B200(ServerSanityMixin, CustomTestCase):
"""LowLatency recipe: TP=4, FP4 (mxfp4), EAGLE spec decoding."""
class TestDSV4FlashFP8H200(ServerSanityMixin, CustomTestCase):
"""LowLatency recipe: TP=4, Marlin FP4, EAGLE spec decoding."""
@classmethod
def setUpClass(cls):
cls.model = try_cached_model(MODEL)
cls.model = try_cached_model(MODEL_FP8)
cls.base_url = DEFAULT_URL_FOR_TEST
cls.process = popen_launch_server(
cls.model,
@@ -42,20 +44,30 @@ class TestDSV4FlashFP4B200(ServerSanityMixin, CustomTestCase):
"--trust-remote-code",
"--tp",
"4",
"--moe-runner-backend",
"flashinfer_mxfp4",
"--dp",
"4",
"--enable-dp-attention",
"--moe-a2a-backend",
"deepep",
"--speculative-algorithm",
"EAGLE",
"--speculative-num-steps",
"3",
"1",
"--speculative-eagle-topk",
"1",
"--speculative-num-draft-tokens",
"4",
"--chunked-prefill-size",
"4096",
"--disable-flashinfer-autotune",
"2",
"--cuda-graph-max-bs",
"128",
"--max-running-requests",
"128",
"--deepep-config",
DEEPEP_CONFIG,
],
env={
"SGLANG_DSV4_FP4_EXPERTS": "0",
"SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK": "256",
},
)
@classmethod
@@ -74,7 +86,7 @@ class TestDSV4FlashFP4B200(ServerSanityMixin, CustomTestCase):
num_threads=128,
)
metrics = run_eval(args)
print(f"[DSV4 Flash FP4 B200] GSM8K {metrics=}")
print(f"[DSV4 Flash FP4 Marlin H200] GSM8K {metrics=}")
self.assertGreater(metrics["score"], 0.93)
test/registered/unit/entrypoints/openai/test_protocol.py
+31
View File
@@ -220,6 +220,37 @@ class TestChatCompletionRequest(unittest.TestCase):
self.assertFalse(request.chat_template_kwargs.get("thinking"))
self.assertFalse(request.chat_template_kwargs.get("enable_thinking"))
def test_chat_completion_reasoning_effort_max(self):
"""`max` is an sglang extension on chat completion's top-level
`reasoning_effort` only; the Responses-API-style nested
`reasoning.effort` path stays aligned with OpenAI's three levels."""
from pydantic import ValidationError
messages = [{"role": "user", "content": "Hello"}]
request = ChatCompletionRequest(
model="test-model",
messages=messages,
reasoning_effort="max",
)
self.assertEqual(request.reasoning_effort, "max")
# Unknown values still rejected.
with self.assertRaises(ValidationError):
ChatCompletionRequest(
model="test-model",
messages=messages,
reasoning_effort="ultra",
)
# Nested reasoning.effort=max is NOT promoted by normalize_reasoning_inputs:
# the Responses API path keeps the OpenAI low/medium/high contract.
request = ChatCompletionRequest(
model="test-model",
messages=messages,
reasoning={"effort": "max"},
)
self.assertNotEqual(request.reasoning_effort, "max")
def test_chat_completion_json_format(self):
"""Test chat completion json format"""
transcript = "Good morning! It's 7:00 AM, and I'm just waking up. Today is going to be a busy day, "
test/registered/unit/function_call/test_function_call_parser.py
+111 -1
View File
@@ -31,7 +31,7 @@ from sglang.srt.function_call.pythonic_detector import PythonicDetector
from sglang.srt.function_call.qwen3_coder_detector import Qwen3CoderDetector
from sglang.test.ci.ci_register import register_cpu_ci
register_cpu_ci(15, "stage-a-test-cpu")
register_cpu_ci(est_time=15, suite="stage-a-test-cpu")
class TestPythonicDetector(unittest.TestCase):
@@ -1686,6 +1686,26 @@ class TestDeepSeekV32Detector(unittest.TestCase):
grammar = xgr.Grammar.from_structural_tag(structural_tag)
self.assertIsInstance(grammar, xgr.Grammar)
def test_self_closing_zero_arg_invoke(self):
"""V32 inherits the same regex; verify self-closing parses to empty
params here too (V32 model rarely emits this shape, but the parser
must agree with V4 since V4 inherits from V32)."""
submit_tool = Tool(
type="function",
function=Function(
name="submit",
parameters={"type": "object", "properties": {}},
),
)
text = (
'<｜DSML｜function_calls>\n<｜DSML｜invoke name="submit"/>\n'
"</｜DSML｜function_calls>"
)
result = self.detector.detect_and_parse(text, [submit_tool])
self.assertEqual(len(result.calls), 1)
self.assertEqual(result.calls[0].name, "submit")
self.assertEqual(json.loads(result.calls[0].parameters), {})
class TestDeepSeekV4Detector(unittest.TestCase):
def setUp(self):
@@ -2111,6 +2131,96 @@ class TestDeepSeekV4Detector(unittest.TestCase):
grammar = xgr.Grammar.from_structural_tag(structural_tag)
self.assertIsInstance(grammar, xgr.Grammar)
def test_self_closing_zero_arg_invoke(self):
"""V4 emits `<｜DSML｜invoke name="x"/>` for zero-arg tools; the
detector must parse it as a complete tool call with empty params
instead of leaking the raw markup back into normal_text."""
submit_tool = Tool(
type="function",
function=Function(
name="submit",
description="Submit the final answer.",
parameters={"type": "object", "properties": {}},
),
)
text = (
"Final answer.\n"
'<｜DSML｜tool_calls>\n<｜DSML｜invoke name="submit"/>\n'
"</｜DSML｜tool_calls>"
)
result = self.detector.detect_and_parse(text, [submit_tool])
self.assertEqual(len(result.calls), 1)
self.assertEqual(result.calls[0].name, "submit")
self.assertEqual(json.loads(result.calls[0].parameters), {})
self.assertNotIn("DSML", result.normal_text)
def test_self_closing_mixed_with_long_form(self):
"""Mix of long-form (with params) and self-closing tags in one block."""
submit_tool = Tool(
type="function",
function=Function(
name="submit",
parameters={"type": "object", "properties": {}},
),
)
text = (
"<｜DSML｜tool_calls>\n"
'<｜DSML｜invoke name="get_favorite_tourist_spot">\n'
'<｜DSML｜parameter name="city" string="true">SF</｜DSML｜parameter>\n'
"</｜DSML｜invoke>\n"
'<｜DSML｜invoke name="submit"/>\n'
"</｜DSML｜tool_calls>"
)
result = self.detector.detect_and_parse(text, self.tools + [submit_tool])
self.assertEqual(len(result.calls), 2)
self.assertEqual(result.calls[0].name, "get_favorite_tourist_spot")
self.assertEqual(json.loads(result.calls[0].parameters), {"city": "SF"})
self.assertEqual(result.calls[1].name, "submit")
self.assertEqual(json.loads(result.calls[1].parameters), {})
def test_streaming_self_closing_invoke(self):
"""Self-closing invoke must terminate cleanly even when `/>` arrives
after the `name=` attribute crosses chunk boundaries."""
submit_tool = Tool(
type="function",
function=Function(
name="submit",
parameters={"type": "object", "properties": {}},
),
)
# Build the prompt and feed it through the tokenizer to exercise the
# same chunk shapes the runtime sees.
text = (
"<｜DSML｜tool_calls>\n"
'<｜DSML｜invoke name="submit"/>\n'
"</｜DSML｜tool_calls>"
)
self.detector = DeepSeekV4Detector()
input_ids = self.tokenizer.encode(text, add_special_tokens=False)
chunks = [
self.tokenizer.decode(input_ids[i : i + self.interval])
for i in range(0, len(input_ids), self.interval)
]
tool_calls_by_index = {}
for chunk in chunks:
result = self.detector.parse_streaming_increment(chunk, [submit_tool])
for call in result.calls:
if call.tool_index is None:
continue
slot = tool_calls_by_index.setdefault(
call.tool_index, {"name": "", "parameters": ""}
)
if call.name:
slot["name"] = call.name
if call.parameters:
slot["parameters"] += call.parameters
self.assertEqual(len(tool_calls_by_index), 1)
self.assertEqual(tool_calls_by_index[0]["name"], "submit")
self.assertEqual(json.loads(tool_calls_by_index[0]["parameters"]), {})
class TestQwen3CoderDetector(unittest.TestCase):
"""Test suite for Qwen3CoderDetector."""