[CI][RFC] Replace black-jupyter with ruff-format (#37210)
Co-authored-by: Alison Shao <a.shao@wustl.edu>
This commit is contained in:
co-authored by
Alison Shao
parent
2641e427be
commit
28262c20df
@@ -36,9 +36,12 @@ def _make_target_verify_batch(bs: int) -> ForwardBatch:
|
||||
def _filter(batch: ForwardBatch, *, lo: int, hi: int) -> ForwardBatch:
|
||||
# filter_batch reads attention_backend (get_server_args) and
|
||||
# moe_dense_tp_size (get_parallel) from the published config.
|
||||
with get_context().override_server_args(
|
||||
attention_backend="fa3", moe_dense_tp_size=None
|
||||
), get_parallel().override(attn_tp_size=1):
|
||||
with (
|
||||
get_context().override_server_args(
|
||||
attention_backend="fa3", moe_dense_tp_size=None
|
||||
),
|
||||
get_parallel().override(attn_tp_size=1),
|
||||
):
|
||||
return TboForwardBatchPreparer.filter_batch(
|
||||
batch,
|
||||
start_token_index=lo,
|
||||
|
||||
@@ -123,12 +123,7 @@ class TestMMMUEvalUtils(CustomTestCase):
|
||||
self,
|
||||
):
|
||||
response = (
|
||||
"The options are:\n"
|
||||
"(A) red\n"
|
||||
"(B) blue\n"
|
||||
"(C) green\n"
|
||||
"(D) yellow\n"
|
||||
"Answer: B"
|
||||
"The options are:\n(A) red\n(B) blue\n(C) green\n(D) yellow\nAnswer: B"
|
||||
)
|
||||
|
||||
pred_ans = self.eval_utils.parse_multi_choice_response(
|
||||
|
||||
@@ -176,7 +176,9 @@ class TestMultimodalPiecewiseCudaGraph(CustomTestCase):
|
||||
args.disable_radix_cache = False
|
||||
args.chunked_prefill_size = 2048
|
||||
|
||||
with (override_platform(is_cuda=True),):
|
||||
with (
|
||||
override_platform(is_cuda=True),
|
||||
):
|
||||
handle_model_capability_adjustments(args)
|
||||
|
||||
self.assertTrue(resolution_result(args, "disable_radix_cache"))
|
||||
|
||||
@@ -303,8 +303,7 @@ class TestDecodeQueueCleanup(CustomTestCase):
|
||||
tail = decode_req(8)
|
||||
queue.pending_reqs.append(tail)
|
||||
with patch(
|
||||
"sglang.srt.disaggregation.decode."
|
||||
"CommonKVReceiver.query_prefill_dp_ranks",
|
||||
"sglang.srt.disaggregation.decode.CommonKVReceiver.query_prefill_dp_ranks",
|
||||
return_value={"8": 2},
|
||||
) as query:
|
||||
queue._resolve_pending_reqs()
|
||||
|
||||
@@ -445,12 +445,13 @@ class TestEagleDsaSeedTransfer(unittest.TestCase):
|
||||
# positions are passed through unremapped.
|
||||
("other", False, False, False, unremapped),
|
||||
):
|
||||
with self.subTest(platform=platform), envs.SGLANG_DSA_FUSE_TOPK.override(
|
||||
True
|
||||
), patch(
|
||||
"sglang.srt.layers.attention.dsa.utils.is_cuda", return_value=cuda
|
||||
), patch(
|
||||
"sglang.srt.layers.attention.dsa.utils.is_hip", return_value=hip
|
||||
with (
|
||||
self.subTest(platform=platform),
|
||||
envs.SGLANG_DSA_FUSE_TOPK.override(True),
|
||||
patch(
|
||||
"sglang.srt.layers.attention.dsa.utils.is_cuda", return_value=cuda
|
||||
),
|
||||
patch("sglang.srt.layers.attention.dsa.utils.is_hip", return_value=hip),
|
||||
):
|
||||
self.assertEqual(
|
||||
should_use_dsa_fused_topk(seed_dsa_topk_from_draft_extend=True),
|
||||
|
||||
@@ -1032,8 +1032,8 @@ class TestNixlStaging(CustomTestCase):
|
||||
staging_total_size=4096,
|
||||
)
|
||||
calls = []
|
||||
mgr.send_kvcache_staged = (
|
||||
lambda *args, **kwargs: calls.append((args, kwargs)) or "handle"
|
||||
mgr.send_kvcache_staged = lambda *args, **kwargs: (
|
||||
calls.append((args, kwargs)) or "handle"
|
||||
)
|
||||
|
||||
handle, deferred = mgr._do_staging_transfer(
|
||||
|
||||
@@ -123,7 +123,6 @@ def test_parallel_group_construction_tp8_attn_cp2():
|
||||
patch("torch.distributed.get_rank", return_value=0),
|
||||
patch("torch.distributed.get_backend", return_value="nccl"),
|
||||
):
|
||||
|
||||
# Mock init_model_parallel_group to capture the groups being created
|
||||
created_groups = {}
|
||||
|
||||
@@ -144,7 +143,6 @@ def test_parallel_group_construction_tp8_attn_cp2():
|
||||
),
|
||||
patch.object(parallel_state, "get_world_group") as mock_world_group,
|
||||
):
|
||||
|
||||
# Mock world group
|
||||
mock_world = Mock()
|
||||
mock_world.device_group = Mock()
|
||||
@@ -174,18 +172,18 @@ def test_parallel_group_construction_tp8_attn_cp2():
|
||||
|
||||
# Verify ATTN_CP groups
|
||||
attn_cp_groups = created_groups.get("attn_cp", [])
|
||||
assert (
|
||||
len(attn_cp_groups) == 4
|
||||
), f"Expected 4 ATTN_CP groups, got {len(attn_cp_groups)}"
|
||||
assert len(attn_cp_groups) == 4, (
|
||||
f"Expected 4 ATTN_CP groups, got {len(attn_cp_groups)}"
|
||||
)
|
||||
expected_attn_cp = [
|
||||
[0, 4],
|
||||
[1, 5],
|
||||
[2, 6],
|
||||
[3, 7],
|
||||
]
|
||||
assert (
|
||||
attn_cp_groups == expected_attn_cp
|
||||
), f"Wrong ATTN_CP groups: {attn_cp_groups}"
|
||||
assert attn_cp_groups == expected_attn_cp, (
|
||||
f"Wrong ATTN_CP groups: {attn_cp_groups}"
|
||||
)
|
||||
|
||||
print("TP=8, Attn CP=2 group construction verified")
|
||||
|
||||
@@ -223,7 +221,6 @@ def test_parallel_group_construction_tp8_moe_ep4_cp2():
|
||||
patch("torch.distributed.get_rank", return_value=0),
|
||||
patch("torch.distributed.get_backend", return_value="nccl"),
|
||||
):
|
||||
|
||||
# Mock init_model_parallel_group to capture the groups being created
|
||||
created_groups = {}
|
||||
|
||||
@@ -244,7 +241,6 @@ def test_parallel_group_construction_tp8_moe_ep4_cp2():
|
||||
),
|
||||
patch.object(parallel_state, "get_world_group") as mock_world_group,
|
||||
):
|
||||
|
||||
# Mock world group
|
||||
mock_world = Mock()
|
||||
mock_world.device_group = Mock()
|
||||
@@ -275,31 +271,31 @@ def test_parallel_group_construction_tp8_moe_ep4_cp2():
|
||||
|
||||
# Verify MOE_EP groups
|
||||
moe_ep_groups = created_groups.get("moe_ep", [])
|
||||
assert (
|
||||
len(moe_ep_groups) == 2
|
||||
), f"Expected 2 MOE_EP groups, got {len(moe_ep_groups)}"
|
||||
assert len(moe_ep_groups) == 2, (
|
||||
f"Expected 2 MOE_EP groups, got {len(moe_ep_groups)}"
|
||||
)
|
||||
expected_moe_ep = [
|
||||
[0, 1, 2, 3],
|
||||
[4, 5, 6, 7],
|
||||
]
|
||||
assert (
|
||||
moe_ep_groups == expected_moe_ep
|
||||
), f"Wrong MOE_EP groups: {moe_ep_groups}"
|
||||
assert moe_ep_groups == expected_moe_ep, (
|
||||
f"Wrong MOE_EP groups: {moe_ep_groups}"
|
||||
)
|
||||
|
||||
# Verify MOE_DP groups
|
||||
moe_dp_groups = created_groups.get("moe_dp", [])
|
||||
assert (
|
||||
len(moe_dp_groups) == 4
|
||||
), f"Expected 4 MOE_DP groups, got {len(moe_dp_groups)}"
|
||||
assert len(moe_dp_groups) == 4, (
|
||||
f"Expected 4 MOE_DP groups, got {len(moe_dp_groups)}"
|
||||
)
|
||||
expected_moe_dp = [
|
||||
[0, 4],
|
||||
[1, 5],
|
||||
[2, 6],
|
||||
[3, 7],
|
||||
]
|
||||
assert (
|
||||
moe_dp_groups == expected_moe_dp
|
||||
), f"Wrong MOE_DP groups: {moe_dp_groups}"
|
||||
assert moe_dp_groups == expected_moe_dp, (
|
||||
f"Wrong MOE_DP groups: {moe_dp_groups}"
|
||||
)
|
||||
|
||||
print("TP=8, MoE EP=4, MoE CP=2 group construction verified")
|
||||
|
||||
|
||||
@@ -369,8 +369,9 @@ class TestChatCompletionRequest(unittest.TestCase):
|
||||
{"reasoning": {"effort": 1.1}},
|
||||
{"reasoning": {"effort": "1.5"}},
|
||||
):
|
||||
with self.subTest(request_kwargs=request_kwargs), self.assertRaises(
|
||||
ValidationError
|
||||
with (
|
||||
self.subTest(request_kwargs=request_kwargs),
|
||||
self.assertRaises(ValidationError),
|
||||
):
|
||||
ChatCompletionRequest(
|
||||
model="test-model", messages=messages, **request_kwargs
|
||||
|
||||
@@ -137,9 +137,9 @@ def _expanded_write_keys(rel: str, tree: ast.AST, call: ast.Call, kw: ast.keywor
|
||||
if isinstance(key, ast.Constant):
|
||||
keys.add(key.value)
|
||||
continue
|
||||
assert isinstance(
|
||||
key, ast.Name
|
||||
), f"non-literal dict key in a writer expansion at {rel}:{call.lineno}"
|
||||
assert isinstance(key, ast.Name), (
|
||||
f"non-literal dict key in a writer expansion at {rel}:{call.lineno}"
|
||||
)
|
||||
bound = loop_bound(key.id)
|
||||
assert bound, (
|
||||
f"dict key {key.id!r} at {rel}:{call.lineno} is not bound by a "
|
||||
@@ -150,12 +150,12 @@ def _expanded_write_keys(rel: str, tree: ast.AST, call: ast.Call, kw: ast.keywor
|
||||
|
||||
if isinstance(kw.value, ast.Dict):
|
||||
return dict_keys(kw.value)
|
||||
assert isinstance(
|
||||
kw.value, ast.Name
|
||||
), f"unresolvable writer expansion at {rel}:{call.lineno}"
|
||||
assert (
|
||||
enclosing is not None
|
||||
), f"writer expansion outside any function at {rel}:{call.lineno}"
|
||||
assert isinstance(kw.value, ast.Name), (
|
||||
f"unresolvable writer expansion at {rel}:{call.lineno}"
|
||||
)
|
||||
assert enclosing is not None, (
|
||||
f"writer expansion outside any function at {rel}:{call.lineno}"
|
||||
)
|
||||
name = kw.value.id
|
||||
if enclosing.args.kwarg is not None and enclosing.args.kwarg.arg == name:
|
||||
return set()
|
||||
@@ -349,7 +349,7 @@ class TestEffectiveStateSurfaces(CustomTestCase):
|
||||
self.assertEqual(
|
||||
missing,
|
||||
{},
|
||||
"a serving surface cannot report what it is running: " f"{missing}",
|
||||
f"a serving surface cannot report what it is running: {missing}",
|
||||
)
|
||||
|
||||
def test_every_surface_reports_the_manager_owned_identity(self):
|
||||
|
||||
@@ -373,7 +373,7 @@ class TestInklingDetector(unittest.TestCase):
|
||||
self.assertEqual(result.normal_text, "Here you go.")
|
||||
|
||||
def test_empty_name_is_allowed_on_the_canonical_path(self):
|
||||
source = "<|content_invoke_tool_json|>" '{"name":"","args":{}}<|end_message|>'
|
||||
source = '<|content_invoke_tool_json|>{"name":"","args":{}}<|end_message|>'
|
||||
result = InklingDetector().detect_and_parse(source, self.tools)
|
||||
self.assertEqual(len(result.calls), 1)
|
||||
self.assertEqual(result.calls[0].name, "")
|
||||
@@ -1305,7 +1305,6 @@ class TestLlama32Detector(unittest.TestCase):
|
||||
|
||||
|
||||
class TestKimiK2Detector(unittest.TestCase):
|
||||
|
||||
def setUp(self):
|
||||
"""Set up test tools and detector."""
|
||||
self.tools = [
|
||||
@@ -1374,7 +1373,6 @@ class TestKimiK2Detector(unittest.TestCase):
|
||||
result = self.detector.parse_streaming_increment(chunk, self.tools)
|
||||
for tool_call_chunk in result.calls:
|
||||
if tool_call_chunk.tool_index is not None:
|
||||
|
||||
while len(tool_calls) <= tool_call_chunk.tool_index:
|
||||
tool_calls.append({"name": "", "parameters": ""})
|
||||
|
||||
@@ -1423,7 +1421,6 @@ class TestKimiK2Detector(unittest.TestCase):
|
||||
result = self.detector.parse_streaming_increment(chunk, self.tools)
|
||||
for tool_call_chunk in result.calls:
|
||||
if tool_call_chunk.tool_index is not None:
|
||||
|
||||
while len(tool_calls) <= tool_call_chunk.tool_index:
|
||||
tool_calls.append({"name": "", "parameters": ""})
|
||||
|
||||
@@ -1458,7 +1455,6 @@ class TestKimiK2Detector(unittest.TestCase):
|
||||
result = self.detector.parse_streaming_increment(chunk, self.tools)
|
||||
for tool_call_chunk in result.calls:
|
||||
if tool_call_chunk.tool_index is not None:
|
||||
|
||||
while len(tool_calls) <= tool_call_chunk.tool_index:
|
||||
tool_calls.append({"name": "", "parameters": ""})
|
||||
|
||||
@@ -1531,7 +1527,7 @@ class TestDeepSeekV3Detector(unittest.TestCase):
|
||||
"function<|tool▁sep|>",
|
||||
"get_tour",
|
||||
"ist_att",
|
||||
"ractions\n```" 'json\n{"',
|
||||
'ractions\n```json\n{"',
|
||||
'city": "',
|
||||
'Beijing"}\n',
|
||||
"```<|tool▁call▁end|>",
|
||||
@@ -1718,9 +1714,9 @@ class TestDeepSeekV32Detector(unittest.TestCase):
|
||||
if call.name:
|
||||
tool_calls_by_index[call.tool_index]["name"] = call.name
|
||||
if call.parameters:
|
||||
tool_calls_by_index[call.tool_index][
|
||||
"parameters"
|
||||
] += call.parameters
|
||||
tool_calls_by_index[call.tool_index]["parameters"] += (
|
||||
call.parameters
|
||||
)
|
||||
|
||||
self.assertGreater(num_tool_call_chunks, 8)
|
||||
|
||||
@@ -1773,9 +1769,9 @@ class TestDeepSeekV32Detector(unittest.TestCase):
|
||||
if call.name:
|
||||
tool_calls_by_index[call.tool_index]["name"] = call.name
|
||||
if call.parameters:
|
||||
tool_calls_by_index[call.tool_index][
|
||||
"parameters"
|
||||
] += call.parameters
|
||||
tool_calls_by_index[call.tool_index]["parameters"] += (
|
||||
call.parameters
|
||||
)
|
||||
|
||||
self.assertGreater(num_tool_call_chunks, 8)
|
||||
self.assertEqual(len(tool_calls_by_index), 1)
|
||||
@@ -1866,9 +1862,9 @@ class TestDeepSeekV32Detector(unittest.TestCase):
|
||||
if call.name:
|
||||
tool_calls_by_index[call.tool_index]["name"] = call.name
|
||||
if call.parameters:
|
||||
tool_calls_by_index[call.tool_index][
|
||||
"parameters"
|
||||
] += call.parameters
|
||||
tool_calls_by_index[call.tool_index]["parameters"] += (
|
||||
call.parameters
|
||||
)
|
||||
|
||||
# Verify that the no-parameter function was correctly parsed
|
||||
self.assertEqual(
|
||||
@@ -1926,9 +1922,9 @@ class TestDeepSeekV32Detector(unittest.TestCase):
|
||||
if call.name:
|
||||
tool_calls_by_index[call.tool_index]["name"] = call.name
|
||||
if call.parameters:
|
||||
tool_calls_by_index[call.tool_index][
|
||||
"parameters"
|
||||
] += call.parameters
|
||||
tool_calls_by_index[call.tool_index]["parameters"] += (
|
||||
call.parameters
|
||||
)
|
||||
|
||||
# Should still parse correctly even with whitespace-only content
|
||||
self.assertEqual(
|
||||
@@ -2128,9 +2124,9 @@ class TestDeepSeekV4Detector(unittest.TestCase):
|
||||
if call.name:
|
||||
tool_calls_by_index[call.tool_index]["name"] = call.name
|
||||
if call.parameters:
|
||||
tool_calls_by_index[call.tool_index][
|
||||
"parameters"
|
||||
] += call.parameters
|
||||
tool_calls_by_index[call.tool_index]["parameters"] += (
|
||||
call.parameters
|
||||
)
|
||||
|
||||
self.assertGreater(num_tool_call_chunks, 8)
|
||||
|
||||
@@ -4500,9 +4496,7 @@ class TestLfm2Detector(unittest.TestCase):
|
||||
def test_reserved_kwarg_with_nested_quote_recovered(self):
|
||||
"""A keyword-named parameter holding a nested-quote command needs
|
||||
the rename and requote rewrites to compose."""
|
||||
text = (
|
||||
"<|tool_call_start|>[search(from='sed -n '1,5p' f.py')]" "<|tool_call_end|>"
|
||||
)
|
||||
text = "<|tool_call_start|>[search(from='sed -n '1,5p' f.py')]<|tool_call_end|>"
|
||||
result = self.detector.detect_and_parse(text, self.tools)
|
||||
|
||||
self.assertEqual(len(result.calls), 1)
|
||||
@@ -4720,9 +4714,9 @@ function call<|role_sep|>
|
||||
if call.name:
|
||||
tool_calls_by_index[call.tool_index]["name"] = call.name
|
||||
if call.parameters:
|
||||
tool_calls_by_index[call.tool_index][
|
||||
"parameters"
|
||||
] += call.parameters
|
||||
tool_calls_by_index[call.tool_index]["parameters"] += (
|
||||
call.parameters
|
||||
)
|
||||
|
||||
self.assertEqual(len(tool_calls_by_index), 1)
|
||||
self.assertEqual(tool_calls_by_index[0]["name"], "manage_user_memory")
|
||||
@@ -4762,9 +4756,9 @@ function call<|role_sep|>
|
||||
if call.name:
|
||||
tool_calls_by_index[call.tool_index]["name"] = call.name
|
||||
if call.parameters:
|
||||
tool_calls_by_index[call.tool_index][
|
||||
"parameters"
|
||||
] += call.parameters
|
||||
tool_calls_by_index[call.tool_index]["parameters"] += (
|
||||
call.parameters
|
||||
)
|
||||
|
||||
self.assertEqual(accumulated_text, "I'll help you.")
|
||||
self.assertEqual(len(tool_calls_by_index), 1)
|
||||
@@ -4802,9 +4796,9 @@ function call<|role_sep|>
|
||||
if call.name:
|
||||
tool_calls_by_index[call.tool_index]["name"] = call.name
|
||||
if call.parameters:
|
||||
tool_calls_by_index[call.tool_index][
|
||||
"parameters"
|
||||
] += call.parameters
|
||||
tool_calls_by_index[call.tool_index]["parameters"] += (
|
||||
call.parameters
|
||||
)
|
||||
|
||||
self.assertEqual(len(tool_calls_by_index), 1)
|
||||
self.assertEqual(tool_calls_by_index[0]["name"], "manage_user_memory")
|
||||
@@ -4841,9 +4835,9 @@ function call<|role_sep|>
|
||||
if call.name:
|
||||
tool_calls_by_index[call.tool_index]["name"] = call.name
|
||||
if call.parameters:
|
||||
tool_calls_by_index[call.tool_index][
|
||||
"parameters"
|
||||
] += call.parameters
|
||||
tool_calls_by_index[call.tool_index]["parameters"] += (
|
||||
call.parameters
|
||||
)
|
||||
|
||||
self.assertEqual(len(tool_calls_by_index), 1)
|
||||
self.assertEqual(tool_calls_by_index[0]["name"], "get_weather")
|
||||
@@ -4878,9 +4872,9 @@ function call<|role_sep|>
|
||||
if call.name:
|
||||
tool_calls_by_index[call.tool_index]["name"] = call.name
|
||||
if call.parameters:
|
||||
tool_calls_by_index[call.tool_index][
|
||||
"parameters"
|
||||
] += call.parameters
|
||||
tool_calls_by_index[call.tool_index]["parameters"] += (
|
||||
call.parameters
|
||||
)
|
||||
|
||||
# Should have name but incomplete parameters
|
||||
self.assertEqual(len(tool_calls_by_index), 1)
|
||||
@@ -4916,9 +4910,9 @@ function call<|role_sep|>
|
||||
if call.name:
|
||||
tool_calls_by_index[call.tool_index]["name"] = call.name
|
||||
if call.parameters:
|
||||
tool_calls_by_index[call.tool_index][
|
||||
"parameters"
|
||||
] += call.parameters
|
||||
tool_calls_by_index[call.tool_index]["parameters"] += (
|
||||
call.parameters
|
||||
)
|
||||
|
||||
self.assertEqual(accumulated_text, "I'll remember that.")
|
||||
self.assertEqual(len(tool_calls_by_index), 1)
|
||||
@@ -4961,9 +4955,9 @@ function call<|role_sep|>
|
||||
if call.name:
|
||||
tool_calls_by_index[call.tool_index]["name"] = call.name
|
||||
if call.parameters:
|
||||
tool_calls_by_index[call.tool_index][
|
||||
"parameters"
|
||||
] += call.parameters
|
||||
tool_calls_by_index[call.tool_index]["parameters"] += (
|
||||
call.parameters
|
||||
)
|
||||
|
||||
self.assertEqual(len(tool_calls_by_index), 1)
|
||||
self.assertEqual(tool_calls_by_index[0]["name"], "get_weather")
|
||||
@@ -5011,9 +5005,9 @@ function call<|role_sep|>
|
||||
if call.name:
|
||||
tool_calls_by_index[call.tool_index]["name"] = call.name
|
||||
if call.parameters:
|
||||
tool_calls_by_index[call.tool_index][
|
||||
"parameters"
|
||||
] += call.parameters
|
||||
tool_calls_by_index[call.tool_index]["parameters"] += (
|
||||
call.parameters
|
||||
)
|
||||
|
||||
self.assertEqual(accumulated_text, "I'll help you.")
|
||||
self.assertEqual(len(tool_calls_by_index), 1)
|
||||
@@ -5265,9 +5259,9 @@ class TestQwen25Detector(unittest.TestCase):
|
||||
if call.name:
|
||||
tool_calls_by_index[call.tool_index]["name"] = call.name
|
||||
if call.parameters:
|
||||
tool_calls_by_index[call.tool_index][
|
||||
"parameters"
|
||||
] += call.parameters
|
||||
tool_calls_by_index[call.tool_index]["parameters"] += (
|
||||
call.parameters
|
||||
)
|
||||
return tool_calls_by_index
|
||||
|
||||
def test_streaming_multiple_tool_calls(self):
|
||||
@@ -5515,9 +5509,9 @@ class TestGemma4Detector(unittest.TestCase):
|
||||
if call.name:
|
||||
tool_calls_by_index[call.tool_index]["name"] = call.name
|
||||
if call.parameters:
|
||||
tool_calls_by_index[call.tool_index][
|
||||
"parameters"
|
||||
] += call.parameters
|
||||
tool_calls_by_index[call.tool_index]["parameters"] += (
|
||||
call.parameters
|
||||
)
|
||||
return normal_text, tool_calls_by_index
|
||||
|
||||
def test_streaming_multiple_tool_calls(self):
|
||||
@@ -5556,9 +5550,9 @@ class TestGemma4Detector(unittest.TestCase):
|
||||
if call.name:
|
||||
tool_calls_by_index[call.tool_index]["name"] = call.name
|
||||
if call.parameters:
|
||||
tool_calls_by_index[call.tool_index][
|
||||
"parameters"
|
||||
] += call.parameters
|
||||
tool_calls_by_index[call.tool_index]["parameters"] += (
|
||||
call.parameters
|
||||
)
|
||||
|
||||
self.assertEqual(len(tool_calls_by_index), 2)
|
||||
self.assertEqual(tool_calls_by_index[0]["name"], "get_weather")
|
||||
@@ -5623,9 +5617,9 @@ class TestGemma4Detector(unittest.TestCase):
|
||||
if call.name:
|
||||
tool_calls_by_index[call.tool_index]["name"] = call.name
|
||||
if call.parameters:
|
||||
tool_calls_by_index[call.tool_index][
|
||||
"parameters"
|
||||
] += call.parameters
|
||||
tool_calls_by_index[call.tool_index]["parameters"] += (
|
||||
call.parameters
|
||||
)
|
||||
self.assertIn("Hello!", normal_text)
|
||||
self.assertIn("Let me also check", normal_text)
|
||||
self.assertEqual(len(tool_calls_by_index), 2)
|
||||
|
||||
@@ -351,9 +351,7 @@ class TestHunyuanDetectorStreaming(CustomTestCase):
|
||||
def test_complete_tool_call_single_chunk(self):
|
||||
detector = self._new_detector()
|
||||
text = (
|
||||
"<tool_calls>"
|
||||
"<tool_call>get_current_date<tool_sep></tool_call>"
|
||||
"</tool_calls>"
|
||||
"<tool_calls><tool_call>get_current_date<tool_sep></tool_call></tool_calls>"
|
||||
)
|
||||
result = detector.parse_streaming_increment(text, self.tools)
|
||||
collected = _collect_streamed_tool_calls(result.calls)
|
||||
|
||||
@@ -197,7 +197,7 @@ def test_strict_schema_rejects_invalid_parameters(arguments):
|
||||
|
||||
def test_required_allows_response_prefix_but_requires_tools():
|
||||
grammar = _grammar([_tool()], tool_choice="required")
|
||||
response = "<|open|>response<|sep|>Checking." "<|close|>response<|sep|>"
|
||||
response = "<|open|>response<|sep|>Checking.<|close|>response<|sep|>"
|
||||
|
||||
assert _accepts(grammar, response + _tools_section(_valid_weather_call()))
|
||||
assert not _accepts(grammar, response)
|
||||
|
||||
@@ -118,7 +118,7 @@ def test_detect_and_parse_cdata_multiline_v3():
|
||||
def test_unknown_tool_block_preserved_v3():
|
||||
detector = MiniCPM5Detector()
|
||||
tools = make_tools_weather()
|
||||
text = '<function name="unknown">' '<param name="x">1</param>' "</function>\n"
|
||||
text = '<function name="unknown"><param name="x">1</param></function>\n'
|
||||
res = detector.detect_and_parse(text, tools)
|
||||
assert len(res.calls) == 0
|
||||
assert "unknown" in res.normal_text
|
||||
@@ -168,7 +168,7 @@ def test_multiple_calls_interleaved_text_v3():
|
||||
def test_incomplete_missing_function_end_v3():
|
||||
detector = MiniCPM5Detector()
|
||||
tools = make_tools_weather()
|
||||
text = '<function name="get_weather">' '<param name="city">北京</param>'
|
||||
text = '<function name="get_weather"><param name="city">北京</param>'
|
||||
res = detector.detect_and_parse(text, tools)
|
||||
assert len(res.calls) == 0
|
||||
assert "get_weather" in res.normal_text
|
||||
@@ -204,11 +204,7 @@ def test_duplicate_param_names_invalid_v3():
|
||||
def test_case_sensitive_param_name_invalid_v3():
|
||||
detector = MiniCPM5Detector()
|
||||
tools = make_tools_weather()
|
||||
text = (
|
||||
'<function name="get_weather">'
|
||||
'<param name="City">北京</param>'
|
||||
"</function>\n"
|
||||
)
|
||||
text = '<function name="get_weather"><param name="City">北京</param></function>\n'
|
||||
res = detector.detect_and_parse(text, tools)
|
||||
assert len(res.calls) == 0
|
||||
|
||||
@@ -243,9 +239,7 @@ def test_streaming_increment_v3():
|
||||
def test_streaming_split_bot_token():
|
||||
detector = MiniCPM5Detector()
|
||||
tools = make_tools_weather()
|
||||
text = (
|
||||
'<function name="get_weather">' '<param name="city">北京</param>' "</function>"
|
||||
)
|
||||
text = '<function name="get_weather"><param name="city">北京</param></function>'
|
||||
|
||||
r1 = detector.parse_streaming_increment("<", tools)
|
||||
assert r1.normal_text == ""
|
||||
@@ -274,9 +268,7 @@ def test_streaming_multiple_complete_blocks_in_one_delta():
|
||||
def test_malformed_xml_with_unescaped_ampersand_falls_back_to_regex():
|
||||
detector = MiniCPM5Detector()
|
||||
tools = make_tools_weather()
|
||||
text = (
|
||||
'<function name="get_weather">' '<param name="city">A & B</param>' "</function>"
|
||||
)
|
||||
text = '<function name="get_weather"><param name="city">A & B</param></function>'
|
||||
|
||||
result = detector.detect_and_parse(text, tools)
|
||||
assert len(result.calls) == 1
|
||||
|
||||
@@ -400,8 +400,7 @@ class TestMuseGlimmerDetector(CustomTestCase):
|
||||
it never will. Goes red if the tool parser loses its stream-end flush
|
||||
(``parse_stream_end`` / detector ``finish``)."""
|
||||
raw = (
|
||||
" to=self<|message|>r<|eom|>"
|
||||
"<|start|>assistant to=user<|message|>answer<|st"
|
||||
" to=self<|message|>r<|eom|><|start|>assistant to=user<|message|>answer<|st"
|
||||
)
|
||||
for chunk_size in (1, 7, 100):
|
||||
_, content, calls = self.pipeline_stream(raw, chunk_size)
|
||||
|
||||
@@ -154,9 +154,7 @@ class TestPoolsideV1Detector(CustomTestCase):
|
||||
in-flight call, matching the old closing-tag-anchored regex behavior.
|
||||
Without the truncated-call filter in detect_and_parse, streaming-as-
|
||||
primitive surfaced a tool call with parameters="{}" on this input."""
|
||||
text = (
|
||||
"<tool_call>get_weather\n<arg_key>location</arg_key>\n" "<arg_value>San Fr"
|
||||
)
|
||||
text = "<tool_call>get_weather\n<arg_key>location</arg_key>\n<arg_value>San Fr"
|
||||
result = self.detector.detect_and_parse(text, self.tools)
|
||||
self.assertEqual(
|
||||
len(result.calls), 0, "truncated mid-arg_value must yield 0 calls"
|
||||
|
||||
@@ -1181,10 +1181,8 @@ class TestMlxOverlapScheduler(unittest.TestCase):
|
||||
spec_algorithm=SpeculativeAlgorithm.NONE,
|
||||
device="cpu",
|
||||
)
|
||||
scheduler.get_next_batch_to_run = (
|
||||
lambda running_batch, last_batch: SimpleNamespace(
|
||||
batch_to_run=batch, running_batch=running_batch
|
||||
)
|
||||
scheduler.get_next_batch_to_run = lambda running_batch, last_batch: (
|
||||
SimpleNamespace(batch_to_run=batch, running_batch=running_batch)
|
||||
)
|
||||
|
||||
with self.assertRaises(_StopLoop):
|
||||
|
||||
@@ -127,7 +127,7 @@ class TestAttentionDpRequestCapacity(CustomTestCase):
|
||||
|
||||
with self.assertRaisesRegex(
|
||||
RuntimeError,
|
||||
"max_mamba_cache_size=15.*per-worker auxiliary-state cap=3.*" "at least 16",
|
||||
"max_mamba_cache_size=15.*per-worker auxiliary-state cap=3.*at least 16",
|
||||
):
|
||||
_initialize_stub(stub, hybrid=True)
|
||||
|
||||
|
||||
@@ -251,9 +251,9 @@ def test_fused_matches_unfused_synthetic():
|
||||
|
||||
assert y_ref.shape == y_fused.shape
|
||||
# A broken kernel must not leak NaN/Inf into the downstream down_proj matmul.
|
||||
assert bool(
|
||||
mx.all(mx.isfinite(y_fused.astype(mx.float32))).item()
|
||||
), f"B={B} hi={hi}: non-finite fused output"
|
||||
assert bool(mx.all(mx.isfinite(y_fused.astype(mx.float32))).item()), (
|
||||
f"B={B} hi={hi}: non-finite fused output"
|
||||
)
|
||||
# Same bf16 bound as the @requires_model kernel test.
|
||||
max_abs, rel = _max_rel_diff(y_ref, y_fused)
|
||||
assert rel < 2e-2, f"B={B} hi={hi}: max_abs={max_abs:.3e} rel={rel:.2%}"
|
||||
|
||||
@@ -226,8 +226,10 @@ class TestMlxHybridInitializeAllocation(CustomTestCase):
|
||||
stub = _hybrid_stub_for_initialize(
|
||||
max_running_requests=4, max_mamba_cache_size=2
|
||||
)
|
||||
with _arch(hybrid=True), _published(stub), self.assertRaisesRegex(
|
||||
RuntimeError, "max_mamba_cache_size"
|
||||
with (
|
||||
_arch(hybrid=True),
|
||||
_published(stub),
|
||||
self.assertRaisesRegex(RuntimeError, "max_mamba_cache_size"),
|
||||
):
|
||||
stub.initialize()
|
||||
|
||||
|
||||
@@ -96,8 +96,9 @@ class TestMetalCaptureProfilerMLX(unittest.TestCase):
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
trace_path = Path(tmp) / "test.gputrace"
|
||||
with patch.object(mx.metal, "start_capture"), patch.object(
|
||||
mx.metal, "stop_capture"
|
||||
with (
|
||||
patch.object(mx.metal, "start_capture"),
|
||||
patch.object(mx.metal, "stop_capture"),
|
||||
):
|
||||
profiler, result = MetalCaptureProfiler.start_mlx(trace_path)
|
||||
|
||||
@@ -131,9 +132,10 @@ class TestMetalCaptureProfilerMLX(unittest.TestCase):
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
trace_path = Path(tmp) / "test.gputrace"
|
||||
with patch.object(mx.metal, "start_capture"), patch.object(
|
||||
mx.metal, "stop_capture"
|
||||
) as mock_stop:
|
||||
with (
|
||||
patch.object(mx.metal, "start_capture"),
|
||||
patch.object(mx.metal, "stop_capture") as mock_stop,
|
||||
):
|
||||
profiler, _ = MetalCaptureProfiler.start_mlx(trace_path)
|
||||
profiler.stop()
|
||||
mock_stop.assert_called_once()
|
||||
@@ -261,9 +263,12 @@ class TestSchedulerProfilerManagerMPS(unittest.TestCase):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
mgr = self._make_manager(tmp)
|
||||
capture_ctx = MagicMock()
|
||||
with mock_patch.object(
|
||||
torch.mps.profiler, "metal_capture", return_value=capture_ctx
|
||||
), mock_patch("torch.distributed.barrier"):
|
||||
with (
|
||||
mock_patch.object(
|
||||
torch.mps.profiler, "metal_capture", return_value=capture_ctx
|
||||
),
|
||||
mock_patch("torch.distributed.barrier"),
|
||||
):
|
||||
result = mgr._start_profile()
|
||||
self.assertTrue(result.success, result.message)
|
||||
self.assertTrue(mgr.profile_in_progress)
|
||||
|
||||
@@ -98,14 +98,13 @@ def _raw_weights(args):
|
||||
p + "post_attn_norm.weight": mx.full((hid,), 0.20),
|
||||
p + "post_attention_layernorm.weight": mx.full((hid,), 0.30),
|
||||
p + "post_ffn_norm.weight": mx.full((hid,), 0.40),
|
||||
p
|
||||
+ "mlp.gate_proj.weight": mx.random.normal(
|
||||
p + "mlp.gate_proj.weight": mx.random.normal(
|
||||
(args.intermediate_size, hid)
|
||||
),
|
||||
p
|
||||
+ "mlp.up_proj.weight": mx.random.normal((args.intermediate_size, hid)),
|
||||
p
|
||||
+ "mlp.down_proj.weight": mx.random.normal(
|
||||
p + "mlp.up_proj.weight": mx.random.normal(
|
||||
(args.intermediate_size, hid)
|
||||
),
|
||||
p + "mlp.down_proj.weight": mx.random.normal(
|
||||
(hid, args.intermediate_size)
|
||||
),
|
||||
}
|
||||
|
||||
@@ -119,8 +119,8 @@ class TestMlxQuantization(unittest.TestCase):
|
||||
self.assertGreater(
|
||||
reduction,
|
||||
0.40,
|
||||
f"expected >40% memory reduction with mlx_q4, got {reduction*100:.1f}% "
|
||||
f"(fp16={mem_fp/1024**3:.2f} GB, q4={mem_q4/1024**3:.2f} GB)",
|
||||
f"expected >40% memory reduction with mlx_q4, got {reduction * 100:.1f}% "
|
||||
f"(fp16={mem_fp / 1024**3:.2f} GB, q4={mem_q4 / 1024**3:.2f} GB)",
|
||||
)
|
||||
|
||||
def test_mlx_q8_creates_quantized_linear_modules(self):
|
||||
|
||||
@@ -27,10 +27,13 @@ HUGE_MEM_BUDGET = 64 * 2**30
|
||||
def _decide(num_q, num_k, mem_budget=HUGE_MEM_BUDGET, is_hip=True):
|
||||
# __new__ skips an __init__ that needs a model config and a device.
|
||||
indexer = dsa_indexer.Indexer.__new__(dsa_indexer.Indexer)
|
||||
with mock.patch.object(dsa_indexer, "_is_hip", is_hip), mock.patch.object(
|
||||
dsa_indexer.Indexer,
|
||||
"_get_mqa_logits_budget_bytes",
|
||||
return_value=mem_budget,
|
||||
with (
|
||||
mock.patch.object(dsa_indexer, "_is_hip", is_hip),
|
||||
mock.patch.object(
|
||||
dsa_indexer.Indexer,
|
||||
"_get_mqa_logits_budget_bytes",
|
||||
return_value=mem_budget,
|
||||
),
|
||||
):
|
||||
return indexer._should_chunk_mqa_logits(num_q, num_k, 0)
|
||||
|
||||
|
||||
@@ -24,8 +24,7 @@ class TestHelionKDADispatcher(unittest.TestCase):
|
||||
return_value=True,
|
||||
),
|
||||
patch(
|
||||
"sglang.srt.layers.attention.linear.kernels.kda_helion."
|
||||
"HelionKDAKernel",
|
||||
"sglang.srt.layers.attention.linear.kernels.kda_helion.HelionKDAKernel",
|
||||
return_value=helion_kernel,
|
||||
) as constructor,
|
||||
):
|
||||
|
||||
@@ -67,9 +67,10 @@ def single_rank(monkeypatch, gloo_world):
|
||||
"_determine_attention_backend",
|
||||
lambda self, passed_backend: passed_backend,
|
||||
)
|
||||
with get_parallel().override(
|
||||
tp_size=1, tp_rank=0, attn_tp_size=1, attn_tp_rank=0
|
||||
), get_context().override_server_args():
|
||||
with (
|
||||
get_parallel().override(tp_size=1, tp_rank=0, attn_tp_size=1, attn_tp_rank=0),
|
||||
get_context().override_server_args(),
|
||||
):
|
||||
yield
|
||||
|
||||
|
||||
|
||||
@@ -106,8 +106,8 @@ class TestDeepGemmMegaMoeApi(CustomTestCase):
|
||||
|
||||
deep_gemm = ModuleType("deep_gemm")
|
||||
deep_gemm.transform_sf_into_required_layout = MagicMock(
|
||||
side_effect=lambda _sf, mn, k, recipe, num_groups, disable_ue8m0_cast: torch.zeros(
|
||||
(num_groups, mn, max(1, k // 32)), dtype=torch.int32
|
||||
side_effect=lambda _sf, mn, k, recipe, num_groups, disable_ue8m0_cast: (
|
||||
torch.zeros((num_groups, mn, max(1, k // 32)), dtype=torch.int32)
|
||||
)
|
||||
)
|
||||
deep_gemm.transform_weights_for_mega_moe = MagicMock(
|
||||
|
||||
@@ -150,18 +150,20 @@ def test_fused_moe_uses_explicit_quant_method_for_full_lifecycle(monkeypatch) ->
|
||||
lambda config: SimpleNamespace(),
|
||||
)
|
||||
|
||||
with get_context().override_server_args(
|
||||
model_path="dummy"
|
||||
), get_flags().moe.override(
|
||||
runner_backend=MoeRunnerBackend.AUTO,
|
||||
a2a_backend=MoeA2ABackend.NONE,
|
||||
), get_parallel().override(
|
||||
moe_ep_size=1,
|
||||
moe_ep_rank=0,
|
||||
moe_tp_size=1,
|
||||
moe_tp_rank=0,
|
||||
tp_size=1,
|
||||
tp_rank=0,
|
||||
with (
|
||||
get_context().override_server_args(model_path="dummy"),
|
||||
get_flags().moe.override(
|
||||
runner_backend=MoeRunnerBackend.AUTO,
|
||||
a2a_backend=MoeA2ABackend.NONE,
|
||||
),
|
||||
get_parallel().override(
|
||||
moe_ep_size=1,
|
||||
moe_ep_rank=0,
|
||||
moe_tp_size=1,
|
||||
moe_tp_rank=0,
|
||||
tp_size=1,
|
||||
tp_rank=0,
|
||||
),
|
||||
):
|
||||
layer = FusedMoE(
|
||||
num_experts=2,
|
||||
@@ -206,10 +208,9 @@ def test_lora_uses_quant_method_contract_for_registered_backend(
|
||||
monkeypatch.setattr(
|
||||
runner_module,
|
||||
"MoeRunner",
|
||||
lambda selected_backend, config, lora_enabled: created_runners.append(
|
||||
(selected_backend, config, lora_enabled)
|
||||
)
|
||||
or object(),
|
||||
lambda selected_backend, config, lora_enabled: (
|
||||
created_runners.append((selected_backend, config, lora_enabled)) or object()
|
||||
),
|
||||
)
|
||||
|
||||
wrapper = FusedMoEWithLoRA(base_layer, lora_backend)
|
||||
@@ -348,18 +349,20 @@ def test_fused_moe_layer_runner_is_none_when_method_builds_no_runner(
|
||||
lambda config: SimpleNamespace(),
|
||||
)
|
||||
|
||||
with get_context().override_server_args(
|
||||
model_path="dummy"
|
||||
), get_flags().moe.override(
|
||||
runner_backend=MoeRunnerBackend.AUTO,
|
||||
a2a_backend=MoeA2ABackend.NONE,
|
||||
), get_parallel().override(
|
||||
moe_ep_size=1,
|
||||
moe_ep_rank=0,
|
||||
moe_tp_size=1,
|
||||
moe_tp_rank=0,
|
||||
tp_size=1,
|
||||
tp_rank=0,
|
||||
with (
|
||||
get_context().override_server_args(model_path="dummy"),
|
||||
get_flags().moe.override(
|
||||
runner_backend=MoeRunnerBackend.AUTO,
|
||||
a2a_backend=MoeA2ABackend.NONE,
|
||||
),
|
||||
get_parallel().override(
|
||||
moe_ep_size=1,
|
||||
moe_ep_rank=0,
|
||||
moe_tp_size=1,
|
||||
moe_tp_rank=0,
|
||||
tp_size=1,
|
||||
tp_rank=0,
|
||||
),
|
||||
):
|
||||
layer = FusedMoE(
|
||||
num_experts=2,
|
||||
|
||||
@@ -39,9 +39,10 @@ class TestDeepGemmUE8M0Requant(CustomTestCase):
|
||||
def test_helper_requants_supported_deepgemm_bf16_once(self):
|
||||
weight, weight_scale = _make_params()
|
||||
|
||||
with self._enabled_deepgemm_ue8m0(), patch.object(
|
||||
fp8_utils, "requant_weight_ue8m0_inplace"
|
||||
) as requant:
|
||||
with (
|
||||
self._enabled_deepgemm_ue8m0(),
|
||||
patch.object(fp8_utils, "requant_weight_ue8m0_inplace") as requant,
|
||||
):
|
||||
fired = fp8_utils.requant_block_scale_ue8m0_for_deepgemm(
|
||||
weight,
|
||||
weight_scale,
|
||||
@@ -67,9 +68,10 @@ class TestDeepGemmUE8M0Requant(CustomTestCase):
|
||||
def test_helper_skips_non_bf16_output(self):
|
||||
weight, weight_scale = _make_params()
|
||||
|
||||
with self._enabled_deepgemm_ue8m0(), patch.object(
|
||||
fp8_utils, "requant_weight_ue8m0_inplace"
|
||||
) as requant:
|
||||
with (
|
||||
self._enabled_deepgemm_ue8m0(),
|
||||
patch.object(fp8_utils, "requant_weight_ue8m0_inplace") as requant,
|
||||
):
|
||||
fired = fp8_utils.requant_block_scale_ue8m0_for_deepgemm(
|
||||
weight,
|
||||
weight_scale,
|
||||
@@ -86,9 +88,10 @@ class TestDeepGemmUE8M0Requant(CustomTestCase):
|
||||
def test_helper_skips_shape_deepgemm_will_not_run(self):
|
||||
weight, weight_scale = _make_params(n=96, k=128)
|
||||
|
||||
with self._enabled_deepgemm_ue8m0(), patch.object(
|
||||
fp8_utils, "requant_weight_ue8m0_inplace"
|
||||
) as requant:
|
||||
with (
|
||||
self._enabled_deepgemm_ue8m0(),
|
||||
patch.object(fp8_utils, "requant_weight_ue8m0_inplace") as requant,
|
||||
):
|
||||
fired = fp8_utils.requant_block_scale_ue8m0_for_deepgemm(
|
||||
weight,
|
||||
weight_scale,
|
||||
@@ -105,9 +108,10 @@ class TestDeepGemmUE8M0Requant(CustomTestCase):
|
||||
def test_helper_skips_non_deepgemm_runner(self):
|
||||
weight, weight_scale = _make_params()
|
||||
|
||||
with self._enabled_deepgemm_ue8m0(), patch.object(
|
||||
fp8_utils, "requant_weight_ue8m0_inplace"
|
||||
) as requant:
|
||||
with (
|
||||
self._enabled_deepgemm_ue8m0(),
|
||||
patch.object(fp8_utils, "requant_weight_ue8m0_inplace") as requant,
|
||||
):
|
||||
fired = fp8_utils.requant_block_scale_ue8m0_for_deepgemm(
|
||||
weight,
|
||||
weight_scale,
|
||||
@@ -125,9 +129,10 @@ class TestDeepGemmUE8M0Requant(CustomTestCase):
|
||||
weight, weight_scale = _make_params()
|
||||
unsupported_block_size = [128, 256]
|
||||
|
||||
with self._enabled_deepgemm_ue8m0(), patch.object(
|
||||
fp8_utils, "requant_weight_ue8m0_inplace"
|
||||
) as requant:
|
||||
with (
|
||||
self._enabled_deepgemm_ue8m0(),
|
||||
patch.object(fp8_utils, "requant_weight_ue8m0_inplace") as requant,
|
||||
):
|
||||
fired = fp8_utils.requant_block_scale_ue8m0_for_deepgemm(
|
||||
weight,
|
||||
weight_scale,
|
||||
@@ -154,9 +159,10 @@ class TestDeepGemmUE8M0Requant(CustomTestCase):
|
||||
layer.weight, layer.weight_scale = _make_params()
|
||||
layer.orig_dtype = torch.bfloat16
|
||||
|
||||
with self._enabled_deepgemm_ue8m0(), patch.object(
|
||||
fp8_utils, "requant_weight_ue8m0_inplace"
|
||||
) as requant:
|
||||
with (
|
||||
self._enabled_deepgemm_ue8m0(),
|
||||
patch.object(fp8_utils, "requant_weight_ue8m0_inplace") as requant,
|
||||
):
|
||||
scheme.process_weights_after_loading(layer)
|
||||
scheme.process_weights_after_loading(layer)
|
||||
|
||||
@@ -179,18 +185,22 @@ class TestDeepGemmUE8M0Requant(CustomTestCase):
|
||||
weight_scale.format_ue8m0 = True
|
||||
return True
|
||||
|
||||
with patch.multiple(
|
||||
fp8_quant,
|
||||
_is_cpu=False,
|
||||
_is_fp8_fnuz=False,
|
||||
_use_aiter=False,
|
||||
), patch.object(
|
||||
method, "is_deepgemm_moe_runner_backend_enabled", return_value=True
|
||||
), patch.object(
|
||||
fp8_quant,
|
||||
"requant_block_scale_ue8m0_for_deepgemm",
|
||||
side_effect=_mark_ue8m0,
|
||||
) as requant:
|
||||
with (
|
||||
patch.multiple(
|
||||
fp8_quant,
|
||||
_is_cpu=False,
|
||||
_is_fp8_fnuz=False,
|
||||
_use_aiter=False,
|
||||
),
|
||||
patch.object(
|
||||
method, "is_deepgemm_moe_runner_backend_enabled", return_value=True
|
||||
),
|
||||
patch.object(
|
||||
fp8_quant,
|
||||
"requant_block_scale_ue8m0_for_deepgemm",
|
||||
side_effect=_mark_ue8m0,
|
||||
) as requant,
|
||||
):
|
||||
method.process_weights_after_loading_block_quant(layer)
|
||||
|
||||
self.assertEqual(
|
||||
|
||||
@@ -54,17 +54,16 @@ class TestFlashinferTrtllmFp8Fallback(CustomTestCase):
|
||||
trtllm_spy = MagicMock(return_value=torch.zeros((M, N), dtype=dtype))
|
||||
quant_spy = MagicMock(return_value=(MagicMock(), MagicMock()))
|
||||
|
||||
with patch.object(
|
||||
fp8_utils,
|
||||
"_get_flashinfer_groupwise_backend",
|
||||
return_value="trtllm",
|
||||
create=True,
|
||||
), patch.object(
|
||||
fp8_utils, "gemm_fp8_nt_groupwise", trtllm_spy, create=True
|
||||
), patch.object(
|
||||
fp8_utils, "triton_w8a8_block_fp8_linear", triton_spy
|
||||
), patch.object(
|
||||
fp8_utils, "sglang_per_token_group_quant_fp8", quant_spy
|
||||
with (
|
||||
patch.object(
|
||||
fp8_utils,
|
||||
"_get_flashinfer_groupwise_backend",
|
||||
return_value="trtllm",
|
||||
create=True,
|
||||
),
|
||||
patch.object(fp8_utils, "gemm_fp8_nt_groupwise", trtllm_spy, create=True),
|
||||
patch.object(fp8_utils, "triton_w8a8_block_fp8_linear", triton_spy),
|
||||
patch.object(fp8_utils, "sglang_per_token_group_quant_fp8", quant_spy),
|
||||
):
|
||||
fp8_utils.flashinfer_gemm_w8a8_block_fp8_linear_with_fallback(
|
||||
input_2d, weight, BLOCK_SIZE, weight_scale
|
||||
|
||||
@@ -25,16 +25,20 @@ class TestMxfp4FlashinferActivationPrep(CustomTestCase):
|
||||
x_quant = torch.empty(3, 64, dtype=torch.float8_e4m3fn)
|
||||
x_scale = torch.arange(6, dtype=torch.uint8).reshape(3, 2)
|
||||
|
||||
with patch(
|
||||
"sglang.srt.layers.moe.route_quant_handoff.take", return_value=None
|
||||
) as take, patch(
|
||||
"sglang.srt.layers.quantization.mxfp4._is_sm107_supported",
|
||||
return_value=True,
|
||||
), patch(
|
||||
"sglang.srt.layers.quantization.fp8_utils.flashinfer_mxfp8_quantize",
|
||||
return_value=(x_quant, x_scale),
|
||||
create=True,
|
||||
) as quantize:
|
||||
with (
|
||||
patch(
|
||||
"sglang.srt.layers.moe.route_quant_handoff.take", return_value=None
|
||||
) as take,
|
||||
patch(
|
||||
"sglang.srt.layers.quantization.mxfp4._is_sm107_supported",
|
||||
return_value=True,
|
||||
),
|
||||
patch(
|
||||
"sglang.srt.layers.quantization.fp8_utils.flashinfer_mxfp8_quantize",
|
||||
return_value=(x_quant, x_scale),
|
||||
create=True,
|
||||
) as quantize,
|
||||
):
|
||||
actual_x, packed_topk, actual_quant, actual_scale = (
|
||||
_prepare_flashinfer_mxfp8_activations(x, 64)
|
||||
)
|
||||
@@ -51,19 +55,24 @@ class TestMxfp4FlashinferActivationPrep(CustomTestCase):
|
||||
x_quant = torch.empty(3, 64, dtype=torch.float8_e4m3fn)
|
||||
x_scale = torch.arange(6, dtype=torch.uint8).reshape(3, 2)
|
||||
|
||||
with patch(
|
||||
"sglang.srt.layers.moe.route_quant_handoff.take", return_value=None
|
||||
) as take, patch(
|
||||
"sglang.srt.layers.quantization.mxfp4._is_sm107_supported",
|
||||
return_value=True,
|
||||
), patch.object(
|
||||
per_token_group_quant_module,
|
||||
"per_token_group_quant",
|
||||
return_value=(x_quant, x_scale),
|
||||
) as quantize, patch(
|
||||
"sglang.srt.layers.quantization.fp8_utils.flashinfer_mxfp8_quantize",
|
||||
create=True,
|
||||
) as flashinfer_quantize:
|
||||
with (
|
||||
patch(
|
||||
"sglang.srt.layers.moe.route_quant_handoff.take", return_value=None
|
||||
) as take,
|
||||
patch(
|
||||
"sglang.srt.layers.quantization.mxfp4._is_sm107_supported",
|
||||
return_value=True,
|
||||
),
|
||||
patch.object(
|
||||
per_token_group_quant_module,
|
||||
"per_token_group_quant",
|
||||
return_value=(x_quant, x_scale),
|
||||
) as quantize,
|
||||
patch(
|
||||
"sglang.srt.layers.quantization.fp8_utils.flashinfer_mxfp8_quantize",
|
||||
create=True,
|
||||
) as flashinfer_quantize,
|
||||
):
|
||||
actual_x, packed_topk, actual_quant, actual_scale = (
|
||||
_prepare_flashinfer_mxfp8_activations(x, 64)
|
||||
)
|
||||
@@ -81,19 +90,22 @@ class TestMxfp4FlashinferActivationPrep(CustomTestCase):
|
||||
x_quant = torch.empty(3, 64, dtype=torch.float8_e4m3fn)
|
||||
x_scale = torch.arange(6, dtype=torch.uint8).reshape(3, 2)
|
||||
|
||||
with patch(
|
||||
"sglang.srt.layers.moe.route_quant_handoff.take", return_value=None
|
||||
), patch(
|
||||
"sglang.srt.layers.quantization.mxfp4._is_sm107_supported",
|
||||
return_value=False,
|
||||
), patch.object(
|
||||
per_token_group_quant_module,
|
||||
"per_token_group_quant",
|
||||
return_value=(x_quant, x_scale),
|
||||
) as quantize, patch(
|
||||
"sglang.srt.layers.quantization.fp8_utils.flashinfer_mxfp8_quantize",
|
||||
create=True,
|
||||
) as flashinfer_quantize:
|
||||
with (
|
||||
patch("sglang.srt.layers.moe.route_quant_handoff.take", return_value=None),
|
||||
patch(
|
||||
"sglang.srt.layers.quantization.mxfp4._is_sm107_supported",
|
||||
return_value=False,
|
||||
),
|
||||
patch.object(
|
||||
per_token_group_quant_module,
|
||||
"per_token_group_quant",
|
||||
return_value=(x_quant, x_scale),
|
||||
) as quantize,
|
||||
patch(
|
||||
"sglang.srt.layers.quantization.fp8_utils.flashinfer_mxfp8_quantize",
|
||||
create=True,
|
||||
) as flashinfer_quantize,
|
||||
):
|
||||
actual_x, packed_topk, actual_quant, actual_scale = (
|
||||
_prepare_flashinfer_mxfp8_activations(x, 64)
|
||||
)
|
||||
@@ -111,11 +123,14 @@ class TestMxfp4FlashinferActivationPrep(CustomTestCase):
|
||||
x_quant = torch.empty(3, 128, dtype=torch.float8_e4m3fn)
|
||||
x_scale = torch.arange(12, dtype=torch.uint8)
|
||||
|
||||
with patch(
|
||||
"sglang.srt.layers.quantization.fp8_utils.flashinfer_mxfp8_quantize",
|
||||
return_value=(x_quant, x_scale),
|
||||
create=True,
|
||||
) as quantize, patch("sglang.srt.layers.moe.route_quant_handoff.take") as take:
|
||||
with (
|
||||
patch(
|
||||
"sglang.srt.layers.quantization.fp8_utils.flashinfer_mxfp8_quantize",
|
||||
return_value=(x_quant, x_scale),
|
||||
create=True,
|
||||
) as quantize,
|
||||
patch("sglang.srt.layers.moe.route_quant_handoff.take") as take,
|
||||
):
|
||||
actual_x, packed_topk, actual_quant, actual_scale = (
|
||||
_prepare_flashinfer_mxfp8_activations(x, 128)
|
||||
)
|
||||
@@ -133,13 +148,16 @@ class TestMxfp4FlashinferActivationPrep(CustomTestCase):
|
||||
x_quant = torch.empty(2, 64, dtype=torch.float8_e4m3fn)
|
||||
x_scale = torch.arange(4, dtype=torch.uint8).reshape(2, 2)
|
||||
|
||||
with patch(
|
||||
"sglang.srt.layers.moe.route_quant_handoff.take",
|
||||
return_value=(packed_topk, x_quant, x_scale),
|
||||
), patch(
|
||||
"sglang.srt.layers.quantization.fp8_utils.flashinfer_mxfp8_quantize",
|
||||
create=True,
|
||||
) as quantize:
|
||||
with (
|
||||
patch(
|
||||
"sglang.srt.layers.moe.route_quant_handoff.take",
|
||||
return_value=(packed_topk, x_quant, x_scale),
|
||||
),
|
||||
patch(
|
||||
"sglang.srt.layers.quantization.fp8_utils.flashinfer_mxfp8_quantize",
|
||||
create=True,
|
||||
) as quantize,
|
||||
):
|
||||
actual_x, actual_packed, actual_quant, actual_scale = (
|
||||
_prepare_flashinfer_mxfp8_activations(x, 64)
|
||||
)
|
||||
|
||||
@@ -585,9 +585,9 @@ def test_humming_range_ignores_prerounded_hidden_tail():
|
||||
"w13 Humming residual changed with the never-written hidden tail; "
|
||||
"the _UE8M0_ONE fill leaked into the per-expert E8M0 range"
|
||||
)
|
||||
assert torch.equal(
|
||||
residuals[0][1], residuals[1][1]
|
||||
), "w2 Humming residual changed with the never-written hidden tail"
|
||||
assert torch.equal(residuals[0][1], residuals[1][1]), (
|
||||
"w2 Humming residual changed with the never-written hidden tail"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
|
||||
@@ -43,17 +43,17 @@ class TestNvFp4MoeBackends(CustomTestCase):
|
||||
quant_config = ModelOptFp4Config(
|
||||
is_checkpoint_nvfp4_serialized=True, group_size=16
|
||||
)
|
||||
with get_context().override_server_args(
|
||||
model_path="dummy"
|
||||
), get_flags().moe.override(
|
||||
runner_backend=MoeRunnerBackend(backend)
|
||||
), get_parallel().override(
|
||||
moe_ep_size=1,
|
||||
moe_ep_rank=0,
|
||||
moe_tp_size=1,
|
||||
moe_tp_rank=0,
|
||||
tp_size=1,
|
||||
tp_rank=0,
|
||||
with (
|
||||
get_context().override_server_args(model_path="dummy"),
|
||||
get_flags().moe.override(runner_backend=MoeRunnerBackend(backend)),
|
||||
get_parallel().override(
|
||||
moe_ep_size=1,
|
||||
moe_ep_rank=0,
|
||||
moe_tp_size=1,
|
||||
moe_tp_rank=0,
|
||||
tp_size=1,
|
||||
tp_rank=0,
|
||||
),
|
||||
):
|
||||
layer = FusedMoE(
|
||||
num_experts=E,
|
||||
|
||||
@@ -19,7 +19,6 @@ def _copy_weights(src, dst_nn):
|
||||
|
||||
|
||||
class TestConv2dLayer(unittest.TestCase):
|
||||
|
||||
def test_basic_patch_embedding(self):
|
||||
layer = Conv2dLayer(3, 768, kernel_size=14, stride=14, bias=False)
|
||||
ref = nn.Conv2d(3, 768, kernel_size=14, stride=14, bias=False)
|
||||
@@ -150,7 +149,6 @@ class TestConv2dLayer(unittest.TestCase):
|
||||
|
||||
|
||||
class TestConvValidation(unittest.TestCase):
|
||||
|
||||
def test_in_channels_not_divisible_by_groups(self):
|
||||
with self.assertRaises(ValueError):
|
||||
Conv2dLayer(3, 64, kernel_size=3, stride=1, groups=2)
|
||||
@@ -204,7 +202,6 @@ class TestConvValidation(unittest.TestCase):
|
||||
|
||||
|
||||
class TestConv3dLayer(unittest.TestCase):
|
||||
|
||||
def test_basic_temporal_patch_embedding(self):
|
||||
layer = Conv3dLayer(
|
||||
3, 1152, kernel_size=[2, 14, 14], stride=[2, 14, 14], bias=False
|
||||
|
||||
@@ -61,9 +61,9 @@ def test_splits_never_increase_with_token_count():
|
||||
for tokens in [1, 2, 4, 8, 16, 32, 64, 128, 256, 512]:
|
||||
splits = _kv_splits_heuristic(tokens, HEADS, BLOCK_H, num_cu=NUM_CU)
|
||||
if prev is not None:
|
||||
assert (
|
||||
splits <= prev
|
||||
), f"splits rose from {prev} to {splits} going to T={tokens}"
|
||||
assert splits <= prev, (
|
||||
f"splits rose from {prev} to {splits} going to T={tokens}"
|
||||
)
|
||||
prev = splits
|
||||
|
||||
|
||||
|
||||
@@ -484,9 +484,12 @@ class TestTagGroupsForFlashInferAllReduceOnly(CustomTestCase):
|
||||
def _tag(self, *, attn_tp, moe_ep, moe_tp):
|
||||
from sglang.srt.distributed import parallel_state as ps
|
||||
|
||||
with patch.object(ps, "_ENABLE_FLASHINFER_ALLREDUCE_ONLY", True), patch.object(
|
||||
ps, "_ATTN_TP", attn_tp
|
||||
), patch.object(ps, "_MOE_EP", moe_ep), patch.object(ps, "_MOE_TP", moe_tp):
|
||||
with (
|
||||
patch.object(ps, "_ENABLE_FLASHINFER_ALLREDUCE_ONLY", True),
|
||||
patch.object(ps, "_ATTN_TP", attn_tp),
|
||||
patch.object(ps, "_MOE_EP", moe_ep),
|
||||
patch.object(ps, "_MOE_TP", moe_tp),
|
||||
):
|
||||
ps._tag_groups_for_flashinfer_allreduce_only()
|
||||
|
||||
def test_hybrid_ep_tp_tags_only_the_ep_group(self):
|
||||
|
||||
@@ -192,8 +192,10 @@ class TestZmqRoundTrip(CustomTestCase):
|
||||
|
||||
loads = _read_until(
|
||||
lambda: reader.read_all(),
|
||||
lambda snaps: len(snaps) == dp_size
|
||||
and all(snap.timestamp == 3.0 for snap in snaps),
|
||||
lambda snaps: (
|
||||
len(snaps) == dp_size
|
||||
and all(snap.timestamp == 3.0 for snap in snaps)
|
||||
),
|
||||
)
|
||||
self.assertEqual(len(loads), dp_size)
|
||||
for load in loads:
|
||||
@@ -373,8 +375,9 @@ class TestZmqReaderOwner(CustomTestCase):
|
||||
override = get_context().override_server_args(**fields)
|
||||
override.install()
|
||||
self.addCleanup(override.restore)
|
||||
with mock.patch.object(rc, "_ROLE_NS_MODE", "enforce"), mock.patch.object(
|
||||
rc._CONTEXT, "_publish_role", "dp_controller"
|
||||
with (
|
||||
mock.patch.object(rc, "_ROLE_NS_MODE", "enforce"),
|
||||
mock.patch.object(rc._CONTEXT, "_publish_role", "dp_controller"),
|
||||
):
|
||||
self.assertTrue(zmq_reader_owner("DataParallelController"))
|
||||
|
||||
@@ -442,8 +445,10 @@ class TestEndToEndZmqSimulation(CustomTestCase):
|
||||
|
||||
loads = _read_until(
|
||||
lambda: reader.read_all(),
|
||||
lambda snaps: len(snaps) == dp_size
|
||||
and all(snap.timestamp == 1.0 for snap in snaps),
|
||||
lambda snaps: (
|
||||
len(snaps) == dp_size
|
||||
and all(snap.timestamp == 1.0 for snap in snaps)
|
||||
),
|
||||
)
|
||||
self.assertEqual(len(loads), dp_size)
|
||||
self.assertEqual(loads[0].num_running_reqs, 10)
|
||||
@@ -464,8 +469,10 @@ class TestEndToEndZmqSimulation(CustomTestCase):
|
||||
|
||||
loads = _read_until(
|
||||
lambda: reader.read_all(),
|
||||
lambda snaps: len(snaps) == dp_size
|
||||
and all(snap.timestamp == 2.0 for snap in snaps),
|
||||
lambda snaps: (
|
||||
len(snaps) == dp_size
|
||||
and all(snap.timestamp == 2.0 for snap in snaps)
|
||||
),
|
||||
)
|
||||
self.assertEqual(loads[0].num_running_reqs, 20)
|
||||
self.assertEqual(loads[1].num_running_reqs, 21)
|
||||
|
||||
@@ -102,8 +102,7 @@ class TestLoadPublisherGating(CustomTestCase):
|
||||
by default (the feature is off without it). dp_size lives on the ps,
|
||||
which the publisher reads (no separate param to disagree with it)."""
|
||||
with patch(
|
||||
"sglang.srt.managers.scheduler_components.load_publisher."
|
||||
"_open_pub_socket"
|
||||
"sglang.srt.managers.scheduler_components.load_publisher._open_pub_socket"
|
||||
) as open_sock:
|
||||
pub = SchedulerLoadPublisher(
|
||||
kv_events_config=config,
|
||||
@@ -284,8 +283,7 @@ class TestLoadPublisherGating(CustomTestCase):
|
||||
import zmq
|
||||
|
||||
with patch(
|
||||
"sglang.srt.managers.scheduler_components.load_publisher."
|
||||
"_open_pub_socket",
|
||||
"sglang.srt.managers.scheduler_components.load_publisher._open_pub_socket",
|
||||
side_effect=zmq.ZMQError,
|
||||
) as open_sock:
|
||||
pub = SchedulerLoadPublisher(
|
||||
|
||||
@@ -451,18 +451,21 @@ class TestDecodePrebuilt(unittest.TestCase):
|
||||
scheduler.waiting_queue[0].priority = 1
|
||||
scheduler.waiting_queue[1].priority = 10
|
||||
scheduler.enable_priority_scheduling = True
|
||||
scheduler.policy.calc_priority.side_effect = (
|
||||
lambda waiting_queue, _: waiting_queue.sort(key=lambda req: -req.priority)
|
||||
scheduler.policy.calc_priority.side_effect = lambda waiting_queue, _: (
|
||||
waiting_queue.sort(key=lambda req: -req.priority)
|
||||
)
|
||||
|
||||
new_batch = MagicMock()
|
||||
# get_new_prebuilt_batch reads the published disagg config
|
||||
# (disaggregation_decode_enable_radix_cache).
|
||||
with patch(
|
||||
"sglang.srt.disaggregation.decode.ScheduleBatch.init_new",
|
||||
return_value=new_batch,
|
||||
) as init_new, get_context().override_server_args(
|
||||
disaggregation_decode_enable_radix_cache=False
|
||||
with (
|
||||
patch(
|
||||
"sglang.srt.disaggregation.decode.ScheduleBatch.init_new",
|
||||
return_value=new_batch,
|
||||
) as init_new,
|
||||
get_context().override_server_args(
|
||||
disaggregation_decode_enable_radix_cache=False
|
||||
),
|
||||
):
|
||||
ret = SchedulerDisaggregationDecodeMixin.get_new_prebuilt_batch(
|
||||
scheduler, scheduler.running_batch
|
||||
@@ -490,11 +493,14 @@ class TestDecodePrebuilt(unittest.TestCase):
|
||||
)
|
||||
new_batch.process_prebuilt.side_effect = lambda *_: call_order.append("process")
|
||||
|
||||
with patch(
|
||||
"sglang.srt.disaggregation.decode.ScheduleBatch.init_new",
|
||||
return_value=new_batch,
|
||||
), get_context().override_server_args(
|
||||
disaggregation_decode_enable_radix_cache=False
|
||||
with (
|
||||
patch(
|
||||
"sglang.srt.disaggregation.decode.ScheduleBatch.init_new",
|
||||
return_value=new_batch,
|
||||
),
|
||||
get_context().override_server_args(
|
||||
disaggregation_decode_enable_radix_cache=False
|
||||
),
|
||||
):
|
||||
ret = SchedulerDisaggregationDecodeMixin.get_new_prebuilt_batch(
|
||||
scheduler, scheduler.running_batch
|
||||
|
||||
@@ -391,9 +391,10 @@ class TestMinFreeSpaceWatermark(HiCacheFileLRUTestBase):
|
||||
free[0] += os.path.getsize(p)
|
||||
return original_remove(p)
|
||||
|
||||
with mock.patch.object(
|
||||
b._evictor, "_fs_stats", side_effect=fake_fs_stats
|
||||
), mock.patch("os.remove", side_effect=tracked_remove):
|
||||
with (
|
||||
mock.patch.object(b._evictor, "_fs_stats", side_effect=fake_fs_stats),
|
||||
mock.patch("os.remove", side_effect=tracked_remove),
|
||||
):
|
||||
self.assertTrue(b.set("newk", _t(60)))
|
||||
self.assertFalse(b.exists("victim"))
|
||||
self.assertTrue(b.exists("newk"))
|
||||
@@ -516,9 +517,10 @@ class TestHiCacheFileMetadataIntegration(HiCacheFileLRUTestBase):
|
||||
b.set("k2", _t(50))
|
||||
|
||||
# Now patch os.scandir and os.path.exists
|
||||
with mock.patch("os.scandir") as mock_scandir, mock.patch(
|
||||
"os.path.exists"
|
||||
) as mock_exists:
|
||||
with (
|
||||
mock.patch("os.scandir") as mock_scandir,
|
||||
mock.patch("os.path.exists") as mock_exists,
|
||||
):
|
||||
mock_exists.return_value = True
|
||||
|
||||
# batch_exists_v2 for k1 and k2 should hit the metadata cache and NOT call os.scandir or os.path.exists
|
||||
|
||||
@@ -49,9 +49,7 @@ class TestDeviceAllocEviction(CustomTestCase):
|
||||
def test_sufficient_capacity_skips_eviction(self):
|
||||
cache = MagicMock()
|
||||
cache.token_to_kv_pool_allocator.swa_available_size.return_value = 10
|
||||
cache.req_to_token_pool.mamba_allocator.schedulable_available_size.return_value = (
|
||||
10
|
||||
)
|
||||
cache.req_to_token_pool.mamba_allocator.schedulable_available_size.return_value = 10
|
||||
|
||||
_evict_swa_for_device_alloc(cache, required_size=10)
|
||||
_evict_mamba_for_device_alloc(cache, required_size=10)
|
||||
|
||||
@@ -182,7 +182,6 @@ def _alloc_and_fill(allocator, ps, lens):
|
||||
|
||||
|
||||
class TestReadTableBuild(unittest.TestCase):
|
||||
|
||||
def test_read_table_matches_reference_across_multipliers(self):
|
||||
"""The load-bearing formula pin: full AND swa read tables equal
|
||||
the independent per-element derivation, across page sizes and both
|
||||
|
||||
@@ -117,7 +117,9 @@ class TestMambaRatioEnvGate(unittest.TestCase):
|
||||
strategy = (
|
||||
"extra_buffer_lazy"
|
||||
if lazy
|
||||
else "extra_buffer" if extra_buffer else "no_buffer"
|
||||
else "extra_buffer"
|
||||
if extra_buffer
|
||||
else "no_buffer"
|
||||
)
|
||||
from sglang.srt import runtime_context as rc
|
||||
|
||||
|
||||
@@ -105,9 +105,12 @@ class TestMambaPathStateCap(unittest.TestCase):
|
||||
def test_server_arg_rejects_zero_and_values_below_negative_one(self):
|
||||
for value in (0, -2):
|
||||
args = ServerArgs(model_path="dummy", mamba_max_states_per_path=value)
|
||||
with self.subTest(value=value), self.assertRaisesRegex(
|
||||
ValueError,
|
||||
"must be -1 \\(unlimited\\) or a positive integer",
|
||||
with (
|
||||
self.subTest(value=value),
|
||||
self.assertRaisesRegex(
|
||||
ValueError,
|
||||
"must be -1 \\(unlimited\\) or a positive integer",
|
||||
),
|
||||
):
|
||||
handle_mamba_backend(args)
|
||||
|
||||
|
||||
@@ -300,17 +300,17 @@ class TestMamba(unittest.TestCase):
|
||||
full_num_tokens = 1
|
||||
print(f"evicting {full_num_tokens} full token")
|
||||
result = tree.evict(EvictParams(num_tokens=full_num_tokens))
|
||||
assert (
|
||||
result.num_tokens_evicted >= full_num_tokens
|
||||
), f"evicted {result.num_tokens_evicted} full tokens, expected {full_num_tokens}"
|
||||
assert result.num_tokens_evicted >= full_num_tokens, (
|
||||
f"evicted {result.num_tokens_evicted} full tokens, expected {full_num_tokens}"
|
||||
)
|
||||
tree.pretty_print()
|
||||
|
||||
mamba_num = 1
|
||||
print(f"evicting {mamba_num} mamba")
|
||||
result = tree.evict(EvictParams(num_tokens=0, mamba_num=mamba_num))
|
||||
assert (
|
||||
result.mamba_num_evicted >= mamba_num
|
||||
), f"evicted {result.mamba_num_evicted} mamba states, expected {mamba_num}"
|
||||
assert result.mamba_num_evicted >= mamba_num, (
|
||||
f"evicted {result.mamba_num_evicted} mamba states, expected {mamba_num}"
|
||||
)
|
||||
tree.pretty_print()
|
||||
|
||||
req5_token_ids = [1, 2, 3, 4, 5]
|
||||
@@ -348,9 +348,9 @@ class TestMamba(unittest.TestCase):
|
||||
mamba_num = 1
|
||||
print(f"evicting {mamba_num} mamba")
|
||||
result = tree.evict(EvictParams(num_tokens=0, mamba_num=mamba_num))
|
||||
assert (
|
||||
result.mamba_num_evicted >= mamba_num
|
||||
), f"evicted {result.mamba_num_evicted} mamba states, expected {mamba_num}"
|
||||
assert result.mamba_num_evicted >= mamba_num, (
|
||||
f"evicted {result.mamba_num_evicted} mamba states, expected {mamba_num}"
|
||||
)
|
||||
tree.pretty_print()
|
||||
|
||||
req8_token_ids = [1, 2, 3, 4, 5, 60, 70]
|
||||
|
||||
@@ -242,10 +242,11 @@ class TestHostMemoryBudget(CustomTestCase):
|
||||
# Deliberate single-accessor stub: isolates the budget math from the
|
||||
# topology derivation, which the ranks_per_host case below covers.
|
||||
fake_mem = unittest.mock.Mock(available=self._AVAILABLE)
|
||||
with unittest.mock.patch.object(
|
||||
base, "ranks_per_host", return_value=ranks
|
||||
), unittest.mock.patch.object(
|
||||
base.psutil, "virtual_memory", return_value=fake_mem
|
||||
with (
|
||||
unittest.mock.patch.object(base, "ranks_per_host", return_value=ranks),
|
||||
unittest.mock.patch.object(
|
||||
base.psutil, "virtual_memory", return_value=fake_mem
|
||||
),
|
||||
):
|
||||
return base.host_memory_budget_bytes()
|
||||
|
||||
@@ -264,9 +265,15 @@ class TestHostMemoryBudget(CustomTestCase):
|
||||
# The launcher slices ranks uniformly across nodes, so the co-located
|
||||
# rank count is world_size // nnodes — no hostname collective.
|
||||
fake_group = unittest.mock.Mock(world_size=16)
|
||||
with get_context().override_server_args(nnodes=2), unittest.mock.patch.object(
|
||||
torch.distributed, "is_initialized", return_value=True
|
||||
), unittest.mock.patch.object(base, "get_world_group", return_value=fake_group):
|
||||
with (
|
||||
get_context().override_server_args(nnodes=2),
|
||||
unittest.mock.patch.object(
|
||||
torch.distributed, "is_initialized", return_value=True
|
||||
),
|
||||
unittest.mock.patch.object(
|
||||
base, "get_world_group", return_value=fake_group
|
||||
),
|
||||
):
|
||||
self.assertEqual(base.ranks_per_host(), 8)
|
||||
|
||||
|
||||
|
||||
@@ -83,9 +83,12 @@ class TestMmapAllocator(unittest.TestCase):
|
||||
|
||||
# MAP_POPULATE is unreachable on a 5.14+ kernel, so CI never runs it;
|
||||
# force the branch or it ships untested.
|
||||
with self.subTest(path="map_populate"), unittest.mock.patch(
|
||||
"sglang.srt.mem_cache.storage.mmap.mmap_allocator._has_madv_populate_write",
|
||||
return_value=False,
|
||||
with (
|
||||
self.subTest(path="map_populate"),
|
||||
unittest.mock.patch(
|
||||
"sglang.srt.mem_cache.storage.mmap.mmap_allocator._has_madv_populate_write",
|
||||
return_value=False,
|
||||
),
|
||||
):
|
||||
mm = _mmap_prefaulted(-1, alloc_bytes, flags)
|
||||
try:
|
||||
|
||||
@@ -3141,12 +3141,16 @@ class TestFloatMultiEndedAllocator(unittest.TestCase):
|
||||
_, _, fla, _, kv = self._build_tri()
|
||||
v = fla.alloc(4)
|
||||
self._stamp(fla, kv, v)
|
||||
with mock.patch.object(
|
||||
torch.Tensor, "tolist", side_effect=AssertionError("tolist = D2H")
|
||||
), mock.patch.object(
|
||||
torch.Tensor, "item", side_effect=AssertionError("item = D2H")
|
||||
), mock.patch.object(
|
||||
torch, "unique", side_effect=AssertionError("unique = host sync")
|
||||
with (
|
||||
mock.patch.object(
|
||||
torch.Tensor, "tolist", side_effect=AssertionError("tolist = D2H")
|
||||
),
|
||||
mock.patch.object(
|
||||
torch.Tensor, "item", side_effect=AssertionError("item = D2H")
|
||||
),
|
||||
mock.patch.object(
|
||||
torch, "unique", side_effect=AssertionError("unique = host sync")
|
||||
),
|
||||
):
|
||||
fla.free(v[:2], _pages=v[:2])
|
||||
|
||||
|
||||
@@ -18,7 +18,6 @@ register_cpu_ci(est_time=8, suite="base-a-test-cpu")
|
||||
|
||||
|
||||
class TestSLRUAccuracy(unittest.TestCase):
|
||||
|
||||
def setUp(self):
|
||||
"""Setup minimal memory pools for testing"""
|
||||
torch.set_default_device(None)
|
||||
|
||||
@@ -882,8 +882,9 @@ def test_prefetch_node_accessors_round_trip():
|
||||
|
||||
assert not core.is_backuped(leaf)
|
||||
assert not core.is_root(leaf)
|
||||
assert core.get_last_hash_value(leaf) == (
|
||||
mem_cache.get_hash_str(array("q", [1, 2]), None, 2)[-1]
|
||||
assert (
|
||||
core.get_last_hash_value(leaf)
|
||||
== (mem_cache.get_hash_str(array("q", [1, 2]), None, 2)[-1])
|
||||
)
|
||||
assert core.get_prefix_hash_values(leaf) == []
|
||||
|
||||
|
||||
@@ -53,7 +53,6 @@ def _recv(rid, input_ids, max_new_tokens=8):
|
||||
|
||||
|
||||
class TestSessionTokenShare(CustomTestCase):
|
||||
|
||||
def setUp(self):
|
||||
self.session = Session(capacity_of_str_len=0, session_id="s", streaming=True)
|
||||
|
||||
|
||||
@@ -134,7 +134,6 @@ def _make_batch(tree, allocator, pool):
|
||||
|
||||
|
||||
class TestSWAEvictionBoundary(unittest.TestCase):
|
||||
|
||||
# -- Eviction formula: page_size > window --
|
||||
|
||||
def test_formula_page_gt_window_sweep(self):
|
||||
|
||||
@@ -727,9 +727,9 @@ class TestSWA(unittest.TestCase):
|
||||
EvictParams(num_tokens=full_num_tokens, swa_num_tokens=swa_num_tokens)
|
||||
)
|
||||
assert isinstance(evict_result, EvictResult)
|
||||
assert (
|
||||
evict_result.swa_num_tokens_evicted >= swa_num_tokens
|
||||
), f"evicted {evict_result.swa_num_tokens_evicted} swa tokens, expected {swa_num_tokens}"
|
||||
assert evict_result.swa_num_tokens_evicted >= swa_num_tokens, (
|
||||
f"evicted {evict_result.swa_num_tokens_evicted} swa tokens, expected {swa_num_tokens}"
|
||||
)
|
||||
tree.pretty_print()
|
||||
|
||||
full_num_tokens, swa_num_tokens = 1, 2
|
||||
@@ -738,12 +738,12 @@ class TestSWA(unittest.TestCase):
|
||||
EvictParams(num_tokens=full_num_tokens, swa_num_tokens=swa_num_tokens)
|
||||
)
|
||||
assert isinstance(evict_result, EvictResult)
|
||||
assert (
|
||||
evict_result.num_tokens_evicted >= full_num_tokens
|
||||
), f"evicted {evict_result.num_tokens_evicted} full tokens, expected {full_num_tokens}"
|
||||
assert (
|
||||
evict_result.swa_num_tokens_evicted >= swa_num_tokens
|
||||
), f"evicted {evict_result.swa_num_tokens_evicted} swa tokens, expected {swa_num_tokens}"
|
||||
assert evict_result.num_tokens_evicted >= full_num_tokens, (
|
||||
f"evicted {evict_result.num_tokens_evicted} full tokens, expected {full_num_tokens}"
|
||||
)
|
||||
assert evict_result.swa_num_tokens_evicted >= swa_num_tokens, (
|
||||
f"evicted {evict_result.swa_num_tokens_evicted} swa tokens, expected {swa_num_tokens}"
|
||||
)
|
||||
tree.pretty_print()
|
||||
|
||||
req5_token_ids = [1, 2, 3, 4, 5]
|
||||
|
||||
@@ -457,10 +457,13 @@ class TestFreeSwaWindowRatchetNoHostSync(unittest.TestCase):
|
||||
alloc = self._swa_composite(lazy=True)
|
||||
v = alloc.alloc(8 * self.PS)
|
||||
self.assertIsNotNone(v)
|
||||
with mock.patch.object(
|
||||
torch, "unique", side_effect=AssertionError("unique = host sync")
|
||||
), mock.patch.object(
|
||||
torch.Tensor, "item", side_effect=AssertionError("item = host sync")
|
||||
with (
|
||||
mock.patch.object(
|
||||
torch, "unique", side_effect=AssertionError("unique = host sync")
|
||||
),
|
||||
mock.patch.object(
|
||||
torch.Tensor, "item", side_effect=AssertionError("item = host sync")
|
||||
),
|
||||
):
|
||||
alloc.free_swa(v[: 4 * self.PS], start_pos=0)
|
||||
alloc.free_swa(v[4 * self.PS :], start_pos=4 * self.PS)
|
||||
|
||||
@@ -424,9 +424,9 @@ def bench_api(
|
||||
(excluded from latency measurement).
|
||||
"""
|
||||
items = setup_fn()
|
||||
assert (
|
||||
len(items) >= num_ops + warmup
|
||||
), f"need {num_ops + warmup} items, got {len(items)}"
|
||||
assert len(items) >= num_ops + warmup, (
|
||||
f"need {num_ops + warmup} items, got {len(items)}"
|
||||
)
|
||||
|
||||
for i in range(warmup):
|
||||
op_fn(items[i])
|
||||
|
||||
@@ -299,11 +299,11 @@ class TestUnifiedTreeCoreLoadBackPending(CustomTestCase):
|
||||
core.node_by_id.side_effect = nodes.__getitem__
|
||||
core.components_by_type = {ComponentType.FULL: mock.Mock()}
|
||||
core.full_host_duplicates = {}
|
||||
core._is_settled_full_host_duplicate.side_effect = (
|
||||
lambda node: UnifiedTreeCore._is_settled_full_host_duplicate(core, node)
|
||||
core._is_settled_full_host_duplicate.side_effect = lambda node: (
|
||||
UnifiedTreeCore._is_settled_full_host_duplicate(core, node)
|
||||
)
|
||||
core._update_duplicate_tracking.side_effect = (
|
||||
lambda node: UnifiedTreeCore._update_duplicate_tracking(core, node)
|
||||
core._update_duplicate_tracking.side_effect = lambda node: (
|
||||
UnifiedTreeCore._update_duplicate_tracking(core, node)
|
||||
)
|
||||
return core, shared, anchor_a, anchor_b
|
||||
|
||||
@@ -1265,7 +1265,6 @@ class TestUnifiedRadixCacheKVEvents(CustomTestCase):
|
||||
|
||||
|
||||
class UnifiedRadixCacheSuite:
|
||||
|
||||
cfg: CacheConfig
|
||||
_rid: int = 0
|
||||
|
||||
@@ -3572,8 +3571,9 @@ class UnifiedRadixCacheSuite:
|
||||
self.assertIn(n, pipeline.inflight_backup_node_ids)
|
||||
self._pump_hicache_until(
|
||||
cache,
|
||||
lambda: not pipeline.inflight_backup_node_ids
|
||||
and not pipeline.ongoing_backup,
|
||||
lambda: (
|
||||
not pipeline.inflight_backup_node_ids and not pipeline.ongoing_backup
|
||||
),
|
||||
"buffer backup pipeline did not drain",
|
||||
)
|
||||
|
||||
@@ -3689,8 +3689,10 @@ class UnifiedRadixCacheSuite:
|
||||
self.assertEqual((stats["attempts"], stats["issued"]), (1, 1))
|
||||
self._pump_hicache_until(
|
||||
cons,
|
||||
lambda: cons.check_prefetch_progress(req_id)
|
||||
and cons.buffer_pipeline.has_staged(req_id),
|
||||
lambda: (
|
||||
cons.check_prefetch_progress(req_id)
|
||||
and cons.buffer_pipeline.has_staged(req_id)
|
||||
),
|
||||
"prefetch did not stage",
|
||||
)
|
||||
# Staged: bounce occupies host staging; nothing device-side; span
|
||||
@@ -3753,8 +3755,10 @@ class UnifiedRadixCacheSuite:
|
||||
# loaded KV bytes equal the producer's.
|
||||
self._pump_hicache_until(
|
||||
cons,
|
||||
lambda: not cons.buffer_pipeline.ongoing_buffer_load_back
|
||||
and self._host_avail_sizes(cons) == avail0,
|
||||
lambda: (
|
||||
not cons.buffer_pipeline.ongoing_buffer_load_back
|
||||
and self._host_avail_sizes(cons) == avail0
|
||||
),
|
||||
"load-back ack did not free the bounce",
|
||||
)
|
||||
mc = cons.match_prefix(MatchPrefixParams(key=RadixKey(array("q", seq))))
|
||||
@@ -3818,8 +3822,10 @@ class UnifiedRadixCacheSuite:
|
||||
)
|
||||
self._pump_hicache_until(
|
||||
cons,
|
||||
lambda: cons.check_prefetch_progress(root_req)
|
||||
and cons.buffer_pipeline.has_staged(root_req),
|
||||
lambda: (
|
||||
cons.check_prefetch_progress(root_req)
|
||||
and cons.buffer_pipeline.has_staged(root_req)
|
||||
),
|
||||
"salted root prefetch did not stage",
|
||||
)
|
||||
held = cons.buffer_pipeline.staged_prefetches[root_req]
|
||||
@@ -3888,8 +3894,10 @@ class UnifiedRadixCacheSuite:
|
||||
)
|
||||
self._pump_hicache_until(
|
||||
cons2,
|
||||
lambda: cons2.check_prefetch_progress(anchored_req)
|
||||
and cons2.buffer_pipeline.has_staged(anchored_req),
|
||||
lambda: (
|
||||
cons2.check_prefetch_progress(anchored_req)
|
||||
and cons2.buffer_pipeline.has_staged(anchored_req)
|
||||
),
|
||||
"salted mid-tree prefetch did not stage",
|
||||
)
|
||||
self._consume_staged_prefetch(
|
||||
@@ -3950,8 +3958,10 @@ class UnifiedRadixCacheSuite:
|
||||
)
|
||||
self._pump_hicache_until(
|
||||
cons,
|
||||
lambda: cons.check_prefetch_progress(req_id)
|
||||
and cons.buffer_pipeline.has_staged(req_id),
|
||||
lambda: (
|
||||
cons.check_prefetch_progress(req_id)
|
||||
and cons.buffer_pipeline.has_staged(req_id)
|
||||
),
|
||||
"retried prefetch did not stage",
|
||||
)
|
||||
self.assertFalse(cons.pop_storage_prefetch_miss(req_id))
|
||||
@@ -4047,8 +4057,10 @@ class UnifiedRadixCacheSuite:
|
||||
)
|
||||
self._pump_hicache_until(
|
||||
cons,
|
||||
lambda: cons.check_prefetch_progress(req_id)
|
||||
and cons.buffer_pipeline.has_staged(req_id),
|
||||
lambda: (
|
||||
cons.check_prefetch_progress(req_id)
|
||||
and cons.buffer_pipeline.has_staged(req_id)
|
||||
),
|
||||
"prefetch did not stage",
|
||||
)
|
||||
|
||||
@@ -4151,8 +4163,10 @@ class UnifiedRadixCacheSuite:
|
||||
)
|
||||
self._pump_hicache_until(
|
||||
cons,
|
||||
lambda: cons.check_prefetch_progress(req_id)
|
||||
and cons.buffer_pipeline.has_staged(req_id),
|
||||
lambda: (
|
||||
cons.check_prefetch_progress(req_id)
|
||||
and cons.buffer_pipeline.has_staged(req_id)
|
||||
),
|
||||
"prefetch did not stage",
|
||||
)
|
||||
cons.pop_prefetch_loaded_tokens(req_id)
|
||||
@@ -4222,8 +4236,10 @@ class UnifiedRadixCacheSuite:
|
||||
)
|
||||
self._pump_hicache_until(
|
||||
cons,
|
||||
lambda: cons.check_prefetch_progress(req_id)
|
||||
and cons.buffer_pipeline.has_staged(req_id),
|
||||
lambda: (
|
||||
cons.check_prefetch_progress(req_id)
|
||||
and cons.buffer_pipeline.has_staged(req_id)
|
||||
),
|
||||
"prefetch did not stage",
|
||||
)
|
||||
cons.pop_prefetch_loaded_tokens(req_id)
|
||||
@@ -4331,8 +4347,10 @@ class UnifiedRadixCacheSuite:
|
||||
)
|
||||
self._pump_hicache_until(
|
||||
cons,
|
||||
lambda: cons.check_prefetch_progress(req_id)
|
||||
and cons.buffer_pipeline.has_staged(req_id),
|
||||
lambda: (
|
||||
cons.check_prefetch_progress(req_id)
|
||||
and cons.buffer_pipeline.has_staged(req_id)
|
||||
),
|
||||
"prefetch did not stage",
|
||||
)
|
||||
cons.pop_prefetch_loaded_tokens(req_id)
|
||||
@@ -4398,8 +4416,10 @@ class UnifiedRadixCacheSuite:
|
||||
)
|
||||
self._pump_hicache_until(
|
||||
cons,
|
||||
lambda: cons.check_prefetch_progress(req_id)
|
||||
and cons.buffer_pipeline.has_staged(req_id),
|
||||
lambda: (
|
||||
cons.check_prefetch_progress(req_id)
|
||||
and cons.buffer_pipeline.has_staged(req_id)
|
||||
),
|
||||
"prefetch did not stage",
|
||||
)
|
||||
cons.pop_prefetch_loaded_tokens(req_id)
|
||||
@@ -4519,8 +4539,10 @@ class UnifiedRadixCacheSuite:
|
||||
)
|
||||
self._pump_hicache_until(
|
||||
cons2,
|
||||
lambda: cons2.check_prefetch_progress("subwin-req")
|
||||
and cons2.buffer_pipeline.has_staged("subwin-req"),
|
||||
lambda: (
|
||||
cons2.check_prefetch_progress("subwin-req")
|
||||
and cons2.buffer_pipeline.has_staged("subwin-req")
|
||||
),
|
||||
"sub-window prefetch did not stage",
|
||||
)
|
||||
self.assertTrue(
|
||||
@@ -7081,7 +7103,6 @@ class UnifiedRadixCacheSuite:
|
||||
|
||||
|
||||
class UnifiedLRUListBoundedRefreshTest(CustomTestCase):
|
||||
|
||||
components = (ComponentType.FULL, ComponentType.SWA)
|
||||
|
||||
def _make_node(self, key_len: int) -> UnifiedTreeNode:
|
||||
|
||||
@@ -465,10 +465,13 @@ class TestTriFreeSwaNoHostSync(unittest.TestCase):
|
||||
self.assertIsNotNone(v)
|
||||
from unittest import mock
|
||||
|
||||
with mock.patch.object(
|
||||
torch, "unique", side_effect=AssertionError("unique = host sync")
|
||||
), mock.patch.object(
|
||||
torch.Tensor, "item", side_effect=AssertionError("item = host sync")
|
||||
with (
|
||||
mock.patch.object(
|
||||
torch, "unique", side_effect=AssertionError("unique = host sync")
|
||||
),
|
||||
mock.patch.object(
|
||||
torch.Tensor, "item", side_effect=AssertionError("item = host sync")
|
||||
),
|
||||
):
|
||||
alloc.free_swa(v[: 4 * self.PS], start_pos=0)
|
||||
self.assertEqual(alloc.verify_byte_accounting(), [])
|
||||
|
||||
@@ -60,13 +60,17 @@ class TestInitProfileBatchMode(CustomTestCase):
|
||||
env = {_BATCH_CAPTURE: "1"}
|
||||
if profiler_dir is not None:
|
||||
env["SGLANG_TORCH_PROFILER_DIR"] = profiler_dir
|
||||
with mock.patch.dict(os.environ, env, clear=False), mock.patch.object(
|
||||
mod, "get_parallel", return_value=SimpleNamespace(tp_rank=rank)
|
||||
), mock.patch.object(mod, "profile") as mock_profile, mock.patch(
|
||||
"torch.profiler.schedule"
|
||||
) as mock_schedule, mock.patch(
|
||||
"torch.cuda.memory._record_memory_history"
|
||||
) as mock_record_history:
|
||||
with (
|
||||
mock.patch.dict(os.environ, env, clear=False),
|
||||
mock.patch.object(
|
||||
mod, "get_parallel", return_value=SimpleNamespace(tp_rank=rank)
|
||||
),
|
||||
mock.patch.object(mod, "profile") as mock_profile,
|
||||
mock.patch("torch.profiler.schedule") as mock_schedule,
|
||||
mock.patch(
|
||||
"torch.cuda.memory._record_memory_history"
|
||||
) as mock_record_history,
|
||||
):
|
||||
os.environ.pop(_CAPTURE_TRACE, None) # original flag off
|
||||
if profiler_dir is None:
|
||||
os.environ.pop("SGLANG_TORCH_PROFILER_DIR", None)
|
||||
@@ -110,19 +114,16 @@ class TestInitProfileBatchMode(CustomTestCase):
|
||||
# No SGLANG_TORCH_PROFILER_DIR -> falls back to the envs default base dir.
|
||||
# Patch makedirs so the test never writes to the cwd.
|
||||
fake_self = _make_fake_self([1])
|
||||
with mock.patch.dict(
|
||||
os.environ, {_BATCH_CAPTURE: "1"}, clear=False
|
||||
), mock.patch.object(
|
||||
mod, "get_parallel", return_value=SimpleNamespace(tp_rank=0)
|
||||
), mock.patch.object(
|
||||
mod, "profile"
|
||||
), mock.patch(
|
||||
"torch.profiler.schedule"
|
||||
), mock.patch(
|
||||
"torch.cuda.memory._record_memory_history"
|
||||
), mock.patch.object(
|
||||
mod.os, "makedirs"
|
||||
) as mock_makedirs:
|
||||
with (
|
||||
mock.patch.dict(os.environ, {_BATCH_CAPTURE: "1"}, clear=False),
|
||||
mock.patch.object(
|
||||
mod, "get_parallel", return_value=SimpleNamespace(tp_rank=0)
|
||||
),
|
||||
mock.patch.object(mod, "profile"),
|
||||
mock.patch("torch.profiler.schedule"),
|
||||
mock.patch("torch.cuda.memory._record_memory_history"),
|
||||
mock.patch.object(mod.os, "makedirs") as mock_makedirs,
|
||||
):
|
||||
os.environ.pop("SGLANG_TORCH_PROFILER_DIR", None)
|
||||
os.environ.pop(_CAPTURE_TRACE, None)
|
||||
DecodeCudaGraphRunner._init_profile_context_and_memory_record(fake_self)
|
||||
@@ -143,12 +144,14 @@ class TestInitProfileOriginalMode(CustomTestCase):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
environ = dict(env)
|
||||
environ["SGLANG_TORCH_PROFILER_DIR"] = tmp
|
||||
with mock.patch.dict(os.environ, environ, clear=False), mock.patch.object(
|
||||
mod, "get_parallel", return_value=SimpleNamespace(tp_rank=0)
|
||||
), mock.patch.object(mod, "profile") as mock_profile, mock.patch(
|
||||
"torch.profiler.schedule"
|
||||
) as mock_schedule, mock.patch(
|
||||
"torch.cuda.memory._record_memory_history"
|
||||
with (
|
||||
mock.patch.dict(os.environ, environ, clear=False),
|
||||
mock.patch.object(
|
||||
mod, "get_parallel", return_value=SimpleNamespace(tp_rank=0)
|
||||
),
|
||||
mock.patch.object(mod, "profile") as mock_profile,
|
||||
mock.patch("torch.profiler.schedule") as mock_schedule,
|
||||
mock.patch("torch.cuda.memory._record_memory_history"),
|
||||
):
|
||||
for k in (_CAPTURE_TRACE, _BATCH_CAPTURE):
|
||||
if k not in environ:
|
||||
@@ -176,18 +179,18 @@ class TestInitProfileOriginalMode(CustomTestCase):
|
||||
class TestOnTraceReadyNaming(CustomTestCase):
|
||||
def _build_on_trace_ready(self, *, capture_bs, rank, tmp):
|
||||
fake_self = _make_fake_self(capture_bs)
|
||||
with mock.patch.dict(
|
||||
os.environ,
|
||||
{"SGLANG_TORCH_PROFILER_DIR": tmp, _BATCH_CAPTURE: "1"},
|
||||
clear=False,
|
||||
), mock.patch.object(
|
||||
mod, "get_parallel", return_value=SimpleNamespace(tp_rank=rank)
|
||||
), mock.patch.object(
|
||||
mod, "profile"
|
||||
) as mock_profile, mock.patch(
|
||||
"torch.profiler.schedule"
|
||||
), mock.patch(
|
||||
"torch.cuda.memory._record_memory_history"
|
||||
with (
|
||||
mock.patch.dict(
|
||||
os.environ,
|
||||
{"SGLANG_TORCH_PROFILER_DIR": tmp, _BATCH_CAPTURE: "1"},
|
||||
clear=False,
|
||||
),
|
||||
mock.patch.object(
|
||||
mod, "get_parallel", return_value=SimpleNamespace(tp_rank=rank)
|
||||
),
|
||||
mock.patch.object(mod, "profile") as mock_profile,
|
||||
mock.patch("torch.profiler.schedule"),
|
||||
mock.patch("torch.cuda.memory._record_memory_history"),
|
||||
):
|
||||
os.environ.pop(_CAPTURE_TRACE, None)
|
||||
DecodeCudaGraphRunner._init_profile_context_and_memory_record(fake_self)
|
||||
|
||||
@@ -118,9 +118,12 @@ class TestHiddenStateGraphRecapture(CustomTestCase):
|
||||
):
|
||||
runner = self._make_runner(runner_cls, CaptureHiddenMode.NULL)
|
||||
|
||||
with self.subTest(runner_cls=runner_cls), self.assertRaisesRegex(
|
||||
RuntimeError,
|
||||
"exceeds the fixed (CUDA|CPU) graph capture mode",
|
||||
with (
|
||||
self.subTest(runner_cls=runner_cls),
|
||||
self.assertRaisesRegex(
|
||||
RuntimeError,
|
||||
"exceeds the fixed (CUDA|CPU) graph capture mode",
|
||||
),
|
||||
):
|
||||
runner._validate_capture_hidden_mode(
|
||||
self._make_forward_batch(CaptureHiddenMode.LAST)
|
||||
|
||||
@@ -139,8 +139,11 @@ class TestCaptureOneWithProfiling(CustomTestCase):
|
||||
rf_names.append(name)
|
||||
return contextlib.nullcontext()
|
||||
|
||||
with mock.patch("torch.cuda.CUDAGraph", return_value="GRAPH"), mock.patch(
|
||||
"torch.profiler.record_function", side_effect=_fake_record_function
|
||||
with (
|
||||
mock.patch("torch.cuda.CUDAGraph", return_value="GRAPH"),
|
||||
mock.patch(
|
||||
"torch.profiler.record_function", side_effect=_fake_record_function
|
||||
),
|
||||
):
|
||||
backend.capture_one(ShapeKey(size=size), forward_fn)
|
||||
|
||||
|
||||
@@ -35,14 +35,18 @@ class TestModelRunnerDecodeRows(unittest.TestCase):
|
||||
spec = SimpleNamespace(
|
||||
speculative_adaptive=True, speculative_adaptive_config=f.name
|
||||
)
|
||||
with patch(
|
||||
"sglang.srt.model_executor.model_runner.get_spec", return_value=spec
|
||||
), patch(
|
||||
"sglang.srt.model_executor.model_runner.max_speculative_num_draft_tokens",
|
||||
return_value=6,
|
||||
), patch(
|
||||
"sglang.srt.model_executor.model_runner.get_batch_sizes_to_capture",
|
||||
side_effect=_alignment_8_capture_bs,
|
||||
with (
|
||||
patch(
|
||||
"sglang.srt.model_executor.model_runner.get_spec", return_value=spec
|
||||
),
|
||||
patch(
|
||||
"sglang.srt.model_executor.model_runner.max_speculative_num_draft_tokens",
|
||||
return_value=6,
|
||||
),
|
||||
patch(
|
||||
"sglang.srt.model_executor.model_runner.get_batch_sizes_to_capture",
|
||||
side_effect=_alignment_8_capture_bs,
|
||||
),
|
||||
):
|
||||
self.assertEqual(runner.max_decode_logits_rows(), 72)
|
||||
|
||||
|
||||
@@ -813,8 +813,9 @@ class TestDSAIndexerAllocationPolicy(CustomTestCase):
|
||||
mr.model_config.hf_config.index_topk_freq = 4
|
||||
mr.model_config.hf_config.index_skip_topk_offset = 3
|
||||
|
||||
with get_memory().override(enable_hierarchical_cache=True), mock_cpu_env(
|
||||
kv_size=1
|
||||
with (
|
||||
get_memory().override(enable_hierarchical_cache=True),
|
||||
mock_cpu_env(kv_size=1),
|
||||
):
|
||||
from sglang.srt.model_executor.pool_configurator import (
|
||||
DefaultPoolConfigurator,
|
||||
|
||||
@@ -374,7 +374,10 @@ class TestBuildDumpPlan(unittest.TestCase):
|
||||
# and depend on (name, content-SHA) pairs of every tensor a rank
|
||||
# owns. Permuting the manifest's insertion order must not change
|
||||
# the rank checksum.
|
||||
with tempfile.TemporaryDirectory() as tmp_a, tempfile.TemporaryDirectory() as tmp_b:
|
||||
with (
|
||||
tempfile.TemporaryDirectory() as tmp_a,
|
||||
tempfile.TemporaryDirectory() as tmp_b,
|
||||
):
|
||||
base_entries = {
|
||||
"alpha.weight": {
|
||||
"checksum": "h_alpha",
|
||||
@@ -403,7 +406,10 @@ class TestBuildDumpPlan(unittest.TestCase):
|
||||
|
||||
def test_rank_checksum_distinguishes_content(self):
|
||||
# Changing one tensor's content-SHA must change the rank checksum.
|
||||
with tempfile.TemporaryDirectory() as tmp_a, tempfile.TemporaryDirectory() as tmp_b:
|
||||
with (
|
||||
tempfile.TemporaryDirectory() as tmp_a,
|
||||
tempfile.TemporaryDirectory() as tmp_b,
|
||||
):
|
||||
entries_a = {
|
||||
"x.weight": {
|
||||
"checksum": "ha",
|
||||
@@ -537,10 +543,10 @@ class TestBuildDumpPlan(unittest.TestCase):
|
||||
# read-only dump root (HF cache mounts). Mock OSError because root
|
||||
# can often still write to mode-0555 dirs under CAP_DAC_OVERRIDE.
|
||||
loader = object.__new__(PreshardedModelLoader)
|
||||
with mock.patch.object(
|
||||
loader, "_world_rank_and_size", return_value=(0, 1)
|
||||
), mock.patch.object(loader, "_world_barrier"), mock.patch(
|
||||
"os.makedirs", side_effect=OSError("Read-only file system")
|
||||
with (
|
||||
mock.patch.object(loader, "_world_rank_and_size", return_value=(0, 1)),
|
||||
mock.patch.object(loader, "_world_barrier"),
|
||||
mock.patch("os.makedirs", side_effect=OSError("Read-only file system")),
|
||||
):
|
||||
with self.assertRaises(RuntimeError) as ctx:
|
||||
loader._ensure_presharded_dir_writable("/ro/presharded")
|
||||
@@ -551,9 +557,10 @@ class TestBuildDumpPlan(unittest.TestCase):
|
||||
loader = object.__new__(PreshardedModelLoader)
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
leaf = os.path.join(tmp, "TP-8-sig-test")
|
||||
with mock.patch.object(
|
||||
loader, "_world_rank_and_size", return_value=(0, 1)
|
||||
), mock.patch.object(loader, "_world_barrier") as barrier:
|
||||
with (
|
||||
mock.patch.object(loader, "_world_rank_and_size", return_value=(0, 1)),
|
||||
mock.patch.object(loader, "_world_barrier") as barrier,
|
||||
):
|
||||
loader._ensure_presharded_dir_writable(leaf)
|
||||
self.assertTrue(os.path.isdir(leaf))
|
||||
barrier.assert_called_once()
|
||||
@@ -605,15 +612,21 @@ class TestStructuralSignature(unittest.TestCase):
|
||||
|
||||
loader = object.__new__(PreshardedModelLoader)
|
||||
loader.load_config = SimpleNamespace()
|
||||
with mock.patch(
|
||||
"sglang.srt.model_loader.loader._get_quantization_config",
|
||||
return_value=None,
|
||||
), mock.patch(
|
||||
"sglang.srt.model_loader.loader._initialize_model",
|
||||
side_effect=_init_stub,
|
||||
), mock.patch(
|
||||
"sglang.srt.model_loader.loader.set_default_torch_dtype",
|
||||
return_value=mock.MagicMock(__enter__=mock.Mock(), __exit__=mock.Mock()),
|
||||
with (
|
||||
mock.patch(
|
||||
"sglang.srt.model_loader.loader._get_quantization_config",
|
||||
return_value=None,
|
||||
),
|
||||
mock.patch(
|
||||
"sglang.srt.model_loader.loader._initialize_model",
|
||||
side_effect=_init_stub,
|
||||
),
|
||||
mock.patch(
|
||||
"sglang.srt.model_loader.loader.set_default_torch_dtype",
|
||||
return_value=mock.MagicMock(
|
||||
__enter__=mock.Mock(), __exit__=mock.Mock()
|
||||
),
|
||||
),
|
||||
):
|
||||
sig_ab = loader._compute_local_structural_signature(
|
||||
SimpleNamespace(
|
||||
@@ -694,15 +707,21 @@ class TestStructuralSignature(unittest.TestCase):
|
||||
def _initialize_model_stub(model_config, load_config, quant_config):
|
||||
return nn.Linear(4, model_config.width, bias=False)
|
||||
|
||||
with mock.patch(
|
||||
"sglang.srt.model_loader.loader._get_quantization_config",
|
||||
return_value=None,
|
||||
), mock.patch(
|
||||
"sglang.srt.model_loader.loader._initialize_model",
|
||||
side_effect=_initialize_model_stub,
|
||||
), mock.patch(
|
||||
"sglang.srt.model_loader.loader.set_default_torch_dtype",
|
||||
return_value=mock.MagicMock(__enter__=mock.Mock(), __exit__=mock.Mock()),
|
||||
with (
|
||||
mock.patch(
|
||||
"sglang.srt.model_loader.loader._get_quantization_config",
|
||||
return_value=None,
|
||||
),
|
||||
mock.patch(
|
||||
"sglang.srt.model_loader.loader._initialize_model",
|
||||
side_effect=_initialize_model_stub,
|
||||
),
|
||||
mock.patch(
|
||||
"sglang.srt.model_loader.loader.set_default_torch_dtype",
|
||||
return_value=mock.MagicMock(
|
||||
__enter__=mock.Mock(), __exit__=mock.Mock()
|
||||
),
|
||||
),
|
||||
):
|
||||
loader_narrow = FakeModelLoader(width=2)
|
||||
loader_wide = FakeModelLoader(width=8)
|
||||
@@ -738,15 +757,21 @@ class TestStructuralSignature(unittest.TestCase):
|
||||
loader = object.__new__(PreshardedModelLoader)
|
||||
loader.load_config = SimpleNamespace()
|
||||
|
||||
with mock.patch(
|
||||
"sglang.srt.model_loader.loader._get_quantization_config",
|
||||
return_value=None,
|
||||
), mock.patch(
|
||||
"sglang.srt.model_loader.loader._initialize_model",
|
||||
side_effect=RuntimeError("simulated init failure"),
|
||||
), mock.patch(
|
||||
"sglang.srt.model_loader.loader.set_default_torch_dtype",
|
||||
return_value=mock.MagicMock(__enter__=mock.Mock(), __exit__=mock.Mock()),
|
||||
with (
|
||||
mock.patch(
|
||||
"sglang.srt.model_loader.loader._get_quantization_config",
|
||||
return_value=None,
|
||||
),
|
||||
mock.patch(
|
||||
"sglang.srt.model_loader.loader._initialize_model",
|
||||
side_effect=RuntimeError("simulated init failure"),
|
||||
),
|
||||
mock.patch(
|
||||
"sglang.srt.model_loader.loader.set_default_torch_dtype",
|
||||
return_value=mock.MagicMock(
|
||||
__enter__=mock.Mock(), __exit__=mock.Mock()
|
||||
),
|
||||
),
|
||||
):
|
||||
result = loader._compute_structural_signature(
|
||||
SimpleNamespace(quantization=None, dtype=torch.float32)
|
||||
@@ -816,23 +841,26 @@ class TestShardConfig(unittest.TestCase):
|
||||
)
|
||||
override.install()
|
||||
self.addCleanup(override.restore)
|
||||
with get_parallel().override(
|
||||
tp_size=8, pp_size=1, moe_dp_size=2, moe_ep_size=4
|
||||
), mock.patch(
|
||||
"sglang.srt.layers.dp_attention.get_moe_cp_size",
|
||||
return_value=2,
|
||||
), mock.patch(
|
||||
"sglang.srt.model_loader.loader.get_exec",
|
||||
return_value=SimpleNamespace(
|
||||
features=SimpleNamespace(enable_fp32_lm_head=True),
|
||||
moe=SimpleNamespace(
|
||||
ep_num_redundant_experts=4,
|
||||
enable_eplb=True,
|
||||
init_expert_location="trivial",
|
||||
with (
|
||||
get_parallel().override(tp_size=8, pp_size=1, moe_dp_size=2, moe_ep_size=4),
|
||||
mock.patch(
|
||||
"sglang.srt.layers.dp_attention.get_moe_cp_size",
|
||||
return_value=2,
|
||||
),
|
||||
mock.patch(
|
||||
"sglang.srt.model_loader.loader.get_exec",
|
||||
return_value=SimpleNamespace(
|
||||
features=SimpleNamespace(enable_fp32_lm_head=True),
|
||||
moe=SimpleNamespace(
|
||||
ep_num_redundant_experts=4,
|
||||
enable_eplb=True,
|
||||
init_expert_location="trivial",
|
||||
),
|
||||
),
|
||||
),
|
||||
), mock.patch.object(
|
||||
loader, "_compute_structural_signature", return_value="sig16"
|
||||
mock.patch.object(
|
||||
loader, "_compute_structural_signature", return_value="sig16"
|
||||
),
|
||||
):
|
||||
cfg = loader._collect_shard_config(model_config)
|
||||
self.assertEqual(required, set(cfg.keys()))
|
||||
@@ -931,23 +959,28 @@ class TestShardConfig(unittest.TestCase):
|
||||
|
||||
# Force rank 0 / world 1 so the method runs the rank-0 prologue
|
||||
# and then fails early on empty state (no need for full dump).
|
||||
with mock.patch.object(
|
||||
PreshardedModelLoader,
|
||||
"_world_rank_and_size",
|
||||
return_value=(0, 1),
|
||||
), mock.patch.object(
|
||||
PreshardedModelLoader, "_world_barrier", return_value=None
|
||||
), mock.patch.object(
|
||||
PreshardedModelLoader,
|
||||
"_build_dump_plan",
|
||||
return_value={
|
||||
"version": PreshardedModelLoader.PLAN_VERSION,
|
||||
"files": [],
|
||||
"rank_to_reads": {"0": []},
|
||||
"rank_checksums": {"0": "0"},
|
||||
},
|
||||
), mock.patch.object(
|
||||
PreshardedModelLoader, "_dump_files_for_rank", return_value=None
|
||||
with (
|
||||
mock.patch.object(
|
||||
PreshardedModelLoader,
|
||||
"_world_rank_and_size",
|
||||
return_value=(0, 1),
|
||||
),
|
||||
mock.patch.object(
|
||||
PreshardedModelLoader, "_world_barrier", return_value=None
|
||||
),
|
||||
mock.patch.object(
|
||||
PreshardedModelLoader,
|
||||
"_build_dump_plan",
|
||||
return_value={
|
||||
"version": PreshardedModelLoader.PLAN_VERSION,
|
||||
"files": [],
|
||||
"rank_to_reads": {"0": []},
|
||||
"rank_checksums": {"0": "0"},
|
||||
},
|
||||
),
|
||||
mock.patch.object(
|
||||
PreshardedModelLoader, "_dump_files_for_rank", return_value=None
|
||||
),
|
||||
):
|
||||
loader._dump_state_to_disk(
|
||||
state_dict={},
|
||||
|
||||
@@ -467,11 +467,13 @@ def test_kimi_k3_encoder_dp_defers_feature_materialization(monkeypatch):
|
||||
# The IPC consumer count asks for the *configured* TP size (matching
|
||||
# MmItemMemoryPool.try_to_recycle), so publish it; the live topology the
|
||||
# sharding helper reads is forced through the context's own override.
|
||||
with mock_patch(
|
||||
"sglang.srt.multimodal.mm_utils.run_dp_sharded_mrope_vision_model",
|
||||
return_value=sharded_embeddings,
|
||||
) as run_dp, get_context().override_server_args(tp_size=1), get_parallel().override(
|
||||
tp_size=1, attn_tp_size=1
|
||||
with (
|
||||
mock_patch(
|
||||
"sglang.srt.multimodal.mm_utils.run_dp_sharded_mrope_vision_model",
|
||||
return_value=sharded_embeddings,
|
||||
) as run_dp,
|
||||
get_context().override_server_args(tp_size=1),
|
||||
get_parallel().override(tp_size=1, attn_tp_size=1),
|
||||
):
|
||||
output = model.get_image_feature(items)
|
||||
# Exercise the loader while the runtime topology is forced.
|
||||
@@ -558,14 +560,17 @@ def test_kimi_k3_preprocesses_only_dp_owner_images(monkeypatch):
|
||||
|
||||
# Configured TP size (the IPC consumer count) comes from the published
|
||||
# bags; the live topology is forced through the context's own override.
|
||||
with mock_patch(
|
||||
"sglang.srt.multimodal.mm_utils.run_dp_sharded_mrope_vision_model",
|
||||
return_value=torch.zeros(1, 2),
|
||||
) as run_dp, get_context().override_server_args(tp_size=1), get_parallel().override(
|
||||
tp_size=1, attn_tp_size=1
|
||||
), mock_patch(
|
||||
"sglang.srt.multimodal.processors.kimi_k25._gpu_preprocess_images",
|
||||
side_effect=fake_preprocess,
|
||||
with (
|
||||
mock_patch(
|
||||
"sglang.srt.multimodal.mm_utils.run_dp_sharded_mrope_vision_model",
|
||||
return_value=torch.zeros(1, 2),
|
||||
) as run_dp,
|
||||
get_context().override_server_args(tp_size=1),
|
||||
get_parallel().override(tp_size=1, attn_tp_size=1),
|
||||
mock_patch(
|
||||
"sglang.srt.multimodal.processors.kimi_k25._gpu_preprocess_images",
|
||||
side_effect=fake_preprocess,
|
||||
),
|
||||
):
|
||||
model.get_image_feature(items)
|
||||
loader = run_dp.call_args.kwargs["load_local_pixel_values"]
|
||||
|
||||
@@ -78,9 +78,12 @@ def _image_item(feature, grid_hws):
|
||||
|
||||
class TestKimiVLEncoderParallelism(CustomTestCase):
|
||||
def test_moonvit_uses_tensor_parallel_layers(self):
|
||||
with get_parallel().override(
|
||||
tp_size=1, tp_rank=0, attn_tp_size=1, attn_tp_rank=0
|
||||
), get_context().override_server_args():
|
||||
with (
|
||||
get_parallel().override(
|
||||
tp_size=1, tp_rank=0, attn_tp_size=1, attn_tp_rank=0
|
||||
),
|
||||
get_context().override_server_args(),
|
||||
):
|
||||
layer = MoonVitEncoderLayer(
|
||||
num_heads=2,
|
||||
hidden_dim=8,
|
||||
|
||||
@@ -160,15 +160,19 @@ class TestQwen3VLFeatureMaterialization(CustomTestCase):
|
||||
)
|
||||
return encoded
|
||||
|
||||
with patch(
|
||||
"sglang.srt.models.qwen3_vl.run_dp_sharded_mrope_vision_model",
|
||||
side_effect=run_dp,
|
||||
), patch(
|
||||
"sglang.srt.models.qwen3_vl.materialize_multimodal_features",
|
||||
return_value=local_features,
|
||||
) as materialize, patch(
|
||||
"sglang.srt.models.qwen3_vl.get_parallel",
|
||||
return_value=SimpleNamespace(tp_size=8),
|
||||
with (
|
||||
patch(
|
||||
"sglang.srt.models.qwen3_vl.run_dp_sharded_mrope_vision_model",
|
||||
side_effect=run_dp,
|
||||
),
|
||||
patch(
|
||||
"sglang.srt.models.qwen3_vl.materialize_multimodal_features",
|
||||
return_value=local_features,
|
||||
) as materialize,
|
||||
patch(
|
||||
"sglang.srt.models.qwen3_vl.get_parallel",
|
||||
return_value=SimpleNamespace(tp_size=8),
|
||||
),
|
||||
):
|
||||
output = feature_method(items)
|
||||
|
||||
|
||||
@@ -78,12 +78,12 @@ def setUpModule():
|
||||
|
||||
if stub_modules:
|
||||
if "sglang.srt.managers.io_struct" in stub_modules:
|
||||
stub_modules["sglang.srt.managers.io_struct"].GenerateReqInput = (
|
||||
_GenerateReqInput
|
||||
)
|
||||
stub_modules["sglang.srt.managers.io_struct"].EmbeddingReqInput = (
|
||||
_EmbeddingReqInput
|
||||
)
|
||||
stub_modules[
|
||||
"sglang.srt.managers.io_struct"
|
||||
].GenerateReqInput = _GenerateReqInput
|
||||
stub_modules[
|
||||
"sglang.srt.managers.io_struct"
|
||||
].EmbeddingReqInput = _EmbeddingReqInput
|
||||
if "sglang.srt.server_args" in stub_modules:
|
||||
stub_modules["sglang.srt.server_args"].ServerArgs = _ServerArgs
|
||||
|
||||
|
||||
@@ -37,7 +37,6 @@ def _patch_hf_transformers_utils(get_tokenizer, get_config=None):
|
||||
|
||||
|
||||
class TestTemplateManagerReasoningDetection(unittest.TestCase):
|
||||
|
||||
def _detect(self, template, vocab):
|
||||
force, config = detect_reasoning_pattern(template)
|
||||
parser = detect_reasoning_parser(
|
||||
@@ -145,15 +144,13 @@ class TestTemplateManagerReasoningDetection(unittest.TestCase):
|
||||
# An explicit boolean=false second argument is equivalent to the
|
||||
# one-argument form.
|
||||
(
|
||||
"{%- set enable_thinking = enable_thinking"
|
||||
" | default(true, false) -%}",
|
||||
"{%- set enable_thinking = enable_thinking | default(true, false) -%}",
|
||||
True,
|
||||
),
|
||||
# Boolean mode with a false default still maps False -> False and
|
||||
# True -> True, so it is a working default-off toggle.
|
||||
(
|
||||
"{%- set enable_thinking = enable_thinking"
|
||||
" | default(false, true) -%}",
|
||||
"{%- set enable_thinking = enable_thinking | default(false, true) -%}",
|
||||
False,
|
||||
),
|
||||
(
|
||||
|
||||
@@ -39,7 +39,6 @@ def _make_req(origin_input_ids=None, output_ids=None):
|
||||
|
||||
# Serialization round-trip
|
||||
class TestCustomLogitProcessorSerialization(CustomTestCase):
|
||||
|
||||
def test_to_str_produces_valid_json(self):
|
||||
"""Test that to_str() produces valid JSON with a 'callable' field."""
|
||||
s = DisallowedTokensLogitsProcessor.to_str()
|
||||
|
||||
@@ -53,7 +53,6 @@ def _make_batch(reqs):
|
||||
|
||||
# BatchedPenalizerOrchestrator
|
||||
class TestBatchedPenalizerOrchestrator(CustomTestCase):
|
||||
|
||||
def test_init_detects_required_penalizers(self):
|
||||
"""Test that orchestrator marks is_required=True when any request has nonzero penalty."""
|
||||
reqs = [_make_req(freq=1.0)]
|
||||
@@ -143,7 +142,6 @@ class TestBatchedPenalizerOrchestrator(CustomTestCase):
|
||||
|
||||
# BatchedFrequencyPenalizer
|
||||
class TestBatchedFrequencyPenalizer(CustomTestCase):
|
||||
|
||||
def _setup(self, freq_values):
|
||||
reqs = [_make_req(freq=f) for f in freq_values]
|
||||
batch = _make_batch(reqs)
|
||||
@@ -225,7 +223,6 @@ class TestBatchedFrequencyPenalizer(CustomTestCase):
|
||||
|
||||
# BatchedPresencePenalizer
|
||||
class TestBatchedPresencePenalizer(CustomTestCase):
|
||||
|
||||
def _setup(self, presence_values):
|
||||
reqs = [_make_req(presence=p) for p in presence_values]
|
||||
batch = _make_batch(reqs)
|
||||
@@ -275,7 +272,6 @@ class TestBatchedPresencePenalizer(CustomTestCase):
|
||||
|
||||
# BatchedMinNewTokensPenalizer
|
||||
class TestBatchedMinNewTokensPenalizer(CustomTestCase):
|
||||
|
||||
def _setup(self, configs):
|
||||
"""configs: list of (min_tokens, stop_ids, eos_id)."""
|
||||
reqs = [_make_req(min_tokens=c[0], stop_ids=c[1], eos_id=c[2]) for c in configs]
|
||||
@@ -388,7 +384,6 @@ class TestBatchedMinNewTokensPenalizer(CustomTestCase):
|
||||
|
||||
# _BatchedPenalizer base class edge cases
|
||||
class TestBatchedPenalizerBase(CustomTestCase):
|
||||
|
||||
def test_filter_when_not_prepared_is_noop(self):
|
||||
"""Test that filter on an unprepared penalizer does not crash."""
|
||||
reqs = [_make_req()]
|
||||
@@ -452,7 +447,6 @@ class TestBatchedPenalizerBase(CustomTestCase):
|
||||
|
||||
# Orchestrator with multiple penalizer types
|
||||
class TestOrchestratorMultiplePenalizers(CustomTestCase):
|
||||
|
||||
def test_all_three_penalizers(self):
|
||||
"""Test orchestrator managing frequency, presence, and min_new_tokens together."""
|
||||
reqs = [_make_req(freq=1.0, presence=0.5, min_tokens=2, eos_id=2)]
|
||||
|
||||
@@ -50,7 +50,6 @@ def _serial_batched_fill(entries, vocab_mask):
|
||||
|
||||
|
||||
class TestMergeBiasTensor(CustomTestCase):
|
||||
|
||||
def test_both_none_returns_none(self):
|
||||
"""Test that merging two None tensors returns None."""
|
||||
result = merge_bias_tensor(None, None, 2, 3, DEVICE, 0.0)
|
||||
@@ -95,7 +94,6 @@ class TestMergeBiasTensor(CustomTestCase):
|
||||
|
||||
# SamplingBatchInfo.__len__
|
||||
class TestSamplingBatchInfoLen(CustomTestCase):
|
||||
|
||||
def test_len_matches_batch_size(self):
|
||||
"""Test that __len__ returns batch size (number of temperature rows)."""
|
||||
info = _make_info(batch_size=5)
|
||||
@@ -103,7 +101,6 @@ class TestSamplingBatchInfoLen(CustomTestCase):
|
||||
|
||||
|
||||
class TestMergeCustomLogitProcessor(CustomTestCase):
|
||||
|
||||
def test_both_none_returns_none(self):
|
||||
"""Test that merging two None processor dicts returns None."""
|
||||
result = SamplingBatchInfo.merge_custom_logit_processor(
|
||||
@@ -150,7 +147,6 @@ class TestMergeCustomLogitProcessor(CustomTestCase):
|
||||
|
||||
# apply_logits_bias
|
||||
class TestApplyLogitsBias(CustomTestCase):
|
||||
|
||||
def test_applies_additive_penalties(self):
|
||||
"""Test that pre-accumulated additive penalties are added to logits."""
|
||||
info = _make_info(batch_size=1)
|
||||
@@ -239,8 +235,8 @@ class TestApplyLogitsBias(CustomTestCase):
|
||||
|
||||
def make_info():
|
||||
grammar = MagicMock()
|
||||
grammar.apply_vocab_mask.side_effect = (
|
||||
lambda logits, vocab_mask: logits.add_(vocab_mask)
|
||||
grammar.apply_vocab_mask.side_effect = lambda logits, vocab_mask: (
|
||||
logits.add_(vocab_mask)
|
||||
)
|
||||
info = _make_info(batch_size=1)
|
||||
info.acc_additive_penalties = torch.linspace(
|
||||
@@ -272,7 +268,6 @@ class TestApplyLogitsBias(CustomTestCase):
|
||||
|
||||
# update_penalties
|
||||
class TestUpdatePenalties(CustomTestCase):
|
||||
|
||||
def test_required_creates_penalties_tensor(self):
|
||||
"""Test that update_penalties allocates a zero tensor and calls orchestrator methods."""
|
||||
orch = MagicMock(is_required=True)
|
||||
@@ -296,7 +291,6 @@ class TestUpdatePenalties(CustomTestCase):
|
||||
|
||||
# update_regex_vocab_mask
|
||||
class TestUpdateRegexVocabMask(CustomTestCase):
|
||||
|
||||
def test_no_grammars_clears_mask(self):
|
||||
"""Test that None grammars clears the grammar_mask."""
|
||||
info = _make_info(batch_size=1)
|
||||
@@ -379,7 +373,6 @@ class TestUpdateRegexVocabMask(CustomTestCase):
|
||||
|
||||
# filter_batch
|
||||
class TestFilterBatch(CustomTestCase):
|
||||
|
||||
def test_filter_keeps_correct_indices(self):
|
||||
"""Test that filter retains rows at indices 0 and 2, dropping index 1."""
|
||||
info = _make_info(batch_size=3)
|
||||
@@ -434,7 +427,6 @@ class TestFilterBatch(CustomTestCase):
|
||||
|
||||
# merge_batch
|
||||
class TestMergeBatch(CustomTestCase):
|
||||
|
||||
def test_merge_concatenates_tensors(self):
|
||||
"""Test that merge concatenates temperature tensors from both batches."""
|
||||
info1 = _make_info(batch_size=2)
|
||||
@@ -513,7 +505,6 @@ class TestMergeBatch(CustomTestCase):
|
||||
|
||||
# copy_for_forward
|
||||
class TestCopyForForward(CustomTestCase):
|
||||
|
||||
def test_returns_copy_without_orchestrator(self):
|
||||
"""Test that copy_for_forward returns a copy with orchestrator set to None."""
|
||||
orch = MagicMock(is_required=False)
|
||||
@@ -526,7 +517,6 @@ class TestCopyForForward(CustomTestCase):
|
||||
|
||||
# from_schedule_batch
|
||||
class TestFromScheduleBatch(CustomTestCase):
|
||||
|
||||
def setUp(self):
|
||||
super().setUp()
|
||||
# from_schedule_batch reads these two flags from the exec bag; give
|
||||
|
||||
@@ -15,7 +15,6 @@ register_cpu_ci(est_time=8, suite="base-a-test-cpu")
|
||||
|
||||
|
||||
class _FakeResponse:
|
||||
|
||||
def __init__(self) -> None:
|
||||
self.read_called = False
|
||||
|
||||
@@ -25,7 +24,6 @@ class _FakeResponse:
|
||||
|
||||
|
||||
class _FakePostCM:
|
||||
|
||||
def __init__(self, response: _FakeResponse) -> None:
|
||||
self._response = response
|
||||
|
||||
@@ -37,7 +35,6 @@ class _FakePostCM:
|
||||
|
||||
|
||||
class _FakeSession:
|
||||
|
||||
def __init__(self) -> None:
|
||||
self.closed = False
|
||||
self.calls: list[tuple[str, object]] = []
|
||||
@@ -52,7 +49,6 @@ class _FakeSession:
|
||||
|
||||
|
||||
class TestBackgroundHttpPosterLifecycle(CustomTestCase):
|
||||
|
||||
def test_init_starts_running_loop_on_daemon_thread(self):
|
||||
poster = BackgroundHttpPoster()
|
||||
self.addCleanup(poster.close)
|
||||
@@ -80,7 +76,6 @@ class TestBackgroundHttpPosterLifecycle(CustomTestCase):
|
||||
|
||||
|
||||
class TestBackgroundHttpPosterSubmitCoro(CustomTestCase):
|
||||
|
||||
def test_submit_coro_runs_on_background_loop_thread(self):
|
||||
poster = BackgroundHttpPoster()
|
||||
self.addCleanup(poster.close)
|
||||
@@ -134,7 +129,6 @@ class TestBackgroundHttpPosterSubmitCoro(CustomTestCase):
|
||||
|
||||
|
||||
class TestBackgroundHttpPosterEnsureSession(CustomTestCase):
|
||||
|
||||
def test_ensure_session_creates_reuses_then_recreates_when_closed(self):
|
||||
poster = BackgroundHttpPoster()
|
||||
self.addCleanup(poster.close)
|
||||
@@ -160,7 +154,6 @@ class TestBackgroundHttpPosterEnsureSession(CustomTestCase):
|
||||
|
||||
|
||||
class TestBackgroundHttpPosterPost(CustomTestCase):
|
||||
|
||||
def _run_on_loop(self, poster: BackgroundHttpPoster, coro) -> None:
|
||||
asyncio.run_coroutine_threadsafe(coro, poster._loop).result(timeout=5.0)
|
||||
|
||||
|
||||
@@ -28,7 +28,6 @@ _NO_REPLY = object()
|
||||
|
||||
|
||||
class _PairSocketHarness:
|
||||
|
||||
def __init__(self, *, reply: object = _NO_REPLY) -> None:
|
||||
self._ctx = zmq.Context()
|
||||
self.server_socket = self._ctx.socket(zmq.PAIR)
|
||||
@@ -62,7 +61,6 @@ class _PairSocketHarness:
|
||||
|
||||
|
||||
class _FakeProcess:
|
||||
|
||||
def __init__(self, *, alive: bool) -> None:
|
||||
self._alive = alive
|
||||
|
||||
@@ -79,7 +77,6 @@ def _make_server(socket: zmq.Socket, process: _FakeProcess) -> ScriptedHttpServe
|
||||
|
||||
|
||||
class TestExecuteScriptReplyMatching(CustomTestCase):
|
||||
|
||||
def test_returns_on_script_succeeded(self):
|
||||
with _PairSocketHarness(reply=ScriptSucceeded()) as pair:
|
||||
server = _make_server(pair.server_socket, _FakeProcess(alive=True))
|
||||
@@ -116,7 +113,6 @@ class TestExecuteScriptReplyMatching(CustomTestCase):
|
||||
|
||||
|
||||
class TestExecuteScriptNoReply(CustomTestCase):
|
||||
|
||||
def test_timeout_when_process_still_alive(self):
|
||||
with _PairSocketHarness() as pair:
|
||||
server = _make_server(pair.server_socket, _FakeProcess(alive=True))
|
||||
@@ -135,7 +131,6 @@ class TestExecuteScriptNoReply(CustomTestCase):
|
||||
|
||||
|
||||
class TestExecuteScriptDirtyGuard(CustomTestCase):
|
||||
|
||||
def test_refuses_to_run_when_already_dirty(self):
|
||||
with _PairSocketHarness() as pair:
|
||||
server = _make_server(pair.server_socket, _FakeProcess(alive=True))
|
||||
|
||||
@@ -26,7 +26,6 @@ def _raising_gen():
|
||||
|
||||
|
||||
class TestAdvanceGenerator(CustomTestCase):
|
||||
|
||||
def test_not_done_when_generator_yields(self):
|
||||
done, exc_tb = scheduler_hook._advance_generator(_yielding_gen())
|
||||
|
||||
|
||||
@@ -13,7 +13,6 @@ register_cpu_ci(est_time=2, suite="base-a-test-cpu")
|
||||
|
||||
|
||||
class TestResolveFn(CustomTestCase):
|
||||
|
||||
def test_resolves_top_level_function(self):
|
||||
self.assertIs(resolve_fn("json:dumps"), json.dumps)
|
||||
|
||||
@@ -46,7 +45,6 @@ class TestResolveFn(CustomTestCase):
|
||||
|
||||
|
||||
class TestEnsureScriptImportable(CustomTestCase):
|
||||
|
||||
_FAKE_ENTRY = "/tmp/__scripted_runtime_ut_fake_sys_path__"
|
||||
|
||||
def setUp(self):
|
||||
|
||||
@@ -18,18 +18,15 @@ import unittest
|
||||
|
||||
@dataclass
|
||||
class _ControlMsg:
|
||||
|
||||
tag: str = "flush"
|
||||
|
||||
|
||||
@dataclass
|
||||
class _StartReq:
|
||||
|
||||
rid: str
|
||||
|
||||
|
||||
class _FakeUnderlyingSocket:
|
||||
|
||||
def __init__(self) -> None:
|
||||
self._ready: deque = deque()
|
||||
self._scheduled: list[list] = []
|
||||
@@ -62,7 +59,6 @@ def _is_start_req(rid: str):
|
||||
|
||||
|
||||
class TestScriptedTokenizerRecvProxyRecv(CustomTestCase):
|
||||
|
||||
def test_recv_pyobj_drains_then_pops_fifo(self):
|
||||
underlying = _FakeUnderlyingSocket()
|
||||
proxy = ScriptedTokenizerRecvProxy(underlying=underlying)
|
||||
@@ -88,7 +84,6 @@ class TestScriptedTokenizerRecvProxyRecv(CustomTestCase):
|
||||
|
||||
|
||||
class TestScriptedTokenizerRecvProxyWaitUntilArrived(CustomTestCase):
|
||||
|
||||
def _proxy_with_stale_control(self):
|
||||
underlying = _FakeUnderlyingSocket()
|
||||
proxy = ScriptedTokenizerRecvProxy(underlying=underlying)
|
||||
|
||||
@@ -714,8 +714,7 @@ class TestResolutionDeclarations(CustomTestCase):
|
||||
current = getattr(server_args, name, None)
|
||||
if current != raw_input[name]:
|
||||
moved.append(
|
||||
f"{shape} -> {name}: raw={raw_input[name]!r} "
|
||||
f"field={current!r}"
|
||||
f"{shape} -> {name}: raw={raw_input[name]!r} field={current!r}"
|
||||
)
|
||||
self.assertEqual(
|
||||
moved,
|
||||
|
||||
@@ -61,7 +61,9 @@ def _holders(fn):
|
||||
else (
|
||||
annotation.id
|
||||
if isinstance(annotation, ast.Name)
|
||||
else annotation.attr if isinstance(annotation, ast.Attribute) else None
|
||||
else annotation.attr
|
||||
if isinstance(annotation, ast.Attribute)
|
||||
else None
|
||||
)
|
||||
)
|
||||
if text == "ServerArgs":
|
||||
|
||||
@@ -120,11 +120,14 @@ class TestDflashVerifyRunsMambaTrackHook(CustomTestCase):
|
||||
calls.append("init_new")
|
||||
return fake_forward_batch
|
||||
|
||||
with mock.patch(
|
||||
"sglang.srt.speculative.spec_utils.prepare_mamba_track_for_verify",
|
||||
side_effect=fake_hook,
|
||||
), mock.patch.object(
|
||||
dflash_info.ForwardBatch, "init_new", side_effect=fake_init_new
|
||||
with (
|
||||
mock.patch(
|
||||
"sglang.srt.speculative.spec_utils.prepare_mamba_track_for_verify",
|
||||
side_effect=fake_hook,
|
||||
),
|
||||
mock.patch.object(
|
||||
dflash_info.ForwardBatch, "init_new", side_effect=fake_init_new
|
||||
),
|
||||
):
|
||||
out, can_run_cuda_graph = self._spec_input().prepare_for_verify(
|
||||
batch, target_worker
|
||||
|
||||
@@ -231,12 +231,15 @@ class TestEagleWorkerV2BackendFallback(CustomTestCase):
|
||||
seq_lens=torch.ones((1,), dtype=torch.int32, device=DEVICE),
|
||||
)
|
||||
|
||||
with patch(
|
||||
"sglang.srt.speculative.eagle_worker_common.build_tree_kernel_efficient",
|
||||
return_value=tree_result,
|
||||
), patch(
|
||||
"sglang.srt.speculative.eagle_worker_v2.prepare_for_draft",
|
||||
return_value=(forward_batch, True),
|
||||
with (
|
||||
patch(
|
||||
"sglang.srt.speculative.eagle_worker_common.build_tree_kernel_efficient",
|
||||
return_value=tree_result,
|
||||
),
|
||||
patch(
|
||||
"sglang.srt.speculative.eagle_worker_v2.prepare_for_draft",
|
||||
return_value=(forward_batch, True),
|
||||
),
|
||||
):
|
||||
worker.draft(batch)
|
||||
|
||||
|
||||
@@ -194,12 +194,15 @@ class TestNgramMambaVerifyUpdate(CustomTestCase):
|
||||
dtype=torch.int32,
|
||||
)
|
||||
|
||||
with patch(
|
||||
"sglang.srt.speculative.spec_utils.mambaish_config",
|
||||
return_value={"some": "config"},
|
||||
), patch(
|
||||
"sglang.srt.speculative.spec_utils.mamba_track_grid",
|
||||
return_value=256,
|
||||
with (
|
||||
patch(
|
||||
"sglang.srt.speculative.spec_utils.mambaish_config",
|
||||
return_value={"some": "config"},
|
||||
),
|
||||
patch(
|
||||
"sglang.srt.speculative.spec_utils.mamba_track_grid",
|
||||
return_value=256,
|
||||
),
|
||||
):
|
||||
commit_mamba_states_after_verify(
|
||||
target_worker,
|
||||
|
||||
@@ -49,10 +49,11 @@ class TestGetLocalSliceBackendBranch(CustomTestCase):
|
||||
|
||||
def test_deepep_v2_reads_buffer_head(self):
|
||||
cap, buf = self._capturer()
|
||||
with mock.patch.object(
|
||||
re_mod, "is_dp_attention_enabled", return_value=True
|
||||
), mock.patch.object(
|
||||
re_mod, "get_moe_a2a_backend", return_value=MoeA2ABackend("deepep_v2")
|
||||
with (
|
||||
mock.patch.object(re_mod, "is_dp_attention_enabled", return_value=True),
|
||||
mock.patch.object(
|
||||
re_mod, "get_moe_a2a_backend", return_value=MoeA2ABackend("deepep_v2")
|
||||
),
|
||||
):
|
||||
out = self._slice(cap, n_local=5)
|
||||
self.assertTrue(torch.equal(out, buf[0:5, :, : self.K]))
|
||||
@@ -61,22 +62,23 @@ class TestGetLocalSliceBackendBranch(CustomTestCase):
|
||||
cap, _ = self._capturer()
|
||||
outs = []
|
||||
for backend in ("deepep", "deepep_v2"):
|
||||
with mock.patch.object(
|
||||
re_mod, "is_dp_attention_enabled", return_value=True
|
||||
), mock.patch.object(
|
||||
re_mod, "get_moe_a2a_backend", return_value=MoeA2ABackend(backend)
|
||||
with (
|
||||
mock.patch.object(re_mod, "is_dp_attention_enabled", return_value=True),
|
||||
mock.patch.object(
|
||||
re_mod, "get_moe_a2a_backend", return_value=MoeA2ABackend(backend)
|
||||
),
|
||||
):
|
||||
outs.append(self._slice(cap, n_local=7))
|
||||
self.assertTrue(torch.equal(outs[0], outs[1]))
|
||||
|
||||
def test_tp_moe_reads_global_offset(self):
|
||||
cap, buf = self._capturer()
|
||||
with mock.patch.object(
|
||||
re_mod, "is_dp_attention_enabled", return_value=True
|
||||
), mock.patch.object(
|
||||
re_mod, "get_moe_a2a_backend", return_value=MoeA2ABackend("none")
|
||||
), mock.patch.object(
|
||||
re_mod, "get_dp_local_slice_cpu", return_value=(6, 4)
|
||||
with (
|
||||
mock.patch.object(re_mod, "is_dp_attention_enabled", return_value=True),
|
||||
mock.patch.object(
|
||||
re_mod, "get_moe_a2a_backend", return_value=MoeA2ABackend("none")
|
||||
),
|
||||
mock.patch.object(re_mod, "get_dp_local_slice_cpu", return_value=(6, 4)),
|
||||
):
|
||||
out = self._slice(cap, n_local=999)
|
||||
self.assertTrue(torch.equal(out, buf[6:10, :, : self.K]))
|
||||
|
||||
@@ -104,9 +104,7 @@ def _byte(rank: int, chunk: int) -> int:
|
||||
def _assert_region(va: int, expected: int, peer: int, chunk: int) -> None:
|
||||
host = np.empty(16, dtype=np.uint8)
|
||||
check_drv(drv.cuMemcpyDtoH(host.ctypes.data, va, host.nbytes), "cuMemcpyDtoH")
|
||||
assert (
|
||||
host == expected
|
||||
).all(), (
|
||||
assert (host == expected).all(), (
|
||||
f"read {host.tolist()} from peer {peer} chunk {chunk}, expected all {expected}"
|
||||
)
|
||||
|
||||
|
||||
@@ -15,7 +15,6 @@ register_cpu_ci(est_time=10, suite="base-a-test-cpu")
|
||||
|
||||
|
||||
class TestDsaTilelangFp8Validation(CustomTestCase):
|
||||
|
||||
def test_cuda_fp8_tilelang_decode_rejected(self):
|
||||
with self.assertRaises(ValueError):
|
||||
_check_tilelang_dsa_fp8_kv("fp8_e4m3", "flashmla_kv", "tilelang", hip=False)
|
||||
|
||||
@@ -62,8 +62,9 @@ class TestEvalKitBackendDispatch(CustomTestCase):
|
||||
host.model = "m"
|
||||
for k, v in attrs.items():
|
||||
setattr(host, k, v)
|
||||
with patch.object(kit, "run_eval", side_effect=fake_run_eval), patch.object(
|
||||
kit.requests, "get", side_effect=_fake_get
|
||||
with (
|
||||
patch.object(kit, "run_eval", side_effect=fake_run_eval),
|
||||
patch.object(kit.requests, "get", side_effect=_fake_get),
|
||||
):
|
||||
host.test_gsm8k()
|
||||
return captured["args"]
|
||||
@@ -108,8 +109,9 @@ class TestEvalKitBackendDispatch(CustomTestCase):
|
||||
host.model = "deployment-model"
|
||||
host.mmmu_pro_score_threshold = 0.75
|
||||
host.mmmu_pro_load_preset_from_model_id = "moonshotai/Kimi-K3"
|
||||
with patch.object(kit, "run_eval", side_effect=fake_run_eval), patch.object(
|
||||
kit.requests, "get", side_effect=_fake_get
|
||||
with (
|
||||
patch.object(kit, "run_eval", side_effect=fake_run_eval),
|
||||
patch.object(kit.requests, "get", side_effect=_fake_get),
|
||||
):
|
||||
host.test_mmmu_pro()
|
||||
return captured["args"]
|
||||
|
||||
@@ -37,27 +37,32 @@ class TestForkTestWorker(CustomTestCase):
|
||||
os.fdopen(result_read_fd) as result_stream,
|
||||
):
|
||||
first = Path(tmpdir) / "first.py"
|
||||
first.write_text(textwrap.dedent("""
|
||||
first.write_text(
|
||||
textwrap.dedent("""
|
||||
import builtins
|
||||
import os
|
||||
|
||||
builtins._sglang_fork_worker_marker = 41
|
||||
os.environ["SGLANG_FORK_WORKER_TEST"] = "leaked"
|
||||
raise SystemExit(0)
|
||||
"""))
|
||||
""")
|
||||
)
|
||||
second = Path(tmpdir) / "second.py"
|
||||
second.write_text(textwrap.dedent("""
|
||||
second.write_text(
|
||||
textwrap.dedent("""
|
||||
import builtins
|
||||
import os
|
||||
|
||||
assert not hasattr(builtins, "_sglang_fork_worker_marker")
|
||||
assert "SGLANG_FORK_WORKER_TEST" not in os.environ
|
||||
raise SystemExit(3)
|
||||
"""))
|
||||
""")
|
||||
)
|
||||
helper = Path(tmpdir) / "sibling_helper.py"
|
||||
helper.write_text("VALUE = 42\n")
|
||||
sibling_import = Path(tmpdir) / "sibling_import.py"
|
||||
sibling_import.write_text(textwrap.dedent("""
|
||||
sibling_import.write_text(
|
||||
textwrap.dedent("""
|
||||
import os
|
||||
import sys
|
||||
|
||||
@@ -65,7 +70,8 @@ class TestForkTestWorker(CustomTestCase):
|
||||
|
||||
assert sys.path[0] == os.path.dirname(__file__)
|
||||
assert VALUE == 42
|
||||
"""))
|
||||
""")
|
||||
)
|
||||
|
||||
results = []
|
||||
for filename in (first, second, sibling_import):
|
||||
|
||||
@@ -375,8 +375,7 @@ class TestNoRenamedAccessorImports(CustomTestCase):
|
||||
base = imported.name.rsplit(".", 1)[-1]
|
||||
if base == "get_server_args":
|
||||
offenders.append(
|
||||
f"{rel}:{node.lineno}: {imported.name} as "
|
||||
f"{imported.asname}"
|
||||
f"{rel}:{node.lineno}: {imported.name} as {imported.asname}"
|
||||
)
|
||||
self.assertFalse(
|
||||
offenders,
|
||||
|
||||
@@ -2350,10 +2350,13 @@ class TestGoldenModelOverrides(_IsolatedPublish):
|
||||
args._model_config = SimpleNamespace(attention_arch=AttentionArch.MHA)
|
||||
return args
|
||||
|
||||
with override_platform(is_sm100=True), patch.object(
|
||||
qwen3_5_module,
|
||||
"get_default_attn_backend",
|
||||
lambda server_args, **_: server_args.default_backend_for_test,
|
||||
with (
|
||||
override_platform(is_sm100=True),
|
||||
patch.object(
|
||||
qwen3_5_module,
|
||||
"get_default_attn_backend",
|
||||
lambda server_args, **_: server_args.default_backend_for_test,
|
||||
),
|
||||
):
|
||||
# radix on + no extra buffer + no spec -> page_size=1 path
|
||||
self.assertEqual(
|
||||
|
||||
@@ -281,8 +281,11 @@ class TestRoleNamespaceEnforcement(CustomTestCase):
|
||||
|
||||
def test_enforce_blocks_reads_outside_the_declared_set(self):
|
||||
self._publish("test")
|
||||
with mock.patch.object(rc, "_ROLE_NS_MODE", "enforce"), mock.patch.dict(
|
||||
rc.ROLE_NAMESPACE_SETS, {"test": frozenset({"serving", "schedule"})}
|
||||
with (
|
||||
mock.patch.object(rc, "_ROLE_NS_MODE", "enforce"),
|
||||
mock.patch.dict(
|
||||
rc.ROLE_NAMESPACE_SETS, {"test": frozenset({"serving", "schedule"})}
|
||||
),
|
||||
):
|
||||
rc.get_serving()
|
||||
rc.get_schedule()
|
||||
@@ -316,8 +319,9 @@ class TestRoleNamespaceEnforcement(CustomTestCase):
|
||||
|
||||
def test_record_mode_collects_the_audit(self):
|
||||
self._publish("test")
|
||||
with mock.patch.object(rc, "_ROLE_NS_MODE", "record"), mock.patch.object(
|
||||
rc, "_RECORDED_NS_READS", set()
|
||||
with (
|
||||
mock.patch.object(rc, "_ROLE_NS_MODE", "record"),
|
||||
mock.patch.object(rc, "_RECORDED_NS_READS", set()),
|
||||
):
|
||||
rc.get_exec()
|
||||
rc.get_disagg()
|
||||
@@ -343,8 +347,9 @@ class TestRoleNamespaceEnforcement(CustomTestCase):
|
||||
def test_record_mode_registers_the_exit_summary_at_publish(self):
|
||||
# A role that reads no bags must still emit its audit line; the exit
|
||||
# hook therefore registers at publish, not at the first read.
|
||||
with mock.patch.object(rc, "_ROLE_NS_MODE", "record"), mock.patch.object(
|
||||
rc, "_RECORD_DUMP_REGISTERED", False
|
||||
with (
|
||||
mock.patch.object(rc, "_ROLE_NS_MODE", "record"),
|
||||
mock.patch.object(rc, "_RECORD_DUMP_REGISTERED", False),
|
||||
):
|
||||
self._publish("test")
|
||||
self.assertTrue(rc._RECORD_DUMP_REGISTERED)
|
||||
@@ -356,8 +361,9 @@ class TestRoleNamespaceEnforcement(CustomTestCase):
|
||||
import torch
|
||||
|
||||
self._publish("test")
|
||||
with mock.patch.object(rc, "_ROLE_NS_MODE", "record"), mock.patch.object(
|
||||
rc, "_RECORDED_NS_READS", set()
|
||||
with (
|
||||
mock.patch.object(rc, "_ROLE_NS_MODE", "record"),
|
||||
mock.patch.object(rc, "_RECORDED_NS_READS", set()),
|
||||
):
|
||||
|
||||
@torch.compile(fullgraph=True, backend="eager", dynamic=False)
|
||||
|
||||
@@ -17,7 +17,6 @@ register_cpu_ci(est_time=10, suite="base-a-test-cpu")
|
||||
|
||||
|
||||
class TestServerArgsAnnotatedCli(CustomTestCase):
|
||||
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
cls.parser = argparse.ArgumentParser()
|
||||
|
||||
@@ -206,17 +206,17 @@ def _expanded_override_keys(rel, tree, call, kw) -> set:
|
||||
return values
|
||||
|
||||
def dict_keys(node) -> set:
|
||||
assert isinstance(
|
||||
node, ast.Dict
|
||||
), f"non-literal dict in override expansion at {rel}:{call.lineno}"
|
||||
assert isinstance(node, ast.Dict), (
|
||||
f"non-literal dict in override expansion at {rel}:{call.lineno}"
|
||||
)
|
||||
keys = set()
|
||||
for key in node.keys:
|
||||
if isinstance(key, ast.Constant):
|
||||
keys.add(key.value)
|
||||
continue
|
||||
assert isinstance(
|
||||
key, ast.Name
|
||||
), f"non-literal dict key in override expansion at {rel}:{call.lineno}"
|
||||
assert isinstance(key, ast.Name), (
|
||||
f"non-literal dict key in override expansion at {rel}:{call.lineno}"
|
||||
)
|
||||
bound = loop_variable_values(key.id)
|
||||
assert bound, (
|
||||
f"dict key {key.id!r} at {rel}:{call.lineno} is not bound by a "
|
||||
@@ -239,9 +239,9 @@ def _expanded_override_keys(rel, tree, call, kw) -> set:
|
||||
f"unresolvable override expansion at {rel}:{call.lineno}"
|
||||
)
|
||||
return keys
|
||||
assert isinstance(
|
||||
kw.value, ast.Name
|
||||
), f"unresolvable override expansion at {rel}:{call.lineno}"
|
||||
assert isinstance(kw.value, ast.Name), (
|
||||
f"unresolvable override expansion at {rel}:{call.lineno}"
|
||||
)
|
||||
name = kw.value.id
|
||||
enclosing = None
|
||||
for fn in ast.walk(tree):
|
||||
@@ -253,9 +253,9 @@ def _expanded_override_keys(rel, tree, call, kw) -> set:
|
||||
):
|
||||
if enclosing is None or fn.lineno > enclosing.lineno:
|
||||
enclosing = fn
|
||||
assert (
|
||||
enclosing is not None
|
||||
), f"override expansion outside any function at {rel}:{call.lineno}"
|
||||
assert enclosing is not None, (
|
||||
f"override expansion outside any function at {rel}:{call.lineno}"
|
||||
)
|
||||
keys = set()
|
||||
found = False
|
||||
for node in ast.walk(enclosing):
|
||||
|
||||
@@ -201,7 +201,6 @@ class TestProfileMerger(CustomTestCase):
|
||||
|
||||
|
||||
class TestProfileMergerIntegration(CustomTestCase):
|
||||
|
||||
def test_data_structures_merge_profiles(self):
|
||||
# Test ProfileReq
|
||||
req = ProfileReq()
|
||||
|
||||
@@ -59,9 +59,9 @@ def _assert_entries_close(
|
||||
"""Compare two streams of (name, should_compare, ComparableWeight)."""
|
||||
actual_list: List[CheckEntry] = list(actual)
|
||||
expected_list: List[CheckEntry] = list(expected)
|
||||
assert len(actual_list) == len(
|
||||
expected_list
|
||||
), f"length mismatch: actual={len(actual_list)} expected={len(expected_list)}"
|
||||
assert len(actual_list) == len(expected_list), (
|
||||
f"length mismatch: actual={len(actual_list)} expected={len(expected_list)}"
|
||||
)
|
||||
for i, ((a_name, a_flag, a_ref), (e_name, e_flag, e_ref)) in enumerate(
|
||||
zip(actual_list, expected_list)
|
||||
):
|
||||
@@ -150,7 +150,6 @@ class _FakeModelRunner:
|
||||
|
||||
|
||||
class TestRandomLike(CustomTestCase):
|
||||
|
||||
def test_floating_point_preserves_dtype_shape_device(self):
|
||||
for dtype in (torch.float32, torch.float16, torch.bfloat16):
|
||||
t = torch.zeros(8, 4, dtype=dtype)
|
||||
@@ -209,7 +208,6 @@ class TestRandomLike(CustomTestCase):
|
||||
|
||||
|
||||
class TestPostprocessTensors(CustomTestCase):
|
||||
|
||||
# --- non-quant / non-skip ---
|
||||
|
||||
def test_no_quant_yields_raw_with_should_compare_true(self):
|
||||
@@ -312,7 +310,6 @@ class TestPostprocessTensors(CustomTestCase):
|
||||
|
||||
|
||||
class TestCheckTensors(CustomTestCase):
|
||||
|
||||
def test_passes_when_all_equal(self):
|
||||
t = torch.ones(2, 2)
|
||||
expect = [
|
||||
@@ -388,7 +385,6 @@ def _quantize_block_fp8(weight: torch.Tensor, scale_margin: float):
|
||||
|
||||
|
||||
class TestCheckTensorsAllowQuantError(CustomTestCase):
|
||||
|
||||
def setUp(self):
|
||||
torch.manual_seed(0)
|
||||
weight = torch.randn(256, 256, device="cuda") * 0.02
|
||||
@@ -442,7 +438,6 @@ class TestCheckTensorsAllowQuantError(CustomTestCase):
|
||||
|
||||
|
||||
class TestBuildQuantizedSet(CustomTestCase):
|
||||
|
||||
def test_fp8_block_module_pairs_weight_and_scale(self):
|
||||
from sglang.srt.layers.quantization.fp8 import Fp8LinearMethod
|
||||
|
||||
@@ -493,7 +488,6 @@ class _WeightCheckerTestBase(CustomTestCase):
|
||||
|
||||
|
||||
class TestSnapshot(_WeightCheckerTestBase):
|
||||
|
||||
def test_captures_params_and_buffers(self):
|
||||
self.checker._snapshot()
|
||||
keys = set(self.checker._snapshot_tensors.keys())
|
||||
@@ -519,7 +513,6 @@ class TestSnapshot(_WeightCheckerTestBase):
|
||||
|
||||
|
||||
class TestResetTensors(_WeightCheckerTestBase):
|
||||
|
||||
def test_changes_normal_params_in_place(self):
|
||||
before_w = self.model.w.clone()
|
||||
before_w_ptr = self.model.w.data_ptr()
|
||||
@@ -547,7 +540,6 @@ class TestResetTensors(_WeightCheckerTestBase):
|
||||
|
||||
|
||||
class TestCompare(_WeightCheckerTestBase):
|
||||
|
||||
def test_without_snapshot_raises(self):
|
||||
with self.assertRaises(AssertionError):
|
||||
self.checker._compare()
|
||||
@@ -586,7 +578,6 @@ class TestCompare(_WeightCheckerTestBase):
|
||||
|
||||
|
||||
class TestHandle(_WeightCheckerTestBase):
|
||||
|
||||
def test_routes_to_actions(self):
|
||||
with (
|
||||
patch.object(self.checker, "_snapshot") as m_snap,
|
||||
@@ -628,7 +619,6 @@ class TestHandle(_WeightCheckerTestBase):
|
||||
|
||||
|
||||
class TestIsNonPersistentBufferName(CustomTestCase):
|
||||
|
||||
def test_matches_cos_sin_cache_substring(self):
|
||||
self.assertTrue(
|
||||
_is_non_persistent_buffer_name("model.rotary_emb.cos_sin_cache")
|
||||
@@ -651,7 +641,6 @@ class TestIsNonPersistentBufferName(CustomTestCase):
|
||||
|
||||
|
||||
class TestHashTensor(CustomTestCase):
|
||||
|
||||
def test_stable_for_same_input(self):
|
||||
t = torch.arange(64, dtype=torch.float32).cuda()
|
||||
self.assertEqual(_hash_tensor(t), _hash_tensor(t.clone()))
|
||||
@@ -680,7 +669,6 @@ class TestHashTensor(CustomTestCase):
|
||||
|
||||
|
||||
class _ChecksumTestBase(CustomTestCase):
|
||||
|
||||
def setUp(self):
|
||||
torch.manual_seed(0)
|
||||
self.model = _TinyModel().cuda()
|
||||
@@ -699,7 +687,6 @@ class _ChecksumTestBase(CustomTestCase):
|
||||
|
||||
|
||||
class TestComputeChecksum(_ChecksumTestBase):
|
||||
|
||||
def test_returns_dict_with_expected_top_level_keys(self):
|
||||
out = self.checker._compute_checksum()
|
||||
self.assertEqual(
|
||||
|
||||
@@ -62,7 +62,6 @@ def _build_fp8_quant_pair(device: str = "cuda"):
|
||||
|
||||
|
||||
class TestQuantUlp(CustomTestCase):
|
||||
|
||||
def test_matches_bruteforce_spacing_for_fp8(self):
|
||||
for dtype in (torch.float8_e4m3fn, torch.float8_e5m2):
|
||||
all_bits = torch.arange(256, dtype=torch.uint8).view(dtype)
|
||||
@@ -183,7 +182,6 @@ class TestCompareQuantPair(CustomTestCase):
|
||||
|
||||
|
||||
class TestSelectComparableWeight(CustomTestCase):
|
||||
|
||||
def test_returns_none_when_not_a_quant_method(self):
|
||||
self.assertIsNone(select_comparable_weight(None))
|
||||
|
||||
|
||||
Reference in New Issue
Block a user