[Config] Round 5.1: the published-side readers ask the bags, and a platform fact gets one address (#37086)

This commit is contained in:
Cheng Wan
2026-08-30 02:18:33 -07:00
committed by GitHub
parent a6e4021368
commit 7e751153eb
148 changed files with 1585 additions and 1114 deletions
@@ -25,6 +25,7 @@ from sglang.srt.model_executor.forward_batch_info import (
from sglang.srt.model_executor.runner.prefill_cuda_graph_runner import (
PrefillCudaGraphRunner,
)
from sglang.srt.runtime_context import override_platform
from sglang.srt.server_args import ServerArgs
from sglang.test.ci.ci_register import register_cpu_ci
from sglang.test.test_utils import CustomTestCase
@@ -84,16 +85,12 @@ class TestMultimodalPiecewiseCudaGraph(CustomTestCase):
)
args._cuda_graph_config_locked = set()
with (
patch(
"sglang.srt.arg_groups.cuda_graph_hook"
".disable_tc_piecewise_cudagraph_if_incompatible"
) as disable_if_incompatible,
patch(
"sglang.srt.arg_groups.overrides.attention_backends_of",
return_value=("fa3", "fa3"),
),
):
args.attention_backend = "fa3"
with patch(
"sglang.srt.arg_groups.cuda_graph_hook"
".disable_tc_piecewise_cudagraph_if_incompatible"
) as disable_if_incompatible:
apply_cuda_graph_compatibility(args)
self.assertEqual(
@@ -118,11 +115,9 @@ class TestMultimodalPiecewiseCudaGraph(CustomTestCase):
)
args._cuda_graph_config_locked = set()
with patch(
"sglang.srt.arg_groups.overrides.attention_backends_of",
return_value=("trtllm_mla", "trtllm_mla"),
):
apply_cuda_graph_compatibility(args)
args.attention_backend = "trtllm_mla"
apply_cuda_graph_compatibility(args)
self.assertEqual(
resolution_result(args, "cuda_graph_config").prefill.backend,
@@ -136,11 +131,9 @@ class TestMultimodalPiecewiseCudaGraph(CustomTestCase):
)
args._cuda_graph_config_locked = {(Phase.PREFILL, "backend")}
with patch(
"sglang.srt.arg_groups.overrides.attention_backends_of",
return_value=("trtllm_mla", "trtllm_mla"),
):
apply_cuda_graph_compatibility(args)
args.attention_backend = "trtllm_mla"
apply_cuda_graph_compatibility(args)
self.assertEqual(
resolution_result(args, "cuda_graph_config").prefill.backend,
@@ -183,7 +176,7 @@ class TestMultimodalPiecewiseCudaGraph(CustomTestCase):
args.disable_radix_cache = False
args.chunked_prefill_size = 2048
with (patch("sglang.srt.arg_groups.model_hook.is_cuda", return_value=True),):
with (override_platform(is_cuda=True),):
handle_model_capability_adjustments(args)
self.assertTrue(resolution_result(args, "disable_radix_cache"))
@@ -20,6 +20,7 @@ import unittest
from concurrent.futures import Future
from unittest.mock import MagicMock, patch
from sglang.srt.arg_groups.overrides import resolution_result
from sglang.srt.constrained.base_grammar_backend import (
GRAMMAR_BACKEND_REGISTRY,
BaseGrammarBackend,
@@ -355,7 +356,10 @@ class TestCreateGrammarBackend(unittest.TestCase):
self.assertEqual(
get_context().resolved_server_args_dict()["grammar_backend"], "none"
)
self.assertEqual(server_args.grammar_backend, "xgrammar")
# The record is not written any more: what the caller asked for is a
# declaration on it, and the runtime fallback to "none" lives in the bag
# (asserted above). The two are meant to differ here.
self.assertEqual(resolution_result(server_args, "grammar_backend"), "xgrammar")
@patch("sglang.srt.constrained.llguidance_backend.GuidanceBackend")
def test_llguidance_backend(self, mock_guidance_cls):
@@ -8,6 +8,7 @@ from sglang.srt.layers.attention.linear.kda_backend import KDAKernelDispatcher
from sglang.srt.layers.attention.linear.kernels.kda_helion import HelionKDAKernel
from sglang.srt.layers.attention.linear.kernels.kda_triton import TritonKDAKernel
from sglang.srt.layers.attention.linear.utils import LinearAttnKernelBackend
from sglang.srt.runtime_context import override_platform
from sglang.srt.server_args import ServerArgs
from sglang.test.ci.ci_register import register_cpu_ci
@@ -160,11 +161,8 @@ class TestHelionKDADispatcher(unittest.TestCase):
def test_replayssm_accepts_helion_and_rejects_other_backends(self):
with (
patch(
"sglang.srt.arg_groups.attention_hook.is_sm100_supported",
return_value=False,
),
patch("sglang.srt.arg_groups.attention_hook.is_cuda", return_value=False),
override_platform(is_sm100=False),
override_platform(is_cuda=False),
):
helion_args = ServerArgs(
model_path="dummy",
@@ -188,11 +186,8 @@ class TestHelionKDADispatcher(unittest.TestCase):
mamba_ssm_dtype="bfloat16",
)
with (
patch(
"sglang.srt.arg_groups.attention_hook.is_sm100_supported",
return_value=True,
),
patch("sglang.srt.arg_groups.attention_hook.is_cuda", return_value=False),
override_platform(is_sm100=True),
override_platform(is_cuda=False),
):
handle_linear_attn_backend(args)
@@ -12,6 +12,7 @@ from sglang.srt.layers.quantization.compressed_tensors.schemes import (
CompressedTensorsWNA16MoE,
CompressedTensorsWNA16TritonMoE,
)
from sglang.srt.runtime_context import override_platform
from sglang.test.ci.ci_register import register_cpu_ci
from sglang.test.test_utils import CustomTestCase
@@ -90,9 +91,7 @@ class TestWNA16MoENoLinearGroup(CustomTestCase):
"get_moe_runner_backend",
return_value=MoeRunnerBackend.AUTO,
),
mock.patch.object(
compressed_tensors, "is_sm100_supported", return_value=True
),
override_platform(is_sm100=True),
):
scheme = quant_config.get_moe_scheme(
torch.nn.Module(), layer_name=EXPERTS_LAYER
@@ -120,9 +119,7 @@ class TestWNA16MoENoLinearGroup(CustomTestCase):
"get_moe_runner_backend",
return_value=MoeRunnerBackend.AUTO,
),
mock.patch.object(
compressed_tensors, "is_sm100_supported", return_value=True
),
override_platform(is_sm100=True),
):
scheme = quant_config.get_moe_scheme(
torch.nn.Module(), layer_name=EXPERTS_LAYER
@@ -157,9 +154,7 @@ class TestWNA16MoENoLinearGroup(CustomTestCase):
"get_moe_runner_backend",
return_value=MoeRunnerBackend.MARLIN,
),
mock.patch.object(
compressed_tensors, "is_sm100_supported", return_value=True
),
override_platform(is_sm100=True),
):
scheme = quant_config.get_moe_scheme(
torch.nn.Module(), layer_name=EXPERTS_LAYER
@@ -178,9 +173,7 @@ class TestWNA16MoENoLinearGroup(CustomTestCase):
"get_moe_runner_backend",
return_value=MoeRunnerBackend.AUTO,
),
mock.patch.object(
compressed_tensors, "is_sm100_supported", return_value=True
),
override_platform(is_sm100=True),
):
scheme = quant_config.get_moe_scheme(
torch.nn.Module(), layer_name=EXPERTS_LAYER
@@ -11,6 +11,7 @@ from types import SimpleNamespace
import pytest
import torch
from sglang.srt.runtime_context import override_platform
from sglang.test.ci.ci_register import register_cuda_ci
register_cuda_ci(est_time=120, stage="base-b", runner_config="1-gpu-small")
@@ -81,10 +82,12 @@ def test_cutlass_adapter_import_does_not_require_flashinfer(monkeypatch):
sys.modules[module_name] = cached_module
def test_dsv4_sm120_load_contract(monkeypatch):
def test_dsv4_sm120_load_contract(monkeypatch, request):
import sglang.srt.layers.quantization.mxfp4_flashinfer_cutlass_moe as adapter_module
monkeypatch.setattr(adapter_module, "is_sm120_supported", lambda: True)
platform = override_platform(is_sm120=True)
platform.install()
request.addfinalizer(platform.restore)
captured = {}
@@ -5,7 +5,7 @@ from unittest.mock import MagicMock, patch
import torch
from sglang.srt.layers import flashinfer_comm_fusion as fusion
from sglang.srt.runtime_context import get_parallel
from sglang.srt.runtime_context import get_parallel, override_platform
from sglang.test.ci.ci_register import register_cuda_ci
from sglang.test.test_utils import CustomTestCase
@@ -101,7 +101,7 @@ class TestFlashInferCommFusion(CustomTestCase):
multi_node = ("auto", True)
# Blackwell: mnnvl on both single-node and multi-node.
with patch.object(fusion, "is_sm100_supported", return_value=True):
with override_platform(is_sm100=True):
self.assertEqual(
fusion._resolve_backend(*single_node),
"mnnvl",
@@ -110,8 +110,8 @@ class TestFlashInferCommFusion(CustomTestCase):
# SM90: auto uses trtllm on single-node, multi-node is unsupported.
with (
patch.object(fusion, "is_sm100_supported", return_value=False),
patch.object(fusion, "is_sm90_supported", return_value=True),
override_platform(is_sm100=False),
override_platform(is_sm90=True),
):
self.assertEqual(
fusion._resolve_backend(*single_node),
@@ -125,8 +125,8 @@ class TestFlashInferCommFusion(CustomTestCase):
for arch in ("pre_sm90", "post_sm10x"):
with (
self.subTest(arch=arch),
patch.object(fusion, "is_sm100_supported", return_value=False),
patch.object(fusion, "is_sm90_supported", return_value=False),
override_platform(is_sm100=False),
override_platform(is_sm90=False),
):
with self.assertRaises(ValueError):
fusion._resolve_backend(*single_node)
@@ -140,8 +140,8 @@ class TestFlashInferCommFusion(CustomTestCase):
multi_node_trtllm = ("trtllm", True)
with (
patch.object(fusion, "is_sm100_supported", return_value=False),
patch.object(fusion, "is_sm90_supported", return_value=True),
override_platform(is_sm100=False),
override_platform(is_sm90=True),
):
self.assertEqual(
fusion._resolve_backend(*single_node_mnnvl),
@@ -156,7 +156,7 @@ class TestFlashInferCommFusion(CustomTestCase):
with self.assertRaises(ValueError):
fusion._resolve_backend(*multi_node_trtllm)
with patch.object(fusion, "is_sm100_supported", return_value=True):
with override_platform(is_sm100=True):
self.assertEqual(
fusion._resolve_backend(*multi_node_mnnvl),
"mnnvl",
@@ -167,8 +167,8 @@ class TestFlashInferCommFusion(CustomTestCase):
for arch in ("pre_sm90", "post_sm10x"):
with (
self.subTest(arch=arch),
patch.object(fusion, "is_sm100_supported", return_value=False),
patch.object(fusion, "is_sm90_supported", return_value=False),
override_platform(is_sm100=False),
override_platform(is_sm90=False),
):
for args in (
single_node_mnnvl,
@@ -6,6 +6,7 @@ from unittest.mock import MagicMock, Mock, patch
import torch
from sglang.srt.runtime_context import override_platform
from sglang.test.ci.ci_register import register_cpu_ci
from sglang.test.test_utils import CustomTestCase
@@ -96,7 +97,7 @@ def _construct_sparse_backend(
with (
get_schedule().override(chunked_prefill_size=chunked_prefill_size),
patch.object(backend_module, "MiniCPMHybridConfig", SimpleNamespace),
patch.object(backend_module, "is_blackwell_supported", return_value=blackwell),
override_platform(is_blackwell=blackwell),
patch.object(
backend_module,
"FlashAttentionBackend",
@@ -292,7 +293,7 @@ class TestMiniCPMSparseMetadata(CustomTestCase):
{"sglang.srt.layers.attention.minicpm.fuse_kernel": fake_fuse_kernel},
),
patch.object(backend_module, "MiniCPMHybridConfig", SimpleNamespace),
patch.object(backend_module, "is_blackwell_supported", return_value=True),
override_platform(is_blackwell=True),
patch.object(
backend_module,
"FlashAttentionBackend",
@@ -318,7 +319,7 @@ class TestMiniCPMSparseMetadata(CustomTestCase):
model_config.num_attention_heads = 8
with (
patch.object(backend_module, "MiniCPMHybridConfig", SimpleNamespace),
patch.object(backend_module, "is_blackwell_supported", return_value=True),
override_platform(is_blackwell=True),
patch.object(
backend_module,
"FlashAttentionBackend",
@@ -337,7 +338,7 @@ class TestMiniCPMSparseMetadata(CustomTestCase):
model_runner.server_args.attention_backend = "minicpm_flashattn"
with (
patch.object(backend_module, "MiniCPMHybridConfig", SimpleNamespace),
patch.object(backend_module, "is_blackwell_supported", return_value=False),
override_platform(is_blackwell=False),
patch.object(
backend_module,
"FlashAttentionBackend",
@@ -397,7 +398,7 @@ class TestMiniCPMSparseMetadata(CustomTestCase):
with (
backend_module.envs.SGLANG_MINICPM_DENSE_AS_SPARSE.override(True),
patch.object(backend_module, "MiniCPMHybridConfig", SimpleNamespace),
patch.object(backend_module, "is_blackwell_supported", return_value=False),
override_platform(is_blackwell=False),
patch.object(
backend_module,
"FlashAttentionBackend",
@@ -403,8 +403,12 @@ class TestStreamOrderedMmFeaturePool(CustomTestCase):
def test_consumer_slot_uses_global_tp_rank(self):
from sglang.srt.multimodal.transport.memory_pool import resolve_consumer_rank
parallel = SimpleNamespace(tp_rank=6, attn_tp_rank=2)
with patch("sglang.srt.runtime_context.get_parallel", return_value=parallel):
# State the topology on the context, not by stubbing the accessor:
# `memory_pool` imports `get_parallel` at module scope, so a patch on
# the defining module never reaches the copy doing the reading.
from sglang.srt.runtime_context import get_parallel
with get_parallel().override(tp_rank=6, attn_tp_rank=2):
self.assertEqual(resolve_consumer_rank(8), 6)
def test_complete_group_acknowledges_each_consumer_slot(self):
@@ -13,6 +13,7 @@ from sglang.srt.mem_cache.base_prefix_cache import (
DecLockRefResult,
IncLockRefResult,
)
from sglang.srt.runtime_context import get_context
from sglang.srt.server_args import ServerArgs, set_global_server_args_for_scheduler
from sglang.srt.utils.common import Range
from sglang.test.ci.ci_register import register_cpu_ci
@@ -79,14 +80,14 @@ class TestPrefillAdder(CustomTestCase):
batch.filter_batch.return_value = None
return batch
def create_server_args(
self, *, schedule_low_priority_values_first: bool
) -> MagicMock:
server_args = MagicMock()
server_args.schedule_low_priority_values_first = (
schedule_low_priority_values_first
def scheduling_order(self, *, schedule_low_priority_values_first: bool):
"""State the policy on the context, which is where the scheduler reads
it: `preempt_to_schedule` takes no record to state it on."""
override = get_context().override_server_args(
schedule_low_priority_values_first=schedule_low_priority_values_first
)
return server_args
override.install()
self.addCleanup(override.restore)
def create_mock_req(self, rid, priority, max_new_tokens, output_len=0, wait_time=0):
req = MagicMock(spec=Req)
@@ -129,9 +130,7 @@ class TestPrefillAdder(CustomTestCase):
self.create_mock_req(rid, priority, max_new_tokens)
for rid, priority, max_new_tokens in params
]
mock_server_args = self.create_server_args(
schedule_low_priority_values_first=False
)
self.scheduling_order(schedule_low_priority_values_first=False)
running_batch = self.create_running_batch(running_reqs)
adder = self.create_adder(running_batch)
@@ -144,7 +143,7 @@ class TestPrefillAdder(CustomTestCase):
new_req = self.create_mock_req("new1", priority=1, max_new_tokens=49)
success = adder.preempt_to_schedule(new_req, mock_server_args)
success = adder.preempt_to_schedule(new_req)
self.assertTrue(success)
self.assertIn(running_reqs[0], adder.preempt_list)
@@ -161,9 +160,7 @@ class TestPrefillAdder(CustomTestCase):
self.create_mock_req(rid, priority, max_new_tokens)
for rid, priority, max_new_tokens in params
]
mock_server_args = self.create_server_args(
schedule_low_priority_values_first=True
)
self.scheduling_order(schedule_low_priority_values_first=True)
running_batch = self.create_running_batch(running_reqs)
adder = self.create_adder(running_batch)
@@ -176,7 +173,7 @@ class TestPrefillAdder(CustomTestCase):
new_req = self.create_mock_req("new1", priority=1, max_new_tokens=49)
success = adder.preempt_to_schedule(new_req, mock_server_args)
success = adder.preempt_to_schedule(new_req)
self.assertTrue(success)
self.assertIn(running_reqs[2], adder.preempt_list)
@@ -193,9 +190,7 @@ class TestPrefillAdder(CustomTestCase):
self.create_mock_req(rid, priority, max_new_tokens)
for rid, priority, max_new_tokens in params
]
mock_server_args = self.create_server_args(
schedule_low_priority_values_first=True
)
self.scheduling_order(schedule_low_priority_values_first=True)
running_batch = self.create_running_batch(running_reqs)
adder = self.create_adder(running_batch)
@@ -211,7 +206,7 @@ class TestPrefillAdder(CustomTestCase):
)
success_by_priority_check = adder.preempt_to_schedule(
new_req_fail_by_priority_check, mock_server_args
new_req_fail_by_priority_check
)
self.assertFalse(success_by_priority_check)
@@ -219,7 +214,7 @@ class TestPrefillAdder(CustomTestCase):
"new2", priority=1, max_new_tokens=110
)
success_by_capacity_check = adder.preempt_to_schedule(
new_req_fail_by_priority_check, mock_server_args
new_req_fail_by_priority_check
)
self.assertFalse(success_by_capacity_check)
@@ -233,9 +228,7 @@ class TestPrefillAdder(CustomTestCase):
self.create_mock_req(rid, priority, max_new_tokens)
for rid, priority, max_new_tokens in params
]
mock_server_args = self.create_server_args(
schedule_low_priority_values_first=False
)
self.scheduling_order(schedule_low_priority_values_first=False)
running_batch = self.create_running_batch(running_reqs)
adder = self.create_adder(running_batch)
@@ -251,7 +244,7 @@ class TestPrefillAdder(CustomTestCase):
)
success_by_priority_check = adder.preempt_to_schedule(
new_req_fail_by_priority_check, mock_server_args
new_req_fail_by_priority_check
)
self.assertFalse(success_by_priority_check)
@@ -259,7 +252,7 @@ class TestPrefillAdder(CustomTestCase):
"new2", priority=-1, max_new_tokens=110
)
success_by_capacity_check = adder.preempt_to_schedule(
new_req_fail_by_priority_check, mock_server_args
new_req_fail_by_priority_check
)
self.assertFalse(success_by_capacity_check)
@@ -273,9 +266,7 @@ class TestPrefillAdder(CustomTestCase):
self.create_mock_req(rid, priority, max_new_tokens)
for rid, priority, max_new_tokens in params
]
mock_server_args = self.create_server_args(
schedule_low_priority_values_first=False
)
self.scheduling_order(schedule_low_priority_values_first=False)
running_batch = self.create_running_batch(running_reqs)
adder = self.create_adder(running_batch)
@@ -288,7 +279,7 @@ class TestPrefillAdder(CustomTestCase):
first_req = self.create_mock_req(
"new_req_prio_1", priority=1, max_new_tokens=49
)
first_success = adder.preempt_to_schedule(first_req, mock_server_args)
first_success = adder.preempt_to_schedule(first_req)
self.assertTrue(first_success)
self.assertIn(running_reqs[0], adder.preempt_list)
self.assertEqual(adder.rem_total_token_offset, 175)
@@ -299,7 +290,7 @@ class TestPrefillAdder(CustomTestCase):
second_req = self.create_mock_req(
"second_new_req_prio_1", priority=1, max_new_tokens=76
)
second_success = adder.preempt_to_schedule(second_req, mock_server_args)
second_success = adder.preempt_to_schedule(second_req)
self.assertFalse(second_success)
self.assertEqual(adder.rem_total_token_offset, 175)
@@ -318,9 +309,7 @@ class TestPrefillAdder(CustomTestCase):
self.create_mock_req(rid, priority, max_new_tokens)
for rid, priority, max_new_tokens in params
]
mock_server_args = self.create_server_args(
schedule_low_priority_values_first=True
)
self.scheduling_order(schedule_low_priority_values_first=True)
running_batch = self.create_running_batch(running_reqs)
adder = self.create_adder(running_batch)
@@ -333,7 +322,7 @@ class TestPrefillAdder(CustomTestCase):
new_req = self.create_mock_req("new1", priority=1, max_new_tokens=75)
success = adder.preempt_to_schedule(new_req, mock_server_args)
success = adder.preempt_to_schedule(new_req)
self.assertTrue(success)
self.assertIn(running_reqs[2], adder.preempt_list)
self.assertEqual(
@@ -353,9 +342,7 @@ class TestPrefillAdder(CustomTestCase):
self.create_mock_req(rid, priority, max_new_tokens)
for rid, priority, max_new_tokens in params
]
mock_server_args = self.create_server_args(
schedule_low_priority_values_first=True
)
self.scheduling_order(schedule_low_priority_values_first=True)
running_batch = self.create_running_batch(running_reqs)
adder = self.create_adder(running_batch)
@@ -368,7 +355,7 @@ class TestPrefillAdder(CustomTestCase):
new_req = self.create_mock_req("new1", priority=1, max_new_tokens=200)
success = adder.preempt_to_schedule(new_req, mock_server_args)
success = adder.preempt_to_schedule(new_req)
self.assertTrue(success)
self.assertIn(running_reqs[2], adder.preempt_list)
self.assertIn(running_reqs[3], adder.preempt_list)
@@ -9,6 +9,7 @@ and the published instance stays as the launcher left it.
import unittest
from types import SimpleNamespace
from sglang.srt.arg_groups.overrides import resolution_result
from sglang.srt.managers.io_struct import (
AttachHiCacheStorageReqInput,
DetachHiCacheStorageReqInput,
@@ -69,7 +70,12 @@ class TestSchedulerHiCacheAttach(CustomTestCase):
self.assertIsNone(
get_context().resolved_server_args_dict()["hicache_storage_backend"]
)
self.assertEqual(self.server_args.hicache_storage_backend, "file")
# The record is not written any more: the attach is a declaration on
# it and the detach is a bag override (asserted above), so the two are
# meant to differ here.
self.assertEqual(
resolution_result(self.server_args, "hicache_storage_backend"), "file"
)
if __name__ == "__main__":
@@ -3,6 +3,7 @@ from unittest import mock
from sglang.srt.environ import envs
from sglang.srt.models.deepseek_common.amd import deepseek_v4_fused_mhc
from sglang.srt.runtime_context import override_platform
from sglang.test.ci.ci_register import register_cpu_ci
register_cpu_ci(est_time=4, suite="base-a-test-cpu")
@@ -19,8 +20,8 @@ class TestAmdFusedMhcCrossLayerGating(unittest.TestCase):
):
self.assertTrue(deepseek_v4_fused_mhc.is_cross_layer_mhc_fusion_enabled())
@mock.patch.object(deepseek_v4_fused_mhc, "is_sm120_supported", return_value=True)
def test_sm120_enables_fusion_with_tilelang_pre_disabled(self, _mock_sm120):
@override_platform(is_sm120=True)
def test_sm120_enables_fusion_with_tilelang_pre_disabled(self):
# Regression (PR review): consolidating _is_fused_mhc_post_pre_enabled into
# this module must preserve the SM120 special case. SM120 disables the
# standalone TileLang pre path, but mhc_fused_post_pre dispatches
@@ -35,8 +36,8 @@ class TestAmdFusedMhcCrossLayerGating(unittest.TestCase):
):
self.assertTrue(deepseek_v4_fused_mhc._is_fused_mhc_post_pre_enabled())
@mock.patch.object(deepseek_v4_fused_mhc, "is_sm120_supported", return_value=False)
def test_no_sm120_still_requires_tilelang_pre(self, _mock_sm120):
@override_platform(is_sm120=False)
def test_no_sm120_still_requires_tilelang_pre(self):
# Negative branch: the (pre OR sm120) clause must not degrade to
# always-true. With SM120 unsupported and the pre flag off, fuse+post
# alone must not enable the standalone TileLang fused path.
@@ -68,11 +69,7 @@ class TestAmdFusedMhcCrossLayerGating(unittest.TestCase):
envs.SGLANG_OPT_FUSE_MHC_POST_PRE.override(fuse),
envs.SGLANG_OPT_USE_TILELANG_MHC_PRE.override(pre),
envs.SGLANG_OPT_USE_TILELANG_MHC_POST.override(post),
mock.patch.object(
deepseek_v4_fused_mhc,
"is_sm120_supported",
return_value=sm120,
),
override_platform(is_sm120=sm120),
):
self.assertEqual(
deepseek_v4_fused_mhc._is_fused_mhc_post_pre_enabled(),
@@ -83,7 +80,6 @@ class TestAmdFusedMhcCrossLayerGating(unittest.TestCase):
@mock.patch.object(deepseek_v4_fused_mhc, "get_bool_env_var", return_value=True)
@mock.patch.object(deepseek_v4_fused_mhc, "_is_hip", True)
def test_aiter_gfx95_enables_cross_layer_fusion(self, _mock_aiter, _mock_gfx95):
# TileLang flags off: fusion must still enable via the aiter gfx95 path.
with (
envs.SGLANG_OPT_FUSE_MHC_POST_PRE.override(False),
envs.SGLANG_OPT_USE_TILELANG_MHC_PRE.override(False),
@@ -55,10 +55,9 @@ def _accessor_names():
for node in tree.body
if isinstance(node, ast.FunctionDef) and node.name.startswith("get_")
}
# The context object itself is not a bag: it exists before anything is
# published, and `declare_late_resolution` calls it deliberately to find
# out whether the record it was handed has been published yet.
return frozenset(names - {"get_context"})
# Two that are not bags: the context object itself, and the platform facts.
# Both answer before anything is published.
return frozenset(names - {"get_context", "get_platform"})
_BAG_ACCESSORS = _accessor_names()
@@ -471,7 +471,7 @@ class TestResolutionReadsTheDeclarations(CustomTestCase):
members = _record_members()
# The floor is here to catch the scan collapsing, not to pin the
# class's size.
self.assertGreater(len(members), 25, f"only {len(members)} members were found")
self.assertGreater(len(members), 15, f"only {len(members)} members were found")
offenders = []
for name, fn in sorted(members.items()):
holders = _holders(fn) | {"self"}
@@ -73,7 +73,12 @@ from sglang.srt.model_executor.cuda_graph_config import (
Phase,
PhaseConfig,
)
from sglang.srt.runtime_context import get_context, get_serving
from sglang.srt.runtime_context import (
describe_kv_events_publisher,
get_context,
get_serving,
override_platform,
)
from sglang.srt.server_args import PortArgs, ServerArgs, prepare_server_args
from sglang.srt.utils.server_args_config_parser import ConfigArgumentMerger
from sglang.test.ci.ci_register import register_cpu_ci
@@ -333,8 +338,8 @@ class TestMultimodalFeatureTransport(CustomTestCase):
def _set_model_type(server_args, *, is_multimodal):
server_args._model_config = SimpleNamespace(is_multimodal=is_multimodal)
@patch("sglang.srt.arg_groups.serving_hook.is_cuda", return_value=True)
def test_cuda_ipc_is_explicit_and_bounded(self, _mock_is_cuda):
@override_platform(is_cuda=True)
def test_cuda_ipc_is_explicit_and_bounded(self):
server_args = ServerArgs(
model_path="dummy",
mm_feature_transport="cuda_ipc",
@@ -355,8 +360,8 @@ class TestMultimodalFeatureTransport(CustomTestCase):
self.assertIn("base GPU 2", output)
self.assertIn("4 tokenizer worker", output)
@patch("sglang.srt.arg_groups.serving_hook.is_cuda", return_value=True)
def test_legacy_keep_flag_maps_to_cuda_ipc(self, _mock_is_cuda):
@override_platform(is_cuda=True)
def test_legacy_keep_flag_maps_to_cuda_ipc(self):
server_args = ServerArgs(model_path="dummy", keep_mm_feature_on_device=True)
with patch.dict(os.environ, {"SGLANG_USE_CUDA_IPC_TRANSPORT": "0"}):
@@ -383,8 +388,8 @@ class TestMultimodalFeatureTransport(CustomTestCase):
with self.assertRaisesRegex(ValueError, "conflicts.*cuda_vmm"):
handle_multimodal_feature_transport(server_args)
@patch("sglang.srt.arg_groups.serving_hook.is_cuda", return_value=True)
def test_explicit_cpu_overrides_legacy_environment(self, _mock_is_cuda):
@override_platform(is_cuda=True)
def test_explicit_cpu_overrides_legacy_environment(self):
server_args = ServerArgs(model_path="dummy", mm_feature_transport="cpu")
with patch.dict(os.environ, {"SGLANG_USE_CUDA_IPC_TRANSPORT": "1"}):
@@ -409,8 +414,8 @@ class TestMultimodalFeatureTransport(CustomTestCase):
)
self.assertFalse(envs.SGLANG_USE_CUDA_IPC_TRANSPORT.get())
@patch("sglang.srt.arg_groups.serving_hook.is_cuda", return_value=True)
def test_default_transport_is_cpu_for_text_only_model(self, _mock_is_cuda):
@override_platform(is_cuda=True)
def test_default_transport_is_cpu_for_text_only_model(self):
server_args = ServerArgs(model_path="dummy")
self._set_model_type(server_args, is_multimodal=False)
@@ -424,8 +429,8 @@ class TestMultimodalFeatureTransport(CustomTestCase):
)
self.assertFalse(envs.SGLANG_USE_CUDA_IPC_TRANSPORT.get())
@patch("sglang.srt.arg_groups.serving_hook.is_cuda", return_value=True)
def test_default_transport_is_cpu_for_multimodal_model(self, _mock_is_cuda):
@override_platform(is_cuda=True)
def test_default_transport_is_cpu_for_multimodal_model(self):
server_args = ServerArgs(model_path="dummy")
self._set_model_type(server_args, is_multimodal=True)
@@ -443,17 +448,13 @@ class TestMultimodalFeatureTransport(CustomTestCase):
@patch(
"sglang.srt.arg_groups.serving_hook.is_mnnvl_fabric_device", return_value=True
)
@patch("sglang.srt.arg_groups.serving_hook.is_cuda", return_value=True)
@override_platform(is_cuda=True)
@patch(
"sglang.srt.model_loader.utils.supports_cuda_vmm_feature_transport",
return_value=True,
)
def test_default_transport_is_cuda_vmm_for_supported_multinode_mnnvl(
self,
_mock_supports_cuda_vmm,
_mock_is_cuda,
_mock_is_mnnvl,
_mock_path_exists,
self, _mock_supports_cuda_vmm, _mock_is_cuda, _mock_is_mnnvl
):
server_args = ServerArgs(model_path="dummy", nnodes=2)
self._set_model_type(server_args, is_multimodal=True)
@@ -476,17 +477,13 @@ class TestMultimodalFeatureTransport(CustomTestCase):
@patch(
"sglang.srt.arg_groups.serving_hook.is_mnnvl_fabric_device", return_value=True
)
@patch("sglang.srt.arg_groups.serving_hook.is_cuda", return_value=True)
@override_platform(is_cuda=True)
@patch(
"sglang.srt.model_loader.utils.supports_cuda_vmm_feature_transport",
return_value=False,
)
def test_default_transport_is_cpu_for_unsupported_multinode_model(
self,
_mock_supports_cuda_vmm,
_mock_is_cuda,
_mock_is_mnnvl,
_mock_path_exists,
self, _mock_supports_cuda_vmm, _mock_is_cuda, _mock_is_mnnvl
):
server_args = ServerArgs(model_path="dummy", nnodes=2)
self._set_model_type(server_args, is_multimodal=True)
@@ -501,9 +498,9 @@ class TestMultimodalFeatureTransport(CustomTestCase):
@patch(
"sglang.srt.arg_groups.serving_hook.is_mnnvl_fabric_device", return_value=True
)
@patch("sglang.srt.arg_groups.serving_hook.is_cuda", return_value=True)
@override_platform(is_cuda=True)
def test_default_transport_is_cpu_without_imex_channel(
self, _mock_is_cuda, _mock_is_mnnvl, _mock_path_exists
self, _mock_is_cuda, _mock_is_mnnvl
):
server_args = ServerArgs(model_path="dummy", nnodes=2)
self._set_model_type(server_args, is_multimodal=True)
@@ -522,10 +519,8 @@ class TestMultimodalFeatureTransport(CustomTestCase):
@patch(
"sglang.srt.arg_groups.serving_hook.is_mnnvl_fabric_device", return_value=False
)
@patch("sglang.srt.arg_groups.serving_hook.is_cuda", return_value=True)
def test_default_transport_is_cpu_for_multinode_non_mnnvl(
self, _mock_is_cuda, _mock_is_mnnvl
):
@override_platform(is_cuda=True)
def test_default_transport_is_cpu_for_multinode_non_mnnvl(self, _mock_is_cuda):
server_args = ServerArgs(model_path="dummy", nnodes=2)
self._set_model_type(server_args, is_multimodal=True)
@@ -538,8 +533,8 @@ class TestMultimodalFeatureTransport(CustomTestCase):
)
self.assertFalse(envs.SGLANG_USE_CUDA_IPC_TRANSPORT.get())
@patch("sglang.srt.arg_groups.serving_hook.is_cuda", return_value=True)
def test_default_transport_is_cpu_for_language_only_model(self, _mock_is_cuda):
@override_platform(is_cuda=True)
def test_default_transport_is_cpu_for_language_only_model(self):
server_args = ServerArgs(model_path="dummy", language_only=True)
self._set_model_type(server_args, is_multimodal=True)
@@ -552,15 +547,15 @@ class TestMultimodalFeatureTransport(CustomTestCase):
)
self.assertFalse(envs.SGLANG_USE_CUDA_IPC_TRANSPORT.get())
@patch("sglang.srt.arg_groups.serving_hook.is_cuda", return_value=False)
def test_cuda_ipc_rejects_non_nvidia_platforms(self, _mock_is_cuda):
@override_platform(is_cuda=False)
def test_cuda_ipc_rejects_non_nvidia_platforms(self):
server_args = ServerArgs(model_path="dummy", mm_feature_transport="cuda_ipc")
with self.assertRaisesRegex(ValueError, "requires NVIDIA CUDA"):
handle_multimodal_feature_transport(server_args)
@patch("sglang.srt.arg_groups.serving_hook.is_cuda", return_value=True)
def test_cuda_ipc_rejects_multi_node(self, _mock_is_cuda):
@override_platform(is_cuda=True)
def test_cuda_ipc_rejects_multi_node(self):
server_args = ServerArgs(
model_path="dummy", mm_feature_transport="cuda_ipc", nnodes=2
)
@@ -568,8 +563,8 @@ class TestMultimodalFeatureTransport(CustomTestCase):
with self.assertRaisesRegex(ValueError, "single node"):
handle_multimodal_feature_transport(server_args)
@patch("sglang.srt.arg_groups.serving_hook.is_cuda", return_value=True)
def test_cuda_vmm_is_explicit_and_uses_shared_budget(self, _mock_is_cuda):
@override_platform(is_cuda=True)
def test_cuda_vmm_is_explicit_and_uses_shared_budget(self):
server_args = ServerArgs(
model_path="dummy",
mm_feature_transport="cuda_vmm",
@@ -595,15 +590,15 @@ class TestMultimodalFeatureTransport(CustomTestCase):
self.assertIn("2 tokenizer worker", output)
self.assertIn("falls back to inline CPU", output)
@patch("sglang.srt.arg_groups.serving_hook.is_cuda", return_value=False)
def test_cuda_vmm_rejects_non_nvidia_platforms(self, _mock_is_cuda):
@override_platform(is_cuda=False)
def test_cuda_vmm_rejects_non_nvidia_platforms(self):
server_args = ServerArgs(model_path="dummy", mm_feature_transport="cuda_vmm")
with self.assertRaisesRegex(ValueError, "requires NVIDIA CUDA"):
handle_multimodal_feature_transport(server_args)
@patch("sglang.srt.arg_groups.serving_hook.is_cuda", return_value=True)
def test_cuda_vmm_rejects_rust_server(self, _mock_is_cuda):
@override_platform(is_cuda=True)
def test_cuda_vmm_rejects_rust_server(self):
server_args = ServerArgs(model_path="dummy", mm_feature_transport="cuda_vmm")
with (
@@ -612,8 +607,8 @@ class TestMultimodalFeatureTransport(CustomTestCase):
):
handle_multimodal_feature_transport(server_args)
@patch("sglang.srt.arg_groups.serving_hook.is_cuda", return_value=True)
def test_cuda_vmm_rejects_pipeline_parallelism(self, _mock_is_cuda):
@override_platform(is_cuda=True)
def test_cuda_vmm_rejects_pipeline_parallelism(self):
server_args = ServerArgs(
model_path="dummy", mm_feature_transport="cuda_vmm", pp_size=2
)
@@ -633,8 +628,8 @@ class TestMambaCacheStochasticRounding(unittest.TestCase):
with self.assertRaisesRegex(ValueError, "--mamba-ssm-dtype float16"):
handle_mamba_backend(server_args)
@patch("sglang.srt.arg_groups.mamba_hook.is_cuda", return_value=False)
def test_rejects_non_cuda(self, _mock_is_cuda):
@override_platform(is_cuda=False)
def test_rejects_non_cuda(self):
server_args = ServerArgs(
model_path="dummy",
mamba_ssm_dtype="float16",
@@ -644,9 +639,9 @@ class TestMambaCacheStochasticRounding(unittest.TestCase):
with self.assertRaisesRegex(ValueError, "NVIDIA CUDA"):
handle_mamba_backend(server_args)
@patch("sglang.srt.arg_groups.mamba_hook.is_cuda", return_value=True)
@patch("sglang.srt.arg_groups.mamba_hook.is_sm100_supported", return_value=False)
def test_rejects_triton_without_sm100(self, _mock_sm100, _mock_is_cuda):
@override_platform(is_cuda=True)
@override_platform(is_sm100=False)
def test_rejects_triton_without_sm100(self):
server_args = ServerArgs(
model_path="dummy",
mamba_ssm_dtype="float16",
@@ -832,8 +827,8 @@ class TestHiSparseDsaBackendPolicy(unittest.TestCase):
)
with (
patch("sglang.srt.configs.model_config.is_deepseek_dsa", return_value=True),
patch("sglang.srt.arg_groups.overrides.is_npu", return_value=False),
patch("sglang.srt.arg_groups.overrides.is_xpu", return_value=False),
override_platform(is_npu=False),
override_platform(is_xpu=False),
patch("torch.cuda.get_device_capability", return_value=(9, 0)),
):
declared = _dsa_split_backend_resolution(view)
@@ -846,22 +841,22 @@ class TestHiSparseDsaBackendPolicy(unittest.TestCase):
),
}
@patch("sglang.srt.arg_groups.hisparse_hook._is_hip", return_value=False)
def test_hisparse_defaults_to_flashmla_sparse_on_cuda_bfloat16(self, _mock_is_hip):
@override_platform(is_hip=False)
def test_hisparse_defaults_to_flashmla_sparse_on_cuda_bfloat16(self):
resolved = self._resolve("bfloat16")
self.assertEqual(resolved["dsa_prefill_backend"], "flashmla_sparse")
self.assertEqual(resolved["dsa_decode_backend"], "flashmla_sparse")
@patch("sglang.srt.arg_groups.hisparse_hook._is_hip", return_value=False)
def test_hisparse_defaults_to_flashmla_kv_on_cuda_fp8(self, _mock_is_hip):
@override_platform(is_hip=False)
def test_hisparse_defaults_to_flashmla_kv_on_cuda_fp8(self):
resolved = self._resolve("fp8_e4m3")
self.assertEqual(resolved["dsa_prefill_backend"], "flashmla_kv")
self.assertEqual(resolved["dsa_decode_backend"], "flashmla_kv")
@patch("sglang.srt.arg_groups.hisparse_hook._is_hip", return_value=False)
def test_hisparse_accepts_flashinfer_sparse_mla_on_cuda_fp8(self, _mock_is_hip):
@override_platform(is_hip=False)
def test_hisparse_accepts_flashinfer_sparse_mla_on_cuda_fp8(self):
"""SM120 GLM DSA resolves both DSA backends to flashinfer_sparse_mla, so
the fp8 hisparse allow-set must admit it or --enable-hisparse cannot
start there at all. The device/arch narrowing happens later, in
@@ -877,24 +872,22 @@ class TestHiSparseDsaBackendPolicy(unittest.TestCase):
validate_hisparse_dsa_backend(server_args, "dsa_prefill_backend", "prefill")
validate_hisparse_dsa_backend(server_args, "dsa_decode_backend", "decode")
@patch("sglang.srt.arg_groups.hisparse_hook._is_hip", return_value=True)
def test_hisparse_defaults_to_tilelang_on_rocm(self, _mock_is_hip):
@override_platform(is_hip=True)
def test_hisparse_defaults_to_tilelang_on_rocm(self):
resolved = self._resolve("bfloat16")
self.assertEqual(resolved["dsa_prefill_backend"], "tilelang")
self.assertEqual(resolved["dsa_decode_backend"], "tilelang")
@patch("sglang.srt.arg_groups.hisparse_hook._is_hip", return_value=True)
def test_hisparse_preserves_rocm_user_backend_and_defaults_missing_side(
self, _mock_is_hip
):
@override_platform(is_hip=True)
def test_hisparse_preserves_rocm_user_backend_and_defaults_missing_side(self):
resolved = self._resolve("bfloat16", dsa_prefill_backend="tilelang")
self.assertEqual(resolved["dsa_prefill_backend"], "tilelang")
self.assertEqual(resolved["dsa_decode_backend"], "tilelang")
@patch("sglang.srt.arg_groups.hisparse_hook._is_hip", return_value=True)
def test_hisparse_accepts_aiter_backend_on_rocm(self, _mock_is_hip):
@override_platform(is_hip=True)
def test_hisparse_accepts_aiter_backend_on_rocm(self):
server_args = ServerArgs(
model_path="dummy",
enable_hisparse=True,
@@ -906,8 +899,8 @@ class TestHiSparseDsaBackendPolicy(unittest.TestCase):
validate_hisparse_dsa_backend(server_args, "dsa_prefill_backend", "prefill")
validate_hisparse_dsa_backend(server_args, "dsa_decode_backend", "decode")
@patch("sglang.srt.arg_groups.hisparse_hook._is_hip", return_value=True)
def test_hisparse_rejects_cuda_backend_on_rocm(self, _mock_is_hip):
@override_platform(is_hip=True)
def test_hisparse_rejects_cuda_backend_on_rocm(self):
server_args = ServerArgs(
model_path="dummy",
enable_hisparse=True,
@@ -918,8 +911,8 @@ class TestHiSparseDsaBackendPolicy(unittest.TestCase):
with self.assertRaisesRegex(ValueError, "tilelang"):
validate_hisparse_dsa_backend(server_args, "dsa_prefill_backend", "prefill")
@patch("sglang.srt.arg_groups.hisparse_hook._is_hip", return_value=False)
def test_hisparse_rejects_rocm_backend_on_cuda(self, _mock_is_hip):
@override_platform(is_hip=False)
def test_hisparse_rejects_rocm_backend_on_cuda(self):
server_args = ServerArgs(
model_path="dummy",
enable_hisparse=True,
@@ -971,15 +964,15 @@ class TestFa4PageSizeAutoForce(CustomTestCase):
args.decode_attention_backend = decode
args.page_size = page_size
# Short-circuit model_config_of(): the fa4 page_size branch only needs
# use_mla_backend() (mocked) and is_sm100_supported() (mocked), not a
# use_mla_backend() (mocked) and override_platform(is_sm100=...), not a
# real model_config. Pre-set the attribute so get_model_config returns
# early without touching ModelConfig.from_server_args.
args._model_config = MagicMock()
args._model_config.hf_config.dual_chunk_attention_config = None
return args
@patch("sglang.srt.arg_groups.overrides.is_sm100_supported", return_value=True)
def test_combined_attention_backend_fa4_forces_page_size_128(self, _mock_sm100):
@override_platform(is_sm100=True)
def test_combined_attention_backend_fa4_forces_page_size_128(self):
# `--attention-backend fa4` (combined): prefill/decode fields stay None.
args = self._make_args(attention_backend="fa4")
@@ -990,8 +983,8 @@ class TestFa4PageSizeAutoForce(CustomTestCase):
self.assertEqual(args.page_size, 1) # the field stays pristine
self.assertEqual(resolved_view(args).page_size, 128)
@patch("sglang.srt.arg_groups.overrides.is_sm100_supported", return_value=True)
def test_explicit_prefill_fa4_forces_page_size_128(self, _mock_sm100):
@override_platform(is_sm100=True)
def test_explicit_prefill_fa4_forces_page_size_128(self):
# `--prefill-attention-backend fa4`: the previously-covered path.
args = self._make_args(attention_backend=None, prefill="fa4", page_size=1)
@@ -1969,11 +1962,11 @@ class TestPrefillCudaGraphLoRACompatibility(CustomTestCase):
prefill=PhaseConfig(backend=Backend.TC_PIECEWISE)
)
with (
patch("sglang.srt.arg_groups.cuda_graph_hook.is_hip", return_value=False),
patch("sglang.srt.arg_groups.cuda_graph_hook.is_npu", return_value=False),
override_platform(is_hip=False),
override_platform(is_npu=False),
patch("sglang.srt.arg_groups.cuda_graph_hook.is_cpu", return_value=False),
patch("sglang.srt.arg_groups.cuda_graph_hook.is_mps", return_value=False),
patch("sglang.srt.arg_groups.cuda_graph_hook.is_xpu", return_value=False),
override_platform(is_xpu=False),
):
disable_tc_piecewise_cudagraph_if_incompatible(args)
@@ -2602,15 +2595,27 @@ class TestGrpcServerArgs(CustomTestCase):
],
host="127.0.0.1",
)
# The port the sidecar dials is the resolved one, off the bag.
override = get_context_for_config().override_server_args(grpc_port=50051)
# Every value the sidecar reads is resolved config, so the case states
# them all through the context rather than half here and half in a
# stand-in the readers no longer consult.
override = get_context_for_config().override_server_args(
grpc_port=50051,
sidecar="example.sidecar",
sidecar_args=[
"--sidecar-shutdown-timeout",
"42",
"--grpc-connections",
"2",
],
host="127.0.0.1",
)
override.install()
self.addCleanup(override.restore)
with (
patch("sglang.srt.entrypoints.sidecar.mp.get_context") as get_context,
patch("sglang.srt.entrypoints.sidecar.Sidecar") as sidecar_class,
):
start_sidecar(server_args)
start_sidecar()
process_kwargs = get_context.return_value.Process.call_args.kwargs
self.assertEqual(process_kwargs["name"], "sglang_sidecar_example.sidecar")
@@ -2826,11 +2831,11 @@ class TestDcpKvEventContract(CustomTestCase):
page_size=64,
kv_events_config=self.KV_EVENTS,
)
self.assertEqual(args.describe_kv_events_publisher()["block_size"], 256)
self.assertEqual(describe_kv_events_publisher(args)["block_size"], 256)
args = ServerArgs(
model_path="dummy", page_size=64, kv_events_config=self.KV_EVENTS
)
self.assertEqual(args.describe_kv_events_publisher()["block_size"], 64)
self.assertEqual(describe_kv_events_publisher(args)["block_size"], 64)
def test_kv_event_block_size_widens_a_single_token_page(self):
# page_size=1 + DCP is a real deployment shape: the allocator is still
@@ -16,6 +16,7 @@ from sglang.test.test_utils import CustomTestCase
register_cuda_ci(est_time=10, stage="base-b", runner_config="1-gpu-small")
from sglang.srt.model_executor.forward_batch_info import ForwardMode
from sglang.srt.runtime_context import override_platform
from sglang.srt.speculative import dflash_info
from sglang.srt.speculative.dflash_info import DFlashVerifyInput
@@ -29,6 +30,9 @@ def _lazy_view(**overrides):
mamba_track_interval=256,
page_size=64,
chunked_prefill_size=None,
# The cases name a supported arch, so the view states the backend
# rather than the answer.
linear_attn_backend="triton",
)
for key, value in overrides.items():
setattr(view, key, value)
@@ -39,15 +43,9 @@ class TestValidateMambaExtraBufferLazyDflash(CustomTestCase):
"""The DFLASH rejection is gone; the neighboring invariants still hold."""
def _validate(self, view):
with mock.patch(
"sglang.srt.arg_groups.overrides.supports_mamba_cache_extra_buffer",
return_value=True,
), mock.patch(
# Keep the test runnable on CPU-only hosts: the platform assert is
# not what is under test here.
"sglang.srt.arg_groups.mamba_hook.is_cuda",
return_value=True,
):
# Keep the test runnable on CPU-only hosts: the platform assert is
# not what is under test here.
with override_platform(is_cuda=True):
validate_mamba_extra_buffer(
view,
"Qwen3NextForCausalLM",
@@ -81,10 +79,7 @@ class TestValidateMambaExtraBufferLazyDflash(CustomTestCase):
def _must_not_be_read():
raise AssertionError("the chunk size was read before page_size resolved")
with mock.patch(
"sglang.srt.arg_groups.overrides.supports_mamba_cache_extra_buffer",
return_value=True,
), mock.patch("sglang.srt.arg_groups.mamba_hook.is_cuda", return_value=True):
with override_platform(is_cuda=True):
validate_mamba_extra_buffer(
_lazy_view(page_size=None),
"Qwen3NextForCausalLM",
+104 -144
View File
@@ -15,7 +15,6 @@ from types import SimpleNamespace
from typing import Optional
from unittest.mock import patch
from sglang.srt.arg_groups import attention_hook
from sglang.srt.arg_groups import overrides as overrides_module
from sglang.srt.arg_groups.arg_utils import A, Arg, resolvable_fields
from sglang.srt.arg_groups.overrides import (
@@ -31,6 +30,7 @@ from sglang.srt.runtime_context import (
get_context,
get_exec,
get_server_args,
override_platform,
reset_context,
)
from sglang.test.test_utils import CustomTestCase
@@ -427,11 +427,7 @@ class TestGoldenModelOverrides(_IsolatedPublish):
)
def test_sparse_minicpm_defaults_to_sparse_attention_backend(self):
with patch.object(
overrides_module,
"is_blackwell_supported",
return_value=False,
):
with override_platform(is_blackwell=False):
for architecture in ("MiniCPMForCausalLM", "MiniCPMSALAForCausalLM"):
with self.subTest(architecture=architecture):
self.assertEqual(
@@ -456,20 +452,14 @@ class TestGoldenModelOverrides(_IsolatedPublish):
has_lightning_layers=False,
)
with patch.object(
overrides_module, "is_blackwell_supported", return_value=False
):
with override_platform(is_blackwell=False):
overrides = overrides_module._minicpm_sala_overrides(args, config)
self.assertTrue(overrides["disable_radix_cache"])
self.assertEqual(overrides["attention_backend"], "minicpm_flashattn")
def test_sparse_minicpm_defaults_to_flashinfer_on_blackwell(self):
with patch.object(
overrides_module,
"is_blackwell_supported",
return_value=True,
):
with override_platform(is_blackwell=True):
self.assertEqual(
self._minicpm_overrides(
"MiniCPMSALAForCausalLM",
@@ -531,11 +521,7 @@ class TestGoldenModelOverrides(_IsolatedPublish):
)["attention_backend"],
"flashinfer",
)
with patch.object(
overrides_module,
"is_blackwell_supported",
return_value=True,
):
with override_platform(is_blackwell=True):
self.assertEqual(
self._minicpm_overrides(
"MiniCPMSALAForCausalLM",
@@ -544,11 +530,7 @@ class TestGoldenModelOverrides(_IsolatedPublish):
)["attention_backend"],
"fa4",
)
with patch.object(
overrides_module,
"is_blackwell_supported",
return_value=False,
):
with override_platform(is_blackwell=False):
split_overrides = self._minicpm_overrides(
"MiniCPMSALAForCausalLM",
sparse_attention=True,
@@ -625,9 +607,7 @@ class TestGoldenModelOverrides(_IsolatedPublish):
"""MiniMax-M2 NVFP4 auto must avoid the unsupported plain TRT-LLM path."""
# Every module that asks: the attention handler validates what the
# override family picks, and each holds its own import.
with patch.object(
overrides_module, "is_sm100_supported", return_value=True
), patch.object(attention_hook, "is_sm100_supported", return_value=True):
with override_platform(is_sm100=True), override_platform(is_sm100=True):
explicit = self._construct(
"MiniMaxM2ForCausalLM",
"llama",
@@ -666,8 +646,8 @@ class TestGoldenModelOverrides(_IsolatedPublish):
# Thor (SM110) and other architectures keep the existing auto behavior.
with (
patch.object(overrides_module, "is_sm100_supported", return_value=False),
patch.object(overrides_module, "is_sm120_supported", return_value=False),
override_platform(is_sm100=False),
override_platform(is_sm120=False),
):
non_sm10x = self._construct(
"MiniMaxM2ForCausalLM", "llama", quantization="modelopt_fp4"
@@ -688,7 +668,7 @@ class TestGoldenModelOverrides(_IsolatedPublish):
return SimpleNamespace(**defaults)
# Non-SM100: the MoE pin must not fire, so hf_config is never inspected.
with patch.object(overrides_module, "is_sm100_supported", return_value=False):
with override_platform(is_sm100=False):
self.assertEqual(
_mimo_v2_overrides(_args(speculative_algorithm="EAGLE"), None),
{"enable_multi_layer_eagle": True},
@@ -704,7 +684,7 @@ class TestGoldenModelOverrides(_IsolatedPublish):
defaults.update(kw)
return SimpleNamespace(**defaults)
with patch.object(overrides_module, "is_sm100_supported", return_value=True):
with override_platform(is_sm100=True):
with patch.object(
overrides_module, "get_quantization_config", return_value="fp8"
):
@@ -724,7 +704,7 @@ class TestGoldenModelOverrides(_IsolatedPublish):
self.assertEqual(_mimo_v2_overrides(_args(), None), {})
def test_mimo_v2_family_is_registered(self):
with patch.object(overrides_module, "is_sm100_supported", return_value=False):
with override_platform(is_sm100=False):
self.assertEqual(
collect_model_override_declarations(
"MiMoV2FlashForCausalLM",
@@ -785,8 +765,8 @@ class TestGoldenModelOverrides(_IsolatedPublish):
)
with (
patch.object(overrides_module, "is_sm100_supported", return_value=True),
patch.object(overrides_module, "is_blackwell_supported", return_value=True),
override_platform(is_sm100=True),
override_platform(is_blackwell=True),
):
self.assertEqual(
_nemotron_h_overrides(server_args, hf_config),
@@ -815,8 +795,8 @@ class TestGoldenModelOverrides(_IsolatedPublish):
)
with (
patch.object(overrides_module, "is_sm100_supported", return_value=True),
patch.object(overrides_module, "is_blackwell_supported", return_value=True),
override_platform(is_sm100=True),
override_platform(is_blackwell=True),
):
self.assertEqual(
_nemotron_h_overrides(server_args, hf_config),
@@ -848,16 +828,8 @@ class TestGoldenModelOverrides(_IsolatedPublish):
server_args.speculative_algorithm = "EAGLE"
with (
patch.object(
overrides_module,
"is_blackwell_supported",
return_value=True,
),
patch.object(
overrides_module,
"is_sm100_supported",
return_value=is_sm100,
),
override_platform(is_blackwell=True),
override_platform(is_sm100=is_sm100),
):
overrides = _nemotron_h_overrides(server_args, hf_config)
for key, value in expected.items():
@@ -871,12 +843,8 @@ class TestGoldenModelOverrides(_IsolatedPublish):
server_args, hf_config = self._nemotron_h_args(quantized_layers={})
server_args.speculative_algorithm = algorithm
with (
patch.object(
overrides_module, "is_blackwell_supported", return_value=True
),
patch.object(
overrides_module, "is_sm100_supported", return_value=True
),
override_platform(is_blackwell=True),
override_platform(is_sm100=True),
):
overrides = _nemotron_h_overrides(server_args, hf_config)
self.assertEqual(overrides["attention_backend"], "trtllm_mha")
@@ -888,8 +856,8 @@ class TestGoldenModelOverrides(_IsolatedPublish):
server_args, hf_config = self._nemotron_h_args(quantized_layers={})
server_args.speculative_algorithm = "DFLASH"
with (
patch.object(overrides_module, "is_blackwell_supported", return_value=True),
patch.object(overrides_module, "is_sm100_supported", return_value=True),
override_platform(is_blackwell=True),
override_platform(is_sm100=True),
):
overrides = _nemotron_h_overrides(server_args, hf_config)
self.assertEqual(overrides["attention_backend"], "trtllm_mha")
@@ -905,8 +873,8 @@ class TestGoldenModelOverrides(_IsolatedPublish):
server_args.speculative_draft_attention_backend = "flashinfer"
with (
patch.object(overrides_module, "is_blackwell_supported", return_value=True),
patch.object(overrides_module, "is_sm100_supported", return_value=True),
override_platform(is_blackwell=True),
override_platform(is_sm100=True),
):
overrides = _nemotron_h_overrides(server_args, hf_config)
@@ -923,8 +891,8 @@ class TestGoldenModelOverrides(_IsolatedPublish):
server_args.speculative_eagle_topk = 4
with (
patch.object(overrides_module, "is_blackwell_supported", return_value=True),
patch.object(overrides_module, "is_sm100_supported", return_value=True),
override_platform(is_blackwell=True),
override_platform(is_sm100=True),
):
overrides = _nemotron_h_overrides(server_args, hf_config)
@@ -939,8 +907,8 @@ class TestGoldenModelOverrides(_IsolatedPublish):
server_args, hf_config = self._nemotron_h_args(quantized_layers={})
with (
patch.object(overrides_module, "is_blackwell_supported", return_value=True),
patch.object(overrides_module, "is_sm100_supported", return_value=False),
override_platform(is_blackwell=True),
override_platform(is_sm100=False),
):
self.assertNotIn(
"attention_backend", _nemotron_h_overrides(server_args, hf_config)
@@ -952,8 +920,8 @@ class TestGoldenModelOverrides(_IsolatedPublish):
server_args, hf_config = self._nemotron_h_args(quantized_layers={})
with (
patch.object(overrides_module, "is_blackwell_supported", return_value=True),
patch.object(overrides_module, "is_sm100_supported", return_value=True),
override_platform(is_blackwell=True),
override_platform(is_sm100=True),
):
self.assertEqual(
_nemotron_h_overrides(server_args, hf_config)["attention_backend"],
@@ -969,8 +937,8 @@ class TestGoldenModelOverrides(_IsolatedPublish):
server_args.speculative_draft_attention_backend = "fa3"
with (
patch.object(overrides_module, "is_blackwell_supported", return_value=True),
patch.object(overrides_module, "is_sm100_supported", return_value=True),
override_platform(is_blackwell=True),
override_platform(is_sm100=True),
):
overrides = _nemotron_h_overrides(server_args, hf_config)
self.assertNotIn("attention_backend", overrides)
@@ -1083,7 +1051,7 @@ class TestGoldenModelOverrides(_IsolatedPublish):
def test_gpt_oss_xpu_dtype_validation_reads_pristine(self):
from sglang.srt.arg_groups.overrides import _gpt_oss_overrides
with patch.object(overrides_module, "is_xpu", return_value=True):
with override_platform(is_xpu=True):
with self.assertRaises(NotImplementedError):
_gpt_oss_overrides(
SimpleNamespace(
@@ -1299,12 +1267,12 @@ class TestGoldenModelOverrides(_IsolatedPublish):
view = ResolvedView(
SimpleNamespace(attention_backend="intel_amx", device="cpu")
)
with patch.object(overrides_module, "cpu_has_amx_support", return_value=False):
with override_platform(has_amx=False):
self.assertEqual(
_attention_backend_platform_fallbacks(view),
{"attention_backend": "torch_native"},
)
with patch.object(overrides_module, "cpu_has_amx_support", return_value=True):
with override_platform(has_amx=True):
self.assertEqual(_attention_backend_platform_fallbacks(view), {})
# dual-chunk config: mismatched explicit backend raises verbatim
@@ -1338,20 +1306,20 @@ class TestGoldenModelOverrides(_IsolatedPublish):
defaults.update(kw)
return ResolvedView(SimpleNamespace(**defaults))
with patch.object(overrides_module, "is_hip", return_value=True):
with override_platform(is_hip=True):
self.assertEqual(
_dllm_attention_backend(_view()), {"attention_backend": "triton"}
)
self.assertEqual(
_dllm_attention_backend(_view(attention_backend="aiter")), {}
)
with patch.object(overrides_module, "is_hip", return_value=False):
with patch.object(overrides_module, "is_npu", return_value=True):
with override_platform(is_hip=False):
with override_platform(is_npu=True):
self.assertEqual(
_dllm_attention_backend(_view()),
{"attention_backend": "ascend"},
)
with patch.object(overrides_module, "is_npu", return_value=False):
with override_platform(is_npu=False):
# cuda graph disabled -> nothing to force
self.assertEqual(_dllm_attention_backend(_view()), {})
self.assertEqual(
@@ -1366,13 +1334,13 @@ class TestGoldenModelOverrides(_IsolatedPublish):
_page_size_default(ResolvedView(SimpleNamespace(page_size=64))), {}
)
# default fill on non-HIP/non-MUSA platforms is 1
with patch.object(overrides_module, "is_hip", return_value=False):
with patch.object(overrides_module, "is_musa", return_value=False):
with override_platform(is_hip=False):
with override_platform(is_musa=False):
self.assertEqual(
_page_size_default(ResolvedView(SimpleNamespace(page_size=None))),
{"page_size": 1},
)
with patch.object(overrides_module, "is_musa", return_value=True):
with override_platform(is_musa=True):
self.assertEqual(
_page_size_default(ResolvedView(SimpleNamespace(page_size=None))),
{"page_size": 64},
@@ -1477,7 +1445,7 @@ class TestGoldenModelOverrides(_IsolatedPublish):
with (
envs.SGLANG_DSV4_FP4_DEQUANT.override(False),
patch.object(overrides_module, "is_sm100_supported", return_value=True),
override_platform(is_sm100=True),
):
self.assertEqual(
_deepseek_v4_overrides(_args(), hf),
@@ -1505,7 +1473,7 @@ class TestGoldenModelOverrides(_IsolatedPublish):
# FlashInfer MXFP4 only supports the standard (non-A2A) dispatcher.
with (
envs.SGLANG_DSV4_FP4_DEQUANT.override(False),
patch.object(overrides_module, "is_sm100_supported", return_value=True),
override_platform(is_sm100=True),
):
self.assertNotIn(
"moe_runner_backend",
@@ -1514,7 +1482,7 @@ class TestGoldenModelOverrides(_IsolatedPublish):
# Runtime FP4-to-FP8 dequantization must retain the generic FP8 runner.
with (
envs.SGLANG_DSV4_FP4_DEQUANT.override(True),
patch.object(overrides_module, "is_sm100_supported", return_value=True),
override_platform(is_sm100=True),
):
self.assertNotIn(
"moe_runner_backend",
@@ -1531,7 +1499,7 @@ class TestGoldenModelOverrides(_IsolatedPublish):
"moe_runner_backend",
_deepseek_v4_overrides(_args(device="npu"), hf),
)
with patch.object(overrides_module, "is_hip", return_value=True):
with override_platform(is_hip=True):
self.assertNotIn(
"moe_runner_backend",
_deepseek_v4_overrides(_args(), hf),
@@ -1539,9 +1507,9 @@ class TestGoldenModelOverrides(_IsolatedPublish):
# Unsupported NVIDIA architectures keep the generic auto-resolution
# path instead of selecting a FlashInfer kernel that cannot launch.
with (
patch.object(overrides_module, "is_sm90_supported", return_value=False),
patch.object(overrides_module, "is_sm100_supported", return_value=False),
patch.object(overrides_module, "is_sm120_supported", return_value=False),
override_platform(is_sm90=False),
override_platform(is_sm100=False),
override_platform(is_sm120=False),
):
self.assertNotIn(
"moe_runner_backend",
@@ -1550,9 +1518,9 @@ class TestGoldenModelOverrides(_IsolatedPublish):
# SM120 uses the same model hook; no later pass is needed.
with (
envs.SGLANG_DSV4_FP4_DEQUANT.override(False),
patch.object(overrides_module, "is_sm90_supported", return_value=False),
patch.object(overrides_module, "is_sm100_supported", return_value=False),
patch.object(overrides_module, "is_sm120_supported", return_value=True),
override_platform(is_sm90=False),
override_platform(is_sm100=False),
override_platform(is_sm120=True),
):
self.assertEqual(
_deepseek_v4_overrides(_args(), hf)["moe_runner_backend"],
@@ -1605,8 +1573,8 @@ class TestGoldenModelOverrides(_IsolatedPublish):
hf = _hf()
with (
patch.object(overrides_module, "is_sm100_supported", return_value=True),
patch.object(overrides_module, "is_blackwell_supported", return_value=True),
override_platform(is_sm100=True),
override_platform(is_blackwell=True),
):
# modelopt checkpoint: quant algo resolution + sm100 defaults
self.assertEqual(
@@ -1625,11 +1593,9 @@ class TestGoldenModelOverrides(_IsolatedPublish):
"modelopt_mixed",
)
with (
patch.object(overrides_module, "is_sm100_supported", return_value=False),
patch.object(overrides_module, "is_cuda", return_value=True),
patch.object(
overrides_module, "get_device_capability", return_value=(9, 0)
),
override_platform(is_sm100=False),
override_platform(is_cuda=True),
override_platform(device_capability=(9, 0)),
):
# SM80-SM90 fp4: marlin
self.assertEqual(
@@ -1651,8 +1617,8 @@ class TestGoldenModelOverrides(_IsolatedPublish):
hf_without_quant_cfg = _hf(include_quantization_config=False)
with (
patch.object(overrides_module, "is_sm100_supported", return_value=True),
patch.object(overrides_module, "is_blackwell_supported", return_value=True),
override_platform(is_sm100=True),
override_platform(is_blackwell=True),
):
for modelopt_quantization in ("modelopt_fp8", "modelopt_fp4"):
with self.subTest(modelopt_quantization=modelopt_quantization):
@@ -1718,9 +1684,9 @@ class TestGoldenModelOverrides(_IsolatedPublish):
with (
patch("sglang.srt.configs.model_config.is_deepseek_dsa", return_value=True),
patch.object(overrides_module, "is_npu", return_value=False),
patch.object(overrides_module, "is_xpu", return_value=False),
patch.object(overrides_module, "is_hip", return_value=False),
override_platform(is_npu=False),
override_platform(is_xpu=False),
override_platform(is_hip=False),
patch("torch.cuda.get_device_capability", return_value=(9, 0)),
):
# Hopper FP8 -> flashmla_kv both
@@ -1758,9 +1724,9 @@ class TestGoldenModelOverrides(_IsolatedPublish):
)
with (
patch("sglang.srt.configs.model_config.is_deepseek_dsa", return_value=True),
patch.object(overrides_module, "is_npu", return_value=False),
patch.object(overrides_module, "is_xpu", return_value=False),
patch.object(overrides_module, "is_hip", return_value=False),
override_platform(is_npu=False),
override_platform(is_xpu=False),
override_platform(is_hip=False),
patch("torch.cuda.get_device_capability", return_value=(12, 0)),
):
self.assertEqual(
@@ -1772,9 +1738,9 @@ class TestGoldenModelOverrides(_IsolatedPublish):
)
with (
patch("sglang.srt.configs.model_config.is_deepseek_dsa", return_value=True),
patch.object(overrides_module, "is_npu", return_value=False),
patch.object(overrides_module, "is_xpu", return_value=False),
patch.object(overrides_module, "is_hip", return_value=True),
override_platform(is_npu=False),
override_platform(is_xpu=False),
override_platform(is_hip=True),
patch("torch.cuda.get_device_capability", return_value=(9, 4)),
):
# ROCm with both unset -> tilelang
@@ -1811,8 +1777,8 @@ class TestGoldenModelOverrides(_IsolatedPublish):
)
with (
patch.object(overrides_module, "is_sm90_supported", return_value=True),
patch.object(overrides_module, "is_sm100_supported", return_value=False),
override_platform(is_sm90=True),
override_platform(is_sm100=False),
):
self.assertEqual(
_flashinfer_allreduce_fusion_auto_enable(_view()),
@@ -1897,7 +1863,7 @@ class TestGoldenModelOverrides(_IsolatedPublish):
defaults.update(kw)
return ResolvedView(SimpleNamespace(**defaults))
with patch.object(overrides_module, "is_sm100_supported", return_value=True):
with override_platform(is_sm100=True):
# decode-only cutedsl: prefill defaults to trtllm_mla
self.assertEqual(
_cutedsl_prefill_backend_fill(_view()),
@@ -1921,7 +1887,7 @@ class TestGoldenModelOverrides(_IsolatedPublish):
_cutedsl_prefill_backend_fill(_view(decode_attention_backend=None)),
{},
)
with patch.object(overrides_module, "is_sm100_supported", return_value=False):
with override_platform(is_sm100=False):
with self.assertRaises(ValueError):
_cutedsl_prefill_backend_fill(_view())
@@ -1971,8 +1937,8 @@ class TestGoldenModelOverrides(_IsolatedPublish):
with (
patch("sglang.srt.configs.model_config.is_deepseek_dsa", return_value=True),
patch.object(overrides_module, "is_npu", return_value=False),
patch.object(overrides_module, "is_xpu", return_value=False),
override_platform(is_npu=False),
override_platform(is_xpu=False),
):
with patch("torch.cuda.get_device_capability", return_value=(9, 0)):
# Hopper: auto -> bfloat16
@@ -2052,7 +2018,7 @@ class TestGoldenModelOverrides(_IsolatedPublish):
SimpleNamespace(_model_config=SimpleNamespace(hf_config=hf), **defaults)
)
with patch.object(overrides_module, "is_hip", return_value=True):
with override_platform(is_hip=True):
with patch.object(
envs.SGLANG_NVFP4_CKPT_FP8_NEXTN_MOE, "get", return_value=False
):
@@ -2093,7 +2059,7 @@ class TestGoldenModelOverrides(_IsolatedPublish):
with self.assertRaises(ValueError):
_deepseek_spec_moe_resolution(_view(ep_size=1))
# the arm is HIP-only
with patch.object(overrides_module, "is_hip", return_value=False):
with override_platform(is_hip=False):
self.assertEqual(_deepseek_spec_moe_resolution(_view()), {})
def test_mamba_radix_cache_resolution_pass(self):
@@ -2234,9 +2200,7 @@ class TestGoldenModelOverrides(_IsolatedPublish):
args._model_config = SimpleNamespace(attention_arch=AttentionArch.MHA)
return args
with patch.object(
overrides_module, "is_sm100_supported", return_value=True
), patch.object(
with override_platform(is_sm100=True), patch.object(
overrides_module,
"get_default_attn_backend",
lambda server_args, **_: server_args.default_backend_for_test,
@@ -2278,13 +2242,13 @@ class TestGoldenModelOverrides(_IsolatedPublish):
),
{"attention_backend": "trtllm_mha", "page_size": 64},
)
with patch.object(overrides_module, "is_sm100_supported", return_value=False):
with override_platform(is_sm100=False):
self.assertEqual(_qwen3_5_hybrid_overrides(_args("fa3"), None), {})
def test_qwen3vl_page_size(self):
from sglang.srt.arg_groups.overrides import _qwen3vl_overrides
with patch.object(overrides_module, "is_hip", return_value=True):
with override_platform(is_hip=True):
with patch("sglang.srt.environ.envs.SGLANG_USE_AITER_UNIFIED_ATTN") as e:
e.get.return_value = True
self.assertEqual(
@@ -2306,7 +2270,7 @@ class TestGoldenModelOverrides(_IsolatedPublish):
defaults.update(kw)
return ResolvedView(SimpleNamespace(**defaults))
with patch.object(overrides_module, "is_sm100_supported", return_value=True):
with override_platform(is_sm100=True):
self.assertEqual(
_moe_runner_backend_quant_constraints(
_view(quantization="nvfp4_online")
@@ -2321,7 +2285,7 @@ class TestGoldenModelOverrides(_IsolatedPublish):
_moe_runner_backend_quant_constraints(_view(quantization="mxfp8")),
{"moe_runner_backend": "flashinfer_trtllm"},
)
with patch.object(overrides_module, "is_sm120_supported", return_value=True):
with override_platform(is_sm120=True):
self.assertEqual(
_moe_runner_backend_quant_constraints(
_view(quantization="modelopt_fp4")
@@ -2366,7 +2330,7 @@ class TestGoldenModelOverrides(_IsolatedPublish):
defaults.update(kw)
return SimpleNamespace(**defaults)
with patch("sglang.srt.utils.common.is_sm100_supported", return_value=True):
with override_platform(is_sm100=True):
# e4m3 + trtllm_mha + SM100: mode active
self.assertTrue(m3_fp8_attn_gemm_enabled(_args()))
# fa4 dense backend: mode inactive (no fp8-q GEMM path)
@@ -2380,7 +2344,7 @@ class TestGoldenModelOverrides(_IsolatedPublish):
# otherwise-active config
with envs.SGLANG_DISABLE_M3_FP8_ATTN_GEMM.override(True):
self.assertFalse(m3_fp8_attn_gemm_enabled(_args()))
with patch("sglang.srt.utils.common.is_sm100_supported", return_value=False):
with override_platform(is_sm100=False):
# non-SM100: mode inactive
self.assertFalse(m3_fp8_attn_gemm_enabled(_args()))
@@ -2400,8 +2364,8 @@ class TestGoldenModelOverrides(_IsolatedPublish):
return ns
hf = SimpleNamespace()
with patch.object(overrides_module, "is_hip", return_value=False), patch.object(
overrides_module, "is_sm100_supported", return_value=True
with override_platform(is_hip=False), override_platform(
is_sm100=True
), patch.object(overrides_module, "get_quantization_config", return_value=None):
# fp8_e4m3 KV: SM100 backend default flips to trtllm_mha (the only
# dense backend with the fp8-q GEMM path); page snaps to 128
@@ -2501,7 +2465,7 @@ class TestGoldenModelOverrides(_IsolatedPublish):
# no matching backend: nothing declared
self.assertEqual(_mla_backend_page_constraints(_view()), {})
with patch.object(overrides_module, "is_sm100_supported", return_value=True):
with override_platform(is_sm100=True):
self.assertEqual(
_fa4_page_constraint(
_view(
@@ -2565,7 +2529,7 @@ class TestGoldenModelOverrides(_IsolatedPublish):
defaults.update(kw)
return SimpleNamespace(**defaults)
with patch.object(overrides_module, "is_sm100_supported", return_value=True):
with override_platform(is_sm100=True):
self.assertEqual(
_llama4_overrides(_args(), None), {"attention_backend": "trtllm_mha"}
)
@@ -2599,9 +2563,9 @@ class TestGoldenModelOverrides(_IsolatedPublish):
self.assertEqual(
_lfm2_overrides(_args(), None), {"attention_backend": "flashinfer"}
)
with patch.object(overrides_module, "is_sm100_supported", return_value=False):
with override_platform(is_sm100=False):
self.assertEqual(_minicpm_v4_6_overrides(_args(), None), {})
with patch.object(overrides_module, "is_sm90_supported", return_value=True):
with override_platform(is_sm90=True):
self.assertEqual(
_llama4_overrides(_args(), None), {"attention_backend": "fa3"}
)
@@ -2609,11 +2573,11 @@ class TestGoldenModelOverrides(_IsolatedPublish):
_gemma4_overrides(_args(), None), {"attention_backend": "triton"}
)
# Glm4Moe: unconditional tf32 declaration + (sm100) quant/moe absorption
with patch.object(overrides_module, "is_sm100_supported", return_value=False):
with override_platform(is_sm100=False):
self.assertEqual(
_glm4_moe_overrides(None, None), {"enable_tf32_matmul": True}
)
with patch.object(overrides_module, "is_sm100_supported", return_value=True):
with override_platform(is_sm100=True):
self.assertEqual(
_glm4_moe_overrides(
SimpleNamespace(
@@ -2654,7 +2618,7 @@ class TestGoldenModelOverrides(_IsolatedPublish):
defaults.update(kw)
return ResolvedView(SimpleNamespace(**defaults))
with patch.object(overrides_module, "is_sm100_supported", return_value=True):
with override_platform(is_sm100=True):
with patch.object(
overrides_module, "get_quantization_config", return_value="fp8"
):
@@ -2670,7 +2634,7 @@ class TestGoldenModelOverrides(_IsolatedPublish):
self.assertEqual(
_deepseek_moe_quant_resolution(_view(arch="LlamaForCausalLM")), {}
)
with patch.object(overrides_module, "is_sm100_supported", return_value=False):
with override_platform(is_sm100=False):
self.assertEqual(_deepseek_moe_quant_resolution(_view()), {})
def test_data_parallelism_and_a2a_passes(self):
@@ -2725,15 +2689,15 @@ class TestGoldenModelOverrides(_IsolatedPublish):
with patch(
"sglang.srt.configs.model_config.is_deepseek_dsa", return_value=True
):
with patch.object(overrides_module, "is_npu", return_value=False):
with patch.object(overrides_module, "is_xpu", return_value=False):
with patch.object(overrides_module, "is_hip", return_value=False):
with override_platform(is_npu=False):
with override_platform(is_xpu=False):
with override_platform(is_hip=False):
self.assertEqual(
_deepseek_family_overrides(_args(), None),
{"attention_backend": "dsa", "page_size": 64},
)
# HIP without the preshuffle path: page 1
with patch.object(overrides_module, "is_hip", return_value=True):
with override_platform(is_hip=True):
with patch(
"sglang.srt.layers.attention.dsa.utils.aiter_can_use_preshuffle_paged_mqa",
return_value=False,
@@ -2746,9 +2710,9 @@ class TestGoldenModelOverrides(_IsolatedPublish):
with patch(
"sglang.srt.configs.model_config.is_deepseek_dsa", return_value=True
):
with patch.object(overrides_module, "is_npu", return_value=False):
with patch.object(overrides_module, "is_xpu", return_value=False):
with patch.object(overrides_module, "is_hip", return_value=False):
with override_platform(is_npu=False):
with override_platform(is_xpu=False):
with override_platform(is_hip=False):
result = _deepseek_family_overrides(
_args(
enable_prefill_cp=True,
@@ -2789,9 +2753,7 @@ class TestGoldenModelOverrides(_IsolatedPublish):
with patch(
"sglang.srt.configs.model_config.is_deepseek_dsa", return_value=False
):
with patch.object(
overrides_module, "is_sm100_supported", return_value=True
):
with override_platform(is_sm100=True):
self.assertEqual(
_deepseek_family_overrides(_args(), None),
{"attention_backend": "trtllm_mla"},
@@ -2802,15 +2764,13 @@ class TestGoldenModelOverrides(_IsolatedPublish):
),
{},
)
with patch.object(
overrides_module, "is_sm100_supported", return_value=False
):
with override_platform(is_sm100=False):
self.assertEqual(_deepseek_family_overrides(_args(), None), {})
def test_qwen3_moe_family_quant_absorption(self):
from sglang.srt.arg_groups.overrides import _qwen3_moe_family_overrides
with patch.object(overrides_module, "is_sm100_supported", return_value=True):
with override_platform(is_sm100=True):
with patch.object(
overrides_module, "get_quantization_config", return_value="fp8"
):
@@ -2829,7 +2789,7 @@ class TestGoldenModelOverrides(_IsolatedPublish):
"moe_runner_backend": "flashinfer_trtllm",
},
)
with patch.object(overrides_module, "is_sm100_supported", return_value=False):
with override_platform(is_sm100=False):
self.assertEqual(_qwen3_moe_family_overrides(None, None), {})
def test_step3p_declarations_at_callable_level(self):
@@ -0,0 +1,81 @@
"""No module-scope name may freeze a platform fact.
The address exists so `override_platform(...)` reaches every reader at once.
A module-level `_is_sm120 = get_platform().is_sm120` defeats that completely:
the value is read when the module is first imported and never again, so whether
an override is visible depends on import order -- and the line *looks* like it
went through the address, which is worse than the bare probe it replaced.
Four of these were written during this refactor's own conversion (three in
`fp8_utils`, one in `deepseek_v4_backend`), by substituting the accessor into
lines that were already frozen. Substituting the call is not the conversion; the
conversion is the reader asking at the point of decision.
"""
from sglang.test.ci.ci_register import register_cpu_ci
register_cpu_ci(est_time=6, suite="base-a-test-cpu")
import ast
import pathlib
import unittest
import sglang
from sglang.test.test_utils import CustomTestCase
_ROOT = pathlib.Path(next(iter(sglang.__path__))) / "srt"
def _frozen_platform_reads():
"""(file, line, name) for each module-scope `x = get_platform().y`."""
found = []
for path in sorted(_ROOT.rglob("*.py")):
source = path.read_text(encoding="utf-8-sig")
if "get_platform" not in source:
continue
try:
tree = ast.parse(source)
except SyntaxError:
continue
# Module scope only: inside a function the call runs per invocation,
# which is the shape the address is for.
for node in tree.body:
if not isinstance(node, ast.Assign):
continue
value = node.value
if not (
isinstance(value, ast.Attribute)
and isinstance(value.value, ast.Call)
and getattr(value.value.func, "id", None) == "get_platform"
):
continue
for target in node.targets:
if isinstance(target, ast.Name):
rel = path.relative_to(_ROOT).as_posix()
found.append(f"{rel}:{node.lineno} {target.id}")
return found
class TestPlatformAddressNotFrozen(CustomTestCase):
def test_the_scan_reaches_the_address(self):
"""The premise: `get_platform()` is used somewhere under srt/."""
users = [
path
for path in _ROOT.rglob("*.py")
if "get_platform()" in path.read_text(encoding="utf-8-sig")
]
self.assertGreater(len(users), 20, "the scan found almost no readers")
def test_no_module_scope_name_freezes_a_platform_fact(self):
frozen = _frozen_platform_reads()
self.assertEqual(
[],
frozen,
"these read a platform fact once at import and keep the answer, so "
"`override_platform(...)` cannot reach them and the result depends "
f"on import order. Ask at the point of decision instead: {frozen}",
)
if __name__ == "__main__":
unittest.main()
@@ -0,0 +1,90 @@
"""A platform fact has one address, and one place to change it.
The value lives in one object, so `override_platform(...)` reaches every reader
at once, and the facts answer before anything is published.
"""
import unittest
from sglang.srt.runtime_context import (
_PLATFORM_PROBES,
_PLATFORM_VALUES,
get_platform,
override_platform,
)
from sglang.srt.utils import common as _common
from sglang.test.ci.ci_register import register_cpu_ci
from sglang.test.test_utils import CustomTestCase
register_cpu_ci(est_time=5, suite="base-a-test-cpu")
class TestPlatformContext(CustomTestCase):
def test_every_name_maps_to_a_real_probe(self):
"""A typo in either table would answer with an AttributeError at runtime."""
for name, probe in {**_PLATFORM_PROBES, **_PLATFORM_VALUES}.items():
self.assertTrue(
callable(getattr(_common, probe, None)),
f"{name} maps to {probe!r}, which is not callable in utils.common",
)
def test_it_answers_before_anything_is_published(self):
"""Unlike a config bag: a launcher asks these before it publishes."""
platform = get_platform()
for name in _PLATFORM_PROBES:
self.assertIsInstance(getattr(platform, name), bool)
def test_the_probe_is_what_it_answers_with(self):
platform = get_platform()
for name, probe in {**_PLATFORM_PROBES, **_PLATFORM_VALUES}.items():
self.assertEqual(getattr(platform, name), getattr(_common, probe)())
def test_an_override_is_scoped_and_restores(self):
platform = get_platform()
before = platform.is_sm100
with override_platform(is_sm100=not before):
self.assertEqual(platform.is_sm100, not before)
self.assertEqual(platform.is_sm100, before)
def test_overrides_nest(self):
platform = get_platform()
with override_platform(is_sm100=True):
self.assertTrue(platform.is_sm100)
with override_platform(is_sm100=False):
self.assertFalse(platform.is_sm100)
self.assertTrue(platform.is_sm100)
def test_an_unknown_fact_is_refused_on_both_paths(self):
with self.assertRaises(AttributeError):
get_platform().is_sm999
with self.assertRaises(ValueError):
override_platform(is_sm999=True).install()
def test_a_fact_is_not_assigned(self):
"""Assignment would move it for one reader, which is the defect."""
with self.assertRaises(AttributeError):
get_platform().is_sm100 = True
def test_one_statement_reaches_every_reader(self):
"""The point of the address.
Two modules that both ask the context see the same answer from one
override -- which is what the round-4 failure needed and could not say.
"""
def module_a():
return get_platform().is_sm100
def module_b():
return get_platform().is_sm100
with override_platform(is_sm100=True):
self.assertTrue(module_a())
self.assertTrue(module_b())
with override_platform(is_sm100=False):
self.assertFalse(module_a())
self.assertFalse(module_b())
if __name__ == "__main__":
unittest.main()
@@ -0,0 +1,134 @@
"""The launch path runs before `publish`, so nothing on it may ask a config bag.
`publish` is what projects the bags; every accessor fails closed until then
(`config namespace 'observability' not published`). Most readers live deep in a
runtime path and are safely downstream of it, so converting one to a bag is
normally free. The launcher's own reads are not: everything
`_launch_subprocesses` calls before its `publish` runs with no bags at all, and
`multimodal_gen` calls into the same code with a `ServerArgs` of its own that
never publishes them.
This is a class no other test in the tree catches: a converted reader is
exercised everywhere by tests that publish first, so it passes unit CI and then
takes the server down on the first real launch -- which is how `configure_logger`
shipped. So the protected set is *derived from the launch path* rather than
listed here: the callees named in `_launch_subprocesses` above its `publish` are
read out of the source, and each is called against a context where nothing has
been published. A conversion of any of them turns this red without a boot.
"""
from sglang.test.ci.ci_register import register_cpu_ci
register_cpu_ci(est_time=5, suite="base-a-test-cpu")
import ast
import logging
import pathlib
import unittest
import sglang
from sglang.srt.entrypoints.engine import _set_envs_and_config
from sglang.srt.runtime_context import get_observability, reset_context
from sglang.srt.server_args import ServerArgs
from sglang.srt.utils.common import configure_logger
from sglang.test.test_utils import CustomTestCase
# Callees this guard can drive directly. `_set_envs_and_config` is here too: it
# is the other one that runs on every launch before anything is published.
_EXERCISED = {
"configure_logger": configure_logger,
"_set_envs_and_config": _set_envs_and_config,
}
# Named by the launcher before `publish`, but none of them reads config out of a
# record. Listed so the set above stays a statement about all of the callees.
_NOT_EXERCISED = {
"load_plugins",
"resolve_auto_parsers",
"snapshot_context",
"resolving_view",
}
def _pre_publish_callees():
"""Functions `_launch_subprocesses` calls before it publishes.
Read from the source so the set cannot go stale: a call added above the
`publish(...)` line joins the protected set on its own.
"""
source = (
pathlib.Path(next(iter(sglang.__path__))) / "srt" / "entrypoints" / "engine.py"
).read_text(encoding="utf-8-sig")
tree = ast.parse(source)
launcher = next(
node
for node in ast.walk(tree)
if isinstance(node, ast.FunctionDef) and node.name == "_launch_subprocesses"
)
publish_line = min(
node.lineno
for node in ast.walk(launcher)
if isinstance(node, ast.Call) and getattr(node.func, "id", None) == "publish"
)
return {
node.func.id
for node in ast.walk(launcher)
if isinstance(node, ast.Call)
and isinstance(node.func, ast.Name)
and node.lineno < publish_line
}
class TestPrePublishReaders(CustomTestCase):
def setUp(self):
self._levels = {
name: logging.getLogger(name).level
for name in (None, "sglang", "httpx", "httpcore")
}
reset_context()
self.addCleanup(self._restore_levels)
self.addCleanup(reset_context)
def _restore_levels(self):
for name, level in self._levels.items():
logging.getLogger(name).setLevel(level)
def test_nothing_is_published_here(self):
"""The premise: this fixture really is a pre-publish context."""
with self.assertRaises(ValueError) as caught:
get_observability()
self.assertIn("not published", str(caught.exception))
def test_the_protected_set_is_what_the_launcher_calls(self):
"""If the launcher stops calling one of these, or starts calling
something new before publishing, this file has to be looked at."""
self.assertEqual(set(_EXERCISED) | _NOT_EXERCISED, _pre_publish_callees())
def test_none_of_them_asks_a_bag(self):
"""Each is called with nothing published. A bag read raises
`config namespace ... not published` -- any other failure is the
callee's own business and does not belong to this guard."""
server_args = ServerArgs(model_path="dummy", log_level="warning")
for name, call in sorted(_EXERCISED.items()):
with self.subTest(callee=name):
reset_context()
try:
call(server_args)
except Exception as exc: # noqa: BLE001 -- see the docstring
self.assertNotIn(
"not published",
str(exc),
f"{name} runs before publish and asked a config bag",
)
def test_configure_logger_reads_the_record_it_was_handed(self):
"""The one that shipped broken, pinned by value rather than by not
raising: `multimodal_gen` hands it a ServerArgs that never publishes."""
configure_logger(ServerArgs(model_path="dummy", log_level="warning"))
self.assertEqual(logging.getLogger().level, logging.WARNING)
configure_logger(ServerArgs(model_path="dummy", log_level="error"))
self.assertEqual(logging.getLogger().level, logging.ERROR)
if __name__ == "__main__":
unittest.main()
+32 -7
View File
@@ -26,6 +26,7 @@ from sglang.srt.arg_groups.overrides import (
max_prefill_buffer_tokens as max_prefill_buffer_tokens_of,
)
from sglang.srt.arg_groups.overrides import (
resolution_result,
resolved_view,
)
from sglang.srt.runtime_context import (
@@ -377,10 +378,12 @@ class TestServerArgsScopedOverride(_IsolatedServerArgs):
)
published = override.install()
self.assertIs(get_server_args(), published)
self.assertEqual(published.attention_backend, "triton")
self.assertEqual(published.chunked_prefill_size, -1)
# The hook declares; the record keeps the operator's input, so the
# values are read where resolution puts them.
self.assertEqual(resolution_result(published, "attention_backend"), "triton")
self.assertEqual(resolution_result(published, "chunked_prefill_size"), -1)
# unnamed fields keep their dataclass defaults
self.assertEqual(published.tp_size, 1)
self.assertEqual(resolution_result(published, "tp_size"), 1)
def test_unknown_fields_are_rejected(self):
with self.assertRaises(ValueError):
@@ -391,7 +394,7 @@ class TestServerArgsScopedOverride(_IsolatedServerArgs):
get_context().set_server_args(previous)
override = get_context().override_server_args(tp_size=8)
override.install()
self.assertEqual(get_server_args().tp_size, 8)
self.assertEqual(get_parallel().tp_size, 8)
override.restore()
self.assertIs(get_server_args(), previous)
@@ -406,9 +409,9 @@ class TestServerArgsScopedOverride(_IsolatedServerArgs):
reset_context()
with get_context().override_server_args(tp_size=2) as outer:
with get_context().override_server_args(tp_size=4):
self.assertEqual(get_server_args().tp_size, 4)
self.assertEqual(get_parallel().tp_size, 4)
self.assertIs(get_server_args(), outer)
self.assertEqual(get_server_args().tp_size, 2)
self.assertEqual(get_parallel().tp_size, 2)
def test_private_attribute_seeding(self):
# Property caches (e.g. _mamba_cache_chunk_size) are seeded through
@@ -418,13 +421,35 @@ class TestServerArgsScopedOverride(_IsolatedServerArgs):
)
self.assertEqual(mamba_cache_chunk_size_of(published), 64)
def test_an_underscore_field_is_declared_like_any_other(self):
"""The split is fields vs not-fields, not the leading underscore.
`_speculative_draft_quantization_explicitly_set` is a real field
published under `spec`. Seeding it as a raw attribute instead of
declaring it would leave the earlier resolution authoritative, so both
the resolution and the bag would keep answering the pre-override value
while the record said otherwise.
"""
from sglang.srt.arg_groups.overrides import resolution_result
from sglang.srt.runtime_context import get_spec
name = "_speculative_draft_quantization_explicitly_set"
self.assertIn(name, ServerArgs.__dataclass_fields__)
published = get_context().override_server_args(**{name: True}).install()
# The record keeps the operator's input, as it does for every other
# field; the override travels as a declaration.
self.assertIsNone(getattr(published, name))
self.assertIs(resolution_result(published, name), True)
self.assertIs(getattr(get_spec(), name), True)
def test_installed_config_arms_the_strict_guard(self):
# The published dummy must behave like a resolved config: bare writes
# raise.
published = get_context().override_server_args(tp_size=2).install()
with self.assertRaises(AttributeError):
published.tp_size = 4
self.assertEqual(published.tp_size, 2)
self.assertEqual(resolution_result(published, "tp_size"), 2)
def test_restore_resets_the_capture_seed(self):
# install() seeds flags.capture from the published dummy; restore()
@@ -98,7 +98,10 @@ class TestNoServerArgsMutationEntry(CustomTestCase):
)
def test_late_resolution_refuses_the_published_config(self):
from sglang.srt.arg_groups.overrides import declare_late_resolution
from sglang.srt.arg_groups.overrides import (
declare_late_resolution,
resolution_result,
)
from sglang.srt.runtime_context import get_context
override = get_context().override_server_args(tp_size=2)
@@ -107,7 +110,9 @@ class TestNoServerArgsMutationEntry(CustomTestCase):
with self.assertRaises(ValueError):
declare_late_resolution(published, "test", tp_size=4)
self.assertEqual(published.tp_size, 2)
# The refusal left the resolution alone: the hook's declaration stands,
# and the record still carries the operator's input.
self.assertEqual(resolution_result(published, "tp_size"), 2)
if __name__ == "__main__":