minke.yu/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 13 Packages Projects Releases Wiki Activity

config: the resolution callbacks into the record go to zero (#36972)

Browse Source 
Co-authored-by: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
Cheng Wan
2026-08-29 04:18:05 -07:00
committed by GitHub
co-authored by Claude Opus 5
parent 48b88e1256
commit b65e677e48
50 changed files with 1087 additions and 979 deletions
Download Patch File Download Diff File Expand all files Collapse all files
test/registered/cpu/test_server_args_backend.py
+2 -2
View File
@@ -58,12 +58,12 @@ class TestServerArgsIBDeviceValidation(unittest.TestCase):
real_listdir = os.listdir
with patch(
"sglang.srt.server_args.os.path.isdir",
"sglang.srt.arg_groups.validation_hook.os.path.isdir",
side_effect=lambda path: (
True if path == "/sys/class/infiniband" else real_isdir(path)
),
), patch(
"sglang.srt.server_args.os.listdir",
"sglang.srt.arg_groups.validation_hook.os.listdir",
side_effect=lambda path: (
available_devices
if path == "/sys/class/infiniband"
test/registered/unit/configs/test_multimodal_piecewise_cuda_graph.py
+10 -14
View File
@@ -88,8 +88,9 @@ class TestMultimodalPiecewiseCudaGraph(CustomTestCase):
"sglang.srt.arg_groups.cuda_graph_hook"
".disable_tc_piecewise_cudagraph_if_incompatible"
) as disable_if_incompatible,
patch.object(
args, "_resolved_attention_backends", return_value=("fa3", "fa3")
patch(
"sglang.srt.arg_groups.overrides.attention_backends_of",
return_value=("fa3", "fa3"),
),
):
apply_cuda_graph_compatibility(args)
@@ -116,12 +117,11 @@ class TestMultimodalPiecewiseCudaGraph(CustomTestCase):
args._cuda_graph_config_locked = set()
with (
patch.object(
args,
"_resolved_attention_backends",
patch(
"sglang.srt.arg_groups.overrides.attention_backends_of",
return_value=("trtllm_mla", "trtllm_mla"),
),
patch.object(args, "use_mla_backend", return_value=True),
patch("sglang.srt.arg_groups.overrides.use_mla_backend", return_value=True),
):
apply_cuda_graph_compatibility(args)
@@ -137,9 +137,8 @@ class TestMultimodalPiecewiseCudaGraph(CustomTestCase):
)
args._cuda_graph_config_locked = {(Phase.PREFILL, "backend")}
with patch.object(
args,
"_resolved_attention_backends",
with patch(
"sglang.srt.arg_groups.overrides.attention_backends_of",
return_value=("trtllm_mla", "trtllm_mla"),
):
apply_cuda_graph_compatibility(args)
@@ -185,10 +184,7 @@ class TestMultimodalPiecewiseCudaGraph(CustomTestCase):
args.disable_radix_cache = False
args.chunked_prefill_size = 2048
with (
patch.object(args, "get_model_config", return_value=args._model_config),
patch("sglang.srt.arg_groups.model_hook.is_cuda", return_value=True),
):
with (patch("sglang.srt.arg_groups.model_hook.is_cuda", return_value=True),):
handle_model_capability_adjustments(args)
self.assertTrue(resolution_result(args, "disable_radix_cache"))
@@ -215,7 +211,7 @@ class TestMultimodalPiecewiseCudaGraph(CustomTestCase):
hf_config=SimpleNamespace(architectures=["BertModel"]),
)
with patch.object(args, "get_model_config", return_value=args._model_config):
if True: # the record already carries the seeded configuration
handle_model_capability_adjustments(args)
self.assertTrue(resolution_result(args, "is_embedding"))
test/registered/unit/mem_cache/test_unified_radix_cache_unittest.py
+7 -4
View File
@@ -87,7 +87,10 @@ from sglang.srt.mem_cache.unified_radix_cache import (
_OngoingPrefetch,
_OngoingWriteThrough,
)
from sglang.srt.runtime_context import get_server_args, get_serving
from sglang.srt.runtime_context import (
get_serving,
mamba_cache_chunk_size,
)
from sglang.srt.sampling.sampling_params import SamplingParams
from sglang.srt.server_args import (
ServerArgs,
@@ -5225,7 +5228,7 @@ class UnifiedRadixCacheSuite:
if not self.cfg.has_mamba or self.cfg.has_swa or self.cfg.page_size != 1:
self.skipTest("requires page_size=1 Full+Mamba")
cache, allocator, req_to_token_pool = build_fixture(self.cfg)
chunk_size = get_server_args().mamba_cache_chunk_size
chunk_size = mamba_cache_chunk_size()
tokens = self._make_seq(1, chunk_size + 1)
self._insert(cache, allocator, req_to_token_pool, tokens)
leaf = cache.match_prefix(
@@ -5258,7 +5261,7 @@ class UnifiedRadixCacheSuite:
if not self.cfg.has_mamba or self.cfg.has_swa or self.cfg.page_size != 1:
self.skipTest("requires page_size=1 Full+Mamba")
cache, allocator, req_to_token_pool = self._build_hicache_fixture()
chunk_size = get_server_args().mamba_cache_chunk_size
chunk_size = mamba_cache_chunk_size()
prefix = self._make_seq(1, chunk_size)
tokens = prefix + self._make_seq(1000, chunk_size + 1)
self._insert(cache, allocator, req_to_token_pool, prefix)
@@ -5283,7 +5286,7 @@ class UnifiedRadixCacheSuite:
if not self.cfg.has_mamba or self.cfg.has_swa or self.cfg.page_size != 1:
self.skipTest("requires page_size=1 Full+Mamba")
cache, allocator, req_to_token_pool = self._build_hicache_fixture()
chunk_size = get_server_args().mamba_cache_chunk_size
chunk_size = mamba_cache_chunk_size()
prefix = self._make_seq(1, chunk_size)
tokens = prefix + self._make_seq(1000, chunk_size + 1)
self._insert(cache, allocator, req_to_token_pool, prefix)
test/registered/unit/model_executor/runner/test_flashinfer_autotune.py
+7 -4
View File
@@ -50,10 +50,7 @@ def test_packed_speculative_extend_is_limited_to_pd_prefill_target(mode, error):
def test_chunked_prefill_disabled_uses_legacy_token_ceiling():
model_runner = SimpleNamespace(
server_args=SimpleNamespace(
max_prefill_buffer_tokens=Mock(return_value=0),
max_prefill_tokens=32768,
),
server_args=SimpleNamespace(),
is_generation=True,
is_draft_worker=False,
spec_algorithm=SimpleNamespace(is_speculative=lambda: False),
@@ -76,6 +73,12 @@ def test_chunked_prefill_disabled_uses_legacy_token_ceiling():
"get_disagg",
return_value=SimpleNamespace(disaggregation_mode="prefill"),
),
patch.object(flashinfer_autotune, "max_prefill_buffer_tokens", return_value=0),
patch.object(
flashinfer_autotune,
"get_schedule",
return_value=SimpleNamespace(max_prefill_tokens=32768),
),
patch.object(flashinfer_autotune, "run_flashinfer_autotune_forward"),
patch.object(flashinfer_autotune.torch.cuda, "empty_cache"),
):
test/registered/unit/server_args/test_model_config_cache.py
+11 -9
View File
@@ -1,4 +1,4 @@
"""`get_model_config()` caches, and the key is the path the record carried.
"""`model_config_of()` caches, and the key is the path the record carried.
Two movements of a `model_path` reach this cache, and only the first one means
the cached configuration describes the wrong checkpoint:
@@ -20,6 +20,7 @@ import tempfile
import unittest
from types import SimpleNamespace
from sglang.srt.arg_groups.overrides import declare_resolution, model_config_of
from sglang.srt.configs.model_config import ModelConfig
from sglang.srt.environ import EnvField, envs
from sglang.srt.server_args import ServerArgs
@@ -117,7 +118,7 @@ class TestTheModelConfigCache(CustomTestCase):
self.assertEqual(server_args.model_path, _OBJECT_STORE_URI)
self.assertEqual(cached.model_path, pulled)
self.assertIs(server_args.get_model_config(), cached)
self.assertIs(model_config_of(server_args), cached)
def test_a_declared_model_path_rebuilds_the_configuration(self):
"""The GGUF and ModelScope shape: the record's own path moved."""
@@ -125,14 +126,15 @@ class TestTheModelConfigCache(CustomTestCase):
second_checkpoint = self._checkpoint()
server_args = ServerArgs(model_path=first_checkpoint, device="cuda")
first = server_args.get_model_config()
first = model_config_of(server_args)
self.assertEqual(first.model_path, first_checkpoint)
server_args._declare(
declare_resolution(
server_args,
"test_a_declared_model_path_rebuilds_the_configuration",
model_path=second_checkpoint,
)
second = server_args.get_model_config()
second = model_config_of(server_args)
self.assertIsNot(second, first)
self.assertEqual(second.model_path, second_checkpoint)
@@ -151,11 +153,11 @@ class TestTheModelConfigCache(CustomTestCase):
model_path=second_checkpoint,
)
rebuilt = copy_.get_model_config()
rebuilt = model_config_of(copy_)
self.assertEqual(rebuilt.model_path, second_checkpoint)
self.assertIs(copy_.get_model_config(), rebuilt)
self.assertIs(model_config_of(copy_), rebuilt)
# The parent keeps the configuration it resolved with.
self.assertEqual(server_args.get_model_config().model_path, first_checkpoint)
self.assertEqual(model_config_of(server_args).model_path, first_checkpoint)
def test_a_supplied_configuration_is_handed_back(self):
"""A configuration nothing in here built carries no key, so nothing
@@ -164,7 +166,7 @@ class TestTheModelConfigCache(CustomTestCase):
stand_in = SimpleNamespace(model_path="somewhere/else")
server_args._model_config = stand_in
self.assertIs(server_args.get_model_config(), stand_in)
self.assertIs(model_config_of(server_args), stand_in)
if __name__ == "__main__":
test/registered/unit/server_args/test_model_config_reads_resolved_input.py
+14 -13
View File
@@ -30,7 +30,7 @@ _SRT = pathlib.Path(sglang.__file__).resolve().parent / "srt"
# Two quantities sharing one name.
_READ_BEFORE_RESOLUTION = frozenset({"is_embedding"})
# Declared after the first `get_model_config()`, so the cached configuration
# Declared after the first `model_config_of()`, so the cached configuration
# holds the earlier value. Nothing reads the stale copy today (its one consumer
# is on the `is_draft_model` branch, built after resolution), and fixing it
# means moving the build or the hook. Pinned so a second field in this position
@@ -109,7 +109,7 @@ def _registry_collection_is_after_the_build():
Handler-local ordering only -- the caller still has to compare against the
pipeline-wide first build, which sits in an *earlier* step: hoisting the
collection above this handler's own `get_model_config()` call does not move
collection above this handler's own `model_config_of()` call does not move
it above the configuration another handler already cached.
"""
handler = None
@@ -145,7 +145,7 @@ def _registry_collection_is_after_the_build():
name = func.id
else:
continue
if name == "get_model_config" and build is None:
if name == "model_config_of" and build is None:
build = node.lineno
if name == "collect_model_override_declarations" and collect is None:
collect = node.lineno
@@ -192,11 +192,12 @@ def _server_args_names(tree, path):
and value.args[0].id in names
)
# `resolved = self._resolved()` is the same view, spelled as the
# record's own member.
# resolution vocabulary.
member = (
isinstance(value, ast.Call)
and isinstance(value.func, ast.Attribute)
and value.func.attr == "_resolved"
and isinstance(value.func, ast.Name)
and value.func.id == "resolved_view"
and isinstance(value.func.value, ast.Name)
and value.func.value.id in names
)
@@ -267,7 +268,7 @@ def _late_resolution_fields():
if isinstance(node.func, ast.Attribute)
else getattr(node.func, "id", "")
)
if called in ("_late_resolution", "declare_late_resolution"):
if called == "declare_late_resolution":
fields |= {kw.arg for kw in node.keywords if kw.arg}
return fields
@@ -488,14 +489,14 @@ def _declaration_positions():
wanted = _constructor_reads()
def build_site():
"""(step index, method name, line) of the first `get_model_config()`."""
"""(step index, method name, line) of the first `model_config_of()`."""
for index, step in enumerate(steps):
for method in reached[step]:
for node in ast.walk(methods[method]):
if (
isinstance(node, ast.Call)
and isinstance(node.func, ast.Attribute)
and node.func.attr == "get_model_config"
and isinstance(node.func, ast.Name)
and node.func.id == "model_config_of"
):
return index, step, method, node.lineno
return None
@@ -532,8 +533,8 @@ def _declaration_positions():
same_body = index == build_index and method == build_method
rank = 0 if same_body and node.lineno < build_line_in_body else 1
if (
isinstance(node.func, ast.Attribute)
and node.func.attr == "_declare"
isinstance(node.func, ast.Name)
and node.func.id == "declare_resolution"
):
fields = {kw.arg for kw in node.keywords if kw.arg}
# A handler that calls an imported hook (the Kimi and DeepSeek
@@ -669,8 +670,8 @@ class TestModelConfigReadsResolvedInput(CustomTestCase):
for method in reached[step]
if any(
isinstance(node, ast.Call)
and isinstance(node.func, ast.Attribute)
and node.func.attr == "get_model_config"
and isinstance(node.func, ast.Name)
and node.func.id == "model_config_of"
for node in ast.walk(methods[method])
)
)
test/registered/unit/server_args/test_no_public_non_field_slot.py
+1 -1
View File
@@ -61,7 +61,7 @@ class TestNoPublicNonFieldSlot(CustomTestCase):
written = _self_written_attributes()
self.assertGreater(
len(written),
5,
3,
f"only {len(written)} self-writes found; the scan is broken, not the "
"record",
)
test/registered/unit/server_args/test_page_major_backend_allowlist.py
+13 -7
View File
@@ -30,6 +30,7 @@ under its own default configuration.
"""
import unittest
from unittest import mock
from sglang.srt.arg_groups.kv_cache_hook import handle_page_major_kv_layout
from sglang.srt.server_args import ServerArgs
@@ -66,13 +67,18 @@ def _accepts(
"mamba_backend": "triton",
}.items():
object.__setattr__(sa, name, value)
sa.use_mla_backend = lambda: use_mla
sa._resolved_attention_backends = lambda: [backend]
try:
handle_page_major_kv_layout(sa)
return True
except AssertionError:
return False
# `use_mla_backend` asks the model configuration, which this stand-in has
# no room for; the case under test is what the handler does with the answer.
# The handler imports it inside the function, so the source module is
# where the patch has to go.
with mock.patch(
"sglang.srt.arg_groups.overrides.use_mla_backend", return_value=use_mla
):
try:
handle_page_major_kv_layout(sa)
return True
except AssertionError:
return False
class TestPageMajorBackendAllowlist(unittest.TestCase):
test/registered/unit/server_args/test_resolution_declarations.py
+4 -3
View File
@@ -24,7 +24,6 @@ import unittest
import unittest.mock
import sglang
from sglang.srt import server_args as server_args_module
from sglang.srt.arg_groups.overrides import resolution_result
from sglang.srt.server_args import ServerArgs
from sglang.test.ci.ci_register import register_cpu_ci
@@ -147,7 +146,7 @@ def _late_resolvers():
if isinstance(node.func, ast.Attribute)
else getattr(node.func, "id", None)
)
if called in ("declare_late_resolution", "_late_resolution"):
if called == "declare_late_resolution":
return True
if called and reaches(called, seen):
return True
@@ -887,7 +886,9 @@ class TestResolutionDeclarations(CustomTestCase):
# The pipeline asks the platform other questions on the way through
# (whether it is out of tree, whether it supports piecewise capture),
# and which of those it reaches depends on the host.
class _Plugin(type(server_args_module.current_platform)):
from sglang.srt.platforms import current_platform
class _Plugin(type(current_platform)):
device_name = "oot"
def apply_server_args_defaults(self, server_args):
test/registered/unit/server_args/test_resolution_is_reproducible.py
+5 -5
View File
@@ -36,7 +36,7 @@ import unittest.mock
import torch
import sglang
from sglang.srt.arg_groups.overrides import resolution_result
from sglang.srt.arg_groups.overrides import model_config_of, resolution_result
from sglang.srt.environ import EnvField, envs
from sglang.srt.server_args import ServerArgs
from sglang.srt.utils import is_cuda
@@ -519,7 +519,7 @@ class TestProgramsResolveBeforeReadingResolution(CustomTestCase):
from sglang.srt.server_args import ServerArgs as _ServerArgs
srt = pathlib.Path(next(iter(sglang.__path__))).resolve() / "srt"
declarers = {"_declare", "declare_resolution", "declare_late_resolution"}
declarers = {"declare_resolution", "declare_late_resolution"}
fields = set()
field_names = {field.name for field in _dataclasses.fields(_ServerArgs)}
# The record plus every module under `arg_groups/`: a handler declares
@@ -826,10 +826,10 @@ class TestACopyStaysResolved(_RestoresProcessState, CustomTestCase):
def test_the_copy_carries_what_resolution_left_on_the_record(self):
"""Not just the stash and the flag.
`get_model_config()` memoizes on the record, and that cache is filled
`model_config_of()` memoizes on the record, and that cache is filled
during resolution. A copy that is marked resolved but arrives without it
cannot fill it -- the read-only guard refuses the cache write -- so the
first `get_model_config()` raises. That is what killed the Ray
first `model_config_of()` raises. That is what killed the Ray
schedulers, and it is why the carry is enumerated from the instance
rather than from a list of names.
"""
@@ -846,7 +846,7 @@ class TestACopyStaysResolved(_RestoresProcessState, CustomTestCase):
[],
f"the copy did not carry what resolution left on the record: {missing}",
)
self.assertIsNotNone(copy_.get_model_config())
self.assertIsNotNone(model_config_of(copy_))
# Containers are copied, so the copy's declaration stays with it.
self.assertEqual(
len(parent._resolved_overrides) + 1, len(copy_._resolved_overrides)
test/registered/unit/server_args/test_resolution_reads_the_declarations.py
+3 -4
View File
@@ -4,8 +4,8 @@
nothing. The fields keep what the caller passed, so a resolver that reads a
field another resolver may have decided reads the raw input -- silently, and
only on the configurations where that other resolver fires. The whole pipeline
therefore reads through `resolving_view` (or `ServerArgs._resolved()`, which is
the same view spelled as the record's own member), and this pins that there is
therefore reads through `resolving_view` (or `resolved_view`, which is
the same view after resolution has finished), and this pins that there is
nothing left reading a field directly.
Subjects: every function in `arg_groups/` that takes a config, every
@@ -83,7 +83,6 @@ def _field_reads(fn, holders):
_DECLARERS = frozenset(
{
"_declare",
"declare_resolution",
"declare_late_resolution",
"declare_direct_writes",
@@ -472,7 +471,7 @@ class TestResolutionReadsTheDeclarations(CustomTestCase):
members = _record_members()
# The floor is here to catch the scan collapsing, not to pin the
# class's size.
self.assertGreater(len(members), 40, f"only {len(members)} members were found")
self.assertGreater(len(members), 25, f"only {len(members)} members were found")
offenders = []
for name, fn in sorted(members.items()):
holders = _holders(fn) | {"self"}
test/registered/unit/server_args/test_server_args.py
+32 -26
View File
@@ -37,7 +37,10 @@ from sglang.srt.arg_groups.moe_hook import (
validate_deepep_v2_dispatch_token_budget,
validate_deepep_v2_speculative_draft,
)
from sglang.srt.arg_groups.overrides import resolution_result
from sglang.srt.arg_groups.overrides import (
cutedsl_moe_max_num_tokens,
resolution_result,
)
from sglang.srt.arg_groups.parallel_hook import (
handle_context_parallelism,
handle_data_parallelism,
@@ -825,9 +828,7 @@ class TestHiSparseDsaBackendPolicy(unittest.TestCase):
)
defaults.update(kw)
view = ResolvedView(
SimpleNamespace(
get_model_config=lambda: SimpleNamespace(hf_config=hf), **defaults
)
SimpleNamespace(_model_config=SimpleNamespace(hf_config=hf), **defaults)
)
with (
patch("sglang.srt.configs.model_config.is_deepseek_dsa", return_value=True),
@@ -845,21 +846,21 @@ class TestHiSparseDsaBackendPolicy(unittest.TestCase):
),
}
@patch("sglang.srt.server_args.is_hip", return_value=False)
@patch("sglang.srt.arg_groups.hisparse_hook._is_hip", return_value=False)
def test_hisparse_defaults_to_flashmla_sparse_on_cuda_bfloat16(self, _mock_is_hip):
resolved = self._resolve("bfloat16")
self.assertEqual(resolved["dsa_prefill_backend"], "flashmla_sparse")
self.assertEqual(resolved["dsa_decode_backend"], "flashmla_sparse")
@patch("sglang.srt.server_args.is_hip", return_value=False)
@patch("sglang.srt.arg_groups.hisparse_hook._is_hip", return_value=False)
def test_hisparse_defaults_to_flashmla_kv_on_cuda_fp8(self, _mock_is_hip):
resolved = self._resolve("fp8_e4m3")
self.assertEqual(resolved["dsa_prefill_backend"], "flashmla_kv")
self.assertEqual(resolved["dsa_decode_backend"], "flashmla_kv")
@patch("sglang.srt.server_args.is_hip", return_value=False)
@patch("sglang.srt.arg_groups.hisparse_hook._is_hip", return_value=False)
def test_hisparse_accepts_flashinfer_sparse_mla_on_cuda_fp8(self, _mock_is_hip):
"""SM120 GLM DSA resolves both DSA backends to flashinfer_sparse_mla, so
the fp8 hisparse allow-set must admit it or --enable-hisparse cannot
@@ -876,14 +877,14 @@ class TestHiSparseDsaBackendPolicy(unittest.TestCase):
validate_hisparse_dsa_backend(server_args, "dsa_prefill_backend", "prefill")
validate_hisparse_dsa_backend(server_args, "dsa_decode_backend", "decode")
@patch("sglang.srt.server_args.is_hip", return_value=True)
@patch("sglang.srt.arg_groups.hisparse_hook._is_hip", return_value=True)
def test_hisparse_defaults_to_tilelang_on_rocm(self, _mock_is_hip):
resolved = self._resolve("bfloat16")
self.assertEqual(resolved["dsa_prefill_backend"], "tilelang")
self.assertEqual(resolved["dsa_decode_backend"], "tilelang")
@patch("sglang.srt.server_args.is_hip", return_value=True)
@patch("sglang.srt.arg_groups.hisparse_hook._is_hip", return_value=True)
def test_hisparse_preserves_rocm_user_backend_and_defaults_missing_side(
self, _mock_is_hip
):
@@ -892,7 +893,7 @@ class TestHiSparseDsaBackendPolicy(unittest.TestCase):
self.assertEqual(resolved["dsa_prefill_backend"], "tilelang")
self.assertEqual(resolved["dsa_decode_backend"], "tilelang")
@patch("sglang.srt.server_args.is_hip", return_value=True)
@patch("sglang.srt.arg_groups.hisparse_hook._is_hip", return_value=True)
def test_hisparse_accepts_aiter_backend_on_rocm(self, _mock_is_hip):
server_args = ServerArgs(
model_path="dummy",
@@ -905,7 +906,7 @@ class TestHiSparseDsaBackendPolicy(unittest.TestCase):
validate_hisparse_dsa_backend(server_args, "dsa_prefill_backend", "prefill")
validate_hisparse_dsa_backend(server_args, "dsa_decode_backend", "decode")
@patch("sglang.srt.server_args.is_hip", return_value=True)
@patch("sglang.srt.arg_groups.hisparse_hook._is_hip", return_value=True)
def test_hisparse_rejects_cuda_backend_on_rocm(self, _mock_is_hip):
server_args = ServerArgs(
model_path="dummy",
@@ -917,7 +918,7 @@ class TestHiSparseDsaBackendPolicy(unittest.TestCase):
with self.assertRaisesRegex(ValueError, "tilelang"):
validate_hisparse_dsa_backend(server_args, "dsa_prefill_backend", "prefill")
@patch("sglang.srt.server_args.is_hip", return_value=False)
@patch("sglang.srt.arg_groups.hisparse_hook._is_hip", return_value=False)
def test_hisparse_rejects_rocm_backend_on_cuda(self, _mock_is_hip):
server_args = ServerArgs(
model_path="dummy",
@@ -969,7 +970,7 @@ class TestFa4PageSizeAutoForce(CustomTestCase):
args.prefill_attention_backend = prefill
args.decode_attention_backend = decode
args.page_size = page_size
# Short-circuit get_model_config(): the fa4 page_size branch only needs
# Short-circuit model_config_of(): the fa4 page_size branch only needs
# use_mla_backend() (mocked) and is_sm100_supported() (mocked), not a
# real model_config. Pre-set the attribute so get_model_config returns
# early without touching ModelConfig.from_server_args.
@@ -978,7 +979,7 @@ class TestFa4PageSizeAutoForce(CustomTestCase):
return args
@patch("sglang.srt.arg_groups.overrides.is_sm100_supported", return_value=True)
@patch("sglang.srt.server_args.ServerArgs.use_mla_backend", return_value=False)
@patch("sglang.srt.arg_groups.overrides.use_mla_backend", return_value=False)
def test_combined_attention_backend_fa4_forces_page_size_128(
self, _mock_mla, _mock_sm100
):
@@ -993,7 +994,7 @@ class TestFa4PageSizeAutoForce(CustomTestCase):
self.assertEqual(resolved_view(args).page_size, 128)
@patch("sglang.srt.arg_groups.overrides.is_sm100_supported", return_value=True)
@patch("sglang.srt.server_args.ServerArgs.use_mla_backend", return_value=False)
@patch("sglang.srt.arg_groups.overrides.use_mla_backend", return_value=False)
def test_explicit_prefill_fa4_forces_page_size_128(self, _mock_mla, _mock_sm100):
# `--prefill-attention-backend fa4`: the previously-covered path.
args = self._make_args(attention_backend=None, prefill="fa4", page_size=1)
@@ -1678,7 +1679,7 @@ class TestAdaptiveSpecArgs(CustomTestCase):
args.speculative_adaptive = True
args.speculative_adaptive_config = f.name
args.device = "cuda"
args.get_model_config = lambda: SimpleNamespace(
args._model_config = SimpleNamespace(
hf_config=SimpleNamespace(
architectures=["LlamaForCausalLM"],
get_text_config=lambda: SimpleNamespace(),
@@ -1870,7 +1871,9 @@ class TestCudaGraphDisaggregationRoles(CustomTestCase):
)
with (
patch("sglang.srt.utils.is_cuda", return_value=True),
patch.object(ServerArgs, "use_mla_backend", return_value=False),
patch(
"sglang.srt.arg_groups.overrides.use_mla_backend", return_value=False
),
):
handle_cuda_graph_config(args)
return args
@@ -1943,7 +1946,9 @@ class TestPrefillCudaGraphLoRACompatibility(CustomTestCase):
)
with (
patch("sglang.srt.utils.is_cuda", return_value=True),
patch.object(ServerArgs, "use_mla_backend", return_value=False),
patch(
"sglang.srt.arg_groups.overrides.use_mla_backend", return_value=False
),
):
handle_cuda_graph_config(args)
return args
@@ -2007,7 +2012,9 @@ class TestBreakableCudaGraphMultimodalAllowlist(CustomTestCase):
)
with (
patch("sglang.srt.utils.is_cuda", return_value=True),
patch.object(ServerArgs, "use_mla_backend", return_value=False),
patch(
"sglang.srt.arg_groups.overrides.use_mla_backend", return_value=False
),
):
handle_cuda_graph_config(args)
return args
@@ -2096,7 +2103,7 @@ class TestCutedslMoeMaxNumTokens(CustomTestCase):
return server_args
def test_prefill_dominates_in_default_config(self):
self.assertEqual(self._args().cutedsl_moe_max_num_tokens(), 16384)
self.assertEqual(cutedsl_moe_max_num_tokens(self._args()), 16384)
def test_speculative_decoding_scales_decode_bound(self):
# decode bound 512 * 8 dominates the small prefill/piecewise bounds
@@ -2106,7 +2113,7 @@ class TestCutedslMoeMaxNumTokens(CustomTestCase):
speculative_algorithm="EAGLE",
speculative_num_draft_tokens=8,
)
self.assertEqual(args.cutedsl_moe_max_num_tokens(), 4096)
self.assertEqual(cutedsl_moe_max_num_tokens(args), 4096)
def test_piecewise_bound_excluded_when_disabled(self):
args = self._args(
@@ -2114,7 +2121,7 @@ class TestCutedslMoeMaxNumTokens(CustomTestCase):
disable_piecewise_cuda_graph=True,
cuda_graph_max_bs=64,
)
self.assertEqual(args.cutedsl_moe_max_num_tokens(), 512)
self.assertEqual(cutedsl_moe_max_num_tokens(args), 512)
class TestSamplingBackendTokenOracleEnvGate(CustomTestCase):
@@ -2466,10 +2473,9 @@ class TestDeepEPv2Args(CustomTestCase):
dp_size=8,
enable_dp_attention=True,
)
with patch.object(
ServerArgs,
"max_speculative_num_draft_tokens",
new=property(lambda _self: 16),
with patch(
"sglang.srt.arg_groups.moe_hook.max_speculative_num_draft_tokens",
return_value=16,
):
with envs.SGLANG_DEEPEP_V2_NUM_MAX_DISPATCH_TOKENS_PER_RANK.override(128):
with self.assertRaisesRegex(ValueError, "tokens/request=16"):
test/registered/unit/spec/test_spec_cpu_overlap_constraint.py
+1 -1
View File
@@ -20,7 +20,7 @@ def _make_spec_args(device: str, algorithm: str = "EAGLE", **overrides) -> Serve
args.speculative_num_steps = 3
args.speculative_eagle_topk = 1
args.speculative_num_draft_tokens = 4
args.get_model_config = lambda: SimpleNamespace(
args._model_config = SimpleNamespace(
hf_config=SimpleNamespace(
architectures=["LlamaForCausalLM"],
get_text_config=lambda: SimpleNamespace(),
test/registered/unit/test_chain_read_ratchet.py
+18 -11
View File
@@ -40,7 +40,7 @@ _OWNERS = ("server_args.py", "runtime_context.py", "arg_groups/")
# startup default wherever it is written, and `benchmark/` ships too.
_READS_SCANNED = _PACKAGE
_DECLARERS = ("_declare", "declare_resolution", "declare_late_resolution")
_DECLARERS = ("declare_resolution", "declare_late_resolution")
def _declared_by_keyword():
@@ -190,10 +190,10 @@ def _declared_by_registry_and_passes():
def _declared_by_late_resolution():
"""Keywords of `self._late_resolution(...)`, the fourth declarer spelling.
"""Keywords of `declare_late_resolution(record, ...)`, the late spelling.
It forwards `**fields` to `declare_late_resolution`, so the keywords sit at
its call sites and a scan for the declarer's own name finds none of them.
The fields sit at the call sites rather than in the declarer, so a scan
that only knew the declarer's own definition would find none of them.
"""
# The record plus `arg_groups/`: a hook calls it on the record it was
# handed, so scanning the record's file alone finds nothing.
@@ -203,8 +203,8 @@ def _declared_by_late_resolution():
for node in ast.walk(ast.parse(source.read_text(encoding="utf-8-sig"))):
if (
isinstance(node, ast.Call)
and isinstance(node.func, ast.Attribute)
and node.func.attr == "_late_resolution"
and isinstance(node.func, ast.Name)
and node.func.id == "declare_late_resolution"
):
fields |= {keyword.arg for keyword in node.keywords if keyword.arg}
return fields
@@ -540,13 +540,20 @@ class TestNoChainReadsOfResolvedConfig(CustomTestCase):
len(by_late),
3,
f"only {len(by_late)} fields are declared late; the "
"`_late_resolution` keyword scan broke",
"`declare_late_resolution` keyword scan broke",
)
# The three mechanisms are not the same set: if any became a subset of
# the keyword scan, that scan would be doing all the work and a
# regression in the others would be invisible.
# The data channel is not the keyword scan's subset: if it became one,
# that scan would be doing all the work and a regression here would be
# invisible. The late channel *is* a subset, and deliberately so --
# `declare_late_resolution` is a keyword declarer like the others now
# that the record hosts no forwarding member, so its own floor above is
# what pins it.
self.assertTrue(by_data - by_keyword, "the data channel adds nothing")
self.assertTrue(by_late - by_keyword, "late resolution adds nothing")
self.assertTrue(
by_late <= by_keyword,
"late resolution declares outside the keyword channel; it is the "
"same spelling, so the two cannot disagree",
)
def test_nothing_reads_a_resolved_field_off_a_borrowed_record(self):
found = _chain_reads(_resolution_written())
test/registered/unit/test_model_overrides.py
+49 -74
View File
@@ -15,6 +15,7 @@ from types import SimpleNamespace
from typing import Optional
from unittest.mock import patch
from sglang.srt.arg_groups import attention_hook
from sglang.srt.arg_groups import overrides as overrides_module
from sglang.srt.arg_groups.arg_utils import A, Arg, resolvable_fields
from sglang.srt.arg_groups.overrides import (
@@ -24,6 +25,7 @@ from sglang.srt.arg_groups.overrides import (
validate_declarations,
)
from sglang.srt.configs.minicpm import MiniCPMHybridConfig
from sglang.srt.configs.model_config import AttentionArch
from sglang.srt.environ import envs
from sglang.srt.runtime_context import (
get_context,
@@ -357,14 +359,6 @@ class TestGoldenModelOverrides(_IsolatedPublish):
enable_dp_attention=enable_dp_attention,
enable_hierarchical_cache=enable_hierarchical_cache,
)
args.is_attention_backend_not_set = lambda: all(
backend is None
for backend in (
args.attention_backend,
args.prefill_attention_backend,
args.decode_attention_backend,
)
)
mixer_types = []
if sparse_attention:
mixer_types.append("minicpm4")
@@ -456,7 +450,6 @@ class TestGoldenModelOverrides(_IsolatedPublish):
disaggregation_mode="null",
enable_dp_attention=False,
enable_hierarchical_cache=False,
is_attention_backend_not_set=lambda: True,
)
config = SimpleNamespace(
has_minicpm_sparse_attention=True,
@@ -630,7 +623,11 @@ class TestGoldenModelOverrides(_IsolatedPublish):
def test_minimax_m2_sm10x_nvfp4_uses_routed_trtllm(self):
"""MiniMax-M2 NVFP4 auto must avoid the unsupported plain TRT-LLM path."""
with patch.object(overrides_module, "is_sm100_supported", return_value=True):
# Every module that asks: the attention handler validates what the
# override family picks, and each holds its own import.
with patch.object(
overrides_module, "is_sm100_supported", return_value=True
), patch.object(attention_hook, "is_sm100_supported", return_value=True):
explicit = self._construct(
"MiniMaxM2ForCausalLM",
"llama",
@@ -765,8 +762,7 @@ class TestGoldenModelOverrides(_IsolatedPublish):
speculative_draft_attention_backend=None,
page_size=None,
mamba_radix_cache_strategy="auto",
is_attention_backend_not_set=lambda: True,
get_model_config=lambda: model_config,
_model_config=model_config,
),
hf_config,
)
@@ -971,7 +967,6 @@ class TestGoldenModelOverrides(_IsolatedPublish):
server_args.speculative_algorithm = "DFLASH"
server_args.prefill_attention_backend = "triton"
server_args.speculative_draft_attention_backend = "fa3"
server_args.is_attention_backend_not_set = lambda: False
with (
patch.object(overrides_module, "is_blackwell_supported", return_value=True),
@@ -1093,7 +1088,9 @@ class TestGoldenModelOverrides(_IsolatedPublish):
_gpt_oss_overrides(
SimpleNamespace(
dtype="float16",
is_attention_backend_not_set=lambda: False,
attention_backend="triton",
prefill_attention_backend=None,
decode_attention_backend=None,
),
SimpleNamespace(architectures=["GptOssForCausalLM"]),
)
@@ -1313,7 +1310,7 @@ class TestGoldenModelOverrides(_IsolatedPublish):
# dual-chunk config: mismatched explicit backend raises verbatim
def _mc(dual):
return SimpleNamespace(
get_model_config=lambda: SimpleNamespace(
_model_config=SimpleNamespace(
hf_config=SimpleNamespace(dual_chunk_attention_config=dual)
),
attention_backend="fa3",
@@ -1473,9 +1470,7 @@ class TestGoldenModelOverrides(_IsolatedPublish):
swa_full_tokens_ratio=ServerArgs.swa_full_tokens_ratio,
moe_a2a_backend="none",
moe_runner_backend="auto",
get_model_config=lambda: SimpleNamespace(
is_fp4_experts=True, nvfp4_moe_meta=None
),
_model_config=SimpleNamespace(is_fp4_experts=True, nvfp4_moe_meta=None),
)
defaults.update(kw)
return SimpleNamespace(**defaults)
@@ -1527,12 +1522,10 @@ class TestGoldenModelOverrides(_IsolatedPublish):
)
# FP8 checkpoints and non-CUDA platforms keep their platform-specific
# auto-resolution paths.
fp8_model_config = lambda: SimpleNamespace(
is_fp4_experts=False, nvfp4_moe_meta=None
)
fp8_model_config = SimpleNamespace(is_fp4_experts=False, nvfp4_moe_meta=None)
self.assertNotIn(
"moe_runner_backend",
_deepseek_v4_overrides(_args(get_model_config=fp8_model_config), hf),
_deepseek_v4_overrides(_args(_model_config=fp8_model_config), hf),
)
self.assertNotIn(
"moe_runner_backend",
@@ -1569,7 +1562,7 @@ class TestGoldenModelOverrides(_IsolatedPublish):
self.assertEqual(
_deepseek_v4_overrides(
_args(
get_model_config=lambda: SimpleNamespace(
_model_config=SimpleNamespace(
is_fp4_experts=False, nvfp4_moe_meta=object()
)
),
@@ -1604,15 +1597,10 @@ class TestGoldenModelOverrides(_IsolatedPublish):
speculative_draft_attention_backend=None,
page_size=None,
mamba_radix_cache_strategy="auto",
get_model_config=lambda: mc,
_model_config=mc,
)
defaults.update(kw)
args = SimpleNamespace(**defaults)
args.is_attention_backend_not_set = lambda: (
args.attention_backend is None
and args.prefill_attention_backend is None
and args.decode_attention_backend is None
)
return args
hf = _hf()
@@ -1725,9 +1713,7 @@ class TestGoldenModelOverrides(_IsolatedPublish):
)
defaults.update(kw)
return ResolvedView(
SimpleNamespace(
get_model_config=lambda: SimpleNamespace(hf_config=hf), **defaults
)
SimpleNamespace(_model_config=SimpleNamespace(hf_config=hf), **defaults)
)
with (
@@ -1821,9 +1807,7 @@ class TestGoldenModelOverrides(_IsolatedPublish):
)
defaults.update(kw)
return ResolvedView(
SimpleNamespace(
get_model_config=lambda: SimpleNamespace(hf_config=hf), **defaults
)
SimpleNamespace(_model_config=SimpleNamespace(hf_config=hf), **defaults)
)
with (
@@ -1952,15 +1936,6 @@ class TestGoldenModelOverrides(_IsolatedPublish):
)
defaults.update(kw)
ns = SimpleNamespace(**defaults)
ns.is_attention_backend_not_set = lambda: (
ns.attention_backend is None
and ns.prefill_attention_backend is None
and ns.decode_attention_backend is None
)
ns.get_attention_backends = lambda: (
ns.prefill_attention_backend or ns.attention_backend,
ns.decode_attention_backend or ns.attention_backend,
)
return ns
# nothing set: prefill defaults to flashinfer
@@ -1991,9 +1966,7 @@ class TestGoldenModelOverrides(_IsolatedPublish):
)
defaults.update(kw)
return ResolvedView(
SimpleNamespace(
get_model_config=lambda: SimpleNamespace(hf_config=hf), **defaults
)
SimpleNamespace(_model_config=SimpleNamespace(hf_config=hf), **defaults)
)
with (
@@ -2037,9 +2010,7 @@ class TestGoldenModelOverrides(_IsolatedPublish):
defaults = dict(kv_cache_dtype="auto", device="cuda")
defaults.update(kw)
return ResolvedView(
SimpleNamespace(
get_model_config=lambda: SimpleNamespace(hf_config=hf), **defaults
)
SimpleNamespace(_model_config=SimpleNamespace(hf_config=hf), **defaults)
)
self.assertEqual(
@@ -2078,9 +2049,7 @@ class TestGoldenModelOverrides(_IsolatedPublish):
)
defaults.update(kw)
return ResolvedView(
SimpleNamespace(
get_model_config=lambda: SimpleNamespace(hf_config=hf), **defaults
)
SimpleNamespace(_model_config=SimpleNamespace(hf_config=hf), **defaults)
)
with patch.object(overrides_module, "is_hip", return_value=True):
@@ -2148,9 +2117,7 @@ class TestGoldenModelOverrides(_IsolatedPublish):
)
defaults.update(kw)
return ResolvedView(
SimpleNamespace(
get_model_config=lambda: SimpleNamespace(hf_config=hf), **defaults
)
SimpleNamespace(_model_config=SimpleNamespace(hf_config=hf), **defaults)
)
# arch guard: non-mamba arch declares nothing
@@ -2255,17 +2222,28 @@ class TestGoldenModelOverrides(_IsolatedPublish):
def _args(default_backend, **kw):
defaults = dict(
attention_backend=None,
_get_default_attn_backend=lambda **_: default_backend,
use_mla_backend=lambda: False,
get_model_config=lambda: None,
prefill_attention_backend=None,
decode_attention_backend=None,
mamba_radix_cache_strategy="auto",
disable_radix_cache=False,
speculative_algorithm=None,
)
defaults.update(kw)
return SimpleNamespace(**defaults)
args = SimpleNamespace(**defaults)
args.default_backend_for_test = default_backend
return args
with patch.object(overrides_module, "is_sm100_supported", return_value=True):
with patch.object(
overrides_module, "is_sm100_supported", return_value=True
), patch.object(
overrides_module,
"get_default_attn_backend",
lambda server_args, **_: server_args.default_backend_for_test,
), patch.object(
overrides_module, "use_mla_backend", return_value=False
), patch.object(
overrides_module, "model_config_of", return_value=None
):
# radix on + no extra buffer + no spec -> page_size=1 path
self.assertEqual(
_qwen3_5_hybrid_overrides(_args("trtllm_mha"), None),
@@ -2422,11 +2400,6 @@ class TestGoldenModelOverrides(_IsolatedPublish):
)
defaults.update(kw)
ns = SimpleNamespace(**defaults)
ns.is_attention_backend_not_set = lambda: (
ns.attention_backend is None
and ns.prefill_attention_backend is None
and ns.decode_attention_backend is None
)
return ns
hf = SimpleNamespace()
@@ -2475,6 +2448,9 @@ class TestGoldenModelOverrides(_IsolatedPublish):
prefill_attention_backend=None,
speculative_draft_attention_backend=None,
page_size=1,
# `use_mla_backend` reads the model configuration; a non-MLA
# one keeps these assertions about the page constraints.
_model_config=SimpleNamespace(attention_arch=None),
)
defaults.update(kw)
return ResolvedView(SimpleNamespace(**defaults))
@@ -2533,7 +2509,6 @@ class TestGoldenModelOverrides(_IsolatedPublish):
_fa4_page_constraint(
_view(
attention_backend="fa4",
use_mla_backend=lambda: False,
speculative_eagle_topk=None,
)
),
@@ -2543,7 +2518,6 @@ class TestGoldenModelOverrides(_IsolatedPublish):
_fa4_page_constraint(
_view(
attention_backend="fa4",
use_mla_backend=lambda: False,
speculative_eagle_topk=2, # EAGLE topk>1 keeps default
)
),
@@ -2554,7 +2528,6 @@ class TestGoldenModelOverrides(_IsolatedPublish):
_intel_xpu_page_constraint(
_view(
decode_attention_backend="intel_xpu",
use_mla_backend=lambda: False,
)
),
{"page_size": 128},
@@ -2563,7 +2536,7 @@ class TestGoldenModelOverrides(_IsolatedPublish):
_intel_xpu_page_constraint(
_view(
decode_attention_backend="intel_xpu",
use_mla_backend=lambda: True,
_model_config=SimpleNamespace(attention_arch=AttentionArch.MLA),
page_size=16, # MLA decode accepts 16
)
),
@@ -2585,7 +2558,8 @@ class TestGoldenModelOverrides(_IsolatedPublish):
defaults = dict(
device="cuda",
attention_backend=None,
is_attention_backend_not_set=lambda: True,
prefill_attention_backend=None,
decode_attention_backend=None,
# keep the (now-absorbed) quant/moe blocks inert so these
# assertions stay attention-only
moe_runner_backend="triton",
@@ -2674,7 +2648,7 @@ class TestGoldenModelOverrides(_IsolatedPublish):
_quantization_explicitly_unset=False,
moe_a2a_backend="none",
moe_runner_backend="auto",
get_model_config=lambda: SimpleNamespace(
_model_config=SimpleNamespace(
hf_config=SimpleNamespace(
architectures=[arch], quantization_config=quant_cfg
)
@@ -2742,7 +2716,6 @@ class TestGoldenModelOverrides(_IsolatedPublish):
def _args(**kw):
defaults = dict(
is_attention_backend_not_set=lambda: True,
attention_backend=None,
prefill_attention_backend=None,
decode_attention_backend=None,
@@ -2869,7 +2842,9 @@ class TestGoldenModelOverrides(_IsolatedPublish):
defaults = dict(
speculative_algorithm=None,
enable_hierarchical_cache=False,
is_attention_backend_not_set=lambda: False,
attention_backend="triton",
prefill_attention_backend=None,
decode_attention_backend=None,
)
defaults.update(kw)
return SimpleNamespace(**defaults)
test/registered/unit/test_runtime_context.py
+15 -3
View File
@@ -16,6 +16,18 @@ from unittest.mock import patch
import sglang as _sglang
import sglang.srt.server_args as server_args_module
from sglang.srt.arg_groups.arg_utils import NS, A, Arg
from sglang.srt.arg_groups.overrides import (
attention_backends_of,
)
from sglang.srt.arg_groups.overrides import (
mamba_cache_chunk_size as mamba_cache_chunk_size_of,
)
from sglang.srt.arg_groups.overrides import (
max_prefill_buffer_tokens as max_prefill_buffer_tokens_of,
)
from sglang.srt.arg_groups.overrides import (
resolved_view,
)
from sglang.srt.runtime_context import (
Flags,
ParallelContext,
@@ -404,7 +416,7 @@ class TestServerArgsScopedOverride(_IsolatedServerArgs):
published = (
get_context().override_server_args(_mamba_cache_chunk_size=64).install()
)
self.assertEqual(published.mamba_cache_chunk_size, 64)
self.assertEqual(mamba_cache_chunk_size_of(published), 64)
def test_installed_config_arms_the_strict_guard(self):
# The published dummy must behave like a resolved config: bare writes
@@ -1156,7 +1168,7 @@ class TestDerivedPredicatesAgreeAcrossTiers(_IsolatedServerArgs):
)
get_context().set_server_args(args)
self.assertEqual(
ServerArgs.max_prefill_buffer_tokens(args),
max_prefill_buffer_tokens_of(args),
max_prefill_buffer_tokens(),
)
@@ -1243,7 +1255,7 @@ class TestDerivedPredicatesAgreeAcrossTiers(_IsolatedServerArgs):
)
get_context().set_server_args(args)
self.assertEqual(
ServerArgs.get_attention_backends(args),
attention_backends_of(resolved_view(args)),
attention_backends(),
)
test/registered/unit/test_split_attention_backend_decisions.py
+2 -1
View File
@@ -34,6 +34,7 @@ from sglang.test.test_utils import CustomTestCase
register_cpu_ci(est_time=2, suite="base-a-test-cpu")
import sglang
from sglang.srt.arg_groups.overrides import attention_backends_of, resolved_view
_PACKAGE_ROOT = Path(next(iter(sglang.__path__))) / "srt"
@@ -181,7 +182,7 @@ class TestSplitBackendsReachTheDecisions(CustomTestCase):
("decode_attention_backend", "flashinfer"),
):
object.__setattr__(args, name, value)
self.assertIn("flashinfer", args.get_attention_backends())
self.assertIn("flashinfer", attention_backends_of(resolved_view(args)))
def test_support_triton_is_the_regression_being_guarded(self):
from sglang.srt.utils.common import support_triton
test/registered/unit/test_supplied_instance_exposure_ratchet.py
+3 -3
View File
@@ -483,8 +483,8 @@ class TestSuppliedInstanceExposure(CustomTestCase):
tgts = [node.target]
elif (
isinstance(node, ast.Call)
and isinstance(node.func, ast.Attribute)
and node.func.attr == "_declare"
and isinstance(node.func, ast.Name)
and node.func.id == "declare_resolution"
):
targets |= {
kw.arg
@@ -622,7 +622,7 @@ class TestSuppliedInstanceExposure(CustomTestCase):
for path in sorted(root.rglob("*.py")):
rel = path.relative_to(root).as_posix()
source = path.read_text(encoding="utf-8-sig")
if "_late_resolution" not in source:
if "declare_late_resolution" not in source:
continue
try:
tree = ast.parse(source)