minke.yu/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 13 Packages Projects Releases Wiki Activity

config: the resolution callbacks into the record go to zero (#36972)

Browse Source 
Co-authored-by: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
Cheng Wan
2026-08-29 04:18:05 -07:00
committed by GitHub
co-authored by Claude Opus 5
parent 48b88e1256
commit b65e677e48
50 changed files with 1087 additions and 979 deletions
Download Patch File Download Diff File Expand all files Collapse all files
test/registered/unit/server_args/test_model_config_cache.py
+11 -9
View File
@@ -1,4 +1,4 @@
"""`get_model_config()` caches, and the key is the path the record carried.
"""`model_config_of()` caches, and the key is the path the record carried.
Two movements of a `model_path` reach this cache, and only the first one means
the cached configuration describes the wrong checkpoint:
@@ -20,6 +20,7 @@ import tempfile
import unittest
from types import SimpleNamespace
from sglang.srt.arg_groups.overrides import declare_resolution, model_config_of
from sglang.srt.configs.model_config import ModelConfig
from sglang.srt.environ import EnvField, envs
from sglang.srt.server_args import ServerArgs
@@ -117,7 +118,7 @@ class TestTheModelConfigCache(CustomTestCase):
self.assertEqual(server_args.model_path, _OBJECT_STORE_URI)
self.assertEqual(cached.model_path, pulled)
self.assertIs(server_args.get_model_config(), cached)
self.assertIs(model_config_of(server_args), cached)
def test_a_declared_model_path_rebuilds_the_configuration(self):
"""The GGUF and ModelScope shape: the record's own path moved."""
@@ -125,14 +126,15 @@ class TestTheModelConfigCache(CustomTestCase):
second_checkpoint = self._checkpoint()
server_args = ServerArgs(model_path=first_checkpoint, device="cuda")
first = server_args.get_model_config()
first = model_config_of(server_args)
self.assertEqual(first.model_path, first_checkpoint)
server_args._declare(
declare_resolution(
server_args,
"test_a_declared_model_path_rebuilds_the_configuration",
model_path=second_checkpoint,
)
second = server_args.get_model_config()
second = model_config_of(server_args)
self.assertIsNot(second, first)
self.assertEqual(second.model_path, second_checkpoint)
@@ -151,11 +153,11 @@ class TestTheModelConfigCache(CustomTestCase):
model_path=second_checkpoint,
)
rebuilt = copy_.get_model_config()
rebuilt = model_config_of(copy_)
self.assertEqual(rebuilt.model_path, second_checkpoint)
self.assertIs(copy_.get_model_config(), rebuilt)
self.assertIs(model_config_of(copy_), rebuilt)
# The parent keeps the configuration it resolved with.
self.assertEqual(server_args.get_model_config().model_path, first_checkpoint)
self.assertEqual(model_config_of(server_args).model_path, first_checkpoint)
def test_a_supplied_configuration_is_handed_back(self):
"""A configuration nothing in here built carries no key, so nothing
@@ -164,7 +166,7 @@ class TestTheModelConfigCache(CustomTestCase):
stand_in = SimpleNamespace(model_path="somewhere/else")
server_args._model_config = stand_in
self.assertIs(server_args.get_model_config(), stand_in)
self.assertIs(model_config_of(server_args), stand_in)
if __name__ == "__main__":
test/registered/unit/server_args/test_model_config_reads_resolved_input.py
+14 -13
View File
@@ -30,7 +30,7 @@ _SRT = pathlib.Path(sglang.__file__).resolve().parent / "srt"
# Two quantities sharing one name.
_READ_BEFORE_RESOLUTION = frozenset({"is_embedding"})
# Declared after the first `get_model_config()`, so the cached configuration
# Declared after the first `model_config_of()`, so the cached configuration
# holds the earlier value. Nothing reads the stale copy today (its one consumer
# is on the `is_draft_model` branch, built after resolution), and fixing it
# means moving the build or the hook. Pinned so a second field in this position
@@ -109,7 +109,7 @@ def _registry_collection_is_after_the_build():
Handler-local ordering only -- the caller still has to compare against the
pipeline-wide first build, which sits in an *earlier* step: hoisting the
collection above this handler's own `get_model_config()` call does not move
collection above this handler's own `model_config_of()` call does not move
it above the configuration another handler already cached.
"""
handler = None
@@ -145,7 +145,7 @@ def _registry_collection_is_after_the_build():
name = func.id
else:
continue
if name == "get_model_config" and build is None:
if name == "model_config_of" and build is None:
build = node.lineno
if name == "collect_model_override_declarations" and collect is None:
collect = node.lineno
@@ -192,11 +192,12 @@ def _server_args_names(tree, path):
and value.args[0].id in names
)
# `resolved = self._resolved()` is the same view, spelled as the
# record's own member.
# resolution vocabulary.
member = (
isinstance(value, ast.Call)
and isinstance(value.func, ast.Attribute)
and value.func.attr == "_resolved"
and isinstance(value.func, ast.Name)
and value.func.id == "resolved_view"
and isinstance(value.func.value, ast.Name)
and value.func.value.id in names
)
@@ -267,7 +268,7 @@ def _late_resolution_fields():
if isinstance(node.func, ast.Attribute)
else getattr(node.func, "id", "")
)
if called in ("_late_resolution", "declare_late_resolution"):
if called == "declare_late_resolution":
fields |= {kw.arg for kw in node.keywords if kw.arg}
return fields
@@ -488,14 +489,14 @@ def _declaration_positions():
wanted = _constructor_reads()
def build_site():
"""(step index, method name, line) of the first `get_model_config()`."""
"""(step index, method name, line) of the first `model_config_of()`."""
for index, step in enumerate(steps):
for method in reached[step]:
for node in ast.walk(methods[method]):
if (
isinstance(node, ast.Call)
and isinstance(node.func, ast.Attribute)
and node.func.attr == "get_model_config"
and isinstance(node.func, ast.Name)
and node.func.id == "model_config_of"
):
return index, step, method, node.lineno
return None
@@ -532,8 +533,8 @@ def _declaration_positions():
same_body = index == build_index and method == build_method
rank = 0 if same_body and node.lineno < build_line_in_body else 1
if (
isinstance(node.func, ast.Attribute)
and node.func.attr == "_declare"
isinstance(node.func, ast.Name)
and node.func.id == "declare_resolution"
):
fields = {kw.arg for kw in node.keywords if kw.arg}
# A handler that calls an imported hook (the Kimi and DeepSeek
@@ -669,8 +670,8 @@ class TestModelConfigReadsResolvedInput(CustomTestCase):
for method in reached[step]
if any(
isinstance(node, ast.Call)
and isinstance(node.func, ast.Attribute)
and node.func.attr == "get_model_config"
and isinstance(node.func, ast.Name)
and node.func.id == "model_config_of"
for node in ast.walk(methods[method])
)
)
test/registered/unit/server_args/test_no_public_non_field_slot.py
+1 -1
View File
@@ -61,7 +61,7 @@ class TestNoPublicNonFieldSlot(CustomTestCase):
written = _self_written_attributes()
self.assertGreater(
len(written),
5,
3,
f"only {len(written)} self-writes found; the scan is broken, not the "
"record",
)
test/registered/unit/server_args/test_page_major_backend_allowlist.py
+13 -7
View File
@@ -30,6 +30,7 @@ under its own default configuration.
"""
import unittest
from unittest import mock
from sglang.srt.arg_groups.kv_cache_hook import handle_page_major_kv_layout
from sglang.srt.server_args import ServerArgs
@@ -66,13 +67,18 @@ def _accepts(
"mamba_backend": "triton",
}.items():
object.__setattr__(sa, name, value)
sa.use_mla_backend = lambda: use_mla
sa._resolved_attention_backends = lambda: [backend]
try:
handle_page_major_kv_layout(sa)
return True
except AssertionError:
return False
# `use_mla_backend` asks the model configuration, which this stand-in has
# no room for; the case under test is what the handler does with the answer.
# The handler imports it inside the function, so the source module is
# where the patch has to go.
with mock.patch(
"sglang.srt.arg_groups.overrides.use_mla_backend", return_value=use_mla
):
try:
handle_page_major_kv_layout(sa)
return True
except AssertionError:
return False
class TestPageMajorBackendAllowlist(unittest.TestCase):
test/registered/unit/server_args/test_resolution_declarations.py
+4 -3
View File
@@ -24,7 +24,6 @@ import unittest
import unittest.mock
import sglang
from sglang.srt import server_args as server_args_module
from sglang.srt.arg_groups.overrides import resolution_result
from sglang.srt.server_args import ServerArgs
from sglang.test.ci.ci_register import register_cpu_ci
@@ -147,7 +146,7 @@ def _late_resolvers():
if isinstance(node.func, ast.Attribute)
else getattr(node.func, "id", None)
)
if called in ("declare_late_resolution", "_late_resolution"):
if called == "declare_late_resolution":
return True
if called and reaches(called, seen):
return True
@@ -887,7 +886,9 @@ class TestResolutionDeclarations(CustomTestCase):
# The pipeline asks the platform other questions on the way through
# (whether it is out of tree, whether it supports piecewise capture),
# and which of those it reaches depends on the host.
class _Plugin(type(server_args_module.current_platform)):
from sglang.srt.platforms import current_platform
class _Plugin(type(current_platform)):
device_name = "oot"
def apply_server_args_defaults(self, server_args):
test/registered/unit/server_args/test_resolution_is_reproducible.py
+5 -5
View File
@@ -36,7 +36,7 @@ import unittest.mock
import torch
import sglang
from sglang.srt.arg_groups.overrides import resolution_result
from sglang.srt.arg_groups.overrides import model_config_of, resolution_result
from sglang.srt.environ import EnvField, envs
from sglang.srt.server_args import ServerArgs
from sglang.srt.utils import is_cuda
@@ -519,7 +519,7 @@ class TestProgramsResolveBeforeReadingResolution(CustomTestCase):
from sglang.srt.server_args import ServerArgs as _ServerArgs
srt = pathlib.Path(next(iter(sglang.__path__))).resolve() / "srt"
declarers = {"_declare", "declare_resolution", "declare_late_resolution"}
declarers = {"declare_resolution", "declare_late_resolution"}
fields = set()
field_names = {field.name for field in _dataclasses.fields(_ServerArgs)}
# The record plus every module under `arg_groups/`: a handler declares
@@ -826,10 +826,10 @@ class TestACopyStaysResolved(_RestoresProcessState, CustomTestCase):
def test_the_copy_carries_what_resolution_left_on_the_record(self):
"""Not just the stash and the flag.
`get_model_config()` memoizes on the record, and that cache is filled
`model_config_of()` memoizes on the record, and that cache is filled
during resolution. A copy that is marked resolved but arrives without it
cannot fill it -- the read-only guard refuses the cache write -- so the
first `get_model_config()` raises. That is what killed the Ray
first `model_config_of()` raises. That is what killed the Ray
schedulers, and it is why the carry is enumerated from the instance
rather than from a list of names.
"""
@@ -846,7 +846,7 @@ class TestACopyStaysResolved(_RestoresProcessState, CustomTestCase):
[],
f"the copy did not carry what resolution left on the record: {missing}",
)
self.assertIsNotNone(copy_.get_model_config())
self.assertIsNotNone(model_config_of(copy_))
# Containers are copied, so the copy's declaration stays with it.
self.assertEqual(
len(parent._resolved_overrides) + 1, len(copy_._resolved_overrides)
test/registered/unit/server_args/test_resolution_reads_the_declarations.py
+3 -4
View File
@@ -4,8 +4,8 @@
nothing. The fields keep what the caller passed, so a resolver that reads a
field another resolver may have decided reads the raw input -- silently, and
only on the configurations where that other resolver fires. The whole pipeline
therefore reads through `resolving_view` (or `ServerArgs._resolved()`, which is
the same view spelled as the record's own member), and this pins that there is
therefore reads through `resolving_view` (or `resolved_view`, which is
the same view after resolution has finished), and this pins that there is
nothing left reading a field directly.
Subjects: every function in `arg_groups/` that takes a config, every
@@ -83,7 +83,6 @@ def _field_reads(fn, holders):
_DECLARERS = frozenset(
{
"_declare",
"declare_resolution",
"declare_late_resolution",
"declare_direct_writes",
@@ -472,7 +471,7 @@ class TestResolutionReadsTheDeclarations(CustomTestCase):
members = _record_members()
# The floor is here to catch the scan collapsing, not to pin the
# class's size.
self.assertGreater(len(members), 40, f"only {len(members)} members were found")
self.assertGreater(len(members), 25, f"only {len(members)} members were found")
offenders = []
for name, fn in sorted(members.items()):
holders = _holders(fn) | {"self"}
test/registered/unit/server_args/test_server_args.py
+32 -26
View File
@@ -37,7 +37,10 @@ from sglang.srt.arg_groups.moe_hook import (
validate_deepep_v2_dispatch_token_budget,
validate_deepep_v2_speculative_draft,
)
from sglang.srt.arg_groups.overrides import resolution_result
from sglang.srt.arg_groups.overrides import (
cutedsl_moe_max_num_tokens,
resolution_result,
)
from sglang.srt.arg_groups.parallel_hook import (
handle_context_parallelism,
handle_data_parallelism,
@@ -825,9 +828,7 @@ class TestHiSparseDsaBackendPolicy(unittest.TestCase):
)
defaults.update(kw)
view = ResolvedView(
SimpleNamespace(
get_model_config=lambda: SimpleNamespace(hf_config=hf), **defaults
)
SimpleNamespace(_model_config=SimpleNamespace(hf_config=hf), **defaults)
)
with (
patch("sglang.srt.configs.model_config.is_deepseek_dsa", return_value=True),
@@ -845,21 +846,21 @@ class TestHiSparseDsaBackendPolicy(unittest.TestCase):
),
}
@patch("sglang.srt.server_args.is_hip", return_value=False)
@patch("sglang.srt.arg_groups.hisparse_hook._is_hip", return_value=False)
def test_hisparse_defaults_to_flashmla_sparse_on_cuda_bfloat16(self, _mock_is_hip):
resolved = self._resolve("bfloat16")
self.assertEqual(resolved["dsa_prefill_backend"], "flashmla_sparse")
self.assertEqual(resolved["dsa_decode_backend"], "flashmla_sparse")
@patch("sglang.srt.server_args.is_hip", return_value=False)
@patch("sglang.srt.arg_groups.hisparse_hook._is_hip", return_value=False)
def test_hisparse_defaults_to_flashmla_kv_on_cuda_fp8(self, _mock_is_hip):
resolved = self._resolve("fp8_e4m3")
self.assertEqual(resolved["dsa_prefill_backend"], "flashmla_kv")
self.assertEqual(resolved["dsa_decode_backend"], "flashmla_kv")
@patch("sglang.srt.server_args.is_hip", return_value=False)
@patch("sglang.srt.arg_groups.hisparse_hook._is_hip", return_value=False)
def test_hisparse_accepts_flashinfer_sparse_mla_on_cuda_fp8(self, _mock_is_hip):
"""SM120 GLM DSA resolves both DSA backends to flashinfer_sparse_mla, so
the fp8 hisparse allow-set must admit it or --enable-hisparse cannot
@@ -876,14 +877,14 @@ class TestHiSparseDsaBackendPolicy(unittest.TestCase):
validate_hisparse_dsa_backend(server_args, "dsa_prefill_backend", "prefill")
validate_hisparse_dsa_backend(server_args, "dsa_decode_backend", "decode")
@patch("sglang.srt.server_args.is_hip", return_value=True)
@patch("sglang.srt.arg_groups.hisparse_hook._is_hip", return_value=True)
def test_hisparse_defaults_to_tilelang_on_rocm(self, _mock_is_hip):
resolved = self._resolve("bfloat16")
self.assertEqual(resolved["dsa_prefill_backend"], "tilelang")
self.assertEqual(resolved["dsa_decode_backend"], "tilelang")
@patch("sglang.srt.server_args.is_hip", return_value=True)
@patch("sglang.srt.arg_groups.hisparse_hook._is_hip", return_value=True)
def test_hisparse_preserves_rocm_user_backend_and_defaults_missing_side(
self, _mock_is_hip
):
@@ -892,7 +893,7 @@ class TestHiSparseDsaBackendPolicy(unittest.TestCase):
self.assertEqual(resolved["dsa_prefill_backend"], "tilelang")
self.assertEqual(resolved["dsa_decode_backend"], "tilelang")
@patch("sglang.srt.server_args.is_hip", return_value=True)
@patch("sglang.srt.arg_groups.hisparse_hook._is_hip", return_value=True)
def test_hisparse_accepts_aiter_backend_on_rocm(self, _mock_is_hip):
server_args = ServerArgs(
model_path="dummy",
@@ -905,7 +906,7 @@ class TestHiSparseDsaBackendPolicy(unittest.TestCase):
validate_hisparse_dsa_backend(server_args, "dsa_prefill_backend", "prefill")
validate_hisparse_dsa_backend(server_args, "dsa_decode_backend", "decode")
@patch("sglang.srt.server_args.is_hip", return_value=True)
@patch("sglang.srt.arg_groups.hisparse_hook._is_hip", return_value=True)
def test_hisparse_rejects_cuda_backend_on_rocm(self, _mock_is_hip):
server_args = ServerArgs(
model_path="dummy",
@@ -917,7 +918,7 @@ class TestHiSparseDsaBackendPolicy(unittest.TestCase):
with self.assertRaisesRegex(ValueError, "tilelang"):
validate_hisparse_dsa_backend(server_args, "dsa_prefill_backend", "prefill")
@patch("sglang.srt.server_args.is_hip", return_value=False)
@patch("sglang.srt.arg_groups.hisparse_hook._is_hip", return_value=False)
def test_hisparse_rejects_rocm_backend_on_cuda(self, _mock_is_hip):
server_args = ServerArgs(
model_path="dummy",
@@ -969,7 +970,7 @@ class TestFa4PageSizeAutoForce(CustomTestCase):
args.prefill_attention_backend = prefill
args.decode_attention_backend = decode
args.page_size = page_size
# Short-circuit get_model_config(): the fa4 page_size branch only needs
# Short-circuit model_config_of(): the fa4 page_size branch only needs
# use_mla_backend() (mocked) and is_sm100_supported() (mocked), not a
# real model_config. Pre-set the attribute so get_model_config returns
# early without touching ModelConfig.from_server_args.
@@ -978,7 +979,7 @@ class TestFa4PageSizeAutoForce(CustomTestCase):
return args
@patch("sglang.srt.arg_groups.overrides.is_sm100_supported", return_value=True)
@patch("sglang.srt.server_args.ServerArgs.use_mla_backend", return_value=False)
@patch("sglang.srt.arg_groups.overrides.use_mla_backend", return_value=False)
def test_combined_attention_backend_fa4_forces_page_size_128(
self, _mock_mla, _mock_sm100
):
@@ -993,7 +994,7 @@ class TestFa4PageSizeAutoForce(CustomTestCase):
self.assertEqual(resolved_view(args).page_size, 128)
@patch("sglang.srt.arg_groups.overrides.is_sm100_supported", return_value=True)
@patch("sglang.srt.server_args.ServerArgs.use_mla_backend", return_value=False)
@patch("sglang.srt.arg_groups.overrides.use_mla_backend", return_value=False)
def test_explicit_prefill_fa4_forces_page_size_128(self, _mock_mla, _mock_sm100):
# `--prefill-attention-backend fa4`: the previously-covered path.
args = self._make_args(attention_backend=None, prefill="fa4", page_size=1)
@@ -1678,7 +1679,7 @@ class TestAdaptiveSpecArgs(CustomTestCase):
args.speculative_adaptive = True
args.speculative_adaptive_config = f.name
args.device = "cuda"
args.get_model_config = lambda: SimpleNamespace(
args._model_config = SimpleNamespace(
hf_config=SimpleNamespace(
architectures=["LlamaForCausalLM"],
get_text_config=lambda: SimpleNamespace(),
@@ -1870,7 +1871,9 @@ class TestCudaGraphDisaggregationRoles(CustomTestCase):
)
with (
patch("sglang.srt.utils.is_cuda", return_value=True),
patch.object(ServerArgs, "use_mla_backend", return_value=False),
patch(
"sglang.srt.arg_groups.overrides.use_mla_backend", return_value=False
),
):
handle_cuda_graph_config(args)
return args
@@ -1943,7 +1946,9 @@ class TestPrefillCudaGraphLoRACompatibility(CustomTestCase):
)
with (
patch("sglang.srt.utils.is_cuda", return_value=True),
patch.object(ServerArgs, "use_mla_backend", return_value=False),
patch(
"sglang.srt.arg_groups.overrides.use_mla_backend", return_value=False
),
):
handle_cuda_graph_config(args)
return args
@@ -2007,7 +2012,9 @@ class TestBreakableCudaGraphMultimodalAllowlist(CustomTestCase):
)
with (
patch("sglang.srt.utils.is_cuda", return_value=True),
patch.object(ServerArgs, "use_mla_backend", return_value=False),
patch(
"sglang.srt.arg_groups.overrides.use_mla_backend", return_value=False
),
):
handle_cuda_graph_config(args)
return args
@@ -2096,7 +2103,7 @@ class TestCutedslMoeMaxNumTokens(CustomTestCase):
return server_args
def test_prefill_dominates_in_default_config(self):
self.assertEqual(self._args().cutedsl_moe_max_num_tokens(), 16384)
self.assertEqual(cutedsl_moe_max_num_tokens(self._args()), 16384)
def test_speculative_decoding_scales_decode_bound(self):
# decode bound 512 * 8 dominates the small prefill/piecewise bounds
@@ -2106,7 +2113,7 @@ class TestCutedslMoeMaxNumTokens(CustomTestCase):
speculative_algorithm="EAGLE",
speculative_num_draft_tokens=8,
)
self.assertEqual(args.cutedsl_moe_max_num_tokens(), 4096)
self.assertEqual(cutedsl_moe_max_num_tokens(args), 4096)
def test_piecewise_bound_excluded_when_disabled(self):
args = self._args(
@@ -2114,7 +2121,7 @@ class TestCutedslMoeMaxNumTokens(CustomTestCase):
disable_piecewise_cuda_graph=True,
cuda_graph_max_bs=64,
)
self.assertEqual(args.cutedsl_moe_max_num_tokens(), 512)
self.assertEqual(cutedsl_moe_max_num_tokens(args), 512)
class TestSamplingBackendTokenOracleEnvGate(CustomTestCase):
@@ -2466,10 +2473,9 @@ class TestDeepEPv2Args(CustomTestCase):
dp_size=8,
enable_dp_attention=True,
)
with patch.object(
ServerArgs,
"max_speculative_num_draft_tokens",
new=property(lambda _self: 16),
with patch(
"sglang.srt.arg_groups.moe_hook.max_speculative_num_draft_tokens",
return_value=16,
):
with envs.SGLANG_DEEPEP_V2_NUM_MAX_DISPATCH_TOKENS_PER_RANK.override(128):
with self.assertRaisesRegex(ValueError, "tokens/request=16"):