From 65824258549671404188450748b90ab7cacab2ae Mon Sep 17 00:00:00 2001 From: Peng Wu Date: Mon, 28 Sep 2026 02:07:35 -0700 Subject: [PATCH] Let predicate-registered linear-attention models carry the mamba radix-cache leaves (#41165) --- python/sglang/srt/arg_groups/mamba_hook.py | 6 +- python/sglang/srt/arg_groups/model_hook.py | 16 +-- python/sglang/srt/arg_groups/overrides.py | 25 ++-- .../srt/configs/linear_attn_model_registry.py | 9 ++ .../spec/test_dflash_extra_buffer_lazy.py | 4 +- test/registered/unit/test_model_overrides.py | 133 +++++++++++++++++- 6 files changed, 168 insertions(+), 25 deletions(-) diff --git a/python/sglang/srt/arg_groups/mamba_hook.py b/python/sglang/srt/arg_groups/mamba_hook.py index 425e1a9421..55fdb92ada 100644 --- a/python/sglang/srt/arg_groups/mamba_hook.py +++ b/python/sglang/srt/arg_groups/mamba_hook.py @@ -98,10 +98,10 @@ def handle_int8_mamba_checkpoint(server_args: Any): ) -def validate_mamba_extra_buffer(view, model_arch: str, *, mamba_cache_chunk_size_of): +def validate_mamba_extra_buffer(view, hf_config: Any, *, mamba_cache_chunk_size_of): - assert supports_mamba_cache_extra_buffer(view, model_arch), ( - f"extra_buffer is not supported for {model_arch}; use no_buffer." + assert supports_mamba_cache_extra_buffer(view, hf_config), ( + f"extra_buffer is not supported for {hf_config.architectures[0]}; use no_buffer." ) assert ( get_platform().is_cuda diff --git a/python/sglang/srt/arg_groups/model_hook.py b/python/sglang/srt/arg_groups/model_hook.py index 3c43f705a1..edfafe900d 100644 --- a/python/sglang/srt/arg_groups/model_hook.py +++ b/python/sglang/srt/arg_groups/model_hook.py @@ -32,7 +32,7 @@ from sglang.srt.arg_groups.overrides import ( ) from sglang.srt.arg_groups.resolution_hooks import run_hook from sglang.srt.configs.embedding_model_spec import BCGPrefillPolicy -from sglang.srt.configs.linear_attn_model_registry import get_linear_attn_spec_by_arch +from sglang.srt.configs.linear_attn_model_registry import get_linear_attn_spec from sglang.srt.connector import ConnectorType from sglang.srt.environ import envs from sglang.srt.hardware_backend.mlx.runtime import use_mlx @@ -194,9 +194,9 @@ def handle_model_specific_adjustments(server_args: Any): f"{sorted(CP_DECODE_ATTN_TP_SUPPORTED_ARCHS)}." ) - _hybrid_spec = get_linear_attn_spec_by_arch(model_arch) + _hybrid_spec = get_linear_attn_spec(hf_config) if _hybrid_spec is not None and _hybrid_spec.uses_mamba_radix_cache: - handle_mamba_radix_cache(server_args, model_arch) + handle_mamba_radix_cache(server_args, hf_config) # Collect the declarative model overrides (registry) on the # pristine config and stash them for publish-time flags resolution; @@ -689,7 +689,7 @@ def handle_model_specific_adjustments(server_args: Any): # for them this re-invocation is an idempotent no-op plus validation. # Kept ahead of the sparse-head pass: the legacy per-branch calls # resolved before that tail write of disable_overlap_schedule. - handle_mamba_radix_cache(server_args, model_arch) + handle_mamba_radix_cache(server_args, hf_config) run_post_process_pass(server_args, _sparse_head_overlap_disable) @@ -933,7 +933,7 @@ def handle_model_capability_adjustments(server_args: Any): ) -def handle_mamba_radix_cache(server_args: Any, model_arch: str): +def handle_mamba_radix_cache(server_args: Any, hf_config: Any): # Resolution moved to the resolution pipeline (arg_groups/overrides.py: # _mamba_radix_cache_resolution), invoked here at each legacy call # slot; this handler keeps the validation. @@ -950,18 +950,18 @@ def handle_mamba_radix_cache(server_args: Any, model_arch: str): if mamba_extra_buffer_of(view): raise ValueError( f"--mamba-radix-cache-strategy {view.mamba_radix_cache_strategy} " - f"needs mamba state, got {model_arch}." + f"needs mamba state, got {hf_config.architectures[0]}." ) return if mamba_extra_buffer_of(view): validate_mamba_extra_buffer( view, - model_arch, + hf_config, mamba_cache_chunk_size_of=lambda: mamba_cache_chunk_size(server_args), ) else: - validate_mamba_no_buffer(view, model_arch) + validate_mamba_no_buffer(view, hf_config.architectures[0]) def handle_language_model_only(server_args: Any): diff --git a/python/sglang/srt/arg_groups/overrides.py b/python/sglang/srt/arg_groups/overrides.py index 306b001a98..a2de0bc293 100644 --- a/python/sglang/srt/arg_groups/overrides.py +++ b/python/sglang/srt/arg_groups/overrides.py @@ -374,7 +374,7 @@ _MAMBA_RADIX_CACHE_ARCHS = frozenset( ) # Architectures that support the extra_buffer mamba radix cache strategy. -# The single source of truth; `supports_mamba_cache_extra_buffer` reads it. +# Registry specs opt in through `support_mamba_cache_extra_buffer` instead. _MAMBA_EXTRA_BUFFER_ARCHS = frozenset( { "KimiLinearForCausalLM", @@ -409,12 +409,17 @@ _MAMBA_EXTRA_BUFFER_ARCHS = frozenset( ) -def supports_mamba_cache_extra_buffer(view: Any, model_arch: str) -> bool: - """Whether ``model_arch`` supports the extra_buffer strategy on the - configured linear-attention backend (pure read).""" +def supports_mamba_cache_extra_buffer(view: Any, hf_config: Any) -> bool: + """Whether the model of ``hf_config`` supports the extra_buffer strategy on + the configured linear-attention backend (pure read).""" + from sglang.srt.configs.linear_attn_model_registry import get_linear_attn_spec + if get_platform().is_xpu: return False - if model_arch in _MAMBA_EXTRA_BUFFER_ARCHS: + spec = get_linear_attn_spec(hf_config) + if hf_config.architectures[0] in _MAMBA_EXTRA_BUFFER_ARCHS or ( + spec is not None and spec.support_mamba_cache_extra_buffer + ): return view.linear_attn_backend == "triton" return False @@ -422,9 +427,7 @@ def supports_mamba_cache_extra_buffer(view: Any, model_arch: str) -> bool: @register_post_process def _mamba_radix_cache_resolution(view: Any) -> dict: """Resolve hybrid-Mamba cache settings using the current page size and overlap policy.""" - from sglang.srt.configs.linear_attn_model_registry import ( - get_linear_attn_spec_by_arch, - ) + from sglang.srt.configs.linear_attn_model_registry import get_linear_attn_spec hf_config = model_config_of(view).hf_config model_arch = hf_config.architectures[0] @@ -435,7 +438,9 @@ def _mamba_radix_cache_resolution(view: Any) -> dict: layer_type == "mamba" for layer_type in getattr(hf_config, "layer_types", []) ) - spec = get_linear_attn_spec_by_arch(model_arch) + # Specs registered by config predicate (archs=[]) are only reachable + # through the HF config, not the architecture name. + spec = get_linear_attn_spec(hf_config) if not ((spec is not None and spec.uses_mamba_radix_cache) or in_branch): return {} @@ -447,7 +452,7 @@ def _mamba_radix_cache_resolution(view: Any) -> dict: wants_overlap = not view.disable_overlap_schedule wants_paging = view.page_size is not None and view.page_size > 1 if (wants_overlap or wants_paging) and supports_mamba_cache_extra_buffer( - view, model_arch + view, hf_config ): declared["mamba_radix_cache_strategy"] = "extra_buffer" else: diff --git a/python/sglang/srt/configs/linear_attn_model_registry.py b/python/sglang/srt/configs/linear_attn_model_registry.py index ba30fcf437..a3f760e6d8 100644 --- a/python/sglang/srt/configs/linear_attn_model_registry.py +++ b/python/sglang/srt/configs/linear_attn_model_registry.py @@ -71,6 +71,15 @@ def get_linear_attn_spec_by_arch(arch_name: str) -> Optional[LinearAttnModelSpec return None +def get_linear_attn_spec(hf_config: Any) -> Optional[LinearAttnModelSpec]: + """Spec for ``hf_config``: by architecture name, else by config class and predicate.""" + spec = get_linear_attn_spec_by_arch(hf_config.architectures[0]) + if spec is not None: + return spec + result = get_linear_attn_config(hf_config) + return result[0] if result is not None else None + + def import_backend_class(dotted_name: str) -> type: module_path, class_name = dotted_name.rsplit(".", 1) module = importlib.import_module(module_path) diff --git a/test/registered/unit/spec/test_dflash_extra_buffer_lazy.py b/test/registered/unit/spec/test_dflash_extra_buffer_lazy.py index fa2a3dca72..4a94c772ba 100644 --- a/test/registered/unit/spec/test_dflash_extra_buffer_lazy.py +++ b/test/registered/unit/spec/test_dflash_extra_buffer_lazy.py @@ -48,7 +48,7 @@ class TestValidateMambaExtraBufferLazyDflash(CustomTestCase): with override_platform(is_cuda=True): validate_mamba_extra_buffer( view, - "Qwen3NextForCausalLM", + SimpleNamespace(architectures=["Qwen3NextForCausalLM"]), mamba_cache_chunk_size_of=lambda: 64, ) @@ -82,7 +82,7 @@ class TestValidateMambaExtraBufferLazyDflash(CustomTestCase): with override_platform(is_cuda=True): validate_mamba_extra_buffer( _lazy_view(page_size=None), - "Qwen3NextForCausalLM", + SimpleNamespace(architectures=["Qwen3NextForCausalLM"]), mamba_cache_chunk_size_of=_must_not_be_read, ) diff --git a/test/registered/unit/test_model_overrides.py b/test/registered/unit/test_model_overrides.py index 1110917241..637a9cadcd 100644 --- a/test/registered/unit/test_model_overrides.py +++ b/test/registered/unit/test_model_overrides.py @@ -609,6 +609,29 @@ class TestGoldenModelOverrides(_IsolatedPublish): ) self.assertFalse(self._resolved(sa, "uses_mamba_radix_cache")) + def test_explicit_extra_buffer_accepts_predicate_registered_mamba_state(self): + from transformers import LlamaConfig + + from sglang.srt.configs import linear_attn_model_registry as registry + + spec = registry.LinearAttnModelSpec( + config_class=LlamaConfig, + backend_class_name="pkg.mod.Backend", + config_predicate=lambda cfg: getattr(cfg, "linear_attn", False), + support_mamba_cache_extra_buffer=True, + ) + with ( + patch.object(registry, "_LINEAR_ATTN_MODEL_REGISTRY", [spec]), + override_platform(is_cuda=True), + ): + sa = self._construct( + "LlamaForCausalLM", + "llama", + config_extra={"linear_attn": True}, + mamba_radix_cache_strategy="extra_buffer", + ) + self.assertTrue(self._resolved(sa, "uses_mamba_radix_cache")) + def test_mistral_large3_forces_bfloat16(self): sa = self._construct("MistralLarge3ForCausalLM", "mistral") self.assertEqual( @@ -2536,7 +2559,8 @@ class TestGoldenModelOverrides(_IsolatedPublish): # extra-buffer support requires the triton linear-attn backend self.assertFalse( supports_mamba_cache_extra_buffer( - SimpleNamespace(linear_attn_backend="fla"), "Qwen3NextForCausalLM" + SimpleNamespace(linear_attn_backend="fla"), + SimpleNamespace(architectures=["Qwen3NextForCausalLM"]), ) ) self.assertTrue( @@ -2545,10 +2569,115 @@ class TestGoldenModelOverrides(_IsolatedPublish): linear_attn_backend="triton", linear_attn_prefill_backend="flashinfer", ), - "Qwen3_5MoeForConditionalGeneration", + SimpleNamespace(architectures=["Qwen3_5MoeForConditionalGeneration"]), ) ) + def test_mamba_radix_cache_resolution_reads_registry_specs(self): + """A spec registered by config predicate (archs=[]) drives the leaves + exactly like an arch-registered one, and the spec's + `support_mamba_cache_extra_buffer` gates the extra_buffer strategy.""" + from sglang.srt.arg_groups.overrides import ( + ResolvedView, + _mamba_radix_cache_resolution, + supports_mamba_cache_extra_buffer, + ) + from sglang.srt.configs import linear_attn_model_registry as registry + + class _PredicateConfig: + def __init__(self, linear_attn): + self.architectures = ["PredicateHybridForCausalLM"] + self.linear_attn = linear_attn + + class _ArchConfig: + architectures = ["ArchHybridForCausalLM"] + + def _view(hf, **kw): + defaults = dict( + disable_radix_cache=False, + mamba_radix_cache_strategy="auto", + disable_overlap_schedule=False, + page_size=None, + linear_attn_backend="triton", + linear_attn_prefill_backend=None, + ) + defaults.update(kw) + return ResolvedView( + SimpleNamespace(_model_config=SimpleNamespace(hf_config=hf), **defaults) + ) + + by_predicate = registry.LinearAttnModelSpec( + config_class=_PredicateConfig, + backend_class_name="pkg.mod.Backend", + config_predicate=lambda cfg: cfg.linear_attn, + support_mamba_cache_extra_buffer=True, + ) + by_arch = registry.LinearAttnModelSpec( + config_class=_ArchConfig, + backend_class_name="pkg.mod.Backend", + arch_names=["ArchHybridForCausalLM"], + ) + with patch.object(registry, "_LINEAR_ATTN_MODEL_REGISTRY", []): + registry.register_linear_attn_model(by_predicate) + registry.register_linear_attn_model(by_arch) + + # predicate hit: the leaf is declared and the spec opts into + # extra_buffer on the triton linear-attn backend only + self.assertEqual( + _mamba_radix_cache_resolution(_view(_PredicateConfig(True))), + { + "uses_mamba_radix_cache": True, + "mamba_radix_cache_strategy": "extra_buffer", + }, + ) + self.assertEqual( + _mamba_radix_cache_resolution( + _view(_PredicateConfig(True), linear_attn_backend="fla") + ), + { + "uses_mamba_radix_cache": True, + "mamba_radix_cache_strategy": "no_buffer", + "disable_overlap_schedule": True, + }, + ) + self.assertTrue( + supports_mamba_cache_extra_buffer( + SimpleNamespace(linear_attn_backend="triton"), + _PredicateConfig(True), + ) + ) + # predicate miss: not a hybrid model at all + self.assertEqual( + _mamba_radix_cache_resolution(_view(_PredicateConfig(False))), {} + ) + self.assertFalse( + supports_mamba_cache_extra_buffer( + SimpleNamespace(linear_attn_backend="triton"), + _PredicateConfig(False), + ) + ) + # arch-registered spec without the opt-in: unchanged, no_buffer + self.assertEqual( + _mamba_radix_cache_resolution(_view(_ArchConfig())), + { + "uses_mamba_radix_cache": True, + "mamba_radix_cache_strategy": "no_buffer", + "disable_overlap_schedule": True, + }, + ) + self.assertFalse( + supports_mamba_cache_extra_buffer( + SimpleNamespace(linear_attn_backend="triton"), _ArchConfig() + ) + ) + # hard-coded archs do not need a spec + self.assertEqual( + _mamba_radix_cache_resolution( + _view(SimpleNamespace(architectures=["Qwen3NextForCausalLM"])) + )["mamba_radix_cache_strategy"], + "extra_buffer", + ) + def test_qwen3_5_hybrid_coupled_declaration(self): from sglang.srt.arg_groups.model_overrides.qwen3_5 import ( _qwen3_5_hybrid_overrides,