Let predicate-registered linear-attention models carry the mamba radix-cache leaves (#41165)

This commit is contained in:
Peng Wu
2026-09-28 17:07:35 +08:00
committed by GitHub
parent df378bca42
commit 6582425854
6 changed files with 168 additions and 25 deletions
+3 -3
View File
@@ -98,10 +98,10 @@ def handle_int8_mamba_checkpoint(server_args: Any):
)
def validate_mamba_extra_buffer(view, model_arch: str, *, mamba_cache_chunk_size_of):
def validate_mamba_extra_buffer(view, hf_config: Any, *, mamba_cache_chunk_size_of):
assert supports_mamba_cache_extra_buffer(view, model_arch), (
f"extra_buffer is not supported for {model_arch}; use no_buffer."
assert supports_mamba_cache_extra_buffer(view, hf_config), (
f"extra_buffer is not supported for {hf_config.architectures[0]}; use no_buffer."
)
assert (
get_platform().is_cuda
+8 -8
View File
@@ -32,7 +32,7 @@ from sglang.srt.arg_groups.overrides import (
)
from sglang.srt.arg_groups.resolution_hooks import run_hook
from sglang.srt.configs.embedding_model_spec import BCGPrefillPolicy
from sglang.srt.configs.linear_attn_model_registry import get_linear_attn_spec_by_arch
from sglang.srt.configs.linear_attn_model_registry import get_linear_attn_spec
from sglang.srt.connector import ConnectorType
from sglang.srt.environ import envs
from sglang.srt.hardware_backend.mlx.runtime import use_mlx
@@ -194,9 +194,9 @@ def handle_model_specific_adjustments(server_args: Any):
f"{sorted(CP_DECODE_ATTN_TP_SUPPORTED_ARCHS)}."
)
_hybrid_spec = get_linear_attn_spec_by_arch(model_arch)
_hybrid_spec = get_linear_attn_spec(hf_config)
if _hybrid_spec is not None and _hybrid_spec.uses_mamba_radix_cache:
handle_mamba_radix_cache(server_args, model_arch)
handle_mamba_radix_cache(server_args, hf_config)
# Collect the declarative model overrides (registry) on the
# pristine config and stash them for publish-time flags resolution;
@@ -689,7 +689,7 @@ def handle_model_specific_adjustments(server_args: Any):
# for them this re-invocation is an idempotent no-op plus validation.
# Kept ahead of the sparse-head pass: the legacy per-branch calls
# resolved before that tail write of disable_overlap_schedule.
handle_mamba_radix_cache(server_args, model_arch)
handle_mamba_radix_cache(server_args, hf_config)
run_post_process_pass(server_args, _sparse_head_overlap_disable)
@@ -933,7 +933,7 @@ def handle_model_capability_adjustments(server_args: Any):
)
def handle_mamba_radix_cache(server_args: Any, model_arch: str):
def handle_mamba_radix_cache(server_args: Any, hf_config: Any):
# Resolution moved to the resolution pipeline (arg_groups/overrides.py:
# _mamba_radix_cache_resolution), invoked here at each legacy call
# slot; this handler keeps the validation.
@@ -950,18 +950,18 @@ def handle_mamba_radix_cache(server_args: Any, model_arch: str):
if mamba_extra_buffer_of(view):
raise ValueError(
f"--mamba-radix-cache-strategy {view.mamba_radix_cache_strategy} "
f"needs mamba state, got {model_arch}."
f"needs mamba state, got {hf_config.architectures[0]}."
)
return
if mamba_extra_buffer_of(view):
validate_mamba_extra_buffer(
view,
model_arch,
hf_config,
mamba_cache_chunk_size_of=lambda: mamba_cache_chunk_size(server_args),
)
else:
validate_mamba_no_buffer(view, model_arch)
validate_mamba_no_buffer(view, hf_config.architectures[0])
def handle_language_model_only(server_args: Any):
+15 -10
View File
@@ -374,7 +374,7 @@ _MAMBA_RADIX_CACHE_ARCHS = frozenset(
)
# Architectures that support the extra_buffer mamba radix cache strategy.
# The single source of truth; `supports_mamba_cache_extra_buffer` reads it.
# Registry specs opt in through `support_mamba_cache_extra_buffer` instead.
_MAMBA_EXTRA_BUFFER_ARCHS = frozenset(
{
"KimiLinearForCausalLM",
@@ -409,12 +409,17 @@ _MAMBA_EXTRA_BUFFER_ARCHS = frozenset(
)
def supports_mamba_cache_extra_buffer(view: Any, model_arch: str) -> bool:
"""Whether ``model_arch`` supports the extra_buffer strategy on the
configured linear-attention backend (pure read)."""
def supports_mamba_cache_extra_buffer(view: Any, hf_config: Any) -> bool:
"""Whether the model of ``hf_config`` supports the extra_buffer strategy on
the configured linear-attention backend (pure read)."""
from sglang.srt.configs.linear_attn_model_registry import get_linear_attn_spec
if get_platform().is_xpu:
return False
if model_arch in _MAMBA_EXTRA_BUFFER_ARCHS:
spec = get_linear_attn_spec(hf_config)
if hf_config.architectures[0] in _MAMBA_EXTRA_BUFFER_ARCHS or (
spec is not None and spec.support_mamba_cache_extra_buffer
):
return view.linear_attn_backend == "triton"
return False
@@ -422,9 +427,7 @@ def supports_mamba_cache_extra_buffer(view: Any, model_arch: str) -> bool:
@register_post_process
def _mamba_radix_cache_resolution(view: Any) -> dict:
"""Resolve hybrid-Mamba cache settings using the current page size and overlap policy."""
from sglang.srt.configs.linear_attn_model_registry import (
get_linear_attn_spec_by_arch,
)
from sglang.srt.configs.linear_attn_model_registry import get_linear_attn_spec
hf_config = model_config_of(view).hf_config
model_arch = hf_config.architectures[0]
@@ -435,7 +438,9 @@ def _mamba_radix_cache_resolution(view: Any) -> dict:
layer_type == "mamba"
for layer_type in getattr(hf_config, "layer_types", [])
)
spec = get_linear_attn_spec_by_arch(model_arch)
# Specs registered by config predicate (archs=[]) are only reachable
# through the HF config, not the architecture name.
spec = get_linear_attn_spec(hf_config)
if not ((spec is not None and spec.uses_mamba_radix_cache) or in_branch):
return {}
@@ -447,7 +452,7 @@ def _mamba_radix_cache_resolution(view: Any) -> dict:
wants_overlap = not view.disable_overlap_schedule
wants_paging = view.page_size is not None and view.page_size > 1
if (wants_overlap or wants_paging) and supports_mamba_cache_extra_buffer(
view, model_arch
view, hf_config
):
declared["mamba_radix_cache_strategy"] = "extra_buffer"
else:
@@ -71,6 +71,15 @@ def get_linear_attn_spec_by_arch(arch_name: str) -> Optional[LinearAttnModelSpec
return None
def get_linear_attn_spec(hf_config: Any) -> Optional[LinearAttnModelSpec]:
"""Spec for ``hf_config``: by architecture name, else by config class and predicate."""
spec = get_linear_attn_spec_by_arch(hf_config.architectures[0])
if spec is not None:
return spec
result = get_linear_attn_config(hf_config)
return result[0] if result is not None else None
def import_backend_class(dotted_name: str) -> type:
module_path, class_name = dotted_name.rsplit(".", 1)
module = importlib.import_module(module_path)