Let predicate-registered linear-attention models carry the mamba radix-cache leaves (#41165)
This commit is contained in:
@@ -98,10 +98,10 @@ def handle_int8_mamba_checkpoint(server_args: Any):
|
||||
)
|
||||
|
||||
|
||||
def validate_mamba_extra_buffer(view, model_arch: str, *, mamba_cache_chunk_size_of):
|
||||
def validate_mamba_extra_buffer(view, hf_config: Any, *, mamba_cache_chunk_size_of):
|
||||
|
||||
assert supports_mamba_cache_extra_buffer(view, model_arch), (
|
||||
f"extra_buffer is not supported for {model_arch}; use no_buffer."
|
||||
assert supports_mamba_cache_extra_buffer(view, hf_config), (
|
||||
f"extra_buffer is not supported for {hf_config.architectures[0]}; use no_buffer."
|
||||
)
|
||||
assert (
|
||||
get_platform().is_cuda
|
||||
|
||||
@@ -32,7 +32,7 @@ from sglang.srt.arg_groups.overrides import (
|
||||
)
|
||||
from sglang.srt.arg_groups.resolution_hooks import run_hook
|
||||
from sglang.srt.configs.embedding_model_spec import BCGPrefillPolicy
|
||||
from sglang.srt.configs.linear_attn_model_registry import get_linear_attn_spec_by_arch
|
||||
from sglang.srt.configs.linear_attn_model_registry import get_linear_attn_spec
|
||||
from sglang.srt.connector import ConnectorType
|
||||
from sglang.srt.environ import envs
|
||||
from sglang.srt.hardware_backend.mlx.runtime import use_mlx
|
||||
@@ -194,9 +194,9 @@ def handle_model_specific_adjustments(server_args: Any):
|
||||
f"{sorted(CP_DECODE_ATTN_TP_SUPPORTED_ARCHS)}."
|
||||
)
|
||||
|
||||
_hybrid_spec = get_linear_attn_spec_by_arch(model_arch)
|
||||
_hybrid_spec = get_linear_attn_spec(hf_config)
|
||||
if _hybrid_spec is not None and _hybrid_spec.uses_mamba_radix_cache:
|
||||
handle_mamba_radix_cache(server_args, model_arch)
|
||||
handle_mamba_radix_cache(server_args, hf_config)
|
||||
|
||||
# Collect the declarative model overrides (registry) on the
|
||||
# pristine config and stash them for publish-time flags resolution;
|
||||
@@ -689,7 +689,7 @@ def handle_model_specific_adjustments(server_args: Any):
|
||||
# for them this re-invocation is an idempotent no-op plus validation.
|
||||
# Kept ahead of the sparse-head pass: the legacy per-branch calls
|
||||
# resolved before that tail write of disable_overlap_schedule.
|
||||
handle_mamba_radix_cache(server_args, model_arch)
|
||||
handle_mamba_radix_cache(server_args, hf_config)
|
||||
|
||||
run_post_process_pass(server_args, _sparse_head_overlap_disable)
|
||||
|
||||
@@ -933,7 +933,7 @@ def handle_model_capability_adjustments(server_args: Any):
|
||||
)
|
||||
|
||||
|
||||
def handle_mamba_radix_cache(server_args: Any, model_arch: str):
|
||||
def handle_mamba_radix_cache(server_args: Any, hf_config: Any):
|
||||
# Resolution moved to the resolution pipeline (arg_groups/overrides.py:
|
||||
# _mamba_radix_cache_resolution), invoked here at each legacy call
|
||||
# slot; this handler keeps the validation.
|
||||
@@ -950,18 +950,18 @@ def handle_mamba_radix_cache(server_args: Any, model_arch: str):
|
||||
if mamba_extra_buffer_of(view):
|
||||
raise ValueError(
|
||||
f"--mamba-radix-cache-strategy {view.mamba_radix_cache_strategy} "
|
||||
f"needs mamba state, got {model_arch}."
|
||||
f"needs mamba state, got {hf_config.architectures[0]}."
|
||||
)
|
||||
return
|
||||
|
||||
if mamba_extra_buffer_of(view):
|
||||
validate_mamba_extra_buffer(
|
||||
view,
|
||||
model_arch,
|
||||
hf_config,
|
||||
mamba_cache_chunk_size_of=lambda: mamba_cache_chunk_size(server_args),
|
||||
)
|
||||
else:
|
||||
validate_mamba_no_buffer(view, model_arch)
|
||||
validate_mamba_no_buffer(view, hf_config.architectures[0])
|
||||
|
||||
|
||||
def handle_language_model_only(server_args: Any):
|
||||
|
||||
@@ -374,7 +374,7 @@ _MAMBA_RADIX_CACHE_ARCHS = frozenset(
|
||||
)
|
||||
|
||||
# Architectures that support the extra_buffer mamba radix cache strategy.
|
||||
# The single source of truth; `supports_mamba_cache_extra_buffer` reads it.
|
||||
# Registry specs opt in through `support_mamba_cache_extra_buffer` instead.
|
||||
_MAMBA_EXTRA_BUFFER_ARCHS = frozenset(
|
||||
{
|
||||
"KimiLinearForCausalLM",
|
||||
@@ -409,12 +409,17 @@ _MAMBA_EXTRA_BUFFER_ARCHS = frozenset(
|
||||
)
|
||||
|
||||
|
||||
def supports_mamba_cache_extra_buffer(view: Any, model_arch: str) -> bool:
|
||||
"""Whether ``model_arch`` supports the extra_buffer strategy on the
|
||||
configured linear-attention backend (pure read)."""
|
||||
def supports_mamba_cache_extra_buffer(view: Any, hf_config: Any) -> bool:
|
||||
"""Whether the model of ``hf_config`` supports the extra_buffer strategy on
|
||||
the configured linear-attention backend (pure read)."""
|
||||
from sglang.srt.configs.linear_attn_model_registry import get_linear_attn_spec
|
||||
|
||||
if get_platform().is_xpu:
|
||||
return False
|
||||
if model_arch in _MAMBA_EXTRA_BUFFER_ARCHS:
|
||||
spec = get_linear_attn_spec(hf_config)
|
||||
if hf_config.architectures[0] in _MAMBA_EXTRA_BUFFER_ARCHS or (
|
||||
spec is not None and spec.support_mamba_cache_extra_buffer
|
||||
):
|
||||
return view.linear_attn_backend == "triton"
|
||||
return False
|
||||
|
||||
@@ -422,9 +427,7 @@ def supports_mamba_cache_extra_buffer(view: Any, model_arch: str) -> bool:
|
||||
@register_post_process
|
||||
def _mamba_radix_cache_resolution(view: Any) -> dict:
|
||||
"""Resolve hybrid-Mamba cache settings using the current page size and overlap policy."""
|
||||
from sglang.srt.configs.linear_attn_model_registry import (
|
||||
get_linear_attn_spec_by_arch,
|
||||
)
|
||||
from sglang.srt.configs.linear_attn_model_registry import get_linear_attn_spec
|
||||
|
||||
hf_config = model_config_of(view).hf_config
|
||||
model_arch = hf_config.architectures[0]
|
||||
@@ -435,7 +438,9 @@ def _mamba_radix_cache_resolution(view: Any) -> dict:
|
||||
layer_type == "mamba"
|
||||
for layer_type in getattr(hf_config, "layer_types", [])
|
||||
)
|
||||
spec = get_linear_attn_spec_by_arch(model_arch)
|
||||
# Specs registered by config predicate (archs=[]) are only reachable
|
||||
# through the HF config, not the architecture name.
|
||||
spec = get_linear_attn_spec(hf_config)
|
||||
if not ((spec is not None and spec.uses_mamba_radix_cache) or in_branch):
|
||||
return {}
|
||||
|
||||
@@ -447,7 +452,7 @@ def _mamba_radix_cache_resolution(view: Any) -> dict:
|
||||
wants_overlap = not view.disable_overlap_schedule
|
||||
wants_paging = view.page_size is not None and view.page_size > 1
|
||||
if (wants_overlap or wants_paging) and supports_mamba_cache_extra_buffer(
|
||||
view, model_arch
|
||||
view, hf_config
|
||||
):
|
||||
declared["mamba_radix_cache_strategy"] = "extra_buffer"
|
||||
else:
|
||||
|
||||
@@ -71,6 +71,15 @@ def get_linear_attn_spec_by_arch(arch_name: str) -> Optional[LinearAttnModelSpec
|
||||
return None
|
||||
|
||||
|
||||
def get_linear_attn_spec(hf_config: Any) -> Optional[LinearAttnModelSpec]:
|
||||
"""Spec for ``hf_config``: by architecture name, else by config class and predicate."""
|
||||
spec = get_linear_attn_spec_by_arch(hf_config.architectures[0])
|
||||
if spec is not None:
|
||||
return spec
|
||||
result = get_linear_attn_config(hf_config)
|
||||
return result[0] if result is not None else None
|
||||
|
||||
|
||||
def import_backend_class(dotted_name: str) -> type:
|
||||
module_path, class_name = dotted_name.rsplit(".", 1)
|
||||
module = importlib.import_module(module_path)
|
||||
|
||||
Reference in New Issue
Block a user