Allow platforms to declare Mamba extra-buffer support (#36410)

This commit is contained in:
Joe
2026-10-06 14:18:49 +08:00
committed by GitHub
parent 9a09b48da7
commit 40fb5935c0
4 changed files with 99 additions and 7 deletions
+17 -7
View File
@@ -10,6 +10,7 @@ from sglang.srt.arg_groups.overrides import (
resolving_view,
supports_mamba_cache_extra_buffer,
)
from sglang.srt.platforms import current_platform
from sglang.srt.runtime_context import get_platform
logger = logging.getLogger(__name__)
@@ -98,18 +99,27 @@ def handle_int8_mamba_checkpoint(server_args: Any):
)
def _platform_supports_mamba_cache_extra_buffer() -> bool:
if current_platform.is_out_of_tree():
return current_platform.support_mamba_cache_extra_buffer()
platform = get_platform()
return (
platform.is_cuda
or platform.is_musa
or platform.is_npu
or platform.is_hip
or platform.is_xpu
)
def validate_mamba_extra_buffer(view, hf_config: Any, *, mamba_cache_chunk_size_of):
assert supports_mamba_cache_extra_buffer(view, hf_config), (
f"extra_buffer is not supported for {hf_config.architectures[0]}; use no_buffer."
)
assert (
get_platform().is_cuda
or get_platform().is_musa
or get_platform().is_npu
or get_platform().is_hip
or get_platform().is_xpu
), "extra_buffer needs CUDA/MUSA/NPU/ROCm/XPU (FLA)."
assert _platform_supports_mamba_cache_extra_buffer(), (
"extra_buffer requires platform support for Mamba cache state snapshots."
)
if view.mamba_radix_cache_strategy == "extra_buffer_lazy":
# The PD-disagg decode pool is not wired for lazy slots.
assert view.disaggregation_mode == "null", (
+4
View File
@@ -129,6 +129,10 @@ class SRTPlatform(DeviceMixin):
"""Whether this platform supports a draft backend for an algorithm."""
return False
def support_mamba_cache_extra_buffer(self) -> bool:
"""Whether this out-of-tree platform supports Mamba cache snapshots."""
return False
def support_cuda_graph(self) -> bool:
"""Whether this platform supports device graph capture and replay.
Controls CUDA graph (CudaGraphRunner) for the decode path.