Allow platforms to declare Mamba extra-buffer support (#36410)
This commit is contained in:
@@ -10,6 +10,7 @@ from sglang.srt.arg_groups.overrides import (
|
||||
resolving_view,
|
||||
supports_mamba_cache_extra_buffer,
|
||||
)
|
||||
from sglang.srt.platforms import current_platform
|
||||
from sglang.srt.runtime_context import get_platform
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
@@ -98,18 +99,27 @@ def handle_int8_mamba_checkpoint(server_args: Any):
|
||||
)
|
||||
|
||||
|
||||
def _platform_supports_mamba_cache_extra_buffer() -> bool:
|
||||
if current_platform.is_out_of_tree():
|
||||
return current_platform.support_mamba_cache_extra_buffer()
|
||||
platform = get_platform()
|
||||
return (
|
||||
platform.is_cuda
|
||||
or platform.is_musa
|
||||
or platform.is_npu
|
||||
or platform.is_hip
|
||||
or platform.is_xpu
|
||||
)
|
||||
|
||||
|
||||
def validate_mamba_extra_buffer(view, hf_config: Any, *, mamba_cache_chunk_size_of):
|
||||
|
||||
assert supports_mamba_cache_extra_buffer(view, hf_config), (
|
||||
f"extra_buffer is not supported for {hf_config.architectures[0]}; use no_buffer."
|
||||
)
|
||||
assert (
|
||||
get_platform().is_cuda
|
||||
or get_platform().is_musa
|
||||
or get_platform().is_npu
|
||||
or get_platform().is_hip
|
||||
or get_platform().is_xpu
|
||||
), "extra_buffer needs CUDA/MUSA/NPU/ROCm/XPU (FLA)."
|
||||
assert _platform_supports_mamba_cache_extra_buffer(), (
|
||||
"extra_buffer requires platform support for Mamba cache state snapshots."
|
||||
)
|
||||
if view.mamba_radix_cache_strategy == "extra_buffer_lazy":
|
||||
# The PD-disagg decode pool is not wired for lazy slots.
|
||||
assert view.disaggregation_mode == "null", (
|
||||
|
||||
@@ -129,6 +129,10 @@ class SRTPlatform(DeviceMixin):
|
||||
"""Whether this platform supports a draft backend for an algorithm."""
|
||||
return False
|
||||
|
||||
def support_mamba_cache_extra_buffer(self) -> bool:
|
||||
"""Whether this out-of-tree platform supports Mamba cache snapshots."""
|
||||
return False
|
||||
|
||||
def support_cuda_graph(self) -> bool:
|
||||
"""Whether this platform supports device graph capture and replay.
|
||||
Controls CUDA graph (CudaGraphRunner) for the decode path.
|
||||
|
||||
Reference in New Issue
Block a user