From 40fb5935c0657a108ab6a851cf82de86ee1c36b5 Mon Sep 17 00:00:00 2001 From: Joe Date: Tue, 6 Oct 2026 02:18:49 -0400 Subject: [PATCH] Allow platforms to declare Mamba extra-buffer support (#36410) --- python/sglang/srt/arg_groups/mamba_hook.py | 24 +++++-- python/sglang/srt/platforms/interface.py | 4 ++ .../unit/platforms/test_platform_interface.py | 11 +++ .../test_mamba_extra_buffer_platform.py | 67 +++++++++++++++++++ 4 files changed, 99 insertions(+), 7 deletions(-) create mode 100644 test/registered/unit/server_args/test_mamba_extra_buffer_platform.py diff --git a/python/sglang/srt/arg_groups/mamba_hook.py b/python/sglang/srt/arg_groups/mamba_hook.py index 55fdb92ada..3376683273 100644 --- a/python/sglang/srt/arg_groups/mamba_hook.py +++ b/python/sglang/srt/arg_groups/mamba_hook.py @@ -10,6 +10,7 @@ from sglang.srt.arg_groups.overrides import ( resolving_view, supports_mamba_cache_extra_buffer, ) +from sglang.srt.platforms import current_platform from sglang.srt.runtime_context import get_platform logger = logging.getLogger(__name__) @@ -98,18 +99,27 @@ def handle_int8_mamba_checkpoint(server_args: Any): ) +def _platform_supports_mamba_cache_extra_buffer() -> bool: + if current_platform.is_out_of_tree(): + return current_platform.support_mamba_cache_extra_buffer() + platform = get_platform() + return ( + platform.is_cuda + or platform.is_musa + or platform.is_npu + or platform.is_hip + or platform.is_xpu + ) + + def validate_mamba_extra_buffer(view, hf_config: Any, *, mamba_cache_chunk_size_of): assert supports_mamba_cache_extra_buffer(view, hf_config), ( f"extra_buffer is not supported for {hf_config.architectures[0]}; use no_buffer." ) - assert ( - get_platform().is_cuda - or get_platform().is_musa - or get_platform().is_npu - or get_platform().is_hip - or get_platform().is_xpu - ), "extra_buffer needs CUDA/MUSA/NPU/ROCm/XPU (FLA)." + assert _platform_supports_mamba_cache_extra_buffer(), ( + "extra_buffer requires platform support for Mamba cache state snapshots." + ) if view.mamba_radix_cache_strategy == "extra_buffer_lazy": # The PD-disagg decode pool is not wired for lazy slots. assert view.disaggregation_mode == "null", ( diff --git a/python/sglang/srt/platforms/interface.py b/python/sglang/srt/platforms/interface.py index 7ec2727234..7bfeec6a7a 100644 --- a/python/sglang/srt/platforms/interface.py +++ b/python/sglang/srt/platforms/interface.py @@ -129,6 +129,10 @@ class SRTPlatform(DeviceMixin): """Whether this platform supports a draft backend for an algorithm.""" return False + def support_mamba_cache_extra_buffer(self) -> bool: + """Whether this out-of-tree platform supports Mamba cache snapshots.""" + return False + def support_cuda_graph(self) -> bool: """Whether this platform supports device graph capture and replay. Controls CUDA graph (CudaGraphRunner) for the decode path. diff --git a/test/registered/unit/platforms/test_platform_interface.py b/test/registered/unit/platforms/test_platform_interface.py index 0edc411926..892443250e 100644 --- a/test/registered/unit/platforms/test_platform_interface.py +++ b/test/registered/unit/platforms/test_platform_interface.py @@ -187,6 +187,17 @@ class TestSRTPlatform(CustomTestCase): ) ) + def test_mamba_cache_extra_buffer_capability(self): + self.assertFalse(SRTPlatform().support_mamba_cache_extra_buffer()) + + class OutOfTreePlatform(SRTPlatform): + _enum = PlatformEnum.OOT + + def support_mamba_cache_extra_buffer(self) -> bool: + return True + + self.assertTrue(OutOfTreePlatform().support_mamba_cache_extra_buffer()) + class TestCudaDeviceMixin(CustomTestCase): """Tests for CUDA device operation defaults.""" diff --git a/test/registered/unit/server_args/test_mamba_extra_buffer_platform.py b/test/registered/unit/server_args/test_mamba_extra_buffer_platform.py new file mode 100644 index 0000000000..18bf5ce260 --- /dev/null +++ b/test/registered/unit/server_args/test_mamba_extra_buffer_platform.py @@ -0,0 +1,67 @@ +import unittest +from types import SimpleNamespace +from unittest.mock import patch + +from sglang.srt.arg_groups.mamba_hook import validate_mamba_extra_buffer +from sglang.srt.runtime_context import override_platform +from sglang.test.ci.ci_register import register_cpu_ci +from sglang.test.test_utils import CustomTestCase + +register_cpu_ci(est_time=3, suite="base-a-test-cpu") + + +def _validate(): + validate_mamba_extra_buffer( + SimpleNamespace( + mamba_radix_cache_strategy="extra_buffer", + speculative_num_draft_tokens=None, + page_size=None, + ), + SimpleNamespace(architectures=["MambaForCausalLM"]), + mamba_cache_chunk_size_of=lambda: None, + ) + + +class TestMambaExtraBufferPlatform(CustomTestCase): + @patch( + "sglang.srt.arg_groups.mamba_hook.supports_mamba_cache_extra_buffer", + return_value=True, + ) + @patch("sglang.srt.arg_groups.mamba_hook.current_platform") + def test_in_tree_validation_uses_platform_context(self, platform, _supports_model): + platform.is_out_of_tree.return_value = False + platform_facts = dict( + is_cuda=False, + is_musa=False, + is_npu=False, + is_hip=False, + is_xpu=False, + ) + with override_platform(**platform_facts): + with self.assertRaisesRegex(AssertionError, "platform support"): + _validate() + for fact in platform_facts: + with self.subTest(fact=fact), override_platform(**{fact: True}): + _validate() + platform.support_mamba_cache_extra_buffer.assert_not_called() + + @patch( + "sglang.srt.arg_groups.mamba_hook.supports_mamba_cache_extra_buffer", + return_value=True, + ) + @patch("sglang.srt.arg_groups.mamba_hook.current_platform") + def test_out_of_tree_validation_uses_platform_capability( + self, platform, _supports_model + ): + platform.is_out_of_tree.return_value = True + with override_platform(is_cuda=True): + platform.support_mamba_cache_extra_buffer.return_value = False + with self.assertRaisesRegex(AssertionError, "platform support"): + _validate() + + platform.support_mamba_cache_extra_buffer.return_value = True + _validate() + + +if __name__ == "__main__": + unittest.main()