From af856717c253f4a86848b7f12d7041a7fe142bc7 Mon Sep 17 00:00:00 2001 From: Mohammad Miadh Angkad <176301910+mmangkad@users.noreply.github.com> Date: Tue, 20 Oct 2026 10:23:17 +0800 Subject: [PATCH] [Fix] mamba: Skip zero-width conv pools in the MTP verify scatter (#45212) Co-authored-by: Mohammad Angkad Co-authored-by: Mohammad Angkad --- .../ops/mamba/mamba_state_scatter_triton.py | 10 +++++-- .../layers/test_mamba_state_scatter_triton.py | 27 +++++++++++++++++++ 2 files changed, 35 insertions(+), 2 deletions(-) diff --git a/python/sglang/kernels/ops/mamba/mamba_state_scatter_triton.py b/python/sglang/kernels/ops/mamba/mamba_state_scatter_triton.py index 65f3f9e143..1e72110cac 100644 --- a/python/sglang/kernels/ops/mamba/mamba_state_scatter_triton.py +++ b/python/sglang/kernels/ops/mamba/mamba_state_scatter_triton.py @@ -422,7 +422,7 @@ def fused_conv_window_scatter_with_mask( ``[layers, cache, dim, K-1]``. """ total_requests = step_indices_raw.shape[0] - if total_requests == 0: + if total_requests == 0 or dst.numel() == 0: return if not (dst.is_cuda or dst.is_xpu) or dst.device != src.device: @@ -730,7 +730,13 @@ def scatter_mamba_states_after_mtp_verify( src_indices_raw, ) - pairs = list(zip(mamba_caches.conv, mamba_caches.intermediate_conv_window)) + pairs = [ + (conv, window) + for conv, window in zip( + mamba_caches.conv, mamba_caches.intermediate_conv_window + ) + if conv.numel() > 0 + ] if not pairs: return if mamba_track_indices is not None: diff --git a/test/registered/unit/layers/test_mamba_state_scatter_triton.py b/test/registered/unit/layers/test_mamba_state_scatter_triton.py index 3d71910935..298499415c 100644 --- a/test/registered/unit/layers/test_mamba_state_scatter_triton.py +++ b/test/registered/unit/layers/test_mamba_state_scatter_triton.py @@ -22,6 +22,7 @@ try: fused_conv_window_scatter_multi, fused_conv_window_scatter_with_mask, fused_mamba_state_scatter_with_mask, + scatter_mamba_states_after_mtp_verify, ) _FUSED_IMPORT_ERROR = None @@ -30,12 +31,14 @@ except Exception as e: # pragma: no cover fused_conv_window_scatter_multi = None fused_conv_window_scatter_with_mask = None fused_mamba_state_scatter_with_mask = None + scatter_mamba_states_after_mtp_verify = None _FUSED_IMPORT_ERROR = e from sglang.srt.mem_cache.layout.token_major import ( build_mamba_entry_views, mamba_entry_bytes, ) +from sglang.srt.mem_cache.memory_pool import MambaPool from sglang.srt.utils import get_device from sglang.srt.utils.common import get_device_module from sglang.test.test_utils import CustomTestCase @@ -539,5 +542,29 @@ class TestFusedConvWindowScatterMulti(CustomTestCase): self._run(num_types=2, n2=4) +@unittest.skipUnless( + DEVICE is not None, + f"verify scatter needs one of {TRITON_DEVICES}", +) +class TestScatterAfterVerifyZeroWidthConv(CustomTestCase): + def test_zero_width_conv_pool(self): + """A model without conv state (Lightning attention) must commit an MTP verify.""" + caches = MambaPool.SpeculativeState( + conv=[torch.empty(2, 8, 0, 0, dtype=torch.bfloat16, device=DEVICE)], + temporal=torch.empty(2, 8, 0, device=DEVICE), + intermediate_ssm=torch.empty(2, 2, 3, 0, device=DEVICE), + intermediate_conv_window=[ + torch.empty(2, 2, 3, 0, 0, dtype=torch.bfloat16, device=DEVICE) + ], + ) + scatter_mamba_states_after_mtp_verify( + mamba_caches=caches, + state_indices_tensor=torch.tensor([1, 5], device=DEVICE), + last_correct_step_indices=torch.tensor([0, 2], device=DEVICE), + mamba_track_indices=None, + mamba_steps_to_track=None, + ) + + if __name__ == "__main__": # pragma: no cover unittest.main()