From 895525f062a303bc7f0457d0d4bef202a4ea2397 Mon Sep 17 00:00:00 2001 From: YAMY <74099316+YAMY1234@users.noreply.github.com> Date: Fri, 2 Oct 2026 05:52:37 -0500 Subject: [PATCH] [Mamba] Warm cache COW kernel before serving (#36266) Signed-off-by: Yangmin Li Co-authored-by: Po-Han Huang (NVIDIA) <53919306+nvpohanh@users.noreply.github.com> --- .../sglang/srt/mem_cache/mamba_slot_fused.py | 6 ++++++ python/sglang/srt/mem_cache/memory_pool.py | 17 ++++++++++++++++ .../layers/mamba/test_mamba_slot_fused.py | 20 +++++++++++++++++++ 3 files changed, 43 insertions(+) diff --git a/python/sglang/srt/mem_cache/mamba_slot_fused.py b/python/sglang/srt/mem_cache/mamba_slot_fused.py index 5689d05057..f51ab67db1 100644 --- a/python/sglang/srt/mem_cache/mamba_slot_fused.py +++ b/python/sglang/srt/mem_cache/mamba_slot_fused.py @@ -171,3 +171,9 @@ def fused_copy_conv_slots( MAX_FEAT_BLOCKS=desc.max_feat_blocks, BLOCK=_BLOCK, ) + + +def warmup_fused_copy_conv_slots(desc: ConvSlotDescriptor) -> None: + """Warm the cache-hit COW specialization with two free pool slots.""" + indices = torch.tensor([0, 1], dtype=torch.int32, device=desc.ptr.device) + fused_copy_conv_slots(desc, indices[:1], indices[1:]) diff --git a/python/sglang/srt/mem_cache/memory_pool.py b/python/sglang/srt/mem_cache/memory_pool.py index a8d521304f..ed9b19f96f 100644 --- a/python/sglang/srt/mem_cache/memory_pool.py +++ b/python/sglang/srt/mem_cache/memory_pool.py @@ -28,6 +28,7 @@ import dataclasses import logging import math import os +import time from contextlib import contextmanager, nullcontext from dataclasses import dataclass, fields from functools import cached_property @@ -962,6 +963,7 @@ class MambaPool: # attn_tp_size (GDN: [key_dim, key_dim, value_dim]); None otherwise. self.conv_shard_groups = getattr(cache_params.shape, "conv_shard_groups", None) self.conv_slice_axis = getattr(cache_params.shape, "conv_slice_axis", 0) + self._warmup_fused_copy_slot_kernel() def get_speculative_mamba2_params_all_layers(self) -> SpeculativeState: assert isinstance(self.mamba_cache, self.SpeculativeState) @@ -996,6 +998,21 @@ class MambaPool: def _should_fuse_slot_ops(self) -> bool: return self._conv_fuse_ok and not envs.SGLANG_DISABLE_FUSED_MAMBA_SLOT_OPS.get() + def _warmup_fused_copy_slot_kernel(self) -> None: + """Warm the cache-hit-only fused COW kernel during pool initialization.""" + if not self._should_fuse_slot_ops() or self.mamba_cache.conv[0].shape[1] < 2: + return + from sglang.srt.mem_cache.mamba_slot_fused import ( + warmup_fused_copy_conv_slots, + ) + + started = time.perf_counter() + warmup_fused_copy_conv_slots(self._conv_slot_desc) + logger.info( + "Warmed fused Mamba slot copy kernel in %.3f ms", + (time.perf_counter() - started) * 1000, + ) + def clear_slots(self, indices: torch.Tensor): """Zero out mamba state at the given pool indices. Must run on forward stream.""" for sibling in self._slot_siblings: diff --git a/test/registered/layers/mamba/test_mamba_slot_fused.py b/test/registered/layers/mamba/test_mamba_slot_fused.py index 484518be42..a0bf16ad44 100644 --- a/test/registered/layers/mamba/test_mamba_slot_fused.py +++ b/test/registered/layers/mamba/test_mamba_slot_fused.py @@ -11,6 +11,7 @@ descriptor packs ``data_ptr()`` values, which only overflows off CUDA (#35047). """ import unittest +from unittest.mock import patch import torch @@ -18,6 +19,7 @@ from sglang.srt.mem_cache.mamba_slot_fused import ( build_conv_slot_descriptor, fused_clear_conv_slots, fused_copy_conv_slots, + warmup_fused_copy_conv_slots, ) from sglang.srt.utils import get_device from sglang.srt.utils.common import get_device_module @@ -57,6 +59,24 @@ CONFIGS = [ ] +class TestMambaSlotFusedWarmup(CustomTestCase): + def test_warmup_uses_disjoint_one_slot_cow_indices(self): + convs = [torch.zeros(1, 2, CONV_LEN, 8, dtype=torch.bfloat16)] + desc = build_conv_slot_descriptor(convs) + with patch( + "sglang.srt.mem_cache.mamba_slot_fused.fused_copy_conv_slots" + ) as copy: + warmup_fused_copy_conv_slots(desc) + + copy.assert_called_once() + called_desc, src, dst = copy.call_args.args + self.assertIs(called_desc, desc) + self.assertEqual(src.dtype, torch.int32) + self.assertEqual(dst.dtype, torch.int32) + self.assertEqual(src.tolist(), [0]) + self.assertEqual(dst.tolist(), [1]) + + def _make_convs(dims, num_layers, pool, device, seed): g = torch.Generator(device=device).manual_seed(seed) return [