[Mamba] Warm cache COW kernel before serving (#36266)

Signed-off-by: Yangmin Li <yangminl@nvidia.com>
Co-authored-by: Po-Han Huang (NVIDIA) <53919306+nvpohanh@users.noreply.github.com>
This commit is contained in:
YAMY
2026-10-02 18:52:37 +08:00
committed by GitHub
co-authored by Po-Han Huang
parent 1db33bc367
commit 895525f062
3 changed files with 43 additions and 0 deletions
@@ -171,3 +171,9 @@ def fused_copy_conv_slots(
MAX_FEAT_BLOCKS=desc.max_feat_blocks,
BLOCK=_BLOCK,
)
def warmup_fused_copy_conv_slots(desc: ConvSlotDescriptor) -> None:
"""Warm the cache-hit COW specialization with two free pool slots."""
indices = torch.tensor([0, 1], dtype=torch.int32, device=desc.ptr.device)
fused_copy_conv_slots(desc, indices[:1], indices[1:])
@@ -28,6 +28,7 @@ import dataclasses
import logging
import math
import os
import time
from contextlib import contextmanager, nullcontext
from dataclasses import dataclass, fields
from functools import cached_property
@@ -962,6 +963,7 @@ class MambaPool:
# attn_tp_size (GDN: [key_dim, key_dim, value_dim]); None otherwise.
self.conv_shard_groups = getattr(cache_params.shape, "conv_shard_groups", None)
self.conv_slice_axis = getattr(cache_params.shape, "conv_slice_axis", 0)
self._warmup_fused_copy_slot_kernel()
def get_speculative_mamba2_params_all_layers(self) -> SpeculativeState:
assert isinstance(self.mamba_cache, self.SpeculativeState)
@@ -996,6 +998,21 @@ class MambaPool:
def _should_fuse_slot_ops(self) -> bool:
return self._conv_fuse_ok and not envs.SGLANG_DISABLE_FUSED_MAMBA_SLOT_OPS.get()
def _warmup_fused_copy_slot_kernel(self) -> None:
"""Warm the cache-hit-only fused COW kernel during pool initialization."""
if not self._should_fuse_slot_ops() or self.mamba_cache.conv[0].shape[1] < 2:
return
from sglang.srt.mem_cache.mamba_slot_fused import (
warmup_fused_copy_conv_slots,
)
started = time.perf_counter()
warmup_fused_copy_conv_slots(self._conv_slot_desc)
logger.info(
"Warmed fused Mamba slot copy kernel in %.3f ms",
(time.perf_counter() - started) * 1000,
)
def clear_slots(self, indices: torch.Tensor):
"""Zero out mamba state at the given pool indices. Must run on forward stream."""
for sibling in self._slot_siblings: