[Mamba] Warm cache COW kernel before serving (#36266)
Signed-off-by: Yangmin Li <yangminl@nvidia.com> Co-authored-by: Po-Han Huang (NVIDIA) <53919306+nvpohanh@users.noreply.github.com>
This commit is contained in:
@@ -171,3 +171,9 @@ def fused_copy_conv_slots(
|
||||
MAX_FEAT_BLOCKS=desc.max_feat_blocks,
|
||||
BLOCK=_BLOCK,
|
||||
)
|
||||
|
||||
|
||||
def warmup_fused_copy_conv_slots(desc: ConvSlotDescriptor) -> None:
|
||||
"""Warm the cache-hit COW specialization with two free pool slots."""
|
||||
indices = torch.tensor([0, 1], dtype=torch.int32, device=desc.ptr.device)
|
||||
fused_copy_conv_slots(desc, indices[:1], indices[1:])
|
||||
|
||||
@@ -28,6 +28,7 @@ import dataclasses
|
||||
import logging
|
||||
import math
|
||||
import os
|
||||
import time
|
||||
from contextlib import contextmanager, nullcontext
|
||||
from dataclasses import dataclass, fields
|
||||
from functools import cached_property
|
||||
@@ -962,6 +963,7 @@ class MambaPool:
|
||||
# attn_tp_size (GDN: [key_dim, key_dim, value_dim]); None otherwise.
|
||||
self.conv_shard_groups = getattr(cache_params.shape, "conv_shard_groups", None)
|
||||
self.conv_slice_axis = getattr(cache_params.shape, "conv_slice_axis", 0)
|
||||
self._warmup_fused_copy_slot_kernel()
|
||||
|
||||
def get_speculative_mamba2_params_all_layers(self) -> SpeculativeState:
|
||||
assert isinstance(self.mamba_cache, self.SpeculativeState)
|
||||
@@ -996,6 +998,21 @@ class MambaPool:
|
||||
def _should_fuse_slot_ops(self) -> bool:
|
||||
return self._conv_fuse_ok and not envs.SGLANG_DISABLE_FUSED_MAMBA_SLOT_OPS.get()
|
||||
|
||||
def _warmup_fused_copy_slot_kernel(self) -> None:
|
||||
"""Warm the cache-hit-only fused COW kernel during pool initialization."""
|
||||
if not self._should_fuse_slot_ops() or self.mamba_cache.conv[0].shape[1] < 2:
|
||||
return
|
||||
from sglang.srt.mem_cache.mamba_slot_fused import (
|
||||
warmup_fused_copy_conv_slots,
|
||||
)
|
||||
|
||||
started = time.perf_counter()
|
||||
warmup_fused_copy_conv_slots(self._conv_slot_desc)
|
||||
logger.info(
|
||||
"Warmed fused Mamba slot copy kernel in %.3f ms",
|
||||
(time.perf_counter() - started) * 1000,
|
||||
)
|
||||
|
||||
def clear_slots(self, indices: torch.Tensor):
|
||||
"""Zero out mamba state at the given pool indices. Must run on forward stream."""
|
||||
for sibling in self._slot_siblings:
|
||||
|
||||
Reference in New Issue
Block a user