[Perf] mamba: Copy tracked decode states at HBM bandwidth in one persistent launch (#45213)

Co-authored-by: metamergebot <metamergebot@users.noreply.github.com>
Co-authored-by: YavorGIvanov <YavorGIvanov@users.noreply.github.com>
This commit is contained in:
metamergebot
2026-10-20 16:52:15 -07:00
committed by GitHub
co-authored by metamergebot YavorGIvanov
parent 5b1dbc2c80
commit b326e8349d
2 changed files with 332 additions and 141 deletions
@@ -6,11 +6,14 @@ This kernel replaces the expensive advanced indexing operations in
avoiding multiple `index_elementwise_kernel` launches.
"""
import functools
import torch
import triton
import triton.language as tl
from sglang.kernels.ops.memory.ptr_table import make_ptr_table
from sglang.srt.utils.common import get_device_core_count
def _require_entry_contiguous_dst(
@@ -33,74 +36,157 @@ def _require_entry_contiguous_dst(
@triton.jit
def _copy_track_chunk(
ptr,
src_off,
dst_off,
start,
numel: tl.constexpr,
CHUNK: tl.constexpr,
BLOCK: tl.constexpr,
):
for off in tl.static_range(0, CHUNK, BLOCK):
offsets = start + off + tl.arange(0, BLOCK)
mask = offsets < numel
data = tl.load(ptr + src_off + offsets, mask=mask)
tl.store(ptr + dst_off + offsets, data, mask=mask)
@triton.jit(do_not_specialize=["batch_size", "num_layers"])
def track_mamba_state_if_needed_kernel(
conv_states_ptr,
ssm_states_ptr,
cache_indices_ptr,
mamba_track_mask_ptr,
mamba_track_indices_ptr,
conv_state_stride_0, # stride for first dimension (batch/pool index)
ssm_state_stride_0, # stride for first dimension (batch/pool index)
conv_state_numel_per_row: tl.constexpr, # total elements per row
ssm_state_numel_per_row: tl.constexpr, # total elements per row
conv_layer_stride,
conv_row_stride,
ssm_layer_stride,
ssm_row_stride,
batch_size,
num_layers,
conv_state_numel_per_row: tl.constexpr,
ssm_state_numel_per_row: tl.constexpr,
CONV_CHUNK: tl.constexpr,
SSM_CHUNK: tl.constexpr,
NUM_CONV_CHUNKS: tl.constexpr,
NUM_SSM_CHUNKS: tl.constexpr,
BS_POW2: tl.constexpr,
BLOCK_SIZE: tl.constexpr,
check_freed_slots: tl.constexpr, # only the shared/unified KV pool emits -1
):
"""
Track conv_states and ssm_states rows based on track mask.
For each batch row with mamba_track_mask set, copy its conv and ssm state
rows from cache_indices[i] to mamba_track_indices[i], in every layer.
This kernel replaces a Python loop that copies state tensors for mamba attention.
For each batch element, if the track mask is True, it copies the entire row from
the source index (cache_indices[i]) to the destination index (mamba_track_indices[i]).
Grid: (batch_size,)
Each block handles one batch element, using multiple threads to copy data in parallel.
Persistent grid: every program reads the whole mask, so the tracked rows
are compacted in-kernel and only their (row, layer, chunk) work items are
spread over the grid. A step with no tracked row reads one mask vector per
program and exits; one tracked row runs at HBM bandwidth on all SMs.
"""
batch_idx = tl.program_id(0)
pid = tl.program_id(0)
num_programs = tl.num_programs(0)
rows = tl.arange(0, BS_POW2)
track = tl.load(mamba_track_mask_ptr + rows, mask=rows < batch_size, other=0).to(
tl.int32
)
num_tracked = tl.sum(track, axis=0)
track_rank = tl.cumsum(track, axis=0)
# Load the copy mask for this batch element
track_mask = tl.load(mamba_track_mask_ptr + batch_idx)
CHUNKS: tl.constexpr = NUM_CONV_CHUNKS + NUM_SSM_CHUNKS
items_per_row = CHUNKS * num_layers
for item in range(pid, num_tracked * items_per_row, num_programs):
rank = item // items_per_row
rem = item - rank * items_per_row
layer_idx = (rem // CHUNKS).to(tl.int64)
chunk = rem % CHUNKS
batch_idx = tl.sum(
tl.where((track_rank == rank + 1) & (track != 0), rows, 0), axis=0
)
# int64 before the row stride multiply: the page-granularity envelope
# layout makes the row stride large enough for int32 to overflow.
src_idx = tl.load(cache_indices_ptr + batch_idx).to(tl.int64)
dst_idx = tl.load(mamba_track_indices_ptr + batch_idx).to(tl.int64)
# Skip freed slots (-1): `state_ptr + (-1)*stride` would fault.
if (not check_freed_slots) or ((src_idx >= 0) & (dst_idx >= 0)):
if chunk < NUM_CONV_CHUNKS:
_copy_track_chunk(
conv_states_ptr + layer_idx * conv_layer_stride,
src_idx * conv_row_stride,
dst_idx * conv_row_stride,
chunk * CONV_CHUNK,
conv_state_numel_per_row,
CONV_CHUNK,
BLOCK_SIZE,
)
else:
_copy_track_chunk(
ssm_states_ptr + layer_idx * ssm_layer_stride,
src_idx * ssm_row_stride,
dst_idx * ssm_row_stride,
(chunk - NUM_CONV_CHUNKS) * SSM_CHUNK,
ssm_state_numel_per_row,
SSM_CHUNK,
BLOCK_SIZE,
)
# Early exit if we don't need to track
if not track_mask:
_TRACK_BLOCK_SIZE = 8192
_TRACK_CHUNK_BYTES = 32768
_TRACK_PROGRAMS_PER_SM = 4
@functools.lru_cache(maxsize=None)
def _num_sms(device_index: int) -> int:
return get_device_core_count(device_index) or 132
def _launch_track_mamba_states(
conv_states: torch.Tensor,
ssm_states: torch.Tensor,
cache_indices: torch.Tensor,
mamba_track_mask: torch.Tensor,
mamba_track_indices: torch.Tensor,
batch_size: int,
num_layers: int,
conv_strides: tuple,
ssm_strides: tuple,
conv_state_numel_per_row: int,
ssm_state_numel_per_row: int,
check_freed_slots: bool,
):
if batch_size == 0:
return
# Cast indices to int64 before they multiply the row stride. The
# page-granularity envelope layout makes the conv/ssm row stride large
# (stride_0 = entry_bytes / itemsize), so an int32 `idx * stride_0` can
# overflow for moderately large idx and wrap to an illegal address. int64 is
# harmless for the small-stride (per-layer) case.
src_idx = tl.load(cache_indices_ptr + batch_idx).to(tl.int64)
dst_idx = tl.load(mamba_track_indices_ptr + batch_idx).to(tl.int64)
# Skip freed slots (-1): `state_ptr + (-1)*stride` would fault. Only the unified
# pool emits -1 tombstones (from the v2p translate); compiled out for static.
if check_freed_slots:
if src_idx < 0 or dst_idx < 0:
return
# Copy conv_states
# Each thread handles BLOCK_SIZE elements
for offset in range(0, conv_state_numel_per_row, BLOCK_SIZE):
element_indices = offset + tl.arange(0, BLOCK_SIZE)
mask = element_indices < conv_state_numel_per_row
src_ptr = conv_states_ptr + src_idx * conv_state_stride_0 + element_indices
dst_ptr = conv_states_ptr + dst_idx * conv_state_stride_0 + element_indices
data = tl.load(src_ptr, mask=mask, other=0.0)
tl.store(dst_ptr, data, mask=mask)
# Copy ssm_states
for offset in range(0, ssm_state_numel_per_row, BLOCK_SIZE):
element_indices = offset + tl.arange(0, BLOCK_SIZE)
mask = element_indices < ssm_state_numel_per_row
src_ptr = ssm_states_ptr + src_idx * ssm_state_stride_0 + element_indices
dst_ptr = ssm_states_ptr + dst_idx * ssm_state_stride_0 + element_indices
data = tl.load(src_ptr, mask=mask, other=0.0)
tl.store(dst_ptr, data, mask=mask)
conv_chunk = max(
_TRACK_BLOCK_SIZE, _TRACK_CHUNK_BYTES // conv_states.element_size()
)
ssm_chunk = max(_TRACK_BLOCK_SIZE, _TRACK_CHUNK_BYTES // ssm_states.element_size())
num_conv_chunks = triton.cdiv(conv_state_numel_per_row, conv_chunk)
num_ssm_chunks = triton.cdiv(ssm_state_numel_per_row, ssm_chunk)
max_items = batch_size * num_layers * (num_conv_chunks + num_ssm_chunks)
device_index = conv_states.device.index or 0
grid = (min(max_items, _TRACK_PROGRAMS_PER_SM * _num_sms(device_index)),)
track_mamba_state_if_needed_kernel[grid](
conv_states,
ssm_states,
cache_indices,
mamba_track_mask,
mamba_track_indices,
*conv_strides,
*ssm_strides,
batch_size,
num_layers,
conv_state_numel_per_row,
ssm_state_numel_per_row,
conv_chunk,
ssm_chunk,
num_conv_chunks,
num_ssm_chunks,
max(16, triton.next_power_of_2(batch_size)),
_TRACK_BLOCK_SIZE,
check_freed_slots,
num_warps=8,
)
def track_mamba_states_if_needed(
@@ -123,25 +209,18 @@ def track_mamba_states_if_needed(
mamba_track_indices: Indices to track for each batch element [batch_size]
batch_size: Number of batch elements
"""
conv_state_numel_per_row = conv_states[0].numel()
ssm_state_numel_per_row = ssm_states[0].numel()
# Choose BLOCK_SIZE based on the size of the data
BLOCK_SIZE = 1024
# Launch kernel with batch_size blocks
grid = (batch_size,)
track_mamba_state_if_needed_kernel[grid](
_launch_track_mamba_states(
conv_states,
ssm_states,
cache_indices,
mamba_track_mask,
mamba_track_indices,
conv_states.stride(0),
ssm_states.stride(0),
conv_state_numel_per_row,
ssm_state_numel_per_row,
BLOCK_SIZE,
batch_size,
1,
(0, conv_states.stride(0)),
(0, ssm_states.stride(0)),
conv_states[0].numel(),
ssm_states[0].numel(),
check_freed_slots,
)
@@ -832,65 +911,6 @@ def fused_commit_track_indices(
return last_correct_step_indices, (mamba_steps_to_track if has_track else None)
@triton.jit
def track_mamba_states_all_layers_kernel(
conv_states_ptr, # [num_layers, pool_size, ...] full conv pool
ssm_states_ptr, # [num_layers, pool_size, ...] full ssm pool
cache_indices_ptr,
mamba_track_mask_ptr,
mamba_track_indices_ptr,
conv_layer_stride,
conv_row_stride,
ssm_layer_stride,
ssm_row_stride,
batch_size,
conv_state_numel_per_row: tl.constexpr,
ssm_state_numel_per_row: tl.constexpr,
BLOCK_SIZE: tl.constexpr,
check_freed_slots: tl.constexpr,
):
"""All-layers variant of track_mamba_state_if_needed_kernel: one launch
covers every mamba layer (grid = num_layers * batch_size) instead of one
launch per layer. The track mask / source / destination indices are shared
across layers, so a single launch at the end of the step is equivalent to
the per-layer launches (each layer's state is final by then)."""
pid = tl.program_id(0)
layer_idx = (pid // batch_size).to(tl.int64)
batch_idx = pid % batch_size
track_mask = tl.load(mamba_track_mask_ptr + batch_idx)
if not track_mask:
return
src_idx = tl.load(cache_indices_ptr + batch_idx).to(tl.int64)
dst_idx = tl.load(mamba_track_indices_ptr + batch_idx).to(tl.int64)
if check_freed_slots:
if src_idx < 0 or dst_idx < 0:
return
conv_base = conv_states_ptr + layer_idx * conv_layer_stride
for offset in range(0, conv_state_numel_per_row, BLOCK_SIZE):
element_indices = offset + tl.arange(0, BLOCK_SIZE)
mask = element_indices < conv_state_numel_per_row
data = tl.load(
conv_base + src_idx * conv_row_stride + element_indices,
mask=mask,
other=0.0,
)
tl.store(
conv_base + dst_idx * conv_row_stride + element_indices, data, mask=mask
)
ssm_base = ssm_states_ptr + layer_idx * ssm_layer_stride
for offset in range(0, ssm_state_numel_per_row, BLOCK_SIZE):
element_indices = offset + tl.arange(0, BLOCK_SIZE)
mask = element_indices < ssm_state_numel_per_row
data = tl.load(
ssm_base + src_idx * ssm_row_stride + element_indices, mask=mask, other=0.0
)
tl.store(ssm_base + dst_idx * ssm_row_stride + element_indices, data, mask=mask)
def track_mamba_states_all_layers(
conv_states_pool: torch.Tensor,
ssm_states_pool: torch.Tensor,
@@ -904,27 +924,21 @@ def track_mamba_states_all_layers(
conv_states_pool / ssm_states_pool are the full [num_layers, pool_size,
...] pools; per-row copy semantics are identical to
track_mamba_states_if_needed applied per layer.
track_mamba_states_if_needed applied per layer. The track mask / source /
destination indices are shared across layers, so a single launch once
every layer's state is final is equivalent to the per-layer launches.
"""
num_layers = conv_states_pool.shape[0]
conv_state_numel_per_row = conv_states_pool[0, 0].numel()
ssm_state_numel_per_row = ssm_states_pool[0, 0].numel()
BLOCK_SIZE = 1024
grid = (num_layers * batch_size,)
track_mamba_states_all_layers_kernel[grid](
_launch_track_mamba_states(
conv_states_pool,
ssm_states_pool,
cache_indices,
mamba_track_mask,
mamba_track_indices,
conv_states_pool.stride(0),
conv_states_pool.stride(1),
ssm_states_pool.stride(0),
ssm_states_pool.stride(1),
batch_size,
conv_state_numel_per_row,
ssm_state_numel_per_row,
BLOCK_SIZE,
conv_states_pool.shape[0],
(conv_states_pool.stride(0), conv_states_pool.stride(1)),
(ssm_states_pool.stride(0), ssm_states_pool.stride(1)),
conv_states_pool[0, 0].numel(),
ssm_states_pool[0, 0].numel(),
check_freed_slots,
)