[Perf] mamba: Copy tracked decode states at HBM bandwidth in one persistent launch (#45213)
Co-authored-by: metamergebot <metamergebot@users.noreply.github.com> Co-authored-by: YavorGIvanov <YavorGIvanov@users.noreply.github.com>
This commit is contained in:
co-authored by
metamergebot
YavorGIvanov
parent
5b1dbc2c80
commit
b326e8349d
@@ -6,11 +6,14 @@ This kernel replaces the expensive advanced indexing operations in
|
||||
avoiding multiple `index_elementwise_kernel` launches.
|
||||
"""
|
||||
|
||||
import functools
|
||||
|
||||
import torch
|
||||
import triton
|
||||
import triton.language as tl
|
||||
|
||||
from sglang.kernels.ops.memory.ptr_table import make_ptr_table
|
||||
from sglang.srt.utils.common import get_device_core_count
|
||||
|
||||
|
||||
def _require_entry_contiguous_dst(
|
||||
@@ -33,74 +36,157 @@ def _require_entry_contiguous_dst(
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _copy_track_chunk(
|
||||
ptr,
|
||||
src_off,
|
||||
dst_off,
|
||||
start,
|
||||
numel: tl.constexpr,
|
||||
CHUNK: tl.constexpr,
|
||||
BLOCK: tl.constexpr,
|
||||
):
|
||||
for off in tl.static_range(0, CHUNK, BLOCK):
|
||||
offsets = start + off + tl.arange(0, BLOCK)
|
||||
mask = offsets < numel
|
||||
data = tl.load(ptr + src_off + offsets, mask=mask)
|
||||
tl.store(ptr + dst_off + offsets, data, mask=mask)
|
||||
|
||||
|
||||
@triton.jit(do_not_specialize=["batch_size", "num_layers"])
|
||||
def track_mamba_state_if_needed_kernel(
|
||||
conv_states_ptr,
|
||||
ssm_states_ptr,
|
||||
cache_indices_ptr,
|
||||
mamba_track_mask_ptr,
|
||||
mamba_track_indices_ptr,
|
||||
conv_state_stride_0, # stride for first dimension (batch/pool index)
|
||||
ssm_state_stride_0, # stride for first dimension (batch/pool index)
|
||||
conv_state_numel_per_row: tl.constexpr, # total elements per row
|
||||
ssm_state_numel_per_row: tl.constexpr, # total elements per row
|
||||
conv_layer_stride,
|
||||
conv_row_stride,
|
||||
ssm_layer_stride,
|
||||
ssm_row_stride,
|
||||
batch_size,
|
||||
num_layers,
|
||||
conv_state_numel_per_row: tl.constexpr,
|
||||
ssm_state_numel_per_row: tl.constexpr,
|
||||
CONV_CHUNK: tl.constexpr,
|
||||
SSM_CHUNK: tl.constexpr,
|
||||
NUM_CONV_CHUNKS: tl.constexpr,
|
||||
NUM_SSM_CHUNKS: tl.constexpr,
|
||||
BS_POW2: tl.constexpr,
|
||||
BLOCK_SIZE: tl.constexpr,
|
||||
check_freed_slots: tl.constexpr, # only the shared/unified KV pool emits -1
|
||||
):
|
||||
"""
|
||||
Track conv_states and ssm_states rows based on track mask.
|
||||
For each batch row with mamba_track_mask set, copy its conv and ssm state
|
||||
rows from cache_indices[i] to mamba_track_indices[i], in every layer.
|
||||
|
||||
This kernel replaces a Python loop that copies state tensors for mamba attention.
|
||||
For each batch element, if the track mask is True, it copies the entire row from
|
||||
the source index (cache_indices[i]) to the destination index (mamba_track_indices[i]).
|
||||
|
||||
Grid: (batch_size,)
|
||||
Each block handles one batch element, using multiple threads to copy data in parallel.
|
||||
Persistent grid: every program reads the whole mask, so the tracked rows
|
||||
are compacted in-kernel and only their (row, layer, chunk) work items are
|
||||
spread over the grid. A step with no tracked row reads one mask vector per
|
||||
program and exits; one tracked row runs at HBM bandwidth on all SMs.
|
||||
"""
|
||||
batch_idx = tl.program_id(0)
|
||||
pid = tl.program_id(0)
|
||||
num_programs = tl.num_programs(0)
|
||||
rows = tl.arange(0, BS_POW2)
|
||||
track = tl.load(mamba_track_mask_ptr + rows, mask=rows < batch_size, other=0).to(
|
||||
tl.int32
|
||||
)
|
||||
num_tracked = tl.sum(track, axis=0)
|
||||
track_rank = tl.cumsum(track, axis=0)
|
||||
|
||||
# Load the copy mask for this batch element
|
||||
track_mask = tl.load(mamba_track_mask_ptr + batch_idx)
|
||||
CHUNKS: tl.constexpr = NUM_CONV_CHUNKS + NUM_SSM_CHUNKS
|
||||
items_per_row = CHUNKS * num_layers
|
||||
for item in range(pid, num_tracked * items_per_row, num_programs):
|
||||
rank = item // items_per_row
|
||||
rem = item - rank * items_per_row
|
||||
layer_idx = (rem // CHUNKS).to(tl.int64)
|
||||
chunk = rem % CHUNKS
|
||||
batch_idx = tl.sum(
|
||||
tl.where((track_rank == rank + 1) & (track != 0), rows, 0), axis=0
|
||||
)
|
||||
# int64 before the row stride multiply: the page-granularity envelope
|
||||
# layout makes the row stride large enough for int32 to overflow.
|
||||
src_idx = tl.load(cache_indices_ptr + batch_idx).to(tl.int64)
|
||||
dst_idx = tl.load(mamba_track_indices_ptr + batch_idx).to(tl.int64)
|
||||
# Skip freed slots (-1): `state_ptr + (-1)*stride` would fault.
|
||||
if (not check_freed_slots) or ((src_idx >= 0) & (dst_idx >= 0)):
|
||||
if chunk < NUM_CONV_CHUNKS:
|
||||
_copy_track_chunk(
|
||||
conv_states_ptr + layer_idx * conv_layer_stride,
|
||||
src_idx * conv_row_stride,
|
||||
dst_idx * conv_row_stride,
|
||||
chunk * CONV_CHUNK,
|
||||
conv_state_numel_per_row,
|
||||
CONV_CHUNK,
|
||||
BLOCK_SIZE,
|
||||
)
|
||||
else:
|
||||
_copy_track_chunk(
|
||||
ssm_states_ptr + layer_idx * ssm_layer_stride,
|
||||
src_idx * ssm_row_stride,
|
||||
dst_idx * ssm_row_stride,
|
||||
(chunk - NUM_CONV_CHUNKS) * SSM_CHUNK,
|
||||
ssm_state_numel_per_row,
|
||||
SSM_CHUNK,
|
||||
BLOCK_SIZE,
|
||||
)
|
||||
|
||||
# Early exit if we don't need to track
|
||||
if not track_mask:
|
||||
|
||||
_TRACK_BLOCK_SIZE = 8192
|
||||
_TRACK_CHUNK_BYTES = 32768
|
||||
_TRACK_PROGRAMS_PER_SM = 4
|
||||
|
||||
|
||||
@functools.lru_cache(maxsize=None)
|
||||
def _num_sms(device_index: int) -> int:
|
||||
return get_device_core_count(device_index) or 132
|
||||
|
||||
|
||||
def _launch_track_mamba_states(
|
||||
conv_states: torch.Tensor,
|
||||
ssm_states: torch.Tensor,
|
||||
cache_indices: torch.Tensor,
|
||||
mamba_track_mask: torch.Tensor,
|
||||
mamba_track_indices: torch.Tensor,
|
||||
batch_size: int,
|
||||
num_layers: int,
|
||||
conv_strides: tuple,
|
||||
ssm_strides: tuple,
|
||||
conv_state_numel_per_row: int,
|
||||
ssm_state_numel_per_row: int,
|
||||
check_freed_slots: bool,
|
||||
):
|
||||
if batch_size == 0:
|
||||
return
|
||||
|
||||
# Cast indices to int64 before they multiply the row stride. The
|
||||
# page-granularity envelope layout makes the conv/ssm row stride large
|
||||
# (stride_0 = entry_bytes / itemsize), so an int32 `idx * stride_0` can
|
||||
# overflow for moderately large idx and wrap to an illegal address. int64 is
|
||||
# harmless for the small-stride (per-layer) case.
|
||||
src_idx = tl.load(cache_indices_ptr + batch_idx).to(tl.int64)
|
||||
dst_idx = tl.load(mamba_track_indices_ptr + batch_idx).to(tl.int64)
|
||||
|
||||
# Skip freed slots (-1): `state_ptr + (-1)*stride` would fault. Only the unified
|
||||
# pool emits -1 tombstones (from the v2p translate); compiled out for static.
|
||||
if check_freed_slots:
|
||||
if src_idx < 0 or dst_idx < 0:
|
||||
return
|
||||
|
||||
# Copy conv_states
|
||||
# Each thread handles BLOCK_SIZE elements
|
||||
for offset in range(0, conv_state_numel_per_row, BLOCK_SIZE):
|
||||
element_indices = offset + tl.arange(0, BLOCK_SIZE)
|
||||
mask = element_indices < conv_state_numel_per_row
|
||||
|
||||
src_ptr = conv_states_ptr + src_idx * conv_state_stride_0 + element_indices
|
||||
dst_ptr = conv_states_ptr + dst_idx * conv_state_stride_0 + element_indices
|
||||
|
||||
data = tl.load(src_ptr, mask=mask, other=0.0)
|
||||
tl.store(dst_ptr, data, mask=mask)
|
||||
|
||||
# Copy ssm_states
|
||||
for offset in range(0, ssm_state_numel_per_row, BLOCK_SIZE):
|
||||
element_indices = offset + tl.arange(0, BLOCK_SIZE)
|
||||
mask = element_indices < ssm_state_numel_per_row
|
||||
|
||||
src_ptr = ssm_states_ptr + src_idx * ssm_state_stride_0 + element_indices
|
||||
dst_ptr = ssm_states_ptr + dst_idx * ssm_state_stride_0 + element_indices
|
||||
|
||||
data = tl.load(src_ptr, mask=mask, other=0.0)
|
||||
tl.store(dst_ptr, data, mask=mask)
|
||||
conv_chunk = max(
|
||||
_TRACK_BLOCK_SIZE, _TRACK_CHUNK_BYTES // conv_states.element_size()
|
||||
)
|
||||
ssm_chunk = max(_TRACK_BLOCK_SIZE, _TRACK_CHUNK_BYTES // ssm_states.element_size())
|
||||
num_conv_chunks = triton.cdiv(conv_state_numel_per_row, conv_chunk)
|
||||
num_ssm_chunks = triton.cdiv(ssm_state_numel_per_row, ssm_chunk)
|
||||
max_items = batch_size * num_layers * (num_conv_chunks + num_ssm_chunks)
|
||||
device_index = conv_states.device.index or 0
|
||||
grid = (min(max_items, _TRACK_PROGRAMS_PER_SM * _num_sms(device_index)),)
|
||||
track_mamba_state_if_needed_kernel[grid](
|
||||
conv_states,
|
||||
ssm_states,
|
||||
cache_indices,
|
||||
mamba_track_mask,
|
||||
mamba_track_indices,
|
||||
*conv_strides,
|
||||
*ssm_strides,
|
||||
batch_size,
|
||||
num_layers,
|
||||
conv_state_numel_per_row,
|
||||
ssm_state_numel_per_row,
|
||||
conv_chunk,
|
||||
ssm_chunk,
|
||||
num_conv_chunks,
|
||||
num_ssm_chunks,
|
||||
max(16, triton.next_power_of_2(batch_size)),
|
||||
_TRACK_BLOCK_SIZE,
|
||||
check_freed_slots,
|
||||
num_warps=8,
|
||||
)
|
||||
|
||||
|
||||
def track_mamba_states_if_needed(
|
||||
@@ -123,25 +209,18 @@ def track_mamba_states_if_needed(
|
||||
mamba_track_indices: Indices to track for each batch element [batch_size]
|
||||
batch_size: Number of batch elements
|
||||
"""
|
||||
conv_state_numel_per_row = conv_states[0].numel()
|
||||
ssm_state_numel_per_row = ssm_states[0].numel()
|
||||
|
||||
# Choose BLOCK_SIZE based on the size of the data
|
||||
BLOCK_SIZE = 1024
|
||||
|
||||
# Launch kernel with batch_size blocks
|
||||
grid = (batch_size,)
|
||||
track_mamba_state_if_needed_kernel[grid](
|
||||
_launch_track_mamba_states(
|
||||
conv_states,
|
||||
ssm_states,
|
||||
cache_indices,
|
||||
mamba_track_mask,
|
||||
mamba_track_indices,
|
||||
conv_states.stride(0),
|
||||
ssm_states.stride(0),
|
||||
conv_state_numel_per_row,
|
||||
ssm_state_numel_per_row,
|
||||
BLOCK_SIZE,
|
||||
batch_size,
|
||||
1,
|
||||
(0, conv_states.stride(0)),
|
||||
(0, ssm_states.stride(0)),
|
||||
conv_states[0].numel(),
|
||||
ssm_states[0].numel(),
|
||||
check_freed_slots,
|
||||
)
|
||||
|
||||
@@ -832,65 +911,6 @@ def fused_commit_track_indices(
|
||||
return last_correct_step_indices, (mamba_steps_to_track if has_track else None)
|
||||
|
||||
|
||||
@triton.jit
|
||||
def track_mamba_states_all_layers_kernel(
|
||||
conv_states_ptr, # [num_layers, pool_size, ...] full conv pool
|
||||
ssm_states_ptr, # [num_layers, pool_size, ...] full ssm pool
|
||||
cache_indices_ptr,
|
||||
mamba_track_mask_ptr,
|
||||
mamba_track_indices_ptr,
|
||||
conv_layer_stride,
|
||||
conv_row_stride,
|
||||
ssm_layer_stride,
|
||||
ssm_row_stride,
|
||||
batch_size,
|
||||
conv_state_numel_per_row: tl.constexpr,
|
||||
ssm_state_numel_per_row: tl.constexpr,
|
||||
BLOCK_SIZE: tl.constexpr,
|
||||
check_freed_slots: tl.constexpr,
|
||||
):
|
||||
"""All-layers variant of track_mamba_state_if_needed_kernel: one launch
|
||||
covers every mamba layer (grid = num_layers * batch_size) instead of one
|
||||
launch per layer. The track mask / source / destination indices are shared
|
||||
across layers, so a single launch at the end of the step is equivalent to
|
||||
the per-layer launches (each layer's state is final by then)."""
|
||||
pid = tl.program_id(0)
|
||||
layer_idx = (pid // batch_size).to(tl.int64)
|
||||
batch_idx = pid % batch_size
|
||||
|
||||
track_mask = tl.load(mamba_track_mask_ptr + batch_idx)
|
||||
if not track_mask:
|
||||
return
|
||||
|
||||
src_idx = tl.load(cache_indices_ptr + batch_idx).to(tl.int64)
|
||||
dst_idx = tl.load(mamba_track_indices_ptr + batch_idx).to(tl.int64)
|
||||
if check_freed_slots:
|
||||
if src_idx < 0 or dst_idx < 0:
|
||||
return
|
||||
|
||||
conv_base = conv_states_ptr + layer_idx * conv_layer_stride
|
||||
for offset in range(0, conv_state_numel_per_row, BLOCK_SIZE):
|
||||
element_indices = offset + tl.arange(0, BLOCK_SIZE)
|
||||
mask = element_indices < conv_state_numel_per_row
|
||||
data = tl.load(
|
||||
conv_base + src_idx * conv_row_stride + element_indices,
|
||||
mask=mask,
|
||||
other=0.0,
|
||||
)
|
||||
tl.store(
|
||||
conv_base + dst_idx * conv_row_stride + element_indices, data, mask=mask
|
||||
)
|
||||
|
||||
ssm_base = ssm_states_ptr + layer_idx * ssm_layer_stride
|
||||
for offset in range(0, ssm_state_numel_per_row, BLOCK_SIZE):
|
||||
element_indices = offset + tl.arange(0, BLOCK_SIZE)
|
||||
mask = element_indices < ssm_state_numel_per_row
|
||||
data = tl.load(
|
||||
ssm_base + src_idx * ssm_row_stride + element_indices, mask=mask, other=0.0
|
||||
)
|
||||
tl.store(ssm_base + dst_idx * ssm_row_stride + element_indices, data, mask=mask)
|
||||
|
||||
|
||||
def track_mamba_states_all_layers(
|
||||
conv_states_pool: torch.Tensor,
|
||||
ssm_states_pool: torch.Tensor,
|
||||
@@ -904,27 +924,21 @@ def track_mamba_states_all_layers(
|
||||
|
||||
conv_states_pool / ssm_states_pool are the full [num_layers, pool_size,
|
||||
...] pools; per-row copy semantics are identical to
|
||||
track_mamba_states_if_needed applied per layer.
|
||||
track_mamba_states_if_needed applied per layer. The track mask / source /
|
||||
destination indices are shared across layers, so a single launch once
|
||||
every layer's state is final is equivalent to the per-layer launches.
|
||||
"""
|
||||
num_layers = conv_states_pool.shape[0]
|
||||
conv_state_numel_per_row = conv_states_pool[0, 0].numel()
|
||||
ssm_state_numel_per_row = ssm_states_pool[0, 0].numel()
|
||||
|
||||
BLOCK_SIZE = 1024
|
||||
grid = (num_layers * batch_size,)
|
||||
track_mamba_states_all_layers_kernel[grid](
|
||||
_launch_track_mamba_states(
|
||||
conv_states_pool,
|
||||
ssm_states_pool,
|
||||
cache_indices,
|
||||
mamba_track_mask,
|
||||
mamba_track_indices,
|
||||
conv_states_pool.stride(0),
|
||||
conv_states_pool.stride(1),
|
||||
ssm_states_pool.stride(0),
|
||||
ssm_states_pool.stride(1),
|
||||
batch_size,
|
||||
conv_state_numel_per_row,
|
||||
ssm_state_numel_per_row,
|
||||
BLOCK_SIZE,
|
||||
conv_states_pool.shape[0],
|
||||
(conv_states_pool.stride(0), conv_states_pool.stride(1)),
|
||||
(ssm_states_pool.stride(0), ssm_states_pool.stride(1)),
|
||||
conv_states_pool[0, 0].numel(),
|
||||
ssm_states_pool[0, 0].numel(),
|
||||
check_freed_slots,
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user