diff --git a/python/sglang/kernels/ops/mamba/mamba_state_scatter_triton.py b/python/sglang/kernels/ops/mamba/mamba_state_scatter_triton.py index 1e72110cac..73201bb313 100644 --- a/python/sglang/kernels/ops/mamba/mamba_state_scatter_triton.py +++ b/python/sglang/kernels/ops/mamba/mamba_state_scatter_triton.py @@ -6,11 +6,14 @@ This kernel replaces the expensive advanced indexing operations in avoiding multiple `index_elementwise_kernel` launches. """ +import functools + import torch import triton import triton.language as tl from sglang.kernels.ops.memory.ptr_table import make_ptr_table +from sglang.srt.utils.common import get_device_core_count def _require_entry_contiguous_dst( @@ -33,74 +36,157 @@ def _require_entry_contiguous_dst( @triton.jit +def _copy_track_chunk( + ptr, + src_off, + dst_off, + start, + numel: tl.constexpr, + CHUNK: tl.constexpr, + BLOCK: tl.constexpr, +): + for off in tl.static_range(0, CHUNK, BLOCK): + offsets = start + off + tl.arange(0, BLOCK) + mask = offsets < numel + data = tl.load(ptr + src_off + offsets, mask=mask) + tl.store(ptr + dst_off + offsets, data, mask=mask) + + +@triton.jit(do_not_specialize=["batch_size", "num_layers"]) def track_mamba_state_if_needed_kernel( conv_states_ptr, ssm_states_ptr, cache_indices_ptr, mamba_track_mask_ptr, mamba_track_indices_ptr, - conv_state_stride_0, # stride for first dimension (batch/pool index) - ssm_state_stride_0, # stride for first dimension (batch/pool index) - conv_state_numel_per_row: tl.constexpr, # total elements per row - ssm_state_numel_per_row: tl.constexpr, # total elements per row + conv_layer_stride, + conv_row_stride, + ssm_layer_stride, + ssm_row_stride, + batch_size, + num_layers, + conv_state_numel_per_row: tl.constexpr, + ssm_state_numel_per_row: tl.constexpr, + CONV_CHUNK: tl.constexpr, + SSM_CHUNK: tl.constexpr, + NUM_CONV_CHUNKS: tl.constexpr, + NUM_SSM_CHUNKS: tl.constexpr, + BS_POW2: tl.constexpr, BLOCK_SIZE: tl.constexpr, check_freed_slots: tl.constexpr, # only the shared/unified KV pool emits -1 ): """ - Track conv_states and ssm_states rows based on track mask. + For each batch row with mamba_track_mask set, copy its conv and ssm state + rows from cache_indices[i] to mamba_track_indices[i], in every layer. - This kernel replaces a Python loop that copies state tensors for mamba attention. - For each batch element, if the track mask is True, it copies the entire row from - the source index (cache_indices[i]) to the destination index (mamba_track_indices[i]). - - Grid: (batch_size,) - Each block handles one batch element, using multiple threads to copy data in parallel. + Persistent grid: every program reads the whole mask, so the tracked rows + are compacted in-kernel and only their (row, layer, chunk) work items are + spread over the grid. A step with no tracked row reads one mask vector per + program and exits; one tracked row runs at HBM bandwidth on all SMs. """ - batch_idx = tl.program_id(0) + pid = tl.program_id(0) + num_programs = tl.num_programs(0) + rows = tl.arange(0, BS_POW2) + track = tl.load(mamba_track_mask_ptr + rows, mask=rows < batch_size, other=0).to( + tl.int32 + ) + num_tracked = tl.sum(track, axis=0) + track_rank = tl.cumsum(track, axis=0) - # Load the copy mask for this batch element - track_mask = tl.load(mamba_track_mask_ptr + batch_idx) + CHUNKS: tl.constexpr = NUM_CONV_CHUNKS + NUM_SSM_CHUNKS + items_per_row = CHUNKS * num_layers + for item in range(pid, num_tracked * items_per_row, num_programs): + rank = item // items_per_row + rem = item - rank * items_per_row + layer_idx = (rem // CHUNKS).to(tl.int64) + chunk = rem % CHUNKS + batch_idx = tl.sum( + tl.where((track_rank == rank + 1) & (track != 0), rows, 0), axis=0 + ) + # int64 before the row stride multiply: the page-granularity envelope + # layout makes the row stride large enough for int32 to overflow. + src_idx = tl.load(cache_indices_ptr + batch_idx).to(tl.int64) + dst_idx = tl.load(mamba_track_indices_ptr + batch_idx).to(tl.int64) + # Skip freed slots (-1): `state_ptr + (-1)*stride` would fault. + if (not check_freed_slots) or ((src_idx >= 0) & (dst_idx >= 0)): + if chunk < NUM_CONV_CHUNKS: + _copy_track_chunk( + conv_states_ptr + layer_idx * conv_layer_stride, + src_idx * conv_row_stride, + dst_idx * conv_row_stride, + chunk * CONV_CHUNK, + conv_state_numel_per_row, + CONV_CHUNK, + BLOCK_SIZE, + ) + else: + _copy_track_chunk( + ssm_states_ptr + layer_idx * ssm_layer_stride, + src_idx * ssm_row_stride, + dst_idx * ssm_row_stride, + (chunk - NUM_CONV_CHUNKS) * SSM_CHUNK, + ssm_state_numel_per_row, + SSM_CHUNK, + BLOCK_SIZE, + ) - # Early exit if we don't need to track - if not track_mask: + +_TRACK_BLOCK_SIZE = 8192 +_TRACK_CHUNK_BYTES = 32768 +_TRACK_PROGRAMS_PER_SM = 4 + + +@functools.lru_cache(maxsize=None) +def _num_sms(device_index: int) -> int: + return get_device_core_count(device_index) or 132 + + +def _launch_track_mamba_states( + conv_states: torch.Tensor, + ssm_states: torch.Tensor, + cache_indices: torch.Tensor, + mamba_track_mask: torch.Tensor, + mamba_track_indices: torch.Tensor, + batch_size: int, + num_layers: int, + conv_strides: tuple, + ssm_strides: tuple, + conv_state_numel_per_row: int, + ssm_state_numel_per_row: int, + check_freed_slots: bool, +): + if batch_size == 0: return - - # Cast indices to int64 before they multiply the row stride. The - # page-granularity envelope layout makes the conv/ssm row stride large - # (stride_0 = entry_bytes / itemsize), so an int32 `idx * stride_0` can - # overflow for moderately large idx and wrap to an illegal address. int64 is - # harmless for the small-stride (per-layer) case. - src_idx = tl.load(cache_indices_ptr + batch_idx).to(tl.int64) - dst_idx = tl.load(mamba_track_indices_ptr + batch_idx).to(tl.int64) - - # Skip freed slots (-1): `state_ptr + (-1)*stride` would fault. Only the unified - # pool emits -1 tombstones (from the v2p translate); compiled out for static. - if check_freed_slots: - if src_idx < 0 or dst_idx < 0: - return - - # Copy conv_states - # Each thread handles BLOCK_SIZE elements - for offset in range(0, conv_state_numel_per_row, BLOCK_SIZE): - element_indices = offset + tl.arange(0, BLOCK_SIZE) - mask = element_indices < conv_state_numel_per_row - - src_ptr = conv_states_ptr + src_idx * conv_state_stride_0 + element_indices - dst_ptr = conv_states_ptr + dst_idx * conv_state_stride_0 + element_indices - - data = tl.load(src_ptr, mask=mask, other=0.0) - tl.store(dst_ptr, data, mask=mask) - - # Copy ssm_states - for offset in range(0, ssm_state_numel_per_row, BLOCK_SIZE): - element_indices = offset + tl.arange(0, BLOCK_SIZE) - mask = element_indices < ssm_state_numel_per_row - - src_ptr = ssm_states_ptr + src_idx * ssm_state_stride_0 + element_indices - dst_ptr = ssm_states_ptr + dst_idx * ssm_state_stride_0 + element_indices - - data = tl.load(src_ptr, mask=mask, other=0.0) - tl.store(dst_ptr, data, mask=mask) + conv_chunk = max( + _TRACK_BLOCK_SIZE, _TRACK_CHUNK_BYTES // conv_states.element_size() + ) + ssm_chunk = max(_TRACK_BLOCK_SIZE, _TRACK_CHUNK_BYTES // ssm_states.element_size()) + num_conv_chunks = triton.cdiv(conv_state_numel_per_row, conv_chunk) + num_ssm_chunks = triton.cdiv(ssm_state_numel_per_row, ssm_chunk) + max_items = batch_size * num_layers * (num_conv_chunks + num_ssm_chunks) + device_index = conv_states.device.index or 0 + grid = (min(max_items, _TRACK_PROGRAMS_PER_SM * _num_sms(device_index)),) + track_mamba_state_if_needed_kernel[grid]( + conv_states, + ssm_states, + cache_indices, + mamba_track_mask, + mamba_track_indices, + *conv_strides, + *ssm_strides, + batch_size, + num_layers, + conv_state_numel_per_row, + ssm_state_numel_per_row, + conv_chunk, + ssm_chunk, + num_conv_chunks, + num_ssm_chunks, + max(16, triton.next_power_of_2(batch_size)), + _TRACK_BLOCK_SIZE, + check_freed_slots, + num_warps=8, + ) def track_mamba_states_if_needed( @@ -123,25 +209,18 @@ def track_mamba_states_if_needed( mamba_track_indices: Indices to track for each batch element [batch_size] batch_size: Number of batch elements """ - conv_state_numel_per_row = conv_states[0].numel() - ssm_state_numel_per_row = ssm_states[0].numel() - - # Choose BLOCK_SIZE based on the size of the data - BLOCK_SIZE = 1024 - - # Launch kernel with batch_size blocks - grid = (batch_size,) - track_mamba_state_if_needed_kernel[grid]( + _launch_track_mamba_states( conv_states, ssm_states, cache_indices, mamba_track_mask, mamba_track_indices, - conv_states.stride(0), - ssm_states.stride(0), - conv_state_numel_per_row, - ssm_state_numel_per_row, - BLOCK_SIZE, + batch_size, + 1, + (0, conv_states.stride(0)), + (0, ssm_states.stride(0)), + conv_states[0].numel(), + ssm_states[0].numel(), check_freed_slots, ) @@ -832,65 +911,6 @@ def fused_commit_track_indices( return last_correct_step_indices, (mamba_steps_to_track if has_track else None) -@triton.jit -def track_mamba_states_all_layers_kernel( - conv_states_ptr, # [num_layers, pool_size, ...] full conv pool - ssm_states_ptr, # [num_layers, pool_size, ...] full ssm pool - cache_indices_ptr, - mamba_track_mask_ptr, - mamba_track_indices_ptr, - conv_layer_stride, - conv_row_stride, - ssm_layer_stride, - ssm_row_stride, - batch_size, - conv_state_numel_per_row: tl.constexpr, - ssm_state_numel_per_row: tl.constexpr, - BLOCK_SIZE: tl.constexpr, - check_freed_slots: tl.constexpr, -): - """All-layers variant of track_mamba_state_if_needed_kernel: one launch - covers every mamba layer (grid = num_layers * batch_size) instead of one - launch per layer. The track mask / source / destination indices are shared - across layers, so a single launch at the end of the step is equivalent to - the per-layer launches (each layer's state is final by then).""" - pid = tl.program_id(0) - layer_idx = (pid // batch_size).to(tl.int64) - batch_idx = pid % batch_size - - track_mask = tl.load(mamba_track_mask_ptr + batch_idx) - if not track_mask: - return - - src_idx = tl.load(cache_indices_ptr + batch_idx).to(tl.int64) - dst_idx = tl.load(mamba_track_indices_ptr + batch_idx).to(tl.int64) - if check_freed_slots: - if src_idx < 0 or dst_idx < 0: - return - - conv_base = conv_states_ptr + layer_idx * conv_layer_stride - for offset in range(0, conv_state_numel_per_row, BLOCK_SIZE): - element_indices = offset + tl.arange(0, BLOCK_SIZE) - mask = element_indices < conv_state_numel_per_row - data = tl.load( - conv_base + src_idx * conv_row_stride + element_indices, - mask=mask, - other=0.0, - ) - tl.store( - conv_base + dst_idx * conv_row_stride + element_indices, data, mask=mask - ) - - ssm_base = ssm_states_ptr + layer_idx * ssm_layer_stride - for offset in range(0, ssm_state_numel_per_row, BLOCK_SIZE): - element_indices = offset + tl.arange(0, BLOCK_SIZE) - mask = element_indices < ssm_state_numel_per_row - data = tl.load( - ssm_base + src_idx * ssm_row_stride + element_indices, mask=mask, other=0.0 - ) - tl.store(ssm_base + dst_idx * ssm_row_stride + element_indices, data, mask=mask) - - def track_mamba_states_all_layers( conv_states_pool: torch.Tensor, ssm_states_pool: torch.Tensor, @@ -904,27 +924,21 @@ def track_mamba_states_all_layers( conv_states_pool / ssm_states_pool are the full [num_layers, pool_size, ...] pools; per-row copy semantics are identical to - track_mamba_states_if_needed applied per layer. + track_mamba_states_if_needed applied per layer. The track mask / source / + destination indices are shared across layers, so a single launch once + every layer's state is final is equivalent to the per-layer launches. """ - num_layers = conv_states_pool.shape[0] - conv_state_numel_per_row = conv_states_pool[0, 0].numel() - ssm_state_numel_per_row = ssm_states_pool[0, 0].numel() - - BLOCK_SIZE = 1024 - grid = (num_layers * batch_size,) - track_mamba_states_all_layers_kernel[grid]( + _launch_track_mamba_states( conv_states_pool, ssm_states_pool, cache_indices, mamba_track_mask, mamba_track_indices, - conv_states_pool.stride(0), - conv_states_pool.stride(1), - ssm_states_pool.stride(0), - ssm_states_pool.stride(1), batch_size, - conv_state_numel_per_row, - ssm_state_numel_per_row, - BLOCK_SIZE, + conv_states_pool.shape[0], + (conv_states_pool.stride(0), conv_states_pool.stride(1)), + (ssm_states_pool.stride(0), ssm_states_pool.stride(1)), + conv_states_pool[0, 0].numel(), + ssm_states_pool[0, 0].numel(), check_freed_slots, ) diff --git a/test/registered/unit/layers/test_mamba_track_states_triton.py b/test/registered/unit/layers/test_mamba_track_states_triton.py new file mode 100644 index 0000000000..47e72865f1 --- /dev/null +++ b/test/registered/unit/layers/test_mamba_track_states_triton.py @@ -0,0 +1,177 @@ +from sglang.test.ci.ci_register import register_cuda_ci + +register_cuda_ci(est_time=20, stage="base-b", runner_config="1-gpu-small") + +import unittest +from unittest import mock + +import torch + +from sglang.kernels.ops.mamba import mamba_state_scatter_triton +from sglang.kernels.ops.mamba.mamba_state_scatter_triton import ( + track_mamba_states_all_layers, + track_mamba_states_if_needed, +) +from sglang.test.test_utils import CustomTestCase + +NUM_LAYERS = 3 +CONV_SHAPE = (3 * 256, 3) +SSM_SHAPE = (4, 64, 64) + + +def _make_pools( + pool_size, + conv_dtype, + ssm_dtype, + gen, + num_layers=NUM_LAYERS, + conv_shape=CONV_SHAPE, + ssm_shape=SSM_SHAPE, +): + conv = torch.randn( + (num_layers, pool_size) + conv_shape, generator=gen, device="cuda" + ).to(conv_dtype) + ssm = torch.randn( + (num_layers, pool_size) + ssm_shape, generator=gen, device="cuda" + ).to(ssm_dtype) + return conv, ssm + + +def _make_batch(bs, real_bs, pool_size, gen, freed=False): + """Rows >= real_bs are graph padding: mask False and index 0, like the + decode graph's zero-padded track slots.""" + perm = torch.randperm(pool_size - 1, generator=gen, device="cuda") + 1 + cache_indices = torch.zeros(bs, dtype=torch.int64, device="cuda") + track_indices = torch.zeros(bs, dtype=torch.int64, device="cuda") + cache_indices[:real_bs] = perm[:real_bs] + track_indices[:real_bs] = perm[real_bs : 2 * real_bs] + mask = torch.zeros(bs, dtype=torch.bool, device="cuda") + mask[:real_bs] = torch.rand(real_bs, generator=gen, device="cuda") < 0.5 + if freed and real_bs > 1: + mask[0] = True + track_indices[0] = -1 + mask[1] = True + cache_indices[1] = -1 + return cache_indices, mask, track_indices + + +def _reference(conv, ssm, cache_indices, mask, track_indices): + conv, ssm = conv.clone(), ssm.clone() + for i in range(mask.numel()): + src, dst = int(cache_indices[i]), int(track_indices[i]) + if bool(mask[i]) and src >= 0 and dst >= 0: + conv[:, dst] = conv[:, src] + ssm[:, dst] = ssm[:, src] + return conv, ssm + + +@unittest.skipUnless(torch.cuda.is_available(), "CUDA is required for this test.") +class TestTrackMambaStates(CustomTestCase): + def _check(self, bs, real_bs, conv_dtype, ssm_dtype, freed=False, **pool_shapes): + gen = torch.Generator(device="cuda").manual_seed(1000 * bs + real_bs) + pool_size = 2 * bs + 2 + conv, ssm = _make_pools(pool_size, conv_dtype, ssm_dtype, gen, **pool_shapes) + cache_indices, mask, track_indices = _make_batch( + bs, real_bs, pool_size, gen, freed + ) + ref_conv, ref_ssm = _reference(conv, ssm, cache_indices, mask, track_indices) + + per_conv, per_ssm = conv.clone(), ssm.clone() + for layer in range(conv.shape[0]): + track_mamba_states_if_needed( + per_conv[layer], + per_ssm[layer], + cache_indices, + mask, + track_indices, + bs, + check_freed_slots=freed, + ) + all_conv, all_ssm = conv.clone(), ssm.clone() + track_mamba_states_all_layers( + all_conv, + all_ssm, + cache_indices, + mask, + track_indices, + bs, + check_freed_slots=freed, + ) + for got_conv, got_ssm in ((per_conv, per_ssm), (all_conv, all_ssm)): + self.assertTrue(torch.equal(got_conv, ref_conv)) + self.assertTrue(torch.equal(got_ssm, ref_ssm)) + + def test_bf16_padded_batches(self): + for bs in range(1, 41): + for real_bs in sorted({1, bs // 2 or 1, bs}): + self._check(bs, real_bs, torch.bfloat16, torch.bfloat16) + + def test_mixed_dtypes(self): + for bs in (1, 7, 17, 40): + self._check(bs, bs, torch.bfloat16, torch.float32) + + def test_freed_slots_skipped(self): + for bs in (2, 9, 33): + self._check(bs, bs, torch.bfloat16, torch.bfloat16, freed=True) + + def test_persistent_loop_and_partial_chunks(self): + # With 4 programs each one loops over many (row, layer, chunk) items, + # and neither state row is a whole number of 32 KB chunks. + shapes = dict(num_layers=4, conv_shape=(3, 6000), ssm_shape=(2, 129, 130)) + with mock.patch.object(mamba_state_scatter_triton, "_num_sms", return_value=1): + for ssm_dtype in (torch.bfloat16, torch.float32): + self._check(9, 6, torch.bfloat16, ssm_dtype, **shapes) + self._check(9, 9, torch.bfloat16, ssm_dtype, freed=True, **shapes) + self._check(9, 9, torch.bfloat16, torch.bfloat16, **shapes) + + def test_all_rows_tracked(self): + gen = torch.Generator(device="cuda").manual_seed(7) + bs = 40 + conv, ssm = _make_pools(2 * bs + 1, torch.bfloat16, torch.bfloat16, gen) + cache_indices = torch.arange(bs, device="cuda") + track_indices = torch.arange(bs, 2 * bs, device="cuda") + mask = torch.ones(bs, dtype=torch.bool, device="cuda") + ref_conv, ref_ssm = _reference(conv, ssm, cache_indices, mask, track_indices) + track_mamba_states_all_layers(conv, ssm, cache_indices, mask, track_indices, bs) + self.assertTrue(torch.equal(conv, ref_conv)) + self.assertTrue(torch.equal(ssm, ref_ssm)) + + def test_cuda_graph_replay_follows_mask(self): + gen = torch.Generator(device="cuda").manual_seed(3) + bs = 24 + pool_size = 2 * bs + 2 + conv, ssm = _make_pools(pool_size, torch.bfloat16, torch.bfloat16, gen) + cache_indices = torch.zeros(bs, dtype=torch.int64, device="cuda") + track_indices = torch.zeros(bs, dtype=torch.int64, device="cuda") + mask = torch.zeros(bs, dtype=torch.bool, device="cuda") + + def run(): + track_mamba_states_all_layers( + conv, ssm, cache_indices, mask, track_indices, bs + ) + + stream = torch.cuda.Stream() + stream.wait_stream(torch.cuda.current_stream()) + with torch.cuda.stream(stream): + run() + torch.cuda.current_stream().wait_stream(stream) + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(graph): + run() + + for real_bs in (0, 1, 5, bs): + new_ci, new_mask, new_ti = _make_batch(bs, real_bs, pool_size, gen) + cache_indices.copy_(new_ci) + mask.copy_(new_mask) + track_indices.copy_(new_ti) + ref_conv, ref_ssm = _reference( + conv, ssm, cache_indices, mask, track_indices + ) + graph.replay() + torch.cuda.synchronize() + self.assertTrue(torch.equal(conv, ref_conv)) + self.assertTrue(torch.equal(ssm, ref_ssm)) + + +if __name__ == "__main__": + unittest.main()