diff --git a/python/sglang/kernels/ops/mamba/__init__.py b/python/sglang/kernels/ops/mamba/__init__.py index d3800b71eb..7a55597e64 100644 --- a/python/sglang/kernels/ops/mamba/__init__.py +++ b/python/sglang/kernels/ops/mamba/__init__.py @@ -149,3 +149,15 @@ register_kernel( capabilities=frozenset({CapabilityRequirement.CUDA}), ) ) + +register_kernel( + KernelSpec( + op="mamba.fill_track_conv_indices", + backend=KernelBackend.TRITON, + target="sglang.kernels.ops.mamba.sconv_tracking:fill_track_conv_indices", + capabilities=_CUDA, + format_signature=FormatSignature( + in_place=True, description="short-convolution checkpoint-window indices" + ), + ) +) diff --git a/python/sglang/kernels/ops/mamba/sconv_tracking.py b/python/sglang/kernels/ops/mamba/sconv_tracking.py new file mode 100644 index 0000000000..4d96e515a8 --- /dev/null +++ b/python/sglang/kernels/ops/mamba/sconv_tracking.py @@ -0,0 +1,91 @@ +"""Prepare short-convolution checkpoint windows in a single Triton launch. + +Live rows select the window ending at the last complete cache-chunk boundary; +padded rows are zeroed so graph replay never reads stale token indices. +""" + +import torch +import triton +import triton.language as tl + + +@triton.jit(do_not_specialize=["live_rows"]) +def _track_conv_indices_kernel( + query_start_loc, + track_seqlens, + prefix_lens, + output, + live_rows, + ROWS: tl.constexpr, + QUERY_END: tl.constexpr, + WIDTH: tl.constexpr, + CHUNK: tl.constexpr, + ZERO_PREFIX: tl.constexpr, + BLOCK: tl.constexpr, +): + offsets = tl.program_id(0) * BLOCK + tl.arange(0, BLOCK) + rows = offsets // WIDTH + window = offsets % WIDTH + live = rows < live_rows + tracked = tl.load(track_seqlens + rows, mask=live, other=0).to(tl.int64) + if ZERO_PREFIX: + prefix = tl.full((BLOCK,), 0, tl.int64) + else: + prefix = tl.load(prefix_lens + rows, mask=live, other=0).to(tl.int64) + delta = tracked - prefix + # Triton's signed integer division truncates toward zero; Torch's // + # floors, including masked requests whose tracked length precedes prefix. + aligned = tl.where( + delta < 0, + -((-delta + CHUNK - 1) // CHUNK) * CHUNK, + (delta // CHUNK) * CHUNK, + ) + query_start = tl.load(query_start_loc + rows, mask=live, other=0).to(tl.int64) + total = tl.load(query_start_loc + QUERY_END).to(tl.int64) + index = query_start + aligned - WIDTH + window + index = tl.minimum(tl.maximum(index, 0), total - 1) + index = tl.where(live, index, 0) + tl.store(output + offsets, index, mask=rows < ROWS) + + +def fill_track_conv_indices( + *, + query_start_loc: torch.Tensor, + track_seqlens: torch.Tensor, + prefix_lens: torch.Tensor | None, + output: torch.Tensor, + live: int, + chunk_size: int, +) -> None: + """Fill a stable track-index buffer with one launch and no temporaries.""" + assert output.ndim == 2 and output.dtype == torch.int64 and output.is_contiguous() + rows, width = output.shape + assert 0 <= live <= rows and width > 0 and chunk_size > 0 + assert query_start_loc.ndim == track_seqlens.ndim == 1 + assert query_start_loc.dtype in (torch.int32, torch.int64) + assert track_seqlens.dtype in (torch.int32, torch.int64) + assert query_start_loc.numel() == rows + 1 and track_seqlens.numel() >= live + assert query_start_loc.is_contiguous() and track_seqlens.is_contiguous() + assert ( + query_start_loc.is_cuda + and track_seqlens.device == output.device == query_start_loc.device + ) + if prefix_lens is not None: + assert prefix_lens.ndim == 1 and prefix_lens.numel() >= live + assert prefix_lens.dtype in (torch.int32, torch.int64) + assert prefix_lens.is_contiguous() and prefix_lens.device == output.device + if rows == 0: + return + _track_conv_indices_kernel[(triton.cdiv(rows * width, 128),)]( + query_start_loc, + track_seqlens, + prefix_lens if prefix_lens is not None else track_seqlens, + output, + live, + ROWS=rows, + QUERY_END=query_start_loc.numel() - 1, + WIDTH=width, + CHUNK=chunk_size, + ZERO_PREFIX=prefix_lens is None, + BLOCK=128, + ) diff --git a/python/sglang/srt/layers/attention/linear/inkling_sconv_backend.py b/python/sglang/srt/layers/attention/linear/inkling_sconv_backend.py index 53ddd7f27d..20fc639a50 100644 --- a/python/sglang/srt/layers/attention/linear/inkling_sconv_backend.py +++ b/python/sglang/srt/layers/attention/linear/inkling_sconv_backend.py @@ -43,6 +43,7 @@ import torch from sglang.kernels.ops.mamba.mamba_state_scatter_triton import ( scatter_mamba_states_after_mtp_verify, ) +from sglang.kernels.ops.mamba.sconv_tracking import fill_track_conv_indices from sglang.srt.layers.attention.hybrid_linear_attn_backend import ( ShortConvHybridAttnBackend, ) @@ -68,6 +69,7 @@ from sglang.srt.runtime_context import ( mamba_cache_chunk_size, ) from sglang.srt.speculative.eagle_info import EagleDraftExtendInput +from sglang.srt.utils import is_cuda if TYPE_CHECKING: from sglang.srt.model_executor.model_runner import ModelRunner @@ -481,20 +483,15 @@ class InklingShortConvAttnBackend(ShortConvAttnBackend): # batch always has them, and must still fail rather than track against # an invented prefix. prefix_lens = forward_batch.extend_prefix_lens - if prefix_lens is None and on_graph_path: - prefix_lens = torch.zeros_like(forward_batch.mamba_track_seqlens) + track_seqlens = forward_batch.mamba_track_seqlens + assert query_start_loc is not None and track_seqlens is not None + assert prefix_lens is not None or on_graph_path live = min( rows, - forward_batch.mamba_track_seqlens.shape[0], - prefix_lens.shape[0], + track_seqlens.shape[0], + prefix_lens.shape[0] if prefix_lens is not None else rows, ) - lens_to_track = forward_batch.mamba_track_seqlens[:live] - prefix_lens[:live] - chunk_aligned = ( - lens_to_track // self.mamba_cache_chunk_size - ) * self.mamba_cache_chunk_size - start_indices = query_start_loc[:live] + chunk_aligned - self.conv_state_len - if on_graph_path: assert rows <= self._graph_track_conv_indices.shape[0], ( f"track-index buffer too small for a captured shape: rows={rows} " @@ -505,8 +502,30 @@ class InklingShortConvAttnBackend(ShortConvAttnBackend): out = torch.empty( (rows, self.conv_state_len), dtype=torch.int64, - device=start_indices.device, + device=query_start_loc.device, ) + + if is_cuda() and query_start_loc.is_cuda: + fill_track_conv_indices( + query_start_loc=query_start_loc, + track_seqlens=track_seqlens, + prefix_lens=prefix_lens, + output=out, + live=live, + chunk_size=self.mamba_cache_chunk_size, + ) + self.sconv_metadata.track_conv_indices = out + return + + if prefix_lens is None: + prefix_lens = torch.zeros_like(track_seqlens) + + lens_to_track = forward_batch.mamba_track_seqlens[:live] - prefix_lens[:live] + chunk_aligned = ( + lens_to_track // self.mamba_cache_chunk_size + ) * self.mamba_cache_chunk_size + start_indices = query_start_loc[:live] + chunk_aligned - self.conv_state_len + torch.add( start_indices.unsqueeze(-1).to(torch.int64), self._track_window_offsets, diff --git a/test/registered/kernels/ops/mamba/test_sconv_tracking.py b/test/registered/kernels/ops/mamba/test_sconv_tracking.py new file mode 100644 index 0000000000..be659c8f73 --- /dev/null +++ b/test/registered/kernels/ops/mamba/test_sconv_tracking.py @@ -0,0 +1,125 @@ +"""Tracking windows preserve Torch's floor division, clamping, and padded rows.""" + +import pytest +import torch + +from sglang.kernels.ops.mamba.sconv_tracking import fill_track_conv_indices +from sglang.test.ci.ci_register import register_cuda_ci + +register_cuda_ci(est_time=10, stage="base-b-kernel-unit", runner_config="1-gpu-large") + +requires_cuda = pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA only") + + +def _reference(query_starts, tracked, prefix, rows, width, live, chunk): + output = torch.zeros((rows, width), dtype=torch.int64, device=query_starts.device) + delta = tracked[:live].long() + if prefix is not None: + delta = delta - prefix[:live].long() + start = query_starts[:live].long() + (delta // chunk) * chunk - width + output[:live] = torch.minimum( + (start[:, None] + torch.arange(width, device=output.device)).clamp_min(0), + query_starts[-1:].long() - 1, + ) + return output + + +@requires_cuda +@pytest.mark.parametrize("rows", [0, 1, 7, 32, 129, 1024]) +@pytest.mark.parametrize("dtype", [torch.int32, torch.int64]) +@pytest.mark.parametrize("has_prefix", [False, True]) +def test_tracking_windows_and_padding(rows, dtype, has_prefix): + query_starts = torch.arange(rows + 1, device="cuda", dtype=dtype) * 256 + for live in sorted({0, rows // 2, rows}): + prefix = torch.arange(live, device="cuda", dtype=dtype) * 256 + 8192 + offsets = torch.tensor([-129, -128, -1, 0, 127, 128, 257], device="cuda") + tracked = prefix + offsets[torch.arange(live, device="cuda") % 7].to(dtype) + prefix = prefix if has_prefix else None + output = torch.full((rows, 3), 999, dtype=torch.int64, device="cuda") + fill_track_conv_indices( + query_start_loc=query_starts, + track_seqlens=tracked, + prefix_lens=prefix, + output=output, + live=live, + chunk_size=128, + ) + torch.testing.assert_close( + output, + _reference(query_starts, tracked, prefix, rows, 3, live, 128), + rtol=0, + atol=0, + ) + + +@requires_cuda +@pytest.mark.parametrize("chunk", [96, 128]) +@pytest.mark.parametrize("total", [0, 1, 8192]) +@pytest.mark.parametrize("width", [1, 3, 7]) +def test_negative_offsets_large_prefixes_and_clamping(chunk, total, width): + query_starts = torch.tensor([0, 0, 0, total], dtype=torch.int64, device="cuda") + prefix = torch.tensor([2**33, 8192, 8192], dtype=torch.int64, device="cuda") + tracked = prefix + torch.tensor([-1, -129, 257], device="cuda") + output = torch.empty((3, width), dtype=torch.int64, device="cuda") + fill_track_conv_indices( + query_start_loc=query_starts, + track_seqlens=tracked, + prefix_lens=prefix, + output=output, + live=3, + chunk_size=chunk, + ) + torch.testing.assert_close( + output, + _reference(query_starts, tracked, prefix, 3, width, 3, chunk), + rtol=0, + atol=0, + ) + + +@requires_cuda +def test_graph_replay_refreshes_indices_and_zeroes_tail(): + rows, live, width = 64, 32, 3 + query_starts = torch.arange(rows + 1, dtype=torch.int32, device="cuda") * 128 + prefix = torch.full((live,), 8192, dtype=torch.int32, device="cuda") + tracked = prefix.long() + 256 + output = torch.empty((rows, width), dtype=torch.int64, device="cuda") + + def fill(): + fill_track_conv_indices( + query_start_loc=query_starts, + track_seqlens=tracked, + prefix_lens=prefix, + output=output, + live=live, + chunk_size=128, + ) + + stream = torch.cuda.Stream() + stream.wait_stream(torch.cuda.current_stream()) + with torch.cuda.stream(stream): + for _ in range(3): + fill() + torch.cuda.current_stream().wait_stream(stream) + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(graph, stream=stream): + fill() + pointer = output.data_ptr() + for shift in (0, 128, -257): + tracked.add_(shift) + query_starts.add_(1) + output.fill_(999) + graph.replay() + torch.testing.assert_close( + output, + _reference(query_starts, tracked, prefix, rows, width, live, 128), + rtol=0, + atol=0, + ) + assert output.data_ptr() == pointer + + +if __name__ == "__main__": + import sys + + sys.exit(pytest.main([__file__, "-v", "-x"]))