[Perf] Fuse short-convolution checkpoint tracking metadata (#43013)

This commit is contained in:
Eric Zhang
2026-10-09 00:26:15 +08:00
committed by GitHub
parent 642e81fbb8
commit 344b8a863b
4 changed files with 258 additions and 11 deletions
@@ -149,3 +149,15 @@ register_kernel(
capabilities=frozenset({CapabilityRequirement.CUDA}),
)
)
register_kernel(
KernelSpec(
op="mamba.fill_track_conv_indices",
backend=KernelBackend.TRITON,
target="sglang.kernels.ops.mamba.sconv_tracking:fill_track_conv_indices",
capabilities=_CUDA,
format_signature=FormatSignature(
in_place=True, description="short-convolution checkpoint-window indices"
),
)
)
@@ -0,0 +1,91 @@
"""Prepare short-convolution checkpoint windows in a single Triton launch.
Live rows select the window ending at the last complete cache-chunk boundary;
padded rows are zeroed so graph replay never reads stale token indices.
"""
import torch
import triton
import triton.language as tl
@triton.jit(do_not_specialize=["live_rows"])
def _track_conv_indices_kernel(
query_start_loc,
track_seqlens,
prefix_lens,
output,
live_rows,
ROWS: tl.constexpr,
QUERY_END: tl.constexpr,
WIDTH: tl.constexpr,
CHUNK: tl.constexpr,
ZERO_PREFIX: tl.constexpr,
BLOCK: tl.constexpr,
):
offsets = tl.program_id(0) * BLOCK + tl.arange(0, BLOCK)
rows = offsets // WIDTH
window = offsets % WIDTH
live = rows < live_rows
tracked = tl.load(track_seqlens + rows, mask=live, other=0).to(tl.int64)
if ZERO_PREFIX:
prefix = tl.full((BLOCK,), 0, tl.int64)
else:
prefix = tl.load(prefix_lens + rows, mask=live, other=0).to(tl.int64)
delta = tracked - prefix
# Triton's signed integer division truncates toward zero; Torch's //
# floors, including masked requests whose tracked length precedes prefix.
aligned = tl.where(
delta < 0,
-((-delta + CHUNK - 1) // CHUNK) * CHUNK,
(delta // CHUNK) * CHUNK,
)
query_start = tl.load(query_start_loc + rows, mask=live, other=0).to(tl.int64)
total = tl.load(query_start_loc + QUERY_END).to(tl.int64)
index = query_start + aligned - WIDTH + window
index = tl.minimum(tl.maximum(index, 0), total - 1)
index = tl.where(live, index, 0)
tl.store(output + offsets, index, mask=rows < ROWS)
def fill_track_conv_indices(
*,
query_start_loc: torch.Tensor,
track_seqlens: torch.Tensor,
prefix_lens: torch.Tensor | None,
output: torch.Tensor,
live: int,
chunk_size: int,
) -> None:
"""Fill a stable track-index buffer with one launch and no temporaries."""
assert output.ndim == 2 and output.dtype == torch.int64 and output.is_contiguous()
rows, width = output.shape
assert 0 <= live <= rows and width > 0 and chunk_size > 0
assert query_start_loc.ndim == track_seqlens.ndim == 1
assert query_start_loc.dtype in (torch.int32, torch.int64)
assert track_seqlens.dtype in (torch.int32, torch.int64)
assert query_start_loc.numel() == rows + 1 and track_seqlens.numel() >= live
assert query_start_loc.is_contiguous() and track_seqlens.is_contiguous()
assert (
query_start_loc.is_cuda
and track_seqlens.device == output.device == query_start_loc.device
)
if prefix_lens is not None:
assert prefix_lens.ndim == 1 and prefix_lens.numel() >= live
assert prefix_lens.dtype in (torch.int32, torch.int64)
assert prefix_lens.is_contiguous() and prefix_lens.device == output.device
if rows == 0:
return
_track_conv_indices_kernel[(triton.cdiv(rows * width, 128),)](
query_start_loc,
track_seqlens,
prefix_lens if prefix_lens is not None else track_seqlens,
output,
live,
ROWS=rows,
QUERY_END=query_start_loc.numel() - 1,
WIDTH=width,
CHUNK=chunk_size,
ZERO_PREFIX=prefix_lens is None,
BLOCK=128,
)
@@ -43,6 +43,7 @@ import torch
from sglang.kernels.ops.mamba.mamba_state_scatter_triton import (
scatter_mamba_states_after_mtp_verify,
)
from sglang.kernels.ops.mamba.sconv_tracking import fill_track_conv_indices
from sglang.srt.layers.attention.hybrid_linear_attn_backend import (
ShortConvHybridAttnBackend,
)
@@ -68,6 +69,7 @@ from sglang.srt.runtime_context import (
mamba_cache_chunk_size,
)
from sglang.srt.speculative.eagle_info import EagleDraftExtendInput
from sglang.srt.utils import is_cuda
if TYPE_CHECKING:
from sglang.srt.model_executor.model_runner import ModelRunner
@@ -481,20 +483,15 @@ class InklingShortConvAttnBackend(ShortConvAttnBackend):
# batch always has them, and must still fail rather than track against
# an invented prefix.
prefix_lens = forward_batch.extend_prefix_lens
if prefix_lens is None and on_graph_path:
prefix_lens = torch.zeros_like(forward_batch.mamba_track_seqlens)
track_seqlens = forward_batch.mamba_track_seqlens
assert query_start_loc is not None and track_seqlens is not None
assert prefix_lens is not None or on_graph_path
live = min(
rows,
forward_batch.mamba_track_seqlens.shape[0],
prefix_lens.shape[0],
track_seqlens.shape[0],
prefix_lens.shape[0] if prefix_lens is not None else rows,
)
lens_to_track = forward_batch.mamba_track_seqlens[:live] - prefix_lens[:live]
chunk_aligned = (
lens_to_track // self.mamba_cache_chunk_size
) * self.mamba_cache_chunk_size
start_indices = query_start_loc[:live] + chunk_aligned - self.conv_state_len
if on_graph_path:
assert rows <= self._graph_track_conv_indices.shape[0], (
f"track-index buffer too small for a captured shape: rows={rows} "
@@ -505,8 +502,30 @@ class InklingShortConvAttnBackend(ShortConvAttnBackend):
out = torch.empty(
(rows, self.conv_state_len),
dtype=torch.int64,
device=start_indices.device,
device=query_start_loc.device,
)
if is_cuda() and query_start_loc.is_cuda:
fill_track_conv_indices(
query_start_loc=query_start_loc,
track_seqlens=track_seqlens,
prefix_lens=prefix_lens,
output=out,
live=live,
chunk_size=self.mamba_cache_chunk_size,
)
self.sconv_metadata.track_conv_indices = out
return
if prefix_lens is None:
prefix_lens = torch.zeros_like(track_seqlens)
lens_to_track = forward_batch.mamba_track_seqlens[:live] - prefix_lens[:live]
chunk_aligned = (
lens_to_track // self.mamba_cache_chunk_size
) * self.mamba_cache_chunk_size
start_indices = query_start_loc[:live] + chunk_aligned - self.conv_state_len
torch.add(
start_indices.unsqueeze(-1).to(torch.int64),
self._track_window_offsets,