[Perf] Fuse short-convolution checkpoint tracking metadata (#43013)
This commit is contained in:
@@ -149,3 +149,15 @@ register_kernel(
|
||||
capabilities=frozenset({CapabilityRequirement.CUDA}),
|
||||
)
|
||||
)
|
||||
|
||||
register_kernel(
|
||||
KernelSpec(
|
||||
op="mamba.fill_track_conv_indices",
|
||||
backend=KernelBackend.TRITON,
|
||||
target="sglang.kernels.ops.mamba.sconv_tracking:fill_track_conv_indices",
|
||||
capabilities=_CUDA,
|
||||
format_signature=FormatSignature(
|
||||
in_place=True, description="short-convolution checkpoint-window indices"
|
||||
),
|
||||
)
|
||||
)
|
||||
|
||||
@@ -0,0 +1,91 @@
|
||||
"""Prepare short-convolution checkpoint windows in a single Triton launch.
|
||||
|
||||
Live rows select the window ending at the last complete cache-chunk boundary;
|
||||
padded rows are zeroed so graph replay never reads stale token indices.
|
||||
"""
|
||||
|
||||
import torch
|
||||
import triton
|
||||
import triton.language as tl
|
||||
|
||||
|
||||
@triton.jit(do_not_specialize=["live_rows"])
|
||||
def _track_conv_indices_kernel(
|
||||
query_start_loc,
|
||||
track_seqlens,
|
||||
prefix_lens,
|
||||
output,
|
||||
live_rows,
|
||||
ROWS: tl.constexpr,
|
||||
QUERY_END: tl.constexpr,
|
||||
WIDTH: tl.constexpr,
|
||||
CHUNK: tl.constexpr,
|
||||
ZERO_PREFIX: tl.constexpr,
|
||||
BLOCK: tl.constexpr,
|
||||
):
|
||||
offsets = tl.program_id(0) * BLOCK + tl.arange(0, BLOCK)
|
||||
rows = offsets // WIDTH
|
||||
window = offsets % WIDTH
|
||||
live = rows < live_rows
|
||||
tracked = tl.load(track_seqlens + rows, mask=live, other=0).to(tl.int64)
|
||||
if ZERO_PREFIX:
|
||||
prefix = tl.full((BLOCK,), 0, tl.int64)
|
||||
else:
|
||||
prefix = tl.load(prefix_lens + rows, mask=live, other=0).to(tl.int64)
|
||||
delta = tracked - prefix
|
||||
# Triton's signed integer division truncates toward zero; Torch's //
|
||||
# floors, including masked requests whose tracked length precedes prefix.
|
||||
aligned = tl.where(
|
||||
delta < 0,
|
||||
-((-delta + CHUNK - 1) // CHUNK) * CHUNK,
|
||||
(delta // CHUNK) * CHUNK,
|
||||
)
|
||||
query_start = tl.load(query_start_loc + rows, mask=live, other=0).to(tl.int64)
|
||||
total = tl.load(query_start_loc + QUERY_END).to(tl.int64)
|
||||
index = query_start + aligned - WIDTH + window
|
||||
index = tl.minimum(tl.maximum(index, 0), total - 1)
|
||||
index = tl.where(live, index, 0)
|
||||
tl.store(output + offsets, index, mask=rows < ROWS)
|
||||
|
||||
|
||||
def fill_track_conv_indices(
|
||||
*,
|
||||
query_start_loc: torch.Tensor,
|
||||
track_seqlens: torch.Tensor,
|
||||
prefix_lens: torch.Tensor | None,
|
||||
output: torch.Tensor,
|
||||
live: int,
|
||||
chunk_size: int,
|
||||
) -> None:
|
||||
"""Fill a stable track-index buffer with one launch and no temporaries."""
|
||||
assert output.ndim == 2 and output.dtype == torch.int64 and output.is_contiguous()
|
||||
rows, width = output.shape
|
||||
assert 0 <= live <= rows and width > 0 and chunk_size > 0
|
||||
assert query_start_loc.ndim == track_seqlens.ndim == 1
|
||||
assert query_start_loc.dtype in (torch.int32, torch.int64)
|
||||
assert track_seqlens.dtype in (torch.int32, torch.int64)
|
||||
assert query_start_loc.numel() == rows + 1 and track_seqlens.numel() >= live
|
||||
assert query_start_loc.is_contiguous() and track_seqlens.is_contiguous()
|
||||
assert (
|
||||
query_start_loc.is_cuda
|
||||
and track_seqlens.device == output.device == query_start_loc.device
|
||||
)
|
||||
if prefix_lens is not None:
|
||||
assert prefix_lens.ndim == 1 and prefix_lens.numel() >= live
|
||||
assert prefix_lens.dtype in (torch.int32, torch.int64)
|
||||
assert prefix_lens.is_contiguous() and prefix_lens.device == output.device
|
||||
if rows == 0:
|
||||
return
|
||||
_track_conv_indices_kernel[(triton.cdiv(rows * width, 128),)](
|
||||
query_start_loc,
|
||||
track_seqlens,
|
||||
prefix_lens if prefix_lens is not None else track_seqlens,
|
||||
output,
|
||||
live,
|
||||
ROWS=rows,
|
||||
QUERY_END=query_start_loc.numel() - 1,
|
||||
WIDTH=width,
|
||||
CHUNK=chunk_size,
|
||||
ZERO_PREFIX=prefix_lens is None,
|
||||
BLOCK=128,
|
||||
)
|
||||
@@ -43,6 +43,7 @@ import torch
|
||||
from sglang.kernels.ops.mamba.mamba_state_scatter_triton import (
|
||||
scatter_mamba_states_after_mtp_verify,
|
||||
)
|
||||
from sglang.kernels.ops.mamba.sconv_tracking import fill_track_conv_indices
|
||||
from sglang.srt.layers.attention.hybrid_linear_attn_backend import (
|
||||
ShortConvHybridAttnBackend,
|
||||
)
|
||||
@@ -68,6 +69,7 @@ from sglang.srt.runtime_context import (
|
||||
mamba_cache_chunk_size,
|
||||
)
|
||||
from sglang.srt.speculative.eagle_info import EagleDraftExtendInput
|
||||
from sglang.srt.utils import is_cuda
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from sglang.srt.model_executor.model_runner import ModelRunner
|
||||
@@ -481,20 +483,15 @@ class InklingShortConvAttnBackend(ShortConvAttnBackend):
|
||||
# batch always has them, and must still fail rather than track against
|
||||
# an invented prefix.
|
||||
prefix_lens = forward_batch.extend_prefix_lens
|
||||
if prefix_lens is None and on_graph_path:
|
||||
prefix_lens = torch.zeros_like(forward_batch.mamba_track_seqlens)
|
||||
track_seqlens = forward_batch.mamba_track_seqlens
|
||||
assert query_start_loc is not None and track_seqlens is not None
|
||||
assert prefix_lens is not None or on_graph_path
|
||||
live = min(
|
||||
rows,
|
||||
forward_batch.mamba_track_seqlens.shape[0],
|
||||
prefix_lens.shape[0],
|
||||
track_seqlens.shape[0],
|
||||
prefix_lens.shape[0] if prefix_lens is not None else rows,
|
||||
)
|
||||
|
||||
lens_to_track = forward_batch.mamba_track_seqlens[:live] - prefix_lens[:live]
|
||||
chunk_aligned = (
|
||||
lens_to_track // self.mamba_cache_chunk_size
|
||||
) * self.mamba_cache_chunk_size
|
||||
start_indices = query_start_loc[:live] + chunk_aligned - self.conv_state_len
|
||||
|
||||
if on_graph_path:
|
||||
assert rows <= self._graph_track_conv_indices.shape[0], (
|
||||
f"track-index buffer too small for a captured shape: rows={rows} "
|
||||
@@ -505,8 +502,30 @@ class InklingShortConvAttnBackend(ShortConvAttnBackend):
|
||||
out = torch.empty(
|
||||
(rows, self.conv_state_len),
|
||||
dtype=torch.int64,
|
||||
device=start_indices.device,
|
||||
device=query_start_loc.device,
|
||||
)
|
||||
|
||||
if is_cuda() and query_start_loc.is_cuda:
|
||||
fill_track_conv_indices(
|
||||
query_start_loc=query_start_loc,
|
||||
track_seqlens=track_seqlens,
|
||||
prefix_lens=prefix_lens,
|
||||
output=out,
|
||||
live=live,
|
||||
chunk_size=self.mamba_cache_chunk_size,
|
||||
)
|
||||
self.sconv_metadata.track_conv_indices = out
|
||||
return
|
||||
|
||||
if prefix_lens is None:
|
||||
prefix_lens = torch.zeros_like(track_seqlens)
|
||||
|
||||
lens_to_track = forward_batch.mamba_track_seqlens[:live] - prefix_lens[:live]
|
||||
chunk_aligned = (
|
||||
lens_to_track // self.mamba_cache_chunk_size
|
||||
) * self.mamba_cache_chunk_size
|
||||
start_indices = query_start_loc[:live] + chunk_aligned - self.conv_state_len
|
||||
|
||||
torch.add(
|
||||
start_indices.unsqueeze(-1).to(torch.int64),
|
||||
self._track_window_offsets,
|
||||
|
||||
Reference in New Issue
Block a user