fix(inkling): translate unified-memory checkpoint destinations to physical slots (#38229)

Co-authored-by: Cheng Wan <cheng.wan@radixark.ai>
This commit is contained in:
SuperSong
2026-10-04 02:45:45 -07:00
committed by GitHub
co-authored by Cheng Wan
parent 1e490772e5
commit 1d02a36bb7
8 changed files with 1098 additions and 23 deletions
@@ -85,6 +85,8 @@ class InklingShortConvMetadata(msgspec.Struct):
# [B, conv_kernel - 1] input positions whose conv window feeds the prefix
# cache. Extend only, and only when tracking is on.
track_conv_indices: Optional[torch.Tensor] = None
# Kernel-facing checkpoint slots. ForwardBatch retains the virtual source.
track_cache_indices: Optional[torch.Tensor] = None
class InklingShortConvAttnBackend(ShortConvAttnBackend):
@@ -144,7 +146,10 @@ class InklingShortConvAttnBackend(ShortConvAttnBackend):
self._graph_track_conv_indices = torch.zeros(
(max_bs, self.conv_state_len), dtype=torch.int64, device=dev
)
# Inert track fields for graph capture: a capture warmup batch that
# Prefill can capture before init_cuda_graph_state. Keep the translated
# checkpoint destinations at one address across both graph phases.
self._graph_track_indices = torch.empty(max_bs, dtype=torch.int64, device=dev)
# Inert prefill track fields (excluding draft extend): a warmup batch that
# carries no tracking metadata must still LAUNCH the track scatter
# (all rows masked off), or the python-level `if` specializes the
# scatter out of the captured graph.
@@ -155,8 +160,6 @@ class InklingShortConvAttnBackend(ShortConvAttnBackend):
self._graph_track_inert_seqlens = torch.zeros(
max_bs, dtype=torch.int64, device=dev
)
# Graph-static: captured track scatters read this address.
self._track_indices_buf = torch.zeros(max_bs, dtype=torch.int64, device=dev)
# Same address-stability requirement; the base only sizes this from
# init_cuda_graph_state, which the prefill graph never calls.
self._alloc_cache_indices_buf(max_bs)
@@ -250,6 +253,7 @@ class InklingShortConvAttnBackend(ShortConvAttnBackend):
def _prepare_slot_indices(self, forward_batch: ForwardBatch):
self._reset_step_state()
self._prepare_track_indices(forward_batch)
req_pool_indices = forward_batch.req_pool_indices
n = req_pool_indices.shape[0]
buf = self._cache_indices_buf
@@ -271,22 +275,22 @@ class InklingShortConvAttnBackend(ShortConvAttnBackend):
return
self.forward_metadata = self._forward_metadata(forward_batch)
self._refresh_cache_indices()
if not self._slot_gather_recordable:
self._translate_track_indices(forward_batch)
def _translate_track_indices(self, forward_batch: ForwardBatch):
"""Translate virtual track ids to the physical slots the conv kernels write."""
track_indices = forward_batch.mamba_track_indices
buf = self._track_indices_buf
# Prep may re-run on the same batch; never translate twice.
if track_indices is None or track_indices.data_ptr() == buf.data_ptr():
def _prepare_track_indices(self, forward_batch: ForwardBatch):
# Every metadata view must explicitly provide virtual IDs or None.
indices = forward_batch.mamba_track_indices
if indices is None or self._slot_gather_recordable:
self.sconv_metadata.track_cache_indices = indices
return
n = track_indices.shape[0]
assert n <= buf.shape[0], (
f"track-index buffer too small: rows={n} vs bound {buf.shape[0]}"
n = indices.shape[0]
assert n <= self._graph_track_indices.shape[0], (
"checkpoint-index buffer too small for the forward batch"
)
buf[:n].copy_(self._translate_mamba_indices(track_indices))
forward_batch.mamba_track_indices = buf[:n]
# Keep the batch virtual across repeated eager metadata initialization
# and across draft backends. Captured consumers read this fixed buffer.
out = self._graph_track_indices[:n]
out.copy_(self._translate_mamba_indices(indices))
self.sconv_metadata.track_cache_indices = out
def _refresh_sconv_metadata(
self, forward_batch: ForwardBatch, *, on_graph_path: bool
@@ -451,6 +455,11 @@ class InklingShortConvAttnBackend(ShortConvAttnBackend):
``batch_size`` rows while the track lengths cover only live requests, and
every row it may read must index inside *this* replay's token buffer.
"""
if forward_batch.forward_mode.is_draft_extend_v2():
# Draft extend snapshots the accepted window in its own cache update.
# Inert prefill tracking would overwrite its checkpoint destinations.
self.sconv_metadata.track_conv_indices = None
return
if forward_batch.mamba_track_mask is None:
if not on_graph_path:
self.sconv_metadata.track_conv_indices = None
@@ -460,6 +469,9 @@ class InklingShortConvAttnBackend(ShortConvAttnBackend):
rows = forward_batch.batch_size
forward_batch.mamba_track_mask = self._graph_track_inert_mask[:rows]
forward_batch.mamba_track_indices = self._graph_track_inert_indices[:rows]
self.sconv_metadata.track_cache_indices = self._graph_track_inert_indices[
:rows
]
forward_batch.mamba_track_seqlens = self._graph_track_inert_seqlens[:rows]
rows = forward_batch.batch_size
query_start_loc = self.sconv_metadata.query_start_loc
@@ -532,6 +544,8 @@ class InklingShortConvAttnBackend(ShortConvAttnBackend):
slot_ids = self._translate_mamba_indices(
pool.get_mamba_indices(req_pool_indices)
)
if mamba_track_indices is not None:
mamba_track_indices = self._translate_mamba_indices(mamba_track_indices)
scatter_mamba_states_after_mtp_verify(
pool.get_speculative_mamba2_params_all_layers(),
slot_ids,
@@ -603,6 +617,7 @@ class InklingShortConvHybridAttnBackend(ShortConvHybridAttnBackend):
and self.supports_draft_extend_metadata_staging
):
self.short_conv_backend._reset_step_state()
self.short_conv_backend._prepare_track_indices(forward_batch)
self.short_conv_backend._refresh_sconv_metadata(
forward_batch, on_graph_path=True
)
@@ -725,7 +725,7 @@ class InklingAttention(nn.Module):
activation=self.k_sconv.activation,
use_residual=self.k_sconv.use_residual,
track_mask=forward_batch.mamba_track_mask,
track_indices=forward_batch.mamba_track_indices,
track_indices=self.k_sconv._conv_state(forward_batch).track_cache_indices,
do_store=do_store,
mxfp8_quant=do_mxfp8_store,
sfk=sfk,
@@ -359,7 +359,7 @@ def ar_sconv_norm_fused(
activation=sconv.activation,
use_residual=sconv.use_residual,
track_mask=forward_batch.mamba_track_mask,
track_indices=forward_batch.mamba_track_indices,
track_indices=sconv._conv_state(forward_batch).track_cache_indices,
shared=shared,
)
res.v5_cur = 1 - cur
@@ -834,7 +834,7 @@ def ar_scattered_sconv_fused(
if fm.is_decode() and forward_batch.mamba_track_mask is not None:
b = forward_batch.batch_size
track_mask = forward_batch.mamba_track_mask[:b]
track_dst = forward_batch.mamba_track_indices[:b]
track_dst = sconv._conv_state(forward_batch).track_cache_indices[:b]
track_from_cache = True
esz = comm.buffer.element_size()
@@ -198,7 +198,7 @@ class ShortConvolution(nn.Module):
activation=self.activation,
use_residual=self.use_residual,
track_mask=forward_batch.mamba_track_mask,
track_indices=forward_batch.mamba_track_indices,
track_indices=self._conv_state(forward_batch).track_cache_indices,
)
def _prepare_extend_sconv_cache(
@@ -216,7 +216,7 @@ class ShortConvolution(nn.Module):
sconv_cache=sconv_cache,
track_conv_indices=track_conv_indices,
mask=forward_batch.mamba_track_mask,
dst_indices=forward_batch.mamba_track_indices,
dst_indices=self._conv_state(forward_batch).track_cache_indices,
)
def _save_intermediate_conv_windows(
@@ -269,7 +269,7 @@ class ShortConvolution(nn.Module):
else:
draft_token_num = hidden_states.shape[1]
mamba_track_indices = getattr(forward_batch, "mamba_track_indices", None)
mamba_track_indices = self._conv_state(forward_batch).track_cache_indices
do_tracking = (
mamba_track_indices is not None
and get_exec().mamba.enable_mamba_extra_buffer
@@ -349,7 +349,7 @@ class ShortConvolution(nn.Module):
if meta.track_conv_indices is not None:
track_rows = meta.track_conv_indices
track_mask = forward_batch.mamba_track_mask
track_dst = forward_batch.mamba_track_indices
track_dst = meta.track_cache_indices
else:
w1 = self.kernel_size[0] - 1
track_rows = torch.empty((0, w1), dtype=torch.int64, device=dev)
@@ -0,0 +1,27 @@
from typing import Optional
import torch
def track_indices(buffer: Optional[torch.Tensor], bs: int):
return None if buffer is None else buffer[:bs]
def refresh_track_indices(buffer, src, *, raw_bs, bs, copy_dsts=None, copy_srcs=None):
"""Refresh virtual destinations without clearing rows that will be copied.
Ordinary EAGLE joins its existing grouped copy. The multi-layer runner
uses a fused preparation kernel for other inputs and copies here instead.
"""
if buffer is None:
return
if src is None:
buffer[:bs].zero_()
return
if raw_bs < bs:
buffer[raw_bs:bs].zero_()
if copy_dsts is None:
buffer[:raw_bs].copy_(src)
else:
copy_dsts.append(buffer[:raw_bs])
copy_srcs.append(src)
@@ -43,6 +43,7 @@ from sglang.srt.runtime_context import (
get_parallel,
get_spec,
)
from sglang.srt.speculative.draft_checkpoint import refresh_track_indices, track_indices
from sglang.srt.speculative.eagle_info import EagleDraftExtendInput
from sglang.srt.speculative.eagle_utils import get_draft_input_from_target_hidden_dim
from sglang.srt.speculative.spec_utils import resolve_num_tokens_per_req
@@ -74,6 +75,7 @@ def resolve_draft_extend_seq_len_fill_value(
class EagleDraftExtendInputBuffers(ForwardInputBuffers):
input_ids: torch.Tensor
req_pool_indices: torch.Tensor
mamba_track_indices: Optional[torch.Tensor]
out_cache_loc: torch.Tensor
positions: torch.Tensor
mrope_positions: torch.Tensor
@@ -268,6 +270,11 @@ class EAGLEDraftExtendCudaGraphRunner(DecodeCudaGraphRunner):
self.buffers = EagleDraftExtendInputBuffers(
input_ids=input_ids,
req_pool_indices=req_pool_indices,
mamba_track_indices=(
torch.zeros(self.max_bs, dtype=torch.int64, device=self.device)
if get_exec().mamba.enable_mamba_extra_buffer
else None
),
out_cache_loc=out_cache_loc,
positions=positions,
mrope_positions=mrope_positions,
@@ -421,6 +428,7 @@ class EAGLEDraftExtendCudaGraphRunner(DecodeCudaGraphRunner):
batch_size=bs,
input_ids=input_ids,
req_pool_indices=req_pool_indices,
mamba_track_indices=track_indices(buffers.mamba_track_indices, bs),
seq_lens=seq_lens,
seq_lens_cpu=seq_lens_cpu,
next_token_logits_buffer=next_token_logits_buffer,
@@ -572,6 +580,14 @@ class EAGLEDraftExtendCudaGraphRunner(DecodeCudaGraphRunner):
copy_srcs.append(forward_batch.spec_info.num_accept_tokens)
copy_dsts.append(buffers.select_index[:raw_bs])
copy_srcs.append(select_index)
refresh_track_indices(
buffers.mamba_track_indices,
forward_batch.mamba_track_indices,
raw_bs=raw_bs,
bs=bs,
copy_dsts=copy_dsts,
copy_srcs=copy_srcs,
)
_grouped_foreach_copy_(copy_dsts, copy_srcs)
# hidden_states is large + contiguous: copy_() uses the cudaMemcpyAsync
@@ -633,6 +649,8 @@ class EAGLEDraftExtendCudaGraphRunner(DecodeCudaGraphRunner):
encoder_lens=None,
out_cache_loc=buffers.out_cache_loc[:num_tokens],
out_cache_loc_dsv4=getattr(forward_batch, "out_cache_loc_dsv4", None),
# Virtual input stays separate from the backend's physical buffer.
mamba_track_indices=track_indices(buffers.mamba_track_indices, bs),
spec_info=forward_batch.spec_info,
)
self.draft_extend_attn_backend.init_forward_metadata_out_graph(fb_view)
@@ -66,6 +66,7 @@ from sglang.srt.runtime_context import (
get_parallel,
get_spec,
)
from sglang.srt.speculative.draft_checkpoint import refresh_track_indices, track_indices
from sglang.srt.speculative.eagle_info import EagleDraftExtendInput
from sglang.srt.speculative.eagle_utils import get_draft_input_from_target_hidden_dim
from sglang.srt.speculative.multi_layer_eagle_utils import (
@@ -116,6 +117,7 @@ class MultiLayerEagleDraftExtendInputBuffers(ForwardInputBuffers):
seq_lens: torch.Tensor
seq_lens_cpu: torch.Tensor
req_pool_indices: torch.Tensor
mamba_track_indices: Optional[torch.Tensor]
num_correct_drafts: torch.Tensor
num_accept_tokens: torch.Tensor
extend_seq_lens: torch.Tensor
@@ -335,6 +337,7 @@ class MultiLayerEagleDraftExtendCudaGraphRunner(DecodeCudaGraphRunner):
batch_size=bs,
input_ids=input_ids,
req_pool_indices=req_pool_indices,
mamba_track_indices=track_indices(buffers.mamba_track_indices, bs),
seq_lens=seq_lens,
seq_lens_cpu=seq_lens_cpu,
next_token_logits_buffer=next_token_logits_buffer,
@@ -493,6 +496,8 @@ class MultiLayerEagleDraftExtendCudaGraphRunner(DecodeCudaGraphRunner):
encoder_lens=None,
# per-step write target; out_cache_loc is frozen at prepare() time.
out_cache_loc=buffers.out_cache_loc[:num_tokens],
# Virtual input stays separate from the backend's physical buffer.
mamba_track_indices=track_indices(buffers.mamba_track_indices, bs),
spec_info=spec_info,
)
if (
@@ -702,6 +707,11 @@ class MultiLayerEagleMultiStepDraftExtendCudaGraphRunner:
seq_lens=seq_lens,
seq_lens_cpu=seq_lens_cpu,
req_pool_indices=req_pool_indices,
mamba_track_indices=(
torch.zeros(self.max_bs, dtype=torch.int64, device=self.device)
if get_exec().mamba.enable_mamba_extra_buffer
else None
),
num_correct_drafts=num_correct_drafts,
num_accept_tokens=num_accept_tokens,
extend_seq_lens=extend_seq_lens,
@@ -755,6 +765,7 @@ class MultiLayerEagleMultiStepDraftExtendCudaGraphRunner:
seq_lens=buffers.seq_lens[:bs],
extend_seq_lens=buffers.extend_seq_lens[:bs],
out_cache_loc=buffers.out_cache_loc[: bs * self.captured_req_width],
mamba_track_indices=track_indices(buffers.mamba_track_indices, bs),
)
for backend in backends:
backend.init_forward_metadata_out_graph(batch)
@@ -773,6 +784,13 @@ class MultiLayerEagleMultiStepDraftExtendCudaGraphRunner:
else:
bs = self.get_runner(0)._pad_to_bucket(raw_bs, self.capture_bs)
refresh_track_indices(
buffers.mamba_track_indices,
forward_batch.mamba_track_indices,
raw_bs=raw_bs,
bs=bs,
)
fill_draft_extend_prepare_buffers(
buffers.input_ids,
buffers.positions,