fix(inkling): translate unified-memory checkpoint destinations to physical slots (#38229)
Co-authored-by: Cheng Wan <cheng.wan@radixark.ai>
This commit is contained in:
@@ -85,6 +85,8 @@ class InklingShortConvMetadata(msgspec.Struct):
|
||||
# [B, conv_kernel - 1] input positions whose conv window feeds the prefix
|
||||
# cache. Extend only, and only when tracking is on.
|
||||
track_conv_indices: Optional[torch.Tensor] = None
|
||||
# Kernel-facing checkpoint slots. ForwardBatch retains the virtual source.
|
||||
track_cache_indices: Optional[torch.Tensor] = None
|
||||
|
||||
|
||||
class InklingShortConvAttnBackend(ShortConvAttnBackend):
|
||||
@@ -144,7 +146,10 @@ class InklingShortConvAttnBackend(ShortConvAttnBackend):
|
||||
self._graph_track_conv_indices = torch.zeros(
|
||||
(max_bs, self.conv_state_len), dtype=torch.int64, device=dev
|
||||
)
|
||||
# Inert track fields for graph capture: a capture warmup batch that
|
||||
# Prefill can capture before init_cuda_graph_state. Keep the translated
|
||||
# checkpoint destinations at one address across both graph phases.
|
||||
self._graph_track_indices = torch.empty(max_bs, dtype=torch.int64, device=dev)
|
||||
# Inert prefill track fields (excluding draft extend): a warmup batch that
|
||||
# carries no tracking metadata must still LAUNCH the track scatter
|
||||
# (all rows masked off), or the python-level `if` specializes the
|
||||
# scatter out of the captured graph.
|
||||
@@ -155,8 +160,6 @@ class InklingShortConvAttnBackend(ShortConvAttnBackend):
|
||||
self._graph_track_inert_seqlens = torch.zeros(
|
||||
max_bs, dtype=torch.int64, device=dev
|
||||
)
|
||||
# Graph-static: captured track scatters read this address.
|
||||
self._track_indices_buf = torch.zeros(max_bs, dtype=torch.int64, device=dev)
|
||||
# Same address-stability requirement; the base only sizes this from
|
||||
# init_cuda_graph_state, which the prefill graph never calls.
|
||||
self._alloc_cache_indices_buf(max_bs)
|
||||
@@ -250,6 +253,7 @@ class InklingShortConvAttnBackend(ShortConvAttnBackend):
|
||||
|
||||
def _prepare_slot_indices(self, forward_batch: ForwardBatch):
|
||||
self._reset_step_state()
|
||||
self._prepare_track_indices(forward_batch)
|
||||
req_pool_indices = forward_batch.req_pool_indices
|
||||
n = req_pool_indices.shape[0]
|
||||
buf = self._cache_indices_buf
|
||||
@@ -271,22 +275,22 @@ class InklingShortConvAttnBackend(ShortConvAttnBackend):
|
||||
return
|
||||
self.forward_metadata = self._forward_metadata(forward_batch)
|
||||
self._refresh_cache_indices()
|
||||
if not self._slot_gather_recordable:
|
||||
self._translate_track_indices(forward_batch)
|
||||
|
||||
def _translate_track_indices(self, forward_batch: ForwardBatch):
|
||||
"""Translate virtual track ids to the physical slots the conv kernels write."""
|
||||
track_indices = forward_batch.mamba_track_indices
|
||||
buf = self._track_indices_buf
|
||||
# Prep may re-run on the same batch; never translate twice.
|
||||
if track_indices is None or track_indices.data_ptr() == buf.data_ptr():
|
||||
def _prepare_track_indices(self, forward_batch: ForwardBatch):
|
||||
# Every metadata view must explicitly provide virtual IDs or None.
|
||||
indices = forward_batch.mamba_track_indices
|
||||
if indices is None or self._slot_gather_recordable:
|
||||
self.sconv_metadata.track_cache_indices = indices
|
||||
return
|
||||
n = track_indices.shape[0]
|
||||
assert n <= buf.shape[0], (
|
||||
f"track-index buffer too small: rows={n} vs bound {buf.shape[0]}"
|
||||
n = indices.shape[0]
|
||||
assert n <= self._graph_track_indices.shape[0], (
|
||||
"checkpoint-index buffer too small for the forward batch"
|
||||
)
|
||||
buf[:n].copy_(self._translate_mamba_indices(track_indices))
|
||||
forward_batch.mamba_track_indices = buf[:n]
|
||||
# Keep the batch virtual across repeated eager metadata initialization
|
||||
# and across draft backends. Captured consumers read this fixed buffer.
|
||||
out = self._graph_track_indices[:n]
|
||||
out.copy_(self._translate_mamba_indices(indices))
|
||||
self.sconv_metadata.track_cache_indices = out
|
||||
|
||||
def _refresh_sconv_metadata(
|
||||
self, forward_batch: ForwardBatch, *, on_graph_path: bool
|
||||
@@ -451,6 +455,11 @@ class InklingShortConvAttnBackend(ShortConvAttnBackend):
|
||||
``batch_size`` rows while the track lengths cover only live requests, and
|
||||
every row it may read must index inside *this* replay's token buffer.
|
||||
"""
|
||||
if forward_batch.forward_mode.is_draft_extend_v2():
|
||||
# Draft extend snapshots the accepted window in its own cache update.
|
||||
# Inert prefill tracking would overwrite its checkpoint destinations.
|
||||
self.sconv_metadata.track_conv_indices = None
|
||||
return
|
||||
if forward_batch.mamba_track_mask is None:
|
||||
if not on_graph_path:
|
||||
self.sconv_metadata.track_conv_indices = None
|
||||
@@ -460,6 +469,9 @@ class InklingShortConvAttnBackend(ShortConvAttnBackend):
|
||||
rows = forward_batch.batch_size
|
||||
forward_batch.mamba_track_mask = self._graph_track_inert_mask[:rows]
|
||||
forward_batch.mamba_track_indices = self._graph_track_inert_indices[:rows]
|
||||
self.sconv_metadata.track_cache_indices = self._graph_track_inert_indices[
|
||||
:rows
|
||||
]
|
||||
forward_batch.mamba_track_seqlens = self._graph_track_inert_seqlens[:rows]
|
||||
rows = forward_batch.batch_size
|
||||
query_start_loc = self.sconv_metadata.query_start_loc
|
||||
@@ -532,6 +544,8 @@ class InklingShortConvAttnBackend(ShortConvAttnBackend):
|
||||
slot_ids = self._translate_mamba_indices(
|
||||
pool.get_mamba_indices(req_pool_indices)
|
||||
)
|
||||
if mamba_track_indices is not None:
|
||||
mamba_track_indices = self._translate_mamba_indices(mamba_track_indices)
|
||||
scatter_mamba_states_after_mtp_verify(
|
||||
pool.get_speculative_mamba2_params_all_layers(),
|
||||
slot_ids,
|
||||
@@ -603,6 +617,7 @@ class InklingShortConvHybridAttnBackend(ShortConvHybridAttnBackend):
|
||||
and self.supports_draft_extend_metadata_staging
|
||||
):
|
||||
self.short_conv_backend._reset_step_state()
|
||||
self.short_conv_backend._prepare_track_indices(forward_batch)
|
||||
self.short_conv_backend._refresh_sconv_metadata(
|
||||
forward_batch, on_graph_path=True
|
||||
)
|
||||
|
||||
@@ -725,7 +725,7 @@ class InklingAttention(nn.Module):
|
||||
activation=self.k_sconv.activation,
|
||||
use_residual=self.k_sconv.use_residual,
|
||||
track_mask=forward_batch.mamba_track_mask,
|
||||
track_indices=forward_batch.mamba_track_indices,
|
||||
track_indices=self.k_sconv._conv_state(forward_batch).track_cache_indices,
|
||||
do_store=do_store,
|
||||
mxfp8_quant=do_mxfp8_store,
|
||||
sfk=sfk,
|
||||
|
||||
@@ -359,7 +359,7 @@ def ar_sconv_norm_fused(
|
||||
activation=sconv.activation,
|
||||
use_residual=sconv.use_residual,
|
||||
track_mask=forward_batch.mamba_track_mask,
|
||||
track_indices=forward_batch.mamba_track_indices,
|
||||
track_indices=sconv._conv_state(forward_batch).track_cache_indices,
|
||||
shared=shared,
|
||||
)
|
||||
res.v5_cur = 1 - cur
|
||||
@@ -834,7 +834,7 @@ def ar_scattered_sconv_fused(
|
||||
if fm.is_decode() and forward_batch.mamba_track_mask is not None:
|
||||
b = forward_batch.batch_size
|
||||
track_mask = forward_batch.mamba_track_mask[:b]
|
||||
track_dst = forward_batch.mamba_track_indices[:b]
|
||||
track_dst = sconv._conv_state(forward_batch).track_cache_indices[:b]
|
||||
track_from_cache = True
|
||||
|
||||
esz = comm.buffer.element_size()
|
||||
|
||||
@@ -198,7 +198,7 @@ class ShortConvolution(nn.Module):
|
||||
activation=self.activation,
|
||||
use_residual=self.use_residual,
|
||||
track_mask=forward_batch.mamba_track_mask,
|
||||
track_indices=forward_batch.mamba_track_indices,
|
||||
track_indices=self._conv_state(forward_batch).track_cache_indices,
|
||||
)
|
||||
|
||||
def _prepare_extend_sconv_cache(
|
||||
@@ -216,7 +216,7 @@ class ShortConvolution(nn.Module):
|
||||
sconv_cache=sconv_cache,
|
||||
track_conv_indices=track_conv_indices,
|
||||
mask=forward_batch.mamba_track_mask,
|
||||
dst_indices=forward_batch.mamba_track_indices,
|
||||
dst_indices=self._conv_state(forward_batch).track_cache_indices,
|
||||
)
|
||||
|
||||
def _save_intermediate_conv_windows(
|
||||
@@ -269,7 +269,7 @@ class ShortConvolution(nn.Module):
|
||||
else:
|
||||
draft_token_num = hidden_states.shape[1]
|
||||
|
||||
mamba_track_indices = getattr(forward_batch, "mamba_track_indices", None)
|
||||
mamba_track_indices = self._conv_state(forward_batch).track_cache_indices
|
||||
do_tracking = (
|
||||
mamba_track_indices is not None
|
||||
and get_exec().mamba.enable_mamba_extra_buffer
|
||||
@@ -349,7 +349,7 @@ class ShortConvolution(nn.Module):
|
||||
if meta.track_conv_indices is not None:
|
||||
track_rows = meta.track_conv_indices
|
||||
track_mask = forward_batch.mamba_track_mask
|
||||
track_dst = forward_batch.mamba_track_indices
|
||||
track_dst = meta.track_cache_indices
|
||||
else:
|
||||
w1 = self.kernel_size[0] - 1
|
||||
track_rows = torch.empty((0, w1), dtype=torch.int64, device=dev)
|
||||
|
||||
@@ -0,0 +1,27 @@
|
||||
from typing import Optional
|
||||
|
||||
import torch
|
||||
|
||||
|
||||
def track_indices(buffer: Optional[torch.Tensor], bs: int):
|
||||
return None if buffer is None else buffer[:bs]
|
||||
|
||||
|
||||
def refresh_track_indices(buffer, src, *, raw_bs, bs, copy_dsts=None, copy_srcs=None):
|
||||
"""Refresh virtual destinations without clearing rows that will be copied.
|
||||
|
||||
Ordinary EAGLE joins its existing grouped copy. The multi-layer runner
|
||||
uses a fused preparation kernel for other inputs and copies here instead.
|
||||
"""
|
||||
if buffer is None:
|
||||
return
|
||||
if src is None:
|
||||
buffer[:bs].zero_()
|
||||
return
|
||||
if raw_bs < bs:
|
||||
buffer[raw_bs:bs].zero_()
|
||||
if copy_dsts is None:
|
||||
buffer[:raw_bs].copy_(src)
|
||||
else:
|
||||
copy_dsts.append(buffer[:raw_bs])
|
||||
copy_srcs.append(src)
|
||||
@@ -43,6 +43,7 @@ from sglang.srt.runtime_context import (
|
||||
get_parallel,
|
||||
get_spec,
|
||||
)
|
||||
from sglang.srt.speculative.draft_checkpoint import refresh_track_indices, track_indices
|
||||
from sglang.srt.speculative.eagle_info import EagleDraftExtendInput
|
||||
from sglang.srt.speculative.eagle_utils import get_draft_input_from_target_hidden_dim
|
||||
from sglang.srt.speculative.spec_utils import resolve_num_tokens_per_req
|
||||
@@ -74,6 +75,7 @@ def resolve_draft_extend_seq_len_fill_value(
|
||||
class EagleDraftExtendInputBuffers(ForwardInputBuffers):
|
||||
input_ids: torch.Tensor
|
||||
req_pool_indices: torch.Tensor
|
||||
mamba_track_indices: Optional[torch.Tensor]
|
||||
out_cache_loc: torch.Tensor
|
||||
positions: torch.Tensor
|
||||
mrope_positions: torch.Tensor
|
||||
@@ -268,6 +270,11 @@ class EAGLEDraftExtendCudaGraphRunner(DecodeCudaGraphRunner):
|
||||
self.buffers = EagleDraftExtendInputBuffers(
|
||||
input_ids=input_ids,
|
||||
req_pool_indices=req_pool_indices,
|
||||
mamba_track_indices=(
|
||||
torch.zeros(self.max_bs, dtype=torch.int64, device=self.device)
|
||||
if get_exec().mamba.enable_mamba_extra_buffer
|
||||
else None
|
||||
),
|
||||
out_cache_loc=out_cache_loc,
|
||||
positions=positions,
|
||||
mrope_positions=mrope_positions,
|
||||
@@ -421,6 +428,7 @@ class EAGLEDraftExtendCudaGraphRunner(DecodeCudaGraphRunner):
|
||||
batch_size=bs,
|
||||
input_ids=input_ids,
|
||||
req_pool_indices=req_pool_indices,
|
||||
mamba_track_indices=track_indices(buffers.mamba_track_indices, bs),
|
||||
seq_lens=seq_lens,
|
||||
seq_lens_cpu=seq_lens_cpu,
|
||||
next_token_logits_buffer=next_token_logits_buffer,
|
||||
@@ -572,6 +580,14 @@ class EAGLEDraftExtendCudaGraphRunner(DecodeCudaGraphRunner):
|
||||
copy_srcs.append(forward_batch.spec_info.num_accept_tokens)
|
||||
copy_dsts.append(buffers.select_index[:raw_bs])
|
||||
copy_srcs.append(select_index)
|
||||
refresh_track_indices(
|
||||
buffers.mamba_track_indices,
|
||||
forward_batch.mamba_track_indices,
|
||||
raw_bs=raw_bs,
|
||||
bs=bs,
|
||||
copy_dsts=copy_dsts,
|
||||
copy_srcs=copy_srcs,
|
||||
)
|
||||
_grouped_foreach_copy_(copy_dsts, copy_srcs)
|
||||
|
||||
# hidden_states is large + contiguous: copy_() uses the cudaMemcpyAsync
|
||||
@@ -633,6 +649,8 @@ class EAGLEDraftExtendCudaGraphRunner(DecodeCudaGraphRunner):
|
||||
encoder_lens=None,
|
||||
out_cache_loc=buffers.out_cache_loc[:num_tokens],
|
||||
out_cache_loc_dsv4=getattr(forward_batch, "out_cache_loc_dsv4", None),
|
||||
# Virtual input stays separate from the backend's physical buffer.
|
||||
mamba_track_indices=track_indices(buffers.mamba_track_indices, bs),
|
||||
spec_info=forward_batch.spec_info,
|
||||
)
|
||||
self.draft_extend_attn_backend.init_forward_metadata_out_graph(fb_view)
|
||||
|
||||
@@ -66,6 +66,7 @@ from sglang.srt.runtime_context import (
|
||||
get_parallel,
|
||||
get_spec,
|
||||
)
|
||||
from sglang.srt.speculative.draft_checkpoint import refresh_track_indices, track_indices
|
||||
from sglang.srt.speculative.eagle_info import EagleDraftExtendInput
|
||||
from sglang.srt.speculative.eagle_utils import get_draft_input_from_target_hidden_dim
|
||||
from sglang.srt.speculative.multi_layer_eagle_utils import (
|
||||
@@ -116,6 +117,7 @@ class MultiLayerEagleDraftExtendInputBuffers(ForwardInputBuffers):
|
||||
seq_lens: torch.Tensor
|
||||
seq_lens_cpu: torch.Tensor
|
||||
req_pool_indices: torch.Tensor
|
||||
mamba_track_indices: Optional[torch.Tensor]
|
||||
num_correct_drafts: torch.Tensor
|
||||
num_accept_tokens: torch.Tensor
|
||||
extend_seq_lens: torch.Tensor
|
||||
@@ -335,6 +337,7 @@ class MultiLayerEagleDraftExtendCudaGraphRunner(DecodeCudaGraphRunner):
|
||||
batch_size=bs,
|
||||
input_ids=input_ids,
|
||||
req_pool_indices=req_pool_indices,
|
||||
mamba_track_indices=track_indices(buffers.mamba_track_indices, bs),
|
||||
seq_lens=seq_lens,
|
||||
seq_lens_cpu=seq_lens_cpu,
|
||||
next_token_logits_buffer=next_token_logits_buffer,
|
||||
@@ -493,6 +496,8 @@ class MultiLayerEagleDraftExtendCudaGraphRunner(DecodeCudaGraphRunner):
|
||||
encoder_lens=None,
|
||||
# per-step write target; out_cache_loc is frozen at prepare() time.
|
||||
out_cache_loc=buffers.out_cache_loc[:num_tokens],
|
||||
# Virtual input stays separate from the backend's physical buffer.
|
||||
mamba_track_indices=track_indices(buffers.mamba_track_indices, bs),
|
||||
spec_info=spec_info,
|
||||
)
|
||||
if (
|
||||
@@ -702,6 +707,11 @@ class MultiLayerEagleMultiStepDraftExtendCudaGraphRunner:
|
||||
seq_lens=seq_lens,
|
||||
seq_lens_cpu=seq_lens_cpu,
|
||||
req_pool_indices=req_pool_indices,
|
||||
mamba_track_indices=(
|
||||
torch.zeros(self.max_bs, dtype=torch.int64, device=self.device)
|
||||
if get_exec().mamba.enable_mamba_extra_buffer
|
||||
else None
|
||||
),
|
||||
num_correct_drafts=num_correct_drafts,
|
||||
num_accept_tokens=num_accept_tokens,
|
||||
extend_seq_lens=extend_seq_lens,
|
||||
@@ -755,6 +765,7 @@ class MultiLayerEagleMultiStepDraftExtendCudaGraphRunner:
|
||||
seq_lens=buffers.seq_lens[:bs],
|
||||
extend_seq_lens=buffers.extend_seq_lens[:bs],
|
||||
out_cache_loc=buffers.out_cache_loc[: bs * self.captured_req_width],
|
||||
mamba_track_indices=track_indices(buffers.mamba_track_indices, bs),
|
||||
)
|
||||
for backend in backends:
|
||||
backend.init_forward_metadata_out_graph(batch)
|
||||
@@ -773,6 +784,13 @@ class MultiLayerEagleMultiStepDraftExtendCudaGraphRunner:
|
||||
else:
|
||||
bs = self.get_runner(0)._pad_to_bucket(raw_bs, self.capture_bs)
|
||||
|
||||
refresh_track_indices(
|
||||
buffers.mamba_track_indices,
|
||||
forward_batch.mamba_track_indices,
|
||||
raw_bs=raw_bs,
|
||||
bs=bs,
|
||||
)
|
||||
|
||||
fill_draft_extend_prepare_buffers(
|
||||
buffers.input_ids,
|
||||
buffers.positions,
|
||||
|
||||
Reference in New Issue
Block a user