fix(inkling): translate unified-memory checkpoint destinations to physical slots (#38229)

Co-authored-by: Cheng Wan <cheng.wan@radixark.ai>
This commit is contained in:
SuperSong
2026-10-04 02:45:45 -07:00
committed by GitHub
co-authored by Cheng Wan
parent 1e490772e5
commit 1d02a36bb7
8 changed files with 1098 additions and 23 deletions
@@ -85,6 +85,8 @@ class InklingShortConvMetadata(msgspec.Struct):
# [B, conv_kernel - 1] input positions whose conv window feeds the prefix
# cache. Extend only, and only when tracking is on.
track_conv_indices: Optional[torch.Tensor] = None
# Kernel-facing checkpoint slots. ForwardBatch retains the virtual source.
track_cache_indices: Optional[torch.Tensor] = None
class InklingShortConvAttnBackend(ShortConvAttnBackend):
@@ -144,7 +146,10 @@ class InklingShortConvAttnBackend(ShortConvAttnBackend):
self._graph_track_conv_indices = torch.zeros(
(max_bs, self.conv_state_len), dtype=torch.int64, device=dev
)
# Inert track fields for graph capture: a capture warmup batch that
# Prefill can capture before init_cuda_graph_state. Keep the translated
# checkpoint destinations at one address across both graph phases.
self._graph_track_indices = torch.empty(max_bs, dtype=torch.int64, device=dev)
# Inert prefill track fields (excluding draft extend): a warmup batch that
# carries no tracking metadata must still LAUNCH the track scatter
# (all rows masked off), or the python-level `if` specializes the
# scatter out of the captured graph.
@@ -155,8 +160,6 @@ class InklingShortConvAttnBackend(ShortConvAttnBackend):
self._graph_track_inert_seqlens = torch.zeros(
max_bs, dtype=torch.int64, device=dev
)
# Graph-static: captured track scatters read this address.
self._track_indices_buf = torch.zeros(max_bs, dtype=torch.int64, device=dev)
# Same address-stability requirement; the base only sizes this from
# init_cuda_graph_state, which the prefill graph never calls.
self._alloc_cache_indices_buf(max_bs)
@@ -250,6 +253,7 @@ class InklingShortConvAttnBackend(ShortConvAttnBackend):
def _prepare_slot_indices(self, forward_batch: ForwardBatch):
self._reset_step_state()
self._prepare_track_indices(forward_batch)
req_pool_indices = forward_batch.req_pool_indices
n = req_pool_indices.shape[0]
buf = self._cache_indices_buf
@@ -271,22 +275,22 @@ class InklingShortConvAttnBackend(ShortConvAttnBackend):
return
self.forward_metadata = self._forward_metadata(forward_batch)
self._refresh_cache_indices()
if not self._slot_gather_recordable:
self._translate_track_indices(forward_batch)
def _translate_track_indices(self, forward_batch: ForwardBatch):
"""Translate virtual track ids to the physical slots the conv kernels write."""
track_indices = forward_batch.mamba_track_indices
buf = self._track_indices_buf
# Prep may re-run on the same batch; never translate twice.
if track_indices is None or track_indices.data_ptr() == buf.data_ptr():
def _prepare_track_indices(self, forward_batch: ForwardBatch):
# Every metadata view must explicitly provide virtual IDs or None.
indices = forward_batch.mamba_track_indices
if indices is None or self._slot_gather_recordable:
self.sconv_metadata.track_cache_indices = indices
return
n = track_indices.shape[0]
assert n <= buf.shape[0], (
f"track-index buffer too small: rows={n} vs bound {buf.shape[0]}"
n = indices.shape[0]
assert n <= self._graph_track_indices.shape[0], (
"checkpoint-index buffer too small for the forward batch"
)
buf[:n].copy_(self._translate_mamba_indices(track_indices))
forward_batch.mamba_track_indices = buf[:n]
# Keep the batch virtual across repeated eager metadata initialization
# and across draft backends. Captured consumers read this fixed buffer.
out = self._graph_track_indices[:n]
out.copy_(self._translate_mamba_indices(indices))
self.sconv_metadata.track_cache_indices = out
def _refresh_sconv_metadata(
self, forward_batch: ForwardBatch, *, on_graph_path: bool
@@ -451,6 +455,11 @@ class InklingShortConvAttnBackend(ShortConvAttnBackend):
``batch_size`` rows while the track lengths cover only live requests, and
every row it may read must index inside *this* replay's token buffer.
"""
if forward_batch.forward_mode.is_draft_extend_v2():
# Draft extend snapshots the accepted window in its own cache update.
# Inert prefill tracking would overwrite its checkpoint destinations.
self.sconv_metadata.track_conv_indices = None
return
if forward_batch.mamba_track_mask is None:
if not on_graph_path:
self.sconv_metadata.track_conv_indices = None
@@ -460,6 +469,9 @@ class InklingShortConvAttnBackend(ShortConvAttnBackend):
rows = forward_batch.batch_size
forward_batch.mamba_track_mask = self._graph_track_inert_mask[:rows]
forward_batch.mamba_track_indices = self._graph_track_inert_indices[:rows]
self.sconv_metadata.track_cache_indices = self._graph_track_inert_indices[
:rows
]
forward_batch.mamba_track_seqlens = self._graph_track_inert_seqlens[:rows]
rows = forward_batch.batch_size
query_start_loc = self.sconv_metadata.query_start_loc
@@ -532,6 +544,8 @@ class InklingShortConvAttnBackend(ShortConvAttnBackend):
slot_ids = self._translate_mamba_indices(
pool.get_mamba_indices(req_pool_indices)
)
if mamba_track_indices is not None:
mamba_track_indices = self._translate_mamba_indices(mamba_track_indices)
scatter_mamba_states_after_mtp_verify(
pool.get_speculative_mamba2_params_all_layers(),
slot_ids,
@@ -603,6 +617,7 @@ class InklingShortConvHybridAttnBackend(ShortConvHybridAttnBackend):
and self.supports_draft_extend_metadata_staging
):
self.short_conv_backend._reset_step_state()
self.short_conv_backend._prepare_track_indices(forward_batch)
self.short_conv_backend._refresh_sconv_metadata(
forward_batch, on_graph_path=True
)
@@ -725,7 +725,7 @@ class InklingAttention(nn.Module):
activation=self.k_sconv.activation,
use_residual=self.k_sconv.use_residual,
track_mask=forward_batch.mamba_track_mask,
track_indices=forward_batch.mamba_track_indices,
track_indices=self.k_sconv._conv_state(forward_batch).track_cache_indices,
do_store=do_store,
mxfp8_quant=do_mxfp8_store,
sfk=sfk,
@@ -359,7 +359,7 @@ def ar_sconv_norm_fused(
activation=sconv.activation,
use_residual=sconv.use_residual,
track_mask=forward_batch.mamba_track_mask,
track_indices=forward_batch.mamba_track_indices,
track_indices=sconv._conv_state(forward_batch).track_cache_indices,
shared=shared,
)
res.v5_cur = 1 - cur
@@ -834,7 +834,7 @@ def ar_scattered_sconv_fused(
if fm.is_decode() and forward_batch.mamba_track_mask is not None:
b = forward_batch.batch_size
track_mask = forward_batch.mamba_track_mask[:b]
track_dst = forward_batch.mamba_track_indices[:b]
track_dst = sconv._conv_state(forward_batch).track_cache_indices[:b]
track_from_cache = True
esz = comm.buffer.element_size()
@@ -198,7 +198,7 @@ class ShortConvolution(nn.Module):
activation=self.activation,
use_residual=self.use_residual,
track_mask=forward_batch.mamba_track_mask,
track_indices=forward_batch.mamba_track_indices,
track_indices=self._conv_state(forward_batch).track_cache_indices,
)
def _prepare_extend_sconv_cache(
@@ -216,7 +216,7 @@ class ShortConvolution(nn.Module):
sconv_cache=sconv_cache,
track_conv_indices=track_conv_indices,
mask=forward_batch.mamba_track_mask,
dst_indices=forward_batch.mamba_track_indices,
dst_indices=self._conv_state(forward_batch).track_cache_indices,
)
def _save_intermediate_conv_windows(
@@ -269,7 +269,7 @@ class ShortConvolution(nn.Module):
else:
draft_token_num = hidden_states.shape[1]
mamba_track_indices = getattr(forward_batch, "mamba_track_indices", None)
mamba_track_indices = self._conv_state(forward_batch).track_cache_indices
do_tracking = (
mamba_track_indices is not None
and get_exec().mamba.enable_mamba_extra_buffer
@@ -349,7 +349,7 @@ class ShortConvolution(nn.Module):
if meta.track_conv_indices is not None:
track_rows = meta.track_conv_indices
track_mask = forward_batch.mamba_track_mask
track_dst = forward_batch.mamba_track_indices
track_dst = meta.track_cache_indices
else:
w1 = self.kernel_size[0] - 1
track_rows = torch.empty((0, w1), dtype=torch.int64, device=dev)
@@ -0,0 +1,27 @@
from typing import Optional
import torch
def track_indices(buffer: Optional[torch.Tensor], bs: int):
return None if buffer is None else buffer[:bs]
def refresh_track_indices(buffer, src, *, raw_bs, bs, copy_dsts=None, copy_srcs=None):
"""Refresh virtual destinations without clearing rows that will be copied.
Ordinary EAGLE joins its existing grouped copy. The multi-layer runner
uses a fused preparation kernel for other inputs and copies here instead.
"""
if buffer is None:
return
if src is None:
buffer[:bs].zero_()
return
if raw_bs < bs:
buffer[raw_bs:bs].zero_()
if copy_dsts is None:
buffer[:raw_bs].copy_(src)
else:
copy_dsts.append(buffer[:raw_bs])
copy_srcs.append(src)
@@ -43,6 +43,7 @@ from sglang.srt.runtime_context import (
get_parallel,
get_spec,
)
from sglang.srt.speculative.draft_checkpoint import refresh_track_indices, track_indices
from sglang.srt.speculative.eagle_info import EagleDraftExtendInput
from sglang.srt.speculative.eagle_utils import get_draft_input_from_target_hidden_dim
from sglang.srt.speculative.spec_utils import resolve_num_tokens_per_req
@@ -74,6 +75,7 @@ def resolve_draft_extend_seq_len_fill_value(
class EagleDraftExtendInputBuffers(ForwardInputBuffers):
input_ids: torch.Tensor
req_pool_indices: torch.Tensor
mamba_track_indices: Optional[torch.Tensor]
out_cache_loc: torch.Tensor
positions: torch.Tensor
mrope_positions: torch.Tensor
@@ -268,6 +270,11 @@ class EAGLEDraftExtendCudaGraphRunner(DecodeCudaGraphRunner):
self.buffers = EagleDraftExtendInputBuffers(
input_ids=input_ids,
req_pool_indices=req_pool_indices,
mamba_track_indices=(
torch.zeros(self.max_bs, dtype=torch.int64, device=self.device)
if get_exec().mamba.enable_mamba_extra_buffer
else None
),
out_cache_loc=out_cache_loc,
positions=positions,
mrope_positions=mrope_positions,
@@ -421,6 +428,7 @@ class EAGLEDraftExtendCudaGraphRunner(DecodeCudaGraphRunner):
batch_size=bs,
input_ids=input_ids,
req_pool_indices=req_pool_indices,
mamba_track_indices=track_indices(buffers.mamba_track_indices, bs),
seq_lens=seq_lens,
seq_lens_cpu=seq_lens_cpu,
next_token_logits_buffer=next_token_logits_buffer,
@@ -572,6 +580,14 @@ class EAGLEDraftExtendCudaGraphRunner(DecodeCudaGraphRunner):
copy_srcs.append(forward_batch.spec_info.num_accept_tokens)
copy_dsts.append(buffers.select_index[:raw_bs])
copy_srcs.append(select_index)
refresh_track_indices(
buffers.mamba_track_indices,
forward_batch.mamba_track_indices,
raw_bs=raw_bs,
bs=bs,
copy_dsts=copy_dsts,
copy_srcs=copy_srcs,
)
_grouped_foreach_copy_(copy_dsts, copy_srcs)
# hidden_states is large + contiguous: copy_() uses the cudaMemcpyAsync
@@ -633,6 +649,8 @@ class EAGLEDraftExtendCudaGraphRunner(DecodeCudaGraphRunner):
encoder_lens=None,
out_cache_loc=buffers.out_cache_loc[:num_tokens],
out_cache_loc_dsv4=getattr(forward_batch, "out_cache_loc_dsv4", None),
# Virtual input stays separate from the backend's physical buffer.
mamba_track_indices=track_indices(buffers.mamba_track_indices, bs),
spec_info=forward_batch.spec_info,
)
self.draft_extend_attn_backend.init_forward_metadata_out_graph(fb_view)
@@ -66,6 +66,7 @@ from sglang.srt.runtime_context import (
get_parallel,
get_spec,
)
from sglang.srt.speculative.draft_checkpoint import refresh_track_indices, track_indices
from sglang.srt.speculative.eagle_info import EagleDraftExtendInput
from sglang.srt.speculative.eagle_utils import get_draft_input_from_target_hidden_dim
from sglang.srt.speculative.multi_layer_eagle_utils import (
@@ -116,6 +117,7 @@ class MultiLayerEagleDraftExtendInputBuffers(ForwardInputBuffers):
seq_lens: torch.Tensor
seq_lens_cpu: torch.Tensor
req_pool_indices: torch.Tensor
mamba_track_indices: Optional[torch.Tensor]
num_correct_drafts: torch.Tensor
num_accept_tokens: torch.Tensor
extend_seq_lens: torch.Tensor
@@ -335,6 +337,7 @@ class MultiLayerEagleDraftExtendCudaGraphRunner(DecodeCudaGraphRunner):
batch_size=bs,
input_ids=input_ids,
req_pool_indices=req_pool_indices,
mamba_track_indices=track_indices(buffers.mamba_track_indices, bs),
seq_lens=seq_lens,
seq_lens_cpu=seq_lens_cpu,
next_token_logits_buffer=next_token_logits_buffer,
@@ -493,6 +496,8 @@ class MultiLayerEagleDraftExtendCudaGraphRunner(DecodeCudaGraphRunner):
encoder_lens=None,
# per-step write target; out_cache_loc is frozen at prepare() time.
out_cache_loc=buffers.out_cache_loc[:num_tokens],
# Virtual input stays separate from the backend's physical buffer.
mamba_track_indices=track_indices(buffers.mamba_track_indices, bs),
spec_info=spec_info,
)
if (
@@ -702,6 +707,11 @@ class MultiLayerEagleMultiStepDraftExtendCudaGraphRunner:
seq_lens=seq_lens,
seq_lens_cpu=seq_lens_cpu,
req_pool_indices=req_pool_indices,
mamba_track_indices=(
torch.zeros(self.max_bs, dtype=torch.int64, device=self.device)
if get_exec().mamba.enable_mamba_extra_buffer
else None
),
num_correct_drafts=num_correct_drafts,
num_accept_tokens=num_accept_tokens,
extend_seq_lens=extend_seq_lens,
@@ -755,6 +765,7 @@ class MultiLayerEagleMultiStepDraftExtendCudaGraphRunner:
seq_lens=buffers.seq_lens[:bs],
extend_seq_lens=buffers.extend_seq_lens[:bs],
out_cache_loc=buffers.out_cache_loc[: bs * self.captured_req_width],
mamba_track_indices=track_indices(buffers.mamba_track_indices, bs),
)
for backend in backends:
backend.init_forward_metadata_out_graph(batch)
@@ -773,6 +784,13 @@ class MultiLayerEagleMultiStepDraftExtendCudaGraphRunner:
else:
bs = self.get_runner(0)._pad_to_bucket(raw_bs, self.capture_bs)
refresh_track_indices(
buffers.mamba_track_indices,
forward_batch.mamba_track_indices,
raw_bs=raw_bs,
bs=bs,
)
fill_draft_extend_prepare_buffers(
buffers.input_ids,
buffers.positions,
@@ -0,0 +1,997 @@
"""Inkling checkpoint writes use live physical slots, including graph replay."""
import unittest
from contextlib import ExitStack, nullcontext
from types import SimpleNamespace
from unittest.mock import Mock, patch
import torch
from sglang.srt.layers.attention.linear.inkling_sconv_backend import (
InklingShortConvAttnBackend,
InklingShortConvHybridAttnBackend,
)
from sglang.srt.mem_cache.allocator.unified_sub_pool import MultiEndedAllocator
from sglang.srt.mem_cache.memory_pool import HybridReqToTokenPool
from sglang.srt.mem_cache.unified_memory_pool import (
MambaSubPoolSpec,
MHASubPoolSpec,
UnifiedHybridReqToTokenPool,
UnifiedKVPool,
UnifiedMambaPool,
UnifiedMambaSlotAllocator,
)
from sglang.srt.model_executor.forward_batch_info import ForwardMode
from sglang.srt.models.inkling_common.sconv import ShortConvolution
from sglang.test.ci.ci_register import register_cuda_ci
from sglang.test.test_utils import CustomTestCase
register_cuda_ci(est_time=30, stage="base-b-kernel-unit", runner_config="1-gpu-large")
_BACKEND = "sglang.srt.layers.attention.linear.inkling_sconv_backend"
@unittest.skipUnless(torch.cuda.is_available(), "requires CUDA")
class TestInklingCheckpointIndices(CustomTestCase):
def make_backend(
self, *, lazy=False, static=False, slot_count=6, draft_token_num=None
):
mamba = MambaSubPoolSpec(
name="mamba",
layer_num=2,
grow_direction="up",
conv_state_shapes=((3, 64),),
conv_dtype=torch.bfloat16,
temporal_state_shape=(0,),
temporal_dtype=torch.float32,
)
full = MHASubPoolSpec(
name="full",
layer_num=1,
head_num=1,
head_dim=8,
store_dtype=torch.bfloat16,
grow_direction="down",
)
shared = UnifiedKVPool(
total_bytes=64 * mamba.entry_bytes(),
sub_pool_specs=[mamba, full],
device="cuda",
enable_memory_saver=False,
)
shared._raw.zero_()
pool = UnifiedMambaPool(
unified_buffer=shared,
sub_pool_name="mamba",
spec_state_size=0,
mamba_layer_ids=[0, 1],
)
allocator = MultiEndedAllocator(
kvcache=pool,
unified_buffer=shared,
sub_pool_name="mamba",
device="cuda",
is_id_owner=True,
lazy_compaction=lazy,
)
slots = allocator.alloc(slot_count)
self.assertIsNotNone(slots)
req_type = HybridReqToTokenPool if static else UnifiedHybridReqToTokenPool
req_pool = req_type.__new__(req_type)
req_pool.size = 4
req_pool.mamba_pool = pool
req_pool.mamba_allocator = UnifiedMambaSlotAllocator(
allocator, max_size=pool.size, device="cuda"
)
req_pool.req_index_to_mamba_index_mapping = slots[1:2].to(torch.int32).repeat(4)
backend = InklingShortConvAttnBackend.__new__(InklingShortConvAttnBackend)
backend.req_to_token_pool = req_pool
backend.device = "cuda"
backend.conv_state_len = 3
backend._cache_indices_buf = None
backend._slot_gather_recordable = static
config = SimpleNamespace(
decode=SimpleNamespace(bs=[1, 2, 4], max_bs=4),
prefill=SimpleNamespace(bs=[8]),
)
with (
patch(
_BACKEND + ".get_exec",
return_value=SimpleNamespace(
graph=SimpleNamespace(cuda_graph_config=config)
),
),
patch(
_BACKEND + ".get_spec",
return_value=SimpleNamespace(
speculative_num_draft_tokens=draft_token_num
),
),
):
backend._alloc_graph_buffers()
return backend, allocator, pool, slots
def batch(self, ids, mode=ForwardMode.EXTEND, mask=None):
n = len(ids)
return SimpleNamespace(
forward_mode=mode,
batch_size=n,
req_pool_indices=torch.arange(n, device="cuda"),
mamba_track_indices=ids,
mamba_track_mask=torch.ones(n, dtype=torch.bool, device="cuda")
if mask is None
else mask,
)
def scatter(self, backend, batch, cache):
hidden = (
torch.arange(3 * 64, device="cuda", dtype=torch.float32)
.reshape(3, 64)
.to(torch.bfloat16)
)
rows = torch.arange(3, device="cuda").repeat(batch.batch_size, 1)
ShortConvolution._prepare_extend_sconv_cache(
SimpleNamespace(_conv_state=lambda _: backend.sconv_metadata),
batch,
cache,
hidden,
rows,
)
return hidden
def test_prefill_checkpoint_after_real_compaction(self):
for lazy in (False, True):
with self.subTest(lazy=lazy):
backend, allocator, pool, slots = self.make_backend(lazy=lazy)
virtual = slots[-1:].clone()
allocator.free(slots[:1].clone())
if lazy:
allocator._flush(urgent=True)
physical = backend._translate_mamba_indices(virtual)
self.assertFalse(torch.equal(virtual, physical.to(virtual.dtype)))
ids = torch.cat([virtual, virtual.new_tensor([-1])])
original = ids.clone()
batch = self.batch(ids, mask=torch.tensor([True, False], device="cuda"))
backend._prepare_slot_indices(batch)
cache = pool.mamba_cache.conv[0][0]
before = cache.clone()
expected = self.scatter(backend, batch, cache)
torch.testing.assert_close(cache[physical[0]], expected, rtol=0, atol=0)
before[physical[0]] = expected
torch.testing.assert_close(cache, before, rtol=0, atol=0)
torch.testing.assert_close(ids, original, rtol=0, atol=0)
self.assertEqual(
backend.sconv_metadata.track_cache_indices.dtype, torch.int64
)
def test_replay_refreshes_captured_destination_after_compaction(self):
backend, allocator, pool, slots = self.make_backend(lazy=True)
virtual = slots[-1:].clone()
batch = self.batch(virtual, mode=ForwardMode.DECODE)
backend.init_forward_metadata_out_graph(batch)
pointer = backend.sconv_metadata.track_cache_indices.data_ptr()
cache = pool.mamba_cache.conv[0][0]
self.scatter(backend, batch, cache) # compile before capture
graph = torch.cuda.CUDAGraph()
with torch.cuda.graph(graph):
self.scatter(backend, batch, cache)
allocator.free(slots[:1].clone())
allocator._flush(urgent=True)
physical = backend._translate_mamba_indices(virtual)
cache.zero_()
fresh = self.batch(virtual, mode=ForwardMode.DECODE)
backend.init_forward_metadata_out_graph(fresh)
self.assertEqual(backend.sconv_metadata.track_cache_indices.data_ptr(), pointer)
graph.replay()
expected = (
torch.arange(3 * 64, device="cuda", dtype=torch.float32)
.reshape(3, 64)
.to(torch.bfloat16)
)
torch.testing.assert_close(cache[physical[0]], expected, rtol=0, atol=0)
self.assertEqual(
torch.count_nonzero(cache).item(), torch.count_nonzero(expected).item()
)
torch.testing.assert_close(virtual, slots[-1:], rtol=0, atol=0)
def test_decode_tracks_the_physical_checkpoint(self):
backend, allocator, pool, slots = self.make_backend()
allocator.free(slots[:1].clone())
batch = self.batch(slots[-1:].clone(), mode=ForwardMode.DECODE)
backend._prepare_slot_indices(batch)
cache = pool.mamba_cache.conv[0][0]
conv = SimpleNamespace(
activation=None,
_conv_state=lambda _: backend.sconv_metadata,
use_residual=True,
_weight_2d=lambda: torch.ones(64, 4, dtype=torch.bfloat16, device="cuda"),
)
ShortConvolution._apply_decode_sconv_kernel(
conv,
torch.ones(1, 64, dtype=torch.bfloat16, device="cuda"),
cache,
backend._cache_indices,
{"cache_mask": torch.ones(1, 1, 1, dtype=torch.bool, device="cuda")},
batch,
)
physical = backend._translate_mamba_indices(slots[-1:])
torch.testing.assert_close(
cache[physical[0]], cache[backend._cache_indices[0]], rtol=0, atol=0
)
self.assertGreater(torch.count_nonzero(cache[physical[0]]).item(), 0)
def test_static_identity_path_and_disabled_tracking(self):
backend, _, _, slots = self.make_backend(static=True)
ids = slots[-1:].clone()
batch = self.batch(ids)
backend._prepare_slot_indices(batch)
self.assertIs(batch.mamba_track_indices, ids)
batch.mamba_track_indices = None
backend._prepare_slot_indices(batch)
self.assertIsNone(batch.mamba_track_indices)
backend._slot_gather_recordable = False
backend._prepare_slot_indices(batch)
self.assertIsNone(batch.mamba_track_indices)
def test_graph_hooks_translate_once_for_each_mode(self):
backend, allocator, _, slots = self.make_backend()
allocator.free(slots[:1].clone())
for mode in (
ForwardMode.DECODE,
ForwardMode.TARGET_VERIFY,
ForwardMode.DRAFT_EXTEND_V2,
):
with self.subTest(mode=mode):
ids = slots[-1:].clone()
batch = self.batch(ids, mode=mode)
with (
patch.object(backend, "_refresh_sconv_metadata"),
patch.object(
backend,
"_translate_mamba_indices",
wraps=backend._translate_mamba_indices,
) as translate,
):
backend.init_forward_metadata_out_graph(batch)
calls = translate.call_count
self.assertEqual(calls, 2) # active and checkpoint slots
backend.init_forward_metadata_in_graph(batch)
self.assertEqual(translate.call_count, calls)
torch.testing.assert_close(
backend.sconv_metadata.track_cache_indices,
backend._translate_mamba_indices(ids).to(torch.int64),
)
torch.testing.assert_close(ids, slots[-1:])
def test_distinct_strided_destinations_keep_static_storage(self):
backend, allocator, pool, slots = self.make_backend()
allocator.free(slots[:1].clone())
selected = slots[2:].flip(0)
storage = torch.stack(
[selected, slots[1:2].expand_as(selected)], dim=1
).flatten()
original = storage.clone()
ids = storage[::2]
self.assertFalse(ids.is_contiguous())
batch = self.batch(ids)
backend._prepare_slot_indices(batch)
pointer = backend.sconv_metadata.track_cache_indices.data_ptr()
physical = backend._translate_mamba_indices(selected).to(torch.int64)
torch.testing.assert_close(
backend.sconv_metadata.track_cache_indices, physical, rtol=0, atol=0
)
cache = pool.mamba_cache.conv[0][0]
expected = cache.clone()
hidden = torch.arange(12 * 64, device="cuda").reshape(12, 64).to(torch.bfloat16)
rows = torch.arange(12, device="cuda").reshape(4, 3)
ShortConvolution._prepare_extend_sconv_cache(
SimpleNamespace(_conv_state=lambda _: backend.sconv_metadata),
batch,
cache,
hidden,
rows,
)
expected[physical] = hidden.reshape(4, 3, 64)
torch.testing.assert_close(cache, expected, rtol=0, atol=0)
smaller = self.batch(ids[:2])
backend._prepare_slot_indices(smaller)
self.assertEqual(backend.sconv_metadata.track_cache_indices.data_ptr(), pointer)
torch.testing.assert_close(storage, original, rtol=0, atol=0)
def test_decode_graph_refreshes_distinct_destinations_and_masks_padding(self):
backend, allocator, pool, slots = self.make_backend(lazy=True, slot_count=12)
req_pool = backend.req_to_token_pool
req_pool.req_index_to_mamba_index_mapping = torch.cat(
[slots.new_zeros(1), slots[1:5]]
).to(torch.int32)
ids = slots[-4:].clone()
track_mask = torch.ones(4, dtype=torch.bool, device="cuda")
batch = self.batch(ids, mode=ForwardMode.DECODE, mask=track_mask)
batch.req_pool_indices = torch.arange(1, 5, device="cuda")
backend.init_forward_metadata_out_graph(batch)
pointer = backend.sconv_metadata.track_cache_indices.data_ptr()
cache = pool.mamba_cache.conv[0][0]
hidden = torch.zeros(4, 64, dtype=torch.bfloat16, device="cuda")
weight = torch.ones(64, 4, dtype=torch.bfloat16, device="cuda")
conv = SimpleNamespace(
activation=None,
use_residual=True,
_weight_2d=lambda: weight,
_conv_state=lambda _: backend.sconv_metadata,
)
def decode():
backend.init_forward_metadata_in_graph(batch)
ShortConvolution._apply_decode_sconv_kernel(
conv,
hidden,
cache,
backend._cache_indices,
backend.sconv_metadata.precomputed,
batch,
)
decode()
graph = torch.cuda.CUDAGraph()
with torch.cuda.graph(graph):
decode()
for step, live in enumerate((3, 1, 4)):
with self.subTest(live=live):
if step < 2:
hole = slots[:1] if step == 0 else slots[5:6]
allocator.free(hole.clone())
allocator._flush(urgent=True)
ids.copy_(slots[-4:].roll(step + 1))
ids[live:].zero_()
original = ids.clone()
track_mask.zero_()
track_mask[:live] = True
if live > 1:
track_mask[1] = False
fresh = self.batch(ids, mode=ForwardMode.DECODE, mask=track_mask)
fresh.req_pool_indices = torch.arange(1, 5, device="cuda")
fresh.req_pool_indices[live:] = 0
backend.init_forward_metadata_out_graph(fresh)
self.assertEqual(
backend.sconv_metadata.track_cache_indices.data_ptr(), pointer
)
physical = backend._translate_mamba_indices(ids).to(torch.int64)
active = backend._cache_indices.clone().to(torch.int64)
cache.fill_(-7)
cache[0].zero_()
hidden.copy_(torch.arange(4, device="cuda")[:, None] + step + 1)
hidden[live:].zero_()
expected = cache.clone()
for row in range(live):
window = torch.cat([cache[active[row], 1:], hidden[row : row + 1]])
expected[active[row]] = window
if row != 1:
expected[physical[row]] = window
graph.replay()
torch.testing.assert_close(cache, expected, rtol=0, atol=0)
torch.testing.assert_close(ids, original, rtol=0, atol=0)
def test_oversized_checkpoint_batch_fails_without_reallocating(self):
backend, _, _, slots = self.make_backend()
pointer = backend._graph_track_indices.data_ptr()
batch = self.batch(slots[-1:].repeat(5))
with self.assertRaisesRegex(
AssertionError, "checkpoint-index buffer too small"
):
backend._prepare_slot_indices(batch)
self.assertEqual(backend._graph_track_indices.data_ptr(), pointer)
def test_verify_commit_translates_both_destinations(self):
backend, allocator, _, slots = self.make_backend()
allocator.free(slots[:1].clone())
indices = torch.tensor([0], device="cuda")
for tracking in (False, True):
with self.subTest(tracking=tracking):
ids = slots[-1:].clone() if tracking else None
with (
patch.object(
backend.req_to_token_pool,
"get_speculative_mamba2_params_all_layers",
return_value=object(),
),
patch(
_BACKEND + ".scatter_mamba_states_after_mtp_verify"
) as scatter,
):
backend.commit_conv_state_after_mtp_verify(
req_pool_indices=indices,
last_correct_step_indices=indices,
mamba_track_indices=ids,
mamba_steps_to_track=indices if tracking else None,
)
if tracking:
torch.testing.assert_close(
scatter.call_args.args[3], backend._translate_mamba_indices(ids)
)
torch.testing.assert_close(ids, slots[-1:], rtol=0, atol=0)
else:
self.assertIsNone(scatter.call_args.args[3])
self.assertIsNone(scatter.call_args.args[4])
torch.testing.assert_close(
scatter.call_args.args[1],
backend._translate_mamba_indices(slots[1:2].to(torch.int32)),
)
def test_explicit_none_tracking_still_refreshes_active_slots(self):
backend, allocator, _, slots = self.make_backend()
allocator.free(slots[:1].clone())
batch = SimpleNamespace(
forward_mode=ForwardMode.DRAFT_EXTEND_V2,
req_pool_indices=torch.tensor([0], device="cuda"),
mamba_track_indices=None,
)
backend.init_forward_metadata_out_graph(batch)
self.assertIsNone(backend.sconv_metadata.track_cache_indices)
torch.testing.assert_close(
backend._cache_indices,
backend._translate_mamba_indices(slots[1:2]).to(torch.int32),
)
def test_missing_tracking_field_rejects_incomplete_metadata_view(self):
for static in (False, True):
with self.subTest(static=static):
backend, _, _, slots = self.make_backend(static=static)
batch = self.batch(slots[-1:].clone())
backend._prepare_slot_indices(batch)
del batch.mamba_track_indices
with self.assertRaisesRegex(AttributeError, "mamba_track_indices"):
backend._prepare_slot_indices(batch)
def test_multilayer_staging_provides_checkpoint_ids_or_none(self):
from sglang.srt.speculative.multi_layer_eagle_draft_extend_cuda_graph_runner import (
MultiLayerEagleMultiStepDraftExtendCudaGraphRunner,
)
_, backend, _, pool, slots, buffers = self._make_draft_checkpoint_runner(
multilayer=True, static=True, tracking=True
)
wrapper = InklingShortConvHybridAttnBackend.__new__(
InklingShortConvHybridAttnBackend
)
wrapper.short_conv_backend = backend
wrapper.full_attn_backend = Mock(supports_draft_extend_metadata_staging=True)
composite = MultiLayerEagleMultiStepDraftExtendCudaGraphRunner.__new__(
MultiLayerEagleMultiStepDraftExtendCudaGraphRunner
)
composite.buffers = buffers
composite.captured_req_width = 4
composite.draft_extend_attn_backend_list = [wrapper]
for step, tracking in enumerate((True, False, True)):
with self.subTest(tracking=tracking, step=step):
ids = slots[-4:].roll(step).clone()
original = ids.clone()
buffers.mamba_track_indices = ids if tracking else None
composite._stage_metadata(bs=3, raw_bs=2)
torch.testing.assert_close(
backend._cache_indices,
torch.cat([slots[1:3], slots.new_zeros(1)]).to(torch.int32),
)
if tracking:
torch.testing.assert_close(
backend.sconv_metadata.track_cache_indices, ids[:3]
)
self.assertEqual(
backend.sconv_metadata.track_cache_indices.data_ptr(),
ids.data_ptr(),
)
batch = self.batch(
ids[:3], mask=torch.tensor([True, True, False], device="cuda")
)
cache = pool.mamba_cache.conv[0][0]
expected = cache.clone()
hidden = self.scatter(backend, batch, cache)
expected[ids[:2].long()] = hidden
torch.testing.assert_close(cache, expected, rtol=0, atol=0)
torch.testing.assert_close(ids, original, rtol=0, atol=0)
else:
self.assertIsNone(backend.sconv_metadata.track_cache_indices)
def test_draft_eager_preplan_and_repeated_steps_preserve_virtual_source(self):
from sglang.srt.speculative import eagle_worker_common as common
from sglang.srt.speculative.eagle_info import EagleDraftExtendInput
for static in (False, True):
for graph_available in (False, True):
with self.subTest(static=static, graph_available=graph_available):
steps = []
for step in range(2):
backend, allocator, pool, slots = self.make_backend(
static=static, slot_count=12, draft_token_num=4
)
backend.mamba_cache_chunk_size = 4
backend.req_to_token_pool.req_index_to_mamba_index_mapping.fill_(
slots[3]
)
if not static:
allocator.free(slots[step : step + 1].clone())
wrapper = InklingShortConvHybridAttnBackend.__new__(
InklingShortConvHybridAttnBackend
)
wrapper.full_attn_backend = Mock()
wrapper.short_conv_backend = backend
wrapper.attn_backend_list = [wrapper.full_attn_backend, backend]
steps.append((wrapper, backend, pool))
ids = slots[-1:].clone()
original = ids.clone()
batch = self.batch(ids, mode=ForwardMode.DRAFT_EXTEND_V2)
batch.seq_lens = torch.tensor([2], device="cuda")
batch.extend_seq_lens = torch.tensor([4], device="cuda")
batch.extend_prefix_lens = None
batch.extend_num_tokens = 4
batch.spec_info = EagleDraftExtendInput(
num_accept_tokens=torch.tensor([3], device="cuda")
)
batch.mark_forward_metadata_ready = Mock()
schedule_batch = SimpleNamespace(
seq_lens=batch.seq_lens,
seq_lens_cpu=None,
forward_mode=batch.forward_mode,
model_config=SimpleNamespace(vocab_size=32),
)
runner = SimpleNamespace(
spec_algorithm=SimpleNamespace(is_standalone=lambda: False),
attn_backend=steps[0][0],
model_config=SimpleNamespace(model_is_mrope=False),
)
graph_runner = (
SimpleNamespace(can_run_graph=lambda _: False)
if graph_available
else None
)
with (
patch.object(
common.ForwardBatch, "init_new", return_value=batch
),
patch.object(common, "maybe_detect_oob"),
patch.object(common, "is_npu", return_value=False),
):
planned = common.prepare_for_draft_extend(
batch.spec_info,
schedule_batch,
torch.zeros(4, dtype=torch.int64, device="cuda"),
4,
runner,
graph_runner,
return_hidden_states_before_norm=False,
)
self.assertIs(planned, batch)
destinations = []
# Inspect the real pre-plan, then repeat step 0 and enter step 1.
for index in (0, 0, 1):
wrapper, backend, pool = steps[index]
if destinations:
wrapper.init_forward_metadata(batch)
self.assertIs(batch.mamba_track_indices, ids)
torch.testing.assert_close(ids, original, rtol=0, atol=0)
physical = backend._translate_mamba_indices(ids).long()
torch.testing.assert_close(
backend.sconv_metadata.track_cache_indices,
physical,
rtol=0,
atol=0,
)
destinations.append(physical.clone())
cache = pool.mamba_cache.conv[0][0]
cache.fill_(-7)
hidden = torch.arange(4, device="cuda")[:, None].expand(4, 64)
hidden = hidden.to(torch.bfloat16)
active = backend._cache_indices.long()
joined = torch.cat([cache[active[0]], hidden])
expected = cache.clone()
expected[physical[0]] = joined[1:4]
expected[active[0]] = joined[3:6]
with patch(
"sglang.srt.models.inkling_common.sconv.get_exec",
return_value=SimpleNamespace(
mamba=SimpleNamespace(
enable_mamba_extra_buffer=True,
mamba_track_interval=4,
)
),
):
ShortConvolution._update_sconv_cache_for_draft_extend(
SimpleNamespace(
_conv_state=lambda _: backend.sconv_metadata
),
batch,
cache,
backend._cache_indices,
hidden,
)
torch.testing.assert_close(cache, expected, rtol=0, atol=0)
if not static:
self.assertFalse(torch.equal(destinations[0], destinations[-1]))
def test_static_draft_staging_keeps_checkpoint_metadata(self):
from sglang.srt.speculative.eagle_info import EagleDraftExtendInput
backend, _, _, slots = self.make_backend(static=True, draft_token_num=4)
wrapper = InklingShortConvHybridAttnBackend.__new__(
InklingShortConvHybridAttnBackend
)
wrapper.short_conv_backend = backend
wrapper.full_attn_backend = Mock(supports_draft_extend_metadata_staging=True)
batch = self.batch(slots[-1:].clone(), mode=ForwardMode.DRAFT_EXTEND_V2)
batch.seq_lens = torch.tensor([6], device="cuda")
batch.extend_seq_lens = torch.tensor([4], device="cuda")
batch.extend_prefix_lens = None
batch.extend_num_tokens = 4
batch.spec_info = EagleDraftExtendInput(
num_accept_tokens=torch.tensor([3], device="cuda")
)
wrapper.init_forward_metadata_out_graph(batch, in_capture=True)
wrapper.init_forward_metadata_in_graph(batch)
self.assertIs(
backend.sconv_metadata.track_cache_indices, batch.mamba_track_indices
)
def test_prefill_capture_reinitializes_inert_checkpoint_metadata(self):
backend, _, pool, _ = self.make_backend()
backend.mamba_cache_chunk_size = 4
batch = self.batch(torch.zeros(1, dtype=torch.int64, device="cuda"))
batch.mamba_track_indices = None
batch.mamba_track_mask = None
batch.seq_lens = torch.tensor([4], device="cuda")
batch.extend_seq_lens = torch.tensor([4], device="cuda")
batch.extend_prefix_lens = None
batch.extend_num_tokens = 4
batch.spec_info = None
cache = pool.mamba_cache.conv[0][0]
expected = cache.clone()
for _ in range(2):
backend.init_forward_metadata_out_graph(batch, in_capture=True)
torch.testing.assert_close(
backend.sconv_metadata.track_cache_indices,
torch.zeros(1, dtype=torch.int64, device="cuda"),
)
self.scatter(backend, batch, cache)
torch.testing.assert_close(cache, expected, rtol=0, atol=0)
def test_draft_runners_capture_and_replay_checkpoint_destinations(self):
for multilayer in (False, True):
for static in (False, True):
for tracking in (False, True):
with self.subTest(
multilayer=multilayer, static=static, tracking=tracking
):
self._check_draft_checkpoint_replay(
multilayer, static, tracking
)
def _allocate_production_draft_buffers(self, multilayer, tracking):
from sglang.srt.speculative import eagle_draft_extend_cuda_graph_runner as eagle
from sglang.srt.speculative import (
multi_layer_eagle_draft_extend_cuda_graph_runner as multi,
)
model_runner = SimpleNamespace(
device="cuda",
tp_size=1,
model_config=SimpleNamespace(
dtype=torch.bfloat16, vocab_size=8, hf_config=SimpleNamespace()
),
graph_shared_output=SimpleNamespace(
get_logits_buffer=lambda vocab, rows: torch.zeros(
rows, vocab, device="cuda"
)
),
)
worker = SimpleNamespace(
draft_runner=model_runner,
speculative_algorithm=SimpleNamespace(is_standalone=lambda: False),
seed_dsa_topk_from_draft_extend=False,
mtp_model_runner=lambda step: model_runner,
)
runtime = SimpleNamespace(
mamba=SimpleNamespace(enable_mamba_extra_buffer=tracking),
graph=SimpleNamespace(
disable_cuda_graph_padding=False, enable_profile_cuda_graph=False
),
)
if multilayer:
composite = (
multi.MultiLayerEagleMultiStepDraftExtendCudaGraphRunner.__new__(
multi.MultiLayerEagleMultiStepDraftExtendCudaGraphRunner
)
)
composite.runners = [SimpleNamespace(model_runner=model_runner)]
composite.eagle_worker = worker
composite.max_bs = 4
composite.captured_req_width = 4
composite.seq_len_fill_value = 4
composite.device = "cuda"
composite.prune_draft_extend_logits = False
composite.rotates_in_graph = False
composite.require_gathered_buffer = False
with (
patch.object(multi, "get_exec", return_value=runtime),
patch.object(
multi, "get_draft_input_from_target_hidden_dim", return_value=64
),
):
return composite._allocate_buffers()
worker.draft_extend_attn_backend = SimpleNamespace(
init_cuda_graph_state=lambda *args: None,
get_cuda_graph_seq_len_fill_value=lambda: 4,
)
# Only model/distributed capture scaffolding is replaced. __init__ and
# its real input-buffer allocation run before the kernel replay test.
with ExitStack() as stack:
replacements = {
"get_exec": runtime,
"get_parallel": SimpleNamespace(pp_size=1),
"deployment_attn_dp_size": 1,
"get_flags": SimpleNamespace(
capture=SimpleNamespace(enable_torch_compile=False)
),
"get_spec": SimpleNamespace(speculative_num_steps=1),
"get_batch_sizes_to_capture": ([4], []),
"resolve_num_tokens_per_req": 4,
"get_draft_input_from_target_hidden_dim": 64,
"require_gathered_buffer": False,
"require_mlp_tp_gather": False,
"require_mlp_sync": False,
"require_attn_tp_gather": False,
"resolve_decode_backend": Mock(),
}
for name, result in replacements.items():
stack.enter_context(patch.object(eagle, name, return_value=result))
stack.enter_context(patch.object(eagle, "model_capture_mode", nullcontext))
stack.enter_context(patch.object(eagle, "DeepEPCudaGraphRunnerAdapter"))
stack.enter_context(
patch.object(eagle.EagleDraftExtendInputBuffers, "share_buffers")
)
stack.enter_context(
patch.object(eagle.EAGLEDraftExtendCudaGraphRunner, "capture")
)
return eagle.EAGLEDraftExtendCudaGraphRunner(worker).buffers
def _make_draft_checkpoint_runner(self, multilayer, static, tracking):
from sglang.srt.speculative import eagle_draft_extend_cuda_graph_runner as eagle
from sglang.srt.speculative import (
multi_layer_eagle_draft_extend_cuda_graph_runner as multi,
)
backend, allocator, pool, slots = self.make_backend(
lazy=True, static=static, slot_count=12, draft_token_num=4
)
backend.mamba_cache_chunk_size = 4
req_pool = backend.req_to_token_pool
req_pool.req_index_to_mamba_index_mapping = torch.cat(
[slots.new_zeros(1), slots[1:5]]
).to(torch.int32)
buffers = self._allocate_production_draft_buffers(multilayer, tracking)
buffers.req_pool_indices.copy_(torch.arange(1, 5, device="cuda"))
buffers.seq_lens.fill_(6)
buffers.seq_lens_cpu.fill_(6)
buffers.num_accept_tokens.fill_(3)
if tracking:
self.assertIsNotNone(buffers.mamba_track_indices)
self.assertEqual(buffers.mamba_track_indices.shape, (4,))
self.assertEqual(buffers.mamba_track_indices.dtype, torch.int64)
self.assertEqual(buffers.mamba_track_indices.device.type, "cuda")
torch.testing.assert_close(
buffers.mamba_track_indices,
torch.zeros(4, dtype=torch.int64, device="cuda"),
)
buffers.mamba_track_indices.copy_(slots[-4:])
else:
self.assertIsNone(buffers.mamba_track_indices)
cls = (
multi.MultiLayerEagleDraftExtendCudaGraphRunner
if multilayer
else eagle.EAGLEDraftExtendCudaGraphRunner
)
runner = cls.__new__(cls)
runner.buffers = buffers
runner.captured_req_width = 4
runner.forward_mode = ForwardMode.DRAFT_EXTEND_V2
runner.extend_seq_lens_cpu = [4] * 4
runner.require_mlp_tp_gather = False
runner.require_attn_tp_gather = False
runner.require_gathered_buffer = False
runner.capture_bs = [4]
runner.seq_len_fill_value = 4
runner.num_front_tokens = 0
runner.prune_draft_extend_logits = False
runner.metadata_captured_in_graph = False
runner.step = 0
runner.deepep_adapter = Mock()
runner.device_module = torch.cuda
runner.attn_backend = SimpleNamespace(
supports_draft_extend_metadata_staging=False
)
runner.draft_extend_attn_backend = backend
runner.eagle_worker = SimpleNamespace(draft_extend_attn_backend_list=[backend])
runner.model_runner = SimpleNamespace(
model_config=SimpleNamespace(model_is_mrope=False),
spec_algorithm=SimpleNamespace(is_standalone=lambda: False),
device_timer=None,
canary_manager=None,
)
return runner, backend, allocator, pool, slots, buffers
def test_static_staged_runner_checkpoint_replay(self):
for tracking in (False, True):
with self.subTest(tracking=tracking):
self._check_draft_checkpoint_replay(True, True, tracking, staging=True)
def _check_draft_checkpoint_replay(
self, multilayer, static, tracking, staging=False
):
from sglang.srt.speculative import eagle_draft_extend_cuda_graph_runner as eagle
from sglang.srt.speculative import (
multi_layer_eagle_draft_extend_cuda_graph_runner as multi,
)
from sglang.srt.speculative.eagle_info import EagleDraftExtendInput
runner, backend, allocator, pool, slots, buffers = (
self._make_draft_checkpoint_runner(multilayer, static, tracking)
)
hook_backend = backend
if staging:
wrapper = InklingShortConvHybridAttnBackend.__new__(
InklingShortConvHybridAttnBackend
)
wrapper.short_conv_backend = backend
wrapper.full_attn_backend = Mock(
supports_draft_extend_metadata_staging=True
)
wrapper.full_attn_backend.draft_extend_metadata_captured_in_graph.return_value = False
hook_backend = wrapper
runner.attn_backend = wrapper
runner.draft_extend_attn_backend = wrapper
runner.eagle_worker.draft_extend_attn_backend_list = [wrapper]
cache = pool.mamba_cache.conv[0][0]
captured = []
graph = torch.cuda.CUDAGraph()
output = SimpleNamespace(
next_token_logits=buffers.next_token_logits_buffer,
hidden_states=buffers.hidden_states,
)
def model_forward(input_ids, positions, batch):
if not captured:
captured.append(batch)
ShortConvolution._update_sconv_cache_for_draft_extend(
SimpleNamespace(_conv_state=lambda _: backend.sconv_metadata),
batch,
cache,
backend._cache_indices,
buffers.hidden_states,
)
return output
def capture_graph(key, run_once, **kwargs):
run_once()
with torch.cuda.graph(graph):
run_once()
runner.model_runner.model = SimpleNamespace(forward=model_forward)
runner.backend = SimpleNamespace(capture_one=capture_graph)
runner._replay_graph = lambda *args: (graph.replay(), output)[1]
sconv_module = "sglang.srt.models.inkling_common.sconv"
with (
patch(
sconv_module + ".get_exec",
return_value=SimpleNamespace(
mamba=SimpleNamespace(
mamba_track_interval=4, enable_mamba_extra_buffer=True
)
),
),
patch.object(eagle, "maybe_flashinfer_autotune_speculative_draft"),
patch.object(eagle, "set_dp_buffer_len"),
patch.object(eagle, "set_is_extend_in_batch"),
):
if multilayer:
batch = runner.get_forward_batch(4)
hook_backend.init_forward_metadata_out_graph(batch, in_capture=True)
def run_once():
hook_backend.init_forward_metadata_in_graph(batch)
return model_forward(batch.input_ids, batch.positions, batch)
capture_graph(None, run_once)
else:
runner.capture_one_shape(4, None)
pointer = (
backend.sconv_metadata.track_cache_indices.data_ptr()
if tracking
else None
)
self.assertIsNone(backend.sconv_metadata.track_conv_indices)
for live in (4, 2, 1, 3, 4):
if live == 2 and not static:
allocator.free(slots[:1].clone())
allocator._flush(urgent=True)
ids = slots[-live:].flip(0).clone()
spec = EagleDraftExtendInput(
hidden_states=buffers.hidden_states[: live * 4],
num_correct_drafts=torch.ones(
live, dtype=torch.int32, device="cuda"
),
num_accept_tokens=torch.full(
(live,), 3, dtype=torch.int32, device="cuda"
),
)
fresh = SimpleNamespace(
batch_size=live,
input_ids=buffers.input_ids[: live * 4],
positions=buffers.positions[: live * 4],
out_cache_loc=buffers.out_cache_loc[: live * 4],
req_pool_indices=torch.arange(1, live + 1, device="cuda"),
seq_lens=torch.full((live,), 6, dtype=torch.int64, device="cuda"),
seq_lens_sum=6 * live,
seq_lens_cpu=None,
extend_seq_lens=torch.full(
(live,), 4, dtype=torch.int32, device="cuda"
),
extend_seq_lens_cpu=None,
mamba_track_indices=ids if tracking and live != 3 else None,
spec_info=spec,
)
cache.fill_(-7)
cache[0].zero_()
buffers.hidden_states.copy_(
torch.arange(16, device="cuda")[:, None] + 1
)
original = ids.clone()
active = backend._translate_mamba_indices(slots[1 : live + 1]).long()
physical = backend._translate_mamba_indices(ids).long()
expected = cache.clone()
for row in range(live):
joined = torch.cat(
[
cache[active[row]],
buffers.hidden_states[row * 4 : (row + 1) * 4],
]
)
if tracking and live != 3:
expected[physical[row]] = joined[1:4]
expected[active[row]] = joined[3:6]
if multilayer:
composite = multi.MultiLayerEagleMultiStepDraftExtendCudaGraphRunner.__new__(
multi.MultiLayerEagleMultiStepDraftExtendCudaGraphRunner
)
composite.buffers = buffers
composite.captured_req_width = 4
composite.require_mlp_tp_gather = False
composite.require_gathered_buffer = False
composite.capture_bs = [4]
composite.num_front_tokens = 0
composite.seq_len_fill_value = 4
composite.runners = [runner]
composite.draft_extend_attn_backend_list = [hook_backend]
if not staging:
composite._stage_metadata = Mock()
composite.prepare(fresh)
runner.replay(
4, composite.seq_lens_sum, composite._replay_spec_info, None
)
else:
runner.execute(fresh, torch.arange(live, device="cuda") * 4 + 1)
torch.testing.assert_close(cache[1:], expected[1:], rtol=0, atol=0)
torch.testing.assert_close(ids, original, rtol=0, atol=0)
if tracking:
expected_ids = torch.zeros_like(buffers.mamba_track_indices)
if live != 3:
expected_ids[:live] = original
torch.testing.assert_close(
buffers.mamba_track_indices, expected_ids, rtol=0, atol=0
)
self.assertEqual(
backend.sconv_metadata.track_cache_indices.data_ptr(), pointer
)
else:
self.assertIsNone(captured[0].mamba_track_indices)
if __name__ == "__main__":
unittest.main()