[unified-memory] Speculative decoding on the unified memory pool: translate every read and write path (#37627)

Co-authored-by: Caihua Li <caihua.li@bytedance.com>
Co-authored-by: Cheng Wan <cheng.wan@radixark.ai>
This commit is contained in:
caihuali95
2026-10-08 00:52:21 -07:00
committed by GitHub
co-authored by Caihua Li Cheng Wan
parent 33883a5fa3
commit 08d30f2300
17 changed files with 895 additions and 167 deletions
@@ -71,6 +71,8 @@ def build_kv_read_indices_kernel(
req_stride, # runtime: req_to_token row stride (elements)
out_stride, # runtime: uniform row stride, used when row_starts is null
item_stride, # runtime: items one program advances per loop trip
seq_len_delta, # runtime: verify widening added to every row's live prefix
max_row_items, # runtime: uniform row width; a longer window stops there
PAGE_SIZE: tl.constexpr,
EMIT_PER_TOKEN: tl.constexpr,
OUT_INT64: tl.constexpr,
@@ -78,7 +80,7 @@ def build_kv_read_indices_kernel(
):
bid = tl.program_id(0)
req = tl.load(req_pool_indices_ptr + bid).to(tl.int64)
seqlen = tl.load(seq_lens_ptr + bid)
seqlen = tl.load(seq_lens_ptr + bid) + seq_len_delta
# Derived here, not on the host: one elementwise op there costs a whole
# launch, which a captured graph then replays every step.
if EMIT_PER_TOKEN:
@@ -93,6 +95,9 @@ def build_kv_read_indices_kernel(
row_out = out_ptr + tl.load(row_starts_ptr + bid).to(tl.int64)
else:
row_out = out_ptr + bid.to(tl.int64) * out_stride
# A widened window can outrun a fixed-width row (a verify near the
# context limit); writing past it would land in the next row.
n_items = tl.minimum(n_items, max_row_items)
for start in range(tl.program_id(1) * BLOCK, n_items, item_stride):
item = start + tl.arange(0, BLOCK)
@@ -132,6 +137,7 @@ def _launch(
row_starts: Optional[torch.Tensor],
kv_start_idx: Optional[torch.Tensor],
emit_per_token: bool,
seq_len_delta: int = 0,
) -> None:
bs = int(req_pool_indices.numel())
item_programs = min(
@@ -148,6 +154,8 @@ def _launch(
req_to_token.stride(0),
out_stride,
item_programs * _BLOCK_ITEMS,
seq_len_delta,
max_items,
PAGE_SIZE=page_size,
EMIT_PER_TOKEN=emit_per_token,
OUT_INT64=out.dtype == torch.int64,
@@ -179,12 +187,18 @@ def build_kv_read_table(
page_size: int,
max_pages: int,
out: torch.Tensor,
seq_len_delta: int = 0,
) -> torch.Tensor:
"""Fill ``out``'s live prefix with PAGE TABLE entries.
``out`` is caller-owned (fresh zeros for the eager path, the module's
capture-stable buffer for replay) and only its ``[:bs, :max_pages]``
region's live prefix is written -- never rebound, never tail-cleared.
``seq_len_delta`` widens every row's live prefix -- the whole-sequence
verify contract (draft KV read back from the pool). Every row stops at
``max_pages``: a widened window that outruns the row is cut there rather
than spilling into the next row, on CUDA as on CPU.
"""
bs = int(req_pool_indices.numel())
assert out.dtype == torch.int32, (
@@ -205,7 +219,7 @@ def build_kv_read_table(
if not req_to_token.is_cuda:
cols = torch.arange(max_pages, device=req_to_token.device)
for b in range(bs):
n_pages = (int(seq_lens[b]) + page_size - 1) // page_size
n_pages = (int(seq_lens[b]) + seq_len_delta + page_size - 1) // page_size
live = min(n_pages, max_pages)
out[b, :live] = _entries(
req_to_token=req_to_token,
@@ -228,6 +242,7 @@ def build_kv_read_table(
row_starts=None,
kv_start_idx=None,
emit_per_token=False,
seq_len_delta=seq_len_delta,
)
return out
+46 -18
View File
@@ -474,6 +474,44 @@ def handle_cache_compatibility(server_args: Any) -> None:
)
# Backends whose speculative verify id rails are translation-audited for
# the unified pool.
_SPEC_VERIFY_AUDITED_BACKENDS = frozenset(
{
"triton",
"trtllm_mla",
"cutedsl_mla",
"tokenspeed_mla",
"flashmla",
"flashinfer",
"fa3",
}
)
def _assert_spec_verify_backends(server_args: Any, *, algorithm: str) -> None:
"""Refuse target backends whose verify id rails are not translation-audited.
Only the target's prefill/decode pair is checked. A draft with a KV pool
of its own indexes that pool directly by virtual id, so its translator is
a passthrough and its backend has nothing to translate."""
allowed = _SPEC_VERIFY_AUDITED_BACKENDS
if resolving_view(server_args).dcp_size > 1:
# flashinfer's spec verify gathers its CSR args with no DCP read
# translation (`translate_dcp_read_ids`); the MLA verify family builds
# its DCP block table itself.
allowed = allowed - {"flashinfer"}
backends = set(attention_backends_of(resolved_view(server_args)))
backends.discard(None)
assert backends <= allowed, (
f"--enable-unified-memory + {algorithm} requires spec-verify-audited "
f"attention backends {sorted(allowed)} for both prefill "
f"and decode; got {sorted(backends)}. Other backends do "
"not translate speculative verify indices to the unified "
"pool's physical ids yet."
)
def handle_unified_memory_pool(server_args: Any) -> None:
cfg = resolving_view(server_args)
@@ -540,25 +578,15 @@ def handle_unified_memory_pool(server_args: Any) -> None:
"audited for the unified pool's virtual-to-physical loc translation. Got "
f"--speculative-algorithm={cfg.speculative_algorithm!r}."
)
assert cfg.speculative_eagle_topk in (None, 1), (
"--enable-unified-memory supports a linear draft chain only "
"(--speculative-eagle-topk in {None, 1}); tree verify relocates "
"accepted tokens one at a time inside the target pool, which the "
"unified pool's page-granular move_kv_cache cannot express. Got "
f"--speculative-eagle-topk={cfg.speculative_eagle_topk!r}."
)
if cfg.speculative_algorithm == "DSPARK":
assert cfg.speculative_eagle_topk in (None, 1), (
"--enable-unified-memory + DSPARK supports a linear draft "
"chain only (--speculative-eagle-topk in {None, 1}); tree "
"verify is not audited for the unified pool. Got "
f"--speculative-eagle-topk={cfg.speculative_eagle_topk!r}."
)
# Both roles: verify routes to either backend depending on
# --speculative-attention-mode.
spec_allowed = {"triton", "trtllm_mla", "cutedsl_mla", "tokenspeed_mla"}
spec_backends = set(attention_backends_of(resolved_view(server_args)))
spec_backends.discard(None)
assert spec_backends <= spec_allowed, (
"--enable-unified-memory + DSPARK requires spec-verify-audited "
f"attention backends {sorted(spec_allowed)} for both prefill "
f"and decode; got {sorted(spec_backends)}. flashinfer / fa3 do "
"not translate speculative verify indices to the unified "
"pool's physical ids yet."
)
_assert_spec_verify_backends(server_args, algorithm="DSPARK")
assert not cfg.enable_two_batch_overlap, (
"--enable-unified-memory does not support --enable-two-batch-overlap: "
"TBO's replay split hands each child a view without the pre-translate "
@@ -344,6 +344,7 @@ class AiterAttnBackend(AttentionBackend):
self.kv_cache_dtype = model_runner.kv_cache_dtype
self.req_to_token = model_runner.req_to_token_pool.req_to_token
self.kv_index_translator = model_runner.kv_index_translator
self.use_mla = model_runner.model_config.attention_arch == AttentionArch.MLA
@@ -1946,10 +1947,10 @@ class AiterAttnBackend(AttentionBackend):
else:
kv_indices, kv_indptr, qo_indptr, _ = (
forward_batch.spec_info.generate_attn_arg_prefill(
forward_batch.req_pool_indices,
forward_batch.seq_lens,
forward_batch.seq_lens_sum,
self.req_to_token,
req_pool_indices=forward_batch.req_pool_indices,
paged_kernel_lens=forward_batch.seq_lens,
paged_kernel_lens_sum=forward_batch.seq_lens_sum,
translator=self.kv_index_translator,
)
)
self.forward_metadata = ForwardMetadata(
@@ -4438,6 +4439,7 @@ class AiterIndicesUpdaterPrefill:
self.kv_last_page_len = attn_backend.kv_last_page_len
self.qo_indptr = attn_backend.qo_indptr
self.req_to_token = model_runner.req_to_token_pool.req_to_token
self.kv_index_translator = model_runner.kv_index_translator
self.update = self.update_single_wrapper
self.kv_indices = None
@@ -4509,10 +4511,10 @@ class AiterIndicesUpdaterPrefill:
else:
kv_indices, kv_indptr, qo_indptr, custom_mask = (
spec_info.generate_attn_arg_prefill(
req_pool_indices,
paged_kernel_lens,
paged_kernel_lens_sum,
self.req_to_token,
req_pool_indices=req_pool_indices,
paged_kernel_lens=paged_kernel_lens,
paged_kernel_lens_sum=paged_kernel_lens_sum,
translator=self.kv_index_translator,
)
)
@@ -4526,6 +4528,7 @@ class AiterMlaIndicesUpdaterPrefill:
# Buffers and wrappers
self.req_to_token = model_runner.req_to_token_pool.req_to_token
self.kv_index_translator = model_runner.kv_index_translator
self.update = self.update_single_wrapper
self.kv_indptr = None
@@ -4587,10 +4590,10 @@ class AiterMlaIndicesUpdaterPrefill:
else:
kv_indices, kv_indptr, qo_indptr, custom_mask = (
spec_info.generate_attn_arg_prefill(
req_pool_indices,
kv_lens,
kv_lens_sum,
self.req_to_token,
req_pool_indices=req_pool_indices,
paged_kernel_lens=kv_lens,
paged_kernel_lens_sum=kv_lens_sum,
translator=self.kv_index_translator,
)
)
@@ -465,6 +465,12 @@ class FlashAttentionBackend(AttentionBackend):
# the persistent metadata.
if not forward_batch.forward_mode.is_draft_extend_v2():
return
if self.kv_index_translator.is_translating:
# The draft-extend runners record this call unconditionally, but a
# translating backend rebuilds these tables out of graph on every
# replay; a captured raw req_to_token gather would then overwrite
# them with virtual ids.
return
bs = forward_batch.batch_size
metadata = self.draft_extend_metadata[bs]
mapping = self._in_graph_full_to_swa_index_mapping()
@@ -511,6 +517,10 @@ class FlashAttentionBackend(AttentionBackend):
return self.token_to_kv_pool.full_to_swa_index_mapping
def draft_extend_metadata_captured_in_graph(self) -> bool:
# A translating backend rebuilds out of graph: the captured gather
# would bake raw req_to_token (virtual) ids into the page table.
if self.kv_index_translator.is_translating:
return False
return (
not self.use_sliding_window_kv_pool
or self.token_to_kv_pool.full_to_swa_index_mapping is not None
@@ -813,6 +823,32 @@ class FlashAttentionBackend(AttentionBackend):
m.max_seq_len_k = self.max_context_len
self.forward_metadata = m
def _spec_read_seq_len_delta(
self, forward_mode: ForwardMode, spec_info: Optional[SpecInput]
) -> int:
"""Columns past ``seq_lens`` this mode's whole-sequence read covers —
the translated table's fill must reach ``cache_seqlens``. Takes the
two fields rather than a ForwardBatch: the captured build slices its
batch and has none. Now read by both builds, so eager and replay
widen identically.
Zero for the prefix-only shapes: normal decode/extend, the topk>1
split (drafts read via the expand metadata), draft-extend whose lens
already include the window, and draft-extend's idle batch. The ragged
verify layout's per-row lens are bounded by the draft window, so its
upper bound rides the same delta."""
if spec_info is None:
return 0
if forward_mode.is_target_verify() and self.topk <= 1:
return self.speculative_num_draft_tokens
if (
forward_mode.is_decode_or_idle()
and self.topk <= 1
and self.speculative_num_steps > 0
):
return self.speculative_step_id + 1
return 0
def init_forward_metadata(self, forward_batch: ForwardBatch):
"""Initialize forward metadata hence all layers in the forward pass can reuse it."""
metadata = FlashAttentionMetadata()
@@ -1201,7 +1237,15 @@ class FlashAttentionBackend(AttentionBackend):
else None
)
if _unified_read:
kv_view = self.kv_index_translator.index_table_for_batch(forward_batch)
delta = self._spec_read_seq_len_delta(
forward_batch.forward_mode, forward_batch.spec_info
)
if delta:
kv_view = self.kv_index_translator.widened_index_table(
forward_batch, seq_len_delta=delta
)
else:
kv_view = self.kv_index_translator.index_table_for_batch(forward_batch)
metadata.page_table = kv_view.ids
if self.use_sliding_window_kv_pool:
metadata.swa_page_table = kv_view.sliding_window_ids
@@ -2892,13 +2936,11 @@ class FlashAttentionBackend(AttentionBackend):
# Page table built on-device (self-guards on cache_seqlens);
# max_seq_len_k left unset -- unread here (scheduler_metadata
# is normal-decode-only).
# Spec is asserted off under the unified pool, so this
# captured view is always the passthrough (req_to_token).
self._set_decode_page_metadata(
metadata,
req_pool_indices,
seq_lens,
self.speculative_step_id + 1,
self._spec_read_seq_len_delta(forward_mode, spec_info),
)
else:
@@ -2999,7 +3041,10 @@ class FlashAttentionBackend(AttentionBackend):
else self.max_context_len
)
self._set_decode_page_metadata(
metadata, req_pool_indices, seq_lens, 0
metadata,
req_pool_indices,
seq_lens,
self._spec_read_seq_len_delta(forward_mode, spec_info),
)
if self.local_attn_builder is not None:
@@ -3034,32 +3079,20 @@ class FlashAttentionBackend(AttentionBackend):
geometry = build_ragged_target_verify_geometry(
seq_lens=seq_lens, layout=padded
)
metadata.cache_seqlens_int32.copy_(geometry.cache_seqlens_int32)
metadata.cu_seqlens_q.copy_(geometry.cu_seqlens_q)
# Per-row verify lens; the builder re-derives
# cache_seqlens from them at delta 0.
verify_lens = geometry.cache_seqlens_int32
verify_delta = 0
else:
metadata.cache_seqlens_int32.copy_(
(seq_lens + self.speculative_num_draft_tokens)
)
verify_lens = seq_lens
verify_delta = self.speculative_num_draft_tokens
# Page table built on-device (self-guards on cache_seqlens);
# max_seq_len_k left unset -- unread here (scheduler_metadata is
# normal-decode-only).
metadata.cu_seqlens_k[1:].copy_(
torch.cumsum(metadata.cache_seqlens_int32, dim=0, dtype=torch.int32)
)
has_swa = self.use_sliding_window_kv_pool
build_trtllm_mha_page_table(
req_to_token=self.req_to_token,
req_pool_indices=req_pool_indices,
cache_seqlens=metadata.cache_seqlens_int32,
page_table=metadata.page_table,
page_size=self.page_size,
swa_page_table=metadata.swa_page_table if has_swa else None,
full_to_swa=(
self.token_to_kv_pool.full_to_swa_index_mapping
if has_swa
else None
),
self._set_decode_page_metadata(
metadata, req_pool_indices, verify_lens, verify_delta
)
else:
# When topk > 1, we need two specific target verify metadata, and then merge states
@@ -3191,18 +3224,44 @@ class FlashAttentionBackend(AttentionBackend):
max_seq_pages = (
metadata.max_seq_len_k + self.page_size - 1
) // self.page_size
page_indices = self.req_to_token[
req_pool_indices[:, None],
self.draft_extend_metadata["strided_indices"][:max_seq_pages],
]
if self.use_sliding_window_kv_pool and metadata.swa_page_table is not None:
swa_page_indices = self.token_to_kv_pool.translate_loc_from_full_to_swa(
page_indices
if self.kv_index_translator.reads_are_translated:
# The translator writes kernel-facing page ids straight into
# the captured tables; `sliding_window_out` fills the swa twin
# in the same pass.
self.kv_index_translator.fill_read_table(
out=metadata.page_table,
sliding_window_out=(
metadata.swa_page_table
if self.use_sliding_window_kv_pool
and metadata.swa_page_table is not None
else None
),
req_pool_indices=req_pool_indices,
seq_lens=seq_lens,
seq_len_delta=self._spec_read_seq_len_delta(
forward_mode, spec_info
),
)
metadata.swa_page_table[:, :max_seq_pages].copy_(
swa_page_indices // self.page_size
else:
page_indices = self.req_to_token[
req_pool_indices[:, None],
self.draft_extend_metadata["strided_indices"][:max_seq_pages],
]
if (
self.use_sliding_window_kv_pool
and metadata.swa_page_table is not None
):
swa_page_indices = (
self.token_to_kv_pool.translate_loc_from_full_to_swa(
page_indices
)
)
metadata.swa_page_table[:, :max_seq_pages].copy_(
swa_page_indices // self.page_size
)
metadata.page_table[:, :max_seq_pages].copy_(
page_indices // self.page_size
)
metadata.page_table[:, :max_seq_pages].copy_(page_indices // self.page_size)
else:
raise ValueError(
@@ -1840,10 +1840,6 @@ class FlashInferIndicesUpdaterPrefill:
self.kv_indptr = attn_backend.kv_indptr
self.kv_last_page_len = attn_backend.kv_last_page_len
self.qo_indptr = attn_backend.qo_indptr
# Kept ONLY for the spec-info branches (generate_attn_arg_prefill),
# which are static-pool-only: unified memory asserts spec off. The
# normal builders source from the per-batch KVIndexTable.
self.req_to_token = model_runner.req_to_token_pool.req_to_token
self._swa_kv_pool = attn_backend._swa_kv_pool
self.prefill_wrapper_ragged = attn_backend.prefill_wrapper_ragged
@@ -2202,20 +2198,22 @@ class FlashInferIndicesUpdaterPrefill:
if spec_info.spec_input_type == SpecInputType.DFLASH_VERIFY:
kv_indices, kv_indptr, qo_indptr, custom_mask = (
spec_info.generate_attn_arg_prefill(
req_pool_indices,
paged_kernel_lens,
paged_kernel_lens_sum,
self.req_to_token,
req_pool_indices=req_pool_indices,
paged_kernel_lens=paged_kernel_lens,
paged_kernel_lens_sum=paged_kernel_lens_sum,
translator=translator,
sliding_window=use_swa_source,
kv_start_idx=kv_start_idx,
)
)
else:
kv_indices, kv_indptr, qo_indptr, custom_mask = (
spec_info.generate_attn_arg_prefill(
req_pool_indices,
paged_kernel_lens,
paged_kernel_lens_sum,
self.req_to_token,
req_pool_indices=req_pool_indices,
paged_kernel_lens=paged_kernel_lens,
paged_kernel_lens_sum=paged_kernel_lens_sum,
translator=translator,
sliding_window=use_swa_source,
)
)
@@ -992,10 +992,6 @@ class FlashInferMLAIndicesUpdaterPrefill:
# Buffers and wrappers
self.kv_indptr = attn_backend.kv_indptr
self.qo_indptr = attn_backend.qo_indptr
# Kept ONLY for the spec-info branch (generate_attn_arg_prefill), which
# is static-pool-only: unified memory asserts spec off. The normal
# builder reads req_to_token through the translator.
self.req_to_token = model_runner.req_to_token_pool.req_to_token
self.prefill_wrapper_ragged = attn_backend.prefill_wrapper_ragged
def update(
@@ -1085,10 +1081,10 @@ class FlashInferMLAIndicesUpdaterPrefill:
elif fast_verify_plan_kwargs is not None:
kv_indices, kv_indptr, qo_indptr, custom_mask = (
spec_info.generate_attn_arg_prefill(
req_pool_indices,
paged_kernel_lens,
paged_kernel_lens_sum,
self.req_to_token,
req_pool_indices=req_pool_indices,
paged_kernel_lens=paged_kernel_lens,
paged_kernel_lens_sum=paged_kernel_lens_sum,
translator=self.attn_backend.kv_index_translator,
kv_indices_buf=fast_verify_plan_kwargs["kv_indices_buf"],
)
)
@@ -1097,10 +1093,10 @@ class FlashInferMLAIndicesUpdaterPrefill:
# TODO: Support topk > 1 with custom mask
kv_indices, kv_indptr, qo_indptr, custom_mask = (
spec_info.generate_attn_arg_prefill(
req_pool_indices,
paged_kernel_lens,
paged_kernel_lens_sum,
self.req_to_token,
req_pool_indices=req_pool_indices,
paged_kernel_lens=paged_kernel_lens,
paged_kernel_lens_sum=paged_kernel_lens_sum,
translator=self.attn_backend.kv_index_translator,
)
)
@@ -163,7 +163,7 @@ class FlashMLABackend(FlashInferMLAAttnBackend):
if forward_batch.forward_mode.is_decode_or_idle():
max_seqlen_pad = triton.cdiv(eager_max_k, PAGE_SIZE)
block_kv_indices = self._eager_block_kv_indices(bs, max_seqlen_pad)
if self.kv_index_translator.is_translating:
if self.kv_index_translator.reads_are_translated:
assert self.page_size == PAGE_SIZE
self.kv_index_translator.fill_read_table(
out=block_kv_indices,
@@ -198,17 +198,25 @@ class FlashMLABackend(FlashInferMLAAttnBackend):
max_seqlen_pad = triton.cdiv(eager_max_k + self.num_draft_tokens, PAGE_SIZE)
block_kv_indices = self._eager_block_kv_indices(bs, max_seqlen_pad)
create_flashmla_kv_indices_triton[
(bs, get_num_kv_index_blocks_flashmla(max_seqlen_pad, PAGE_SIZE))
](
self.req_to_token,
forward_batch.req_pool_indices,
seq_lens,
None,
block_kv_indices,
self.req_to_token.stride(0),
block_kv_indices.stride(0),
)
if self.kv_index_translator.reads_are_translated:
assert self.page_size == PAGE_SIZE
self.kv_index_translator.fill_read_table(
out=block_kv_indices,
req_pool_indices=forward_batch.req_pool_indices,
seq_lens=seq_lens,
)
else:
create_flashmla_kv_indices_triton[
(bs, get_num_kv_index_blocks_flashmla(max_seqlen_pad, PAGE_SIZE))
](
self.req_to_token,
forward_batch.req_pool_indices,
seq_lens,
None,
block_kv_indices,
self.req_to_token.stride(0),
block_kv_indices.stride(0),
)
mla_metadata, num_splits = get_mla_metadata(
seq_lens.to(torch.int32),
self.num_draft_tokens * self.num_q_heads,
@@ -230,17 +238,25 @@ class FlashMLABackend(FlashInferMLAAttnBackend):
max_seqlen_pad = triton.cdiv(eager_max_k + window, PAGE_SIZE)
block_kv_indices = self._eager_block_kv_indices(bs, max_seqlen_pad)
create_flashmla_kv_indices_triton[
(bs, get_num_kv_index_blocks_flashmla(max_seqlen_pad, PAGE_SIZE))
](
self.req_to_token,
forward_batch.req_pool_indices,
seq_lens_k,
None,
block_kv_indices,
self.req_to_token.stride(0),
block_kv_indices.stride(0),
)
if self.kv_index_translator.reads_are_translated:
assert self.page_size == PAGE_SIZE
self.kv_index_translator.fill_read_table(
out=block_kv_indices,
req_pool_indices=forward_batch.req_pool_indices,
seq_lens=seq_lens_k,
)
else:
create_flashmla_kv_indices_triton[
(bs, get_num_kv_index_blocks_flashmla(max_seqlen_pad, PAGE_SIZE))
](
self.req_to_token,
forward_batch.req_pool_indices,
seq_lens_k,
None,
block_kv_indices,
self.req_to_token.stride(0),
block_kv_indices.stride(0),
)
mla_metadata, num_splits = get_mla_metadata(
seq_lens_k,
window * self.num_q_heads,
@@ -337,7 +353,7 @@ class FlashMLABackend(FlashInferMLAAttnBackend):
else:
max_seqlen_pad = self.cuda_graph_kv_indices.shape[1]
if self.kv_index_translator.is_translating:
if self.kv_index_translator.reads_are_translated:
assert self.page_size == PAGE_SIZE
self.kv_index_translator.fill_read_table(
out=self.cuda_graph_kv_indices,
@@ -257,6 +257,7 @@ class KVIndexTranslator:
seq_lens: torch.Tensor,
max_pages: Optional[int] = None,
into: Optional[KVReadTables] = None,
seq_len_delta: int = 0,
) -> KVIndexTable:
"""The one per-batch entry point.
@@ -265,6 +266,10 @@ class KVIndexTranslator:
returns the table WHOLE, so a caller needing a stable pointer (a
captured graph bakes it) passes its own tables in ``into``;
``into=None`` allocates of width ``max_pages`` instead.
``seq_len_delta`` widens every row's live prefix -- the whole-sequence
verify contract (draft KV read back from the pool). An eager caller's
``max_pages`` must already cover the delta.
"""
if not self.is_translating or self.defer_read_translate:
return KVIndexTable(
@@ -308,6 +313,7 @@ class KVIndexTranslator:
page_size=self.page_size,
max_pages=width,
out=out_full,
seq_len_delta=seq_len_delta,
)
if out_swa is not None:
build_kv_read_table(
@@ -318,6 +324,7 @@ class KVIndexTranslator:
page_size=self.page_size,
max_pages=width,
out=out_swa,
seq_len_delta=seq_len_delta,
)
return KVIndexTable(
ids=out_full,
@@ -335,6 +342,7 @@ class KVIndexTranslator:
req_pool_indices: torch.Tensor,
seq_lens: torch.Tensor,
sliding_window_out: Optional[torch.Tensor] = None,
seq_len_delta: int = 0,
) -> None:
"""`build_index_table(into=...)` for a caller that owns a bare block
table rather than a KVReadTables: the page-table consumers read that
@@ -359,6 +367,7 @@ class KVIndexTranslator:
req_pool_indices=req_pool_indices,
seq_lens=seq_lens,
into=KVReadTables(full=out, sliding_window=sliding_window_out),
seq_len_delta=seq_len_delta,
)
def index_table_for_batch(self, forward_batch) -> KVIndexTable:
@@ -404,6 +413,38 @@ class KVIndexTranslator:
"""
return self._full_v2p_table
def widened_index_table(self, forward_batch, *, seq_len_delta: int) -> KVIndexTable:
"""Whole-sequence-verify view: every row's live prefix widened by
`seq_len_delta` columns (the drafts are read back from the pool).
Not memoized -- verify is one consumer, and the batch's memoized
prefix table stays valid for the others.
The width comes from the LIVE prefix: a DSPARK/DFLASH verify batch
carries `seq_lens_cpu` already expanded by the window and keeps the
live lens on `spec_info.live_seq_lens_cpu`, so widening the expanded
lens would count the window twice. The width never exceeds the
`req_to_token` row it is read from."""
max_pages = None
if self.is_translating:
row_pages = -(-self.req_to_token.shape[1] // self.page_size)
live = getattr(forward_batch.spec_info, "live_seq_lens_cpu", None)
slc = live if live is not None else forward_batch.seq_lens_cpu
if (
forward_batch.seq_lens_sum is not None
and slc is not None
and slc.numel() > 0
):
max_seq = int(slc.max()) + seq_len_delta
max_pages = min(max(-(-max_seq // self.page_size), 1), row_pages)
else:
max_pages = row_pages
return self.build_index_table(
req_pool_indices=forward_batch.req_pool_indices,
seq_lens=forward_batch.seq_lens,
max_pages=max_pages,
seq_len_delta=seq_len_delta,
)
def bind_and_verify_backends(self, backends) -> None:
"""Boot: make every reachable backend carry THIS translator.
+1
View File
@@ -349,6 +349,7 @@ class FalconH1HybridAttentionDecoderLayer(nn.Module):
layer_id=self.layer_id,
forward_batch=forward_batch,
mup_vector=self.mup_vector,
use_triton_causal_conv=forward_batch.forward_mode.is_target_verify(),
)
mamba_hidden_states = mamba_hidden_states * self.ssm_out_multiplier
+16 -11
View File
@@ -5,8 +5,8 @@ from typing import TYPE_CHECKING, Optional
import torch
from sglang.kernels.ops.attention.utils import create_flashinfer_kv_indices_triton
from sglang.srt.managers.schedule_batch import ScheduleBatch
from sglang.srt.mem_cache.kv_index_translator import KVIndexTranslator
from sglang.srt.model_executor.forward_batch_info import (
CaptureHiddenMode,
ForwardBatch,
@@ -128,15 +128,20 @@ class DFlashVerifyInput(SpecInput):
def generate_attn_arg_prefill(
self,
*,
req_pool_indices: torch.Tensor,
paged_kernel_lens: torch.Tensor,
paged_kernel_lens_sum: int,
req_to_token: torch.Tensor,
translator: KVIndexTranslator,
kv_start_idx: Optional[torch.Tensor] = None,
kv_indices_buf: Optional[torch.Tensor] = None,
sliding_window: bool = False,
):
"""CSR verify args, gathered straight into the packed stream. The lens
are widened here and handed to the translator, so nothing materializes
a ``[bs, max_pages]`` rectangle to repack from."""
device = req_pool_indices.device
bs = len(req_pool_indices)
bs = req_pool_indices.numel()
layout = self.ragged_verify_layout
if layout is not None and layout.bs != bs:
@@ -174,14 +179,14 @@ class DFlashVerifyInput(SpecInput):
dtype=torch.int32,
device=device,
)
create_flashinfer_kv_indices_triton[(bs,)](
req_to_token,
req_pool_indices,
paged_kernel_lens,
cum_kv_seq_len,
kv_start_idx,
kv_indices,
req_to_token.size(1),
translator.fill_packed_read_stream(
req_pool_indices=req_pool_indices,
seq_lens=paged_kernel_lens,
indptr=cum_kv_seq_len,
total_tokens=paged_kernel_lens_sum + kv_indices_extra,
out=kv_indices,
kv_start_idx=kv_start_idx,
sliding_window=sliding_window,
)
mask = self.custom_mask
if mask is not None:
+29 -25
View File
@@ -4,7 +4,7 @@ from typing import Callable, List, Optional
import torch
from sglang.kernels.ops.attention.utils import create_flashinfer_kv_indices_triton
from sglang.srt.mem_cache.kv_index_translator import KVIndexTranslator
from sglang.srt.model_executor.forward_batch_info import CaptureHiddenMode
from sglang.srt.runtime_context import get_spec
from sglang.srt.speculative.spec_info import SpecInput, SpecInputType
@@ -85,13 +85,18 @@ class EagleVerifyInput(SpecInput):
def generate_attn_arg_prefill(
self,
*,
req_pool_indices: torch.Tensor,
paged_kernel_lens: torch.Tensor,
paged_kernel_lens_sum: int,
req_to_token: torch.Tensor,
translator: KVIndexTranslator,
sliding_window: bool = False,
):
"""CSR verify args, gathered straight into the packed stream. The lens
are widened here and handed to the translator, so nothing materializes
a ``[bs, max_pages]`` rectangle to repack from."""
device = req_pool_indices.device
batch_size = len(req_pool_indices)
batch_size = req_pool_indices.numel()
qo_indptr = torch.arange(
0,
(1 + batch_size) * self.draft_token_num,
@@ -106,19 +111,15 @@ class EagleVerifyInput(SpecInput):
paged_kernel_lens = paged_kernel_lens + self.draft_token_num
cum_kv_seq_len[1:] = torch.cumsum(paged_kernel_lens, dim=0)
kv_indices = torch.empty(
paged_kernel_lens_sum + self.draft_token_num * batch_size,
dtype=torch.int32,
device=device,
)
create_flashinfer_kv_indices_triton[(batch_size,)](
req_to_token,
req_pool_indices,
paged_kernel_lens,
cum_kv_seq_len,
None,
kv_indices,
req_to_token.size(1),
total_tokens = paged_kernel_lens_sum + self.draft_token_num * batch_size
kv_indices = torch.empty(total_tokens, dtype=torch.int32, device=device)
translator.fill_packed_read_stream(
req_pool_indices=req_pool_indices,
seq_lens=paged_kernel_lens,
indptr=cum_kv_seq_len,
total_tokens=total_tokens,
out=kv_indices,
sliding_window=sliding_window,
)
mask_numel = (
paged_kernel_lens_sum * self.draft_token_num
@@ -396,11 +397,15 @@ class EagleDraftExtendInput(SpecInput):
def generate_attn_arg_prefill(
self,
*,
req_pool_indices: torch.Tensor,
paged_kernel_lens: torch.Tensor,
paged_kernel_lens_sum: Optional[int],
req_to_token: torch.Tensor,
translator: KVIndexTranslator,
sliding_window: bool = False,
):
"""Draft-extend CSR args. The lens already include the window, so
unlike verify nothing is widened here."""
device = req_pool_indices.device
bs = self.num_correct_drafts.numel()
# Constant num_tokens_per_req qo layout (required for cuda-graph capture).
@@ -421,13 +426,12 @@ class EagleDraftExtendInput(SpecInput):
paged_kernel_lens_sum, dtype=torch.int32, device=device
)
create_flashinfer_kv_indices_triton[(bs,)](
req_to_token,
req_pool_indices,
paged_kernel_lens,
cum_kv_seq_len,
None,
kv_indices,
req_to_token.size(1),
translator.fill_packed_read_stream(
req_pool_indices=req_pool_indices,
seq_lens=paged_kernel_lens,
indptr=cum_kv_seq_len,
total_tokens=paged_kernel_lens_sum,
out=kv_indices,
sliding_window=sliding_window,
)
return kv_indices, cum_kv_seq_len, qo_indptr, None
+17 -16
View File
@@ -4,7 +4,7 @@ from typing import List, Optional
import torch
from sglang.kernels.ops.attention.utils import create_flashinfer_kv_indices_triton
from sglang.srt.mem_cache.kv_index_translator import KVIndexTranslator
from sglang.srt.speculative.spec_info import SpecInput, SpecInputType
@@ -58,12 +58,17 @@ class NgramVerifyInput(SpecInput):
def generate_attn_arg_prefill(
self,
*,
req_pool_indices: torch.Tensor,
paged_kernel_lens: torch.Tensor,
paged_kernel_lens_sum: int,
req_to_token: torch.Tensor,
translator: KVIndexTranslator,
sliding_window: bool = False,
):
bs = len(req_pool_indices)
"""CSR verify args, gathered straight into the packed stream. The lens
are widened here and handed to the translator, so nothing materializes
a ``[bs, max_pages]`` rectangle to repack from."""
bs = req_pool_indices.numel()
cum_kv_seq_len = torch.zeros((bs + 1,), dtype=torch.int32, device=self.device)
@@ -75,20 +80,16 @@ class NgramVerifyInput(SpecInput):
* self.draft_token_num
)
kv_indices = torch.empty(
paged_kernel_lens_sum + self.draft_token_num * bs,
dtype=torch.int32,
device=self.device,
)
total_tokens = paged_kernel_lens_sum + self.draft_token_num * bs
kv_indices = torch.empty(total_tokens, dtype=torch.int32, device=self.device)
create_flashinfer_kv_indices_triton[(bs,)](
req_to_token,
req_pool_indices,
paged_kernel_lens,
cum_kv_seq_len,
None,
kv_indices,
req_to_token.size(1),
translator.fill_packed_read_stream(
req_pool_indices=req_pool_indices,
seq_lens=paged_kernel_lens,
indptr=cum_kv_seq_len,
total_tokens=total_tokens,
out=kv_indices,
sliding_window=sliding_window,
)
# Pad custom_mask when CUDA graph pads batch size beyond the actual number of requests.