[unified-memory] Speculative decoding on the unified memory pool: translate every read and write path (#37627)
Co-authored-by: Caihua Li <caihua.li@bytedance.com> Co-authored-by: Cheng Wan <cheng.wan@radixark.ai>
This commit is contained in:
co-authored by
Caihua Li
Cheng Wan
parent
33883a5fa3
commit
08d30f2300
@@ -71,6 +71,8 @@ def build_kv_read_indices_kernel(
|
||||
req_stride, # runtime: req_to_token row stride (elements)
|
||||
out_stride, # runtime: uniform row stride, used when row_starts is null
|
||||
item_stride, # runtime: items one program advances per loop trip
|
||||
seq_len_delta, # runtime: verify widening added to every row's live prefix
|
||||
max_row_items, # runtime: uniform row width; a longer window stops there
|
||||
PAGE_SIZE: tl.constexpr,
|
||||
EMIT_PER_TOKEN: tl.constexpr,
|
||||
OUT_INT64: tl.constexpr,
|
||||
@@ -78,7 +80,7 @@ def build_kv_read_indices_kernel(
|
||||
):
|
||||
bid = tl.program_id(0)
|
||||
req = tl.load(req_pool_indices_ptr + bid).to(tl.int64)
|
||||
seqlen = tl.load(seq_lens_ptr + bid)
|
||||
seqlen = tl.load(seq_lens_ptr + bid) + seq_len_delta
|
||||
# Derived here, not on the host: one elementwise op there costs a whole
|
||||
# launch, which a captured graph then replays every step.
|
||||
if EMIT_PER_TOKEN:
|
||||
@@ -93,6 +95,9 @@ def build_kv_read_indices_kernel(
|
||||
row_out = out_ptr + tl.load(row_starts_ptr + bid).to(tl.int64)
|
||||
else:
|
||||
row_out = out_ptr + bid.to(tl.int64) * out_stride
|
||||
# A widened window can outrun a fixed-width row (a verify near the
|
||||
# context limit); writing past it would land in the next row.
|
||||
n_items = tl.minimum(n_items, max_row_items)
|
||||
|
||||
for start in range(tl.program_id(1) * BLOCK, n_items, item_stride):
|
||||
item = start + tl.arange(0, BLOCK)
|
||||
@@ -132,6 +137,7 @@ def _launch(
|
||||
row_starts: Optional[torch.Tensor],
|
||||
kv_start_idx: Optional[torch.Tensor],
|
||||
emit_per_token: bool,
|
||||
seq_len_delta: int = 0,
|
||||
) -> None:
|
||||
bs = int(req_pool_indices.numel())
|
||||
item_programs = min(
|
||||
@@ -148,6 +154,8 @@ def _launch(
|
||||
req_to_token.stride(0),
|
||||
out_stride,
|
||||
item_programs * _BLOCK_ITEMS,
|
||||
seq_len_delta,
|
||||
max_items,
|
||||
PAGE_SIZE=page_size,
|
||||
EMIT_PER_TOKEN=emit_per_token,
|
||||
OUT_INT64=out.dtype == torch.int64,
|
||||
@@ -179,12 +187,18 @@ def build_kv_read_table(
|
||||
page_size: int,
|
||||
max_pages: int,
|
||||
out: torch.Tensor,
|
||||
seq_len_delta: int = 0,
|
||||
) -> torch.Tensor:
|
||||
"""Fill ``out``'s live prefix with PAGE TABLE entries.
|
||||
|
||||
``out`` is caller-owned (fresh zeros for the eager path, the module's
|
||||
capture-stable buffer for replay) and only its ``[:bs, :max_pages]``
|
||||
region's live prefix is written -- never rebound, never tail-cleared.
|
||||
|
||||
``seq_len_delta`` widens every row's live prefix -- the whole-sequence
|
||||
verify contract (draft KV read back from the pool). Every row stops at
|
||||
``max_pages``: a widened window that outruns the row is cut there rather
|
||||
than spilling into the next row, on CUDA as on CPU.
|
||||
"""
|
||||
bs = int(req_pool_indices.numel())
|
||||
assert out.dtype == torch.int32, (
|
||||
@@ -205,7 +219,7 @@ def build_kv_read_table(
|
||||
if not req_to_token.is_cuda:
|
||||
cols = torch.arange(max_pages, device=req_to_token.device)
|
||||
for b in range(bs):
|
||||
n_pages = (int(seq_lens[b]) + page_size - 1) // page_size
|
||||
n_pages = (int(seq_lens[b]) + seq_len_delta + page_size - 1) // page_size
|
||||
live = min(n_pages, max_pages)
|
||||
out[b, :live] = _entries(
|
||||
req_to_token=req_to_token,
|
||||
@@ -228,6 +242,7 @@ def build_kv_read_table(
|
||||
row_starts=None,
|
||||
kv_start_idx=None,
|
||||
emit_per_token=False,
|
||||
seq_len_delta=seq_len_delta,
|
||||
)
|
||||
return out
|
||||
|
||||
|
||||
@@ -474,6 +474,44 @@ def handle_cache_compatibility(server_args: Any) -> None:
|
||||
)
|
||||
|
||||
|
||||
# Backends whose speculative verify id rails are translation-audited for
|
||||
# the unified pool.
|
||||
_SPEC_VERIFY_AUDITED_BACKENDS = frozenset(
|
||||
{
|
||||
"triton",
|
||||
"trtllm_mla",
|
||||
"cutedsl_mla",
|
||||
"tokenspeed_mla",
|
||||
"flashmla",
|
||||
"flashinfer",
|
||||
"fa3",
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def _assert_spec_verify_backends(server_args: Any, *, algorithm: str) -> None:
|
||||
"""Refuse target backends whose verify id rails are not translation-audited.
|
||||
|
||||
Only the target's prefill/decode pair is checked. A draft with a KV pool
|
||||
of its own indexes that pool directly by virtual id, so its translator is
|
||||
a passthrough and its backend has nothing to translate."""
|
||||
allowed = _SPEC_VERIFY_AUDITED_BACKENDS
|
||||
if resolving_view(server_args).dcp_size > 1:
|
||||
# flashinfer's spec verify gathers its CSR args with no DCP read
|
||||
# translation (`translate_dcp_read_ids`); the MLA verify family builds
|
||||
# its DCP block table itself.
|
||||
allowed = allowed - {"flashinfer"}
|
||||
backends = set(attention_backends_of(resolved_view(server_args)))
|
||||
backends.discard(None)
|
||||
assert backends <= allowed, (
|
||||
f"--enable-unified-memory + {algorithm} requires spec-verify-audited "
|
||||
f"attention backends {sorted(allowed)} for both prefill "
|
||||
f"and decode; got {sorted(backends)}. Other backends do "
|
||||
"not translate speculative verify indices to the unified "
|
||||
"pool's physical ids yet."
|
||||
)
|
||||
|
||||
|
||||
def handle_unified_memory_pool(server_args: Any) -> None:
|
||||
|
||||
cfg = resolving_view(server_args)
|
||||
@@ -540,25 +578,15 @@ def handle_unified_memory_pool(server_args: Any) -> None:
|
||||
"audited for the unified pool's virtual-to-physical loc translation. Got "
|
||||
f"--speculative-algorithm={cfg.speculative_algorithm!r}."
|
||||
)
|
||||
assert cfg.speculative_eagle_topk in (None, 1), (
|
||||
"--enable-unified-memory supports a linear draft chain only "
|
||||
"(--speculative-eagle-topk in {None, 1}); tree verify relocates "
|
||||
"accepted tokens one at a time inside the target pool, which the "
|
||||
"unified pool's page-granular move_kv_cache cannot express. Got "
|
||||
f"--speculative-eagle-topk={cfg.speculative_eagle_topk!r}."
|
||||
)
|
||||
if cfg.speculative_algorithm == "DSPARK":
|
||||
assert cfg.speculative_eagle_topk in (None, 1), (
|
||||
"--enable-unified-memory + DSPARK supports a linear draft "
|
||||
"chain only (--speculative-eagle-topk in {None, 1}); tree "
|
||||
"verify is not audited for the unified pool. Got "
|
||||
f"--speculative-eagle-topk={cfg.speculative_eagle_topk!r}."
|
||||
)
|
||||
# Both roles: verify routes to either backend depending on
|
||||
# --speculative-attention-mode.
|
||||
spec_allowed = {"triton", "trtllm_mla", "cutedsl_mla", "tokenspeed_mla"}
|
||||
spec_backends = set(attention_backends_of(resolved_view(server_args)))
|
||||
spec_backends.discard(None)
|
||||
assert spec_backends <= spec_allowed, (
|
||||
"--enable-unified-memory + DSPARK requires spec-verify-audited "
|
||||
f"attention backends {sorted(spec_allowed)} for both prefill "
|
||||
f"and decode; got {sorted(spec_backends)}. flashinfer / fa3 do "
|
||||
"not translate speculative verify indices to the unified "
|
||||
"pool's physical ids yet."
|
||||
)
|
||||
_assert_spec_verify_backends(server_args, algorithm="DSPARK")
|
||||
assert not cfg.enable_two_batch_overlap, (
|
||||
"--enable-unified-memory does not support --enable-two-batch-overlap: "
|
||||
"TBO's replay split hands each child a view without the pre-translate "
|
||||
|
||||
@@ -344,6 +344,7 @@ class AiterAttnBackend(AttentionBackend):
|
||||
self.kv_cache_dtype = model_runner.kv_cache_dtype
|
||||
|
||||
self.req_to_token = model_runner.req_to_token_pool.req_to_token
|
||||
self.kv_index_translator = model_runner.kv_index_translator
|
||||
|
||||
self.use_mla = model_runner.model_config.attention_arch == AttentionArch.MLA
|
||||
|
||||
@@ -1946,10 +1947,10 @@ class AiterAttnBackend(AttentionBackend):
|
||||
else:
|
||||
kv_indices, kv_indptr, qo_indptr, _ = (
|
||||
forward_batch.spec_info.generate_attn_arg_prefill(
|
||||
forward_batch.req_pool_indices,
|
||||
forward_batch.seq_lens,
|
||||
forward_batch.seq_lens_sum,
|
||||
self.req_to_token,
|
||||
req_pool_indices=forward_batch.req_pool_indices,
|
||||
paged_kernel_lens=forward_batch.seq_lens,
|
||||
paged_kernel_lens_sum=forward_batch.seq_lens_sum,
|
||||
translator=self.kv_index_translator,
|
||||
)
|
||||
)
|
||||
self.forward_metadata = ForwardMetadata(
|
||||
@@ -4438,6 +4439,7 @@ class AiterIndicesUpdaterPrefill:
|
||||
self.kv_last_page_len = attn_backend.kv_last_page_len
|
||||
self.qo_indptr = attn_backend.qo_indptr
|
||||
self.req_to_token = model_runner.req_to_token_pool.req_to_token
|
||||
self.kv_index_translator = model_runner.kv_index_translator
|
||||
self.update = self.update_single_wrapper
|
||||
|
||||
self.kv_indices = None
|
||||
@@ -4509,10 +4511,10 @@ class AiterIndicesUpdaterPrefill:
|
||||
else:
|
||||
kv_indices, kv_indptr, qo_indptr, custom_mask = (
|
||||
spec_info.generate_attn_arg_prefill(
|
||||
req_pool_indices,
|
||||
paged_kernel_lens,
|
||||
paged_kernel_lens_sum,
|
||||
self.req_to_token,
|
||||
req_pool_indices=req_pool_indices,
|
||||
paged_kernel_lens=paged_kernel_lens,
|
||||
paged_kernel_lens_sum=paged_kernel_lens_sum,
|
||||
translator=self.kv_index_translator,
|
||||
)
|
||||
)
|
||||
|
||||
@@ -4526,6 +4528,7 @@ class AiterMlaIndicesUpdaterPrefill:
|
||||
|
||||
# Buffers and wrappers
|
||||
self.req_to_token = model_runner.req_to_token_pool.req_to_token
|
||||
self.kv_index_translator = model_runner.kv_index_translator
|
||||
self.update = self.update_single_wrapper
|
||||
|
||||
self.kv_indptr = None
|
||||
@@ -4587,10 +4590,10 @@ class AiterMlaIndicesUpdaterPrefill:
|
||||
else:
|
||||
kv_indices, kv_indptr, qo_indptr, custom_mask = (
|
||||
spec_info.generate_attn_arg_prefill(
|
||||
req_pool_indices,
|
||||
kv_lens,
|
||||
kv_lens_sum,
|
||||
self.req_to_token,
|
||||
req_pool_indices=req_pool_indices,
|
||||
paged_kernel_lens=kv_lens,
|
||||
paged_kernel_lens_sum=kv_lens_sum,
|
||||
translator=self.kv_index_translator,
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
@@ -465,6 +465,12 @@ class FlashAttentionBackend(AttentionBackend):
|
||||
# the persistent metadata.
|
||||
if not forward_batch.forward_mode.is_draft_extend_v2():
|
||||
return
|
||||
if self.kv_index_translator.is_translating:
|
||||
# The draft-extend runners record this call unconditionally, but a
|
||||
# translating backend rebuilds these tables out of graph on every
|
||||
# replay; a captured raw req_to_token gather would then overwrite
|
||||
# them with virtual ids.
|
||||
return
|
||||
bs = forward_batch.batch_size
|
||||
metadata = self.draft_extend_metadata[bs]
|
||||
mapping = self._in_graph_full_to_swa_index_mapping()
|
||||
@@ -511,6 +517,10 @@ class FlashAttentionBackend(AttentionBackend):
|
||||
return self.token_to_kv_pool.full_to_swa_index_mapping
|
||||
|
||||
def draft_extend_metadata_captured_in_graph(self) -> bool:
|
||||
# A translating backend rebuilds out of graph: the captured gather
|
||||
# would bake raw req_to_token (virtual) ids into the page table.
|
||||
if self.kv_index_translator.is_translating:
|
||||
return False
|
||||
return (
|
||||
not self.use_sliding_window_kv_pool
|
||||
or self.token_to_kv_pool.full_to_swa_index_mapping is not None
|
||||
@@ -813,6 +823,32 @@ class FlashAttentionBackend(AttentionBackend):
|
||||
m.max_seq_len_k = self.max_context_len
|
||||
self.forward_metadata = m
|
||||
|
||||
def _spec_read_seq_len_delta(
|
||||
self, forward_mode: ForwardMode, spec_info: Optional[SpecInput]
|
||||
) -> int:
|
||||
"""Columns past ``seq_lens`` this mode's whole-sequence read covers —
|
||||
the translated table's fill must reach ``cache_seqlens``. Takes the
|
||||
two fields rather than a ForwardBatch: the captured build slices its
|
||||
batch and has none. Now read by both builds, so eager and replay
|
||||
widen identically.
|
||||
|
||||
Zero for the prefix-only shapes: normal decode/extend, the topk>1
|
||||
split (drafts read via the expand metadata), draft-extend whose lens
|
||||
already include the window, and draft-extend's idle batch. The ragged
|
||||
verify layout's per-row lens are bounded by the draft window, so its
|
||||
upper bound rides the same delta."""
|
||||
if spec_info is None:
|
||||
return 0
|
||||
if forward_mode.is_target_verify() and self.topk <= 1:
|
||||
return self.speculative_num_draft_tokens
|
||||
if (
|
||||
forward_mode.is_decode_or_idle()
|
||||
and self.topk <= 1
|
||||
and self.speculative_num_steps > 0
|
||||
):
|
||||
return self.speculative_step_id + 1
|
||||
return 0
|
||||
|
||||
def init_forward_metadata(self, forward_batch: ForwardBatch):
|
||||
"""Initialize forward metadata hence all layers in the forward pass can reuse it."""
|
||||
metadata = FlashAttentionMetadata()
|
||||
@@ -1201,7 +1237,15 @@ class FlashAttentionBackend(AttentionBackend):
|
||||
else None
|
||||
)
|
||||
if _unified_read:
|
||||
kv_view = self.kv_index_translator.index_table_for_batch(forward_batch)
|
||||
delta = self._spec_read_seq_len_delta(
|
||||
forward_batch.forward_mode, forward_batch.spec_info
|
||||
)
|
||||
if delta:
|
||||
kv_view = self.kv_index_translator.widened_index_table(
|
||||
forward_batch, seq_len_delta=delta
|
||||
)
|
||||
else:
|
||||
kv_view = self.kv_index_translator.index_table_for_batch(forward_batch)
|
||||
metadata.page_table = kv_view.ids
|
||||
if self.use_sliding_window_kv_pool:
|
||||
metadata.swa_page_table = kv_view.sliding_window_ids
|
||||
@@ -2892,13 +2936,11 @@ class FlashAttentionBackend(AttentionBackend):
|
||||
# Page table built on-device (self-guards on cache_seqlens);
|
||||
# max_seq_len_k left unset -- unread here (scheduler_metadata
|
||||
# is normal-decode-only).
|
||||
# Spec is asserted off under the unified pool, so this
|
||||
# captured view is always the passthrough (req_to_token).
|
||||
self._set_decode_page_metadata(
|
||||
metadata,
|
||||
req_pool_indices,
|
||||
seq_lens,
|
||||
self.speculative_step_id + 1,
|
||||
self._spec_read_seq_len_delta(forward_mode, spec_info),
|
||||
)
|
||||
|
||||
else:
|
||||
@@ -2999,7 +3041,10 @@ class FlashAttentionBackend(AttentionBackend):
|
||||
else self.max_context_len
|
||||
)
|
||||
self._set_decode_page_metadata(
|
||||
metadata, req_pool_indices, seq_lens, 0
|
||||
metadata,
|
||||
req_pool_indices,
|
||||
seq_lens,
|
||||
self._spec_read_seq_len_delta(forward_mode, spec_info),
|
||||
)
|
||||
|
||||
if self.local_attn_builder is not None:
|
||||
@@ -3034,32 +3079,20 @@ class FlashAttentionBackend(AttentionBackend):
|
||||
geometry = build_ragged_target_verify_geometry(
|
||||
seq_lens=seq_lens, layout=padded
|
||||
)
|
||||
metadata.cache_seqlens_int32.copy_(geometry.cache_seqlens_int32)
|
||||
metadata.cu_seqlens_q.copy_(geometry.cu_seqlens_q)
|
||||
# Per-row verify lens; the builder re-derives
|
||||
# cache_seqlens from them at delta 0.
|
||||
verify_lens = geometry.cache_seqlens_int32
|
||||
verify_delta = 0
|
||||
else:
|
||||
metadata.cache_seqlens_int32.copy_(
|
||||
(seq_lens + self.speculative_num_draft_tokens)
|
||||
)
|
||||
verify_lens = seq_lens
|
||||
verify_delta = self.speculative_num_draft_tokens
|
||||
|
||||
# Page table built on-device (self-guards on cache_seqlens);
|
||||
# max_seq_len_k left unset -- unread here (scheduler_metadata is
|
||||
# normal-decode-only).
|
||||
metadata.cu_seqlens_k[1:].copy_(
|
||||
torch.cumsum(metadata.cache_seqlens_int32, dim=0, dtype=torch.int32)
|
||||
)
|
||||
has_swa = self.use_sliding_window_kv_pool
|
||||
build_trtllm_mha_page_table(
|
||||
req_to_token=self.req_to_token,
|
||||
req_pool_indices=req_pool_indices,
|
||||
cache_seqlens=metadata.cache_seqlens_int32,
|
||||
page_table=metadata.page_table,
|
||||
page_size=self.page_size,
|
||||
swa_page_table=metadata.swa_page_table if has_swa else None,
|
||||
full_to_swa=(
|
||||
self.token_to_kv_pool.full_to_swa_index_mapping
|
||||
if has_swa
|
||||
else None
|
||||
),
|
||||
self._set_decode_page_metadata(
|
||||
metadata, req_pool_indices, verify_lens, verify_delta
|
||||
)
|
||||
else:
|
||||
# When topk > 1, we need two specific target verify metadata, and then merge states
|
||||
@@ -3191,18 +3224,44 @@ class FlashAttentionBackend(AttentionBackend):
|
||||
max_seq_pages = (
|
||||
metadata.max_seq_len_k + self.page_size - 1
|
||||
) // self.page_size
|
||||
page_indices = self.req_to_token[
|
||||
req_pool_indices[:, None],
|
||||
self.draft_extend_metadata["strided_indices"][:max_seq_pages],
|
||||
]
|
||||
if self.use_sliding_window_kv_pool and metadata.swa_page_table is not None:
|
||||
swa_page_indices = self.token_to_kv_pool.translate_loc_from_full_to_swa(
|
||||
page_indices
|
||||
if self.kv_index_translator.reads_are_translated:
|
||||
# The translator writes kernel-facing page ids straight into
|
||||
# the captured tables; `sliding_window_out` fills the swa twin
|
||||
# in the same pass.
|
||||
self.kv_index_translator.fill_read_table(
|
||||
out=metadata.page_table,
|
||||
sliding_window_out=(
|
||||
metadata.swa_page_table
|
||||
if self.use_sliding_window_kv_pool
|
||||
and metadata.swa_page_table is not None
|
||||
else None
|
||||
),
|
||||
req_pool_indices=req_pool_indices,
|
||||
seq_lens=seq_lens,
|
||||
seq_len_delta=self._spec_read_seq_len_delta(
|
||||
forward_mode, spec_info
|
||||
),
|
||||
)
|
||||
metadata.swa_page_table[:, :max_seq_pages].copy_(
|
||||
swa_page_indices // self.page_size
|
||||
else:
|
||||
page_indices = self.req_to_token[
|
||||
req_pool_indices[:, None],
|
||||
self.draft_extend_metadata["strided_indices"][:max_seq_pages],
|
||||
]
|
||||
if (
|
||||
self.use_sliding_window_kv_pool
|
||||
and metadata.swa_page_table is not None
|
||||
):
|
||||
swa_page_indices = (
|
||||
self.token_to_kv_pool.translate_loc_from_full_to_swa(
|
||||
page_indices
|
||||
)
|
||||
)
|
||||
metadata.swa_page_table[:, :max_seq_pages].copy_(
|
||||
swa_page_indices // self.page_size
|
||||
)
|
||||
metadata.page_table[:, :max_seq_pages].copy_(
|
||||
page_indices // self.page_size
|
||||
)
|
||||
metadata.page_table[:, :max_seq_pages].copy_(page_indices // self.page_size)
|
||||
|
||||
else:
|
||||
raise ValueError(
|
||||
|
||||
@@ -1840,10 +1840,6 @@ class FlashInferIndicesUpdaterPrefill:
|
||||
self.kv_indptr = attn_backend.kv_indptr
|
||||
self.kv_last_page_len = attn_backend.kv_last_page_len
|
||||
self.qo_indptr = attn_backend.qo_indptr
|
||||
# Kept ONLY for the spec-info branches (generate_attn_arg_prefill),
|
||||
# which are static-pool-only: unified memory asserts spec off. The
|
||||
# normal builders source from the per-batch KVIndexTable.
|
||||
self.req_to_token = model_runner.req_to_token_pool.req_to_token
|
||||
self._swa_kv_pool = attn_backend._swa_kv_pool
|
||||
self.prefill_wrapper_ragged = attn_backend.prefill_wrapper_ragged
|
||||
|
||||
@@ -2202,20 +2198,22 @@ class FlashInferIndicesUpdaterPrefill:
|
||||
if spec_info.spec_input_type == SpecInputType.DFLASH_VERIFY:
|
||||
kv_indices, kv_indptr, qo_indptr, custom_mask = (
|
||||
spec_info.generate_attn_arg_prefill(
|
||||
req_pool_indices,
|
||||
paged_kernel_lens,
|
||||
paged_kernel_lens_sum,
|
||||
self.req_to_token,
|
||||
req_pool_indices=req_pool_indices,
|
||||
paged_kernel_lens=paged_kernel_lens,
|
||||
paged_kernel_lens_sum=paged_kernel_lens_sum,
|
||||
translator=translator,
|
||||
sliding_window=use_swa_source,
|
||||
kv_start_idx=kv_start_idx,
|
||||
)
|
||||
)
|
||||
else:
|
||||
kv_indices, kv_indptr, qo_indptr, custom_mask = (
|
||||
spec_info.generate_attn_arg_prefill(
|
||||
req_pool_indices,
|
||||
paged_kernel_lens,
|
||||
paged_kernel_lens_sum,
|
||||
self.req_to_token,
|
||||
req_pool_indices=req_pool_indices,
|
||||
paged_kernel_lens=paged_kernel_lens,
|
||||
paged_kernel_lens_sum=paged_kernel_lens_sum,
|
||||
translator=translator,
|
||||
sliding_window=use_swa_source,
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
@@ -992,10 +992,6 @@ class FlashInferMLAIndicesUpdaterPrefill:
|
||||
# Buffers and wrappers
|
||||
self.kv_indptr = attn_backend.kv_indptr
|
||||
self.qo_indptr = attn_backend.qo_indptr
|
||||
# Kept ONLY for the spec-info branch (generate_attn_arg_prefill), which
|
||||
# is static-pool-only: unified memory asserts spec off. The normal
|
||||
# builder reads req_to_token through the translator.
|
||||
self.req_to_token = model_runner.req_to_token_pool.req_to_token
|
||||
self.prefill_wrapper_ragged = attn_backend.prefill_wrapper_ragged
|
||||
|
||||
def update(
|
||||
@@ -1085,10 +1081,10 @@ class FlashInferMLAIndicesUpdaterPrefill:
|
||||
elif fast_verify_plan_kwargs is not None:
|
||||
kv_indices, kv_indptr, qo_indptr, custom_mask = (
|
||||
spec_info.generate_attn_arg_prefill(
|
||||
req_pool_indices,
|
||||
paged_kernel_lens,
|
||||
paged_kernel_lens_sum,
|
||||
self.req_to_token,
|
||||
req_pool_indices=req_pool_indices,
|
||||
paged_kernel_lens=paged_kernel_lens,
|
||||
paged_kernel_lens_sum=paged_kernel_lens_sum,
|
||||
translator=self.attn_backend.kv_index_translator,
|
||||
kv_indices_buf=fast_verify_plan_kwargs["kv_indices_buf"],
|
||||
)
|
||||
)
|
||||
@@ -1097,10 +1093,10 @@ class FlashInferMLAIndicesUpdaterPrefill:
|
||||
# TODO: Support topk > 1 with custom mask
|
||||
kv_indices, kv_indptr, qo_indptr, custom_mask = (
|
||||
spec_info.generate_attn_arg_prefill(
|
||||
req_pool_indices,
|
||||
paged_kernel_lens,
|
||||
paged_kernel_lens_sum,
|
||||
self.req_to_token,
|
||||
req_pool_indices=req_pool_indices,
|
||||
paged_kernel_lens=paged_kernel_lens,
|
||||
paged_kernel_lens_sum=paged_kernel_lens_sum,
|
||||
translator=self.attn_backend.kv_index_translator,
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
@@ -163,7 +163,7 @@ class FlashMLABackend(FlashInferMLAAttnBackend):
|
||||
if forward_batch.forward_mode.is_decode_or_idle():
|
||||
max_seqlen_pad = triton.cdiv(eager_max_k, PAGE_SIZE)
|
||||
block_kv_indices = self._eager_block_kv_indices(bs, max_seqlen_pad)
|
||||
if self.kv_index_translator.is_translating:
|
||||
if self.kv_index_translator.reads_are_translated:
|
||||
assert self.page_size == PAGE_SIZE
|
||||
self.kv_index_translator.fill_read_table(
|
||||
out=block_kv_indices,
|
||||
@@ -198,17 +198,25 @@ class FlashMLABackend(FlashInferMLAAttnBackend):
|
||||
|
||||
max_seqlen_pad = triton.cdiv(eager_max_k + self.num_draft_tokens, PAGE_SIZE)
|
||||
block_kv_indices = self._eager_block_kv_indices(bs, max_seqlen_pad)
|
||||
create_flashmla_kv_indices_triton[
|
||||
(bs, get_num_kv_index_blocks_flashmla(max_seqlen_pad, PAGE_SIZE))
|
||||
](
|
||||
self.req_to_token,
|
||||
forward_batch.req_pool_indices,
|
||||
seq_lens,
|
||||
None,
|
||||
block_kv_indices,
|
||||
self.req_to_token.stride(0),
|
||||
block_kv_indices.stride(0),
|
||||
)
|
||||
if self.kv_index_translator.reads_are_translated:
|
||||
assert self.page_size == PAGE_SIZE
|
||||
self.kv_index_translator.fill_read_table(
|
||||
out=block_kv_indices,
|
||||
req_pool_indices=forward_batch.req_pool_indices,
|
||||
seq_lens=seq_lens,
|
||||
)
|
||||
else:
|
||||
create_flashmla_kv_indices_triton[
|
||||
(bs, get_num_kv_index_blocks_flashmla(max_seqlen_pad, PAGE_SIZE))
|
||||
](
|
||||
self.req_to_token,
|
||||
forward_batch.req_pool_indices,
|
||||
seq_lens,
|
||||
None,
|
||||
block_kv_indices,
|
||||
self.req_to_token.stride(0),
|
||||
block_kv_indices.stride(0),
|
||||
)
|
||||
mla_metadata, num_splits = get_mla_metadata(
|
||||
seq_lens.to(torch.int32),
|
||||
self.num_draft_tokens * self.num_q_heads,
|
||||
@@ -230,17 +238,25 @@ class FlashMLABackend(FlashInferMLAAttnBackend):
|
||||
|
||||
max_seqlen_pad = triton.cdiv(eager_max_k + window, PAGE_SIZE)
|
||||
block_kv_indices = self._eager_block_kv_indices(bs, max_seqlen_pad)
|
||||
create_flashmla_kv_indices_triton[
|
||||
(bs, get_num_kv_index_blocks_flashmla(max_seqlen_pad, PAGE_SIZE))
|
||||
](
|
||||
self.req_to_token,
|
||||
forward_batch.req_pool_indices,
|
||||
seq_lens_k,
|
||||
None,
|
||||
block_kv_indices,
|
||||
self.req_to_token.stride(0),
|
||||
block_kv_indices.stride(0),
|
||||
)
|
||||
if self.kv_index_translator.reads_are_translated:
|
||||
assert self.page_size == PAGE_SIZE
|
||||
self.kv_index_translator.fill_read_table(
|
||||
out=block_kv_indices,
|
||||
req_pool_indices=forward_batch.req_pool_indices,
|
||||
seq_lens=seq_lens_k,
|
||||
)
|
||||
else:
|
||||
create_flashmla_kv_indices_triton[
|
||||
(bs, get_num_kv_index_blocks_flashmla(max_seqlen_pad, PAGE_SIZE))
|
||||
](
|
||||
self.req_to_token,
|
||||
forward_batch.req_pool_indices,
|
||||
seq_lens_k,
|
||||
None,
|
||||
block_kv_indices,
|
||||
self.req_to_token.stride(0),
|
||||
block_kv_indices.stride(0),
|
||||
)
|
||||
mla_metadata, num_splits = get_mla_metadata(
|
||||
seq_lens_k,
|
||||
window * self.num_q_heads,
|
||||
@@ -337,7 +353,7 @@ class FlashMLABackend(FlashInferMLAAttnBackend):
|
||||
else:
|
||||
max_seqlen_pad = self.cuda_graph_kv_indices.shape[1]
|
||||
|
||||
if self.kv_index_translator.is_translating:
|
||||
if self.kv_index_translator.reads_are_translated:
|
||||
assert self.page_size == PAGE_SIZE
|
||||
self.kv_index_translator.fill_read_table(
|
||||
out=self.cuda_graph_kv_indices,
|
||||
|
||||
@@ -257,6 +257,7 @@ class KVIndexTranslator:
|
||||
seq_lens: torch.Tensor,
|
||||
max_pages: Optional[int] = None,
|
||||
into: Optional[KVReadTables] = None,
|
||||
seq_len_delta: int = 0,
|
||||
) -> KVIndexTable:
|
||||
"""The one per-batch entry point.
|
||||
|
||||
@@ -265,6 +266,10 @@ class KVIndexTranslator:
|
||||
returns the table WHOLE, so a caller needing a stable pointer (a
|
||||
captured graph bakes it) passes its own tables in ``into``;
|
||||
``into=None`` allocates of width ``max_pages`` instead.
|
||||
|
||||
``seq_len_delta`` widens every row's live prefix -- the whole-sequence
|
||||
verify contract (draft KV read back from the pool). An eager caller's
|
||||
``max_pages`` must already cover the delta.
|
||||
"""
|
||||
if not self.is_translating or self.defer_read_translate:
|
||||
return KVIndexTable(
|
||||
@@ -308,6 +313,7 @@ class KVIndexTranslator:
|
||||
page_size=self.page_size,
|
||||
max_pages=width,
|
||||
out=out_full,
|
||||
seq_len_delta=seq_len_delta,
|
||||
)
|
||||
if out_swa is not None:
|
||||
build_kv_read_table(
|
||||
@@ -318,6 +324,7 @@ class KVIndexTranslator:
|
||||
page_size=self.page_size,
|
||||
max_pages=width,
|
||||
out=out_swa,
|
||||
seq_len_delta=seq_len_delta,
|
||||
)
|
||||
return KVIndexTable(
|
||||
ids=out_full,
|
||||
@@ -335,6 +342,7 @@ class KVIndexTranslator:
|
||||
req_pool_indices: torch.Tensor,
|
||||
seq_lens: torch.Tensor,
|
||||
sliding_window_out: Optional[torch.Tensor] = None,
|
||||
seq_len_delta: int = 0,
|
||||
) -> None:
|
||||
"""`build_index_table(into=...)` for a caller that owns a bare block
|
||||
table rather than a KVReadTables: the page-table consumers read that
|
||||
@@ -359,6 +367,7 @@ class KVIndexTranslator:
|
||||
req_pool_indices=req_pool_indices,
|
||||
seq_lens=seq_lens,
|
||||
into=KVReadTables(full=out, sliding_window=sliding_window_out),
|
||||
seq_len_delta=seq_len_delta,
|
||||
)
|
||||
|
||||
def index_table_for_batch(self, forward_batch) -> KVIndexTable:
|
||||
@@ -404,6 +413,38 @@ class KVIndexTranslator:
|
||||
"""
|
||||
return self._full_v2p_table
|
||||
|
||||
def widened_index_table(self, forward_batch, *, seq_len_delta: int) -> KVIndexTable:
|
||||
"""Whole-sequence-verify view: every row's live prefix widened by
|
||||
`seq_len_delta` columns (the drafts are read back from the pool).
|
||||
Not memoized -- verify is one consumer, and the batch's memoized
|
||||
prefix table stays valid for the others.
|
||||
|
||||
The width comes from the LIVE prefix: a DSPARK/DFLASH verify batch
|
||||
carries `seq_lens_cpu` already expanded by the window and keeps the
|
||||
live lens on `spec_info.live_seq_lens_cpu`, so widening the expanded
|
||||
lens would count the window twice. The width never exceeds the
|
||||
`req_to_token` row it is read from."""
|
||||
max_pages = None
|
||||
if self.is_translating:
|
||||
row_pages = -(-self.req_to_token.shape[1] // self.page_size)
|
||||
live = getattr(forward_batch.spec_info, "live_seq_lens_cpu", None)
|
||||
slc = live if live is not None else forward_batch.seq_lens_cpu
|
||||
if (
|
||||
forward_batch.seq_lens_sum is not None
|
||||
and slc is not None
|
||||
and slc.numel() > 0
|
||||
):
|
||||
max_seq = int(slc.max()) + seq_len_delta
|
||||
max_pages = min(max(-(-max_seq // self.page_size), 1), row_pages)
|
||||
else:
|
||||
max_pages = row_pages
|
||||
return self.build_index_table(
|
||||
req_pool_indices=forward_batch.req_pool_indices,
|
||||
seq_lens=forward_batch.seq_lens,
|
||||
max_pages=max_pages,
|
||||
seq_len_delta=seq_len_delta,
|
||||
)
|
||||
|
||||
def bind_and_verify_backends(self, backends) -> None:
|
||||
"""Boot: make every reachable backend carry THIS translator.
|
||||
|
||||
|
||||
@@ -349,6 +349,7 @@ class FalconH1HybridAttentionDecoderLayer(nn.Module):
|
||||
layer_id=self.layer_id,
|
||||
forward_batch=forward_batch,
|
||||
mup_vector=self.mup_vector,
|
||||
use_triton_causal_conv=forward_batch.forward_mode.is_target_verify(),
|
||||
)
|
||||
mamba_hidden_states = mamba_hidden_states * self.ssm_out_multiplier
|
||||
|
||||
|
||||
@@ -5,8 +5,8 @@ from typing import TYPE_CHECKING, Optional
|
||||
|
||||
import torch
|
||||
|
||||
from sglang.kernels.ops.attention.utils import create_flashinfer_kv_indices_triton
|
||||
from sglang.srt.managers.schedule_batch import ScheduleBatch
|
||||
from sglang.srt.mem_cache.kv_index_translator import KVIndexTranslator
|
||||
from sglang.srt.model_executor.forward_batch_info import (
|
||||
CaptureHiddenMode,
|
||||
ForwardBatch,
|
||||
@@ -128,15 +128,20 @@ class DFlashVerifyInput(SpecInput):
|
||||
|
||||
def generate_attn_arg_prefill(
|
||||
self,
|
||||
*,
|
||||
req_pool_indices: torch.Tensor,
|
||||
paged_kernel_lens: torch.Tensor,
|
||||
paged_kernel_lens_sum: int,
|
||||
req_to_token: torch.Tensor,
|
||||
translator: KVIndexTranslator,
|
||||
kv_start_idx: Optional[torch.Tensor] = None,
|
||||
kv_indices_buf: Optional[torch.Tensor] = None,
|
||||
sliding_window: bool = False,
|
||||
):
|
||||
"""CSR verify args, gathered straight into the packed stream. The lens
|
||||
are widened here and handed to the translator, so nothing materializes
|
||||
a ``[bs, max_pages]`` rectangle to repack from."""
|
||||
device = req_pool_indices.device
|
||||
bs = len(req_pool_indices)
|
||||
bs = req_pool_indices.numel()
|
||||
|
||||
layout = self.ragged_verify_layout
|
||||
if layout is not None and layout.bs != bs:
|
||||
@@ -174,14 +179,14 @@ class DFlashVerifyInput(SpecInput):
|
||||
dtype=torch.int32,
|
||||
device=device,
|
||||
)
|
||||
create_flashinfer_kv_indices_triton[(bs,)](
|
||||
req_to_token,
|
||||
req_pool_indices,
|
||||
paged_kernel_lens,
|
||||
cum_kv_seq_len,
|
||||
kv_start_idx,
|
||||
kv_indices,
|
||||
req_to_token.size(1),
|
||||
translator.fill_packed_read_stream(
|
||||
req_pool_indices=req_pool_indices,
|
||||
seq_lens=paged_kernel_lens,
|
||||
indptr=cum_kv_seq_len,
|
||||
total_tokens=paged_kernel_lens_sum + kv_indices_extra,
|
||||
out=kv_indices,
|
||||
kv_start_idx=kv_start_idx,
|
||||
sliding_window=sliding_window,
|
||||
)
|
||||
mask = self.custom_mask
|
||||
if mask is not None:
|
||||
|
||||
@@ -4,7 +4,7 @@ from typing import Callable, List, Optional
|
||||
|
||||
import torch
|
||||
|
||||
from sglang.kernels.ops.attention.utils import create_flashinfer_kv_indices_triton
|
||||
from sglang.srt.mem_cache.kv_index_translator import KVIndexTranslator
|
||||
from sglang.srt.model_executor.forward_batch_info import CaptureHiddenMode
|
||||
from sglang.srt.runtime_context import get_spec
|
||||
from sglang.srt.speculative.spec_info import SpecInput, SpecInputType
|
||||
@@ -85,13 +85,18 @@ class EagleVerifyInput(SpecInput):
|
||||
|
||||
def generate_attn_arg_prefill(
|
||||
self,
|
||||
*,
|
||||
req_pool_indices: torch.Tensor,
|
||||
paged_kernel_lens: torch.Tensor,
|
||||
paged_kernel_lens_sum: int,
|
||||
req_to_token: torch.Tensor,
|
||||
translator: KVIndexTranslator,
|
||||
sliding_window: bool = False,
|
||||
):
|
||||
"""CSR verify args, gathered straight into the packed stream. The lens
|
||||
are widened here and handed to the translator, so nothing materializes
|
||||
a ``[bs, max_pages]`` rectangle to repack from."""
|
||||
device = req_pool_indices.device
|
||||
batch_size = len(req_pool_indices)
|
||||
batch_size = req_pool_indices.numel()
|
||||
qo_indptr = torch.arange(
|
||||
0,
|
||||
(1 + batch_size) * self.draft_token_num,
|
||||
@@ -106,19 +111,15 @@ class EagleVerifyInput(SpecInput):
|
||||
paged_kernel_lens = paged_kernel_lens + self.draft_token_num
|
||||
cum_kv_seq_len[1:] = torch.cumsum(paged_kernel_lens, dim=0)
|
||||
|
||||
kv_indices = torch.empty(
|
||||
paged_kernel_lens_sum + self.draft_token_num * batch_size,
|
||||
dtype=torch.int32,
|
||||
device=device,
|
||||
)
|
||||
create_flashinfer_kv_indices_triton[(batch_size,)](
|
||||
req_to_token,
|
||||
req_pool_indices,
|
||||
paged_kernel_lens,
|
||||
cum_kv_seq_len,
|
||||
None,
|
||||
kv_indices,
|
||||
req_to_token.size(1),
|
||||
total_tokens = paged_kernel_lens_sum + self.draft_token_num * batch_size
|
||||
kv_indices = torch.empty(total_tokens, dtype=torch.int32, device=device)
|
||||
translator.fill_packed_read_stream(
|
||||
req_pool_indices=req_pool_indices,
|
||||
seq_lens=paged_kernel_lens,
|
||||
indptr=cum_kv_seq_len,
|
||||
total_tokens=total_tokens,
|
||||
out=kv_indices,
|
||||
sliding_window=sliding_window,
|
||||
)
|
||||
mask_numel = (
|
||||
paged_kernel_lens_sum * self.draft_token_num
|
||||
@@ -396,11 +397,15 @@ class EagleDraftExtendInput(SpecInput):
|
||||
|
||||
def generate_attn_arg_prefill(
|
||||
self,
|
||||
*,
|
||||
req_pool_indices: torch.Tensor,
|
||||
paged_kernel_lens: torch.Tensor,
|
||||
paged_kernel_lens_sum: Optional[int],
|
||||
req_to_token: torch.Tensor,
|
||||
translator: KVIndexTranslator,
|
||||
sliding_window: bool = False,
|
||||
):
|
||||
"""Draft-extend CSR args. The lens already include the window, so
|
||||
unlike verify nothing is widened here."""
|
||||
device = req_pool_indices.device
|
||||
bs = self.num_correct_drafts.numel()
|
||||
# Constant num_tokens_per_req qo layout (required for cuda-graph capture).
|
||||
@@ -421,13 +426,12 @@ class EagleDraftExtendInput(SpecInput):
|
||||
paged_kernel_lens_sum, dtype=torch.int32, device=device
|
||||
)
|
||||
|
||||
create_flashinfer_kv_indices_triton[(bs,)](
|
||||
req_to_token,
|
||||
req_pool_indices,
|
||||
paged_kernel_lens,
|
||||
cum_kv_seq_len,
|
||||
None,
|
||||
kv_indices,
|
||||
req_to_token.size(1),
|
||||
translator.fill_packed_read_stream(
|
||||
req_pool_indices=req_pool_indices,
|
||||
seq_lens=paged_kernel_lens,
|
||||
indptr=cum_kv_seq_len,
|
||||
total_tokens=paged_kernel_lens_sum,
|
||||
out=kv_indices,
|
||||
sliding_window=sliding_window,
|
||||
)
|
||||
return kv_indices, cum_kv_seq_len, qo_indptr, None
|
||||
|
||||
@@ -4,7 +4,7 @@ from typing import List, Optional
|
||||
|
||||
import torch
|
||||
|
||||
from sglang.kernels.ops.attention.utils import create_flashinfer_kv_indices_triton
|
||||
from sglang.srt.mem_cache.kv_index_translator import KVIndexTranslator
|
||||
from sglang.srt.speculative.spec_info import SpecInput, SpecInputType
|
||||
|
||||
|
||||
@@ -58,12 +58,17 @@ class NgramVerifyInput(SpecInput):
|
||||
|
||||
def generate_attn_arg_prefill(
|
||||
self,
|
||||
*,
|
||||
req_pool_indices: torch.Tensor,
|
||||
paged_kernel_lens: torch.Tensor,
|
||||
paged_kernel_lens_sum: int,
|
||||
req_to_token: torch.Tensor,
|
||||
translator: KVIndexTranslator,
|
||||
sliding_window: bool = False,
|
||||
):
|
||||
bs = len(req_pool_indices)
|
||||
"""CSR verify args, gathered straight into the packed stream. The lens
|
||||
are widened here and handed to the translator, so nothing materializes
|
||||
a ``[bs, max_pages]`` rectangle to repack from."""
|
||||
bs = req_pool_indices.numel()
|
||||
|
||||
cum_kv_seq_len = torch.zeros((bs + 1,), dtype=torch.int32, device=self.device)
|
||||
|
||||
@@ -75,20 +80,16 @@ class NgramVerifyInput(SpecInput):
|
||||
* self.draft_token_num
|
||||
)
|
||||
|
||||
kv_indices = torch.empty(
|
||||
paged_kernel_lens_sum + self.draft_token_num * bs,
|
||||
dtype=torch.int32,
|
||||
device=self.device,
|
||||
)
|
||||
total_tokens = paged_kernel_lens_sum + self.draft_token_num * bs
|
||||
kv_indices = torch.empty(total_tokens, dtype=torch.int32, device=self.device)
|
||||
|
||||
create_flashinfer_kv_indices_triton[(bs,)](
|
||||
req_to_token,
|
||||
req_pool_indices,
|
||||
paged_kernel_lens,
|
||||
cum_kv_seq_len,
|
||||
None,
|
||||
kv_indices,
|
||||
req_to_token.size(1),
|
||||
translator.fill_packed_read_stream(
|
||||
req_pool_indices=req_pool_indices,
|
||||
seq_lens=paged_kernel_lens,
|
||||
indptr=cum_kv_seq_len,
|
||||
total_tokens=total_tokens,
|
||||
out=kv_indices,
|
||||
sliding_window=sliding_window,
|
||||
)
|
||||
|
||||
# Pad custom_mask when CUDA graph pads batch size beyond the actual number of requests.
|
||||
|
||||
Reference in New Issue
Block a user