[Perf] attention: Capture translated FA4 verify metadata for DSPARK (#44973)

This commit is contained in:
Cheng Wan
2026-10-18 05:42:16 -07:00
committed by GitHub
parent 2e9b2cacda
commit 57db5089c2
8 changed files with 293 additions and 50 deletions
+52 -16
View File
@@ -164,15 +164,27 @@ def _resolve_metadata_page_ids(
token_ids,
mask,
PAGE_SIZE: tl.constexpr,
page_mapping,
page_mapping_stride,
token_mapping=None,
token_mapping_stride=0,
):
# Static SWA maps token slots. An absent mapping compiles to an identity.
# Static SWA maps token slots; unified pools map pages. Callers supply
# the appropriate mapping, and absent mappings compile to an identity.
if token_mapping is not None:
token_ids = tl.load(
token_mapping + token_ids * token_mapping_stride, mask=mask, other=0
)
page_ids = token_ids // PAGE_SIZE
if page_mapping is not None:
page_ids = tl.maximum(
tl.load(
page_mapping + tl.maximum(page_ids, 0) * page_mapping_stride,
mask=mask,
other=0,
),
0,
)
return page_ids
@@ -197,12 +209,17 @@ def _fused_decode_metadata_kernel(
swa_page_table_stride_1,
full_to_swa_index_mapping,
full_to_swa_mapping_stride_0,
full_v2p,
full_v2p_stride_0,
swa_v2p,
swa_v2p_stride_0,
B,
max_seq_pages,
PAGE_SIZE: tl.constexpr,
SEQ_LEN_DELTA: tl.constexpr,
BLOCK_COLS: tl.constexpr,
SKIP_PAGE_TABLE: tl.constexpr,
ZERO_TAIL: tl.constexpr,
):
pid_b = tl.program_id(0)
pid_c = tl.program_id(1)
@@ -228,7 +245,7 @@ def _fused_decode_metadata_kernel(
tl.cdiv(seq_len + SEQ_LEN_DELTA, PAGE_SIZE), max_seq_pages
)
col_start = pid_c * BLOCK_COLS
if col_start >= num_live_pages:
if not ZERO_TAIL and col_start >= num_live_pages:
return
row_idx = tl.load(req_pool_indices + pid_b * req_pool_indices_stride_0)
@@ -242,11 +259,14 @@ def _fused_decode_metadata_kernel(
other=0,
cache_modifier=".cg",
)
full_pages = _resolve_metadata_page_ids(token_ids, live_mask, PAGE_SIZE)
store_mask = cols < max_seq_pages if ZERO_TAIL else live_mask
full_pages = _resolve_metadata_page_ids(
token_ids, live_mask, PAGE_SIZE, full_v2p, full_v2p_stride_0
)
tl.store(
page_table + pid_b * page_table_stride_0 + cols * page_table_stride_1,
full_pages,
mask=live_mask,
mask=store_mask,
cache_modifier=".cg",
)
if swa_page_table is not None:
@@ -254,6 +274,8 @@ def _fused_decode_metadata_kernel(
token_ids,
live_mask,
PAGE_SIZE,
swa_v2p,
swa_v2p_stride_0,
full_to_swa_index_mapping,
full_to_swa_mapping_stride_0,
)
@@ -262,7 +284,7 @@ def _fused_decode_metadata_kernel(
+ pid_b * swa_page_table_stride_0
+ cols * swa_page_table_stride_1,
swa_pages,
mask=live_mask,
mask=store_mask,
cache_modifier=".cg",
)
@@ -490,30 +512,39 @@ def normal_decode_set_metadata(
swa_page_table: Optional[torch.Tensor] = None,
full_to_swa_index_mapping: Optional[torch.Tensor] = None,
skip_page_table: bool = False,
full_v2p: Optional[torch.Tensor] = None,
swa_v2p: Optional[torch.Tensor] = None,
zero_tail: bool = False,
):
"""Fill sequence lengths, their prefix sum and page tables in one launch.
``req_to_token`` contains token slots. Static SWA uses
``full_to_swa_index_mapping`` to map FULL token slots to SWA token slots.
Runtime callers resolve this tensor from their pool before calling.
``req_to_token`` contains token slots. ``full_v2p`` / ``swa_v2p`` map
virtual pages to physical pages; an unbound (-1) page maps to the sink.
Static SWA instead uses ``full_to_swa_index_mapping`` to map token slots.
Supply exactly one SWA mapping when writing ``swa_page_table``. Mappings
are read on device on every invocation, including CUDA graph replay.
Only the live prefix (ceil((seq_len + delta) / page_size)) is rewritten;
consumers must bound reads by ``cache_seqlens_int32``. For pretranslated
tables, ``skip_page_table=True`` leaves both page tables untouched and
computes only lengths and the prefix sum.
Only the live prefix (ceil((seq_len + delta) / page_size)) is rewritten
by default; consumers must bound reads by ``cache_seqlens_int32``.
``zero_tail=True`` also clears columns up to ``max_seq_pages`` to zero,
independently of address translation. ``skip_page_table=True`` leaves
both page tables untouched and computes only lengths and the prefix sum.
"""
assert page_size > 0 and (page_size & (page_size - 1)) == 0, (
f"page_size must be a power of two, got {page_size}"
)
if swa_page_table is not None and not skip_page_table:
assert full_to_swa_index_mapping is not None, (
"SWA output requires a token-slot mapping"
assert (full_to_swa_index_mapping is None) != (swa_v2p is None), (
"SWA output requires exactly one token-slot or page mapping"
)
batch_size = cache_seqlens_int32.shape[0]
# Lengths-only/token gathers use 256 columns; page/SWA gathers use 128.
# Lengths-only/unmapped token gathers use 256 columns, other gathers 128.
block_cols = (
256 if skip_page_table or (page_size == 1 and swa_page_table is None) else 128
256
if skip_page_table
or (page_size == 1 and swa_page_table is None and full_v2p is None)
else 128
)
grid = (
(1, 1)
@@ -542,12 +573,17 @@ def normal_decode_set_metadata(
full_to_swa_index_mapping.stride(0)
if full_to_swa_index_mapping is not None
else 0,
full_v2p,
full_v2p.stride(0) if full_v2p is not None else 0,
swa_v2p,
swa_v2p.stride(0) if swa_v2p is not None else 0,
batch_size,
max_seq_pages,
PAGE_SIZE=page_size,
SEQ_LEN_DELTA=seq_len_delta,
BLOCK_COLS=block_cols,
SKIP_PAGE_TABLE=skip_page_table,
ZERO_TAIL=zero_tail,
num_warps=block_cols // 32,
num_stages=3,
)
+14 -6
View File
@@ -488,10 +488,14 @@ _SPEC_VERIFY_AUDITED_BACKENDS = frozenset(
}
)
_SPEC_MHA_AUDITED_BACKENDS = frozenset({"triton", "flashinfer", "fa3", "trtllm_mha"})
# The MHA backends that read through the KV-index translator on every path a
# draft forward takes. A fused draft region holds dense K/V rows, so a fused
# draft runs only on these.
TRANSLATED_MHA_RAILS = frozenset({"triton", "flashinfer", "fa3", "trtllm_mha"})
# FA4 carries translated draft ids, but speculative admission is audited
# per algorithm below; currently only DSPARK is validated with FA4.
TRANSLATED_MHA_RAILS = _SPEC_MHA_AUDITED_BACKENDS | {"fa4"}
def _assert_spec_verify_backends(
@@ -613,7 +617,7 @@ def handle_unified_memory_pool(server_args: Any) -> None:
eagle_allowed = (
_SPEC_VERIFY_AUDITED_BACKENDS
if use_mla_backend(server_args)
else TRANSLATED_MHA_RAILS
else _SPEC_MHA_AUDITED_BACKENDS
)
eagle_backends = set(attention_backends_of(resolved_view(server_args)))
assert (
@@ -628,20 +632,24 @@ def handle_unified_memory_pool(server_args: Any) -> None:
# An unset draft backend inherits the target's pair.
draft_backend = cfg.speculative_draft_attention_backend
draft_backends = {draft_backend} if draft_backend else eagle_backends
assert draft_backends <= TRANSLATED_MHA_RAILS, (
assert draft_backends <= _SPEC_MHA_AUDITED_BACKENDS, (
"--enable-unified-memory + EAGLE/EAGLE3 runs the draft on "
f"{sorted(draft_backends)}, but a fused draft is MHA-shaped and "
"reads its KV through the translated MHA rails "
f"{sorted(TRANSLATED_MHA_RAILS)}. "
f"{sorted(_SPEC_MHA_AUDITED_BACKENDS)}. "
"Set --speculative-draft-attention-backend to one of them."
)
if cfg.speculative_algorithm == "DSPARK":
_assert_spec_verify_backends(server_args, algorithm="DSPARK")
_assert_spec_verify_backends(
server_args,
algorithm="DSPARK",
allowed=_SPEC_VERIFY_AUDITED_BACKENDS | {"fa4"},
)
if cfg.speculative_algorithm == "DFLASH":
_assert_spec_verify_backends(
server_args,
algorithm="DFLASH",
allowed=TRANSLATED_MHA_RAILS,
allowed=_SPEC_MHA_AUDITED_BACKENDS,
)
assert not cfg.enable_two_batch_overlap, (
"--enable-unified-memory does not support --enable-two-batch-overlap: "
@@ -463,7 +463,39 @@ class FlashAttentionBackend(AttentionBackend):
k_sf, v_sf = self.token_to_kv_pool.get_kv_scale_buffer(layer.layer_id)
return {"sfq": q_descale, "sfk": k_sf, "sfv": v_sf}
def _capture_unified_verify_metadata(self, forward_batch: ForwardBatch) -> bool:
# Fixed-width verify uses only capture-stable request/length buffers.
# Ragged layouts and encoder offsets still use their eager builders.
return (
forward_batch.forward_mode.is_target_verify()
and self.fa_impl_ver == 4
and self.kv_index_translator.reads_are_translated
and self.topk <= 1
and forward_batch.encoder_lens is None
and getattr(forward_batch.spec_info, "ragged_verify_layout", None) is None
)
def init_forward_metadata_in_graph(self, forward_batch: ForwardBatch) -> None:
if self._capture_unified_verify_metadata(forward_batch):
metadata = self.target_verify_metadata[forward_batch.batch_size]
translator = self.kv_index_translator
swa = translator.space(IdSpaceKind.SLIDING_WINDOW)
normal_decode_set_metadata(
metadata.cache_seqlens_int32,
metadata.cu_seqlens_k,
metadata.page_table,
self.req_to_token,
forward_batch.req_pool_indices,
self.max_num_pages,
forward_batch.seq_lens,
self.speculative_num_draft_tokens,
self.page_size,
metadata.swa_page_table,
full_v2p=translator.space(IdSpaceKind.FULL).read_v2p,
swa_v2p=None if swa is None else swa.read_v2p,
zero_tail=True,
)
return
# Single-CG has no Python between steps, so one capturable kernel updates
# the persistent metadata.
if not forward_batch.forward_mode.is_draft_extend_v2():
@@ -594,6 +626,23 @@ class FlashAttentionBackend(AttentionBackend):
else:
self.cuda_graph_swa_out_cache_loc[:n].copy_(swa_out_cache_loc)
if self._capture_unified_verify_metadata(forward_batch):
if in_capture:
self._bind_metadata_buffers(
bs,
forward_batch.positions.numel(),
encoder_lens,
forward_mode,
spec_info,
seq_lens.device,
)
self.forward_metadata = self.target_verify_metadata[bs]
self.forward_metadata_spec_decode_expand = None
# The runner records the device-side update at the start of the
# model graph. Do not populate a live plan's read-table cache with
# this graph-owned buffer: another bucket can overwrite it later.
return
if in_capture:
num_tokens = forward_batch.positions.numel()
seq_lens_cpu = seq_lens.cpu()
@@ -1258,16 +1258,11 @@ class DecodeCudaGraphRunner(BaseCudaGraphRunner):
attn_backend.init_forward_metadata_out_graph(forward_batch, in_capture=True)
def run_once():
# Graph-recordable metadata-prep hook. The unified memory pool
# records ZERO translate nodes here: all its read/write translates
# run eagerly in `init_forward_metadata_out_graph` (replay-prep), so
# the captured graph reads already-physical locs. Base no-op for triton.
# Backends may gather live pool mappings into their static
# metadata here; these nodes run before any model KV reads.
attn_backend.init_forward_metadata_in_graph(forward_batch)
self._record_in_graph_metadata_prep_done()
# No invalidate_loc_cache() here: the unified pool translates its
# locs in `init_forward_metadata_out_graph`, so no cache to invalidate.
forward_batch.dp_local_start_pos = forward_batch.dp_local_num_tokens = (
None
)