[Perf] attention: Capture translated FA4 verify metadata for DSPARK (#44973)
This commit is contained in:
@@ -164,15 +164,27 @@ def _resolve_metadata_page_ids(
|
||||
token_ids,
|
||||
mask,
|
||||
PAGE_SIZE: tl.constexpr,
|
||||
page_mapping,
|
||||
page_mapping_stride,
|
||||
token_mapping=None,
|
||||
token_mapping_stride=0,
|
||||
):
|
||||
# Static SWA maps token slots. An absent mapping compiles to an identity.
|
||||
# Static SWA maps token slots; unified pools map pages. Callers supply
|
||||
# the appropriate mapping, and absent mappings compile to an identity.
|
||||
if token_mapping is not None:
|
||||
token_ids = tl.load(
|
||||
token_mapping + token_ids * token_mapping_stride, mask=mask, other=0
|
||||
)
|
||||
page_ids = token_ids // PAGE_SIZE
|
||||
if page_mapping is not None:
|
||||
page_ids = tl.maximum(
|
||||
tl.load(
|
||||
page_mapping + tl.maximum(page_ids, 0) * page_mapping_stride,
|
||||
mask=mask,
|
||||
other=0,
|
||||
),
|
||||
0,
|
||||
)
|
||||
return page_ids
|
||||
|
||||
|
||||
@@ -197,12 +209,17 @@ def _fused_decode_metadata_kernel(
|
||||
swa_page_table_stride_1,
|
||||
full_to_swa_index_mapping,
|
||||
full_to_swa_mapping_stride_0,
|
||||
full_v2p,
|
||||
full_v2p_stride_0,
|
||||
swa_v2p,
|
||||
swa_v2p_stride_0,
|
||||
B,
|
||||
max_seq_pages,
|
||||
PAGE_SIZE: tl.constexpr,
|
||||
SEQ_LEN_DELTA: tl.constexpr,
|
||||
BLOCK_COLS: tl.constexpr,
|
||||
SKIP_PAGE_TABLE: tl.constexpr,
|
||||
ZERO_TAIL: tl.constexpr,
|
||||
):
|
||||
pid_b = tl.program_id(0)
|
||||
pid_c = tl.program_id(1)
|
||||
@@ -228,7 +245,7 @@ def _fused_decode_metadata_kernel(
|
||||
tl.cdiv(seq_len + SEQ_LEN_DELTA, PAGE_SIZE), max_seq_pages
|
||||
)
|
||||
col_start = pid_c * BLOCK_COLS
|
||||
if col_start >= num_live_pages:
|
||||
if not ZERO_TAIL and col_start >= num_live_pages:
|
||||
return
|
||||
|
||||
row_idx = tl.load(req_pool_indices + pid_b * req_pool_indices_stride_0)
|
||||
@@ -242,11 +259,14 @@ def _fused_decode_metadata_kernel(
|
||||
other=0,
|
||||
cache_modifier=".cg",
|
||||
)
|
||||
full_pages = _resolve_metadata_page_ids(token_ids, live_mask, PAGE_SIZE)
|
||||
store_mask = cols < max_seq_pages if ZERO_TAIL else live_mask
|
||||
full_pages = _resolve_metadata_page_ids(
|
||||
token_ids, live_mask, PAGE_SIZE, full_v2p, full_v2p_stride_0
|
||||
)
|
||||
tl.store(
|
||||
page_table + pid_b * page_table_stride_0 + cols * page_table_stride_1,
|
||||
full_pages,
|
||||
mask=live_mask,
|
||||
mask=store_mask,
|
||||
cache_modifier=".cg",
|
||||
)
|
||||
if swa_page_table is not None:
|
||||
@@ -254,6 +274,8 @@ def _fused_decode_metadata_kernel(
|
||||
token_ids,
|
||||
live_mask,
|
||||
PAGE_SIZE,
|
||||
swa_v2p,
|
||||
swa_v2p_stride_0,
|
||||
full_to_swa_index_mapping,
|
||||
full_to_swa_mapping_stride_0,
|
||||
)
|
||||
@@ -262,7 +284,7 @@ def _fused_decode_metadata_kernel(
|
||||
+ pid_b * swa_page_table_stride_0
|
||||
+ cols * swa_page_table_stride_1,
|
||||
swa_pages,
|
||||
mask=live_mask,
|
||||
mask=store_mask,
|
||||
cache_modifier=".cg",
|
||||
)
|
||||
|
||||
@@ -490,30 +512,39 @@ def normal_decode_set_metadata(
|
||||
swa_page_table: Optional[torch.Tensor] = None,
|
||||
full_to_swa_index_mapping: Optional[torch.Tensor] = None,
|
||||
skip_page_table: bool = False,
|
||||
full_v2p: Optional[torch.Tensor] = None,
|
||||
swa_v2p: Optional[torch.Tensor] = None,
|
||||
zero_tail: bool = False,
|
||||
):
|
||||
"""Fill sequence lengths, their prefix sum and page tables in one launch.
|
||||
|
||||
``req_to_token`` contains token slots. Static SWA uses
|
||||
``full_to_swa_index_mapping`` to map FULL token slots to SWA token slots.
|
||||
Runtime callers resolve this tensor from their pool before calling.
|
||||
``req_to_token`` contains token slots. ``full_v2p`` / ``swa_v2p`` map
|
||||
virtual pages to physical pages; an unbound (-1) page maps to the sink.
|
||||
Static SWA instead uses ``full_to_swa_index_mapping`` to map token slots.
|
||||
Supply exactly one SWA mapping when writing ``swa_page_table``. Mappings
|
||||
are read on device on every invocation, including CUDA graph replay.
|
||||
|
||||
Only the live prefix (ceil((seq_len + delta) / page_size)) is rewritten;
|
||||
consumers must bound reads by ``cache_seqlens_int32``. For pretranslated
|
||||
tables, ``skip_page_table=True`` leaves both page tables untouched and
|
||||
computes only lengths and the prefix sum.
|
||||
Only the live prefix (ceil((seq_len + delta) / page_size)) is rewritten
|
||||
by default; consumers must bound reads by ``cache_seqlens_int32``.
|
||||
``zero_tail=True`` also clears columns up to ``max_seq_pages`` to zero,
|
||||
independently of address translation. ``skip_page_table=True`` leaves
|
||||
both page tables untouched and computes only lengths and the prefix sum.
|
||||
"""
|
||||
assert page_size > 0 and (page_size & (page_size - 1)) == 0, (
|
||||
f"page_size must be a power of two, got {page_size}"
|
||||
)
|
||||
if swa_page_table is not None and not skip_page_table:
|
||||
assert full_to_swa_index_mapping is not None, (
|
||||
"SWA output requires a token-slot mapping"
|
||||
assert (full_to_swa_index_mapping is None) != (swa_v2p is None), (
|
||||
"SWA output requires exactly one token-slot or page mapping"
|
||||
)
|
||||
|
||||
batch_size = cache_seqlens_int32.shape[0]
|
||||
# Lengths-only/token gathers use 256 columns; page/SWA gathers use 128.
|
||||
# Lengths-only/unmapped token gathers use 256 columns, other gathers 128.
|
||||
block_cols = (
|
||||
256 if skip_page_table or (page_size == 1 and swa_page_table is None) else 128
|
||||
256
|
||||
if skip_page_table
|
||||
or (page_size == 1 and swa_page_table is None and full_v2p is None)
|
||||
else 128
|
||||
)
|
||||
grid = (
|
||||
(1, 1)
|
||||
@@ -542,12 +573,17 @@ def normal_decode_set_metadata(
|
||||
full_to_swa_index_mapping.stride(0)
|
||||
if full_to_swa_index_mapping is not None
|
||||
else 0,
|
||||
full_v2p,
|
||||
full_v2p.stride(0) if full_v2p is not None else 0,
|
||||
swa_v2p,
|
||||
swa_v2p.stride(0) if swa_v2p is not None else 0,
|
||||
batch_size,
|
||||
max_seq_pages,
|
||||
PAGE_SIZE=page_size,
|
||||
SEQ_LEN_DELTA=seq_len_delta,
|
||||
BLOCK_COLS=block_cols,
|
||||
SKIP_PAGE_TABLE=skip_page_table,
|
||||
ZERO_TAIL=zero_tail,
|
||||
num_warps=block_cols // 32,
|
||||
num_stages=3,
|
||||
)
|
||||
|
||||
@@ -488,10 +488,14 @@ _SPEC_VERIFY_AUDITED_BACKENDS = frozenset(
|
||||
}
|
||||
)
|
||||
|
||||
_SPEC_MHA_AUDITED_BACKENDS = frozenset({"triton", "flashinfer", "fa3", "trtllm_mha"})
|
||||
|
||||
# The MHA backends that read through the KV-index translator on every path a
|
||||
# draft forward takes. A fused draft region holds dense K/V rows, so a fused
|
||||
# draft runs only on these.
|
||||
TRANSLATED_MHA_RAILS = frozenset({"triton", "flashinfer", "fa3", "trtllm_mha"})
|
||||
# FA4 carries translated draft ids, but speculative admission is audited
|
||||
# per algorithm below; currently only DSPARK is validated with FA4.
|
||||
TRANSLATED_MHA_RAILS = _SPEC_MHA_AUDITED_BACKENDS | {"fa4"}
|
||||
|
||||
|
||||
def _assert_spec_verify_backends(
|
||||
@@ -613,7 +617,7 @@ def handle_unified_memory_pool(server_args: Any) -> None:
|
||||
eagle_allowed = (
|
||||
_SPEC_VERIFY_AUDITED_BACKENDS
|
||||
if use_mla_backend(server_args)
|
||||
else TRANSLATED_MHA_RAILS
|
||||
else _SPEC_MHA_AUDITED_BACKENDS
|
||||
)
|
||||
eagle_backends = set(attention_backends_of(resolved_view(server_args)))
|
||||
assert (
|
||||
@@ -628,20 +632,24 @@ def handle_unified_memory_pool(server_args: Any) -> None:
|
||||
# An unset draft backend inherits the target's pair.
|
||||
draft_backend = cfg.speculative_draft_attention_backend
|
||||
draft_backends = {draft_backend} if draft_backend else eagle_backends
|
||||
assert draft_backends <= TRANSLATED_MHA_RAILS, (
|
||||
assert draft_backends <= _SPEC_MHA_AUDITED_BACKENDS, (
|
||||
"--enable-unified-memory + EAGLE/EAGLE3 runs the draft on "
|
||||
f"{sorted(draft_backends)}, but a fused draft is MHA-shaped and "
|
||||
"reads its KV through the translated MHA rails "
|
||||
f"{sorted(TRANSLATED_MHA_RAILS)}. "
|
||||
f"{sorted(_SPEC_MHA_AUDITED_BACKENDS)}. "
|
||||
"Set --speculative-draft-attention-backend to one of them."
|
||||
)
|
||||
if cfg.speculative_algorithm == "DSPARK":
|
||||
_assert_spec_verify_backends(server_args, algorithm="DSPARK")
|
||||
_assert_spec_verify_backends(
|
||||
server_args,
|
||||
algorithm="DSPARK",
|
||||
allowed=_SPEC_VERIFY_AUDITED_BACKENDS | {"fa4"},
|
||||
)
|
||||
if cfg.speculative_algorithm == "DFLASH":
|
||||
_assert_spec_verify_backends(
|
||||
server_args,
|
||||
algorithm="DFLASH",
|
||||
allowed=TRANSLATED_MHA_RAILS,
|
||||
allowed=_SPEC_MHA_AUDITED_BACKENDS,
|
||||
)
|
||||
assert not cfg.enable_two_batch_overlap, (
|
||||
"--enable-unified-memory does not support --enable-two-batch-overlap: "
|
||||
|
||||
@@ -463,7 +463,39 @@ class FlashAttentionBackend(AttentionBackend):
|
||||
k_sf, v_sf = self.token_to_kv_pool.get_kv_scale_buffer(layer.layer_id)
|
||||
return {"sfq": q_descale, "sfk": k_sf, "sfv": v_sf}
|
||||
|
||||
def _capture_unified_verify_metadata(self, forward_batch: ForwardBatch) -> bool:
|
||||
# Fixed-width verify uses only capture-stable request/length buffers.
|
||||
# Ragged layouts and encoder offsets still use their eager builders.
|
||||
return (
|
||||
forward_batch.forward_mode.is_target_verify()
|
||||
and self.fa_impl_ver == 4
|
||||
and self.kv_index_translator.reads_are_translated
|
||||
and self.topk <= 1
|
||||
and forward_batch.encoder_lens is None
|
||||
and getattr(forward_batch.spec_info, "ragged_verify_layout", None) is None
|
||||
)
|
||||
|
||||
def init_forward_metadata_in_graph(self, forward_batch: ForwardBatch) -> None:
|
||||
if self._capture_unified_verify_metadata(forward_batch):
|
||||
metadata = self.target_verify_metadata[forward_batch.batch_size]
|
||||
translator = self.kv_index_translator
|
||||
swa = translator.space(IdSpaceKind.SLIDING_WINDOW)
|
||||
normal_decode_set_metadata(
|
||||
metadata.cache_seqlens_int32,
|
||||
metadata.cu_seqlens_k,
|
||||
metadata.page_table,
|
||||
self.req_to_token,
|
||||
forward_batch.req_pool_indices,
|
||||
self.max_num_pages,
|
||||
forward_batch.seq_lens,
|
||||
self.speculative_num_draft_tokens,
|
||||
self.page_size,
|
||||
metadata.swa_page_table,
|
||||
full_v2p=translator.space(IdSpaceKind.FULL).read_v2p,
|
||||
swa_v2p=None if swa is None else swa.read_v2p,
|
||||
zero_tail=True,
|
||||
)
|
||||
return
|
||||
# Single-CG has no Python between steps, so one capturable kernel updates
|
||||
# the persistent metadata.
|
||||
if not forward_batch.forward_mode.is_draft_extend_v2():
|
||||
@@ -594,6 +626,23 @@ class FlashAttentionBackend(AttentionBackend):
|
||||
else:
|
||||
self.cuda_graph_swa_out_cache_loc[:n].copy_(swa_out_cache_loc)
|
||||
|
||||
if self._capture_unified_verify_metadata(forward_batch):
|
||||
if in_capture:
|
||||
self._bind_metadata_buffers(
|
||||
bs,
|
||||
forward_batch.positions.numel(),
|
||||
encoder_lens,
|
||||
forward_mode,
|
||||
spec_info,
|
||||
seq_lens.device,
|
||||
)
|
||||
self.forward_metadata = self.target_verify_metadata[bs]
|
||||
self.forward_metadata_spec_decode_expand = None
|
||||
# The runner records the device-side update at the start of the
|
||||
# model graph. Do not populate a live plan's read-table cache with
|
||||
# this graph-owned buffer: another bucket can overwrite it later.
|
||||
return
|
||||
|
||||
if in_capture:
|
||||
num_tokens = forward_batch.positions.numel()
|
||||
seq_lens_cpu = seq_lens.cpu()
|
||||
|
||||
@@ -1258,16 +1258,11 @@ class DecodeCudaGraphRunner(BaseCudaGraphRunner):
|
||||
attn_backend.init_forward_metadata_out_graph(forward_batch, in_capture=True)
|
||||
|
||||
def run_once():
|
||||
# Graph-recordable metadata-prep hook. The unified memory pool
|
||||
# records ZERO translate nodes here: all its read/write translates
|
||||
# run eagerly in `init_forward_metadata_out_graph` (replay-prep), so
|
||||
# the captured graph reads already-physical locs. Base no-op for triton.
|
||||
# Backends may gather live pool mappings into their static
|
||||
# metadata here; these nodes run before any model KV reads.
|
||||
attn_backend.init_forward_metadata_in_graph(forward_batch)
|
||||
self._record_in_graph_metadata_prep_done()
|
||||
|
||||
# No invalidate_loc_cache() here: the unified pool translates its
|
||||
# locs in `init_forward_metadata_out_graph`, so no cache to invalidate.
|
||||
|
||||
forward_batch.dp_local_start_pos = forward_batch.dp_local_num_tokens = (
|
||||
None
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user