[Dsv4.1] Bounded replay with sparse mla path (#42273)

This commit is contained in:
Yuwei An
2026-10-04 17:19:14 -07:00
committed by GitHub
parent b792228b35
commit f70e8c68fc
5 changed files with 247 additions and 21 deletions
@@ -49,9 +49,13 @@ def _combine_topk_swa_indices_kernel(
gather_lens_ptr,
compressed_base_ptr,
swa_base_ptr,
swa_indices_ptr,
swa_indices_stride,
swa_lengths_ptr,
num_tokens,
num_reqs,
top_k,
EXPLICIT_SWA: tl.constexpr,
COMPRESS_RATIO: tl.constexpr,
WINDOW_SIZE: tl.constexpr,
WIDTH: tl.constexpr,
@@ -92,7 +96,10 @@ def _combine_topk_swa_indices_kernel(
# -1 entries inside the top-k span stay -1 (attention skips them).
# top_k=0 disables the compressed portion for SWA-only layers.
topk_len = tl.where(owned, tl.minimum((pos + 1) // COMPRESS_RATIO, top_k), 0)
swa_len = tl.where(owned, tl.minimum(pos + 1, WINDOW_SIZE), 0)
if EXPLICIT_SWA:
swa_len = tl.load(swa_lengths_ptr + token, mask=owned, other=0)
else:
swa_len = tl.where(owned, tl.minimum(pos + 1, WINDOW_SIZE), 0)
combined_row = token.to(tl.int64)[:, None] * combined_indices_stride
topk_row = token.to(tl.int64)[:, None] * topk_indices_stride
@@ -107,9 +114,19 @@ def _combine_topk_swa_indices_kernel(
# Workspace SWA index: swa_base[r] + (gather_offset_in_buffer).
# For positions [pos - swa_len + 1, pos], the buffer offsets are
# [pos - swa_len + 1 - gather_start, pos - gather_start].
swa_start = swa_base + pos - swa_len + 1 - gather_start
swa_vals = swa_start[:, None] + (offset - topk_len[:, None])
in_swa = offset < (topk_len + swa_len)[:, None]
if EXPLICIT_SWA:
swa_vals = tl.load(
swa_indices_ptr
+ token.to(tl.int64)[:, None] * swa_indices_stride
+ (offset - topk_len[:, None]),
mask=owned[:, None] & ~in_topk & in_swa,
other=-1,
)
swa_vals = tl.where(swa_vals >= 0, swa_base[:, None] + swa_vals, -1)
else:
swa_start = swa_base + pos - swa_len + 1 - gather_start
swa_vals = swa_start[:, None] + (offset - topk_len[:, None])
vals = tl.where(
in_topk,
tl.where(topk_vals >= 0, topk_vals + compressed_base[:, None], -1),
@@ -232,6 +232,8 @@ def _maybe_precompute_flashmla_sched_meta(
b, s_q = q.shape[0], q.shape[1]
num_sm_parts = max(_num_sms(q.device.index) // s_q, 1)
if 4 * (5 * b + 1 + num_sm_parts * META_INTS) > 48 * 1024:
return
meta = torch.empty((num_sm_parts, META_INTS), dtype=torch.int32, device=q.device)
num_splits = torch.empty((b + 1,), dtype=torch.int32, device=q.device)
flashmla_sched_meta(
@@ -2382,11 +2384,15 @@ class DeepseekV4AttnBackend(
) -> SparsePrefillChunkCache:
seq_lens_cpu = forward_batch.seq_lens_cpu
assert seq_lens_cpu is not None
# The chunk cache gathers the W-1 positions before the chunk; under the
# tail those are late-layer window slots this prefill never wrote.
assert self.forward_metadata.late_layer_tail is None
# Decoder tails require explicit SWA indices to exclude unwritten history.
tail = self.forward_metadata.late_layer_tail
request_layout = core_attn_metadata.request_window_layout
assert tail is None or request_layout is not None
extend_seq_lens = forward_batch.extend_seq_lens
extend_seq_lens_cpu = forward_batch.extend_seq_lens_cpu
if tail is not None:
extend_seq_lens = tail.extend_seq_lens
extend_seq_lens_cpu = tail.extend_seq_lens_cpu
assert extend_seq_lens_cpu is not None
seq_lens_cpu_list = seq_lens_cpu.tolist()
total_swa = sum(
@@ -2409,6 +2415,9 @@ class DeepseekV4AttnBackend(
query_lens = extend_seq_lens.to(torch.int32)
# padding rows are never combined
query_pos = core_attn_metadata.seq_lens_casual[:num_qo_tokens] - 1
if request_layout is not None:
assert not is_cp_active(forward_batch)
query_pos = request_layout.pos.to(torch.int32)
if query_pos.shape[0] < num_qo_tokens:
query_pos = _pad_tensor_to_size(query_pos, num_qo_tokens, value=0)
return SparsePrefillChunkCache.build(
@@ -2424,6 +2433,7 @@ class DeepseekV4AttnBackend(
num_qo_tokens=num_qo_tokens,
max_seq_len=max(seq_lens_cpu_list),
total_swa=total_swa,
request_window_layout=request_layout,
)
def _build_forward_metadata(
@@ -3419,13 +3429,20 @@ class DeepseekV4AttnBackend(
f"{extra_indices.shape=}'s last dimension is not aligned to 64"
)
# sparse_prefill_fwd does not support SM120. The tail stays dense: its
# window floor lives in swa_page_indices, which the chunk cache ignores.
# RequestWindow sparse gathering does not support CP yet.
if (
forward_batch.forward_mode.is_extend_without_speculative()
and not get_platform().is_sm120
and self.forward_metadata.late_layer_tail is None
and token_to_kv_pool.request_window is None
and (
(
token_to_kv_pool.request_window is None
and self.forward_metadata.late_layer_tail is None
)
or (
token_to_kv_pool.request_window is not None
and not is_cp_active(forward_batch)
)
)
and (
q.shape[0] > _LARGE_INDEXER_QUERY_THRESHOLD
or envs.SGLANG_OPT_FLASHMLA_SPARSE_PREFILL.get()
@@ -124,6 +124,8 @@ def combine_topk_swa_indices(
topk: int,
out_indices: Optional[torch.Tensor] = None,
out_lens: Optional[torch.Tensor] = None,
swa_indices: Optional[torch.Tensor] = None,
swa_lengths: Optional[torch.Tensor] = None,
) -> tuple[torch.Tensor, torch.Tensor]:
"""Combine top-k and SWA indices for flash_mla_sparse_fwd.
@@ -132,6 +134,7 @@ def combine_topk_swa_indices(
query_start_loc can include a cross-chunk offset; query_pos is absolute.
compressed_base and swa_base address the flat workspace. SWA-only layers use
topk == 0, but compress_ratio must still be positive.
Optional SWA indices are relative to swa_base and preserve replay floors.
Returns int32 indices [num_tokens, padded_topk_swa] and per-token scanned-prefix
lengths, including -1 entries skipped by attention. Width is padded to 128.
@@ -151,6 +154,10 @@ def combine_topk_swa_indices(
)
num_tokens = topk_indices.shape[0]
if swa_indices is not None:
assert swa_indices.shape == (num_tokens, window_size)
assert swa_lengths is not None and swa_lengths.shape == (num_tokens,)
assert swa_indices.dtype == swa_lengths.dtype == torch.int32
num_reqs = seq_lens.shape[0]
combined_topk = combined_topk_width(topk, window_size)
if out_indices is None:
@@ -188,9 +195,13 @@ def combine_topk_swa_indices(
gather_lens,
compressed_base,
swa_base,
swa_indices,
0 if swa_indices is None else swa_indices.stride(0),
swa_lengths,
num_tokens,
num_reqs,
top_k=topk,
EXPLICIT_SWA=swa_indices is not None,
COMPRESS_RATIO=compress_ratio,
WINDOW_SIZE=window_size,
WIDTH=combined_topk,
@@ -325,6 +336,8 @@ class SparsePrefillChunkCache:
# Compressed caches keyed by compress ratio: c128 (every block, combined once
# per chunk) and the top-k ratios (combined per layer).
compressed: Dict[int, CompressedGather] = field(default_factory=dict)
swa_indices: Optional[torch.Tensor] = None
swa_lengths: Optional[torch.Tensor] = None
@classmethod
def build(
@@ -341,6 +354,7 @@ class SparsePrefillChunkCache:
num_qo_tokens: int,
max_seq_len: int,
total_swa: int,
request_window_layout=None,
) -> "SparsePrefillChunkCache":
"""``query_lens`` / ``query_pos``: the rows this forward runs (the extend, or
a CP rank's interleaved share of it); the SWA gather spans the whole extend."""
@@ -350,17 +364,29 @@ class SparsePrefillChunkCache:
query_start_loc = torch.zeros(num_reqs + 1, dtype=torch.int32, device=device)
query_start_loc[1:] = torch.cumsum(query_lens, dim=0).to(torch.int32)
swa_token_ids, swa_first_pos, swa_gather_lens, swa_offsets = (
build_swa_token_ids(
seq_lens=seq_lens,
extend_seq_lens=extend_seq_lens,
req_pool_indices=req_pool_indices,
req_to_token=req_to_token,
full_to_swa=full_to_swa,
swa_window=swa_window_size,
total_swa=total_swa,
if request_window_layout is None:
swa_token_ids, swa_first_pos, swa_gather_lens, swa_offsets = (
build_swa_token_ids(
seq_lens=seq_lens,
extend_seq_lens=extend_seq_lens,
req_pool_indices=req_pool_indices,
req_to_token=req_to_token,
full_to_swa=full_to_swa,
swa_window=swa_window_size,
total_swa=total_swa,
)
)
)
swa_indices = swa_lengths = None
else:
swa_token_ids = torch.arange(
request_window_layout.size, dtype=torch.int32, device=device
)
swa_first_pos = torch.zeros_like(seq_lens)
swa_gather_lens = torch.zeros_like(seq_lens)
# RequestWindow indices already address the shared workspace.
swa_offsets = torch.zeros(num_reqs + 1, dtype=torch.int32, device=device)
swa_indices = request_window_layout.indices
swa_lengths = request_window_layout.lengths
cache = cls(
num_reqs=num_reqs,
@@ -375,6 +401,8 @@ class SparsePrefillChunkCache:
swa_first_pos=swa_first_pos,
swa_gather_lens=swa_gather_lens,
swa_offsets=swa_offsets,
swa_indices=swa_indices,
swa_lengths=swa_lengths,
)
# Pre-compute the c0 combine output: TOPK=0, compressed_base=0,
@@ -393,6 +421,8 @@ class SparsePrefillChunkCache:
window_size=swa_window_size,
compress_ratio=1,
topk=0,
swa_indices=swa_indices,
swa_lengths=swa_lengths,
)
return cache
@@ -484,6 +514,8 @@ class SparsePrefillChunkCache:
window_size=self.swa_window_size,
compress_ratio=128,
topk=c128_max,
swa_indices=self.swa_indices,
swa_lengths=self.swa_lengths,
)
gather = CompressedGather(
@@ -577,4 +609,6 @@ class SparsePrefillChunkCache:
topk=topk,
out_indices=gather.combined_indices,
out_lens=gather.combined_lens,
swa_indices=self.swa_indices,
swa_lengths=self.swa_lengths,
)