[Dsv4.1] Bounded replay with sparse mla path (#42273)
This commit is contained in:
@@ -49,9 +49,13 @@ def _combine_topk_swa_indices_kernel(
|
||||
gather_lens_ptr,
|
||||
compressed_base_ptr,
|
||||
swa_base_ptr,
|
||||
swa_indices_ptr,
|
||||
swa_indices_stride,
|
||||
swa_lengths_ptr,
|
||||
num_tokens,
|
||||
num_reqs,
|
||||
top_k,
|
||||
EXPLICIT_SWA: tl.constexpr,
|
||||
COMPRESS_RATIO: tl.constexpr,
|
||||
WINDOW_SIZE: tl.constexpr,
|
||||
WIDTH: tl.constexpr,
|
||||
@@ -92,7 +96,10 @@ def _combine_topk_swa_indices_kernel(
|
||||
# -1 entries inside the top-k span stay -1 (attention skips them).
|
||||
# top_k=0 disables the compressed portion for SWA-only layers.
|
||||
topk_len = tl.where(owned, tl.minimum((pos + 1) // COMPRESS_RATIO, top_k), 0)
|
||||
swa_len = tl.where(owned, tl.minimum(pos + 1, WINDOW_SIZE), 0)
|
||||
if EXPLICIT_SWA:
|
||||
swa_len = tl.load(swa_lengths_ptr + token, mask=owned, other=0)
|
||||
else:
|
||||
swa_len = tl.where(owned, tl.minimum(pos + 1, WINDOW_SIZE), 0)
|
||||
|
||||
combined_row = token.to(tl.int64)[:, None] * combined_indices_stride
|
||||
topk_row = token.to(tl.int64)[:, None] * topk_indices_stride
|
||||
@@ -107,9 +114,19 @@ def _combine_topk_swa_indices_kernel(
|
||||
# Workspace SWA index: swa_base[r] + (gather_offset_in_buffer).
|
||||
# For positions [pos - swa_len + 1, pos], the buffer offsets are
|
||||
# [pos - swa_len + 1 - gather_start, pos - gather_start].
|
||||
swa_start = swa_base + pos - swa_len + 1 - gather_start
|
||||
swa_vals = swa_start[:, None] + (offset - topk_len[:, None])
|
||||
in_swa = offset < (topk_len + swa_len)[:, None]
|
||||
if EXPLICIT_SWA:
|
||||
swa_vals = tl.load(
|
||||
swa_indices_ptr
|
||||
+ token.to(tl.int64)[:, None] * swa_indices_stride
|
||||
+ (offset - topk_len[:, None]),
|
||||
mask=owned[:, None] & ~in_topk & in_swa,
|
||||
other=-1,
|
||||
)
|
||||
swa_vals = tl.where(swa_vals >= 0, swa_base[:, None] + swa_vals, -1)
|
||||
else:
|
||||
swa_start = swa_base + pos - swa_len + 1 - gather_start
|
||||
swa_vals = swa_start[:, None] + (offset - topk_len[:, None])
|
||||
vals = tl.where(
|
||||
in_topk,
|
||||
tl.where(topk_vals >= 0, topk_vals + compressed_base[:, None], -1),
|
||||
|
||||
@@ -232,6 +232,8 @@ def _maybe_precompute_flashmla_sched_meta(
|
||||
|
||||
b, s_q = q.shape[0], q.shape[1]
|
||||
num_sm_parts = max(_num_sms(q.device.index) // s_q, 1)
|
||||
if 4 * (5 * b + 1 + num_sm_parts * META_INTS) > 48 * 1024:
|
||||
return
|
||||
meta = torch.empty((num_sm_parts, META_INTS), dtype=torch.int32, device=q.device)
|
||||
num_splits = torch.empty((b + 1,), dtype=torch.int32, device=q.device)
|
||||
flashmla_sched_meta(
|
||||
@@ -2382,11 +2384,15 @@ class DeepseekV4AttnBackend(
|
||||
) -> SparsePrefillChunkCache:
|
||||
seq_lens_cpu = forward_batch.seq_lens_cpu
|
||||
assert seq_lens_cpu is not None
|
||||
# The chunk cache gathers the W-1 positions before the chunk; under the
|
||||
# tail those are late-layer window slots this prefill never wrote.
|
||||
assert self.forward_metadata.late_layer_tail is None
|
||||
# Decoder tails require explicit SWA indices to exclude unwritten history.
|
||||
tail = self.forward_metadata.late_layer_tail
|
||||
request_layout = core_attn_metadata.request_window_layout
|
||||
assert tail is None or request_layout is not None
|
||||
extend_seq_lens = forward_batch.extend_seq_lens
|
||||
extend_seq_lens_cpu = forward_batch.extend_seq_lens_cpu
|
||||
if tail is not None:
|
||||
extend_seq_lens = tail.extend_seq_lens
|
||||
extend_seq_lens_cpu = tail.extend_seq_lens_cpu
|
||||
assert extend_seq_lens_cpu is not None
|
||||
seq_lens_cpu_list = seq_lens_cpu.tolist()
|
||||
total_swa = sum(
|
||||
@@ -2409,6 +2415,9 @@ class DeepseekV4AttnBackend(
|
||||
query_lens = extend_seq_lens.to(torch.int32)
|
||||
# padding rows are never combined
|
||||
query_pos = core_attn_metadata.seq_lens_casual[:num_qo_tokens] - 1
|
||||
if request_layout is not None:
|
||||
assert not is_cp_active(forward_batch)
|
||||
query_pos = request_layout.pos.to(torch.int32)
|
||||
if query_pos.shape[0] < num_qo_tokens:
|
||||
query_pos = _pad_tensor_to_size(query_pos, num_qo_tokens, value=0)
|
||||
return SparsePrefillChunkCache.build(
|
||||
@@ -2424,6 +2433,7 @@ class DeepseekV4AttnBackend(
|
||||
num_qo_tokens=num_qo_tokens,
|
||||
max_seq_len=max(seq_lens_cpu_list),
|
||||
total_swa=total_swa,
|
||||
request_window_layout=request_layout,
|
||||
)
|
||||
|
||||
def _build_forward_metadata(
|
||||
@@ -3419,13 +3429,20 @@ class DeepseekV4AttnBackend(
|
||||
f"{extra_indices.shape=}'s last dimension is not aligned to 64"
|
||||
)
|
||||
|
||||
# sparse_prefill_fwd does not support SM120. The tail stays dense: its
|
||||
# window floor lives in swa_page_indices, which the chunk cache ignores.
|
||||
# RequestWindow sparse gathering does not support CP yet.
|
||||
if (
|
||||
forward_batch.forward_mode.is_extend_without_speculative()
|
||||
and not get_platform().is_sm120
|
||||
and self.forward_metadata.late_layer_tail is None
|
||||
and token_to_kv_pool.request_window is None
|
||||
and (
|
||||
(
|
||||
token_to_kv_pool.request_window is None
|
||||
and self.forward_metadata.late_layer_tail is None
|
||||
)
|
||||
or (
|
||||
token_to_kv_pool.request_window is not None
|
||||
and not is_cp_active(forward_batch)
|
||||
)
|
||||
)
|
||||
and (
|
||||
q.shape[0] > _LARGE_INDEXER_QUERY_THRESHOLD
|
||||
or envs.SGLANG_OPT_FLASHMLA_SPARSE_PREFILL.get()
|
||||
|
||||
@@ -124,6 +124,8 @@ def combine_topk_swa_indices(
|
||||
topk: int,
|
||||
out_indices: Optional[torch.Tensor] = None,
|
||||
out_lens: Optional[torch.Tensor] = None,
|
||||
swa_indices: Optional[torch.Tensor] = None,
|
||||
swa_lengths: Optional[torch.Tensor] = None,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""Combine top-k and SWA indices for flash_mla_sparse_fwd.
|
||||
|
||||
@@ -132,6 +134,7 @@ def combine_topk_swa_indices(
|
||||
query_start_loc can include a cross-chunk offset; query_pos is absolute.
|
||||
compressed_base and swa_base address the flat workspace. SWA-only layers use
|
||||
topk == 0, but compress_ratio must still be positive.
|
||||
Optional SWA indices are relative to swa_base and preserve replay floors.
|
||||
|
||||
Returns int32 indices [num_tokens, padded_topk_swa] and per-token scanned-prefix
|
||||
lengths, including -1 entries skipped by attention. Width is padded to 128.
|
||||
@@ -151,6 +154,10 @@ def combine_topk_swa_indices(
|
||||
)
|
||||
|
||||
num_tokens = topk_indices.shape[0]
|
||||
if swa_indices is not None:
|
||||
assert swa_indices.shape == (num_tokens, window_size)
|
||||
assert swa_lengths is not None and swa_lengths.shape == (num_tokens,)
|
||||
assert swa_indices.dtype == swa_lengths.dtype == torch.int32
|
||||
num_reqs = seq_lens.shape[0]
|
||||
combined_topk = combined_topk_width(topk, window_size)
|
||||
if out_indices is None:
|
||||
@@ -188,9 +195,13 @@ def combine_topk_swa_indices(
|
||||
gather_lens,
|
||||
compressed_base,
|
||||
swa_base,
|
||||
swa_indices,
|
||||
0 if swa_indices is None else swa_indices.stride(0),
|
||||
swa_lengths,
|
||||
num_tokens,
|
||||
num_reqs,
|
||||
top_k=topk,
|
||||
EXPLICIT_SWA=swa_indices is not None,
|
||||
COMPRESS_RATIO=compress_ratio,
|
||||
WINDOW_SIZE=window_size,
|
||||
WIDTH=combined_topk,
|
||||
@@ -325,6 +336,8 @@ class SparsePrefillChunkCache:
|
||||
# Compressed caches keyed by compress ratio: c128 (every block, combined once
|
||||
# per chunk) and the top-k ratios (combined per layer).
|
||||
compressed: Dict[int, CompressedGather] = field(default_factory=dict)
|
||||
swa_indices: Optional[torch.Tensor] = None
|
||||
swa_lengths: Optional[torch.Tensor] = None
|
||||
|
||||
@classmethod
|
||||
def build(
|
||||
@@ -341,6 +354,7 @@ class SparsePrefillChunkCache:
|
||||
num_qo_tokens: int,
|
||||
max_seq_len: int,
|
||||
total_swa: int,
|
||||
request_window_layout=None,
|
||||
) -> "SparsePrefillChunkCache":
|
||||
"""``query_lens`` / ``query_pos``: the rows this forward runs (the extend, or
|
||||
a CP rank's interleaved share of it); the SWA gather spans the whole extend."""
|
||||
@@ -350,17 +364,29 @@ class SparsePrefillChunkCache:
|
||||
query_start_loc = torch.zeros(num_reqs + 1, dtype=torch.int32, device=device)
|
||||
query_start_loc[1:] = torch.cumsum(query_lens, dim=0).to(torch.int32)
|
||||
|
||||
swa_token_ids, swa_first_pos, swa_gather_lens, swa_offsets = (
|
||||
build_swa_token_ids(
|
||||
seq_lens=seq_lens,
|
||||
extend_seq_lens=extend_seq_lens,
|
||||
req_pool_indices=req_pool_indices,
|
||||
req_to_token=req_to_token,
|
||||
full_to_swa=full_to_swa,
|
||||
swa_window=swa_window_size,
|
||||
total_swa=total_swa,
|
||||
if request_window_layout is None:
|
||||
swa_token_ids, swa_first_pos, swa_gather_lens, swa_offsets = (
|
||||
build_swa_token_ids(
|
||||
seq_lens=seq_lens,
|
||||
extend_seq_lens=extend_seq_lens,
|
||||
req_pool_indices=req_pool_indices,
|
||||
req_to_token=req_to_token,
|
||||
full_to_swa=full_to_swa,
|
||||
swa_window=swa_window_size,
|
||||
total_swa=total_swa,
|
||||
)
|
||||
)
|
||||
)
|
||||
swa_indices = swa_lengths = None
|
||||
else:
|
||||
swa_token_ids = torch.arange(
|
||||
request_window_layout.size, dtype=torch.int32, device=device
|
||||
)
|
||||
swa_first_pos = torch.zeros_like(seq_lens)
|
||||
swa_gather_lens = torch.zeros_like(seq_lens)
|
||||
# RequestWindow indices already address the shared workspace.
|
||||
swa_offsets = torch.zeros(num_reqs + 1, dtype=torch.int32, device=device)
|
||||
swa_indices = request_window_layout.indices
|
||||
swa_lengths = request_window_layout.lengths
|
||||
|
||||
cache = cls(
|
||||
num_reqs=num_reqs,
|
||||
@@ -375,6 +401,8 @@ class SparsePrefillChunkCache:
|
||||
swa_first_pos=swa_first_pos,
|
||||
swa_gather_lens=swa_gather_lens,
|
||||
swa_offsets=swa_offsets,
|
||||
swa_indices=swa_indices,
|
||||
swa_lengths=swa_lengths,
|
||||
)
|
||||
|
||||
# Pre-compute the c0 combine output: TOPK=0, compressed_base=0,
|
||||
@@ -393,6 +421,8 @@ class SparsePrefillChunkCache:
|
||||
window_size=swa_window_size,
|
||||
compress_ratio=1,
|
||||
topk=0,
|
||||
swa_indices=swa_indices,
|
||||
swa_lengths=swa_lengths,
|
||||
)
|
||||
return cache
|
||||
|
||||
@@ -484,6 +514,8 @@ class SparsePrefillChunkCache:
|
||||
window_size=self.swa_window_size,
|
||||
compress_ratio=128,
|
||||
topk=c128_max,
|
||||
swa_indices=self.swa_indices,
|
||||
swa_lengths=self.swa_lengths,
|
||||
)
|
||||
|
||||
gather = CompressedGather(
|
||||
@@ -577,4 +609,6 @@ class SparsePrefillChunkCache:
|
||||
topk=topk,
|
||||
out_indices=gather.combined_indices,
|
||||
out_lens=gather.combined_lens,
|
||||
swa_indices=self.swa_indices,
|
||||
swa_lengths=self.swa_lengths,
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user