[DSv4.1] Fold encoder-SWA replay into the extend forward (#43166)

Co-authored-by: Yuwei An <ayw.sirius19@gmail.com>
This commit is contained in:
YAMY
2026-10-14 17:27:07 -07:00
committed by GitHub
co-authored by Yuwei An
parent 3c28614f12
commit c891c5c7ca
6 changed files with 678 additions and 54 deletions
@@ -61,6 +61,9 @@ class AttentionBackend(ABC):
decode_attention_backend_str: Optional[str] = None
supports_ragged_verify_graph: bool = False
# DeepSeek-V4.1 encoder SWA replay: True if the backend floors the window and
# skips cached-row compression for replay rows folded into the extend.
folds_encoder_swa_replay: bool = False
# Compute / KV-cache dtype. Only backends that need them (MLA/MHA fp8
# fuse-rope checks) set these in __init__; declared here as None so callers
# can read them off ANY backend — including hybrid wrappers that don't set
@@ -941,6 +941,13 @@ def _tail_rows(
return t[token_indices]
def _pad_rows(t: torch.Tensor, *, num_rows: int) -> torch.Tensor:
assert t.shape[0] <= num_rows, (t.shape[0], num_rows)
if t.shape[0] == num_rows:
return t
return torch.nn.functional.pad(t, (0, num_rows - t.shape[0]))
# Rows per logits chunk for the ratio-1/2 indexer inside the prefill CUDA graph;
# its width is the graph's max_seq_len, and longer contexts replay eagerly.
_PREFILL_GRAPH_INDEXER_ROW_CHUNK = 2048
@@ -1103,6 +1110,7 @@ class DeepseekV4AttnBackend(
use_captured_forward_metadata_for_breakable_cuda_graph: bool = True
supports_prefill_cuda_graph_max_context_size: bool = True
supports_ragged_verify_graph: bool = True
folds_encoder_swa_replay: bool = True
needs_cpu_seq_lens: bool = False
trtllm_attn: bool = False
@@ -1134,6 +1142,7 @@ class DeepseekV4AttnBackend(
super().__init__()
self.model_runner = model_runner
self.encoder_replay = False
self.encoder_row_floor = None
self.device = torch.device(model_runner.device)
self.max_context_len = model_runner.model_config.context_len
head_dim = model_runner.model_config.head_dim
@@ -1632,6 +1641,12 @@ class DeepseekV4AttnBackend(
tail_len=SWA_WINDOW,
device=device,
)
if self.encoder_row_floor is not None:
# Folded hits: a tail row never reads below its request's replay start.
swa_replay_start = torch.maximum(
swa_replay_start,
self.encoder_row_floor[token_indices].to(swa_replay_start.dtype),
)
contiguous_start = (
extend_lens_cpu[0] - tail_lens_cpu[0] if len(extend_lens_cpu) == 1 else None
)
@@ -2541,6 +2556,7 @@ class DeepseekV4AttnBackend(
return
self.encoder_replay = forward_batch.encoder_swa_replay
self.encoder_row_floor = forward_batch.encoder_swa_row_floor
self.forward_metadata = self._build_forward_metadata(forward_batch)
self.init_forward_metadata_in_graph(forward_batch)
self.tail_forward_metadata = (
@@ -3092,6 +3108,11 @@ class DeepseekV4AttnBackend(
# length is the request count; padded rows belong to no request.
start_loc = forward_batch.extend_start_loc
seq_lens = forward_batch.extend_seq_lens[: start_loc.shape[0]]
skip = forward_batch.encoder_swa_compress_skip
if skip is not None:
# Folded replay rows are cached already; compress the new rows only.
start_loc = start_loc + skip
seq_lens = seq_lens - skip
self._low_ratio_compress_fused(
layer,
x,
@@ -3100,6 +3121,9 @@ class DeepseekV4AttnBackend(
extend_offsets=(start_loc.to(torch.int32), seq_lens.to(torch.int32)),
)
else:
rows = forward_batch.encoder_swa_compress_rows
if rows is not None and forward_batch.forward_mode.is_extend():
x, req, pos = x[rows], req[rows], pos[rows]
self._low_ratio_compress_torch(
layer,
x,
@@ -4218,6 +4242,12 @@ class DeepseekV4AttnBackend(
)
group_first = torch.cummax(torch.where(starts, offset, 0), dim=0).values
swa_replay_start = raw_positions - (offset - group_first)
elif swa_replay_start is None and self.encoder_row_floor is not None:
# Folded replay: hits floor every row at their replay start;
# MLP-sync padding rows follow the real ones with floor 0.
swa_replay_start = _pad_rows(
self.encoder_row_floor, num_rows=raw_positions.shape[0]
)
request_layout = window_layout(
req_pool_indices_repeated,
raw_positions,
+27 -4
View File
@@ -612,24 +612,33 @@ class TpModelWorker(BaseTpWorker):
return_kv_loc_plan: bool = False,
) -> GenerationBatchResult:
# Get forward batch from schedule batch
folded = None
if batch is not None:
# update the consumer index of hicache to the running batch
self.set_hicache_consumer(batch.hicache_consumer_index)
if get_exec().features.enable_encoder_swa_bounded_replay:
from sglang.srt.model_executor.encoder_swa_replay import (
apply_folded_extend,
fold_encoder_swa_replay,
run_encoder_swa_replay,
)
# Replay reads restored main/indexer KV before the normal extend.
run_encoder_swa_replay(self, batch)
if self.model_runner.attn_backend.folds_encoder_swa_replay:
# Hits extend from their replay start, rebuilding the window in-pass.
folded = fold_encoder_swa_replay(self, batch)
else:
# Replay reads restored main/indexer KV before the normal extend.
run_encoder_swa_replay(self, batch)
forward_batch = ForwardBatch.init_new(
batch,
batch if folded is None else folded.batch,
self.model_runner,
capture_hidden_mode=capture_hidden_mode,
return_hidden_states_before_norm=False,
)
if folded is not None:
apply_folded_extend(folded, forward_batch)
else:
# FIXME(lsyin): unify the interface of forward_batch
assert forward_batch is not None
@@ -649,13 +658,27 @@ class TpModelWorker(BaseTpWorker):
pp_proxy_tensors=pp_proxy_tensors,
)
logits_output, can_run_cuda_graph = out.logits_output, out.can_run_graph
if folded is not None:
from sglang.srt.model_executor.encoder_swa_replay import (
drop_folded_rows,
)
drop_folded_rows(logits_output=logits_output, folded=folded)
batch_result = GenerationBatchResult(
logits_output=logits_output,
can_run_cuda_graph=can_run_cuda_graph,
expert_distribution_metrics=out.expert_distribution_metrics,
routed_experts_output=out.routed_experts_output,
indexer_topk_output=out.indexer_topk_output,
kv_loc_plan=forward_batch.kv_loc_plan if return_kv_loc_plan else None,
kv_loc_plan=(
None
if not return_kv_loc_plan
# A folded forward also wrote the replay rows; the caller's
# plan covers the batch it scheduled.
else forward_batch.kv_loc_plan
if folded is None
else self.model_runner.kv_index_translator.own_plan(batch)
),
)
capture_pre_sample_logits(batch, forward_batch, logits_output)
@@ -1,8 +1,72 @@
from copy import copy
from typing import Any, Optional
import msgspec
import torch
class FoldedExtend(msgspec.Struct, frozen=True):
"""An extend batch whose prefix hits start at their replay start."""
batch: Any # ScheduleBatch copy fed to ForwardBatch.init_new
keep_rows: torch.Tensor # [original rows] int64, folded-row index of each
scheduled_row: torch.Tensor # [folded rows] int64, original row, -1 if replayed
row_floor: torch.Tensor # [folded rows] int64
compress_skip: torch.Tensor # [bs] int32, leading replay rows per request
replay_lens: list[int] # [bs] leading replay rows per request
num_rows: int
def fold_encoder_swa_replay(worker, batch) -> Optional[FoldedExtend]:
"""Prepend each hit's <=128 replay tokens to its extend instead of replaying."""
runner = worker.model_runner
window = runner.token_to_kv_pool.request_window
if window is None or not batch.forward_mode.is_extend_without_speculative():
return None
rows = _reset_windows_and_collect_hits(batch=batch, window=window)
if not rows:
return None
return _fold_batch(batch=batch, runner=runner, rows=rows)
def apply_folded_extend(folded: FoldedExtend, forward_batch) -> None:
forward_batch.encoder_swa_row_floor = folded.row_floor
forward_batch.encoder_swa_compress_skip = folded.compress_skip
forward_batch.encoder_swa_compress_rows = folded.keep_rows
def drop_folded_rows(*, logits_output, folded: FoldedExtend) -> None:
"""Hand downstream consumers (the DSpark draft) only the original extend rows."""
hidden = logits_output.hidden_states
token_indices = logits_output.hidden_states_token_indices
if token_indices is not None:
# Decoder-SWA tail rows index the folded extend; keep those on original rows.
keep = _tail_rows_on_original(
folded=folded, num_tail_rows=token_indices.shape[0]
).to(token_indices.device, non_blocking=True)
logits_output.hidden_states_token_indices = folded.scheduled_row[
token_indices[keep]
]
if hidden is not None:
logits_output.hidden_states = hidden[keep]
elif hidden is not None and hidden.shape[0] >= folded.num_rows:
# MLP-sync padding rows, if any, follow the real rows and are dropped too.
logits_output.hidden_states = hidden[folded.keep_rows]
def _tail_rows_on_original(*, folded: FoldedExtend, num_tail_rows: int):
# Each request's tail is its last SWA_WINDOW folded rows; replay rows lead it.
from sglang.srt.layers.attention.deepseek_v4_backend import SWA_WINDOW
keep, off = [], 0
for n, r in zip(folded.batch.extend_lens, folded.replay_lens):
tail = min(SWA_WINDOW, n)
keep.append(torch.arange(off + max(0, tail - (n - r)), off + tail))
off += tail
assert off == num_tail_rows, (off, num_tail_rows)
return torch.cat(keep)
def run_encoder_swa_replay(worker, batch):
from sglang.srt.model_executor.forward_batch_info import (
CaptureHiddenMode,
@@ -13,11 +77,28 @@ def run_encoder_swa_replay(worker, batch):
window = runner.token_to_kv_pool.request_window
if window is None or not batch.forward_mode.is_extend_without_speculative():
return
rows = _reset_windows_and_collect_hits(batch=batch, window=window)
if not rows:
return
# One forward replays every hitting request; each request's window is
# floored at its own start, so the result matches per-request replays.
replay = _build_replay_batch(batch=batch, runner=runner, rows=rows)
fb = ForwardBatch.init_new(
replay,
runner,
capture_hidden_mode=CaptureHiddenMode.NULL,
return_hidden_states_before_norm=False,
)
fb.encoder_swa_replay = True
runner.forward(fb)
def _reset_windows_and_collect_hits(*, batch, window):
rows = []
for i, reset in enumerate(batch.encoder_swa_reset):
if not reset:
continue
slot = batch.req_pool_indices[i : i + 1]
window.reset(slot)
window.reset(batch.req_pool_indices[i : i + 1])
end = batch.prefix_lens[i]
if not end:
continue
@@ -25,53 +106,157 @@ def run_encoder_swa_replay(worker, batch):
raise ValueError(
"encoder SWA replay requires an even cached-prefix boundary"
)
start = max(0, end - 128)
req = batch.reqs[i]
replay = copy(batch)
replay.reqs = [req]
replay.input_ids = torch.tensor(
list(req.full_untruncated_fill_ids[start:end]),
dtype=torch.int64,
device=runner.device,
)
replay.prefill_input_ids_cpu = None
replay.req_pool_indices = slot
replay.req_pool_indices_cpu = batch.req_pool_indices_cpu[i : i + 1]
replay.prefix_lens = [start]
replay.extend_lens = [end - start]
replay.extend_num_tokens = end - start
replay.seq_lens = torch.tensor([end], dtype=torch.int64, device=runner.device)
replay.seq_lens_cpu = torch.tensor([end], dtype=torch.int64)
replay.seq_lens_sum = end
replay.orig_seq_lens = replay.seq_lens
replay.out_cache_loc = runner.req_to_token_pool.req_to_token[
slot[0], start:end
].long()
replay.return_logprob = False
replay.top_logprobs_nums = None
replay.token_ids_logprobs = None
replay.extend_logprob_start_lens = [end - start]
replay.extend_input_logprob_token_ids = None
replay.is_prefill_only = True
replay.spec_info = None
replay.sampling_info = None
replay.has_grammar = False
replay.multimodal_inputs = [None]
replay.engram_history = None
hasher = runner.model.model.engram_hasher
if hasher is not None:
n = hasher.max_ngram_size - 1
ids = list(req.full_untruncated_fill_ids[max(0, start - n) : start])
replay.engram_history = torch.tensor(
[[0] * (n - len(ids)) + ids],
dtype=torch.int32,
device=runner.device,
rows.append((i, max(0, end - 128), end))
return rows
def _build_replay_batch(*, batch, runner, rows):
idx = [i for i, _, _ in rows]
starts = [s for _, s, _ in rows]
ends = [e for _, _, e in rows]
lens = [e - s for s, e in zip(starts, ends)]
reqs = [batch.reqs[i] for i in idx]
slots = batch.req_pool_indices[idx]
req_to_token = runner.req_to_token_pool.req_to_token
replay = copy(batch)
replay.reqs = reqs
replay.input_ids = torch.tensor(
[
t
for r, s, e in zip(reqs, starts, ends)
for t in r.full_untruncated_fill_ids[s:e]
],
dtype=torch.int64,
device=runner.device,
)
replay.prefill_input_ids_cpu = None
replay.req_pool_indices = slots
replay.req_pool_indices_cpu = batch.req_pool_indices_cpu[idx]
replay.prefix_lens = starts
replay.extend_lens = lens
replay.extend_num_tokens = sum(lens)
replay.seq_lens = torch.tensor(ends, dtype=torch.int64, device=runner.device)
replay.seq_lens_cpu = torch.tensor(ends, dtype=torch.int64)
replay.seq_lens_sum = sum(ends)
replay.orig_seq_lens = replay.seq_lens
replay.out_cache_loc = torch.cat(
[req_to_token[slots[k], s:e] for k, (s, e) in enumerate(zip(starts, ends))]
).long()
replay.return_logprob = False
replay.top_logprobs_nums = None
replay.token_ids_logprobs = None
replay.extend_logprob_start_lens = lens
replay.extend_input_logprob_token_ids = None
replay.is_prefill_only = True
replay.spec_info = None
replay.sampling_info = None
replay.has_grammar = False
replay.multimodal_inputs = [None] * len(reqs)
replay.engram_history = _engram_history(reqs=reqs, starts=starts, runner=runner)
_set_execution_counts(batch=replay, scheduled=batch)
return replay
def _engram_history(*, reqs, starts, runner):
hasher = runner.model.model.engram_hasher
if hasher is None:
return None
n = hasher.max_ngram_size - 1
hist = []
for r, s in zip(reqs, starts):
ids = list(r.full_untruncated_fill_ids[max(0, s - n) : s])
hist.append([0] * (n - len(ids)) + ids)
return torch.tensor(hist, dtype=torch.int32, device=runner.device)
def _fold_batch(*, batch, runner, rows) -> FoldedExtend:
bs = len(batch.reqs)
device = runner.device
replay = [0] * bs
for i, s, e in rows:
replay[i] = e - s
ext = list(batch.extend_lens)
pre = [p - r for p, r in zip(batch.prefix_lens, replay)]
new_ext = [x + r for x, r in zip(ext, replay)]
req_to_token = runner.req_to_token_pool.req_to_token
ids, locs, keep, cpu_ids = [], [], [], []
off = fold_off = 0
for i, req in enumerate(batch.reqs):
r, n = replay[i], ext[i]
if r:
head = list(req.full_untruncated_fill_ids[pre[i] : pre[i] + r])
ids.append(torch.tensor(head, dtype=batch.input_ids.dtype, device=device))
locs.append(
req_to_token[batch.req_pool_indices[i], pre[i] : pre[i] + r].long()
)
fb = ForwardBatch.init_new(
replay,
runner,
capture_hidden_mode=CaptureHiddenMode.NULL,
return_hidden_states_before_norm=False,
cpu_ids.append(torch.tensor(head, dtype=torch.int64))
ids.append(batch.input_ids[off : off + n])
locs.append(batch.out_cache_loc[off : off + n].long())
if batch.prefill_input_ids_cpu is not None:
cpu_ids.append(batch.prefill_input_ids_cpu[off : off + n].to(torch.int64))
keep.append(torch.arange(fold_off + r, fold_off + r + n))
off += n
fold_off += r + n
folded = copy(batch)
folded.input_ids = torch.cat(ids)
folded.out_cache_loc = torch.cat(locs)
folded.prefill_input_ids_cpu = (
torch.cat(cpu_ids) if batch.prefill_input_ids_cpu is not None else None
)
folded.prefix_lens = pre
folded.extend_lens = new_ext
folded.extend_num_tokens = fold_off
if batch.extend_logprob_start_lens is not None:
folded.extend_logprob_start_lens = [
x + r for x, r in zip(batch.extend_logprob_start_lens, replay)
]
if batch.engram_history is not None:
folded.engram_history = _engram_history(
reqs=batch.reqs, starts=pre, runner=runner
)
fb.encoder_swa_replay = True
runner.forward(fb)
_set_execution_counts(batch=folded, scheduled=batch)
floor = torch.tensor(
[p if r else 0 for p, r in zip(pre, replay)], dtype=torch.int64
)
row_floor = torch.repeat_interleave(
floor, torch.tensor(new_ext), output_size=fold_off
)
keep_rows = torch.cat(keep)
scheduled_row = torch.full((fold_off,), -1, dtype=torch.int64)
scheduled_row[keep_rows] = torch.arange(keep_rows.shape[0])
return FoldedExtend(
batch=folded,
keep_rows=keep_rows.to(device, non_blocking=True),
scheduled_row=scheduled_row.to(device, non_blocking=True),
row_floor=row_floor.to(device, non_blocking=True),
compress_skip=torch.tensor(replay, dtype=torch.int32).to(
device, non_blocking=True
),
replay_lens=replay,
num_rows=fold_off,
)
def _set_execution_counts(*, batch, scheduled) -> None:
"""MLP-sync counts of a batch copied from `scheduled` with different rows."""
if scheduled.global_num_tokens is None:
return
# The flag rejects attention DP, so the gather holds only this rank's count.
if (
len(scheduled.global_num_tokens) != 1
or scheduled.global_num_tokens[0] != scheduled.extend_num_tokens
):
raise ValueError(
"encoder SWA replay expects one MLP-sync count equal to the scheduled "
f"extend ({scheduled.extend_num_tokens}), got {scheduled.global_num_tokens}"
)
batch.global_num_tokens = [batch.extend_num_tokens]
# The scheduler's formula; the draft counts stay with the scheduled batch.
batch.global_num_tokens_for_logprob = [
sum(
max(n - start, 1)
for n, start in zip(batch.extend_lens, batch.extend_logprob_start_lens)
)
]
@@ -772,6 +772,12 @@ class ForwardBatch(ForwardBatchDeepSeekMHAMixin):
# For ngram embedding
ngram_embedding_info: Optional[NgramEmbeddingInfo] = None
encoder_swa_replay: bool = False
# Encoder replay folded into the extend: per-row window floor [num_tokens]
# int64 (the request's replay start, 0 for other requests), and per-request
# leading replay rows the low-ratio compressors skip [bs] int32.
encoder_swa_row_floor: Optional[torch.Tensor] = None
encoder_swa_compress_skip: Optional[torch.Tensor] = None
encoder_swa_compress_rows: Optional[torch.Tensor] = None # non-replay rows, int64
# DeepSeek-V4.1 engram, extend only: the n - 1 tokens before each request's
# first extend token, oldest first, [bs, n - 1] int32 (see EngramHasher).