[Perf] deepseek-v4.1: Run encoder SWA bounded replay in the breakable prefill graph (#44930)

Co-authored-by: Vincent Zhong <207368749+vincentzed@users.noreply.github.com>
This commit is contained in:
Liangsheng Yin
2026-10-17 20:51:18 -07:00
committed by GitHub
co-authored by Vincent Zhong
parent a0b77db84d
commit ff73d63121
11 changed files with 285 additions and 49 deletions
@@ -281,8 +281,15 @@ def validate_deepseek_v41_features(server_args: ServerArgs) -> None:
),
),
(
"prefill CUDA graphs",
cfg.cuda_graph_config.prefill.backend != Backend.DISABLED,
"prefill CUDA graphs other than breakable",
cfg.cuda_graph_config.prefill.backend
not in (Backend.DISABLED, Backend.BREAKABLE),
),
# ROCm replays hits in a forward of their own, outside the graph.
(
"the prefill CUDA graph on ROCm",
cfg.cuda_graph_config.prefill.backend != Backend.DISABLED
and get_platform().is_hip,
),
("context parallelism", cfg.attn_cp_size > 1),
# ROCm replays hits in a forward of their own, which only the hitting
@@ -700,6 +700,8 @@ class DSV4AttnMetadata:
"trtllm_prefill_c4_indices",
"trtllm_prefill_c128",
]
if self.request_window_layout is not None:
self.request_window_layout.copy_(other.request_window_layout)
# Keep graph-captured tensor objects alive for fields that captured
# kernels read by address; overwrite only their contents.
for field_name in tensor_copy_fields:
@@ -1480,6 +1482,9 @@ class DeepseekV4AttnBackend(
extend_seq_lens_tensor=extend_seq_lens,
extend_start_loc=extend_start_loc,
)
graph_window = (
use_prefill_cuda_graph and self.token_to_kv_pool.request_window is not None
)
core_attn_metadata = self.make_core_attn_metadata(
req_to_token=self.req_to_token,
req_pool_indices_repeated=req_pool_indices_repeated,
@@ -1492,7 +1497,14 @@ class DeepseekV4AttnBackend(
dspark_swa_buffers=dspark_swa_buffers,
num_tokens=num_write_tokens if cp_active else None,
swa_replay_start=swa_replay_start,
num_groups=len(extend_seq_lens_cpu),
# Captured kernels read the window history by address, so a graph
# keeps one group per request slot, at most one per row.
num_groups=(
min(self.req_to_token.shape[0], padded_num_tokens)
if graph_window
else len(extend_seq_lens_cpu)
),
request_window_live_rows=num_tokens if graph_window else None,
)
if cp_active:
core_attn_metadata.apply_cp_reindex(
@@ -2649,8 +2661,6 @@ class DeepseekV4AttnBackend(
self.online_c128_mtp.clear()
return
self.encoder_replay = forward_batch.encoder_swa_replay
self.encoder_row_floor = forward_batch.encoder_swa_row_floor
self.forward_metadata = self._build_forward_metadata(forward_batch)
self.init_forward_metadata_in_graph(forward_batch)
self.tail_forward_metadata = (
@@ -2660,6 +2670,9 @@ class DeepseekV4AttnBackend(
else None
)
self._activate_request_window()
def _activate_request_window(self) -> None:
if self.token_to_kv_pool.request_window is not None:
self.token_to_kv_pool.request_window.activate(
self.forward_metadata.core_attn_metadata.request_window_layout
@@ -2736,6 +2749,8 @@ class DeepseekV4AttnBackend(
# Reject undersized query buffers before accessing attention metadata.
request_layout = core_attn_metadata.request_window_layout
assert tail is None or request_layout is not None
if request_layout is not None:
request_layout = request_layout.leading_rows(num_qo_tokens)
seq_lens_cpu_list = seq_lens_cpu.tolist()
total_swa = sum(
min(int(seq_len), int(extend_len) + SWA_WINDOW - 1)
@@ -2780,13 +2795,17 @@ class DeepseekV4AttnBackend(
)
@staticmethod
def _compressor_out_loc(forward_batch: ForwardBatch) -> torch.Tensor:
def _compressor_out_loc(
forward_batch: ForwardBatch, *, own_buffer: bool = False
) -> torch.Tensor:
"""Slot 0 for folded replay rows: their main KV, compressed groups and index
keys are cached, so the compressors write only the new rows."""
keys are cached, so the compressors write only the new rows. A prefill graph
refreshes the captured slots in place, so it takes its own buffer rather
than the static out_cache_loc."""
loc = forward_batch.out_cache_loc
keep = forward_batch.encoder_swa_compress_rows
if keep is None:
return loc
return loc.clone() if own_buffer else loc
return torch.zeros_like(loc).index_copy_(0, keep, loc[keep])
def _build_forward_metadata(
@@ -2796,6 +2815,9 @@ class DeepseekV4AttnBackend(
max_seq_len_override: Optional[int] = None,
use_prefill_cuda_graph: bool = False,
):
# make_core_attn_metadata reads the fold's window state from here.
self.encoder_replay = forward_batch.encoder_swa_replay
self.encoder_row_floor = forward_batch.encoder_swa_row_floor
logical_forward_mode = _get_logical_forward_mode(forward_batch)
req_pool_indices = forward_batch.req_pool_indices
seq_lens = forward_batch.seq_lens.to(torch.int32)
@@ -2889,7 +2911,9 @@ class DeepseekV4AttnBackend(
req_pool_indices=req_pool_indices,
seq_lens=seq_lens,
seq_lens_cpu=seq_lens_cpu.tolist(),
out_cache_loc=self._compressor_out_loc(forward_batch),
out_cache_loc=self._compressor_out_loc(
forward_batch, own_buffer=use_prefill_cuda_graph
),
num_tokens=sum(extend_seq_lens_cpu),
extend_seq_lens=extend_seq_lens,
extend_seq_lens_cpu=extend_seq_lens_cpu,
@@ -2916,6 +2940,7 @@ class DeepseekV4AttnBackend(
max_seq_len_override=max_seq_len,
use_prefill_cuda_graph=True,
)
self._activate_request_window()
with_tail = (
self.enable_decoder_swa_bounded_replay
and forward_batch.forward_mode.is_extend_without_speculative()
@@ -3008,6 +3033,7 @@ class DeepseekV4AttnBackend(
assert isinstance(capture_metadata, DSV4Metadata)
capture_metadata.refresh_for_breakable_cuda_graph_replay_(static_metadata)
self.forward_metadata = capture_metadata
self._activate_request_window()
self.tail_forward_metadata = (
self._prefill_graph_tail_metadata(metadata_batch, max_seq_len)
if capture_metadata.prefill_graph_tail
@@ -4356,6 +4382,7 @@ class DeepseekV4AttnBackend(
swa_replay_start: Optional[torch.Tensor] = None,
num_groups: Optional[int] = None,
dspark_swa_buffers: Optional[Tuple[torch.Tensor, torch.Tensor]] = None,
request_window_live_rows: Optional[int] = None,
) -> DSV4AttnMetadata:
small_metadata = (
not is_prefill
@@ -4409,12 +4436,14 @@ class DeepseekV4AttnBackend(
swa_replay_start = _pad_rows(
self.encoder_row_floor, num_rows=raw_positions.shape[0]
)
live = request_window_live_rows
request_layout = window_layout(
req_pool_indices_repeated,
raw_positions,
req_pool_indices_repeated[:live],
raw_positions[:live],
capacity=self.token_to_kv_pool.request_window.capacity,
floor=swa_replay_start,
floor=None if swa_replay_start is None else swa_replay_start[:live],
num_groups=num_groups,
padded_rows=None if live is None else raw_positions.numel(),
)
swa_page_indices = _pad_last_dim(request_layout.indices)
swa_topk_lengths = request_layout.lengths
@@ -384,6 +384,7 @@ def _local_decode_cuda_graph_vote(
def _local_prefill_cuda_graph_vote(
*,
local_batch: Optional[ScheduleBatch],
num_tokens: int,
prefill_graph_runner,
coordinated_prefill: bool,
breakable_prefill: bool,
@@ -393,7 +394,9 @@ def _local_prefill_cuda_graph_vote(
"""This rank's vote for the prefill graph (min-reduced across dp
ranks). Extend and mixed batches share the runner's rank-local replay
policy. A decode batch eligible for the decode->extend conversion votes as
its 1-token-extend view, so the vote and post-sync conversion agree."""
its 1-token-extend view, so the vote and post-sync conversion agree.
num_tokens is the rows this rank gathers, encoder replay rows folded in,
which an extend replay runs."""
if local_batch is None or local_batch.forward_mode.is_idle():
return True
if not coordinated_prefill:
@@ -401,7 +404,6 @@ def _local_prefill_cuda_graph_vote(
mode = local_batch.forward_mode
if mode in (ForwardMode.EXTEND, ForwardMode.MIXED):
num_tokens = local_batch.extend_num_tokens
input_embeds = local_batch.input_embeds
replace_embeds = local_batch.replace_embeds
prefix_lens = local_batch.prefix_lens
@@ -579,6 +581,7 @@ def prepare_mlp_sync_batch_raw(
)
can_run_prefill_cuda_graph = _local_prefill_cuda_graph_vote(
local_batch=local_batch,
num_tokens=num_tokens,
prefill_graph_runner=prefill_graph_runner,
coordinated_prefill=coordinated_prefill,
breakable_prefill=breakable_prefill,
@@ -39,8 +39,10 @@ from sglang.srt.layers.attention.hip_flash_mla import resolve_hip_flashmla_backe
from sglang.srt.mem_cache.base_swa_memory_pool import BaseSWAKVPool
from sglang.srt.mem_cache.deepseek_v4_compress_state import CompressStatePool
from sglang.srt.mem_cache.memory_pool import KVCache
from sglang.srt.model_executor.cuda_graph_config import Backend
from sglang.srt.runtime_context import get_exec, get_platform, get_spec
from sglang.srt.utils import ceil_div, is_gfx95_supported, is_hip
from sglang.srt.utils.common import get_cuda_graph_max_batch_size
logger = logging.getLogger(__name__)
@@ -1138,6 +1140,17 @@ class DeepSeekV4TokenToKVPool(BaseSWAKVPool):
from sglang.srt.runtime_context import get_schedule
chunk = get_schedule().chunked_prefill_size or 0
# A prefill graph pads its layouts to the captured bucket, which can
# exceed this rank's chunk (DP attention divides the chunk budget).
graph_config = get_exec().graph.cuda_graph_config
prefill_graph = graph_config.prefill if graph_config is not None else None
graph_rows = (
get_cuda_graph_max_batch_size(max(prefill_graph.bs))
if prefill_graph is not None
and prefill_graph.backend == Backend.BREAKABLE
and prefill_graph.bs
else 0
)
self.request_window = RequestWindow(
make_window_pool,
num_slots=self.num_req_slots,
@@ -1147,6 +1160,7 @@ class DeepSeekV4TokenToKVPool(BaseSWAKVPool):
workspace_rows=(self.num_req_slots + 1) * self.sliding_window
+ max(
chunk,
graph_rows,
(self.num_req_slots + 1) * (1 + (online_mtp_max_draft_tokens or 0)),
),
)
@@ -24,6 +24,20 @@ class WindowLayout(msgspec.Struct, frozen=True):
commit_mask: torch.Tensor
size: int
def leading_rows(self, num_rows: int) -> "WindowLayout":
"""The layout of the live queries; a prefill CUDA graph pads the rest."""
if num_rows == self.pos.numel():
return self
return msgspec.structs.replace(
self,
req=self.req[:num_rows],
pos=self.pos[:num_rows],
write_loc=self.write_loc[:num_rows],
indices=self.indices[:num_rows],
lengths=self.lengths[:num_rows],
commit_mask=self.commit_mask[:num_rows],
)
def copy_(self, other: "WindowLayout") -> None:
# Captured copy kernels read these tensors by address, so a graph replay
# must refresh their contents in place, not rebind the object.
@@ -65,7 +79,10 @@ def window_layout(
capacity: int = 256,
floor: Optional[torch.Tensor] = None,
num_groups: Optional[int] = None,
padded_rows: Optional[int] = None,
):
"""padded_rows is set for the prefill CUDA graph, which pads the layout to that
many query rows past the live ones in req and pos."""
n = pos.numel()
if n == 0:
raise ValueError("request-window layout needs at least one query")
@@ -111,6 +128,26 @@ def window_layout(
history_loc = torch.arange(history_rows, device=device)
commit_mask = (group_last - offset) < capacity
if padded_rows is not None and padded_rows > n:
# Padding rows write the workspace past the live rows and commit nowhere.
padding = padded_rows - n
zeros = req.new_zeros(padding)
req, pos = torch.cat([req, zeros]), torch.cat([pos, zeros])
write_loc = torch.cat(
[
write_loc,
torch.arange(
history_rows + n,
history_rows + padded_rows,
dtype=torch.int32,
device=device,
),
]
)
indices = torch.cat([indices, indices.new_full((padding, window), -1)])
lengths = torch.cat([lengths, lengths.new_zeros(padding)])
commit_mask = torch.cat([commit_mask, commit_mask.new_zeros(padding)])
n = padded_rows
return WindowLayout(
req,
pos,
@@ -2004,6 +2004,13 @@ class PrefillCudaGraphRunner(BaseCudaGraphRunner):
# The n-gram hasher runs outside the graph and reads this at replay.
static_forward_batch.ngram_embedding_info = forward_batch.ngram_embedding_info
static_forward_batch.engram_history = forward_batch.engram_history
# The encoder replay fold: a replay builds the same window floors and
# compressor slots as an eager prefill.
static_forward_batch.encoder_swa_replay = forward_batch.encoder_swa_replay
static_forward_batch.encoder_swa_row_floor = forward_batch.encoder_swa_row_floor
static_forward_batch.encoder_swa_compress_rows = (
forward_batch.encoder_swa_compress_rows
)
static_forward_batch = self.model_runner.prepare_dummy_forward_batch(
static_forward_batch
)