[Perf] deepseek-v4.1: Run encoder SWA bounded replay in the breakable prefill graph (#44930)
Co-authored-by: Vincent Zhong <207368749+vincentzed@users.noreply.github.com>
This commit is contained in:
co-authored by
Vincent Zhong
parent
a0b77db84d
commit
ff73d63121
@@ -281,8 +281,15 @@ def validate_deepseek_v41_features(server_args: ServerArgs) -> None:
|
||||
),
|
||||
),
|
||||
(
|
||||
"prefill CUDA graphs",
|
||||
cfg.cuda_graph_config.prefill.backend != Backend.DISABLED,
|
||||
"prefill CUDA graphs other than breakable",
|
||||
cfg.cuda_graph_config.prefill.backend
|
||||
not in (Backend.DISABLED, Backend.BREAKABLE),
|
||||
),
|
||||
# ROCm replays hits in a forward of their own, outside the graph.
|
||||
(
|
||||
"the prefill CUDA graph on ROCm",
|
||||
cfg.cuda_graph_config.prefill.backend != Backend.DISABLED
|
||||
and get_platform().is_hip,
|
||||
),
|
||||
("context parallelism", cfg.attn_cp_size > 1),
|
||||
# ROCm replays hits in a forward of their own, which only the hitting
|
||||
|
||||
@@ -700,6 +700,8 @@ class DSV4AttnMetadata:
|
||||
"trtllm_prefill_c4_indices",
|
||||
"trtllm_prefill_c128",
|
||||
]
|
||||
if self.request_window_layout is not None:
|
||||
self.request_window_layout.copy_(other.request_window_layout)
|
||||
# Keep graph-captured tensor objects alive for fields that captured
|
||||
# kernels read by address; overwrite only their contents.
|
||||
for field_name in tensor_copy_fields:
|
||||
@@ -1480,6 +1482,9 @@ class DeepseekV4AttnBackend(
|
||||
extend_seq_lens_tensor=extend_seq_lens,
|
||||
extend_start_loc=extend_start_loc,
|
||||
)
|
||||
graph_window = (
|
||||
use_prefill_cuda_graph and self.token_to_kv_pool.request_window is not None
|
||||
)
|
||||
core_attn_metadata = self.make_core_attn_metadata(
|
||||
req_to_token=self.req_to_token,
|
||||
req_pool_indices_repeated=req_pool_indices_repeated,
|
||||
@@ -1492,7 +1497,14 @@ class DeepseekV4AttnBackend(
|
||||
dspark_swa_buffers=dspark_swa_buffers,
|
||||
num_tokens=num_write_tokens if cp_active else None,
|
||||
swa_replay_start=swa_replay_start,
|
||||
num_groups=len(extend_seq_lens_cpu),
|
||||
# Captured kernels read the window history by address, so a graph
|
||||
# keeps one group per request slot, at most one per row.
|
||||
num_groups=(
|
||||
min(self.req_to_token.shape[0], padded_num_tokens)
|
||||
if graph_window
|
||||
else len(extend_seq_lens_cpu)
|
||||
),
|
||||
request_window_live_rows=num_tokens if graph_window else None,
|
||||
)
|
||||
if cp_active:
|
||||
core_attn_metadata.apply_cp_reindex(
|
||||
@@ -2649,8 +2661,6 @@ class DeepseekV4AttnBackend(
|
||||
self.online_c128_mtp.clear()
|
||||
return
|
||||
|
||||
self.encoder_replay = forward_batch.encoder_swa_replay
|
||||
self.encoder_row_floor = forward_batch.encoder_swa_row_floor
|
||||
self.forward_metadata = self._build_forward_metadata(forward_batch)
|
||||
self.init_forward_metadata_in_graph(forward_batch)
|
||||
self.tail_forward_metadata = (
|
||||
@@ -2660,6 +2670,9 @@ class DeepseekV4AttnBackend(
|
||||
else None
|
||||
)
|
||||
|
||||
self._activate_request_window()
|
||||
|
||||
def _activate_request_window(self) -> None:
|
||||
if self.token_to_kv_pool.request_window is not None:
|
||||
self.token_to_kv_pool.request_window.activate(
|
||||
self.forward_metadata.core_attn_metadata.request_window_layout
|
||||
@@ -2736,6 +2749,8 @@ class DeepseekV4AttnBackend(
|
||||
# Reject undersized query buffers before accessing attention metadata.
|
||||
request_layout = core_attn_metadata.request_window_layout
|
||||
assert tail is None or request_layout is not None
|
||||
if request_layout is not None:
|
||||
request_layout = request_layout.leading_rows(num_qo_tokens)
|
||||
seq_lens_cpu_list = seq_lens_cpu.tolist()
|
||||
total_swa = sum(
|
||||
min(int(seq_len), int(extend_len) + SWA_WINDOW - 1)
|
||||
@@ -2780,13 +2795,17 @@ class DeepseekV4AttnBackend(
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _compressor_out_loc(forward_batch: ForwardBatch) -> torch.Tensor:
|
||||
def _compressor_out_loc(
|
||||
forward_batch: ForwardBatch, *, own_buffer: bool = False
|
||||
) -> torch.Tensor:
|
||||
"""Slot 0 for folded replay rows: their main KV, compressed groups and index
|
||||
keys are cached, so the compressors write only the new rows."""
|
||||
keys are cached, so the compressors write only the new rows. A prefill graph
|
||||
refreshes the captured slots in place, so it takes its own buffer rather
|
||||
than the static out_cache_loc."""
|
||||
loc = forward_batch.out_cache_loc
|
||||
keep = forward_batch.encoder_swa_compress_rows
|
||||
if keep is None:
|
||||
return loc
|
||||
return loc.clone() if own_buffer else loc
|
||||
return torch.zeros_like(loc).index_copy_(0, keep, loc[keep])
|
||||
|
||||
def _build_forward_metadata(
|
||||
@@ -2796,6 +2815,9 @@ class DeepseekV4AttnBackend(
|
||||
max_seq_len_override: Optional[int] = None,
|
||||
use_prefill_cuda_graph: bool = False,
|
||||
):
|
||||
# make_core_attn_metadata reads the fold's window state from here.
|
||||
self.encoder_replay = forward_batch.encoder_swa_replay
|
||||
self.encoder_row_floor = forward_batch.encoder_swa_row_floor
|
||||
logical_forward_mode = _get_logical_forward_mode(forward_batch)
|
||||
req_pool_indices = forward_batch.req_pool_indices
|
||||
seq_lens = forward_batch.seq_lens.to(torch.int32)
|
||||
@@ -2889,7 +2911,9 @@ class DeepseekV4AttnBackend(
|
||||
req_pool_indices=req_pool_indices,
|
||||
seq_lens=seq_lens,
|
||||
seq_lens_cpu=seq_lens_cpu.tolist(),
|
||||
out_cache_loc=self._compressor_out_loc(forward_batch),
|
||||
out_cache_loc=self._compressor_out_loc(
|
||||
forward_batch, own_buffer=use_prefill_cuda_graph
|
||||
),
|
||||
num_tokens=sum(extend_seq_lens_cpu),
|
||||
extend_seq_lens=extend_seq_lens,
|
||||
extend_seq_lens_cpu=extend_seq_lens_cpu,
|
||||
@@ -2916,6 +2940,7 @@ class DeepseekV4AttnBackend(
|
||||
max_seq_len_override=max_seq_len,
|
||||
use_prefill_cuda_graph=True,
|
||||
)
|
||||
self._activate_request_window()
|
||||
with_tail = (
|
||||
self.enable_decoder_swa_bounded_replay
|
||||
and forward_batch.forward_mode.is_extend_without_speculative()
|
||||
@@ -3008,6 +3033,7 @@ class DeepseekV4AttnBackend(
|
||||
assert isinstance(capture_metadata, DSV4Metadata)
|
||||
capture_metadata.refresh_for_breakable_cuda_graph_replay_(static_metadata)
|
||||
self.forward_metadata = capture_metadata
|
||||
self._activate_request_window()
|
||||
self.tail_forward_metadata = (
|
||||
self._prefill_graph_tail_metadata(metadata_batch, max_seq_len)
|
||||
if capture_metadata.prefill_graph_tail
|
||||
@@ -4356,6 +4382,7 @@ class DeepseekV4AttnBackend(
|
||||
swa_replay_start: Optional[torch.Tensor] = None,
|
||||
num_groups: Optional[int] = None,
|
||||
dspark_swa_buffers: Optional[Tuple[torch.Tensor, torch.Tensor]] = None,
|
||||
request_window_live_rows: Optional[int] = None,
|
||||
) -> DSV4AttnMetadata:
|
||||
small_metadata = (
|
||||
not is_prefill
|
||||
@@ -4409,12 +4436,14 @@ class DeepseekV4AttnBackend(
|
||||
swa_replay_start = _pad_rows(
|
||||
self.encoder_row_floor, num_rows=raw_positions.shape[0]
|
||||
)
|
||||
live = request_window_live_rows
|
||||
request_layout = window_layout(
|
||||
req_pool_indices_repeated,
|
||||
raw_positions,
|
||||
req_pool_indices_repeated[:live],
|
||||
raw_positions[:live],
|
||||
capacity=self.token_to_kv_pool.request_window.capacity,
|
||||
floor=swa_replay_start,
|
||||
floor=None if swa_replay_start is None else swa_replay_start[:live],
|
||||
num_groups=num_groups,
|
||||
padded_rows=None if live is None else raw_positions.numel(),
|
||||
)
|
||||
swa_page_indices = _pad_last_dim(request_layout.indices)
|
||||
swa_topk_lengths = request_layout.lengths
|
||||
|
||||
@@ -384,6 +384,7 @@ def _local_decode_cuda_graph_vote(
|
||||
def _local_prefill_cuda_graph_vote(
|
||||
*,
|
||||
local_batch: Optional[ScheduleBatch],
|
||||
num_tokens: int,
|
||||
prefill_graph_runner,
|
||||
coordinated_prefill: bool,
|
||||
breakable_prefill: bool,
|
||||
@@ -393,7 +394,9 @@ def _local_prefill_cuda_graph_vote(
|
||||
"""This rank's vote for the prefill graph (min-reduced across dp
|
||||
ranks). Extend and mixed batches share the runner's rank-local replay
|
||||
policy. A decode batch eligible for the decode->extend conversion votes as
|
||||
its 1-token-extend view, so the vote and post-sync conversion agree."""
|
||||
its 1-token-extend view, so the vote and post-sync conversion agree.
|
||||
num_tokens is the rows this rank gathers, encoder replay rows folded in,
|
||||
which an extend replay runs."""
|
||||
if local_batch is None or local_batch.forward_mode.is_idle():
|
||||
return True
|
||||
if not coordinated_prefill:
|
||||
@@ -401,7 +404,6 @@ def _local_prefill_cuda_graph_vote(
|
||||
|
||||
mode = local_batch.forward_mode
|
||||
if mode in (ForwardMode.EXTEND, ForwardMode.MIXED):
|
||||
num_tokens = local_batch.extend_num_tokens
|
||||
input_embeds = local_batch.input_embeds
|
||||
replace_embeds = local_batch.replace_embeds
|
||||
prefix_lens = local_batch.prefix_lens
|
||||
@@ -579,6 +581,7 @@ def prepare_mlp_sync_batch_raw(
|
||||
)
|
||||
can_run_prefill_cuda_graph = _local_prefill_cuda_graph_vote(
|
||||
local_batch=local_batch,
|
||||
num_tokens=num_tokens,
|
||||
prefill_graph_runner=prefill_graph_runner,
|
||||
coordinated_prefill=coordinated_prefill,
|
||||
breakable_prefill=breakable_prefill,
|
||||
|
||||
@@ -39,8 +39,10 @@ from sglang.srt.layers.attention.hip_flash_mla import resolve_hip_flashmla_backe
|
||||
from sglang.srt.mem_cache.base_swa_memory_pool import BaseSWAKVPool
|
||||
from sglang.srt.mem_cache.deepseek_v4_compress_state import CompressStatePool
|
||||
from sglang.srt.mem_cache.memory_pool import KVCache
|
||||
from sglang.srt.model_executor.cuda_graph_config import Backend
|
||||
from sglang.srt.runtime_context import get_exec, get_platform, get_spec
|
||||
from sglang.srt.utils import ceil_div, is_gfx95_supported, is_hip
|
||||
from sglang.srt.utils.common import get_cuda_graph_max_batch_size
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -1138,6 +1140,17 @@ class DeepSeekV4TokenToKVPool(BaseSWAKVPool):
|
||||
from sglang.srt.runtime_context import get_schedule
|
||||
|
||||
chunk = get_schedule().chunked_prefill_size or 0
|
||||
# A prefill graph pads its layouts to the captured bucket, which can
|
||||
# exceed this rank's chunk (DP attention divides the chunk budget).
|
||||
graph_config = get_exec().graph.cuda_graph_config
|
||||
prefill_graph = graph_config.prefill if graph_config is not None else None
|
||||
graph_rows = (
|
||||
get_cuda_graph_max_batch_size(max(prefill_graph.bs))
|
||||
if prefill_graph is not None
|
||||
and prefill_graph.backend == Backend.BREAKABLE
|
||||
and prefill_graph.bs
|
||||
else 0
|
||||
)
|
||||
self.request_window = RequestWindow(
|
||||
make_window_pool,
|
||||
num_slots=self.num_req_slots,
|
||||
@@ -1147,6 +1160,7 @@ class DeepSeekV4TokenToKVPool(BaseSWAKVPool):
|
||||
workspace_rows=(self.num_req_slots + 1) * self.sliding_window
|
||||
+ max(
|
||||
chunk,
|
||||
graph_rows,
|
||||
(self.num_req_slots + 1) * (1 + (online_mtp_max_draft_tokens or 0)),
|
||||
),
|
||||
)
|
||||
|
||||
@@ -24,6 +24,20 @@ class WindowLayout(msgspec.Struct, frozen=True):
|
||||
commit_mask: torch.Tensor
|
||||
size: int
|
||||
|
||||
def leading_rows(self, num_rows: int) -> "WindowLayout":
|
||||
"""The layout of the live queries; a prefill CUDA graph pads the rest."""
|
||||
if num_rows == self.pos.numel():
|
||||
return self
|
||||
return msgspec.structs.replace(
|
||||
self,
|
||||
req=self.req[:num_rows],
|
||||
pos=self.pos[:num_rows],
|
||||
write_loc=self.write_loc[:num_rows],
|
||||
indices=self.indices[:num_rows],
|
||||
lengths=self.lengths[:num_rows],
|
||||
commit_mask=self.commit_mask[:num_rows],
|
||||
)
|
||||
|
||||
def copy_(self, other: "WindowLayout") -> None:
|
||||
# Captured copy kernels read these tensors by address, so a graph replay
|
||||
# must refresh their contents in place, not rebind the object.
|
||||
@@ -65,7 +79,10 @@ def window_layout(
|
||||
capacity: int = 256,
|
||||
floor: Optional[torch.Tensor] = None,
|
||||
num_groups: Optional[int] = None,
|
||||
padded_rows: Optional[int] = None,
|
||||
):
|
||||
"""padded_rows is set for the prefill CUDA graph, which pads the layout to that
|
||||
many query rows past the live ones in req and pos."""
|
||||
n = pos.numel()
|
||||
if n == 0:
|
||||
raise ValueError("request-window layout needs at least one query")
|
||||
@@ -111,6 +128,26 @@ def window_layout(
|
||||
history_loc = torch.arange(history_rows, device=device)
|
||||
|
||||
commit_mask = (group_last - offset) < capacity
|
||||
if padded_rows is not None and padded_rows > n:
|
||||
# Padding rows write the workspace past the live rows and commit nowhere.
|
||||
padding = padded_rows - n
|
||||
zeros = req.new_zeros(padding)
|
||||
req, pos = torch.cat([req, zeros]), torch.cat([pos, zeros])
|
||||
write_loc = torch.cat(
|
||||
[
|
||||
write_loc,
|
||||
torch.arange(
|
||||
history_rows + n,
|
||||
history_rows + padded_rows,
|
||||
dtype=torch.int32,
|
||||
device=device,
|
||||
),
|
||||
]
|
||||
)
|
||||
indices = torch.cat([indices, indices.new_full((padding, window), -1)])
|
||||
lengths = torch.cat([lengths, lengths.new_zeros(padding)])
|
||||
commit_mask = torch.cat([commit_mask, commit_mask.new_zeros(padding)])
|
||||
n = padded_rows
|
||||
return WindowLayout(
|
||||
req,
|
||||
pos,
|
||||
|
||||
@@ -2004,6 +2004,13 @@ class PrefillCudaGraphRunner(BaseCudaGraphRunner):
|
||||
# The n-gram hasher runs outside the graph and reads this at replay.
|
||||
static_forward_batch.ngram_embedding_info = forward_batch.ngram_embedding_info
|
||||
static_forward_batch.engram_history = forward_batch.engram_history
|
||||
# The encoder replay fold: a replay builds the same window floors and
|
||||
# compressor slots as an eager prefill.
|
||||
static_forward_batch.encoder_swa_replay = forward_batch.encoder_swa_replay
|
||||
static_forward_batch.encoder_swa_row_floor = forward_batch.encoder_swa_row_floor
|
||||
static_forward_batch.encoder_swa_compress_rows = (
|
||||
forward_batch.encoder_swa_compress_rows
|
||||
)
|
||||
static_forward_batch = self.model_runner.prepare_dummy_forward_batch(
|
||||
static_forward_batch
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user