From ff73d63121e9fc28f430a37700920115762b9938 Mon Sep 17 00:00:00 2001 From: Liangsheng Yin Date: Sat, 17 Oct 2026 20:51:18 -0700 Subject: [PATCH] [Perf] deepseek-v4.1: Run encoder SWA bounded replay in the breakable prefill graph (#44930) Co-authored-by: Vincent Zhong <207368749+vincentzed@users.noreply.github.com> --- .../sglang/srt/arg_groups/deepseek_v4_hook.py | 11 +- .../layers/attention/deepseek_v4_backend.py | 49 ++++- .../managers/scheduler_components/dp_attn.py | 7 +- .../srt/mem_cache/deepseek_v4_memory_pool.py | 14 ++ .../srt/mem_cache/dsv41_request_window.py | 37 ++++ .../runner/prefill_cuda_graph_runner.py | 7 + .../scheduler_components/test_dp_attn.py | 3 + .../mem_cache/test_dsv4_compressed_pools.py | 28 ++- .../runner/test_prefill_cuda_graph_padding.py | 1 + .../model_executor/test_encoder_swa_replay.py | 176 ++++++++++++++---- .../test_prefill_cuda_graph_runner.py | 1 + 11 files changed, 285 insertions(+), 49 deletions(-) diff --git a/python/sglang/srt/arg_groups/deepseek_v4_hook.py b/python/sglang/srt/arg_groups/deepseek_v4_hook.py index d8a87e0c72..9dbc45d8cc 100644 --- a/python/sglang/srt/arg_groups/deepseek_v4_hook.py +++ b/python/sglang/srt/arg_groups/deepseek_v4_hook.py @@ -281,8 +281,15 @@ def validate_deepseek_v41_features(server_args: ServerArgs) -> None: ), ), ( - "prefill CUDA graphs", - cfg.cuda_graph_config.prefill.backend != Backend.DISABLED, + "prefill CUDA graphs other than breakable", + cfg.cuda_graph_config.prefill.backend + not in (Backend.DISABLED, Backend.BREAKABLE), + ), + # ROCm replays hits in a forward of their own, outside the graph. + ( + "the prefill CUDA graph on ROCm", + cfg.cuda_graph_config.prefill.backend != Backend.DISABLED + and get_platform().is_hip, ), ("context parallelism", cfg.attn_cp_size > 1), # ROCm replays hits in a forward of their own, which only the hitting diff --git a/python/sglang/srt/layers/attention/deepseek_v4_backend.py b/python/sglang/srt/layers/attention/deepseek_v4_backend.py index 75af46e40c..9dbdfeb31b 100644 --- a/python/sglang/srt/layers/attention/deepseek_v4_backend.py +++ b/python/sglang/srt/layers/attention/deepseek_v4_backend.py @@ -700,6 +700,8 @@ class DSV4AttnMetadata: "trtllm_prefill_c4_indices", "trtllm_prefill_c128", ] + if self.request_window_layout is not None: + self.request_window_layout.copy_(other.request_window_layout) # Keep graph-captured tensor objects alive for fields that captured # kernels read by address; overwrite only their contents. for field_name in tensor_copy_fields: @@ -1480,6 +1482,9 @@ class DeepseekV4AttnBackend( extend_seq_lens_tensor=extend_seq_lens, extend_start_loc=extend_start_loc, ) + graph_window = ( + use_prefill_cuda_graph and self.token_to_kv_pool.request_window is not None + ) core_attn_metadata = self.make_core_attn_metadata( req_to_token=self.req_to_token, req_pool_indices_repeated=req_pool_indices_repeated, @@ -1492,7 +1497,14 @@ class DeepseekV4AttnBackend( dspark_swa_buffers=dspark_swa_buffers, num_tokens=num_write_tokens if cp_active else None, swa_replay_start=swa_replay_start, - num_groups=len(extend_seq_lens_cpu), + # Captured kernels read the window history by address, so a graph + # keeps one group per request slot, at most one per row. + num_groups=( + min(self.req_to_token.shape[0], padded_num_tokens) + if graph_window + else len(extend_seq_lens_cpu) + ), + request_window_live_rows=num_tokens if graph_window else None, ) if cp_active: core_attn_metadata.apply_cp_reindex( @@ -2649,8 +2661,6 @@ class DeepseekV4AttnBackend( self.online_c128_mtp.clear() return - self.encoder_replay = forward_batch.encoder_swa_replay - self.encoder_row_floor = forward_batch.encoder_swa_row_floor self.forward_metadata = self._build_forward_metadata(forward_batch) self.init_forward_metadata_in_graph(forward_batch) self.tail_forward_metadata = ( @@ -2660,6 +2670,9 @@ class DeepseekV4AttnBackend( else None ) + self._activate_request_window() + + def _activate_request_window(self) -> None: if self.token_to_kv_pool.request_window is not None: self.token_to_kv_pool.request_window.activate( self.forward_metadata.core_attn_metadata.request_window_layout @@ -2736,6 +2749,8 @@ class DeepseekV4AttnBackend( # Reject undersized query buffers before accessing attention metadata. request_layout = core_attn_metadata.request_window_layout assert tail is None or request_layout is not None + if request_layout is not None: + request_layout = request_layout.leading_rows(num_qo_tokens) seq_lens_cpu_list = seq_lens_cpu.tolist() total_swa = sum( min(int(seq_len), int(extend_len) + SWA_WINDOW - 1) @@ -2780,13 +2795,17 @@ class DeepseekV4AttnBackend( ) @staticmethod - def _compressor_out_loc(forward_batch: ForwardBatch) -> torch.Tensor: + def _compressor_out_loc( + forward_batch: ForwardBatch, *, own_buffer: bool = False + ) -> torch.Tensor: """Slot 0 for folded replay rows: their main KV, compressed groups and index - keys are cached, so the compressors write only the new rows.""" + keys are cached, so the compressors write only the new rows. A prefill graph + refreshes the captured slots in place, so it takes its own buffer rather + than the static out_cache_loc.""" loc = forward_batch.out_cache_loc keep = forward_batch.encoder_swa_compress_rows if keep is None: - return loc + return loc.clone() if own_buffer else loc return torch.zeros_like(loc).index_copy_(0, keep, loc[keep]) def _build_forward_metadata( @@ -2796,6 +2815,9 @@ class DeepseekV4AttnBackend( max_seq_len_override: Optional[int] = None, use_prefill_cuda_graph: bool = False, ): + # make_core_attn_metadata reads the fold's window state from here. + self.encoder_replay = forward_batch.encoder_swa_replay + self.encoder_row_floor = forward_batch.encoder_swa_row_floor logical_forward_mode = _get_logical_forward_mode(forward_batch) req_pool_indices = forward_batch.req_pool_indices seq_lens = forward_batch.seq_lens.to(torch.int32) @@ -2889,7 +2911,9 @@ class DeepseekV4AttnBackend( req_pool_indices=req_pool_indices, seq_lens=seq_lens, seq_lens_cpu=seq_lens_cpu.tolist(), - out_cache_loc=self._compressor_out_loc(forward_batch), + out_cache_loc=self._compressor_out_loc( + forward_batch, own_buffer=use_prefill_cuda_graph + ), num_tokens=sum(extend_seq_lens_cpu), extend_seq_lens=extend_seq_lens, extend_seq_lens_cpu=extend_seq_lens_cpu, @@ -2916,6 +2940,7 @@ class DeepseekV4AttnBackend( max_seq_len_override=max_seq_len, use_prefill_cuda_graph=True, ) + self._activate_request_window() with_tail = ( self.enable_decoder_swa_bounded_replay and forward_batch.forward_mode.is_extend_without_speculative() @@ -3008,6 +3033,7 @@ class DeepseekV4AttnBackend( assert isinstance(capture_metadata, DSV4Metadata) capture_metadata.refresh_for_breakable_cuda_graph_replay_(static_metadata) self.forward_metadata = capture_metadata + self._activate_request_window() self.tail_forward_metadata = ( self._prefill_graph_tail_metadata(metadata_batch, max_seq_len) if capture_metadata.prefill_graph_tail @@ -4356,6 +4382,7 @@ class DeepseekV4AttnBackend( swa_replay_start: Optional[torch.Tensor] = None, num_groups: Optional[int] = None, dspark_swa_buffers: Optional[Tuple[torch.Tensor, torch.Tensor]] = None, + request_window_live_rows: Optional[int] = None, ) -> DSV4AttnMetadata: small_metadata = ( not is_prefill @@ -4409,12 +4436,14 @@ class DeepseekV4AttnBackend( swa_replay_start = _pad_rows( self.encoder_row_floor, num_rows=raw_positions.shape[0] ) + live = request_window_live_rows request_layout = window_layout( - req_pool_indices_repeated, - raw_positions, + req_pool_indices_repeated[:live], + raw_positions[:live], capacity=self.token_to_kv_pool.request_window.capacity, - floor=swa_replay_start, + floor=None if swa_replay_start is None else swa_replay_start[:live], num_groups=num_groups, + padded_rows=None if live is None else raw_positions.numel(), ) swa_page_indices = _pad_last_dim(request_layout.indices) swa_topk_lengths = request_layout.lengths diff --git a/python/sglang/srt/managers/scheduler_components/dp_attn.py b/python/sglang/srt/managers/scheduler_components/dp_attn.py index 36f0ce00ba..bd72eaf906 100644 --- a/python/sglang/srt/managers/scheduler_components/dp_attn.py +++ b/python/sglang/srt/managers/scheduler_components/dp_attn.py @@ -384,6 +384,7 @@ def _local_decode_cuda_graph_vote( def _local_prefill_cuda_graph_vote( *, local_batch: Optional[ScheduleBatch], + num_tokens: int, prefill_graph_runner, coordinated_prefill: bool, breakable_prefill: bool, @@ -393,7 +394,9 @@ def _local_prefill_cuda_graph_vote( """This rank's vote for the prefill graph (min-reduced across dp ranks). Extend and mixed batches share the runner's rank-local replay policy. A decode batch eligible for the decode->extend conversion votes as - its 1-token-extend view, so the vote and post-sync conversion agree.""" + its 1-token-extend view, so the vote and post-sync conversion agree. + num_tokens is the rows this rank gathers, encoder replay rows folded in, + which an extend replay runs.""" if local_batch is None or local_batch.forward_mode.is_idle(): return True if not coordinated_prefill: @@ -401,7 +404,6 @@ def _local_prefill_cuda_graph_vote( mode = local_batch.forward_mode if mode in (ForwardMode.EXTEND, ForwardMode.MIXED): - num_tokens = local_batch.extend_num_tokens input_embeds = local_batch.input_embeds replace_embeds = local_batch.replace_embeds prefix_lens = local_batch.prefix_lens @@ -579,6 +581,7 @@ def prepare_mlp_sync_batch_raw( ) can_run_prefill_cuda_graph = _local_prefill_cuda_graph_vote( local_batch=local_batch, + num_tokens=num_tokens, prefill_graph_runner=prefill_graph_runner, coordinated_prefill=coordinated_prefill, breakable_prefill=breakable_prefill, diff --git a/python/sglang/srt/mem_cache/deepseek_v4_memory_pool.py b/python/sglang/srt/mem_cache/deepseek_v4_memory_pool.py index 59a9e74056..ba2885a649 100644 --- a/python/sglang/srt/mem_cache/deepseek_v4_memory_pool.py +++ b/python/sglang/srt/mem_cache/deepseek_v4_memory_pool.py @@ -39,8 +39,10 @@ from sglang.srt.layers.attention.hip_flash_mla import resolve_hip_flashmla_backe from sglang.srt.mem_cache.base_swa_memory_pool import BaseSWAKVPool from sglang.srt.mem_cache.deepseek_v4_compress_state import CompressStatePool from sglang.srt.mem_cache.memory_pool import KVCache +from sglang.srt.model_executor.cuda_graph_config import Backend from sglang.srt.runtime_context import get_exec, get_platform, get_spec from sglang.srt.utils import ceil_div, is_gfx95_supported, is_hip +from sglang.srt.utils.common import get_cuda_graph_max_batch_size logger = logging.getLogger(__name__) @@ -1138,6 +1140,17 @@ class DeepSeekV4TokenToKVPool(BaseSWAKVPool): from sglang.srt.runtime_context import get_schedule chunk = get_schedule().chunked_prefill_size or 0 + # A prefill graph pads its layouts to the captured bucket, which can + # exceed this rank's chunk (DP attention divides the chunk budget). + graph_config = get_exec().graph.cuda_graph_config + prefill_graph = graph_config.prefill if graph_config is not None else None + graph_rows = ( + get_cuda_graph_max_batch_size(max(prefill_graph.bs)) + if prefill_graph is not None + and prefill_graph.backend == Backend.BREAKABLE + and prefill_graph.bs + else 0 + ) self.request_window = RequestWindow( make_window_pool, num_slots=self.num_req_slots, @@ -1147,6 +1160,7 @@ class DeepSeekV4TokenToKVPool(BaseSWAKVPool): workspace_rows=(self.num_req_slots + 1) * self.sliding_window + max( chunk, + graph_rows, (self.num_req_slots + 1) * (1 + (online_mtp_max_draft_tokens or 0)), ), ) diff --git a/python/sglang/srt/mem_cache/dsv41_request_window.py b/python/sglang/srt/mem_cache/dsv41_request_window.py index 5180aec572..ea8cad7d4c 100644 --- a/python/sglang/srt/mem_cache/dsv41_request_window.py +++ b/python/sglang/srt/mem_cache/dsv41_request_window.py @@ -24,6 +24,20 @@ class WindowLayout(msgspec.Struct, frozen=True): commit_mask: torch.Tensor size: int + def leading_rows(self, num_rows: int) -> "WindowLayout": + """The layout of the live queries; a prefill CUDA graph pads the rest.""" + if num_rows == self.pos.numel(): + return self + return msgspec.structs.replace( + self, + req=self.req[:num_rows], + pos=self.pos[:num_rows], + write_loc=self.write_loc[:num_rows], + indices=self.indices[:num_rows], + lengths=self.lengths[:num_rows], + commit_mask=self.commit_mask[:num_rows], + ) + def copy_(self, other: "WindowLayout") -> None: # Captured copy kernels read these tensors by address, so a graph replay # must refresh their contents in place, not rebind the object. @@ -65,7 +79,10 @@ def window_layout( capacity: int = 256, floor: Optional[torch.Tensor] = None, num_groups: Optional[int] = None, + padded_rows: Optional[int] = None, ): + """padded_rows is set for the prefill CUDA graph, which pads the layout to that + many query rows past the live ones in req and pos.""" n = pos.numel() if n == 0: raise ValueError("request-window layout needs at least one query") @@ -111,6 +128,26 @@ def window_layout( history_loc = torch.arange(history_rows, device=device) commit_mask = (group_last - offset) < capacity + if padded_rows is not None and padded_rows > n: + # Padding rows write the workspace past the live rows and commit nowhere. + padding = padded_rows - n + zeros = req.new_zeros(padding) + req, pos = torch.cat([req, zeros]), torch.cat([pos, zeros]) + write_loc = torch.cat( + [ + write_loc, + torch.arange( + history_rows + n, + history_rows + padded_rows, + dtype=torch.int32, + device=device, + ), + ] + ) + indices = torch.cat([indices, indices.new_full((padding, window), -1)]) + lengths = torch.cat([lengths, lengths.new_zeros(padding)]) + commit_mask = torch.cat([commit_mask, commit_mask.new_zeros(padding)]) + n = padded_rows return WindowLayout( req, pos, diff --git a/python/sglang/srt/model_executor/runner/prefill_cuda_graph_runner.py b/python/sglang/srt/model_executor/runner/prefill_cuda_graph_runner.py index 57ddf3194a..c093f6e621 100644 --- a/python/sglang/srt/model_executor/runner/prefill_cuda_graph_runner.py +++ b/python/sglang/srt/model_executor/runner/prefill_cuda_graph_runner.py @@ -2004,6 +2004,13 @@ class PrefillCudaGraphRunner(BaseCudaGraphRunner): # The n-gram hasher runs outside the graph and reads this at replay. static_forward_batch.ngram_embedding_info = forward_batch.ngram_embedding_info static_forward_batch.engram_history = forward_batch.engram_history + # The encoder replay fold: a replay builds the same window floors and + # compressor slots as an eager prefill. + static_forward_batch.encoder_swa_replay = forward_batch.encoder_swa_replay + static_forward_batch.encoder_swa_row_floor = forward_batch.encoder_swa_row_floor + static_forward_batch.encoder_swa_compress_rows = ( + forward_batch.encoder_swa_compress_rows + ) static_forward_batch = self.model_runner.prepare_dummy_forward_batch( static_forward_batch ) diff --git a/test/registered/unit/managers/scheduler_components/test_dp_attn.py b/test/registered/unit/managers/scheduler_components/test_dp_attn.py index c54042397c..e28a68c3a6 100644 --- a/test/registered/unit/managers/scheduler_components/test_dp_attn.py +++ b/test/registered/unit/managers/scheduler_components/test_dp_attn.py @@ -444,6 +444,7 @@ class TestDecodeToExtendConversionVote(CustomTestCase): ): return dp_attn._local_prefill_cuda_graph_vote( local_batch=batch, + num_tokens=2, prefill_graph_runner=runner, coordinated_prefill=True, breakable_prefill=True, @@ -483,6 +484,7 @@ class TestPrefillCudaGraphVote(CustomTestCase): ) vote = dp_attn._local_prefill_cuda_graph_vote( local_batch=batch, + num_tokens=4, prefill_graph_runner=runner, coordinated_prefill=True, breakable_prefill=True, @@ -542,6 +544,7 @@ class TestPrefillCudaGraphVote(CustomTestCase): ) return dp_attn._local_prefill_cuda_graph_vote( local_batch=batch, + num_tokens=48, prefill_graph_runner=runner, coordinated_prefill=True, breakable_prefill=True, diff --git a/test/registered/unit/mem_cache/test_dsv4_compressed_pools.py b/test/registered/unit/mem_cache/test_dsv4_compressed_pools.py index aee211099d..281c6480f6 100644 --- a/test/registered/unit/mem_cache/test_dsv4_compressed_pools.py +++ b/test/registered/unit/mem_cache/test_dsv4_compressed_pools.py @@ -26,6 +26,11 @@ from sglang.srt.mem_cache.deepseek_v4_memory_pool import ( _num_dsv4_physical_kv_pages, ) from sglang.srt.mem_cache.dsv41_request_window import window_layout +from sglang.srt.model_executor.cuda_graph_config import ( + Backend, + CudaGraphConfig, + PhaseConfig, +) from sglang.srt.model_executor.forward_batch_info import ForwardMode from sglang.srt.runtime_context import get_context from sglang.srt.speculative.dspark_components.dspark_kv_inject import ( @@ -544,8 +549,8 @@ class TestEncoderReplayWithoutSpeculation(CustomTestCase): override.install() self.addCleanup(override.restore) - def test_sparse_prefill_chunk_cache_builds_without_mapping(self): - pool = DeepSeekV4TokenToKVPool( + def _make_pool(self): + return DeepSeekV4TokenToKVPool( max_num_reqs=2, num_req_slots=3, swa_size=0, @@ -567,6 +572,25 @@ class TestEncoderReplayWithoutSpeculation(CustomTestCase): compression_ratios=[0, 0, 0], full_size=2048, ) + + def test_workspace_covers_the_largest_prefill_graph_bucket(self): + override = get_context().override_server_args( + enable_encoder_swa_bounded_replay=True, + page_size=256, + max_running_requests=2, + chunked_prefill_size=256, + cuda_graph_config=CudaGraphConfig( + prefill=PhaseConfig(backend=Backend.BREAKABLE, bs=[64, 1024]) + ), + ) + override.install() + self.addCleanup(override.restore) + history = (3 + 1) * 128 # (num_req_slots + 1) * sliding_window + workspace = self._make_pool().request_window.workspace + self.assertGreaterEqual(workspace.size, history + 1024) + + def test_sparse_prefill_chunk_cache_builds_without_mapping(self): + pool = self._make_pool() self.assertIsNotNone(pool.request_window) self.assertFalse(pool.needs_paged_swa_allocator) self.assertIsNone(pool.full_to_swa_index_mapping) diff --git a/test/registered/unit/model_executor/runner/test_prefill_cuda_graph_padding.py b/test/registered/unit/model_executor/runner/test_prefill_cuda_graph_padding.py index 97127c191a..427ecfed5a 100644 --- a/test/registered/unit/model_executor/runner/test_prefill_cuda_graph_padding.py +++ b/test/registered/unit/model_executor/runner/test_prefill_cuda_graph_padding.py @@ -264,6 +264,7 @@ class TestPrefillCudaGraphPadding(CustomTestCase): votes = [ _local_prefill_cuda_graph_vote( local_batch=batch, + num_tokens=4, prefill_graph_runner=runner, coordinated_prefill=True, breakable_prefill=True, diff --git a/test/registered/unit/model_executor/test_encoder_swa_replay.py b/test/registered/unit/model_executor/test_encoder_swa_replay.py index 3a6e4e84f8..ce084f28c0 100644 --- a/test/registered/unit/model_executor/test_encoder_swa_replay.py +++ b/test/registered/unit/model_executor/test_encoder_swa_replay.py @@ -10,6 +10,7 @@ import torch from sglang.kernels.ops.attention.dsv4_attn_metadata_kernels import ( late_layer_tail_layout, ) +from sglang.srt.arg_groups import deepseek_v4_hook from sglang.srt.distributed import parallel_state from sglang.srt.distributed.parallel_state import GroupCoordinator from sglang.srt.layers.attention.deepseek_v4_backend import ( @@ -18,7 +19,11 @@ from sglang.srt.layers.attention.deepseek_v4_backend import ( DSV4AttnMetadata, ) from sglang.srt.mem_cache.dsv41_request_window import window_layout -from sglang.srt.model_executor.cuda_graph_config import CudaGraphConfig, PhaseConfig +from sglang.srt.model_executor.cuda_graph_config import ( + Backend, + CudaGraphConfig, + PhaseConfig, +) from sglang.srt.model_executor.encoder_swa_replay import ( _build_replay_batch, _fold_batch, @@ -30,13 +35,15 @@ from sglang.srt.model_executor.forward_batch_info import ( ForwardBatch, ForwardMode, ) -from sglang.srt.runtime_context import get_context, get_parallel +from sglang.srt.runtime_context import get_context, get_parallel, override_platform +from sglang.srt.server_args import ServerArgs from sglang.srt.speculative.spec_info import SpeculativeAlgorithm from sglang.test.ci.ci_register import register_cpu_ci from sglang.test.test_utils import CustomTestCase register_cpu_ci(est_time=10, suite="base-a-test-cpu") +_ENV_GATE = "sglang.kernels.ops.attention.dsv4.unified_kv_kernels.env_gate" MAX_CTX = 1024 @@ -130,9 +137,23 @@ def _backend(fx): backend.token_to_kv_pool = SimpleNamespace( request_window=SimpleNamespace(capacity=2 * SWA_WINDOW) ) + backend.req_to_token_pool = SimpleNamespace(req_to_token=fx.req_to_token) + backend.online_c128_mtp = SimpleNamespace(prepare_forward=lambda *_, **__: 0) + backend.is_dspark_draft = False + backend.topk = backend.speculative_num_steps = 0 return backend +def _build_metadata(fx, forward_batch, *, use_prefill_cuda_graph=False): + with ( + patch.object(DSV4AttnMetadata, "init_compression_metadata"), + patch.object(DSV4AttnMetadata, "init_flashmla_related"), + ): + return _backend(fx)._build_forward_metadata( + forward_batch, use_prefill_cuda_graph=use_prefill_cuda_graph + ) + + class TestFoldedExtendRows(CustomTestCase): def test_mixed_hit_and_miss_fold_maps_every_row(self): """Hits extend from their replay start into their cached KV slots; every @@ -278,30 +299,34 @@ class TestFoldWithDecoderSwaTail(CustomTestCase): ) +def _extend_batch(fx): + """The folded extend's forward batch, as the backend's metadata reads it.""" + fb = fx.folded.batch + seq_lens = fx.folded_seq_lens() + lens = torch.tensor(fb.extend_lens) + return SimpleNamespace( + forward_mode=ForwardMode.EXTEND, + batch_size=len(fb.extend_lens), + spec_info=None, + max_seq_len_override=None, + extend_seq_lens=lens, + extend_seq_lens_cpu=list(fb.extend_lens), + extend_start_loc=torch.cumsum(lens, 0) - lens, + seq_lens_cpu=torch.tensor(seq_lens), + seq_lens=torch.tensor(seq_lens), + req_pool_indices=fb.req_pool_indices, + out_cache_loc=fb.out_cache_loc, + positions=fx.folded_positions(), + encoder_swa_replay=False, + encoder_swa_row_floor=fx.folded.row_floor, + encoder_swa_compress_rows=fx.folded.keep_rows, + ) + + class TestFoldedReplayCompressorWrites(CustomTestCase): """Folded replay rows rebuild only their SWA window: their main KV, compressed groups and index keys are cached, so the compressors write only the new rows.""" - def _forward_batch(self, fx): - fb = fx.folded.batch - seq_lens = fx.folded_seq_lens() - lens = torch.tensor(fb.extend_lens) - return SimpleNamespace( - forward_mode=ForwardMode.EXTEND, - batch_size=len(fb.extend_lens), - spec_info=None, - max_seq_len_override=None, - extend_seq_lens=lens, - extend_seq_lens_cpu=list(fb.extend_lens), - extend_start_loc=torch.cumsum(lens, 0) - lens, - seq_lens_cpu=torch.tensor(seq_lens), - seq_lens=torch.tensor(seq_lens), - req_pool_indices=fb.req_pool_indices, - out_cache_loc=fb.out_cache_loc, - positions=fx.folded_positions(), - encoder_swa_compress_rows=fx.folded.keep_rows, - ) - def _expected_out_loc(self, fx): """Replay rows on the padding slot, new rows on their own KV slots.""" fb = fx.folded.batch @@ -317,16 +342,7 @@ class TestFoldedReplayCompressorWrites(CustomTestCase): def test_extend_metadata_puts_replay_rows_on_the_padding_slot(self): fx = _Fixture() - backend = _backend(fx) - backend.req_to_token_pool = SimpleNamespace(req_to_token=fx.req_to_token) - backend.online_c128_mtp = SimpleNamespace(prepare_forward=lambda *_, **__: 0) - backend.is_dspark_draft = False - backend.topk = backend.speculative_num_steps = 0 - with ( - patch.object(DSV4AttnMetadata, "init_compression_metadata"), - patch.object(DSV4AttnMetadata, "init_flashmla_related"), - ): - metadata = backend._build_forward_metadata(self._forward_batch(fx)) + metadata = _build_metadata(fx, _extend_batch(fx)) raw_out_loc = metadata.core_attn_metadata.raw_out_loc self.assertEqual(int((raw_out_loc == 0).sum()), REPLAY_ROWS) @@ -353,7 +369,7 @@ class TestFoldedReplayCompressorWrites(CustomTestCase): fb = fx.folded.batch req = fb.req_pool_indices.repeat_interleave(torch.tensor(fb.extend_lens)) rows = torch.arange(out_loc.shape[0], dtype=torch.float32)[:, None] - forward_batch = self._forward_batch(fx) + forward_batch = _extend_batch(fx) backend._low_ratio_compress( layer, rows, req, forward_batch.positions, forward_batch ) @@ -363,6 +379,100 @@ class TestFoldedReplayCompressorWrites(CustomTestCase): torch.testing.assert_close(slots, out_loc) +class TestFoldedReplayUnderPrefillGraph(CustomTestCase): + """The breakable prefill graph replays a folded batch padded to its captured + bucket and refreshes the captured metadata in place before each replay.""" + + PAD = 32 + + def _graph_batch(self, fx, *, compress_rows): + fb = _extend_batch(fx) + pad = torch.zeros(self.PAD, dtype=fb.out_cache_loc.dtype) + fb.out_cache_loc = torch.cat([fb.out_cache_loc, pad]) + fb.positions = torch.cat([fb.positions, pad]) + fb.encoder_swa_compress_rows = compress_rows + return fb + + def test_padding_rows_get_no_window_and_commit_nowhere(self): + fx = _Fixture() + live = sum(fx.folded.batch.extend_lens) + graph = _build_metadata( + fx, + self._graph_batch(fx, compress_rows=fx.folded.keep_rows), + use_prefill_cuda_graph=True, + ).core_attn_metadata.request_window_layout + eager = _build_metadata( + fx, _extend_batch(fx) + ).core_attn_metadata.request_window_layout + + self.assertEqual(graph.pos.numel(), live + self.PAD) + # The graph fixes its window groups, so workspace offsets differ from + # eager; each live row still sees the same window and commits the same. + live_rows = graph.leading_rows(live) + torch.testing.assert_close(live_rows.lengths, eager.lengths) + torch.testing.assert_close(live_rows.commit_mask, eager.commit_mask) + torch.testing.assert_close( + (live_rows.indices >= 0).sum(-1), (eager.indices >= 0).sum(-1) + ) + self.assertEqual(live_rows.write_loc.unique().numel(), live) + self.assertFalse(bool(graph.commit_mask[live:].any())) + self.assertTrue(bool((graph.lengths[live:] == 0).all())) + self.assertTrue(bool((graph.indices[live:] == -1).all())) + self.assertTrue( + bool((graph.write_loc[live:] > graph.write_loc[:live].max()).all()) + ) + + def test_graph_slots_do_not_alias_the_static_out_cache_loc(self): + """A capture has no replay rows; the replay that has them refreshes the + captured slots in place, which must leave the batch's input untouched.""" + fx = _Fixture() + fb = self._graph_batch(fx, compress_rows=None) + raw = _build_metadata( + fx, fb, use_prefill_cuda_graph=True + ).core_attn_metadata.raw_out_loc + self.assertNotEqual( + raw.untyped_storage().data_ptr(), + fb.out_cache_loc.untyped_storage().data_ptr(), + ) + torch.testing.assert_close(raw, fb.out_cache_loc) + + +class TestEncoderReplayPrefillGraphGate(CustomTestCase): + def _validate(self, prefill_backend): + args = ServerArgs( + model_path="local-deepseek-v41", + attention_backend="dsv4", + enable_encoder_swa_bounded_replay=True, + enable_decoder_swa_bounded_replay=False, + cuda_graph_config=CudaGraphConfig( + prefill=PhaseConfig(backend=prefill_backend) + ), + ) + args._model_config = SimpleNamespace( + hf_config=SimpleNamespace(model_type="deepseek_v41") + ) + with patch(f"{_ENV_GATE}.is_unified_kv_triton", return_value=False): + deepseek_v4_hook.validate_deepseek_v41_features(args) + + @override_platform(is_cuda=True, is_hip=False) + def test_cuda_accepts_only_the_breakable_prefill_graph(self): + self._validate(Backend.BREAKABLE) + for backend in (Backend.FULL, Backend.TC_PIECEWISE): + with ( + self.subTest(backend=backend), + self.assertRaisesRegex(ValueError, "other than breakable"), + ): + self._validate(backend) + + @override_platform(is_cuda=False, is_hip=True) + def test_rocm_rejects_the_prefill_graph(self): + with ( + patch.object(deepseek_v4_hook, "is_gfx95_supported", return_value=True), + self.assertRaisesRegex(ValueError, "prefill CUDA graph on ROCm"), + ): + self._validate(Backend.BREAKABLE) + + class TestFoldMlpSync(CustomTestCase): """Pure TP that still syncs the MLP (e.g. TP4 with --moe-dense-tp-size 1).""" diff --git a/test/registered/unit/model_executor/test_prefill_cuda_graph_runner.py b/test/registered/unit/model_executor/test_prefill_cuda_graph_runner.py index 0e3e0f07f4..834e57baa9 100644 --- a/test/registered/unit/model_executor/test_prefill_cuda_graph_runner.py +++ b/test/registered/unit/model_executor/test_prefill_cuda_graph_runner.py @@ -688,6 +688,7 @@ class TestPrefillCudaGraphRunnerChunkedPrefix(CustomTestCase): self.assertEqual( _local_prefill_cuda_graph_vote( local_batch=schedule_batch, + num_tokens=4, prefill_graph_runner=runner, coordinated_prefill=True, breakable_prefill=backend == Backend.BREAKABLE,