diff --git a/python/sglang/srt/layers/attention/deepseek_v4_backend.py b/python/sglang/srt/layers/attention/deepseek_v4_backend.py index d812abb29f..65677ec50f 100644 --- a/python/sglang/srt/layers/attention/deepseek_v4_backend.py +++ b/python/sglang/srt/layers/attention/deepseek_v4_backend.py @@ -3151,6 +3151,13 @@ class DeepseekV4AttnBackend( max_seq_len_override=max_seq_len, use_prefill_cuda_graph=True, ) + if self._prefill_graph_translates_swa_loc(self.forward_metadata): + # Every layer's store reads this buffer; replay refreshes it in place. + self.forward_metadata.core_attn_metadata.swa_out_cache_loc = ( + self.token_to_kv_pool.translate_loc_from_full_to_swa( + forward_batch.out_cache_loc + ).to(torch.int32) + ) self._activate_request_window() with_tail = ( self.enable_decoder_swa_bounded_replay @@ -3171,6 +3178,13 @@ class DeepseekV4AttnBackend( ) return self.forward_metadata + def _prefill_graph_translates_swa_loc(self, metadata: DSV4Metadata) -> bool: + return ( + metadata.core_attn_metadata.swa_out_cache_loc is None + and self.token_to_kv_pool.request_window is None + and self.token_to_kv_pool.swa_req_ring_size is None + ) + def _prefill_graph_context_width(self, forward_batch: ForwardBatch) -> int: """One context width for the full rows' and the late-layer tail's metadata: the live context, unless c4/c128 compress metadata is captured at a width.""" @@ -3250,6 +3264,12 @@ class DeepseekV4AttnBackend( ) assert isinstance(capture_metadata, DSV4Metadata) capture_metadata.refresh_for_breakable_cuda_graph_replay_(static_metadata) + if self._prefill_graph_translates_swa_loc(static_metadata): + capture_metadata.core_attn_metadata.swa_out_cache_loc.copy_( + self.token_to_kv_pool.translate_loc_from_full_to_swa( + metadata_batch.out_cache_loc + ) + ) self.prefill_graph_replay_batch = forward_batch self.forward_metadata = capture_metadata self._activate_request_window() diff --git a/python/sglang/srt/models/deepseek_v4_replay_tail.py b/python/sglang/srt/models/deepseek_v4_replay_tail.py index bfb2dc8d2e..85ae2cd1ac 100644 --- a/python/sglang/srt/models/deepseek_v4_replay_tail.py +++ b/python/sglang/srt/models/deepseek_v4_replay_tail.py @@ -497,7 +497,9 @@ class DecoderReplayTail: def _bridge_outputs( self, block_rows: int, outputs: List[Optional[torch.Tensor]] ) -> Tuple[Optional[torch.Tensor], ...]: - """Block-sized bridge views of the tail rows; rows past the tail are zero.""" + """Block-sized bridge views of the tail rows. Rows past the tail hold stale + values, which reach only rows that _scatter_tail_rows and the + real_rows_count cut drop.""" key = tuple(None if t is None else (t.dtype, t.shape[1:]) for t in outputs) sets = self._bridge_buffers.setdefault(key, []) if not sets or sets[0][0].shape[0] < block_rows: @@ -515,7 +517,6 @@ class DecoderReplayTail: continue view = buf[:block_rows] view[: t.shape[0]].copy_(t) - view[t.shape[0] :].zero_() views.append(view) return tuple(views) diff --git a/test/registered/attention/unittests/dsv4/test_deepseek_v4.py b/test/registered/attention/unittests/dsv4/test_deepseek_v4.py index 7656299504..0c34604362 100644 --- a/test/registered/attention/unittests/dsv4/test_deepseek_v4.py +++ b/test/registered/attention/unittests/dsv4/test_deepseek_v4.py @@ -653,13 +653,19 @@ class TestDSV4BreakableCudaGraphMetadataContract(CustomTestCase): self._make_core_metadata(0), indexer_metadata=None ) capture_metadata.sparse_prefill_cache = object() + captured_swa_loc = torch.zeros(2, dtype=torch.int32) + capture_metadata.core_attn_metadata.swa_out_cache_loc = captured_swa_loc replay_metadata = DSV4Metadata( self._make_core_metadata(1000), indexer_metadata=None ) backend = object.__new__(DeepseekV4AttnBackend) backend.MAX_SEQ_LEN_FOR_CAPTURE = 4096 backend.prefill_candidates = object.__new__(DenseBlocksBackend) - backend.token_to_kv_pool = SimpleNamespace(request_window=None) + backend.token_to_kv_pool = SimpleNamespace( + request_window=None, + swa_req_ring_size=None, + translate_loc_from_full_to_swa=lambda loc: loc + 100, + ) backend.low_ratios = () backend.has_c4 = backend.has_c128 = True calls = [] @@ -672,7 +678,11 @@ class TestDSV4BreakableCudaGraphMetadataContract(CustomTestCase): backend._build_forward_metadata = fake_build_forward_metadata forward_batch = SimpleNamespace(name="live", max_seq_len_override=None) - static_forward_batch = SimpleNamespace(name="static", max_seq_len_override=None) + static_forward_batch = SimpleNamespace( + name="static", + max_seq_len_override=None, + out_cache_loc=torch.tensor([5, 6], dtype=torch.int64), + ) backend.prepare_forward_metadata_for_breakable_cuda_graph_replay( capture_metadata, @@ -685,6 +695,11 @@ class TestDSV4BreakableCudaGraphMetadataContract(CustomTestCase): self.assertTrue(calls[0][2]) self.assertIs(backend.forward_metadata, capture_metadata) self.assertIsNone(capture_metadata.sparse_prefill_cache) + # The captured layer stores read the SWA locations by address. + self.assertIs( + capture_metadata.core_attn_metadata.swa_out_cache_loc, captured_swa_loc + ) + self.assertEqual(captured_swa_loc.tolist(), [105, 106]) self.assertTrue( torch.equal( capture_metadata.core_attn_metadata.seq_lens_casual, diff --git a/test/registered/unit/layers/test_dsv41_late_layer_tail_readers.py b/test/registered/unit/layers/test_dsv41_late_layer_tail_readers.py index 8f0596b990..189756a693 100644 --- a/test/registered/unit/layers/test_dsv41_late_layer_tail_readers.py +++ b/test/registered/unit/layers/test_dsv41_late_layer_tail_readers.py @@ -290,6 +290,7 @@ def _prefill_graph_replay(*, ratios, seq_len): backend.has_c4 = 4 in ratios backend.has_c128 = 128 in ratios backend._build_forward_metadata = build_full + backend._prefill_graph_translates_swa_loc = lambda metadata: False backend._prefill_graph_tail_metadata = build_tail backend._activate_request_window = lambda: None capture = object.__new__(DSV4Metadata)