From ef26f6ad49b03d85bea33d080c2c7d46e2f84097 Mon Sep 17 00:00:00 2001 From: Liangsheng Yin Date: Sat, 17 Oct 2026 00:56:18 -0700 Subject: [PATCH] [Fix] deepseek-v4.1: Keep the cached KV of folded encoder replay rows (#44857) --- .../layers/attention/deepseek_v4_backend.py | 20 +-- .../srt/model_executor/encoder_swa_replay.py | 5 - .../srt/model_executor/forward_batch_info.py | 4 +- .../model_executor/test_encoder_swa_replay.py | 125 +++++++++++++++--- 4 files changed, 116 insertions(+), 38 deletions(-) diff --git a/python/sglang/srt/layers/attention/deepseek_v4_backend.py b/python/sglang/srt/layers/attention/deepseek_v4_backend.py index 5eb17b471c..15f0d08b57 100644 --- a/python/sglang/srt/layers/attention/deepseek_v4_backend.py +++ b/python/sglang/srt/layers/attention/deepseek_v4_backend.py @@ -2704,6 +2704,16 @@ class DeepseekV4AttnBackend( request_window_layout=request_layout, ) + @staticmethod + def _compressor_out_loc(forward_batch: ForwardBatch) -> torch.Tensor: + """Slot 0 for folded replay rows: their main KV, compressed groups and index + keys are cached, so the compressors write only the new rows.""" + loc = forward_batch.out_cache_loc + keep = forward_batch.encoder_swa_compress_rows + if keep is None: + return loc + return torch.zeros_like(loc).index_copy_(0, keep, loc[keep]) + def _build_forward_metadata( self, forward_batch: ForwardBatch, @@ -2804,7 +2814,7 @@ class DeepseekV4AttnBackend( req_pool_indices=req_pool_indices, seq_lens=seq_lens, seq_lens_cpu=seq_lens_cpu.tolist(), - out_cache_loc=forward_batch.out_cache_loc, + out_cache_loc=self._compressor_out_loc(forward_batch), num_tokens=sum(extend_seq_lens_cpu), extend_seq_lens=extend_seq_lens, extend_seq_lens_cpu=extend_seq_lens_cpu, @@ -3127,11 +3137,6 @@ class DeepseekV4AttnBackend( # length is the request count; padded rows belong to no request. start_loc = forward_batch.extend_start_loc seq_lens = forward_batch.extend_seq_lens[: start_loc.shape[0]] - skip = forward_batch.encoder_swa_compress_skip - if skip is not None: - # Folded replay rows are cached already; compress the new rows only. - start_loc = start_loc + skip - seq_lens = seq_lens - skip self._low_ratio_compress_fused( layer, x, @@ -3140,9 +3145,6 @@ class DeepseekV4AttnBackend( extend_offsets=(start_loc.to(torch.int32), seq_lens.to(torch.int32)), ) else: - rows = forward_batch.encoder_swa_compress_rows - if rows is not None and forward_batch.forward_mode.is_extend(): - x, req, pos = x[rows], req[rows], pos[rows] self._low_ratio_compress_torch( layer, x, diff --git a/python/sglang/srt/model_executor/encoder_swa_replay.py b/python/sglang/srt/model_executor/encoder_swa_replay.py index cf0585354c..10df46d669 100644 --- a/python/sglang/srt/model_executor/encoder_swa_replay.py +++ b/python/sglang/srt/model_executor/encoder_swa_replay.py @@ -12,7 +12,6 @@ class FoldedExtend(msgspec.Struct, frozen=True): keep_rows: torch.Tensor # [original rows] int64, folded-row index of each scheduled_row: torch.Tensor # [folded rows] int64, original row, -1 if replayed row_floor: torch.Tensor # [folded rows] int64 - compress_skip: torch.Tensor # [bs] int32, leading replay rows per request replay_lens: list[int] # [bs] leading replay rows per request num_rows: int @@ -41,7 +40,6 @@ def folded_replay_rows(batch) -> int: def apply_folded_extend(folded: FoldedExtend, forward_batch) -> None: forward_batch.encoder_swa_row_floor = folded.row_floor - forward_batch.encoder_swa_compress_skip = folded.compress_skip forward_batch.encoder_swa_compress_rows = folded.keep_rows @@ -240,9 +238,6 @@ def _fold_batch(*, batch, runner, rows) -> FoldedExtend: keep_rows=keep_rows.to(device, non_blocking=True), scheduled_row=scheduled_row.to(device, non_blocking=True), row_floor=row_floor.to(device, non_blocking=True), - compress_skip=torch.tensor(replay, dtype=torch.int32).to( - device, non_blocking=True - ), replay_lens=replay, num_rows=fold_off, ) diff --git a/python/sglang/srt/model_executor/forward_batch_info.py b/python/sglang/srt/model_executor/forward_batch_info.py index 0f10d5f2ab..8cdb79d1ec 100644 --- a/python/sglang/srt/model_executor/forward_batch_info.py +++ b/python/sglang/srt/model_executor/forward_batch_info.py @@ -773,10 +773,8 @@ class ForwardBatch(ForwardBatchDeepSeekMHAMixin): ngram_embedding_info: Optional[NgramEmbeddingInfo] = None encoder_swa_replay: bool = False # Encoder replay folded into the extend: per-row window floor [num_tokens] - # int64 (the request's replay start, 0 for other requests), and per-request - # leading replay rows the low-ratio compressors skip [bs] int32. + # int64 (the request's replay start, 0 for other requests). encoder_swa_row_floor: Optional[torch.Tensor] = None - encoder_swa_compress_skip: Optional[torch.Tensor] = None encoder_swa_compress_rows: Optional[torch.Tensor] = None # non-replay rows, int64 # DeepSeek-V4.1 engram, extend only: the n - 1 tokens before each request's diff --git a/test/registered/unit/model_executor/test_encoder_swa_replay.py b/test/registered/unit/model_executor/test_encoder_swa_replay.py index 264c819c93..982c698703 100644 --- a/test/registered/unit/model_executor/test_encoder_swa_replay.py +++ b/test/registered/unit/model_executor/test_encoder_swa_replay.py @@ -109,6 +109,24 @@ class _Fixture: return [p + n for p, n in zip(fb.prefix_lens, fb.extend_lens)] +def _backend(fx): + backend = object.__new__(DeepseekV4AttnBackend) + backend.encoder_replay = False + backend.encoder_row_floor = fx.folded.row_floor + backend.req_to_token = fx.req_to_token + backend.page_size = 256 + backend.index_topk = 512 + backend.present_ratios = (0,) + backend.low_ratios = () + backend.has_c4 = backend.has_c128 = False + backend.trtllm_attn = False + backend.cuda_int32_kwargs = {"dtype": torch.int32, "device": "cpu"} + backend.token_to_kv_pool = SimpleNamespace( + request_window=SimpleNamespace(capacity=2 * SWA_WINDOW) + ) + return backend + + class TestFoldedExtendRows(CustomTestCase): def test_mixed_hit_and_miss_fold_maps_every_row(self): """Hits extend from their replay start into their cached KV slots; every @@ -135,9 +153,6 @@ class TestFoldedExtendRows(CustomTestCase): folded.row_floor, torch.tensor([0] * 40 + [384] * 158 + [0] * 84), ) - torch.testing.assert_close( - folded.compress_skip, torch.tensor([0, 128, 64], dtype=torch.int32) - ) # Input logprobs cover the same tokens, so their row count is unchanged. self.assertEqual(fb.extend_logprob_start_lens, [40, 158, 69]) logprob_rows = [ @@ -167,23 +182,6 @@ class TestFoldWithDecoderSwaTail(CustomTestCase): """Encoder SWA bounded replay combined with decoder SWA bounded replay: the late layers run only each request's last SWA_WINDOW folded rows.""" - def _backend(self, fx): - backend = object.__new__(DeepseekV4AttnBackend) - backend.encoder_replay = False - backend.encoder_row_floor = fx.folded.row_floor - backend.req_to_token = fx.req_to_token - backend.page_size = 256 - backend.index_topk = 512 - backend.present_ratios = (0,) - backend.low_ratios = () - backend.has_c4 = backend.has_c128 = False - backend.trtllm_attn = False - backend.cuda_int32_kwargs = {"dtype": torch.int32, "device": "cpu"} - backend.token_to_kv_pool = SimpleNamespace( - request_window=SimpleNamespace(capacity=2 * SWA_WINDOW) - ) - return backend - def test_tail_keeps_its_own_floor_under_fold(self): """A prefix hit with both flags on builds the tail metadata; each tail row is floored one window back or at its request's replay start, whichever is @@ -205,7 +203,7 @@ class TestFoldWithDecoderSwaTail(CustomTestCase): patch.object(DSV4AttnMetadata, "init_compression_metadata"), patch.object(DSV4AttnMetadata, "init_flashmla_related"), ): - metadata = self._backend(fx)._build_late_layer_tail_metadata(forward_batch) + metadata = _backend(fx)._build_late_layer_tail_metadata(forward_batch) # Tails: the miss's 40 rows, the last 128 of the 158-row hit (from 414, # above its replay start 384), and the whole 84-row hit (from 0). @@ -272,6 +270,91 @@ class TestFoldWithDecoderSwaTail(CustomTestCase): ) +class TestFoldedReplayCompressorWrites(CustomTestCase): + """Folded replay rows rebuild only their SWA window: their main KV, compressed + groups and index keys are cached, so the compressors write only the new rows.""" + + def _forward_batch(self, fx): + fb = fx.folded.batch + seq_lens = fx.folded_seq_lens() + lens = torch.tensor(fb.extend_lens) + return SimpleNamespace( + forward_mode=ForwardMode.EXTEND, + batch_size=len(fb.extend_lens), + spec_info=None, + max_seq_len_override=None, + extend_seq_lens=lens, + extend_seq_lens_cpu=list(fb.extend_lens), + extend_start_loc=torch.cumsum(lens, 0) - lens, + seq_lens_cpu=torch.tensor(seq_lens), + seq_lens=torch.tensor(seq_lens), + req_pool_indices=fb.req_pool_indices, + out_cache_loc=fb.out_cache_loc, + positions=fx.folded_positions(), + encoder_swa_compress_rows=fx.folded.keep_rows, + ) + + def _expected_out_loc(self, fx): + """Replay rows on the padding slot, new rows on their own KV slots.""" + fb = fx.folded.batch + replayed = torch.cat( + [ + torch.arange(start, start + n) < prefix + for start, n, (_, prefix, _) in zip( + fb.prefix_lens, fb.extend_lens, REQS + ) + ] + ) + return fb.out_cache_loc.masked_fill(replayed, 0) + + def test_extend_metadata_puts_replay_rows_on_the_padding_slot(self): + fx = _Fixture() + backend = _backend(fx) + backend.req_to_token_pool = SimpleNamespace(req_to_token=fx.req_to_token) + backend.online_c128_mtp = SimpleNamespace(prepare_forward=lambda *_, **__: 0) + backend.is_dspark_draft = False + backend.topk = backend.speculative_num_steps = 0 + with ( + patch.object(DSV4AttnMetadata, "init_compression_metadata"), + patch.object(DSV4AttnMetadata, "init_flashmla_related"), + ): + metadata = backend._build_forward_metadata(self._forward_batch(fx)) + + raw_out_loc = metadata.core_attn_metadata.raw_out_loc + self.assertEqual(int((raw_out_loc == 0).sum()), REPLAY_ROWS) + torch.testing.assert_close(raw_out_loc, self._expected_out_loc(fx)) + + def test_unfused_compressor_writes_each_row_at_its_own_slot(self): + fx = _Fixture() + backend = _backend(fx) + out_loc = self._expected_out_loc(fx) + backend.forward_metadata = SimpleNamespace( + core_metadata=SimpleNamespace(c1_out_loc=out_loc, raw_out_loc=out_loc) + ) + written = [] + backend._low_ratio_write_group = lambda layer, kv, slots, pos, **_: ( + written.append((kv, slots)) + ) + layer = SimpleNamespace( + compress_ratio=1, + layer_id=0, + compressor=SimpleNamespace( + use_fused_compress=False, project=lambda x: (x, None) + ), + ) + fb = fx.folded.batch + req = fb.req_pool_indices.repeat_interleave(torch.tensor(fb.extend_lens)) + rows = torch.arange(out_loc.shape[0], dtype=torch.float32)[:, None] + forward_batch = self._forward_batch(fx) + backend._low_ratio_compress( + layer, rows, req, forward_batch.positions, forward_batch + ) + + ((kv, slots),) = written + torch.testing.assert_close(kv, rows) + torch.testing.assert_close(slots, out_loc) + + class TestFoldMlpSync(CustomTestCase): """Pure TP that still syncs the MLP (e.g. TP4 with --moe-dense-tp-size 1)."""