[Fix] deepseek-v4.1: Keep the cached KV of folded encoder replay rows (#44857)

This commit is contained in:
Liangsheng Yin
2026-10-17 00:56:18 -07:00
committed by GitHub
parent bce7b27aa6
commit ef26f6ad49
4 changed files with 116 additions and 38 deletions
@@ -2704,6 +2704,16 @@ class DeepseekV4AttnBackend(
request_window_layout=request_layout,
)
@staticmethod
def _compressor_out_loc(forward_batch: ForwardBatch) -> torch.Tensor:
"""Slot 0 for folded replay rows: their main KV, compressed groups and index
keys are cached, so the compressors write only the new rows."""
loc = forward_batch.out_cache_loc
keep = forward_batch.encoder_swa_compress_rows
if keep is None:
return loc
return torch.zeros_like(loc).index_copy_(0, keep, loc[keep])
def _build_forward_metadata(
self,
forward_batch: ForwardBatch,
@@ -2804,7 +2814,7 @@ class DeepseekV4AttnBackend(
req_pool_indices=req_pool_indices,
seq_lens=seq_lens,
seq_lens_cpu=seq_lens_cpu.tolist(),
out_cache_loc=forward_batch.out_cache_loc,
out_cache_loc=self._compressor_out_loc(forward_batch),
num_tokens=sum(extend_seq_lens_cpu),
extend_seq_lens=extend_seq_lens,
extend_seq_lens_cpu=extend_seq_lens_cpu,
@@ -3127,11 +3137,6 @@ class DeepseekV4AttnBackend(
# length is the request count; padded rows belong to no request.
start_loc = forward_batch.extend_start_loc
seq_lens = forward_batch.extend_seq_lens[: start_loc.shape[0]]
skip = forward_batch.encoder_swa_compress_skip
if skip is not None:
# Folded replay rows are cached already; compress the new rows only.
start_loc = start_loc + skip
seq_lens = seq_lens - skip
self._low_ratio_compress_fused(
layer,
x,
@@ -3140,9 +3145,6 @@ class DeepseekV4AttnBackend(
extend_offsets=(start_loc.to(torch.int32), seq_lens.to(torch.int32)),
)
else:
rows = forward_batch.encoder_swa_compress_rows
if rows is not None and forward_batch.forward_mode.is_extend():
x, req, pos = x[rows], req[rows], pos[rows]
self._low_ratio_compress_torch(
layer,
x,
@@ -12,7 +12,6 @@ class FoldedExtend(msgspec.Struct, frozen=True):
keep_rows: torch.Tensor # [original rows] int64, folded-row index of each
scheduled_row: torch.Tensor # [folded rows] int64, original row, -1 if replayed
row_floor: torch.Tensor # [folded rows] int64
compress_skip: torch.Tensor # [bs] int32, leading replay rows per request
replay_lens: list[int] # [bs] leading replay rows per request
num_rows: int
@@ -41,7 +40,6 @@ def folded_replay_rows(batch) -> int:
def apply_folded_extend(folded: FoldedExtend, forward_batch) -> None:
forward_batch.encoder_swa_row_floor = folded.row_floor
forward_batch.encoder_swa_compress_skip = folded.compress_skip
forward_batch.encoder_swa_compress_rows = folded.keep_rows
@@ -240,9 +238,6 @@ def _fold_batch(*, batch, runner, rows) -> FoldedExtend:
keep_rows=keep_rows.to(device, non_blocking=True),
scheduled_row=scheduled_row.to(device, non_blocking=True),
row_floor=row_floor.to(device, non_blocking=True),
compress_skip=torch.tensor(replay, dtype=torch.int32).to(
device, non_blocking=True
),
replay_lens=replay,
num_rows=fold_off,
)
@@ -773,10 +773,8 @@ class ForwardBatch(ForwardBatchDeepSeekMHAMixin):
ngram_embedding_info: Optional[NgramEmbeddingInfo] = None
encoder_swa_replay: bool = False
# Encoder replay folded into the extend: per-row window floor [num_tokens]
# int64 (the request's replay start, 0 for other requests), and per-request
# leading replay rows the low-ratio compressors skip [bs] int32.
# int64 (the request's replay start, 0 for other requests).
encoder_swa_row_floor: Optional[torch.Tensor] = None
encoder_swa_compress_skip: Optional[torch.Tensor] = None
encoder_swa_compress_rows: Optional[torch.Tensor] = None # non-replay rows, int64
# DeepSeek-V4.1 engram, extend only: the n - 1 tokens before each request's
@@ -109,6 +109,24 @@ class _Fixture:
return [p + n for p, n in zip(fb.prefix_lens, fb.extend_lens)]
def _backend(fx):
backend = object.__new__(DeepseekV4AttnBackend)
backend.encoder_replay = False
backend.encoder_row_floor = fx.folded.row_floor
backend.req_to_token = fx.req_to_token
backend.page_size = 256
backend.index_topk = 512
backend.present_ratios = (0,)
backend.low_ratios = ()
backend.has_c4 = backend.has_c128 = False
backend.trtllm_attn = False
backend.cuda_int32_kwargs = {"dtype": torch.int32, "device": "cpu"}
backend.token_to_kv_pool = SimpleNamespace(
request_window=SimpleNamespace(capacity=2 * SWA_WINDOW)
)
return backend
class TestFoldedExtendRows(CustomTestCase):
def test_mixed_hit_and_miss_fold_maps_every_row(self):
"""Hits extend from their replay start into their cached KV slots; every
@@ -135,9 +153,6 @@ class TestFoldedExtendRows(CustomTestCase):
folded.row_floor,
torch.tensor([0] * 40 + [384] * 158 + [0] * 84),
)
torch.testing.assert_close(
folded.compress_skip, torch.tensor([0, 128, 64], dtype=torch.int32)
)
# Input logprobs cover the same tokens, so their row count is unchanged.
self.assertEqual(fb.extend_logprob_start_lens, [40, 158, 69])
logprob_rows = [
@@ -167,23 +182,6 @@ class TestFoldWithDecoderSwaTail(CustomTestCase):
"""Encoder SWA bounded replay combined with decoder SWA bounded replay:
the late layers run only each request's last SWA_WINDOW folded rows."""
def _backend(self, fx):
backend = object.__new__(DeepseekV4AttnBackend)
backend.encoder_replay = False
backend.encoder_row_floor = fx.folded.row_floor
backend.req_to_token = fx.req_to_token
backend.page_size = 256
backend.index_topk = 512
backend.present_ratios = (0,)
backend.low_ratios = ()
backend.has_c4 = backend.has_c128 = False
backend.trtllm_attn = False
backend.cuda_int32_kwargs = {"dtype": torch.int32, "device": "cpu"}
backend.token_to_kv_pool = SimpleNamespace(
request_window=SimpleNamespace(capacity=2 * SWA_WINDOW)
)
return backend
def test_tail_keeps_its_own_floor_under_fold(self):
"""A prefix hit with both flags on builds the tail metadata; each tail row
is floored one window back or at its request's replay start, whichever is
@@ -205,7 +203,7 @@ class TestFoldWithDecoderSwaTail(CustomTestCase):
patch.object(DSV4AttnMetadata, "init_compression_metadata"),
patch.object(DSV4AttnMetadata, "init_flashmla_related"),
):
metadata = self._backend(fx)._build_late_layer_tail_metadata(forward_batch)
metadata = _backend(fx)._build_late_layer_tail_metadata(forward_batch)
# Tails: the miss's 40 rows, the last 128 of the 158-row hit (from 414,
# above its replay start 384), and the whole 84-row hit (from 0).
@@ -272,6 +270,91 @@ class TestFoldWithDecoderSwaTail(CustomTestCase):
)
class TestFoldedReplayCompressorWrites(CustomTestCase):
"""Folded replay rows rebuild only their SWA window: their main KV, compressed
groups and index keys are cached, so the compressors write only the new rows."""
def _forward_batch(self, fx):
fb = fx.folded.batch
seq_lens = fx.folded_seq_lens()
lens = torch.tensor(fb.extend_lens)
return SimpleNamespace(
forward_mode=ForwardMode.EXTEND,
batch_size=len(fb.extend_lens),
spec_info=None,
max_seq_len_override=None,
extend_seq_lens=lens,
extend_seq_lens_cpu=list(fb.extend_lens),
extend_start_loc=torch.cumsum(lens, 0) - lens,
seq_lens_cpu=torch.tensor(seq_lens),
seq_lens=torch.tensor(seq_lens),
req_pool_indices=fb.req_pool_indices,
out_cache_loc=fb.out_cache_loc,
positions=fx.folded_positions(),
encoder_swa_compress_rows=fx.folded.keep_rows,
)
def _expected_out_loc(self, fx):
"""Replay rows on the padding slot, new rows on their own KV slots."""
fb = fx.folded.batch
replayed = torch.cat(
[
torch.arange(start, start + n) < prefix
for start, n, (_, prefix, _) in zip(
fb.prefix_lens, fb.extend_lens, REQS
)
]
)
return fb.out_cache_loc.masked_fill(replayed, 0)
def test_extend_metadata_puts_replay_rows_on_the_padding_slot(self):
fx = _Fixture()
backend = _backend(fx)
backend.req_to_token_pool = SimpleNamespace(req_to_token=fx.req_to_token)
backend.online_c128_mtp = SimpleNamespace(prepare_forward=lambda *_, **__: 0)
backend.is_dspark_draft = False
backend.topk = backend.speculative_num_steps = 0
with (
patch.object(DSV4AttnMetadata, "init_compression_metadata"),
patch.object(DSV4AttnMetadata, "init_flashmla_related"),
):
metadata = backend._build_forward_metadata(self._forward_batch(fx))
raw_out_loc = metadata.core_attn_metadata.raw_out_loc
self.assertEqual(int((raw_out_loc == 0).sum()), REPLAY_ROWS)
torch.testing.assert_close(raw_out_loc, self._expected_out_loc(fx))
def test_unfused_compressor_writes_each_row_at_its_own_slot(self):
fx = _Fixture()
backend = _backend(fx)
out_loc = self._expected_out_loc(fx)
backend.forward_metadata = SimpleNamespace(
core_metadata=SimpleNamespace(c1_out_loc=out_loc, raw_out_loc=out_loc)
)
written = []
backend._low_ratio_write_group = lambda layer, kv, slots, pos, **_: (
written.append((kv, slots))
)
layer = SimpleNamespace(
compress_ratio=1,
layer_id=0,
compressor=SimpleNamespace(
use_fused_compress=False, project=lambda x: (x, None)
),
)
fb = fx.folded.batch
req = fb.req_pool_indices.repeat_interleave(torch.tensor(fb.extend_lens))
rows = torch.arange(out_loc.shape[0], dtype=torch.float32)[:, None]
forward_batch = self._forward_batch(fx)
backend._low_ratio_compress(
layer, rows, req, forward_batch.positions, forward_batch
)
((kv, slots),) = written
torch.testing.assert_close(kv, rows)
torch.testing.assert_close(slots, out_loc)
class TestFoldMlpSync(CustomTestCase):
"""Pure TP that still syncs the MLP (e.g. TP4 with --moe-dense-tp-size 1)."""