[Fix] deepseek-v4.1: Keep the cached KV of folded encoder replay rows (#44857)

This commit is contained in:
Liangsheng Yin
2026-10-17 00:56:18 -07:00
committed by GitHub
parent bce7b27aa6
commit ef26f6ad49
4 changed files with 116 additions and 38 deletions
@@ -2704,6 +2704,16 @@ class DeepseekV4AttnBackend(
request_window_layout=request_layout,
)
@staticmethod
def _compressor_out_loc(forward_batch: ForwardBatch) -> torch.Tensor:
"""Slot 0 for folded replay rows: their main KV, compressed groups and index
keys are cached, so the compressors write only the new rows."""
loc = forward_batch.out_cache_loc
keep = forward_batch.encoder_swa_compress_rows
if keep is None:
return loc
return torch.zeros_like(loc).index_copy_(0, keep, loc[keep])
def _build_forward_metadata(
self,
forward_batch: ForwardBatch,
@@ -2804,7 +2814,7 @@ class DeepseekV4AttnBackend(
req_pool_indices=req_pool_indices,
seq_lens=seq_lens,
seq_lens_cpu=seq_lens_cpu.tolist(),
out_cache_loc=forward_batch.out_cache_loc,
out_cache_loc=self._compressor_out_loc(forward_batch),
num_tokens=sum(extend_seq_lens_cpu),
extend_seq_lens=extend_seq_lens,
extend_seq_lens_cpu=extend_seq_lens_cpu,
@@ -3127,11 +3137,6 @@ class DeepseekV4AttnBackend(
# length is the request count; padded rows belong to no request.
start_loc = forward_batch.extend_start_loc
seq_lens = forward_batch.extend_seq_lens[: start_loc.shape[0]]
skip = forward_batch.encoder_swa_compress_skip
if skip is not None:
# Folded replay rows are cached already; compress the new rows only.
start_loc = start_loc + skip
seq_lens = seq_lens - skip
self._low_ratio_compress_fused(
layer,
x,
@@ -3140,9 +3145,6 @@ class DeepseekV4AttnBackend(
extend_offsets=(start_loc.to(torch.int32), seq_lens.to(torch.int32)),
)
else:
rows = forward_batch.encoder_swa_compress_rows
if rows is not None and forward_batch.forward_mode.is_extend():
x, req, pos = x[rows], req[rows], pos[rows]
self._low_ratio_compress_torch(
layer,
x,
@@ -12,7 +12,6 @@ class FoldedExtend(msgspec.Struct, frozen=True):
keep_rows: torch.Tensor # [original rows] int64, folded-row index of each
scheduled_row: torch.Tensor # [folded rows] int64, original row, -1 if replayed
row_floor: torch.Tensor # [folded rows] int64
compress_skip: torch.Tensor # [bs] int32, leading replay rows per request
replay_lens: list[int] # [bs] leading replay rows per request
num_rows: int
@@ -41,7 +40,6 @@ def folded_replay_rows(batch) -> int:
def apply_folded_extend(folded: FoldedExtend, forward_batch) -> None:
forward_batch.encoder_swa_row_floor = folded.row_floor
forward_batch.encoder_swa_compress_skip = folded.compress_skip
forward_batch.encoder_swa_compress_rows = folded.keep_rows
@@ -240,9 +238,6 @@ def _fold_batch(*, batch, runner, rows) -> FoldedExtend:
keep_rows=keep_rows.to(device, non_blocking=True),
scheduled_row=scheduled_row.to(device, non_blocking=True),
row_floor=row_floor.to(device, non_blocking=True),
compress_skip=torch.tensor(replay, dtype=torch.int32).to(
device, non_blocking=True
),
replay_lens=replay,
num_rows=fold_off,
)
@@ -773,10 +773,8 @@ class ForwardBatch(ForwardBatchDeepSeekMHAMixin):
ngram_embedding_info: Optional[NgramEmbeddingInfo] = None
encoder_swa_replay: bool = False
# Encoder replay folded into the extend: per-row window floor [num_tokens]
# int64 (the request's replay start, 0 for other requests), and per-request
# leading replay rows the low-ratio compressors skip [bs] int32.
# int64 (the request's replay start, 0 for other requests).
encoder_swa_row_floor: Optional[torch.Tensor] = None
encoder_swa_compress_skip: Optional[torch.Tensor] = None
encoder_swa_compress_rows: Optional[torch.Tensor] = None # non-replay rows, int64
# DeepSeek-V4.1 engram, extend only: the n - 1 tokens before each request's