[Fix] deepseek-v4.1: Keep the cached KV of folded encoder replay rows (#44857)
This commit is contained in:
@@ -2704,6 +2704,16 @@ class DeepseekV4AttnBackend(
|
||||
request_window_layout=request_layout,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _compressor_out_loc(forward_batch: ForwardBatch) -> torch.Tensor:
|
||||
"""Slot 0 for folded replay rows: their main KV, compressed groups and index
|
||||
keys are cached, so the compressors write only the new rows."""
|
||||
loc = forward_batch.out_cache_loc
|
||||
keep = forward_batch.encoder_swa_compress_rows
|
||||
if keep is None:
|
||||
return loc
|
||||
return torch.zeros_like(loc).index_copy_(0, keep, loc[keep])
|
||||
|
||||
def _build_forward_metadata(
|
||||
self,
|
||||
forward_batch: ForwardBatch,
|
||||
@@ -2804,7 +2814,7 @@ class DeepseekV4AttnBackend(
|
||||
req_pool_indices=req_pool_indices,
|
||||
seq_lens=seq_lens,
|
||||
seq_lens_cpu=seq_lens_cpu.tolist(),
|
||||
out_cache_loc=forward_batch.out_cache_loc,
|
||||
out_cache_loc=self._compressor_out_loc(forward_batch),
|
||||
num_tokens=sum(extend_seq_lens_cpu),
|
||||
extend_seq_lens=extend_seq_lens,
|
||||
extend_seq_lens_cpu=extend_seq_lens_cpu,
|
||||
@@ -3127,11 +3137,6 @@ class DeepseekV4AttnBackend(
|
||||
# length is the request count; padded rows belong to no request.
|
||||
start_loc = forward_batch.extend_start_loc
|
||||
seq_lens = forward_batch.extend_seq_lens[: start_loc.shape[0]]
|
||||
skip = forward_batch.encoder_swa_compress_skip
|
||||
if skip is not None:
|
||||
# Folded replay rows are cached already; compress the new rows only.
|
||||
start_loc = start_loc + skip
|
||||
seq_lens = seq_lens - skip
|
||||
self._low_ratio_compress_fused(
|
||||
layer,
|
||||
x,
|
||||
@@ -3140,9 +3145,6 @@ class DeepseekV4AttnBackend(
|
||||
extend_offsets=(start_loc.to(torch.int32), seq_lens.to(torch.int32)),
|
||||
)
|
||||
else:
|
||||
rows = forward_batch.encoder_swa_compress_rows
|
||||
if rows is not None and forward_batch.forward_mode.is_extend():
|
||||
x, req, pos = x[rows], req[rows], pos[rows]
|
||||
self._low_ratio_compress_torch(
|
||||
layer,
|
||||
x,
|
||||
|
||||
@@ -12,7 +12,6 @@ class FoldedExtend(msgspec.Struct, frozen=True):
|
||||
keep_rows: torch.Tensor # [original rows] int64, folded-row index of each
|
||||
scheduled_row: torch.Tensor # [folded rows] int64, original row, -1 if replayed
|
||||
row_floor: torch.Tensor # [folded rows] int64
|
||||
compress_skip: torch.Tensor # [bs] int32, leading replay rows per request
|
||||
replay_lens: list[int] # [bs] leading replay rows per request
|
||||
num_rows: int
|
||||
|
||||
@@ -41,7 +40,6 @@ def folded_replay_rows(batch) -> int:
|
||||
|
||||
def apply_folded_extend(folded: FoldedExtend, forward_batch) -> None:
|
||||
forward_batch.encoder_swa_row_floor = folded.row_floor
|
||||
forward_batch.encoder_swa_compress_skip = folded.compress_skip
|
||||
forward_batch.encoder_swa_compress_rows = folded.keep_rows
|
||||
|
||||
|
||||
@@ -240,9 +238,6 @@ def _fold_batch(*, batch, runner, rows) -> FoldedExtend:
|
||||
keep_rows=keep_rows.to(device, non_blocking=True),
|
||||
scheduled_row=scheduled_row.to(device, non_blocking=True),
|
||||
row_floor=row_floor.to(device, non_blocking=True),
|
||||
compress_skip=torch.tensor(replay, dtype=torch.int32).to(
|
||||
device, non_blocking=True
|
||||
),
|
||||
replay_lens=replay,
|
||||
num_rows=fold_off,
|
||||
)
|
||||
|
||||
@@ -773,10 +773,8 @@ class ForwardBatch(ForwardBatchDeepSeekMHAMixin):
|
||||
ngram_embedding_info: Optional[NgramEmbeddingInfo] = None
|
||||
encoder_swa_replay: bool = False
|
||||
# Encoder replay folded into the extend: per-row window floor [num_tokens]
|
||||
# int64 (the request's replay start, 0 for other requests), and per-request
|
||||
# leading replay rows the low-ratio compressors skip [bs] int32.
|
||||
# int64 (the request's replay start, 0 for other requests).
|
||||
encoder_swa_row_floor: Optional[torch.Tensor] = None
|
||||
encoder_swa_compress_skip: Optional[torch.Tensor] = None
|
||||
encoder_swa_compress_rows: Optional[torch.Tensor] = None # non-replay rows, int64
|
||||
|
||||
# DeepSeek-V4.1 engram, extend only: the n - 1 tokens before each request's
|
||||
|
||||
Reference in New Issue
Block a user