[Fix] deepseek-v4.1: Keep the cached KV of folded encoder replay rows (#44857)
This commit is contained in:
@@ -2704,6 +2704,16 @@ class DeepseekV4AttnBackend(
|
||||
request_window_layout=request_layout,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _compressor_out_loc(forward_batch: ForwardBatch) -> torch.Tensor:
|
||||
"""Slot 0 for folded replay rows: their main KV, compressed groups and index
|
||||
keys are cached, so the compressors write only the new rows."""
|
||||
loc = forward_batch.out_cache_loc
|
||||
keep = forward_batch.encoder_swa_compress_rows
|
||||
if keep is None:
|
||||
return loc
|
||||
return torch.zeros_like(loc).index_copy_(0, keep, loc[keep])
|
||||
|
||||
def _build_forward_metadata(
|
||||
self,
|
||||
forward_batch: ForwardBatch,
|
||||
@@ -2804,7 +2814,7 @@ class DeepseekV4AttnBackend(
|
||||
req_pool_indices=req_pool_indices,
|
||||
seq_lens=seq_lens,
|
||||
seq_lens_cpu=seq_lens_cpu.tolist(),
|
||||
out_cache_loc=forward_batch.out_cache_loc,
|
||||
out_cache_loc=self._compressor_out_loc(forward_batch),
|
||||
num_tokens=sum(extend_seq_lens_cpu),
|
||||
extend_seq_lens=extend_seq_lens,
|
||||
extend_seq_lens_cpu=extend_seq_lens_cpu,
|
||||
@@ -3127,11 +3137,6 @@ class DeepseekV4AttnBackend(
|
||||
# length is the request count; padded rows belong to no request.
|
||||
start_loc = forward_batch.extend_start_loc
|
||||
seq_lens = forward_batch.extend_seq_lens[: start_loc.shape[0]]
|
||||
skip = forward_batch.encoder_swa_compress_skip
|
||||
if skip is not None:
|
||||
# Folded replay rows are cached already; compress the new rows only.
|
||||
start_loc = start_loc + skip
|
||||
seq_lens = seq_lens - skip
|
||||
self._low_ratio_compress_fused(
|
||||
layer,
|
||||
x,
|
||||
@@ -3140,9 +3145,6 @@ class DeepseekV4AttnBackend(
|
||||
extend_offsets=(start_loc.to(torch.int32), seq_lens.to(torch.int32)),
|
||||
)
|
||||
else:
|
||||
rows = forward_batch.encoder_swa_compress_rows
|
||||
if rows is not None and forward_batch.forward_mode.is_extend():
|
||||
x, req, pos = x[rows], req[rows], pos[rows]
|
||||
self._low_ratio_compress_torch(
|
||||
layer,
|
||||
x,
|
||||
|
||||
@@ -12,7 +12,6 @@ class FoldedExtend(msgspec.Struct, frozen=True):
|
||||
keep_rows: torch.Tensor # [original rows] int64, folded-row index of each
|
||||
scheduled_row: torch.Tensor # [folded rows] int64, original row, -1 if replayed
|
||||
row_floor: torch.Tensor # [folded rows] int64
|
||||
compress_skip: torch.Tensor # [bs] int32, leading replay rows per request
|
||||
replay_lens: list[int] # [bs] leading replay rows per request
|
||||
num_rows: int
|
||||
|
||||
@@ -41,7 +40,6 @@ def folded_replay_rows(batch) -> int:
|
||||
|
||||
def apply_folded_extend(folded: FoldedExtend, forward_batch) -> None:
|
||||
forward_batch.encoder_swa_row_floor = folded.row_floor
|
||||
forward_batch.encoder_swa_compress_skip = folded.compress_skip
|
||||
forward_batch.encoder_swa_compress_rows = folded.keep_rows
|
||||
|
||||
|
||||
@@ -240,9 +238,6 @@ def _fold_batch(*, batch, runner, rows) -> FoldedExtend:
|
||||
keep_rows=keep_rows.to(device, non_blocking=True),
|
||||
scheduled_row=scheduled_row.to(device, non_blocking=True),
|
||||
row_floor=row_floor.to(device, non_blocking=True),
|
||||
compress_skip=torch.tensor(replay, dtype=torch.int32).to(
|
||||
device, non_blocking=True
|
||||
),
|
||||
replay_lens=replay,
|
||||
num_rows=fold_off,
|
||||
)
|
||||
|
||||
@@ -773,10 +773,8 @@ class ForwardBatch(ForwardBatchDeepSeekMHAMixin):
|
||||
ngram_embedding_info: Optional[NgramEmbeddingInfo] = None
|
||||
encoder_swa_replay: bool = False
|
||||
# Encoder replay folded into the extend: per-row window floor [num_tokens]
|
||||
# int64 (the request's replay start, 0 for other requests), and per-request
|
||||
# leading replay rows the low-ratio compressors skip [bs] int32.
|
||||
# int64 (the request's replay start, 0 for other requests).
|
||||
encoder_swa_row_floor: Optional[torch.Tensor] = None
|
||||
encoder_swa_compress_skip: Optional[torch.Tensor] = None
|
||||
encoder_swa_compress_rows: Optional[torch.Tensor] = None # non-replay rows, int64
|
||||
|
||||
# DeepSeek-V4.1 engram, extend only: the n - 1 tokens before each request's
|
||||
|
||||
@@ -109,6 +109,24 @@ class _Fixture:
|
||||
return [p + n for p, n in zip(fb.prefix_lens, fb.extend_lens)]
|
||||
|
||||
|
||||
def _backend(fx):
|
||||
backend = object.__new__(DeepseekV4AttnBackend)
|
||||
backend.encoder_replay = False
|
||||
backend.encoder_row_floor = fx.folded.row_floor
|
||||
backend.req_to_token = fx.req_to_token
|
||||
backend.page_size = 256
|
||||
backend.index_topk = 512
|
||||
backend.present_ratios = (0,)
|
||||
backend.low_ratios = ()
|
||||
backend.has_c4 = backend.has_c128 = False
|
||||
backend.trtllm_attn = False
|
||||
backend.cuda_int32_kwargs = {"dtype": torch.int32, "device": "cpu"}
|
||||
backend.token_to_kv_pool = SimpleNamespace(
|
||||
request_window=SimpleNamespace(capacity=2 * SWA_WINDOW)
|
||||
)
|
||||
return backend
|
||||
|
||||
|
||||
class TestFoldedExtendRows(CustomTestCase):
|
||||
def test_mixed_hit_and_miss_fold_maps_every_row(self):
|
||||
"""Hits extend from their replay start into their cached KV slots; every
|
||||
@@ -135,9 +153,6 @@ class TestFoldedExtendRows(CustomTestCase):
|
||||
folded.row_floor,
|
||||
torch.tensor([0] * 40 + [384] * 158 + [0] * 84),
|
||||
)
|
||||
torch.testing.assert_close(
|
||||
folded.compress_skip, torch.tensor([0, 128, 64], dtype=torch.int32)
|
||||
)
|
||||
# Input logprobs cover the same tokens, so their row count is unchanged.
|
||||
self.assertEqual(fb.extend_logprob_start_lens, [40, 158, 69])
|
||||
logprob_rows = [
|
||||
@@ -167,23 +182,6 @@ class TestFoldWithDecoderSwaTail(CustomTestCase):
|
||||
"""Encoder SWA bounded replay combined with decoder SWA bounded replay:
|
||||
the late layers run only each request's last SWA_WINDOW folded rows."""
|
||||
|
||||
def _backend(self, fx):
|
||||
backend = object.__new__(DeepseekV4AttnBackend)
|
||||
backend.encoder_replay = False
|
||||
backend.encoder_row_floor = fx.folded.row_floor
|
||||
backend.req_to_token = fx.req_to_token
|
||||
backend.page_size = 256
|
||||
backend.index_topk = 512
|
||||
backend.present_ratios = (0,)
|
||||
backend.low_ratios = ()
|
||||
backend.has_c4 = backend.has_c128 = False
|
||||
backend.trtllm_attn = False
|
||||
backend.cuda_int32_kwargs = {"dtype": torch.int32, "device": "cpu"}
|
||||
backend.token_to_kv_pool = SimpleNamespace(
|
||||
request_window=SimpleNamespace(capacity=2 * SWA_WINDOW)
|
||||
)
|
||||
return backend
|
||||
|
||||
def test_tail_keeps_its_own_floor_under_fold(self):
|
||||
"""A prefix hit with both flags on builds the tail metadata; each tail row
|
||||
is floored one window back or at its request's replay start, whichever is
|
||||
@@ -205,7 +203,7 @@ class TestFoldWithDecoderSwaTail(CustomTestCase):
|
||||
patch.object(DSV4AttnMetadata, "init_compression_metadata"),
|
||||
patch.object(DSV4AttnMetadata, "init_flashmla_related"),
|
||||
):
|
||||
metadata = self._backend(fx)._build_late_layer_tail_metadata(forward_batch)
|
||||
metadata = _backend(fx)._build_late_layer_tail_metadata(forward_batch)
|
||||
|
||||
# Tails: the miss's 40 rows, the last 128 of the 158-row hit (from 414,
|
||||
# above its replay start 384), and the whole 84-row hit (from 0).
|
||||
@@ -272,6 +270,91 @@ class TestFoldWithDecoderSwaTail(CustomTestCase):
|
||||
)
|
||||
|
||||
|
||||
class TestFoldedReplayCompressorWrites(CustomTestCase):
|
||||
"""Folded replay rows rebuild only their SWA window: their main KV, compressed
|
||||
groups and index keys are cached, so the compressors write only the new rows."""
|
||||
|
||||
def _forward_batch(self, fx):
|
||||
fb = fx.folded.batch
|
||||
seq_lens = fx.folded_seq_lens()
|
||||
lens = torch.tensor(fb.extend_lens)
|
||||
return SimpleNamespace(
|
||||
forward_mode=ForwardMode.EXTEND,
|
||||
batch_size=len(fb.extend_lens),
|
||||
spec_info=None,
|
||||
max_seq_len_override=None,
|
||||
extend_seq_lens=lens,
|
||||
extend_seq_lens_cpu=list(fb.extend_lens),
|
||||
extend_start_loc=torch.cumsum(lens, 0) - lens,
|
||||
seq_lens_cpu=torch.tensor(seq_lens),
|
||||
seq_lens=torch.tensor(seq_lens),
|
||||
req_pool_indices=fb.req_pool_indices,
|
||||
out_cache_loc=fb.out_cache_loc,
|
||||
positions=fx.folded_positions(),
|
||||
encoder_swa_compress_rows=fx.folded.keep_rows,
|
||||
)
|
||||
|
||||
def _expected_out_loc(self, fx):
|
||||
"""Replay rows on the padding slot, new rows on their own KV slots."""
|
||||
fb = fx.folded.batch
|
||||
replayed = torch.cat(
|
||||
[
|
||||
torch.arange(start, start + n) < prefix
|
||||
for start, n, (_, prefix, _) in zip(
|
||||
fb.prefix_lens, fb.extend_lens, REQS
|
||||
)
|
||||
]
|
||||
)
|
||||
return fb.out_cache_loc.masked_fill(replayed, 0)
|
||||
|
||||
def test_extend_metadata_puts_replay_rows_on_the_padding_slot(self):
|
||||
fx = _Fixture()
|
||||
backend = _backend(fx)
|
||||
backend.req_to_token_pool = SimpleNamespace(req_to_token=fx.req_to_token)
|
||||
backend.online_c128_mtp = SimpleNamespace(prepare_forward=lambda *_, **__: 0)
|
||||
backend.is_dspark_draft = False
|
||||
backend.topk = backend.speculative_num_steps = 0
|
||||
with (
|
||||
patch.object(DSV4AttnMetadata, "init_compression_metadata"),
|
||||
patch.object(DSV4AttnMetadata, "init_flashmla_related"),
|
||||
):
|
||||
metadata = backend._build_forward_metadata(self._forward_batch(fx))
|
||||
|
||||
raw_out_loc = metadata.core_attn_metadata.raw_out_loc
|
||||
self.assertEqual(int((raw_out_loc == 0).sum()), REPLAY_ROWS)
|
||||
torch.testing.assert_close(raw_out_loc, self._expected_out_loc(fx))
|
||||
|
||||
def test_unfused_compressor_writes_each_row_at_its_own_slot(self):
|
||||
fx = _Fixture()
|
||||
backend = _backend(fx)
|
||||
out_loc = self._expected_out_loc(fx)
|
||||
backend.forward_metadata = SimpleNamespace(
|
||||
core_metadata=SimpleNamespace(c1_out_loc=out_loc, raw_out_loc=out_loc)
|
||||
)
|
||||
written = []
|
||||
backend._low_ratio_write_group = lambda layer, kv, slots, pos, **_: (
|
||||
written.append((kv, slots))
|
||||
)
|
||||
layer = SimpleNamespace(
|
||||
compress_ratio=1,
|
||||
layer_id=0,
|
||||
compressor=SimpleNamespace(
|
||||
use_fused_compress=False, project=lambda x: (x, None)
|
||||
),
|
||||
)
|
||||
fb = fx.folded.batch
|
||||
req = fb.req_pool_indices.repeat_interleave(torch.tensor(fb.extend_lens))
|
||||
rows = torch.arange(out_loc.shape[0], dtype=torch.float32)[:, None]
|
||||
forward_batch = self._forward_batch(fx)
|
||||
backend._low_ratio_compress(
|
||||
layer, rows, req, forward_batch.positions, forward_batch
|
||||
)
|
||||
|
||||
((kv, slots),) = written
|
||||
torch.testing.assert_close(kv, rows)
|
||||
torch.testing.assert_close(slots, out_loc)
|
||||
|
||||
|
||||
class TestFoldMlpSync(CustomTestCase):
|
||||
"""Pure TP that still syncs the MLP (e.g. TP4 with --moe-dense-tp-size 1)."""
|
||||
|
||||
|
||||
Reference in New Issue
Block a user