From c891c5c7ca5c1bf2229ee3020074514f86cb60f6 Mon Sep 17 00:00:00 2001 From: YAMY <74099316+YAMY1234@users.noreply.github.com> Date: Wed, 14 Oct 2026 17:27:07 -0700 Subject: [PATCH] [DSv4.1] Fold encoder-SWA replay into the extend forward (#43166) Co-authored-by: Yuwei An --- .../srt/layers/attention/base_attn_backend.py | 3 + .../layers/attention/deepseek_v4_backend.py | 30 ++ python/sglang/srt/managers/tp_worker.py | 31 +- .../srt/model_executor/encoder_swa_replay.py | 285 ++++++++++--- .../srt/model_executor/forward_batch_info.py | 6 + .../model_executor/test_encoder_swa_replay.py | 377 ++++++++++++++++++ 6 files changed, 678 insertions(+), 54 deletions(-) create mode 100644 test/registered/unit/model_executor/test_encoder_swa_replay.py diff --git a/python/sglang/srt/layers/attention/base_attn_backend.py b/python/sglang/srt/layers/attention/base_attn_backend.py index 1502623ef6..fa79775581 100644 --- a/python/sglang/srt/layers/attention/base_attn_backend.py +++ b/python/sglang/srt/layers/attention/base_attn_backend.py @@ -61,6 +61,9 @@ class AttentionBackend(ABC): decode_attention_backend_str: Optional[str] = None supports_ragged_verify_graph: bool = False + # DeepSeek-V4.1 encoder SWA replay: True if the backend floors the window and + # skips cached-row compression for replay rows folded into the extend. + folds_encoder_swa_replay: bool = False # Compute / KV-cache dtype. Only backends that need them (MLA/MHA fp8 # fuse-rope checks) set these in __init__; declared here as None so callers # can read them off ANY backend — including hybrid wrappers that don't set diff --git a/python/sglang/srt/layers/attention/deepseek_v4_backend.py b/python/sglang/srt/layers/attention/deepseek_v4_backend.py index 32e34d2ae0..ecb84751d1 100644 --- a/python/sglang/srt/layers/attention/deepseek_v4_backend.py +++ b/python/sglang/srt/layers/attention/deepseek_v4_backend.py @@ -941,6 +941,13 @@ def _tail_rows( return t[token_indices] +def _pad_rows(t: torch.Tensor, *, num_rows: int) -> torch.Tensor: + assert t.shape[0] <= num_rows, (t.shape[0], num_rows) + if t.shape[0] == num_rows: + return t + return torch.nn.functional.pad(t, (0, num_rows - t.shape[0])) + + # Rows per logits chunk for the ratio-1/2 indexer inside the prefill CUDA graph; # its width is the graph's max_seq_len, and longer contexts replay eagerly. _PREFILL_GRAPH_INDEXER_ROW_CHUNK = 2048 @@ -1103,6 +1110,7 @@ class DeepseekV4AttnBackend( use_captured_forward_metadata_for_breakable_cuda_graph: bool = True supports_prefill_cuda_graph_max_context_size: bool = True supports_ragged_verify_graph: bool = True + folds_encoder_swa_replay: bool = True needs_cpu_seq_lens: bool = False trtllm_attn: bool = False @@ -1134,6 +1142,7 @@ class DeepseekV4AttnBackend( super().__init__() self.model_runner = model_runner self.encoder_replay = False + self.encoder_row_floor = None self.device = torch.device(model_runner.device) self.max_context_len = model_runner.model_config.context_len head_dim = model_runner.model_config.head_dim @@ -1632,6 +1641,12 @@ class DeepseekV4AttnBackend( tail_len=SWA_WINDOW, device=device, ) + if self.encoder_row_floor is not None: + # Folded hits: a tail row never reads below its request's replay start. + swa_replay_start = torch.maximum( + swa_replay_start, + self.encoder_row_floor[token_indices].to(swa_replay_start.dtype), + ) contiguous_start = ( extend_lens_cpu[0] - tail_lens_cpu[0] if len(extend_lens_cpu) == 1 else None ) @@ -2541,6 +2556,7 @@ class DeepseekV4AttnBackend( return self.encoder_replay = forward_batch.encoder_swa_replay + self.encoder_row_floor = forward_batch.encoder_swa_row_floor self.forward_metadata = self._build_forward_metadata(forward_batch) self.init_forward_metadata_in_graph(forward_batch) self.tail_forward_metadata = ( @@ -3092,6 +3108,11 @@ class DeepseekV4AttnBackend( # length is the request count; padded rows belong to no request. start_loc = forward_batch.extend_start_loc seq_lens = forward_batch.extend_seq_lens[: start_loc.shape[0]] + skip = forward_batch.encoder_swa_compress_skip + if skip is not None: + # Folded replay rows are cached already; compress the new rows only. + start_loc = start_loc + skip + seq_lens = seq_lens - skip self._low_ratio_compress_fused( layer, x, @@ -3100,6 +3121,9 @@ class DeepseekV4AttnBackend( extend_offsets=(start_loc.to(torch.int32), seq_lens.to(torch.int32)), ) else: + rows = forward_batch.encoder_swa_compress_rows + if rows is not None and forward_batch.forward_mode.is_extend(): + x, req, pos = x[rows], req[rows], pos[rows] self._low_ratio_compress_torch( layer, x, @@ -4218,6 +4242,12 @@ class DeepseekV4AttnBackend( ) group_first = torch.cummax(torch.where(starts, offset, 0), dim=0).values swa_replay_start = raw_positions - (offset - group_first) + elif swa_replay_start is None and self.encoder_row_floor is not None: + # Folded replay: hits floor every row at their replay start; + # MLP-sync padding rows follow the real ones with floor 0. + swa_replay_start = _pad_rows( + self.encoder_row_floor, num_rows=raw_positions.shape[0] + ) request_layout = window_layout( req_pool_indices_repeated, raw_positions, diff --git a/python/sglang/srt/managers/tp_worker.py b/python/sglang/srt/managers/tp_worker.py index c6b424ce8a..bd962fec5b 100644 --- a/python/sglang/srt/managers/tp_worker.py +++ b/python/sglang/srt/managers/tp_worker.py @@ -612,24 +612,33 @@ class TpModelWorker(BaseTpWorker): return_kv_loc_plan: bool = False, ) -> GenerationBatchResult: # Get forward batch from schedule batch + folded = None if batch is not None: # update the consumer index of hicache to the running batch self.set_hicache_consumer(batch.hicache_consumer_index) if get_exec().features.enable_encoder_swa_bounded_replay: from sglang.srt.model_executor.encoder_swa_replay import ( + apply_folded_extend, + fold_encoder_swa_replay, run_encoder_swa_replay, ) - # Replay reads restored main/indexer KV before the normal extend. - run_encoder_swa_replay(self, batch) + if self.model_runner.attn_backend.folds_encoder_swa_replay: + # Hits extend from their replay start, rebuilding the window in-pass. + folded = fold_encoder_swa_replay(self, batch) + else: + # Replay reads restored main/indexer KV before the normal extend. + run_encoder_swa_replay(self, batch) forward_batch = ForwardBatch.init_new( - batch, + batch if folded is None else folded.batch, self.model_runner, capture_hidden_mode=capture_hidden_mode, return_hidden_states_before_norm=False, ) + if folded is not None: + apply_folded_extend(folded, forward_batch) else: # FIXME(lsyin): unify the interface of forward_batch assert forward_batch is not None @@ -649,13 +658,27 @@ class TpModelWorker(BaseTpWorker): pp_proxy_tensors=pp_proxy_tensors, ) logits_output, can_run_cuda_graph = out.logits_output, out.can_run_graph + if folded is not None: + from sglang.srt.model_executor.encoder_swa_replay import ( + drop_folded_rows, + ) + + drop_folded_rows(logits_output=logits_output, folded=folded) batch_result = GenerationBatchResult( logits_output=logits_output, can_run_cuda_graph=can_run_cuda_graph, expert_distribution_metrics=out.expert_distribution_metrics, routed_experts_output=out.routed_experts_output, indexer_topk_output=out.indexer_topk_output, - kv_loc_plan=forward_batch.kv_loc_plan if return_kv_loc_plan else None, + kv_loc_plan=( + None + if not return_kv_loc_plan + # A folded forward also wrote the replay rows; the caller's + # plan covers the batch it scheduled. + else forward_batch.kv_loc_plan + if folded is None + else self.model_runner.kv_index_translator.own_plan(batch) + ), ) capture_pre_sample_logits(batch, forward_batch, logits_output) diff --git a/python/sglang/srt/model_executor/encoder_swa_replay.py b/python/sglang/srt/model_executor/encoder_swa_replay.py index 3f96b7d6e6..4110047e97 100644 --- a/python/sglang/srt/model_executor/encoder_swa_replay.py +++ b/python/sglang/srt/model_executor/encoder_swa_replay.py @@ -1,8 +1,72 @@ from copy import copy +from typing import Any, Optional +import msgspec import torch +class FoldedExtend(msgspec.Struct, frozen=True): + """An extend batch whose prefix hits start at their replay start.""" + + batch: Any # ScheduleBatch copy fed to ForwardBatch.init_new + keep_rows: torch.Tensor # [original rows] int64, folded-row index of each + scheduled_row: torch.Tensor # [folded rows] int64, original row, -1 if replayed + row_floor: torch.Tensor # [folded rows] int64 + compress_skip: torch.Tensor # [bs] int32, leading replay rows per request + replay_lens: list[int] # [bs] leading replay rows per request + num_rows: int + + +def fold_encoder_swa_replay(worker, batch) -> Optional[FoldedExtend]: + """Prepend each hit's <=128 replay tokens to its extend instead of replaying.""" + runner = worker.model_runner + window = runner.token_to_kv_pool.request_window + if window is None or not batch.forward_mode.is_extend_without_speculative(): + return None + rows = _reset_windows_and_collect_hits(batch=batch, window=window) + if not rows: + return None + return _fold_batch(batch=batch, runner=runner, rows=rows) + + +def apply_folded_extend(folded: FoldedExtend, forward_batch) -> None: + forward_batch.encoder_swa_row_floor = folded.row_floor + forward_batch.encoder_swa_compress_skip = folded.compress_skip + forward_batch.encoder_swa_compress_rows = folded.keep_rows + + +def drop_folded_rows(*, logits_output, folded: FoldedExtend) -> None: + """Hand downstream consumers (the DSpark draft) only the original extend rows.""" + hidden = logits_output.hidden_states + token_indices = logits_output.hidden_states_token_indices + if token_indices is not None: + # Decoder-SWA tail rows index the folded extend; keep those on original rows. + keep = _tail_rows_on_original( + folded=folded, num_tail_rows=token_indices.shape[0] + ).to(token_indices.device, non_blocking=True) + logits_output.hidden_states_token_indices = folded.scheduled_row[ + token_indices[keep] + ] + if hidden is not None: + logits_output.hidden_states = hidden[keep] + elif hidden is not None and hidden.shape[0] >= folded.num_rows: + # MLP-sync padding rows, if any, follow the real rows and are dropped too. + logits_output.hidden_states = hidden[folded.keep_rows] + + +def _tail_rows_on_original(*, folded: FoldedExtend, num_tail_rows: int): + # Each request's tail is its last SWA_WINDOW folded rows; replay rows lead it. + from sglang.srt.layers.attention.deepseek_v4_backend import SWA_WINDOW + + keep, off = [], 0 + for n, r in zip(folded.batch.extend_lens, folded.replay_lens): + tail = min(SWA_WINDOW, n) + keep.append(torch.arange(off + max(0, tail - (n - r)), off + tail)) + off += tail + assert off == num_tail_rows, (off, num_tail_rows) + return torch.cat(keep) + + def run_encoder_swa_replay(worker, batch): from sglang.srt.model_executor.forward_batch_info import ( CaptureHiddenMode, @@ -13,11 +77,28 @@ def run_encoder_swa_replay(worker, batch): window = runner.token_to_kv_pool.request_window if window is None or not batch.forward_mode.is_extend_without_speculative(): return + rows = _reset_windows_and_collect_hits(batch=batch, window=window) + if not rows: + return + # One forward replays every hitting request; each request's window is + # floored at its own start, so the result matches per-request replays. + replay = _build_replay_batch(batch=batch, runner=runner, rows=rows) + fb = ForwardBatch.init_new( + replay, + runner, + capture_hidden_mode=CaptureHiddenMode.NULL, + return_hidden_states_before_norm=False, + ) + fb.encoder_swa_replay = True + runner.forward(fb) + + +def _reset_windows_and_collect_hits(*, batch, window): + rows = [] for i, reset in enumerate(batch.encoder_swa_reset): if not reset: continue - slot = batch.req_pool_indices[i : i + 1] - window.reset(slot) + window.reset(batch.req_pool_indices[i : i + 1]) end = batch.prefix_lens[i] if not end: continue @@ -25,53 +106,157 @@ def run_encoder_swa_replay(worker, batch): raise ValueError( "encoder SWA replay requires an even cached-prefix boundary" ) - start = max(0, end - 128) - req = batch.reqs[i] - replay = copy(batch) - replay.reqs = [req] - replay.input_ids = torch.tensor( - list(req.full_untruncated_fill_ids[start:end]), - dtype=torch.int64, - device=runner.device, - ) - replay.prefill_input_ids_cpu = None - replay.req_pool_indices = slot - replay.req_pool_indices_cpu = batch.req_pool_indices_cpu[i : i + 1] - replay.prefix_lens = [start] - replay.extend_lens = [end - start] - replay.extend_num_tokens = end - start - replay.seq_lens = torch.tensor([end], dtype=torch.int64, device=runner.device) - replay.seq_lens_cpu = torch.tensor([end], dtype=torch.int64) - replay.seq_lens_sum = end - replay.orig_seq_lens = replay.seq_lens - replay.out_cache_loc = runner.req_to_token_pool.req_to_token[ - slot[0], start:end - ].long() - replay.return_logprob = False - replay.top_logprobs_nums = None - replay.token_ids_logprobs = None - replay.extend_logprob_start_lens = [end - start] - replay.extend_input_logprob_token_ids = None - replay.is_prefill_only = True - replay.spec_info = None - replay.sampling_info = None - replay.has_grammar = False - replay.multimodal_inputs = [None] - replay.engram_history = None - hasher = runner.model.model.engram_hasher - if hasher is not None: - n = hasher.max_ngram_size - 1 - ids = list(req.full_untruncated_fill_ids[max(0, start - n) : start]) - replay.engram_history = torch.tensor( - [[0] * (n - len(ids)) + ids], - dtype=torch.int32, - device=runner.device, + rows.append((i, max(0, end - 128), end)) + return rows + + +def _build_replay_batch(*, batch, runner, rows): + idx = [i for i, _, _ in rows] + starts = [s for _, s, _ in rows] + ends = [e for _, _, e in rows] + lens = [e - s for s, e in zip(starts, ends)] + reqs = [batch.reqs[i] for i in idx] + slots = batch.req_pool_indices[idx] + req_to_token = runner.req_to_token_pool.req_to_token + + replay = copy(batch) + replay.reqs = reqs + replay.input_ids = torch.tensor( + [ + t + for r, s, e in zip(reqs, starts, ends) + for t in r.full_untruncated_fill_ids[s:e] + ], + dtype=torch.int64, + device=runner.device, + ) + replay.prefill_input_ids_cpu = None + replay.req_pool_indices = slots + replay.req_pool_indices_cpu = batch.req_pool_indices_cpu[idx] + replay.prefix_lens = starts + replay.extend_lens = lens + replay.extend_num_tokens = sum(lens) + replay.seq_lens = torch.tensor(ends, dtype=torch.int64, device=runner.device) + replay.seq_lens_cpu = torch.tensor(ends, dtype=torch.int64) + replay.seq_lens_sum = sum(ends) + replay.orig_seq_lens = replay.seq_lens + replay.out_cache_loc = torch.cat( + [req_to_token[slots[k], s:e] for k, (s, e) in enumerate(zip(starts, ends))] + ).long() + replay.return_logprob = False + replay.top_logprobs_nums = None + replay.token_ids_logprobs = None + replay.extend_logprob_start_lens = lens + replay.extend_input_logprob_token_ids = None + replay.is_prefill_only = True + replay.spec_info = None + replay.sampling_info = None + replay.has_grammar = False + replay.multimodal_inputs = [None] * len(reqs) + replay.engram_history = _engram_history(reqs=reqs, starts=starts, runner=runner) + _set_execution_counts(batch=replay, scheduled=batch) + return replay + + +def _engram_history(*, reqs, starts, runner): + hasher = runner.model.model.engram_hasher + if hasher is None: + return None + n = hasher.max_ngram_size - 1 + hist = [] + for r, s in zip(reqs, starts): + ids = list(r.full_untruncated_fill_ids[max(0, s - n) : s]) + hist.append([0] * (n - len(ids)) + ids) + return torch.tensor(hist, dtype=torch.int32, device=runner.device) + + +def _fold_batch(*, batch, runner, rows) -> FoldedExtend: + bs = len(batch.reqs) + device = runner.device + replay = [0] * bs + for i, s, e in rows: + replay[i] = e - s + ext = list(batch.extend_lens) + pre = [p - r for p, r in zip(batch.prefix_lens, replay)] + new_ext = [x + r for x, r in zip(ext, replay)] + req_to_token = runner.req_to_token_pool.req_to_token + ids, locs, keep, cpu_ids = [], [], [], [] + off = fold_off = 0 + for i, req in enumerate(batch.reqs): + r, n = replay[i], ext[i] + if r: + head = list(req.full_untruncated_fill_ids[pre[i] : pre[i] + r]) + ids.append(torch.tensor(head, dtype=batch.input_ids.dtype, device=device)) + locs.append( + req_to_token[batch.req_pool_indices[i], pre[i] : pre[i] + r].long() ) - fb = ForwardBatch.init_new( - replay, - runner, - capture_hidden_mode=CaptureHiddenMode.NULL, - return_hidden_states_before_norm=False, + cpu_ids.append(torch.tensor(head, dtype=torch.int64)) + ids.append(batch.input_ids[off : off + n]) + locs.append(batch.out_cache_loc[off : off + n].long()) + if batch.prefill_input_ids_cpu is not None: + cpu_ids.append(batch.prefill_input_ids_cpu[off : off + n].to(torch.int64)) + keep.append(torch.arange(fold_off + r, fold_off + r + n)) + off += n + fold_off += r + n + + folded = copy(batch) + folded.input_ids = torch.cat(ids) + folded.out_cache_loc = torch.cat(locs) + folded.prefill_input_ids_cpu = ( + torch.cat(cpu_ids) if batch.prefill_input_ids_cpu is not None else None + ) + folded.prefix_lens = pre + folded.extend_lens = new_ext + folded.extend_num_tokens = fold_off + if batch.extend_logprob_start_lens is not None: + folded.extend_logprob_start_lens = [ + x + r for x, r in zip(batch.extend_logprob_start_lens, replay) + ] + if batch.engram_history is not None: + folded.engram_history = _engram_history( + reqs=batch.reqs, starts=pre, runner=runner ) - fb.encoder_swa_replay = True - runner.forward(fb) + _set_execution_counts(batch=folded, scheduled=batch) + floor = torch.tensor( + [p if r else 0 for p, r in zip(pre, replay)], dtype=torch.int64 + ) + row_floor = torch.repeat_interleave( + floor, torch.tensor(new_ext), output_size=fold_off + ) + keep_rows = torch.cat(keep) + scheduled_row = torch.full((fold_off,), -1, dtype=torch.int64) + scheduled_row[keep_rows] = torch.arange(keep_rows.shape[0]) + return FoldedExtend( + batch=folded, + keep_rows=keep_rows.to(device, non_blocking=True), + scheduled_row=scheduled_row.to(device, non_blocking=True), + row_floor=row_floor.to(device, non_blocking=True), + compress_skip=torch.tensor(replay, dtype=torch.int32).to( + device, non_blocking=True + ), + replay_lens=replay, + num_rows=fold_off, + ) + + +def _set_execution_counts(*, batch, scheduled) -> None: + """MLP-sync counts of a batch copied from `scheduled` with different rows.""" + if scheduled.global_num_tokens is None: + return + # The flag rejects attention DP, so the gather holds only this rank's count. + if ( + len(scheduled.global_num_tokens) != 1 + or scheduled.global_num_tokens[0] != scheduled.extend_num_tokens + ): + raise ValueError( + "encoder SWA replay expects one MLP-sync count equal to the scheduled " + f"extend ({scheduled.extend_num_tokens}), got {scheduled.global_num_tokens}" + ) + batch.global_num_tokens = [batch.extend_num_tokens] + # The scheduler's formula; the draft counts stay with the scheduled batch. + batch.global_num_tokens_for_logprob = [ + sum( + max(n - start, 1) + for n, start in zip(batch.extend_lens, batch.extend_logprob_start_lens) + ) + ] diff --git a/python/sglang/srt/model_executor/forward_batch_info.py b/python/sglang/srt/model_executor/forward_batch_info.py index 72ff61acb3..30fe44c4bc 100644 --- a/python/sglang/srt/model_executor/forward_batch_info.py +++ b/python/sglang/srt/model_executor/forward_batch_info.py @@ -772,6 +772,12 @@ class ForwardBatch(ForwardBatchDeepSeekMHAMixin): # For ngram embedding ngram_embedding_info: Optional[NgramEmbeddingInfo] = None encoder_swa_replay: bool = False + # Encoder replay folded into the extend: per-row window floor [num_tokens] + # int64 (the request's replay start, 0 for other requests), and per-request + # leading replay rows the low-ratio compressors skip [bs] int32. + encoder_swa_row_floor: Optional[torch.Tensor] = None + encoder_swa_compress_skip: Optional[torch.Tensor] = None + encoder_swa_compress_rows: Optional[torch.Tensor] = None # non-replay rows, int64 # DeepSeek-V4.1 engram, extend only: the n - 1 tokens before each request's # first extend token, oldest first, [bs, n - 1] int32 (see EngramHasher). diff --git a/test/registered/unit/model_executor/test_encoder_swa_replay.py b/test/registered/unit/model_executor/test_encoder_swa_replay.py new file mode 100644 index 0000000000..b37302c804 --- /dev/null +++ b/test/registered/unit/model_executor/test_encoder_swa_replay.py @@ -0,0 +1,377 @@ +"""Encoder-SWA replay folded into the extend forward: row bookkeeping of the +folded batch and the paths that consume it. CPU only.""" + +import unittest +from types import SimpleNamespace +from unittest.mock import MagicMock, patch + +import torch + +from sglang.kernels.ops.attention.dsv4_attn_metadata_kernels import ( + late_layer_tail_layout, +) +from sglang.srt.distributed import parallel_state +from sglang.srt.distributed.parallel_state import GroupCoordinator +from sglang.srt.layers.attention.deepseek_v4_backend import ( + SWA_WINDOW, + DeepseekV4AttnBackend, + DSV4AttnMetadata, +) +from sglang.srt.mem_cache.dsv41_request_window import window_layout +from sglang.srt.model_executor.cuda_graph_config import CudaGraphConfig, PhaseConfig +from sglang.srt.model_executor.encoder_swa_replay import ( + _build_replay_batch, + _fold_batch, + drop_folded_rows, +) +from sglang.srt.model_executor.forward_batch_info import ForwardBatch, ForwardMode +from sglang.srt.runtime_context import get_context, get_parallel +from sglang.test.ci.ci_register import register_cpu_ci +from sglang.test.test_utils import CustomTestCase + +register_cpu_ci(est_time=10, suite="base-a-test-cpu") + +MAX_CTX = 1024 + + +def _loc(slot, pos): + return slot * MAX_CTX + pos + + +def _tok(slot, pos): + return 1000 * (slot + 1) + pos + + +# (req slot, cached prefix, new tokens). Slot 1 misses; slot 2 hits with a full +# 128-token replay and a folded extend longer than the 128-row tail; slot 3 hits +# a 64-token prefix (replay from 0) and its whole folded extend is its tail. +REQS = [(1, 0, 40), (2, 512, 30), (3, 64, 20)] +HITS = [(1, 384, 512), (2, 0, 64)] # (batch index, replay start, prefix) + + +class _Fixture: + def __init__(self): + self.req_to_token = torch.tensor( + [[_loc(s, p) for p in range(MAX_CTX)] for s in range(4)], dtype=torch.int32 + ) + self.runner = SimpleNamespace( + device="cpu", + req_to_token_pool=SimpleNamespace(req_to_token=self.req_to_token), + ) + reqs = [ + SimpleNamespace( + full_untruncated_fill_ids=[_tok(s, p) for p in range(p + n)] + ) + for s, p, n in REQS + ] + rows = [(s, p, p + n) for s, p, n in REQS] + self.extend_num_tokens = sum(n for _, _, n in REQS) + self.batch = SimpleNamespace( + reqs=reqs, + req_pool_indices=torch.tensor([s for s, _, _ in REQS]), + prefix_lens=[p for _, p, _ in REQS], + extend_lens=[n for _, _, n in REQS], + extend_num_tokens=self.extend_num_tokens, + input_ids=torch.tensor( + [_tok(s, q) for s, a, b in rows for q in range(a, b)] + ), + out_cache_loc=torch.tensor( + [_loc(s, q) for s, a, b in rows for q in range(a, b)] + ), + prefill_input_ids_cpu=None, + # Slot 3 asks for input logprobs from its 6th new token on. + extend_logprob_start_lens=[40, 30, 5], + engram_history=None, + global_num_tokens=[self.extend_num_tokens], + global_num_tokens_for_logprob=[1 + 1 + 15], + can_run_decode_cuda_graph=False, + can_run_dp_draft_cuda_graph=False, + dp_spec_prefill_coordination_applied=False, + ) + self.folded = _fold_batch( + batch=self.batch, + runner=self.runner, + rows=[(i, start, end) for i, start, end in HITS], + ) + + def folded_positions(self): + fb = self.folded.batch + return torch.cat( + [torch.arange(p, p + n) for p, n in zip(fb.prefix_lens, fb.extend_lens)] + ) + + def folded_seq_lens(self): + fb = self.folded.batch + return [p + n for p, n in zip(fb.prefix_lens, fb.extend_lens)] + + +class TestFoldedExtendRows(CustomTestCase): + def test_mixed_hit_and_miss_fold_maps_every_row(self): + """Hits extend from their replay start into their cached KV slots; every + per-row view (tokens, KV slots, window floor, logprob range, hidden rows) + stays aligned with the scheduled batch.""" + fx = _Fixture() + fb, folded = fx.folded.batch, fx.folded + starts = [0, 384, 0] + self.assertEqual(fb.prefix_lens, starts) + self.assertEqual(fb.extend_lens, [40, 158, 84]) + self.assertEqual(fb.extend_num_tokens, 282) + spans = [(s, a, p + n) for (s, p, n), a in zip(REQS, starts)] + torch.testing.assert_close( + fb.input_ids, + torch.tensor([_tok(s, q) for s, a, b in spans for q in range(a, b)]), + ) + torch.testing.assert_close( + fb.out_cache_loc, + torch.tensor([_loc(s, q) for s, a, b in spans for q in range(a, b)]), + ) + # A miss is never floored; a hit's rows all floor at its replay start. + torch.testing.assert_close( + folded.row_floor, + torch.tensor([0] * 40 + [384] * 158 + [0] * 84), + ) + torch.testing.assert_close( + folded.compress_skip, torch.tensor([0, 128, 64], dtype=torch.int32) + ) + # Input logprobs cover the same tokens, so their row count is unchanged. + self.assertEqual(fb.extend_logprob_start_lens, [40, 158, 69]) + logprob_rows = [ + max(n - s, 1) for n, s in zip(fb.extend_lens, fb.extend_logprob_start_lens) + ] + self.assertEqual(sum(logprob_rows), fx.batch.global_num_tokens_for_logprob[0]) + # Full hidden states of the folded forward reduce to the scheduled rows. + out = SimpleNamespace( + hidden_states=fb.input_ids[:, None].float(), + hidden_states_token_indices=None, + ) + drop_folded_rows(logits_output=out, folded=folded) + torch.testing.assert_close( + out.hidden_states, fx.batch.input_ids[:, None].float() + ) + + +class TestFoldWithDecoderSwaTail(CustomTestCase): + """--enable-encoder-swa-bounded-replay with --enable-decoder-swa-bounded-replay: + the late layers run only each request's last SWA_WINDOW folded rows.""" + + def _backend(self, fx): + backend = object.__new__(DeepseekV4AttnBackend) + backend.encoder_replay = False + backend.encoder_row_floor = fx.folded.row_floor + backend.req_to_token = fx.req_to_token + backend.page_size = 256 + backend.index_topk = 512 + backend.present_ratios = (0,) + backend.low_ratios = () + backend.has_c4 = backend.has_c128 = False + backend.trtllm_attn = False + backend.cuda_int32_kwargs = {"dtype": torch.int32, "device": "cpu"} + backend.token_to_kv_pool = SimpleNamespace( + request_window=SimpleNamespace(capacity=2 * SWA_WINDOW) + ) + return backend + + def test_tail_keeps_its_own_floor_under_fold(self): + """A prefix hit with both flags on builds the tail metadata; each tail row + is floored at its tail start or its request's replay start, whichever is + later. It used to take the fold's per-row floor of all folded rows and + fail a row-count assert.""" + fx = _Fixture() + fb = fx.folded.batch + seq_lens = fx.folded_seq_lens() + forward_batch = SimpleNamespace( + forward_mode=ForwardMode.EXTEND, + extend_seq_lens_cpu=list(fb.extend_lens), + seq_lens_cpu=torch.tensor(seq_lens), + seq_lens=torch.tensor(seq_lens), + req_pool_indices=fb.req_pool_indices, + out_cache_loc=fb.out_cache_loc, + positions=fx.folded_positions(), + ) + with ( + patch.object(DSV4AttnMetadata, "init_compression_metadata"), + patch.object(DSV4AttnMetadata, "init_flashmla_related"), + ): + metadata = self._backend(fx)._build_late_layer_tail_metadata(forward_batch) + + # Tails: the miss's 40 rows, the last 128 of the 158-row hit (from 414, + # above its replay start 384), and the whole 84-row hit (from 0). + tail = metadata.late_layer_tail + self.assertEqual(tail.extend_seq_lens_cpu, [40, 128, 84]) + floor = torch.tensor([0] * 40 + [414] * 128 + [0] * 84) + req = torch.tensor([1] * 40 + [2] * 128 + [3] * 84) + expected = window_layout( + req, + tail.positions, + capacity=2 * SWA_WINDOW, + floor=floor, + num_groups=3, + ) + layout = metadata.core_attn_metadata.request_window_layout + torch.testing.assert_close(layout.lengths, expected.lengths) + torch.testing.assert_close(layout.indices, expected.indices) + + def test_tail_hidden_rows_map_to_scheduled_rows(self): + """With DSpark the model returns the tail's hidden rows and their indices + into the folded extend. The draft indexes scheduled-row cache locations + with them, so they must come back in scheduled-row space without the + replayed rows.""" + fx = _Fixture() + fb = fx.folded.batch + token_indices, _, _ = late_layer_tail_layout( + extend_lens_cpu=list(fb.extend_lens), + seq_lens_cpu=fx.folded_seq_lens(), + tail_len=SWA_WINDOW, + device=torch.device("cpu"), + ) + out = SimpleNamespace( + hidden_states=fb.input_ids[token_indices][:, None].float(), + hidden_states_token_indices=token_indices, + ) + drop_folded_rows(logits_output=out, folded=fx.folded) + + # Kept: all 40 miss rows, the hit's 30 new rows, the small hit's 20. + expected = torch.cat( + [torch.arange(0, 40), torch.arange(40, 70), torch.arange(70, 90)] + ) + torch.testing.assert_close(out.hidden_states_token_indices, expected) + torch.testing.assert_close( + out.hidden_states, fx.batch.input_ids[expected][:, None].float() + ) + + +class TestFoldMlpSync(CustomTestCase): + """Pure TP that still syncs the MLP (e.g. TP4 with --moe-dense-tp-size 1).""" + + ATTN_TP = 4 + + def setUp(self): + override = get_context().override_server_args( + tp_size=self.ATTN_TP, + cuda_graph_config=CudaGraphConfig(prefill=PhaseConfig(bs=[])), + ) + override.install() + self.addCleanup(override.restore) + + def _forward_batch(self, batch): + num_rows = batch.input_ids.shape[0] + seq_lens = [p + n for p, n in zip(batch.prefix_lens, batch.extend_lens)] + fb = ForwardBatch( + forward_mode=ForwardMode.EXTEND, + batch_size=len(seq_lens), + input_ids=batch.input_ids, + req_pool_indices=batch.req_pool_indices, + seq_lens=torch.tensor(seq_lens), + out_cache_loc=batch.out_cache_loc, + seq_lens_sum=sum(seq_lens), + positions=torch.zeros(num_rows, dtype=torch.int64), + seq_lens_cpu=torch.tensor(seq_lens), + extend_num_tokens=batch.extend_num_tokens, + extend_seq_lens=torch.tensor(batch.extend_lens), + extend_seq_lens_cpu=list(batch.extend_lens), + extend_prefix_lens_cpu=list(batch.prefix_lens), + is_extend_in_batch=True, + ) + fb.init_mlp_sync_metadata(batch, torch.device("cpu")) + return fb + + def _mlp_sync(self, fb): + runner = MagicMock() + runner.attn_backend.get_cuda_graph_seq_len_fill_value.return_value = 1 + runner.attn_backend.get_cpu_graph_seq_len_fill_value.return_value = 1 + runner.is_draft_worker = False + runner.attn_tp_sequence_sharded.return_value = False + tp_group = GroupCoordinator.__new__(GroupCoordinator) + tp_group.world_size = self.ATTN_TP + tp_group.rank_in_group = 0 + with ( + get_parallel().override(tp_rank=0, attn_tp_rank=0), + patch.object(parallel_state, "_TP", tp_group), + # CPU runners have no driver to pin the synced token counts with. + patch("sglang.srt.model_executor.forward_batch_info._is_cpu", True), + ): + fb.prepare_mlp_sync_batch(runner) + + def test_folded_rows_count_in_mlp_sync_totals(self): + """The scheduler's gathered count covers the scheduled rows only. A folded + batch must sync its folded row count, or the padding to that count + shrinks input_ids (a negative pad); logprob rows keep their count.""" + fx = _Fixture() + fb = self._forward_batch(fx.folded.batch) + self._mlp_sync(fb) + + padded = 284 # 282 folded rows aligned to the attention TP size + self.assertEqual(fb.global_num_tokens_cpu, [padded]) + self.assertEqual(fb.extend_num_tokens, padded) + self.assertEqual(fb.input_ids.shape[0], padded) + self.assertEqual(fb.out_cache_loc.shape[0], padded) + torch.testing.assert_close(fb.input_ids[:282], fx.folded.batch.input_ids) + self.assertEqual(fb.global_num_tokens_for_logprob_cpu, [17]) + # Hidden states of the padded forward still reduce to the scheduled rows. + out = SimpleNamespace( + hidden_states=fb.input_ids[:, None].float(), + hidden_states_token_indices=None, + ) + drop_folded_rows(logits_output=out, folded=fx.folded) + torch.testing.assert_close( + out.hidden_states, fx.batch.input_ids[:, None].float() + ) + + def test_batched_replay_rows_count_in_mlp_sync_totals(self): + """Backends that do not fold run one replay forward for all hits of a step. + Its rows (128 per hit) are not the scheduled rows the gather counted, so it + must sync its own count, or the padding shrinks input_ids.""" + fx = _Fixture() + hits = [(1, 512, 64), (2, 640, 64)] # (req slot, cached prefix, new tokens) + rows = [(s, p, p + n) for s, p, n in hits] + batch = SimpleNamespace( + **{ + **vars(fx.batch), + "reqs": [ + SimpleNamespace( + full_untruncated_fill_ids=[_tok(s, q) for q in range(p + n)] + ) + for s, p, n in hits + ], + "req_pool_indices": torch.tensor([s for s, _, _ in hits]), + "req_pool_indices_cpu": torch.tensor([s for s, _, _ in hits]), + "prefix_lens": [p for _, p, _ in hits], + "extend_lens": [n for _, _, n in hits], + "extend_num_tokens": 128, + "input_ids": torch.tensor( + [_tok(s, q) for s, a, b in rows for q in range(a, b)] + ), + "out_cache_loc": torch.tensor( + [_loc(s, q) for s, a, b in rows for q in range(a, b)] + ), + "extend_logprob_start_lens": [64, 64], + "global_num_tokens": [128], + "global_num_tokens_for_logprob": [2], + } + ) + runner = SimpleNamespace( + **vars(fx.runner), + model=SimpleNamespace(model=SimpleNamespace(engram_hasher=None)), + ) + replay = _build_replay_batch( + batch=batch, runner=runner, rows=[(0, 384, 512), (1, 512, 640)] + ) + fb = self._forward_batch(replay) + self._mlp_sync(fb) + + self.assertEqual(fb.global_num_tokens_cpu, [256]) + self.assertEqual(fb.extend_num_tokens, 256) + self.assertEqual(fb.input_ids.shape[0], 256) + torch.testing.assert_close( + fb.input_ids, + torch.tensor( + [_tok(1, q) for q in range(384, 512)] + + [_tok(2, q) for q in range(512, 640)] + ), + ) + # One sampled row per replayed request, as the scheduler would count it. + self.assertEqual(fb.global_num_tokens_for_logprob_cpu, [2]) + + +if __name__ == "__main__": + unittest.main()