Compare commits

...
Author SHA1 Message Date
Florian Wunderlich d5a998f346 test: import SWA eviction helper directly
(cherry picked from commit 5729f61e175bde3ef2b67dc3a95abb1042f4bf1b)
2026-11-04 09:45:08 +08:00
Florian Wunderlich a3401b3e65 Fix SWA eviction behind the replay-trimmed prefix frontier
(cherry picked from commit f399d1e9a2bf7c33c7244ecf405e756292ddaf2b)
2026-11-04 09:45:08 +08:00
Yangmin Li 01087e689c [DSv4.1] Carry decoder tail counts through DP scheduler synchronization
On CUDA, take every rank's late-layer MoE gather size from the scheduled
per-rank trim counts, which ride the existing scheduler all-gather, and switch
and restore them with #43077's late-layer DP helpers; ROCm keeps its eager
count gather. Make the helpers' count tensor copy asynchronous so the switch
does not synchronize the stream. Restrict to attention TP 1 and TP MoE.

(cherry picked from commit a6c2d5f40efec4b491fcdd2dbcedf14ec70d3861)
2026-11-04 09:45:07 +08:00
alphabetc1 0b45c07637 perf: dsv41 decoder tail runs only the prompt's last window rows of each prefill chunk (opt-in)
(cherry picked from commit 1b778e6df01a138a35403627e209cf33967246e6)
2026-11-04 09:44:11 +08:00
17 changed files with 664 additions and 25 deletions
@@ -443,15 +443,23 @@ def late_layer_tail_lens(
extend_lens_cpu: list[int],
tail_len: int,
whole_extend: Optional[list[bool]] = None,
prompt_rest_lens: Optional[list[int]] = None,
) -> list[int]:
"""Each request's late-layer tail: its last min(tail_len, extend_len) tokens,
or its whole extend where whole_extend is set."""
if whole_extend is None:
return [min(tail_len, n) for n in extend_lens_cpu]
return [
n if whole else min(tail_len, n)
for n, whole in zip(extend_lens_cpu, whole_extend)
]
or its whole extend where whole_extend is set. With prompt_rest_lens (prompt
tokens left after each extend), only its rows in the prompt's last tail_len
tokens, and at least its last row."""
if whole_extend is not None and any(whole_extend):
return [
n if whole else min(tail_len, n)
for n, whole in zip(extend_lens_cpu, whole_extend)
]
if prompt_rest_lens is not None:
return [
min(n, max(1, tail_len - rest))
for n, rest in zip(extend_lens_cpu, prompt_rest_lens, strict=True)
]
return [min(tail_len, n) for n in extend_lens_cpu]
def late_layer_tail_layout(
@@ -461,13 +469,15 @@ def late_layer_tail_layout(
tail_len: int,
device: torch.device,
whole_extend: Optional[list[bool]] = None,
prompt_rest_lens: Optional[list[int]] = None,
) -> tuple[torch.Tensor, list[int], torch.Tensor]:
"""Tail rows of each prefill extend: its last min(tail_len, extend_len) tokens,
or its whole extend where whole_extend is set.
Returns (token indices into the extend, per-request tail lengths, per-row
absolute window floor). The floor is seq_len - tail_len, so a tail shorter
than tail_len reads rows the request wrote before this extend. A whole
extend keeps every row's plain window."""
extend keeps every row's plain window. With prompt_rest_lens, tails and
floors follow the prompt's end instead (late_layer_tail_lens, _tail_floors)."""
if whole_extend is not None and any(whole_extend):
return _whole_extend_tail_layout(
extend_lens_cpu=extend_lens_cpu,
@@ -476,16 +486,19 @@ def late_layer_tail_layout(
device=device,
whole_extend=whole_extend,
)
tail_lens_cpu = late_layer_tail_lens(extend_lens_cpu, tail_len)
tail_lens_cpu = late_layer_tail_lens(
extend_lens_cpu, tail_len, prompt_rest_lens=prompt_rest_lens
)
floors_cpu = _tail_floors(seq_lens_cpu, tail_len, prompt_rest_lens)
if len(extend_lens_cpu) == 1:
n, t, s = extend_lens_cpu[0], tail_lens_cpu[0], seq_lens_cpu[0]
floor = torch.full((t,), max(0, s - tail_len), dtype=torch.int32, device=device)
n, t = extend_lens_cpu[0], tail_lens_cpu[0]
floor = torch.full((t,), floors_cpu[0], dtype=torch.int32, device=device)
return torch.arange(n - t, n, device=device), tail_lens_cpu, floor
# One H2D copy for the three length vectors; launch count does not grow with bs.
lens = async_h2d(
[extend_lens_cpu, tail_lens_cpu, seq_lens_cpu], dtype=torch.int64, device=device
[extend_lens_cpu, tail_lens_cpu, floors_cpu], dtype=torch.int64, device=device
)
extend_lens, tail_lens, seq_lens = lens[0], lens[1], lens[2]
extend_lens, tail_lens, floors = lens[0], lens[1], lens[2]
total = sum(tail_lens_cpu)
req = torch.repeat_interleave(
torch.arange(len(tail_lens_cpu), device=device), tail_lens, output_size=total
@@ -495,10 +508,23 @@ def late_layer_tail_layout(
- (torch.cumsum(tail_lens, 0) - tail_lens)[req]
)
token_indices = (torch.cumsum(extend_lens, 0) - tail_lens)[req] + offs
floor = (seq_lens - tail_len).clamp_min(0)[req].to(torch.int32)
floor = floors[req].to(torch.int32)
return token_indices, tail_lens_cpu, floor
def _tail_floors(
seq_lens_cpu: list[int], tail_len: int, prompt_rest_lens: Optional[list[int]]
) -> list[int]:
"""Each request's window floor: tail_len below its extend's end, or below its
prompt's end, clamped to its last row, whose own KV a lone tail row reads."""
if prompt_rest_lens is None:
return [max(0, s - tail_len) for s in seq_lens_cpu]
return [
min(max(0, s + rest - tail_len), max(0, s - 1))
for s, rest in zip(seq_lens_cpu, prompt_rest_lens, strict=True)
]
def _whole_extend_tail_layout(
*,
extend_lens_cpu: list[int],
@@ -728,6 +728,7 @@ class TboForwardBatchPreparer:
"extend_prefix_lens_cpu",
"extend_seq_lens_cpu",
"extend_logprob_start_lens_cpu",
"extend_prompt_rest_lens_cpu",
"lora_ids",
"rids",
]:
+3
View File
@@ -1719,6 +1719,9 @@ class Envs:
# Prefill-graph shapes of at least this many tokens run the tail from the tail
# graphs in a break (measured crossover 1536-2048 tokens on GB300 TP4).
SGLANG_DSV4_DECODER_TAIL_GRAPH_MIN_PREFILL_TOKENS = EnvInt(2048)
# Decoder SWA bounded replay: a prefill chunk runs the tail layers only over its
# rows in the prompt's last window, else over its last row.
SGLANG_OPT_DSV41_PROMPT_WINDOW_TAIL = EnvBool(False)
# Paged KV layout of the DeepSeek-V4 family pools: "v4" (584 B/token, every
# GPU), "v41" (the SM100 FlashMLA V4.1 formats: 528 B fp8 SWA cache, fp8 or
# fp4 compressed caches), "v41_new" (the same bytes interleaved per token, for
@@ -1024,7 +1024,10 @@ def late_layer_num_tokens(forward_batch: ForwardBatch, *, dspark: bool) -> int:
whole_extend = late_layer_whole_extend(forward_batch, dspark=dspark)
return sum(
late_layer_tail_lens(
forward_batch.extend_seq_lens_cpu, SWA_WINDOW, whole_extend
forward_batch.extend_seq_lens_cpu,
SWA_WINDOW,
whole_extend,
prompt_rest_lens=forward_batch.extend_prompt_rest_lens_cpu,
)
)
@@ -1834,6 +1837,7 @@ class DeepseekV4AttnBackend(
whole_extend=late_layer_whole_extend(
forward_batch, dspark=self.model_runner.spec_algorithm.is_dspark()
),
prompt_rest_lens=forward_batch.extend_prompt_rest_lens_cpu,
)
if self.encoder_row_floor is not None:
# Folded hits: a tail row never reads below its request's replay start.
@@ -2252,6 +2252,7 @@ class DeepseekV4HipRadixBackend(
tail_len=SWA_WINDOW,
device=device,
whole_extend=late_layer_whole_extend(forward_batch, dspark=self.is_dspark),
prompt_rest_lens=forward_batch.extend_prompt_rest_lens_cpu,
)
contiguous_start = (
extend_lens_cpu[0] - tail_lens_cpu[0] if len(extend_lens_cpu) == 1 else None
@@ -2649,6 +2649,8 @@ class ScheduleBatch(ScheduleBatchDisaggregationDecodeMixin):
# For DP attention
global_num_tokens: Optional[List[int]] = None
global_num_tokens_for_logprob: Optional[List[int]] = None
# Per DP rank, the rows the decoder SWA tail drops; None when no rank trims.
global_decoder_trim_rows: Optional[List[int]] = None
# The draft model can use a different MoE A2A backend than the target.
draft_global_num_tokens: Optional[List[int]] = None
draft_global_num_tokens_for_logprob: Optional[List[int]] = None
@@ -32,8 +32,15 @@ from sglang.srt.model_executor.cuda_graph_config import (
check_cuda_graph_backend,
cuda_graph_fully_disabled,
)
from sglang.srt.model_executor.encoder_swa_replay import folded_replay_rows
from sglang.srt.model_executor.forward_batch_info import ForwardMode
from sglang.srt.model_executor.encoder_swa_replay import (
decoder_swa_trim_rows,
folded_replay_rows,
)
from sglang.srt.model_executor.forward_batch_info import (
CaptureHiddenMode,
ForwardMode,
get_server_return_hidden_states_mode,
)
from sglang.srt.model_executor.runner import PrefillCudaGraphRunner
from sglang.srt.observability.metrics_collector import (
DPBalanceStats,
@@ -161,6 +168,8 @@ class MLPSyncBatchInfo:
is_health_check: bool = False
# Causal (query, key) pairs this rank attends this step.
attention_pairs: int = 0
# Rows the decoder SWA tail drops before the late layers.
decoder_trim_rows: int = 0
# some gathered elements
tp0_info_cpu: torch.Tensor = None
@@ -169,6 +178,7 @@ class MLPSyncBatchInfo:
global_num_tokens: list[int] = None
global_num_tokens_for_logprob: list[int] = None
global_attention_pairs: list[int] = None
global_decoder_trim_rows: list[int] = None
tbo_split_seq_index: torch.Tensor = None
global_forward_mode: int = None
dp_cooperation_info: Optional[DPCooperationInfo] = None
@@ -187,6 +197,7 @@ class MLPSyncBatchInfo:
int(self.can_run_draft_cuda_graph),
int(self.is_health_check),
self.attention_pairs,
self.decoder_trim_rows,
],
device=device,
dtype=dtype,
@@ -206,6 +217,7 @@ class MLPSyncBatchInfo:
1, # can_run_draft_cuda_graph
0, # is_health_check
0, # attention_pairs
0, # decoder_trim_rows
],
device=device,
dtype=dtype,
@@ -217,6 +229,7 @@ class MLPSyncBatchInfo:
self.global_num_tokens = [self.num_tokens]
self.global_num_tokens_for_logprob = [self.num_tokens_for_logprob]
self.global_attention_pairs = [self.attention_pairs]
self.global_decoder_trim_rows = [self.decoder_trim_rows]
self.any_health_check = self.is_health_check
if _ENABLE_METRICS_DP_ATTENTION:
self.dp_cooperation_info = DPCooperationInfo.create(
@@ -294,6 +307,7 @@ class MLPSyncBatchInfo:
self.can_run_draft_cuda_graph = bool(tp0_info_cpu[:, 8].min())
self.any_health_check = bool(tp0_info_cpu[:, 9].max())
self.global_attention_pairs = tp0_info_cpu[:, 10].tolist()
self.global_decoder_trim_rows = tp0_info_cpu[:, 11].tolist()
self.sync_wait_seconds = time.perf_counter() - sync_start
if _ENABLE_METRICS_DP_ATTENTION:
self.dp_cooperation_info = DPCooperationInfo.create(
@@ -308,6 +322,7 @@ def _update_gather_batch(
draft_require_mlp_tp_gather: Optional[bool] = None,
skip_global_metadata=False,
):
batch.global_decoder_trim_rows = None
if not require_mlp_tp_gather:
batch.global_num_tokens = [mlp_sync_info.num_tokens]
batch.global_num_tokens_for_logprob = [mlp_sync_info.num_tokens_for_logprob]
@@ -316,6 +331,12 @@ def _update_gather_batch(
batch.global_num_tokens_for_logprob = (
mlp_sync_info.global_num_tokens_for_logprob
)
if (
any(mlp_sync_info.global_decoder_trim_rows or ())
and min(mlp_sync_info.global_decoder_trim_rows) >= 0
):
# Every rank resizes its late-layer DP gather when any rank trims.
batch.global_decoder_trim_rows = mlp_sync_info.global_decoder_trim_rows
# Reuse the same all-gather result for a draft model whose A2A backend
# requires a different local/full token-count representation.
if draft_require_mlp_tp_gather is not None:
@@ -532,6 +553,7 @@ def prepare_mlp_sync_batch_raw(
attn_tp_size = parallel.attn_tp_size
tp_group = parallel.tp_group
# Check if other DP workers have running batches
decoder_trim_rows = 0
if (
local_batch is None
or local_batch.forward_mode.is_prebuilt()
@@ -549,6 +571,15 @@ def prepare_mlp_sync_batch_raw(
and model_runner.attn_backend.folds_encoder_swa_replay
):
num_tokens += folded_replay_rows(local_batch)
if get_exec().features.enable_decoder_swa_bounded_replay:
decoder_trim_rows = (
-1
if get_server_return_hidden_states_mode() == CaptureHiddenMode.FULL
else decoder_swa_trim_rows(
local_batch,
folds_encoder=model_runner.attn_backend.folds_encoder_swa_replay,
)
)
num_tokens_for_logprob = sum(
# We should have at least 1 token for sample in every case.
max(extend_len - logprob_start_len, 1)
@@ -641,6 +672,7 @@ def prepare_mlp_sync_batch_raw(
# Gathered unconditionally like the other columns, so a per-process
# metrics flag cannot make ranks disagree on what the tensor holds.
attention_pairs=_local_attention_pairs(local_batch),
decoder_trim_rows=decoder_trim_rows,
)
if num_dp_ranks == 1:
@@ -703,6 +703,7 @@ class BasePrefixCache(ABC, PrefixCacheTrait):
supports_prefix_sharing=self.supports_prefix_sharing(),
retain_floor=self.swa_retain_floor(req),
eviction_interval=eviction_interval,
reprefill_tail_tokens=self.swa_reprefill_tail_tokens(req),
)
def swa_retain_floor(self, req) -> int | None:
+9
View File
@@ -84,6 +84,7 @@ def free_swa_out_of_window_slots(
component_type: ComponentType = ComponentType.SWA,
free_segment: Callable[..., None] | None = None,
eviction_interval: int = 1,
reprefill_tail_tokens: int = 0,
) -> None:
if not req.kv.holds_kv:
return
@@ -115,6 +116,14 @@ def free_swa_out_of_window_slots(
evict_threshold = pre_len - ceil_align(
max(sliding_window_size, page_size), page_size
)
if reprefill_tail_tokens:
# Matching excludes the replay tail before page alignment. Keep the
# SWA window behind that earlier boundary too, or a first replay
# falls back to an older checkpoint despite a Full-KV cache hit.
replay_boundary = (pre_len - reprefill_tail_tokens) // page_size * page_size
evict_threshold = min(
evict_threshold, replay_boundary - sliding_window_size
)
if retain_floor is not None and supports_prefix_sharing:
# The caller owns where the floor is (see BasePrefixCache.swa_retain_floor);
# this only promises not to free past it. Without prefix sharing a retained
@@ -1046,6 +1046,7 @@ class SWAComponent(TreeComponent):
req_to_token_pool=self.cache.req_to_token_pool,
token_to_kv_pool_allocator=self.cache.token_to_kv_pool_allocator,
retain_floor=self.cache.swa_retain_floor(req),
reprefill_tail_tokens=self.cache.swa_reprefill_tail_tokens(req),
)
def free_out_of_window_slots(
@@ -52,6 +52,23 @@ def folded_replay_rows(batch) -> int:
)
def decoder_swa_trim_rows(batch, *, folds_encoder: bool) -> int:
"""Rows removed by a plain extend's decoder tail; -1 requests the backend
exchange when prompt outputs require whole-extend rows."""
if not batch.forward_mode.is_extend_without_speculative():
return 0
if batch.return_logprob or batch.return_hidden_states:
return -1
lens = list(batch.extend_lens)
if folds_encoder and batch.encoder_swa_reset is not None:
for i, (reset, prefix) in enumerate(
zip(batch.encoder_swa_reset, batch.prefix_lens)
):
if reset:
lens[i] += replay_len(prefix)
return sum(max(0, n - 128) for n in lens)
def apply_folded_extend(folded: FoldedExtend, forward_batch) -> None:
forward_batch.encoder_swa_row_floor = folded.row_floor
forward_batch.encoder_swa_compress_rows = folded.keep_rows
@@ -51,6 +51,7 @@ from sglang.srt.layers.dp_attention import (
set_is_extend_in_batch,
world_dp_gather_enabled,
)
from sglang.srt.model_executor.cuda_graph_config import Backend
from sglang.srt.model_executor.forward_batch_deepseek_mha_mixin import (
ForwardBatchDeepSeekMHAMixin,
)
@@ -59,6 +60,7 @@ from sglang.srt.runtime_context import (
get_flags,
get_lora,
get_parallel,
get_platform,
mamba_cache_chunk_size,
)
from sglang.srt.speculative.spec_info import SpecInputType
@@ -78,7 +80,7 @@ if TYPE_CHECKING:
from sglang.srt.layers.engram_staging import StagedRows
from sglang.srt.layers.layer_boundary.residual.stream import ResidualStream
from sglang.srt.layers.logits_processor import LogitsProcessorOutput
from sglang.srt.managers.schedule_batch import MultimodalInputs, ScheduleBatch
from sglang.srt.managers.schedule_batch import MultimodalInputs, Req, ScheduleBatch
from sglang.srt.mem_cache.kv_loc_plan import Cols, KVLocPlan
from sglang.srt.model_executor.model_runner import ModelRunner
from sglang.srt.sampling.sampling_batch_info import SamplingBatchInfo
@@ -489,6 +491,17 @@ class NgramEmbeddingInfo:
)
def prompt_rest_lens_of(
reqs: List[Req], seq_lens: List[int], decoding_req_ids: Set[int]
) -> List[int]:
"""Prompt tokens each request prefills after this extend, from the batch's own
ends (an encoder replay batch ends at its prefix); a decode row has none."""
return [
0 if id(req) in decoding_req_ids else len(req.full_untruncated_fill_ids) - end
for req, end in zip(reqs, seq_lens, strict=True)
]
@dataclass
class ForwardBatch(ForwardBatchDeepSeekMHAMixin):
"""Store all inputs of a forward pass."""
@@ -669,6 +682,8 @@ class ForwardBatch(ForwardBatchDeepSeekMHAMixin):
extend_prefix_lens_cpu: Optional[List[int]] = None
extend_seq_lens_cpu: Optional[List[int]] = None
extend_logprob_start_lens_cpu: Optional[List[int]] = None
# Prompt tokens each request prefills after this extend (decoder SWA tail rows).
extend_prompt_rest_lens_cpu: Optional[List[int]] = None
extend_input_logprob_token_ids_gpu: Optional[torch.Tensor] = None
# For DP attention (MLP sync sizes)
@@ -681,6 +696,9 @@ class ForwardBatch(ForwardBatchDeepSeekMHAMixin):
# Has to be None when cuda graph is captured.
global_num_tokens_for_logprob_cpu: Optional[List[int]] = None
global_num_tokens_for_logprob_gpu: Optional[torch.Tensor] = None
# Per DP rank, the rows the decoder SWA tail drops before the late layers;
# None when no rank trims (see DeepseekV4Model's late-layer DP switch).
global_decoder_trim_rows_cpu: Optional[List[int]] = None
# Real (non-padding) token count, held at two scopes whose meaning never
# changes once set:
@@ -1015,6 +1033,7 @@ class ForwardBatch(ForwardBatchDeepSeekMHAMixin):
batch.seq_lens_sum = int(seq_lens_cpu.sum())
contains_last_prefill_chunk = False
prompt_rest_lens = None
if batch.forward_mode.is_extend_without_speculative():
decoding_req_ids = {id(req) for req in (batch.decoding_reqs or ())}
contains_last_prefill_chunk = any(
@@ -1022,6 +1041,14 @@ class ForwardBatch(ForwardBatchDeepSeekMHAMixin):
for req in batch.reqs
if id(req) not in decoding_req_ids
)
if (
envs.SGLANG_OPT_DSV41_PROMPT_WINDOW_TAIL.get()
and get_parallel().attn_cp_size == 1
and seq_lens_cpu is not None
):
prompt_rest_lens = prompt_rest_lens_of(
batch.reqs, seq_lens_cpu.tolist(), decoding_req_ids
)
from sglang.srt.model_executor.model_runner import ModelRunner
@@ -1082,6 +1109,7 @@ class ForwardBatch(ForwardBatchDeepSeekMHAMixin):
global_forward_mode=batch.global_forward_mode,
is_prefill_only=batch.is_prefill_only,
contains_last_prefill_chunk=contains_last_prefill_chunk,
extend_prompt_rest_lens_cpu=prompt_rest_lens,
spec_algorithm=batch.spec_algorithm,
capture_hidden_mode=capture_hidden_mode,
return_hidden_states_before_norm=return_hidden_states_before_norm,
@@ -1166,6 +1194,8 @@ class ForwardBatch(ForwardBatchDeepSeekMHAMixin):
ret.init_mlp_sync_metadata(
batch, device, is_draft_worker=model_runner.is_draft_worker
)
if not model_runner.is_draft_worker:
ret.global_decoder_trim_rows_cpu = batch.global_decoder_trim_rows
if ret.forward_mode.is_idle():
ret.positions = torch.empty((0,), dtype=torch.int64, device=device)
@@ -1945,13 +1975,29 @@ class ForwardBatch(ForwardBatchDeepSeekMHAMixin):
):
local_late_rows = model_runner.attn_backend.late_layer_num_tokens(self)
if local_late_rows is not None:
late_rows = torch.empty(sync_group_size, dtype=torch.int64)
all_gather_single(
late_rows,
torch.tensor([local_late_rows], dtype=torch.int64),
group=get_parallel().tp_group.cpu_group,
)
self.global_late_layer_num_tokens_cpu = late_rows.tolist()
trims = self.global_decoder_trim_rows_cpu
if (
trims is not None
and get_platform().is_cuda
and get_parallel().attn_tp_size == 1
and get_exec().graph.cuda_graph_config.prefill.backend
== Backend.DISABLED
and not self.dp_padding_mode.is_max_len()
):
counts = [
n - t for n, t in zip(global_num_tokens, trims, strict=True)
]
assert min(counts) >= 0, (global_num_tokens, trims)
assert counts[dp_slot_in(counts)] == local_late_rows
self.global_late_layer_num_tokens_cpu = counts
else:
late_rows = torch.empty(sync_group_size, dtype=torch.int64)
all_gather_single(
late_rows,
torch.tensor([local_late_rows], dtype=torch.int64),
group=get_parallel().tp_group.cpu_group,
)
self.global_late_layer_num_tokens_cpu = late_rows.tolist()
TboForwardBatchPreparer.prepare(
batch=self, is_draft_worker=model_runner.is_draft_worker
@@ -2074,6 +2120,10 @@ class ForwardBatch(ForwardBatchDeepSeekMHAMixin):
self.extend_logprob_start_lens_cpu.extend(
[0] * (bs - len(self.extend_logprob_start_lens_cpu))
)
if self.extend_prompt_rest_lens_cpu is not None:
self.extend_prompt_rest_lens_cpu.extend(
[0] * (bs - len(self.extend_prompt_rest_lens_cpu))
)
if self.rids_int is not None:
self.rids_int = self._pad_tensor_to_size(self.rids_int, bs)
@@ -0,0 +1,181 @@
"""Decoder SWA bounded replay with SGLANG_OPT_DSV41_PROMPT_WINDOW_TAIL: a prefill
chunk runs the late layers over its rows in the prompt's last window only, so any
chunking of a prompt computes exactly the rows and floors of the unchunked prompt
there, plus one placeholder row per chunk outside it. CPU only."""
import unittest
from types import SimpleNamespace
import torch
from sglang.kernels.ops.attention.dsv4.attn_metadata_kernels import (
late_layer_tail_layout,
late_layer_tail_lens,
)
from sglang.srt.layers.attention.deepseek_v4_backend import (
SWA_WINDOW,
late_layer_num_tokens,
)
from sglang.srt.model_executor.forward_batch_info import (
CaptureHiddenMode,
ForwardMode,
prompt_rest_lens_of,
)
from sglang.test.ci.ci_register import register_cpu_ci
from sglang.test.test_utils import CustomTestCase
register_cpu_ci(est_time=5, suite="base-a-test-cpu")
CPU = torch.device("cpu")
def _layout(extend_lens, seq_lens, rest=None, whole_extend=None):
return late_layer_tail_layout(
extend_lens_cpu=extend_lens,
seq_lens_cpu=seq_lens,
tail_len=SWA_WINDOW,
device=CPU,
whole_extend=whole_extend,
prompt_rest_lens=rest,
)
def _chunked_rows(prompt_len, chunks):
"""(absolute position, floor) of every tail row over a prompt's chunks, each
chunk one single-request step."""
rows, start = [], 0
for n in chunks:
end = start + n
token_indices, _, floor = _layout([n], [end], rest=[prompt_len - end])
rows += list(zip((start + token_indices).tolist(), floor.tolist()))
start = end
assert start == prompt_len
return rows
class TestTailLens(CustomTestCase):
def test_final_chunks_keep_the_window_tail(self):
extend_lens = [300, 50, 128]
self.assertEqual(
late_layer_tail_lens(extend_lens, SWA_WINDOW, prompt_rest_lens=[0, 0, 0]),
late_layer_tail_lens(extend_lens, SWA_WINDOW),
)
def test_earlier_chunks_keep_their_rows_in_the_prompt_window(self):
# Far from the end: one row. Straddling the window: the rows inside it.
# Inside the window: the whole chunk.
self.assertEqual(
late_layer_tail_lens(
[32768, 300, 40, 300],
SWA_WINDOW,
prompt_rest_lens=[5000, 50, 20, SWA_WINDOW],
),
[1, SWA_WINDOW - 50, 40, 1],
)
def test_whole_extend_wins(self):
self.assertEqual(
late_layer_tail_lens(
[300, 300], SWA_WINDOW, [True, False], prompt_rest_lens=[5000, 5000]
),
[300, SWA_WINDOW],
)
# No whole request: the prompt window applies, as in the layout.
self.assertEqual(
late_layer_tail_lens([300], SWA_WINDOW, [False], prompt_rest_lens=[5000]),
[1],
)
class TestTailLayout(CustomTestCase):
def test_final_chunks_match_the_window_layout(self):
# Negative control: final chunks (nothing left) change no row or floor.
for extend_lens, seq_lens in (([300], [900]), ([300, 50, 40], [900, 50, 400])):
with self.subTest(extend_lens=extend_lens):
got = _layout(extend_lens, seq_lens, rest=[0] * len(extend_lens))
want = _layout(extend_lens, seq_lens)
torch.testing.assert_close(got[0], want[0])
self.assertEqual(got[1], want[1])
torch.testing.assert_close(got[2], want[2])
def test_placeholder_row_reads_only_itself(self):
token_indices, tail_lens, floor = _layout([32768], [65536], rest=[70000])
self.assertEqual(tail_lens, [1])
self.assertEqual(token_indices.tolist(), [32767])
self.assertEqual(floor.tolist(), [65535])
def test_batched_layout_matches_single_requests(self):
extend_lens = [32768, 300, 40, 200]
seq_lens = [40000, 900, 60, 200]
rest = [9000, 50, 20, 0]
token_indices, tail_lens, floor = _layout(extend_lens, seq_lens, rest)
want_indices, want_floor, start = [], [], 0
for n, s, r in zip(extend_lens, seq_lens, rest):
idx, _, fl = _layout([n], [s], rest=[r])
want_indices.append(start + idx)
want_floor.append(fl)
start += n
self.assertEqual(tail_lens, [1, SWA_WINDOW - 50, 40, SWA_WINDOW])
torch.testing.assert_close(token_indices, torch.cat(want_indices))
torch.testing.assert_close(floor, torch.cat(want_floor))
class TestChunkingInvariance(CustomTestCase):
"""The rows in the prompt's last window, and the floor each reads from, do
not depend on how the prompt was chunked."""
def test_any_chunking_computes_the_unchunked_window(self):
prompt_len = 1000
whole = _chunked_rows(prompt_len, [prompt_len])
floor = prompt_len - SWA_WINDOW
self.assertEqual(whole, [(p, floor) for p in range(floor, prompt_len)])
for chunks in (
[400, 400, 200],
[900, 60, 40],
[800, 100, 60, 40],
[872, 1, 127],
[700, 200, 100],
):
with self.subTest(chunks=chunks):
rows = _chunked_rows(prompt_len, chunks)
window = [(p, f) for p, f in rows if p >= floor]
self.assertEqual(window, whole)
# Placeholders sit below the window and read only their own row.
for p, f in rows:
if p < floor:
self.assertEqual(f, p)
def test_window_rows_never_read_below_the_window(self):
rows = _chunked_rows(1000, [900, 60, 40])
self.assertTrue(all(f == 1000 - SWA_WINDOW for p, f in rows if p >= 872))
class TestPromptRestLens(CustomTestCase):
def test_rest_from_the_batch_ends(self):
prompt = SimpleNamespace(full_untruncated_fill_ids=list(range(1000)))
decode = SimpleNamespace(full_untruncated_fill_ids=list(range(70)))
# A chunk ending at 400, a replay batch ending at its prefix 128, and a
# decode row carried in the extend.
self.assertEqual(
prompt_rest_lens_of([prompt, prompt, decode], [400, 128, 70], {id(decode)}),
[600, 872, 0],
)
def test_rows_counted_for_dp_match_the_layout(self):
extend_lens, rest = [32768, 300, 40], [9000, 50, 20]
fb = SimpleNamespace(
forward_mode=ForwardMode.EXTEND,
extend_seq_lens_cpu=extend_lens,
extend_logprob_start_lens_cpu=None,
return_logprob=False,
capture_hidden_mode=CaptureHiddenMode.FULL,
extend_prompt_rest_lens_cpu=rest,
)
_, tail_lens, _ = _layout(extend_lens, [40000, 900, 60], rest)
self.assertEqual(late_layer_num_tokens(fb, dspark=True), sum(tail_lens))
fb.extend_prompt_rest_lens_cpu = None
self.assertEqual(late_layer_num_tokens(fb, dspark=True), 2 * SWA_WINDOW + 40)
if __name__ == "__main__":
unittest.main()
@@ -0,0 +1,128 @@
"""Exercise SWA eviction frontiers without accelerator execution or model weights."""
import ast
import unittest
from pathlib import Path
from types import SimpleNamespace
from unittest.mock import Mock
from sglang.srt.mem_cache import common as cache_common
from sglang.test.ci.ci_register import register_cpu_ci
from sglang.test.test_utils import CustomTestCase
register_cpu_ci(est_time=5, suite="base-a-test-cpu")
class ReplayRetentionTest(CustomTestCase):
@classmethod
def setUpClass(cls):
cls.source = Path(cache_common.__file__).parent
cls.free_window = staticmethod(cache_common.free_swa_out_of_window_slots)
def frontier(
self,
length,
window=128,
page=256,
replay=128,
sharing=True,
retain_floor=None,
previous=0,
):
kv = SimpleNamespace(
holds_kv=True,
cache_protected_len=0,
req_pool_idx=0,
swa_dead_lo=lambda _: 0,
get_evicted_seqlen=lambda _: previous,
)
kv.set_evicted_seqlen = lambda _, value: setattr(kv, "evicted", value)
table = Mock()
table.__getitem__ = Mock(return_value=object())
allocator = SimpleNamespace(free_swa_segment=Mock())
self.free_window(
SimpleNamespace(kv=kv),
length - 1,
sliding_window_size=window,
page_size=page,
req_to_token_pool=SimpleNamespace(req_to_token=table),
token_to_kv_pool_allocator=allocator,
supports_prefix_sharing=sharing,
retain_floor=retain_floor,
reprefill_tail_tokens=replay,
)
return kv.evicted
def test_first_replay_keeps_window_before_trimmed_match(self):
for length in (32576, 32448, 269913, 1029986):
with self.subTest(length=length):
boundary = (length - 128) // 256 * 256
self.assertLessEqual(self.frontier(length), boundary - 128)
def test_all_page_alignments_preserve_replay_history(self):
for page, window, replay in (
(256, 128, 128),
(64, 128, 128),
(16, 8, 8),
(1, 4, 4),
):
for remainder in range(page):
length = 64 * page + 2 * window + remainder
with self.subTest(page=page, window=window, remainder=remainder):
boundary = (length - replay) // page * page
self.assertLessEqual(
self.frontier(length, window, page, replay), boundary - window
)
def test_no_replay_keeps_existing_eviction_policy(self):
for length in range(32000, 32512):
expected = (length - 1 - 256) // 256 * 256
self.assertEqual(self.frontier(length, replay=0), expected)
def test_256_token_pages_retain_at_most_one_extra_page(self):
for length in range(32000, 32512):
before = self.frontier(length, replay=0)
after = self.frontier(length)
self.assertGreaterEqual(before - after, 0)
self.assertLessEqual(before - after, 256)
def test_bad_alignment_exposes_original_frontier_mismatch(self):
self.assertEqual(self.frontier(32576, replay=0), 32256)
self.assertEqual(self.frontier(32576), 32000)
self.assertEqual(self.frontier(32448), 32000)
def test_disabled_prefix_sharing_does_not_retain_replay_history(self):
for replay in (0, 128):
self.assertEqual(self.frontier(32576, replay=replay, sharing=False), 32256)
def test_checkpoint_floor_and_eviction_cursor_still_apply(self):
self.assertEqual(self.frontier(32576, retain_floor=30000), 29952)
# Previously freed slots cannot be recovered by changing a frontier.
self.assertEqual(self.frontier(32576, previous=32256), 32256)
def test_scheduler_and_checkpoint_paths_pass_the_replay_guard(self):
for relative, expected in (
("base_prefix_cache.py", "self.swa_reprefill_tail_tokens(req)"),
(
"unified_cache/components/swa.py",
"self.cache.swa_reprefill_tail_tokens(req)",
),
):
tree = ast.parse((self.source / relative).read_text())
calls = [
node
for node in ast.walk(tree)
if isinstance(node, ast.Call)
and ast.unparse(node.func) == "free_swa_out_of_window_slots"
]
self.assertEqual(len(calls), 1)
keyword = next(
(kw for kw in calls[0].keywords if kw.arg == "reprefill_tail_tokens"),
None,
)
self.assertIsNotNone(keyword, relative)
self.assertEqual(ast.unparse(keyword.value), expected)
if __name__ == "__main__":
unittest.main()
@@ -0,0 +1,181 @@
"""Decoder tail sizes reuse the scheduler exchange on eager CUDA DP steps."""
from types import SimpleNamespace
from unittest.mock import MagicMock, patch
import pytest
import torch
from sglang.srt.managers.scheduler_components import dp_attn
from sglang.srt.managers.scheduler_components.dp_attn import _update_gather_batch
from sglang.srt.model_executor import forward_batch_info as fbi
from sglang.srt.model_executor.cuda_graph_config import Backend
from sglang.srt.model_executor.encoder_swa_replay import decoder_swa_trim_rows
from sglang.srt.model_executor.forward_batch_info import ForwardBatch, ForwardMode
from sglang.test.ci.ci_register import register_cpu_ci
register_cpu_ci(est_time=5, suite="base-a-test-cpu")
@pytest.mark.parametrize("folds,expected", [(False, 3968), (True, 3998)])
def test_trim_includes_folded_encoder_rows(folds, expected):
batch = SimpleNamespace(
forward_mode=ForwardMode.EXTEND,
extend_lens=[4096, 30, 40],
prefix_lens=[0, 512, 64],
encoder_swa_reset=[True, True, True],
return_logprob=False,
return_hidden_states=False,
)
assert decoder_swa_trim_rows(batch, folds_encoder=folds) == expected
batch.return_logprob = True
assert decoder_swa_trim_rows(batch, folds_encoder=folds) == -1
batch.return_logprob = False
batch.return_hidden_states = True
assert decoder_swa_trim_rows(batch, folds_encoder=folds) == -1
@pytest.mark.parametrize(
"mode", [ForwardMode.DECODE, ForwardMode.IDLE, ForwardMode.TARGET_VERIFY]
)
def test_non_extend_peers_do_not_trim(mode):
assert (
decoder_swa_trim_rows(SimpleNamespace(forward_mode=mode), folds_encoder=True)
== 0
)
@pytest.mark.parametrize(
"trims,expected",
[
([3968, 30, 0, 0], [3968, 30, 0, 0]),
([0, 0, 0, 0], None),
([3968, -1, 0, 0], None),
],
)
def test_whole_output_peer_falls_back_on_every_rank(trims, expected):
info = MagicMock()
info.global_decoder_trim_rows = trims
batch = SimpleNamespace()
_update_gather_batch(batch, info, True, skip_global_metadata=True)
assert batch.global_decoder_trim_rows == expected
@pytest.mark.parametrize(
"rank,mode",
[
(0, ForwardMode.EXTEND),
(1, ForwardMode.EXTEND),
(2, ForwardMode.IDLE),
(3, ForwardMode.DECODE),
],
)
@pytest.mark.parametrize("fallback", [None, "graph", "hip", "whole_outputs"])
def test_preparation_uses_scheduler_counts_or_preserves_backend_exchange(
rank, mode, fallback
):
rows, trims, late = [4096, 282, 0, 130], [3968, 30, 0, 0], [128, 252, 0, 130]
n = rows[rank]
batch = ForwardBatch(
forward_mode=mode,
batch_size=1 if n else 0,
input_ids=torch.zeros(n, dtype=torch.long),
req_pool_indices=torch.zeros(1 if n else 0, dtype=torch.long),
seq_lens=torch.tensor([n] if n else []),
out_cache_loc=torch.arange(n),
seq_lens_sum=n,
positions=torch.arange(n),
is_extend_in_batch=True,
global_num_tokens_cpu=rows,
global_num_tokens_for_logprob_cpu=rows,
global_num_tokens_gpu=torch.zeros(4, dtype=torch.int64),
global_decoder_trim_rows_cpu=None if fallback == "whole_outputs" else trims,
)
runner = MagicMock(is_draft_worker=False)
runner.attn_tp_sequence_sharded.return_value = False
runner.attn_backend.late_layer_num_tokens.return_value = late[rank]
cfg = SimpleNamespace(
graph=SimpleNamespace(
cuda_graph_config=SimpleNamespace(
prefill=SimpleNamespace(
bs=[],
backend=Backend.BREAKABLE
if fallback == "graph"
else Backend.DISABLED,
)
)
)
)
def exchange(output, local, **kwargs):
assert local.item() == late[rank]
output.copy_(torch.tensor(late))
with (
patch.multiple(
fbi,
get_parallel=lambda: SimpleNamespace(
attn_tp_size=1, tp_group=SimpleNamespace(cpu_group=None)
),
get_exec=lambda: cfg,
get_platform=lambda: SimpleNamespace(is_cuda=fallback != "hip"),
_elastic_should_preserve_local_token_counts=lambda **kwargs: False,
_mega_moe_materializes_idle_rank=lambda batch: False,
dp_slot_in=lambda counts: rank,
set_dp_buffer_len_from_batch=lambda *args: None,
set_is_extend_in_batch=lambda *args: None,
_is_cpu=True,
),
patch.object(fbi, "mambaish_config", return_value=None, create=True),
patch.object(
fbi.DpPaddingMode,
"get_dp_padding_mode",
return_value=fbi.DpPaddingMode.SUM_LEN,
),
patch.object(ForwardBatch, "_pad_inputs_to_size"),
patch(
"sglang.srt.batch_overlap.two_batch_overlap.TboForwardBatchPreparer.prepare"
),
patch.object(fbi, "all_gather_single", side_effect=exchange) as gather,
):
batch.prepare_mlp_sync_batch(runner)
assert batch.global_late_layer_num_tokens_cpu == late
assert gather.call_count == (0 if fallback is None else 1)
def test_scheduler_exchange_carries_peer_trims_and_inactive_slots():
infos = [
dp_attn.MLPSyncBatchInfo(
num_dp_ranks=4,
tp_size=1,
cp_size=1,
num_tokens=n,
num_tokens_for_logprob=n,
can_run_decode_cuda_graph=False,
can_run_draft_cuda_graph=False,
can_run_prefill_cuda_graph=False,
is_extend_in_batch=True,
local_can_run_tbo=False,
local_forward_mode=ForwardMode.EXTEND.value,
decoder_trim_rows=t,
)
for n, t in zip([4096, 282, 0, 130], [3968, 30, 0, 0])
]
gathered = torch.cat([info._get_local_tensor("cpu") for info in infos])
with (
patch.object(
dp_attn,
"exchange_sync_rows",
side_effect=lambda out, *a, **kw: out.copy_(gathered),
),
patch.object(
dp_attn,
"get_parallel",
return_value=SimpleNamespace(
tp_group=SimpleNamespace(active_ranks_cpu=torch.tensor([1, 1, 0, 1]))
),
),
):
for info in infos:
info.all_gather(device="cpu", group=None)
assert info.global_decoder_trim_rows == [3968, 30, 0, 0]
assert info.global_num_tokens == [4096, 282, 0, 130]
@@ -292,6 +292,7 @@ class TestFoldWithDecoderSwaTail(CustomTestCase):
positions=fx.folded_positions(),
capture_hidden_mode=CaptureHiddenMode.NULL,
return_logprob=False,
extend_prompt_rest_lens_cpu=None,
)
with (
patch.object(DSV4AttnMetadata, "init_compression_metadata"),
@@ -133,6 +133,7 @@ class TestDPSpecPrefillCoordinationPlan(CustomTestCase):
num_tokens_for_logprob=32,
global_num_tokens=[32, 7],
global_num_tokens_for_logprob=[32, 7],
global_decoder_trim_rows=[0, 0],
tp0_info_cpu=torch.tensor([[32, 32, 1, 0], [7, 7, 1, 0]]),
is_extend_in_batch=False,
tbo_split_seq_index=None,