perf: dsv41 decoder tail runs only the prompt's last window rows of each prefill chunk (opt-in)

(cherry picked from commit 1b778e6df01a138a35403627e209cf33967246e6)
This commit is contained in:
alphabetc1
2026-11-04 09:44:11 +08:00
committed by minke.yu
parent ee9170e7bb
commit 0b45c07637
8 changed files with 260 additions and 16 deletions
@@ -443,15 +443,23 @@ def late_layer_tail_lens(
extend_lens_cpu: list[int],
tail_len: int,
whole_extend: Optional[list[bool]] = None,
prompt_rest_lens: Optional[list[int]] = None,
) -> list[int]:
"""Each request's late-layer tail: its last min(tail_len, extend_len) tokens,
or its whole extend where whole_extend is set."""
if whole_extend is None:
return [min(tail_len, n) for n in extend_lens_cpu]
return [
n if whole else min(tail_len, n)
for n, whole in zip(extend_lens_cpu, whole_extend)
]
or its whole extend where whole_extend is set. With prompt_rest_lens (prompt
tokens left after each extend), only its rows in the prompt's last tail_len
tokens, and at least its last row."""
if whole_extend is not None and any(whole_extend):
return [
n if whole else min(tail_len, n)
for n, whole in zip(extend_lens_cpu, whole_extend)
]
if prompt_rest_lens is not None:
return [
min(n, max(1, tail_len - rest))
for n, rest in zip(extend_lens_cpu, prompt_rest_lens, strict=True)
]
return [min(tail_len, n) for n in extend_lens_cpu]
def late_layer_tail_layout(
@@ -461,13 +469,15 @@ def late_layer_tail_layout(
tail_len: int,
device: torch.device,
whole_extend: Optional[list[bool]] = None,
prompt_rest_lens: Optional[list[int]] = None,
) -> tuple[torch.Tensor, list[int], torch.Tensor]:
"""Tail rows of each prefill extend: its last min(tail_len, extend_len) tokens,
or its whole extend where whole_extend is set.
Returns (token indices into the extend, per-request tail lengths, per-row
absolute window floor). The floor is seq_len - tail_len, so a tail shorter
than tail_len reads rows the request wrote before this extend. A whole
extend keeps every row's plain window."""
extend keeps every row's plain window. With prompt_rest_lens, tails and
floors follow the prompt's end instead (late_layer_tail_lens, _tail_floors)."""
if whole_extend is not None and any(whole_extend):
return _whole_extend_tail_layout(
extend_lens_cpu=extend_lens_cpu,
@@ -476,16 +486,19 @@ def late_layer_tail_layout(
device=device,
whole_extend=whole_extend,
)
tail_lens_cpu = late_layer_tail_lens(extend_lens_cpu, tail_len)
tail_lens_cpu = late_layer_tail_lens(
extend_lens_cpu, tail_len, prompt_rest_lens=prompt_rest_lens
)
floors_cpu = _tail_floors(seq_lens_cpu, tail_len, prompt_rest_lens)
if len(extend_lens_cpu) == 1:
n, t, s = extend_lens_cpu[0], tail_lens_cpu[0], seq_lens_cpu[0]
floor = torch.full((t,), max(0, s - tail_len), dtype=torch.int32, device=device)
n, t = extend_lens_cpu[0], tail_lens_cpu[0]
floor = torch.full((t,), floors_cpu[0], dtype=torch.int32, device=device)
return torch.arange(n - t, n, device=device), tail_lens_cpu, floor
# One H2D copy for the three length vectors; launch count does not grow with bs.
lens = async_h2d(
[extend_lens_cpu, tail_lens_cpu, seq_lens_cpu], dtype=torch.int64, device=device
[extend_lens_cpu, tail_lens_cpu, floors_cpu], dtype=torch.int64, device=device
)
extend_lens, tail_lens, seq_lens = lens[0], lens[1], lens[2]
extend_lens, tail_lens, floors = lens[0], lens[1], lens[2]
total = sum(tail_lens_cpu)
req = torch.repeat_interleave(
torch.arange(len(tail_lens_cpu), device=device), tail_lens, output_size=total
@@ -495,10 +508,23 @@ def late_layer_tail_layout(
- (torch.cumsum(tail_lens, 0) - tail_lens)[req]
)
token_indices = (torch.cumsum(extend_lens, 0) - tail_lens)[req] + offs
floor = (seq_lens - tail_len).clamp_min(0)[req].to(torch.int32)
floor = floors[req].to(torch.int32)
return token_indices, tail_lens_cpu, floor
def _tail_floors(
seq_lens_cpu: list[int], tail_len: int, prompt_rest_lens: Optional[list[int]]
) -> list[int]:
"""Each request's window floor: tail_len below its extend's end, or below its
prompt's end, clamped to its last row, whose own KV a lone tail row reads."""
if prompt_rest_lens is None:
return [max(0, s - tail_len) for s in seq_lens_cpu]
return [
min(max(0, s + rest - tail_len), max(0, s - 1))
for s, rest in zip(seq_lens_cpu, prompt_rest_lens, strict=True)
]
def _whole_extend_tail_layout(
*,
extend_lens_cpu: list[int],
@@ -728,6 +728,7 @@ class TboForwardBatchPreparer:
"extend_prefix_lens_cpu",
"extend_seq_lens_cpu",
"extend_logprob_start_lens_cpu",
"extend_prompt_rest_lens_cpu",
"lora_ids",
"rids",
]:
+3
View File
@@ -1719,6 +1719,9 @@ class Envs:
# Prefill-graph shapes of at least this many tokens run the tail from the tail
# graphs in a break (measured crossover 1536-2048 tokens on GB300 TP4).
SGLANG_DSV4_DECODER_TAIL_GRAPH_MIN_PREFILL_TOKENS = EnvInt(2048)
# Decoder SWA bounded replay: a prefill chunk runs the tail layers only over its
# rows in the prompt's last window, else over its last row.
SGLANG_OPT_DSV41_PROMPT_WINDOW_TAIL = EnvBool(False)
# Paged KV layout of the DeepSeek-V4 family pools: "v4" (584 B/token, every
# GPU), "v41" (the SM100 FlashMLA V4.1 formats: 528 B fp8 SWA cache, fp8 or
# fp4 compressed caches), "v41_new" (the same bytes interleaved per token, for
@@ -1024,7 +1024,10 @@ def late_layer_num_tokens(forward_batch: ForwardBatch, *, dspark: bool) -> int:
whole_extend = late_layer_whole_extend(forward_batch, dspark=dspark)
return sum(
late_layer_tail_lens(
forward_batch.extend_seq_lens_cpu, SWA_WINDOW, whole_extend
forward_batch.extend_seq_lens_cpu,
SWA_WINDOW,
whole_extend,
prompt_rest_lens=forward_batch.extend_prompt_rest_lens_cpu,
)
)
@@ -1834,6 +1837,7 @@ class DeepseekV4AttnBackend(
whole_extend=late_layer_whole_extend(
forward_batch, dspark=self.model_runner.spec_algorithm.is_dspark()
),
prompt_rest_lens=forward_batch.extend_prompt_rest_lens_cpu,
)
if self.encoder_row_floor is not None:
# Folded hits: a tail row never reads below its request's replay start.
@@ -2252,6 +2252,7 @@ class DeepseekV4HipRadixBackend(
tail_len=SWA_WINDOW,
device=device,
whole_extend=late_layer_whole_extend(forward_batch, dspark=self.is_dspark),
prompt_rest_lens=forward_batch.extend_prompt_rest_lens_cpu,
)
contiguous_start = (
extend_lens_cpu[0] - tail_lens_cpu[0] if len(extend_lens_cpu) == 1 else None
@@ -78,7 +78,7 @@ if TYPE_CHECKING:
from sglang.srt.layers.engram_staging import StagedRows
from sglang.srt.layers.layer_boundary.residual.stream import ResidualStream
from sglang.srt.layers.logits_processor import LogitsProcessorOutput
from sglang.srt.managers.schedule_batch import MultimodalInputs, ScheduleBatch
from sglang.srt.managers.schedule_batch import MultimodalInputs, Req, ScheduleBatch
from sglang.srt.mem_cache.kv_loc_plan import Cols, KVLocPlan
from sglang.srt.model_executor.model_runner import ModelRunner
from sglang.srt.sampling.sampling_batch_info import SamplingBatchInfo
@@ -489,6 +489,17 @@ class NgramEmbeddingInfo:
)
def prompt_rest_lens_of(
reqs: List[Req], seq_lens: List[int], decoding_req_ids: Set[int]
) -> List[int]:
"""Prompt tokens each request prefills after this extend, from the batch's own
ends (an encoder replay batch ends at its prefix); a decode row has none."""
return [
0 if id(req) in decoding_req_ids else len(req.full_untruncated_fill_ids) - end
for req, end in zip(reqs, seq_lens, strict=True)
]
@dataclass
class ForwardBatch(ForwardBatchDeepSeekMHAMixin):
"""Store all inputs of a forward pass."""
@@ -669,6 +680,8 @@ class ForwardBatch(ForwardBatchDeepSeekMHAMixin):
extend_prefix_lens_cpu: Optional[List[int]] = None
extend_seq_lens_cpu: Optional[List[int]] = None
extend_logprob_start_lens_cpu: Optional[List[int]] = None
# Prompt tokens each request prefills after this extend (decoder SWA tail rows).
extend_prompt_rest_lens_cpu: Optional[List[int]] = None
extend_input_logprob_token_ids_gpu: Optional[torch.Tensor] = None
# For DP attention (MLP sync sizes)
@@ -1015,6 +1028,7 @@ class ForwardBatch(ForwardBatchDeepSeekMHAMixin):
batch.seq_lens_sum = int(seq_lens_cpu.sum())
contains_last_prefill_chunk = False
prompt_rest_lens = None
if batch.forward_mode.is_extend_without_speculative():
decoding_req_ids = {id(req) for req in (batch.decoding_reqs or ())}
contains_last_prefill_chunk = any(
@@ -1022,6 +1036,14 @@ class ForwardBatch(ForwardBatchDeepSeekMHAMixin):
for req in batch.reqs
if id(req) not in decoding_req_ids
)
if (
envs.SGLANG_OPT_DSV41_PROMPT_WINDOW_TAIL.get()
and get_parallel().attn_cp_size == 1
and seq_lens_cpu is not None
):
prompt_rest_lens = prompt_rest_lens_of(
batch.reqs, seq_lens_cpu.tolist(), decoding_req_ids
)
from sglang.srt.model_executor.model_runner import ModelRunner
@@ -1082,6 +1104,7 @@ class ForwardBatch(ForwardBatchDeepSeekMHAMixin):
global_forward_mode=batch.global_forward_mode,
is_prefill_only=batch.is_prefill_only,
contains_last_prefill_chunk=contains_last_prefill_chunk,
extend_prompt_rest_lens_cpu=prompt_rest_lens,
spec_algorithm=batch.spec_algorithm,
capture_hidden_mode=capture_hidden_mode,
return_hidden_states_before_norm=return_hidden_states_before_norm,
@@ -2074,6 +2097,10 @@ class ForwardBatch(ForwardBatchDeepSeekMHAMixin):
self.extend_logprob_start_lens_cpu.extend(
[0] * (bs - len(self.extend_logprob_start_lens_cpu))
)
if self.extend_prompt_rest_lens_cpu is not None:
self.extend_prompt_rest_lens_cpu.extend(
[0] * (bs - len(self.extend_prompt_rest_lens_cpu))
)
if self.rids_int is not None:
self.rids_int = self._pad_tensor_to_size(self.rids_int, bs)