perf: dsv41 decoder tail runs only the prompt's last window rows of each prefill chunk (opt-in)
(cherry picked from commit 1b778e6df01a138a35403627e209cf33967246e6)
This commit is contained in:
@@ -443,15 +443,23 @@ def late_layer_tail_lens(
|
||||
extend_lens_cpu: list[int],
|
||||
tail_len: int,
|
||||
whole_extend: Optional[list[bool]] = None,
|
||||
prompt_rest_lens: Optional[list[int]] = None,
|
||||
) -> list[int]:
|
||||
"""Each request's late-layer tail: its last min(tail_len, extend_len) tokens,
|
||||
or its whole extend where whole_extend is set."""
|
||||
if whole_extend is None:
|
||||
return [min(tail_len, n) for n in extend_lens_cpu]
|
||||
return [
|
||||
n if whole else min(tail_len, n)
|
||||
for n, whole in zip(extend_lens_cpu, whole_extend)
|
||||
]
|
||||
or its whole extend where whole_extend is set. With prompt_rest_lens (prompt
|
||||
tokens left after each extend), only its rows in the prompt's last tail_len
|
||||
tokens, and at least its last row."""
|
||||
if whole_extend is not None and any(whole_extend):
|
||||
return [
|
||||
n if whole else min(tail_len, n)
|
||||
for n, whole in zip(extend_lens_cpu, whole_extend)
|
||||
]
|
||||
if prompt_rest_lens is not None:
|
||||
return [
|
||||
min(n, max(1, tail_len - rest))
|
||||
for n, rest in zip(extend_lens_cpu, prompt_rest_lens, strict=True)
|
||||
]
|
||||
return [min(tail_len, n) for n in extend_lens_cpu]
|
||||
|
||||
|
||||
def late_layer_tail_layout(
|
||||
@@ -461,13 +469,15 @@ def late_layer_tail_layout(
|
||||
tail_len: int,
|
||||
device: torch.device,
|
||||
whole_extend: Optional[list[bool]] = None,
|
||||
prompt_rest_lens: Optional[list[int]] = None,
|
||||
) -> tuple[torch.Tensor, list[int], torch.Tensor]:
|
||||
"""Tail rows of each prefill extend: its last min(tail_len, extend_len) tokens,
|
||||
or its whole extend where whole_extend is set.
|
||||
Returns (token indices into the extend, per-request tail lengths, per-row
|
||||
absolute window floor). The floor is seq_len - tail_len, so a tail shorter
|
||||
than tail_len reads rows the request wrote before this extend. A whole
|
||||
extend keeps every row's plain window."""
|
||||
extend keeps every row's plain window. With prompt_rest_lens, tails and
|
||||
floors follow the prompt's end instead (late_layer_tail_lens, _tail_floors)."""
|
||||
if whole_extend is not None and any(whole_extend):
|
||||
return _whole_extend_tail_layout(
|
||||
extend_lens_cpu=extend_lens_cpu,
|
||||
@@ -476,16 +486,19 @@ def late_layer_tail_layout(
|
||||
device=device,
|
||||
whole_extend=whole_extend,
|
||||
)
|
||||
tail_lens_cpu = late_layer_tail_lens(extend_lens_cpu, tail_len)
|
||||
tail_lens_cpu = late_layer_tail_lens(
|
||||
extend_lens_cpu, tail_len, prompt_rest_lens=prompt_rest_lens
|
||||
)
|
||||
floors_cpu = _tail_floors(seq_lens_cpu, tail_len, prompt_rest_lens)
|
||||
if len(extend_lens_cpu) == 1:
|
||||
n, t, s = extend_lens_cpu[0], tail_lens_cpu[0], seq_lens_cpu[0]
|
||||
floor = torch.full((t,), max(0, s - tail_len), dtype=torch.int32, device=device)
|
||||
n, t = extend_lens_cpu[0], tail_lens_cpu[0]
|
||||
floor = torch.full((t,), floors_cpu[0], dtype=torch.int32, device=device)
|
||||
return torch.arange(n - t, n, device=device), tail_lens_cpu, floor
|
||||
# One H2D copy for the three length vectors; launch count does not grow with bs.
|
||||
lens = async_h2d(
|
||||
[extend_lens_cpu, tail_lens_cpu, seq_lens_cpu], dtype=torch.int64, device=device
|
||||
[extend_lens_cpu, tail_lens_cpu, floors_cpu], dtype=torch.int64, device=device
|
||||
)
|
||||
extend_lens, tail_lens, seq_lens = lens[0], lens[1], lens[2]
|
||||
extend_lens, tail_lens, floors = lens[0], lens[1], lens[2]
|
||||
total = sum(tail_lens_cpu)
|
||||
req = torch.repeat_interleave(
|
||||
torch.arange(len(tail_lens_cpu), device=device), tail_lens, output_size=total
|
||||
@@ -495,10 +508,23 @@ def late_layer_tail_layout(
|
||||
- (torch.cumsum(tail_lens, 0) - tail_lens)[req]
|
||||
)
|
||||
token_indices = (torch.cumsum(extend_lens, 0) - tail_lens)[req] + offs
|
||||
floor = (seq_lens - tail_len).clamp_min(0)[req].to(torch.int32)
|
||||
floor = floors[req].to(torch.int32)
|
||||
return token_indices, tail_lens_cpu, floor
|
||||
|
||||
|
||||
def _tail_floors(
|
||||
seq_lens_cpu: list[int], tail_len: int, prompt_rest_lens: Optional[list[int]]
|
||||
) -> list[int]:
|
||||
"""Each request's window floor: tail_len below its extend's end, or below its
|
||||
prompt's end, clamped to its last row, whose own KV a lone tail row reads."""
|
||||
if prompt_rest_lens is None:
|
||||
return [max(0, s - tail_len) for s in seq_lens_cpu]
|
||||
return [
|
||||
min(max(0, s + rest - tail_len), max(0, s - 1))
|
||||
for s, rest in zip(seq_lens_cpu, prompt_rest_lens, strict=True)
|
||||
]
|
||||
|
||||
|
||||
def _whole_extend_tail_layout(
|
||||
*,
|
||||
extend_lens_cpu: list[int],
|
||||
|
||||
@@ -728,6 +728,7 @@ class TboForwardBatchPreparer:
|
||||
"extend_prefix_lens_cpu",
|
||||
"extend_seq_lens_cpu",
|
||||
"extend_logprob_start_lens_cpu",
|
||||
"extend_prompt_rest_lens_cpu",
|
||||
"lora_ids",
|
||||
"rids",
|
||||
]:
|
||||
|
||||
@@ -1719,6 +1719,9 @@ class Envs:
|
||||
# Prefill-graph shapes of at least this many tokens run the tail from the tail
|
||||
# graphs in a break (measured crossover 1536-2048 tokens on GB300 TP4).
|
||||
SGLANG_DSV4_DECODER_TAIL_GRAPH_MIN_PREFILL_TOKENS = EnvInt(2048)
|
||||
# Decoder SWA bounded replay: a prefill chunk runs the tail layers only over its
|
||||
# rows in the prompt's last window, else over its last row.
|
||||
SGLANG_OPT_DSV41_PROMPT_WINDOW_TAIL = EnvBool(False)
|
||||
# Paged KV layout of the DeepSeek-V4 family pools: "v4" (584 B/token, every
|
||||
# GPU), "v41" (the SM100 FlashMLA V4.1 formats: 528 B fp8 SWA cache, fp8 or
|
||||
# fp4 compressed caches), "v41_new" (the same bytes interleaved per token, for
|
||||
|
||||
@@ -1024,7 +1024,10 @@ def late_layer_num_tokens(forward_batch: ForwardBatch, *, dspark: bool) -> int:
|
||||
whole_extend = late_layer_whole_extend(forward_batch, dspark=dspark)
|
||||
return sum(
|
||||
late_layer_tail_lens(
|
||||
forward_batch.extend_seq_lens_cpu, SWA_WINDOW, whole_extend
|
||||
forward_batch.extend_seq_lens_cpu,
|
||||
SWA_WINDOW,
|
||||
whole_extend,
|
||||
prompt_rest_lens=forward_batch.extend_prompt_rest_lens_cpu,
|
||||
)
|
||||
)
|
||||
|
||||
@@ -1834,6 +1837,7 @@ class DeepseekV4AttnBackend(
|
||||
whole_extend=late_layer_whole_extend(
|
||||
forward_batch, dspark=self.model_runner.spec_algorithm.is_dspark()
|
||||
),
|
||||
prompt_rest_lens=forward_batch.extend_prompt_rest_lens_cpu,
|
||||
)
|
||||
if self.encoder_row_floor is not None:
|
||||
# Folded hits: a tail row never reads below its request's replay start.
|
||||
|
||||
@@ -2252,6 +2252,7 @@ class DeepseekV4HipRadixBackend(
|
||||
tail_len=SWA_WINDOW,
|
||||
device=device,
|
||||
whole_extend=late_layer_whole_extend(forward_batch, dspark=self.is_dspark),
|
||||
prompt_rest_lens=forward_batch.extend_prompt_rest_lens_cpu,
|
||||
)
|
||||
contiguous_start = (
|
||||
extend_lens_cpu[0] - tail_lens_cpu[0] if len(extend_lens_cpu) == 1 else None
|
||||
|
||||
@@ -78,7 +78,7 @@ if TYPE_CHECKING:
|
||||
from sglang.srt.layers.engram_staging import StagedRows
|
||||
from sglang.srt.layers.layer_boundary.residual.stream import ResidualStream
|
||||
from sglang.srt.layers.logits_processor import LogitsProcessorOutput
|
||||
from sglang.srt.managers.schedule_batch import MultimodalInputs, ScheduleBatch
|
||||
from sglang.srt.managers.schedule_batch import MultimodalInputs, Req, ScheduleBatch
|
||||
from sglang.srt.mem_cache.kv_loc_plan import Cols, KVLocPlan
|
||||
from sglang.srt.model_executor.model_runner import ModelRunner
|
||||
from sglang.srt.sampling.sampling_batch_info import SamplingBatchInfo
|
||||
@@ -489,6 +489,17 @@ class NgramEmbeddingInfo:
|
||||
)
|
||||
|
||||
|
||||
def prompt_rest_lens_of(
|
||||
reqs: List[Req], seq_lens: List[int], decoding_req_ids: Set[int]
|
||||
) -> List[int]:
|
||||
"""Prompt tokens each request prefills after this extend, from the batch's own
|
||||
ends (an encoder replay batch ends at its prefix); a decode row has none."""
|
||||
return [
|
||||
0 if id(req) in decoding_req_ids else len(req.full_untruncated_fill_ids) - end
|
||||
for req, end in zip(reqs, seq_lens, strict=True)
|
||||
]
|
||||
|
||||
|
||||
@dataclass
|
||||
class ForwardBatch(ForwardBatchDeepSeekMHAMixin):
|
||||
"""Store all inputs of a forward pass."""
|
||||
@@ -669,6 +680,8 @@ class ForwardBatch(ForwardBatchDeepSeekMHAMixin):
|
||||
extend_prefix_lens_cpu: Optional[List[int]] = None
|
||||
extend_seq_lens_cpu: Optional[List[int]] = None
|
||||
extend_logprob_start_lens_cpu: Optional[List[int]] = None
|
||||
# Prompt tokens each request prefills after this extend (decoder SWA tail rows).
|
||||
extend_prompt_rest_lens_cpu: Optional[List[int]] = None
|
||||
extend_input_logprob_token_ids_gpu: Optional[torch.Tensor] = None
|
||||
|
||||
# For DP attention (MLP sync sizes)
|
||||
@@ -1015,6 +1028,7 @@ class ForwardBatch(ForwardBatchDeepSeekMHAMixin):
|
||||
batch.seq_lens_sum = int(seq_lens_cpu.sum())
|
||||
|
||||
contains_last_prefill_chunk = False
|
||||
prompt_rest_lens = None
|
||||
if batch.forward_mode.is_extend_without_speculative():
|
||||
decoding_req_ids = {id(req) for req in (batch.decoding_reqs or ())}
|
||||
contains_last_prefill_chunk = any(
|
||||
@@ -1022,6 +1036,14 @@ class ForwardBatch(ForwardBatchDeepSeekMHAMixin):
|
||||
for req in batch.reqs
|
||||
if id(req) not in decoding_req_ids
|
||||
)
|
||||
if (
|
||||
envs.SGLANG_OPT_DSV41_PROMPT_WINDOW_TAIL.get()
|
||||
and get_parallel().attn_cp_size == 1
|
||||
and seq_lens_cpu is not None
|
||||
):
|
||||
prompt_rest_lens = prompt_rest_lens_of(
|
||||
batch.reqs, seq_lens_cpu.tolist(), decoding_req_ids
|
||||
)
|
||||
|
||||
from sglang.srt.model_executor.model_runner import ModelRunner
|
||||
|
||||
@@ -1082,6 +1104,7 @@ class ForwardBatch(ForwardBatchDeepSeekMHAMixin):
|
||||
global_forward_mode=batch.global_forward_mode,
|
||||
is_prefill_only=batch.is_prefill_only,
|
||||
contains_last_prefill_chunk=contains_last_prefill_chunk,
|
||||
extend_prompt_rest_lens_cpu=prompt_rest_lens,
|
||||
spec_algorithm=batch.spec_algorithm,
|
||||
capture_hidden_mode=capture_hidden_mode,
|
||||
return_hidden_states_before_norm=return_hidden_states_before_norm,
|
||||
@@ -2074,6 +2097,10 @@ class ForwardBatch(ForwardBatchDeepSeekMHAMixin):
|
||||
self.extend_logprob_start_lens_cpu.extend(
|
||||
[0] * (bs - len(self.extend_logprob_start_lens_cpu))
|
||||
)
|
||||
if self.extend_prompt_rest_lens_cpu is not None:
|
||||
self.extend_prompt_rest_lens_cpu.extend(
|
||||
[0] * (bs - len(self.extend_prompt_rest_lens_cpu))
|
||||
)
|
||||
|
||||
if self.rids_int is not None:
|
||||
self.rids_int = self._pad_tensor_to_size(self.rids_int, bs)
|
||||
|
||||
Reference in New Issue
Block a user