[Feat] dsv4.1: Return prompt logprobs under encoder SWA bounded replay (#45050)

This commit is contained in:
Khoa Pham
2026-10-18 19:08:20 -07:00
committed by GitHub
parent 69076b9f9e
commit a7fd2e1158
5 changed files with 57 additions and 26 deletions
@@ -2875,13 +2875,6 @@ class ScheduleBatch(ScheduleBatchDisaggregationDecodeMixin):
"encoder SWA replay does not support input_embeds or "
"positional_embed_overrides"
)
if req.return_logprob and req.logprob_start_len not in (
-1,
len(req.origin_input_ids),
):
raise ValueError(
"encoder SWA replay cannot return cached prompt logprobs"
)
self.encoder_swa_reset = [resets_swa_window(r) for r in reqs]
# Allocate memory
out_cache_loc, req_pool_indices_tensor, req_pool_indices_cpu = alloc_for_extend(
@@ -1342,14 +1342,6 @@ class TokenizerManager(TokenizerControlMixin, TokenizerManagerScoreMixin):
"encoder SWA replay does not support video, audio, input_embeds, "
"or positional_embed_overrides"
)
if (
isinstance(obj, GenerateReqInput)
and obj.return_logprob
and obj.logprob_start_len not in (None, -1, len(input_ids))
):
raise ValueError(
"encoder SWA replay cannot return cached prompt logprobs"
)
# A decision layout indexes the whole prompt, so it is checked before any
# truncation below could cut the prompt.
self._validate_joint_schema_request(obj, input_ids)
+10 -1
View File
@@ -663,7 +663,16 @@ class TpModelWorker(BaseTpWorker):
drop_folded_rows,
)
drop_folded_rows(logits_output=logits_output, folded=folded)
tail = self.model_runner.attn_backend.tail_forward_metadata
drop_folded_rows(
logits_output=logits_output,
folded=folded,
tail_lens=(
None
if tail is None
else tail.late_layer_tail.extend_seq_lens_cpu
),
)
batch_result = GenerationBatchResult(
logits_output=logits_output,
can_run_cuda_graph=can_run_cuda_graph,
@@ -55,14 +55,16 @@ def apply_folded_extend(folded: FoldedExtend, forward_batch) -> None:
forward_batch.encoder_swa_compress_rows = folded.keep_rows
def drop_folded_rows(*, logits_output, folded: FoldedExtend) -> None:
def drop_folded_rows(
*, logits_output, folded: FoldedExtend, tail_lens: Optional[list[int]] = None
) -> None:
"""Hand downstream consumers (the DSpark draft) only the original extend rows."""
hidden = logits_output.hidden_states
token_indices = logits_output.hidden_states_token_indices
if token_indices is not None:
# Decoder-SWA tail rows index the folded extend; keep those on original rows.
keep = _tail_rows_on_original(
folded=folded, num_tail_rows=token_indices.shape[0]
folded=folded, tail_lens=tail_lens, num_tail_rows=token_indices.shape[0]
).to(token_indices.device, non_blocking=True)
logits_output.hidden_states_token_indices = folded.scheduled_row[
token_indices[keep]
@@ -74,13 +76,11 @@ def drop_folded_rows(*, logits_output, folded: FoldedExtend) -> None:
logits_output.hidden_states = hidden[folded.keep_rows]
def _tail_rows_on_original(*, folded: FoldedExtend, num_tail_rows: int):
# Each request's tail is its last SWA_WINDOW folded rows; replay rows lead it.
from sglang.srt.layers.attention.deepseek_v4_backend import SWA_WINDOW
def _tail_rows_on_original(
*, folded: FoldedExtend, tail_lens: list[int], num_tail_rows: int
):
keep, off = [], 0
for n, r in zip(folded.batch.extend_lens, folded.replay_lens):
tail = min(SWA_WINDOW, n)
for n, r, tail in zip(folded.batch.extend_lens, folded.replay_lens, tail_lens):
keep.append(torch.arange(off + max(0, tail - (n - r)), off + tail))
off += tail
assert off == num_tail_rows, (off, num_tail_rows)