[Feat] dsv4.1: Return prompt logprobs under encoder SWA bounded replay (#45050)
This commit is contained in:
@@ -2875,13 +2875,6 @@ class ScheduleBatch(ScheduleBatchDisaggregationDecodeMixin):
|
||||
"encoder SWA replay does not support input_embeds or "
|
||||
"positional_embed_overrides"
|
||||
)
|
||||
if req.return_logprob and req.logprob_start_len not in (
|
||||
-1,
|
||||
len(req.origin_input_ids),
|
||||
):
|
||||
raise ValueError(
|
||||
"encoder SWA replay cannot return cached prompt logprobs"
|
||||
)
|
||||
self.encoder_swa_reset = [resets_swa_window(r) for r in reqs]
|
||||
# Allocate memory
|
||||
out_cache_loc, req_pool_indices_tensor, req_pool_indices_cpu = alloc_for_extend(
|
||||
|
||||
@@ -1342,14 +1342,6 @@ class TokenizerManager(TokenizerControlMixin, TokenizerManagerScoreMixin):
|
||||
"encoder SWA replay does not support video, audio, input_embeds, "
|
||||
"or positional_embed_overrides"
|
||||
)
|
||||
if (
|
||||
isinstance(obj, GenerateReqInput)
|
||||
and obj.return_logprob
|
||||
and obj.logprob_start_len not in (None, -1, len(input_ids))
|
||||
):
|
||||
raise ValueError(
|
||||
"encoder SWA replay cannot return cached prompt logprobs"
|
||||
)
|
||||
# A decision layout indexes the whole prompt, so it is checked before any
|
||||
# truncation below could cut the prompt.
|
||||
self._validate_joint_schema_request(obj, input_ids)
|
||||
|
||||
@@ -663,7 +663,16 @@ class TpModelWorker(BaseTpWorker):
|
||||
drop_folded_rows,
|
||||
)
|
||||
|
||||
drop_folded_rows(logits_output=logits_output, folded=folded)
|
||||
tail = self.model_runner.attn_backend.tail_forward_metadata
|
||||
drop_folded_rows(
|
||||
logits_output=logits_output,
|
||||
folded=folded,
|
||||
tail_lens=(
|
||||
None
|
||||
if tail is None
|
||||
else tail.late_layer_tail.extend_seq_lens_cpu
|
||||
),
|
||||
)
|
||||
batch_result = GenerationBatchResult(
|
||||
logits_output=logits_output,
|
||||
can_run_cuda_graph=can_run_cuda_graph,
|
||||
|
||||
@@ -55,14 +55,16 @@ def apply_folded_extend(folded: FoldedExtend, forward_batch) -> None:
|
||||
forward_batch.encoder_swa_compress_rows = folded.keep_rows
|
||||
|
||||
|
||||
def drop_folded_rows(*, logits_output, folded: FoldedExtend) -> None:
|
||||
def drop_folded_rows(
|
||||
*, logits_output, folded: FoldedExtend, tail_lens: Optional[list[int]] = None
|
||||
) -> None:
|
||||
"""Hand downstream consumers (the DSpark draft) only the original extend rows."""
|
||||
hidden = logits_output.hidden_states
|
||||
token_indices = logits_output.hidden_states_token_indices
|
||||
if token_indices is not None:
|
||||
# Decoder-SWA tail rows index the folded extend; keep those on original rows.
|
||||
keep = _tail_rows_on_original(
|
||||
folded=folded, num_tail_rows=token_indices.shape[0]
|
||||
folded=folded, tail_lens=tail_lens, num_tail_rows=token_indices.shape[0]
|
||||
).to(token_indices.device, non_blocking=True)
|
||||
logits_output.hidden_states_token_indices = folded.scheduled_row[
|
||||
token_indices[keep]
|
||||
@@ -74,13 +76,11 @@ def drop_folded_rows(*, logits_output, folded: FoldedExtend) -> None:
|
||||
logits_output.hidden_states = hidden[folded.keep_rows]
|
||||
|
||||
|
||||
def _tail_rows_on_original(*, folded: FoldedExtend, num_tail_rows: int):
|
||||
# Each request's tail is its last SWA_WINDOW folded rows; replay rows lead it.
|
||||
from sglang.srt.layers.attention.deepseek_v4_backend import SWA_WINDOW
|
||||
|
||||
def _tail_rows_on_original(
|
||||
*, folded: FoldedExtend, tail_lens: list[int], num_tail_rows: int
|
||||
):
|
||||
keep, off = [], 0
|
||||
for n, r in zip(folded.batch.extend_lens, folded.replay_lens):
|
||||
tail = min(SWA_WINDOW, n)
|
||||
for n, r, tail in zip(folded.batch.extend_lens, folded.replay_lens, tail_lens):
|
||||
keep.append(torch.arange(off + max(0, tail - (n - r)), off + tail))
|
||||
off += tail
|
||||
assert off == num_tail_rows, (off, num_tail_rows)
|
||||
|
||||
Reference in New Issue
Block a user