On CUDA, take every rank's late-layer MoE gather size from the scheduled per-rank trim counts, which ride the existing scheduler all-gather, and switch and restore them with #43077's late-layer DP helpers; ROCm keeps its eager count gather. Make the helpers' count tensor copy asynchronous so the switch does not synchronize the stream. Restrict to attention TP 1 and TP MoE. (cherry picked from commit a6c2d5f40efec4b491fcdd2dbcedf14ec70d3861)
309 lines
12 KiB
Python
309 lines
12 KiB
Python
from copy import copy
|
|
from typing import Any, Optional
|
|
|
|
import msgspec
|
|
import torch
|
|
|
|
from sglang.srt.utils.common import async_h2d, is_pin_memory_available
|
|
|
|
|
|
class FoldedExtend(msgspec.Struct, frozen=True):
|
|
"""An extend batch whose prefix hits start at their replay start."""
|
|
|
|
batch: Any # ScheduleBatch copy fed to ForwardBatch.init_new
|
|
keep_rows: torch.Tensor # [original rows] int64, folded-row index of each
|
|
scheduled_row: torch.Tensor # [folded rows] int64, original row, -1 if replayed
|
|
row_floor: torch.Tensor # [folded rows] int64
|
|
replay_lens: list[int] # [bs] leading replay rows per request
|
|
num_rows: int
|
|
|
|
|
|
def fold_encoder_swa_replay(worker, batch) -> Optional[FoldedExtend]:
|
|
"""Prepend each hit's <=128 replay tokens to its extend instead of replaying."""
|
|
runner = worker.model_runner
|
|
window = runner.token_to_kv_pool.request_window
|
|
if window is None or not batch.forward_mode.is_extend_without_speculative():
|
|
return None
|
|
rows = _reset_windows_and_collect_hits(batch=batch, window=window)
|
|
if not rows:
|
|
return None
|
|
return _fold_batch(batch=batch, runner=runner, rows=rows)
|
|
|
|
|
|
def resets_swa_window(req) -> bool:
|
|
"""Whether this step starts the request's SWA window: a new request slot or a
|
|
retraction. The prefill budget and `encoder_swa_reset` both read this."""
|
|
return req.kv.req_pool_idx is None or req.is_retracted
|
|
|
|
|
|
def replay_len(prefix_len: int) -> int:
|
|
"""Cached tokens replayed in front of a hit's extend: the last SWA window.
|
|
The prefill budget, the MLP-sync row count and the fold all read this."""
|
|
return min(prefix_len, 128)
|
|
|
|
|
|
def folded_replay_rows(batch) -> int:
|
|
if not batch.forward_mode.is_extend_without_speculative():
|
|
return 0
|
|
return sum(
|
|
replay_len(end)
|
|
for reset, end in zip(batch.encoder_swa_reset, batch.prefix_lens)
|
|
if reset
|
|
)
|
|
|
|
|
|
def decoder_swa_trim_rows(batch, *, folds_encoder: bool) -> int:
|
|
"""Rows removed by a plain extend's decoder tail; -1 requests the backend
|
|
exchange when prompt outputs require whole-extend rows."""
|
|
if not batch.forward_mode.is_extend_without_speculative():
|
|
return 0
|
|
if batch.return_logprob or batch.return_hidden_states:
|
|
return -1
|
|
lens = list(batch.extend_lens)
|
|
if folds_encoder and batch.encoder_swa_reset is not None:
|
|
for i, (reset, prefix) in enumerate(
|
|
zip(batch.encoder_swa_reset, batch.prefix_lens)
|
|
):
|
|
if reset:
|
|
lens[i] += replay_len(prefix)
|
|
return sum(max(0, n - 128) for n in lens)
|
|
|
|
|
|
def apply_folded_extend(folded: FoldedExtend, forward_batch) -> None:
|
|
forward_batch.encoder_swa_row_floor = folded.row_floor
|
|
forward_batch.encoder_swa_compress_rows = folded.keep_rows
|
|
|
|
|
|
def drop_folded_rows(
|
|
*, logits_output, folded: FoldedExtend, tail_lens: Optional[list[int]] = None
|
|
) -> None:
|
|
"""Hand downstream consumers (the DSpark draft) only the original extend rows."""
|
|
hidden = logits_output.hidden_states
|
|
token_indices = logits_output.hidden_states_token_indices
|
|
if token_indices is not None:
|
|
# Decoder-SWA tail rows index the folded extend; keep those on original rows.
|
|
keep = _tail_rows_on_original(
|
|
folded=folded, tail_lens=tail_lens, num_tail_rows=token_indices.shape[0]
|
|
)
|
|
keep = _to_device(keep, token_indices.device)
|
|
logits_output.hidden_states_token_indices = folded.scheduled_row[
|
|
token_indices[keep]
|
|
]
|
|
if hidden is not None:
|
|
logits_output.hidden_states = hidden[keep]
|
|
elif hidden is not None and hidden.shape[0] >= folded.num_rows:
|
|
# MLP-sync padding rows, if any, follow the real rows and are dropped too.
|
|
logits_output.hidden_states = hidden[folded.keep_rows]
|
|
|
|
|
|
def _tail_rows_on_original(
|
|
*, folded: FoldedExtend, tail_lens: list[int], num_tail_rows: int
|
|
):
|
|
keep, off = [], 0
|
|
for n, r, tail in zip(folded.batch.extend_lens, folded.replay_lens, tail_lens):
|
|
keep.append(torch.arange(off + max(0, tail - (n - r)), off + tail))
|
|
off += tail
|
|
assert off == num_tail_rows, (off, num_tail_rows)
|
|
return torch.cat(keep)
|
|
|
|
|
|
def run_encoder_swa_replay(worker, batch):
|
|
from sglang.srt.model_executor.forward_batch_info import (
|
|
CaptureHiddenMode,
|
|
ForwardBatch,
|
|
)
|
|
|
|
runner = worker.model_runner
|
|
window = runner.token_to_kv_pool.request_window
|
|
if window is None or not batch.forward_mode.is_extend_without_speculative():
|
|
return
|
|
rows = _reset_windows_and_collect_hits(batch=batch, window=window)
|
|
if not rows:
|
|
return
|
|
assert batch.engram_staged_rows is None, "staged engram rows need a folded replay"
|
|
# One forward replays every hitting request; each request's window is
|
|
# floored at its own start, so the result matches per-request replays.
|
|
replay = _build_replay_batch(batch=batch, runner=runner, rows=rows)
|
|
fb = ForwardBatch.init_new(
|
|
replay,
|
|
runner,
|
|
capture_hidden_mode=CaptureHiddenMode.NULL,
|
|
return_hidden_states_before_norm=False,
|
|
)
|
|
fb.encoder_swa_replay = True
|
|
runner.forward(fb)
|
|
|
|
|
|
def _reset_windows_and_collect_hits(*, batch, window):
|
|
rows = []
|
|
for i, reset in enumerate(batch.encoder_swa_reset):
|
|
if not reset:
|
|
continue
|
|
window.reset(batch.req_pool_indices[i : i + 1])
|
|
end = batch.prefix_lens[i]
|
|
if not end:
|
|
continue
|
|
if end % 2:
|
|
raise ValueError(
|
|
"encoder SWA replay requires an even cached-prefix boundary"
|
|
)
|
|
rows.append((i, end - replay_len(end), end))
|
|
return rows
|
|
|
|
|
|
def _build_replay_batch(*, batch, runner, rows):
|
|
idx = [i for i, _, _ in rows]
|
|
starts = [s for _, s, _ in rows]
|
|
ends = [e for _, _, e in rows]
|
|
lens = [e - s for s, e in zip(starts, ends)]
|
|
reqs = [batch.reqs[i] for i in idx]
|
|
slots = batch.req_pool_indices[idx]
|
|
req_to_token = runner.req_to_token_pool.req_to_token
|
|
|
|
replay = copy(batch)
|
|
replay.reqs = reqs
|
|
replay.input_ids = torch.tensor(
|
|
[
|
|
t
|
|
for r, s, e in zip(reqs, starts, ends)
|
|
for t in r.full_untruncated_fill_ids[s:e]
|
|
],
|
|
dtype=torch.int64,
|
|
device=runner.device,
|
|
)
|
|
replay.prefill_input_ids_cpu = None
|
|
replay.req_pool_indices = slots
|
|
replay.req_pool_indices_cpu = batch.req_pool_indices_cpu[idx]
|
|
replay.prefix_lens = starts
|
|
replay.extend_lens = lens
|
|
replay.extend_num_tokens = sum(lens)
|
|
replay.seq_lens = torch.tensor(ends, dtype=torch.int64, device=runner.device)
|
|
replay.seq_lens_cpu = torch.tensor(ends, dtype=torch.int64)
|
|
replay.seq_lens_sum = sum(ends)
|
|
replay.orig_seq_lens = replay.seq_lens
|
|
replay.out_cache_loc = torch.cat(
|
|
[req_to_token[slots[k], s:e] for k, (s, e) in enumerate(zip(starts, ends))]
|
|
).long()
|
|
replay.return_logprob = False
|
|
replay.top_logprobs_nums = None
|
|
replay.token_ids_logprobs = None
|
|
replay.extend_logprob_start_lens = lens
|
|
replay.extend_input_logprob_token_ids = None
|
|
replay.is_prefill_only = True
|
|
replay.spec_info = None
|
|
replay.sampling_info = None
|
|
replay.has_grammar = False
|
|
replay.multimodal_inputs = [None] * len(reqs)
|
|
replay.engram_history = _engram_history(reqs=reqs, starts=starts, runner=runner)
|
|
_set_execution_counts(batch=replay, scheduled=batch)
|
|
return replay
|
|
|
|
|
|
def _engram_history(*, reqs, starts, runner):
|
|
hasher = runner.model.model.engram_hasher
|
|
if hasher is None:
|
|
return None
|
|
n = hasher.max_ngram_size - 1
|
|
hist = []
|
|
for r, s in zip(reqs, starts):
|
|
ids = r.full_untruncated_fill_ids[max(0, s - n) : s]
|
|
hist.append(hasher.history_row(ids))
|
|
return async_h2d(hist, dtype=torch.int32, device=runner.device)
|
|
|
|
|
|
def _fold_batch(*, batch, runner, rows) -> FoldedExtend:
|
|
bs = len(batch.reqs)
|
|
device = runner.device
|
|
replay = [0] * bs
|
|
for i, s, e in rows:
|
|
replay[i] = e - s
|
|
ext = list(batch.extend_lens)
|
|
pre = [p - r for p, r in zip(batch.prefix_lens, replay)]
|
|
new_ext = [x + r for x, r in zip(ext, replay)]
|
|
req_to_token = runner.req_to_token_pool.req_to_token
|
|
slots = batch.req_pool_indices_cpu.tolist()
|
|
ids, locs, keep, cpu_ids = [], [], [], []
|
|
off = fold_off = 0
|
|
for i, req in enumerate(batch.reqs):
|
|
r, n = replay[i], ext[i]
|
|
if r:
|
|
head = list(req.full_untruncated_fill_ids[pre[i] : pre[i] + r])
|
|
head_ids = async_h2d(head, dtype=batch.input_ids.dtype, device=device)
|
|
ids.append(head_ids)
|
|
locs.append(req_to_token[slots[i], pre[i] : pre[i] + r].long())
|
|
cpu_ids.append(torch.tensor(head, dtype=torch.int64))
|
|
ids.append(batch.input_ids[off : off + n])
|
|
locs.append(batch.out_cache_loc[off : off + n].long())
|
|
if batch.prefill_input_ids_cpu is not None:
|
|
cpu_ids.append(batch.prefill_input_ids_cpu[off : off + n].to(torch.int64))
|
|
keep.append(torch.arange(fold_off + r, fold_off + r + n))
|
|
off += n
|
|
fold_off += r + n
|
|
|
|
folded = copy(batch)
|
|
folded.input_ids = torch.cat(ids)
|
|
folded.out_cache_loc = torch.cat(locs)
|
|
folded.prefill_input_ids_cpu = (
|
|
torch.cat(cpu_ids) if batch.prefill_input_ids_cpu is not None else None
|
|
)
|
|
folded.prefix_lens = pre
|
|
folded.extend_lens = new_ext
|
|
folded.extend_num_tokens = fold_off
|
|
if batch.extend_logprob_start_lens is not None:
|
|
folded.extend_logprob_start_lens = [
|
|
x + r for x, r in zip(batch.extend_logprob_start_lens, replay)
|
|
]
|
|
if batch.engram_history is not None:
|
|
folded.engram_history = _engram_history(
|
|
reqs=batch.reqs, starts=pre, runner=runner
|
|
)
|
|
floor = torch.tensor(
|
|
[p if r else 0 for p, r in zip(pre, replay)], dtype=torch.int64
|
|
)
|
|
row_floor = torch.repeat_interleave(
|
|
floor, torch.tensor(new_ext), output_size=fold_off
|
|
)
|
|
keep_rows = torch.cat(keep)
|
|
scheduled_row = torch.full((fold_off,), -1, dtype=torch.int64)
|
|
scheduled_row[keep_rows] = torch.arange(keep_rows.shape[0])
|
|
keep_rows = _to_device(keep_rows, device)
|
|
scheduled_row = _to_device(scheduled_row, device)
|
|
row_floor = _to_device(row_floor, device)
|
|
return FoldedExtend(
|
|
batch=folded,
|
|
keep_rows=keep_rows,
|
|
scheduled_row=scheduled_row,
|
|
row_floor=row_floor,
|
|
replay_lens=replay,
|
|
num_rows=fold_off,
|
|
)
|
|
|
|
|
|
def _to_device(t: torch.Tensor, device) -> torch.Tensor:
|
|
# A pageable host tensor's copy waits for the stream even with non_blocking.
|
|
pin = is_pin_memory_available(device)
|
|
return (t.pin_memory() if pin else t).to(device, non_blocking=pin)
|
|
|
|
|
|
def _set_execution_counts(*, batch, scheduled) -> None:
|
|
"""MLP-sync counts of a batch copied from `scheduled` with different rows."""
|
|
if scheduled.global_num_tokens is None:
|
|
return
|
|
# Attention DP is admitted only with the fold, so the gather holds one count.
|
|
if (
|
|
len(scheduled.global_num_tokens) != 1
|
|
or scheduled.global_num_tokens[0] != scheduled.extend_num_tokens
|
|
):
|
|
raise ValueError(
|
|
"encoder SWA replay expects one MLP-sync count equal to the scheduled "
|
|
f"extend ({scheduled.extend_num_tokens}), got {scheduled.global_num_tokens}"
|
|
)
|
|
batch.global_num_tokens = [batch.extend_num_tokens]
|
|
# The scheduler's formula; the draft counts stay with the scheduled batch.
|
|
batch.global_num_tokens_for_logprob = [
|
|
sum(
|
|
max(n - start, 1)
|
|
for n, start in zip(batch.extend_lens, batch.extend_logprob_start_lens)
|
|
)
|
|
]
|