Compare commits
4
Commits
main
...
b300-pick-261102
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
d5a998f346 | ||
|
|
a3401b3e65 | ||
|
|
01087e689c | ||
|
|
0b45c07637 |
@@ -443,15 +443,23 @@ def late_layer_tail_lens(
|
||||
extend_lens_cpu: list[int],
|
||||
tail_len: int,
|
||||
whole_extend: Optional[list[bool]] = None,
|
||||
prompt_rest_lens: Optional[list[int]] = None,
|
||||
) -> list[int]:
|
||||
"""Each request's late-layer tail: its last min(tail_len, extend_len) tokens,
|
||||
or its whole extend where whole_extend is set."""
|
||||
if whole_extend is None:
|
||||
return [min(tail_len, n) for n in extend_lens_cpu]
|
||||
return [
|
||||
n if whole else min(tail_len, n)
|
||||
for n, whole in zip(extend_lens_cpu, whole_extend)
|
||||
]
|
||||
or its whole extend where whole_extend is set. With prompt_rest_lens (prompt
|
||||
tokens left after each extend), only its rows in the prompt's last tail_len
|
||||
tokens, and at least its last row."""
|
||||
if whole_extend is not None and any(whole_extend):
|
||||
return [
|
||||
n if whole else min(tail_len, n)
|
||||
for n, whole in zip(extend_lens_cpu, whole_extend)
|
||||
]
|
||||
if prompt_rest_lens is not None:
|
||||
return [
|
||||
min(n, max(1, tail_len - rest))
|
||||
for n, rest in zip(extend_lens_cpu, prompt_rest_lens, strict=True)
|
||||
]
|
||||
return [min(tail_len, n) for n in extend_lens_cpu]
|
||||
|
||||
|
||||
def late_layer_tail_layout(
|
||||
@@ -461,13 +469,15 @@ def late_layer_tail_layout(
|
||||
tail_len: int,
|
||||
device: torch.device,
|
||||
whole_extend: Optional[list[bool]] = None,
|
||||
prompt_rest_lens: Optional[list[int]] = None,
|
||||
) -> tuple[torch.Tensor, list[int], torch.Tensor]:
|
||||
"""Tail rows of each prefill extend: its last min(tail_len, extend_len) tokens,
|
||||
or its whole extend where whole_extend is set.
|
||||
Returns (token indices into the extend, per-request tail lengths, per-row
|
||||
absolute window floor). The floor is seq_len - tail_len, so a tail shorter
|
||||
than tail_len reads rows the request wrote before this extend. A whole
|
||||
extend keeps every row's plain window."""
|
||||
extend keeps every row's plain window. With prompt_rest_lens, tails and
|
||||
floors follow the prompt's end instead (late_layer_tail_lens, _tail_floors)."""
|
||||
if whole_extend is not None and any(whole_extend):
|
||||
return _whole_extend_tail_layout(
|
||||
extend_lens_cpu=extend_lens_cpu,
|
||||
@@ -476,16 +486,19 @@ def late_layer_tail_layout(
|
||||
device=device,
|
||||
whole_extend=whole_extend,
|
||||
)
|
||||
tail_lens_cpu = late_layer_tail_lens(extend_lens_cpu, tail_len)
|
||||
tail_lens_cpu = late_layer_tail_lens(
|
||||
extend_lens_cpu, tail_len, prompt_rest_lens=prompt_rest_lens
|
||||
)
|
||||
floors_cpu = _tail_floors(seq_lens_cpu, tail_len, prompt_rest_lens)
|
||||
if len(extend_lens_cpu) == 1:
|
||||
n, t, s = extend_lens_cpu[0], tail_lens_cpu[0], seq_lens_cpu[0]
|
||||
floor = torch.full((t,), max(0, s - tail_len), dtype=torch.int32, device=device)
|
||||
n, t = extend_lens_cpu[0], tail_lens_cpu[0]
|
||||
floor = torch.full((t,), floors_cpu[0], dtype=torch.int32, device=device)
|
||||
return torch.arange(n - t, n, device=device), tail_lens_cpu, floor
|
||||
# One H2D copy for the three length vectors; launch count does not grow with bs.
|
||||
lens = async_h2d(
|
||||
[extend_lens_cpu, tail_lens_cpu, seq_lens_cpu], dtype=torch.int64, device=device
|
||||
[extend_lens_cpu, tail_lens_cpu, floors_cpu], dtype=torch.int64, device=device
|
||||
)
|
||||
extend_lens, tail_lens, seq_lens = lens[0], lens[1], lens[2]
|
||||
extend_lens, tail_lens, floors = lens[0], lens[1], lens[2]
|
||||
total = sum(tail_lens_cpu)
|
||||
req = torch.repeat_interleave(
|
||||
torch.arange(len(tail_lens_cpu), device=device), tail_lens, output_size=total
|
||||
@@ -495,10 +508,23 @@ def late_layer_tail_layout(
|
||||
- (torch.cumsum(tail_lens, 0) - tail_lens)[req]
|
||||
)
|
||||
token_indices = (torch.cumsum(extend_lens, 0) - tail_lens)[req] + offs
|
||||
floor = (seq_lens - tail_len).clamp_min(0)[req].to(torch.int32)
|
||||
floor = floors[req].to(torch.int32)
|
||||
return token_indices, tail_lens_cpu, floor
|
||||
|
||||
|
||||
def _tail_floors(
|
||||
seq_lens_cpu: list[int], tail_len: int, prompt_rest_lens: Optional[list[int]]
|
||||
) -> list[int]:
|
||||
"""Each request's window floor: tail_len below its extend's end, or below its
|
||||
prompt's end, clamped to its last row, whose own KV a lone tail row reads."""
|
||||
if prompt_rest_lens is None:
|
||||
return [max(0, s - tail_len) for s in seq_lens_cpu]
|
||||
return [
|
||||
min(max(0, s + rest - tail_len), max(0, s - 1))
|
||||
for s, rest in zip(seq_lens_cpu, prompt_rest_lens, strict=True)
|
||||
]
|
||||
|
||||
|
||||
def _whole_extend_tail_layout(
|
||||
*,
|
||||
extend_lens_cpu: list[int],
|
||||
|
||||
@@ -728,6 +728,7 @@ class TboForwardBatchPreparer:
|
||||
"extend_prefix_lens_cpu",
|
||||
"extend_seq_lens_cpu",
|
||||
"extend_logprob_start_lens_cpu",
|
||||
"extend_prompt_rest_lens_cpu",
|
||||
"lora_ids",
|
||||
"rids",
|
||||
]:
|
||||
|
||||
@@ -1719,6 +1719,9 @@ class Envs:
|
||||
# Prefill-graph shapes of at least this many tokens run the tail from the tail
|
||||
# graphs in a break (measured crossover 1536-2048 tokens on GB300 TP4).
|
||||
SGLANG_DSV4_DECODER_TAIL_GRAPH_MIN_PREFILL_TOKENS = EnvInt(2048)
|
||||
# Decoder SWA bounded replay: a prefill chunk runs the tail layers only over its
|
||||
# rows in the prompt's last window, else over its last row.
|
||||
SGLANG_OPT_DSV41_PROMPT_WINDOW_TAIL = EnvBool(False)
|
||||
# Paged KV layout of the DeepSeek-V4 family pools: "v4" (584 B/token, every
|
||||
# GPU), "v41" (the SM100 FlashMLA V4.1 formats: 528 B fp8 SWA cache, fp8 or
|
||||
# fp4 compressed caches), "v41_new" (the same bytes interleaved per token, for
|
||||
|
||||
@@ -1024,7 +1024,10 @@ def late_layer_num_tokens(forward_batch: ForwardBatch, *, dspark: bool) -> int:
|
||||
whole_extend = late_layer_whole_extend(forward_batch, dspark=dspark)
|
||||
return sum(
|
||||
late_layer_tail_lens(
|
||||
forward_batch.extend_seq_lens_cpu, SWA_WINDOW, whole_extend
|
||||
forward_batch.extend_seq_lens_cpu,
|
||||
SWA_WINDOW,
|
||||
whole_extend,
|
||||
prompt_rest_lens=forward_batch.extend_prompt_rest_lens_cpu,
|
||||
)
|
||||
)
|
||||
|
||||
@@ -1834,6 +1837,7 @@ class DeepseekV4AttnBackend(
|
||||
whole_extend=late_layer_whole_extend(
|
||||
forward_batch, dspark=self.model_runner.spec_algorithm.is_dspark()
|
||||
),
|
||||
prompt_rest_lens=forward_batch.extend_prompt_rest_lens_cpu,
|
||||
)
|
||||
if self.encoder_row_floor is not None:
|
||||
# Folded hits: a tail row never reads below its request's replay start.
|
||||
|
||||
@@ -2252,6 +2252,7 @@ class DeepseekV4HipRadixBackend(
|
||||
tail_len=SWA_WINDOW,
|
||||
device=device,
|
||||
whole_extend=late_layer_whole_extend(forward_batch, dspark=self.is_dspark),
|
||||
prompt_rest_lens=forward_batch.extend_prompt_rest_lens_cpu,
|
||||
)
|
||||
contiguous_start = (
|
||||
extend_lens_cpu[0] - tail_lens_cpu[0] if len(extend_lens_cpu) == 1 else None
|
||||
|
||||
@@ -2649,6 +2649,8 @@ class ScheduleBatch(ScheduleBatchDisaggregationDecodeMixin):
|
||||
# For DP attention
|
||||
global_num_tokens: Optional[List[int]] = None
|
||||
global_num_tokens_for_logprob: Optional[List[int]] = None
|
||||
# Per DP rank, the rows the decoder SWA tail drops; None when no rank trims.
|
||||
global_decoder_trim_rows: Optional[List[int]] = None
|
||||
# The draft model can use a different MoE A2A backend than the target.
|
||||
draft_global_num_tokens: Optional[List[int]] = None
|
||||
draft_global_num_tokens_for_logprob: Optional[List[int]] = None
|
||||
|
||||
@@ -32,8 +32,15 @@ from sglang.srt.model_executor.cuda_graph_config import (
|
||||
check_cuda_graph_backend,
|
||||
cuda_graph_fully_disabled,
|
||||
)
|
||||
from sglang.srt.model_executor.encoder_swa_replay import folded_replay_rows
|
||||
from sglang.srt.model_executor.forward_batch_info import ForwardMode
|
||||
from sglang.srt.model_executor.encoder_swa_replay import (
|
||||
decoder_swa_trim_rows,
|
||||
folded_replay_rows,
|
||||
)
|
||||
from sglang.srt.model_executor.forward_batch_info import (
|
||||
CaptureHiddenMode,
|
||||
ForwardMode,
|
||||
get_server_return_hidden_states_mode,
|
||||
)
|
||||
from sglang.srt.model_executor.runner import PrefillCudaGraphRunner
|
||||
from sglang.srt.observability.metrics_collector import (
|
||||
DPBalanceStats,
|
||||
@@ -161,6 +168,8 @@ class MLPSyncBatchInfo:
|
||||
is_health_check: bool = False
|
||||
# Causal (query, key) pairs this rank attends this step.
|
||||
attention_pairs: int = 0
|
||||
# Rows the decoder SWA tail drops before the late layers.
|
||||
decoder_trim_rows: int = 0
|
||||
|
||||
# some gathered elements
|
||||
tp0_info_cpu: torch.Tensor = None
|
||||
@@ -169,6 +178,7 @@ class MLPSyncBatchInfo:
|
||||
global_num_tokens: list[int] = None
|
||||
global_num_tokens_for_logprob: list[int] = None
|
||||
global_attention_pairs: list[int] = None
|
||||
global_decoder_trim_rows: list[int] = None
|
||||
tbo_split_seq_index: torch.Tensor = None
|
||||
global_forward_mode: int = None
|
||||
dp_cooperation_info: Optional[DPCooperationInfo] = None
|
||||
@@ -187,6 +197,7 @@ class MLPSyncBatchInfo:
|
||||
int(self.can_run_draft_cuda_graph),
|
||||
int(self.is_health_check),
|
||||
self.attention_pairs,
|
||||
self.decoder_trim_rows,
|
||||
],
|
||||
device=device,
|
||||
dtype=dtype,
|
||||
@@ -206,6 +217,7 @@ class MLPSyncBatchInfo:
|
||||
1, # can_run_draft_cuda_graph
|
||||
0, # is_health_check
|
||||
0, # attention_pairs
|
||||
0, # decoder_trim_rows
|
||||
],
|
||||
device=device,
|
||||
dtype=dtype,
|
||||
@@ -217,6 +229,7 @@ class MLPSyncBatchInfo:
|
||||
self.global_num_tokens = [self.num_tokens]
|
||||
self.global_num_tokens_for_logprob = [self.num_tokens_for_logprob]
|
||||
self.global_attention_pairs = [self.attention_pairs]
|
||||
self.global_decoder_trim_rows = [self.decoder_trim_rows]
|
||||
self.any_health_check = self.is_health_check
|
||||
if _ENABLE_METRICS_DP_ATTENTION:
|
||||
self.dp_cooperation_info = DPCooperationInfo.create(
|
||||
@@ -294,6 +307,7 @@ class MLPSyncBatchInfo:
|
||||
self.can_run_draft_cuda_graph = bool(tp0_info_cpu[:, 8].min())
|
||||
self.any_health_check = bool(tp0_info_cpu[:, 9].max())
|
||||
self.global_attention_pairs = tp0_info_cpu[:, 10].tolist()
|
||||
self.global_decoder_trim_rows = tp0_info_cpu[:, 11].tolist()
|
||||
self.sync_wait_seconds = time.perf_counter() - sync_start
|
||||
if _ENABLE_METRICS_DP_ATTENTION:
|
||||
self.dp_cooperation_info = DPCooperationInfo.create(
|
||||
@@ -308,6 +322,7 @@ def _update_gather_batch(
|
||||
draft_require_mlp_tp_gather: Optional[bool] = None,
|
||||
skip_global_metadata=False,
|
||||
):
|
||||
batch.global_decoder_trim_rows = None
|
||||
if not require_mlp_tp_gather:
|
||||
batch.global_num_tokens = [mlp_sync_info.num_tokens]
|
||||
batch.global_num_tokens_for_logprob = [mlp_sync_info.num_tokens_for_logprob]
|
||||
@@ -316,6 +331,12 @@ def _update_gather_batch(
|
||||
batch.global_num_tokens_for_logprob = (
|
||||
mlp_sync_info.global_num_tokens_for_logprob
|
||||
)
|
||||
if (
|
||||
any(mlp_sync_info.global_decoder_trim_rows or ())
|
||||
and min(mlp_sync_info.global_decoder_trim_rows) >= 0
|
||||
):
|
||||
# Every rank resizes its late-layer DP gather when any rank trims.
|
||||
batch.global_decoder_trim_rows = mlp_sync_info.global_decoder_trim_rows
|
||||
# Reuse the same all-gather result for a draft model whose A2A backend
|
||||
# requires a different local/full token-count representation.
|
||||
if draft_require_mlp_tp_gather is not None:
|
||||
@@ -532,6 +553,7 @@ def prepare_mlp_sync_batch_raw(
|
||||
attn_tp_size = parallel.attn_tp_size
|
||||
tp_group = parallel.tp_group
|
||||
# Check if other DP workers have running batches
|
||||
decoder_trim_rows = 0
|
||||
if (
|
||||
local_batch is None
|
||||
or local_batch.forward_mode.is_prebuilt()
|
||||
@@ -549,6 +571,15 @@ def prepare_mlp_sync_batch_raw(
|
||||
and model_runner.attn_backend.folds_encoder_swa_replay
|
||||
):
|
||||
num_tokens += folded_replay_rows(local_batch)
|
||||
if get_exec().features.enable_decoder_swa_bounded_replay:
|
||||
decoder_trim_rows = (
|
||||
-1
|
||||
if get_server_return_hidden_states_mode() == CaptureHiddenMode.FULL
|
||||
else decoder_swa_trim_rows(
|
||||
local_batch,
|
||||
folds_encoder=model_runner.attn_backend.folds_encoder_swa_replay,
|
||||
)
|
||||
)
|
||||
num_tokens_for_logprob = sum(
|
||||
# We should have at least 1 token for sample in every case.
|
||||
max(extend_len - logprob_start_len, 1)
|
||||
@@ -641,6 +672,7 @@ def prepare_mlp_sync_batch_raw(
|
||||
# Gathered unconditionally like the other columns, so a per-process
|
||||
# metrics flag cannot make ranks disagree on what the tensor holds.
|
||||
attention_pairs=_local_attention_pairs(local_batch),
|
||||
decoder_trim_rows=decoder_trim_rows,
|
||||
)
|
||||
|
||||
if num_dp_ranks == 1:
|
||||
|
||||
@@ -703,6 +703,7 @@ class BasePrefixCache(ABC, PrefixCacheTrait):
|
||||
supports_prefix_sharing=self.supports_prefix_sharing(),
|
||||
retain_floor=self.swa_retain_floor(req),
|
||||
eviction_interval=eviction_interval,
|
||||
reprefill_tail_tokens=self.swa_reprefill_tail_tokens(req),
|
||||
)
|
||||
|
||||
def swa_retain_floor(self, req) -> int | None:
|
||||
|
||||
@@ -84,6 +84,7 @@ def free_swa_out_of_window_slots(
|
||||
component_type: ComponentType = ComponentType.SWA,
|
||||
free_segment: Callable[..., None] | None = None,
|
||||
eviction_interval: int = 1,
|
||||
reprefill_tail_tokens: int = 0,
|
||||
) -> None:
|
||||
if not req.kv.holds_kv:
|
||||
return
|
||||
@@ -115,6 +116,14 @@ def free_swa_out_of_window_slots(
|
||||
evict_threshold = pre_len - ceil_align(
|
||||
max(sliding_window_size, page_size), page_size
|
||||
)
|
||||
if reprefill_tail_tokens:
|
||||
# Matching excludes the replay tail before page alignment. Keep the
|
||||
# SWA window behind that earlier boundary too, or a first replay
|
||||
# falls back to an older checkpoint despite a Full-KV cache hit.
|
||||
replay_boundary = (pre_len - reprefill_tail_tokens) // page_size * page_size
|
||||
evict_threshold = min(
|
||||
evict_threshold, replay_boundary - sliding_window_size
|
||||
)
|
||||
if retain_floor is not None and supports_prefix_sharing:
|
||||
# The caller owns where the floor is (see BasePrefixCache.swa_retain_floor);
|
||||
# this only promises not to free past it. Without prefix sharing a retained
|
||||
|
||||
@@ -1046,6 +1046,7 @@ class SWAComponent(TreeComponent):
|
||||
req_to_token_pool=self.cache.req_to_token_pool,
|
||||
token_to_kv_pool_allocator=self.cache.token_to_kv_pool_allocator,
|
||||
retain_floor=self.cache.swa_retain_floor(req),
|
||||
reprefill_tail_tokens=self.cache.swa_reprefill_tail_tokens(req),
|
||||
)
|
||||
|
||||
def free_out_of_window_slots(
|
||||
|
||||
@@ -52,6 +52,23 @@ def folded_replay_rows(batch) -> int:
|
||||
)
|
||||
|
||||
|
||||
def decoder_swa_trim_rows(batch, *, folds_encoder: bool) -> int:
|
||||
"""Rows removed by a plain extend's decoder tail; -1 requests the backend
|
||||
exchange when prompt outputs require whole-extend rows."""
|
||||
if not batch.forward_mode.is_extend_without_speculative():
|
||||
return 0
|
||||
if batch.return_logprob or batch.return_hidden_states:
|
||||
return -1
|
||||
lens = list(batch.extend_lens)
|
||||
if folds_encoder and batch.encoder_swa_reset is not None:
|
||||
for i, (reset, prefix) in enumerate(
|
||||
zip(batch.encoder_swa_reset, batch.prefix_lens)
|
||||
):
|
||||
if reset:
|
||||
lens[i] += replay_len(prefix)
|
||||
return sum(max(0, n - 128) for n in lens)
|
||||
|
||||
|
||||
def apply_folded_extend(folded: FoldedExtend, forward_batch) -> None:
|
||||
forward_batch.encoder_swa_row_floor = folded.row_floor
|
||||
forward_batch.encoder_swa_compress_rows = folded.keep_rows
|
||||
|
||||
@@ -51,6 +51,7 @@ from sglang.srt.layers.dp_attention import (
|
||||
set_is_extend_in_batch,
|
||||
world_dp_gather_enabled,
|
||||
)
|
||||
from sglang.srt.model_executor.cuda_graph_config import Backend
|
||||
from sglang.srt.model_executor.forward_batch_deepseek_mha_mixin import (
|
||||
ForwardBatchDeepSeekMHAMixin,
|
||||
)
|
||||
@@ -59,6 +60,7 @@ from sglang.srt.runtime_context import (
|
||||
get_flags,
|
||||
get_lora,
|
||||
get_parallel,
|
||||
get_platform,
|
||||
mamba_cache_chunk_size,
|
||||
)
|
||||
from sglang.srt.speculative.spec_info import SpecInputType
|
||||
@@ -78,7 +80,7 @@ if TYPE_CHECKING:
|
||||
from sglang.srt.layers.engram_staging import StagedRows
|
||||
from sglang.srt.layers.layer_boundary.residual.stream import ResidualStream
|
||||
from sglang.srt.layers.logits_processor import LogitsProcessorOutput
|
||||
from sglang.srt.managers.schedule_batch import MultimodalInputs, ScheduleBatch
|
||||
from sglang.srt.managers.schedule_batch import MultimodalInputs, Req, ScheduleBatch
|
||||
from sglang.srt.mem_cache.kv_loc_plan import Cols, KVLocPlan
|
||||
from sglang.srt.model_executor.model_runner import ModelRunner
|
||||
from sglang.srt.sampling.sampling_batch_info import SamplingBatchInfo
|
||||
@@ -489,6 +491,17 @@ class NgramEmbeddingInfo:
|
||||
)
|
||||
|
||||
|
||||
def prompt_rest_lens_of(
|
||||
reqs: List[Req], seq_lens: List[int], decoding_req_ids: Set[int]
|
||||
) -> List[int]:
|
||||
"""Prompt tokens each request prefills after this extend, from the batch's own
|
||||
ends (an encoder replay batch ends at its prefix); a decode row has none."""
|
||||
return [
|
||||
0 if id(req) in decoding_req_ids else len(req.full_untruncated_fill_ids) - end
|
||||
for req, end in zip(reqs, seq_lens, strict=True)
|
||||
]
|
||||
|
||||
|
||||
@dataclass
|
||||
class ForwardBatch(ForwardBatchDeepSeekMHAMixin):
|
||||
"""Store all inputs of a forward pass."""
|
||||
@@ -669,6 +682,8 @@ class ForwardBatch(ForwardBatchDeepSeekMHAMixin):
|
||||
extend_prefix_lens_cpu: Optional[List[int]] = None
|
||||
extend_seq_lens_cpu: Optional[List[int]] = None
|
||||
extend_logprob_start_lens_cpu: Optional[List[int]] = None
|
||||
# Prompt tokens each request prefills after this extend (decoder SWA tail rows).
|
||||
extend_prompt_rest_lens_cpu: Optional[List[int]] = None
|
||||
extend_input_logprob_token_ids_gpu: Optional[torch.Tensor] = None
|
||||
|
||||
# For DP attention (MLP sync sizes)
|
||||
@@ -681,6 +696,9 @@ class ForwardBatch(ForwardBatchDeepSeekMHAMixin):
|
||||
# Has to be None when cuda graph is captured.
|
||||
global_num_tokens_for_logprob_cpu: Optional[List[int]] = None
|
||||
global_num_tokens_for_logprob_gpu: Optional[torch.Tensor] = None
|
||||
# Per DP rank, the rows the decoder SWA tail drops before the late layers;
|
||||
# None when no rank trims (see DeepseekV4Model's late-layer DP switch).
|
||||
global_decoder_trim_rows_cpu: Optional[List[int]] = None
|
||||
|
||||
# Real (non-padding) token count, held at two scopes whose meaning never
|
||||
# changes once set:
|
||||
@@ -1015,6 +1033,7 @@ class ForwardBatch(ForwardBatchDeepSeekMHAMixin):
|
||||
batch.seq_lens_sum = int(seq_lens_cpu.sum())
|
||||
|
||||
contains_last_prefill_chunk = False
|
||||
prompt_rest_lens = None
|
||||
if batch.forward_mode.is_extend_without_speculative():
|
||||
decoding_req_ids = {id(req) for req in (batch.decoding_reqs or ())}
|
||||
contains_last_prefill_chunk = any(
|
||||
@@ -1022,6 +1041,14 @@ class ForwardBatch(ForwardBatchDeepSeekMHAMixin):
|
||||
for req in batch.reqs
|
||||
if id(req) not in decoding_req_ids
|
||||
)
|
||||
if (
|
||||
envs.SGLANG_OPT_DSV41_PROMPT_WINDOW_TAIL.get()
|
||||
and get_parallel().attn_cp_size == 1
|
||||
and seq_lens_cpu is not None
|
||||
):
|
||||
prompt_rest_lens = prompt_rest_lens_of(
|
||||
batch.reqs, seq_lens_cpu.tolist(), decoding_req_ids
|
||||
)
|
||||
|
||||
from sglang.srt.model_executor.model_runner import ModelRunner
|
||||
|
||||
@@ -1082,6 +1109,7 @@ class ForwardBatch(ForwardBatchDeepSeekMHAMixin):
|
||||
global_forward_mode=batch.global_forward_mode,
|
||||
is_prefill_only=batch.is_prefill_only,
|
||||
contains_last_prefill_chunk=contains_last_prefill_chunk,
|
||||
extend_prompt_rest_lens_cpu=prompt_rest_lens,
|
||||
spec_algorithm=batch.spec_algorithm,
|
||||
capture_hidden_mode=capture_hidden_mode,
|
||||
return_hidden_states_before_norm=return_hidden_states_before_norm,
|
||||
@@ -1166,6 +1194,8 @@ class ForwardBatch(ForwardBatchDeepSeekMHAMixin):
|
||||
ret.init_mlp_sync_metadata(
|
||||
batch, device, is_draft_worker=model_runner.is_draft_worker
|
||||
)
|
||||
if not model_runner.is_draft_worker:
|
||||
ret.global_decoder_trim_rows_cpu = batch.global_decoder_trim_rows
|
||||
|
||||
if ret.forward_mode.is_idle():
|
||||
ret.positions = torch.empty((0,), dtype=torch.int64, device=device)
|
||||
@@ -1945,13 +1975,29 @@ class ForwardBatch(ForwardBatchDeepSeekMHAMixin):
|
||||
):
|
||||
local_late_rows = model_runner.attn_backend.late_layer_num_tokens(self)
|
||||
if local_late_rows is not None:
|
||||
late_rows = torch.empty(sync_group_size, dtype=torch.int64)
|
||||
all_gather_single(
|
||||
late_rows,
|
||||
torch.tensor([local_late_rows], dtype=torch.int64),
|
||||
group=get_parallel().tp_group.cpu_group,
|
||||
)
|
||||
self.global_late_layer_num_tokens_cpu = late_rows.tolist()
|
||||
trims = self.global_decoder_trim_rows_cpu
|
||||
if (
|
||||
trims is not None
|
||||
and get_platform().is_cuda
|
||||
and get_parallel().attn_tp_size == 1
|
||||
and get_exec().graph.cuda_graph_config.prefill.backend
|
||||
== Backend.DISABLED
|
||||
and not self.dp_padding_mode.is_max_len()
|
||||
):
|
||||
counts = [
|
||||
n - t for n, t in zip(global_num_tokens, trims, strict=True)
|
||||
]
|
||||
assert min(counts) >= 0, (global_num_tokens, trims)
|
||||
assert counts[dp_slot_in(counts)] == local_late_rows
|
||||
self.global_late_layer_num_tokens_cpu = counts
|
||||
else:
|
||||
late_rows = torch.empty(sync_group_size, dtype=torch.int64)
|
||||
all_gather_single(
|
||||
late_rows,
|
||||
torch.tensor([local_late_rows], dtype=torch.int64),
|
||||
group=get_parallel().tp_group.cpu_group,
|
||||
)
|
||||
self.global_late_layer_num_tokens_cpu = late_rows.tolist()
|
||||
|
||||
TboForwardBatchPreparer.prepare(
|
||||
batch=self, is_draft_worker=model_runner.is_draft_worker
|
||||
@@ -2074,6 +2120,10 @@ class ForwardBatch(ForwardBatchDeepSeekMHAMixin):
|
||||
self.extend_logprob_start_lens_cpu.extend(
|
||||
[0] * (bs - len(self.extend_logprob_start_lens_cpu))
|
||||
)
|
||||
if self.extend_prompt_rest_lens_cpu is not None:
|
||||
self.extend_prompt_rest_lens_cpu.extend(
|
||||
[0] * (bs - len(self.extend_prompt_rest_lens_cpu))
|
||||
)
|
||||
|
||||
if self.rids_int is not None:
|
||||
self.rids_int = self._pad_tensor_to_size(self.rids_int, bs)
|
||||
|
||||
@@ -0,0 +1,181 @@
|
||||
"""Decoder SWA bounded replay with SGLANG_OPT_DSV41_PROMPT_WINDOW_TAIL: a prefill
|
||||
chunk runs the late layers over its rows in the prompt's last window only, so any
|
||||
chunking of a prompt computes exactly the rows and floors of the unchunked prompt
|
||||
there, plus one placeholder row per chunk outside it. CPU only."""
|
||||
|
||||
import unittest
|
||||
from types import SimpleNamespace
|
||||
|
||||
import torch
|
||||
|
||||
from sglang.kernels.ops.attention.dsv4.attn_metadata_kernels import (
|
||||
late_layer_tail_layout,
|
||||
late_layer_tail_lens,
|
||||
)
|
||||
from sglang.srt.layers.attention.deepseek_v4_backend import (
|
||||
SWA_WINDOW,
|
||||
late_layer_num_tokens,
|
||||
)
|
||||
from sglang.srt.model_executor.forward_batch_info import (
|
||||
CaptureHiddenMode,
|
||||
ForwardMode,
|
||||
prompt_rest_lens_of,
|
||||
)
|
||||
from sglang.test.ci.ci_register import register_cpu_ci
|
||||
from sglang.test.test_utils import CustomTestCase
|
||||
|
||||
register_cpu_ci(est_time=5, suite="base-a-test-cpu")
|
||||
|
||||
CPU = torch.device("cpu")
|
||||
|
||||
|
||||
def _layout(extend_lens, seq_lens, rest=None, whole_extend=None):
|
||||
return late_layer_tail_layout(
|
||||
extend_lens_cpu=extend_lens,
|
||||
seq_lens_cpu=seq_lens,
|
||||
tail_len=SWA_WINDOW,
|
||||
device=CPU,
|
||||
whole_extend=whole_extend,
|
||||
prompt_rest_lens=rest,
|
||||
)
|
||||
|
||||
|
||||
def _chunked_rows(prompt_len, chunks):
|
||||
"""(absolute position, floor) of every tail row over a prompt's chunks, each
|
||||
chunk one single-request step."""
|
||||
rows, start = [], 0
|
||||
for n in chunks:
|
||||
end = start + n
|
||||
token_indices, _, floor = _layout([n], [end], rest=[prompt_len - end])
|
||||
rows += list(zip((start + token_indices).tolist(), floor.tolist()))
|
||||
start = end
|
||||
assert start == prompt_len
|
||||
return rows
|
||||
|
||||
|
||||
class TestTailLens(CustomTestCase):
|
||||
def test_final_chunks_keep_the_window_tail(self):
|
||||
extend_lens = [300, 50, 128]
|
||||
self.assertEqual(
|
||||
late_layer_tail_lens(extend_lens, SWA_WINDOW, prompt_rest_lens=[0, 0, 0]),
|
||||
late_layer_tail_lens(extend_lens, SWA_WINDOW),
|
||||
)
|
||||
|
||||
def test_earlier_chunks_keep_their_rows_in_the_prompt_window(self):
|
||||
# Far from the end: one row. Straddling the window: the rows inside it.
|
||||
# Inside the window: the whole chunk.
|
||||
self.assertEqual(
|
||||
late_layer_tail_lens(
|
||||
[32768, 300, 40, 300],
|
||||
SWA_WINDOW,
|
||||
prompt_rest_lens=[5000, 50, 20, SWA_WINDOW],
|
||||
),
|
||||
[1, SWA_WINDOW - 50, 40, 1],
|
||||
)
|
||||
|
||||
def test_whole_extend_wins(self):
|
||||
self.assertEqual(
|
||||
late_layer_tail_lens(
|
||||
[300, 300], SWA_WINDOW, [True, False], prompt_rest_lens=[5000, 5000]
|
||||
),
|
||||
[300, SWA_WINDOW],
|
||||
)
|
||||
# No whole request: the prompt window applies, as in the layout.
|
||||
self.assertEqual(
|
||||
late_layer_tail_lens([300], SWA_WINDOW, [False], prompt_rest_lens=[5000]),
|
||||
[1],
|
||||
)
|
||||
|
||||
|
||||
class TestTailLayout(CustomTestCase):
|
||||
def test_final_chunks_match_the_window_layout(self):
|
||||
# Negative control: final chunks (nothing left) change no row or floor.
|
||||
for extend_lens, seq_lens in (([300], [900]), ([300, 50, 40], [900, 50, 400])):
|
||||
with self.subTest(extend_lens=extend_lens):
|
||||
got = _layout(extend_lens, seq_lens, rest=[0] * len(extend_lens))
|
||||
want = _layout(extend_lens, seq_lens)
|
||||
torch.testing.assert_close(got[0], want[0])
|
||||
self.assertEqual(got[1], want[1])
|
||||
torch.testing.assert_close(got[2], want[2])
|
||||
|
||||
def test_placeholder_row_reads_only_itself(self):
|
||||
token_indices, tail_lens, floor = _layout([32768], [65536], rest=[70000])
|
||||
self.assertEqual(tail_lens, [1])
|
||||
self.assertEqual(token_indices.tolist(), [32767])
|
||||
self.assertEqual(floor.tolist(), [65535])
|
||||
|
||||
def test_batched_layout_matches_single_requests(self):
|
||||
extend_lens = [32768, 300, 40, 200]
|
||||
seq_lens = [40000, 900, 60, 200]
|
||||
rest = [9000, 50, 20, 0]
|
||||
token_indices, tail_lens, floor = _layout(extend_lens, seq_lens, rest)
|
||||
want_indices, want_floor, start = [], [], 0
|
||||
for n, s, r in zip(extend_lens, seq_lens, rest):
|
||||
idx, _, fl = _layout([n], [s], rest=[r])
|
||||
want_indices.append(start + idx)
|
||||
want_floor.append(fl)
|
||||
start += n
|
||||
self.assertEqual(tail_lens, [1, SWA_WINDOW - 50, 40, SWA_WINDOW])
|
||||
torch.testing.assert_close(token_indices, torch.cat(want_indices))
|
||||
torch.testing.assert_close(floor, torch.cat(want_floor))
|
||||
|
||||
|
||||
class TestChunkingInvariance(CustomTestCase):
|
||||
"""The rows in the prompt's last window, and the floor each reads from, do
|
||||
not depend on how the prompt was chunked."""
|
||||
|
||||
def test_any_chunking_computes_the_unchunked_window(self):
|
||||
prompt_len = 1000
|
||||
whole = _chunked_rows(prompt_len, [prompt_len])
|
||||
floor = prompt_len - SWA_WINDOW
|
||||
self.assertEqual(whole, [(p, floor) for p in range(floor, prompt_len)])
|
||||
for chunks in (
|
||||
[400, 400, 200],
|
||||
[900, 60, 40],
|
||||
[800, 100, 60, 40],
|
||||
[872, 1, 127],
|
||||
[700, 200, 100],
|
||||
):
|
||||
with self.subTest(chunks=chunks):
|
||||
rows = _chunked_rows(prompt_len, chunks)
|
||||
window = [(p, f) for p, f in rows if p >= floor]
|
||||
self.assertEqual(window, whole)
|
||||
# Placeholders sit below the window and read only their own row.
|
||||
for p, f in rows:
|
||||
if p < floor:
|
||||
self.assertEqual(f, p)
|
||||
|
||||
def test_window_rows_never_read_below_the_window(self):
|
||||
rows = _chunked_rows(1000, [900, 60, 40])
|
||||
self.assertTrue(all(f == 1000 - SWA_WINDOW for p, f in rows if p >= 872))
|
||||
|
||||
|
||||
class TestPromptRestLens(CustomTestCase):
|
||||
def test_rest_from_the_batch_ends(self):
|
||||
prompt = SimpleNamespace(full_untruncated_fill_ids=list(range(1000)))
|
||||
decode = SimpleNamespace(full_untruncated_fill_ids=list(range(70)))
|
||||
# A chunk ending at 400, a replay batch ending at its prefix 128, and a
|
||||
# decode row carried in the extend.
|
||||
self.assertEqual(
|
||||
prompt_rest_lens_of([prompt, prompt, decode], [400, 128, 70], {id(decode)}),
|
||||
[600, 872, 0],
|
||||
)
|
||||
|
||||
def test_rows_counted_for_dp_match_the_layout(self):
|
||||
extend_lens, rest = [32768, 300, 40], [9000, 50, 20]
|
||||
fb = SimpleNamespace(
|
||||
forward_mode=ForwardMode.EXTEND,
|
||||
extend_seq_lens_cpu=extend_lens,
|
||||
extend_logprob_start_lens_cpu=None,
|
||||
return_logprob=False,
|
||||
capture_hidden_mode=CaptureHiddenMode.FULL,
|
||||
extend_prompt_rest_lens_cpu=rest,
|
||||
)
|
||||
_, tail_lens, _ = _layout(extend_lens, [40000, 900, 60], rest)
|
||||
self.assertEqual(late_layer_num_tokens(fb, dspark=True), sum(tail_lens))
|
||||
fb.extend_prompt_rest_lens_cpu = None
|
||||
self.assertEqual(late_layer_num_tokens(fb, dspark=True), 2 * SWA_WINDOW + 40)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,128 @@
|
||||
"""Exercise SWA eviction frontiers without accelerator execution or model weights."""
|
||||
|
||||
import ast
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import Mock
|
||||
|
||||
from sglang.srt.mem_cache import common as cache_common
|
||||
from sglang.test.ci.ci_register import register_cpu_ci
|
||||
from sglang.test.test_utils import CustomTestCase
|
||||
|
||||
register_cpu_ci(est_time=5, suite="base-a-test-cpu")
|
||||
|
||||
|
||||
class ReplayRetentionTest(CustomTestCase):
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
cls.source = Path(cache_common.__file__).parent
|
||||
cls.free_window = staticmethod(cache_common.free_swa_out_of_window_slots)
|
||||
|
||||
def frontier(
|
||||
self,
|
||||
length,
|
||||
window=128,
|
||||
page=256,
|
||||
replay=128,
|
||||
sharing=True,
|
||||
retain_floor=None,
|
||||
previous=0,
|
||||
):
|
||||
kv = SimpleNamespace(
|
||||
holds_kv=True,
|
||||
cache_protected_len=0,
|
||||
req_pool_idx=0,
|
||||
swa_dead_lo=lambda _: 0,
|
||||
get_evicted_seqlen=lambda _: previous,
|
||||
)
|
||||
kv.set_evicted_seqlen = lambda _, value: setattr(kv, "evicted", value)
|
||||
table = Mock()
|
||||
table.__getitem__ = Mock(return_value=object())
|
||||
allocator = SimpleNamespace(free_swa_segment=Mock())
|
||||
self.free_window(
|
||||
SimpleNamespace(kv=kv),
|
||||
length - 1,
|
||||
sliding_window_size=window,
|
||||
page_size=page,
|
||||
req_to_token_pool=SimpleNamespace(req_to_token=table),
|
||||
token_to_kv_pool_allocator=allocator,
|
||||
supports_prefix_sharing=sharing,
|
||||
retain_floor=retain_floor,
|
||||
reprefill_tail_tokens=replay,
|
||||
)
|
||||
return kv.evicted
|
||||
|
||||
def test_first_replay_keeps_window_before_trimmed_match(self):
|
||||
for length in (32576, 32448, 269913, 1029986):
|
||||
with self.subTest(length=length):
|
||||
boundary = (length - 128) // 256 * 256
|
||||
self.assertLessEqual(self.frontier(length), boundary - 128)
|
||||
|
||||
def test_all_page_alignments_preserve_replay_history(self):
|
||||
for page, window, replay in (
|
||||
(256, 128, 128),
|
||||
(64, 128, 128),
|
||||
(16, 8, 8),
|
||||
(1, 4, 4),
|
||||
):
|
||||
for remainder in range(page):
|
||||
length = 64 * page + 2 * window + remainder
|
||||
with self.subTest(page=page, window=window, remainder=remainder):
|
||||
boundary = (length - replay) // page * page
|
||||
self.assertLessEqual(
|
||||
self.frontier(length, window, page, replay), boundary - window
|
||||
)
|
||||
|
||||
def test_no_replay_keeps_existing_eviction_policy(self):
|
||||
for length in range(32000, 32512):
|
||||
expected = (length - 1 - 256) // 256 * 256
|
||||
self.assertEqual(self.frontier(length, replay=0), expected)
|
||||
|
||||
def test_256_token_pages_retain_at_most_one_extra_page(self):
|
||||
for length in range(32000, 32512):
|
||||
before = self.frontier(length, replay=0)
|
||||
after = self.frontier(length)
|
||||
self.assertGreaterEqual(before - after, 0)
|
||||
self.assertLessEqual(before - after, 256)
|
||||
|
||||
def test_bad_alignment_exposes_original_frontier_mismatch(self):
|
||||
self.assertEqual(self.frontier(32576, replay=0), 32256)
|
||||
self.assertEqual(self.frontier(32576), 32000)
|
||||
self.assertEqual(self.frontier(32448), 32000)
|
||||
|
||||
def test_disabled_prefix_sharing_does_not_retain_replay_history(self):
|
||||
for replay in (0, 128):
|
||||
self.assertEqual(self.frontier(32576, replay=replay, sharing=False), 32256)
|
||||
|
||||
def test_checkpoint_floor_and_eviction_cursor_still_apply(self):
|
||||
self.assertEqual(self.frontier(32576, retain_floor=30000), 29952)
|
||||
# Previously freed slots cannot be recovered by changing a frontier.
|
||||
self.assertEqual(self.frontier(32576, previous=32256), 32256)
|
||||
|
||||
def test_scheduler_and_checkpoint_paths_pass_the_replay_guard(self):
|
||||
for relative, expected in (
|
||||
("base_prefix_cache.py", "self.swa_reprefill_tail_tokens(req)"),
|
||||
(
|
||||
"unified_cache/components/swa.py",
|
||||
"self.cache.swa_reprefill_tail_tokens(req)",
|
||||
),
|
||||
):
|
||||
tree = ast.parse((self.source / relative).read_text())
|
||||
calls = [
|
||||
node
|
||||
for node in ast.walk(tree)
|
||||
if isinstance(node, ast.Call)
|
||||
and ast.unparse(node.func) == "free_swa_out_of_window_slots"
|
||||
]
|
||||
self.assertEqual(len(calls), 1)
|
||||
keyword = next(
|
||||
(kw for kw in calls[0].keywords if kw.arg == "reprefill_tail_tokens"),
|
||||
None,
|
||||
)
|
||||
self.assertIsNotNone(keyword, relative)
|
||||
self.assertEqual(ast.unparse(keyword.value), expected)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,181 @@
|
||||
"""Decoder tail sizes reuse the scheduler exchange on eager CUDA DP steps."""
|
||||
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
from sglang.srt.managers.scheduler_components import dp_attn
|
||||
from sglang.srt.managers.scheduler_components.dp_attn import _update_gather_batch
|
||||
from sglang.srt.model_executor import forward_batch_info as fbi
|
||||
from sglang.srt.model_executor.cuda_graph_config import Backend
|
||||
from sglang.srt.model_executor.encoder_swa_replay import decoder_swa_trim_rows
|
||||
from sglang.srt.model_executor.forward_batch_info import ForwardBatch, ForwardMode
|
||||
from sglang.test.ci.ci_register import register_cpu_ci
|
||||
|
||||
register_cpu_ci(est_time=5, suite="base-a-test-cpu")
|
||||
|
||||
|
||||
@pytest.mark.parametrize("folds,expected", [(False, 3968), (True, 3998)])
|
||||
def test_trim_includes_folded_encoder_rows(folds, expected):
|
||||
batch = SimpleNamespace(
|
||||
forward_mode=ForwardMode.EXTEND,
|
||||
extend_lens=[4096, 30, 40],
|
||||
prefix_lens=[0, 512, 64],
|
||||
encoder_swa_reset=[True, True, True],
|
||||
return_logprob=False,
|
||||
return_hidden_states=False,
|
||||
)
|
||||
assert decoder_swa_trim_rows(batch, folds_encoder=folds) == expected
|
||||
batch.return_logprob = True
|
||||
assert decoder_swa_trim_rows(batch, folds_encoder=folds) == -1
|
||||
batch.return_logprob = False
|
||||
batch.return_hidden_states = True
|
||||
assert decoder_swa_trim_rows(batch, folds_encoder=folds) == -1
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"mode", [ForwardMode.DECODE, ForwardMode.IDLE, ForwardMode.TARGET_VERIFY]
|
||||
)
|
||||
def test_non_extend_peers_do_not_trim(mode):
|
||||
assert (
|
||||
decoder_swa_trim_rows(SimpleNamespace(forward_mode=mode), folds_encoder=True)
|
||||
== 0
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"trims,expected",
|
||||
[
|
||||
([3968, 30, 0, 0], [3968, 30, 0, 0]),
|
||||
([0, 0, 0, 0], None),
|
||||
([3968, -1, 0, 0], None),
|
||||
],
|
||||
)
|
||||
def test_whole_output_peer_falls_back_on_every_rank(trims, expected):
|
||||
info = MagicMock()
|
||||
info.global_decoder_trim_rows = trims
|
||||
batch = SimpleNamespace()
|
||||
_update_gather_batch(batch, info, True, skip_global_metadata=True)
|
||||
assert batch.global_decoder_trim_rows == expected
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"rank,mode",
|
||||
[
|
||||
(0, ForwardMode.EXTEND),
|
||||
(1, ForwardMode.EXTEND),
|
||||
(2, ForwardMode.IDLE),
|
||||
(3, ForwardMode.DECODE),
|
||||
],
|
||||
)
|
||||
@pytest.mark.parametrize("fallback", [None, "graph", "hip", "whole_outputs"])
|
||||
def test_preparation_uses_scheduler_counts_or_preserves_backend_exchange(
|
||||
rank, mode, fallback
|
||||
):
|
||||
rows, trims, late = [4096, 282, 0, 130], [3968, 30, 0, 0], [128, 252, 0, 130]
|
||||
n = rows[rank]
|
||||
batch = ForwardBatch(
|
||||
forward_mode=mode,
|
||||
batch_size=1 if n else 0,
|
||||
input_ids=torch.zeros(n, dtype=torch.long),
|
||||
req_pool_indices=torch.zeros(1 if n else 0, dtype=torch.long),
|
||||
seq_lens=torch.tensor([n] if n else []),
|
||||
out_cache_loc=torch.arange(n),
|
||||
seq_lens_sum=n,
|
||||
positions=torch.arange(n),
|
||||
is_extend_in_batch=True,
|
||||
global_num_tokens_cpu=rows,
|
||||
global_num_tokens_for_logprob_cpu=rows,
|
||||
global_num_tokens_gpu=torch.zeros(4, dtype=torch.int64),
|
||||
global_decoder_trim_rows_cpu=None if fallback == "whole_outputs" else trims,
|
||||
)
|
||||
runner = MagicMock(is_draft_worker=False)
|
||||
runner.attn_tp_sequence_sharded.return_value = False
|
||||
runner.attn_backend.late_layer_num_tokens.return_value = late[rank]
|
||||
cfg = SimpleNamespace(
|
||||
graph=SimpleNamespace(
|
||||
cuda_graph_config=SimpleNamespace(
|
||||
prefill=SimpleNamespace(
|
||||
bs=[],
|
||||
backend=Backend.BREAKABLE
|
||||
if fallback == "graph"
|
||||
else Backend.DISABLED,
|
||||
)
|
||||
)
|
||||
)
|
||||
)
|
||||
|
||||
def exchange(output, local, **kwargs):
|
||||
assert local.item() == late[rank]
|
||||
output.copy_(torch.tensor(late))
|
||||
|
||||
with (
|
||||
patch.multiple(
|
||||
fbi,
|
||||
get_parallel=lambda: SimpleNamespace(
|
||||
attn_tp_size=1, tp_group=SimpleNamespace(cpu_group=None)
|
||||
),
|
||||
get_exec=lambda: cfg,
|
||||
get_platform=lambda: SimpleNamespace(is_cuda=fallback != "hip"),
|
||||
_elastic_should_preserve_local_token_counts=lambda **kwargs: False,
|
||||
_mega_moe_materializes_idle_rank=lambda batch: False,
|
||||
dp_slot_in=lambda counts: rank,
|
||||
set_dp_buffer_len_from_batch=lambda *args: None,
|
||||
set_is_extend_in_batch=lambda *args: None,
|
||||
_is_cpu=True,
|
||||
),
|
||||
patch.object(fbi, "mambaish_config", return_value=None, create=True),
|
||||
patch.object(
|
||||
fbi.DpPaddingMode,
|
||||
"get_dp_padding_mode",
|
||||
return_value=fbi.DpPaddingMode.SUM_LEN,
|
||||
),
|
||||
patch.object(ForwardBatch, "_pad_inputs_to_size"),
|
||||
patch(
|
||||
"sglang.srt.batch_overlap.two_batch_overlap.TboForwardBatchPreparer.prepare"
|
||||
),
|
||||
patch.object(fbi, "all_gather_single", side_effect=exchange) as gather,
|
||||
):
|
||||
batch.prepare_mlp_sync_batch(runner)
|
||||
assert batch.global_late_layer_num_tokens_cpu == late
|
||||
assert gather.call_count == (0 if fallback is None else 1)
|
||||
|
||||
|
||||
def test_scheduler_exchange_carries_peer_trims_and_inactive_slots():
|
||||
infos = [
|
||||
dp_attn.MLPSyncBatchInfo(
|
||||
num_dp_ranks=4,
|
||||
tp_size=1,
|
||||
cp_size=1,
|
||||
num_tokens=n,
|
||||
num_tokens_for_logprob=n,
|
||||
can_run_decode_cuda_graph=False,
|
||||
can_run_draft_cuda_graph=False,
|
||||
can_run_prefill_cuda_graph=False,
|
||||
is_extend_in_batch=True,
|
||||
local_can_run_tbo=False,
|
||||
local_forward_mode=ForwardMode.EXTEND.value,
|
||||
decoder_trim_rows=t,
|
||||
)
|
||||
for n, t in zip([4096, 282, 0, 130], [3968, 30, 0, 0])
|
||||
]
|
||||
gathered = torch.cat([info._get_local_tensor("cpu") for info in infos])
|
||||
with (
|
||||
patch.object(
|
||||
dp_attn,
|
||||
"exchange_sync_rows",
|
||||
side_effect=lambda out, *a, **kw: out.copy_(gathered),
|
||||
),
|
||||
patch.object(
|
||||
dp_attn,
|
||||
"get_parallel",
|
||||
return_value=SimpleNamespace(
|
||||
tp_group=SimpleNamespace(active_ranks_cpu=torch.tensor([1, 1, 0, 1]))
|
||||
),
|
||||
),
|
||||
):
|
||||
for info in infos:
|
||||
info.all_gather(device="cpu", group=None)
|
||||
assert info.global_decoder_trim_rows == [3968, 30, 0, 0]
|
||||
assert info.global_num_tokens == [4096, 282, 0, 130]
|
||||
@@ -292,6 +292,7 @@ class TestFoldWithDecoderSwaTail(CustomTestCase):
|
||||
positions=fx.folded_positions(),
|
||||
capture_hidden_mode=CaptureHiddenMode.NULL,
|
||||
return_logprob=False,
|
||||
extend_prompt_rest_lens_cpu=None,
|
||||
)
|
||||
with (
|
||||
patch.object(DSV4AttnMetadata, "init_compression_metadata"),
|
||||
|
||||
@@ -133,6 +133,7 @@ class TestDPSpecPrefillCoordinationPlan(CustomTestCase):
|
||||
num_tokens_for_logprob=32,
|
||||
global_num_tokens=[32, 7],
|
||||
global_num_tokens_for_logprob=[32, 7],
|
||||
global_decoder_trim_rows=[0, 0],
|
||||
tp0_info_cpu=torch.tensor([[32, 32, 1, 0], [7, 7, 1, 0]]),
|
||||
is_extend_in_batch=False,
|
||||
tbo_split_seq_index=None,
|
||||
|
||||
Reference in New Issue
Block a user