diff --git a/python/sglang/srt/environ.py b/python/sglang/srt/environ.py index 268e6fad24..90b5ab122e 100644 --- a/python/sglang/srt/environ.py +++ b/python/sglang/srt/environ.py @@ -1633,6 +1633,9 @@ class Envs: # Decoder SWA bounded replay: eager prefill steps with up to this many tail rows # run the tail layers from CUDA graphs (buckets of 128 rows); 0 keeps them eager. SGLANG_DSV4_DECODER_TAIL_GRAPH_MAX_ROWS = EnvInt(2048) + # Prefill-graph shapes of at least this many tokens run the tail from the tail + # graphs in a break (measured crossover 1536-2048 tokens on GB300 TP4). + SGLANG_DSV4_DECODER_TAIL_GRAPH_MIN_PREFILL_TOKENS = EnvInt(2048) # Paged KV layout of the DeepSeek-V4 family pools: "v4" (584 B/token, every # GPU), "v41" (the SM100 FlashMLA V4.1 formats: 528 B fp8 SWA cache, fp8 or # fp4 compressed caches) or "auto" (v41 on SM100, v4 elsewhere). diff --git a/python/sglang/srt/layers/attention/deepseek_v4_backend.py b/python/sglang/srt/layers/attention/deepseek_v4_backend.py index 0ac54d0ad8..75c8bb74c2 100644 --- a/python/sglang/srt/layers/attention/deepseek_v4_backend.py +++ b/python/sglang/srt/layers/attention/deepseek_v4_backend.py @@ -1279,6 +1279,8 @@ class DeepseekV4AttnBackend( self.tail_forward_metadata: Optional[DSV4Metadata] = None # Set by write_tail_source_kv until exit_late_layer_tail. self.tail_source_kv_layer_id: Optional[int] = None + # The live batch of the prefill-graph replay; a tail break needs its exact rows. + self.prefill_graph_replay_batch: Optional[ForwardBatch] = None # Prefill CUDA graph tail block (bounded replay inside the graph): row # capacity, persistent index/location buffers, held metadata per block size. self._bcg_tail_capacity: Optional[int] = None @@ -3081,6 +3083,7 @@ class DeepseekV4AttnBackend( static_metadata, two_level=self._prefill_graph_two_level(), ) + self.prefill_graph_replay_batch = forward_batch self.forward_metadata = capture_metadata self._activate_request_window() self.tail_forward_metadata = ( diff --git a/python/sglang/srt/model_executor/runner/row_bucket_graph_runner.py b/python/sglang/srt/model_executor/runner/row_bucket_graph_runner.py index 157a9a0aae..b2b4d9bc3a 100644 --- a/python/sglang/srt/model_executor/runner/row_bucket_graph_runner.py +++ b/python/sglang/srt/model_executor/runner/row_bucket_graph_runner.py @@ -11,6 +11,7 @@ from __future__ import annotations import copy import logging import time +from contextlib import nullcontext from typing import TYPE_CHECKING, Callable, ContextManager, List, Optional import torch @@ -38,6 +39,12 @@ if TYPE_CHECKING: logger = logging.getLogger(__name__) Body = Callable[[List[torch.Tensor], "ForwardBatch"], List[Optional[torch.Tensor]]] +# Wraps capture and replay with the body's batch and the bucket's static inputs. +BatchScope = Callable[["ForwardBatch", List[torch.Tensor]], ContextManager] + + +def _no_scope(batch: ForwardBatch, inputs: List[torch.Tensor]) -> ContextManager: + return nullcontext() class _Graph: @@ -90,15 +97,20 @@ class RowBucketGraphRunner: forward_batch: ForwardBatch, body: Body, context: Callable[[ForwardBatch], ContextManager], + batch_scope: BatchScope = _no_scope, + bucket_from: Optional[int] = None, + out_rows: Optional[int] = None, ) -> List[Optional[torch.Tensor]]: """Run ``body`` on the ``live`` rows from their bucket's graph, capturing - it on first use. ``key`` names whatever besides row count and input shapes - changes what ``body`` records; ``context`` wraps capture and replay with - the batch the eager breaks read. Returns the outputs cut to the live - rows, as views of graph buffers valid until the next replay.""" + it on first use. ``key`` names what besides row count and input shapes + changes the recording; ``context`` and ``batch_scope`` wrap capture and + replay with the breaks' and the body's batch; ``bucket_from`` picks the + bucket from another row count. Returns views valid until the next replay, + cut to ``out_rows`` rows.""" num_rows = live[0].shape[0] - rows = self.bucket_rows(num_rows) - assert rows is not None, num_rows + out_rows = num_rows if out_rows is None else out_rows + rows = self.bucket_rows(num_rows if bucket_from is None else bucket_from) + assert rows is not None, (num_rows, bucket_from) # Inputs of one shape share their static buffers across keys. input_key = tuple((t.dtype, t.shape[1:]) for t in live) static = self._static_inputs(input_key, live) @@ -116,14 +128,15 @@ class RowBucketGraphRunner: for r in range(self.max_rows, rows - 1, -self.row_step): if (r, graph_key) not in self._graphs: self._graphs[(r, graph_key)] = self._capture( - r, num_rows, static, forward_batch, body + r, num_rows, static, forward_batch, body, batch_scope ) graph = self._graphs[(rows, graph_key)] - if graph.num_token_non_padded is not None: - graph.num_token_non_padded.fill_(num_rows) - with graph_pool_replay_scope(): - graph.graph.replay() - return [None if t is None else t[:num_rows] for t in graph.outputs] + with batch_scope(step_batch, [buf[:rows] for buf in static]): + if graph.num_token_non_padded is not None: + graph.num_token_non_padded.fill_(num_rows) + with graph_pool_replay_scope(): + graph.graph.replay() + return [None if t is None else t[:out_rows] for t in graph.outputs] def _static_inputs(self, input_key, live) -> List[torch.Tensor]: static = self._static.get(input_key) @@ -140,6 +153,7 @@ class RowBucketGraphRunner: static: List[torch.Tensor], forward_batch: ForwardBatch, body: Body, + batch_scope: BatchScope, ) -> _Graph: start = time.perf_counter() free_before = torch.cuda.mem_get_info()[0] @@ -154,14 +168,18 @@ class RowBucketGraphRunner: forward_batch.num_token_non_padded, num_rows ) capture_batch.num_token_non_padded = num_token_non_padded - return self._record( - lambda: body(inputs, capture_batch), - rows, - num_token_non_padded, - capture_batch, - start, - free_before, - ) + with batch_scope(capture_batch, inputs): + graph = self._record( + lambda: body(inputs, capture_batch), + rows, + num_token_non_padded, + capture_batch, + start, + free_before, + ) + # The graph reads device tensors the scope put on the batch. + graph.batch += (dict(vars(capture_batch)),) + return graph def _record( self, diff --git a/python/sglang/srt/models/deepseek_v4.py b/python/sglang/srt/models/deepseek_v4.py index bdb56a1bc1..4099c05649 100644 --- a/python/sglang/srt/models/deepseek_v4.py +++ b/python/sglang/srt/models/deepseek_v4.py @@ -825,7 +825,17 @@ def deepseek_v4_low_ratio_sources(layer, x, q_lora, positions) -> None: ) -bcg_deepseek_v4_low_ratio_sources = eager_on_graph(True)(deepseek_v4_low_ratio_sources) +def _low_ratio_sources_capture_stub(layer, x, q_lora, positions) -> None: + # A decoder tail graph's capture pass hands the break activations its + # segments never computed; the indexer would select candidate blocks from + # them and index pages past the live context. Its replay runs the body. + if not in_replay_tail_graph(): + deepseek_v4_low_ratio_sources(layer, x, q_lora, positions) + + +bcg_deepseek_v4_low_ratio_sources = eager_on_graph( + deepseek_v4_low_ratio_sources, capture_stub=_low_ratio_sources_capture_stub +) def deepseek_v4_engram_hash_ids(hasher, input_ids: torch.Tensor) -> torch.Tensor: @@ -3846,13 +3856,15 @@ class DeepseekV4Model(nn.Module): ), in_graph=tail_in_graph, ) - use_tail_graph = ( - self.decoder_replay_tail is not None - and tail is not None - and self.decoder_replay_tail.plan( - tail, num_tokens=input_ids.shape[0], tail_in_graph=tail_in_graph + use_tail_graph, tail_graph_dp_rows = False, None + if self.decoder_replay_tail is not None: + use_tail_graph, tail_graph_dp_rows = self.decoder_replay_tail.plan( + tail, + forward_batch=forward_batch, + num_tokens=input_ids.shape[0], + tail_in_graph=tail_in_graph, + late_dp_counts=late_dp_counts, ) - ) tail_sentinel = positions.new_zeros((1,)) if tail_in_graph else None saved_full = None # The HIP defer_post gate excludes residual readers. CUDA materializes @@ -3867,7 +3879,10 @@ class DeepseekV4Model(nn.Module): if _is_cuda: layer = self.layers[i] if ( - (tail is not None and i == self.late_layer_start) + ( + (tail is not None or use_tail_graph) + and i == self.late_layer_start + ) or layer.engram is not None or (capture_dspark and i in self.dspark_layers_to_capture) ): @@ -3899,6 +3914,31 @@ class DeepseekV4Model(nn.Module): # against the persistent index buffers. tail_sentinel = bcg_deepseek_v4_enter_late_layer_tail(tail_sentinel) saved_full = attn_backend.prefill_graph_late_layer_saved + if ( + self.decoder_replay_tail is not None + and self.decoder_replay_tail.breaks_out_of_prefill_graph( + tail, num_tokens=input_ids.shape[0] + ) + ): + residual, pre, aux = ( + self.decoder_replay_tail.run_in_prefill_graph( + state=state, + positions=positions, + input_ids=input_ids, + input_ids_global=input_ids_global, + capture_dspark=capture_dspark, + block_rows=tail.token_indices.shape[0], + source_kv_layer_id=( + self.late_layer_start + if self.tail_writes_source_kv + else None + ), + ) + ) + dspark_aux_hidden_states.extend(aux) + residual_batch.clear(forward_batch) + bcg_deepseek_v4_exit_late_layer_tail(tail_sentinel) + return residual, pre, tail else: saved_full = attn_backend.enter_late_layer_tail(forward_batch) state = state.take_rows(tail.rows) @@ -3920,11 +3960,25 @@ class DeepseekV4Model(nn.Module): input_ids_global=input_ids_global, forward_batch=forward_batch, capture_dspark=capture_dspark, + dp_rows=tail_graph_dp_rows, ) dspark_aux_hidden_states.extend(aux) residual_batch.clear(forward_batch) attn_backend.exit_late_layer_tail(saved_full, forward_batch) return residual, pre, tail + if use_tail_graph and tail is None and i == self.late_layer_start: + residual, pre, aux = self.decoder_replay_tail.run_idle( + state=state, + positions=positions, + input_ids=input_ids, + input_ids_global=input_ids_global, + forward_batch=forward_batch, + capture_dspark=capture_dspark, + dp_rows=tail_graph_dp_rows, + ) + dspark_aux_hidden_states.extend(aux) + residual_batch.clear(forward_batch) + return residual, pre, None if late_dp_counts is not None and i == self.late_layer_start: saved_dp = enter_late_layer_dp(forward_batch, late_dp_counts) if get_moe_a2a_backend().is_none(): diff --git a/python/sglang/srt/models/deepseek_v4_replay_tail.py b/python/sglang/srt/models/deepseek_v4_replay_tail.py index 1327615099..77819d45fb 100644 --- a/python/sglang/srt/models/deepseek_v4_replay_tail.py +++ b/python/sglang/srt/models/deepseek_v4_replay_tail.py @@ -1,23 +1,33 @@ -"""DeepSeek-V4.1's decoder-replay tail layers (``late_layer_start`` on), replayed -from `RowBucketGraphRunner` graphs on eager prefill steps. The SWA KV store, the -attention and the low-ratio sources stay eager breaks of those graphs. +"""DeepSeek-V4.1's decoder-replay tail layers, run from row-bucket graphs. + +Eager prefill steps replay them directly. A prefill-graph shape of at least +``SGLANG_DSV4_DECODER_TAIL_GRAPH_MIN_PREFILL_TOKENS`` tokens replays them from one +eager break, through bridge buffers; smaller shapes keep the tail captured. + +Under DP attention every rank replays the largest rank's bucket. MegaMoE agrees +on it through a host gather, so it needs the prefill graph off. """ from __future__ import annotations +import copy from contextlib import contextmanager, nullcontext -from typing import TYPE_CHECKING, List, Optional, Tuple +from typing import TYPE_CHECKING, Callable, List, Optional, Tuple import torch +from sglang.srt.distributed.utils import all_gather_single from sglang.srt.environ import envs from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder +from sglang.srt.layers.dp_attention import dp_gather_replicate from sglang.srt.layers.layer_boundary.residual import batch as residual_batch +from sglang.srt.layers.moe import get_moe_a2a_backend from sglang.srt.model_executor.cuda_graph_config import ( Backend, Phase, check_cuda_graph_backend, ) +from sglang.srt.model_executor.forward_batch_info import ForwardMode from sglang.srt.model_executor.forward_context import get_attn_backend from sglang.srt.model_executor.model_runner_components.layer_setup import ( compute_attention_and_moe_layers, @@ -25,6 +35,9 @@ from sglang.srt.model_executor.model_runner_components.layer_setup import ( from sglang.srt.model_executor.runner.row_bucket_graph_runner import ( RowBucketGraphRunner, ) +from sglang.srt.model_executor.runner_backend_utils.breakable_cuda_graph import ( + breakable_cuda_graph as bcg_module, +) from sglang.srt.model_executor.runner_backend_utils.breakable_cuda_graph import ( eager_on_graph, enable_breakable_cuda_graph, @@ -35,8 +48,12 @@ from sglang.srt.model_executor.runner_backend_utils.tc_piecewise_cuda_graph.cont ) from sglang.srt.model_executor.runner_utils.capture_mode import get_is_capture_mode from sglang.srt.models import deepseek_v4_boundary as hc_boundary +from sglang.srt.models.deepseek_common.decoder_swa_bounded_replay import ( + enter_late_layer_dp, + exit_late_layer_dp, +) from sglang.srt.models.deepseek_v4_mhc import HcState -from sglang.srt.runtime_context import get_parallel +from sglang.srt.runtime_context import get_flags, get_parallel from sglang.srt.utils import is_cuda if TYPE_CHECKING: @@ -61,6 +78,20 @@ def _replay_tail_scope(): _in_replay_tail = False +def scattered_dp_counts(num_tokens: int, num_rows: int) -> Tuple[int, List[int]]: + """The largest step token count and every rank's tail rows, gathered on the + host; safe only when every rank runs the step eagerly.""" + group = get_parallel().tp_group.cpu_group + counts = torch.empty( + 2 * torch.distributed.get_world_size(group=group), dtype=torch.int64 + ) + all_gather_single( + counts, torch.tensor([num_tokens, num_rows], dtype=torch.int64), group=group + ) + tokens, rows = counts.view(-1, 2).T.tolist() + return max(tokens), rows + + def _tail_kv_store(attention, x, positions, qkv_a) -> None: # The SWA store writes the step's live tail slots, so a tail graph breaks here. forward_batch = get_tc_piecewise_forward_context().forward_batch @@ -108,6 +139,9 @@ class DecoderReplayTail: self.runner = RowBucketGraphRunner( row_step=row_step, max_rows=max_rows, label="Decoder tail graph" ) + self._static_ids_global: dict[torch.dtype, torch.Tensor] = {} + # Older, smaller sets stay alive for the graphs that read them. + self._bridge_buffers: dict[tuple, List[List[Optional[torch.Tensor]]]] = {} @classmethod def maybe_create(cls, model: torch.nn.Module) -> Optional[DecoderReplayTail]: @@ -130,24 +164,114 @@ class DecoderReplayTail: ) def plan( - self, tail: LateLayerTail, *, num_tokens: int, tail_in_graph: bool - ) -> bool: - """Whether this step runs its tail layers from a graph; also reports the - step's token count to the warm gate.""" + self, + tail: Optional[LateLayerTail], + *, + forward_batch: ForwardBatch, + num_tokens: int, + tail_in_graph: bool, + late_dp_counts: Optional[List[int]], + ) -> Tuple[bool, Optional[List[int]]]: + """Whether this step uses a tail graph and, under DP attention, every rank's + tail rows; also feeds the warm gate.""" from sglang.kernels.ops.attention.dsv4.unified_kv_kernels.env_gate import ( is_unified_kv_triton, ) - self.runner.ready(num_tokens) - return not ( + dp = get_parallel().attn_dp_size > 1 + if not (tail is not None or (dp and forward_batch.is_extend_in_batch)): + return False, None + scattered = self._scattered_dp( + tail, + tail_in_graph=tail_in_graph, + forward_batch=forward_batch, + num_tokens=num_tokens, + ) + if scattered is not None: + self.runner.ready(scattered[0]) + else: + counts = forward_batch.global_num_tokens_cpu + self.runner.ready(max(counts) if dp and counts else num_tokens) + if ( tail_in_graph - or get_parallel().attn_dp_size > 1 or get_is_capture_mode() or is_unified_kv_triton() or get_global_expert_distribution_recorder().recording or not self.runner.warm - or tail.cp_metadata is not None - or self.runner.bucket_rows(tail.positions.shape[0]) is None + or (tail is not None and tail.cp_metadata is not None) + ): + return False, None + if not dp: + return ( + tail is not None + and self.runner.bucket_rows(tail.positions.shape[0]) is not None, + None, + ) + dp_rows = scattered[1] if scattered is not None else None + if dp_rows is None: + dp_rows = self._gathered_dp_rows(late_dp_counts, forward_batch) + if dp_rows is None or self.runner.bucket_rows(max(dp_rows)) is None: + return False, None + return True, dp_rows + + def _scattered_dp( + self, + tail: Optional[LateLayerTail], + *, + tail_in_graph: bool, + forward_batch: ForwardBatch, + num_tokens: int, + ) -> Optional[Tuple[int, List[int]]]: + """`scattered_dp_counts` for a MoE routing each rank's own rows. Such a MoE + lets ranks pick prefill graph or eager independently, so it needs the + prefill graph disabled.""" + if ( + get_parallel().attn_dp_size == 1 + or get_moe_a2a_backend().is_none() + or not check_cuda_graph_backend(Phase.PREFILL, Backend.DISABLED) + or not forward_batch.is_extend_in_batch + or tail_in_graph + or get_is_capture_mode() + ): + return None + return scattered_dp_counts( + num_tokens, 0 if tail is None else tail.positions.shape[0] + ) + + @staticmethod + def _gathered_dp_rows( + late_dp_counts: Optional[List[int]], forward_batch: ForwardBatch + ) -> Optional[List[int]]: + """Every rank's tail rows for a TP MoE, or None without a uniform layout.""" + if not get_moe_a2a_backend().is_none() or not forward_batch.is_extend_in_batch: + return None + # Exchanged before the forward (ForwardBatch.prepare_mlp_sync_batch). + if forward_batch.global_late_layer_num_tokens_cpu is not None: + return list(forward_batch.global_late_layer_num_tokens_cpu) + if late_dp_counts is not None: + return late_dp_counts + # No re-layout was needed: every rank's tail is its whole extend. + early = forward_batch.global_num_tokens_padded_cpu + if early is None: + early = forward_batch.global_num_tokens_cpu + if early is None or len(early) != get_parallel().attn_dp_size: + return None + return list(early) + + def breaks_out_of_prefill_graph( + self, tail: LateLayerTail, *, num_tokens: int + ) -> bool: + from sglang.kernels.ops.attention.dsv4.unified_kv_kernels.env_gate import ( + is_unified_kv_triton, + ) + + return ( + num_tokens >= envs.SGLANG_DSV4_DECODER_TAIL_GRAPH_MIN_PREFILL_TOKENS.get() + and (get_parallel().attn_dp_size == 1 or get_moe_a2a_backend().is_none()) + and tail.cp_metadata is None + and not is_unified_kv_triton() + # Encoder replay's folded rows are not checked against the exact tail. + and get_attn_backend().token_to_kv_pool.request_window is None ) def run( @@ -156,37 +280,227 @@ class DecoderReplayTail: state: HcState, positions: torch.Tensor, input_ids: torch.Tensor, - input_ids_global: torch.Tensor, + input_ids_global: Optional[torch.Tensor], forward_batch: ForwardBatch, capture_dspark: bool, + dp_rows: Optional[List[int]] = None, + out_rows: Optional[int] = None, ) -> Tuple[torch.Tensor, Optional[torch.Tensor], List[torch.Tensor]]: - """Returns the tail rows' materialized residual and pre and the DSpark - auxiliary states, as views of graph buffers valid until the next replay.""" + """Returns views of graph buffers valid until the next replay, cut to + ``out_rows`` rows. Under DP every rank replays the bucket of the largest + ``dp_rows``.""" + if forward_batch.forward_mode.is_idle(): + # An idle DP rank runs the extend's graph, with no rows of its own. + forward_batch = copy.copy(forward_batch) + forward_batch.forward_mode = ForwardMode.EXTEND + gather = dp_rows is not None and get_moe_a2a_backend().is_none() + # Ranks in the DP layout the MoE sees: all of them when it gathers. + layout = None if dp_rows is None else (len(dp_rows) if gather else 1) + gather_ids = gather and self._model.late_layer_vl_routing + if input_ids_global is None: + input_ids_global = input_ids leaves, has_pre, rebuild = _flatten_state(state) num_state = len(leaves) + live = [*leaves, positions, input_ids] + if not gather: + live.append(input_ids_global) + ids_global = ( + self._ids_global_buffer(input_ids.dtype, len(dp_rows)) + if gather_ids + else None + ) def body(inputs: List[torch.Tensor], batch: ForwardBatch): + rows = inputs[0].shape[0] + if not gather: + ids = inputs[num_state + 2] + else: + ids = None if ids_global is None else ids_global[: rows * layout] residual, pre, aux = self._layer_stack( rebuild(inputs[:num_state]), positions=inputs[num_state], input_ids=inputs[num_state + 1], - input_ids_global=inputs[num_state + 2], + input_ids_global=ids, forward_batch=batch, capture_dspark=capture_dspark, ) return [residual, pre, *aux] + @contextmanager + def dp_layout(batch: ForwardBatch, inputs: List[torch.Tensor]): + if layout is None: + yield + return + rows = inputs[0].shape[0] + graph_count = batch.num_token_non_padded + saved = enter_late_layer_dp(batch, [rows] * layout) + if layout == 1 and graph_count is not None: + # A local MoE masks pad rows by the graph's count, not the bucket. + batch.global_num_token_non_padded = graph_count + batch.num_token_non_padded = graph_count + try: + if ids_global is not None: + dp_gather_replicate( + ids_global[: batch.global_dp_buffer_len, None], + inputs[num_state + 1][:, None].clone(), + batch, + ) + yield + finally: + exit_late_layer_dp(batch, saved) + residual, pre, *aux = self.runner.run( - key=(capture_dspark, has_pre), - live=[*leaves, positions, input_ids, input_ids_global], + key=(capture_dspark, has_pre, layout, gather_ids), + live=live, forward_batch=forward_batch, body=body, context=self._break_context, + batch_scope=dp_layout, + bucket_from=None if dp_rows is None else max(dp_rows), + out_rows=out_rows, ) return residual, pre, aux + def run_idle( + self, + *, + state: HcState, + positions: torch.Tensor, + input_ids: torch.Tensor, + input_ids_global: Optional[torch.Tensor], + forward_batch: ForwardBatch, + capture_dspark: bool, + dp_rows: List[int], + ) -> Tuple[torch.Tensor, Optional[torch.Tensor], List[torch.Tensor]]: + """An idle DP rank replays the shared bucket with no rows of its own.""" + return self.run( + state=state.take_rows(lambda t: t[:0]), + positions=positions[:0], + input_ids=input_ids[:0], + input_ids_global=( + None if input_ids_global is None else input_ids_global[:0] + ), + forward_batch=forward_batch, + capture_dspark=capture_dspark, + dp_rows=dp_rows, + out_rows=state.residual.shape[0], + ) + + def _run_layers_eagerly( + self, + *, + state: HcState, + positions: torch.Tensor, + input_ids: torch.Tensor, + input_ids_global: Optional[torch.Tensor], + forward_batch: ForwardBatch, + capture_dspark: bool, + dp_rows: Optional[List[int]] = None, + ) -> Tuple[torch.Tensor, Optional[torch.Tensor], List[torch.Tensor]]: + """For a prefill-graph step whose tail has no graph to replay.""" + if forward_batch.forward_mode.is_idle(): + forward_batch = copy.copy(forward_batch) + forward_batch.forward_mode = ForwardMode.EXTEND + tail_batch = copy.copy(forward_batch) + tail_batch.global_num_token_non_padded_cpu = positions.shape[0] + gather = dp_rows is not None and get_moe_a2a_backend().is_none() + if input_ids_global is None or gather: + input_ids_global = input_ids if not gather else None + saved_dp = None + if dp_rows is not None: + saved_dp = enter_late_layer_dp(tail_batch, dp_rows) + try: + if gather and self._model.late_layer_vl_routing: + gathered = input_ids.new_empty((sum(dp_rows), 1)) + dp_gather_replicate(gathered, input_ids[:, None].clone(), tail_batch) + input_ids_global = gathered.squeeze(-1) + with self._break_context(tail_batch): + return self._layer_stack( + state, + positions=positions, + input_ids=input_ids, + input_ids_global=input_ids_global, + forward_batch=tail_batch, + capture_dspark=capture_dspark, + ) + finally: + if saved_dp is not None: + exit_late_layer_dp(tail_batch, saved_dp) + + def run_in_prefill_graph( + self, + *, + state: HcState, + positions: torch.Tensor, + input_ids: torch.Tensor, + input_ids_global: Optional[torch.Tensor], + capture_dspark: bool, + block_rows: int, + source_kv_layer_id: Optional[int], + ) -> Tuple[torch.Tensor, Optional[torch.Tensor], List[torch.Tensor]]: + """Runs the tail in one break of the outer prefill graph, bridged into its + ``block_rows`` block.""" + leaves, _, rebuild = _flatten_state(state) + residual, pre, *aux = bcg_tail_layers_in_prefill_graph( + self, + rebuild, + capture_dspark, + block_rows, + source_kv_layer_id, + get_parallel().attn_dp_size > 1, + leaves, + positions, + input_ids, + input_ids if input_ids_global is None else input_ids_global, + ) + return residual, pre, aux + + def _bridge_outputs( + self, block_rows: int, outputs: List[Optional[torch.Tensor]] + ) -> Tuple[Optional[torch.Tensor], ...]: + """Block-sized bridge views of the tail rows; rows past the tail are zero.""" + key = tuple(None if t is None else (t.dtype, t.shape[1:]) for t in outputs) + sets = self._bridge_buffers.setdefault(key, []) + if not sets or sets[0][0].shape[0] < block_rows: + sets.insert( + 0, + [ + None if t is None else t.new_zeros((block_rows, *t.shape[1:])) + for t in outputs + ], + ) + views = [] + for buf, t in zip(sets[0], outputs): + if t is None: + views.append(None) + continue + view = buf[:block_rows] + view[: t.shape[0]].copy_(t) + view[t.shape[0] :].zero_() + views.append(view) + return tuple(views) + + def _ids_global_buffer(self, dtype: torch.dtype, dp_size: int) -> torch.Tensor: + buf = self._static_ids_global.get(dtype) + if buf is None: + buf = self._static_ids_global[dtype] = torch.zeros( + (self.runner.max_rows * dp_size,), dtype=dtype, device="cuda" + ) + return buf + @contextmanager def _break_context(self, tail_batch: ForwardBatch): + outer = get_tc_piecewise_forward_context() + if outer is not None: + # Nested in a prefill-graph break: swap only the batch the breaks read. + saved_batch = outer.forward_batch + outer.forward_batch = tail_batch + try: + with _replay_tail_scope(): + yield + finally: + outer.forward_batch = saved_batch + return if self._layers is None: self._layers = compute_attention_and_moe_layers(self._model) layers = self._layers @@ -256,3 +570,127 @@ class DecoderReplayTail: residual_batch.clear(forward_batch) state = state.materialized(model.layers[self._layer_ids[-1]].hc_cfg) return state.residual, state.pre, aux + + +def _unpadded_batch(forward_batch: ForwardBatch) -> ForwardBatch: + """The batch cut to its live requests and tokens (the graph pads both).""" + extend_lens = forward_batch.extend_seq_lens_cpu + if extend_lens is None: + return forward_batch + bs, num_tokens = len(extend_lens), int(sum(extend_lens)) + view = copy.copy(forward_batch) + view.batch_size = bs + for name in ( + "req_pool_indices", + "seq_lens", + "seq_lens_cpu", + "extend_seq_lens", + "extend_prefix_lens", + "extend_start_loc", + ): + value = getattr(forward_batch, name, None) + if value is not None: + setattr(view, name, value[:bs]) + for name in ("positions", "out_cache_loc", "input_ids"): + value = getattr(forward_batch, name, None) + if value is not None: + setattr(view, name, value[:num_tokens]) + return view + + +def _enter_eager_tail_in_prefill_graph( + backend, graph_batch: ForwardBatch, source_kv_layer_id: Optional[int] +) -> Tuple[tuple, ForwardBatch]: + """Switches onto exact-row tail metadata; returns the exit state and the + batch the tail runs on (the capture batch while the graph captures).""" + forward_batch = _unpadded_batch(backend.prefill_graph_replay_batch or graph_batch) + held = backend.tail_forward_metadata + backend.tail_forward_metadata = backend._build_late_layer_tail_metadata( + forward_batch + ) + try: + saved = backend.enter_late_layer_tail(forward_batch) + finally: + backend.tail_forward_metadata = held + backend.tail_source_kv_layer_id = source_kv_layer_id + return saved, forward_batch + + +def _tail_layers_in_prefill_graph( + tail_layers: DecoderReplayTail, + rebuild: Callable[[List[torch.Tensor]], HcState], + capture_dspark: bool, + block_rows: int, + source_kv_layer_id: Optional[int], + dp: bool, + leaves: List[torch.Tensor], + positions: torch.Tensor, + input_ids: torch.Tensor, + input_ids_global: torch.Tensor, +) -> Tuple[Optional[torch.Tensor], ...]: + backend = get_attn_backend() + saved, forward_batch = _enter_eager_tail_in_prefill_graph( + backend, get_tc_piecewise_forward_context().forward_batch, source_kv_layer_id + ) + # Runs between the outer graph's segments, also while it captures, so the + # inner breaks must run directly. + capture_token = bcg_module._current_capture_var.set(None) + try: + tail = backend.forward_metadata.late_layer_tail + # The graph's rows are padded to its shape; the tail indexes live rows. + num_tokens = forward_batch.input_ids.shape[0] + + def rows(t: torch.Tensor) -> torch.Tensor: + return tail.rows(t[:num_tokens]) + + num_rows = tail.positions.shape[0] + dp_rows = None + if dp: + # Exchanged before the forward, except while the prefill graphs capture + # (every rank together), when it gathers here. + dp_rows = forward_batch.global_late_layer_num_tokens_cpu + if dp_rows is None: + counts = torch.empty(get_parallel().tp_size, dtype=torch.int64) + all_gather_single( + counts, + torch.tensor([num_rows], dtype=torch.int64), + group=get_parallel().tp_group.cpu_group, + ) + dp_rows = counts.tolist() + assert dp_rows[get_parallel().attn_dp_rank] == num_rows, ( + dp_rows, + num_rows, + ) + tail_ids = rows(input_ids) + # A TP MoE under DP gathers every rank's IDs inside the tail layers. + if dp and get_moe_a2a_backend().is_none(): + ids_global = None + else: + ids_global = rows(input_ids_global) + kwargs = dict( + state=rebuild([rows(t) for t in leaves]), + positions=tail.positions, + input_ids=tail_ids, + input_ids_global=ids_global, + forward_batch=forward_batch, + capture_dspark=capture_dspark, + dp_rows=dp_rows, + ) + # Capture stream and pool are busy while the prefill graphs capture. + runner = tail_layers.runner + if ( + not get_flags().dp.capturing_prefill_graph + and runner.warm + and runner.bucket_rows(max(dp_rows) if dp else num_rows) is not None + ): + residual, pre, aux = tail_layers.run(**kwargs) + else: + residual, pre, aux = tail_layers._run_layers_eagerly(**kwargs) + return tail_layers._bridge_outputs(block_rows, [residual, pre, *aux]) + finally: + bcg_module._current_capture_var.reset(capture_token) + backend.exit_late_layer_tail(saved, forward_batch) + backend.prefill_graph_replay_batch = None + + +bcg_tail_layers_in_prefill_graph = eager_on_graph(True)(_tail_layers_in_prefill_graph)