[Perf] deepseek-v4.1: Run decoder replay tail graphs in prefill-graph steps and under DP attention (#45107)

This commit is contained in:
Liangsheng Yin
2026-10-19 13:02:07 -07:00
committed by GitHub
parent 5385866108
commit f1a2ee763d
5 changed files with 564 additions and 48 deletions
+3
View File
@@ -1633,6 +1633,9 @@ class Envs:
# Decoder SWA bounded replay: eager prefill steps with up to this many tail rows
# run the tail layers from CUDA graphs (buckets of 128 rows); 0 keeps them eager.
SGLANG_DSV4_DECODER_TAIL_GRAPH_MAX_ROWS = EnvInt(2048)
# Prefill-graph shapes of at least this many tokens run the tail from the tail
# graphs in a break (measured crossover 1536-2048 tokens on GB300 TP4).
SGLANG_DSV4_DECODER_TAIL_GRAPH_MIN_PREFILL_TOKENS = EnvInt(2048)
# Paged KV layout of the DeepSeek-V4 family pools: "v4" (584 B/token, every
# GPU), "v41" (the SM100 FlashMLA V4.1 formats: 528 B fp8 SWA cache, fp8 or
# fp4 compressed caches) or "auto" (v41 on SM100, v4 elsewhere).
@@ -1279,6 +1279,8 @@ class DeepseekV4AttnBackend(
self.tail_forward_metadata: Optional[DSV4Metadata] = None
# Set by write_tail_source_kv until exit_late_layer_tail.
self.tail_source_kv_layer_id: Optional[int] = None
# The live batch of the prefill-graph replay; a tail break needs its exact rows.
self.prefill_graph_replay_batch: Optional[ForwardBatch] = None
# Prefill CUDA graph tail block (bounded replay inside the graph): row
# capacity, persistent index/location buffers, held metadata per block size.
self._bcg_tail_capacity: Optional[int] = None
@@ -3081,6 +3083,7 @@ class DeepseekV4AttnBackend(
static_metadata,
two_level=self._prefill_graph_two_level(),
)
self.prefill_graph_replay_batch = forward_batch
self.forward_metadata = capture_metadata
self._activate_request_window()
self.tail_forward_metadata = (
@@ -11,6 +11,7 @@ from __future__ import annotations
import copy
import logging
import time
from contextlib import nullcontext
from typing import TYPE_CHECKING, Callable, ContextManager, List, Optional
import torch
@@ -38,6 +39,12 @@ if TYPE_CHECKING:
logger = logging.getLogger(__name__)
Body = Callable[[List[torch.Tensor], "ForwardBatch"], List[Optional[torch.Tensor]]]
# Wraps capture and replay with the body's batch and the bucket's static inputs.
BatchScope = Callable[["ForwardBatch", List[torch.Tensor]], ContextManager]
def _no_scope(batch: ForwardBatch, inputs: List[torch.Tensor]) -> ContextManager:
return nullcontext()
class _Graph:
@@ -90,15 +97,20 @@ class RowBucketGraphRunner:
forward_batch: ForwardBatch,
body: Body,
context: Callable[[ForwardBatch], ContextManager],
batch_scope: BatchScope = _no_scope,
bucket_from: Optional[int] = None,
out_rows: Optional[int] = None,
) -> List[Optional[torch.Tensor]]:
"""Run ``body`` on the ``live`` rows from their bucket's graph, capturing
it on first use. ``key`` names whatever besides row count and input shapes
changes what ``body`` records; ``context`` wraps capture and replay with
the batch the eager breaks read. Returns the outputs cut to the live
rows, as views of graph buffers valid until the next replay."""
it on first use. ``key`` names what besides row count and input shapes
changes the recording; ``context`` and ``batch_scope`` wrap capture and
replay with the breaks' and the body's batch; ``bucket_from`` picks the
bucket from another row count. Returns views valid until the next replay,
cut to ``out_rows`` rows."""
num_rows = live[0].shape[0]
rows = self.bucket_rows(num_rows)
assert rows is not None, num_rows
out_rows = num_rows if out_rows is None else out_rows
rows = self.bucket_rows(num_rows if bucket_from is None else bucket_from)
assert rows is not None, (num_rows, bucket_from)
# Inputs of one shape share their static buffers across keys.
input_key = tuple((t.dtype, t.shape[1:]) for t in live)
static = self._static_inputs(input_key, live)
@@ -116,14 +128,15 @@ class RowBucketGraphRunner:
for r in range(self.max_rows, rows - 1, -self.row_step):
if (r, graph_key) not in self._graphs:
self._graphs[(r, graph_key)] = self._capture(
r, num_rows, static, forward_batch, body
r, num_rows, static, forward_batch, body, batch_scope
)
graph = self._graphs[(rows, graph_key)]
if graph.num_token_non_padded is not None:
graph.num_token_non_padded.fill_(num_rows)
with graph_pool_replay_scope():
graph.graph.replay()
return [None if t is None else t[:num_rows] for t in graph.outputs]
with batch_scope(step_batch, [buf[:rows] for buf in static]):
if graph.num_token_non_padded is not None:
graph.num_token_non_padded.fill_(num_rows)
with graph_pool_replay_scope():
graph.graph.replay()
return [None if t is None else t[:out_rows] for t in graph.outputs]
def _static_inputs(self, input_key, live) -> List[torch.Tensor]:
static = self._static.get(input_key)
@@ -140,6 +153,7 @@ class RowBucketGraphRunner:
static: List[torch.Tensor],
forward_batch: ForwardBatch,
body: Body,
batch_scope: BatchScope,
) -> _Graph:
start = time.perf_counter()
free_before = torch.cuda.mem_get_info()[0]
@@ -154,14 +168,18 @@ class RowBucketGraphRunner:
forward_batch.num_token_non_padded, num_rows
)
capture_batch.num_token_non_padded = num_token_non_padded
return self._record(
lambda: body(inputs, capture_batch),
rows,
num_token_non_padded,
capture_batch,
start,
free_before,
)
with batch_scope(capture_batch, inputs):
graph = self._record(
lambda: body(inputs, capture_batch),
rows,
num_token_non_padded,
capture_batch,
start,
free_before,
)
# The graph reads device tensors the scope put on the batch.
graph.batch += (dict(vars(capture_batch)),)
return graph
def _record(
self,
+62 -8
View File
@@ -825,7 +825,17 @@ def deepseek_v4_low_ratio_sources(layer, x, q_lora, positions) -> None:
)
bcg_deepseek_v4_low_ratio_sources = eager_on_graph(True)(deepseek_v4_low_ratio_sources)
def _low_ratio_sources_capture_stub(layer, x, q_lora, positions) -> None:
# A decoder tail graph's capture pass hands the break activations its
# segments never computed; the indexer would select candidate blocks from
# them and index pages past the live context. Its replay runs the body.
if not in_replay_tail_graph():
deepseek_v4_low_ratio_sources(layer, x, q_lora, positions)
bcg_deepseek_v4_low_ratio_sources = eager_on_graph(
deepseek_v4_low_ratio_sources, capture_stub=_low_ratio_sources_capture_stub
)
def deepseek_v4_engram_hash_ids(hasher, input_ids: torch.Tensor) -> torch.Tensor:
@@ -3846,13 +3856,15 @@ class DeepseekV4Model(nn.Module):
),
in_graph=tail_in_graph,
)
use_tail_graph = (
self.decoder_replay_tail is not None
and tail is not None
and self.decoder_replay_tail.plan(
tail, num_tokens=input_ids.shape[0], tail_in_graph=tail_in_graph
use_tail_graph, tail_graph_dp_rows = False, None
if self.decoder_replay_tail is not None:
use_tail_graph, tail_graph_dp_rows = self.decoder_replay_tail.plan(
tail,
forward_batch=forward_batch,
num_tokens=input_ids.shape[0],
tail_in_graph=tail_in_graph,
late_dp_counts=late_dp_counts,
)
)
tail_sentinel = positions.new_zeros((1,)) if tail_in_graph else None
saved_full = None
# The HIP defer_post gate excludes residual readers. CUDA materializes
@@ -3867,7 +3879,10 @@ class DeepseekV4Model(nn.Module):
if _is_cuda:
layer = self.layers[i]
if (
(tail is not None and i == self.late_layer_start)
(
(tail is not None or use_tail_graph)
and i == self.late_layer_start
)
or layer.engram is not None
or (capture_dspark and i in self.dspark_layers_to_capture)
):
@@ -3899,6 +3914,31 @@ class DeepseekV4Model(nn.Module):
# against the persistent index buffers.
tail_sentinel = bcg_deepseek_v4_enter_late_layer_tail(tail_sentinel)
saved_full = attn_backend.prefill_graph_late_layer_saved
if (
self.decoder_replay_tail is not None
and self.decoder_replay_tail.breaks_out_of_prefill_graph(
tail, num_tokens=input_ids.shape[0]
)
):
residual, pre, aux = (
self.decoder_replay_tail.run_in_prefill_graph(
state=state,
positions=positions,
input_ids=input_ids,
input_ids_global=input_ids_global,
capture_dspark=capture_dspark,
block_rows=tail.token_indices.shape[0],
source_kv_layer_id=(
self.late_layer_start
if self.tail_writes_source_kv
else None
),
)
)
dspark_aux_hidden_states.extend(aux)
residual_batch.clear(forward_batch)
bcg_deepseek_v4_exit_late_layer_tail(tail_sentinel)
return residual, pre, tail
else:
saved_full = attn_backend.enter_late_layer_tail(forward_batch)
state = state.take_rows(tail.rows)
@@ -3920,11 +3960,25 @@ class DeepseekV4Model(nn.Module):
input_ids_global=input_ids_global,
forward_batch=forward_batch,
capture_dspark=capture_dspark,
dp_rows=tail_graph_dp_rows,
)
dspark_aux_hidden_states.extend(aux)
residual_batch.clear(forward_batch)
attn_backend.exit_late_layer_tail(saved_full, forward_batch)
return residual, pre, tail
if use_tail_graph and tail is None and i == self.late_layer_start:
residual, pre, aux = self.decoder_replay_tail.run_idle(
state=state,
positions=positions,
input_ids=input_ids,
input_ids_global=input_ids_global,
forward_batch=forward_batch,
capture_dspark=capture_dspark,
dp_rows=tail_graph_dp_rows,
)
dspark_aux_hidden_states.extend(aux)
residual_batch.clear(forward_batch)
return residual, pre, None
if late_dp_counts is not None and i == self.late_layer_start:
saved_dp = enter_late_layer_dp(forward_batch, late_dp_counts)
if get_moe_a2a_backend().is_none():
@@ -1,23 +1,33 @@
"""DeepSeek-V4.1's decoder-replay tail layers (``late_layer_start`` on), replayed
from `RowBucketGraphRunner` graphs on eager prefill steps. The SWA KV store, the
attention and the low-ratio sources stay eager breaks of those graphs.
"""DeepSeek-V4.1's decoder-replay tail layers, run from row-bucket graphs.
Eager prefill steps replay them directly. A prefill-graph shape of at least
``SGLANG_DSV4_DECODER_TAIL_GRAPH_MIN_PREFILL_TOKENS`` tokens replays them from one
eager break, through bridge buffers; smaller shapes keep the tail captured.
Under DP attention every rank replays the largest rank's bucket. MegaMoE agrees
on it through a host gather, so it needs the prefill graph off.
"""
from __future__ import annotations
import copy
from contextlib import contextmanager, nullcontext
from typing import TYPE_CHECKING, List, Optional, Tuple
from typing import TYPE_CHECKING, Callable, List, Optional, Tuple
import torch
from sglang.srt.distributed.utils import all_gather_single
from sglang.srt.environ import envs
from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder
from sglang.srt.layers.dp_attention import dp_gather_replicate
from sglang.srt.layers.layer_boundary.residual import batch as residual_batch
from sglang.srt.layers.moe import get_moe_a2a_backend
from sglang.srt.model_executor.cuda_graph_config import (
Backend,
Phase,
check_cuda_graph_backend,
)
from sglang.srt.model_executor.forward_batch_info import ForwardMode
from sglang.srt.model_executor.forward_context import get_attn_backend
from sglang.srt.model_executor.model_runner_components.layer_setup import (
compute_attention_and_moe_layers,
@@ -25,6 +35,9 @@ from sglang.srt.model_executor.model_runner_components.layer_setup import (
from sglang.srt.model_executor.runner.row_bucket_graph_runner import (
RowBucketGraphRunner,
)
from sglang.srt.model_executor.runner_backend_utils.breakable_cuda_graph import (
breakable_cuda_graph as bcg_module,
)
from sglang.srt.model_executor.runner_backend_utils.breakable_cuda_graph import (
eager_on_graph,
enable_breakable_cuda_graph,
@@ -35,8 +48,12 @@ from sglang.srt.model_executor.runner_backend_utils.tc_piecewise_cuda_graph.cont
)
from sglang.srt.model_executor.runner_utils.capture_mode import get_is_capture_mode
from sglang.srt.models import deepseek_v4_boundary as hc_boundary
from sglang.srt.models.deepseek_common.decoder_swa_bounded_replay import (
enter_late_layer_dp,
exit_late_layer_dp,
)
from sglang.srt.models.deepseek_v4_mhc import HcState
from sglang.srt.runtime_context import get_parallel
from sglang.srt.runtime_context import get_flags, get_parallel
from sglang.srt.utils import is_cuda
if TYPE_CHECKING:
@@ -61,6 +78,20 @@ def _replay_tail_scope():
_in_replay_tail = False
def scattered_dp_counts(num_tokens: int, num_rows: int) -> Tuple[int, List[int]]:
"""The largest step token count and every rank's tail rows, gathered on the
host; safe only when every rank runs the step eagerly."""
group = get_parallel().tp_group.cpu_group
counts = torch.empty(
2 * torch.distributed.get_world_size(group=group), dtype=torch.int64
)
all_gather_single(
counts, torch.tensor([num_tokens, num_rows], dtype=torch.int64), group=group
)
tokens, rows = counts.view(-1, 2).T.tolist()
return max(tokens), rows
def _tail_kv_store(attention, x, positions, qkv_a) -> None:
# The SWA store writes the step's live tail slots, so a tail graph breaks here.
forward_batch = get_tc_piecewise_forward_context().forward_batch
@@ -108,6 +139,9 @@ class DecoderReplayTail:
self.runner = RowBucketGraphRunner(
row_step=row_step, max_rows=max_rows, label="Decoder tail graph"
)
self._static_ids_global: dict[torch.dtype, torch.Tensor] = {}
# Older, smaller sets stay alive for the graphs that read them.
self._bridge_buffers: dict[tuple, List[List[Optional[torch.Tensor]]]] = {}
@classmethod
def maybe_create(cls, model: torch.nn.Module) -> Optional[DecoderReplayTail]:
@@ -130,24 +164,114 @@ class DecoderReplayTail:
)
def plan(
self, tail: LateLayerTail, *, num_tokens: int, tail_in_graph: bool
) -> bool:
"""Whether this step runs its tail layers from a graph; also reports the
step's token count to the warm gate."""
self,
tail: Optional[LateLayerTail],
*,
forward_batch: ForwardBatch,
num_tokens: int,
tail_in_graph: bool,
late_dp_counts: Optional[List[int]],
) -> Tuple[bool, Optional[List[int]]]:
"""Whether this step uses a tail graph and, under DP attention, every rank's
tail rows; also feeds the warm gate."""
from sglang.kernels.ops.attention.dsv4.unified_kv_kernels.env_gate import (
is_unified_kv_triton,
)
self.runner.ready(num_tokens)
return not (
dp = get_parallel().attn_dp_size > 1
if not (tail is not None or (dp and forward_batch.is_extend_in_batch)):
return False, None
scattered = self._scattered_dp(
tail,
tail_in_graph=tail_in_graph,
forward_batch=forward_batch,
num_tokens=num_tokens,
)
if scattered is not None:
self.runner.ready(scattered[0])
else:
counts = forward_batch.global_num_tokens_cpu
self.runner.ready(max(counts) if dp and counts else num_tokens)
if (
tail_in_graph
or get_parallel().attn_dp_size > 1
or get_is_capture_mode()
or is_unified_kv_triton()
or get_global_expert_distribution_recorder().recording
or not self.runner.warm
or tail.cp_metadata is not None
or self.runner.bucket_rows(tail.positions.shape[0]) is None
or (tail is not None and tail.cp_metadata is not None)
):
return False, None
if not dp:
return (
tail is not None
and self.runner.bucket_rows(tail.positions.shape[0]) is not None,
None,
)
dp_rows = scattered[1] if scattered is not None else None
if dp_rows is None:
dp_rows = self._gathered_dp_rows(late_dp_counts, forward_batch)
if dp_rows is None or self.runner.bucket_rows(max(dp_rows)) is None:
return False, None
return True, dp_rows
def _scattered_dp(
self,
tail: Optional[LateLayerTail],
*,
tail_in_graph: bool,
forward_batch: ForwardBatch,
num_tokens: int,
) -> Optional[Tuple[int, List[int]]]:
"""`scattered_dp_counts` for a MoE routing each rank's own rows. Such a MoE
lets ranks pick prefill graph or eager independently, so it needs the
prefill graph disabled."""
if (
get_parallel().attn_dp_size == 1
or get_moe_a2a_backend().is_none()
or not check_cuda_graph_backend(Phase.PREFILL, Backend.DISABLED)
or not forward_batch.is_extend_in_batch
or tail_in_graph
or get_is_capture_mode()
):
return None
return scattered_dp_counts(
num_tokens, 0 if tail is None else tail.positions.shape[0]
)
@staticmethod
def _gathered_dp_rows(
late_dp_counts: Optional[List[int]], forward_batch: ForwardBatch
) -> Optional[List[int]]:
"""Every rank's tail rows for a TP MoE, or None without a uniform layout."""
if not get_moe_a2a_backend().is_none() or not forward_batch.is_extend_in_batch:
return None
# Exchanged before the forward (ForwardBatch.prepare_mlp_sync_batch).
if forward_batch.global_late_layer_num_tokens_cpu is not None:
return list(forward_batch.global_late_layer_num_tokens_cpu)
if late_dp_counts is not None:
return late_dp_counts
# No re-layout was needed: every rank's tail is its whole extend.
early = forward_batch.global_num_tokens_padded_cpu
if early is None:
early = forward_batch.global_num_tokens_cpu
if early is None or len(early) != get_parallel().attn_dp_size:
return None
return list(early)
def breaks_out_of_prefill_graph(
self, tail: LateLayerTail, *, num_tokens: int
) -> bool:
from sglang.kernels.ops.attention.dsv4.unified_kv_kernels.env_gate import (
is_unified_kv_triton,
)
return (
num_tokens >= envs.SGLANG_DSV4_DECODER_TAIL_GRAPH_MIN_PREFILL_TOKENS.get()
and (get_parallel().attn_dp_size == 1 or get_moe_a2a_backend().is_none())
and tail.cp_metadata is None
and not is_unified_kv_triton()
# Encoder replay's folded rows are not checked against the exact tail.
and get_attn_backend().token_to_kv_pool.request_window is None
)
def run(
@@ -156,37 +280,227 @@ class DecoderReplayTail:
state: HcState,
positions: torch.Tensor,
input_ids: torch.Tensor,
input_ids_global: torch.Tensor,
input_ids_global: Optional[torch.Tensor],
forward_batch: ForwardBatch,
capture_dspark: bool,
dp_rows: Optional[List[int]] = None,
out_rows: Optional[int] = None,
) -> Tuple[torch.Tensor, Optional[torch.Tensor], List[torch.Tensor]]:
"""Returns the tail rows' materialized residual and pre and the DSpark
auxiliary states, as views of graph buffers valid until the next replay."""
"""Returns views of graph buffers valid until the next replay, cut to
``out_rows`` rows. Under DP every rank replays the bucket of the largest
``dp_rows``."""
if forward_batch.forward_mode.is_idle():
# An idle DP rank runs the extend's graph, with no rows of its own.
forward_batch = copy.copy(forward_batch)
forward_batch.forward_mode = ForwardMode.EXTEND
gather = dp_rows is not None and get_moe_a2a_backend().is_none()
# Ranks in the DP layout the MoE sees: all of them when it gathers.
layout = None if dp_rows is None else (len(dp_rows) if gather else 1)
gather_ids = gather and self._model.late_layer_vl_routing
if input_ids_global is None:
input_ids_global = input_ids
leaves, has_pre, rebuild = _flatten_state(state)
num_state = len(leaves)
live = [*leaves, positions, input_ids]
if not gather:
live.append(input_ids_global)
ids_global = (
self._ids_global_buffer(input_ids.dtype, len(dp_rows))
if gather_ids
else None
)
def body(inputs: List[torch.Tensor], batch: ForwardBatch):
rows = inputs[0].shape[0]
if not gather:
ids = inputs[num_state + 2]
else:
ids = None if ids_global is None else ids_global[: rows * layout]
residual, pre, aux = self._layer_stack(
rebuild(inputs[:num_state]),
positions=inputs[num_state],
input_ids=inputs[num_state + 1],
input_ids_global=inputs[num_state + 2],
input_ids_global=ids,
forward_batch=batch,
capture_dspark=capture_dspark,
)
return [residual, pre, *aux]
@contextmanager
def dp_layout(batch: ForwardBatch, inputs: List[torch.Tensor]):
if layout is None:
yield
return
rows = inputs[0].shape[0]
graph_count = batch.num_token_non_padded
saved = enter_late_layer_dp(batch, [rows] * layout)
if layout == 1 and graph_count is not None:
# A local MoE masks pad rows by the graph's count, not the bucket.
batch.global_num_token_non_padded = graph_count
batch.num_token_non_padded = graph_count
try:
if ids_global is not None:
dp_gather_replicate(
ids_global[: batch.global_dp_buffer_len, None],
inputs[num_state + 1][:, None].clone(),
batch,
)
yield
finally:
exit_late_layer_dp(batch, saved)
residual, pre, *aux = self.runner.run(
key=(capture_dspark, has_pre),
live=[*leaves, positions, input_ids, input_ids_global],
key=(capture_dspark, has_pre, layout, gather_ids),
live=live,
forward_batch=forward_batch,
body=body,
context=self._break_context,
batch_scope=dp_layout,
bucket_from=None if dp_rows is None else max(dp_rows),
out_rows=out_rows,
)
return residual, pre, aux
def run_idle(
self,
*,
state: HcState,
positions: torch.Tensor,
input_ids: torch.Tensor,
input_ids_global: Optional[torch.Tensor],
forward_batch: ForwardBatch,
capture_dspark: bool,
dp_rows: List[int],
) -> Tuple[torch.Tensor, Optional[torch.Tensor], List[torch.Tensor]]:
"""An idle DP rank replays the shared bucket with no rows of its own."""
return self.run(
state=state.take_rows(lambda t: t[:0]),
positions=positions[:0],
input_ids=input_ids[:0],
input_ids_global=(
None if input_ids_global is None else input_ids_global[:0]
),
forward_batch=forward_batch,
capture_dspark=capture_dspark,
dp_rows=dp_rows,
out_rows=state.residual.shape[0],
)
def _run_layers_eagerly(
self,
*,
state: HcState,
positions: torch.Tensor,
input_ids: torch.Tensor,
input_ids_global: Optional[torch.Tensor],
forward_batch: ForwardBatch,
capture_dspark: bool,
dp_rows: Optional[List[int]] = None,
) -> Tuple[torch.Tensor, Optional[torch.Tensor], List[torch.Tensor]]:
"""For a prefill-graph step whose tail has no graph to replay."""
if forward_batch.forward_mode.is_idle():
forward_batch = copy.copy(forward_batch)
forward_batch.forward_mode = ForwardMode.EXTEND
tail_batch = copy.copy(forward_batch)
tail_batch.global_num_token_non_padded_cpu = positions.shape[0]
gather = dp_rows is not None and get_moe_a2a_backend().is_none()
if input_ids_global is None or gather:
input_ids_global = input_ids if not gather else None
saved_dp = None
if dp_rows is not None:
saved_dp = enter_late_layer_dp(tail_batch, dp_rows)
try:
if gather and self._model.late_layer_vl_routing:
gathered = input_ids.new_empty((sum(dp_rows), 1))
dp_gather_replicate(gathered, input_ids[:, None].clone(), tail_batch)
input_ids_global = gathered.squeeze(-1)
with self._break_context(tail_batch):
return self._layer_stack(
state,
positions=positions,
input_ids=input_ids,
input_ids_global=input_ids_global,
forward_batch=tail_batch,
capture_dspark=capture_dspark,
)
finally:
if saved_dp is not None:
exit_late_layer_dp(tail_batch, saved_dp)
def run_in_prefill_graph(
self,
*,
state: HcState,
positions: torch.Tensor,
input_ids: torch.Tensor,
input_ids_global: Optional[torch.Tensor],
capture_dspark: bool,
block_rows: int,
source_kv_layer_id: Optional[int],
) -> Tuple[torch.Tensor, Optional[torch.Tensor], List[torch.Tensor]]:
"""Runs the tail in one break of the outer prefill graph, bridged into its
``block_rows`` block."""
leaves, _, rebuild = _flatten_state(state)
residual, pre, *aux = bcg_tail_layers_in_prefill_graph(
self,
rebuild,
capture_dspark,
block_rows,
source_kv_layer_id,
get_parallel().attn_dp_size > 1,
leaves,
positions,
input_ids,
input_ids if input_ids_global is None else input_ids_global,
)
return residual, pre, aux
def _bridge_outputs(
self, block_rows: int, outputs: List[Optional[torch.Tensor]]
) -> Tuple[Optional[torch.Tensor], ...]:
"""Block-sized bridge views of the tail rows; rows past the tail are zero."""
key = tuple(None if t is None else (t.dtype, t.shape[1:]) for t in outputs)
sets = self._bridge_buffers.setdefault(key, [])
if not sets or sets[0][0].shape[0] < block_rows:
sets.insert(
0,
[
None if t is None else t.new_zeros((block_rows, *t.shape[1:]))
for t in outputs
],
)
views = []
for buf, t in zip(sets[0], outputs):
if t is None:
views.append(None)
continue
view = buf[:block_rows]
view[: t.shape[0]].copy_(t)
view[t.shape[0] :].zero_()
views.append(view)
return tuple(views)
def _ids_global_buffer(self, dtype: torch.dtype, dp_size: int) -> torch.Tensor:
buf = self._static_ids_global.get(dtype)
if buf is None:
buf = self._static_ids_global[dtype] = torch.zeros(
(self.runner.max_rows * dp_size,), dtype=dtype, device="cuda"
)
return buf
@contextmanager
def _break_context(self, tail_batch: ForwardBatch):
outer = get_tc_piecewise_forward_context()
if outer is not None:
# Nested in a prefill-graph break: swap only the batch the breaks read.
saved_batch = outer.forward_batch
outer.forward_batch = tail_batch
try:
with _replay_tail_scope():
yield
finally:
outer.forward_batch = saved_batch
return
if self._layers is None:
self._layers = compute_attention_and_moe_layers(self._model)
layers = self._layers
@@ -256,3 +570,127 @@ class DecoderReplayTail:
residual_batch.clear(forward_batch)
state = state.materialized(model.layers[self._layer_ids[-1]].hc_cfg)
return state.residual, state.pre, aux
def _unpadded_batch(forward_batch: ForwardBatch) -> ForwardBatch:
"""The batch cut to its live requests and tokens (the graph pads both)."""
extend_lens = forward_batch.extend_seq_lens_cpu
if extend_lens is None:
return forward_batch
bs, num_tokens = len(extend_lens), int(sum(extend_lens))
view = copy.copy(forward_batch)
view.batch_size = bs
for name in (
"req_pool_indices",
"seq_lens",
"seq_lens_cpu",
"extend_seq_lens",
"extend_prefix_lens",
"extend_start_loc",
):
value = getattr(forward_batch, name, None)
if value is not None:
setattr(view, name, value[:bs])
for name in ("positions", "out_cache_loc", "input_ids"):
value = getattr(forward_batch, name, None)
if value is not None:
setattr(view, name, value[:num_tokens])
return view
def _enter_eager_tail_in_prefill_graph(
backend, graph_batch: ForwardBatch, source_kv_layer_id: Optional[int]
) -> Tuple[tuple, ForwardBatch]:
"""Switches onto exact-row tail metadata; returns the exit state and the
batch the tail runs on (the capture batch while the graph captures)."""
forward_batch = _unpadded_batch(backend.prefill_graph_replay_batch or graph_batch)
held = backend.tail_forward_metadata
backend.tail_forward_metadata = backend._build_late_layer_tail_metadata(
forward_batch
)
try:
saved = backend.enter_late_layer_tail(forward_batch)
finally:
backend.tail_forward_metadata = held
backend.tail_source_kv_layer_id = source_kv_layer_id
return saved, forward_batch
def _tail_layers_in_prefill_graph(
tail_layers: DecoderReplayTail,
rebuild: Callable[[List[torch.Tensor]], HcState],
capture_dspark: bool,
block_rows: int,
source_kv_layer_id: Optional[int],
dp: bool,
leaves: List[torch.Tensor],
positions: torch.Tensor,
input_ids: torch.Tensor,
input_ids_global: torch.Tensor,
) -> Tuple[Optional[torch.Tensor], ...]:
backend = get_attn_backend()
saved, forward_batch = _enter_eager_tail_in_prefill_graph(
backend, get_tc_piecewise_forward_context().forward_batch, source_kv_layer_id
)
# Runs between the outer graph's segments, also while it captures, so the
# inner breaks must run directly.
capture_token = bcg_module._current_capture_var.set(None)
try:
tail = backend.forward_metadata.late_layer_tail
# The graph's rows are padded to its shape; the tail indexes live rows.
num_tokens = forward_batch.input_ids.shape[0]
def rows(t: torch.Tensor) -> torch.Tensor:
return tail.rows(t[:num_tokens])
num_rows = tail.positions.shape[0]
dp_rows = None
if dp:
# Exchanged before the forward, except while the prefill graphs capture
# (every rank together), when it gathers here.
dp_rows = forward_batch.global_late_layer_num_tokens_cpu
if dp_rows is None:
counts = torch.empty(get_parallel().tp_size, dtype=torch.int64)
all_gather_single(
counts,
torch.tensor([num_rows], dtype=torch.int64),
group=get_parallel().tp_group.cpu_group,
)
dp_rows = counts.tolist()
assert dp_rows[get_parallel().attn_dp_rank] == num_rows, (
dp_rows,
num_rows,
)
tail_ids = rows(input_ids)
# A TP MoE under DP gathers every rank's IDs inside the tail layers.
if dp and get_moe_a2a_backend().is_none():
ids_global = None
else:
ids_global = rows(input_ids_global)
kwargs = dict(
state=rebuild([rows(t) for t in leaves]),
positions=tail.positions,
input_ids=tail_ids,
input_ids_global=ids_global,
forward_batch=forward_batch,
capture_dspark=capture_dspark,
dp_rows=dp_rows,
)
# Capture stream and pool are busy while the prefill graphs capture.
runner = tail_layers.runner
if (
not get_flags().dp.capturing_prefill_graph
and runner.warm
and runner.bucket_rows(max(dp_rows) if dp else num_rows) is not None
):
residual, pre, aux = tail_layers.run(**kwargs)
else:
residual, pre, aux = tail_layers._run_layers_eagerly(**kwargs)
return tail_layers._bridge_outputs(block_rows, [residual, pre, *aux])
finally:
bcg_module._current_capture_var.reset(capture_token)
backend.exit_late_layer_tail(saved, forward_batch)
backend.prefill_graph_replay_batch = None
bcg_tail_layers_in_prefill_graph = eager_on_graph(True)(_tail_layers_in_prefill_graph)