[Perf] deepseek-v4.1: Run decoder replay tail graphs in prefill-graph steps and under DP attention (#45107)
This commit is contained in:
@@ -1633,6 +1633,9 @@ class Envs:
|
||||
# Decoder SWA bounded replay: eager prefill steps with up to this many tail rows
|
||||
# run the tail layers from CUDA graphs (buckets of 128 rows); 0 keeps them eager.
|
||||
SGLANG_DSV4_DECODER_TAIL_GRAPH_MAX_ROWS = EnvInt(2048)
|
||||
# Prefill-graph shapes of at least this many tokens run the tail from the tail
|
||||
# graphs in a break (measured crossover 1536-2048 tokens on GB300 TP4).
|
||||
SGLANG_DSV4_DECODER_TAIL_GRAPH_MIN_PREFILL_TOKENS = EnvInt(2048)
|
||||
# Paged KV layout of the DeepSeek-V4 family pools: "v4" (584 B/token, every
|
||||
# GPU), "v41" (the SM100 FlashMLA V4.1 formats: 528 B fp8 SWA cache, fp8 or
|
||||
# fp4 compressed caches) or "auto" (v41 on SM100, v4 elsewhere).
|
||||
|
||||
@@ -1279,6 +1279,8 @@ class DeepseekV4AttnBackend(
|
||||
self.tail_forward_metadata: Optional[DSV4Metadata] = None
|
||||
# Set by write_tail_source_kv until exit_late_layer_tail.
|
||||
self.tail_source_kv_layer_id: Optional[int] = None
|
||||
# The live batch of the prefill-graph replay; a tail break needs its exact rows.
|
||||
self.prefill_graph_replay_batch: Optional[ForwardBatch] = None
|
||||
# Prefill CUDA graph tail block (bounded replay inside the graph): row
|
||||
# capacity, persistent index/location buffers, held metadata per block size.
|
||||
self._bcg_tail_capacity: Optional[int] = None
|
||||
@@ -3081,6 +3083,7 @@ class DeepseekV4AttnBackend(
|
||||
static_metadata,
|
||||
two_level=self._prefill_graph_two_level(),
|
||||
)
|
||||
self.prefill_graph_replay_batch = forward_batch
|
||||
self.forward_metadata = capture_metadata
|
||||
self._activate_request_window()
|
||||
self.tail_forward_metadata = (
|
||||
|
||||
@@ -11,6 +11,7 @@ from __future__ import annotations
|
||||
import copy
|
||||
import logging
|
||||
import time
|
||||
from contextlib import nullcontext
|
||||
from typing import TYPE_CHECKING, Callable, ContextManager, List, Optional
|
||||
|
||||
import torch
|
||||
@@ -38,6 +39,12 @@ if TYPE_CHECKING:
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
Body = Callable[[List[torch.Tensor], "ForwardBatch"], List[Optional[torch.Tensor]]]
|
||||
# Wraps capture and replay with the body's batch and the bucket's static inputs.
|
||||
BatchScope = Callable[["ForwardBatch", List[torch.Tensor]], ContextManager]
|
||||
|
||||
|
||||
def _no_scope(batch: ForwardBatch, inputs: List[torch.Tensor]) -> ContextManager:
|
||||
return nullcontext()
|
||||
|
||||
|
||||
class _Graph:
|
||||
@@ -90,15 +97,20 @@ class RowBucketGraphRunner:
|
||||
forward_batch: ForwardBatch,
|
||||
body: Body,
|
||||
context: Callable[[ForwardBatch], ContextManager],
|
||||
batch_scope: BatchScope = _no_scope,
|
||||
bucket_from: Optional[int] = None,
|
||||
out_rows: Optional[int] = None,
|
||||
) -> List[Optional[torch.Tensor]]:
|
||||
"""Run ``body`` on the ``live`` rows from their bucket's graph, capturing
|
||||
it on first use. ``key`` names whatever besides row count and input shapes
|
||||
changes what ``body`` records; ``context`` wraps capture and replay with
|
||||
the batch the eager breaks read. Returns the outputs cut to the live
|
||||
rows, as views of graph buffers valid until the next replay."""
|
||||
it on first use. ``key`` names what besides row count and input shapes
|
||||
changes the recording; ``context`` and ``batch_scope`` wrap capture and
|
||||
replay with the breaks' and the body's batch; ``bucket_from`` picks the
|
||||
bucket from another row count. Returns views valid until the next replay,
|
||||
cut to ``out_rows`` rows."""
|
||||
num_rows = live[0].shape[0]
|
||||
rows = self.bucket_rows(num_rows)
|
||||
assert rows is not None, num_rows
|
||||
out_rows = num_rows if out_rows is None else out_rows
|
||||
rows = self.bucket_rows(num_rows if bucket_from is None else bucket_from)
|
||||
assert rows is not None, (num_rows, bucket_from)
|
||||
# Inputs of one shape share their static buffers across keys.
|
||||
input_key = tuple((t.dtype, t.shape[1:]) for t in live)
|
||||
static = self._static_inputs(input_key, live)
|
||||
@@ -116,14 +128,15 @@ class RowBucketGraphRunner:
|
||||
for r in range(self.max_rows, rows - 1, -self.row_step):
|
||||
if (r, graph_key) not in self._graphs:
|
||||
self._graphs[(r, graph_key)] = self._capture(
|
||||
r, num_rows, static, forward_batch, body
|
||||
r, num_rows, static, forward_batch, body, batch_scope
|
||||
)
|
||||
graph = self._graphs[(rows, graph_key)]
|
||||
if graph.num_token_non_padded is not None:
|
||||
graph.num_token_non_padded.fill_(num_rows)
|
||||
with graph_pool_replay_scope():
|
||||
graph.graph.replay()
|
||||
return [None if t is None else t[:num_rows] for t in graph.outputs]
|
||||
with batch_scope(step_batch, [buf[:rows] for buf in static]):
|
||||
if graph.num_token_non_padded is not None:
|
||||
graph.num_token_non_padded.fill_(num_rows)
|
||||
with graph_pool_replay_scope():
|
||||
graph.graph.replay()
|
||||
return [None if t is None else t[:out_rows] for t in graph.outputs]
|
||||
|
||||
def _static_inputs(self, input_key, live) -> List[torch.Tensor]:
|
||||
static = self._static.get(input_key)
|
||||
@@ -140,6 +153,7 @@ class RowBucketGraphRunner:
|
||||
static: List[torch.Tensor],
|
||||
forward_batch: ForwardBatch,
|
||||
body: Body,
|
||||
batch_scope: BatchScope,
|
||||
) -> _Graph:
|
||||
start = time.perf_counter()
|
||||
free_before = torch.cuda.mem_get_info()[0]
|
||||
@@ -154,14 +168,18 @@ class RowBucketGraphRunner:
|
||||
forward_batch.num_token_non_padded, num_rows
|
||||
)
|
||||
capture_batch.num_token_non_padded = num_token_non_padded
|
||||
return self._record(
|
||||
lambda: body(inputs, capture_batch),
|
||||
rows,
|
||||
num_token_non_padded,
|
||||
capture_batch,
|
||||
start,
|
||||
free_before,
|
||||
)
|
||||
with batch_scope(capture_batch, inputs):
|
||||
graph = self._record(
|
||||
lambda: body(inputs, capture_batch),
|
||||
rows,
|
||||
num_token_non_padded,
|
||||
capture_batch,
|
||||
start,
|
||||
free_before,
|
||||
)
|
||||
# The graph reads device tensors the scope put on the batch.
|
||||
graph.batch += (dict(vars(capture_batch)),)
|
||||
return graph
|
||||
|
||||
def _record(
|
||||
self,
|
||||
|
||||
@@ -825,7 +825,17 @@ def deepseek_v4_low_ratio_sources(layer, x, q_lora, positions) -> None:
|
||||
)
|
||||
|
||||
|
||||
bcg_deepseek_v4_low_ratio_sources = eager_on_graph(True)(deepseek_v4_low_ratio_sources)
|
||||
def _low_ratio_sources_capture_stub(layer, x, q_lora, positions) -> None:
|
||||
# A decoder tail graph's capture pass hands the break activations its
|
||||
# segments never computed; the indexer would select candidate blocks from
|
||||
# them and index pages past the live context. Its replay runs the body.
|
||||
if not in_replay_tail_graph():
|
||||
deepseek_v4_low_ratio_sources(layer, x, q_lora, positions)
|
||||
|
||||
|
||||
bcg_deepseek_v4_low_ratio_sources = eager_on_graph(
|
||||
deepseek_v4_low_ratio_sources, capture_stub=_low_ratio_sources_capture_stub
|
||||
)
|
||||
|
||||
|
||||
def deepseek_v4_engram_hash_ids(hasher, input_ids: torch.Tensor) -> torch.Tensor:
|
||||
@@ -3846,13 +3856,15 @@ class DeepseekV4Model(nn.Module):
|
||||
),
|
||||
in_graph=tail_in_graph,
|
||||
)
|
||||
use_tail_graph = (
|
||||
self.decoder_replay_tail is not None
|
||||
and tail is not None
|
||||
and self.decoder_replay_tail.plan(
|
||||
tail, num_tokens=input_ids.shape[0], tail_in_graph=tail_in_graph
|
||||
use_tail_graph, tail_graph_dp_rows = False, None
|
||||
if self.decoder_replay_tail is not None:
|
||||
use_tail_graph, tail_graph_dp_rows = self.decoder_replay_tail.plan(
|
||||
tail,
|
||||
forward_batch=forward_batch,
|
||||
num_tokens=input_ids.shape[0],
|
||||
tail_in_graph=tail_in_graph,
|
||||
late_dp_counts=late_dp_counts,
|
||||
)
|
||||
)
|
||||
tail_sentinel = positions.new_zeros((1,)) if tail_in_graph else None
|
||||
saved_full = None
|
||||
# The HIP defer_post gate excludes residual readers. CUDA materializes
|
||||
@@ -3867,7 +3879,10 @@ class DeepseekV4Model(nn.Module):
|
||||
if _is_cuda:
|
||||
layer = self.layers[i]
|
||||
if (
|
||||
(tail is not None and i == self.late_layer_start)
|
||||
(
|
||||
(tail is not None or use_tail_graph)
|
||||
and i == self.late_layer_start
|
||||
)
|
||||
or layer.engram is not None
|
||||
or (capture_dspark and i in self.dspark_layers_to_capture)
|
||||
):
|
||||
@@ -3899,6 +3914,31 @@ class DeepseekV4Model(nn.Module):
|
||||
# against the persistent index buffers.
|
||||
tail_sentinel = bcg_deepseek_v4_enter_late_layer_tail(tail_sentinel)
|
||||
saved_full = attn_backend.prefill_graph_late_layer_saved
|
||||
if (
|
||||
self.decoder_replay_tail is not None
|
||||
and self.decoder_replay_tail.breaks_out_of_prefill_graph(
|
||||
tail, num_tokens=input_ids.shape[0]
|
||||
)
|
||||
):
|
||||
residual, pre, aux = (
|
||||
self.decoder_replay_tail.run_in_prefill_graph(
|
||||
state=state,
|
||||
positions=positions,
|
||||
input_ids=input_ids,
|
||||
input_ids_global=input_ids_global,
|
||||
capture_dspark=capture_dspark,
|
||||
block_rows=tail.token_indices.shape[0],
|
||||
source_kv_layer_id=(
|
||||
self.late_layer_start
|
||||
if self.tail_writes_source_kv
|
||||
else None
|
||||
),
|
||||
)
|
||||
)
|
||||
dspark_aux_hidden_states.extend(aux)
|
||||
residual_batch.clear(forward_batch)
|
||||
bcg_deepseek_v4_exit_late_layer_tail(tail_sentinel)
|
||||
return residual, pre, tail
|
||||
else:
|
||||
saved_full = attn_backend.enter_late_layer_tail(forward_batch)
|
||||
state = state.take_rows(tail.rows)
|
||||
@@ -3920,11 +3960,25 @@ class DeepseekV4Model(nn.Module):
|
||||
input_ids_global=input_ids_global,
|
||||
forward_batch=forward_batch,
|
||||
capture_dspark=capture_dspark,
|
||||
dp_rows=tail_graph_dp_rows,
|
||||
)
|
||||
dspark_aux_hidden_states.extend(aux)
|
||||
residual_batch.clear(forward_batch)
|
||||
attn_backend.exit_late_layer_tail(saved_full, forward_batch)
|
||||
return residual, pre, tail
|
||||
if use_tail_graph and tail is None and i == self.late_layer_start:
|
||||
residual, pre, aux = self.decoder_replay_tail.run_idle(
|
||||
state=state,
|
||||
positions=positions,
|
||||
input_ids=input_ids,
|
||||
input_ids_global=input_ids_global,
|
||||
forward_batch=forward_batch,
|
||||
capture_dspark=capture_dspark,
|
||||
dp_rows=tail_graph_dp_rows,
|
||||
)
|
||||
dspark_aux_hidden_states.extend(aux)
|
||||
residual_batch.clear(forward_batch)
|
||||
return residual, pre, None
|
||||
if late_dp_counts is not None and i == self.late_layer_start:
|
||||
saved_dp = enter_late_layer_dp(forward_batch, late_dp_counts)
|
||||
if get_moe_a2a_backend().is_none():
|
||||
|
||||
@@ -1,23 +1,33 @@
|
||||
"""DeepSeek-V4.1's decoder-replay tail layers (``late_layer_start`` on), replayed
|
||||
from `RowBucketGraphRunner` graphs on eager prefill steps. The SWA KV store, the
|
||||
attention and the low-ratio sources stay eager breaks of those graphs.
|
||||
"""DeepSeek-V4.1's decoder-replay tail layers, run from row-bucket graphs.
|
||||
|
||||
Eager prefill steps replay them directly. A prefill-graph shape of at least
|
||||
``SGLANG_DSV4_DECODER_TAIL_GRAPH_MIN_PREFILL_TOKENS`` tokens replays them from one
|
||||
eager break, through bridge buffers; smaller shapes keep the tail captured.
|
||||
|
||||
Under DP attention every rank replays the largest rank's bucket. MegaMoE agrees
|
||||
on it through a host gather, so it needs the prefill graph off.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import copy
|
||||
from contextlib import contextmanager, nullcontext
|
||||
from typing import TYPE_CHECKING, List, Optional, Tuple
|
||||
from typing import TYPE_CHECKING, Callable, List, Optional, Tuple
|
||||
|
||||
import torch
|
||||
|
||||
from sglang.srt.distributed.utils import all_gather_single
|
||||
from sglang.srt.environ import envs
|
||||
from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder
|
||||
from sglang.srt.layers.dp_attention import dp_gather_replicate
|
||||
from sglang.srt.layers.layer_boundary.residual import batch as residual_batch
|
||||
from sglang.srt.layers.moe import get_moe_a2a_backend
|
||||
from sglang.srt.model_executor.cuda_graph_config import (
|
||||
Backend,
|
||||
Phase,
|
||||
check_cuda_graph_backend,
|
||||
)
|
||||
from sglang.srt.model_executor.forward_batch_info import ForwardMode
|
||||
from sglang.srt.model_executor.forward_context import get_attn_backend
|
||||
from sglang.srt.model_executor.model_runner_components.layer_setup import (
|
||||
compute_attention_and_moe_layers,
|
||||
@@ -25,6 +35,9 @@ from sglang.srt.model_executor.model_runner_components.layer_setup import (
|
||||
from sglang.srt.model_executor.runner.row_bucket_graph_runner import (
|
||||
RowBucketGraphRunner,
|
||||
)
|
||||
from sglang.srt.model_executor.runner_backend_utils.breakable_cuda_graph import (
|
||||
breakable_cuda_graph as bcg_module,
|
||||
)
|
||||
from sglang.srt.model_executor.runner_backend_utils.breakable_cuda_graph import (
|
||||
eager_on_graph,
|
||||
enable_breakable_cuda_graph,
|
||||
@@ -35,8 +48,12 @@ from sglang.srt.model_executor.runner_backend_utils.tc_piecewise_cuda_graph.cont
|
||||
)
|
||||
from sglang.srt.model_executor.runner_utils.capture_mode import get_is_capture_mode
|
||||
from sglang.srt.models import deepseek_v4_boundary as hc_boundary
|
||||
from sglang.srt.models.deepseek_common.decoder_swa_bounded_replay import (
|
||||
enter_late_layer_dp,
|
||||
exit_late_layer_dp,
|
||||
)
|
||||
from sglang.srt.models.deepseek_v4_mhc import HcState
|
||||
from sglang.srt.runtime_context import get_parallel
|
||||
from sglang.srt.runtime_context import get_flags, get_parallel
|
||||
from sglang.srt.utils import is_cuda
|
||||
|
||||
if TYPE_CHECKING:
|
||||
@@ -61,6 +78,20 @@ def _replay_tail_scope():
|
||||
_in_replay_tail = False
|
||||
|
||||
|
||||
def scattered_dp_counts(num_tokens: int, num_rows: int) -> Tuple[int, List[int]]:
|
||||
"""The largest step token count and every rank's tail rows, gathered on the
|
||||
host; safe only when every rank runs the step eagerly."""
|
||||
group = get_parallel().tp_group.cpu_group
|
||||
counts = torch.empty(
|
||||
2 * torch.distributed.get_world_size(group=group), dtype=torch.int64
|
||||
)
|
||||
all_gather_single(
|
||||
counts, torch.tensor([num_tokens, num_rows], dtype=torch.int64), group=group
|
||||
)
|
||||
tokens, rows = counts.view(-1, 2).T.tolist()
|
||||
return max(tokens), rows
|
||||
|
||||
|
||||
def _tail_kv_store(attention, x, positions, qkv_a) -> None:
|
||||
# The SWA store writes the step's live tail slots, so a tail graph breaks here.
|
||||
forward_batch = get_tc_piecewise_forward_context().forward_batch
|
||||
@@ -108,6 +139,9 @@ class DecoderReplayTail:
|
||||
self.runner = RowBucketGraphRunner(
|
||||
row_step=row_step, max_rows=max_rows, label="Decoder tail graph"
|
||||
)
|
||||
self._static_ids_global: dict[torch.dtype, torch.Tensor] = {}
|
||||
# Older, smaller sets stay alive for the graphs that read them.
|
||||
self._bridge_buffers: dict[tuple, List[List[Optional[torch.Tensor]]]] = {}
|
||||
|
||||
@classmethod
|
||||
def maybe_create(cls, model: torch.nn.Module) -> Optional[DecoderReplayTail]:
|
||||
@@ -130,24 +164,114 @@ class DecoderReplayTail:
|
||||
)
|
||||
|
||||
def plan(
|
||||
self, tail: LateLayerTail, *, num_tokens: int, tail_in_graph: bool
|
||||
) -> bool:
|
||||
"""Whether this step runs its tail layers from a graph; also reports the
|
||||
step's token count to the warm gate."""
|
||||
self,
|
||||
tail: Optional[LateLayerTail],
|
||||
*,
|
||||
forward_batch: ForwardBatch,
|
||||
num_tokens: int,
|
||||
tail_in_graph: bool,
|
||||
late_dp_counts: Optional[List[int]],
|
||||
) -> Tuple[bool, Optional[List[int]]]:
|
||||
"""Whether this step uses a tail graph and, under DP attention, every rank's
|
||||
tail rows; also feeds the warm gate."""
|
||||
from sglang.kernels.ops.attention.dsv4.unified_kv_kernels.env_gate import (
|
||||
is_unified_kv_triton,
|
||||
)
|
||||
|
||||
self.runner.ready(num_tokens)
|
||||
return not (
|
||||
dp = get_parallel().attn_dp_size > 1
|
||||
if not (tail is not None or (dp and forward_batch.is_extend_in_batch)):
|
||||
return False, None
|
||||
scattered = self._scattered_dp(
|
||||
tail,
|
||||
tail_in_graph=tail_in_graph,
|
||||
forward_batch=forward_batch,
|
||||
num_tokens=num_tokens,
|
||||
)
|
||||
if scattered is not None:
|
||||
self.runner.ready(scattered[0])
|
||||
else:
|
||||
counts = forward_batch.global_num_tokens_cpu
|
||||
self.runner.ready(max(counts) if dp and counts else num_tokens)
|
||||
if (
|
||||
tail_in_graph
|
||||
or get_parallel().attn_dp_size > 1
|
||||
or get_is_capture_mode()
|
||||
or is_unified_kv_triton()
|
||||
or get_global_expert_distribution_recorder().recording
|
||||
or not self.runner.warm
|
||||
or tail.cp_metadata is not None
|
||||
or self.runner.bucket_rows(tail.positions.shape[0]) is None
|
||||
or (tail is not None and tail.cp_metadata is not None)
|
||||
):
|
||||
return False, None
|
||||
if not dp:
|
||||
return (
|
||||
tail is not None
|
||||
and self.runner.bucket_rows(tail.positions.shape[0]) is not None,
|
||||
None,
|
||||
)
|
||||
dp_rows = scattered[1] if scattered is not None else None
|
||||
if dp_rows is None:
|
||||
dp_rows = self._gathered_dp_rows(late_dp_counts, forward_batch)
|
||||
if dp_rows is None or self.runner.bucket_rows(max(dp_rows)) is None:
|
||||
return False, None
|
||||
return True, dp_rows
|
||||
|
||||
def _scattered_dp(
|
||||
self,
|
||||
tail: Optional[LateLayerTail],
|
||||
*,
|
||||
tail_in_graph: bool,
|
||||
forward_batch: ForwardBatch,
|
||||
num_tokens: int,
|
||||
) -> Optional[Tuple[int, List[int]]]:
|
||||
"""`scattered_dp_counts` for a MoE routing each rank's own rows. Such a MoE
|
||||
lets ranks pick prefill graph or eager independently, so it needs the
|
||||
prefill graph disabled."""
|
||||
if (
|
||||
get_parallel().attn_dp_size == 1
|
||||
or get_moe_a2a_backend().is_none()
|
||||
or not check_cuda_graph_backend(Phase.PREFILL, Backend.DISABLED)
|
||||
or not forward_batch.is_extend_in_batch
|
||||
or tail_in_graph
|
||||
or get_is_capture_mode()
|
||||
):
|
||||
return None
|
||||
return scattered_dp_counts(
|
||||
num_tokens, 0 if tail is None else tail.positions.shape[0]
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _gathered_dp_rows(
|
||||
late_dp_counts: Optional[List[int]], forward_batch: ForwardBatch
|
||||
) -> Optional[List[int]]:
|
||||
"""Every rank's tail rows for a TP MoE, or None without a uniform layout."""
|
||||
if not get_moe_a2a_backend().is_none() or not forward_batch.is_extend_in_batch:
|
||||
return None
|
||||
# Exchanged before the forward (ForwardBatch.prepare_mlp_sync_batch).
|
||||
if forward_batch.global_late_layer_num_tokens_cpu is not None:
|
||||
return list(forward_batch.global_late_layer_num_tokens_cpu)
|
||||
if late_dp_counts is not None:
|
||||
return late_dp_counts
|
||||
# No re-layout was needed: every rank's tail is its whole extend.
|
||||
early = forward_batch.global_num_tokens_padded_cpu
|
||||
if early is None:
|
||||
early = forward_batch.global_num_tokens_cpu
|
||||
if early is None or len(early) != get_parallel().attn_dp_size:
|
||||
return None
|
||||
return list(early)
|
||||
|
||||
def breaks_out_of_prefill_graph(
|
||||
self, tail: LateLayerTail, *, num_tokens: int
|
||||
) -> bool:
|
||||
from sglang.kernels.ops.attention.dsv4.unified_kv_kernels.env_gate import (
|
||||
is_unified_kv_triton,
|
||||
)
|
||||
|
||||
return (
|
||||
num_tokens >= envs.SGLANG_DSV4_DECODER_TAIL_GRAPH_MIN_PREFILL_TOKENS.get()
|
||||
and (get_parallel().attn_dp_size == 1 or get_moe_a2a_backend().is_none())
|
||||
and tail.cp_metadata is None
|
||||
and not is_unified_kv_triton()
|
||||
# Encoder replay's folded rows are not checked against the exact tail.
|
||||
and get_attn_backend().token_to_kv_pool.request_window is None
|
||||
)
|
||||
|
||||
def run(
|
||||
@@ -156,37 +280,227 @@ class DecoderReplayTail:
|
||||
state: HcState,
|
||||
positions: torch.Tensor,
|
||||
input_ids: torch.Tensor,
|
||||
input_ids_global: torch.Tensor,
|
||||
input_ids_global: Optional[torch.Tensor],
|
||||
forward_batch: ForwardBatch,
|
||||
capture_dspark: bool,
|
||||
dp_rows: Optional[List[int]] = None,
|
||||
out_rows: Optional[int] = None,
|
||||
) -> Tuple[torch.Tensor, Optional[torch.Tensor], List[torch.Tensor]]:
|
||||
"""Returns the tail rows' materialized residual and pre and the DSpark
|
||||
auxiliary states, as views of graph buffers valid until the next replay."""
|
||||
"""Returns views of graph buffers valid until the next replay, cut to
|
||||
``out_rows`` rows. Under DP every rank replays the bucket of the largest
|
||||
``dp_rows``."""
|
||||
if forward_batch.forward_mode.is_idle():
|
||||
# An idle DP rank runs the extend's graph, with no rows of its own.
|
||||
forward_batch = copy.copy(forward_batch)
|
||||
forward_batch.forward_mode = ForwardMode.EXTEND
|
||||
gather = dp_rows is not None and get_moe_a2a_backend().is_none()
|
||||
# Ranks in the DP layout the MoE sees: all of them when it gathers.
|
||||
layout = None if dp_rows is None else (len(dp_rows) if gather else 1)
|
||||
gather_ids = gather and self._model.late_layer_vl_routing
|
||||
if input_ids_global is None:
|
||||
input_ids_global = input_ids
|
||||
leaves, has_pre, rebuild = _flatten_state(state)
|
||||
num_state = len(leaves)
|
||||
live = [*leaves, positions, input_ids]
|
||||
if not gather:
|
||||
live.append(input_ids_global)
|
||||
ids_global = (
|
||||
self._ids_global_buffer(input_ids.dtype, len(dp_rows))
|
||||
if gather_ids
|
||||
else None
|
||||
)
|
||||
|
||||
def body(inputs: List[torch.Tensor], batch: ForwardBatch):
|
||||
rows = inputs[0].shape[0]
|
||||
if not gather:
|
||||
ids = inputs[num_state + 2]
|
||||
else:
|
||||
ids = None if ids_global is None else ids_global[: rows * layout]
|
||||
residual, pre, aux = self._layer_stack(
|
||||
rebuild(inputs[:num_state]),
|
||||
positions=inputs[num_state],
|
||||
input_ids=inputs[num_state + 1],
|
||||
input_ids_global=inputs[num_state + 2],
|
||||
input_ids_global=ids,
|
||||
forward_batch=batch,
|
||||
capture_dspark=capture_dspark,
|
||||
)
|
||||
return [residual, pre, *aux]
|
||||
|
||||
@contextmanager
|
||||
def dp_layout(batch: ForwardBatch, inputs: List[torch.Tensor]):
|
||||
if layout is None:
|
||||
yield
|
||||
return
|
||||
rows = inputs[0].shape[0]
|
||||
graph_count = batch.num_token_non_padded
|
||||
saved = enter_late_layer_dp(batch, [rows] * layout)
|
||||
if layout == 1 and graph_count is not None:
|
||||
# A local MoE masks pad rows by the graph's count, not the bucket.
|
||||
batch.global_num_token_non_padded = graph_count
|
||||
batch.num_token_non_padded = graph_count
|
||||
try:
|
||||
if ids_global is not None:
|
||||
dp_gather_replicate(
|
||||
ids_global[: batch.global_dp_buffer_len, None],
|
||||
inputs[num_state + 1][:, None].clone(),
|
||||
batch,
|
||||
)
|
||||
yield
|
||||
finally:
|
||||
exit_late_layer_dp(batch, saved)
|
||||
|
||||
residual, pre, *aux = self.runner.run(
|
||||
key=(capture_dspark, has_pre),
|
||||
live=[*leaves, positions, input_ids, input_ids_global],
|
||||
key=(capture_dspark, has_pre, layout, gather_ids),
|
||||
live=live,
|
||||
forward_batch=forward_batch,
|
||||
body=body,
|
||||
context=self._break_context,
|
||||
batch_scope=dp_layout,
|
||||
bucket_from=None if dp_rows is None else max(dp_rows),
|
||||
out_rows=out_rows,
|
||||
)
|
||||
return residual, pre, aux
|
||||
|
||||
def run_idle(
|
||||
self,
|
||||
*,
|
||||
state: HcState,
|
||||
positions: torch.Tensor,
|
||||
input_ids: torch.Tensor,
|
||||
input_ids_global: Optional[torch.Tensor],
|
||||
forward_batch: ForwardBatch,
|
||||
capture_dspark: bool,
|
||||
dp_rows: List[int],
|
||||
) -> Tuple[torch.Tensor, Optional[torch.Tensor], List[torch.Tensor]]:
|
||||
"""An idle DP rank replays the shared bucket with no rows of its own."""
|
||||
return self.run(
|
||||
state=state.take_rows(lambda t: t[:0]),
|
||||
positions=positions[:0],
|
||||
input_ids=input_ids[:0],
|
||||
input_ids_global=(
|
||||
None if input_ids_global is None else input_ids_global[:0]
|
||||
),
|
||||
forward_batch=forward_batch,
|
||||
capture_dspark=capture_dspark,
|
||||
dp_rows=dp_rows,
|
||||
out_rows=state.residual.shape[0],
|
||||
)
|
||||
|
||||
def _run_layers_eagerly(
|
||||
self,
|
||||
*,
|
||||
state: HcState,
|
||||
positions: torch.Tensor,
|
||||
input_ids: torch.Tensor,
|
||||
input_ids_global: Optional[torch.Tensor],
|
||||
forward_batch: ForwardBatch,
|
||||
capture_dspark: bool,
|
||||
dp_rows: Optional[List[int]] = None,
|
||||
) -> Tuple[torch.Tensor, Optional[torch.Tensor], List[torch.Tensor]]:
|
||||
"""For a prefill-graph step whose tail has no graph to replay."""
|
||||
if forward_batch.forward_mode.is_idle():
|
||||
forward_batch = copy.copy(forward_batch)
|
||||
forward_batch.forward_mode = ForwardMode.EXTEND
|
||||
tail_batch = copy.copy(forward_batch)
|
||||
tail_batch.global_num_token_non_padded_cpu = positions.shape[0]
|
||||
gather = dp_rows is not None and get_moe_a2a_backend().is_none()
|
||||
if input_ids_global is None or gather:
|
||||
input_ids_global = input_ids if not gather else None
|
||||
saved_dp = None
|
||||
if dp_rows is not None:
|
||||
saved_dp = enter_late_layer_dp(tail_batch, dp_rows)
|
||||
try:
|
||||
if gather and self._model.late_layer_vl_routing:
|
||||
gathered = input_ids.new_empty((sum(dp_rows), 1))
|
||||
dp_gather_replicate(gathered, input_ids[:, None].clone(), tail_batch)
|
||||
input_ids_global = gathered.squeeze(-1)
|
||||
with self._break_context(tail_batch):
|
||||
return self._layer_stack(
|
||||
state,
|
||||
positions=positions,
|
||||
input_ids=input_ids,
|
||||
input_ids_global=input_ids_global,
|
||||
forward_batch=tail_batch,
|
||||
capture_dspark=capture_dspark,
|
||||
)
|
||||
finally:
|
||||
if saved_dp is not None:
|
||||
exit_late_layer_dp(tail_batch, saved_dp)
|
||||
|
||||
def run_in_prefill_graph(
|
||||
self,
|
||||
*,
|
||||
state: HcState,
|
||||
positions: torch.Tensor,
|
||||
input_ids: torch.Tensor,
|
||||
input_ids_global: Optional[torch.Tensor],
|
||||
capture_dspark: bool,
|
||||
block_rows: int,
|
||||
source_kv_layer_id: Optional[int],
|
||||
) -> Tuple[torch.Tensor, Optional[torch.Tensor], List[torch.Tensor]]:
|
||||
"""Runs the tail in one break of the outer prefill graph, bridged into its
|
||||
``block_rows`` block."""
|
||||
leaves, _, rebuild = _flatten_state(state)
|
||||
residual, pre, *aux = bcg_tail_layers_in_prefill_graph(
|
||||
self,
|
||||
rebuild,
|
||||
capture_dspark,
|
||||
block_rows,
|
||||
source_kv_layer_id,
|
||||
get_parallel().attn_dp_size > 1,
|
||||
leaves,
|
||||
positions,
|
||||
input_ids,
|
||||
input_ids if input_ids_global is None else input_ids_global,
|
||||
)
|
||||
return residual, pre, aux
|
||||
|
||||
def _bridge_outputs(
|
||||
self, block_rows: int, outputs: List[Optional[torch.Tensor]]
|
||||
) -> Tuple[Optional[torch.Tensor], ...]:
|
||||
"""Block-sized bridge views of the tail rows; rows past the tail are zero."""
|
||||
key = tuple(None if t is None else (t.dtype, t.shape[1:]) for t in outputs)
|
||||
sets = self._bridge_buffers.setdefault(key, [])
|
||||
if not sets or sets[0][0].shape[0] < block_rows:
|
||||
sets.insert(
|
||||
0,
|
||||
[
|
||||
None if t is None else t.new_zeros((block_rows, *t.shape[1:]))
|
||||
for t in outputs
|
||||
],
|
||||
)
|
||||
views = []
|
||||
for buf, t in zip(sets[0], outputs):
|
||||
if t is None:
|
||||
views.append(None)
|
||||
continue
|
||||
view = buf[:block_rows]
|
||||
view[: t.shape[0]].copy_(t)
|
||||
view[t.shape[0] :].zero_()
|
||||
views.append(view)
|
||||
return tuple(views)
|
||||
|
||||
def _ids_global_buffer(self, dtype: torch.dtype, dp_size: int) -> torch.Tensor:
|
||||
buf = self._static_ids_global.get(dtype)
|
||||
if buf is None:
|
||||
buf = self._static_ids_global[dtype] = torch.zeros(
|
||||
(self.runner.max_rows * dp_size,), dtype=dtype, device="cuda"
|
||||
)
|
||||
return buf
|
||||
|
||||
@contextmanager
|
||||
def _break_context(self, tail_batch: ForwardBatch):
|
||||
outer = get_tc_piecewise_forward_context()
|
||||
if outer is not None:
|
||||
# Nested in a prefill-graph break: swap only the batch the breaks read.
|
||||
saved_batch = outer.forward_batch
|
||||
outer.forward_batch = tail_batch
|
||||
try:
|
||||
with _replay_tail_scope():
|
||||
yield
|
||||
finally:
|
||||
outer.forward_batch = saved_batch
|
||||
return
|
||||
if self._layers is None:
|
||||
self._layers = compute_attention_and_moe_layers(self._model)
|
||||
layers = self._layers
|
||||
@@ -256,3 +570,127 @@ class DecoderReplayTail:
|
||||
residual_batch.clear(forward_batch)
|
||||
state = state.materialized(model.layers[self._layer_ids[-1]].hc_cfg)
|
||||
return state.residual, state.pre, aux
|
||||
|
||||
|
||||
def _unpadded_batch(forward_batch: ForwardBatch) -> ForwardBatch:
|
||||
"""The batch cut to its live requests and tokens (the graph pads both)."""
|
||||
extend_lens = forward_batch.extend_seq_lens_cpu
|
||||
if extend_lens is None:
|
||||
return forward_batch
|
||||
bs, num_tokens = len(extend_lens), int(sum(extend_lens))
|
||||
view = copy.copy(forward_batch)
|
||||
view.batch_size = bs
|
||||
for name in (
|
||||
"req_pool_indices",
|
||||
"seq_lens",
|
||||
"seq_lens_cpu",
|
||||
"extend_seq_lens",
|
||||
"extend_prefix_lens",
|
||||
"extend_start_loc",
|
||||
):
|
||||
value = getattr(forward_batch, name, None)
|
||||
if value is not None:
|
||||
setattr(view, name, value[:bs])
|
||||
for name in ("positions", "out_cache_loc", "input_ids"):
|
||||
value = getattr(forward_batch, name, None)
|
||||
if value is not None:
|
||||
setattr(view, name, value[:num_tokens])
|
||||
return view
|
||||
|
||||
|
||||
def _enter_eager_tail_in_prefill_graph(
|
||||
backend, graph_batch: ForwardBatch, source_kv_layer_id: Optional[int]
|
||||
) -> Tuple[tuple, ForwardBatch]:
|
||||
"""Switches onto exact-row tail metadata; returns the exit state and the
|
||||
batch the tail runs on (the capture batch while the graph captures)."""
|
||||
forward_batch = _unpadded_batch(backend.prefill_graph_replay_batch or graph_batch)
|
||||
held = backend.tail_forward_metadata
|
||||
backend.tail_forward_metadata = backend._build_late_layer_tail_metadata(
|
||||
forward_batch
|
||||
)
|
||||
try:
|
||||
saved = backend.enter_late_layer_tail(forward_batch)
|
||||
finally:
|
||||
backend.tail_forward_metadata = held
|
||||
backend.tail_source_kv_layer_id = source_kv_layer_id
|
||||
return saved, forward_batch
|
||||
|
||||
|
||||
def _tail_layers_in_prefill_graph(
|
||||
tail_layers: DecoderReplayTail,
|
||||
rebuild: Callable[[List[torch.Tensor]], HcState],
|
||||
capture_dspark: bool,
|
||||
block_rows: int,
|
||||
source_kv_layer_id: Optional[int],
|
||||
dp: bool,
|
||||
leaves: List[torch.Tensor],
|
||||
positions: torch.Tensor,
|
||||
input_ids: torch.Tensor,
|
||||
input_ids_global: torch.Tensor,
|
||||
) -> Tuple[Optional[torch.Tensor], ...]:
|
||||
backend = get_attn_backend()
|
||||
saved, forward_batch = _enter_eager_tail_in_prefill_graph(
|
||||
backend, get_tc_piecewise_forward_context().forward_batch, source_kv_layer_id
|
||||
)
|
||||
# Runs between the outer graph's segments, also while it captures, so the
|
||||
# inner breaks must run directly.
|
||||
capture_token = bcg_module._current_capture_var.set(None)
|
||||
try:
|
||||
tail = backend.forward_metadata.late_layer_tail
|
||||
# The graph's rows are padded to its shape; the tail indexes live rows.
|
||||
num_tokens = forward_batch.input_ids.shape[0]
|
||||
|
||||
def rows(t: torch.Tensor) -> torch.Tensor:
|
||||
return tail.rows(t[:num_tokens])
|
||||
|
||||
num_rows = tail.positions.shape[0]
|
||||
dp_rows = None
|
||||
if dp:
|
||||
# Exchanged before the forward, except while the prefill graphs capture
|
||||
# (every rank together), when it gathers here.
|
||||
dp_rows = forward_batch.global_late_layer_num_tokens_cpu
|
||||
if dp_rows is None:
|
||||
counts = torch.empty(get_parallel().tp_size, dtype=torch.int64)
|
||||
all_gather_single(
|
||||
counts,
|
||||
torch.tensor([num_rows], dtype=torch.int64),
|
||||
group=get_parallel().tp_group.cpu_group,
|
||||
)
|
||||
dp_rows = counts.tolist()
|
||||
assert dp_rows[get_parallel().attn_dp_rank] == num_rows, (
|
||||
dp_rows,
|
||||
num_rows,
|
||||
)
|
||||
tail_ids = rows(input_ids)
|
||||
# A TP MoE under DP gathers every rank's IDs inside the tail layers.
|
||||
if dp and get_moe_a2a_backend().is_none():
|
||||
ids_global = None
|
||||
else:
|
||||
ids_global = rows(input_ids_global)
|
||||
kwargs = dict(
|
||||
state=rebuild([rows(t) for t in leaves]),
|
||||
positions=tail.positions,
|
||||
input_ids=tail_ids,
|
||||
input_ids_global=ids_global,
|
||||
forward_batch=forward_batch,
|
||||
capture_dspark=capture_dspark,
|
||||
dp_rows=dp_rows,
|
||||
)
|
||||
# Capture stream and pool are busy while the prefill graphs capture.
|
||||
runner = tail_layers.runner
|
||||
if (
|
||||
not get_flags().dp.capturing_prefill_graph
|
||||
and runner.warm
|
||||
and runner.bucket_rows(max(dp_rows) if dp else num_rows) is not None
|
||||
):
|
||||
residual, pre, aux = tail_layers.run(**kwargs)
|
||||
else:
|
||||
residual, pre, aux = tail_layers._run_layers_eagerly(**kwargs)
|
||||
return tail_layers._bridge_outputs(block_rows, [residual, pre, *aux])
|
||||
finally:
|
||||
bcg_module._current_capture_var.reset(capture_token)
|
||||
backend.exit_late_layer_tail(saved, forward_batch)
|
||||
backend.prefill_graph_replay_batch = None
|
||||
|
||||
|
||||
bcg_tail_layers_in_prefill_graph = eager_on_graph(True)(_tail_layers_in_prefill_graph)
|
||||
|
||||
Reference in New Issue
Block a user