[Perf] deepseek-v4.1: Run decoder replay tail layers from CUDA graphs on eager prefill steps (#45059)
This commit is contained in:
@@ -1630,6 +1630,9 @@ class Envs:
|
||||
# Quantize the SWA fp8 KV cache from bf16-rounded values (matches
|
||||
# trainer-side QAT and the DSA-CP path) instead of fp32 registers.
|
||||
SGLANG_DSV4_USE_BF16_KV_QUANT_SOURCE = EnvBool(False)
|
||||
# Decoder SWA bounded replay: eager prefill steps with up to this many tail rows
|
||||
# run the tail layers from CUDA graphs (buckets of 128 rows); 0 keeps them eager.
|
||||
SGLANG_DSV4_DECODER_TAIL_GRAPH_MAX_ROWS = EnvInt(2048)
|
||||
# Paged KV layout of the DeepSeek-V4 family pools: "v4" (584 B/token, every
|
||||
# GPU), "v41" (the SM100 FlashMLA V4.1 formats: 528 B fp8 SWA cache, fp8 or
|
||||
# fp4 compressed caches) or "auto" (v41 on SM100, v4 elsewhere).
|
||||
|
||||
@@ -3314,8 +3314,14 @@ class DeepseekV4AttnBackend(
|
||||
from sglang.srt.model_executor.runner_backend_utils.breakable_cuda_graph.context import (
|
||||
is_in_breakable_cuda_graph,
|
||||
)
|
||||
from sglang.srt.models.deepseek_v4_replay_tail import in_replay_tail_graph
|
||||
|
||||
return self.low_ratio_prefill_graph and is_in_breakable_cuda_graph()
|
||||
# Decoder tail graphs run the low-ratio sources as an eager break.
|
||||
return (
|
||||
self.low_ratio_prefill_graph
|
||||
and is_in_breakable_cuda_graph()
|
||||
and not in_replay_tail_graph()
|
||||
)
|
||||
|
||||
def _low_ratio_compress_decode(self, layer, x, req, pos) -> None:
|
||||
# Projection layout and fused-write support are fixed together at load time.
|
||||
|
||||
@@ -0,0 +1,220 @@
|
||||
"""Breakable CUDA graphs of a layer range that runs over a varying number of rows,
|
||||
one graph per row bucket (a multiple of ``row_step``).
|
||||
|
||||
Ops of the body that read per-step state (KV slots, attention metadata) must be
|
||||
eager breaks; they read the step's real row count from
|
||||
``forward_batch.global_num_token_non_padded_cpu``.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import copy
|
||||
import logging
|
||||
import time
|
||||
from typing import TYPE_CHECKING, Callable, ContextManager, List, Optional
|
||||
|
||||
import torch
|
||||
|
||||
from sglang.srt.distributed.device_communicators.pynccl_allocator import (
|
||||
set_graph_pool_id,
|
||||
)
|
||||
from sglang.srt.distributed.parallel_state import graph_capture
|
||||
from sglang.srt.model_executor.forward_context import get_attn_backend
|
||||
from sglang.srt.model_executor.runner_backend_utils.breakable_cuda_graph import (
|
||||
BreakableCUDAGraph,
|
||||
BreakableCUDAGraphCapture,
|
||||
)
|
||||
from sglang.srt.model_executor.runner_utils.pool import (
|
||||
get_or_create_global_graph_capture_stream,
|
||||
get_or_create_global_graph_memory_pool,
|
||||
graph_pool_capture_scope,
|
||||
graph_pool_replay_scope,
|
||||
)
|
||||
from sglang.srt.runtime_context import get_parallel, get_schedule
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from sglang.srt.model_executor.forward_batch_info import ForwardBatch
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
Body = Callable[[List[torch.Tensor], "ForwardBatch"], List[Optional[torch.Tensor]]]
|
||||
|
||||
|
||||
class _Graph:
|
||||
__slots__ = ("graph", "num_token_non_padded", "outputs", "batch")
|
||||
|
||||
def __init__(self, graph, num_token_non_padded, outputs, batch):
|
||||
self.graph = graph
|
||||
# MoE top-k masks rows at or past this count; captured by address.
|
||||
self.num_token_non_padded = num_token_non_padded
|
||||
self.outputs = outputs
|
||||
# Kernels write buffers cached on the capture batch and attention metadata.
|
||||
self.batch = batch
|
||||
|
||||
|
||||
class RowBucketGraphRunner:
|
||||
def __init__(self, *, row_step: int, max_rows: int, label: str) -> None:
|
||||
self.row_step = row_step
|
||||
self.max_rows = max_rows - max_rows % row_step
|
||||
self.label = label
|
||||
self._graphs: dict[tuple, _Graph] = {}
|
||||
self._static: dict[tuple, List[torch.Tensor]] = {}
|
||||
self._warm = False
|
||||
self.capture_seconds = 0.0
|
||||
self.capture_bytes = 0
|
||||
|
||||
def ready(self, num_tokens: int) -> bool:
|
||||
"""Whether graphs may run from this step on; every extend forward reports
|
||||
its token count here. Kernels that size cached workspaces lazily by row
|
||||
count reallocate them on a bigger step, which would leave earlier captures
|
||||
reading freed memory; a full prefill chunk has grown them all."""
|
||||
if not self._warm:
|
||||
chunk = get_schedule().chunked_prefill_size
|
||||
self._warm = num_tokens >= (chunk if chunk and chunk > 0 else 8192)
|
||||
return self._warm
|
||||
|
||||
@property
|
||||
def warm(self) -> bool:
|
||||
return self._warm
|
||||
|
||||
def bucket_rows(self, num_rows: int) -> Optional[int]:
|
||||
if num_rows == 0 or num_rows > self.max_rows:
|
||||
return None
|
||||
return -(-num_rows // self.row_step) * self.row_step
|
||||
|
||||
def run(
|
||||
self,
|
||||
*,
|
||||
key: tuple,
|
||||
live: List[torch.Tensor],
|
||||
forward_batch: ForwardBatch,
|
||||
body: Body,
|
||||
context: Callable[[ForwardBatch], ContextManager],
|
||||
) -> List[Optional[torch.Tensor]]:
|
||||
"""Run ``body`` on the ``live`` rows from their bucket's graph, capturing
|
||||
it on first use. ``key`` names whatever besides row count and input shapes
|
||||
changes what ``body`` records; ``context`` wraps capture and replay with
|
||||
the batch the eager breaks read. Returns the outputs cut to the live
|
||||
rows, as views of graph buffers valid until the next replay."""
|
||||
num_rows = live[0].shape[0]
|
||||
rows = self.bucket_rows(num_rows)
|
||||
assert rows is not None, num_rows
|
||||
# Inputs of one shape share their static buffers across keys.
|
||||
input_key = tuple((t.dtype, t.shape[1:]) for t in live)
|
||||
static = self._static_inputs(input_key, live)
|
||||
for buf, t in zip(static, live):
|
||||
buf[:num_rows].copy_(t)
|
||||
graph_key = (key, input_key)
|
||||
# The eager breaks read the step's real row count off this view.
|
||||
step_batch = copy.copy(forward_batch)
|
||||
step_batch.global_num_token_non_padded_cpu = num_rows
|
||||
with context(step_batch):
|
||||
if (rows, graph_key) not in self._graphs:
|
||||
# Largest first, so smaller buckets reuse the pool memory the
|
||||
# larger ones took; a bucket below this step's rows cannot be
|
||||
# captured here (the breaks write every live row).
|
||||
for r in range(self.max_rows, rows - 1, -self.row_step):
|
||||
if (r, graph_key) not in self._graphs:
|
||||
self._graphs[(r, graph_key)] = self._capture(
|
||||
r, num_rows, static, forward_batch, body
|
||||
)
|
||||
graph = self._graphs[(rows, graph_key)]
|
||||
if graph.num_token_non_padded is not None:
|
||||
graph.num_token_non_padded.fill_(num_rows)
|
||||
with graph_pool_replay_scope():
|
||||
graph.graph.replay()
|
||||
return [None if t is None else t[:num_rows] for t in graph.outputs]
|
||||
|
||||
def _static_inputs(self, input_key, live) -> List[torch.Tensor]:
|
||||
static = self._static.get(input_key)
|
||||
if static is None:
|
||||
static = self._static[input_key] = [
|
||||
t.new_zeros((self.max_rows, *t.shape[1:])) for t in live
|
||||
]
|
||||
return static
|
||||
|
||||
def _capture(
|
||||
self,
|
||||
rows: int,
|
||||
num_rows: int,
|
||||
static: List[torch.Tensor],
|
||||
forward_batch: ForwardBatch,
|
||||
body: Body,
|
||||
) -> _Graph:
|
||||
start = time.perf_counter()
|
||||
free_before = torch.cuda.mem_get_info()[0]
|
||||
inputs = [buf[:rows] for buf in static]
|
||||
# The layers read the step's batch; the copy's count tensor is the graph's.
|
||||
# Host-side row counts read during capture cover the whole bucket.
|
||||
capture_batch = copy.copy(forward_batch)
|
||||
capture_batch.global_num_token_non_padded_cpu = rows
|
||||
num_token_non_padded = None
|
||||
if forward_batch.num_token_non_padded is not None:
|
||||
num_token_non_padded = torch.full_like(
|
||||
forward_batch.num_token_non_padded, num_rows
|
||||
)
|
||||
capture_batch.num_token_non_padded = num_token_non_padded
|
||||
return self._record(
|
||||
lambda: body(inputs, capture_batch),
|
||||
rows,
|
||||
num_token_non_padded,
|
||||
capture_batch,
|
||||
start,
|
||||
free_before,
|
||||
)
|
||||
|
||||
def _record(
|
||||
self,
|
||||
body,
|
||||
rows,
|
||||
num_token_non_padded,
|
||||
capture_batch,
|
||||
start,
|
||||
free_before,
|
||||
) -> _Graph:
|
||||
pool = get_or_create_global_graph_memory_pool(torch.cuda)
|
||||
set_graph_pool_id(pool)
|
||||
tp_group = get_parallel().tp_group
|
||||
graph = BreakableCUDAGraph()
|
||||
with graph_capture(stream=get_or_create_global_graph_capture_stream()) as ctx:
|
||||
# The warmups run eagerly; their KV writes repeat the replay's.
|
||||
for _ in range(2):
|
||||
torch.cuda.synchronize()
|
||||
tp_group.barrier()
|
||||
body()
|
||||
with (
|
||||
graph_pool_capture_scope(),
|
||||
BreakableCUDAGraphCapture(
|
||||
cuda_graph=graph,
|
||||
pool=pool,
|
||||
stream=ctx.stream,
|
||||
barrier_fn=tp_group.barrier,
|
||||
),
|
||||
):
|
||||
outputs = body()
|
||||
torch.cuda.current_stream().wait_stream(ctx.stream)
|
||||
torch.cuda.synchronize()
|
||||
recorded = _Graph(
|
||||
graph,
|
||||
num_token_non_padded,
|
||||
outputs,
|
||||
(capture_batch, get_attn_backend().forward_metadata),
|
||||
)
|
||||
seconds = time.perf_counter() - start
|
||||
used = free_before - torch.cuda.mem_get_info()[0]
|
||||
self.capture_seconds += seconds
|
||||
self.capture_bytes += used
|
||||
if get_parallel().tp_rank == 0:
|
||||
logger.info(
|
||||
"%s: captured %d rows (%d segments) in %.2f s, %.3f GiB; "
|
||||
"%d graphs, %.2f s, %.3f GiB",
|
||||
self.label,
|
||||
rows,
|
||||
len(graph._segments),
|
||||
seconds,
|
||||
used / 2**30,
|
||||
len(self._graphs) + 1,
|
||||
self.capture_seconds,
|
||||
self.capture_bytes / 2**30,
|
||||
)
|
||||
return recorded
|
||||
@@ -172,6 +172,11 @@ from sglang.srt.models.deepseek_v4_platform_stage import (
|
||||
NonCudaDecoderMixin,
|
||||
forward_layer_fused_boundary,
|
||||
)
|
||||
from sglang.srt.models.deepseek_v4_replay_tail import (
|
||||
DecoderReplayTail,
|
||||
bcg_tail_kv_store,
|
||||
in_replay_tail_graph,
|
||||
)
|
||||
from sglang.srt.models.deepseek_v41_vit import Aligner, ViT
|
||||
from sglang.srt.models.utils import WeightsMapper
|
||||
from sglang.srt.multimodal.deepseek_v41_image_processing import (
|
||||
@@ -2305,6 +2310,10 @@ class MQALayer(MqaAttentionBase):
|
||||
positions=global_positions,
|
||||
)
|
||||
kv = None
|
||||
elif in_replay_tail_graph():
|
||||
assert not fuse_q_rope
|
||||
bcg_tail_kv_store(self, x_linear, positions, qkv_a)
|
||||
kv = None
|
||||
else:
|
||||
self._compute_kv_to_cache(
|
||||
x_linear,
|
||||
@@ -2324,7 +2333,10 @@ class MQALayer(MqaAttentionBase):
|
||||
if (
|
||||
forward_batch.forward_mode.is_extend()
|
||||
and is_in_breakable_cuda_graph()
|
||||
and not getattr(attn_backend, "low_ratio_prefill_graph", False)
|
||||
and (
|
||||
not getattr(attn_backend, "low_ratio_prefill_graph", False)
|
||||
or in_replay_tail_graph()
|
||||
)
|
||||
):
|
||||
bcg_deepseek_v4_low_ratio_sources(self, x, q_lora, positions)
|
||||
else:
|
||||
@@ -3532,6 +3544,8 @@ DeepseekV4DecoderLayer = (
|
||||
|
||||
class DeepseekV4Model(nn.Module):
|
||||
fall_back_to_pt_during_load = False
|
||||
# Class-level default for instances built without running __init__.
|
||||
decoder_replay_tail: Optional[DecoderReplayTail] = None
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
@@ -3698,6 +3712,7 @@ class DeepseekV4Model(nn.Module):
|
||||
and config.vision_n_layers > 0
|
||||
and not getattr(config, "language_model_only", False)
|
||||
)
|
||||
self.decoder_replay_tail = DecoderReplayTail.maybe_create(self)
|
||||
|
||||
def get_input_embeddings(self) -> nn.Module:
|
||||
return self.embed_tokens
|
||||
@@ -3831,6 +3846,13 @@ class DeepseekV4Model(nn.Module):
|
||||
),
|
||||
in_graph=tail_in_graph,
|
||||
)
|
||||
use_tail_graph = (
|
||||
self.decoder_replay_tail is not None
|
||||
and tail is not None
|
||||
and self.decoder_replay_tail.plan(
|
||||
tail, num_tokens=input_ids.shape[0], tail_in_graph=tail_in_graph
|
||||
)
|
||||
)
|
||||
tail_sentinel = positions.new_zeros((1,)) if tail_in_graph else None
|
||||
saved_full = None
|
||||
# The HIP defer_post gate excludes residual readers. CUDA materializes
|
||||
@@ -3890,6 +3912,19 @@ class DeepseekV4Model(nn.Module):
|
||||
positions = tail.positions
|
||||
if hash_ids is not None:
|
||||
hash_ids = tail.rows(hash_ids)
|
||||
if use_tail_graph:
|
||||
residual, pre, aux = self.decoder_replay_tail.run(
|
||||
state=state,
|
||||
positions=positions,
|
||||
input_ids=input_ids,
|
||||
input_ids_global=input_ids_global,
|
||||
forward_batch=forward_batch,
|
||||
capture_dspark=capture_dspark,
|
||||
)
|
||||
dspark_aux_hidden_states.extend(aux)
|
||||
residual_batch.clear(forward_batch)
|
||||
attn_backend.exit_late_layer_tail(saved_full, forward_batch)
|
||||
return residual, pre, tail
|
||||
if late_dp_counts is not None and i == self.late_layer_start:
|
||||
saved_dp = enter_late_layer_dp(forward_batch, late_dp_counts)
|
||||
if get_moe_a2a_backend().is_none():
|
||||
|
||||
@@ -0,0 +1,258 @@
|
||||
"""DeepSeek-V4.1's decoder-replay tail layers (``late_layer_start`` on), replayed
|
||||
from `RowBucketGraphRunner` graphs on eager prefill steps. The SWA KV store, the
|
||||
attention and the low-ratio sources stay eager breaks of those graphs.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from contextlib import contextmanager, nullcontext
|
||||
from typing import TYPE_CHECKING, List, Optional, Tuple
|
||||
|
||||
import torch
|
||||
|
||||
from sglang.srt.environ import envs
|
||||
from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder
|
||||
from sglang.srt.layers.layer_boundary.residual import batch as residual_batch
|
||||
from sglang.srt.model_executor.cuda_graph_config import (
|
||||
Backend,
|
||||
Phase,
|
||||
check_cuda_graph_backend,
|
||||
)
|
||||
from sglang.srt.model_executor.forward_context import get_attn_backend
|
||||
from sglang.srt.model_executor.model_runner_components.layer_setup import (
|
||||
compute_attention_and_moe_layers,
|
||||
)
|
||||
from sglang.srt.model_executor.runner.row_bucket_graph_runner import (
|
||||
RowBucketGraphRunner,
|
||||
)
|
||||
from sglang.srt.model_executor.runner_backend_utils.breakable_cuda_graph import (
|
||||
eager_on_graph,
|
||||
enable_breakable_cuda_graph,
|
||||
)
|
||||
from sglang.srt.model_executor.runner_backend_utils.tc_piecewise_cuda_graph.context_manager import (
|
||||
get_tc_piecewise_forward_context,
|
||||
set_tc_piecewise_forward_context,
|
||||
)
|
||||
from sglang.srt.model_executor.runner_utils.capture_mode import get_is_capture_mode
|
||||
from sglang.srt.models import deepseek_v4_boundary as hc_boundary
|
||||
from sglang.srt.models.deepseek_v4_mhc import HcState
|
||||
from sglang.srt.runtime_context import get_parallel
|
||||
from sglang.srt.utils import is_cuda
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from sglang.srt.layers.attention.deepseek_v4_backend import LateLayerTail
|
||||
from sglang.srt.model_executor.forward_batch_info import ForwardBatch
|
||||
|
||||
_in_replay_tail = False
|
||||
|
||||
|
||||
def in_replay_tail_graph() -> bool:
|
||||
"""True while the tail layers capture or replay a decoder tail graph."""
|
||||
return _in_replay_tail
|
||||
|
||||
|
||||
@contextmanager
|
||||
def _replay_tail_scope():
|
||||
global _in_replay_tail
|
||||
_in_replay_tail = True
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
_in_replay_tail = False
|
||||
|
||||
|
||||
def _tail_kv_store(attention, x, positions, qkv_a) -> None:
|
||||
# The SWA store writes the step's live tail slots, so a tail graph breaks here.
|
||||
forward_batch = get_tc_piecewise_forward_context().forward_batch
|
||||
num_rows = forward_batch.global_num_token_non_padded_cpu
|
||||
if num_rows == 0:
|
||||
return
|
||||
attention._compute_kv_to_cache(
|
||||
x[:num_rows],
|
||||
positions[:num_rows],
|
||||
forward_batch,
|
||||
get_attn_backend(),
|
||||
qkv_a=None if qkv_a is None else qkv_a[:num_rows],
|
||||
)
|
||||
|
||||
|
||||
bcg_tail_kv_store = eager_on_graph(True)(_tail_kv_store)
|
||||
|
||||
|
||||
def _flatten_state(state: HcState):
|
||||
"""The state's row tensors, whether ``pre`` is among them, and the inverse.
|
||||
The normed input and mix stats are dropped (the first tail layer rebuilds
|
||||
them), so every step keys the same graphs."""
|
||||
assert isinstance(state.streams, torch.Tensor), "tail graphs are CUDA only"
|
||||
has_pre = state.pre is not None
|
||||
leaves = [state.streams, state.pre] if has_pre else [state.streams]
|
||||
|
||||
def rebuild(tensors: List[torch.Tensor]) -> HcState:
|
||||
return HcState(tensors[0], tensors[1] if has_pre else None)
|
||||
|
||||
return leaves, has_pre, rebuild
|
||||
|
||||
|
||||
class DecoderReplayTail:
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
model: torch.nn.Module,
|
||||
layer_ids: range,
|
||||
row_step: int,
|
||||
max_rows: int,
|
||||
) -> None:
|
||||
self._model = model
|
||||
self._layer_ids = layer_ids
|
||||
self._layers = None
|
||||
self.runner = RowBucketGraphRunner(
|
||||
row_step=row_step, max_rows=max_rows, label="Decoder tail graph"
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def maybe_create(cls, model: torch.nn.Module) -> Optional[DecoderReplayTail]:
|
||||
max_rows = envs.SGLANG_DSV4_DECODER_TAIL_GRAPH_MAX_ROWS.get()
|
||||
if not (
|
||||
model.tail_writes_source_kv
|
||||
and is_cuda()
|
||||
and max_rows > 0
|
||||
# Borrowed graph-pool storage forbids capturing mid-serving.
|
||||
and not envs.SGLANG_ENABLE_GRAPH_POOL_BORROW.get()
|
||||
):
|
||||
return None
|
||||
from sglang.srt.layers.attention.deepseek_v4_backend import SWA_WINDOW
|
||||
|
||||
return cls(
|
||||
model=model,
|
||||
layer_ids=range(model.late_layer_start, model.end_layer),
|
||||
row_step=SWA_WINDOW,
|
||||
max_rows=max_rows,
|
||||
)
|
||||
|
||||
def plan(
|
||||
self, tail: LateLayerTail, *, num_tokens: int, tail_in_graph: bool
|
||||
) -> bool:
|
||||
"""Whether this step runs its tail layers from a graph; also reports the
|
||||
step's token count to the warm gate."""
|
||||
from sglang.kernels.ops.attention.dsv4.unified_kv_kernels.env_gate import (
|
||||
is_unified_kv_triton,
|
||||
)
|
||||
|
||||
self.runner.ready(num_tokens)
|
||||
return not (
|
||||
tail_in_graph
|
||||
or get_parallel().attn_dp_size > 1
|
||||
or get_is_capture_mode()
|
||||
or is_unified_kv_triton()
|
||||
or get_global_expert_distribution_recorder().recording
|
||||
or not self.runner.warm
|
||||
or tail.cp_metadata is not None
|
||||
or self.runner.bucket_rows(tail.positions.shape[0]) is None
|
||||
)
|
||||
|
||||
def run(
|
||||
self,
|
||||
*,
|
||||
state: HcState,
|
||||
positions: torch.Tensor,
|
||||
input_ids: torch.Tensor,
|
||||
input_ids_global: torch.Tensor,
|
||||
forward_batch: ForwardBatch,
|
||||
capture_dspark: bool,
|
||||
) -> Tuple[torch.Tensor, Optional[torch.Tensor], List[torch.Tensor]]:
|
||||
"""Returns the tail rows' materialized residual and pre and the DSpark
|
||||
auxiliary states, as views of graph buffers valid until the next replay."""
|
||||
leaves, has_pre, rebuild = _flatten_state(state)
|
||||
num_state = len(leaves)
|
||||
|
||||
def body(inputs: List[torch.Tensor], batch: ForwardBatch):
|
||||
residual, pre, aux = self._layer_stack(
|
||||
rebuild(inputs[:num_state]),
|
||||
positions=inputs[num_state],
|
||||
input_ids=inputs[num_state + 1],
|
||||
input_ids_global=inputs[num_state + 2],
|
||||
forward_batch=batch,
|
||||
capture_dspark=capture_dspark,
|
||||
)
|
||||
return [residual, pre, *aux]
|
||||
|
||||
residual, pre, *aux = self.runner.run(
|
||||
key=(capture_dspark, has_pre),
|
||||
live=[*leaves, positions, input_ids, input_ids_global],
|
||||
forward_batch=forward_batch,
|
||||
body=body,
|
||||
context=self._break_context,
|
||||
)
|
||||
return residual, pre, aux
|
||||
|
||||
@contextmanager
|
||||
def _break_context(self, tail_batch: ForwardBatch):
|
||||
if self._layers is None:
|
||||
self._layers = compute_attention_and_moe_layers(self._model)
|
||||
layers = self._layers
|
||||
with (
|
||||
set_tc_piecewise_forward_context(
|
||||
tail_batch,
|
||||
layers.attention_layers,
|
||||
None,
|
||||
layers.moe_layers,
|
||||
layers.moe_fusions,
|
||||
dsa_indexers=layers.dsa_indexers,
|
||||
mha_companion_layers=layers.mha_companion_layers,
|
||||
),
|
||||
enable_breakable_cuda_graph(),
|
||||
_replay_tail_scope(),
|
||||
):
|
||||
yield
|
||||
|
||||
def _layer_stack(
|
||||
self,
|
||||
state: HcState,
|
||||
*,
|
||||
positions: torch.Tensor,
|
||||
input_ids: torch.Tensor,
|
||||
input_ids_global: torch.Tensor,
|
||||
forward_batch: ForwardBatch,
|
||||
capture_dspark: bool,
|
||||
) -> Tuple[torch.Tensor, Optional[torch.Tensor], List[torch.Tensor]]:
|
||||
"""The graphs' body; mirrors the model loop past the tail switch."""
|
||||
model = self._model
|
||||
aux: List[torch.Tensor] = []
|
||||
residual_batch.start(forward_batch)
|
||||
hidden_states = residual_batch.set_written(state, forward_batch)
|
||||
try:
|
||||
for i in self._layer_ids:
|
||||
layer = model.layers[i]
|
||||
assert layer.engram is None
|
||||
if capture_dspark and i in model.dspark_layers_to_capture:
|
||||
call = hc_boundary.LayerCall(input_ids, input_ids_global)
|
||||
next_boundary = (
|
||||
hc_boundary.HcRead(layer, True)
|
||||
.bind(forward_batch, call)
|
||||
.next_boundary
|
||||
)
|
||||
hidden_states = state = hc_boundary.materialize(
|
||||
hidden_states, forward_batch, next=next_boundary
|
||||
)
|
||||
# The draft head reads the attention input of its target layers.
|
||||
aux.append(state.residual.mean(dim=1))
|
||||
ctx = (
|
||||
nullcontext()
|
||||
if check_cuda_graph_backend(Phase.PREFILL, Backend.TC_PIECEWISE)
|
||||
else get_global_expert_distribution_recorder().with_current_layer(i)
|
||||
)
|
||||
with ctx:
|
||||
hidden_states = layer(
|
||||
positions=positions,
|
||||
hidden_states=hidden_states,
|
||||
input_ids=input_ids,
|
||||
forward_batch=forward_batch,
|
||||
input_ids_global=input_ids_global,
|
||||
)
|
||||
state = residual_batch.read_output(
|
||||
hidden_states, forward_batch, hc_boundary.FinalRead(), None
|
||||
)
|
||||
finally:
|
||||
residual_batch.clear(forward_batch)
|
||||
state = state.materialized(model.layers[self._layer_ids[-1]].hc_cfg)
|
||||
return state.residual, state.pre, aux
|
||||
Reference in New Issue
Block a user