[Refactor] deepseek-v4: Use explicit stage boundaries and row transport (#43607)

This commit is contained in:
Cheng Wan
2026-10-15 02:05:28 -07:00
committed by GitHub
parent 3a835cf417
commit f7ad4345b8
33 changed files with 3896 additions and 1237 deletions
@@ -63,7 +63,10 @@ class OperationsStrategy:
for layer in layers
]
)
elif layer_name == "DeepseekV4DecoderLayer":
# Other model families do not need to import the V4 implementation.
from sglang.srt.models.deepseek_v4 import DeepseekV4DecoderLayerBase
if isinstance(layers[0], DeepseekV4DecoderLayerBase):
return OperationsStrategy.concat(
[
_compute_moe_deepseek_v4_layer_operations_strategy_tbo(
@@ -72,8 +75,7 @@ class OperationsStrategy:
for layer in layers
]
)
else:
raise NotImplementedError
raise NotImplementedError
def _assert_all_same(items: List):
@@ -32,6 +32,7 @@ from sglang.srt.layers.layer_boundary.contracts import (
CpMoves,
EdgeContract,
EntryPath,
ExitFacts,
ProducerReduction,
StageKind,
StagePath,
@@ -126,6 +127,7 @@ class StagePlan:
is_branch=False,
terminal=False,
writes_at_handoff=False,
row_transport=None,
finishes_directly=False,
qkv_latent_func=None,
fusions=None,
@@ -201,9 +203,15 @@ class StagePlan:
returns_over_dp=out.returns_over_dp,
writes_at_handoff=writes_at_handoff,
)
self.paths[variant] = msgspec.structs.replace(
path, exit=exit_facts(kind, self, variant, path)
fixed_exit = (
ExitFacts(
writes_residual=path.output.update.applied_at_exit
or writes_at_handoff
)
if row_transport is not None
else exit_facts(kind, self, variant, path)
)
self.paths[variant] = msgspec.structs.replace(path, exit=fixed_exit)
@property
def incoming_residual_rows(self):
@@ -278,10 +286,16 @@ def _bind_stage(
if incoming.entries.keys() != outgoing.exits.keys():
raise ValueError("incoming and outgoing batch variants disagree")
update = declaration.update
if pipeline_handoff and not (update.is_plain_add or update.applied_at_exit):
if pipeline_handoff and not (
update.is_plain_add
or update.applied_at_exit
or (declaration.writes_at_handoff and declaration.row_transport is not None)
):
raise NotImplementedError(
"pipeline boundaries require a producer-written residual"
)
if declaration.row_transport is not None and declaration.row_transport.bind is None:
raise ValueError("a local stage requires its compute transport")
if declaration.sum_impl is not None and declaration.sum_impl.run is None:
raise ValueError("a local stage requires its custom sum implementation")
if declaration.output_complete and declaration.sum_impl is not None:
@@ -311,7 +325,10 @@ def _bind_stage(
"an update applied at the stage's exit with a gather over attention CP"
)
attn_input_adapter = None
if declaration.kind is StageKind.ATTENTION:
if (
declaration.kind is StageKind.ATTENTION
and declaration.row_transport is None
):
# On an input-scattered batch the step gathers the rows itself for
# an attention whose QKV hook does not gather them after the
# projection.
@@ -332,6 +349,7 @@ def _bind_stage(
is_branch=declaration.prepared_from is not None,
terminal=declaration.terminal,
writes_at_handoff=declaration.writes_at_handoff,
row_transport=declaration.row_transport,
attn_tp_gather=declaration.attn_tp_gather,
# The exit runs its consumer's gather: the next stage's, or the
# final read's.
@@ -314,6 +314,7 @@ class ExitFacts(msgspec.Struct, frozen=True):
is not one all-reduce, may still be left to a fused consumer.
"""
writes_residual: bool = False
may_defer_sum: bool = False
defers_mixer_sum: bool = False
sum_in_reduce_scatter: bool = False
@@ -55,6 +55,7 @@ from sglang.srt.layers.layer_boundary.residual.add_norm import (
NORM_READOUT,
PLAIN_ADD,
)
from sglang.srt.layers.layer_boundary.row_transport import RowTransport
from sglang.srt.layers.layer_boundary.stage import StageBoundary
from sglang.srt.layers.moe import is_moe_input_scattered_across_dp_ranks
from sglang.srt.runtime_context import get_exec, get_parallel
@@ -305,6 +306,9 @@ class StageDeclaration:
sum_impl: Optional implementation of the declared output sum using
the model's custom collective. Its batch predicate, when supplied,
preserves eligibility for carrying that sum to the consumer.
row_transport: Explicit source, compute and return rows for a stage
whose operand moves independently of its residual. The bound
invocation owns output completion and inverse transport.
attn_tp_gather: Optional implementation of the gather over attention
TP that brings rows into this stage: at its own entry, or at its
producer's exit, which runs its consumer's gather. Given this rank's
@@ -340,6 +344,7 @@ class StageDeclaration:
output_complete: bool = False
attn_tp_gather: Optional[Callable] = None
sum_impl: Optional[SumImplementation] = None
row_transport: Optional[RowTransport] = None
# Only declarations participate in construction, never executable stages.
previous: Optional[StageDeclaration] = None
prepared_from: Optional[StageDeclaration] = None
@@ -436,6 +441,7 @@ def declare_attn(
attn_tp_gather=None,
in_token_order=False,
sum_impl=None,
row_transport=None,
):
"""Declare attention or a mixer; construct its executable boundary later.
@@ -479,6 +485,7 @@ def declare_attn(
attn_tp_gather=attn_tp_gather,
in_token_order=in_token_order,
sum_impl=sum_impl,
row_transport=row_transport,
)
@@ -493,6 +500,7 @@ def declare_ffn(
exit_rows=None,
output_complete=False,
sum_impl=None,
row_transport=None,
):
"""Declare a dense or MoE FFN independently of its compute module.
@@ -526,6 +534,7 @@ def declare_ffn(
exit_rows=exit_rows or tbo_exit_rows(sparse, next_layer_sparse),
output_complete=output_complete,
sum_impl=sum_impl,
row_transport=row_transport,
)
@@ -620,6 +629,39 @@ def _resolve_stage(stage, variant, arrival, following=None, *, final_read=None):
and the stage after it (None when it ends the stack or a branch)."""
axes, attention, local, full = _row_layouts(variant)
_reject_unsupported(stage)
if stage.row_transport is not None:
if (
stage.sum_impl is not None
or stage.output_transform is not None
or stage.output_complete
or stage.in_token_order
):
raise NotImplementedError(
"operand transport cannot bypass another output contract"
)
if stage.exit_rows not in (None, ExitRows.ATTENTION):
raise NotImplementedError(
"operand transport declares its return rows explicitly"
)
try:
rows = stage.row_transport.rows[variant]
except KeyError:
raise NotImplementedError(
"operand transport does not support this batch variant"
) from None
if rows.group is not None and not isinstance(rows.group, SumGroup):
raise TypeError("operand transport requires a declared SumGroup")
if rows.decode_group is not None and not isinstance(
rows.decode_group, SumGroup
):
raise TypeError("operand transport requires a declared decode SumGroup")
if rows.source != arrival.produced.layout or rows.returned != rows.source:
raise NotImplementedError("compute transport must return its source rows")
declaration = StageContract(
InputContract(rows.source, read=stage.read),
OutputContract(rows.returned, group=rows.group, update=stage.update),
)
return _Flow(declaration, arrival, arrival.residual, rows.returned)
if stage.kind is StageKind.FFN:
declaration, residual, returned = _resolve_ffn(
variant,
@@ -811,8 +853,13 @@ def _connect_line(
# their row choices still determine what crosses the pipeline boundary.
needs_unpadded = any(
stage is not None
and stage.kind is StageKind.FFN
and _ffn_on_rank_rows(stage.sparse, stage.dense_tp_size)
and (
stage.row_transport is not None
or (
stage.kind is StageKind.FFN
and _ffn_on_rank_rows(stage.sparse, stage.dense_tp_size)
)
)
for stage in (before, *line, after)
)
variants = tuple(
@@ -1074,8 +1121,9 @@ def _handed_off(declaration):
return replace(
declaration,
exit_rows=ExitRows.ATTENTION,
writes_at_handoff=_ffn_on_rank_rows(
declaration.sparse, declaration.dense_tp_size
writes_at_handoff=(
declaration.row_transport is not None
or _ffn_on_rank_rows(declaration.sparse, declaration.dense_tp_size)
),
)
@@ -114,6 +114,9 @@ def facts_of(declaration):
sum_impl=declaration.sum_impl.facts()
if declaration.sum_impl is not None
else None,
row_transport=declaration.row_transport.facts()
if declaration.row_transport is not None
else None,
previous=None,
prepared_from=None,
)
@@ -48,9 +48,11 @@ def clear(forward_batch):
"""Release invocation state on completion or an exception."""
stream = forward_batch.residual_stream
if stream is not None:
if stream.transport is not None:
stream.transport[1].release()
if stream.pending is not None:
stream.pending.release()
stream.pending = stream.residual = None
stream.pending = stream.residual = stream.transport = None
forward_batch.residual_stream = None
@@ -101,11 +101,12 @@ class ResidualStream:
selection live on the bound stage, not on tensor values.
"""
__slots__ = ("residual", "pending")
__slots__ = ("residual", "pending", "transport")
def __init__(self, residual=None):
self.residual = residual
self.pending = None
self.transport = None
@classmethod
def from_handoff(cls, hidden, residual, update):
@@ -0,0 +1,161 @@
"""Transport a compute operand without moving its source residual."""
from contextlib import nullcontext
from typing import Callable, Mapping, Optional
import msgspec
from sglang.srt.layers.layer_boundary.contracts import BatchVariant
from sglang.srt.layers.layer_boundary.layout import Layout, SumGroup, _sum_group
from sglang.srt.layers.layer_boundary.output import (
DeferredFinalize,
UnreducedOutput,
complete_owed,
)
from sglang.srt.layers.layer_boundary.residual import bind_update
class TransportRows(msgspec.Struct, frozen=True):
"""The source/read, compute, and completed output rows of one adapter.
The residual stays on source rows. ``group`` names the compute output's
outstanding sum; None means the computation (e.g. EP combine) completes it.
The boundary completes that sum and returns source rows, or publishes an
explicit completion handle on source rows. It never applies the update.
"""
source: Layout
compute: Layout
returned: Layout
group: Optional[SumGroup] = None
decode_group: Optional[SumGroup] = None
class RowTransport(msgspec.Struct, frozen=True):
"""Declare separate compute/residual rows and bind one invocation's moves.
``bind(input, batch, context, rows, overlapped_rows)`` returns an invocation
with ``input`` and ``scope()``. The boundary checks its selected rows,
prepares its output, completes owed work and runs its return move.
Facts retain row contracts only.
"""
rows: Mapping[BatchVariant, TransportRows]
bind: Optional[Callable] = None
overlapped_rows: Optional[Mapping[BatchVariant, TransportRows]] = None
def facts(self):
return RowTransport(
dict(self.rows),
overlapped_rows=(
dict(self.overlapped_rows) if self.overlapped_rows is not None else None
),
)
class ComputeTransport:
"""One prepared operand with declared rows and output completion policy.
``prepare_output`` exposes the compute result's outstanding work.
The boundary completes it before ``return_output`` moves the result back;
a move that performs the sum declares ``sum_in_return`` before compute.
Residual update and consumer normalization stay with the boundary.
"""
@staticmethod
def scope():
return nullcontext()
is_overlapped = False
defers_sum = False
sum_in_return = False
def release(self):
pass
def prepare_output(self, output):
return output
def return_output(self, output):
return output
def validate_invocation(invocation, ordinary, overlapped):
expected = overlapped if invocation.is_overlapped else ordinary
if expected is None or invocation.rows != expected:
raise ValueError("compute transport does not match its declared rows and sum")
if invocation.defers_sum and (
invocation.sum_in_return or expected.compute != expected.returned
):
raise ValueError("a deferred transport sum must already be on its return rows")
if invocation.sum_in_return and expected.group is None:
raise ValueError("a transport move cannot complete an undeclared sum")
class TransportExit:
"""Consume a prepared transport once, then publish the residual update."""
def __init__(self, stage, batch, *, update_context=None):
self.stage = stage
self.batch = batch
self.stream = batch.residual_stream
owner, self.invocation = self.stream.transport
if owner is not stage:
raise RuntimeError("compute transport belongs to another stage")
self.path = stage.plan.path_for(batch)
self.update = bind_update(self.path.output.update, batch, update_context)
self.publish = (
TransportExit._write
if self.path.exit.writes_residual
else TransportExit._record
)
self.scope = self.invocation.scope()
self.used = False
def __enter__(self):
try:
self.scope.__enter__()
except BaseException:
self.stream.transport = None
raise
return self
def __exit__(self, *exc):
try:
result = self.scope.__exit__(*exc)
finally:
self.invocation.release()
self.stream.transport = None
if exc[0] is None and not self.used:
raise RuntimeError("compute transport exited without finish")
return result
def finish(self, output):
if self.used:
raise RuntimeError("compute transport was already consumed")
self.used = True
output = self.invocation.prepare_output(output)
rows = self.invocation.rows
groups = tuple(
group for group in (rows.group, rows.decode_group) if group is not None
)
if isinstance(output, UnreducedOutput):
if output.complete_sum is not None or output.reduce_to_dp_local is not None:
raise ValueError("operand transport requires its declared sum group")
if not any(_sum_group(group) is output.group for group in groups):
raise ValueError("transport output owes an undeclared sum group")
elif (
isinstance(output, DeferredFinalize)
and output.completion_group not in groups
):
raise ValueError("transport finalize owes an undeclared sum group")
if not self.invocation.defers_sum or self.path.exit.writes_residual:
output = complete_owed(output)
output = self.invocation.return_output(output)
return self.publish(self, output)
def _write(self, output):
return self.stream.write(self.update.update(output, self.stream.residual))
def _record(self, output):
return self.stream.record(output, self.update)
@@ -40,6 +40,11 @@ from sglang.srt.layers.layer_boundary.residual.add_norm import (
)
from sglang.srt.layers.layer_boundary.residual.batch import stream_of
from sglang.srt.layers.layer_boundary.residual.stream import DeclaredSum, ResidualStream
from sglang.srt.layers.layer_boundary.row_transport import (
ComputeTransport,
TransportExit,
validate_invocation,
)
from sglang.srt.model_executor.forward_batch_info import ForwardBatch
from sglang.srt.runtime_context import get_forward
@@ -96,6 +101,7 @@ class StageBoundary:
(``call``: what the stage's read takes, e.g. the attention's
``quant_format``)."""
entry = self.entry(forward_batch, steps)
context = call.get("read_context")
if (
stream.pending is None
and stream.residual is None
@@ -123,6 +129,25 @@ class StageBoundary:
hidden_states, forward_batch, self.plan.qkv_latent_func
)
stream.write(residual)
transport = self.declaration.row_transport
if transport is not None:
if stream.transport is not None:
raise RuntimeError("cannot replace an unconsumed compute transport")
variant = self.plan.variant_for(forward_batch)
ordinary = transport.rows[variant]
overlapped = (
transport.overlapped_rows.get(variant)
if transport.overlapped_rows is not None
else None
)
invocation = transport.bind(
hidden_states, forward_batch, context, ordinary, overlapped
)
if not isinstance(invocation, ComputeTransport):
raise TypeError("row transport must bind a ComputeTransport invocation")
validate_invocation(invocation, ordinary, overlapped)
stream.transport = (self, invocation)
hidden_states = invocation.input
return hidden_states, stream
@property
@@ -175,6 +200,9 @@ class StageBoundary:
actual contribution, or an FFN's or a mixer's output with its sum
completed or carried on. An FFN whose boundary has fusions (which may
take a MoE finalize handoff) finishes through exit() instead."""
if self.declaration.row_transport is not None:
with self.exit(forward_batch, update_context=update_context) as output:
return output.finish(hidden_states)
if self.kind is StageKind.ATTENTION and self.plan.finishes_directly:
produced = self.plan.path_for(forward_batch).output
completion = (
@@ -201,6 +229,13 @@ class StageBoundary:
hidden_states
)
def transport_context(self, forward_batch):
"""Borrow this prepared compute's operand transport until finish()."""
stream = stream_of(forward_batch)
if stream.transport is None or stream.transport[0] is not self:
raise RuntimeError("no prepared compute transport for this stage")
return stream.transport[1]
def exit(self, forward_batch, *, update_context=None):
"""Select and scope one FFN or single-stage mixer's output decision,
for an FFN whose MoE may hand off its finalize (see finish()).
@@ -215,6 +250,8 @@ class StageBoundary:
when compute is skipped for an empty input. It completes or carries work
using the same decision rather than selecting a second path.
"""
if self.declaration.row_transport is not None:
return TransportExit(self, forward_batch, update_context=update_context)
result = (
self.plan.output.ffn_exit(
forward_batch,
@@ -2155,7 +2155,7 @@ def enable_num_token_non_padded():
return True
# DP attention + TP MoE gathers each rank's whole padded chunk into the MoE,
# so the model needs the real count to zero the pad rows (see
# deepseek_v4 _run_moe_ffn_dp_sync).
# deepseek_common.moe_transport).
return _dp_tp_moe_masks_pad_rows()
@@ -11,8 +11,6 @@ from sglang.srt.batch_invariant_ops import is_batch_invariant_mode_enabled
from sglang.srt.environ import envs
from sglang.srt.layers.moe import get_moe_a2a_backend
from sglang.srt.layers.moe.utils import should_skip_post_experts_all_reduce
from sglang.srt.models.deepseek_v2 import MoEOutput
from sglang.srt.models.deepseek_v4_mhc import AttnOutput
from sglang.srt.runtime_context import (
get_exec,
get_forward,
@@ -572,83 +570,6 @@ def apply_moe_mhc(x: torch.Tensor, state: MhcPostOperands) -> torch.Tensor:
)
def forward_hc_pre_from_prev_fused_boundary(
layer,
positions: torch.Tensor,
hidden_states: Optional[torch.Tensor],
input_ids: torch.Tensor,
forward_batch,
input_ids_global: torch.Tensor,
prev_pre: Optional[torch.Tensor],
pending_post: Optional[Tuple[torch.Tensor, ...]],
defer_post: bool,
) -> Tuple[Optional[torch.Tensor], torch.Tensor, Optional[Tuple[torch.Tensor, ...]]]:
"""ROCm form of DeepseekV4DecoderLayer.forward_hc_pre_from_prev.
pending_post is the previous layer's unapplied FFN hc_post (x, residual, post,
comb); with defer_post this layer's is returned the same way and hidden_states is None."""
if pending_post is None:
pending_post = (None, hidden_states, None, None)
residual, x, attn_coefficients = hc_boundary(
layer,
*pending_post,
prev_pre,
layer.hc_attn_fn,
layer.hc_attn_scale,
layer.hc_attn_base,
)
# the boundary's reduce + sinkhorn rides in the norm launch
x, x_quant = layer._input_norm(
x, allow_aiter_quant=False, coefficients=attn_coefficients
)
with layer.self_attn.maybe_use_decode_attn_tp(forward_batch):
mhc = attention_mhc_fusion(layer, residual, attn_coefficients, forward_batch)
x = layer.self_attn(
x=x,
positions=positions,
forward_batch=forward_batch,
x_quant=x_quant,
defer_all_reduce=mhc is not None,
)
if mhc is not None:
# wo_b honors the deferred reduce under attention_mhc_fusion's gates;
# fail loud rather than fall back to an unfused reduce.
assert isinstance(x, AttnOutput)
residual = apply_attention_mhc(x.partial, mhc)
x = None
residual, x, ffn_coefficients = hc_boundary(
layer,
x,
residual,
attn_coefficients.post if x is not None else None,
attn_coefficients.comb if x is not None else None,
attn_coefficients.pre,
layer.hc_ffn_fn,
layer.hc_ffn_scale,
layer.hc_ffn_base,
)
x = _gfx95_dense_post_attention_norm(layer, x, ffn_coefficients)
mhc = moe_mhc_fusion(layer, residual, ffn_coefficients, forward_batch)
x = layer._run_moe_ffn_dp_sync(
x,
forward_batch,
input_ids=input_ids,
input_ids_global=input_ids_global,
return_moe_output=mhc is not None,
)
ffn_pre, ffn_post, ffn_comb = ffn_coefficients.tensors()
if mhc is not None:
# The MoE TP path honors the deferred output under moe_mhc_fusion's gates;
# fail loud rather than fall back to an unfused reduce. The fused post
# carries the reduce, so take the merged rows without one.
assert isinstance(x, MoEOutput)
# Reduction already applied post. The next boundary consumes this
# materialized residual, including when it would normally defer post.
return apply_moe_mhc(x.get_merged(), mhc), ffn_pre, None
if defer_post:
return None, ffn_pre, (x, residual, ffn_post, ffn_comb)
return layer.hc_post(x, residual, ffn_post, ffn_comb), ffn_pre, None
def _gfx95_dense_post_attention_norm(layer, x: torch.Tensor, coefficients):
"""layer.post_attention_layernorm(x) hosting the pending reduce + sinkhorn where
the gfx950 norm kernel is available; the module norm plus a standalone reduce +
@@ -21,7 +21,6 @@ from sglang.srt.model_executor.runner_backend_utils.breakable_cuda_graph.context
from sglang.srt.models.deepseek_common.amd import deepseek_v4_gfx95_dense as gfx95_dense
from sglang.srt.models.deepseek_common.amd.deepseek_v4_fused_mhc import ( # noqa: F401 deepseek_v4 reaches apply_attention_mhc through this module
apply_attention_mhc,
forward_hc_pre_from_prev_fused_boundary,
)
from sglang.srt.runtime_context import get_exec
from sglang.srt.utils import is_gfx95_supported
@@ -150,7 +149,7 @@ def attention_inv_rope(
def init_decoder_layer(layer, quant_config) -> None:
"""The ROCm state of DeepseekV4DecoderLayer.__init__. input_layernorm also emits the
"""The ROCm state of DeepseekV4DecoderLayerBase.__init__. input_layernorm also emits the
pre-quantized operand of the dense projections (whether wqkv_a consumes native MXFP8
resolves on first use), and on gfx950 one launch runs hc_post, the pre-collapse and
the mixing stats; the kernel only supports hc_mult 4."""
@@ -203,48 +202,3 @@ def input_norm(
if coefficients is not None:
coefficients.materialize()
return layer.input_layernorm(hidden_states), None
# ---- DeepseekV4Model layer loop ----
def forward_layer_fused_boundary(
model,
i: int,
state,
*,
positions: torch.Tensor,
input_ids: torch.Tensor,
forward_batch,
input_ids_global: torch.Tensor,
capture_dspark: bool,
):
"""Layer i through the fused mHC boundary. Its FFN hc_post stays pending for the next
layer's boundary launch unless that layer reads the residual stream first (Engram gate,
DSpark capture) or there is none; hidden_states is None while a post is pending."""
nxt = i + 1
defer_post = (
nxt < model.end_layer
and model.layers[nxt].engram is None
and not (capture_dspark and nxt in model.dspark_layers_to_capture)
)
# The boundary speaks its own (hidden_states, prev_pre, pending_post) triple;
# the loop speaks HcState. This is the only place the two meet.
from sglang.srt.models.deepseek_v4_mhc import HcPending, HcState
pending = isinstance(state.streams, HcPending)
hidden_states, prev_pre, pending_post = forward_hc_pre_from_prev_fused_boundary(
model.layers[i],
positions=positions,
hidden_states=None if pending else state.streams,
input_ids=input_ids,
forward_batch=forward_batch,
input_ids_global=input_ids_global,
prev_pre=state.pre,
pending_post=tuple(state.streams) if pending else None,
defer_post=defer_post,
)
return HcState(
hidden_states if pending_post is None else HcPending(*pending_post),
prev_pre,
)
@@ -0,0 +1,433 @@
"""MoE operand transport and completion, preserving source residual rows."""
from contextlib import contextmanager, nullcontext
import torch
from sglang.srt.environ import envs
from sglang.srt.layers.cp.interleave import attn_cp_interleave_gather
from sglang.srt.layers.cp.utils import is_cp_active
from sglang.srt.layers.dp_attention import (
_tbo_event,
attn_tp_all_gather,
attn_tp_all_gather_into_tensor,
dp_gather_partial,
dp_gather_replicate,
dp_reduce_scatter_tensor,
dp_reduce_scatterv_async,
dp_scatter,
get_dp_global_num_tokens,
get_dp_tbo_comm_stream,
get_global_dp_buffer,
get_global_dp_buffer_len,
get_local_dp_buffer,
get_local_dp_buffer_len,
get_tbo_persistent_buffer,
is_dp_gatherv_active,
)
from sglang.srt.layers.layer_boundary.ops import attn_cp_interleave_reduce_scatter
from sglang.srt.layers.layer_boundary.output import DeferredFinalize
from sglang.srt.layers.layer_boundary.row_transport import ComputeTransport
from sglang.srt.layers.moe import get_moe_a2a_backend, should_use_dp_reduce_scatterv
from sglang.srt.runtime_context import get_forward, get_parallel
from sglang.srt.utils import get_bool_env_var, is_hip
_is_hip = is_hip()
SHARED_EXPERT_LOCAL = get_bool_env_var("SGLANG_DP_SHARED_EXPERT_LOCAL")
def shared_expert_runs_locally(moe):
"""Whether the replicated shared expert can run before the DP gather."""
return (
SHARED_EXPERT_LOCAL
and getattr(moe, "shared_experts", None) is not None
and getattr(moe, "_shared_expert_tp1", False)
)
@contextmanager
def _every_row_routed(forward_batch, num_rows: int):
# Under CP the real rows are not a prefix, so every row must be routed.
saved = (
forward_batch.num_token_non_padded,
forward_batch.global_num_token_non_padded_cpu,
)
if saved[0] is not None:
forward_batch.num_token_non_padded = torch.full_like(saved[0], num_rows)
forward_batch.global_num_token_non_padded_cpu = num_rows
try:
yield
finally:
(
forward_batch.num_token_non_padded,
forward_batch.global_num_token_non_padded_cpu,
) = saved
def _mask_dp_pad_rows(local_hidden_states, forward_batch) -> None:
n_real = forward_batch.num_token_non_padded
num_rows = local_hidden_states.shape[0]
if n_real is None or num_rows == 0:
return
pad = torch.arange(num_rows, device=local_hidden_states.device) >= n_real
local_hidden_states.masked_fill_(
pad.view(-1, *([1] * (local_hidden_states.dim() - 1))), 0
)
def _attn_tp_all_gather_chunks(
local: torch.Tensor, chunk_rows: list[int]
) -> torch.Tensor:
# Tensor all_gather stays on aiter AG / pynccl under graph capture; the list
# all_gather falls back to ProcessGroupNCCL, which invalidates HIP capture.
max_rows = max(chunk_rows)
padded = local
if local.shape[0] != max_rows:
padded = local.new_zeros((max_rows, *local.shape[1:]))
padded[: local.shape[0]].copy_(local)
gathered = local.new_empty((len(chunk_rows) * max_rows, *local.shape[1:]))
attn_tp_all_gather_into_tensor(gathered, padded.contiguous())
if all(rows == max_rows for rows in chunk_rows):
return gathered
return torch.cat(
[
gathered[i * max_rows : i * max_rows + rows]
for i, rows in enumerate(chunk_rows)
]
)
class MoeRowTransport(ComputeTransport):
def __init__(
self,
moe,
hidden_states,
forward_batch,
input_ids,
input_ids_global,
*,
cp_return_contiguous=False,
use_cp=None,
moe_gather=None,
attn_scatter=None,
):
_use_cp = is_cp_active(forward_batch) if use_cp is None else use_cp
_use_tp_moe_gather = (
moe_gather
if moe_gather is not None
else (
not _use_cp
and get_parallel().attn_dp_size > 1
and get_moe_a2a_backend().is_none()
)
)
_use_tp_attn_a2a_scatter = (
attn_scatter
if attn_scatter is not None
else (
not _use_cp
and get_parallel().attn_tp_size > 1
and not get_moe_a2a_backend().is_none()
)
)
# Variable-length combine sums TP-sharded expert output while returning
# each DP rank's rows. The producer leaves that sum to the transport.
_use_reduce_scatterv = (
_use_tp_moe_gather
and is_dp_gatherv_active()
and forward_batch.dp_padding_mode is not None
and not forward_batch.dp_padding_mode.is_max_len()
)
# Equal-chunk combine requires MAX_LEN padding and one attention rank
# per DP shard; the variable-length path takes precedence.
_use_reduce_scatter = (
envs.SGLANG_DP_USE_REDUCE_SCATTER.get()
and _use_tp_moe_gather
and not _use_reduce_scatterv
and not should_use_dp_reduce_scatterv()
and forward_batch.dp_padding_mode is not None
and forward_batch.dp_padding_mode.is_max_len()
and get_parallel().tp_size == get_parallel().attn_dp_size
)
mlp_reduce_scatter = (
_use_cp
or _use_reduce_scatterv
or _use_reduce_scatter
or (_use_tp_moe_gather and should_use_dp_reduce_scatterv())
)
# A replicated shared expert can run on local rows before the gather;
# its output is added after returning the routed experts' local slice.
_shared_local = None
do_shared_local = _use_tp_moe_gather and shared_expert_runs_locally(moe)
if _use_cp:
moe_a2a_backend = get_moe_a2a_backend()
if moe_a2a_backend.is_none():
hidden_states = attn_cp_interleave_gather(hidden_states)
else:
assert (
moe_a2a_backend.is_deepep()
or moe_a2a_backend.is_megamoe()
or moe_a2a_backend.is_mori()
), (
"CP requires moe_a2a_backend in ('deepep', 'megamoe', 'mori'), "
f"got {moe_a2a_backend.value!r}."
)
elif _use_tp_moe_gather:
hidden_states, local_hidden_states = (
get_global_dp_buffer(get_parallel().tp_group),
hidden_states,
)
if do_shared_local and local_hidden_states.shape[0] > 0:
_shared_local = moe._forward_shared_experts(local_hidden_states)
# Pad rows are not refreshed during graph replay. Stale bf16-max
# values can overflow the MoE and contaminate the combine output.
if _is_hip:
_mask_dp_pad_rows(local_hidden_states, forward_batch)
# self_attn has already reduced across attention TP, so these hidden
# states are replicated and must not be summed by a partial gather.
dp_gather_replicate(hidden_states, local_hidden_states, forward_batch)
_a2a_scatter_chunks = None
if _use_tp_attn_a2a_scatter:
s, r = get_parallel().attn_tp_size, get_parallel().attn_tp_rank
_a2a_scatter_chunks = list(hidden_states.tensor_split(s))
hidden_states = _a2a_scatter_chunks[r].contiguous()
# DSpark next-token layers are not hash-routed and intentionally do not
# carry token IDs. Only split IDs for callers that actually provide them.
if input_ids is not None:
input_ids = input_ids.tensor_split(s)[r].contiguous()
if input_ids_global is not None:
input_ids_global = input_ids_global.tensor_split(s)[r].contiguous()
gathered_rows = (
_every_row_routed(forward_batch, hidden_states.shape[0])
if _use_cp and get_moe_a2a_backend().is_none()
else nullcontext()
)
self.input = hidden_states
self.input_ids, self.input_ids_global = input_ids, input_ids_global
self.batch = forward_batch
self.cp_return_contiguous = cp_return_contiguous
self._use_cp = _use_cp
self._use_tp_moe_gather = _use_tp_moe_gather
self._use_tp_attn_a2a_scatter = _use_tp_attn_a2a_scatter
self._use_reduce_scatterv = _use_reduce_scatterv
self._use_reduce_scatter = _use_reduce_scatter
self.mlp_reduce_scatter = mlp_reduce_scatter
self._shared_local = _shared_local
self.do_shared_local = do_shared_local
self._a2a_scatter_chunks = _a2a_scatter_chunks
self.gathered_rows = gathered_rows
def return_output(self, hidden_states):
forward_batch = self.batch
_use_cp = self._use_cp
_use_tp_moe_gather = self._use_tp_moe_gather
_use_tp_attn_a2a_scatter = self._use_tp_attn_a2a_scatter
_use_reduce_scatterv = self._use_reduce_scatterv
_use_reduce_scatter = self._use_reduce_scatter
_shared_local = self._shared_local
_a2a_scatter_chunks = self._a2a_scatter_chunks
if _use_cp and get_moe_a2a_backend().is_none():
if self.cp_return_contiguous:
parallel = get_parallel()
hidden_states = parallel.tp_group.all_reduce(hidden_states)
hidden_states = hidden_states.tensor_split(parallel.attn_cp_size)[
parallel.attn_cp_rank
].contiguous()
else:
hidden_states = attn_cp_interleave_reduce_scatter(hidden_states)
elif _use_tp_moe_gather:
hidden_states, global_hidden_states = (
get_local_dp_buffer(get_parallel().tp_group),
hidden_states,
)
if should_use_dp_reduce_scatterv() or _use_reduce_scatterv:
get_parallel().tp_group.reduce_scatterv(
global_hidden_states,
output=hidden_states,
sizes=get_dp_global_num_tokens(),
)
elif _use_reduce_scatter:
dp_reduce_scatter_tensor(hidden_states, global_hidden_states)
else:
dp_scatter(hidden_states, global_hidden_states, forward_batch)
if _shared_local is not None:
n = hidden_states.shape[0]
hidden_states = hidden_states + _shared_local[:n]
if _use_tp_attn_a2a_scatter:
assert _a2a_scatter_chunks is not None
if _is_hip:
hidden_states = _attn_tp_all_gather_chunks(
hidden_states, chunk_rows=[t.shape[0] for t in _a2a_scatter_chunks]
)
else:
gathered = [torch.empty_like(t) for t in _a2a_scatter_chunks]
attn_tp_all_gather(gathered, hidden_states.contiguous())
hidden_states = torch.cat(gathered)
return hidden_states
@contextmanager
def scope(self):
with (
get_forward().scoped(
mlp_reduce_scatter=self.mlp_reduce_scatter, defer_moe_finalize=False
),
self.gathered_rows,
):
yield
def release(self):
self.input = self._shared_local = self._a2a_scatter_chunks = None
def _tbo_gather_moe_input(moe, local, batch, sub, do_shared_local):
shared_local = (
moe._forward_shared_experts(local)
if do_shared_local and local.shape[0] > 0
else None
)
global_hidden = get_tbo_persistent_buffer(
("gh", sub),
get_global_dp_buffer_len(),
local.shape[1],
local.dtype,
local.device,
)
comm = get_dp_tbo_comm_stream()
compute = torch.cuda.current_stream()
with torch.cuda.stream(comm):
comm.wait_stream(compute)
dp_gather_partial(global_hidden, local, batch)
event = _tbo_event(("gather", sub))
event.record(comm)
return global_hidden, event, shared_local
def _tbo_combine_moe_output(output, sub):
local = get_tbo_persistent_buffer(
("lo", sub),
get_local_dp_buffer_len(),
output.shape[1],
output.dtype,
output.device,
)
event = dp_reduce_scatterv_async(
local, output, get_dp_global_num_tokens(), event_key=("combine", sub)
)
return local, event
def _tbo_wait_moe_output(local, event, shared_local):
torch.cuda.current_stream().wait_event(event)
return local + shared_local[: local.shape[0]] if shared_local is not None else local
class MoeFinalize(DeferredFinalize, frozen=True):
pieces: object = None
partial: object = None
def complete(self, *, reduce=True):
from sglang.srt.layers.moe import post_experts_all_reduce
from sglang.srt.layers.moe.utils import should_add_replicated_moe_output
with get_forward().scoped(mlp_reduce_scatter=not reduce):
value = (
self.pieces.get_merged() if self.pieces is not None else self.partial
)
if reduce:
value = post_experts_all_reduce(value)
if (
self.pieces is not None
and self.pieces.shared is not None
and self.pieces.shared_is_replicated
and should_add_replicated_moe_output()
):
value += self.pieces.shared
return value
class OverlappedMoeTransport(ComputeTransport):
"""The TBO issue/wait protocol, owned by one microbatch.
EP ops already return source rows. The TP path gathers normalized 2-D
input and sums/scatters expert output asynchronously; residual coefficients
stay on the source rows throughout. Neither finish path sums again.
"""
is_overlapped = True
def __init__(self, moe, hidden_states, batch, subbatch_index, rows):
self.rows = rows
self.moe, self.input, self.batch = moe, hidden_states, batch
self.ep = not get_moe_a2a_backend().is_none()
self.sub = subbatch_index
self.do_shared_local = not self.ep and shared_expert_runs_locally(moe)
self.shared_local = None
self.gather_event = self.combine_event = None
self.gather_keepalive = self.combine_keepalive = None
self.global_hidden = self.local_out = None
self.combined = False
def release(self):
events = (self.gather_event, self.combine_event)
if any(event is not None for event in events):
current = torch.cuda.current_stream()
for event in events:
if event is not None:
current.wait_event(event)
self.gather_event = self.combine_event = None
self.gather_keepalive = self.combine_keepalive = None
self.global_hidden = self.local_out = self.shared_local = self.input = None
def issue_gather(self):
if self.ep or self.gather_event is not None or self.global_hidden is not None:
raise RuntimeError("TBO gather cannot be issued twice")
local = self.input
self.global_hidden, self.gather_event, self.shared_local = (
_tbo_gather_moe_input(
self.moe, local, self.batch, self.sub, self.do_shared_local
)
)
self.gather_keepalive = local
self.input = None
def wait_gather(self):
if self.gather_event is None:
raise RuntimeError("TBO gather was not issued")
torch.cuda.current_stream().wait_event(self.gather_event)
self.gather_event = self.gather_keepalive = None
return self.global_hidden
def issue_combine(self, output):
if self.ep or self.combine_event is not None or self.combined:
raise RuntimeError("TBO combine cannot be issued twice")
self.local_out, self.combine_event = _tbo_combine_moe_output(output, self.sub)
self.combine_keepalive = output
self.global_hidden = None
def wait_combine(self):
if self.combine_event is None:
raise RuntimeError("TBO combine was not issued")
value = _tbo_wait_moe_output(
self.local_out, self.combine_event, self.shared_local
)
self.combine_event = self.combine_keepalive = None
self.local_out = self.shared_local = None
self.combined = True
return value
def prepare_output(self, output):
if not self.ep and not self.combined:
raise RuntimeError("TBO finish requires the combine wait")
return output
def _tbo_run_moe(layer, hidden, batch, do_shared_local):
global_ids = batch._tbo_global_input_ids
with get_forward().scoped(mlp_reduce_scatter=True):
return layer.mlp(
hidden,
batch,
input_ids=global_ids,
input_ids_global=global_ids,
skip_shared_experts=do_shared_local,
)
+75 -3
View File
@@ -618,6 +618,13 @@ class MoEOutput(NamedTuple):
)
class MergedMoEOutput(MoEOutput):
"""Merged routed rows with a replicated shared output still separate."""
def get_merged(self) -> torch.Tensor:
return self.routed
class DeepseekV2MoE(nn.Module):
def __init__(
self,
@@ -982,10 +989,13 @@ class DeepseekV2MoE(nn.Module):
input_ids_global: Optional[torch.Tensor] = None,
skip_shared_experts: bool = False,
return_moe_output: bool = False,
leave_output_sum: bool = False,
) -> Union[torch.Tensor, MoEOutput]:
"""``return_moe_output`` asks for the pieces unmerged, as `MoEOutput`; only
the TP paths honor it. Distinct from `get_forward().defer_moe_finalize`,
the V2/V3 layer-boundary handoff."""
the V2/V3 layer-boundary handoff. ``leave_output_sum`` returns those
pieces with the selected finalize kernel, for
a boundary that owns reduction and the replicated shared add."""
from sglang.srt.layers.moe.mega_moe import forward_mega_moe, should_use_mega_moe
if should_use_mega_moe(self, hidden_states):
@@ -1007,6 +1017,20 @@ class DeepseekV2MoE(nn.Module):
if not self._enable_a2a_moe:
if self._can_dual_stream_graph(hidden_states):
fwd = get_forward()
if leave_output_sum:
routed, shared = dsv4_flashinfer_moe_dual_stream_outputs(
hidden_states,
self.layer_id,
fwd.mlp_reduce_scatter,
self._shared_expert_tp1,
)
return MergedMoEOutput(
routed,
shared if self._shared_expert_tp1 else None,
self.experts,
1.0,
self._shared_expert_tp1,
)
return dsv2_flashinfer_moe_dual_stream_graph(
hidden_states,
self.layer_id,
@@ -1027,6 +1051,7 @@ class DeepseekV2MoE(nn.Module):
num_token_non_padded=num_token_non_padded,
use_vision_topk=use_vision_topk,
return_moe_output=return_moe_output,
leave_output_sum=leave_output_sum,
)
else:
return self.forward_normal(
@@ -1038,6 +1063,7 @@ class DeepseekV2MoE(nn.Module):
num_token_non_padded=num_token_non_padded,
use_vision_topk=use_vision_topk,
return_moe_output=return_moe_output,
leave_output_sum=leave_output_sum,
)
else:
return self.forward_deepep(
@@ -1053,6 +1079,7 @@ class DeepseekV2MoE(nn.Module):
num_token_non_padded: Optional[torch.Tensor] = None,
use_vision_topk: bool = False,
return_moe_output: bool = False,
leave_output_sum: bool = False,
) -> Union[torch.Tensor, MoEOutput]:
# Note(kpham-sgl): issue order satisfies 3 constraints:
# - no stream explosion: main (routed) issued before alt block -> capture reuses 1 alt stream;
@@ -1195,7 +1222,7 @@ class DeepseekV2MoE(nn.Module):
reduce=sum_post_experts_output,
)
if return_moe_output:
if return_moe_output or leave_output_sum:
return MoEOutput(
routed=final_hidden_states,
shared=shared_output,
@@ -1238,6 +1265,7 @@ class DeepseekV2MoE(nn.Module):
num_token_non_padded: Optional[torch.Tensor] = None,
use_vision_topk: bool = False,
return_moe_output: bool = False,
leave_output_sum: bool = False,
) -> Union[torch.Tensor, MoEOutput]:
if hasattr(self, "shared_experts") and use_intel_amx_backend(
self.shared_experts.gate_up_proj
@@ -1373,7 +1401,7 @@ class DeepseekV2MoE(nn.Module):
pre_quant_input=pre_quant_input,
)
if return_moe_output:
if return_moe_output or leave_output_sum:
return MoEOutput(
routed=final_hidden_states,
shared=shared_output,
@@ -3494,4 +3522,48 @@ def dsv2_flashinfer_moe_dual_stream_graph(
return moe_fusion.forward_normal_dual_stream(hidden_states)
def _fake_dsv4_dual_stream_outputs(
hidden_states, layer_id, mlp_reduce_scatter, shared_replicated
):
return torch.empty_like(hidden_states), (
torch.empty_like(hidden_states)
if shared_replicated
else hidden_states.new_empty((0,))
)
@register_custom_op(fake_impl=_fake_dsv4_dual_stream_outputs)
def dsv4_flashinfer_moe_dual_stream_outputs(
hidden_states: torch.Tensor,
layer_id: int,
mlp_reduce_scatter: bool,
shared_replicated: bool,
) -> Tuple[torch.Tensor, torch.Tensor]:
return _run_dsv4_dual_stream_outputs(
hidden_states, layer_id, mlp_reduce_scatter, shared_replicated
)
def _run_dsv4_dual_stream_outputs(
hidden_states: torch.Tensor,
layer_id: int,
mlp_reduce_scatter: bool,
shared_replicated: bool,
) -> Tuple[torch.Tensor, torch.Tensor]:
forward_context = get_tc_piecewise_forward_context()
assert forward_context is not None and forward_context.moe_fusions is not None
moe = forward_context.moe_fusions[layer_id]
assert moe is not None
with get_forward().scoped(
mlp_reduce_scatter=mlp_reduce_scatter,
flashinfer_trtllm_bypass=True,
lora_batch_layout=LoRABatchLayout.TP_GLOBAL,
defer_moe_finalize=False,
):
pieces = moe.forward_normal_dual_stream(hidden_states, leave_output_sum=True)
assert pieces.shared_is_replicated == shared_replicated
merged = pieces.get_merged()
return merged, pieces.shared if shared_replicated else merged.new_empty((0,))
EntryClass = [DeepseekV2ForCausalLM, DeepseekV3ForCausalLM, DeepseekV32ForCausalLM]
File diff suppressed because it is too large Load Diff
@@ -0,0 +1,540 @@
"""DeepSeek hyper-connection reads, updates and operand row transport."""
from functools import partial
import msgspec
import torch
from sglang.srt.layers.cp.cp_decode_attn_tp import get_cp_decode_attn_tp_ctx
from sglang.srt.layers.layer_boundary.contracts import BatchVariant, ReadoutFusion
from sglang.srt.layers.layer_boundary.factories import declare_attn, declare_ffn
from sglang.srt.layers.layer_boundary.facts import ReadFacts, UpdateFacts
from sglang.srt.layers.layer_boundary.layout import (
Layout,
SumGroup,
TokenAxis,
_sum_group,
)
from sglang.srt.layers.layer_boundary.output import UnreducedOutput
from sglang.srt.layers.layer_boundary.residual import PendingUpdateRead
from sglang.srt.layers.layer_boundary.residual import batch as residual_batch
from sglang.srt.layers.layer_boundary.row_transport import (
ComputeTransport,
RowTransport,
TransportRows,
)
from sglang.srt.layers.moe import get_moe_a2a_backend
from sglang.srt.layers.quantization.mxfp4_flashinfer_trtllm_moe import (
can_fuse_all_reduce,
)
from sglang.srt.models import deepseek_v4_mhc as mhc
from sglang.srt.models.deepseek_common.moe_transport import (
MoeFinalize,
MoeRowTransport,
)
from sglang.srt.models.deepseek_v2 import MoEOutput
from sglang.srt.runtime_context import get_parallel
from sglang.srt.utils import is_gfx95_supported
_is_gfx95_supported = is_gfx95_supported()
class LayerCall(msgspec.Struct, frozen=True):
input_ids: object = None
input_ids_global: object = None
allow_cross_layer_fusion: bool = True
tbo: bool = False
tbo_subbatch_index: object = None
class ReadState(msgspec.Struct):
"""The source residual and coefficients of one sublayer invocation."""
streams: torch.Tensor
post: object = None
comb: object = None
quantized: object = None
stats: object = None
stats_stream: object = None
call: object = None
class HcUpdate:
bind = None
replay_update = None
is_plain_add = False
outlives_layer = True
quantized_sum = False
def __init__(self, layer, attention, *, applied_at_exit=False):
self.layer = layer
self.attention = attention
self.applied_at_exit = applied_at_exit
self.writes_stream = False
def slice_residual_attn_tp(self, residual):
raise NotImplementedError("mHC coefficients remain on their source rows")
def gather_residual_attn_tp(self, residual):
raise NotImplementedError("mHC coefficients remain on their source rows")
def post(self, output, state, *, next=None):
if not self.layer.boundary_pre_from_prev:
return self.layer.hc_post(output, state.streams, state.post, state.comb)
hc = self.layer.attn_hc if self.attention else self.layer.ffn_hc
if self.attention:
return mhc.run_attn_post(
hc,
output,
state.streams,
stats_stream=state.stats_stream,
next=next,
world_size=self.layer.self_attn.attn_tp_size,
precomputed=state.stats,
)
return mhc.run_moe_post(
hc,
output,
state.streams,
stats_stream=state.stats_stream,
next=next if state.call.allow_cross_layer_fusion else None,
world_size=self.layer.mlp.tp_size,
precomputed=state.stats,
)
def update(self, output, state):
return self.post(output, state)
class HcRead:
is_plain_norm = False
reads_before_dp_gather = True
reads_after_attn_tp_gather = True
gathering_reads = ()
def __init__(self, layer, attention):
self.layer = layer
self.attention = attention
self.completing_fusions = tuple(
ReadoutFusion(
group,
self._complete_read,
reads=True,
carried=True,
accepts_update=lambda update: isinstance(update, HcUpdate),
)
for group in (SumGroup.ATTN_TP, SumGroup.MOE_OUTPUT)
)
def bind(self, batch, call):
return HcReadCall(self, batch, call)
@staticmethod
def init_residual(hidden_states):
return hidden_states
def _complete_read(self, output, state, batch, norm, *, update, read_context=None):
if not self.layer.boundary_pre_from_prev:
return None
if isinstance(output, MoeFinalize):
if output.pieces is None:
return None
output = output.pieces
elif isinstance(output, UnreducedOutput):
if not update.attention:
return None
output = mhc.AttnOutput(output.partial)
else:
return None
call = read_context if read_context is not None else state.call
read = self.bind(batch, call)
return read._after_post(
update.post(output, state, next=read.next_boundary), norm
)
class HcReadCall(HcRead):
def __init__(self, spec, batch, call):
self.spec = spec
self.layer = spec.layer
self.attention = spec.attention
self.batch = batch
self.call = call
self.completing_fusions = tuple(
msgspec.structs.replace(
f, run=partial(spec._complete_read, read_context=call)
)
for f in spec.completing_fusions
)
def bind(self, batch, call=None):
if batch is not self.batch or (call is not None and call is not self.call):
raise ValueError("an invocation read cannot bind another call")
return self
@property
def next_boundary(self):
if not self.layer.boundary_mhc_fusions or (
self.attention and self.layer.engram is not None
):
return None
hc = self.layer.attn_hc if self.attention else self.layer.ffn_hc
return mhc.make_boundary(
self.layer.input_layernorm
if self.attention
else self.layer.post_attention_layernorm,
accepts_mxfp8=self.attention
and self.layer.self_attn.accepts_mxfp8_swizzled_input(),
hc=hc,
)
def _after_post(self, residual, norm):
return self.read(residual, norm)
def read(
self,
residual,
norm,
quant_format="",
post_residual_addition=None,
):
if quant_format or post_residual_addition is not None:
raise NotImplementedError("mHC read supplies its own projection format")
layer = self.layer
if layer.boundary_pre_from_prev:
if not isinstance(residual, mhc.HcState):
residual = mhc.HcState(residual)
streams = residual.residual
stream = (
layer.hc_stats_stream
if mhc.use_stats_stream(layer.hc_cfg, self.batch, streams)
else None
)
mhc.fork_stats_stream(stream)
quantized = (
[]
if self.attention and layer.self_attn.accepts_mxfp8_swizzled_input()
else None
)
hc = layer.attn_hc if self.attention else layer.ffn_hc
x = mhc.combine(hc, residual, quantized)
stats = residual.stats
residual.release()
return x, ReadState(
streams,
quantized=quantized[0] if quantized else None,
stats=stats,
stats_stream=stream,
call=self.call,
)
fn, scale, base = self._parameters()
x, post, comb, norm_fused = layer.hc_pre(
residual,
fn,
scale,
base,
norm=norm if layer.boundary_mhc_fusions else None,
forward_batch=self.batch,
)
quantized = None
if not norm_fused:
if self.attention:
x, quantized = layer._input_norm(
x, allow_aiter_quant=_is_gfx95_supported
)
else:
x = norm(x)
return x, ReadState(residual, post, comb, quantized, call=self.call)
def _parameters(self):
layer = self.layer
if self.attention:
return layer.hc_attn_fn, layer.hc_attn_scale, layer.hc_attn_base
return layer.hc_ffn_fn, layer.hc_ffn_scale, layer.hc_ffn_base
def update_and_read(self, update, value, residual, norm, **kwargs):
if not isinstance(update, HcUpdate):
raise TypeError("mHC read requires its producer's update")
if self.layer.boundary_pre_from_prev:
return self.read(
update.post(value, residual, next=self.next_boundary), norm
)
layer = self.layer
if layer.boundary_mhc_fusions and layer.use_fused_mhc_post_pre:
from sglang.srt.models.deepseek_common.amd.deepseek_v4_fused_mhc import (
apply_mhc_post_pre_boundary,
)
fn, scale, base = self._parameters()
cached = (
layer._input_layernorm_weight_bf16
if self.attention
else layer._post_attention_layernorm_weight_bf16
)
fused = apply_mhc_post_pre_boundary(
value,
residual.streams,
residual.post,
residual.comb,
fn,
scale,
base,
layer.hc_mult,
layer.rms_norm_eps,
layer.hc_eps,
mhc.MHC_POST_MULT_VALUE,
layer.hc_sinkhorn_iters,
cached if cached is not None else norm.weight.data,
norm.variance_epsilon,
fn_transpose=not self.attention,
)
if fused is not None:
streams, x, post, comb, norm_fused = fused
quantized = None
if not norm_fused:
if self.attention:
x, quantized = layer._input_norm(
x, allow_aiter_quant=_is_gfx95_supported
)
else:
x = norm(x)
return x, ReadState(streams, post, comb, quantized, call=self.call)
return self.read(update.update(value, residual), norm)
class FinalRead(PendingUpdateRead):
bind = None
@staticmethod
def _complete(output, residual, batch, norm, *, update):
if (
not isinstance(output, MoeFinalize)
or output.pieces is None
or not update.layer.boundary_pre_from_prev
):
return None
return update.post(output.pieces, residual)
completing_fusions = (
ReadoutFusion(
SumGroup.MOE_OUTPUT,
_complete.__func__,
reads=True,
carried=True,
accepts_update=lambda update: isinstance(update, HcUpdate),
),
)
@staticmethod
def accepts_pending_update(update):
return isinstance(update, HcUpdate)
@staticmethod
def init_residual(value):
return value
@staticmethod
def read(value, norm):
return value
@staticmethod
def update_and_read(update, value, residual, norm):
return update.update(value, residual)
def materialize(hidden_states, batch, *, consume=True, next=None):
"""Read mHC streams at a PP, head, Engram or capture seam."""
stream = residual_batch.stream_of(batch)
stream.check(hidden_states)
pending = stream.pending
if pending is None:
return hidden_states
value = pending.for_boundary()
if isinstance(value, MoeFinalize) and value.pieces is not None:
result = pending.update.post(value.pieces, stream.residual, next=next)
elif isinstance(value, UnreducedOutput):
result = pending.update.post(
mhc.AttnOutput(value.partial), stream.residual, next=next
)
else:
result = pending.update.post(pending.complete(), stream.residual, next=next)
if consume:
stream.write(result)
return result
def stages(config, layer_id, *, layer=None):
from sglang.srt.layers.layer_boundary.layout import token_axis_sizes
ep = not get_moe_a2a_backend().is_none()
table_attn, table_ffn, overlapped_ffn = {}, {}, {}
for variant in (BatchVariant.ORDINARY, BatchVariant.CONTEXT_PARALLEL):
cp = variant is BatchVariant.CONTEXT_PARALLEL
source = Layout.sharded_over(
TokenAxis.ATTN_DP,
*((TokenAxis.ATTN_CP,) if cp else ()),
axis_sizes=token_axis_sizes(cp_active=cp),
)
compute = Layout.sharded_over(
*(
(TokenAxis.ATTN_DP, TokenAxis.ATTN_CP)
if cp and ep
else ((TokenAxis.ATTN_DP, TokenAxis.ATTN_TP) if ep else ())
),
axis_sizes=token_axis_sizes(cp_active=cp),
)
table_attn[variant] = TransportRows(
source,
source,
source,
SumGroup.ATTN_TP,
SumGroup.ATTN_CP if get_parallel().attn_cp_size > 1 else None,
)
table_ffn[variant] = TransportRows(
source, compute, source, None if ep else SumGroup.MOE_OUTPUT
)
if not cp:
overlapped_ffn[variant] = TransportRows(
source,
source if ep else compute,
source,
None if ep else SumGroup.MOE_OUTPUT,
)
if layer is None:
from sglang.srt.models.deepseek_v4 import (
_is_fused_mhc_post_pre_enabled_xpu,
is_cross_layer_mhc_fusion_enabled,
)
lagged = config.hc_pre_from_prev_sublayer
fused = (
is_cross_layer_mhc_fusion_enabled() or _is_fused_mhc_post_pre_enabled_xpu()
)
else:
lagged = layer.boundary_pre_from_prev
fused = layer.boundary_mhc_fusions and layer.use_fused_mhc_post_pre
applied = False if lagged else not fused
if layer is None:
read = ReadFacts.of(HcRead(None, True))
au = UpdateFacts.of(HcUpdate(None, True))
fu = UpdateFacts.of(HcUpdate(None, False, applied_at_exit=applied))
else:
read = HcRead(layer, True)
au = HcUpdate(layer, True)
fu = HcUpdate(layer, False, applied_at_exit=applied)
for variant in (BatchVariant.UNPADDED, BatchVariant.INPUT_SCATTERED):
table_attn[variant] = table_attn[BatchVariant.ORDINARY]
table_ffn[variant] = table_ffn[BatchVariant.ORDINARY]
# TBO keeps source rows until expert dispatch, including an uneven final
# microbatch. It does not shard these rows across attention TP.
overlapped_ffn[BatchVariant.UNPADDED] = overlapped_ffn[BatchVariant.ORDINARY]
return (
declare_attn(
read=read,
update=au,
gathers_attn_tp_input=False,
row_transport=RowTransport(
table_attn,
layer._bind_attention_transport if layer is not None else None,
),
),
declare_ffn(
sparse=True,
read=HcRead(layer, False) if layer is not None else read,
update=fu,
row_transport=RowTransport(
table_ffn,
layer._bind_moe_transport if layer is not None else None,
overlapped_rows=overlapped_ffn,
),
),
)
class AttentionTransport(ComputeTransport):
def __init__(self, layer, hidden_states, batch, call, rows):
if rows.source != rows.compute or rows.source != rows.returned:
raise ValueError("attention transport must preserve its source rows")
if rows.group is not SumGroup.ATTN_TP or rows.decode_group not in (
None,
SumGroup.ATTN_CP,
):
raise ValueError("attention transport does not match its projection groups")
self.num_rows = hidden_states.shape[0]
self.rows = rows
self.input = hidden_states
self.defers_sum = (
layer.boundary_mhc_fusions
and layer.boundary_pre_from_prev
and layer._can_fuse_attn_mhc
and can_fuse_all_reduce(hidden_states.shape[0], layer.hc_cfg.hidden)
)
def release(self):
self.input = None
def prepare_output(self, output):
self.input = None
value = output.partial if isinstance(output, mhc.AttnOutput) else output
if value.shape[0] != self.num_rows:
raise ValueError("attention output changed its source rows")
if not isinstance(output, mhc.AttnOutput):
return output
ctx = get_cp_decode_attn_tp_ctx()
group = self.rows.decode_group if ctx.use_decode_attn_tp else self.rows.group
if group is None:
raise ValueError("attention output owes no declared transport sum")
return UnreducedOutput(output.partial, group=_sum_group(group))
class MoeTransport(MoeRowTransport):
def __init__(self, layer, hidden_states, forward_batch, call, rows):
self.source_rows = hidden_states.shape[0]
self.rows = rows
if (rows.group is None) != (not get_moe_a2a_backend().is_none()):
raise ValueError("MoE backend does not match its transport output contract")
cp = TokenAxis.ATTN_CP in rows.source.sharded
super().__init__(
layer.mlp,
hidden_states,
forward_batch,
call.input_ids,
call.input_ids_global,
cp_return_contiguous=layer.config.model_type == "deepseek_v41",
use_cp=cp,
moe_gather=not cp
and TokenAxis.ATTN_DP in rows.source.sharded - rows.compute.sharded,
attn_scatter=not cp
and TokenAxis.ATTN_TP in rows.compute.sharded - rows.source.sharded,
)
self.sum_in_return = self.mlp_reduce_scatter and rows.group is not None
self.return_moe_output = self.defers_sum = (
layer.boundary_mhc_fusions
and layer.boundary_pre_from_prev
and layer._can_fuse_ffn_mhc
and can_fuse_all_reduce(self.input.shape[0], layer.hc_cfg.hidden)
and not self._use_cp
and not self._use_tp_moe_gather
and not self._use_tp_attn_a2a_scatter
)
def prepare_output(self, hidden_states):
self.input = None
if isinstance(hidden_states, MoEOutput):
completion = MoeFinalize(hidden_states)
return (
completion.complete(reduce=False)
if self.mlp_reduce_scatter
else completion
)
if self.rows.group is not None and not self.mlp_reduce_scatter:
return MoeFinalize(partial=hidden_states)
return hidden_states
def return_output(self, hidden_states):
if self.defers_sum:
return hidden_states
value = super().return_output(hidden_states)
if value.shape[0] != self.source_rows:
raise ValueError("MoE transport did not return its source rows")
return value
+86 -110
View File
@@ -2,6 +2,7 @@ from __future__ import annotations
import copy
import logging
from contextlib import nullcontext
from typing import Iterable, List, Optional, Tuple
import msgspec
@@ -19,6 +20,8 @@ from sglang.srt.distributed.device_communicators.vocab_gather import make_vocab_
from sglang.srt.environ import envs
from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder
from sglang.srt.layers.dp_attention import is_dp_attention_enabled
from sglang.srt.layers.layer_boundary import layer_stack
from sglang.srt.layers.layer_boundary.residual import batch as residual_batch
from sglang.srt.layers.layernorm import RMSNorm
from sglang.srt.layers.logits_processor import LogitsProcessorOutput
from sglang.srt.layers.moe.utils import is_shared_experts_fusion_disabled
@@ -33,18 +36,22 @@ from sglang.srt.model_executor.forward_batch_info import ForwardBatch
from sglang.srt.model_executor.forward_context import get_token_to_kv_pool
from sglang.srt.model_executor.runner import get_is_capture_mode
from sglang.srt.model_loader.weight_utils import default_weight_loader
from sglang.srt.models import deepseek_v4_boundary as hc_boundary
from sglang.srt.models import deepseek_v4_mhc as mhc
from sglang.srt.models.dbrx import ReplicatedLinear
from sglang.srt.models.deepseek_v4 import (
DEEPSEEK_V4_STACKED_PARAMS_MAPPING,
DeepseekV4DecoderLayer,
DeepseekV4CudaDecoderLayer,
DeepseekV4ForCausalLM,
DeepseekV4PlatformDecoderLayer,
MqaAttentionBase,
_apply_wo_a_bf16_matmul,
_dequant_fp8_wo_a_streaming,
_is_cuda,
hc_head_torch,
make_hc_head_params,
)
from sglang.srt.models.deepseek_v4_platform_stage import NonCudaDSparkMixin
from sglang.srt.models.dspark import (
BaseDSparkMarkovHead,
DSparkConfidenceHead,
@@ -249,6 +256,8 @@ class DSparkAttention(MqaAttentionBase):
positions: torch.Tensor,
hidden_states: torch.Tensor,
forward_batch: ForwardBatch,
*,
defer_all_reduce: bool = False,
) -> torch.Tensor:
# Idle ranks have no draft KV rows or attention metadata, but still
# participate in the shared MoE forward later in the draft stage.
@@ -370,8 +379,17 @@ class DSparkAttention(MqaAttentionBase):
)
else:
o = torch.einsum("bgd,grd->bgr", o.float(), wo_a.float()).to(q.dtype)
out, _ = self.wo_b(o.reshape(o.shape[0], o.shape[1] * o.shape[2]))
return out
value = o.reshape(o.shape[0], o.shape[1] * o.shape[2])
out, _ = self.wo_b(value, skip_all_reduce=defer_all_reduce)
return (
mhc.AttnOutput(out)
if _is_cuda and defer_all_reduce and self.attn_tp_size > 1
else out
)
@staticmethod
def accepts_mxfp8_swizzled_input():
return False
def _resolve_dspark_pool() -> DeepSeekV4TokenToKVPool:
@@ -555,7 +573,7 @@ def _dspark_stage_config(config: DeepSeekV4Config) -> DeepSeekV4Config:
return stage_config
class DSparkV4Stage(DeepseekV4DecoderLayer):
class DSparkV4StageMixin:
def __init__(
self,
config: DeepSeekV4Config,
@@ -575,6 +593,8 @@ class DSparkV4Stage(DeepseekV4DecoderLayer):
quant_config=quant_config,
prefix=prefix,
is_nextn=True,
boundary_pre_from_prev=config.hc_pre_from_prev_sublayer,
boundary_mhc_fusions=False,
alt_streams=alt_streams,
hc_stats_stream=hc_stats_stream,
moe_routed_quant_stream=moe_routed_quant_stream,
@@ -628,99 +648,33 @@ class DSparkV4Stage(DeepseekV4DecoderLayer):
alt_streams=alt_streams,
)
def _hc_pre_block(
self,
x: torch.Tensor,
hc_fn: torch.Tensor,
hc_scale: torch.Tensor,
hc_base: torch.Tensor,
) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
y, post, comb, _ = self.hc_pre(x, hc_fn, hc_scale, hc_base)
return y, post, comb
def _hc_post_block(
self,
x: torch.Tensor,
residual: torch.Tensor,
post: torch.Tensor,
comb: torch.Tensor,
) -> torch.Tensor:
return self.hc_post(x, residual, post, comb)
def forward(
self,
positions: torch.Tensor,
hidden_states: torch.Tensor,
forward_batch: ForwardBatch,
prev_pre: Optional[torch.Tensor] = None,
) -> Tuple[torch.Tensor, Optional[torch.Tensor]]:
if self.hc_pre_from_prev_sublayer:
return self._forward_hc_pre_from_prev(
positions, hidden_states, forward_batch, prev_pre
)
residual = hidden_states
x, post, comb = self._hc_pre_block(
hidden_states, self.hc_attn_fn, self.hc_attn_scale, self.hc_attn_base
class DSparkV4CudaStage(DSparkV4StageMixin, DeepseekV4CudaDecoderLayer):
def _run_boundary_attention(
self, hidden_states, positions, forward_batch, quantized
):
return self.self_attn(
positions, hidden_states, forward_batch, defer_all_reduce=True
)
x = self.input_layernorm(x)
with self.self_attn.maybe_use_decode_attn_tp(forward_batch):
x = self.self_attn(positions, x, forward_batch)
x = self._hc_post_block(x, residual, post, comb)
residual = x
x, post, comb = self._hc_pre_block(
x, self.hc_ffn_fn, self.hc_ffn_scale, self.hc_ffn_base
def forward(self, positions, hidden_states, forward_batch):
return super().forward(
positions,
hidden_states,
None,
forward_batch,
None,
allow_cross_layer_fusion=False,
)
x = self.post_attention_layernorm(x)
x = self._run_ffn(x, forward_batch)
x = self._hc_post_block(x, residual, post, comb)
return x, None
def _forward_hc_pre_from_prev(
self,
positions: torch.Tensor,
hidden_states: torch.Tensor,
forward_batch: ForwardBatch,
prev_pre: Optional[torch.Tensor],
) -> Tuple[torch.Tensor, torch.Tensor]:
# The draft stages run the plain four steps: no fused post, no seams, so
# they need nothing from `bind_next` and never hand anything forward.
assert self.attn_hc is not None and self.ffn_hc is not None
stats_stream = None
if mhc.use_stats_stream(self.hc_cfg, forward_batch, hidden_states):
stats_stream = self.hc_stats_stream
residual = hidden_states
mhc.fork_stats_stream(stats_stream)
x = mhc.combine(self.attn_hc, mhc.HcState(hidden_states, prev_pre))
with self.self_attn.maybe_use_decode_attn_tp(forward_batch):
x = self.self_attn(positions, x, forward_batch)
attn_pre, attn_post, attn_comb = mhc.mix_stats(
self.attn_hc, hidden_states, stats_stream
)
if stats_stream is not None:
torch.cuda.current_stream().wait_stream(stats_stream)
hidden_states = self.hc_post(x, residual, attn_post, attn_comb)
class DSparkV4PlatformStage(
NonCudaDSparkMixin, DSparkV4StageMixin, DeepseekV4PlatformDecoderLayer
):
"""Non-CUDA draft with platform mHC operations."""
residual = hidden_states
mhc.fork_stats_stream(stats_stream)
x = mhc.combine(self.ffn_hc, mhc.HcState(hidden_states, attn_pre))
x = self._run_ffn(x, forward_batch)
ffn_pre, ffn_post, ffn_comb = mhc.mix_stats(
self.ffn_hc, hidden_states, stats_stream
)
if stats_stream is not None:
torch.cuda.current_stream().wait_stream(stats_stream)
hidden_states = self.hc_post(x, residual, ffn_post, ffn_comb)
return hidden_states, ffn_pre
def _run_ffn(self, x: torch.Tensor, forward_batch: ForwardBatch) -> torch.Tensor:
shape = x.shape
x = x.reshape(-1, self.dim)
y = self._run_moe_ffn_dp_sync(
x, forward_batch, input_ids=None, input_ids_global=None
)
return y.view(shape)
DSparkV4Stage = DSparkV4CudaStage if _is_cuda else DSparkV4PlatformStage
class DeepseekV4ForCausalLMDSpark(nn.Module):
@@ -812,23 +766,28 @@ class DeepseekV4ForCausalLMDSpark(nn.Module):
and getattr(config, "hc_pre_from_prev_sublayer", False)
else None
)
self.stages = nn.ModuleList(
[
DSparkV4Stage(
config=config,
layer_id=stage_id,
stage_id=stage_id,
num_stages=self.num_stages,
num_target_layers=self.num_target_features,
quant_config=quant_config,
prefix=add_prefix(f"stages.{stage_id}", prefix),
alt_streams=self.alt_streams,
hc_stats_stream=self.hc_stats_stream,
moe_routed_quant_stream=self.moe_routed_quant_stream,
)
for stage_id in range(self.num_stages)
]
)
with (
layer_stack(final_read=hc_boundary.FinalRead())
if _is_cuda
else nullcontext()
):
self.stages = nn.ModuleList(
[
DSparkV4Stage(
config=config,
layer_id=stage_id,
stage_id=stage_id,
num_stages=self.num_stages,
num_target_layers=self.num_target_features,
quant_config=quant_config,
prefix=add_prefix(f"stages.{stage_id}", prefix),
alt_streams=self.alt_streams,
hc_stats_stream=self.hc_stats_stream,
moe_routed_quant_stream=self.moe_routed_quant_stream,
)
for stage_id in range(self.num_stages)
]
)
self.markov_head = DSparkV4MarkovHead(
vocab_size=int(config.vocab_size),
markov_rank=int(dspark_config.markov_rank),
@@ -942,20 +901,37 @@ class DeepseekV4ForCausalLMDSpark(nn.Module):
get_embedding: bool = False,
pp_proxy_tensors=None,
) -> LogitsProcessorOutput:
del get_embedding, pp_proxy_tensors
if not _is_cuda:
return self._forward_impl(input_ids, positions, forward_batch, input_embeds)
try:
return self._forward_impl(input_ids, positions, forward_batch, input_embeds)
finally:
residual_batch.clear(forward_batch)
def _forward_impl(self, input_ids, positions, forward_batch, input_embeds):
if input_embeds is None:
input_embeds = self.forward_embed(input_ids)
x = input_embeds
pre = None
# Draft MoE layers have no recorder layer index, so keep them out of the stats.
if _is_cuda:
residual_batch.start(forward_batch)
# Draft stages do not have recorder layer indices.
with get_global_expert_distribution_recorder().disable_this_region():
for stage in self.stages:
x, pre = stage(positions, x, forward_batch, pre)
if _is_cuda:
x = stage(positions, x, forward_batch)
else:
x, pre = stage(positions, x, forward_batch, pre)
if _is_cuda:
x = residual_batch.read_output(
x, forward_batch, hc_boundary.FinalRead(), None
)
if self.hc_pre_from_prev_sublayer:
pre, x = x.pre, x.residual
if self.hc_pre_from_prev_sublayer:
from sglang.kernels.ops.layernorm.mhc import hc_combine
x = hc_combine(x.flatten(1).float(), pre, self.hc_mult, x.dtype)
return LogitsProcessorOutput(next_token_logits=None, hidden_states=x)
def collapse_hc_head(self, x: torch.Tensor) -> torch.Tensor:
@@ -52,6 +52,9 @@ _is_xpu = is_xpu()
HcTriplet = Tuple[torch.Tensor, torch.Tensor, torch.Tensor]
MHC_POST_MULT_VALUE = 2.0
class HcConfig(NamedTuple):
"""Everything about a layer's mHC that is fixed for the model's lifetime."""
+48 -20
View File
@@ -1,4 +1,5 @@
import logging
from contextlib import nullcontext
from typing import Iterable, Optional, Tuple
import torch
@@ -18,6 +19,8 @@ from sglang.srt.layers.dp_attention import (
get_global_dp_buffer_len,
is_dp_attention_enabled,
)
from sglang.srt.layers.layer_boundary import layer_stack
from sglang.srt.layers.layer_boundary.residual import batch as residual_batch
from sglang.srt.layers.layernorm import RMSNorm
from sglang.srt.layers.linear import ReplicatedLinear
from sglang.srt.layers.logits_processor import LogitsProcessor
@@ -30,9 +33,11 @@ from sglang.srt.layers.vocab_parallel_embedding import (
)
from sglang.srt.model_executor.forward_batch_info import ForwardBatch
from sglang.srt.model_executor.forward_context import get_attn_backend
from sglang.srt.models import deepseek_v4_boundary as hc_boundary
from sglang.srt.models.deepseek_v4 import (
DeepseekV4DecoderLayer,
DeepseekV4ForCausalLM,
_is_cuda,
_is_npu,
wo_a_fp8_gemm_enabled,
)
@@ -95,15 +100,20 @@ class DeepseekV4ModelNextN(nn.Module):
else:
prefix = add_prefix("decoder", prefix)
self.decoder = DeepseekV4DecoderLayer(
config,
layer_id=0,
quant_config=quant_config,
is_nextn=True,
prefix=prefix,
alt_streams=None,
compress_ratio_override=COMPRESS_RATIO_NEXTN_LAYER,
)
with (
layer_stack(final_read=hc_boundary.FinalRead())
if _is_cuda
else nullcontext()
):
self.decoder = DeepseekV4DecoderLayer(
config,
layer_id=0,
quant_config=quant_config,
is_nextn=True,
prefix=prefix,
alt_streams=None,
compress_ratio_override=COMPRESS_RATIO_NEXTN_LAYER,
)
self.shared_head = nn.Module()
self.shared_head.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
@@ -187,17 +197,35 @@ class DeepseekV4ModelNextN(nn.Module):
# decoder layer reads the memoized gather instead of re-gathering.
prime_rope_cos_sin([self.decoder.self_attn], forward_batch, positions)
hidden_states, residual, post, comb = self.decoder(
positions=positions,
hidden_states=hidden_states,
forward_batch=forward_batch,
input_ids=input_ids,
input_ids_global=input_ids_global,
)
if residual is not None:
# NextN has a single decoder layer, so no later layer can consume a
# deferred fused hc_post state.
hidden_states = self.decoder.hc_post(hidden_states, residual, post, comb)
if _is_cuda:
residual_batch.start(forward_batch)
try:
hidden_states = self.decoder(
positions=positions,
hidden_states=hidden_states,
forward_batch=forward_batch,
input_ids=input_ids,
input_ids_global=input_ids_global,
)
hidden_states = residual_batch.read_output(
hidden_states, forward_batch, hc_boundary.FinalRead(), None
)
finally:
residual_batch.clear(forward_batch)
else:
hidden_states, residual, post, comb = self.decoder(
positions=positions,
hidden_states=hidden_states,
forward_batch=forward_batch,
input_ids=input_ids,
input_ids_global=input_ids_global,
)
if residual is not None:
# NextN has a single decoder layer, so no later layer can consume a
# deferred fused hc_post state.
hidden_states = self.decoder.hc_post(
hidden_states, residual, post, comb
)
pre_hc_head = hidden_states.flatten(1)
@@ -0,0 +1,716 @@
"""Non-CUDA DeepSeek V4 decoder and draft execution.
The mixins own platform mHC state, fusion, and TBO operations.
"""
from __future__ import annotations
from functools import cached_property
from typing import TYPE_CHECKING, Optional, Tuple, Union
import torch
from sglang.srt.layers.moe import get_moe_a2a_backend
from sglang.srt.layers.quantization.mxfp4_flashinfer_trtllm_moe import (
can_fuse_all_reduce,
)
from sglang.srt.models import deepseek_v4_mhc as mhc
from sglang.srt.models.deepseek_common.amd.deepseek_v4_fused_mhc import (
_gfx95_dense_post_attention_norm,
apply_attention_mhc,
apply_mhc_post_pre_boundary,
apply_moe_mhc,
attention_mhc_fusion,
)
from sglang.srt.models.deepseek_common.amd.deepseek_v4_fused_mhc import (
hc_boundary as make_amd_hc_boundary,
)
from sglang.srt.models.deepseek_common.amd.deepseek_v4_fused_mhc import (
moe_mhc_fusion,
)
from sglang.srt.models.deepseek_common.moe_transport import (
MoeRowTransport,
_tbo_combine_moe_output,
_tbo_gather_moe_input,
_tbo_run_moe,
_tbo_wait_moe_output,
shared_expert_runs_locally,
)
from sglang.srt.models.deepseek_v2 import MoEOutput
from sglang.srt.models.deepseek_v4_mhc import MHC_POST_MULT_VALUE, AttnOutput
from sglang.srt.runtime_context import get_parallel
from sglang.srt.utils import is_gfx95_supported
if TYPE_CHECKING:
from sglang.srt.model_executor.forward_batch_info import ForwardBatch
from sglang.srt.models import deepseek_v2
_is_gfx95_supported = is_gfx95_supported()
def _prepare_ffn_input(layer, hidden_states, residual, post, comb, forward_batch):
"""Close attention and prepare the FFN with the platform's mHC policy."""
if layer.use_fused_mhc_post_pre:
post_attn_norm_weight = (
layer._post_attention_layernorm_weight_bf16
if layer._post_attention_layernorm_weight_bf16 is not None
else layer.post_attention_layernorm.weight.data
)
fused = apply_mhc_post_pre_boundary(
hidden_states,
residual,
post,
comb,
layer.hc_ffn_fn,
layer.hc_ffn_scale,
layer.hc_ffn_base,
layer.hc_mult,
layer.rms_norm_eps,
layer.hc_eps,
MHC_POST_MULT_VALUE,
layer.hc_sinkhorn_iters,
post_attn_norm_weight,
layer.post_attention_layernorm.variance_epsilon,
fn_transpose=True,
)
if fused is not None:
residual, hidden_states, post, comb, norm_fused = fused
if not norm_fused:
hidden_states = layer.post_attention_layernorm(hidden_states)
return residual, hidden_states, post, comb
hidden_states = layer.hc_post(hidden_states, residual, post, comb)
residual = hidden_states
hidden_states, post, comb, norm_fused = layer.hc_pre(
hidden_states,
layer.hc_ffn_fn,
layer.hc_ffn_scale,
layer.hc_ffn_base,
norm=layer.post_attention_layernorm,
forward_batch=forward_batch,
)
if not norm_fused:
hidden_states = layer.post_attention_layernorm(hidden_states)
return residual, hidden_states, post, comb
class NonCudaDecoderMixin:
"""Platform tuple and HcState execution for target and NextN decoders.
CUDA mHC completion lives in deepseek_v4_boundary; operand movement and
finalization live in deepseek_common.moe_transport. Changes to completion
ordering must be checked in both implementations.
"""
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self._next_layer = None
self.local_boundary = None
self.next_boundary = None
def bind_next(self, next_layer):
if not self.hc_pre_from_prev_sublayer:
return
self._next_layer = None
if isinstance(next_layer, NonCudaDecoderMixin) and next_layer.engram is None:
self._next_layer = next_layer
def refresh_mhc_norm_weight_cache(self):
super().refresh_mhc_norm_weight_cache()
self.local_boundary = None
self.next_boundary = None
@cached_property
def _can_fuse_attn_mhc(self):
return (
get_parallel().tp_size == self.self_attn.attn_tp_size == 4
and self.self_attn.wo_b.reduce_results
and self.local_boundary.norm_fusable
and mhc.can_fuse_post(self.hc_cfg)
)
def forward_hc_pre_from_prev(
self,
positions: torch.Tensor,
state: mhc.HcState,
input_ids: torch.Tensor,
forward_batch: ForwardBatch,
input_ids_global: torch.Tensor,
) -> mhc.HcState:
"""The layer's two hyper-connections, each collapsing with the previous one's
pre-mix."""
self._init_boundaries()
stats_stream = None
if mhc.use_stats_stream(self.hc_cfg, forward_batch, state.residual):
stats_stream = self.hc_stats_stream
def run_attn_hc(state: mhc.HcState) -> mhc.HcState:
assert self.attn_hc is not None
residual = state.residual
quantized = [] if self.self_attn.accepts_mxfp8_swizzled_input() else None
mhc.fork_stats_stream(stats_stream)
x = mhc.combine(self.attn_hc, state, quantized)
precomputed = state.stats
state.release()
del state
world_size = self.self_attn.attn_tp_size
fuse_all_reduce_mhc = self._can_fuse_attn_mhc and can_fuse_all_reduce(
x.shape[0], self.hc_cfg.hidden
)
with self.self_attn.maybe_use_decode_attn_tp(forward_batch):
y = self.self_attn(
x=x,
positions=positions,
forward_batch=forward_batch,
x_quant=quantized[0] if quantized else None,
defer_all_reduce=fuse_all_reduce_mhc,
)
del x
return mhc.run_attn_post(
self.attn_hc,
y,
residual,
stats_stream=stats_stream,
next=self.local_boundary,
world_size=world_size,
precomputed=precomputed,
)
def run_ffn_hc(state: mhc.HcState) -> mhc.HcState:
assert self.ffn_hc is not None
residual = state.residual
mhc.fork_stats_stream(stats_stream)
x = mhc.combine(self.ffn_hc, state)
precomputed = state.stats
state.release()
del state
fuse_all_reduce_mhc = self._can_fuse_ffn_mhc and can_fuse_all_reduce(
x.shape[0], self.hc_cfg.hidden
)
y = self._run_moe_ffn_dp_sync(
x,
forward_batch,
input_ids=input_ids,
input_ids_global=input_ids_global,
return_moe_output=fuse_all_reduce_mhc,
)
del x
return mhc.run_moe_post(
self.ffn_hc,
y,
residual,
stats_stream=stats_stream,
next=self.next_boundary,
world_size=self.mlp.tp_size,
precomputed=precomputed,
)
return run_ffn_hc(run_attn_hc(state))
def _init_boundaries(self) -> None:
"""Resolved on first forward: `accepts_mxfp8_swizzled_input` is not settled
until `process_weights_after_loading`."""
if self.local_boundary is not None:
return
# ATTN[i] -> MOE[i]
self.local_boundary = mhc.make_boundary(
self.post_attention_layernorm,
accepts_mxfp8=False,
hc=self.ffn_hc,
)
# MOE[i] -> ATTN[i+1]
if self._next_layer is not None:
self.next_boundary = mhc.make_boundary(
self._next_layer.input_layernorm,
accepts_mxfp8=self._next_layer.self_attn.accepts_mxfp8_swizzled_input(),
hc=self._next_layer.attn_hc,
)
def forward(
self,
positions: torch.Tensor,
hidden_states: torch.Tensor,
input_ids: torch.Tensor,
forward_batch: ForwardBatch,
input_ids_global: torch.Tensor,
prev_residual: Optional[torch.Tensor] = None,
prev_post: Optional[torch.Tensor] = None,
prev_comb: Optional[torch.Tensor] = None,
) -> Tuple[
torch.Tensor,
Optional[torch.Tensor],
Optional[torch.Tensor],
Optional[torch.Tensor],
]:
use_fused = self.use_fused_mhc_post_pre
fused = None
if prev_residual is not None and use_fused:
# Dispatch cascade: aiter HIP (gfx95) -> Triton (gfx95 small-batch
# <=64 tokens, or gfx1250 all sizes) -> TileLang -> None.
input_norm_weight = (
self._input_layernorm_weight_bf16
if self._input_layernorm_weight_bf16 is not None
else self.input_layernorm.weight.data
)
fused = apply_mhc_post_pre_boundary(
hidden_states,
prev_residual,
prev_post,
prev_comb,
self.hc_attn_fn,
self.hc_attn_scale,
self.hc_attn_base,
self.hc_mult,
self.rms_norm_eps,
self.hc_eps,
MHC_POST_MULT_VALUE,
self.hc_sinkhorn_iters,
input_norm_weight,
self.input_layernorm.variance_epsilon,
fn_transpose=False,
)
if fused is not None:
residual, hidden_states, post, comb, norm_fused = fused
else:
if prev_residual is not None and use_fused:
hidden_states = self.hc_post(
hidden_states, prev_residual, prev_post, prev_comb
)
residual = hidden_states
hidden_states, post, comb, norm_fused = self.hc_pre(
hidden_states,
self.hc_attn_fn,
self.hc_attn_scale,
self.hc_attn_base,
norm=self.input_layernorm,
forward_batch=forward_batch,
)
if not norm_fused:
# Triton post+pre can leave norm separate: gfx95 may quantize it,
# while gfx1250 must keep the plain layernorm path.
hidden_states, x_quant = self._input_norm(
hidden_states, allow_aiter_quant=_is_gfx95_supported
)
else:
x_quant = None
with self.self_attn.maybe_use_decode_attn_tp(forward_batch):
hidden_states = self.self_attn(
x=hidden_states,
positions=positions,
forward_batch=forward_batch,
x_quant=x_quant,
)
del fused
residual, hidden_states, post, comb = _prepare_ffn_input(
self, hidden_states, residual, post, comb, forward_batch
)
hidden_states = self._run_moe_ffn_dp_sync(
hidden_states,
forward_batch,
input_ids=input_ids,
input_ids_global=input_ids_global,
)
if not use_fused:
hidden_states = self.hc_post(hidden_states, residual, post, comb)
return hidden_states, None, None, None
# Return the deferred FFN hc_post state; the next layer consumes it with
# cross-layer fusion, and the final layer is completed in DeepseekV4Model.
return hidden_states, residual, post, comb
def _run_moe_ffn_dp_sync(
self,
hidden_states: torch.Tensor,
forward_batch: ForwardBatch,
*,
input_ids: Optional[torch.Tensor],
input_ids_global: Optional[torch.Tensor],
return_moe_output: bool = False,
) -> Union[torch.Tensor, deepseek_v2.MoEOutput]:
prepared = MoeRowTransport(
self.mlp,
hidden_states,
forward_batch,
input_ids,
input_ids_global,
cp_return_contiguous=self.config.model_type == "deepseek_v41",
)
with prepared.scope():
return_moe_output = (
return_moe_output
and not prepared._use_cp
and not prepared._use_tp_moe_gather
and not prepared._use_tp_attn_a2a_scatter
)
hidden_states = self.mlp(
prepared.input,
forward_batch,
input_ids=prepared.input_ids,
input_ids_global=prepared.input_ids_global,
skip_shared_experts=prepared.do_shared_local,
return_moe_output=return_moe_output,
)
prepared.input = None
return prepared.return_output(hidden_states)
def op_mhc_prepare_attn(
self,
state,
positions: torch.Tensor,
hidden_states: torch.Tensor,
forward_batch: ForwardBatch,
residual: Optional[torch.Tensor] = None,
tbo_subbatch_index: Optional[int] = None,
**kwargs,
):
# Non-fused attention-side mHC pre + input layernorm.
attn_residual = hidden_states
hidden_states, post, comb, norm_fused = self.hc_pre(
hidden_states,
self.hc_attn_fn,
self.hc_attn_scale,
self.hc_attn_base,
norm=self.input_layernorm,
forward_batch=forward_batch,
)
if not norm_fused:
hidden_states, x_quant = self._input_norm(hidden_states)
else:
x_quant = None
state.attn_residual = attn_residual
state.attn_post = post
state.attn_comb = comb
state.hidden_states_after_input_norm = hidden_states
state.attn_x_quant = x_quant
# mori's op_output slices final_hidden_states[:num_tokens].
if get_moe_a2a_backend().is_mori():
state.num_tokens = attn_residual.shape[0]
state.update(
dict(
forward_batch=forward_batch,
positions=positions,
tbo_subbatch_index=tbo_subbatch_index,
)
)
def op_mhc_post_attn_pre_mlp(self, state):
# Close the attention mHC (hc_post), then open the FFN-side mHC pre +
# post-attention layernorm. Produces the 2D MoE input.
#
# Pop each boundary tensor from the state EXACTLY ONCE, up front, and
# reuse the locals for both the fused attempt and the non-fused
# fallback. apply_mhc_post_pre_boundary() returns None when it declines
# to fuse -- most importantly for the 0-token DP two-batch-overlap idle
# ubatch -- in which case control must fall through to the unfused
# hc_post. Popping in the fused call's arguments and again in the
# fallback would double-pop -> KeyError: 'hidden_states_after_attn' on
# every idle DP rank. use_fused_mhc_post_pre is on whenever the aiter
# gfx95 mHC path is available (is_cross_layer_mhc_fusion_enabled), so
# this fallback is reached under DP regardless of the TileLang env.
hidden_states_after_attn = state.pop("hidden_states_after_attn")
attn_residual = state.pop("attn_residual")
attn_post = state.pop("attn_post")
attn_comb = state.pop("attn_comb")
residual, hidden_states, post, comb = _prepare_ffn_input(
self,
hidden_states_after_attn,
attn_residual,
attn_post,
attn_comb,
state.forward_batch,
)
state.ffn_residual = residual
state.ffn_post = post
state.ffn_comb = comb
state.hidden_states_mlp_input = hidden_states
def op_mhc_postprocess(self, state):
# Close the FFN mHC (hc_post) and emit the next layer's input dict.
hidden_states = self.hc_post(
state.pop("hidden_states_mlp_output"),
state.pop("ffn_residual"),
state.pop("ffn_post"),
state.pop("ffn_comb"),
)
output = dict(
positions=state.positions,
hidden_states=hidden_states,
# DSV4 non-fused layers carry no residual across layers; the key is
# required by the next layer's op_mhc_prepare_attn (ignored).
residual=None,
forward_batch=state.forward_batch,
tbo_subbatch_index=state.tbo_subbatch_index,
)
state.clear(
expect_keys={
"positions",
"forward_batch",
"tbo_subbatch_index",
}
)
return output
def op_gather_a(self, state):
local = state.pop("hidden_states_mlp_input")
state.do_shared_local = shared_expert_runs_locally(self.mlp)
state.global_hidden, state.gather_event, state.shared_local = (
_tbo_gather_moe_input(
self.mlp,
local,
state.forward_batch,
state.tbo_subbatch_index,
state.do_shared_local,
)
)
state.gather_keepalive = local
def op_gather_b(self, state):
torch.cuda.current_stream().wait_event(state.pop("gather_event"))
# Compute now ordered after the gather -> the gather input is safe to
# release (freed on the compute stream, no record_stream deferral).
state.pop("gather_keepalive")
def op_moe(self, state):
state.global_expert_out = _tbo_run_moe(
self, state.pop("global_hidden"), state.forward_batch, state.do_shared_local
)
def op_combine_a(self, state):
output = state.pop("global_expert_out")
state.local_out, state.combine_event = _tbo_combine_moe_output(
output, state.tbo_subbatch_index
)
state.combine_keepalive = output
def op_combine_b(self, state):
state.hidden_states_mlp_output = _tbo_wait_moe_output(
state.pop("local_out"),
state.pop("combine_event"),
state.pop("shared_local"),
)
state.pop("combine_keepalive")
state.pop("do_shared_local")
class NonCudaDSparkMixin(NonCudaDecoderMixin):
"""Draft stage execution with plain norms and its attention call."""
def forward(
self,
positions: torch.Tensor,
hidden_states: torch.Tensor,
forward_batch: ForwardBatch,
prev_pre: Optional[torch.Tensor] = None,
) -> Tuple[torch.Tensor, Optional[torch.Tensor]]:
if self.hc_pre_from_prev_sublayer:
return self._forward_hc_pre_from_prev(
positions, hidden_states, forward_batch, prev_pre
)
residual = hidden_states
x, post, comb = self._hc_pre_block(
hidden_states, self.hc_attn_fn, self.hc_attn_scale, self.hc_attn_base
)
x = self.input_layernorm(x)
with self.self_attn.maybe_use_decode_attn_tp(forward_batch):
x = self.self_attn(positions, x, forward_batch)
x = self._hc_post_block(x, residual, post, comb)
residual = x
x, post, comb = self._hc_pre_block(
x, self.hc_ffn_fn, self.hc_ffn_scale, self.hc_ffn_base
)
x = self.post_attention_layernorm(x)
x = self._run_ffn(x, forward_batch)
x = self._hc_post_block(x, residual, post, comb)
return x, None
def _hc_pre_block(
self,
x: torch.Tensor,
hc_fn: torch.Tensor,
hc_scale: torch.Tensor,
hc_base: torch.Tensor,
) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
y, post, comb, _ = self.hc_pre(x, hc_fn, hc_scale, hc_base)
return y, post, comb
def _hc_post_block(
self,
x: torch.Tensor,
residual: torch.Tensor,
post: torch.Tensor,
comb: torch.Tensor,
) -> torch.Tensor:
return self.hc_post(x, residual, post, comb)
def _forward_hc_pre_from_prev(
self,
positions: torch.Tensor,
hidden_states: torch.Tensor,
forward_batch: ForwardBatch,
prev_pre: Optional[torch.Tensor],
) -> Tuple[torch.Tensor, torch.Tensor]:
# The draft stages run the plain four steps: no fused post, no seams, so
# they need nothing from `bind_next` and never hand anything forward.
assert self.attn_hc is not None and self.ffn_hc is not None
stats_stream = None
if mhc.use_stats_stream(self.hc_cfg, forward_batch, hidden_states):
stats_stream = self.hc_stats_stream
residual = hidden_states
mhc.fork_stats_stream(stats_stream)
x = mhc.combine(self.attn_hc, mhc.HcState(hidden_states, prev_pre))
with self.self_attn.maybe_use_decode_attn_tp(forward_batch):
x = self.self_attn(positions, x, forward_batch)
attn_pre, attn_post, attn_comb = mhc.mix_stats(
self.attn_hc, hidden_states, stats_stream
)
if stats_stream is not None:
torch.cuda.current_stream().wait_stream(stats_stream)
hidden_states = self.hc_post(x, residual, attn_post, attn_comb)
residual = hidden_states
mhc.fork_stats_stream(stats_stream)
x = mhc.combine(self.ffn_hc, mhc.HcState(hidden_states, attn_pre))
x = self._run_ffn(x, forward_batch)
ffn_pre, ffn_post, ffn_comb = mhc.mix_stats(
self.ffn_hc, hidden_states, stats_stream
)
if stats_stream is not None:
torch.cuda.current_stream().wait_stream(stats_stream)
hidden_states = self.hc_post(x, residual, ffn_post, ffn_comb)
return hidden_states, ffn_pre
def _run_ffn(self, x: torch.Tensor, forward_batch: ForwardBatch) -> torch.Tensor:
shape = x.shape
x = x.reshape(-1, self.dim)
y = self._run_moe_ffn_dp_sync(
x, forward_batch, input_ids=None, input_ids_global=None
)
return y.view(shape)
def forward_layer_fused_boundary(
model,
i: int,
state,
*,
positions: torch.Tensor,
input_ids: torch.Tensor,
forward_batch,
input_ids_global: torch.Tensor,
capture_dspark: bool,
):
"""Layer i through the fused mHC boundary. Its FFN hc_post stays pending for the next
layer's boundary launch unless that layer reads the residual stream first (Engram gate,
DSpark capture) or there is none; hidden_states is None while a post is pending."""
nxt = i + 1
defer_post = (
nxt < model.end_layer
and model.layers[nxt].engram is None
and not (capture_dspark and nxt in model.dspark_layers_to_capture)
)
# The boundary speaks its own (hidden_states, prev_pre, pending_post) triple;
# the loop speaks HcState. This is the only place the two meet.
from sglang.srt.models.deepseek_v4_mhc import HcPending, HcState
pending = isinstance(state.streams, HcPending)
hidden_states, prev_pre, pending_post = forward_hc_pre_from_prev_fused_boundary(
model.layers[i],
positions=positions,
hidden_states=None if pending else state.streams,
input_ids=input_ids,
forward_batch=forward_batch,
input_ids_global=input_ids_global,
prev_pre=state.pre,
pending_post=tuple(state.streams) if pending else None,
defer_post=defer_post,
)
return HcState(
hidden_states if pending_post is None else HcPending(*pending_post),
prev_pre,
)
def forward_hc_pre_from_prev_fused_boundary(
layer,
positions: torch.Tensor,
hidden_states: Optional[torch.Tensor],
input_ids: torch.Tensor,
forward_batch,
input_ids_global: torch.Tensor,
prev_pre: Optional[torch.Tensor],
pending_post: Optional[Tuple[torch.Tensor, ...]],
defer_post: bool,
) -> Tuple[Optional[torch.Tensor], torch.Tensor, Optional[Tuple[torch.Tensor, ...]]]:
"""ROCm fused attention and FFN hyper-connections.
pending_post is the previous layer's unapplied FFN hc_post (x, residual, post,
comb); with defer_post this layer's is returned the same way and hidden_states is None."""
if pending_post is None:
pending_post = (None, hidden_states, None, None)
residual, x, attn_coefficients = make_amd_hc_boundary(
layer,
*pending_post,
prev_pre,
layer.hc_attn_fn,
layer.hc_attn_scale,
layer.hc_attn_base,
)
# the boundary's reduce + sinkhorn rides in the norm launch
x, x_quant = layer._input_norm(
x, allow_aiter_quant=False, coefficients=attn_coefficients
)
with layer.self_attn.maybe_use_decode_attn_tp(forward_batch):
mhc = attention_mhc_fusion(layer, residual, attn_coefficients, forward_batch)
x = layer.self_attn(
x=x,
positions=positions,
forward_batch=forward_batch,
x_quant=x_quant,
defer_all_reduce=mhc is not None,
)
if mhc is not None:
# wo_b honors the deferred reduce under attention_mhc_fusion's gates;
# fail loud rather than fall back to an unfused reduce.
assert isinstance(x, AttnOutput)
residual = apply_attention_mhc(x.partial, mhc)
x = None
residual, x, ffn_coefficients = make_amd_hc_boundary(
layer,
x,
residual,
attn_coefficients.post if x is not None else None,
attn_coefficients.comb if x is not None else None,
attn_coefficients.pre,
layer.hc_ffn_fn,
layer.hc_ffn_scale,
layer.hc_ffn_base,
)
x = _gfx95_dense_post_attention_norm(layer, x, ffn_coefficients)
mhc = moe_mhc_fusion(layer, residual, ffn_coefficients, forward_batch)
x = layer._run_moe_ffn_dp_sync(
x,
forward_batch,
input_ids=input_ids,
input_ids_global=input_ids_global,
return_moe_output=mhc is not None,
)
ffn_pre, ffn_post, ffn_comb = ffn_coefficients.tensors()
if mhc is not None:
# The MoE TP path honors the deferred output under moe_mhc_fusion's gates;
# fail loud rather than fall back to an unfused reduce. The fused post
# carries the reduce, so take the merged rows without one.
assert isinstance(x, MoEOutput)
# Reduction already applied post. The next boundary consumes this
# materialized residual, including when it would normally defer post.
return apply_moe_mhc(x.get_merged(), mhc), ffn_pre, None
if defer_post:
return None, ffn_pre, (x, residual, ffn_post, ffn_comb)
return layer.hc_post(x, residual, ffn_post, ffn_comb), ffn_pre, None
+1 -1
View File
@@ -137,7 +137,7 @@ if _use_aiter_gfx95:
logger = logging.getLogger(__name__)
# Matches DeepSeek-V4's _MHC_POST_MULT_VALUE; the fused and unfused boundaries
# Matches deepseek_v4_mhc.MHC_POST_MULT_VALUE; the fused and unfused boundaries
# must agree on it.
_MHC_POST_MULT_VALUE = 2.0