[Refactor] deepseek-v4: Use explicit stage boundaries and row transport (#43607)
This commit is contained in:
@@ -63,7 +63,10 @@ class OperationsStrategy:
|
||||
for layer in layers
|
||||
]
|
||||
)
|
||||
elif layer_name == "DeepseekV4DecoderLayer":
|
||||
# Other model families do not need to import the V4 implementation.
|
||||
from sglang.srt.models.deepseek_v4 import DeepseekV4DecoderLayerBase
|
||||
|
||||
if isinstance(layers[0], DeepseekV4DecoderLayerBase):
|
||||
return OperationsStrategy.concat(
|
||||
[
|
||||
_compute_moe_deepseek_v4_layer_operations_strategy_tbo(
|
||||
@@ -72,8 +75,7 @@ class OperationsStrategy:
|
||||
for layer in layers
|
||||
]
|
||||
)
|
||||
else:
|
||||
raise NotImplementedError
|
||||
raise NotImplementedError
|
||||
|
||||
|
||||
def _assert_all_same(items: List):
|
||||
|
||||
@@ -32,6 +32,7 @@ from sglang.srt.layers.layer_boundary.contracts import (
|
||||
CpMoves,
|
||||
EdgeContract,
|
||||
EntryPath,
|
||||
ExitFacts,
|
||||
ProducerReduction,
|
||||
StageKind,
|
||||
StagePath,
|
||||
@@ -126,6 +127,7 @@ class StagePlan:
|
||||
is_branch=False,
|
||||
terminal=False,
|
||||
writes_at_handoff=False,
|
||||
row_transport=None,
|
||||
finishes_directly=False,
|
||||
qkv_latent_func=None,
|
||||
fusions=None,
|
||||
@@ -201,9 +203,15 @@ class StagePlan:
|
||||
returns_over_dp=out.returns_over_dp,
|
||||
writes_at_handoff=writes_at_handoff,
|
||||
)
|
||||
self.paths[variant] = msgspec.structs.replace(
|
||||
path, exit=exit_facts(kind, self, variant, path)
|
||||
fixed_exit = (
|
||||
ExitFacts(
|
||||
writes_residual=path.output.update.applied_at_exit
|
||||
or writes_at_handoff
|
||||
)
|
||||
if row_transport is not None
|
||||
else exit_facts(kind, self, variant, path)
|
||||
)
|
||||
self.paths[variant] = msgspec.structs.replace(path, exit=fixed_exit)
|
||||
|
||||
@property
|
||||
def incoming_residual_rows(self):
|
||||
@@ -278,10 +286,16 @@ def _bind_stage(
|
||||
if incoming.entries.keys() != outgoing.exits.keys():
|
||||
raise ValueError("incoming and outgoing batch variants disagree")
|
||||
update = declaration.update
|
||||
if pipeline_handoff and not (update.is_plain_add or update.applied_at_exit):
|
||||
if pipeline_handoff and not (
|
||||
update.is_plain_add
|
||||
or update.applied_at_exit
|
||||
or (declaration.writes_at_handoff and declaration.row_transport is not None)
|
||||
):
|
||||
raise NotImplementedError(
|
||||
"pipeline boundaries require a producer-written residual"
|
||||
)
|
||||
if declaration.row_transport is not None and declaration.row_transport.bind is None:
|
||||
raise ValueError("a local stage requires its compute transport")
|
||||
if declaration.sum_impl is not None and declaration.sum_impl.run is None:
|
||||
raise ValueError("a local stage requires its custom sum implementation")
|
||||
if declaration.output_complete and declaration.sum_impl is not None:
|
||||
@@ -311,7 +325,10 @@ def _bind_stage(
|
||||
"an update applied at the stage's exit with a gather over attention CP"
|
||||
)
|
||||
attn_input_adapter = None
|
||||
if declaration.kind is StageKind.ATTENTION:
|
||||
if (
|
||||
declaration.kind is StageKind.ATTENTION
|
||||
and declaration.row_transport is None
|
||||
):
|
||||
# On an input-scattered batch the step gathers the rows itself for
|
||||
# an attention whose QKV hook does not gather them after the
|
||||
# projection.
|
||||
@@ -332,6 +349,7 @@ def _bind_stage(
|
||||
is_branch=declaration.prepared_from is not None,
|
||||
terminal=declaration.terminal,
|
||||
writes_at_handoff=declaration.writes_at_handoff,
|
||||
row_transport=declaration.row_transport,
|
||||
attn_tp_gather=declaration.attn_tp_gather,
|
||||
# The exit runs its consumer's gather: the next stage's, or the
|
||||
# final read's.
|
||||
|
||||
@@ -314,6 +314,7 @@ class ExitFacts(msgspec.Struct, frozen=True):
|
||||
is not one all-reduce, may still be left to a fused consumer.
|
||||
"""
|
||||
|
||||
writes_residual: bool = False
|
||||
may_defer_sum: bool = False
|
||||
defers_mixer_sum: bool = False
|
||||
sum_in_reduce_scatter: bool = False
|
||||
|
||||
@@ -55,6 +55,7 @@ from sglang.srt.layers.layer_boundary.residual.add_norm import (
|
||||
NORM_READOUT,
|
||||
PLAIN_ADD,
|
||||
)
|
||||
from sglang.srt.layers.layer_boundary.row_transport import RowTransport
|
||||
from sglang.srt.layers.layer_boundary.stage import StageBoundary
|
||||
from sglang.srt.layers.moe import is_moe_input_scattered_across_dp_ranks
|
||||
from sglang.srt.runtime_context import get_exec, get_parallel
|
||||
@@ -305,6 +306,9 @@ class StageDeclaration:
|
||||
sum_impl: Optional implementation of the declared output sum using
|
||||
the model's custom collective. Its batch predicate, when supplied,
|
||||
preserves eligibility for carrying that sum to the consumer.
|
||||
row_transport: Explicit source, compute and return rows for a stage
|
||||
whose operand moves independently of its residual. The bound
|
||||
invocation owns output completion and inverse transport.
|
||||
attn_tp_gather: Optional implementation of the gather over attention
|
||||
TP that brings rows into this stage: at its own entry, or at its
|
||||
producer's exit, which runs its consumer's gather. Given this rank's
|
||||
@@ -340,6 +344,7 @@ class StageDeclaration:
|
||||
output_complete: bool = False
|
||||
attn_tp_gather: Optional[Callable] = None
|
||||
sum_impl: Optional[SumImplementation] = None
|
||||
row_transport: Optional[RowTransport] = None
|
||||
# Only declarations participate in construction, never executable stages.
|
||||
previous: Optional[StageDeclaration] = None
|
||||
prepared_from: Optional[StageDeclaration] = None
|
||||
@@ -436,6 +441,7 @@ def declare_attn(
|
||||
attn_tp_gather=None,
|
||||
in_token_order=False,
|
||||
sum_impl=None,
|
||||
row_transport=None,
|
||||
):
|
||||
"""Declare attention or a mixer; construct its executable boundary later.
|
||||
|
||||
@@ -479,6 +485,7 @@ def declare_attn(
|
||||
attn_tp_gather=attn_tp_gather,
|
||||
in_token_order=in_token_order,
|
||||
sum_impl=sum_impl,
|
||||
row_transport=row_transport,
|
||||
)
|
||||
|
||||
|
||||
@@ -493,6 +500,7 @@ def declare_ffn(
|
||||
exit_rows=None,
|
||||
output_complete=False,
|
||||
sum_impl=None,
|
||||
row_transport=None,
|
||||
):
|
||||
"""Declare a dense or MoE FFN independently of its compute module.
|
||||
|
||||
@@ -526,6 +534,7 @@ def declare_ffn(
|
||||
exit_rows=exit_rows or tbo_exit_rows(sparse, next_layer_sparse),
|
||||
output_complete=output_complete,
|
||||
sum_impl=sum_impl,
|
||||
row_transport=row_transport,
|
||||
)
|
||||
|
||||
|
||||
@@ -620,6 +629,39 @@ def _resolve_stage(stage, variant, arrival, following=None, *, final_read=None):
|
||||
and the stage after it (None when it ends the stack or a branch)."""
|
||||
axes, attention, local, full = _row_layouts(variant)
|
||||
_reject_unsupported(stage)
|
||||
if stage.row_transport is not None:
|
||||
if (
|
||||
stage.sum_impl is not None
|
||||
or stage.output_transform is not None
|
||||
or stage.output_complete
|
||||
or stage.in_token_order
|
||||
):
|
||||
raise NotImplementedError(
|
||||
"operand transport cannot bypass another output contract"
|
||||
)
|
||||
if stage.exit_rows not in (None, ExitRows.ATTENTION):
|
||||
raise NotImplementedError(
|
||||
"operand transport declares its return rows explicitly"
|
||||
)
|
||||
try:
|
||||
rows = stage.row_transport.rows[variant]
|
||||
except KeyError:
|
||||
raise NotImplementedError(
|
||||
"operand transport does not support this batch variant"
|
||||
) from None
|
||||
if rows.group is not None and not isinstance(rows.group, SumGroup):
|
||||
raise TypeError("operand transport requires a declared SumGroup")
|
||||
if rows.decode_group is not None and not isinstance(
|
||||
rows.decode_group, SumGroup
|
||||
):
|
||||
raise TypeError("operand transport requires a declared decode SumGroup")
|
||||
if rows.source != arrival.produced.layout or rows.returned != rows.source:
|
||||
raise NotImplementedError("compute transport must return its source rows")
|
||||
declaration = StageContract(
|
||||
InputContract(rows.source, read=stage.read),
|
||||
OutputContract(rows.returned, group=rows.group, update=stage.update),
|
||||
)
|
||||
return _Flow(declaration, arrival, arrival.residual, rows.returned)
|
||||
if stage.kind is StageKind.FFN:
|
||||
declaration, residual, returned = _resolve_ffn(
|
||||
variant,
|
||||
@@ -811,8 +853,13 @@ def _connect_line(
|
||||
# their row choices still determine what crosses the pipeline boundary.
|
||||
needs_unpadded = any(
|
||||
stage is not None
|
||||
and stage.kind is StageKind.FFN
|
||||
and _ffn_on_rank_rows(stage.sparse, stage.dense_tp_size)
|
||||
and (
|
||||
stage.row_transport is not None
|
||||
or (
|
||||
stage.kind is StageKind.FFN
|
||||
and _ffn_on_rank_rows(stage.sparse, stage.dense_tp_size)
|
||||
)
|
||||
)
|
||||
for stage in (before, *line, after)
|
||||
)
|
||||
variants = tuple(
|
||||
@@ -1074,8 +1121,9 @@ def _handed_off(declaration):
|
||||
return replace(
|
||||
declaration,
|
||||
exit_rows=ExitRows.ATTENTION,
|
||||
writes_at_handoff=_ffn_on_rank_rows(
|
||||
declaration.sparse, declaration.dense_tp_size
|
||||
writes_at_handoff=(
|
||||
declaration.row_transport is not None
|
||||
or _ffn_on_rank_rows(declaration.sparse, declaration.dense_tp_size)
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
@@ -114,6 +114,9 @@ def facts_of(declaration):
|
||||
sum_impl=declaration.sum_impl.facts()
|
||||
if declaration.sum_impl is not None
|
||||
else None,
|
||||
row_transport=declaration.row_transport.facts()
|
||||
if declaration.row_transport is not None
|
||||
else None,
|
||||
previous=None,
|
||||
prepared_from=None,
|
||||
)
|
||||
|
||||
@@ -48,9 +48,11 @@ def clear(forward_batch):
|
||||
"""Release invocation state on completion or an exception."""
|
||||
stream = forward_batch.residual_stream
|
||||
if stream is not None:
|
||||
if stream.transport is not None:
|
||||
stream.transport[1].release()
|
||||
if stream.pending is not None:
|
||||
stream.pending.release()
|
||||
stream.pending = stream.residual = None
|
||||
stream.pending = stream.residual = stream.transport = None
|
||||
forward_batch.residual_stream = None
|
||||
|
||||
|
||||
|
||||
@@ -101,11 +101,12 @@ class ResidualStream:
|
||||
selection live on the bound stage, not on tensor values.
|
||||
"""
|
||||
|
||||
__slots__ = ("residual", "pending")
|
||||
__slots__ = ("residual", "pending", "transport")
|
||||
|
||||
def __init__(self, residual=None):
|
||||
self.residual = residual
|
||||
self.pending = None
|
||||
self.transport = None
|
||||
|
||||
@classmethod
|
||||
def from_handoff(cls, hidden, residual, update):
|
||||
|
||||
@@ -0,0 +1,161 @@
|
||||
"""Transport a compute operand without moving its source residual."""
|
||||
|
||||
from contextlib import nullcontext
|
||||
from typing import Callable, Mapping, Optional
|
||||
|
||||
import msgspec
|
||||
|
||||
from sglang.srt.layers.layer_boundary.contracts import BatchVariant
|
||||
from sglang.srt.layers.layer_boundary.layout import Layout, SumGroup, _sum_group
|
||||
from sglang.srt.layers.layer_boundary.output import (
|
||||
DeferredFinalize,
|
||||
UnreducedOutput,
|
||||
complete_owed,
|
||||
)
|
||||
from sglang.srt.layers.layer_boundary.residual import bind_update
|
||||
|
||||
|
||||
class TransportRows(msgspec.Struct, frozen=True):
|
||||
"""The source/read, compute, and completed output rows of one adapter.
|
||||
|
||||
The residual stays on source rows. ``group`` names the compute output's
|
||||
outstanding sum; None means the computation (e.g. EP combine) completes it.
|
||||
The boundary completes that sum and returns source rows, or publishes an
|
||||
explicit completion handle on source rows. It never applies the update.
|
||||
"""
|
||||
|
||||
source: Layout
|
||||
compute: Layout
|
||||
returned: Layout
|
||||
group: Optional[SumGroup] = None
|
||||
decode_group: Optional[SumGroup] = None
|
||||
|
||||
|
||||
class RowTransport(msgspec.Struct, frozen=True):
|
||||
"""Declare separate compute/residual rows and bind one invocation's moves.
|
||||
|
||||
``bind(input, batch, context, rows, overlapped_rows)`` returns an invocation
|
||||
with ``input`` and ``scope()``. The boundary checks its selected rows,
|
||||
prepares its output, completes owed work and runs its return move.
|
||||
Facts retain row contracts only.
|
||||
"""
|
||||
|
||||
rows: Mapping[BatchVariant, TransportRows]
|
||||
bind: Optional[Callable] = None
|
||||
overlapped_rows: Optional[Mapping[BatchVariant, TransportRows]] = None
|
||||
|
||||
def facts(self):
|
||||
return RowTransport(
|
||||
dict(self.rows),
|
||||
overlapped_rows=(
|
||||
dict(self.overlapped_rows) if self.overlapped_rows is not None else None
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
class ComputeTransport:
|
||||
"""One prepared operand with declared rows and output completion policy.
|
||||
|
||||
``prepare_output`` exposes the compute result's outstanding work.
|
||||
The boundary completes it before ``return_output`` moves the result back;
|
||||
a move that performs the sum declares ``sum_in_return`` before compute.
|
||||
Residual update and consumer normalization stay with the boundary.
|
||||
"""
|
||||
|
||||
@staticmethod
|
||||
def scope():
|
||||
return nullcontext()
|
||||
|
||||
is_overlapped = False
|
||||
defers_sum = False
|
||||
sum_in_return = False
|
||||
|
||||
def release(self):
|
||||
pass
|
||||
|
||||
def prepare_output(self, output):
|
||||
return output
|
||||
|
||||
def return_output(self, output):
|
||||
return output
|
||||
|
||||
|
||||
def validate_invocation(invocation, ordinary, overlapped):
|
||||
expected = overlapped if invocation.is_overlapped else ordinary
|
||||
if expected is None or invocation.rows != expected:
|
||||
raise ValueError("compute transport does not match its declared rows and sum")
|
||||
if invocation.defers_sum and (
|
||||
invocation.sum_in_return or expected.compute != expected.returned
|
||||
):
|
||||
raise ValueError("a deferred transport sum must already be on its return rows")
|
||||
if invocation.sum_in_return and expected.group is None:
|
||||
raise ValueError("a transport move cannot complete an undeclared sum")
|
||||
|
||||
|
||||
class TransportExit:
|
||||
"""Consume a prepared transport once, then publish the residual update."""
|
||||
|
||||
def __init__(self, stage, batch, *, update_context=None):
|
||||
self.stage = stage
|
||||
self.batch = batch
|
||||
self.stream = batch.residual_stream
|
||||
owner, self.invocation = self.stream.transport
|
||||
if owner is not stage:
|
||||
raise RuntimeError("compute transport belongs to another stage")
|
||||
self.path = stage.plan.path_for(batch)
|
||||
self.update = bind_update(self.path.output.update, batch, update_context)
|
||||
self.publish = (
|
||||
TransportExit._write
|
||||
if self.path.exit.writes_residual
|
||||
else TransportExit._record
|
||||
)
|
||||
self.scope = self.invocation.scope()
|
||||
self.used = False
|
||||
|
||||
def __enter__(self):
|
||||
try:
|
||||
self.scope.__enter__()
|
||||
except BaseException:
|
||||
self.stream.transport = None
|
||||
raise
|
||||
return self
|
||||
|
||||
def __exit__(self, *exc):
|
||||
try:
|
||||
result = self.scope.__exit__(*exc)
|
||||
finally:
|
||||
self.invocation.release()
|
||||
self.stream.transport = None
|
||||
if exc[0] is None and not self.used:
|
||||
raise RuntimeError("compute transport exited without finish")
|
||||
return result
|
||||
|
||||
def finish(self, output):
|
||||
if self.used:
|
||||
raise RuntimeError("compute transport was already consumed")
|
||||
self.used = True
|
||||
output = self.invocation.prepare_output(output)
|
||||
rows = self.invocation.rows
|
||||
groups = tuple(
|
||||
group for group in (rows.group, rows.decode_group) if group is not None
|
||||
)
|
||||
if isinstance(output, UnreducedOutput):
|
||||
if output.complete_sum is not None or output.reduce_to_dp_local is not None:
|
||||
raise ValueError("operand transport requires its declared sum group")
|
||||
if not any(_sum_group(group) is output.group for group in groups):
|
||||
raise ValueError("transport output owes an undeclared sum group")
|
||||
elif (
|
||||
isinstance(output, DeferredFinalize)
|
||||
and output.completion_group not in groups
|
||||
):
|
||||
raise ValueError("transport finalize owes an undeclared sum group")
|
||||
if not self.invocation.defers_sum or self.path.exit.writes_residual:
|
||||
output = complete_owed(output)
|
||||
output = self.invocation.return_output(output)
|
||||
return self.publish(self, output)
|
||||
|
||||
def _write(self, output):
|
||||
return self.stream.write(self.update.update(output, self.stream.residual))
|
||||
|
||||
def _record(self, output):
|
||||
return self.stream.record(output, self.update)
|
||||
@@ -40,6 +40,11 @@ from sglang.srt.layers.layer_boundary.residual.add_norm import (
|
||||
)
|
||||
from sglang.srt.layers.layer_boundary.residual.batch import stream_of
|
||||
from sglang.srt.layers.layer_boundary.residual.stream import DeclaredSum, ResidualStream
|
||||
from sglang.srt.layers.layer_boundary.row_transport import (
|
||||
ComputeTransport,
|
||||
TransportExit,
|
||||
validate_invocation,
|
||||
)
|
||||
from sglang.srt.model_executor.forward_batch_info import ForwardBatch
|
||||
from sglang.srt.runtime_context import get_forward
|
||||
|
||||
@@ -96,6 +101,7 @@ class StageBoundary:
|
||||
(``call``: what the stage's read takes, e.g. the attention's
|
||||
``quant_format``)."""
|
||||
entry = self.entry(forward_batch, steps)
|
||||
context = call.get("read_context")
|
||||
if (
|
||||
stream.pending is None
|
||||
and stream.residual is None
|
||||
@@ -123,6 +129,25 @@ class StageBoundary:
|
||||
hidden_states, forward_batch, self.plan.qkv_latent_func
|
||||
)
|
||||
stream.write(residual)
|
||||
transport = self.declaration.row_transport
|
||||
if transport is not None:
|
||||
if stream.transport is not None:
|
||||
raise RuntimeError("cannot replace an unconsumed compute transport")
|
||||
variant = self.plan.variant_for(forward_batch)
|
||||
ordinary = transport.rows[variant]
|
||||
overlapped = (
|
||||
transport.overlapped_rows.get(variant)
|
||||
if transport.overlapped_rows is not None
|
||||
else None
|
||||
)
|
||||
invocation = transport.bind(
|
||||
hidden_states, forward_batch, context, ordinary, overlapped
|
||||
)
|
||||
if not isinstance(invocation, ComputeTransport):
|
||||
raise TypeError("row transport must bind a ComputeTransport invocation")
|
||||
validate_invocation(invocation, ordinary, overlapped)
|
||||
stream.transport = (self, invocation)
|
||||
hidden_states = invocation.input
|
||||
return hidden_states, stream
|
||||
|
||||
@property
|
||||
@@ -175,6 +200,9 @@ class StageBoundary:
|
||||
actual contribution, or an FFN's or a mixer's output with its sum
|
||||
completed or carried on. An FFN whose boundary has fusions (which may
|
||||
take a MoE finalize handoff) finishes through exit() instead."""
|
||||
if self.declaration.row_transport is not None:
|
||||
with self.exit(forward_batch, update_context=update_context) as output:
|
||||
return output.finish(hidden_states)
|
||||
if self.kind is StageKind.ATTENTION and self.plan.finishes_directly:
|
||||
produced = self.plan.path_for(forward_batch).output
|
||||
completion = (
|
||||
@@ -201,6 +229,13 @@ class StageBoundary:
|
||||
hidden_states
|
||||
)
|
||||
|
||||
def transport_context(self, forward_batch):
|
||||
"""Borrow this prepared compute's operand transport until finish()."""
|
||||
stream = stream_of(forward_batch)
|
||||
if stream.transport is None or stream.transport[0] is not self:
|
||||
raise RuntimeError("no prepared compute transport for this stage")
|
||||
return stream.transport[1]
|
||||
|
||||
def exit(self, forward_batch, *, update_context=None):
|
||||
"""Select and scope one FFN or single-stage mixer's output decision,
|
||||
for an FFN whose MoE may hand off its finalize (see finish()).
|
||||
@@ -215,6 +250,8 @@ class StageBoundary:
|
||||
when compute is skipped for an empty input. It completes or carries work
|
||||
using the same decision rather than selecting a second path.
|
||||
"""
|
||||
if self.declaration.row_transport is not None:
|
||||
return TransportExit(self, forward_batch, update_context=update_context)
|
||||
result = (
|
||||
self.plan.output.ffn_exit(
|
||||
forward_batch,
|
||||
|
||||
@@ -2155,7 +2155,7 @@ def enable_num_token_non_padded():
|
||||
return True
|
||||
# DP attention + TP MoE gathers each rank's whole padded chunk into the MoE,
|
||||
# so the model needs the real count to zero the pad rows (see
|
||||
# deepseek_v4 _run_moe_ffn_dp_sync).
|
||||
# deepseek_common.moe_transport).
|
||||
return _dp_tp_moe_masks_pad_rows()
|
||||
|
||||
|
||||
|
||||
@@ -11,8 +11,6 @@ from sglang.srt.batch_invariant_ops import is_batch_invariant_mode_enabled
|
||||
from sglang.srt.environ import envs
|
||||
from sglang.srt.layers.moe import get_moe_a2a_backend
|
||||
from sglang.srt.layers.moe.utils import should_skip_post_experts_all_reduce
|
||||
from sglang.srt.models.deepseek_v2 import MoEOutput
|
||||
from sglang.srt.models.deepseek_v4_mhc import AttnOutput
|
||||
from sglang.srt.runtime_context import (
|
||||
get_exec,
|
||||
get_forward,
|
||||
@@ -572,83 +570,6 @@ def apply_moe_mhc(x: torch.Tensor, state: MhcPostOperands) -> torch.Tensor:
|
||||
)
|
||||
|
||||
|
||||
def forward_hc_pre_from_prev_fused_boundary(
|
||||
layer,
|
||||
positions: torch.Tensor,
|
||||
hidden_states: Optional[torch.Tensor],
|
||||
input_ids: torch.Tensor,
|
||||
forward_batch,
|
||||
input_ids_global: torch.Tensor,
|
||||
prev_pre: Optional[torch.Tensor],
|
||||
pending_post: Optional[Tuple[torch.Tensor, ...]],
|
||||
defer_post: bool,
|
||||
) -> Tuple[Optional[torch.Tensor], torch.Tensor, Optional[Tuple[torch.Tensor, ...]]]:
|
||||
"""ROCm form of DeepseekV4DecoderLayer.forward_hc_pre_from_prev.
|
||||
pending_post is the previous layer's unapplied FFN hc_post (x, residual, post,
|
||||
comb); with defer_post this layer's is returned the same way and hidden_states is None."""
|
||||
if pending_post is None:
|
||||
pending_post = (None, hidden_states, None, None)
|
||||
residual, x, attn_coefficients = hc_boundary(
|
||||
layer,
|
||||
*pending_post,
|
||||
prev_pre,
|
||||
layer.hc_attn_fn,
|
||||
layer.hc_attn_scale,
|
||||
layer.hc_attn_base,
|
||||
)
|
||||
# the boundary's reduce + sinkhorn rides in the norm launch
|
||||
x, x_quant = layer._input_norm(
|
||||
x, allow_aiter_quant=False, coefficients=attn_coefficients
|
||||
)
|
||||
with layer.self_attn.maybe_use_decode_attn_tp(forward_batch):
|
||||
mhc = attention_mhc_fusion(layer, residual, attn_coefficients, forward_batch)
|
||||
x = layer.self_attn(
|
||||
x=x,
|
||||
positions=positions,
|
||||
forward_batch=forward_batch,
|
||||
x_quant=x_quant,
|
||||
defer_all_reduce=mhc is not None,
|
||||
)
|
||||
if mhc is not None:
|
||||
# wo_b honors the deferred reduce under attention_mhc_fusion's gates;
|
||||
# fail loud rather than fall back to an unfused reduce.
|
||||
assert isinstance(x, AttnOutput)
|
||||
residual = apply_attention_mhc(x.partial, mhc)
|
||||
x = None
|
||||
residual, x, ffn_coefficients = hc_boundary(
|
||||
layer,
|
||||
x,
|
||||
residual,
|
||||
attn_coefficients.post if x is not None else None,
|
||||
attn_coefficients.comb if x is not None else None,
|
||||
attn_coefficients.pre,
|
||||
layer.hc_ffn_fn,
|
||||
layer.hc_ffn_scale,
|
||||
layer.hc_ffn_base,
|
||||
)
|
||||
x = _gfx95_dense_post_attention_norm(layer, x, ffn_coefficients)
|
||||
mhc = moe_mhc_fusion(layer, residual, ffn_coefficients, forward_batch)
|
||||
x = layer._run_moe_ffn_dp_sync(
|
||||
x,
|
||||
forward_batch,
|
||||
input_ids=input_ids,
|
||||
input_ids_global=input_ids_global,
|
||||
return_moe_output=mhc is not None,
|
||||
)
|
||||
ffn_pre, ffn_post, ffn_comb = ffn_coefficients.tensors()
|
||||
if mhc is not None:
|
||||
# The MoE TP path honors the deferred output under moe_mhc_fusion's gates;
|
||||
# fail loud rather than fall back to an unfused reduce. The fused post
|
||||
# carries the reduce, so take the merged rows without one.
|
||||
assert isinstance(x, MoEOutput)
|
||||
# Reduction already applied post. The next boundary consumes this
|
||||
# materialized residual, including when it would normally defer post.
|
||||
return apply_moe_mhc(x.get_merged(), mhc), ffn_pre, None
|
||||
if defer_post:
|
||||
return None, ffn_pre, (x, residual, ffn_post, ffn_comb)
|
||||
return layer.hc_post(x, residual, ffn_post, ffn_comb), ffn_pre, None
|
||||
|
||||
|
||||
def _gfx95_dense_post_attention_norm(layer, x: torch.Tensor, coefficients):
|
||||
"""layer.post_attention_layernorm(x) hosting the pending reduce + sinkhorn where
|
||||
the gfx950 norm kernel is available; the module norm plus a standalone reduce +
|
||||
|
||||
@@ -21,7 +21,6 @@ from sglang.srt.model_executor.runner_backend_utils.breakable_cuda_graph.context
|
||||
from sglang.srt.models.deepseek_common.amd import deepseek_v4_gfx95_dense as gfx95_dense
|
||||
from sglang.srt.models.deepseek_common.amd.deepseek_v4_fused_mhc import ( # noqa: F401 deepseek_v4 reaches apply_attention_mhc through this module
|
||||
apply_attention_mhc,
|
||||
forward_hc_pre_from_prev_fused_boundary,
|
||||
)
|
||||
from sglang.srt.runtime_context import get_exec
|
||||
from sglang.srt.utils import is_gfx95_supported
|
||||
@@ -150,7 +149,7 @@ def attention_inv_rope(
|
||||
|
||||
|
||||
def init_decoder_layer(layer, quant_config) -> None:
|
||||
"""The ROCm state of DeepseekV4DecoderLayer.__init__. input_layernorm also emits the
|
||||
"""The ROCm state of DeepseekV4DecoderLayerBase.__init__. input_layernorm also emits the
|
||||
pre-quantized operand of the dense projections (whether wqkv_a consumes native MXFP8
|
||||
resolves on first use), and on gfx950 one launch runs hc_post, the pre-collapse and
|
||||
the mixing stats; the kernel only supports hc_mult 4."""
|
||||
@@ -203,48 +202,3 @@ def input_norm(
|
||||
if coefficients is not None:
|
||||
coefficients.materialize()
|
||||
return layer.input_layernorm(hidden_states), None
|
||||
|
||||
|
||||
# ---- DeepseekV4Model layer loop ----
|
||||
|
||||
|
||||
def forward_layer_fused_boundary(
|
||||
model,
|
||||
i: int,
|
||||
state,
|
||||
*,
|
||||
positions: torch.Tensor,
|
||||
input_ids: torch.Tensor,
|
||||
forward_batch,
|
||||
input_ids_global: torch.Tensor,
|
||||
capture_dspark: bool,
|
||||
):
|
||||
"""Layer i through the fused mHC boundary. Its FFN hc_post stays pending for the next
|
||||
layer's boundary launch unless that layer reads the residual stream first (Engram gate,
|
||||
DSpark capture) or there is none; hidden_states is None while a post is pending."""
|
||||
nxt = i + 1
|
||||
defer_post = (
|
||||
nxt < model.end_layer
|
||||
and model.layers[nxt].engram is None
|
||||
and not (capture_dspark and nxt in model.dspark_layers_to_capture)
|
||||
)
|
||||
# The boundary speaks its own (hidden_states, prev_pre, pending_post) triple;
|
||||
# the loop speaks HcState. This is the only place the two meet.
|
||||
from sglang.srt.models.deepseek_v4_mhc import HcPending, HcState
|
||||
|
||||
pending = isinstance(state.streams, HcPending)
|
||||
hidden_states, prev_pre, pending_post = forward_hc_pre_from_prev_fused_boundary(
|
||||
model.layers[i],
|
||||
positions=positions,
|
||||
hidden_states=None if pending else state.streams,
|
||||
input_ids=input_ids,
|
||||
forward_batch=forward_batch,
|
||||
input_ids_global=input_ids_global,
|
||||
prev_pre=state.pre,
|
||||
pending_post=tuple(state.streams) if pending else None,
|
||||
defer_post=defer_post,
|
||||
)
|
||||
return HcState(
|
||||
hidden_states if pending_post is None else HcPending(*pending_post),
|
||||
prev_pre,
|
||||
)
|
||||
|
||||
@@ -0,0 +1,433 @@
|
||||
"""MoE operand transport and completion, preserving source residual rows."""
|
||||
|
||||
from contextlib import contextmanager, nullcontext
|
||||
|
||||
import torch
|
||||
|
||||
from sglang.srt.environ import envs
|
||||
from sglang.srt.layers.cp.interleave import attn_cp_interleave_gather
|
||||
from sglang.srt.layers.cp.utils import is_cp_active
|
||||
from sglang.srt.layers.dp_attention import (
|
||||
_tbo_event,
|
||||
attn_tp_all_gather,
|
||||
attn_tp_all_gather_into_tensor,
|
||||
dp_gather_partial,
|
||||
dp_gather_replicate,
|
||||
dp_reduce_scatter_tensor,
|
||||
dp_reduce_scatterv_async,
|
||||
dp_scatter,
|
||||
get_dp_global_num_tokens,
|
||||
get_dp_tbo_comm_stream,
|
||||
get_global_dp_buffer,
|
||||
get_global_dp_buffer_len,
|
||||
get_local_dp_buffer,
|
||||
get_local_dp_buffer_len,
|
||||
get_tbo_persistent_buffer,
|
||||
is_dp_gatherv_active,
|
||||
)
|
||||
from sglang.srt.layers.layer_boundary.ops import attn_cp_interleave_reduce_scatter
|
||||
from sglang.srt.layers.layer_boundary.output import DeferredFinalize
|
||||
from sglang.srt.layers.layer_boundary.row_transport import ComputeTransport
|
||||
from sglang.srt.layers.moe import get_moe_a2a_backend, should_use_dp_reduce_scatterv
|
||||
from sglang.srt.runtime_context import get_forward, get_parallel
|
||||
from sglang.srt.utils import get_bool_env_var, is_hip
|
||||
|
||||
_is_hip = is_hip()
|
||||
SHARED_EXPERT_LOCAL = get_bool_env_var("SGLANG_DP_SHARED_EXPERT_LOCAL")
|
||||
|
||||
|
||||
def shared_expert_runs_locally(moe):
|
||||
"""Whether the replicated shared expert can run before the DP gather."""
|
||||
return (
|
||||
SHARED_EXPERT_LOCAL
|
||||
and getattr(moe, "shared_experts", None) is not None
|
||||
and getattr(moe, "_shared_expert_tp1", False)
|
||||
)
|
||||
|
||||
|
||||
@contextmanager
|
||||
def _every_row_routed(forward_batch, num_rows: int):
|
||||
# Under CP the real rows are not a prefix, so every row must be routed.
|
||||
saved = (
|
||||
forward_batch.num_token_non_padded,
|
||||
forward_batch.global_num_token_non_padded_cpu,
|
||||
)
|
||||
if saved[0] is not None:
|
||||
forward_batch.num_token_non_padded = torch.full_like(saved[0], num_rows)
|
||||
forward_batch.global_num_token_non_padded_cpu = num_rows
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
(
|
||||
forward_batch.num_token_non_padded,
|
||||
forward_batch.global_num_token_non_padded_cpu,
|
||||
) = saved
|
||||
|
||||
|
||||
def _mask_dp_pad_rows(local_hidden_states, forward_batch) -> None:
|
||||
n_real = forward_batch.num_token_non_padded
|
||||
num_rows = local_hidden_states.shape[0]
|
||||
if n_real is None or num_rows == 0:
|
||||
return
|
||||
pad = torch.arange(num_rows, device=local_hidden_states.device) >= n_real
|
||||
local_hidden_states.masked_fill_(
|
||||
pad.view(-1, *([1] * (local_hidden_states.dim() - 1))), 0
|
||||
)
|
||||
|
||||
|
||||
def _attn_tp_all_gather_chunks(
|
||||
local: torch.Tensor, chunk_rows: list[int]
|
||||
) -> torch.Tensor:
|
||||
# Tensor all_gather stays on aiter AG / pynccl under graph capture; the list
|
||||
# all_gather falls back to ProcessGroupNCCL, which invalidates HIP capture.
|
||||
max_rows = max(chunk_rows)
|
||||
padded = local
|
||||
if local.shape[0] != max_rows:
|
||||
padded = local.new_zeros((max_rows, *local.shape[1:]))
|
||||
padded[: local.shape[0]].copy_(local)
|
||||
gathered = local.new_empty((len(chunk_rows) * max_rows, *local.shape[1:]))
|
||||
attn_tp_all_gather_into_tensor(gathered, padded.contiguous())
|
||||
if all(rows == max_rows for rows in chunk_rows):
|
||||
return gathered
|
||||
return torch.cat(
|
||||
[
|
||||
gathered[i * max_rows : i * max_rows + rows]
|
||||
for i, rows in enumerate(chunk_rows)
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
class MoeRowTransport(ComputeTransport):
|
||||
def __init__(
|
||||
self,
|
||||
moe,
|
||||
hidden_states,
|
||||
forward_batch,
|
||||
input_ids,
|
||||
input_ids_global,
|
||||
*,
|
||||
cp_return_contiguous=False,
|
||||
use_cp=None,
|
||||
moe_gather=None,
|
||||
attn_scatter=None,
|
||||
):
|
||||
_use_cp = is_cp_active(forward_batch) if use_cp is None else use_cp
|
||||
_use_tp_moe_gather = (
|
||||
moe_gather
|
||||
if moe_gather is not None
|
||||
else (
|
||||
not _use_cp
|
||||
and get_parallel().attn_dp_size > 1
|
||||
and get_moe_a2a_backend().is_none()
|
||||
)
|
||||
)
|
||||
_use_tp_attn_a2a_scatter = (
|
||||
attn_scatter
|
||||
if attn_scatter is not None
|
||||
else (
|
||||
not _use_cp
|
||||
and get_parallel().attn_tp_size > 1
|
||||
and not get_moe_a2a_backend().is_none()
|
||||
)
|
||||
)
|
||||
# Variable-length combine sums TP-sharded expert output while returning
|
||||
# each DP rank's rows. The producer leaves that sum to the transport.
|
||||
_use_reduce_scatterv = (
|
||||
_use_tp_moe_gather
|
||||
and is_dp_gatherv_active()
|
||||
and forward_batch.dp_padding_mode is not None
|
||||
and not forward_batch.dp_padding_mode.is_max_len()
|
||||
)
|
||||
# Equal-chunk combine requires MAX_LEN padding and one attention rank
|
||||
# per DP shard; the variable-length path takes precedence.
|
||||
_use_reduce_scatter = (
|
||||
envs.SGLANG_DP_USE_REDUCE_SCATTER.get()
|
||||
and _use_tp_moe_gather
|
||||
and not _use_reduce_scatterv
|
||||
and not should_use_dp_reduce_scatterv()
|
||||
and forward_batch.dp_padding_mode is not None
|
||||
and forward_batch.dp_padding_mode.is_max_len()
|
||||
and get_parallel().tp_size == get_parallel().attn_dp_size
|
||||
)
|
||||
mlp_reduce_scatter = (
|
||||
_use_cp
|
||||
or _use_reduce_scatterv
|
||||
or _use_reduce_scatter
|
||||
or (_use_tp_moe_gather and should_use_dp_reduce_scatterv())
|
||||
)
|
||||
# A replicated shared expert can run on local rows before the gather;
|
||||
# its output is added after returning the routed experts' local slice.
|
||||
_shared_local = None
|
||||
do_shared_local = _use_tp_moe_gather and shared_expert_runs_locally(moe)
|
||||
if _use_cp:
|
||||
moe_a2a_backend = get_moe_a2a_backend()
|
||||
if moe_a2a_backend.is_none():
|
||||
hidden_states = attn_cp_interleave_gather(hidden_states)
|
||||
else:
|
||||
assert (
|
||||
moe_a2a_backend.is_deepep()
|
||||
or moe_a2a_backend.is_megamoe()
|
||||
or moe_a2a_backend.is_mori()
|
||||
), (
|
||||
"CP requires moe_a2a_backend in ('deepep', 'megamoe', 'mori'), "
|
||||
f"got {moe_a2a_backend.value!r}."
|
||||
)
|
||||
elif _use_tp_moe_gather:
|
||||
hidden_states, local_hidden_states = (
|
||||
get_global_dp_buffer(get_parallel().tp_group),
|
||||
hidden_states,
|
||||
)
|
||||
if do_shared_local and local_hidden_states.shape[0] > 0:
|
||||
_shared_local = moe._forward_shared_experts(local_hidden_states)
|
||||
# Pad rows are not refreshed during graph replay. Stale bf16-max
|
||||
# values can overflow the MoE and contaminate the combine output.
|
||||
if _is_hip:
|
||||
_mask_dp_pad_rows(local_hidden_states, forward_batch)
|
||||
# self_attn has already reduced across attention TP, so these hidden
|
||||
# states are replicated and must not be summed by a partial gather.
|
||||
dp_gather_replicate(hidden_states, local_hidden_states, forward_batch)
|
||||
_a2a_scatter_chunks = None
|
||||
if _use_tp_attn_a2a_scatter:
|
||||
s, r = get_parallel().attn_tp_size, get_parallel().attn_tp_rank
|
||||
_a2a_scatter_chunks = list(hidden_states.tensor_split(s))
|
||||
hidden_states = _a2a_scatter_chunks[r].contiguous()
|
||||
# DSpark next-token layers are not hash-routed and intentionally do not
|
||||
# carry token IDs. Only split IDs for callers that actually provide them.
|
||||
if input_ids is not None:
|
||||
input_ids = input_ids.tensor_split(s)[r].contiguous()
|
||||
if input_ids_global is not None:
|
||||
input_ids_global = input_ids_global.tensor_split(s)[r].contiguous()
|
||||
gathered_rows = (
|
||||
_every_row_routed(forward_batch, hidden_states.shape[0])
|
||||
if _use_cp and get_moe_a2a_backend().is_none()
|
||||
else nullcontext()
|
||||
)
|
||||
self.input = hidden_states
|
||||
self.input_ids, self.input_ids_global = input_ids, input_ids_global
|
||||
self.batch = forward_batch
|
||||
self.cp_return_contiguous = cp_return_contiguous
|
||||
self._use_cp = _use_cp
|
||||
self._use_tp_moe_gather = _use_tp_moe_gather
|
||||
self._use_tp_attn_a2a_scatter = _use_tp_attn_a2a_scatter
|
||||
self._use_reduce_scatterv = _use_reduce_scatterv
|
||||
self._use_reduce_scatter = _use_reduce_scatter
|
||||
self.mlp_reduce_scatter = mlp_reduce_scatter
|
||||
self._shared_local = _shared_local
|
||||
self.do_shared_local = do_shared_local
|
||||
self._a2a_scatter_chunks = _a2a_scatter_chunks
|
||||
self.gathered_rows = gathered_rows
|
||||
|
||||
def return_output(self, hidden_states):
|
||||
forward_batch = self.batch
|
||||
_use_cp = self._use_cp
|
||||
_use_tp_moe_gather = self._use_tp_moe_gather
|
||||
_use_tp_attn_a2a_scatter = self._use_tp_attn_a2a_scatter
|
||||
_use_reduce_scatterv = self._use_reduce_scatterv
|
||||
_use_reduce_scatter = self._use_reduce_scatter
|
||||
_shared_local = self._shared_local
|
||||
_a2a_scatter_chunks = self._a2a_scatter_chunks
|
||||
if _use_cp and get_moe_a2a_backend().is_none():
|
||||
if self.cp_return_contiguous:
|
||||
parallel = get_parallel()
|
||||
hidden_states = parallel.tp_group.all_reduce(hidden_states)
|
||||
hidden_states = hidden_states.tensor_split(parallel.attn_cp_size)[
|
||||
parallel.attn_cp_rank
|
||||
].contiguous()
|
||||
else:
|
||||
hidden_states = attn_cp_interleave_reduce_scatter(hidden_states)
|
||||
elif _use_tp_moe_gather:
|
||||
hidden_states, global_hidden_states = (
|
||||
get_local_dp_buffer(get_parallel().tp_group),
|
||||
hidden_states,
|
||||
)
|
||||
if should_use_dp_reduce_scatterv() or _use_reduce_scatterv:
|
||||
get_parallel().tp_group.reduce_scatterv(
|
||||
global_hidden_states,
|
||||
output=hidden_states,
|
||||
sizes=get_dp_global_num_tokens(),
|
||||
)
|
||||
elif _use_reduce_scatter:
|
||||
dp_reduce_scatter_tensor(hidden_states, global_hidden_states)
|
||||
else:
|
||||
dp_scatter(hidden_states, global_hidden_states, forward_batch)
|
||||
if _shared_local is not None:
|
||||
n = hidden_states.shape[0]
|
||||
hidden_states = hidden_states + _shared_local[:n]
|
||||
if _use_tp_attn_a2a_scatter:
|
||||
assert _a2a_scatter_chunks is not None
|
||||
if _is_hip:
|
||||
hidden_states = _attn_tp_all_gather_chunks(
|
||||
hidden_states, chunk_rows=[t.shape[0] for t in _a2a_scatter_chunks]
|
||||
)
|
||||
else:
|
||||
gathered = [torch.empty_like(t) for t in _a2a_scatter_chunks]
|
||||
attn_tp_all_gather(gathered, hidden_states.contiguous())
|
||||
hidden_states = torch.cat(gathered)
|
||||
return hidden_states
|
||||
|
||||
@contextmanager
|
||||
def scope(self):
|
||||
with (
|
||||
get_forward().scoped(
|
||||
mlp_reduce_scatter=self.mlp_reduce_scatter, defer_moe_finalize=False
|
||||
),
|
||||
self.gathered_rows,
|
||||
):
|
||||
yield
|
||||
|
||||
def release(self):
|
||||
self.input = self._shared_local = self._a2a_scatter_chunks = None
|
||||
|
||||
|
||||
def _tbo_gather_moe_input(moe, local, batch, sub, do_shared_local):
|
||||
shared_local = (
|
||||
moe._forward_shared_experts(local)
|
||||
if do_shared_local and local.shape[0] > 0
|
||||
else None
|
||||
)
|
||||
global_hidden = get_tbo_persistent_buffer(
|
||||
("gh", sub),
|
||||
get_global_dp_buffer_len(),
|
||||
local.shape[1],
|
||||
local.dtype,
|
||||
local.device,
|
||||
)
|
||||
comm = get_dp_tbo_comm_stream()
|
||||
compute = torch.cuda.current_stream()
|
||||
with torch.cuda.stream(comm):
|
||||
comm.wait_stream(compute)
|
||||
dp_gather_partial(global_hidden, local, batch)
|
||||
event = _tbo_event(("gather", sub))
|
||||
event.record(comm)
|
||||
return global_hidden, event, shared_local
|
||||
|
||||
|
||||
def _tbo_combine_moe_output(output, sub):
|
||||
local = get_tbo_persistent_buffer(
|
||||
("lo", sub),
|
||||
get_local_dp_buffer_len(),
|
||||
output.shape[1],
|
||||
output.dtype,
|
||||
output.device,
|
||||
)
|
||||
event = dp_reduce_scatterv_async(
|
||||
local, output, get_dp_global_num_tokens(), event_key=("combine", sub)
|
||||
)
|
||||
return local, event
|
||||
|
||||
|
||||
def _tbo_wait_moe_output(local, event, shared_local):
|
||||
torch.cuda.current_stream().wait_event(event)
|
||||
return local + shared_local[: local.shape[0]] if shared_local is not None else local
|
||||
|
||||
|
||||
class MoeFinalize(DeferredFinalize, frozen=True):
|
||||
pieces: object = None
|
||||
partial: object = None
|
||||
|
||||
def complete(self, *, reduce=True):
|
||||
from sglang.srt.layers.moe import post_experts_all_reduce
|
||||
from sglang.srt.layers.moe.utils import should_add_replicated_moe_output
|
||||
|
||||
with get_forward().scoped(mlp_reduce_scatter=not reduce):
|
||||
value = (
|
||||
self.pieces.get_merged() if self.pieces is not None else self.partial
|
||||
)
|
||||
if reduce:
|
||||
value = post_experts_all_reduce(value)
|
||||
if (
|
||||
self.pieces is not None
|
||||
and self.pieces.shared is not None
|
||||
and self.pieces.shared_is_replicated
|
||||
and should_add_replicated_moe_output()
|
||||
):
|
||||
value += self.pieces.shared
|
||||
return value
|
||||
|
||||
|
||||
class OverlappedMoeTransport(ComputeTransport):
|
||||
"""The TBO issue/wait protocol, owned by one microbatch.
|
||||
|
||||
EP ops already return source rows. The TP path gathers normalized 2-D
|
||||
input and sums/scatters expert output asynchronously; residual coefficients
|
||||
stay on the source rows throughout. Neither finish path sums again.
|
||||
"""
|
||||
|
||||
is_overlapped = True
|
||||
|
||||
def __init__(self, moe, hidden_states, batch, subbatch_index, rows):
|
||||
self.rows = rows
|
||||
self.moe, self.input, self.batch = moe, hidden_states, batch
|
||||
self.ep = not get_moe_a2a_backend().is_none()
|
||||
self.sub = subbatch_index
|
||||
self.do_shared_local = not self.ep and shared_expert_runs_locally(moe)
|
||||
self.shared_local = None
|
||||
self.gather_event = self.combine_event = None
|
||||
self.gather_keepalive = self.combine_keepalive = None
|
||||
self.global_hidden = self.local_out = None
|
||||
self.combined = False
|
||||
|
||||
def release(self):
|
||||
events = (self.gather_event, self.combine_event)
|
||||
if any(event is not None for event in events):
|
||||
current = torch.cuda.current_stream()
|
||||
for event in events:
|
||||
if event is not None:
|
||||
current.wait_event(event)
|
||||
self.gather_event = self.combine_event = None
|
||||
self.gather_keepalive = self.combine_keepalive = None
|
||||
self.global_hidden = self.local_out = self.shared_local = self.input = None
|
||||
|
||||
def issue_gather(self):
|
||||
if self.ep or self.gather_event is not None or self.global_hidden is not None:
|
||||
raise RuntimeError("TBO gather cannot be issued twice")
|
||||
local = self.input
|
||||
self.global_hidden, self.gather_event, self.shared_local = (
|
||||
_tbo_gather_moe_input(
|
||||
self.moe, local, self.batch, self.sub, self.do_shared_local
|
||||
)
|
||||
)
|
||||
self.gather_keepalive = local
|
||||
self.input = None
|
||||
|
||||
def wait_gather(self):
|
||||
if self.gather_event is None:
|
||||
raise RuntimeError("TBO gather was not issued")
|
||||
torch.cuda.current_stream().wait_event(self.gather_event)
|
||||
self.gather_event = self.gather_keepalive = None
|
||||
return self.global_hidden
|
||||
|
||||
def issue_combine(self, output):
|
||||
if self.ep or self.combine_event is not None or self.combined:
|
||||
raise RuntimeError("TBO combine cannot be issued twice")
|
||||
self.local_out, self.combine_event = _tbo_combine_moe_output(output, self.sub)
|
||||
self.combine_keepalive = output
|
||||
self.global_hidden = None
|
||||
|
||||
def wait_combine(self):
|
||||
if self.combine_event is None:
|
||||
raise RuntimeError("TBO combine was not issued")
|
||||
value = _tbo_wait_moe_output(
|
||||
self.local_out, self.combine_event, self.shared_local
|
||||
)
|
||||
self.combine_event = self.combine_keepalive = None
|
||||
self.local_out = self.shared_local = None
|
||||
self.combined = True
|
||||
return value
|
||||
|
||||
def prepare_output(self, output):
|
||||
if not self.ep and not self.combined:
|
||||
raise RuntimeError("TBO finish requires the combine wait")
|
||||
return output
|
||||
|
||||
|
||||
def _tbo_run_moe(layer, hidden, batch, do_shared_local):
|
||||
global_ids = batch._tbo_global_input_ids
|
||||
with get_forward().scoped(mlp_reduce_scatter=True):
|
||||
return layer.mlp(
|
||||
hidden,
|
||||
batch,
|
||||
input_ids=global_ids,
|
||||
input_ids_global=global_ids,
|
||||
skip_shared_experts=do_shared_local,
|
||||
)
|
||||
@@ -618,6 +618,13 @@ class MoEOutput(NamedTuple):
|
||||
)
|
||||
|
||||
|
||||
class MergedMoEOutput(MoEOutput):
|
||||
"""Merged routed rows with a replicated shared output still separate."""
|
||||
|
||||
def get_merged(self) -> torch.Tensor:
|
||||
return self.routed
|
||||
|
||||
|
||||
class DeepseekV2MoE(nn.Module):
|
||||
def __init__(
|
||||
self,
|
||||
@@ -982,10 +989,13 @@ class DeepseekV2MoE(nn.Module):
|
||||
input_ids_global: Optional[torch.Tensor] = None,
|
||||
skip_shared_experts: bool = False,
|
||||
return_moe_output: bool = False,
|
||||
leave_output_sum: bool = False,
|
||||
) -> Union[torch.Tensor, MoEOutput]:
|
||||
"""``return_moe_output`` asks for the pieces unmerged, as `MoEOutput`; only
|
||||
the TP paths honor it. Distinct from `get_forward().defer_moe_finalize`,
|
||||
the V2/V3 layer-boundary handoff."""
|
||||
the V2/V3 layer-boundary handoff. ``leave_output_sum`` returns those
|
||||
pieces with the selected finalize kernel, for
|
||||
a boundary that owns reduction and the replicated shared add."""
|
||||
from sglang.srt.layers.moe.mega_moe import forward_mega_moe, should_use_mega_moe
|
||||
|
||||
if should_use_mega_moe(self, hidden_states):
|
||||
@@ -1007,6 +1017,20 @@ class DeepseekV2MoE(nn.Module):
|
||||
if not self._enable_a2a_moe:
|
||||
if self._can_dual_stream_graph(hidden_states):
|
||||
fwd = get_forward()
|
||||
if leave_output_sum:
|
||||
routed, shared = dsv4_flashinfer_moe_dual_stream_outputs(
|
||||
hidden_states,
|
||||
self.layer_id,
|
||||
fwd.mlp_reduce_scatter,
|
||||
self._shared_expert_tp1,
|
||||
)
|
||||
return MergedMoEOutput(
|
||||
routed,
|
||||
shared if self._shared_expert_tp1 else None,
|
||||
self.experts,
|
||||
1.0,
|
||||
self._shared_expert_tp1,
|
||||
)
|
||||
return dsv2_flashinfer_moe_dual_stream_graph(
|
||||
hidden_states,
|
||||
self.layer_id,
|
||||
@@ -1027,6 +1051,7 @@ class DeepseekV2MoE(nn.Module):
|
||||
num_token_non_padded=num_token_non_padded,
|
||||
use_vision_topk=use_vision_topk,
|
||||
return_moe_output=return_moe_output,
|
||||
leave_output_sum=leave_output_sum,
|
||||
)
|
||||
else:
|
||||
return self.forward_normal(
|
||||
@@ -1038,6 +1063,7 @@ class DeepseekV2MoE(nn.Module):
|
||||
num_token_non_padded=num_token_non_padded,
|
||||
use_vision_topk=use_vision_topk,
|
||||
return_moe_output=return_moe_output,
|
||||
leave_output_sum=leave_output_sum,
|
||||
)
|
||||
else:
|
||||
return self.forward_deepep(
|
||||
@@ -1053,6 +1079,7 @@ class DeepseekV2MoE(nn.Module):
|
||||
num_token_non_padded: Optional[torch.Tensor] = None,
|
||||
use_vision_topk: bool = False,
|
||||
return_moe_output: bool = False,
|
||||
leave_output_sum: bool = False,
|
||||
) -> Union[torch.Tensor, MoEOutput]:
|
||||
# Note(kpham-sgl): issue order satisfies 3 constraints:
|
||||
# - no stream explosion: main (routed) issued before alt block -> capture reuses 1 alt stream;
|
||||
@@ -1195,7 +1222,7 @@ class DeepseekV2MoE(nn.Module):
|
||||
reduce=sum_post_experts_output,
|
||||
)
|
||||
|
||||
if return_moe_output:
|
||||
if return_moe_output or leave_output_sum:
|
||||
return MoEOutput(
|
||||
routed=final_hidden_states,
|
||||
shared=shared_output,
|
||||
@@ -1238,6 +1265,7 @@ class DeepseekV2MoE(nn.Module):
|
||||
num_token_non_padded: Optional[torch.Tensor] = None,
|
||||
use_vision_topk: bool = False,
|
||||
return_moe_output: bool = False,
|
||||
leave_output_sum: bool = False,
|
||||
) -> Union[torch.Tensor, MoEOutput]:
|
||||
if hasattr(self, "shared_experts") and use_intel_amx_backend(
|
||||
self.shared_experts.gate_up_proj
|
||||
@@ -1373,7 +1401,7 @@ class DeepseekV2MoE(nn.Module):
|
||||
pre_quant_input=pre_quant_input,
|
||||
)
|
||||
|
||||
if return_moe_output:
|
||||
if return_moe_output or leave_output_sum:
|
||||
return MoEOutput(
|
||||
routed=final_hidden_states,
|
||||
shared=shared_output,
|
||||
@@ -3494,4 +3522,48 @@ def dsv2_flashinfer_moe_dual_stream_graph(
|
||||
return moe_fusion.forward_normal_dual_stream(hidden_states)
|
||||
|
||||
|
||||
def _fake_dsv4_dual_stream_outputs(
|
||||
hidden_states, layer_id, mlp_reduce_scatter, shared_replicated
|
||||
):
|
||||
return torch.empty_like(hidden_states), (
|
||||
torch.empty_like(hidden_states)
|
||||
if shared_replicated
|
||||
else hidden_states.new_empty((0,))
|
||||
)
|
||||
|
||||
|
||||
@register_custom_op(fake_impl=_fake_dsv4_dual_stream_outputs)
|
||||
def dsv4_flashinfer_moe_dual_stream_outputs(
|
||||
hidden_states: torch.Tensor,
|
||||
layer_id: int,
|
||||
mlp_reduce_scatter: bool,
|
||||
shared_replicated: bool,
|
||||
) -> Tuple[torch.Tensor, torch.Tensor]:
|
||||
return _run_dsv4_dual_stream_outputs(
|
||||
hidden_states, layer_id, mlp_reduce_scatter, shared_replicated
|
||||
)
|
||||
|
||||
|
||||
def _run_dsv4_dual_stream_outputs(
|
||||
hidden_states: torch.Tensor,
|
||||
layer_id: int,
|
||||
mlp_reduce_scatter: bool,
|
||||
shared_replicated: bool,
|
||||
) -> Tuple[torch.Tensor, torch.Tensor]:
|
||||
forward_context = get_tc_piecewise_forward_context()
|
||||
assert forward_context is not None and forward_context.moe_fusions is not None
|
||||
moe = forward_context.moe_fusions[layer_id]
|
||||
assert moe is not None
|
||||
with get_forward().scoped(
|
||||
mlp_reduce_scatter=mlp_reduce_scatter,
|
||||
flashinfer_trtllm_bypass=True,
|
||||
lora_batch_layout=LoRABatchLayout.TP_GLOBAL,
|
||||
defer_moe_finalize=False,
|
||||
):
|
||||
pieces = moe.forward_normal_dual_stream(hidden_states, leave_output_sum=True)
|
||||
assert pieces.shared_is_replicated == shared_replicated
|
||||
merged = pieces.get_merged()
|
||||
return merged, pieces.shared if shared_replicated else merged.new_empty((0,))
|
||||
|
||||
|
||||
EntryClass = [DeepseekV2ForCausalLM, DeepseekV3ForCausalLM, DeepseekV32ForCausalLM]
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,540 @@
|
||||
"""DeepSeek hyper-connection reads, updates and operand row transport."""
|
||||
|
||||
from functools import partial
|
||||
|
||||
import msgspec
|
||||
import torch
|
||||
|
||||
from sglang.srt.layers.cp.cp_decode_attn_tp import get_cp_decode_attn_tp_ctx
|
||||
from sglang.srt.layers.layer_boundary.contracts import BatchVariant, ReadoutFusion
|
||||
from sglang.srt.layers.layer_boundary.factories import declare_attn, declare_ffn
|
||||
from sglang.srt.layers.layer_boundary.facts import ReadFacts, UpdateFacts
|
||||
from sglang.srt.layers.layer_boundary.layout import (
|
||||
Layout,
|
||||
SumGroup,
|
||||
TokenAxis,
|
||||
_sum_group,
|
||||
)
|
||||
from sglang.srt.layers.layer_boundary.output import UnreducedOutput
|
||||
from sglang.srt.layers.layer_boundary.residual import PendingUpdateRead
|
||||
from sglang.srt.layers.layer_boundary.residual import batch as residual_batch
|
||||
from sglang.srt.layers.layer_boundary.row_transport import (
|
||||
ComputeTransport,
|
||||
RowTransport,
|
||||
TransportRows,
|
||||
)
|
||||
from sglang.srt.layers.moe import get_moe_a2a_backend
|
||||
from sglang.srt.layers.quantization.mxfp4_flashinfer_trtllm_moe import (
|
||||
can_fuse_all_reduce,
|
||||
)
|
||||
from sglang.srt.models import deepseek_v4_mhc as mhc
|
||||
from sglang.srt.models.deepseek_common.moe_transport import (
|
||||
MoeFinalize,
|
||||
MoeRowTransport,
|
||||
)
|
||||
from sglang.srt.models.deepseek_v2 import MoEOutput
|
||||
from sglang.srt.runtime_context import get_parallel
|
||||
from sglang.srt.utils import is_gfx95_supported
|
||||
|
||||
_is_gfx95_supported = is_gfx95_supported()
|
||||
|
||||
|
||||
class LayerCall(msgspec.Struct, frozen=True):
|
||||
input_ids: object = None
|
||||
input_ids_global: object = None
|
||||
allow_cross_layer_fusion: bool = True
|
||||
tbo: bool = False
|
||||
tbo_subbatch_index: object = None
|
||||
|
||||
|
||||
class ReadState(msgspec.Struct):
|
||||
"""The source residual and coefficients of one sublayer invocation."""
|
||||
|
||||
streams: torch.Tensor
|
||||
post: object = None
|
||||
comb: object = None
|
||||
quantized: object = None
|
||||
stats: object = None
|
||||
stats_stream: object = None
|
||||
call: object = None
|
||||
|
||||
|
||||
class HcUpdate:
|
||||
bind = None
|
||||
replay_update = None
|
||||
is_plain_add = False
|
||||
outlives_layer = True
|
||||
quantized_sum = False
|
||||
|
||||
def __init__(self, layer, attention, *, applied_at_exit=False):
|
||||
self.layer = layer
|
||||
self.attention = attention
|
||||
self.applied_at_exit = applied_at_exit
|
||||
self.writes_stream = False
|
||||
|
||||
def slice_residual_attn_tp(self, residual):
|
||||
raise NotImplementedError("mHC coefficients remain on their source rows")
|
||||
|
||||
def gather_residual_attn_tp(self, residual):
|
||||
raise NotImplementedError("mHC coefficients remain on their source rows")
|
||||
|
||||
def post(self, output, state, *, next=None):
|
||||
if not self.layer.boundary_pre_from_prev:
|
||||
return self.layer.hc_post(output, state.streams, state.post, state.comb)
|
||||
hc = self.layer.attn_hc if self.attention else self.layer.ffn_hc
|
||||
if self.attention:
|
||||
return mhc.run_attn_post(
|
||||
hc,
|
||||
output,
|
||||
state.streams,
|
||||
stats_stream=state.stats_stream,
|
||||
next=next,
|
||||
world_size=self.layer.self_attn.attn_tp_size,
|
||||
precomputed=state.stats,
|
||||
)
|
||||
return mhc.run_moe_post(
|
||||
hc,
|
||||
output,
|
||||
state.streams,
|
||||
stats_stream=state.stats_stream,
|
||||
next=next if state.call.allow_cross_layer_fusion else None,
|
||||
world_size=self.layer.mlp.tp_size,
|
||||
precomputed=state.stats,
|
||||
)
|
||||
|
||||
def update(self, output, state):
|
||||
return self.post(output, state)
|
||||
|
||||
|
||||
class HcRead:
|
||||
is_plain_norm = False
|
||||
reads_before_dp_gather = True
|
||||
reads_after_attn_tp_gather = True
|
||||
gathering_reads = ()
|
||||
|
||||
def __init__(self, layer, attention):
|
||||
self.layer = layer
|
||||
self.attention = attention
|
||||
self.completing_fusions = tuple(
|
||||
ReadoutFusion(
|
||||
group,
|
||||
self._complete_read,
|
||||
reads=True,
|
||||
carried=True,
|
||||
accepts_update=lambda update: isinstance(update, HcUpdate),
|
||||
)
|
||||
for group in (SumGroup.ATTN_TP, SumGroup.MOE_OUTPUT)
|
||||
)
|
||||
|
||||
def bind(self, batch, call):
|
||||
return HcReadCall(self, batch, call)
|
||||
|
||||
@staticmethod
|
||||
def init_residual(hidden_states):
|
||||
return hidden_states
|
||||
|
||||
def _complete_read(self, output, state, batch, norm, *, update, read_context=None):
|
||||
if not self.layer.boundary_pre_from_prev:
|
||||
return None
|
||||
if isinstance(output, MoeFinalize):
|
||||
if output.pieces is None:
|
||||
return None
|
||||
output = output.pieces
|
||||
elif isinstance(output, UnreducedOutput):
|
||||
if not update.attention:
|
||||
return None
|
||||
output = mhc.AttnOutput(output.partial)
|
||||
else:
|
||||
return None
|
||||
call = read_context if read_context is not None else state.call
|
||||
read = self.bind(batch, call)
|
||||
return read._after_post(
|
||||
update.post(output, state, next=read.next_boundary), norm
|
||||
)
|
||||
|
||||
|
||||
class HcReadCall(HcRead):
|
||||
def __init__(self, spec, batch, call):
|
||||
self.spec = spec
|
||||
self.layer = spec.layer
|
||||
self.attention = spec.attention
|
||||
self.batch = batch
|
||||
self.call = call
|
||||
self.completing_fusions = tuple(
|
||||
msgspec.structs.replace(
|
||||
f, run=partial(spec._complete_read, read_context=call)
|
||||
)
|
||||
for f in spec.completing_fusions
|
||||
)
|
||||
|
||||
def bind(self, batch, call=None):
|
||||
if batch is not self.batch or (call is not None and call is not self.call):
|
||||
raise ValueError("an invocation read cannot bind another call")
|
||||
return self
|
||||
|
||||
@property
|
||||
def next_boundary(self):
|
||||
if not self.layer.boundary_mhc_fusions or (
|
||||
self.attention and self.layer.engram is not None
|
||||
):
|
||||
return None
|
||||
hc = self.layer.attn_hc if self.attention else self.layer.ffn_hc
|
||||
return mhc.make_boundary(
|
||||
self.layer.input_layernorm
|
||||
if self.attention
|
||||
else self.layer.post_attention_layernorm,
|
||||
accepts_mxfp8=self.attention
|
||||
and self.layer.self_attn.accepts_mxfp8_swizzled_input(),
|
||||
hc=hc,
|
||||
)
|
||||
|
||||
def _after_post(self, residual, norm):
|
||||
return self.read(residual, norm)
|
||||
|
||||
def read(
|
||||
self,
|
||||
residual,
|
||||
norm,
|
||||
quant_format="",
|
||||
post_residual_addition=None,
|
||||
):
|
||||
if quant_format or post_residual_addition is not None:
|
||||
raise NotImplementedError("mHC read supplies its own projection format")
|
||||
layer = self.layer
|
||||
if layer.boundary_pre_from_prev:
|
||||
if not isinstance(residual, mhc.HcState):
|
||||
residual = mhc.HcState(residual)
|
||||
streams = residual.residual
|
||||
stream = (
|
||||
layer.hc_stats_stream
|
||||
if mhc.use_stats_stream(layer.hc_cfg, self.batch, streams)
|
||||
else None
|
||||
)
|
||||
mhc.fork_stats_stream(stream)
|
||||
quantized = (
|
||||
[]
|
||||
if self.attention and layer.self_attn.accepts_mxfp8_swizzled_input()
|
||||
else None
|
||||
)
|
||||
hc = layer.attn_hc if self.attention else layer.ffn_hc
|
||||
x = mhc.combine(hc, residual, quantized)
|
||||
stats = residual.stats
|
||||
residual.release()
|
||||
return x, ReadState(
|
||||
streams,
|
||||
quantized=quantized[0] if quantized else None,
|
||||
stats=stats,
|
||||
stats_stream=stream,
|
||||
call=self.call,
|
||||
)
|
||||
fn, scale, base = self._parameters()
|
||||
x, post, comb, norm_fused = layer.hc_pre(
|
||||
residual,
|
||||
fn,
|
||||
scale,
|
||||
base,
|
||||
norm=norm if layer.boundary_mhc_fusions else None,
|
||||
forward_batch=self.batch,
|
||||
)
|
||||
quantized = None
|
||||
if not norm_fused:
|
||||
if self.attention:
|
||||
x, quantized = layer._input_norm(
|
||||
x, allow_aiter_quant=_is_gfx95_supported
|
||||
)
|
||||
else:
|
||||
x = norm(x)
|
||||
return x, ReadState(residual, post, comb, quantized, call=self.call)
|
||||
|
||||
def _parameters(self):
|
||||
layer = self.layer
|
||||
if self.attention:
|
||||
return layer.hc_attn_fn, layer.hc_attn_scale, layer.hc_attn_base
|
||||
return layer.hc_ffn_fn, layer.hc_ffn_scale, layer.hc_ffn_base
|
||||
|
||||
def update_and_read(self, update, value, residual, norm, **kwargs):
|
||||
if not isinstance(update, HcUpdate):
|
||||
raise TypeError("mHC read requires its producer's update")
|
||||
if self.layer.boundary_pre_from_prev:
|
||||
return self.read(
|
||||
update.post(value, residual, next=self.next_boundary), norm
|
||||
)
|
||||
layer = self.layer
|
||||
if layer.boundary_mhc_fusions and layer.use_fused_mhc_post_pre:
|
||||
from sglang.srt.models.deepseek_common.amd.deepseek_v4_fused_mhc import (
|
||||
apply_mhc_post_pre_boundary,
|
||||
)
|
||||
|
||||
fn, scale, base = self._parameters()
|
||||
cached = (
|
||||
layer._input_layernorm_weight_bf16
|
||||
if self.attention
|
||||
else layer._post_attention_layernorm_weight_bf16
|
||||
)
|
||||
fused = apply_mhc_post_pre_boundary(
|
||||
value,
|
||||
residual.streams,
|
||||
residual.post,
|
||||
residual.comb,
|
||||
fn,
|
||||
scale,
|
||||
base,
|
||||
layer.hc_mult,
|
||||
layer.rms_norm_eps,
|
||||
layer.hc_eps,
|
||||
mhc.MHC_POST_MULT_VALUE,
|
||||
layer.hc_sinkhorn_iters,
|
||||
cached if cached is not None else norm.weight.data,
|
||||
norm.variance_epsilon,
|
||||
fn_transpose=not self.attention,
|
||||
)
|
||||
if fused is not None:
|
||||
streams, x, post, comb, norm_fused = fused
|
||||
quantized = None
|
||||
if not norm_fused:
|
||||
if self.attention:
|
||||
x, quantized = layer._input_norm(
|
||||
x, allow_aiter_quant=_is_gfx95_supported
|
||||
)
|
||||
else:
|
||||
x = norm(x)
|
||||
return x, ReadState(streams, post, comb, quantized, call=self.call)
|
||||
return self.read(update.update(value, residual), norm)
|
||||
|
||||
|
||||
class FinalRead(PendingUpdateRead):
|
||||
bind = None
|
||||
|
||||
@staticmethod
|
||||
def _complete(output, residual, batch, norm, *, update):
|
||||
if (
|
||||
not isinstance(output, MoeFinalize)
|
||||
or output.pieces is None
|
||||
or not update.layer.boundary_pre_from_prev
|
||||
):
|
||||
return None
|
||||
return update.post(output.pieces, residual)
|
||||
|
||||
completing_fusions = (
|
||||
ReadoutFusion(
|
||||
SumGroup.MOE_OUTPUT,
|
||||
_complete.__func__,
|
||||
reads=True,
|
||||
carried=True,
|
||||
accepts_update=lambda update: isinstance(update, HcUpdate),
|
||||
),
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def accepts_pending_update(update):
|
||||
return isinstance(update, HcUpdate)
|
||||
|
||||
@staticmethod
|
||||
def init_residual(value):
|
||||
return value
|
||||
|
||||
@staticmethod
|
||||
def read(value, norm):
|
||||
return value
|
||||
|
||||
@staticmethod
|
||||
def update_and_read(update, value, residual, norm):
|
||||
return update.update(value, residual)
|
||||
|
||||
|
||||
def materialize(hidden_states, batch, *, consume=True, next=None):
|
||||
"""Read mHC streams at a PP, head, Engram or capture seam."""
|
||||
stream = residual_batch.stream_of(batch)
|
||||
stream.check(hidden_states)
|
||||
pending = stream.pending
|
||||
if pending is None:
|
||||
return hidden_states
|
||||
value = pending.for_boundary()
|
||||
if isinstance(value, MoeFinalize) and value.pieces is not None:
|
||||
result = pending.update.post(value.pieces, stream.residual, next=next)
|
||||
elif isinstance(value, UnreducedOutput):
|
||||
result = pending.update.post(
|
||||
mhc.AttnOutput(value.partial), stream.residual, next=next
|
||||
)
|
||||
else:
|
||||
result = pending.update.post(pending.complete(), stream.residual, next=next)
|
||||
if consume:
|
||||
stream.write(result)
|
||||
return result
|
||||
|
||||
|
||||
def stages(config, layer_id, *, layer=None):
|
||||
from sglang.srt.layers.layer_boundary.layout import token_axis_sizes
|
||||
|
||||
ep = not get_moe_a2a_backend().is_none()
|
||||
table_attn, table_ffn, overlapped_ffn = {}, {}, {}
|
||||
for variant in (BatchVariant.ORDINARY, BatchVariant.CONTEXT_PARALLEL):
|
||||
cp = variant is BatchVariant.CONTEXT_PARALLEL
|
||||
source = Layout.sharded_over(
|
||||
TokenAxis.ATTN_DP,
|
||||
*((TokenAxis.ATTN_CP,) if cp else ()),
|
||||
axis_sizes=token_axis_sizes(cp_active=cp),
|
||||
)
|
||||
compute = Layout.sharded_over(
|
||||
*(
|
||||
(TokenAxis.ATTN_DP, TokenAxis.ATTN_CP)
|
||||
if cp and ep
|
||||
else ((TokenAxis.ATTN_DP, TokenAxis.ATTN_TP) if ep else ())
|
||||
),
|
||||
axis_sizes=token_axis_sizes(cp_active=cp),
|
||||
)
|
||||
table_attn[variant] = TransportRows(
|
||||
source,
|
||||
source,
|
||||
source,
|
||||
SumGroup.ATTN_TP,
|
||||
SumGroup.ATTN_CP if get_parallel().attn_cp_size > 1 else None,
|
||||
)
|
||||
table_ffn[variant] = TransportRows(
|
||||
source, compute, source, None if ep else SumGroup.MOE_OUTPUT
|
||||
)
|
||||
if not cp:
|
||||
overlapped_ffn[variant] = TransportRows(
|
||||
source,
|
||||
source if ep else compute,
|
||||
source,
|
||||
None if ep else SumGroup.MOE_OUTPUT,
|
||||
)
|
||||
if layer is None:
|
||||
from sglang.srt.models.deepseek_v4 import (
|
||||
_is_fused_mhc_post_pre_enabled_xpu,
|
||||
is_cross_layer_mhc_fusion_enabled,
|
||||
)
|
||||
|
||||
lagged = config.hc_pre_from_prev_sublayer
|
||||
fused = (
|
||||
is_cross_layer_mhc_fusion_enabled() or _is_fused_mhc_post_pre_enabled_xpu()
|
||||
)
|
||||
else:
|
||||
lagged = layer.boundary_pre_from_prev
|
||||
fused = layer.boundary_mhc_fusions and layer.use_fused_mhc_post_pre
|
||||
applied = False if lagged else not fused
|
||||
if layer is None:
|
||||
read = ReadFacts.of(HcRead(None, True))
|
||||
au = UpdateFacts.of(HcUpdate(None, True))
|
||||
fu = UpdateFacts.of(HcUpdate(None, False, applied_at_exit=applied))
|
||||
else:
|
||||
read = HcRead(layer, True)
|
||||
au = HcUpdate(layer, True)
|
||||
fu = HcUpdate(layer, False, applied_at_exit=applied)
|
||||
for variant in (BatchVariant.UNPADDED, BatchVariant.INPUT_SCATTERED):
|
||||
table_attn[variant] = table_attn[BatchVariant.ORDINARY]
|
||||
table_ffn[variant] = table_ffn[BatchVariant.ORDINARY]
|
||||
# TBO keeps source rows until expert dispatch, including an uneven final
|
||||
# microbatch. It does not shard these rows across attention TP.
|
||||
overlapped_ffn[BatchVariant.UNPADDED] = overlapped_ffn[BatchVariant.ORDINARY]
|
||||
return (
|
||||
declare_attn(
|
||||
read=read,
|
||||
update=au,
|
||||
gathers_attn_tp_input=False,
|
||||
row_transport=RowTransport(
|
||||
table_attn,
|
||||
layer._bind_attention_transport if layer is not None else None,
|
||||
),
|
||||
),
|
||||
declare_ffn(
|
||||
sparse=True,
|
||||
read=HcRead(layer, False) if layer is not None else read,
|
||||
update=fu,
|
||||
row_transport=RowTransport(
|
||||
table_ffn,
|
||||
layer._bind_moe_transport if layer is not None else None,
|
||||
overlapped_rows=overlapped_ffn,
|
||||
),
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
class AttentionTransport(ComputeTransport):
|
||||
def __init__(self, layer, hidden_states, batch, call, rows):
|
||||
if rows.source != rows.compute or rows.source != rows.returned:
|
||||
raise ValueError("attention transport must preserve its source rows")
|
||||
if rows.group is not SumGroup.ATTN_TP or rows.decode_group not in (
|
||||
None,
|
||||
SumGroup.ATTN_CP,
|
||||
):
|
||||
raise ValueError("attention transport does not match its projection groups")
|
||||
self.num_rows = hidden_states.shape[0]
|
||||
self.rows = rows
|
||||
self.input = hidden_states
|
||||
self.defers_sum = (
|
||||
layer.boundary_mhc_fusions
|
||||
and layer.boundary_pre_from_prev
|
||||
and layer._can_fuse_attn_mhc
|
||||
and can_fuse_all_reduce(hidden_states.shape[0], layer.hc_cfg.hidden)
|
||||
)
|
||||
|
||||
def release(self):
|
||||
self.input = None
|
||||
|
||||
def prepare_output(self, output):
|
||||
self.input = None
|
||||
value = output.partial if isinstance(output, mhc.AttnOutput) else output
|
||||
if value.shape[0] != self.num_rows:
|
||||
raise ValueError("attention output changed its source rows")
|
||||
if not isinstance(output, mhc.AttnOutput):
|
||||
return output
|
||||
ctx = get_cp_decode_attn_tp_ctx()
|
||||
group = self.rows.decode_group if ctx.use_decode_attn_tp else self.rows.group
|
||||
if group is None:
|
||||
raise ValueError("attention output owes no declared transport sum")
|
||||
return UnreducedOutput(output.partial, group=_sum_group(group))
|
||||
|
||||
|
||||
class MoeTransport(MoeRowTransport):
|
||||
def __init__(self, layer, hidden_states, forward_batch, call, rows):
|
||||
self.source_rows = hidden_states.shape[0]
|
||||
self.rows = rows
|
||||
if (rows.group is None) != (not get_moe_a2a_backend().is_none()):
|
||||
raise ValueError("MoE backend does not match its transport output contract")
|
||||
cp = TokenAxis.ATTN_CP in rows.source.sharded
|
||||
super().__init__(
|
||||
layer.mlp,
|
||||
hidden_states,
|
||||
forward_batch,
|
||||
call.input_ids,
|
||||
call.input_ids_global,
|
||||
cp_return_contiguous=layer.config.model_type == "deepseek_v41",
|
||||
use_cp=cp,
|
||||
moe_gather=not cp
|
||||
and TokenAxis.ATTN_DP in rows.source.sharded - rows.compute.sharded,
|
||||
attn_scatter=not cp
|
||||
and TokenAxis.ATTN_TP in rows.compute.sharded - rows.source.sharded,
|
||||
)
|
||||
self.sum_in_return = self.mlp_reduce_scatter and rows.group is not None
|
||||
self.return_moe_output = self.defers_sum = (
|
||||
layer.boundary_mhc_fusions
|
||||
and layer.boundary_pre_from_prev
|
||||
and layer._can_fuse_ffn_mhc
|
||||
and can_fuse_all_reduce(self.input.shape[0], layer.hc_cfg.hidden)
|
||||
and not self._use_cp
|
||||
and not self._use_tp_moe_gather
|
||||
and not self._use_tp_attn_a2a_scatter
|
||||
)
|
||||
|
||||
def prepare_output(self, hidden_states):
|
||||
self.input = None
|
||||
if isinstance(hidden_states, MoEOutput):
|
||||
completion = MoeFinalize(hidden_states)
|
||||
return (
|
||||
completion.complete(reduce=False)
|
||||
if self.mlp_reduce_scatter
|
||||
else completion
|
||||
)
|
||||
if self.rows.group is not None and not self.mlp_reduce_scatter:
|
||||
return MoeFinalize(partial=hidden_states)
|
||||
return hidden_states
|
||||
|
||||
def return_output(self, hidden_states):
|
||||
if self.defers_sum:
|
||||
return hidden_states
|
||||
value = super().return_output(hidden_states)
|
||||
if value.shape[0] != self.source_rows:
|
||||
raise ValueError("MoE transport did not return its source rows")
|
||||
return value
|
||||
@@ -2,6 +2,7 @@ from __future__ import annotations
|
||||
|
||||
import copy
|
||||
import logging
|
||||
from contextlib import nullcontext
|
||||
from typing import Iterable, List, Optional, Tuple
|
||||
|
||||
import msgspec
|
||||
@@ -19,6 +20,8 @@ from sglang.srt.distributed.device_communicators.vocab_gather import make_vocab_
|
||||
from sglang.srt.environ import envs
|
||||
from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder
|
||||
from sglang.srt.layers.dp_attention import is_dp_attention_enabled
|
||||
from sglang.srt.layers.layer_boundary import layer_stack
|
||||
from sglang.srt.layers.layer_boundary.residual import batch as residual_batch
|
||||
from sglang.srt.layers.layernorm import RMSNorm
|
||||
from sglang.srt.layers.logits_processor import LogitsProcessorOutput
|
||||
from sglang.srt.layers.moe.utils import is_shared_experts_fusion_disabled
|
||||
@@ -33,18 +36,22 @@ from sglang.srt.model_executor.forward_batch_info import ForwardBatch
|
||||
from sglang.srt.model_executor.forward_context import get_token_to_kv_pool
|
||||
from sglang.srt.model_executor.runner import get_is_capture_mode
|
||||
from sglang.srt.model_loader.weight_utils import default_weight_loader
|
||||
from sglang.srt.models import deepseek_v4_boundary as hc_boundary
|
||||
from sglang.srt.models import deepseek_v4_mhc as mhc
|
||||
from sglang.srt.models.dbrx import ReplicatedLinear
|
||||
from sglang.srt.models.deepseek_v4 import (
|
||||
DEEPSEEK_V4_STACKED_PARAMS_MAPPING,
|
||||
DeepseekV4DecoderLayer,
|
||||
DeepseekV4CudaDecoderLayer,
|
||||
DeepseekV4ForCausalLM,
|
||||
DeepseekV4PlatformDecoderLayer,
|
||||
MqaAttentionBase,
|
||||
_apply_wo_a_bf16_matmul,
|
||||
_dequant_fp8_wo_a_streaming,
|
||||
_is_cuda,
|
||||
hc_head_torch,
|
||||
make_hc_head_params,
|
||||
)
|
||||
from sglang.srt.models.deepseek_v4_platform_stage import NonCudaDSparkMixin
|
||||
from sglang.srt.models.dspark import (
|
||||
BaseDSparkMarkovHead,
|
||||
DSparkConfidenceHead,
|
||||
@@ -249,6 +256,8 @@ class DSparkAttention(MqaAttentionBase):
|
||||
positions: torch.Tensor,
|
||||
hidden_states: torch.Tensor,
|
||||
forward_batch: ForwardBatch,
|
||||
*,
|
||||
defer_all_reduce: bool = False,
|
||||
) -> torch.Tensor:
|
||||
# Idle ranks have no draft KV rows or attention metadata, but still
|
||||
# participate in the shared MoE forward later in the draft stage.
|
||||
@@ -370,8 +379,17 @@ class DSparkAttention(MqaAttentionBase):
|
||||
)
|
||||
else:
|
||||
o = torch.einsum("bgd,grd->bgr", o.float(), wo_a.float()).to(q.dtype)
|
||||
out, _ = self.wo_b(o.reshape(o.shape[0], o.shape[1] * o.shape[2]))
|
||||
return out
|
||||
value = o.reshape(o.shape[0], o.shape[1] * o.shape[2])
|
||||
out, _ = self.wo_b(value, skip_all_reduce=defer_all_reduce)
|
||||
return (
|
||||
mhc.AttnOutput(out)
|
||||
if _is_cuda and defer_all_reduce and self.attn_tp_size > 1
|
||||
else out
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def accepts_mxfp8_swizzled_input():
|
||||
return False
|
||||
|
||||
|
||||
def _resolve_dspark_pool() -> DeepSeekV4TokenToKVPool:
|
||||
@@ -555,7 +573,7 @@ def _dspark_stage_config(config: DeepSeekV4Config) -> DeepSeekV4Config:
|
||||
return stage_config
|
||||
|
||||
|
||||
class DSparkV4Stage(DeepseekV4DecoderLayer):
|
||||
class DSparkV4StageMixin:
|
||||
def __init__(
|
||||
self,
|
||||
config: DeepSeekV4Config,
|
||||
@@ -575,6 +593,8 @@ class DSparkV4Stage(DeepseekV4DecoderLayer):
|
||||
quant_config=quant_config,
|
||||
prefix=prefix,
|
||||
is_nextn=True,
|
||||
boundary_pre_from_prev=config.hc_pre_from_prev_sublayer,
|
||||
boundary_mhc_fusions=False,
|
||||
alt_streams=alt_streams,
|
||||
hc_stats_stream=hc_stats_stream,
|
||||
moe_routed_quant_stream=moe_routed_quant_stream,
|
||||
@@ -628,99 +648,33 @@ class DSparkV4Stage(DeepseekV4DecoderLayer):
|
||||
alt_streams=alt_streams,
|
||||
)
|
||||
|
||||
def _hc_pre_block(
|
||||
self,
|
||||
x: torch.Tensor,
|
||||
hc_fn: torch.Tensor,
|
||||
hc_scale: torch.Tensor,
|
||||
hc_base: torch.Tensor,
|
||||
) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
y, post, comb, _ = self.hc_pre(x, hc_fn, hc_scale, hc_base)
|
||||
return y, post, comb
|
||||
|
||||
def _hc_post_block(
|
||||
self,
|
||||
x: torch.Tensor,
|
||||
residual: torch.Tensor,
|
||||
post: torch.Tensor,
|
||||
comb: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
return self.hc_post(x, residual, post, comb)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
positions: torch.Tensor,
|
||||
hidden_states: torch.Tensor,
|
||||
forward_batch: ForwardBatch,
|
||||
prev_pre: Optional[torch.Tensor] = None,
|
||||
) -> Tuple[torch.Tensor, Optional[torch.Tensor]]:
|
||||
if self.hc_pre_from_prev_sublayer:
|
||||
return self._forward_hc_pre_from_prev(
|
||||
positions, hidden_states, forward_batch, prev_pre
|
||||
)
|
||||
residual = hidden_states
|
||||
x, post, comb = self._hc_pre_block(
|
||||
hidden_states, self.hc_attn_fn, self.hc_attn_scale, self.hc_attn_base
|
||||
class DSparkV4CudaStage(DSparkV4StageMixin, DeepseekV4CudaDecoderLayer):
|
||||
def _run_boundary_attention(
|
||||
self, hidden_states, positions, forward_batch, quantized
|
||||
):
|
||||
return self.self_attn(
|
||||
positions, hidden_states, forward_batch, defer_all_reduce=True
|
||||
)
|
||||
x = self.input_layernorm(x)
|
||||
with self.self_attn.maybe_use_decode_attn_tp(forward_batch):
|
||||
x = self.self_attn(positions, x, forward_batch)
|
||||
x = self._hc_post_block(x, residual, post, comb)
|
||||
|
||||
residual = x
|
||||
x, post, comb = self._hc_pre_block(
|
||||
x, self.hc_ffn_fn, self.hc_ffn_scale, self.hc_ffn_base
|
||||
def forward(self, positions, hidden_states, forward_batch):
|
||||
return super().forward(
|
||||
positions,
|
||||
hidden_states,
|
||||
None,
|
||||
forward_batch,
|
||||
None,
|
||||
allow_cross_layer_fusion=False,
|
||||
)
|
||||
x = self.post_attention_layernorm(x)
|
||||
x = self._run_ffn(x, forward_batch)
|
||||
x = self._hc_post_block(x, residual, post, comb)
|
||||
return x, None
|
||||
|
||||
def _forward_hc_pre_from_prev(
|
||||
self,
|
||||
positions: torch.Tensor,
|
||||
hidden_states: torch.Tensor,
|
||||
forward_batch: ForwardBatch,
|
||||
prev_pre: Optional[torch.Tensor],
|
||||
) -> Tuple[torch.Tensor, torch.Tensor]:
|
||||
# The draft stages run the plain four steps: no fused post, no seams, so
|
||||
# they need nothing from `bind_next` and never hand anything forward.
|
||||
assert self.attn_hc is not None and self.ffn_hc is not None
|
||||
stats_stream = None
|
||||
if mhc.use_stats_stream(self.hc_cfg, forward_batch, hidden_states):
|
||||
stats_stream = self.hc_stats_stream
|
||||
|
||||
residual = hidden_states
|
||||
mhc.fork_stats_stream(stats_stream)
|
||||
x = mhc.combine(self.attn_hc, mhc.HcState(hidden_states, prev_pre))
|
||||
with self.self_attn.maybe_use_decode_attn_tp(forward_batch):
|
||||
x = self.self_attn(positions, x, forward_batch)
|
||||
attn_pre, attn_post, attn_comb = mhc.mix_stats(
|
||||
self.attn_hc, hidden_states, stats_stream
|
||||
)
|
||||
if stats_stream is not None:
|
||||
torch.cuda.current_stream().wait_stream(stats_stream)
|
||||
hidden_states = self.hc_post(x, residual, attn_post, attn_comb)
|
||||
class DSparkV4PlatformStage(
|
||||
NonCudaDSparkMixin, DSparkV4StageMixin, DeepseekV4PlatformDecoderLayer
|
||||
):
|
||||
"""Non-CUDA draft with platform mHC operations."""
|
||||
|
||||
residual = hidden_states
|
||||
mhc.fork_stats_stream(stats_stream)
|
||||
x = mhc.combine(self.ffn_hc, mhc.HcState(hidden_states, attn_pre))
|
||||
x = self._run_ffn(x, forward_batch)
|
||||
ffn_pre, ffn_post, ffn_comb = mhc.mix_stats(
|
||||
self.ffn_hc, hidden_states, stats_stream
|
||||
)
|
||||
if stats_stream is not None:
|
||||
torch.cuda.current_stream().wait_stream(stats_stream)
|
||||
hidden_states = self.hc_post(x, residual, ffn_post, ffn_comb)
|
||||
return hidden_states, ffn_pre
|
||||
|
||||
def _run_ffn(self, x: torch.Tensor, forward_batch: ForwardBatch) -> torch.Tensor:
|
||||
shape = x.shape
|
||||
x = x.reshape(-1, self.dim)
|
||||
y = self._run_moe_ffn_dp_sync(
|
||||
x, forward_batch, input_ids=None, input_ids_global=None
|
||||
)
|
||||
return y.view(shape)
|
||||
DSparkV4Stage = DSparkV4CudaStage if _is_cuda else DSparkV4PlatformStage
|
||||
|
||||
|
||||
class DeepseekV4ForCausalLMDSpark(nn.Module):
|
||||
@@ -812,23 +766,28 @@ class DeepseekV4ForCausalLMDSpark(nn.Module):
|
||||
and getattr(config, "hc_pre_from_prev_sublayer", False)
|
||||
else None
|
||||
)
|
||||
self.stages = nn.ModuleList(
|
||||
[
|
||||
DSparkV4Stage(
|
||||
config=config,
|
||||
layer_id=stage_id,
|
||||
stage_id=stage_id,
|
||||
num_stages=self.num_stages,
|
||||
num_target_layers=self.num_target_features,
|
||||
quant_config=quant_config,
|
||||
prefix=add_prefix(f"stages.{stage_id}", prefix),
|
||||
alt_streams=self.alt_streams,
|
||||
hc_stats_stream=self.hc_stats_stream,
|
||||
moe_routed_quant_stream=self.moe_routed_quant_stream,
|
||||
)
|
||||
for stage_id in range(self.num_stages)
|
||||
]
|
||||
)
|
||||
with (
|
||||
layer_stack(final_read=hc_boundary.FinalRead())
|
||||
if _is_cuda
|
||||
else nullcontext()
|
||||
):
|
||||
self.stages = nn.ModuleList(
|
||||
[
|
||||
DSparkV4Stage(
|
||||
config=config,
|
||||
layer_id=stage_id,
|
||||
stage_id=stage_id,
|
||||
num_stages=self.num_stages,
|
||||
num_target_layers=self.num_target_features,
|
||||
quant_config=quant_config,
|
||||
prefix=add_prefix(f"stages.{stage_id}", prefix),
|
||||
alt_streams=self.alt_streams,
|
||||
hc_stats_stream=self.hc_stats_stream,
|
||||
moe_routed_quant_stream=self.moe_routed_quant_stream,
|
||||
)
|
||||
for stage_id in range(self.num_stages)
|
||||
]
|
||||
)
|
||||
self.markov_head = DSparkV4MarkovHead(
|
||||
vocab_size=int(config.vocab_size),
|
||||
markov_rank=int(dspark_config.markov_rank),
|
||||
@@ -942,20 +901,37 @@ class DeepseekV4ForCausalLMDSpark(nn.Module):
|
||||
get_embedding: bool = False,
|
||||
pp_proxy_tensors=None,
|
||||
) -> LogitsProcessorOutput:
|
||||
del get_embedding, pp_proxy_tensors
|
||||
if not _is_cuda:
|
||||
return self._forward_impl(input_ids, positions, forward_batch, input_embeds)
|
||||
try:
|
||||
return self._forward_impl(input_ids, positions, forward_batch, input_embeds)
|
||||
finally:
|
||||
residual_batch.clear(forward_batch)
|
||||
|
||||
def _forward_impl(self, input_ids, positions, forward_batch, input_embeds):
|
||||
if input_embeds is None:
|
||||
input_embeds = self.forward_embed(input_ids)
|
||||
x = input_embeds
|
||||
pre = None
|
||||
# Draft MoE layers have no recorder layer index, so keep them out of the stats.
|
||||
if _is_cuda:
|
||||
residual_batch.start(forward_batch)
|
||||
# Draft stages do not have recorder layer indices.
|
||||
with get_global_expert_distribution_recorder().disable_this_region():
|
||||
for stage in self.stages:
|
||||
x, pre = stage(positions, x, forward_batch, pre)
|
||||
if _is_cuda:
|
||||
x = stage(positions, x, forward_batch)
|
||||
else:
|
||||
x, pre = stage(positions, x, forward_batch, pre)
|
||||
if _is_cuda:
|
||||
x = residual_batch.read_output(
|
||||
x, forward_batch, hc_boundary.FinalRead(), None
|
||||
)
|
||||
if self.hc_pre_from_prev_sublayer:
|
||||
pre, x = x.pre, x.residual
|
||||
if self.hc_pre_from_prev_sublayer:
|
||||
from sglang.kernels.ops.layernorm.mhc import hc_combine
|
||||
|
||||
x = hc_combine(x.flatten(1).float(), pre, self.hc_mult, x.dtype)
|
||||
|
||||
return LogitsProcessorOutput(next_token_logits=None, hidden_states=x)
|
||||
|
||||
def collapse_hc_head(self, x: torch.Tensor) -> torch.Tensor:
|
||||
|
||||
@@ -52,6 +52,9 @@ _is_xpu = is_xpu()
|
||||
HcTriplet = Tuple[torch.Tensor, torch.Tensor, torch.Tensor]
|
||||
|
||||
|
||||
MHC_POST_MULT_VALUE = 2.0
|
||||
|
||||
|
||||
class HcConfig(NamedTuple):
|
||||
"""Everything about a layer's mHC that is fixed for the model's lifetime."""
|
||||
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import logging
|
||||
from contextlib import nullcontext
|
||||
from typing import Iterable, Optional, Tuple
|
||||
|
||||
import torch
|
||||
@@ -18,6 +19,8 @@ from sglang.srt.layers.dp_attention import (
|
||||
get_global_dp_buffer_len,
|
||||
is_dp_attention_enabled,
|
||||
)
|
||||
from sglang.srt.layers.layer_boundary import layer_stack
|
||||
from sglang.srt.layers.layer_boundary.residual import batch as residual_batch
|
||||
from sglang.srt.layers.layernorm import RMSNorm
|
||||
from sglang.srt.layers.linear import ReplicatedLinear
|
||||
from sglang.srt.layers.logits_processor import LogitsProcessor
|
||||
@@ -30,9 +33,11 @@ from sglang.srt.layers.vocab_parallel_embedding import (
|
||||
)
|
||||
from sglang.srt.model_executor.forward_batch_info import ForwardBatch
|
||||
from sglang.srt.model_executor.forward_context import get_attn_backend
|
||||
from sglang.srt.models import deepseek_v4_boundary as hc_boundary
|
||||
from sglang.srt.models.deepseek_v4 import (
|
||||
DeepseekV4DecoderLayer,
|
||||
DeepseekV4ForCausalLM,
|
||||
_is_cuda,
|
||||
_is_npu,
|
||||
wo_a_fp8_gemm_enabled,
|
||||
)
|
||||
@@ -95,15 +100,20 @@ class DeepseekV4ModelNextN(nn.Module):
|
||||
else:
|
||||
prefix = add_prefix("decoder", prefix)
|
||||
|
||||
self.decoder = DeepseekV4DecoderLayer(
|
||||
config,
|
||||
layer_id=0,
|
||||
quant_config=quant_config,
|
||||
is_nextn=True,
|
||||
prefix=prefix,
|
||||
alt_streams=None,
|
||||
compress_ratio_override=COMPRESS_RATIO_NEXTN_LAYER,
|
||||
)
|
||||
with (
|
||||
layer_stack(final_read=hc_boundary.FinalRead())
|
||||
if _is_cuda
|
||||
else nullcontext()
|
||||
):
|
||||
self.decoder = DeepseekV4DecoderLayer(
|
||||
config,
|
||||
layer_id=0,
|
||||
quant_config=quant_config,
|
||||
is_nextn=True,
|
||||
prefix=prefix,
|
||||
alt_streams=None,
|
||||
compress_ratio_override=COMPRESS_RATIO_NEXTN_LAYER,
|
||||
)
|
||||
|
||||
self.shared_head = nn.Module()
|
||||
self.shared_head.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
|
||||
@@ -187,17 +197,35 @@ class DeepseekV4ModelNextN(nn.Module):
|
||||
# decoder layer reads the memoized gather instead of re-gathering.
|
||||
prime_rope_cos_sin([self.decoder.self_attn], forward_batch, positions)
|
||||
|
||||
hidden_states, residual, post, comb = self.decoder(
|
||||
positions=positions,
|
||||
hidden_states=hidden_states,
|
||||
forward_batch=forward_batch,
|
||||
input_ids=input_ids,
|
||||
input_ids_global=input_ids_global,
|
||||
)
|
||||
if residual is not None:
|
||||
# NextN has a single decoder layer, so no later layer can consume a
|
||||
# deferred fused hc_post state.
|
||||
hidden_states = self.decoder.hc_post(hidden_states, residual, post, comb)
|
||||
if _is_cuda:
|
||||
residual_batch.start(forward_batch)
|
||||
try:
|
||||
hidden_states = self.decoder(
|
||||
positions=positions,
|
||||
hidden_states=hidden_states,
|
||||
forward_batch=forward_batch,
|
||||
input_ids=input_ids,
|
||||
input_ids_global=input_ids_global,
|
||||
)
|
||||
hidden_states = residual_batch.read_output(
|
||||
hidden_states, forward_batch, hc_boundary.FinalRead(), None
|
||||
)
|
||||
finally:
|
||||
residual_batch.clear(forward_batch)
|
||||
else:
|
||||
hidden_states, residual, post, comb = self.decoder(
|
||||
positions=positions,
|
||||
hidden_states=hidden_states,
|
||||
forward_batch=forward_batch,
|
||||
input_ids=input_ids,
|
||||
input_ids_global=input_ids_global,
|
||||
)
|
||||
if residual is not None:
|
||||
# NextN has a single decoder layer, so no later layer can consume a
|
||||
# deferred fused hc_post state.
|
||||
hidden_states = self.decoder.hc_post(
|
||||
hidden_states, residual, post, comb
|
||||
)
|
||||
|
||||
pre_hc_head = hidden_states.flatten(1)
|
||||
|
||||
|
||||
@@ -0,0 +1,716 @@
|
||||
"""Non-CUDA DeepSeek V4 decoder and draft execution.
|
||||
|
||||
The mixins own platform mHC state, fusion, and TBO operations.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from functools import cached_property
|
||||
from typing import TYPE_CHECKING, Optional, Tuple, Union
|
||||
|
||||
import torch
|
||||
|
||||
from sglang.srt.layers.moe import get_moe_a2a_backend
|
||||
from sglang.srt.layers.quantization.mxfp4_flashinfer_trtllm_moe import (
|
||||
can_fuse_all_reduce,
|
||||
)
|
||||
from sglang.srt.models import deepseek_v4_mhc as mhc
|
||||
from sglang.srt.models.deepseek_common.amd.deepseek_v4_fused_mhc import (
|
||||
_gfx95_dense_post_attention_norm,
|
||||
apply_attention_mhc,
|
||||
apply_mhc_post_pre_boundary,
|
||||
apply_moe_mhc,
|
||||
attention_mhc_fusion,
|
||||
)
|
||||
from sglang.srt.models.deepseek_common.amd.deepseek_v4_fused_mhc import (
|
||||
hc_boundary as make_amd_hc_boundary,
|
||||
)
|
||||
from sglang.srt.models.deepseek_common.amd.deepseek_v4_fused_mhc import (
|
||||
moe_mhc_fusion,
|
||||
)
|
||||
from sglang.srt.models.deepseek_common.moe_transport import (
|
||||
MoeRowTransport,
|
||||
_tbo_combine_moe_output,
|
||||
_tbo_gather_moe_input,
|
||||
_tbo_run_moe,
|
||||
_tbo_wait_moe_output,
|
||||
shared_expert_runs_locally,
|
||||
)
|
||||
from sglang.srt.models.deepseek_v2 import MoEOutput
|
||||
from sglang.srt.models.deepseek_v4_mhc import MHC_POST_MULT_VALUE, AttnOutput
|
||||
from sglang.srt.runtime_context import get_parallel
|
||||
from sglang.srt.utils import is_gfx95_supported
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from sglang.srt.model_executor.forward_batch_info import ForwardBatch
|
||||
from sglang.srt.models import deepseek_v2
|
||||
|
||||
_is_gfx95_supported = is_gfx95_supported()
|
||||
|
||||
|
||||
def _prepare_ffn_input(layer, hidden_states, residual, post, comb, forward_batch):
|
||||
"""Close attention and prepare the FFN with the platform's mHC policy."""
|
||||
if layer.use_fused_mhc_post_pre:
|
||||
post_attn_norm_weight = (
|
||||
layer._post_attention_layernorm_weight_bf16
|
||||
if layer._post_attention_layernorm_weight_bf16 is not None
|
||||
else layer.post_attention_layernorm.weight.data
|
||||
)
|
||||
fused = apply_mhc_post_pre_boundary(
|
||||
hidden_states,
|
||||
residual,
|
||||
post,
|
||||
comb,
|
||||
layer.hc_ffn_fn,
|
||||
layer.hc_ffn_scale,
|
||||
layer.hc_ffn_base,
|
||||
layer.hc_mult,
|
||||
layer.rms_norm_eps,
|
||||
layer.hc_eps,
|
||||
MHC_POST_MULT_VALUE,
|
||||
layer.hc_sinkhorn_iters,
|
||||
post_attn_norm_weight,
|
||||
layer.post_attention_layernorm.variance_epsilon,
|
||||
fn_transpose=True,
|
||||
)
|
||||
if fused is not None:
|
||||
residual, hidden_states, post, comb, norm_fused = fused
|
||||
if not norm_fused:
|
||||
hidden_states = layer.post_attention_layernorm(hidden_states)
|
||||
return residual, hidden_states, post, comb
|
||||
hidden_states = layer.hc_post(hidden_states, residual, post, comb)
|
||||
residual = hidden_states
|
||||
hidden_states, post, comb, norm_fused = layer.hc_pre(
|
||||
hidden_states,
|
||||
layer.hc_ffn_fn,
|
||||
layer.hc_ffn_scale,
|
||||
layer.hc_ffn_base,
|
||||
norm=layer.post_attention_layernorm,
|
||||
forward_batch=forward_batch,
|
||||
)
|
||||
if not norm_fused:
|
||||
hidden_states = layer.post_attention_layernorm(hidden_states)
|
||||
|
||||
return residual, hidden_states, post, comb
|
||||
|
||||
|
||||
class NonCudaDecoderMixin:
|
||||
"""Platform tuple and HcState execution for target and NextN decoders.
|
||||
|
||||
CUDA mHC completion lives in deepseek_v4_boundary; operand movement and
|
||||
finalization live in deepseek_common.moe_transport. Changes to completion
|
||||
ordering must be checked in both implementations.
|
||||
"""
|
||||
|
||||
def __init__(self, *args, **kwargs):
|
||||
super().__init__(*args, **kwargs)
|
||||
self._next_layer = None
|
||||
self.local_boundary = None
|
||||
self.next_boundary = None
|
||||
|
||||
def bind_next(self, next_layer):
|
||||
if not self.hc_pre_from_prev_sublayer:
|
||||
return
|
||||
self._next_layer = None
|
||||
if isinstance(next_layer, NonCudaDecoderMixin) and next_layer.engram is None:
|
||||
self._next_layer = next_layer
|
||||
|
||||
def refresh_mhc_norm_weight_cache(self):
|
||||
super().refresh_mhc_norm_weight_cache()
|
||||
self.local_boundary = None
|
||||
self.next_boundary = None
|
||||
|
||||
@cached_property
|
||||
def _can_fuse_attn_mhc(self):
|
||||
return (
|
||||
get_parallel().tp_size == self.self_attn.attn_tp_size == 4
|
||||
and self.self_attn.wo_b.reduce_results
|
||||
and self.local_boundary.norm_fusable
|
||||
and mhc.can_fuse_post(self.hc_cfg)
|
||||
)
|
||||
|
||||
def forward_hc_pre_from_prev(
|
||||
self,
|
||||
positions: torch.Tensor,
|
||||
state: mhc.HcState,
|
||||
input_ids: torch.Tensor,
|
||||
forward_batch: ForwardBatch,
|
||||
input_ids_global: torch.Tensor,
|
||||
) -> mhc.HcState:
|
||||
"""The layer's two hyper-connections, each collapsing with the previous one's
|
||||
pre-mix."""
|
||||
self._init_boundaries()
|
||||
stats_stream = None
|
||||
if mhc.use_stats_stream(self.hc_cfg, forward_batch, state.residual):
|
||||
stats_stream = self.hc_stats_stream
|
||||
|
||||
def run_attn_hc(state: mhc.HcState) -> mhc.HcState:
|
||||
assert self.attn_hc is not None
|
||||
residual = state.residual
|
||||
quantized = [] if self.self_attn.accepts_mxfp8_swizzled_input() else None
|
||||
mhc.fork_stats_stream(stats_stream)
|
||||
x = mhc.combine(self.attn_hc, state, quantized)
|
||||
precomputed = state.stats
|
||||
state.release()
|
||||
del state
|
||||
world_size = self.self_attn.attn_tp_size
|
||||
fuse_all_reduce_mhc = self._can_fuse_attn_mhc and can_fuse_all_reduce(
|
||||
x.shape[0], self.hc_cfg.hidden
|
||||
)
|
||||
with self.self_attn.maybe_use_decode_attn_tp(forward_batch):
|
||||
y = self.self_attn(
|
||||
x=x,
|
||||
positions=positions,
|
||||
forward_batch=forward_batch,
|
||||
x_quant=quantized[0] if quantized else None,
|
||||
defer_all_reduce=fuse_all_reduce_mhc,
|
||||
)
|
||||
del x
|
||||
return mhc.run_attn_post(
|
||||
self.attn_hc,
|
||||
y,
|
||||
residual,
|
||||
stats_stream=stats_stream,
|
||||
next=self.local_boundary,
|
||||
world_size=world_size,
|
||||
precomputed=precomputed,
|
||||
)
|
||||
|
||||
def run_ffn_hc(state: mhc.HcState) -> mhc.HcState:
|
||||
assert self.ffn_hc is not None
|
||||
residual = state.residual
|
||||
mhc.fork_stats_stream(stats_stream)
|
||||
x = mhc.combine(self.ffn_hc, state)
|
||||
precomputed = state.stats
|
||||
state.release()
|
||||
del state
|
||||
fuse_all_reduce_mhc = self._can_fuse_ffn_mhc and can_fuse_all_reduce(
|
||||
x.shape[0], self.hc_cfg.hidden
|
||||
)
|
||||
y = self._run_moe_ffn_dp_sync(
|
||||
x,
|
||||
forward_batch,
|
||||
input_ids=input_ids,
|
||||
input_ids_global=input_ids_global,
|
||||
return_moe_output=fuse_all_reduce_mhc,
|
||||
)
|
||||
del x
|
||||
return mhc.run_moe_post(
|
||||
self.ffn_hc,
|
||||
y,
|
||||
residual,
|
||||
stats_stream=stats_stream,
|
||||
next=self.next_boundary,
|
||||
world_size=self.mlp.tp_size,
|
||||
precomputed=precomputed,
|
||||
)
|
||||
|
||||
return run_ffn_hc(run_attn_hc(state))
|
||||
|
||||
def _init_boundaries(self) -> None:
|
||||
"""Resolved on first forward: `accepts_mxfp8_swizzled_input` is not settled
|
||||
until `process_weights_after_loading`."""
|
||||
if self.local_boundary is not None:
|
||||
return
|
||||
# ATTN[i] -> MOE[i]
|
||||
self.local_boundary = mhc.make_boundary(
|
||||
self.post_attention_layernorm,
|
||||
accepts_mxfp8=False,
|
||||
hc=self.ffn_hc,
|
||||
)
|
||||
# MOE[i] -> ATTN[i+1]
|
||||
if self._next_layer is not None:
|
||||
self.next_boundary = mhc.make_boundary(
|
||||
self._next_layer.input_layernorm,
|
||||
accepts_mxfp8=self._next_layer.self_attn.accepts_mxfp8_swizzled_input(),
|
||||
hc=self._next_layer.attn_hc,
|
||||
)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
positions: torch.Tensor,
|
||||
hidden_states: torch.Tensor,
|
||||
input_ids: torch.Tensor,
|
||||
forward_batch: ForwardBatch,
|
||||
input_ids_global: torch.Tensor,
|
||||
prev_residual: Optional[torch.Tensor] = None,
|
||||
prev_post: Optional[torch.Tensor] = None,
|
||||
prev_comb: Optional[torch.Tensor] = None,
|
||||
) -> Tuple[
|
||||
torch.Tensor,
|
||||
Optional[torch.Tensor],
|
||||
Optional[torch.Tensor],
|
||||
Optional[torch.Tensor],
|
||||
]:
|
||||
use_fused = self.use_fused_mhc_post_pre
|
||||
|
||||
fused = None
|
||||
if prev_residual is not None and use_fused:
|
||||
# Dispatch cascade: aiter HIP (gfx95) -> Triton (gfx95 small-batch
|
||||
# <=64 tokens, or gfx1250 all sizes) -> TileLang -> None.
|
||||
input_norm_weight = (
|
||||
self._input_layernorm_weight_bf16
|
||||
if self._input_layernorm_weight_bf16 is not None
|
||||
else self.input_layernorm.weight.data
|
||||
)
|
||||
fused = apply_mhc_post_pre_boundary(
|
||||
hidden_states,
|
||||
prev_residual,
|
||||
prev_post,
|
||||
prev_comb,
|
||||
self.hc_attn_fn,
|
||||
self.hc_attn_scale,
|
||||
self.hc_attn_base,
|
||||
self.hc_mult,
|
||||
self.rms_norm_eps,
|
||||
self.hc_eps,
|
||||
MHC_POST_MULT_VALUE,
|
||||
self.hc_sinkhorn_iters,
|
||||
input_norm_weight,
|
||||
self.input_layernorm.variance_epsilon,
|
||||
fn_transpose=False,
|
||||
)
|
||||
if fused is not None:
|
||||
residual, hidden_states, post, comb, norm_fused = fused
|
||||
else:
|
||||
if prev_residual is not None and use_fused:
|
||||
hidden_states = self.hc_post(
|
||||
hidden_states, prev_residual, prev_post, prev_comb
|
||||
)
|
||||
residual = hidden_states
|
||||
hidden_states, post, comb, norm_fused = self.hc_pre(
|
||||
hidden_states,
|
||||
self.hc_attn_fn,
|
||||
self.hc_attn_scale,
|
||||
self.hc_attn_base,
|
||||
norm=self.input_layernorm,
|
||||
forward_batch=forward_batch,
|
||||
)
|
||||
if not norm_fused:
|
||||
# Triton post+pre can leave norm separate: gfx95 may quantize it,
|
||||
# while gfx1250 must keep the plain layernorm path.
|
||||
hidden_states, x_quant = self._input_norm(
|
||||
hidden_states, allow_aiter_quant=_is_gfx95_supported
|
||||
)
|
||||
else:
|
||||
x_quant = None
|
||||
|
||||
with self.self_attn.maybe_use_decode_attn_tp(forward_batch):
|
||||
hidden_states = self.self_attn(
|
||||
x=hidden_states,
|
||||
positions=positions,
|
||||
forward_batch=forward_batch,
|
||||
x_quant=x_quant,
|
||||
)
|
||||
|
||||
del fused
|
||||
residual, hidden_states, post, comb = _prepare_ffn_input(
|
||||
self, hidden_states, residual, post, comb, forward_batch
|
||||
)
|
||||
|
||||
hidden_states = self._run_moe_ffn_dp_sync(
|
||||
hidden_states,
|
||||
forward_batch,
|
||||
input_ids=input_ids,
|
||||
input_ids_global=input_ids_global,
|
||||
)
|
||||
|
||||
if not use_fused:
|
||||
hidden_states = self.hc_post(hidden_states, residual, post, comb)
|
||||
return hidden_states, None, None, None
|
||||
|
||||
# Return the deferred FFN hc_post state; the next layer consumes it with
|
||||
# cross-layer fusion, and the final layer is completed in DeepseekV4Model.
|
||||
return hidden_states, residual, post, comb
|
||||
|
||||
def _run_moe_ffn_dp_sync(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
forward_batch: ForwardBatch,
|
||||
*,
|
||||
input_ids: Optional[torch.Tensor],
|
||||
input_ids_global: Optional[torch.Tensor],
|
||||
return_moe_output: bool = False,
|
||||
) -> Union[torch.Tensor, deepseek_v2.MoEOutput]:
|
||||
prepared = MoeRowTransport(
|
||||
self.mlp,
|
||||
hidden_states,
|
||||
forward_batch,
|
||||
input_ids,
|
||||
input_ids_global,
|
||||
cp_return_contiguous=self.config.model_type == "deepseek_v41",
|
||||
)
|
||||
with prepared.scope():
|
||||
return_moe_output = (
|
||||
return_moe_output
|
||||
and not prepared._use_cp
|
||||
and not prepared._use_tp_moe_gather
|
||||
and not prepared._use_tp_attn_a2a_scatter
|
||||
)
|
||||
hidden_states = self.mlp(
|
||||
prepared.input,
|
||||
forward_batch,
|
||||
input_ids=prepared.input_ids,
|
||||
input_ids_global=prepared.input_ids_global,
|
||||
skip_shared_experts=prepared.do_shared_local,
|
||||
return_moe_output=return_moe_output,
|
||||
)
|
||||
prepared.input = None
|
||||
return prepared.return_output(hidden_states)
|
||||
|
||||
def op_mhc_prepare_attn(
|
||||
self,
|
||||
state,
|
||||
positions: torch.Tensor,
|
||||
hidden_states: torch.Tensor,
|
||||
forward_batch: ForwardBatch,
|
||||
residual: Optional[torch.Tensor] = None,
|
||||
tbo_subbatch_index: Optional[int] = None,
|
||||
**kwargs,
|
||||
):
|
||||
# Non-fused attention-side mHC pre + input layernorm.
|
||||
attn_residual = hidden_states
|
||||
hidden_states, post, comb, norm_fused = self.hc_pre(
|
||||
hidden_states,
|
||||
self.hc_attn_fn,
|
||||
self.hc_attn_scale,
|
||||
self.hc_attn_base,
|
||||
norm=self.input_layernorm,
|
||||
forward_batch=forward_batch,
|
||||
)
|
||||
if not norm_fused:
|
||||
hidden_states, x_quant = self._input_norm(hidden_states)
|
||||
else:
|
||||
x_quant = None
|
||||
|
||||
state.attn_residual = attn_residual
|
||||
state.attn_post = post
|
||||
state.attn_comb = comb
|
||||
state.hidden_states_after_input_norm = hidden_states
|
||||
state.attn_x_quant = x_quant
|
||||
# mori's op_output slices final_hidden_states[:num_tokens].
|
||||
if get_moe_a2a_backend().is_mori():
|
||||
state.num_tokens = attn_residual.shape[0]
|
||||
state.update(
|
||||
dict(
|
||||
forward_batch=forward_batch,
|
||||
positions=positions,
|
||||
tbo_subbatch_index=tbo_subbatch_index,
|
||||
)
|
||||
)
|
||||
|
||||
def op_mhc_post_attn_pre_mlp(self, state):
|
||||
# Close the attention mHC (hc_post), then open the FFN-side mHC pre +
|
||||
# post-attention layernorm. Produces the 2D MoE input.
|
||||
#
|
||||
# Pop each boundary tensor from the state EXACTLY ONCE, up front, and
|
||||
# reuse the locals for both the fused attempt and the non-fused
|
||||
# fallback. apply_mhc_post_pre_boundary() returns None when it declines
|
||||
# to fuse -- most importantly for the 0-token DP two-batch-overlap idle
|
||||
# ubatch -- in which case control must fall through to the unfused
|
||||
# hc_post. Popping in the fused call's arguments and again in the
|
||||
# fallback would double-pop -> KeyError: 'hidden_states_after_attn' on
|
||||
# every idle DP rank. use_fused_mhc_post_pre is on whenever the aiter
|
||||
# gfx95 mHC path is available (is_cross_layer_mhc_fusion_enabled), so
|
||||
# this fallback is reached under DP regardless of the TileLang env.
|
||||
hidden_states_after_attn = state.pop("hidden_states_after_attn")
|
||||
attn_residual = state.pop("attn_residual")
|
||||
attn_post = state.pop("attn_post")
|
||||
attn_comb = state.pop("attn_comb")
|
||||
|
||||
residual, hidden_states, post, comb = _prepare_ffn_input(
|
||||
self,
|
||||
hidden_states_after_attn,
|
||||
attn_residual,
|
||||
attn_post,
|
||||
attn_comb,
|
||||
state.forward_batch,
|
||||
)
|
||||
state.ffn_residual = residual
|
||||
state.ffn_post = post
|
||||
state.ffn_comb = comb
|
||||
state.hidden_states_mlp_input = hidden_states
|
||||
|
||||
def op_mhc_postprocess(self, state):
|
||||
# Close the FFN mHC (hc_post) and emit the next layer's input dict.
|
||||
hidden_states = self.hc_post(
|
||||
state.pop("hidden_states_mlp_output"),
|
||||
state.pop("ffn_residual"),
|
||||
state.pop("ffn_post"),
|
||||
state.pop("ffn_comb"),
|
||||
)
|
||||
output = dict(
|
||||
positions=state.positions,
|
||||
hidden_states=hidden_states,
|
||||
# DSV4 non-fused layers carry no residual across layers; the key is
|
||||
# required by the next layer's op_mhc_prepare_attn (ignored).
|
||||
residual=None,
|
||||
forward_batch=state.forward_batch,
|
||||
tbo_subbatch_index=state.tbo_subbatch_index,
|
||||
)
|
||||
state.clear(
|
||||
expect_keys={
|
||||
"positions",
|
||||
"forward_batch",
|
||||
"tbo_subbatch_index",
|
||||
}
|
||||
)
|
||||
return output
|
||||
|
||||
def op_gather_a(self, state):
|
||||
local = state.pop("hidden_states_mlp_input")
|
||||
state.do_shared_local = shared_expert_runs_locally(self.mlp)
|
||||
state.global_hidden, state.gather_event, state.shared_local = (
|
||||
_tbo_gather_moe_input(
|
||||
self.mlp,
|
||||
local,
|
||||
state.forward_batch,
|
||||
state.tbo_subbatch_index,
|
||||
state.do_shared_local,
|
||||
)
|
||||
)
|
||||
state.gather_keepalive = local
|
||||
|
||||
def op_gather_b(self, state):
|
||||
torch.cuda.current_stream().wait_event(state.pop("gather_event"))
|
||||
# Compute now ordered after the gather -> the gather input is safe to
|
||||
# release (freed on the compute stream, no record_stream deferral).
|
||||
state.pop("gather_keepalive")
|
||||
|
||||
def op_moe(self, state):
|
||||
state.global_expert_out = _tbo_run_moe(
|
||||
self, state.pop("global_hidden"), state.forward_batch, state.do_shared_local
|
||||
)
|
||||
|
||||
def op_combine_a(self, state):
|
||||
output = state.pop("global_expert_out")
|
||||
state.local_out, state.combine_event = _tbo_combine_moe_output(
|
||||
output, state.tbo_subbatch_index
|
||||
)
|
||||
state.combine_keepalive = output
|
||||
|
||||
def op_combine_b(self, state):
|
||||
state.hidden_states_mlp_output = _tbo_wait_moe_output(
|
||||
state.pop("local_out"),
|
||||
state.pop("combine_event"),
|
||||
state.pop("shared_local"),
|
||||
)
|
||||
state.pop("combine_keepalive")
|
||||
state.pop("do_shared_local")
|
||||
|
||||
|
||||
class NonCudaDSparkMixin(NonCudaDecoderMixin):
|
||||
"""Draft stage execution with plain norms and its attention call."""
|
||||
|
||||
def forward(
|
||||
self,
|
||||
positions: torch.Tensor,
|
||||
hidden_states: torch.Tensor,
|
||||
forward_batch: ForwardBatch,
|
||||
prev_pre: Optional[torch.Tensor] = None,
|
||||
) -> Tuple[torch.Tensor, Optional[torch.Tensor]]:
|
||||
if self.hc_pre_from_prev_sublayer:
|
||||
return self._forward_hc_pre_from_prev(
|
||||
positions, hidden_states, forward_batch, prev_pre
|
||||
)
|
||||
residual = hidden_states
|
||||
x, post, comb = self._hc_pre_block(
|
||||
hidden_states, self.hc_attn_fn, self.hc_attn_scale, self.hc_attn_base
|
||||
)
|
||||
x = self.input_layernorm(x)
|
||||
with self.self_attn.maybe_use_decode_attn_tp(forward_batch):
|
||||
x = self.self_attn(positions, x, forward_batch)
|
||||
x = self._hc_post_block(x, residual, post, comb)
|
||||
|
||||
residual = x
|
||||
x, post, comb = self._hc_pre_block(
|
||||
x, self.hc_ffn_fn, self.hc_ffn_scale, self.hc_ffn_base
|
||||
)
|
||||
x = self.post_attention_layernorm(x)
|
||||
x = self._run_ffn(x, forward_batch)
|
||||
x = self._hc_post_block(x, residual, post, comb)
|
||||
return x, None
|
||||
|
||||
def _hc_pre_block(
|
||||
self,
|
||||
x: torch.Tensor,
|
||||
hc_fn: torch.Tensor,
|
||||
hc_scale: torch.Tensor,
|
||||
hc_base: torch.Tensor,
|
||||
) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
y, post, comb, _ = self.hc_pre(x, hc_fn, hc_scale, hc_base)
|
||||
return y, post, comb
|
||||
|
||||
def _hc_post_block(
|
||||
self,
|
||||
x: torch.Tensor,
|
||||
residual: torch.Tensor,
|
||||
post: torch.Tensor,
|
||||
comb: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
return self.hc_post(x, residual, post, comb)
|
||||
|
||||
def _forward_hc_pre_from_prev(
|
||||
self,
|
||||
positions: torch.Tensor,
|
||||
hidden_states: torch.Tensor,
|
||||
forward_batch: ForwardBatch,
|
||||
prev_pre: Optional[torch.Tensor],
|
||||
) -> Tuple[torch.Tensor, torch.Tensor]:
|
||||
# The draft stages run the plain four steps: no fused post, no seams, so
|
||||
# they need nothing from `bind_next` and never hand anything forward.
|
||||
assert self.attn_hc is not None and self.ffn_hc is not None
|
||||
stats_stream = None
|
||||
if mhc.use_stats_stream(self.hc_cfg, forward_batch, hidden_states):
|
||||
stats_stream = self.hc_stats_stream
|
||||
|
||||
residual = hidden_states
|
||||
mhc.fork_stats_stream(stats_stream)
|
||||
x = mhc.combine(self.attn_hc, mhc.HcState(hidden_states, prev_pre))
|
||||
with self.self_attn.maybe_use_decode_attn_tp(forward_batch):
|
||||
x = self.self_attn(positions, x, forward_batch)
|
||||
attn_pre, attn_post, attn_comb = mhc.mix_stats(
|
||||
self.attn_hc, hidden_states, stats_stream
|
||||
)
|
||||
if stats_stream is not None:
|
||||
torch.cuda.current_stream().wait_stream(stats_stream)
|
||||
hidden_states = self.hc_post(x, residual, attn_post, attn_comb)
|
||||
|
||||
residual = hidden_states
|
||||
mhc.fork_stats_stream(stats_stream)
|
||||
x = mhc.combine(self.ffn_hc, mhc.HcState(hidden_states, attn_pre))
|
||||
x = self._run_ffn(x, forward_batch)
|
||||
ffn_pre, ffn_post, ffn_comb = mhc.mix_stats(
|
||||
self.ffn_hc, hidden_states, stats_stream
|
||||
)
|
||||
if stats_stream is not None:
|
||||
torch.cuda.current_stream().wait_stream(stats_stream)
|
||||
hidden_states = self.hc_post(x, residual, ffn_post, ffn_comb)
|
||||
return hidden_states, ffn_pre
|
||||
|
||||
def _run_ffn(self, x: torch.Tensor, forward_batch: ForwardBatch) -> torch.Tensor:
|
||||
shape = x.shape
|
||||
x = x.reshape(-1, self.dim)
|
||||
y = self._run_moe_ffn_dp_sync(
|
||||
x, forward_batch, input_ids=None, input_ids_global=None
|
||||
)
|
||||
return y.view(shape)
|
||||
|
||||
|
||||
def forward_layer_fused_boundary(
|
||||
model,
|
||||
i: int,
|
||||
state,
|
||||
*,
|
||||
positions: torch.Tensor,
|
||||
input_ids: torch.Tensor,
|
||||
forward_batch,
|
||||
input_ids_global: torch.Tensor,
|
||||
capture_dspark: bool,
|
||||
):
|
||||
"""Layer i through the fused mHC boundary. Its FFN hc_post stays pending for the next
|
||||
layer's boundary launch unless that layer reads the residual stream first (Engram gate,
|
||||
DSpark capture) or there is none; hidden_states is None while a post is pending."""
|
||||
nxt = i + 1
|
||||
defer_post = (
|
||||
nxt < model.end_layer
|
||||
and model.layers[nxt].engram is None
|
||||
and not (capture_dspark and nxt in model.dspark_layers_to_capture)
|
||||
)
|
||||
# The boundary speaks its own (hidden_states, prev_pre, pending_post) triple;
|
||||
# the loop speaks HcState. This is the only place the two meet.
|
||||
from sglang.srt.models.deepseek_v4_mhc import HcPending, HcState
|
||||
|
||||
pending = isinstance(state.streams, HcPending)
|
||||
hidden_states, prev_pre, pending_post = forward_hc_pre_from_prev_fused_boundary(
|
||||
model.layers[i],
|
||||
positions=positions,
|
||||
hidden_states=None if pending else state.streams,
|
||||
input_ids=input_ids,
|
||||
forward_batch=forward_batch,
|
||||
input_ids_global=input_ids_global,
|
||||
prev_pre=state.pre,
|
||||
pending_post=tuple(state.streams) if pending else None,
|
||||
defer_post=defer_post,
|
||||
)
|
||||
return HcState(
|
||||
hidden_states if pending_post is None else HcPending(*pending_post),
|
||||
prev_pre,
|
||||
)
|
||||
|
||||
|
||||
def forward_hc_pre_from_prev_fused_boundary(
|
||||
layer,
|
||||
positions: torch.Tensor,
|
||||
hidden_states: Optional[torch.Tensor],
|
||||
input_ids: torch.Tensor,
|
||||
forward_batch,
|
||||
input_ids_global: torch.Tensor,
|
||||
prev_pre: Optional[torch.Tensor],
|
||||
pending_post: Optional[Tuple[torch.Tensor, ...]],
|
||||
defer_post: bool,
|
||||
) -> Tuple[Optional[torch.Tensor], torch.Tensor, Optional[Tuple[torch.Tensor, ...]]]:
|
||||
"""ROCm fused attention and FFN hyper-connections.
|
||||
pending_post is the previous layer's unapplied FFN hc_post (x, residual, post,
|
||||
comb); with defer_post this layer's is returned the same way and hidden_states is None."""
|
||||
if pending_post is None:
|
||||
pending_post = (None, hidden_states, None, None)
|
||||
residual, x, attn_coefficients = make_amd_hc_boundary(
|
||||
layer,
|
||||
*pending_post,
|
||||
prev_pre,
|
||||
layer.hc_attn_fn,
|
||||
layer.hc_attn_scale,
|
||||
layer.hc_attn_base,
|
||||
)
|
||||
# the boundary's reduce + sinkhorn rides in the norm launch
|
||||
x, x_quant = layer._input_norm(
|
||||
x, allow_aiter_quant=False, coefficients=attn_coefficients
|
||||
)
|
||||
with layer.self_attn.maybe_use_decode_attn_tp(forward_batch):
|
||||
mhc = attention_mhc_fusion(layer, residual, attn_coefficients, forward_batch)
|
||||
x = layer.self_attn(
|
||||
x=x,
|
||||
positions=positions,
|
||||
forward_batch=forward_batch,
|
||||
x_quant=x_quant,
|
||||
defer_all_reduce=mhc is not None,
|
||||
)
|
||||
if mhc is not None:
|
||||
# wo_b honors the deferred reduce under attention_mhc_fusion's gates;
|
||||
# fail loud rather than fall back to an unfused reduce.
|
||||
assert isinstance(x, AttnOutput)
|
||||
residual = apply_attention_mhc(x.partial, mhc)
|
||||
x = None
|
||||
residual, x, ffn_coefficients = make_amd_hc_boundary(
|
||||
layer,
|
||||
x,
|
||||
residual,
|
||||
attn_coefficients.post if x is not None else None,
|
||||
attn_coefficients.comb if x is not None else None,
|
||||
attn_coefficients.pre,
|
||||
layer.hc_ffn_fn,
|
||||
layer.hc_ffn_scale,
|
||||
layer.hc_ffn_base,
|
||||
)
|
||||
x = _gfx95_dense_post_attention_norm(layer, x, ffn_coefficients)
|
||||
mhc = moe_mhc_fusion(layer, residual, ffn_coefficients, forward_batch)
|
||||
x = layer._run_moe_ffn_dp_sync(
|
||||
x,
|
||||
forward_batch,
|
||||
input_ids=input_ids,
|
||||
input_ids_global=input_ids_global,
|
||||
return_moe_output=mhc is not None,
|
||||
)
|
||||
ffn_pre, ffn_post, ffn_comb = ffn_coefficients.tensors()
|
||||
if mhc is not None:
|
||||
# The MoE TP path honors the deferred output under moe_mhc_fusion's gates;
|
||||
# fail loud rather than fall back to an unfused reduce. The fused post
|
||||
# carries the reduce, so take the merged rows without one.
|
||||
assert isinstance(x, MoEOutput)
|
||||
# Reduction already applied post. The next boundary consumes this
|
||||
# materialized residual, including when it would normally defer post.
|
||||
return apply_moe_mhc(x.get_merged(), mhc), ffn_pre, None
|
||||
if defer_post:
|
||||
return None, ffn_pre, (x, residual, ffn_post, ffn_comb)
|
||||
return layer.hc_post(x, residual, ffn_post, ffn_comb), ffn_pre, None
|
||||
@@ -137,7 +137,7 @@ if _use_aiter_gfx95:
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Matches DeepSeek-V4's _MHC_POST_MULT_VALUE; the fused and unfused boundaries
|
||||
# Matches deepseek_v4_mhc.MHC_POST_MULT_VALUE; the fused and unfused boundaries
|
||||
# must agree on it.
|
||||
_MHC_POST_MULT_VALUE = 2.0
|
||||
|
||||
|
||||
Reference in New Issue
Block a user