[Refactor] Drop the reduction-skip mechanisms stage boundaries no longer use (#42312)
This commit is contained in:
@@ -115,9 +115,6 @@ class ExitMove(NamedTuple):
|
||||
# Whether output_move also completes the sum the producer leaves.
|
||||
output_move_completes_sum: bool = False
|
||||
returns_over_dp: bool = False
|
||||
# When output_move completes the sum: the move of an output compute has
|
||||
# already reduced onto the same rows.
|
||||
complete_output_move: Optional[Callable] = None
|
||||
|
||||
|
||||
def input_rows(edge: EdgeContract) -> Layout:
|
||||
@@ -599,9 +596,8 @@ def _select_exit_move(
|
||||
) -> ExitMove:
|
||||
"""How the FFN output reaches the rows the layer hands on: by undoing the
|
||||
attention-DP gather (the FFN exit and finish run that step), or else the
|
||||
move that takes it there, None when there is none to choose here; whether
|
||||
that move also completes the sum the FFN leaves, and if so the move for an
|
||||
output compute already reduced."""
|
||||
move that takes it there, None when there is none to choose here; and
|
||||
whether that move also completes the sum the FFN leaves."""
|
||||
|
||||
update = produced.update
|
||||
if produced.layout == residual:
|
||||
@@ -634,12 +630,6 @@ def _select_exit_move(
|
||||
update=update,
|
||||
),
|
||||
output_move_completes_sum=True,
|
||||
complete_output_move=partial(
|
||||
residual_slice_output,
|
||||
sums=False,
|
||||
gathers_back=to != residual,
|
||||
update=update,
|
||||
),
|
||||
)
|
||||
if to != residual or not produced.layout.sharded <= residual.sharded:
|
||||
raise NotImplementedError(f"{produced=} {residual=} {to=}")
|
||||
@@ -655,11 +645,7 @@ def _select_exit_move(
|
||||
_sum_group(produced.group), cp_moves.reduce_scatter_group()
|
||||
):
|
||||
raise NotImplementedError(f"{produced=} {residual=} {to=}")
|
||||
return ExitMove(
|
||||
cp_moves.reduce_scatter,
|
||||
output_move_completes_sum=True,
|
||||
complete_output_move=cp_moves.take_back,
|
||||
)
|
||||
return ExitMove(cp_moves.reduce_scatter, output_move_completes_sum=True)
|
||||
if returned == {TokenAxis.ATTN_DP, TokenAxis.ATTN_CP}:
|
||||
# This rank's CP shard, from where the DP gather put it.
|
||||
return ExitMove(dp_cp_take_back_output)
|
||||
|
||||
@@ -214,7 +214,6 @@ class StagePlan:
|
||||
output_move=out.output_move,
|
||||
output_move_completes_sum=out.output_move_completes_sum,
|
||||
returns_over_dp=out.returns_over_dp,
|
||||
complete_output_move=out.complete_output_move,
|
||||
)
|
||||
|
||||
@property
|
||||
|
||||
@@ -27,19 +27,16 @@ from sglang.srt.layers.layer_boundary.residual.add_norm import NORM_READOUT, PLA
|
||||
|
||||
|
||||
class ProducerReduction(Enum):
|
||||
"""Describe compute's cooperation with its output boundary.
|
||||
"""Who completes the sum a stage's output owes; compute never does.
|
||||
|
||||
ALWAYS_PARTIAL is attention-only: finish() publishes its partial sum. EXIT_SCOPED
|
||||
follows exit() flags for an FFN or single-stage mixer. TAIL_AFTER_SUM is FFN-only:
|
||||
compute adds a replicated component after its internal sum, so the
|
||||
boundary never defers that sum to the next layer. A selected reduce-scatter
|
||||
still applies, so compute must add the tail only when it completes the sum.
|
||||
ALWAYS_PARTIAL is attention-only: finish() hands the partial sum to the
|
||||
next stage's input as its declared sum. EXIT_SCOPED, for an FFN or a
|
||||
single-stage mixer: the exit completes the sum, or carries it to the next
|
||||
stage's input.
|
||||
"""
|
||||
|
||||
ALWAYS_PARTIAL = auto()
|
||||
EXIT_SCOPED = auto()
|
||||
# A replicated component is added after the sum inside compute.
|
||||
TAIL_AFTER_SUM = auto()
|
||||
|
||||
|
||||
class ExitRows(Enum):
|
||||
@@ -80,30 +77,30 @@ class OutputContract(msgspec.Struct, frozen=True):
|
||||
|
||||
Fields:
|
||||
layout: Token sharding of the producer contribution.
|
||||
group: Named sum group, or None when there is no reduction.
|
||||
always_partial: Compute always returns a partial sum.
|
||||
may_defer_to_next: Compute can skip reduction under the exit scope
|
||||
and let the following layer complete it.
|
||||
may_reduce_scatter: Compute can leave reduction to a fixed-size
|
||||
reduce-scatter selected by the boundary.
|
||||
may_reduce_scatterv: Compute can leave reduction to the selected
|
||||
variable-size attention-DP combine.
|
||||
group: The group the output owes its sum over, or None when it is
|
||||
complete.
|
||||
always_partial: The sum is handed to the next stage's input as its
|
||||
declared sum.
|
||||
may_defer_to_next: The exit may carry the sum to the following
|
||||
stage's input instead of completing it.
|
||||
may_reduce_scatter: A fixed-size reduce-scatter selected by the
|
||||
boundary may complete the sum.
|
||||
may_reduce_scatterv: The selected variable-size attention-DP combine
|
||||
may complete the sum.
|
||||
update: Producer residual operation; None on an arrival contract that
|
||||
declares capabilities and obtains the actual update from the stream.
|
||||
transform: Optional operation on the contribution before residual update.
|
||||
|
||||
These permissions are not evidence that a particular output is partial.
|
||||
The exit decision and ResidualStream record what that output actually owes.
|
||||
These are permissions, not a record of a particular output. The exit
|
||||
decision and ResidualStream record what that output actually owes.
|
||||
"""
|
||||
|
||||
layout: Layout
|
||||
# None when there is nothing to sum.
|
||||
group: Optional[SumGroup] = None
|
||||
# The producer never reduces its output, e.g. a row-parallel projection
|
||||
# built with reduce_results=False.
|
||||
# The sum is always handed to the next stage's input as a declared sum.
|
||||
always_partial: bool = False
|
||||
# Otherwise it leaves the sum only when the boundary publishes the flag for
|
||||
# it: fuse_mlp_allreduce, or mlp_reduce_scatter.
|
||||
# Otherwise the exit completes it, or may carry it to the next input.
|
||||
may_defer_to_next: bool = False
|
||||
may_reduce_scatter: bool = False
|
||||
# Whether the selected attention-DP combine may complete this sum.
|
||||
@@ -236,8 +233,6 @@ class StagePath(msgspec.Struct, frozen=True):
|
||||
batch by the attention-DP exit path.
|
||||
output_move_completes_sum: Whether that move also reduces the output.
|
||||
returns_over_dp: Whether output uses batch-dependent attention-DP transport.
|
||||
complete_output_move: The move for an output compute already reduced,
|
||||
run instead of an output_move that also reduces it.
|
||||
"""
|
||||
|
||||
entry: EntryPath
|
||||
@@ -247,7 +242,6 @@ class StagePath(msgspec.Struct, frozen=True):
|
||||
output_move_completes_sum: bool = False
|
||||
|
||||
returns_over_dp: bool = False
|
||||
complete_output_move: Optional[Callable] = None
|
||||
|
||||
|
||||
class StageKind(Enum):
|
||||
|
||||
@@ -129,11 +129,11 @@ class ExitPolicy:
|
||||
return stream.record(hidden_states, update, declared_sum=declared_sum)
|
||||
|
||||
@staticmethod
|
||||
def _sum_owed_after_skip(steps, skipped_reduction):
|
||||
# The input-scattered path leaves the sum for the next input's TP
|
||||
# reduce-scatter. Other skip paths complete it in the output move.
|
||||
def _sum_left_to_next_input(steps, sum_in_reduce_scatter):
|
||||
# On an input-scattered batch the next input's TP reduce-scatter
|
||||
# completes the sum; the other reduce-scatters are this exit's move.
|
||||
if (
|
||||
skipped_reduction
|
||||
sum_in_reduce_scatter
|
||||
and not steps.returns_over_dp
|
||||
and not steps.output_move_completes_sum
|
||||
):
|
||||
@@ -160,10 +160,9 @@ class ExitPolicy:
|
||||
may_reduce_scatterv=steps.output.may_reduce_scatterv,
|
||||
)
|
||||
|
||||
def _skips_sum_for_reduce_scatter(self, steps, dp_step: Optional[Callable]) -> bool:
|
||||
"""Whether the FFN leaves its sum out because a reduce-scatter completes
|
||||
it: the attention-DP one ``dp_step`` names, or the CP / input-scattered
|
||||
one."""
|
||||
def _sum_in_reduce_scatter(self, steps, dp_step: Optional[Callable]) -> bool:
|
||||
"""Whether a reduce-scatter completes the FFN output's sum: the
|
||||
attention-DP one ``dp_step`` names, or the CP / input-scattered one."""
|
||||
if dp_step is not None:
|
||||
return True
|
||||
if not steps.output.may_reduce_scatter:
|
||||
@@ -178,10 +177,10 @@ class ExitPolicy:
|
||||
return _sum_group(steps.output.group)
|
||||
|
||||
def _decide(self, forward_batch: ForwardBatch, steps) -> ExitDecision:
|
||||
"""Decide once, before the FFN runs, what it skips and what completes its
|
||||
output: the next layer's input, or this layer's postprocess step."""
|
||||
"""Decide once, before the FFN runs, what completes its output's sum:
|
||||
the next layer's input, or this layer's postprocess step."""
|
||||
dp_step = self._dp_reduce_scatter_step(forward_batch, steps)
|
||||
mlp_reduce_scatter = self._skips_sum_for_reduce_scatter(steps, dp_step)
|
||||
sum_in_reduce_scatter = self._sum_in_reduce_scatter(steps, dp_step)
|
||||
complete_now = partial(
|
||||
self._complete_now,
|
||||
forward_batch=forward_batch,
|
||||
@@ -189,7 +188,7 @@ class ExitPolicy:
|
||||
steps=steps,
|
||||
output_move=steps.output_move,
|
||||
# A reduce-scatter completes the sum on its way back.
|
||||
owes_sum=steps.output.group is not None and not mlp_reduce_scatter,
|
||||
owes_sum=steps.output.group is not None and not sum_in_reduce_scatter,
|
||||
)
|
||||
defer_moe_finalize = (
|
||||
self.plan.fusions is not None
|
||||
@@ -200,16 +199,18 @@ class ExitPolicy:
|
||||
):
|
||||
return ExitDecision(
|
||||
defer_moe_finalize=False,
|
||||
fuse_mlp_allreduce=False,
|
||||
mlp_reduce_scatter=mlp_reduce_scatter,
|
||||
sum_in_reduce_scatter=sum_in_reduce_scatter,
|
||||
complete=complete_now,
|
||||
)
|
||||
# Producers declare remaining work independently of the kernel chosen
|
||||
# by the consumer. Every handoff also carries an unfused completion.
|
||||
fuse_mlp_allreduce = defer_moe_finalize or self._defers_sum(
|
||||
forward_batch, steps, mlp_reduce_scatter=mlp_reduce_scatter, dp_step=dp_step
|
||||
defers = defer_moe_finalize or self._defers_sum(
|
||||
forward_batch,
|
||||
steps,
|
||||
sum_in_reduce_scatter=sum_in_reduce_scatter,
|
||||
dp_step=dp_step,
|
||||
)
|
||||
if fuse_mlp_allreduce:
|
||||
if defers:
|
||||
group = self.ffn_reduction_group(steps)
|
||||
if steps.returns_over_dp:
|
||||
# Under attention DP the next layer also brings the sum back to
|
||||
@@ -239,8 +240,7 @@ class ExitPolicy:
|
||||
complete = complete_now
|
||||
return ExitDecision(
|
||||
defer_moe_finalize=defer_moe_finalize,
|
||||
fuse_mlp_allreduce=fuse_mlp_allreduce,
|
||||
mlp_reduce_scatter=mlp_reduce_scatter,
|
||||
sum_in_reduce_scatter=sum_in_reduce_scatter,
|
||||
complete=complete,
|
||||
)
|
||||
|
||||
@@ -288,15 +288,16 @@ class ExitPolicy:
|
||||
self, forward_batch: ForwardBatch, *, stream: ResidualStream
|
||||
) -> MixerExit:
|
||||
"""Decide once whether this stage's mixer (an attention-like stage)
|
||||
skips its output all-reduce. Use the result as a context manager around
|
||||
the mixer, then call ``finish``."""
|
||||
output's sum is completed here or carried on. Call ``finish`` on the
|
||||
result with the mixer's output."""
|
||||
return MixerExit(self, forward_batch, stream=stream)
|
||||
|
||||
def ffn_exit(
|
||||
self, forward_batch: ForwardBatch, *, stream: ResidualStream
|
||||
) -> FfnExit:
|
||||
"""Decide once how this layer's FFN output reduction completes. Use the
|
||||
result as a context manager around the FFN call, then call ``finish``."""
|
||||
result as a context manager around the FFN call when the FFN may hand
|
||||
off its MoE finalize, then call ``finish``."""
|
||||
return FfnExit(self, forward_batch, stream=stream)
|
||||
|
||||
def _sum_deferral_allowed(self, steps) -> bool:
|
||||
@@ -331,7 +332,7 @@ class ExitPolicy:
|
||||
forward_batch: ForwardBatch,
|
||||
steps,
|
||||
*,
|
||||
mlp_reduce_scatter: bool,
|
||||
sum_in_reduce_scatter: bool,
|
||||
dp_step: Optional[Callable],
|
||||
) -> bool:
|
||||
"""Whether the FFN leaves its output's all-reduce to the next layer's
|
||||
@@ -342,7 +343,7 @@ class ExitPolicy:
|
||||
and not self.plan.terminal
|
||||
and self._sum_deferral_allowed(steps)
|
||||
and _batch_allows_deferred_sum(forward_batch, self.plan)
|
||||
and not mlp_reduce_scatter
|
||||
and not sum_in_reduce_scatter
|
||||
# Under attention DP the next layer must also run postprocess's
|
||||
# scatter back to this rank's tokens, and nothing more.
|
||||
and (
|
||||
@@ -392,9 +393,7 @@ class ExitDecision(msgspec.Struct, frozen=True):
|
||||
|
||||
Fields:
|
||||
defer_moe_finalize: Compute may return a producer-specific finalize handoff.
|
||||
fuse_mlp_allreduce: The sum is carried to the next consumer, which
|
||||
completes it, fused or unfused.
|
||||
mlp_reduce_scatter: A reduce-scatter on the way back completes the sum.
|
||||
sum_in_reduce_scatter: A reduce-scatter on the way back completes the sum.
|
||||
complete: Callable(output, residual) returning the completed or wrapped
|
||||
output and its corresponding residual rows.
|
||||
|
||||
@@ -402,8 +401,7 @@ class ExitDecision(msgspec.Struct, frozen=True):
|
||||
"""
|
||||
|
||||
defer_moe_finalize: bool
|
||||
fuse_mlp_allreduce: bool
|
||||
mlp_reduce_scatter: bool
|
||||
sum_in_reduce_scatter: bool
|
||||
complete: Callable[[torch.Tensor, torch.Tensor], Tuple]
|
||||
|
||||
|
||||
@@ -519,8 +517,8 @@ class FfnExit:
|
||||
self.defer_moe_finalize = completion.defer_moe_finalize
|
||||
self._complete = completion.complete
|
||||
self._update = steps.output.update
|
||||
self._declared_sum = boundary._sum_owed_after_skip(
|
||||
steps, completion.mlp_reduce_scatter
|
||||
self._declared_sum = boundary._sum_left_to_next_input(
|
||||
steps, completion.sum_in_reduce_scatter
|
||||
)
|
||||
self._scope = get_forward().scoped(defer_moe_finalize=self.defer_moe_finalize)
|
||||
|
||||
|
||||
@@ -82,7 +82,6 @@ def _resolve_ffn(
|
||||
read=NORM_READOUT,
|
||||
update=PLAIN_ADD,
|
||||
dense_tp_size=None,
|
||||
reduction=ProducerReduction.EXIT_SCOPED,
|
||||
):
|
||||
parallel = get_parallel()
|
||||
if dense_tp_size not in (None, 1, parallel.tp_size):
|
||||
@@ -169,8 +168,7 @@ def _resolve_ffn(
|
||||
and not terminal
|
||||
and not update.applied_at_exit
|
||||
and update.outlives_layer
|
||||
and output_transform is None
|
||||
and reduction is ProducerReduction.EXIT_SCOPED,
|
||||
and output_transform is None,
|
||||
may_reduce_scatter=use_reduce_scatter and may_scatter,
|
||||
may_reduce_scatterv=use_reduce_scatterv and may_leave,
|
||||
update=update,
|
||||
@@ -203,8 +201,8 @@ class StageDeclaration:
|
||||
residual update. An FFN's exit runs it under an explicit
|
||||
reduction-order contract; an attention's is run by the input of
|
||||
the stage that follows, once the attention's sum is complete.
|
||||
reduction: Whether compute always leaves a partial sum, obeys the
|
||||
exit scope, or adds a replicated component after its own sum.
|
||||
reduction: Whether the next stage's input always completes the sum
|
||||
(an attention's), or the exit decides (an FFN's or a mixer's).
|
||||
gathers_attn_tp_input: Whether attention gathers TP-sharded input itself.
|
||||
dense_tp_size: Dense FFN compute width: None uses the configured width,
|
||||
1 means local compute, and the full TP size means TP compute.
|
||||
@@ -282,9 +280,8 @@ def declare_attn(
|
||||
read: Input operation; defaults to normalization with quantization support.
|
||||
update: Operation that adds this stage's output to the residual.
|
||||
terminal: Whether this stage ends the model's layer stack.
|
||||
reduction: ALWAYS_PARTIAL for an output projection that always skips reduction;
|
||||
EXIT_SCOPED for a mixer that follows its exit scope's reduction decision.
|
||||
TAIL_AFTER_SUM is rejected for attention stages.
|
||||
reduction: ALWAYS_PARTIAL for an attention whose sum the next stage's
|
||||
input always completes; EXIT_SCOPED for a mixer whose exit decides.
|
||||
gathers_attn_tp_input: Whether compute gathers attention-TP input slices itself.
|
||||
output_transform: Optional operation on the output once its sum is
|
||||
complete, before the residual update (a sandwich norm). The next
|
||||
@@ -294,8 +291,6 @@ def declare_attn(
|
||||
Returns:
|
||||
A StageDeclaration with no norm, tensors or execution plan.
|
||||
"""
|
||||
if reduction is ProducerReduction.TAIL_AFTER_SUM:
|
||||
raise ValueError("TAIL_AFTER_SUM is not supported for attention stages")
|
||||
if (
|
||||
output_transform is not None
|
||||
and reduction is not ProducerReduction.ALWAYS_PARTIAL
|
||||
@@ -325,7 +320,6 @@ def declare_ffn(
|
||||
output_transform=None,
|
||||
next_layer_sparse=False,
|
||||
dense_tp_size=None,
|
||||
reduction=ProducerReduction.EXIT_SCOPED,
|
||||
exit_rows=None,
|
||||
):
|
||||
"""Declare a dense or MoE FFN independently of its compute module.
|
||||
@@ -342,17 +336,12 @@ def declare_ffn(
|
||||
to derive the TBO exit rows when exit_rows is not supplied.
|
||||
dense_tp_size: Dense compute width: None for configuration, 1 for local
|
||||
compute, or the full TP size.
|
||||
reduction: How compute cooperates with the exit's reduction decision:
|
||||
EXIT_SCOPED (default) follows the exit scope; TAIL_AFTER_SUM marks a
|
||||
replicated tail after the sum. ALWAYS_PARTIAL is rejected for FFN stages.
|
||||
exit_rows: Explicit output-row requirement; otherwise derived from
|
||||
the adjacent FFN kinds and TBO configuration.
|
||||
|
||||
Returns:
|
||||
A StageDeclaration with no norm, tensors or execution plan.
|
||||
"""
|
||||
if reduction is ProducerReduction.ALWAYS_PARTIAL:
|
||||
raise ValueError("ALWAYS_PARTIAL is not supported for ffn stages")
|
||||
return StageDeclaration(
|
||||
StageKind.FFN,
|
||||
read,
|
||||
@@ -363,7 +352,6 @@ def declare_ffn(
|
||||
terminal=terminal,
|
||||
output_transform=output_transform,
|
||||
dense_tp_size=dense_tp_size,
|
||||
reduction=reduction,
|
||||
exit_rows=exit_rows or tbo_exit_rows(sparse, next_layer_sparse),
|
||||
)
|
||||
|
||||
@@ -388,7 +376,6 @@ def _resolve_stage(stage, variant, following=None):
|
||||
read=stage.read,
|
||||
update=stage.update,
|
||||
dense_tp_size=stage.dense_tp_size,
|
||||
reduction=stage.reduction,
|
||||
)
|
||||
if resolve_exit_rows(stage.exit_rows) is ExitRows.ATTENTION:
|
||||
returned = attention
|
||||
|
||||
@@ -428,10 +428,9 @@ def moe_cp_take_back_output(
|
||||
):
|
||||
"""Return a MoE output computed on the MoE-CP-gathered rows to this rank's attention rows.
|
||||
|
||||
After moe_tensor_model_parallel_all_reduce (which runs unconditionally since
|
||||
mlp_reduce_scatter=False for this path), all ranks in the moe_cp group hold the
|
||||
full MoE result for all cp_per_moe token chunks. We simply slice out this rank's
|
||||
CP-local portion.
|
||||
Once the exit has completed the MoE output's sum, all ranks in the moe_cp
|
||||
group hold the full MoE result for all cp_per_moe token chunks. We simply
|
||||
slice out this rank's CP-local portion.
|
||||
|
||||
If DP>1, further scatter back to the local DP slice.
|
||||
"""
|
||||
|
||||
@@ -170,20 +170,26 @@ class StageBoundary:
|
||||
return hidden_states
|
||||
|
||||
def finish(self, hidden_states, forward_batch):
|
||||
"""Hand this attention's actual contribution to the following stage."""
|
||||
if self.kind is not StageKind.ATTENTION or not self.plan.finishes_directly:
|
||||
raise RuntimeError(
|
||||
"use the stage exit scope for an FFN or single-stage mixer"
|
||||
"""Hand this stage's output to the following stage: an attention's
|
||||
actual contribution, or an FFN's or a mixer's output with its sum
|
||||
completed or carried on. An FFN whose boundary has fusions (which may
|
||||
take a MoE finalize handoff) finishes through exit() instead."""
|
||||
if self.kind is StageKind.ATTENTION and self.plan.finishes_directly:
|
||||
produced = self.plan.produced(forward_batch)
|
||||
return stream_of(forward_batch).record(
|
||||
hidden_states,
|
||||
produced.update,
|
||||
declared_sum=produced.group if produced.always_partial else None,
|
||||
)
|
||||
produced = self.plan.produced(forward_batch)
|
||||
return stream_of(forward_batch).record(
|
||||
hidden_states,
|
||||
produced.update,
|
||||
declared_sum=produced.group if produced.always_partial else None,
|
||||
)
|
||||
if self.kind is StageKind.FFN and self.plan.fusions is not None:
|
||||
raise RuntimeError(
|
||||
"an FFN whose boundary has fusions finishes through exit()"
|
||||
)
|
||||
return self.exit(forward_batch).finish(hidden_states)
|
||||
|
||||
def exit(self, forward_batch):
|
||||
"""Select and scope one FFN or single-stage mixer's output decision.
|
||||
"""Select and scope one FFN or single-stage mixer's output decision,
|
||||
for an FFN whose MoE may hand off its finalize (see finish()).
|
||||
|
||||
Args:
|
||||
forward_batch: Active batch whose stream receives the compute result.
|
||||
|
||||
@@ -1661,9 +1661,9 @@ class RowParallelLinear(LinearBase):
|
||||
self, input_parallel, output_tensor, bias=bias_
|
||||
)
|
||||
|
||||
# skip_all_reduce: explicit call-site override. Also honor
|
||||
# ForwardFlags (fuse_mlp_allreduce / mlp_reduce_scatter) published by
|
||||
# the decoder — callers should not thread those flags into modules.
|
||||
# skip_all_reduce: explicit call-site override. Also honor the
|
||||
# mlp_reduce_scatter ForwardFlag published by the decoder — callers
|
||||
# should not thread it into modules.
|
||||
if (
|
||||
((self.reduce_results and self.tp_size > 1) or self.use_decode_attn_tp)
|
||||
and not skip_all_reduce
|
||||
|
||||
@@ -730,12 +730,10 @@ def should_use_dp_reduce_scatterv():
|
||||
def should_skip_mlp_all_reduce() -> bool:
|
||||
"""Whether dense MLP / row-parallel projections should skip their all-reduce.
|
||||
|
||||
True when the decoder published ``fuse_mlp_allreduce`` (next residual+LN
|
||||
absorbs the AR) or ``mlp_reduce_scatter`` (postprocess will reduce-scatter)
|
||||
on ``get_forward()``.
|
||||
True when the decoder published ``mlp_reduce_scatter`` (postprocess will
|
||||
reduce-scatter) on ``get_forward()``.
|
||||
"""
|
||||
f = get_forward()
|
||||
return f.fuse_mlp_allreduce or f.mlp_reduce_scatter
|
||||
return get_forward().mlp_reduce_scatter
|
||||
|
||||
|
||||
def post_experts_output_is_complete(*, is_tp_path: bool) -> bool:
|
||||
|
||||
@@ -295,9 +295,8 @@ class ApertusDecoderLayer(nn.Module):
|
||||
# Fully Connected
|
||||
hidden_states = self.attn_boundary.finish(hidden_states, forward_batch)
|
||||
hidden_states = self.ffn_boundary.prepare(hidden_states, forward_batch)
|
||||
with self.ffn_boundary.exit(forward_batch) as ffn_exit:
|
||||
hidden_states = self.mlp(hidden_states)
|
||||
return ffn_exit.finish(hidden_states)
|
||||
hidden_states = self.mlp(hidden_states)
|
||||
return self.ffn_boundary.finish(hidden_states, forward_batch)
|
||||
|
||||
|
||||
class ApertusModel(nn.Module):
|
||||
|
||||
@@ -271,9 +271,8 @@ class ArceeDecoderLayer(nn.Module):
|
||||
# Fully Connected
|
||||
hidden_states = self.attn_boundary.finish(hidden_states, forward_batch)
|
||||
hidden_states = self.ffn_boundary.prepare(hidden_states, forward_batch)
|
||||
with self.ffn_boundary.exit(forward_batch) as ffn_exit:
|
||||
hidden_states = self.mlp(hidden_states)
|
||||
return ffn_exit.finish(hidden_states)
|
||||
hidden_states = self.mlp(hidden_states)
|
||||
return self.ffn_boundary.finish(hidden_states, forward_batch)
|
||||
|
||||
|
||||
class ArceeModel(nn.Module):
|
||||
|
||||
@@ -748,9 +748,8 @@ class BailingMoEBlock(nn.Module):
|
||||
forward_batch=forward_batch,
|
||||
)
|
||||
|
||||
with self.ffn_boundary.exit(forward_batch) as ffn_exit:
|
||||
hidden_states = self.mlp(hidden_states, forward_batch)
|
||||
hidden_states = ffn_exit.finish(hidden_states)
|
||||
hidden_states = self.mlp(hidden_states, forward_batch)
|
||||
hidden_states = self.ffn_boundary.finish(hidden_states, forward_batch)
|
||||
|
||||
return hidden_states
|
||||
|
||||
|
||||
@@ -883,9 +883,8 @@ class BailingMoELinearDecoderLayer(nn.Module):
|
||||
# logger.warning(
|
||||
# f"===={self.layer_id=}, 3 shape= {hidden_states.shape}, {residual.shape}"
|
||||
# )
|
||||
with self.ffn_boundary.exit(forward_batch) as ffn_exit:
|
||||
hidden_states = self.mlp(hidden_states)
|
||||
hidden_states = ffn_exit.finish(hidden_states)
|
||||
hidden_states = self.mlp(hidden_states)
|
||||
hidden_states = self.ffn_boundary.finish(hidden_states, forward_batch)
|
||||
return hidden_states
|
||||
|
||||
@staticmethod
|
||||
|
||||
@@ -1196,17 +1196,16 @@ class BailingMoELinearDecoderLayer(nn.Module):
|
||||
hidden_states = self.attn_boundary.finish(hidden_states, forward_batch)
|
||||
hidden_states = self.ffn_boundary.prepare(hidden_states, forward_batch)
|
||||
|
||||
with self.ffn_boundary.exit(forward_batch) as ffn_exit:
|
||||
if not (
|
||||
is_dense_ffn_fully_dp()
|
||||
and (not self.is_layer_sparse)
|
||||
and hidden_states.shape[0] == 0
|
||||
):
|
||||
hidden_states = self.mlp(
|
||||
hidden_states,
|
||||
forward_batch=forward_batch,
|
||||
)
|
||||
hidden_states = ffn_exit.finish(hidden_states)
|
||||
if not (
|
||||
is_dense_ffn_fully_dp()
|
||||
and (not self.is_layer_sparse)
|
||||
and hidden_states.shape[0] == 0
|
||||
):
|
||||
hidden_states = self.mlp(
|
||||
hidden_states,
|
||||
forward_batch=forward_batch,
|
||||
)
|
||||
hidden_states = self.ffn_boundary.finish(hidden_states, forward_batch)
|
||||
|
||||
return hidden_states
|
||||
|
||||
|
||||
@@ -982,7 +982,6 @@ class DeepseekV2MoE(nn.Module):
|
||||
return dsv2_flashinfer_moe_dual_stream_graph(
|
||||
hidden_states,
|
||||
self.layer_id,
|
||||
fwd.fuse_mlp_allreduce,
|
||||
fwd.mlp_reduce_scatter,
|
||||
)
|
||||
elif (
|
||||
@@ -3535,7 +3534,6 @@ class DeepseekV32ForCausalLM(DeepseekV2ForCausalLM):
|
||||
def dsv2_flashinfer_moe_dual_stream_graph(
|
||||
hidden_states: torch.Tensor,
|
||||
layer_id: int,
|
||||
fuse_mlp_allreduce: bool,
|
||||
mlp_reduce_scatter: bool,
|
||||
) -> torch.Tensor:
|
||||
forward_context = get_tc_piecewise_forward_context()
|
||||
@@ -3548,7 +3546,6 @@ def dsv2_flashinfer_moe_dual_stream_graph(
|
||||
# torch.compile. Carry graph-varying control state as scalar operands and
|
||||
# republish it for the nested MoE/linear consumers.
|
||||
with get_forward().scoped(
|
||||
fuse_mlp_allreduce=fuse_mlp_allreduce,
|
||||
mlp_reduce_scatter=mlp_reduce_scatter,
|
||||
flashinfer_trtllm_bypass=True,
|
||||
lora_batch_layout=LoRABatchLayout.TP_GLOBAL,
|
||||
|
||||
@@ -1464,9 +1464,8 @@ class Dots3DecoderLayer(nn.Module):
|
||||
hidden_states = self.attn_boundary.finish(hidden_states, forward_batch)
|
||||
hidden_states = self.ffn_boundary.prepare(hidden_states, forward_batch)
|
||||
|
||||
with self.ffn_boundary.exit(forward_batch) as ffn_exit:
|
||||
hidden_states = self.mlp(hidden_states, forward_batch)
|
||||
return ffn_exit.finish(hidden_states)
|
||||
hidden_states = self.mlp(hidden_states, forward_batch)
|
||||
return self.ffn_boundary.finish(hidden_states, forward_batch)
|
||||
|
||||
|
||||
class Dots3Model(nn.Module):
|
||||
|
||||
@@ -472,12 +472,11 @@ class Ernie4_5_VLMoeDecoderLayer(nn.Module):
|
||||
# Fully Connected
|
||||
hidden_states = self.attn_boundary.finish(hidden_states, forward_batch)
|
||||
hidden_states = self.ffn_boundary.prepare(hidden_states, forward_batch)
|
||||
with self.ffn_boundary.exit(forward_batch) as ffn_exit:
|
||||
if isinstance(self.mlp, Ernie4_5_VLMoeMoE):
|
||||
hidden_states = self.mlp(hidden_states, visual_token_mask, **kwargs)
|
||||
else:
|
||||
hidden_states = self.mlp(hidden_states)
|
||||
return ffn_exit.finish(hidden_states)
|
||||
if isinstance(self.mlp, Ernie4_5_VLMoeMoE):
|
||||
hidden_states = self.mlp(hidden_states, visual_token_mask, **kwargs)
|
||||
else:
|
||||
hidden_states = self.mlp(hidden_states)
|
||||
return self.ffn_boundary.finish(hidden_states, forward_batch)
|
||||
|
||||
|
||||
# only used as text backbone for ernie4.5 vl
|
||||
|
||||
@@ -313,9 +313,8 @@ class Exaone4DecoderLayer(nn.Module):
|
||||
# Fully Connected
|
||||
hidden_states = self.attn_boundary.finish(hidden_states, forward_batch)
|
||||
hidden_states = self.ffn_boundary.prepare(hidden_states, forward_batch)
|
||||
with self.ffn_boundary.exit(forward_batch) as ffn_exit:
|
||||
hidden_states = self.mlp(hidden_states)
|
||||
return ffn_exit.finish(hidden_states)
|
||||
hidden_states = self.mlp(hidden_states)
|
||||
return self.ffn_boundary.finish(hidden_states, forward_batch)
|
||||
|
||||
|
||||
class Exaone4Model(nn.Module):
|
||||
|
||||
@@ -533,9 +533,8 @@ class ExaoneMoEDecoderLayer(nn.Module):
|
||||
hidden_states = self.attn_boundary.finish(hidden_states, forward_batch)
|
||||
hidden_states = self.ffn_boundary.prepare(hidden_states, forward_batch)
|
||||
# Fully Connected
|
||||
with self.ffn_boundary.exit(forward_batch) as ffn_exit:
|
||||
hidden_states = self.mlp(hidden_states, forward_batch)
|
||||
return ffn_exit.finish(hidden_states)
|
||||
hidden_states = self.mlp(hidden_states, forward_batch)
|
||||
return self.ffn_boundary.finish(hidden_states, forward_batch)
|
||||
|
||||
|
||||
class ExaoneMoEModel(nn.Module):
|
||||
|
||||
@@ -366,9 +366,8 @@ class FalconH1HybridAttentionDecoderLayer(nn.Module):
|
||||
# Fully Connected
|
||||
hidden_states = self.attn_boundary.finish(hidden_states, forward_batch)
|
||||
hidden_states = self.ffn_boundary.prepare(hidden_states, forward_batch)
|
||||
with self.ffn_boundary.exit(forward_batch) as ffn_exit:
|
||||
hidden_states = self.feed_forward(hidden_states, forward_batch)
|
||||
return ffn_exit.finish(hidden_states)
|
||||
hidden_states = self.feed_forward(hidden_states, forward_batch)
|
||||
return self.ffn_boundary.finish(hidden_states, forward_batch)
|
||||
|
||||
|
||||
ALL_DECODER_LAYER_TYPES = {
|
||||
|
||||
@@ -719,11 +719,10 @@ class Gemma4DecoderLayer(nn.Module):
|
||||
hidden_states = self.attn_boundary.finish(hidden_states, forward_batch)
|
||||
hidden_states = self.ffn_boundary.prepare(hidden_states, forward_batch)
|
||||
residual = residual_batch.written_residual(forward_batch)
|
||||
with self.ffn_boundary.exit(forward_batch) as ffn_exit:
|
||||
hidden_states = self._next_stream(
|
||||
hidden_states, residual, per_layer_input, forward_batch
|
||||
)
|
||||
return ffn_exit.finish(hidden_states)
|
||||
hidden_states = self._next_stream(
|
||||
hidden_states, residual, per_layer_input, forward_batch
|
||||
)
|
||||
return self.ffn_boundary.finish(hidden_states, forward_batch)
|
||||
|
||||
def _next_stream(
|
||||
self,
|
||||
|
||||
@@ -302,9 +302,8 @@ class Glm4DecoderLayer(nn.Module):
|
||||
# Fully Connected
|
||||
hidden_states = self.attn_boundary.finish(hidden_states, forward_batch)
|
||||
hidden_states = self.ffn_boundary.prepare(hidden_states, forward_batch)
|
||||
with self.ffn_boundary.exit(forward_batch) as ffn_exit:
|
||||
hidden_states = self.mlp(hidden_states)
|
||||
return ffn_exit.finish(hidden_states)
|
||||
hidden_states = self.mlp(hidden_states)
|
||||
return self.ffn_boundary.finish(hidden_states, forward_batch)
|
||||
|
||||
|
||||
class Glm4Model(nn.Module):
|
||||
|
||||
@@ -865,9 +865,8 @@ class Glm4MoeDecoderLayer(nn.Module):
|
||||
hidden_states = self.attn_boundary.finish(hidden_states, forward_batch)
|
||||
hidden_states = self.ffn_boundary.prepare(hidden_states, forward_batch)
|
||||
|
||||
with self.ffn_boundary.exit(forward_batch) as ffn_exit:
|
||||
hidden_states = self.mlp(hidden_states, forward_batch)
|
||||
hidden_states = ffn_exit.finish(hidden_states)
|
||||
hidden_states = self.mlp(hidden_states, forward_batch)
|
||||
hidden_states = self.ffn_boundary.finish(hidden_states, forward_batch)
|
||||
|
||||
return hidden_states
|
||||
|
||||
|
||||
@@ -556,9 +556,8 @@ class Glm4MoeLiteDecoderLayer(nn.Module):
|
||||
hidden_states = self.attn_boundary.finish(hidden_states, forward_batch)
|
||||
hidden_states = self.ffn_boundary.prepare(hidden_states, forward_batch)
|
||||
|
||||
with self.ffn_boundary.exit(forward_batch) as ffn_exit:
|
||||
hidden_states = self.mlp(hidden_states, forward_batch)
|
||||
hidden_states = ffn_exit.finish(hidden_states)
|
||||
hidden_states = self.mlp(hidden_states, forward_batch)
|
||||
hidden_states = self.ffn_boundary.finish(hidden_states, forward_batch)
|
||||
|
||||
return hidden_states
|
||||
|
||||
|
||||
@@ -964,13 +964,13 @@ class Glm5NextDecoderLayer(nn.Module):
|
||||
else:
|
||||
_mlp_ctx = nullcontext()
|
||||
|
||||
with self.ffn_boundary.exit(forward_batch) as ffn_exit, _mlp_ctx:
|
||||
with _mlp_ctx:
|
||||
hidden_states = self.mlp(
|
||||
hidden_states,
|
||||
forward_batch,
|
||||
gemm_output_zero_allocator,
|
||||
)
|
||||
hidden_states = ffn_exit.finish(hidden_states)
|
||||
hidden_states = self.ffn_boundary.finish(hidden_states, forward_batch)
|
||||
|
||||
return (hidden_states, topk_indices)
|
||||
|
||||
|
||||
@@ -947,9 +947,8 @@ class GlmImageTextDecoderLayer(nn.Module):
|
||||
# Fully Connected
|
||||
hidden_states = self.attn_boundary.finish(hidden_states, forward_batch)
|
||||
hidden_states = self.ffn_boundary.prepare(hidden_states, forward_batch)
|
||||
with self.ffn_boundary.exit(forward_batch) as ffn_exit:
|
||||
hidden_states = self.mlp(hidden_states)
|
||||
return ffn_exit.finish(hidden_states)
|
||||
hidden_states = self.mlp(hidden_states)
|
||||
return self.ffn_boundary.finish(hidden_states, forward_batch)
|
||||
|
||||
|
||||
class GlmImageTextModel(nn.Module):
|
||||
|
||||
@@ -629,9 +629,8 @@ class GptOssDecoderLayer(nn.Module):
|
||||
hidden_states = self.attn_boundary.finish(hidden_states, forward_batch)
|
||||
hidden_states = self.ffn_boundary.prepare(hidden_states, forward_batch)
|
||||
|
||||
with self.ffn_boundary.exit(forward_batch) as ffn_exit:
|
||||
hidden_states = self.mlp(hidden_states, forward_batch)
|
||||
hidden_states = ffn_exit.finish(hidden_states)
|
||||
hidden_states = self.mlp(hidden_states, forward_batch)
|
||||
hidden_states = self.ffn_boundary.finish(hidden_states, forward_batch)
|
||||
|
||||
return hidden_states
|
||||
|
||||
|
||||
@@ -155,21 +155,20 @@ class GraniteMoeHybridMambaDecoderLayer(nn.Module):
|
||||
|
||||
hidden_states = self.attn_boundary.finish(output, forward_batch)
|
||||
hidden_states = self.ffn_boundary.prepare(hidden_states, forward_batch)
|
||||
with self.ffn_boundary.exit(forward_batch) as ffn_exit:
|
||||
if self.shared_mlp is None:
|
||||
if self.block_sparse_moe is not None:
|
||||
hidden_states = self.block_sparse_moe(hidden_states)
|
||||
# else: skip
|
||||
if self.shared_mlp is None:
|
||||
if self.block_sparse_moe is not None:
|
||||
hidden_states = self.block_sparse_moe(hidden_states)
|
||||
# else: skip
|
||||
else:
|
||||
# create a copy since block_sparse_moe modifies in-place
|
||||
if self.block_sparse_moe is not None:
|
||||
moe_hidden_states = hidden_states.clone()
|
||||
moe_hidden_states = self.block_sparse_moe(moe_hidden_states)
|
||||
hidden_states = moe_hidden_states + self.shared_mlp(hidden_states)
|
||||
del moe_hidden_states
|
||||
else:
|
||||
# create a copy since block_sparse_moe modifies in-place
|
||||
if self.block_sparse_moe is not None:
|
||||
moe_hidden_states = hidden_states.clone()
|
||||
moe_hidden_states = self.block_sparse_moe(moe_hidden_states)
|
||||
hidden_states = moe_hidden_states + self.shared_mlp(hidden_states)
|
||||
del moe_hidden_states
|
||||
else:
|
||||
hidden_states = self.shared_mlp(hidden_states)
|
||||
return ffn_exit.finish(hidden_states)
|
||||
hidden_states = self.shared_mlp(hidden_states)
|
||||
return self.ffn_boundary.finish(hidden_states, forward_batch)
|
||||
|
||||
|
||||
class GraniteMoeHybridAttention(nn.Module):
|
||||
@@ -339,21 +338,20 @@ class GraniteMoeHybridAttentionDecoderLayer(nn.Module):
|
||||
)
|
||||
hidden_states = self.attn_boundary.finish(hidden_states, forward_batch)
|
||||
hidden_states = self.ffn_boundary.prepare(hidden_states, forward_batch)
|
||||
with self.ffn_boundary.exit(forward_batch) as ffn_exit:
|
||||
if self.shared_mlp is None:
|
||||
if self.block_sparse_moe is not None:
|
||||
hidden_states = self.block_sparse_moe(hidden_states)
|
||||
# else: skip
|
||||
if self.shared_mlp is None:
|
||||
if self.block_sparse_moe is not None:
|
||||
hidden_states = self.block_sparse_moe(hidden_states)
|
||||
# else: skip
|
||||
else:
|
||||
# create a copy since block_sparse_moe modifies in-place
|
||||
if self.block_sparse_moe is not None:
|
||||
moe_hidden_states = hidden_states.clone()
|
||||
moe_hidden_states = self.block_sparse_moe(moe_hidden_states)
|
||||
hidden_states = moe_hidden_states + self.shared_mlp(hidden_states)
|
||||
del moe_hidden_states
|
||||
else:
|
||||
# create a copy since block_sparse_moe modifies in-place
|
||||
if self.block_sparse_moe is not None:
|
||||
moe_hidden_states = hidden_states.clone()
|
||||
moe_hidden_states = self.block_sparse_moe(moe_hidden_states)
|
||||
hidden_states = moe_hidden_states + self.shared_mlp(hidden_states)
|
||||
del moe_hidden_states
|
||||
else:
|
||||
hidden_states = self.shared_mlp(hidden_states)
|
||||
return ffn_exit.finish(hidden_states)
|
||||
hidden_states = self.shared_mlp(hidden_states)
|
||||
return self.ffn_boundary.finish(hidden_states, forward_batch)
|
||||
|
||||
|
||||
ALL_DECODER_LAYER_TYPES = {
|
||||
|
||||
@@ -429,11 +429,8 @@ class _InternS2MobiusDecoderMixin:
|
||||
) -> torch.Tensor:
|
||||
hidden_states = self.attn_boundary.finish(hidden_states, forward_batch)
|
||||
hidden_states = self.ffn_boundary.prepare(hidden_states, forward_batch)
|
||||
with self.ffn_boundary.exit(forward_batch) as ffn_exit:
|
||||
hidden_states = self._forward_mobius_mlp(
|
||||
hidden_states, forward_batch, meta_mlp
|
||||
)
|
||||
hidden_states = ffn_exit.finish(hidden_states)
|
||||
hidden_states = self._forward_mobius_mlp(hidden_states, forward_batch, meta_mlp)
|
||||
hidden_states = self.ffn_boundary.finish(hidden_states, forward_batch)
|
||||
return hidden_states
|
||||
|
||||
|
||||
|
||||
@@ -532,9 +532,8 @@ class IQuestQ1DecoderLayer(nn.Module):
|
||||
attn_output = self.self_attn(positions, hidden_states, forward_batch)
|
||||
hidden_states = self.attn_boundary.finish(attn_output, forward_batch)
|
||||
hidden_states = self.ffn_boundary.prepare(hidden_states, forward_batch)
|
||||
with self.ffn_boundary.exit(forward_batch) as ffn_exit:
|
||||
mlp_output = self.mlp(hidden_states)
|
||||
return ffn_exit.finish(mlp_output)
|
||||
mlp_output = self.mlp(hidden_states)
|
||||
return self.ffn_boundary.finish(mlp_output, forward_batch)
|
||||
|
||||
|
||||
class IQuestQ1Model(nn.Module):
|
||||
|
||||
@@ -652,9 +652,8 @@ class KimiDecoderLayer(nn.Module):
|
||||
# Fully Connected
|
||||
hidden_states = self.attn_boundary.finish(hidden_states, forward_batch)
|
||||
hidden_states = self.ffn_boundary.prepare(hidden_states, forward_batch)
|
||||
with self.ffn_boundary.exit(forward_batch) as ffn_exit:
|
||||
hidden_states = self.mlp(hidden_states)
|
||||
return ffn_exit.finish(hidden_states)
|
||||
hidden_states = self.mlp(hidden_states)
|
||||
return self.ffn_boundary.finish(hidden_states, forward_batch)
|
||||
|
||||
|
||||
class KimiLinearModel(nn.Module):
|
||||
|
||||
@@ -104,8 +104,6 @@ class LagunaMLP(nn.Module):
|
||||
) -> torch.Tensor:
|
||||
gate_up, _ = self.gate_up_proj(x)
|
||||
x = self.act_fn(gate_up)
|
||||
# RowParallelLinear honors ForwardFlags (fuse_mlp_allreduce /
|
||||
# mlp_reduce_scatter) published by the decoder via scoped().
|
||||
x, _ = self.down_proj(x)
|
||||
return x
|
||||
|
||||
@@ -491,12 +489,11 @@ class LagunaDecoderLayer(nn.Module):
|
||||
hidden_states = self.attn_boundary.finish(hidden_states, forward_batch)
|
||||
hidden_states = self.ffn_boundary.prepare(hidden_states, forward_batch)
|
||||
|
||||
with self.ffn_boundary.exit(forward_batch) as ffn_exit:
|
||||
hidden_states = self.mlp(
|
||||
hidden_states,
|
||||
forward_batch=forward_batch,
|
||||
)
|
||||
hidden_states = ffn_exit.finish(hidden_states)
|
||||
hidden_states = self.mlp(
|
||||
hidden_states,
|
||||
forward_batch=forward_batch,
|
||||
)
|
||||
hidden_states = self.ffn_boundary.finish(hidden_states, forward_batch)
|
||||
return hidden_states
|
||||
|
||||
|
||||
|
||||
@@ -675,9 +675,8 @@ class LLaDA2MoeBlock(nn.Module):
|
||||
forward_batch=forward_batch,
|
||||
)
|
||||
|
||||
with self.ffn_boundary.exit(forward_batch) as ffn_exit:
|
||||
hidden_states = self.mlp(hidden_states, forward_batch)
|
||||
hidden_states = ffn_exit.finish(hidden_states)
|
||||
hidden_states = self.mlp(hidden_states, forward_batch)
|
||||
hidden_states = self.ffn_boundary.finish(hidden_states, forward_batch)
|
||||
|
||||
return hidden_states
|
||||
|
||||
|
||||
@@ -394,9 +394,8 @@ class LlamaDecoderLayer(nn.Module):
|
||||
# Fully Connected
|
||||
hidden_states = self.attn_boundary.finish(hidden_states, forward_batch)
|
||||
hidden_states = self.ffn_boundary.prepare(hidden_states, forward_batch)
|
||||
with self.ffn_boundary.exit(forward_batch) as ffn_exit:
|
||||
hidden_states = self.mlp(hidden_states)
|
||||
return ffn_exit.finish(hidden_states)
|
||||
hidden_states = self.mlp(hidden_states)
|
||||
return self.ffn_boundary.finish(hidden_states, forward_batch)
|
||||
|
||||
|
||||
class LlamaModel(nn.Module):
|
||||
|
||||
@@ -472,9 +472,8 @@ class Llama4DecoderLayer(nn.Module):
|
||||
hidden_states = self.ffn_boundary.prepare(hidden_states, forward_batch)
|
||||
|
||||
# Fully Connected
|
||||
with self.ffn_boundary.exit(forward_batch) as ffn_exit:
|
||||
hidden_states = self.feed_forward(hidden_states, forward_batch)
|
||||
hidden_states = ffn_exit.finish(hidden_states)
|
||||
hidden_states = self.feed_forward(hidden_states, forward_batch)
|
||||
hidden_states = self.ffn_boundary.finish(hidden_states, forward_batch)
|
||||
|
||||
return hidden_states
|
||||
|
||||
|
||||
@@ -504,9 +504,8 @@ class LongcatFlashDecoderLayer(nn.Module):
|
||||
hidden_states = self.first_ffn_boundary.branch_input(
|
||||
self.moe_boundary, hidden_states, forward_batch
|
||||
)
|
||||
with self.first_ffn_boundary.exit(forward_batch) as ffn_exit:
|
||||
hidden_states = self.mlps[0](hidden_states)
|
||||
hidden_states = ffn_exit.finish(hidden_states)
|
||||
hidden_states = self.mlps[0](hidden_states)
|
||||
hidden_states = self.first_ffn_boundary.finish(hidden_states, forward_batch)
|
||||
|
||||
# second_attn_boundary
|
||||
hidden_states = self.second_attn_boundary.prepare(hidden_states, forward_batch)
|
||||
@@ -526,9 +525,8 @@ class LongcatFlashDecoderLayer(nn.Module):
|
||||
# second_mlp
|
||||
hidden_states = self.second_attn_boundary.finish(hidden_states, forward_batch)
|
||||
hidden_states = self.second_ffn_boundary.prepare(hidden_states, forward_batch)
|
||||
with self.second_ffn_boundary.exit(forward_batch) as ffn_exit:
|
||||
hidden_states = self.mlps[1](hidden_states)
|
||||
hidden_states = ffn_exit.finish(hidden_states)
|
||||
hidden_states = self.mlps[1](hidden_states)
|
||||
hidden_states = self.second_ffn_boundary.finish(hidden_states, forward_batch)
|
||||
|
||||
return hidden_states, prev_topk_indices
|
||||
|
||||
|
||||
@@ -191,9 +191,8 @@ class LongcatFlashDenseDecoderLayer(nn.Module):
|
||||
|
||||
hidden_states = self.attn_boundary.finish(hidden_states, forward_batch)
|
||||
hidden_states = self.ffn_boundary.prepare(hidden_states, forward_batch)
|
||||
with self.ffn_boundary.exit(forward_batch) as ffn_exit:
|
||||
hidden_states = self.mlp(hidden_states)
|
||||
hidden_states = ffn_exit.finish(hidden_states)
|
||||
hidden_states = self.mlp(hidden_states)
|
||||
hidden_states = self.ffn_boundary.finish(hidden_states, forward_batch)
|
||||
return hidden_states
|
||||
|
||||
|
||||
|
||||
@@ -898,9 +898,8 @@ class MiMoV2DecoderLayer(nn.Module):
|
||||
hidden_states = self.attn_boundary.finish(hidden_states, forward_batch)
|
||||
hidden_states = self.ffn_boundary.prepare(hidden_states, forward_batch)
|
||||
|
||||
with self.ffn_boundary.exit(forward_batch) as ffn_exit:
|
||||
hidden_states = self.mlp(hidden_states, forward_batch)
|
||||
hidden_states = ffn_exit.finish(hidden_states)
|
||||
hidden_states = self.mlp(hidden_states, forward_batch)
|
||||
hidden_states = self.ffn_boundary.finish(hidden_states, forward_batch)
|
||||
|
||||
return hidden_states
|
||||
|
||||
|
||||
@@ -155,12 +155,9 @@ class MiMoV2MTPLayer(nn.Module):
|
||||
|
||||
hidden_states = self.attn_boundary.finish(hidden_states, forward_batch)
|
||||
hidden_states = self.ffn_boundary.prepare(hidden_states, forward_batch)
|
||||
with (
|
||||
self.ffn_boundary.exit(forward_batch) as ffn_exit,
|
||||
get_global_expert_distribution_recorder().disable_this_region(),
|
||||
):
|
||||
with get_global_expert_distribution_recorder().disable_this_region():
|
||||
hidden_states = self.mlp(hidden_states)
|
||||
hidden_states = ffn_exit.finish(hidden_states)
|
||||
hidden_states = self.ffn_boundary.finish(hidden_states, forward_batch)
|
||||
|
||||
return hidden_states
|
||||
|
||||
|
||||
@@ -912,9 +912,8 @@ class MiniMaxM2DecoderLayer(nn.Module):
|
||||
hidden_states = self.attn_boundary.finish(hidden_states, forward_batch)
|
||||
hidden_states = self.ffn_boundary.prepare(hidden_states, forward_batch)
|
||||
|
||||
with self.ffn_boundary.exit(forward_batch) as ffn_exit:
|
||||
hidden_states = self.block_sparse_moe(hidden_states, forward_batch)
|
||||
hidden_states = ffn_exit.finish(hidden_states)
|
||||
hidden_states = self.block_sparse_moe(hidden_states, forward_batch)
|
||||
hidden_states = self.ffn_boundary.finish(hidden_states, forward_batch)
|
||||
|
||||
return hidden_states
|
||||
|
||||
|
||||
@@ -1374,10 +1374,9 @@ class MiniMaxM3DecoderLayer(nn.Module):
|
||||
hidden_states = self.attn_boundary.finish(hidden_states, forward_batch)
|
||||
hidden_states = self.ffn_boundary.prepare(hidden_states, forward_batch)
|
||||
|
||||
with self.ffn_boundary.exit(forward_batch) as ffn_exit:
|
||||
if self.is_layer_sparse or hidden_states.shape[0] != 0:
|
||||
hidden_states = self.mlp(hidden_states, forward_batch=forward_batch)
|
||||
return ffn_exit.finish(hidden_states)
|
||||
if self.is_layer_sparse or hidden_states.shape[0] != 0:
|
||||
hidden_states = self.mlp(hidden_states, forward_batch=forward_batch)
|
||||
return self.ffn_boundary.finish(hidden_states, forward_batch)
|
||||
|
||||
|
||||
class MiniMaxM3Model(nn.Module):
|
||||
|
||||
@@ -264,9 +264,8 @@ class MixtralDecoderLayer(nn.Module):
|
||||
# Fully Connected
|
||||
hidden_states = self.attn_boundary.finish(hidden_states, forward_batch)
|
||||
hidden_states = self.ffn_boundary.prepare(hidden_states, forward_batch)
|
||||
with self.ffn_boundary.exit(forward_batch) as ffn_exit:
|
||||
hidden_states = self.block_sparse_moe(hidden_states)
|
||||
return ffn_exit.finish(hidden_states)
|
||||
hidden_states = self.block_sparse_moe(hidden_states)
|
||||
return self.ffn_boundary.finish(hidden_states, forward_batch)
|
||||
|
||||
|
||||
class MixtralModel(nn.Module):
|
||||
|
||||
@@ -1045,9 +1045,8 @@ class MossVLSelfAttentionDecoderLayer(nn.Module):
|
||||
# MLP
|
||||
hidden_states = self.attn_boundary.finish(hidden_states, forward_batch)
|
||||
hidden_states = self.ffn_boundary.prepare(hidden_states, forward_batch)
|
||||
with self.ffn_boundary.exit(forward_batch) as ffn_exit:
|
||||
hidden_states = self.mlp(hidden_states)
|
||||
hidden_states = ffn_exit.finish(hidden_states)
|
||||
hidden_states = self.mlp(hidden_states)
|
||||
hidden_states = self.ffn_boundary.finish(hidden_states, forward_batch)
|
||||
return hidden_states
|
||||
|
||||
|
||||
|
||||
@@ -262,9 +262,8 @@ class NanbeigeDecoderLayer(nn.Module):
|
||||
# Fully Connected
|
||||
hidden_states = self.attn_boundary.finish(hidden_states, forward_batch)
|
||||
hidden_states = self.ffn_boundary.prepare(hidden_states, forward_batch)
|
||||
with self.ffn_boundary.exit(forward_batch) as ffn_exit:
|
||||
hidden_states = self.mlp(hidden_states)
|
||||
return ffn_exit.finish(hidden_states)
|
||||
hidden_states = self.mlp(hidden_states)
|
||||
return self.ffn_boundary.finish(hidden_states, forward_batch)
|
||||
|
||||
|
||||
class NanbeigeModel(nn.Module):
|
||||
|
||||
@@ -384,9 +384,8 @@ class NemotronHMLPLikeDecoderLayer(nn.Module):
|
||||
forward_batch: ForwardBatch,
|
||||
):
|
||||
hidden_states = self.boundary.prepare(hidden_states, forward_batch)
|
||||
with self.boundary.exit(forward_batch) as ffn_exit:
|
||||
hidden_states = self.mixer.forward(hidden_states)
|
||||
return ffn_exit.finish(hidden_states)
|
||||
hidden_states = self.mixer.forward(hidden_states)
|
||||
return self.boundary.finish(hidden_states, forward_batch)
|
||||
|
||||
|
||||
class NemotronHMLPDecoderLayer(NemotronHMLPLikeDecoderLayer):
|
||||
@@ -467,9 +466,8 @@ class NemotronHAttnLikeDecoderLayer(nn.Module):
|
||||
if forward_batch.forward_mode.is_idle():
|
||||
return hidden_states
|
||||
|
||||
with self.boundary.exit(forward_batch) as mixer_exit:
|
||||
hidden_states = self._forward_mixer(hidden_states, forward_batch)
|
||||
return mixer_exit.finish(hidden_states)
|
||||
hidden_states = self._forward_mixer(hidden_states, forward_batch)
|
||||
return self.boundary.finish(hidden_states, forward_batch)
|
||||
|
||||
|
||||
class NemotronHMambaDecoderLayer(NemotronHAttnLikeDecoderLayer):
|
||||
|
||||
@@ -194,9 +194,8 @@ class DeciLMDecoderLayer(nn.Module):
|
||||
# Fully Connected
|
||||
if not self._is_no_op_ffn:
|
||||
hidden_states = self.ffn_boundary.prepare(hidden_states, forward_batch)
|
||||
with self.ffn_boundary.exit(forward_batch) as ffn_exit:
|
||||
hidden_states = self.mlp(hidden_states)
|
||||
hidden_states = ffn_exit.finish(hidden_states)
|
||||
hidden_states = self.mlp(hidden_states)
|
||||
hidden_states = self.ffn_boundary.finish(hidden_states, forward_batch)
|
||||
return hidden_states
|
||||
|
||||
|
||||
|
||||
@@ -325,9 +325,8 @@ class Qwen2DecoderLayer(nn.Module):
|
||||
# Fully Connected
|
||||
hidden_states = self.attn_boundary.finish(hidden_states, forward_batch)
|
||||
hidden_states = self.ffn_boundary.prepare(hidden_states, forward_batch)
|
||||
with self.ffn_boundary.exit(forward_batch) as ffn_exit:
|
||||
hidden_states = self.mlp(hidden_states)
|
||||
return ffn_exit.finish(hidden_states)
|
||||
hidden_states = self.mlp(hidden_states)
|
||||
return self.ffn_boundary.finish(hidden_states, forward_batch)
|
||||
|
||||
|
||||
class Qwen2Model(nn.Module):
|
||||
|
||||
@@ -1095,9 +1095,8 @@ class Qwen2MoeDecoderLayer(nn.Module):
|
||||
hidden_states = self.attn_boundary.finish(hidden_states, forward_batch)
|
||||
hidden_states = self.ffn_boundary.prepare(hidden_states, forward_batch)
|
||||
|
||||
with self.ffn_boundary.exit(forward_batch) as ffn_exit:
|
||||
hidden_states = self.mlp(hidden_states, forward_batch)
|
||||
hidden_states = ffn_exit.finish(hidden_states)
|
||||
hidden_states = self.mlp(hidden_states, forward_batch)
|
||||
hidden_states = self.ffn_boundary.finish(hidden_states, forward_batch)
|
||||
|
||||
return hidden_states
|
||||
|
||||
|
||||
@@ -423,11 +423,10 @@ class Qwen3DecoderLayer(nn.Module):
|
||||
else None
|
||||
),
|
||||
)
|
||||
with self.ffn_boundary.exit(forward_batch) as ffn_exit:
|
||||
hidden_states = self.mlp(hidden_states, forward_batch=forward_batch)
|
||||
hidden_states = self.mlp(hidden_states, forward_batch=forward_batch)
|
||||
if _is_npu and get_cmo_stream():
|
||||
wait_cmo_stream()
|
||||
hidden_states = ffn_exit.finish(hidden_states)
|
||||
hidden_states = self.ffn_boundary.finish(hidden_states, forward_batch)
|
||||
return hidden_states
|
||||
|
||||
|
||||
|
||||
@@ -904,9 +904,8 @@ class Qwen3MoeDecoderLayer(nn.Module):
|
||||
hidden_states = self.attn_boundary.finish(hidden_states, forward_batch)
|
||||
hidden_states = self.ffn_boundary.prepare(hidden_states, forward_batch)
|
||||
|
||||
with self.ffn_boundary.exit(forward_batch) as ffn_exit:
|
||||
hidden_states = self.mlp(hidden_states, forward_batch)
|
||||
hidden_states = ffn_exit.finish(hidden_states)
|
||||
hidden_states = self.mlp(hidden_states, forward_batch)
|
||||
hidden_states = self.ffn_boundary.finish(hidden_states, forward_batch)
|
||||
|
||||
return hidden_states
|
||||
|
||||
|
||||
@@ -479,15 +479,14 @@ def _apply_qwen3_next_mlp(
|
||||
) -> torch.Tensor:
|
||||
hidden_states = layer.attn_boundary.finish(hidden_states, forward_batch)
|
||||
hidden_states = layer.ffn_boundary.prepare(hidden_states, forward_batch)
|
||||
with layer.ffn_boundary.exit(forward_batch) as ffn_exit:
|
||||
if isinstance(layer.mlp, Qwen2MoeSparseMoeBlock):
|
||||
hidden_states = layer.mlp(
|
||||
hidden_states,
|
||||
forward_batch=forward_batch,
|
||||
)
|
||||
else:
|
||||
hidden_states = layer.mlp(hidden_states)
|
||||
hidden_states = ffn_exit.finish(hidden_states)
|
||||
if isinstance(layer.mlp, Qwen2MoeSparseMoeBlock):
|
||||
hidden_states = layer.mlp(
|
||||
hidden_states,
|
||||
forward_batch=forward_batch,
|
||||
)
|
||||
else:
|
||||
hidden_states = layer.mlp(hidden_states)
|
||||
hidden_states = layer.ffn_boundary.finish(hidden_states, forward_batch)
|
||||
|
||||
return hidden_states
|
||||
|
||||
|
||||
@@ -1080,9 +1080,8 @@ class SarvamMoEMLADecoderLayer(nn.Module):
|
||||
)
|
||||
hidden_states = self.attn_boundary.finish(hidden_states, forward_batch)
|
||||
hidden_states = self.ffn_boundary.prepare(hidden_states, forward_batch)
|
||||
with self.ffn_boundary.exit(forward_batch) as ffn_exit:
|
||||
hidden_states = self.mlp(hidden_states, forward_batch)
|
||||
hidden_states = ffn_exit.finish(hidden_states)
|
||||
hidden_states = self.mlp(hidden_states, forward_batch)
|
||||
hidden_states = self.ffn_boundary.finish(hidden_states, forward_batch)
|
||||
return hidden_states
|
||||
|
||||
|
||||
|
||||
@@ -325,9 +325,8 @@ class SDARBlock(nn.Module):
|
||||
hidden_states = self.attn_boundary.finish(hidden_states, forward_batch)
|
||||
hidden_states = self.ffn_boundary.prepare(hidden_states, forward_batch)
|
||||
|
||||
with self.ffn_boundary.exit(forward_batch) as ffn_exit:
|
||||
hidden_states = self.mlp(hidden_states)
|
||||
hidden_states = ffn_exit.finish(hidden_states)
|
||||
hidden_states = self.mlp(hidden_states)
|
||||
hidden_states = self.ffn_boundary.finish(hidden_states, forward_batch)
|
||||
|
||||
return hidden_states
|
||||
|
||||
|
||||
@@ -384,12 +384,11 @@ class SDARMoeBlock(nn.Module):
|
||||
hidden_states = self.attn_boundary.finish(hidden_states, forward_batch)
|
||||
hidden_states = self.ffn_boundary.prepare(hidden_states, forward_batch)
|
||||
|
||||
with self.ffn_boundary.exit(forward_batch) as ffn_exit:
|
||||
hidden_states = self.mlp(
|
||||
hidden_states,
|
||||
forward_batch=forward_batch,
|
||||
)
|
||||
hidden_states = ffn_exit.finish(hidden_states)
|
||||
hidden_states = self.mlp(
|
||||
hidden_states,
|
||||
forward_batch=forward_batch,
|
||||
)
|
||||
hidden_states = self.ffn_boundary.finish(hidden_states, forward_batch)
|
||||
|
||||
return hidden_states
|
||||
|
||||
|
||||
@@ -296,9 +296,8 @@ class Spark2_5DecoderLayer(nn.Module):
|
||||
)
|
||||
hidden_states = self.attn_boundary.finish(hidden_states, forward_batch)
|
||||
hidden_states = self.ffn_boundary.prepare(hidden_states, forward_batch)
|
||||
with self.ffn_boundary.exit(forward_batch) as ffn_exit:
|
||||
hidden_states = self.mlp(hidden_states)
|
||||
return ffn_exit.finish(hidden_states)
|
||||
hidden_states = self.mlp(hidden_states)
|
||||
return self.ffn_boundary.finish(hidden_states, forward_batch)
|
||||
|
||||
|
||||
class Spark2_5Model(nn.Module):
|
||||
|
||||
@@ -417,12 +417,11 @@ class Step3TextDecoderLayer(nn.Module):
|
||||
|
||||
hidden_states = self.attn_boundary.finish(hidden_states, forward_batch)
|
||||
hidden_states = self.ffn_boundary.prepare(hidden_states, forward_batch)
|
||||
with self.ffn_boundary.exit(forward_batch) as ffn_exit:
|
||||
if self.use_moe:
|
||||
hidden_states = self.moe_mlp_forward(hidden_states)
|
||||
else:
|
||||
hidden_states = self.mlp(hidden_states)
|
||||
hidden_states = ffn_exit.finish(hidden_states)
|
||||
if self.use_moe:
|
||||
hidden_states = self.moe_mlp_forward(hidden_states)
|
||||
else:
|
||||
hidden_states = self.mlp(hidden_states)
|
||||
hidden_states = self.ffn_boundary.finish(hidden_states, forward_batch)
|
||||
|
||||
return hidden_states
|
||||
|
||||
|
||||
@@ -565,16 +565,14 @@ class Step3p5DecoderLayer(nn.Module):
|
||||
hidden_states = self.ffn_boundary.prepare(hidden_states, forward_batch)
|
||||
|
||||
if self.use_moe:
|
||||
with self.ffn_boundary.exit(forward_batch) as ffn_exit:
|
||||
share_output = self.share_expert(hidden_states)
|
||||
hidden_states = self.moe(
|
||||
hidden_states, forward_batch, shared_output=share_output
|
||||
)
|
||||
return ffn_exit.finish(hidden_states)
|
||||
share_output = self.share_expert(hidden_states)
|
||||
hidden_states = self.moe(
|
||||
hidden_states, forward_batch, shared_output=share_output
|
||||
)
|
||||
return self.ffn_boundary.finish(hidden_states, forward_batch)
|
||||
|
||||
with self.ffn_boundary.exit(forward_batch) as ffn_exit:
|
||||
hidden_states = self.mlp(hidden_states)
|
||||
return ffn_exit.finish(hidden_states)
|
||||
hidden_states = self.mlp(hidden_states)
|
||||
return self.ffn_boundary.finish(hidden_states, forward_batch)
|
||||
|
||||
|
||||
class Step3p5Model(nn.Module):
|
||||
|
||||
@@ -1558,12 +1558,11 @@ class XllmDecoderLayer(nn.Module):
|
||||
hidden_states = self.attn_boundary.finish(hidden_states, forward_batch)
|
||||
hidden_states = self.ffn_boundary.prepare(hidden_states, forward_batch)
|
||||
|
||||
with self.ffn_boundary.exit(forward_batch) as ffn_exit:
|
||||
if isinstance(self.mlp, XllmMLP):
|
||||
hidden_states = self.mlp(hidden_states)
|
||||
else:
|
||||
hidden_states = self.mlp(hidden_states, forward_batch)
|
||||
hidden_states = ffn_exit.finish(hidden_states)
|
||||
if isinstance(self.mlp, XllmMLP):
|
||||
hidden_states = self.mlp(hidden_states)
|
||||
else:
|
||||
hidden_states = self.mlp(hidden_states, forward_batch)
|
||||
hidden_states = self.ffn_boundary.finish(hidden_states, forward_batch)
|
||||
|
||||
return hidden_states
|
||||
|
||||
|
||||
@@ -1397,11 +1397,10 @@ class ZayaDecoderMLPLayer(nn.Module):
|
||||
prev_router_hidden_states: Optional[torch.Tensor] = None,
|
||||
) -> tuple[torch.Tensor, Optional[torch.Tensor]]:
|
||||
hidden_states = self.ffn_boundary.prepare(hidden_states, forward_batch)
|
||||
with self.ffn_boundary.exit(forward_batch) as ffn_exit:
|
||||
hidden_states, prev_router_hidden_states = self.zaya_block(
|
||||
hidden_states, prev_router_hidden_states
|
||||
)
|
||||
hidden_states = ffn_exit.finish(hidden_states)
|
||||
hidden_states, prev_router_hidden_states = self.zaya_block(
|
||||
hidden_states, prev_router_hidden_states
|
||||
)
|
||||
hidden_states = self.ffn_boundary.finish(hidden_states, forward_batch)
|
||||
return hidden_states, prev_router_hidden_states
|
||||
|
||||
|
||||
|
||||
@@ -670,11 +670,9 @@ class ForwardFlags:
|
||||
"is_extend_in_batch": False,
|
||||
# Per-layer MLP collective control (set by decoder via scoped()
|
||||
# around the MLP / MoE / hybrid mixer call).
|
||||
# fuse_mlp_allreduce: next residual+LN absorbs the post-MLP all-reduce.
|
||||
# mlp_reduce_scatter: postprocess will reduce-scatter (skip MLP AR).
|
||||
# flashinfer_trtllm_bypass: deepseek dual-stream graph topk bypass.
|
||||
# defer_moe_finalize: next layer's fused collective absorbs the finalize.
|
||||
"fuse_mlp_allreduce": False,
|
||||
"mlp_reduce_scatter": False,
|
||||
"flashinfer_trtllm_bypass": False,
|
||||
"defer_moe_finalize": False,
|
||||
@@ -694,7 +692,6 @@ class ForwardFlags:
|
||||
"attn_input_scattered",
|
||||
"attn_inputs",
|
||||
"is_extend_in_batch",
|
||||
"fuse_mlp_allreduce",
|
||||
"mlp_reduce_scatter",
|
||||
"flashinfer_trtllm_bypass",
|
||||
"defer_moe_finalize",
|
||||
|
||||
Reference in New Issue
Block a user