[Refactor] Drop the reduction-skip mechanisms stage boundaries no longer use (#42312)

This commit is contained in:
Cheng Wan
2026-10-03 02:21:55 -07:00
committed by GitHub
parent 71b04e02cf
commit 7be5e3473c
73 changed files with 291 additions and 430 deletions
@@ -115,9 +115,6 @@ class ExitMove(NamedTuple):
# Whether output_move also completes the sum the producer leaves.
output_move_completes_sum: bool = False
returns_over_dp: bool = False
# When output_move completes the sum: the move of an output compute has
# already reduced onto the same rows.
complete_output_move: Optional[Callable] = None
def input_rows(edge: EdgeContract) -> Layout:
@@ -599,9 +596,8 @@ def _select_exit_move(
) -> ExitMove:
"""How the FFN output reaches the rows the layer hands on: by undoing the
attention-DP gather (the FFN exit and finish run that step), or else the
move that takes it there, None when there is none to choose here; whether
that move also completes the sum the FFN leaves, and if so the move for an
output compute already reduced."""
move that takes it there, None when there is none to choose here; and
whether that move also completes the sum the FFN leaves."""
update = produced.update
if produced.layout == residual:
@@ -634,12 +630,6 @@ def _select_exit_move(
update=update,
),
output_move_completes_sum=True,
complete_output_move=partial(
residual_slice_output,
sums=False,
gathers_back=to != residual,
update=update,
),
)
if to != residual or not produced.layout.sharded <= residual.sharded:
raise NotImplementedError(f"{produced=} {residual=} {to=}")
@@ -655,11 +645,7 @@ def _select_exit_move(
_sum_group(produced.group), cp_moves.reduce_scatter_group()
):
raise NotImplementedError(f"{produced=} {residual=} {to=}")
return ExitMove(
cp_moves.reduce_scatter,
output_move_completes_sum=True,
complete_output_move=cp_moves.take_back,
)
return ExitMove(cp_moves.reduce_scatter, output_move_completes_sum=True)
if returned == {TokenAxis.ATTN_DP, TokenAxis.ATTN_CP}:
# This rank's CP shard, from where the DP gather put it.
return ExitMove(dp_cp_take_back_output)
@@ -214,7 +214,6 @@ class StagePlan:
output_move=out.output_move,
output_move_completes_sum=out.output_move_completes_sum,
returns_over_dp=out.returns_over_dp,
complete_output_move=out.complete_output_move,
)
@property
@@ -27,19 +27,16 @@ from sglang.srt.layers.layer_boundary.residual.add_norm import NORM_READOUT, PLA
class ProducerReduction(Enum):
"""Describe compute's cooperation with its output boundary.
"""Who completes the sum a stage's output owes; compute never does.
ALWAYS_PARTIAL is attention-only: finish() publishes its partial sum. EXIT_SCOPED
follows exit() flags for an FFN or single-stage mixer. TAIL_AFTER_SUM is FFN-only:
compute adds a replicated component after its internal sum, so the
boundary never defers that sum to the next layer. A selected reduce-scatter
still applies, so compute must add the tail only when it completes the sum.
ALWAYS_PARTIAL is attention-only: finish() hands the partial sum to the
next stage's input as its declared sum. EXIT_SCOPED, for an FFN or a
single-stage mixer: the exit completes the sum, or carries it to the next
stage's input.
"""
ALWAYS_PARTIAL = auto()
EXIT_SCOPED = auto()
# A replicated component is added after the sum inside compute.
TAIL_AFTER_SUM = auto()
class ExitRows(Enum):
@@ -80,30 +77,30 @@ class OutputContract(msgspec.Struct, frozen=True):
Fields:
layout: Token sharding of the producer contribution.
group: Named sum group, or None when there is no reduction.
always_partial: Compute always returns a partial sum.
may_defer_to_next: Compute can skip reduction under the exit scope
and let the following layer complete it.
may_reduce_scatter: Compute can leave reduction to a fixed-size
reduce-scatter selected by the boundary.
may_reduce_scatterv: Compute can leave reduction to the selected
variable-size attention-DP combine.
group: The group the output owes its sum over, or None when it is
complete.
always_partial: The sum is handed to the next stage's input as its
declared sum.
may_defer_to_next: The exit may carry the sum to the following
stage's input instead of completing it.
may_reduce_scatter: A fixed-size reduce-scatter selected by the
boundary may complete the sum.
may_reduce_scatterv: The selected variable-size attention-DP combine
may complete the sum.
update: Producer residual operation; None on an arrival contract that
declares capabilities and obtains the actual update from the stream.
transform: Optional operation on the contribution before residual update.
These permissions are not evidence that a particular output is partial.
The exit decision and ResidualStream record what that output actually owes.
These are permissions, not a record of a particular output. The exit
decision and ResidualStream record what that output actually owes.
"""
layout: Layout
# None when there is nothing to sum.
group: Optional[SumGroup] = None
# The producer never reduces its output, e.g. a row-parallel projection
# built with reduce_results=False.
# The sum is always handed to the next stage's input as a declared sum.
always_partial: bool = False
# Otherwise it leaves the sum only when the boundary publishes the flag for
# it: fuse_mlp_allreduce, or mlp_reduce_scatter.
# Otherwise the exit completes it, or may carry it to the next input.
may_defer_to_next: bool = False
may_reduce_scatter: bool = False
# Whether the selected attention-DP combine may complete this sum.
@@ -236,8 +233,6 @@ class StagePath(msgspec.Struct, frozen=True):
batch by the attention-DP exit path.
output_move_completes_sum: Whether that move also reduces the output.
returns_over_dp: Whether output uses batch-dependent attention-DP transport.
complete_output_move: The move for an output compute already reduced,
run instead of an output_move that also reduces it.
"""
entry: EntryPath
@@ -247,7 +242,6 @@ class StagePath(msgspec.Struct, frozen=True):
output_move_completes_sum: bool = False
returns_over_dp: bool = False
complete_output_move: Optional[Callable] = None
class StageKind(Enum):
+29 -31
View File
@@ -129,11 +129,11 @@ class ExitPolicy:
return stream.record(hidden_states, update, declared_sum=declared_sum)
@staticmethod
def _sum_owed_after_skip(steps, skipped_reduction):
# The input-scattered path leaves the sum for the next input's TP
# reduce-scatter. Other skip paths complete it in the output move.
def _sum_left_to_next_input(steps, sum_in_reduce_scatter):
# On an input-scattered batch the next input's TP reduce-scatter
# completes the sum; the other reduce-scatters are this exit's move.
if (
skipped_reduction
sum_in_reduce_scatter
and not steps.returns_over_dp
and not steps.output_move_completes_sum
):
@@ -160,10 +160,9 @@ class ExitPolicy:
may_reduce_scatterv=steps.output.may_reduce_scatterv,
)
def _skips_sum_for_reduce_scatter(self, steps, dp_step: Optional[Callable]) -> bool:
"""Whether the FFN leaves its sum out because a reduce-scatter completes
it: the attention-DP one ``dp_step`` names, or the CP / input-scattered
one."""
def _sum_in_reduce_scatter(self, steps, dp_step: Optional[Callable]) -> bool:
"""Whether a reduce-scatter completes the FFN output's sum: the
attention-DP one ``dp_step`` names, or the CP / input-scattered one."""
if dp_step is not None:
return True
if not steps.output.may_reduce_scatter:
@@ -178,10 +177,10 @@ class ExitPolicy:
return _sum_group(steps.output.group)
def _decide(self, forward_batch: ForwardBatch, steps) -> ExitDecision:
"""Decide once, before the FFN runs, what it skips and what completes its
output: the next layer's input, or this layer's postprocess step."""
"""Decide once, before the FFN runs, what completes its output's sum:
the next layer's input, or this layer's postprocess step."""
dp_step = self._dp_reduce_scatter_step(forward_batch, steps)
mlp_reduce_scatter = self._skips_sum_for_reduce_scatter(steps, dp_step)
sum_in_reduce_scatter = self._sum_in_reduce_scatter(steps, dp_step)
complete_now = partial(
self._complete_now,
forward_batch=forward_batch,
@@ -189,7 +188,7 @@ class ExitPolicy:
steps=steps,
output_move=steps.output_move,
# A reduce-scatter completes the sum on its way back.
owes_sum=steps.output.group is not None and not mlp_reduce_scatter,
owes_sum=steps.output.group is not None and not sum_in_reduce_scatter,
)
defer_moe_finalize = (
self.plan.fusions is not None
@@ -200,16 +199,18 @@ class ExitPolicy:
):
return ExitDecision(
defer_moe_finalize=False,
fuse_mlp_allreduce=False,
mlp_reduce_scatter=mlp_reduce_scatter,
sum_in_reduce_scatter=sum_in_reduce_scatter,
complete=complete_now,
)
# Producers declare remaining work independently of the kernel chosen
# by the consumer. Every handoff also carries an unfused completion.
fuse_mlp_allreduce = defer_moe_finalize or self._defers_sum(
forward_batch, steps, mlp_reduce_scatter=mlp_reduce_scatter, dp_step=dp_step
defers = defer_moe_finalize or self._defers_sum(
forward_batch,
steps,
sum_in_reduce_scatter=sum_in_reduce_scatter,
dp_step=dp_step,
)
if fuse_mlp_allreduce:
if defers:
group = self.ffn_reduction_group(steps)
if steps.returns_over_dp:
# Under attention DP the next layer also brings the sum back to
@@ -239,8 +240,7 @@ class ExitPolicy:
complete = complete_now
return ExitDecision(
defer_moe_finalize=defer_moe_finalize,
fuse_mlp_allreduce=fuse_mlp_allreduce,
mlp_reduce_scatter=mlp_reduce_scatter,
sum_in_reduce_scatter=sum_in_reduce_scatter,
complete=complete,
)
@@ -288,15 +288,16 @@ class ExitPolicy:
self, forward_batch: ForwardBatch, *, stream: ResidualStream
) -> MixerExit:
"""Decide once whether this stage's mixer (an attention-like stage)
skips its output all-reduce. Use the result as a context manager around
the mixer, then call ``finish``."""
output's sum is completed here or carried on. Call ``finish`` on the
result with the mixer's output."""
return MixerExit(self, forward_batch, stream=stream)
def ffn_exit(
self, forward_batch: ForwardBatch, *, stream: ResidualStream
) -> FfnExit:
"""Decide once how this layer's FFN output reduction completes. Use the
result as a context manager around the FFN call, then call ``finish``."""
result as a context manager around the FFN call when the FFN may hand
off its MoE finalize, then call ``finish``."""
return FfnExit(self, forward_batch, stream=stream)
def _sum_deferral_allowed(self, steps) -> bool:
@@ -331,7 +332,7 @@ class ExitPolicy:
forward_batch: ForwardBatch,
steps,
*,
mlp_reduce_scatter: bool,
sum_in_reduce_scatter: bool,
dp_step: Optional[Callable],
) -> bool:
"""Whether the FFN leaves its output's all-reduce to the next layer's
@@ -342,7 +343,7 @@ class ExitPolicy:
and not self.plan.terminal
and self._sum_deferral_allowed(steps)
and _batch_allows_deferred_sum(forward_batch, self.plan)
and not mlp_reduce_scatter
and not sum_in_reduce_scatter
# Under attention DP the next layer must also run postprocess's
# scatter back to this rank's tokens, and nothing more.
and (
@@ -392,9 +393,7 @@ class ExitDecision(msgspec.Struct, frozen=True):
Fields:
defer_moe_finalize: Compute may return a producer-specific finalize handoff.
fuse_mlp_allreduce: The sum is carried to the next consumer, which
completes it, fused or unfused.
mlp_reduce_scatter: A reduce-scatter on the way back completes the sum.
sum_in_reduce_scatter: A reduce-scatter on the way back completes the sum.
complete: Callable(output, residual) returning the completed or wrapped
output and its corresponding residual rows.
@@ -402,8 +401,7 @@ class ExitDecision(msgspec.Struct, frozen=True):
"""
defer_moe_finalize: bool
fuse_mlp_allreduce: bool
mlp_reduce_scatter: bool
sum_in_reduce_scatter: bool
complete: Callable[[torch.Tensor, torch.Tensor], Tuple]
@@ -519,8 +517,8 @@ class FfnExit:
self.defer_moe_finalize = completion.defer_moe_finalize
self._complete = completion.complete
self._update = steps.output.update
self._declared_sum = boundary._sum_owed_after_skip(
steps, completion.mlp_reduce_scatter
self._declared_sum = boundary._sum_left_to_next_input(
steps, completion.sum_in_reduce_scatter
)
self._scope = get_forward().scoped(defer_moe_finalize=self.defer_moe_finalize)
@@ -82,7 +82,6 @@ def _resolve_ffn(
read=NORM_READOUT,
update=PLAIN_ADD,
dense_tp_size=None,
reduction=ProducerReduction.EXIT_SCOPED,
):
parallel = get_parallel()
if dense_tp_size not in (None, 1, parallel.tp_size):
@@ -169,8 +168,7 @@ def _resolve_ffn(
and not terminal
and not update.applied_at_exit
and update.outlives_layer
and output_transform is None
and reduction is ProducerReduction.EXIT_SCOPED,
and output_transform is None,
may_reduce_scatter=use_reduce_scatter and may_scatter,
may_reduce_scatterv=use_reduce_scatterv and may_leave,
update=update,
@@ -203,8 +201,8 @@ class StageDeclaration:
residual update. An FFN's exit runs it under an explicit
reduction-order contract; an attention's is run by the input of
the stage that follows, once the attention's sum is complete.
reduction: Whether compute always leaves a partial sum, obeys the
exit scope, or adds a replicated component after its own sum.
reduction: Whether the next stage's input always completes the sum
(an attention's), or the exit decides (an FFN's or a mixer's).
gathers_attn_tp_input: Whether attention gathers TP-sharded input itself.
dense_tp_size: Dense FFN compute width: None uses the configured width,
1 means local compute, and the full TP size means TP compute.
@@ -282,9 +280,8 @@ def declare_attn(
read: Input operation; defaults to normalization with quantization support.
update: Operation that adds this stage's output to the residual.
terminal: Whether this stage ends the model's layer stack.
reduction: ALWAYS_PARTIAL for an output projection that always skips reduction;
EXIT_SCOPED for a mixer that follows its exit scope's reduction decision.
TAIL_AFTER_SUM is rejected for attention stages.
reduction: ALWAYS_PARTIAL for an attention whose sum the next stage's
input always completes; EXIT_SCOPED for a mixer whose exit decides.
gathers_attn_tp_input: Whether compute gathers attention-TP input slices itself.
output_transform: Optional operation on the output once its sum is
complete, before the residual update (a sandwich norm). The next
@@ -294,8 +291,6 @@ def declare_attn(
Returns:
A StageDeclaration with no norm, tensors or execution plan.
"""
if reduction is ProducerReduction.TAIL_AFTER_SUM:
raise ValueError("TAIL_AFTER_SUM is not supported for attention stages")
if (
output_transform is not None
and reduction is not ProducerReduction.ALWAYS_PARTIAL
@@ -325,7 +320,6 @@ def declare_ffn(
output_transform=None,
next_layer_sparse=False,
dense_tp_size=None,
reduction=ProducerReduction.EXIT_SCOPED,
exit_rows=None,
):
"""Declare a dense or MoE FFN independently of its compute module.
@@ -342,17 +336,12 @@ def declare_ffn(
to derive the TBO exit rows when exit_rows is not supplied.
dense_tp_size: Dense compute width: None for configuration, 1 for local
compute, or the full TP size.
reduction: How compute cooperates with the exit's reduction decision:
EXIT_SCOPED (default) follows the exit scope; TAIL_AFTER_SUM marks a
replicated tail after the sum. ALWAYS_PARTIAL is rejected for FFN stages.
exit_rows: Explicit output-row requirement; otherwise derived from
the adjacent FFN kinds and TBO configuration.
Returns:
A StageDeclaration with no norm, tensors or execution plan.
"""
if reduction is ProducerReduction.ALWAYS_PARTIAL:
raise ValueError("ALWAYS_PARTIAL is not supported for ffn stages")
return StageDeclaration(
StageKind.FFN,
read,
@@ -363,7 +352,6 @@ def declare_ffn(
terminal=terminal,
output_transform=output_transform,
dense_tp_size=dense_tp_size,
reduction=reduction,
exit_rows=exit_rows or tbo_exit_rows(sparse, next_layer_sparse),
)
@@ -388,7 +376,6 @@ def _resolve_stage(stage, variant, following=None):
read=stage.read,
update=stage.update,
dense_tp_size=stage.dense_tp_size,
reduction=stage.reduction,
)
if resolve_exit_rows(stage.exit_rows) is ExitRows.ATTENTION:
returned = attention
@@ -428,10 +428,9 @@ def moe_cp_take_back_output(
):
"""Return a MoE output computed on the MoE-CP-gathered rows to this rank's attention rows.
After moe_tensor_model_parallel_all_reduce (which runs unconditionally since
mlp_reduce_scatter=False for this path), all ranks in the moe_cp group hold the
full MoE result for all cp_per_moe token chunks. We simply slice out this rank's
CP-local portion.
Once the exit has completed the MoE output's sum, all ranks in the moe_cp
group hold the full MoE result for all cp_per_moe token chunks. We simply
slice out this rank's CP-local portion.
If DP>1, further scatter back to the local DP slice.
"""
@@ -170,20 +170,26 @@ class StageBoundary:
return hidden_states
def finish(self, hidden_states, forward_batch):
"""Hand this attention's actual contribution to the following stage."""
if self.kind is not StageKind.ATTENTION or not self.plan.finishes_directly:
raise RuntimeError(
"use the stage exit scope for an FFN or single-stage mixer"
"""Hand this stage's output to the following stage: an attention's
actual contribution, or an FFN's or a mixer's output with its sum
completed or carried on. An FFN whose boundary has fusions (which may
take a MoE finalize handoff) finishes through exit() instead."""
if self.kind is StageKind.ATTENTION and self.plan.finishes_directly:
produced = self.plan.produced(forward_batch)
return stream_of(forward_batch).record(
hidden_states,
produced.update,
declared_sum=produced.group if produced.always_partial else None,
)
produced = self.plan.produced(forward_batch)
return stream_of(forward_batch).record(
hidden_states,
produced.update,
declared_sum=produced.group if produced.always_partial else None,
)
if self.kind is StageKind.FFN and self.plan.fusions is not None:
raise RuntimeError(
"an FFN whose boundary has fusions finishes through exit()"
)
return self.exit(forward_batch).finish(hidden_states)
def exit(self, forward_batch):
"""Select and scope one FFN or single-stage mixer's output decision.
"""Select and scope one FFN or single-stage mixer's output decision,
for an FFN whose MoE may hand off its finalize (see finish()).
Args:
forward_batch: Active batch whose stream receives the compute result.
+3 -3
View File
@@ -1661,9 +1661,9 @@ class RowParallelLinear(LinearBase):
self, input_parallel, output_tensor, bias=bias_
)
# skip_all_reduce: explicit call-site override. Also honor
# ForwardFlags (fuse_mlp_allreduce / mlp_reduce_scatter) published by
# the decoder — callers should not thread those flags into modules.
# skip_all_reduce: explicit call-site override. Also honor the
# mlp_reduce_scatter ForwardFlag published by the decoder — callers
# should not thread it into modules.
if (
((self.reduce_results and self.tp_size > 1) or self.use_decode_attn_tp)
and not skip_all_reduce
+3 -5
View File
@@ -730,12 +730,10 @@ def should_use_dp_reduce_scatterv():
def should_skip_mlp_all_reduce() -> bool:
"""Whether dense MLP / row-parallel projections should skip their all-reduce.
True when the decoder published ``fuse_mlp_allreduce`` (next residual+LN
absorbs the AR) or ``mlp_reduce_scatter`` (postprocess will reduce-scatter)
on ``get_forward()``.
True when the decoder published ``mlp_reduce_scatter`` (postprocess will
reduce-scatter) on ``get_forward()``.
"""
f = get_forward()
return f.fuse_mlp_allreduce or f.mlp_reduce_scatter
return get_forward().mlp_reduce_scatter
def post_experts_output_is_complete(*, is_tp_path: bool) -> bool:
+2 -3
View File
@@ -295,9 +295,8 @@ class ApertusDecoderLayer(nn.Module):
# Fully Connected
hidden_states = self.attn_boundary.finish(hidden_states, forward_batch)
hidden_states = self.ffn_boundary.prepare(hidden_states, forward_batch)
with self.ffn_boundary.exit(forward_batch) as ffn_exit:
hidden_states = self.mlp(hidden_states)
return ffn_exit.finish(hidden_states)
hidden_states = self.mlp(hidden_states)
return self.ffn_boundary.finish(hidden_states, forward_batch)
class ApertusModel(nn.Module):
+2 -3
View File
@@ -271,9 +271,8 @@ class ArceeDecoderLayer(nn.Module):
# Fully Connected
hidden_states = self.attn_boundary.finish(hidden_states, forward_batch)
hidden_states = self.ffn_boundary.prepare(hidden_states, forward_batch)
with self.ffn_boundary.exit(forward_batch) as ffn_exit:
hidden_states = self.mlp(hidden_states)
return ffn_exit.finish(hidden_states)
hidden_states = self.mlp(hidden_states)
return self.ffn_boundary.finish(hidden_states, forward_batch)
class ArceeModel(nn.Module):
+2 -3
View File
@@ -748,9 +748,8 @@ class BailingMoEBlock(nn.Module):
forward_batch=forward_batch,
)
with self.ffn_boundary.exit(forward_batch) as ffn_exit:
hidden_states = self.mlp(hidden_states, forward_batch)
hidden_states = ffn_exit.finish(hidden_states)
hidden_states = self.mlp(hidden_states, forward_batch)
hidden_states = self.ffn_boundary.finish(hidden_states, forward_batch)
return hidden_states
@@ -883,9 +883,8 @@ class BailingMoELinearDecoderLayer(nn.Module):
# logger.warning(
# f"===={self.layer_id=}, 3 shape= {hidden_states.shape}, {residual.shape}"
# )
with self.ffn_boundary.exit(forward_batch) as ffn_exit:
hidden_states = self.mlp(hidden_states)
hidden_states = ffn_exit.finish(hidden_states)
hidden_states = self.mlp(hidden_states)
hidden_states = self.ffn_boundary.finish(hidden_states, forward_batch)
return hidden_states
@staticmethod
+10 -11
View File
@@ -1196,17 +1196,16 @@ class BailingMoELinearDecoderLayer(nn.Module):
hidden_states = self.attn_boundary.finish(hidden_states, forward_batch)
hidden_states = self.ffn_boundary.prepare(hidden_states, forward_batch)
with self.ffn_boundary.exit(forward_batch) as ffn_exit:
if not (
is_dense_ffn_fully_dp()
and (not self.is_layer_sparse)
and hidden_states.shape[0] == 0
):
hidden_states = self.mlp(
hidden_states,
forward_batch=forward_batch,
)
hidden_states = ffn_exit.finish(hidden_states)
if not (
is_dense_ffn_fully_dp()
and (not self.is_layer_sparse)
and hidden_states.shape[0] == 0
):
hidden_states = self.mlp(
hidden_states,
forward_batch=forward_batch,
)
hidden_states = self.ffn_boundary.finish(hidden_states, forward_batch)
return hidden_states
-3
View File
@@ -982,7 +982,6 @@ class DeepseekV2MoE(nn.Module):
return dsv2_flashinfer_moe_dual_stream_graph(
hidden_states,
self.layer_id,
fwd.fuse_mlp_allreduce,
fwd.mlp_reduce_scatter,
)
elif (
@@ -3535,7 +3534,6 @@ class DeepseekV32ForCausalLM(DeepseekV2ForCausalLM):
def dsv2_flashinfer_moe_dual_stream_graph(
hidden_states: torch.Tensor,
layer_id: int,
fuse_mlp_allreduce: bool,
mlp_reduce_scatter: bool,
) -> torch.Tensor:
forward_context = get_tc_piecewise_forward_context()
@@ -3548,7 +3546,6 @@ def dsv2_flashinfer_moe_dual_stream_graph(
# torch.compile. Carry graph-varying control state as scalar operands and
# republish it for the nested MoE/linear consumers.
with get_forward().scoped(
fuse_mlp_allreduce=fuse_mlp_allreduce,
mlp_reduce_scatter=mlp_reduce_scatter,
flashinfer_trtllm_bypass=True,
lora_batch_layout=LoRABatchLayout.TP_GLOBAL,
@@ -1464,9 +1464,8 @@ class Dots3DecoderLayer(nn.Module):
hidden_states = self.attn_boundary.finish(hidden_states, forward_batch)
hidden_states = self.ffn_boundary.prepare(hidden_states, forward_batch)
with self.ffn_boundary.exit(forward_batch) as ffn_exit:
hidden_states = self.mlp(hidden_states, forward_batch)
return ffn_exit.finish(hidden_states)
hidden_states = self.mlp(hidden_states, forward_batch)
return self.ffn_boundary.finish(hidden_states, forward_batch)
class Dots3Model(nn.Module):
+5 -6
View File
@@ -472,12 +472,11 @@ class Ernie4_5_VLMoeDecoderLayer(nn.Module):
# Fully Connected
hidden_states = self.attn_boundary.finish(hidden_states, forward_batch)
hidden_states = self.ffn_boundary.prepare(hidden_states, forward_batch)
with self.ffn_boundary.exit(forward_batch) as ffn_exit:
if isinstance(self.mlp, Ernie4_5_VLMoeMoE):
hidden_states = self.mlp(hidden_states, visual_token_mask, **kwargs)
else:
hidden_states = self.mlp(hidden_states)
return ffn_exit.finish(hidden_states)
if isinstance(self.mlp, Ernie4_5_VLMoeMoE):
hidden_states = self.mlp(hidden_states, visual_token_mask, **kwargs)
else:
hidden_states = self.mlp(hidden_states)
return self.ffn_boundary.finish(hidden_states, forward_batch)
# only used as text backbone for ernie4.5 vl
+2 -3
View File
@@ -313,9 +313,8 @@ class Exaone4DecoderLayer(nn.Module):
# Fully Connected
hidden_states = self.attn_boundary.finish(hidden_states, forward_batch)
hidden_states = self.ffn_boundary.prepare(hidden_states, forward_batch)
with self.ffn_boundary.exit(forward_batch) as ffn_exit:
hidden_states = self.mlp(hidden_states)
return ffn_exit.finish(hidden_states)
hidden_states = self.mlp(hidden_states)
return self.ffn_boundary.finish(hidden_states, forward_batch)
class Exaone4Model(nn.Module):
+2 -3
View File
@@ -533,9 +533,8 @@ class ExaoneMoEDecoderLayer(nn.Module):
hidden_states = self.attn_boundary.finish(hidden_states, forward_batch)
hidden_states = self.ffn_boundary.prepare(hidden_states, forward_batch)
# Fully Connected
with self.ffn_boundary.exit(forward_batch) as ffn_exit:
hidden_states = self.mlp(hidden_states, forward_batch)
return ffn_exit.finish(hidden_states)
hidden_states = self.mlp(hidden_states, forward_batch)
return self.ffn_boundary.finish(hidden_states, forward_batch)
class ExaoneMoEModel(nn.Module):
+2 -3
View File
@@ -366,9 +366,8 @@ class FalconH1HybridAttentionDecoderLayer(nn.Module):
# Fully Connected
hidden_states = self.attn_boundary.finish(hidden_states, forward_batch)
hidden_states = self.ffn_boundary.prepare(hidden_states, forward_batch)
with self.ffn_boundary.exit(forward_batch) as ffn_exit:
hidden_states = self.feed_forward(hidden_states, forward_batch)
return ffn_exit.finish(hidden_states)
hidden_states = self.feed_forward(hidden_states, forward_batch)
return self.ffn_boundary.finish(hidden_states, forward_batch)
ALL_DECODER_LAYER_TYPES = {
+4 -5
View File
@@ -719,11 +719,10 @@ class Gemma4DecoderLayer(nn.Module):
hidden_states = self.attn_boundary.finish(hidden_states, forward_batch)
hidden_states = self.ffn_boundary.prepare(hidden_states, forward_batch)
residual = residual_batch.written_residual(forward_batch)
with self.ffn_boundary.exit(forward_batch) as ffn_exit:
hidden_states = self._next_stream(
hidden_states, residual, per_layer_input, forward_batch
)
return ffn_exit.finish(hidden_states)
hidden_states = self._next_stream(
hidden_states, residual, per_layer_input, forward_batch
)
return self.ffn_boundary.finish(hidden_states, forward_batch)
def _next_stream(
self,
+2 -3
View File
@@ -302,9 +302,8 @@ class Glm4DecoderLayer(nn.Module):
# Fully Connected
hidden_states = self.attn_boundary.finish(hidden_states, forward_batch)
hidden_states = self.ffn_boundary.prepare(hidden_states, forward_batch)
with self.ffn_boundary.exit(forward_batch) as ffn_exit:
hidden_states = self.mlp(hidden_states)
return ffn_exit.finish(hidden_states)
hidden_states = self.mlp(hidden_states)
return self.ffn_boundary.finish(hidden_states, forward_batch)
class Glm4Model(nn.Module):
+2 -3
View File
@@ -865,9 +865,8 @@ class Glm4MoeDecoderLayer(nn.Module):
hidden_states = self.attn_boundary.finish(hidden_states, forward_batch)
hidden_states = self.ffn_boundary.prepare(hidden_states, forward_batch)
with self.ffn_boundary.exit(forward_batch) as ffn_exit:
hidden_states = self.mlp(hidden_states, forward_batch)
hidden_states = ffn_exit.finish(hidden_states)
hidden_states = self.mlp(hidden_states, forward_batch)
hidden_states = self.ffn_boundary.finish(hidden_states, forward_batch)
return hidden_states
+2 -3
View File
@@ -556,9 +556,8 @@ class Glm4MoeLiteDecoderLayer(nn.Module):
hidden_states = self.attn_boundary.finish(hidden_states, forward_batch)
hidden_states = self.ffn_boundary.prepare(hidden_states, forward_batch)
with self.ffn_boundary.exit(forward_batch) as ffn_exit:
hidden_states = self.mlp(hidden_states, forward_batch)
hidden_states = ffn_exit.finish(hidden_states)
hidden_states = self.mlp(hidden_states, forward_batch)
hidden_states = self.ffn_boundary.finish(hidden_states, forward_batch)
return hidden_states
+2 -2
View File
@@ -964,13 +964,13 @@ class Glm5NextDecoderLayer(nn.Module):
else:
_mlp_ctx = nullcontext()
with self.ffn_boundary.exit(forward_batch) as ffn_exit, _mlp_ctx:
with _mlp_ctx:
hidden_states = self.mlp(
hidden_states,
forward_batch,
gemm_output_zero_allocator,
)
hidden_states = ffn_exit.finish(hidden_states)
hidden_states = self.ffn_boundary.finish(hidden_states, forward_batch)
return (hidden_states, topk_indices)
+2 -3
View File
@@ -947,9 +947,8 @@ class GlmImageTextDecoderLayer(nn.Module):
# Fully Connected
hidden_states = self.attn_boundary.finish(hidden_states, forward_batch)
hidden_states = self.ffn_boundary.prepare(hidden_states, forward_batch)
with self.ffn_boundary.exit(forward_batch) as ffn_exit:
hidden_states = self.mlp(hidden_states)
return ffn_exit.finish(hidden_states)
hidden_states = self.mlp(hidden_states)
return self.ffn_boundary.finish(hidden_states, forward_batch)
class GlmImageTextModel(nn.Module):
+2 -3
View File
@@ -629,9 +629,8 @@ class GptOssDecoderLayer(nn.Module):
hidden_states = self.attn_boundary.finish(hidden_states, forward_batch)
hidden_states = self.ffn_boundary.prepare(hidden_states, forward_batch)
with self.ffn_boundary.exit(forward_batch) as ffn_exit:
hidden_states = self.mlp(hidden_states, forward_batch)
hidden_states = ffn_exit.finish(hidden_states)
hidden_states = self.mlp(hidden_states, forward_batch)
hidden_states = self.ffn_boundary.finish(hidden_states, forward_batch)
return hidden_states
+26 -28
View File
@@ -155,21 +155,20 @@ class GraniteMoeHybridMambaDecoderLayer(nn.Module):
hidden_states = self.attn_boundary.finish(output, forward_batch)
hidden_states = self.ffn_boundary.prepare(hidden_states, forward_batch)
with self.ffn_boundary.exit(forward_batch) as ffn_exit:
if self.shared_mlp is None:
if self.block_sparse_moe is not None:
hidden_states = self.block_sparse_moe(hidden_states)
# else: skip
if self.shared_mlp is None:
if self.block_sparse_moe is not None:
hidden_states = self.block_sparse_moe(hidden_states)
# else: skip
else:
# create a copy since block_sparse_moe modifies in-place
if self.block_sparse_moe is not None:
moe_hidden_states = hidden_states.clone()
moe_hidden_states = self.block_sparse_moe(moe_hidden_states)
hidden_states = moe_hidden_states + self.shared_mlp(hidden_states)
del moe_hidden_states
else:
# create a copy since block_sparse_moe modifies in-place
if self.block_sparse_moe is not None:
moe_hidden_states = hidden_states.clone()
moe_hidden_states = self.block_sparse_moe(moe_hidden_states)
hidden_states = moe_hidden_states + self.shared_mlp(hidden_states)
del moe_hidden_states
else:
hidden_states = self.shared_mlp(hidden_states)
return ffn_exit.finish(hidden_states)
hidden_states = self.shared_mlp(hidden_states)
return self.ffn_boundary.finish(hidden_states, forward_batch)
class GraniteMoeHybridAttention(nn.Module):
@@ -339,21 +338,20 @@ class GraniteMoeHybridAttentionDecoderLayer(nn.Module):
)
hidden_states = self.attn_boundary.finish(hidden_states, forward_batch)
hidden_states = self.ffn_boundary.prepare(hidden_states, forward_batch)
with self.ffn_boundary.exit(forward_batch) as ffn_exit:
if self.shared_mlp is None:
if self.block_sparse_moe is not None:
hidden_states = self.block_sparse_moe(hidden_states)
# else: skip
if self.shared_mlp is None:
if self.block_sparse_moe is not None:
hidden_states = self.block_sparse_moe(hidden_states)
# else: skip
else:
# create a copy since block_sparse_moe modifies in-place
if self.block_sparse_moe is not None:
moe_hidden_states = hidden_states.clone()
moe_hidden_states = self.block_sparse_moe(moe_hidden_states)
hidden_states = moe_hidden_states + self.shared_mlp(hidden_states)
del moe_hidden_states
else:
# create a copy since block_sparse_moe modifies in-place
if self.block_sparse_moe is not None:
moe_hidden_states = hidden_states.clone()
moe_hidden_states = self.block_sparse_moe(moe_hidden_states)
hidden_states = moe_hidden_states + self.shared_mlp(hidden_states)
del moe_hidden_states
else:
hidden_states = self.shared_mlp(hidden_states)
return ffn_exit.finish(hidden_states)
hidden_states = self.shared_mlp(hidden_states)
return self.ffn_boundary.finish(hidden_states, forward_batch)
ALL_DECODER_LAYER_TYPES = {
+2 -5
View File
@@ -429,11 +429,8 @@ class _InternS2MobiusDecoderMixin:
) -> torch.Tensor:
hidden_states = self.attn_boundary.finish(hidden_states, forward_batch)
hidden_states = self.ffn_boundary.prepare(hidden_states, forward_batch)
with self.ffn_boundary.exit(forward_batch) as ffn_exit:
hidden_states = self._forward_mobius_mlp(
hidden_states, forward_batch, meta_mlp
)
hidden_states = ffn_exit.finish(hidden_states)
hidden_states = self._forward_mobius_mlp(hidden_states, forward_batch, meta_mlp)
hidden_states = self.ffn_boundary.finish(hidden_states, forward_batch)
return hidden_states
+2 -3
View File
@@ -532,9 +532,8 @@ class IQuestQ1DecoderLayer(nn.Module):
attn_output = self.self_attn(positions, hidden_states, forward_batch)
hidden_states = self.attn_boundary.finish(attn_output, forward_batch)
hidden_states = self.ffn_boundary.prepare(hidden_states, forward_batch)
with self.ffn_boundary.exit(forward_batch) as ffn_exit:
mlp_output = self.mlp(hidden_states)
return ffn_exit.finish(mlp_output)
mlp_output = self.mlp(hidden_states)
return self.ffn_boundary.finish(mlp_output, forward_batch)
class IQuestQ1Model(nn.Module):
+2 -3
View File
@@ -652,9 +652,8 @@ class KimiDecoderLayer(nn.Module):
# Fully Connected
hidden_states = self.attn_boundary.finish(hidden_states, forward_batch)
hidden_states = self.ffn_boundary.prepare(hidden_states, forward_batch)
with self.ffn_boundary.exit(forward_batch) as ffn_exit:
hidden_states = self.mlp(hidden_states)
return ffn_exit.finish(hidden_states)
hidden_states = self.mlp(hidden_states)
return self.ffn_boundary.finish(hidden_states, forward_batch)
class KimiLinearModel(nn.Module):
+5 -8
View File
@@ -104,8 +104,6 @@ class LagunaMLP(nn.Module):
) -> torch.Tensor:
gate_up, _ = self.gate_up_proj(x)
x = self.act_fn(gate_up)
# RowParallelLinear honors ForwardFlags (fuse_mlp_allreduce /
# mlp_reduce_scatter) published by the decoder via scoped().
x, _ = self.down_proj(x)
return x
@@ -491,12 +489,11 @@ class LagunaDecoderLayer(nn.Module):
hidden_states = self.attn_boundary.finish(hidden_states, forward_batch)
hidden_states = self.ffn_boundary.prepare(hidden_states, forward_batch)
with self.ffn_boundary.exit(forward_batch) as ffn_exit:
hidden_states = self.mlp(
hidden_states,
forward_batch=forward_batch,
)
hidden_states = ffn_exit.finish(hidden_states)
hidden_states = self.mlp(
hidden_states,
forward_batch=forward_batch,
)
hidden_states = self.ffn_boundary.finish(hidden_states, forward_batch)
return hidden_states
+2 -3
View File
@@ -675,9 +675,8 @@ class LLaDA2MoeBlock(nn.Module):
forward_batch=forward_batch,
)
with self.ffn_boundary.exit(forward_batch) as ffn_exit:
hidden_states = self.mlp(hidden_states, forward_batch)
hidden_states = ffn_exit.finish(hidden_states)
hidden_states = self.mlp(hidden_states, forward_batch)
hidden_states = self.ffn_boundary.finish(hidden_states, forward_batch)
return hidden_states
+2 -3
View File
@@ -394,9 +394,8 @@ class LlamaDecoderLayer(nn.Module):
# Fully Connected
hidden_states = self.attn_boundary.finish(hidden_states, forward_batch)
hidden_states = self.ffn_boundary.prepare(hidden_states, forward_batch)
with self.ffn_boundary.exit(forward_batch) as ffn_exit:
hidden_states = self.mlp(hidden_states)
return ffn_exit.finish(hidden_states)
hidden_states = self.mlp(hidden_states)
return self.ffn_boundary.finish(hidden_states, forward_batch)
class LlamaModel(nn.Module):
+2 -3
View File
@@ -472,9 +472,8 @@ class Llama4DecoderLayer(nn.Module):
hidden_states = self.ffn_boundary.prepare(hidden_states, forward_batch)
# Fully Connected
with self.ffn_boundary.exit(forward_batch) as ffn_exit:
hidden_states = self.feed_forward(hidden_states, forward_batch)
hidden_states = ffn_exit.finish(hidden_states)
hidden_states = self.feed_forward(hidden_states, forward_batch)
hidden_states = self.ffn_boundary.finish(hidden_states, forward_batch)
return hidden_states
+4 -6
View File
@@ -504,9 +504,8 @@ class LongcatFlashDecoderLayer(nn.Module):
hidden_states = self.first_ffn_boundary.branch_input(
self.moe_boundary, hidden_states, forward_batch
)
with self.first_ffn_boundary.exit(forward_batch) as ffn_exit:
hidden_states = self.mlps[0](hidden_states)
hidden_states = ffn_exit.finish(hidden_states)
hidden_states = self.mlps[0](hidden_states)
hidden_states = self.first_ffn_boundary.finish(hidden_states, forward_batch)
# second_attn_boundary
hidden_states = self.second_attn_boundary.prepare(hidden_states, forward_batch)
@@ -526,9 +525,8 @@ class LongcatFlashDecoderLayer(nn.Module):
# second_mlp
hidden_states = self.second_attn_boundary.finish(hidden_states, forward_batch)
hidden_states = self.second_ffn_boundary.prepare(hidden_states, forward_batch)
with self.second_ffn_boundary.exit(forward_batch) as ffn_exit:
hidden_states = self.mlps[1](hidden_states)
hidden_states = ffn_exit.finish(hidden_states)
hidden_states = self.mlps[1](hidden_states)
hidden_states = self.second_ffn_boundary.finish(hidden_states, forward_batch)
return hidden_states, prev_topk_indices
@@ -191,9 +191,8 @@ class LongcatFlashDenseDecoderLayer(nn.Module):
hidden_states = self.attn_boundary.finish(hidden_states, forward_batch)
hidden_states = self.ffn_boundary.prepare(hidden_states, forward_batch)
with self.ffn_boundary.exit(forward_batch) as ffn_exit:
hidden_states = self.mlp(hidden_states)
hidden_states = ffn_exit.finish(hidden_states)
hidden_states = self.mlp(hidden_states)
hidden_states = self.ffn_boundary.finish(hidden_states, forward_batch)
return hidden_states
+2 -3
View File
@@ -898,9 +898,8 @@ class MiMoV2DecoderLayer(nn.Module):
hidden_states = self.attn_boundary.finish(hidden_states, forward_batch)
hidden_states = self.ffn_boundary.prepare(hidden_states, forward_batch)
with self.ffn_boundary.exit(forward_batch) as ffn_exit:
hidden_states = self.mlp(hidden_states, forward_batch)
hidden_states = ffn_exit.finish(hidden_states)
hidden_states = self.mlp(hidden_states, forward_batch)
hidden_states = self.ffn_boundary.finish(hidden_states, forward_batch)
return hidden_states
+2 -5
View File
@@ -155,12 +155,9 @@ class MiMoV2MTPLayer(nn.Module):
hidden_states = self.attn_boundary.finish(hidden_states, forward_batch)
hidden_states = self.ffn_boundary.prepare(hidden_states, forward_batch)
with (
self.ffn_boundary.exit(forward_batch) as ffn_exit,
get_global_expert_distribution_recorder().disable_this_region(),
):
with get_global_expert_distribution_recorder().disable_this_region():
hidden_states = self.mlp(hidden_states)
hidden_states = ffn_exit.finish(hidden_states)
hidden_states = self.ffn_boundary.finish(hidden_states, forward_batch)
return hidden_states
+2 -3
View File
@@ -912,9 +912,8 @@ class MiniMaxM2DecoderLayer(nn.Module):
hidden_states = self.attn_boundary.finish(hidden_states, forward_batch)
hidden_states = self.ffn_boundary.prepare(hidden_states, forward_batch)
with self.ffn_boundary.exit(forward_batch) as ffn_exit:
hidden_states = self.block_sparse_moe(hidden_states, forward_batch)
hidden_states = ffn_exit.finish(hidden_states)
hidden_states = self.block_sparse_moe(hidden_states, forward_batch)
hidden_states = self.ffn_boundary.finish(hidden_states, forward_batch)
return hidden_states
+3 -4
View File
@@ -1374,10 +1374,9 @@ class MiniMaxM3DecoderLayer(nn.Module):
hidden_states = self.attn_boundary.finish(hidden_states, forward_batch)
hidden_states = self.ffn_boundary.prepare(hidden_states, forward_batch)
with self.ffn_boundary.exit(forward_batch) as ffn_exit:
if self.is_layer_sparse or hidden_states.shape[0] != 0:
hidden_states = self.mlp(hidden_states, forward_batch=forward_batch)
return ffn_exit.finish(hidden_states)
if self.is_layer_sparse or hidden_states.shape[0] != 0:
hidden_states = self.mlp(hidden_states, forward_batch=forward_batch)
return self.ffn_boundary.finish(hidden_states, forward_batch)
class MiniMaxM3Model(nn.Module):
+2 -3
View File
@@ -264,9 +264,8 @@ class MixtralDecoderLayer(nn.Module):
# Fully Connected
hidden_states = self.attn_boundary.finish(hidden_states, forward_batch)
hidden_states = self.ffn_boundary.prepare(hidden_states, forward_batch)
with self.ffn_boundary.exit(forward_batch) as ffn_exit:
hidden_states = self.block_sparse_moe(hidden_states)
return ffn_exit.finish(hidden_states)
hidden_states = self.block_sparse_moe(hidden_states)
return self.ffn_boundary.finish(hidden_states, forward_batch)
class MixtralModel(nn.Module):
+2 -3
View File
@@ -1045,9 +1045,8 @@ class MossVLSelfAttentionDecoderLayer(nn.Module):
# MLP
hidden_states = self.attn_boundary.finish(hidden_states, forward_batch)
hidden_states = self.ffn_boundary.prepare(hidden_states, forward_batch)
with self.ffn_boundary.exit(forward_batch) as ffn_exit:
hidden_states = self.mlp(hidden_states)
hidden_states = ffn_exit.finish(hidden_states)
hidden_states = self.mlp(hidden_states)
hidden_states = self.ffn_boundary.finish(hidden_states, forward_batch)
return hidden_states
+2 -3
View File
@@ -262,9 +262,8 @@ class NanbeigeDecoderLayer(nn.Module):
# Fully Connected
hidden_states = self.attn_boundary.finish(hidden_states, forward_batch)
hidden_states = self.ffn_boundary.prepare(hidden_states, forward_batch)
with self.ffn_boundary.exit(forward_batch) as ffn_exit:
hidden_states = self.mlp(hidden_states)
return ffn_exit.finish(hidden_states)
hidden_states = self.mlp(hidden_states)
return self.ffn_boundary.finish(hidden_states, forward_batch)
class NanbeigeModel(nn.Module):
+4 -6
View File
@@ -384,9 +384,8 @@ class NemotronHMLPLikeDecoderLayer(nn.Module):
forward_batch: ForwardBatch,
):
hidden_states = self.boundary.prepare(hidden_states, forward_batch)
with self.boundary.exit(forward_batch) as ffn_exit:
hidden_states = self.mixer.forward(hidden_states)
return ffn_exit.finish(hidden_states)
hidden_states = self.mixer.forward(hidden_states)
return self.boundary.finish(hidden_states, forward_batch)
class NemotronHMLPDecoderLayer(NemotronHMLPLikeDecoderLayer):
@@ -467,9 +466,8 @@ class NemotronHAttnLikeDecoderLayer(nn.Module):
if forward_batch.forward_mode.is_idle():
return hidden_states
with self.boundary.exit(forward_batch) as mixer_exit:
hidden_states = self._forward_mixer(hidden_states, forward_batch)
return mixer_exit.finish(hidden_states)
hidden_states = self._forward_mixer(hidden_states, forward_batch)
return self.boundary.finish(hidden_states, forward_batch)
class NemotronHMambaDecoderLayer(NemotronHAttnLikeDecoderLayer):
+2 -3
View File
@@ -194,9 +194,8 @@ class DeciLMDecoderLayer(nn.Module):
# Fully Connected
if not self._is_no_op_ffn:
hidden_states = self.ffn_boundary.prepare(hidden_states, forward_batch)
with self.ffn_boundary.exit(forward_batch) as ffn_exit:
hidden_states = self.mlp(hidden_states)
hidden_states = ffn_exit.finish(hidden_states)
hidden_states = self.mlp(hidden_states)
hidden_states = self.ffn_boundary.finish(hidden_states, forward_batch)
return hidden_states
+2 -3
View File
@@ -325,9 +325,8 @@ class Qwen2DecoderLayer(nn.Module):
# Fully Connected
hidden_states = self.attn_boundary.finish(hidden_states, forward_batch)
hidden_states = self.ffn_boundary.prepare(hidden_states, forward_batch)
with self.ffn_boundary.exit(forward_batch) as ffn_exit:
hidden_states = self.mlp(hidden_states)
return ffn_exit.finish(hidden_states)
hidden_states = self.mlp(hidden_states)
return self.ffn_boundary.finish(hidden_states, forward_batch)
class Qwen2Model(nn.Module):
+2 -3
View File
@@ -1095,9 +1095,8 @@ class Qwen2MoeDecoderLayer(nn.Module):
hidden_states = self.attn_boundary.finish(hidden_states, forward_batch)
hidden_states = self.ffn_boundary.prepare(hidden_states, forward_batch)
with self.ffn_boundary.exit(forward_batch) as ffn_exit:
hidden_states = self.mlp(hidden_states, forward_batch)
hidden_states = ffn_exit.finish(hidden_states)
hidden_states = self.mlp(hidden_states, forward_batch)
hidden_states = self.ffn_boundary.finish(hidden_states, forward_batch)
return hidden_states
+2 -3
View File
@@ -423,11 +423,10 @@ class Qwen3DecoderLayer(nn.Module):
else None
),
)
with self.ffn_boundary.exit(forward_batch) as ffn_exit:
hidden_states = self.mlp(hidden_states, forward_batch=forward_batch)
hidden_states = self.mlp(hidden_states, forward_batch=forward_batch)
if _is_npu and get_cmo_stream():
wait_cmo_stream()
hidden_states = ffn_exit.finish(hidden_states)
hidden_states = self.ffn_boundary.finish(hidden_states, forward_batch)
return hidden_states
+2 -3
View File
@@ -904,9 +904,8 @@ class Qwen3MoeDecoderLayer(nn.Module):
hidden_states = self.attn_boundary.finish(hidden_states, forward_batch)
hidden_states = self.ffn_boundary.prepare(hidden_states, forward_batch)
with self.ffn_boundary.exit(forward_batch) as ffn_exit:
hidden_states = self.mlp(hidden_states, forward_batch)
hidden_states = ffn_exit.finish(hidden_states)
hidden_states = self.mlp(hidden_states, forward_batch)
hidden_states = self.ffn_boundary.finish(hidden_states, forward_batch)
return hidden_states
+8 -9
View File
@@ -479,15 +479,14 @@ def _apply_qwen3_next_mlp(
) -> torch.Tensor:
hidden_states = layer.attn_boundary.finish(hidden_states, forward_batch)
hidden_states = layer.ffn_boundary.prepare(hidden_states, forward_batch)
with layer.ffn_boundary.exit(forward_batch) as ffn_exit:
if isinstance(layer.mlp, Qwen2MoeSparseMoeBlock):
hidden_states = layer.mlp(
hidden_states,
forward_batch=forward_batch,
)
else:
hidden_states = layer.mlp(hidden_states)
hidden_states = ffn_exit.finish(hidden_states)
if isinstance(layer.mlp, Qwen2MoeSparseMoeBlock):
hidden_states = layer.mlp(
hidden_states,
forward_batch=forward_batch,
)
else:
hidden_states = layer.mlp(hidden_states)
hidden_states = layer.ffn_boundary.finish(hidden_states, forward_batch)
return hidden_states
+2 -3
View File
@@ -1080,9 +1080,8 @@ class SarvamMoEMLADecoderLayer(nn.Module):
)
hidden_states = self.attn_boundary.finish(hidden_states, forward_batch)
hidden_states = self.ffn_boundary.prepare(hidden_states, forward_batch)
with self.ffn_boundary.exit(forward_batch) as ffn_exit:
hidden_states = self.mlp(hidden_states, forward_batch)
hidden_states = ffn_exit.finish(hidden_states)
hidden_states = self.mlp(hidden_states, forward_batch)
hidden_states = self.ffn_boundary.finish(hidden_states, forward_batch)
return hidden_states
+2 -3
View File
@@ -325,9 +325,8 @@ class SDARBlock(nn.Module):
hidden_states = self.attn_boundary.finish(hidden_states, forward_batch)
hidden_states = self.ffn_boundary.prepare(hidden_states, forward_batch)
with self.ffn_boundary.exit(forward_batch) as ffn_exit:
hidden_states = self.mlp(hidden_states)
hidden_states = ffn_exit.finish(hidden_states)
hidden_states = self.mlp(hidden_states)
hidden_states = self.ffn_boundary.finish(hidden_states, forward_batch)
return hidden_states
+5 -6
View File
@@ -384,12 +384,11 @@ class SDARMoeBlock(nn.Module):
hidden_states = self.attn_boundary.finish(hidden_states, forward_batch)
hidden_states = self.ffn_boundary.prepare(hidden_states, forward_batch)
with self.ffn_boundary.exit(forward_batch) as ffn_exit:
hidden_states = self.mlp(
hidden_states,
forward_batch=forward_batch,
)
hidden_states = ffn_exit.finish(hidden_states)
hidden_states = self.mlp(
hidden_states,
forward_batch=forward_batch,
)
hidden_states = self.ffn_boundary.finish(hidden_states, forward_batch)
return hidden_states
+2 -3
View File
@@ -296,9 +296,8 @@ class Spark2_5DecoderLayer(nn.Module):
)
hidden_states = self.attn_boundary.finish(hidden_states, forward_batch)
hidden_states = self.ffn_boundary.prepare(hidden_states, forward_batch)
with self.ffn_boundary.exit(forward_batch) as ffn_exit:
hidden_states = self.mlp(hidden_states)
return ffn_exit.finish(hidden_states)
hidden_states = self.mlp(hidden_states)
return self.ffn_boundary.finish(hidden_states, forward_batch)
class Spark2_5Model(nn.Module):
+5 -6
View File
@@ -417,12 +417,11 @@ class Step3TextDecoderLayer(nn.Module):
hidden_states = self.attn_boundary.finish(hidden_states, forward_batch)
hidden_states = self.ffn_boundary.prepare(hidden_states, forward_batch)
with self.ffn_boundary.exit(forward_batch) as ffn_exit:
if self.use_moe:
hidden_states = self.moe_mlp_forward(hidden_states)
else:
hidden_states = self.mlp(hidden_states)
hidden_states = ffn_exit.finish(hidden_states)
if self.use_moe:
hidden_states = self.moe_mlp_forward(hidden_states)
else:
hidden_states = self.mlp(hidden_states)
hidden_states = self.ffn_boundary.finish(hidden_states, forward_batch)
return hidden_states
+7 -9
View File
@@ -565,16 +565,14 @@ class Step3p5DecoderLayer(nn.Module):
hidden_states = self.ffn_boundary.prepare(hidden_states, forward_batch)
if self.use_moe:
with self.ffn_boundary.exit(forward_batch) as ffn_exit:
share_output = self.share_expert(hidden_states)
hidden_states = self.moe(
hidden_states, forward_batch, shared_output=share_output
)
return ffn_exit.finish(hidden_states)
share_output = self.share_expert(hidden_states)
hidden_states = self.moe(
hidden_states, forward_batch, shared_output=share_output
)
return self.ffn_boundary.finish(hidden_states, forward_batch)
with self.ffn_boundary.exit(forward_batch) as ffn_exit:
hidden_states = self.mlp(hidden_states)
return ffn_exit.finish(hidden_states)
hidden_states = self.mlp(hidden_states)
return self.ffn_boundary.finish(hidden_states, forward_batch)
class Step3p5Model(nn.Module):
+5 -6
View File
@@ -1558,12 +1558,11 @@ class XllmDecoderLayer(nn.Module):
hidden_states = self.attn_boundary.finish(hidden_states, forward_batch)
hidden_states = self.ffn_boundary.prepare(hidden_states, forward_batch)
with self.ffn_boundary.exit(forward_batch) as ffn_exit:
if isinstance(self.mlp, XllmMLP):
hidden_states = self.mlp(hidden_states)
else:
hidden_states = self.mlp(hidden_states, forward_batch)
hidden_states = ffn_exit.finish(hidden_states)
if isinstance(self.mlp, XllmMLP):
hidden_states = self.mlp(hidden_states)
else:
hidden_states = self.mlp(hidden_states, forward_batch)
hidden_states = self.ffn_boundary.finish(hidden_states, forward_batch)
return hidden_states
+4 -5
View File
@@ -1397,11 +1397,10 @@ class ZayaDecoderMLPLayer(nn.Module):
prev_router_hidden_states: Optional[torch.Tensor] = None,
) -> tuple[torch.Tensor, Optional[torch.Tensor]]:
hidden_states = self.ffn_boundary.prepare(hidden_states, forward_batch)
with self.ffn_boundary.exit(forward_batch) as ffn_exit:
hidden_states, prev_router_hidden_states = self.zaya_block(
hidden_states, prev_router_hidden_states
)
hidden_states = ffn_exit.finish(hidden_states)
hidden_states, prev_router_hidden_states = self.zaya_block(
hidden_states, prev_router_hidden_states
)
hidden_states = self.ffn_boundary.finish(hidden_states, forward_batch)
return hidden_states, prev_router_hidden_states
-3
View File
@@ -670,11 +670,9 @@ class ForwardFlags:
"is_extend_in_batch": False,
# Per-layer MLP collective control (set by decoder via scoped()
# around the MLP / MoE / hybrid mixer call).
# fuse_mlp_allreduce: next residual+LN absorbs the post-MLP all-reduce.
# mlp_reduce_scatter: postprocess will reduce-scatter (skip MLP AR).
# flashinfer_trtllm_bypass: deepseek dual-stream graph topk bypass.
# defer_moe_finalize: next layer's fused collective absorbs the finalize.
"fuse_mlp_allreduce": False,
"mlp_reduce_scatter": False,
"flashinfer_trtllm_bypass": False,
"defer_moe_finalize": False,
@@ -694,7 +692,6 @@ class ForwardFlags:
"attn_input_scattered",
"attn_inputs",
"is_extend_in_batch",
"fuse_mlp_allreduce",
"mlp_reduce_scatter",
"flashinfer_trtllm_bypass",
"defer_moe_finalize",