[Refactor] Select reduction fusion at the consumer (#41551)
This commit is contained in:
@@ -40,7 +40,6 @@ from sglang.srt.layers.communicator.boundary import (
|
||||
from sglang.srt.layers.communicator.layer import (
|
||||
FfnCompletion,
|
||||
FfnExit,
|
||||
FfnExitFusion,
|
||||
LayerCommunicator,
|
||||
LayerFacts,
|
||||
MHCLayerCommunicator,
|
||||
@@ -91,7 +90,6 @@ __all__ = [
|
||||
"FUSE_ALLREDUCE_MAX_BATCH_SIZE",
|
||||
"FfnCompletion",
|
||||
"FfnExit",
|
||||
"FfnExitFusion",
|
||||
"FusedMlpInput",
|
||||
"HandoffOutput",
|
||||
"LayerCommunicator",
|
||||
|
||||
@@ -49,6 +49,7 @@ from sglang.srt.layers.communicator.ops import (
|
||||
_read_input,
|
||||
move_rows,
|
||||
tp_reduce_scatter,
|
||||
tp_slice,
|
||||
)
|
||||
from sglang.srt.layers.communicator.residual import (
|
||||
LayerResidual,
|
||||
@@ -551,6 +552,7 @@ class StageEntry(msgspec.Struct, frozen=True):
|
||||
capture_move: Optional[Callable] = None
|
||||
capture_move_allocates: bool = False
|
||||
capture_preserves_residual: Optional[Callable] = None
|
||||
input_sum: Optional[SumGroup] = None
|
||||
|
||||
|
||||
class BoundarySteps(msgspec.Struct, frozen=True):
|
||||
@@ -724,6 +726,20 @@ def _bind_consumer(
|
||||
cp_moves=cp_moves,
|
||||
enters_stack=enters_stack,
|
||||
)
|
||||
completed_step = None
|
||||
if edge.produced.always_leaves:
|
||||
completed_step, _, _ = _select_input_steps(
|
||||
msgspec.structs.replace(edge.produced, always_leaves=False),
|
||||
residual=edge.residual,
|
||||
residual_to=edge.residual_to,
|
||||
need=edge.need,
|
||||
adds_plainly=adds_plainly,
|
||||
fusions=(),
|
||||
force_layernorm_before_gather=force_layernorm_before_gather,
|
||||
residual_joins_sum=False,
|
||||
cp_moves=cp_moves,
|
||||
enters_stack=enters_stack,
|
||||
)
|
||||
# A written stream entering the first physical layer must not run enter
|
||||
# again (e.g. MHC expansion). Both alternatives are bound at construction.
|
||||
written_step = None
|
||||
@@ -747,7 +763,8 @@ def _bind_consumer(
|
||||
step=step,
|
||||
adds_plainly=adds_plainly,
|
||||
carried_fusions=carried_fusions if plain else (),
|
||||
owes_by_construction=edge.produced.always_leaves,
|
||||
expected_sum=edge.produced.group if edge.produced.always_leaves else None,
|
||||
completed_step=completed_step,
|
||||
written_step=written_step,
|
||||
),
|
||||
input_move=input_move,
|
||||
@@ -826,12 +843,18 @@ def _select_boundary_steps(
|
||||
handoff=attention_handoff,
|
||||
capture_move=into_attention.capture_move,
|
||||
capture_move_allocates=into_attention.capture_move_allocates,
|
||||
input_sum=edges.into_attention.produced.group
|
||||
if edges.into_attention.produced.always_leaves
|
||||
else None,
|
||||
),
|
||||
ffn=StageEntry(
|
||||
prepare=into_ffn.prepare,
|
||||
input_rows=into_ffn.input_rows,
|
||||
input_move=into_ffn.input_move,
|
||||
fused=into_ffn.fused,
|
||||
input_sum=edges.into_ffn.produced.group
|
||||
if edges.into_ffn.produced.always_leaves
|
||||
else None,
|
||||
),
|
||||
ffn_output=edges.out_of_ffn.produced,
|
||||
ffn_output_move=out_of_ffn.output_move,
|
||||
@@ -895,17 +918,19 @@ def _select_input_steps(
|
||||
if residual == residual_to:
|
||||
# The residual stays on each rank's slice while the stage takes the
|
||||
# rows around it (MHC on an input-scattered batch).
|
||||
if gathered or owes is not SumGroup.ATTN_TP:
|
||||
if gathered or owes not in (None, SumGroup.ATTN_TP):
|
||||
raise NotImplementedError(f"{produced=} {residual=} {need=}")
|
||||
return (
|
||||
partial(_mlp_input_on_residual_shard, read=read),
|
||||
partial(
|
||||
_mlp_input_on_residual_shard, read=read, reduces=owes is not None
|
||||
),
|
||||
(),
|
||||
None,
|
||||
)
|
||||
# A reduce-scatter completes the TP sum onto each rank's slice, which
|
||||
# the stage takes: its group is the TP group without attention DP or CP.
|
||||
if (
|
||||
owes is not SumGroup.TP
|
||||
owes not in (None, SumGroup.TP)
|
||||
or residual.sharded
|
||||
or TokenAxis.ATTN_TP_SCATTER not in need.gathers_itself
|
||||
or residual_to.sharded != {TokenAxis.ATTN_TP_SCATTER}
|
||||
@@ -914,7 +939,7 @@ def _select_input_steps(
|
||||
return (
|
||||
partial(
|
||||
_read_input,
|
||||
layer_input=tp_reduce_scatter,
|
||||
layer_input=tp_reduce_scatter if owes is not None else tp_slice,
|
||||
enters_stack=enters_stack,
|
||||
read=read,
|
||||
),
|
||||
@@ -974,7 +999,17 @@ def _select_input_steps(
|
||||
if not gathered:
|
||||
if owes is None:
|
||||
if gathers_residual:
|
||||
raise NotImplementedError(f"{produced=} {residual=} {need=}")
|
||||
return (
|
||||
partial(
|
||||
_mlp_input_without_dp,
|
||||
gathers_residual=True,
|
||||
fusions=(),
|
||||
group=None,
|
||||
read=read,
|
||||
),
|
||||
(),
|
||||
None,
|
||||
)
|
||||
return (
|
||||
partial(
|
||||
_read_input,
|
||||
|
||||
@@ -14,7 +14,6 @@ from typing import Callable, Optional, Sequence
|
||||
import torch
|
||||
|
||||
from sglang.srt.layers.communicator import (
|
||||
FfnExitFusion,
|
||||
FusedMlpInput,
|
||||
HandoffOutput,
|
||||
LayerCommunicator,
|
||||
@@ -201,33 +200,27 @@ class CuteDSLFusion:
|
||||
construction. At the attention input: the MoE finalize + AR + add + norm of
|
||||
a handoff the previous layer left, and the AR + add + norm of a sum it left.
|
||||
At the FFN input: the attention output's AR + add + norm. At the FFN exit:
|
||||
handing the MoE finalize, or the all-reduce, to the next layer's kernel.
|
||||
install_cutedsl_fusion gives it the model's service and chooses the exit
|
||||
from the layer after this one; until then no kernel takes a batch."""
|
||||
handing off a deferred MoE finalize with its unfused completion.
|
||||
install_cutedsl_fusion supplies the service and producer capability;
|
||||
consumer eligibility is checked only where the output is consumed."""
|
||||
|
||||
def __init__(self) -> None:
|
||||
self.service: CuteDSLFusionService | None = None
|
||||
# The FFN exit's choices, from what install() knows of the next layer.
|
||||
# Producer capabilities; consumer kernel selection is independent.
|
||||
self.hands_off_finalize = False
|
||||
self.next_input_absorbs = False
|
||||
self.requires_local_reduction = False
|
||||
|
||||
def install(
|
||||
self,
|
||||
service: CuteDSLFusionService,
|
||||
*,
|
||||
hands_off_finalize: bool,
|
||||
next_input_absorbs: bool,
|
||||
output_is_replicated: bool,
|
||||
) -> None:
|
||||
"""Take the model's fusion service and choose the FFN exit's kernels:
|
||||
the MoE may hand off its finalize when it can and something after it
|
||||
takes the handoff (``hands_off_finalize``); the FFN may leave its
|
||||
all-reduce to the next layer's AR + norm when that layer has one, unless
|
||||
a replicated output follows the reduction (moving it would scale that
|
||||
by tp)."""
|
||||
"""Install the shared workspace and this producer's capabilities."""
|
||||
self.service = service
|
||||
self.hands_off_finalize = hands_off_finalize
|
||||
self.next_input_absorbs = next_input_absorbs and not output_is_replicated
|
||||
self.requires_local_reduction = output_is_replicated
|
||||
|
||||
def attention_input(self, layer: LayerCommunicator) -> tuple:
|
||||
return (
|
||||
@@ -254,12 +247,6 @@ class CuteDSLFusion:
|
||||
)
|
||||
return ()
|
||||
|
||||
def ffn_exit(self, layer: LayerCommunicator) -> tuple:
|
||||
return (
|
||||
partial(self._defer_moe_finalize, layer),
|
||||
partial(self._absorb_all_reduce, layer),
|
||||
)
|
||||
|
||||
def _finalize_output_and_update_and_read_residual(
|
||||
self, layer, owed, residual, forward_batch, post_residual_addition
|
||||
):
|
||||
@@ -330,26 +317,28 @@ class CuteDSLFusion:
|
||||
and not get_exec().comm.enable_quant_communications
|
||||
)
|
||||
|
||||
def _defer_moe_finalize(
|
||||
self, layer, forward_batch: ForwardBatch
|
||||
) -> Optional[FfnExitFusion]:
|
||||
"""The MoE hands its unfinalized output to the next layer's finalize +
|
||||
all-reduce + norm."""
|
||||
if self.hands_off_finalize and self._should_use_finalize(
|
||||
layer, forward_batch, int(forward_batch.input_ids.shape[0])
|
||||
):
|
||||
return FfnExitFusion.DEFER_MOE_FINALIZE
|
||||
return None
|
||||
def can_defer_all_reduce(self, layer, forward_batch: ForwardBatch) -> bool:
|
||||
"""Preserve the producer's fused sum path when this workspace is usable.
|
||||
|
||||
def _absorb_all_reduce(
|
||||
self, layer, forward_batch: ForwardBatch
|
||||
) -> Optional[FfnExitFusion]:
|
||||
"""The next layer's AR + norm takes the post-experts all-reduce."""
|
||||
if self.next_input_absorbs and self._can_consume_post_moe_all_reduce(
|
||||
layer, forward_batch, int(forward_batch.input_ids.shape[0])
|
||||
):
|
||||
return FfnExitFusion.NEXT_INPUT
|
||||
return None
|
||||
The next consumer still selects its kernel and completes the sum with
|
||||
an ordinary all-reduce if that kernel declines the actual input.
|
||||
"""
|
||||
return (
|
||||
self._common_eligible(
|
||||
layer, forward_batch, int(forward_batch.input_ids.shape[0])
|
||||
)
|
||||
and not get_exec().comm.enable_quant_communications
|
||||
)
|
||||
|
||||
def can_defer_finalize(self, layer, forward_batch: ForwardBatch) -> bool:
|
||||
"""Whether this producer can emit a handoff with an unfused fallback."""
|
||||
return (
|
||||
self.hands_off_finalize
|
||||
and not self.requires_local_reduction
|
||||
and self._should_use_finalize(
|
||||
layer, forward_batch, int(forward_batch.input_ids.shape[0])
|
||||
)
|
||||
)
|
||||
|
||||
def _common_eligible(self, layer, forward_batch: ForwardBatch, m: int) -> bool:
|
||||
parallel = get_parallel()
|
||||
@@ -383,7 +372,6 @@ def install_cutedsl_fusion(
|
||||
rms_epsilon: float,
|
||||
can_defer_finalize: _LayerPredicate,
|
||||
requires_local_reduction: _LayerPredicate | None = None,
|
||||
final_norm_consumes_handoff: bool = False,
|
||||
label: str,
|
||||
) -> CuteDSLFusionService | None:
|
||||
"""One shared workspace handle per fusion-enabled layer, or None.
|
||||
@@ -419,23 +407,15 @@ def install_cutedsl_fusion(
|
||||
rms_epsilon=rms_epsilon,
|
||||
)
|
||||
hands_off = 0
|
||||
for index, layer in enumerate(layers):
|
||||
for layer in layers:
|
||||
fusion = _fusion_of(layer)
|
||||
if fusion is None:
|
||||
continue
|
||||
successor = layers[index + 1] if index + 1 < len(layers) else None
|
||||
if successor is None:
|
||||
has_consumer = final_norm_consumes_handoff
|
||||
else:
|
||||
has_consumer = _fusion_of(successor) is not None
|
||||
hands_off_finalize = bool(can_defer_finalize(layer)) and has_consumer
|
||||
hands_off_finalize = bool(can_defer_finalize(layer))
|
||||
hands_off += hands_off_finalize
|
||||
fusion.install(
|
||||
service,
|
||||
hands_off_finalize=hands_off_finalize,
|
||||
# False on the last layer: the final norm does not all-reduce.
|
||||
next_input_absorbs=successor is not None
|
||||
and _fusion_of(successor) is not None,
|
||||
output_is_replicated=requires_local_reduction is not None
|
||||
and bool(requires_local_reduction(layer)),
|
||||
)
|
||||
|
||||
@@ -14,7 +14,6 @@
|
||||
"""The communicator a model layer uses."""
|
||||
|
||||
from dataclasses import dataclass
|
||||
from enum import Enum, auto
|
||||
from functools import cached_property, partial
|
||||
from typing import Callable, Optional, Protocol, Tuple, Union
|
||||
|
||||
@@ -49,7 +48,6 @@ from sglang.srt.layers.communicator.layout import (
|
||||
SumGroup,
|
||||
TokenAxis,
|
||||
_batch_shards_over_cp,
|
||||
_batch_size,
|
||||
_ffn_has_tokens,
|
||||
_gathers_over_attention_cp,
|
||||
_generic_prefill_cp_shards_tokens,
|
||||
@@ -86,13 +84,13 @@ from sglang.srt.layers.communicator.residual.access import (
|
||||
written,
|
||||
)
|
||||
from sglang.srt.layers.communicator.residual.add_norm import (
|
||||
ADD,
|
||||
PLAIN_RESIDUAL,
|
||||
aiter_all_reduce_fusion_enabled_for,
|
||||
apply_aiter_all_reduce_fusion,
|
||||
apply_flashinfer_allreduce_fusion,
|
||||
)
|
||||
from sglang.srt.layers.communicator.residual.mhc import MHCState
|
||||
from sglang.srt.layers.communicator.residual.stream import ResidualStream
|
||||
from sglang.srt.layers.communicator.residual.stream import DeclaredSum, ResidualStream
|
||||
from sglang.srt.layers.dp_attention import (
|
||||
is_dp_attention_enabled,
|
||||
is_enable_moe_cp_allgather,
|
||||
@@ -156,11 +154,39 @@ class LayerFacts:
|
||||
)
|
||||
|
||||
|
||||
def _unfused_completion_matches_the_ffn(forward_batch: ForwardBatch) -> bool:
|
||||
"""Whether the next layer's all-reduce is the one the FFN would have run
|
||||
itself, as the MoE declares it (post_experts_sum_is_one_all_reduce), on a
|
||||
batch the FFN runs."""
|
||||
return _ffn_has_tokens(forward_batch) and post_experts_sum_is_one_all_reduce()
|
||||
def _can_defer_ffn_reduction(forward_batch: ForwardBatch, boundary=None) -> bool:
|
||||
"""Admit ordinary single-sum outputs and the existing fused LoRA path.
|
||||
|
||||
A fused-kernel fallback completes the partial-output contract selected
|
||||
before the producer ran, without recomputing the producer's LoRA path.
|
||||
"""
|
||||
if not _ffn_has_tokens(forward_batch):
|
||||
return False
|
||||
if post_experts_sum_is_one_all_reduce():
|
||||
return True
|
||||
# LoRA-B is replicated and linear. TP1 shared experts add on rank zero
|
||||
# when the sum is deferred. Preserve these fused paths, but keep their
|
||||
# producer-side reduction order when no fused consumer is enabled.
|
||||
if not (
|
||||
(get_lora().enable_lora or envs.SGLANG_SHARED_EXPERT_TP1.get())
|
||||
and get_moe_a2a_backend().is_none()
|
||||
and not get_exec().comm.enable_quant_communications
|
||||
and post_experts_reduction_group() is get_parallel().tp_group
|
||||
):
|
||||
return False
|
||||
if (
|
||||
boundary is not None
|
||||
and boundary.fusions is not None
|
||||
and boundary.fusions.can_defer_all_reduce(boundary, forward_batch)
|
||||
):
|
||||
return True
|
||||
if apply_flashinfer_allreduce_fusion(forward_batch.input_ids.shape[0]):
|
||||
return True
|
||||
# Aiter also checks width and bytes, so use the actual residual storage.
|
||||
residual = forward_batch.residual_stream.residual
|
||||
return residual is not None and apply_aiter_all_reduce_fusion(
|
||||
residual, forward_batch
|
||||
)
|
||||
|
||||
|
||||
class StageCommunicator:
|
||||
@@ -205,12 +231,20 @@ class StageCommunicator:
|
||||
stream = residual_batch.current(forward_batch)
|
||||
if residual is not stream:
|
||||
raise RuntimeError("residual alias belongs to a different invocation")
|
||||
if stream is not None:
|
||||
if stream.pending is not None:
|
||||
call["update"] = stream.pending.update
|
||||
if stream.pending is None and stream.residual is not None:
|
||||
call["written"] = True
|
||||
hidden_states, residual = stream.input(hidden_states)
|
||||
if (
|
||||
stream.pending is None
|
||||
and stream.residual is None
|
||||
and entry.input_sum is not None
|
||||
):
|
||||
hidden_states = stream.leave(
|
||||
hidden_states, ADD, declared_sum=entry.input_sum
|
||||
)
|
||||
if stream.pending is not None:
|
||||
call["update"] = stream.pending.update
|
||||
call["pending"] = stream.pending
|
||||
if stream.pending is None and stream.residual is not None:
|
||||
call["written"] = True
|
||||
hidden_states, residual = stream.input(hidden_states)
|
||||
hidden_states, residual = entry.prepare(
|
||||
hidden_states, residual, forward_batch, self.norm, context, **call
|
||||
)
|
||||
@@ -239,9 +273,15 @@ class LayerFusions(Protocol):
|
||||
|
||||
def ffn_input(self, layer: "LayerCommunicator") -> Tuple["FusedMlpInput", ...]: ...
|
||||
|
||||
def ffn_exit(
|
||||
self, layer: "LayerCommunicator"
|
||||
) -> Tuple[Callable[[ForwardBatch], Optional["FfnExitFusion"]], ...]: ...
|
||||
requires_local_reduction: bool
|
||||
|
||||
def can_defer_all_reduce(
|
||||
self, layer: "LayerCommunicator", forward_batch: ForwardBatch
|
||||
) -> bool: ...
|
||||
|
||||
def can_defer_finalize(
|
||||
self, layer: "LayerCommunicator", forward_batch: ForwardBatch
|
||||
) -> bool: ...
|
||||
|
||||
|
||||
class LayerCommunicator:
|
||||
@@ -255,7 +295,6 @@ class LayerCommunicator:
|
||||
# Communicators built without __init__ (e.g. test doubles) publish no LoRA
|
||||
# layout and try no fused kernel at the FFN exit.
|
||||
_publish_lora_layout: bool = False
|
||||
_ffn_exit_fusions: Tuple[Callable, ...] = ()
|
||||
# A plain residual unless the layer is built with its own.
|
||||
_residual: LayerResidual = PLAIN_RESIDUAL
|
||||
# The fused kernels a backend gives the layer, if any.
|
||||
@@ -303,8 +342,6 @@ class LayerCommunicator:
|
||||
)
|
||||
# The fused kernels every batch's attention input tries first.
|
||||
self._attn_input_fusions = self._select_attn_input_fusions()
|
||||
# The fused kernels of the next layer's input the FFN exit tries first.
|
||||
self._ffn_exit_fusions = self._select_ffn_exit_fusions()
|
||||
self._speculative_algo = SpeculativeAlgorithm.from_string(
|
||||
get_spec().speculative_algorithm
|
||||
)
|
||||
@@ -433,6 +470,9 @@ class LayerCommunicator:
|
||||
handoff=None if is_ffn else _hand_qkv_hook_its_input,
|
||||
fused=into.fused,
|
||||
capture_move=into.capture_move,
|
||||
input_sum=into_edge.produced.group
|
||||
if into_edge.produced.always_leaves
|
||||
else None,
|
||||
)
|
||||
self._steps = BoundarySteps(
|
||||
attention=None if is_ffn else entry,
|
||||
@@ -641,7 +681,11 @@ class LayerCommunicator:
|
||||
|
||||
def snapshot(self, hidden_states, residual, *, at_input: bool = False):
|
||||
group = None
|
||||
if residual is not None and self.stage_edges is not None:
|
||||
if (
|
||||
residual is not None
|
||||
and not isinstance(residual, ResidualStream)
|
||||
and self.stage_edges is not None
|
||||
):
|
||||
edge = self.stage_edges[0 if at_input else 1]
|
||||
if edge.produced.always_leaves:
|
||||
group = _sum_group(edge.produced.group)
|
||||
@@ -663,7 +707,12 @@ class LayerCommunicator:
|
||||
if storage is not None and storage.shape[0] == 0:
|
||||
return hidden_states, None
|
||||
hidden_states = (
|
||||
residual.complete(hidden_states)
|
||||
(
|
||||
hidden_states
|
||||
if residual.pending is not None
|
||||
and isinstance(residual.pending.owed, DeclaredSum)
|
||||
else residual.complete(hidden_states)
|
||||
)
|
||||
if isinstance(residual, ResidualStream)
|
||||
else reduce_output(hidden_states)
|
||||
)
|
||||
@@ -675,8 +724,10 @@ class LayerCommunicator:
|
||||
residual_in_hidden=self._residual.ffn_update.at_producer,
|
||||
allow_missing_residual=allow_missing_residual,
|
||||
)
|
||||
steps = self._batch_steps(forward_batch)
|
||||
entry = steps.attention or steps.ffn
|
||||
hidden_states, stream = ResidualStream.arrive(
|
||||
hidden_states, residual, self._residual.ffn_update
|
||||
hidden_states, residual, ADD, declared_sum=entry.input_sum
|
||||
)
|
||||
forward_batch.residual_stream = stream
|
||||
return hidden_states, stream
|
||||
@@ -934,7 +985,9 @@ class LayerCommunicator:
|
||||
if isinstance(residual, ResidualStream) and self.stage_edges is None:
|
||||
# The paired boundary records the attention contribution once.
|
||||
hidden_states = residual.leave(
|
||||
hidden_states, self._residual.attention_update
|
||||
hidden_states,
|
||||
self._residual.attention_update,
|
||||
declared_sum=steps.ffn.input_sum,
|
||||
)
|
||||
return self.ffn.prepare(
|
||||
hidden_states,
|
||||
@@ -969,16 +1022,34 @@ class LayerCommunicator:
|
||||
residual,
|
||||
stream,
|
||||
self._batch_steps(forward_batch).ffn_output.update,
|
||||
self._declared_ffn_sum(
|
||||
self._batch_steps(forward_batch),
|
||||
self._ffn_leaves_sum_to_reduce_scatter(
|
||||
forward_batch, self._postprocess_dp_step(forward_batch)
|
||||
),
|
||||
),
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _leave_ffn_output(hidden_states, residual, stream, update):
|
||||
def _leave_ffn_output(hidden_states, residual, stream, update, declared_sum=None):
|
||||
if stream is None:
|
||||
return hidden_states, residual
|
||||
stream.residual = residual
|
||||
if update.at_producer:
|
||||
return stream.write(hidden_states), stream
|
||||
return stream.leave(hidden_states, update), stream
|
||||
return stream.leave(hidden_states, update, declared_sum=declared_sum), stream
|
||||
|
||||
@staticmethod
|
||||
def _declared_ffn_sum(steps, skipped_reduction):
|
||||
# The input-scattered path leaves the sum for the next input's TP
|
||||
# reduce-scatter. Other skip paths complete it in the output move.
|
||||
if (
|
||||
skipped_reduction
|
||||
and not steps.returns_over_dp
|
||||
and not steps.ffn_output_move_completes_sum
|
||||
):
|
||||
return SumGroup.TP
|
||||
return None
|
||||
|
||||
def _local_token_move_can_go_to_next_layer(
|
||||
self, forward_batch: ForwardBatch
|
||||
@@ -1042,14 +1113,13 @@ class LayerCommunicator:
|
||||
mlp_reduce_scatter=mlp_reduce_scatter,
|
||||
complete=complete_now,
|
||||
)
|
||||
fusion = next(
|
||||
filter(None, (fused(forward_batch) for fused in self._ffn_exit_fusions)),
|
||||
None,
|
||||
defer_moe_finalize = (
|
||||
self.fusions is not None
|
||||
and self.fusions.can_defer_finalize(self, forward_batch)
|
||||
)
|
||||
defer_moe_finalize = fusion is FfnExitFusion.DEFER_MOE_FINALIZE
|
||||
# A fused kernel that takes the sum, a handoff included, skips the
|
||||
# post-experts all-reduce.
|
||||
fuse_mlp_allreduce = fusion is not None or self._ffn_sum_moves_to_next_layer(
|
||||
# Producers declare remaining work independently of the kernel chosen
|
||||
# by the consumer. Every handoff also carries an unfused completion.
|
||||
fuse_mlp_allreduce = defer_moe_finalize or self._ffn_sum_moves_to_next_layer(
|
||||
forward_batch, mlp_reduce_scatter=mlp_reduce_scatter, dp_step=dp_step
|
||||
)
|
||||
if fuse_mlp_allreduce:
|
||||
@@ -1205,28 +1275,12 @@ class LayerCommunicator:
|
||||
return stream.leave(output, update), stream
|
||||
return output, residual
|
||||
|
||||
def _select_ffn_exit_fusions(
|
||||
self,
|
||||
) -> Tuple[Callable[[ForwardBatch], Optional["FfnExitFusion"]], ...]:
|
||||
"""The fused kernels of the next layer's input that may take this layer's
|
||||
FFN sum, in the order they are tried. Each returns what the exit does
|
||||
when its kernel takes this batch, or None. A backend's come first."""
|
||||
given = self.fusions.ffn_exit(self) if self.fusions else ()
|
||||
return (*given, self._next_input_norm_takes_ffn_sum)
|
||||
|
||||
def _next_input_norm_takes_ffn_sum(
|
||||
self, forward_batch: ForwardBatch
|
||||
) -> Optional["FfnExitFusion"]:
|
||||
"""The aiter / flashinfer AR + add + norm of the next layer's input."""
|
||||
if self.should_fuse_mlp_allreduce_with_next_layer(forward_batch):
|
||||
return FfnExitFusion.NEXT_INPUT
|
||||
return None
|
||||
|
||||
def _ffn_sum_can_move_to_next_layer(self, forward_batch: ForwardBatch) -> bool:
|
||||
# Under the MoE-CP all-gather the fusion path would skip postprocess_layer
|
||||
# and its MoE-CP scatter, leaving hidden_states longer than the residual.
|
||||
if (
|
||||
is_enable_moe_cp_allgather()
|
||||
(self.fusions is not None and self.fusions.requires_local_reduction)
|
||||
or is_enable_moe_cp_allgather()
|
||||
or not self._batch_steps(forward_batch).ffn_sum_is_movable
|
||||
):
|
||||
return False
|
||||
@@ -1252,29 +1306,6 @@ class LayerCommunicator:
|
||||
|
||||
return not get_attn_tp_context().input_scattered
|
||||
|
||||
# NOTE: This function will cause torch recompilation
|
||||
def should_fuse_mlp_allreduce_with_next_layer(
|
||||
self, forward_batch: ForwardBatch
|
||||
) -> bool:
|
||||
if not self._ffn_sum_can_move_to_next_layer(forward_batch):
|
||||
return False
|
||||
batch_size = _batch_size(forward_batch)
|
||||
return (
|
||||
(
|
||||
apply_flashinfer_allreduce_fusion(batch_size)
|
||||
or (
|
||||
_use_aiter
|
||||
and batch_size > 0
|
||||
and get_parallel().tp_size != 6
|
||||
and not is_dp_attention_enabled()
|
||||
and get_moe_a2a_backend().is_none()
|
||||
and aiter_all_reduce_fusion_enabled_for(forward_batch.forward_mode)
|
||||
)
|
||||
)
|
||||
and (not self.is_last_layer)
|
||||
and (self._context.tp_size > 1)
|
||||
)
|
||||
|
||||
def _ffn_sum_moves_to_next_layer(
|
||||
self,
|
||||
forward_batch: ForwardBatch,
|
||||
@@ -1289,7 +1320,7 @@ class LayerCommunicator:
|
||||
self._context.tp_size > 1
|
||||
and not self.is_last_layer
|
||||
and self._ffn_sum_can_move_to_next_layer(forward_batch)
|
||||
and _unfused_completion_matches_the_ffn(forward_batch)
|
||||
and _can_defer_ffn_reduction(forward_batch, self)
|
||||
and not mlp_reduce_scatter
|
||||
# Under attention DP the next layer must also run postprocess's
|
||||
# scatter back to this rank's tokens, and nothing more.
|
||||
@@ -1303,16 +1334,6 @@ class LayerCommunicator:
|
||||
)
|
||||
|
||||
|
||||
class FfnExitFusion(Enum):
|
||||
"""What an FFN exit does when a fused kernel of the next layer's input takes
|
||||
its sum."""
|
||||
|
||||
# The MoE hands its unfinalized output on; the next input finalizes it.
|
||||
DEFER_MOE_FINALIZE = auto()
|
||||
# The FFN leaves its all-reduce to the next input norm.
|
||||
NEXT_INPUT = auto()
|
||||
|
||||
|
||||
class FfnCompletion(msgspec.Struct, frozen=True):
|
||||
"""One FFN's reduction decision. The flags are published while the FFN runs;
|
||||
``complete(hidden_states, residual)`` then either wraps the output for the
|
||||
@@ -1337,17 +1358,20 @@ class MixerExit:
|
||||
``with`` block ``fuse_mlp_allreduce`` on ``get_forward()`` tells its
|
||||
row-parallel output projection to skip the all-reduce. It skips when the
|
||||
stage's output always leaves its sum (to an FFN stage, which completes it in
|
||||
its input), and when it may leave it and the fused kernel takes it into the
|
||||
next input norm."""
|
||||
its input), and when its output declaration permits deferring the sum. The
|
||||
consumer decides whether to fuse its completion with the input norm."""
|
||||
|
||||
__slots__ = ("skips_reduction", "_hands_on", "_scope", "_update")
|
||||
__slots__ = ("skips_reduction", "_hands_on", "_scope", "_update", "_declared_sum")
|
||||
|
||||
def __init__(self, communicator: LayerCommunicator, forward_batch: ForwardBatch):
|
||||
produced = communicator._batch_steps(forward_batch).ffn_output
|
||||
self._update = produced.update
|
||||
self._declared_sum = produced.group if produced.always_leaves else None
|
||||
self._hands_on = (
|
||||
produced.leaves_for_next_layer
|
||||
and communicator.should_fuse_mlp_allreduce_with_next_layer(forward_batch)
|
||||
and communicator._context.tp_size > 1
|
||||
and not is_dp_attention_enabled()
|
||||
and communicator._ffn_sum_can_move_to_next_layer(forward_batch)
|
||||
)
|
||||
self.skips_reduction = produced.always_leaves or self._hands_on
|
||||
self._scope = get_forward().scoped(fuse_mlp_allreduce=self.skips_reduction)
|
||||
@@ -1367,7 +1391,9 @@ class MixerExit:
|
||||
hidden_states, group=get_parallel().tp_group
|
||||
)
|
||||
if isinstance(residual, ResidualStream):
|
||||
return residual.leave(hidden_states, self._update), residual
|
||||
return residual.leave(
|
||||
hidden_states, self._update, declared_sum=self._declared_sum
|
||||
), residual
|
||||
return hidden_states
|
||||
|
||||
|
||||
@@ -1384,6 +1410,7 @@ class FfnExit:
|
||||
"mlp_reduce_scatter",
|
||||
"_complete",
|
||||
"_update",
|
||||
"_declared_sum",
|
||||
"_scope",
|
||||
)
|
||||
|
||||
@@ -1395,7 +1422,11 @@ class FfnExit:
|
||||
self.fuse_mlp_allreduce = completion.fuse_mlp_allreduce
|
||||
self.mlp_reduce_scatter = completion.mlp_reduce_scatter
|
||||
self._complete = completion.complete
|
||||
self._update = communicator._batch_steps(forward_batch).ffn_output.update
|
||||
steps = communicator._batch_steps(forward_batch)
|
||||
self._update = steps.ffn_output.update
|
||||
self._declared_sum = communicator._declared_ffn_sum(
|
||||
steps, completion.mlp_reduce_scatter
|
||||
)
|
||||
self._scope = get_forward().scoped(
|
||||
fuse_mlp_allreduce=self.fuse_mlp_allreduce,
|
||||
mlp_reduce_scatter=self.mlp_reduce_scatter,
|
||||
@@ -1422,7 +1453,7 @@ class FfnExit:
|
||||
else:
|
||||
hidden_states, residual = self._complete(hidden_states, residual)
|
||||
return self.communicator._leave_ffn_output(
|
||||
hidden_states, residual, stream, self._update
|
||||
hidden_states, residual, stream, self._update, self._declared_sum
|
||||
)
|
||||
|
||||
|
||||
|
||||
@@ -52,6 +52,7 @@ from sglang.srt.layers.communicator.output import (
|
||||
)
|
||||
from sglang.srt.layers.communicator.residual import StageRead, StageUpdate
|
||||
from sglang.srt.layers.communicator.residual.add_norm import ADD, NORM_READ
|
||||
from sglang.srt.layers.communicator.residual.stream import DeclaredSum
|
||||
from sglang.srt.layers.dp_attention import (
|
||||
attn_tp_all_gather_into_tensor,
|
||||
attn_tp_reduce_scatter_tensor,
|
||||
@@ -102,6 +103,14 @@ def tp_reduce_scatter(
|
||||
return output, residual
|
||||
|
||||
|
||||
def tp_slice(hidden_states, residual, context):
|
||||
"""The rows reduce-scatter would return, when the sum is already complete."""
|
||||
hidden_states = hidden_states.tensor_split(context.tp_size)[context.tp_rank].clone()
|
||||
if residual is not None:
|
||||
residual = residual.tensor_split(context.tp_size)[context.tp_rank]
|
||||
return hidden_states, residual
|
||||
|
||||
|
||||
class CommunicateSimpleFn:
|
||||
@staticmethod
|
||||
def _trivial(
|
||||
@@ -263,7 +272,7 @@ def _mlp_input_without_dp(
|
||||
hidden_states = _mlp_input_reduce_output(
|
||||
hidden_states, forward_batch, may_quantize=update.adds_plainly
|
||||
)
|
||||
else:
|
||||
elif group is SumGroup.TP:
|
||||
hidden_states = tensor_model_parallel_all_reduce(hidden_states)
|
||||
if _is_npu and context.cache is not None:
|
||||
_ = prepare_weight_cache(hidden_states, context.cache)
|
||||
@@ -411,7 +420,9 @@ def _consumer_step(
|
||||
step: Callable,
|
||||
adds_plainly: bool,
|
||||
carried_fusions: Tuple[Callable, ...],
|
||||
owes_by_construction: bool = False,
|
||||
expected_sum: Optional[SumGroup] = None,
|
||||
completed_step: Optional[Callable] = None,
|
||||
pending=None,
|
||||
written_step: Optional[Callable] = None,
|
||||
written: bool = False,
|
||||
update: StageUpdate = ADD,
|
||||
@@ -422,18 +433,26 @@ def _consumer_step(
|
||||
complete value. Under attention DP the reduction back to this rank's tokens
|
||||
comes first; otherwise one of ``carried_fusions`` may complete it with the
|
||||
residual add and the read, and an empty batch has nothing to sum. An input
|
||||
that owes its sum by construction (``owes_by_construction``) is that sum's
|
||||
only carrier. ``call`` is what the stage's read takes (the attention's
|
||||
whose declared sum is already completed uses ``completed_step`` instead;
|
||||
a pending declared sum must match ``expected_sum``. ``call`` is what the stage's read takes (the attention's
|
||||
``quant_format`` and ``post_residual_addition``)."""
|
||||
if residual is not None and update.adds_plainly != adds_plainly:
|
||||
raise RuntimeError("producer update does not match the boundary's capability")
|
||||
if pending is not None:
|
||||
if isinstance(pending.owed, DeclaredSum):
|
||||
if pending.owed.group is not expected_sum:
|
||||
raise RuntimeError("producer sum does not match the input declaration")
|
||||
elif expected_sum is not None:
|
||||
if pending.owed is not None:
|
||||
raise RuntimeError("a declared input sum carried another completion")
|
||||
step = completed_step
|
||||
if written and written_step is not None:
|
||||
step = written_step
|
||||
if not isinstance(hidden_states, torch.Tensor):
|
||||
owed = hidden_states
|
||||
if residual is None:
|
||||
raise RuntimeError(f"{type(owed).__name__} requires residual input")
|
||||
if owes_by_construction:
|
||||
if expected_sum is not None:
|
||||
raise RuntimeError(
|
||||
f"an input that owes its sum by construction arrived as "
|
||||
f"{type(owed).__name__}"
|
||||
@@ -538,6 +557,7 @@ def _mlp_input_on_residual_shard(
|
||||
layernorm: torch.nn.Module,
|
||||
context: CommunicateContext,
|
||||
*,
|
||||
reduces: bool = True,
|
||||
read: StageRead,
|
||||
update: StageUpdate,
|
||||
):
|
||||
@@ -548,7 +568,10 @@ def _mlp_input_on_residual_shard(
|
||||
if hidden_states.shape[0] == 0:
|
||||
return hidden_states, hidden_states
|
||||
shard = hidden_states.tensor_split(context.tp_size)[context.tp_rank]
|
||||
get_parallel().tp_group.reduce_scatter_tensor(shard, hidden_states)
|
||||
if reduces:
|
||||
get_parallel().tp_group.reduce_scatter_tensor(shard, hidden_states)
|
||||
else:
|
||||
shard = shard.clone()
|
||||
shard, residual = read.update_and_read(update, shard, residual, layernorm)
|
||||
attn_tp_all_gather_into_tensor(hidden_states, shard)
|
||||
return hidden_states, residual
|
||||
|
||||
@@ -82,6 +82,7 @@ def finish_layer_stack(
|
||||
forward_batch: ForwardBatch,
|
||||
*,
|
||||
final_norm_takes_handoff: bool = False,
|
||||
preserve_declared: bool = False,
|
||||
) -> Tuple[Union[torch.Tensor, HandoffOutput], Optional[torch.Tensor]]:
|
||||
"""Complete what this layer left for a next layer. Call it on the last
|
||||
layer of this rank before its output reaches the final norm, the next
|
||||
@@ -92,7 +93,11 @@ def finish_layer_stack(
|
||||
stream = forward_batch.residual_stream
|
||||
if residual is not stream:
|
||||
raise RuntimeError("residual alias belongs to a different invocation")
|
||||
output = stream.finish(hidden_states, takes_handoff=final_norm_takes_handoff)
|
||||
output = stream.finish(
|
||||
hidden_states,
|
||||
takes_handoff=final_norm_takes_handoff,
|
||||
preserve_declared=preserve_declared,
|
||||
)
|
||||
forward_batch.residual_stream = None
|
||||
return output
|
||||
if final_norm_takes_handoff and isinstance(hidden_states, HandoffOutput):
|
||||
|
||||
@@ -19,6 +19,7 @@ import msgspec
|
||||
import torch
|
||||
|
||||
from sglang.srt.distributed import GroupCoordinator
|
||||
from sglang.srt.layers.communicator.layout import SumGroup, _sum_group
|
||||
from sglang.srt.layers.communicator.output import HandoffOutput, UnreducedOutput
|
||||
from sglang.srt.layers.communicator.residual import StageUpdate
|
||||
|
||||
@@ -35,12 +36,21 @@ class CarriedSum(msgspec.Struct, frozen=True):
|
||||
return self.group.all_reduce(value)
|
||||
|
||||
|
||||
class DeclaredSum(msgspec.Struct, frozen=True):
|
||||
"""A sum every output of this producer owes to its declared input edge."""
|
||||
|
||||
group: SumGroup
|
||||
|
||||
def complete(self, value):
|
||||
return _sum_group(self.group).all_reduce(value)
|
||||
|
||||
|
||||
class Contribution(msgspec.Struct):
|
||||
"""The sole owner of an output's value, update and remaining work."""
|
||||
|
||||
value: Optional[torch.Tensor]
|
||||
update: StageUpdate
|
||||
owed: Union[CarriedSum, HandoffOutput, None] = None
|
||||
owed: Union[CarriedSum, DeclaredSum, HandoffOutput, None] = None
|
||||
|
||||
def for_boundary(self):
|
||||
# These forms are private inputs to the existing fused-kernel adapters.
|
||||
@@ -53,7 +63,7 @@ class Contribution(msgspec.Struct):
|
||||
return self.value
|
||||
|
||||
def complete(self):
|
||||
if isinstance(self.owed, CarriedSum):
|
||||
if isinstance(self.owed, (CarriedSum, DeclaredSum)):
|
||||
value = self.owed.complete(self.value)
|
||||
elif isinstance(self.owed, HandoffOutput):
|
||||
value = self.owed.complete()
|
||||
@@ -84,22 +94,26 @@ class ResidualStream:
|
||||
self.pending = None
|
||||
|
||||
@classmethod
|
||||
def arrive(cls, hidden, residual, update):
|
||||
def arrive(cls, hidden, residual, update, *, declared_sum=None):
|
||||
"""Rebuild an explicit tensor handoff, including each TBO microbatch."""
|
||||
stream = cls(residual)
|
||||
if residual is None:
|
||||
return stream.write(hidden), stream
|
||||
return stream.leave(hidden, update), stream
|
||||
return stream.leave(hidden, update, declared_sum=declared_sum), stream
|
||||
|
||||
def write(self, residual):
|
||||
self.residual = residual
|
||||
self.pending = None
|
||||
return residual
|
||||
|
||||
def leave(self, output, update):
|
||||
def leave(self, output, update, *, declared_sum=None):
|
||||
if self.pending is not None:
|
||||
raise RuntimeError("cannot replace an unconsumed producer contribution")
|
||||
if isinstance(output, UnreducedOutput):
|
||||
if declared_sum is not None:
|
||||
if not isinstance(output, torch.Tensor):
|
||||
raise RuntimeError("a declared sum cannot carry another completion")
|
||||
contribution = Contribution(output, update, DeclaredSum(declared_sum))
|
||||
elif isinstance(output, UnreducedOutput):
|
||||
contribution = Contribution(
|
||||
output.partial,
|
||||
update,
|
||||
@@ -151,7 +165,7 @@ class ResidualStream:
|
||||
return hidden.clone()
|
||||
if pending.owed is None:
|
||||
value = pending.value
|
||||
elif isinstance(pending.owed, CarriedSum):
|
||||
elif isinstance(pending.owed, (CarriedSum, DeclaredSum)):
|
||||
value = pending.owed.complete(pending.value.clone())
|
||||
else:
|
||||
raise NotImplementedError("a finalize handoff requires main-output capture")
|
||||
@@ -159,10 +173,12 @@ class ResidualStream:
|
||||
value = group.all_reduce(value.clone())
|
||||
return value.clone() if self.residual is None else value + self.residual
|
||||
|
||||
def finish(self, hidden, *, takes_handoff=False):
|
||||
def finish(self, hidden, *, takes_handoff=False, preserve_declared=False):
|
||||
self.check(hidden)
|
||||
if self.pending is None:
|
||||
return hidden, None
|
||||
if preserve_declared and isinstance(self.pending.owed, DeclaredSum):
|
||||
return self.pending.value, self.residual
|
||||
if takes_handoff and isinstance(self.pending.owed, HandoffOutput):
|
||||
return self.pending.owed, self.residual
|
||||
return self.pending.complete(), self.residual
|
||||
|
||||
@@ -848,7 +848,10 @@ class BailingMoEModel(nn.Module):
|
||||
)
|
||||
last_layer = self.layers[self.end_layer - 1]
|
||||
hidden_states, residual = last_layer.layer_communicator.finish_layer_stack(
|
||||
hidden_states, residual, forward_batch
|
||||
hidden_states,
|
||||
residual,
|
||||
forward_batch,
|
||||
preserve_declared=not self.pp_group.is_last_rank,
|
||||
)
|
||||
if not self.pp_group.is_last_rank:
|
||||
return PPProxyTensors(
|
||||
|
||||
@@ -1008,7 +1008,10 @@ class BailingMoELinearModel(nn.Module):
|
||||
)
|
||||
last_layer = self.layers[self.end_layer - 1]
|
||||
hidden_states, residual = last_layer.layer_communicator.finish_layer_stack(
|
||||
hidden_states, residual, forward_batch
|
||||
hidden_states,
|
||||
residual,
|
||||
forward_batch,
|
||||
preserve_declared=not self.pp_group.is_last_rank,
|
||||
)
|
||||
if not self.pp_group.is_last_rank:
|
||||
return PPProxyTensors(
|
||||
|
||||
@@ -1401,7 +1401,10 @@ class BailingMoELinearModel(nn.Module):
|
||||
|
||||
last_layer = self.layers[self.end_layer - 1]
|
||||
hidden_states, residual = last_layer.layer_communicator.finish_layer_stack(
|
||||
hidden_states, residual, forward_batch
|
||||
hidden_states,
|
||||
residual,
|
||||
forward_batch,
|
||||
preserve_declared=not self.pp_group.is_last_rank,
|
||||
)
|
||||
if not self.pp_group.is_last_rank:
|
||||
return PPProxyTensors(
|
||||
|
||||
@@ -2979,7 +2979,7 @@ class DeepseekV2Model(nn.Module):
|
||||
|
||||
install_cutedsl_fusion(
|
||||
# PP pads self.layers with PPMissingLayer, which has no communicator.
|
||||
# A stage's last layer has no successor, so it keeps its reduction.
|
||||
# A pipeline exit completes any handoff before sending its tensors.
|
||||
self.layers[self.start_layer : self.end_layer],
|
||||
hidden_size=config.hidden_size,
|
||||
top_k=config.num_experts_per_tok,
|
||||
@@ -2999,7 +2999,6 @@ class DeepseekV2Model(nn.Module):
|
||||
if isinstance(layer.mlp, DeepseekV2MoE)
|
||||
else layer.mlp.tp_size == 1
|
||||
),
|
||||
final_norm_consumes_handoff=False,
|
||||
label="DeepSeek-V3/GLM",
|
||||
)
|
||||
|
||||
@@ -3135,7 +3134,10 @@ class DeepseekV2Model(nn.Module):
|
||||
|
||||
last_layer = self.layers[self.end_layer - 1]
|
||||
hidden_states, residual = last_layer.layer_communicator.finish_layer_stack(
|
||||
hidden_states, residual, forward_batch
|
||||
hidden_states,
|
||||
residual,
|
||||
forward_batch,
|
||||
preserve_declared=not self.pp_group.is_last_rank,
|
||||
)
|
||||
if not self.pp_group.is_last_rank:
|
||||
proxy_tensors = {
|
||||
|
||||
@@ -584,7 +584,12 @@ class GigaChat35Model(nn.Module):
|
||||
|
||||
hidden_states, residual = self.layers[
|
||||
self.end_layer - 1
|
||||
].layer_communicator.finish_layer_stack(hidden_states, residual, forward_batch)
|
||||
].layer_communicator.finish_layer_stack(
|
||||
hidden_states,
|
||||
residual,
|
||||
forward_batch,
|
||||
preserve_declared=not self.pp_group.is_last_rank,
|
||||
)
|
||||
|
||||
if not self.pp_group.is_last_rank:
|
||||
return PPProxyTensors(
|
||||
|
||||
@@ -1134,7 +1134,10 @@ class Glm4MoeModel(nn.Module):
|
||||
|
||||
last_layer = self.layers[self.end_layer - 1]
|
||||
hidden_states, residual = last_layer.layer_communicator.finish_layer_stack(
|
||||
hidden_states, residual, forward_batch
|
||||
hidden_states,
|
||||
residual,
|
||||
forward_batch,
|
||||
preserve_declared=not self.pp_group.is_last_rank,
|
||||
)
|
||||
if not self.pp_group.is_last_rank:
|
||||
return PPProxyTensors(
|
||||
|
||||
@@ -833,7 +833,10 @@ class Glm4MoeLiteModel(nn.Module):
|
||||
|
||||
last_layer = self.layers[self.end_layer - 1]
|
||||
hidden_states, residual = last_layer.layer_communicator.finish_layer_stack(
|
||||
hidden_states, residual, forward_batch
|
||||
hidden_states,
|
||||
residual,
|
||||
forward_batch,
|
||||
preserve_declared=not self.pp_group.is_last_rank,
|
||||
)
|
||||
if not self.pp_group.is_last_rank:
|
||||
return PPProxyTensors(
|
||||
|
||||
@@ -1191,7 +1191,10 @@ class Glm5NextModel(nn.Module):
|
||||
|
||||
last_layer = self.layers[self.end_layer - 1]
|
||||
hidden_states, residual = last_layer.layer_communicator.finish_layer_stack(
|
||||
hidden_states, residual, forward_batch
|
||||
hidden_states,
|
||||
residual,
|
||||
forward_batch,
|
||||
preserve_declared=not self.pp_group.is_last_rank,
|
||||
)
|
||||
if not self.pp_group.is_last_rank:
|
||||
if self.config.mhc:
|
||||
|
||||
@@ -732,7 +732,10 @@ class GptOssModel(nn.Module):
|
||||
)
|
||||
last_layer = self.layers[self.end_layer - 1]
|
||||
hidden_states, residual = last_layer.layer_communicator.finish_layer_stack(
|
||||
hidden_states, residual, forward_batch
|
||||
hidden_states,
|
||||
residual,
|
||||
forward_batch,
|
||||
preserve_declared=not self.pp_group.is_last_rank,
|
||||
)
|
||||
if not self.pp_group.is_last_rank:
|
||||
return PPProxyTensors(
|
||||
|
||||
@@ -611,7 +611,10 @@ class LagunaModel(nn.Module):
|
||||
|
||||
last_layer = self.layers[self.end_layer - 1]
|
||||
hidden_states, residual = last_layer.layer_communicator.finish_layer_stack(
|
||||
hidden_states, residual, forward_batch
|
||||
hidden_states,
|
||||
residual,
|
||||
forward_batch,
|
||||
preserve_declared=not self.pp_group.is_last_rank,
|
||||
)
|
||||
if not self.pp_group.is_last_rank:
|
||||
return PPProxyTensors(
|
||||
|
||||
@@ -766,7 +766,12 @@ class LLaDA2MoeModel(nn.Module):
|
||||
|
||||
hidden_states, residual = self.layers[
|
||||
self.end_layer - 1
|
||||
].layer_communicator.finish_layer_stack(hidden_states, residual, forward_batch)
|
||||
].layer_communicator.finish_layer_stack(
|
||||
hidden_states,
|
||||
residual,
|
||||
forward_batch,
|
||||
preserve_declared=not self.pp_group.is_last_rank,
|
||||
)
|
||||
if not self.pp_group.is_last_rank:
|
||||
return PPProxyTensors(
|
||||
{
|
||||
|
||||
@@ -1081,7 +1081,10 @@ class MiMoV2Model(nn.Module):
|
||||
|
||||
last_layer = self.layers[self.end_layer - 1]
|
||||
hidden_states, residual = last_layer.layer_communicator.finish_layer_stack(
|
||||
hidden_states, residual, forward_batch
|
||||
hidden_states,
|
||||
residual,
|
||||
forward_batch,
|
||||
preserve_declared=not self.pp_group.is_last_rank,
|
||||
)
|
||||
|
||||
# A draft targeting the final layer ("after layer
|
||||
|
||||
@@ -1184,7 +1184,10 @@ class MiniMaxM2Model(nn.Module):
|
||||
|
||||
last_layer = self.layers[self.end_layer - 1]
|
||||
hidden_states, residual = last_layer.layer_communicator.finish_layer_stack(
|
||||
hidden_states, residual, forward_batch
|
||||
hidden_states,
|
||||
residual,
|
||||
forward_batch,
|
||||
preserve_declared=not self.pp_group.is_last_rank,
|
||||
)
|
||||
if not self.pp_group.is_last_rank:
|
||||
return PPProxyTensors(
|
||||
|
||||
@@ -1522,7 +1522,10 @@ class MiniMaxM3Model(nn.Module):
|
||||
|
||||
last_layer = self.layers[self.end_layer - 1]
|
||||
hidden_states, residual = last_layer.layer_communicator.finish_layer_stack(
|
||||
hidden_states, residual, forward_batch
|
||||
hidden_states,
|
||||
residual,
|
||||
forward_batch,
|
||||
preserve_declared=not self.pp_group.is_last_rank,
|
||||
)
|
||||
if not self.pp_group.is_last_rank:
|
||||
return PPProxyTensors(
|
||||
|
||||
@@ -760,7 +760,10 @@ class NemotronHModel(nn.Module):
|
||||
|
||||
last_layer = self.layers[self.end_layer - 1]
|
||||
hidden_states, residual = last_layer.layer_communicator.finish_layer_stack(
|
||||
hidden_states, residual, forward_batch
|
||||
hidden_states,
|
||||
residual,
|
||||
forward_batch,
|
||||
preserve_declared=not self.pp_group.is_last_rank,
|
||||
)
|
||||
if not self.pp_group.is_last_rank:
|
||||
return PPProxyTensors(
|
||||
|
||||
@@ -1220,7 +1220,10 @@ class Qwen2MoeModel(nn.Module):
|
||||
|
||||
last_layer = self.layers[self.end_layer - 1]
|
||||
hidden_states, residual = last_layer.layer_communicator.finish_layer_stack(
|
||||
hidden_states, residual, forward_batch
|
||||
hidden_states,
|
||||
residual,
|
||||
forward_batch,
|
||||
preserve_declared=not self.pp_group.is_last_rank,
|
||||
)
|
||||
if not self.pp_group.is_last_rank:
|
||||
return PPProxyTensors(
|
||||
|
||||
@@ -1746,8 +1746,6 @@ class Qwen3_5ForCausalLM(nn.Module):
|
||||
rms_epsilon=config.rms_norm_eps,
|
||||
# Every layer was checked above.
|
||||
can_defer_finalize=lambda layer: True,
|
||||
# The final GemmaRMSNorm closes out the last layer's handoff.
|
||||
final_norm_consumes_handoff=True,
|
||||
label="Qwen3.5",
|
||||
)
|
||||
|
||||
@@ -1870,6 +1868,7 @@ class Qwen3_5ForCausalLM(nn.Module):
|
||||
# The final norm below finalizes a deferred MoE output in its kernel.
|
||||
final_norm_takes_handoff=self.flashinfer_mnnvl_cutedsl_fusion is not None
|
||||
and self.pp_group.is_last_rank,
|
||||
preserve_declared=not self.pp_group.is_last_rank,
|
||||
)
|
||||
|
||||
# Return intermediate tensors for pipeline parallelism
|
||||
|
||||
@@ -1258,7 +1258,12 @@ class Qwen3LLMModel(Qwen3Model):
|
||||
|
||||
hidden_states, residual = self.layers[
|
||||
self.end_layer - 1
|
||||
].layer_communicator.finish_layer_stack(hidden_states, residual, forward_batch)
|
||||
].layer_communicator.finish_layer_stack(
|
||||
hidden_states,
|
||||
residual,
|
||||
forward_batch,
|
||||
preserve_declared=not self.pp_group.is_last_rank,
|
||||
)
|
||||
|
||||
# Handle deepstack for the last processed layer (HF-order path only).
|
||||
last_deepstack = (
|
||||
|
||||
@@ -158,7 +158,10 @@ class Qwen3MoeLLMModel(Qwen3MoeModel):
|
||||
|
||||
last_layer = self.layers[self.end_layer - 1]
|
||||
hidden_states, residual = last_layer.layer_communicator.finish_layer_stack(
|
||||
hidden_states, residual, forward_batch
|
||||
hidden_states,
|
||||
residual,
|
||||
forward_batch,
|
||||
preserve_declared=not self.pp_group.is_last_rank,
|
||||
)
|
||||
if not self.pp_group.is_last_rank:
|
||||
return PPProxyTensors(
|
||||
|
||||
@@ -1167,7 +1167,10 @@ class SarvamMLAModel(nn.Module):
|
||||
|
||||
last_layer = self.layers[self.end_layer - 1]
|
||||
hidden_states, residual = last_layer.layer_communicator.finish_layer_stack(
|
||||
hidden_states, residual, forward_batch
|
||||
hidden_states,
|
||||
residual,
|
||||
forward_batch,
|
||||
preserve_declared=not self.pp_group.is_last_rank,
|
||||
)
|
||||
if not self.pp_group.is_last_rank:
|
||||
return PPProxyTensors(
|
||||
|
||||
@@ -421,7 +421,12 @@ class SDARModel(nn.Module):
|
||||
|
||||
hidden_states, residual = self.layers[
|
||||
self.end_layer - 1
|
||||
].layer_communicator.finish_layer_stack(hidden_states, residual, forward_batch)
|
||||
].layer_communicator.finish_layer_stack(
|
||||
hidden_states,
|
||||
residual,
|
||||
forward_batch,
|
||||
preserve_declared=not self.pp_group.is_last_rank,
|
||||
)
|
||||
|
||||
if not self.pp_group.is_last_rank:
|
||||
return PPProxyTensors(
|
||||
|
||||
@@ -486,7 +486,10 @@ class SDARMoeModel(nn.Module):
|
||||
|
||||
last_layer = self.layers[self.end_layer - 1]
|
||||
hidden_states, residual = last_layer.layer_communicator.finish_layer_stack(
|
||||
hidden_states, residual, forward_batch
|
||||
hidden_states,
|
||||
residual,
|
||||
forward_batch,
|
||||
preserve_declared=not self.pp_group.is_last_rank,
|
||||
)
|
||||
if not self.pp_group.is_last_rank:
|
||||
return PPProxyTensors(
|
||||
|
||||
@@ -716,7 +716,10 @@ class Step3p5Model(nn.Module):
|
||||
# break
|
||||
last_layer = self.layers[self.end_layer - 1]
|
||||
hidden_states, residual = last_layer.layer_communicator.finish_layer_stack(
|
||||
hidden_states, residual, forward_batch
|
||||
hidden_states,
|
||||
residual,
|
||||
forward_batch,
|
||||
preserve_declared=not self.pp_group.is_last_rank,
|
||||
)
|
||||
if not self.pp_group.is_last_rank:
|
||||
return PPProxyTensors(
|
||||
|
||||
@@ -1662,7 +1662,12 @@ class XllmModel(nn.Module):
|
||||
|
||||
hidden_states, residual = self.layers[
|
||||
self.end_layer - 1
|
||||
].layer_communicator.finish_layer_stack(hidden_states, residual, forward_batch)
|
||||
].layer_communicator.finish_layer_stack(
|
||||
hidden_states,
|
||||
residual,
|
||||
forward_batch,
|
||||
preserve_declared=not self.pp_group.is_last_rank,
|
||||
)
|
||||
if not self.pp_group.is_last_rank:
|
||||
return PPProxyTensors(
|
||||
{
|
||||
|
||||
Reference in New Issue
Block a user