[Refactor] Let stage boundaries complete every stage-output sum (#42301)
This commit is contained in:
@@ -19,7 +19,7 @@ from typing import TYPE_CHECKING, Tuple
|
||||
|
||||
import torch
|
||||
|
||||
from sglang.srt.layers.layer_boundary.ops import move_rows
|
||||
from sglang.srt.layers.layer_boundary.ops import move_rows, sum_output
|
||||
from sglang.srt.layers.layer_boundary.residual.stream import ResidualStream
|
||||
from sglang.srt.model_executor.forward_batch_info import ForwardBatch
|
||||
|
||||
@@ -53,9 +53,14 @@ def branch_input(
|
||||
def branch_output(
|
||||
plan: StagePlan, hidden_states: torch.Tensor, forward_batch: ForwardBatch
|
||||
) -> torch.Tensor:
|
||||
"""This layer's complete FFN output as a branch's contribution, which
|
||||
adds to the layer's output without writing the residual: moved to the
|
||||
rows the layer hands on."""
|
||||
"""This layer's FFN output as a branch's contribution, which adds to the
|
||||
layer's output without writing the residual: the sum it owes completed,
|
||||
then moved to the rows the layer hands on."""
|
||||
group = plan.path_for(forward_batch).output.group
|
||||
if group is not None:
|
||||
hidden_states = sum_output(
|
||||
hidden_states, group, forward_batch, may_quantize=False
|
||||
)
|
||||
rows, _, to = plan.branch_rows(forward_batch)
|
||||
return move_rows(hidden_states, rows, to, forward_batch)
|
||||
|
||||
|
||||
@@ -39,6 +39,7 @@ from sglang.srt.layers.layer_boundary.ops import (
|
||||
all_reduce_to_dp_local,
|
||||
dp_reduce_scatter,
|
||||
dp_reduce_scatterv,
|
||||
sum_output,
|
||||
to_dp_local,
|
||||
)
|
||||
from sglang.srt.layers.layer_boundary.output import (
|
||||
@@ -94,13 +95,14 @@ class ExitPolicy:
|
||||
def __init__(self, plan):
|
||||
self.plan = plan
|
||||
|
||||
def finish_complete_output(
|
||||
def complete_now(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
stream: ResidualStream,
|
||||
forward_batch: ForwardBatch,
|
||||
):
|
||||
"""Move a complete output from the operation-scheduled producer path."""
|
||||
"""Complete the output of the operation-scheduled producer path, which
|
||||
never defers: the sum it owes, then its move and write-back."""
|
||||
steps = self.plan.path_for(forward_batch)
|
||||
hidden_states, residual = self._complete_now(
|
||||
hidden_states,
|
||||
@@ -108,11 +110,9 @@ class ExitPolicy:
|
||||
forward_batch=forward_batch,
|
||||
dp_step=None,
|
||||
steps=steps,
|
||||
output_move=(
|
||||
steps.complete_output_move
|
||||
if steps.output_move_completes_sum
|
||||
else steps.output_move
|
||||
),
|
||||
output_move=steps.output_move,
|
||||
owes_sum=steps.output.group is not None
|
||||
and not steps.output_move_completes_sum,
|
||||
)
|
||||
return self._record_output(
|
||||
hidden_states,
|
||||
@@ -188,6 +188,8 @@ class ExitPolicy:
|
||||
dp_step=dp_step,
|
||||
steps=steps,
|
||||
output_move=steps.output_move,
|
||||
# A reduce-scatter completes the sum on its way back.
|
||||
owes_sum=steps.output.group is not None and not mlp_reduce_scatter,
|
||||
)
|
||||
defer_moe_finalize = (
|
||||
self.plan.fusions is not None
|
||||
@@ -251,10 +253,19 @@ class ExitPolicy:
|
||||
dp_step: Optional[Callable],
|
||||
steps,
|
||||
output_move: Optional[Callable],
|
||||
owes_sum: bool,
|
||||
) -> Tuple[torch.Tensor, torch.Tensor]:
|
||||
"""This layer's postprocess, run with the attention-DP step already
|
||||
chosen: the move back to where the next layer reads the FFN output, then
|
||||
the write-back into the residual for a layer that does it itself."""
|
||||
chosen: the sum the output owes, the move back to where the next layer
|
||||
reads it, then the write-back into the residual for a layer that does
|
||||
it itself."""
|
||||
if owes_sum:
|
||||
hidden_states = sum_output(
|
||||
hidden_states,
|
||||
steps.output.group,
|
||||
forward_batch,
|
||||
may_quantize=steps.output.update.is_plain_add,
|
||||
)
|
||||
if steps.output.transform is not None:
|
||||
hidden_states = steps.output.transform.apply(hidden_states)
|
||||
if steps.returns_over_dp:
|
||||
@@ -289,7 +300,7 @@ class ExitPolicy:
|
||||
return FfnExit(self, forward_batch, stream=stream)
|
||||
|
||||
def _sum_deferral_allowed(self, steps) -> bool:
|
||||
# Under the MoE-CP all-gather the fusion path would skip finish_complete_output
|
||||
# Under the MoE-CP all-gather the fusion path would skip complete_now
|
||||
# and its MoE-CP scatter, leaving hidden_states longer than the residual.
|
||||
if is_enable_moe_cp_allgather() or steps.output.group is None:
|
||||
return False
|
||||
@@ -377,17 +388,17 @@ def _batch_allows_deferred_sum(forward_batch: ForwardBatch, boundary=None) -> bo
|
||||
|
||||
|
||||
class ExitDecision(msgspec.Struct, frozen=True):
|
||||
"""One decision shared by compute flags and the matching output completion.
|
||||
"""One decision for an FFN output, made before compute runs.
|
||||
|
||||
Fields:
|
||||
defer_moe_finalize: Compute may return a producer-specific finalize handoff.
|
||||
fuse_mlp_allreduce: Compute skips its all-reduce; completion runs it later
|
||||
or carries it to the next consumer, whether fused there or unfused.
|
||||
mlp_reduce_scatter: Compute leaves reduction to the selected scatter path.
|
||||
fuse_mlp_allreduce: The sum is carried to the next consumer, which
|
||||
completes it, fused or unfused.
|
||||
mlp_reduce_scatter: A reduce-scatter on the way back completes the sum.
|
||||
complete: Callable(output, residual) returning the completed or wrapped
|
||||
output and its corresponding residual rows.
|
||||
|
||||
Do not independently reselect completion after compute has used these flags.
|
||||
Do not independently reselect completion after compute has run.
|
||||
"""
|
||||
|
||||
defer_moe_finalize: bool
|
||||
@@ -405,22 +416,21 @@ def _defer(
|
||||
|
||||
|
||||
class MixerExit:
|
||||
"""The scope that publishes a mixer's decision while it runs: inside the
|
||||
``with`` block ``fuse_mlp_allreduce`` on ``get_forward()`` tells its
|
||||
row-parallel output projection to skip the all-reduce. It skips when the
|
||||
stage's output always leaves its sum (to an FFN stage, which completes it in
|
||||
its input), or when its declaration permits deferring the sum to the next
|
||||
attention stage and the batch allows it: TP > 1, no attention DP, and no
|
||||
input-scattered or MoE-CP all-gather layout. The consumer decides whether
|
||||
to fuse its completion with the input norm."""
|
||||
"""A mixer's (an attention-like stage's) output decision. The mixer leaves
|
||||
its output's sum; the stage either carries it to the next attention stage
|
||||
(when the declaration permits deferring and the batch allows it: TP > 1,
|
||||
no attention DP, and no input-scattered or MoE-CP all-gather layout),
|
||||
hands it on as the declared sum an FFN stage completes in its input, or
|
||||
completes it here. The consumer decides whether to fuse its completion
|
||||
with the input norm."""
|
||||
|
||||
__slots__ = (
|
||||
"skips_reduction",
|
||||
"_defers",
|
||||
"_scope",
|
||||
"_group",
|
||||
"_update",
|
||||
"_declared_sum",
|
||||
"_stream",
|
||||
"_forward_batch",
|
||||
)
|
||||
|
||||
def __init__(
|
||||
@@ -433,7 +443,9 @@ class MixerExit:
|
||||
steps = boundary.plan.path_for(forward_batch)
|
||||
produced = steps.output
|
||||
self._stream = stream
|
||||
self._forward_batch = forward_batch
|
||||
self._update = produced.update
|
||||
self._group = produced.group
|
||||
self._declared_sum = produced.group if produced.always_partial else None
|
||||
self._defers = (
|
||||
produced.may_defer_to_next
|
||||
@@ -441,15 +453,12 @@ class MixerExit:
|
||||
and not is_dp_attention_enabled()
|
||||
and boundary._sum_deferral_allowed(steps)
|
||||
)
|
||||
self.skips_reduction = produced.always_partial or self._defers
|
||||
self._scope = get_forward().scoped(fuse_mlp_allreduce=self.skips_reduction)
|
||||
|
||||
def __enter__(self) -> MixerExit:
|
||||
self._scope.__enter__()
|
||||
return self
|
||||
|
||||
def __exit__(self, *exc_info):
|
||||
return self._scope.__exit__(*exc_info)
|
||||
return None
|
||||
|
||||
def finish(self, hidden_states: torch.Tensor):
|
||||
"""Hand the actual mixer result to the next boundary through the stream."""
|
||||
@@ -457,6 +466,13 @@ class MixerExit:
|
||||
hidden_states = UnreducedOutput(
|
||||
hidden_states, group=get_parallel().tp_group
|
||||
)
|
||||
elif self._declared_sum is None and self._group is not None:
|
||||
hidden_states = sum_output(
|
||||
hidden_states,
|
||||
self._group,
|
||||
self._forward_batch,
|
||||
may_quantize=self._update.is_plain_add,
|
||||
)
|
||||
return self._stream.record(
|
||||
hidden_states, self._update, declared_sum=self._declared_sum
|
||||
)
|
||||
@@ -473,18 +489,15 @@ class FfnExit:
|
||||
Fields:
|
||||
boundary: The owning output boundary.
|
||||
defer_moe_finalize: Whether compute may return a finalize handoff.
|
||||
fuse_mlp_allreduce: Whether compute leaves its all-reduce to completion.
|
||||
mlp_reduce_scatter: Whether compute leaves reduction to a scatter path.
|
||||
|
||||
The three flags are published on get_forward() only inside the context.
|
||||
finish(output) uses the same selected action after successful compute.
|
||||
The FFN leaves its output's sum; finish(output) completes it or carries
|
||||
it on with the action selected before compute. ``defer_moe_finalize`` is
|
||||
published on get_forward() only inside the context.
|
||||
"""
|
||||
|
||||
__slots__ = (
|
||||
"boundary",
|
||||
"defer_moe_finalize",
|
||||
"fuse_mlp_allreduce",
|
||||
"mlp_reduce_scatter",
|
||||
"_complete",
|
||||
"_update",
|
||||
"_declared_sum",
|
||||
@@ -504,18 +517,12 @@ class FfnExit:
|
||||
steps = boundary.plan.path_for(forward_batch)
|
||||
completion = boundary._decide(forward_batch, steps)
|
||||
self.defer_moe_finalize = completion.defer_moe_finalize
|
||||
self.fuse_mlp_allreduce = completion.fuse_mlp_allreduce
|
||||
self.mlp_reduce_scatter = completion.mlp_reduce_scatter
|
||||
self._complete = completion.complete
|
||||
self._update = steps.output.update
|
||||
self._declared_sum = boundary._sum_owed_after_skip(
|
||||
steps, completion.mlp_reduce_scatter
|
||||
)
|
||||
self._scope = get_forward().scoped(
|
||||
fuse_mlp_allreduce=self.fuse_mlp_allreduce,
|
||||
mlp_reduce_scatter=self.mlp_reduce_scatter,
|
||||
defer_moe_finalize=self.defer_moe_finalize,
|
||||
)
|
||||
self._scope = get_forward().scoped(defer_moe_finalize=self.defer_moe_finalize)
|
||||
|
||||
def __enter__(self) -> FfnExit:
|
||||
self._scope.__enter__()
|
||||
|
||||
@@ -22,6 +22,8 @@ from sglang.srt.distributed import (
|
||||
GroupCoordinator,
|
||||
attention_tensor_model_parallel_all_reduce,
|
||||
attention_tensor_model_parallel_quant_all_reduce,
|
||||
tensor_model_parallel_all_reduce,
|
||||
tensor_model_parallel_quant_all_reduce,
|
||||
)
|
||||
from sglang.srt.distributed.device_communicators.pynccl_allocator import (
|
||||
use_symmetric_memory,
|
||||
@@ -47,12 +49,14 @@ from sglang.srt.layers.layer_boundary.adapters.attention import (
|
||||
)
|
||||
from sglang.srt.layers.layer_boundary.layout import (
|
||||
Layout,
|
||||
SumGroup,
|
||||
TokenAxis,
|
||||
_cp_shard_token_rows,
|
||||
moe_cp_gathered_rows,
|
||||
)
|
||||
from sglang.srt.layers.layer_boundary.residual import ResidualUpdate
|
||||
from sglang.srt.layers.layer_boundary.residual.add_norm import PLAIN_ADD
|
||||
from sglang.srt.layers.moe.utils import sum_post_experts_output
|
||||
from sglang.srt.model_executor.forward_batch_info import ForwardBatch
|
||||
from sglang.srt.runtime_context import get_exec, get_parallel
|
||||
|
||||
@@ -173,6 +177,34 @@ def attn_tp_all_reduce(
|
||||
return attention_tensor_model_parallel_all_reduce(hidden_states)
|
||||
|
||||
|
||||
def sum_output(
|
||||
hidden_states: torch.Tensor,
|
||||
group: SumGroup,
|
||||
forward_batch: ForwardBatch,
|
||||
*,
|
||||
may_quantize: bool,
|
||||
) -> torch.Tensor:
|
||||
"""Complete the sum an output owes over ``group`` on the rows it is on."""
|
||||
parallel = get_parallel()
|
||||
if group is SumGroup.ATTN_TP:
|
||||
if parallel.attn_tp_size == 1:
|
||||
return hidden_states
|
||||
return attn_tp_all_reduce(hidden_states, forward_batch, may_quantize)
|
||||
if group is SumGroup.TP:
|
||||
if parallel.tp_size == 1:
|
||||
return hidden_states
|
||||
if (
|
||||
may_quantize
|
||||
and not forward_batch.forward_mode.is_decode_or_idle()
|
||||
and get_exec().comm.enable_quant_communications
|
||||
):
|
||||
return tensor_model_parallel_quant_all_reduce(hidden_states)
|
||||
return tensor_model_parallel_all_reduce(hidden_states)
|
||||
if group is SumGroup.MOE_OUTPUT:
|
||||
return sum_post_experts_output(hidden_states)
|
||||
raise ValueError(f"unsupported output sum group: {group!r}")
|
||||
|
||||
|
||||
def dp_gather(
|
||||
hidden_states: torch.Tensor,
|
||||
forward_batch: ForwardBatch,
|
||||
|
||||
@@ -187,7 +187,8 @@ class StageBoundary:
|
||||
forward_batch: Active batch whose stream receives the compute result.
|
||||
|
||||
Returns:
|
||||
A context manager publishing reduction/finalize flags during compute.
|
||||
A context manager publishing the producer's finalize choice during
|
||||
compute.
|
||||
Call its finish(output) exactly once after successful compute, including
|
||||
when compute is skipped for an empty input. It completes or carries work
|
||||
using the same decision rather than selecting a second path.
|
||||
@@ -253,11 +254,11 @@ class StageBoundary:
|
||||
hidden_states = stream.complete(hidden_states)
|
||||
return hidden_states, stream.snapshot(hidden_states)
|
||||
|
||||
def finish_complete_output(self, hidden_states, forward_batch):
|
||||
"""Move an FFN output that compute already completed outside exit()
|
||||
(the operation-scheduled TBO path) onto the rows the layer hands on;
|
||||
it chooses no reduction step."""
|
||||
return self.plan.output.finish_complete_output(
|
||||
def complete_now(self, hidden_states, forward_batch):
|
||||
"""Complete an FFN output outside exit() (the operation-scheduled TBO
|
||||
path, which never defers): the sum it owes, then the move onto the
|
||||
rows the layer hands on."""
|
||||
return self.plan.output.complete_now(
|
||||
hidden_states, stream_of(forward_batch), forward_batch
|
||||
)
|
||||
|
||||
@@ -379,3 +380,26 @@ class StageBoundary:
|
||||
),
|
||||
)
|
||||
return hidden_states, stream
|
||||
|
||||
|
||||
def check_stage_producers(model: torch.nn.Module) -> None:
|
||||
"""Reject a model whose stage producers complete their own output sums.
|
||||
|
||||
In a layer built from stage boundaries, the boundary completes every sum a
|
||||
stage output owes. A submodule there constructed with
|
||||
``reduce_results=True`` (a row-parallel projection, a MoE block) would
|
||||
have its output summed twice. Called once, after the model is built.
|
||||
"""
|
||||
offenders = [
|
||||
f"{name}.{child_name}"
|
||||
for name, layer in model.named_modules()
|
||||
if any(isinstance(v, StageBoundary) for v in vars(layer).values())
|
||||
for child_name, child in layer.named_modules()
|
||||
if child_name and getattr(child, "reduce_results", False) is True
|
||||
]
|
||||
if offenders:
|
||||
raise ValueError(
|
||||
"stage boundaries complete their producers' output sums, but these "
|
||||
f"submodules sum their own output: {', '.join(offenders[:8])}"
|
||||
+ (f" (and {len(offenders) - 8} more)" if len(offenders) > 8 else "")
|
||||
)
|
||||
|
||||
@@ -797,6 +797,18 @@ def should_add_replicated_moe_output() -> bool:
|
||||
return not (parallel.tp_size > 1 and summed_later and parallel.tp_rank != 0)
|
||||
|
||||
|
||||
def adds_replicated_output_to_partial() -> bool:
|
||||
"""For a MoE block whose stage boundary completes its sum: whether this rank
|
||||
adds an output every TP rank holds in full, such as a shared expert
|
||||
replicated with tp_size=1, to its MoE output. While the output still owes a
|
||||
TP sum, only TP rank 0 adds it, so the sum counts it once."""
|
||||
parallel = get_parallel()
|
||||
owes_sum = parallel.tp_size > 1 and not post_experts_output_is_complete(
|
||||
is_tp_path=True
|
||||
)
|
||||
return not owes_sum or parallel.tp_rank == 0
|
||||
|
||||
|
||||
def can_merge_post_experts_all_reduce() -> bool:
|
||||
"""Whether the EP and MoE-TP reductions can collapse into one _TP all-reduce.
|
||||
|
||||
@@ -818,20 +830,38 @@ def post_experts_all_reduce(hidden_states: torch.Tensor) -> torch.Tensor:
|
||||
sequential ones, which also restores the invariant the fused residual+LN path
|
||||
depends on.
|
||||
"""
|
||||
parallel = get_parallel()
|
||||
return _post_experts_sum(
|
||||
hidden_states,
|
||||
reduce_ep=parallel.moe_ep_size > 1
|
||||
and not should_skip_post_experts_all_reduce(is_tp_path=False),
|
||||
reduce_tp=parallel.moe_tp_size > 1
|
||||
and not should_skip_post_experts_all_reduce(is_tp_path=True),
|
||||
)
|
||||
|
||||
|
||||
def sum_post_experts_output(hidden_states: torch.Tensor) -> torch.Tensor:
|
||||
"""Complete the sum a MoE output owes over the EP and MoE-TP groups, for the
|
||||
boundary that owns it; a path the combine already summed is left alone."""
|
||||
parallel = get_parallel()
|
||||
return _post_experts_sum(
|
||||
hidden_states,
|
||||
reduce_ep=parallel.moe_ep_size > 1
|
||||
and not post_experts_output_is_complete(is_tp_path=False),
|
||||
reduce_tp=parallel.moe_tp_size > 1
|
||||
and not post_experts_output_is_complete(is_tp_path=True),
|
||||
)
|
||||
|
||||
|
||||
def _post_experts_sum(
|
||||
hidden_states: torch.Tensor, *, reduce_ep: bool, reduce_tp: bool
|
||||
) -> torch.Tensor:
|
||||
from sglang.srt.distributed.communication_op import (
|
||||
moe_expert_parallel_all_reduce,
|
||||
moe_tensor_model_parallel_all_reduce,
|
||||
tensor_model_parallel_all_reduce,
|
||||
)
|
||||
|
||||
parallel = get_parallel()
|
||||
reduce_ep = parallel.moe_ep_size > 1 and not should_skip_post_experts_all_reduce(
|
||||
is_tp_path=False
|
||||
)
|
||||
reduce_tp = parallel.moe_tp_size > 1 and not should_skip_post_experts_all_reduce(
|
||||
is_tp_path=True
|
||||
)
|
||||
|
||||
if reduce_ep and reduce_tp and can_merge_post_experts_all_reduce():
|
||||
return tensor_model_parallel_all_reduce(hidden_states)
|
||||
|
||||
|
||||
@@ -83,6 +83,7 @@ from sglang.srt.connector.utils import parse_model_name
|
||||
from sglang.srt.distributed import (
|
||||
model_parallel_is_initialized,
|
||||
)
|
||||
from sglang.srt.layers.layer_boundary.stage import check_stage_producers
|
||||
from sglang.srt.layers.modelopt_utils import QUANT_CFG_CHOICES
|
||||
from sglang.srt.layers.moe.utils import (
|
||||
install_shared_experts_fusion_decision,
|
||||
@@ -304,7 +305,9 @@ def _initialize_model(
|
||||
if load_config.draft_model_idx is not None:
|
||||
kwargs["draft_model_idx"] = load_config.draft_model_idx
|
||||
|
||||
return model_class(**kwargs)
|
||||
model = model_class(**kwargs)
|
||||
check_stage_producers(model)
|
||||
return model
|
||||
|
||||
|
||||
def post_load_weights(model: nn.Module) -> None:
|
||||
|
||||
@@ -259,6 +259,7 @@ class ApertusDecoderLayer(nn.Module):
|
||||
quant_config=quant_config,
|
||||
bias=getattr(config, "mlp_bias", False),
|
||||
prefix=add_prefix("mlp", prefix),
|
||||
reduce_results=False,
|
||||
)
|
||||
self.attention_layernorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
|
||||
self.feedforward_layernorm = RMSNorm(
|
||||
@@ -294,8 +295,9 @@ class ApertusDecoderLayer(nn.Module):
|
||||
# Fully Connected
|
||||
hidden_states = self.attn_boundary.finish(hidden_states, forward_batch)
|
||||
hidden_states = self.ffn_boundary.prepare(hidden_states, forward_batch)
|
||||
hidden_states = self.mlp(hidden_states)
|
||||
return self.ffn_boundary.finish_complete_output(hidden_states, forward_batch)
|
||||
with self.ffn_boundary.exit(forward_batch) as ffn_exit:
|
||||
hidden_states = self.mlp(hidden_states)
|
||||
return ffn_exit.finish(hidden_states)
|
||||
|
||||
|
||||
class ApertusModel(nn.Module):
|
||||
|
||||
@@ -235,6 +235,7 @@ class ArceeDecoderLayer(nn.Module):
|
||||
hidden_act=config.hidden_act,
|
||||
quant_config=quant_config,
|
||||
prefix=add_prefix("mlp", prefix),
|
||||
reduce_results=False,
|
||||
)
|
||||
self.input_layernorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
|
||||
self.post_attention_layernorm = RMSNorm(
|
||||
@@ -270,8 +271,9 @@ class ArceeDecoderLayer(nn.Module):
|
||||
# Fully Connected
|
||||
hidden_states = self.attn_boundary.finish(hidden_states, forward_batch)
|
||||
hidden_states = self.ffn_boundary.prepare(hidden_states, forward_batch)
|
||||
hidden_states = self.mlp(hidden_states)
|
||||
return self.ffn_boundary.finish_complete_output(hidden_states, forward_batch)
|
||||
with self.ffn_boundary.exit(forward_batch) as ffn_exit:
|
||||
hidden_states = self.mlp(hidden_states)
|
||||
return ffn_exit.finish(hidden_states)
|
||||
|
||||
|
||||
class ArceeModel(nn.Module):
|
||||
|
||||
@@ -55,7 +55,6 @@ from sglang.srt.layers.logits_processor import LogitsProcessor
|
||||
from sglang.srt.layers.moe import (
|
||||
get_deepep_mode,
|
||||
get_moe_a2a_backend,
|
||||
reduce_moe_output,
|
||||
)
|
||||
from sglang.srt.layers.moe.ep_moe.layer import get_moe_impl_class
|
||||
from sglang.srt.layers.moe.fused_moe_triton.layer import FusedMoE
|
||||
@@ -460,7 +459,6 @@ class BailingMoESparseMoeBlock(nn.Module):
|
||||
if self.num_shared_experts > 0:
|
||||
final_hidden_states = final_hidden_states + shared_output
|
||||
|
||||
final_hidden_states = reduce_moe_output(final_hidden_states)
|
||||
return final_hidden_states.view(num_tokens, hidden_size)
|
||||
|
||||
def forward_deepep(
|
||||
@@ -693,6 +691,7 @@ class BailingMoEBlock(nn.Module):
|
||||
prefix=add_prefix("mlp", prefix),
|
||||
tp_rank=mlp_tp_rank,
|
||||
tp_size=mlp_tp_size,
|
||||
reduce_results=False,
|
||||
)
|
||||
|
||||
self.post_attention_layernorm = RMSNorm(hidden_size, eps=config.rms_norm_eps)
|
||||
|
||||
@@ -32,7 +32,6 @@ from sglang.srt.layers.linear import (
|
||||
RowParallelLinear,
|
||||
)
|
||||
from sglang.srt.layers.logits_processor import LogitsProcessor
|
||||
from sglang.srt.layers.moe import reduce_moe_output
|
||||
from sglang.srt.layers.moe.ep_moe.layer import DeepEPMoE, get_moe_impl_class
|
||||
from sglang.srt.layers.moe.fused_moe_triton.layer import FusedMoE
|
||||
from sglang.srt.layers.moe.topk import TopK
|
||||
@@ -365,7 +364,6 @@ class BailingMoE(nn.Module):
|
||||
if self.num_shared_experts > 0:
|
||||
final_hidden_states = final_hidden_states + shared_output
|
||||
|
||||
final_hidden_states = reduce_moe_output(final_hidden_states)
|
||||
return final_hidden_states
|
||||
|
||||
|
||||
@@ -671,6 +669,7 @@ class BailingMoEAttention(nn.Module):
|
||||
bias=config.use_bias,
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.o_proj",
|
||||
reduce_results=False,
|
||||
)
|
||||
if hasattr(config, "rotary_dim"):
|
||||
self.rotary_dim = config.rotary_dim
|
||||
@@ -792,6 +791,7 @@ class BailingMoELinearDecoderLayer(nn.Module):
|
||||
intermediate_size=config.intermediate_size,
|
||||
quant_config=quant_config,
|
||||
prefix=add_prefix("mlp", prefix),
|
||||
reduce_results=False,
|
||||
)
|
||||
else:
|
||||
if is_nextn or self.layer_id >= config.first_k_dense_replace:
|
||||
@@ -810,6 +810,7 @@ class BailingMoELinearDecoderLayer(nn.Module):
|
||||
intermediate_size=config.intermediate_size,
|
||||
quant_config=quant_config,
|
||||
prefix=add_prefix("mlp", prefix),
|
||||
reduce_results=False,
|
||||
)
|
||||
rms_norm_eps = float(getattr(config, "rms_norm_eps", 1e-5))
|
||||
self.input_layernorm = RMSNorm(self.hidden_size, eps=rms_norm_eps)
|
||||
|
||||
@@ -19,10 +19,6 @@ from sglang.kernels.ops.quantization.fp8_kernel import (
|
||||
)
|
||||
from sglang.srt.configs import KimiLinearConfig
|
||||
from sglang.srt.configs.bailing_hybrid import is_bailing_multi_gate_enabled
|
||||
from sglang.srt.distributed import (
|
||||
moe_expert_parallel_all_reduce,
|
||||
moe_tensor_model_parallel_all_reduce,
|
||||
)
|
||||
from sglang.srt.environ import envs
|
||||
from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder
|
||||
from sglang.srt.eplb.expert_location_dispatch import ExpertLocationDispatchInfo
|
||||
@@ -47,7 +43,6 @@ from sglang.srt.layers.linear import (
|
||||
from sglang.srt.layers.logits_processor import LogitsProcessor
|
||||
from sglang.srt.layers.moe import (
|
||||
get_moe_a2a_backend,
|
||||
should_skip_post_experts_all_reduce,
|
||||
)
|
||||
from sglang.srt.layers.moe.ep_moe.layer import get_moe_impl_class
|
||||
from sglang.srt.layers.moe.fused_moe_triton.layer import FusedMoE
|
||||
@@ -751,18 +746,6 @@ class BailingMoE(nn.Module):
|
||||
|
||||
if shared_output is not None:
|
||||
final_hidden_states = final_hidden_states + shared_output
|
||||
|
||||
if self.moe_ep_size > 1 and not should_skip_post_experts_all_reduce(
|
||||
is_tp_path=False,
|
||||
):
|
||||
final_hidden_states = moe_expert_parallel_all_reduce(final_hidden_states)
|
||||
|
||||
if self.moe_tp_size > 1 and not should_skip_post_experts_all_reduce(
|
||||
is_tp_path=True,
|
||||
):
|
||||
final_hidden_states = moe_tensor_model_parallel_all_reduce(
|
||||
final_hidden_states
|
||||
)
|
||||
return final_hidden_states
|
||||
|
||||
def _forward_shared_experts(
|
||||
@@ -978,6 +961,7 @@ class BailingMoEAttention(nn.Module):
|
||||
bias=config.use_bias,
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.o_proj",
|
||||
reduce_results=False,
|
||||
)
|
||||
if hasattr(config, "rotary_dim"):
|
||||
self.rotary_dim = config.rotary_dim
|
||||
@@ -1121,6 +1105,7 @@ class BailingMoELinearDecoderLayer(nn.Module):
|
||||
prefix=prefix,
|
||||
tp_rank=mlp_tp_rank,
|
||||
tp_size=mlp_tp_size,
|
||||
reduce_results=False,
|
||||
)
|
||||
else:
|
||||
if is_nextn or self.layer_id >= config.first_k_dense_replace:
|
||||
@@ -1141,6 +1126,7 @@ class BailingMoELinearDecoderLayer(nn.Module):
|
||||
prefix=prefix,
|
||||
tp_rank=mlp_tp_rank,
|
||||
tp_size=mlp_tp_size,
|
||||
reduce_results=False,
|
||||
)
|
||||
rms_norm_eps = float(getattr(config, "rms_norm_eps", 1e-5))
|
||||
self.input_layernorm = RMSNorm(self.hidden_size, eps=rms_norm_eps)
|
||||
|
||||
@@ -111,6 +111,7 @@ from sglang.srt.layers.moe.token_dispatcher.base import (
|
||||
from sglang.srt.layers.moe.topk import BypassedTopKOutput, TopK, TopKOutputFormat
|
||||
from sglang.srt.layers.moe.utils import (
|
||||
RoutingMethodType,
|
||||
adds_replicated_output_to_partial,
|
||||
filter_moe_weight_param_global_expert,
|
||||
has_per_rank_fused_shared_slots,
|
||||
is_deepep_class_backend,
|
||||
@@ -118,6 +119,7 @@ from sglang.srt.layers.moe.utils import (
|
||||
is_shared_experts_fusion_disabled,
|
||||
is_tbo_enabled,
|
||||
should_add_replicated_moe_output,
|
||||
sum_post_experts_output,
|
||||
)
|
||||
from sglang.srt.layers.quantization.base_config import QuantizationConfig
|
||||
from sglang.srt.layers.quantization.fp8 import Fp8Config
|
||||
@@ -275,10 +277,15 @@ class DeepseekV2MLP(nn.Module):
|
||||
tp_rank: Optional[int] = None,
|
||||
tp_size: Optional[int] = None,
|
||||
swiglu_limit: Optional[float] = None,
|
||||
allow_fused_down: Optional[bool] = None,
|
||||
) -> None:
|
||||
super().__init__()
|
||||
self.tp_size = tp_size
|
||||
self.swiglu_limit = swiglu_limit
|
||||
# The fused down path calls deepgemm directly and runs no all-reduce.
|
||||
self.allow_fused_down = (
|
||||
not reduce_results if allow_fused_down is None else allow_fused_down
|
||||
)
|
||||
|
||||
self.gate_up_proj = MergedColumnParallelLinear(
|
||||
hidden_size,
|
||||
@@ -375,7 +382,7 @@ class DeepseekV2MLP(nn.Module):
|
||||
# are fp8 (uint8 storage with weight_scale_inv).
|
||||
if (
|
||||
self.swiglu_limit is not None
|
||||
and not self.down_proj.reduce_results
|
||||
and self.allow_fused_down
|
||||
and getattr(self.down_proj, "weight", None) is not None
|
||||
and self.down_proj.weight.dtype == torch.uint8
|
||||
and hasattr(self.down_proj, "weight_scale_inv")
|
||||
@@ -592,8 +599,11 @@ class DeepseekV2MoE(nn.Module):
|
||||
is_nextn: bool = False,
|
||||
is_deepseek_v4: bool = False,
|
||||
vl_correction_bias: bool = False,
|
||||
reduce_results: bool = True,
|
||||
):
|
||||
super().__init__()
|
||||
# False when a stage boundary completes this output's sum.
|
||||
self.reduce_results = reduce_results
|
||||
self.tp_size = get_parallel().tp_size
|
||||
self.moe_ep_size = get_parallel().moe_ep_size
|
||||
self.routed_scaling_factor = config.routed_scaling_factor
|
||||
@@ -1160,7 +1170,7 @@ class DeepseekV2MoE(nn.Module):
|
||||
final_hidden_states,
|
||||
gated_shared_output=shared_output,
|
||||
m=hidden_states.shape[0],
|
||||
reduce=post_experts_all_reduce,
|
||||
reduce=sum_post_experts_output,
|
||||
)
|
||||
|
||||
all_reduce_done = False
|
||||
@@ -1256,7 +1266,7 @@ class DeepseekV2MoE(nn.Module):
|
||||
self.routed_scaling_factor,
|
||||
)
|
||||
|
||||
if not all_reduce_done:
|
||||
if not all_reduce_done and self.reduce_results:
|
||||
if (
|
||||
self.is_deepseek_v4
|
||||
and self.tp_size > 1
|
||||
@@ -1274,9 +1284,9 @@ class DeepseekV2MoE(nn.Module):
|
||||
if mhc is not None:
|
||||
mhc.start_stats_before_all_reduce()
|
||||
final_hidden_states = post_experts_all_reduce(final_hidden_states)
|
||||
# TP1 shared experts are replicated, so add them after all-reduce to
|
||||
# avoid summing the same shared output once per TP rank.
|
||||
if self._shared_expert_tp1 and should_add_replicated_moe_output():
|
||||
# TP1 shared experts are replicated: add them after this block's
|
||||
# all-reduce, or on TP rank 0 only when the stage boundary sums.
|
||||
if self._shared_expert_tp1 and self._adds_replicated_output():
|
||||
final_hidden_states += shared_output
|
||||
return final_hidden_states
|
||||
|
||||
@@ -1431,31 +1441,40 @@ class DeepseekV2MoE(nn.Module):
|
||||
self.routed_scaling_factor,
|
||||
)
|
||||
|
||||
if (
|
||||
self.is_deepseek_v4
|
||||
and self.tp_size > 1
|
||||
and not should_skip_post_experts_all_reduce(is_tp_path=True)
|
||||
):
|
||||
from sglang.srt.layers.moe.mhc_post_fusion import current_mhc_post_fusion
|
||||
|
||||
mhc = current_mhc_post_fusion()
|
||||
if _is_hip and _hip_moe.fused_all_reduce_mhc(
|
||||
self, mhc, final_hidden_states
|
||||
if self.reduce_results:
|
||||
if (
|
||||
self.is_deepseek_v4
|
||||
and self.tp_size > 1
|
||||
and not should_skip_post_experts_all_reduce(is_tp_path=True)
|
||||
):
|
||||
return final_hidden_states
|
||||
if mhc is not None:
|
||||
mhc.start_stats_before_all_reduce()
|
||||
final_hidden_states = post_experts_all_reduce(final_hidden_states)
|
||||
# TP1 shared experts are replicated, so add them after all-reduce to
|
||||
# avoid summing the same shared output once per TP rank.
|
||||
from sglang.srt.layers.moe.mhc_post_fusion import (
|
||||
current_mhc_post_fusion,
|
||||
)
|
||||
|
||||
mhc = current_mhc_post_fusion()
|
||||
if _is_hip and _hip_moe.fused_all_reduce_mhc(
|
||||
self, mhc, final_hidden_states
|
||||
):
|
||||
return final_hidden_states
|
||||
if mhc is not None:
|
||||
mhc.start_stats_before_all_reduce()
|
||||
final_hidden_states = post_experts_all_reduce(final_hidden_states)
|
||||
# TP1 shared experts are replicated: add them after this block's
|
||||
# all-reduce, or on TP rank 0 only when the stage boundary sums.
|
||||
if (
|
||||
shared_output is not None
|
||||
and self._shared_expert_tp1
|
||||
and should_add_replicated_moe_output()
|
||||
and self._adds_replicated_output()
|
||||
):
|
||||
final_hidden_states += shared_output
|
||||
return final_hidden_states
|
||||
|
||||
def _adds_replicated_output(self) -> bool:
|
||||
"""Whether this rank adds the replicated TP1 shared-expert output."""
|
||||
if self.reduce_results:
|
||||
return should_add_replicated_moe_output()
|
||||
return adds_replicated_output_to_partial()
|
||||
|
||||
def forward_cpu(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
@@ -1507,7 +1526,8 @@ class DeepseekV2MoE(nn.Module):
|
||||
), # block_size
|
||||
True, # is_vnni
|
||||
)
|
||||
final_hidden_states = post_experts_all_reduce(final_hidden_states)
|
||||
if self.reduce_results:
|
||||
final_hidden_states = post_experts_all_reduce(final_hidden_states)
|
||||
return final_hidden_states
|
||||
|
||||
def forward_deepep(
|
||||
@@ -2661,6 +2681,7 @@ class DeepseekV2DecoderLayer(nn.Module):
|
||||
layer_id=self.layer_id,
|
||||
alt_stream=alt_stream,
|
||||
is_nextn=is_nextn,
|
||||
reduce_results=False,
|
||||
)
|
||||
else:
|
||||
if is_dense_ffn_fully_dp():
|
||||
@@ -2676,6 +2697,8 @@ class DeepseekV2DecoderLayer(nn.Module):
|
||||
tp_rank=mlp_tp_rank,
|
||||
tp_size=mlp_tp_size,
|
||||
swiglu_limit=getattr(config, "swiglu_limit", None),
|
||||
reduce_results=False,
|
||||
allow_fused_down=False,
|
||||
)
|
||||
|
||||
self.input_layernorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
|
||||
@@ -2876,7 +2899,7 @@ class DeepseekV2DecoderLayer(nn.Module):
|
||||
)
|
||||
|
||||
def op_comm_postprocess_layer(self, state):
|
||||
hidden_states = self.ffn_boundary.finish_complete_output(
|
||||
hidden_states = self.ffn_boundary.complete_now(
|
||||
state.pop("hidden_states_mlp_output"), state.forward_batch
|
||||
)
|
||||
|
||||
|
||||
@@ -45,9 +45,6 @@ from sglang.srt.batch_overlap.two_batch_overlap import (
|
||||
model_forward_stages,
|
||||
)
|
||||
from sglang.srt.configs.dots3 import Dots3Config
|
||||
from sglang.srt.distributed import (
|
||||
tensor_model_parallel_all_reduce,
|
||||
)
|
||||
from sglang.srt.distributed.device_communicators.pynccl_allocator import (
|
||||
use_symmetric_memory,
|
||||
)
|
||||
@@ -228,21 +225,13 @@ class Dots3MLP(nn.Module):
|
||||
)
|
||||
self.act_fn = SiluAndMul()
|
||||
|
||||
def forward(
|
||||
self,
|
||||
x,
|
||||
forward_batch=None,
|
||||
should_allreduce_fusion: bool = False,
|
||||
use_reduce_scatter: bool = False,
|
||||
):
|
||||
def forward(self, x, forward_batch=None):
|
||||
if (self.tp_size == 1) and x.shape[0] == 0:
|
||||
return x
|
||||
|
||||
gate_up, _ = self.gate_up_proj(x)
|
||||
x = self.act_fn(gate_up)
|
||||
x, _ = self.down_proj(
|
||||
x, skip_all_reduce=should_allreduce_fusion or use_reduce_scatter
|
||||
)
|
||||
x, _ = self.down_proj(x)
|
||||
return x
|
||||
|
||||
|
||||
@@ -416,8 +405,6 @@ class Dots3MoE(nn.Module):
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
forward_batch: Optional[ForwardBatch] = None,
|
||||
should_allreduce_fusion: bool = False,
|
||||
use_reduce_scatter: bool = False,
|
||||
) -> torch.Tensor:
|
||||
if not self._enable_deepep_moe:
|
||||
DUAL_STREAM_TOKEN_THRESHOLD = 1024
|
||||
@@ -427,26 +414,13 @@ class Dots3MoE(nn.Module):
|
||||
and hidden_states.shape[0] > 0
|
||||
and hidden_states.shape[0] <= DUAL_STREAM_TOKEN_THRESHOLD
|
||||
):
|
||||
return self.forward_normal_dual_stream(
|
||||
hidden_states,
|
||||
should_allreduce_fusion,
|
||||
use_reduce_scatter,
|
||||
)
|
||||
return self.forward_normal_dual_stream(hidden_states)
|
||||
else:
|
||||
return self.forward_normal(
|
||||
hidden_states,
|
||||
should_allreduce_fusion,
|
||||
use_reduce_scatter,
|
||||
)
|
||||
return self.forward_normal(hidden_states)
|
||||
else:
|
||||
return self.forward_deepep(hidden_states, forward_batch)
|
||||
|
||||
def forward_normal_dual_stream(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
should_allreduce_fusion: bool = False,
|
||||
use_reduce_scatter: bool = False,
|
||||
) -> torch.Tensor:
|
||||
def forward_normal_dual_stream(self, hidden_states: torch.Tensor) -> torch.Tensor:
|
||||
|
||||
current_stream = torch.cuda.current_stream()
|
||||
self.alt_stream.wait_stream(current_stream)
|
||||
@@ -465,21 +439,9 @@ class Dots3MoE(nn.Module):
|
||||
torch.add(final_hidden_states, shared_output, out=final_hidden_states_out)
|
||||
final_hidden_states = final_hidden_states_out
|
||||
sm.tag(final_hidden_states)
|
||||
if (
|
||||
self.tp_size > 1
|
||||
and not should_allreduce_fusion
|
||||
and not use_reduce_scatter
|
||||
and not should_use_flashinfer_cutlass_moe_fp4_allgather()
|
||||
):
|
||||
final_hidden_states = tensor_model_parallel_all_reduce(final_hidden_states)
|
||||
return final_hidden_states
|
||||
|
||||
def forward_normal(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
should_allreduce_fusion: bool = False,
|
||||
use_reduce_scatter: bool = False,
|
||||
) -> torch.Tensor:
|
||||
def forward_normal(self, hidden_states: torch.Tensor) -> torch.Tensor:
|
||||
if hidden_states.shape[0] > 0:
|
||||
shared_output = self._forward_shared_experts(hidden_states)
|
||||
# router_logits: (num_tokens, n_experts)
|
||||
@@ -496,13 +458,6 @@ class Dots3MoE(nn.Module):
|
||||
torch.add(final_hidden_states, shared_output, out=final_hidden_states_out)
|
||||
final_hidden_states = final_hidden_states_out
|
||||
sm.tag(final_hidden_states)
|
||||
if (
|
||||
self.tp_size > 1
|
||||
and not should_allreduce_fusion
|
||||
and not use_reduce_scatter
|
||||
and not should_use_flashinfer_cutlass_moe_fp4_allgather()
|
||||
):
|
||||
final_hidden_states = tensor_model_parallel_all_reduce(final_hidden_states)
|
||||
return final_hidden_states
|
||||
|
||||
def forward_deepep(
|
||||
@@ -1454,6 +1409,7 @@ class Dots3DecoderLayer(nn.Module):
|
||||
prefix=add_prefix("mlp", prefix),
|
||||
tp_rank=mlp_tp_rank,
|
||||
tp_size=mlp_tp_size,
|
||||
reduce_results=False,
|
||||
)
|
||||
|
||||
self.input_layernorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
|
||||
@@ -1509,12 +1465,7 @@ class Dots3DecoderLayer(nn.Module):
|
||||
hidden_states = self.ffn_boundary.prepare(hidden_states, forward_batch)
|
||||
|
||||
with self.ffn_boundary.exit(forward_batch) as ffn_exit:
|
||||
hidden_states = self.mlp(
|
||||
hidden_states,
|
||||
forward_batch,
|
||||
ffn_exit.fuse_mlp_allreduce,
|
||||
ffn_exit.mlp_reduce_scatter,
|
||||
)
|
||||
hidden_states = self.mlp(hidden_states, forward_batch)
|
||||
return ffn_exit.finish(hidden_states)
|
||||
|
||||
|
||||
|
||||
@@ -74,6 +74,7 @@ class Exaone4GatedMLP(nn.Module):
|
||||
hidden_size,
|
||||
bias=bias,
|
||||
quant_config=quant_config,
|
||||
reduce_results=False,
|
||||
prefix=add_prefix("down_proj", prefix),
|
||||
)
|
||||
if hidden_act != "silu":
|
||||
@@ -312,8 +313,9 @@ class Exaone4DecoderLayer(nn.Module):
|
||||
# Fully Connected
|
||||
hidden_states = self.attn_boundary.finish(hidden_states, forward_batch)
|
||||
hidden_states = self.ffn_boundary.prepare(hidden_states, forward_batch)
|
||||
hidden_states = self.mlp(hidden_states)
|
||||
return self.ffn_boundary.finish_complete_output(hidden_states, forward_batch)
|
||||
with self.ffn_boundary.exit(forward_batch) as ffn_exit:
|
||||
hidden_states = self.mlp(hidden_states)
|
||||
return ffn_exit.finish(hidden_states)
|
||||
|
||||
|
||||
class Exaone4Model(nn.Module):
|
||||
|
||||
@@ -214,6 +214,7 @@ class FalconH1HybridAttentionDecoderLayer(nn.Module):
|
||||
mlp_multipliers=config.mlp_multipliers,
|
||||
quant_config=quant_config,
|
||||
prefix=add_prefix("mlp", prefix),
|
||||
reduce_results=False,
|
||||
)
|
||||
|
||||
self.input_layernorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
|
||||
|
||||
@@ -59,7 +59,6 @@ from sglang.srt.layers.linear import (
|
||||
from sglang.srt.layers.logits_processor import LogitsProcessor
|
||||
from sglang.srt.layers.moe import (
|
||||
get_moe_a2a_backend,
|
||||
reduce_moe_output,
|
||||
should_use_flashinfer_cutlass_moe_fp4_allgather,
|
||||
)
|
||||
from sglang.srt.layers.moe.ep_moe.layer import get_moe_impl_class
|
||||
@@ -607,7 +606,6 @@ class Glm4MoeSparseMoeBlock(nn.Module):
|
||||
|
||||
current_stream.wait_stream(self.alt_stream)
|
||||
final_hidden_states += shared_output
|
||||
final_hidden_states = reduce_moe_output(final_hidden_states)
|
||||
return final_hidden_states
|
||||
|
||||
def forward_normal(
|
||||
@@ -633,7 +631,6 @@ class Glm4MoeSparseMoeBlock(nn.Module):
|
||||
final_hidden_states_out = torch.empty_like(final_hidden_states)
|
||||
torch.add(final_hidden_states, shared_output, out=final_hidden_states_out)
|
||||
final_hidden_states = final_hidden_states_out
|
||||
final_hidden_states = reduce_moe_output(final_hidden_states)
|
||||
return final_hidden_states
|
||||
|
||||
def forward_deepep(
|
||||
@@ -768,6 +765,7 @@ class Glm4MoeDecoderLayer(nn.Module):
|
||||
prefix=add_prefix("mlp", prefix),
|
||||
tp_rank=mlp_tp_rank,
|
||||
tp_size=mlp_tp_size,
|
||||
reduce_results=False,
|
||||
)
|
||||
|
||||
self.input_layernorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
|
||||
|
||||
@@ -50,7 +50,6 @@ from sglang.srt.layers.linear import MergedColumnParallelLinear, RowParallelLine
|
||||
from sglang.srt.layers.logits_processor import LogitsProcessor
|
||||
from sglang.srt.layers.moe import (
|
||||
get_moe_a2a_backend,
|
||||
reduce_moe_output,
|
||||
should_use_flashinfer_cutlass_moe_fp4_allgather,
|
||||
)
|
||||
from sglang.srt.layers.moe.ep_moe.layer import get_moe_impl_class
|
||||
@@ -338,7 +337,6 @@ class Glm4MoeLiteSparseMoeBlock(nn.Module):
|
||||
|
||||
current_stream.wait_stream(self.alt_stream)
|
||||
final_hidden_states += shared_output
|
||||
final_hidden_states = reduce_moe_output(final_hidden_states)
|
||||
return final_hidden_states
|
||||
|
||||
def forward_normal(
|
||||
@@ -364,7 +362,6 @@ class Glm4MoeLiteSparseMoeBlock(nn.Module):
|
||||
final_hidden_states_out = torch.empty_like(final_hidden_states)
|
||||
torch.add(final_hidden_states, shared_output, out=final_hidden_states_out)
|
||||
final_hidden_states = final_hidden_states_out
|
||||
final_hidden_states = reduce_moe_output(final_hidden_states)
|
||||
return final_hidden_states
|
||||
|
||||
def forward_deepep(
|
||||
@@ -476,6 +473,7 @@ class Glm4MoeLiteDecoderLayer(nn.Module):
|
||||
prefix=add_prefix("mlp", prefix),
|
||||
tp_rank=mlp_tp_rank,
|
||||
tp_size=mlp_tp_size,
|
||||
reduce_results=False,
|
||||
)
|
||||
|
||||
self.input_layernorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
|
||||
|
||||
@@ -734,6 +734,7 @@ class Glm5NextDecoderLayer(nn.Module):
|
||||
layer_id=self.layer_id,
|
||||
alt_stream=alt_stream,
|
||||
is_nextn=is_nextn,
|
||||
reduce_results=False,
|
||||
)
|
||||
else:
|
||||
if is_dense_ffn_fully_dp():
|
||||
@@ -749,6 +750,8 @@ class Glm5NextDecoderLayer(nn.Module):
|
||||
tp_rank=mlp_tp_rank,
|
||||
tp_size=mlp_tp_size,
|
||||
swiglu_limit=config.swiglu_limit,
|
||||
reduce_results=False,
|
||||
allow_fused_down=False,
|
||||
)
|
||||
|
||||
self.input_layernorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
|
||||
|
||||
@@ -48,7 +48,6 @@ from sglang.srt.layers.linear import (
|
||||
from sglang.srt.layers.logits_processor import LogitsProcessor
|
||||
from sglang.srt.layers.moe import (
|
||||
get_moe_a2a_backend,
|
||||
reduce_moe_output,
|
||||
)
|
||||
from sglang.srt.layers.moe.ep_moe.layer import get_moe_impl_class
|
||||
from sglang.srt.layers.moe.fused_moe_triton.layer import FusedMoE
|
||||
@@ -335,8 +334,6 @@ class GptOssSparseMoeBlock(nn.Module):
|
||||
topk_output = self.topk(router_input, router_logits)
|
||||
final_hidden_states = self.experts(hidden_states, topk_output)
|
||||
|
||||
final_hidden_states = reduce_moe_output(final_hidden_states)
|
||||
|
||||
# When input was pre-padded, FusedMoE.forward_impl captured the
|
||||
# padded width as `origin_hidden_states_dim` and skipped its own
|
||||
# output-trim contiguous() — so the experts output is still
|
||||
|
||||
@@ -31,9 +31,6 @@ from sglang.srt.layers.linear import (
|
||||
ReplicatedLinear,
|
||||
RowParallelLinear,
|
||||
)
|
||||
from sglang.srt.layers.moe import (
|
||||
reduce_moe_output,
|
||||
)
|
||||
from sglang.srt.layers.moe.ep_moe.layer import get_moe_impl_class
|
||||
from sglang.srt.layers.moe.topk import TopK
|
||||
from sglang.srt.layers.moe.utils import (
|
||||
@@ -422,7 +419,7 @@ class _InternS2MobiusDecoderMixin:
|
||||
routed = _get_mobius_routed_bank(meta_mlp, self.layer_id).forward_routed(
|
||||
hidden_states, forward_batch
|
||||
)
|
||||
return reduce_moe_output(routed + shared)
|
||||
return routed + shared
|
||||
|
||||
def _forward_after_attention(
|
||||
self,
|
||||
|
||||
@@ -12,7 +12,6 @@ from sglang.kernels.ops.attention.fla.fused_norm_gate import FusedRMSNormGated
|
||||
from sglang.srt.configs.kimi_linear import KimiLinearConfig
|
||||
from sglang.srt.distributed import (
|
||||
divide,
|
||||
tensor_model_parallel_all_reduce,
|
||||
)
|
||||
from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder
|
||||
from sglang.srt.layers.dcp.planner import prepare_decode_context_parallel_metadata
|
||||
@@ -185,9 +184,6 @@ class KimiMoE(nn.Module):
|
||||
|
||||
if shared_output is not None:
|
||||
final_hidden_states = final_hidden_states + shared_output
|
||||
|
||||
if self.tp_size > 1:
|
||||
final_hidden_states = tensor_model_parallel_all_reduce(final_hidden_states)
|
||||
return final_hidden_states.view(num_tokens, hidden_size)
|
||||
|
||||
|
||||
@@ -613,6 +609,7 @@ class KimiDecoderLayer(nn.Module):
|
||||
hidden_act=config.hidden_act,
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.mlp",
|
||||
reduce_results=False,
|
||||
)
|
||||
self.input_layernorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
|
||||
self.post_attention_layernorm = RMSNorm(
|
||||
@@ -655,8 +652,9 @@ class KimiDecoderLayer(nn.Module):
|
||||
# Fully Connected
|
||||
hidden_states = self.attn_boundary.finish(hidden_states, forward_batch)
|
||||
hidden_states = self.ffn_boundary.prepare(hidden_states, forward_batch)
|
||||
hidden_states = self.mlp(hidden_states)
|
||||
return self.ffn_boundary.finish_complete_output(hidden_states, forward_batch)
|
||||
with self.ffn_boundary.exit(forward_batch) as ffn_exit:
|
||||
hidden_states = self.mlp(hidden_states)
|
||||
return ffn_exit.finish(hidden_states)
|
||||
|
||||
|
||||
class KimiLinearModel(nn.Module):
|
||||
|
||||
@@ -37,11 +37,10 @@ from sglang.srt.layers.linear import (
|
||||
RowParallelLinear,
|
||||
)
|
||||
from sglang.srt.layers.logits_processor import LogitsProcessor
|
||||
from sglang.srt.layers.moe import reduce_moe_output
|
||||
from sglang.srt.layers.moe.ep_moe.layer import get_moe_impl_class
|
||||
from sglang.srt.layers.moe.fused_moe_triton.layer import FusedMoE
|
||||
from sglang.srt.layers.moe.topk import TopK
|
||||
from sglang.srt.layers.moe.utils import should_add_replicated_moe_output
|
||||
from sglang.srt.layers.moe.utils import adds_replicated_output_to_partial
|
||||
from sglang.srt.layers.quantization.base_config import QuantizationConfig
|
||||
from sglang.srt.layers.radix_attention import RadixAttention
|
||||
from sglang.srt.layers.rotary_embedding import get_rope
|
||||
@@ -226,8 +225,7 @@ class LagunaMoE(nn.Module):
|
||||
else:
|
||||
final = routed_out + shared_out
|
||||
|
||||
final = reduce_moe_output(final)
|
||||
if self._shared_expert_tp1 and should_add_replicated_moe_output():
|
||||
if self._shared_expert_tp1 and adds_replicated_output_to_partial():
|
||||
final = final + shared_out
|
||||
return final
|
||||
|
||||
@@ -448,7 +446,7 @@ class LagunaDecoderLayer(nn.Module):
|
||||
intermediate_size=config.intermediate_size,
|
||||
hidden_act=config.hidden_act,
|
||||
quant_config=quant_config,
|
||||
reduce_results=True,
|
||||
reduce_results=False,
|
||||
prefix=add_prefix("mlp", prefix),
|
||||
)
|
||||
|
||||
|
||||
@@ -51,7 +51,6 @@ from sglang.srt.layers.logits_processor import LogitsProcessor
|
||||
from sglang.srt.layers.moe import (
|
||||
get_deepep_mode,
|
||||
get_moe_a2a_backend,
|
||||
reduce_moe_output,
|
||||
)
|
||||
from sglang.srt.layers.moe.ep_moe.layer import get_moe_impl_class
|
||||
from sglang.srt.layers.moe.fused_moe_triton.layer import FusedMoE
|
||||
@@ -381,7 +380,6 @@ class LLaDA2MoeSparseMoeBlock(nn.Module):
|
||||
if self.num_shared_experts > 0:
|
||||
final_hidden_states = final_hidden_states + shared_output
|
||||
|
||||
final_hidden_states = reduce_moe_output(final_hidden_states)
|
||||
return final_hidden_states.view(num_tokens, hidden_size)
|
||||
|
||||
def forward_deepep(
|
||||
@@ -628,6 +626,7 @@ class LLaDA2MoeBlock(nn.Module):
|
||||
prefix=add_prefix("mlp", prefix),
|
||||
tp_rank=mlp_tp_rank,
|
||||
tp_size=mlp_tp_size,
|
||||
reduce_results=False,
|
||||
)
|
||||
|
||||
self.post_attention_layernorm = RMSNorm(hidden_size, eps=config.rms_norm_eps)
|
||||
|
||||
@@ -41,7 +41,6 @@ from sglang.srt.layers.linear import (
|
||||
ReplicatedLinear,
|
||||
RowParallelLinear,
|
||||
)
|
||||
from sglang.srt.layers.moe import reduce_moe_output
|
||||
from sglang.srt.layers.moe.fused_moe_triton import FusedMoE
|
||||
from sglang.srt.layers.moe.topk import TopK
|
||||
from sglang.srt.layers.quantization.base_config import QuantizationConfig
|
||||
@@ -148,8 +147,6 @@ class Llama4MoE(nn.Module):
|
||||
|
||||
out_aD = routed_out + shared_out
|
||||
|
||||
out_aD = reduce_moe_output(out_aD)
|
||||
|
||||
return out_aD
|
||||
|
||||
def _forward_core(self, hidden_states, forward_mode: ForwardMode):
|
||||
@@ -418,6 +415,7 @@ class Llama4DecoderLayer(nn.Module):
|
||||
hidden_act="silu",
|
||||
quant_config=quant_config,
|
||||
prefix=add_prefix("feed_forward", prefix),
|
||||
reduce_results=False,
|
||||
)
|
||||
self.input_layernorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
|
||||
self.post_attention_layernorm = RMSNorm(
|
||||
|
||||
@@ -44,9 +44,6 @@ from sglang.kernels.ops.gemm.bf16_fp32 import (
|
||||
from sglang.kernels.ops.moe.ep_moe_kernels import zero_experts_compute_triton
|
||||
from sglang.kernels.ops.quantization.fp8_kernel import is_fp8_fnuz
|
||||
from sglang.srt.configs import LongcatFlashConfig
|
||||
from sglang.srt.distributed import (
|
||||
tensor_model_parallel_all_reduce,
|
||||
)
|
||||
from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder
|
||||
from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation
|
||||
from sglang.srt.layers import deep_gemm_wrapper
|
||||
@@ -72,7 +69,6 @@ from sglang.srt.layers.moe.fused_moe_triton.layer import FusedMoE
|
||||
from sglang.srt.layers.moe.topk import StandardTopKOutput, TopK
|
||||
from sglang.srt.layers.moe.utils import (
|
||||
filter_moe_weight_param_global_expert,
|
||||
should_skip_mlp_all_reduce,
|
||||
)
|
||||
from sglang.srt.layers.n_gram_embedding import NgramEmbedding
|
||||
from sglang.srt.layers.quantization.base_config import QuantizationConfig
|
||||
@@ -325,18 +321,6 @@ class LongcatFlashMoE(nn.Module):
|
||||
if self.zero_expert_type is not None and hidden_states.shape[0] > 0:
|
||||
final_hidden_states += zero_expert_result.to(final_hidden_states.device)
|
||||
|
||||
# LONGCAT_MOE_A2A_SKIP_ALLREDUCE: skip the post-experts TP all-reduce when a
|
||||
# real EP a2a backend is active -- self.experts (DeepEPMoE) already combined
|
||||
# expert outputs across EP ranks, so an extra all-reduce double-counts.
|
||||
from sglang.srt.layers.moe.utils import get_moe_a2a_backend as _lc_gab
|
||||
|
||||
if (
|
||||
self.tp_size > 1
|
||||
and _lc_gab().is_none()
|
||||
and not should_skip_mlp_all_reduce()
|
||||
):
|
||||
final_hidden_states = tensor_model_parallel_all_reduce(final_hidden_states)
|
||||
|
||||
return final_hidden_states.view(num_tokens, hidden_dim)
|
||||
|
||||
def get_moe_weights(self):
|
||||
@@ -414,7 +398,7 @@ class LongcatFlashDecoderLayer(nn.Module):
|
||||
if "mlps" in getattr(config, "disable_quant_module", [])
|
||||
else quant_config
|
||||
),
|
||||
reduce_results=True,
|
||||
reduce_results=False,
|
||||
prefix=add_prefix(f"mlps.{i}", prefix),
|
||||
)
|
||||
for i in range(2)
|
||||
|
||||
@@ -154,7 +154,7 @@ class LongcatFlashDenseDecoderLayer(nn.Module):
|
||||
intermediate_size=config.intermediate_size,
|
||||
hidden_act=config.hidden_act,
|
||||
quant_config=quant_config,
|
||||
reduce_results=True,
|
||||
reduce_results=False,
|
||||
prefix=add_prefix(f"mlps", prefix),
|
||||
)
|
||||
self.input_layernorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
|
||||
@@ -191,10 +191,9 @@ class LongcatFlashDenseDecoderLayer(nn.Module):
|
||||
|
||||
hidden_states = self.attn_boundary.finish(hidden_states, forward_batch)
|
||||
hidden_states = self.ffn_boundary.prepare(hidden_states, forward_batch)
|
||||
hidden_states = self.mlp(hidden_states)
|
||||
hidden_states = self.ffn_boundary.finish_complete_output(
|
||||
hidden_states, forward_batch
|
||||
)
|
||||
with self.ffn_boundary.exit(forward_batch) as ffn_exit:
|
||||
hidden_states = self.mlp(hidden_states)
|
||||
hidden_states = ffn_exit.finish(hidden_states)
|
||||
return hidden_states
|
||||
|
||||
|
||||
|
||||
@@ -436,6 +436,7 @@ class MellumDecoderLayer(Qwen3MoeDecoderLayer):
|
||||
hidden_act=cfg.hidden_act,
|
||||
quant_config=quant_config,
|
||||
prefix=add_prefix("mlp", prefix),
|
||||
reduce_results=False,
|
||||
)
|
||||
|
||||
is_previous_layer_sparse = _is_sparse(layer_id - 1)
|
||||
|
||||
@@ -52,7 +52,6 @@ from sglang.srt.layers.logits_processor import LogitsProcessor
|
||||
from sglang.srt.layers.moe import (
|
||||
get_moe_a2a_backend,
|
||||
get_moe_runner_backend,
|
||||
reduce_moe_output,
|
||||
)
|
||||
from sglang.srt.layers.moe.ep_moe.layer import DeepEPMoE, get_moe_impl_class
|
||||
from sglang.srt.layers.moe.topk import TopK, TopKOutputFormat
|
||||
@@ -499,8 +498,6 @@ class MiMoV2MoE(nn.Module):
|
||||
|
||||
final_hidden_states = self.experts(hidden_states, topk_output)
|
||||
|
||||
final_hidden_states = reduce_moe_output(final_hidden_states)
|
||||
|
||||
return final_hidden_states
|
||||
|
||||
def forward_deepep(
|
||||
@@ -852,6 +849,7 @@ class MiMoV2DecoderLayer(nn.Module):
|
||||
prefix=add_prefix("mlp", prefix),
|
||||
tp_rank=mlp_tp_rank,
|
||||
tp_size=mlp_tp_size,
|
||||
reduce_results=False,
|
||||
)
|
||||
|
||||
self.input_layernorm = RMSNorm(config.hidden_size, eps=config.layernorm_epsilon)
|
||||
@@ -945,7 +943,7 @@ class MiMoV2DecoderLayer(nn.Module):
|
||||
)
|
||||
|
||||
def op_comm_postprocess_layer(self, state):
|
||||
hidden_states = self.ffn_boundary.finish_complete_output(
|
||||
hidden_states = self.ffn_boundary.complete_now(
|
||||
state.pop("hidden_states_mlp_output"), state.forward_batch
|
||||
)
|
||||
|
||||
|
||||
@@ -113,6 +113,7 @@ class MiMoV2MTPLayer(nn.Module):
|
||||
prefix=add_prefix("mlp", prefix),
|
||||
tp_rank=mlp_tp_rank,
|
||||
tp_size=mlp_tp_size,
|
||||
reduce_results=False,
|
||||
)
|
||||
self.input_layernorm = RMSNorm(config.hidden_size, eps=config.layernorm_epsilon)
|
||||
self.post_attention_layernorm = RMSNorm(
|
||||
|
||||
@@ -53,7 +53,6 @@ from sglang.srt.layers.linear import (
|
||||
from sglang.srt.layers.logits_processor import LogitsProcessor
|
||||
from sglang.srt.layers.moe import (
|
||||
get_moe_a2a_backend,
|
||||
reduce_moe_output,
|
||||
)
|
||||
from sglang.srt.layers.moe.ep_moe.layer import get_moe_impl_class
|
||||
from sglang.srt.layers.moe.fused_moe_triton.layer import FusedMoE
|
||||
@@ -602,7 +601,6 @@ class MiniMaxM2MoE(nn.Module):
|
||||
topk_output = self.topk.empty_topk_output(hidden_states.device)
|
||||
|
||||
final_hidden_states = self.experts(hidden_states, topk_output)
|
||||
final_hidden_states = reduce_moe_output(final_hidden_states)
|
||||
|
||||
return final_hidden_states.view(num_tokens, hidden_dim)
|
||||
|
||||
|
||||
@@ -28,9 +28,6 @@ from sglang.srt.configs.model_config import (
|
||||
get_minimax_sparse_disable_value_layer_ids,
|
||||
get_minimax_sparse_layer_ids,
|
||||
)
|
||||
from sglang.srt.distributed import (
|
||||
tensor_model_parallel_all_reduce,
|
||||
)
|
||||
from sglang.srt.environ import envs
|
||||
from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder
|
||||
from sglang.srt.eplb.expert_location_dispatch import ExpertLocationDispatchInfo
|
||||
@@ -315,15 +312,10 @@ class MiniMaxM3MLP(nn.Module):
|
||||
self,
|
||||
x,
|
||||
forward_batch: Optional[ForwardBatch] = None,
|
||||
should_allreduce_fusion: bool = False,
|
||||
use_reduce_scatter: bool = False,
|
||||
):
|
||||
gate_up, _ = self.gate_up_proj(x)
|
||||
x = self.act_fn(gate_up)
|
||||
x, _ = self.down_proj(
|
||||
x,
|
||||
skip_all_reduce=should_allreduce_fusion or use_reduce_scatter,
|
||||
)
|
||||
x, _ = self.down_proj(x)
|
||||
return x
|
||||
|
||||
|
||||
@@ -432,22 +424,13 @@ class MiniMaxM3MoE(nn.Module):
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
forward_batch: ForwardBatch,
|
||||
should_allreduce_fusion: bool = False,
|
||||
use_reduce_scatter: bool = False,
|
||||
) -> torch.Tensor:
|
||||
if get_moe_a2a_backend().is_deepep():
|
||||
return self.forward_deepep(hidden_states, forward_batch)
|
||||
else:
|
||||
return self.forward_normal(
|
||||
hidden_states, should_allreduce_fusion, use_reduce_scatter
|
||||
)
|
||||
return self.forward_normal(hidden_states)
|
||||
|
||||
def forward_normal(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
should_allreduce_fusion: bool = False,
|
||||
use_reduce_scatter: bool = False,
|
||||
) -> torch.Tensor:
|
||||
def forward_normal(self, hidden_states: torch.Tensor) -> torch.Tensor:
|
||||
if hidden_states.shape[0] > 0:
|
||||
if (
|
||||
self.alt_stream is not None
|
||||
@@ -470,9 +453,6 @@ class MiniMaxM3MoE(nn.Module):
|
||||
|
||||
if shared_output is not None:
|
||||
final_hidden_states = final_hidden_states + shared_output
|
||||
if self.tp_size > 1 and not should_allreduce_fusion and not use_reduce_scatter:
|
||||
final_hidden_states = tensor_model_parallel_all_reduce(final_hidden_states)
|
||||
|
||||
return final_hidden_states
|
||||
|
||||
def _forward_router_experts(self, hidden_states: torch.Tensor) -> torch.Tensor:
|
||||
@@ -1325,6 +1305,7 @@ class MiniMaxM3DecoderLayer(nn.Module):
|
||||
intermediate_size=config.dense_intermediate_size,
|
||||
tp_rank=mlp_tp_rank,
|
||||
tp_size=mlp_tp_size,
|
||||
reduce_results=False,
|
||||
)
|
||||
|
||||
self.use_gemma_norm = getattr(config, "use_gemma_norm", False)
|
||||
@@ -1395,12 +1376,7 @@ class MiniMaxM3DecoderLayer(nn.Module):
|
||||
|
||||
with self.ffn_boundary.exit(forward_batch) as ffn_exit:
|
||||
if self.is_layer_sparse or hidden_states.shape[0] != 0:
|
||||
hidden_states = self.mlp(
|
||||
hidden_states,
|
||||
forward_batch=forward_batch,
|
||||
should_allreduce_fusion=ffn_exit.fuse_mlp_allreduce,
|
||||
use_reduce_scatter=ffn_exit.mlp_reduce_scatter,
|
||||
)
|
||||
hidden_states = self.mlp(hidden_states, forward_batch=forward_batch)
|
||||
return ffn_exit.finish(hidden_states)
|
||||
|
||||
|
||||
|
||||
@@ -25,9 +25,6 @@ import torch
|
||||
from torch import nn
|
||||
from transformers import MixtralConfig
|
||||
|
||||
from sglang.srt.distributed import (
|
||||
tensor_model_parallel_all_reduce,
|
||||
)
|
||||
from sglang.srt.layers.layer_boundary import (
|
||||
declare_attn,
|
||||
declare_ffn,
|
||||
@@ -118,8 +115,6 @@ class MixtralMoE(nn.Module):
|
||||
router_logits, _ = self.gate(hidden_states)
|
||||
topk_output = self.topk(hidden_states, router_logits)
|
||||
final_hidden_states = self.experts(hidden_states, topk_output)
|
||||
if self.tp_size > 1:
|
||||
final_hidden_states = tensor_model_parallel_all_reduce(final_hidden_states)
|
||||
return final_hidden_states.view(orig_shape)
|
||||
|
||||
|
||||
@@ -269,8 +264,9 @@ class MixtralDecoderLayer(nn.Module):
|
||||
# Fully Connected
|
||||
hidden_states = self.attn_boundary.finish(hidden_states, forward_batch)
|
||||
hidden_states = self.ffn_boundary.prepare(hidden_states, forward_batch)
|
||||
hidden_states = self.block_sparse_moe(hidden_states)
|
||||
return self.ffn_boundary.finish_complete_output(hidden_states, forward_batch)
|
||||
with self.ffn_boundary.exit(forward_batch) as ffn_exit:
|
||||
hidden_states = self.block_sparse_moe(hidden_states)
|
||||
return ffn_exit.finish(hidden_states)
|
||||
|
||||
|
||||
class MixtralModel(nn.Module):
|
||||
|
||||
@@ -849,6 +849,7 @@ class MossVLTextMLP(nn.Module):
|
||||
hidden_act: str = "silu",
|
||||
quant_config: Optional[QuantizationConfig] = None,
|
||||
prefix: str = "",
|
||||
reduce_results: bool = True,
|
||||
):
|
||||
super().__init__()
|
||||
self.gate_up_proj = MergedColumnParallelLinear(
|
||||
@@ -863,6 +864,7 @@ class MossVLTextMLP(nn.Module):
|
||||
hidden_size,
|
||||
bias=False,
|
||||
quant_config=quant_config,
|
||||
reduce_results=reduce_results,
|
||||
prefix=add_prefix("down_proj", prefix),
|
||||
)
|
||||
if hidden_act != "silu":
|
||||
@@ -996,6 +998,7 @@ class MossVLSelfAttentionDecoderLayer(nn.Module):
|
||||
hidden_act=config.hidden_act,
|
||||
quant_config=quant_config,
|
||||
prefix=add_prefix("mlp", prefix),
|
||||
reduce_results=False,
|
||||
)
|
||||
norm_kwargs = (
|
||||
dict(
|
||||
|
||||
@@ -77,6 +77,7 @@ class NanbeigeMLP(nn.Module):
|
||||
hidden_size,
|
||||
bias=False,
|
||||
quant_config=quant_config,
|
||||
reduce_results=False,
|
||||
prefix=add_prefix("down_proj", prefix),
|
||||
)
|
||||
if hidden_act != "silu":
|
||||
@@ -261,8 +262,9 @@ class NanbeigeDecoderLayer(nn.Module):
|
||||
# Fully Connected
|
||||
hidden_states = self.attn_boundary.finish(hidden_states, forward_batch)
|
||||
hidden_states = self.ffn_boundary.prepare(hidden_states, forward_batch)
|
||||
hidden_states = self.mlp(hidden_states)
|
||||
return self.ffn_boundary.finish_complete_output(hidden_states, forward_batch)
|
||||
with self.ffn_boundary.exit(forward_batch) as ffn_exit:
|
||||
hidden_states = self.mlp(hidden_states)
|
||||
return ffn_exit.finish(hidden_states)
|
||||
|
||||
|
||||
class NanbeigeModel(nn.Module):
|
||||
|
||||
@@ -49,7 +49,6 @@ from sglang.srt.layers.moe.topk import TopK
|
||||
from sglang.srt.layers.moe.utils import (
|
||||
RoutingMethodType,
|
||||
get_moe_a2a_backend,
|
||||
reduce_moe_output,
|
||||
)
|
||||
from sglang.srt.layers.quantization import QuantizationConfig
|
||||
from sglang.srt.layers.quantization.unquant import UnquantizedLinearMethod
|
||||
@@ -79,7 +78,7 @@ from sglang.srt.model_loader.weight_utils import (
|
||||
)
|
||||
from sglang.srt.models.nemotron_h_utils import make_stage_boundary
|
||||
from sglang.srt.models.utils import WeightsMapper
|
||||
from sglang.srt.runtime_context import get_exec, get_forward, get_parallel
|
||||
from sglang.srt.runtime_context import get_exec, get_parallel
|
||||
from sglang.srt.utils import (
|
||||
add_prefix,
|
||||
get_current_device_stream_fast,
|
||||
@@ -365,8 +364,6 @@ class NemotronHMoE(nn.Module):
|
||||
elif shared_output is not None:
|
||||
final_hidden_states += shared_output
|
||||
|
||||
final_hidden_states = reduce_moe_output(final_hidden_states)
|
||||
|
||||
return final_hidden_states.view(num_tokens, hidden_dim)
|
||||
|
||||
|
||||
@@ -420,6 +417,7 @@ class NemotronHMLPDecoderLayer(NemotronHMLPLikeDecoderLayer):
|
||||
quant_config=quant_config,
|
||||
bias=config.mlp_bias,
|
||||
prefix=f"{prefix}.mixer",
|
||||
reduce_results=False,
|
||||
)
|
||||
|
||||
self.norm = RMSNorm(config.hidden_size, eps=config.layer_norm_epsilon)
|
||||
@@ -470,9 +468,7 @@ class NemotronHAttnLikeDecoderLayer(nn.Module):
|
||||
return hidden_states
|
||||
|
||||
with self.boundary.exit(forward_batch) as mixer_exit:
|
||||
hidden_states = self._forward_mixer(
|
||||
hidden_states, forward_batch, mixer_exit.skips_reduction
|
||||
)
|
||||
hidden_states = self._forward_mixer(hidden_states, forward_batch)
|
||||
return mixer_exit.finish(hidden_states)
|
||||
|
||||
|
||||
@@ -496,7 +492,7 @@ class NemotronHMambaDecoderLayer(NemotronHAttnLikeDecoderLayer):
|
||||
rms_norm_eps=config.layer_norm_epsilon,
|
||||
activation=config.mamba_hidden_act,
|
||||
quant_config=quant_config,
|
||||
reduce_results=True,
|
||||
reduce_results=False,
|
||||
prefix=f"{prefix}.mixer",
|
||||
)
|
||||
|
||||
@@ -525,19 +521,14 @@ class NemotronHMambaDecoderLayer(NemotronHAttnLikeDecoderLayer):
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
forward_batch: ForwardBatch,
|
||||
skip_reduce: bool,
|
||||
) -> torch.Tensor:
|
||||
if is_in_breakable_cuda_graph():
|
||||
output = torch.empty_like(hidden_states)
|
||||
breakable_nemotron_mamba2_with_output(
|
||||
hidden_states, output, self.layer_id, skip_reduce
|
||||
)
|
||||
breakable_nemotron_mamba2_with_output(hidden_states, output, self.layer_id)
|
||||
return output
|
||||
if is_in_tc_piecewise_cuda_graph():
|
||||
output = torch.empty_like(hidden_states)
|
||||
nemotron_mamba2_with_output(
|
||||
hidden_states, output, self.layer_id, skip_reduce
|
||||
)
|
||||
nemotron_mamba2_with_output(hidden_states, output, self.layer_id)
|
||||
return output
|
||||
return self._forward_mamba(hidden_states, forward_batch)
|
||||
|
||||
@@ -593,6 +584,7 @@ class NemotronHAttention(nn.Module):
|
||||
quant_config=quant_config,
|
||||
tp_rank=tp_rank,
|
||||
tp_size=tp_size,
|
||||
reduce_results=False,
|
||||
use_dp_attention_reduce=is_dp_attention_enabled(),
|
||||
prefix=f"{prefix}.o_proj",
|
||||
)
|
||||
@@ -645,7 +637,6 @@ class NemotronHAttentionDecoderLayer(NemotronHAttnLikeDecoderLayer):
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
forward_batch: ForwardBatch,
|
||||
skip_reduce: bool,
|
||||
) -> torch.Tensor:
|
||||
return self.mixer.forward(
|
||||
hidden_states=hidden_states, forward_batch=forward_batch
|
||||
@@ -1152,7 +1143,6 @@ def nemotron_mamba2_with_output(
|
||||
hidden_states: torch.Tensor,
|
||||
output: torch.Tensor,
|
||||
layer_id: int,
|
||||
fuse_mlp_allreduce: bool = False,
|
||||
) -> None:
|
||||
"""Split op for Mamba2 forward in piecewise CUDA graph mode."""
|
||||
context = get_tc_piecewise_forward_context()
|
||||
@@ -1172,12 +1162,7 @@ def nemotron_mamba2_with_output(
|
||||
if hidden_states.shape[0] != num_actual_tokens:
|
||||
hidden_states = hidden_states[:num_actual_tokens]
|
||||
|
||||
# This function is an opaque custom op under torch.compile. The caller's
|
||||
# ForwardFlags scope is Python control-plane state and is no longer active
|
||||
# when the compiled graph invokes this implementation. Carry the scalar
|
||||
# across the graph boundary and republish it for RowParallelLinear.
|
||||
with get_forward().scoped(fuse_mlp_allreduce=fuse_mlp_allreduce):
|
||||
ret = mamba_layer._forward_mamba(hidden_states, forward_batch)
|
||||
ret = mamba_layer._forward_mamba(hidden_states, forward_batch)
|
||||
|
||||
# Copy result back; output may be larger (padded) so only fill actual tokens
|
||||
output[:num_actual_tokens].view(ret.shape).copy_(ret)
|
||||
|
||||
@@ -72,6 +72,7 @@ class Qwen2MLP(nn.Module):
|
||||
hidden_act: str,
|
||||
quant_config: Optional[QuantizationConfig] = None,
|
||||
prefix: str = "",
|
||||
reduce_results: bool = True,
|
||||
) -> None:
|
||||
super().__init__()
|
||||
self.gate_up_proj = MergedColumnParallelLinear(
|
||||
@@ -86,6 +87,7 @@ class Qwen2MLP(nn.Module):
|
||||
hidden_size,
|
||||
bias=False,
|
||||
quant_config=quant_config,
|
||||
reduce_results=reduce_results,
|
||||
prefix=add_prefix("down_proj", prefix),
|
||||
)
|
||||
if hidden_act != "silu":
|
||||
|
||||
@@ -62,10 +62,7 @@ from sglang.srt.layers.linear import (
|
||||
RowParallelLinear,
|
||||
)
|
||||
from sglang.srt.layers.logits_processor import LogitsProcessor
|
||||
from sglang.srt.layers.moe import (
|
||||
get_moe_a2a_backend,
|
||||
reduce_moe_output,
|
||||
)
|
||||
from sglang.srt.layers.moe import get_moe_a2a_backend, reduce_moe_output
|
||||
from sglang.srt.layers.moe.ep_moe.layer import get_moe_impl_class
|
||||
from sglang.srt.layers.moe.fused_moe_triton import FusedMoE
|
||||
from sglang.srt.layers.moe.topk import StandardTopKOutput, TopK, TopKOutputChecker
|
||||
@@ -73,6 +70,7 @@ from sglang.srt.layers.moe.utils import (
|
||||
RoutingMethodType,
|
||||
filter_moe_weight_param_global_expert,
|
||||
is_deepep_class_backend,
|
||||
sum_post_experts_output,
|
||||
uses_per_rank_fused_shared_slots,
|
||||
)
|
||||
from sglang.srt.layers.quantization.base_config import QuantizationConfig
|
||||
@@ -266,8 +264,11 @@ class Qwen2MoeSparseMoeBlock(nn.Module):
|
||||
is_nextn: bool = False,
|
||||
support_shared_expert_fusion: bool = False,
|
||||
enable_cuda_shared_expert_fusion: bool = False,
|
||||
reduce_results: bool = True,
|
||||
):
|
||||
super().__init__()
|
||||
# False when a stage boundary completes this output's sum.
|
||||
self.reduce_results = reduce_results
|
||||
self.tp_size = get_parallel().tp_size
|
||||
self.layer_id = layer_id
|
||||
self.alt_stream = alt_stream
|
||||
@@ -871,7 +872,7 @@ class Qwen2MoeSparseMoeBlock(nn.Module):
|
||||
final_hidden_states,
|
||||
gated_shared_output=shared_output,
|
||||
m=num_tokens,
|
||||
reduce=reduce_moe_output,
|
||||
reduce=sum_post_experts_output,
|
||||
)
|
||||
|
||||
if shared_output is not None:
|
||||
@@ -884,7 +885,8 @@ class Qwen2MoeSparseMoeBlock(nn.Module):
|
||||
)
|
||||
else:
|
||||
final_hidden_states += shared_output
|
||||
final_hidden_states = reduce_moe_output(final_hidden_states)
|
||||
if self.reduce_results:
|
||||
final_hidden_states = reduce_moe_output(final_hidden_states)
|
||||
|
||||
# Debug removed - was causing issues during CUDA graph capture
|
||||
|
||||
@@ -1035,6 +1037,7 @@ class Qwen2MoeDecoderLayer(nn.Module):
|
||||
quant_config=quant_config,
|
||||
alt_stream=alt_stream,
|
||||
prefix=add_prefix("mlp", prefix),
|
||||
reduce_results=False,
|
||||
)
|
||||
else:
|
||||
self.mlp = Qwen2MoeMLP(
|
||||
@@ -1043,6 +1046,7 @@ class Qwen2MoeDecoderLayer(nn.Module):
|
||||
hidden_act=config.hidden_act,
|
||||
quant_config=quant_config,
|
||||
prefix=add_prefix("mlp", prefix),
|
||||
reduce_results=False,
|
||||
)
|
||||
self.input_layernorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
|
||||
self.post_attention_layernorm = RMSNorm(
|
||||
|
||||
@@ -355,6 +355,7 @@ class Qwen3DecoderLayer(nn.Module):
|
||||
hidden_act=config.hidden_act,
|
||||
quant_config=quant_config,
|
||||
prefix=add_prefix("mlp", prefix),
|
||||
reduce_results=False,
|
||||
)
|
||||
|
||||
norm_kwargs = (
|
||||
|
||||
@@ -1041,6 +1041,9 @@ class Qwen3_5GatedDeltaNet(nn.Module):
|
||||
class Qwen3_5LinearDecoderLayer(nn.Module):
|
||||
"""Qwen3.5 Decoder Layer with Linear Attention (GatedDeltaNet)."""
|
||||
|
||||
# True in a subclass built without stage boundaries.
|
||||
_ffn_sums_itself = False
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
config: Qwen3_5TextConfig,
|
||||
@@ -1073,6 +1076,7 @@ class Qwen3_5LinearDecoderLayer(nn.Module):
|
||||
prefix=add_prefix("mlp", prefix.replace(".linear_attn", "")),
|
||||
is_nextn=is_nextn,
|
||||
support_shared_expert_fusion=not _disable_shared_experts_fusion(),
|
||||
reduce_results=self._ffn_sums_itself,
|
||||
)
|
||||
is_layer_sparse = True
|
||||
is_previous_layer_sparse = True
|
||||
@@ -1084,6 +1088,7 @@ class Qwen3_5LinearDecoderLayer(nn.Module):
|
||||
hidden_act=config.hidden_act,
|
||||
quant_config=quant_config,
|
||||
prefix=add_prefix("mlp", prefix.replace(".linear_attn", "")),
|
||||
reduce_results=self._ffn_sums_itself,
|
||||
)
|
||||
_maybe_enable_silu_fp4_quant_fusion(self.mlp)
|
||||
is_layer_sparse = False
|
||||
@@ -1169,6 +1174,9 @@ class Qwen3_5LinearDecoderLayer(nn.Module):
|
||||
class Qwen3_5AttentionDecoderLayer(nn.Module):
|
||||
"""Qwen3.5 Decoder Layer with Full Attention."""
|
||||
|
||||
# See Qwen3_5LinearDecoderLayer._ffn_sums_itself.
|
||||
_ffn_sums_itself = False
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
config: Qwen3_5TextConfig,
|
||||
@@ -1271,6 +1279,7 @@ class Qwen3_5AttentionDecoderLayer(nn.Module):
|
||||
hidden_act=config.hidden_act,
|
||||
quant_config=quant_config,
|
||||
prefix=add_prefix("mlp", prefix.replace(".self_attn", "")),
|
||||
reduce_results=self._ffn_sums_itself,
|
||||
)
|
||||
is_layer_sparse = False
|
||||
is_previous_layer_sparse = False
|
||||
@@ -1288,6 +1297,7 @@ class Qwen3_5AttentionDecoderLayer(nn.Module):
|
||||
prefix=add_prefix("mlp", prefix.replace(".self_attn", "")),
|
||||
is_nextn=is_nextn,
|
||||
support_shared_expert_fusion=not _disable_shared_experts_fusion(),
|
||||
reduce_results=self._ffn_sums_itself,
|
||||
)
|
||||
is_layer_sparse = True
|
||||
is_previous_layer_sparse = True
|
||||
|
||||
@@ -45,7 +45,6 @@ from sglang.srt.layers.linear import (
|
||||
from sglang.srt.layers.logits_processor import LogitsProcessor
|
||||
from sglang.srt.layers.moe import (
|
||||
get_moe_a2a_backend,
|
||||
post_experts_all_reduce,
|
||||
)
|
||||
from sglang.srt.layers.moe.ep_moe.layer import get_moe_impl_class
|
||||
from sglang.srt.layers.moe.fused_moe_triton.layer import FusedMoE
|
||||
@@ -342,9 +341,6 @@ class Qwen3MoeSparseMoeBlock(nn.Module):
|
||||
else:
|
||||
topk_output = self.topk.empty_topk_output(hidden_states.device)
|
||||
final_hidden_states = self.experts(hidden_states, topk_output)
|
||||
|
||||
final_hidden_states = post_experts_all_reduce(final_hidden_states)
|
||||
|
||||
return final_hidden_states.view(num_tokens, hidden_dim)
|
||||
|
||||
def forward_deepep(
|
||||
@@ -856,6 +852,7 @@ class Qwen3MoeDecoderLayer(nn.Module):
|
||||
hidden_act=config.hidden_act,
|
||||
quant_config=quant_config,
|
||||
prefix=add_prefix("mlp", prefix),
|
||||
reduce_results=False,
|
||||
)
|
||||
self.input_layernorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
|
||||
self.post_attention_layernorm = RMSNorm(
|
||||
@@ -941,7 +938,7 @@ class Qwen3MoeDecoderLayer(nn.Module):
|
||||
)
|
||||
|
||||
def op_comm_postprocess_layer(self, state):
|
||||
hidden_states = self.ffn_boundary.finish_complete_output(
|
||||
hidden_states = self.ffn_boundary.complete_now(
|
||||
state.pop("hidden_states_mlp_output"), state.forward_batch
|
||||
)
|
||||
|
||||
|
||||
@@ -524,6 +524,7 @@ class Qwen3HybridLinearDecoderLayer(nn.Module):
|
||||
is_nextn=is_nextn,
|
||||
support_shared_expert_fusion=True,
|
||||
enable_cuda_shared_expert_fusion=True,
|
||||
reduce_results=False,
|
||||
)
|
||||
else:
|
||||
self.mlp = Qwen2MoeMLP(
|
||||
@@ -532,6 +533,7 @@ class Qwen3HybridLinearDecoderLayer(nn.Module):
|
||||
hidden_act=config.hidden_act,
|
||||
quant_config=quant_config,
|
||||
prefix=add_prefix("mlp", prefix.replace(".linear_attn", "")),
|
||||
reduce_results=False,
|
||||
)
|
||||
self.input_layernorm = GemmaRMSNorm(config.hidden_size, eps=config.rms_norm_eps)
|
||||
self.post_attention_layernorm = GemmaRMSNorm(
|
||||
@@ -688,6 +690,7 @@ class Qwen3HybridAttentionDecoderLayer(nn.Module):
|
||||
is_nextn=is_nextn,
|
||||
support_shared_expert_fusion=True,
|
||||
enable_cuda_shared_expert_fusion=True,
|
||||
reduce_results=False,
|
||||
)
|
||||
else:
|
||||
self.mlp = Qwen2MoeMLP(
|
||||
@@ -696,6 +699,7 @@ class Qwen3HybridAttentionDecoderLayer(nn.Module):
|
||||
hidden_act=config.hidden_act,
|
||||
quant_config=quant_config,
|
||||
prefix=add_prefix("mlp", prefix.replace(".self_attn", "")),
|
||||
reduce_results=False,
|
||||
)
|
||||
self.input_layernorm = GemmaRMSNorm(config.hidden_size, eps=config.rms_norm_eps)
|
||||
self.post_attention_layernorm = GemmaRMSNorm(
|
||||
|
||||
@@ -1410,6 +1410,9 @@ class Qwen4ExpPLELayer(nn.Module):
|
||||
|
||||
|
||||
class Qwen4ExpLayerExtensionMixin:
|
||||
# These layers drop the stage boundaries they inherit.
|
||||
_ffn_sums_itself = True
|
||||
|
||||
def _init_qwen4_exp_layer_extensions(
|
||||
self,
|
||||
config: Qwen4ExpTextConfig,
|
||||
|
||||
@@ -34,7 +34,6 @@ from sglang.srt.layers.linear import (
|
||||
RowParallelLinear,
|
||||
)
|
||||
from sglang.srt.layers.logits_processor import LogitsProcessor, LogitsProcessorOutput
|
||||
from sglang.srt.layers.moe import reduce_moe_output
|
||||
from sglang.srt.layers.moe.ep_moe.layer import get_moe_impl_class
|
||||
from sglang.srt.layers.moe.fused_moe_triton.layer import FusedMoE
|
||||
from sglang.srt.layers.moe.topk import TopK
|
||||
@@ -368,7 +367,6 @@ class SarvamMoESparseMoeBlock(nn.Module):
|
||||
final_hidden_states = final_hidden_states * self.routed_scaling_factor
|
||||
current_stream.wait_stream(self.alt_stream)
|
||||
final_hidden_states = final_hidden_states + shared_out
|
||||
final_hidden_states = reduce_moe_output(final_hidden_states)
|
||||
return final_hidden_states.view(num_tokens, hidden_dim)
|
||||
|
||||
def forward_normal(
|
||||
@@ -403,8 +401,6 @@ class SarvamMoESparseMoeBlock(nn.Module):
|
||||
elif self.routed_scaling_factor != 1.0:
|
||||
final_hidden_states = final_hidden_states * self.routed_scaling_factor
|
||||
|
||||
final_hidden_states = reduce_moe_output(final_hidden_states)
|
||||
|
||||
return final_hidden_states.view(num_tokens, hidden_dim)
|
||||
|
||||
|
||||
@@ -1038,6 +1034,7 @@ class SarvamMoEMLADecoderLayer(nn.Module):
|
||||
prefix=add_prefix("mlp", prefix),
|
||||
tp_rank=mlp_tp_rank,
|
||||
tp_size=mlp_tp_size,
|
||||
reduce_results=False,
|
||||
)
|
||||
|
||||
self.input_layernorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
|
||||
|
||||
@@ -290,7 +290,7 @@ class SDARBlock(nn.Module):
|
||||
self.mlp = SDARMLP(
|
||||
config=config,
|
||||
quant_config=quant_config,
|
||||
reduce_results=True,
|
||||
reduce_results=False,
|
||||
prefix=add_prefix("mlp", prefix),
|
||||
)
|
||||
|
||||
|
||||
@@ -31,7 +31,6 @@ from sglang.srt.layers.linear import (
|
||||
from sglang.srt.layers.logits_processor import LogitsProcessor
|
||||
from sglang.srt.layers.moe import (
|
||||
get_moe_a2a_backend,
|
||||
reduce_moe_output,
|
||||
)
|
||||
from sglang.srt.layers.moe.ep_moe.layer import get_moe_impl_class
|
||||
from sglang.srt.layers.moe.fused_moe_triton.layer import FusedMoE
|
||||
@@ -147,8 +146,6 @@ class SDARMoeSparseMoeBlock(nn.Module):
|
||||
topk_output = self.topk(hidden_states, router_logits)
|
||||
out = self.experts(hidden_states, topk_output) # (T, H)
|
||||
|
||||
out = reduce_moe_output(out)
|
||||
|
||||
return out.view(num_tokens, hidden_dim)
|
||||
|
||||
def forward_deepep(self, hidden_states: torch.Tensor, forward_batch: ForwardBatch):
|
||||
|
||||
@@ -264,6 +264,7 @@ class Spark2_5DecoderLayer(nn.Module):
|
||||
intermediate_size=config.intermediate_size,
|
||||
quant_config=quant_config,
|
||||
prefix=add_prefix("mlp", prefix),
|
||||
reduce_results=False,
|
||||
)
|
||||
|
||||
self.input_layernorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
|
||||
@@ -295,8 +296,9 @@ class Spark2_5DecoderLayer(nn.Module):
|
||||
)
|
||||
hidden_states = self.attn_boundary.finish(hidden_states, forward_batch)
|
||||
hidden_states = self.ffn_boundary.prepare(hidden_states, forward_batch)
|
||||
hidden_states = self.mlp(hidden_states)
|
||||
return self.ffn_boundary.finish_complete_output(hidden_states, forward_batch)
|
||||
with self.ffn_boundary.exit(forward_batch) as ffn_exit:
|
||||
hidden_states = self.mlp(hidden_states)
|
||||
return ffn_exit.finish(hidden_states)
|
||||
|
||||
|
||||
class Spark2_5Model(nn.Module):
|
||||
|
||||
@@ -41,7 +41,6 @@ from sglang.srt.layers.moe import get_moe_a2a_backend
|
||||
from sglang.srt.layers.moe.ep_moe.layer import get_moe_impl_class
|
||||
from sglang.srt.layers.moe.fused_moe_triton import FusedMoE
|
||||
from sglang.srt.layers.moe.topk import TopK
|
||||
from sglang.srt.layers.moe.utils import reduce_moe_output
|
||||
from sglang.srt.layers.quantization.base_config import QuantizationConfig
|
||||
from sglang.srt.layers.radix_attention import RadixAttention
|
||||
from sglang.srt.layers.rotary_embedding import get_rope
|
||||
@@ -346,6 +345,7 @@ class Step3TextDecoderLayer(nn.Module):
|
||||
hidden_act="silu",
|
||||
quant_config=quant_config,
|
||||
prefix=add_prefix("mlp", prefix),
|
||||
reduce_results=False,
|
||||
)
|
||||
else:
|
||||
self.use_moe = True
|
||||
@@ -397,7 +397,7 @@ class Step3TextDecoderLayer(nn.Module):
|
||||
hidden_states += self.share_expert(h)
|
||||
else:
|
||||
hidden_states = self.moe(hidden_states)
|
||||
return reduce_moe_output(hidden_states)
|
||||
return hidden_states
|
||||
|
||||
def forward(
|
||||
self,
|
||||
|
||||
@@ -27,7 +27,6 @@ from sglang.srt.layers.linear import (
|
||||
from sglang.srt.layers.logits_processor import LogitsProcessor
|
||||
from sglang.srt.layers.moe import (
|
||||
get_moe_a2a_backend,
|
||||
reduce_moe_output,
|
||||
should_use_flashinfer_cutlass_moe_fp4_allgather,
|
||||
)
|
||||
from sglang.srt.layers.moe.ep_moe.layer import get_moe_impl_class
|
||||
@@ -226,7 +225,6 @@ class Step3p5MoEMLP(nn.Module):
|
||||
final_hidden_states = self.experts(hidden_states, topk_output)
|
||||
if shared_output is not None:
|
||||
final_hidden_states += shared_output
|
||||
final_hidden_states = reduce_moe_output(final_hidden_states)
|
||||
|
||||
return final_hidden_states.view(num_tokens, hidden_dim)
|
||||
|
||||
@@ -506,6 +504,7 @@ class Step3p5DecoderLayer(nn.Module):
|
||||
swiglu_limit=swiglu_limit_shared,
|
||||
quant_config=quant_config,
|
||||
prefix=add_prefix("mlp", prefix),
|
||||
reduce_results=False,
|
||||
)
|
||||
|
||||
self.input_layernorm = GemmaRMSNorm(config.hidden_size, eps=config.rms_norm_eps)
|
||||
|
||||
@@ -58,7 +58,6 @@ from sglang.srt.layers.linear import (
|
||||
from sglang.srt.layers.logits_processor import LogitsProcessor
|
||||
from sglang.srt.layers.moe import (
|
||||
get_moe_a2a_backend,
|
||||
reduce_moe_output,
|
||||
)
|
||||
from sglang.srt.layers.moe.ep_moe.layer import get_moe_impl_class
|
||||
from sglang.srt.layers.moe.fused_moe_triton import FusedMoE
|
||||
@@ -1059,7 +1058,6 @@ class XllmSparseMoeBlock(nn.Module):
|
||||
|
||||
if shared_output is not None:
|
||||
final_hidden_states += shared_output
|
||||
final_hidden_states = reduce_moe_output(final_hidden_states)
|
||||
|
||||
return final_hidden_states.view(num_tokens, hidden_dim)
|
||||
|
||||
@@ -1520,6 +1518,7 @@ class XllmDecoderLayer(nn.Module):
|
||||
prefix=add_prefix("mlp", prefix),
|
||||
tp_rank=mlp_tp_rank,
|
||||
tp_size=mlp_tp_size,
|
||||
reduce_results=False,
|
||||
)
|
||||
|
||||
self.input_layernorm = _make_norm(config)
|
||||
|
||||
@@ -152,7 +152,7 @@ def postprocess_output(boundary, hidden, residual, forward_batch):
|
||||
stream = (
|
||||
residual if isinstance(residual, ResidualStream) else ResidualStream(residual)
|
||||
)
|
||||
hidden = boundary.finish_complete_output(hidden, stream, forward_batch)
|
||||
hidden = boundary.complete_now(hidden, stream, forward_batch)
|
||||
if isinstance(residual, ResidualStream):
|
||||
return hidden, stream
|
||||
return (hidden, None) if stream.pending is None else stream.input(hidden)
|
||||
|
||||
Reference in New Issue
Block a user