[Refactor] Replace LayerScatterModes with LayerFacts and remove ScatterMode (#41443)

This commit is contained in:
Cheng Wan
2026-09-27 16:56:11 -07:00
committed by GitHub
parent 06d012eaca
commit 81f27fb3a7
49 changed files with 294 additions and 471 deletions
@@ -42,14 +42,13 @@ from sglang.srt.layers.communicator.layer import (
FfnExit,
FfnExitFusion,
LayerCommunicator,
LayerScatterModes,
LayerFacts,
MHCLayerCommunicator,
MixerExit,
)
from sglang.srt.layers.communicator.layout import (
CommunicateContext,
Layout,
ScatterMode,
SumGroup,
TokenAxis,
enable_moe_dense_fully_dp,
@@ -98,7 +97,7 @@ __all__ = [
"HandoffOutput",
"LayerCommunicator",
"LayerResidual",
"LayerScatterModes",
"LayerFacts",
"LayerStage",
"Layout",
"MHCLayerCommunicator",
@@ -107,7 +106,6 @@ __all__ = [
"NORM_QUANT_READ",
"NORM_READ",
"PLAIN_RESIDUAL",
"ScatterMode",
"StageDecl",
"StageEntry",
"StageInput",
+32 -136
View File
@@ -24,7 +24,6 @@ import torch
from sglang.srt.distributed import GroupCoordinator
from sglang.srt.environ import envs
from sglang.srt.layers import layernorm_sp
from sglang.srt.layers.attention.dsa.utils import is_dsa_enable_prefill_cp
from sglang.srt.layers.aux_hidden_states import AuxHiddenStateAccumulator
from sglang.srt.layers.communicator.adapters.attention import get_attn_tp_context
from sglang.srt.layers.communicator.boundary import (
@@ -47,7 +46,6 @@ from sglang.srt.layers.communicator.boundary import (
from sglang.srt.layers.communicator.layout import (
CommunicateContext,
Layout,
ScatterMode,
SumGroup,
TokenAxis,
_batch_shards_over_cp,
@@ -56,7 +54,6 @@ from sglang.srt.layers.communicator.layout import (
_gathers_over_attention_cp,
_generic_prefill_cp_shards_tokens,
_sum_group,
enable_dwdp,
enable_moe_dense_fully_dp,
sparse_moe_gathers_over_moe_cp,
token_axis_sizes,
@@ -83,7 +80,6 @@ from sglang.srt.layers.communicator.residual.add_norm import (
apply_flashinfer_allreduce_fusion,
)
from sglang.srt.layers.communicator.residual.mhc import MHCState
from sglang.srt.layers.cp.utils import is_mla_cp_enabled
from sglang.srt.layers.dp_attention import (
is_dp_attention_enabled,
is_enable_moe_cp_allgather,
@@ -94,7 +90,6 @@ from sglang.srt.layers.moe import (
is_moe_input_scattered_across_dp_ranks,
post_experts_reduction_group,
post_experts_sum_is_one_all_reduce,
should_use_flashinfer_cutlass_moe_fp4_allgather,
)
from sglang.srt.model_executor.forward_batch_info import ForwardBatch
from sglang.srt.runtime_context import (
@@ -113,131 +108,39 @@ _use_ag_after_qlora = envs.SGLANG_USE_AG_AFTER_QLORA.get()
@dataclass
class _LayerModeComputationContext:
num_layers: int
layer_id: int
is_layer_sparse: bool
is_previous_layer_sparse: Optional[bool]
is_next_layer_sparse: Optional[bool]
class LayerFacts:
"""Where a layer sits in the model and whether it and its neighbours have a
sparse MLP: the facts its boundaries are declared from."""
def previous_layer(self):
assert self.is_previous_layer_sparse is not None
return _LayerModeComputationContext(
num_layers=self.num_layers,
layer_id=self.layer_id - 1,
is_layer_sparse=self.is_previous_layer_sparse,
is_previous_layer_sparse=None,
is_next_layer_sparse=self.is_layer_sparse,
)
def sparse_mlp_scatter_mode() -> ScatterMode:
"""SCATTERED hands a sparse MLP this rank's own token shard; FULL and
MOE_FULL hand it a buffer gathered over the attn-TP or MoE-CP group."""
if (
# Token dispatch/combine will be handled outside of LayerCommunicator for these modes.
not get_moe_a2a_backend().is_none()
or should_use_flashinfer_cutlass_moe_fp4_allgather()
or enable_dwdp()
):
return ScatterMode.SCATTERED
# DSA CP and MLA CP both don't support MOE_FULL yet; fall back to FULL.
if is_enable_moe_cp_allgather() and not (
is_dsa_enable_prefill_cp() or is_mla_cp_enabled()
):
return ScatterMode.MOE_FULL
return ScatterMode.FULL
@dataclass
class LayerScatterModes:
layer_input_mode: ScatterMode
attn_mode: ScatterMode
# Can be further split into e.g. mlp_input_mode and mlp_output_mode if needed
mlp_mode: ScatterMode
middle_residual_mode: ScatterMode
layer_output_mode: ScatterMode
is_layer_sparse: bool = False
# The model's first layer: its input is the embedding, not a layer output.
is_first_layer: bool = False
# The model's last layer: its output goes to the final norm, not a next layer.
is_last_layer: bool = False
# Whether the layer before this one has a sparse MLP; None when the modes
# Whether the layer before this one has a sparse MLP; None when the facts
# were given directly, not planned from the layer sequence.
is_previous_layer_sparse: Optional[bool] = None
# Whether the layer after this one has a sparse MLP; None likewise.
is_next_layer_sparse: Optional[bool] = None
@classmethod
def init_new(cls, **kwargs):
context = _LayerModeComputationContext(**kwargs)
def init_new(
cls,
*,
layer_id: int,
num_layers: int,
is_layer_sparse: bool,
is_previous_layer_sparse: Optional[bool],
is_next_layer_sparse: Optional[bool],
) -> "LayerFacts":
return cls(
layer_input_mode=cls._compute_layer_input_mode(context),
attn_mode=ScatterMode.TP_ATTN_FULL,
mlp_mode=cls._compute_mlp_mode(context),
middle_residual_mode=cls._compute_middle_residual_mode(context),
layer_output_mode=cls._compute_layer_output_mode(context),
is_layer_sparse=context.is_layer_sparse,
is_first_layer=context.layer_id == 0,
is_last_layer=context.layer_id == context.num_layers - 1,
is_previous_layer_sparse=context.is_previous_layer_sparse,
is_next_layer_sparse=context.is_next_layer_sparse,
is_layer_sparse=is_layer_sparse,
is_first_layer=layer_id == 0,
is_last_layer=layer_id == num_layers - 1,
is_previous_layer_sparse=is_previous_layer_sparse,
is_next_layer_sparse=is_next_layer_sparse,
)
@classmethod
def _compute_layer_input_mode(cls, context: _LayerModeComputationContext):
if context.layer_id == 0:
return ScatterMode.model_input_output()
return cls._compute_layer_output_mode(context.previous_layer())
@classmethod
def _compute_mlp_mode(cls, context: _LayerModeComputationContext):
if context.is_layer_sparse:
return sparse_mlp_scatter_mode()
else:
if enable_moe_dense_fully_dp():
return ScatterMode.SCATTERED
# A TP-sharded dense MLP reduces over the whole TP group, which spans
# every CP rank; a CP-sharded prefill must gather tokens across CP
# first or the all-reduce sums different tokens' partial outputs.
# MLA/DSA CP models gather over the attention-CP group instead.
if _generic_prefill_cp_shards_tokens() and not (
is_dsa_enable_prefill_cp() or is_mla_cp_enabled()
):
return ScatterMode.MOE_FULL
return ScatterMode.FULL
@classmethod
def _should_gather_for_tbo(cls, context: _LayerModeComputationContext):
return (
not context.is_layer_sparse
and context.is_next_layer_sparse
and enable_moe_dense_fully_dp()
and get_exec().overlap.enable_two_batch_overlap
)
@classmethod
def _compute_middle_residual_mode(cls, context: _LayerModeComputationContext):
mlp_mode = cls._compute_mlp_mode(context)
if mlp_mode == ScatterMode.SCATTERED:
return ScatterMode.SCATTERED
if mlp_mode in (ScatterMode.FULL, ScatterMode.MOE_FULL):
return ScatterMode.TP_ATTN_FULL
raise NotImplementedError
@classmethod
def _compute_layer_output_mode(cls, context: _LayerModeComputationContext):
mlp_mode = cls._compute_mlp_mode(context)
if context.layer_id == context.num_layers - 1:
return ScatterMode.model_input_output()
if mlp_mode == ScatterMode.SCATTERED:
if cls._should_gather_for_tbo(context):
return ScatterMode.TP_ATTN_FULL
return ScatterMode.SCATTERED
if mlp_mode in (ScatterMode.FULL, ScatterMode.MOE_FULL):
return ScatterMode.TP_ATTN_FULL
raise NotImplementedError
def _unfused_completion_matches_the_ffn(forward_batch: ForwardBatch) -> bool:
"""Whether the next layer's all-reduce is the one the FFN would have run
@@ -329,7 +232,7 @@ class LayerCommunicator:
def __init__(
self,
layer_scatter_modes: LayerScatterModes,
layer_facts: LayerFacts,
input_layernorm: torch.nn.Module,
post_attention_layernorm: torch.nn.Module,
# Reduce scatter requires skipping all-reduce in model code after MoE/MLP, so only enable for models which have that implemented. Remove flag once done for all models that use LayerCommunicator.
@@ -349,11 +252,11 @@ class LayerCommunicator:
# The fused kernels a backend gives the layer, tried before its own.
fusions: Optional[LayerFusions] = None,
):
self.layer_scatter_modes = layer_scatter_modes
self.layer_facts = layer_facts
self.input_layernorm = input_layernorm
self.post_attention_layernorm = post_attention_layernorm
self.allow_reduce_scatter = allow_reduce_scatter
self.is_last_layer = layer_scatter_modes.is_last_layer
self.is_last_layer = layer_facts.is_last_layer
self.qkv_latent_func = qkv_latent_func
self.force_layernorm_before_dp_gather = force_layernorm_before_dp_gather
self.enable_fused_ar_quant = enable_fused_ar_quant
@@ -425,7 +328,7 @@ class LayerCommunicator:
residual,
),
attention_fusions=self._attn_input_fusions,
enters_stack=self.layer_scatter_modes.is_first_layer,
enters_stack=self.layer_facts.is_first_layer,
)
if layernorm_sp.layernorm_sp_enabled()
else None
@@ -514,7 +417,7 @@ class LayerCommunicator:
hold every token on each CP rank. An active LayerNorm SP region and
input-scattered attention have their own. Raises NotImplementedError
for the combinations the steps do not cover."""
modes = self.layer_scatter_modes
modes = self.layer_facts
parallel = get_parallel()
if not (modes.is_first_layer or modes.is_previous_layer_sparse is not None):
raise NotImplementedError(
@@ -639,7 +542,7 @@ class LayerCommunicator:
scattered = scattered_residual_layer_sides(
axis_sizes=token_axis_sizes(),
ffn_group=sides.ffn_output.group,
is_first_layer=self.layer_scatter_modes.is_first_layer,
is_first_layer=self.layer_facts.is_first_layer,
is_last_layer=self.is_last_layer,
leaves_for_reduce_scatter=self.allow_reduce_scatter,
)
@@ -649,7 +552,7 @@ class LayerCommunicator:
with_residual(scattered, self._residual),
attention_handoff=handoff,
attention_fusions=self._attn_input_fusions,
enters_stack=self.layer_scatter_modes.is_first_layer,
enters_stack=self.layer_facts.is_first_layer,
)
def _steps_from_declarations(
@@ -659,7 +562,7 @@ class LayerCommunicator:
return _select_boundary_steps(
sides,
attention_fusions=self._attn_input_fusions,
enters_stack=self.layer_scatter_modes.is_first_layer,
enters_stack=self.layer_facts.is_first_layer,
**kwargs,
)
@@ -748,7 +651,7 @@ class LayerCommunicator:
# The SP region opens at the first layer, re-evaluated per forward so a
# crash mid-loop cannot leak into the next one. It sets what the batch's
# steps are chosen from, and the first layer's input owes nothing.
if self._sp_steps is not None and self.layer_scatter_modes.is_first_layer:
if self._sp_steps is not None and self.layer_facts.is_first_layer:
get_forward().set(
"sp_active", layernorm_sp.runs_sp(forward_batch.forward_mode)
)
@@ -1041,7 +944,7 @@ class LayerCommunicator:
forward_batch=forward_batch,
context=self._context,
allow_reduce_scatter=self.allow_reduce_scatter,
is_layer_sparse=self.layer_scatter_modes.is_layer_sparse,
is_layer_sparse=self.layer_facts.is_layer_sparse,
)
update = steps.ffn_output.update
if residual is not None and update.at_producer:
@@ -1136,13 +1039,6 @@ class LayerCommunicator:
return hidden_states, residual
return reduce_output(hidden_states), residual
def should_use_reduce_scatter(self, forward_batch: ForwardBatch):
"""Whether the FFN leaves its sum to a reduce-scatter, for layers that run
their FFN outside ffn_exit."""
return self._ffn_leaves_sum_to_reduce_scatter(
forward_batch, self._postprocess_dp_step(forward_batch)
)
def _select_ffn_exit_fusions(
self,
) -> Tuple[Callable[[ForwardBatch], Optional["FfnExitFusion"]], ...]:
@@ -1359,7 +1255,7 @@ class MHCLayerCommunicator(LayerCommunicator):
def __init__(
self,
layer_scatter_modes: LayerScatterModes,
layer_facts: LayerFacts,
input_layernorm: torch.nn.Module,
post_attention_layernorm: torch.nn.Module,
allow_reduce_scatter: bool = False,
@@ -1377,9 +1273,9 @@ class MHCLayerCommunicator(LayerCommunicator):
hc_ffn_pre=hc_ffn_pre,
hc_post=hc_post,
hc_ffn_post_pre=hc_ffn_post_pre,
is_last_layer=layer_scatter_modes.is_last_layer,
is_last_layer=layer_facts.is_last_layer,
)
if layer_scatter_modes.is_layer_sparse and sparse_moe_gathers_over_moe_cp():
if layer_facts.is_layer_sparse and sparse_moe_gathers_over_moe_cp():
raise NotImplementedError(
"MHCLayerCommunicator does not support a MoE gathered over the "
"MoE-CP group (moe_dp_size < attention_context_parallel_size). "
@@ -1388,7 +1284,7 @@ class MHCLayerCommunicator(LayerCommunicator):
# The postprocess writes the FFN output into the streams, so the FFN's
# sum never waits for the next layer.
super().__init__(
layer_scatter_modes,
layer_facts,
input_layernorm,
post_attention_layernorm,
allow_reduce_scatter,
@@ -71,30 +71,6 @@ class SumGroup(Enum):
MOE_OUTPUT = auto()
class ScatterMode(Enum):
"""
Suppose we have TP=4, DP=2, enable-dp-attention, and the system handles seq a,b,c,d
Model input/output: [ab, ab, cd, cd] for four ranks respectively
SCATTERED: [a, b, c, d]
TP_ATTN_FULL: [ab, ab, cd, cd], i.e. all ranks inside a TP attn group have full data of the group
FULL: [abcd, abcd, abcd, abcd]
MOE_FULL: full within the MoE group (cp_per_moe CP chunks), used when moe_dp_size < attn_cp_size
"""
SCATTERED = auto()
TP_ATTN_FULL = auto()
FULL = auto()
MOE_FULL = auto()
@staticmethod
def model_input_output():
"""The scatter mode for model forward pass input and output data"""
if is_dsa_enable_prefill_cp() or is_mla_cp_enabled():
return ScatterMode.SCATTERED
return ScatterMode.TP_ATTN_FULL
def enable_moe_dense_fully_dp():
return get_parallel().moe_dense_tp_size == 1
@@ -105,10 +81,6 @@ def _generic_prefill_cp_shards_tokens() -> bool:
return parallel.attn_cp_size > 1 and parallel.enable_prefill_cp
def enable_dwdp():
return get_parallel().dwdp_size > 1
def _batch_size(forward_batch: ForwardBatch) -> int:
return (
forward_batch.input_ids.shape[0] if hasattr(forward_batch, "input_ids") else 0
@@ -124,7 +96,6 @@ def _ffn_has_tokens(forward_batch: ForwardBatch) -> bool:
@dataclass
class CommunicateContext:
process_group_sizes: Dict[ScatterMode, int]
attn_tp_rank: int
attn_tp_size: int
attn_dp_size: int
@@ -144,18 +115,7 @@ class CommunicateContext:
attn_cp_rank = get_parallel().attn_cp_rank
tp_size = get_parallel().tp_size
tp_rank = get_parallel().tp_rank
moe_cp_size = get_moe_cp_size()
process_group_sizes = {
ScatterMode.SCATTERED: 1,
ScatterMode.TP_ATTN_FULL: attn_tp_size,
# TODO: support --moe-dense-tp-size > 1
# With context parallel enabled, we should exclude
# the attn_cp_size from the total tp_size
ScatterMode.FULL: tp_size // attn_cp_size,
ScatterMode.MOE_FULL: tp_size // (attn_cp_size // moe_cp_size),
}
return cls(
process_group_sizes=process_group_sizes,
attn_tp_rank=attn_tp_rank,
attn_tp_size=attn_tp_size,
attn_dp_size=attn_dp_size,
+3 -3
View File
@@ -33,7 +33,7 @@ from sglang.srt.eplb.expert_location_dispatch import ExpertLocationDispatchInfo
from sglang.srt.layers.activation import SiluAndMul
from sglang.srt.layers.communicator import (
LayerCommunicator,
LayerScatterModes,
LayerFacts,
enable_moe_dense_fully_dp,
reduce_output,
)
@@ -669,7 +669,7 @@ class BailingMoEBlock(nn.Module):
config, layer_id=layer_id + 1, is_nextn=False
)
self.layer_scatter_modes = LayerScatterModes.init_new(
self.layer_facts = LayerFacts.init_new(
layer_id=layer_id,
# A NextN draft is a one-layer model.
num_layers=1 if is_nextn else config.num_hidden_layers,
@@ -703,7 +703,7 @@ class BailingMoEBlock(nn.Module):
self.post_attention_layernorm = RMSNorm(hidden_size, eps=config.rms_norm_eps)
self.layer_communicator = LayerCommunicator(
layer_scatter_modes=self.layer_scatter_modes,
layer_facts=self.layer_facts,
input_layernorm=self.input_layernorm,
post_attention_layernorm=self.post_attention_layernorm,
allow_reduce_scatter=True,
@@ -17,7 +17,7 @@ from sglang.srt.layers import deep_gemm_wrapper
from sglang.srt.layers.activation import SiluAndMul
from sglang.srt.layers.communicator import (
LayerCommunicator,
LayerScatterModes,
LayerFacts,
)
from sglang.srt.layers.dp_attention import (
is_dp_attention_enabled,
@@ -813,7 +813,7 @@ class BailingMoELinearDecoderLayer(nn.Module):
self.input_layernorm = RMSNorm(self.hidden_size, eps=rms_norm_eps)
self.post_attention_layernorm = RMSNorm(self.hidden_size, eps=rms_norm_eps)
self.layer_scatter_modes = LayerScatterModes.init_new(
self.layer_facts = LayerFacts.init_new(
layer_id=layer_id,
# A NextN draft is a one-layer model.
num_layers=1 if is_nextn else config.num_hidden_layers,
@@ -828,7 +828,7 @@ class BailingMoELinearDecoderLayer(nn.Module):
else None
)
self.layer_communicator = LayerCommunicator(
layer_scatter_modes=self.layer_scatter_modes,
layer_facts=self.layer_facts,
input_layernorm=self.input_layernorm,
post_attention_layernorm=self.post_attention_layernorm,
allow_reduce_scatter=False,
+3 -3
View File
@@ -30,7 +30,7 @@ from sglang.srt.layers import deep_gemm_wrapper
from sglang.srt.layers.activation import SiluAndMul
from sglang.srt.layers.communicator import (
LayerCommunicator,
LayerScatterModes,
LayerFacts,
enable_moe_dense_fully_dp,
layer_input_buffer,
reduce_output,
@@ -1148,7 +1148,7 @@ class BailingMoELinearDecoderLayer(nn.Module):
self.input_layernorm = RMSNorm(self.hidden_size, eps=rms_norm_eps)
self.post_attention_layernorm = RMSNorm(self.hidden_size, eps=rms_norm_eps)
self.layer_scatter_modes = LayerScatterModes.init_new(
self.layer_facts = LayerFacts.init_new(
layer_id=layer_id,
# NextN wraps a single decoder layer whose checkpoint prefix is the
# post-model layer id. Treat it as a one-layer model for scatter-mode
@@ -1160,7 +1160,7 @@ class BailingMoELinearDecoderLayer(nn.Module):
)
self.layer_communicator = LayerCommunicator(
layer_scatter_modes=self.layer_scatter_modes,
layer_facts=self.layer_facts,
input_layernorm=self.input_layernorm,
post_attention_layernorm=self.post_attention_layernorm,
allow_reduce_scatter=True,
+3 -3
View File
@@ -69,7 +69,7 @@ from sglang.srt.layers.aux_hidden_states import (
)
from sglang.srt.layers.communicator import (
LayerCommunicator,
LayerScatterModes,
LayerFacts,
enable_moe_dense_fully_dp,
get_attn_tp_context,
layer_input_buffer,
@@ -2576,7 +2576,7 @@ class DeepseekV2DecoderLayer(nn.Module):
is_previous_layer_sparse = self._is_layer_sparse(layer_id - 1, is_nextn=False)
is_next_layer_sparse = self._is_layer_sparse(layer_id + 1, is_nextn=False)
self.layer_scatter_modes = LayerScatterModes.init_new(
self.layer_facts = LayerFacts.init_new(
layer_id=layer_id,
num_layers=1 if is_nextn else config.num_hidden_layers,
is_layer_sparse=self.is_layer_sparse,
@@ -2643,7 +2643,7 @@ class DeepseekV2DecoderLayer(nn.Module):
fusions = CuteDSLFusion()
return LayerCommunicator(
layer_scatter_modes=self.layer_scatter_modes,
layer_facts=self.layer_facts,
input_layernorm=input_layernorm,
post_attention_layernorm=post_attention_layernorm,
allow_reduce_scatter=True,
@@ -59,7 +59,7 @@ from sglang.srt.layers.activation import SiluAndMul
from sglang.srt.layers.attention.dsa.dsa_indexer import Indexer
from sglang.srt.layers.communicator import (
LayerCommunicator,
LayerScatterModes,
LayerFacts,
enable_moe_dense_fully_dp,
)
from sglang.srt.layers.dp_attention import is_dp_attention_enabled
@@ -1541,7 +1541,7 @@ class Dots3DecoderLayer(nn.Module):
is_previous_layer_sparse = self._is_layer_sparse(layer_id - 1, is_nextn=False)
is_next_layer_sparse = self._is_layer_sparse(layer_id + 1, is_nextn=False)
self.layer_scatter_modes = LayerScatterModes.init_new(
self.layer_facts = LayerFacts.init_new(
layer_id=layer_id,
num_layers=1 if is_nextn else config.num_hidden_layers,
is_layer_sparse=self.is_layer_sparse,
@@ -1579,7 +1579,7 @@ class Dots3DecoderLayer(nn.Module):
)
self.layer_communicator = LayerCommunicator(
layer_scatter_modes=self.layer_scatter_modes,
layer_facts=self.layer_facts,
input_layernorm=self.input_layernorm,
post_attention_layernorm=self.post_attention_layernorm,
allow_reduce_scatter=True,
+3 -3
View File
@@ -11,7 +11,7 @@ from sglang.srt.layers.attention.hybrid_linear_attn_backend import (
Mamba2AttnBackend,
)
from sglang.srt.layers.attention.mamba.mamba import MambaMixer2
from sglang.srt.layers.communicator import LayerCommunicator, LayerScatterModes
from sglang.srt.layers.communicator import LayerCommunicator, LayerFacts
from sglang.srt.layers.dp_attention import (
is_dp_attention_enabled,
)
@@ -198,7 +198,7 @@ class FalconH1HybridAttentionDecoderLayer(nn.Module):
is_previous_layer_sparse = False
is_next_layer_sparse = False
self.layer_scatter_modes = LayerScatterModes.init_new(
self.layer_facts = LayerFacts.init_new(
layer_id=layer_id,
num_layers=config.num_hidden_layers,
is_layer_sparse=self.is_layer_sparse,
@@ -223,7 +223,7 @@ class FalconH1HybridAttentionDecoderLayer(nn.Module):
self.k_norm = RMSNorm(self.head_dim, eps=config.rms_norm_eps)
self.layer_communicator = LayerCommunicator(
layer_scatter_modes=self.layer_scatter_modes,
layer_facts=self.layer_facts,
input_layernorm=self.input_layernorm,
post_attention_layernorm=self.pre_ff_layernorm,
allow_reduce_scatter=True,
+3 -3
View File
@@ -40,7 +40,7 @@ from sglang.srt.eplb.expert_location_dispatch import ExpertLocationDispatchInfo
from sglang.srt.layers.activation import SiluAndMul
from sglang.srt.layers.communicator import (
LayerCommunicator,
LayerScatterModes,
LayerFacts,
enable_moe_dense_fully_dp,
reduce_output,
)
@@ -836,7 +836,7 @@ class Glm4MoeDecoderLayer(nn.Module):
is_previous_layer_sparse = self._is_layer_sparse(layer_id - 1, is_nextn=False)
is_next_layer_sparse = self._is_layer_sparse(layer_id + 1, is_nextn=False)
self.layer_scatter_modes = LayerScatterModes.init_new(
self.layer_facts = LayerFacts.init_new(
layer_id=layer_id,
num_layers=1 if is_nextn else config.num_hidden_layers,
is_layer_sparse=self.is_layer_sparse,
@@ -873,7 +873,7 @@ class Glm4MoeDecoderLayer(nn.Module):
)
self.layer_communicator = LayerCommunicator(
layer_scatter_modes=self.layer_scatter_modes,
layer_facts=self.layer_facts,
input_layernorm=self.input_layernorm,
post_attention_layernorm=self.post_attention_layernorm,
allow_reduce_scatter=True,
+3 -3
View File
@@ -34,7 +34,7 @@ from sglang.srt.eplb.expert_location_dispatch import ExpertLocationDispatchInfo
from sglang.srt.layers.activation import SiluAndMul
from sglang.srt.layers.communicator import (
LayerCommunicator,
LayerScatterModes,
LayerFacts,
enable_moe_dense_fully_dp,
get_attn_tp_context,
reduce_output,
@@ -548,7 +548,7 @@ class Glm4MoeLiteDecoderLayer(nn.Module):
is_previous_layer_sparse = self._is_layer_sparse(layer_id - 1, is_nextn=False)
is_next_layer_sparse = self._is_layer_sparse(layer_id + 1, is_nextn=False)
self.layer_scatter_modes = LayerScatterModes.init_new(
self.layer_facts = LayerFacts.init_new(
layer_id=layer_id,
num_layers=1 if is_nextn else config.num_hidden_layers,
is_layer_sparse=self.is_layer_sparse,
@@ -588,7 +588,7 @@ class Glm4MoeLiteDecoderLayer(nn.Module):
self._gfx95_quant_format = self._detect_gfx95_quant_format()
self.layer_communicator = LayerCommunicator(
layer_scatter_modes=self.layer_scatter_modes,
layer_facts=self.layer_facts,
input_layernorm=self.input_layernorm,
post_attention_layernorm=self.post_attention_layernorm,
allow_reduce_scatter=True,
+3 -3
View File
@@ -27,7 +27,7 @@ from sglang.srt.layers.attention import vision_utils
from sglang.srt.layers.attention.vision import VisionAttention
from sglang.srt.layers.communicator import (
LayerCommunicator,
LayerScatterModes,
LayerFacts,
MHCLayerCommunicator,
enable_moe_dense_fully_dp,
get_attn_tp_context,
@@ -726,7 +726,7 @@ class Glm5NextDecoderLayer(nn.Module):
is_previous_layer_sparse = self._is_layer_sparse(layer_id - 1, is_nextn=False)
is_next_layer_sparse = self._is_layer_sparse(layer_id + 1, is_nextn=False)
self.layer_scatter_modes = LayerScatterModes.init_new(
self.layer_facts = LayerFacts.init_new(
layer_id=layer_id,
num_layers=1 if is_nextn else config.num_hidden_layers,
is_layer_sparse=self.is_layer_sparse,
@@ -784,7 +784,7 @@ class Glm5NextDecoderLayer(nn.Module):
)
shared_kwargs: Dict[str, Any] = dict(
layer_scatter_modes=self.layer_scatter_modes,
layer_facts=self.layer_facts,
input_layernorm=self.input_layernorm,
post_attention_layernorm=self.post_attention_layernorm,
allow_reduce_scatter=True,
+3 -3
View File
@@ -31,7 +31,7 @@ from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_r
from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation
from sglang.srt.layers.communicator import (
LayerCommunicator,
LayerScatterModes,
LayerFacts,
reduce_output,
)
from sglang.srt.layers.dp_attention import (
@@ -569,7 +569,7 @@ class GptOssDecoderLayer(nn.Module):
is_previous_layer_sparse = True
is_next_layer_sparse = True
self.layer_scatter_modes = LayerScatterModes.init_new(
self.layer_facts = LayerFacts.init_new(
layer_id=layer_id,
num_layers=config.num_hidden_layers,
is_layer_sparse=self.is_layer_sparse,
@@ -606,7 +606,7 @@ class GptOssDecoderLayer(nn.Module):
)
self.layer_communicator = LayerCommunicator(
layer_scatter_modes=self.layer_scatter_modes,
layer_facts=self.layer_facts,
input_layernorm=self.input_layernorm,
post_attention_layernorm=self.post_attention_layernorm,
allow_reduce_scatter=True,
+5 -5
View File
@@ -10,7 +10,7 @@ from sglang.srt.configs.interns2_mobius import (
InternS2MobiusConfig,
InternS2MobiusTextConfig,
)
from sglang.srt.layers.communicator import LayerCommunicator, LayerScatterModes
from sglang.srt.layers.communicator import LayerCommunicator, LayerFacts
from sglang.srt.layers.dp_attention import is_dp_attention_enabled
from sglang.srt.layers.layernorm import GemmaRMSNorm
from sglang.srt.layers.linear import (
@@ -476,7 +476,7 @@ class InternS2MobiusLinearDecoderLayer(_InternS2MobiusDecoderMixin, nn.Module):
layer_prefix = prefix.removesuffix(".linear_attn")
self._init_mobius_mlp(config, quant_config, layer_prefix)
self.layer_scatter_modes = LayerScatterModes.init_new(
self.layer_facts = LayerFacts.init_new(
layer_id=layer_id,
num_layers=config.num_hidden_layers,
is_layer_sparse=True,
@@ -492,7 +492,7 @@ class InternS2MobiusLinearDecoderLayer(_InternS2MobiusDecoderMixin, nn.Module):
and _linear_accepts_fp8_tuple(self.linear_attn.in_proj_qkvz)
)
self.layer_communicator = LayerCommunicator(
layer_scatter_modes=self.layer_scatter_modes,
layer_facts=self.layer_facts,
input_layernorm=self.input_layernorm,
post_attention_layernorm=self.post_attention_layernorm,
allow_reduce_scatter=True,
@@ -609,7 +609,7 @@ class InternS2MobiusAttentionDecoderLayer(
layer_prefix = prefix.removesuffix(".self_attn")
self._init_mobius_mlp(config, quant_config, layer_prefix)
self.layer_scatter_modes = LayerScatterModes.init_new(
self.layer_facts = LayerFacts.init_new(
layer_id=layer_id,
num_layers=config.num_hidden_layers,
is_layer_sparse=True,
@@ -626,7 +626,7 @@ class InternS2MobiusAttentionDecoderLayer(
_enable_qwen35_fused_ar_quant() and _linear_accepts_fp8_tuple(self.qkv_proj)
)
self.layer_communicator = LayerCommunicator(
layer_scatter_modes=self.layer_scatter_modes,
layer_facts=self.layer_facts,
input_layernorm=self.input_layernorm,
post_attention_layernorm=self.post_attention_layernorm,
allow_reduce_scatter=True,
+3 -3
View File
@@ -21,7 +21,7 @@ from sglang.srt.environ import envs
from sglang.srt.layers.activation import SiluAndMul
from sglang.srt.layers.communicator import (
LayerCommunicator,
LayerScatterModes,
LayerFacts,
reduce_output,
)
from sglang.srt.layers.dp_attention import (
@@ -455,7 +455,7 @@ class LagunaDecoderLayer(nn.Module):
config.hidden_size, eps=config.rms_norm_eps
)
self.layer_scatter_modes = LayerScatterModes.init_new(
self.layer_facts = LayerFacts.init_new(
layer_id=layer_id,
num_layers=config.num_hidden_layers,
is_layer_sparse=self.is_layer_sparse,
@@ -463,7 +463,7 @@ class LagunaDecoderLayer(nn.Module):
is_next_layer_sparse=is_next_layer_sparse,
)
self.layer_communicator = LayerCommunicator(
layer_scatter_modes=self.layer_scatter_modes,
layer_facts=self.layer_facts,
input_layernorm=self.input_layernorm,
post_attention_layernorm=self.post_attention_layernorm,
allow_reduce_scatter=True,
+3 -3
View File
@@ -33,7 +33,7 @@ from sglang.srt.eplb.expert_location_dispatch import ExpertLocationDispatchInfo
from sglang.srt.layers.activation import SiluAndMul
from sglang.srt.layers.communicator import (
LayerCommunicator,
LayerScatterModes,
LayerFacts,
enable_moe_dense_fully_dp,
)
from sglang.srt.layers.dp_attention import (
@@ -612,7 +612,7 @@ class LLaDA2MoeBlock(nn.Module):
is_previous_layer_sparse = self._is_layer_sparse(config, layer_id=layer_id - 1)
is_next_layer_sparse = self._is_layer_sparse(config, layer_id=layer_id + 1)
self.layer_scatter_modes = LayerScatterModes.init_new(
self.layer_facts = LayerFacts.init_new(
layer_id=layer_id,
num_layers=config.num_hidden_layers,
is_layer_sparse=self.is_layer_sparse,
@@ -645,7 +645,7 @@ class LLaDA2MoeBlock(nn.Module):
self.post_attention_layernorm = RMSNorm(hidden_size, eps=config.rms_norm_eps)
self.layer_communicator = LayerCommunicator(
layer_scatter_modes=self.layer_scatter_modes,
layer_facts=self.layer_facts,
input_layernorm=self.input_layernorm,
post_attention_layernorm=self.post_attention_layernorm,
allow_reduce_scatter=True,
+3 -3
View File
@@ -25,7 +25,7 @@ import torch
from torch import nn
from transformers import Llama4TextConfig
from sglang.srt.layers.communicator import LayerCommunicator, LayerScatterModes
from sglang.srt.layers.communicator import LayerCommunicator, LayerFacts
from sglang.srt.layers.dp_attention import (
is_dp_attention_enabled,
)
@@ -420,7 +420,7 @@ class Llama4DecoderLayer(nn.Module):
config.hidden_size, eps=config.rms_norm_eps
)
self.layer_scatter_modes = LayerScatterModes.init_new(
self.layer_facts = LayerFacts.init_new(
layer_id=layer_id,
num_layers=config.num_hidden_layers,
is_layer_sparse=is_moe_layer,
@@ -429,7 +429,7 @@ class Llama4DecoderLayer(nn.Module):
)
self.layer_communicator = LayerCommunicator(
layer_scatter_modes=self.layer_scatter_modes,
layer_facts=self.layer_facts,
input_layernorm=self.input_layernorm,
post_attention_layernorm=self.post_attention_layernorm,
allow_reduce_scatter=True,
+6 -6
View File
@@ -51,7 +51,7 @@ from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_r
from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation
from sglang.srt.layers import deep_gemm_wrapper
from sglang.srt.layers.activation import SiluAndMul
from sglang.srt.layers.communicator import LayerCommunicator, LayerScatterModes
from sglang.srt.layers.communicator import LayerCommunicator, LayerFacts
from sglang.srt.layers.dp_attention import (
is_dp_attention_enabled,
)
@@ -419,8 +419,8 @@ class LongcatFlashDecoderLayer(nn.Module):
self.attn_tp_size = get_parallel().attn_tp_size
self.attn_tp_rank = get_parallel().attn_tp_rank
self.mlp_layer_scatter_modes = [
LayerScatterModes.init_new(
self.mlp_layer_facts = [
LayerFacts.init_new(
layer_id=self.layer_id * 2 + i,
num_layers=config.num_hidden_layers,
is_layer_sparse=False,
@@ -432,7 +432,7 @@ class LongcatFlashDecoderLayer(nn.Module):
]
self.mlp_layer_communicator = [
LayerCommunicator(
layer_scatter_modes=self.mlp_layer_scatter_modes[i],
layer_facts=self.mlp_layer_facts[i],
input_layernorm=self.input_layernorm[i],
post_attention_layernorm=self.post_attention_layernorm[i],
qkv_latent_func=self.self_attn[i].prepare_qkv_latent,
@@ -441,7 +441,7 @@ class LongcatFlashDecoderLayer(nn.Module):
for i in range(2)
]
self.moe_layer_scatter_modes = LayerScatterModes.init_new(
self.moe_layer_facts = LayerFacts.init_new(
layer_id=self.layer_id,
num_layers=config.num_hidden_layers,
is_layer_sparse=True,
@@ -450,7 +450,7 @@ class LongcatFlashDecoderLayer(nn.Module):
is_next_layer_sparse=True,
)
self.moe_layer_communicator = LayerCommunicator(
layer_scatter_modes=self.moe_layer_scatter_modes,
layer_facts=self.moe_layer_facts,
input_layernorm=self.input_layernorm[0],
post_attention_layernorm=self.post_attention_layernorm[0],
qkv_latent_func=self.self_attn[0].prepare_qkv_latent,
@@ -41,7 +41,7 @@ from sglang.kernels.ops.quantization.fp8_kernel import is_fp8_fnuz
from sglang.srt.configs import LongcatFlashConfig
from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder
from sglang.srt.layers import deep_gemm_wrapper
from sglang.srt.layers.communicator import LayerCommunicator, LayerScatterModes
from sglang.srt.layers.communicator import LayerCommunicator, LayerFacts
from sglang.srt.layers.dp_attention import (
is_dp_attention_enabled,
)
@@ -159,7 +159,7 @@ class LongcatFlashDenseDecoderLayer(nn.Module):
self.attn_tp_size = get_parallel().attn_tp_size
self.attn_tp_rank = get_parallel().attn_tp_rank
self.layer_scatter_modes = LayerScatterModes.init_new(
self.layer_facts = LayerFacts.init_new(
layer_id=self.layer_id,
num_layers=config.num_hidden_layers,
is_layer_sparse=False,
@@ -167,7 +167,7 @@ class LongcatFlashDenseDecoderLayer(nn.Module):
is_next_layer_sparse=False,
)
self.layer_communicator = LayerCommunicator(
layer_scatter_modes=self.layer_scatter_modes,
layer_facts=self.layer_facts,
input_layernorm=self.input_layernorm,
post_attention_layernorm=self.post_attention_layernorm,
)
+3 -3
View File
@@ -25,7 +25,7 @@ import torch
from torch import nn
from transformers import PretrainedConfig
from sglang.srt.layers.communicator import LayerCommunicator, LayerScatterModes
from sglang.srt.layers.communicator import LayerCommunicator, LayerFacts
from sglang.srt.layers.layernorm import RMSNorm
from sglang.srt.layers.linear import QKVParallelLinear, RowParallelLinear
from sglang.srt.layers.quantization.base_config import QuantizationConfig
@@ -442,7 +442,7 @@ class MellumDecoderLayer(Qwen3MoeDecoderLayer):
is_previous_layer_sparse = _is_sparse(layer_id - 1)
is_next_layer_sparse = _is_sparse(layer_id + 1)
self.layer_scatter_modes = LayerScatterModes.init_new(
self.layer_facts = LayerFacts.init_new(
layer_id=layer_id,
num_layers=cfg.num_hidden_layers,
is_layer_sparse=self.is_layer_sparse,
@@ -454,7 +454,7 @@ class MellumDecoderLayer(Qwen3MoeDecoderLayer):
self.post_attention_layernorm = RMSNorm(cfg.hidden_size, eps=rms_norm_eps)
self.layer_communicator = LayerCommunicator(
layer_scatter_modes=self.layer_scatter_modes,
layer_facts=self.layer_facts,
input_layernorm=self.input_layernorm,
post_attention_layernorm=self.post_attention_layernorm,
allow_reduce_scatter=True,
+3 -3
View File
@@ -34,7 +34,7 @@ from sglang.srt.layers.aux_hidden_states import (
)
from sglang.srt.layers.communicator import (
LayerCommunicator,
LayerScatterModes,
LayerFacts,
enable_moe_dense_fully_dp,
)
from sglang.srt.layers.dp_attention import (
@@ -856,7 +856,7 @@ class MiMoV2DecoderLayer(nn.Module):
config.hidden_size, eps=config.layernorm_epsilon
)
self.layer_scatter_modes = LayerScatterModes.init_new(
self.layer_facts = LayerFacts.init_new(
layer_id=layer_id,
num_layers=config.num_hidden_layers,
is_layer_sparse=self.is_layer_sparse,
@@ -864,7 +864,7 @@ class MiMoV2DecoderLayer(nn.Module):
is_next_layer_sparse=is_next_layer_sparse,
)
self.layer_communicator = LayerCommunicator(
layer_scatter_modes=self.layer_scatter_modes,
layer_facts=self.layer_facts,
input_layernorm=self.input_layernorm,
post_attention_layernorm=self.post_attention_layernorm,
allow_reduce_scatter=True,
+3 -3
View File
@@ -23,7 +23,7 @@ from sglang.srt.configs.model_config import get_mimo_v2_fused_qkv_expected_tp_si
from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder
from sglang.srt.layers.communicator import (
LayerCommunicator,
LayerScatterModes,
LayerFacts,
enable_moe_dense_fully_dp,
)
from sglang.srt.layers.dp_attention import (
@@ -116,7 +116,7 @@ class MiMoV2MTPLayer(nn.Module):
self.post_attention_layernorm = RMSNorm(
config.hidden_size, eps=config.layernorm_epsilon
)
self.layer_scatter_modes = LayerScatterModes.init_new(
self.layer_facts = LayerFacts.init_new(
layer_id=layer_id,
num_layers=1,
is_layer_sparse=self.is_layer_sparse,
@@ -124,7 +124,7 @@ class MiMoV2MTPLayer(nn.Module):
is_next_layer_sparse=is_next_layer_sparse,
)
self.layer_communicator = LayerCommunicator(
layer_scatter_modes=self.layer_scatter_modes,
layer_facts=self.layer_facts,
input_layernorm=self.input_layernorm,
post_attention_layernorm=self.post_attention_layernorm,
allow_deferred_ffn_reduction=False,
+3 -3
View File
@@ -32,7 +32,7 @@ from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_r
from sglang.srt.eplb.expert_location_dispatch import ExpertLocationDispatchInfo
from sglang.srt.layers.communicator import (
LayerCommunicator,
LayerScatterModes,
LayerFacts,
)
from sglang.srt.layers.dp_attention import (
attn_tp_all_reduce,
@@ -975,7 +975,7 @@ class MiniMaxM2DecoderLayer(nn.Module):
is_previous_layer_sparse = True
is_next_layer_sparse = True
self.layer_scatter_modes = LayerScatterModes.init_new(
self.layer_facts = LayerFacts.init_new(
layer_id=layer_id,
num_layers=config.num_hidden_layers,
is_layer_sparse=self.is_layer_sparse,
@@ -984,7 +984,7 @@ class MiniMaxM2DecoderLayer(nn.Module):
)
self.layer_communicator = LayerCommunicator(
layer_scatter_modes=self.layer_scatter_modes,
layer_facts=self.layer_facts,
input_layernorm=self.input_layernorm,
post_attention_layernorm=self.post_attention_layernorm,
allow_reduce_scatter=True,
+4 -4
View File
@@ -37,7 +37,7 @@ from sglang.srt.eplb.expert_location_dispatch import ExpertLocationDispatchInfo
from sglang.srt.layers.activation import SiluAndMul
from sglang.srt.layers.communicator import (
LayerCommunicator,
LayerScatterModes,
LayerFacts,
enable_moe_dense_fully_dp,
)
from sglang.srt.layers.dp_attention import (
@@ -1297,7 +1297,7 @@ class MiniMaxM3DecoderLayer(nn.Module):
moe_layer_freq = getattr(config, "moe_layer_freq", None)
# Means "MLP is a sparse MoE", not attention sparsity. Kept as ``is_layer_sparse``
# because LayerCommunicator / LayerScatterModes / other models read this attr.
# because LayerCommunicator / LayerFacts / other models read this attr.
self.is_layer_sparse = (
moe_layer_freq[layer_id] != 0 if moe_layer_freq is not None else True
)
@@ -1347,7 +1347,7 @@ class MiniMaxM3DecoderLayer(nn.Module):
is_previous_layer_sparse = _is_layer_sparse(layer_id - 1)
is_next_layer_sparse = _is_layer_sparse(layer_id + 1)
self.layer_scatter_modes = LayerScatterModes.init_new(
self.layer_facts = LayerFacts.init_new(
layer_id=layer_id,
num_layers=config.num_hidden_layers,
is_layer_sparse=self.is_layer_sparse,
@@ -1356,7 +1356,7 @@ class MiniMaxM3DecoderLayer(nn.Module):
)
self.layer_communicator = LayerCommunicator(
layer_scatter_modes=self.layer_scatter_modes,
layer_facts=self.layer_facts,
input_layernorm=self.input_layernorm,
post_attention_layernorm=self.post_attention_layernorm,
allow_reduce_scatter=True,
+3 -3
View File
@@ -22,7 +22,7 @@ from sglang.srt.layers.attention.vision import (
VisionAttentionMetadata,
prepare_vision_attention_metadata,
)
from sglang.srt.layers.communicator import LayerCommunicator, LayerScatterModes
from sglang.srt.layers.communicator import LayerCommunicator, LayerFacts
from sglang.srt.layers.conv import Conv3dLayer
from sglang.srt.layers.layernorm import RMSNorm
from sglang.srt.layers.linear import (
@@ -1011,7 +1011,7 @@ class MossVLSelfAttentionDecoderLayer(nn.Module):
self.post_attention_layernorm = RMSNorm(
config.hidden_size, eps=config.rms_norm_eps, **norm_kwargs
)
self.layer_scatter_modes = LayerScatterModes.init_new(
self.layer_facts = LayerFacts.init_new(
layer_id=layer_id,
num_layers=config.num_hidden_layers,
is_layer_sparse=False,
@@ -1019,7 +1019,7 @@ class MossVLSelfAttentionDecoderLayer(nn.Module):
is_next_layer_sparse=False,
)
self.layer_communicator = LayerCommunicator(
layer_scatter_modes=self.layer_scatter_modes,
layer_facts=self.layer_facts,
input_layernorm=self.input_layernorm,
post_attention_layernorm=self.post_attention_layernorm,
allow_deferred_ffn_reduction=False,
+4 -23
View File
@@ -6,10 +6,9 @@ from sglang.srt.configs.nemotron_h import ATTENTION, MAMBA, MOE
from sglang.srt.layers.communicator import (
NORM_QUANT_READ,
LayerCommunicator,
LayerScatterModes,
LayerFacts,
LayerStage,
Layout,
ScatterMode,
StageDecl,
StageInput,
StageKind,
@@ -102,25 +101,6 @@ def layer_stage(pattern: str, layer_idx: int) -> LayerStage:
)
def _build_layer_scatter_modes(
is_sparse: bool = False, is_last_layer: bool = False
) -> LayerScatterModes:
scatter_mlp = is_sparse and not get_moe_a2a_backend().is_none()
mlp_mode = ScatterMode.SCATTERED if scatter_mlp else ScatterMode.FULL
middle_residual_mode = (
ScatterMode.SCATTERED if scatter_mlp else ScatterMode.TP_ATTN_FULL
)
return LayerScatterModes(
layer_input_mode=ScatterMode.TP_ATTN_FULL,
attn_mode=ScatterMode.TP_ATTN_FULL,
mlp_mode=mlp_mode,
middle_residual_mode=middle_residual_mode,
layer_output_mode=ScatterMode.TP_ATTN_FULL,
is_layer_sparse=is_sparse,
is_last_layer=is_last_layer,
)
def make_layer_communicator(
layer_norm: RMSNorm, *, pattern: str, layer_idx: int
) -> LayerCommunicator:
@@ -129,8 +109,9 @@ def make_layer_communicator(
stage = layer_stage(pattern, layer_idx)
for_attn = stage.kind is StageKind.ATTENTION
return LayerCommunicator(
layer_scatter_modes=_build_layer_scatter_modes(
pattern[layer_idx] == MOE, is_last_layer=layer_idx == len(pattern) - 1
layer_facts=LayerFacts(
is_layer_sparse=pattern[layer_idx] == MOE,
is_last_layer=layer_idx == len(pattern) - 1,
),
input_layernorm=layer_norm if for_attn else None,
post_attention_layernorm=None if for_attn else layer_norm,
+3 -3
View File
@@ -42,7 +42,7 @@ from sglang.srt.eplb.expert_location_dispatch import ExpertLocationDispatchInfo
from sglang.srt.layers.activation import SiluAndMul
from sglang.srt.layers.communicator import (
LayerCommunicator,
LayerScatterModes,
LayerFacts,
)
from sglang.srt.layers.dp_attention import (
is_dp_attention_enabled,
@@ -1025,7 +1025,7 @@ class Qwen2MoeDecoderLayer(nn.Module):
is_previous_layer_sparse = True
is_next_layer_sparse = True
self.layer_scatter_modes = LayerScatterModes.init_new(
self.layer_facts = LayerFacts.init_new(
layer_id=layer_id,
num_layers=config.num_hidden_layers,
is_layer_sparse=self.is_layer_sparse,
@@ -1054,7 +1054,7 @@ class Qwen2MoeDecoderLayer(nn.Module):
config.hidden_size, eps=config.rms_norm_eps
)
self.layer_communicator = LayerCommunicator(
layer_scatter_modes=self.layer_scatter_modes,
layer_facts=self.layer_facts,
input_layernorm=self.input_layernorm,
post_attention_layernorm=self.post_attention_layernorm,
allow_reduce_scatter=True,
+3 -3
View File
@@ -5,7 +5,7 @@ from typing import Any, Dict, Iterable, List, Optional, Tuple
import torch
from torch import nn
from sglang.srt.layers.communicator import LayerCommunicator, LayerScatterModes
from sglang.srt.layers.communicator import LayerCommunicator, LayerFacts
from sglang.srt.layers.layernorm import RMSNorm
from sglang.srt.layers.linear import QKVParallelLinear, RowParallelLinear
from sglang.srt.layers.logits_processor import LogitsProcessor
@@ -370,7 +370,7 @@ class Qwen3DecoderLayer(nn.Module):
config.hidden_size, eps=config.rms_norm_eps, **norm_kwargs
)
self.layer_scatter_modes = LayerScatterModes.init_new(
self.layer_facts = LayerFacts.init_new(
layer_id=layer_id,
num_layers=config.num_hidden_layers,
is_layer_sparse=False,
@@ -378,7 +378,7 @@ class Qwen3DecoderLayer(nn.Module):
is_next_layer_sparse=False,
)
self.layer_communicator = LayerCommunicator(
layer_scatter_modes=self.layer_scatter_modes,
layer_facts=self.layer_facts,
input_layernorm=self.input_layernorm,
post_attention_layernorm=self.post_attention_layernorm,
allow_deferred_ffn_reduction=False,
+5 -5
View File
@@ -45,7 +45,7 @@ from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation
from sglang.srt.layers.attention.mamba.mamba import mamba_v2_sharded_weight_loader
from sglang.srt.layers.communicator import (
LayerCommunicator,
LayerScatterModes,
LayerFacts,
reduce_output,
)
from sglang.srt.layers.dp_attention import (
@@ -1104,7 +1104,7 @@ class Qwen3_5LinearDecoderLayer(nn.Module):
else:
raise ValueError(f"Invalid model type: {config.model_type}")
self.layer_scatter_modes = LayerScatterModes.init_new(
self.layer_facts = LayerFacts.init_new(
layer_id=layer_id,
num_layers=config.num_hidden_layers,
is_layer_sparse=is_layer_sparse,
@@ -1124,7 +1124,7 @@ class Qwen3_5LinearDecoderLayer(nn.Module):
and _linear_accepts_fp8_tuple(self.linear_attn.in_proj_qkvz)
)
self.layer_communicator = LayerCommunicator(
layer_scatter_modes=self.layer_scatter_modes,
layer_facts=self.layer_facts,
input_layernorm=self.input_layernorm,
post_attention_layernorm=self.post_attention_layernorm,
allow_reduce_scatter=True,
@@ -1306,7 +1306,7 @@ class Qwen3_5AttentionDecoderLayer(nn.Module):
else:
raise ValueError(f"Invalid model type: {config.model_type}")
self.layer_scatter_modes = LayerScatterModes.init_new(
self.layer_facts = LayerFacts.init_new(
layer_id=layer_id,
num_layers=config.num_hidden_layers,
is_layer_sparse=is_layer_sparse,
@@ -1328,7 +1328,7 @@ class Qwen3_5AttentionDecoderLayer(nn.Module):
_enable_qwen35_fused_ar_quant() and _linear_accepts_fp8_tuple(self.qkv_proj)
)
self.layer_communicator = LayerCommunicator(
layer_scatter_modes=self.layer_scatter_modes,
layer_facts=self.layer_facts,
input_layernorm=self.input_layernorm,
post_attention_layernorm=self.post_attention_layernorm,
allow_reduce_scatter=True,
+3 -3
View File
@@ -29,7 +29,7 @@ from transformers import PretrainedConfig
from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder
from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation
from sglang.srt.eplb.expert_location_dispatch import ExpertLocationDispatchInfo
from sglang.srt.layers.communicator import LayerCommunicator, LayerScatterModes
from sglang.srt.layers.communicator import LayerCommunicator, LayerFacts
from sglang.srt.layers.layernorm import RMSNorm
from sglang.srt.layers.linear import (
QKVParallelLinear,
@@ -840,7 +840,7 @@ class Qwen3MoeDecoderLayer(nn.Module):
is_previous_layer_sparse = True
is_next_layer_sparse = True
self.layer_scatter_modes = LayerScatterModes.init_new(
self.layer_facts = LayerFacts.init_new(
layer_id=layer_id,
num_layers=config.num_hidden_layers,
is_layer_sparse=self.is_layer_sparse,
@@ -869,7 +869,7 @@ class Qwen3MoeDecoderLayer(nn.Module):
)
self.layer_communicator = LayerCommunicator(
layer_scatter_modes=self.layer_scatter_modes,
layer_facts=self.layer_facts,
input_layernorm=self.input_layernorm,
post_attention_layernorm=self.post_attention_layernorm,
allow_reduce_scatter=True,
+5 -5
View File
@@ -15,7 +15,7 @@ from sglang.srt.configs.qwen3_next import Qwen3NextConfig
from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder
from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation
from sglang.srt.layers.attention.mamba.mamba import mamba_v2_sharded_weight_loader
from sglang.srt.layers.communicator import LayerCommunicator, LayerScatterModes
from sglang.srt.layers.communicator import LayerCommunicator, LayerFacts
from sglang.srt.layers.dp_attention import (
is_dp_attention_enabled,
)
@@ -507,7 +507,7 @@ class Qwen3HybridLinearDecoderLayer(nn.Module):
is_next_layer_sparse = True
self.layer_id = layer_id
self.layer_scatter_modes = LayerScatterModes.init_new(
self.layer_facts = LayerFacts.init_new(
layer_id=layer_id,
num_layers=config.num_hidden_layers,
is_layer_sparse=self.is_layer_sparse,
@@ -539,7 +539,7 @@ class Qwen3HybridLinearDecoderLayer(nn.Module):
config.hidden_size, eps=config.rms_norm_eps
)
self.layer_communicator = LayerCommunicator(
layer_scatter_modes=self.layer_scatter_modes,
layer_facts=self.layer_facts,
input_layernorm=self.input_layernorm,
post_attention_layernorm=self.post_attention_layernorm,
allow_reduce_scatter=True,
@@ -675,7 +675,7 @@ class Qwen3HybridAttentionDecoderLayer(nn.Module):
is_previous_layer_sparse = True
is_next_layer_sparse = True
self.layer_scatter_modes = LayerScatterModes.init_new(
self.layer_facts = LayerFacts.init_new(
layer_id=layer_id,
num_layers=config.num_hidden_layers,
is_layer_sparse=self.is_layer_sparse,
@@ -711,7 +711,7 @@ class Qwen3HybridAttentionDecoderLayer(nn.Module):
self.k_norm = GemmaRMSNorm(self.head_dim, eps=config.rms_norm_eps)
self.layer_communicator = LayerCommunicator(
layer_scatter_modes=self.layer_scatter_modes,
layer_facts=self.layer_facts,
input_layernorm=self.input_layernorm,
post_attention_layernorm=self.post_attention_layernorm,
allow_reduce_scatter=True,
+3 -3
View File
@@ -21,7 +21,7 @@ from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation
from sglang.srt.layers.activation import SiluAndMul
from sglang.srt.layers.communicator import (
LayerCommunicator,
LayerScatterModes,
LayerFacts,
enable_moe_dense_fully_dp,
)
from sglang.srt.layers.dp_attention import (
@@ -1047,7 +1047,7 @@ class SarvamMoEMLADecoderLayer(nn.Module):
)
self.attn_tp_size = get_parallel().attn_tp_size
self.layer_scatter_modes = LayerScatterModes.init_new(
self.layer_facts = LayerFacts.init_new(
layer_id=layer_id,
num_layers=config.num_hidden_layers,
is_layer_sparse=self.is_layer_sparse,
@@ -1055,7 +1055,7 @@ class SarvamMoEMLADecoderLayer(nn.Module):
is_next_layer_sparse=is_next_layer_sparse,
)
self.layer_communicator = LayerCommunicator(
layer_scatter_modes=self.layer_scatter_modes,
layer_facts=self.layer_facts,
input_layernorm=self.input_layernorm,
post_attention_layernorm=self.post_attention_layernorm,
qkv_latent_func=self.self_attn.prepare_qkv_latent,
+3 -3
View File
@@ -11,7 +11,7 @@ from torch import nn
from transformers import PretrainedConfig
from sglang.srt.layers.activation import SiluAndMul
from sglang.srt.layers.communicator import LayerCommunicator, LayerScatterModes
from sglang.srt.layers.communicator import LayerCommunicator, LayerFacts
from sglang.srt.layers.dp_attention import (
is_dp_attention_enabled,
)
@@ -290,7 +290,7 @@ class SDARBlock(nn.Module):
prefix=add_prefix("mlp", prefix),
)
self.layer_scatter_modes = LayerScatterModes.init_new(
self.layer_facts = LayerFacts.init_new(
layer_id=layer_id,
num_layers=config.num_hidden_layers,
is_layer_sparse=False,
@@ -298,7 +298,7 @@ class SDARBlock(nn.Module):
is_next_layer_sparse=False,
)
self.layer_communicator = LayerCommunicator(
layer_scatter_modes=self.layer_scatter_modes,
layer_facts=self.layer_facts,
input_layernorm=self.input_layernorm,
post_attention_layernorm=self.post_attention_layernorm,
allow_reduce_scatter=True,
+3 -3
View File
@@ -15,7 +15,7 @@ from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation
from sglang.srt.eplb.expert_location_dispatch import ExpertLocationDispatchInfo
from sglang.srt.layers.communicator import (
LayerCommunicator,
LayerScatterModes,
LayerFacts,
)
from sglang.srt.layers.dp_attention import (
is_dp_attention_enabled,
@@ -355,7 +355,7 @@ class SDARMoeBlock(nn.Module):
prefix=add_prefix("mlp", prefix),
)
self.layer_scatter_modes = LayerScatterModes.init_new(
self.layer_facts = LayerFacts.init_new(
layer_id=layer_id,
num_layers=config.num_hidden_layers,
is_layer_sparse=True,
@@ -364,7 +364,7 @@ class SDARMoeBlock(nn.Module):
)
self.layer_communicator = LayerCommunicator(
layer_scatter_modes=self.layer_scatter_modes,
layer_facts=self.layer_facts,
input_layernorm=self.input_layernorm,
post_attention_layernorm=self.post_attention_layernorm,
allow_reduce_scatter=True,
+3 -3
View File
@@ -22,7 +22,7 @@ from sglang.srt.distributed import (
from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation
from sglang.srt.layers.activation import SiluAndMul
from sglang.srt.layers.attention.vision import VisionAttention
from sglang.srt.layers.communicator import LayerCommunicator, LayerScatterModes
from sglang.srt.layers.communicator import LayerCommunicator, LayerFacts
from sglang.srt.layers.conv import Conv2dLayer
from sglang.srt.layers.dp_attention import (
is_dp_attention_enabled,
@@ -338,7 +338,7 @@ class Step3TextDecoderLayer(nn.Module):
)
self.is_next_layer_sparse = True if layer_id + 1 in moe_layers_idx else False
self.layer_scatter_modes = LayerScatterModes.init_new(
self.layer_facts = LayerFacts.init_new(
layer_id=layer_id,
num_layers=config.num_hidden_layers,
is_layer_sparse=self.is_layer_sparse,
@@ -379,7 +379,7 @@ class Step3TextDecoderLayer(nn.Module):
)
self.layer_communicator = LayerCommunicator(
layer_scatter_modes=self.layer_scatter_modes,
layer_facts=self.layer_facts,
input_layernorm=self.input_layernorm,
post_attention_layernorm=self.post_attention_layernorm,
allow_deferred_ffn_reduction=False,
+3 -3
View File
@@ -11,7 +11,7 @@ from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_r
from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation
from sglang.srt.eplb.expert_location_dispatch import ExpertLocationDispatchInfo
from sglang.srt.layers.activation import SiluAndMul
from sglang.srt.layers.communicator import LayerCommunicator, LayerScatterModes
from sglang.srt.layers.communicator import LayerCommunicator, LayerFacts
from sglang.srt.layers.dp_attention import (
is_dp_attention_enabled,
)
@@ -560,7 +560,7 @@ class Step3p5DecoderLayer(nn.Module):
)
# An MTP draft is a one-layer model; layer_id still indexes its config.
self.layer_scatter_modes = LayerScatterModes.init_new(
self.layer_facts = LayerFacts.init_new(
layer_id=0 if is_nextn else layer_id,
num_layers=1 if is_nextn else config.num_hidden_layers,
is_layer_sparse=self.is_moe_layer,
@@ -568,7 +568,7 @@ class Step3p5DecoderLayer(nn.Module):
is_next_layer_sparse=self.is_next_layer_sparse,
)
self.layer_communicator = LayerCommunicator(
layer_scatter_modes=self.layer_scatter_modes,
layer_facts=self.layer_facts,
input_layernorm=self.input_layernorm,
post_attention_layernorm=self.post_attention_layernorm,
allow_reduce_scatter=True,
+3 -3
View File
@@ -41,7 +41,7 @@ from sglang.srt.eplb.expert_location_dispatch import ExpertLocationDispatchInfo
from sglang.srt.layers.activation import SiluAndMul
from sglang.srt.layers.communicator import (
LayerCommunicator,
LayerScatterModes,
LayerFacts,
enable_moe_dense_fully_dp,
)
from sglang.srt.layers.dp_attention import is_dp_attention_enabled
@@ -1505,7 +1505,7 @@ class XllmDecoderLayer(nn.Module):
is_previous_layer_sparse = _is_sparse(layer_id - 1)
is_next_layer_sparse = _is_sparse(layer_id + 1)
self.layer_scatter_modes = LayerScatterModes.init_new(
self.layer_facts = LayerFacts.init_new(
layer_id=layer_id,
num_layers=config.num_hidden_layers,
is_layer_sparse=self.is_layer_sparse,
@@ -1538,7 +1538,7 @@ class XllmDecoderLayer(nn.Module):
self.input_layernorm = _make_norm(config)
self.post_attention_layernorm = _make_norm(config)
self.layer_communicator = LayerCommunicator(
layer_scatter_modes=self.layer_scatter_modes,
layer_facts=self.layer_facts,
input_layernorm=self.input_layernorm,
post_attention_layernorm=self.post_attention_layernorm,
allow_reduce_scatter=True,