From 81f27fb3a71d3504eab821a9d39352c924c730cf Mon Sep 17 00:00:00 2001 From: Cheng Wan <54331508+ch-wan@users.noreply.github.com> Date: Sun, 27 Sep 2026 16:56:11 -0700 Subject: [PATCH] [Refactor] Replace LayerScatterModes with LayerFacts and remove ScatterMode (#41443) --- .../srt/layers/communicator/__init__.py | 6 +- .../sglang/srt/layers/communicator/layer.py | 168 ++++-------------- .../sglang/srt/layers/communicator/layout.py | 40 ----- python/sglang/srt/models/bailing_moe.py | 6 +- .../sglang/srt/models/bailing_moe_linear.py | 6 +- python/sglang/srt/models/bailing_moe_v3.py | 6 +- python/sglang/srt/models/deepseek_v2.py | 6 +- .../srt/models/dots3_common/modeling.py | 6 +- python/sglang/srt/models/falcon_h1.py | 6 +- python/sglang/srt/models/glm4_moe.py | 6 +- python/sglang/srt/models/glm4_moe_lite.py | 6 +- python/sglang/srt/models/glm5_next.py | 6 +- python/sglang/srt/models/gpt_oss.py | 6 +- python/sglang/srt/models/interns2_mobius.py | 10 +- python/sglang/srt/models/laguna.py | 6 +- python/sglang/srt/models/llada2.py | 6 +- python/sglang/srt/models/llama4.py | 6 +- python/sglang/srt/models/longcat_flash.py | 12 +- .../sglang/srt/models/longcat_flash_nextn.py | 6 +- python/sglang/srt/models/mellum.py | 6 +- python/sglang/srt/models/mimo_v2.py | 6 +- python/sglang/srt/models/mimo_v2_nextn.py | 6 +- python/sglang/srt/models/minimax_m2.py | 6 +- python/sglang/srt/models/minimax_m3.py | 8 +- python/sglang/srt/models/moss_vl.py | 6 +- python/sglang/srt/models/nemotron_h_utils.py | 27 +-- python/sglang/srt/models/qwen2_moe.py | 6 +- python/sglang/srt/models/qwen3.py | 6 +- python/sglang/srt/models/qwen3_5.py | 10 +- python/sglang/srt/models/qwen3_moe.py | 6 +- python/sglang/srt/models/qwen3_next.py | 10 +- python/sglang/srt/models/sarvam_moe.py | 6 +- python/sglang/srt/models/sdar.py | 6 +- python/sglang/srt/models/sdar_moe.py | 6 +- python/sglang/srt/models/step3_vl.py | 6 +- python/sglang/srt/models/step3p5.py | 6 +- python/sglang/srt/models/xllm.py | 6 +- .../ops/test_aiter_allreduce_fusion_amd.py | 10 +- .../unit/layers/test_declared_attention_cp.py | 6 +- .../layers/test_declared_decoder_boundary.py | 127 +++++++------ .../layers/test_dense_mlp_under_prefill_cp.py | 56 ++++++ .../layers/test_dp_attention_cp_gather.py | 2 +- .../test_layer_communicator_fusion_gate.py | 28 +-- .../test_layer_scatter_modes_cp_dense_mlp.py | 35 ---- .../unit/layers/test_layernorm_sp.py | 2 +- .../test_postprocess_reduce_scatterv.py | 24 +-- .../unit/layers/test_prepare_attn_steps.py | 2 +- .../lora/test_triton_dp_attention_unit.py | 2 +- .../models/test_last_layer_communicator.py | 12 +- 49 files changed, 294 insertions(+), 471 deletions(-) create mode 100644 test/registered/unit/layers/test_dense_mlp_under_prefill_cp.py delete mode 100644 test/registered/unit/layers/test_layer_scatter_modes_cp_dense_mlp.py diff --git a/python/sglang/srt/layers/communicator/__init__.py b/python/sglang/srt/layers/communicator/__init__.py index 0458a9894e..97713e78cc 100644 --- a/python/sglang/srt/layers/communicator/__init__.py +++ b/python/sglang/srt/layers/communicator/__init__.py @@ -42,14 +42,13 @@ from sglang.srt.layers.communicator.layer import ( FfnExit, FfnExitFusion, LayerCommunicator, - LayerScatterModes, + LayerFacts, MHCLayerCommunicator, MixerExit, ) from sglang.srt.layers.communicator.layout import ( CommunicateContext, Layout, - ScatterMode, SumGroup, TokenAxis, enable_moe_dense_fully_dp, @@ -98,7 +97,7 @@ __all__ = [ "HandoffOutput", "LayerCommunicator", "LayerResidual", - "LayerScatterModes", + "LayerFacts", "LayerStage", "Layout", "MHCLayerCommunicator", @@ -107,7 +106,6 @@ __all__ = [ "NORM_QUANT_READ", "NORM_READ", "PLAIN_RESIDUAL", - "ScatterMode", "StageDecl", "StageEntry", "StageInput", diff --git a/python/sglang/srt/layers/communicator/layer.py b/python/sglang/srt/layers/communicator/layer.py index 0f5ce9e8c7..e2722e360f 100644 --- a/python/sglang/srt/layers/communicator/layer.py +++ b/python/sglang/srt/layers/communicator/layer.py @@ -24,7 +24,6 @@ import torch from sglang.srt.distributed import GroupCoordinator from sglang.srt.environ import envs from sglang.srt.layers import layernorm_sp -from sglang.srt.layers.attention.dsa.utils import is_dsa_enable_prefill_cp from sglang.srt.layers.aux_hidden_states import AuxHiddenStateAccumulator from sglang.srt.layers.communicator.adapters.attention import get_attn_tp_context from sglang.srt.layers.communicator.boundary import ( @@ -47,7 +46,6 @@ from sglang.srt.layers.communicator.boundary import ( from sglang.srt.layers.communicator.layout import ( CommunicateContext, Layout, - ScatterMode, SumGroup, TokenAxis, _batch_shards_over_cp, @@ -56,7 +54,6 @@ from sglang.srt.layers.communicator.layout import ( _gathers_over_attention_cp, _generic_prefill_cp_shards_tokens, _sum_group, - enable_dwdp, enable_moe_dense_fully_dp, sparse_moe_gathers_over_moe_cp, token_axis_sizes, @@ -83,7 +80,6 @@ from sglang.srt.layers.communicator.residual.add_norm import ( apply_flashinfer_allreduce_fusion, ) from sglang.srt.layers.communicator.residual.mhc import MHCState -from sglang.srt.layers.cp.utils import is_mla_cp_enabled from sglang.srt.layers.dp_attention import ( is_dp_attention_enabled, is_enable_moe_cp_allgather, @@ -94,7 +90,6 @@ from sglang.srt.layers.moe import ( is_moe_input_scattered_across_dp_ranks, post_experts_reduction_group, post_experts_sum_is_one_all_reduce, - should_use_flashinfer_cutlass_moe_fp4_allgather, ) from sglang.srt.model_executor.forward_batch_info import ForwardBatch from sglang.srt.runtime_context import ( @@ -113,131 +108,39 @@ _use_ag_after_qlora = envs.SGLANG_USE_AG_AFTER_QLORA.get() @dataclass -class _LayerModeComputationContext: - num_layers: int - layer_id: int - is_layer_sparse: bool - is_previous_layer_sparse: Optional[bool] - is_next_layer_sparse: Optional[bool] +class LayerFacts: + """Where a layer sits in the model and whether it and its neighbours have a + sparse MLP: the facts its boundaries are declared from.""" - def previous_layer(self): - assert self.is_previous_layer_sparse is not None - return _LayerModeComputationContext( - num_layers=self.num_layers, - layer_id=self.layer_id - 1, - is_layer_sparse=self.is_previous_layer_sparse, - is_previous_layer_sparse=None, - is_next_layer_sparse=self.is_layer_sparse, - ) - - -def sparse_mlp_scatter_mode() -> ScatterMode: - """SCATTERED hands a sparse MLP this rank's own token shard; FULL and - MOE_FULL hand it a buffer gathered over the attn-TP or MoE-CP group.""" - if ( - # Token dispatch/combine will be handled outside of LayerCommunicator for these modes. - not get_moe_a2a_backend().is_none() - or should_use_flashinfer_cutlass_moe_fp4_allgather() - or enable_dwdp() - ): - return ScatterMode.SCATTERED - # DSA CP and MLA CP both don't support MOE_FULL yet; fall back to FULL. - if is_enable_moe_cp_allgather() and not ( - is_dsa_enable_prefill_cp() or is_mla_cp_enabled() - ): - return ScatterMode.MOE_FULL - return ScatterMode.FULL - - -@dataclass -class LayerScatterModes: - layer_input_mode: ScatterMode - attn_mode: ScatterMode - # Can be further split into e.g. mlp_input_mode and mlp_output_mode if needed - mlp_mode: ScatterMode - middle_residual_mode: ScatterMode - layer_output_mode: ScatterMode is_layer_sparse: bool = False # The model's first layer: its input is the embedding, not a layer output. is_first_layer: bool = False # The model's last layer: its output goes to the final norm, not a next layer. is_last_layer: bool = False - # Whether the layer before this one has a sparse MLP; None when the modes + # Whether the layer before this one has a sparse MLP; None when the facts # were given directly, not planned from the layer sequence. is_previous_layer_sparse: Optional[bool] = None # Whether the layer after this one has a sparse MLP; None likewise. is_next_layer_sparse: Optional[bool] = None @classmethod - def init_new(cls, **kwargs): - context = _LayerModeComputationContext(**kwargs) + def init_new( + cls, + *, + layer_id: int, + num_layers: int, + is_layer_sparse: bool, + is_previous_layer_sparse: Optional[bool], + is_next_layer_sparse: Optional[bool], + ) -> "LayerFacts": return cls( - layer_input_mode=cls._compute_layer_input_mode(context), - attn_mode=ScatterMode.TP_ATTN_FULL, - mlp_mode=cls._compute_mlp_mode(context), - middle_residual_mode=cls._compute_middle_residual_mode(context), - layer_output_mode=cls._compute_layer_output_mode(context), - is_layer_sparse=context.is_layer_sparse, - is_first_layer=context.layer_id == 0, - is_last_layer=context.layer_id == context.num_layers - 1, - is_previous_layer_sparse=context.is_previous_layer_sparse, - is_next_layer_sparse=context.is_next_layer_sparse, + is_layer_sparse=is_layer_sparse, + is_first_layer=layer_id == 0, + is_last_layer=layer_id == num_layers - 1, + is_previous_layer_sparse=is_previous_layer_sparse, + is_next_layer_sparse=is_next_layer_sparse, ) - @classmethod - def _compute_layer_input_mode(cls, context: _LayerModeComputationContext): - if context.layer_id == 0: - return ScatterMode.model_input_output() - return cls._compute_layer_output_mode(context.previous_layer()) - - @classmethod - def _compute_mlp_mode(cls, context: _LayerModeComputationContext): - if context.is_layer_sparse: - return sparse_mlp_scatter_mode() - else: - if enable_moe_dense_fully_dp(): - return ScatterMode.SCATTERED - # A TP-sharded dense MLP reduces over the whole TP group, which spans - # every CP rank; a CP-sharded prefill must gather tokens across CP - # first or the all-reduce sums different tokens' partial outputs. - # MLA/DSA CP models gather over the attention-CP group instead. - if _generic_prefill_cp_shards_tokens() and not ( - is_dsa_enable_prefill_cp() or is_mla_cp_enabled() - ): - return ScatterMode.MOE_FULL - return ScatterMode.FULL - - @classmethod - def _should_gather_for_tbo(cls, context: _LayerModeComputationContext): - return ( - not context.is_layer_sparse - and context.is_next_layer_sparse - and enable_moe_dense_fully_dp() - and get_exec().overlap.enable_two_batch_overlap - ) - - @classmethod - def _compute_middle_residual_mode(cls, context: _LayerModeComputationContext): - mlp_mode = cls._compute_mlp_mode(context) - if mlp_mode == ScatterMode.SCATTERED: - return ScatterMode.SCATTERED - if mlp_mode in (ScatterMode.FULL, ScatterMode.MOE_FULL): - return ScatterMode.TP_ATTN_FULL - raise NotImplementedError - - @classmethod - def _compute_layer_output_mode(cls, context: _LayerModeComputationContext): - mlp_mode = cls._compute_mlp_mode(context) - if context.layer_id == context.num_layers - 1: - return ScatterMode.model_input_output() - if mlp_mode == ScatterMode.SCATTERED: - if cls._should_gather_for_tbo(context): - return ScatterMode.TP_ATTN_FULL - return ScatterMode.SCATTERED - if mlp_mode in (ScatterMode.FULL, ScatterMode.MOE_FULL): - return ScatterMode.TP_ATTN_FULL - raise NotImplementedError - def _unfused_completion_matches_the_ffn(forward_batch: ForwardBatch) -> bool: """Whether the next layer's all-reduce is the one the FFN would have run @@ -329,7 +232,7 @@ class LayerCommunicator: def __init__( self, - layer_scatter_modes: LayerScatterModes, + layer_facts: LayerFacts, input_layernorm: torch.nn.Module, post_attention_layernorm: torch.nn.Module, # Reduce scatter requires skipping all-reduce in model code after MoE/MLP, so only enable for models which have that implemented. Remove flag once done for all models that use LayerCommunicator. @@ -349,11 +252,11 @@ class LayerCommunicator: # The fused kernels a backend gives the layer, tried before its own. fusions: Optional[LayerFusions] = None, ): - self.layer_scatter_modes = layer_scatter_modes + self.layer_facts = layer_facts self.input_layernorm = input_layernorm self.post_attention_layernorm = post_attention_layernorm self.allow_reduce_scatter = allow_reduce_scatter - self.is_last_layer = layer_scatter_modes.is_last_layer + self.is_last_layer = layer_facts.is_last_layer self.qkv_latent_func = qkv_latent_func self.force_layernorm_before_dp_gather = force_layernorm_before_dp_gather self.enable_fused_ar_quant = enable_fused_ar_quant @@ -425,7 +328,7 @@ class LayerCommunicator: residual, ), attention_fusions=self._attn_input_fusions, - enters_stack=self.layer_scatter_modes.is_first_layer, + enters_stack=self.layer_facts.is_first_layer, ) if layernorm_sp.layernorm_sp_enabled() else None @@ -514,7 +417,7 @@ class LayerCommunicator: hold every token on each CP rank. An active LayerNorm SP region and input-scattered attention have their own. Raises NotImplementedError for the combinations the steps do not cover.""" - modes = self.layer_scatter_modes + modes = self.layer_facts parallel = get_parallel() if not (modes.is_first_layer or modes.is_previous_layer_sparse is not None): raise NotImplementedError( @@ -639,7 +542,7 @@ class LayerCommunicator: scattered = scattered_residual_layer_sides( axis_sizes=token_axis_sizes(), ffn_group=sides.ffn_output.group, - is_first_layer=self.layer_scatter_modes.is_first_layer, + is_first_layer=self.layer_facts.is_first_layer, is_last_layer=self.is_last_layer, leaves_for_reduce_scatter=self.allow_reduce_scatter, ) @@ -649,7 +552,7 @@ class LayerCommunicator: with_residual(scattered, self._residual), attention_handoff=handoff, attention_fusions=self._attn_input_fusions, - enters_stack=self.layer_scatter_modes.is_first_layer, + enters_stack=self.layer_facts.is_first_layer, ) def _steps_from_declarations( @@ -659,7 +562,7 @@ class LayerCommunicator: return _select_boundary_steps( sides, attention_fusions=self._attn_input_fusions, - enters_stack=self.layer_scatter_modes.is_first_layer, + enters_stack=self.layer_facts.is_first_layer, **kwargs, ) @@ -748,7 +651,7 @@ class LayerCommunicator: # The SP region opens at the first layer, re-evaluated per forward so a # crash mid-loop cannot leak into the next one. It sets what the batch's # steps are chosen from, and the first layer's input owes nothing. - if self._sp_steps is not None and self.layer_scatter_modes.is_first_layer: + if self._sp_steps is not None and self.layer_facts.is_first_layer: get_forward().set( "sp_active", layernorm_sp.runs_sp(forward_batch.forward_mode) ) @@ -1041,7 +944,7 @@ class LayerCommunicator: forward_batch=forward_batch, context=self._context, allow_reduce_scatter=self.allow_reduce_scatter, - is_layer_sparse=self.layer_scatter_modes.is_layer_sparse, + is_layer_sparse=self.layer_facts.is_layer_sparse, ) update = steps.ffn_output.update if residual is not None and update.at_producer: @@ -1136,13 +1039,6 @@ class LayerCommunicator: return hidden_states, residual return reduce_output(hidden_states), residual - def should_use_reduce_scatter(self, forward_batch: ForwardBatch): - """Whether the FFN leaves its sum to a reduce-scatter, for layers that run - their FFN outside ffn_exit.""" - return self._ffn_leaves_sum_to_reduce_scatter( - forward_batch, self._postprocess_dp_step(forward_batch) - ) - def _select_ffn_exit_fusions( self, ) -> Tuple[Callable[[ForwardBatch], Optional["FfnExitFusion"]], ...]: @@ -1359,7 +1255,7 @@ class MHCLayerCommunicator(LayerCommunicator): def __init__( self, - layer_scatter_modes: LayerScatterModes, + layer_facts: LayerFacts, input_layernorm: torch.nn.Module, post_attention_layernorm: torch.nn.Module, allow_reduce_scatter: bool = False, @@ -1377,9 +1273,9 @@ class MHCLayerCommunicator(LayerCommunicator): hc_ffn_pre=hc_ffn_pre, hc_post=hc_post, hc_ffn_post_pre=hc_ffn_post_pre, - is_last_layer=layer_scatter_modes.is_last_layer, + is_last_layer=layer_facts.is_last_layer, ) - if layer_scatter_modes.is_layer_sparse and sparse_moe_gathers_over_moe_cp(): + if layer_facts.is_layer_sparse and sparse_moe_gathers_over_moe_cp(): raise NotImplementedError( "MHCLayerCommunicator does not support a MoE gathered over the " "MoE-CP group (moe_dp_size < attention_context_parallel_size). " @@ -1388,7 +1284,7 @@ class MHCLayerCommunicator(LayerCommunicator): # The postprocess writes the FFN output into the streams, so the FFN's # sum never waits for the next layer. super().__init__( - layer_scatter_modes, + layer_facts, input_layernorm, post_attention_layernorm, allow_reduce_scatter, diff --git a/python/sglang/srt/layers/communicator/layout.py b/python/sglang/srt/layers/communicator/layout.py index e2f7b603df..87a800e89b 100644 --- a/python/sglang/srt/layers/communicator/layout.py +++ b/python/sglang/srt/layers/communicator/layout.py @@ -71,30 +71,6 @@ class SumGroup(Enum): MOE_OUTPUT = auto() -class ScatterMode(Enum): - """ - Suppose we have TP=4, DP=2, enable-dp-attention, and the system handles seq a,b,c,d - Model input/output: [ab, ab, cd, cd] for four ranks respectively - SCATTERED: [a, b, c, d] - TP_ATTN_FULL: [ab, ab, cd, cd], i.e. all ranks inside a TP attn group have full data of the group - FULL: [abcd, abcd, abcd, abcd] - MOE_FULL: full within the MoE group (cp_per_moe CP chunks), used when moe_dp_size < attn_cp_size - """ - - SCATTERED = auto() - TP_ATTN_FULL = auto() - FULL = auto() - MOE_FULL = auto() - - @staticmethod - def model_input_output(): - """The scatter mode for model forward pass input and output data""" - if is_dsa_enable_prefill_cp() or is_mla_cp_enabled(): - return ScatterMode.SCATTERED - - return ScatterMode.TP_ATTN_FULL - - def enable_moe_dense_fully_dp(): return get_parallel().moe_dense_tp_size == 1 @@ -105,10 +81,6 @@ def _generic_prefill_cp_shards_tokens() -> bool: return parallel.attn_cp_size > 1 and parallel.enable_prefill_cp -def enable_dwdp(): - return get_parallel().dwdp_size > 1 - - def _batch_size(forward_batch: ForwardBatch) -> int: return ( forward_batch.input_ids.shape[0] if hasattr(forward_batch, "input_ids") else 0 @@ -124,7 +96,6 @@ def _ffn_has_tokens(forward_batch: ForwardBatch) -> bool: @dataclass class CommunicateContext: - process_group_sizes: Dict[ScatterMode, int] attn_tp_rank: int attn_tp_size: int attn_dp_size: int @@ -144,18 +115,7 @@ class CommunicateContext: attn_cp_rank = get_parallel().attn_cp_rank tp_size = get_parallel().tp_size tp_rank = get_parallel().tp_rank - moe_cp_size = get_moe_cp_size() - process_group_sizes = { - ScatterMode.SCATTERED: 1, - ScatterMode.TP_ATTN_FULL: attn_tp_size, - # TODO: support --moe-dense-tp-size > 1 - # With context parallel enabled, we should exclude - # the attn_cp_size from the total tp_size - ScatterMode.FULL: tp_size // attn_cp_size, - ScatterMode.MOE_FULL: tp_size // (attn_cp_size // moe_cp_size), - } return cls( - process_group_sizes=process_group_sizes, attn_tp_rank=attn_tp_rank, attn_tp_size=attn_tp_size, attn_dp_size=attn_dp_size, diff --git a/python/sglang/srt/models/bailing_moe.py b/python/sglang/srt/models/bailing_moe.py index b01ec580c9..f9187514c9 100644 --- a/python/sglang/srt/models/bailing_moe.py +++ b/python/sglang/srt/models/bailing_moe.py @@ -33,7 +33,7 @@ from sglang.srt.eplb.expert_location_dispatch import ExpertLocationDispatchInfo from sglang.srt.layers.activation import SiluAndMul from sglang.srt.layers.communicator import ( LayerCommunicator, - LayerScatterModes, + LayerFacts, enable_moe_dense_fully_dp, reduce_output, ) @@ -669,7 +669,7 @@ class BailingMoEBlock(nn.Module): config, layer_id=layer_id + 1, is_nextn=False ) - self.layer_scatter_modes = LayerScatterModes.init_new( + self.layer_facts = LayerFacts.init_new( layer_id=layer_id, # A NextN draft is a one-layer model. num_layers=1 if is_nextn else config.num_hidden_layers, @@ -703,7 +703,7 @@ class BailingMoEBlock(nn.Module): self.post_attention_layernorm = RMSNorm(hidden_size, eps=config.rms_norm_eps) self.layer_communicator = LayerCommunicator( - layer_scatter_modes=self.layer_scatter_modes, + layer_facts=self.layer_facts, input_layernorm=self.input_layernorm, post_attention_layernorm=self.post_attention_layernorm, allow_reduce_scatter=True, diff --git a/python/sglang/srt/models/bailing_moe_linear.py b/python/sglang/srt/models/bailing_moe_linear.py index f099c5d0e0..5bcf27a284 100644 --- a/python/sglang/srt/models/bailing_moe_linear.py +++ b/python/sglang/srt/models/bailing_moe_linear.py @@ -17,7 +17,7 @@ from sglang.srt.layers import deep_gemm_wrapper from sglang.srt.layers.activation import SiluAndMul from sglang.srt.layers.communicator import ( LayerCommunicator, - LayerScatterModes, + LayerFacts, ) from sglang.srt.layers.dp_attention import ( is_dp_attention_enabled, @@ -813,7 +813,7 @@ class BailingMoELinearDecoderLayer(nn.Module): self.input_layernorm = RMSNorm(self.hidden_size, eps=rms_norm_eps) self.post_attention_layernorm = RMSNorm(self.hidden_size, eps=rms_norm_eps) - self.layer_scatter_modes = LayerScatterModes.init_new( + self.layer_facts = LayerFacts.init_new( layer_id=layer_id, # A NextN draft is a one-layer model. num_layers=1 if is_nextn else config.num_hidden_layers, @@ -828,7 +828,7 @@ class BailingMoELinearDecoderLayer(nn.Module): else None ) self.layer_communicator = LayerCommunicator( - layer_scatter_modes=self.layer_scatter_modes, + layer_facts=self.layer_facts, input_layernorm=self.input_layernorm, post_attention_layernorm=self.post_attention_layernorm, allow_reduce_scatter=False, diff --git a/python/sglang/srt/models/bailing_moe_v3.py b/python/sglang/srt/models/bailing_moe_v3.py index 333479224a..23a0729a50 100644 --- a/python/sglang/srt/models/bailing_moe_v3.py +++ b/python/sglang/srt/models/bailing_moe_v3.py @@ -30,7 +30,7 @@ from sglang.srt.layers import deep_gemm_wrapper from sglang.srt.layers.activation import SiluAndMul from sglang.srt.layers.communicator import ( LayerCommunicator, - LayerScatterModes, + LayerFacts, enable_moe_dense_fully_dp, layer_input_buffer, reduce_output, @@ -1148,7 +1148,7 @@ class BailingMoELinearDecoderLayer(nn.Module): self.input_layernorm = RMSNorm(self.hidden_size, eps=rms_norm_eps) self.post_attention_layernorm = RMSNorm(self.hidden_size, eps=rms_norm_eps) - self.layer_scatter_modes = LayerScatterModes.init_new( + self.layer_facts = LayerFacts.init_new( layer_id=layer_id, # NextN wraps a single decoder layer whose checkpoint prefix is the # post-model layer id. Treat it as a one-layer model for scatter-mode @@ -1160,7 +1160,7 @@ class BailingMoELinearDecoderLayer(nn.Module): ) self.layer_communicator = LayerCommunicator( - layer_scatter_modes=self.layer_scatter_modes, + layer_facts=self.layer_facts, input_layernorm=self.input_layernorm, post_attention_layernorm=self.post_attention_layernorm, allow_reduce_scatter=True, diff --git a/python/sglang/srt/models/deepseek_v2.py b/python/sglang/srt/models/deepseek_v2.py index 0408a9ec7e..4f23371e53 100644 --- a/python/sglang/srt/models/deepseek_v2.py +++ b/python/sglang/srt/models/deepseek_v2.py @@ -69,7 +69,7 @@ from sglang.srt.layers.aux_hidden_states import ( ) from sglang.srt.layers.communicator import ( LayerCommunicator, - LayerScatterModes, + LayerFacts, enable_moe_dense_fully_dp, get_attn_tp_context, layer_input_buffer, @@ -2576,7 +2576,7 @@ class DeepseekV2DecoderLayer(nn.Module): is_previous_layer_sparse = self._is_layer_sparse(layer_id - 1, is_nextn=False) is_next_layer_sparse = self._is_layer_sparse(layer_id + 1, is_nextn=False) - self.layer_scatter_modes = LayerScatterModes.init_new( + self.layer_facts = LayerFacts.init_new( layer_id=layer_id, num_layers=1 if is_nextn else config.num_hidden_layers, is_layer_sparse=self.is_layer_sparse, @@ -2643,7 +2643,7 @@ class DeepseekV2DecoderLayer(nn.Module): fusions = CuteDSLFusion() return LayerCommunicator( - layer_scatter_modes=self.layer_scatter_modes, + layer_facts=self.layer_facts, input_layernorm=input_layernorm, post_attention_layernorm=post_attention_layernorm, allow_reduce_scatter=True, diff --git a/python/sglang/srt/models/dots3_common/modeling.py b/python/sglang/srt/models/dots3_common/modeling.py index 170bd39e82..4ca0c649a5 100644 --- a/python/sglang/srt/models/dots3_common/modeling.py +++ b/python/sglang/srt/models/dots3_common/modeling.py @@ -59,7 +59,7 @@ from sglang.srt.layers.activation import SiluAndMul from sglang.srt.layers.attention.dsa.dsa_indexer import Indexer from sglang.srt.layers.communicator import ( LayerCommunicator, - LayerScatterModes, + LayerFacts, enable_moe_dense_fully_dp, ) from sglang.srt.layers.dp_attention import is_dp_attention_enabled @@ -1541,7 +1541,7 @@ class Dots3DecoderLayer(nn.Module): is_previous_layer_sparse = self._is_layer_sparse(layer_id - 1, is_nextn=False) is_next_layer_sparse = self._is_layer_sparse(layer_id + 1, is_nextn=False) - self.layer_scatter_modes = LayerScatterModes.init_new( + self.layer_facts = LayerFacts.init_new( layer_id=layer_id, num_layers=1 if is_nextn else config.num_hidden_layers, is_layer_sparse=self.is_layer_sparse, @@ -1579,7 +1579,7 @@ class Dots3DecoderLayer(nn.Module): ) self.layer_communicator = LayerCommunicator( - layer_scatter_modes=self.layer_scatter_modes, + layer_facts=self.layer_facts, input_layernorm=self.input_layernorm, post_attention_layernorm=self.post_attention_layernorm, allow_reduce_scatter=True, diff --git a/python/sglang/srt/models/falcon_h1.py b/python/sglang/srt/models/falcon_h1.py index 954abebe3a..243fb83070 100644 --- a/python/sglang/srt/models/falcon_h1.py +++ b/python/sglang/srt/models/falcon_h1.py @@ -11,7 +11,7 @@ from sglang.srt.layers.attention.hybrid_linear_attn_backend import ( Mamba2AttnBackend, ) from sglang.srt.layers.attention.mamba.mamba import MambaMixer2 -from sglang.srt.layers.communicator import LayerCommunicator, LayerScatterModes +from sglang.srt.layers.communicator import LayerCommunicator, LayerFacts from sglang.srt.layers.dp_attention import ( is_dp_attention_enabled, ) @@ -198,7 +198,7 @@ class FalconH1HybridAttentionDecoderLayer(nn.Module): is_previous_layer_sparse = False is_next_layer_sparse = False - self.layer_scatter_modes = LayerScatterModes.init_new( + self.layer_facts = LayerFacts.init_new( layer_id=layer_id, num_layers=config.num_hidden_layers, is_layer_sparse=self.is_layer_sparse, @@ -223,7 +223,7 @@ class FalconH1HybridAttentionDecoderLayer(nn.Module): self.k_norm = RMSNorm(self.head_dim, eps=config.rms_norm_eps) self.layer_communicator = LayerCommunicator( - layer_scatter_modes=self.layer_scatter_modes, + layer_facts=self.layer_facts, input_layernorm=self.input_layernorm, post_attention_layernorm=self.pre_ff_layernorm, allow_reduce_scatter=True, diff --git a/python/sglang/srt/models/glm4_moe.py b/python/sglang/srt/models/glm4_moe.py index c12375f409..a165b4b07c 100644 --- a/python/sglang/srt/models/glm4_moe.py +++ b/python/sglang/srt/models/glm4_moe.py @@ -40,7 +40,7 @@ from sglang.srt.eplb.expert_location_dispatch import ExpertLocationDispatchInfo from sglang.srt.layers.activation import SiluAndMul from sglang.srt.layers.communicator import ( LayerCommunicator, - LayerScatterModes, + LayerFacts, enable_moe_dense_fully_dp, reduce_output, ) @@ -836,7 +836,7 @@ class Glm4MoeDecoderLayer(nn.Module): is_previous_layer_sparse = self._is_layer_sparse(layer_id - 1, is_nextn=False) is_next_layer_sparse = self._is_layer_sparse(layer_id + 1, is_nextn=False) - self.layer_scatter_modes = LayerScatterModes.init_new( + self.layer_facts = LayerFacts.init_new( layer_id=layer_id, num_layers=1 if is_nextn else config.num_hidden_layers, is_layer_sparse=self.is_layer_sparse, @@ -873,7 +873,7 @@ class Glm4MoeDecoderLayer(nn.Module): ) self.layer_communicator = LayerCommunicator( - layer_scatter_modes=self.layer_scatter_modes, + layer_facts=self.layer_facts, input_layernorm=self.input_layernorm, post_attention_layernorm=self.post_attention_layernorm, allow_reduce_scatter=True, diff --git a/python/sglang/srt/models/glm4_moe_lite.py b/python/sglang/srt/models/glm4_moe_lite.py index 6e21969626..fb9e290ed7 100644 --- a/python/sglang/srt/models/glm4_moe_lite.py +++ b/python/sglang/srt/models/glm4_moe_lite.py @@ -34,7 +34,7 @@ from sglang.srt.eplb.expert_location_dispatch import ExpertLocationDispatchInfo from sglang.srt.layers.activation import SiluAndMul from sglang.srt.layers.communicator import ( LayerCommunicator, - LayerScatterModes, + LayerFacts, enable_moe_dense_fully_dp, get_attn_tp_context, reduce_output, @@ -548,7 +548,7 @@ class Glm4MoeLiteDecoderLayer(nn.Module): is_previous_layer_sparse = self._is_layer_sparse(layer_id - 1, is_nextn=False) is_next_layer_sparse = self._is_layer_sparse(layer_id + 1, is_nextn=False) - self.layer_scatter_modes = LayerScatterModes.init_new( + self.layer_facts = LayerFacts.init_new( layer_id=layer_id, num_layers=1 if is_nextn else config.num_hidden_layers, is_layer_sparse=self.is_layer_sparse, @@ -588,7 +588,7 @@ class Glm4MoeLiteDecoderLayer(nn.Module): self._gfx95_quant_format = self._detect_gfx95_quant_format() self.layer_communicator = LayerCommunicator( - layer_scatter_modes=self.layer_scatter_modes, + layer_facts=self.layer_facts, input_layernorm=self.input_layernorm, post_attention_layernorm=self.post_attention_layernorm, allow_reduce_scatter=True, diff --git a/python/sglang/srt/models/glm5_next.py b/python/sglang/srt/models/glm5_next.py index ed71974c24..81ce2b67b7 100644 --- a/python/sglang/srt/models/glm5_next.py +++ b/python/sglang/srt/models/glm5_next.py @@ -27,7 +27,7 @@ from sglang.srt.layers.attention import vision_utils from sglang.srt.layers.attention.vision import VisionAttention from sglang.srt.layers.communicator import ( LayerCommunicator, - LayerScatterModes, + LayerFacts, MHCLayerCommunicator, enable_moe_dense_fully_dp, get_attn_tp_context, @@ -726,7 +726,7 @@ class Glm5NextDecoderLayer(nn.Module): is_previous_layer_sparse = self._is_layer_sparse(layer_id - 1, is_nextn=False) is_next_layer_sparse = self._is_layer_sparse(layer_id + 1, is_nextn=False) - self.layer_scatter_modes = LayerScatterModes.init_new( + self.layer_facts = LayerFacts.init_new( layer_id=layer_id, num_layers=1 if is_nextn else config.num_hidden_layers, is_layer_sparse=self.is_layer_sparse, @@ -784,7 +784,7 @@ class Glm5NextDecoderLayer(nn.Module): ) shared_kwargs: Dict[str, Any] = dict( - layer_scatter_modes=self.layer_scatter_modes, + layer_facts=self.layer_facts, input_layernorm=self.input_layernorm, post_attention_layernorm=self.post_attention_layernorm, allow_reduce_scatter=True, diff --git a/python/sglang/srt/models/gpt_oss.py b/python/sglang/srt/models/gpt_oss.py index b08c459d3c..b06a2b7f8a 100644 --- a/python/sglang/srt/models/gpt_oss.py +++ b/python/sglang/srt/models/gpt_oss.py @@ -31,7 +31,7 @@ from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_r from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation from sglang.srt.layers.communicator import ( LayerCommunicator, - LayerScatterModes, + LayerFacts, reduce_output, ) from sglang.srt.layers.dp_attention import ( @@ -569,7 +569,7 @@ class GptOssDecoderLayer(nn.Module): is_previous_layer_sparse = True is_next_layer_sparse = True - self.layer_scatter_modes = LayerScatterModes.init_new( + self.layer_facts = LayerFacts.init_new( layer_id=layer_id, num_layers=config.num_hidden_layers, is_layer_sparse=self.is_layer_sparse, @@ -606,7 +606,7 @@ class GptOssDecoderLayer(nn.Module): ) self.layer_communicator = LayerCommunicator( - layer_scatter_modes=self.layer_scatter_modes, + layer_facts=self.layer_facts, input_layernorm=self.input_layernorm, post_attention_layernorm=self.post_attention_layernorm, allow_reduce_scatter=True, diff --git a/python/sglang/srt/models/interns2_mobius.py b/python/sglang/srt/models/interns2_mobius.py index d30935644a..ff477e564c 100644 --- a/python/sglang/srt/models/interns2_mobius.py +++ b/python/sglang/srt/models/interns2_mobius.py @@ -10,7 +10,7 @@ from sglang.srt.configs.interns2_mobius import ( InternS2MobiusConfig, InternS2MobiusTextConfig, ) -from sglang.srt.layers.communicator import LayerCommunicator, LayerScatterModes +from sglang.srt.layers.communicator import LayerCommunicator, LayerFacts from sglang.srt.layers.dp_attention import is_dp_attention_enabled from sglang.srt.layers.layernorm import GemmaRMSNorm from sglang.srt.layers.linear import ( @@ -476,7 +476,7 @@ class InternS2MobiusLinearDecoderLayer(_InternS2MobiusDecoderMixin, nn.Module): layer_prefix = prefix.removesuffix(".linear_attn") self._init_mobius_mlp(config, quant_config, layer_prefix) - self.layer_scatter_modes = LayerScatterModes.init_new( + self.layer_facts = LayerFacts.init_new( layer_id=layer_id, num_layers=config.num_hidden_layers, is_layer_sparse=True, @@ -492,7 +492,7 @@ class InternS2MobiusLinearDecoderLayer(_InternS2MobiusDecoderMixin, nn.Module): and _linear_accepts_fp8_tuple(self.linear_attn.in_proj_qkvz) ) self.layer_communicator = LayerCommunicator( - layer_scatter_modes=self.layer_scatter_modes, + layer_facts=self.layer_facts, input_layernorm=self.input_layernorm, post_attention_layernorm=self.post_attention_layernorm, allow_reduce_scatter=True, @@ -609,7 +609,7 @@ class InternS2MobiusAttentionDecoderLayer( layer_prefix = prefix.removesuffix(".self_attn") self._init_mobius_mlp(config, quant_config, layer_prefix) - self.layer_scatter_modes = LayerScatterModes.init_new( + self.layer_facts = LayerFacts.init_new( layer_id=layer_id, num_layers=config.num_hidden_layers, is_layer_sparse=True, @@ -626,7 +626,7 @@ class InternS2MobiusAttentionDecoderLayer( _enable_qwen35_fused_ar_quant() and _linear_accepts_fp8_tuple(self.qkv_proj) ) self.layer_communicator = LayerCommunicator( - layer_scatter_modes=self.layer_scatter_modes, + layer_facts=self.layer_facts, input_layernorm=self.input_layernorm, post_attention_layernorm=self.post_attention_layernorm, allow_reduce_scatter=True, diff --git a/python/sglang/srt/models/laguna.py b/python/sglang/srt/models/laguna.py index df3bdf4ac8..a84fb9ed56 100644 --- a/python/sglang/srt/models/laguna.py +++ b/python/sglang/srt/models/laguna.py @@ -21,7 +21,7 @@ from sglang.srt.environ import envs from sglang.srt.layers.activation import SiluAndMul from sglang.srt.layers.communicator import ( LayerCommunicator, - LayerScatterModes, + LayerFacts, reduce_output, ) from sglang.srt.layers.dp_attention import ( @@ -455,7 +455,7 @@ class LagunaDecoderLayer(nn.Module): config.hidden_size, eps=config.rms_norm_eps ) - self.layer_scatter_modes = LayerScatterModes.init_new( + self.layer_facts = LayerFacts.init_new( layer_id=layer_id, num_layers=config.num_hidden_layers, is_layer_sparse=self.is_layer_sparse, @@ -463,7 +463,7 @@ class LagunaDecoderLayer(nn.Module): is_next_layer_sparse=is_next_layer_sparse, ) self.layer_communicator = LayerCommunicator( - layer_scatter_modes=self.layer_scatter_modes, + layer_facts=self.layer_facts, input_layernorm=self.input_layernorm, post_attention_layernorm=self.post_attention_layernorm, allow_reduce_scatter=True, diff --git a/python/sglang/srt/models/llada2.py b/python/sglang/srt/models/llada2.py index 760a72ec01..d41b22b735 100644 --- a/python/sglang/srt/models/llada2.py +++ b/python/sglang/srt/models/llada2.py @@ -33,7 +33,7 @@ from sglang.srt.eplb.expert_location_dispatch import ExpertLocationDispatchInfo from sglang.srt.layers.activation import SiluAndMul from sglang.srt.layers.communicator import ( LayerCommunicator, - LayerScatterModes, + LayerFacts, enable_moe_dense_fully_dp, ) from sglang.srt.layers.dp_attention import ( @@ -612,7 +612,7 @@ class LLaDA2MoeBlock(nn.Module): is_previous_layer_sparse = self._is_layer_sparse(config, layer_id=layer_id - 1) is_next_layer_sparse = self._is_layer_sparse(config, layer_id=layer_id + 1) - self.layer_scatter_modes = LayerScatterModes.init_new( + self.layer_facts = LayerFacts.init_new( layer_id=layer_id, num_layers=config.num_hidden_layers, is_layer_sparse=self.is_layer_sparse, @@ -645,7 +645,7 @@ class LLaDA2MoeBlock(nn.Module): self.post_attention_layernorm = RMSNorm(hidden_size, eps=config.rms_norm_eps) self.layer_communicator = LayerCommunicator( - layer_scatter_modes=self.layer_scatter_modes, + layer_facts=self.layer_facts, input_layernorm=self.input_layernorm, post_attention_layernorm=self.post_attention_layernorm, allow_reduce_scatter=True, diff --git a/python/sglang/srt/models/llama4.py b/python/sglang/srt/models/llama4.py index a5f4e1cb9e..5acf60cf9a 100644 --- a/python/sglang/srt/models/llama4.py +++ b/python/sglang/srt/models/llama4.py @@ -25,7 +25,7 @@ import torch from torch import nn from transformers import Llama4TextConfig -from sglang.srt.layers.communicator import LayerCommunicator, LayerScatterModes +from sglang.srt.layers.communicator import LayerCommunicator, LayerFacts from sglang.srt.layers.dp_attention import ( is_dp_attention_enabled, ) @@ -420,7 +420,7 @@ class Llama4DecoderLayer(nn.Module): config.hidden_size, eps=config.rms_norm_eps ) - self.layer_scatter_modes = LayerScatterModes.init_new( + self.layer_facts = LayerFacts.init_new( layer_id=layer_id, num_layers=config.num_hidden_layers, is_layer_sparse=is_moe_layer, @@ -429,7 +429,7 @@ class Llama4DecoderLayer(nn.Module): ) self.layer_communicator = LayerCommunicator( - layer_scatter_modes=self.layer_scatter_modes, + layer_facts=self.layer_facts, input_layernorm=self.input_layernorm, post_attention_layernorm=self.post_attention_layernorm, allow_reduce_scatter=True, diff --git a/python/sglang/srt/models/longcat_flash.py b/python/sglang/srt/models/longcat_flash.py index 61f77a5632..0d1c4aeeb7 100644 --- a/python/sglang/srt/models/longcat_flash.py +++ b/python/sglang/srt/models/longcat_flash.py @@ -51,7 +51,7 @@ from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_r from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation from sglang.srt.layers import deep_gemm_wrapper from sglang.srt.layers.activation import SiluAndMul -from sglang.srt.layers.communicator import LayerCommunicator, LayerScatterModes +from sglang.srt.layers.communicator import LayerCommunicator, LayerFacts from sglang.srt.layers.dp_attention import ( is_dp_attention_enabled, ) @@ -419,8 +419,8 @@ class LongcatFlashDecoderLayer(nn.Module): self.attn_tp_size = get_parallel().attn_tp_size self.attn_tp_rank = get_parallel().attn_tp_rank - self.mlp_layer_scatter_modes = [ - LayerScatterModes.init_new( + self.mlp_layer_facts = [ + LayerFacts.init_new( layer_id=self.layer_id * 2 + i, num_layers=config.num_hidden_layers, is_layer_sparse=False, @@ -432,7 +432,7 @@ class LongcatFlashDecoderLayer(nn.Module): ] self.mlp_layer_communicator = [ LayerCommunicator( - layer_scatter_modes=self.mlp_layer_scatter_modes[i], + layer_facts=self.mlp_layer_facts[i], input_layernorm=self.input_layernorm[i], post_attention_layernorm=self.post_attention_layernorm[i], qkv_latent_func=self.self_attn[i].prepare_qkv_latent, @@ -441,7 +441,7 @@ class LongcatFlashDecoderLayer(nn.Module): for i in range(2) ] - self.moe_layer_scatter_modes = LayerScatterModes.init_new( + self.moe_layer_facts = LayerFacts.init_new( layer_id=self.layer_id, num_layers=config.num_hidden_layers, is_layer_sparse=True, @@ -450,7 +450,7 @@ class LongcatFlashDecoderLayer(nn.Module): is_next_layer_sparse=True, ) self.moe_layer_communicator = LayerCommunicator( - layer_scatter_modes=self.moe_layer_scatter_modes, + layer_facts=self.moe_layer_facts, input_layernorm=self.input_layernorm[0], post_attention_layernorm=self.post_attention_layernorm[0], qkv_latent_func=self.self_attn[0].prepare_qkv_latent, diff --git a/python/sglang/srt/models/longcat_flash_nextn.py b/python/sglang/srt/models/longcat_flash_nextn.py index dd48b333a8..363c7e3e77 100644 --- a/python/sglang/srt/models/longcat_flash_nextn.py +++ b/python/sglang/srt/models/longcat_flash_nextn.py @@ -41,7 +41,7 @@ from sglang.kernels.ops.quantization.fp8_kernel import is_fp8_fnuz from sglang.srt.configs import LongcatFlashConfig from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder from sglang.srt.layers import deep_gemm_wrapper -from sglang.srt.layers.communicator import LayerCommunicator, LayerScatterModes +from sglang.srt.layers.communicator import LayerCommunicator, LayerFacts from sglang.srt.layers.dp_attention import ( is_dp_attention_enabled, ) @@ -159,7 +159,7 @@ class LongcatFlashDenseDecoderLayer(nn.Module): self.attn_tp_size = get_parallel().attn_tp_size self.attn_tp_rank = get_parallel().attn_tp_rank - self.layer_scatter_modes = LayerScatterModes.init_new( + self.layer_facts = LayerFacts.init_new( layer_id=self.layer_id, num_layers=config.num_hidden_layers, is_layer_sparse=False, @@ -167,7 +167,7 @@ class LongcatFlashDenseDecoderLayer(nn.Module): is_next_layer_sparse=False, ) self.layer_communicator = LayerCommunicator( - layer_scatter_modes=self.layer_scatter_modes, + layer_facts=self.layer_facts, input_layernorm=self.input_layernorm, post_attention_layernorm=self.post_attention_layernorm, ) diff --git a/python/sglang/srt/models/mellum.py b/python/sglang/srt/models/mellum.py index 1e678c9a26..8bbf656978 100644 --- a/python/sglang/srt/models/mellum.py +++ b/python/sglang/srt/models/mellum.py @@ -25,7 +25,7 @@ import torch from torch import nn from transformers import PretrainedConfig -from sglang.srt.layers.communicator import LayerCommunicator, LayerScatterModes +from sglang.srt.layers.communicator import LayerCommunicator, LayerFacts from sglang.srt.layers.layernorm import RMSNorm from sglang.srt.layers.linear import QKVParallelLinear, RowParallelLinear from sglang.srt.layers.quantization.base_config import QuantizationConfig @@ -442,7 +442,7 @@ class MellumDecoderLayer(Qwen3MoeDecoderLayer): is_previous_layer_sparse = _is_sparse(layer_id - 1) is_next_layer_sparse = _is_sparse(layer_id + 1) - self.layer_scatter_modes = LayerScatterModes.init_new( + self.layer_facts = LayerFacts.init_new( layer_id=layer_id, num_layers=cfg.num_hidden_layers, is_layer_sparse=self.is_layer_sparse, @@ -454,7 +454,7 @@ class MellumDecoderLayer(Qwen3MoeDecoderLayer): self.post_attention_layernorm = RMSNorm(cfg.hidden_size, eps=rms_norm_eps) self.layer_communicator = LayerCommunicator( - layer_scatter_modes=self.layer_scatter_modes, + layer_facts=self.layer_facts, input_layernorm=self.input_layernorm, post_attention_layernorm=self.post_attention_layernorm, allow_reduce_scatter=True, diff --git a/python/sglang/srt/models/mimo_v2.py b/python/sglang/srt/models/mimo_v2.py index 144a21f18e..07ff88fe60 100644 --- a/python/sglang/srt/models/mimo_v2.py +++ b/python/sglang/srt/models/mimo_v2.py @@ -34,7 +34,7 @@ from sglang.srt.layers.aux_hidden_states import ( ) from sglang.srt.layers.communicator import ( LayerCommunicator, - LayerScatterModes, + LayerFacts, enable_moe_dense_fully_dp, ) from sglang.srt.layers.dp_attention import ( @@ -856,7 +856,7 @@ class MiMoV2DecoderLayer(nn.Module): config.hidden_size, eps=config.layernorm_epsilon ) - self.layer_scatter_modes = LayerScatterModes.init_new( + self.layer_facts = LayerFacts.init_new( layer_id=layer_id, num_layers=config.num_hidden_layers, is_layer_sparse=self.is_layer_sparse, @@ -864,7 +864,7 @@ class MiMoV2DecoderLayer(nn.Module): is_next_layer_sparse=is_next_layer_sparse, ) self.layer_communicator = LayerCommunicator( - layer_scatter_modes=self.layer_scatter_modes, + layer_facts=self.layer_facts, input_layernorm=self.input_layernorm, post_attention_layernorm=self.post_attention_layernorm, allow_reduce_scatter=True, diff --git a/python/sglang/srt/models/mimo_v2_nextn.py b/python/sglang/srt/models/mimo_v2_nextn.py index 4ae075531d..f317b6b610 100644 --- a/python/sglang/srt/models/mimo_v2_nextn.py +++ b/python/sglang/srt/models/mimo_v2_nextn.py @@ -23,7 +23,7 @@ from sglang.srt.configs.model_config import get_mimo_v2_fused_qkv_expected_tp_si from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder from sglang.srt.layers.communicator import ( LayerCommunicator, - LayerScatterModes, + LayerFacts, enable_moe_dense_fully_dp, ) from sglang.srt.layers.dp_attention import ( @@ -116,7 +116,7 @@ class MiMoV2MTPLayer(nn.Module): self.post_attention_layernorm = RMSNorm( config.hidden_size, eps=config.layernorm_epsilon ) - self.layer_scatter_modes = LayerScatterModes.init_new( + self.layer_facts = LayerFacts.init_new( layer_id=layer_id, num_layers=1, is_layer_sparse=self.is_layer_sparse, @@ -124,7 +124,7 @@ class MiMoV2MTPLayer(nn.Module): is_next_layer_sparse=is_next_layer_sparse, ) self.layer_communicator = LayerCommunicator( - layer_scatter_modes=self.layer_scatter_modes, + layer_facts=self.layer_facts, input_layernorm=self.input_layernorm, post_attention_layernorm=self.post_attention_layernorm, allow_deferred_ffn_reduction=False, diff --git a/python/sglang/srt/models/minimax_m2.py b/python/sglang/srt/models/minimax_m2.py index 28c695d6fc..faa1a06f45 100644 --- a/python/sglang/srt/models/minimax_m2.py +++ b/python/sglang/srt/models/minimax_m2.py @@ -32,7 +32,7 @@ from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_r from sglang.srt.eplb.expert_location_dispatch import ExpertLocationDispatchInfo from sglang.srt.layers.communicator import ( LayerCommunicator, - LayerScatterModes, + LayerFacts, ) from sglang.srt.layers.dp_attention import ( attn_tp_all_reduce, @@ -975,7 +975,7 @@ class MiniMaxM2DecoderLayer(nn.Module): is_previous_layer_sparse = True is_next_layer_sparse = True - self.layer_scatter_modes = LayerScatterModes.init_new( + self.layer_facts = LayerFacts.init_new( layer_id=layer_id, num_layers=config.num_hidden_layers, is_layer_sparse=self.is_layer_sparse, @@ -984,7 +984,7 @@ class MiniMaxM2DecoderLayer(nn.Module): ) self.layer_communicator = LayerCommunicator( - layer_scatter_modes=self.layer_scatter_modes, + layer_facts=self.layer_facts, input_layernorm=self.input_layernorm, post_attention_layernorm=self.post_attention_layernorm, allow_reduce_scatter=True, diff --git a/python/sglang/srt/models/minimax_m3.py b/python/sglang/srt/models/minimax_m3.py index d3900dcbbf..d1d22bd532 100644 --- a/python/sglang/srt/models/minimax_m3.py +++ b/python/sglang/srt/models/minimax_m3.py @@ -37,7 +37,7 @@ from sglang.srt.eplb.expert_location_dispatch import ExpertLocationDispatchInfo from sglang.srt.layers.activation import SiluAndMul from sglang.srt.layers.communicator import ( LayerCommunicator, - LayerScatterModes, + LayerFacts, enable_moe_dense_fully_dp, ) from sglang.srt.layers.dp_attention import ( @@ -1297,7 +1297,7 @@ class MiniMaxM3DecoderLayer(nn.Module): moe_layer_freq = getattr(config, "moe_layer_freq", None) # Means "MLP is a sparse MoE", not attention sparsity. Kept as ``is_layer_sparse`` - # because LayerCommunicator / LayerScatterModes / other models read this attr. + # because LayerCommunicator / LayerFacts / other models read this attr. self.is_layer_sparse = ( moe_layer_freq[layer_id] != 0 if moe_layer_freq is not None else True ) @@ -1347,7 +1347,7 @@ class MiniMaxM3DecoderLayer(nn.Module): is_previous_layer_sparse = _is_layer_sparse(layer_id - 1) is_next_layer_sparse = _is_layer_sparse(layer_id + 1) - self.layer_scatter_modes = LayerScatterModes.init_new( + self.layer_facts = LayerFacts.init_new( layer_id=layer_id, num_layers=config.num_hidden_layers, is_layer_sparse=self.is_layer_sparse, @@ -1356,7 +1356,7 @@ class MiniMaxM3DecoderLayer(nn.Module): ) self.layer_communicator = LayerCommunicator( - layer_scatter_modes=self.layer_scatter_modes, + layer_facts=self.layer_facts, input_layernorm=self.input_layernorm, post_attention_layernorm=self.post_attention_layernorm, allow_reduce_scatter=True, diff --git a/python/sglang/srt/models/moss_vl.py b/python/sglang/srt/models/moss_vl.py index dac1ca0be7..1b9968a53b 100644 --- a/python/sglang/srt/models/moss_vl.py +++ b/python/sglang/srt/models/moss_vl.py @@ -22,7 +22,7 @@ from sglang.srt.layers.attention.vision import ( VisionAttentionMetadata, prepare_vision_attention_metadata, ) -from sglang.srt.layers.communicator import LayerCommunicator, LayerScatterModes +from sglang.srt.layers.communicator import LayerCommunicator, LayerFacts from sglang.srt.layers.conv import Conv3dLayer from sglang.srt.layers.layernorm import RMSNorm from sglang.srt.layers.linear import ( @@ -1011,7 +1011,7 @@ class MossVLSelfAttentionDecoderLayer(nn.Module): self.post_attention_layernorm = RMSNorm( config.hidden_size, eps=config.rms_norm_eps, **norm_kwargs ) - self.layer_scatter_modes = LayerScatterModes.init_new( + self.layer_facts = LayerFacts.init_new( layer_id=layer_id, num_layers=config.num_hidden_layers, is_layer_sparse=False, @@ -1019,7 +1019,7 @@ class MossVLSelfAttentionDecoderLayer(nn.Module): is_next_layer_sparse=False, ) self.layer_communicator = LayerCommunicator( - layer_scatter_modes=self.layer_scatter_modes, + layer_facts=self.layer_facts, input_layernorm=self.input_layernorm, post_attention_layernorm=self.post_attention_layernorm, allow_deferred_ffn_reduction=False, diff --git a/python/sglang/srt/models/nemotron_h_utils.py b/python/sglang/srt/models/nemotron_h_utils.py index 6c36aee4ec..b2270968d0 100644 --- a/python/sglang/srt/models/nemotron_h_utils.py +++ b/python/sglang/srt/models/nemotron_h_utils.py @@ -6,10 +6,9 @@ from sglang.srt.configs.nemotron_h import ATTENTION, MAMBA, MOE from sglang.srt.layers.communicator import ( NORM_QUANT_READ, LayerCommunicator, - LayerScatterModes, + LayerFacts, LayerStage, Layout, - ScatterMode, StageDecl, StageInput, StageKind, @@ -102,25 +101,6 @@ def layer_stage(pattern: str, layer_idx: int) -> LayerStage: ) -def _build_layer_scatter_modes( - is_sparse: bool = False, is_last_layer: bool = False -) -> LayerScatterModes: - scatter_mlp = is_sparse and not get_moe_a2a_backend().is_none() - mlp_mode = ScatterMode.SCATTERED if scatter_mlp else ScatterMode.FULL - middle_residual_mode = ( - ScatterMode.SCATTERED if scatter_mlp else ScatterMode.TP_ATTN_FULL - ) - return LayerScatterModes( - layer_input_mode=ScatterMode.TP_ATTN_FULL, - attn_mode=ScatterMode.TP_ATTN_FULL, - mlp_mode=mlp_mode, - middle_residual_mode=middle_residual_mode, - layer_output_mode=ScatterMode.TP_ATTN_FULL, - is_layer_sparse=is_sparse, - is_last_layer=is_last_layer, - ) - - def make_layer_communicator( layer_norm: RMSNorm, *, pattern: str, layer_idx: int ) -> LayerCommunicator: @@ -129,8 +109,9 @@ def make_layer_communicator( stage = layer_stage(pattern, layer_idx) for_attn = stage.kind is StageKind.ATTENTION return LayerCommunicator( - layer_scatter_modes=_build_layer_scatter_modes( - pattern[layer_idx] == MOE, is_last_layer=layer_idx == len(pattern) - 1 + layer_facts=LayerFacts( + is_layer_sparse=pattern[layer_idx] == MOE, + is_last_layer=layer_idx == len(pattern) - 1, ), input_layernorm=layer_norm if for_attn else None, post_attention_layernorm=None if for_attn else layer_norm, diff --git a/python/sglang/srt/models/qwen2_moe.py b/python/sglang/srt/models/qwen2_moe.py index f4875b8920..91a9d6e2f8 100644 --- a/python/sglang/srt/models/qwen2_moe.py +++ b/python/sglang/srt/models/qwen2_moe.py @@ -42,7 +42,7 @@ from sglang.srt.eplb.expert_location_dispatch import ExpertLocationDispatchInfo from sglang.srt.layers.activation import SiluAndMul from sglang.srt.layers.communicator import ( LayerCommunicator, - LayerScatterModes, + LayerFacts, ) from sglang.srt.layers.dp_attention import ( is_dp_attention_enabled, @@ -1025,7 +1025,7 @@ class Qwen2MoeDecoderLayer(nn.Module): is_previous_layer_sparse = True is_next_layer_sparse = True - self.layer_scatter_modes = LayerScatterModes.init_new( + self.layer_facts = LayerFacts.init_new( layer_id=layer_id, num_layers=config.num_hidden_layers, is_layer_sparse=self.is_layer_sparse, @@ -1054,7 +1054,7 @@ class Qwen2MoeDecoderLayer(nn.Module): config.hidden_size, eps=config.rms_norm_eps ) self.layer_communicator = LayerCommunicator( - layer_scatter_modes=self.layer_scatter_modes, + layer_facts=self.layer_facts, input_layernorm=self.input_layernorm, post_attention_layernorm=self.post_attention_layernorm, allow_reduce_scatter=True, diff --git a/python/sglang/srt/models/qwen3.py b/python/sglang/srt/models/qwen3.py index d9f9ca1dc0..88371e9138 100644 --- a/python/sglang/srt/models/qwen3.py +++ b/python/sglang/srt/models/qwen3.py @@ -5,7 +5,7 @@ from typing import Any, Dict, Iterable, List, Optional, Tuple import torch from torch import nn -from sglang.srt.layers.communicator import LayerCommunicator, LayerScatterModes +from sglang.srt.layers.communicator import LayerCommunicator, LayerFacts from sglang.srt.layers.layernorm import RMSNorm from sglang.srt.layers.linear import QKVParallelLinear, RowParallelLinear from sglang.srt.layers.logits_processor import LogitsProcessor @@ -370,7 +370,7 @@ class Qwen3DecoderLayer(nn.Module): config.hidden_size, eps=config.rms_norm_eps, **norm_kwargs ) - self.layer_scatter_modes = LayerScatterModes.init_new( + self.layer_facts = LayerFacts.init_new( layer_id=layer_id, num_layers=config.num_hidden_layers, is_layer_sparse=False, @@ -378,7 +378,7 @@ class Qwen3DecoderLayer(nn.Module): is_next_layer_sparse=False, ) self.layer_communicator = LayerCommunicator( - layer_scatter_modes=self.layer_scatter_modes, + layer_facts=self.layer_facts, input_layernorm=self.input_layernorm, post_attention_layernorm=self.post_attention_layernorm, allow_deferred_ffn_reduction=False, diff --git a/python/sglang/srt/models/qwen3_5.py b/python/sglang/srt/models/qwen3_5.py index 92a6211753..67d347cbff 100644 --- a/python/sglang/srt/models/qwen3_5.py +++ b/python/sglang/srt/models/qwen3_5.py @@ -45,7 +45,7 @@ from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation from sglang.srt.layers.attention.mamba.mamba import mamba_v2_sharded_weight_loader from sglang.srt.layers.communicator import ( LayerCommunicator, - LayerScatterModes, + LayerFacts, reduce_output, ) from sglang.srt.layers.dp_attention import ( @@ -1104,7 +1104,7 @@ class Qwen3_5LinearDecoderLayer(nn.Module): else: raise ValueError(f"Invalid model type: {config.model_type}") - self.layer_scatter_modes = LayerScatterModes.init_new( + self.layer_facts = LayerFacts.init_new( layer_id=layer_id, num_layers=config.num_hidden_layers, is_layer_sparse=is_layer_sparse, @@ -1124,7 +1124,7 @@ class Qwen3_5LinearDecoderLayer(nn.Module): and _linear_accepts_fp8_tuple(self.linear_attn.in_proj_qkvz) ) self.layer_communicator = LayerCommunicator( - layer_scatter_modes=self.layer_scatter_modes, + layer_facts=self.layer_facts, input_layernorm=self.input_layernorm, post_attention_layernorm=self.post_attention_layernorm, allow_reduce_scatter=True, @@ -1306,7 +1306,7 @@ class Qwen3_5AttentionDecoderLayer(nn.Module): else: raise ValueError(f"Invalid model type: {config.model_type}") - self.layer_scatter_modes = LayerScatterModes.init_new( + self.layer_facts = LayerFacts.init_new( layer_id=layer_id, num_layers=config.num_hidden_layers, is_layer_sparse=is_layer_sparse, @@ -1328,7 +1328,7 @@ class Qwen3_5AttentionDecoderLayer(nn.Module): _enable_qwen35_fused_ar_quant() and _linear_accepts_fp8_tuple(self.qkv_proj) ) self.layer_communicator = LayerCommunicator( - layer_scatter_modes=self.layer_scatter_modes, + layer_facts=self.layer_facts, input_layernorm=self.input_layernorm, post_attention_layernorm=self.post_attention_layernorm, allow_reduce_scatter=True, diff --git a/python/sglang/srt/models/qwen3_moe.py b/python/sglang/srt/models/qwen3_moe.py index 481808bc6c..f8d268b353 100644 --- a/python/sglang/srt/models/qwen3_moe.py +++ b/python/sglang/srt/models/qwen3_moe.py @@ -29,7 +29,7 @@ from transformers import PretrainedConfig from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation from sglang.srt.eplb.expert_location_dispatch import ExpertLocationDispatchInfo -from sglang.srt.layers.communicator import LayerCommunicator, LayerScatterModes +from sglang.srt.layers.communicator import LayerCommunicator, LayerFacts from sglang.srt.layers.layernorm import RMSNorm from sglang.srt.layers.linear import ( QKVParallelLinear, @@ -840,7 +840,7 @@ class Qwen3MoeDecoderLayer(nn.Module): is_previous_layer_sparse = True is_next_layer_sparse = True - self.layer_scatter_modes = LayerScatterModes.init_new( + self.layer_facts = LayerFacts.init_new( layer_id=layer_id, num_layers=config.num_hidden_layers, is_layer_sparse=self.is_layer_sparse, @@ -869,7 +869,7 @@ class Qwen3MoeDecoderLayer(nn.Module): ) self.layer_communicator = LayerCommunicator( - layer_scatter_modes=self.layer_scatter_modes, + layer_facts=self.layer_facts, input_layernorm=self.input_layernorm, post_attention_layernorm=self.post_attention_layernorm, allow_reduce_scatter=True, diff --git a/python/sglang/srt/models/qwen3_next.py b/python/sglang/srt/models/qwen3_next.py index cdc3b4778a..0806421f2b 100644 --- a/python/sglang/srt/models/qwen3_next.py +++ b/python/sglang/srt/models/qwen3_next.py @@ -15,7 +15,7 @@ from sglang.srt.configs.qwen3_next import Qwen3NextConfig from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation from sglang.srt.layers.attention.mamba.mamba import mamba_v2_sharded_weight_loader -from sglang.srt.layers.communicator import LayerCommunicator, LayerScatterModes +from sglang.srt.layers.communicator import LayerCommunicator, LayerFacts from sglang.srt.layers.dp_attention import ( is_dp_attention_enabled, ) @@ -507,7 +507,7 @@ class Qwen3HybridLinearDecoderLayer(nn.Module): is_next_layer_sparse = True self.layer_id = layer_id - self.layer_scatter_modes = LayerScatterModes.init_new( + self.layer_facts = LayerFacts.init_new( layer_id=layer_id, num_layers=config.num_hidden_layers, is_layer_sparse=self.is_layer_sparse, @@ -539,7 +539,7 @@ class Qwen3HybridLinearDecoderLayer(nn.Module): config.hidden_size, eps=config.rms_norm_eps ) self.layer_communicator = LayerCommunicator( - layer_scatter_modes=self.layer_scatter_modes, + layer_facts=self.layer_facts, input_layernorm=self.input_layernorm, post_attention_layernorm=self.post_attention_layernorm, allow_reduce_scatter=True, @@ -675,7 +675,7 @@ class Qwen3HybridAttentionDecoderLayer(nn.Module): is_previous_layer_sparse = True is_next_layer_sparse = True - self.layer_scatter_modes = LayerScatterModes.init_new( + self.layer_facts = LayerFacts.init_new( layer_id=layer_id, num_layers=config.num_hidden_layers, is_layer_sparse=self.is_layer_sparse, @@ -711,7 +711,7 @@ class Qwen3HybridAttentionDecoderLayer(nn.Module): self.k_norm = GemmaRMSNorm(self.head_dim, eps=config.rms_norm_eps) self.layer_communicator = LayerCommunicator( - layer_scatter_modes=self.layer_scatter_modes, + layer_facts=self.layer_facts, input_layernorm=self.input_layernorm, post_attention_layernorm=self.post_attention_layernorm, allow_reduce_scatter=True, diff --git a/python/sglang/srt/models/sarvam_moe.py b/python/sglang/srt/models/sarvam_moe.py index 20cc72335d..d5b9fbc81a 100644 --- a/python/sglang/srt/models/sarvam_moe.py +++ b/python/sglang/srt/models/sarvam_moe.py @@ -21,7 +21,7 @@ from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation from sglang.srt.layers.activation import SiluAndMul from sglang.srt.layers.communicator import ( LayerCommunicator, - LayerScatterModes, + LayerFacts, enable_moe_dense_fully_dp, ) from sglang.srt.layers.dp_attention import ( @@ -1047,7 +1047,7 @@ class SarvamMoEMLADecoderLayer(nn.Module): ) self.attn_tp_size = get_parallel().attn_tp_size - self.layer_scatter_modes = LayerScatterModes.init_new( + self.layer_facts = LayerFacts.init_new( layer_id=layer_id, num_layers=config.num_hidden_layers, is_layer_sparse=self.is_layer_sparse, @@ -1055,7 +1055,7 @@ class SarvamMoEMLADecoderLayer(nn.Module): is_next_layer_sparse=is_next_layer_sparse, ) self.layer_communicator = LayerCommunicator( - layer_scatter_modes=self.layer_scatter_modes, + layer_facts=self.layer_facts, input_layernorm=self.input_layernorm, post_attention_layernorm=self.post_attention_layernorm, qkv_latent_func=self.self_attn.prepare_qkv_latent, diff --git a/python/sglang/srt/models/sdar.py b/python/sglang/srt/models/sdar.py index 49e3744514..9513062360 100644 --- a/python/sglang/srt/models/sdar.py +++ b/python/sglang/srt/models/sdar.py @@ -11,7 +11,7 @@ from torch import nn from transformers import PretrainedConfig from sglang.srt.layers.activation import SiluAndMul -from sglang.srt.layers.communicator import LayerCommunicator, LayerScatterModes +from sglang.srt.layers.communicator import LayerCommunicator, LayerFacts from sglang.srt.layers.dp_attention import ( is_dp_attention_enabled, ) @@ -290,7 +290,7 @@ class SDARBlock(nn.Module): prefix=add_prefix("mlp", prefix), ) - self.layer_scatter_modes = LayerScatterModes.init_new( + self.layer_facts = LayerFacts.init_new( layer_id=layer_id, num_layers=config.num_hidden_layers, is_layer_sparse=False, @@ -298,7 +298,7 @@ class SDARBlock(nn.Module): is_next_layer_sparse=False, ) self.layer_communicator = LayerCommunicator( - layer_scatter_modes=self.layer_scatter_modes, + layer_facts=self.layer_facts, input_layernorm=self.input_layernorm, post_attention_layernorm=self.post_attention_layernorm, allow_reduce_scatter=True, diff --git a/python/sglang/srt/models/sdar_moe.py b/python/sglang/srt/models/sdar_moe.py index d8043cb8f0..aab0af6905 100644 --- a/python/sglang/srt/models/sdar_moe.py +++ b/python/sglang/srt/models/sdar_moe.py @@ -15,7 +15,7 @@ from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation from sglang.srt.eplb.expert_location_dispatch import ExpertLocationDispatchInfo from sglang.srt.layers.communicator import ( LayerCommunicator, - LayerScatterModes, + LayerFacts, ) from sglang.srt.layers.dp_attention import ( is_dp_attention_enabled, @@ -355,7 +355,7 @@ class SDARMoeBlock(nn.Module): prefix=add_prefix("mlp", prefix), ) - self.layer_scatter_modes = LayerScatterModes.init_new( + self.layer_facts = LayerFacts.init_new( layer_id=layer_id, num_layers=config.num_hidden_layers, is_layer_sparse=True, @@ -364,7 +364,7 @@ class SDARMoeBlock(nn.Module): ) self.layer_communicator = LayerCommunicator( - layer_scatter_modes=self.layer_scatter_modes, + layer_facts=self.layer_facts, input_layernorm=self.input_layernorm, post_attention_layernorm=self.post_attention_layernorm, allow_reduce_scatter=True, diff --git a/python/sglang/srt/models/step3_vl.py b/python/sglang/srt/models/step3_vl.py index 36e41f1a94..d0645e47c7 100644 --- a/python/sglang/srt/models/step3_vl.py +++ b/python/sglang/srt/models/step3_vl.py @@ -22,7 +22,7 @@ from sglang.srt.distributed import ( from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation from sglang.srt.layers.activation import SiluAndMul from sglang.srt.layers.attention.vision import VisionAttention -from sglang.srt.layers.communicator import LayerCommunicator, LayerScatterModes +from sglang.srt.layers.communicator import LayerCommunicator, LayerFacts from sglang.srt.layers.conv import Conv2dLayer from sglang.srt.layers.dp_attention import ( is_dp_attention_enabled, @@ -338,7 +338,7 @@ class Step3TextDecoderLayer(nn.Module): ) self.is_next_layer_sparse = True if layer_id + 1 in moe_layers_idx else False - self.layer_scatter_modes = LayerScatterModes.init_new( + self.layer_facts = LayerFacts.init_new( layer_id=layer_id, num_layers=config.num_hidden_layers, is_layer_sparse=self.is_layer_sparse, @@ -379,7 +379,7 @@ class Step3TextDecoderLayer(nn.Module): ) self.layer_communicator = LayerCommunicator( - layer_scatter_modes=self.layer_scatter_modes, + layer_facts=self.layer_facts, input_layernorm=self.input_layernorm, post_attention_layernorm=self.post_attention_layernorm, allow_deferred_ffn_reduction=False, diff --git a/python/sglang/srt/models/step3p5.py b/python/sglang/srt/models/step3p5.py index 83e2b23b06..272d418bb6 100644 --- a/python/sglang/srt/models/step3p5.py +++ b/python/sglang/srt/models/step3p5.py @@ -11,7 +11,7 @@ from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_r from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation from sglang.srt.eplb.expert_location_dispatch import ExpertLocationDispatchInfo from sglang.srt.layers.activation import SiluAndMul -from sglang.srt.layers.communicator import LayerCommunicator, LayerScatterModes +from sglang.srt.layers.communicator import LayerCommunicator, LayerFacts from sglang.srt.layers.dp_attention import ( is_dp_attention_enabled, ) @@ -560,7 +560,7 @@ class Step3p5DecoderLayer(nn.Module): ) # An MTP draft is a one-layer model; layer_id still indexes its config. - self.layer_scatter_modes = LayerScatterModes.init_new( + self.layer_facts = LayerFacts.init_new( layer_id=0 if is_nextn else layer_id, num_layers=1 if is_nextn else config.num_hidden_layers, is_layer_sparse=self.is_moe_layer, @@ -568,7 +568,7 @@ class Step3p5DecoderLayer(nn.Module): is_next_layer_sparse=self.is_next_layer_sparse, ) self.layer_communicator = LayerCommunicator( - layer_scatter_modes=self.layer_scatter_modes, + layer_facts=self.layer_facts, input_layernorm=self.input_layernorm, post_attention_layernorm=self.post_attention_layernorm, allow_reduce_scatter=True, diff --git a/python/sglang/srt/models/xllm.py b/python/sglang/srt/models/xllm.py index f29c623a2b..44975120c3 100644 --- a/python/sglang/srt/models/xllm.py +++ b/python/sglang/srt/models/xllm.py @@ -41,7 +41,7 @@ from sglang.srt.eplb.expert_location_dispatch import ExpertLocationDispatchInfo from sglang.srt.layers.activation import SiluAndMul from sglang.srt.layers.communicator import ( LayerCommunicator, - LayerScatterModes, + LayerFacts, enable_moe_dense_fully_dp, ) from sglang.srt.layers.dp_attention import is_dp_attention_enabled @@ -1505,7 +1505,7 @@ class XllmDecoderLayer(nn.Module): is_previous_layer_sparse = _is_sparse(layer_id - 1) is_next_layer_sparse = _is_sparse(layer_id + 1) - self.layer_scatter_modes = LayerScatterModes.init_new( + self.layer_facts = LayerFacts.init_new( layer_id=layer_id, num_layers=config.num_hidden_layers, is_layer_sparse=self.is_layer_sparse, @@ -1538,7 +1538,7 @@ class XllmDecoderLayer(nn.Module): self.input_layernorm = _make_norm(config) self.post_attention_layernorm = _make_norm(config) self.layer_communicator = LayerCommunicator( - layer_scatter_modes=self.layer_scatter_modes, + layer_facts=self.layer_facts, input_layernorm=self.input_layernorm, post_attention_layernorm=self.post_attention_layernorm, allow_reduce_scatter=True, diff --git a/test/registered/ops/test_aiter_allreduce_fusion_amd.py b/test/registered/ops/test_aiter_allreduce_fusion_amd.py index e5bb9121ed..ccddf7aecd 100755 --- a/test/registered/ops/test_aiter_allreduce_fusion_amd.py +++ b/test/registered/ops/test_aiter_allreduce_fusion_amd.py @@ -10,7 +10,7 @@ from pathlib import Path import torch -from sglang.srt.layers.communicator import LayerCommunicator, ScatterMode +from sglang.srt.layers.communicator import LayerCommunicator from sglang.srt.model_executor.forward_batch_info import ForwardMode from sglang.test.ci.ci_register import register_amd_ci from sglang.test.communicator_patch import patch_communicator @@ -348,11 +348,10 @@ class TestAiterAllreduceFusionAmd(unittest.TestCase): ) -def _fake_self(*, mlp_mode=ScatterMode.TP_ATTN_FULL, is_last_layer=False, tp_size=8): +def _fake_self(*, is_last_layer=False, tp_size=8): """Minimal stand-in for a LayerCommunicator with the fields the gate reads.""" return types.SimpleNamespace( _speculative_algo=None, - layer_scatter_modes=types.SimpleNamespace(mlp_mode=mlp_mode), is_last_layer=is_last_layer, _context=types.SimpleNamespace(tp_size=tp_size), ) @@ -387,7 +386,6 @@ class TestAiterAllreduceFusionGate(CustomTestCase): aiter_enabled=True, use_aiter=True, tp_world_size=8, - mlp_mode=ScatterMode.TP_ATTN_FULL, is_last_layer=False, tp_size=8, forward_mode=ForwardMode.DECODE, @@ -440,9 +438,7 @@ class TestAiterAllreduceFusionGate(CustomTestCase): patch_communicator("get_moe_a2a_backend", lambda: a2a_backend) ) - fake_self = _fake_self( - mlp_mode=mlp_mode, is_last_layer=is_last_layer, tp_size=tp_size - ) + fake_self = _fake_self(is_last_layer=is_last_layer, tp_size=tp_size) return LayerCommunicator.should_fuse_mlp_allreduce_with_next_layer( fake_self, _fake_forward_batch(forward_mode=forward_mode) ) diff --git a/test/registered/unit/layers/test_declared_attention_cp.py b/test/registered/unit/layers/test_declared_attention_cp.py index cb71fb34c2..cf245d6e08 100644 --- a/test/registered/unit/layers/test_declared_attention_cp.py +++ b/test/registered/unit/layers/test_declared_attention_cp.py @@ -119,10 +119,6 @@ class TestAttentionCpBoundary(CustomTestCase): (comm, "get_moe_a2a_backend"), lambda: SimpleNamespace(is_none=lambda: True), ), - ( - (comm, "should_use_flashinfer_cutlass_moe_fp4_allgather"), - lambda: False, - ), ( (comm, "post_experts_reduction_group"), lambda: moe_group or parallel.tp_group, @@ -165,7 +161,7 @@ class TestAttentionCpBoundary(CustomTestCase): def build(self, allow_reduce_scatter): # A MoE layer after a dense one; dense layers run on every rank here. return comm.LayerCommunicator( - layer_scatter_modes=SimpleNamespace( + layer_facts=SimpleNamespace( is_first_layer=False, is_last_layer=False, is_layer_sparse=True, diff --git a/test/registered/unit/layers/test_declared_decoder_boundary.py b/test/registered/unit/layers/test_declared_decoder_boundary.py index c3261523fa..65daed5c8b 100644 --- a/test/registered/unit/layers/test_declared_decoder_boundary.py +++ b/test/registered/unit/layers/test_declared_decoder_boundary.py @@ -5,8 +5,8 @@ rows (a MoE dispatched per DP shard). The numeric checks run every rank of a DP x attention-TP world as a thread over fake collectives that wait for all members of their group, build the layers' -communicators for real, and pass them scatter modes that fail on any read of a -mode field. +communicators for real, and pass them layer facts that fail on any read of +something else. """ import itertools @@ -24,9 +24,8 @@ from sglang.srt.layers import communicator as comm from sglang.srt.layers import layernorm_sp from sglang.srt.layers.communicator import ( LayerCommunicator, - LayerScatterModes, + LayerFacts, MHCLayerCommunicator, - ScatterMode, SumGroup, TokenAxis, UnreducedOutput, @@ -109,9 +108,6 @@ def planning(parallel, *, sp=False, a2a=False, dsa_cp=False): lambda: SimpleNamespace(is_none=lambda: not a2a), ), patch_communicator("is_moe_input_scattered_across_dp_ranks", lambda: a2a), - patch_communicator( - "should_use_flashinfer_cutlass_moe_fp4_allgather", lambda: False - ), patch_communicator("is_enable_moe_cp_allgather", moe_cp_gathers), patch_communicator("get_lora", lambda: SimpleNamespace(enable_lora=False)), # A MoE whose EP and TP sums merge: its output's group is the TP group. @@ -148,14 +144,14 @@ class FusableNorm(Norm): class FactsOnly: - """Scatter modes whose mode fields fail when read: only the layer facts - are there.""" + """Layer facts that fail on any read of something else: construction reads + only the layer facts.""" def __init__(self, **facts): self.__dict__.update(facts) def __getattr__(self, name): - raise AssertionError(f"read scatter mode {name!r}") + raise AssertionError(f"read {name!r}, which is not a layer fact") def layer_facts( @@ -194,7 +190,7 @@ def sides_of( def build( - modes, + facts, parallel, *, cls=LayerCommunicator, @@ -205,14 +201,14 @@ def build( ): with planning(parallel, sp=sp, a2a=a2a, dsa_cp=dsa_cp): return cls( - layer_scatter_modes=modes, + layer_facts=facts, input_layernorm=Norm(), post_attention_layernorm=Norm(), **kwargs, ) -def planned_modes( +def planned_facts( layer_id, num_layers, *, @@ -223,7 +219,7 @@ def planned_modes( dsa_cp=False, ): with planning(parallel, a2a=a2a, dsa_cp=dsa_cp): - return LayerScatterModes.init_new( + return LayerFacts.init_new( layer_id=layer_id, num_layers=num_layers, is_layer_sparse=sparse, @@ -260,7 +256,7 @@ class TestWhichLayersUseDeclarations(CustomTestCase): with self.subTest(norm=norm.__name__): with planning(parallel_of(attn_dp=1, attn_tp=2)): communicator = LayerCommunicator( - layer_scatter_modes=layer_facts(1, 3), + layer_facts=layer_facts(1, 3), input_layernorm=norm(), post_attention_layernorm=norm(), ) @@ -319,7 +315,7 @@ class TestWhichLayersUseDeclarations(CustomTestCase): with self.subTest(a2a=a2a): layers = [ build( - planned_modes( + planned_facts( i, 4, sparse=sparse, @@ -388,7 +384,7 @@ class TestWhichLayersUseDeclarations(CustomTestCase): ): layers = [ build( - planned_modes( + planned_facts( i, 4, sparse=sparse, @@ -420,7 +416,7 @@ class TestWhichLayersUseDeclarations(CustomTestCase): def test_the_last_a2a_layer_folds_the_residual_back(self): parallel = parallel_of(attn_dp=2, attn_tp=2) last = build( - planned_modes( + planned_facts( 3, 4, sparse=True, previous_sparse=True, parallel=parallel, a2a=True ), parallel, @@ -433,11 +429,11 @@ class TestWhichLayersUseDeclarations(CustomTestCase): def test_an_attention_that_gathers_its_slice_itself(self): parallel = parallel_of(attn_dp=2, attn_tp=2) - modes = planned_modes( + facts = planned_facts( 2, 4, sparse=True, previous_sparse=True, parallel=parallel, a2a=True ) with patch_communicator("_use_ag_after_qlora", True): - layer = build(modes, parallel, a2a=True) + layer = build(facts, parallel, a2a=True) self.assertIsNone(layer._steps.attention.input_move) def test_cp_without_prefill_cp(self): @@ -461,19 +457,13 @@ class TestWhichLayersUseDeclarations(CustomTestCase): self.assertIsNotNone(without_dp._input_scattered_steps) def test_a_layer_without_the_previous_layer_s_facts_is_refused(self): - direct = LayerScatterModes( - layer_input_mode=ScatterMode.TP_ATTN_FULL, - attn_mode=ScatterMode.TP_ATTN_FULL, - mlp_mode=ScatterMode.FULL, - middle_residual_mode=ScatterMode.TP_ATTN_FULL, - layer_output_mode=ScatterMode.TP_ATTN_FULL, - ) + direct = LayerFacts() with self.assertRaises(NotImplementedError): build(direct, parallel_of(attn_dp=2, attn_tp=2)) def build_mhc( - modes, parallel, *, a2a=False, dsa_cp=False, two_batch_overlap=False, **kwargs + facts, parallel, *, a2a=False, dsa_cp=False, two_batch_overlap=False, **kwargs ): overlap = SimpleNamespace( overlap=SimpleNamespace(enable_two_batch_overlap=two_batch_overlap) @@ -483,7 +473,7 @@ def build_mhc( patch_communicator("get_exec", lambda: overlap), ): return MHCLayerCommunicator( - layer_scatter_modes=modes, + layer_facts=facts, input_layernorm=Norm(), post_attention_layernorm=Norm(), hc_mult=2, @@ -564,7 +554,10 @@ class TestMhcOnTheDeclarations(CustomTestCase): patch_communicator("should_use_dp_reduce_scatterv", lambda: False), patch_communicator("can_use_dp_reduce_scatter", lambda: True), ): - self.assertTrue(communicator.should_use_reduce_scatter(max_len)) + step = communicator._postprocess_dp_step(max_len) + self.assertTrue( + communicator._ffn_leaves_sum_to_reduce_scatter(max_len, step) + ) def test_the_exit_runs_the_move_back_over_dp_it_chose(self): parallel = parallel_of(attn_dp=2, attn_tp=2) @@ -614,7 +607,7 @@ class TestMhcOnTheDeclarations(CustomTestCase): parallel = parallel_of(attn_dp=2, attn_tp=2) layers = [ build_mhc( - planned_modes( + planned_facts( i, 3, sparse=sparse, @@ -751,30 +744,30 @@ class TestMhcOnTheDeclarations(CustomTestCase): parallel = parallel_of( attn_dp=1, attn_tp=1, attn_cp=2, enable_prefill_cp=True, moe_dense_tp_size=1 ) - modes = planned_modes( + facts = planned_facts( 1, 3, sparse=True, previous_sparse=False, parallel=parallel, dsa_cp=True ) with self.assertRaises(NotImplementedError): - build_mhc(modes, parallel, dsa_cp=True, allow_reduce_scatter=True) + build_mhc(facts, parallel, dsa_cp=True, allow_reduce_scatter=True) # A dense layer on every rank computes on its own shard: nothing moves. - modes = planned_modes( + facts = planned_facts( 1, 3, sparse=False, previous_sparse=False, parallel=parallel, dsa_cp=True ) - build_mhc(modes, parallel, dsa_cp=True, allow_reduce_scatter=True) + build_mhc(facts, parallel, dsa_cp=True, allow_reduce_scatter=True) def test_input_scattered_attention_under_attention_cp_is_rejected(self): scattered = dict(enable_attn_tp_input_scattered=True) parallel = parallel_of(attn_dp=1, attn_tp=2, attn_cp=2, **scattered) with self.assertRaisesRegex(NotImplementedError, "input-scattered"): build_mhc( - planned_modes( + planned_facts( 1, 3, sparse=False, previous_sparse=False, parallel=parallel ), parallel, ) parallel = parallel_of(attn_dp=1, attn_tp=2, **scattered) build_mhc( - planned_modes(1, 3, sparse=False, previous_sparse=False, parallel=parallel), + planned_facts(1, 3, sparse=False, previous_sparse=False, parallel=parallel), parallel, ) @@ -785,22 +778,22 @@ class TestMhcOnTheDeclarations(CustomTestCase): parallel = parallel_of( attn_dp=1, attn_tp=2, attn_cp=2, enable_prefill_cp=prefill_cp ) - modes = planned_modes( + facts = planned_facts( 1, 3, sparse=True, previous_sparse=True, parallel=parallel ) with self.assertRaisesRegex(NotImplementedError, "MoE-CP group"): - build_mhc(modes, parallel) + build_mhc(facts, parallel) # A dense layer builds, and so does a MoE on its own CP shard (MoE DP = CP). parallel = parallel_of(attn_dp=1, attn_tp=2, attn_cp=2) build_mhc( - planned_modes(1, 3, sparse=False, previous_sparse=False, parallel=parallel), + planned_facts(1, 3, sparse=False, previous_sparse=False, parallel=parallel), parallel, ) parallel = parallel_of( attn_dp=1, attn_tp=2, attn_cp=2, moe_dp_size=2, moe_tp_size=2 ) build_mhc( - planned_modes(1, 3, sparse=True, previous_sparse=True, parallel=parallel), + planned_facts(1, 3, sparse=True, previous_sparse=True, parallel=parallel), parallel, ) @@ -816,7 +809,7 @@ class TestTwoBatchOverlap(CustomTestCase): layers = [] with planning(parallel), patch_communicator("get_exec", lambda: overlap): for i, sparse in enumerate((False, False, True, True)): - modes = LayerScatterModes.init_new( + facts = LayerFacts.init_new( layer_id=i, num_layers=4, is_layer_sparse=sparse, @@ -825,7 +818,7 @@ class TestTwoBatchOverlap(CustomTestCase): ) layers.append( LayerCommunicator( - layer_scatter_modes=modes, + layer_facts=facts, input_layernorm=Norm(), post_attention_layernorm=Norm(), ) @@ -869,7 +862,7 @@ class TestTwoBatchOverlap(CustomTestCase): with self.assertRaises(NotImplementedError): comm.tbo_split_moves(comm.Layout(frozenset())) - def build(self, modes, parallel, *, a2a, two_batch_overlap): + def build(self, facts, parallel, *, a2a, two_batch_overlap): overlap = SimpleNamespace( overlap=SimpleNamespace(enable_two_batch_overlap=two_batch_overlap) ) @@ -878,7 +871,7 @@ class TestTwoBatchOverlap(CustomTestCase): patch_communicator("get_exec", lambda: overlap), ): communicator = LayerCommunicator( - layer_scatter_modes=modes, + layer_facts=facts, input_layernorm=Norm(), post_attention_layernorm=Norm(), ) @@ -922,7 +915,7 @@ class TestTwoBatchOverlap(CustomTestCase): attn_tp=attn_tp, moe_dense_tp_size=1 if dense_fully_dp else None, ) - modes = planned_modes( + facts = planned_facts( layer_id, 4, sparse=sparse, @@ -931,7 +924,7 @@ class TestTwoBatchOverlap(CustomTestCase): a2a=a2a, ) communicator = self.build( - modes, + facts, parallel, a2a=a2a, two_batch_overlap=two_batch_overlap, @@ -947,14 +940,16 @@ class TestTwoBatchOverlap(CustomTestCase): ): communicator.publish_mlp_lora_layout(steps) # LoRA admits attention DP only with attention TP 1, where the - # gather over attention DP is the FULL mode. + # FFN takes every row unless it runs on each rank's own: a MoE + # dispatched by an a2a backend, or a dense MLP on every rank. + on_local_rows = a2a if sparse else dense_fully_dp if attn_dp > 1 and attn_tp == 1: self.assertIs( published["lora_batch_layout"], ( - LoRABatchLayout.TP_GLOBAL - if modes.mlp_mode is ScatterMode.FULL - else LoRABatchLayout.DP_LOCAL + LoRABatchLayout.DP_LOCAL + if on_local_rows + else LoRABatchLayout.TP_GLOBAL ), ) @@ -1155,7 +1150,7 @@ class TestFusedKernelsTakeOnlyTheStepsTheyComplete(CustomTestCase): with planning(parallel_of(attn_dp=1, attn_tp=2)): communicator = Declaring( - layer_scatter_modes=layer_facts(1, 3), + layer_facts=layer_facts(1, 3), input_layernorm=Norm(), post_attention_layernorm=Norm(), ) @@ -1241,7 +1236,7 @@ class TestTheSequenceParallelRegion(CustomTestCase): with self.subTest(layer_id=layer_id): with planning(parallel, sp=True): communicator = LayerCommunicator( - layer_scatter_modes=layer_facts(layer_id, 3), + layer_facts=layer_facts(layer_id, 3), input_layernorm=Norm(), post_attention_layernorm=Norm(), ) @@ -1275,7 +1270,7 @@ class TestTheSequenceParallelRegion(CustomTestCase): def test_without_sp_there_is_no_region(self): with planning(parallel_of(attn_dp=1, attn_tp=2)): communicator = LayerCommunicator( - layer_scatter_modes=layer_facts(1, 3), + layer_facts=layer_facts(1, 3), input_layernorm=Norm(), post_attention_layernorm=Norm(), ) @@ -1509,10 +1504,10 @@ class TestTheAttentionInputHalf(CustomTestCase): the layer chose, and starts the residual only on the stack's first layer.""" - def build_fusable(self, modes, parallel, **planning_kwargs): + def build_fusable(self, facts, parallel, **planning_kwargs): with planning(parallel, **planning_kwargs): return LayerCommunicator( - layer_scatter_modes=modes, + layer_facts=facts, input_layernorm=FusableNorm(), post_attention_layernorm=Norm(), ) @@ -1530,14 +1525,14 @@ class TestTheAttentionInputHalf(CustomTestCase): ): for layer_id in (0, 1): with self.subTest(name, layer_id=layer_id): - modes = planned_modes( + facts = planned_facts( layer_id, 3, sparse=False, previous_sparse=False, parallel=parallel, ) - communicator = self.build_fusable(modes, parallel, **kwargs) + communicator = self.build_fusable(facts, parallel, **kwargs) fusions = communicator._attn_input_fusions self.assertTrue(fusions) variants = [ @@ -1582,10 +1577,10 @@ class TestPrefillCP(CustomTestCase): def test_a_dsa_cp_extend_leaves_its_sum_to_the_reduce_scatter(self): # A MoE on the TP group, as under DSA interleave CP without a2a. parallel = self.dsa_parallel() - modes = planned_modes( + facts = planned_facts( 1, 3, sparse=True, previous_sparse=False, parallel=parallel, dsa_cp=True ) - communicator = build(modes, parallel, dsa_cp=True, allow_reduce_scatter=True) + communicator = build(facts, parallel, dsa_cp=True, allow_reduce_scatter=True) cp, ordinary = communicator._cp_steps, communicator._steps self.assertIs( cp.ffn.prepare.keywords["step"].func, @@ -1657,10 +1652,10 @@ class TestPrefillCP(CustomTestCase): def test_dsa_cp_dense_layers_run_on_their_shard(self): parallel = self.dsa_parallel() - modes = planned_modes( + facts = planned_facts( 1, 3, sparse=False, previous_sparse=False, parallel=parallel, dsa_cp=True ) - communicator = build(modes, parallel, dsa_cp=True, allow_reduce_scatter=True) + communicator = build(facts, parallel, dsa_cp=True, allow_reduce_scatter=True) for steps in (communicator._steps, communicator._cp_steps): self.assertIs(steps.ffn.prepare.keywords["step"].func, comm_ops._read_input) self.assertIsNone(steps.ffn_output.group) @@ -1722,7 +1717,7 @@ class TestPrefillCP(CustomTestCase): def test_the_fused_kernels_run_on_each_chunk(self): with planning(self.cp_parallel()): communicator = LayerCommunicator( - layer_scatter_modes=layer_facts(1, 3), + layer_facts=layer_facts(1, 3), input_layernorm=FusableNorm(), post_attention_layernorm=FusableNorm(), ) @@ -1735,7 +1730,7 @@ class TestPrefillCP(CustomTestCase): def cp_outcome(self, parallel, *, sparse=True, a2a=False, dsa_cp=False): """A layer under attention CP: "cp steps" for the batches that shard their tokens, "ordinary" when no batch does, or "refused".""" - modes = planned_modes( + facts = planned_facts( 1, 3, sparse=sparse, @@ -1746,7 +1741,7 @@ class TestPrefillCP(CustomTestCase): ) try: layer = build( - modes, parallel, a2a=a2a, dsa_cp=dsa_cp, allow_reduce_scatter=True + facts, parallel, a2a=a2a, dsa_cp=dsa_cp, allow_reduce_scatter=True ) except NotImplementedError: return "refused" @@ -2328,7 +2323,7 @@ class TestTwoLayers(CustomTestCase): s = states[rank] layers = [ LayerCommunicator( - layer_scatter_modes=layer_facts( + layer_facts=layer_facts( i, 2, sparse=sparse[i], diff --git a/test/registered/unit/layers/test_dense_mlp_under_prefill_cp.py b/test/registered/unit/layers/test_dense_mlp_under_prefill_cp.py new file mode 100644 index 0000000000..e5a7d452da --- /dev/null +++ b/test/registered/unit/layers/test_dense_mlp_under_prefill_cp.py @@ -0,0 +1,56 @@ +import unittest + +from sglang.srt.layers import communicator as comm +from sglang.srt.layers.communicator import ( + SumGroup, + TokenAxis, + decoder_layer_edges, + decoder_layer_sides, + make_boundary, + make_output_boundary, +) +from sglang.srt.layers.communicator import ops as comm_ops +from sglang.srt.layers.communicator.boundary import CpMoves +from sglang.test.ci.ci_register import register_cpu_ci +from sglang.test.test_utils import CustomTestCase + +register_cpu_ci(est_time=7, suite="base-a-test-cpu") + + +class TestDenseMlpUnderPrefillCP(CustomTestCase): + """A TP-sharded dense MLP under prefill CP must gather tokens across CP ranks + before its all-reduce, or CP pairs sum partial outputs of different tokens + (issue #38019: Qwen3-32B emitted garbage that never reached EOS).""" + + def test_dense_mlp_gathers_across_cp(self): + sides = decoder_layer_sides( + axis_sizes={ + TokenAxis.ATTN_DP: 1, + TokenAxis.ATTN_CP: 2, + TokenAxis.ATTN_TP_SCATTER: 2, + }, + ffn_on_local_rows=False, + previous_on_local_rows=False, + is_last_layer=False, + attention_gathers_local_rows=False, + ffn_group=SumGroup.TP, + leaves_for_next_layer=False, + leaves_for_reduce_scatter=False, + leaves_for_reduce_scatterv=False, + ) + # The TP group spans every CP rank, so the MLP takes every row. + self.assertIn(TokenAxis.ATTN_CP, sides.attention.layout.sharded) + self.assertNotIn(TokenAxis.ATTN_CP, sides.ffn.layout.sharded) + moves = CpMoves( + gather=comm_ops._mlp_input_gather_moe_cp, + take_back=comm.CommunicateSummableTensorPairFn._scatter_hidden_states_moe, + ) + edges = decoder_layer_edges(sides) + step = make_boundary(edges.into_ffn, cp_moves=moves).prepare.keywords["step"] + self.assertIs(step.func, comm_ops._mlp_input_gather_moe_cp) + out = make_output_boundary(edges.out_of_ffn, cp_moves=moves) + self.assertIs(out.output_move, moves.take_back) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/unit/layers/test_dp_attention_cp_gather.py b/test/registered/unit/layers/test_dp_attention_cp_gather.py index cfadbbacf0..c9cf5ebeb6 100644 --- a/test/registered/unit/layers/test_dp_attention_cp_gather.py +++ b/test/registered/unit/layers/test_dp_attention_cp_gather.py @@ -245,7 +245,7 @@ class TestDpCpGather(CustomTestCase): else patch.object(*target, value) ) communicator = comm.LayerCommunicator( - layer_scatter_modes=SimpleNamespace( + layer_facts=SimpleNamespace( is_first_layer=False, is_last_layer=False, is_layer_sparse=sparse, diff --git a/test/registered/unit/layers/test_layer_communicator_fusion_gate.py b/test/registered/unit/layers/test_layer_communicator_fusion_gate.py index 85ac55b872..24dff2b096 100644 --- a/test/registered/unit/layers/test_layer_communicator_fusion_gate.py +++ b/test/registered/unit/layers/test_layer_communicator_fusion_gate.py @@ -10,7 +10,6 @@ from sglang.srt.layers import communicator as comm from sglang.srt.layers.communicator import ( LayerCommunicator, Layout, - ScatterMode, StageOutput, SumGroup, TokenAxis, @@ -83,18 +82,11 @@ def _steps(*, ffn_output=None, returns_over_dp=False, ffn_sum_is_movable=True): ) -def _fake_communicator(mlp_mode=ScatterMode.TP_ATTN_FULL): +def _fake_communicator(ffn_sum_is_movable=True): communicator = LayerCommunicator.__new__(LayerCommunicator) communicator._speculative_algo = None - communicator.layer_scatter_modes = types.SimpleNamespace(mlp_mode=mlp_mode) - # Fixed at construction by the scatter-mode path from the MLP's mode. - communicator._steps = _steps( - ffn_sum_is_movable=mlp_mode - not in ( - ScatterMode.MOE_FULL, - ScatterMode.SCATTERED, - ) - ) + # An FFN gathered over MoE-CP or on each rank's own rows has no movable sum. + communicator._steps = _steps(ffn_sum_is_movable=ffn_sum_is_movable) communicator._sp_steps = None communicator._input_scattered_steps = None communicator._cp_steps = None @@ -320,7 +312,7 @@ class TestFuseMlpAllReduceGate(CustomTestCase): moe_ep_size, moe_tp_size, moe_dp_size=1, - mlp_mode=ScatterMode.TP_ATTN_FULL, + ffn_sum_is_movable=True, ): forward_batch = types.SimpleNamespace( input_ids=types.SimpleNamespace(shape=(8,)) @@ -340,7 +332,7 @@ class TestFuseMlpAllReduceGate(CustomTestCase): ), ): return LayerCommunicator.should_fuse_mlp_allreduce_with_next_layer( - _fake_communicator(mlp_mode), forward_batch + _fake_communicator(ffn_sum_is_movable), forward_batch ) def test_hybrid_ep_tp_fuses_when_mergeable(self): @@ -358,14 +350,12 @@ class TestFuseMlpAllReduceGate(CustomTestCase): def test_pure_ep_still_fuses(self): self.assertTrue(self._should_fuse(moe_ep_size=4, moe_tp_size=1)) - def test_moe_full_layer_does_not_fuse(self): + def test_a_layer_gathered_over_moe_cp_does_not_fuse(self): # Fusion skips postprocess_layer, which holds the CP scatter; a dense - # MOE_FULL layer (moe_dp_size == attn_cp_size) is not caught by the - # is_enable_moe_cp_allgather gate. + # layer gathered over the MoE-CP group (moe_dp_size == attn_cp_size) is + # not caught by the is_enable_moe_cp_allgather gate. self.assertFalse( - self._should_fuse( - moe_ep_size=1, moe_tp_size=4, mlp_mode=ScatterMode.MOE_FULL - ) + self._should_fuse(moe_ep_size=1, moe_tp_size=4, ffn_sum_is_movable=False) ) diff --git a/test/registered/unit/layers/test_layer_scatter_modes_cp_dense_mlp.py b/test/registered/unit/layers/test_layer_scatter_modes_cp_dense_mlp.py deleted file mode 100644 index f086b8a47c..0000000000 --- a/test/registered/unit/layers/test_layer_scatter_modes_cp_dense_mlp.py +++ /dev/null @@ -1,35 +0,0 @@ -import unittest - -from sglang.srt.layers.communicator import LayerScatterModes, ScatterMode -from sglang.test.ci.ci_register import register_cpu_ci -from sglang.test.communicator_patch import patch_communicator -from sglang.test.test_utils import CustomTestCase - -register_cpu_ci(est_time=7, suite="base-a-test-cpu") - - -class TestDenseMlpScatterModeUnderPrefillCP(CustomTestCase): - """A TP-sharded dense MLP under prefill CP must gather tokens across CP ranks - before its all-reduce, or CP pairs sum partial outputs of different tokens - (issue #38019: Qwen3-32B emitted garbage that never reached EOS).""" - - def test_dense_mlp_gathers_across_cp(self): - with ( - patch_communicator("_generic_prefill_cp_shards_tokens", return_value=True), - patch_communicator("is_dsa_enable_prefill_cp", return_value=False), - patch_communicator("is_mla_cp_enabled", return_value=False), - patch_communicator("enable_moe_dense_fully_dp", return_value=False), - ): - modes = LayerScatterModes.init_new( - layer_id=1, - num_layers=4, - is_layer_sparse=False, - is_previous_layer_sparse=False, - is_next_layer_sparse=False, - ) - self.assertEqual(modes.mlp_mode, ScatterMode.MOE_FULL) - self.assertEqual(modes.layer_output_mode, ScatterMode.TP_ATTN_FULL) - - -if __name__ == "__main__": - unittest.main() diff --git a/test/registered/unit/layers/test_layernorm_sp.py b/test/registered/unit/layers/test_layernorm_sp.py index 7546d4d547..b3c0af90f4 100644 --- a/test/registered/unit/layers/test_layernorm_sp.py +++ b/test/registered/unit/layers/test_layernorm_sp.py @@ -177,7 +177,7 @@ class TestSpRegionSteps(CustomTestCase): c._sp_steps = sp_region_steps() c._input_scattered_steps = None c._cp_steps = None - c.layer_scatter_modes = SimpleNamespace( + c.layer_facts = SimpleNamespace( is_first_layer=first_layer, is_layer_sparse=False ) c.allow_reduce_scatter = False diff --git a/test/registered/unit/layers/test_postprocess_reduce_scatterv.py b/test/registered/unit/layers/test_postprocess_reduce_scatterv.py index 7243375a46..d73d2eac3a 100644 --- a/test/registered/unit/layers/test_postprocess_reduce_scatterv.py +++ b/test/registered/unit/layers/test_postprocess_reduce_scatterv.py @@ -15,8 +15,7 @@ from sglang.srt.layers import communicator as comm from sglang.srt.layers.communicator import ( CommunicateSummableTensorPairFn, LayerCommunicator, - LayerScatterModes, - ScatterMode, + LayerFacts, ) from sglang.srt.layers.communicator import ops as comm_ops from sglang.srt.layers.moe import utils as moe_utils @@ -155,26 +154,17 @@ class TestPostprocessReduceScatterv(CustomTestCase): postprocess_sums(allow_reduce_scatter=False, is_layer_sparse=False) ) - def test_layer_modes_record_whether_the_layer_is_sparse(self): + def test_layer_facts_record_whether_the_layer_is_sparse(self): for sparse in (False, True): - with ( - self.subTest(sparse=sparse), - patch_communicator( - "sparse_mlp_scatter_mode", return_value=ScatterMode.FULL - ), - patch_communicator("enable_moe_dense_fully_dp", return_value=False), - patch_communicator( - "_generic_prefill_cp_shards_tokens", return_value=False - ), - ): - modes = LayerScatterModes.init_new( + with self.subTest(sparse=sparse): + facts = LayerFacts.init_new( layer_id=1, num_layers=4, is_layer_sparse=sparse, is_previous_layer_sparse=sparse, is_next_layer_sparse=sparse, ) - self.assertEqual(modes.is_layer_sparse, sparse) + self.assertEqual(facts.is_layer_sparse, sparse) def test_postprocess_passes_the_layer_sparsity(self): seen = {} @@ -184,7 +174,7 @@ class TestPostprocessReduceScatterv(CustomTestCase): communicator._cp_steps = None communicator._context = None communicator.allow_reduce_scatter = False - communicator.layer_scatter_modes = types.SimpleNamespace(is_layer_sparse=True) + communicator.layer_facts = types.SimpleNamespace(is_layer_sparse=True) communicator._steps = steps( ffn_output_move=lambda **kwargs: seen.update(kwargs) or (None, None) ) @@ -282,7 +272,7 @@ class TestLongcatNextnReducesItsMlp(CustomTestCase): ), patch.object(nextn, "RMSNorm"), patch.object(nextn, "get_parallel"), - patch.object(nextn, "LayerScatterModes"), + patch.object(nextn, "LayerFacts"), patch.object(nextn, "LayerCommunicator"), ): nextn.LongcatFlashDenseDecoderLayer(config, layer_id=0) diff --git a/test/registered/unit/layers/test_prepare_attn_steps.py b/test/registered/unit/layers/test_prepare_attn_steps.py index 8c10d9e7cf..331cd9c995 100644 --- a/test/registered/unit/layers/test_prepare_attn_steps.py +++ b/test/registered/unit/layers/test_prepare_attn_steps.py @@ -47,7 +47,7 @@ def communicator(norm): c = comm.LayerCommunicator.__new__(comm.LayerCommunicator) c.input_layernorm = norm # A layer inside the stack: an absent residual was written back before. - c.layer_scatter_modes = SimpleNamespace(is_first_layer=False) + c.layer_facts = SimpleNamespace(is_first_layer=False) c._sp_steps = None c._input_scattered_steps = None c._cp_steps = None diff --git a/test/registered/unit/lora/test_triton_dp_attention_unit.py b/test/registered/unit/lora/test_triton_dp_attention_unit.py index 8a00033a62..44765a022b 100644 --- a/test/registered/unit/lora/test_triton_dp_attention_unit.py +++ b/test/registered/unit/lora/test_triton_dp_attention_unit.py @@ -230,7 +230,7 @@ def test_communicator_publishes_layout_at_each_transition( expected_mlp = expected_attn = initial communicator = LayerCommunicator.__new__(LayerCommunicator) communicator._publish_lora_layout = publish_lora_layout - communicator.layer_scatter_modes = SimpleNamespace(is_first_layer=False) + communicator.layer_facts = SimpleNamespace(is_first_layer=False) communicator._context = SimpleNamespace() communicator._sp_steps = None communicator.post_attention_layernorm = None diff --git a/test/registered/unit/models/test_last_layer_communicator.py b/test/registered/unit/models/test_last_layer_communicator.py index de93b53d35..4f230a5b6b 100644 --- a/test/registered/unit/models/test_last_layer_communicator.py +++ b/test/registered/unit/models/test_last_layer_communicator.py @@ -161,12 +161,12 @@ CASES = { def build(case, num_layers, layer_id, config=None, **kwargs): """Construct one decoder layer with its submodules stubbed. Returns the - LayerCommunicator kwargs, the LayerScatterModes.init_new kwargs, and the + LayerCommunicator kwargs, the LayerFacts.init_new kwargs, and the names of the stubbed submodules it built.""" module_name, class_name, make_config, stubs, _ = CASES[case] module = import_model(module_name) communicator = MagicMock() - scatter_modes = MagicMock() + facts = MagicMock() built = [] def recording_stub(name): @@ -176,7 +176,7 @@ def build(case, num_layers, layer_id, config=None, **kwargs): return make - patches = dict(LayerCommunicator=communicator, LayerScatterModes=scatter_modes) + patches = dict(LayerCommunicator=communicator, LayerFacts=facts) patches.update({name: recording_stub(name) for name in stubs}) if hasattr(module, "get_parallel"): patches["get_parallel"] = lambda: PARALLEL @@ -185,7 +185,7 @@ def build(case, num_layers, layer_id, config=None, **kwargs): config or make_config(num_layers), layer_id=layer_id, **kwargs ) communicator.assert_called_once() - return communicator.call_args.kwargs, scatter_modes.init_new.call_args.kwargs, built + return communicator.call_args.kwargs, facts.init_new.call_args.kwargs, built def planned_as_last(case, num_layers, layer_id, **kwargs): @@ -268,7 +268,7 @@ class TestLastLayerCommunicator(CustomTestCase): self.assertTrue(planned["is_layer_sparse"]) -class TestLayerScatterModesLastLayer(CustomTestCase): +class TestLayerFactsLastLayer(CustomTestCase): def test_the_plan_marks_the_last_layer(self): from sglang.srt.layers import communicator as comm @@ -285,7 +285,7 @@ class TestLayerScatterModesLastLayer(CustomTestCase): (1, 0), ): with self.subTest(num_layers=num_layers, layer_id=layer_id): - modes = comm.LayerScatterModes.init_new( + modes = comm.LayerFacts.init_new( layer_id=layer_id, num_layers=num_layers, is_layer_sparse=False,