From 26a3214cdae3713feefae0ca67a85baa6a80b188 Mon Sep 17 00:00:00 2001 From: Cheng Wan <54331508+ch-wan@users.noreply.github.com> Date: Fri, 25 Sep 2026 01:49:33 -0700 Subject: [PATCH] [Refactor] Take a layer's last-layer fact from its scatter-mode plan (#41199) --- python/sglang/srt/layers/communicator.py | 8 ++-- .../sglang/srt/layers/communicator_dsa_cp.py | 2 - python/sglang/srt/layers/communicator_mhc.py | 2 - python/sglang/srt/models/bailing_moe.py | 3 -- .../sglang/srt/models/bailing_moe_linear.py | 1 - python/sglang/srt/models/bailing_moe_v3.py | 1 - python/sglang/srt/models/deepseek_v2.py | 3 -- .../srt/models/dots3_common/modeling.py | 3 -- python/sglang/srt/models/glm4_moe.py | 3 -- python/sglang/srt/models/glm4_moe_lite.py | 3 -- python/sglang/srt/models/glm5_next.py | 3 -- python/sglang/srt/models/gpt_oss.py | 6 +-- python/sglang/srt/models/interns2_mobius.py | 2 - python/sglang/srt/models/laguna.py | 1 - python/sglang/srt/models/llada2.py | 2 - python/sglang/srt/models/mellum.py | 1 - python/sglang/srt/models/mimo_v2.py | 1 - python/sglang/srt/models/minimax_m2.py | 1 - python/sglang/srt/models/minimax_m3.py | 1 - python/sglang/srt/models/nemotron_h_utils.py | 8 ++-- python/sglang/srt/models/qwen3_5.py | 2 - python/sglang/srt/models/qwen3_moe.py | 1 - python/sglang/srt/models/qwen3_next.py | 2 - python/sglang/srt/models/sarvam_moe.py | 1 - python/sglang/srt/models/sdar_moe.py | 1 - python/sglang/srt/models/step3p5.py | 1 - python/sglang/srt/models/xllm.py | 1 - .../models/test_last_layer_communicator.py | 39 +++++++++++++++++-- 28 files changed, 46 insertions(+), 57 deletions(-) diff --git a/python/sglang/srt/layers/communicator.py b/python/sglang/srt/layers/communicator.py index e586e43580..89a772bb7a 100644 --- a/python/sglang/srt/layers/communicator.py +++ b/python/sglang/srt/layers/communicator.py @@ -443,6 +443,8 @@ class LayerScatterModes: middle_residual_mode: ScatterMode layer_output_mode: ScatterMode is_layer_sparse: bool = False + # The model's last layer: its output goes to the final norm, not a next layer. + is_last_layer: bool = False @classmethod def init_new(cls, **kwargs): @@ -454,6 +456,7 @@ class LayerScatterModes: middle_residual_mode=cls._compute_middle_residual_mode(context), layer_output_mode=cls._compute_layer_output_mode(context), is_layer_sparse=context.is_layer_sparse, + is_last_layer=context.layer_id == context.num_layers - 1, ) @classmethod @@ -705,7 +708,6 @@ class LayerCommunicator: post_attention_layernorm: torch.nn.Module, # Reduce scatter requires skipping all-reduce in model code after MoE/MLP, so only enable for models which have that implemented. Remove flag once done for all models that use LayerCommunicator. allow_reduce_scatter: bool = False, - is_last_layer: bool = False, qkv_latent_func: Optional[Callable] = None, force_layernorm_before_dp_gather: bool = False, enable_fused_ar_quant: bool = False, @@ -716,7 +718,7 @@ class LayerCommunicator: self.input_layernorm = input_layernorm self.post_attention_layernorm = post_attention_layernorm self.allow_reduce_scatter = allow_reduce_scatter - self.is_last_layer = is_last_layer + self.is_last_layer = layer_scatter_modes.is_last_layer self.qkv_latent_func = qkv_latent_func self.force_layernorm_before_dp_gather = force_layernorm_before_dp_gather self.enable_fused_ar_quant = enable_fused_ar_quant @@ -743,11 +745,11 @@ class LayerCommunicator: mlp_mode=ScatterMode.SCATTERED, middle_residual_mode=ScatterMode.SCATTERED, layer_output_mode=ScatterMode.SCATTERED, + is_last_layer=layer_scatter_modes.is_last_layer, ), input_layernorm=input_layernorm, post_attention_layernorm=post_attention_layernorm, allow_reduce_scatter=allow_reduce_scatter, - is_last_layer=is_last_layer, qkv_latent_func=qkv_latent_func, force_layernorm_before_dp_gather=force_layernorm_before_dp_gather, enable_fused_ar_quant=enable_fused_ar_quant, diff --git a/python/sglang/srt/layers/communicator_dsa_cp.py b/python/sglang/srt/layers/communicator_dsa_cp.py index c64c012734..1b2e5c3f42 100644 --- a/python/sglang/srt/layers/communicator_dsa_cp.py +++ b/python/sglang/srt/layers/communicator_dsa_cp.py @@ -92,7 +92,6 @@ class DSACPLayerCommunicator(LayerCommunicator): post_attention_layernorm: torch.nn.Module, # Reduce scatter requires skipping all-reduce in model code after MoE/MLP, so only enable for models which have that implemented. Remove flag once done for all models that use LayerCommunicator. allow_reduce_scatter: bool = False, - is_last_layer: bool = False, qkv_latent_func: Optional[Callable] = None, ): super().__init__( @@ -100,7 +99,6 @@ class DSACPLayerCommunicator(LayerCommunicator): input_layernorm, post_attention_layernorm, allow_reduce_scatter, - is_last_layer, qkv_latent_func, ) diff --git a/python/sglang/srt/layers/communicator_mhc.py b/python/sglang/srt/layers/communicator_mhc.py index 178ef50f33..f664c9d002 100644 --- a/python/sglang/srt/layers/communicator_mhc.py +++ b/python/sglang/srt/layers/communicator_mhc.py @@ -416,7 +416,6 @@ class MHCLayerCommunicator(LayerCommunicator): input_layernorm: torch.nn.Module, post_attention_layernorm: torch.nn.Module, allow_reduce_scatter: bool = False, - is_last_layer: bool = False, qkv_latent_func: Optional[Callable] = None, *, is_first_layer: bool, @@ -440,7 +439,6 @@ class MHCLayerCommunicator(LayerCommunicator): input_layernorm, post_attention_layernorm, allow_reduce_scatter, - is_last_layer, qkv_latent_func, ) diff --git a/python/sglang/srt/models/bailing_moe.py b/python/sglang/srt/models/bailing_moe.py index c7d12ecb6c..b01ec580c9 100644 --- a/python/sglang/srt/models/bailing_moe.py +++ b/python/sglang/srt/models/bailing_moe.py @@ -678,8 +678,6 @@ class BailingMoEBlock(nn.Module): is_next_layer_sparse=is_next_layer_sparse, ) - self.is_last_layer = is_nextn or (self.layer_id == config.num_hidden_layers - 1) - if self.is_layer_sparse: self.mlp = BailingMoESparseMoeBlock( layer_id=layer_id, @@ -709,7 +707,6 @@ class BailingMoEBlock(nn.Module): input_layernorm=self.input_layernorm, post_attention_layernorm=self.post_attention_layernorm, allow_reduce_scatter=True, - is_last_layer=self.is_last_layer, ) def _is_layer_sparse( diff --git a/python/sglang/srt/models/bailing_moe_linear.py b/python/sglang/srt/models/bailing_moe_linear.py index be2f3efd4a..f099c5d0e0 100644 --- a/python/sglang/srt/models/bailing_moe_linear.py +++ b/python/sglang/srt/models/bailing_moe_linear.py @@ -832,7 +832,6 @@ class BailingMoELinearDecoderLayer(nn.Module): input_layernorm=self.input_layernorm, post_attention_layernorm=self.post_attention_layernorm, allow_reduce_scatter=False, - is_last_layer=(is_nextn or layer_id == config.num_hidden_layers - 1), qkv_latent_func=qkv_latent_func, ) diff --git a/python/sglang/srt/models/bailing_moe_v3.py b/python/sglang/srt/models/bailing_moe_v3.py index 4d42bc53c7..333479224a 100644 --- a/python/sglang/srt/models/bailing_moe_v3.py +++ b/python/sglang/srt/models/bailing_moe_v3.py @@ -1164,7 +1164,6 @@ class BailingMoELinearDecoderLayer(nn.Module): input_layernorm=self.input_layernorm, post_attention_layernorm=self.post_attention_layernorm, allow_reduce_scatter=True, - is_last_layer=(is_nextn or layer_id == config.num_hidden_layers - 1), qkv_latent_func=( self.attention.prepare_qkv_latent if self.attention_type == 1 and self.use_mla diff --git a/python/sglang/srt/models/deepseek_v2.py b/python/sglang/srt/models/deepseek_v2.py index 0e8c1674d0..f5174f7977 100644 --- a/python/sglang/srt/models/deepseek_v2.py +++ b/python/sglang/srt/models/deepseek_v2.py @@ -2631,9 +2631,6 @@ class DeepseekV2DecoderLayer(nn.Module): input_layernorm=self.input_layernorm, post_attention_layernorm=self.post_attention_layernorm, allow_reduce_scatter=True, - is_last_layer=( - is_nextn or (self.layer_id == self.config.num_hidden_layers - 1) - ), qkv_latent_func=self.self_attn.prepare_qkv_latent, ) diff --git a/python/sglang/srt/models/dots3_common/modeling.py b/python/sglang/srt/models/dots3_common/modeling.py index e699f70da3..6cfc2109d3 100644 --- a/python/sglang/srt/models/dots3_common/modeling.py +++ b/python/sglang/srt/models/dots3_common/modeling.py @@ -1583,9 +1583,6 @@ class Dots3DecoderLayer(nn.Module): input_layernorm=self.input_layernorm, post_attention_layernorm=self.post_attention_layernorm, allow_reduce_scatter=True, - is_last_layer=( - is_nextn or (self.layer_id == self.config.num_hidden_layers - 1) - ), ) def _is_layer_sparse(self, layer_id: int, is_nextn: bool) -> bool: diff --git a/python/sglang/srt/models/glm4_moe.py b/python/sglang/srt/models/glm4_moe.py index e304818513..2b20e64f84 100644 --- a/python/sglang/srt/models/glm4_moe.py +++ b/python/sglang/srt/models/glm4_moe.py @@ -877,9 +877,6 @@ class Glm4MoeDecoderLayer(nn.Module): input_layernorm=self.input_layernorm, post_attention_layernorm=self.post_attention_layernorm, allow_reduce_scatter=True, - is_last_layer=( - is_nextn or (self.layer_id == self.config.num_hidden_layers - 1) - ), ) # Detect if QKV uses aiter FP8 per-token quant so we can fuse diff --git a/python/sglang/srt/models/glm4_moe_lite.py b/python/sglang/srt/models/glm4_moe_lite.py index 3e95bf2067..3b4f46ce60 100644 --- a/python/sglang/srt/models/glm4_moe_lite.py +++ b/python/sglang/srt/models/glm4_moe_lite.py @@ -592,9 +592,6 @@ class Glm4MoeLiteDecoderLayer(nn.Module): input_layernorm=self.input_layernorm, post_attention_layernorm=self.post_attention_layernorm, allow_reduce_scatter=True, - is_last_layer=( - is_nextn or (self.layer_id == self.config.num_hidden_layers - 1) - ), qkv_latent_func=self.self_attn.prepare_qkv_latent, ) diff --git a/python/sglang/srt/models/glm5_next.py b/python/sglang/srt/models/glm5_next.py index bd1492381b..c634008bd4 100644 --- a/python/sglang/srt/models/glm5_next.py +++ b/python/sglang/srt/models/glm5_next.py @@ -788,9 +788,6 @@ class Glm5NextDecoderLayer(nn.Module): input_layernorm=self.input_layernorm, post_attention_layernorm=self.post_attention_layernorm, allow_reduce_scatter=True, - is_last_layer=( - is_nextn or (self.layer_id == self.config.num_hidden_layers - 1) - ), qkv_latent_func=( self.self_attn.prepare_qkv_latent if not self.is_linear_attn else None ), diff --git a/python/sglang/srt/models/gpt_oss.py b/python/sglang/srt/models/gpt_oss.py index 32859deb2c..b08c459d3c 100644 --- a/python/sglang/srt/models/gpt_oss.py +++ b/python/sglang/srt/models/gpt_oss.py @@ -564,9 +564,8 @@ class GptOssDecoderLayer(nn.Module): self.attn_tp_size = get_parallel().attn_tp_size self.attn_tp_rank = get_parallel().attn_tp_rank - # GptOss all layers are sparse and have no nextn now + # GptOss all layers are sparse self.is_layer_sparse = True - self.is_nextn = False is_previous_layer_sparse = True is_next_layer_sparse = True @@ -611,9 +610,6 @@ class GptOssDecoderLayer(nn.Module): input_layernorm=self.input_layernorm, post_attention_layernorm=self.post_attention_layernorm, allow_reduce_scatter=True, - is_last_layer=( - self.is_nextn or (self.layer_id == self.config.num_hidden_layers - 1) - ), ) def forward( diff --git a/python/sglang/srt/models/interns2_mobius.py b/python/sglang/srt/models/interns2_mobius.py index b4bedd6de5..79fedb4f58 100644 --- a/python/sglang/srt/models/interns2_mobius.py +++ b/python/sglang/srt/models/interns2_mobius.py @@ -505,7 +505,6 @@ class InternS2MobiusLinearDecoderLayer(_InternS2MobiusDecoderMixin, nn.Module): input_layernorm=self.input_layernorm, post_attention_layernorm=self.post_attention_layernorm, allow_reduce_scatter=True, - is_last_layer=(layer_id == config.num_hidden_layers - 1), enable_fused_ar_quant=enable_fused_ar_quant, fused_ar_quant_keep_bf16=enable_fused_ar_quant, ) @@ -639,7 +638,6 @@ class InternS2MobiusAttentionDecoderLayer( input_layernorm=self.input_layernorm, post_attention_layernorm=self.post_attention_layernorm, allow_reduce_scatter=True, - is_last_layer=(layer_id == config.num_hidden_layers - 1), enable_fused_ar_quant=enable_fused_ar_quant, fused_ar_quant_keep_bf16=False, ) diff --git a/python/sglang/srt/models/laguna.py b/python/sglang/srt/models/laguna.py index b9d875ef64..df3bdf4ac8 100644 --- a/python/sglang/srt/models/laguna.py +++ b/python/sglang/srt/models/laguna.py @@ -467,7 +467,6 @@ class LagunaDecoderLayer(nn.Module): input_layernorm=self.input_layernorm, post_attention_layernorm=self.post_attention_layernorm, allow_reduce_scatter=True, - is_last_layer=(layer_id == config.num_hidden_layers - 1), ) def forward( diff --git a/python/sglang/srt/models/llada2.py b/python/sglang/srt/models/llada2.py index 4f4774c354..def449acf6 100644 --- a/python/sglang/srt/models/llada2.py +++ b/python/sglang/srt/models/llada2.py @@ -620,8 +620,6 @@ class LLaDA2MoeBlock(nn.Module): is_next_layer_sparse=is_next_layer_sparse, ) - self.is_last_layer = self.layer_id == config.num_hidden_layers - 1 - if self.is_layer_sparse: self.mlp = LLaDA2MoeSparseMoeBlock( layer_id=layer_id, diff --git a/python/sglang/srt/models/mellum.py b/python/sglang/srt/models/mellum.py index be64381f63..1e678c9a26 100644 --- a/python/sglang/srt/models/mellum.py +++ b/python/sglang/srt/models/mellum.py @@ -458,7 +458,6 @@ class MellumDecoderLayer(Qwen3MoeDecoderLayer): input_layernorm=self.input_layernorm, post_attention_layernorm=self.post_attention_layernorm, allow_reduce_scatter=True, - is_last_layer=(layer_id == cfg.num_hidden_layers - 1), ) diff --git a/python/sglang/srt/models/mimo_v2.py b/python/sglang/srt/models/mimo_v2.py index e1f27f560d..dcdefbf80b 100644 --- a/python/sglang/srt/models/mimo_v2.py +++ b/python/sglang/srt/models/mimo_v2.py @@ -869,7 +869,6 @@ class MiMoV2DecoderLayer(nn.Module): input_layernorm=self.input_layernorm, post_attention_layernorm=self.post_attention_layernorm, allow_reduce_scatter=True, - is_last_layer=(self.layer_id == self.config.num_hidden_layers - 1), ) def forward( diff --git a/python/sglang/srt/models/minimax_m2.py b/python/sglang/srt/models/minimax_m2.py index b0c968ba13..c680ee8439 100644 --- a/python/sglang/srt/models/minimax_m2.py +++ b/python/sglang/srt/models/minimax_m2.py @@ -989,7 +989,6 @@ class MiniMaxM2DecoderLayer(nn.Module): input_layernorm=self.input_layernorm, post_attention_layernorm=self.post_attention_layernorm, allow_reduce_scatter=True, - is_last_layer=(layer_id == config.num_hidden_layers - 1), ) def forward( diff --git a/python/sglang/srt/models/minimax_m3.py b/python/sglang/srt/models/minimax_m3.py index ab5faa7d85..883eb99dd7 100644 --- a/python/sglang/srt/models/minimax_m3.py +++ b/python/sglang/srt/models/minimax_m3.py @@ -1361,7 +1361,6 @@ class MiniMaxM3DecoderLayer(nn.Module): input_layernorm=self.input_layernorm, post_attention_layernorm=self.post_attention_layernorm, allow_reduce_scatter=True, - is_last_layer=(layer_id == config.num_hidden_layers - 1), ) def forward( diff --git a/python/sglang/srt/models/nemotron_h_utils.py b/python/sglang/srt/models/nemotron_h_utils.py index c2222127d9..6c89fe22ec 100644 --- a/python/sglang/srt/models/nemotron_h_utils.py +++ b/python/sglang/srt/models/nemotron_h_utils.py @@ -23,7 +23,9 @@ def feeds_mlp_layer(pattern: str, layer_idx: int) -> bool: return next_idx < len(pattern) and not is_attn_layer(pattern[next_idx]) -def _build_layer_scatter_modes(is_sparse: bool = False) -> LayerScatterModes: +def _build_layer_scatter_modes( + is_sparse: bool = False, is_last_layer: bool = False +) -> LayerScatterModes: scatter_mlp = is_sparse and not get_moe_a2a_backend().is_none() mlp_mode = ScatterMode.SCATTERED if scatter_mlp else ScatterMode.FULL middle_residual_mode = ( @@ -36,6 +38,7 @@ def _build_layer_scatter_modes(is_sparse: bool = False) -> LayerScatterModes: middle_residual_mode=middle_residual_mode, layer_output_mode=ScatterMode.TP_ATTN_FULL, is_layer_sparse=is_sparse, + is_last_layer=is_last_layer, ) @@ -48,12 +51,11 @@ def make_layer_communicator( is_last_layer: bool = False, ) -> LayerCommunicator: return LayerCommunicator( - layer_scatter_modes=_build_layer_scatter_modes(is_sparse), + layer_scatter_modes=_build_layer_scatter_modes(is_sparse, is_last_layer), input_layernorm=layer_norm if for_attn else nn.Identity(), post_attention_layernorm=nn.Identity() if for_attn else layer_norm, # With attention TP > 1, the default gather adds the residual to one # rank's partial in bf16 before the cross-rank sum. force_layernorm_before_dp_gather=True, allow_reduce_scatter=allow_reduce_scatter, - is_last_layer=is_last_layer, ) diff --git a/python/sglang/srt/models/qwen3_5.py b/python/sglang/srt/models/qwen3_5.py index c1124edea2..a144068857 100644 --- a/python/sglang/srt/models/qwen3_5.py +++ b/python/sglang/srt/models/qwen3_5.py @@ -1129,7 +1129,6 @@ class Qwen3_5LinearDecoderLayer(nn.Module): input_layernorm=self.input_layernorm, post_attention_layernorm=self.post_attention_layernorm, allow_reduce_scatter=True, - is_last_layer=(layer_id == config.num_hidden_layers - 1), enable_fused_ar_quant=enable_fused_ar_quant, fused_ar_quant_keep_bf16=enable_fused_ar_quant, ) @@ -1333,7 +1332,6 @@ class Qwen3_5AttentionDecoderLayer(nn.Module): input_layernorm=self.input_layernorm, post_attention_layernorm=self.post_attention_layernorm, allow_reduce_scatter=True, - is_last_layer=(layer_id == config.num_hidden_layers - 1), enable_fused_ar_quant=enable_fused_ar_quant, fused_ar_quant_keep_bf16=False, ) diff --git a/python/sglang/srt/models/qwen3_moe.py b/python/sglang/srt/models/qwen3_moe.py index f443bf7fb0..481808bc6c 100644 --- a/python/sglang/srt/models/qwen3_moe.py +++ b/python/sglang/srt/models/qwen3_moe.py @@ -873,7 +873,6 @@ class Qwen3MoeDecoderLayer(nn.Module): input_layernorm=self.input_layernorm, post_attention_layernorm=self.post_attention_layernorm, allow_reduce_scatter=True, - is_last_layer=(self.layer_id == self.config.num_hidden_layers - 1), ) def forward( diff --git a/python/sglang/srt/models/qwen3_next.py b/python/sglang/srt/models/qwen3_next.py index cce93f3f86..cdc3b4778a 100644 --- a/python/sglang/srt/models/qwen3_next.py +++ b/python/sglang/srt/models/qwen3_next.py @@ -543,7 +543,6 @@ class Qwen3HybridLinearDecoderLayer(nn.Module): input_layernorm=self.input_layernorm, post_attention_layernorm=self.post_attention_layernorm, allow_reduce_scatter=True, - is_last_layer=(layer_id == config.num_hidden_layers - 1), ) def forward( @@ -716,7 +715,6 @@ class Qwen3HybridAttentionDecoderLayer(nn.Module): input_layernorm=self.input_layernorm, post_attention_layernorm=self.post_attention_layernorm, allow_reduce_scatter=True, - is_last_layer=(layer_id == config.num_hidden_layers - 1), ) self.alt_stream = alt_stream diff --git a/python/sglang/srt/models/sarvam_moe.py b/python/sglang/srt/models/sarvam_moe.py index 9af712f12c..20cc72335d 100644 --- a/python/sglang/srt/models/sarvam_moe.py +++ b/python/sglang/srt/models/sarvam_moe.py @@ -1060,7 +1060,6 @@ class SarvamMoEMLADecoderLayer(nn.Module): post_attention_layernorm=self.post_attention_layernorm, qkv_latent_func=self.self_attn.prepare_qkv_latent, allow_reduce_scatter=True, - is_last_layer=(layer_id == config.num_hidden_layers - 1), ) def forward( diff --git a/python/sglang/srt/models/sdar_moe.py b/python/sglang/srt/models/sdar_moe.py index ec5eeb8bfb..d8043cb8f0 100644 --- a/python/sglang/srt/models/sdar_moe.py +++ b/python/sglang/srt/models/sdar_moe.py @@ -368,7 +368,6 @@ class SDARMoeBlock(nn.Module): input_layernorm=self.input_layernorm, post_attention_layernorm=self.post_attention_layernorm, allow_reduce_scatter=True, - is_last_layer=(layer_id == config.num_hidden_layers - 1), ) def forward( diff --git a/python/sglang/srt/models/step3p5.py b/python/sglang/srt/models/step3p5.py index c57b84798c..35dcf19878 100644 --- a/python/sglang/srt/models/step3p5.py +++ b/python/sglang/srt/models/step3p5.py @@ -572,7 +572,6 @@ class Step3p5DecoderLayer(nn.Module): input_layernorm=self.input_layernorm, post_attention_layernorm=self.post_attention_layernorm, allow_reduce_scatter=True, - is_last_layer=(is_nextn or layer_id == config.num_hidden_layers - 1), ) self.layer_id = layer_id diff --git a/python/sglang/srt/models/xllm.py b/python/sglang/srt/models/xllm.py index 33faedf085..c0c9eb5b77 100644 --- a/python/sglang/srt/models/xllm.py +++ b/python/sglang/srt/models/xllm.py @@ -1542,7 +1542,6 @@ class XllmDecoderLayer(nn.Module): input_layernorm=self.input_layernorm, post_attention_layernorm=self.post_attention_layernorm, allow_reduce_scatter=True, - is_last_layer=(self.layer_id == config.num_hidden_layers - 1), ) def forward( diff --git a/test/registered/unit/models/test_last_layer_communicator.py b/test/registered/unit/models/test_last_layer_communicator.py index 2de04dd971..2bf4d313b2 100644 --- a/test/registered/unit/models/test_last_layer_communicator.py +++ b/test/registered/unit/models/test_last_layer_communicator.py @@ -187,8 +187,12 @@ def build(case, num_layers, layer_id, config=None, **kwargs): return communicator.call_args.kwargs, scatter_modes.init_new.call_args.kwargs, built -def is_last_layer_passed(case, num_layers, layer_id, **kwargs): - return build(case, num_layers, layer_id, **kwargs)[0].get("is_last_layer", False) +def planned_as_last(case, num_layers, layer_id, **kwargs): + """Whether the layer's layout plan makes it the model's last layer, the one + fact the communicator reads the last layer from.""" + passed, planned, _ = build(case, num_layers, layer_id, **kwargs) + assert "is_last_layer" not in passed, "the plan is the only source" + return planned["layer_id"] == planned["num_layers"] - 1 class TestLastLayerCommunicator(CustomTestCase): @@ -199,7 +203,7 @@ class TestLastLayerCommunicator(CustomTestCase): for layer_id in range(NUM_LAYERS): with self.subTest(case=case, layer_id=layer_id): self.assertEqual( - is_last_layer_passed(case, NUM_LAYERS, layer_id), + planned_as_last(case, NUM_LAYERS, layer_id), layer_id == NUM_LAYERS - 1, ) @@ -212,7 +216,7 @@ class TestLastLayerCommunicator(CustomTestCase): num_layers, layer_id = draft with self.subTest(case=case): self.assertTrue( - is_last_layer_passed(case, num_layers, layer_id, is_nextn=True) + planned_as_last(case, num_layers, layer_id, is_nextn=True) ) def test_draft_model_layer_is_planned_as_a_one_layer_model(self): @@ -252,5 +256,32 @@ class TestLastLayerCommunicator(CustomTestCase): self.assertTrue(planned["is_layer_sparse"]) +class TestLayerScatterModesLastLayer(CustomTestCase): + def test_the_plan_marks_the_last_layer(self): + from sglang.srt.layers import communicator as comm + + with ( + patch.object(comm, "enable_moe_dense_fully_dp", return_value=False), + patch.object(comm, "_generic_prefill_cp_shards_tokens", return_value=False), + patch.object(comm, "is_dsa_enable_prefill_cp", return_value=False), + patch.object(comm, "is_mla_cp_enabled", return_value=False), + ): + for num_layers, layer_id in ( + (NUM_LAYERS, 0), + (NUM_LAYERS, 2), + (NUM_LAYERS, 3), + (1, 0), + ): + with self.subTest(num_layers=num_layers, layer_id=layer_id): + modes = comm.LayerScatterModes.init_new( + layer_id=layer_id, + num_layers=num_layers, + is_layer_sparse=False, + is_previous_layer_sparse=False, + is_next_layer_sparse=False, + ) + self.assertEqual(modes.is_last_layer, layer_id == num_layers - 1) + + if __name__ == "__main__": unittest.main()