From 234ccba7f625e12639e621044496bcc6197c3758 Mon Sep 17 00:00:00 2001 From: Cheng Wan <54331508+ch-wan@users.noreply.github.com> Date: Sat, 10 Oct 2026 12:43:09 -0700 Subject: [PATCH] [Refactor] Declare every model's stages from a shared function and drop the meta-device neighbour build (#43460) --- docs/docs/developer_guide/layer_boundary.mdx | 2 +- .../srt/layers/layer_boundary/factories.py | 52 +--- .../sglang/srt/layers/layer_boundary/facts.py | 12 + .../layer_boundary/residual/attn_bank.py | 17 ++ .../layers/layer_boundary/residual/gated.py | 17 ++ .../srt/layers/layer_boundary/residual/mhc.py | 14 + .../srt/layers/moe/fused_moe_triton/layer.py | 3 - python/sglang/srt/layers/moe/kt_ep_wrapper.py | 3 +- python/sglang/srt/models/apertus.py | 15 +- python/sglang/srt/models/arcee.py | 15 +- python/sglang/srt/models/bailing_moe.py | 36 ++- .../sglang/srt/models/bailing_moe_linear.py | 35 ++- python/sglang/srt/models/bailing_moe_v3.py | 39 +-- python/sglang/srt/models/deepseek_v2.py | 66 +++-- .../srt/models/dots3_common/modeling.py | 38 ++- python/sglang/srt/models/ernie45_moe_vl.py | 24 +- python/sglang/srt/models/exaone4.py | 46 ++-- python/sglang/srt/models/exaone_moe.py | 28 +- python/sglang/srt/models/gemma4_causal.py | 46 +++- python/sglang/srt/models/gigachat35.py | 67 +++-- python/sglang/srt/models/glm4.py | 41 ++- python/sglang/srt/models/glm4_moe.py | 40 ++- python/sglang/srt/models/glm4_moe_lite.py | 42 ++- python/sglang/srt/models/glm5_next.py | 65 +++-- python/sglang/srt/models/gpt_oss.py | 29 +- python/sglang/srt/models/granitemoehybrid.py | 56 ++-- python/sglang/srt/models/jet_nemotron.py | 6 + python/sglang/srt/models/kimi_k3.py | 248 +++++++++++------- python/sglang/srt/models/kimi_linear.py | 24 +- python/sglang/srt/models/laguna.py | 35 ++- python/sglang/srt/models/llada2.py | 28 +- python/sglang/srt/models/llama.py | 22 +- python/sglang/srt/models/llama_eagle.py | 3 +- python/sglang/srt/models/mellum.py | 50 ++-- python/sglang/srt/models/mimo_v2.py | 38 ++- python/sglang/srt/models/minimax_m2.py | 38 ++- python/sglang/srt/models/minimax_m3.py | 53 ++-- python/sglang/srt/models/ministral3.py | 1 + python/sglang/srt/models/mixtral.py | 20 +- python/sglang/srt/models/nanbeige.py | 20 +- python/sglang/srt/models/nemotron_nas.py | 28 +- python/sglang/srt/models/qwen2.py | 22 +- python/sglang/srt/models/qwen2_eagle.py | 3 +- python/sglang/srt/models/qwen2_moe.py | 30 ++- python/sglang/srt/models/qwen3.py | 18 +- python/sglang/srt/models/qwen3_5.py | 100 +++---- python/sglang/srt/models/qwen3_moe.py | 29 +- python/sglang/srt/models/qwen4_exp.py | 68 +++-- python/sglang/srt/models/sarvam_moe.py | 62 +++-- python/sglang/srt/models/sdar.py | 24 +- python/sglang/srt/models/sdar_moe.py | 20 +- python/sglang/srt/models/spark2_5.py | 20 +- python/sglang/srt/models/step3p5.py | 36 ++- python/sglang/srt/models/xllm.py | 68 +++-- python/sglang/srt/models/zaya.py | 62 +++-- python/sglang/srt/utils/common.py | 125 +-------- python/sglang/test/boundary_fixtures.py | 14 +- .../unit/layer_boundary/test_append_stages.py | 185 +++++-------- .../layer_boundary/test_gated_residual_ops.py | 4 +- .../test_independent_stage_construction.py | 12 +- .../test_kimi_k3_stage_kernels.py | 22 +- .../test_nemotron_h_mtp_reduction.py | 10 +- .../layer_boundary/test_qwen4_exp_ple_rows.py | 7 +- .../unit/layers/test_neighbour_layer_build.py | 76 ------ test/registered/unit/layers/test_xielu.py | 3 +- 65 files changed, 1407 insertions(+), 1075 deletions(-) delete mode 100644 test/registered/unit/layers/test_neighbour_layer_build.py diff --git a/docs/docs/developer_guide/layer_boundary.mdx b/docs/docs/developer_guide/layer_boundary.mdx index 336620778b..3d5811b226 100644 --- a/docs/docs/developer_guide/layer_boundary.mdx +++ b/docs/docs/developer_guide/layer_boundary.mdx @@ -122,7 +122,7 @@ with layer_stack(): Every stage binds when the stack closes, once its producer and its consumer are known: the stage appended before it and the one appended after it. The last stage ends the model's layer stack and is marked terminal, unless a later layer declares a stage. A boundary's `declaration` is usable at once; when the stack closes, it is replaced by the declaration with its place in the stack (its `previous` and `terminal`), and the plan is bound. An error raised while binding carries a note naming the `append_stages()` call it came from. A terminal stage leaves no work for a following layer, except the final-norm finalize handoff described in [Enter and leave the layer stack](#enter-and-leave-the-layer-stack). A NextN/MTP draft module is its own stack: its layer enters the stack and ends it, even when it reuses the target's decoder class. -Pipeline parallelism splits one layer stack across ranks, so the first and last local stages have neighbours that another rank builds. The model gives `make_layers()` (or `make_pp_layers()`) its shared declaration function, `stage_facts`: given a global layer index, it returns the stages that layer declares, in order (an empty tuple for a layer that declares none), from the config, the layer index, construction options known before the layer is built, and the fixed parallel topology, never from built modules or provider state. The layer declares its own stages with the same function, so both ranks derive a boundary from one source; under pipeline parallelism `make_layers()` also checks each layer it builds against the function. When the stack closes, it asks the function for the stages next to it, nearest layer first, passing over a layer that declares none, and keeps only what binding reads of them (`facts_of()`: the capabilities a read, update and output transform declare, not the objects). A model that does not give the function yet is on a transitional list: its neighbours are built again with the model's own layer function inside `building_neighbour_layer()` (on the meta device), which must then skip the host and device resources only a running layer needs when `is_building_neighbour_layer()` is true and must not depend on which pipeline rank builds it. A model off that list must give the function to run under pipeline parallelism. +Pipeline parallelism splits one layer stack across ranks, so the first and last local stages have neighbours that another rank builds. The model gives `make_layers()` (or `make_pp_layers()`) its shared declaration function, `stage_facts`: given a global layer index, it returns the stages that layer declares, in order (an empty tuple for a layer that declares none), from the config, the layer index, construction options known before the layer is built, and the fixed parallel topology, never from built modules or provider state. The layer declares its own stages with the same function, so both ranks derive a boundary from one source; under pipeline parallelism `make_layers()` also checks each layer it builds against the function. When the stack closes, it asks the function for the stages next to it, nearest layer first, passing over a layer that declares none, and keeps only what binding reads of them (`facts_of()`: the capabilities a read, update and output transform declare, not the objects). A model whose layers declare stages must give the function to run under pipeline parallelism; no rank builds a layer another rank holds. Each item may include a third element containing constructor options: an attention stage accepts `qkv_latent_func` (for example `{"qkv_latent_func": self.self_attn.prepare_qkv_latent}`) and `fusions`; an FFN stage accepts only `fusions` (see [Fusion providers](#fusion-providers-and-output-decisions)). An option a stage does not take is rejected when appended. Pass declarations fresh from `declare_attn()` / `declare_ffn()`: one taken from a bound boundary already has its place in a stack, and is rejected. diff --git a/python/sglang/srt/layers/layer_boundary/factories.py b/python/sglang/srt/layers/layer_boundary/factories.py index 69a47cbde9..caa6c01f48 100644 --- a/python/sglang/srt/layers/layer_boundary/factories.py +++ b/python/sglang/srt/layers/layer_boundary/factories.py @@ -7,7 +7,7 @@ import os import sys from dataclasses import dataclass, fields, is_dataclass, replace from types import MappingProxyType -from typing import Callable, Mapping, NamedTuple, Optional, Tuple +from typing import Callable, Mapping, NamedTuple, Optional from sglang.srt.environ import envs from sglang.srt.layers import layernorm_sp @@ -302,11 +302,10 @@ class StageDeclaration: contiguous slice of the rows, it returns them all, in rank order, or None to leave the gather to the boundary. Called on every batch, so it must be CUDA-graph safe. Across a pipeline boundary the - producer's rank takes it from the neighbouring stage's declaration - (the model's shared declaration function, or, for a model on the - transitional list, the layer built again on the meta device), so it - may use only the communication state of the rank it runs on, not - the declaring layer's weights or buffers. + producer's rank takes it from the model's shared declaration + function, without the declaring layer, so it may use only the + communication state of the rank it runs on, not that layer's + weights or buffers. previous: The stage whose output this stage consumes, as the stack records it: only the facts binding reads of it (see facts_of), on this rank or another. @@ -849,12 +848,13 @@ def layer_stack(*, previous_layers=(), next_layers=(), final_read=None): Args: previous_layers: For the layers before this stack that another pipeline rank holds, nearest first, callables that return the - stages the layer declares as DeclaredStages (from the model's - shared declaration function), or else build the layer so that it - appends them. Called only if this stack appended stages, after its - own layers are built, until one of them declares a stage: its last - stage is the producer of this stack's first. Only what binding - reads of that stage is kept (see facts_of). + stages the layer declares, in order, without building it (the + model's shared declaration function at that layer's index; an + empty sequence for a layer that declares none). Called only if + this stack appended stages, after its own layers are built, until + one of them declares a stage: its last stage is the producer of + this stack's first. Only what binding reads of that stage is kept + (see facts_of). next_layers: Likewise for the layers after this stack, whose first declared stage is the consumer of this stack's last. final_read: The model's final read of the stack's output (a @@ -881,13 +881,6 @@ def layer_stack(*, previous_layers=(), next_layers=(), final_read=None): _stack = outer -class DeclaredStages(NamedTuple): - """The stages a neighbouring layer declares, from the model's shared - declaration function, which stand in for building the layer.""" - - stages: Tuple[StageDeclaration, ...] - - def check_declared_stages(stack, appended: int, expected, where: str) -> None: """Check that what a layer appended to ``stack`` since it held ``appended`` appends is what its shared declaration function says it @@ -947,26 +940,9 @@ def _differing_fields(value, given, prefix=""): def _neighbour_stage(layers, *, last): """The stage a neighbouring layer declares next to this stack: the last - one of the nearest layer before it, or the first of the nearest after. - Branches are side paths, so they never stand next to the stack.""" - global _stack + one of the nearest layer before it, or the first of the nearest after.""" for layer in layers: - outer = _stack - _stack = _LayerStack() - try: - result = layer() - declared = ( - result.stages - if isinstance(result, DeclaredStages) - else [ - declaration - for append in _stack.appends - if append.prepared_from is None - for declaration in append.declarations - ] - ) - finally: - _stack = outer + declared = tuple(layer()) if declared: return facts_of(declared[-1] if last else declared[0]) return None diff --git a/python/sglang/srt/layers/layer_boundary/facts.py b/python/sglang/srt/layers/layer_boundary/facts.py index 7849d877f7..a956f65013 100644 --- a/python/sglang/srt/layers/layer_boundary/facts.py +++ b/python/sglang/srt/layers/layer_boundary/facts.py @@ -81,6 +81,18 @@ class TransformFacts(msgspec.Struct, frozen=True): return cls(before_reduce_scatter=transform.before_reduce_scatter) +def residual_facts(ops): + """A layer's reads and updates (a LayerResidualOps) reduced to the + capabilities they declare, in the same shape: what its stages declare + without the modules and state the reads and updates hold.""" + return type(ops)( + attn_readout=ReadFacts.of(ops.attn_readout), + attn_update=UpdateFacts.of(ops.attn_update), + ffn_readout=ReadFacts.of(ops.ffn_readout), + ffn_update=UpdateFacts.of(ops.ffn_update), + ) + + def facts_of(declaration): """``declaration`` with its read, update and output transform reduced to the capabilities they declare, and without the declarations it was diff --git a/python/sglang/srt/layers/layer_boundary/residual/attn_bank.py b/python/sglang/srt/layers/layer_boundary/residual/attn_bank.py index dbaa02324b..2caef8bc5f 100644 --- a/python/sglang/srt/layers/layer_boundary/residual/attn_bank.py +++ b/python/sglang/srt/layers/layer_boundary/residual/attn_bank.py @@ -21,6 +21,7 @@ import torch from sglang.srt.layers.attn_residual import AttnResidual from sglang.srt.layers.layer_boundary import ops from sglang.srt.layers.layer_boundary.contracts import ReadoutFusion +from sglang.srt.layers.layer_boundary.facts import residual_facts from sglang.srt.layers.layer_boundary.layout import SumGroup from sglang.srt.layers.layer_boundary.residual import LayerResidualOps from sglang.srt.layers.layer_boundary.residual.add_norm import PLAIN_ADD @@ -187,6 +188,22 @@ class AttnBankState: ffn_update=PLAIN_ADD, ) + @classmethod + def facts(cls, *, reads_slices: bool = False) -> LayerResidualOps: + """What residual_ops() declares (see facts_of) for a layer whose bank + stays on each rank's attention-TP slice when ``reads_slices``. It does + not depend on the bank or the layer's modules, so the stages of a + layer that is not built declare it too.""" + state = cls( + bank=None, + attn_score_proj=None, + attn_score_norm=None, + ffn_score_proj=None, + ffn_score_norm=None, + reads_slices=reads_slices, + ) + return residual_facts(state.residual_ops()) + class _BankReadout: """A read that mixes the banked snapshots into the residual's norm, so it diff --git a/python/sglang/srt/layers/layer_boundary/residual/gated.py b/python/sglang/srt/layers/layer_boundary/residual/gated.py index ae176797c3..afa85675fa 100644 --- a/python/sglang/srt/layers/layer_boundary/residual/gated.py +++ b/python/sglang/srt/layers/layer_boundary/residual/gated.py @@ -18,6 +18,7 @@ from typing import Callable, Optional import torch +from sglang.srt.layers.layer_boundary.facts import residual_facts from sglang.srt.layers.layer_boundary.residual import LayerResidualOps from sglang.srt.runtime_context import get_parallel @@ -111,6 +112,22 @@ class GatedResidualState: ffn_update=_FfnUpdate(self), ) + @classmethod + def facts(cls, *, attn_reads_every_row: bool = False) -> LayerResidualOps: + """What residual_ops() declares (see facts_of) for a layer whose + attention read needs every row when ``attn_reads_every_row``. It does + not depend on the layer's modules, so the stages of a layer that is + not built declare it too.""" + state = cls( + expand=None, + attn_mix=None, + ffn_mix=None, + attn_combine=None, + ffn_combine=None, + attn_reads_every_row=attn_reads_every_row, + ) + return residual_facts(state.residual_ops()) + class _AttnReadout: """The normalized, gated mix of the streams, from streams that already hold diff --git a/python/sglang/srt/layers/layer_boundary/residual/mhc.py b/python/sglang/srt/layers/layer_boundary/residual/mhc.py index 18f799ee45..3ad6206faf 100644 --- a/python/sglang/srt/layers/layer_boundary/residual/mhc.py +++ b/python/sglang/srt/layers/layer_boundary/residual/mhc.py @@ -19,6 +19,7 @@ from typing import Callable, Optional import torch from sglang.kernels.ops.layernorm.mhc import hc_contract, hc_expand +from sglang.srt.layers.layer_boundary.facts import ReadFacts, UpdateFacts from sglang.srt.layers.layer_boundary.residual import LayerResidualOps from sglang.srt.runtime_context import get_parallel @@ -119,6 +120,19 @@ class MHCState: ffn_update=_FfnUpdate(self), ) + @staticmethod + def facts() -> LayerResidualOps: + """What residual_ops() declares (see facts_of): the class constants + of its reads and updates, the same for every layer whatever + parameters it holds, so the stages of a layer that is not built + declare them too.""" + return LayerResidualOps( + attn_readout=ReadFacts.of(_AttnReadout), + attn_update=UpdateFacts.of(_AttnUpdate), + ffn_readout=ReadFacts.of(_FfnReadout), + ffn_update=UpdateFacts.of(_FfnUpdate), + ) + class _AttnReadout: """hc_pre and the input norm, from streams that already hold the previous diff --git a/python/sglang/srt/layers/moe/fused_moe_triton/layer.py b/python/sglang/srt/layers/moe/fused_moe_triton/layer.py index 2654f7f0f3..1c09781a12 100644 --- a/python/sglang/srt/layers/moe/fused_moe_triton/layer.py +++ b/python/sglang/srt/layers/moe/fused_moe_triton/layer.py @@ -87,7 +87,6 @@ from sglang.srt.utils import ( is_npu, round_up, ) -from sglang.srt.utils.common import is_building_neighbour_layer from sglang.srt.utils.custom_op import register_custom_op _is_hip = is_hip() @@ -164,8 +163,6 @@ def create_moe_dispatcher( moe_runner_config: MoeRunnerConfig, quant_method: FusedMoEMethodBase, ) -> BaseDispatcher: - if is_building_neighbour_layer(): - return StandardDispatcher(moe_runner_config) a2a_backend = get_moe_a2a_backend() if a2a_backend.is_none() and is_npu(): return AscendTPDispatcher(moe_runner_config) diff --git a/python/sglang/srt/layers/moe/kt_ep_wrapper.py b/python/sglang/srt/layers/moe/kt_ep_wrapper.py index 563546840c..36c15cf139 100644 --- a/python/sglang/srt/layers/moe/kt_ep_wrapper.py +++ b/python/sglang/srt/layers/moe/kt_ep_wrapper.py @@ -20,7 +20,6 @@ from sglang.srt.runtime_context import ( get_schedule, ) from sglang.srt.utils import get_compiler_backend -from sglang.srt.utils.common import is_building_neighbour_layer if TYPE_CHECKING: from sglang.srt.layers.moe import MoeRunnerConfig @@ -216,7 +215,7 @@ class KTEPWrapperMethod(FusedMoEMethodBase): # 2. Initialize KT wrapper for CPU experts # CPU experts: num_gpu_experts to num_experts-1 - if self.tp_rank == 0 and not is_building_neighbour_layer(): + if self.tp_rank == 0: self.wrapper = KTMoEWrapper( layer_idx=self.kt_config.layer_idx, num_experts=num_experts, diff --git a/python/sglang/srt/models/apertus.py b/python/sglang/srt/models/apertus.py index 641cfbd68f..d281abe670 100644 --- a/python/sglang/srt/models/apertus.py +++ b/python/sglang/srt/models/apertus.py @@ -266,14 +266,18 @@ class ApertusDecoderLayer(nn.Module): self.feedforward_layernorm = RMSNorm( config.hidden_size, eps=config.rms_norm_eps ) + attn, ffn = self.stage_facts(layer_id) self.attn_boundary, self.ffn_boundary = append_stages( - (declare_attn(), self.attention_layernorm), - ( - declare_ffn(sparse=False, next_layer_sparse=False), - self.feedforward_layernorm, - ), + (attn, self.attention_layernorm), + (ffn, self.feedforward_layernorm), ) + @staticmethod + def stage_facts(layer_id: int): + """The stages every Apertus layer declares, an attention and a dense + FFN: the model's shared declaration function (see make_layers).""" + return (declare_attn(), declare_ffn(sparse=False, next_layer_sparse=False)) + def forward( self, positions: torch.Tensor, @@ -329,6 +333,7 @@ class ApertusModel(nn.Module): config=config, quant_config=quant_config, layer_id=idx, prefix=prefix ), prefix="model.layers", + stage_facts=ApertusDecoderLayer.stage_facts, ) if self.pp_group.is_last_rank: diff --git a/python/sglang/srt/models/arcee.py b/python/sglang/srt/models/arcee.py index 1624f18f46..d5d27b3188 100644 --- a/python/sglang/srt/models/arcee.py +++ b/python/sglang/srt/models/arcee.py @@ -242,14 +242,18 @@ class ArceeDecoderLayer(nn.Module): self.post_attention_layernorm = RMSNorm( config.hidden_size, eps=config.rms_norm_eps ) + attn, ffn = self.stage_facts(layer_id) self.attn_boundary, self.ffn_boundary = append_stages( - (declare_attn(), self.input_layernorm), - ( - declare_ffn(sparse=False, next_layer_sparse=False), - self.post_attention_layernorm, - ), + (attn, self.input_layernorm), + (ffn, self.post_attention_layernorm), ) + @staticmethod + def stage_facts(layer_id: int): + """The stages every Arcee layer declares, an attention and a dense FFN: + the model's shared declaration function (see make_layers).""" + return (declare_attn(), declare_ffn(sparse=False, next_layer_sparse=False)) + def forward( self, positions: torch.Tensor, @@ -303,6 +307,7 @@ class ArceeModel(nn.Module): config=config, quant_config=quant_config, layer_id=idx, prefix=prefix ), prefix="model.layers", + stage_facts=ArceeDecoderLayer.stage_facts, ) if self.pp_group.is_last_rank: diff --git a/python/sglang/srt/models/bailing_moe.py b/python/sglang/srt/models/bailing_moe.py index 25e7ed8ca0..fef8818a9d 100644 --- a/python/sglang/srt/models/bailing_moe.py +++ b/python/sglang/srt/models/bailing_moe.py @@ -655,12 +655,8 @@ class BailingMoEBlock(nn.Module): ) self.layer_id = layer_id - self.is_layer_sparse = self._is_layer_sparse( - config, layer_id=layer_id, is_nextn=is_nextn - ) - is_next_layer_sparse = self._is_layer_sparse( - config, layer_id=layer_id + 1, is_nextn=False - ) + attn, ffn = self.stage_facts(config, layer_id, is_nextn=is_nextn) + self.is_layer_sparse = ffn.sparse if self.is_layer_sparse: self.mlp = BailingMoESparseMoeBlock( @@ -684,18 +680,31 @@ class BailingMoEBlock(nn.Module): self.post_attention_layernorm = RMSNorm(hidden_size, eps=config.rms_norm_eps) self.attn_boundary, self.ffn_boundary = append_stages( - (declare_attn(), self.input_layernorm), - ( - declare_ffn( - sparse=self.is_layer_sparse, - next_layer_sparse=is_next_layer_sparse, + (attn, self.input_layernorm), + (ffn, self.post_attention_layernorm), + ) + + @classmethod + def stage_facts( + cls, config: PretrainedConfig, layer_id: int, is_nextn: bool = False + ): + """The stages the layer at ``layer_id`` declares, from the config: the + model's shared declaration function (see make_layers).""" + return ( + declare_attn(), + declare_ffn( + sparse=cls._is_layer_sparse( + config, layer_id=layer_id, is_nextn=is_nextn + ), + next_layer_sparse=cls._is_layer_sparse( + config, layer_id=layer_id + 1, is_nextn=False ), - self.post_attention_layernorm, ), ) + @staticmethod def _is_layer_sparse( - self, config: PretrainedConfig, layer_id: int, is_nextn: bool + config: PretrainedConfig, layer_id: int, is_nextn: bool ) -> bool: return is_nextn or ( config.num_experts is not None and layer_id >= config.first_k_dense_replace @@ -774,6 +783,7 @@ class BailingMoEModel(nn.Module): alt_stream=alt_stream, ), prefix=add_prefix("layers", prefix), + stage_facts=lambda idx: BailingMoEBlock.stage_facts(config, idx), ) if self.pp_group.is_last_rank: self.norm = RMSNorm(self.embed_dim, eps=config.rms_norm_eps) diff --git a/python/sglang/srt/models/bailing_moe_linear.py b/python/sglang/srt/models/bailing_moe_linear.py index f297615166..ce8f877058 100644 --- a/python/sglang/srt/models/bailing_moe_linear.py +++ b/python/sglang/srt/models/bailing_moe_linear.py @@ -779,8 +779,7 @@ class BailingMoELinearDecoderLayer(nn.Module): self.expert_num = config.num_experts self.hidden_size = config.hidden_size - is_moe_layer = self._is_layer_sparse(config, self.layer_id, is_nextn=is_nextn) - is_next_layer_moe_layer = self._is_layer_sparse(config, self.layer_id + 1) + attn, ffn = self.stage_facts(config, self.layer_id, is_nextn=is_nextn) if self.expert_num == 1: self.mlp = BailingMLP( hidden_size=self.hidden_size, @@ -818,22 +817,27 @@ class BailingMoELinearDecoderLayer(nn.Module): else None ) self.attn_boundary, self.ffn_boundary = append_stages( - ( - declare_attn(), - self.input_layernorm, - {"qkv_latent_func": qkv_latent_func}, - ), - ( - declare_ffn( - sparse=is_moe_layer, - next_layer_sparse=is_next_layer_moe_layer, - ), - self.post_attention_layernorm, + (attn, self.input_layernorm, {"qkv_latent_func": qkv_latent_func}), + (ffn, self.post_attention_layernorm), + ) + + @classmethod + def stage_facts( + cls, config: PretrainedConfig, layer_id: int, is_nextn: bool = False + ): + """The stages the layer at ``layer_id`` declares, from the config: the + model's shared declaration function (see make_layers).""" + return ( + declare_attn(), + declare_ffn( + sparse=cls._is_layer_sparse(config, layer_id, is_nextn=is_nextn), + next_layer_sparse=cls._is_layer_sparse(config, layer_id + 1), ), ) + @staticmethod def _is_layer_sparse( - self, config: PretrainedConfig, layer_id: int, is_nextn: bool = False + config: PretrainedConfig, layer_id: int, is_nextn: bool = False ) -> bool: return is_nextn or ( config.num_experts is not None and layer_id >= config.first_k_dense_replace @@ -945,6 +949,9 @@ class BailingMoELinearModel(nn.Module): self.num_layers, layer_fn, prefix=f"{prefix}.layers", + stage_facts=lambda idx: BailingMoELinearDecoderLayer.stage_facts( + config, idx + ), ) norm_kwargs = {} diff --git a/python/sglang/srt/models/bailing_moe_v3.py b/python/sglang/srt/models/bailing_moe_v3.py index 3cdc40ea4c..b67d4d8411 100644 --- a/python/sglang/srt/models/bailing_moe_v3.py +++ b/python/sglang/srt/models/bailing_moe_v3.py @@ -1073,14 +1073,8 @@ class BailingMoELinearDecoderLayer(nn.Module): self.expert_num = config.num_experts self.hidden_size = config.hidden_size - is_moe_layer = is_nextn or ( - not (self.expert_num == 1) - and (self.layer_id >= config.first_k_dense_replace) - ) - self.is_layer_sparse = is_moe_layer - is_next_layer_sparse = not (self.expert_num == 1) and ( - self.layer_id + 1 >= config.first_k_dense_replace - ) + attn, ffn = self.stage_facts(config, self.layer_id, is_nextn=is_nextn) + self.is_layer_sparse = ffn.sparse mlp_parallel_group = "replicated" if is_dense_ffn_fully_dp() else "tp" if self.expert_num == 1: @@ -1119,7 +1113,7 @@ class BailingMoELinearDecoderLayer(nn.Module): self.attn_boundary, self.ffn_boundary = append_stages( ( - declare_attn(), + attn, self.input_layernorm, { "qkv_latent_func": self.attention.prepare_qkv_latent @@ -1127,12 +1121,24 @@ class BailingMoELinearDecoderLayer(nn.Module): else None }, ), - ( - declare_ffn( - sparse=is_moe_layer, - next_layer_sparse=is_next_layer_sparse, - ), - self.post_attention_layernorm, + (ffn, self.post_attention_layernorm), + ) + + @staticmethod + def stage_facts(config: PretrainedConfig, layer_id: int, is_nextn: bool = False): + """The stages the layer at ``layer_id`` declares, from the config: the + model's shared declaration function (see make_layers).""" + is_moe_layer = is_nextn or ( + not (config.num_experts == 1) and (layer_id >= config.first_k_dense_replace) + ) + is_next_layer_sparse = not (config.num_experts == 1) and ( + layer_id + 1 >= config.first_k_dense_replace + ) + return ( + declare_attn(), + declare_ffn( + sparse=is_moe_layer, + next_layer_sparse=is_next_layer_sparse, ), ) @@ -1265,6 +1271,9 @@ class BailingMoELinearModel(nn.Module): self.num_layers, layer_fn, prefix=f"{prefix}.layers", + stage_facts=lambda idx: BailingMoELinearDecoderLayer.stage_facts( + config, idx + ), ) linear_layer_nums = sum( diff --git a/python/sglang/srt/models/deepseek_v2.py b/python/sglang/srt/models/deepseek_v2.py index 9bc5c84ddb..e5d85047b9 100644 --- a/python/sglang/srt/models/deepseek_v2.py +++ b/python/sglang/srt/models/deepseek_v2.py @@ -2590,10 +2590,9 @@ class DeepseekV2DecoderLayer(nn.Module): "SGLANG_USE_AG_AFTER_QLORA only supports the model with q_lora_rank" ) - self.is_layer_sparse = self._is_layer_sparse(layer_id, is_nextn=is_nextn) - is_next_layer_sparse = self._is_layer_sparse(layer_id + 1, is_nextn=False) - - self._stage_next_sparse = is_next_layer_sparse + self.is_layer_sparse = self._is_layer_sparse( + config, layer_id, is_nextn=is_nextn + ) if self.is_layer_sparse: self.mlp = DeepseekV2MoE( @@ -2634,18 +2633,42 @@ class DeepseekV2DecoderLayer(nn.Module): qkv_latent_func=self.self_attn.prepare_qkv_latent, ) + @classmethod + def stage_facts( + cls, + config: PretrainedConfig, + layer_id: int, + *, + is_nextn: bool = False, + attn_output=None, + output=None, + ): + """The stages a layer of this class declares, from the config alone: + the model's shared declaration function, which the layer declares + with too (see make_layers). ``attn_output`` and ``output``: the + attention's and the FFN's output transforms, if any.""" + return ( + declare_attn(output_transform=attn_output), + declare_ffn( + sparse=cls._is_layer_sparse(config, layer_id, is_nextn=is_nextn), + next_layer_sparse=cls._is_layer_sparse( + config, layer_id + 1, is_nextn=False + ), + output_transform=output, + ), + ) + def _build_stages( self, *, input_layernorm: nn.Module, post_attention_layernorm: nn.Module, qkv_latent_func: Optional[Callable], - output=None, - attn_output=None, + **modules, ): """The stage boundaries for this layer's norms; they choose their - steps from them at construction. ``attn_output`` and ``output``: the - attention's and the FFN's output transforms, if any.""" + steps from them at construction. ``modules``: what this class's + stage_facts binds the stages' output transforms to, if anything.""" fusions = None if ( not get_parallel().enable_prefill_cp @@ -2656,21 +2679,16 @@ class DeepseekV2DecoderLayer(nn.Module): from sglang.srt.layers.layer_boundary.fusions.cutedsl import CuteDSLFusion fusions = CuteDSLFusion() + attn, ffn = self.stage_facts( + self.config, self.layer_id, is_nextn=self.is_nextn, **modules + ) attn_boundary, ffn_boundary = append_stages( ( - declare_attn(output_transform=attn_output), + attn, input_layernorm, {"qkv_latent_func": qkv_latent_func, "fusions": fusions}, ), - ( - declare_ffn( - sparse=self.is_layer_sparse, - next_layer_sparse=self._stage_next_sparse, - output_transform=output, - ), - post_attention_layernorm, - {"fusions": fusions}, - ), + (ffn, post_attention_layernorm, {"fusions": fusions}), ) return attn_boundary, ffn_boundary @@ -2706,11 +2724,14 @@ class DeepseekV2DecoderLayer(nn.Module): self._gfx95_quant_format = fmt return fmt - def _is_layer_sparse(self, layer_id: int, is_nextn: bool) -> bool: + @staticmethod + def _is_layer_sparse( + config: PretrainedConfig, layer_id: int, is_nextn: bool + ) -> bool: return is_nextn or ( - self.config.n_routed_experts is not None - and layer_id >= self.config.first_k_dense_replace - and layer_id % self.config.moe_layer_freq == 0 + config.n_routed_experts is not None + and layer_id >= config.first_k_dense_replace + and layer_id % config.moe_layer_freq == 0 ) def forward( @@ -2916,6 +2937,7 @@ class DeepseekV2Model(nn.Module): else [] ), ), + stage_facts=lambda idx: DeepseekV2DecoderLayer.stage_facts(config, idx), ) local_layer_ids = list(range(self.start_layer, self.end_layer)) diff --git a/python/sglang/srt/models/dots3_common/modeling.py b/python/sglang/srt/models/dots3_common/modeling.py index a7c13da00e..a06112f2c4 100644 --- a/python/sglang/srt/models/dots3_common/modeling.py +++ b/python/sglang/srt/models/dots3_common/modeling.py @@ -1377,8 +1377,8 @@ class Dots3DecoderLayer(nn.Module): prefix=add_prefix("self_attn", prefix), alt_stream=alt_stream, ) - self.is_layer_sparse = self._is_layer_sparse(layer_id, is_nextn=is_nextn) - is_next_layer_sparse = self._is_layer_sparse(layer_id + 1, is_nextn=False) + attn, ffn = self.stage_facts(config, layer_id, is_nextn=is_nextn) + self.is_layer_sparse = ffn.sparse if self.is_layer_sparse: self.mlp = Dots3MoE( @@ -1407,24 +1407,37 @@ class Dots3DecoderLayer(nn.Module): ) self.attn_boundary, self.ffn_boundary = append_stages( - (declare_attn(), self.input_layernorm), - ( - declare_ffn( - sparse=self.is_layer_sparse, - next_layer_sparse=is_next_layer_sparse, + (attn, self.input_layernorm), + (ffn, self.post_attention_layernorm), + ) + + @classmethod + def stage_facts( + cls, config: PretrainedConfig, layer_id: int, is_nextn: bool = False + ): + """The stages the layer at ``layer_id`` declares, from the config: the + model's shared declaration function (see make_layers).""" + return ( + declare_attn(), + declare_ffn( + sparse=cls._is_layer_sparse(config, layer_id, is_nextn=is_nextn), + next_layer_sparse=cls._is_layer_sparse( + config, layer_id + 1, is_nextn=False ), - self.post_attention_layernorm, ), ) - def _is_layer_sparse(self, layer_id: int, is_nextn: bool) -> bool: + @staticmethod + def _is_layer_sparse( + config: PretrainedConfig, layer_id: int, is_nextn: bool + ) -> bool: # The MTP block uses a dense MLP. if is_nextn: return False return ( - self.config.n_routed_experts is not None - and layer_id >= self.config.first_k_dense_replace - and layer_id % self.config.moe_layer_freq == 0 + config.n_routed_experts is not None + and layer_id >= config.first_k_dense_replace + and layer_id % config.moe_layer_freq == 0 ) def forward( @@ -1485,6 +1498,7 @@ class Dots3Model(nn.Module): alt_stream=self.alt_stream, ), prefix=add_prefix("layers", prefix), + stage_facts=lambda idx: Dots3DecoderLayer.stage_facts(config, idx), ) if self.pp_group.is_last_rank: self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps) diff --git a/python/sglang/srt/models/ernie45_moe_vl.py b/python/sglang/srt/models/ernie45_moe_vl.py index 562f0d357a..29761990b4 100644 --- a/python/sglang/srt/models/ernie45_moe_vl.py +++ b/python/sglang/srt/models/ernie45_moe_vl.py @@ -412,8 +412,8 @@ class Ernie4_5_VLMoeDecoderLayer(nn.Module): # MoE # MLP - sparse = _is_moe_layer(config, layer_id) - if sparse: + attn, ffn = self.stage_facts(config, layer_id) + if ffn.sparse: self.mlp = Ernie4_5_VLMoeMoE( config=config, layer_id=layer_id, @@ -435,15 +435,22 @@ class Ernie4_5_VLMoeDecoderLayer(nn.Module): self.post_attention_layernorm = RMSNorm( config.hidden_size, eps=config.rms_norm_eps ) + self.attn_boundary, self.ffn_boundary = append_stages( + (attn, self.input_layernorm), + (ffn, self.post_attention_layernorm), + ) + + @staticmethod + def stage_facts(config: PretrainedConfig, layer_id: int): + """The stages the layer at ``layer_id`` declares, from the config: the + model's shared declaration function (see make_layers).""" + sparse = _is_moe_layer(config, layer_id) next_sparse = layer_id + 1 < config.num_hidden_layers and _is_moe_layer( config, layer_id + 1 ) - self.attn_boundary, self.ffn_boundary = append_stages( - (declare_attn(), self.input_layernorm), - ( - declare_ffn(sparse=sparse, next_layer_sparse=next_sparse), - self.post_attention_layernorm, - ), + return ( + declare_attn(), + declare_ffn(sparse=sparse, next_layer_sparse=next_sparse), ) def forward( @@ -502,6 +509,7 @@ class Ernie4_5_VLMoeModel(nn.Module): prefix=prefix, ), prefix=add_prefix("layers", prefix), + stage_facts=lambda idx: Ernie4_5_VLMoeDecoderLayer.stage_facts(config, idx), ) if self.pp_group.is_last_rank: self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps) diff --git a/python/sglang/srt/models/exaone4.py b/python/sglang/srt/models/exaone4.py index e0a1ed5935..20d58e68f2 100644 --- a/python/sglang/srt/models/exaone4.py +++ b/python/sglang/srt/models/exaone4.py @@ -267,25 +267,38 @@ class Exaone4DecoderLayer(nn.Module): self.post_feedforward_layernorm = RMSNorm( self.hidden_size, eps=config.rms_norm_eps ) + attn, ffn = self.stage_facts( + config, + layer_id, + post_attention_layernorm=self.post_attention_layernorm, + post_feedforward_layernorm=self.post_feedforward_layernorm, + ) + self.attn_boundary, self.ffn_boundary = append_stages((attn, None), (ffn, None)) + + @staticmethod + def stage_facts( + config: Exaone4Config, + layer_id: int, + *, + post_attention_layernorm=None, + post_feedforward_layernorm=None, + ): + """The stages an EXAONE 4.0 layer declares, from the config alone: the + model's shared declaration function, which the layer declares with + too (see make_layers). The output norms are what the updates add.""" # Post-LN: each stage reads the residual as it is, and its output is # normalized before it is added. The layer writes the FFN's itself. - ffn_update = PostNormAdd(self.post_feedforward_layernorm, applied_at_exit=True) - self.attn_boundary, self.ffn_boundary = append_stages( - ( - declare_attn( - read=PLAIN_READOUT, - update=PostNormAdd(self.post_attention_layernorm), - ), - None, + ffn_update = PostNormAdd(post_feedforward_layernorm, applied_at_exit=True) + return ( + declare_attn( + read=PLAIN_READOUT, + update=PostNormAdd(post_attention_layernorm), ), - ( - declare_ffn( - sparse=False, - next_layer_sparse=False, - read=PLAIN_READOUT, - update=ffn_update, - ), - None, + declare_ffn( + sparse=False, + next_layer_sparse=False, + read=PLAIN_READOUT, + update=ffn_update, ), ) @@ -341,6 +354,7 @@ class Exaone4Model(nn.Module): prefix=prefix, ), prefix=add_prefix("layers", prefix), + stage_facts=lambda idx: Exaone4DecoderLayer.stage_facts(config, idx), ) if self.pp_group.is_last_rank: self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps) diff --git a/python/sglang/srt/models/exaone_moe.py b/python/sglang/srt/models/exaone_moe.py index 503d1c6e9f..4d759482ae 100755 --- a/python/sglang/srt/models/exaone_moe.py +++ b/python/sglang/srt/models/exaone_moe.py @@ -464,7 +464,8 @@ class ExaoneMoEDecoderLayer(nn.Module): prefix=add_prefix("self_attn", prefix), ) - if config.is_moe_layer[layer_id]: + attn, ffn = self.stage_facts(config, layer_id) + if ffn.sparse: self.mlp = ExaoneMoESparseMoEBlock( layer_id=layer_id, config=config, @@ -485,17 +486,23 @@ class ExaoneMoEDecoderLayer(nn.Module): self.post_attention_layernorm = RMSNorm( config.hidden_size, eps=config.rms_norm_eps ) + self.attn_boundary, self.ffn_boundary = append_stages( + (attn, self.input_layernorm), + (ffn, self.post_attention_layernorm), + ) + + @staticmethod + def stage_facts(config: PretrainedConfig, layer_id: int): + """The stages the layer at ``layer_id`` declares, from the config: the + model's shared declaration function (see make_layers).""" is_moe_layer = config.is_moe_layer num_layers = config.num_hidden_layers - self.attn_boundary, self.ffn_boundary = append_stages( - (declare_attn(), self.input_layernorm), - ( - declare_ffn( - sparse=is_moe_layer[layer_id], - next_layer_sparse=layer_id + 1 < num_layers - and is_moe_layer[layer_id + 1], - ), - self.post_attention_layernorm, + return ( + declare_attn(), + declare_ffn( + sparse=is_moe_layer[layer_id], + next_layer_sparse=layer_id + 1 < num_layers + and is_moe_layer[layer_id + 1], ), ) @@ -559,6 +566,7 @@ class ExaoneMoEModel(nn.Module): alt_stream=alt_stream, ), prefix=add_prefix("layers", prefix), + stage_facts=lambda idx: ExaoneMoEDecoderLayer.stage_facts(config, idx), ) if self.pp_group.is_last_rank: diff --git a/python/sglang/srt/models/gemma4_causal.py b/python/sglang/srt/models/gemma4_causal.py index 67e6ee7429..f0983560b9 100644 --- a/python/sglang/srt/models/gemma4_causal.py +++ b/python/sglang/srt/models/gemma4_causal.py @@ -636,7 +636,7 @@ class Gemma4DecoderLayer(nn.Module): self.post_per_layer_input_norm = None # Parallel MoE - self.enable_moe_block = getattr(config, "enable_moe_block", False) + self.enable_moe_block = self._enable_moe_block(config) if self.enable_moe_block: self.router = Gemma4Router( config, @@ -671,20 +671,37 @@ class Gemma4DecoderLayer(nn.Module): self.register_buffer("layer_scalar", torch.ones(1), persistent=True) self.has_ple = self.hidden_size_per_layer_input > 0 self.prefix = prefix + attn, ffn = self.stage_facts( + config, layer_id, post_attention_layernorm=self.post_attention_layernorm + ) self.attn_boundary, self.ffn_boundary = append_stages( - ( - declare_attn( - output_transform=OutputTransform(self.post_attention_layernorm) - ), - self.input_layernorm, - ), - ( - declare_ffn( - sparse=self.enable_moe_block, - next_layer_sparse=self.enable_moe_block, - update=REPLACE_AT_EXIT, - ), - self.pre_feedforward_layernorm, + (attn, self.input_layernorm), + (ffn, self.pre_feedforward_layernorm), + ) + + @staticmethod + def _enable_moe_block(config: PretrainedConfig) -> bool: + return getattr(config, "enable_moe_block", False) + + @classmethod + def stage_facts( + cls, + config: PretrainedConfig, + layer_id: int, + *, + post_attention_layernorm=None, + ): + """The stages a Gemma 4 layer declares, from the config alone: the + model's shared declaration function, which the layer declares with + too (see make_layers). The attention's output norm is its output + transform.""" + enable_moe_block = cls._enable_moe_block(config) + return ( + declare_attn(output_transform=OutputTransform(post_attention_layernorm)), + declare_ffn( + sparse=enable_moe_block, + next_layer_sparse=enable_moe_block, + update=REPLACE_AT_EXIT, ), ) @@ -919,6 +936,7 @@ class Gemma4TextModel(PreTrainedModel): prefix=prefix, ), prefix=add_prefix("layers", prefix), + stage_facts=lambda idx: Gemma4DecoderLayer.stage_facts(config, idx), ) if self.pp_group.is_last_rank: diff --git a/python/sglang/srt/models/gigachat35.py b/python/sglang/srt/models/gigachat35.py index 13dbf0612d..1d643bfc60 100644 --- a/python/sglang/srt/models/gigachat35.py +++ b/python/sglang/srt/models/gigachat35.py @@ -410,9 +410,7 @@ class GigaChat35DecoderLayer(deepseek_v2.DeepseekV2DecoderLayer): if swiglu_limit > 0 and runner_config is not None: runner_config.gemm1_clamp_limit = swiglu_limit - layernorm_type = getattr(config, "layernorm_type", "pre") - self._use_pre = layernorm_type in ("pre", "pre_post") - self._use_post = layernorm_type in ("post", "pre_post") + self._use_pre, self._use_post = self._norm_placement(config) self.input_layernorm = build_norm(config, config.hidden_size) self.post_attention_layernorm = build_norm(config, config.hidden_size) @@ -434,25 +432,55 @@ class GigaChat35DecoderLayer(deepseek_v2.DeepseekV2DecoderLayer): input_layernorm=attn_prepare_layernorm, post_attention_layernorm=mlp_prepare_layernorm, qkv_latent_func=qkv_latent_func, - attn_output=( - OutputTransform(self.post_self_attn_layernorm) - if self.post_self_attn_layernorm is not None - else None - ), - output=( - OutputTransform(self.post_feedforward_layernorm) - if self.post_feedforward_layernorm is not None - else None - ), + post_self_attn_layernorm=self.post_self_attn_layernorm, + post_feedforward_layernorm=self.post_feedforward_layernorm, ) - def _is_layer_sparse(self, layer_id: int, is_nextn: bool) -> bool: - if is_nextn: - return bool(getattr(self.config, "nextn_is_sparse", False)) + @staticmethod + def _norm_placement(config: GigaChat35Config): + """Whether the layer norms each sublayer's input, and its output.""" + layernorm_type = getattr(config, "layernorm_type", "pre") return ( - self.config.n_routed_experts is not None - and layer_id >= self.config.first_k_dense_replace - and layer_id % self.config.moe_layer_freq == 0 + layernorm_type in ("pre", "pre_post"), + layernorm_type in ("post", "pre_post"), + ) + + @classmethod + def stage_facts( + cls, + config: GigaChat35Config, + layer_id: int, + *, + is_nextn: bool = False, + post_self_attn_layernorm=None, + post_feedforward_layernorm=None, + ): + """The stages a GigaChat 3.5 layer declares, from the config alone: + the model's shared declaration function, which the layer declares + with too (see make_layers). The output norms, which the layer has + only when it norms the sublayers' outputs, are the output + transforms.""" + _, use_post = cls._norm_placement(config) + return super().stage_facts( + config, + layer_id, + is_nextn=is_nextn, + attn_output=( + OutputTransform(post_self_attn_layernorm) if use_post else None + ), + output=OutputTransform(post_feedforward_layernorm) if use_post else None, + ) + + @staticmethod + def _is_layer_sparse( + config: GigaChat35Config, layer_id: int, is_nextn: bool + ) -> bool: + if is_nextn: + return bool(getattr(config, "nextn_is_sparse", False)) + return ( + config.n_routed_experts is not None + and layer_id >= config.first_k_dense_replace + and layer_id % config.moe_layer_freq == 0 ) def forward( @@ -527,6 +555,7 @@ class GigaChat35Model(nn.Module): alt_stream=self.alt_stream, ), prefix=add_prefix("layers", prefix), + stage_facts=lambda idx: GigaChat35DecoderLayer.stage_facts(config, idx), ) if self.pp_group.is_last_rank: diff --git a/python/sglang/srt/models/glm4.py b/python/sglang/srt/models/glm4.py index 8695046f0a..7c79b8f99c 100644 --- a/python/sglang/srt/models/glm4.py +++ b/python/sglang/srt/models/glm4.py @@ -264,20 +264,34 @@ class Glm4DecoderLayer(nn.Module): config.hidden_size, eps=config.rms_norm_eps ) self.post_mlp_layernorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps) + attn, ffn = self.stage_facts( + config, + layer_id, + post_self_attn_layernorm=self.post_self_attn_layernorm, + post_mlp_layernorm=self.post_mlp_layernorm, + ) self.attn_boundary, self.ffn_boundary = append_stages( - ( - declare_attn( - output_transform=OutputTransform(self.post_self_attn_layernorm) - ), - self.input_layernorm, - ), - ( - declare_ffn( - sparse=False, - next_layer_sparse=False, - output_transform=OutputTransform(self.post_mlp_layernorm), - ), - self.post_attention_layernorm, + (attn, self.input_layernorm), + (ffn, self.post_attention_layernorm), + ) + + @staticmethod + def stage_facts( + config: Glm4Config, + layer_id: int, + *, + post_self_attn_layernorm=None, + post_mlp_layernorm=None, + ): + """The stages a GLM-4 layer declares, from the config alone: the + model's shared declaration function, which the layer declares with + too (see make_layers). Its output norms are the output transforms.""" + return ( + declare_attn(output_transform=OutputTransform(post_self_attn_layernorm)), + declare_ffn( + sparse=False, + next_layer_sparse=False, + output_transform=OutputTransform(post_mlp_layernorm), ), ) @@ -344,6 +358,7 @@ class Glm4Model(nn.Module): alt_stream=alt_stream, ), prefix=add_prefix("layers", prefix), + stage_facts=lambda idx: decoder_layer_type.stage_facts(config, idx), ) if self.pp_group.is_last_rank: self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps) diff --git a/python/sglang/srt/models/glm4_moe.py b/python/sglang/srt/models/glm4_moe.py index 03e3d0a7b5..fc8ebbb43e 100644 --- a/python/sglang/srt/models/glm4_moe.py +++ b/python/sglang/srt/models/glm4_moe.py @@ -734,8 +734,8 @@ class Glm4MoeDecoderLayer(nn.Module): alt_stream=alt_stream, ) - self.is_layer_sparse = self._is_layer_sparse(layer_id, is_nextn=is_nextn) - is_next_layer_sparse = self._is_layer_sparse(layer_id + 1, is_nextn=False) + attn, ffn = self.stage_facts(config, layer_id, is_nextn=is_nextn) + self.is_layer_sparse = ffn.sparse if self.is_layer_sparse: self.mlp = Glm4MoeSparseMoeBlock( @@ -763,14 +763,8 @@ class Glm4MoeDecoderLayer(nn.Module): ) self.attn_boundary, self.ffn_boundary = append_stages( - (declare_attn(), self.input_layernorm), - ( - declare_ffn( - sparse=self.is_layer_sparse, - next_layer_sparse=is_next_layer_sparse, - ), - self.post_attention_layernorm, - ), + (attn, self.input_layernorm), + (ffn, self.post_attention_layernorm), ) # Detect if QKV uses aiter FP8 per-token quant so we can fuse @@ -818,10 +812,29 @@ class Glm4MoeDecoderLayer(nn.Module): self.self_attn.qkv_proj, "attn" ) - def _is_layer_sparse(self, layer_id: int, is_nextn: bool) -> bool: + @classmethod + def stage_facts( + cls, config: PretrainedConfig, layer_id: int, is_nextn: bool = False + ): + """The stages the layer at ``layer_id`` declares, from the config: the + model's shared declaration function (see make_layers).""" + return ( + declare_attn(), + declare_ffn( + sparse=cls._is_layer_sparse(config, layer_id, is_nextn=is_nextn), + next_layer_sparse=cls._is_layer_sparse( + config, layer_id + 1, is_nextn=False + ), + ), + ) + + @staticmethod + def _is_layer_sparse( + config: PretrainedConfig, layer_id: int, is_nextn: bool + ) -> bool: return is_nextn or ( - self.config.n_routed_experts is not None - and layer_id >= self.config.first_k_dense_replace + config.n_routed_experts is not None + and layer_id >= config.first_k_dense_replace ) def forward( @@ -893,6 +906,7 @@ class Glm4MoeModel(nn.Module): alt_stream=self.alt_stream, ), prefix=add_prefix("layers", prefix), + stage_facts=lambda idx: Glm4MoeDecoderLayer.stage_facts(config, idx), ) if self.pp_group.is_last_rank: self.norm = RMSNorm(self.embed_dim, eps=config.rms_norm_eps) diff --git a/python/sglang/srt/models/glm4_moe_lite.py b/python/sglang/srt/models/glm4_moe_lite.py index ba81f79366..01e196281a 100644 --- a/python/sglang/srt/models/glm4_moe_lite.py +++ b/python/sglang/srt/models/glm4_moe_lite.py @@ -447,8 +447,8 @@ class Glm4MoeLiteDecoderLayer(nn.Module): prefix=add_prefix("self_attn", prefix), ) - self.is_layer_sparse = self._is_layer_sparse(layer_id, is_nextn=is_nextn) - is_next_layer_sparse = self._is_layer_sparse(layer_id + 1, is_nextn=False) + attn, ffn = self.stage_facts(config, layer_id, is_nextn=is_nextn) + self.is_layer_sparse = ffn.sparse if self.is_layer_sparse: self.mlp = Glm4MoeLiteSparseMoeBlock( @@ -480,17 +480,11 @@ class Glm4MoeLiteDecoderLayer(nn.Module): self.attn_boundary, self.ffn_boundary = append_stages( ( - declare_attn(), + attn, self.input_layernorm, {"qkv_latent_func": self.self_attn.prepare_qkv_latent}, ), - ( - declare_ffn( - sparse=self.is_layer_sparse, - next_layer_sparse=is_next_layer_sparse, - ), - self.post_attention_layernorm, - ), + (ffn, self.post_attention_layernorm), ) def _detect_gfx95_quant_format(self) -> str: @@ -509,11 +503,30 @@ class Glm4MoeLiteDecoderLayer(nn.Module): return "fp8" return "" - def _is_layer_sparse(self, layer_id: int, is_nextn: bool) -> bool: + @classmethod + def stage_facts( + cls, config: PretrainedConfig, layer_id: int, is_nextn: bool = False + ): + """The stages the layer at ``layer_id`` declares, from the config: the + model's shared declaration function (see make_layers).""" + return ( + declare_attn(), + declare_ffn( + sparse=cls._is_layer_sparse(config, layer_id, is_nextn=is_nextn), + next_layer_sparse=cls._is_layer_sparse( + config, layer_id + 1, is_nextn=False + ), + ), + ) + + @staticmethod + def _is_layer_sparse( + config: PretrainedConfig, layer_id: int, is_nextn: bool + ) -> bool: return is_nextn or ( - self.config.n_routed_experts is not None - and layer_id >= self.config.first_k_dense_replace - and layer_id % self.config.moe_layer_freq == 0 + config.n_routed_experts is not None + and layer_id >= config.first_k_dense_replace + and layer_id % config.moe_layer_freq == 0 ) def forward( @@ -586,6 +599,7 @@ class Glm4MoeLiteModel(nn.Module): alt_stream=self.alt_stream, ), prefix=add_prefix("layers", prefix), + stage_facts=lambda idx: Glm4MoeLiteDecoderLayer.stage_facts(config, idx), ) if self.pp_group.is_last_rank: self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps) diff --git a/python/sglang/srt/models/glm5_next.py b/python/sglang/srt/models/glm5_next.py index 7fce8b7393..40f4ea3501 100644 --- a/python/sglang/srt/models/glm5_next.py +++ b/python/sglang/srt/models/glm5_next.py @@ -743,8 +743,9 @@ class Glm5NextDecoderLayer(nn.Module): "SGLANG_USE_AG_AFTER_QLORA only supports the model with q_lora_rank" ) - self.is_layer_sparse = self._is_layer_sparse(layer_id, is_nextn=is_nextn) - is_next_layer_sparse = self._is_layer_sparse(layer_id + 1, is_nextn=False) + self.is_layer_sparse = self._is_layer_sparse( + config, layer_id, is_nextn=is_nextn + ) if self.is_layer_sparse: self.mlp = Glm5NextMoE( @@ -795,9 +796,9 @@ class Glm5NextDecoderLayer(nn.Module): ) terminal = layer_id == (1 if is_nextn else config.num_hidden_layers) - 1 - residual = PLAIN_RESIDUAL_OPS + mhc = None if self.config.mhc: - residual = MHCState( + mhc = MHCState( hc_mult=config.hc_mult, hc_attn_pre=self.hc_attn_pre, hc_ffn_pre=self.hc_ffn_pre, @@ -809,15 +810,10 @@ class Glm5NextDecoderLayer(nn.Module): ), is_last_layer=terminal, ).residual_ops() + attn, ffn = self.stage_facts(config, layer_id, is_nextn=is_nextn, mhc=mhc) self.attn_boundary, self.ffn_boundary = append_stages( ( - declare_attn( - read=residual.attn_readout, - update=residual.attn_update, - tp_group=SumGroup.TP - if self.is_linear_attn and get_parallel().enable_cp_tp_group_sharing - else SumGroup.ATTN_TP, - ), + attn, self.input_layernorm, { "qkv_latent_func": self.self_attn.prepare_qkv_latent @@ -825,14 +821,37 @@ class Glm5NextDecoderLayer(nn.Module): else None, }, ), - ( - declare_ffn( - sparse=self.is_layer_sparse, - next_layer_sparse=is_next_layer_sparse, - read=residual.ffn_readout, - update=residual.ffn_update, + (ffn, self.post_attention_layernorm), + ) + + @classmethod + def stage_facts(cls, config, layer_id, *, is_nextn=False, mhc=None): + """The attention and FFN stages the layer at ``layer_id`` declares: + the model's shared declaration function (see make_layers), which the + layer declares itself with. The config selects the residual: a plain + one, or hyper-connection streams, read and updated by ``mhc``, the + layer's own reads and updates of them; without it, the stages + declare what those do.""" + if config.mhc: + residual = MHCState.facts() if mhc is None else mhc + else: + residual = PLAIN_RESIDUAL_OPS + return ( + declare_attn( + read=residual.attn_readout, + update=residual.attn_update, + tp_group=SumGroup.TP + if config.is_kda_layer(layer_id) + and get_parallel().enable_cp_tp_group_sharing + else SumGroup.ATTN_TP, + ), + declare_ffn( + sparse=cls._is_layer_sparse(config, layer_id, is_nextn=is_nextn), + next_layer_sparse=cls._is_layer_sparse( + config, layer_id + 1, is_nextn=False ), - self.post_attention_layernorm, + read=residual.ffn_readout, + update=residual.ffn_update, ), ) @@ -924,11 +943,12 @@ class Glm5NextDecoderLayer(nn.Module): hc_mult=self.config.hc_mult, ) - def _is_layer_sparse(self, layer_id: int, is_nextn: bool) -> bool: + @staticmethod + def _is_layer_sparse(config, layer_id: int, is_nextn: bool) -> bool: return is_nextn or ( - self.config.n_routed_experts is not None - and layer_id >= self.config.first_k_dense_replace - and layer_id % self.config.moe_layer_freq == 0 + config.n_routed_experts is not None + and layer_id >= config.first_k_dense_replace + and layer_id % config.moe_layer_freq == 0 ) def forward( @@ -1035,6 +1055,7 @@ class Glm5NextModel(nn.Module): alt_stream=self.alt_stream, ), prefix=add_prefix("layers", prefix), + stage_facts=lambda idx: Glm5NextDecoderLayer.stage_facts(config, idx), ) if self.pp_group.is_last_rank: self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps) diff --git a/python/sglang/srt/models/gpt_oss.py b/python/sglang/srt/models/gpt_oss.py index 2dc60d7889..49088157f8 100644 --- a/python/sglang/srt/models/gpt_oss.py +++ b/python/sglang/srt/models/gpt_oss.py @@ -557,9 +557,8 @@ class GptOssDecoderLayer(nn.Module): self.layer_id = layer_id - # GptOss all layers are sparse - self.is_layer_sparse = True - is_next_layer_sparse = True + attn, ffn = self.stage_facts(config, layer_id) + self.is_layer_sparse = ffn.sparse if self.is_layer_sparse: self.mlp = GptOssSparseMoeBlock( @@ -590,13 +589,22 @@ class GptOssDecoderLayer(nn.Module): ) self.attn_boundary, self.ffn_boundary = append_stages( - (declare_attn(), self.input_layernorm), - ( - declare_ffn( - sparse=self.is_layer_sparse, - next_layer_sparse=is_next_layer_sparse, - ), - self.post_attention_layernorm, + (attn, self.input_layernorm), + (ffn, self.post_attention_layernorm), + ) + + @staticmethod + def stage_facts(config: GptOssConfig, layer_id: int): + """The stages the layer at ``layer_id`` declares: the model's shared + declaration function (see make_layers).""" + # GptOss all layers are sparse + is_layer_sparse = True + is_next_layer_sparse = True + return ( + declare_attn(), + declare_ffn( + sparse=is_layer_sparse, + next_layer_sparse=is_next_layer_sparse, ), ) @@ -664,6 +672,7 @@ class GptOssModel(nn.Module): prefix=prefix, ), prefix=add_prefix("layers", prefix), + stage_facts=lambda idx: decoder_layer_type.stage_facts(config, idx), ) if self.pp_group.is_last_rank: self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps) diff --git a/python/sglang/srt/models/granitemoehybrid.py b/python/sglang/srt/models/granitemoehybrid.py index 42f2706835..bd7493c694 100644 --- a/python/sglang/srt/models/granitemoehybrid.py +++ b/python/sglang/srt/models/granitemoehybrid.py @@ -1,3 +1,4 @@ +from functools import partial from typing import Iterable, Optional import torch @@ -40,26 +41,34 @@ from sglang.srt.utils import make_pp_layers from .granitemoe import GraniteMoeMoE, GraniteMoeSharedMLP -def _make_stages(layer): - """Both kinds of layer scale their mixer and FFN outputs by - residual_multiplier, add them in the activation dtype and normalize the sum - as a separate step.""" - scale = OutputTransform(layer._scale_output) - sparse = layer.block_sparse_moe is not None +def _has_moe(config: GraniteMoeHybridConfig) -> bool: + return getattr(config, "num_local_experts", 0) > 0 + + +def stage_facts(config: GraniteMoeHybridConfig, layer_idx: int, scale=None): + """The stages a layer of either kind declares, from the config alone: the + model's shared declaration function, which the layers declare with too + (see make_layers). Both kinds of layer scale their mixer and FFN outputs + by residual_multiplier (``scale``), add them in the activation dtype and + normalize the sum as a separate step.""" + scale = OutputTransform(scale) + sparse = _has_moe(config) + return ( + declare_attn(read=UNFUSED_NORM_READOUT, output_transform=scale), + declare_ffn( + sparse=sparse, + next_layer_sparse=sparse, + read=UNFUSED_NORM_READOUT, + output_transform=scale, + ), + ) + + +def _make_stages(layer, config: GraniteMoeHybridConfig, layer_idx: int): + attn, ffn = stage_facts(config, layer_idx, scale=layer._scale_output) return append_stages( - ( - declare_attn(read=UNFUSED_NORM_READOUT, output_transform=scale), - layer.input_layernorm, - ), - ( - declare_ffn( - sparse=sparse, - next_layer_sparse=sparse, - read=UNFUSED_NORM_READOUT, - output_transform=scale, - ), - layer.post_attention_layernorm, - ), + (attn, layer.input_layernorm), + (ffn, layer.post_attention_layernorm), ) @@ -91,7 +100,7 @@ class GraniteMoeHybridMambaDecoderLayer(nn.Module): ) self.block_sparse_moe = None - if getattr(config, "num_local_experts", 0) > 0: + if _has_moe(config): self.block_sparse_moe = GraniteMoeMoE( num_experts=config.num_local_experts, top_k=config.num_experts_per_tok, @@ -118,7 +127,7 @@ class GraniteMoeHybridMambaDecoderLayer(nn.Module): self.post_attention_layernorm = RMSNorm( config.hidden_size, eps=config.rms_norm_eps ) - self.attn_boundary, self.ffn_boundary = _make_stages(self) + self.attn_boundary, self.ffn_boundary = _make_stages(self, config, layer_idx) def _scale_output(self, hidden_states: torch.Tensor) -> torch.Tensor: return hidden_states * self.residual_multiplier @@ -282,7 +291,7 @@ class GraniteMoeHybridAttentionDecoderLayer(nn.Module): ) self.block_sparse_moe = None - if getattr(config, "num_local_experts", 0) > 0: + if _has_moe(config): self.block_sparse_moe = GraniteMoeMoE( num_experts=config.num_local_experts, top_k=config.num_experts_per_tok, @@ -309,7 +318,7 @@ class GraniteMoeHybridAttentionDecoderLayer(nn.Module): self.post_attention_layernorm = RMSNorm( config.hidden_size, eps=config.rms_norm_eps ) - self.attn_boundary, self.ffn_boundary = _make_stages(self) + self.attn_boundary, self.ffn_boundary = _make_stages(self, config, layer_idx) def _scale_output(self, hidden_states: torch.Tensor) -> torch.Tensor: return hidden_states * self.residual_multiplier @@ -395,6 +404,7 @@ class GraniteMoeHybridModel(nn.Module): config.num_hidden_layers, get_layer, prefix=f"{prefix}.layers", + stage_facts=partial(stage_facts, config), ) if self.pp_group.is_last_rank: diff --git a/python/sglang/srt/models/jet_nemotron.py b/python/sglang/srt/models/jet_nemotron.py index d7b779aa70..5c22aed464 100644 --- a/python/sglang/srt/models/jet_nemotron.py +++ b/python/sglang/srt/models/jet_nemotron.py @@ -461,6 +461,12 @@ class JetNemotronDecoderLayer(nn.Module): config.hidden_size, eps=config.rms_norm_eps ) + @staticmethod + def stage_facts(config: JetNemotronConfig, layer_id: int): + """None: the layer adds its own residual, so it declares no stages. + The model's shared declaration function (see make_layers).""" + return () + def forward( self, positions: torch.Tensor, diff --git a/python/sglang/srt/models/kimi_k3.py b/python/sglang/srt/models/kimi_k3.py index 696ac59978..47aa75712f 100644 --- a/python/sglang/srt/models/kimi_k3.py +++ b/python/sglang/srt/models/kimi_k3.py @@ -28,7 +28,6 @@ from sglang.srt.distributed import ( from sglang.srt.distributed.device_communicators.pynccl_allocator import ( use_symmetric_memory, ) -from sglang.srt.distributed.utils import get_group_rank_size from sglang.srt.environ import envs from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder from sglang.srt.layers import zero_copy_context @@ -52,6 +51,7 @@ from sglang.srt.layers.layer_boundary import ( declare_attn, declare_ffn, ) +from sglang.srt.layers.layer_boundary.residual import LayerResidualOps from sglang.srt.layers.layer_boundary.residual import batch as residual_batch from sglang.srt.layers.layer_boundary.residual.add_norm import ( REPLACE_AT_EXIT, @@ -72,6 +72,7 @@ from sglang.srt.layers.linear import ( QKVParallelLinear, ReplicatedLinear, RowParallelLinear, + resolve_linear_parallel_group, ) from sglang.srt.layers.logits_processor import LogitsProcessor from sglang.srt.layers.moe import route_quant_handoff @@ -273,6 +274,20 @@ def _merge_weights_as_views( return merged, sizes +def _dense_mlp_parallel_group() -> str: + """The group the dense layers' MLP shards over: a K3 MLP's group when it + is not given one. The Ascend path shards the dense MLP inside each + attention-TP replica. The GPU K3 refactor instead gathers all DP rows + and shards this one dense layer over the full TP group. Keep the GPU + default, but allow the NPU launcher to retain the proven attention-TP + layout without a device-type branch in shared model code.""" + return ( + "attn_tp" + if get_parallel().enable_dense_mlp_attn_tp and is_dp_attention_enabled() + else "tp" + ) + + class KimiK3MLP(nn.Module): """K3 MLP; SiLU or SiTU activation.""" @@ -288,18 +303,10 @@ class KimiK3MLP(nn.Module): parallel_group: Optional[LinearParallelGroup] = None, ) -> None: super().__init__() - # The Ascend path shards the dense MLP inside each attention-TP - # replica. The GPU K3 refactor instead gathers all DP rows and shards - # this one dense layer over the full TP group. Keep the GPU default, - # but allow the NPU launcher to retain the proven attention-TP layout - # without a device-type branch in shared model code. - self._dense_attn_tp = ( - get_parallel().enable_dense_mlp_attn_tp - and is_dp_attention_enabled() - and parallel_group is None - ) - if parallel_group is None: - parallel_group = "attn_tp" if self._dense_attn_tp else "tp" + default_group = parallel_group is None + if default_group: + parallel_group = _dense_mlp_parallel_group() + self._dense_attn_tp = default_group and parallel_group == "attn_tp" self.gate_up_proj = MergedColumnParallelLinear( hidden_size, [intermediate_size] * 2, @@ -417,6 +424,27 @@ def _k3_symm_o_proj_out(o_proj: RowParallelLinear, x: torch.Tensor) -> torch.Ten ) +def _uses_latent_moe(config: KimiLinearConfig) -> bool: + """Whether the MoE layers' routed experts run in a latent space narrower + than the hidden size (Latent MoE).""" + return config.routed_expert_hidden_size is not None + + +def _uses_ep_a2a() -> bool: + """Whether the MoE all-to-all backend is an EP a2a one, which moves each + row to its experts directly: the MoE region then consumes whatever rows + this rank holds (SP-MoE shard or DP-local batch), with no DP gather and + no TP reduce anywhere in it.""" + _a2a_backend = get_moe_a2a_backend() + return ( + _a2a_backend.is_megamoe() + or _a2a_backend.is_deepep() + or _a2a_backend.is_mooncake() + or _a2a_backend.is_ascend_fuseep() + or _a2a_backend.is_mori() + ) + + class KimiK3MoE(nn.Module): """K3 MoE with Latent MoE (experts run in moe_hidden_size space).""" @@ -438,7 +466,7 @@ class KimiK3MoE(nn.Module): self.layer_idx = layer_idx self.alt_stream = alt_stream - self.use_latent_moe = config.routed_expert_hidden_size is not None + self.use_latent_moe = _uses_latent_moe(config) # Merged front weight ([H, gate_up + E + latent]), built after weight # loading by _merge_front_weights(). self._front_w: Optional[torch.Tensor] = None @@ -541,18 +569,7 @@ class KimiK3MoE(nn.Module): "got a checkpoint with different constants" ) - # EP a2a backends move each row to its experts directly, so the MoE - # region consumes whatever rows this rank holds (SP-MoE shard or - # DP-local batch). No DP gather and no TP reduce anywhere in the - # region. - _a2a_backend = get_moe_a2a_backend() - self._ep_a2a = ( - _a2a_backend.is_megamoe() - or _a2a_backend.is_deepep() - or _a2a_backend.is_mooncake() - or _a2a_backend.is_ascend_fuseep() - or _a2a_backend.is_mori() - ) + self._ep_a2a = _uses_ep_a2a() # Defer the trtllm-gen finalize (top-k weighted unpermute) into the # push all-reduce's staging pass (k3_ar_fusion.finalize_all_reduce_push_norm) @@ -2358,14 +2375,7 @@ def _is_moe_layer(config: KimiLinearConfig, layer_idx: int) -> bool: def _shards_moe_rows() -> bool: """Whether an MoE layer runs on its attention-TP token shard (SP-MoE): an EP a2a backend with attention TP.""" - backend = get_moe_a2a_backend() - return ( - backend.is_megamoe() - or backend.is_deepep() - or backend.is_mooncake() - or backend.is_ascend_fuseep() - or backend.is_mori() - ) and get_parallel().attn_tp_group.world_size > 1 + return _uses_ep_a2a() and get_parallel().attn_tp_group.world_size > 1 def _fuses_attn_all_reduce(config: KimiLinearConfig) -> bool: @@ -2380,7 +2390,7 @@ def _fuses_attn_all_reduce(config: KimiLinearConfig) -> bool: ) -def _carries_bank_slices(config: KimiLinearConfig) -> bool: +def _carries_bank_slices(config: KimiLinearConfig, sp_collective: bool) -> bool: """Whether consecutive SP-MoE layers keep the stream and the attention-residual bank on each rank's attention-TP shard of the rows (SGLANG_K3_SP_ATTN_RES): each layer reads its attention input there, then @@ -2388,11 +2398,12 @@ def _carries_bank_slices(config: KimiLinearConfig) -> bool: neither a later pipeline rank nor a draft model's capture of the target's hidden states can take them; it is decided at construction. Nor can a dense layer after an SP-MoE one, whose FFN reads every row of the bank: - every layer from the first MoE layer on must be one.""" + every layer from the first MoE layer on must be one. ``sp_collective``: + whether K3's SP collective is enabled on this rank.""" if not ( _shards_moe_rows() and config.attn_res_block_size is not None - and k3_sp_collective.enabled() + and sp_collective and envs.SGLANG_K3_SP_ATTN_RES.get() and get_parallel().pp_size == 1 ): @@ -2404,6 +2415,84 @@ def _carries_bank_slices(config: KimiLinearConfig) -> bool: return not (spec.is_eagle3() or spec.is_dflash_family()) +def _latent_moe_writes_stream(config: KimiLinearConfig, layer_idx: int) -> bool: + """Whether a layer's FFN writes the next stream itself: a latent MoE on + this rank's rows adds the residual in its tail add and writes the next + stream itself; one gathered over attention DP returns its output to this + rank's tokens first.""" + return ( + config.attn_res_block_size is not None + and _is_moe_layer(config, layer_idx) + and _uses_latent_moe(config) + and not (is_dp_attention_enabled() and not _uses_ep_a2a()) + ) + + +def _stage_facts( + config: KimiLinearConfig, + layer_idx: int, + *, + sp_collective: bool, + bank: Optional[LayerResidualOps] = None, +): + """The attention and FFN stages the K3 layer at ``layer_idx`` declares: + the model's shared declaration function (see make_layers), which the + layer declares itself with. ``sp_collective`` is whether K3's SP + collective is enabled on this rank, which the model settles before it + builds a layer. ``bank`` is the layer's own reads and updates of the + attention-residual bank; without it, the stages declare what those do.""" + is_moe = _is_moe_layer(config, layer_idx) + # Under attention DP the FFN input is read on this rank's rows + # once the attention's sum is complete, then gathered. + attn_ops = {} + ffn_ops = dict(read=NormReadout(reads_before_dp_gather=True)) + if config.attn_res_block_size is not None: + if bank is None: + bank = AttnBankState.facts( + reads_slices=_carries_bank_slices(config, sp_collective) + ) + attn_ops = dict( + read=bank.attn_readout, + update=bank.attn_update, + # The rows an SP-MoE layer left on each rank's shard are + # gathered in K3's tuned all-gather when it takes the batch: + # before this read, or after it while the bank stays on the + # shard. + attn_tp_gather=( + k3_sp_collective.all_gather + if sp_collective + and _shards_moe_rows() + and _is_moe_layer(config, layer_idx - 1) + else None + ), + ) + ffn_ops = dict( + read=bank.ffn_readout, + update=( + REPLACE_AT_EXIT + if _latent_moe_writes_stream(config, layer_idx) + else bank.ffn_update + ), + ) + return ( + declare_attn(**attn_ops), + declare_ffn( + **ffn_ops, + sparse=is_moe, + next_layer_sparse=_is_moe_layer(config, layer_idx + 1), + # The TP width the dense MLP is built with. + dense_tp_size=( + None + if is_moe + else resolve_linear_parallel_group(_dense_mlp_parallel_group())[1] + ), + # A latent MoE completes its output sum together with + # the latent reduction its norm needs. + output_complete=is_moe and _uses_latent_moe(config), + ), + ) + + class KimiK3DecoderLayer(nn.Module): """Decoder layer carrying the K3 attention-residual stream.""" @@ -2419,7 +2508,6 @@ class KimiK3DecoderLayer(nn.Module): super().__init__() self.hidden_size = config.hidden_size self.layer_idx = layer_idx - self._dp_attention = is_dp_attention_enabled() self._is_moe_layer = _is_moe_layer(config, layer_idx) # SP-MoE: with an EP a2a backend and attention TP, a MoE layer runs on # this rank's attention-TP shard of the rows. @@ -2506,31 +2594,17 @@ class KimiK3DecoderLayer(nn.Module): prefix=f"{prefix}.mlp_res_proj", ) - # A latent MoE on this rank's rows adds the residual in its tail add and - # writes the next stream itself; one gathered over attention DP returns - # its output to this rank's tokens first. - self._ffn_writes_stream = ( - self.use_attn_residuals - and self._is_moe_layer - and self.mlp.use_latent_moe - and not (self._dp_attention and not self.mlp._ep_a2a) - ) + self._ffn_writes_stream = _latent_moe_writes_stream(config, layer_idx) self._declare_stages(config, layer_idx, attn_bank) def _declare_stages(self, config, layer_idx, attn_bank): - """Declare this layer's attention and FFN stages: the reads and updates - around them, the rows the FFN takes, and the kernels K3 supplies for - the bank path.""" - # Under attention DP the FFN input is read on this rank's rows - # once the attention's sum is complete, then gathered. - attn_ops = {} - ffn_ops = dict(read=NormReadout(reads_before_dp_gather=True)) - carries = _carries_bank_slices(config) - tuned_gather = ( - k3_sp_collective.all_gather if k3_sp_collective.enabled() else None - ) + """Declare this layer's attention and FFN stages as _stage_facts gives + them, with this layer's reads and updates of the bank and the kernels + K3 supplies for the bank path.""" + sp_collective = k3_sp_collective.enabled() + bank = None if self.use_attn_residuals: - bank_ops = AttnBankState( + bank = AttnBankState( attn_bank, self.self_attention_res_proj, self.self_attention_res_norm, @@ -2539,48 +2613,16 @@ class KimiK3DecoderLayer(nn.Module): writes_block=self.is_block_write_layer, ffn_input_fusions=self._ffn_input_fusions(), fuses_slice_collectives=self._sp_moe - and k3_sp_collective.enabled() + and sp_collective and envs.SGLANG_K3_SP_ATTN_RES.get(), - reads_slices=carries, + reads_slices=_carries_bank_slices(config, sp_collective), ).residual_ops() - attn_ops = dict( - read=bank_ops.attn_readout, - update=bank_ops.attn_update, - # The rows an SP-MoE layer left on each rank's shard are - # gathered in K3's tuned all-gather when it takes the batch: - # before this read, or after it while the bank stays on the - # shard. - attn_tp_gather=( - tuned_gather - if _shards_moe_rows() and _is_moe_layer(config, layer_idx - 1) - else None - ), - ) - ffn_ops = dict( - read=bank_ops.ffn_readout, - update=( - REPLACE_AT_EXIT if self._ffn_writes_stream else bank_ops.ffn_update - ), - ) + attn, ffn = _stage_facts( + config, layer_idx, sp_collective=sp_collective, bank=bank + ) self.attn_boundary, self.ffn_boundary = append_stages( - (declare_attn(**attn_ops), self.input_layernorm), - ( - declare_ffn( - **ffn_ops, - sparse=self._is_moe_layer, - next_layer_sparse=_is_moe_layer(config, layer_idx + 1), - # The TP width the dense MLP is built with. - dense_tp_size=( - None - if self._is_moe_layer - else get_group_rank_size(self.mlp.down_proj.tp_group)[1] - ), - # A latent MoE completes its output sum together with - # the latent reduction its norm needs. - output_complete=self._is_moe_layer and self.mlp.use_latent_moe, - ), - self.post_attention_layernorm, - ), + (attn, self.input_layernorm), + (ffn, self.post_attention_layernorm), ) def _ffn_input_fusions(self): @@ -2673,7 +2715,10 @@ class KimiK3LinearModel(nn.Module): self.pp_group = get_parallel().pp_group self.dspark_layers_to_capture: Optional[list[int]] = None self._dp_attention = is_dp_attention_enabled() - self.carries_bank_slices = _carries_bank_slices(config) + # Whether K3's SP collective is enabled on this rank, which the layers' + # stage declarations depend on: settled once, before any is built. + sp_collective = k3_sp_collective.enabled() + self.carries_bank_slices = _carries_bank_slices(config, sp_collective) if self.pp_group.is_first_rank: embedding_quant_config = ( @@ -2731,7 +2776,7 @@ class KimiK3LinearModel(nn.Module): # takes the batch. attn_tp_gather=( k3_sp_collective.all_gather - if k3_sp_collective.enabled() + if sp_collective and _shards_moe_rows() and _is_moe_layer(config, config.num_hidden_layers - 1) else None @@ -2753,6 +2798,9 @@ class KimiK3LinearModel(nn.Module): ), prefix=f"{prefix}.layers", final_read=self._final_read if self.pp_group.is_last_rank else None, + stage_facts=lambda idx: _stage_facts( + config, idx, sp_collective=sp_collective + ), ) def forward( diff --git a/python/sglang/srt/models/kimi_linear.py b/python/sglang/srt/models/kimi_linear.py index d1a29f2f9d..b0feaa7998 100644 --- a/python/sglang/srt/models/kimi_linear.py +++ b/python/sglang/srt/models/kimi_linear.py @@ -580,8 +580,8 @@ class KimiDecoderLayer(nn.Module): reduce_results=False, ) - sparse = _is_sparse_layer(config, layer_idx) - if sparse: + attn, ffn = self.stage_facts(config, layer_idx) + if ffn.sparse: self.block_sparse_moe = KimiMoE( config=config, quant_config=quant_config, @@ -603,15 +603,22 @@ class KimiDecoderLayer(nn.Module): self.post_attention_layernorm = RMSNorm( config.hidden_size, eps=config.rms_norm_eps ) + self.attn_boundary, self.ffn_boundary = append_stages( + (attn, self.input_layernorm), + (ffn, self.post_attention_layernorm), + ) + + @staticmethod + def stage_facts(config: KimiLinearConfig, layer_idx: int): + """The stages the layer at ``layer_idx`` declares, from the config: the + model's shared declaration function (see make_layers).""" + sparse = _is_sparse_layer(config, layer_idx) next_sparse = layer_idx + 1 < config.num_hidden_layers and _is_sparse_layer( config, layer_idx + 1 ) - self.attn_boundary, self.ffn_boundary = append_stages( - (declare_attn(), self.input_layernorm), - ( - declare_ffn(sparse=sparse, next_layer_sparse=next_sparse), - self.post_attention_layernorm, - ), + return ( + declare_attn(), + declare_ffn(sparse=sparse, next_layer_sparse=next_sparse), ) def forward( @@ -674,6 +681,7 @@ class KimiLinearModel(nn.Module): alt_stream=self.alt_stream, ), prefix=f"{prefix}.layers", + stage_facts=lambda idx: KimiDecoderLayer.stage_facts(config, idx), ) if self.pp_group.is_last_rank: diff --git a/python/sglang/srt/models/laguna.py b/python/sglang/srt/models/laguna.py index 99c3059aeb..71bfdac86f 100644 --- a/python/sglang/srt/models/laguna.py +++ b/python/sglang/srt/models/laguna.py @@ -418,12 +418,8 @@ class LagunaDecoderLayer(nn.Module): prefix=add_prefix("self_attn", prefix), ) - mlp_types = config.mlp_layer_types - self.is_layer_sparse = mlp_types[layer_id] == "sparse" - is_next_layer_sparse = ( - layer_id + 1 < config.num_hidden_layers - and mlp_types[layer_id + 1] == "sparse" - ) + attn, ffn = self.stage_facts(config, layer_id) + self.is_layer_sparse = ffn.sparse if self.is_layer_sparse: self.mlp = LagunaMoE( @@ -448,13 +444,25 @@ class LagunaDecoderLayer(nn.Module): ) self.attn_boundary, self.ffn_boundary = append_stages( - (declare_attn(), self.input_layernorm), - ( - declare_ffn( - sparse=self.is_layer_sparse, - next_layer_sparse=is_next_layer_sparse, - ), - self.post_attention_layernorm, + (attn, self.input_layernorm), + (ffn, self.post_attention_layernorm), + ) + + @staticmethod + def stage_facts(config: LagunaConfig, layer_id: int): + """The stages the layer at ``layer_id`` declares, from the config: the + model's shared declaration function (see make_layers).""" + mlp_types = config.mlp_layer_types + is_layer_sparse = mlp_types[layer_id] == "sparse" + is_next_layer_sparse = ( + layer_id + 1 < config.num_hidden_layers + and mlp_types[layer_id + 1] == "sparse" + ) + return ( + declare_attn(), + declare_ffn( + sparse=is_layer_sparse, + next_layer_sparse=is_next_layer_sparse, ), ) @@ -519,6 +527,7 @@ class LagunaModel(nn.Module): prefix=prefix, ), prefix=add_prefix("layers", prefix), + stage_facts=lambda idx: decoder_layer_type.stage_facts(config, idx), ) if self.pp_group.is_last_rank: self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps) diff --git a/python/sglang/srt/models/llada2.py b/python/sglang/srt/models/llada2.py index e57db99096..5ce01d3e6e 100644 --- a/python/sglang/srt/models/llada2.py +++ b/python/sglang/srt/models/llada2.py @@ -20,6 +20,7 @@ """SGLang LLaDA2MoeModelLM model.""" import logging +from functools import partial from typing import Iterable, Optional, Tuple, Union import torch @@ -1150,7 +1151,6 @@ class LLaDA2MoeBlock(nn.Module): self.layer_id = layer_id self.is_layer_sparse = self._is_layer_sparse(config, layer_id=layer_id) - is_next_layer_sparse = self._is_layer_sparse(config, layer_id=layer_id + 1) if self.is_layer_sparse: self.mlp = LLaDA2MoeSparseMoeBlock( @@ -1173,18 +1173,27 @@ class LLaDA2MoeBlock(nn.Module): self.post_attention_layernorm = RMSNorm(hidden_size, eps=config.rms_norm_eps) + attn, ffn = self.stage_facts(config, layer_id) self.attn_boundary, self.ffn_boundary = append_stages( - (declare_attn(), self.input_layernorm), - ( - declare_ffn( - sparse=self.is_layer_sparse, - next_layer_sparse=is_next_layer_sparse, - ), - self.post_attention_layernorm, + (attn, self.input_layernorm), + (ffn, self.post_attention_layernorm), + ) + + @classmethod + def stage_facts(cls, config: PretrainedConfig, layer_id: int): + """The stages the layer at ``layer_id`` declares, in order: the model's + shared declaration function, which the layer declares with too (see + make_layers).""" + return ( + declare_attn(), + declare_ffn( + sparse=cls._is_layer_sparse(config, layer_id=layer_id), + next_layer_sparse=cls._is_layer_sparse(config, layer_id=layer_id + 1), ), ) - def _is_layer_sparse(self, config: PretrainedConfig, layer_id: int) -> bool: + @staticmethod + def _is_layer_sparse(config: PretrainedConfig, layer_id: int) -> bool: return ( config.num_experts is not None and layer_id >= config.first_k_dense_replace ) @@ -1253,6 +1262,7 @@ class LLaDA2MoeModel(nn.Module): alt_stream=alt_stream, ), prefix=add_prefix("layers", prefix), + stage_facts=partial(LLaDA2MoeBlock.stage_facts, config), ) if self.pp_group.is_last_rank: self.norm = RMSNorm(self.embed_dim, eps=config.rms_norm_eps) diff --git a/python/sglang/srt/models/llama.py b/python/sglang/srt/models/llama.py index ccdd054274..07cf4a1ae5 100644 --- a/python/sglang/srt/models/llama.py +++ b/python/sglang/srt/models/llama.py @@ -364,15 +364,24 @@ class LlamaDecoderLayer(nn.Module): return self.input_layernorm.fuse_input_quant(self.self_attn.qkv_proj) self.post_attention_layernorm.fuse_input_quant(self.mlp.gate_up_proj) + attn, ffn = self.stage_facts(config, layer_id) self.attn_boundary, self.ffn_boundary = append_stages( - (declare_attn(read=self._attn_readout(layer_id)), self.input_layernorm), - ( - declare_ffn(sparse=False, next_layer_sparse=False), - self.post_attention_layernorm, - ), + (attn, self.input_layernorm), + (ffn, self.post_attention_layernorm), ) - def _attn_readout(self, layer_id: int): + @classmethod + def stage_facts(cls, config: LlamaConfig, layer_id: int): + """The stages a layer of this class declares, from the config alone: + the model's shared declaration function, which the layer declares + with too (see make_layers). A subclass's attention read takes part.""" + return ( + declare_attn(read=cls._attn_readout(layer_id)), + declare_ffn(sparse=False, next_layer_sparse=False), + ) + + @classmethod + def _attn_readout(cls, layer_id: int): return NORM_QUANT_READOUT def forward( @@ -437,6 +446,7 @@ class LlamaModel(nn.Module): prefix=prefix, ), prefix="model.layers", + stage_facts=lambda idx: LlamaDecoderLayer.stage_facts(config, idx), ) if self.pp_group.is_last_rank: diff --git a/python/sglang/srt/models/llama_eagle.py b/python/sglang/srt/models/llama_eagle.py index 6b4b7eca3e..a2c288bd2b 100644 --- a/python/sglang/srt/models/llama_eagle.py +++ b/python/sglang/srt/models/llama_eagle.py @@ -54,7 +54,8 @@ class LlamaDecoderLayer(LlamaDecoderLayer): if layer_id == 0: del self.input_layernorm - def _attn_readout(self, layer_id: int): + @classmethod + def _attn_readout(cls, layer_id: int): if layer_id == 0: return PLAIN_READOUT return super()._attn_readout(layer_id) diff --git a/python/sglang/srt/models/mellum.py b/python/sglang/srt/models/mellum.py index bb9e7677ee..a2afff72b3 100644 --- a/python/sglang/srt/models/mellum.py +++ b/python/sglang/srt/models/mellum.py @@ -402,18 +402,8 @@ class MellumDecoderLayer(Qwen3MoeDecoderLayer): f"{len(mlp_layer_types)} and {cfg.num_hidden_layers}" ) - def _is_sparse(lid: int) -> bool: - if lid < 0 or lid >= cfg.num_hidden_layers: - return False - mlp_type = mlp_layer_types[lid] - if mlp_type not in ("sparse", "dense"): - raise ValueError( - f"Unsupported mlp_layer_types[{lid}]={mlp_type}; " - "expected 'sparse' or 'dense'" - ) - return mlp_type == "sparse" - - self.is_layer_sparse = _is_sparse(layer_id) + attn, ffn = self.stage_facts(config, layer_id) + self.is_layer_sparse = ffn.sparse if self.is_layer_sparse: if num_experts <= 0: @@ -436,19 +426,37 @@ class MellumDecoderLayer(Qwen3MoeDecoderLayer): reduce_results=False, ) - is_next_layer_sparse = _is_sparse(layer_id + 1) - self.input_layernorm = RMSNorm(cfg.hidden_size, eps=rms_norm_eps) self.post_attention_layernorm = RMSNorm(cfg.hidden_size, eps=rms_norm_eps) self.attn_boundary, self.ffn_boundary = append_stages( - (declare_attn(), self.input_layernorm), - ( - declare_ffn( - sparse=self.is_layer_sparse, - next_layer_sparse=is_next_layer_sparse, - ), - self.post_attention_layernorm, + (attn, self.input_layernorm), + (ffn, self.post_attention_layernorm), + ) + + @staticmethod + def _is_sparse(config: PretrainedConfig, lid: int) -> bool: + cfg = cast(Any, config) + if lid < 0 or lid >= cfg.num_hidden_layers: + return False + mlp_type = cfg.mlp_layer_types[lid] + if mlp_type not in ("sparse", "dense"): + raise ValueError( + f"Unsupported mlp_layer_types[{lid}]={mlp_type}; " + "expected 'sparse' or 'dense'" + ) + return mlp_type == "sparse" + + @classmethod + def stage_facts(cls, config: PretrainedConfig, layer_id: int): + """The stages a Mellum layer declares, from the config alone: the + model's shared declaration function, which the layer declares with + too (see make_layers).""" + return ( + declare_attn(), + declare_ffn( + sparse=cls._is_sparse(config, layer_id), + next_layer_sparse=cls._is_sparse(config, layer_id + 1), ), ) diff --git a/python/sglang/srt/models/mimo_v2.py b/python/sglang/srt/models/mimo_v2.py index a45ddbb2c8..f6f40c22f3 100644 --- a/python/sglang/srt/models/mimo_v2.py +++ b/python/sglang/srt/models/mimo_v2.py @@ -16,6 +16,7 @@ import logging import math import re from array import array +from functools import partial from typing import Any, Dict, Iterable, List, Optional, Tuple, Union import torch @@ -822,8 +823,7 @@ class MiMoV2DecoderLayer(nn.Module): prefix=add_prefix("self_attn", prefix), ) - self.is_layer_sparse = self.is_moe_layer(layer_id) - is_next_layer_sparse = self.is_moe_layer(layer_id + 1) + self.is_layer_sparse = self.is_moe_layer(config, layer_id) if self.is_layer_sparse: self.mlp = MiMoV2MoE( @@ -849,14 +849,22 @@ class MiMoV2DecoderLayer(nn.Module): config.hidden_size, eps=config.layernorm_epsilon ) + attn, ffn = self.stage_facts(config, layer_id) self.attn_boundary, self.ffn_boundary = append_stages( - (declare_attn(), self.input_layernorm), - ( - declare_ffn( - sparse=self.is_layer_sparse, - next_layer_sparse=is_next_layer_sparse, - ), - self.post_attention_layernorm, + (attn, self.input_layernorm), + (ffn, self.post_attention_layernorm), + ) + + @classmethod + def stage_facts(cls, config, layer_id: int): + """The stages the layer at ``layer_id`` declares, in order: the model's + shared declaration function, which the layer declares with too (see + make_layers).""" + return ( + declare_attn(), + declare_ffn( + sparse=cls.is_moe_layer(config, layer_id), + next_layer_sparse=cls.is_moe_layer(config, layer_id + 1), ), ) @@ -889,12 +897,13 @@ class MiMoV2DecoderLayer(nn.Module): return hidden_states - def is_moe_layer(self, layer_idx: int) -> bool: + @staticmethod + def is_moe_layer(config, layer_idx: int) -> bool: return ( - hasattr(self.config, "moe_layer_freq") - and 0 <= layer_idx < len(self.config.moe_layer_freq) - and not isinstance(self.config.moe_layer_freq, int) - and self.config.moe_layer_freq[layer_idx] + hasattr(config, "moe_layer_freq") + and 0 <= layer_idx < len(config.moe_layer_freq) + and not isinstance(config.moe_layer_freq, int) + and config.moe_layer_freq[layer_idx] ) def is_swa_layer(self) -> bool: @@ -987,6 +996,7 @@ class MiMoV2Model(nn.Module): prefix=prefix, ), prefix=add_prefix("layers", prefix), + stage_facts=partial(decoder_layer_type.stage_facts, config), ) if self.pp_group.is_last_rank: self.norm = RMSNorm(config.hidden_size, eps=config.layernorm_epsilon) diff --git a/python/sglang/srt/models/minimax_m2.py b/python/sglang/srt/models/minimax_m2.py index 1b04676ae9..6f06bea80f 100644 --- a/python/sglang/srt/models/minimax_m2.py +++ b/python/sglang/srt/models/minimax_m2.py @@ -17,7 +17,7 @@ import logging from contextlib import nullcontext -from functools import lru_cache +from functools import lru_cache, partial from typing import Any, Dict, Iterable, Optional, Set, Tuple, Union import torch @@ -99,7 +99,6 @@ from sglang.srt.utils import ( is_xpu, make_pp_layers, ) -from sglang.srt.utils.common import is_building_neighbour_layer from sglang.srt.utils.custom_op import register_custom_op from sglang.srt.utils.hf_transformers_utils import get_rope_config @@ -409,12 +408,7 @@ class MiniMaxM2QKRMSNorm: use_fused_norm = get_bool_env_var("SGLANG_USE_FUSED_PARALLEL_QKNORM") self._forward_impl = self._forward_naive - if ( - self._world_size > 1 - and _is_cuda - and use_fused_norm - and not is_building_neighbour_layer() - ): + if self._world_size > 1 and _is_cuda and use_fused_norm: occupancy = get_fused_parallel_qknorm_max_occupancy( q_norm.weight.dtype, self._world_size, @@ -846,8 +840,8 @@ class MiniMaxM2DecoderLayer(nn.Module): self.hidden_size = config.hidden_size self.layer_id = layer_id - # TBO support: All MiniMax layers are sparse (MoE) - self.is_layer_sparse = True + attn, ffn = self.stage_facts(config, layer_id) + self.is_layer_sparse = ffn.sparse self.self_attn = MiniMaxM2Attention( config=config, @@ -870,17 +864,20 @@ class MiniMaxM2DecoderLayer(nn.Module): config.hidden_size, eps=getattr(config, "rms_norm_eps", 1e-6) ) - is_next_layer_sparse = True - self.attn_boundary, self.ffn_boundary = append_stages( - (declare_attn(), self.input_layernorm), - ( - declare_ffn( - sparse=self.is_layer_sparse, - next_layer_sparse=is_next_layer_sparse, - ), - self.post_attention_layernorm, - ), + (attn, self.input_layernorm), + (ffn, self.post_attention_layernorm), + ) + + @staticmethod + def stage_facts(config: PretrainedConfig, layer_id: int): + """The stages the layer at ``layer_id`` declares, in order: the model's + shared declaration function, which the layer declares with too (see + make_layers).""" + # TBO support: All MiniMax layers are sparse (MoE) + return ( + declare_attn(), + declare_ffn(sparse=True, next_layer_sparse=True), ) def forward( @@ -951,6 +948,7 @@ class MiniMaxM2Model(nn.Module): config.num_hidden_layers, layer_fn, prefix=add_prefix("layers", prefix), + stage_facts=partial(MiniMaxM2DecoderLayer.stage_facts, config), ) if self.pp_group.is_last_rank: self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps) diff --git a/python/sglang/srt/models/minimax_m3.py b/python/sglang/srt/models/minimax_m3.py index 860bfcbb7b..dff0a590e6 100644 --- a/python/sglang/srt/models/minimax_m3.py +++ b/python/sglang/srt/models/minimax_m3.py @@ -17,6 +17,7 @@ import logging from contextlib import nullcontext +from functools import partial from typing import Iterable, List, Optional, Set, Tuple, Union import torch @@ -1520,12 +1521,9 @@ class MiniMaxM3DecoderLayer(nn.Module): disable_index_value=disable_index_value, ) - moe_layer_freq = getattr(config, "moe_layer_freq", None) # Means "MLP is a sparse MoE", not attention sparsity. Kept as ``is_layer_sparse`` # because model construction and downstream integrations read this attr. - self.is_layer_sparse = ( - moe_layer_freq[layer_id] != 0 if moe_layer_freq is not None else True - ) + self.is_layer_sparse = self._is_layer_sparse(config, layer_id) if self.is_layer_sparse: self.mlp = MiniMaxM3MoE( @@ -1560,26 +1558,40 @@ class MiniMaxM3DecoderLayer(nn.Module): config.hidden_size, eps=config.rms_norm_eps ) - def _is_layer_sparse(lid): - if moe_layer_freq is None: - return True - if lid < 0 or lid >= config.num_hidden_layers: - return True - return moe_layer_freq[lid] != 0 - - is_next_layer_sparse = _is_layer_sparse(layer_id + 1) - + attn, ffn = self.stage_facts(config, layer_id) self.attn_boundary, self.ffn_boundary = append_stages( - (declare_attn(), self.input_layernorm), - ( - declare_ffn( - sparse=self.is_layer_sparse, - next_layer_sparse=is_next_layer_sparse, - ), - self.post_attention_layernorm, + (attn, self.input_layernorm), + (ffn, self.post_attention_layernorm), + ) + + @classmethod + def stage_facts(cls, config: PretrainedConfig, layer_id: int): + """The stages the layer at ``layer_id`` declares, in order: the model's + shared declaration function, which the layer declares with too (see + make_layers).""" + return ( + declare_attn(), + declare_ffn( + sparse=cls._is_layer_sparse(config, layer_id), + next_layer_sparse=cls._is_next_layer_sparse(config, layer_id), ), ) + @staticmethod + def _is_layer_sparse(config: PretrainedConfig, layer_id: int) -> bool: + moe_layer_freq = getattr(config, "moe_layer_freq", None) + return moe_layer_freq[layer_id] != 0 if moe_layer_freq is not None else True + + @staticmethod + def _is_next_layer_sparse(config: PretrainedConfig, layer_id: int) -> bool: + moe_layer_freq = getattr(config, "moe_layer_freq", None) + lid = layer_id + 1 + if moe_layer_freq is None: + return True + if lid < 0 or lid >= config.num_hidden_layers: + return True + return moe_layer_freq[lid] != 0 + def forward( self, positions: torch.Tensor, @@ -1653,6 +1665,7 @@ class MiniMaxM3Model(nn.Module): config.num_hidden_layers, layer_fn, prefix=add_prefix("layers", prefix), + stage_facts=partial(MiniMaxM3DecoderLayer.stage_facts, config), ) if self.pp_group.is_last_rank: if self.use_gemma_norm: diff --git a/python/sglang/srt/models/ministral3.py b/python/sglang/srt/models/ministral3.py index 51335242ef..e99ed624fa 100644 --- a/python/sglang/srt/models/ministral3.py +++ b/python/sglang/srt/models/ministral3.py @@ -155,6 +155,7 @@ class Ministral3Model(LlamaModel): prefix=prefix, ), prefix="model.layers", + stage_facts=lambda idx: Ministral3DecoderLayer.stage_facts(config, idx), ) diff --git a/python/sglang/srt/models/mixtral.py b/python/sglang/srt/models/mixtral.py index 2ef687ebc1..ac0e8b4e4d 100644 --- a/python/sglang/srt/models/mixtral.py +++ b/python/sglang/srt/models/mixtral.py @@ -19,6 +19,7 @@ """Inference-only Mixtral model.""" import logging +from functools import partial from typing import Iterable, Optional, Tuple, Union import torch @@ -235,12 +236,20 @@ class MixtralDecoderLayer(nn.Module): self.post_attention_layernorm = RMSNorm( config.hidden_size, eps=config.rms_norm_eps ) + attn, ffn = self.stage_facts(config, layer_id) self.attn_boundary, self.ffn_boundary = append_stages( - (declare_attn(), self.input_layernorm), - ( - declare_ffn(sparse=True, next_layer_sparse=True), - self.post_attention_layernorm, - ), + (attn, self.input_layernorm), + (ffn, self.post_attention_layernorm), + ) + + @staticmethod + def stage_facts(config: MixtralConfig, layer_id: int): + """The stages the layer at ``layer_id`` declares, in order: the model's + shared declaration function, which the layer declares with too (see + make_layers).""" + return ( + declare_attn(), + declare_ffn(sparse=True, next_layer_sparse=True), ) def forward( @@ -292,6 +301,7 @@ class MixtralModel(nn.Module): ), prefix="layers", return_tuple=True, + stage_facts=partial(MixtralDecoderLayer.stage_facts, config), ) if self.pp_group.is_last_rank: diff --git a/python/sglang/srt/models/nanbeige.py b/python/sglang/srt/models/nanbeige.py index dc2244a323..a01d6460f3 100644 --- a/python/sglang/srt/models/nanbeige.py +++ b/python/sglang/srt/models/nanbeige.py @@ -1,4 +1,5 @@ import logging +from functools import partial from typing import Iterable, List, Optional, Tuple, Union import torch @@ -231,12 +232,20 @@ class NanbeigeDecoderLayer(nn.Module): ) # Each loop runs the physical layers as one stack: the model folds the # residual back in and re-enters at layer 0. + attn, ffn = self.stage_facts(config, layer_id) self.attn_boundary, self.ffn_boundary = append_stages( - (declare_attn(), self.input_layernorm), - ( - declare_ffn(sparse=False, next_layer_sparse=False), - self.post_attention_layernorm, - ), + (attn, self.input_layernorm), + (ffn, self.post_attention_layernorm), + ) + + @staticmethod + def stage_facts(config: NanbeigeConfig, layer_id: int): + """The stages the layer at ``layer_id`` declares, in order: the model's + shared declaration function, which the layer declares with too (see + make_layers).""" + return ( + declare_attn(), + declare_ffn(sparse=False, next_layer_sparse=False), ) def forward( @@ -305,6 +314,7 @@ class NanbeigeModel(nn.Module): alt_stream=alt_stream, ), prefix=add_prefix("layers", prefix), + stage_facts=partial(decoder_layer_type.stage_facts, config), ) if self.pp_group.is_last_rank: self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps) diff --git a/python/sglang/srt/models/nemotron_nas.py b/python/sglang/srt/models/nemotron_nas.py index 54a6688734..75a5351183 100644 --- a/python/sglang/srt/models/nemotron_nas.py +++ b/python/sglang/srt/models/nemotron_nas.py @@ -134,17 +134,12 @@ class DeciLMDecoderLayer(nn.Module): config.hidden_size, eps=config.rms_norm_eps ) - # A layer whose attention and FFN are both no-ops has no stages. - stages = [] + norms = [] if not self._is_no_op_attention: - stages.append((declare_attn(), self.input_layernorm)) + norms.append(self.input_layernorm) if not self._is_no_op_ffn: - stages.append( - ( - declare_ffn(sparse=False, next_layer_sparse=False), - self.post_attention_layernorm, - ) - ) + norms.append(self.post_attention_layernorm) + stages = list(zip(self.stage_facts(config, layer_idx), norms, strict=True)) self.entry_boundary = None if stages: boundaries = append_stages( @@ -156,6 +151,20 @@ class DeciLMDecoderLayer(nn.Module): if not self._is_no_op_ffn: self.ffn_boundary = boundaries[-1] + @staticmethod + def stage_facts(config: LlamaConfig, layer_idx: int): + """The stages a DeciLM layer declares, from its block config alone: + the model's shared declaration function, which the layer declares + with too (see make_layers).""" + block_config = config.block_configs[layer_idx] + # A layer whose attention and FFN are both no-ops has no stages. + stages = [] + if not block_config.attention.no_op: + stages.append(declare_attn()) + if not block_config.ffn.no_op: + stages.append(declare_ffn(sparse=False, next_layer_sparse=False)) + return tuple(stages) + def forward( self, positions: torch.Tensor, @@ -223,6 +232,7 @@ class DeciModel(nn.Module): config.num_hidden_layers, get_layer, prefix=add_prefix("layers", prefix), + stage_facts=lambda idx: layer_type.stage_facts(config, idx), ) if get_parallel().pp_group.is_last_rank: self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps) diff --git a/python/sglang/srt/models/qwen2.py b/python/sglang/srt/models/qwen2.py index 94d1f24f41..9f80581db0 100644 --- a/python/sglang/srt/models/qwen2.py +++ b/python/sglang/srt/models/qwen2.py @@ -299,15 +299,24 @@ class Qwen2DecoderLayer(nn.Module): ) self.input_layernorm.fuse_input_quant(self.self_attn.qkv_proj) self.post_attention_layernorm.fuse_input_quant(self.mlp.gate_up_proj) + attn, ffn = self.stage_facts(config, layer_id) self.attn_boundary, self.ffn_boundary = append_stages( - (declare_attn(read=self._attn_readout(layer_id)), self.input_layernorm), - ( - declare_ffn(sparse=False, next_layer_sparse=False), - self.post_attention_layernorm, - ), + (attn, self.input_layernorm), + (ffn, self.post_attention_layernorm), ) - def _attn_readout(self, layer_id: int): + @classmethod + def stage_facts(cls, config: Qwen2Config, layer_id: int): + """The stages a layer of this class declares, from the config alone: + the model's shared declaration function, which the layer declares + with too (see make_layers). A subclass's attention read takes part.""" + return ( + declare_attn(read=cls._attn_readout(layer_id)), + declare_ffn(sparse=False, next_layer_sparse=False), + ) + + @classmethod + def _attn_readout(cls, layer_id: int): return NORM_QUANT_READOUT def forward( @@ -384,6 +393,7 @@ class Qwen2Model(nn.Module): alt_stream=alt_stream, ), prefix=add_prefix("layers", prefix), + stage_facts=lambda idx: decoder_layer_type.stage_facts(config, idx), ) if self.pp_group.is_last_rank: norm_kwargs = ( diff --git a/python/sglang/srt/models/qwen2_eagle.py b/python/sglang/srt/models/qwen2_eagle.py index 195816fb02..cab607d5f9 100644 --- a/python/sglang/srt/models/qwen2_eagle.py +++ b/python/sglang/srt/models/qwen2_eagle.py @@ -55,7 +55,8 @@ class Qwen2DecoderLayer(Qwen2DecoderLayer): if layer_id == 0: del self.input_layernorm - def _attn_readout(self, layer_id: int): + @classmethod + def _attn_readout(cls, layer_id: int): if layer_id == 0: return PLAIN_READOUT return super()._attn_readout(layer_id) diff --git a/python/sglang/srt/models/qwen2_moe.py b/python/sglang/srt/models/qwen2_moe.py index 03d7d7a5df..bb1719595e 100644 --- a/python/sglang/srt/models/qwen2_moe.py +++ b/python/sglang/srt/models/qwen2_moe.py @@ -1026,9 +1026,8 @@ class Qwen2MoeDecoderLayer(nn.Module): self.layer_id = layer_id - # Qwen2MoE all layers are sparse and have no nextn now - self.is_layer_sparse = True - is_next_layer_sparse = True + attn, ffn = self.stage_facts(config, layer_id) + self.is_layer_sparse = ffn.sparse if self.is_layer_sparse: self.mlp = Qwen2MoeSparseMoeBlock( @@ -1053,13 +1052,23 @@ class Qwen2MoeDecoderLayer(nn.Module): config.hidden_size, eps=config.rms_norm_eps ) self.attn_boundary, self.ffn_boundary = append_stages( - (declare_attn(), self.input_layernorm), - ( - declare_ffn( - sparse=self.is_layer_sparse, - next_layer_sparse=is_next_layer_sparse, - ), - self.post_attention_layernorm, + (attn, self.input_layernorm), + (ffn, self.post_attention_layernorm), + ) + + @staticmethod + def stage_facts(config: PretrainedConfig, layer_id: int): + """The stages a Qwen2-MoE layer declares, from the config alone: the + model's shared declaration function, which the layer declares with + too (see make_layers).""" + # Qwen2MoE all layers are sparse and have no nextn now + is_layer_sparse = True + is_next_layer_sparse = True + return ( + declare_attn(), + declare_ffn( + sparse=is_layer_sparse, + next_layer_sparse=is_next_layer_sparse, ), ) @@ -1138,6 +1147,7 @@ class Qwen2MoeModel(nn.Module): alt_stream=alt_stream, ), prefix=add_prefix("layers", prefix), + stage_facts=lambda idx: decoder_layer_type.stage_facts(config, idx), ) if self.pp_group.is_last_rank: self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps) diff --git a/python/sglang/srt/models/qwen3.py b/python/sglang/srt/models/qwen3.py index 78574b6eef..9552a15ba7 100644 --- a/python/sglang/srt/models/qwen3.py +++ b/python/sglang/srt/models/qwen3.py @@ -373,12 +373,20 @@ class Qwen3DecoderLayer(nn.Module): config.hidden_size, eps=config.rms_norm_eps, **norm_kwargs ) + attn, ffn = self.stage_facts(config, layer_id) self.attn_boundary, self.ffn_boundary = append_stages( - (declare_attn(), self.input_layernorm), - ( - declare_ffn(sparse=False, next_layer_sparse=False), - self.post_attention_layernorm, - ), + (attn, self.input_layernorm), + (ffn, self.post_attention_layernorm), + ) + + @staticmethod + def stage_facts(config: Qwen3Config, layer_id: int): + """The stages a Qwen3 layer declares, from the config alone: the + model's shared declaration function, which the layer declares with + too (see make_layers).""" + return ( + declare_attn(), + declare_ffn(sparse=False, next_layer_sparse=False), ) def forward( diff --git a/python/sglang/srt/models/qwen3_5.py b/python/sglang/srt/models/qwen3_5.py index 1f1e30dd52..5ff875082a 100644 --- a/python/sglang/srt/models/qwen3_5.py +++ b/python/sglang/srt/models/qwen3_5.py @@ -69,6 +69,7 @@ from sglang.srt.layers.layer_boundary.residual import batch as residual_batch # Layers - Others from sglang.srt.layers.layer_boundary.residual.add_norm import ( + NORM_QUANT_READOUT, Fp8Input, NormQuantReadout, ) @@ -252,6 +253,27 @@ def _layer_fusions(config: Qwen3_5TextConfig, is_nextn: bool): return None +def _qwen3_5_is_moe(config: Qwen3_5TextConfig) -> bool: + """Whether the model's layers run sparse MoE blocks rather than dense MLPs: + every layer of either kind alike.""" + return config.model_type in _QWEN3_5_MOE_TEXT_MODEL_TYPES + + +def _qwen3_5_stage_facts( + config: Qwen3_5TextConfig, layer_id: int, *, attn_read=NORM_QUANT_READOUT +): + """The attention and FFN stages a Qwen3.5 layer of either kind declares: + the model's shared declaration function (see make_layers), which the + layer declares itself with. ``attn_read`` is the layer's own attention + read, which declares what the plain one does and also hands on the + quantized input its projection takes.""" + sparse = _qwen3_5_is_moe(config) + return ( + declare_attn(read=attn_read), + declare_ffn(sparse=sparse, next_layer_sparse=sparse), + ) + + if _is_cuda: from sglang.kernels.ops.attention.fused_qk_rmsnorm_rope_gate import ( fused_qk_gemma_rmsnorm_rope_gate, @@ -1087,6 +1109,8 @@ class Qwen3_5GatedDeltaNet(nn.Module): class Qwen3_5LinearDecoderLayer(nn.Module): """Qwen3.5 Decoder Layer with Linear Attention (GatedDeltaNet).""" + stage_facts = staticmethod(_qwen3_5_stage_facts) + def __init__( self, config: Qwen3_5TextConfig, @@ -1107,7 +1131,7 @@ class Qwen3_5LinearDecoderLayer(nn.Module): # NOTE: Determine the MLP type based on the model type # Qwen3.5 use all layers for MLP / Qwen3.5-MoE use sparse MoE blocks - if config.model_type in _QWEN3_5_MOE_TEXT_MODEL_TYPES: + if _qwen3_5_is_moe(config): self.mlp = Qwen2MoeSparseMoeBlock( layer_id=layer_id, config=config, @@ -1123,8 +1147,6 @@ class Qwen3_5LinearDecoderLayer(nn.Module): # The stage boundary completes this stage's sum. reduce_results=False, ) - is_layer_sparse = True - is_next_layer_sparse = True elif config.model_type == "qwen3_5_text": self.mlp = Qwen2MoeMLP( hidden_size=config.hidden_size, @@ -1136,8 +1158,6 @@ class Qwen3_5LinearDecoderLayer(nn.Module): reduce_results=False, ) _maybe_enable_silu_fp4_quant_fusion(self.mlp) - is_layer_sparse = False - is_next_layer_sparse = False else: raise ValueError(f"Invalid model type: {config.model_type}") @@ -1152,28 +1172,16 @@ class Qwen3_5LinearDecoderLayer(nn.Module): boundary_fusions = _layer_fusions(config, is_nextn) # A subclass that brings its own residual builds the stages itself. if build_stages: + attn, ffn = self.stage_facts( + config, + layer_id, + attn_read=NormQuantReadout( + fp8_input=Fp8Input.TUPLE_AND_BF16 if accepts_fp8_input else None + ), + ) self.attn_boundary, self.ffn_boundary = append_stages( - ( - declare_attn( - read=NormQuantReadout( - fp8_input=Fp8Input.TUPLE_AND_BF16 - if accepts_fp8_input - else None - ) - ), - self.input_layernorm, - { - "fusions": boundary_fusions, - }, - ), - ( - declare_ffn( - sparse=is_layer_sparse, - next_layer_sparse=is_next_layer_sparse, - ), - self.post_attention_layernorm, - {"fusions": boundary_fusions}, - ), + (attn, self.input_layernorm, {"fusions": boundary_fusions}), + (ffn, self.post_attention_layernorm, {"fusions": boundary_fusions}), ) def forward( @@ -1216,6 +1224,8 @@ class Qwen3_5LinearDecoderLayer(nn.Module): class Qwen3_5AttentionDecoderLayer(nn.Module): """Qwen3.5 Decoder Layer with Full Attention.""" + stage_facts = staticmethod(_qwen3_5_stage_facts) + def __init__( self, config: Qwen3_5TextConfig, @@ -1327,9 +1337,7 @@ class Qwen3_5AttentionDecoderLayer(nn.Module): # The stage boundary completes this stage's sum. reduce_results=False, ) - is_layer_sparse = False - is_next_layer_sparse = False - elif config.model_type in _QWEN3_5_MOE_TEXT_MODEL_TYPES: + elif _qwen3_5_is_moe(config): self.mlp = Qwen2MoeSparseMoeBlock( layer_id=layer_id, config=config, @@ -1345,8 +1353,6 @@ class Qwen3_5AttentionDecoderLayer(nn.Module): # The stage boundary completes this stage's sum. reduce_results=False, ) - is_layer_sparse = True - is_next_layer_sparse = True else: raise ValueError(f"Invalid model type: {config.model_type}") @@ -1364,26 +1370,16 @@ class Qwen3_5AttentionDecoderLayer(nn.Module): boundary_fusions = _layer_fusions(config, is_nextn) # A subclass that brings its own residual builds the stages itself. if build_stages: + attn, ffn = self.stage_facts( + config, + layer_id, + attn_read=NormQuantReadout( + fp8_input=Fp8Input.TUPLE if accepts_fp8_input else None + ), + ) self.attn_boundary, self.ffn_boundary = append_stages( - ( - declare_attn( - read=NormQuantReadout( - fp8_input=Fp8Input.TUPLE if accepts_fp8_input else None - ) - ), - self.input_layernorm, - { - "fusions": boundary_fusions, - }, - ), - ( - declare_ffn( - sparse=is_layer_sparse, - next_layer_sparse=is_next_layer_sparse, - ), - self.post_attention_layernorm, - {"fusions": boundary_fusions}, - ), + (attn, self.input_layernorm, {"fusions": boundary_fusions}), + (ffn, self.post_attention_layernorm, {"fusions": boundary_fusions}), ) self.alt_stream = alt_stream @@ -1752,6 +1748,10 @@ class Qwen3_5ForCausalLM(nn.Module): config.num_hidden_layers, get_layer, prefix=f"{prefix}.layers", + # The layer type chosen for each index declares its stages. + stage_facts=lambda idx: self.decoder_layer_types[ + config.layers_block_type[idx] + ].stage_facts(config, idx), ) self.flashinfer_mnnvl_cutedsl_fusion = None diff --git a/python/sglang/srt/models/qwen3_moe.py b/python/sglang/srt/models/qwen3_moe.py index a5698e177a..5cb3b37161 100644 --- a/python/sglang/srt/models/qwen3_moe.py +++ b/python/sglang/srt/models/qwen3_moe.py @@ -853,9 +853,8 @@ class Qwen3MoeDecoderLayer(nn.Module): self.layer_id = layer_id - # Qwen3MoE all layers are sparse and have no nextn now - self.is_layer_sparse = True - is_next_layer_sparse = True + attn, ffn = self.stage_facts(config, layer_id) + self.is_layer_sparse = ffn.sparse if self.is_layer_sparse: self.mlp = Qwen3MoeSparseMoeBlock( @@ -879,13 +878,23 @@ class Qwen3MoeDecoderLayer(nn.Module): ) self.attn_boundary, self.ffn_boundary = append_stages( - (declare_attn(), self.input_layernorm), - ( - declare_ffn( - sparse=self.is_layer_sparse, - next_layer_sparse=is_next_layer_sparse, - ), - self.post_attention_layernorm, + (attn, self.input_layernorm), + (ffn, self.post_attention_layernorm), + ) + + @staticmethod + def stage_facts(config: Qwen3MoeConfig, layer_id: int): + """The stages a Qwen3-MoE layer declares, from the config alone: the + model's shared declaration function, which the layer declares with + too (see make_layers).""" + # Qwen3MoE all layers are sparse and have no nextn now + is_layer_sparse = True + is_next_layer_sparse = True + return ( + declare_attn(), + declare_ffn( + sparse=is_layer_sparse, + next_layer_sparse=is_next_layer_sparse, ), ) diff --git a/python/sglang/srt/models/qwen4_exp.py b/python/sglang/srt/models/qwen4_exp.py index 4fd980b651..faebe4d497 100644 --- a/python/sglang/srt/models/qwen4_exp.py +++ b/python/sglang/srt/models/qwen4_exp.py @@ -81,6 +81,7 @@ from sglang.srt.models.qwen3_5 import ( Qwen3_5ForCausalLM, Qwen3_5GatedDeltaNet, Qwen3_5LinearDecoderLayer, + _qwen3_5_is_moe, ) from sglang.srt.models.qwen3_vl import Qwen3VLForConditionalGeneration from sglang.srt.models.qwen4_exp_ple_table import ( @@ -90,7 +91,6 @@ from sglang.srt.models.qwen4_exp_ple_table import ( ) from sglang.srt.runtime_context import get_parallel from sglang.srt.utils import get_bool_env_var, is_hip, logger -from sglang.srt.utils.common import is_building_neighbour_layer _use_aiter = get_bool_env_var("SGLANG_USE_AITER") and is_hip() @@ -537,13 +537,6 @@ class Qwen4ExpPLEGroupedNorm(nn.Module): return (x_norm * weight).to(compute_dtype) -def _offloads_ple(config) -> bool: - """Whether this layer keeps its PLE table in host memory. A pipeline - neighbour layer is never loaded or run, so it keeps the table on the meta - device and allocates no host memory or stream for it.""" - return bool(config.ple_offload_embedding) and not is_building_neighbour_layer() - - class Qwen4ExpNGramEmbedding(nn.Module): _MASK64 = (1 << 64) - 1 _SPLITMIX_GAMMA = 0x9E3779B97F4A7C15 @@ -615,7 +608,7 @@ class Qwen4ExpNGramEmbedding(nn.Module): and not self.use_attn_tp_ngram ) ngram_prefix = f"{prefix}.ngram_embedding" if prefix else "ngram_embedding" - offload_embedding = _offloads_ple(config) + offload_embedding = bool(config.ple_offload_embedding) # Offload only needs this embedding's metadata: build it on meta so the # shard is never allocated on the device. with torch.device("meta") if offload_embedding else nullcontext(): @@ -1098,7 +1091,9 @@ class Qwen4ExpPLELayer(nn.Module): bias=False, ) nn.init.zeros_(self.conv1d.weight) - self._prefetch_stream = torch.cuda.Stream() if _offloads_ple(config) else None + self._prefetch_stream = ( + torch.cuda.Stream() if config.ple_offload_embedding else None + ) self._graph_prefetch_buffers = {} self._eager_prefetch_buffer = None self._prefetch_state = None @@ -1449,31 +1444,45 @@ def _has_ple(layer_id, config) -> bool: return (layer_id + 1) in config.ple_layer_ids -def _build_qwen4_exp_stages(residual, *, sparse): - """The attention and FFN stage boundaries of one gated hyper-connection layer. +def _qwen4_exp_stage_facts(config, layer_id, *, residual=None): + """The attention and FFN stages a gated hyper-connection layer of either + kind declares: the model's shared declaration function (see make_layers), + which the layer declares itself with. ``residual`` is the layer's own + reads and updates of the streams; without it, the stages declare what + those do. A PLE layer's attention read needs every row, for the PLE + embedding is computed for all of them.""" + if residual is None: + residual = GatedResidualState.facts( + attn_reads_every_row=_has_ple(layer_id, config) + ) + sparse = _qwen3_5_is_moe(config) + return ( + declare_attn(read=residual.attn_readout, update=residual.attn_update), + declare_ffn( + sparse=sparse, + next_layer_sparse=sparse, + read=residual.ffn_readout, + update=residual.ffn_update, + ), + ) + + +def _build_qwen4_exp_stages(config, layer_id, residual): + """The attention and FFN stage boundaries of one gated hyper-connection + layer, declared by _qwen4_exp_stage_facts with the layer's own reads and + updates (``residual``). Each read normalizes the streams itself, so neither stage binds a norm, and the writes are gated injections, so neither stage offers a fused add-and-norm candidate. """ - return append_stages( - ( - declare_attn(read=residual.attn_readout, update=residual.attn_update), - None, - ), - ( - declare_ffn( - sparse=sparse, - next_layer_sparse=sparse, - read=residual.ffn_readout, - update=residual.ffn_update, - ), - None, - ), - ) + attn, ffn = _qwen4_exp_stage_facts(config, layer_id, residual=residual) + return append_stages((attn, None), (ffn, None)) class Qwen4ExpLayerExtensionMixin: + stage_facts = staticmethod(_qwen4_exp_stage_facts) + def _init_qwen4_exp_layer_extensions( self, config: Qwen4ExpTextConfig, @@ -1529,6 +1538,8 @@ class Qwen4ExpLayerExtensionMixin: use_combine=True, ) self.attn_boundary, self.ffn_boundary = _build_qwen4_exp_stages( + config, + layer_id, GatedResidualState( expand=self._widen_streams, attn_mix=self._attn_mix, @@ -1536,9 +1547,8 @@ class Qwen4ExpLayerExtensionMixin: attn_combine=self._attn_combine, ffn_combine=self._ffn_combine, # The PLE embedding is computed for every row. - attn_reads_every_row=self.ple is not None, + attn_reads_every_row=_has_ple(layer_id, config), ).residual_ops(), - sparse=isinstance(self.mlp, Qwen2MoeSparseMoeBlock), ) from sglang.srt.layers.moe.qwen4_decode import prepare_qwen4_decode_comm diff --git a/python/sglang/srt/models/sarvam_moe.py b/python/sglang/srt/models/sarvam_moe.py index 6763a03a1e..2477ce2837 100644 --- a/python/sglang/srt/models/sarvam_moe.py +++ b/python/sglang/srt/models/sarvam_moe.py @@ -5,6 +5,7 @@ import math from enum import IntEnum, auto +from functools import partial from typing import Any, Dict, Iterable, Optional, Tuple import torch @@ -986,20 +987,7 @@ class SarvamMoEMLADecoderLayer(nn.Module): alt_stream=alt_stream, ) - first_k_dense = getattr(config, "first_k_dense_replace", 1) - moe_layer_freq = getattr(config, "moe_layer_freq", 1) - has_moe = getattr(config, "num_experts", None) is not None - self.is_layer_sparse = ( - has_moe - and layer_id >= first_k_dense - and (layer_id - first_k_dense) % moe_layer_freq == 0 - ) - is_next_layer_sparse = ( - has_moe - and layer_id < config.num_hidden_layers - 1 - and (layer_id + 1) >= first_k_dense - and (layer_id + 1 - first_k_dense) % moe_layer_freq == 0 - ) + self.is_layer_sparse = self._is_layer_sparse(config, layer_id) if self.is_layer_sparse: self.mlp = SarvamMoESparseMoeBlock( @@ -1026,21 +1014,52 @@ class SarvamMoEMLADecoderLayer(nn.Module): config.hidden_size, eps=config.rms_norm_eps ) + attn, ffn = self.stage_facts(config, layer_id) self.attn_boundary, self.ffn_boundary = append_stages( ( - declare_attn(), + attn, self.input_layernorm, {"qkv_latent_func": self.self_attn.prepare_qkv_latent}, ), - ( - declare_ffn( - sparse=self.is_layer_sparse, - next_layer_sparse=is_next_layer_sparse, - ), - self.post_attention_layernorm, + (ffn, self.post_attention_layernorm), + ) + + @classmethod + def stage_facts(cls, config: PretrainedConfig, layer_id: int): + """The stages the layer at ``layer_id`` declares, in order: the model's + shared declaration function, which the layer declares with too (see + make_layers).""" + return ( + declare_attn(), + declare_ffn( + sparse=cls._is_layer_sparse(config, layer_id), + next_layer_sparse=cls._is_next_layer_sparse(config, layer_id), ), ) + @staticmethod + def _is_layer_sparse(config: PretrainedConfig, layer_id: int) -> bool: + first_k_dense = getattr(config, "first_k_dense_replace", 1) + moe_layer_freq = getattr(config, "moe_layer_freq", 1) + has_moe = getattr(config, "num_experts", None) is not None + return ( + has_moe + and layer_id >= first_k_dense + and (layer_id - first_k_dense) % moe_layer_freq == 0 + ) + + @staticmethod + def _is_next_layer_sparse(config: PretrainedConfig, layer_id: int) -> bool: + first_k_dense = getattr(config, "first_k_dense_replace", 1) + moe_layer_freq = getattr(config, "moe_layer_freq", 1) + has_moe = getattr(config, "num_experts", None) is not None + return ( + has_moe + and layer_id < config.num_hidden_layers - 1 + and (layer_id + 1) >= first_k_dense + and (layer_id + 1 - first_k_dense) % moe_layer_freq == 0 + ) + def forward( self, positions: torch.Tensor, @@ -1096,6 +1115,7 @@ class SarvamMLAModel(nn.Module): alt_stream=self.alt_stream, ), prefix="model.layers", + stage_facts=partial(SarvamMoEMLADecoderLayer.stage_facts, config), ) if self.pp_group.is_last_rank: diff --git a/python/sglang/srt/models/sdar.py b/python/sglang/srt/models/sdar.py index 87d2c5f8d3..bf02f090e5 100644 --- a/python/sglang/srt/models/sdar.py +++ b/python/sglang/srt/models/sdar.py @@ -4,6 +4,7 @@ SGLang SDARModelLM (block diffusion / dLLM-style forward). """ import logging +from functools import partial from typing import Iterable, Optional, Tuple, Union import torch @@ -291,14 +292,22 @@ class SDARBlock(nn.Module): prefix=add_prefix("mlp", prefix), ) + attn, ffn = self.stage_facts(config, layer_id) self.attn_boundary, self.ffn_boundary = append_stages( - (declare_attn(), self.input_layernorm), - ( - declare_ffn( - sparse=False, - next_layer_sparse=False, - ), - self.post_attention_layernorm, + (attn, self.input_layernorm), + (ffn, self.post_attention_layernorm), + ) + + @staticmethod + def stage_facts(config: PretrainedConfig, layer_id: int): + """The stages the layer at ``layer_id`` declares, in order: the model's + shared declaration function, which the layer declares with too (see + make_layers).""" + return ( + declare_attn(), + declare_ffn( + sparse=False, + next_layer_sparse=False, ), ) @@ -361,6 +370,7 @@ class SDARModel(nn.Module): alt_stream=alt_stream, ), prefix=add_prefix("layers", prefix), + stage_facts=partial(SDARBlock.stage_facts, config), ) if self.pp_group.is_last_rank: norm_kwargs = ( diff --git a/python/sglang/srt/models/sdar_moe.py b/python/sglang/srt/models/sdar_moe.py index 90095df243..df9738f4cd 100644 --- a/python/sglang/srt/models/sdar_moe.py +++ b/python/sglang/srt/models/sdar_moe.py @@ -4,6 +4,7 @@ SGLang SDARMoeModelLM (block diffusion / dLLM-style forward) with MoE MLP. """ import logging +from functools import partial from typing import Iterable, Optional, Tuple, Union import torch @@ -351,12 +352,20 @@ class SDARMoeBlock(nn.Module): prefix=add_prefix("mlp", prefix), ) + attn, ffn = self.stage_facts(config, layer_id) self.attn_boundary, self.ffn_boundary = append_stages( - (declare_attn(), self.input_layernorm), - ( - declare_ffn(sparse=True, next_layer_sparse=True), - self.post_attention_layernorm, - ), + (attn, self.input_layernorm), + (ffn, self.post_attention_layernorm), + ) + + @staticmethod + def stage_facts(config: PretrainedConfig, layer_id: int): + """The stages the layer at ``layer_id`` declares, in order: the model's + shared declaration function, which the layer declares with too (see + make_layers).""" + return ( + declare_attn(), + declare_ffn(sparse=True, next_layer_sparse=True), ) def forward( @@ -422,6 +431,7 @@ class SDARMoeModel(nn.Module): alt_stream=alt_stream, ), prefix=add_prefix("layers", prefix), + stage_facts=partial(SDARMoeBlock.stage_facts, config), ) if self.pp_group.is_last_rank: diff --git a/python/sglang/srt/models/spark2_5.py b/python/sglang/srt/models/spark2_5.py index c945a3a596..393715ea4a 100644 --- a/python/sglang/srt/models/spark2_5.py +++ b/python/sglang/srt/models/spark2_5.py @@ -1,4 +1,5 @@ import logging +from functools import partial from typing import Iterable, Optional, Tuple, Union import torch @@ -267,12 +268,20 @@ class Spark2_5DecoderLayer(nn.Module): self.post_attention_layernorm = RMSNorm( config.hidden_size, eps=config.rms_norm_eps ) + attn, ffn = self.stage_facts(config, layer_id) self.attn_boundary, self.ffn_boundary = append_stages( - (declare_attn(), self.input_layernorm), - ( - declare_ffn(sparse=False, next_layer_sparse=False), - self.post_attention_layernorm, - ), + (attn, self.input_layernorm), + (ffn, self.post_attention_layernorm), + ) + + @staticmethod + def stage_facts(config: Spark2_5Config, layer_id: int): + """The stages the layer at ``layer_id`` declares, in order: the model's + shared declaration function, which the layer declares with too (see + make_layers).""" + return ( + declare_attn(), + declare_ffn(sparse=False, next_layer_sparse=False), ) def forward( @@ -326,6 +335,7 @@ class Spark2_5Model(nn.Module): prefix=prefix, ), prefix=add_prefix("layers", prefix), + stage_facts=partial(Spark2_5DecoderLayer.stage_facts, config), ) if self.pp_group.is_last_rank: self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps) diff --git a/python/sglang/srt/models/step3p5.py b/python/sglang/srt/models/step3p5.py index 43a95b772b..7c7c45b8d4 100644 --- a/python/sglang/srt/models/step3p5.py +++ b/python/sglang/srt/models/step3p5.py @@ -1,3 +1,4 @@ +from functools import partial from typing import Any, Dict, Iterable, Optional, Tuple, Union import torch @@ -422,11 +423,10 @@ class Step3p5DecoderLayer(nn.Module): rope_theta = config.rope_theta max_position_embeddings = config.max_position_embeddings head_dim = config.head_dim - moe_layers_set = {int(x) for x in config.moe_layers_enum.split(",")} self.num_attention_heads = config.num_attention_heads self.num_key_value_heads = config.num_attention_groups - self.is_moe_layer = layer_id in moe_layers_set - self.is_next_layer_sparse = (layer_id + 1) in moe_layers_set + self.is_moe_layer = self._is_moe_layer(config, layer_id) + self.is_next_layer_sparse = self._is_moe_layer(config, layer_id + 1) num_hidden_layers = config.num_hidden_layers if ( @@ -519,19 +519,32 @@ class Step3p5DecoderLayer(nn.Module): config.hidden_size, eps=config.rms_norm_eps ) + attn, ffn = self.stage_facts(config, layer_id) self.attn_boundary, self.ffn_boundary = append_stages( - (declare_attn(), self.input_layernorm), - ( - declare_ffn( - sparse=self.is_moe_layer, - next_layer_sparse=self.is_next_layer_sparse, - ), - self.post_attention_layernorm, - ), + (attn, self.input_layernorm), + (ffn, self.post_attention_layernorm), ) self.layer_id = layer_id + @classmethod + def stage_facts(cls, config: Step3p5Config, layer_id: int): + """The stages the layer at ``layer_id`` declares, in order: the model's + shared declaration function, which the layer declares with too (see + make_layers).""" + return ( + declare_attn(), + declare_ffn( + sparse=cls._is_moe_layer(config, layer_id), + next_layer_sparse=cls._is_moe_layer(config, layer_id + 1), + ), + ) + + @staticmethod + def _is_moe_layer(config: Step3p5Config, layer_id: int) -> bool: + moe_layers_set = {int(x) for x in config.moe_layers_enum.split(",")} + return layer_id in moe_layers_set + def forward( self, positions: torch.Tensor, @@ -605,6 +618,7 @@ class Step3p5Model(nn.Module): alt_stream=alt_stream, ), prefix=add_prefix("layers", prefix), + stage_facts=partial(Step3p5DecoderLayer.stage_facts, config), ) if self.pp_group.is_last_rank: self.norm = GemmaRMSNorm(config.hidden_size, eps=config.rms_norm_eps) diff --git a/python/sglang/srt/models/xllm.py b/python/sglang/srt/models/xllm.py index d8c9f411a0..7c75d842ea 100644 --- a/python/sglang/srt/models/xllm.py +++ b/python/sglang/srt/models/xllm.py @@ -28,6 +28,7 @@ import math from contextlib import nullcontext +from functools import partial from typing import Any, Dict, Iterable, Optional, Tuple, Union import torch @@ -1433,15 +1434,7 @@ class XllmDecoderLayer(nn.Module): self.layer_id = layer_id - # Determine if this layer is sparse (MoE) or dense - mlp_only_layers = getattr(config, "mlp_only_layers", []) - decoder_sparse_step = getattr(config, "decoder_sparse_step", 1) - if (layer_id not in mlp_only_layers) and ( - config.num_experts > 0 and (layer_id + 1) % decoder_sparse_step == 0 - ): - self.is_layer_sparse = True - else: - self.is_layer_sparse = False + self.is_layer_sparse = self._is_layer_sparse(config, layer_id) is_mova_config = getattr(config, "num_values", 0) > 0 is_mova_attention = is_mova_config and layer_id >= config.num_dense_layers @@ -1475,16 +1468,6 @@ class XllmDecoderLayer(nn.Module): prefix=add_prefix("self_attn", prefix), ) - # Check neighbors for scatter modes - def _is_sparse(lid): - if lid < 0 or lid >= config.num_hidden_layers: - return False - return (lid not in mlp_only_layers) and ( - config.num_experts > 0 and (lid + 1) % decoder_sparse_step == 0 - ) - - is_next_layer_sparse = _is_sparse(layer_id + 1) - if self.is_layer_sparse: self.mlp = XllmSparseMoeBlock( layer_id=layer_id, @@ -1506,17 +1489,49 @@ class XllmDecoderLayer(nn.Module): self.input_layernorm = _make_norm(config) self.post_attention_layernorm = _make_norm(config) + attn, ffn = self.stage_facts(config, layer_id) self.attn_boundary, self.ffn_boundary = append_stages( - (declare_attn(), self.input_layernorm), - ( - declare_ffn( - sparse=self.is_layer_sparse, - next_layer_sparse=is_next_layer_sparse, - ), - self.post_attention_layernorm, + (attn, self.input_layernorm), + (ffn, self.post_attention_layernorm), + ) + + @classmethod + def stage_facts(cls, config: PretrainedConfig, layer_id: int): + """The stages the layer at ``layer_id`` declares, in order: the model's + shared declaration function, which the layer declares with too (see + make_layers).""" + return ( + declare_attn(), + declare_ffn( + sparse=cls._is_layer_sparse(config, layer_id), + next_layer_sparse=cls._is_next_layer_sparse(config, layer_id), ), ) + @staticmethod + def _is_layer_sparse(config: PretrainedConfig, layer_id: int) -> bool: + # Determine if this layer is sparse (MoE) or dense + mlp_only_layers = getattr(config, "mlp_only_layers", []) + decoder_sparse_step = getattr(config, "decoder_sparse_step", 1) + if (layer_id not in mlp_only_layers) and ( + config.num_experts > 0 and (layer_id + 1) % decoder_sparse_step == 0 + ): + return True + else: + return False + + @staticmethod + def _is_next_layer_sparse(config: PretrainedConfig, layer_id: int) -> bool: + # Check neighbors for scatter modes + mlp_only_layers = getattr(config, "mlp_only_layers", []) + decoder_sparse_step = getattr(config, "decoder_sparse_step", 1) + lid = layer_id + 1 + if lid < 0 or lid >= config.num_hidden_layers: + return False + return (lid not in mlp_only_layers) and ( + config.num_experts > 0 and (lid + 1) % decoder_sparse_step == 0 + ) + def forward( self, positions: torch.Tensor, @@ -1576,6 +1591,7 @@ class XllmModel(nn.Module): prefix=prefix, ), prefix=add_prefix("layers", prefix), + stage_facts=partial(XllmDecoderLayer.stage_facts, config), ) if self.pp_group.is_last_rank: self.norm = _make_norm(config) diff --git a/python/sglang/srt/models/zaya.py b/python/sglang/srt/models/zaya.py index 5da953b8a6..fc9c68ac15 100644 --- a/python/sglang/srt/models/zaya.py +++ b/python/sglang/srt/models/zaya.py @@ -1317,11 +1317,16 @@ class ZayaDecoderATTLayer(nn.Module): self.res_scale = ResidualScaling(config, layer_id) else: self.res_scale = None - (self.attn_boundary,) = append_stages( - (declare_attn(read=_ResidualMergeRead(self.res_scale)), self.input_norm), - ) + (attn,) = self.stage_facts(res_scale=self.res_scale) + (self.attn_boundary,) = append_stages((attn, self.input_norm)) self.entry_boundary = self.attn_boundary + @staticmethod + def stage_facts(*, res_scale: Optional[ResidualScaling] = None): + """The one stage an attention layer declares: its attention, read + through ``res_scale``.""" + return (declare_attn(read=_ResidualMergeRead(res_scale)),) + @staticmethod def _build_norm(config: ZayaConfig) -> nn.Module: if config.normalization == "RMSNorm": @@ -1368,17 +1373,21 @@ class ZayaDecoderMLPLayer(nn.Module): self.res_scale = ResidualScaling(config, layer_id) else: self.res_scale = None - (self.ffn_boundary,) = append_stages( - ( - declare_ffn( - sparse=True, - next_layer_sparse=True, - read=_ResidualMergeRead(self.res_scale), - ), - self.input_norm, + (ffn,) = self.stage_facts(res_scale=self.res_scale) + (self.ffn_boundary,) = append_stages((ffn, self.input_norm)) + self.entry_boundary = self.ffn_boundary + + @staticmethod + def stage_facts(*, res_scale: Optional[ResidualScaling] = None): + """The one stage an MoE layer declares: its MoE, read through + ``res_scale``.""" + return ( + declare_ffn( + sparse=True, + next_layer_sparse=True, + read=_ResidualMergeRead(res_scale), ), ) - self.entry_boundary = self.ffn_boundary def forward( self, @@ -1400,23 +1409,22 @@ class ZayaDecoderMLPLayer(nn.Module): # --------------------------------------------------------------------------- +def _layer_class(layer_id: int): + # Even layer ids are attention, odd layer ids are MoE. This matches the HF + # checkpoint keys: ``model.layers.<2k>.self_attn.*`` (CCA) versus + # ``model.layers.<2k+1>.zaya_block.*`` (MoE). + if layer_id % 2 == 0: + return ZayaDecoderATTLayer + return ZayaDecoderMLPLayer + + def _build_layer( layer_id: int, config: ZayaConfig, quant_config: Optional[QuantizationConfig], prefix: str, ) -> nn.Module: - # Even layer ids are attention, odd layer ids are MoE. This matches the HF - # checkpoint keys: ``model.layers.<2k>.self_attn.*`` (CCA) versus - # ``model.layers.<2k+1>.zaya_block.*`` (MoE). - if layer_id % 2 == 0: - return ZayaDecoderATTLayer( - config=config, - layer_id=layer_id, - quant_config=quant_config, - prefix=prefix, - ) - return ZayaDecoderMLPLayer( + return _layer_class(layer_id)( config=config, layer_id=layer_id, quant_config=quant_config, @@ -1424,6 +1432,13 @@ def _build_layer( ) +def stage_facts(layer_id: int): + """The one stage the layer at ``layer_id`` declares, from its kind alone: + the model's shared declaration function, which its layers declare with + too (see make_layers).""" + return _layer_class(layer_id).stage_facts() + + class ZayaModel(nn.Module): def __init__( self, @@ -1456,6 +1471,7 @@ class ZayaModel(nn.Module): prefix=prefix, ), prefix=add_prefix("layers", prefix), + stage_facts=stage_facts, ) if self.pp_group.is_last_rank: diff --git a/python/sglang/srt/utils/common.py b/python/sglang/srt/utils/common.py index 9ea4928408..e14043ed2d 100644 --- a/python/sglang/srt/utils/common.py +++ b/python/sglang/srt/utils/common.py @@ -1551,33 +1551,6 @@ def mark_end(name): time_infos[name].pretty_print() -# Set while a layer another pipeline stage holds is built, only for the stage -# boundaries it declares. -_building_neighbour_layer = False - - -def is_building_neighbour_layer() -> bool: - """Whether the layer under construction belongs to another pipeline stage - and is built here only to read the stage boundaries it declares. It is - built on the meta device and never loaded or run, so its constructor skips - the host and device resources a running layer needs: tables, streams, - engines and communicators.""" - return _building_neighbour_layer - - -@contextmanager -def building_neighbour_layer(): - """Build a pipeline neighbour layer: on the meta device, with - is_building_neighbour_layer() true.""" - global _building_neighbour_layer - outer, _building_neighbour_layer = _building_neighbour_layer, True - try: - with torch.device("meta"): - yield - finally: - _building_neighbour_layer = outer - - class LayerFn(Protocol): def __call__(self, idx: int, prefix: str) -> torch.nn.Module: ... @@ -1601,10 +1574,9 @@ def make_layers( ``stage_facts``: the model's shared declaration function, which returns the stages the layer at a global index declares without building it (an empty sequence when it declares none), and which the layer itself uses - for its own declarations. A model on the transitional list may omit it; - the stack then builds the layer again on the meta device to read them. - ``final_read`` is the stack's terminal read when it is not a plain final - norm (see layer_stack). + for its own declarations. A model whose layers declare stages must give + it to run under pipeline parallelism. ``final_read`` is the stack's + terminal read when it is not a plain final norm (see layer_stack). """ # circular imports from sglang.srt.distributed import get_pp_indices @@ -1628,10 +1600,8 @@ def make_layers( def neighbour(idx): if stage_facts is not None: - return functools.partial(_declared_stages, stage_facts, idx) - return functools.partial( - _build_neighbour_layer, layer_fn, idx, add_prefix(idx, prefix) - ) + return functools.partial(stage_facts, idx) + return functools.partial(_no_stage_facts, layer_fn) def build(stack, idx): appended = len(stack.appends) @@ -1690,84 +1660,15 @@ def make_pp_layers( ) -# The models whose layers a pipeline rank still builds on the meta device to -# read the stages they declare, until each gives make_layers its shared -# declaration function (stage_facts). Any other model must give one to run -# under pipeline parallelism. -_NEIGHBOUR_BUILD_MODULES = frozenset( - ( - "sglang.srt.models.apertus", - "sglang.srt.models.arcee", - "sglang.srt.models.bailing_moe", - "sglang.srt.models.bailing_moe_linear", - "sglang.srt.models.bailing_moe_v3", - "sglang.srt.models.deepseek_v2", - "sglang.srt.models.dots3_common.modeling", - "sglang.srt.models.ernie45_moe_vl", - "sglang.srt.models.exaone4", - "sglang.srt.models.exaone_moe", - "sglang.srt.models.gemma4_causal", - "sglang.srt.models.gigachat35", - "sglang.srt.models.glm4", - "sglang.srt.models.glm4_moe", - "sglang.srt.models.glm4_moe_lite", - "sglang.srt.models.glm5_next", - "sglang.srt.models.gpt_oss", - "sglang.srt.models.granitemoehybrid", - "sglang.srt.models.kimi_k3", - "sglang.srt.models.kimi_linear", - "sglang.srt.models.laguna", - "sglang.srt.models.llada2", - "sglang.srt.models.llama", - "sglang.srt.models.mimo_v2", - "sglang.srt.models.minimax_m2", - "sglang.srt.models.minimax_m3", - "sglang.srt.models.ministral3", - "sglang.srt.models.mixtral", - "sglang.srt.models.nanbeige", - "sglang.srt.models.nemotron_nas", - "sglang.srt.models.qwen2", - "sglang.srt.models.qwen2_moe", - "sglang.srt.models.qwen3_5", - "sglang.srt.models.sarvam_moe", - "sglang.srt.models.sdar", - "sglang.srt.models.sdar_moe", - "sglang.srt.models.spark2_5", - "sglang.srt.models.step3p5", - "sglang.srt.models.xllm", - "sglang.srt.models.zaya", - ) -) - - -def _declared_stages(stage_facts, idx: int): - """The stages the layer at ``idx`` declares, from the model's shared - declaration function, standing in for building it.""" - from sglang.srt.layers.layer_boundary.factories import DeclaredStages - - return DeclaredStages(tuple(stage_facts(idx))) - - -def _build_neighbour_layer(layer_fn: LayerFn, idx: int, prefix: str) -> None: - """Build a layer another pipeline stage holds, only for the stage - boundaries it declares (see building_neighbour_layer). RoPE modules it - adds to the shared cache are meta, so they are dropped again.""" - from sglang.srt.layers.rotary_embedding.factory import _ROPE_DICT - +def _no_stage_facts(layer_fn: LayerFn) -> None: + """Stands in for the shared declaration function of a model that gives + none: called only if this pipeline rank's layers declare stages.""" module = getattr(getattr(layer_fn, "func", layer_fn), "__module__", None) - if module not in _NEIGHBOUR_BUILD_MODULES: - raise ValueError( - f"{module} declares stage boundaries under pipeline parallelism " - "without a shared declaration function: give make_layers its " - "stage_facts, the stages a layer declares without building it" - ) - cached = set(_ROPE_DICT) - try: - with building_neighbour_layer(): - layer_fn(idx=idx, prefix=prefix) - finally: - for key in set(_ROPE_DICT) - cached: - del _ROPE_DICT[key] + raise ValueError( + f"{module} declares stage boundaries under pipeline parallelism " + "without a shared declaration function: give make_layers its " + "stage_facts, the stages a layer declares without building it" + ) def set_random_seed(seed: int) -> None: diff --git a/python/sglang/test/boundary_fixtures.py b/python/sglang/test/boundary_fixtures.py index e810ea5655..cce4292993 100644 --- a/python/sglang/test/boundary_fixtures.py +++ b/python/sglang/test/boundary_fixtures.py @@ -28,19 +28,13 @@ def build_stages(*stages, previous=None, terminal=False): terminal: Whether the last stage ends the model's layer stack; otherwise the next layer's attention follows it on the same rank. """ - # The stack's neighbour hooks build layers another pipeline rank holds; - # here they stand for this rank's, so nothing is handed off to them. + # The stack's neighbours are layers another pipeline rank holds; here + # they stand for this rank's, so nothing is handed off to them. with ( mock.patch.object(factories, "_handed_off", lambda declaration: declaration), layer_stack( - previous_layers=( - [lambda: append_stages((previous, None))] - if previous is not None - else [] - ), - next_layers=( - [] if terminal else [lambda: append_stages((declare_attn(), None))] - ), + previous_layers=[lambda: (previous,)] if previous is not None else [], + next_layers=[] if terminal else [lambda: (declare_attn(),)], ), ): return append_stages(*stages) diff --git a/test/registered/unit/layer_boundary/test_append_stages.py b/test/registered/unit/layer_boundary/test_append_stages.py index fac952076b..205ae3b5d0 100644 --- a/test/registered/unit/layer_boundary/test_append_stages.py +++ b/test/registered/unit/layer_boundary/test_append_stages.py @@ -36,10 +36,8 @@ from sglang.srt.layers.layer_boundary.residual.add_norm import ( NormQuantReadout, ) from sglang.srt.layers.layer_boundary.residual.stream import DeclaredSum -from sglang.srt.layers.rotary_embedding import factory as rope_factory from sglang.srt.model_executor.forward_batch_info import PPProxyTensors -from sglang.srt.utils import common -from sglang.srt.utils.common import is_building_neighbour_layer, make_layers +from sglang.srt.utils.common import make_layers from sglang.test.ci.ci_register import register_cpu_ci from sglang.test.communicator_patch import patch_communicator from sglang.test.test_utils import CustomTestCase @@ -47,11 +45,19 @@ from sglang.test.test_utils import CustomTestCase register_cpu_ci(est_time=5, suite="base-a-test-cpu") +def layer_stages(sparse=False): + """The stages one decoder layer declares: attention, then an FFN. A + pipeline rank reads them for a layer another rank holds.""" + return ( + declare_attn(), + declare_ffn(sparse=sparse, next_layer_sparse=sparse), + ) + + def layer(sparse=False): - """One decoder layer's stages: attention, then an FFN.""" + """One decoder layer, declaring layer_stages.""" return append_stages( - (declare_attn(), fixture.Norm()), - (declare_ffn(sparse=sparse, next_layer_sparse=sparse), fixture.Norm()), + *((declaration, fixture.Norm()) for declaration in layer_stages(sparse)) ) @@ -94,41 +100,39 @@ class TestAppendStages(CustomTestCase): layer() def test_the_previous_layer_gives_the_first_stage_its_producer(self): - built = [] + read = [] def previous_layer(): - built.append(None) - layer(sparse=True) + read.append(None) + return layer_stages(sparse=True) with layer_stack(previous_layers=[previous_layer]): first, _ = layer() - self.assertEqual(len(built), 1) + self.assertEqual(len(read), 1) self.assertTrue(first.declaration.previous.sparse) self.assertFalse(first.plan.enters_stack) def test_the_next_layer_gives_the_last_stage_its_consumer(self): - with layer_stack( - next_layers=[lambda: append_stages((declare_ffn(), fixture.Norm()))] - ): + with layer_stack(next_layers=[lambda: (declare_ffn(),)]): (stage,) = append_stages((mixer(), fixture.Norm())) self.assertFalse(stage.plan.terminal) # It leaves its sum to the FFN that follows it, which completes it. for edge in stage.plan.edges.values(): self.assertTrue(edge.outgoing.produced.always_partial) - def test_a_stack_without_stages_builds_no_neighbour(self): + def test_a_stack_without_stages_reads_no_neighbour(self): def unexpected(): - self.fail("a neighbour was built for a stack without stages") + self.fail("a neighbour was read for a stack without stages") with layer_stack(previous_layers=[unexpected], next_layers=[unexpected]): pass - def test_neighbours_are_built_after_the_stacks_own_layers(self): + def test_neighbours_are_read_after_the_stacks_own_layers(self): order = [] def neighbour(): order.append("neighbour") - layer() + return layer_stages() with layer_stack(previous_layers=[neighbour], next_layers=[neighbour]): order.append("own") @@ -136,10 +140,12 @@ class TestAppendStages(CustomTestCase): self.assertEqual(order, ["own", "neighbour", "neighbour"]) def test_a_neighbour_without_stages_is_passed_over(self): - with layer_stack(previous_layers=[lambda: None, lambda: layer(sparse=True)]): + with layer_stack( + previous_layers=[lambda: (), lambda: layer_stages(sparse=True)] + ): first, _ = layer() self.assertTrue(first.declaration.previous.sparse) - with layer_stack(previous_layers=[lambda: None], next_layers=[lambda: None]): + with layer_stack(previous_layers=[lambda: ()], next_layers=[lambda: ()]): first, last = layer() # No earlier layer declares a stage: the stack starts here, and ends # here when no later one does. @@ -148,10 +154,9 @@ class TestAppendStages(CustomTestCase): def test_a_neighbour_is_read_without_joining_the_stack(self): def previous_layer(): - # A layer with a dense FFN, then one with a MoE: the producer is - # the nearest stage, and neither joins this stack. - layer(sparse=False) - layer(sparse=True) + # A dense FFN, then a MoE: the producer is the nearest stage, and + # neither joins this stack. + return (declare_ffn(sparse=False), declare_ffn(sparse=True)) with layer_stack(previous_layers=[previous_layer]): stages = layer() @@ -271,7 +276,7 @@ class TestOneStagePerAppend(CustomTestCase): # or the final read, takes it on the rows that attention ran on, so # the a2a MoE before it returns it on the attention's rows. def ffn_layer(): - append_stages((declare_ffn(sparse=True), fixture.Norm())) + return (declare_ffn(sparse=True),) for next_layers in ((), (ffn_layer,)): with self.subTest(hands_off=bool(next_layers)): @@ -296,41 +301,43 @@ class TestOneStagePerAppend(CustomTestCase): ) +def stage_facts(idx): + """The shared declaration function of TestMakeLayers' layers: layer 1 + has a MoE.""" + return layer_stages(sparse=idx == 1) + + class TestMakeLayers(CustomTestCase): """make_layers builds its layers in one stack, and on a pipeline stage - reads the stages the other stages' layers declare next to it.""" + reads the stages the other stages' layers declare next to it from the + model's shared declaration function.""" def setUp(self): self.planning = fixture.planning(fixture.parallel_of(attn_dp=1, attn_tp=2)) self.planning.__enter__() self.addCleanup(self.planning.__exit__, None, None, None) - # These layers are built again on the meta device, as a model's on the - # transitional list are, unless a test gives a declaration function. - allowed = patch.object( - common, - "_NEIGHBOUR_BUILD_MODULES", - common._NEIGHBOUR_BUILD_MODULES | {__name__}, - ) - allowed.start() - self.addCleanup(allowed.stop) - def build(self, pp_rank=None, pp_size=None, *, declares=True): - """Build the layers; layer 1 has a MoE. Returns each layer_fn call as - (index, prefix, device, built as a neighbour) and the stages of the - layers built here.""" - calls, stages = [], {} + def build(self, pp_rank=None, pp_size=None, *, declares=True, facts=stage_facts): + """Build the layers. Returns the indices layer_fn built and the stages + of the layers built here.""" + built, stages = [], {} def layer_fn(idx, prefix): - device = torch.empty(()).device.type - calls.append((idx, prefix, device, is_building_neighbour_layer())) + self.assertEqual(prefix, f"layers.{idx}") + built.append(idx) if declares: - stages.setdefault(idx, layer(sparse=idx == 1)) + stages[idx] = layer(sparse=idx == 1) return nn.Identity() make_layers( - NUM_LAYERS, layer_fn, pp_rank=pp_rank, pp_size=pp_size, prefix="layers" + NUM_LAYERS, + layer_fn, + pp_rank=pp_rank, + pp_size=pp_size, + prefix="layers", + stage_facts=facts, ) - return calls, stages + return built, stages def test_the_layers_form_one_stack(self): _, stages = self.build() @@ -338,27 +345,11 @@ class TestMakeLayers(CustomTestCase): self.assertEqual([s.plan.terminal for s in stages], [False] * 7 + [True]) self.assertEqual([s.plan.enters_stack for s in stages], [True] + [False] * 7) - def test_a_pipeline_stage_builds_its_neighbours_on_meta_after_its_layers(self): - calls, _ = self.build(pp_rank=0, pp_size=2) - self.assertEqual( - calls, - [ - (0, "layers.0", "cpu", False), - (1, "layers.1", "cpu", False), - (2, "layers.2", "meta", True), - ], - ) - calls, _ = self.build(pp_rank=1, pp_size=2) - self.assertEqual( - calls, - [ - (2, "layers.2", "cpu", False), - (3, "layers.3", "cpu", False), - (1, "layers.1", "meta", True), - ], - ) + def test_a_pipeline_stage_builds_only_its_own_layers(self): + self.assertEqual(self.build(pp_rank=0, pp_size=2)[0], [0, 1]) + self.assertEqual(self.build(pp_rank=1, pp_size=2)[0], [2, 3]) - def test_the_neighbours_connect_the_stack_across_pipeline_stages(self): + def test_the_declaration_function_connects_the_stack_across_stages(self): _, first = self.build(pp_rank=0, pp_size=2) self.assertTrue(first[0][0].plan.enters_stack) self.assertFalse(first[1][1].plan.terminal) @@ -368,33 +359,6 @@ class TestMakeLayers(CustomTestCase): self.assertFalse(last[2][0].plan.enters_stack) self.assertTrue(last[3][1].plan.terminal) - def test_a_shared_declaration_function_stands_in_for_the_neighbours(self): - built, stages = [], {} - - def layer_fn(idx, prefix): - built.append(idx) - stages[idx] = layer(sparse=idx == 1) - return nn.Identity() - - def stage_facts(idx): - # The stages layer() declares, without building the layer. - return ( - declare_attn(), - declare_ffn(sparse=idx == 1, next_layer_sparse=idx == 1), - ) - - make_layers( - NUM_LAYERS, - layer_fn, - pp_rank=1, - pp_size=2, - prefix="layers", - stage_facts=stage_facts, - ) - self.assertEqual(built, [2, 3]) - # Layer 2's attention follows layer 1's MoE, on the other stage. - self.assertTrue(stages[2][0].declaration.previous.sparse) - def test_a_layer_must_declare_what_its_declaration_function_says(self): def layer_fn(idx, prefix): layer(sparse=idx == 2) @@ -413,25 +377,13 @@ class TestMakeLayers(CustomTestCase): stage_facts=lambda idx: (declare_attn(), declare_ffn()), ) - def test_a_model_off_the_list_needs_a_declaration_function(self): - with patch.object(common, "_NEIGHBOUR_BUILD_MODULES", frozenset()): - with self.assertRaisesRegex(ValueError, "stage_facts"): - self.build(pp_rank=1, pp_size=2) + def test_layers_with_stages_need_a_declaration_function(self): + with self.assertRaisesRegex(ValueError, "stage_facts"): + self.build(pp_rank=1, pp_size=2, facts=None) - def test_layers_without_stages_build_no_neighbour(self): - calls, _ = self.build(pp_rank=1, pp_size=2, declares=False) - self.assertEqual([call[0] for call in calls], [2, 3]) - - def test_a_neighbour_s_rope_modules_leave_the_cache(self): - def layer_fn(idx, prefix): - rope_factory._ROPE_DICT[("test", idx)] = None - layer() - return nn.Identity() - - with patch.dict(rope_factory._ROPE_DICT): - make_layers(NUM_LAYERS, layer_fn, pp_rank=0, pp_size=2) - added = {key for key in rope_factory._ROPE_DICT if key[0] == "test"} - self.assertEqual(added, {("test", 0), ("test", 1)}) + def test_layers_without_stages_need_no_declaration_function(self): + built, _ = self.build(pp_rank=1, pp_size=2, declares=False, facts=None) + self.assertEqual(built, [2, 3]) class TestPipelineHandoff(CustomTestCase): @@ -453,7 +405,7 @@ class TestPipelineHandoff(CustomTestCase): return stage.plan.paths[BatchVariant.ORDINARY] def test_the_sending_rank_returns_to_the_attention_rows(self): - with layer_stack(next_layers=[partial(layer, sparse=True)]): + with layer_stack(next_layers=[partial(layer_stages, sparse=True)]): first_attn, first_ffn, second_attn, second_ffn = [ s for _ in range(2) for s in layer(sparse=True) ] @@ -470,7 +422,7 @@ class TestPipelineHandoff(CustomTestCase): def test_the_handoff_writes_what_the_move_leaves(self): # A move that leaves the residual, as for an FFN on an unpadded batch's # rows: the exit still writes the output into it. - with layer_stack(next_layers=[partial(layer, sparse=True)]): + with layer_stack(next_layers=[partial(layer_stages, sparse=True)]): ffn = [s for s in layer(sparse=True)][-1] steps = msgspec.structs.replace(self.path(ffn), output_move=keep_output) hidden, residual = torch.ones(4, 4), torch.full((4, 4), 2.0) @@ -488,7 +440,7 @@ class TestPipelineHandoff(CustomTestCase): torch.testing.assert_close(written, want) def test_the_receiving_rank_reads_the_attention_rows(self): - with layer_stack(previous_layers=[partial(layer, sparse=True)]): + with layer_stack(previous_layers=[partial(layer_stages, sparse=True)]): attention, _ = layer(sparse=True) self.assertIsNone(self.path(attention).entry.input_move) self.assertIs(attention.declaration.previous.exit_rows, ExitRows.ATTENTION) @@ -496,7 +448,7 @@ class TestPipelineHandoff(CustomTestCase): def test_the_receiving_rank_reads_the_stream_as_written(self): # A residual beside it, as a captured graph's input buffer holds, is # not read. - with layer_stack(previous_layers=[partial(layer, sparse=True)]): + with layer_stack(previous_layers=[partial(layer_stages, sparse=True)]): attention, _ = layer(sparse=True) attention.plan.path_for = lambda _: self.path(attention) hidden, stale = torch.ones(4, 4), torch.full((4, 4), 2.0) @@ -513,7 +465,7 @@ class TestPipelineHandoff(CustomTestCase): def test_a_handoff_from_the_attention_rows_carries_its_residual(self): # A dense FFN over the TP group computes on the attention's rows and # leaves the residual add to the receiver, which requires it. - with layer_stack(previous_layers=[partial(layer, sparse=False)]): + with layer_stack(previous_layers=[partial(layer_stages, sparse=False)]): attention, _ = layer(sparse=False) self.assertFalse(attention.declaration.previous.writes_at_handoff) attention.plan.path_for = lambda _: self.path(attention) @@ -537,7 +489,7 @@ class TestInputScatteredHandoff(CustomTestCase): attn_dp=1, attn_tp=4, enable_attn_tp_input_scattered=True ) with fixture.planning(parallel): - with layer_stack(next_layers=[layer]): + with layer_stack(next_layers=[layer_stages]): stages = [s for _ in range(2) for s in layer(sparse=True)] for stage in stages: for variant, edges in stage.plan.edges.items(): @@ -565,8 +517,7 @@ def bind_cut(kinds, *, before=(), after=()): the pipeline ranks next to it.""" def held_by_another_rank(kinds): - stages = [(CUT_STAGES[kind](), fixture.Norm()) for kind in kinds] - return [lambda: append_stages(*stages)] if kinds else [] + return [lambda: tuple(CUT_STAGES[kind]() for kind in kinds)] if kinds else [] with ( BINDING, diff --git a/test/registered/unit/layer_boundary/test_gated_residual_ops.py b/test/registered/unit/layer_boundary/test_gated_residual_ops.py index 734292317b..47147910fd 100644 --- a/test/registered/unit/layer_boundary/test_gated_residual_ops.py +++ b/test/registered/unit/layer_boundary/test_gated_residual_ops.py @@ -180,9 +180,11 @@ class TestGatedResidualOps(CustomTestCase): fixture.planning(fixture.parallel_of(attn_dp=1, attn_tp=4)), layer_stack(), ): + # A MoE model type, whose layers run sparse FFNs. _, ffn = _build_qwen4_exp_stages( + SimpleNamespace(model_type="qwen4_exp_text"), + 0, state.residual_ops(), - sparse=True, ) with patch( "sglang.srt.layers.layer_boundary.exit.sum_output", diff --git a/test/registered/unit/layer_boundary/test_independent_stage_construction.py b/test/registered/unit/layer_boundary/test_independent_stage_construction.py index c4f8a7b47b..cba81e7dd3 100644 --- a/test/registered/unit/layer_boundary/test_independent_stage_construction.py +++ b/test/registered/unit/layer_boundary/test_independent_stage_construction.py @@ -36,7 +36,7 @@ register_cpu_ci(est_time=12, suite="base-a-test-cpu") def next_layer(): """The next layer's attention, which a stack that does not end the model hands its output to.""" - append_stages((declare_attn(), fixture.Norm())) + return (declare_attn(),) class TestIndependentStageConstruction(CustomTestCase): @@ -174,7 +174,7 @@ class TestIndependentStageConstruction(CustomTestCase): )[1] def previous_layer(): - append_stages((declare_ffn(sparse=True), fixture.Norm())) + return (declare_ffn(sparse=True),) neighbours = dict( previous_layers=[previous_layer], next_layers=[next_layer] @@ -294,7 +294,7 @@ def cut_stages(kinds): def held_by_another_rank(kinds): """The layers another pipeline rank holds, declaring ``kinds``: the neighbours of a stack cut next to them.""" - return [lambda: append_stages(*cut_stages(kinds))] if kinds else [] + return [lambda: tuple(CUT_STAGES[kind]() for kind in kinds)] if kinds else [] def bind_rank(kinds, *, before=(), after=()): @@ -473,11 +473,7 @@ class TestPipelineCuts(CustomTestCase): "a pipeline rank that ends on an attention transforming " "the output whose sum it leaves", ), - layer_stack( - previous_layers=[ - lambda: append_stages((transformed(), fixture.Norm())) - ] - ), + layer_stack(previous_layers=[lambda: (transformed(),)]), ): append_stages(*cut_stages((kind,))) diff --git a/test/registered/unit/layer_boundary/test_kimi_k3_stage_kernels.py b/test/registered/unit/layer_boundary/test_kimi_k3_stage_kernels.py index e939b7dff9..d7990f6c08 100644 --- a/test/registered/unit/layer_boundary/test_kimi_k3_stage_kernels.py +++ b/test/registered/unit/layer_boundary/test_kimi_k3_stage_kernels.py @@ -26,8 +26,13 @@ from sglang.test.test_utils import CustomTestCase register_cpu_ci(est_time=5, suite="base-a-test-cpu") +# A latent MoE on every layer after the first. CONFIG = SimpleNamespace( - is_moe=True, num_experts=8, first_k_dense_replace=1, moe_layer_freq=1 + is_moe=True, + num_experts=8, + first_k_dense_replace=1, + moe_layer_freq=1, + routed_expert_hidden_size=2, ) @@ -37,11 +42,8 @@ def moe_layer(*, bank, sp_moe, all_reduce_fusion): nn.Module.__init__(layer) layer.use_attn_residuals = bank layer.is_block_write_layer = False - layer._is_moe_layer = True layer._sp_moe = sp_moe layer.all_reduce_fusion = all_reduce_fusion - layer._ffn_writes_stream = bank - layer.mlp = SimpleNamespace(use_latent_moe=True, _ep_a2a=sp_moe) layer.input_layernorm = fixture.Norm() layer.post_attention_layernorm = fixture.Norm() for name in ( @@ -60,6 +62,11 @@ def declare(layers, *, sp_moe, carries=False, config=CONFIG): model's final read of the bank, which gathers an SP-MoE layer's rows in K3's tuned all-gather.""" bank = AttnBank() + has_bank = any(layer.use_attn_residuals for layer in layers.values()) + # Every layer of a model with a bank block size carries the bank. + config = SimpleNamespace( + **{**vars(config), "attn_res_block_size": 4 if has_bank else None} + ) final_read = ( AttnBankOutputRead( bank, @@ -69,7 +76,7 @@ def declare(layers, *, sp_moe, carries=False, config=CONFIG): attn_tp_gather=k3_sp_collective.all_gather if sp_moe else None, reads_attn_tp_slices=carries, ) - if any(layer.use_attn_residuals for layer in layers.values()) + if has_bank else None ) with ( @@ -204,7 +211,6 @@ class TestKimiK3BankOnShards(CustomTestCase): moe_layer_freq=moe_layer_freq, ), patch.object(kimi_k3, "_shards_moe_rows", return_value=True), - patch.object(k3_sp_collective, "enabled", return_value=True), envs.SGLANG_K3_SP_ATTN_RES.override(True), patch.object( kimi_k3, "get_parallel", lambda: SimpleNamespace(pp_size=pp_size) @@ -215,7 +221,9 @@ class TestKimiK3BankOnShards(CustomTestCase): lambda: SimpleNamespace(speculative_algorithm=algorithm), ), ): - self.assertEqual(kimi_k3._carries_bank_slices(config), carries) + self.assertEqual( + kimi_k3._carries_bank_slices(config, sp_collective=True), carries + ) if __name__ == "__main__": diff --git a/test/registered/unit/layer_boundary/test_nemotron_h_mtp_reduction.py b/test/registered/unit/layer_boundary/test_nemotron_h_mtp_reduction.py index b48f80ffa7..fdbb24db20 100644 --- a/test/registered/unit/layer_boundary/test_nemotron_h_mtp_reduction.py +++ b/test/registered/unit/layer_boundary/test_nemotron_h_mtp_reduction.py @@ -11,7 +11,7 @@ from sglang.srt.layers.layer_boundary.residual.add_norm import PLAIN_ADD from sglang.srt.layers.layer_boundary.residual.stream import ResidualStream from sglang.srt.model_executor.forward_batch_info import ForwardMode from sglang.srt.models import nemotron_h_mtp -from sglang.srt.models.nemotron_h_utils import make_stage_boundary +from sglang.srt.models.nemotron_h_utils import stage_facts from sglang.srt.runtime_context import get_context, get_flags, get_parallel from sglang.test.ci.ci_register import register_cpu_ci from sglang.test.communicator_patch import patch_communicator @@ -86,13 +86,7 @@ class TestNemotronMTPReduction(CustomTestCase): layer.norm = _Norm() # The MoE layer follows the pattern's attention layer, # built elsewhere: the stack reads it as a neighbour. - with layer_stack( - previous_layers=[ - lambda: make_stage_boundary( - _Norm(), pattern="*E", layer_idx=0 - ) - ] - ): + with layer_stack(previous_layers=[lambda: stage_facts("*E", 0)]): layer._init_stage_boundary( SimpleNamespace(hybrid_override_pattern="*E"), 1 ) diff --git a/test/registered/unit/layer_boundary/test_qwen4_exp_ple_rows.py b/test/registered/unit/layer_boundary/test_qwen4_exp_ple_rows.py index e0555bb020..42bb2274c6 100644 --- a/test/registered/unit/layer_boundary/test_qwen4_exp_ple_rows.py +++ b/test/registered/unit/layer_boundary/test_qwen4_exp_ple_rows.py @@ -39,15 +39,18 @@ class TestQwen4ExpPleRows(CustomTestCase): # An all-to-all MoE with attention TP keeps the residual on this # rank's slice of the rows between layers; the PLE embedding is # computed for every row, so its layer must read the full rows. - config = SimpleNamespace(num_hidden_layers=4, ple_layer_ids=[3]) + config = SimpleNamespace( + model_type="qwen4_exp_text", num_hidden_layers=4, ple_layer_ids=[3] + ) with ( fixture.planning(fixture.parallel_of(attn_dp=1, attn_tp=2), a2a=True), layer_stack(), ): layers = [ _build_qwen4_exp_stages( + config, + layer_id, _residual_ops(attn_reads_every_row=_has_ple(layer_id, config)), - sparse=True, ) for layer_id in range(config.num_hidden_layers) ] diff --git a/test/registered/unit/layers/test_neighbour_layer_build.py b/test/registered/unit/layers/test_neighbour_layer_build.py deleted file mode 100644 index 5605e17fac..0000000000 --- a/test/registered/unit/layers/test_neighbour_layer_build.py +++ /dev/null @@ -1,76 +0,0 @@ -"""A pipeline neighbour layer is built on the meta device and allocates nothing.""" - -import unittest -from types import SimpleNamespace -from unittest.mock import patch - -import torch - -from sglang.srt.layers.hyperconnection import GatedResidual, HyperConnectionConfig -from sglang.srt.layers.moe.fused_moe_triton import layer as fused_moe -from sglang.srt.models.qwen4_exp import _offloads_ple -from sglang.srt.utils.common import ( - building_neighbour_layer, - is_building_neighbour_layer, -) -from sglang.test.ci.ci_register import register_cpu_ci -from sglang.test.test_utils import CustomTestCase - -register_cpu_ci(est_time=5, suite="base-a-test-cpu") - - -class TestNeighbourLayerBuild(CustomTestCase): - def test_the_flag_holds_only_inside_the_build(self): - self.assertFalse(is_building_neighbour_layer()) - with building_neighbour_layer(): - self.assertTrue(is_building_neighbour_layer()) - self.assertEqual(torch.empty(1).device.type, "meta") - self.assertFalse(is_building_neighbour_layer()) - - def test_a_gated_residual_allocates_nothing(self): - config = HyperConnectionConfig(hc_count=4, hidden_size=64, hc_lowrank=16) - with building_neighbour_layer(): - module = GatedResidual(config, use_mix=True, use_combine=True) - tensors = [*module.parameters(), *module.buffers()] - self.assertTrue(tensors) - self.assertEqual({t.device.type for t in tensors}, {"meta"}) - - def test_a_neighbour_keeps_its_ple_table_off_the_host(self): - offloaded = SimpleNamespace(ple_offload_embedding=True) - self.assertTrue(_offloads_ple(offloaded)) - with building_neighbour_layer(): - self.assertFalse(_offloads_ple(offloaded)) - self.assertFalse(_offloads_ple(SimpleNamespace(ple_offload_embedding=False))) - - def test_a_neighbour_moe_sets_up_no_all_to_all(self): - class FlashinferA2A: - def is_flashinfer(self): - return True - - def __getattr__(self, name): - if name.startswith("is_"): - return lambda: False - raise AttributeError(name) - - def unexpected(**kwargs): - raise AssertionError("a neighbour layer built an all-to-all dispatcher") - - config = SimpleNamespace( - top_k=2, num_experts=4, num_local_experts=4, hidden_size=8 - ) - parallel = SimpleNamespace(tp_group=SimpleNamespace(device_group=None)) - standard = object() - with ( - patch.object(fused_moe, "get_moe_a2a_backend", FlashinferA2A), - patch.object(fused_moe, "get_parallel", lambda: parallel), - patch.object(fused_moe, "FlashinferDispatcher", unexpected), - patch.object(fused_moe, "StandardDispatcher", lambda config: standard), - ): - with building_neighbour_layer(): - self.assertIs(fused_moe.create_moe_dispatcher(config, None), standard) - with self.assertRaisesRegex(AssertionError, "all-to-all dispatcher"): - fused_moe.create_moe_dispatcher(config, None) - - -if __name__ == "__main__": - unittest.main() diff --git a/test/registered/unit/layers/test_xielu.py b/test/registered/unit/layers/test_xielu.py index 99b1f0af9c..0c1b44df24 100644 --- a/test/registered/unit/layers/test_xielu.py +++ b/test/registered/unit/layers/test_xielu.py @@ -11,8 +11,7 @@ register_cpu_ci(est_time=2, suite="base-a-test-cpu") class TestXIELU(CustomTestCase): def test_builds_on_the_meta_device_with_the_same_scalars(self): - # A layer built only for the stage boundaries it declares is built on - # the meta device, where nothing can be read back from a buffer. + # Built on the meta device, nothing can be read back from a buffer. for dtype in (torch.bfloat16, torch.float32): with self.subTest(dtype=dtype): on_cpu = XIELU(beta=0.3, eps=-1e-6, dtype=dtype)