[Refactor] Declare every model's stages from a shared function and drop the meta-device neighbour build (#43460)

This commit is contained in:
Cheng Wan
2026-10-10 12:43:09 -07:00
committed by GitHub
parent 3b16eeedce
commit 234ccba7f6
65 changed files with 1407 additions and 1075 deletions
@@ -7,7 +7,7 @@ import os
import sys
from dataclasses import dataclass, fields, is_dataclass, replace
from types import MappingProxyType
from typing import Callable, Mapping, NamedTuple, Optional, Tuple
from typing import Callable, Mapping, NamedTuple, Optional
from sglang.srt.environ import envs
from sglang.srt.layers import layernorm_sp
@@ -302,11 +302,10 @@ class StageDeclaration:
contiguous slice of the rows, it returns them all, in rank order,
or None to leave the gather to the boundary. Called on every batch,
so it must be CUDA-graph safe. Across a pipeline boundary the
producer's rank takes it from the neighbouring stage's declaration
(the model's shared declaration function, or, for a model on the
transitional list, the layer built again on the meta device), so it
may use only the communication state of the rank it runs on, not
the declaring layer's weights or buffers.
producer's rank takes it from the model's shared declaration
function, without the declaring layer, so it may use only the
communication state of the rank it runs on, not that layer's
weights or buffers.
previous: The stage whose output this stage consumes, as the stack
records it: only the facts binding reads of it (see facts_of),
on this rank or another.
@@ -849,12 +848,13 @@ def layer_stack(*, previous_layers=(), next_layers=(), final_read=None):
Args:
previous_layers: For the layers before this stack that another
pipeline rank holds, nearest first, callables that return the
stages the layer declares as DeclaredStages (from the model's
shared declaration function), or else build the layer so that it
appends them. Called only if this stack appended stages, after its
own layers are built, until one of them declares a stage: its last
stage is the producer of this stack's first. Only what binding
reads of that stage is kept (see facts_of).
stages the layer declares, in order, without building it (the
model's shared declaration function at that layer's index; an
empty sequence for a layer that declares none). Called only if
this stack appended stages, after its own layers are built, until
one of them declares a stage: its last stage is the producer of
this stack's first. Only what binding reads of that stage is kept
(see facts_of).
next_layers: Likewise for the layers after this stack, whose first
declared stage is the consumer of this stack's last.
final_read: The model's final read of the stack's output (a
@@ -881,13 +881,6 @@ def layer_stack(*, previous_layers=(), next_layers=(), final_read=None):
_stack = outer
class DeclaredStages(NamedTuple):
"""The stages a neighbouring layer declares, from the model's shared
declaration function, which stand in for building the layer."""
stages: Tuple[StageDeclaration, ...]
def check_declared_stages(stack, appended: int, expected, where: str) -> None:
"""Check that what a layer appended to ``stack`` since it held
``appended`` appends is what its shared declaration function says it
@@ -947,26 +940,9 @@ def _differing_fields(value, given, prefix=""):
def _neighbour_stage(layers, *, last):
"""The stage a neighbouring layer declares next to this stack: the last
one of the nearest layer before it, or the first of the nearest after.
Branches are side paths, so they never stand next to the stack."""
global _stack
one of the nearest layer before it, or the first of the nearest after."""
for layer in layers:
outer = _stack
_stack = _LayerStack()
try:
result = layer()
declared = (
result.stages
if isinstance(result, DeclaredStages)
else [
declaration
for append in _stack.appends
if append.prepared_from is None
for declaration in append.declarations
]
)
finally:
_stack = outer
declared = tuple(layer())
if declared:
return facts_of(declared[-1] if last else declared[0])
return None
@@ -81,6 +81,18 @@ class TransformFacts(msgspec.Struct, frozen=True):
return cls(before_reduce_scatter=transform.before_reduce_scatter)
def residual_facts(ops):
"""A layer's reads and updates (a LayerResidualOps) reduced to the
capabilities they declare, in the same shape: what its stages declare
without the modules and state the reads and updates hold."""
return type(ops)(
attn_readout=ReadFacts.of(ops.attn_readout),
attn_update=UpdateFacts.of(ops.attn_update),
ffn_readout=ReadFacts.of(ops.ffn_readout),
ffn_update=UpdateFacts.of(ops.ffn_update),
)
def facts_of(declaration):
"""``declaration`` with its read, update and output transform reduced to
the capabilities they declare, and without the declarations it was
@@ -21,6 +21,7 @@ import torch
from sglang.srt.layers.attn_residual import AttnResidual
from sglang.srt.layers.layer_boundary import ops
from sglang.srt.layers.layer_boundary.contracts import ReadoutFusion
from sglang.srt.layers.layer_boundary.facts import residual_facts
from sglang.srt.layers.layer_boundary.layout import SumGroup
from sglang.srt.layers.layer_boundary.residual import LayerResidualOps
from sglang.srt.layers.layer_boundary.residual.add_norm import PLAIN_ADD
@@ -187,6 +188,22 @@ class AttnBankState:
ffn_update=PLAIN_ADD,
)
@classmethod
def facts(cls, *, reads_slices: bool = False) -> LayerResidualOps:
"""What residual_ops() declares (see facts_of) for a layer whose bank
stays on each rank's attention-TP slice when ``reads_slices``. It does
not depend on the bank or the layer's modules, so the stages of a
layer that is not built declare it too."""
state = cls(
bank=None,
attn_score_proj=None,
attn_score_norm=None,
ffn_score_proj=None,
ffn_score_norm=None,
reads_slices=reads_slices,
)
return residual_facts(state.residual_ops())
class _BankReadout:
"""A read that mixes the banked snapshots into the residual's norm, so it
@@ -18,6 +18,7 @@ from typing import Callable, Optional
import torch
from sglang.srt.layers.layer_boundary.facts import residual_facts
from sglang.srt.layers.layer_boundary.residual import LayerResidualOps
from sglang.srt.runtime_context import get_parallel
@@ -111,6 +112,22 @@ class GatedResidualState:
ffn_update=_FfnUpdate(self),
)
@classmethod
def facts(cls, *, attn_reads_every_row: bool = False) -> LayerResidualOps:
"""What residual_ops() declares (see facts_of) for a layer whose
attention read needs every row when ``attn_reads_every_row``. It does
not depend on the layer's modules, so the stages of a layer that is
not built declare it too."""
state = cls(
expand=None,
attn_mix=None,
ffn_mix=None,
attn_combine=None,
ffn_combine=None,
attn_reads_every_row=attn_reads_every_row,
)
return residual_facts(state.residual_ops())
class _AttnReadout:
"""The normalized, gated mix of the streams, from streams that already hold
@@ -19,6 +19,7 @@ from typing import Callable, Optional
import torch
from sglang.kernels.ops.layernorm.mhc import hc_contract, hc_expand
from sglang.srt.layers.layer_boundary.facts import ReadFacts, UpdateFacts
from sglang.srt.layers.layer_boundary.residual import LayerResidualOps
from sglang.srt.runtime_context import get_parallel
@@ -119,6 +120,19 @@ class MHCState:
ffn_update=_FfnUpdate(self),
)
@staticmethod
def facts() -> LayerResidualOps:
"""What residual_ops() declares (see facts_of): the class constants
of its reads and updates, the same for every layer whatever
parameters it holds, so the stages of a layer that is not built
declare them too."""
return LayerResidualOps(
attn_readout=ReadFacts.of(_AttnReadout),
attn_update=UpdateFacts.of(_AttnUpdate),
ffn_readout=ReadFacts.of(_FfnReadout),
ffn_update=UpdateFacts.of(_FfnUpdate),
)
class _AttnReadout:
"""hc_pre and the input norm, from streams that already hold the previous
@@ -87,7 +87,6 @@ from sglang.srt.utils import (
is_npu,
round_up,
)
from sglang.srt.utils.common import is_building_neighbour_layer
from sglang.srt.utils.custom_op import register_custom_op
_is_hip = is_hip()
@@ -164,8 +163,6 @@ def create_moe_dispatcher(
moe_runner_config: MoeRunnerConfig,
quant_method: FusedMoEMethodBase,
) -> BaseDispatcher:
if is_building_neighbour_layer():
return StandardDispatcher(moe_runner_config)
a2a_backend = get_moe_a2a_backend()
if a2a_backend.is_none() and is_npu():
return AscendTPDispatcher(moe_runner_config)
@@ -20,7 +20,6 @@ from sglang.srt.runtime_context import (
get_schedule,
)
from sglang.srt.utils import get_compiler_backend
from sglang.srt.utils.common import is_building_neighbour_layer
if TYPE_CHECKING:
from sglang.srt.layers.moe import MoeRunnerConfig
@@ -216,7 +215,7 @@ class KTEPWrapperMethod(FusedMoEMethodBase):
# 2. Initialize KT wrapper for CPU experts
# CPU experts: num_gpu_experts to num_experts-1
if self.tp_rank == 0 and not is_building_neighbour_layer():
if self.tp_rank == 0:
self.wrapper = KTMoEWrapper(
layer_idx=self.kt_config.layer_idx,
num_experts=num_experts,
+10 -5
View File
@@ -266,14 +266,18 @@ class ApertusDecoderLayer(nn.Module):
self.feedforward_layernorm = RMSNorm(
config.hidden_size, eps=config.rms_norm_eps
)
attn, ffn = self.stage_facts(layer_id)
self.attn_boundary, self.ffn_boundary = append_stages(
(declare_attn(), self.attention_layernorm),
(
declare_ffn(sparse=False, next_layer_sparse=False),
self.feedforward_layernorm,
),
(attn, self.attention_layernorm),
(ffn, self.feedforward_layernorm),
)
@staticmethod
def stage_facts(layer_id: int):
"""The stages every Apertus layer declares, an attention and a dense
FFN: the model's shared declaration function (see make_layers)."""
return (declare_attn(), declare_ffn(sparse=False, next_layer_sparse=False))
def forward(
self,
positions: torch.Tensor,
@@ -329,6 +333,7 @@ class ApertusModel(nn.Module):
config=config, quant_config=quant_config, layer_id=idx, prefix=prefix
),
prefix="model.layers",
stage_facts=ApertusDecoderLayer.stage_facts,
)
if self.pp_group.is_last_rank:
+10 -5
View File
@@ -242,14 +242,18 @@ class ArceeDecoderLayer(nn.Module):
self.post_attention_layernorm = RMSNorm(
config.hidden_size, eps=config.rms_norm_eps
)
attn, ffn = self.stage_facts(layer_id)
self.attn_boundary, self.ffn_boundary = append_stages(
(declare_attn(), self.input_layernorm),
(
declare_ffn(sparse=False, next_layer_sparse=False),
self.post_attention_layernorm,
),
(attn, self.input_layernorm),
(ffn, self.post_attention_layernorm),
)
@staticmethod
def stage_facts(layer_id: int):
"""The stages every Arcee layer declares, an attention and a dense FFN:
the model's shared declaration function (see make_layers)."""
return (declare_attn(), declare_ffn(sparse=False, next_layer_sparse=False))
def forward(
self,
positions: torch.Tensor,
@@ -303,6 +307,7 @@ class ArceeModel(nn.Module):
config=config, quant_config=quant_config, layer_id=idx, prefix=prefix
),
prefix="model.layers",
stage_facts=ArceeDecoderLayer.stage_facts,
)
if self.pp_group.is_last_rank:
+23 -13
View File
@@ -655,12 +655,8 @@ class BailingMoEBlock(nn.Module):
)
self.layer_id = layer_id
self.is_layer_sparse = self._is_layer_sparse(
config, layer_id=layer_id, is_nextn=is_nextn
)
is_next_layer_sparse = self._is_layer_sparse(
config, layer_id=layer_id + 1, is_nextn=False
)
attn, ffn = self.stage_facts(config, layer_id, is_nextn=is_nextn)
self.is_layer_sparse = ffn.sparse
if self.is_layer_sparse:
self.mlp = BailingMoESparseMoeBlock(
@@ -684,18 +680,31 @@ class BailingMoEBlock(nn.Module):
self.post_attention_layernorm = RMSNorm(hidden_size, eps=config.rms_norm_eps)
self.attn_boundary, self.ffn_boundary = append_stages(
(declare_attn(), self.input_layernorm),
(
declare_ffn(
sparse=self.is_layer_sparse,
next_layer_sparse=is_next_layer_sparse,
(attn, self.input_layernorm),
(ffn, self.post_attention_layernorm),
)
@classmethod
def stage_facts(
cls, config: PretrainedConfig, layer_id: int, is_nextn: bool = False
):
"""The stages the layer at ``layer_id`` declares, from the config: the
model's shared declaration function (see make_layers)."""
return (
declare_attn(),
declare_ffn(
sparse=cls._is_layer_sparse(
config, layer_id=layer_id, is_nextn=is_nextn
),
next_layer_sparse=cls._is_layer_sparse(
config, layer_id=layer_id + 1, is_nextn=False
),
self.post_attention_layernorm,
),
)
@staticmethod
def _is_layer_sparse(
self, config: PretrainedConfig, layer_id: int, is_nextn: bool
config: PretrainedConfig, layer_id: int, is_nextn: bool
) -> bool:
return is_nextn or (
config.num_experts is not None and layer_id >= config.first_k_dense_replace
@@ -774,6 +783,7 @@ class BailingMoEModel(nn.Module):
alt_stream=alt_stream,
),
prefix=add_prefix("layers", prefix),
stage_facts=lambda idx: BailingMoEBlock.stage_facts(config, idx),
)
if self.pp_group.is_last_rank:
self.norm = RMSNorm(self.embed_dim, eps=config.rms_norm_eps)
+21 -14
View File
@@ -779,8 +779,7 @@ class BailingMoELinearDecoderLayer(nn.Module):
self.expert_num = config.num_experts
self.hidden_size = config.hidden_size
is_moe_layer = self._is_layer_sparse(config, self.layer_id, is_nextn=is_nextn)
is_next_layer_moe_layer = self._is_layer_sparse(config, self.layer_id + 1)
attn, ffn = self.stage_facts(config, self.layer_id, is_nextn=is_nextn)
if self.expert_num == 1:
self.mlp = BailingMLP(
hidden_size=self.hidden_size,
@@ -818,22 +817,27 @@ class BailingMoELinearDecoderLayer(nn.Module):
else None
)
self.attn_boundary, self.ffn_boundary = append_stages(
(
declare_attn(),
self.input_layernorm,
{"qkv_latent_func": qkv_latent_func},
),
(
declare_ffn(
sparse=is_moe_layer,
next_layer_sparse=is_next_layer_moe_layer,
),
self.post_attention_layernorm,
(attn, self.input_layernorm, {"qkv_latent_func": qkv_latent_func}),
(ffn, self.post_attention_layernorm),
)
@classmethod
def stage_facts(
cls, config: PretrainedConfig, layer_id: int, is_nextn: bool = False
):
"""The stages the layer at ``layer_id`` declares, from the config: the
model's shared declaration function (see make_layers)."""
return (
declare_attn(),
declare_ffn(
sparse=cls._is_layer_sparse(config, layer_id, is_nextn=is_nextn),
next_layer_sparse=cls._is_layer_sparse(config, layer_id + 1),
),
)
@staticmethod
def _is_layer_sparse(
self, config: PretrainedConfig, layer_id: int, is_nextn: bool = False
config: PretrainedConfig, layer_id: int, is_nextn: bool = False
) -> bool:
return is_nextn or (
config.num_experts is not None and layer_id >= config.first_k_dense_replace
@@ -945,6 +949,9 @@ class BailingMoELinearModel(nn.Module):
self.num_layers,
layer_fn,
prefix=f"{prefix}.layers",
stage_facts=lambda idx: BailingMoELinearDecoderLayer.stage_facts(
config, idx
),
)
norm_kwargs = {}
+24 -15
View File
@@ -1073,14 +1073,8 @@ class BailingMoELinearDecoderLayer(nn.Module):
self.expert_num = config.num_experts
self.hidden_size = config.hidden_size
is_moe_layer = is_nextn or (
not (self.expert_num == 1)
and (self.layer_id >= config.first_k_dense_replace)
)
self.is_layer_sparse = is_moe_layer
is_next_layer_sparse = not (self.expert_num == 1) and (
self.layer_id + 1 >= config.first_k_dense_replace
)
attn, ffn = self.stage_facts(config, self.layer_id, is_nextn=is_nextn)
self.is_layer_sparse = ffn.sparse
mlp_parallel_group = "replicated" if is_dense_ffn_fully_dp() else "tp"
if self.expert_num == 1:
@@ -1119,7 +1113,7 @@ class BailingMoELinearDecoderLayer(nn.Module):
self.attn_boundary, self.ffn_boundary = append_stages(
(
declare_attn(),
attn,
self.input_layernorm,
{
"qkv_latent_func": self.attention.prepare_qkv_latent
@@ -1127,12 +1121,24 @@ class BailingMoELinearDecoderLayer(nn.Module):
else None
},
),
(
declare_ffn(
sparse=is_moe_layer,
next_layer_sparse=is_next_layer_sparse,
),
self.post_attention_layernorm,
(ffn, self.post_attention_layernorm),
)
@staticmethod
def stage_facts(config: PretrainedConfig, layer_id: int, is_nextn: bool = False):
"""The stages the layer at ``layer_id`` declares, from the config: the
model's shared declaration function (see make_layers)."""
is_moe_layer = is_nextn or (
not (config.num_experts == 1) and (layer_id >= config.first_k_dense_replace)
)
is_next_layer_sparse = not (config.num_experts == 1) and (
layer_id + 1 >= config.first_k_dense_replace
)
return (
declare_attn(),
declare_ffn(
sparse=is_moe_layer,
next_layer_sparse=is_next_layer_sparse,
),
)
@@ -1265,6 +1271,9 @@ class BailingMoELinearModel(nn.Module):
self.num_layers,
layer_fn,
prefix=f"{prefix}.layers",
stage_facts=lambda idx: BailingMoELinearDecoderLayer.stage_facts(
config, idx
),
)
linear_layer_nums = sum(
+44 -22
View File
@@ -2590,10 +2590,9 @@ class DeepseekV2DecoderLayer(nn.Module):
"SGLANG_USE_AG_AFTER_QLORA only supports the model with q_lora_rank"
)
self.is_layer_sparse = self._is_layer_sparse(layer_id, is_nextn=is_nextn)
is_next_layer_sparse = self._is_layer_sparse(layer_id + 1, is_nextn=False)
self._stage_next_sparse = is_next_layer_sparse
self.is_layer_sparse = self._is_layer_sparse(
config, layer_id, is_nextn=is_nextn
)
if self.is_layer_sparse:
self.mlp = DeepseekV2MoE(
@@ -2634,18 +2633,42 @@ class DeepseekV2DecoderLayer(nn.Module):
qkv_latent_func=self.self_attn.prepare_qkv_latent,
)
@classmethod
def stage_facts(
cls,
config: PretrainedConfig,
layer_id: int,
*,
is_nextn: bool = False,
attn_output=None,
output=None,
):
"""The stages a layer of this class declares, from the config alone:
the model's shared declaration function, which the layer declares
with too (see make_layers). ``attn_output`` and ``output``: the
attention's and the FFN's output transforms, if any."""
return (
declare_attn(output_transform=attn_output),
declare_ffn(
sparse=cls._is_layer_sparse(config, layer_id, is_nextn=is_nextn),
next_layer_sparse=cls._is_layer_sparse(
config, layer_id + 1, is_nextn=False
),
output_transform=output,
),
)
def _build_stages(
self,
*,
input_layernorm: nn.Module,
post_attention_layernorm: nn.Module,
qkv_latent_func: Optional[Callable],
output=None,
attn_output=None,
**modules,
):
"""The stage boundaries for this layer's norms; they choose their
steps from them at construction. ``attn_output`` and ``output``: the
attention's and the FFN's output transforms, if any."""
steps from them at construction. ``modules``: what this class's
stage_facts binds the stages' output transforms to, if anything."""
fusions = None
if (
not get_parallel().enable_prefill_cp
@@ -2656,21 +2679,16 @@ class DeepseekV2DecoderLayer(nn.Module):
from sglang.srt.layers.layer_boundary.fusions.cutedsl import CuteDSLFusion
fusions = CuteDSLFusion()
attn, ffn = self.stage_facts(
self.config, self.layer_id, is_nextn=self.is_nextn, **modules
)
attn_boundary, ffn_boundary = append_stages(
(
declare_attn(output_transform=attn_output),
attn,
input_layernorm,
{"qkv_latent_func": qkv_latent_func, "fusions": fusions},
),
(
declare_ffn(
sparse=self.is_layer_sparse,
next_layer_sparse=self._stage_next_sparse,
output_transform=output,
),
post_attention_layernorm,
{"fusions": fusions},
),
(ffn, post_attention_layernorm, {"fusions": fusions}),
)
return attn_boundary, ffn_boundary
@@ -2706,11 +2724,14 @@ class DeepseekV2DecoderLayer(nn.Module):
self._gfx95_quant_format = fmt
return fmt
def _is_layer_sparse(self, layer_id: int, is_nextn: bool) -> bool:
@staticmethod
def _is_layer_sparse(
config: PretrainedConfig, layer_id: int, is_nextn: bool
) -> bool:
return is_nextn or (
self.config.n_routed_experts is not None
and layer_id >= self.config.first_k_dense_replace
and layer_id % self.config.moe_layer_freq == 0
config.n_routed_experts is not None
and layer_id >= config.first_k_dense_replace
and layer_id % config.moe_layer_freq == 0
)
def forward(
@@ -2916,6 +2937,7 @@ class DeepseekV2Model(nn.Module):
else []
),
),
stage_facts=lambda idx: DeepseekV2DecoderLayer.stage_facts(config, idx),
)
local_layer_ids = list(range(self.start_layer, self.end_layer))
@@ -1377,8 +1377,8 @@ class Dots3DecoderLayer(nn.Module):
prefix=add_prefix("self_attn", prefix),
alt_stream=alt_stream,
)
self.is_layer_sparse = self._is_layer_sparse(layer_id, is_nextn=is_nextn)
is_next_layer_sparse = self._is_layer_sparse(layer_id + 1, is_nextn=False)
attn, ffn = self.stage_facts(config, layer_id, is_nextn=is_nextn)
self.is_layer_sparse = ffn.sparse
if self.is_layer_sparse:
self.mlp = Dots3MoE(
@@ -1407,24 +1407,37 @@ class Dots3DecoderLayer(nn.Module):
)
self.attn_boundary, self.ffn_boundary = append_stages(
(declare_attn(), self.input_layernorm),
(
declare_ffn(
sparse=self.is_layer_sparse,
next_layer_sparse=is_next_layer_sparse,
(attn, self.input_layernorm),
(ffn, self.post_attention_layernorm),
)
@classmethod
def stage_facts(
cls, config: PretrainedConfig, layer_id: int, is_nextn: bool = False
):
"""The stages the layer at ``layer_id`` declares, from the config: the
model's shared declaration function (see make_layers)."""
return (
declare_attn(),
declare_ffn(
sparse=cls._is_layer_sparse(config, layer_id, is_nextn=is_nextn),
next_layer_sparse=cls._is_layer_sparse(
config, layer_id + 1, is_nextn=False
),
self.post_attention_layernorm,
),
)
def _is_layer_sparse(self, layer_id: int, is_nextn: bool) -> bool:
@staticmethod
def _is_layer_sparse(
config: PretrainedConfig, layer_id: int, is_nextn: bool
) -> bool:
# The MTP block uses a dense MLP.
if is_nextn:
return False
return (
self.config.n_routed_experts is not None
and layer_id >= self.config.first_k_dense_replace
and layer_id % self.config.moe_layer_freq == 0
config.n_routed_experts is not None
and layer_id >= config.first_k_dense_replace
and layer_id % config.moe_layer_freq == 0
)
def forward(
@@ -1485,6 +1498,7 @@ class Dots3Model(nn.Module):
alt_stream=self.alt_stream,
),
prefix=add_prefix("layers", prefix),
stage_facts=lambda idx: Dots3DecoderLayer.stage_facts(config, idx),
)
if self.pp_group.is_last_rank:
self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
+16 -8
View File
@@ -412,8 +412,8 @@ class Ernie4_5_VLMoeDecoderLayer(nn.Module):
# MoE
# MLP
sparse = _is_moe_layer(config, layer_id)
if sparse:
attn, ffn = self.stage_facts(config, layer_id)
if ffn.sparse:
self.mlp = Ernie4_5_VLMoeMoE(
config=config,
layer_id=layer_id,
@@ -435,15 +435,22 @@ class Ernie4_5_VLMoeDecoderLayer(nn.Module):
self.post_attention_layernorm = RMSNorm(
config.hidden_size, eps=config.rms_norm_eps
)
self.attn_boundary, self.ffn_boundary = append_stages(
(attn, self.input_layernorm),
(ffn, self.post_attention_layernorm),
)
@staticmethod
def stage_facts(config: PretrainedConfig, layer_id: int):
"""The stages the layer at ``layer_id`` declares, from the config: the
model's shared declaration function (see make_layers)."""
sparse = _is_moe_layer(config, layer_id)
next_sparse = layer_id + 1 < config.num_hidden_layers and _is_moe_layer(
config, layer_id + 1
)
self.attn_boundary, self.ffn_boundary = append_stages(
(declare_attn(), self.input_layernorm),
(
declare_ffn(sparse=sparse, next_layer_sparse=next_sparse),
self.post_attention_layernorm,
),
return (
declare_attn(),
declare_ffn(sparse=sparse, next_layer_sparse=next_sparse),
)
def forward(
@@ -502,6 +509,7 @@ class Ernie4_5_VLMoeModel(nn.Module):
prefix=prefix,
),
prefix=add_prefix("layers", prefix),
stage_facts=lambda idx: Ernie4_5_VLMoeDecoderLayer.stage_facts(config, idx),
)
if self.pp_group.is_last_rank:
self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
+30 -16
View File
@@ -267,25 +267,38 @@ class Exaone4DecoderLayer(nn.Module):
self.post_feedforward_layernorm = RMSNorm(
self.hidden_size, eps=config.rms_norm_eps
)
attn, ffn = self.stage_facts(
config,
layer_id,
post_attention_layernorm=self.post_attention_layernorm,
post_feedforward_layernorm=self.post_feedforward_layernorm,
)
self.attn_boundary, self.ffn_boundary = append_stages((attn, None), (ffn, None))
@staticmethod
def stage_facts(
config: Exaone4Config,
layer_id: int,
*,
post_attention_layernorm=None,
post_feedforward_layernorm=None,
):
"""The stages an EXAONE 4.0 layer declares, from the config alone: the
model's shared declaration function, which the layer declares with
too (see make_layers). The output norms are what the updates add."""
# Post-LN: each stage reads the residual as it is, and its output is
# normalized before it is added. The layer writes the FFN's itself.
ffn_update = PostNormAdd(self.post_feedforward_layernorm, applied_at_exit=True)
self.attn_boundary, self.ffn_boundary = append_stages(
(
declare_attn(
read=PLAIN_READOUT,
update=PostNormAdd(self.post_attention_layernorm),
),
None,
ffn_update = PostNormAdd(post_feedforward_layernorm, applied_at_exit=True)
return (
declare_attn(
read=PLAIN_READOUT,
update=PostNormAdd(post_attention_layernorm),
),
(
declare_ffn(
sparse=False,
next_layer_sparse=False,
read=PLAIN_READOUT,
update=ffn_update,
),
None,
declare_ffn(
sparse=False,
next_layer_sparse=False,
read=PLAIN_READOUT,
update=ffn_update,
),
)
@@ -341,6 +354,7 @@ class Exaone4Model(nn.Module):
prefix=prefix,
),
prefix=add_prefix("layers", prefix),
stage_facts=lambda idx: Exaone4DecoderLayer.stage_facts(config, idx),
)
if self.pp_group.is_last_rank:
self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
+18 -10
View File
@@ -464,7 +464,8 @@ class ExaoneMoEDecoderLayer(nn.Module):
prefix=add_prefix("self_attn", prefix),
)
if config.is_moe_layer[layer_id]:
attn, ffn = self.stage_facts(config, layer_id)
if ffn.sparse:
self.mlp = ExaoneMoESparseMoEBlock(
layer_id=layer_id,
config=config,
@@ -485,17 +486,23 @@ class ExaoneMoEDecoderLayer(nn.Module):
self.post_attention_layernorm = RMSNorm(
config.hidden_size, eps=config.rms_norm_eps
)
self.attn_boundary, self.ffn_boundary = append_stages(
(attn, self.input_layernorm),
(ffn, self.post_attention_layernorm),
)
@staticmethod
def stage_facts(config: PretrainedConfig, layer_id: int):
"""The stages the layer at ``layer_id`` declares, from the config: the
model's shared declaration function (see make_layers)."""
is_moe_layer = config.is_moe_layer
num_layers = config.num_hidden_layers
self.attn_boundary, self.ffn_boundary = append_stages(
(declare_attn(), self.input_layernorm),
(
declare_ffn(
sparse=is_moe_layer[layer_id],
next_layer_sparse=layer_id + 1 < num_layers
and is_moe_layer[layer_id + 1],
),
self.post_attention_layernorm,
return (
declare_attn(),
declare_ffn(
sparse=is_moe_layer[layer_id],
next_layer_sparse=layer_id + 1 < num_layers
and is_moe_layer[layer_id + 1],
),
)
@@ -559,6 +566,7 @@ class ExaoneMoEModel(nn.Module):
alt_stream=alt_stream,
),
prefix=add_prefix("layers", prefix),
stage_facts=lambda idx: ExaoneMoEDecoderLayer.stage_facts(config, idx),
)
if self.pp_group.is_last_rank:
+32 -14
View File
@@ -636,7 +636,7 @@ class Gemma4DecoderLayer(nn.Module):
self.post_per_layer_input_norm = None
# Parallel MoE
self.enable_moe_block = getattr(config, "enable_moe_block", False)
self.enable_moe_block = self._enable_moe_block(config)
if self.enable_moe_block:
self.router = Gemma4Router(
config,
@@ -671,20 +671,37 @@ class Gemma4DecoderLayer(nn.Module):
self.register_buffer("layer_scalar", torch.ones(1), persistent=True)
self.has_ple = self.hidden_size_per_layer_input > 0
self.prefix = prefix
attn, ffn = self.stage_facts(
config, layer_id, post_attention_layernorm=self.post_attention_layernorm
)
self.attn_boundary, self.ffn_boundary = append_stages(
(
declare_attn(
output_transform=OutputTransform(self.post_attention_layernorm)
),
self.input_layernorm,
),
(
declare_ffn(
sparse=self.enable_moe_block,
next_layer_sparse=self.enable_moe_block,
update=REPLACE_AT_EXIT,
),
self.pre_feedforward_layernorm,
(attn, self.input_layernorm),
(ffn, self.pre_feedforward_layernorm),
)
@staticmethod
def _enable_moe_block(config: PretrainedConfig) -> bool:
return getattr(config, "enable_moe_block", False)
@classmethod
def stage_facts(
cls,
config: PretrainedConfig,
layer_id: int,
*,
post_attention_layernorm=None,
):
"""The stages a Gemma 4 layer declares, from the config alone: the
model's shared declaration function, which the layer declares with
too (see make_layers). The attention's output norm is its output
transform."""
enable_moe_block = cls._enable_moe_block(config)
return (
declare_attn(output_transform=OutputTransform(post_attention_layernorm)),
declare_ffn(
sparse=enable_moe_block,
next_layer_sparse=enable_moe_block,
update=REPLACE_AT_EXIT,
),
)
@@ -919,6 +936,7 @@ class Gemma4TextModel(PreTrainedModel):
prefix=prefix,
),
prefix=add_prefix("layers", prefix),
stage_facts=lambda idx: Gemma4DecoderLayer.stage_facts(config, idx),
)
if self.pp_group.is_last_rank:
+48 -19
View File
@@ -410,9 +410,7 @@ class GigaChat35DecoderLayer(deepseek_v2.DeepseekV2DecoderLayer):
if swiglu_limit > 0 and runner_config is not None:
runner_config.gemm1_clamp_limit = swiglu_limit
layernorm_type = getattr(config, "layernorm_type", "pre")
self._use_pre = layernorm_type in ("pre", "pre_post")
self._use_post = layernorm_type in ("post", "pre_post")
self._use_pre, self._use_post = self._norm_placement(config)
self.input_layernorm = build_norm(config, config.hidden_size)
self.post_attention_layernorm = build_norm(config, config.hidden_size)
@@ -434,25 +432,55 @@ class GigaChat35DecoderLayer(deepseek_v2.DeepseekV2DecoderLayer):
input_layernorm=attn_prepare_layernorm,
post_attention_layernorm=mlp_prepare_layernorm,
qkv_latent_func=qkv_latent_func,
attn_output=(
OutputTransform(self.post_self_attn_layernorm)
if self.post_self_attn_layernorm is not None
else None
),
output=(
OutputTransform(self.post_feedforward_layernorm)
if self.post_feedforward_layernorm is not None
else None
),
post_self_attn_layernorm=self.post_self_attn_layernorm,
post_feedforward_layernorm=self.post_feedforward_layernorm,
)
def _is_layer_sparse(self, layer_id: int, is_nextn: bool) -> bool:
if is_nextn:
return bool(getattr(self.config, "nextn_is_sparse", False))
@staticmethod
def _norm_placement(config: GigaChat35Config):
"""Whether the layer norms each sublayer's input, and its output."""
layernorm_type = getattr(config, "layernorm_type", "pre")
return (
self.config.n_routed_experts is not None
and layer_id >= self.config.first_k_dense_replace
and layer_id % self.config.moe_layer_freq == 0
layernorm_type in ("pre", "pre_post"),
layernorm_type in ("post", "pre_post"),
)
@classmethod
def stage_facts(
cls,
config: GigaChat35Config,
layer_id: int,
*,
is_nextn: bool = False,
post_self_attn_layernorm=None,
post_feedforward_layernorm=None,
):
"""The stages a GigaChat 3.5 layer declares, from the config alone:
the model's shared declaration function, which the layer declares
with too (see make_layers). The output norms, which the layer has
only when it norms the sublayers' outputs, are the output
transforms."""
_, use_post = cls._norm_placement(config)
return super().stage_facts(
config,
layer_id,
is_nextn=is_nextn,
attn_output=(
OutputTransform(post_self_attn_layernorm) if use_post else None
),
output=OutputTransform(post_feedforward_layernorm) if use_post else None,
)
@staticmethod
def _is_layer_sparse(
config: GigaChat35Config, layer_id: int, is_nextn: bool
) -> bool:
if is_nextn:
return bool(getattr(config, "nextn_is_sparse", False))
return (
config.n_routed_experts is not None
and layer_id >= config.first_k_dense_replace
and layer_id % config.moe_layer_freq == 0
)
def forward(
@@ -527,6 +555,7 @@ class GigaChat35Model(nn.Module):
alt_stream=self.alt_stream,
),
prefix=add_prefix("layers", prefix),
stage_facts=lambda idx: GigaChat35DecoderLayer.stage_facts(config, idx),
)
if self.pp_group.is_last_rank:
+28 -13
View File
@@ -264,20 +264,34 @@ class Glm4DecoderLayer(nn.Module):
config.hidden_size, eps=config.rms_norm_eps
)
self.post_mlp_layernorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
attn, ffn = self.stage_facts(
config,
layer_id,
post_self_attn_layernorm=self.post_self_attn_layernorm,
post_mlp_layernorm=self.post_mlp_layernorm,
)
self.attn_boundary, self.ffn_boundary = append_stages(
(
declare_attn(
output_transform=OutputTransform(self.post_self_attn_layernorm)
),
self.input_layernorm,
),
(
declare_ffn(
sparse=False,
next_layer_sparse=False,
output_transform=OutputTransform(self.post_mlp_layernorm),
),
self.post_attention_layernorm,
(attn, self.input_layernorm),
(ffn, self.post_attention_layernorm),
)
@staticmethod
def stage_facts(
config: Glm4Config,
layer_id: int,
*,
post_self_attn_layernorm=None,
post_mlp_layernorm=None,
):
"""The stages a GLM-4 layer declares, from the config alone: the
model's shared declaration function, which the layer declares with
too (see make_layers). Its output norms are the output transforms."""
return (
declare_attn(output_transform=OutputTransform(post_self_attn_layernorm)),
declare_ffn(
sparse=False,
next_layer_sparse=False,
output_transform=OutputTransform(post_mlp_layernorm),
),
)
@@ -344,6 +358,7 @@ class Glm4Model(nn.Module):
alt_stream=alt_stream,
),
prefix=add_prefix("layers", prefix),
stage_facts=lambda idx: decoder_layer_type.stage_facts(config, idx),
)
if self.pp_group.is_last_rank:
self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
+27 -13
View File
@@ -734,8 +734,8 @@ class Glm4MoeDecoderLayer(nn.Module):
alt_stream=alt_stream,
)
self.is_layer_sparse = self._is_layer_sparse(layer_id, is_nextn=is_nextn)
is_next_layer_sparse = self._is_layer_sparse(layer_id + 1, is_nextn=False)
attn, ffn = self.stage_facts(config, layer_id, is_nextn=is_nextn)
self.is_layer_sparse = ffn.sparse
if self.is_layer_sparse:
self.mlp = Glm4MoeSparseMoeBlock(
@@ -763,14 +763,8 @@ class Glm4MoeDecoderLayer(nn.Module):
)
self.attn_boundary, self.ffn_boundary = append_stages(
(declare_attn(), self.input_layernorm),
(
declare_ffn(
sparse=self.is_layer_sparse,
next_layer_sparse=is_next_layer_sparse,
),
self.post_attention_layernorm,
),
(attn, self.input_layernorm),
(ffn, self.post_attention_layernorm),
)
# Detect if QKV uses aiter FP8 per-token quant so we can fuse
@@ -818,10 +812,29 @@ class Glm4MoeDecoderLayer(nn.Module):
self.self_attn.qkv_proj, "attn"
)
def _is_layer_sparse(self, layer_id: int, is_nextn: bool) -> bool:
@classmethod
def stage_facts(
cls, config: PretrainedConfig, layer_id: int, is_nextn: bool = False
):
"""The stages the layer at ``layer_id`` declares, from the config: the
model's shared declaration function (see make_layers)."""
return (
declare_attn(),
declare_ffn(
sparse=cls._is_layer_sparse(config, layer_id, is_nextn=is_nextn),
next_layer_sparse=cls._is_layer_sparse(
config, layer_id + 1, is_nextn=False
),
),
)
@staticmethod
def _is_layer_sparse(
config: PretrainedConfig, layer_id: int, is_nextn: bool
) -> bool:
return is_nextn or (
self.config.n_routed_experts is not None
and layer_id >= self.config.first_k_dense_replace
config.n_routed_experts is not None
and layer_id >= config.first_k_dense_replace
)
def forward(
@@ -893,6 +906,7 @@ class Glm4MoeModel(nn.Module):
alt_stream=self.alt_stream,
),
prefix=add_prefix("layers", prefix),
stage_facts=lambda idx: Glm4MoeDecoderLayer.stage_facts(config, idx),
)
if self.pp_group.is_last_rank:
self.norm = RMSNorm(self.embed_dim, eps=config.rms_norm_eps)
+28 -14
View File
@@ -447,8 +447,8 @@ class Glm4MoeLiteDecoderLayer(nn.Module):
prefix=add_prefix("self_attn", prefix),
)
self.is_layer_sparse = self._is_layer_sparse(layer_id, is_nextn=is_nextn)
is_next_layer_sparse = self._is_layer_sparse(layer_id + 1, is_nextn=False)
attn, ffn = self.stage_facts(config, layer_id, is_nextn=is_nextn)
self.is_layer_sparse = ffn.sparse
if self.is_layer_sparse:
self.mlp = Glm4MoeLiteSparseMoeBlock(
@@ -480,17 +480,11 @@ class Glm4MoeLiteDecoderLayer(nn.Module):
self.attn_boundary, self.ffn_boundary = append_stages(
(
declare_attn(),
attn,
self.input_layernorm,
{"qkv_latent_func": self.self_attn.prepare_qkv_latent},
),
(
declare_ffn(
sparse=self.is_layer_sparse,
next_layer_sparse=is_next_layer_sparse,
),
self.post_attention_layernorm,
),
(ffn, self.post_attention_layernorm),
)
def _detect_gfx95_quant_format(self) -> str:
@@ -509,11 +503,30 @@ class Glm4MoeLiteDecoderLayer(nn.Module):
return "fp8"
return ""
def _is_layer_sparse(self, layer_id: int, is_nextn: bool) -> bool:
@classmethod
def stage_facts(
cls, config: PretrainedConfig, layer_id: int, is_nextn: bool = False
):
"""The stages the layer at ``layer_id`` declares, from the config: the
model's shared declaration function (see make_layers)."""
return (
declare_attn(),
declare_ffn(
sparse=cls._is_layer_sparse(config, layer_id, is_nextn=is_nextn),
next_layer_sparse=cls._is_layer_sparse(
config, layer_id + 1, is_nextn=False
),
),
)
@staticmethod
def _is_layer_sparse(
config: PretrainedConfig, layer_id: int, is_nextn: bool
) -> bool:
return is_nextn or (
self.config.n_routed_experts is not None
and layer_id >= self.config.first_k_dense_replace
and layer_id % self.config.moe_layer_freq == 0
config.n_routed_experts is not None
and layer_id >= config.first_k_dense_replace
and layer_id % config.moe_layer_freq == 0
)
def forward(
@@ -586,6 +599,7 @@ class Glm4MoeLiteModel(nn.Module):
alt_stream=self.alt_stream,
),
prefix=add_prefix("layers", prefix),
stage_facts=lambda idx: Glm4MoeLiteDecoderLayer.stage_facts(config, idx),
)
if self.pp_group.is_last_rank:
self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
+43 -22
View File
@@ -743,8 +743,9 @@ class Glm5NextDecoderLayer(nn.Module):
"SGLANG_USE_AG_AFTER_QLORA only supports the model with q_lora_rank"
)
self.is_layer_sparse = self._is_layer_sparse(layer_id, is_nextn=is_nextn)
is_next_layer_sparse = self._is_layer_sparse(layer_id + 1, is_nextn=False)
self.is_layer_sparse = self._is_layer_sparse(
config, layer_id, is_nextn=is_nextn
)
if self.is_layer_sparse:
self.mlp = Glm5NextMoE(
@@ -795,9 +796,9 @@ class Glm5NextDecoderLayer(nn.Module):
)
terminal = layer_id == (1 if is_nextn else config.num_hidden_layers) - 1
residual = PLAIN_RESIDUAL_OPS
mhc = None
if self.config.mhc:
residual = MHCState(
mhc = MHCState(
hc_mult=config.hc_mult,
hc_attn_pre=self.hc_attn_pre,
hc_ffn_pre=self.hc_ffn_pre,
@@ -809,15 +810,10 @@ class Glm5NextDecoderLayer(nn.Module):
),
is_last_layer=terminal,
).residual_ops()
attn, ffn = self.stage_facts(config, layer_id, is_nextn=is_nextn, mhc=mhc)
self.attn_boundary, self.ffn_boundary = append_stages(
(
declare_attn(
read=residual.attn_readout,
update=residual.attn_update,
tp_group=SumGroup.TP
if self.is_linear_attn and get_parallel().enable_cp_tp_group_sharing
else SumGroup.ATTN_TP,
),
attn,
self.input_layernorm,
{
"qkv_latent_func": self.self_attn.prepare_qkv_latent
@@ -825,14 +821,37 @@ class Glm5NextDecoderLayer(nn.Module):
else None,
},
),
(
declare_ffn(
sparse=self.is_layer_sparse,
next_layer_sparse=is_next_layer_sparse,
read=residual.ffn_readout,
update=residual.ffn_update,
(ffn, self.post_attention_layernorm),
)
@classmethod
def stage_facts(cls, config, layer_id, *, is_nextn=False, mhc=None):
"""The attention and FFN stages the layer at ``layer_id`` declares:
the model's shared declaration function (see make_layers), which the
layer declares itself with. The config selects the residual: a plain
one, or hyper-connection streams, read and updated by ``mhc``, the
layer's own reads and updates of them; without it, the stages
declare what those do."""
if config.mhc:
residual = MHCState.facts() if mhc is None else mhc
else:
residual = PLAIN_RESIDUAL_OPS
return (
declare_attn(
read=residual.attn_readout,
update=residual.attn_update,
tp_group=SumGroup.TP
if config.is_kda_layer(layer_id)
and get_parallel().enable_cp_tp_group_sharing
else SumGroup.ATTN_TP,
),
declare_ffn(
sparse=cls._is_layer_sparse(config, layer_id, is_nextn=is_nextn),
next_layer_sparse=cls._is_layer_sparse(
config, layer_id + 1, is_nextn=False
),
self.post_attention_layernorm,
read=residual.ffn_readout,
update=residual.ffn_update,
),
)
@@ -924,11 +943,12 @@ class Glm5NextDecoderLayer(nn.Module):
hc_mult=self.config.hc_mult,
)
def _is_layer_sparse(self, layer_id: int, is_nextn: bool) -> bool:
@staticmethod
def _is_layer_sparse(config, layer_id: int, is_nextn: bool) -> bool:
return is_nextn or (
self.config.n_routed_experts is not None
and layer_id >= self.config.first_k_dense_replace
and layer_id % self.config.moe_layer_freq == 0
config.n_routed_experts is not None
and layer_id >= config.first_k_dense_replace
and layer_id % config.moe_layer_freq == 0
)
def forward(
@@ -1035,6 +1055,7 @@ class Glm5NextModel(nn.Module):
alt_stream=self.alt_stream,
),
prefix=add_prefix("layers", prefix),
stage_facts=lambda idx: Glm5NextDecoderLayer.stage_facts(config, idx),
)
if self.pp_group.is_last_rank:
self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
+19 -10
View File
@@ -557,9 +557,8 @@ class GptOssDecoderLayer(nn.Module):
self.layer_id = layer_id
# GptOss all layers are sparse
self.is_layer_sparse = True
is_next_layer_sparse = True
attn, ffn = self.stage_facts(config, layer_id)
self.is_layer_sparse = ffn.sparse
if self.is_layer_sparse:
self.mlp = GptOssSparseMoeBlock(
@@ -590,13 +589,22 @@ class GptOssDecoderLayer(nn.Module):
)
self.attn_boundary, self.ffn_boundary = append_stages(
(declare_attn(), self.input_layernorm),
(
declare_ffn(
sparse=self.is_layer_sparse,
next_layer_sparse=is_next_layer_sparse,
),
self.post_attention_layernorm,
(attn, self.input_layernorm),
(ffn, self.post_attention_layernorm),
)
@staticmethod
def stage_facts(config: GptOssConfig, layer_id: int):
"""The stages the layer at ``layer_id`` declares: the model's shared
declaration function (see make_layers)."""
# GptOss all layers are sparse
is_layer_sparse = True
is_next_layer_sparse = True
return (
declare_attn(),
declare_ffn(
sparse=is_layer_sparse,
next_layer_sparse=is_next_layer_sparse,
),
)
@@ -664,6 +672,7 @@ class GptOssModel(nn.Module):
prefix=prefix,
),
prefix=add_prefix("layers", prefix),
stage_facts=lambda idx: decoder_layer_type.stage_facts(config, idx),
)
if self.pp_group.is_last_rank:
self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
+33 -23
View File
@@ -1,3 +1,4 @@
from functools import partial
from typing import Iterable, Optional
import torch
@@ -40,26 +41,34 @@ from sglang.srt.utils import make_pp_layers
from .granitemoe import GraniteMoeMoE, GraniteMoeSharedMLP
def _make_stages(layer):
"""Both kinds of layer scale their mixer and FFN outputs by
residual_multiplier, add them in the activation dtype and normalize the sum
as a separate step."""
scale = OutputTransform(layer._scale_output)
sparse = layer.block_sparse_moe is not None
def _has_moe(config: GraniteMoeHybridConfig) -> bool:
return getattr(config, "num_local_experts", 0) > 0
def stage_facts(config: GraniteMoeHybridConfig, layer_idx: int, scale=None):
"""The stages a layer of either kind declares, from the config alone: the
model's shared declaration function, which the layers declare with too
(see make_layers). Both kinds of layer scale their mixer and FFN outputs
by residual_multiplier (``scale``), add them in the activation dtype and
normalize the sum as a separate step."""
scale = OutputTransform(scale)
sparse = _has_moe(config)
return (
declare_attn(read=UNFUSED_NORM_READOUT, output_transform=scale),
declare_ffn(
sparse=sparse,
next_layer_sparse=sparse,
read=UNFUSED_NORM_READOUT,
output_transform=scale,
),
)
def _make_stages(layer, config: GraniteMoeHybridConfig, layer_idx: int):
attn, ffn = stage_facts(config, layer_idx, scale=layer._scale_output)
return append_stages(
(
declare_attn(read=UNFUSED_NORM_READOUT, output_transform=scale),
layer.input_layernorm,
),
(
declare_ffn(
sparse=sparse,
next_layer_sparse=sparse,
read=UNFUSED_NORM_READOUT,
output_transform=scale,
),
layer.post_attention_layernorm,
),
(attn, layer.input_layernorm),
(ffn, layer.post_attention_layernorm),
)
@@ -91,7 +100,7 @@ class GraniteMoeHybridMambaDecoderLayer(nn.Module):
)
self.block_sparse_moe = None
if getattr(config, "num_local_experts", 0) > 0:
if _has_moe(config):
self.block_sparse_moe = GraniteMoeMoE(
num_experts=config.num_local_experts,
top_k=config.num_experts_per_tok,
@@ -118,7 +127,7 @@ class GraniteMoeHybridMambaDecoderLayer(nn.Module):
self.post_attention_layernorm = RMSNorm(
config.hidden_size, eps=config.rms_norm_eps
)
self.attn_boundary, self.ffn_boundary = _make_stages(self)
self.attn_boundary, self.ffn_boundary = _make_stages(self, config, layer_idx)
def _scale_output(self, hidden_states: torch.Tensor) -> torch.Tensor:
return hidden_states * self.residual_multiplier
@@ -282,7 +291,7 @@ class GraniteMoeHybridAttentionDecoderLayer(nn.Module):
)
self.block_sparse_moe = None
if getattr(config, "num_local_experts", 0) > 0:
if _has_moe(config):
self.block_sparse_moe = GraniteMoeMoE(
num_experts=config.num_local_experts,
top_k=config.num_experts_per_tok,
@@ -309,7 +318,7 @@ class GraniteMoeHybridAttentionDecoderLayer(nn.Module):
self.post_attention_layernorm = RMSNorm(
config.hidden_size, eps=config.rms_norm_eps
)
self.attn_boundary, self.ffn_boundary = _make_stages(self)
self.attn_boundary, self.ffn_boundary = _make_stages(self, config, layer_idx)
def _scale_output(self, hidden_states: torch.Tensor) -> torch.Tensor:
return hidden_states * self.residual_multiplier
@@ -395,6 +404,7 @@ class GraniteMoeHybridModel(nn.Module):
config.num_hidden_layers,
get_layer,
prefix=f"{prefix}.layers",
stage_facts=partial(stage_facts, config),
)
if self.pp_group.is_last_rank:
+6
View File
@@ -461,6 +461,12 @@ class JetNemotronDecoderLayer(nn.Module):
config.hidden_size, eps=config.rms_norm_eps
)
@staticmethod
def stage_facts(config: JetNemotronConfig, layer_id: int):
"""None: the layer adds its own residual, so it declares no stages.
The model's shared declaration function (see make_layers)."""
return ()
def forward(
self,
positions: torch.Tensor,
+148 -100
View File
@@ -28,7 +28,6 @@ from sglang.srt.distributed import (
from sglang.srt.distributed.device_communicators.pynccl_allocator import (
use_symmetric_memory,
)
from sglang.srt.distributed.utils import get_group_rank_size
from sglang.srt.environ import envs
from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder
from sglang.srt.layers import zero_copy_context
@@ -52,6 +51,7 @@ from sglang.srt.layers.layer_boundary import (
declare_attn,
declare_ffn,
)
from sglang.srt.layers.layer_boundary.residual import LayerResidualOps
from sglang.srt.layers.layer_boundary.residual import batch as residual_batch
from sglang.srt.layers.layer_boundary.residual.add_norm import (
REPLACE_AT_EXIT,
@@ -72,6 +72,7 @@ from sglang.srt.layers.linear import (
QKVParallelLinear,
ReplicatedLinear,
RowParallelLinear,
resolve_linear_parallel_group,
)
from sglang.srt.layers.logits_processor import LogitsProcessor
from sglang.srt.layers.moe import route_quant_handoff
@@ -273,6 +274,20 @@ def _merge_weights_as_views(
return merged, sizes
def _dense_mlp_parallel_group() -> str:
"""The group the dense layers' MLP shards over: a K3 MLP's group when it
is not given one. The Ascend path shards the dense MLP inside each
attention-TP replica. The GPU K3 refactor instead gathers all DP rows
and shards this one dense layer over the full TP group. Keep the GPU
default, but allow the NPU launcher to retain the proven attention-TP
layout without a device-type branch in shared model code."""
return (
"attn_tp"
if get_parallel().enable_dense_mlp_attn_tp and is_dp_attention_enabled()
else "tp"
)
class KimiK3MLP(nn.Module):
"""K3 MLP; SiLU or SiTU activation."""
@@ -288,18 +303,10 @@ class KimiK3MLP(nn.Module):
parallel_group: Optional[LinearParallelGroup] = None,
) -> None:
super().__init__()
# The Ascend path shards the dense MLP inside each attention-TP
# replica. The GPU K3 refactor instead gathers all DP rows and shards
# this one dense layer over the full TP group. Keep the GPU default,
# but allow the NPU launcher to retain the proven attention-TP layout
# without a device-type branch in shared model code.
self._dense_attn_tp = (
get_parallel().enable_dense_mlp_attn_tp
and is_dp_attention_enabled()
and parallel_group is None
)
if parallel_group is None:
parallel_group = "attn_tp" if self._dense_attn_tp else "tp"
default_group = parallel_group is None
if default_group:
parallel_group = _dense_mlp_parallel_group()
self._dense_attn_tp = default_group and parallel_group == "attn_tp"
self.gate_up_proj = MergedColumnParallelLinear(
hidden_size,
[intermediate_size] * 2,
@@ -417,6 +424,27 @@ def _k3_symm_o_proj_out(o_proj: RowParallelLinear, x: torch.Tensor) -> torch.Ten
)
def _uses_latent_moe(config: KimiLinearConfig) -> bool:
"""Whether the MoE layers' routed experts run in a latent space narrower
than the hidden size (Latent MoE)."""
return config.routed_expert_hidden_size is not None
def _uses_ep_a2a() -> bool:
"""Whether the MoE all-to-all backend is an EP a2a one, which moves each
row to its experts directly: the MoE region then consumes whatever rows
this rank holds (SP-MoE shard or DP-local batch), with no DP gather and
no TP reduce anywhere in it."""
_a2a_backend = get_moe_a2a_backend()
return (
_a2a_backend.is_megamoe()
or _a2a_backend.is_deepep()
or _a2a_backend.is_mooncake()
or _a2a_backend.is_ascend_fuseep()
or _a2a_backend.is_mori()
)
class KimiK3MoE(nn.Module):
"""K3 MoE with Latent MoE (experts run in moe_hidden_size space)."""
@@ -438,7 +466,7 @@ class KimiK3MoE(nn.Module):
self.layer_idx = layer_idx
self.alt_stream = alt_stream
self.use_latent_moe = config.routed_expert_hidden_size is not None
self.use_latent_moe = _uses_latent_moe(config)
# Merged front weight ([H, gate_up + E + latent]), built after weight
# loading by _merge_front_weights().
self._front_w: Optional[torch.Tensor] = None
@@ -541,18 +569,7 @@ class KimiK3MoE(nn.Module):
"got a checkpoint with different constants"
)
# EP a2a backends move each row to its experts directly, so the MoE
# region consumes whatever rows this rank holds (SP-MoE shard or
# DP-local batch). No DP gather and no TP reduce anywhere in the
# region.
_a2a_backend = get_moe_a2a_backend()
self._ep_a2a = (
_a2a_backend.is_megamoe()
or _a2a_backend.is_deepep()
or _a2a_backend.is_mooncake()
or _a2a_backend.is_ascend_fuseep()
or _a2a_backend.is_mori()
)
self._ep_a2a = _uses_ep_a2a()
# Defer the trtllm-gen finalize (top-k weighted unpermute) into the
# push all-reduce's staging pass (k3_ar_fusion.finalize_all_reduce_push_norm)
@@ -2358,14 +2375,7 @@ def _is_moe_layer(config: KimiLinearConfig, layer_idx: int) -> bool:
def _shards_moe_rows() -> bool:
"""Whether an MoE layer runs on its attention-TP token shard (SP-MoE): an
EP a2a backend with attention TP."""
backend = get_moe_a2a_backend()
return (
backend.is_megamoe()
or backend.is_deepep()
or backend.is_mooncake()
or backend.is_ascend_fuseep()
or backend.is_mori()
) and get_parallel().attn_tp_group.world_size > 1
return _uses_ep_a2a() and get_parallel().attn_tp_group.world_size > 1
def _fuses_attn_all_reduce(config: KimiLinearConfig) -> bool:
@@ -2380,7 +2390,7 @@ def _fuses_attn_all_reduce(config: KimiLinearConfig) -> bool:
)
def _carries_bank_slices(config: KimiLinearConfig) -> bool:
def _carries_bank_slices(config: KimiLinearConfig, sp_collective: bool) -> bool:
"""Whether consecutive SP-MoE layers keep the stream and the
attention-residual bank on each rank's attention-TP shard of the rows
(SGLANG_K3_SP_ATTN_RES): each layer reads its attention input there, then
@@ -2388,11 +2398,12 @@ def _carries_bank_slices(config: KimiLinearConfig) -> bool:
neither a later pipeline rank nor a draft model's capture of the target's
hidden states can take them; it is decided at construction. Nor can a
dense layer after an SP-MoE one, whose FFN reads every row of the bank:
every layer from the first MoE layer on must be one."""
every layer from the first MoE layer on must be one. ``sp_collective``:
whether K3's SP collective is enabled on this rank."""
if not (
_shards_moe_rows()
and config.attn_res_block_size is not None
and k3_sp_collective.enabled()
and sp_collective
and envs.SGLANG_K3_SP_ATTN_RES.get()
and get_parallel().pp_size == 1
):
@@ -2404,6 +2415,84 @@ def _carries_bank_slices(config: KimiLinearConfig) -> bool:
return not (spec.is_eagle3() or spec.is_dflash_family())
def _latent_moe_writes_stream(config: KimiLinearConfig, layer_idx: int) -> bool:
"""Whether a layer's FFN writes the next stream itself: a latent MoE on
this rank's rows adds the residual in its tail add and writes the next
stream itself; one gathered over attention DP returns its output to this
rank's tokens first."""
return (
config.attn_res_block_size is not None
and _is_moe_layer(config, layer_idx)
and _uses_latent_moe(config)
and not (is_dp_attention_enabled() and not _uses_ep_a2a())
)
def _stage_facts(
config: KimiLinearConfig,
layer_idx: int,
*,
sp_collective: bool,
bank: Optional[LayerResidualOps] = None,
):
"""The attention and FFN stages the K3 layer at ``layer_idx`` declares:
the model's shared declaration function (see make_layers), which the
layer declares itself with. ``sp_collective`` is whether K3's SP
collective is enabled on this rank, which the model settles before it
builds a layer. ``bank`` is the layer's own reads and updates of the
attention-residual bank; without it, the stages declare what those do."""
is_moe = _is_moe_layer(config, layer_idx)
# Under attention DP the FFN input is read on this rank's rows
# once the attention's sum is complete, then gathered.
attn_ops = {}
ffn_ops = dict(read=NormReadout(reads_before_dp_gather=True))
if config.attn_res_block_size is not None:
if bank is None:
bank = AttnBankState.facts(
reads_slices=_carries_bank_slices(config, sp_collective)
)
attn_ops = dict(
read=bank.attn_readout,
update=bank.attn_update,
# The rows an SP-MoE layer left on each rank's shard are
# gathered in K3's tuned all-gather when it takes the batch:
# before this read, or after it while the bank stays on the
# shard.
attn_tp_gather=(
k3_sp_collective.all_gather
if sp_collective
and _shards_moe_rows()
and _is_moe_layer(config, layer_idx - 1)
else None
),
)
ffn_ops = dict(
read=bank.ffn_readout,
update=(
REPLACE_AT_EXIT
if _latent_moe_writes_stream(config, layer_idx)
else bank.ffn_update
),
)
return (
declare_attn(**attn_ops),
declare_ffn(
**ffn_ops,
sparse=is_moe,
next_layer_sparse=_is_moe_layer(config, layer_idx + 1),
# The TP width the dense MLP is built with.
dense_tp_size=(
None
if is_moe
else resolve_linear_parallel_group(_dense_mlp_parallel_group())[1]
),
# A latent MoE completes its output sum together with
# the latent reduction its norm needs.
output_complete=is_moe and _uses_latent_moe(config),
),
)
class KimiK3DecoderLayer(nn.Module):
"""Decoder layer carrying the K3 attention-residual stream."""
@@ -2419,7 +2508,6 @@ class KimiK3DecoderLayer(nn.Module):
super().__init__()
self.hidden_size = config.hidden_size
self.layer_idx = layer_idx
self._dp_attention = is_dp_attention_enabled()
self._is_moe_layer = _is_moe_layer(config, layer_idx)
# SP-MoE: with an EP a2a backend and attention TP, a MoE layer runs on
# this rank's attention-TP shard of the rows.
@@ -2506,31 +2594,17 @@ class KimiK3DecoderLayer(nn.Module):
prefix=f"{prefix}.mlp_res_proj",
)
# A latent MoE on this rank's rows adds the residual in its tail add and
# writes the next stream itself; one gathered over attention DP returns
# its output to this rank's tokens first.
self._ffn_writes_stream = (
self.use_attn_residuals
and self._is_moe_layer
and self.mlp.use_latent_moe
and not (self._dp_attention and not self.mlp._ep_a2a)
)
self._ffn_writes_stream = _latent_moe_writes_stream(config, layer_idx)
self._declare_stages(config, layer_idx, attn_bank)
def _declare_stages(self, config, layer_idx, attn_bank):
"""Declare this layer's attention and FFN stages: the reads and updates
around them, the rows the FFN takes, and the kernels K3 supplies for
the bank path."""
# Under attention DP the FFN input is read on this rank's rows
# once the attention's sum is complete, then gathered.
attn_ops = {}
ffn_ops = dict(read=NormReadout(reads_before_dp_gather=True))
carries = _carries_bank_slices(config)
tuned_gather = (
k3_sp_collective.all_gather if k3_sp_collective.enabled() else None
)
"""Declare this layer's attention and FFN stages as _stage_facts gives
them, with this layer's reads and updates of the bank and the kernels
K3 supplies for the bank path."""
sp_collective = k3_sp_collective.enabled()
bank = None
if self.use_attn_residuals:
bank_ops = AttnBankState(
bank = AttnBankState(
attn_bank,
self.self_attention_res_proj,
self.self_attention_res_norm,
@@ -2539,48 +2613,16 @@ class KimiK3DecoderLayer(nn.Module):
writes_block=self.is_block_write_layer,
ffn_input_fusions=self._ffn_input_fusions(),
fuses_slice_collectives=self._sp_moe
and k3_sp_collective.enabled()
and sp_collective
and envs.SGLANG_K3_SP_ATTN_RES.get(),
reads_slices=carries,
reads_slices=_carries_bank_slices(config, sp_collective),
).residual_ops()
attn_ops = dict(
read=bank_ops.attn_readout,
update=bank_ops.attn_update,
# The rows an SP-MoE layer left on each rank's shard are
# gathered in K3's tuned all-gather when it takes the batch:
# before this read, or after it while the bank stays on the
# shard.
attn_tp_gather=(
tuned_gather
if _shards_moe_rows() and _is_moe_layer(config, layer_idx - 1)
else None
),
)
ffn_ops = dict(
read=bank_ops.ffn_readout,
update=(
REPLACE_AT_EXIT if self._ffn_writes_stream else bank_ops.ffn_update
),
)
attn, ffn = _stage_facts(
config, layer_idx, sp_collective=sp_collective, bank=bank
)
self.attn_boundary, self.ffn_boundary = append_stages(
(declare_attn(**attn_ops), self.input_layernorm),
(
declare_ffn(
**ffn_ops,
sparse=self._is_moe_layer,
next_layer_sparse=_is_moe_layer(config, layer_idx + 1),
# The TP width the dense MLP is built with.
dense_tp_size=(
None
if self._is_moe_layer
else get_group_rank_size(self.mlp.down_proj.tp_group)[1]
),
# A latent MoE completes its output sum together with
# the latent reduction its norm needs.
output_complete=self._is_moe_layer and self.mlp.use_latent_moe,
),
self.post_attention_layernorm,
),
(attn, self.input_layernorm),
(ffn, self.post_attention_layernorm),
)
def _ffn_input_fusions(self):
@@ -2673,7 +2715,10 @@ class KimiK3LinearModel(nn.Module):
self.pp_group = get_parallel().pp_group
self.dspark_layers_to_capture: Optional[list[int]] = None
self._dp_attention = is_dp_attention_enabled()
self.carries_bank_slices = _carries_bank_slices(config)
# Whether K3's SP collective is enabled on this rank, which the layers'
# stage declarations depend on: settled once, before any is built.
sp_collective = k3_sp_collective.enabled()
self.carries_bank_slices = _carries_bank_slices(config, sp_collective)
if self.pp_group.is_first_rank:
embedding_quant_config = (
@@ -2731,7 +2776,7 @@ class KimiK3LinearModel(nn.Module):
# takes the batch.
attn_tp_gather=(
k3_sp_collective.all_gather
if k3_sp_collective.enabled()
if sp_collective
and _shards_moe_rows()
and _is_moe_layer(config, config.num_hidden_layers - 1)
else None
@@ -2753,6 +2798,9 @@ class KimiK3LinearModel(nn.Module):
),
prefix=f"{prefix}.layers",
final_read=self._final_read if self.pp_group.is_last_rank else None,
stage_facts=lambda idx: _stage_facts(
config, idx, sp_collective=sp_collective
),
)
def forward(
+16 -8
View File
@@ -580,8 +580,8 @@ class KimiDecoderLayer(nn.Module):
reduce_results=False,
)
sparse = _is_sparse_layer(config, layer_idx)
if sparse:
attn, ffn = self.stage_facts(config, layer_idx)
if ffn.sparse:
self.block_sparse_moe = KimiMoE(
config=config,
quant_config=quant_config,
@@ -603,15 +603,22 @@ class KimiDecoderLayer(nn.Module):
self.post_attention_layernorm = RMSNorm(
config.hidden_size, eps=config.rms_norm_eps
)
self.attn_boundary, self.ffn_boundary = append_stages(
(attn, self.input_layernorm),
(ffn, self.post_attention_layernorm),
)
@staticmethod
def stage_facts(config: KimiLinearConfig, layer_idx: int):
"""The stages the layer at ``layer_idx`` declares, from the config: the
model's shared declaration function (see make_layers)."""
sparse = _is_sparse_layer(config, layer_idx)
next_sparse = layer_idx + 1 < config.num_hidden_layers and _is_sparse_layer(
config, layer_idx + 1
)
self.attn_boundary, self.ffn_boundary = append_stages(
(declare_attn(), self.input_layernorm),
(
declare_ffn(sparse=sparse, next_layer_sparse=next_sparse),
self.post_attention_layernorm,
),
return (
declare_attn(),
declare_ffn(sparse=sparse, next_layer_sparse=next_sparse),
)
def forward(
@@ -674,6 +681,7 @@ class KimiLinearModel(nn.Module):
alt_stream=self.alt_stream,
),
prefix=f"{prefix}.layers",
stage_facts=lambda idx: KimiDecoderLayer.stage_facts(config, idx),
)
if self.pp_group.is_last_rank:
+22 -13
View File
@@ -418,12 +418,8 @@ class LagunaDecoderLayer(nn.Module):
prefix=add_prefix("self_attn", prefix),
)
mlp_types = config.mlp_layer_types
self.is_layer_sparse = mlp_types[layer_id] == "sparse"
is_next_layer_sparse = (
layer_id + 1 < config.num_hidden_layers
and mlp_types[layer_id + 1] == "sparse"
)
attn, ffn = self.stage_facts(config, layer_id)
self.is_layer_sparse = ffn.sparse
if self.is_layer_sparse:
self.mlp = LagunaMoE(
@@ -448,13 +444,25 @@ class LagunaDecoderLayer(nn.Module):
)
self.attn_boundary, self.ffn_boundary = append_stages(
(declare_attn(), self.input_layernorm),
(
declare_ffn(
sparse=self.is_layer_sparse,
next_layer_sparse=is_next_layer_sparse,
),
self.post_attention_layernorm,
(attn, self.input_layernorm),
(ffn, self.post_attention_layernorm),
)
@staticmethod
def stage_facts(config: LagunaConfig, layer_id: int):
"""The stages the layer at ``layer_id`` declares, from the config: the
model's shared declaration function (see make_layers)."""
mlp_types = config.mlp_layer_types
is_layer_sparse = mlp_types[layer_id] == "sparse"
is_next_layer_sparse = (
layer_id + 1 < config.num_hidden_layers
and mlp_types[layer_id + 1] == "sparse"
)
return (
declare_attn(),
declare_ffn(
sparse=is_layer_sparse,
next_layer_sparse=is_next_layer_sparse,
),
)
@@ -519,6 +527,7 @@ class LagunaModel(nn.Module):
prefix=prefix,
),
prefix=add_prefix("layers", prefix),
stage_facts=lambda idx: decoder_layer_type.stage_facts(config, idx),
)
if self.pp_group.is_last_rank:
self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
+19 -9
View File
@@ -20,6 +20,7 @@
"""SGLang LLaDA2MoeModelLM model."""
import logging
from functools import partial
from typing import Iterable, Optional, Tuple, Union
import torch
@@ -1150,7 +1151,6 @@ class LLaDA2MoeBlock(nn.Module):
self.layer_id = layer_id
self.is_layer_sparse = self._is_layer_sparse(config, layer_id=layer_id)
is_next_layer_sparse = self._is_layer_sparse(config, layer_id=layer_id + 1)
if self.is_layer_sparse:
self.mlp = LLaDA2MoeSparseMoeBlock(
@@ -1173,18 +1173,27 @@ class LLaDA2MoeBlock(nn.Module):
self.post_attention_layernorm = RMSNorm(hidden_size, eps=config.rms_norm_eps)
attn, ffn = self.stage_facts(config, layer_id)
self.attn_boundary, self.ffn_boundary = append_stages(
(declare_attn(), self.input_layernorm),
(
declare_ffn(
sparse=self.is_layer_sparse,
next_layer_sparse=is_next_layer_sparse,
),
self.post_attention_layernorm,
(attn, self.input_layernorm),
(ffn, self.post_attention_layernorm),
)
@classmethod
def stage_facts(cls, config: PretrainedConfig, layer_id: int):
"""The stages the layer at ``layer_id`` declares, in order: the model's
shared declaration function, which the layer declares with too (see
make_layers)."""
return (
declare_attn(),
declare_ffn(
sparse=cls._is_layer_sparse(config, layer_id=layer_id),
next_layer_sparse=cls._is_layer_sparse(config, layer_id=layer_id + 1),
),
)
def _is_layer_sparse(self, config: PretrainedConfig, layer_id: int) -> bool:
@staticmethod
def _is_layer_sparse(config: PretrainedConfig, layer_id: int) -> bool:
return (
config.num_experts is not None and layer_id >= config.first_k_dense_replace
)
@@ -1253,6 +1262,7 @@ class LLaDA2MoeModel(nn.Module):
alt_stream=alt_stream,
),
prefix=add_prefix("layers", prefix),
stage_facts=partial(LLaDA2MoeBlock.stage_facts, config),
)
if self.pp_group.is_last_rank:
self.norm = RMSNorm(self.embed_dim, eps=config.rms_norm_eps)
+16 -6
View File
@@ -364,15 +364,24 @@ class LlamaDecoderLayer(nn.Module):
return
self.input_layernorm.fuse_input_quant(self.self_attn.qkv_proj)
self.post_attention_layernorm.fuse_input_quant(self.mlp.gate_up_proj)
attn, ffn = self.stage_facts(config, layer_id)
self.attn_boundary, self.ffn_boundary = append_stages(
(declare_attn(read=self._attn_readout(layer_id)), self.input_layernorm),
(
declare_ffn(sparse=False, next_layer_sparse=False),
self.post_attention_layernorm,
),
(attn, self.input_layernorm),
(ffn, self.post_attention_layernorm),
)
def _attn_readout(self, layer_id: int):
@classmethod
def stage_facts(cls, config: LlamaConfig, layer_id: int):
"""The stages a layer of this class declares, from the config alone:
the model's shared declaration function, which the layer declares
with too (see make_layers). A subclass's attention read takes part."""
return (
declare_attn(read=cls._attn_readout(layer_id)),
declare_ffn(sparse=False, next_layer_sparse=False),
)
@classmethod
def _attn_readout(cls, layer_id: int):
return NORM_QUANT_READOUT
def forward(
@@ -437,6 +446,7 @@ class LlamaModel(nn.Module):
prefix=prefix,
),
prefix="model.layers",
stage_facts=lambda idx: LlamaDecoderLayer.stage_facts(config, idx),
)
if self.pp_group.is_last_rank:
+2 -1
View File
@@ -54,7 +54,8 @@ class LlamaDecoderLayer(LlamaDecoderLayer):
if layer_id == 0:
del self.input_layernorm
def _attn_readout(self, layer_id: int):
@classmethod
def _attn_readout(cls, layer_id: int):
if layer_id == 0:
return PLAIN_READOUT
return super()._attn_readout(layer_id)
+29 -21
View File
@@ -402,18 +402,8 @@ class MellumDecoderLayer(Qwen3MoeDecoderLayer):
f"{len(mlp_layer_types)} and {cfg.num_hidden_layers}"
)
def _is_sparse(lid: int) -> bool:
if lid < 0 or lid >= cfg.num_hidden_layers:
return False
mlp_type = mlp_layer_types[lid]
if mlp_type not in ("sparse", "dense"):
raise ValueError(
f"Unsupported mlp_layer_types[{lid}]={mlp_type}; "
"expected 'sparse' or 'dense'"
)
return mlp_type == "sparse"
self.is_layer_sparse = _is_sparse(layer_id)
attn, ffn = self.stage_facts(config, layer_id)
self.is_layer_sparse = ffn.sparse
if self.is_layer_sparse:
if num_experts <= 0:
@@ -436,19 +426,37 @@ class MellumDecoderLayer(Qwen3MoeDecoderLayer):
reduce_results=False,
)
is_next_layer_sparse = _is_sparse(layer_id + 1)
self.input_layernorm = RMSNorm(cfg.hidden_size, eps=rms_norm_eps)
self.post_attention_layernorm = RMSNorm(cfg.hidden_size, eps=rms_norm_eps)
self.attn_boundary, self.ffn_boundary = append_stages(
(declare_attn(), self.input_layernorm),
(
declare_ffn(
sparse=self.is_layer_sparse,
next_layer_sparse=is_next_layer_sparse,
),
self.post_attention_layernorm,
(attn, self.input_layernorm),
(ffn, self.post_attention_layernorm),
)
@staticmethod
def _is_sparse(config: PretrainedConfig, lid: int) -> bool:
cfg = cast(Any, config)
if lid < 0 or lid >= cfg.num_hidden_layers:
return False
mlp_type = cfg.mlp_layer_types[lid]
if mlp_type not in ("sparse", "dense"):
raise ValueError(
f"Unsupported mlp_layer_types[{lid}]={mlp_type}; "
"expected 'sparse' or 'dense'"
)
return mlp_type == "sparse"
@classmethod
def stage_facts(cls, config: PretrainedConfig, layer_id: int):
"""The stages a Mellum layer declares, from the config alone: the
model's shared declaration function, which the layer declares with
too (see make_layers)."""
return (
declare_attn(),
declare_ffn(
sparse=cls._is_sparse(config, layer_id),
next_layer_sparse=cls._is_sparse(config, layer_id + 1),
),
)
+24 -14
View File
@@ -16,6 +16,7 @@ import logging
import math
import re
from array import array
from functools import partial
from typing import Any, Dict, Iterable, List, Optional, Tuple, Union
import torch
@@ -822,8 +823,7 @@ class MiMoV2DecoderLayer(nn.Module):
prefix=add_prefix("self_attn", prefix),
)
self.is_layer_sparse = self.is_moe_layer(layer_id)
is_next_layer_sparse = self.is_moe_layer(layer_id + 1)
self.is_layer_sparse = self.is_moe_layer(config, layer_id)
if self.is_layer_sparse:
self.mlp = MiMoV2MoE(
@@ -849,14 +849,22 @@ class MiMoV2DecoderLayer(nn.Module):
config.hidden_size, eps=config.layernorm_epsilon
)
attn, ffn = self.stage_facts(config, layer_id)
self.attn_boundary, self.ffn_boundary = append_stages(
(declare_attn(), self.input_layernorm),
(
declare_ffn(
sparse=self.is_layer_sparse,
next_layer_sparse=is_next_layer_sparse,
),
self.post_attention_layernorm,
(attn, self.input_layernorm),
(ffn, self.post_attention_layernorm),
)
@classmethod
def stage_facts(cls, config, layer_id: int):
"""The stages the layer at ``layer_id`` declares, in order: the model's
shared declaration function, which the layer declares with too (see
make_layers)."""
return (
declare_attn(),
declare_ffn(
sparse=cls.is_moe_layer(config, layer_id),
next_layer_sparse=cls.is_moe_layer(config, layer_id + 1),
),
)
@@ -889,12 +897,13 @@ class MiMoV2DecoderLayer(nn.Module):
return hidden_states
def is_moe_layer(self, layer_idx: int) -> bool:
@staticmethod
def is_moe_layer(config, layer_idx: int) -> bool:
return (
hasattr(self.config, "moe_layer_freq")
and 0 <= layer_idx < len(self.config.moe_layer_freq)
and not isinstance(self.config.moe_layer_freq, int)
and self.config.moe_layer_freq[layer_idx]
hasattr(config, "moe_layer_freq")
and 0 <= layer_idx < len(config.moe_layer_freq)
and not isinstance(config.moe_layer_freq, int)
and config.moe_layer_freq[layer_idx]
)
def is_swa_layer(self) -> bool:
@@ -987,6 +996,7 @@ class MiMoV2Model(nn.Module):
prefix=prefix,
),
prefix=add_prefix("layers", prefix),
stage_facts=partial(decoder_layer_type.stage_facts, config),
)
if self.pp_group.is_last_rank:
self.norm = RMSNorm(config.hidden_size, eps=config.layernorm_epsilon)
+18 -20
View File
@@ -17,7 +17,7 @@
import logging
from contextlib import nullcontext
from functools import lru_cache
from functools import lru_cache, partial
from typing import Any, Dict, Iterable, Optional, Set, Tuple, Union
import torch
@@ -99,7 +99,6 @@ from sglang.srt.utils import (
is_xpu,
make_pp_layers,
)
from sglang.srt.utils.common import is_building_neighbour_layer
from sglang.srt.utils.custom_op import register_custom_op
from sglang.srt.utils.hf_transformers_utils import get_rope_config
@@ -409,12 +408,7 @@ class MiniMaxM2QKRMSNorm:
use_fused_norm = get_bool_env_var("SGLANG_USE_FUSED_PARALLEL_QKNORM")
self._forward_impl = self._forward_naive
if (
self._world_size > 1
and _is_cuda
and use_fused_norm
and not is_building_neighbour_layer()
):
if self._world_size > 1 and _is_cuda and use_fused_norm:
occupancy = get_fused_parallel_qknorm_max_occupancy(
q_norm.weight.dtype,
self._world_size,
@@ -846,8 +840,8 @@ class MiniMaxM2DecoderLayer(nn.Module):
self.hidden_size = config.hidden_size
self.layer_id = layer_id
# TBO support: All MiniMax layers are sparse (MoE)
self.is_layer_sparse = True
attn, ffn = self.stage_facts(config, layer_id)
self.is_layer_sparse = ffn.sparse
self.self_attn = MiniMaxM2Attention(
config=config,
@@ -870,17 +864,20 @@ class MiniMaxM2DecoderLayer(nn.Module):
config.hidden_size, eps=getattr(config, "rms_norm_eps", 1e-6)
)
is_next_layer_sparse = True
self.attn_boundary, self.ffn_boundary = append_stages(
(declare_attn(), self.input_layernorm),
(
declare_ffn(
sparse=self.is_layer_sparse,
next_layer_sparse=is_next_layer_sparse,
),
self.post_attention_layernorm,
),
(attn, self.input_layernorm),
(ffn, self.post_attention_layernorm),
)
@staticmethod
def stage_facts(config: PretrainedConfig, layer_id: int):
"""The stages the layer at ``layer_id`` declares, in order: the model's
shared declaration function, which the layer declares with too (see
make_layers)."""
# TBO support: All MiniMax layers are sparse (MoE)
return (
declare_attn(),
declare_ffn(sparse=True, next_layer_sparse=True),
)
def forward(
@@ -951,6 +948,7 @@ class MiniMaxM2Model(nn.Module):
config.num_hidden_layers,
layer_fn,
prefix=add_prefix("layers", prefix),
stage_facts=partial(MiniMaxM2DecoderLayer.stage_facts, config),
)
if self.pp_group.is_last_rank:
self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
+33 -20
View File
@@ -17,6 +17,7 @@
import logging
from contextlib import nullcontext
from functools import partial
from typing import Iterable, List, Optional, Set, Tuple, Union
import torch
@@ -1520,12 +1521,9 @@ class MiniMaxM3DecoderLayer(nn.Module):
disable_index_value=disable_index_value,
)
moe_layer_freq = getattr(config, "moe_layer_freq", None)
# Means "MLP is a sparse MoE", not attention sparsity. Kept as ``is_layer_sparse``
# because model construction and downstream integrations read this attr.
self.is_layer_sparse = (
moe_layer_freq[layer_id] != 0 if moe_layer_freq is not None else True
)
self.is_layer_sparse = self._is_layer_sparse(config, layer_id)
if self.is_layer_sparse:
self.mlp = MiniMaxM3MoE(
@@ -1560,26 +1558,40 @@ class MiniMaxM3DecoderLayer(nn.Module):
config.hidden_size, eps=config.rms_norm_eps
)
def _is_layer_sparse(lid):
if moe_layer_freq is None:
return True
if lid < 0 or lid >= config.num_hidden_layers:
return True
return moe_layer_freq[lid] != 0
is_next_layer_sparse = _is_layer_sparse(layer_id + 1)
attn, ffn = self.stage_facts(config, layer_id)
self.attn_boundary, self.ffn_boundary = append_stages(
(declare_attn(), self.input_layernorm),
(
declare_ffn(
sparse=self.is_layer_sparse,
next_layer_sparse=is_next_layer_sparse,
),
self.post_attention_layernorm,
(attn, self.input_layernorm),
(ffn, self.post_attention_layernorm),
)
@classmethod
def stage_facts(cls, config: PretrainedConfig, layer_id: int):
"""The stages the layer at ``layer_id`` declares, in order: the model's
shared declaration function, which the layer declares with too (see
make_layers)."""
return (
declare_attn(),
declare_ffn(
sparse=cls._is_layer_sparse(config, layer_id),
next_layer_sparse=cls._is_next_layer_sparse(config, layer_id),
),
)
@staticmethod
def _is_layer_sparse(config: PretrainedConfig, layer_id: int) -> bool:
moe_layer_freq = getattr(config, "moe_layer_freq", None)
return moe_layer_freq[layer_id] != 0 if moe_layer_freq is not None else True
@staticmethod
def _is_next_layer_sparse(config: PretrainedConfig, layer_id: int) -> bool:
moe_layer_freq = getattr(config, "moe_layer_freq", None)
lid = layer_id + 1
if moe_layer_freq is None:
return True
if lid < 0 or lid >= config.num_hidden_layers:
return True
return moe_layer_freq[lid] != 0
def forward(
self,
positions: torch.Tensor,
@@ -1653,6 +1665,7 @@ class MiniMaxM3Model(nn.Module):
config.num_hidden_layers,
layer_fn,
prefix=add_prefix("layers", prefix),
stage_facts=partial(MiniMaxM3DecoderLayer.stage_facts, config),
)
if self.pp_group.is_last_rank:
if self.use_gemma_norm:
+1
View File
@@ -155,6 +155,7 @@ class Ministral3Model(LlamaModel):
prefix=prefix,
),
prefix="model.layers",
stage_facts=lambda idx: Ministral3DecoderLayer.stage_facts(config, idx),
)
+15 -5
View File
@@ -19,6 +19,7 @@
"""Inference-only Mixtral model."""
import logging
from functools import partial
from typing import Iterable, Optional, Tuple, Union
import torch
@@ -235,12 +236,20 @@ class MixtralDecoderLayer(nn.Module):
self.post_attention_layernorm = RMSNorm(
config.hidden_size, eps=config.rms_norm_eps
)
attn, ffn = self.stage_facts(config, layer_id)
self.attn_boundary, self.ffn_boundary = append_stages(
(declare_attn(), self.input_layernorm),
(
declare_ffn(sparse=True, next_layer_sparse=True),
self.post_attention_layernorm,
),
(attn, self.input_layernorm),
(ffn, self.post_attention_layernorm),
)
@staticmethod
def stage_facts(config: MixtralConfig, layer_id: int):
"""The stages the layer at ``layer_id`` declares, in order: the model's
shared declaration function, which the layer declares with too (see
make_layers)."""
return (
declare_attn(),
declare_ffn(sparse=True, next_layer_sparse=True),
)
def forward(
@@ -292,6 +301,7 @@ class MixtralModel(nn.Module):
),
prefix="layers",
return_tuple=True,
stage_facts=partial(MixtralDecoderLayer.stage_facts, config),
)
if self.pp_group.is_last_rank:
+15 -5
View File
@@ -1,4 +1,5 @@
import logging
from functools import partial
from typing import Iterable, List, Optional, Tuple, Union
import torch
@@ -231,12 +232,20 @@ class NanbeigeDecoderLayer(nn.Module):
)
# Each loop runs the physical layers as one stack: the model folds the
# residual back in and re-enters at layer 0.
attn, ffn = self.stage_facts(config, layer_id)
self.attn_boundary, self.ffn_boundary = append_stages(
(declare_attn(), self.input_layernorm),
(
declare_ffn(sparse=False, next_layer_sparse=False),
self.post_attention_layernorm,
),
(attn, self.input_layernorm),
(ffn, self.post_attention_layernorm),
)
@staticmethod
def stage_facts(config: NanbeigeConfig, layer_id: int):
"""The stages the layer at ``layer_id`` declares, in order: the model's
shared declaration function, which the layer declares with too (see
make_layers)."""
return (
declare_attn(),
declare_ffn(sparse=False, next_layer_sparse=False),
)
def forward(
@@ -305,6 +314,7 @@ class NanbeigeModel(nn.Module):
alt_stream=alt_stream,
),
prefix=add_prefix("layers", prefix),
stage_facts=partial(decoder_layer_type.stage_facts, config),
)
if self.pp_group.is_last_rank:
self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
+19 -9
View File
@@ -134,17 +134,12 @@ class DeciLMDecoderLayer(nn.Module):
config.hidden_size, eps=config.rms_norm_eps
)
# A layer whose attention and FFN are both no-ops has no stages.
stages = []
norms = []
if not self._is_no_op_attention:
stages.append((declare_attn(), self.input_layernorm))
norms.append(self.input_layernorm)
if not self._is_no_op_ffn:
stages.append(
(
declare_ffn(sparse=False, next_layer_sparse=False),
self.post_attention_layernorm,
)
)
norms.append(self.post_attention_layernorm)
stages = list(zip(self.stage_facts(config, layer_idx), norms, strict=True))
self.entry_boundary = None
if stages:
boundaries = append_stages(
@@ -156,6 +151,20 @@ class DeciLMDecoderLayer(nn.Module):
if not self._is_no_op_ffn:
self.ffn_boundary = boundaries[-1]
@staticmethod
def stage_facts(config: LlamaConfig, layer_idx: int):
"""The stages a DeciLM layer declares, from its block config alone:
the model's shared declaration function, which the layer declares
with too (see make_layers)."""
block_config = config.block_configs[layer_idx]
# A layer whose attention and FFN are both no-ops has no stages.
stages = []
if not block_config.attention.no_op:
stages.append(declare_attn())
if not block_config.ffn.no_op:
stages.append(declare_ffn(sparse=False, next_layer_sparse=False))
return tuple(stages)
def forward(
self,
positions: torch.Tensor,
@@ -223,6 +232,7 @@ class DeciModel(nn.Module):
config.num_hidden_layers,
get_layer,
prefix=add_prefix("layers", prefix),
stage_facts=lambda idx: layer_type.stage_facts(config, idx),
)
if get_parallel().pp_group.is_last_rank:
self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
+16 -6
View File
@@ -299,15 +299,24 @@ class Qwen2DecoderLayer(nn.Module):
)
self.input_layernorm.fuse_input_quant(self.self_attn.qkv_proj)
self.post_attention_layernorm.fuse_input_quant(self.mlp.gate_up_proj)
attn, ffn = self.stage_facts(config, layer_id)
self.attn_boundary, self.ffn_boundary = append_stages(
(declare_attn(read=self._attn_readout(layer_id)), self.input_layernorm),
(
declare_ffn(sparse=False, next_layer_sparse=False),
self.post_attention_layernorm,
),
(attn, self.input_layernorm),
(ffn, self.post_attention_layernorm),
)
def _attn_readout(self, layer_id: int):
@classmethod
def stage_facts(cls, config: Qwen2Config, layer_id: int):
"""The stages a layer of this class declares, from the config alone:
the model's shared declaration function, which the layer declares
with too (see make_layers). A subclass's attention read takes part."""
return (
declare_attn(read=cls._attn_readout(layer_id)),
declare_ffn(sparse=False, next_layer_sparse=False),
)
@classmethod
def _attn_readout(cls, layer_id: int):
return NORM_QUANT_READOUT
def forward(
@@ -384,6 +393,7 @@ class Qwen2Model(nn.Module):
alt_stream=alt_stream,
),
prefix=add_prefix("layers", prefix),
stage_facts=lambda idx: decoder_layer_type.stage_facts(config, idx),
)
if self.pp_group.is_last_rank:
norm_kwargs = (
+2 -1
View File
@@ -55,7 +55,8 @@ class Qwen2DecoderLayer(Qwen2DecoderLayer):
if layer_id == 0:
del self.input_layernorm
def _attn_readout(self, layer_id: int):
@classmethod
def _attn_readout(cls, layer_id: int):
if layer_id == 0:
return PLAIN_READOUT
return super()._attn_readout(layer_id)
+20 -10
View File
@@ -1026,9 +1026,8 @@ class Qwen2MoeDecoderLayer(nn.Module):
self.layer_id = layer_id
# Qwen2MoE all layers are sparse and have no nextn now
self.is_layer_sparse = True
is_next_layer_sparse = True
attn, ffn = self.stage_facts(config, layer_id)
self.is_layer_sparse = ffn.sparse
if self.is_layer_sparse:
self.mlp = Qwen2MoeSparseMoeBlock(
@@ -1053,13 +1052,23 @@ class Qwen2MoeDecoderLayer(nn.Module):
config.hidden_size, eps=config.rms_norm_eps
)
self.attn_boundary, self.ffn_boundary = append_stages(
(declare_attn(), self.input_layernorm),
(
declare_ffn(
sparse=self.is_layer_sparse,
next_layer_sparse=is_next_layer_sparse,
),
self.post_attention_layernorm,
(attn, self.input_layernorm),
(ffn, self.post_attention_layernorm),
)
@staticmethod
def stage_facts(config: PretrainedConfig, layer_id: int):
"""The stages a Qwen2-MoE layer declares, from the config alone: the
model's shared declaration function, which the layer declares with
too (see make_layers)."""
# Qwen2MoE all layers are sparse and have no nextn now
is_layer_sparse = True
is_next_layer_sparse = True
return (
declare_attn(),
declare_ffn(
sparse=is_layer_sparse,
next_layer_sparse=is_next_layer_sparse,
),
)
@@ -1138,6 +1147,7 @@ class Qwen2MoeModel(nn.Module):
alt_stream=alt_stream,
),
prefix=add_prefix("layers", prefix),
stage_facts=lambda idx: decoder_layer_type.stage_facts(config, idx),
)
if self.pp_group.is_last_rank:
self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
+13 -5
View File
@@ -373,12 +373,20 @@ class Qwen3DecoderLayer(nn.Module):
config.hidden_size, eps=config.rms_norm_eps, **norm_kwargs
)
attn, ffn = self.stage_facts(config, layer_id)
self.attn_boundary, self.ffn_boundary = append_stages(
(declare_attn(), self.input_layernorm),
(
declare_ffn(sparse=False, next_layer_sparse=False),
self.post_attention_layernorm,
),
(attn, self.input_layernorm),
(ffn, self.post_attention_layernorm),
)
@staticmethod
def stage_facts(config: Qwen3Config, layer_id: int):
"""The stages a Qwen3 layer declares, from the config alone: the
model's shared declaration function, which the layer declares with
too (see make_layers)."""
return (
declare_attn(),
declare_ffn(sparse=False, next_layer_sparse=False),
)
def forward(
+50 -50
View File
@@ -69,6 +69,7 @@ from sglang.srt.layers.layer_boundary.residual import batch as residual_batch
# Layers - Others
from sglang.srt.layers.layer_boundary.residual.add_norm import (
NORM_QUANT_READOUT,
Fp8Input,
NormQuantReadout,
)
@@ -252,6 +253,27 @@ def _layer_fusions(config: Qwen3_5TextConfig, is_nextn: bool):
return None
def _qwen3_5_is_moe(config: Qwen3_5TextConfig) -> bool:
"""Whether the model's layers run sparse MoE blocks rather than dense MLPs:
every layer of either kind alike."""
return config.model_type in _QWEN3_5_MOE_TEXT_MODEL_TYPES
def _qwen3_5_stage_facts(
config: Qwen3_5TextConfig, layer_id: int, *, attn_read=NORM_QUANT_READOUT
):
"""The attention and FFN stages a Qwen3.5 layer of either kind declares:
the model's shared declaration function (see make_layers), which the
layer declares itself with. ``attn_read`` is the layer's own attention
read, which declares what the plain one does and also hands on the
quantized input its projection takes."""
sparse = _qwen3_5_is_moe(config)
return (
declare_attn(read=attn_read),
declare_ffn(sparse=sparse, next_layer_sparse=sparse),
)
if _is_cuda:
from sglang.kernels.ops.attention.fused_qk_rmsnorm_rope_gate import (
fused_qk_gemma_rmsnorm_rope_gate,
@@ -1087,6 +1109,8 @@ class Qwen3_5GatedDeltaNet(nn.Module):
class Qwen3_5LinearDecoderLayer(nn.Module):
"""Qwen3.5 Decoder Layer with Linear Attention (GatedDeltaNet)."""
stage_facts = staticmethod(_qwen3_5_stage_facts)
def __init__(
self,
config: Qwen3_5TextConfig,
@@ -1107,7 +1131,7 @@ class Qwen3_5LinearDecoderLayer(nn.Module):
# NOTE: Determine the MLP type based on the model type
# Qwen3.5 use all layers for MLP / Qwen3.5-MoE use sparse MoE blocks
if config.model_type in _QWEN3_5_MOE_TEXT_MODEL_TYPES:
if _qwen3_5_is_moe(config):
self.mlp = Qwen2MoeSparseMoeBlock(
layer_id=layer_id,
config=config,
@@ -1123,8 +1147,6 @@ class Qwen3_5LinearDecoderLayer(nn.Module):
# The stage boundary completes this stage's sum.
reduce_results=False,
)
is_layer_sparse = True
is_next_layer_sparse = True
elif config.model_type == "qwen3_5_text":
self.mlp = Qwen2MoeMLP(
hidden_size=config.hidden_size,
@@ -1136,8 +1158,6 @@ class Qwen3_5LinearDecoderLayer(nn.Module):
reduce_results=False,
)
_maybe_enable_silu_fp4_quant_fusion(self.mlp)
is_layer_sparse = False
is_next_layer_sparse = False
else:
raise ValueError(f"Invalid model type: {config.model_type}")
@@ -1152,28 +1172,16 @@ class Qwen3_5LinearDecoderLayer(nn.Module):
boundary_fusions = _layer_fusions(config, is_nextn)
# A subclass that brings its own residual builds the stages itself.
if build_stages:
attn, ffn = self.stage_facts(
config,
layer_id,
attn_read=NormQuantReadout(
fp8_input=Fp8Input.TUPLE_AND_BF16 if accepts_fp8_input else None
),
)
self.attn_boundary, self.ffn_boundary = append_stages(
(
declare_attn(
read=NormQuantReadout(
fp8_input=Fp8Input.TUPLE_AND_BF16
if accepts_fp8_input
else None
)
),
self.input_layernorm,
{
"fusions": boundary_fusions,
},
),
(
declare_ffn(
sparse=is_layer_sparse,
next_layer_sparse=is_next_layer_sparse,
),
self.post_attention_layernorm,
{"fusions": boundary_fusions},
),
(attn, self.input_layernorm, {"fusions": boundary_fusions}),
(ffn, self.post_attention_layernorm, {"fusions": boundary_fusions}),
)
def forward(
@@ -1216,6 +1224,8 @@ class Qwen3_5LinearDecoderLayer(nn.Module):
class Qwen3_5AttentionDecoderLayer(nn.Module):
"""Qwen3.5 Decoder Layer with Full Attention."""
stage_facts = staticmethod(_qwen3_5_stage_facts)
def __init__(
self,
config: Qwen3_5TextConfig,
@@ -1327,9 +1337,7 @@ class Qwen3_5AttentionDecoderLayer(nn.Module):
# The stage boundary completes this stage's sum.
reduce_results=False,
)
is_layer_sparse = False
is_next_layer_sparse = False
elif config.model_type in _QWEN3_5_MOE_TEXT_MODEL_TYPES:
elif _qwen3_5_is_moe(config):
self.mlp = Qwen2MoeSparseMoeBlock(
layer_id=layer_id,
config=config,
@@ -1345,8 +1353,6 @@ class Qwen3_5AttentionDecoderLayer(nn.Module):
# The stage boundary completes this stage's sum.
reduce_results=False,
)
is_layer_sparse = True
is_next_layer_sparse = True
else:
raise ValueError(f"Invalid model type: {config.model_type}")
@@ -1364,26 +1370,16 @@ class Qwen3_5AttentionDecoderLayer(nn.Module):
boundary_fusions = _layer_fusions(config, is_nextn)
# A subclass that brings its own residual builds the stages itself.
if build_stages:
attn, ffn = self.stage_facts(
config,
layer_id,
attn_read=NormQuantReadout(
fp8_input=Fp8Input.TUPLE if accepts_fp8_input else None
),
)
self.attn_boundary, self.ffn_boundary = append_stages(
(
declare_attn(
read=NormQuantReadout(
fp8_input=Fp8Input.TUPLE if accepts_fp8_input else None
)
),
self.input_layernorm,
{
"fusions": boundary_fusions,
},
),
(
declare_ffn(
sparse=is_layer_sparse,
next_layer_sparse=is_next_layer_sparse,
),
self.post_attention_layernorm,
{"fusions": boundary_fusions},
),
(attn, self.input_layernorm, {"fusions": boundary_fusions}),
(ffn, self.post_attention_layernorm, {"fusions": boundary_fusions}),
)
self.alt_stream = alt_stream
@@ -1752,6 +1748,10 @@ class Qwen3_5ForCausalLM(nn.Module):
config.num_hidden_layers,
get_layer,
prefix=f"{prefix}.layers",
# The layer type chosen for each index declares its stages.
stage_facts=lambda idx: self.decoder_layer_types[
config.layers_block_type[idx]
].stage_facts(config, idx),
)
self.flashinfer_mnnvl_cutedsl_fusion = None
+19 -10
View File
@@ -853,9 +853,8 @@ class Qwen3MoeDecoderLayer(nn.Module):
self.layer_id = layer_id
# Qwen3MoE all layers are sparse and have no nextn now
self.is_layer_sparse = True
is_next_layer_sparse = True
attn, ffn = self.stage_facts(config, layer_id)
self.is_layer_sparse = ffn.sparse
if self.is_layer_sparse:
self.mlp = Qwen3MoeSparseMoeBlock(
@@ -879,13 +878,23 @@ class Qwen3MoeDecoderLayer(nn.Module):
)
self.attn_boundary, self.ffn_boundary = append_stages(
(declare_attn(), self.input_layernorm),
(
declare_ffn(
sparse=self.is_layer_sparse,
next_layer_sparse=is_next_layer_sparse,
),
self.post_attention_layernorm,
(attn, self.input_layernorm),
(ffn, self.post_attention_layernorm),
)
@staticmethod
def stage_facts(config: Qwen3MoeConfig, layer_id: int):
"""The stages a Qwen3-MoE layer declares, from the config alone: the
model's shared declaration function, which the layer declares with
too (see make_layers)."""
# Qwen3MoE all layers are sparse and have no nextn now
is_layer_sparse = True
is_next_layer_sparse = True
return (
declare_attn(),
declare_ffn(
sparse=is_layer_sparse,
next_layer_sparse=is_next_layer_sparse,
),
)
+39 -29
View File
@@ -81,6 +81,7 @@ from sglang.srt.models.qwen3_5 import (
Qwen3_5ForCausalLM,
Qwen3_5GatedDeltaNet,
Qwen3_5LinearDecoderLayer,
_qwen3_5_is_moe,
)
from sglang.srt.models.qwen3_vl import Qwen3VLForConditionalGeneration
from sglang.srt.models.qwen4_exp_ple_table import (
@@ -90,7 +91,6 @@ from sglang.srt.models.qwen4_exp_ple_table import (
)
from sglang.srt.runtime_context import get_parallel
from sglang.srt.utils import get_bool_env_var, is_hip, logger
from sglang.srt.utils.common import is_building_neighbour_layer
_use_aiter = get_bool_env_var("SGLANG_USE_AITER") and is_hip()
@@ -537,13 +537,6 @@ class Qwen4ExpPLEGroupedNorm(nn.Module):
return (x_norm * weight).to(compute_dtype)
def _offloads_ple(config) -> bool:
"""Whether this layer keeps its PLE table in host memory. A pipeline
neighbour layer is never loaded or run, so it keeps the table on the meta
device and allocates no host memory or stream for it."""
return bool(config.ple_offload_embedding) and not is_building_neighbour_layer()
class Qwen4ExpNGramEmbedding(nn.Module):
_MASK64 = (1 << 64) - 1
_SPLITMIX_GAMMA = 0x9E3779B97F4A7C15
@@ -615,7 +608,7 @@ class Qwen4ExpNGramEmbedding(nn.Module):
and not self.use_attn_tp_ngram
)
ngram_prefix = f"{prefix}.ngram_embedding" if prefix else "ngram_embedding"
offload_embedding = _offloads_ple(config)
offload_embedding = bool(config.ple_offload_embedding)
# Offload only needs this embedding's metadata: build it on meta so the
# shard is never allocated on the device.
with torch.device("meta") if offload_embedding else nullcontext():
@@ -1098,7 +1091,9 @@ class Qwen4ExpPLELayer(nn.Module):
bias=False,
)
nn.init.zeros_(self.conv1d.weight)
self._prefetch_stream = torch.cuda.Stream() if _offloads_ple(config) else None
self._prefetch_stream = (
torch.cuda.Stream() if config.ple_offload_embedding else None
)
self._graph_prefetch_buffers = {}
self._eager_prefetch_buffer = None
self._prefetch_state = None
@@ -1449,31 +1444,45 @@ def _has_ple(layer_id, config) -> bool:
return (layer_id + 1) in config.ple_layer_ids
def _build_qwen4_exp_stages(residual, *, sparse):
"""The attention and FFN stage boundaries of one gated hyper-connection layer.
def _qwen4_exp_stage_facts(config, layer_id, *, residual=None):
"""The attention and FFN stages a gated hyper-connection layer of either
kind declares: the model's shared declaration function (see make_layers),
which the layer declares itself with. ``residual`` is the layer's own
reads and updates of the streams; without it, the stages declare what
those do. A PLE layer's attention read needs every row, for the PLE
embedding is computed for all of them."""
if residual is None:
residual = GatedResidualState.facts(
attn_reads_every_row=_has_ple(layer_id, config)
)
sparse = _qwen3_5_is_moe(config)
return (
declare_attn(read=residual.attn_readout, update=residual.attn_update),
declare_ffn(
sparse=sparse,
next_layer_sparse=sparse,
read=residual.ffn_readout,
update=residual.ffn_update,
),
)
def _build_qwen4_exp_stages(config, layer_id, residual):
"""The attention and FFN stage boundaries of one gated hyper-connection
layer, declared by _qwen4_exp_stage_facts with the layer's own reads and
updates (``residual``).
Each read normalizes the streams itself, so neither stage binds a norm, and
the writes are gated injections, so neither stage offers a fused
add-and-norm candidate.
"""
return append_stages(
(
declare_attn(read=residual.attn_readout, update=residual.attn_update),
None,
),
(
declare_ffn(
sparse=sparse,
next_layer_sparse=sparse,
read=residual.ffn_readout,
update=residual.ffn_update,
),
None,
),
)
attn, ffn = _qwen4_exp_stage_facts(config, layer_id, residual=residual)
return append_stages((attn, None), (ffn, None))
class Qwen4ExpLayerExtensionMixin:
stage_facts = staticmethod(_qwen4_exp_stage_facts)
def _init_qwen4_exp_layer_extensions(
self,
config: Qwen4ExpTextConfig,
@@ -1529,6 +1538,8 @@ class Qwen4ExpLayerExtensionMixin:
use_combine=True,
)
self.attn_boundary, self.ffn_boundary = _build_qwen4_exp_stages(
config,
layer_id,
GatedResidualState(
expand=self._widen_streams,
attn_mix=self._attn_mix,
@@ -1536,9 +1547,8 @@ class Qwen4ExpLayerExtensionMixin:
attn_combine=self._attn_combine,
ffn_combine=self._ffn_combine,
# The PLE embedding is computed for every row.
attn_reads_every_row=self.ple is not None,
attn_reads_every_row=_has_ple(layer_id, config),
).residual_ops(),
sparse=isinstance(self.mlp, Qwen2MoeSparseMoeBlock),
)
from sglang.srt.layers.moe.qwen4_decode import prepare_qwen4_decode_comm
+41 -21
View File
@@ -5,6 +5,7 @@
import math
from enum import IntEnum, auto
from functools import partial
from typing import Any, Dict, Iterable, Optional, Tuple
import torch
@@ -986,20 +987,7 @@ class SarvamMoEMLADecoderLayer(nn.Module):
alt_stream=alt_stream,
)
first_k_dense = getattr(config, "first_k_dense_replace", 1)
moe_layer_freq = getattr(config, "moe_layer_freq", 1)
has_moe = getattr(config, "num_experts", None) is not None
self.is_layer_sparse = (
has_moe
and layer_id >= first_k_dense
and (layer_id - first_k_dense) % moe_layer_freq == 0
)
is_next_layer_sparse = (
has_moe
and layer_id < config.num_hidden_layers - 1
and (layer_id + 1) >= first_k_dense
and (layer_id + 1 - first_k_dense) % moe_layer_freq == 0
)
self.is_layer_sparse = self._is_layer_sparse(config, layer_id)
if self.is_layer_sparse:
self.mlp = SarvamMoESparseMoeBlock(
@@ -1026,21 +1014,52 @@ class SarvamMoEMLADecoderLayer(nn.Module):
config.hidden_size, eps=config.rms_norm_eps
)
attn, ffn = self.stage_facts(config, layer_id)
self.attn_boundary, self.ffn_boundary = append_stages(
(
declare_attn(),
attn,
self.input_layernorm,
{"qkv_latent_func": self.self_attn.prepare_qkv_latent},
),
(
declare_ffn(
sparse=self.is_layer_sparse,
next_layer_sparse=is_next_layer_sparse,
),
self.post_attention_layernorm,
(ffn, self.post_attention_layernorm),
)
@classmethod
def stage_facts(cls, config: PretrainedConfig, layer_id: int):
"""The stages the layer at ``layer_id`` declares, in order: the model's
shared declaration function, which the layer declares with too (see
make_layers)."""
return (
declare_attn(),
declare_ffn(
sparse=cls._is_layer_sparse(config, layer_id),
next_layer_sparse=cls._is_next_layer_sparse(config, layer_id),
),
)
@staticmethod
def _is_layer_sparse(config: PretrainedConfig, layer_id: int) -> bool:
first_k_dense = getattr(config, "first_k_dense_replace", 1)
moe_layer_freq = getattr(config, "moe_layer_freq", 1)
has_moe = getattr(config, "num_experts", None) is not None
return (
has_moe
and layer_id >= first_k_dense
and (layer_id - first_k_dense) % moe_layer_freq == 0
)
@staticmethod
def _is_next_layer_sparse(config: PretrainedConfig, layer_id: int) -> bool:
first_k_dense = getattr(config, "first_k_dense_replace", 1)
moe_layer_freq = getattr(config, "moe_layer_freq", 1)
has_moe = getattr(config, "num_experts", None) is not None
return (
has_moe
and layer_id < config.num_hidden_layers - 1
and (layer_id + 1) >= first_k_dense
and (layer_id + 1 - first_k_dense) % moe_layer_freq == 0
)
def forward(
self,
positions: torch.Tensor,
@@ -1096,6 +1115,7 @@ class SarvamMLAModel(nn.Module):
alt_stream=self.alt_stream,
),
prefix="model.layers",
stage_facts=partial(SarvamMoEMLADecoderLayer.stage_facts, config),
)
if self.pp_group.is_last_rank:
+17 -7
View File
@@ -4,6 +4,7 @@ SGLang SDARModelLM (block diffusion / dLLM-style forward).
"""
import logging
from functools import partial
from typing import Iterable, Optional, Tuple, Union
import torch
@@ -291,14 +292,22 @@ class SDARBlock(nn.Module):
prefix=add_prefix("mlp", prefix),
)
attn, ffn = self.stage_facts(config, layer_id)
self.attn_boundary, self.ffn_boundary = append_stages(
(declare_attn(), self.input_layernorm),
(
declare_ffn(
sparse=False,
next_layer_sparse=False,
),
self.post_attention_layernorm,
(attn, self.input_layernorm),
(ffn, self.post_attention_layernorm),
)
@staticmethod
def stage_facts(config: PretrainedConfig, layer_id: int):
"""The stages the layer at ``layer_id`` declares, in order: the model's
shared declaration function, which the layer declares with too (see
make_layers)."""
return (
declare_attn(),
declare_ffn(
sparse=False,
next_layer_sparse=False,
),
)
@@ -361,6 +370,7 @@ class SDARModel(nn.Module):
alt_stream=alt_stream,
),
prefix=add_prefix("layers", prefix),
stage_facts=partial(SDARBlock.stage_facts, config),
)
if self.pp_group.is_last_rank:
norm_kwargs = (
+15 -5
View File
@@ -4,6 +4,7 @@ SGLang SDARMoeModelLM (block diffusion / dLLM-style forward) with MoE MLP.
"""
import logging
from functools import partial
from typing import Iterable, Optional, Tuple, Union
import torch
@@ -351,12 +352,20 @@ class SDARMoeBlock(nn.Module):
prefix=add_prefix("mlp", prefix),
)
attn, ffn = self.stage_facts(config, layer_id)
self.attn_boundary, self.ffn_boundary = append_stages(
(declare_attn(), self.input_layernorm),
(
declare_ffn(sparse=True, next_layer_sparse=True),
self.post_attention_layernorm,
),
(attn, self.input_layernorm),
(ffn, self.post_attention_layernorm),
)
@staticmethod
def stage_facts(config: PretrainedConfig, layer_id: int):
"""The stages the layer at ``layer_id`` declares, in order: the model's
shared declaration function, which the layer declares with too (see
make_layers)."""
return (
declare_attn(),
declare_ffn(sparse=True, next_layer_sparse=True),
)
def forward(
@@ -422,6 +431,7 @@ class SDARMoeModel(nn.Module):
alt_stream=alt_stream,
),
prefix=add_prefix("layers", prefix),
stage_facts=partial(SDARMoeBlock.stage_facts, config),
)
if self.pp_group.is_last_rank:
+15 -5
View File
@@ -1,4 +1,5 @@
import logging
from functools import partial
from typing import Iterable, Optional, Tuple, Union
import torch
@@ -267,12 +268,20 @@ class Spark2_5DecoderLayer(nn.Module):
self.post_attention_layernorm = RMSNorm(
config.hidden_size, eps=config.rms_norm_eps
)
attn, ffn = self.stage_facts(config, layer_id)
self.attn_boundary, self.ffn_boundary = append_stages(
(declare_attn(), self.input_layernorm),
(
declare_ffn(sparse=False, next_layer_sparse=False),
self.post_attention_layernorm,
),
(attn, self.input_layernorm),
(ffn, self.post_attention_layernorm),
)
@staticmethod
def stage_facts(config: Spark2_5Config, layer_id: int):
"""The stages the layer at ``layer_id`` declares, in order: the model's
shared declaration function, which the layer declares with too (see
make_layers)."""
return (
declare_attn(),
declare_ffn(sparse=False, next_layer_sparse=False),
)
def forward(
@@ -326,6 +335,7 @@ class Spark2_5Model(nn.Module):
prefix=prefix,
),
prefix=add_prefix("layers", prefix),
stage_facts=partial(Spark2_5DecoderLayer.stage_facts, config),
)
if self.pp_group.is_last_rank:
self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
+25 -11
View File
@@ -1,3 +1,4 @@
from functools import partial
from typing import Any, Dict, Iterable, Optional, Tuple, Union
import torch
@@ -422,11 +423,10 @@ class Step3p5DecoderLayer(nn.Module):
rope_theta = config.rope_theta
max_position_embeddings = config.max_position_embeddings
head_dim = config.head_dim
moe_layers_set = {int(x) for x in config.moe_layers_enum.split(",")}
self.num_attention_heads = config.num_attention_heads
self.num_key_value_heads = config.num_attention_groups
self.is_moe_layer = layer_id in moe_layers_set
self.is_next_layer_sparse = (layer_id + 1) in moe_layers_set
self.is_moe_layer = self._is_moe_layer(config, layer_id)
self.is_next_layer_sparse = self._is_moe_layer(config, layer_id + 1)
num_hidden_layers = config.num_hidden_layers
if (
@@ -519,19 +519,32 @@ class Step3p5DecoderLayer(nn.Module):
config.hidden_size, eps=config.rms_norm_eps
)
attn, ffn = self.stage_facts(config, layer_id)
self.attn_boundary, self.ffn_boundary = append_stages(
(declare_attn(), self.input_layernorm),
(
declare_ffn(
sparse=self.is_moe_layer,
next_layer_sparse=self.is_next_layer_sparse,
),
self.post_attention_layernorm,
),
(attn, self.input_layernorm),
(ffn, self.post_attention_layernorm),
)
self.layer_id = layer_id
@classmethod
def stage_facts(cls, config: Step3p5Config, layer_id: int):
"""The stages the layer at ``layer_id`` declares, in order: the model's
shared declaration function, which the layer declares with too (see
make_layers)."""
return (
declare_attn(),
declare_ffn(
sparse=cls._is_moe_layer(config, layer_id),
next_layer_sparse=cls._is_moe_layer(config, layer_id + 1),
),
)
@staticmethod
def _is_moe_layer(config: Step3p5Config, layer_id: int) -> bool:
moe_layers_set = {int(x) for x in config.moe_layers_enum.split(",")}
return layer_id in moe_layers_set
def forward(
self,
positions: torch.Tensor,
@@ -605,6 +618,7 @@ class Step3p5Model(nn.Module):
alt_stream=alt_stream,
),
prefix=add_prefix("layers", prefix),
stage_facts=partial(Step3p5DecoderLayer.stage_facts, config),
)
if self.pp_group.is_last_rank:
self.norm = GemmaRMSNorm(config.hidden_size, eps=config.rms_norm_eps)
+42 -26
View File
@@ -28,6 +28,7 @@
import math
from contextlib import nullcontext
from functools import partial
from typing import Any, Dict, Iterable, Optional, Tuple, Union
import torch
@@ -1433,15 +1434,7 @@ class XllmDecoderLayer(nn.Module):
self.layer_id = layer_id
# Determine if this layer is sparse (MoE) or dense
mlp_only_layers = getattr(config, "mlp_only_layers", [])
decoder_sparse_step = getattr(config, "decoder_sparse_step", 1)
if (layer_id not in mlp_only_layers) and (
config.num_experts > 0 and (layer_id + 1) % decoder_sparse_step == 0
):
self.is_layer_sparse = True
else:
self.is_layer_sparse = False
self.is_layer_sparse = self._is_layer_sparse(config, layer_id)
is_mova_config = getattr(config, "num_values", 0) > 0
is_mova_attention = is_mova_config and layer_id >= config.num_dense_layers
@@ -1475,16 +1468,6 @@ class XllmDecoderLayer(nn.Module):
prefix=add_prefix("self_attn", prefix),
)
# Check neighbors for scatter modes
def _is_sparse(lid):
if lid < 0 or lid >= config.num_hidden_layers:
return False
return (lid not in mlp_only_layers) and (
config.num_experts > 0 and (lid + 1) % decoder_sparse_step == 0
)
is_next_layer_sparse = _is_sparse(layer_id + 1)
if self.is_layer_sparse:
self.mlp = XllmSparseMoeBlock(
layer_id=layer_id,
@@ -1506,17 +1489,49 @@ class XllmDecoderLayer(nn.Module):
self.input_layernorm = _make_norm(config)
self.post_attention_layernorm = _make_norm(config)
attn, ffn = self.stage_facts(config, layer_id)
self.attn_boundary, self.ffn_boundary = append_stages(
(declare_attn(), self.input_layernorm),
(
declare_ffn(
sparse=self.is_layer_sparse,
next_layer_sparse=is_next_layer_sparse,
),
self.post_attention_layernorm,
(attn, self.input_layernorm),
(ffn, self.post_attention_layernorm),
)
@classmethod
def stage_facts(cls, config: PretrainedConfig, layer_id: int):
"""The stages the layer at ``layer_id`` declares, in order: the model's
shared declaration function, which the layer declares with too (see
make_layers)."""
return (
declare_attn(),
declare_ffn(
sparse=cls._is_layer_sparse(config, layer_id),
next_layer_sparse=cls._is_next_layer_sparse(config, layer_id),
),
)
@staticmethod
def _is_layer_sparse(config: PretrainedConfig, layer_id: int) -> bool:
# Determine if this layer is sparse (MoE) or dense
mlp_only_layers = getattr(config, "mlp_only_layers", [])
decoder_sparse_step = getattr(config, "decoder_sparse_step", 1)
if (layer_id not in mlp_only_layers) and (
config.num_experts > 0 and (layer_id + 1) % decoder_sparse_step == 0
):
return True
else:
return False
@staticmethod
def _is_next_layer_sparse(config: PretrainedConfig, layer_id: int) -> bool:
# Check neighbors for scatter modes
mlp_only_layers = getattr(config, "mlp_only_layers", [])
decoder_sparse_step = getattr(config, "decoder_sparse_step", 1)
lid = layer_id + 1
if lid < 0 or lid >= config.num_hidden_layers:
return False
return (lid not in mlp_only_layers) and (
config.num_experts > 0 and (lid + 1) % decoder_sparse_step == 0
)
def forward(
self,
positions: torch.Tensor,
@@ -1576,6 +1591,7 @@ class XllmModel(nn.Module):
prefix=prefix,
),
prefix=add_prefix("layers", prefix),
stage_facts=partial(XllmDecoderLayer.stage_facts, config),
)
if self.pp_group.is_last_rank:
self.norm = _make_norm(config)
+39 -23
View File
@@ -1317,11 +1317,16 @@ class ZayaDecoderATTLayer(nn.Module):
self.res_scale = ResidualScaling(config, layer_id)
else:
self.res_scale = None
(self.attn_boundary,) = append_stages(
(declare_attn(read=_ResidualMergeRead(self.res_scale)), self.input_norm),
)
(attn,) = self.stage_facts(res_scale=self.res_scale)
(self.attn_boundary,) = append_stages((attn, self.input_norm))
self.entry_boundary = self.attn_boundary
@staticmethod
def stage_facts(*, res_scale: Optional[ResidualScaling] = None):
"""The one stage an attention layer declares: its attention, read
through ``res_scale``."""
return (declare_attn(read=_ResidualMergeRead(res_scale)),)
@staticmethod
def _build_norm(config: ZayaConfig) -> nn.Module:
if config.normalization == "RMSNorm":
@@ -1368,17 +1373,21 @@ class ZayaDecoderMLPLayer(nn.Module):
self.res_scale = ResidualScaling(config, layer_id)
else:
self.res_scale = None
(self.ffn_boundary,) = append_stages(
(
declare_ffn(
sparse=True,
next_layer_sparse=True,
read=_ResidualMergeRead(self.res_scale),
),
self.input_norm,
(ffn,) = self.stage_facts(res_scale=self.res_scale)
(self.ffn_boundary,) = append_stages((ffn, self.input_norm))
self.entry_boundary = self.ffn_boundary
@staticmethod
def stage_facts(*, res_scale: Optional[ResidualScaling] = None):
"""The one stage an MoE layer declares: its MoE, read through
``res_scale``."""
return (
declare_ffn(
sparse=True,
next_layer_sparse=True,
read=_ResidualMergeRead(res_scale),
),
)
self.entry_boundary = self.ffn_boundary
def forward(
self,
@@ -1400,23 +1409,22 @@ class ZayaDecoderMLPLayer(nn.Module):
# ---------------------------------------------------------------------------
def _layer_class(layer_id: int):
# Even layer ids are attention, odd layer ids are MoE. This matches the HF
# checkpoint keys: ``model.layers.<2k>.self_attn.*`` (CCA) versus
# ``model.layers.<2k+1>.zaya_block.*`` (MoE).
if layer_id % 2 == 0:
return ZayaDecoderATTLayer
return ZayaDecoderMLPLayer
def _build_layer(
layer_id: int,
config: ZayaConfig,
quant_config: Optional[QuantizationConfig],
prefix: str,
) -> nn.Module:
# Even layer ids are attention, odd layer ids are MoE. This matches the HF
# checkpoint keys: ``model.layers.<2k>.self_attn.*`` (CCA) versus
# ``model.layers.<2k+1>.zaya_block.*`` (MoE).
if layer_id % 2 == 0:
return ZayaDecoderATTLayer(
config=config,
layer_id=layer_id,
quant_config=quant_config,
prefix=prefix,
)
return ZayaDecoderMLPLayer(
return _layer_class(layer_id)(
config=config,
layer_id=layer_id,
quant_config=quant_config,
@@ -1424,6 +1432,13 @@ def _build_layer(
)
def stage_facts(layer_id: int):
"""The one stage the layer at ``layer_id`` declares, from its kind alone:
the model's shared declaration function, which its layers declare with
too (see make_layers)."""
return _layer_class(layer_id).stage_facts()
class ZayaModel(nn.Module):
def __init__(
self,
@@ -1456,6 +1471,7 @@ class ZayaModel(nn.Module):
prefix=prefix,
),
prefix=add_prefix("layers", prefix),
stage_facts=stage_facts,
)
if self.pp_group.is_last_rank:
+13 -112
View File
@@ -1551,33 +1551,6 @@ def mark_end(name):
time_infos[name].pretty_print()
# Set while a layer another pipeline stage holds is built, only for the stage
# boundaries it declares.
_building_neighbour_layer = False
def is_building_neighbour_layer() -> bool:
"""Whether the layer under construction belongs to another pipeline stage
and is built here only to read the stage boundaries it declares. It is
built on the meta device and never loaded or run, so its constructor skips
the host and device resources a running layer needs: tables, streams,
engines and communicators."""
return _building_neighbour_layer
@contextmanager
def building_neighbour_layer():
"""Build a pipeline neighbour layer: on the meta device, with
is_building_neighbour_layer() true."""
global _building_neighbour_layer
outer, _building_neighbour_layer = _building_neighbour_layer, True
try:
with torch.device("meta"):
yield
finally:
_building_neighbour_layer = outer
class LayerFn(Protocol):
def __call__(self, idx: int, prefix: str) -> torch.nn.Module: ...
@@ -1601,10 +1574,9 @@ def make_layers(
``stage_facts``: the model's shared declaration function, which returns
the stages the layer at a global index declares without building it (an
empty sequence when it declares none), and which the layer itself uses
for its own declarations. A model on the transitional list may omit it;
the stack then builds the layer again on the meta device to read them.
``final_read`` is the stack's terminal read when it is not a plain final
norm (see layer_stack).
for its own declarations. A model whose layers declare stages must give
it to run under pipeline parallelism. ``final_read`` is the stack's
terminal read when it is not a plain final norm (see layer_stack).
"""
# circular imports
from sglang.srt.distributed import get_pp_indices
@@ -1628,10 +1600,8 @@ def make_layers(
def neighbour(idx):
if stage_facts is not None:
return functools.partial(_declared_stages, stage_facts, idx)
return functools.partial(
_build_neighbour_layer, layer_fn, idx, add_prefix(idx, prefix)
)
return functools.partial(stage_facts, idx)
return functools.partial(_no_stage_facts, layer_fn)
def build(stack, idx):
appended = len(stack.appends)
@@ -1690,84 +1660,15 @@ def make_pp_layers(
)
# The models whose layers a pipeline rank still builds on the meta device to
# read the stages they declare, until each gives make_layers its shared
# declaration function (stage_facts). Any other model must give one to run
# under pipeline parallelism.
_NEIGHBOUR_BUILD_MODULES = frozenset(
(
"sglang.srt.models.apertus",
"sglang.srt.models.arcee",
"sglang.srt.models.bailing_moe",
"sglang.srt.models.bailing_moe_linear",
"sglang.srt.models.bailing_moe_v3",
"sglang.srt.models.deepseek_v2",
"sglang.srt.models.dots3_common.modeling",
"sglang.srt.models.ernie45_moe_vl",
"sglang.srt.models.exaone4",
"sglang.srt.models.exaone_moe",
"sglang.srt.models.gemma4_causal",
"sglang.srt.models.gigachat35",
"sglang.srt.models.glm4",
"sglang.srt.models.glm4_moe",
"sglang.srt.models.glm4_moe_lite",
"sglang.srt.models.glm5_next",
"sglang.srt.models.gpt_oss",
"sglang.srt.models.granitemoehybrid",
"sglang.srt.models.kimi_k3",
"sglang.srt.models.kimi_linear",
"sglang.srt.models.laguna",
"sglang.srt.models.llada2",
"sglang.srt.models.llama",
"sglang.srt.models.mimo_v2",
"sglang.srt.models.minimax_m2",
"sglang.srt.models.minimax_m3",
"sglang.srt.models.ministral3",
"sglang.srt.models.mixtral",
"sglang.srt.models.nanbeige",
"sglang.srt.models.nemotron_nas",
"sglang.srt.models.qwen2",
"sglang.srt.models.qwen2_moe",
"sglang.srt.models.qwen3_5",
"sglang.srt.models.sarvam_moe",
"sglang.srt.models.sdar",
"sglang.srt.models.sdar_moe",
"sglang.srt.models.spark2_5",
"sglang.srt.models.step3p5",
"sglang.srt.models.xllm",
"sglang.srt.models.zaya",
)
)
def _declared_stages(stage_facts, idx: int):
"""The stages the layer at ``idx`` declares, from the model's shared
declaration function, standing in for building it."""
from sglang.srt.layers.layer_boundary.factories import DeclaredStages
return DeclaredStages(tuple(stage_facts(idx)))
def _build_neighbour_layer(layer_fn: LayerFn, idx: int, prefix: str) -> None:
"""Build a layer another pipeline stage holds, only for the stage
boundaries it declares (see building_neighbour_layer). RoPE modules it
adds to the shared cache are meta, so they are dropped again."""
from sglang.srt.layers.rotary_embedding.factory import _ROPE_DICT
def _no_stage_facts(layer_fn: LayerFn) -> None:
"""Stands in for the shared declaration function of a model that gives
none: called only if this pipeline rank's layers declare stages."""
module = getattr(getattr(layer_fn, "func", layer_fn), "__module__", None)
if module not in _NEIGHBOUR_BUILD_MODULES:
raise ValueError(
f"{module} declares stage boundaries under pipeline parallelism "
"without a shared declaration function: give make_layers its "
"stage_facts, the stages a layer declares without building it"
)
cached = set(_ROPE_DICT)
try:
with building_neighbour_layer():
layer_fn(idx=idx, prefix=prefix)
finally:
for key in set(_ROPE_DICT) - cached:
del _ROPE_DICT[key]
raise ValueError(
f"{module} declares stage boundaries under pipeline parallelism "
"without a shared declaration function: give make_layers its "
"stage_facts, the stages a layer declares without building it"
)
def set_random_seed(seed: int) -> None:
+4 -10
View File
@@ -28,19 +28,13 @@ def build_stages(*stages, previous=None, terminal=False):
terminal: Whether the last stage ends the model's layer stack;
otherwise the next layer's attention follows it on the same rank.
"""
# The stack's neighbour hooks build layers another pipeline rank holds;
# here they stand for this rank's, so nothing is handed off to them.
# The stack's neighbours are layers another pipeline rank holds; here
# they stand for this rank's, so nothing is handed off to them.
with (
mock.patch.object(factories, "_handed_off", lambda declaration: declaration),
layer_stack(
previous_layers=(
[lambda: append_stages((previous, None))]
if previous is not None
else []
),
next_layers=(
[] if terminal else [lambda: append_stages((declare_attn(), None))]
),
previous_layers=[lambda: (previous,)] if previous is not None else [],
next_layers=[] if terminal else [lambda: (declare_attn(),)],
),
):
return append_stages(*stages)