[Refactor] Declare every model's stages from a shared function and drop the meta-device neighbour build (#43460)
This commit is contained in:
@@ -7,7 +7,7 @@ import os
|
||||
import sys
|
||||
from dataclasses import dataclass, fields, is_dataclass, replace
|
||||
from types import MappingProxyType
|
||||
from typing import Callable, Mapping, NamedTuple, Optional, Tuple
|
||||
from typing import Callable, Mapping, NamedTuple, Optional
|
||||
|
||||
from sglang.srt.environ import envs
|
||||
from sglang.srt.layers import layernorm_sp
|
||||
@@ -302,11 +302,10 @@ class StageDeclaration:
|
||||
contiguous slice of the rows, it returns them all, in rank order,
|
||||
or None to leave the gather to the boundary. Called on every batch,
|
||||
so it must be CUDA-graph safe. Across a pipeline boundary the
|
||||
producer's rank takes it from the neighbouring stage's declaration
|
||||
(the model's shared declaration function, or, for a model on the
|
||||
transitional list, the layer built again on the meta device), so it
|
||||
may use only the communication state of the rank it runs on, not
|
||||
the declaring layer's weights or buffers.
|
||||
producer's rank takes it from the model's shared declaration
|
||||
function, without the declaring layer, so it may use only the
|
||||
communication state of the rank it runs on, not that layer's
|
||||
weights or buffers.
|
||||
previous: The stage whose output this stage consumes, as the stack
|
||||
records it: only the facts binding reads of it (see facts_of),
|
||||
on this rank or another.
|
||||
@@ -849,12 +848,13 @@ def layer_stack(*, previous_layers=(), next_layers=(), final_read=None):
|
||||
Args:
|
||||
previous_layers: For the layers before this stack that another
|
||||
pipeline rank holds, nearest first, callables that return the
|
||||
stages the layer declares as DeclaredStages (from the model's
|
||||
shared declaration function), or else build the layer so that it
|
||||
appends them. Called only if this stack appended stages, after its
|
||||
own layers are built, until one of them declares a stage: its last
|
||||
stage is the producer of this stack's first. Only what binding
|
||||
reads of that stage is kept (see facts_of).
|
||||
stages the layer declares, in order, without building it (the
|
||||
model's shared declaration function at that layer's index; an
|
||||
empty sequence for a layer that declares none). Called only if
|
||||
this stack appended stages, after its own layers are built, until
|
||||
one of them declares a stage: its last stage is the producer of
|
||||
this stack's first. Only what binding reads of that stage is kept
|
||||
(see facts_of).
|
||||
next_layers: Likewise for the layers after this stack, whose first
|
||||
declared stage is the consumer of this stack's last.
|
||||
final_read: The model's final read of the stack's output (a
|
||||
@@ -881,13 +881,6 @@ def layer_stack(*, previous_layers=(), next_layers=(), final_read=None):
|
||||
_stack = outer
|
||||
|
||||
|
||||
class DeclaredStages(NamedTuple):
|
||||
"""The stages a neighbouring layer declares, from the model's shared
|
||||
declaration function, which stand in for building the layer."""
|
||||
|
||||
stages: Tuple[StageDeclaration, ...]
|
||||
|
||||
|
||||
def check_declared_stages(stack, appended: int, expected, where: str) -> None:
|
||||
"""Check that what a layer appended to ``stack`` since it held
|
||||
``appended`` appends is what its shared declaration function says it
|
||||
@@ -947,26 +940,9 @@ def _differing_fields(value, given, prefix=""):
|
||||
|
||||
def _neighbour_stage(layers, *, last):
|
||||
"""The stage a neighbouring layer declares next to this stack: the last
|
||||
one of the nearest layer before it, or the first of the nearest after.
|
||||
Branches are side paths, so they never stand next to the stack."""
|
||||
global _stack
|
||||
one of the nearest layer before it, or the first of the nearest after."""
|
||||
for layer in layers:
|
||||
outer = _stack
|
||||
_stack = _LayerStack()
|
||||
try:
|
||||
result = layer()
|
||||
declared = (
|
||||
result.stages
|
||||
if isinstance(result, DeclaredStages)
|
||||
else [
|
||||
declaration
|
||||
for append in _stack.appends
|
||||
if append.prepared_from is None
|
||||
for declaration in append.declarations
|
||||
]
|
||||
)
|
||||
finally:
|
||||
_stack = outer
|
||||
declared = tuple(layer())
|
||||
if declared:
|
||||
return facts_of(declared[-1] if last else declared[0])
|
||||
return None
|
||||
|
||||
@@ -81,6 +81,18 @@ class TransformFacts(msgspec.Struct, frozen=True):
|
||||
return cls(before_reduce_scatter=transform.before_reduce_scatter)
|
||||
|
||||
|
||||
def residual_facts(ops):
|
||||
"""A layer's reads and updates (a LayerResidualOps) reduced to the
|
||||
capabilities they declare, in the same shape: what its stages declare
|
||||
without the modules and state the reads and updates hold."""
|
||||
return type(ops)(
|
||||
attn_readout=ReadFacts.of(ops.attn_readout),
|
||||
attn_update=UpdateFacts.of(ops.attn_update),
|
||||
ffn_readout=ReadFacts.of(ops.ffn_readout),
|
||||
ffn_update=UpdateFacts.of(ops.ffn_update),
|
||||
)
|
||||
|
||||
|
||||
def facts_of(declaration):
|
||||
"""``declaration`` with its read, update and output transform reduced to
|
||||
the capabilities they declare, and without the declarations it was
|
||||
|
||||
@@ -21,6 +21,7 @@ import torch
|
||||
from sglang.srt.layers.attn_residual import AttnResidual
|
||||
from sglang.srt.layers.layer_boundary import ops
|
||||
from sglang.srt.layers.layer_boundary.contracts import ReadoutFusion
|
||||
from sglang.srt.layers.layer_boundary.facts import residual_facts
|
||||
from sglang.srt.layers.layer_boundary.layout import SumGroup
|
||||
from sglang.srt.layers.layer_boundary.residual import LayerResidualOps
|
||||
from sglang.srt.layers.layer_boundary.residual.add_norm import PLAIN_ADD
|
||||
@@ -187,6 +188,22 @@ class AttnBankState:
|
||||
ffn_update=PLAIN_ADD,
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def facts(cls, *, reads_slices: bool = False) -> LayerResidualOps:
|
||||
"""What residual_ops() declares (see facts_of) for a layer whose bank
|
||||
stays on each rank's attention-TP slice when ``reads_slices``. It does
|
||||
not depend on the bank or the layer's modules, so the stages of a
|
||||
layer that is not built declare it too."""
|
||||
state = cls(
|
||||
bank=None,
|
||||
attn_score_proj=None,
|
||||
attn_score_norm=None,
|
||||
ffn_score_proj=None,
|
||||
ffn_score_norm=None,
|
||||
reads_slices=reads_slices,
|
||||
)
|
||||
return residual_facts(state.residual_ops())
|
||||
|
||||
|
||||
class _BankReadout:
|
||||
"""A read that mixes the banked snapshots into the residual's norm, so it
|
||||
|
||||
@@ -18,6 +18,7 @@ from typing import Callable, Optional
|
||||
|
||||
import torch
|
||||
|
||||
from sglang.srt.layers.layer_boundary.facts import residual_facts
|
||||
from sglang.srt.layers.layer_boundary.residual import LayerResidualOps
|
||||
from sglang.srt.runtime_context import get_parallel
|
||||
|
||||
@@ -111,6 +112,22 @@ class GatedResidualState:
|
||||
ffn_update=_FfnUpdate(self),
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def facts(cls, *, attn_reads_every_row: bool = False) -> LayerResidualOps:
|
||||
"""What residual_ops() declares (see facts_of) for a layer whose
|
||||
attention read needs every row when ``attn_reads_every_row``. It does
|
||||
not depend on the layer's modules, so the stages of a layer that is
|
||||
not built declare it too."""
|
||||
state = cls(
|
||||
expand=None,
|
||||
attn_mix=None,
|
||||
ffn_mix=None,
|
||||
attn_combine=None,
|
||||
ffn_combine=None,
|
||||
attn_reads_every_row=attn_reads_every_row,
|
||||
)
|
||||
return residual_facts(state.residual_ops())
|
||||
|
||||
|
||||
class _AttnReadout:
|
||||
"""The normalized, gated mix of the streams, from streams that already hold
|
||||
|
||||
@@ -19,6 +19,7 @@ from typing import Callable, Optional
|
||||
import torch
|
||||
|
||||
from sglang.kernels.ops.layernorm.mhc import hc_contract, hc_expand
|
||||
from sglang.srt.layers.layer_boundary.facts import ReadFacts, UpdateFacts
|
||||
from sglang.srt.layers.layer_boundary.residual import LayerResidualOps
|
||||
from sglang.srt.runtime_context import get_parallel
|
||||
|
||||
@@ -119,6 +120,19 @@ class MHCState:
|
||||
ffn_update=_FfnUpdate(self),
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def facts() -> LayerResidualOps:
|
||||
"""What residual_ops() declares (see facts_of): the class constants
|
||||
of its reads and updates, the same for every layer whatever
|
||||
parameters it holds, so the stages of a layer that is not built
|
||||
declare them too."""
|
||||
return LayerResidualOps(
|
||||
attn_readout=ReadFacts.of(_AttnReadout),
|
||||
attn_update=UpdateFacts.of(_AttnUpdate),
|
||||
ffn_readout=ReadFacts.of(_FfnReadout),
|
||||
ffn_update=UpdateFacts.of(_FfnUpdate),
|
||||
)
|
||||
|
||||
|
||||
class _AttnReadout:
|
||||
"""hc_pre and the input norm, from streams that already hold the previous
|
||||
|
||||
@@ -87,7 +87,6 @@ from sglang.srt.utils import (
|
||||
is_npu,
|
||||
round_up,
|
||||
)
|
||||
from sglang.srt.utils.common import is_building_neighbour_layer
|
||||
from sglang.srt.utils.custom_op import register_custom_op
|
||||
|
||||
_is_hip = is_hip()
|
||||
@@ -164,8 +163,6 @@ def create_moe_dispatcher(
|
||||
moe_runner_config: MoeRunnerConfig,
|
||||
quant_method: FusedMoEMethodBase,
|
||||
) -> BaseDispatcher:
|
||||
if is_building_neighbour_layer():
|
||||
return StandardDispatcher(moe_runner_config)
|
||||
a2a_backend = get_moe_a2a_backend()
|
||||
if a2a_backend.is_none() and is_npu():
|
||||
return AscendTPDispatcher(moe_runner_config)
|
||||
|
||||
@@ -20,7 +20,6 @@ from sglang.srt.runtime_context import (
|
||||
get_schedule,
|
||||
)
|
||||
from sglang.srt.utils import get_compiler_backend
|
||||
from sglang.srt.utils.common import is_building_neighbour_layer
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from sglang.srt.layers.moe import MoeRunnerConfig
|
||||
@@ -216,7 +215,7 @@ class KTEPWrapperMethod(FusedMoEMethodBase):
|
||||
|
||||
# 2. Initialize KT wrapper for CPU experts
|
||||
# CPU experts: num_gpu_experts to num_experts-1
|
||||
if self.tp_rank == 0 and not is_building_neighbour_layer():
|
||||
if self.tp_rank == 0:
|
||||
self.wrapper = KTMoEWrapper(
|
||||
layer_idx=self.kt_config.layer_idx,
|
||||
num_experts=num_experts,
|
||||
|
||||
@@ -266,14 +266,18 @@ class ApertusDecoderLayer(nn.Module):
|
||||
self.feedforward_layernorm = RMSNorm(
|
||||
config.hidden_size, eps=config.rms_norm_eps
|
||||
)
|
||||
attn, ffn = self.stage_facts(layer_id)
|
||||
self.attn_boundary, self.ffn_boundary = append_stages(
|
||||
(declare_attn(), self.attention_layernorm),
|
||||
(
|
||||
declare_ffn(sparse=False, next_layer_sparse=False),
|
||||
self.feedforward_layernorm,
|
||||
),
|
||||
(attn, self.attention_layernorm),
|
||||
(ffn, self.feedforward_layernorm),
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def stage_facts(layer_id: int):
|
||||
"""The stages every Apertus layer declares, an attention and a dense
|
||||
FFN: the model's shared declaration function (see make_layers)."""
|
||||
return (declare_attn(), declare_ffn(sparse=False, next_layer_sparse=False))
|
||||
|
||||
def forward(
|
||||
self,
|
||||
positions: torch.Tensor,
|
||||
@@ -329,6 +333,7 @@ class ApertusModel(nn.Module):
|
||||
config=config, quant_config=quant_config, layer_id=idx, prefix=prefix
|
||||
),
|
||||
prefix="model.layers",
|
||||
stage_facts=ApertusDecoderLayer.stage_facts,
|
||||
)
|
||||
|
||||
if self.pp_group.is_last_rank:
|
||||
|
||||
@@ -242,14 +242,18 @@ class ArceeDecoderLayer(nn.Module):
|
||||
self.post_attention_layernorm = RMSNorm(
|
||||
config.hidden_size, eps=config.rms_norm_eps
|
||||
)
|
||||
attn, ffn = self.stage_facts(layer_id)
|
||||
self.attn_boundary, self.ffn_boundary = append_stages(
|
||||
(declare_attn(), self.input_layernorm),
|
||||
(
|
||||
declare_ffn(sparse=False, next_layer_sparse=False),
|
||||
self.post_attention_layernorm,
|
||||
),
|
||||
(attn, self.input_layernorm),
|
||||
(ffn, self.post_attention_layernorm),
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def stage_facts(layer_id: int):
|
||||
"""The stages every Arcee layer declares, an attention and a dense FFN:
|
||||
the model's shared declaration function (see make_layers)."""
|
||||
return (declare_attn(), declare_ffn(sparse=False, next_layer_sparse=False))
|
||||
|
||||
def forward(
|
||||
self,
|
||||
positions: torch.Tensor,
|
||||
@@ -303,6 +307,7 @@ class ArceeModel(nn.Module):
|
||||
config=config, quant_config=quant_config, layer_id=idx, prefix=prefix
|
||||
),
|
||||
prefix="model.layers",
|
||||
stage_facts=ArceeDecoderLayer.stage_facts,
|
||||
)
|
||||
|
||||
if self.pp_group.is_last_rank:
|
||||
|
||||
@@ -655,12 +655,8 @@ class BailingMoEBlock(nn.Module):
|
||||
)
|
||||
self.layer_id = layer_id
|
||||
|
||||
self.is_layer_sparse = self._is_layer_sparse(
|
||||
config, layer_id=layer_id, is_nextn=is_nextn
|
||||
)
|
||||
is_next_layer_sparse = self._is_layer_sparse(
|
||||
config, layer_id=layer_id + 1, is_nextn=False
|
||||
)
|
||||
attn, ffn = self.stage_facts(config, layer_id, is_nextn=is_nextn)
|
||||
self.is_layer_sparse = ffn.sparse
|
||||
|
||||
if self.is_layer_sparse:
|
||||
self.mlp = BailingMoESparseMoeBlock(
|
||||
@@ -684,18 +680,31 @@ class BailingMoEBlock(nn.Module):
|
||||
self.post_attention_layernorm = RMSNorm(hidden_size, eps=config.rms_norm_eps)
|
||||
|
||||
self.attn_boundary, self.ffn_boundary = append_stages(
|
||||
(declare_attn(), self.input_layernorm),
|
||||
(
|
||||
declare_ffn(
|
||||
sparse=self.is_layer_sparse,
|
||||
next_layer_sparse=is_next_layer_sparse,
|
||||
(attn, self.input_layernorm),
|
||||
(ffn, self.post_attention_layernorm),
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def stage_facts(
|
||||
cls, config: PretrainedConfig, layer_id: int, is_nextn: bool = False
|
||||
):
|
||||
"""The stages the layer at ``layer_id`` declares, from the config: the
|
||||
model's shared declaration function (see make_layers)."""
|
||||
return (
|
||||
declare_attn(),
|
||||
declare_ffn(
|
||||
sparse=cls._is_layer_sparse(
|
||||
config, layer_id=layer_id, is_nextn=is_nextn
|
||||
),
|
||||
next_layer_sparse=cls._is_layer_sparse(
|
||||
config, layer_id=layer_id + 1, is_nextn=False
|
||||
),
|
||||
self.post_attention_layernorm,
|
||||
),
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _is_layer_sparse(
|
||||
self, config: PretrainedConfig, layer_id: int, is_nextn: bool
|
||||
config: PretrainedConfig, layer_id: int, is_nextn: bool
|
||||
) -> bool:
|
||||
return is_nextn or (
|
||||
config.num_experts is not None and layer_id >= config.first_k_dense_replace
|
||||
@@ -774,6 +783,7 @@ class BailingMoEModel(nn.Module):
|
||||
alt_stream=alt_stream,
|
||||
),
|
||||
prefix=add_prefix("layers", prefix),
|
||||
stage_facts=lambda idx: BailingMoEBlock.stage_facts(config, idx),
|
||||
)
|
||||
if self.pp_group.is_last_rank:
|
||||
self.norm = RMSNorm(self.embed_dim, eps=config.rms_norm_eps)
|
||||
|
||||
@@ -779,8 +779,7 @@ class BailingMoELinearDecoderLayer(nn.Module):
|
||||
|
||||
self.expert_num = config.num_experts
|
||||
self.hidden_size = config.hidden_size
|
||||
is_moe_layer = self._is_layer_sparse(config, self.layer_id, is_nextn=is_nextn)
|
||||
is_next_layer_moe_layer = self._is_layer_sparse(config, self.layer_id + 1)
|
||||
attn, ffn = self.stage_facts(config, self.layer_id, is_nextn=is_nextn)
|
||||
if self.expert_num == 1:
|
||||
self.mlp = BailingMLP(
|
||||
hidden_size=self.hidden_size,
|
||||
@@ -818,22 +817,27 @@ class BailingMoELinearDecoderLayer(nn.Module):
|
||||
else None
|
||||
)
|
||||
self.attn_boundary, self.ffn_boundary = append_stages(
|
||||
(
|
||||
declare_attn(),
|
||||
self.input_layernorm,
|
||||
{"qkv_latent_func": qkv_latent_func},
|
||||
),
|
||||
(
|
||||
declare_ffn(
|
||||
sparse=is_moe_layer,
|
||||
next_layer_sparse=is_next_layer_moe_layer,
|
||||
),
|
||||
self.post_attention_layernorm,
|
||||
(attn, self.input_layernorm, {"qkv_latent_func": qkv_latent_func}),
|
||||
(ffn, self.post_attention_layernorm),
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def stage_facts(
|
||||
cls, config: PretrainedConfig, layer_id: int, is_nextn: bool = False
|
||||
):
|
||||
"""The stages the layer at ``layer_id`` declares, from the config: the
|
||||
model's shared declaration function (see make_layers)."""
|
||||
return (
|
||||
declare_attn(),
|
||||
declare_ffn(
|
||||
sparse=cls._is_layer_sparse(config, layer_id, is_nextn=is_nextn),
|
||||
next_layer_sparse=cls._is_layer_sparse(config, layer_id + 1),
|
||||
),
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _is_layer_sparse(
|
||||
self, config: PretrainedConfig, layer_id: int, is_nextn: bool = False
|
||||
config: PretrainedConfig, layer_id: int, is_nextn: bool = False
|
||||
) -> bool:
|
||||
return is_nextn or (
|
||||
config.num_experts is not None and layer_id >= config.first_k_dense_replace
|
||||
@@ -945,6 +949,9 @@ class BailingMoELinearModel(nn.Module):
|
||||
self.num_layers,
|
||||
layer_fn,
|
||||
prefix=f"{prefix}.layers",
|
||||
stage_facts=lambda idx: BailingMoELinearDecoderLayer.stage_facts(
|
||||
config, idx
|
||||
),
|
||||
)
|
||||
|
||||
norm_kwargs = {}
|
||||
|
||||
@@ -1073,14 +1073,8 @@ class BailingMoELinearDecoderLayer(nn.Module):
|
||||
|
||||
self.expert_num = config.num_experts
|
||||
self.hidden_size = config.hidden_size
|
||||
is_moe_layer = is_nextn or (
|
||||
not (self.expert_num == 1)
|
||||
and (self.layer_id >= config.first_k_dense_replace)
|
||||
)
|
||||
self.is_layer_sparse = is_moe_layer
|
||||
is_next_layer_sparse = not (self.expert_num == 1) and (
|
||||
self.layer_id + 1 >= config.first_k_dense_replace
|
||||
)
|
||||
attn, ffn = self.stage_facts(config, self.layer_id, is_nextn=is_nextn)
|
||||
self.is_layer_sparse = ffn.sparse
|
||||
mlp_parallel_group = "replicated" if is_dense_ffn_fully_dp() else "tp"
|
||||
|
||||
if self.expert_num == 1:
|
||||
@@ -1119,7 +1113,7 @@ class BailingMoELinearDecoderLayer(nn.Module):
|
||||
|
||||
self.attn_boundary, self.ffn_boundary = append_stages(
|
||||
(
|
||||
declare_attn(),
|
||||
attn,
|
||||
self.input_layernorm,
|
||||
{
|
||||
"qkv_latent_func": self.attention.prepare_qkv_latent
|
||||
@@ -1127,12 +1121,24 @@ class BailingMoELinearDecoderLayer(nn.Module):
|
||||
else None
|
||||
},
|
||||
),
|
||||
(
|
||||
declare_ffn(
|
||||
sparse=is_moe_layer,
|
||||
next_layer_sparse=is_next_layer_sparse,
|
||||
),
|
||||
self.post_attention_layernorm,
|
||||
(ffn, self.post_attention_layernorm),
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def stage_facts(config: PretrainedConfig, layer_id: int, is_nextn: bool = False):
|
||||
"""The stages the layer at ``layer_id`` declares, from the config: the
|
||||
model's shared declaration function (see make_layers)."""
|
||||
is_moe_layer = is_nextn or (
|
||||
not (config.num_experts == 1) and (layer_id >= config.first_k_dense_replace)
|
||||
)
|
||||
is_next_layer_sparse = not (config.num_experts == 1) and (
|
||||
layer_id + 1 >= config.first_k_dense_replace
|
||||
)
|
||||
return (
|
||||
declare_attn(),
|
||||
declare_ffn(
|
||||
sparse=is_moe_layer,
|
||||
next_layer_sparse=is_next_layer_sparse,
|
||||
),
|
||||
)
|
||||
|
||||
@@ -1265,6 +1271,9 @@ class BailingMoELinearModel(nn.Module):
|
||||
self.num_layers,
|
||||
layer_fn,
|
||||
prefix=f"{prefix}.layers",
|
||||
stage_facts=lambda idx: BailingMoELinearDecoderLayer.stage_facts(
|
||||
config, idx
|
||||
),
|
||||
)
|
||||
|
||||
linear_layer_nums = sum(
|
||||
|
||||
@@ -2590,10 +2590,9 @@ class DeepseekV2DecoderLayer(nn.Module):
|
||||
"SGLANG_USE_AG_AFTER_QLORA only supports the model with q_lora_rank"
|
||||
)
|
||||
|
||||
self.is_layer_sparse = self._is_layer_sparse(layer_id, is_nextn=is_nextn)
|
||||
is_next_layer_sparse = self._is_layer_sparse(layer_id + 1, is_nextn=False)
|
||||
|
||||
self._stage_next_sparse = is_next_layer_sparse
|
||||
self.is_layer_sparse = self._is_layer_sparse(
|
||||
config, layer_id, is_nextn=is_nextn
|
||||
)
|
||||
|
||||
if self.is_layer_sparse:
|
||||
self.mlp = DeepseekV2MoE(
|
||||
@@ -2634,18 +2633,42 @@ class DeepseekV2DecoderLayer(nn.Module):
|
||||
qkv_latent_func=self.self_attn.prepare_qkv_latent,
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def stage_facts(
|
||||
cls,
|
||||
config: PretrainedConfig,
|
||||
layer_id: int,
|
||||
*,
|
||||
is_nextn: bool = False,
|
||||
attn_output=None,
|
||||
output=None,
|
||||
):
|
||||
"""The stages a layer of this class declares, from the config alone:
|
||||
the model's shared declaration function, which the layer declares
|
||||
with too (see make_layers). ``attn_output`` and ``output``: the
|
||||
attention's and the FFN's output transforms, if any."""
|
||||
return (
|
||||
declare_attn(output_transform=attn_output),
|
||||
declare_ffn(
|
||||
sparse=cls._is_layer_sparse(config, layer_id, is_nextn=is_nextn),
|
||||
next_layer_sparse=cls._is_layer_sparse(
|
||||
config, layer_id + 1, is_nextn=False
|
||||
),
|
||||
output_transform=output,
|
||||
),
|
||||
)
|
||||
|
||||
def _build_stages(
|
||||
self,
|
||||
*,
|
||||
input_layernorm: nn.Module,
|
||||
post_attention_layernorm: nn.Module,
|
||||
qkv_latent_func: Optional[Callable],
|
||||
output=None,
|
||||
attn_output=None,
|
||||
**modules,
|
||||
):
|
||||
"""The stage boundaries for this layer's norms; they choose their
|
||||
steps from them at construction. ``attn_output`` and ``output``: the
|
||||
attention's and the FFN's output transforms, if any."""
|
||||
steps from them at construction. ``modules``: what this class's
|
||||
stage_facts binds the stages' output transforms to, if anything."""
|
||||
fusions = None
|
||||
if (
|
||||
not get_parallel().enable_prefill_cp
|
||||
@@ -2656,21 +2679,16 @@ class DeepseekV2DecoderLayer(nn.Module):
|
||||
from sglang.srt.layers.layer_boundary.fusions.cutedsl import CuteDSLFusion
|
||||
|
||||
fusions = CuteDSLFusion()
|
||||
attn, ffn = self.stage_facts(
|
||||
self.config, self.layer_id, is_nextn=self.is_nextn, **modules
|
||||
)
|
||||
attn_boundary, ffn_boundary = append_stages(
|
||||
(
|
||||
declare_attn(output_transform=attn_output),
|
||||
attn,
|
||||
input_layernorm,
|
||||
{"qkv_latent_func": qkv_latent_func, "fusions": fusions},
|
||||
),
|
||||
(
|
||||
declare_ffn(
|
||||
sparse=self.is_layer_sparse,
|
||||
next_layer_sparse=self._stage_next_sparse,
|
||||
output_transform=output,
|
||||
),
|
||||
post_attention_layernorm,
|
||||
{"fusions": fusions},
|
||||
),
|
||||
(ffn, post_attention_layernorm, {"fusions": fusions}),
|
||||
)
|
||||
return attn_boundary, ffn_boundary
|
||||
|
||||
@@ -2706,11 +2724,14 @@ class DeepseekV2DecoderLayer(nn.Module):
|
||||
self._gfx95_quant_format = fmt
|
||||
return fmt
|
||||
|
||||
def _is_layer_sparse(self, layer_id: int, is_nextn: bool) -> bool:
|
||||
@staticmethod
|
||||
def _is_layer_sparse(
|
||||
config: PretrainedConfig, layer_id: int, is_nextn: bool
|
||||
) -> bool:
|
||||
return is_nextn or (
|
||||
self.config.n_routed_experts is not None
|
||||
and layer_id >= self.config.first_k_dense_replace
|
||||
and layer_id % self.config.moe_layer_freq == 0
|
||||
config.n_routed_experts is not None
|
||||
and layer_id >= config.first_k_dense_replace
|
||||
and layer_id % config.moe_layer_freq == 0
|
||||
)
|
||||
|
||||
def forward(
|
||||
@@ -2916,6 +2937,7 @@ class DeepseekV2Model(nn.Module):
|
||||
else []
|
||||
),
|
||||
),
|
||||
stage_facts=lambda idx: DeepseekV2DecoderLayer.stage_facts(config, idx),
|
||||
)
|
||||
|
||||
local_layer_ids = list(range(self.start_layer, self.end_layer))
|
||||
|
||||
@@ -1377,8 +1377,8 @@ class Dots3DecoderLayer(nn.Module):
|
||||
prefix=add_prefix("self_attn", prefix),
|
||||
alt_stream=alt_stream,
|
||||
)
|
||||
self.is_layer_sparse = self._is_layer_sparse(layer_id, is_nextn=is_nextn)
|
||||
is_next_layer_sparse = self._is_layer_sparse(layer_id + 1, is_nextn=False)
|
||||
attn, ffn = self.stage_facts(config, layer_id, is_nextn=is_nextn)
|
||||
self.is_layer_sparse = ffn.sparse
|
||||
|
||||
if self.is_layer_sparse:
|
||||
self.mlp = Dots3MoE(
|
||||
@@ -1407,24 +1407,37 @@ class Dots3DecoderLayer(nn.Module):
|
||||
)
|
||||
|
||||
self.attn_boundary, self.ffn_boundary = append_stages(
|
||||
(declare_attn(), self.input_layernorm),
|
||||
(
|
||||
declare_ffn(
|
||||
sparse=self.is_layer_sparse,
|
||||
next_layer_sparse=is_next_layer_sparse,
|
||||
(attn, self.input_layernorm),
|
||||
(ffn, self.post_attention_layernorm),
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def stage_facts(
|
||||
cls, config: PretrainedConfig, layer_id: int, is_nextn: bool = False
|
||||
):
|
||||
"""The stages the layer at ``layer_id`` declares, from the config: the
|
||||
model's shared declaration function (see make_layers)."""
|
||||
return (
|
||||
declare_attn(),
|
||||
declare_ffn(
|
||||
sparse=cls._is_layer_sparse(config, layer_id, is_nextn=is_nextn),
|
||||
next_layer_sparse=cls._is_layer_sparse(
|
||||
config, layer_id + 1, is_nextn=False
|
||||
),
|
||||
self.post_attention_layernorm,
|
||||
),
|
||||
)
|
||||
|
||||
def _is_layer_sparse(self, layer_id: int, is_nextn: bool) -> bool:
|
||||
@staticmethod
|
||||
def _is_layer_sparse(
|
||||
config: PretrainedConfig, layer_id: int, is_nextn: bool
|
||||
) -> bool:
|
||||
# The MTP block uses a dense MLP.
|
||||
if is_nextn:
|
||||
return False
|
||||
return (
|
||||
self.config.n_routed_experts is not None
|
||||
and layer_id >= self.config.first_k_dense_replace
|
||||
and layer_id % self.config.moe_layer_freq == 0
|
||||
config.n_routed_experts is not None
|
||||
and layer_id >= config.first_k_dense_replace
|
||||
and layer_id % config.moe_layer_freq == 0
|
||||
)
|
||||
|
||||
def forward(
|
||||
@@ -1485,6 +1498,7 @@ class Dots3Model(nn.Module):
|
||||
alt_stream=self.alt_stream,
|
||||
),
|
||||
prefix=add_prefix("layers", prefix),
|
||||
stage_facts=lambda idx: Dots3DecoderLayer.stage_facts(config, idx),
|
||||
)
|
||||
if self.pp_group.is_last_rank:
|
||||
self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
|
||||
|
||||
@@ -412,8 +412,8 @@ class Ernie4_5_VLMoeDecoderLayer(nn.Module):
|
||||
|
||||
# MoE
|
||||
# MLP
|
||||
sparse = _is_moe_layer(config, layer_id)
|
||||
if sparse:
|
||||
attn, ffn = self.stage_facts(config, layer_id)
|
||||
if ffn.sparse:
|
||||
self.mlp = Ernie4_5_VLMoeMoE(
|
||||
config=config,
|
||||
layer_id=layer_id,
|
||||
@@ -435,15 +435,22 @@ class Ernie4_5_VLMoeDecoderLayer(nn.Module):
|
||||
self.post_attention_layernorm = RMSNorm(
|
||||
config.hidden_size, eps=config.rms_norm_eps
|
||||
)
|
||||
self.attn_boundary, self.ffn_boundary = append_stages(
|
||||
(attn, self.input_layernorm),
|
||||
(ffn, self.post_attention_layernorm),
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def stage_facts(config: PretrainedConfig, layer_id: int):
|
||||
"""The stages the layer at ``layer_id`` declares, from the config: the
|
||||
model's shared declaration function (see make_layers)."""
|
||||
sparse = _is_moe_layer(config, layer_id)
|
||||
next_sparse = layer_id + 1 < config.num_hidden_layers and _is_moe_layer(
|
||||
config, layer_id + 1
|
||||
)
|
||||
self.attn_boundary, self.ffn_boundary = append_stages(
|
||||
(declare_attn(), self.input_layernorm),
|
||||
(
|
||||
declare_ffn(sparse=sparse, next_layer_sparse=next_sparse),
|
||||
self.post_attention_layernorm,
|
||||
),
|
||||
return (
|
||||
declare_attn(),
|
||||
declare_ffn(sparse=sparse, next_layer_sparse=next_sparse),
|
||||
)
|
||||
|
||||
def forward(
|
||||
@@ -502,6 +509,7 @@ class Ernie4_5_VLMoeModel(nn.Module):
|
||||
prefix=prefix,
|
||||
),
|
||||
prefix=add_prefix("layers", prefix),
|
||||
stage_facts=lambda idx: Ernie4_5_VLMoeDecoderLayer.stage_facts(config, idx),
|
||||
)
|
||||
if self.pp_group.is_last_rank:
|
||||
self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
|
||||
|
||||
@@ -267,25 +267,38 @@ class Exaone4DecoderLayer(nn.Module):
|
||||
self.post_feedforward_layernorm = RMSNorm(
|
||||
self.hidden_size, eps=config.rms_norm_eps
|
||||
)
|
||||
attn, ffn = self.stage_facts(
|
||||
config,
|
||||
layer_id,
|
||||
post_attention_layernorm=self.post_attention_layernorm,
|
||||
post_feedforward_layernorm=self.post_feedforward_layernorm,
|
||||
)
|
||||
self.attn_boundary, self.ffn_boundary = append_stages((attn, None), (ffn, None))
|
||||
|
||||
@staticmethod
|
||||
def stage_facts(
|
||||
config: Exaone4Config,
|
||||
layer_id: int,
|
||||
*,
|
||||
post_attention_layernorm=None,
|
||||
post_feedforward_layernorm=None,
|
||||
):
|
||||
"""The stages an EXAONE 4.0 layer declares, from the config alone: the
|
||||
model's shared declaration function, which the layer declares with
|
||||
too (see make_layers). The output norms are what the updates add."""
|
||||
# Post-LN: each stage reads the residual as it is, and its output is
|
||||
# normalized before it is added. The layer writes the FFN's itself.
|
||||
ffn_update = PostNormAdd(self.post_feedforward_layernorm, applied_at_exit=True)
|
||||
self.attn_boundary, self.ffn_boundary = append_stages(
|
||||
(
|
||||
declare_attn(
|
||||
read=PLAIN_READOUT,
|
||||
update=PostNormAdd(self.post_attention_layernorm),
|
||||
),
|
||||
None,
|
||||
ffn_update = PostNormAdd(post_feedforward_layernorm, applied_at_exit=True)
|
||||
return (
|
||||
declare_attn(
|
||||
read=PLAIN_READOUT,
|
||||
update=PostNormAdd(post_attention_layernorm),
|
||||
),
|
||||
(
|
||||
declare_ffn(
|
||||
sparse=False,
|
||||
next_layer_sparse=False,
|
||||
read=PLAIN_READOUT,
|
||||
update=ffn_update,
|
||||
),
|
||||
None,
|
||||
declare_ffn(
|
||||
sparse=False,
|
||||
next_layer_sparse=False,
|
||||
read=PLAIN_READOUT,
|
||||
update=ffn_update,
|
||||
),
|
||||
)
|
||||
|
||||
@@ -341,6 +354,7 @@ class Exaone4Model(nn.Module):
|
||||
prefix=prefix,
|
||||
),
|
||||
prefix=add_prefix("layers", prefix),
|
||||
stage_facts=lambda idx: Exaone4DecoderLayer.stage_facts(config, idx),
|
||||
)
|
||||
if self.pp_group.is_last_rank:
|
||||
self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
|
||||
|
||||
@@ -464,7 +464,8 @@ class ExaoneMoEDecoderLayer(nn.Module):
|
||||
prefix=add_prefix("self_attn", prefix),
|
||||
)
|
||||
|
||||
if config.is_moe_layer[layer_id]:
|
||||
attn, ffn = self.stage_facts(config, layer_id)
|
||||
if ffn.sparse:
|
||||
self.mlp = ExaoneMoESparseMoEBlock(
|
||||
layer_id=layer_id,
|
||||
config=config,
|
||||
@@ -485,17 +486,23 @@ class ExaoneMoEDecoderLayer(nn.Module):
|
||||
self.post_attention_layernorm = RMSNorm(
|
||||
config.hidden_size, eps=config.rms_norm_eps
|
||||
)
|
||||
self.attn_boundary, self.ffn_boundary = append_stages(
|
||||
(attn, self.input_layernorm),
|
||||
(ffn, self.post_attention_layernorm),
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def stage_facts(config: PretrainedConfig, layer_id: int):
|
||||
"""The stages the layer at ``layer_id`` declares, from the config: the
|
||||
model's shared declaration function (see make_layers)."""
|
||||
is_moe_layer = config.is_moe_layer
|
||||
num_layers = config.num_hidden_layers
|
||||
self.attn_boundary, self.ffn_boundary = append_stages(
|
||||
(declare_attn(), self.input_layernorm),
|
||||
(
|
||||
declare_ffn(
|
||||
sparse=is_moe_layer[layer_id],
|
||||
next_layer_sparse=layer_id + 1 < num_layers
|
||||
and is_moe_layer[layer_id + 1],
|
||||
),
|
||||
self.post_attention_layernorm,
|
||||
return (
|
||||
declare_attn(),
|
||||
declare_ffn(
|
||||
sparse=is_moe_layer[layer_id],
|
||||
next_layer_sparse=layer_id + 1 < num_layers
|
||||
and is_moe_layer[layer_id + 1],
|
||||
),
|
||||
)
|
||||
|
||||
@@ -559,6 +566,7 @@ class ExaoneMoEModel(nn.Module):
|
||||
alt_stream=alt_stream,
|
||||
),
|
||||
prefix=add_prefix("layers", prefix),
|
||||
stage_facts=lambda idx: ExaoneMoEDecoderLayer.stage_facts(config, idx),
|
||||
)
|
||||
|
||||
if self.pp_group.is_last_rank:
|
||||
|
||||
@@ -636,7 +636,7 @@ class Gemma4DecoderLayer(nn.Module):
|
||||
self.post_per_layer_input_norm = None
|
||||
|
||||
# Parallel MoE
|
||||
self.enable_moe_block = getattr(config, "enable_moe_block", False)
|
||||
self.enable_moe_block = self._enable_moe_block(config)
|
||||
if self.enable_moe_block:
|
||||
self.router = Gemma4Router(
|
||||
config,
|
||||
@@ -671,20 +671,37 @@ class Gemma4DecoderLayer(nn.Module):
|
||||
self.register_buffer("layer_scalar", torch.ones(1), persistent=True)
|
||||
self.has_ple = self.hidden_size_per_layer_input > 0
|
||||
self.prefix = prefix
|
||||
attn, ffn = self.stage_facts(
|
||||
config, layer_id, post_attention_layernorm=self.post_attention_layernorm
|
||||
)
|
||||
self.attn_boundary, self.ffn_boundary = append_stages(
|
||||
(
|
||||
declare_attn(
|
||||
output_transform=OutputTransform(self.post_attention_layernorm)
|
||||
),
|
||||
self.input_layernorm,
|
||||
),
|
||||
(
|
||||
declare_ffn(
|
||||
sparse=self.enable_moe_block,
|
||||
next_layer_sparse=self.enable_moe_block,
|
||||
update=REPLACE_AT_EXIT,
|
||||
),
|
||||
self.pre_feedforward_layernorm,
|
||||
(attn, self.input_layernorm),
|
||||
(ffn, self.pre_feedforward_layernorm),
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _enable_moe_block(config: PretrainedConfig) -> bool:
|
||||
return getattr(config, "enable_moe_block", False)
|
||||
|
||||
@classmethod
|
||||
def stage_facts(
|
||||
cls,
|
||||
config: PretrainedConfig,
|
||||
layer_id: int,
|
||||
*,
|
||||
post_attention_layernorm=None,
|
||||
):
|
||||
"""The stages a Gemma 4 layer declares, from the config alone: the
|
||||
model's shared declaration function, which the layer declares with
|
||||
too (see make_layers). The attention's output norm is its output
|
||||
transform."""
|
||||
enable_moe_block = cls._enable_moe_block(config)
|
||||
return (
|
||||
declare_attn(output_transform=OutputTransform(post_attention_layernorm)),
|
||||
declare_ffn(
|
||||
sparse=enable_moe_block,
|
||||
next_layer_sparse=enable_moe_block,
|
||||
update=REPLACE_AT_EXIT,
|
||||
),
|
||||
)
|
||||
|
||||
@@ -919,6 +936,7 @@ class Gemma4TextModel(PreTrainedModel):
|
||||
prefix=prefix,
|
||||
),
|
||||
prefix=add_prefix("layers", prefix),
|
||||
stage_facts=lambda idx: Gemma4DecoderLayer.stage_facts(config, idx),
|
||||
)
|
||||
|
||||
if self.pp_group.is_last_rank:
|
||||
|
||||
@@ -410,9 +410,7 @@ class GigaChat35DecoderLayer(deepseek_v2.DeepseekV2DecoderLayer):
|
||||
if swiglu_limit > 0 and runner_config is not None:
|
||||
runner_config.gemm1_clamp_limit = swiglu_limit
|
||||
|
||||
layernorm_type = getattr(config, "layernorm_type", "pre")
|
||||
self._use_pre = layernorm_type in ("pre", "pre_post")
|
||||
self._use_post = layernorm_type in ("post", "pre_post")
|
||||
self._use_pre, self._use_post = self._norm_placement(config)
|
||||
|
||||
self.input_layernorm = build_norm(config, config.hidden_size)
|
||||
self.post_attention_layernorm = build_norm(config, config.hidden_size)
|
||||
@@ -434,25 +432,55 @@ class GigaChat35DecoderLayer(deepseek_v2.DeepseekV2DecoderLayer):
|
||||
input_layernorm=attn_prepare_layernorm,
|
||||
post_attention_layernorm=mlp_prepare_layernorm,
|
||||
qkv_latent_func=qkv_latent_func,
|
||||
attn_output=(
|
||||
OutputTransform(self.post_self_attn_layernorm)
|
||||
if self.post_self_attn_layernorm is not None
|
||||
else None
|
||||
),
|
||||
output=(
|
||||
OutputTransform(self.post_feedforward_layernorm)
|
||||
if self.post_feedforward_layernorm is not None
|
||||
else None
|
||||
),
|
||||
post_self_attn_layernorm=self.post_self_attn_layernorm,
|
||||
post_feedforward_layernorm=self.post_feedforward_layernorm,
|
||||
)
|
||||
|
||||
def _is_layer_sparse(self, layer_id: int, is_nextn: bool) -> bool:
|
||||
if is_nextn:
|
||||
return bool(getattr(self.config, "nextn_is_sparse", False))
|
||||
@staticmethod
|
||||
def _norm_placement(config: GigaChat35Config):
|
||||
"""Whether the layer norms each sublayer's input, and its output."""
|
||||
layernorm_type = getattr(config, "layernorm_type", "pre")
|
||||
return (
|
||||
self.config.n_routed_experts is not None
|
||||
and layer_id >= self.config.first_k_dense_replace
|
||||
and layer_id % self.config.moe_layer_freq == 0
|
||||
layernorm_type in ("pre", "pre_post"),
|
||||
layernorm_type in ("post", "pre_post"),
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def stage_facts(
|
||||
cls,
|
||||
config: GigaChat35Config,
|
||||
layer_id: int,
|
||||
*,
|
||||
is_nextn: bool = False,
|
||||
post_self_attn_layernorm=None,
|
||||
post_feedforward_layernorm=None,
|
||||
):
|
||||
"""The stages a GigaChat 3.5 layer declares, from the config alone:
|
||||
the model's shared declaration function, which the layer declares
|
||||
with too (see make_layers). The output norms, which the layer has
|
||||
only when it norms the sublayers' outputs, are the output
|
||||
transforms."""
|
||||
_, use_post = cls._norm_placement(config)
|
||||
return super().stage_facts(
|
||||
config,
|
||||
layer_id,
|
||||
is_nextn=is_nextn,
|
||||
attn_output=(
|
||||
OutputTransform(post_self_attn_layernorm) if use_post else None
|
||||
),
|
||||
output=OutputTransform(post_feedforward_layernorm) if use_post else None,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _is_layer_sparse(
|
||||
config: GigaChat35Config, layer_id: int, is_nextn: bool
|
||||
) -> bool:
|
||||
if is_nextn:
|
||||
return bool(getattr(config, "nextn_is_sparse", False))
|
||||
return (
|
||||
config.n_routed_experts is not None
|
||||
and layer_id >= config.first_k_dense_replace
|
||||
and layer_id % config.moe_layer_freq == 0
|
||||
)
|
||||
|
||||
def forward(
|
||||
@@ -527,6 +555,7 @@ class GigaChat35Model(nn.Module):
|
||||
alt_stream=self.alt_stream,
|
||||
),
|
||||
prefix=add_prefix("layers", prefix),
|
||||
stage_facts=lambda idx: GigaChat35DecoderLayer.stage_facts(config, idx),
|
||||
)
|
||||
|
||||
if self.pp_group.is_last_rank:
|
||||
|
||||
@@ -264,20 +264,34 @@ class Glm4DecoderLayer(nn.Module):
|
||||
config.hidden_size, eps=config.rms_norm_eps
|
||||
)
|
||||
self.post_mlp_layernorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
|
||||
attn, ffn = self.stage_facts(
|
||||
config,
|
||||
layer_id,
|
||||
post_self_attn_layernorm=self.post_self_attn_layernorm,
|
||||
post_mlp_layernorm=self.post_mlp_layernorm,
|
||||
)
|
||||
self.attn_boundary, self.ffn_boundary = append_stages(
|
||||
(
|
||||
declare_attn(
|
||||
output_transform=OutputTransform(self.post_self_attn_layernorm)
|
||||
),
|
||||
self.input_layernorm,
|
||||
),
|
||||
(
|
||||
declare_ffn(
|
||||
sparse=False,
|
||||
next_layer_sparse=False,
|
||||
output_transform=OutputTransform(self.post_mlp_layernorm),
|
||||
),
|
||||
self.post_attention_layernorm,
|
||||
(attn, self.input_layernorm),
|
||||
(ffn, self.post_attention_layernorm),
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def stage_facts(
|
||||
config: Glm4Config,
|
||||
layer_id: int,
|
||||
*,
|
||||
post_self_attn_layernorm=None,
|
||||
post_mlp_layernorm=None,
|
||||
):
|
||||
"""The stages a GLM-4 layer declares, from the config alone: the
|
||||
model's shared declaration function, which the layer declares with
|
||||
too (see make_layers). Its output norms are the output transforms."""
|
||||
return (
|
||||
declare_attn(output_transform=OutputTransform(post_self_attn_layernorm)),
|
||||
declare_ffn(
|
||||
sparse=False,
|
||||
next_layer_sparse=False,
|
||||
output_transform=OutputTransform(post_mlp_layernorm),
|
||||
),
|
||||
)
|
||||
|
||||
@@ -344,6 +358,7 @@ class Glm4Model(nn.Module):
|
||||
alt_stream=alt_stream,
|
||||
),
|
||||
prefix=add_prefix("layers", prefix),
|
||||
stage_facts=lambda idx: decoder_layer_type.stage_facts(config, idx),
|
||||
)
|
||||
if self.pp_group.is_last_rank:
|
||||
self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
|
||||
|
||||
@@ -734,8 +734,8 @@ class Glm4MoeDecoderLayer(nn.Module):
|
||||
alt_stream=alt_stream,
|
||||
)
|
||||
|
||||
self.is_layer_sparse = self._is_layer_sparse(layer_id, is_nextn=is_nextn)
|
||||
is_next_layer_sparse = self._is_layer_sparse(layer_id + 1, is_nextn=False)
|
||||
attn, ffn = self.stage_facts(config, layer_id, is_nextn=is_nextn)
|
||||
self.is_layer_sparse = ffn.sparse
|
||||
|
||||
if self.is_layer_sparse:
|
||||
self.mlp = Glm4MoeSparseMoeBlock(
|
||||
@@ -763,14 +763,8 @@ class Glm4MoeDecoderLayer(nn.Module):
|
||||
)
|
||||
|
||||
self.attn_boundary, self.ffn_boundary = append_stages(
|
||||
(declare_attn(), self.input_layernorm),
|
||||
(
|
||||
declare_ffn(
|
||||
sparse=self.is_layer_sparse,
|
||||
next_layer_sparse=is_next_layer_sparse,
|
||||
),
|
||||
self.post_attention_layernorm,
|
||||
),
|
||||
(attn, self.input_layernorm),
|
||||
(ffn, self.post_attention_layernorm),
|
||||
)
|
||||
|
||||
# Detect if QKV uses aiter FP8 per-token quant so we can fuse
|
||||
@@ -818,10 +812,29 @@ class Glm4MoeDecoderLayer(nn.Module):
|
||||
self.self_attn.qkv_proj, "attn"
|
||||
)
|
||||
|
||||
def _is_layer_sparse(self, layer_id: int, is_nextn: bool) -> bool:
|
||||
@classmethod
|
||||
def stage_facts(
|
||||
cls, config: PretrainedConfig, layer_id: int, is_nextn: bool = False
|
||||
):
|
||||
"""The stages the layer at ``layer_id`` declares, from the config: the
|
||||
model's shared declaration function (see make_layers)."""
|
||||
return (
|
||||
declare_attn(),
|
||||
declare_ffn(
|
||||
sparse=cls._is_layer_sparse(config, layer_id, is_nextn=is_nextn),
|
||||
next_layer_sparse=cls._is_layer_sparse(
|
||||
config, layer_id + 1, is_nextn=False
|
||||
),
|
||||
),
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _is_layer_sparse(
|
||||
config: PretrainedConfig, layer_id: int, is_nextn: bool
|
||||
) -> bool:
|
||||
return is_nextn or (
|
||||
self.config.n_routed_experts is not None
|
||||
and layer_id >= self.config.first_k_dense_replace
|
||||
config.n_routed_experts is not None
|
||||
and layer_id >= config.first_k_dense_replace
|
||||
)
|
||||
|
||||
def forward(
|
||||
@@ -893,6 +906,7 @@ class Glm4MoeModel(nn.Module):
|
||||
alt_stream=self.alt_stream,
|
||||
),
|
||||
prefix=add_prefix("layers", prefix),
|
||||
stage_facts=lambda idx: Glm4MoeDecoderLayer.stage_facts(config, idx),
|
||||
)
|
||||
if self.pp_group.is_last_rank:
|
||||
self.norm = RMSNorm(self.embed_dim, eps=config.rms_norm_eps)
|
||||
|
||||
@@ -447,8 +447,8 @@ class Glm4MoeLiteDecoderLayer(nn.Module):
|
||||
prefix=add_prefix("self_attn", prefix),
|
||||
)
|
||||
|
||||
self.is_layer_sparse = self._is_layer_sparse(layer_id, is_nextn=is_nextn)
|
||||
is_next_layer_sparse = self._is_layer_sparse(layer_id + 1, is_nextn=False)
|
||||
attn, ffn = self.stage_facts(config, layer_id, is_nextn=is_nextn)
|
||||
self.is_layer_sparse = ffn.sparse
|
||||
|
||||
if self.is_layer_sparse:
|
||||
self.mlp = Glm4MoeLiteSparseMoeBlock(
|
||||
@@ -480,17 +480,11 @@ class Glm4MoeLiteDecoderLayer(nn.Module):
|
||||
|
||||
self.attn_boundary, self.ffn_boundary = append_stages(
|
||||
(
|
||||
declare_attn(),
|
||||
attn,
|
||||
self.input_layernorm,
|
||||
{"qkv_latent_func": self.self_attn.prepare_qkv_latent},
|
||||
),
|
||||
(
|
||||
declare_ffn(
|
||||
sparse=self.is_layer_sparse,
|
||||
next_layer_sparse=is_next_layer_sparse,
|
||||
),
|
||||
self.post_attention_layernorm,
|
||||
),
|
||||
(ffn, self.post_attention_layernorm),
|
||||
)
|
||||
|
||||
def _detect_gfx95_quant_format(self) -> str:
|
||||
@@ -509,11 +503,30 @@ class Glm4MoeLiteDecoderLayer(nn.Module):
|
||||
return "fp8"
|
||||
return ""
|
||||
|
||||
def _is_layer_sparse(self, layer_id: int, is_nextn: bool) -> bool:
|
||||
@classmethod
|
||||
def stage_facts(
|
||||
cls, config: PretrainedConfig, layer_id: int, is_nextn: bool = False
|
||||
):
|
||||
"""The stages the layer at ``layer_id`` declares, from the config: the
|
||||
model's shared declaration function (see make_layers)."""
|
||||
return (
|
||||
declare_attn(),
|
||||
declare_ffn(
|
||||
sparse=cls._is_layer_sparse(config, layer_id, is_nextn=is_nextn),
|
||||
next_layer_sparse=cls._is_layer_sparse(
|
||||
config, layer_id + 1, is_nextn=False
|
||||
),
|
||||
),
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _is_layer_sparse(
|
||||
config: PretrainedConfig, layer_id: int, is_nextn: bool
|
||||
) -> bool:
|
||||
return is_nextn or (
|
||||
self.config.n_routed_experts is not None
|
||||
and layer_id >= self.config.first_k_dense_replace
|
||||
and layer_id % self.config.moe_layer_freq == 0
|
||||
config.n_routed_experts is not None
|
||||
and layer_id >= config.first_k_dense_replace
|
||||
and layer_id % config.moe_layer_freq == 0
|
||||
)
|
||||
|
||||
def forward(
|
||||
@@ -586,6 +599,7 @@ class Glm4MoeLiteModel(nn.Module):
|
||||
alt_stream=self.alt_stream,
|
||||
),
|
||||
prefix=add_prefix("layers", prefix),
|
||||
stage_facts=lambda idx: Glm4MoeLiteDecoderLayer.stage_facts(config, idx),
|
||||
)
|
||||
if self.pp_group.is_last_rank:
|
||||
self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
|
||||
|
||||
@@ -743,8 +743,9 @@ class Glm5NextDecoderLayer(nn.Module):
|
||||
"SGLANG_USE_AG_AFTER_QLORA only supports the model with q_lora_rank"
|
||||
)
|
||||
|
||||
self.is_layer_sparse = self._is_layer_sparse(layer_id, is_nextn=is_nextn)
|
||||
is_next_layer_sparse = self._is_layer_sparse(layer_id + 1, is_nextn=False)
|
||||
self.is_layer_sparse = self._is_layer_sparse(
|
||||
config, layer_id, is_nextn=is_nextn
|
||||
)
|
||||
|
||||
if self.is_layer_sparse:
|
||||
self.mlp = Glm5NextMoE(
|
||||
@@ -795,9 +796,9 @@ class Glm5NextDecoderLayer(nn.Module):
|
||||
)
|
||||
|
||||
terminal = layer_id == (1 if is_nextn else config.num_hidden_layers) - 1
|
||||
residual = PLAIN_RESIDUAL_OPS
|
||||
mhc = None
|
||||
if self.config.mhc:
|
||||
residual = MHCState(
|
||||
mhc = MHCState(
|
||||
hc_mult=config.hc_mult,
|
||||
hc_attn_pre=self.hc_attn_pre,
|
||||
hc_ffn_pre=self.hc_ffn_pre,
|
||||
@@ -809,15 +810,10 @@ class Glm5NextDecoderLayer(nn.Module):
|
||||
),
|
||||
is_last_layer=terminal,
|
||||
).residual_ops()
|
||||
attn, ffn = self.stage_facts(config, layer_id, is_nextn=is_nextn, mhc=mhc)
|
||||
self.attn_boundary, self.ffn_boundary = append_stages(
|
||||
(
|
||||
declare_attn(
|
||||
read=residual.attn_readout,
|
||||
update=residual.attn_update,
|
||||
tp_group=SumGroup.TP
|
||||
if self.is_linear_attn and get_parallel().enable_cp_tp_group_sharing
|
||||
else SumGroup.ATTN_TP,
|
||||
),
|
||||
attn,
|
||||
self.input_layernorm,
|
||||
{
|
||||
"qkv_latent_func": self.self_attn.prepare_qkv_latent
|
||||
@@ -825,14 +821,37 @@ class Glm5NextDecoderLayer(nn.Module):
|
||||
else None,
|
||||
},
|
||||
),
|
||||
(
|
||||
declare_ffn(
|
||||
sparse=self.is_layer_sparse,
|
||||
next_layer_sparse=is_next_layer_sparse,
|
||||
read=residual.ffn_readout,
|
||||
update=residual.ffn_update,
|
||||
(ffn, self.post_attention_layernorm),
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def stage_facts(cls, config, layer_id, *, is_nextn=False, mhc=None):
|
||||
"""The attention and FFN stages the layer at ``layer_id`` declares:
|
||||
the model's shared declaration function (see make_layers), which the
|
||||
layer declares itself with. The config selects the residual: a plain
|
||||
one, or hyper-connection streams, read and updated by ``mhc``, the
|
||||
layer's own reads and updates of them; without it, the stages
|
||||
declare what those do."""
|
||||
if config.mhc:
|
||||
residual = MHCState.facts() if mhc is None else mhc
|
||||
else:
|
||||
residual = PLAIN_RESIDUAL_OPS
|
||||
return (
|
||||
declare_attn(
|
||||
read=residual.attn_readout,
|
||||
update=residual.attn_update,
|
||||
tp_group=SumGroup.TP
|
||||
if config.is_kda_layer(layer_id)
|
||||
and get_parallel().enable_cp_tp_group_sharing
|
||||
else SumGroup.ATTN_TP,
|
||||
),
|
||||
declare_ffn(
|
||||
sparse=cls._is_layer_sparse(config, layer_id, is_nextn=is_nextn),
|
||||
next_layer_sparse=cls._is_layer_sparse(
|
||||
config, layer_id + 1, is_nextn=False
|
||||
),
|
||||
self.post_attention_layernorm,
|
||||
read=residual.ffn_readout,
|
||||
update=residual.ffn_update,
|
||||
),
|
||||
)
|
||||
|
||||
@@ -924,11 +943,12 @@ class Glm5NextDecoderLayer(nn.Module):
|
||||
hc_mult=self.config.hc_mult,
|
||||
)
|
||||
|
||||
def _is_layer_sparse(self, layer_id: int, is_nextn: bool) -> bool:
|
||||
@staticmethod
|
||||
def _is_layer_sparse(config, layer_id: int, is_nextn: bool) -> bool:
|
||||
return is_nextn or (
|
||||
self.config.n_routed_experts is not None
|
||||
and layer_id >= self.config.first_k_dense_replace
|
||||
and layer_id % self.config.moe_layer_freq == 0
|
||||
config.n_routed_experts is not None
|
||||
and layer_id >= config.first_k_dense_replace
|
||||
and layer_id % config.moe_layer_freq == 0
|
||||
)
|
||||
|
||||
def forward(
|
||||
@@ -1035,6 +1055,7 @@ class Glm5NextModel(nn.Module):
|
||||
alt_stream=self.alt_stream,
|
||||
),
|
||||
prefix=add_prefix("layers", prefix),
|
||||
stage_facts=lambda idx: Glm5NextDecoderLayer.stage_facts(config, idx),
|
||||
)
|
||||
if self.pp_group.is_last_rank:
|
||||
self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
|
||||
|
||||
@@ -557,9 +557,8 @@ class GptOssDecoderLayer(nn.Module):
|
||||
|
||||
self.layer_id = layer_id
|
||||
|
||||
# GptOss all layers are sparse
|
||||
self.is_layer_sparse = True
|
||||
is_next_layer_sparse = True
|
||||
attn, ffn = self.stage_facts(config, layer_id)
|
||||
self.is_layer_sparse = ffn.sparse
|
||||
|
||||
if self.is_layer_sparse:
|
||||
self.mlp = GptOssSparseMoeBlock(
|
||||
@@ -590,13 +589,22 @@ class GptOssDecoderLayer(nn.Module):
|
||||
)
|
||||
|
||||
self.attn_boundary, self.ffn_boundary = append_stages(
|
||||
(declare_attn(), self.input_layernorm),
|
||||
(
|
||||
declare_ffn(
|
||||
sparse=self.is_layer_sparse,
|
||||
next_layer_sparse=is_next_layer_sparse,
|
||||
),
|
||||
self.post_attention_layernorm,
|
||||
(attn, self.input_layernorm),
|
||||
(ffn, self.post_attention_layernorm),
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def stage_facts(config: GptOssConfig, layer_id: int):
|
||||
"""The stages the layer at ``layer_id`` declares: the model's shared
|
||||
declaration function (see make_layers)."""
|
||||
# GptOss all layers are sparse
|
||||
is_layer_sparse = True
|
||||
is_next_layer_sparse = True
|
||||
return (
|
||||
declare_attn(),
|
||||
declare_ffn(
|
||||
sparse=is_layer_sparse,
|
||||
next_layer_sparse=is_next_layer_sparse,
|
||||
),
|
||||
)
|
||||
|
||||
@@ -664,6 +672,7 @@ class GptOssModel(nn.Module):
|
||||
prefix=prefix,
|
||||
),
|
||||
prefix=add_prefix("layers", prefix),
|
||||
stage_facts=lambda idx: decoder_layer_type.stage_facts(config, idx),
|
||||
)
|
||||
if self.pp_group.is_last_rank:
|
||||
self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
from functools import partial
|
||||
from typing import Iterable, Optional
|
||||
|
||||
import torch
|
||||
@@ -40,26 +41,34 @@ from sglang.srt.utils import make_pp_layers
|
||||
from .granitemoe import GraniteMoeMoE, GraniteMoeSharedMLP
|
||||
|
||||
|
||||
def _make_stages(layer):
|
||||
"""Both kinds of layer scale their mixer and FFN outputs by
|
||||
residual_multiplier, add them in the activation dtype and normalize the sum
|
||||
as a separate step."""
|
||||
scale = OutputTransform(layer._scale_output)
|
||||
sparse = layer.block_sparse_moe is not None
|
||||
def _has_moe(config: GraniteMoeHybridConfig) -> bool:
|
||||
return getattr(config, "num_local_experts", 0) > 0
|
||||
|
||||
|
||||
def stage_facts(config: GraniteMoeHybridConfig, layer_idx: int, scale=None):
|
||||
"""The stages a layer of either kind declares, from the config alone: the
|
||||
model's shared declaration function, which the layers declare with too
|
||||
(see make_layers). Both kinds of layer scale their mixer and FFN outputs
|
||||
by residual_multiplier (``scale``), add them in the activation dtype and
|
||||
normalize the sum as a separate step."""
|
||||
scale = OutputTransform(scale)
|
||||
sparse = _has_moe(config)
|
||||
return (
|
||||
declare_attn(read=UNFUSED_NORM_READOUT, output_transform=scale),
|
||||
declare_ffn(
|
||||
sparse=sparse,
|
||||
next_layer_sparse=sparse,
|
||||
read=UNFUSED_NORM_READOUT,
|
||||
output_transform=scale,
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def _make_stages(layer, config: GraniteMoeHybridConfig, layer_idx: int):
|
||||
attn, ffn = stage_facts(config, layer_idx, scale=layer._scale_output)
|
||||
return append_stages(
|
||||
(
|
||||
declare_attn(read=UNFUSED_NORM_READOUT, output_transform=scale),
|
||||
layer.input_layernorm,
|
||||
),
|
||||
(
|
||||
declare_ffn(
|
||||
sparse=sparse,
|
||||
next_layer_sparse=sparse,
|
||||
read=UNFUSED_NORM_READOUT,
|
||||
output_transform=scale,
|
||||
),
|
||||
layer.post_attention_layernorm,
|
||||
),
|
||||
(attn, layer.input_layernorm),
|
||||
(ffn, layer.post_attention_layernorm),
|
||||
)
|
||||
|
||||
|
||||
@@ -91,7 +100,7 @@ class GraniteMoeHybridMambaDecoderLayer(nn.Module):
|
||||
)
|
||||
|
||||
self.block_sparse_moe = None
|
||||
if getattr(config, "num_local_experts", 0) > 0:
|
||||
if _has_moe(config):
|
||||
self.block_sparse_moe = GraniteMoeMoE(
|
||||
num_experts=config.num_local_experts,
|
||||
top_k=config.num_experts_per_tok,
|
||||
@@ -118,7 +127,7 @@ class GraniteMoeHybridMambaDecoderLayer(nn.Module):
|
||||
self.post_attention_layernorm = RMSNorm(
|
||||
config.hidden_size, eps=config.rms_norm_eps
|
||||
)
|
||||
self.attn_boundary, self.ffn_boundary = _make_stages(self)
|
||||
self.attn_boundary, self.ffn_boundary = _make_stages(self, config, layer_idx)
|
||||
|
||||
def _scale_output(self, hidden_states: torch.Tensor) -> torch.Tensor:
|
||||
return hidden_states * self.residual_multiplier
|
||||
@@ -282,7 +291,7 @@ class GraniteMoeHybridAttentionDecoderLayer(nn.Module):
|
||||
)
|
||||
|
||||
self.block_sparse_moe = None
|
||||
if getattr(config, "num_local_experts", 0) > 0:
|
||||
if _has_moe(config):
|
||||
self.block_sparse_moe = GraniteMoeMoE(
|
||||
num_experts=config.num_local_experts,
|
||||
top_k=config.num_experts_per_tok,
|
||||
@@ -309,7 +318,7 @@ class GraniteMoeHybridAttentionDecoderLayer(nn.Module):
|
||||
self.post_attention_layernorm = RMSNorm(
|
||||
config.hidden_size, eps=config.rms_norm_eps
|
||||
)
|
||||
self.attn_boundary, self.ffn_boundary = _make_stages(self)
|
||||
self.attn_boundary, self.ffn_boundary = _make_stages(self, config, layer_idx)
|
||||
|
||||
def _scale_output(self, hidden_states: torch.Tensor) -> torch.Tensor:
|
||||
return hidden_states * self.residual_multiplier
|
||||
@@ -395,6 +404,7 @@ class GraniteMoeHybridModel(nn.Module):
|
||||
config.num_hidden_layers,
|
||||
get_layer,
|
||||
prefix=f"{prefix}.layers",
|
||||
stage_facts=partial(stage_facts, config),
|
||||
)
|
||||
|
||||
if self.pp_group.is_last_rank:
|
||||
|
||||
@@ -461,6 +461,12 @@ class JetNemotronDecoderLayer(nn.Module):
|
||||
config.hidden_size, eps=config.rms_norm_eps
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def stage_facts(config: JetNemotronConfig, layer_id: int):
|
||||
"""None: the layer adds its own residual, so it declares no stages.
|
||||
The model's shared declaration function (see make_layers)."""
|
||||
return ()
|
||||
|
||||
def forward(
|
||||
self,
|
||||
positions: torch.Tensor,
|
||||
|
||||
+148
-100
@@ -28,7 +28,6 @@ from sglang.srt.distributed import (
|
||||
from sglang.srt.distributed.device_communicators.pynccl_allocator import (
|
||||
use_symmetric_memory,
|
||||
)
|
||||
from sglang.srt.distributed.utils import get_group_rank_size
|
||||
from sglang.srt.environ import envs
|
||||
from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder
|
||||
from sglang.srt.layers import zero_copy_context
|
||||
@@ -52,6 +51,7 @@ from sglang.srt.layers.layer_boundary import (
|
||||
declare_attn,
|
||||
declare_ffn,
|
||||
)
|
||||
from sglang.srt.layers.layer_boundary.residual import LayerResidualOps
|
||||
from sglang.srt.layers.layer_boundary.residual import batch as residual_batch
|
||||
from sglang.srt.layers.layer_boundary.residual.add_norm import (
|
||||
REPLACE_AT_EXIT,
|
||||
@@ -72,6 +72,7 @@ from sglang.srt.layers.linear import (
|
||||
QKVParallelLinear,
|
||||
ReplicatedLinear,
|
||||
RowParallelLinear,
|
||||
resolve_linear_parallel_group,
|
||||
)
|
||||
from sglang.srt.layers.logits_processor import LogitsProcessor
|
||||
from sglang.srt.layers.moe import route_quant_handoff
|
||||
@@ -273,6 +274,20 @@ def _merge_weights_as_views(
|
||||
return merged, sizes
|
||||
|
||||
|
||||
def _dense_mlp_parallel_group() -> str:
|
||||
"""The group the dense layers' MLP shards over: a K3 MLP's group when it
|
||||
is not given one. The Ascend path shards the dense MLP inside each
|
||||
attention-TP replica. The GPU K3 refactor instead gathers all DP rows
|
||||
and shards this one dense layer over the full TP group. Keep the GPU
|
||||
default, but allow the NPU launcher to retain the proven attention-TP
|
||||
layout without a device-type branch in shared model code."""
|
||||
return (
|
||||
"attn_tp"
|
||||
if get_parallel().enable_dense_mlp_attn_tp and is_dp_attention_enabled()
|
||||
else "tp"
|
||||
)
|
||||
|
||||
|
||||
class KimiK3MLP(nn.Module):
|
||||
"""K3 MLP; SiLU or SiTU activation."""
|
||||
|
||||
@@ -288,18 +303,10 @@ class KimiK3MLP(nn.Module):
|
||||
parallel_group: Optional[LinearParallelGroup] = None,
|
||||
) -> None:
|
||||
super().__init__()
|
||||
# The Ascend path shards the dense MLP inside each attention-TP
|
||||
# replica. The GPU K3 refactor instead gathers all DP rows and shards
|
||||
# this one dense layer over the full TP group. Keep the GPU default,
|
||||
# but allow the NPU launcher to retain the proven attention-TP layout
|
||||
# without a device-type branch in shared model code.
|
||||
self._dense_attn_tp = (
|
||||
get_parallel().enable_dense_mlp_attn_tp
|
||||
and is_dp_attention_enabled()
|
||||
and parallel_group is None
|
||||
)
|
||||
if parallel_group is None:
|
||||
parallel_group = "attn_tp" if self._dense_attn_tp else "tp"
|
||||
default_group = parallel_group is None
|
||||
if default_group:
|
||||
parallel_group = _dense_mlp_parallel_group()
|
||||
self._dense_attn_tp = default_group and parallel_group == "attn_tp"
|
||||
self.gate_up_proj = MergedColumnParallelLinear(
|
||||
hidden_size,
|
||||
[intermediate_size] * 2,
|
||||
@@ -417,6 +424,27 @@ def _k3_symm_o_proj_out(o_proj: RowParallelLinear, x: torch.Tensor) -> torch.Ten
|
||||
)
|
||||
|
||||
|
||||
def _uses_latent_moe(config: KimiLinearConfig) -> bool:
|
||||
"""Whether the MoE layers' routed experts run in a latent space narrower
|
||||
than the hidden size (Latent MoE)."""
|
||||
return config.routed_expert_hidden_size is not None
|
||||
|
||||
|
||||
def _uses_ep_a2a() -> bool:
|
||||
"""Whether the MoE all-to-all backend is an EP a2a one, which moves each
|
||||
row to its experts directly: the MoE region then consumes whatever rows
|
||||
this rank holds (SP-MoE shard or DP-local batch), with no DP gather and
|
||||
no TP reduce anywhere in it."""
|
||||
_a2a_backend = get_moe_a2a_backend()
|
||||
return (
|
||||
_a2a_backend.is_megamoe()
|
||||
or _a2a_backend.is_deepep()
|
||||
or _a2a_backend.is_mooncake()
|
||||
or _a2a_backend.is_ascend_fuseep()
|
||||
or _a2a_backend.is_mori()
|
||||
)
|
||||
|
||||
|
||||
class KimiK3MoE(nn.Module):
|
||||
"""K3 MoE with Latent MoE (experts run in moe_hidden_size space)."""
|
||||
|
||||
@@ -438,7 +466,7 @@ class KimiK3MoE(nn.Module):
|
||||
self.layer_idx = layer_idx
|
||||
self.alt_stream = alt_stream
|
||||
|
||||
self.use_latent_moe = config.routed_expert_hidden_size is not None
|
||||
self.use_latent_moe = _uses_latent_moe(config)
|
||||
# Merged front weight ([H, gate_up + E + latent]), built after weight
|
||||
# loading by _merge_front_weights().
|
||||
self._front_w: Optional[torch.Tensor] = None
|
||||
@@ -541,18 +569,7 @@ class KimiK3MoE(nn.Module):
|
||||
"got a checkpoint with different constants"
|
||||
)
|
||||
|
||||
# EP a2a backends move each row to its experts directly, so the MoE
|
||||
# region consumes whatever rows this rank holds (SP-MoE shard or
|
||||
# DP-local batch). No DP gather and no TP reduce anywhere in the
|
||||
# region.
|
||||
_a2a_backend = get_moe_a2a_backend()
|
||||
self._ep_a2a = (
|
||||
_a2a_backend.is_megamoe()
|
||||
or _a2a_backend.is_deepep()
|
||||
or _a2a_backend.is_mooncake()
|
||||
or _a2a_backend.is_ascend_fuseep()
|
||||
or _a2a_backend.is_mori()
|
||||
)
|
||||
self._ep_a2a = _uses_ep_a2a()
|
||||
|
||||
# Defer the trtllm-gen finalize (top-k weighted unpermute) into the
|
||||
# push all-reduce's staging pass (k3_ar_fusion.finalize_all_reduce_push_norm)
|
||||
@@ -2358,14 +2375,7 @@ def _is_moe_layer(config: KimiLinearConfig, layer_idx: int) -> bool:
|
||||
def _shards_moe_rows() -> bool:
|
||||
"""Whether an MoE layer runs on its attention-TP token shard (SP-MoE): an
|
||||
EP a2a backend with attention TP."""
|
||||
backend = get_moe_a2a_backend()
|
||||
return (
|
||||
backend.is_megamoe()
|
||||
or backend.is_deepep()
|
||||
or backend.is_mooncake()
|
||||
or backend.is_ascend_fuseep()
|
||||
or backend.is_mori()
|
||||
) and get_parallel().attn_tp_group.world_size > 1
|
||||
return _uses_ep_a2a() and get_parallel().attn_tp_group.world_size > 1
|
||||
|
||||
|
||||
def _fuses_attn_all_reduce(config: KimiLinearConfig) -> bool:
|
||||
@@ -2380,7 +2390,7 @@ def _fuses_attn_all_reduce(config: KimiLinearConfig) -> bool:
|
||||
)
|
||||
|
||||
|
||||
def _carries_bank_slices(config: KimiLinearConfig) -> bool:
|
||||
def _carries_bank_slices(config: KimiLinearConfig, sp_collective: bool) -> bool:
|
||||
"""Whether consecutive SP-MoE layers keep the stream and the
|
||||
attention-residual bank on each rank's attention-TP shard of the rows
|
||||
(SGLANG_K3_SP_ATTN_RES): each layer reads its attention input there, then
|
||||
@@ -2388,11 +2398,12 @@ def _carries_bank_slices(config: KimiLinearConfig) -> bool:
|
||||
neither a later pipeline rank nor a draft model's capture of the target's
|
||||
hidden states can take them; it is decided at construction. Nor can a
|
||||
dense layer after an SP-MoE one, whose FFN reads every row of the bank:
|
||||
every layer from the first MoE layer on must be one."""
|
||||
every layer from the first MoE layer on must be one. ``sp_collective``:
|
||||
whether K3's SP collective is enabled on this rank."""
|
||||
if not (
|
||||
_shards_moe_rows()
|
||||
and config.attn_res_block_size is not None
|
||||
and k3_sp_collective.enabled()
|
||||
and sp_collective
|
||||
and envs.SGLANG_K3_SP_ATTN_RES.get()
|
||||
and get_parallel().pp_size == 1
|
||||
):
|
||||
@@ -2404,6 +2415,84 @@ def _carries_bank_slices(config: KimiLinearConfig) -> bool:
|
||||
return not (spec.is_eagle3() or spec.is_dflash_family())
|
||||
|
||||
|
||||
def _latent_moe_writes_stream(config: KimiLinearConfig, layer_idx: int) -> bool:
|
||||
"""Whether a layer's FFN writes the next stream itself: a latent MoE on
|
||||
this rank's rows adds the residual in its tail add and writes the next
|
||||
stream itself; one gathered over attention DP returns its output to this
|
||||
rank's tokens first."""
|
||||
return (
|
||||
config.attn_res_block_size is not None
|
||||
and _is_moe_layer(config, layer_idx)
|
||||
and _uses_latent_moe(config)
|
||||
and not (is_dp_attention_enabled() and not _uses_ep_a2a())
|
||||
)
|
||||
|
||||
|
||||
def _stage_facts(
|
||||
config: KimiLinearConfig,
|
||||
layer_idx: int,
|
||||
*,
|
||||
sp_collective: bool,
|
||||
bank: Optional[LayerResidualOps] = None,
|
||||
):
|
||||
"""The attention and FFN stages the K3 layer at ``layer_idx`` declares:
|
||||
the model's shared declaration function (see make_layers), which the
|
||||
layer declares itself with. ``sp_collective`` is whether K3's SP
|
||||
collective is enabled on this rank, which the model settles before it
|
||||
builds a layer. ``bank`` is the layer's own reads and updates of the
|
||||
attention-residual bank; without it, the stages declare what those do."""
|
||||
is_moe = _is_moe_layer(config, layer_idx)
|
||||
# Under attention DP the FFN input is read on this rank's rows
|
||||
# once the attention's sum is complete, then gathered.
|
||||
attn_ops = {}
|
||||
ffn_ops = dict(read=NormReadout(reads_before_dp_gather=True))
|
||||
if config.attn_res_block_size is not None:
|
||||
if bank is None:
|
||||
bank = AttnBankState.facts(
|
||||
reads_slices=_carries_bank_slices(config, sp_collective)
|
||||
)
|
||||
attn_ops = dict(
|
||||
read=bank.attn_readout,
|
||||
update=bank.attn_update,
|
||||
# The rows an SP-MoE layer left on each rank's shard are
|
||||
# gathered in K3's tuned all-gather when it takes the batch:
|
||||
# before this read, or after it while the bank stays on the
|
||||
# shard.
|
||||
attn_tp_gather=(
|
||||
k3_sp_collective.all_gather
|
||||
if sp_collective
|
||||
and _shards_moe_rows()
|
||||
and _is_moe_layer(config, layer_idx - 1)
|
||||
else None
|
||||
),
|
||||
)
|
||||
ffn_ops = dict(
|
||||
read=bank.ffn_readout,
|
||||
update=(
|
||||
REPLACE_AT_EXIT
|
||||
if _latent_moe_writes_stream(config, layer_idx)
|
||||
else bank.ffn_update
|
||||
),
|
||||
)
|
||||
return (
|
||||
declare_attn(**attn_ops),
|
||||
declare_ffn(
|
||||
**ffn_ops,
|
||||
sparse=is_moe,
|
||||
next_layer_sparse=_is_moe_layer(config, layer_idx + 1),
|
||||
# The TP width the dense MLP is built with.
|
||||
dense_tp_size=(
|
||||
None
|
||||
if is_moe
|
||||
else resolve_linear_parallel_group(_dense_mlp_parallel_group())[1]
|
||||
),
|
||||
# A latent MoE completes its output sum together with
|
||||
# the latent reduction its norm needs.
|
||||
output_complete=is_moe and _uses_latent_moe(config),
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
class KimiK3DecoderLayer(nn.Module):
|
||||
"""Decoder layer carrying the K3 attention-residual stream."""
|
||||
|
||||
@@ -2419,7 +2508,6 @@ class KimiK3DecoderLayer(nn.Module):
|
||||
super().__init__()
|
||||
self.hidden_size = config.hidden_size
|
||||
self.layer_idx = layer_idx
|
||||
self._dp_attention = is_dp_attention_enabled()
|
||||
self._is_moe_layer = _is_moe_layer(config, layer_idx)
|
||||
# SP-MoE: with an EP a2a backend and attention TP, a MoE layer runs on
|
||||
# this rank's attention-TP shard of the rows.
|
||||
@@ -2506,31 +2594,17 @@ class KimiK3DecoderLayer(nn.Module):
|
||||
prefix=f"{prefix}.mlp_res_proj",
|
||||
)
|
||||
|
||||
# A latent MoE on this rank's rows adds the residual in its tail add and
|
||||
# writes the next stream itself; one gathered over attention DP returns
|
||||
# its output to this rank's tokens first.
|
||||
self._ffn_writes_stream = (
|
||||
self.use_attn_residuals
|
||||
and self._is_moe_layer
|
||||
and self.mlp.use_latent_moe
|
||||
and not (self._dp_attention and not self.mlp._ep_a2a)
|
||||
)
|
||||
self._ffn_writes_stream = _latent_moe_writes_stream(config, layer_idx)
|
||||
self._declare_stages(config, layer_idx, attn_bank)
|
||||
|
||||
def _declare_stages(self, config, layer_idx, attn_bank):
|
||||
"""Declare this layer's attention and FFN stages: the reads and updates
|
||||
around them, the rows the FFN takes, and the kernels K3 supplies for
|
||||
the bank path."""
|
||||
# Under attention DP the FFN input is read on this rank's rows
|
||||
# once the attention's sum is complete, then gathered.
|
||||
attn_ops = {}
|
||||
ffn_ops = dict(read=NormReadout(reads_before_dp_gather=True))
|
||||
carries = _carries_bank_slices(config)
|
||||
tuned_gather = (
|
||||
k3_sp_collective.all_gather if k3_sp_collective.enabled() else None
|
||||
)
|
||||
"""Declare this layer's attention and FFN stages as _stage_facts gives
|
||||
them, with this layer's reads and updates of the bank and the kernels
|
||||
K3 supplies for the bank path."""
|
||||
sp_collective = k3_sp_collective.enabled()
|
||||
bank = None
|
||||
if self.use_attn_residuals:
|
||||
bank_ops = AttnBankState(
|
||||
bank = AttnBankState(
|
||||
attn_bank,
|
||||
self.self_attention_res_proj,
|
||||
self.self_attention_res_norm,
|
||||
@@ -2539,48 +2613,16 @@ class KimiK3DecoderLayer(nn.Module):
|
||||
writes_block=self.is_block_write_layer,
|
||||
ffn_input_fusions=self._ffn_input_fusions(),
|
||||
fuses_slice_collectives=self._sp_moe
|
||||
and k3_sp_collective.enabled()
|
||||
and sp_collective
|
||||
and envs.SGLANG_K3_SP_ATTN_RES.get(),
|
||||
reads_slices=carries,
|
||||
reads_slices=_carries_bank_slices(config, sp_collective),
|
||||
).residual_ops()
|
||||
attn_ops = dict(
|
||||
read=bank_ops.attn_readout,
|
||||
update=bank_ops.attn_update,
|
||||
# The rows an SP-MoE layer left on each rank's shard are
|
||||
# gathered in K3's tuned all-gather when it takes the batch:
|
||||
# before this read, or after it while the bank stays on the
|
||||
# shard.
|
||||
attn_tp_gather=(
|
||||
tuned_gather
|
||||
if _shards_moe_rows() and _is_moe_layer(config, layer_idx - 1)
|
||||
else None
|
||||
),
|
||||
)
|
||||
ffn_ops = dict(
|
||||
read=bank_ops.ffn_readout,
|
||||
update=(
|
||||
REPLACE_AT_EXIT if self._ffn_writes_stream else bank_ops.ffn_update
|
||||
),
|
||||
)
|
||||
attn, ffn = _stage_facts(
|
||||
config, layer_idx, sp_collective=sp_collective, bank=bank
|
||||
)
|
||||
self.attn_boundary, self.ffn_boundary = append_stages(
|
||||
(declare_attn(**attn_ops), self.input_layernorm),
|
||||
(
|
||||
declare_ffn(
|
||||
**ffn_ops,
|
||||
sparse=self._is_moe_layer,
|
||||
next_layer_sparse=_is_moe_layer(config, layer_idx + 1),
|
||||
# The TP width the dense MLP is built with.
|
||||
dense_tp_size=(
|
||||
None
|
||||
if self._is_moe_layer
|
||||
else get_group_rank_size(self.mlp.down_proj.tp_group)[1]
|
||||
),
|
||||
# A latent MoE completes its output sum together with
|
||||
# the latent reduction its norm needs.
|
||||
output_complete=self._is_moe_layer and self.mlp.use_latent_moe,
|
||||
),
|
||||
self.post_attention_layernorm,
|
||||
),
|
||||
(attn, self.input_layernorm),
|
||||
(ffn, self.post_attention_layernorm),
|
||||
)
|
||||
|
||||
def _ffn_input_fusions(self):
|
||||
@@ -2673,7 +2715,10 @@ class KimiK3LinearModel(nn.Module):
|
||||
self.pp_group = get_parallel().pp_group
|
||||
self.dspark_layers_to_capture: Optional[list[int]] = None
|
||||
self._dp_attention = is_dp_attention_enabled()
|
||||
self.carries_bank_slices = _carries_bank_slices(config)
|
||||
# Whether K3's SP collective is enabled on this rank, which the layers'
|
||||
# stage declarations depend on: settled once, before any is built.
|
||||
sp_collective = k3_sp_collective.enabled()
|
||||
self.carries_bank_slices = _carries_bank_slices(config, sp_collective)
|
||||
|
||||
if self.pp_group.is_first_rank:
|
||||
embedding_quant_config = (
|
||||
@@ -2731,7 +2776,7 @@ class KimiK3LinearModel(nn.Module):
|
||||
# takes the batch.
|
||||
attn_tp_gather=(
|
||||
k3_sp_collective.all_gather
|
||||
if k3_sp_collective.enabled()
|
||||
if sp_collective
|
||||
and _shards_moe_rows()
|
||||
and _is_moe_layer(config, config.num_hidden_layers - 1)
|
||||
else None
|
||||
@@ -2753,6 +2798,9 @@ class KimiK3LinearModel(nn.Module):
|
||||
),
|
||||
prefix=f"{prefix}.layers",
|
||||
final_read=self._final_read if self.pp_group.is_last_rank else None,
|
||||
stage_facts=lambda idx: _stage_facts(
|
||||
config, idx, sp_collective=sp_collective
|
||||
),
|
||||
)
|
||||
|
||||
def forward(
|
||||
|
||||
@@ -580,8 +580,8 @@ class KimiDecoderLayer(nn.Module):
|
||||
reduce_results=False,
|
||||
)
|
||||
|
||||
sparse = _is_sparse_layer(config, layer_idx)
|
||||
if sparse:
|
||||
attn, ffn = self.stage_facts(config, layer_idx)
|
||||
if ffn.sparse:
|
||||
self.block_sparse_moe = KimiMoE(
|
||||
config=config,
|
||||
quant_config=quant_config,
|
||||
@@ -603,15 +603,22 @@ class KimiDecoderLayer(nn.Module):
|
||||
self.post_attention_layernorm = RMSNorm(
|
||||
config.hidden_size, eps=config.rms_norm_eps
|
||||
)
|
||||
self.attn_boundary, self.ffn_boundary = append_stages(
|
||||
(attn, self.input_layernorm),
|
||||
(ffn, self.post_attention_layernorm),
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def stage_facts(config: KimiLinearConfig, layer_idx: int):
|
||||
"""The stages the layer at ``layer_idx`` declares, from the config: the
|
||||
model's shared declaration function (see make_layers)."""
|
||||
sparse = _is_sparse_layer(config, layer_idx)
|
||||
next_sparse = layer_idx + 1 < config.num_hidden_layers and _is_sparse_layer(
|
||||
config, layer_idx + 1
|
||||
)
|
||||
self.attn_boundary, self.ffn_boundary = append_stages(
|
||||
(declare_attn(), self.input_layernorm),
|
||||
(
|
||||
declare_ffn(sparse=sparse, next_layer_sparse=next_sparse),
|
||||
self.post_attention_layernorm,
|
||||
),
|
||||
return (
|
||||
declare_attn(),
|
||||
declare_ffn(sparse=sparse, next_layer_sparse=next_sparse),
|
||||
)
|
||||
|
||||
def forward(
|
||||
@@ -674,6 +681,7 @@ class KimiLinearModel(nn.Module):
|
||||
alt_stream=self.alt_stream,
|
||||
),
|
||||
prefix=f"{prefix}.layers",
|
||||
stage_facts=lambda idx: KimiDecoderLayer.stage_facts(config, idx),
|
||||
)
|
||||
|
||||
if self.pp_group.is_last_rank:
|
||||
|
||||
@@ -418,12 +418,8 @@ class LagunaDecoderLayer(nn.Module):
|
||||
prefix=add_prefix("self_attn", prefix),
|
||||
)
|
||||
|
||||
mlp_types = config.mlp_layer_types
|
||||
self.is_layer_sparse = mlp_types[layer_id] == "sparse"
|
||||
is_next_layer_sparse = (
|
||||
layer_id + 1 < config.num_hidden_layers
|
||||
and mlp_types[layer_id + 1] == "sparse"
|
||||
)
|
||||
attn, ffn = self.stage_facts(config, layer_id)
|
||||
self.is_layer_sparse = ffn.sparse
|
||||
|
||||
if self.is_layer_sparse:
|
||||
self.mlp = LagunaMoE(
|
||||
@@ -448,13 +444,25 @@ class LagunaDecoderLayer(nn.Module):
|
||||
)
|
||||
|
||||
self.attn_boundary, self.ffn_boundary = append_stages(
|
||||
(declare_attn(), self.input_layernorm),
|
||||
(
|
||||
declare_ffn(
|
||||
sparse=self.is_layer_sparse,
|
||||
next_layer_sparse=is_next_layer_sparse,
|
||||
),
|
||||
self.post_attention_layernorm,
|
||||
(attn, self.input_layernorm),
|
||||
(ffn, self.post_attention_layernorm),
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def stage_facts(config: LagunaConfig, layer_id: int):
|
||||
"""The stages the layer at ``layer_id`` declares, from the config: the
|
||||
model's shared declaration function (see make_layers)."""
|
||||
mlp_types = config.mlp_layer_types
|
||||
is_layer_sparse = mlp_types[layer_id] == "sparse"
|
||||
is_next_layer_sparse = (
|
||||
layer_id + 1 < config.num_hidden_layers
|
||||
and mlp_types[layer_id + 1] == "sparse"
|
||||
)
|
||||
return (
|
||||
declare_attn(),
|
||||
declare_ffn(
|
||||
sparse=is_layer_sparse,
|
||||
next_layer_sparse=is_next_layer_sparse,
|
||||
),
|
||||
)
|
||||
|
||||
@@ -519,6 +527,7 @@ class LagunaModel(nn.Module):
|
||||
prefix=prefix,
|
||||
),
|
||||
prefix=add_prefix("layers", prefix),
|
||||
stage_facts=lambda idx: decoder_layer_type.stage_facts(config, idx),
|
||||
)
|
||||
if self.pp_group.is_last_rank:
|
||||
self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
|
||||
|
||||
@@ -20,6 +20,7 @@
|
||||
"""SGLang LLaDA2MoeModelLM model."""
|
||||
|
||||
import logging
|
||||
from functools import partial
|
||||
from typing import Iterable, Optional, Tuple, Union
|
||||
|
||||
import torch
|
||||
@@ -1150,7 +1151,6 @@ class LLaDA2MoeBlock(nn.Module):
|
||||
self.layer_id = layer_id
|
||||
|
||||
self.is_layer_sparse = self._is_layer_sparse(config, layer_id=layer_id)
|
||||
is_next_layer_sparse = self._is_layer_sparse(config, layer_id=layer_id + 1)
|
||||
|
||||
if self.is_layer_sparse:
|
||||
self.mlp = LLaDA2MoeSparseMoeBlock(
|
||||
@@ -1173,18 +1173,27 @@ class LLaDA2MoeBlock(nn.Module):
|
||||
|
||||
self.post_attention_layernorm = RMSNorm(hidden_size, eps=config.rms_norm_eps)
|
||||
|
||||
attn, ffn = self.stage_facts(config, layer_id)
|
||||
self.attn_boundary, self.ffn_boundary = append_stages(
|
||||
(declare_attn(), self.input_layernorm),
|
||||
(
|
||||
declare_ffn(
|
||||
sparse=self.is_layer_sparse,
|
||||
next_layer_sparse=is_next_layer_sparse,
|
||||
),
|
||||
self.post_attention_layernorm,
|
||||
(attn, self.input_layernorm),
|
||||
(ffn, self.post_attention_layernorm),
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def stage_facts(cls, config: PretrainedConfig, layer_id: int):
|
||||
"""The stages the layer at ``layer_id`` declares, in order: the model's
|
||||
shared declaration function, which the layer declares with too (see
|
||||
make_layers)."""
|
||||
return (
|
||||
declare_attn(),
|
||||
declare_ffn(
|
||||
sparse=cls._is_layer_sparse(config, layer_id=layer_id),
|
||||
next_layer_sparse=cls._is_layer_sparse(config, layer_id=layer_id + 1),
|
||||
),
|
||||
)
|
||||
|
||||
def _is_layer_sparse(self, config: PretrainedConfig, layer_id: int) -> bool:
|
||||
@staticmethod
|
||||
def _is_layer_sparse(config: PretrainedConfig, layer_id: int) -> bool:
|
||||
return (
|
||||
config.num_experts is not None and layer_id >= config.first_k_dense_replace
|
||||
)
|
||||
@@ -1253,6 +1262,7 @@ class LLaDA2MoeModel(nn.Module):
|
||||
alt_stream=alt_stream,
|
||||
),
|
||||
prefix=add_prefix("layers", prefix),
|
||||
stage_facts=partial(LLaDA2MoeBlock.stage_facts, config),
|
||||
)
|
||||
if self.pp_group.is_last_rank:
|
||||
self.norm = RMSNorm(self.embed_dim, eps=config.rms_norm_eps)
|
||||
|
||||
@@ -364,15 +364,24 @@ class LlamaDecoderLayer(nn.Module):
|
||||
return
|
||||
self.input_layernorm.fuse_input_quant(self.self_attn.qkv_proj)
|
||||
self.post_attention_layernorm.fuse_input_quant(self.mlp.gate_up_proj)
|
||||
attn, ffn = self.stage_facts(config, layer_id)
|
||||
self.attn_boundary, self.ffn_boundary = append_stages(
|
||||
(declare_attn(read=self._attn_readout(layer_id)), self.input_layernorm),
|
||||
(
|
||||
declare_ffn(sparse=False, next_layer_sparse=False),
|
||||
self.post_attention_layernorm,
|
||||
),
|
||||
(attn, self.input_layernorm),
|
||||
(ffn, self.post_attention_layernorm),
|
||||
)
|
||||
|
||||
def _attn_readout(self, layer_id: int):
|
||||
@classmethod
|
||||
def stage_facts(cls, config: LlamaConfig, layer_id: int):
|
||||
"""The stages a layer of this class declares, from the config alone:
|
||||
the model's shared declaration function, which the layer declares
|
||||
with too (see make_layers). A subclass's attention read takes part."""
|
||||
return (
|
||||
declare_attn(read=cls._attn_readout(layer_id)),
|
||||
declare_ffn(sparse=False, next_layer_sparse=False),
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def _attn_readout(cls, layer_id: int):
|
||||
return NORM_QUANT_READOUT
|
||||
|
||||
def forward(
|
||||
@@ -437,6 +446,7 @@ class LlamaModel(nn.Module):
|
||||
prefix=prefix,
|
||||
),
|
||||
prefix="model.layers",
|
||||
stage_facts=lambda idx: LlamaDecoderLayer.stage_facts(config, idx),
|
||||
)
|
||||
|
||||
if self.pp_group.is_last_rank:
|
||||
|
||||
@@ -54,7 +54,8 @@ class LlamaDecoderLayer(LlamaDecoderLayer):
|
||||
if layer_id == 0:
|
||||
del self.input_layernorm
|
||||
|
||||
def _attn_readout(self, layer_id: int):
|
||||
@classmethod
|
||||
def _attn_readout(cls, layer_id: int):
|
||||
if layer_id == 0:
|
||||
return PLAIN_READOUT
|
||||
return super()._attn_readout(layer_id)
|
||||
|
||||
@@ -402,18 +402,8 @@ class MellumDecoderLayer(Qwen3MoeDecoderLayer):
|
||||
f"{len(mlp_layer_types)} and {cfg.num_hidden_layers}"
|
||||
)
|
||||
|
||||
def _is_sparse(lid: int) -> bool:
|
||||
if lid < 0 or lid >= cfg.num_hidden_layers:
|
||||
return False
|
||||
mlp_type = mlp_layer_types[lid]
|
||||
if mlp_type not in ("sparse", "dense"):
|
||||
raise ValueError(
|
||||
f"Unsupported mlp_layer_types[{lid}]={mlp_type}; "
|
||||
"expected 'sparse' or 'dense'"
|
||||
)
|
||||
return mlp_type == "sparse"
|
||||
|
||||
self.is_layer_sparse = _is_sparse(layer_id)
|
||||
attn, ffn = self.stage_facts(config, layer_id)
|
||||
self.is_layer_sparse = ffn.sparse
|
||||
|
||||
if self.is_layer_sparse:
|
||||
if num_experts <= 0:
|
||||
@@ -436,19 +426,37 @@ class MellumDecoderLayer(Qwen3MoeDecoderLayer):
|
||||
reduce_results=False,
|
||||
)
|
||||
|
||||
is_next_layer_sparse = _is_sparse(layer_id + 1)
|
||||
|
||||
self.input_layernorm = RMSNorm(cfg.hidden_size, eps=rms_norm_eps)
|
||||
self.post_attention_layernorm = RMSNorm(cfg.hidden_size, eps=rms_norm_eps)
|
||||
|
||||
self.attn_boundary, self.ffn_boundary = append_stages(
|
||||
(declare_attn(), self.input_layernorm),
|
||||
(
|
||||
declare_ffn(
|
||||
sparse=self.is_layer_sparse,
|
||||
next_layer_sparse=is_next_layer_sparse,
|
||||
),
|
||||
self.post_attention_layernorm,
|
||||
(attn, self.input_layernorm),
|
||||
(ffn, self.post_attention_layernorm),
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _is_sparse(config: PretrainedConfig, lid: int) -> bool:
|
||||
cfg = cast(Any, config)
|
||||
if lid < 0 or lid >= cfg.num_hidden_layers:
|
||||
return False
|
||||
mlp_type = cfg.mlp_layer_types[lid]
|
||||
if mlp_type not in ("sparse", "dense"):
|
||||
raise ValueError(
|
||||
f"Unsupported mlp_layer_types[{lid}]={mlp_type}; "
|
||||
"expected 'sparse' or 'dense'"
|
||||
)
|
||||
return mlp_type == "sparse"
|
||||
|
||||
@classmethod
|
||||
def stage_facts(cls, config: PretrainedConfig, layer_id: int):
|
||||
"""The stages a Mellum layer declares, from the config alone: the
|
||||
model's shared declaration function, which the layer declares with
|
||||
too (see make_layers)."""
|
||||
return (
|
||||
declare_attn(),
|
||||
declare_ffn(
|
||||
sparse=cls._is_sparse(config, layer_id),
|
||||
next_layer_sparse=cls._is_sparse(config, layer_id + 1),
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
@@ -16,6 +16,7 @@ import logging
|
||||
import math
|
||||
import re
|
||||
from array import array
|
||||
from functools import partial
|
||||
from typing import Any, Dict, Iterable, List, Optional, Tuple, Union
|
||||
|
||||
import torch
|
||||
@@ -822,8 +823,7 @@ class MiMoV2DecoderLayer(nn.Module):
|
||||
prefix=add_prefix("self_attn", prefix),
|
||||
)
|
||||
|
||||
self.is_layer_sparse = self.is_moe_layer(layer_id)
|
||||
is_next_layer_sparse = self.is_moe_layer(layer_id + 1)
|
||||
self.is_layer_sparse = self.is_moe_layer(config, layer_id)
|
||||
|
||||
if self.is_layer_sparse:
|
||||
self.mlp = MiMoV2MoE(
|
||||
@@ -849,14 +849,22 @@ class MiMoV2DecoderLayer(nn.Module):
|
||||
config.hidden_size, eps=config.layernorm_epsilon
|
||||
)
|
||||
|
||||
attn, ffn = self.stage_facts(config, layer_id)
|
||||
self.attn_boundary, self.ffn_boundary = append_stages(
|
||||
(declare_attn(), self.input_layernorm),
|
||||
(
|
||||
declare_ffn(
|
||||
sparse=self.is_layer_sparse,
|
||||
next_layer_sparse=is_next_layer_sparse,
|
||||
),
|
||||
self.post_attention_layernorm,
|
||||
(attn, self.input_layernorm),
|
||||
(ffn, self.post_attention_layernorm),
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def stage_facts(cls, config, layer_id: int):
|
||||
"""The stages the layer at ``layer_id`` declares, in order: the model's
|
||||
shared declaration function, which the layer declares with too (see
|
||||
make_layers)."""
|
||||
return (
|
||||
declare_attn(),
|
||||
declare_ffn(
|
||||
sparse=cls.is_moe_layer(config, layer_id),
|
||||
next_layer_sparse=cls.is_moe_layer(config, layer_id + 1),
|
||||
),
|
||||
)
|
||||
|
||||
@@ -889,12 +897,13 @@ class MiMoV2DecoderLayer(nn.Module):
|
||||
|
||||
return hidden_states
|
||||
|
||||
def is_moe_layer(self, layer_idx: int) -> bool:
|
||||
@staticmethod
|
||||
def is_moe_layer(config, layer_idx: int) -> bool:
|
||||
return (
|
||||
hasattr(self.config, "moe_layer_freq")
|
||||
and 0 <= layer_idx < len(self.config.moe_layer_freq)
|
||||
and not isinstance(self.config.moe_layer_freq, int)
|
||||
and self.config.moe_layer_freq[layer_idx]
|
||||
hasattr(config, "moe_layer_freq")
|
||||
and 0 <= layer_idx < len(config.moe_layer_freq)
|
||||
and not isinstance(config.moe_layer_freq, int)
|
||||
and config.moe_layer_freq[layer_idx]
|
||||
)
|
||||
|
||||
def is_swa_layer(self) -> bool:
|
||||
@@ -987,6 +996,7 @@ class MiMoV2Model(nn.Module):
|
||||
prefix=prefix,
|
||||
),
|
||||
prefix=add_prefix("layers", prefix),
|
||||
stage_facts=partial(decoder_layer_type.stage_facts, config),
|
||||
)
|
||||
if self.pp_group.is_last_rank:
|
||||
self.norm = RMSNorm(config.hidden_size, eps=config.layernorm_epsilon)
|
||||
|
||||
@@ -17,7 +17,7 @@
|
||||
|
||||
import logging
|
||||
from contextlib import nullcontext
|
||||
from functools import lru_cache
|
||||
from functools import lru_cache, partial
|
||||
from typing import Any, Dict, Iterable, Optional, Set, Tuple, Union
|
||||
|
||||
import torch
|
||||
@@ -99,7 +99,6 @@ from sglang.srt.utils import (
|
||||
is_xpu,
|
||||
make_pp_layers,
|
||||
)
|
||||
from sglang.srt.utils.common import is_building_neighbour_layer
|
||||
from sglang.srt.utils.custom_op import register_custom_op
|
||||
from sglang.srt.utils.hf_transformers_utils import get_rope_config
|
||||
|
||||
@@ -409,12 +408,7 @@ class MiniMaxM2QKRMSNorm:
|
||||
use_fused_norm = get_bool_env_var("SGLANG_USE_FUSED_PARALLEL_QKNORM")
|
||||
|
||||
self._forward_impl = self._forward_naive
|
||||
if (
|
||||
self._world_size > 1
|
||||
and _is_cuda
|
||||
and use_fused_norm
|
||||
and not is_building_neighbour_layer()
|
||||
):
|
||||
if self._world_size > 1 and _is_cuda and use_fused_norm:
|
||||
occupancy = get_fused_parallel_qknorm_max_occupancy(
|
||||
q_norm.weight.dtype,
|
||||
self._world_size,
|
||||
@@ -846,8 +840,8 @@ class MiniMaxM2DecoderLayer(nn.Module):
|
||||
self.hidden_size = config.hidden_size
|
||||
self.layer_id = layer_id
|
||||
|
||||
# TBO support: All MiniMax layers are sparse (MoE)
|
||||
self.is_layer_sparse = True
|
||||
attn, ffn = self.stage_facts(config, layer_id)
|
||||
self.is_layer_sparse = ffn.sparse
|
||||
|
||||
self.self_attn = MiniMaxM2Attention(
|
||||
config=config,
|
||||
@@ -870,17 +864,20 @@ class MiniMaxM2DecoderLayer(nn.Module):
|
||||
config.hidden_size, eps=getattr(config, "rms_norm_eps", 1e-6)
|
||||
)
|
||||
|
||||
is_next_layer_sparse = True
|
||||
|
||||
self.attn_boundary, self.ffn_boundary = append_stages(
|
||||
(declare_attn(), self.input_layernorm),
|
||||
(
|
||||
declare_ffn(
|
||||
sparse=self.is_layer_sparse,
|
||||
next_layer_sparse=is_next_layer_sparse,
|
||||
),
|
||||
self.post_attention_layernorm,
|
||||
),
|
||||
(attn, self.input_layernorm),
|
||||
(ffn, self.post_attention_layernorm),
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def stage_facts(config: PretrainedConfig, layer_id: int):
|
||||
"""The stages the layer at ``layer_id`` declares, in order: the model's
|
||||
shared declaration function, which the layer declares with too (see
|
||||
make_layers)."""
|
||||
# TBO support: All MiniMax layers are sparse (MoE)
|
||||
return (
|
||||
declare_attn(),
|
||||
declare_ffn(sparse=True, next_layer_sparse=True),
|
||||
)
|
||||
|
||||
def forward(
|
||||
@@ -951,6 +948,7 @@ class MiniMaxM2Model(nn.Module):
|
||||
config.num_hidden_layers,
|
||||
layer_fn,
|
||||
prefix=add_prefix("layers", prefix),
|
||||
stage_facts=partial(MiniMaxM2DecoderLayer.stage_facts, config),
|
||||
)
|
||||
if self.pp_group.is_last_rank:
|
||||
self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
|
||||
|
||||
@@ -17,6 +17,7 @@
|
||||
|
||||
import logging
|
||||
from contextlib import nullcontext
|
||||
from functools import partial
|
||||
from typing import Iterable, List, Optional, Set, Tuple, Union
|
||||
|
||||
import torch
|
||||
@@ -1520,12 +1521,9 @@ class MiniMaxM3DecoderLayer(nn.Module):
|
||||
disable_index_value=disable_index_value,
|
||||
)
|
||||
|
||||
moe_layer_freq = getattr(config, "moe_layer_freq", None)
|
||||
# Means "MLP is a sparse MoE", not attention sparsity. Kept as ``is_layer_sparse``
|
||||
# because model construction and downstream integrations read this attr.
|
||||
self.is_layer_sparse = (
|
||||
moe_layer_freq[layer_id] != 0 if moe_layer_freq is not None else True
|
||||
)
|
||||
self.is_layer_sparse = self._is_layer_sparse(config, layer_id)
|
||||
|
||||
if self.is_layer_sparse:
|
||||
self.mlp = MiniMaxM3MoE(
|
||||
@@ -1560,26 +1558,40 @@ class MiniMaxM3DecoderLayer(nn.Module):
|
||||
config.hidden_size, eps=config.rms_norm_eps
|
||||
)
|
||||
|
||||
def _is_layer_sparse(lid):
|
||||
if moe_layer_freq is None:
|
||||
return True
|
||||
if lid < 0 or lid >= config.num_hidden_layers:
|
||||
return True
|
||||
return moe_layer_freq[lid] != 0
|
||||
|
||||
is_next_layer_sparse = _is_layer_sparse(layer_id + 1)
|
||||
|
||||
attn, ffn = self.stage_facts(config, layer_id)
|
||||
self.attn_boundary, self.ffn_boundary = append_stages(
|
||||
(declare_attn(), self.input_layernorm),
|
||||
(
|
||||
declare_ffn(
|
||||
sparse=self.is_layer_sparse,
|
||||
next_layer_sparse=is_next_layer_sparse,
|
||||
),
|
||||
self.post_attention_layernorm,
|
||||
(attn, self.input_layernorm),
|
||||
(ffn, self.post_attention_layernorm),
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def stage_facts(cls, config: PretrainedConfig, layer_id: int):
|
||||
"""The stages the layer at ``layer_id`` declares, in order: the model's
|
||||
shared declaration function, which the layer declares with too (see
|
||||
make_layers)."""
|
||||
return (
|
||||
declare_attn(),
|
||||
declare_ffn(
|
||||
sparse=cls._is_layer_sparse(config, layer_id),
|
||||
next_layer_sparse=cls._is_next_layer_sparse(config, layer_id),
|
||||
),
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _is_layer_sparse(config: PretrainedConfig, layer_id: int) -> bool:
|
||||
moe_layer_freq = getattr(config, "moe_layer_freq", None)
|
||||
return moe_layer_freq[layer_id] != 0 if moe_layer_freq is not None else True
|
||||
|
||||
@staticmethod
|
||||
def _is_next_layer_sparse(config: PretrainedConfig, layer_id: int) -> bool:
|
||||
moe_layer_freq = getattr(config, "moe_layer_freq", None)
|
||||
lid = layer_id + 1
|
||||
if moe_layer_freq is None:
|
||||
return True
|
||||
if lid < 0 or lid >= config.num_hidden_layers:
|
||||
return True
|
||||
return moe_layer_freq[lid] != 0
|
||||
|
||||
def forward(
|
||||
self,
|
||||
positions: torch.Tensor,
|
||||
@@ -1653,6 +1665,7 @@ class MiniMaxM3Model(nn.Module):
|
||||
config.num_hidden_layers,
|
||||
layer_fn,
|
||||
prefix=add_prefix("layers", prefix),
|
||||
stage_facts=partial(MiniMaxM3DecoderLayer.stage_facts, config),
|
||||
)
|
||||
if self.pp_group.is_last_rank:
|
||||
if self.use_gemma_norm:
|
||||
|
||||
@@ -155,6 +155,7 @@ class Ministral3Model(LlamaModel):
|
||||
prefix=prefix,
|
||||
),
|
||||
prefix="model.layers",
|
||||
stage_facts=lambda idx: Ministral3DecoderLayer.stage_facts(config, idx),
|
||||
)
|
||||
|
||||
|
||||
|
||||
@@ -19,6 +19,7 @@
|
||||
"""Inference-only Mixtral model."""
|
||||
|
||||
import logging
|
||||
from functools import partial
|
||||
from typing import Iterable, Optional, Tuple, Union
|
||||
|
||||
import torch
|
||||
@@ -235,12 +236,20 @@ class MixtralDecoderLayer(nn.Module):
|
||||
self.post_attention_layernorm = RMSNorm(
|
||||
config.hidden_size, eps=config.rms_norm_eps
|
||||
)
|
||||
attn, ffn = self.stage_facts(config, layer_id)
|
||||
self.attn_boundary, self.ffn_boundary = append_stages(
|
||||
(declare_attn(), self.input_layernorm),
|
||||
(
|
||||
declare_ffn(sparse=True, next_layer_sparse=True),
|
||||
self.post_attention_layernorm,
|
||||
),
|
||||
(attn, self.input_layernorm),
|
||||
(ffn, self.post_attention_layernorm),
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def stage_facts(config: MixtralConfig, layer_id: int):
|
||||
"""The stages the layer at ``layer_id`` declares, in order: the model's
|
||||
shared declaration function, which the layer declares with too (see
|
||||
make_layers)."""
|
||||
return (
|
||||
declare_attn(),
|
||||
declare_ffn(sparse=True, next_layer_sparse=True),
|
||||
)
|
||||
|
||||
def forward(
|
||||
@@ -292,6 +301,7 @@ class MixtralModel(nn.Module):
|
||||
),
|
||||
prefix="layers",
|
||||
return_tuple=True,
|
||||
stage_facts=partial(MixtralDecoderLayer.stage_facts, config),
|
||||
)
|
||||
|
||||
if self.pp_group.is_last_rank:
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import logging
|
||||
from functools import partial
|
||||
from typing import Iterable, List, Optional, Tuple, Union
|
||||
|
||||
import torch
|
||||
@@ -231,12 +232,20 @@ class NanbeigeDecoderLayer(nn.Module):
|
||||
)
|
||||
# Each loop runs the physical layers as one stack: the model folds the
|
||||
# residual back in and re-enters at layer 0.
|
||||
attn, ffn = self.stage_facts(config, layer_id)
|
||||
self.attn_boundary, self.ffn_boundary = append_stages(
|
||||
(declare_attn(), self.input_layernorm),
|
||||
(
|
||||
declare_ffn(sparse=False, next_layer_sparse=False),
|
||||
self.post_attention_layernorm,
|
||||
),
|
||||
(attn, self.input_layernorm),
|
||||
(ffn, self.post_attention_layernorm),
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def stage_facts(config: NanbeigeConfig, layer_id: int):
|
||||
"""The stages the layer at ``layer_id`` declares, in order: the model's
|
||||
shared declaration function, which the layer declares with too (see
|
||||
make_layers)."""
|
||||
return (
|
||||
declare_attn(),
|
||||
declare_ffn(sparse=False, next_layer_sparse=False),
|
||||
)
|
||||
|
||||
def forward(
|
||||
@@ -305,6 +314,7 @@ class NanbeigeModel(nn.Module):
|
||||
alt_stream=alt_stream,
|
||||
),
|
||||
prefix=add_prefix("layers", prefix),
|
||||
stage_facts=partial(decoder_layer_type.stage_facts, config),
|
||||
)
|
||||
if self.pp_group.is_last_rank:
|
||||
self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
|
||||
|
||||
@@ -134,17 +134,12 @@ class DeciLMDecoderLayer(nn.Module):
|
||||
config.hidden_size, eps=config.rms_norm_eps
|
||||
)
|
||||
|
||||
# A layer whose attention and FFN are both no-ops has no stages.
|
||||
stages = []
|
||||
norms = []
|
||||
if not self._is_no_op_attention:
|
||||
stages.append((declare_attn(), self.input_layernorm))
|
||||
norms.append(self.input_layernorm)
|
||||
if not self._is_no_op_ffn:
|
||||
stages.append(
|
||||
(
|
||||
declare_ffn(sparse=False, next_layer_sparse=False),
|
||||
self.post_attention_layernorm,
|
||||
)
|
||||
)
|
||||
norms.append(self.post_attention_layernorm)
|
||||
stages = list(zip(self.stage_facts(config, layer_idx), norms, strict=True))
|
||||
self.entry_boundary = None
|
||||
if stages:
|
||||
boundaries = append_stages(
|
||||
@@ -156,6 +151,20 @@ class DeciLMDecoderLayer(nn.Module):
|
||||
if not self._is_no_op_ffn:
|
||||
self.ffn_boundary = boundaries[-1]
|
||||
|
||||
@staticmethod
|
||||
def stage_facts(config: LlamaConfig, layer_idx: int):
|
||||
"""The stages a DeciLM layer declares, from its block config alone:
|
||||
the model's shared declaration function, which the layer declares
|
||||
with too (see make_layers)."""
|
||||
block_config = config.block_configs[layer_idx]
|
||||
# A layer whose attention and FFN are both no-ops has no stages.
|
||||
stages = []
|
||||
if not block_config.attention.no_op:
|
||||
stages.append(declare_attn())
|
||||
if not block_config.ffn.no_op:
|
||||
stages.append(declare_ffn(sparse=False, next_layer_sparse=False))
|
||||
return tuple(stages)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
positions: torch.Tensor,
|
||||
@@ -223,6 +232,7 @@ class DeciModel(nn.Module):
|
||||
config.num_hidden_layers,
|
||||
get_layer,
|
||||
prefix=add_prefix("layers", prefix),
|
||||
stage_facts=lambda idx: layer_type.stage_facts(config, idx),
|
||||
)
|
||||
if get_parallel().pp_group.is_last_rank:
|
||||
self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
|
||||
|
||||
@@ -299,15 +299,24 @@ class Qwen2DecoderLayer(nn.Module):
|
||||
)
|
||||
self.input_layernorm.fuse_input_quant(self.self_attn.qkv_proj)
|
||||
self.post_attention_layernorm.fuse_input_quant(self.mlp.gate_up_proj)
|
||||
attn, ffn = self.stage_facts(config, layer_id)
|
||||
self.attn_boundary, self.ffn_boundary = append_stages(
|
||||
(declare_attn(read=self._attn_readout(layer_id)), self.input_layernorm),
|
||||
(
|
||||
declare_ffn(sparse=False, next_layer_sparse=False),
|
||||
self.post_attention_layernorm,
|
||||
),
|
||||
(attn, self.input_layernorm),
|
||||
(ffn, self.post_attention_layernorm),
|
||||
)
|
||||
|
||||
def _attn_readout(self, layer_id: int):
|
||||
@classmethod
|
||||
def stage_facts(cls, config: Qwen2Config, layer_id: int):
|
||||
"""The stages a layer of this class declares, from the config alone:
|
||||
the model's shared declaration function, which the layer declares
|
||||
with too (see make_layers). A subclass's attention read takes part."""
|
||||
return (
|
||||
declare_attn(read=cls._attn_readout(layer_id)),
|
||||
declare_ffn(sparse=False, next_layer_sparse=False),
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def _attn_readout(cls, layer_id: int):
|
||||
return NORM_QUANT_READOUT
|
||||
|
||||
def forward(
|
||||
@@ -384,6 +393,7 @@ class Qwen2Model(nn.Module):
|
||||
alt_stream=alt_stream,
|
||||
),
|
||||
prefix=add_prefix("layers", prefix),
|
||||
stage_facts=lambda idx: decoder_layer_type.stage_facts(config, idx),
|
||||
)
|
||||
if self.pp_group.is_last_rank:
|
||||
norm_kwargs = (
|
||||
|
||||
@@ -55,7 +55,8 @@ class Qwen2DecoderLayer(Qwen2DecoderLayer):
|
||||
if layer_id == 0:
|
||||
del self.input_layernorm
|
||||
|
||||
def _attn_readout(self, layer_id: int):
|
||||
@classmethod
|
||||
def _attn_readout(cls, layer_id: int):
|
||||
if layer_id == 0:
|
||||
return PLAIN_READOUT
|
||||
return super()._attn_readout(layer_id)
|
||||
|
||||
@@ -1026,9 +1026,8 @@ class Qwen2MoeDecoderLayer(nn.Module):
|
||||
|
||||
self.layer_id = layer_id
|
||||
|
||||
# Qwen2MoE all layers are sparse and have no nextn now
|
||||
self.is_layer_sparse = True
|
||||
is_next_layer_sparse = True
|
||||
attn, ffn = self.stage_facts(config, layer_id)
|
||||
self.is_layer_sparse = ffn.sparse
|
||||
|
||||
if self.is_layer_sparse:
|
||||
self.mlp = Qwen2MoeSparseMoeBlock(
|
||||
@@ -1053,13 +1052,23 @@ class Qwen2MoeDecoderLayer(nn.Module):
|
||||
config.hidden_size, eps=config.rms_norm_eps
|
||||
)
|
||||
self.attn_boundary, self.ffn_boundary = append_stages(
|
||||
(declare_attn(), self.input_layernorm),
|
||||
(
|
||||
declare_ffn(
|
||||
sparse=self.is_layer_sparse,
|
||||
next_layer_sparse=is_next_layer_sparse,
|
||||
),
|
||||
self.post_attention_layernorm,
|
||||
(attn, self.input_layernorm),
|
||||
(ffn, self.post_attention_layernorm),
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def stage_facts(config: PretrainedConfig, layer_id: int):
|
||||
"""The stages a Qwen2-MoE layer declares, from the config alone: the
|
||||
model's shared declaration function, which the layer declares with
|
||||
too (see make_layers)."""
|
||||
# Qwen2MoE all layers are sparse and have no nextn now
|
||||
is_layer_sparse = True
|
||||
is_next_layer_sparse = True
|
||||
return (
|
||||
declare_attn(),
|
||||
declare_ffn(
|
||||
sparse=is_layer_sparse,
|
||||
next_layer_sparse=is_next_layer_sparse,
|
||||
),
|
||||
)
|
||||
|
||||
@@ -1138,6 +1147,7 @@ class Qwen2MoeModel(nn.Module):
|
||||
alt_stream=alt_stream,
|
||||
),
|
||||
prefix=add_prefix("layers", prefix),
|
||||
stage_facts=lambda idx: decoder_layer_type.stage_facts(config, idx),
|
||||
)
|
||||
if self.pp_group.is_last_rank:
|
||||
self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
|
||||
|
||||
@@ -373,12 +373,20 @@ class Qwen3DecoderLayer(nn.Module):
|
||||
config.hidden_size, eps=config.rms_norm_eps, **norm_kwargs
|
||||
)
|
||||
|
||||
attn, ffn = self.stage_facts(config, layer_id)
|
||||
self.attn_boundary, self.ffn_boundary = append_stages(
|
||||
(declare_attn(), self.input_layernorm),
|
||||
(
|
||||
declare_ffn(sparse=False, next_layer_sparse=False),
|
||||
self.post_attention_layernorm,
|
||||
),
|
||||
(attn, self.input_layernorm),
|
||||
(ffn, self.post_attention_layernorm),
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def stage_facts(config: Qwen3Config, layer_id: int):
|
||||
"""The stages a Qwen3 layer declares, from the config alone: the
|
||||
model's shared declaration function, which the layer declares with
|
||||
too (see make_layers)."""
|
||||
return (
|
||||
declare_attn(),
|
||||
declare_ffn(sparse=False, next_layer_sparse=False),
|
||||
)
|
||||
|
||||
def forward(
|
||||
|
||||
@@ -69,6 +69,7 @@ from sglang.srt.layers.layer_boundary.residual import batch as residual_batch
|
||||
|
||||
# Layers - Others
|
||||
from sglang.srt.layers.layer_boundary.residual.add_norm import (
|
||||
NORM_QUANT_READOUT,
|
||||
Fp8Input,
|
||||
NormQuantReadout,
|
||||
)
|
||||
@@ -252,6 +253,27 @@ def _layer_fusions(config: Qwen3_5TextConfig, is_nextn: bool):
|
||||
return None
|
||||
|
||||
|
||||
def _qwen3_5_is_moe(config: Qwen3_5TextConfig) -> bool:
|
||||
"""Whether the model's layers run sparse MoE blocks rather than dense MLPs:
|
||||
every layer of either kind alike."""
|
||||
return config.model_type in _QWEN3_5_MOE_TEXT_MODEL_TYPES
|
||||
|
||||
|
||||
def _qwen3_5_stage_facts(
|
||||
config: Qwen3_5TextConfig, layer_id: int, *, attn_read=NORM_QUANT_READOUT
|
||||
):
|
||||
"""The attention and FFN stages a Qwen3.5 layer of either kind declares:
|
||||
the model's shared declaration function (see make_layers), which the
|
||||
layer declares itself with. ``attn_read`` is the layer's own attention
|
||||
read, which declares what the plain one does and also hands on the
|
||||
quantized input its projection takes."""
|
||||
sparse = _qwen3_5_is_moe(config)
|
||||
return (
|
||||
declare_attn(read=attn_read),
|
||||
declare_ffn(sparse=sparse, next_layer_sparse=sparse),
|
||||
)
|
||||
|
||||
|
||||
if _is_cuda:
|
||||
from sglang.kernels.ops.attention.fused_qk_rmsnorm_rope_gate import (
|
||||
fused_qk_gemma_rmsnorm_rope_gate,
|
||||
@@ -1087,6 +1109,8 @@ class Qwen3_5GatedDeltaNet(nn.Module):
|
||||
class Qwen3_5LinearDecoderLayer(nn.Module):
|
||||
"""Qwen3.5 Decoder Layer with Linear Attention (GatedDeltaNet)."""
|
||||
|
||||
stage_facts = staticmethod(_qwen3_5_stage_facts)
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
config: Qwen3_5TextConfig,
|
||||
@@ -1107,7 +1131,7 @@ class Qwen3_5LinearDecoderLayer(nn.Module):
|
||||
|
||||
# NOTE: Determine the MLP type based on the model type
|
||||
# Qwen3.5 use all layers for MLP / Qwen3.5-MoE use sparse MoE blocks
|
||||
if config.model_type in _QWEN3_5_MOE_TEXT_MODEL_TYPES:
|
||||
if _qwen3_5_is_moe(config):
|
||||
self.mlp = Qwen2MoeSparseMoeBlock(
|
||||
layer_id=layer_id,
|
||||
config=config,
|
||||
@@ -1123,8 +1147,6 @@ class Qwen3_5LinearDecoderLayer(nn.Module):
|
||||
# The stage boundary completes this stage's sum.
|
||||
reduce_results=False,
|
||||
)
|
||||
is_layer_sparse = True
|
||||
is_next_layer_sparse = True
|
||||
elif config.model_type == "qwen3_5_text":
|
||||
self.mlp = Qwen2MoeMLP(
|
||||
hidden_size=config.hidden_size,
|
||||
@@ -1136,8 +1158,6 @@ class Qwen3_5LinearDecoderLayer(nn.Module):
|
||||
reduce_results=False,
|
||||
)
|
||||
_maybe_enable_silu_fp4_quant_fusion(self.mlp)
|
||||
is_layer_sparse = False
|
||||
is_next_layer_sparse = False
|
||||
else:
|
||||
raise ValueError(f"Invalid model type: {config.model_type}")
|
||||
|
||||
@@ -1152,28 +1172,16 @@ class Qwen3_5LinearDecoderLayer(nn.Module):
|
||||
boundary_fusions = _layer_fusions(config, is_nextn)
|
||||
# A subclass that brings its own residual builds the stages itself.
|
||||
if build_stages:
|
||||
attn, ffn = self.stage_facts(
|
||||
config,
|
||||
layer_id,
|
||||
attn_read=NormQuantReadout(
|
||||
fp8_input=Fp8Input.TUPLE_AND_BF16 if accepts_fp8_input else None
|
||||
),
|
||||
)
|
||||
self.attn_boundary, self.ffn_boundary = append_stages(
|
||||
(
|
||||
declare_attn(
|
||||
read=NormQuantReadout(
|
||||
fp8_input=Fp8Input.TUPLE_AND_BF16
|
||||
if accepts_fp8_input
|
||||
else None
|
||||
)
|
||||
),
|
||||
self.input_layernorm,
|
||||
{
|
||||
"fusions": boundary_fusions,
|
||||
},
|
||||
),
|
||||
(
|
||||
declare_ffn(
|
||||
sparse=is_layer_sparse,
|
||||
next_layer_sparse=is_next_layer_sparse,
|
||||
),
|
||||
self.post_attention_layernorm,
|
||||
{"fusions": boundary_fusions},
|
||||
),
|
||||
(attn, self.input_layernorm, {"fusions": boundary_fusions}),
|
||||
(ffn, self.post_attention_layernorm, {"fusions": boundary_fusions}),
|
||||
)
|
||||
|
||||
def forward(
|
||||
@@ -1216,6 +1224,8 @@ class Qwen3_5LinearDecoderLayer(nn.Module):
|
||||
class Qwen3_5AttentionDecoderLayer(nn.Module):
|
||||
"""Qwen3.5 Decoder Layer with Full Attention."""
|
||||
|
||||
stage_facts = staticmethod(_qwen3_5_stage_facts)
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
config: Qwen3_5TextConfig,
|
||||
@@ -1327,9 +1337,7 @@ class Qwen3_5AttentionDecoderLayer(nn.Module):
|
||||
# The stage boundary completes this stage's sum.
|
||||
reduce_results=False,
|
||||
)
|
||||
is_layer_sparse = False
|
||||
is_next_layer_sparse = False
|
||||
elif config.model_type in _QWEN3_5_MOE_TEXT_MODEL_TYPES:
|
||||
elif _qwen3_5_is_moe(config):
|
||||
self.mlp = Qwen2MoeSparseMoeBlock(
|
||||
layer_id=layer_id,
|
||||
config=config,
|
||||
@@ -1345,8 +1353,6 @@ class Qwen3_5AttentionDecoderLayer(nn.Module):
|
||||
# The stage boundary completes this stage's sum.
|
||||
reduce_results=False,
|
||||
)
|
||||
is_layer_sparse = True
|
||||
is_next_layer_sparse = True
|
||||
else:
|
||||
raise ValueError(f"Invalid model type: {config.model_type}")
|
||||
|
||||
@@ -1364,26 +1370,16 @@ class Qwen3_5AttentionDecoderLayer(nn.Module):
|
||||
boundary_fusions = _layer_fusions(config, is_nextn)
|
||||
# A subclass that brings its own residual builds the stages itself.
|
||||
if build_stages:
|
||||
attn, ffn = self.stage_facts(
|
||||
config,
|
||||
layer_id,
|
||||
attn_read=NormQuantReadout(
|
||||
fp8_input=Fp8Input.TUPLE if accepts_fp8_input else None
|
||||
),
|
||||
)
|
||||
self.attn_boundary, self.ffn_boundary = append_stages(
|
||||
(
|
||||
declare_attn(
|
||||
read=NormQuantReadout(
|
||||
fp8_input=Fp8Input.TUPLE if accepts_fp8_input else None
|
||||
)
|
||||
),
|
||||
self.input_layernorm,
|
||||
{
|
||||
"fusions": boundary_fusions,
|
||||
},
|
||||
),
|
||||
(
|
||||
declare_ffn(
|
||||
sparse=is_layer_sparse,
|
||||
next_layer_sparse=is_next_layer_sparse,
|
||||
),
|
||||
self.post_attention_layernorm,
|
||||
{"fusions": boundary_fusions},
|
||||
),
|
||||
(attn, self.input_layernorm, {"fusions": boundary_fusions}),
|
||||
(ffn, self.post_attention_layernorm, {"fusions": boundary_fusions}),
|
||||
)
|
||||
|
||||
self.alt_stream = alt_stream
|
||||
@@ -1752,6 +1748,10 @@ class Qwen3_5ForCausalLM(nn.Module):
|
||||
config.num_hidden_layers,
|
||||
get_layer,
|
||||
prefix=f"{prefix}.layers",
|
||||
# The layer type chosen for each index declares its stages.
|
||||
stage_facts=lambda idx: self.decoder_layer_types[
|
||||
config.layers_block_type[idx]
|
||||
].stage_facts(config, idx),
|
||||
)
|
||||
|
||||
self.flashinfer_mnnvl_cutedsl_fusion = None
|
||||
|
||||
@@ -853,9 +853,8 @@ class Qwen3MoeDecoderLayer(nn.Module):
|
||||
|
||||
self.layer_id = layer_id
|
||||
|
||||
# Qwen3MoE all layers are sparse and have no nextn now
|
||||
self.is_layer_sparse = True
|
||||
is_next_layer_sparse = True
|
||||
attn, ffn = self.stage_facts(config, layer_id)
|
||||
self.is_layer_sparse = ffn.sparse
|
||||
|
||||
if self.is_layer_sparse:
|
||||
self.mlp = Qwen3MoeSparseMoeBlock(
|
||||
@@ -879,13 +878,23 @@ class Qwen3MoeDecoderLayer(nn.Module):
|
||||
)
|
||||
|
||||
self.attn_boundary, self.ffn_boundary = append_stages(
|
||||
(declare_attn(), self.input_layernorm),
|
||||
(
|
||||
declare_ffn(
|
||||
sparse=self.is_layer_sparse,
|
||||
next_layer_sparse=is_next_layer_sparse,
|
||||
),
|
||||
self.post_attention_layernorm,
|
||||
(attn, self.input_layernorm),
|
||||
(ffn, self.post_attention_layernorm),
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def stage_facts(config: Qwen3MoeConfig, layer_id: int):
|
||||
"""The stages a Qwen3-MoE layer declares, from the config alone: the
|
||||
model's shared declaration function, which the layer declares with
|
||||
too (see make_layers)."""
|
||||
# Qwen3MoE all layers are sparse and have no nextn now
|
||||
is_layer_sparse = True
|
||||
is_next_layer_sparse = True
|
||||
return (
|
||||
declare_attn(),
|
||||
declare_ffn(
|
||||
sparse=is_layer_sparse,
|
||||
next_layer_sparse=is_next_layer_sparse,
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
@@ -81,6 +81,7 @@ from sglang.srt.models.qwen3_5 import (
|
||||
Qwen3_5ForCausalLM,
|
||||
Qwen3_5GatedDeltaNet,
|
||||
Qwen3_5LinearDecoderLayer,
|
||||
_qwen3_5_is_moe,
|
||||
)
|
||||
from sglang.srt.models.qwen3_vl import Qwen3VLForConditionalGeneration
|
||||
from sglang.srt.models.qwen4_exp_ple_table import (
|
||||
@@ -90,7 +91,6 @@ from sglang.srt.models.qwen4_exp_ple_table import (
|
||||
)
|
||||
from sglang.srt.runtime_context import get_parallel
|
||||
from sglang.srt.utils import get_bool_env_var, is_hip, logger
|
||||
from sglang.srt.utils.common import is_building_neighbour_layer
|
||||
|
||||
_use_aiter = get_bool_env_var("SGLANG_USE_AITER") and is_hip()
|
||||
|
||||
@@ -537,13 +537,6 @@ class Qwen4ExpPLEGroupedNorm(nn.Module):
|
||||
return (x_norm * weight).to(compute_dtype)
|
||||
|
||||
|
||||
def _offloads_ple(config) -> bool:
|
||||
"""Whether this layer keeps its PLE table in host memory. A pipeline
|
||||
neighbour layer is never loaded or run, so it keeps the table on the meta
|
||||
device and allocates no host memory or stream for it."""
|
||||
return bool(config.ple_offload_embedding) and not is_building_neighbour_layer()
|
||||
|
||||
|
||||
class Qwen4ExpNGramEmbedding(nn.Module):
|
||||
_MASK64 = (1 << 64) - 1
|
||||
_SPLITMIX_GAMMA = 0x9E3779B97F4A7C15
|
||||
@@ -615,7 +608,7 @@ class Qwen4ExpNGramEmbedding(nn.Module):
|
||||
and not self.use_attn_tp_ngram
|
||||
)
|
||||
ngram_prefix = f"{prefix}.ngram_embedding" if prefix else "ngram_embedding"
|
||||
offload_embedding = _offloads_ple(config)
|
||||
offload_embedding = bool(config.ple_offload_embedding)
|
||||
# Offload only needs this embedding's metadata: build it on meta so the
|
||||
# shard is never allocated on the device.
|
||||
with torch.device("meta") if offload_embedding else nullcontext():
|
||||
@@ -1098,7 +1091,9 @@ class Qwen4ExpPLELayer(nn.Module):
|
||||
bias=False,
|
||||
)
|
||||
nn.init.zeros_(self.conv1d.weight)
|
||||
self._prefetch_stream = torch.cuda.Stream() if _offloads_ple(config) else None
|
||||
self._prefetch_stream = (
|
||||
torch.cuda.Stream() if config.ple_offload_embedding else None
|
||||
)
|
||||
self._graph_prefetch_buffers = {}
|
||||
self._eager_prefetch_buffer = None
|
||||
self._prefetch_state = None
|
||||
@@ -1449,31 +1444,45 @@ def _has_ple(layer_id, config) -> bool:
|
||||
return (layer_id + 1) in config.ple_layer_ids
|
||||
|
||||
|
||||
def _build_qwen4_exp_stages(residual, *, sparse):
|
||||
"""The attention and FFN stage boundaries of one gated hyper-connection layer.
|
||||
def _qwen4_exp_stage_facts(config, layer_id, *, residual=None):
|
||||
"""The attention and FFN stages a gated hyper-connection layer of either
|
||||
kind declares: the model's shared declaration function (see make_layers),
|
||||
which the layer declares itself with. ``residual`` is the layer's own
|
||||
reads and updates of the streams; without it, the stages declare what
|
||||
those do. A PLE layer's attention read needs every row, for the PLE
|
||||
embedding is computed for all of them."""
|
||||
if residual is None:
|
||||
residual = GatedResidualState.facts(
|
||||
attn_reads_every_row=_has_ple(layer_id, config)
|
||||
)
|
||||
sparse = _qwen3_5_is_moe(config)
|
||||
return (
|
||||
declare_attn(read=residual.attn_readout, update=residual.attn_update),
|
||||
declare_ffn(
|
||||
sparse=sparse,
|
||||
next_layer_sparse=sparse,
|
||||
read=residual.ffn_readout,
|
||||
update=residual.ffn_update,
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def _build_qwen4_exp_stages(config, layer_id, residual):
|
||||
"""The attention and FFN stage boundaries of one gated hyper-connection
|
||||
layer, declared by _qwen4_exp_stage_facts with the layer's own reads and
|
||||
updates (``residual``).
|
||||
|
||||
Each read normalizes the streams itself, so neither stage binds a norm, and
|
||||
the writes are gated injections, so neither stage offers a fused
|
||||
add-and-norm candidate.
|
||||
"""
|
||||
return append_stages(
|
||||
(
|
||||
declare_attn(read=residual.attn_readout, update=residual.attn_update),
|
||||
None,
|
||||
),
|
||||
(
|
||||
declare_ffn(
|
||||
sparse=sparse,
|
||||
next_layer_sparse=sparse,
|
||||
read=residual.ffn_readout,
|
||||
update=residual.ffn_update,
|
||||
),
|
||||
None,
|
||||
),
|
||||
)
|
||||
attn, ffn = _qwen4_exp_stage_facts(config, layer_id, residual=residual)
|
||||
return append_stages((attn, None), (ffn, None))
|
||||
|
||||
|
||||
class Qwen4ExpLayerExtensionMixin:
|
||||
stage_facts = staticmethod(_qwen4_exp_stage_facts)
|
||||
|
||||
def _init_qwen4_exp_layer_extensions(
|
||||
self,
|
||||
config: Qwen4ExpTextConfig,
|
||||
@@ -1529,6 +1538,8 @@ class Qwen4ExpLayerExtensionMixin:
|
||||
use_combine=True,
|
||||
)
|
||||
self.attn_boundary, self.ffn_boundary = _build_qwen4_exp_stages(
|
||||
config,
|
||||
layer_id,
|
||||
GatedResidualState(
|
||||
expand=self._widen_streams,
|
||||
attn_mix=self._attn_mix,
|
||||
@@ -1536,9 +1547,8 @@ class Qwen4ExpLayerExtensionMixin:
|
||||
attn_combine=self._attn_combine,
|
||||
ffn_combine=self._ffn_combine,
|
||||
# The PLE embedding is computed for every row.
|
||||
attn_reads_every_row=self.ple is not None,
|
||||
attn_reads_every_row=_has_ple(layer_id, config),
|
||||
).residual_ops(),
|
||||
sparse=isinstance(self.mlp, Qwen2MoeSparseMoeBlock),
|
||||
)
|
||||
|
||||
from sglang.srt.layers.moe.qwen4_decode import prepare_qwen4_decode_comm
|
||||
|
||||
@@ -5,6 +5,7 @@
|
||||
|
||||
import math
|
||||
from enum import IntEnum, auto
|
||||
from functools import partial
|
||||
from typing import Any, Dict, Iterable, Optional, Tuple
|
||||
|
||||
import torch
|
||||
@@ -986,20 +987,7 @@ class SarvamMoEMLADecoderLayer(nn.Module):
|
||||
alt_stream=alt_stream,
|
||||
)
|
||||
|
||||
first_k_dense = getattr(config, "first_k_dense_replace", 1)
|
||||
moe_layer_freq = getattr(config, "moe_layer_freq", 1)
|
||||
has_moe = getattr(config, "num_experts", None) is not None
|
||||
self.is_layer_sparse = (
|
||||
has_moe
|
||||
and layer_id >= first_k_dense
|
||||
and (layer_id - first_k_dense) % moe_layer_freq == 0
|
||||
)
|
||||
is_next_layer_sparse = (
|
||||
has_moe
|
||||
and layer_id < config.num_hidden_layers - 1
|
||||
and (layer_id + 1) >= first_k_dense
|
||||
and (layer_id + 1 - first_k_dense) % moe_layer_freq == 0
|
||||
)
|
||||
self.is_layer_sparse = self._is_layer_sparse(config, layer_id)
|
||||
|
||||
if self.is_layer_sparse:
|
||||
self.mlp = SarvamMoESparseMoeBlock(
|
||||
@@ -1026,21 +1014,52 @@ class SarvamMoEMLADecoderLayer(nn.Module):
|
||||
config.hidden_size, eps=config.rms_norm_eps
|
||||
)
|
||||
|
||||
attn, ffn = self.stage_facts(config, layer_id)
|
||||
self.attn_boundary, self.ffn_boundary = append_stages(
|
||||
(
|
||||
declare_attn(),
|
||||
attn,
|
||||
self.input_layernorm,
|
||||
{"qkv_latent_func": self.self_attn.prepare_qkv_latent},
|
||||
),
|
||||
(
|
||||
declare_ffn(
|
||||
sparse=self.is_layer_sparse,
|
||||
next_layer_sparse=is_next_layer_sparse,
|
||||
),
|
||||
self.post_attention_layernorm,
|
||||
(ffn, self.post_attention_layernorm),
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def stage_facts(cls, config: PretrainedConfig, layer_id: int):
|
||||
"""The stages the layer at ``layer_id`` declares, in order: the model's
|
||||
shared declaration function, which the layer declares with too (see
|
||||
make_layers)."""
|
||||
return (
|
||||
declare_attn(),
|
||||
declare_ffn(
|
||||
sparse=cls._is_layer_sparse(config, layer_id),
|
||||
next_layer_sparse=cls._is_next_layer_sparse(config, layer_id),
|
||||
),
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _is_layer_sparse(config: PretrainedConfig, layer_id: int) -> bool:
|
||||
first_k_dense = getattr(config, "first_k_dense_replace", 1)
|
||||
moe_layer_freq = getattr(config, "moe_layer_freq", 1)
|
||||
has_moe = getattr(config, "num_experts", None) is not None
|
||||
return (
|
||||
has_moe
|
||||
and layer_id >= first_k_dense
|
||||
and (layer_id - first_k_dense) % moe_layer_freq == 0
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _is_next_layer_sparse(config: PretrainedConfig, layer_id: int) -> bool:
|
||||
first_k_dense = getattr(config, "first_k_dense_replace", 1)
|
||||
moe_layer_freq = getattr(config, "moe_layer_freq", 1)
|
||||
has_moe = getattr(config, "num_experts", None) is not None
|
||||
return (
|
||||
has_moe
|
||||
and layer_id < config.num_hidden_layers - 1
|
||||
and (layer_id + 1) >= first_k_dense
|
||||
and (layer_id + 1 - first_k_dense) % moe_layer_freq == 0
|
||||
)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
positions: torch.Tensor,
|
||||
@@ -1096,6 +1115,7 @@ class SarvamMLAModel(nn.Module):
|
||||
alt_stream=self.alt_stream,
|
||||
),
|
||||
prefix="model.layers",
|
||||
stage_facts=partial(SarvamMoEMLADecoderLayer.stage_facts, config),
|
||||
)
|
||||
|
||||
if self.pp_group.is_last_rank:
|
||||
|
||||
@@ -4,6 +4,7 @@ SGLang SDARModelLM (block diffusion / dLLM-style forward).
|
||||
"""
|
||||
|
||||
import logging
|
||||
from functools import partial
|
||||
from typing import Iterable, Optional, Tuple, Union
|
||||
|
||||
import torch
|
||||
@@ -291,14 +292,22 @@ class SDARBlock(nn.Module):
|
||||
prefix=add_prefix("mlp", prefix),
|
||||
)
|
||||
|
||||
attn, ffn = self.stage_facts(config, layer_id)
|
||||
self.attn_boundary, self.ffn_boundary = append_stages(
|
||||
(declare_attn(), self.input_layernorm),
|
||||
(
|
||||
declare_ffn(
|
||||
sparse=False,
|
||||
next_layer_sparse=False,
|
||||
),
|
||||
self.post_attention_layernorm,
|
||||
(attn, self.input_layernorm),
|
||||
(ffn, self.post_attention_layernorm),
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def stage_facts(config: PretrainedConfig, layer_id: int):
|
||||
"""The stages the layer at ``layer_id`` declares, in order: the model's
|
||||
shared declaration function, which the layer declares with too (see
|
||||
make_layers)."""
|
||||
return (
|
||||
declare_attn(),
|
||||
declare_ffn(
|
||||
sparse=False,
|
||||
next_layer_sparse=False,
|
||||
),
|
||||
)
|
||||
|
||||
@@ -361,6 +370,7 @@ class SDARModel(nn.Module):
|
||||
alt_stream=alt_stream,
|
||||
),
|
||||
prefix=add_prefix("layers", prefix),
|
||||
stage_facts=partial(SDARBlock.stage_facts, config),
|
||||
)
|
||||
if self.pp_group.is_last_rank:
|
||||
norm_kwargs = (
|
||||
|
||||
@@ -4,6 +4,7 @@ SGLang SDARMoeModelLM (block diffusion / dLLM-style forward) with MoE MLP.
|
||||
"""
|
||||
|
||||
import logging
|
||||
from functools import partial
|
||||
from typing import Iterable, Optional, Tuple, Union
|
||||
|
||||
import torch
|
||||
@@ -351,12 +352,20 @@ class SDARMoeBlock(nn.Module):
|
||||
prefix=add_prefix("mlp", prefix),
|
||||
)
|
||||
|
||||
attn, ffn = self.stage_facts(config, layer_id)
|
||||
self.attn_boundary, self.ffn_boundary = append_stages(
|
||||
(declare_attn(), self.input_layernorm),
|
||||
(
|
||||
declare_ffn(sparse=True, next_layer_sparse=True),
|
||||
self.post_attention_layernorm,
|
||||
),
|
||||
(attn, self.input_layernorm),
|
||||
(ffn, self.post_attention_layernorm),
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def stage_facts(config: PretrainedConfig, layer_id: int):
|
||||
"""The stages the layer at ``layer_id`` declares, in order: the model's
|
||||
shared declaration function, which the layer declares with too (see
|
||||
make_layers)."""
|
||||
return (
|
||||
declare_attn(),
|
||||
declare_ffn(sparse=True, next_layer_sparse=True),
|
||||
)
|
||||
|
||||
def forward(
|
||||
@@ -422,6 +431,7 @@ class SDARMoeModel(nn.Module):
|
||||
alt_stream=alt_stream,
|
||||
),
|
||||
prefix=add_prefix("layers", prefix),
|
||||
stage_facts=partial(SDARMoeBlock.stage_facts, config),
|
||||
)
|
||||
|
||||
if self.pp_group.is_last_rank:
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import logging
|
||||
from functools import partial
|
||||
from typing import Iterable, Optional, Tuple, Union
|
||||
|
||||
import torch
|
||||
@@ -267,12 +268,20 @@ class Spark2_5DecoderLayer(nn.Module):
|
||||
self.post_attention_layernorm = RMSNorm(
|
||||
config.hidden_size, eps=config.rms_norm_eps
|
||||
)
|
||||
attn, ffn = self.stage_facts(config, layer_id)
|
||||
self.attn_boundary, self.ffn_boundary = append_stages(
|
||||
(declare_attn(), self.input_layernorm),
|
||||
(
|
||||
declare_ffn(sparse=False, next_layer_sparse=False),
|
||||
self.post_attention_layernorm,
|
||||
),
|
||||
(attn, self.input_layernorm),
|
||||
(ffn, self.post_attention_layernorm),
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def stage_facts(config: Spark2_5Config, layer_id: int):
|
||||
"""The stages the layer at ``layer_id`` declares, in order: the model's
|
||||
shared declaration function, which the layer declares with too (see
|
||||
make_layers)."""
|
||||
return (
|
||||
declare_attn(),
|
||||
declare_ffn(sparse=False, next_layer_sparse=False),
|
||||
)
|
||||
|
||||
def forward(
|
||||
@@ -326,6 +335,7 @@ class Spark2_5Model(nn.Module):
|
||||
prefix=prefix,
|
||||
),
|
||||
prefix=add_prefix("layers", prefix),
|
||||
stage_facts=partial(Spark2_5DecoderLayer.stage_facts, config),
|
||||
)
|
||||
if self.pp_group.is_last_rank:
|
||||
self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
from functools import partial
|
||||
from typing import Any, Dict, Iterable, Optional, Tuple, Union
|
||||
|
||||
import torch
|
||||
@@ -422,11 +423,10 @@ class Step3p5DecoderLayer(nn.Module):
|
||||
rope_theta = config.rope_theta
|
||||
max_position_embeddings = config.max_position_embeddings
|
||||
head_dim = config.head_dim
|
||||
moe_layers_set = {int(x) for x in config.moe_layers_enum.split(",")}
|
||||
self.num_attention_heads = config.num_attention_heads
|
||||
self.num_key_value_heads = config.num_attention_groups
|
||||
self.is_moe_layer = layer_id in moe_layers_set
|
||||
self.is_next_layer_sparse = (layer_id + 1) in moe_layers_set
|
||||
self.is_moe_layer = self._is_moe_layer(config, layer_id)
|
||||
self.is_next_layer_sparse = self._is_moe_layer(config, layer_id + 1)
|
||||
num_hidden_layers = config.num_hidden_layers
|
||||
|
||||
if (
|
||||
@@ -519,19 +519,32 @@ class Step3p5DecoderLayer(nn.Module):
|
||||
config.hidden_size, eps=config.rms_norm_eps
|
||||
)
|
||||
|
||||
attn, ffn = self.stage_facts(config, layer_id)
|
||||
self.attn_boundary, self.ffn_boundary = append_stages(
|
||||
(declare_attn(), self.input_layernorm),
|
||||
(
|
||||
declare_ffn(
|
||||
sparse=self.is_moe_layer,
|
||||
next_layer_sparse=self.is_next_layer_sparse,
|
||||
),
|
||||
self.post_attention_layernorm,
|
||||
),
|
||||
(attn, self.input_layernorm),
|
||||
(ffn, self.post_attention_layernorm),
|
||||
)
|
||||
|
||||
self.layer_id = layer_id
|
||||
|
||||
@classmethod
|
||||
def stage_facts(cls, config: Step3p5Config, layer_id: int):
|
||||
"""The stages the layer at ``layer_id`` declares, in order: the model's
|
||||
shared declaration function, which the layer declares with too (see
|
||||
make_layers)."""
|
||||
return (
|
||||
declare_attn(),
|
||||
declare_ffn(
|
||||
sparse=cls._is_moe_layer(config, layer_id),
|
||||
next_layer_sparse=cls._is_moe_layer(config, layer_id + 1),
|
||||
),
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _is_moe_layer(config: Step3p5Config, layer_id: int) -> bool:
|
||||
moe_layers_set = {int(x) for x in config.moe_layers_enum.split(",")}
|
||||
return layer_id in moe_layers_set
|
||||
|
||||
def forward(
|
||||
self,
|
||||
positions: torch.Tensor,
|
||||
@@ -605,6 +618,7 @@ class Step3p5Model(nn.Module):
|
||||
alt_stream=alt_stream,
|
||||
),
|
||||
prefix=add_prefix("layers", prefix),
|
||||
stage_facts=partial(Step3p5DecoderLayer.stage_facts, config),
|
||||
)
|
||||
if self.pp_group.is_last_rank:
|
||||
self.norm = GemmaRMSNorm(config.hidden_size, eps=config.rms_norm_eps)
|
||||
|
||||
@@ -28,6 +28,7 @@
|
||||
|
||||
import math
|
||||
from contextlib import nullcontext
|
||||
from functools import partial
|
||||
from typing import Any, Dict, Iterable, Optional, Tuple, Union
|
||||
|
||||
import torch
|
||||
@@ -1433,15 +1434,7 @@ class XllmDecoderLayer(nn.Module):
|
||||
|
||||
self.layer_id = layer_id
|
||||
|
||||
# Determine if this layer is sparse (MoE) or dense
|
||||
mlp_only_layers = getattr(config, "mlp_only_layers", [])
|
||||
decoder_sparse_step = getattr(config, "decoder_sparse_step", 1)
|
||||
if (layer_id not in mlp_only_layers) and (
|
||||
config.num_experts > 0 and (layer_id + 1) % decoder_sparse_step == 0
|
||||
):
|
||||
self.is_layer_sparse = True
|
||||
else:
|
||||
self.is_layer_sparse = False
|
||||
self.is_layer_sparse = self._is_layer_sparse(config, layer_id)
|
||||
|
||||
is_mova_config = getattr(config, "num_values", 0) > 0
|
||||
is_mova_attention = is_mova_config and layer_id >= config.num_dense_layers
|
||||
@@ -1475,16 +1468,6 @@ class XllmDecoderLayer(nn.Module):
|
||||
prefix=add_prefix("self_attn", prefix),
|
||||
)
|
||||
|
||||
# Check neighbors for scatter modes
|
||||
def _is_sparse(lid):
|
||||
if lid < 0 or lid >= config.num_hidden_layers:
|
||||
return False
|
||||
return (lid not in mlp_only_layers) and (
|
||||
config.num_experts > 0 and (lid + 1) % decoder_sparse_step == 0
|
||||
)
|
||||
|
||||
is_next_layer_sparse = _is_sparse(layer_id + 1)
|
||||
|
||||
if self.is_layer_sparse:
|
||||
self.mlp = XllmSparseMoeBlock(
|
||||
layer_id=layer_id,
|
||||
@@ -1506,17 +1489,49 @@ class XllmDecoderLayer(nn.Module):
|
||||
|
||||
self.input_layernorm = _make_norm(config)
|
||||
self.post_attention_layernorm = _make_norm(config)
|
||||
attn, ffn = self.stage_facts(config, layer_id)
|
||||
self.attn_boundary, self.ffn_boundary = append_stages(
|
||||
(declare_attn(), self.input_layernorm),
|
||||
(
|
||||
declare_ffn(
|
||||
sparse=self.is_layer_sparse,
|
||||
next_layer_sparse=is_next_layer_sparse,
|
||||
),
|
||||
self.post_attention_layernorm,
|
||||
(attn, self.input_layernorm),
|
||||
(ffn, self.post_attention_layernorm),
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def stage_facts(cls, config: PretrainedConfig, layer_id: int):
|
||||
"""The stages the layer at ``layer_id`` declares, in order: the model's
|
||||
shared declaration function, which the layer declares with too (see
|
||||
make_layers)."""
|
||||
return (
|
||||
declare_attn(),
|
||||
declare_ffn(
|
||||
sparse=cls._is_layer_sparse(config, layer_id),
|
||||
next_layer_sparse=cls._is_next_layer_sparse(config, layer_id),
|
||||
),
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _is_layer_sparse(config: PretrainedConfig, layer_id: int) -> bool:
|
||||
# Determine if this layer is sparse (MoE) or dense
|
||||
mlp_only_layers = getattr(config, "mlp_only_layers", [])
|
||||
decoder_sparse_step = getattr(config, "decoder_sparse_step", 1)
|
||||
if (layer_id not in mlp_only_layers) and (
|
||||
config.num_experts > 0 and (layer_id + 1) % decoder_sparse_step == 0
|
||||
):
|
||||
return True
|
||||
else:
|
||||
return False
|
||||
|
||||
@staticmethod
|
||||
def _is_next_layer_sparse(config: PretrainedConfig, layer_id: int) -> bool:
|
||||
# Check neighbors for scatter modes
|
||||
mlp_only_layers = getattr(config, "mlp_only_layers", [])
|
||||
decoder_sparse_step = getattr(config, "decoder_sparse_step", 1)
|
||||
lid = layer_id + 1
|
||||
if lid < 0 or lid >= config.num_hidden_layers:
|
||||
return False
|
||||
return (lid not in mlp_only_layers) and (
|
||||
config.num_experts > 0 and (lid + 1) % decoder_sparse_step == 0
|
||||
)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
positions: torch.Tensor,
|
||||
@@ -1576,6 +1591,7 @@ class XllmModel(nn.Module):
|
||||
prefix=prefix,
|
||||
),
|
||||
prefix=add_prefix("layers", prefix),
|
||||
stage_facts=partial(XllmDecoderLayer.stage_facts, config),
|
||||
)
|
||||
if self.pp_group.is_last_rank:
|
||||
self.norm = _make_norm(config)
|
||||
|
||||
@@ -1317,11 +1317,16 @@ class ZayaDecoderATTLayer(nn.Module):
|
||||
self.res_scale = ResidualScaling(config, layer_id)
|
||||
else:
|
||||
self.res_scale = None
|
||||
(self.attn_boundary,) = append_stages(
|
||||
(declare_attn(read=_ResidualMergeRead(self.res_scale)), self.input_norm),
|
||||
)
|
||||
(attn,) = self.stage_facts(res_scale=self.res_scale)
|
||||
(self.attn_boundary,) = append_stages((attn, self.input_norm))
|
||||
self.entry_boundary = self.attn_boundary
|
||||
|
||||
@staticmethod
|
||||
def stage_facts(*, res_scale: Optional[ResidualScaling] = None):
|
||||
"""The one stage an attention layer declares: its attention, read
|
||||
through ``res_scale``."""
|
||||
return (declare_attn(read=_ResidualMergeRead(res_scale)),)
|
||||
|
||||
@staticmethod
|
||||
def _build_norm(config: ZayaConfig) -> nn.Module:
|
||||
if config.normalization == "RMSNorm":
|
||||
@@ -1368,17 +1373,21 @@ class ZayaDecoderMLPLayer(nn.Module):
|
||||
self.res_scale = ResidualScaling(config, layer_id)
|
||||
else:
|
||||
self.res_scale = None
|
||||
(self.ffn_boundary,) = append_stages(
|
||||
(
|
||||
declare_ffn(
|
||||
sparse=True,
|
||||
next_layer_sparse=True,
|
||||
read=_ResidualMergeRead(self.res_scale),
|
||||
),
|
||||
self.input_norm,
|
||||
(ffn,) = self.stage_facts(res_scale=self.res_scale)
|
||||
(self.ffn_boundary,) = append_stages((ffn, self.input_norm))
|
||||
self.entry_boundary = self.ffn_boundary
|
||||
|
||||
@staticmethod
|
||||
def stage_facts(*, res_scale: Optional[ResidualScaling] = None):
|
||||
"""The one stage an MoE layer declares: its MoE, read through
|
||||
``res_scale``."""
|
||||
return (
|
||||
declare_ffn(
|
||||
sparse=True,
|
||||
next_layer_sparse=True,
|
||||
read=_ResidualMergeRead(res_scale),
|
||||
),
|
||||
)
|
||||
self.entry_boundary = self.ffn_boundary
|
||||
|
||||
def forward(
|
||||
self,
|
||||
@@ -1400,23 +1409,22 @@ class ZayaDecoderMLPLayer(nn.Module):
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _layer_class(layer_id: int):
|
||||
# Even layer ids are attention, odd layer ids are MoE. This matches the HF
|
||||
# checkpoint keys: ``model.layers.<2k>.self_attn.*`` (CCA) versus
|
||||
# ``model.layers.<2k+1>.zaya_block.*`` (MoE).
|
||||
if layer_id % 2 == 0:
|
||||
return ZayaDecoderATTLayer
|
||||
return ZayaDecoderMLPLayer
|
||||
|
||||
|
||||
def _build_layer(
|
||||
layer_id: int,
|
||||
config: ZayaConfig,
|
||||
quant_config: Optional[QuantizationConfig],
|
||||
prefix: str,
|
||||
) -> nn.Module:
|
||||
# Even layer ids are attention, odd layer ids are MoE. This matches the HF
|
||||
# checkpoint keys: ``model.layers.<2k>.self_attn.*`` (CCA) versus
|
||||
# ``model.layers.<2k+1>.zaya_block.*`` (MoE).
|
||||
if layer_id % 2 == 0:
|
||||
return ZayaDecoderATTLayer(
|
||||
config=config,
|
||||
layer_id=layer_id,
|
||||
quant_config=quant_config,
|
||||
prefix=prefix,
|
||||
)
|
||||
return ZayaDecoderMLPLayer(
|
||||
return _layer_class(layer_id)(
|
||||
config=config,
|
||||
layer_id=layer_id,
|
||||
quant_config=quant_config,
|
||||
@@ -1424,6 +1432,13 @@ def _build_layer(
|
||||
)
|
||||
|
||||
|
||||
def stage_facts(layer_id: int):
|
||||
"""The one stage the layer at ``layer_id`` declares, from its kind alone:
|
||||
the model's shared declaration function, which its layers declare with
|
||||
too (see make_layers)."""
|
||||
return _layer_class(layer_id).stage_facts()
|
||||
|
||||
|
||||
class ZayaModel(nn.Module):
|
||||
def __init__(
|
||||
self,
|
||||
@@ -1456,6 +1471,7 @@ class ZayaModel(nn.Module):
|
||||
prefix=prefix,
|
||||
),
|
||||
prefix=add_prefix("layers", prefix),
|
||||
stage_facts=stage_facts,
|
||||
)
|
||||
|
||||
if self.pp_group.is_last_rank:
|
||||
|
||||
@@ -1551,33 +1551,6 @@ def mark_end(name):
|
||||
time_infos[name].pretty_print()
|
||||
|
||||
|
||||
# Set while a layer another pipeline stage holds is built, only for the stage
|
||||
# boundaries it declares.
|
||||
_building_neighbour_layer = False
|
||||
|
||||
|
||||
def is_building_neighbour_layer() -> bool:
|
||||
"""Whether the layer under construction belongs to another pipeline stage
|
||||
and is built here only to read the stage boundaries it declares. It is
|
||||
built on the meta device and never loaded or run, so its constructor skips
|
||||
the host and device resources a running layer needs: tables, streams,
|
||||
engines and communicators."""
|
||||
return _building_neighbour_layer
|
||||
|
||||
|
||||
@contextmanager
|
||||
def building_neighbour_layer():
|
||||
"""Build a pipeline neighbour layer: on the meta device, with
|
||||
is_building_neighbour_layer() true."""
|
||||
global _building_neighbour_layer
|
||||
outer, _building_neighbour_layer = _building_neighbour_layer, True
|
||||
try:
|
||||
with torch.device("meta"):
|
||||
yield
|
||||
finally:
|
||||
_building_neighbour_layer = outer
|
||||
|
||||
|
||||
class LayerFn(Protocol):
|
||||
def __call__(self, idx: int, prefix: str) -> torch.nn.Module: ...
|
||||
|
||||
@@ -1601,10 +1574,9 @@ def make_layers(
|
||||
``stage_facts``: the model's shared declaration function, which returns
|
||||
the stages the layer at a global index declares without building it (an
|
||||
empty sequence when it declares none), and which the layer itself uses
|
||||
for its own declarations. A model on the transitional list may omit it;
|
||||
the stack then builds the layer again on the meta device to read them.
|
||||
``final_read`` is the stack's terminal read when it is not a plain final
|
||||
norm (see layer_stack).
|
||||
for its own declarations. A model whose layers declare stages must give
|
||||
it to run under pipeline parallelism. ``final_read`` is the stack's
|
||||
terminal read when it is not a plain final norm (see layer_stack).
|
||||
"""
|
||||
# circular imports
|
||||
from sglang.srt.distributed import get_pp_indices
|
||||
@@ -1628,10 +1600,8 @@ def make_layers(
|
||||
|
||||
def neighbour(idx):
|
||||
if stage_facts is not None:
|
||||
return functools.partial(_declared_stages, stage_facts, idx)
|
||||
return functools.partial(
|
||||
_build_neighbour_layer, layer_fn, idx, add_prefix(idx, prefix)
|
||||
)
|
||||
return functools.partial(stage_facts, idx)
|
||||
return functools.partial(_no_stage_facts, layer_fn)
|
||||
|
||||
def build(stack, idx):
|
||||
appended = len(stack.appends)
|
||||
@@ -1690,84 +1660,15 @@ def make_pp_layers(
|
||||
)
|
||||
|
||||
|
||||
# The models whose layers a pipeline rank still builds on the meta device to
|
||||
# read the stages they declare, until each gives make_layers its shared
|
||||
# declaration function (stage_facts). Any other model must give one to run
|
||||
# under pipeline parallelism.
|
||||
_NEIGHBOUR_BUILD_MODULES = frozenset(
|
||||
(
|
||||
"sglang.srt.models.apertus",
|
||||
"sglang.srt.models.arcee",
|
||||
"sglang.srt.models.bailing_moe",
|
||||
"sglang.srt.models.bailing_moe_linear",
|
||||
"sglang.srt.models.bailing_moe_v3",
|
||||
"sglang.srt.models.deepseek_v2",
|
||||
"sglang.srt.models.dots3_common.modeling",
|
||||
"sglang.srt.models.ernie45_moe_vl",
|
||||
"sglang.srt.models.exaone4",
|
||||
"sglang.srt.models.exaone_moe",
|
||||
"sglang.srt.models.gemma4_causal",
|
||||
"sglang.srt.models.gigachat35",
|
||||
"sglang.srt.models.glm4",
|
||||
"sglang.srt.models.glm4_moe",
|
||||
"sglang.srt.models.glm4_moe_lite",
|
||||
"sglang.srt.models.glm5_next",
|
||||
"sglang.srt.models.gpt_oss",
|
||||
"sglang.srt.models.granitemoehybrid",
|
||||
"sglang.srt.models.kimi_k3",
|
||||
"sglang.srt.models.kimi_linear",
|
||||
"sglang.srt.models.laguna",
|
||||
"sglang.srt.models.llada2",
|
||||
"sglang.srt.models.llama",
|
||||
"sglang.srt.models.mimo_v2",
|
||||
"sglang.srt.models.minimax_m2",
|
||||
"sglang.srt.models.minimax_m3",
|
||||
"sglang.srt.models.ministral3",
|
||||
"sglang.srt.models.mixtral",
|
||||
"sglang.srt.models.nanbeige",
|
||||
"sglang.srt.models.nemotron_nas",
|
||||
"sglang.srt.models.qwen2",
|
||||
"sglang.srt.models.qwen2_moe",
|
||||
"sglang.srt.models.qwen3_5",
|
||||
"sglang.srt.models.sarvam_moe",
|
||||
"sglang.srt.models.sdar",
|
||||
"sglang.srt.models.sdar_moe",
|
||||
"sglang.srt.models.spark2_5",
|
||||
"sglang.srt.models.step3p5",
|
||||
"sglang.srt.models.xllm",
|
||||
"sglang.srt.models.zaya",
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def _declared_stages(stage_facts, idx: int):
|
||||
"""The stages the layer at ``idx`` declares, from the model's shared
|
||||
declaration function, standing in for building it."""
|
||||
from sglang.srt.layers.layer_boundary.factories import DeclaredStages
|
||||
|
||||
return DeclaredStages(tuple(stage_facts(idx)))
|
||||
|
||||
|
||||
def _build_neighbour_layer(layer_fn: LayerFn, idx: int, prefix: str) -> None:
|
||||
"""Build a layer another pipeline stage holds, only for the stage
|
||||
boundaries it declares (see building_neighbour_layer). RoPE modules it
|
||||
adds to the shared cache are meta, so they are dropped again."""
|
||||
from sglang.srt.layers.rotary_embedding.factory import _ROPE_DICT
|
||||
|
||||
def _no_stage_facts(layer_fn: LayerFn) -> None:
|
||||
"""Stands in for the shared declaration function of a model that gives
|
||||
none: called only if this pipeline rank's layers declare stages."""
|
||||
module = getattr(getattr(layer_fn, "func", layer_fn), "__module__", None)
|
||||
if module not in _NEIGHBOUR_BUILD_MODULES:
|
||||
raise ValueError(
|
||||
f"{module} declares stage boundaries under pipeline parallelism "
|
||||
"without a shared declaration function: give make_layers its "
|
||||
"stage_facts, the stages a layer declares without building it"
|
||||
)
|
||||
cached = set(_ROPE_DICT)
|
||||
try:
|
||||
with building_neighbour_layer():
|
||||
layer_fn(idx=idx, prefix=prefix)
|
||||
finally:
|
||||
for key in set(_ROPE_DICT) - cached:
|
||||
del _ROPE_DICT[key]
|
||||
raise ValueError(
|
||||
f"{module} declares stage boundaries under pipeline parallelism "
|
||||
"without a shared declaration function: give make_layers its "
|
||||
"stage_facts, the stages a layer declares without building it"
|
||||
)
|
||||
|
||||
|
||||
def set_random_seed(seed: int) -> None:
|
||||
|
||||
@@ -28,19 +28,13 @@ def build_stages(*stages, previous=None, terminal=False):
|
||||
terminal: Whether the last stage ends the model's layer stack;
|
||||
otherwise the next layer's attention follows it on the same rank.
|
||||
"""
|
||||
# The stack's neighbour hooks build layers another pipeline rank holds;
|
||||
# here they stand for this rank's, so nothing is handed off to them.
|
||||
# The stack's neighbours are layers another pipeline rank holds; here
|
||||
# they stand for this rank's, so nothing is handed off to them.
|
||||
with (
|
||||
mock.patch.object(factories, "_handed_off", lambda declaration: declaration),
|
||||
layer_stack(
|
||||
previous_layers=(
|
||||
[lambda: append_stages((previous, None))]
|
||||
if previous is not None
|
||||
else []
|
||||
),
|
||||
next_layers=(
|
||||
[] if terminal else [lambda: append_stages((declare_attn(), None))]
|
||||
),
|
||||
previous_layers=[lambda: (previous,)] if previous is not None else [],
|
||||
next_layers=[] if terminal else [lambda: (declare_attn(),)],
|
||||
),
|
||||
):
|
||||
return append_stages(*stages)
|
||||
|
||||
Reference in New Issue
Block a user