[Refactor] Build layer stacks in order with append_stages (#42481)

This commit is contained in:
Cheng Wan
2026-10-04 16:40:24 -07:00
committed by GitHub
parent a99567ab74
commit 1f299e9f2f
96 changed files with 1848 additions and 1293 deletions
+2 -2
View File
@@ -338,8 +338,8 @@ class XIELU(BaseFusedOp):
self.register_buffer("eps", torch.tensor(eps, dtype=dtype))
self.with_vector_loads = with_vector_loads
# Temporary until xIELU CUDA fully implemented
self._beta_scalar = float(self.beta.detach().cpu().float().item())
self._eps_scalar = float(self.eps.detach().cpu().float().item())
self._beta_scalar = torch.tensor(beta, dtype=dtype, device="cpu").item()
self._eps_scalar = torch.tensor(eps, dtype=dtype, device="cpu").item()
self._xielu_cuda_obj = None
try:
@@ -139,14 +139,12 @@ class GatedResidual(HyperConnectionBase):
self.hidden_size * self.hc_count,
self.config.hc_lowrank,
bias=False,
device=torch.get_device_module().current_device(),
dtype=config.params_dtype,
)
self.input_mix_weight_up = nn.Linear(
self.config.hc_lowrank,
self.hc_count * self.hidden_size,
bias=False,
device=torch.get_device_module().current_device(),
dtype=config.params_dtype,
)
lowrank = self.config.hc_lowrank
@@ -166,7 +164,6 @@ class GatedResidual(HyperConnectionBase):
self.hidden_size * self.hc_count,
self.hc_count,
bias=False,
device=torch.get_device_module().current_device(),
dtype=config.params_dtype,
)
# hc_combine rejects other shapes; device and dtype are checked per call.
@@ -40,11 +40,10 @@ from sglang.srt.layers.layer_boundary.contracts import (
)
from sglang.srt.layers.layer_boundary.exit import ExitDecision, FfnExit, MixerExit
from sglang.srt.layers.layer_boundary.factories import (
append_stages,
declare_attn,
declare_ffn,
make_attn_stage,
make_ffn_stage,
make_stages,
layer_stack,
)
from sglang.srt.layers.layer_boundary.layout import (
Layout,
@@ -85,9 +84,8 @@ from sglang.srt.layers.layer_boundary.residual.mhc import (
__all__ = [
"declare_attn",
"declare_ffn",
"make_attn_stage",
"make_ffn_stage",
"make_stages",
"append_stages",
"layer_stack",
"PLAIN_ADD",
"AttentionInputs",
"StagePath",
@@ -145,8 +145,11 @@ class StagePlan:
finishes_directly: Attention can publish its output with finish instead of
an exit scope and output transport.
qkv_latent_func: Optional hook for prepared attention input.
fusions: Optional backend provider of ordered consumer fusion candidates
and the producer deferral policies (see make_attn_stage).
fusions: Optional backend provider. Consumer side: ordered
attention_input(plan) and ffn_input(plan) candidates. Producer side
(FFN exit): can_defer_finalize(plan, batch), called on every exit,
and can_defer_all_reduce(plan, batch), called when LoRA or TP1
shared experts are enabled.
The plan owns static paths, not per-forward tensors or a neighbour's norm.
Runtime residual state belongs to the ForwardBatch's ResidualStream.
@@ -2,6 +2,9 @@
from __future__ import annotations
import contextlib
import os
import sys
from dataclasses import dataclass, replace
from types import MappingProxyType
from typing import Mapping, Optional
@@ -48,6 +51,7 @@ from sglang.srt.layers.layer_boundary.residual.add_norm import (
PLAIN_ADD,
REPLACE_AT_EXIT,
)
from sglang.srt.layers.layer_boundary.stage import StageBoundary
from sglang.srt.layers.moe import is_moe_input_scattered_across_dp_ranks
from sglang.srt.runtime_context import get_exec, get_parallel
@@ -194,9 +198,10 @@ class StageDeclaration:
update: Producer operation that writes compute output into the residual.
sparse: Whether this FFN is a MoE; used to resolve input/output rows.
Expert routing and all-to-all remain inside the MoE computation.
terminal: Whether this stage ends the model's layer stack. Prevents
leaving work that requires a following layer; a finalize handoff
may still reach the terminal norm when the fusion provider allows it.
terminal: Whether this stage ends the model's layer stack, as the
stack records it. Prevents leaving work that requires a following
layer; a finalize handoff may still reach the terminal norm when
the fusion provider allows it.
output_transform: Optional operation on the contribution before the
residual update. An FFN's exit runs it under an explicit
reduction-order contract; an attention's is run by the input of
@@ -207,8 +212,8 @@ class StageDeclaration:
dense_tp_size: Dense FFN compute width: None uses the configured width,
1 means local compute, and the full TP size means TP compute.
exit_rows: Required FFN output rows at the layer or branch exit.
previous: Declaration whose output this stage consumes. It may be
reconstructed locally, including across pipeline ranks.
previous: Declaration whose output this stage consumes, as the stack
records it; across pipeline ranks it is built locally.
prepared_from: Declaration whose already-read input a branch reuses.
Mutually exclusive with previous; avoids a second update/read.
@@ -263,11 +268,8 @@ class StageConnection:
def declare_attn(
*,
previous: Optional[StageDeclaration] = None,
prepared_from: Optional[StageDeclaration] = None,
read=NORM_QUANT_READOUT,
update=PLAIN_ADD,
terminal=False,
reduction=ProducerReduction.ALWAYS_PARTIAL,
gathers_attn_tp_input=True,
output_transform=None,
@@ -275,11 +277,8 @@ def declare_attn(
"""Declare attention or a mixer; construct its executable boundary later.
Args:
previous: Producer declaration whose output this stage consumes.
prepared_from: Source of an already-read branch input, instead of previous.
read: Input operation; defaults to normalization with quantization support.
update: Operation that adds this stage's output to the residual.
terminal: Whether this stage ends the model's layer stack.
reduction: ALWAYS_PARTIAL for an attention whose sum the next stage's
input always completes; EXIT_SCOPED for a mixer whose exit decides.
gathers_attn_tp_input: Whether compute gathers attention-TP input slices itself.
@@ -300,9 +299,6 @@ def declare_attn(
StageKind.ATTENTION,
read,
update,
previous=previous,
prepared_from=prepared_from,
terminal=terminal,
output_transform=output_transform,
reduction=reduction,
gathers_attn_tp_input=gathers_attn_tp_input,
@@ -311,12 +307,9 @@ def declare_attn(
def declare_ffn(
*,
previous: Optional[StageDeclaration] = None,
prepared_from: Optional[StageDeclaration] = None,
sparse=False,
read=NORM_READOUT,
update=PLAIN_ADD,
terminal=False,
output_transform=None,
next_layer_sparse=False,
dense_tp_size=None,
@@ -325,12 +318,9 @@ def declare_ffn(
"""Declare a dense or MoE FFN independently of its compute module.
Args:
previous: Producer declaration whose output this stage consumes.
prepared_from: Source of an already-read branch input, instead of previous.
sparse: Whether the FFN is a MoE; EP dispatch stays inside compute.
read: Operation deriving FFN input from the residual.
update: Operation writing FFN output into the residual.
terminal: Whether this stage ends the model's layer stack.
output_transform: Optional contribution transform before residual update.
next_layer_sparse: Whether the next decoder layer's FFN is sparse; used only
to derive the TBO exit rows when exit_rows is not supplied.
@@ -347,9 +337,6 @@ def declare_ffn(
read,
update,
sparse=sparse,
previous=previous,
prepared_from=prepared_from,
terminal=terminal,
output_transform=output_transform,
dense_tp_size=dense_tp_size,
exit_rows=exit_rows or tbo_exit_rows(sparse, next_layer_sparse),
@@ -569,167 +556,364 @@ def _incoming(stage):
)
def _connections(stage, following=None):
if following is not None and following.previous != stage:
raise ValueError("the following declaration must consume this stage's output")
incoming = _incoming(stage)
outgoing = _connect(
stage,
following,
residual_from=incoming,
)
return incoming, outgoing
class _Append:
"""One append_stages call: its validated stages, the boundaries it
returned, which the stack fills in when it closes, and where it was made."""
__slots__ = ("declarations", "bindings", "prepared_from", "boundaries", "origin")
def __init__(self, declarations, bindings, prepared_from, boundaries, origin):
self.declarations = declarations
self.bindings = bindings
self.prepared_from = prepared_from
self.boundaries = boundaries
self.origin = origin
def make_attn_stage(
*,
declaration,
norm,
following: Optional[StageDeclaration] = None,
qkv_latent_func=None,
fusions=None,
):
"""Resolve one attention/mixer's boundaries and bind its input norm.
class _LayerStack:
"""A layer stack under construction: what was appended, in order, and how
to reach the layers other pipeline ranks hold on either side of it."""
__slots__ = ("appends", "previous_layers", "next_layers")
def __init__(self, previous_layers=(), next_layers=()):
self.appends = []
self.previous_layers = previous_layers
self.next_layers = next_layers
# The stack being built; layer_stack saves and restores an outer one.
_stack: Optional[_LayerStack] = None
@contextlib.contextmanager
def layer_stack(*, previous_layers=(), next_layers=()):
"""Open a layer stack that append_stages extends in order.
Every stage binds when the stack closes, once its producer and its
consumer are both known: the stage appended before and after it. The last
stage ends the model's layer stack unless a later layer declares a stage.
Args:
declaration: Attention StageDeclaration, including its input source.
norm: This consumer's normalization module, never its neighbour's norm.
following: Local consumer declaration whose previous is declaration.
None leaves a layer or stack exit for an independently bound reader.
qkv_latent_func: Optional attention input hook, invoked after preparation
and movement onto the compute input rows.
fusions: Optional backend provider. Consumer side: ordered
attention_input(plan) and ffn_input(plan) candidates. Producer side
(FFN exit): can_defer_finalize(plan, batch), called on every exit,
and can_defer_all_reduce(plan, batch), called when LoRA or TP1 shared
experts are enabled.
Returns:
A StageBoundary with precomputed paths for supported batch variants.
previous_layers: Callables that build, nearest first, the layers before
this stack that another pipeline rank holds. Called only if this
stack appended stages, after its own layers are built, until one of
them declares a stage: its last stage is the producer of this
stack's first. What they build is discarded.
next_layers: Likewise for the layers after this stack, whose first
declared stage is the consumer of this stack's last.
"""
if declaration.kind is not StageKind.ATTENTION:
raise TypeError("make_attn_stage requires an attention declaration")
incoming, outgoing = _connections(declaration, following)
return _bind_stage(
global _stack
outer = _stack
stack = _LayerStack(previous_layers, next_layers)
_stack = stack
try:
yield stack
if stack.appends:
_bind_stack(
stack.appends,
previous=_neighbour_stage(stack.previous_layers, last=True),
following=_neighbour_stage(stack.next_layers, last=False),
)
finally:
_stack = outer
def _neighbour_stage(build_layers, *, last):
"""The stage a neighbouring layer declares next to this stack: the last
one of the nearest layer before it, or the first of the nearest after.
Branches are side paths, so they never stand next to the stack."""
global _stack
for build_layer in build_layers:
outer = _stack
_stack = _LayerStack()
try:
build_layer()
appends = [a for a in _stack.appends if a.prepared_from is None]
finally:
_stack = outer
if appends:
return appends[-1].declarations[-1] if last else appends[0].declarations[0]
return None
def _require_stack() -> _LayerStack:
if _stack is None:
raise RuntimeError(
"append_stages needs an open layer stack; build the layers inside "
"make_layers or layer_stack"
)
return _stack
def _note_origin(error: Exception, origin: str) -> None:
add_note = getattr(error, "add_note", None)
if add_note is not None:
add_note(f"while binding the stages appended at {origin}")
class _PendingStage:
"""The last stage of a linear append. Its consumer is the next append's
first stage, or the stack exit, so it binds only once that is known."""
__slots__ = (
"boundary",
"declaration",
"norm",
"options",
"predecessor",
"predecessor_incoming",
"predecessor_boundary",
"origin",
)
def __init__(
self,
boundary,
declaration,
norm,
incoming,
outgoing,
qkv_latent_func=qkv_latent_func,
fusions=fusions,
)
options,
predecessor,
predecessor_incoming,
predecessor_boundary,
origin,
):
self.boundary = boundary
self.declaration = declaration
self.norm = norm
self.options = options
# The stage before it within the same append, or None when it is the
# append's only stage and takes its input from the stack.
self.predecessor = predecessor
self.predecessor_incoming = predecessor_incoming
self.predecessor_boundary = predecessor_boundary
self.origin = origin
def make_ffn_stage(
*,
declaration,
norm,
following: Optional[StageDeclaration] = None,
fusions=None,
):
"""Resolve one FFN's boundaries and bind its input norm.
class _Chain:
"""The stage a following append extends, and the one stage still waiting
for its consumer, while the stack binds its appends in order."""
Args:
declaration: FFN StageDeclaration, including its input source.
norm: This FFN's input normalization module.
following: Local consumer declaration whose previous is declaration;
None leaves a layer or stack exit for an independently bound reader.
fusions: Optional backend fusion provider, as in make_attn_stage.
__slots__ = ("previous", "pending")
Returns:
A StageBoundary. Expert routing and all-to-all stay inside compute.
def __init__(self, previous):
self.previous = _detached(previous)
self.pending = None
def _detached(declaration):
"""The declaration a following append extends, without its own history.
A stage's incoming edge reads its producer's own incoming edge only when
the producer is an attention that always leaves its sum; the copy has no
history, so that lookback stops at the producer.
"""
if declaration.kind is not StageKind.FFN:
raise TypeError("make_ffn_stage requires an FFN declaration")
incoming, outgoing = _connections(declaration, following)
return _bind_stage(
declaration,
norm,
incoming,
outgoing,
fusions=fusions,
)
if declaration is None:
return None
return replace(declaration, previous=None, prepared_from=None)
def make_stages(
*stages, previous=None, prepared_from=None, following=None, terminal=False
):
"""Bind a local linear sequence of any positive number of stages.
def _bind_stack(appends, *, previous, following):
"""Bind every appended stage, in order, and fill in the boundaries each
append returned."""
chain = _Chain(previous)
# A returned declaration's boundary as bound, for the branches that read it.
sources = {}
bound = []
for append in appends:
prepared_from = append.prepared_from
if prepared_from is not None:
source = sources.get(id(prepared_from))
if source is None:
raise ValueError(
"prepared_from must be the declaration of a stage appended "
f"earlier to the same layer stack (at {append.origin})"
)
prepared_from = source.declaration
boundaries = _extend(chain, append, prepared_from)
for returned, boundary in zip(append.boundaries, boundaries):
sources[id(returned.declaration)] = boundary
bound.append(boundaries)
# The last stage's consumer is the next rank's first stage, if any;
# without one it ends the model's layer stack.
_bind_pending(chain, consumer=following, terminal=following is None)
for append, boundaries in zip(appends, bound):
for returned, boundary in zip(append.boundaries, boundaries):
returned.plan = boundary.plan
returned.declaration = boundary.declaration
Args:
*stages: Items of (declaration, norm) or (declaration, norm, options).
Declarations must have no source or terminal flag. Options are
constructor keywords: fusions, and qkv_latent_func for attention.
previous: External producer declaration consumed by the first stage.
prepared_from: Already-read input declaration reused by the first stage
of a branch; mutually exclusive with previous.
following: External consumer declaration after the last local stage.
None denotes a layer or stack exit with an independently bound read.
terminal: Marks only the final stage as the end of the model's stack.
Returns:
A tuple of independent StageBoundary objects in declaration order.
Sources are connected on copied declarations; caller inputs are unchanged.
No sequence object or runtime routing is retained.
"""
if not stages:
raise ValueError("make_stages needs at least one stage")
if previous is not None and prepared_from is not None:
raise ValueError("choose a previous output or a prepared branch input")
declarations = []
bindings = []
for index, item in enumerate(stages):
def _bind_pending(chain: _Chain, *, consumer, terminal):
pending = chain.pending
if pending is None:
return
chain.pending = None
try:
declaration = replace(pending.declaration, terminal=terminal)
if pending.predecessor is None:
incoming = _incoming(declaration)
else:
incoming = _connect(
pending.predecessor,
declaration,
residual_from=pending.predecessor_incoming,
)
outgoing = _connect(declaration, consumer, residual_from=incoming)
bound = _bind_stage(
declaration, pending.norm, incoming, outgoing, **pending.options
)
except Exception as error:
_note_origin(error, pending.origin)
raise
pending.boundary.plan = bound.plan
pending.boundary.declaration = declaration
if pending.predecessor_boundary is not None:
_carry_capture(pending.predecessor_boundary, pending.boundary)
def _carry_capture(producer, consumer):
"""Let an attention that always leaves its sum preserve the residual its
FFN's entry needs for capture. Only between stages of one append."""
if not (
producer.kind is StageKind.ATTENTION
and producer.declaration.reduction is ProducerReduction.ALWAYS_PARTIAL
and consumer.kind is StageKind.FFN
):
return
for variant, steps in producer.plan.paths.items():
next_steps = consumer.plan.paths[variant]
predicate = next_steps.entry.preserves_residual
if predicate is not None:
producer.plan.paths[variant] = msgspec.structs.replace(
steps,
entry=msgspec.structs.replace(
steps.entry, capture_preserves_residual=predicate
),
)
_STAGE_OPTIONS = {
StageKind.ATTENTION: frozenset({"qkv_latent_func", "fusions"}),
StageKind.FFN: frozenset({"fusions"}),
}
def _stage_items(stages):
declarations, bindings = [], []
for item in stages:
if len(item) not in (2, 3):
raise ValueError("a stage needs (declaration, norm[, options])")
declaration, norm = item[:2]
if not isinstance(declaration, StageDeclaration):
raise TypeError("make_stages requires stage declarations")
raise TypeError("append_stages requires stage declarations")
if declaration.previous is not None or declaration.prepared_from is not None:
raise ValueError("pass input sources to make_stages, not its declarations")
raise ValueError("a declaration takes its source from the layer stack")
if declaration.terminal:
raise ValueError("pass terminal to make_stages, not its declarations")
declaration = replace(
declaration,
previous=previous if index == 0 else declarations[-1],
prepared_from=prepared_from if index == 0 else None,
terminal=terminal and index == len(stages) - 1,
)
raise ValueError("the layer stack marks the terminal stage")
options = dict(item[2] if len(item) == 3 else {})
unexpected = options.keys() - _STAGE_OPTIONS[declaration.kind]
if unexpected:
raise TypeError(
f"unsupported {declaration.kind.name} stage options: "
f"{sorted(unexpected)}"
)
declarations.append(declaration)
bindings.append((norm, item[2] if len(item) == 3 else {}))
external_consumer = following
boundaries = []
incoming = _incoming(declarations[0])
for index, (declaration, (norm, options)) in enumerate(zip(declarations, bindings)):
following = (
declarations[index + 1]
if index + 1 < len(declarations)
else external_consumer
)
outgoing = _connect(declaration, following, residual_from=incoming)
options = dict(options)
if declaration.kind is StageKind.FFN:
unexpected = options.keys() - {
"fusions",
}
if unexpected:
raise TypeError(f"unsupported FFN options: {sorted(unexpected)}")
boundaries.append(_bind_stage(declaration, norm, incoming, outgoing, **options))
incoming = outgoing
for producer, consumer in zip(boundaries, boundaries[1:]):
if not (
producer.kind is StageKind.ATTENTION
and producer.declaration.reduction is ProducerReduction.ALWAYS_PARTIAL
and consumer.kind is StageKind.FFN
):
continue
for variant, steps in producer.plan.paths.items():
next_steps = consumer.plan.paths[variant]
predicate = next_steps.entry.preserves_residual
if predicate is not None:
producer.plan.paths[variant] = msgspec.structs.replace(
steps,
entry=msgspec.structs.replace(
steps.entry, capture_preserves_residual=predicate
bindings.append((norm, options))
return declarations, bindings
def _origin(stack: _LayerStack) -> str:
"""Where the current append_stages call was made, and its place in the stack."""
frame = sys._getframe(2)
return (
f"{os.path.basename(frame.f_code.co_filename)}:{frame.f_lineno} "
f"(append {len(stack.appends)} of its layer stack)"
)
def append_stages(*stages, prepared_from=None):
"""Append a local linear sequence of stages to the open layer stack.
Args:
*stages: Items of (declaration, norm) or (declaration, norm, options).
Declarations carry no source or terminal flag. Options are
constructor keywords: fusions, and qkv_latent_func for attention.
prepared_from: The ``declaration`` of a boundary an earlier call
returned on this stack: the already-read stage this sequence
branches from. A branch is a side path the model merges back
explicitly: it neither extends the stack nor waits for a consumer,
so each of its stages binds with only the stages of the branch.
Returns:
A tuple of StageBoundary objects in declaration order. Their
declarations are usable at once and gain their place in the stack when
it closes, which is also when their plans are bound.
"""
if not stages:
raise ValueError("append_stages needs at least one stage")
stack = _require_stack()
declarations, bindings = _stage_items(stages)
declarations = [replace(d) for d in declarations]
boundaries = tuple(StageBoundary(None, declaration=d) for d in declarations)
stack.appends.append(
_Append(declarations, bindings, prepared_from, boundaries, _origin(stack))
)
return boundaries
def _extend(chain, append, prepared_from):
"""Chain one append onto the stack and bind what can be bound."""
declarations, bindings = append.declarations, append.bindings
branch = prepared_from is not None
if not branch:
# This append's first stage is the consumer the pending stage waited for.
_bind_pending(chain, consumer=declarations[0], terminal=False)
try:
chained = []
for index, declaration in enumerate(declarations):
chained.append(
replace(
declaration,
previous=(
chained[-1] if index else (None if branch else chain.previous)
),
prepared_from=prepared_from if index == 0 else None,
)
)
boundaries = []
last = len(chained) - 1
# incoming feeds the stage being bound; previous_incoming fed the one
# before.
previous_incoming, incoming = None, _incoming(chained[0])
for index, (declaration, (norm, options)) in enumerate(zip(chained, bindings)):
if index == last and not branch:
boundaries.append(StageBoundary(None, declaration=declaration))
chain.pending = _PendingStage(
boundaries[-1],
declaration,
norm,
options,
predecessor=chained[index - 1] if index else None,
predecessor_incoming=previous_incoming,
predecessor_boundary=boundaries[-2] if index else None,
origin=append.origin,
)
break
following = chained[index + 1] if index < last else None
outgoing = _connect(declaration, following, residual_from=incoming)
boundaries.append(
_bind_stage(declaration, norm, incoming, outgoing, **options)
)
previous_incoming, incoming = incoming, outgoing
except Exception as error:
_note_origin(error, append.origin)
raise
if branch:
for producer, consumer in zip(boundaries, boundaries[1:]):
_carry_capture(producer, consumer)
else:
# The pair that ends on the pending stage is carried when it binds.
for producer, consumer in zip(boundaries[:-2], boundaries[1:-1]):
_carry_capture(producer, consumer)
chain.previous = _detached(chained[-1])
return tuple(boundaries)
@@ -87,6 +87,7 @@ from sglang.srt.utils import (
is_npu,
round_up,
)
from sglang.srt.utils.common import is_building_neighbour_layer
from sglang.srt.utils.custom_op import register_custom_op
_is_hip = is_hip()
@@ -163,6 +164,8 @@ def create_moe_dispatcher(
moe_runner_config: MoeRunnerConfig,
quant_method: FusedMoEMethodBase,
) -> BaseDispatcher:
if is_building_neighbour_layer():
return StandardDispatcher(moe_runner_config)
a2a_backend = get_moe_a2a_backend()
if a2a_backend.is_none() and is_npu():
return AscendTPDispatcher(moe_runner_config)
@@ -20,6 +20,7 @@ from sglang.srt.runtime_context import (
get_schedule,
)
from sglang.srt.utils import get_compiler_backend
from sglang.srt.utils.common import is_building_neighbour_layer
if TYPE_CHECKING:
from sglang.srt.layers.moe import MoeRunnerConfig
@@ -215,7 +216,7 @@ class KTEPWrapperMethod(FusedMoEMethodBase):
# 2. Initialize KT wrapper for CPU experts
# CPU experts: num_gpu_experts to num_experts-1
if self.tp_rank == 0:
if self.tp_rank == 0 and not is_building_neighbour_layer():
self.wrapper = KTMoEWrapper(
layer_idx=self.kt_config.layer_idx,
num_experts=num_experts,
+2 -4
View File
@@ -29,9 +29,9 @@ from transformers import ApertusConfig
from sglang.srt.layers.activation import XIELU
from sglang.srt.layers.aux_hidden_states import AuxHiddenStateList
from sglang.srt.layers.layer_boundary import (
append_stages,
declare_attn,
declare_ffn,
make_stages,
)
from sglang.srt.layers.layer_boundary.residual import batch as residual_batch
from sglang.srt.layers.layernorm import RMSNorm
@@ -265,14 +265,12 @@ class ApertusDecoderLayer(nn.Module):
self.feedforward_layernorm = RMSNorm(
config.hidden_size, eps=config.rms_norm_eps
)
self.attn_boundary, self.ffn_boundary = make_stages(
self.attn_boundary, self.ffn_boundary = append_stages(
(declare_attn(), self.attention_layernorm),
(
declare_ffn(sparse=False, next_layer_sparse=False),
self.feedforward_layernorm,
),
previous=declare_ffn() if layer_id != 0 else None,
terminal=layer_id == config.num_hidden_layers - 1,
)
def forward(
+2 -4
View File
@@ -23,9 +23,9 @@ from transformers import LlamaConfig
from sglang.srt.layers.activation import get_act_fn
from sglang.srt.layers.aux_hidden_states import AuxHiddenStateList
from sglang.srt.layers.layer_boundary import (
append_stages,
declare_attn,
declare_ffn,
make_stages,
)
from sglang.srt.layers.layer_boundary.residual import batch as residual_batch
from sglang.srt.layers.layernorm import RMSNorm
@@ -241,14 +241,12 @@ class ArceeDecoderLayer(nn.Module):
self.post_attention_layernorm = RMSNorm(
config.hidden_size, eps=config.rms_norm_eps
)
self.attn_boundary, self.ffn_boundary = make_stages(
self.attn_boundary, self.ffn_boundary = append_stages(
(declare_attn(), self.input_layernorm),
(
declare_ffn(sparse=False, next_layer_sparse=False),
self.post_attention_layernorm,
),
previous=declare_ffn() if layer_id != 0 else None,
terminal=layer_id == config.num_hidden_layers - 1,
)
def forward(
+2 -11
View File
@@ -39,10 +39,10 @@ from sglang.srt.layers.dp_attention import (
is_dp_attention_enabled,
)
from sglang.srt.layers.layer_boundary import (
append_stages,
declare_attn,
declare_ffn,
is_dense_ffn_fully_dp,
make_stages,
)
from sglang.srt.layers.layer_boundary.residual import batch as residual_batch
from sglang.srt.layers.layernorm import RMSNorm
@@ -664,9 +664,6 @@ class BailingMoEBlock(nn.Module):
self.is_layer_sparse = self._is_layer_sparse(
config, layer_id=layer_id, is_nextn=is_nextn
)
is_previous_layer_sparse = self._is_layer_sparse(
config, layer_id=layer_id - 1, is_nextn=False
)
is_next_layer_sparse = self._is_layer_sparse(
config, layer_id=layer_id + 1, is_nextn=False
)
@@ -696,7 +693,7 @@ class BailingMoEBlock(nn.Module):
self.post_attention_layernorm = RMSNorm(hidden_size, eps=config.rms_norm_eps)
self.attn_boundary, self.ffn_boundary = make_stages(
self.attn_boundary, self.ffn_boundary = append_stages(
(declare_attn(), self.input_layernorm),
(
declare_ffn(
@@ -705,12 +702,6 @@ class BailingMoEBlock(nn.Module):
),
self.post_attention_layernorm,
),
previous=declare_ffn(
sparse=is_previous_layer_sparse, next_layer_sparse=self.is_layer_sparse
)
if layer_id != 0
else None,
terminal=layer_id == (1 if is_nextn else config.num_hidden_layers) - 1,
)
def _is_layer_sparse(
@@ -19,9 +19,9 @@ from sglang.srt.layers.dp_attention import (
is_dp_attention_enabled,
)
from sglang.srt.layers.layer_boundary import (
append_stages,
declare_attn,
declare_ffn,
make_stages,
)
from sglang.srt.layers.layer_boundary.residual import batch as residual_batch
from sglang.srt.layers.layernorm import RMSNorm
@@ -783,7 +783,6 @@ class BailingMoELinearDecoderLayer(nn.Module):
self.expert_num = config.num_experts
self.hidden_size = config.hidden_size
is_moe_layer = self._is_layer_sparse(config, self.layer_id, is_nextn=is_nextn)
is_previous_moe_layer = self._is_layer_sparse(config, self.layer_id - 1)
is_next_layer_moe_layer = self._is_layer_sparse(config, self.layer_id + 1)
if self.expert_num == 1:
self.mlp = BailingMLP(
@@ -821,7 +820,7 @@ class BailingMoELinearDecoderLayer(nn.Module):
if config.attention_type == 1 and self.use_mla
else None
)
self.attn_boundary, self.ffn_boundary = make_stages(
self.attn_boundary, self.ffn_boundary = append_stages(
(
declare_attn(),
self.input_layernorm,
@@ -834,12 +833,6 @@ class BailingMoELinearDecoderLayer(nn.Module):
),
self.post_attention_layernorm,
),
previous=declare_ffn(
sparse=is_previous_moe_layer, next_layer_sparse=is_moe_layer
)
if layer_id != 0
else None,
terminal=layer_id == (1 if is_nextn else config.num_hidden_layers) - 1,
)
def _is_layer_sparse(
+25 -21
View File
@@ -27,6 +27,7 @@ from torch import nn
from transformers import PretrainedConfig
from sglang.srt.layers.dp_attention import is_dp_attention_enabled
from sglang.srt.layers.layer_boundary import layer_stack
from sglang.srt.layers.layer_boundary.residual import batch as residual_batch
from sglang.srt.layers.layernorm import RMSNorm
from sglang.srt.layers.linear import ReplicatedLinear
@@ -109,27 +110,30 @@ class BailingMoEModelNextN(nn.Module):
self.is_hybrid = (
hasattr(config, "model_type") and config.model_type == "bailing_hybrid"
)
if self.is_hybrid:
config.attention_type = 1
decoder_layer_cls = BailingMoELinearDecoderLayer
decoder_kwargs = {
"quant_config": quant_config,
"layer_id": 0,
"is_nextn": True,
"prefix": add_prefix(f"layers.{config.num_hidden_layers}", prefix),
}
if _is_bailing_moe_v3_config(config):
decoder_layer_cls = BailingMoeV3DecoderLayer
decoder_kwargs["num_fused_shared_experts"] = num_fused_shared_experts
self.decoder = decoder_layer_cls(config, **decoder_kwargs)
else:
self.decoder = BailingMoEBlock(
config,
0,
quant_config=quant_config,
prefix=add_prefix("decoder", prefix),
is_nextn=True,
)
with layer_stack():
if self.is_hybrid:
config.attention_type = 1
decoder_layer_cls = BailingMoELinearDecoderLayer
decoder_kwargs = {
"quant_config": quant_config,
"layer_id": 0,
"is_nextn": True,
"prefix": add_prefix(f"layers.{config.num_hidden_layers}", prefix),
}
if _is_bailing_moe_v3_config(config):
decoder_layer_cls = BailingMoeV3DecoderLayer
decoder_kwargs["num_fused_shared_experts"] = (
num_fused_shared_experts
)
self.decoder = decoder_layer_cls(config, **decoder_kwargs)
else:
self.decoder = BailingMoEBlock(
config,
0,
quant_config=quant_config,
prefix=add_prefix("decoder", prefix),
is_nextn=True,
)
self.shared_head = nn.Module()
self.final_layernorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
+2 -11
View File
@@ -27,10 +27,10 @@ from sglang.srt.layers.activation import SiluAndMul
from sglang.srt.layers.aux_hidden_states import AuxHiddenStateList
from sglang.srt.layers.dp_attention import is_dp_attention_enabled
from sglang.srt.layers.layer_boundary import (
append_stages,
declare_attn,
declare_ffn,
is_dense_ffn_fully_dp,
make_stages,
)
from sglang.srt.layers.layer_boundary.residual import batch as residual_batch
from sglang.srt.layers.layernorm import RMSNorm
@@ -1085,9 +1085,6 @@ class BailingMoELinearDecoderLayer(nn.Module):
and (self.layer_id >= config.first_k_dense_replace)
)
self.is_layer_sparse = is_moe_layer
is_previous_moe_layer = not (self.expert_num == 1) and (
self.layer_id - 1 >= config.first_k_dense_replace
)
is_next_layer_sparse = not (self.expert_num == 1) and (
self.layer_id + 1 >= config.first_k_dense_replace
)
@@ -1132,7 +1129,7 @@ class BailingMoELinearDecoderLayer(nn.Module):
self.input_layernorm = RMSNorm(self.hidden_size, eps=rms_norm_eps)
self.post_attention_layernorm = RMSNorm(self.hidden_size, eps=rms_norm_eps)
self.attn_boundary, self.ffn_boundary = make_stages(
self.attn_boundary, self.ffn_boundary = append_stages(
(
declare_attn(),
self.input_layernorm,
@@ -1149,12 +1146,6 @@ class BailingMoELinearDecoderLayer(nn.Module):
),
self.post_attention_layernorm,
),
previous=declare_ffn(
sparse=is_previous_moe_layer, next_layer_sparse=is_moe_layer
)
if layer_id != 0
else None,
terminal=layer_id == (1 if is_nextn else config.num_hidden_layers) - 1,
)
@torch.inference_mode()
+12 -10
View File
@@ -28,6 +28,7 @@ from sglang.kernels.ops.layernorm.fused_eh_norm import fused_eh_norm
from sglang.srt.environ import envs
from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder
from sglang.srt.layers.attention.index_topk_share import IndexTopKShareState
from sglang.srt.layers.layer_boundary import layer_stack
from sglang.srt.layers.layer_boundary.residual import batch as residual_batch
from sglang.srt.layers.layernorm import RMSNorm
from sglang.srt.layers.linear import ReplicatedLinear
@@ -121,16 +122,17 @@ class DeepseekModelNextN(nn.Module):
layer_name = "layers." + str(config.num_hidden_layers)
self.quant_config = quant_config
self.decoder = DeepseekV2DecoderLayer(
config,
0,
quant_config=quant_config,
moe_quant_config_override=moe_quant_config_override,
is_nextn=True,
prefix=add_prefix(layer_name, prefix),
alt_stream=self.alt_stream,
skip_rope=config.qk_rope_head_dim == 0,
)
with layer_stack():
self.decoder = DeepseekV2DecoderLayer(
config,
0,
quant_config=quant_config,
moe_quant_config_override=moe_quant_config_override,
is_nextn=True,
prefix=add_prefix(layer_name, prefix),
alt_stream=self.alt_stream,
skip_rope=config.qk_rope_head_dim == 0,
)
self.shared_head = nn.Module()
self.shared_head.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
+2 -15
View File
@@ -76,11 +76,11 @@ from sglang.srt.layers.dcp.planner import (
)
from sglang.srt.layers.flashinfer_comm_fusion import uses_cutedsl_ar_fusion
from sglang.srt.layers.layer_boundary import (
append_stages,
declare_attn,
declare_ffn,
get_attn_tp_context,
is_dense_ffn_fully_dp,
make_stages,
)
from sglang.srt.layers.layer_boundary.residual import access as residual_access
from sglang.srt.layers.layer_boundary.residual import batch as residual_batch
@@ -2665,14 +2665,8 @@ class DeepseekV2DecoderLayer(nn.Module):
)
self.is_layer_sparse = self._is_layer_sparse(layer_id, is_nextn=is_nextn)
is_previous_layer_sparse = self._is_layer_sparse(layer_id - 1, is_nextn=False)
is_next_layer_sparse = self._is_layer_sparse(layer_id + 1, is_nextn=False)
self._stage_enters_stack = (layer_id) == 0
self._stage_terminal = (layer_id) == (
1 if is_nextn else config.num_hidden_layers
) - 1
self._stage_previous_sparse = is_previous_layer_sparse
self._stage_next_sparse = is_next_layer_sparse
if self.is_layer_sparse:
@@ -2740,7 +2734,7 @@ class DeepseekV2DecoderLayer(nn.Module):
from sglang.srt.layers.layer_boundary.fusions.cutedsl import CuteDSLFusion
fusions = CuteDSLFusion()
attn_boundary, ffn_boundary = make_stages(
attn_boundary, ffn_boundary = append_stages(
(
declare_attn(output_transform=attn_output),
input_layernorm,
@@ -2755,13 +2749,6 @@ class DeepseekV2DecoderLayer(nn.Module):
post_attention_layernorm,
{"fusions": fusions},
),
previous=declare_ffn(
sparse=self._stage_previous_sparse,
next_layer_sparse=self.is_layer_sparse,
)
if not self._stage_enters_stack
else None,
terminal=self._stage_terminal,
)
return attn_boundary, ffn_boundary
@@ -56,10 +56,10 @@ from sglang.srt.layers.activation import SiluAndMul
from sglang.srt.layers.attention.dsa.dsa_indexer import Indexer
from sglang.srt.layers.dp_attention import is_dp_attention_enabled
from sglang.srt.layers.layer_boundary import (
append_stages,
declare_attn,
declare_ffn,
is_dense_ffn_fully_dp,
make_stages,
)
from sglang.srt.layers.layer_boundary.residual import batch as residual_batch
from sglang.srt.layers.layernorm import RMSNorm
@@ -1384,7 +1384,6 @@ class Dots3DecoderLayer(nn.Module):
alt_stream=alt_stream,
)
self.is_layer_sparse = self._is_layer_sparse(layer_id, is_nextn=is_nextn)
is_previous_layer_sparse = self._is_layer_sparse(layer_id - 1, is_nextn=False)
is_next_layer_sparse = self._is_layer_sparse(layer_id + 1, is_nextn=False)
if self.is_layer_sparse:
@@ -1417,7 +1416,7 @@ class Dots3DecoderLayer(nn.Module):
config.hidden_size, eps=config.rms_norm_eps
)
self.attn_boundary, self.ffn_boundary = make_stages(
self.attn_boundary, self.ffn_boundary = append_stages(
(declare_attn(), self.input_layernorm),
(
declare_ffn(
@@ -1426,12 +1425,6 @@ class Dots3DecoderLayer(nn.Module):
),
self.post_attention_layernorm,
),
previous=declare_ffn(
sparse=is_previous_layer_sparse, next_layer_sparse=self.is_layer_sparse
)
if layer_id != 0
else None,
terminal=layer_id == (1 if is_nextn else config.num_hidden_layers) - 1,
)
def _is_layer_sparse(self, layer_id: int, is_nextn: bool) -> bool:
@@ -9,6 +9,7 @@ from transformers import PretrainedConfig
from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder
from sglang.srt.layers.dp_attention import is_dp_attention_enabled
from sglang.srt.layers.layer_boundary import layer_stack
from sglang.srt.layers.layer_boundary.residual import batch as residual_batch
from sglang.srt.layers.layernorm import RMSNorm
from sglang.srt.layers.linear import ReplicatedLinear
@@ -48,13 +49,14 @@ class Dots3MTPHead(nn.Module):
quant_config=quant_config,
prefix=add_prefix("eh_proj", prefix),
)
self.decoder = Dots3DecoderLayer(
config,
layer_id=0,
quant_config=quant_config,
is_nextn=True,
prefix=add_prefix("decoder", prefix),
)
with layer_stack():
self.decoder = Dots3DecoderLayer(
config,
layer_id=0,
quant_config=quant_config,
is_nextn=True,
prefix=add_prefix("decoder", prefix),
)
self.shared_head = nn.Module()
self.shared_head.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
+2 -9
View File
@@ -23,9 +23,9 @@ from torch import nn
from transformers import PretrainedConfig
from sglang.srt.layers.layer_boundary import (
append_stages,
declare_attn,
declare_ffn,
make_stages,
)
from sglang.srt.layers.layer_boundary.residual import batch as residual_batch
from sglang.srt.layers.layernorm import RMSNorm
@@ -438,19 +438,12 @@ class Ernie4_5_VLMoeDecoderLayer(nn.Module):
next_sparse = layer_id + 1 < config.num_hidden_layers and _is_moe_layer(
config, layer_id + 1
)
self.attn_boundary, self.ffn_boundary = make_stages(
self.attn_boundary, self.ffn_boundary = append_stages(
(declare_attn(), self.input_layernorm),
(
declare_ffn(sparse=sparse, next_layer_sparse=next_sparse),
self.post_attention_layernorm,
),
previous=declare_ffn(
sparse=_is_moe_layer(config, layer_id - 1),
next_layer_sparse=sparse,
)
if layer_id != 0
else None,
terminal=layer_id == config.num_hidden_layers - 1,
)
def forward(
+2 -4
View File
@@ -7,9 +7,9 @@ from transformers import Exaone4Config
from sglang.srt.layers.activation import SiluAndMul
from sglang.srt.layers.layer_boundary import (
append_stages,
declare_attn,
declare_ffn,
make_stages,
)
from sglang.srt.layers.layer_boundary.residual import batch as residual_batch
from sglang.srt.layers.layer_boundary.residual.post_norm import (
@@ -275,7 +275,7 @@ class Exaone4DecoderLayer(nn.Module):
# Post-LN: each stage reads the residual as it is, and its output is
# normalized before it is added. The layer writes the FFN's itself.
ffn_update = PostNormAdd(self.post_feedforward_layernorm, applied_at_exit=True)
self.attn_boundary, self.ffn_boundary = make_stages(
self.attn_boundary, self.ffn_boundary = append_stages(
(
declare_attn(
read=PLAIN_READOUT,
@@ -292,8 +292,6 @@ class Exaone4DecoderLayer(nn.Module):
),
None,
),
previous=declare_ffn(update=ffn_update) if layer_id != 0 else None,
terminal=layer_id == config.num_hidden_layers - 1,
)
def forward(
+2 -9
View File
@@ -31,9 +31,9 @@ from sglang.srt.layers.activation import SiluAndMul
from sglang.srt.layers.aux_hidden_states import AuxHiddenStateList
from sglang.srt.layers.dp_attention import is_dp_attention_enabled
from sglang.srt.layers.layer_boundary import (
append_stages,
declare_attn,
declare_ffn,
make_stages,
)
from sglang.srt.layers.layer_boundary.residual import batch as residual_batch
from sglang.srt.layers.layernorm import RMSNorm
@@ -491,7 +491,7 @@ class ExaoneMoEDecoderLayer(nn.Module):
)
is_moe_layer = config.is_moe_layer
num_layers = config.num_hidden_layers
self.attn_boundary, self.ffn_boundary = make_stages(
self.attn_boundary, self.ffn_boundary = append_stages(
(declare_attn(), self.input_layernorm),
(
declare_ffn(
@@ -501,13 +501,6 @@ class ExaoneMoEDecoderLayer(nn.Module):
),
self.post_attention_layernorm,
),
previous=declare_ffn(
sparse=is_moe_layer[layer_id - 1],
next_layer_sparse=is_moe_layer[layer_id],
)
if layer_id != 0
else None,
terminal=layer_id == num_layers - 1,
)
def forward(
+2 -9
View File
@@ -15,9 +15,9 @@ from sglang.srt.layers.dp_attention import (
is_dp_attention_enabled,
)
from sglang.srt.layers.layer_boundary import (
append_stages,
declare_attn,
declare_ffn,
make_stages,
)
from sglang.srt.layers.layer_boundary.output import OutputTransform
from sglang.srt.layers.layer_boundary.residual import batch as residual_batch
@@ -203,7 +203,6 @@ class FalconH1HybridAttentionDecoderLayer(nn.Module):
# FalconH1 all layers are dense and have no nextn now
self.is_layer_sparse = False
is_previous_layer_sparse = False
is_next_layer_sparse = False
self.feed_forward = FalconH1MLP(
@@ -223,7 +222,7 @@ class FalconH1HybridAttentionDecoderLayer(nn.Module):
self.q_norm = RMSNorm(self.head_dim, eps=config.rms_norm_eps)
self.k_norm = RMSNorm(self.head_dim, eps=config.rms_norm_eps)
self.attn_boundary, self.ffn_boundary = make_stages(
self.attn_boundary, self.ffn_boundary = append_stages(
(declare_attn(), self.input_layernorm),
(
declare_ffn(
@@ -235,12 +234,6 @@ class FalconH1HybridAttentionDecoderLayer(nn.Module):
),
self.pre_ff_layernorm,
),
previous=declare_ffn(
sparse=is_previous_layer_sparse, next_layer_sparse=self.is_layer_sparse
)
if layer_id != 0
else None,
terminal=layer_id == config.num_hidden_layers - 1,
)
self.alt_stream = alt_stream
+2 -10
View File
@@ -36,9 +36,9 @@ from sglang.kernels.ops.moe.gemma4_routing import (
from sglang.srt.layers.aux_hidden_states import AuxHiddenStateList
from sglang.srt.layers.layer_boundary import (
SumGroup,
append_stages,
declare_attn,
declare_ffn,
make_stages,
)
from sglang.srt.layers.layer_boundary.output import OutputTransform
from sglang.srt.layers.layer_boundary.residual import batch as residual_batch
@@ -666,7 +666,7 @@ class Gemma4DecoderLayer(nn.Module):
self.register_buffer("layer_scalar", torch.ones(1), persistent=True)
self.has_ple = self.hidden_size_per_layer_input > 0
self.prefix = prefix
self.attn_boundary, self.ffn_boundary = make_stages(
self.attn_boundary, self.ffn_boundary = append_stages(
(
declare_attn(
output_transform=OutputTransform(self.post_attention_layernorm)
@@ -681,14 +681,6 @@ class Gemma4DecoderLayer(nn.Module):
),
self.pre_feedforward_layernorm,
),
previous=declare_ffn(
sparse=self.enable_moe_block,
next_layer_sparse=self.enable_moe_block,
update=REPLACE_AT_EXIT,
)
if layer_id != 0
else None,
terminal=layer_id == config.num_hidden_layers - 1,
)
def forward(
+10 -8
View File
@@ -18,6 +18,7 @@ from typing import Iterable, Optional
import torch
from torch import nn
from sglang.srt.layers.layer_boundary import layer_stack
from sglang.srt.layers.layer_boundary.residual import batch as residual_batch
from sglang.srt.layers.logits_processor import LogitsProcessor
from sglang.srt.layers.quantization.base_config import QuantizationConfig
@@ -64,14 +65,15 @@ class GigaChat35ModelNextN(nn.Module):
self.alt_stream = torch.cuda.Stream() if torch.cuda.is_available() else None
self.decoder = GigaChat35DecoderLayer(
config=config,
layer_id=0,
quant_config=quant_config,
prefix=add_prefix("decoder", prefix),
alt_stream=self.alt_stream,
is_nextn=True,
)
with layer_stack():
self.decoder = GigaChat35DecoderLayer(
config=config,
layer_id=0,
quant_config=quant_config,
prefix=add_prefix("decoder", prefix),
alt_stream=self.alt_stream,
is_nextn=True,
)
self.shared_head = nn.Module()
self.shared_head.norm = build_norm(config, config.hidden_size)
+2 -4
View File
@@ -27,9 +27,9 @@ from sglang.srt.layers.activation import SiluAndMul
from sglang.srt.layers.aux_hidden_states import AuxHiddenStateList
from sglang.srt.layers.dp_attention import is_dp_attention_enabled
from sglang.srt.layers.layer_boundary import (
append_stages,
declare_attn,
declare_ffn,
make_stages,
)
from sglang.srt.layers.layer_boundary.output import OutputTransform
from sglang.srt.layers.layer_boundary.residual import batch as residual_batch
@@ -263,7 +263,7 @@ class Glm4DecoderLayer(nn.Module):
config.hidden_size, eps=config.rms_norm_eps
)
self.post_mlp_layernorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
self.attn_boundary, self.ffn_boundary = make_stages(
self.attn_boundary, self.ffn_boundary = append_stages(
(
declare_attn(
output_transform=OutputTransform(self.post_self_attn_layernorm)
@@ -278,8 +278,6 @@ class Glm4DecoderLayer(nn.Module):
),
self.post_attention_layernorm,
),
previous=declare_ffn() if layer_id != 0 else None,
terminal=layer_id == config.num_hidden_layers - 1,
)
def forward(
+2 -9
View File
@@ -44,10 +44,10 @@ from sglang.srt.layers.dp_attention import (
is_dp_attention_enabled,
)
from sglang.srt.layers.layer_boundary import (
append_stages,
declare_attn,
declare_ffn,
is_dense_ffn_fully_dp,
make_stages,
)
from sglang.srt.layers.layer_boundary.residual import batch as residual_batch
from sglang.srt.layers.layernorm import RMSNorm
@@ -741,7 +741,6 @@ class Glm4MoeDecoderLayer(nn.Module):
)
self.is_layer_sparse = self._is_layer_sparse(layer_id, is_nextn=is_nextn)
is_previous_layer_sparse = self._is_layer_sparse(layer_id - 1, is_nextn=False)
is_next_layer_sparse = self._is_layer_sparse(layer_id + 1, is_nextn=False)
if self.is_layer_sparse:
@@ -773,7 +772,7 @@ class Glm4MoeDecoderLayer(nn.Module):
config.hidden_size, eps=config.rms_norm_eps
)
self.attn_boundary, self.ffn_boundary = make_stages(
self.attn_boundary, self.ffn_boundary = append_stages(
(declare_attn(), self.input_layernorm),
(
declare_ffn(
@@ -782,12 +781,6 @@ class Glm4MoeDecoderLayer(nn.Module):
),
self.post_attention_layernorm,
),
previous=declare_ffn(
sparse=is_previous_layer_sparse, next_layer_sparse=self.is_layer_sparse
)
if layer_id != 0
else None,
terminal=layer_id == (1 if is_nextn else config.num_hidden_layers) - 1,
)
# Detect if QKV uses aiter FP8 per-token quant so we can fuse
+2 -9
View File
@@ -38,11 +38,11 @@ from sglang.srt.layers.dp_attention import (
is_dp_attention_enabled,
)
from sglang.srt.layers.layer_boundary import (
append_stages,
declare_attn,
declare_ffn,
get_attn_tp_context,
is_dense_ffn_fully_dp,
make_stages,
)
from sglang.srt.layers.layer_boundary.residual import batch as residual_batch
from sglang.srt.layers.layernorm import RMSNorm
@@ -448,7 +448,6 @@ class Glm4MoeLiteDecoderLayer(nn.Module):
)
self.is_layer_sparse = self._is_layer_sparse(layer_id, is_nextn=is_nextn)
is_previous_layer_sparse = self._is_layer_sparse(layer_id - 1, is_nextn=False)
is_next_layer_sparse = self._is_layer_sparse(layer_id + 1, is_nextn=False)
if self.is_layer_sparse:
@@ -483,7 +482,7 @@ class Glm4MoeLiteDecoderLayer(nn.Module):
self._gfx95_quant_format = self._detect_gfx95_quant_format()
self.attn_boundary, self.ffn_boundary = make_stages(
self.attn_boundary, self.ffn_boundary = append_stages(
(
declare_attn(),
self.input_layernorm,
@@ -496,12 +495,6 @@ class Glm4MoeLiteDecoderLayer(nn.Module):
),
self.post_attention_layernorm,
),
previous=declare_ffn(
sparse=is_previous_layer_sparse, next_layer_sparse=self.is_layer_sparse
)
if layer_id != 0
else None,
terminal=layer_id == (1 if is_nextn else config.num_hidden_layers) - 1,
)
def _detect_gfx95_quant_format(self) -> str:
@@ -23,6 +23,7 @@ from transformers import PretrainedConfig
from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder
from sglang.srt.layers.dp_attention import is_dp_attention_enabled
from sglang.srt.layers.layer_boundary import layer_stack
from sglang.srt.layers.layer_boundary.residual import batch as residual_batch
from sglang.srt.layers.layernorm import RMSNorm
from sglang.srt.layers.logits_processor import LogitsProcessor
@@ -71,13 +72,14 @@ class Glm4MoeLiteModelNextN(nn.Module):
self.eh_proj = nn.Linear(2 * config.hidden_size, config.hidden_size, bias=False)
self.decoder = Glm4MoeLiteDecoderLayer(
config,
0,
quant_config=quant_config,
is_nextn=True,
prefix=add_prefix("decoder", prefix),
)
with layer_stack():
self.decoder = Glm4MoeLiteDecoderLayer(
config,
0,
quant_config=quant_config,
is_nextn=True,
prefix=add_prefix("decoder", prefix),
)
self.shared_head = nn.Module()
self.shared_head.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
+9 -7
View File
@@ -23,6 +23,7 @@ from transformers import PretrainedConfig
from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder
from sglang.srt.layers.dp_attention import is_dp_attention_enabled
from sglang.srt.layers.layer_boundary import layer_stack
from sglang.srt.layers.layer_boundary.residual import batch as residual_batch
from sglang.srt.layers.layernorm import RMSNorm
from sglang.srt.layers.logits_processor import LogitsProcessor
@@ -67,13 +68,14 @@ class Glm4MoeModelNextN(nn.Module):
self.eh_proj = nn.Linear(2 * config.hidden_size, config.hidden_size, bias=False)
self.decoder = Glm4MoeDecoderLayer(
config,
0,
quant_config=quant_config,
is_nextn=True,
prefix=add_prefix("decoder", prefix),
)
with layer_stack():
self.decoder = Glm4MoeDecoderLayer(
config,
0,
quant_config=quant_config,
is_nextn=True,
prefix=add_prefix("decoder", prefix),
)
self.shared_head = nn.Module()
self.shared_head.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
+2 -11
View File
@@ -29,11 +29,11 @@ from sglang.srt.layers.conv import Conv2dLayer
from sglang.srt.layers.layer_boundary import (
PLAIN_RESIDUAL_OPS,
MHCState,
append_stages,
declare_attn,
declare_ffn,
get_attn_tp_context,
is_dense_ffn_fully_dp,
make_stages,
)
from sglang.srt.layers.layer_boundary.residual import access as residual_access
from sglang.srt.layers.layer_boundary.residual import batch as residual_batch
@@ -723,7 +723,6 @@ class Glm5NextDecoderLayer(nn.Module):
)
self.is_layer_sparse = self._is_layer_sparse(layer_id, is_nextn=is_nextn)
is_previous_layer_sparse = self._is_layer_sparse(layer_id - 1, is_nextn=False)
is_next_layer_sparse = self._is_layer_sparse(layer_id + 1, is_nextn=False)
if self.is_layer_sparse:
@@ -793,7 +792,7 @@ class Glm5NextDecoderLayer(nn.Module):
),
is_last_layer=terminal,
).residual_ops()
self.attn_boundary, self.ffn_boundary = make_stages(
self.attn_boundary, self.ffn_boundary = append_stages(
(
declare_attn(read=residual.attn_readout, update=residual.attn_update),
self.input_layernorm,
@@ -812,14 +811,6 @@ class Glm5NextDecoderLayer(nn.Module):
),
self.post_attention_layernorm,
),
previous=declare_ffn(
sparse=is_previous_layer_sparse,
next_layer_sparse=self.is_layer_sparse,
update=residual.ffn_update,
)
if layer_id != 0
else None,
terminal=terminal,
)
def _hc_pre(
+15 -15
View File
@@ -35,9 +35,10 @@ from sglang.srt.layers.attention.vision import (
)
from sglang.srt.layers.dp_attention import is_dp_attention_enabled
from sglang.srt.layers.layer_boundary import (
append_stages,
declare_attn,
declare_ffn,
make_stages,
layer_stack,
)
from sglang.srt.layers.layer_boundary.output import OutputTransform
from sglang.srt.layers.layer_boundary.residual import batch as residual_batch
@@ -907,7 +908,7 @@ class GlmImageTextDecoderLayer(nn.Module):
config.hidden_size, eps=config.rms_norm_eps
)
self.post_mlp_layernorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
self.attn_boundary, self.ffn_boundary = make_stages(
self.attn_boundary, self.ffn_boundary = append_stages(
(
declare_attn(
read=UNFUSED_NORM_READOUT,
@@ -924,8 +925,6 @@ class GlmImageTextDecoderLayer(nn.Module):
),
self.post_attention_layernorm,
),
previous=declare_ffn() if layer_id != 0 else None,
terminal=layer_id == config.num_hidden_layers - 1,
)
def forward(
@@ -971,17 +970,18 @@ class GlmImageTextModel(nn.Module):
prefix=add_prefix("embed_tokens", prefix),
)
self.layers = nn.ModuleList(
[
GlmImageTextDecoderLayer(
layer_id=i,
config=config,
quant_config=self.quant_config,
prefix=add_prefix(f"layers.{i}", getattr(config, "prefix", "")),
)
for i in range(config.num_hidden_layers)
]
)
with layer_stack():
self.layers = nn.ModuleList(
[
GlmImageTextDecoderLayer(
layer_id=i,
config=config,
quant_config=self.quant_config,
prefix=add_prefix(f"layers.{i}", getattr(config, "prefix", "")),
)
for i in range(config.num_hidden_layers)
]
)
self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
def forward(
+8 -6
View File
@@ -23,6 +23,7 @@ from transformers import PretrainedConfig
from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder
from sglang.srt.layers.dp_attention import is_dp_attention_enabled
from sglang.srt.layers.layer_boundary import layer_stack
from sglang.srt.layers.layer_boundary.residual import batch as residual_batch
from sglang.srt.layers.layernorm import RMSNorm
from sglang.srt.layers.logits_processor import LogitsProcessor
@@ -70,12 +71,13 @@ class GlmOcrModelNextN(nn.Module):
self.eh_proj = nn.Linear(2 * config.hidden_size, config.hidden_size, bias=False)
self.decoder = Glm4DecoderLayer(
config,
0,
quant_config=quant_config,
prefix=add_prefix("decoder", prefix),
)
with layer_stack():
self.decoder = Glm4DecoderLayer(
config,
0,
quant_config=quant_config,
prefix=add_prefix("decoder", prefix),
)
self.shared_head = nn.Module()
self.shared_head.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
+2 -9
View File
@@ -34,9 +34,9 @@ from sglang.srt.layers.dp_attention import (
is_dp_attention_enabled,
)
from sglang.srt.layers.layer_boundary import (
append_stages,
declare_attn,
declare_ffn,
make_stages,
)
from sglang.srt.layers.layer_boundary.residual import batch as residual_batch
from sglang.srt.layers.layernorm import RMSNorm
@@ -560,7 +560,6 @@ class GptOssDecoderLayer(nn.Module):
# GptOss all layers are sparse
self.is_layer_sparse = True
is_previous_layer_sparse = True
is_next_layer_sparse = True
if self.is_layer_sparse:
@@ -591,7 +590,7 @@ class GptOssDecoderLayer(nn.Module):
x_pad_to_multiple=post_attn_pad_multiple,
)
self.attn_boundary, self.ffn_boundary = make_stages(
self.attn_boundary, self.ffn_boundary = append_stages(
(declare_attn(), self.input_layernorm),
(
declare_ffn(
@@ -600,12 +599,6 @@ class GptOssDecoderLayer(nn.Module):
),
self.post_attention_layernorm,
),
previous=declare_ffn(
sparse=is_previous_layer_sparse, next_layer_sparse=self.is_layer_sparse
)
if layer_id != 0
else None,
terminal=layer_id == config.num_hidden_layers - 1,
)
def forward(
+5 -11
View File
@@ -11,9 +11,9 @@ from sglang.srt.layers.attention.hybrid_linear_attn_backend import (
from sglang.srt.layers.attention.mamba.mamba import MambaMixer2
from sglang.srt.layers.aux_hidden_states import AuxHiddenStateList
from sglang.srt.layers.layer_boundary import (
append_stages,
declare_attn,
declare_ffn,
make_stages,
)
from sglang.srt.layers.layer_boundary.output import OutputTransform
from sglang.srt.layers.layer_boundary.residual import batch as residual_batch
@@ -40,13 +40,13 @@ from sglang.srt.utils import make_pp_layers
from .granitemoe import GraniteMoeMoE, GraniteMoeSharedMLP
def _make_stages(layer, config, layer_idx: int):
def _make_stages(layer):
"""Both kinds of layer scale their mixer and FFN outputs by
residual_multiplier, add them in the activation dtype and normalize the sum
as a separate step."""
scale = OutputTransform(layer._scale_output)
sparse = layer.block_sparse_moe is not None
return make_stages(
return append_stages(
(
declare_attn(read=UNFUSED_NORM_READOUT, output_transform=scale),
layer.input_layernorm,
@@ -60,12 +60,6 @@ def _make_stages(layer, config, layer_idx: int):
),
layer.post_attention_layernorm,
),
previous=(
declare_ffn(sparse=sparse, next_layer_sparse=sparse)
if layer_idx != 0
else None
),
terminal=layer_idx == config.num_hidden_layers - 1,
)
@@ -124,7 +118,7 @@ class GraniteMoeHybridMambaDecoderLayer(nn.Module):
self.post_attention_layernorm = RMSNorm(
config.hidden_size, eps=config.rms_norm_eps
)
self.attn_boundary, self.ffn_boundary = _make_stages(self, config, layer_idx)
self.attn_boundary, self.ffn_boundary = _make_stages(self)
def _scale_output(self, hidden_states: torch.Tensor) -> torch.Tensor:
return hidden_states * self.residual_multiplier
@@ -315,7 +309,7 @@ class GraniteMoeHybridAttentionDecoderLayer(nn.Module):
self.post_attention_layernorm = RMSNorm(
config.hidden_size, eps=config.rms_norm_eps
)
self.attn_boundary, self.ffn_boundary = _make_stages(self, config, layer_idx)
self.attn_boundary, self.ffn_boundary = _make_stages(self)
def _scale_output(self, hidden_states: torch.Tensor) -> torch.Tensor:
return hidden_states * self.residual_multiplier
+17 -25
View File
@@ -9,10 +9,11 @@ from transformers import PretrainedConfig
from sglang.srt.layers.attention.index_topk_share import IndexTopKShareState
from sglang.srt.layers.layer_boundary import (
IHCState,
append_stages,
declare_attn,
declare_ffn,
get_attn_tp_context,
make_stages,
layer_stack,
)
from sglang.srt.layers.layer_boundary.residual import batch as residual_batch
from sglang.srt.layers.layernorm import RMSNorm
@@ -583,8 +584,8 @@ def _build_stages(
layer_types = config.mlp_layer_types
is_sparse = layer_types[layer_id] != "dense"
next_id, previous_id = layer_id + 1, layer_id - 1
return make_stages(
next_id = layer_id + 1
return append_stages(
(
declare_attn(read=residual.attn_readout, update=residual.attn_update),
input_layernorm,
@@ -601,16 +602,6 @@ def _build_stages(
),
post_attention_layernorm,
),
previous=(
declare_ffn(
sparse=layer_types[previous_id] != "dense",
next_layer_sparse=is_sparse,
update=residual.ffn_update,
)
if layer_id != 0
else None
),
terminal=layer_id == config.num_hidden_layers - 1,
)
@@ -703,18 +694,19 @@ class HYV4Model(nn.Module):
**get_embedding_tp_kwargs(),
)
self.alt_stream = get_stream("alt") if is_cuda() else None
self.layers = nn.ModuleList(
[
HYV4DecoderLayer(
config,
i,
quant_config,
f"{prefix}.layers.{i}",
self.alt_stream,
)
for i in range(config.num_hidden_layers)
]
)
with layer_stack():
self.layers = nn.ModuleList(
[
HYV4DecoderLayer(
config,
i,
quant_config,
f"{prefix}.layers.{i}",
self.alt_stream,
)
for i in range(config.num_hidden_layers)
]
)
self.hc_head = HYV4HCHeadLayer(config, f"{prefix}.hc_head")
self.norm = RMSNorm(config.hidden_size, config.rms_norm_eps)
# The head mixes the streams down and fuses the final norm into one kernel.
+10 -9
View File
@@ -6,10 +6,11 @@ from torch import nn
from sglang.srt.layers.attention.index_topk_share import IndexTopKShareState
from sglang.srt.layers.layer_boundary import (
append_stages,
declare_attn,
declare_ffn,
get_attn_tp_context,
make_stages,
layer_stack,
)
from sglang.srt.layers.layer_boundary.residual import batch as residual_batch
from sglang.srt.layers.layernorm import RMSNorm
@@ -71,14 +72,13 @@ class HYV4MTPDecoderLayer(nn.Module):
)
if hasattr(self.mlp, "shared_experts"):
self.mlp.shared_experts.swiglu_limit = None
self.attn_boundary, self.ffn_boundary = make_stages(
self.attn_boundary, self.ffn_boundary = append_stages(
(
declare_attn(),
self.input_layernorm,
{"qkv_latent_func": self.self_attn.prepare_qkv_latent},
),
(declare_ffn(sparse=True), self.post_attention_layernorm),
terminal=True,
)
def forward(
@@ -124,12 +124,13 @@ class HYV4ModelNextN(nn.Module):
self.hnorm = RMSNorm(config.hidden_size, config.rms_norm_eps)
self.eh_proj = nn.Linear(2 * config.hidden_size, config.hidden_size, bias=False)
self.alt_stream = get_stream("alt") if is_cuda() else None
self.decoder = HYV4MTPDecoderLayer(
config,
quant_config,
f"{prefix}.decoder",
self.alt_stream,
)
with layer_stack():
self.decoder = HYV4MTPDecoderLayer(
config,
quant_config,
f"{prefix}.decoder",
self.alt_stream,
)
self.shared_head = nn.Module()
self.shared_head.norm = RMSNorm(config.hidden_size, config.rms_norm_eps)
+3 -11
View File
@@ -16,9 +16,9 @@ from sglang.srt.layers.aux_hidden_states import (
)
from sglang.srt.layers.dp_attention import is_dp_attention_enabled
from sglang.srt.layers.layer_boundary import (
append_stages,
declare_attn,
declare_ffn,
make_stages,
)
from sglang.srt.layers.layer_boundary.residual import batch as residual_batch
from sglang.srt.layers.layer_boundary.residual.add_norm import (
@@ -485,7 +485,7 @@ class InternS2MobiusLinearDecoderLayer(_InternS2MobiusDecoderMixin, nn.Module):
config.hidden_size, eps=config.rms_norm_eps
)
accepts_fp8_input = _linear_accepts_fp8_tuple(self.linear_attn.in_proj_qkvz)
self.attn_boundary, self.ffn_boundary = make_stages(
self.attn_boundary, self.ffn_boundary = append_stages(
(
declare_attn(
read=NormQuantReadout(
@@ -501,10 +501,6 @@ class InternS2MobiusLinearDecoderLayer(_InternS2MobiusDecoderMixin, nn.Module):
),
self.post_attention_layernorm,
),
previous=declare_ffn(sparse=True, next_layer_sparse=True)
if layer_id != 0
else None,
terminal=layer_id == config.num_hidden_layers - 1,
)
def forward(
@@ -617,7 +613,7 @@ class InternS2MobiusAttentionDecoderLayer(
self.q_norm = GemmaRMSNorm(self.head_dim, eps=config.rms_norm_eps)
self.k_norm = GemmaRMSNorm(self.head_dim, eps=config.rms_norm_eps)
accepts_fp8_input = _linear_accepts_fp8_tuple(self.qkv_proj)
self.attn_boundary, self.ffn_boundary = make_stages(
self.attn_boundary, self.ffn_boundary = append_stages(
(
declare_attn(
read=NormQuantReadout(
@@ -633,10 +629,6 @@ class InternS2MobiusAttentionDecoderLayer(
),
self.post_attention_layernorm,
),
previous=declare_ffn(sparse=True, next_layer_sparse=True)
if layer_id != 0
else None,
terminal=layer_id == config.num_hidden_layers - 1,
)
self.alt_stream = alt_stream
+2 -8
View File
@@ -11,9 +11,9 @@ from triton.language.extra import libdevice
from sglang.srt.layers.activation import SiluAndMul
from sglang.srt.layers.dp_attention import reject_attn_tp_shard_with_tp_reduce
from sglang.srt.layers.layer_boundary import (
append_stages,
declare_attn,
declare_ffn,
make_stages,
)
from sglang.srt.layers.layer_boundary.output import OutputTransform
from sglang.srt.layers.layer_boundary.residual import batch as residual_batch
@@ -496,7 +496,7 @@ class IQuestQ1DecoderLayer(nn.Module):
# Intentional: layer 0 adds the raw input and an FFN output norm, later layers
# add the normalized input without one; the MTP draft follows layer 0.
moe = _is_moe_layer(config, layer_id)
self.attn_boundary, self.ffn_boundary = make_stages(
self.attn_boundary, self.ffn_boundary = append_stages(
(
declare_attn(
read=UNFUSED_NORM_READOUT
@@ -515,12 +515,6 @@ class IQuestQ1DecoderLayer(nn.Module):
),
self.feed_forward_norm,
),
previous=declare_ffn(
sparse=_is_moe_layer(config, layer_id - 1), next_layer_sparse=moe
)
if layer_id != 0
else None,
terminal=layer_id == config.num_hidden_layers - 1,
)
def _attn_output(self, attn_output: torch.Tensor) -> torch.Tensor:
+2 -9
View File
@@ -16,9 +16,9 @@ from sglang.srt.distributed import (
from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder
from sglang.srt.layers.dcp.planner import prepare_decode_context_parallel_metadata
from sglang.srt.layers.layer_boundary import (
append_stages,
declare_attn,
declare_ffn,
make_stages,
)
from sglang.srt.layers.layer_boundary.residual import batch as residual_batch
from sglang.srt.layers.layernorm import RMSNorm
@@ -618,19 +618,12 @@ class KimiDecoderLayer(nn.Module):
next_sparse = layer_idx + 1 < config.num_hidden_layers and _is_sparse_layer(
config, layer_idx + 1
)
self.attn_boundary, self.ffn_boundary = make_stages(
self.attn_boundary, self.ffn_boundary = append_stages(
(declare_attn(), self.input_layernorm),
(
declare_ffn(sparse=sparse, next_layer_sparse=next_sparse),
self.post_attention_layernorm,
),
previous=declare_ffn(
sparse=_is_sparse_layer(config, layer_idx - 1),
next_layer_sparse=sparse,
)
if layer_idx != 0
else None,
terminal=layer_idx == config.num_hidden_layers - 1,
)
def forward(
+2 -9
View File
@@ -24,9 +24,9 @@ from sglang.srt.layers.dp_attention import (
is_dp_attention_enabled,
)
from sglang.srt.layers.layer_boundary import (
append_stages,
declare_attn,
declare_ffn,
make_stages,
)
from sglang.srt.layers.layer_boundary.residual import batch as residual_batch
from sglang.srt.layers.layernorm import RMSNorm
@@ -425,7 +425,6 @@ class LagunaDecoderLayer(nn.Module):
mlp_types = config.mlp_layer_types
self.is_layer_sparse = mlp_types[layer_id] == "sparse"
is_previous_layer_sparse = layer_id > 0 and mlp_types[layer_id - 1] == "sparse"
is_next_layer_sparse = (
layer_id + 1 < config.num_hidden_layers
and mlp_types[layer_id + 1] == "sparse"
@@ -453,7 +452,7 @@ class LagunaDecoderLayer(nn.Module):
config.hidden_size, eps=config.rms_norm_eps
)
self.attn_boundary, self.ffn_boundary = make_stages(
self.attn_boundary, self.ffn_boundary = append_stages(
(declare_attn(), self.input_layernorm),
(
declare_ffn(
@@ -462,12 +461,6 @@ class LagunaDecoderLayer(nn.Module):
),
self.post_attention_layernorm,
),
previous=declare_ffn(
sparse=is_previous_layer_sparse, next_layer_sparse=self.is_layer_sparse
)
if layer_id != 0
else None,
terminal=layer_id == config.num_hidden_layers - 1,
)
def forward(
+2 -9
View File
@@ -35,10 +35,10 @@ from sglang.srt.layers.dp_attention import (
is_dp_attention_enabled,
)
from sglang.srt.layers.layer_boundary import (
append_stages,
declare_attn,
declare_ffn,
is_dense_ffn_fully_dp,
make_stages,
)
from sglang.srt.layers.layer_boundary.residual import batch as residual_batch
from sglang.srt.layers.layernorm import RMSNorm
@@ -603,7 +603,6 @@ class LLaDA2MoeBlock(nn.Module):
self.layer_id = layer_id
self.is_layer_sparse = self._is_layer_sparse(config, layer_id=layer_id)
is_previous_layer_sparse = self._is_layer_sparse(config, layer_id=layer_id - 1)
is_next_layer_sparse = self._is_layer_sparse(config, layer_id=layer_id + 1)
if self.is_layer_sparse:
@@ -631,7 +630,7 @@ class LLaDA2MoeBlock(nn.Module):
self.post_attention_layernorm = RMSNorm(hidden_size, eps=config.rms_norm_eps)
self.attn_boundary, self.ffn_boundary = make_stages(
self.attn_boundary, self.ffn_boundary = append_stages(
(declare_attn(), self.input_layernorm),
(
declare_ffn(
@@ -640,12 +639,6 @@ class LLaDA2MoeBlock(nn.Module):
),
self.post_attention_layernorm,
),
previous=declare_ffn(
sparse=is_previous_layer_sparse, next_layer_sparse=self.is_layer_sparse
)
if layer_id != 0
else None,
terminal=layer_id == config.num_hidden_layers - 1,
)
def _is_layer_sparse(self, config: PretrainedConfig, layer_id: int) -> bool:
+2 -4
View File
@@ -31,9 +31,9 @@ from sglang.srt.distributed import (
from sglang.srt.layers.activation import SiluAndMul
from sglang.srt.layers.aux_hidden_states import AuxHiddenStateList
from sglang.srt.layers.layer_boundary import (
append_stages,
declare_attn,
declare_ffn,
make_stages,
)
from sglang.srt.layers.layer_boundary.residual import batch as residual_batch
from sglang.srt.layers.layer_boundary.residual.add_norm import NORM_QUANT_READOUT
@@ -361,14 +361,12 @@ class LlamaDecoderLayer(nn.Module):
return
self.input_layernorm.fuse_input_quant(self.self_attn.qkv_proj)
self.post_attention_layernorm.fuse_input_quant(self.mlp.gate_up_proj)
self.attn_boundary, self.ffn_boundary = make_stages(
self.attn_boundary, self.ffn_boundary = append_stages(
(declare_attn(read=self._attn_readout(layer_id)), self.input_layernorm),
(
declare_ffn(sparse=False, next_layer_sparse=False),
self.post_attention_layernorm,
),
previous=declare_ffn() if layer_id != 0 else None,
terminal=layer_id == config.num_hidden_layers - 1,
)
def _attn_readout(self, layer_id: int):
+6 -10
View File
@@ -30,9 +30,9 @@ from sglang.srt.layers.dp_attention import (
is_dp_attention_enabled,
)
from sglang.srt.layers.layer_boundary import (
append_stages,
declare_attn,
declare_ffn,
make_stages,
)
from sglang.srt.layers.layer_boundary.residual import batch as residual_batch
from sglang.srt.layers.layernorm import RMSNorm
@@ -398,7 +398,6 @@ class Llama4DecoderLayer(nn.Module):
)
self.config = config
is_moe_layer = self._is_moe_layer(layer_id)
is_previous_moe_layer = self._is_moe_layer(layer_id - 1)
is_next_moe_layer = self._is_moe_layer(layer_id + 1)
if is_moe_layer:
@@ -422,7 +421,7 @@ class Llama4DecoderLayer(nn.Module):
config.hidden_size, eps=config.rms_norm_eps
)
self.attn_boundary, self.ffn_boundary = make_stages(
self.attn_boundary, self.ffn_boundary = append_stages(
(declare_attn(), self.input_layernorm),
(
declare_ffn(
@@ -431,12 +430,6 @@ class Llama4DecoderLayer(nn.Module):
),
self.post_attention_layernorm,
),
previous=declare_ffn(
sparse=is_previous_moe_layer, next_layer_sparse=is_moe_layer
)
if layer_id != 0
else None,
terminal=layer_id == config.num_hidden_layers - 1,
)
def _is_moe_layer(self, layer_id: int) -> bool:
@@ -499,7 +492,10 @@ class Llama4Model(nn.Module):
self.layers = make_layers(
config.num_hidden_layers,
lambda idx, prefix: Llama4DecoderLayer(
config=config, layer_id=idx, quant_config=quant_config, prefix=prefix
config=config,
layer_id=idx,
quant_config=quant_config,
prefix=prefix,
),
prefix=add_prefix("layers", prefix),
)
+13 -11
View File
@@ -25,6 +25,7 @@ import torch
from torch import nn
from transformers import LlamaConfig
from sglang.srt.layers.layer_boundary import layer_stack
from sglang.srt.layers.layer_boundary.residual import batch as residual_batch
from sglang.srt.layers.layer_boundary.residual.post_norm import PLAIN_READOUT
from sglang.srt.layers.logits_processor import LogitsProcessor
@@ -74,17 +75,18 @@ class LlamaModel(nn.Module):
config.hidden_size,
prefix=add_prefix("embed_tokens", prefix),
)
self.layers = nn.ModuleList(
[
LlamaDecoderLayer(
config,
i,
quant_config=quant_config,
prefix=add_prefix(f"layers.{i}", prefix),
)
for i in range(config.num_hidden_layers)
]
)
with layer_stack():
self.layers = nn.ModuleList(
[
LlamaDecoderLayer(
config,
i,
quant_config=quant_config,
prefix=add_prefix(f"layers.{i}", prefix),
)
for i in range(config.num_hidden_layers)
]
)
self.fc = torch.nn.Linear(config.hidden_size * 2, config.hidden_size)
def forward(
+8 -6
View File
@@ -27,6 +27,7 @@ import torch
from torch import nn
from transformers import LlamaConfig
from sglang.srt.layers.layer_boundary import layer_stack
from sglang.srt.layers.layernorm import RMSNorm
from sglang.srt.layers.linear import QKVParallelLinear
from sglang.srt.layers.logits_processor import LogitsProcessor
@@ -193,12 +194,13 @@ class LlamaModel(nn.Module):
else:
self.fc_norm = None
self.layers = nn.ModuleList(
[
LlamaDecoderLayer(config, i, quant_config, prefix)
for i in range(config.num_hidden_layers)
]
)
with layer_stack():
self.layers = nn.ModuleList(
[
LlamaDecoderLayer(config, i, quant_config, prefix)
for i in range(config.num_hidden_layers)
]
)
self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
self.norm_output = getattr(config, "norm_output", False)
+17 -19
View File
@@ -52,9 +52,10 @@ from sglang.srt.layers.dp_attention import (
is_dp_attention_enabled,
)
from sglang.srt.layers.layer_boundary import (
append_stages,
declare_attn,
declare_ffn,
make_stages,
layer_stack,
)
from sglang.srt.layers.layer_boundary.residual import batch as residual_batch
from sglang.srt.layers.layernorm import RMSNorm
@@ -412,7 +413,7 @@ class LongcatFlashDecoderLayer(nn.Module):
prefix=add_prefix("mlp", prefix),
)
self.attn_boundary, self.moe_boundary = make_stages(
self.attn_boundary, self.moe_boundary = append_stages(
(
declare_attn(),
self.input_layernorm[0],
@@ -422,13 +423,9 @@ class LongcatFlashDecoderLayer(nn.Module):
declare_ffn(sparse=True, next_layer_sparse=True),
self.post_attention_layernorm[0],
),
previous=declare_ffn(sparse=True, next_layer_sparse=True)
if self.layer_id != 0
else None,
terminal=self.layer_id == config.num_hidden_layers - 1,
)
self.first_ffn_boundary, self.second_attn_boundary, self.second_ffn_boundary = (
make_stages(
append_stages(
(
declare_ffn(),
self.post_attention_layernorm[0],
@@ -562,18 +559,19 @@ class LongcatFlashModel(nn.Module):
)
self.alt_stream = get_stream("alt")
self.layers = nn.ModuleList(
[
LongcatFlashDecoderLayer(
config,
layer_id,
quant_config=quant_config,
prefix=add_prefix(f"layers.{layer_id}", prefix),
alt_stream=self.alt_stream,
)
for layer_id in range(config.num_hidden_layers)
]
)
with layer_stack():
self.layers = nn.ModuleList(
[
LongcatFlashDecoderLayer(
config,
layer_id,
quant_config=quant_config,
prefix=add_prefix(f"layers.{layer_id}", prefix),
alt_stream=self.alt_stream,
)
for layer_id in range(config.num_hidden_layers)
]
)
self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
self.layers_to_capture = []
@@ -45,9 +45,10 @@ from sglang.srt.layers.dp_attention import (
is_dp_attention_enabled,
)
from sglang.srt.layers.layer_boundary import (
append_stages,
declare_attn,
declare_ffn,
make_stages,
layer_stack,
)
from sglang.srt.layers.layer_boundary.residual import batch as residual_batch
from sglang.srt.layers.layernorm import RMSNorm
@@ -162,14 +163,12 @@ class LongcatFlashDenseDecoderLayer(nn.Module):
config.hidden_size, eps=config.rms_norm_eps
)
self.attn_boundary, self.ffn_boundary = make_stages(
self.attn_boundary, self.ffn_boundary = append_stages(
(declare_attn(), self.input_layernorm),
(
declare_ffn(sparse=False, next_layer_sparse=False),
self.post_attention_layernorm,
),
previous=declare_ffn() if self.layer_id != 0 else None,
terminal=self.layer_id == config.num_hidden_layers - 1,
)
def forward(
@@ -224,9 +223,10 @@ class LongcatFlashModelNextN(nn.Module):
quant_config=quant_config,
prefix=add_prefix("eh_proj", ""),
)
self.decoder = LongcatFlashDenseDecoderLayer(
config, 0, quant_config=quant_config, alt_stream=self.alt_stream
)
with layer_stack():
self.decoder = LongcatFlashDenseDecoderLayer(
config, 0, quant_config=quant_config, alt_stream=self.alt_stream
)
self.final_layernorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
+2 -9
View File
@@ -26,9 +26,9 @@ from torch import nn
from transformers import PretrainedConfig
from sglang.srt.layers.layer_boundary import (
append_stages,
declare_attn,
declare_ffn,
make_stages,
)
from sglang.srt.layers.layernorm import RMSNorm
from sglang.srt.layers.linear import QKVParallelLinear, RowParallelLinear
@@ -439,13 +439,12 @@ class MellumDecoderLayer(Qwen3MoeDecoderLayer):
reduce_results=False,
)
is_previous_layer_sparse = _is_sparse(layer_id - 1)
is_next_layer_sparse = _is_sparse(layer_id + 1)
self.input_layernorm = RMSNorm(cfg.hidden_size, eps=rms_norm_eps)
self.post_attention_layernorm = RMSNorm(cfg.hidden_size, eps=rms_norm_eps)
self.attn_boundary, self.ffn_boundary = make_stages(
self.attn_boundary, self.ffn_boundary = append_stages(
(declare_attn(), self.input_layernorm),
(
declare_ffn(
@@ -454,12 +453,6 @@ class MellumDecoderLayer(Qwen3MoeDecoderLayer):
),
self.post_attention_layernorm,
),
previous=declare_ffn(
sparse=is_previous_layer_sparse, next_layer_sparse=self.is_layer_sparse
)
if layer_id != 0
else None,
terminal=layer_id == cfg.num_hidden_layers - 1,
)
+5 -3
View File
@@ -8,6 +8,7 @@ import torch
from torch import nn
from transformers import PretrainedConfig
from sglang.srt.layers.layer_boundary import layer_stack
from sglang.srt.layers.layer_boundary.residual import batch as residual_batch
from sglang.srt.layers.layernorm import RMSNorm
from sglang.srt.layers.logits_processor import LogitsProcessor
@@ -40,9 +41,10 @@ class MiMoMultiTokenPredictorLayer(nn.Module):
self.input_proj = nn.Linear(
config.hidden_size * 2, config.hidden_size, bias=False
)
self.mtp_block = Qwen2DecoderLayer(
config=config, quant_config=quant_config, prefix=prefix
)
with layer_stack():
self.mtp_block = Qwen2DecoderLayer(
config=config, quant_config=quant_config, prefix=prefix
)
self.final_layernorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
def forward(
+2 -9
View File
@@ -36,10 +36,10 @@ from sglang.srt.layers.dp_attention import (
is_dp_attention_enabled,
)
from sglang.srt.layers.layer_boundary import (
append_stages,
declare_attn,
declare_ffn,
is_dense_ffn_fully_dp,
make_stages,
)
from sglang.srt.layers.layer_boundary.residual import batch as residual_batch
from sglang.srt.layers.layernorm import RMSNorm
@@ -826,7 +826,6 @@ class MiMoV2DecoderLayer(nn.Module):
)
self.is_layer_sparse = self.is_moe_layer(layer_id)
is_previous_layer_sparse = self.is_moe_layer(layer_id - 1)
is_next_layer_sparse = self.is_moe_layer(layer_id + 1)
if self.is_layer_sparse:
@@ -857,7 +856,7 @@ class MiMoV2DecoderLayer(nn.Module):
config.hidden_size, eps=config.layernorm_epsilon
)
self.attn_boundary, self.ffn_boundary = make_stages(
self.attn_boundary, self.ffn_boundary = append_stages(
(declare_attn(), self.input_layernorm),
(
declare_ffn(
@@ -866,12 +865,6 @@ class MiMoV2DecoderLayer(nn.Module):
),
self.post_attention_layernorm,
),
previous=declare_ffn(
sparse=is_previous_layer_sparse, next_layer_sparse=self.is_layer_sparse
)
if layer_id != 0
else None,
terminal=layer_id == config.num_hidden_layers - 1,
)
def forward(
+10 -15
View File
@@ -25,10 +25,11 @@ from sglang.srt.layers.dp_attention import (
is_dp_attention_enabled,
)
from sglang.srt.layers.layer_boundary import (
append_stages,
declare_attn,
declare_ffn,
is_dense_ffn_fully_dp,
make_stages,
layer_stack,
)
from sglang.srt.layers.layer_boundary.residual import batch as residual_batch
from sglang.srt.layers.layernorm import RMSNorm
@@ -98,7 +99,6 @@ class MiMoV2MTPLayer(nn.Module):
prefix=add_prefix("self_attn", prefix),
)
self.is_layer_sparse = False
is_previous_layer_sparse = True
is_next_layer_sparse = False
if is_dense_ffn_fully_dp():
@@ -120,7 +120,7 @@ class MiMoV2MTPLayer(nn.Module):
config.hidden_size, eps=config.layernorm_epsilon
)
self.attn_boundary, self.ffn_boundary = make_stages(
self.attn_boundary, self.ffn_boundary = append_stages(
(declare_attn(), self.input_layernorm),
(
declare_ffn(
@@ -129,12 +129,6 @@ class MiMoV2MTPLayer(nn.Module):
),
self.post_attention_layernorm,
),
previous=declare_ffn(
sparse=is_previous_layer_sparse, next_layer_sparse=self.is_layer_sparse
)
if layer_id != 0
else None,
terminal=layer_id == 1 - 1,
)
def forward(
@@ -185,12 +179,13 @@ class MiMoV2ModelNextN(nn.Module):
self.eh_proj = nn.Linear(2 * config.hidden_size, config.hidden_size, bias=False)
self.mtp_block = MiMoV2MTPLayer(
config,
0,
quant_config=quant_config,
prefix=add_prefix("decoder", prefix),
)
with layer_stack():
self.mtp_block = MiMoV2MTPLayer(
config,
0,
quant_config=quant_config,
prefix=add_prefix("decoder", prefix),
)
self.final_layernorm = RMSNorm(config.hidden_size, eps=config.layernorm_epsilon)
def forward(
+9 -10
View File
@@ -39,9 +39,9 @@ from sglang.srt.layers.dp_attention import (
is_dp_attention_enabled,
)
from sglang.srt.layers.layer_boundary import (
append_stages,
declare_attn,
declare_ffn,
make_stages,
)
from sglang.srt.layers.layer_boundary.residual import batch as residual_batch
from sglang.srt.layers.layernorm import RMSNorm
@@ -99,6 +99,7 @@ from sglang.srt.utils import (
is_xpu,
make_pp_layers,
)
from sglang.srt.utils.common import is_building_neighbour_layer
from sglang.srt.utils.custom_op import register_custom_op
from sglang.srt.utils.hf_transformers_utils import get_rope_config
@@ -408,7 +409,12 @@ class MiniMaxM2QKRMSNorm:
use_fused_norm = get_bool_env_var("SGLANG_USE_FUSED_PARALLEL_QKNORM")
self._forward_impl = self._forward_naive
if self._world_size > 1 and _is_cuda and use_fused_norm:
if (
self._world_size > 1
and _is_cuda
and use_fused_norm
and not is_building_neighbour_layer()
):
occupancy = get_fused_parallel_qknorm_max_occupancy(
q_norm.weight.dtype,
self._world_size,
@@ -867,10 +873,9 @@ class MiniMaxM2DecoderLayer(nn.Module):
config.hidden_size, eps=getattr(config, "rms_norm_eps", 1e-6)
)
is_previous_layer_sparse = True
is_next_layer_sparse = True
self.attn_boundary, self.ffn_boundary = make_stages(
self.attn_boundary, self.ffn_boundary = append_stages(
(declare_attn(), self.input_layernorm),
(
declare_ffn(
@@ -879,12 +884,6 @@ class MiniMaxM2DecoderLayer(nn.Module):
),
self.post_attention_layernorm,
),
previous=declare_ffn(
sparse=is_previous_layer_sparse, next_layer_sparse=self.is_layer_sparse
)
if layer_id != 0
else None,
terminal=layer_id == config.num_hidden_layers - 1,
)
def forward(
+2 -9
View File
@@ -40,10 +40,10 @@ from sglang.srt.layers.dp_attention import (
is_dp_attention_enabled,
)
from sglang.srt.layers.layer_boundary import (
append_stages,
declare_attn,
declare_ffn,
is_dense_ffn_fully_dp,
make_stages,
)
from sglang.srt.layers.layer_boundary.residual import batch as residual_batch
from sglang.srt.layers.layernorm import GemmaRMSNorm, RMSNorm
@@ -1517,10 +1517,9 @@ class MiniMaxM3DecoderLayer(nn.Module):
return True
return moe_layer_freq[lid] != 0
is_previous_layer_sparse = _is_layer_sparse(layer_id - 1)
is_next_layer_sparse = _is_layer_sparse(layer_id + 1)
self.attn_boundary, self.ffn_boundary = make_stages(
self.attn_boundary, self.ffn_boundary = append_stages(
(declare_attn(), self.input_layernorm),
(
declare_ffn(
@@ -1529,12 +1528,6 @@ class MiniMaxM3DecoderLayer(nn.Module):
),
self.post_attention_layernorm,
),
previous=declare_ffn(
sparse=is_previous_layer_sparse, next_layer_sparse=self.is_layer_sparse
)
if layer_id != 0
else None,
terminal=layer_id == config.num_hidden_layers - 1,
)
def forward(
+13 -11
View File
@@ -40,6 +40,7 @@ import torch
from torch import nn
from transformers import PretrainedConfig
from sglang.srt.layers.layer_boundary import layer_stack
from sglang.srt.layers.layer_boundary.residual import batch as residual_batch
from sglang.srt.layers.layernorm import RMSNorm
from sglang.srt.layers.linear import RowParallelLinear
@@ -75,17 +76,18 @@ class MistralEagleModel(nn.Module):
config.hidden_size,
prefix=add_prefix("embed_tokens", prefix),
)
self.layers = nn.ModuleList(
[
LlamaDecoderLayer(
config=config,
layer_id=i,
prefix=add_prefix(f"layers.{i}", prefix),
quant_config=quant_config,
)
for i in range(config.num_hidden_layers)
]
)
with layer_stack():
self.layers = nn.ModuleList(
[
LlamaDecoderLayer(
config=config,
layer_id=i,
prefix=add_prefix(f"layers.{i}", prefix),
quant_config=quant_config,
)
for i in range(config.num_hidden_layers)
]
)
self.start_layer = 0
self.end_layer = config.num_hidden_layers
self.fc = RowParallelLinear(
+2 -6
View File
@@ -26,9 +26,9 @@ from torch import nn
from transformers import MixtralConfig
from sglang.srt.layers.layer_boundary import (
append_stages,
declare_attn,
declare_ffn,
make_stages,
)
from sglang.srt.layers.layer_boundary.residual import batch as residual_batch
from sglang.srt.layers.layernorm import RMSNorm
@@ -235,16 +235,12 @@ class MixtralDecoderLayer(nn.Module):
self.post_attention_layernorm = RMSNorm(
config.hidden_size, eps=config.rms_norm_eps
)
self.attn_boundary, self.ffn_boundary = make_stages(
self.attn_boundary, self.ffn_boundary = append_stages(
(declare_attn(), self.input_layernorm),
(
declare_ffn(sparse=True, next_layer_sparse=True),
self.post_attention_layernorm,
),
previous=declare_ffn(sparse=True, next_layer_sparse=True)
if layer_id != 0
else None,
terminal=layer_id == config.num_hidden_layers - 1,
)
def forward(
+20 -18
View File
@@ -22,6 +22,7 @@ from transformers.models.mllama.modeling_mllama import (
from sglang.srt.layers.activation import get_act_fn
from sglang.srt.layers.attention.vision import VisionAttention
from sglang.srt.layers.layer_boundary import layer_stack
from sglang.srt.layers.layer_boundary.residual import batch as residual_batch
from sglang.srt.layers.layernorm import RMSNorm
from sglang.srt.layers.linear import (
@@ -678,26 +679,27 @@ class MllamaTextModel(nn.Module):
self.cross_attention_layers = config.cross_attention_layers
layers = []
for layer_id in range(config.num_hidden_layers):
if layer_id in self.cross_attention_layers:
layers.append(
MllamaCrossAttentionDecoderLayer(
config,
layer_id,
quant_config=quant_config,
prefix=add_prefix(f"layers.{layer_id}", prefix),
with layer_stack():
for layer_id in range(config.num_hidden_layers):
if layer_id in self.cross_attention_layers:
layers.append(
MllamaCrossAttentionDecoderLayer(
config,
layer_id,
quant_config=quant_config,
prefix=add_prefix(f"layers.{layer_id}", prefix),
)
)
)
else:
# TODO: force LlamaDecoderLayer to config.attention_bias=False
layers.append(
LlamaDecoderLayer(
config,
quant_config=quant_config,
layer_id=layer_id,
prefix=add_prefix(f"layers.{layer_id}", prefix),
else:
# TODO: force LlamaDecoderLayer to config.attention_bias=False
layers.append(
LlamaDecoderLayer(
config,
quant_config=quant_config,
layer_id=layer_id,
prefix=add_prefix(f"layers.{layer_id}", prefix),
)
)
)
self.layers = nn.ModuleList(layers)
self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
+21 -21
View File
@@ -21,9 +21,10 @@ from sglang.srt.layers.attention.vision import (
)
from sglang.srt.layers.conv import Conv3dLayer
from sglang.srt.layers.layer_boundary import (
append_stages,
declare_attn,
declare_ffn,
make_stages,
layer_stack,
)
from sglang.srt.layers.layer_boundary.residual import batch as residual_batch
from sglang.srt.layers.layernorm import RMSNorm
@@ -1017,14 +1018,12 @@ class MossVLSelfAttentionDecoderLayer(nn.Module):
config.hidden_size, eps=config.rms_norm_eps, **norm_kwargs
)
self.attn_boundary, self.ffn_boundary = make_stages(
self.attn_boundary, self.ffn_boundary = append_stages(
(declare_attn(), self.input_layernorm),
(
declare_ffn(sparse=False, next_layer_sparse=False),
self.post_attention_layernorm,
),
previous=declare_ffn() if layer_id != 0 else None,
terminal=layer_id == config.num_hidden_layers - 1,
)
def forward(
@@ -1070,25 +1069,26 @@ class MossVLTextModel(nn.Module):
self.cross_attention_layers = config.cross_attention_layers
layers = []
for layer_id in range(config.num_hidden_layers):
if layer_id in self.cross_attention_layers:
layers.append(
MossVLCrossAttentionDecoderLayer(
config,
layer_id,
quant_config=quant_config,
prefix=add_prefix(f"layers.{layer_id}", prefix),
with layer_stack():
for layer_id in range(config.num_hidden_layers):
if layer_id in self.cross_attention_layers:
layers.append(
MossVLCrossAttentionDecoderLayer(
config,
layer_id,
quant_config=quant_config,
prefix=add_prefix(f"layers.{layer_id}", prefix),
)
)
)
else:
layers.append(
MossVLSelfAttentionDecoderLayer(
config,
layer_id,
quant_config=quant_config,
prefix=add_prefix(f"layers.{layer_id}", prefix),
else:
layers.append(
MossVLSelfAttentionDecoderLayer(
config,
layer_id,
quant_config=quant_config,
prefix=add_prefix(f"layers.{layer_id}", prefix),
)
)
)
self.layers = nn.ModuleList(layers)
self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
+2 -4
View File
@@ -9,9 +9,9 @@ from sglang.srt.layers.activation import SiluAndMul
from sglang.srt.layers.aux_hidden_states import AuxHiddenStateList
from sglang.srt.layers.dp_attention import is_dp_attention_enabled
from sglang.srt.layers.layer_boundary import (
append_stages,
declare_attn,
declare_ffn,
make_stages,
)
from sglang.srt.layers.layer_boundary.residual import batch as residual_batch
from sglang.srt.layers.layernorm import RMSNorm
@@ -231,14 +231,12 @@ class NanbeigeDecoderLayer(nn.Module):
)
# Each loop runs the physical layers as one stack: the model folds the
# residual back in and re-enters at layer 0.
self.attn_boundary, self.ffn_boundary = make_stages(
self.attn_boundary, self.ffn_boundary = append_stages(
(declare_attn(), self.input_layernorm),
(
declare_ffn(sparse=False, next_layer_sparse=False),
self.post_attention_layernorm,
),
previous=declare_ffn() if layer_id != 0 else None,
terminal=layer_id == config.num_hidden_layers - 1,
)
def forward(
+26 -22
View File
@@ -19,6 +19,7 @@ from torch import nn
from sglang.srt.configs import NemotronHConfig
from sglang.srt.layers.dp_attention import is_dp_attention_enabled
from sglang.srt.layers.layer_boundary import layer_stack
from sglang.srt.layers.layer_boundary.residual import batch as residual_batch
from sglang.srt.layers.layernorm import RMSNorm
from sglang.srt.layers.linear import ColumnParallelLinear
@@ -239,34 +240,37 @@ class NemotronHMultiTokenPredictor(nn.Module):
# Total number of physical layers = num_steps * pattern_len
total_layers = self.num_mtp_layers * self.pattern_len
for i in range(total_layers):
step_rel_idx = i % self.pattern_len
with layer_stack():
for i in range(total_layers):
step_rel_idx = i % self.pattern_len
char = self.pattern_str[step_rel_idx]
char = self.pattern_str[step_rel_idx]
is_start_of_step = step_rel_idx == 0
is_end_of_step = step_rel_idx == self.pattern_len - 1
is_start_of_step = step_rel_idx == 0
is_end_of_step = step_rel_idx == self.pattern_len - 1
layer_prefix = f"{prefix}.layers.{i}"
layer_prefix = f"{prefix}.layers.{i}"
common_kwargs = dict(
config=config,
layer_idx=i,
quant_config=quant_config,
prefix=layer_prefix,
has_start_projections=is_start_of_step,
has_end_norm=is_end_of_step,
)
if char == "*":
self.layers[str(i)] = NemotronHMTPAttentionDecoderLayer(**common_kwargs)
elif char == "E":
self.layers[str(i)] = NemotronHMTPMoEDecoderLayer(**common_kwargs)
else:
raise NotImplementedError(
f"Pattern char '{char}' in {self.pattern_str} not implemented"
common_kwargs = dict(
config=config,
layer_idx=i,
quant_config=quant_config,
prefix=layer_prefix,
has_start_projections=is_start_of_step,
has_end_norm=is_end_of_step,
)
if char == "*":
self.layers[str(i)] = NemotronHMTPAttentionDecoderLayer(
**common_kwargs
)
elif char == "E":
self.layers[str(i)] = NemotronHMTPMoEDecoderLayer(**common_kwargs)
else:
raise NotImplementedError(
f"Pattern char '{char}' in {self.pattern_str} not implemented"
)
def get_input_embeddings(self, input_ids: torch.Tensor) -> torch.Tensor:
assert self.embed_tokens is not None, (
"embed_tokens not initialized - must be shared from target model"
+2 -8
View File
@@ -4,9 +4,9 @@ from sglang.srt.configs.nemotron_h import ATTENTION, MAMBA, MOE
from sglang.srt.layers.layer_boundary import (
ExitRows,
ProducerReduction,
append_stages,
declare_attn,
declare_ffn,
make_stages,
)
from sglang.srt.layers.layer_boundary.residual.add_norm import (
NormQuantReadout,
@@ -42,16 +42,10 @@ def make_stage_boundary(layer_norm: RMSNorm, *, pattern: str, layer_idx: int):
if get_parallel().attn_cp_size > 1 or layernorm_sp.layernorm_sp_enabled():
raise NotImplementedError("a Nemotron stage with attention CP or LayerNorm SP")
previous = _declaration(pattern, layer_idx - 1) if layer_idx > 0 else None
(boundary,) = make_stages(
(boundary,) = append_stages(
(
_declaration(pattern, layer_idx),
layer_norm,
),
previous=previous,
following=_declaration(pattern, layer_idx + 1)
if layer_idx + 1 < len(pattern)
else None,
terminal=layer_idx == len(pattern) - 1,
)
return boundary
+2 -21
View File
@@ -24,9 +24,9 @@ from torch import nn
from transformers import LlamaConfig
from sglang.srt.layers.layer_boundary import (
append_stages,
declare_attn,
declare_ffn,
make_stages,
)
from sglang.srt.layers.layer_boundary.residual import batch as residual_batch
from sglang.srt.layers.layernorm import RMSNorm
@@ -63,21 +63,6 @@ def _find_multiple(n: int, k: int) -> int:
return n + k - (n % k)
def _previous_stage(block_configs, layer_idx: int):
"""The declaration of the last attention or FFN before this layer, or None
when this layer's first stage starts the layer stack."""
for block in reversed(block_configs[:layer_idx]):
if not block.ffn.no_op:
return declare_ffn()
if not block.attention.no_op:
return declare_attn()
return None
def _has_stage(block) -> bool:
return not (block.attention.no_op and block.ffn.no_op)
class DeciLMDecoderLayer(nn.Module):
def __init__(
self,
@@ -162,12 +147,8 @@ class DeciLMDecoderLayer(nn.Module):
)
self.entry_boundary = None
if stages:
boundaries = make_stages(
boundaries = append_stages(
*stages,
previous=_previous_stage(config.block_configs, layer_idx),
terminal=not any(
_has_stage(block) for block in config.block_configs[layer_idx + 1 :]
),
)
self.entry_boundary = boundaries[0]
if not self._is_no_op_attention:
+2 -4
View File
@@ -29,9 +29,9 @@ from sglang.srt.layers.activation import SiluAndMul
from sglang.srt.layers.aux_hidden_states import AuxHiddenStateList
from sglang.srt.layers.dp_attention import is_dp_attention_enabled
from sglang.srt.layers.layer_boundary import (
append_stages,
declare_attn,
declare_ffn,
make_stages,
)
from sglang.srt.layers.layer_boundary.residual import batch as residual_batch
from sglang.srt.layers.layer_boundary.residual.add_norm import NORM_QUANT_READOUT
@@ -292,14 +292,12 @@ class Qwen2DecoderLayer(nn.Module):
)
self.input_layernorm.fuse_input_quant(self.self_attn.qkv_proj)
self.post_attention_layernorm.fuse_input_quant(self.mlp.gate_up_proj)
self.attn_boundary, self.ffn_boundary = make_stages(
self.attn_boundary, self.ffn_boundary = append_stages(
(declare_attn(read=self._attn_readout(layer_id)), self.input_layernorm),
(
declare_ffn(sparse=False, next_layer_sparse=False),
self.post_attention_layernorm,
),
previous=declare_ffn() if layer_id != 0 else None,
terminal=layer_id == config.num_hidden_layers - 1,
)
def _attn_readout(self, layer_id: int):
+13 -11
View File
@@ -24,6 +24,7 @@ from typing import Iterable, Optional, Tuple
import torch
from torch import nn
from sglang.srt.layers.layer_boundary import layer_stack
from sglang.srt.layers.layer_boundary.residual import batch as residual_batch
from sglang.srt.layers.layer_boundary.residual.post_norm import PLAIN_READOUT
from sglang.srt.layers.logits_processor import LogitsProcessor
@@ -75,17 +76,18 @@ class Qwen2Model(nn.Module):
config.hidden_size,
prefix=add_prefix("embed_tokens", prefix),
)
self.layers = nn.ModuleList(
[
Qwen2DecoderLayer(
config,
i,
quant_config=quant_config,
prefix=add_prefix(f"layers.{i}", prefix),
)
for i in range(config.num_hidden_layers)
]
)
with layer_stack():
self.layers = nn.ModuleList(
[
Qwen2DecoderLayer(
config,
i,
quant_config=quant_config,
prefix=add_prefix(f"layers.{i}", prefix),
)
for i in range(config.num_hidden_layers)
]
)
self.fc = torch.nn.Linear(config.hidden_size * 2, config.hidden_size)
def forward(
+2 -9
View File
@@ -49,9 +49,9 @@ from sglang.srt.layers.dp_attention import (
)
from sglang.srt.layers.flashinfer_comm_fusion import uses_cutedsl_ar_fusion
from sglang.srt.layers.layer_boundary import (
append_stages,
declare_attn,
declare_ffn,
make_stages,
)
from sglang.srt.layers.layer_boundary.residual import batch as residual_batch
from sglang.srt.layers.layernorm import RMSNorm
@@ -1027,7 +1027,6 @@ class Qwen2MoeDecoderLayer(nn.Module):
# Qwen2MoE all layers are sparse and have no nextn now
self.is_layer_sparse = True
is_previous_layer_sparse = True
is_next_layer_sparse = True
if self.is_layer_sparse:
@@ -1052,7 +1051,7 @@ class Qwen2MoeDecoderLayer(nn.Module):
self.post_attention_layernorm = RMSNorm(
config.hidden_size, eps=config.rms_norm_eps
)
self.attn_boundary, self.ffn_boundary = make_stages(
self.attn_boundary, self.ffn_boundary = append_stages(
(declare_attn(), self.input_layernorm),
(
declare_ffn(
@@ -1061,12 +1060,6 @@ class Qwen2MoeDecoderLayer(nn.Module):
),
self.post_attention_layernorm,
),
previous=declare_ffn(
sparse=is_previous_layer_sparse, next_layer_sparse=self.is_layer_sparse
)
if layer_id != 0
else None,
terminal=layer_id == config.num_hidden_layers - 1,
)
def forward(
+2 -4
View File
@@ -7,9 +7,9 @@ from torch import nn
from sglang.srt.layers.aux_hidden_states import AuxHiddenStateList
from sglang.srt.layers.layer_boundary import (
append_stages,
declare_attn,
declare_ffn,
make_stages,
)
from sglang.srt.layers.layer_boundary.residual import batch as residual_batch
from sglang.srt.layers.layernorm import RMSNorm
@@ -375,14 +375,12 @@ class Qwen3DecoderLayer(nn.Module):
config.hidden_size, eps=config.rms_norm_eps, **norm_kwargs
)
self.attn_boundary, self.ffn_boundary = make_stages(
self.attn_boundary, self.ffn_boundary = append_stages(
(declare_attn(), self.input_layernorm),
(
declare_ffn(sparse=False, next_layer_sparse=False),
self.post_attention_layernorm,
),
previous=declare_ffn() if layer_id != 0 else None,
terminal=layer_id == config.num_hidden_layers - 1,
)
def forward(
+3 -20
View File
@@ -54,9 +54,9 @@ from sglang.srt.layers.dp_attention import (
)
from sglang.srt.layers.flashinfer_comm_fusion import uses_cutedsl_ar_fusion
from sglang.srt.layers.layer_boundary import (
append_stages,
declare_attn,
declare_ffn,
make_stages,
)
# Layers - Attention
@@ -447,7 +447,6 @@ class Qwen3_5GatedDeltaNet(nn.Module):
eps=self.layer_norm_epsilon,
group_size=None,
norm_before_gate=True,
device=torch.get_device_module().current_device(),
dtype=torch.get_default_dtype(),
**(
{"activation": self.output_gate_type}
@@ -1078,7 +1077,6 @@ class Qwen3_5LinearDecoderLayer(nn.Module):
reduce_results=False,
)
is_layer_sparse = True
is_previous_layer_sparse = True
is_next_layer_sparse = True
elif config.model_type == "qwen3_5_text":
self.mlp = Qwen2MoeMLP(
@@ -1092,7 +1090,6 @@ class Qwen3_5LinearDecoderLayer(nn.Module):
)
_maybe_enable_silu_fp4_quant_fusion(self.mlp)
is_layer_sparse = False
is_previous_layer_sparse = False
is_next_layer_sparse = False
else:
raise ValueError(f"Invalid model type: {config.model_type}")
@@ -1108,7 +1105,7 @@ class Qwen3_5LinearDecoderLayer(nn.Module):
boundary_fusions = _layer_fusions(config, is_nextn)
# A subclass that brings its own residual builds the stages itself.
if build_stages:
self.attn_boundary, self.ffn_boundary = make_stages(
self.attn_boundary, self.ffn_boundary = append_stages(
(
declare_attn(
read=NormQuantReadout(
@@ -1130,12 +1127,6 @@ class Qwen3_5LinearDecoderLayer(nn.Module):
self.post_attention_layernorm,
{"fusions": boundary_fusions},
),
previous=declare_ffn(
sparse=is_previous_layer_sparse, next_layer_sparse=is_layer_sparse
)
if layer_id != 0
else None,
terminal=layer_id == config.num_hidden_layers - 1,
)
def forward(
@@ -1285,7 +1276,6 @@ class Qwen3_5AttentionDecoderLayer(nn.Module):
reduce_results=False,
)
is_layer_sparse = False
is_previous_layer_sparse = False
is_next_layer_sparse = False
elif config.model_type in _QWEN3_5_MOE_TEXT_MODEL_TYPES:
self.mlp = Qwen2MoeSparseMoeBlock(
@@ -1304,7 +1294,6 @@ class Qwen3_5AttentionDecoderLayer(nn.Module):
reduce_results=False,
)
is_layer_sparse = True
is_previous_layer_sparse = True
is_next_layer_sparse = True
else:
raise ValueError(f"Invalid model type: {config.model_type}")
@@ -1323,7 +1312,7 @@ class Qwen3_5AttentionDecoderLayer(nn.Module):
boundary_fusions = _layer_fusions(config, is_nextn)
# A subclass that brings its own residual builds the stages itself.
if build_stages:
self.attn_boundary, self.ffn_boundary = make_stages(
self.attn_boundary, self.ffn_boundary = append_stages(
(
declare_attn(
read=NormQuantReadout(
@@ -1343,12 +1332,6 @@ class Qwen3_5AttentionDecoderLayer(nn.Module):
self.post_attention_layernorm,
{"fusions": boundary_fusions},
),
previous=declare_ffn(
sparse=is_previous_layer_sparse, next_layer_sparse=is_layer_sparse
)
if layer_id != 0
else None,
terminal=layer_id == config.num_hidden_layers - 1,
)
self.alt_stream = alt_stream
+2 -9
View File
@@ -31,9 +31,9 @@ from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation
from sglang.srt.eplb.expert_location_dispatch import ExpertLocationDispatchInfo
from sglang.srt.layers.aux_hidden_states import AuxHiddenStateAccumulator
from sglang.srt.layers.layer_boundary import (
append_stages,
declare_attn,
declare_ffn,
make_stages,
)
from sglang.srt.layers.layer_boundary.residual import batch as residual_batch
from sglang.srt.layers.layernorm import RMSNorm
@@ -858,7 +858,6 @@ class Qwen3MoeDecoderLayer(nn.Module):
# Qwen3MoE all layers are sparse and have no nextn now
self.is_layer_sparse = True
is_previous_layer_sparse = True
is_next_layer_sparse = True
if self.is_layer_sparse:
@@ -882,7 +881,7 @@ class Qwen3MoeDecoderLayer(nn.Module):
config.hidden_size, eps=config.rms_norm_eps
)
self.attn_boundary, self.ffn_boundary = make_stages(
self.attn_boundary, self.ffn_boundary = append_stages(
(declare_attn(), self.input_layernorm),
(
declare_ffn(
@@ -891,12 +890,6 @@ class Qwen3MoeDecoderLayer(nn.Module):
),
self.post_attention_layernorm,
),
previous=declare_ffn(
sparse=is_previous_layer_sparse, next_layer_sparse=self.is_layer_sparse
)
if layer_id != 0
else None,
terminal=layer_id == config.num_hidden_layers - 1,
)
def forward(
+3 -17
View File
@@ -23,9 +23,9 @@ from sglang.srt.layers.dp_attention import (
is_dp_attention_enabled,
)
from sglang.srt.layers.layer_boundary import (
append_stages,
declare_attn,
declare_ffn,
make_stages,
)
from sglang.srt.layers.layer_boundary.residual import batch as residual_batch
from sglang.srt.layers.layernorm import GemmaRMSNorm
@@ -509,7 +509,6 @@ class Qwen3HybridLinearDecoderLayer(nn.Module):
# Qwen3Next all layers are sparse and have no nextn now
self.is_layer_sparse = True
is_previous_layer_sparse = True
is_next_layer_sparse = True
self.layer_id = layer_id
@@ -538,7 +537,7 @@ class Qwen3HybridLinearDecoderLayer(nn.Module):
self.post_attention_layernorm = GemmaRMSNorm(
config.hidden_size, eps=config.rms_norm_eps
)
self.attn_boundary, self.ffn_boundary = make_stages(
self.attn_boundary, self.ffn_boundary = append_stages(
(declare_attn(), self.input_layernorm),
(
declare_ffn(
@@ -547,12 +546,6 @@ class Qwen3HybridLinearDecoderLayer(nn.Module):
),
self.post_attention_layernorm,
),
previous=declare_ffn(
sparse=is_previous_layer_sparse, next_layer_sparse=self.is_layer_sparse
)
if layer_id != 0
else None,
terminal=layer_id == config.num_hidden_layers - 1,
)
def forward(
@@ -676,7 +669,6 @@ class Qwen3HybridAttentionDecoderLayer(nn.Module):
# Qwen3Next all layers are sparse and have no nextn now
self.is_layer_sparse = True
is_previous_layer_sparse = True
is_next_layer_sparse = True
if self.is_layer_sparse:
@@ -708,7 +700,7 @@ class Qwen3HybridAttentionDecoderLayer(nn.Module):
self.q_norm = GemmaRMSNorm(self.head_dim, eps=config.rms_norm_eps)
self.k_norm = GemmaRMSNorm(self.head_dim, eps=config.rms_norm_eps)
self.attn_boundary, self.ffn_boundary = make_stages(
self.attn_boundary, self.ffn_boundary = append_stages(
(declare_attn(), self.input_layernorm),
(
declare_ffn(
@@ -717,12 +709,6 @@ class Qwen3HybridAttentionDecoderLayer(nn.Module):
),
self.post_attention_layernorm,
),
previous=declare_ffn(
sparse=is_previous_layer_sparse, next_layer_sparse=self.is_layer_sparse
)
if layer_id != 0
else None,
terminal=layer_id == config.num_hidden_layers - 1,
)
self.alt_stream = alt_stream
+12 -17
View File
@@ -39,10 +39,10 @@ from sglang.srt.layers.hyperconnection import (
from sglang.srt.layers.layer_boundary import (
ExitRows,
GatedResidualState,
append_stages,
declare_attn,
declare_ffn,
get_attn_tp_context,
make_stages,
)
from sglang.srt.layers.layer_boundary.residual import batch as residual_batch
from sglang.srt.layers.linear import ReplicatedLinear
@@ -88,6 +88,7 @@ from sglang.srt.models.qwen4_exp_ple_table import (
)
from sglang.srt.runtime_context import get_parallel
from sglang.srt.utils import get_bool_env_var, is_hip, logger
from sglang.srt.utils.common import is_building_neighbour_layer
_use_aiter = get_bool_env_var("SGLANG_USE_AITER") and is_hip()
@@ -534,6 +535,13 @@ class Qwen4ExpPLEGroupedNorm(nn.Module):
return (x_norm * weight).to(compute_dtype)
def _offloads_ple(config) -> bool:
"""Whether this layer keeps its PLE table in host memory. A pipeline
neighbour layer is never loaded or run, so it keeps the table on the meta
device and allocates no host memory or stream for it."""
return bool(config.ple_offload_embedding) and not is_building_neighbour_layer()
class Qwen4ExpNGramEmbedding(nn.Module):
_MASK64 = (1 << 64) - 1
_SPLITMIX_GAMMA = 0x9E3779B97F4A7C15
@@ -605,7 +613,7 @@ class Qwen4ExpNGramEmbedding(nn.Module):
and not self.use_attn_tp_ngram
)
ngram_prefix = f"{prefix}.ngram_embedding" if prefix else "ngram_embedding"
offload_embedding = bool(config.ple_offload_embedding)
offload_embedding = _offloads_ple(config)
# Offload only needs this embedding's metadata: build it on meta so the
# shard is never allocated on the device.
with torch.device("meta") if offload_embedding else nullcontext():
@@ -1088,9 +1096,7 @@ class Qwen4ExpPLELayer(nn.Module):
bias=False,
)
nn.init.zeros_(self.conv1d.weight)
self._prefetch_stream = (
torch.cuda.Stream() if config.ple_offload_embedding else None
)
self._prefetch_stream = torch.cuda.Stream() if _offloads_ple(config) else None
self._graph_prefetch_buffers = {}
self._eager_prefetch_buffer = None
self._prefetch_state = None
@@ -1441,7 +1447,7 @@ def _build_qwen4_exp_stages(residual, *, sparse, layer_id, config):
the writes are gated injections, so neither stage offers a fused
add-and-norm candidate.
"""
return make_stages(
return append_stages(
(
declare_attn(read=residual.attn_readout, update=residual.attn_update),
None,
@@ -1456,17 +1462,6 @@ def _build_qwen4_exp_stages(residual, *, sparse, layer_id, config):
),
None,
),
previous=(
declare_ffn(
sparse=sparse,
next_layer_sparse=sparse,
update=residual.ffn_update,
exit_rows=_ffn_exit_rows(layer_id - 1, config),
)
if layer_id != 0
else None
),
terminal=layer_id == config.num_hidden_layers - 1,
)
+2 -14
View File
@@ -20,10 +20,10 @@ from sglang.srt.layers.dp_attention import (
is_dp_attention_enabled,
)
from sglang.srt.layers.layer_boundary import (
append_stages,
declare_attn,
declare_ffn,
is_dense_ffn_fully_dp,
make_stages,
)
from sglang.srt.layers.layer_boundary.residual import batch as residual_batch
from sglang.srt.layers.layernorm import RMSNorm
@@ -1002,12 +1002,6 @@ class SarvamMoEMLADecoderLayer(nn.Module):
and layer_id >= first_k_dense
and (layer_id - first_k_dense) % moe_layer_freq == 0
)
is_previous_layer_sparse = (
has_moe
and layer_id > 0
and (layer_id - 1) >= first_k_dense
and (layer_id - 1 - first_k_dense) % moe_layer_freq == 0
)
is_next_layer_sparse = (
has_moe
and layer_id < config.num_hidden_layers - 1
@@ -1044,7 +1038,7 @@ class SarvamMoEMLADecoderLayer(nn.Module):
config.hidden_size, eps=config.rms_norm_eps
)
self.attn_boundary, self.ffn_boundary = make_stages(
self.attn_boundary, self.ffn_boundary = append_stages(
(
declare_attn(),
self.input_layernorm,
@@ -1057,12 +1051,6 @@ class SarvamMoEMLADecoderLayer(nn.Module):
),
self.post_attention_layernorm,
),
previous=declare_ffn(
sparse=is_previous_layer_sparse, next_layer_sparse=self.is_layer_sparse
)
if layer_id != 0
else None,
terminal=layer_id == config.num_hidden_layers - 1,
)
def forward(
+2 -4
View File
@@ -15,9 +15,9 @@ from sglang.srt.layers.dp_attention import (
is_dp_attention_enabled,
)
from sglang.srt.layers.layer_boundary import (
append_stages,
declare_attn,
declare_ffn,
make_stages,
)
from sglang.srt.layers.layer_boundary.residual import batch as residual_batch
from sglang.srt.layers.layernorm import RMSNorm
@@ -294,7 +294,7 @@ class SDARBlock(nn.Module):
prefix=add_prefix("mlp", prefix),
)
self.attn_boundary, self.ffn_boundary = make_stages(
self.attn_boundary, self.ffn_boundary = append_stages(
(declare_attn(), self.input_layernorm),
(
declare_ffn(
@@ -303,8 +303,6 @@ class SDARBlock(nn.Module):
),
self.post_attention_layernorm,
),
previous=declare_ffn() if layer_id != 0 else None,
terminal=layer_id == config.num_hidden_layers - 1,
)
def forward(
+2 -6
View File
@@ -17,9 +17,9 @@ from sglang.srt.layers.dp_attention import (
is_dp_attention_enabled,
)
from sglang.srt.layers.layer_boundary import (
append_stages,
declare_attn,
declare_ffn,
make_stages,
)
from sglang.srt.layers.layer_boundary.residual import batch as residual_batch
from sglang.srt.layers.layernorm import RMSNorm
@@ -353,16 +353,12 @@ class SDARMoeBlock(nn.Module):
prefix=add_prefix("mlp", prefix),
)
self.attn_boundary, self.ffn_boundary = make_stages(
self.attn_boundary, self.ffn_boundary = append_stages(
(declare_attn(), self.input_layernorm),
(
declare_ffn(sparse=True, next_layer_sparse=True),
self.post_attention_layernorm,
),
previous=declare_ffn(sparse=True, next_layer_sparse=True)
if layer_id != 0
else None,
terminal=layer_id == config.num_hidden_layers - 1,
)
def forward(
+2 -4
View File
@@ -7,9 +7,9 @@ from torch import nn
from sglang.srt.layers.activation import GeluAndMul
from sglang.srt.layers.dp_attention import is_dp_attention_enabled
from sglang.srt.layers.layer_boundary import (
append_stages,
declare_attn,
declare_ffn,
make_stages,
)
from sglang.srt.layers.layer_boundary.residual import batch as residual_batch
from sglang.srt.layers.layernorm import RMSNorm
@@ -271,14 +271,12 @@ class Spark2_5DecoderLayer(nn.Module):
self.post_attention_layernorm = RMSNorm(
config.hidden_size, eps=config.rms_norm_eps
)
self.attn_boundary, self.ffn_boundary = make_stages(
self.attn_boundary, self.ffn_boundary = append_stages(
(declare_attn(), self.input_layernorm),
(
declare_ffn(sparse=False, next_layer_sparse=False),
self.post_attention_layernorm,
),
previous=declare_ffn() if layer_id != 0 else None,
terminal=layer_id == config.num_hidden_layers - 1,
)
def forward(
+2 -12
View File
@@ -25,9 +25,9 @@ from sglang.srt.layers.dp_attention import (
reject_attn_tp_shard_with_tp_reduce,
)
from sglang.srt.layers.layer_boundary import (
append_stages,
declare_attn,
declare_ffn,
make_stages,
)
from sglang.srt.layers.layer_boundary.residual import batch as residual_batch
from sglang.srt.layers.layernorm import RMSNorm
@@ -334,9 +334,6 @@ class Step3TextDecoderLayer(nn.Module):
self.layer_id = layer_id
self.is_layer_sparse = True if layer_id in moe_layers_idx else False
self.is_previous_layer_sparse = (
True if layer_id - 1 in moe_layers_idx else False
)
self.is_next_layer_sparse = True if layer_id + 1 in moe_layers_idx else False
if not self.is_layer_sparse:
@@ -373,7 +370,7 @@ class Step3TextDecoderLayer(nn.Module):
prefix=add_prefix("mlp", prefix),
)
self.attn_boundary, self.ffn_boundary = make_stages(
self.attn_boundary, self.ffn_boundary = append_stages(
(declare_attn(), self.input_layernorm),
(
declare_ffn(
@@ -382,13 +379,6 @@ class Step3TextDecoderLayer(nn.Module):
),
self.post_attention_layernorm,
),
previous=declare_ffn(
sparse=self.is_previous_layer_sparse,
next_layer_sparse=self.is_layer_sparse,
)
if layer_id != 0
else None,
terminal=layer_id == config.num_hidden_layers - 1,
)
def moe_mlp_forward(self, hidden_states):
+2 -14
View File
@@ -11,9 +11,9 @@ from sglang.srt.layers.dp_attention import (
is_dp_attention_enabled,
)
from sglang.srt.layers.layer_boundary import (
append_stages,
declare_attn,
declare_ffn,
make_stages,
)
from sglang.srt.layers.layer_boundary.residual import batch as residual_batch
from sglang.srt.layers.layernorm import GemmaRMSNorm
@@ -409,7 +409,6 @@ class Step3p5DecoderLayer(nn.Module):
quant_config: Optional[QuantizationConfig] = None,
prefix: str = "",
alt_stream: Optional[torch.cuda.Stream] = None,
is_nextn: bool = False,
) -> None:
super().__init__()
self.hidden_size = config.hidden_size
@@ -426,7 +425,6 @@ class Step3p5DecoderLayer(nn.Module):
self.num_attention_heads = config.num_attention_heads
self.num_key_value_heads = config.num_attention_groups
self.is_moe_layer = layer_id in moe_layers_set
self.is_previous_layer_sparse = (layer_id - 1) in moe_layers_set
self.is_next_layer_sparse = (layer_id + 1) in moe_layers_set
num_hidden_layers = config.num_hidden_layers
@@ -520,9 +518,7 @@ class Step3p5DecoderLayer(nn.Module):
config.hidden_size, eps=config.rms_norm_eps
)
# An MTP draft is a one-layer model; layer_id still indexes its config.
self.attn_boundary, self.ffn_boundary = make_stages(
self.attn_boundary, self.ffn_boundary = append_stages(
(declare_attn(), self.input_layernorm),
(
declare_ffn(
@@ -531,14 +527,6 @@ class Step3p5DecoderLayer(nn.Module):
),
self.post_attention_layernorm,
),
previous=declare_ffn(
sparse=self.is_previous_layer_sparse,
next_layer_sparse=self.is_moe_layer,
)
if not (0 if is_nextn else layer_id) == 0
else None,
terminal=(0 if is_nextn else layer_id)
== (1 if is_nextn else config.num_hidden_layers) - 1,
)
self.layer_id = layer_id
+7 -6
View File
@@ -6,6 +6,7 @@ import torch
import torch.nn as nn
from transformers import PretrainedConfig
from sglang.srt.layers.layer_boundary import layer_stack
from sglang.srt.layers.layer_boundary.residual import batch as residual_batch
from sglang.srt.layers.layernorm import GemmaRMSNorm
from sglang.srt.layers.logits_processor import LogitsProcessor
@@ -79,12 +80,12 @@ class Step3p5AMultiTokenPredictor(nn.Module):
self.hnorm = GemmaRMSNorm(config.hidden_size, config.rms_norm_eps)
self.eh_proj = nn.Linear(config.hidden_size * 2, config.hidden_size, bias=False)
self.shared_head = SharedHead(config=config, quant_config=quant_config)
self.mtp_block = Step3p5DecoderLayer(
config=config,
layer_id=layer_id,
prefix=f"{prefix}.mtp_block",
is_nextn=True,
)
with layer_stack():
self.mtp_block = Step3p5DecoderLayer(
config=config,
layer_id=layer_id,
prefix=f"{prefix}.mtp_block",
)
self.lm_head = self.shared_head.head
def forward(
+2 -9
View File
@@ -41,10 +41,10 @@ from sglang.srt.eplb.expert_location_dispatch import ExpertLocationDispatchInfo
from sglang.srt.layers.activation import SiluAndMul
from sglang.srt.layers.dp_attention import is_dp_attention_enabled
from sglang.srt.layers.layer_boundary import (
append_stages,
declare_attn,
declare_ffn,
is_dense_ffn_fully_dp,
make_stages,
)
from sglang.srt.layers.layer_boundary.residual import batch as residual_batch
from sglang.srt.layers.layernorm import RMSNorm
@@ -1495,7 +1495,6 @@ class XllmDecoderLayer(nn.Module):
config.num_experts > 0 and (lid + 1) % decoder_sparse_step == 0
)
is_previous_layer_sparse = _is_sparse(layer_id - 1)
is_next_layer_sparse = _is_sparse(layer_id + 1)
if self.is_layer_sparse:
@@ -1523,7 +1522,7 @@ class XllmDecoderLayer(nn.Module):
self.input_layernorm = _make_norm(config)
self.post_attention_layernorm = _make_norm(config)
self.attn_boundary, self.ffn_boundary = make_stages(
self.attn_boundary, self.ffn_boundary = append_stages(
(declare_attn(), self.input_layernorm),
(
declare_ffn(
@@ -1532,12 +1531,6 @@ class XllmDecoderLayer(nn.Module):
),
self.post_attention_layernorm,
),
previous=declare_ffn(
sparse=is_previous_layer_sparse, next_layer_sparse=self.is_layer_sparse
)
if layer_id != 0
else None,
terminal=layer_id == config.num_hidden_layers - 1,
)
def forward(
+3 -9
View File
@@ -53,9 +53,9 @@ from torch import nn
from sglang.srt.configs.zaya import ZayaConfig
from sglang.srt.layers.layer_boundary import (
append_stages,
declare_attn,
declare_ffn,
make_stages,
)
from sglang.srt.layers.layer_boundary.residual import batch as residual_batch
from sglang.srt.layers.layer_boundary.residual.add_norm import PLAIN_ADD
@@ -1320,12 +1320,8 @@ class ZayaDecoderATTLayer(nn.Module):
self.res_scale = ResidualScaling(config, layer_id)
else:
self.res_scale = None
(self.attn_boundary,) = make_stages(
(self.attn_boundary,) = append_stages(
(declare_attn(read=_ResidualMergeRead(self.res_scale)), self.input_norm),
previous=declare_ffn(sparse=True, next_layer_sparse=True)
if layer_id != 0
else None,
terminal=layer_id == config.num_hidden_layers - 1,
)
self.entry_boundary = self.attn_boundary
@@ -1375,7 +1371,7 @@ class ZayaDecoderMLPLayer(nn.Module):
self.res_scale = ResidualScaling(config, layer_id)
else:
self.res_scale = None
(self.ffn_boundary,) = make_stages(
(self.ffn_boundary,) = append_stages(
(
declare_ffn(
sparse=True,
@@ -1384,8 +1380,6 @@ class ZayaDecoderMLPLayer(nn.Module):
),
self.input_norm,
),
previous=declare_attn(),
terminal=layer_id == config.num_hidden_layers - 1,
)
self.entry_boundary = self.ffn_boundary
+73 -14
View File
@@ -1542,6 +1542,33 @@ def mark_end(name):
time_infos[name].pretty_print()
# Set while a layer another pipeline stage holds is built, only for the stage
# boundaries it declares.
_building_neighbour_layer = False
def is_building_neighbour_layer() -> bool:
"""Whether the layer under construction belongs to another pipeline stage
and is built here only to read the stage boundaries it declares. It is
built on the meta device and never loaded or run, so its constructor skips
the host and device resources a running layer needs: tables, streams,
engines and communicators."""
return _building_neighbour_layer
@contextmanager
def building_neighbour_layer():
"""Build a pipeline neighbour layer: on the meta device, with
is_building_neighbour_layer() true."""
global _building_neighbour_layer
outer, _building_neighbour_layer = _building_neighbour_layer, True
try:
with torch.device("meta"):
yield
finally:
_building_neighbour_layer = outer
class LayerFn(Protocol):
def __call__(self, idx: int, prefix: str) -> torch.nn.Module: ...
@@ -1555,9 +1582,16 @@ def make_layers(
return_tuple: bool = False,
offloader_kwargs: Optional[Dict[str, Any]] = None,
) -> Tuple[torch.nn.Module, int, int]:
"""Make a list of layers with the given layer function"""
"""Make a list of layers with the given layer function.
The local layers are built inside one layer stack, so layers that declare
stage boundaries connect in order without naming their neighbours. Across
a pipeline stage boundary the stack learns the neighbouring stage from the
layer itself, built again on the meta device.
"""
# circular imports
from sglang.srt.distributed import get_pp_indices
from sglang.srt.layers.layer_boundary.factories import layer_stack
from sglang.srt.layers.utils import PPMissingLayer
from sglang.srt.utils.offloader import get_offloader
@@ -1571,20 +1605,30 @@ def make_layers(
if pp_rank is not None and pp_size is not None
else (0, num_hidden_layers)
)
modules = torch.nn.ModuleList(
[PPMissingLayer(return_tuple=return_tuple) for _ in range(start_layer)]
+ get_offloader().wrap_modules(
(
layer_fn(idx=idx, prefix=add_prefix(idx, prefix))
for idx in range(start_layer, end_layer)
),
**(offloader_kwargs or {}),
def neighbour(idx):
return functools.partial(
_build_neighbour_layer, layer_fn, idx, add_prefix(idx, prefix)
)
with layer_stack(
previous_layers=[neighbour(idx) for idx in reversed(range(start_layer))],
next_layers=[neighbour(idx) for idx in range(end_layer, num_hidden_layers)],
):
modules = torch.nn.ModuleList(
[PPMissingLayer(return_tuple=return_tuple) for _ in range(start_layer)]
+ get_offloader().wrap_modules(
(
layer_fn(idx=idx, prefix=add_prefix(idx, prefix))
for idx in range(start_layer, end_layer)
),
**(offloader_kwargs or {}),
)
+ [
PPMissingLayer(return_tuple=return_tuple)
for _ in range(end_layer, num_hidden_layers)
]
)
+ [
PPMissingLayer(return_tuple=return_tuple)
for _ in range(end_layer, num_hidden_layers)
]
)
if pp_rank is None or pp_size is None:
return modules
return modules, start_layer, end_layer
@@ -1613,6 +1657,21 @@ def make_pp_layers(
)
def _build_neighbour_layer(layer_fn: LayerFn, idx: int, prefix: str) -> None:
"""Build a layer another pipeline stage holds, only for the stage
boundaries it declares (see building_neighbour_layer). RoPE modules it
adds to the shared cache are meta, so they are dropped again."""
from sglang.srt.layers.rotary_embedding.factory import _ROPE_DICT
cached = set(_ROPE_DICT)
try:
with building_neighbour_layer():
layer_fn(idx=idx, prefix=prefix)
finally:
for key in set(_ROPE_DICT) - cached:
del _ROPE_DICT[key]
def set_random_seed(seed: int) -> None:
"""Set the random seed for all libraries."""
random.seed(seed)
+23 -2
View File
@@ -4,15 +4,36 @@ from types import SimpleNamespace
from sglang.srt.layers.layer_boundary.construction import StagePlan
from sglang.srt.layers.layer_boundary.factories import (
append_stages,
declare_attn,
declare_ffn,
make_stages,
layer_stack,
)
from sglang.srt.layers.layer_boundary.ops import keep_output
from sglang.srt.layers.layer_boundary.residual.add_norm import PLAIN_RESIDUAL_OPS
from sglang.srt.layers.layer_boundary.stage import StageBoundary
def build_stages(*stages, previous=None, terminal=False):
"""Bind one sequence of stages in a layer stack of its own.
Args:
*stages: Items of (declaration, norm) or (declaration, norm, options),
as for append_stages.
previous: Declaration of the stage before the first one, standing for
a layer the test does not build; None starts the stack.
terminal: Whether the last stage ends the model's layer stack;
otherwise the next layer's attention follows it.
"""
with layer_stack(
previous_layers=(
[lambda: append_stages((previous, None))] if previous is not None else []
),
next_layers=[] if terminal else [lambda: append_stages((declare_attn(), None))],
):
return append_stages(*stages)
def make_test_stages(
*,
attention_norm,
@@ -42,7 +63,7 @@ def make_test_stages(
output_transform=output,
)
common = {key: options.pop(key) for key in ("fusions",) if key in options}
attn, ffn = make_stages(
attn, ffn = build_stages(
(
attention,
attention_norm,