[Refactor] Build layer stacks in order with append_stages (#42481)
This commit is contained in:
@@ -338,8 +338,8 @@ class XIELU(BaseFusedOp):
|
||||
self.register_buffer("eps", torch.tensor(eps, dtype=dtype))
|
||||
self.with_vector_loads = with_vector_loads
|
||||
# Temporary until xIELU CUDA fully implemented
|
||||
self._beta_scalar = float(self.beta.detach().cpu().float().item())
|
||||
self._eps_scalar = float(self.eps.detach().cpu().float().item())
|
||||
self._beta_scalar = torch.tensor(beta, dtype=dtype, device="cpu").item()
|
||||
self._eps_scalar = torch.tensor(eps, dtype=dtype, device="cpu").item()
|
||||
|
||||
self._xielu_cuda_obj = None
|
||||
try:
|
||||
|
||||
@@ -139,14 +139,12 @@ class GatedResidual(HyperConnectionBase):
|
||||
self.hidden_size * self.hc_count,
|
||||
self.config.hc_lowrank,
|
||||
bias=False,
|
||||
device=torch.get_device_module().current_device(),
|
||||
dtype=config.params_dtype,
|
||||
)
|
||||
self.input_mix_weight_up = nn.Linear(
|
||||
self.config.hc_lowrank,
|
||||
self.hc_count * self.hidden_size,
|
||||
bias=False,
|
||||
device=torch.get_device_module().current_device(),
|
||||
dtype=config.params_dtype,
|
||||
)
|
||||
lowrank = self.config.hc_lowrank
|
||||
@@ -166,7 +164,6 @@ class GatedResidual(HyperConnectionBase):
|
||||
self.hidden_size * self.hc_count,
|
||||
self.hc_count,
|
||||
bias=False,
|
||||
device=torch.get_device_module().current_device(),
|
||||
dtype=config.params_dtype,
|
||||
)
|
||||
# hc_combine rejects other shapes; device and dtype are checked per call.
|
||||
|
||||
@@ -40,11 +40,10 @@ from sglang.srt.layers.layer_boundary.contracts import (
|
||||
)
|
||||
from sglang.srt.layers.layer_boundary.exit import ExitDecision, FfnExit, MixerExit
|
||||
from sglang.srt.layers.layer_boundary.factories import (
|
||||
append_stages,
|
||||
declare_attn,
|
||||
declare_ffn,
|
||||
make_attn_stage,
|
||||
make_ffn_stage,
|
||||
make_stages,
|
||||
layer_stack,
|
||||
)
|
||||
from sglang.srt.layers.layer_boundary.layout import (
|
||||
Layout,
|
||||
@@ -85,9 +84,8 @@ from sglang.srt.layers.layer_boundary.residual.mhc import (
|
||||
__all__ = [
|
||||
"declare_attn",
|
||||
"declare_ffn",
|
||||
"make_attn_stage",
|
||||
"make_ffn_stage",
|
||||
"make_stages",
|
||||
"append_stages",
|
||||
"layer_stack",
|
||||
"PLAIN_ADD",
|
||||
"AttentionInputs",
|
||||
"StagePath",
|
||||
|
||||
@@ -145,8 +145,11 @@ class StagePlan:
|
||||
finishes_directly: Attention can publish its output with finish instead of
|
||||
an exit scope and output transport.
|
||||
qkv_latent_func: Optional hook for prepared attention input.
|
||||
fusions: Optional backend provider of ordered consumer fusion candidates
|
||||
and the producer deferral policies (see make_attn_stage).
|
||||
fusions: Optional backend provider. Consumer side: ordered
|
||||
attention_input(plan) and ffn_input(plan) candidates. Producer side
|
||||
(FFN exit): can_defer_finalize(plan, batch), called on every exit,
|
||||
and can_defer_all_reduce(plan, batch), called when LoRA or TP1
|
||||
shared experts are enabled.
|
||||
|
||||
The plan owns static paths, not per-forward tensors or a neighbour's norm.
|
||||
Runtime residual state belongs to the ForwardBatch's ResidualStream.
|
||||
|
||||
@@ -2,6 +2,9 @@
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import contextlib
|
||||
import os
|
||||
import sys
|
||||
from dataclasses import dataclass, replace
|
||||
from types import MappingProxyType
|
||||
from typing import Mapping, Optional
|
||||
@@ -48,6 +51,7 @@ from sglang.srt.layers.layer_boundary.residual.add_norm import (
|
||||
PLAIN_ADD,
|
||||
REPLACE_AT_EXIT,
|
||||
)
|
||||
from sglang.srt.layers.layer_boundary.stage import StageBoundary
|
||||
from sglang.srt.layers.moe import is_moe_input_scattered_across_dp_ranks
|
||||
from sglang.srt.runtime_context import get_exec, get_parallel
|
||||
|
||||
@@ -194,9 +198,10 @@ class StageDeclaration:
|
||||
update: Producer operation that writes compute output into the residual.
|
||||
sparse: Whether this FFN is a MoE; used to resolve input/output rows.
|
||||
Expert routing and all-to-all remain inside the MoE computation.
|
||||
terminal: Whether this stage ends the model's layer stack. Prevents
|
||||
leaving work that requires a following layer; a finalize handoff
|
||||
may still reach the terminal norm when the fusion provider allows it.
|
||||
terminal: Whether this stage ends the model's layer stack, as the
|
||||
stack records it. Prevents leaving work that requires a following
|
||||
layer; a finalize handoff may still reach the terminal norm when
|
||||
the fusion provider allows it.
|
||||
output_transform: Optional operation on the contribution before the
|
||||
residual update. An FFN's exit runs it under an explicit
|
||||
reduction-order contract; an attention's is run by the input of
|
||||
@@ -207,8 +212,8 @@ class StageDeclaration:
|
||||
dense_tp_size: Dense FFN compute width: None uses the configured width,
|
||||
1 means local compute, and the full TP size means TP compute.
|
||||
exit_rows: Required FFN output rows at the layer or branch exit.
|
||||
previous: Declaration whose output this stage consumes. It may be
|
||||
reconstructed locally, including across pipeline ranks.
|
||||
previous: Declaration whose output this stage consumes, as the stack
|
||||
records it; across pipeline ranks it is built locally.
|
||||
prepared_from: Declaration whose already-read input a branch reuses.
|
||||
Mutually exclusive with previous; avoids a second update/read.
|
||||
|
||||
@@ -263,11 +268,8 @@ class StageConnection:
|
||||
|
||||
def declare_attn(
|
||||
*,
|
||||
previous: Optional[StageDeclaration] = None,
|
||||
prepared_from: Optional[StageDeclaration] = None,
|
||||
read=NORM_QUANT_READOUT,
|
||||
update=PLAIN_ADD,
|
||||
terminal=False,
|
||||
reduction=ProducerReduction.ALWAYS_PARTIAL,
|
||||
gathers_attn_tp_input=True,
|
||||
output_transform=None,
|
||||
@@ -275,11 +277,8 @@ def declare_attn(
|
||||
"""Declare attention or a mixer; construct its executable boundary later.
|
||||
|
||||
Args:
|
||||
previous: Producer declaration whose output this stage consumes.
|
||||
prepared_from: Source of an already-read branch input, instead of previous.
|
||||
read: Input operation; defaults to normalization with quantization support.
|
||||
update: Operation that adds this stage's output to the residual.
|
||||
terminal: Whether this stage ends the model's layer stack.
|
||||
reduction: ALWAYS_PARTIAL for an attention whose sum the next stage's
|
||||
input always completes; EXIT_SCOPED for a mixer whose exit decides.
|
||||
gathers_attn_tp_input: Whether compute gathers attention-TP input slices itself.
|
||||
@@ -300,9 +299,6 @@ def declare_attn(
|
||||
StageKind.ATTENTION,
|
||||
read,
|
||||
update,
|
||||
previous=previous,
|
||||
prepared_from=prepared_from,
|
||||
terminal=terminal,
|
||||
output_transform=output_transform,
|
||||
reduction=reduction,
|
||||
gathers_attn_tp_input=gathers_attn_tp_input,
|
||||
@@ -311,12 +307,9 @@ def declare_attn(
|
||||
|
||||
def declare_ffn(
|
||||
*,
|
||||
previous: Optional[StageDeclaration] = None,
|
||||
prepared_from: Optional[StageDeclaration] = None,
|
||||
sparse=False,
|
||||
read=NORM_READOUT,
|
||||
update=PLAIN_ADD,
|
||||
terminal=False,
|
||||
output_transform=None,
|
||||
next_layer_sparse=False,
|
||||
dense_tp_size=None,
|
||||
@@ -325,12 +318,9 @@ def declare_ffn(
|
||||
"""Declare a dense or MoE FFN independently of its compute module.
|
||||
|
||||
Args:
|
||||
previous: Producer declaration whose output this stage consumes.
|
||||
prepared_from: Source of an already-read branch input, instead of previous.
|
||||
sparse: Whether the FFN is a MoE; EP dispatch stays inside compute.
|
||||
read: Operation deriving FFN input from the residual.
|
||||
update: Operation writing FFN output into the residual.
|
||||
terminal: Whether this stage ends the model's layer stack.
|
||||
output_transform: Optional contribution transform before residual update.
|
||||
next_layer_sparse: Whether the next decoder layer's FFN is sparse; used only
|
||||
to derive the TBO exit rows when exit_rows is not supplied.
|
||||
@@ -347,9 +337,6 @@ def declare_ffn(
|
||||
read,
|
||||
update,
|
||||
sparse=sparse,
|
||||
previous=previous,
|
||||
prepared_from=prepared_from,
|
||||
terminal=terminal,
|
||||
output_transform=output_transform,
|
||||
dense_tp_size=dense_tp_size,
|
||||
exit_rows=exit_rows or tbo_exit_rows(sparse, next_layer_sparse),
|
||||
@@ -569,167 +556,364 @@ def _incoming(stage):
|
||||
)
|
||||
|
||||
|
||||
def _connections(stage, following=None):
|
||||
if following is not None and following.previous != stage:
|
||||
raise ValueError("the following declaration must consume this stage's output")
|
||||
incoming = _incoming(stage)
|
||||
outgoing = _connect(
|
||||
stage,
|
||||
following,
|
||||
residual_from=incoming,
|
||||
)
|
||||
return incoming, outgoing
|
||||
class _Append:
|
||||
"""One append_stages call: its validated stages, the boundaries it
|
||||
returned, which the stack fills in when it closes, and where it was made."""
|
||||
|
||||
__slots__ = ("declarations", "bindings", "prepared_from", "boundaries", "origin")
|
||||
|
||||
def __init__(self, declarations, bindings, prepared_from, boundaries, origin):
|
||||
self.declarations = declarations
|
||||
self.bindings = bindings
|
||||
self.prepared_from = prepared_from
|
||||
self.boundaries = boundaries
|
||||
self.origin = origin
|
||||
|
||||
|
||||
def make_attn_stage(
|
||||
*,
|
||||
declaration,
|
||||
norm,
|
||||
following: Optional[StageDeclaration] = None,
|
||||
qkv_latent_func=None,
|
||||
fusions=None,
|
||||
):
|
||||
"""Resolve one attention/mixer's boundaries and bind its input norm.
|
||||
class _LayerStack:
|
||||
"""A layer stack under construction: what was appended, in order, and how
|
||||
to reach the layers other pipeline ranks hold on either side of it."""
|
||||
|
||||
__slots__ = ("appends", "previous_layers", "next_layers")
|
||||
|
||||
def __init__(self, previous_layers=(), next_layers=()):
|
||||
self.appends = []
|
||||
self.previous_layers = previous_layers
|
||||
self.next_layers = next_layers
|
||||
|
||||
|
||||
# The stack being built; layer_stack saves and restores an outer one.
|
||||
_stack: Optional[_LayerStack] = None
|
||||
|
||||
|
||||
@contextlib.contextmanager
|
||||
def layer_stack(*, previous_layers=(), next_layers=()):
|
||||
"""Open a layer stack that append_stages extends in order.
|
||||
|
||||
Every stage binds when the stack closes, once its producer and its
|
||||
consumer are both known: the stage appended before and after it. The last
|
||||
stage ends the model's layer stack unless a later layer declares a stage.
|
||||
|
||||
Args:
|
||||
declaration: Attention StageDeclaration, including its input source.
|
||||
norm: This consumer's normalization module, never its neighbour's norm.
|
||||
following: Local consumer declaration whose previous is declaration.
|
||||
None leaves a layer or stack exit for an independently bound reader.
|
||||
qkv_latent_func: Optional attention input hook, invoked after preparation
|
||||
and movement onto the compute input rows.
|
||||
fusions: Optional backend provider. Consumer side: ordered
|
||||
attention_input(plan) and ffn_input(plan) candidates. Producer side
|
||||
(FFN exit): can_defer_finalize(plan, batch), called on every exit,
|
||||
and can_defer_all_reduce(plan, batch), called when LoRA or TP1 shared
|
||||
experts are enabled.
|
||||
|
||||
Returns:
|
||||
A StageBoundary with precomputed paths for supported batch variants.
|
||||
previous_layers: Callables that build, nearest first, the layers before
|
||||
this stack that another pipeline rank holds. Called only if this
|
||||
stack appended stages, after its own layers are built, until one of
|
||||
them declares a stage: its last stage is the producer of this
|
||||
stack's first. What they build is discarded.
|
||||
next_layers: Likewise for the layers after this stack, whose first
|
||||
declared stage is the consumer of this stack's last.
|
||||
"""
|
||||
if declaration.kind is not StageKind.ATTENTION:
|
||||
raise TypeError("make_attn_stage requires an attention declaration")
|
||||
incoming, outgoing = _connections(declaration, following)
|
||||
return _bind_stage(
|
||||
global _stack
|
||||
outer = _stack
|
||||
stack = _LayerStack(previous_layers, next_layers)
|
||||
_stack = stack
|
||||
try:
|
||||
yield stack
|
||||
if stack.appends:
|
||||
_bind_stack(
|
||||
stack.appends,
|
||||
previous=_neighbour_stage(stack.previous_layers, last=True),
|
||||
following=_neighbour_stage(stack.next_layers, last=False),
|
||||
)
|
||||
finally:
|
||||
_stack = outer
|
||||
|
||||
|
||||
def _neighbour_stage(build_layers, *, last):
|
||||
"""The stage a neighbouring layer declares next to this stack: the last
|
||||
one of the nearest layer before it, or the first of the nearest after.
|
||||
Branches are side paths, so they never stand next to the stack."""
|
||||
global _stack
|
||||
for build_layer in build_layers:
|
||||
outer = _stack
|
||||
_stack = _LayerStack()
|
||||
try:
|
||||
build_layer()
|
||||
appends = [a for a in _stack.appends if a.prepared_from is None]
|
||||
finally:
|
||||
_stack = outer
|
||||
if appends:
|
||||
return appends[-1].declarations[-1] if last else appends[0].declarations[0]
|
||||
return None
|
||||
|
||||
|
||||
def _require_stack() -> _LayerStack:
|
||||
if _stack is None:
|
||||
raise RuntimeError(
|
||||
"append_stages needs an open layer stack; build the layers inside "
|
||||
"make_layers or layer_stack"
|
||||
)
|
||||
return _stack
|
||||
|
||||
|
||||
def _note_origin(error: Exception, origin: str) -> None:
|
||||
add_note = getattr(error, "add_note", None)
|
||||
if add_note is not None:
|
||||
add_note(f"while binding the stages appended at {origin}")
|
||||
|
||||
|
||||
class _PendingStage:
|
||||
"""The last stage of a linear append. Its consumer is the next append's
|
||||
first stage, or the stack exit, so it binds only once that is known."""
|
||||
|
||||
__slots__ = (
|
||||
"boundary",
|
||||
"declaration",
|
||||
"norm",
|
||||
"options",
|
||||
"predecessor",
|
||||
"predecessor_incoming",
|
||||
"predecessor_boundary",
|
||||
"origin",
|
||||
)
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
boundary,
|
||||
declaration,
|
||||
norm,
|
||||
incoming,
|
||||
outgoing,
|
||||
qkv_latent_func=qkv_latent_func,
|
||||
fusions=fusions,
|
||||
)
|
||||
options,
|
||||
predecessor,
|
||||
predecessor_incoming,
|
||||
predecessor_boundary,
|
||||
origin,
|
||||
):
|
||||
self.boundary = boundary
|
||||
self.declaration = declaration
|
||||
self.norm = norm
|
||||
self.options = options
|
||||
# The stage before it within the same append, or None when it is the
|
||||
# append's only stage and takes its input from the stack.
|
||||
self.predecessor = predecessor
|
||||
self.predecessor_incoming = predecessor_incoming
|
||||
self.predecessor_boundary = predecessor_boundary
|
||||
self.origin = origin
|
||||
|
||||
|
||||
def make_ffn_stage(
|
||||
*,
|
||||
declaration,
|
||||
norm,
|
||||
following: Optional[StageDeclaration] = None,
|
||||
fusions=None,
|
||||
):
|
||||
"""Resolve one FFN's boundaries and bind its input norm.
|
||||
class _Chain:
|
||||
"""The stage a following append extends, and the one stage still waiting
|
||||
for its consumer, while the stack binds its appends in order."""
|
||||
|
||||
Args:
|
||||
declaration: FFN StageDeclaration, including its input source.
|
||||
norm: This FFN's input normalization module.
|
||||
following: Local consumer declaration whose previous is declaration;
|
||||
None leaves a layer or stack exit for an independently bound reader.
|
||||
fusions: Optional backend fusion provider, as in make_attn_stage.
|
||||
__slots__ = ("previous", "pending")
|
||||
|
||||
Returns:
|
||||
A StageBoundary. Expert routing and all-to-all stay inside compute.
|
||||
def __init__(self, previous):
|
||||
self.previous = _detached(previous)
|
||||
self.pending = None
|
||||
|
||||
|
||||
def _detached(declaration):
|
||||
"""The declaration a following append extends, without its own history.
|
||||
|
||||
A stage's incoming edge reads its producer's own incoming edge only when
|
||||
the producer is an attention that always leaves its sum; the copy has no
|
||||
history, so that lookback stops at the producer.
|
||||
"""
|
||||
if declaration.kind is not StageKind.FFN:
|
||||
raise TypeError("make_ffn_stage requires an FFN declaration")
|
||||
incoming, outgoing = _connections(declaration, following)
|
||||
return _bind_stage(
|
||||
declaration,
|
||||
norm,
|
||||
incoming,
|
||||
outgoing,
|
||||
fusions=fusions,
|
||||
)
|
||||
if declaration is None:
|
||||
return None
|
||||
return replace(declaration, previous=None, prepared_from=None)
|
||||
|
||||
|
||||
def make_stages(
|
||||
*stages, previous=None, prepared_from=None, following=None, terminal=False
|
||||
):
|
||||
"""Bind a local linear sequence of any positive number of stages.
|
||||
def _bind_stack(appends, *, previous, following):
|
||||
"""Bind every appended stage, in order, and fill in the boundaries each
|
||||
append returned."""
|
||||
chain = _Chain(previous)
|
||||
# A returned declaration's boundary as bound, for the branches that read it.
|
||||
sources = {}
|
||||
bound = []
|
||||
for append in appends:
|
||||
prepared_from = append.prepared_from
|
||||
if prepared_from is not None:
|
||||
source = sources.get(id(prepared_from))
|
||||
if source is None:
|
||||
raise ValueError(
|
||||
"prepared_from must be the declaration of a stage appended "
|
||||
f"earlier to the same layer stack (at {append.origin})"
|
||||
)
|
||||
prepared_from = source.declaration
|
||||
boundaries = _extend(chain, append, prepared_from)
|
||||
for returned, boundary in zip(append.boundaries, boundaries):
|
||||
sources[id(returned.declaration)] = boundary
|
||||
bound.append(boundaries)
|
||||
# The last stage's consumer is the next rank's first stage, if any;
|
||||
# without one it ends the model's layer stack.
|
||||
_bind_pending(chain, consumer=following, terminal=following is None)
|
||||
for append, boundaries in zip(appends, bound):
|
||||
for returned, boundary in zip(append.boundaries, boundaries):
|
||||
returned.plan = boundary.plan
|
||||
returned.declaration = boundary.declaration
|
||||
|
||||
Args:
|
||||
*stages: Items of (declaration, norm) or (declaration, norm, options).
|
||||
Declarations must have no source or terminal flag. Options are
|
||||
constructor keywords: fusions, and qkv_latent_func for attention.
|
||||
previous: External producer declaration consumed by the first stage.
|
||||
prepared_from: Already-read input declaration reused by the first stage
|
||||
of a branch; mutually exclusive with previous.
|
||||
following: External consumer declaration after the last local stage.
|
||||
None denotes a layer or stack exit with an independently bound read.
|
||||
terminal: Marks only the final stage as the end of the model's stack.
|
||||
|
||||
Returns:
|
||||
A tuple of independent StageBoundary objects in declaration order.
|
||||
Sources are connected on copied declarations; caller inputs are unchanged.
|
||||
No sequence object or runtime routing is retained.
|
||||
"""
|
||||
if not stages:
|
||||
raise ValueError("make_stages needs at least one stage")
|
||||
if previous is not None and prepared_from is not None:
|
||||
raise ValueError("choose a previous output or a prepared branch input")
|
||||
declarations = []
|
||||
bindings = []
|
||||
for index, item in enumerate(stages):
|
||||
def _bind_pending(chain: _Chain, *, consumer, terminal):
|
||||
pending = chain.pending
|
||||
if pending is None:
|
||||
return
|
||||
chain.pending = None
|
||||
try:
|
||||
declaration = replace(pending.declaration, terminal=terminal)
|
||||
if pending.predecessor is None:
|
||||
incoming = _incoming(declaration)
|
||||
else:
|
||||
incoming = _connect(
|
||||
pending.predecessor,
|
||||
declaration,
|
||||
residual_from=pending.predecessor_incoming,
|
||||
)
|
||||
outgoing = _connect(declaration, consumer, residual_from=incoming)
|
||||
bound = _bind_stage(
|
||||
declaration, pending.norm, incoming, outgoing, **pending.options
|
||||
)
|
||||
except Exception as error:
|
||||
_note_origin(error, pending.origin)
|
||||
raise
|
||||
pending.boundary.plan = bound.plan
|
||||
pending.boundary.declaration = declaration
|
||||
if pending.predecessor_boundary is not None:
|
||||
_carry_capture(pending.predecessor_boundary, pending.boundary)
|
||||
|
||||
|
||||
def _carry_capture(producer, consumer):
|
||||
"""Let an attention that always leaves its sum preserve the residual its
|
||||
FFN's entry needs for capture. Only between stages of one append."""
|
||||
if not (
|
||||
producer.kind is StageKind.ATTENTION
|
||||
and producer.declaration.reduction is ProducerReduction.ALWAYS_PARTIAL
|
||||
and consumer.kind is StageKind.FFN
|
||||
):
|
||||
return
|
||||
for variant, steps in producer.plan.paths.items():
|
||||
next_steps = consumer.plan.paths[variant]
|
||||
predicate = next_steps.entry.preserves_residual
|
||||
if predicate is not None:
|
||||
producer.plan.paths[variant] = msgspec.structs.replace(
|
||||
steps,
|
||||
entry=msgspec.structs.replace(
|
||||
steps.entry, capture_preserves_residual=predicate
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
_STAGE_OPTIONS = {
|
||||
StageKind.ATTENTION: frozenset({"qkv_latent_func", "fusions"}),
|
||||
StageKind.FFN: frozenset({"fusions"}),
|
||||
}
|
||||
|
||||
|
||||
def _stage_items(stages):
|
||||
declarations, bindings = [], []
|
||||
for item in stages:
|
||||
if len(item) not in (2, 3):
|
||||
raise ValueError("a stage needs (declaration, norm[, options])")
|
||||
declaration, norm = item[:2]
|
||||
if not isinstance(declaration, StageDeclaration):
|
||||
raise TypeError("make_stages requires stage declarations")
|
||||
raise TypeError("append_stages requires stage declarations")
|
||||
if declaration.previous is not None or declaration.prepared_from is not None:
|
||||
raise ValueError("pass input sources to make_stages, not its declarations")
|
||||
raise ValueError("a declaration takes its source from the layer stack")
|
||||
if declaration.terminal:
|
||||
raise ValueError("pass terminal to make_stages, not its declarations")
|
||||
declaration = replace(
|
||||
declaration,
|
||||
previous=previous if index == 0 else declarations[-1],
|
||||
prepared_from=prepared_from if index == 0 else None,
|
||||
terminal=terminal and index == len(stages) - 1,
|
||||
)
|
||||
raise ValueError("the layer stack marks the terminal stage")
|
||||
options = dict(item[2] if len(item) == 3 else {})
|
||||
unexpected = options.keys() - _STAGE_OPTIONS[declaration.kind]
|
||||
if unexpected:
|
||||
raise TypeError(
|
||||
f"unsupported {declaration.kind.name} stage options: "
|
||||
f"{sorted(unexpected)}"
|
||||
)
|
||||
declarations.append(declaration)
|
||||
bindings.append((norm, item[2] if len(item) == 3 else {}))
|
||||
external_consumer = following
|
||||
boundaries = []
|
||||
incoming = _incoming(declarations[0])
|
||||
for index, (declaration, (norm, options)) in enumerate(zip(declarations, bindings)):
|
||||
following = (
|
||||
declarations[index + 1]
|
||||
if index + 1 < len(declarations)
|
||||
else external_consumer
|
||||
)
|
||||
outgoing = _connect(declaration, following, residual_from=incoming)
|
||||
options = dict(options)
|
||||
if declaration.kind is StageKind.FFN:
|
||||
unexpected = options.keys() - {
|
||||
"fusions",
|
||||
}
|
||||
if unexpected:
|
||||
raise TypeError(f"unsupported FFN options: {sorted(unexpected)}")
|
||||
boundaries.append(_bind_stage(declaration, norm, incoming, outgoing, **options))
|
||||
incoming = outgoing
|
||||
for producer, consumer in zip(boundaries, boundaries[1:]):
|
||||
if not (
|
||||
producer.kind is StageKind.ATTENTION
|
||||
and producer.declaration.reduction is ProducerReduction.ALWAYS_PARTIAL
|
||||
and consumer.kind is StageKind.FFN
|
||||
):
|
||||
continue
|
||||
for variant, steps in producer.plan.paths.items():
|
||||
next_steps = consumer.plan.paths[variant]
|
||||
predicate = next_steps.entry.preserves_residual
|
||||
if predicate is not None:
|
||||
producer.plan.paths[variant] = msgspec.structs.replace(
|
||||
steps,
|
||||
entry=msgspec.structs.replace(
|
||||
steps.entry, capture_preserves_residual=predicate
|
||||
bindings.append((norm, options))
|
||||
return declarations, bindings
|
||||
|
||||
|
||||
def _origin(stack: _LayerStack) -> str:
|
||||
"""Where the current append_stages call was made, and its place in the stack."""
|
||||
frame = sys._getframe(2)
|
||||
return (
|
||||
f"{os.path.basename(frame.f_code.co_filename)}:{frame.f_lineno} "
|
||||
f"(append {len(stack.appends)} of its layer stack)"
|
||||
)
|
||||
|
||||
|
||||
def append_stages(*stages, prepared_from=None):
|
||||
"""Append a local linear sequence of stages to the open layer stack.
|
||||
|
||||
Args:
|
||||
*stages: Items of (declaration, norm) or (declaration, norm, options).
|
||||
Declarations carry no source or terminal flag. Options are
|
||||
constructor keywords: fusions, and qkv_latent_func for attention.
|
||||
prepared_from: The ``declaration`` of a boundary an earlier call
|
||||
returned on this stack: the already-read stage this sequence
|
||||
branches from. A branch is a side path the model merges back
|
||||
explicitly: it neither extends the stack nor waits for a consumer,
|
||||
so each of its stages binds with only the stages of the branch.
|
||||
|
||||
Returns:
|
||||
A tuple of StageBoundary objects in declaration order. Their
|
||||
declarations are usable at once and gain their place in the stack when
|
||||
it closes, which is also when their plans are bound.
|
||||
"""
|
||||
if not stages:
|
||||
raise ValueError("append_stages needs at least one stage")
|
||||
stack = _require_stack()
|
||||
declarations, bindings = _stage_items(stages)
|
||||
declarations = [replace(d) for d in declarations]
|
||||
boundaries = tuple(StageBoundary(None, declaration=d) for d in declarations)
|
||||
stack.appends.append(
|
||||
_Append(declarations, bindings, prepared_from, boundaries, _origin(stack))
|
||||
)
|
||||
return boundaries
|
||||
|
||||
|
||||
def _extend(chain, append, prepared_from):
|
||||
"""Chain one append onto the stack and bind what can be bound."""
|
||||
declarations, bindings = append.declarations, append.bindings
|
||||
branch = prepared_from is not None
|
||||
if not branch:
|
||||
# This append's first stage is the consumer the pending stage waited for.
|
||||
_bind_pending(chain, consumer=declarations[0], terminal=False)
|
||||
try:
|
||||
chained = []
|
||||
for index, declaration in enumerate(declarations):
|
||||
chained.append(
|
||||
replace(
|
||||
declaration,
|
||||
previous=(
|
||||
chained[-1] if index else (None if branch else chain.previous)
|
||||
),
|
||||
prepared_from=prepared_from if index == 0 else None,
|
||||
)
|
||||
)
|
||||
boundaries = []
|
||||
last = len(chained) - 1
|
||||
# incoming feeds the stage being bound; previous_incoming fed the one
|
||||
# before.
|
||||
previous_incoming, incoming = None, _incoming(chained[0])
|
||||
for index, (declaration, (norm, options)) in enumerate(zip(chained, bindings)):
|
||||
if index == last and not branch:
|
||||
boundaries.append(StageBoundary(None, declaration=declaration))
|
||||
chain.pending = _PendingStage(
|
||||
boundaries[-1],
|
||||
declaration,
|
||||
norm,
|
||||
options,
|
||||
predecessor=chained[index - 1] if index else None,
|
||||
predecessor_incoming=previous_incoming,
|
||||
predecessor_boundary=boundaries[-2] if index else None,
|
||||
origin=append.origin,
|
||||
)
|
||||
break
|
||||
following = chained[index + 1] if index < last else None
|
||||
outgoing = _connect(declaration, following, residual_from=incoming)
|
||||
boundaries.append(
|
||||
_bind_stage(declaration, norm, incoming, outgoing, **options)
|
||||
)
|
||||
previous_incoming, incoming = incoming, outgoing
|
||||
except Exception as error:
|
||||
_note_origin(error, append.origin)
|
||||
raise
|
||||
if branch:
|
||||
for producer, consumer in zip(boundaries, boundaries[1:]):
|
||||
_carry_capture(producer, consumer)
|
||||
else:
|
||||
# The pair that ends on the pending stage is carried when it binds.
|
||||
for producer, consumer in zip(boundaries[:-2], boundaries[1:-1]):
|
||||
_carry_capture(producer, consumer)
|
||||
chain.previous = _detached(chained[-1])
|
||||
return tuple(boundaries)
|
||||
|
||||
@@ -87,6 +87,7 @@ from sglang.srt.utils import (
|
||||
is_npu,
|
||||
round_up,
|
||||
)
|
||||
from sglang.srt.utils.common import is_building_neighbour_layer
|
||||
from sglang.srt.utils.custom_op import register_custom_op
|
||||
|
||||
_is_hip = is_hip()
|
||||
@@ -163,6 +164,8 @@ def create_moe_dispatcher(
|
||||
moe_runner_config: MoeRunnerConfig,
|
||||
quant_method: FusedMoEMethodBase,
|
||||
) -> BaseDispatcher:
|
||||
if is_building_neighbour_layer():
|
||||
return StandardDispatcher(moe_runner_config)
|
||||
a2a_backend = get_moe_a2a_backend()
|
||||
if a2a_backend.is_none() and is_npu():
|
||||
return AscendTPDispatcher(moe_runner_config)
|
||||
|
||||
@@ -20,6 +20,7 @@ from sglang.srt.runtime_context import (
|
||||
get_schedule,
|
||||
)
|
||||
from sglang.srt.utils import get_compiler_backend
|
||||
from sglang.srt.utils.common import is_building_neighbour_layer
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from sglang.srt.layers.moe import MoeRunnerConfig
|
||||
@@ -215,7 +216,7 @@ class KTEPWrapperMethod(FusedMoEMethodBase):
|
||||
|
||||
# 2. Initialize KT wrapper for CPU experts
|
||||
# CPU experts: num_gpu_experts to num_experts-1
|
||||
if self.tp_rank == 0:
|
||||
if self.tp_rank == 0 and not is_building_neighbour_layer():
|
||||
self.wrapper = KTMoEWrapper(
|
||||
layer_idx=self.kt_config.layer_idx,
|
||||
num_experts=num_experts,
|
||||
|
||||
@@ -29,9 +29,9 @@ from transformers import ApertusConfig
|
||||
from sglang.srt.layers.activation import XIELU
|
||||
from sglang.srt.layers.aux_hidden_states import AuxHiddenStateList
|
||||
from sglang.srt.layers.layer_boundary import (
|
||||
append_stages,
|
||||
declare_attn,
|
||||
declare_ffn,
|
||||
make_stages,
|
||||
)
|
||||
from sglang.srt.layers.layer_boundary.residual import batch as residual_batch
|
||||
from sglang.srt.layers.layernorm import RMSNorm
|
||||
@@ -265,14 +265,12 @@ class ApertusDecoderLayer(nn.Module):
|
||||
self.feedforward_layernorm = RMSNorm(
|
||||
config.hidden_size, eps=config.rms_norm_eps
|
||||
)
|
||||
self.attn_boundary, self.ffn_boundary = make_stages(
|
||||
self.attn_boundary, self.ffn_boundary = append_stages(
|
||||
(declare_attn(), self.attention_layernorm),
|
||||
(
|
||||
declare_ffn(sparse=False, next_layer_sparse=False),
|
||||
self.feedforward_layernorm,
|
||||
),
|
||||
previous=declare_ffn() if layer_id != 0 else None,
|
||||
terminal=layer_id == config.num_hidden_layers - 1,
|
||||
)
|
||||
|
||||
def forward(
|
||||
|
||||
@@ -23,9 +23,9 @@ from transformers import LlamaConfig
|
||||
from sglang.srt.layers.activation import get_act_fn
|
||||
from sglang.srt.layers.aux_hidden_states import AuxHiddenStateList
|
||||
from sglang.srt.layers.layer_boundary import (
|
||||
append_stages,
|
||||
declare_attn,
|
||||
declare_ffn,
|
||||
make_stages,
|
||||
)
|
||||
from sglang.srt.layers.layer_boundary.residual import batch as residual_batch
|
||||
from sglang.srt.layers.layernorm import RMSNorm
|
||||
@@ -241,14 +241,12 @@ class ArceeDecoderLayer(nn.Module):
|
||||
self.post_attention_layernorm = RMSNorm(
|
||||
config.hidden_size, eps=config.rms_norm_eps
|
||||
)
|
||||
self.attn_boundary, self.ffn_boundary = make_stages(
|
||||
self.attn_boundary, self.ffn_boundary = append_stages(
|
||||
(declare_attn(), self.input_layernorm),
|
||||
(
|
||||
declare_ffn(sparse=False, next_layer_sparse=False),
|
||||
self.post_attention_layernorm,
|
||||
),
|
||||
previous=declare_ffn() if layer_id != 0 else None,
|
||||
terminal=layer_id == config.num_hidden_layers - 1,
|
||||
)
|
||||
|
||||
def forward(
|
||||
|
||||
@@ -39,10 +39,10 @@ from sglang.srt.layers.dp_attention import (
|
||||
is_dp_attention_enabled,
|
||||
)
|
||||
from sglang.srt.layers.layer_boundary import (
|
||||
append_stages,
|
||||
declare_attn,
|
||||
declare_ffn,
|
||||
is_dense_ffn_fully_dp,
|
||||
make_stages,
|
||||
)
|
||||
from sglang.srt.layers.layer_boundary.residual import batch as residual_batch
|
||||
from sglang.srt.layers.layernorm import RMSNorm
|
||||
@@ -664,9 +664,6 @@ class BailingMoEBlock(nn.Module):
|
||||
self.is_layer_sparse = self._is_layer_sparse(
|
||||
config, layer_id=layer_id, is_nextn=is_nextn
|
||||
)
|
||||
is_previous_layer_sparse = self._is_layer_sparse(
|
||||
config, layer_id=layer_id - 1, is_nextn=False
|
||||
)
|
||||
is_next_layer_sparse = self._is_layer_sparse(
|
||||
config, layer_id=layer_id + 1, is_nextn=False
|
||||
)
|
||||
@@ -696,7 +693,7 @@ class BailingMoEBlock(nn.Module):
|
||||
|
||||
self.post_attention_layernorm = RMSNorm(hidden_size, eps=config.rms_norm_eps)
|
||||
|
||||
self.attn_boundary, self.ffn_boundary = make_stages(
|
||||
self.attn_boundary, self.ffn_boundary = append_stages(
|
||||
(declare_attn(), self.input_layernorm),
|
||||
(
|
||||
declare_ffn(
|
||||
@@ -705,12 +702,6 @@ class BailingMoEBlock(nn.Module):
|
||||
),
|
||||
self.post_attention_layernorm,
|
||||
),
|
||||
previous=declare_ffn(
|
||||
sparse=is_previous_layer_sparse, next_layer_sparse=self.is_layer_sparse
|
||||
)
|
||||
if layer_id != 0
|
||||
else None,
|
||||
terminal=layer_id == (1 if is_nextn else config.num_hidden_layers) - 1,
|
||||
)
|
||||
|
||||
def _is_layer_sparse(
|
||||
|
||||
@@ -19,9 +19,9 @@ from sglang.srt.layers.dp_attention import (
|
||||
is_dp_attention_enabled,
|
||||
)
|
||||
from sglang.srt.layers.layer_boundary import (
|
||||
append_stages,
|
||||
declare_attn,
|
||||
declare_ffn,
|
||||
make_stages,
|
||||
)
|
||||
from sglang.srt.layers.layer_boundary.residual import batch as residual_batch
|
||||
from sglang.srt.layers.layernorm import RMSNorm
|
||||
@@ -783,7 +783,6 @@ class BailingMoELinearDecoderLayer(nn.Module):
|
||||
self.expert_num = config.num_experts
|
||||
self.hidden_size = config.hidden_size
|
||||
is_moe_layer = self._is_layer_sparse(config, self.layer_id, is_nextn=is_nextn)
|
||||
is_previous_moe_layer = self._is_layer_sparse(config, self.layer_id - 1)
|
||||
is_next_layer_moe_layer = self._is_layer_sparse(config, self.layer_id + 1)
|
||||
if self.expert_num == 1:
|
||||
self.mlp = BailingMLP(
|
||||
@@ -821,7 +820,7 @@ class BailingMoELinearDecoderLayer(nn.Module):
|
||||
if config.attention_type == 1 and self.use_mla
|
||||
else None
|
||||
)
|
||||
self.attn_boundary, self.ffn_boundary = make_stages(
|
||||
self.attn_boundary, self.ffn_boundary = append_stages(
|
||||
(
|
||||
declare_attn(),
|
||||
self.input_layernorm,
|
||||
@@ -834,12 +833,6 @@ class BailingMoELinearDecoderLayer(nn.Module):
|
||||
),
|
||||
self.post_attention_layernorm,
|
||||
),
|
||||
previous=declare_ffn(
|
||||
sparse=is_previous_moe_layer, next_layer_sparse=is_moe_layer
|
||||
)
|
||||
if layer_id != 0
|
||||
else None,
|
||||
terminal=layer_id == (1 if is_nextn else config.num_hidden_layers) - 1,
|
||||
)
|
||||
|
||||
def _is_layer_sparse(
|
||||
|
||||
@@ -27,6 +27,7 @@ from torch import nn
|
||||
from transformers import PretrainedConfig
|
||||
|
||||
from sglang.srt.layers.dp_attention import is_dp_attention_enabled
|
||||
from sglang.srt.layers.layer_boundary import layer_stack
|
||||
from sglang.srt.layers.layer_boundary.residual import batch as residual_batch
|
||||
from sglang.srt.layers.layernorm import RMSNorm
|
||||
from sglang.srt.layers.linear import ReplicatedLinear
|
||||
@@ -109,27 +110,30 @@ class BailingMoEModelNextN(nn.Module):
|
||||
self.is_hybrid = (
|
||||
hasattr(config, "model_type") and config.model_type == "bailing_hybrid"
|
||||
)
|
||||
if self.is_hybrid:
|
||||
config.attention_type = 1
|
||||
decoder_layer_cls = BailingMoELinearDecoderLayer
|
||||
decoder_kwargs = {
|
||||
"quant_config": quant_config,
|
||||
"layer_id": 0,
|
||||
"is_nextn": True,
|
||||
"prefix": add_prefix(f"layers.{config.num_hidden_layers}", prefix),
|
||||
}
|
||||
if _is_bailing_moe_v3_config(config):
|
||||
decoder_layer_cls = BailingMoeV3DecoderLayer
|
||||
decoder_kwargs["num_fused_shared_experts"] = num_fused_shared_experts
|
||||
self.decoder = decoder_layer_cls(config, **decoder_kwargs)
|
||||
else:
|
||||
self.decoder = BailingMoEBlock(
|
||||
config,
|
||||
0,
|
||||
quant_config=quant_config,
|
||||
prefix=add_prefix("decoder", prefix),
|
||||
is_nextn=True,
|
||||
)
|
||||
with layer_stack():
|
||||
if self.is_hybrid:
|
||||
config.attention_type = 1
|
||||
decoder_layer_cls = BailingMoELinearDecoderLayer
|
||||
decoder_kwargs = {
|
||||
"quant_config": quant_config,
|
||||
"layer_id": 0,
|
||||
"is_nextn": True,
|
||||
"prefix": add_prefix(f"layers.{config.num_hidden_layers}", prefix),
|
||||
}
|
||||
if _is_bailing_moe_v3_config(config):
|
||||
decoder_layer_cls = BailingMoeV3DecoderLayer
|
||||
decoder_kwargs["num_fused_shared_experts"] = (
|
||||
num_fused_shared_experts
|
||||
)
|
||||
self.decoder = decoder_layer_cls(config, **decoder_kwargs)
|
||||
else:
|
||||
self.decoder = BailingMoEBlock(
|
||||
config,
|
||||
0,
|
||||
quant_config=quant_config,
|
||||
prefix=add_prefix("decoder", prefix),
|
||||
is_nextn=True,
|
||||
)
|
||||
|
||||
self.shared_head = nn.Module()
|
||||
self.final_layernorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
|
||||
|
||||
@@ -27,10 +27,10 @@ from sglang.srt.layers.activation import SiluAndMul
|
||||
from sglang.srt.layers.aux_hidden_states import AuxHiddenStateList
|
||||
from sglang.srt.layers.dp_attention import is_dp_attention_enabled
|
||||
from sglang.srt.layers.layer_boundary import (
|
||||
append_stages,
|
||||
declare_attn,
|
||||
declare_ffn,
|
||||
is_dense_ffn_fully_dp,
|
||||
make_stages,
|
||||
)
|
||||
from sglang.srt.layers.layer_boundary.residual import batch as residual_batch
|
||||
from sglang.srt.layers.layernorm import RMSNorm
|
||||
@@ -1085,9 +1085,6 @@ class BailingMoELinearDecoderLayer(nn.Module):
|
||||
and (self.layer_id >= config.first_k_dense_replace)
|
||||
)
|
||||
self.is_layer_sparse = is_moe_layer
|
||||
is_previous_moe_layer = not (self.expert_num == 1) and (
|
||||
self.layer_id - 1 >= config.first_k_dense_replace
|
||||
)
|
||||
is_next_layer_sparse = not (self.expert_num == 1) and (
|
||||
self.layer_id + 1 >= config.first_k_dense_replace
|
||||
)
|
||||
@@ -1132,7 +1129,7 @@ class BailingMoELinearDecoderLayer(nn.Module):
|
||||
self.input_layernorm = RMSNorm(self.hidden_size, eps=rms_norm_eps)
|
||||
self.post_attention_layernorm = RMSNorm(self.hidden_size, eps=rms_norm_eps)
|
||||
|
||||
self.attn_boundary, self.ffn_boundary = make_stages(
|
||||
self.attn_boundary, self.ffn_boundary = append_stages(
|
||||
(
|
||||
declare_attn(),
|
||||
self.input_layernorm,
|
||||
@@ -1149,12 +1146,6 @@ class BailingMoELinearDecoderLayer(nn.Module):
|
||||
),
|
||||
self.post_attention_layernorm,
|
||||
),
|
||||
previous=declare_ffn(
|
||||
sparse=is_previous_moe_layer, next_layer_sparse=is_moe_layer
|
||||
)
|
||||
if layer_id != 0
|
||||
else None,
|
||||
terminal=layer_id == (1 if is_nextn else config.num_hidden_layers) - 1,
|
||||
)
|
||||
|
||||
@torch.inference_mode()
|
||||
|
||||
@@ -28,6 +28,7 @@ from sglang.kernels.ops.layernorm.fused_eh_norm import fused_eh_norm
|
||||
from sglang.srt.environ import envs
|
||||
from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder
|
||||
from sglang.srt.layers.attention.index_topk_share import IndexTopKShareState
|
||||
from sglang.srt.layers.layer_boundary import layer_stack
|
||||
from sglang.srt.layers.layer_boundary.residual import batch as residual_batch
|
||||
from sglang.srt.layers.layernorm import RMSNorm
|
||||
from sglang.srt.layers.linear import ReplicatedLinear
|
||||
@@ -121,16 +122,17 @@ class DeepseekModelNextN(nn.Module):
|
||||
layer_name = "layers." + str(config.num_hidden_layers)
|
||||
|
||||
self.quant_config = quant_config
|
||||
self.decoder = DeepseekV2DecoderLayer(
|
||||
config,
|
||||
0,
|
||||
quant_config=quant_config,
|
||||
moe_quant_config_override=moe_quant_config_override,
|
||||
is_nextn=True,
|
||||
prefix=add_prefix(layer_name, prefix),
|
||||
alt_stream=self.alt_stream,
|
||||
skip_rope=config.qk_rope_head_dim == 0,
|
||||
)
|
||||
with layer_stack():
|
||||
self.decoder = DeepseekV2DecoderLayer(
|
||||
config,
|
||||
0,
|
||||
quant_config=quant_config,
|
||||
moe_quant_config_override=moe_quant_config_override,
|
||||
is_nextn=True,
|
||||
prefix=add_prefix(layer_name, prefix),
|
||||
alt_stream=self.alt_stream,
|
||||
skip_rope=config.qk_rope_head_dim == 0,
|
||||
)
|
||||
|
||||
self.shared_head = nn.Module()
|
||||
self.shared_head.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
|
||||
|
||||
@@ -76,11 +76,11 @@ from sglang.srt.layers.dcp.planner import (
|
||||
)
|
||||
from sglang.srt.layers.flashinfer_comm_fusion import uses_cutedsl_ar_fusion
|
||||
from sglang.srt.layers.layer_boundary import (
|
||||
append_stages,
|
||||
declare_attn,
|
||||
declare_ffn,
|
||||
get_attn_tp_context,
|
||||
is_dense_ffn_fully_dp,
|
||||
make_stages,
|
||||
)
|
||||
from sglang.srt.layers.layer_boundary.residual import access as residual_access
|
||||
from sglang.srt.layers.layer_boundary.residual import batch as residual_batch
|
||||
@@ -2665,14 +2665,8 @@ class DeepseekV2DecoderLayer(nn.Module):
|
||||
)
|
||||
|
||||
self.is_layer_sparse = self._is_layer_sparse(layer_id, is_nextn=is_nextn)
|
||||
is_previous_layer_sparse = self._is_layer_sparse(layer_id - 1, is_nextn=False)
|
||||
is_next_layer_sparse = self._is_layer_sparse(layer_id + 1, is_nextn=False)
|
||||
|
||||
self._stage_enters_stack = (layer_id) == 0
|
||||
self._stage_terminal = (layer_id) == (
|
||||
1 if is_nextn else config.num_hidden_layers
|
||||
) - 1
|
||||
self._stage_previous_sparse = is_previous_layer_sparse
|
||||
self._stage_next_sparse = is_next_layer_sparse
|
||||
|
||||
if self.is_layer_sparse:
|
||||
@@ -2740,7 +2734,7 @@ class DeepseekV2DecoderLayer(nn.Module):
|
||||
from sglang.srt.layers.layer_boundary.fusions.cutedsl import CuteDSLFusion
|
||||
|
||||
fusions = CuteDSLFusion()
|
||||
attn_boundary, ffn_boundary = make_stages(
|
||||
attn_boundary, ffn_boundary = append_stages(
|
||||
(
|
||||
declare_attn(output_transform=attn_output),
|
||||
input_layernorm,
|
||||
@@ -2755,13 +2749,6 @@ class DeepseekV2DecoderLayer(nn.Module):
|
||||
post_attention_layernorm,
|
||||
{"fusions": fusions},
|
||||
),
|
||||
previous=declare_ffn(
|
||||
sparse=self._stage_previous_sparse,
|
||||
next_layer_sparse=self.is_layer_sparse,
|
||||
)
|
||||
if not self._stage_enters_stack
|
||||
else None,
|
||||
terminal=self._stage_terminal,
|
||||
)
|
||||
return attn_boundary, ffn_boundary
|
||||
|
||||
|
||||
@@ -56,10 +56,10 @@ from sglang.srt.layers.activation import SiluAndMul
|
||||
from sglang.srt.layers.attention.dsa.dsa_indexer import Indexer
|
||||
from sglang.srt.layers.dp_attention import is_dp_attention_enabled
|
||||
from sglang.srt.layers.layer_boundary import (
|
||||
append_stages,
|
||||
declare_attn,
|
||||
declare_ffn,
|
||||
is_dense_ffn_fully_dp,
|
||||
make_stages,
|
||||
)
|
||||
from sglang.srt.layers.layer_boundary.residual import batch as residual_batch
|
||||
from sglang.srt.layers.layernorm import RMSNorm
|
||||
@@ -1384,7 +1384,6 @@ class Dots3DecoderLayer(nn.Module):
|
||||
alt_stream=alt_stream,
|
||||
)
|
||||
self.is_layer_sparse = self._is_layer_sparse(layer_id, is_nextn=is_nextn)
|
||||
is_previous_layer_sparse = self._is_layer_sparse(layer_id - 1, is_nextn=False)
|
||||
is_next_layer_sparse = self._is_layer_sparse(layer_id + 1, is_nextn=False)
|
||||
|
||||
if self.is_layer_sparse:
|
||||
@@ -1417,7 +1416,7 @@ class Dots3DecoderLayer(nn.Module):
|
||||
config.hidden_size, eps=config.rms_norm_eps
|
||||
)
|
||||
|
||||
self.attn_boundary, self.ffn_boundary = make_stages(
|
||||
self.attn_boundary, self.ffn_boundary = append_stages(
|
||||
(declare_attn(), self.input_layernorm),
|
||||
(
|
||||
declare_ffn(
|
||||
@@ -1426,12 +1425,6 @@ class Dots3DecoderLayer(nn.Module):
|
||||
),
|
||||
self.post_attention_layernorm,
|
||||
),
|
||||
previous=declare_ffn(
|
||||
sparse=is_previous_layer_sparse, next_layer_sparse=self.is_layer_sparse
|
||||
)
|
||||
if layer_id != 0
|
||||
else None,
|
||||
terminal=layer_id == (1 if is_nextn else config.num_hidden_layers) - 1,
|
||||
)
|
||||
|
||||
def _is_layer_sparse(self, layer_id: int, is_nextn: bool) -> bool:
|
||||
|
||||
@@ -9,6 +9,7 @@ from transformers import PretrainedConfig
|
||||
|
||||
from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder
|
||||
from sglang.srt.layers.dp_attention import is_dp_attention_enabled
|
||||
from sglang.srt.layers.layer_boundary import layer_stack
|
||||
from sglang.srt.layers.layer_boundary.residual import batch as residual_batch
|
||||
from sglang.srt.layers.layernorm import RMSNorm
|
||||
from sglang.srt.layers.linear import ReplicatedLinear
|
||||
@@ -48,13 +49,14 @@ class Dots3MTPHead(nn.Module):
|
||||
quant_config=quant_config,
|
||||
prefix=add_prefix("eh_proj", prefix),
|
||||
)
|
||||
self.decoder = Dots3DecoderLayer(
|
||||
config,
|
||||
layer_id=0,
|
||||
quant_config=quant_config,
|
||||
is_nextn=True,
|
||||
prefix=add_prefix("decoder", prefix),
|
||||
)
|
||||
with layer_stack():
|
||||
self.decoder = Dots3DecoderLayer(
|
||||
config,
|
||||
layer_id=0,
|
||||
quant_config=quant_config,
|
||||
is_nextn=True,
|
||||
prefix=add_prefix("decoder", prefix),
|
||||
)
|
||||
self.shared_head = nn.Module()
|
||||
self.shared_head.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
|
||||
|
||||
|
||||
@@ -23,9 +23,9 @@ from torch import nn
|
||||
from transformers import PretrainedConfig
|
||||
|
||||
from sglang.srt.layers.layer_boundary import (
|
||||
append_stages,
|
||||
declare_attn,
|
||||
declare_ffn,
|
||||
make_stages,
|
||||
)
|
||||
from sglang.srt.layers.layer_boundary.residual import batch as residual_batch
|
||||
from sglang.srt.layers.layernorm import RMSNorm
|
||||
@@ -438,19 +438,12 @@ class Ernie4_5_VLMoeDecoderLayer(nn.Module):
|
||||
next_sparse = layer_id + 1 < config.num_hidden_layers and _is_moe_layer(
|
||||
config, layer_id + 1
|
||||
)
|
||||
self.attn_boundary, self.ffn_boundary = make_stages(
|
||||
self.attn_boundary, self.ffn_boundary = append_stages(
|
||||
(declare_attn(), self.input_layernorm),
|
||||
(
|
||||
declare_ffn(sparse=sparse, next_layer_sparse=next_sparse),
|
||||
self.post_attention_layernorm,
|
||||
),
|
||||
previous=declare_ffn(
|
||||
sparse=_is_moe_layer(config, layer_id - 1),
|
||||
next_layer_sparse=sparse,
|
||||
)
|
||||
if layer_id != 0
|
||||
else None,
|
||||
terminal=layer_id == config.num_hidden_layers - 1,
|
||||
)
|
||||
|
||||
def forward(
|
||||
|
||||
@@ -7,9 +7,9 @@ from transformers import Exaone4Config
|
||||
|
||||
from sglang.srt.layers.activation import SiluAndMul
|
||||
from sglang.srt.layers.layer_boundary import (
|
||||
append_stages,
|
||||
declare_attn,
|
||||
declare_ffn,
|
||||
make_stages,
|
||||
)
|
||||
from sglang.srt.layers.layer_boundary.residual import batch as residual_batch
|
||||
from sglang.srt.layers.layer_boundary.residual.post_norm import (
|
||||
@@ -275,7 +275,7 @@ class Exaone4DecoderLayer(nn.Module):
|
||||
# Post-LN: each stage reads the residual as it is, and its output is
|
||||
# normalized before it is added. The layer writes the FFN's itself.
|
||||
ffn_update = PostNormAdd(self.post_feedforward_layernorm, applied_at_exit=True)
|
||||
self.attn_boundary, self.ffn_boundary = make_stages(
|
||||
self.attn_boundary, self.ffn_boundary = append_stages(
|
||||
(
|
||||
declare_attn(
|
||||
read=PLAIN_READOUT,
|
||||
@@ -292,8 +292,6 @@ class Exaone4DecoderLayer(nn.Module):
|
||||
),
|
||||
None,
|
||||
),
|
||||
previous=declare_ffn(update=ffn_update) if layer_id != 0 else None,
|
||||
terminal=layer_id == config.num_hidden_layers - 1,
|
||||
)
|
||||
|
||||
def forward(
|
||||
|
||||
@@ -31,9 +31,9 @@ from sglang.srt.layers.activation import SiluAndMul
|
||||
from sglang.srt.layers.aux_hidden_states import AuxHiddenStateList
|
||||
from sglang.srt.layers.dp_attention import is_dp_attention_enabled
|
||||
from sglang.srt.layers.layer_boundary import (
|
||||
append_stages,
|
||||
declare_attn,
|
||||
declare_ffn,
|
||||
make_stages,
|
||||
)
|
||||
from sglang.srt.layers.layer_boundary.residual import batch as residual_batch
|
||||
from sglang.srt.layers.layernorm import RMSNorm
|
||||
@@ -491,7 +491,7 @@ class ExaoneMoEDecoderLayer(nn.Module):
|
||||
)
|
||||
is_moe_layer = config.is_moe_layer
|
||||
num_layers = config.num_hidden_layers
|
||||
self.attn_boundary, self.ffn_boundary = make_stages(
|
||||
self.attn_boundary, self.ffn_boundary = append_stages(
|
||||
(declare_attn(), self.input_layernorm),
|
||||
(
|
||||
declare_ffn(
|
||||
@@ -501,13 +501,6 @@ class ExaoneMoEDecoderLayer(nn.Module):
|
||||
),
|
||||
self.post_attention_layernorm,
|
||||
),
|
||||
previous=declare_ffn(
|
||||
sparse=is_moe_layer[layer_id - 1],
|
||||
next_layer_sparse=is_moe_layer[layer_id],
|
||||
)
|
||||
if layer_id != 0
|
||||
else None,
|
||||
terminal=layer_id == num_layers - 1,
|
||||
)
|
||||
|
||||
def forward(
|
||||
|
||||
@@ -15,9 +15,9 @@ from sglang.srt.layers.dp_attention import (
|
||||
is_dp_attention_enabled,
|
||||
)
|
||||
from sglang.srt.layers.layer_boundary import (
|
||||
append_stages,
|
||||
declare_attn,
|
||||
declare_ffn,
|
||||
make_stages,
|
||||
)
|
||||
from sglang.srt.layers.layer_boundary.output import OutputTransform
|
||||
from sglang.srt.layers.layer_boundary.residual import batch as residual_batch
|
||||
@@ -203,7 +203,6 @@ class FalconH1HybridAttentionDecoderLayer(nn.Module):
|
||||
|
||||
# FalconH1 all layers are dense and have no nextn now
|
||||
self.is_layer_sparse = False
|
||||
is_previous_layer_sparse = False
|
||||
is_next_layer_sparse = False
|
||||
|
||||
self.feed_forward = FalconH1MLP(
|
||||
@@ -223,7 +222,7 @@ class FalconH1HybridAttentionDecoderLayer(nn.Module):
|
||||
self.q_norm = RMSNorm(self.head_dim, eps=config.rms_norm_eps)
|
||||
self.k_norm = RMSNorm(self.head_dim, eps=config.rms_norm_eps)
|
||||
|
||||
self.attn_boundary, self.ffn_boundary = make_stages(
|
||||
self.attn_boundary, self.ffn_boundary = append_stages(
|
||||
(declare_attn(), self.input_layernorm),
|
||||
(
|
||||
declare_ffn(
|
||||
@@ -235,12 +234,6 @@ class FalconH1HybridAttentionDecoderLayer(nn.Module):
|
||||
),
|
||||
self.pre_ff_layernorm,
|
||||
),
|
||||
previous=declare_ffn(
|
||||
sparse=is_previous_layer_sparse, next_layer_sparse=self.is_layer_sparse
|
||||
)
|
||||
if layer_id != 0
|
||||
else None,
|
||||
terminal=layer_id == config.num_hidden_layers - 1,
|
||||
)
|
||||
|
||||
self.alt_stream = alt_stream
|
||||
|
||||
@@ -36,9 +36,9 @@ from sglang.kernels.ops.moe.gemma4_routing import (
|
||||
from sglang.srt.layers.aux_hidden_states import AuxHiddenStateList
|
||||
from sglang.srt.layers.layer_boundary import (
|
||||
SumGroup,
|
||||
append_stages,
|
||||
declare_attn,
|
||||
declare_ffn,
|
||||
make_stages,
|
||||
)
|
||||
from sglang.srt.layers.layer_boundary.output import OutputTransform
|
||||
from sglang.srt.layers.layer_boundary.residual import batch as residual_batch
|
||||
@@ -666,7 +666,7 @@ class Gemma4DecoderLayer(nn.Module):
|
||||
self.register_buffer("layer_scalar", torch.ones(1), persistent=True)
|
||||
self.has_ple = self.hidden_size_per_layer_input > 0
|
||||
self.prefix = prefix
|
||||
self.attn_boundary, self.ffn_boundary = make_stages(
|
||||
self.attn_boundary, self.ffn_boundary = append_stages(
|
||||
(
|
||||
declare_attn(
|
||||
output_transform=OutputTransform(self.post_attention_layernorm)
|
||||
@@ -681,14 +681,6 @@ class Gemma4DecoderLayer(nn.Module):
|
||||
),
|
||||
self.pre_feedforward_layernorm,
|
||||
),
|
||||
previous=declare_ffn(
|
||||
sparse=self.enable_moe_block,
|
||||
next_layer_sparse=self.enable_moe_block,
|
||||
update=REPLACE_AT_EXIT,
|
||||
)
|
||||
if layer_id != 0
|
||||
else None,
|
||||
terminal=layer_id == config.num_hidden_layers - 1,
|
||||
)
|
||||
|
||||
def forward(
|
||||
|
||||
@@ -18,6 +18,7 @@ from typing import Iterable, Optional
|
||||
import torch
|
||||
from torch import nn
|
||||
|
||||
from sglang.srt.layers.layer_boundary import layer_stack
|
||||
from sglang.srt.layers.layer_boundary.residual import batch as residual_batch
|
||||
from sglang.srt.layers.logits_processor import LogitsProcessor
|
||||
from sglang.srt.layers.quantization.base_config import QuantizationConfig
|
||||
@@ -64,14 +65,15 @@ class GigaChat35ModelNextN(nn.Module):
|
||||
|
||||
self.alt_stream = torch.cuda.Stream() if torch.cuda.is_available() else None
|
||||
|
||||
self.decoder = GigaChat35DecoderLayer(
|
||||
config=config,
|
||||
layer_id=0,
|
||||
quant_config=quant_config,
|
||||
prefix=add_prefix("decoder", prefix),
|
||||
alt_stream=self.alt_stream,
|
||||
is_nextn=True,
|
||||
)
|
||||
with layer_stack():
|
||||
self.decoder = GigaChat35DecoderLayer(
|
||||
config=config,
|
||||
layer_id=0,
|
||||
quant_config=quant_config,
|
||||
prefix=add_prefix("decoder", prefix),
|
||||
alt_stream=self.alt_stream,
|
||||
is_nextn=True,
|
||||
)
|
||||
|
||||
self.shared_head = nn.Module()
|
||||
self.shared_head.norm = build_norm(config, config.hidden_size)
|
||||
|
||||
@@ -27,9 +27,9 @@ from sglang.srt.layers.activation import SiluAndMul
|
||||
from sglang.srt.layers.aux_hidden_states import AuxHiddenStateList
|
||||
from sglang.srt.layers.dp_attention import is_dp_attention_enabled
|
||||
from sglang.srt.layers.layer_boundary import (
|
||||
append_stages,
|
||||
declare_attn,
|
||||
declare_ffn,
|
||||
make_stages,
|
||||
)
|
||||
from sglang.srt.layers.layer_boundary.output import OutputTransform
|
||||
from sglang.srt.layers.layer_boundary.residual import batch as residual_batch
|
||||
@@ -263,7 +263,7 @@ class Glm4DecoderLayer(nn.Module):
|
||||
config.hidden_size, eps=config.rms_norm_eps
|
||||
)
|
||||
self.post_mlp_layernorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
|
||||
self.attn_boundary, self.ffn_boundary = make_stages(
|
||||
self.attn_boundary, self.ffn_boundary = append_stages(
|
||||
(
|
||||
declare_attn(
|
||||
output_transform=OutputTransform(self.post_self_attn_layernorm)
|
||||
@@ -278,8 +278,6 @@ class Glm4DecoderLayer(nn.Module):
|
||||
),
|
||||
self.post_attention_layernorm,
|
||||
),
|
||||
previous=declare_ffn() if layer_id != 0 else None,
|
||||
terminal=layer_id == config.num_hidden_layers - 1,
|
||||
)
|
||||
|
||||
def forward(
|
||||
|
||||
@@ -44,10 +44,10 @@ from sglang.srt.layers.dp_attention import (
|
||||
is_dp_attention_enabled,
|
||||
)
|
||||
from sglang.srt.layers.layer_boundary import (
|
||||
append_stages,
|
||||
declare_attn,
|
||||
declare_ffn,
|
||||
is_dense_ffn_fully_dp,
|
||||
make_stages,
|
||||
)
|
||||
from sglang.srt.layers.layer_boundary.residual import batch as residual_batch
|
||||
from sglang.srt.layers.layernorm import RMSNorm
|
||||
@@ -741,7 +741,6 @@ class Glm4MoeDecoderLayer(nn.Module):
|
||||
)
|
||||
|
||||
self.is_layer_sparse = self._is_layer_sparse(layer_id, is_nextn=is_nextn)
|
||||
is_previous_layer_sparse = self._is_layer_sparse(layer_id - 1, is_nextn=False)
|
||||
is_next_layer_sparse = self._is_layer_sparse(layer_id + 1, is_nextn=False)
|
||||
|
||||
if self.is_layer_sparse:
|
||||
@@ -773,7 +772,7 @@ class Glm4MoeDecoderLayer(nn.Module):
|
||||
config.hidden_size, eps=config.rms_norm_eps
|
||||
)
|
||||
|
||||
self.attn_boundary, self.ffn_boundary = make_stages(
|
||||
self.attn_boundary, self.ffn_boundary = append_stages(
|
||||
(declare_attn(), self.input_layernorm),
|
||||
(
|
||||
declare_ffn(
|
||||
@@ -782,12 +781,6 @@ class Glm4MoeDecoderLayer(nn.Module):
|
||||
),
|
||||
self.post_attention_layernorm,
|
||||
),
|
||||
previous=declare_ffn(
|
||||
sparse=is_previous_layer_sparse, next_layer_sparse=self.is_layer_sparse
|
||||
)
|
||||
if layer_id != 0
|
||||
else None,
|
||||
terminal=layer_id == (1 if is_nextn else config.num_hidden_layers) - 1,
|
||||
)
|
||||
|
||||
# Detect if QKV uses aiter FP8 per-token quant so we can fuse
|
||||
|
||||
@@ -38,11 +38,11 @@ from sglang.srt.layers.dp_attention import (
|
||||
is_dp_attention_enabled,
|
||||
)
|
||||
from sglang.srt.layers.layer_boundary import (
|
||||
append_stages,
|
||||
declare_attn,
|
||||
declare_ffn,
|
||||
get_attn_tp_context,
|
||||
is_dense_ffn_fully_dp,
|
||||
make_stages,
|
||||
)
|
||||
from sglang.srt.layers.layer_boundary.residual import batch as residual_batch
|
||||
from sglang.srt.layers.layernorm import RMSNorm
|
||||
@@ -448,7 +448,6 @@ class Glm4MoeLiteDecoderLayer(nn.Module):
|
||||
)
|
||||
|
||||
self.is_layer_sparse = self._is_layer_sparse(layer_id, is_nextn=is_nextn)
|
||||
is_previous_layer_sparse = self._is_layer_sparse(layer_id - 1, is_nextn=False)
|
||||
is_next_layer_sparse = self._is_layer_sparse(layer_id + 1, is_nextn=False)
|
||||
|
||||
if self.is_layer_sparse:
|
||||
@@ -483,7 +482,7 @@ class Glm4MoeLiteDecoderLayer(nn.Module):
|
||||
|
||||
self._gfx95_quant_format = self._detect_gfx95_quant_format()
|
||||
|
||||
self.attn_boundary, self.ffn_boundary = make_stages(
|
||||
self.attn_boundary, self.ffn_boundary = append_stages(
|
||||
(
|
||||
declare_attn(),
|
||||
self.input_layernorm,
|
||||
@@ -496,12 +495,6 @@ class Glm4MoeLiteDecoderLayer(nn.Module):
|
||||
),
|
||||
self.post_attention_layernorm,
|
||||
),
|
||||
previous=declare_ffn(
|
||||
sparse=is_previous_layer_sparse, next_layer_sparse=self.is_layer_sparse
|
||||
)
|
||||
if layer_id != 0
|
||||
else None,
|
||||
terminal=layer_id == (1 if is_nextn else config.num_hidden_layers) - 1,
|
||||
)
|
||||
|
||||
def _detect_gfx95_quant_format(self) -> str:
|
||||
|
||||
@@ -23,6 +23,7 @@ from transformers import PretrainedConfig
|
||||
|
||||
from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder
|
||||
from sglang.srt.layers.dp_attention import is_dp_attention_enabled
|
||||
from sglang.srt.layers.layer_boundary import layer_stack
|
||||
from sglang.srt.layers.layer_boundary.residual import batch as residual_batch
|
||||
from sglang.srt.layers.layernorm import RMSNorm
|
||||
from sglang.srt.layers.logits_processor import LogitsProcessor
|
||||
@@ -71,13 +72,14 @@ class Glm4MoeLiteModelNextN(nn.Module):
|
||||
|
||||
self.eh_proj = nn.Linear(2 * config.hidden_size, config.hidden_size, bias=False)
|
||||
|
||||
self.decoder = Glm4MoeLiteDecoderLayer(
|
||||
config,
|
||||
0,
|
||||
quant_config=quant_config,
|
||||
is_nextn=True,
|
||||
prefix=add_prefix("decoder", prefix),
|
||||
)
|
||||
with layer_stack():
|
||||
self.decoder = Glm4MoeLiteDecoderLayer(
|
||||
config,
|
||||
0,
|
||||
quant_config=quant_config,
|
||||
is_nextn=True,
|
||||
prefix=add_prefix("decoder", prefix),
|
||||
)
|
||||
|
||||
self.shared_head = nn.Module()
|
||||
self.shared_head.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
|
||||
|
||||
@@ -23,6 +23,7 @@ from transformers import PretrainedConfig
|
||||
|
||||
from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder
|
||||
from sglang.srt.layers.dp_attention import is_dp_attention_enabled
|
||||
from sglang.srt.layers.layer_boundary import layer_stack
|
||||
from sglang.srt.layers.layer_boundary.residual import batch as residual_batch
|
||||
from sglang.srt.layers.layernorm import RMSNorm
|
||||
from sglang.srt.layers.logits_processor import LogitsProcessor
|
||||
@@ -67,13 +68,14 @@ class Glm4MoeModelNextN(nn.Module):
|
||||
|
||||
self.eh_proj = nn.Linear(2 * config.hidden_size, config.hidden_size, bias=False)
|
||||
|
||||
self.decoder = Glm4MoeDecoderLayer(
|
||||
config,
|
||||
0,
|
||||
quant_config=quant_config,
|
||||
is_nextn=True,
|
||||
prefix=add_prefix("decoder", prefix),
|
||||
)
|
||||
with layer_stack():
|
||||
self.decoder = Glm4MoeDecoderLayer(
|
||||
config,
|
||||
0,
|
||||
quant_config=quant_config,
|
||||
is_nextn=True,
|
||||
prefix=add_prefix("decoder", prefix),
|
||||
)
|
||||
|
||||
self.shared_head = nn.Module()
|
||||
self.shared_head.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
|
||||
|
||||
@@ -29,11 +29,11 @@ from sglang.srt.layers.conv import Conv2dLayer
|
||||
from sglang.srt.layers.layer_boundary import (
|
||||
PLAIN_RESIDUAL_OPS,
|
||||
MHCState,
|
||||
append_stages,
|
||||
declare_attn,
|
||||
declare_ffn,
|
||||
get_attn_tp_context,
|
||||
is_dense_ffn_fully_dp,
|
||||
make_stages,
|
||||
)
|
||||
from sglang.srt.layers.layer_boundary.residual import access as residual_access
|
||||
from sglang.srt.layers.layer_boundary.residual import batch as residual_batch
|
||||
@@ -723,7 +723,6 @@ class Glm5NextDecoderLayer(nn.Module):
|
||||
)
|
||||
|
||||
self.is_layer_sparse = self._is_layer_sparse(layer_id, is_nextn=is_nextn)
|
||||
is_previous_layer_sparse = self._is_layer_sparse(layer_id - 1, is_nextn=False)
|
||||
is_next_layer_sparse = self._is_layer_sparse(layer_id + 1, is_nextn=False)
|
||||
|
||||
if self.is_layer_sparse:
|
||||
@@ -793,7 +792,7 @@ class Glm5NextDecoderLayer(nn.Module):
|
||||
),
|
||||
is_last_layer=terminal,
|
||||
).residual_ops()
|
||||
self.attn_boundary, self.ffn_boundary = make_stages(
|
||||
self.attn_boundary, self.ffn_boundary = append_stages(
|
||||
(
|
||||
declare_attn(read=residual.attn_readout, update=residual.attn_update),
|
||||
self.input_layernorm,
|
||||
@@ -812,14 +811,6 @@ class Glm5NextDecoderLayer(nn.Module):
|
||||
),
|
||||
self.post_attention_layernorm,
|
||||
),
|
||||
previous=declare_ffn(
|
||||
sparse=is_previous_layer_sparse,
|
||||
next_layer_sparse=self.is_layer_sparse,
|
||||
update=residual.ffn_update,
|
||||
)
|
||||
if layer_id != 0
|
||||
else None,
|
||||
terminal=terminal,
|
||||
)
|
||||
|
||||
def _hc_pre(
|
||||
|
||||
@@ -35,9 +35,10 @@ from sglang.srt.layers.attention.vision import (
|
||||
)
|
||||
from sglang.srt.layers.dp_attention import is_dp_attention_enabled
|
||||
from sglang.srt.layers.layer_boundary import (
|
||||
append_stages,
|
||||
declare_attn,
|
||||
declare_ffn,
|
||||
make_stages,
|
||||
layer_stack,
|
||||
)
|
||||
from sglang.srt.layers.layer_boundary.output import OutputTransform
|
||||
from sglang.srt.layers.layer_boundary.residual import batch as residual_batch
|
||||
@@ -907,7 +908,7 @@ class GlmImageTextDecoderLayer(nn.Module):
|
||||
config.hidden_size, eps=config.rms_norm_eps
|
||||
)
|
||||
self.post_mlp_layernorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
|
||||
self.attn_boundary, self.ffn_boundary = make_stages(
|
||||
self.attn_boundary, self.ffn_boundary = append_stages(
|
||||
(
|
||||
declare_attn(
|
||||
read=UNFUSED_NORM_READOUT,
|
||||
@@ -924,8 +925,6 @@ class GlmImageTextDecoderLayer(nn.Module):
|
||||
),
|
||||
self.post_attention_layernorm,
|
||||
),
|
||||
previous=declare_ffn() if layer_id != 0 else None,
|
||||
terminal=layer_id == config.num_hidden_layers - 1,
|
||||
)
|
||||
|
||||
def forward(
|
||||
@@ -971,17 +970,18 @@ class GlmImageTextModel(nn.Module):
|
||||
prefix=add_prefix("embed_tokens", prefix),
|
||||
)
|
||||
|
||||
self.layers = nn.ModuleList(
|
||||
[
|
||||
GlmImageTextDecoderLayer(
|
||||
layer_id=i,
|
||||
config=config,
|
||||
quant_config=self.quant_config,
|
||||
prefix=add_prefix(f"layers.{i}", getattr(config, "prefix", "")),
|
||||
)
|
||||
for i in range(config.num_hidden_layers)
|
||||
]
|
||||
)
|
||||
with layer_stack():
|
||||
self.layers = nn.ModuleList(
|
||||
[
|
||||
GlmImageTextDecoderLayer(
|
||||
layer_id=i,
|
||||
config=config,
|
||||
quant_config=self.quant_config,
|
||||
prefix=add_prefix(f"layers.{i}", getattr(config, "prefix", "")),
|
||||
)
|
||||
for i in range(config.num_hidden_layers)
|
||||
]
|
||||
)
|
||||
self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
|
||||
|
||||
def forward(
|
||||
|
||||
@@ -23,6 +23,7 @@ from transformers import PretrainedConfig
|
||||
|
||||
from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder
|
||||
from sglang.srt.layers.dp_attention import is_dp_attention_enabled
|
||||
from sglang.srt.layers.layer_boundary import layer_stack
|
||||
from sglang.srt.layers.layer_boundary.residual import batch as residual_batch
|
||||
from sglang.srt.layers.layernorm import RMSNorm
|
||||
from sglang.srt.layers.logits_processor import LogitsProcessor
|
||||
@@ -70,12 +71,13 @@ class GlmOcrModelNextN(nn.Module):
|
||||
|
||||
self.eh_proj = nn.Linear(2 * config.hidden_size, config.hidden_size, bias=False)
|
||||
|
||||
self.decoder = Glm4DecoderLayer(
|
||||
config,
|
||||
0,
|
||||
quant_config=quant_config,
|
||||
prefix=add_prefix("decoder", prefix),
|
||||
)
|
||||
with layer_stack():
|
||||
self.decoder = Glm4DecoderLayer(
|
||||
config,
|
||||
0,
|
||||
quant_config=quant_config,
|
||||
prefix=add_prefix("decoder", prefix),
|
||||
)
|
||||
|
||||
self.shared_head = nn.Module()
|
||||
self.shared_head.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
|
||||
|
||||
@@ -34,9 +34,9 @@ from sglang.srt.layers.dp_attention import (
|
||||
is_dp_attention_enabled,
|
||||
)
|
||||
from sglang.srt.layers.layer_boundary import (
|
||||
append_stages,
|
||||
declare_attn,
|
||||
declare_ffn,
|
||||
make_stages,
|
||||
)
|
||||
from sglang.srt.layers.layer_boundary.residual import batch as residual_batch
|
||||
from sglang.srt.layers.layernorm import RMSNorm
|
||||
@@ -560,7 +560,6 @@ class GptOssDecoderLayer(nn.Module):
|
||||
|
||||
# GptOss all layers are sparse
|
||||
self.is_layer_sparse = True
|
||||
is_previous_layer_sparse = True
|
||||
is_next_layer_sparse = True
|
||||
|
||||
if self.is_layer_sparse:
|
||||
@@ -591,7 +590,7 @@ class GptOssDecoderLayer(nn.Module):
|
||||
x_pad_to_multiple=post_attn_pad_multiple,
|
||||
)
|
||||
|
||||
self.attn_boundary, self.ffn_boundary = make_stages(
|
||||
self.attn_boundary, self.ffn_boundary = append_stages(
|
||||
(declare_attn(), self.input_layernorm),
|
||||
(
|
||||
declare_ffn(
|
||||
@@ -600,12 +599,6 @@ class GptOssDecoderLayer(nn.Module):
|
||||
),
|
||||
self.post_attention_layernorm,
|
||||
),
|
||||
previous=declare_ffn(
|
||||
sparse=is_previous_layer_sparse, next_layer_sparse=self.is_layer_sparse
|
||||
)
|
||||
if layer_id != 0
|
||||
else None,
|
||||
terminal=layer_id == config.num_hidden_layers - 1,
|
||||
)
|
||||
|
||||
def forward(
|
||||
|
||||
@@ -11,9 +11,9 @@ from sglang.srt.layers.attention.hybrid_linear_attn_backend import (
|
||||
from sglang.srt.layers.attention.mamba.mamba import MambaMixer2
|
||||
from sglang.srt.layers.aux_hidden_states import AuxHiddenStateList
|
||||
from sglang.srt.layers.layer_boundary import (
|
||||
append_stages,
|
||||
declare_attn,
|
||||
declare_ffn,
|
||||
make_stages,
|
||||
)
|
||||
from sglang.srt.layers.layer_boundary.output import OutputTransform
|
||||
from sglang.srt.layers.layer_boundary.residual import batch as residual_batch
|
||||
@@ -40,13 +40,13 @@ from sglang.srt.utils import make_pp_layers
|
||||
from .granitemoe import GraniteMoeMoE, GraniteMoeSharedMLP
|
||||
|
||||
|
||||
def _make_stages(layer, config, layer_idx: int):
|
||||
def _make_stages(layer):
|
||||
"""Both kinds of layer scale their mixer and FFN outputs by
|
||||
residual_multiplier, add them in the activation dtype and normalize the sum
|
||||
as a separate step."""
|
||||
scale = OutputTransform(layer._scale_output)
|
||||
sparse = layer.block_sparse_moe is not None
|
||||
return make_stages(
|
||||
return append_stages(
|
||||
(
|
||||
declare_attn(read=UNFUSED_NORM_READOUT, output_transform=scale),
|
||||
layer.input_layernorm,
|
||||
@@ -60,12 +60,6 @@ def _make_stages(layer, config, layer_idx: int):
|
||||
),
|
||||
layer.post_attention_layernorm,
|
||||
),
|
||||
previous=(
|
||||
declare_ffn(sparse=sparse, next_layer_sparse=sparse)
|
||||
if layer_idx != 0
|
||||
else None
|
||||
),
|
||||
terminal=layer_idx == config.num_hidden_layers - 1,
|
||||
)
|
||||
|
||||
|
||||
@@ -124,7 +118,7 @@ class GraniteMoeHybridMambaDecoderLayer(nn.Module):
|
||||
self.post_attention_layernorm = RMSNorm(
|
||||
config.hidden_size, eps=config.rms_norm_eps
|
||||
)
|
||||
self.attn_boundary, self.ffn_boundary = _make_stages(self, config, layer_idx)
|
||||
self.attn_boundary, self.ffn_boundary = _make_stages(self)
|
||||
|
||||
def _scale_output(self, hidden_states: torch.Tensor) -> torch.Tensor:
|
||||
return hidden_states * self.residual_multiplier
|
||||
@@ -315,7 +309,7 @@ class GraniteMoeHybridAttentionDecoderLayer(nn.Module):
|
||||
self.post_attention_layernorm = RMSNorm(
|
||||
config.hidden_size, eps=config.rms_norm_eps
|
||||
)
|
||||
self.attn_boundary, self.ffn_boundary = _make_stages(self, config, layer_idx)
|
||||
self.attn_boundary, self.ffn_boundary = _make_stages(self)
|
||||
|
||||
def _scale_output(self, hidden_states: torch.Tensor) -> torch.Tensor:
|
||||
return hidden_states * self.residual_multiplier
|
||||
|
||||
@@ -9,10 +9,11 @@ from transformers import PretrainedConfig
|
||||
from sglang.srt.layers.attention.index_topk_share import IndexTopKShareState
|
||||
from sglang.srt.layers.layer_boundary import (
|
||||
IHCState,
|
||||
append_stages,
|
||||
declare_attn,
|
||||
declare_ffn,
|
||||
get_attn_tp_context,
|
||||
make_stages,
|
||||
layer_stack,
|
||||
)
|
||||
from sglang.srt.layers.layer_boundary.residual import batch as residual_batch
|
||||
from sglang.srt.layers.layernorm import RMSNorm
|
||||
@@ -583,8 +584,8 @@ def _build_stages(
|
||||
|
||||
layer_types = config.mlp_layer_types
|
||||
is_sparse = layer_types[layer_id] != "dense"
|
||||
next_id, previous_id = layer_id + 1, layer_id - 1
|
||||
return make_stages(
|
||||
next_id = layer_id + 1
|
||||
return append_stages(
|
||||
(
|
||||
declare_attn(read=residual.attn_readout, update=residual.attn_update),
|
||||
input_layernorm,
|
||||
@@ -601,16 +602,6 @@ def _build_stages(
|
||||
),
|
||||
post_attention_layernorm,
|
||||
),
|
||||
previous=(
|
||||
declare_ffn(
|
||||
sparse=layer_types[previous_id] != "dense",
|
||||
next_layer_sparse=is_sparse,
|
||||
update=residual.ffn_update,
|
||||
)
|
||||
if layer_id != 0
|
||||
else None
|
||||
),
|
||||
terminal=layer_id == config.num_hidden_layers - 1,
|
||||
)
|
||||
|
||||
|
||||
@@ -703,18 +694,19 @@ class HYV4Model(nn.Module):
|
||||
**get_embedding_tp_kwargs(),
|
||||
)
|
||||
self.alt_stream = get_stream("alt") if is_cuda() else None
|
||||
self.layers = nn.ModuleList(
|
||||
[
|
||||
HYV4DecoderLayer(
|
||||
config,
|
||||
i,
|
||||
quant_config,
|
||||
f"{prefix}.layers.{i}",
|
||||
self.alt_stream,
|
||||
)
|
||||
for i in range(config.num_hidden_layers)
|
||||
]
|
||||
)
|
||||
with layer_stack():
|
||||
self.layers = nn.ModuleList(
|
||||
[
|
||||
HYV4DecoderLayer(
|
||||
config,
|
||||
i,
|
||||
quant_config,
|
||||
f"{prefix}.layers.{i}",
|
||||
self.alt_stream,
|
||||
)
|
||||
for i in range(config.num_hidden_layers)
|
||||
]
|
||||
)
|
||||
self.hc_head = HYV4HCHeadLayer(config, f"{prefix}.hc_head")
|
||||
self.norm = RMSNorm(config.hidden_size, config.rms_norm_eps)
|
||||
# The head mixes the streams down and fuses the final norm into one kernel.
|
||||
|
||||
@@ -6,10 +6,11 @@ from torch import nn
|
||||
|
||||
from sglang.srt.layers.attention.index_topk_share import IndexTopKShareState
|
||||
from sglang.srt.layers.layer_boundary import (
|
||||
append_stages,
|
||||
declare_attn,
|
||||
declare_ffn,
|
||||
get_attn_tp_context,
|
||||
make_stages,
|
||||
layer_stack,
|
||||
)
|
||||
from sglang.srt.layers.layer_boundary.residual import batch as residual_batch
|
||||
from sglang.srt.layers.layernorm import RMSNorm
|
||||
@@ -71,14 +72,13 @@ class HYV4MTPDecoderLayer(nn.Module):
|
||||
)
|
||||
if hasattr(self.mlp, "shared_experts"):
|
||||
self.mlp.shared_experts.swiglu_limit = None
|
||||
self.attn_boundary, self.ffn_boundary = make_stages(
|
||||
self.attn_boundary, self.ffn_boundary = append_stages(
|
||||
(
|
||||
declare_attn(),
|
||||
self.input_layernorm,
|
||||
{"qkv_latent_func": self.self_attn.prepare_qkv_latent},
|
||||
),
|
||||
(declare_ffn(sparse=True), self.post_attention_layernorm),
|
||||
terminal=True,
|
||||
)
|
||||
|
||||
def forward(
|
||||
@@ -124,12 +124,13 @@ class HYV4ModelNextN(nn.Module):
|
||||
self.hnorm = RMSNorm(config.hidden_size, config.rms_norm_eps)
|
||||
self.eh_proj = nn.Linear(2 * config.hidden_size, config.hidden_size, bias=False)
|
||||
self.alt_stream = get_stream("alt") if is_cuda() else None
|
||||
self.decoder = HYV4MTPDecoderLayer(
|
||||
config,
|
||||
quant_config,
|
||||
f"{prefix}.decoder",
|
||||
self.alt_stream,
|
||||
)
|
||||
with layer_stack():
|
||||
self.decoder = HYV4MTPDecoderLayer(
|
||||
config,
|
||||
quant_config,
|
||||
f"{prefix}.decoder",
|
||||
self.alt_stream,
|
||||
)
|
||||
self.shared_head = nn.Module()
|
||||
self.shared_head.norm = RMSNorm(config.hidden_size, config.rms_norm_eps)
|
||||
|
||||
|
||||
@@ -16,9 +16,9 @@ from sglang.srt.layers.aux_hidden_states import (
|
||||
)
|
||||
from sglang.srt.layers.dp_attention import is_dp_attention_enabled
|
||||
from sglang.srt.layers.layer_boundary import (
|
||||
append_stages,
|
||||
declare_attn,
|
||||
declare_ffn,
|
||||
make_stages,
|
||||
)
|
||||
from sglang.srt.layers.layer_boundary.residual import batch as residual_batch
|
||||
from sglang.srt.layers.layer_boundary.residual.add_norm import (
|
||||
@@ -485,7 +485,7 @@ class InternS2MobiusLinearDecoderLayer(_InternS2MobiusDecoderMixin, nn.Module):
|
||||
config.hidden_size, eps=config.rms_norm_eps
|
||||
)
|
||||
accepts_fp8_input = _linear_accepts_fp8_tuple(self.linear_attn.in_proj_qkvz)
|
||||
self.attn_boundary, self.ffn_boundary = make_stages(
|
||||
self.attn_boundary, self.ffn_boundary = append_stages(
|
||||
(
|
||||
declare_attn(
|
||||
read=NormQuantReadout(
|
||||
@@ -501,10 +501,6 @@ class InternS2MobiusLinearDecoderLayer(_InternS2MobiusDecoderMixin, nn.Module):
|
||||
),
|
||||
self.post_attention_layernorm,
|
||||
),
|
||||
previous=declare_ffn(sparse=True, next_layer_sparse=True)
|
||||
if layer_id != 0
|
||||
else None,
|
||||
terminal=layer_id == config.num_hidden_layers - 1,
|
||||
)
|
||||
|
||||
def forward(
|
||||
@@ -617,7 +613,7 @@ class InternS2MobiusAttentionDecoderLayer(
|
||||
self.q_norm = GemmaRMSNorm(self.head_dim, eps=config.rms_norm_eps)
|
||||
self.k_norm = GemmaRMSNorm(self.head_dim, eps=config.rms_norm_eps)
|
||||
accepts_fp8_input = _linear_accepts_fp8_tuple(self.qkv_proj)
|
||||
self.attn_boundary, self.ffn_boundary = make_stages(
|
||||
self.attn_boundary, self.ffn_boundary = append_stages(
|
||||
(
|
||||
declare_attn(
|
||||
read=NormQuantReadout(
|
||||
@@ -633,10 +629,6 @@ class InternS2MobiusAttentionDecoderLayer(
|
||||
),
|
||||
self.post_attention_layernorm,
|
||||
),
|
||||
previous=declare_ffn(sparse=True, next_layer_sparse=True)
|
||||
if layer_id != 0
|
||||
else None,
|
||||
terminal=layer_id == config.num_hidden_layers - 1,
|
||||
)
|
||||
self.alt_stream = alt_stream
|
||||
|
||||
|
||||
@@ -11,9 +11,9 @@ from triton.language.extra import libdevice
|
||||
from sglang.srt.layers.activation import SiluAndMul
|
||||
from sglang.srt.layers.dp_attention import reject_attn_tp_shard_with_tp_reduce
|
||||
from sglang.srt.layers.layer_boundary import (
|
||||
append_stages,
|
||||
declare_attn,
|
||||
declare_ffn,
|
||||
make_stages,
|
||||
)
|
||||
from sglang.srt.layers.layer_boundary.output import OutputTransform
|
||||
from sglang.srt.layers.layer_boundary.residual import batch as residual_batch
|
||||
@@ -496,7 +496,7 @@ class IQuestQ1DecoderLayer(nn.Module):
|
||||
# Intentional: layer 0 adds the raw input and an FFN output norm, later layers
|
||||
# add the normalized input without one; the MTP draft follows layer 0.
|
||||
moe = _is_moe_layer(config, layer_id)
|
||||
self.attn_boundary, self.ffn_boundary = make_stages(
|
||||
self.attn_boundary, self.ffn_boundary = append_stages(
|
||||
(
|
||||
declare_attn(
|
||||
read=UNFUSED_NORM_READOUT
|
||||
@@ -515,12 +515,6 @@ class IQuestQ1DecoderLayer(nn.Module):
|
||||
),
|
||||
self.feed_forward_norm,
|
||||
),
|
||||
previous=declare_ffn(
|
||||
sparse=_is_moe_layer(config, layer_id - 1), next_layer_sparse=moe
|
||||
)
|
||||
if layer_id != 0
|
||||
else None,
|
||||
terminal=layer_id == config.num_hidden_layers - 1,
|
||||
)
|
||||
|
||||
def _attn_output(self, attn_output: torch.Tensor) -> torch.Tensor:
|
||||
|
||||
@@ -16,9 +16,9 @@ from sglang.srt.distributed import (
|
||||
from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder
|
||||
from sglang.srt.layers.dcp.planner import prepare_decode_context_parallel_metadata
|
||||
from sglang.srt.layers.layer_boundary import (
|
||||
append_stages,
|
||||
declare_attn,
|
||||
declare_ffn,
|
||||
make_stages,
|
||||
)
|
||||
from sglang.srt.layers.layer_boundary.residual import batch as residual_batch
|
||||
from sglang.srt.layers.layernorm import RMSNorm
|
||||
@@ -618,19 +618,12 @@ class KimiDecoderLayer(nn.Module):
|
||||
next_sparse = layer_idx + 1 < config.num_hidden_layers and _is_sparse_layer(
|
||||
config, layer_idx + 1
|
||||
)
|
||||
self.attn_boundary, self.ffn_boundary = make_stages(
|
||||
self.attn_boundary, self.ffn_boundary = append_stages(
|
||||
(declare_attn(), self.input_layernorm),
|
||||
(
|
||||
declare_ffn(sparse=sparse, next_layer_sparse=next_sparse),
|
||||
self.post_attention_layernorm,
|
||||
),
|
||||
previous=declare_ffn(
|
||||
sparse=_is_sparse_layer(config, layer_idx - 1),
|
||||
next_layer_sparse=sparse,
|
||||
)
|
||||
if layer_idx != 0
|
||||
else None,
|
||||
terminal=layer_idx == config.num_hidden_layers - 1,
|
||||
)
|
||||
|
||||
def forward(
|
||||
|
||||
@@ -24,9 +24,9 @@ from sglang.srt.layers.dp_attention import (
|
||||
is_dp_attention_enabled,
|
||||
)
|
||||
from sglang.srt.layers.layer_boundary import (
|
||||
append_stages,
|
||||
declare_attn,
|
||||
declare_ffn,
|
||||
make_stages,
|
||||
)
|
||||
from sglang.srt.layers.layer_boundary.residual import batch as residual_batch
|
||||
from sglang.srt.layers.layernorm import RMSNorm
|
||||
@@ -425,7 +425,6 @@ class LagunaDecoderLayer(nn.Module):
|
||||
|
||||
mlp_types = config.mlp_layer_types
|
||||
self.is_layer_sparse = mlp_types[layer_id] == "sparse"
|
||||
is_previous_layer_sparse = layer_id > 0 and mlp_types[layer_id - 1] == "sparse"
|
||||
is_next_layer_sparse = (
|
||||
layer_id + 1 < config.num_hidden_layers
|
||||
and mlp_types[layer_id + 1] == "sparse"
|
||||
@@ -453,7 +452,7 @@ class LagunaDecoderLayer(nn.Module):
|
||||
config.hidden_size, eps=config.rms_norm_eps
|
||||
)
|
||||
|
||||
self.attn_boundary, self.ffn_boundary = make_stages(
|
||||
self.attn_boundary, self.ffn_boundary = append_stages(
|
||||
(declare_attn(), self.input_layernorm),
|
||||
(
|
||||
declare_ffn(
|
||||
@@ -462,12 +461,6 @@ class LagunaDecoderLayer(nn.Module):
|
||||
),
|
||||
self.post_attention_layernorm,
|
||||
),
|
||||
previous=declare_ffn(
|
||||
sparse=is_previous_layer_sparse, next_layer_sparse=self.is_layer_sparse
|
||||
)
|
||||
if layer_id != 0
|
||||
else None,
|
||||
terminal=layer_id == config.num_hidden_layers - 1,
|
||||
)
|
||||
|
||||
def forward(
|
||||
|
||||
@@ -35,10 +35,10 @@ from sglang.srt.layers.dp_attention import (
|
||||
is_dp_attention_enabled,
|
||||
)
|
||||
from sglang.srt.layers.layer_boundary import (
|
||||
append_stages,
|
||||
declare_attn,
|
||||
declare_ffn,
|
||||
is_dense_ffn_fully_dp,
|
||||
make_stages,
|
||||
)
|
||||
from sglang.srt.layers.layer_boundary.residual import batch as residual_batch
|
||||
from sglang.srt.layers.layernorm import RMSNorm
|
||||
@@ -603,7 +603,6 @@ class LLaDA2MoeBlock(nn.Module):
|
||||
self.layer_id = layer_id
|
||||
|
||||
self.is_layer_sparse = self._is_layer_sparse(config, layer_id=layer_id)
|
||||
is_previous_layer_sparse = self._is_layer_sparse(config, layer_id=layer_id - 1)
|
||||
is_next_layer_sparse = self._is_layer_sparse(config, layer_id=layer_id + 1)
|
||||
|
||||
if self.is_layer_sparse:
|
||||
@@ -631,7 +630,7 @@ class LLaDA2MoeBlock(nn.Module):
|
||||
|
||||
self.post_attention_layernorm = RMSNorm(hidden_size, eps=config.rms_norm_eps)
|
||||
|
||||
self.attn_boundary, self.ffn_boundary = make_stages(
|
||||
self.attn_boundary, self.ffn_boundary = append_stages(
|
||||
(declare_attn(), self.input_layernorm),
|
||||
(
|
||||
declare_ffn(
|
||||
@@ -640,12 +639,6 @@ class LLaDA2MoeBlock(nn.Module):
|
||||
),
|
||||
self.post_attention_layernorm,
|
||||
),
|
||||
previous=declare_ffn(
|
||||
sparse=is_previous_layer_sparse, next_layer_sparse=self.is_layer_sparse
|
||||
)
|
||||
if layer_id != 0
|
||||
else None,
|
||||
terminal=layer_id == config.num_hidden_layers - 1,
|
||||
)
|
||||
|
||||
def _is_layer_sparse(self, config: PretrainedConfig, layer_id: int) -> bool:
|
||||
|
||||
@@ -31,9 +31,9 @@ from sglang.srt.distributed import (
|
||||
from sglang.srt.layers.activation import SiluAndMul
|
||||
from sglang.srt.layers.aux_hidden_states import AuxHiddenStateList
|
||||
from sglang.srt.layers.layer_boundary import (
|
||||
append_stages,
|
||||
declare_attn,
|
||||
declare_ffn,
|
||||
make_stages,
|
||||
)
|
||||
from sglang.srt.layers.layer_boundary.residual import batch as residual_batch
|
||||
from sglang.srt.layers.layer_boundary.residual.add_norm import NORM_QUANT_READOUT
|
||||
@@ -361,14 +361,12 @@ class LlamaDecoderLayer(nn.Module):
|
||||
return
|
||||
self.input_layernorm.fuse_input_quant(self.self_attn.qkv_proj)
|
||||
self.post_attention_layernorm.fuse_input_quant(self.mlp.gate_up_proj)
|
||||
self.attn_boundary, self.ffn_boundary = make_stages(
|
||||
self.attn_boundary, self.ffn_boundary = append_stages(
|
||||
(declare_attn(read=self._attn_readout(layer_id)), self.input_layernorm),
|
||||
(
|
||||
declare_ffn(sparse=False, next_layer_sparse=False),
|
||||
self.post_attention_layernorm,
|
||||
),
|
||||
previous=declare_ffn() if layer_id != 0 else None,
|
||||
terminal=layer_id == config.num_hidden_layers - 1,
|
||||
)
|
||||
|
||||
def _attn_readout(self, layer_id: int):
|
||||
|
||||
@@ -30,9 +30,9 @@ from sglang.srt.layers.dp_attention import (
|
||||
is_dp_attention_enabled,
|
||||
)
|
||||
from sglang.srt.layers.layer_boundary import (
|
||||
append_stages,
|
||||
declare_attn,
|
||||
declare_ffn,
|
||||
make_stages,
|
||||
)
|
||||
from sglang.srt.layers.layer_boundary.residual import batch as residual_batch
|
||||
from sglang.srt.layers.layernorm import RMSNorm
|
||||
@@ -398,7 +398,6 @@ class Llama4DecoderLayer(nn.Module):
|
||||
)
|
||||
self.config = config
|
||||
is_moe_layer = self._is_moe_layer(layer_id)
|
||||
is_previous_moe_layer = self._is_moe_layer(layer_id - 1)
|
||||
is_next_moe_layer = self._is_moe_layer(layer_id + 1)
|
||||
|
||||
if is_moe_layer:
|
||||
@@ -422,7 +421,7 @@ class Llama4DecoderLayer(nn.Module):
|
||||
config.hidden_size, eps=config.rms_norm_eps
|
||||
)
|
||||
|
||||
self.attn_boundary, self.ffn_boundary = make_stages(
|
||||
self.attn_boundary, self.ffn_boundary = append_stages(
|
||||
(declare_attn(), self.input_layernorm),
|
||||
(
|
||||
declare_ffn(
|
||||
@@ -431,12 +430,6 @@ class Llama4DecoderLayer(nn.Module):
|
||||
),
|
||||
self.post_attention_layernorm,
|
||||
),
|
||||
previous=declare_ffn(
|
||||
sparse=is_previous_moe_layer, next_layer_sparse=is_moe_layer
|
||||
)
|
||||
if layer_id != 0
|
||||
else None,
|
||||
terminal=layer_id == config.num_hidden_layers - 1,
|
||||
)
|
||||
|
||||
def _is_moe_layer(self, layer_id: int) -> bool:
|
||||
@@ -499,7 +492,10 @@ class Llama4Model(nn.Module):
|
||||
self.layers = make_layers(
|
||||
config.num_hidden_layers,
|
||||
lambda idx, prefix: Llama4DecoderLayer(
|
||||
config=config, layer_id=idx, quant_config=quant_config, prefix=prefix
|
||||
config=config,
|
||||
layer_id=idx,
|
||||
quant_config=quant_config,
|
||||
prefix=prefix,
|
||||
),
|
||||
prefix=add_prefix("layers", prefix),
|
||||
)
|
||||
|
||||
@@ -25,6 +25,7 @@ import torch
|
||||
from torch import nn
|
||||
from transformers import LlamaConfig
|
||||
|
||||
from sglang.srt.layers.layer_boundary import layer_stack
|
||||
from sglang.srt.layers.layer_boundary.residual import batch as residual_batch
|
||||
from sglang.srt.layers.layer_boundary.residual.post_norm import PLAIN_READOUT
|
||||
from sglang.srt.layers.logits_processor import LogitsProcessor
|
||||
@@ -74,17 +75,18 @@ class LlamaModel(nn.Module):
|
||||
config.hidden_size,
|
||||
prefix=add_prefix("embed_tokens", prefix),
|
||||
)
|
||||
self.layers = nn.ModuleList(
|
||||
[
|
||||
LlamaDecoderLayer(
|
||||
config,
|
||||
i,
|
||||
quant_config=quant_config,
|
||||
prefix=add_prefix(f"layers.{i}", prefix),
|
||||
)
|
||||
for i in range(config.num_hidden_layers)
|
||||
]
|
||||
)
|
||||
with layer_stack():
|
||||
self.layers = nn.ModuleList(
|
||||
[
|
||||
LlamaDecoderLayer(
|
||||
config,
|
||||
i,
|
||||
quant_config=quant_config,
|
||||
prefix=add_prefix(f"layers.{i}", prefix),
|
||||
)
|
||||
for i in range(config.num_hidden_layers)
|
||||
]
|
||||
)
|
||||
self.fc = torch.nn.Linear(config.hidden_size * 2, config.hidden_size)
|
||||
|
||||
def forward(
|
||||
|
||||
@@ -27,6 +27,7 @@ import torch
|
||||
from torch import nn
|
||||
from transformers import LlamaConfig
|
||||
|
||||
from sglang.srt.layers.layer_boundary import layer_stack
|
||||
from sglang.srt.layers.layernorm import RMSNorm
|
||||
from sglang.srt.layers.linear import QKVParallelLinear
|
||||
from sglang.srt.layers.logits_processor import LogitsProcessor
|
||||
@@ -193,12 +194,13 @@ class LlamaModel(nn.Module):
|
||||
else:
|
||||
self.fc_norm = None
|
||||
|
||||
self.layers = nn.ModuleList(
|
||||
[
|
||||
LlamaDecoderLayer(config, i, quant_config, prefix)
|
||||
for i in range(config.num_hidden_layers)
|
||||
]
|
||||
)
|
||||
with layer_stack():
|
||||
self.layers = nn.ModuleList(
|
||||
[
|
||||
LlamaDecoderLayer(config, i, quant_config, prefix)
|
||||
for i in range(config.num_hidden_layers)
|
||||
]
|
||||
)
|
||||
|
||||
self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
|
||||
self.norm_output = getattr(config, "norm_output", False)
|
||||
|
||||
@@ -52,9 +52,10 @@ from sglang.srt.layers.dp_attention import (
|
||||
is_dp_attention_enabled,
|
||||
)
|
||||
from sglang.srt.layers.layer_boundary import (
|
||||
append_stages,
|
||||
declare_attn,
|
||||
declare_ffn,
|
||||
make_stages,
|
||||
layer_stack,
|
||||
)
|
||||
from sglang.srt.layers.layer_boundary.residual import batch as residual_batch
|
||||
from sglang.srt.layers.layernorm import RMSNorm
|
||||
@@ -412,7 +413,7 @@ class LongcatFlashDecoderLayer(nn.Module):
|
||||
prefix=add_prefix("mlp", prefix),
|
||||
)
|
||||
|
||||
self.attn_boundary, self.moe_boundary = make_stages(
|
||||
self.attn_boundary, self.moe_boundary = append_stages(
|
||||
(
|
||||
declare_attn(),
|
||||
self.input_layernorm[0],
|
||||
@@ -422,13 +423,9 @@ class LongcatFlashDecoderLayer(nn.Module):
|
||||
declare_ffn(sparse=True, next_layer_sparse=True),
|
||||
self.post_attention_layernorm[0],
|
||||
),
|
||||
previous=declare_ffn(sparse=True, next_layer_sparse=True)
|
||||
if self.layer_id != 0
|
||||
else None,
|
||||
terminal=self.layer_id == config.num_hidden_layers - 1,
|
||||
)
|
||||
self.first_ffn_boundary, self.second_attn_boundary, self.second_ffn_boundary = (
|
||||
make_stages(
|
||||
append_stages(
|
||||
(
|
||||
declare_ffn(),
|
||||
self.post_attention_layernorm[0],
|
||||
@@ -562,18 +559,19 @@ class LongcatFlashModel(nn.Module):
|
||||
)
|
||||
|
||||
self.alt_stream = get_stream("alt")
|
||||
self.layers = nn.ModuleList(
|
||||
[
|
||||
LongcatFlashDecoderLayer(
|
||||
config,
|
||||
layer_id,
|
||||
quant_config=quant_config,
|
||||
prefix=add_prefix(f"layers.{layer_id}", prefix),
|
||||
alt_stream=self.alt_stream,
|
||||
)
|
||||
for layer_id in range(config.num_hidden_layers)
|
||||
]
|
||||
)
|
||||
with layer_stack():
|
||||
self.layers = nn.ModuleList(
|
||||
[
|
||||
LongcatFlashDecoderLayer(
|
||||
config,
|
||||
layer_id,
|
||||
quant_config=quant_config,
|
||||
prefix=add_prefix(f"layers.{layer_id}", prefix),
|
||||
alt_stream=self.alt_stream,
|
||||
)
|
||||
for layer_id in range(config.num_hidden_layers)
|
||||
]
|
||||
)
|
||||
self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
|
||||
self.layers_to_capture = []
|
||||
|
||||
|
||||
@@ -45,9 +45,10 @@ from sglang.srt.layers.dp_attention import (
|
||||
is_dp_attention_enabled,
|
||||
)
|
||||
from sglang.srt.layers.layer_boundary import (
|
||||
append_stages,
|
||||
declare_attn,
|
||||
declare_ffn,
|
||||
make_stages,
|
||||
layer_stack,
|
||||
)
|
||||
from sglang.srt.layers.layer_boundary.residual import batch as residual_batch
|
||||
from sglang.srt.layers.layernorm import RMSNorm
|
||||
@@ -162,14 +163,12 @@ class LongcatFlashDenseDecoderLayer(nn.Module):
|
||||
config.hidden_size, eps=config.rms_norm_eps
|
||||
)
|
||||
|
||||
self.attn_boundary, self.ffn_boundary = make_stages(
|
||||
self.attn_boundary, self.ffn_boundary = append_stages(
|
||||
(declare_attn(), self.input_layernorm),
|
||||
(
|
||||
declare_ffn(sparse=False, next_layer_sparse=False),
|
||||
self.post_attention_layernorm,
|
||||
),
|
||||
previous=declare_ffn() if self.layer_id != 0 else None,
|
||||
terminal=self.layer_id == config.num_hidden_layers - 1,
|
||||
)
|
||||
|
||||
def forward(
|
||||
@@ -224,9 +223,10 @@ class LongcatFlashModelNextN(nn.Module):
|
||||
quant_config=quant_config,
|
||||
prefix=add_prefix("eh_proj", ""),
|
||||
)
|
||||
self.decoder = LongcatFlashDenseDecoderLayer(
|
||||
config, 0, quant_config=quant_config, alt_stream=self.alt_stream
|
||||
)
|
||||
with layer_stack():
|
||||
self.decoder = LongcatFlashDenseDecoderLayer(
|
||||
config, 0, quant_config=quant_config, alt_stream=self.alt_stream
|
||||
)
|
||||
|
||||
self.final_layernorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
|
||||
|
||||
|
||||
@@ -26,9 +26,9 @@ from torch import nn
|
||||
from transformers import PretrainedConfig
|
||||
|
||||
from sglang.srt.layers.layer_boundary import (
|
||||
append_stages,
|
||||
declare_attn,
|
||||
declare_ffn,
|
||||
make_stages,
|
||||
)
|
||||
from sglang.srt.layers.layernorm import RMSNorm
|
||||
from sglang.srt.layers.linear import QKVParallelLinear, RowParallelLinear
|
||||
@@ -439,13 +439,12 @@ class MellumDecoderLayer(Qwen3MoeDecoderLayer):
|
||||
reduce_results=False,
|
||||
)
|
||||
|
||||
is_previous_layer_sparse = _is_sparse(layer_id - 1)
|
||||
is_next_layer_sparse = _is_sparse(layer_id + 1)
|
||||
|
||||
self.input_layernorm = RMSNorm(cfg.hidden_size, eps=rms_norm_eps)
|
||||
self.post_attention_layernorm = RMSNorm(cfg.hidden_size, eps=rms_norm_eps)
|
||||
|
||||
self.attn_boundary, self.ffn_boundary = make_stages(
|
||||
self.attn_boundary, self.ffn_boundary = append_stages(
|
||||
(declare_attn(), self.input_layernorm),
|
||||
(
|
||||
declare_ffn(
|
||||
@@ -454,12 +453,6 @@ class MellumDecoderLayer(Qwen3MoeDecoderLayer):
|
||||
),
|
||||
self.post_attention_layernorm,
|
||||
),
|
||||
previous=declare_ffn(
|
||||
sparse=is_previous_layer_sparse, next_layer_sparse=self.is_layer_sparse
|
||||
)
|
||||
if layer_id != 0
|
||||
else None,
|
||||
terminal=layer_id == cfg.num_hidden_layers - 1,
|
||||
)
|
||||
|
||||
|
||||
|
||||
@@ -8,6 +8,7 @@ import torch
|
||||
from torch import nn
|
||||
from transformers import PretrainedConfig
|
||||
|
||||
from sglang.srt.layers.layer_boundary import layer_stack
|
||||
from sglang.srt.layers.layer_boundary.residual import batch as residual_batch
|
||||
from sglang.srt.layers.layernorm import RMSNorm
|
||||
from sglang.srt.layers.logits_processor import LogitsProcessor
|
||||
@@ -40,9 +41,10 @@ class MiMoMultiTokenPredictorLayer(nn.Module):
|
||||
self.input_proj = nn.Linear(
|
||||
config.hidden_size * 2, config.hidden_size, bias=False
|
||||
)
|
||||
self.mtp_block = Qwen2DecoderLayer(
|
||||
config=config, quant_config=quant_config, prefix=prefix
|
||||
)
|
||||
with layer_stack():
|
||||
self.mtp_block = Qwen2DecoderLayer(
|
||||
config=config, quant_config=quant_config, prefix=prefix
|
||||
)
|
||||
self.final_layernorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
|
||||
|
||||
def forward(
|
||||
|
||||
@@ -36,10 +36,10 @@ from sglang.srt.layers.dp_attention import (
|
||||
is_dp_attention_enabled,
|
||||
)
|
||||
from sglang.srt.layers.layer_boundary import (
|
||||
append_stages,
|
||||
declare_attn,
|
||||
declare_ffn,
|
||||
is_dense_ffn_fully_dp,
|
||||
make_stages,
|
||||
)
|
||||
from sglang.srt.layers.layer_boundary.residual import batch as residual_batch
|
||||
from sglang.srt.layers.layernorm import RMSNorm
|
||||
@@ -826,7 +826,6 @@ class MiMoV2DecoderLayer(nn.Module):
|
||||
)
|
||||
|
||||
self.is_layer_sparse = self.is_moe_layer(layer_id)
|
||||
is_previous_layer_sparse = self.is_moe_layer(layer_id - 1)
|
||||
is_next_layer_sparse = self.is_moe_layer(layer_id + 1)
|
||||
|
||||
if self.is_layer_sparse:
|
||||
@@ -857,7 +856,7 @@ class MiMoV2DecoderLayer(nn.Module):
|
||||
config.hidden_size, eps=config.layernorm_epsilon
|
||||
)
|
||||
|
||||
self.attn_boundary, self.ffn_boundary = make_stages(
|
||||
self.attn_boundary, self.ffn_boundary = append_stages(
|
||||
(declare_attn(), self.input_layernorm),
|
||||
(
|
||||
declare_ffn(
|
||||
@@ -866,12 +865,6 @@ class MiMoV2DecoderLayer(nn.Module):
|
||||
),
|
||||
self.post_attention_layernorm,
|
||||
),
|
||||
previous=declare_ffn(
|
||||
sparse=is_previous_layer_sparse, next_layer_sparse=self.is_layer_sparse
|
||||
)
|
||||
if layer_id != 0
|
||||
else None,
|
||||
terminal=layer_id == config.num_hidden_layers - 1,
|
||||
)
|
||||
|
||||
def forward(
|
||||
|
||||
@@ -25,10 +25,11 @@ from sglang.srt.layers.dp_attention import (
|
||||
is_dp_attention_enabled,
|
||||
)
|
||||
from sglang.srt.layers.layer_boundary import (
|
||||
append_stages,
|
||||
declare_attn,
|
||||
declare_ffn,
|
||||
is_dense_ffn_fully_dp,
|
||||
make_stages,
|
||||
layer_stack,
|
||||
)
|
||||
from sglang.srt.layers.layer_boundary.residual import batch as residual_batch
|
||||
from sglang.srt.layers.layernorm import RMSNorm
|
||||
@@ -98,7 +99,6 @@ class MiMoV2MTPLayer(nn.Module):
|
||||
prefix=add_prefix("self_attn", prefix),
|
||||
)
|
||||
self.is_layer_sparse = False
|
||||
is_previous_layer_sparse = True
|
||||
is_next_layer_sparse = False
|
||||
|
||||
if is_dense_ffn_fully_dp():
|
||||
@@ -120,7 +120,7 @@ class MiMoV2MTPLayer(nn.Module):
|
||||
config.hidden_size, eps=config.layernorm_epsilon
|
||||
)
|
||||
|
||||
self.attn_boundary, self.ffn_boundary = make_stages(
|
||||
self.attn_boundary, self.ffn_boundary = append_stages(
|
||||
(declare_attn(), self.input_layernorm),
|
||||
(
|
||||
declare_ffn(
|
||||
@@ -129,12 +129,6 @@ class MiMoV2MTPLayer(nn.Module):
|
||||
),
|
||||
self.post_attention_layernorm,
|
||||
),
|
||||
previous=declare_ffn(
|
||||
sparse=is_previous_layer_sparse, next_layer_sparse=self.is_layer_sparse
|
||||
)
|
||||
if layer_id != 0
|
||||
else None,
|
||||
terminal=layer_id == 1 - 1,
|
||||
)
|
||||
|
||||
def forward(
|
||||
@@ -185,12 +179,13 @@ class MiMoV2ModelNextN(nn.Module):
|
||||
|
||||
self.eh_proj = nn.Linear(2 * config.hidden_size, config.hidden_size, bias=False)
|
||||
|
||||
self.mtp_block = MiMoV2MTPLayer(
|
||||
config,
|
||||
0,
|
||||
quant_config=quant_config,
|
||||
prefix=add_prefix("decoder", prefix),
|
||||
)
|
||||
with layer_stack():
|
||||
self.mtp_block = MiMoV2MTPLayer(
|
||||
config,
|
||||
0,
|
||||
quant_config=quant_config,
|
||||
prefix=add_prefix("decoder", prefix),
|
||||
)
|
||||
self.final_layernorm = RMSNorm(config.hidden_size, eps=config.layernorm_epsilon)
|
||||
|
||||
def forward(
|
||||
|
||||
@@ -39,9 +39,9 @@ from sglang.srt.layers.dp_attention import (
|
||||
is_dp_attention_enabled,
|
||||
)
|
||||
from sglang.srt.layers.layer_boundary import (
|
||||
append_stages,
|
||||
declare_attn,
|
||||
declare_ffn,
|
||||
make_stages,
|
||||
)
|
||||
from sglang.srt.layers.layer_boundary.residual import batch as residual_batch
|
||||
from sglang.srt.layers.layernorm import RMSNorm
|
||||
@@ -99,6 +99,7 @@ from sglang.srt.utils import (
|
||||
is_xpu,
|
||||
make_pp_layers,
|
||||
)
|
||||
from sglang.srt.utils.common import is_building_neighbour_layer
|
||||
from sglang.srt.utils.custom_op import register_custom_op
|
||||
from sglang.srt.utils.hf_transformers_utils import get_rope_config
|
||||
|
||||
@@ -408,7 +409,12 @@ class MiniMaxM2QKRMSNorm:
|
||||
use_fused_norm = get_bool_env_var("SGLANG_USE_FUSED_PARALLEL_QKNORM")
|
||||
|
||||
self._forward_impl = self._forward_naive
|
||||
if self._world_size > 1 and _is_cuda and use_fused_norm:
|
||||
if (
|
||||
self._world_size > 1
|
||||
and _is_cuda
|
||||
and use_fused_norm
|
||||
and not is_building_neighbour_layer()
|
||||
):
|
||||
occupancy = get_fused_parallel_qknorm_max_occupancy(
|
||||
q_norm.weight.dtype,
|
||||
self._world_size,
|
||||
@@ -867,10 +873,9 @@ class MiniMaxM2DecoderLayer(nn.Module):
|
||||
config.hidden_size, eps=getattr(config, "rms_norm_eps", 1e-6)
|
||||
)
|
||||
|
||||
is_previous_layer_sparse = True
|
||||
is_next_layer_sparse = True
|
||||
|
||||
self.attn_boundary, self.ffn_boundary = make_stages(
|
||||
self.attn_boundary, self.ffn_boundary = append_stages(
|
||||
(declare_attn(), self.input_layernorm),
|
||||
(
|
||||
declare_ffn(
|
||||
@@ -879,12 +884,6 @@ class MiniMaxM2DecoderLayer(nn.Module):
|
||||
),
|
||||
self.post_attention_layernorm,
|
||||
),
|
||||
previous=declare_ffn(
|
||||
sparse=is_previous_layer_sparse, next_layer_sparse=self.is_layer_sparse
|
||||
)
|
||||
if layer_id != 0
|
||||
else None,
|
||||
terminal=layer_id == config.num_hidden_layers - 1,
|
||||
)
|
||||
|
||||
def forward(
|
||||
|
||||
@@ -40,10 +40,10 @@ from sglang.srt.layers.dp_attention import (
|
||||
is_dp_attention_enabled,
|
||||
)
|
||||
from sglang.srt.layers.layer_boundary import (
|
||||
append_stages,
|
||||
declare_attn,
|
||||
declare_ffn,
|
||||
is_dense_ffn_fully_dp,
|
||||
make_stages,
|
||||
)
|
||||
from sglang.srt.layers.layer_boundary.residual import batch as residual_batch
|
||||
from sglang.srt.layers.layernorm import GemmaRMSNorm, RMSNorm
|
||||
@@ -1517,10 +1517,9 @@ class MiniMaxM3DecoderLayer(nn.Module):
|
||||
return True
|
||||
return moe_layer_freq[lid] != 0
|
||||
|
||||
is_previous_layer_sparse = _is_layer_sparse(layer_id - 1)
|
||||
is_next_layer_sparse = _is_layer_sparse(layer_id + 1)
|
||||
|
||||
self.attn_boundary, self.ffn_boundary = make_stages(
|
||||
self.attn_boundary, self.ffn_boundary = append_stages(
|
||||
(declare_attn(), self.input_layernorm),
|
||||
(
|
||||
declare_ffn(
|
||||
@@ -1529,12 +1528,6 @@ class MiniMaxM3DecoderLayer(nn.Module):
|
||||
),
|
||||
self.post_attention_layernorm,
|
||||
),
|
||||
previous=declare_ffn(
|
||||
sparse=is_previous_layer_sparse, next_layer_sparse=self.is_layer_sparse
|
||||
)
|
||||
if layer_id != 0
|
||||
else None,
|
||||
terminal=layer_id == config.num_hidden_layers - 1,
|
||||
)
|
||||
|
||||
def forward(
|
||||
|
||||
@@ -40,6 +40,7 @@ import torch
|
||||
from torch import nn
|
||||
from transformers import PretrainedConfig
|
||||
|
||||
from sglang.srt.layers.layer_boundary import layer_stack
|
||||
from sglang.srt.layers.layer_boundary.residual import batch as residual_batch
|
||||
from sglang.srt.layers.layernorm import RMSNorm
|
||||
from sglang.srt.layers.linear import RowParallelLinear
|
||||
@@ -75,17 +76,18 @@ class MistralEagleModel(nn.Module):
|
||||
config.hidden_size,
|
||||
prefix=add_prefix("embed_tokens", prefix),
|
||||
)
|
||||
self.layers = nn.ModuleList(
|
||||
[
|
||||
LlamaDecoderLayer(
|
||||
config=config,
|
||||
layer_id=i,
|
||||
prefix=add_prefix(f"layers.{i}", prefix),
|
||||
quant_config=quant_config,
|
||||
)
|
||||
for i in range(config.num_hidden_layers)
|
||||
]
|
||||
)
|
||||
with layer_stack():
|
||||
self.layers = nn.ModuleList(
|
||||
[
|
||||
LlamaDecoderLayer(
|
||||
config=config,
|
||||
layer_id=i,
|
||||
prefix=add_prefix(f"layers.{i}", prefix),
|
||||
quant_config=quant_config,
|
||||
)
|
||||
for i in range(config.num_hidden_layers)
|
||||
]
|
||||
)
|
||||
self.start_layer = 0
|
||||
self.end_layer = config.num_hidden_layers
|
||||
self.fc = RowParallelLinear(
|
||||
|
||||
@@ -26,9 +26,9 @@ from torch import nn
|
||||
from transformers import MixtralConfig
|
||||
|
||||
from sglang.srt.layers.layer_boundary import (
|
||||
append_stages,
|
||||
declare_attn,
|
||||
declare_ffn,
|
||||
make_stages,
|
||||
)
|
||||
from sglang.srt.layers.layer_boundary.residual import batch as residual_batch
|
||||
from sglang.srt.layers.layernorm import RMSNorm
|
||||
@@ -235,16 +235,12 @@ class MixtralDecoderLayer(nn.Module):
|
||||
self.post_attention_layernorm = RMSNorm(
|
||||
config.hidden_size, eps=config.rms_norm_eps
|
||||
)
|
||||
self.attn_boundary, self.ffn_boundary = make_stages(
|
||||
self.attn_boundary, self.ffn_boundary = append_stages(
|
||||
(declare_attn(), self.input_layernorm),
|
||||
(
|
||||
declare_ffn(sparse=True, next_layer_sparse=True),
|
||||
self.post_attention_layernorm,
|
||||
),
|
||||
previous=declare_ffn(sparse=True, next_layer_sparse=True)
|
||||
if layer_id != 0
|
||||
else None,
|
||||
terminal=layer_id == config.num_hidden_layers - 1,
|
||||
)
|
||||
|
||||
def forward(
|
||||
|
||||
@@ -22,6 +22,7 @@ from transformers.models.mllama.modeling_mllama import (
|
||||
|
||||
from sglang.srt.layers.activation import get_act_fn
|
||||
from sglang.srt.layers.attention.vision import VisionAttention
|
||||
from sglang.srt.layers.layer_boundary import layer_stack
|
||||
from sglang.srt.layers.layer_boundary.residual import batch as residual_batch
|
||||
from sglang.srt.layers.layernorm import RMSNorm
|
||||
from sglang.srt.layers.linear import (
|
||||
@@ -678,26 +679,27 @@ class MllamaTextModel(nn.Module):
|
||||
self.cross_attention_layers = config.cross_attention_layers
|
||||
|
||||
layers = []
|
||||
for layer_id in range(config.num_hidden_layers):
|
||||
if layer_id in self.cross_attention_layers:
|
||||
layers.append(
|
||||
MllamaCrossAttentionDecoderLayer(
|
||||
config,
|
||||
layer_id,
|
||||
quant_config=quant_config,
|
||||
prefix=add_prefix(f"layers.{layer_id}", prefix),
|
||||
with layer_stack():
|
||||
for layer_id in range(config.num_hidden_layers):
|
||||
if layer_id in self.cross_attention_layers:
|
||||
layers.append(
|
||||
MllamaCrossAttentionDecoderLayer(
|
||||
config,
|
||||
layer_id,
|
||||
quant_config=quant_config,
|
||||
prefix=add_prefix(f"layers.{layer_id}", prefix),
|
||||
)
|
||||
)
|
||||
)
|
||||
else:
|
||||
# TODO: force LlamaDecoderLayer to config.attention_bias=False
|
||||
layers.append(
|
||||
LlamaDecoderLayer(
|
||||
config,
|
||||
quant_config=quant_config,
|
||||
layer_id=layer_id,
|
||||
prefix=add_prefix(f"layers.{layer_id}", prefix),
|
||||
else:
|
||||
# TODO: force LlamaDecoderLayer to config.attention_bias=False
|
||||
layers.append(
|
||||
LlamaDecoderLayer(
|
||||
config,
|
||||
quant_config=quant_config,
|
||||
layer_id=layer_id,
|
||||
prefix=add_prefix(f"layers.{layer_id}", prefix),
|
||||
)
|
||||
)
|
||||
)
|
||||
|
||||
self.layers = nn.ModuleList(layers)
|
||||
self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
|
||||
|
||||
@@ -21,9 +21,10 @@ from sglang.srt.layers.attention.vision import (
|
||||
)
|
||||
from sglang.srt.layers.conv import Conv3dLayer
|
||||
from sglang.srt.layers.layer_boundary import (
|
||||
append_stages,
|
||||
declare_attn,
|
||||
declare_ffn,
|
||||
make_stages,
|
||||
layer_stack,
|
||||
)
|
||||
from sglang.srt.layers.layer_boundary.residual import batch as residual_batch
|
||||
from sglang.srt.layers.layernorm import RMSNorm
|
||||
@@ -1017,14 +1018,12 @@ class MossVLSelfAttentionDecoderLayer(nn.Module):
|
||||
config.hidden_size, eps=config.rms_norm_eps, **norm_kwargs
|
||||
)
|
||||
|
||||
self.attn_boundary, self.ffn_boundary = make_stages(
|
||||
self.attn_boundary, self.ffn_boundary = append_stages(
|
||||
(declare_attn(), self.input_layernorm),
|
||||
(
|
||||
declare_ffn(sparse=False, next_layer_sparse=False),
|
||||
self.post_attention_layernorm,
|
||||
),
|
||||
previous=declare_ffn() if layer_id != 0 else None,
|
||||
terminal=layer_id == config.num_hidden_layers - 1,
|
||||
)
|
||||
|
||||
def forward(
|
||||
@@ -1070,25 +1069,26 @@ class MossVLTextModel(nn.Module):
|
||||
self.cross_attention_layers = config.cross_attention_layers
|
||||
|
||||
layers = []
|
||||
for layer_id in range(config.num_hidden_layers):
|
||||
if layer_id in self.cross_attention_layers:
|
||||
layers.append(
|
||||
MossVLCrossAttentionDecoderLayer(
|
||||
config,
|
||||
layer_id,
|
||||
quant_config=quant_config,
|
||||
prefix=add_prefix(f"layers.{layer_id}", prefix),
|
||||
with layer_stack():
|
||||
for layer_id in range(config.num_hidden_layers):
|
||||
if layer_id in self.cross_attention_layers:
|
||||
layers.append(
|
||||
MossVLCrossAttentionDecoderLayer(
|
||||
config,
|
||||
layer_id,
|
||||
quant_config=quant_config,
|
||||
prefix=add_prefix(f"layers.{layer_id}", prefix),
|
||||
)
|
||||
)
|
||||
)
|
||||
else:
|
||||
layers.append(
|
||||
MossVLSelfAttentionDecoderLayer(
|
||||
config,
|
||||
layer_id,
|
||||
quant_config=quant_config,
|
||||
prefix=add_prefix(f"layers.{layer_id}", prefix),
|
||||
else:
|
||||
layers.append(
|
||||
MossVLSelfAttentionDecoderLayer(
|
||||
config,
|
||||
layer_id,
|
||||
quant_config=quant_config,
|
||||
prefix=add_prefix(f"layers.{layer_id}", prefix),
|
||||
)
|
||||
)
|
||||
)
|
||||
self.layers = nn.ModuleList(layers)
|
||||
self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
|
||||
|
||||
|
||||
@@ -9,9 +9,9 @@ from sglang.srt.layers.activation import SiluAndMul
|
||||
from sglang.srt.layers.aux_hidden_states import AuxHiddenStateList
|
||||
from sglang.srt.layers.dp_attention import is_dp_attention_enabled
|
||||
from sglang.srt.layers.layer_boundary import (
|
||||
append_stages,
|
||||
declare_attn,
|
||||
declare_ffn,
|
||||
make_stages,
|
||||
)
|
||||
from sglang.srt.layers.layer_boundary.residual import batch as residual_batch
|
||||
from sglang.srt.layers.layernorm import RMSNorm
|
||||
@@ -231,14 +231,12 @@ class NanbeigeDecoderLayer(nn.Module):
|
||||
)
|
||||
# Each loop runs the physical layers as one stack: the model folds the
|
||||
# residual back in and re-enters at layer 0.
|
||||
self.attn_boundary, self.ffn_boundary = make_stages(
|
||||
self.attn_boundary, self.ffn_boundary = append_stages(
|
||||
(declare_attn(), self.input_layernorm),
|
||||
(
|
||||
declare_ffn(sparse=False, next_layer_sparse=False),
|
||||
self.post_attention_layernorm,
|
||||
),
|
||||
previous=declare_ffn() if layer_id != 0 else None,
|
||||
terminal=layer_id == config.num_hidden_layers - 1,
|
||||
)
|
||||
|
||||
def forward(
|
||||
|
||||
@@ -19,6 +19,7 @@ from torch import nn
|
||||
|
||||
from sglang.srt.configs import NemotronHConfig
|
||||
from sglang.srt.layers.dp_attention import is_dp_attention_enabled
|
||||
from sglang.srt.layers.layer_boundary import layer_stack
|
||||
from sglang.srt.layers.layer_boundary.residual import batch as residual_batch
|
||||
from sglang.srt.layers.layernorm import RMSNorm
|
||||
from sglang.srt.layers.linear import ColumnParallelLinear
|
||||
@@ -239,34 +240,37 @@ class NemotronHMultiTokenPredictor(nn.Module):
|
||||
|
||||
# Total number of physical layers = num_steps * pattern_len
|
||||
total_layers = self.num_mtp_layers * self.pattern_len
|
||||
for i in range(total_layers):
|
||||
step_rel_idx = i % self.pattern_len
|
||||
with layer_stack():
|
||||
for i in range(total_layers):
|
||||
step_rel_idx = i % self.pattern_len
|
||||
|
||||
char = self.pattern_str[step_rel_idx]
|
||||
char = self.pattern_str[step_rel_idx]
|
||||
|
||||
is_start_of_step = step_rel_idx == 0
|
||||
is_end_of_step = step_rel_idx == self.pattern_len - 1
|
||||
is_start_of_step = step_rel_idx == 0
|
||||
is_end_of_step = step_rel_idx == self.pattern_len - 1
|
||||
|
||||
layer_prefix = f"{prefix}.layers.{i}"
|
||||
layer_prefix = f"{prefix}.layers.{i}"
|
||||
|
||||
common_kwargs = dict(
|
||||
config=config,
|
||||
layer_idx=i,
|
||||
quant_config=quant_config,
|
||||
prefix=layer_prefix,
|
||||
has_start_projections=is_start_of_step,
|
||||
has_end_norm=is_end_of_step,
|
||||
)
|
||||
|
||||
if char == "*":
|
||||
self.layers[str(i)] = NemotronHMTPAttentionDecoderLayer(**common_kwargs)
|
||||
elif char == "E":
|
||||
self.layers[str(i)] = NemotronHMTPMoEDecoderLayer(**common_kwargs)
|
||||
else:
|
||||
raise NotImplementedError(
|
||||
f"Pattern char '{char}' in {self.pattern_str} not implemented"
|
||||
common_kwargs = dict(
|
||||
config=config,
|
||||
layer_idx=i,
|
||||
quant_config=quant_config,
|
||||
prefix=layer_prefix,
|
||||
has_start_projections=is_start_of_step,
|
||||
has_end_norm=is_end_of_step,
|
||||
)
|
||||
|
||||
if char == "*":
|
||||
self.layers[str(i)] = NemotronHMTPAttentionDecoderLayer(
|
||||
**common_kwargs
|
||||
)
|
||||
elif char == "E":
|
||||
self.layers[str(i)] = NemotronHMTPMoEDecoderLayer(**common_kwargs)
|
||||
else:
|
||||
raise NotImplementedError(
|
||||
f"Pattern char '{char}' in {self.pattern_str} not implemented"
|
||||
)
|
||||
|
||||
def get_input_embeddings(self, input_ids: torch.Tensor) -> torch.Tensor:
|
||||
assert self.embed_tokens is not None, (
|
||||
"embed_tokens not initialized - must be shared from target model"
|
||||
|
||||
@@ -4,9 +4,9 @@ from sglang.srt.configs.nemotron_h import ATTENTION, MAMBA, MOE
|
||||
from sglang.srt.layers.layer_boundary import (
|
||||
ExitRows,
|
||||
ProducerReduction,
|
||||
append_stages,
|
||||
declare_attn,
|
||||
declare_ffn,
|
||||
make_stages,
|
||||
)
|
||||
from sglang.srt.layers.layer_boundary.residual.add_norm import (
|
||||
NormQuantReadout,
|
||||
@@ -42,16 +42,10 @@ def make_stage_boundary(layer_norm: RMSNorm, *, pattern: str, layer_idx: int):
|
||||
|
||||
if get_parallel().attn_cp_size > 1 or layernorm_sp.layernorm_sp_enabled():
|
||||
raise NotImplementedError("a Nemotron stage with attention CP or LayerNorm SP")
|
||||
previous = _declaration(pattern, layer_idx - 1) if layer_idx > 0 else None
|
||||
(boundary,) = make_stages(
|
||||
(boundary,) = append_stages(
|
||||
(
|
||||
_declaration(pattern, layer_idx),
|
||||
layer_norm,
|
||||
),
|
||||
previous=previous,
|
||||
following=_declaration(pattern, layer_idx + 1)
|
||||
if layer_idx + 1 < len(pattern)
|
||||
else None,
|
||||
terminal=layer_idx == len(pattern) - 1,
|
||||
)
|
||||
return boundary
|
||||
|
||||
@@ -24,9 +24,9 @@ from torch import nn
|
||||
from transformers import LlamaConfig
|
||||
|
||||
from sglang.srt.layers.layer_boundary import (
|
||||
append_stages,
|
||||
declare_attn,
|
||||
declare_ffn,
|
||||
make_stages,
|
||||
)
|
||||
from sglang.srt.layers.layer_boundary.residual import batch as residual_batch
|
||||
from sglang.srt.layers.layernorm import RMSNorm
|
||||
@@ -63,21 +63,6 @@ def _find_multiple(n: int, k: int) -> int:
|
||||
return n + k - (n % k)
|
||||
|
||||
|
||||
def _previous_stage(block_configs, layer_idx: int):
|
||||
"""The declaration of the last attention or FFN before this layer, or None
|
||||
when this layer's first stage starts the layer stack."""
|
||||
for block in reversed(block_configs[:layer_idx]):
|
||||
if not block.ffn.no_op:
|
||||
return declare_ffn()
|
||||
if not block.attention.no_op:
|
||||
return declare_attn()
|
||||
return None
|
||||
|
||||
|
||||
def _has_stage(block) -> bool:
|
||||
return not (block.attention.no_op and block.ffn.no_op)
|
||||
|
||||
|
||||
class DeciLMDecoderLayer(nn.Module):
|
||||
def __init__(
|
||||
self,
|
||||
@@ -162,12 +147,8 @@ class DeciLMDecoderLayer(nn.Module):
|
||||
)
|
||||
self.entry_boundary = None
|
||||
if stages:
|
||||
boundaries = make_stages(
|
||||
boundaries = append_stages(
|
||||
*stages,
|
||||
previous=_previous_stage(config.block_configs, layer_idx),
|
||||
terminal=not any(
|
||||
_has_stage(block) for block in config.block_configs[layer_idx + 1 :]
|
||||
),
|
||||
)
|
||||
self.entry_boundary = boundaries[0]
|
||||
if not self._is_no_op_attention:
|
||||
|
||||
@@ -29,9 +29,9 @@ from sglang.srt.layers.activation import SiluAndMul
|
||||
from sglang.srt.layers.aux_hidden_states import AuxHiddenStateList
|
||||
from sglang.srt.layers.dp_attention import is_dp_attention_enabled
|
||||
from sglang.srt.layers.layer_boundary import (
|
||||
append_stages,
|
||||
declare_attn,
|
||||
declare_ffn,
|
||||
make_stages,
|
||||
)
|
||||
from sglang.srt.layers.layer_boundary.residual import batch as residual_batch
|
||||
from sglang.srt.layers.layer_boundary.residual.add_norm import NORM_QUANT_READOUT
|
||||
@@ -292,14 +292,12 @@ class Qwen2DecoderLayer(nn.Module):
|
||||
)
|
||||
self.input_layernorm.fuse_input_quant(self.self_attn.qkv_proj)
|
||||
self.post_attention_layernorm.fuse_input_quant(self.mlp.gate_up_proj)
|
||||
self.attn_boundary, self.ffn_boundary = make_stages(
|
||||
self.attn_boundary, self.ffn_boundary = append_stages(
|
||||
(declare_attn(read=self._attn_readout(layer_id)), self.input_layernorm),
|
||||
(
|
||||
declare_ffn(sparse=False, next_layer_sparse=False),
|
||||
self.post_attention_layernorm,
|
||||
),
|
||||
previous=declare_ffn() if layer_id != 0 else None,
|
||||
terminal=layer_id == config.num_hidden_layers - 1,
|
||||
)
|
||||
|
||||
def _attn_readout(self, layer_id: int):
|
||||
|
||||
@@ -24,6 +24,7 @@ from typing import Iterable, Optional, Tuple
|
||||
import torch
|
||||
from torch import nn
|
||||
|
||||
from sglang.srt.layers.layer_boundary import layer_stack
|
||||
from sglang.srt.layers.layer_boundary.residual import batch as residual_batch
|
||||
from sglang.srt.layers.layer_boundary.residual.post_norm import PLAIN_READOUT
|
||||
from sglang.srt.layers.logits_processor import LogitsProcessor
|
||||
@@ -75,17 +76,18 @@ class Qwen2Model(nn.Module):
|
||||
config.hidden_size,
|
||||
prefix=add_prefix("embed_tokens", prefix),
|
||||
)
|
||||
self.layers = nn.ModuleList(
|
||||
[
|
||||
Qwen2DecoderLayer(
|
||||
config,
|
||||
i,
|
||||
quant_config=quant_config,
|
||||
prefix=add_prefix(f"layers.{i}", prefix),
|
||||
)
|
||||
for i in range(config.num_hidden_layers)
|
||||
]
|
||||
)
|
||||
with layer_stack():
|
||||
self.layers = nn.ModuleList(
|
||||
[
|
||||
Qwen2DecoderLayer(
|
||||
config,
|
||||
i,
|
||||
quant_config=quant_config,
|
||||
prefix=add_prefix(f"layers.{i}", prefix),
|
||||
)
|
||||
for i in range(config.num_hidden_layers)
|
||||
]
|
||||
)
|
||||
self.fc = torch.nn.Linear(config.hidden_size * 2, config.hidden_size)
|
||||
|
||||
def forward(
|
||||
|
||||
@@ -49,9 +49,9 @@ from sglang.srt.layers.dp_attention import (
|
||||
)
|
||||
from sglang.srt.layers.flashinfer_comm_fusion import uses_cutedsl_ar_fusion
|
||||
from sglang.srt.layers.layer_boundary import (
|
||||
append_stages,
|
||||
declare_attn,
|
||||
declare_ffn,
|
||||
make_stages,
|
||||
)
|
||||
from sglang.srt.layers.layer_boundary.residual import batch as residual_batch
|
||||
from sglang.srt.layers.layernorm import RMSNorm
|
||||
@@ -1027,7 +1027,6 @@ class Qwen2MoeDecoderLayer(nn.Module):
|
||||
|
||||
# Qwen2MoE all layers are sparse and have no nextn now
|
||||
self.is_layer_sparse = True
|
||||
is_previous_layer_sparse = True
|
||||
is_next_layer_sparse = True
|
||||
|
||||
if self.is_layer_sparse:
|
||||
@@ -1052,7 +1051,7 @@ class Qwen2MoeDecoderLayer(nn.Module):
|
||||
self.post_attention_layernorm = RMSNorm(
|
||||
config.hidden_size, eps=config.rms_norm_eps
|
||||
)
|
||||
self.attn_boundary, self.ffn_boundary = make_stages(
|
||||
self.attn_boundary, self.ffn_boundary = append_stages(
|
||||
(declare_attn(), self.input_layernorm),
|
||||
(
|
||||
declare_ffn(
|
||||
@@ -1061,12 +1060,6 @@ class Qwen2MoeDecoderLayer(nn.Module):
|
||||
),
|
||||
self.post_attention_layernorm,
|
||||
),
|
||||
previous=declare_ffn(
|
||||
sparse=is_previous_layer_sparse, next_layer_sparse=self.is_layer_sparse
|
||||
)
|
||||
if layer_id != 0
|
||||
else None,
|
||||
terminal=layer_id == config.num_hidden_layers - 1,
|
||||
)
|
||||
|
||||
def forward(
|
||||
|
||||
@@ -7,9 +7,9 @@ from torch import nn
|
||||
|
||||
from sglang.srt.layers.aux_hidden_states import AuxHiddenStateList
|
||||
from sglang.srt.layers.layer_boundary import (
|
||||
append_stages,
|
||||
declare_attn,
|
||||
declare_ffn,
|
||||
make_stages,
|
||||
)
|
||||
from sglang.srt.layers.layer_boundary.residual import batch as residual_batch
|
||||
from sglang.srt.layers.layernorm import RMSNorm
|
||||
@@ -375,14 +375,12 @@ class Qwen3DecoderLayer(nn.Module):
|
||||
config.hidden_size, eps=config.rms_norm_eps, **norm_kwargs
|
||||
)
|
||||
|
||||
self.attn_boundary, self.ffn_boundary = make_stages(
|
||||
self.attn_boundary, self.ffn_boundary = append_stages(
|
||||
(declare_attn(), self.input_layernorm),
|
||||
(
|
||||
declare_ffn(sparse=False, next_layer_sparse=False),
|
||||
self.post_attention_layernorm,
|
||||
),
|
||||
previous=declare_ffn() if layer_id != 0 else None,
|
||||
terminal=layer_id == config.num_hidden_layers - 1,
|
||||
)
|
||||
|
||||
def forward(
|
||||
|
||||
@@ -54,9 +54,9 @@ from sglang.srt.layers.dp_attention import (
|
||||
)
|
||||
from sglang.srt.layers.flashinfer_comm_fusion import uses_cutedsl_ar_fusion
|
||||
from sglang.srt.layers.layer_boundary import (
|
||||
append_stages,
|
||||
declare_attn,
|
||||
declare_ffn,
|
||||
make_stages,
|
||||
)
|
||||
|
||||
# Layers - Attention
|
||||
@@ -447,7 +447,6 @@ class Qwen3_5GatedDeltaNet(nn.Module):
|
||||
eps=self.layer_norm_epsilon,
|
||||
group_size=None,
|
||||
norm_before_gate=True,
|
||||
device=torch.get_device_module().current_device(),
|
||||
dtype=torch.get_default_dtype(),
|
||||
**(
|
||||
{"activation": self.output_gate_type}
|
||||
@@ -1078,7 +1077,6 @@ class Qwen3_5LinearDecoderLayer(nn.Module):
|
||||
reduce_results=False,
|
||||
)
|
||||
is_layer_sparse = True
|
||||
is_previous_layer_sparse = True
|
||||
is_next_layer_sparse = True
|
||||
elif config.model_type == "qwen3_5_text":
|
||||
self.mlp = Qwen2MoeMLP(
|
||||
@@ -1092,7 +1090,6 @@ class Qwen3_5LinearDecoderLayer(nn.Module):
|
||||
)
|
||||
_maybe_enable_silu_fp4_quant_fusion(self.mlp)
|
||||
is_layer_sparse = False
|
||||
is_previous_layer_sparse = False
|
||||
is_next_layer_sparse = False
|
||||
else:
|
||||
raise ValueError(f"Invalid model type: {config.model_type}")
|
||||
@@ -1108,7 +1105,7 @@ class Qwen3_5LinearDecoderLayer(nn.Module):
|
||||
boundary_fusions = _layer_fusions(config, is_nextn)
|
||||
# A subclass that brings its own residual builds the stages itself.
|
||||
if build_stages:
|
||||
self.attn_boundary, self.ffn_boundary = make_stages(
|
||||
self.attn_boundary, self.ffn_boundary = append_stages(
|
||||
(
|
||||
declare_attn(
|
||||
read=NormQuantReadout(
|
||||
@@ -1130,12 +1127,6 @@ class Qwen3_5LinearDecoderLayer(nn.Module):
|
||||
self.post_attention_layernorm,
|
||||
{"fusions": boundary_fusions},
|
||||
),
|
||||
previous=declare_ffn(
|
||||
sparse=is_previous_layer_sparse, next_layer_sparse=is_layer_sparse
|
||||
)
|
||||
if layer_id != 0
|
||||
else None,
|
||||
terminal=layer_id == config.num_hidden_layers - 1,
|
||||
)
|
||||
|
||||
def forward(
|
||||
@@ -1285,7 +1276,6 @@ class Qwen3_5AttentionDecoderLayer(nn.Module):
|
||||
reduce_results=False,
|
||||
)
|
||||
is_layer_sparse = False
|
||||
is_previous_layer_sparse = False
|
||||
is_next_layer_sparse = False
|
||||
elif config.model_type in _QWEN3_5_MOE_TEXT_MODEL_TYPES:
|
||||
self.mlp = Qwen2MoeSparseMoeBlock(
|
||||
@@ -1304,7 +1294,6 @@ class Qwen3_5AttentionDecoderLayer(nn.Module):
|
||||
reduce_results=False,
|
||||
)
|
||||
is_layer_sparse = True
|
||||
is_previous_layer_sparse = True
|
||||
is_next_layer_sparse = True
|
||||
else:
|
||||
raise ValueError(f"Invalid model type: {config.model_type}")
|
||||
@@ -1323,7 +1312,7 @@ class Qwen3_5AttentionDecoderLayer(nn.Module):
|
||||
boundary_fusions = _layer_fusions(config, is_nextn)
|
||||
# A subclass that brings its own residual builds the stages itself.
|
||||
if build_stages:
|
||||
self.attn_boundary, self.ffn_boundary = make_stages(
|
||||
self.attn_boundary, self.ffn_boundary = append_stages(
|
||||
(
|
||||
declare_attn(
|
||||
read=NormQuantReadout(
|
||||
@@ -1343,12 +1332,6 @@ class Qwen3_5AttentionDecoderLayer(nn.Module):
|
||||
self.post_attention_layernorm,
|
||||
{"fusions": boundary_fusions},
|
||||
),
|
||||
previous=declare_ffn(
|
||||
sparse=is_previous_layer_sparse, next_layer_sparse=is_layer_sparse
|
||||
)
|
||||
if layer_id != 0
|
||||
else None,
|
||||
terminal=layer_id == config.num_hidden_layers - 1,
|
||||
)
|
||||
|
||||
self.alt_stream = alt_stream
|
||||
|
||||
@@ -31,9 +31,9 @@ from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation
|
||||
from sglang.srt.eplb.expert_location_dispatch import ExpertLocationDispatchInfo
|
||||
from sglang.srt.layers.aux_hidden_states import AuxHiddenStateAccumulator
|
||||
from sglang.srt.layers.layer_boundary import (
|
||||
append_stages,
|
||||
declare_attn,
|
||||
declare_ffn,
|
||||
make_stages,
|
||||
)
|
||||
from sglang.srt.layers.layer_boundary.residual import batch as residual_batch
|
||||
from sglang.srt.layers.layernorm import RMSNorm
|
||||
@@ -858,7 +858,6 @@ class Qwen3MoeDecoderLayer(nn.Module):
|
||||
|
||||
# Qwen3MoE all layers are sparse and have no nextn now
|
||||
self.is_layer_sparse = True
|
||||
is_previous_layer_sparse = True
|
||||
is_next_layer_sparse = True
|
||||
|
||||
if self.is_layer_sparse:
|
||||
@@ -882,7 +881,7 @@ class Qwen3MoeDecoderLayer(nn.Module):
|
||||
config.hidden_size, eps=config.rms_norm_eps
|
||||
)
|
||||
|
||||
self.attn_boundary, self.ffn_boundary = make_stages(
|
||||
self.attn_boundary, self.ffn_boundary = append_stages(
|
||||
(declare_attn(), self.input_layernorm),
|
||||
(
|
||||
declare_ffn(
|
||||
@@ -891,12 +890,6 @@ class Qwen3MoeDecoderLayer(nn.Module):
|
||||
),
|
||||
self.post_attention_layernorm,
|
||||
),
|
||||
previous=declare_ffn(
|
||||
sparse=is_previous_layer_sparse, next_layer_sparse=self.is_layer_sparse
|
||||
)
|
||||
if layer_id != 0
|
||||
else None,
|
||||
terminal=layer_id == config.num_hidden_layers - 1,
|
||||
)
|
||||
|
||||
def forward(
|
||||
|
||||
@@ -23,9 +23,9 @@ from sglang.srt.layers.dp_attention import (
|
||||
is_dp_attention_enabled,
|
||||
)
|
||||
from sglang.srt.layers.layer_boundary import (
|
||||
append_stages,
|
||||
declare_attn,
|
||||
declare_ffn,
|
||||
make_stages,
|
||||
)
|
||||
from sglang.srt.layers.layer_boundary.residual import batch as residual_batch
|
||||
from sglang.srt.layers.layernorm import GemmaRMSNorm
|
||||
@@ -509,7 +509,6 @@ class Qwen3HybridLinearDecoderLayer(nn.Module):
|
||||
|
||||
# Qwen3Next all layers are sparse and have no nextn now
|
||||
self.is_layer_sparse = True
|
||||
is_previous_layer_sparse = True
|
||||
is_next_layer_sparse = True
|
||||
self.layer_id = layer_id
|
||||
|
||||
@@ -538,7 +537,7 @@ class Qwen3HybridLinearDecoderLayer(nn.Module):
|
||||
self.post_attention_layernorm = GemmaRMSNorm(
|
||||
config.hidden_size, eps=config.rms_norm_eps
|
||||
)
|
||||
self.attn_boundary, self.ffn_boundary = make_stages(
|
||||
self.attn_boundary, self.ffn_boundary = append_stages(
|
||||
(declare_attn(), self.input_layernorm),
|
||||
(
|
||||
declare_ffn(
|
||||
@@ -547,12 +546,6 @@ class Qwen3HybridLinearDecoderLayer(nn.Module):
|
||||
),
|
||||
self.post_attention_layernorm,
|
||||
),
|
||||
previous=declare_ffn(
|
||||
sparse=is_previous_layer_sparse, next_layer_sparse=self.is_layer_sparse
|
||||
)
|
||||
if layer_id != 0
|
||||
else None,
|
||||
terminal=layer_id == config.num_hidden_layers - 1,
|
||||
)
|
||||
|
||||
def forward(
|
||||
@@ -676,7 +669,6 @@ class Qwen3HybridAttentionDecoderLayer(nn.Module):
|
||||
|
||||
# Qwen3Next all layers are sparse and have no nextn now
|
||||
self.is_layer_sparse = True
|
||||
is_previous_layer_sparse = True
|
||||
is_next_layer_sparse = True
|
||||
|
||||
if self.is_layer_sparse:
|
||||
@@ -708,7 +700,7 @@ class Qwen3HybridAttentionDecoderLayer(nn.Module):
|
||||
self.q_norm = GemmaRMSNorm(self.head_dim, eps=config.rms_norm_eps)
|
||||
self.k_norm = GemmaRMSNorm(self.head_dim, eps=config.rms_norm_eps)
|
||||
|
||||
self.attn_boundary, self.ffn_boundary = make_stages(
|
||||
self.attn_boundary, self.ffn_boundary = append_stages(
|
||||
(declare_attn(), self.input_layernorm),
|
||||
(
|
||||
declare_ffn(
|
||||
@@ -717,12 +709,6 @@ class Qwen3HybridAttentionDecoderLayer(nn.Module):
|
||||
),
|
||||
self.post_attention_layernorm,
|
||||
),
|
||||
previous=declare_ffn(
|
||||
sparse=is_previous_layer_sparse, next_layer_sparse=self.is_layer_sparse
|
||||
)
|
||||
if layer_id != 0
|
||||
else None,
|
||||
terminal=layer_id == config.num_hidden_layers - 1,
|
||||
)
|
||||
|
||||
self.alt_stream = alt_stream
|
||||
|
||||
@@ -39,10 +39,10 @@ from sglang.srt.layers.hyperconnection import (
|
||||
from sglang.srt.layers.layer_boundary import (
|
||||
ExitRows,
|
||||
GatedResidualState,
|
||||
append_stages,
|
||||
declare_attn,
|
||||
declare_ffn,
|
||||
get_attn_tp_context,
|
||||
make_stages,
|
||||
)
|
||||
from sglang.srt.layers.layer_boundary.residual import batch as residual_batch
|
||||
from sglang.srt.layers.linear import ReplicatedLinear
|
||||
@@ -88,6 +88,7 @@ from sglang.srt.models.qwen4_exp_ple_table import (
|
||||
)
|
||||
from sglang.srt.runtime_context import get_parallel
|
||||
from sglang.srt.utils import get_bool_env_var, is_hip, logger
|
||||
from sglang.srt.utils.common import is_building_neighbour_layer
|
||||
|
||||
_use_aiter = get_bool_env_var("SGLANG_USE_AITER") and is_hip()
|
||||
|
||||
@@ -534,6 +535,13 @@ class Qwen4ExpPLEGroupedNorm(nn.Module):
|
||||
return (x_norm * weight).to(compute_dtype)
|
||||
|
||||
|
||||
def _offloads_ple(config) -> bool:
|
||||
"""Whether this layer keeps its PLE table in host memory. A pipeline
|
||||
neighbour layer is never loaded or run, so it keeps the table on the meta
|
||||
device and allocates no host memory or stream for it."""
|
||||
return bool(config.ple_offload_embedding) and not is_building_neighbour_layer()
|
||||
|
||||
|
||||
class Qwen4ExpNGramEmbedding(nn.Module):
|
||||
_MASK64 = (1 << 64) - 1
|
||||
_SPLITMIX_GAMMA = 0x9E3779B97F4A7C15
|
||||
@@ -605,7 +613,7 @@ class Qwen4ExpNGramEmbedding(nn.Module):
|
||||
and not self.use_attn_tp_ngram
|
||||
)
|
||||
ngram_prefix = f"{prefix}.ngram_embedding" if prefix else "ngram_embedding"
|
||||
offload_embedding = bool(config.ple_offload_embedding)
|
||||
offload_embedding = _offloads_ple(config)
|
||||
# Offload only needs this embedding's metadata: build it on meta so the
|
||||
# shard is never allocated on the device.
|
||||
with torch.device("meta") if offload_embedding else nullcontext():
|
||||
@@ -1088,9 +1096,7 @@ class Qwen4ExpPLELayer(nn.Module):
|
||||
bias=False,
|
||||
)
|
||||
nn.init.zeros_(self.conv1d.weight)
|
||||
self._prefetch_stream = (
|
||||
torch.cuda.Stream() if config.ple_offload_embedding else None
|
||||
)
|
||||
self._prefetch_stream = torch.cuda.Stream() if _offloads_ple(config) else None
|
||||
self._graph_prefetch_buffers = {}
|
||||
self._eager_prefetch_buffer = None
|
||||
self._prefetch_state = None
|
||||
@@ -1441,7 +1447,7 @@ def _build_qwen4_exp_stages(residual, *, sparse, layer_id, config):
|
||||
the writes are gated injections, so neither stage offers a fused
|
||||
add-and-norm candidate.
|
||||
"""
|
||||
return make_stages(
|
||||
return append_stages(
|
||||
(
|
||||
declare_attn(read=residual.attn_readout, update=residual.attn_update),
|
||||
None,
|
||||
@@ -1456,17 +1462,6 @@ def _build_qwen4_exp_stages(residual, *, sparse, layer_id, config):
|
||||
),
|
||||
None,
|
||||
),
|
||||
previous=(
|
||||
declare_ffn(
|
||||
sparse=sparse,
|
||||
next_layer_sparse=sparse,
|
||||
update=residual.ffn_update,
|
||||
exit_rows=_ffn_exit_rows(layer_id - 1, config),
|
||||
)
|
||||
if layer_id != 0
|
||||
else None
|
||||
),
|
||||
terminal=layer_id == config.num_hidden_layers - 1,
|
||||
)
|
||||
|
||||
|
||||
|
||||
@@ -20,10 +20,10 @@ from sglang.srt.layers.dp_attention import (
|
||||
is_dp_attention_enabled,
|
||||
)
|
||||
from sglang.srt.layers.layer_boundary import (
|
||||
append_stages,
|
||||
declare_attn,
|
||||
declare_ffn,
|
||||
is_dense_ffn_fully_dp,
|
||||
make_stages,
|
||||
)
|
||||
from sglang.srt.layers.layer_boundary.residual import batch as residual_batch
|
||||
from sglang.srt.layers.layernorm import RMSNorm
|
||||
@@ -1002,12 +1002,6 @@ class SarvamMoEMLADecoderLayer(nn.Module):
|
||||
and layer_id >= first_k_dense
|
||||
and (layer_id - first_k_dense) % moe_layer_freq == 0
|
||||
)
|
||||
is_previous_layer_sparse = (
|
||||
has_moe
|
||||
and layer_id > 0
|
||||
and (layer_id - 1) >= first_k_dense
|
||||
and (layer_id - 1 - first_k_dense) % moe_layer_freq == 0
|
||||
)
|
||||
is_next_layer_sparse = (
|
||||
has_moe
|
||||
and layer_id < config.num_hidden_layers - 1
|
||||
@@ -1044,7 +1038,7 @@ class SarvamMoEMLADecoderLayer(nn.Module):
|
||||
config.hidden_size, eps=config.rms_norm_eps
|
||||
)
|
||||
|
||||
self.attn_boundary, self.ffn_boundary = make_stages(
|
||||
self.attn_boundary, self.ffn_boundary = append_stages(
|
||||
(
|
||||
declare_attn(),
|
||||
self.input_layernorm,
|
||||
@@ -1057,12 +1051,6 @@ class SarvamMoEMLADecoderLayer(nn.Module):
|
||||
),
|
||||
self.post_attention_layernorm,
|
||||
),
|
||||
previous=declare_ffn(
|
||||
sparse=is_previous_layer_sparse, next_layer_sparse=self.is_layer_sparse
|
||||
)
|
||||
if layer_id != 0
|
||||
else None,
|
||||
terminal=layer_id == config.num_hidden_layers - 1,
|
||||
)
|
||||
|
||||
def forward(
|
||||
|
||||
@@ -15,9 +15,9 @@ from sglang.srt.layers.dp_attention import (
|
||||
is_dp_attention_enabled,
|
||||
)
|
||||
from sglang.srt.layers.layer_boundary import (
|
||||
append_stages,
|
||||
declare_attn,
|
||||
declare_ffn,
|
||||
make_stages,
|
||||
)
|
||||
from sglang.srt.layers.layer_boundary.residual import batch as residual_batch
|
||||
from sglang.srt.layers.layernorm import RMSNorm
|
||||
@@ -294,7 +294,7 @@ class SDARBlock(nn.Module):
|
||||
prefix=add_prefix("mlp", prefix),
|
||||
)
|
||||
|
||||
self.attn_boundary, self.ffn_boundary = make_stages(
|
||||
self.attn_boundary, self.ffn_boundary = append_stages(
|
||||
(declare_attn(), self.input_layernorm),
|
||||
(
|
||||
declare_ffn(
|
||||
@@ -303,8 +303,6 @@ class SDARBlock(nn.Module):
|
||||
),
|
||||
self.post_attention_layernorm,
|
||||
),
|
||||
previous=declare_ffn() if layer_id != 0 else None,
|
||||
terminal=layer_id == config.num_hidden_layers - 1,
|
||||
)
|
||||
|
||||
def forward(
|
||||
|
||||
@@ -17,9 +17,9 @@ from sglang.srt.layers.dp_attention import (
|
||||
is_dp_attention_enabled,
|
||||
)
|
||||
from sglang.srt.layers.layer_boundary import (
|
||||
append_stages,
|
||||
declare_attn,
|
||||
declare_ffn,
|
||||
make_stages,
|
||||
)
|
||||
from sglang.srt.layers.layer_boundary.residual import batch as residual_batch
|
||||
from sglang.srt.layers.layernorm import RMSNorm
|
||||
@@ -353,16 +353,12 @@ class SDARMoeBlock(nn.Module):
|
||||
prefix=add_prefix("mlp", prefix),
|
||||
)
|
||||
|
||||
self.attn_boundary, self.ffn_boundary = make_stages(
|
||||
self.attn_boundary, self.ffn_boundary = append_stages(
|
||||
(declare_attn(), self.input_layernorm),
|
||||
(
|
||||
declare_ffn(sparse=True, next_layer_sparse=True),
|
||||
self.post_attention_layernorm,
|
||||
),
|
||||
previous=declare_ffn(sparse=True, next_layer_sparse=True)
|
||||
if layer_id != 0
|
||||
else None,
|
||||
terminal=layer_id == config.num_hidden_layers - 1,
|
||||
)
|
||||
|
||||
def forward(
|
||||
|
||||
@@ -7,9 +7,9 @@ from torch import nn
|
||||
from sglang.srt.layers.activation import GeluAndMul
|
||||
from sglang.srt.layers.dp_attention import is_dp_attention_enabled
|
||||
from sglang.srt.layers.layer_boundary import (
|
||||
append_stages,
|
||||
declare_attn,
|
||||
declare_ffn,
|
||||
make_stages,
|
||||
)
|
||||
from sglang.srt.layers.layer_boundary.residual import batch as residual_batch
|
||||
from sglang.srt.layers.layernorm import RMSNorm
|
||||
@@ -271,14 +271,12 @@ class Spark2_5DecoderLayer(nn.Module):
|
||||
self.post_attention_layernorm = RMSNorm(
|
||||
config.hidden_size, eps=config.rms_norm_eps
|
||||
)
|
||||
self.attn_boundary, self.ffn_boundary = make_stages(
|
||||
self.attn_boundary, self.ffn_boundary = append_stages(
|
||||
(declare_attn(), self.input_layernorm),
|
||||
(
|
||||
declare_ffn(sparse=False, next_layer_sparse=False),
|
||||
self.post_attention_layernorm,
|
||||
),
|
||||
previous=declare_ffn() if layer_id != 0 else None,
|
||||
terminal=layer_id == config.num_hidden_layers - 1,
|
||||
)
|
||||
|
||||
def forward(
|
||||
|
||||
@@ -25,9 +25,9 @@ from sglang.srt.layers.dp_attention import (
|
||||
reject_attn_tp_shard_with_tp_reduce,
|
||||
)
|
||||
from sglang.srt.layers.layer_boundary import (
|
||||
append_stages,
|
||||
declare_attn,
|
||||
declare_ffn,
|
||||
make_stages,
|
||||
)
|
||||
from sglang.srt.layers.layer_boundary.residual import batch as residual_batch
|
||||
from sglang.srt.layers.layernorm import RMSNorm
|
||||
@@ -334,9 +334,6 @@ class Step3TextDecoderLayer(nn.Module):
|
||||
|
||||
self.layer_id = layer_id
|
||||
self.is_layer_sparse = True if layer_id in moe_layers_idx else False
|
||||
self.is_previous_layer_sparse = (
|
||||
True if layer_id - 1 in moe_layers_idx else False
|
||||
)
|
||||
self.is_next_layer_sparse = True if layer_id + 1 in moe_layers_idx else False
|
||||
|
||||
if not self.is_layer_sparse:
|
||||
@@ -373,7 +370,7 @@ class Step3TextDecoderLayer(nn.Module):
|
||||
prefix=add_prefix("mlp", prefix),
|
||||
)
|
||||
|
||||
self.attn_boundary, self.ffn_boundary = make_stages(
|
||||
self.attn_boundary, self.ffn_boundary = append_stages(
|
||||
(declare_attn(), self.input_layernorm),
|
||||
(
|
||||
declare_ffn(
|
||||
@@ -382,13 +379,6 @@ class Step3TextDecoderLayer(nn.Module):
|
||||
),
|
||||
self.post_attention_layernorm,
|
||||
),
|
||||
previous=declare_ffn(
|
||||
sparse=self.is_previous_layer_sparse,
|
||||
next_layer_sparse=self.is_layer_sparse,
|
||||
)
|
||||
if layer_id != 0
|
||||
else None,
|
||||
terminal=layer_id == config.num_hidden_layers - 1,
|
||||
)
|
||||
|
||||
def moe_mlp_forward(self, hidden_states):
|
||||
|
||||
@@ -11,9 +11,9 @@ from sglang.srt.layers.dp_attention import (
|
||||
is_dp_attention_enabled,
|
||||
)
|
||||
from sglang.srt.layers.layer_boundary import (
|
||||
append_stages,
|
||||
declare_attn,
|
||||
declare_ffn,
|
||||
make_stages,
|
||||
)
|
||||
from sglang.srt.layers.layer_boundary.residual import batch as residual_batch
|
||||
from sglang.srt.layers.layernorm import GemmaRMSNorm
|
||||
@@ -409,7 +409,6 @@ class Step3p5DecoderLayer(nn.Module):
|
||||
quant_config: Optional[QuantizationConfig] = None,
|
||||
prefix: str = "",
|
||||
alt_stream: Optional[torch.cuda.Stream] = None,
|
||||
is_nextn: bool = False,
|
||||
) -> None:
|
||||
super().__init__()
|
||||
self.hidden_size = config.hidden_size
|
||||
@@ -426,7 +425,6 @@ class Step3p5DecoderLayer(nn.Module):
|
||||
self.num_attention_heads = config.num_attention_heads
|
||||
self.num_key_value_heads = config.num_attention_groups
|
||||
self.is_moe_layer = layer_id in moe_layers_set
|
||||
self.is_previous_layer_sparse = (layer_id - 1) in moe_layers_set
|
||||
self.is_next_layer_sparse = (layer_id + 1) in moe_layers_set
|
||||
num_hidden_layers = config.num_hidden_layers
|
||||
|
||||
@@ -520,9 +518,7 @@ class Step3p5DecoderLayer(nn.Module):
|
||||
config.hidden_size, eps=config.rms_norm_eps
|
||||
)
|
||||
|
||||
# An MTP draft is a one-layer model; layer_id still indexes its config.
|
||||
|
||||
self.attn_boundary, self.ffn_boundary = make_stages(
|
||||
self.attn_boundary, self.ffn_boundary = append_stages(
|
||||
(declare_attn(), self.input_layernorm),
|
||||
(
|
||||
declare_ffn(
|
||||
@@ -531,14 +527,6 @@ class Step3p5DecoderLayer(nn.Module):
|
||||
),
|
||||
self.post_attention_layernorm,
|
||||
),
|
||||
previous=declare_ffn(
|
||||
sparse=self.is_previous_layer_sparse,
|
||||
next_layer_sparse=self.is_moe_layer,
|
||||
)
|
||||
if not (0 if is_nextn else layer_id) == 0
|
||||
else None,
|
||||
terminal=(0 if is_nextn else layer_id)
|
||||
== (1 if is_nextn else config.num_hidden_layers) - 1,
|
||||
)
|
||||
|
||||
self.layer_id = layer_id
|
||||
|
||||
@@ -6,6 +6,7 @@ import torch
|
||||
import torch.nn as nn
|
||||
from transformers import PretrainedConfig
|
||||
|
||||
from sglang.srt.layers.layer_boundary import layer_stack
|
||||
from sglang.srt.layers.layer_boundary.residual import batch as residual_batch
|
||||
from sglang.srt.layers.layernorm import GemmaRMSNorm
|
||||
from sglang.srt.layers.logits_processor import LogitsProcessor
|
||||
@@ -79,12 +80,12 @@ class Step3p5AMultiTokenPredictor(nn.Module):
|
||||
self.hnorm = GemmaRMSNorm(config.hidden_size, config.rms_norm_eps)
|
||||
self.eh_proj = nn.Linear(config.hidden_size * 2, config.hidden_size, bias=False)
|
||||
self.shared_head = SharedHead(config=config, quant_config=quant_config)
|
||||
self.mtp_block = Step3p5DecoderLayer(
|
||||
config=config,
|
||||
layer_id=layer_id,
|
||||
prefix=f"{prefix}.mtp_block",
|
||||
is_nextn=True,
|
||||
)
|
||||
with layer_stack():
|
||||
self.mtp_block = Step3p5DecoderLayer(
|
||||
config=config,
|
||||
layer_id=layer_id,
|
||||
prefix=f"{prefix}.mtp_block",
|
||||
)
|
||||
self.lm_head = self.shared_head.head
|
||||
|
||||
def forward(
|
||||
|
||||
@@ -41,10 +41,10 @@ from sglang.srt.eplb.expert_location_dispatch import ExpertLocationDispatchInfo
|
||||
from sglang.srt.layers.activation import SiluAndMul
|
||||
from sglang.srt.layers.dp_attention import is_dp_attention_enabled
|
||||
from sglang.srt.layers.layer_boundary import (
|
||||
append_stages,
|
||||
declare_attn,
|
||||
declare_ffn,
|
||||
is_dense_ffn_fully_dp,
|
||||
make_stages,
|
||||
)
|
||||
from sglang.srt.layers.layer_boundary.residual import batch as residual_batch
|
||||
from sglang.srt.layers.layernorm import RMSNorm
|
||||
@@ -1495,7 +1495,6 @@ class XllmDecoderLayer(nn.Module):
|
||||
config.num_experts > 0 and (lid + 1) % decoder_sparse_step == 0
|
||||
)
|
||||
|
||||
is_previous_layer_sparse = _is_sparse(layer_id - 1)
|
||||
is_next_layer_sparse = _is_sparse(layer_id + 1)
|
||||
|
||||
if self.is_layer_sparse:
|
||||
@@ -1523,7 +1522,7 @@ class XllmDecoderLayer(nn.Module):
|
||||
|
||||
self.input_layernorm = _make_norm(config)
|
||||
self.post_attention_layernorm = _make_norm(config)
|
||||
self.attn_boundary, self.ffn_boundary = make_stages(
|
||||
self.attn_boundary, self.ffn_boundary = append_stages(
|
||||
(declare_attn(), self.input_layernorm),
|
||||
(
|
||||
declare_ffn(
|
||||
@@ -1532,12 +1531,6 @@ class XllmDecoderLayer(nn.Module):
|
||||
),
|
||||
self.post_attention_layernorm,
|
||||
),
|
||||
previous=declare_ffn(
|
||||
sparse=is_previous_layer_sparse, next_layer_sparse=self.is_layer_sparse
|
||||
)
|
||||
if layer_id != 0
|
||||
else None,
|
||||
terminal=layer_id == config.num_hidden_layers - 1,
|
||||
)
|
||||
|
||||
def forward(
|
||||
|
||||
@@ -53,9 +53,9 @@ from torch import nn
|
||||
|
||||
from sglang.srt.configs.zaya import ZayaConfig
|
||||
from sglang.srt.layers.layer_boundary import (
|
||||
append_stages,
|
||||
declare_attn,
|
||||
declare_ffn,
|
||||
make_stages,
|
||||
)
|
||||
from sglang.srt.layers.layer_boundary.residual import batch as residual_batch
|
||||
from sglang.srt.layers.layer_boundary.residual.add_norm import PLAIN_ADD
|
||||
@@ -1320,12 +1320,8 @@ class ZayaDecoderATTLayer(nn.Module):
|
||||
self.res_scale = ResidualScaling(config, layer_id)
|
||||
else:
|
||||
self.res_scale = None
|
||||
(self.attn_boundary,) = make_stages(
|
||||
(self.attn_boundary,) = append_stages(
|
||||
(declare_attn(read=_ResidualMergeRead(self.res_scale)), self.input_norm),
|
||||
previous=declare_ffn(sparse=True, next_layer_sparse=True)
|
||||
if layer_id != 0
|
||||
else None,
|
||||
terminal=layer_id == config.num_hidden_layers - 1,
|
||||
)
|
||||
self.entry_boundary = self.attn_boundary
|
||||
|
||||
@@ -1375,7 +1371,7 @@ class ZayaDecoderMLPLayer(nn.Module):
|
||||
self.res_scale = ResidualScaling(config, layer_id)
|
||||
else:
|
||||
self.res_scale = None
|
||||
(self.ffn_boundary,) = make_stages(
|
||||
(self.ffn_boundary,) = append_stages(
|
||||
(
|
||||
declare_ffn(
|
||||
sparse=True,
|
||||
@@ -1384,8 +1380,6 @@ class ZayaDecoderMLPLayer(nn.Module):
|
||||
),
|
||||
self.input_norm,
|
||||
),
|
||||
previous=declare_attn(),
|
||||
terminal=layer_id == config.num_hidden_layers - 1,
|
||||
)
|
||||
self.entry_boundary = self.ffn_boundary
|
||||
|
||||
|
||||
@@ -1542,6 +1542,33 @@ def mark_end(name):
|
||||
time_infos[name].pretty_print()
|
||||
|
||||
|
||||
# Set while a layer another pipeline stage holds is built, only for the stage
|
||||
# boundaries it declares.
|
||||
_building_neighbour_layer = False
|
||||
|
||||
|
||||
def is_building_neighbour_layer() -> bool:
|
||||
"""Whether the layer under construction belongs to another pipeline stage
|
||||
and is built here only to read the stage boundaries it declares. It is
|
||||
built on the meta device and never loaded or run, so its constructor skips
|
||||
the host and device resources a running layer needs: tables, streams,
|
||||
engines and communicators."""
|
||||
return _building_neighbour_layer
|
||||
|
||||
|
||||
@contextmanager
|
||||
def building_neighbour_layer():
|
||||
"""Build a pipeline neighbour layer: on the meta device, with
|
||||
is_building_neighbour_layer() true."""
|
||||
global _building_neighbour_layer
|
||||
outer, _building_neighbour_layer = _building_neighbour_layer, True
|
||||
try:
|
||||
with torch.device("meta"):
|
||||
yield
|
||||
finally:
|
||||
_building_neighbour_layer = outer
|
||||
|
||||
|
||||
class LayerFn(Protocol):
|
||||
def __call__(self, idx: int, prefix: str) -> torch.nn.Module: ...
|
||||
|
||||
@@ -1555,9 +1582,16 @@ def make_layers(
|
||||
return_tuple: bool = False,
|
||||
offloader_kwargs: Optional[Dict[str, Any]] = None,
|
||||
) -> Tuple[torch.nn.Module, int, int]:
|
||||
"""Make a list of layers with the given layer function"""
|
||||
"""Make a list of layers with the given layer function.
|
||||
|
||||
The local layers are built inside one layer stack, so layers that declare
|
||||
stage boundaries connect in order without naming their neighbours. Across
|
||||
a pipeline stage boundary the stack learns the neighbouring stage from the
|
||||
layer itself, built again on the meta device.
|
||||
"""
|
||||
# circular imports
|
||||
from sglang.srt.distributed import get_pp_indices
|
||||
from sglang.srt.layers.layer_boundary.factories import layer_stack
|
||||
from sglang.srt.layers.utils import PPMissingLayer
|
||||
from sglang.srt.utils.offloader import get_offloader
|
||||
|
||||
@@ -1571,20 +1605,30 @@ def make_layers(
|
||||
if pp_rank is not None and pp_size is not None
|
||||
else (0, num_hidden_layers)
|
||||
)
|
||||
modules = torch.nn.ModuleList(
|
||||
[PPMissingLayer(return_tuple=return_tuple) for _ in range(start_layer)]
|
||||
+ get_offloader().wrap_modules(
|
||||
(
|
||||
layer_fn(idx=idx, prefix=add_prefix(idx, prefix))
|
||||
for idx in range(start_layer, end_layer)
|
||||
),
|
||||
**(offloader_kwargs or {}),
|
||||
|
||||
def neighbour(idx):
|
||||
return functools.partial(
|
||||
_build_neighbour_layer, layer_fn, idx, add_prefix(idx, prefix)
|
||||
)
|
||||
|
||||
with layer_stack(
|
||||
previous_layers=[neighbour(idx) for idx in reversed(range(start_layer))],
|
||||
next_layers=[neighbour(idx) for idx in range(end_layer, num_hidden_layers)],
|
||||
):
|
||||
modules = torch.nn.ModuleList(
|
||||
[PPMissingLayer(return_tuple=return_tuple) for _ in range(start_layer)]
|
||||
+ get_offloader().wrap_modules(
|
||||
(
|
||||
layer_fn(idx=idx, prefix=add_prefix(idx, prefix))
|
||||
for idx in range(start_layer, end_layer)
|
||||
),
|
||||
**(offloader_kwargs or {}),
|
||||
)
|
||||
+ [
|
||||
PPMissingLayer(return_tuple=return_tuple)
|
||||
for _ in range(end_layer, num_hidden_layers)
|
||||
]
|
||||
)
|
||||
+ [
|
||||
PPMissingLayer(return_tuple=return_tuple)
|
||||
for _ in range(end_layer, num_hidden_layers)
|
||||
]
|
||||
)
|
||||
if pp_rank is None or pp_size is None:
|
||||
return modules
|
||||
return modules, start_layer, end_layer
|
||||
@@ -1613,6 +1657,21 @@ def make_pp_layers(
|
||||
)
|
||||
|
||||
|
||||
def _build_neighbour_layer(layer_fn: LayerFn, idx: int, prefix: str) -> None:
|
||||
"""Build a layer another pipeline stage holds, only for the stage
|
||||
boundaries it declares (see building_neighbour_layer). RoPE modules it
|
||||
adds to the shared cache are meta, so they are dropped again."""
|
||||
from sglang.srt.layers.rotary_embedding.factory import _ROPE_DICT
|
||||
|
||||
cached = set(_ROPE_DICT)
|
||||
try:
|
||||
with building_neighbour_layer():
|
||||
layer_fn(idx=idx, prefix=prefix)
|
||||
finally:
|
||||
for key in set(_ROPE_DICT) - cached:
|
||||
del _ROPE_DICT[key]
|
||||
|
||||
|
||||
def set_random_seed(seed: int) -> None:
|
||||
"""Set the random seed for all libraries."""
|
||||
random.seed(seed)
|
||||
|
||||
@@ -4,15 +4,36 @@ from types import SimpleNamespace
|
||||
|
||||
from sglang.srt.layers.layer_boundary.construction import StagePlan
|
||||
from sglang.srt.layers.layer_boundary.factories import (
|
||||
append_stages,
|
||||
declare_attn,
|
||||
declare_ffn,
|
||||
make_stages,
|
||||
layer_stack,
|
||||
)
|
||||
from sglang.srt.layers.layer_boundary.ops import keep_output
|
||||
from sglang.srt.layers.layer_boundary.residual.add_norm import PLAIN_RESIDUAL_OPS
|
||||
from sglang.srt.layers.layer_boundary.stage import StageBoundary
|
||||
|
||||
|
||||
def build_stages(*stages, previous=None, terminal=False):
|
||||
"""Bind one sequence of stages in a layer stack of its own.
|
||||
|
||||
Args:
|
||||
*stages: Items of (declaration, norm) or (declaration, norm, options),
|
||||
as for append_stages.
|
||||
previous: Declaration of the stage before the first one, standing for
|
||||
a layer the test does not build; None starts the stack.
|
||||
terminal: Whether the last stage ends the model's layer stack;
|
||||
otherwise the next layer's attention follows it.
|
||||
"""
|
||||
with layer_stack(
|
||||
previous_layers=(
|
||||
[lambda: append_stages((previous, None))] if previous is not None else []
|
||||
),
|
||||
next_layers=[] if terminal else [lambda: append_stages((declare_attn(), None))],
|
||||
):
|
||||
return append_stages(*stages)
|
||||
|
||||
|
||||
def make_test_stages(
|
||||
*,
|
||||
attention_norm,
|
||||
@@ -42,7 +63,7 @@ def make_test_stages(
|
||||
output_transform=output,
|
||||
)
|
||||
common = {key: options.pop(key) for key in ("fusions",) if key in options}
|
||||
attn, ffn = make_stages(
|
||||
attn, ffn = build_stages(
|
||||
(
|
||||
attention,
|
||||
attention_norm,
|
||||
|
||||
Reference in New Issue
Block a user