[Refactor] Declare pipeline neighbours from a shared declaration function (#43459)
This commit is contained in:
@@ -24,7 +24,6 @@ from sglang.srt.layers.layer_boundary.boundary import (
|
||||
)
|
||||
from sglang.srt.layers.layer_boundary.construction import (
|
||||
BatchVariant,
|
||||
VariantEdges,
|
||||
)
|
||||
from sglang.srt.layers.layer_boundary.contracts import (
|
||||
EdgeContract,
|
||||
@@ -35,11 +34,10 @@ from sglang.srt.layers.layer_boundary.contracts import (
|
||||
OutputContract,
|
||||
ProducerReduction,
|
||||
ReadoutFusion,
|
||||
StageContract,
|
||||
StageKind,
|
||||
StagePath,
|
||||
)
|
||||
from sglang.srt.layers.layer_boundary.exit import ExitDecision, FfnExit, MixerExit
|
||||
from sglang.srt.layers.layer_boundary.exit import MixerExit
|
||||
from sglang.srt.layers.layer_boundary.factories import (
|
||||
append_stages,
|
||||
declare_attn,
|
||||
@@ -52,8 +50,6 @@ from sglang.srt.layers.layer_boundary.layout import (
|
||||
TokenAxis,
|
||||
batch_gathers_over_moe_cp,
|
||||
is_dense_ffn_fully_dp,
|
||||
moe_gathers_over_moe_cp,
|
||||
token_axis_sizes,
|
||||
)
|
||||
from sglang.srt.layers.layer_boundary.ops import (
|
||||
move_rows,
|
||||
@@ -64,11 +60,9 @@ from sglang.srt.layers.layer_boundary.output import (
|
||||
UnreducedOutput,
|
||||
complete_owed,
|
||||
)
|
||||
from sglang.srt.layers.layer_boundary.residual import LayerResidualOps
|
||||
from sglang.srt.layers.layer_boundary.residual.add_norm import (
|
||||
FUSE_ALLREDUCE_MAX_BATCH_SIZE,
|
||||
NORM_QUANT_READOUT,
|
||||
NORM_READOUT,
|
||||
PLAIN_ADD,
|
||||
PLAIN_RESIDUAL_OPS,
|
||||
)
|
||||
@@ -94,21 +88,16 @@ __all__ = [
|
||||
"ExitRows",
|
||||
"ProducerReduction",
|
||||
"FUSE_ALLREDUCE_MAX_BATCH_SIZE",
|
||||
"ExitDecision",
|
||||
"FfnExit",
|
||||
"FfnInputFusion",
|
||||
"ReadoutFusion",
|
||||
"DeferredFinalize",
|
||||
"GatedResidualState",
|
||||
"IHCState",
|
||||
"LayerResidualOps",
|
||||
"Layout",
|
||||
"MHCState",
|
||||
"MixerExit",
|
||||
"NORM_QUANT_READOUT",
|
||||
"NORM_READOUT",
|
||||
"PLAIN_RESIDUAL_OPS",
|
||||
"StageContract",
|
||||
"EntryPath",
|
||||
"InputContract",
|
||||
"StageKind",
|
||||
@@ -123,8 +112,6 @@ __all__ = [
|
||||
"batch_gathers_over_moe_cp",
|
||||
"move_rows",
|
||||
"complete_owed",
|
||||
"moe_gathers_over_moe_cp",
|
||||
"tbo_split_moves",
|
||||
"token_axis_sizes",
|
||||
"tp_reduce_scatter",
|
||||
]
|
||||
|
||||
@@ -235,9 +235,6 @@ class StagePlan:
|
||||
return _bound_for(self._next_input_rows, self.variant_for(forward_batch))
|
||||
return self.path_for(forward_batch).entry.input_rows
|
||||
|
||||
def produced(self, forward_batch):
|
||||
return self.path_for(forward_batch).output
|
||||
|
||||
@cached_property
|
||||
def output(self):
|
||||
return ExitPolicy(self)
|
||||
|
||||
@@ -27,7 +27,9 @@ from sglang.srt.layers.layer_boundary.residual.add_norm import NORM_READOUT, PLA
|
||||
|
||||
|
||||
class ProducerReduction(Enum):
|
||||
"""Who completes the sum a stage's output owes; compute never does.
|
||||
"""Who completes the sum a stage's output owes: exactly one completer for
|
||||
each output, the boundary. Compute completes it only where an FFN fuses
|
||||
it with a reduction its computation needs (``output_complete``).
|
||||
|
||||
ALWAYS_PARTIAL is attention-only: finish() hands the partial sum to the
|
||||
next stage's input as its declared sum. EXIT_SCOPED, for an FFN or a
|
||||
|
||||
@@ -5,9 +5,9 @@ from __future__ import annotations
|
||||
import contextlib
|
||||
import os
|
||||
import sys
|
||||
from dataclasses import dataclass, replace
|
||||
from dataclasses import dataclass, fields, is_dataclass, replace
|
||||
from types import MappingProxyType
|
||||
from typing import Callable, Mapping, NamedTuple, Optional
|
||||
from typing import Callable, Mapping, NamedTuple, Optional, Tuple
|
||||
|
||||
from sglang.srt.environ import envs
|
||||
from sglang.srt.layers import layernorm_sp
|
||||
@@ -29,6 +29,7 @@ from sglang.srt.layers.layer_boundary.contracts import (
|
||||
StageContract,
|
||||
StageKind,
|
||||
)
|
||||
from sglang.srt.layers.layer_boundary.facts import facts_of
|
||||
from sglang.srt.layers.layer_boundary.layout import (
|
||||
Layout,
|
||||
SumGroup,
|
||||
@@ -301,11 +302,14 @@ class StageDeclaration:
|
||||
contiguous slice of the rows, it returns them all, in rank order,
|
||||
or None to leave the gather to the boundary. Called on every batch,
|
||||
so it must be CUDA-graph safe. Across a pipeline boundary the
|
||||
producer's rank takes it from the neighbouring layer it builds on
|
||||
the meta device, so it may use only the communication state of the
|
||||
rank it runs on, not the declaring layer's weights or buffers.
|
||||
previous: Declaration whose output this stage consumes, as the stack
|
||||
records it; across pipeline ranks it is built locally.
|
||||
producer's rank takes it from the neighbouring stage's declaration
|
||||
(the model's shared declaration function, or, for a model on the
|
||||
transitional list, the layer built again on the meta device), so it
|
||||
may use only the communication state of the rank it runs on, not
|
||||
the declaring layer's weights or buffers.
|
||||
previous: The stage whose output this stage consumes, as the stack
|
||||
records it: only the facts binding reads of it (see facts_of),
|
||||
on this rank or another.
|
||||
prepared_from: Declaration whose already-read input a branch reuses.
|
||||
Mutually exclusive with previous; avoids a second update/read.
|
||||
|
||||
@@ -738,6 +742,19 @@ def _connect_line(line, origins, *, before=None, after=None, arrivals=None):
|
||||
if arrivals is None:
|
||||
arrivals = {v: _stack_arrival(v) for v in variants}
|
||||
if before is not None:
|
||||
if (
|
||||
before.output_transform is not None
|
||||
and before.kind is StageKind.ATTENTION
|
||||
and before.reduction is ProducerReduction.ALWAYS_PARTIAL
|
||||
):
|
||||
# Its transform runs at the next stage's input, which would be
|
||||
# on this rank, without the module that declares it.
|
||||
error = NotImplementedError(
|
||||
"a pipeline rank that ends on an attention transforming the "
|
||||
"output whose sum it leaves"
|
||||
)
|
||||
_note_origin(error, origins[0])
|
||||
raise error
|
||||
flow = resolve(0, before, arrivals, line[0])
|
||||
arrivals = {v: _arrival(before, flow[v], v) for v in variants}
|
||||
flows = []
|
||||
@@ -830,11 +847,14 @@ def layer_stack(*, previous_layers=(), next_layers=(), final_read=None):
|
||||
stage ends the model's layer stack unless a later layer declares a stage.
|
||||
|
||||
Args:
|
||||
previous_layers: Callables that build, nearest first, the layers before
|
||||
this stack that another pipeline rank holds. Called only if this
|
||||
stack appended stages, after its own layers are built, until one of
|
||||
them declares a stage: its last stage is the producer of this
|
||||
stack's first. What they build is discarded.
|
||||
previous_layers: For the layers before this stack that another
|
||||
pipeline rank holds, nearest first, callables that return the
|
||||
stages the layer declares as DeclaredStages (from the model's
|
||||
shared declaration function), or else build the layer so that it
|
||||
appends them. Called only if this stack appended stages, after its
|
||||
own layers are built, until one of them declares a stage: its last
|
||||
stage is the producer of this stack's first. Only what binding
|
||||
reads of that stage is kept (see facts_of).
|
||||
next_layers: Likewise for the layers after this stack, whose first
|
||||
declared stage is the consumer of this stack's last.
|
||||
final_read: The model's final read of the stack's output (a
|
||||
@@ -861,21 +881,94 @@ def layer_stack(*, previous_layers=(), next_layers=(), final_read=None):
|
||||
_stack = outer
|
||||
|
||||
|
||||
def _neighbour_stage(build_layers, *, last):
|
||||
class DeclaredStages(NamedTuple):
|
||||
"""The stages a neighbouring layer declares, from the model's shared
|
||||
declaration function, which stand in for building the layer."""
|
||||
|
||||
stages: Tuple[StageDeclaration, ...]
|
||||
|
||||
|
||||
def check_declared_stages(stack, appended: int, expected, where: str) -> None:
|
||||
"""Check that what a layer appended to ``stack`` since it held
|
||||
``appended`` appends is what its shared declaration function says it
|
||||
declares, as far as binding reads it: another pipeline rank binds this
|
||||
layer's stages from that function alone."""
|
||||
declared = [
|
||||
facts_of(declaration)
|
||||
for append in stack.appends[appended:]
|
||||
if append.prepared_from is None
|
||||
for declaration in append.declarations
|
||||
]
|
||||
expected = [facts_of(declaration) for declaration in expected]
|
||||
if declared != expected:
|
||||
raise ValueError(
|
||||
f"{where} declares stages its shared declaration function does "
|
||||
f"not: {_first_difference(declared, expected)}"
|
||||
)
|
||||
|
||||
|
||||
def _first_difference(declared, expected) -> str:
|
||||
"""Where the stages a layer declares first differ from those its shared
|
||||
declaration function gives, field by field."""
|
||||
if len(declared) != len(expected):
|
||||
return (
|
||||
f"it declares {[stage.kind.name for stage in declared]}, the "
|
||||
f"function {[stage.kind.name for stage in expected]}"
|
||||
)
|
||||
index, (stage, given) = next(
|
||||
(index, pair)
|
||||
for index, pair in enumerate(zip(declared, expected))
|
||||
if pair[0] != pair[1]
|
||||
)
|
||||
return f"stage {index} ({stage.kind.name}): " + "; ".join(
|
||||
f"{name} is {value!r}, the function says {given_value!r}"
|
||||
for name, value, given_value in _differing_fields(stage, given)
|
||||
)
|
||||
|
||||
|
||||
def _differing_fields(value, given, prefix=""):
|
||||
"""(dotted name, value, given value) for each leaf field that differs,
|
||||
descending into declarations and the facts they hold."""
|
||||
if is_dataclass(value):
|
||||
names = [field.name for field in fields(value)]
|
||||
else:
|
||||
names = getattr(type(value), "__struct_fields__", None)
|
||||
if names is None or type(value) is not type(given):
|
||||
return [(prefix.rstrip("."), value, given)]
|
||||
return [
|
||||
difference
|
||||
for name in names
|
||||
if getattr(value, name) != getattr(given, name)
|
||||
for difference in _differing_fields(
|
||||
getattr(value, name), getattr(given, name), f"{prefix}{name}."
|
||||
)
|
||||
]
|
||||
|
||||
|
||||
def _neighbour_stage(layers, *, last):
|
||||
"""The stage a neighbouring layer declares next to this stack: the last
|
||||
one of the nearest layer before it, or the first of the nearest after.
|
||||
Branches are side paths, so they never stand next to the stack."""
|
||||
global _stack
|
||||
for build_layer in build_layers:
|
||||
for layer in layers:
|
||||
outer = _stack
|
||||
_stack = _LayerStack()
|
||||
try:
|
||||
build_layer()
|
||||
appends = [a for a in _stack.appends if a.prepared_from is None]
|
||||
result = layer()
|
||||
declared = (
|
||||
result.stages
|
||||
if isinstance(result, DeclaredStages)
|
||||
else [
|
||||
declaration
|
||||
for append in _stack.appends
|
||||
if append.prepared_from is None
|
||||
for declaration in append.declarations
|
||||
]
|
||||
)
|
||||
finally:
|
||||
_stack = outer
|
||||
if appends:
|
||||
return appends[-1].declarations[-1] if last else appends[0].declarations[0]
|
||||
if declared:
|
||||
return facts_of(declared[-1] if last else declared[0])
|
||||
return None
|
||||
|
||||
|
||||
@@ -938,15 +1031,6 @@ def _return_before_trailing_attention(line):
|
||||
return
|
||||
|
||||
|
||||
def _detached(declaration):
|
||||
"""The declaration as the stage after it records it: without its own
|
||||
history, so a bound stage holds its producer's facts and not the chain of
|
||||
declarations before it."""
|
||||
if declaration is None:
|
||||
return None
|
||||
return replace(declaration, previous=None, prepared_from=None)
|
||||
|
||||
|
||||
def _bind_stack(appends, *, previous, following, final_read=None):
|
||||
"""Bind every appended stage and fill in the boundaries each append
|
||||
returned.
|
||||
@@ -973,13 +1057,13 @@ def _bind_stack(appends, *, previous, following, final_read=None):
|
||||
bound = {id(append): [None] * len(append.bindings) for append in appends}
|
||||
# A returned declaration's stage as bound, for the branches that start from it.
|
||||
sources = {}
|
||||
producer = _detached(_handed_off(previous))
|
||||
producer = facts_of(_handed_off(previous))
|
||||
chained = []
|
||||
for append, index in stages:
|
||||
chained.append(
|
||||
replace(append.declarations[index], previous=producer, prepared_from=None)
|
||||
)
|
||||
producer = _detached(chained[-1])
|
||||
producer = facts_of(chained[-1])
|
||||
connections = _connect_line(
|
||||
chained,
|
||||
[append.origin for append, _ in stages],
|
||||
@@ -1010,8 +1094,8 @@ def _bind_stack(appends, *, previous, following, final_read=None):
|
||||
chained.append(
|
||||
replace(
|
||||
declaration,
|
||||
previous=chained[-1] if chained else None,
|
||||
prepared_from=None if chained else source_declaration,
|
||||
previous=facts_of(chained[-1]) if chained else None,
|
||||
prepared_from=None if chained else facts_of(source_declaration),
|
||||
)
|
||||
)
|
||||
try:
|
||||
|
||||
@@ -0,0 +1,99 @@
|
||||
# Copyright 2023-2024 SGLang Team
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
# ==============================================================================
|
||||
"""What binding reads of a stage it does not run: the stage before or after
|
||||
a stack on another pipeline rank, and the producer a bound stage records.
|
||||
|
||||
A stage's read, update and output transform are objects that hold the
|
||||
layer's modules and state. Binding a neighbouring stage reads only the
|
||||
capabilities they declare, so a neighbour, and the producer a stage keeps,
|
||||
is recorded as those values alone: no module, bound method or declaration
|
||||
history stays referenced once the stack is bound."""
|
||||
|
||||
from typing import Optional
|
||||
|
||||
import msgspec
|
||||
|
||||
|
||||
class UpdateFacts(msgspec.Struct, frozen=True):
|
||||
"""The capabilities a ResidualUpdate declares (see ResidualUpdate)."""
|
||||
|
||||
is_plain_add: bool
|
||||
applied_at_exit: bool
|
||||
outlives_layer: bool
|
||||
writes_stream: bool
|
||||
quantized_sum: bool = False
|
||||
|
||||
@classmethod
|
||||
def of(cls, update) -> "UpdateFacts":
|
||||
if isinstance(update, cls):
|
||||
return update
|
||||
return cls(
|
||||
is_plain_add=update.is_plain_add,
|
||||
applied_at_exit=update.applied_at_exit,
|
||||
outlives_layer=update.outlives_layer,
|
||||
writes_stream=update.writes_stream,
|
||||
quantized_sum=update.quantized_sum,
|
||||
)
|
||||
|
||||
|
||||
class ReadFacts(msgspec.Struct, frozen=True):
|
||||
"""The capabilities a ResidualReadout declares (see ResidualReadout).
|
||||
Binding takes kernels only from the reads of the stages it binds, never
|
||||
from a stage recorded by its facts, so these supply none."""
|
||||
|
||||
is_plain_norm: bool
|
||||
reads_before_dp_gather: bool
|
||||
reads_after_attn_tp_gather: bool
|
||||
completing_fusions = ()
|
||||
gathering_reads = ()
|
||||
|
||||
@classmethod
|
||||
def of(cls, read) -> "ReadFacts":
|
||||
if isinstance(read, cls):
|
||||
return read
|
||||
return cls(
|
||||
is_plain_norm=read.is_plain_norm,
|
||||
reads_before_dp_gather=read.reads_before_dp_gather,
|
||||
reads_after_attn_tp_gather=read.reads_after_attn_tp_gather,
|
||||
)
|
||||
|
||||
|
||||
class TransformFacts(msgspec.Struct, frozen=True):
|
||||
"""What an OutputTransform declares (see OutputTransform)."""
|
||||
|
||||
before_reduce_scatter: bool = False
|
||||
|
||||
@classmethod
|
||||
def of(cls, transform) -> Optional["TransformFacts"]:
|
||||
if transform is None or isinstance(transform, cls):
|
||||
return transform
|
||||
return cls(before_reduce_scatter=transform.before_reduce_scatter)
|
||||
|
||||
|
||||
def facts_of(declaration):
|
||||
"""``declaration`` with its read, update and output transform reduced to
|
||||
the capabilities they declare, and without the declarations it was
|
||||
chained to. None stays None."""
|
||||
if declaration is None:
|
||||
return None
|
||||
from dataclasses import replace
|
||||
|
||||
return replace(
|
||||
declaration,
|
||||
read=ReadFacts.of(declaration.read),
|
||||
update=UpdateFacts.of(declaration.update),
|
||||
output_transform=TransformFacts.of(declaration.output_transform),
|
||||
previous=None,
|
||||
prepared_from=None,
|
||||
)
|
||||
@@ -335,7 +335,6 @@ def keep_output(
|
||||
hidden_states: torch.Tensor,
|
||||
residual: torch.Tensor,
|
||||
forward_batch: ForwardBatch,
|
||||
**kwargs,
|
||||
):
|
||||
return hidden_states, residual
|
||||
|
||||
@@ -346,7 +345,6 @@ def update_attn_tp_gather_output(
|
||||
forward_batch: ForwardBatch,
|
||||
update: ResidualUpdate = PLAIN_ADD,
|
||||
gather: Optional[Callable] = None,
|
||||
**kwargs,
|
||||
):
|
||||
"""Write the output into the residual on this rank's slice, then gather it
|
||||
over attention TP (see attn_tp_gather_with)."""
|
||||
@@ -371,7 +369,6 @@ def residual_slice_output(
|
||||
sums: bool,
|
||||
gathers_back: bool,
|
||||
update: ResidualUpdate,
|
||||
**kwargs,
|
||||
):
|
||||
"""Bring the FFN output onto the slice of the rows the residual is on:
|
||||
a reduce-scatter that also completes its sum when ``sums``, else this
|
||||
@@ -405,7 +402,6 @@ def attn_cp_take_back_output(
|
||||
hidden_states: torch.Tensor,
|
||||
residual: torch.Tensor,
|
||||
forward_batch: ForwardBatch,
|
||||
**kwargs,
|
||||
):
|
||||
"""DSA and MLA CP: this rank's shard of a complete output gathered in
|
||||
equal shards over the attention-CP group."""
|
||||
@@ -418,7 +414,6 @@ def attn_cp_reduce_scatter_output(
|
||||
hidden_states: torch.Tensor,
|
||||
residual: torch.Tensor,
|
||||
forward_batch: ForwardBatch,
|
||||
**kwargs,
|
||||
):
|
||||
"""DSA and MLA CP: sum the FFN output over the attention-CP group and
|
||||
keep this rank's shard."""
|
||||
@@ -429,7 +424,6 @@ def dp_cp_take_back_output(
|
||||
hidden_states: torch.Tensor,
|
||||
residual: torch.Tensor,
|
||||
forward_batch: ForwardBatch,
|
||||
**kwargs,
|
||||
):
|
||||
"""This rank's CP shard of the rows the DP gather put in its DP group's
|
||||
slot, at the shard's padded length with the padding zeroed."""
|
||||
@@ -450,7 +444,6 @@ def moe_cp_take_back_output(
|
||||
hidden_states: torch.Tensor,
|
||||
residual: torch.Tensor,
|
||||
forward_batch: ForwardBatch,
|
||||
**kwargs,
|
||||
):
|
||||
"""Return a MoE output computed on the MoE-CP-gathered rows to this rank's attention rows.
|
||||
|
||||
|
||||
@@ -174,7 +174,7 @@ class StageBoundary:
|
||||
completed or carried on. An FFN whose boundary has fusions (which may
|
||||
take a MoE finalize handoff) finishes through exit() instead."""
|
||||
if self.kind is StageKind.ATTENTION and self.plan.finishes_directly:
|
||||
produced = self.plan.produced(forward_batch)
|
||||
produced = self.plan.path_for(forward_batch).output
|
||||
return stream_of(forward_batch).record(
|
||||
hidden_states,
|
||||
produced.update,
|
||||
@@ -238,9 +238,6 @@ class StageBoundary:
|
||||
)
|
||||
return hidden_states
|
||||
|
||||
def snapshot(self, hidden_states, forward_batch):
|
||||
return stream_of(forward_batch).snapshot(hidden_states)
|
||||
|
||||
def capture_output(self, hidden_states, forward_batch, *, skip_empty=False):
|
||||
"""Capture an output while preserving the caller's reduction timing.
|
||||
|
||||
|
||||
@@ -607,7 +607,7 @@ class LongcatFlashModel(nn.Module):
|
||||
for i in range(total_num_layers):
|
||||
if i in self.layers_to_capture:
|
||||
aux_hidden_states.append(
|
||||
self.layers[i].attn_boundary.snapshot(hidden_states, forward_batch)
|
||||
residual_batch.snapshot(hidden_states, forward_batch)
|
||||
)
|
||||
with get_global_expert_distribution_recorder().with_current_layer(i):
|
||||
layer = self.layers[i]
|
||||
|
||||
@@ -18,6 +18,7 @@
|
||||
"""Inference-only NemotronH model."""
|
||||
|
||||
from collections.abc import Iterable
|
||||
from functools import partial
|
||||
|
||||
import torch
|
||||
from torch import nn
|
||||
@@ -77,7 +78,7 @@ from sglang.srt.model_loader.weight_utils import (
|
||||
replace_prefix,
|
||||
replace_substrings,
|
||||
)
|
||||
from sglang.srt.models.nemotron_h_utils import make_stage_boundary
|
||||
from sglang.srt.models.nemotron_h_utils import make_stage_boundary, stage_facts
|
||||
from sglang.srt.models.utils import WeightsMapper
|
||||
from sglang.srt.runtime_context import get_exec, get_parallel
|
||||
from sglang.srt.utils import (
|
||||
@@ -690,6 +691,7 @@ class NemotronHModel(nn.Module):
|
||||
len(config.hybrid_override_pattern),
|
||||
get_layer,
|
||||
prefix=f"{prefix}.layers",
|
||||
stage_facts=partial(stage_facts, config.hybrid_override_pattern),
|
||||
)
|
||||
if self.pp_group.is_last_rank:
|
||||
self.norm_f = RMSNorm(config.hidden_size, eps=config.layer_norm_epsilon)
|
||||
|
||||
@@ -37,15 +37,18 @@ def _declaration(pattern: str, layer_idx: int):
|
||||
)
|
||||
|
||||
|
||||
def stage_facts(pattern: str, layer_idx: int):
|
||||
"""The one stage each Nemotron-H layer declares, from the pattern alone:
|
||||
the model's shared declaration function, which its layers declare with
|
||||
too (see make_layers)."""
|
||||
return (_declaration(pattern, layer_idx),)
|
||||
|
||||
|
||||
def make_stage_boundary(layer_norm: RMSNorm, *, pattern: str, layer_idx: int):
|
||||
from sglang.srt.layers import layernorm_sp
|
||||
|
||||
if get_parallel().attn_cp_size > 1 or layernorm_sp.layernorm_sp_enabled():
|
||||
raise NotImplementedError("a Nemotron stage with attention CP or LayerNorm SP")
|
||||
(boundary,) = append_stages(
|
||||
(
|
||||
_declaration(pattern, layer_idx),
|
||||
layer_norm,
|
||||
),
|
||||
)
|
||||
(declaration,) = stage_facts(pattern, layer_idx)
|
||||
(boundary,) = append_stages((declaration, layer_norm))
|
||||
return boundary
|
||||
|
||||
@@ -1591,18 +1591,27 @@ def make_layers(
|
||||
return_tuple: bool = False,
|
||||
offloader_kwargs: Optional[Dict[str, Any]] = None,
|
||||
final_read: Optional[Any] = None,
|
||||
stage_facts: Optional[Callable[[int], Sequence[Any]]] = None,
|
||||
) -> Tuple[torch.nn.Module, int, int]:
|
||||
"""Make a list of layers with the given layer function.
|
||||
|
||||
The local layers are built inside one layer stack, so layers that declare
|
||||
stage boundaries connect in order without naming their neighbours. Across
|
||||
a pipeline stage boundary the stack learns the neighbouring stage from the
|
||||
layer itself, built again on the meta device. ``final_read`` is the
|
||||
stack's terminal read when it is not a plain final norm (see layer_stack).
|
||||
a pipeline stage boundary the stack learns the neighbouring stage from
|
||||
``stage_facts``: the model's shared declaration function, which returns
|
||||
the stages the layer at a global index declares without building it (an
|
||||
empty sequence when it declares none), and which the layer itself uses
|
||||
for its own declarations. A model on the transitional list may omit it;
|
||||
the stack then builds the layer again on the meta device to read them.
|
||||
``final_read`` is the stack's terminal read when it is not a plain final
|
||||
norm (see layer_stack).
|
||||
"""
|
||||
# circular imports
|
||||
from sglang.srt.distributed import get_pp_indices
|
||||
from sglang.srt.layers.layer_boundary.factories import layer_stack
|
||||
from sglang.srt.layers.layer_boundary.factories import (
|
||||
check_declared_stages,
|
||||
layer_stack,
|
||||
)
|
||||
from sglang.srt.layers.utils import PPMissingLayer
|
||||
from sglang.srt.utils.offloader import get_offloader
|
||||
|
||||
@@ -1618,22 +1627,29 @@ def make_layers(
|
||||
)
|
||||
|
||||
def neighbour(idx):
|
||||
if stage_facts is not None:
|
||||
return functools.partial(_declared_stages, stage_facts, idx)
|
||||
return functools.partial(
|
||||
_build_neighbour_layer, layer_fn, idx, add_prefix(idx, prefix)
|
||||
)
|
||||
|
||||
def build(stack, idx):
|
||||
appended = len(stack.appends)
|
||||
layer = layer_fn(idx=idx, prefix=add_prefix(idx, prefix))
|
||||
if stage_facts is not None and (pp_size or 1) > 1:
|
||||
# Another rank binds this layer's stages from stage_facts alone.
|
||||
check_declared_stages(stack, appended, stage_facts(idx), f"layer {idx}")
|
||||
return layer
|
||||
|
||||
with layer_stack(
|
||||
previous_layers=[neighbour(idx) for idx in reversed(range(start_layer))],
|
||||
next_layers=[neighbour(idx) for idx in range(end_layer, num_hidden_layers)],
|
||||
final_read=final_read,
|
||||
):
|
||||
) as stack:
|
||||
modules = torch.nn.ModuleList(
|
||||
[PPMissingLayer(return_tuple=return_tuple) for _ in range(start_layer)]
|
||||
+ get_offloader().wrap_modules(
|
||||
(
|
||||
layer_fn(idx=idx, prefix=add_prefix(idx, prefix))
|
||||
for idx in range(start_layer, end_layer)
|
||||
),
|
||||
(build(stack, idx) for idx in range(start_layer, end_layer)),
|
||||
**(offloader_kwargs or {}),
|
||||
)
|
||||
+ [
|
||||
@@ -1653,10 +1669,12 @@ def make_pp_layers(
|
||||
return_tuple: bool = False,
|
||||
offloader_kwargs: Optional[Dict[str, Any]] = None,
|
||||
final_read: Optional[Any] = None,
|
||||
stage_facts: Optional[Callable[[int], Sequence[Any]]] = None,
|
||||
) -> Tuple[torch.nn.Module, int, int]:
|
||||
"""Make this pipeline stage's layers, and return them with the stage's range.
|
||||
|
||||
Layers outside ``[start_layer, end_layer)`` are ``PPMissingLayer`` stand-ins.
|
||||
``stage_facts`` is the model's shared declaration function (see make_layers).
|
||||
"""
|
||||
parallel = get_parallel()
|
||||
return make_layers(
|
||||
@@ -1668,15 +1686,81 @@ def make_pp_layers(
|
||||
return_tuple=return_tuple,
|
||||
offloader_kwargs=offloader_kwargs,
|
||||
final_read=final_read,
|
||||
stage_facts=stage_facts,
|
||||
)
|
||||
|
||||
|
||||
# The models whose layers a pipeline rank still builds on the meta device to
|
||||
# read the stages they declare, until each gives make_layers its shared
|
||||
# declaration function (stage_facts). Any other model must give one to run
|
||||
# under pipeline parallelism.
|
||||
_NEIGHBOUR_BUILD_MODULES = frozenset(
|
||||
(
|
||||
"sglang.srt.models.apertus",
|
||||
"sglang.srt.models.arcee",
|
||||
"sglang.srt.models.bailing_moe",
|
||||
"sglang.srt.models.bailing_moe_linear",
|
||||
"sglang.srt.models.bailing_moe_v3",
|
||||
"sglang.srt.models.deepseek_v2",
|
||||
"sglang.srt.models.dots3_common.modeling",
|
||||
"sglang.srt.models.ernie45_moe_vl",
|
||||
"sglang.srt.models.exaone4",
|
||||
"sglang.srt.models.exaone_moe",
|
||||
"sglang.srt.models.gemma4_causal",
|
||||
"sglang.srt.models.gigachat35",
|
||||
"sglang.srt.models.glm4",
|
||||
"sglang.srt.models.glm4_moe",
|
||||
"sglang.srt.models.glm4_moe_lite",
|
||||
"sglang.srt.models.glm5_next",
|
||||
"sglang.srt.models.gpt_oss",
|
||||
"sglang.srt.models.granitemoehybrid",
|
||||
"sglang.srt.models.kimi_k3",
|
||||
"sglang.srt.models.kimi_linear",
|
||||
"sglang.srt.models.laguna",
|
||||
"sglang.srt.models.llada2",
|
||||
"sglang.srt.models.llama",
|
||||
"sglang.srt.models.mimo_v2",
|
||||
"sglang.srt.models.minimax_m2",
|
||||
"sglang.srt.models.minimax_m3",
|
||||
"sglang.srt.models.ministral3",
|
||||
"sglang.srt.models.mixtral",
|
||||
"sglang.srt.models.nanbeige",
|
||||
"sglang.srt.models.nemotron_nas",
|
||||
"sglang.srt.models.qwen2",
|
||||
"sglang.srt.models.qwen2_moe",
|
||||
"sglang.srt.models.qwen3_5",
|
||||
"sglang.srt.models.sarvam_moe",
|
||||
"sglang.srt.models.sdar",
|
||||
"sglang.srt.models.sdar_moe",
|
||||
"sglang.srt.models.spark2_5",
|
||||
"sglang.srt.models.step3p5",
|
||||
"sglang.srt.models.xllm",
|
||||
"sglang.srt.models.zaya",
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def _declared_stages(stage_facts, idx: int):
|
||||
"""The stages the layer at ``idx`` declares, from the model's shared
|
||||
declaration function, standing in for building it."""
|
||||
from sglang.srt.layers.layer_boundary.factories import DeclaredStages
|
||||
|
||||
return DeclaredStages(tuple(stage_facts(idx)))
|
||||
|
||||
|
||||
def _build_neighbour_layer(layer_fn: LayerFn, idx: int, prefix: str) -> None:
|
||||
"""Build a layer another pipeline stage holds, only for the stage
|
||||
boundaries it declares (see building_neighbour_layer). RoPE modules it
|
||||
adds to the shared cache are meta, so they are dropped again."""
|
||||
from sglang.srt.layers.rotary_embedding.factory import _ROPE_DICT
|
||||
|
||||
module = getattr(getattr(layer_fn, "func", layer_fn), "__module__", None)
|
||||
if module not in _NEIGHBOUR_BUILD_MODULES:
|
||||
raise ValueError(
|
||||
f"{module} declares stage boundaries under pipeline parallelism "
|
||||
"without a shared declaration function: give make_layers its "
|
||||
"stage_facts, the stages a layer declares without building it"
|
||||
)
|
||||
cached = set(_ROPE_DICT)
|
||||
try:
|
||||
with building_neighbour_layer():
|
||||
|
||||
Reference in New Issue
Block a user