[Refactor] Declare pipeline neighbours from a shared declaration function (#43459)

This commit is contained in:
Cheng Wan
2026-10-10 12:42:46 -07:00
committed by GitHub
parent 3b717c0f13
commit 3b16eeedce
14 changed files with 438 additions and 85 deletions
@@ -24,7 +24,6 @@ from sglang.srt.layers.layer_boundary.boundary import (
)
from sglang.srt.layers.layer_boundary.construction import (
BatchVariant,
VariantEdges,
)
from sglang.srt.layers.layer_boundary.contracts import (
EdgeContract,
@@ -35,11 +34,10 @@ from sglang.srt.layers.layer_boundary.contracts import (
OutputContract,
ProducerReduction,
ReadoutFusion,
StageContract,
StageKind,
StagePath,
)
from sglang.srt.layers.layer_boundary.exit import ExitDecision, FfnExit, MixerExit
from sglang.srt.layers.layer_boundary.exit import MixerExit
from sglang.srt.layers.layer_boundary.factories import (
append_stages,
declare_attn,
@@ -52,8 +50,6 @@ from sglang.srt.layers.layer_boundary.layout import (
TokenAxis,
batch_gathers_over_moe_cp,
is_dense_ffn_fully_dp,
moe_gathers_over_moe_cp,
token_axis_sizes,
)
from sglang.srt.layers.layer_boundary.ops import (
move_rows,
@@ -64,11 +60,9 @@ from sglang.srt.layers.layer_boundary.output import (
UnreducedOutput,
complete_owed,
)
from sglang.srt.layers.layer_boundary.residual import LayerResidualOps
from sglang.srt.layers.layer_boundary.residual.add_norm import (
FUSE_ALLREDUCE_MAX_BATCH_SIZE,
NORM_QUANT_READOUT,
NORM_READOUT,
PLAIN_ADD,
PLAIN_RESIDUAL_OPS,
)
@@ -94,21 +88,16 @@ __all__ = [
"ExitRows",
"ProducerReduction",
"FUSE_ALLREDUCE_MAX_BATCH_SIZE",
"ExitDecision",
"FfnExit",
"FfnInputFusion",
"ReadoutFusion",
"DeferredFinalize",
"GatedResidualState",
"IHCState",
"LayerResidualOps",
"Layout",
"MHCState",
"MixerExit",
"NORM_QUANT_READOUT",
"NORM_READOUT",
"PLAIN_RESIDUAL_OPS",
"StageContract",
"EntryPath",
"InputContract",
"StageKind",
@@ -123,8 +112,6 @@ __all__ = [
"batch_gathers_over_moe_cp",
"move_rows",
"complete_owed",
"moe_gathers_over_moe_cp",
"tbo_split_moves",
"token_axis_sizes",
"tp_reduce_scatter",
]
@@ -235,9 +235,6 @@ class StagePlan:
return _bound_for(self._next_input_rows, self.variant_for(forward_batch))
return self.path_for(forward_batch).entry.input_rows
def produced(self, forward_batch):
return self.path_for(forward_batch).output
@cached_property
def output(self):
return ExitPolicy(self)
@@ -27,7 +27,9 @@ from sglang.srt.layers.layer_boundary.residual.add_norm import NORM_READOUT, PLA
class ProducerReduction(Enum):
"""Who completes the sum a stage's output owes; compute never does.
"""Who completes the sum a stage's output owes: exactly one completer for
each output, the boundary. Compute completes it only where an FFN fuses
it with a reduction its computation needs (``output_complete``).
ALWAYS_PARTIAL is attention-only: finish() hands the partial sum to the
next stage's input as its declared sum. EXIT_SCOPED, for an FFN or a
@@ -5,9 +5,9 @@ from __future__ import annotations
import contextlib
import os
import sys
from dataclasses import dataclass, replace
from dataclasses import dataclass, fields, is_dataclass, replace
from types import MappingProxyType
from typing import Callable, Mapping, NamedTuple, Optional
from typing import Callable, Mapping, NamedTuple, Optional, Tuple
from sglang.srt.environ import envs
from sglang.srt.layers import layernorm_sp
@@ -29,6 +29,7 @@ from sglang.srt.layers.layer_boundary.contracts import (
StageContract,
StageKind,
)
from sglang.srt.layers.layer_boundary.facts import facts_of
from sglang.srt.layers.layer_boundary.layout import (
Layout,
SumGroup,
@@ -301,11 +302,14 @@ class StageDeclaration:
contiguous slice of the rows, it returns them all, in rank order,
or None to leave the gather to the boundary. Called on every batch,
so it must be CUDA-graph safe. Across a pipeline boundary the
producer's rank takes it from the neighbouring layer it builds on
the meta device, so it may use only the communication state of the
rank it runs on, not the declaring layer's weights or buffers.
previous: Declaration whose output this stage consumes, as the stack
records it; across pipeline ranks it is built locally.
producer's rank takes it from the neighbouring stage's declaration
(the model's shared declaration function, or, for a model on the
transitional list, the layer built again on the meta device), so it
may use only the communication state of the rank it runs on, not
the declaring layer's weights or buffers.
previous: The stage whose output this stage consumes, as the stack
records it: only the facts binding reads of it (see facts_of),
on this rank or another.
prepared_from: Declaration whose already-read input a branch reuses.
Mutually exclusive with previous; avoids a second update/read.
@@ -738,6 +742,19 @@ def _connect_line(line, origins, *, before=None, after=None, arrivals=None):
if arrivals is None:
arrivals = {v: _stack_arrival(v) for v in variants}
if before is not None:
if (
before.output_transform is not None
and before.kind is StageKind.ATTENTION
and before.reduction is ProducerReduction.ALWAYS_PARTIAL
):
# Its transform runs at the next stage's input, which would be
# on this rank, without the module that declares it.
error = NotImplementedError(
"a pipeline rank that ends on an attention transforming the "
"output whose sum it leaves"
)
_note_origin(error, origins[0])
raise error
flow = resolve(0, before, arrivals, line[0])
arrivals = {v: _arrival(before, flow[v], v) for v in variants}
flows = []
@@ -830,11 +847,14 @@ def layer_stack(*, previous_layers=(), next_layers=(), final_read=None):
stage ends the model's layer stack unless a later layer declares a stage.
Args:
previous_layers: Callables that build, nearest first, the layers before
this stack that another pipeline rank holds. Called only if this
stack appended stages, after its own layers are built, until one of
them declares a stage: its last stage is the producer of this
stack's first. What they build is discarded.
previous_layers: For the layers before this stack that another
pipeline rank holds, nearest first, callables that return the
stages the layer declares as DeclaredStages (from the model's
shared declaration function), or else build the layer so that it
appends them. Called only if this stack appended stages, after its
own layers are built, until one of them declares a stage: its last
stage is the producer of this stack's first. Only what binding
reads of that stage is kept (see facts_of).
next_layers: Likewise for the layers after this stack, whose first
declared stage is the consumer of this stack's last.
final_read: The model's final read of the stack's output (a
@@ -861,21 +881,94 @@ def layer_stack(*, previous_layers=(), next_layers=(), final_read=None):
_stack = outer
def _neighbour_stage(build_layers, *, last):
class DeclaredStages(NamedTuple):
"""The stages a neighbouring layer declares, from the model's shared
declaration function, which stand in for building the layer."""
stages: Tuple[StageDeclaration, ...]
def check_declared_stages(stack, appended: int, expected, where: str) -> None:
"""Check that what a layer appended to ``stack`` since it held
``appended`` appends is what its shared declaration function says it
declares, as far as binding reads it: another pipeline rank binds this
layer's stages from that function alone."""
declared = [
facts_of(declaration)
for append in stack.appends[appended:]
if append.prepared_from is None
for declaration in append.declarations
]
expected = [facts_of(declaration) for declaration in expected]
if declared != expected:
raise ValueError(
f"{where} declares stages its shared declaration function does "
f"not: {_first_difference(declared, expected)}"
)
def _first_difference(declared, expected) -> str:
"""Where the stages a layer declares first differ from those its shared
declaration function gives, field by field."""
if len(declared) != len(expected):
return (
f"it declares {[stage.kind.name for stage in declared]}, the "
f"function {[stage.kind.name for stage in expected]}"
)
index, (stage, given) = next(
(index, pair)
for index, pair in enumerate(zip(declared, expected))
if pair[0] != pair[1]
)
return f"stage {index} ({stage.kind.name}): " + "; ".join(
f"{name} is {value!r}, the function says {given_value!r}"
for name, value, given_value in _differing_fields(stage, given)
)
def _differing_fields(value, given, prefix=""):
"""(dotted name, value, given value) for each leaf field that differs,
descending into declarations and the facts they hold."""
if is_dataclass(value):
names = [field.name for field in fields(value)]
else:
names = getattr(type(value), "__struct_fields__", None)
if names is None or type(value) is not type(given):
return [(prefix.rstrip("."), value, given)]
return [
difference
for name in names
if getattr(value, name) != getattr(given, name)
for difference in _differing_fields(
getattr(value, name), getattr(given, name), f"{prefix}{name}."
)
]
def _neighbour_stage(layers, *, last):
"""The stage a neighbouring layer declares next to this stack: the last
one of the nearest layer before it, or the first of the nearest after.
Branches are side paths, so they never stand next to the stack."""
global _stack
for build_layer in build_layers:
for layer in layers:
outer = _stack
_stack = _LayerStack()
try:
build_layer()
appends = [a for a in _stack.appends if a.prepared_from is None]
result = layer()
declared = (
result.stages
if isinstance(result, DeclaredStages)
else [
declaration
for append in _stack.appends
if append.prepared_from is None
for declaration in append.declarations
]
)
finally:
_stack = outer
if appends:
return appends[-1].declarations[-1] if last else appends[0].declarations[0]
if declared:
return facts_of(declared[-1] if last else declared[0])
return None
@@ -938,15 +1031,6 @@ def _return_before_trailing_attention(line):
return
def _detached(declaration):
"""The declaration as the stage after it records it: without its own
history, so a bound stage holds its producer's facts and not the chain of
declarations before it."""
if declaration is None:
return None
return replace(declaration, previous=None, prepared_from=None)
def _bind_stack(appends, *, previous, following, final_read=None):
"""Bind every appended stage and fill in the boundaries each append
returned.
@@ -973,13 +1057,13 @@ def _bind_stack(appends, *, previous, following, final_read=None):
bound = {id(append): [None] * len(append.bindings) for append in appends}
# A returned declaration's stage as bound, for the branches that start from it.
sources = {}
producer = _detached(_handed_off(previous))
producer = facts_of(_handed_off(previous))
chained = []
for append, index in stages:
chained.append(
replace(append.declarations[index], previous=producer, prepared_from=None)
)
producer = _detached(chained[-1])
producer = facts_of(chained[-1])
connections = _connect_line(
chained,
[append.origin for append, _ in stages],
@@ -1010,8 +1094,8 @@ def _bind_stack(appends, *, previous, following, final_read=None):
chained.append(
replace(
declaration,
previous=chained[-1] if chained else None,
prepared_from=None if chained else source_declaration,
previous=facts_of(chained[-1]) if chained else None,
prepared_from=None if chained else facts_of(source_declaration),
)
)
try:
@@ -0,0 +1,99 @@
# Copyright 2023-2024 SGLang Team
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""What binding reads of a stage it does not run: the stage before or after
a stack on another pipeline rank, and the producer a bound stage records.
A stage's read, update and output transform are objects that hold the
layer's modules and state. Binding a neighbouring stage reads only the
capabilities they declare, so a neighbour, and the producer a stage keeps,
is recorded as those values alone: no module, bound method or declaration
history stays referenced once the stack is bound."""
from typing import Optional
import msgspec
class UpdateFacts(msgspec.Struct, frozen=True):
"""The capabilities a ResidualUpdate declares (see ResidualUpdate)."""
is_plain_add: bool
applied_at_exit: bool
outlives_layer: bool
writes_stream: bool
quantized_sum: bool = False
@classmethod
def of(cls, update) -> "UpdateFacts":
if isinstance(update, cls):
return update
return cls(
is_plain_add=update.is_plain_add,
applied_at_exit=update.applied_at_exit,
outlives_layer=update.outlives_layer,
writes_stream=update.writes_stream,
quantized_sum=update.quantized_sum,
)
class ReadFacts(msgspec.Struct, frozen=True):
"""The capabilities a ResidualReadout declares (see ResidualReadout).
Binding takes kernels only from the reads of the stages it binds, never
from a stage recorded by its facts, so these supply none."""
is_plain_norm: bool
reads_before_dp_gather: bool
reads_after_attn_tp_gather: bool
completing_fusions = ()
gathering_reads = ()
@classmethod
def of(cls, read) -> "ReadFacts":
if isinstance(read, cls):
return read
return cls(
is_plain_norm=read.is_plain_norm,
reads_before_dp_gather=read.reads_before_dp_gather,
reads_after_attn_tp_gather=read.reads_after_attn_tp_gather,
)
class TransformFacts(msgspec.Struct, frozen=True):
"""What an OutputTransform declares (see OutputTransform)."""
before_reduce_scatter: bool = False
@classmethod
def of(cls, transform) -> Optional["TransformFacts"]:
if transform is None or isinstance(transform, cls):
return transform
return cls(before_reduce_scatter=transform.before_reduce_scatter)
def facts_of(declaration):
"""``declaration`` with its read, update and output transform reduced to
the capabilities they declare, and without the declarations it was
chained to. None stays None."""
if declaration is None:
return None
from dataclasses import replace
return replace(
declaration,
read=ReadFacts.of(declaration.read),
update=UpdateFacts.of(declaration.update),
output_transform=TransformFacts.of(declaration.output_transform),
previous=None,
prepared_from=None,
)
@@ -335,7 +335,6 @@ def keep_output(
hidden_states: torch.Tensor,
residual: torch.Tensor,
forward_batch: ForwardBatch,
**kwargs,
):
return hidden_states, residual
@@ -346,7 +345,6 @@ def update_attn_tp_gather_output(
forward_batch: ForwardBatch,
update: ResidualUpdate = PLAIN_ADD,
gather: Optional[Callable] = None,
**kwargs,
):
"""Write the output into the residual on this rank's slice, then gather it
over attention TP (see attn_tp_gather_with)."""
@@ -371,7 +369,6 @@ def residual_slice_output(
sums: bool,
gathers_back: bool,
update: ResidualUpdate,
**kwargs,
):
"""Bring the FFN output onto the slice of the rows the residual is on:
a reduce-scatter that also completes its sum when ``sums``, else this
@@ -405,7 +402,6 @@ def attn_cp_take_back_output(
hidden_states: torch.Tensor,
residual: torch.Tensor,
forward_batch: ForwardBatch,
**kwargs,
):
"""DSA and MLA CP: this rank's shard of a complete output gathered in
equal shards over the attention-CP group."""
@@ -418,7 +414,6 @@ def attn_cp_reduce_scatter_output(
hidden_states: torch.Tensor,
residual: torch.Tensor,
forward_batch: ForwardBatch,
**kwargs,
):
"""DSA and MLA CP: sum the FFN output over the attention-CP group and
keep this rank's shard."""
@@ -429,7 +424,6 @@ def dp_cp_take_back_output(
hidden_states: torch.Tensor,
residual: torch.Tensor,
forward_batch: ForwardBatch,
**kwargs,
):
"""This rank's CP shard of the rows the DP gather put in its DP group's
slot, at the shard's padded length with the padding zeroed."""
@@ -450,7 +444,6 @@ def moe_cp_take_back_output(
hidden_states: torch.Tensor,
residual: torch.Tensor,
forward_batch: ForwardBatch,
**kwargs,
):
"""Return a MoE output computed on the MoE-CP-gathered rows to this rank's attention rows.
@@ -174,7 +174,7 @@ class StageBoundary:
completed or carried on. An FFN whose boundary has fusions (which may
take a MoE finalize handoff) finishes through exit() instead."""
if self.kind is StageKind.ATTENTION and self.plan.finishes_directly:
produced = self.plan.produced(forward_batch)
produced = self.plan.path_for(forward_batch).output
return stream_of(forward_batch).record(
hidden_states,
produced.update,
@@ -238,9 +238,6 @@ class StageBoundary:
)
return hidden_states
def snapshot(self, hidden_states, forward_batch):
return stream_of(forward_batch).snapshot(hidden_states)
def capture_output(self, hidden_states, forward_batch, *, skip_empty=False):
"""Capture an output while preserving the caller's reduction timing.
+1 -1
View File
@@ -607,7 +607,7 @@ class LongcatFlashModel(nn.Module):
for i in range(total_num_layers):
if i in self.layers_to_capture:
aux_hidden_states.append(
self.layers[i].attn_boundary.snapshot(hidden_states, forward_batch)
residual_batch.snapshot(hidden_states, forward_batch)
)
with get_global_expert_distribution_recorder().with_current_layer(i):
layer = self.layers[i]
+3 -1
View File
@@ -18,6 +18,7 @@
"""Inference-only NemotronH model."""
from collections.abc import Iterable
from functools import partial
import torch
from torch import nn
@@ -77,7 +78,7 @@ from sglang.srt.model_loader.weight_utils import (
replace_prefix,
replace_substrings,
)
from sglang.srt.models.nemotron_h_utils import make_stage_boundary
from sglang.srt.models.nemotron_h_utils import make_stage_boundary, stage_facts
from sglang.srt.models.utils import WeightsMapper
from sglang.srt.runtime_context import get_exec, get_parallel
from sglang.srt.utils import (
@@ -690,6 +691,7 @@ class NemotronHModel(nn.Module):
len(config.hybrid_override_pattern),
get_layer,
prefix=f"{prefix}.layers",
stage_facts=partial(stage_facts, config.hybrid_override_pattern),
)
if self.pp_group.is_last_rank:
self.norm_f = RMSNorm(config.hidden_size, eps=config.layer_norm_epsilon)
+9 -6
View File
@@ -37,15 +37,18 @@ def _declaration(pattern: str, layer_idx: int):
)
def stage_facts(pattern: str, layer_idx: int):
"""The one stage each Nemotron-H layer declares, from the pattern alone:
the model's shared declaration function, which its layers declare with
too (see make_layers)."""
return (_declaration(pattern, layer_idx),)
def make_stage_boundary(layer_norm: RMSNorm, *, pattern: str, layer_idx: int):
from sglang.srt.layers import layernorm_sp
if get_parallel().attn_cp_size > 1 or layernorm_sp.layernorm_sp_enabled():
raise NotImplementedError("a Nemotron stage with attention CP or LayerNorm SP")
(boundary,) = append_stages(
(
_declaration(pattern, layer_idx),
layer_norm,
),
)
(declaration,) = stage_facts(pattern, layer_idx)
(boundary,) = append_stages((declaration, layer_norm))
return boundary
+93 -9
View File
@@ -1591,18 +1591,27 @@ def make_layers(
return_tuple: bool = False,
offloader_kwargs: Optional[Dict[str, Any]] = None,
final_read: Optional[Any] = None,
stage_facts: Optional[Callable[[int], Sequence[Any]]] = None,
) -> Tuple[torch.nn.Module, int, int]:
"""Make a list of layers with the given layer function.
The local layers are built inside one layer stack, so layers that declare
stage boundaries connect in order without naming their neighbours. Across
a pipeline stage boundary the stack learns the neighbouring stage from the
layer itself, built again on the meta device. ``final_read`` is the
stack's terminal read when it is not a plain final norm (see layer_stack).
a pipeline stage boundary the stack learns the neighbouring stage from
``stage_facts``: the model's shared declaration function, which returns
the stages the layer at a global index declares without building it (an
empty sequence when it declares none), and which the layer itself uses
for its own declarations. A model on the transitional list may omit it;
the stack then builds the layer again on the meta device to read them.
``final_read`` is the stack's terminal read when it is not a plain final
norm (see layer_stack).
"""
# circular imports
from sglang.srt.distributed import get_pp_indices
from sglang.srt.layers.layer_boundary.factories import layer_stack
from sglang.srt.layers.layer_boundary.factories import (
check_declared_stages,
layer_stack,
)
from sglang.srt.layers.utils import PPMissingLayer
from sglang.srt.utils.offloader import get_offloader
@@ -1618,22 +1627,29 @@ def make_layers(
)
def neighbour(idx):
if stage_facts is not None:
return functools.partial(_declared_stages, stage_facts, idx)
return functools.partial(
_build_neighbour_layer, layer_fn, idx, add_prefix(idx, prefix)
)
def build(stack, idx):
appended = len(stack.appends)
layer = layer_fn(idx=idx, prefix=add_prefix(idx, prefix))
if stage_facts is not None and (pp_size or 1) > 1:
# Another rank binds this layer's stages from stage_facts alone.
check_declared_stages(stack, appended, stage_facts(idx), f"layer {idx}")
return layer
with layer_stack(
previous_layers=[neighbour(idx) for idx in reversed(range(start_layer))],
next_layers=[neighbour(idx) for idx in range(end_layer, num_hidden_layers)],
final_read=final_read,
):
) as stack:
modules = torch.nn.ModuleList(
[PPMissingLayer(return_tuple=return_tuple) for _ in range(start_layer)]
+ get_offloader().wrap_modules(
(
layer_fn(idx=idx, prefix=add_prefix(idx, prefix))
for idx in range(start_layer, end_layer)
),
(build(stack, idx) for idx in range(start_layer, end_layer)),
**(offloader_kwargs or {}),
)
+ [
@@ -1653,10 +1669,12 @@ def make_pp_layers(
return_tuple: bool = False,
offloader_kwargs: Optional[Dict[str, Any]] = None,
final_read: Optional[Any] = None,
stage_facts: Optional[Callable[[int], Sequence[Any]]] = None,
) -> Tuple[torch.nn.Module, int, int]:
"""Make this pipeline stage's layers, and return them with the stage's range.
Layers outside ``[start_layer, end_layer)`` are ``PPMissingLayer`` stand-ins.
``stage_facts`` is the model's shared declaration function (see make_layers).
"""
parallel = get_parallel()
return make_layers(
@@ -1668,15 +1686,81 @@ def make_pp_layers(
return_tuple=return_tuple,
offloader_kwargs=offloader_kwargs,
final_read=final_read,
stage_facts=stage_facts,
)
# The models whose layers a pipeline rank still builds on the meta device to
# read the stages they declare, until each gives make_layers its shared
# declaration function (stage_facts). Any other model must give one to run
# under pipeline parallelism.
_NEIGHBOUR_BUILD_MODULES = frozenset(
(
"sglang.srt.models.apertus",
"sglang.srt.models.arcee",
"sglang.srt.models.bailing_moe",
"sglang.srt.models.bailing_moe_linear",
"sglang.srt.models.bailing_moe_v3",
"sglang.srt.models.deepseek_v2",
"sglang.srt.models.dots3_common.modeling",
"sglang.srt.models.ernie45_moe_vl",
"sglang.srt.models.exaone4",
"sglang.srt.models.exaone_moe",
"sglang.srt.models.gemma4_causal",
"sglang.srt.models.gigachat35",
"sglang.srt.models.glm4",
"sglang.srt.models.glm4_moe",
"sglang.srt.models.glm4_moe_lite",
"sglang.srt.models.glm5_next",
"sglang.srt.models.gpt_oss",
"sglang.srt.models.granitemoehybrid",
"sglang.srt.models.kimi_k3",
"sglang.srt.models.kimi_linear",
"sglang.srt.models.laguna",
"sglang.srt.models.llada2",
"sglang.srt.models.llama",
"sglang.srt.models.mimo_v2",
"sglang.srt.models.minimax_m2",
"sglang.srt.models.minimax_m3",
"sglang.srt.models.ministral3",
"sglang.srt.models.mixtral",
"sglang.srt.models.nanbeige",
"sglang.srt.models.nemotron_nas",
"sglang.srt.models.qwen2",
"sglang.srt.models.qwen2_moe",
"sglang.srt.models.qwen3_5",
"sglang.srt.models.sarvam_moe",
"sglang.srt.models.sdar",
"sglang.srt.models.sdar_moe",
"sglang.srt.models.spark2_5",
"sglang.srt.models.step3p5",
"sglang.srt.models.xllm",
"sglang.srt.models.zaya",
)
)
def _declared_stages(stage_facts, idx: int):
"""The stages the layer at ``idx`` declares, from the model's shared
declaration function, standing in for building it."""
from sglang.srt.layers.layer_boundary.factories import DeclaredStages
return DeclaredStages(tuple(stage_facts(idx)))
def _build_neighbour_layer(layer_fn: LayerFn, idx: int, prefix: str) -> None:
"""Build a layer another pipeline stage holds, only for the stage
boundaries it declares (see building_neighbour_layer). RoPE modules it
adds to the shared cache are meta, so they are dropped again."""
from sglang.srt.layers.rotary_embedding.factory import _ROPE_DICT
module = getattr(getattr(layer_fn, "func", layer_fn), "__module__", None)
if module not in _NEIGHBOUR_BUILD_MODULES:
raise ValueError(
f"{module} declares stage boundaries under pipeline parallelism "
"without a shared declaration function: give make_layers its "
"stage_facts, the stages a layer declares without building it"
)
cached = set(_ROPE_DICT)
try:
with building_neighbour_layer():