Files
sglang/python/sglang/srt/layers/layer_boundary/contracts.py
T

256 lines
11 KiB
Python

# Copyright 2023-2024 SGLang Team
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Immutable stage declarations and bound input/output paths."""
from enum import Enum, auto
from typing import Callable, FrozenSet, Optional, Tuple
import msgspec
import torch
from sglang.srt.distributed import GroupCoordinator
from sglang.srt.layers.layer_boundary.layout import Layout, SumGroup, TokenAxis
from sglang.srt.layers.layer_boundary.output import OutputTransform
from sglang.srt.layers.layer_boundary.residual import ResidualReadout, ResidualUpdate
from sglang.srt.layers.layer_boundary.residual.add_norm import NORM_READOUT, PLAIN_ADD
class ProducerReduction(Enum):
"""Who completes the sum a stage's output owes; compute never does.
ALWAYS_PARTIAL is attention-only: finish() hands the partial sum to the
next stage's input as its declared sum. EXIT_SCOPED, for an FFN or a
single-stage mixer: the exit completes the sum, or carries it to the next
stage's input.
"""
ALWAYS_PARTIAL = auto()
EXIT_SCOPED = auto()
class ExitRows(Enum):
"""The rows a layer or branch must leave its output on, independent of
compute's input rows."""
ATTENTION = auto()
TBO_SPLIT = auto()
class BatchVariant(Enum):
ORDINARY = auto()
CONTEXT_PARALLEL = auto()
INPUT_SCATTERED = auto()
SEQUENCE_PARALLEL = auto()
class InputContract(msgspec.Struct, frozen=True):
"""Declare the consumer's input rows and read operation.
Fields:
layout: Token sharding required by compute.
gathered_by_compute: Token axes compute can gather internally; the boundary
may hand it input still sharded over those axes.
read: Operation that derives input from the updated residual.
"""
layout: Layout
# Token axes the consumer gathers over itself when its input arrives
# sharded over them.
gathered_by_compute: FrozenSet[TokenAxis] = frozenset()
# How the consumer reads its input from the residual.
read: ResidualReadout = NORM_READOUT
class OutputContract(msgspec.Struct, frozen=True):
"""Declare producer rows and permitted reduction handoffs at construction.
Fields:
layout: Token sharding of the producer contribution.
group: The group the output owes its sum over, or None when it is
complete.
always_partial: The sum is handed to the next stage's input as its
declared sum.
may_defer_to_next: The exit may carry the sum to the following
stage's input instead of completing it.
may_reduce_scatter: A fixed-size reduce-scatter selected by the
boundary may complete the sum.
may_reduce_scatterv: The selected variable-size attention-DP combine
may complete the sum.
update: Producer residual operation; None on an arrival contract that
declares capabilities and obtains the actual update from the stream.
transform: Optional operation on the contribution before residual update.
These are permissions, not a record of a particular output. The exit
decision and ResidualStream record what that output actually owes.
"""
layout: Layout
# None when there is nothing to sum.
group: Optional[SumGroup] = None
# The sum is always handed to the next stage's input as a declared sum.
always_partial: bool = False
# Otherwise the exit completes it, or may carry it to the next input.
may_defer_to_next: bool = False
may_reduce_scatter: bool = False
# Whether the selected attention-DP combine may complete this sum.
may_reduce_scatterv: bool = False
# How the producer's output is written into the residual. An arrival
# description has no producer object; its edge declares capabilities only.
update: Optional[ResidualUpdate] = PLAIN_ADD
transform: Optional[OutputTransform] = None
class StageContract(msgspec.Struct, frozen=True):
"""Resolved input and output contracts for one stage and batch variant.
Fields:
input: Consumer row requirement and read operation.
output: Producer row, reduction and residual-update contract.
"""
input: InputContract
output: OutputContract
class EdgeContract(msgspec.Struct, frozen=True):
"""Describe both sides of a boundary and the residual's row movement.
Fields:
produced: Producer output contract as seen by this side of the edge.
need: Consumer input contract.
residual: Residual layout before the boundary.
residual_to: Residual layout after the boundary.
residual_joins_sum: Whether one rank may add the residual into a partial
before reduction; valid only for an eligible plain-add update.
arriving_plain_add: Allowed values of ResidualUpdate.is_plain_add for
arriving contributions. Empty means use produced.update's capability.
The actual update object travels with the residual stream.
"""
produced: OutputContract
need: InputContract
residual: Layout
residual_to: Layout
# Whether the residual is added into one rank's share of the produced sum
# before that sum completes, instead of after it.
residual_joins_sum: bool = False
# Capabilities allowed to arrive from another layer, not its update object.
arriving_plain_add: Tuple[bool, ...] = ()
class FfnInputFusion(msgspec.Struct, frozen=True):
"""A kernel that completes the sum the attention output owes together with
the residual add and the post-attention norm, in that order.
``run(hidden_states, residual, forward_batch)`` returns the FFN's input and
the residual, or None when it does not take the batch; it returns None only
before touching its inputs or starting a collective."""
# The group whose sum it completes.
completes: SumGroup
run: Callable[..., Optional[Tuple[torch.Tensor, torch.Tensor]]]
# Callable(residual, forward_batch): True guarantees this candidate is
# selected and does not mutate residual, including backend fallback.
preserves_residual: Optional[Callable] = None
class CpMoves(msgspec.Struct, frozen=True):
"""How a CP extend's rows reach an FFN that needs all of them and come back,
chosen once for the kind of prefill CP: ``gather`` gathers the FFN input
after each rank has completed its own block, ``take_back`` returns this
rank's block of a complete output, and ``reduce_scatter``, where there is
one, completes a sum left over the ranks of ``reduce_scatter_group()`` and
returns the block in the same collective."""
gather: Callable
take_back: Callable
reduce_scatter: Optional[Callable] = None
reduce_scatter_group: Optional[Callable[[], GroupCoordinator]] = None
class EntryPath(msgspec.Struct, frozen=True):
"""Bound consumer operations for one batch variant.
Fields:
prepare: Callable(hidden_states, residual, forward_batch, norm, **call)
returning compute input and updated residual; finishes owed work,
applies the producer update and performs the consumer read.
input_rows: Layout handed to compute after preparation and input_move.
input_move: Optional movement after prepare, before compute takes the input.
attn_input_adapter: Optional callable(input, forward_batch, qkv_latent_func) that
adapts already-placed input for attention.
capture_move: Optional movement of the updated residual back onto the
producer's rows for auxiliary capture.
capture_move_allocates: Whether capture_move returns fresh storage (a
gather) rather than a view (a cut); capture then retains it
without copying.
declared_sum: Statically owed sum group for an otherwise raw input tensor.
preserves_residual: Optional (residual, batch) predicate certifying this
input path leaves residual untouched, including backend fallback.
capture_preserves_residual: Same guarantee for the local following stage,
copied at construction so auxiliary capture can retain its input.
"""
# Completes what the input owes, writes the previous stage's output into
# the residual and reads this stage's input:
# (hidden_states, residual, forward_batch, norm, **call).
prepare: Callable
# The rows prepare hands the stage.
input_rows: Layout
# Moves the input onto the stage's rows after prepare, when prepare does
# not: (hidden_states, forward_batch) -> hidden_states.
input_move: Optional[Callable] = None
# Hands the stage its input once it is on its rows:
# (hidden_states, forward_batch, qkv_latent_func) -> hidden_states.
attn_input_adapter: Optional[Callable] = None
# Return the updated residual to the producer's rows for aux capture.
capture_move: Optional[Callable] = None
capture_move_allocates: bool = False
declared_sum: Optional[SumGroup] = None
preserves_residual: Optional[Callable] = None
# Bound from a local successor at construction; no runtime plan lookup.
capture_preserves_residual: Optional[Callable] = None
class StagePath(msgspec.Struct, frozen=True):
"""Precomputed entry and exit work for one stage and batch variant.
Fields:
entry: Bound consumer half: preparation, input move and input adapter.
output: Producer contract used by the exit decision.
output_move: Fixed output transport, or None when absent or chosen per
batch by the attention-DP exit path.
output_move_completes_sum: Whether that move also reduces the output.
returns_over_dp: Whether output uses batch-dependent attention-DP transport.
"""
entry: EntryPath
output: OutputContract
# None means no fixed move. returns_over_dp selects batch-dependent DP transport.
output_move: Optional[Callable]
output_move_completes_sum: bool = False
returns_over_dp: bool = False
class StageKind(Enum):
"""Select attention/mixer or FFN input/output adapters.
This is a boundary role, not a compute implementation or a requirement
that layers contain exactly two alternating stages.
"""
ATTENTION = auto()
FFN = auto()