diff --git a/docs/docs/references/nightly_precision_regression.mdx b/docs/docs/references/nightly_precision_regression.mdx index 5040b4889c..70a4c8cde1 100644 --- a/docs/docs/references/nightly_precision_regression.mdx +++ b/docs/docs/references/nightly_precision_regression.mdx @@ -100,8 +100,8 @@ A `capture_signature` (SHA-1 hash of schema version, max_tokens, ignore_eos, TP The harness pins FlashInfer all-reduce fusion **on** (`--flashinfer-allreduce-fusion-backend trtllm`; the SM90 auto-enable was dropped in #23402). Two names are captured per layer: -- `inputs.1` — hidden states entering the layer. `LayerCommunicator` defers the cross-layer all-reduce, so with fusion active these are rank-local TP-partial sums. CPU-side reduction of them is not the kernel's semantic output and drifts across self-hosted runners, so they are **not** compared. `_assert_fused_tp_layout()` keeps them only as a guard: if they come back replicated, fusion silently fell back and the test fails. -- `self_attn.inputs.hidden_states` — the value attention consumes after `prepare_attn()`, i.e. post fused all-reduce and residual RMSNorm. This is replicated across ranks and is what the comparator actually diffs. +- `inputs.1` — hidden states entering the layer. The FFN boundary defers the cross-layer all-reduce, so with fusion active these are rank-local TP-partial sums. CPU-side reduction of them is not the kernel's semantic output and drifts across self-hosted runners, so they are **not** compared. `_assert_fused_tp_layout()` keeps them only as a guard: if they come back replicated, fusion silently fell back and the test fails. +- `self_attn.inputs.hidden_states` — the value attention consumes after `attn_stage.prepare()`, i.e. post fused all-reduce and residual RMSNorm. This is replicated across ranks and is what the comparator actually diffs. This is why a stale baseline captured with fusion *not* initialized is incompatible: it holds replicated layer-entry tensors where the target holds TP-partial ones, which reads as an approximately `tp_size`x mismatch. Fusion initialization failures are tracked in [FlashInfer #3676](https://github.com/flashinfer-ai/flashinfer/issues/3676) and [SGLang #30875](https://github.com/sgl-project/sglang/issues/30875). diff --git a/python/sglang/srt/arg_groups/fields/parallel.py b/python/sglang/srt/arg_groups/fields/parallel.py index 239da21a60..3e0e28e6d4 100644 --- a/python/sglang/srt/arg_groups/fields/parallel.py +++ b/python/sglang/srt/arg_groups/fields/parallel.py @@ -195,7 +195,10 @@ class Parallel(msgspec.Struct): ] = None enable_attn_tp_input_scattered: A[ bool, - "Allow input of attention to be scattered when only using tensor parallelism, to reduce the computational load of operations such as qkv latent.", + Arg( + help="Allow input of attention to be scattered when only using tensor parallelism, to reduce the computational load of operations such as qkv latent.", + resolvable=True, + ), ] = False enable_shared_experts_attn_tp: A[ bool, diff --git a/python/sglang/srt/arg_groups/parallel_hook.py b/python/sglang/srt/arg_groups/parallel_hook.py index 7cd2fb3bb1..1a396935fb 100644 --- a/python/sglang/srt/arg_groups/parallel_hook.py +++ b/python/sglang/srt/arg_groups/parallel_hook.py @@ -29,6 +29,27 @@ from sglang.srt.utils.common import parse_connector_type logger = logging.getLogger(__name__) +def _boundary_parallelism_overrides(cfg, model_type: str) -> dict: + """Resolve the token-row modes the model's forward actually enters.""" + nemotron = model_type in ("nemotron_h", "nemotron_h_puzzle") + longcat = model_type == "longcat_flash" + if nemotron and cfg.attn_cp_size > 1: + raise ValueError("Nemotron-H does not support --attn-cp-size > 1") + if longcat and cfg.enable_prefill_cp and cfg.attn_cp_size > 1: + raise ValueError( + "LongCat-Flash does not support --enable-prefill-cp with --attn-cp-size > 1" + ) + if (nemotron or longcat) and cfg.enable_attn_tp_input_scattered: + # Neither model enters the input-scattered attention scope. Preserve + # their existing ordinary execution and make the effective flag explicit. + logger.warning( + "Disabling input-scattered attention for %s: its forward does not enter that scope", + model_type, + ) + return {"enable_attn_tp_input_scattered": False} + return {} + + def handle_context_parallelism(server_args: Any): # Through the registry, not a bare call: an out-of-tree replacement of # `validate_prefill_cp_platform` registered at its own (earlier) pipeline @@ -42,6 +63,13 @@ def handle_context_parallelism(server_args: Any): model_config = model_config_of(server_args) hf_config = model_config.hf_config model_arch = hf_config.architectures[0] + declare_resolution( + server_args, + "boundary_parallelism", + **_boundary_parallelism_overrides( + cfg, model_config.hf_text_config.model_type + ), + ) if ( cfg.enable_prefill_cp and model_arch == "DeepseekV32ForCausalLM" diff --git a/python/sglang/srt/batch_overlap/two_batch_overlap.py b/python/sglang/srt/batch_overlap/two_batch_overlap.py index 3399f0950e..fc207eefb2 100644 --- a/python/sglang/srt/batch_overlap/two_batch_overlap.py +++ b/python/sglang/srt/batch_overlap/two_batch_overlap.py @@ -957,7 +957,9 @@ def model_forward_stages( hidden_states, residual = stream.finish(hidden_states) inputs["hidden_states"] = hidden_states parts = _model_forward_tbo_split_inputs( - **inputs, residual=residual, layer_input_rows=layers[0].attn_stage.input_rows + **inputs, + residual=residual, + layer_input_rows=layers[0].attn_stage.incoming_residual_rows, ) for part in parts: part["hidden_states"], child_stream = ResidualStream.arrive( @@ -992,89 +994,6 @@ def model_forward_stages( return hidden_states -def model_forward_maybe_tbo( - layers, - enable_tbo: bool, - positions: torch.Tensor, - forward_batch: ForwardBatch, - hidden_states: torch.Tensor, - residual: Optional[torch.Tensor], - zero_allocator: Optional[BumpAllocator] = None, -): - inputs = dict( - positions=positions, - hidden_states=hidden_states, - forward_batch=forward_batch, - residual=residual, - zero_allocator=zero_allocator, - ) - operations_strategy = OperationsStrategy.init_new_tbo( - layers, forward_batch.global_forward_mode - ) - if enable_tbo: - return _model_forward_tbo( - inputs=inputs, - operations_strategy=operations_strategy, - layer_input_rows=layers[0].layer_communicator.input_rows, - ) - else: - return _model_forward_non_tbo(inputs, operations_strategy) - - -def _model_forward_tbo( - inputs, - operations_strategy: OperationsStrategy, - layer_input_rows: Layout, -): - parent_batch = inputs["forward_batch"] - stream = inputs["residual"] - pending = stream.pending if isinstance(stream, ResidualStream) else None - inputs["hidden_states"], inputs["residual"] = finish_layer_stack( - inputs["hidden_states"], inputs["residual"], inputs["forward_batch"] - ) - inputs_arr = _model_forward_tbo_split_inputs( - **inputs, layer_input_rows=layer_input_rows - ) - original_hidden_states_len = inputs["hidden_states"].shape[0] - if isinstance(stream, ResidualStream): - for part in inputs_arr: - part["hidden_states"], part["residual"] = ResidualStream.arrive( - part["hidden_states"], - part["residual"], - pending.update if pending is not None else None, - ) - part["forward_batch"].residual_stream = part["residual"] - del inputs, stream, pending - - context = ( - empty_context() - if _is_hip - else deep_gemm_wrapper.configure_deep_gemm_num_sms( - operations_strategy.deep_gemm_num_sms - ) - ) - - with context: - outputs_arr = execute_overlapped_operations( - inputs_arr=inputs_arr, - operations_arr=[operations_strategy.operations] * 2, - delta_stages=[0, operations_strategy.tbo_delta_stages], - ) - - hidden_states, residual = _model_forward_tbo_merge_outputs( - *outputs_arr, original_hidden_states_len - ) - parent_batch.residual_stream = ( - residual if isinstance(residual, ResidualStream) else None - ) - return hidden_states, residual - - -def _model_forward_non_tbo(inputs, operations_strategy: OperationsStrategy): - outputs = execute_operations(inputs, operations_strategy.operations) - return outputs["hidden_states"], outputs["residual"] - - def _model_forward_tbo_split_inputs( hidden_states: torch.Tensor, residual: torch.Tensor, diff --git a/python/sglang/srt/layers/communicator/__init__.py b/python/sglang/srt/layers/communicator/__init__.py index 0829ab71a1..74c48c9389 100644 --- a/python/sglang/srt/layers/communicator/__init__.py +++ b/python/sglang/srt/layers/communicator/__init__.py @@ -18,24 +18,25 @@ from sglang.srt.layers.communicator.adapters.attention import ( get_attn_tp_context, ) from sglang.srt.layers.communicator.boundary import ( - BoundarySteps, - EdgeDecl, - FusedMlpInput, - LayerStage, - StageDecl, - StageEntry, - StageInput, - StageKind, - StageOutput, make_boundary, make_output_boundary, - stage_edges, tbo_split_moves, ) from sglang.srt.layers.communicator.construction import ( BatchVariant, StageEdges, - make_stage, +) +from sglang.srt.layers.communicator.contracts import ( + EdgeDecl, + FusedMlpInput, + HandoffRows, + ProducerReduction, + StageDecl, + StageEntry, + StageInput, + StageKind, + StageOutput, + StageSteps, ) from sglang.srt.layers.communicator.exit import FfnCompletion, FfnExit, MixerExit from sglang.srt.layers.communicator.factories import ( @@ -55,8 +56,6 @@ from sglang.srt.layers.communicator.layout import ( token_axis_sizes, ) from sglang.srt.layers.communicator.ops import ( - CommunicateSimpleFn, - CommunicateSummableTensorPairFn, move_rows, tp_reduce_scatter, ) @@ -78,9 +77,6 @@ from sglang.srt.layers.communicator.residual.mhc import ( ) __all__ = [ - "LayerCommunicator", - "LayerFacts", - "MHCLayerCommunicator", "declare_attn", "declare_ffn", "make_attn_stage", @@ -88,17 +84,16 @@ __all__ = [ "make_stages", "ADD", "AttentionInputs", - "BoundarySteps", - "CommunicateSimpleFn", - "CommunicateSummableTensorPairFn", + "StageSteps", "EdgeDecl", + "HandoffRows", + "ProducerReduction", "FUSE_ALLREDUCE_MAX_BATCH_SIZE", "FfnCompletion", "FfnExit", "FusedMlpInput", "HandoffOutput", "LayerResidual", - "LayerStage", "Layout", "MHCState", "MixerExit", @@ -121,14 +116,7 @@ __all__ = [ "move_rows", "reduce_output", "sparse_moe_gathers_over_moe_cp", - "stage_edges", "tbo_split_moves", "token_axis_sizes", "tp_reduce_scatter", ] - -from sglang.srt.layers.communicator.layer import ( - LayerCommunicator, - LayerFacts, - MHCLayerCommunicator, -) diff --git a/python/sglang/srt/layers/communicator/adapters/attention.py b/python/sglang/srt/layers/communicator/adapters/attention.py index 3027a45203..bc4c7098f8 100644 --- a/python/sglang/srt/layers/communicator/adapters/attention.py +++ b/python/sglang/srt/layers/communicator/adapters/attention.py @@ -135,11 +135,6 @@ class AttnTpContext: def set_attn_inputs(self, attn_inputs: AttentionInputs): get_forward().set("attn_inputs", attn_inputs) - def set_hidden_states_local(self, hidden_states: torch.Tensor) -> None: - attn_inputs = get_forward().attn_inputs - if attn_inputs is not None: - attn_inputs.hidden_states_local = hidden_states - def fetch_qkv_latent(self): attn_inputs = get_forward().attn_inputs assert attn_inputs is not None diff --git a/python/sglang/srt/layers/communicator/adapters/branch.py b/python/sglang/srt/layers/communicator/adapters/branch.py index 1bef5817c5..5fa4fd724b 100644 --- a/python/sglang/srt/layers/communicator/adapters/branch.py +++ b/python/sglang/srt/layers/communicator/adapters/branch.py @@ -15,7 +15,7 @@ from __future__ import annotations -from typing import Tuple +from typing import TYPE_CHECKING, Tuple import torch @@ -23,65 +23,63 @@ from sglang.srt.layers.communicator.ops import move_rows from sglang.srt.layers.communicator.residual.stream import ResidualStream from sglang.srt.model_executor.forward_batch_info import ForwardBatch +if TYPE_CHECKING: + from sglang.srt.layers.communicator.construction import StagePlan + def branch_input( - self, - source: object, + plan: StagePlan, + source: StagePlan, hidden_states: torch.Tensor, - residual: torch.Tensor, + stream: ResidualStream, forward_batch: ForwardBatch, -) -> Tuple[torch.Tensor, torch.Tensor]: +) -> Tuple[torch.Tensor, ResidualStream]: """The FFN input and residual that ``source``'s boundary read for its own FFN, for this layer's FFN, which branches from the same input: moved to the rows this FFN needs and its residual's rows.""" rows, residual_rows, _ = source._branch_rows(forward_batch) - to, residual_to, _ = self._branch_rows(forward_batch) - stream = residual if isinstance(residual, ResidualStream) else None - if stream is not None: - if stream.pending is not None: - raise RuntimeError("a branch must start from a prepared stage input") - residual = stream.residual + to, residual_to, _ = plan._branch_rows(forward_batch) + if stream.pending is not None: + raise RuntimeError("a branch must start from a prepared stage input") + residual = stream.residual hidden_states = move_rows(hidden_states, rows, to, forward_batch) residual = move_rows(residual, residual_rows, residual_to, forward_batch) return ( hidden_states, - ResidualStream(residual) if stream is not None else residual, + ResidualStream(residual), ) def branch_output( - self, hidden_states: torch.Tensor, forward_batch: ForwardBatch + plan: StagePlan, hidden_states: torch.Tensor, forward_batch: ForwardBatch ) -> torch.Tensor: """This layer's complete FFN output as a branch's contribution, which adds to the layer's output without writing the residual: moved to the rows the layer hands on.""" - rows, _, to = self._branch_rows(forward_batch) + rows, _, to = plan._branch_rows(forward_batch) return move_rows(hidden_states, rows, to, forward_batch) def merge_branch( - self, + plan: StagePlan, contribution: torch.Tensor, hidden_states: torch.Tensor, - residual: torch.Tensor, - source: object, + stream: ResidualStream, + source: StagePlan, forward_batch: ForwardBatch, -) -> Tuple[torch.Tensor, torch.Tensor]: +) -> Tuple[torch.Tensor, ResidualStream]: """A contribution from ``branch_output`` summed with what ``source``'s layer hands on, ``hidden_states`` and ``residual``, moved to the rows this layer hands on.""" _, _, rows = source._branch_rows(forward_batch) - _, _, to = self._branch_rows(forward_batch) - stream = residual if isinstance(residual, ResidualStream) else None - update = None - if stream is not None: - stream.check(hidden_states) - update = stream.pending.update - hidden_states, residual = stream.finish(hidden_states) + _, _, to = plan._branch_rows(forward_batch) + stream.check(hidden_states) + if stream.pending is None: + raise RuntimeError("branch merge requires a pending producer contribution") + update = stream.pending.update + hidden_states, residual = stream.finish(hidden_states) hidden_states = move_rows(hidden_states, rows, to, forward_batch) residual = move_rows(residual, rows, to, forward_batch) output = contribution + hidden_states - if stream is not None: - stream.write(residual) - return stream.leave(output, update), stream - return output, residual + stream.write(residual) + return stream.leave(output, update), stream diff --git a/python/sglang/srt/layers/communicator/adapters/overlap.py b/python/sglang/srt/layers/communicator/adapters/overlap.py new file mode 100644 index 0000000000..010547bedc --- /dev/null +++ b/python/sglang/srt/layers/communicator/adapters/overlap.py @@ -0,0 +1,26 @@ +"""Row requirements at the entrance to two-batch overlap.""" + +from sglang.srt.layers.communicator.contracts import HandoffRows +from sglang.srt.layers.communicator.layout import enable_moe_dense_fully_dp +from sglang.srt.runtime_context import get_exec + + +def tbo_handoff(sparse, next_sparse): + """Record a potential split between a dense layer and a sparse layer.""" + return HandoffRows.TBO_SPLIT if not sparse and next_sparse else None + + +def resolve_handoff_rows(requirement): + """Bind a handoff requirement with the construction-time parallel config. + + Dense-local output must reach attention rows before the TBO split. + Moving that gather after the split changes padding and collective sizes. + """ + if requirement is HandoffRows.TBO_SPLIT: + return ( + HandoffRows.ATTENTION + if enable_moe_dense_fully_dp() + and get_exec().overlap.enable_two_batch_overlap + else None + ) + return requirement diff --git a/python/sglang/srt/layers/communicator/boundary.py b/python/sglang/srt/layers/communicator/boundary.py index 3fb4b4438d..49df1be154 100644 --- a/python/sglang/srt/layers/communicator/boundary.py +++ b/python/sglang/srt/layers/communicator/boundary.py @@ -14,14 +14,18 @@ """What each stage produces and needs, and the boundary steps chosen from those declarations.""" -from enum import Enum, auto from functools import partial -from typing import Callable, FrozenSet, Mapping, Optional, Tuple +from typing import Callable, Optional, Tuple import msgspec -import torch -from sglang.srt.distributed import GroupCoordinator +from sglang.srt.layers.communicator.contracts import ( + CpMoves, + EdgeDecl, + FusedMlpInput, + StageInput, + StageOutput, +) from sglang.srt.layers.communicator.layout import ( Layout, SumGroup, @@ -32,11 +36,22 @@ from sglang.srt.layers.communicator.layout import ( token_axis_sizes, ) from sglang.srt.layers.communicator.ops import ( - CommunicateSimpleFn, - CommunicateSummableTensorPairFn, + gather_attention_tp, + identity_output, + move_rows, + output_on_residual_shard, + reduce_scatter_over_cp, + scatter_moe_cp_output, + scatter_output, + take_back_attention_cp_shard, + take_back_cp_shard, + tp_reduce_scatter, + tp_slice, + update_and_gather, +) +from sglang.srt.layers.communicator.prepare import ( _consumer_step, _dispatch_consumer, - _hand_qkv_hook_its_input, _mlp_input_dp_partial, _mlp_input_dp_replicate, _mlp_input_gather_attention_cp, @@ -47,445 +62,10 @@ from sglang.srt.layers.communicator.ops import ( _mlp_input_slice, _mlp_input_without_dp, _read_input, - move_rows, - tp_reduce_scatter, - tp_slice, -) -from sglang.srt.layers.communicator.output import OutputTransform -from sglang.srt.layers.communicator.residual import ( - LayerResidual, - StageRead, - StageUpdate, -) -from sglang.srt.layers.communicator.residual.add_norm import ( - ADD, - NORM_QUANT_READ, - NORM_READ, ) from sglang.srt.runtime_context import get_parallel -class StageInput(msgspec.Struct, frozen=True): - """The rows a stage's consumer needs: sharded over the token axes its - compute group does not span.""" - - layout: Layout - # Token axes the consumer gathers over itself when its input arrives - # sharded over them. - gathers_itself: FrozenSet[TokenAxis] = frozenset() - # How the consumer reads its input from the residual. - read: StageRead = NORM_READ - - -class StageOutput(msgspec.Struct, frozen=True): - """What a stage's producer hands the boundary after it, fixed at - construction: its rows, the group its output is summed over, and when it - leaves that sum to the boundary instead of completing it. - - What one output actually owes is carried with the value: a plain - tensor handed across layers is complete, an ``UnreducedOutput`` names what - is left. A raw compute output entering a boundary is read with this - declaration and the boundary's decision for the batch.""" - - layout: Layout - # None when there is nothing to sum. - group: Optional[SumGroup] = None - # The producer never reduces its output, e.g. a row-parallel projection - # built with reduce_results=False. - always_leaves: bool = False - # Otherwise it leaves the sum only when the boundary publishes the flag for - # it: fuse_mlp_allreduce, or mlp_reduce_scatter. - leaves_for_next_layer: bool = False - leaves_for_reduce_scatter: bool = False - # Whether it leaves the sum to the attention-DP reduce_scatterv whenever that - # combine applies, as a MoE block does without any flag. - leaves_for_reduce_scatterv: bool = False - # How the producer's output is written into the residual. An arrival - # description has no producer object; its edge declares capabilities only. - update: Optional[StageUpdate] = ADD - transform: Optional[OutputTransform] = None - - -class StageDecl(msgspec.Struct, frozen=True): - """A computing stage's two sides: the rows its input must be on, and what - its output is.""" - - input: StageInput - output: StageOutput - - -class DecoderLayerSides(msgspec.Struct, frozen=True): - """What a decoder layer declares about its attention, its FFN and the rows - between them; ``decoder_layer_edges`` turns it into the layer's - boundaries.""" - - # The rows the layer's input and residual arrive on. - input_rows: Layout - attention: StageDecl - ffn: StageDecl - # The rows the residual is on while the FFN runs. - ffn_residual_rows: Layout - # The rows the layer hands on. - output_rows: Layout - # The group the layer's input arrives as a partial sum over, if any. - input_owes: Optional[SumGroup] = None - # Whether the residual is added into one rank's share of the attention - # output's sum before that sum completes, instead of after it. - residual_joins_attention_sum: bool = False - - -class EdgeDecl(msgspec.Struct, frozen=True): - """One boundary between two stages, as the layer that runs one side of it - sees it: what arrives from the producer, what the consumer needs, and the - rows the residual is on before and after the boundary.""" - - produced: StageOutput - need: StageInput - residual: Layout - residual_to: Layout - # Whether the residual is added into one rank's share of the produced sum - # before that sum completes, instead of after it. - residual_joins_sum: bool = False - # Capabilities allowed to arrive from another layer, not its update object. - update_capabilities: Tuple[bool, ...] = () - - -class DecoderLayerEdges(msgspec.Struct, frozen=True): - """A decoder layer's three boundaries: from the rows the previous layer - handed on into the attention, from the attention into the FFN, and from - the FFN onto the rows the layer hands on. The next layer runs the rest of - the last one.""" - - into_attention: EdgeDecl - into_ffn: EdgeDecl - out_of_ffn: EdgeDecl - - -def decoder_layer_edges(sides: DecoderLayerSides) -> DecoderLayerEdges: - """The boundaries of a decoder layer, each with the declarations of its two - sides. What arrives at the layer is complete unless it owes a sum by - construction (``input_owes``); a sum the previous layer leaves for a batch - comes with the value.""" - owes = sides.input_owes is not None - # Cross-layer arrival describes rows and owed work. The actual update - # arrives in the stream; this model family supplies only its capability. - arrived = StageOutput( - sides.input_rows, - group=sides.input_owes, - always_leaves=owes, - update=None, - ) - # Completing what the input owes leaves it and the residual on each - # attention-TP rank's slice. - attention_rows = ( - Layout(sides.input_rows.sharded | {TokenAxis.ATTN_TP_SCATTER}) - if owes - else sides.input_rows - ) - return DecoderLayerEdges( - into_attention=EdgeDecl( - produced=arrived, - need=sides.attention.input, - residual=sides.input_rows, - residual_to=attention_rows, - update_capabilities=(sides.ffn.output.update.adds_plainly,), - ), - into_ffn=EdgeDecl( - produced=sides.attention.output, - need=sides.ffn.input, - residual=attention_rows, - residual_to=sides.ffn_residual_rows, - residual_joins_sum=sides.residual_joins_attention_sum, - ), - out_of_ffn=EdgeDecl( - produced=sides.ffn.output, - need=StageInput(sides.output_rows), - residual=sides.ffn_residual_rows, - residual_to=sides.output_rows, - ), - ) - - -def with_residual( - sides: DecoderLayerSides, residual: LayerResidual -) -> DecoderLayerSides: - """``sides`` with the reads and updates the layer declares for its - attention and its FFN.""" - replace = msgspec.structs.replace - return replace( - sides, - attention=StageDecl( - input=replace(sides.attention.input, read=residual.attention_read), - output=replace(sides.attention.output, update=residual.attention_update), - ), - ffn=StageDecl( - input=replace(sides.ffn.input, read=residual.ffn_read), - output=replace(sides.ffn.output, update=residual.ffn_update), - ), - ) - - -def stage_edges( - *, previous: Optional[StageOutput], stage: StageDecl, rows: Layout -) -> Tuple[EdgeDecl, EdgeDecl]: - """The two boundaries of a layer that is one stage of a sequence of stages: - into it from the previous stage's output (None at the start of the layer - stack) and out of it onto ``rows``, the rows every layer hands on and the - residual is on between stages. The previous output arrives on those rows: a - stage whose output is elsewhere (an FFN on the TP group) moves it back there - first, and what it may leave of its sum comes with the value; otherwise the - value is complete. The residual follows the input onto a finer slice and - stays where it is when the input is gathered.""" - adds_plainly = previous.update.adds_plainly if previous is not None else True - arrived = ( - StageOutput( - rows, - group=previous.group, - always_leaves=previous.always_leaves, - leaves_for_next_layer=previous.leaves_for_next_layer, - update=None, - ) - if previous is not None - and (previous.always_leaves or previous.leaves_for_next_layer) - else StageOutput(rows, update=None) - ) - during = stage.input.layout if rows.sharded <= stage.input.layout.sharded else rows - return ( - EdgeDecl( - produced=arrived, - need=stage.input, - residual=rows, - residual_to=during, - update_capabilities=(adds_plainly,), - ), - EdgeDecl( - produced=stage.output, - need=StageInput(rows), - residual=during, - residual_to=rows, - ), - ) - - -def sequence_parallel_layer_sides( - *, axis_sizes: Mapping[TokenAxis, int] -) -> DecoderLayerSides: - """A decoder layer while a LayerNorm SP region is active. Its linears - all-gather their input and reduce-scatter their output themselves, so the - layer's rows, its residual and both outputs stay on each TP rank's slice.""" - attention = Layout.sharded_over( - TokenAxis.ATTN_DP, TokenAxis.ATTN_CP, axis_sizes=axis_sizes - ) - local = Layout.sharded_over( - TokenAxis.ATTN_DP, - TokenAxis.ATTN_CP, - TokenAxis.ATTN_TP_SCATTER, - axis_sizes=axis_sizes, - ) - gathers = frozenset({TokenAxis.ATTN_TP_SCATTER}) - return DecoderLayerSides( - input_rows=local, - # qkv and gate_up gather into GEMM; o_proj and down reduce-scatter out. - attention=StageDecl( - input=StageInput(attention, gathers_itself=gathers, read=NORM_QUANT_READ), - output=StageOutput(local), - ), - ffn=StageDecl( - input=StageInput( - Layout.sharded_over(axis_sizes=axis_sizes), gathers_itself=gathers - ), - output=StageOutput(local), - ), - ffn_residual_rows=local, - output_rows=local, - ) - - -def scattered_residual_layer_sides( - *, - axis_sizes: Mapping[TokenAxis, int], - ffn_group: SumGroup, - is_first_layer: bool, - is_last_layer: bool, - leaves_for_reduce_scatter: bool, -) -> DecoderLayerSides: - """A decoder layer on an input-scattered batch whose residual stays on - each attention-TP rank's slice (MHC). The attention and the FFN compute on - the full rows. The attention takes the slice and gathers it itself (its - QKV hook, or the handoff before an attention that needs the rows); the - boundary brings the FFN its rows and moves both outputs back onto the - slice: the attention output's TP sum by a reduce-scatter, the FFN's too - when it leaves it (``leaves_for_reduce_scatter``). The first layer's input - is the embedding's partial sum on the full rows; the last layer hands on - the full rows again.""" - attention = Layout.sharded_over( - TokenAxis.ATTN_DP, TokenAxis.ATTN_CP, axis_sizes=axis_sizes - ) - local = Layout.sharded_over( - TokenAxis.ATTN_DP, - TokenAxis.ATTN_CP, - TokenAxis.ATTN_TP_SCATTER, - axis_sizes=axis_sizes, - ) - return DecoderLayerSides( - input_rows=attention if is_first_layer else local, - attention=StageDecl( - input=StageInput( - attention, - gathers_itself=frozenset({TokenAxis.ATTN_TP_SCATTER}), - read=NORM_QUANT_READ, - ), - output=StageOutput(attention, group=SumGroup.ATTN_TP, always_leaves=True), - ), - ffn=StageDecl( - input=StageInput(Layout.sharded_over(axis_sizes=axis_sizes)), - output=StageOutput( - attention, - group=ffn_group, - leaves_for_reduce_scatter=leaves_for_reduce_scatter, - ), - ), - ffn_residual_rows=local, - output_rows=attention if is_last_layer else local, - input_owes=SumGroup.TP if is_first_layer else None, - ) - - -def input_scattered_layer_sides( - *, - axis_sizes: Mapping[TokenAxis, int], - ffn_group: SumGroup, - hands_on_partial: bool, -) -> DecoderLayerSides: - """A decoder layer on a batch whose attention input is scattered over - attention TP. Each layer's input is a partial sum over TP on the full rows: - the embedding and every FFN before it leave that sum, and a reduce-scatter - completes it onto each rank's slice, which the attention gathers itself. - The residual comes back to the full rows inside the attention output's sum.""" - attention = Layout.sharded_over( - TokenAxis.ATTN_DP, TokenAxis.ATTN_CP, axis_sizes=axis_sizes - ) - return DecoderLayerSides( - input_rows=attention, - attention=StageDecl( - input=StageInput( - attention, - gathers_itself=frozenset({TokenAxis.ATTN_TP_SCATTER}), - read=NORM_QUANT_READ, - ), - output=StageOutput(attention, group=SumGroup.ATTN_TP, always_leaves=True), - ), - ffn=StageDecl( - input=StageInput(Layout.sharded_over(axis_sizes=axis_sizes)), - # The next input completes the sum; the last FFN completes its own. - output=StageOutput( - attention, group=ffn_group, leaves_for_reduce_scatter=hands_on_partial - ), - ), - ffn_residual_rows=attention, - output_rows=attention, - input_owes=SumGroup.TP, - residual_joins_attention_sum=True, - ) - - -def decoder_layer_sides( - *, - axis_sizes: Mapping[TokenAxis, int], - ffn_on_local_rows: bool, - previous_on_local_rows: bool, - is_last_layer: bool, - attention_gathers_local_rows: bool, - ffn_group: SumGroup, - leaves_for_next_layer: bool, - leaves_for_reduce_scatter: bool, - leaves_for_reduce_scatterv: bool, - hands_on_attention_rows: bool = False, - ffn_shards_over_cp: bool = False, -) -> DecoderLayerSides: - """An attention followed by an FFN, derived from the groups each computes - over. The FFN runs either on the TP group (a dense MLP, or a MoE not - dispatched per DP shard) or on this rank's local rows (a MoE dispatched per - DP shard, which completes its own combine, or a dense MLP on every rank). - A MoE whose data-parallel groups are the CP ranks (``ffn_shards_over_cp``) - computes each CP shard on its own ranks: its rows stay sharded over CP. - A layer on local rows hands on the attention's rows when it is the last, or - with ``hands_on_attention_rows``.""" - # Attention computes over the attention-TP ranks of one DP (and CP) shard. - attention = Layout.sharded_over( - TokenAxis.ATTN_DP, TokenAxis.ATTN_CP, axis_sizes=axis_sizes - ) - # Each attention-TP rank's own slice of those rows. - local = Layout.sharded_over( - TokenAxis.ATTN_DP, - TokenAxis.ATTN_CP, - TokenAxis.ATTN_TP_SCATTER, - axis_sizes=axis_sizes, - ) - # The TP group spans every token axis, so an FFN on it needs every row. - ffn = ( - local - if ffn_on_local_rows - else Layout.sharded_over( - *((TokenAxis.ATTN_CP,) if ffn_shards_over_cp else ()), - axis_sizes=axis_sizes, - ) - ) - attention_tp = axis_sizes[TokenAxis.ATTN_TP_SCATTER] > 1 - return DecoderLayerSides( - input_rows=local if previous_on_local_rows else attention, - attention=StageDecl( - input=StageInput( - attention, - gathers_itself=( - frozenset({TokenAxis.ATTN_TP_SCATTER}) - if attention_gathers_local_rows - else frozenset() - ), - read=NORM_QUANT_READ, - ), - # The output projection leaves the attention-TP sum to prepare_mlp. - output=StageOutput( - attention, - group=SumGroup.ATTN_TP if attention_tp else None, - always_leaves=attention_tp, - ), - ), - ffn=StageDecl( - input=StageInput(ffn), - output=StageOutput(ffn) - if ffn_on_local_rows - else StageOutput( - ffn, - group=ffn_group, - leaves_for_next_layer=leaves_for_next_layer, - leaves_for_reduce_scatter=leaves_for_reduce_scatter, - leaves_for_reduce_scatterv=leaves_for_reduce_scatterv, - ), - ), - ffn_residual_rows=local if ffn_on_local_rows else attention, - output_rows=( - local - if ffn_on_local_rows and not (is_last_layer or hands_on_attention_rows) - else attention - ), - ) - - -class FusedMlpInput(msgspec.Struct, frozen=True): - """A kernel that completes the sum the attention output owes together with - the residual add and the post-attention norm, in that order. - - ``run(hidden_states, residual, forward_batch)`` returns the FFN's input and - the residual, or None when it does not take the batch; it returns None only - before touching its inputs or starting a collective.""" - - # The group whose sum it completes. - completes: SumGroup - run: Callable[..., Optional[Tuple[torch.Tensor, torch.Tensor]]] - preserves_residual: Optional[Callable] = None - - def tbo_split_moves(layer_input_rows: Layout) -> Tuple[Callable, Callable]: """The moves around the two-batch-overlap split, which cuts the attention's rows: from the rows the first overlapped layer takes to the attention's, @@ -493,30 +73,16 @@ def tbo_split_moves(layer_input_rows: Layout) -> Tuple[Callable, Callable]: attention = Layout.sharded_over( TokenAxis.ATTN_DP, TokenAxis.ATTN_CP, axis_sizes=token_axis_sizes() ) - pair = CommunicateSummableTensorPairFn + if layer_input_rows == attention: - return pair._trivial, pair._trivial + return identity_output, identity_output if layer_input_rows.sharded - attention.sharded == {TokenAxis.ATTN_TP_SCATTER}: # Each rank's slice: write the residual in and gather over attention # TP, then take the slice of each half. - return pair._gather, pair._scatter + return update_and_gather, scatter_output raise NotImplementedError(f"{layer_input_rows=}") -class CpMoves(msgspec.Struct, frozen=True): - """How a CP extend's rows reach an FFN that needs all of them and come back, - chosen once for the kind of prefill CP: ``gather`` gathers the FFN input - after each rank has completed its own block, ``take_back`` returns this - rank's block of a complete output, and ``reduce_scatter``, where there is - one, completes a sum left over the ranks of ``reduce_scatter_group()`` and - returns the block in the same collective.""" - - gather: Callable - take_back: Callable - reduce_scatter: Optional[Callable] = None - reduce_scatter_group: Optional[Callable[[], GroupCoordinator]] = None - - def _cp_moves() -> CpMoves: """DSA and MLA CP gather equal shards over the attention-CP group and can complete a sum over it. GQA prefill CP gathers blocks padded to the longest @@ -524,85 +90,16 @@ def _cp_moves() -> CpMoves: if _gathers_over_attention_cp(): return CpMoves( gather=_mlp_input_gather_attention_cp, - take_back=CommunicateSummableTensorPairFn._take_back_attention_cp_shard, - reduce_scatter=CommunicateSummableTensorPairFn._reduce_scatter_over_cp, + take_back=take_back_attention_cp_shard, + reduce_scatter=reduce_scatter_over_cp, reduce_scatter_group=lambda: get_parallel().attn_cp_group, ) return CpMoves( gather=_mlp_input_gather_moe_cp, - take_back=CommunicateSummableTensorPairFn._scatter_hidden_states_moe, + take_back=scatter_moe_cp_output, ) -class StageEntry(msgspec.Struct, frozen=True): - """The boundary into one of a layer's stages, as the layer runs it.""" - - # Completes what the input owes, writes the previous stage's output into - # the residual and reads this stage's input: - # (hidden_states, residual, forward_batch, norm, **call). - prepare: Callable - # The rows prepare hands the stage. - input_rows: Layout - # Moves the input onto the stage's rows after prepare, when prepare does - # not: (hidden_states, forward_batch) -> hidden_states. - input_move: Optional[Callable] = None - # Hands the stage its input once it is on its rows: - # (hidden_states, forward_batch, qkv_latent_func) -> hidden_states. - handoff: Optional[Callable] = None - # The fused kernels prepare tries first. - fused: Tuple["FusedMlpInput", ...] = () - # Return the updated residual to the producer's rows for aux capture. - capture_move: Optional[Callable] = None - capture_move_allocates: bool = False - preserves_residual: Optional[Callable] = None - capture_preserves_residual: Optional[Callable] = None - input_sum: Optional[SumGroup] = None - - -class BoundarySteps(msgspec.Struct, frozen=True): - """The steps a batch runs at a layer's boundaries: into each of its - stages, and the last stage's output on to the rows the layer hands on.""" - - # The boundary into the layer's attention and into its FFN; None for a - # stage a layer that is one stage does not have. - attention: Optional[StageEntry] - ffn: Optional[StageEntry] - # What the FFN exit reads: the FFN output's group and what it may leave. - ffn_output: StageOutput - # The postprocess that moves the FFN output on; None when it goes back over - # attention DP, whose step the FFN exit and postprocess choose per batch. - ffn_output_move: Optional[Callable] - # Whether the next layer's input can take the FFN's sum. - ffn_sum_is_movable: bool - # Whether ffn_output_move also completes the sum the FFN leaves. - ffn_output_move_completes_sum: bool = False - - @property - def returns_over_dp(self) -> bool: - return self.ffn_output_move is None - - -class StageKind(Enum): - """Which of a decoder layer's two stages a layer that is one stage takes - the place of: its norm, its read and update, its fused kernels and its - entry in the layer's steps.""" - - ATTENTION = auto() - FFN = auto() - - -class LayerStage(msgspec.Struct, frozen=True): - """A layer that is one stage of a sequence of stages, each an - attention-like mixer or an FFN: which of the two it is, its two boundaries - (into it, and out of it onto the rows every layer hands on, as - ``stage_edges`` gives them), and whether the layer stack - starts at it.""" - - kind: StageKind - edges: Tuple[EdgeDecl, EdgeDecl] - enters_stack: bool = False - - class Boundary(msgspec.Struct, frozen=True): """The steps one layer runs at one boundary, chosen from both sides' declarations. A layer runs the consumer's half of a boundary into one of @@ -615,24 +112,20 @@ class Boundary(msgspec.Struct, frozen=True): prepare: Optional[Callable] = None # The move onto the consumer's rows after prepare; None when there is none. input_move: Optional[Callable] = None - # The fused kernels prepare tries first on a sum the input always owes. - fused: Tuple["FusedMlpInput", ...] = () # The producer's half: the postprocess that moves the output onto the rows # the layer hands on; None when it goes back over attention DP, whose step # the FFN exit and postprocess choose per batch. output_move: Optional[Callable] = None # Whether output_move also completes the sum the producer leaves. output_move_completes_sum: bool = False + returns_over_dp: bool = False preserves_residual: Optional[Callable] = None @property def input_rows(self) -> Layout: """The rows the consumer is handed: what it needs, still sharded over the axes it gathers itself as the rows its input is read on are.""" - need = self.edge.need - return Layout( - need.layout.sharded | (self.edge.residual_to.sharded & need.gathers_itself) - ) + return input_rows(self.edge) @property def capture_move_allocates(self) -> bool: @@ -648,12 +141,18 @@ class Boundary(msgspec.Struct, frozen=True): ) +def input_rows(edge: EdgeDecl) -> Layout: + """Consumer rows, retaining the axes it gathers inside its computation.""" + return Layout( + edge.need.layout.sharded | (edge.residual_to.sharded & edge.need.gathers_itself) + ) + + def make_boundary( edge: EdgeDecl, *, fusions: Tuple["FusedMlpInput", ...] = (), carried_fusions: Tuple[Callable, ...] = (), - force_layernorm_before_gather: bool = False, cp_moves: Optional[CpMoves] = None, enters_stack: bool = False, ) -> Boundary: @@ -674,7 +173,6 @@ def make_boundary( adds_plainly=capability, fusions=fusions, carried_fusions=carried_fusions, - force_layernorm_before_gather=force_layernorm_before_gather, cp_moves=cp_moves, enters_stack=enters_stack, ) @@ -692,9 +190,6 @@ def make_boundary( _dispatch_consumer, paths={capability: path.prepare for capability, path in paths.items()}, ), - fused=tuple( - f for f in fusions if any(f in path.fused for path in paths.values()) - ), ) @@ -704,7 +199,6 @@ def _bind_consumer( adds_plainly: bool, fusions: Tuple["FusedMlpInput", ...] = (), carried_fusions: Tuple[Callable, ...] = (), - force_layernorm_before_gather: bool = False, cp_moves: Optional[CpMoves] = None, enters_stack: bool = False, ) -> Boundary: @@ -720,28 +214,26 @@ def _bind_consumer( producer chose for a batch.""" # A fused kernel runs the add and the norm itself. plain = adds_plainly and edge.need.read.norms_plainly - step, fused, input_move = _select_input_steps( + step, input_move = _select_input_steps( edge.produced, residual=edge.residual, residual_to=edge.residual_to, need=edge.need, adds_plainly=adds_plainly, fusions=fusions if plain else (), - force_layernorm_before_gather=force_layernorm_before_gather, residual_joins_sum=edge.residual_joins_sum, cp_moves=cp_moves, enters_stack=enters_stack, ) completed_step = None if edge.produced.always_leaves: - completed_step, _, _ = _select_input_steps( + completed_step, _ = _select_input_steps( msgspec.structs.replace(edge.produced, always_leaves=False), residual=edge.residual, residual_to=edge.residual_to, need=edge.need, adds_plainly=adds_plainly, fusions=(), - force_layernorm_before_gather=force_layernorm_before_gather, residual_joins_sum=False, cp_moves=cp_moves, enters_stack=enters_stack, @@ -750,14 +242,13 @@ def _bind_consumer( # again (e.g. MHC expansion). Both alternatives are bound at construction. written_step = None if enters_stack: - written_step, _, _ = _select_input_steps( + written_step, _ = _select_input_steps( edge.produced, residual=edge.residual, residual_to=edge.residual_to, need=edge.need, adds_plainly=adds_plainly, fusions=fusions if plain else (), - force_layernorm_before_gather=force_layernorm_before_gather, residual_joins_sum=edge.residual_joins_sum, cp_moves=cp_moves, enters_stack=False, @@ -790,7 +281,6 @@ def _bind_consumer( written_step=written_step, ), input_move=input_move, - fused=fused, ) @@ -818,73 +308,15 @@ def make_output_boundary( residual=edge.residual, to=edge.residual_to, cp_moves=cp_moves, - update=edge.produced.update, ) return Boundary( edge, - output_move=None if returns_over_dp else output_move, + output_move=output_move, + returns_over_dp=returns_over_dp, output_move_completes_sum=completes_sum, ) -def _select_boundary_steps( - sides: DecoderLayerSides, - *, - fusions: Tuple["FusedMlpInput", ...] = (), - force_layernorm_before_gather: bool = False, - cp_moves: Optional[CpMoves] = None, - attention_handoff: Callable = _hand_qkv_hook_its_input, - attention_fusions: Tuple[Callable, ...] = (), - enters_stack: bool = False, -) -> BoundarySteps: - """The steps of a decoder layer: its three boundaries, each chosen from the - declarations of its two sides, around the layer's residual operations. Both - edges that cross attention CP take the same ``cp_moves``; the attention's - input tries ``attention_fusions``, and the layer stack's first layer - (``enters_stack``) starts its residual there.""" - edges = decoder_layer_edges(sides) - out_of_ffn = make_output_boundary(edges.out_of_ffn, cp_moves=cp_moves) - into_ffn = make_boundary( - edges.into_ffn, - fusions=fusions, - force_layernorm_before_gather=force_layernorm_before_gather, - cp_moves=cp_moves, - ) - into_attention = make_boundary( - edges.into_attention, - carried_fusions=attention_fusions, - force_layernorm_before_gather=force_layernorm_before_gather, - cp_moves=cp_moves, - enters_stack=enters_stack, - ) - return BoundarySteps( - attention=StageEntry( - prepare=into_attention.prepare, - input_rows=into_attention.input_rows, - input_move=into_attention.input_move, - handoff=attention_handoff, - capture_move=into_attention.capture_move, - capture_move_allocates=into_attention.capture_move_allocates, - input_sum=edges.into_attention.produced.group - if edges.into_attention.produced.always_leaves - else None, - ), - ffn=StageEntry( - prepare=into_ffn.prepare, - input_rows=into_ffn.input_rows, - input_move=into_ffn.input_move, - fused=into_ffn.fused, - input_sum=edges.into_ffn.produced.group - if edges.into_ffn.produced.always_leaves - else None, - ), - ffn_output=edges.out_of_ffn.produced, - ffn_output_move=out_of_ffn.output_move, - ffn_output_move_completes_sum=out_of_ffn.output_move_completes_sum, - ffn_sum_is_movable=edges.out_of_ffn.produced.group is not None, - ) - - def _select_input_steps( produced: StageOutput, *, @@ -893,11 +325,10 @@ def _select_input_steps( need: StageInput, adds_plainly: bool, fusions: Tuple[FusedMlpInput, ...], - force_layernorm_before_gather: bool, residual_joins_sum: bool, cp_moves: Optional[CpMoves], enters_stack: bool, -) -> Tuple[Callable, Tuple[FusedMlpInput, ...], Optional[Callable]]: +) -> Tuple[Callable, Optional[Callable]]: """The steps into a stage for a value that is complete or owes the sum its producer always leaves, the fused kernels they try first, and the move after them: complete that sum, move the residual to the rows it has while @@ -933,7 +364,6 @@ def _select_input_steps( scatters_residual=residual != residual_to, read=read, ), - (), None, ) if residual_to.sharded - produced.layout.sharded == {TokenAxis.ATTN_TP_SCATTER}: @@ -946,7 +376,6 @@ def _select_input_steps( partial( _mlp_input_on_residual_shard, read=read, reduces=owes is not None ), - (), None, ) # A reduce-scatter completes the TP sum onto each rank's slice, which @@ -965,26 +394,24 @@ def _select_input_steps( enters_stack=enters_stack, read=read, ), - (), None, ) if gathered == {TokenAxis.ATTN_CP}: # Each CP rank completes its own chunk, then the CP moves gather them. if cp_moves is None: raise NotImplementedError(f"{produced=} {need=}") - on_chunk, fused, _ = _select_input_steps( + on_chunk, _ = _select_input_steps( produced, residual=residual, residual_to=residual_to, need=StageInput(produced.layout, read=read), adds_plainly=adds_plainly, fusions=fusions, - force_layernorm_before_gather=force_layernorm_before_gather, residual_joins_sum=residual_joins_sum, cp_moves=cp_moves, enters_stack=enters_stack, ) - return partial(cp_moves.gather, gather=on_chunk), fused, None + return (partial(cp_moves.gather, gather=on_chunk), None) if gathered == {TokenAxis.ATTN_TP_SCATTER}: # A complete input on each rank's slice, gathered over attention TP # once it is read. @@ -1001,8 +428,7 @@ def _select_input_steps( enters_stack=enters_stack, read=read, ), - (), - CommunicateSimpleFn._scattered_to_tp_attn_full, + gather_attention_tp, ) if ( residual_to != produced.layout @@ -1029,7 +455,6 @@ def _select_input_steps( group=None, read=read, ), - (), None, ) return ( @@ -1039,7 +464,6 @@ def _select_input_steps( enters_stack=enters_stack, read=read, ), - (), None, ) if gathers_residual and residual_joins_sum: @@ -1047,7 +471,7 @@ def _select_input_steps( # sum, so the all-reduce also brings the residual back to every row. if owes is not SumGroup.ATTN_TP or not adds_plainly: raise NotImplementedError(f"{produced=} {residual=} {need=}") - return partial(_mlp_input_residual_into_sum, read=read), (), None + return (partial(_mlp_input_residual_into_sum, read=read), None) # A sum over TP completes only on rows every TP rank holds: the TP group # spans attention DP and CP. if owes is not SumGroup.ATTN_TP and ( @@ -1063,7 +487,6 @@ def _select_input_steps( group=owes, read=read, ), - fused, None, ) if owes not in (None, SumGroup.ATTN_TP): @@ -1077,7 +500,7 @@ def _select_input_steps( places_cp_shards = TokenAxis.ATTN_CP in gathered if ( owes_attention_tp - and not force_layernorm_before_gather + and not read.before_gather and adds_plainly and read.norms_plainly ): @@ -1088,7 +511,6 @@ def _select_input_steps( places_cp_shards=places_cp_shards, read=read, ), - (), None, ) return ( @@ -1099,7 +521,6 @@ def _select_input_steps( places_cp_shards=places_cp_shards, read=read, ), - (), None, ) @@ -1110,21 +531,21 @@ def _select_ffn_output_move( residual: Layout, to: Layout, cp_moves: Optional[CpMoves] = None, - update: StageUpdate = ADD, ) -> Tuple[bool, Optional[Callable], bool]: """How the FFN output reaches the rows the layer hands on: whether it goes back by undoing the attention-DP gather (the FFN exit and postprocess run that step), or else the postprocess that moves it, None when there is none to choose here; and whether that move also completes the sum the FFN leaves.""" - pair = CommunicateSummableTensorPairFn + + update = produced.update if produced.layout == residual: if to == residual: - return False, pair._trivial, False + return False, identity_output, False if to.sharded == residual.sharded - {TokenAxis.ATTN_TP_SCATTER}: # Each rank's slice back to the attention's rows: write the output # into the residual, then gather over attention TP. - return False, partial(pair._gather, update=update), False + return False, partial(update_and_gather, update=update), False raise NotImplementedError(f"{produced=} {residual=} {to=}") returned = residual.sharded - produced.layout.sharded if returned == {TokenAxis.ATTN_TP_SCATTER} and to in (residual, produced.layout): @@ -1135,7 +556,7 @@ def _select_ffn_output_move( return ( False, partial( - pair._onto_residual_shard, + output_on_residual_shard, sums=sums, gathers_back=to != residual, update=update, @@ -1159,7 +580,7 @@ def _select_ffn_output_move( return False, cp_moves.reduce_scatter, True if returned == {TokenAxis.ATTN_DP, TokenAxis.ATTN_CP}: # This rank's CP shard, from where the DP gather put it. - return False, CommunicateSummableTensorPairFn._take_back_cp_shard, False + return False, take_back_cp_shard, False if returned != {TokenAxis.ATTN_DP}: raise NotImplementedError(f"{produced=} {residual=} {to=}") return True, None, False diff --git a/python/sglang/srt/layers/communicator/construction.py b/python/sglang/srt/layers/communicator/construction.py index 91ca1f04d7..f740d7a336 100644 --- a/python/sglang/srt/layers/communicator/construction.py +++ b/python/sglang/srt/layers/communicator/construction.py @@ -14,22 +14,25 @@ """Bind one stage to predeclared boundaries and reusable batch paths.""" from dataclasses import dataclass -from enum import Enum, auto from functools import cached_property -from typing import Callable, Optional, Protocol, Tuple +from typing import Callable, Optional from sglang.srt.environ import envs from sglang.srt.layers.communicator.adapters.attention import get_attn_tp_context from sglang.srt.layers.communicator.boundary import ( - BoundarySteps, - EdgeDecl, - FusedMlpInput, - StageDecl, - StageEntry, - StageKind, + _cp_moves, make_boundary, make_output_boundary, ) +from sglang.srt.layers.communicator.contracts import ( + BatchVariant, + CpMoves, + EdgeDecl, + ProducerReduction, + StageEntry, + StageKind, + StageSteps, +) from sglang.srt.layers.communicator.exit import OutputBoundary from sglang.srt.layers.communicator.fusions.allreduce import ( attention_fusions, @@ -42,11 +45,14 @@ from sglang.srt.layers.communicator.layout import ( _generic_prefill_cp_shards_tokens, enable_moe_dense_fully_dp, ) +from sglang.srt.layers.communicator.prepare import ( + _hand_qkv_hook_its_input, + _hand_scattered_input_to_attention, +) from sglang.srt.layers.communicator.stage import StageCommunicator from sglang.srt.layers.moe import ( get_moe_a2a_backend, ) -from sglang.srt.model_executor.forward_batch_info import ForwardBatch from sglang.srt.runtime_context import ( get_forward, get_lora, @@ -58,24 +64,6 @@ from sglang.srt.speculative.spec_info import SpeculativeAlgorithm _use_ag_after_qlora = envs.SGLANG_USE_AG_AFTER_QLORA.get() -class BoundaryFusions(Protocol): - """The fused kernels a fusion backend gives a layer, bound to it and tried - before the layer's own: at its attention input (each takes what the previous - layer left, the residual, the batch and a post-residual addition), at its - FFN input, and at its FFN exit (each says what the exit does when its kernel - takes the batch).""" - - def attention_input(self, layer: "StagePlan") -> Tuple[Callable, ...]: ... - - def ffn_input(self, layer: "StagePlan") -> Tuple["FusedMlpInput", ...]: ... - - requires_local_reduction: bool - - def can_defer_finalize( - self, layer: "StagePlan", forward_batch: ForwardBatch - ) -> bool: ... - - def _refuse_uncovered_cp_moe(moe_on_local_rows: bool, cp_shards: bool) -> None: """A MoE layer under attention CP whose tokens the steps cannot bring to it: under a prefill CP, one dispatched per DP shard under attention DP @@ -118,13 +106,6 @@ def _input_can_be_scattered() -> bool: ) -class BatchVariant(Enum): - ORDINARY = auto() - CONTEXT_PARALLEL = auto() - INPUT_SCATTERED = auto() - SEQUENCE_PARALLEL = auto() - - @dataclass(frozen=True) class StageEdges: """One stage's incoming and outgoing declarations for one batch variant. @@ -136,7 +117,7 @@ class StageEdges: incoming: EdgeDecl outgoing: EdgeDecl handoff: Optional[Callable] = None - cp_moves: object = None + cp_moves: Optional[CpMoves] = None def _requires_branch_input(*args, **kwargs): @@ -155,26 +136,16 @@ class StagePlan: enters_stack=False, prepared_input=False, terminal=False, - fixed_output=False, - is_sparse=False, + direct_handoff=False, qkv_latent_func=None, - force_layernorm_before_dp_gather=False, - enable_fused_ar_quant=False, - fused_ar_quant_keep_bf16=False, - residual_in_hidden=False, fusions=None, ): - self.kind = kind self.norm = norm self.variants = dict(variants) - self.is_first_layer = enters_stack - self.is_last_layer = terminal - self.fixed_output = fixed_output - self.is_sparse = is_sparse + self.enters_stack = enters_stack + self.terminal = terminal + self.direct_handoff = direct_handoff self.qkv_latent_func = qkv_latent_func - self.enable_fused_ar_quant = enable_fused_ar_quant - self.fused_ar_quant_keep_bf16 = fused_ar_quant_keep_bf16 - self.residual_in_hidden = residual_in_hidden self.fusions = fusions self._speculative_algo = SpeculativeAlgorithm.from_string( get_spec().speculative_algorithm @@ -183,8 +154,13 @@ class StagePlan: get_lora().enable_lora ) self._fusion_rows = None - carried = attention_fusions(self) if kind is StageKind.ATTENTION else () - self._carried_fusions = carried + carried = ( + attention_fusions( + self, next(iter(self.variants.values())).incoming.need.read + ) + if kind is StageKind.ATTENTION + else () + ) fused = ffn_fusions(self) if kind is StageKind.FFN else () self._paths = {} for variant, edges in self.variants.items(): @@ -200,7 +176,6 @@ class StagePlan: edges.incoming, fusions=fused, carried_fusions=carried, - force_layernorm_before_gather=force_layernorm_before_dp_gather, cp_moves=edges.cp_moves, enters_stack=enters_stack, ) @@ -209,7 +184,6 @@ class StagePlan: input_rows=into.input_rows, input_move=into.input_move, handoff=edges.handoff, - fused=into.fused, capture_move=into.capture_move, capture_move_allocates=into.capture_move_allocates, preserves_residual=into.preserves_residual, @@ -219,55 +193,54 @@ class StagePlan: ) out = ( None - if fixed_output + if direct_handoff else make_output_boundary(edges.outgoing, cp_moves=edges.cp_moves) ) - self._paths[variant] = BoundarySteps( - attention=entry if kind is StageKind.ATTENTION else None, - ffn=entry if kind is StageKind.FFN else None, - ffn_output=edges.outgoing.produced, - ffn_output_move=None if out is None else out.output_move, - ffn_output_move_completes_sum=False + self._paths[variant] = StageSteps( + entry=entry, + output=edges.outgoing.produced, + output_move=None if out is None else out.output_move, + returns_over_dp=out is not None and out.returns_over_dp, + output_move_completes_sum=False if out is None else out.output_move_completes_sum, - ffn_sum_is_movable=edges.outgoing.produced.group is not None, ) - self._steps = self._paths[BatchVariant.ORDINARY] - self._cp_steps = self._paths.get(BatchVariant.CONTEXT_PARALLEL) - self._input_scattered_steps = self._paths.get(BatchVariant.INPUT_SCATTERED) - self._sp_steps = self._paths.get(BatchVariant.SEQUENCE_PARALLEL) @property - def input_rows(self): + def incoming_residual_rows(self): return self.variants[BatchVariant.ORDINARY].incoming.residual @property def input_on_attention_tp_slices(self): - return TokenAxis.ATTN_TP_SCATTER in self.input_rows.sharded + return TokenAxis.ATTN_TP_SCATTER in self.incoming_residual_rows.sharded def _variant(self, forward_batch): - if self._sp_steps is not None and get_forward().sp_active: + # The batch determines its rows; missing paths must not change them. + if get_forward().sp_active: return BatchVariant.SEQUENCE_PARALLEL - if ( - self._input_scattered_steps is not None - and get_attn_tp_context().input_scattered - ): + if get_attn_tp_context().input_scattered: return BatchVariant.INPUT_SCATTERED - if self._cp_steps is not None and _batch_shards_over_cp(forward_batch): + if _batch_shards_over_cp(forward_batch): return BatchVariant.CONTEXT_PARALLEL return BatchVariant.ORDINARY def _batch_steps(self, forward_batch): - return self._paths[self._variant(forward_batch)] + variant = self._variant(forward_batch) + try: + return self._paths[variant] + except KeyError: + raise NotImplementedError( + f"no stage boundary path for the active {variant.name} batch" + ) from None def fusion_rows(self, forward_batch): if self._fusion_rows is not None: return self._fusion_rows[self._variant(forward_batch)] entry = self._batch_steps(forward_batch) - return (entry.attention or entry.ffn).input_rows + return entry.entry.input_rows - def produced(self, kind, forward_batch): - return self._batch_steps(forward_batch).ffn_output + def produced(self, forward_batch): + return self._batch_steps(forward_batch).output @cached_property def output(self): @@ -284,24 +257,47 @@ class StagePlan: ) -def make_stage( - declaration: StageDecl, - *, - kind: StageKind, - norm, - incoming: EdgeDecl, - outgoing: EdgeDecl, - variants=None, - handoff=None, - **options, -) -> StageCommunicator: - """Build one real stage from its own declaration and adjacent edges. +def _bind_stage(declaration, norm, incoming, outgoing, **options): + if incoming.consumer != declaration or outgoing.producer != declaration: + raise ValueError("connections do not match the stage declaration") + if incoming.entries.keys() != outgoing.exits.keys(): + raise ValueError("incoming and outgoing batch variants disagree") + variants = {} + for variant, edge in incoming.entries.items(): + if ( + declaration.update.at_producer + and TokenAxis.ATTN_CP + in edge.produced.layout.sharded - edge.need.layout.sharded + ): + raise NotImplementedError("MHC with a gather over attention CP") + handoff = None + if declaration.kind is StageKind.ATTENTION: + handoff = ( + _hand_scattered_input_to_attention + if variant is BatchVariant.INPUT_SCATTERED + and not declaration.update.adds_plainly + else _hand_qkv_hook_its_input + ) + moves = _cp_moves() if variant is BatchVariant.CONTEXT_PARALLEL else None + variants[variant] = StageEdges(edge, outgoing.exits[variant], handoff, moves) + plan = StagePlan( + declaration.kind, + norm, + variants, + enters_stack=incoming.producer is None, + prepared_input=declaration.prepared_from is not None, + terminal=declaration.terminal, + direct_handoff=declaration.kind is StageKind.ATTENTION + and declaration.reduction is ProducerReduction.PARTIAL, + **options, + ) + if declaration.kind is StageKind.ATTENTION and outgoing.consumer is not None: + # Layout eligibility comes from the connected consumer, not a mutable + # link to its execution plan. Kernel binding remains consumer-owned. + from sglang.srt.layers.communicator.boundary import input_rows - The optional variants supply the same stage's CP/SP/input-scattered - declarations. Neither a paired stage nor its layer facts/norm is needed. - """ - if incoming.need != declaration.input or outgoing.produced != declaration.output: - raise ValueError("stage declaration disagrees with its boundaries") - paths = dict(variants or {}) - paths[BatchVariant.ORDINARY] = StageEdges(incoming, outgoing, handoff) - return StageCommunicator(StagePlan(kind, norm, paths, **options), kind, norm) + plan._fusion_rows = { + v: input_rows(edge) for v, edge in outgoing.entries.items() + } + + return StageCommunicator(plan, declaration=declaration) diff --git a/python/sglang/srt/layers/communicator/contracts.py b/python/sglang/srt/layers/communicator/contracts.py new file mode 100644 index 0000000000..e9c1d3d23b --- /dev/null +++ b/python/sglang/srt/layers/communicator/contracts.py @@ -0,0 +1,189 @@ +# Copyright 2023-2024 SGLang Team +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Immutable stage declarations and bound input/output paths.""" + +from enum import Enum, auto +from typing import Callable, FrozenSet, Optional, Tuple + +import msgspec +import torch + +from sglang.srt.distributed import GroupCoordinator +from sglang.srt.layers.communicator.layout import Layout, SumGroup, TokenAxis +from sglang.srt.layers.communicator.output import OutputTransform +from sglang.srt.layers.communicator.residual import StageRead, StageUpdate +from sglang.srt.layers.communicator.residual.add_norm import ADD, NORM_READ + + +class ProducerReduction(Enum): + """Whether compute always leaves its sum or follows the boundary scope.""" + + PARTIAL = auto() + SCOPED = auto() + # A replicated component is added after the sum inside compute. + LOCAL_TAIL = auto() + + +class HandoffRows(Enum): + """A required layer/branch handoff, independent of compute's input rows.""" + + ATTENTION = auto() + TBO_SPLIT = auto() + + +class BatchVariant(Enum): + ORDINARY = auto() + CONTEXT_PARALLEL = auto() + INPUT_SCATTERED = auto() + SEQUENCE_PARALLEL = auto() + + +class StageInput(msgspec.Struct, frozen=True): + """The rows a stage's consumer needs: sharded over the token axes its + compute group does not span.""" + + layout: Layout + # Token axes the consumer gathers over itself when its input arrives + # sharded over them. + gathers_itself: FrozenSet[TokenAxis] = frozenset() + # How the consumer reads its input from the residual. + read: StageRead = NORM_READ + + +class StageOutput(msgspec.Struct, frozen=True): + """What a stage's producer hands the boundary after it, fixed at + construction: its rows, the group its output is summed over, and when it + leaves that sum to the boundary instead of completing it. + + What one output actually owes is carried with the value: a plain + tensor handed across layers is complete, an ``UnreducedOutput`` names what + is left. A raw compute output entering a boundary is read with this + declaration and the boundary's decision for the batch.""" + + layout: Layout + # None when there is nothing to sum. + group: Optional[SumGroup] = None + # The producer never reduces its output, e.g. a row-parallel projection + # built with reduce_results=False. + always_leaves: bool = False + # Otherwise it leaves the sum only when the boundary publishes the flag for + # it: fuse_mlp_allreduce, or mlp_reduce_scatter. + leaves_for_next_layer: bool = False + leaves_for_reduce_scatter: bool = False + # Whether it leaves the sum to the attention-DP reduce_scatterv whenever that + # combine applies, as a MoE block does without any flag. + leaves_for_reduce_scatterv: bool = False + # How the producer's output is written into the residual. An arrival + # description has no producer object; its edge declares capabilities only. + update: Optional[StageUpdate] = ADD + transform: Optional[OutputTransform] = None + + +class StageDecl(msgspec.Struct, frozen=True): + """A computing stage's two sides: the rows its input must be on, and what + its output is.""" + + input: StageInput + output: StageOutput + + +class EdgeDecl(msgspec.Struct, frozen=True): + """One boundary between two stages, as the layer that runs one side of it + sees it: what arrives from the producer, what the consumer needs, and the + rows the residual is on before and after the boundary.""" + + produced: StageOutput + need: StageInput + residual: Layout + residual_to: Layout + # Whether the residual is added into one rank's share of the produced sum + # before that sum completes, instead of after it. + residual_joins_sum: bool = False + # Capabilities allowed to arrive from another layer, not its update object. + update_capabilities: Tuple[bool, ...] = () + + +class FusedMlpInput(msgspec.Struct, frozen=True): + """A kernel that completes the sum the attention output owes together with + the residual add and the post-attention norm, in that order. + + ``run(hidden_states, residual, forward_batch)`` returns the FFN's input and + the residual, or None when it does not take the batch; it returns None only + before touching its inputs or starting a collective.""" + + # The group whose sum it completes. + completes: SumGroup + run: Callable[..., Optional[Tuple[torch.Tensor, torch.Tensor]]] + # Callable(residual, forward_batch): True guarantees this candidate is + # selected and does not mutate residual, including backend fallback. + preserves_residual: Optional[Callable] = None + + +class CpMoves(msgspec.Struct, frozen=True): + """How a CP extend's rows reach an FFN that needs all of them and come back, + chosen once for the kind of prefill CP: ``gather`` gathers the FFN input + after each rank has completed its own block, ``take_back`` returns this + rank's block of a complete output, and ``reduce_scatter``, where there is + one, completes a sum left over the ranks of ``reduce_scatter_group()`` and + returns the block in the same collective.""" + + gather: Callable + take_back: Callable + reduce_scatter: Optional[Callable] = None + reduce_scatter_group: Optional[Callable[[], GroupCoordinator]] = None + + +class StageEntry(msgspec.Struct, frozen=True): + """The boundary into one of a layer's stages, as the layer runs it.""" + + # Completes what the input owes, writes the previous stage's output into + # the residual and reads this stage's input: + # (hidden_states, residual, forward_batch, norm, **call). + prepare: Callable + # The rows prepare hands the stage. + input_rows: Layout + # Moves the input onto the stage's rows after prepare, when prepare does + # not: (hidden_states, forward_batch) -> hidden_states. + input_move: Optional[Callable] = None + # Hands the stage its input once it is on its rows: + # (hidden_states, forward_batch, qkv_latent_func) -> hidden_states. + handoff: Optional[Callable] = None + # Return the updated residual to the producer's rows for aux capture. + capture_move: Optional[Callable] = None + capture_move_allocates: bool = False + input_sum: Optional[SumGroup] = None + preserves_residual: Optional[Callable] = None + # Bound from a local successor at construction; no runtime plan lookup. + capture_preserves_residual: Optional[Callable] = None + + +class StageSteps(msgspec.Struct, frozen=True): + """One batch variant of one stage: its input and output boundary paths.""" + + entry: StageEntry + output: StageOutput + # None means no fixed move. returns_over_dp selects batch-dependent DP transport. + output_move: Optional[Callable] + output_move_completes_sum: bool = False + + returns_over_dp: bool = False + + +class StageKind(Enum): + """Which of a decoder layer's two stages a layer that is one stage takes + the place of: its norm, its read and update, its fused kernels and its + entry in the layer's steps.""" + + ATTENTION = auto() + FFN = auto() diff --git a/python/sglang/srt/layers/communicator/exit.py b/python/sglang/srt/layers/communicator/exit.py index 4f8dbe2093..a55b5cd0b9 100644 --- a/python/sglang/srt/layers/communicator/exit.py +++ b/python/sglang/srt/layers/communicator/exit.py @@ -16,7 +16,7 @@ from __future__ import annotations from functools import partial -from typing import Callable, Optional, Tuple, Union +from typing import Callable, Optional, Tuple import msgspec import torch @@ -51,11 +51,13 @@ from sglang.srt.layers.dp_attention import ( ) from sglang.srt.layers.moe import ( can_merge_post_experts_all_reduce, - get_moe_a2a_backend, - post_experts_reduction_group, post_experts_sum_is_one_all_reduce, should_use_dp_reduce_scatterv, ) +from sglang.srt.layers.moe.utils import ( + get_moe_a2a_backend, + post_experts_reduction_group, +) from sglang.srt.model_executor.forward_batch_info import ForwardBatch from sglang.srt.runtime_context import ( get_exec, @@ -95,32 +97,31 @@ class OutputBoundary: def postprocess_layer( self, hidden_states: torch.Tensor, - residual: torch.Tensor, + stream: ResidualStream, forward_batch: ForwardBatch, ): """Move a complete output from the operation-scheduled producer path.""" - stream = residual if isinstance(residual, ResidualStream) else None + steps = self.plan._batch_steps(forward_batch) hidden_states, residual = self._complete_ffn_output_now( hidden_states, - stream.residual if stream is not None else residual, + stream.residual, forward_batch=forward_batch, dp_step=None, + steps=steps, ) return self._leave_ffn_output( hidden_states, residual, stream, - self.plan._batch_steps(forward_batch).ffn_output.update, + steps.output.update, ) @staticmethod def _leave_ffn_output(hidden_states, residual, stream, update, declared_sum=None): - if stream is None: - return hidden_states, residual stream.residual = residual if update.at_producer: - return stream.write(hidden_states), stream - return stream.leave(hidden_states, update, declared_sum=declared_sum), stream + return stream.write(hidden_states) + return stream.leave(hidden_states, update, declared_sum=declared_sum) @staticmethod def _declared_ffn_sum(steps, skipped_reduction): @@ -129,74 +130,69 @@ class OutputBoundary: if ( skipped_reduction and not steps.returns_over_dp - and not steps.ffn_output_move_completes_sum + and not steps.output_move_completes_sum ): return SumGroup.TP return None - def _local_token_move_can_go_to_next_layer( - self, forward_batch: ForwardBatch - ) -> bool: + def _local_token_move_can_go_to_next_layer(self, steps) -> bool: """Whether the next layer's input can run this layer's move of its FFN output back to this rank's tokens: the base postprocess scatter, when the next layer's input also writes the output into the residual.""" - steps = self.plan._batch_steps(forward_batch) - return steps.returns_over_dp and not steps.ffn_output.update.at_producer + return steps.returns_over_dp and not steps.output.update.at_producer def _postprocess_dp_step( - self, forward_batch: ForwardBatch + self, forward_batch: ForwardBatch, steps ) -> Optional[Callable[[torch.Tensor, torch.Tensor, ForwardBatch], None]]: """The reduce-scatter that brings this layer's FFN output back to this rank's tokens under attention DP; None when the base postprocess would only scatter, or does not move tokens.""" - steps = self.plan._batch_steps(forward_batch) if not steps.returns_over_dp: return None return _reduce_and_redistribute_output_step( forward_batch, - leaves_for_reduce_scatter=steps.ffn_output.leaves_for_reduce_scatter, - leaves_for_reduce_scatterv=steps.ffn_output.leaves_for_reduce_scatterv, + leaves_for_reduce_scatter=steps.output.leaves_for_reduce_scatter, + leaves_for_reduce_scatterv=steps.output.leaves_for_reduce_scatterv, ) def _ffn_leaves_sum_to_reduce_scatter( - self, forward_batch: ForwardBatch, dp_step: Optional[Callable] + self, steps, dp_step: Optional[Callable] ) -> bool: """Whether the FFN leaves its sum out because a reduce-scatter completes it: the attention-DP one ``dp_step`` names, or the CP / input-scattered one.""" - steps = self.plan._batch_steps(forward_batch) if dp_step is not None: return True - if not steps.ffn_output.leaves_for_reduce_scatter: + if not steps.output.leaves_for_reduce_scatter: return False - if steps.ffn_output_move_completes_sum: + if steps.output_move_completes_sum: return True - return get_attn_tp_context().input_scattered and not self.plan.is_last_layer + return get_attn_tp_context().input_scattered and not self.plan.terminal - def ffn_reduction_group(self, forward_batch: ForwardBatch) -> GroupCoordinator: + def ffn_reduction_group(self, steps) -> GroupCoordinator: """The group this layer's FFN output owes its sum over: the MoE output's group on a sparse layer, the TP group a dense MLP reduces over.""" - return _sum_group(self.plan._batch_steps(forward_batch).ffn_output.group) + return _sum_group(steps.output.group) - def _select_ffn_completion(self, forward_batch: ForwardBatch) -> FfnCompletion: + def _select_ffn_completion( + self, forward_batch: ForwardBatch, steps + ) -> FfnCompletion: """Decide once, before the FFN runs, what it skips and what completes its output: the next layer's input, or this layer's postprocess step.""" - dp_step = self._postprocess_dp_step(forward_batch) - mlp_reduce_scatter = self._ffn_leaves_sum_to_reduce_scatter( - forward_batch, dp_step - ) + dp_step = self._postprocess_dp_step(forward_batch, steps) + mlp_reduce_scatter = self._ffn_leaves_sum_to_reduce_scatter(steps, dp_step) complete_now = partial( self._complete_ffn_output_now, forward_batch=forward_batch, dp_step=dp_step, + steps=steps, ) - steps = self.plan._batch_steps(forward_batch) defer_moe_finalize = ( self.plan.fusions is not None and self.plan.fusions.can_defer_finalize(self.plan, forward_batch) ) - if not steps.ffn_output.leaves_for_next_layer and not ( - self.plan.is_last_layer and defer_moe_finalize + if not steps.output.leaves_for_next_layer and not ( + self.plan.terminal and defer_moe_finalize ): return FfnCompletion( defer_moe_finalize=False, @@ -207,10 +203,10 @@ class OutputBoundary: # Producers declare remaining work independently of the kernel chosen # by the consumer. Every handoff also carries an unfused completion. fuse_mlp_allreduce = defer_moe_finalize or self._ffn_sum_moves_to_next_layer( - forward_batch, mlp_reduce_scatter=mlp_reduce_scatter, dp_step=dp_step + forward_batch, steps, mlp_reduce_scatter=mlp_reduce_scatter, dp_step=dp_step ) if fuse_mlp_allreduce: - group = self.ffn_reduction_group(forward_batch) + group = self.ffn_reduction_group(steps) if steps.returns_over_dp: # Under attention DP the next layer also brings the sum back to # this rank's tokens. @@ -225,8 +221,8 @@ class OutputBoundary: complete = partial(_leave_to_next_layer, wrap) elif ( dp_step is not None - and not self.plan.is_last_layer - and self._local_token_move_can_go_to_next_layer(forward_batch) + and not self.plan.terminal + and self._local_token_move_can_go_to_next_layer(steps) ): complete = partial( _leave_to_next_layer, @@ -253,51 +249,48 @@ class OutputBoundary: *, forward_batch: ForwardBatch, dp_step: Optional[Callable], + steps, ) -> Tuple[torch.Tensor, torch.Tensor]: """This layer's postprocess, run with the attention-DP step already chosen: the move back to where the next layer reads the FFN output, then the write-back into the residual for a layer that does it itself.""" - steps = self.plan._batch_steps(forward_batch) - if steps.ffn_output.transform is not None: - hidden_states = steps.ffn_output.transform.apply(hidden_states) + if steps.output.transform is not None: + hidden_states = steps.output.transform.apply(hidden_states) if steps.returns_over_dp: hidden_states = _to_local_tokens( dp_step or _redistribute_output, forward_batch, hidden_states ) else: - hidden_states, residual = steps.ffn_output_move( + hidden_states, residual = steps.output_move( hidden_states=hidden_states, residual=residual, forward_batch=forward_batch, ) - update = steps.ffn_output.update + update = steps.output.update if residual is not None and update.at_producer: hidden_states = update.update(hidden_states, residual) residual = None return hidden_states, residual - def mixer_exit(self, forward_batch: ForwardBatch) -> MixerExit: + def mixer_exit( + self, forward_batch: ForwardBatch, *, stream: ResidualStream + ) -> MixerExit: """Decide once whether this stage's mixer (an attention-like stage) skips its output all-reduce. Use the result as a context manager around the mixer, then call ``finish``.""" - return MixerExit(self, forward_batch) + return MixerExit(self, forward_batch, stream=stream) - def ffn_exit(self, forward_batch: ForwardBatch) -> FfnExit: + def ffn_exit( + self, forward_batch: ForwardBatch, *, stream: ResidualStream + ) -> FfnExit: """Decide once how this layer's FFN output reduction completes. Use the result as a context manager around the FFN call, then call ``finish``.""" - return FfnExit(self, forward_batch) + return FfnExit(self, forward_batch, stream=stream) - def _ffn_sum_can_move_to_next_layer(self, forward_batch: ForwardBatch) -> bool: + def _ffn_sum_can_move_to_next_layer(self, steps) -> bool: # Under the MoE-CP all-gather the fusion path would skip postprocess_layer # and its MoE-CP scatter, leaving hidden_states longer than the residual. - if ( - ( - self.plan.fusions is not None - and self.plan.fusions.requires_local_reduction - ) - or is_enable_moe_cp_allgather() - or not self.plan._batch_steps(forward_batch).ffn_sum_is_movable - ): + if is_enable_moe_cp_allgather() or steps.output.group is None: return False # The fused residual+LN reduces over a single group. Hybrid EP+TP spans @@ -324,6 +317,7 @@ class OutputBoundary: def _ffn_sum_moves_to_next_layer( self, forward_batch: ForwardBatch, + steps, *, mlp_reduce_scatter: bool, dp_step: Optional[Callable], @@ -333,8 +327,8 @@ class OutputBoundary: same all-reduce the FFN itself would have.""" return ( get_parallel().tp_size > 1 - and not self.plan.is_last_layer - and self._ffn_sum_can_move_to_next_layer(forward_batch) + and not self.plan.terminal + and self._ffn_sum_can_move_to_next_layer(steps) and _can_defer_ffn_reduction(forward_batch, self.plan) and not mlp_reduce_scatter # Under attention DP the next layer must also run postprocess's @@ -342,7 +336,7 @@ class OutputBoundary: and ( not is_dp_attention_enabled() or ( - self._local_token_move_can_go_to_next_layer(forward_batch) + self._local_token_move_can_go_to_next_layer(steps) and dp_step is None ) ) @@ -420,16 +414,23 @@ class MixerExit: "_stream", ) - def __init__(self, communicator: OutputBoundary, forward_batch: ForwardBatch): - produced = communicator.plan._batch_steps(forward_batch).ffn_output - self._stream = None + def __init__( + self, + boundary: OutputBoundary, + forward_batch: ForwardBatch, + *, + stream: ResidualStream, + ): + steps = boundary.plan._batch_steps(forward_batch) + produced = steps.output + self._stream = stream self._update = produced.update self._declared_sum = produced.group if produced.always_leaves else None self._hands_on = ( produced.leaves_for_next_layer and get_parallel().tp_size > 1 and not is_dp_attention_enabled() - and communicator._ffn_sum_can_move_to_next_layer(forward_batch) + and boundary._ffn_sum_can_move_to_next_layer(steps) ) self.skips_reduction = produced.always_leaves or self._hands_on self._scope = get_forward().scoped(fuse_mlp_allreduce=self.skips_reduction) @@ -441,21 +442,15 @@ class MixerExit: def __exit__(self, *exc_info): return self._scope.__exit__(*exc_info) - def finish(self, hidden_states: torch.Tensor, residual=None): - """The mixer's output: its partial sum to the FFN stage after it, as a - value that owes the sum to a mixer after it, or complete.""" - if self._stream is not None: - residual = self._stream + def finish(self, hidden_states: torch.Tensor): + """Hand the actual mixer result to the next boundary through the stream.""" if self._hands_on: hidden_states = UnreducedOutput( hidden_states, group=get_parallel().tp_group ) - if isinstance(residual, ResidualStream): - output = residual.leave( - hidden_states, self._update, declared_sum=self._declared_sum - ) - return output if self._stream is not None else (output, residual) - return hidden_states + return self._stream.leave( + hidden_states, self._update, declared_sum=self._declared_sum + ) class FfnExit: @@ -464,8 +459,7 @@ class FfnExit: ``defer_moe_finalize`` on ``get_forward()``.""" __slots__ = ( - "communicator", - "forward_batch", + "boundary", "defer_moe_finalize", "fuse_mlp_allreduce", "mlp_reduce_scatter", @@ -476,18 +470,23 @@ class FfnExit: "_stream", ) - def __init__(self, communicator: OutputBoundary, forward_batch: ForwardBatch): - self._stream = None - self.communicator = communicator - self.forward_batch = forward_batch - completion = communicator._select_ffn_completion(forward_batch) + def __init__( + self, + boundary: OutputBoundary, + forward_batch: ForwardBatch, + *, + stream: ResidualStream, + ): + self._stream = stream + self.boundary = boundary + steps = boundary.plan._batch_steps(forward_batch) + completion = boundary._select_ffn_completion(forward_batch, steps) self.defer_moe_finalize = completion.defer_moe_finalize self.fuse_mlp_allreduce = completion.fuse_mlp_allreduce self.mlp_reduce_scatter = completion.mlp_reduce_scatter self._complete = completion.complete - steps = communicator.plan._batch_steps(forward_batch) - self._update = steps.ffn_output.update - self._declared_sum = communicator._declared_ffn_sum( + self._update = steps.output.update + self._declared_sum = boundary._declared_ffn_sum( steps, completion.mlp_reduce_scatter ) self._scope = get_forward().scoped( @@ -503,21 +502,14 @@ class FfnExit: def __exit__(self, *exc_info): return self._scope.__exit__(*exc_info) - def finish( - self, hidden_states: torch.Tensor, residual: torch.Tensor = None - ) -> Tuple[Union[torch.Tensor, UnreducedOutput], torch.Tensor]: - """Leave the reduction to the next layer's input, or postprocess.""" - if self._stream is not None: - residual = self._stream - stream = residual if isinstance(residual, ResidualStream) else None - if stream is not None: - residual = stream.residual + def finish(self, hidden_states: torch.Tensor): + """Complete or carry this output, preserving its producer update.""" + residual = self._stream.residual if not isinstance(hidden_states, torch.Tensor): - # A deferred MoE finalize handoff, consumed by the next prepare_attn. + # A deferred MoE finalize handoff, consumed by the next attention input. assert self.defer_moe_finalize, "unrequested deferred MoE handoff" else: hidden_states, residual = self._complete(hidden_states, residual) - result = self.communicator._leave_ffn_output( - hidden_states, residual, stream, self._update, self._declared_sum + return self.boundary._leave_ffn_output( + hidden_states, residual, self._stream, self._update, self._declared_sum ) - return result[0] if self._stream is not None else result diff --git a/python/sglang/srt/layers/communicator/factories.py b/python/sglang/srt/layers/communicator/factories.py index 03038f9772..1babf5e10d 100644 --- a/python/sglang/srt/layers/communicator/factories.py +++ b/python/sglang/srt/layers/communicator/factories.py @@ -9,21 +9,26 @@ from typing import Mapping, Optional import msgspec from sglang.srt.layers import layernorm_sp -from sglang.srt.layers.communicator.boundary import ( +from sglang.srt.layers.communicator.adapters.overlap import ( + resolve_handoff_rows, + tbo_handoff, +) +from sglang.srt.layers.communicator.boundary import _cp_moves +from sglang.srt.layers.communicator.construction import ( + BatchVariant, + _bind_stage, + _input_can_be_scattered, + _refuse_uncovered_cp_moe, + _use_ag_after_qlora, +) +from sglang.srt.layers.communicator.contracts import ( EdgeDecl, + HandoffRows, + ProducerReduction, StageDecl, StageInput, StageKind, StageOutput, - _cp_moves, -) -from sglang.srt.layers.communicator.construction import ( - BatchVariant, - StageEdges, - StagePlan, - _input_can_be_scattered, - _refuse_uncovered_cp_moe, - _use_ag_after_qlora, ) from sglang.srt.layers.communicator.layout import ( Layout, @@ -35,10 +40,6 @@ from sglang.srt.layers.communicator.layout import ( sparse_moe_gathers_over_moe_cp, token_axis_sizes, ) -from sglang.srt.layers.communicator.ops import ( - _hand_qkv_hook_its_input, - _hand_scattered_input_to_attention, -) from sglang.srt.layers.communicator.output import OutputTransform from sglang.srt.layers.communicator.residual import StageRead, StageUpdate from sglang.srt.layers.communicator.residual.add_norm import ( @@ -46,15 +47,12 @@ from sglang.srt.layers.communicator.residual.add_norm import ( NORM_QUANT_READ, NORM_READ, ) -from sglang.srt.layers.communicator.stage import StageCommunicator from sglang.srt.layers.moe import is_moe_input_scattered_across_dp_ranks from sglang.srt.runtime_context import get_exec, get_parallel -def _variants(*, ordinary_only=False): +def _variants(): yield BatchVariant.ORDINARY - if ordinary_only: - return if _generic_prefill_cp_shards_tokens(): yield BatchVariant.CONTEXT_PARALLEL if _input_can_be_scattered(): @@ -79,27 +77,31 @@ def _ffn_decl( *, sparse, terminal=False, - next_sparse=False, - output=None, + output_transform=None, read=NORM_READ, update=ADD, - local_dense=True, + dense_tp_size=None, + reduction=ProducerReduction.SCOPED, ): parallel = get_parallel() - reduction = get_exec().comm.boundary_reduction - if reduction not in ("ar", "rs", "rsv", "rs+rsv"): + if dense_tp_size not in (None, 1, parallel.tp_size): + raise ValueError("dense FFN rows support local compute or the full TP group") + strategy = get_exec().comm.boundary_reduction + if strategy not in ("ar", "rs", "rsv", "rs+rsv"): raise ValueError( "boundary_reduction must be resolved before model construction" ) - can_move_output = output is None or output.before_reduce_scatter - use_reduce_scatter = reduction in ("rs", "rs+rsv") and can_move_output - use_reduce_scatterv = reduction in ("rsv", "rs+rsv") and can_move_output + can_move_output = output_transform is None or output_transform.before_reduce_scatter + use_reduce_scatter = strategy in ("rs", "rs+rsv") and can_move_output + use_reduce_scatterv = strategy in ("rsv", "rs+rsv") and can_move_output cp_shards = _generic_prefill_cp_shards_tokens() axes, attention, local, full = _rows(variant) on_local = ( is_moe_input_scattered_across_dp_ranks() if sparse - else (local_dense and enable_moe_dense_fully_dp()) + else ( + enable_moe_dense_fully_dp() if dense_tp_size is None else dense_tp_size == 1 + ) ) if parallel.attn_cp_size > 1 and sparse: _refuse_uncovered_cp_moe(on_local, cp_shards) @@ -118,7 +120,7 @@ def _ffn_decl( gathers_itself=frozenset({TokenAxis.ATTN_TP_SCATTER}), read=read, ), - StageOutput(local, update=update, transform=output), + StageOutput(local, update=update, transform=output_transform), ), local, local, @@ -136,7 +138,7 @@ def _ffn_decl( leaves_for_reduce_scatter=use_reduce_scatter and (scattered_residual or not terminal), update=update, - transform=output, + transform=output_transform, ), ), residual, @@ -156,8 +158,8 @@ def _ffn_decl( *((TokenAxis.ATTN_CP,) if on_cp else ()), axis_sizes=axes ) ) - output = ( - StageOutput(rows, update=update, transform=output) + produced = ( + StageOutput(rows, update=update, transform=output_transform) if on_local else StageOutput( rows, @@ -166,22 +168,17 @@ def _ffn_decl( and not terminal and not update.at_producer and update.can_defer_across_layers - and output is None, + and output_transform is None + and reduction is ProducerReduction.SCOPED, leaves_for_reduce_scatter=use_reduce_scatter and may_scatter, leaves_for_reduce_scatterv=use_reduce_scatterv and may_leave, update=update, - transform=output, + transform=output_transform, ) ) - gathers_for_tbo = ( - enable_moe_dense_fully_dp() - and get_exec().overlap.enable_two_batch_overlap - and not sparse - and bool(next_sparse) - ) - returned = local if on_local and not terminal and not gathers_for_tbo else attention + returned = local if on_local and not terminal else attention return ( - StageDecl(StageInput(rows, read=read), output), + StageDecl(StageInput(rows, read=read), produced), local if on_local else attention, returned, ) @@ -201,13 +198,11 @@ class StageDeclaration: update: StageUpdate sparse: bool = False terminal: bool = False - next_sparse: bool = False - output: Optional[OutputTransform] = None - ordinary_only: bool = False - # A mixer with its own exit publishes reduction decisions before compute. - mixer_exit: bool = False - next_kind: Optional[StageKind] = None - return_to_attention: bool = False + output_transform: Optional[OutputTransform] = None + reduction: ProducerReduction = ProducerReduction.SCOPED + gathers_tp_input: bool = False + dense_tp_size: Optional[int] = None + handoff_rows: Optional[HandoffRows] = None # Only declarations participate in construction, never executable stages. previous: Optional[StageDeclaration] = None prepared_from: Optional[StageDeclaration] = None @@ -245,10 +240,11 @@ def declare_attn( read=NORM_QUANT_READ, update=ADD, terminal=False, - ordinary_only=False, - mixer_exit=False, - next_kind=None, + reduction=ProducerReduction.PARTIAL, + gathers_tp_input=True, ): + if reduction is ProducerReduction.LOCAL_TAIL: + raise ValueError("LOCAL_TAIL is not supported for attention stages") return StageDeclaration( StageKind.ATTENTION, read, @@ -256,9 +252,8 @@ def declare_attn( previous=previous, prepared_from=prepared_from, terminal=terminal, - ordinary_only=ordinary_only, - mixer_exit=mixer_exit, - next_kind=next_kind, + reduction=reduction, + gathers_tp_input=gathers_tp_input, ) @@ -270,11 +265,14 @@ def declare_ffn( read=NORM_READ, update=ADD, terminal=False, + output_transform=None, next_sparse=False, - output=None, - ordinary_only=False, - return_to_attention=False, + dense_tp_size=None, + reduction=ProducerReduction.SCOPED, + handoff_rows=None, ): + if reduction is ProducerReduction.PARTIAL: + raise ValueError("PARTIAL is not supported for ffn stages") return StageDeclaration( StageKind.FFN, read, @@ -283,43 +281,43 @@ def declare_ffn( previous=previous, prepared_from=prepared_from, terminal=terminal, - next_sparse=next_sparse, - output=output, - ordinary_only=ordinary_only, - return_to_attention=return_to_attention, + output_transform=output_transform, + dense_tp_size=dense_tp_size, + reduction=reduction, + handoff_rows=handoff_rows or tbo_handoff(sparse, next_sparse), ) -def _resolve(stage, variant): +def _resolve(stage, variant, following=None): axes, attention, local, full = _rows(variant) + if stage.update.at_producer: + if stage.sparse and sparse_moe_gathers_over_moe_cp(): + raise NotImplementedError( + "MHC does not support a MoE gathered over the MoE-CP group" + ) + if get_parallel().attn_cp_size > 1 and _input_can_be_scattered(): + raise NotImplementedError( + "MHC with input-scattered attention under attention CP" + ) if stage.kind is StageKind.FFN: - if stage.update.at_producer: - if stage.sparse and sparse_moe_gathers_over_moe_cp(): - raise NotImplementedError( - "MHC does not support a MoE gathered over the MoE-CP group" - ) - if get_parallel().attn_cp_size > 1 and _input_can_be_scattered(): - raise NotImplementedError( - "MHC with input-scattered attention under attention CP" - ) declaration, residual, returned = _ffn_decl( variant, sparse=stage.sparse, terminal=stage.terminal, - next_sparse=stage.next_sparse, - output=stage.output, + output_transform=stage.output_transform, read=stage.read, update=stage.update, - local_dense=not stage.return_to_attention, + dense_tp_size=stage.dense_tp_size, + reduction=stage.reduction, ) - if stage.return_to_attention: + if resolve_handoff_rows(stage.handoff_rows) is HandoffRows.ATTENTION: returned = attention return declaration, residual, returned sp = variant is BatchVariant.SEQUENCE_PARALLEL scattered = variant is BatchVariant.INPUT_SCATTERED gathers = ( frozenset({TokenAxis.ATTN_TP_SCATTER}) - if not stage.mixer_exit and (sp or scattered or _use_ag_after_qlora) + if stage.gathers_tp_input and (sp or scattered or _use_ag_after_qlora) else frozenset() ) owes = not sp and axes[TokenAxis.ATTN_TP_SCATTER] > 1 @@ -329,10 +327,14 @@ def _resolve(stage, variant): local if sp else attention, group=SumGroup.ATTN_TP if owes else None, always_leaves=owes - and (not stage.mixer_exit or stage.next_kind is StageKind.FFN), + and ( + stage.reduction is ProducerReduction.PARTIAL + or (following is not None and following.kind is StageKind.FFN) + ), leaves_for_next_layer=owes - and stage.mixer_exit - and stage.next_kind is StageKind.ATTENTION, + and stage.reduction is ProducerReduction.SCOPED + and following is not None + and following.kind is StageKind.ATTENTION, update=stage.update, ), ) @@ -351,9 +353,8 @@ def _connect(producer, consumer, *, residual_from=None): if producer is None and consumer is None: raise ValueError("a boundary needs at least one declared side") before, after = producer, consumer - ordinary_only = any(s.ordinary_only for s in (before, after) if s is not None) exits, entries = {}, {} - for variant in _variants(ordinary_only=ordinary_only): + for variant in _variants(): _, attention, local, _ = _rows(variant) if before is None: rows = local if variant is BatchVariant.SEQUENCE_PARALLEL else attention @@ -367,13 +368,16 @@ def _connect(producer, consumer, *, residual_from=None): residual = rows capabilities = (True,) else: - decl, during, returned = _resolve(before, variant) + decl, during, returned = _resolve(before, variant, following=after) if during is None: if residual_from is not None: if residual_from.consumer != producer: raise ValueError("residual source must enter the producer") during = residual_from.entries[variant].residual_to - elif before.mixer_exit: + elif ( + before.kind is StageKind.ATTENTION + and before.reduction is ProducerReduction.SCOPED + ): during = attention else: raise ValueError( @@ -382,16 +386,24 @@ def _connect(producer, consumer, *, residual_from=None): exits[variant] = EdgeDecl( decl.output, StageInput(returned), during, returned ) - if before.kind is StageKind.ATTENTION and not before.mixer_exit: + if ( + before.kind is StageKind.ATTENTION + and before.reduction is ProducerReduction.PARTIAL + ): arrived, residual, capabilities = decl.output, during, () else: owes = ( variant is BatchVariant.INPUT_SCATTERED and not before.update.at_producer ) - carries = (before.mixer_exit or before.return_to_attention) and ( - decl.output.always_leaves or decl.output.leaves_for_next_layer - ) + carries = ( + ( + before.kind is StageKind.ATTENTION + and before.reduction is ProducerReduction.SCOPED + ) + or resolve_handoff_rows(before.handoff_rows) + is HandoffRows.ATTENTION + ) and (decl.output.always_leaves or decl.output.leaves_for_next_layer) arrived = StageOutput( returned, group=decl.output.group @@ -413,10 +425,10 @@ def _connect(producer, consumer, *, residual_from=None): if variant is BatchVariant.INPUT_SCATTERED and arrived.always_leaves and arrived.update is None - and not after.mixer_exit + and after.reduction is ProducerReduction.PARTIAL else residual ) - elif after.return_to_attention: + elif resolve_handoff_rows(after.handoff_rows) is HandoffRows.ATTENTION: during = ( decl.input.layout if residual.sharded <= decl.input.layout.sharded @@ -440,7 +452,7 @@ def _connect(producer, consumer, *, residual_from=None): if ( before is not None and before.kind is StageKind.ATTENTION - and not before.mixer_exit + and before.reduction is ProducerReduction.PARTIAL ): exits[variant] = edge return StageConnection(producer, consumer, exits, entries) @@ -453,7 +465,11 @@ def _fork_input(prepared, consumer): execute the branch's input norm again. """ entries = {} - for variant in _variants(ordinary_only=consumer.ordinary_only): + for variant in _variants(): + if variant is not BatchVariant.ORDINARY: + raise NotImplementedError( + "prepared branch transport requires ordinary token rows" + ) source = prepared.entries[variant] declaration, during, _ = _resolve(consumer, variant) entries[variant] = EdgeDecl( @@ -476,7 +492,7 @@ def _incoming(stage): _incoming(previous) if previous is not None and previous.kind is StageKind.ATTENTION - and not previous.mixer_exit + and previous.reduction is ProducerReduction.PARTIAL else None ) return _connect( @@ -498,78 +514,12 @@ def _connections(stage, following=None): return incoming, outgoing -def _bind_stage(declaration, norm, incoming, outgoing, **options): - if incoming.consumer != declaration or outgoing.producer != declaration: - raise ValueError("connections do not match the stage declaration") - if incoming.entries.keys() != outgoing.exits.keys(): - raise ValueError("incoming and outgoing batch variants disagree") - if declaration.update.at_producer: - if declaration.sparse and sparse_moe_gathers_over_moe_cp(): - raise NotImplementedError( - "MHC does not support a MoE gathered over the MoE-CP group" - ) - if get_parallel().attn_cp_size > 1 and _input_can_be_scattered(): - raise NotImplementedError( - "MHC with input-scattered attention under attention CP" - ) - variants = {} - for variant, edge in incoming.entries.items(): - if ( - declaration.update.at_producer - and TokenAxis.ATTN_CP - in edge.produced.layout.sharded - edge.need.layout.sharded - ): - raise NotImplementedError("MHC with a gather over attention CP") - handoff = None - if declaration.kind is StageKind.ATTENTION: - handoff = ( - _hand_scattered_input_to_attention - if variant is BatchVariant.INPUT_SCATTERED - and not declaration.update.adds_plainly - else _hand_qkv_hook_its_input - ) - moves = _cp_moves() if variant is BatchVariant.CONTEXT_PARALLEL else None - variants[variant] = StageEdges(edge, outgoing.exits[variant], handoff, moves) - plan = StagePlan( - declaration.kind, - norm, - variants, - enters_stack=incoming.producer is None, - prepared_input=declaration.prepared_from is not None, - terminal=declaration.terminal, - fixed_output=declaration.kind is StageKind.ATTENTION - and not declaration.mixer_exit, - is_sparse=declaration.sparse, - **options, - ) - if declaration.kind is StageKind.ATTENTION and outgoing.consumer is not None: - # Layout eligibility comes from the connected consumer, not a mutable - # link to its execution plan. Kernel binding remains consumer-owned. - from sglang.srt.layers.communicator.boundary import make_boundary - - plan._fusion_rows = { - v: make_boundary( - edge, - cp_moves=_cp_moves() if v is BatchVariant.CONTEXT_PARALLEL else None, - force_layernorm_before_gather=options.get( - "force_layernorm_before_dp_gather", False - ), - ).input_rows - for v, edge in outgoing.entries.items() - } - return StageCommunicator(plan, declaration.kind, norm, declaration=declaration) - - def make_attn_stage( *, declaration, norm, following: Optional[StageDeclaration] = None, qkv_latent_func=None, - force_layernorm_before_dp_gather=False, - enable_fused_ar_quant=False, - fused_ar_quant_keep_bf16=False, - residual_in_hidden=False, fusions=None, ): """Resolve this attention's boundaries, then bind its norm and hooks. @@ -587,10 +537,6 @@ def make_attn_stage( incoming, outgoing, qkv_latent_func=qkv_latent_func, - force_layernorm_before_dp_gather=force_layernorm_before_dp_gather, - enable_fused_ar_quant=enable_fused_ar_quant, - fused_ar_quant_keep_bf16=fused_ar_quant_keep_bf16, - residual_in_hidden=residual_in_hidden, fusions=fusions, ) @@ -600,7 +546,6 @@ def make_ffn_stage( declaration, norm, following: Optional[StageDeclaration] = None, - force_layernorm_before_dp_gather=False, fusions=None, ): """Resolve this FFN's boundaries and bind its norm; EP stays inside compute.""" @@ -612,13 +557,13 @@ def make_ffn_stage( norm, incoming, outgoing, - force_layernorm_before_dp_gather=force_layernorm_before_dp_gather, - residual_in_hidden=declaration.update.at_producer, fusions=fusions, ) -def make_stages(*stages, previous=None, prepared_from=None, terminal=False): +def make_stages( + *stages, previous=None, prepared_from=None, following=None, terminal=False +): """Bind a local linear sequence and return its independent boundaries. Each item is ``(declaration, norm)`` or ``(declaration, norm, options)``. @@ -653,53 +598,43 @@ def make_stages(*stages, previous=None, prepared_from=None, terminal=False): previous=previous if index == 0 else declarations[-1], prepared_from=prepared_from if index == 0 else None, terminal=terminal and index == len(stages) - 1, - next_kind=stages[index + 1][0].kind - if declaration.mixer_exit and index + 1 < len(stages) - else declaration.next_kind, ) declarations.append(declaration) bindings.append((norm, item[2] if len(item) == 3 else {})) + external_consumer = following boundaries = [] + incoming = _incoming(declarations[0]) for index, (declaration, (norm, options)) in enumerate(zip(declarations, bindings)): - build = ( - make_attn_stage - if declaration.kind is StageKind.ATTENTION - else make_ffn_stage - ) - boundaries.append( - build( - declaration=declaration, - norm=norm, - following=declarations[index + 1] - if index + 1 < len(declarations) - else None, - **options, - ) + following = ( + declarations[index + 1] + if index + 1 < len(declarations) + else external_consumer ) + outgoing = _connect(declaration, following, residual_from=incoming) + options = dict(options) + if declaration.kind is StageKind.FFN: + unexpected = options.keys() - { + "fusions", + } + if unexpected: + raise TypeError(f"unsupported FFN options: {sorted(unexpected)}") + boundaries.append(_bind_stage(declaration, norm, incoming, outgoing, **options)) + incoming = outgoing for producer, consumer in zip(boundaries, boundaries[1:]): - if ( - producer.kind is not StageKind.ATTENTION - or producer.declaration.mixer_exit - or consumer.kind is not StageKind.FFN + if not ( + producer.kind is StageKind.ATTENTION + and producer.declaration.reduction is ProducerReduction.PARTIAL + and consumer.kind is StageKind.FFN ): continue for variant, steps in producer.plan._paths.items(): - predicate = consumer.plan._paths[variant].ffn.preserves_residual + next_steps = consumer.plan._paths[variant] + predicate = next_steps.entry.preserves_residual if predicate is not None: producer.plan._paths[variant] = msgspec.structs.replace( steps, - attention=msgspec.structs.replace( - steps.attention, capture_preserves_residual=predicate + entry=msgspec.structs.replace( + steps.entry, capture_preserves_residual=predicate ), ) - producer.plan._steps = producer.plan._paths[BatchVariant.ORDINARY] - producer.plan._cp_steps = producer.plan._paths.get( - BatchVariant.CONTEXT_PARALLEL - ) - producer.plan._input_scattered_steps = producer.plan._paths.get( - BatchVariant.INPUT_SCATTERED - ) - producer.plan._sp_steps = producer.plan._paths.get( - BatchVariant.SEQUENCE_PARALLEL - ) return tuple(boundaries) diff --git a/python/sglang/srt/layers/communicator/fusions/allreduce.py b/python/sglang/srt/layers/communicator/fusions/allreduce.py index 76bdb3b191..e9e558b059 100644 --- a/python/sglang/srt/layers/communicator/fusions/allreduce.py +++ b/python/sglang/srt/layers/communicator/fusions/allreduce.py @@ -19,23 +19,26 @@ from typing import Callable, Optional, Tuple import torch -from sglang.srt.layers.communicator.boundary import FusedMlpInput +from sglang.srt.layers.communicator.contracts import FusedMlpInput from sglang.srt.layers.communicator.layout import SumGroup from sglang.srt.layers.communicator.output import UnreducedOutput from sglang.srt.layers.communicator.residual.add_norm import ( + NORM_QUANT_READ, + Fp8Input, + NormQuantRead, apply_aiter_all_reduce_fusion, apply_flashinfer_allreduce_fusion, ) from sglang.srt.layers.layernorm import GemmaRMSNorm, RMSNorm from sglang.srt.layers.moe import post_experts_reduction_group from sglang.srt.model_executor.forward_batch_info import ForwardBatch -from sglang.srt.runtime_context import get_parallel +from sglang.srt.runtime_context import get_exec, get_parallel from sglang.srt.utils import get_bool_env_var, is_hip _use_aiter = get_bool_env_var("SGLANG_USE_AITER") and is_hip() -def attention_fusions(plan) -> Tuple[Callable, ...]: +def attention_fusions(plan, read=NORM_QUANT_READ) -> Tuple[Callable, ...]: """The fused kernels that complete what the previous layer left together with the residual update and the input norm, in the order they are tried. Each takes (owed, residual, forward_batch, post_residual_addition) and @@ -45,12 +48,23 @@ def attention_fusions(plan) -> Tuple[Callable, ...]: given = plan.fusions.attention_input(plan) if plan.fusions else () if not hasattr(plan.norm, "forward_with_allreduce_fusion"): return given - plan._attn_input_fuses_quant = ( - plan.enable_fused_ar_quant + fuses_quant = ( + isinstance(read, NormQuantRead) + and read.fp8_input is not None and _use_aiter + and not get_bool_env_var("SGLANG_DISABLE_FUSED_AR_QUANT", default="false") + and get_exec().comm.enable_aiter_allreduce_fusion and hasattr(plan.norm, "forward_with_allreduce_fusion_quant_per_group") ) - return (*given, partial(complete_attention_input, plan)) + return ( + *given, + partial( + complete_attention_input, + plan, + fuses_quant=fuses_quant, + keep_bf16=fuses_quant and read.fp8_input is Fp8Input.TUPLE_AND_BF16, + ), + ) def complete_attention_input( @@ -59,11 +73,14 @@ def complete_attention_input( residual: torch.Tensor, forward_batch: ForwardBatch, post_residual_addition: Optional[torch.Tensor], + *, + fuses_quant: bool, + keep_bf16: bool, ) -> Optional[Tuple[torch.Tensor, torch.Tensor]]: """Complete the sum the previous layer left, add it to the residual and apply the input norm in one aiter or flashinfer kernel; None when the - kernel does not take this batch. The result is not quantized for - ``quant_format``.""" + kernel does not take this batch. The optional FP8 result follows the consumer read declaration; + the separate ``quant_format`` path remains the read adapter's responsibility.""" if ( not isinstance(owed, UnreducedOutput) # The kernel does not add it. @@ -78,14 +95,14 @@ def complete_attention_input( or apply_flashinfer_allreduce_fusion(hidden_states.shape[0]) ): return None - if plan._attn_input_fuses_quant: + if fuses_quant: # Falls back to AR+RMSNorm + separate quant internally when the # fully-fused kernel cannot service the shape. quant_result = plan.norm.forward_with_allreduce_fusion_quant_per_group( hidden_states, residual, use_attn_tp_group=False, - keep_bf16=plan.fused_ar_quant_keep_bf16, + keep_bf16=keep_bf16, ) if quant_result is not None: return quant_result diff --git a/python/sglang/srt/layers/communicator/fusions/cutedsl.py b/python/sglang/srt/layers/communicator/fusions/cutedsl.py index a6ec616b43..551c4453ea 100644 --- a/python/sglang/srt/layers/communicator/fusions/cutedsl.py +++ b/python/sglang/srt/layers/communicator/fusions/cutedsl.py @@ -208,21 +208,18 @@ class CuteDSLFusion: # Producer capabilities; consumer kernel selection is independent. self.hands_off_finalize = False self.terminal_finalize = False - self.requires_local_reduction = False def install( self, service: CuteDSLFusionService, *, hands_off_finalize: bool, - output_is_replicated: bool, terminal_finalize: bool = False, ) -> None: """Install the shared workspace and this producer's capabilities.""" self.service = service self.hands_off_finalize = hands_off_finalize self.terminal_finalize = terminal_finalize - self.requires_local_reduction = output_is_replicated def attention_input(self, layer) -> tuple: return ( @@ -233,7 +230,7 @@ class CuteDSLFusion: def ffn_input(self, layer) -> tuple: parallel = get_parallel() if ( - TokenAxis.ATTN_TP_SCATTER not in layer.input_rows.sharded + TokenAxis.ATTN_TP_SCATTER not in layer.incoming_residual_rows.sharded # The workspace sums over TP, which is then the attention-TP group. and parallel.attn_tp_size == parallel.tp_size and _fused_norm_gamma(layer.norm) is not None @@ -336,11 +333,7 @@ class CuteDSLFusion: """Whether this producer can emit a handoff with an unfused fallback.""" return ( self.hands_off_finalize - and (not layer.is_last_layer or self.terminal_finalize) - and ( - not self.requires_local_reduction - or (layer.is_last_layer and self.terminal_finalize) - ) + and (not layer.terminal or self.terminal_finalize) and self._should_use_finalize( layer, forward_batch, int(forward_batch.input_ids.shape[0]) ) @@ -377,7 +370,6 @@ def install_cutedsl_fusion( top_k: int, rms_epsilon: float, can_defer_finalize: _LayerPredicate, - requires_local_reduction: _LayerPredicate | None = None, label: str, terminal_finalize: bool = False, ) -> CuteDSLFusionService | None: @@ -424,8 +416,6 @@ def install_cutedsl_fusion( service, hands_off_finalize=hands_off_finalize, terminal_finalize=terminal_finalize, - output_is_replicated=requires_local_reduction is not None - and bool(requires_local_reduction(layer)), ) logger.info( "Installed one %s FlashInfer MNNVL CuTe DSL fusion handle for %d of %d layers " diff --git a/python/sglang/srt/layers/communicator/layer.py b/python/sglang/srt/layers/communicator/layer.py deleted file mode 100644 index 91df3effbb..0000000000 --- a/python/sglang/srt/layers/communicator/layer.py +++ /dev/null @@ -1,1454 +0,0 @@ -# Copyright 2023-2024 SGLang Team -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""The communicator a model layer uses.""" - -from dataclasses import dataclass -from functools import cached_property, partial -from typing import Callable, Optional, Protocol, Tuple, Union - -import msgspec -import torch - -from sglang.srt.distributed import GroupCoordinator -from sglang.srt.environ import envs -from sglang.srt.layers import layernorm_sp -from sglang.srt.layers.aux_hidden_states import AuxHiddenStateAccumulator -from sglang.srt.layers.communicator.adapters.attention import get_attn_tp_context -from sglang.srt.layers.communicator.boundary import ( - BoundarySteps, - DecoderLayerSides, - FusedMlpInput, - LayerStage, - StageEntry, - StageKind, - _cp_moves, - _select_boundary_steps, - decoder_layer_sides, - input_scattered_layer_sides, - make_boundary, - make_output_boundary, - scattered_residual_layer_sides, - sequence_parallel_layer_sides, - with_residual, -) -from sglang.srt.layers.communicator.exit import _reduce_and_redistribute_output_step -from sglang.srt.layers.communicator.layout import ( - Layout, - SumGroup, - TokenAxis, - _batch_shards_over_cp, - _ffn_has_tokens, - _gathers_over_attention_cp, - _generic_prefill_cp_shards_tokens, - _sum_group, - enable_moe_dense_fully_dp, - sparse_moe_gathers_over_moe_cp, - token_axis_sizes, -) -from sglang.srt.layers.communicator.legacy_stage import StageCommunicator -from sglang.srt.layers.communicator.ops import ( - CommunicateSimpleFn, - _all_reduce_then_to_local_tokens, - _hand_qkv_hook_its_input, - _hand_scattered_input_to_attention, - _mlp_input_without_dp, - _redistribute_output, - _to_local_tokens, - move_rows, -) -from sglang.srt.layers.communicator.output import ( - UnreducedOutput, - reduce_output, -) -from sglang.srt.layers.communicator.residual import LayerResidual -from sglang.srt.layers.communicator.residual import batch as residual_batch -from sglang.srt.layers.communicator.residual.access import ( - add_to_output, - buffer, - finish_layer_stack, - fold, - from_pp, - norm_output, - snapshot, - written, -) -from sglang.srt.layers.communicator.residual.add_norm import ( - ADD, - PLAIN_RESIDUAL, - apply_aiter_all_reduce_fusion, - apply_flashinfer_allreduce_fusion, -) -from sglang.srt.layers.communicator.residual.mhc import MHCState -from sglang.srt.layers.communicator.residual.stream import DeclaredSum, ResidualStream -from sglang.srt.layers.dp_attention import ( - is_dp_attention_enabled, - is_enable_moe_cp_allgather, -) -from sglang.srt.layers.layernorm import GemmaRMSNorm, RMSNorm -from sglang.srt.layers.moe import ( - can_merge_post_experts_all_reduce, - get_moe_a2a_backend, - is_moe_input_scattered_across_dp_ranks, - post_experts_reduction_group, - post_experts_sum_is_one_all_reduce, -) -from sglang.srt.model_executor.forward_batch_info import ForwardBatch -from sglang.srt.runtime_context import ( - LoRABatchLayout, - get_exec, - get_forward, - get_lora, - get_parallel, - get_spec, -) -from sglang.srt.speculative.spec_info import SpeculativeAlgorithm -from sglang.srt.utils import get_bool_env_var, is_hip - -_use_aiter = get_bool_env_var("SGLANG_USE_AITER") and is_hip() -_use_ag_after_qlora = envs.SGLANG_USE_AG_AFTER_QLORA.get() - - -@dataclass -class LayerFacts: - """Where a layer sits in the model and whether it and its neighbours have a - sparse MLP: the facts its boundaries are declared from.""" - - is_layer_sparse: bool = False - # The model's first layer: its input is the embedding, not a layer output. - is_first_layer: bool = False - # The model's last layer: its output goes to the final norm, not a next layer. - is_last_layer: bool = False - # Whether the layer before this one has a sparse MLP; None when the facts - # were given directly, not planned from the layer sequence. - is_previous_layer_sparse: Optional[bool] = None - # Whether the layer after this one has a sparse MLP; None likewise. - is_next_layer_sparse: Optional[bool] = None - - @classmethod - def init_new( - cls, - *, - layer_id: int, - num_layers: int, - is_layer_sparse: bool, - is_previous_layer_sparse: Optional[bool], - is_next_layer_sparse: Optional[bool], - ) -> "LayerFacts": - return cls( - is_layer_sparse=is_layer_sparse, - is_first_layer=layer_id == 0, - is_last_layer=layer_id == num_layers - 1, - is_previous_layer_sparse=is_previous_layer_sparse, - is_next_layer_sparse=is_next_layer_sparse, - ) - - -def _can_defer_ffn_reduction(forward_batch: ForwardBatch, boundary=None) -> bool: - """Admit ordinary single-sum outputs and the existing fused LoRA path. - - A fused-kernel fallback completes the partial-output contract selected - before the producer ran, without recomputing the producer's LoRA path. - """ - if not _ffn_has_tokens(forward_batch): - return False - if post_experts_sum_is_one_all_reduce(): - return True - # LoRA-B is replicated and linear. TP1 shared experts add on rank zero - # when the sum is deferred. Preserve these fused paths, but keep their - # producer-side reduction order when no fused consumer is enabled. - if not ( - (get_lora().enable_lora or envs.SGLANG_SHARED_EXPERT_TP1.get()) - and get_moe_a2a_backend().is_none() - and not get_exec().comm.enable_quant_communications - and post_experts_reduction_group() is get_parallel().tp_group - ): - return False - if ( - boundary is not None - and boundary.fusions is not None - and boundary.fusions.can_defer_all_reduce(boundary, forward_batch) - ): - return True - if apply_flashinfer_allreduce_fusion(forward_batch.input_ids.shape[0]): - return True - # Aiter also checks width and bytes, so use the actual residual storage. - residual = forward_batch.residual_stream.residual - return residual is not None and apply_aiter_all_reduce_fusion( - residual, forward_batch - ) - - -class LayerFusions(Protocol): - """The fused kernels a fusion backend gives a layer, bound to it and tried - before the layer's own: at its attention input (each takes what the previous - layer left, the residual, the batch and a post-residual addition), at its - FFN input, and at its FFN exit (each says what the exit does when its kernel - takes the batch).""" - - def attention_input(self, layer: "LayerCommunicator") -> Tuple[Callable, ...]: ... - - def ffn_input(self, layer: "LayerCommunicator") -> Tuple["FusedMlpInput", ...]: ... - - requires_local_reduction: bool - - def can_defer_all_reduce( - self, layer: "LayerCommunicator", forward_batch: ForwardBatch - ) -> bool: ... - - def can_defer_finalize( - self, layer: "LayerCommunicator", forward_batch: ForwardBatch - ) -> bool: ... - - -class LayerCommunicator: - add_to_output = staticmethod(add_to_output) - buffer = staticmethod(buffer) - fold = staticmethod(fold) - norm_output = staticmethod(norm_output) - written = staticmethod(written) - finish_layer_stack = staticmethod(finish_layer_stack) - - # Communicators built without __init__ (e.g. test doubles) publish no LoRA - # layout and try no fused kernel at the FFN exit. - _publish_lora_layout: bool = False - # A plain residual unless the layer is built with its own. - _residual: LayerResidual = PLAIN_RESIDUAL - # The fused kernels a backend gives the layer, if any. - fusions: Optional[LayerFusions] = None - stage_edges = None - - def __init__( - self, - layer_facts: LayerFacts, - input_layernorm: torch.nn.Module, - post_attention_layernorm: torch.nn.Module, - # Reduce scatter requires skipping all-reduce in model code after MoE/MLP, so only enable for models which have that implemented. Remove flag once done for all models that use LayerCommunicator. - allow_reduce_scatter: bool = False, - qkv_latent_func: Optional[Callable] = None, - force_layernorm_before_dp_gather: bool = False, - enable_fused_ar_quant: bool = False, - fused_ar_quant_keep_bf16: bool = False, - # False for a layer whose FFN always completes its own reduction. - allow_deferred_ffn_reduction: bool = True, - # How the layer's attention and FFN read their inputs from the residual - # and write their outputs into it. - residual: LayerResidual = PLAIN_RESIDUAL, - # A layer that is one stage of a sequence of stages, instead of an - # attention followed by an FFN. - stage: Optional["LayerStage"] = None, - # The fused kernels a backend gives the layer, tried before its own. - fusions: Optional[LayerFusions] = None, - ): - self.layer_facts = layer_facts - self.input_layernorm = input_layernorm - self.post_attention_layernorm = post_attention_layernorm - self.allow_reduce_scatter = allow_reduce_scatter - self.is_last_layer = layer_facts.is_last_layer - self.qkv_latent_func = qkv_latent_func - self.force_layernorm_before_dp_gather = force_layernorm_before_dp_gather - self.enable_fused_ar_quant = enable_fused_ar_quant - self.fused_ar_quant_keep_bf16 = fused_ar_quant_keep_bf16 - self.allow_deferred_ffn_reduction = allow_deferred_ffn_reduction - self._residual = residual - self.fusions = fusions - - # The fused kernels every batch's attention input tries first. - self._attn_input_fusions = self._select_attn_input_fusions() - self._speculative_algo = SpeculativeAlgorithm.from_string( - get_spec().speculative_algorithm - ) - # LoRA kernels need the per-layer token layout only under DP attention. - self._publish_lora_layout = get_parallel().enable_dp_attention and bool( - get_lora().enable_lora - ) - if stage is not None: - if residual != PLAIN_RESIDUAL: - raise ValueError( - "a layer that is one stage takes its reads and updates from " - "the stage's declarations" - ) - self._init_stage(stage) - return - # Its two boundaries, for a layer that is one stage. - self.stage_edges = None - # The steps the layer's ordinary batches run. - sides = self._declared_sides() - self._declared = sides - self._steps = self._steps_from_declarations( - sides, - fusions=self._select_mlp_input_fusions(), - force_layernorm_before_gather=force_layernorm_before_dp_gather, - ) - # The steps a batch that shards its tokens over attention CP runs; None - # when no batch does. - self._cp_steps = ( - self._steps_from_declarations( - self._declared_sides(cp_active=True), - fusions=self._select_mlp_input_fusions(), - force_layernorm_before_gather=force_layernorm_before_dp_gather, - cp_moves=_cp_moves(), - ) - if _generic_prefill_cp_shards_tokens() - else None - ) - # The steps a batch with input-scattered attention runs; None where it - # cannot run. - self._input_scattered_steps = ( - self._steps_for_input_scattered(sides) - if self._input_can_be_scattered() - else None - ) - - # Under LayerNorm SP, the steps the layer runs while the region is - # active; None without SP. The two are exclusive: SP runs a model without - # q_lora, which input-scattered attention needs. - self._sp_steps = ( - _select_boundary_steps( - with_residual( - sequence_parallel_layer_sides(axis_sizes=token_axis_sizes()), - residual, - ), - attention_fusions=self._attn_input_fusions, - enters_stack=self.layer_facts.is_first_layer, - ) - if layernorm_sp.layernorm_sp_enabled() - else None - ) - - # Bind aux ownership to the selected input implementation once. - for name in ("_steps", "_cp_steps", "_input_scattered_steps", "_sp_steps"): - steps = self.__dict__[name] - if steps is None or steps.ffn is None: - continue - prepare = steps.ffn.prepare - if isinstance(prepare, partial): - prepare = prepare.keywords.get("step", prepare) - norm = self.post_attention_layernorm - known_norm = isinstance(norm, (RMSNorm, GemmaRMSNorm)) and type( - norm - ).forward_with_allreduce_fusion in ( - RMSNorm.forward_with_allreduce_fusion, - GemmaRMSNorm.forward_with_allreduce_fusion, - ) - if ( - known_norm - and isinstance(prepare, partial) - and prepare.func is _mlp_input_without_dp - and not prepare.keywords["gathers_residual"] - and steps.ffn.fused - and steps.ffn.fused[0].run - == self._mlp_input_reduce_output_and_update_and_read_residual - ): - self.__dict__[name] = msgspec.structs.replace( - steps, - attention=msgspec.structs.replace( - steps.attention, - capture_preserves_residual=self._capture_keeps_residual, - ), - ) - - def _capture_keeps_residual(self, value, forward_batch): - # The CUDA wrapper allocates a fresh residual even on backend fallback. - return ( - not _use_aiter - and get_parallel().attn_tp_size > 1 - and apply_flashinfer_allreduce_fusion(value.shape[0]) - ) - - def _init_stage(self, stage: "LayerStage") -> None: - """A layer that is one stage: every batch runs the two boundaries its - declarations give.""" - if get_parallel().attn_cp_size > 1 or layernorm_sp.layernorm_sp_enabled(): - raise NotImplementedError( - "a layer that is one stage with attention CP or LayerNorm SP" - ) - self._declared = None - self._cp_steps = self._input_scattered_steps = self._sp_steps = None - into_edge, out_edge = stage.edges - is_ffn = stage.kind is StageKind.FFN - self.stage_edges = stage.edges - into = make_boundary( - into_edge, - fusions=self._select_mlp_input_fusions() if is_ffn else (), - carried_fusions=() if is_ffn else self._attn_input_fusions, - force_layernorm_before_gather=self.force_layernorm_before_dp_gather, - enters_stack=stage.enters_stack, - ) - out = make_output_boundary(out_edge) - entry = StageEntry( - prepare=into.prepare, - input_rows=into.input_rows, - input_move=into.input_move, - handoff=None if is_ffn else _hand_qkv_hook_its_input, - fused=into.fused, - capture_move=into.capture_move, - input_sum=into_edge.produced.group - if into_edge.produced.always_leaves - else None, - ) - self._steps = BoundarySteps( - attention=None if is_ffn else entry, - ffn=entry if is_ffn else None, - ffn_output=out_edge.produced, - ffn_output_move=out.output_move, - ffn_output_move_completes_sum=out.output_move_completes_sum, - ffn_sum_is_movable=out_edge.produced.group is not None, - ) - - @cached_property - def attn(self) -> StageCommunicator: - """The layer's attention stage.""" - return StageCommunicator(self, "attention", "input_layernorm") - - @cached_property - def ffn(self) -> StageCommunicator: - """The layer's FFN stage.""" - return StageCommunicator(self, "ffn", "post_attention_layernorm") - - @property - def input_rows(self) -> Layout: - """The rows the layer's input and residual arrive on in a batch that - runs its ordinary steps.""" - if self._declared is not None: - return self._declared.input_rows - return self.stage_edges[0].residual - - @property - def input_on_attention_tp_slices(self) -> bool: - """Whether the layer's input arrives on each attention-TP rank's slice - of its rows (after a layer on local rows), in a batch that runs its - ordinary steps.""" - return TokenAxis.ATTN_TP_SCATTER in self.input_rows.sharded - - def _input_can_be_scattered(self) -> bool: - """Whether a batch may run this layer with input-scattered attention: - configured, on TP without attention DP, a prefill CP, an a2a backend or - a dense MLP on every rank. The rest of what ``AttnTpContext.init_context`` requires is only - known once the model is built.""" - parallel = get_parallel() - return ( - parallel.enable_attn_tp_input_scattered - and parallel.tp_size > 1 - and parallel.attn_dp_size == 1 - and not _generic_prefill_cp_shards_tokens() - and get_moe_a2a_backend().is_none() - and not enable_moe_dense_fully_dp() - ) - - def _declared_sides(self, *, cp_active: bool = False) -> DecoderLayerSides: - """The declarations this layer's boundaries are chosen from: an - attention and an FFN, in a layer whose previous layer is known. - ``cp_active`` gives a batch that shards its tokens over CP; the others - hold every token on each CP rank. An active LayerNorm SP region and - input-scattered attention have their own. Raises NotImplementedError - for the combinations the steps do not cover.""" - modes = self.layer_facts - parallel = get_parallel() - if not (modes.is_first_layer or modes.is_previous_layer_sparse is not None): - raise NotImplementedError( - "a layer built without the facts of the layer before it" - ) - # A MoE dispatched per DP shard computes on this rank's local rows and - # hands its layer's output on there; so does a dense MLP on every rank. - moe_on_local_rows = is_moe_input_scattered_across_dp_ranks() - dense_on_local_rows = enable_moe_dense_fully_dp() - - # Some batch shards its tokens over attention CP. - cp_shards = _generic_prefill_cp_shards_tokens() - if parallel.attn_cp_size > 1 and modes.is_layer_sparse: - self._refuse_uncovered_cp_moe(moe_on_local_rows, cp_shards) - # A MoE whose data-parallel groups are the CP ranks computes each CP - # shard of a GQA prefill CP on its own ranks. - ffn_on_cp_shards = ( - modes.is_layer_sparse - and parallel.attn_cp_size > 1 - and parallel.moe_dp_size == parallel.attn_cp_size - and not _gathers_over_attention_cp() - ) - - def on_local_rows(sparse: bool) -> bool: - return moe_on_local_rows if sparse else dense_on_local_rows - - def gathers_for_tbo(sparse: bool, next_sparse: Optional[bool]) -> bool: - # Under two-batch overlap a dense layer on local rows hands the - # sparse layer after it, where the split happens, the attention's - # rows. - return ( - dense_on_local_rows - and get_exec().overlap.enable_two_batch_overlap - and not sparse - and bool(next_sparse) - ) - - if cp_shards and _cp_moves().reduce_scatter is not None: - # A CP extend's FFN may leave its sum to the reduce-scatter that - # takes each rank's shard back (DSA and MLA CP). - may_leave = not cp_active - may_leave_to_reduce_scatter = True - elif cp_shards or parallel.attn_dp_size > 1: - # Otherwise under CP an FFN over every CP shard completes its own sum; - # one on its own CP shard hands on its rows as without CP. - may_leave = may_leave_to_reduce_scatter = ( - parallel.attn_cp_size == 1 or ffn_on_cp_shards - ) - else: - # No batch shards its tokens and there is no attention DP: as - # without CP. - may_leave = may_leave_to_reduce_scatter = True - sides = decoder_layer_sides( - axis_sizes=token_axis_sizes(cp_active=cp_active), - ffn_on_local_rows=on_local_rows(modes.is_layer_sparse), - ffn_shards_over_cp=ffn_on_cp_shards, - previous_on_local_rows=( - not modes.is_first_layer - and on_local_rows(modes.is_previous_layer_sparse) - and not gathers_for_tbo( - modes.is_previous_layer_sparse, modes.is_layer_sparse - ) - ), - is_last_layer=modes.is_last_layer, - hands_on_attention_rows=gathers_for_tbo( - modes.is_layer_sparse, modes.is_next_layer_sparse - ), - attention_gathers_local_rows=_use_ag_after_qlora, - ffn_group=SumGroup.MOE_OUTPUT if modes.is_layer_sparse else SumGroup.TP, - leaves_for_next_layer=self.allow_deferred_ffn_reduction and may_leave, - leaves_for_reduce_scatter=self.allow_reduce_scatter - and may_leave_to_reduce_scatter, - # A MoE block leaves its sum to reduce_scatterv whenever that combine - # applies (should_skip_post_experts_all_reduce). - leaves_for_reduce_scatterv=( - self.allow_reduce_scatter or modes.is_layer_sparse - ) - and may_leave, - ) - return with_residual(sides, self._residual) - - @staticmethod - def _refuse_uncovered_cp_moe(moe_on_local_rows: bool, cp_shards: bool) -> None: - """A MoE layer under attention CP whose tokens the steps cannot bring - to it: under a prefill CP, one dispatched per DP shard under attention DP - and GQA CP, and one on the TP group whose data-parallel groups are the CP - ranks under DSA or MLA CP; and under attention DP, one on the TP group - whose data-parallel groups are the CP ranks.""" - parallel = get_parallel() - gqa = not _gathers_over_attention_cp() - if moe_on_local_rows: - if cp_shards and gqa and parallel.attn_dp_size > 1: - raise NotImplementedError( - "a MoE dispatched per DP shard under attention DP and GQA " - "prefill CP" - ) - elif parallel.moe_dp_size == parallel.attn_cp_size: - if cp_shards and not gqa: - raise NotImplementedError( - "a MoE on the TP group with moe_dp_size == attn_cp_size under " - "DSA or MLA prefill CP" - ) - if parallel.attn_dp_size > 1: - raise NotImplementedError( - "a MoE on the TP group with moe_dp_size == attn_cp_size under " - "attention DP and attention CP" - ) - - def _steps_for_input_scattered(self, sides: DecoderLayerSides) -> "BoundarySteps": - """The steps a batch with input-scattered attention runs at this layer, - for the layer's ordinary declarations ``sides``. A plain residual comes - back to the full rows inside the attention output's sum; one whose - write-back is not a plain add stays on each rank's slice.""" - if self._residual.attention_update.adds_plainly: - scattered = input_scattered_layer_sides( - axis_sizes=token_axis_sizes(), - ffn_group=sides.ffn.output.group, - hands_on_partial=self.allow_reduce_scatter and not self.is_last_layer, - ) - handoff = _hand_qkv_hook_its_input - else: - scattered = scattered_residual_layer_sides( - axis_sizes=token_axis_sizes(), - ffn_group=sides.ffn.output.group, - is_first_layer=self.layer_facts.is_first_layer, - is_last_layer=self.is_last_layer, - leaves_for_reduce_scatter=self.allow_reduce_scatter, - ) - # DSA and hook-less attention take the slice gathered. - handoff = _hand_scattered_input_to_attention - return _select_boundary_steps( - with_residual(scattered, self._residual), - attention_handoff=handoff, - attention_fusions=self._attn_input_fusions, - enters_stack=self.layer_facts.is_first_layer, - ) - - def _steps_from_declarations( - self, sides: DecoderLayerSides, **kwargs - ) -> "BoundarySteps": - """The steps this layer runs for a set of declarations.""" - return _select_boundary_steps( - sides, - attention_fusions=self._attn_input_fusions, - enters_stack=self.layer_facts.is_first_layer, - **kwargs, - ) - - def snapshot(self, hidden_states, residual, *, at_input: bool = False): - group = None - if ( - residual is not None - and not isinstance(residual, ResidualStream) - and self.stage_edges is not None - ): - edge = self.stage_edges[0 if at_input else 1] - if edge.produced.always_leaves: - group = _sum_group(edge.produced.group) - return snapshot(hidden_states, residual, group=group) - - def capture_output( - self, - hidden_states, - residual, - *, - at_input: bool = False, - skip_empty: bool = False, - ): - """Complete the carried output on the main path, then snapshot it. - Keep this explicit completion distinct from a read-only snapshot: the - next prepare must not select a fusion for a sum already completed here.""" - if skip_empty: - storage = buffer(hidden_states) - if storage is not None and storage.shape[0] == 0: - return hidden_states, None - hidden_states = ( - ( - hidden_states - if residual.pending is not None - and isinstance(residual.pending.owed, DeclaredSum) - else residual.complete(hidden_states) - ) - if isinstance(residual, ResidualStream) - else reduce_output(hidden_states) - ) - return hidden_states, self.snapshot(hidden_states, residual, at_input=at_input) - - def from_pp(self, tensors, forward_batch, *, allow_missing_residual: bool = False): - hidden_states, residual = from_pp( - tensors, - residual_in_hidden=self._residual.ffn_update.at_producer, - allow_missing_residual=allow_missing_residual, - ) - steps = self._batch_steps(forward_batch) - entry = steps.attention or steps.ffn - hidden_states, stream = ResidualStream.arrive( - hidden_states, residual, ADD, declared_sum=entry.input_sum - ) - forward_batch.residual_stream = stream - return hidden_states, stream - - def prepare_attn_and_capture_last_layer_outputs( - self, - hidden_states: torch.Tensor, - residual: torch.Tensor, - forward_batch: ForwardBatch, - captured_last_layer_outputs: Optional[AuxHiddenStateAccumulator] = None, - post_residual_addition: Optional[torch.Tensor] = None, - quant_format: str = "", - capture_output: Optional[Callable] = None, - ): - # Aux consumers need a materialized output before the input norm. - # Complete communication here, preserving the existing add+norm kernel - # and its FP32 accumulation. Its residual result also supplies capture. - capture_before_read = capture_output is not None and ( - residual is None - or ( - isinstance(residual, ResidualStream) - and residual.residual is None - and residual.pending is None - ) - or post_residual_addition is not None - ) - if capture_output is not None: - hidden_states = ( - residual.complete(hidden_states) - if isinstance(residual, ResidualStream) - else reduce_output(hidden_states) - ) - if capture_before_read: - # Embeddings precede enter; HF deepstack capture precedes the - # extra addition. Neither is the residual returned by the read. - value, previous = ( - residual.finish(hidden_states) - if isinstance(residual, ResidualStream) - else (hidden_states, residual) - ) - if previous is None: - capture_output(value) - else: - capture_output(value + previous, owned=True) - hidden_states, residual = self.prepare_attn( - hidden_states, - residual, - forward_batch, - quant_format=quant_format, - post_residual_addition=post_residual_addition, - ) - if capture_output is not None and not capture_before_read: - value = residual.residual - move = self.attn.entry(forward_batch).capture_move - if move is not None: - value = move(value, forward_batch=forward_batch) - keeps = self.attn.entry(forward_batch).capture_preserves_residual - capture_output( - value, - owned=self.attn.entry(forward_batch).capture_move_allocates - or (move is None and keeps is not None and keeps(value, forward_batch)), - ) - if captured_last_layer_outputs is not None: - residual_value = residual.residual - move = self.attn.entry(forward_batch).input_move - gathered_last_layer_output = ( - residual_value - if move is None - else move( - hidden_states=residual_value, - forward_batch=forward_batch, - ) - ) - keeps = self.attn.entry(forward_batch).capture_preserves_residual - captured_last_layer_outputs.capture( - gathered_last_layer_output, - owned=move is CommunicateSimpleFn._scattered_to_tp_attn_full - or ( - move is None - and keeps is not None - and keeps(residual_value, forward_batch) - ), - ) - return hidden_states, residual - - def publish_attn_lora_layout(self) -> None: - """Attention consumes the DP-local token batch.""" - if self._publish_lora_layout: - get_forward().set("lora_batch_layout", LoRABatchLayout.DP_LOCAL) - - def publish_mlp_lora_layout(self, steps: "BoundarySteps") -> None: - """The MLP consumes the TP-global batch only when the batch's FFN input - is gathered over attention DP.""" - if self._publish_lora_layout: - get_forward().set( - "lora_batch_layout", - ( - LoRABatchLayout.DP_LOCAL - if TokenAxis.ATTN_DP in steps.ffn.input_rows.sharded - else LoRABatchLayout.TP_GLOBAL - ), - ) - - def prepare_attn( - self, - hidden_states: Union[torch.Tensor, UnreducedOutput], - residual: torch.Tensor, - forward_batch: ForwardBatch, - quant_format: str = "", - post_residual_addition: Optional[torch.Tensor] = None, - ): - self.publish_attn_lora_layout() - # The SP region opens at the first layer, re-evaluated per forward so a - # crash mid-loop cannot leak into the next one. It sets what the batch's - # steps are chosen from, and the first layer's input owes nothing. - if self._sp_steps is not None and self.layer_facts.is_first_layer: - get_forward().set( - "sp_active", layernorm_sp.runs_sp(forward_batch.forward_mode) - ) - if get_forward().sp_active: - hidden_states = layernorm_sp.sp_entry_scatter(hidden_states) - return self.attn._prepare( - hidden_states, - residual, - forward_batch, - quant_format=quant_format, - post_residual_addition=post_residual_addition, - ) - - def _in_sp_region(self) -> bool: - return self._sp_steps is not None and get_forward().sp_active - - def _batch_steps(self, forward_batch: ForwardBatch) -> "BoundarySteps": - """The steps this batch runs: an active LayerNorm SP region's, - input-scattered attention's, a CP extend's, or the layer's ordinary - ones.""" - if self._in_sp_region(): - return self._sp_steps - if ( - self._input_scattered_steps is not None - and get_attn_tp_context().input_scattered - ): - return self._input_scattered_steps - if self._cp_steps is not None and _batch_shards_over_cp(forward_batch): - return self._cp_steps - return self._steps - - def _select_attn_input_fusions(self) -> Tuple[Callable, ...]: - """The fused kernels that complete what the previous layer left together - with the residual update and the input norm, in the order they are tried. - Each takes (owed, residual, forward_batch, post_residual_addition) and - returns None when it does not take the batch. They add the residual - plainly; the boundary tries them only when the update it writes in is a - plain add. A backend's come first.""" - given = self.fusions.attention_input(self) if self.fusions else () - if not hasattr(self.input_layernorm, "forward_with_allreduce_fusion"): - return given - self._attn_input_fuses_quant = ( - self.enable_fused_ar_quant - and _use_aiter - and hasattr( - self.input_layernorm, "forward_with_allreduce_fusion_quant_per_group" - ) - ) - return (*given, self._reduce_output_and_update_and_read_residual) - - def _reduce_output_and_update_and_read_residual( - self, - owed: UnreducedOutput, - residual: torch.Tensor, - forward_batch: ForwardBatch, - post_residual_addition: Optional[torch.Tensor], - ) -> Optional[Tuple[torch.Tensor, torch.Tensor]]: - """Complete the sum the previous layer left, add it to the residual and - apply the input norm in one aiter or flashinfer kernel; None when the - kernel does not take this batch. The result is not quantized for - ``quant_format``.""" - if ( - not isinstance(owed, UnreducedOutput) - # The kernel does not add it. - or post_residual_addition is not None - # The kernel reduces over the MoE output's group. - or owed.group is not post_experts_reduction_group() - ): - return None - hidden_states = owed.partial - if not ( - apply_aiter_all_reduce_fusion(hidden_states, forward_batch) - or apply_flashinfer_allreduce_fusion(hidden_states.shape[0]) - ): - return None - if self._attn_input_fuses_quant: - # Falls back to AR+RMSNorm + separate quant internally when the - # fully-fused kernel cannot service the shape. - quant_result = ( - self.input_layernorm.forward_with_allreduce_fusion_quant_per_group( - hidden_states, - residual, - use_attn_tp_group=False, - keep_bf16=self.fused_ar_quant_keep_bf16, - ) - ) - if quant_result is not None: - return quant_result - return self.input_layernorm.forward_with_allreduce_fusion( - hidden_states, residual, use_attn_tp_group=False - ) - - def _select_mlp_input_fusions(self) -> Tuple["FusedMlpInput", ...]: - """The fused kernels that can take the attention -> FFN steps, in the - order they are tried. They add the residual plainly; the boundary tries - them only when the update it writes in is a plain add. A backend's come - first.""" - given = self.fusions.ffn_input(self) if self.fusions else () - if not hasattr(self.post_attention_layernorm, "forward_with_allreduce_fusion"): - return given - return ( - *given, - FusedMlpInput( - completes=SumGroup.ATTN_TP, - run=self._mlp_input_reduce_output_and_update_and_read_residual, - ), - ) - - def _mlp_input_reduce_output_and_update_and_read_residual( - self, - hidden_states: torch.Tensor, - residual: torch.Tensor, - forward_batch: ForwardBatch, - ) -> Optional[Tuple[torch.Tensor, torch.Tensor]]: - """The attention-TP all-reduce, residual add and norm in one aiter or - flashinfer kernel; None when neither takes the batch.""" - if not ( - apply_aiter_all_reduce_fusion(hidden_states, forward_batch) - or apply_flashinfer_allreduce_fusion(hidden_states.shape[0]) - ): - return None - return self.post_attention_layernorm.forward_with_allreduce_fusion( - hidden_states, residual, use_attn_tp_group=True - ) - - def prepare_mlp( - self, - hidden_states: Union[torch.Tensor, UnreducedOutput], - residual: torch.Tensor, - forward_batch: ForwardBatch, - cache=None, - ): - steps = self._batch_steps(forward_batch) - self.publish_mlp_lora_layout(steps) - - if isinstance(residual, ResidualStream) and self.stage_edges is None: - # The paired boundary records the attention contribution once. - hidden_states = residual.leave( - hidden_states, - self._residual.attention_update, - declared_sum=steps.ffn.input_sum, - ) - return self.ffn._prepare( - hidden_states, - residual, - forward_batch, - steps, - update=self._residual.attention_update, - cache=cache, - ) - - def maybe_prefetch_next_full_attention_kv( - self, - forward_batch: ForwardBatch, - next_full_attention_layer_id: Optional[int], - ) -> None: - return - - def postprocess_layer( - self, - hidden_states: torch.Tensor, - residual: torch.Tensor, - forward_batch: ForwardBatch, - ): - stream = residual if isinstance(residual, ResidualStream) else None - hidden_states, residual = self._complete_ffn_output_now( - hidden_states, - stream.residual if stream is not None else residual, - forward_batch=forward_batch, - dp_step=self._postprocess_dp_step(forward_batch), - ) - return self._leave_ffn_output( - hidden_states, - residual, - stream, - self._batch_steps(forward_batch).ffn_output.update, - self._declared_ffn_sum( - self._batch_steps(forward_batch), - self._ffn_leaves_sum_to_reduce_scatter( - forward_batch, self._postprocess_dp_step(forward_batch) - ), - ), - ) - - @staticmethod - def _leave_ffn_output(hidden_states, residual, stream, update, declared_sum=None): - if stream is None: - return hidden_states, residual - stream.residual = residual - if update.at_producer: - return stream.write(hidden_states), stream - return stream.leave(hidden_states, update, declared_sum=declared_sum), stream - - @staticmethod - def _declared_ffn_sum(steps, skipped_reduction): - # The input-scattered path leaves the sum for the next input's TP - # reduce-scatter. Other skip paths complete it in the output move. - if ( - skipped_reduction - and not steps.returns_over_dp - and not steps.ffn_output_move_completes_sum - ): - return SumGroup.TP - return None - - def _local_token_move_can_go_to_next_layer( - self, forward_batch: ForwardBatch - ) -> bool: - """Whether the next layer's input can run this layer's move of its FFN - output back to this rank's tokens: the base postprocess scatter, when - the next layer's input also writes the output into the residual.""" - steps = self._batch_steps(forward_batch) - return steps.returns_over_dp and not steps.ffn_output.update.at_producer - - def _postprocess_dp_step( - self, forward_batch: ForwardBatch - ) -> Optional[Callable[[torch.Tensor, torch.Tensor, ForwardBatch], None]]: - """The reduce-scatter that brings this layer's FFN output back to this - rank's tokens under attention DP; None when the base postprocess would - only scatter, or does not move tokens.""" - steps = self._batch_steps(forward_batch) - if not steps.returns_over_dp: - return None - return _reduce_and_redistribute_output_step( - forward_batch, - leaves_for_reduce_scatter=steps.ffn_output.leaves_for_reduce_scatter, - leaves_for_reduce_scatterv=steps.ffn_output.leaves_for_reduce_scatterv, - ) - - def _ffn_leaves_sum_to_reduce_scatter( - self, forward_batch: ForwardBatch, dp_step: Optional[Callable] - ) -> bool: - """Whether the FFN leaves its sum out because a reduce-scatter completes - it: the attention-DP one ``dp_step`` names, or the CP / input-scattered - one.""" - steps = self._batch_steps(forward_batch) - if not steps.ffn_output.leaves_for_reduce_scatter: - return False - if dp_step is not None or steps.ffn_output_move_completes_sum: - return True - return get_attn_tp_context().input_scattered and not self.is_last_layer - - def ffn_reduction_group(self, forward_batch: ForwardBatch) -> GroupCoordinator: - """The group this layer's FFN output owes its sum over: the MoE output's - group on a sparse layer, the TP group a dense MLP reduces over.""" - return _sum_group(self._batch_steps(forward_batch).ffn_output.group) - - def _select_ffn_completion(self, forward_batch: ForwardBatch) -> "FfnCompletion": - """Decide once, before the FFN runs, what it skips and what completes its - output: the next layer's input, or this layer's postprocess step.""" - dp_step = self._postprocess_dp_step(forward_batch) - mlp_reduce_scatter = self._ffn_leaves_sum_to_reduce_scatter( - forward_batch, dp_step - ) - complete_now = partial( - self._complete_ffn_output_now, - forward_batch=forward_batch, - dp_step=dp_step, - ) - steps = self._batch_steps(forward_batch) - if not steps.ffn_output.leaves_for_next_layer: - return FfnCompletion( - defer_moe_finalize=False, - fuse_mlp_allreduce=False, - mlp_reduce_scatter=mlp_reduce_scatter, - complete=complete_now, - ) - defer_moe_finalize = ( - self.fusions is not None - and self.fusions.can_defer_finalize(self, forward_batch) - ) - # Producers declare remaining work independently of the kernel chosen - # by the consumer. Every handoff also carries an unfused completion. - fuse_mlp_allreduce = defer_moe_finalize or self._ffn_sum_moves_to_next_layer( - forward_batch, mlp_reduce_scatter=mlp_reduce_scatter, dp_step=dp_step - ) - if fuse_mlp_allreduce: - group = self.ffn_reduction_group(forward_batch) - if steps.returns_over_dp: - # Under attention DP the next layer also brings the sum back to - # this rank's tokens. - wrap = partial( - UnreducedOutput, - reduce_and_redistribute=partial( - _all_reduce_then_to_local_tokens, group, forward_batch - ), - ) - else: - wrap = partial(UnreducedOutput, group=group) - complete = partial(_leave_to_next_layer, wrap) - elif ( - dp_step is not None - and not self.is_last_layer - and self._local_token_move_can_go_to_next_layer(forward_batch) - ): - complete = partial( - _leave_to_next_layer, - partial( - UnreducedOutput, - reduce_and_redistribute=partial( - _to_local_tokens, dp_step, forward_batch - ), - ), - ) - else: - complete = complete_now - return FfnCompletion( - defer_moe_finalize=defer_moe_finalize, - fuse_mlp_allreduce=fuse_mlp_allreduce, - mlp_reduce_scatter=mlp_reduce_scatter, - complete=complete, - ) - - def _complete_ffn_output_now( - self, - hidden_states: torch.Tensor, - residual: torch.Tensor, - *, - forward_batch: ForwardBatch, - dp_step: Optional[Callable], - ) -> Tuple[torch.Tensor, torch.Tensor]: - """This layer's postprocess, run with the attention-DP step already - chosen: the move back to where the next layer reads the FFN output, then - the write-back into the residual for a layer that does it itself.""" - steps = self._batch_steps(forward_batch) - if steps.returns_over_dp: - hidden_states = _to_local_tokens( - dp_step or _redistribute_output, forward_batch, hidden_states - ) - else: - hidden_states, residual = steps.ffn_output_move( - hidden_states=hidden_states, - residual=residual, - forward_batch=forward_batch, - allow_reduce_scatter=self.allow_reduce_scatter, - is_layer_sparse=self.layer_facts.is_layer_sparse, - ) - update = steps.ffn_output.update - if residual is not None and update.at_producer: - hidden_states = update.update(hidden_states, residual) - residual = None - return hidden_states, residual - - def mixer_exit(self, forward_batch: ForwardBatch) -> "MixerExit": - """Decide once whether this stage's mixer (an attention-like stage) - skips its output all-reduce. Use the result as a context manager around - the mixer, then call ``finish``.""" - return MixerExit(self, forward_batch) - - def ffn_exit(self, forward_batch: ForwardBatch) -> "FfnExit": - """Decide once how this layer's FFN output reduction completes. Use the - result as a context manager around the FFN call, then call ``finish``.""" - return FfnExit(self, forward_batch) - - def _branch_rows( - self, forward_batch: ForwardBatch - ) -> Tuple[Layout, Layout, Layout]: - """The rows of this layer's FFN input, of its residual while the FFN - runs, and of what the layer hands on, for a batch that runs its ordinary - steps.""" - if self._batch_steps(forward_batch) is not self._steps: - raise NotImplementedError( - "a branch on a batch with attention CP, LayerNorm SP or " - "input-scattered attention" - ) - if self._declared is None: - raise NotImplementedError("a branch on a layer that is one stage") - sides = self._declared - return sides.ffn.input.layout, sides.ffn_residual_rows, sides.output_rows - - def branch_input( - self, - source: "LayerCommunicator", - hidden_states: torch.Tensor, - residual: torch.Tensor, - forward_batch: ForwardBatch, - ) -> Tuple[torch.Tensor, torch.Tensor]: - """The FFN input and residual that ``source``'s boundary read for its - own FFN, for this layer's FFN, which branches from the same input: moved - to the rows this FFN needs and its residual's rows.""" - rows, residual_rows, _ = source._branch_rows(forward_batch) - to, residual_to, _ = self._branch_rows(forward_batch) - stream = residual if isinstance(residual, ResidualStream) else None - if stream is not None: - if stream.pending is not None: - raise RuntimeError("a branch must start from a prepared stage input") - residual = stream.residual - hidden_states = move_rows(hidden_states, rows, to, forward_batch) - residual = move_rows(residual, residual_rows, residual_to, forward_batch) - forward_batch.residual_stream = ResidualStream(residual) - return hidden_states, forward_batch.residual_stream - - def branch_output( - self, hidden_states: torch.Tensor, forward_batch: ForwardBatch - ) -> torch.Tensor: - """This layer's complete FFN output as a branch's contribution, which - adds to the layer's output without writing the residual: moved to the - rows the layer hands on.""" - rows, _, to = self._branch_rows(forward_batch) - return move_rows(hidden_states, rows, to, forward_batch) - - def merge_branch( - self, - contribution: torch.Tensor, - hidden_states: torch.Tensor, - residual: torch.Tensor, - source: "LayerCommunicator", - forward_batch: ForwardBatch, - ) -> Tuple[torch.Tensor, torch.Tensor]: - """A contribution from ``branch_output`` summed with what ``source``'s - layer hands on, ``hidden_states`` and ``residual``, moved to the rows - this layer hands on.""" - _, _, rows = source._branch_rows(forward_batch) - _, _, to = self._branch_rows(forward_batch) - stream = residual if isinstance(residual, ResidualStream) else None - update = None - if stream is not None: - stream.check(hidden_states) - update = stream.pending.update - hidden_states, residual = stream.finish(hidden_states) - hidden_states = move_rows(hidden_states, rows, to, forward_batch) - residual = move_rows(residual, rows, to, forward_batch) - output = contribution + hidden_states - if stream is not None: - stream.write(residual) - return stream.leave(output, update), stream - return output, residual - - def _ffn_sum_can_move_to_next_layer(self, forward_batch: ForwardBatch) -> bool: - # Under the MoE-CP all-gather the fusion path would skip postprocess_layer - # and its MoE-CP scatter, leaving hidden_states longer than the residual. - if ( - (self.fusions is not None and self.fusions.requires_local_reduction) - or is_enable_moe_cp_allgather() - or not self._batch_steps(forward_batch).ffn_sum_is_movable - ): - return False - - # The fused residual+LN reduces over a single group. Hybrid EP+TP spans - # two disjoint groups; post_experts_all_reduce() merges them into one - # _TP reduction when moe_dp_size == 1, which the fused kernel can absorb. - # When merging is blocked, no single group covers both, so fusion stays off. - parallel = get_parallel() - if ( - parallel.moe_ep_size > 1 - and parallel.moe_tp_size > 1 - and not can_merge_post_experts_all_reduce() - ): - return False - - if ( - is_dp_attention_enabled() - and self._speculative_algo is not None - and self._speculative_algo.is_eagle() - ): - return False - - return not get_attn_tp_context().input_scattered - - def _ffn_sum_moves_to_next_layer( - self, - forward_batch: ForwardBatch, - *, - mlp_reduce_scatter: bool, - dp_step: Optional[Callable], - ) -> bool: - """Whether the FFN leaves its output's all-reduce to the next layer's - input when no fused kernel takes it: when the next layer would run the - same all-reduce the FFN itself would have.""" - return ( - get_parallel().tp_size > 1 - and not self.is_last_layer - and self._ffn_sum_can_move_to_next_layer(forward_batch) - and _can_defer_ffn_reduction(forward_batch, self) - and not mlp_reduce_scatter - # Under attention DP the next layer must also run postprocess's - # scatter back to this rank's tokens, and nothing more. - and ( - not is_dp_attention_enabled() - or ( - self._local_token_move_can_go_to_next_layer(forward_batch) - and dp_step is None - ) - ) - ) - - -class FfnCompletion(msgspec.Struct, frozen=True): - """One FFN's reduction decision. The flags are published while the FFN runs; - ``complete(hidden_states, residual)`` then either wraps the output for the - next layer's input to complete, or runs this layer's postprocess step.""" - - defer_moe_finalize: bool - fuse_mlp_allreduce: bool - mlp_reduce_scatter: bool - complete: Callable[[torch.Tensor, torch.Tensor], Tuple] - - -def _leave_to_next_layer( - wrap: Callable[[torch.Tensor], UnreducedOutput], - hidden_states: torch.Tensor, - residual: torch.Tensor, -) -> Tuple[UnreducedOutput, torch.Tensor]: - return wrap(hidden_states), residual - - -class MixerExit: - """The scope that publishes a mixer's decision while it runs: inside the - ``with`` block ``fuse_mlp_allreduce`` on ``get_forward()`` tells its - row-parallel output projection to skip the all-reduce. It skips when the - stage's output always leaves its sum (to an FFN stage, which completes it in - its input), and when its output declaration permits deferring the sum. The - consumer decides whether to fuse its completion with the input norm.""" - - __slots__ = ( - "skips_reduction", - "_hands_on", - "_scope", - "_update", - "_declared_sum", - "_stream", - ) - - def __init__(self, communicator: LayerCommunicator, forward_batch: ForwardBatch): - produced = communicator._batch_steps(forward_batch).ffn_output - self._stream = None - self._update = produced.update - self._declared_sum = produced.group if produced.always_leaves else None - self._hands_on = ( - produced.leaves_for_next_layer - and get_parallel().tp_size > 1 - and not is_dp_attention_enabled() - and communicator._ffn_sum_can_move_to_next_layer(forward_batch) - ) - self.skips_reduction = produced.always_leaves or self._hands_on - self._scope = get_forward().scoped(fuse_mlp_allreduce=self.skips_reduction) - - def __enter__(self) -> "MixerExit": - self._scope.__enter__() - return self - - def __exit__(self, *exc_info): - return self._scope.__exit__(*exc_info) - - def finish(self, hidden_states: torch.Tensor, residual=None): - """The mixer's output: its partial sum to the FFN stage after it, as a - value that owes the sum to a mixer after it, or complete.""" - if self._stream is not None: - residual = self._stream - if self._hands_on: - hidden_states = UnreducedOutput( - hidden_states, group=get_parallel().tp_group - ) - if isinstance(residual, ResidualStream): - output = residual.leave( - hidden_states, self._update, declared_sum=self._declared_sum - ) - return output if self._stream is not None else (output, residual) - return hidden_states - - -class FfnExit: - """The scope that publishes an FfnCompletion while the FFN runs: inside the - ``with`` block it is ``fuse_mlp_allreduce`` / ``mlp_reduce_scatter`` / - ``defer_moe_finalize`` on ``get_forward()``.""" - - __slots__ = ( - "communicator", - "forward_batch", - "defer_moe_finalize", - "fuse_mlp_allreduce", - "mlp_reduce_scatter", - "_complete", - "_update", - "_declared_sum", - "_scope", - "_stream", - ) - - def __init__(self, communicator: LayerCommunicator, forward_batch: ForwardBatch): - self._stream = None - self.communicator = communicator - self.forward_batch = forward_batch - completion = communicator._select_ffn_completion(forward_batch) - self.defer_moe_finalize = completion.defer_moe_finalize - self.fuse_mlp_allreduce = completion.fuse_mlp_allreduce - self.mlp_reduce_scatter = completion.mlp_reduce_scatter - self._complete = completion.complete - steps = communicator._batch_steps(forward_batch) - self._update = steps.ffn_output.update - self._declared_sum = communicator._declared_ffn_sum( - steps, completion.mlp_reduce_scatter - ) - self._scope = get_forward().scoped( - fuse_mlp_allreduce=self.fuse_mlp_allreduce, - mlp_reduce_scatter=self.mlp_reduce_scatter, - defer_moe_finalize=self.defer_moe_finalize, - ) - - def __enter__(self) -> "FfnExit": - self._scope.__enter__() - return self - - def __exit__(self, *exc_info): - return self._scope.__exit__(*exc_info) - - def finish( - self, hidden_states: torch.Tensor, residual: torch.Tensor = None - ) -> Tuple[Union[torch.Tensor, UnreducedOutput], torch.Tensor]: - """Leave the reduction to the next layer's input, or postprocess.""" - stream = residual_batch.current(self.forward_batch) - if stream is not None: - residual = stream.residual - if not isinstance(hidden_states, torch.Tensor): - # A deferred MoE finalize handoff, consumed by the next prepare_attn. - assert self.defer_moe_finalize, "unrequested deferred MoE handoff" - else: - hidden_states, residual = self._complete(hidden_states, residual) - result = self.communicator._leave_ffn_output( - hidden_states, residual, stream, self._update, self._declared_sum - ) - return result[0] if self._stream is not None else result - - -class MHCLayerCommunicator(LayerCommunicator): - """A layer whose residual is hyper-connection streams: the shared boundary - steps, run with MHCState's residual operations.""" - - def __init__( - self, - layer_facts: LayerFacts, - input_layernorm: torch.nn.Module, - post_attention_layernorm: torch.nn.Module, - allow_reduce_scatter: bool = False, - qkv_latent_func: Optional[Callable] = None, - *, - hc_mult: int, - hc_attn_pre: Callable, - hc_ffn_pre: Callable, - hc_post: Callable, - hc_ffn_post_pre: Optional[Callable] = None, - ): - self.mhc = MHCState( - hc_mult=hc_mult, - hc_attn_pre=hc_attn_pre, - hc_ffn_pre=hc_ffn_pre, - hc_post=hc_post, - hc_ffn_post_pre=hc_ffn_post_pre, - is_last_layer=layer_facts.is_last_layer, - ) - if layer_facts.is_layer_sparse and sparse_moe_gathers_over_moe_cp(): - raise NotImplementedError( - "MHCLayerCommunicator does not support a MoE gathered over the " - "MoE-CP group (moe_dp_size < attention_context_parallel_size). " - "Increase moe_dp_size to match attention_context_parallel_size." - ) - # The postprocess writes the FFN output into the streams, so the FFN's - # sum never waits for the next layer. - super().__init__( - layer_facts, - input_layernorm, - post_attention_layernorm, - allow_reduce_scatter, - qkv_latent_func, - allow_deferred_ffn_reduction=False, - residual=self.mhc.layer_residual(), - ) - # MHC has not been run with input-scattered attention under attention CP. - if get_parallel().attn_cp_size > 1 and self._input_can_be_scattered(): - raise NotImplementedError( - "MHCLayerCommunicator with input-scattered attention under attention CP" - ) - - def _steps_from_declarations( - self, sides: DecoderLayerSides, **kwargs - ) -> BoundarySteps: - # MHC has not been run with an FFN input gathered over attention CP. - if ( - TokenAxis.ATTN_CP - in sides.attention.output.layout.sharded - sides.ffn.input.layout.sharded - ): - raise NotImplementedError( - f"MHCLayerCommunicator with a gather over attention CP: {sides=}" - ) - return super()._steps_from_declarations(sides, **kwargs) diff --git a/python/sglang/srt/layers/communicator/layout.py b/python/sglang/srt/layers/communicator/layout.py index afa617479d..0d28a6d07d 100644 --- a/python/sglang/srt/layers/communicator/layout.py +++ b/python/sglang/srt/layers/communicator/layout.py @@ -142,7 +142,9 @@ def _sum_group(group: SumGroup) -> GroupCoordinator: return parallel.attn_tp_group if group is SumGroup.TP: return parallel.tp_group - return post_experts_reduction_group() + if group is SumGroup.MOE_OUTPUT: + return post_experts_reduction_group() + raise ValueError(f"unsupported output sum group: {group!r}") def token_axis_sizes(*, cp_active: bool = False) -> Dict[TokenAxis, int]: diff --git a/python/sglang/srt/layers/communicator/legacy_stage.py b/python/sglang/srt/layers/communicator/legacy_stage.py deleted file mode 100644 index 476b9bbc90..0000000000 --- a/python/sglang/srt/layers/communicator/legacy_stage.py +++ /dev/null @@ -1,160 +0,0 @@ -# Copyright 2023-2024 SGLang Team -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""A stage's model-facing input and output boundary.""" - -from typing import TYPE_CHECKING, Optional - -from sglang.srt.layers.communicator.boundary import BoundarySteps, StageEntry -from sglang.srt.layers.communicator.residual.add_norm import ADD -from sglang.srt.layers.communicator.residual.batch import current -from sglang.srt.model_executor.forward_batch_info import ForwardBatch - -if TYPE_CHECKING: - from sglang.srt.layers.communicator.layer import LayerCommunicator - - -class StageCommunicator: - """One stage of a layer, its attention or its FFN: the boundary into it, - run with the stage's norm on the steps the layer chose for the batch.""" - - def __init__(self, layer: "LayerCommunicator", name: str, norm_name: str): - self._layer = layer - self._name = name - self.norm = getattr(layer, norm_name) - - def entry( - self, forward_batch: ForwardBatch, steps: Optional[BoundarySteps] = None - ) -> StageEntry: - if steps is None: - steps = self._layer._batch_steps(forward_batch) - entry = getattr(steps, self._name) - if entry is None: - raise NotImplementedError( - f"a layer that is one stage has no {self._name} stage" - ) - return entry - - def _prepare( - self, - hidden_states, - residual, - forward_batch: ForwardBatch, - steps: Optional[BoundarySteps] = None, - **call, - ): - """The stage's input and the residual, from the previous stage's output - (``call``: what the stage's read takes, e.g. the attention's - ``quant_format``).""" - entry = self.entry(forward_batch, steps) - stream = current(forward_batch) - if residual is not stream: - raise RuntimeError("residual alias belongs to a different invocation") - if ( - stream.pending is None - and stream.residual is None - and entry.input_sum is not None - ): - hidden_states = stream.leave( - hidden_states, ADD, declared_sum=entry.input_sum - ) - if stream.pending is not None: - call["update"] = stream.pending.update - call["pending"] = stream.pending - if stream.pending is None and stream.residual is not None: - call["written"] = True - hidden_states, residual = stream.input(hidden_states) - hidden_states, residual = entry.prepare( - hidden_states, residual, forward_batch, self.norm, **call - ) - if entry.input_move is not None: - hidden_states = entry.input_move( - hidden_states=hidden_states, - forward_batch=forward_batch, - ) - if entry.handoff is not None: - hidden_states = entry.handoff( - hidden_states, forward_batch, self._layer.qkv_latent_func - ) - stream.write(residual) - return hidden_states, stream - - @property - def input_rows(self): - return self._layer.input_rows - - @property - def input_on_attention_tp_slices(self): - return self._layer.input_on_attention_tp_slices - - def prepare(self, hidden_states, forward_batch, *, cache=None, **call): - stream = current(forward_batch) - if self._name == "attention": - hidden_states, stream = ( - self._layer.prepare_attn_and_capture_last_layer_outputs( - hidden_states, stream, forward_batch, **call - ) - ) - else: - steps = self._layer._batch_steps(forward_batch) - self._layer.publish_mlp_lora_layout(steps) - if cache is not None: - call["cache"] = cache - hidden_states, stream = self._prepare( - hidden_states, stream, forward_batch, steps, **call - ) - forward_batch.residual_stream = stream - return hidden_states - - def finish(self, hidden_states, forward_batch): - """Hand this attention's actual contribution to the following stage.""" - if self._name != "attention" or self._layer.stage_edges is not None: - raise RuntimeError( - "use the stage exit scope for an FFN or single-stage mixer" - ) - return current(forward_batch).leave( - hidden_states, - self._layer._residual.attention_update, - declared_sum=self._layer._batch_steps(forward_batch).ffn.input_sum, - ) - - def exit(self, forward_batch): - result = ( - self._layer.ffn_exit(forward_batch) - if self._name == "ffn" - else self._layer.mixer_exit(forward_batch) - ) - result._stream = current(forward_batch) - return result - - def from_pp(self, tensors, forward_batch, **kwargs): - hidden_states, stream = self._layer.from_pp(tensors, forward_batch, **kwargs) - forward_batch.residual_stream = stream - return hidden_states - - def snapshot(self, hidden_states, forward_batch, *, at_input=False): - return self._layer.snapshot( - hidden_states, current(forward_batch), at_input=at_input - ) - - def capture_output(self, hidden_states, forward_batch, **kwargs): - return self._layer.capture_output( - hidden_states, current(forward_batch), **kwargs - ) - - def postprocess(self, hidden_states, forward_batch): - hidden_states, stream = self._layer.postprocess_layer( - hidden_states, current(forward_batch), forward_batch - ) - forward_batch.residual_stream = stream - return hidden_states diff --git a/python/sglang/srt/layers/communicator/ops.py b/python/sglang/srt/layers/communicator/ops.py index a9a3810654..1c6be8b3e9 100644 --- a/python/sglang/srt/layers/communicator/ops.py +++ b/python/sglang/srt/layers/communicator/ops.py @@ -22,36 +22,25 @@ from sglang.srt.distributed import ( GroupCoordinator, attention_tensor_model_parallel_all_reduce, attention_tensor_model_parallel_quant_all_reduce, - tensor_model_parallel_all_reduce, ) from sglang.srt.distributed.device_communicators.pynccl_allocator import ( use_symmetric_memory, ) from sglang.srt.layers.communicator.adapters.attention import ( - AttentionInputs, _redistribute_from_attn_tp_shards, _redistribute_to_attn_tp_shards, - get_attn_tp_context, ) from sglang.srt.layers.communicator.adapters.context_parallel import ( - dsa_cp_gather_hidden_states, dsa_cp_reduce_scatter_hidden_states, ) from sglang.srt.layers.communicator.layout import ( Layout, - SumGroup, TokenAxis, _cp_shard_token_rows, moe_cp_gathered_rows, ) -from sglang.srt.layers.communicator.output import ( - HandoffOutput, - UnreducedOutput, - reduce_output, -) -from sglang.srt.layers.communicator.residual import StageRead, StageUpdate -from sglang.srt.layers.communicator.residual.add_norm import ADD, NORM_READ -from sglang.srt.layers.communicator.residual.stream import DeclaredSum +from sglang.srt.layers.communicator.residual import StageUpdate +from sglang.srt.layers.communicator.residual.add_norm import ADD from sglang.srt.layers.dp_attention import ( attn_tp_all_gather_into_tensor, attn_tp_reduce_scatter_tensor, @@ -63,21 +52,11 @@ from sglang.srt.layers.dp_attention import ( get_global_dp_buffer, get_local_dp_buffer, get_moe_cp_rank, - get_moe_cp_size, is_allocation_symmetric, moe_cp_all_gather_into_tensor, ) from sglang.srt.model_executor.forward_batch_info import ForwardBatch from sglang.srt.runtime_context import get_exec, get_parallel -from sglang.srt.utils import ( - is_npu, -) - -_is_npu = is_npu() - - -if _is_npu: - from sglang.srt.hardware_backend.npu.cmo import prepare_weight_cache def tp_reduce_scatter( @@ -110,43 +89,34 @@ def tp_slice(hidden_states, residual): return hidden_states, residual -class CommunicateSimpleFn: - @staticmethod - def _trivial( - hidden_states: torch.Tensor, - forward_batch: ForwardBatch, - ) -> torch.Tensor: - return hidden_states - - @staticmethod - def _scattered_to_tp_attn_full( - hidden_states: Union[torch.Tensor, Tuple[torch.Tensor, torch.Tensor]], - forward_batch: ForwardBatch, - ) -> Union[torch.Tensor, Tuple[torch.Tensor, torch.Tensor]]: - parallel = get_parallel() - if isinstance(hidden_states, tuple): - gathered_hidden_states = [] - for local_hidden_states in hidden_states: - with use_symmetric_memory( - parallel.tp_group, - disabled=not is_allocation_symmetric(), - ): - output = torch.empty( - ( - local_hidden_states.shape[0] * parallel.attn_tp_size, - *local_hidden_states.shape[1:], - ), - dtype=local_hidden_states.dtype, - device=local_hidden_states.device, - ) - attn_tp_all_gather_into_tensor( - output, - local_hidden_states, +def gather_attention_tp( + hidden_states: Union[torch.Tensor, Tuple[torch.Tensor, torch.Tensor]], + forward_batch: ForwardBatch, +) -> Union[torch.Tensor, Tuple[torch.Tensor, torch.Tensor]]: + parallel = get_parallel() + if isinstance(hidden_states, tuple): + gathered_hidden_states = [] + for local_hidden_states in hidden_states: + with use_symmetric_memory( + parallel.tp_group, + disabled=not is_allocation_symmetric(), + ): + output = torch.empty( + ( + local_hidden_states.shape[0] * parallel.attn_tp_size, + *local_hidden_states.shape[1:], + ), + dtype=local_hidden_states.dtype, + device=local_hidden_states.device, ) - gathered_hidden_states.append(output) - return tuple(gathered_hidden_states) + attn_tp_all_gather_into_tensor( + output, + local_hidden_states, + ) + gathered_hidden_states.append(output) + return tuple(gathered_hidden_states) - return _redistribute_from_attn_tp_shards(hidden_states) + return _redistribute_from_attn_tp_shards(hidden_states) def _reduce_and_redistribute_output_to_attn_tp_shards( @@ -244,145 +214,6 @@ def move_rows( return hidden_states -def _mlp_input_without_dp( - hidden_states: torch.Tensor, - residual: torch.Tensor, - forward_batch: ForwardBatch, - layernorm: torch.nn.Module, - *, - cache=None, - gathers_residual: bool, - fusions: Tuple[Callable, ...], - group: SumGroup = SumGroup.ATTN_TP, - read: StageRead = NORM_READ, - update: StageUpdate = ADD, -): - """Complete the sum the input owes over ``group`` on the rows it is on, - unless one of ``fusions`` does it with the residual add and the norm, then - write it into the residual and read the input.""" - if gathers_residual: - residual = update.residual_from_attn_tp_shards(residual) - for fused in fusions: - result = fused(hidden_states, residual, forward_batch) - if result is not None: - return result - if group is SumGroup.ATTN_TP: - # MHC sums its streams in full precision. - hidden_states = _mlp_input_reduce_output( - hidden_states, forward_batch, may_quantize=update.adds_plainly - ) - elif group is SumGroup.TP: - hidden_states = tensor_model_parallel_all_reduce(hidden_states) - if _is_npu and cache is not None: - _ = prepare_weight_cache(hidden_states, cache) - return read.update_and_read(update, hidden_states, residual, layernorm) - - -def _mlp_input_dp_replicate( - hidden_states: torch.Tensor, - residual: torch.Tensor, - forward_batch: ForwardBatch, - layernorm: torch.nn.Module, - *, - cache=None, - gathers_residual: bool, - reduces_attention_tp: bool, - places_cp_shards: bool = False, - read: StageRead = NORM_READ, - update: StageUpdate = ADD, -): - """Attention DP: complete the attention-TP sum if it is owed, write the - output into the residual and read the FFN input locally, then gather. With - ``places_cp_shards`` each CP rank puts its shard of the DP group's tokens - beside the others' in the group's slot.""" - if gathers_residual: - residual = update.residual_from_attn_tp_shards(residual) - if hidden_states.shape[0] != 0: - if reduces_attention_tp: - hidden_states = attention_tensor_model_parallel_all_reduce(hidden_states) - with use_symmetric_memory( - get_parallel().tp_group, - disabled=not is_allocation_symmetric(), - ): - hidden_states, residual = read.update_and_read( - update, hidden_states, residual, layernorm - ) - else: - hidden_states, residual = read.update_and_read( - update, hidden_states, residual, layernorm - ) - cp_shard_counts = _cp_shard_token_rows(forward_batch) if places_cp_shards else None - return ( - _redistribute_input_to_dp(hidden_states, forward_batch, cp_shard_counts), - residual, - ) - - -def _mlp_input_dp_partial( - hidden_states: torch.Tensor, - residual: torch.Tensor, - forward_batch: ForwardBatch, - layernorm: torch.nn.Module, - *, - cache=None, - gathers_residual: bool, - places_cp_shards: bool = False, - read: StageRead = NORM_READ, - update: StageUpdate = ADD, -): - """Attention DP: one rank adds the residual, the gather sums it, then read - the input from the sum. With ``places_cp_shards`` each CP rank puts its - shard of the DP group's tokens beside the others' in the group's slot.""" - if gathers_residual: - residual = _redistribute_from_attn_tp_shards(residual) - if get_parallel().attn_tp_rank == 0: - hidden_states += residual - cp_shard_counts = _cp_shard_token_rows(forward_batch) if places_cp_shards else None - hidden_states = _reduce_and_redistribute_output_to_dp( - hidden_states, forward_batch, cp_shard_counts - ) - dp_scatter(residual, hidden_states, forward_batch, cp_shard_counts) - hidden_states, _ = read.read(hidden_states, layernorm) - return hidden_states, residual - - -def _hand_qkv_hook_its_input( - hidden_states: torch.Tensor, - forward_batch: ForwardBatch, - qkv_latent_func: Optional[Callable], -) -> torch.Tensor: - """Give the attention's QKV hook the attention input.""" - if qkv_latent_func is not None: - get_attn_tp_context().set_attn_inputs( - AttentionInputs(hidden_states, forward_batch, qkv_latent_func) - ) - return hidden_states - - -def _hand_scattered_input_to_attention( - hidden_states: torch.Tensor, - forward_batch: ForwardBatch, - qkv_latent_func: Optional[Callable], -) -> torch.Tensor: - """Input-scattered attention takes each rank's slice, and its QKV hook - gathers the rows after the projection. DSA and attention without a hook - consume full hidden states, so those are gathered here, and the hook is - told they are.""" - ctx = get_attn_tp_context() - if ctx.is_dsa or qkv_latent_func is None: - hidden_states = _tp_all_gather_scattered_rows(hidden_states, forward_batch) - if qkv_latent_func is not None: - ctx.set_attn_inputs( - AttentionInputs( - hidden_states, - forward_batch, - qkv_latent_func, - is_pre_gathered=ctx.is_dsa, - ) - ) - return hidden_states - - def _tp_all_gather_scattered_rows( hidden_states: torch.Tensor, forward_batch: ForwardBatch ) -> torch.Tensor: @@ -393,267 +224,6 @@ def _tp_all_gather_scattered_rows( return output -def _dispatch_consumer( - hidden_states, residual, forward_batch, norm, *, paths, update=ADD, **call -): - if residual is None: - # Enter/read-only paths do not execute an update. - prepare = paths[False] - else: - try: - prepare = paths[update.adds_plainly] - except KeyError: - raise RuntimeError("producer update has no bound input path") from None - return prepare(hidden_states, residual, forward_batch, norm, update=update, **call) - - -def _consumer_step( - hidden_states: Union[torch.Tensor, UnreducedOutput, HandoffOutput], - residual: Optional[torch.Tensor], - forward_batch: ForwardBatch, - norm: torch.nn.Module, - *, - step: Callable, - adds_plainly: bool, - carried_fusions: Tuple[Callable, ...], - expected_sum: Optional[SumGroup] = None, - completed_step: Optional[Callable] = None, - pending=None, - written_step: Optional[Callable] = None, - written: bool = False, - update: StageUpdate = ADD, - **call, -): - """A boundary's half into a stage: complete what the value carries, a sum - or a handoff its producer left for this batch, then run ``step`` on the - complete value. Under attention DP the reduction back to this rank's tokens - comes first; otherwise one of ``carried_fusions`` may complete it with the - residual add and the read, and an empty batch has nothing to sum. An input - whose declared sum is already completed uses ``completed_step`` instead; - a pending declared sum must match ``expected_sum``. ``call`` is what the stage's read takes (the attention's - ``quant_format`` and ``post_residual_addition``).""" - if residual is not None and update.adds_plainly != adds_plainly: - raise RuntimeError("producer update does not match the boundary's capability") - if pending is not None: - if isinstance(pending.owed, DeclaredSum): - if pending.owed.group is not expected_sum: - raise RuntimeError("producer sum does not match the input declaration") - elif expected_sum is not None: - if pending.owed is not None: - raise RuntimeError("a declared input sum carried another completion") - step = completed_step - if written and written_step is not None: - step = written_step - if not isinstance(hidden_states, torch.Tensor): - owed = hidden_states - if residual is None: - raise RuntimeError(f"{type(owed).__name__} requires residual input") - if expected_sum is not None: - raise RuntimeError( - f"an input that owes its sum by construction arrived as " - f"{type(owed).__name__}" - ) - if ( - isinstance(owed, UnreducedOutput) - and owed.reduce_and_redistribute is not None - ): - hidden_states = reduce_output(owed) - else: - for fused in carried_fusions: - result = fused( - owed, residual, forward_batch, call.get("post_residual_addition") - ) - if result is not None: - return result - if isinstance(owed, HandoffOutput) or owed.partial.shape[0] != 0: - hidden_states = reduce_output(owed) - else: - hidden_states = owed.partial - return step(hidden_states, residual, forward_batch, norm, update=update, **call) - - -def _read_input( - hidden_states: torch.Tensor, - residual: Optional[torch.Tensor], - forward_batch: ForwardBatch, - norm: torch.nn.Module, - *, - cache=None, - layer_input: Optional[Callable], - enters_stack: bool, - read: StageRead, - update: StageUpdate = ADD, - quant_format: str = "", - post_residual_addition: Optional[torch.Tensor] = None, -): - """Complete what the input owes by construction (``layer_input``), then - write the previous stage's output into the residual and read the stage's - input with ``norm``. The layer stack's first stage (``enters_stack``) - starts its residual from its input.""" - enters = residual is None and enters_stack - if layer_input is not None: - hidden_states, residual = layer_input(hidden_states, residual) - if enters: - hidden_states, residual = read.enter(hidden_states), None - if residual is None: - # The previous layer already wrote its output into the residual. - return read.read(hidden_states, norm, quant_format) - return read.update_and_read( - update, - hidden_states, - residual, - norm, - quant_format=quant_format, - post_residual_addition=post_residual_addition, - ) - - -def _mlp_input_scatter( - hidden_states: torch.Tensor, - residual: torch.Tensor, - forward_batch: ForwardBatch, - layernorm: torch.nn.Module, - *, - cache=None, - scatters_residual: bool, - read: StageRead = NORM_READ, - update: StageUpdate = ADD, -): - hidden_states = _reduce_and_redistribute_output_to_attn_tp_shards(hidden_states) - if scatters_residual: - residual = update.residual_to_attn_tp_shard(residual) - return read.update_and_read(update, hidden_states, residual, layernorm) - - -def _mlp_input_slice( - hidden_states: torch.Tensor, - residual: Optional[torch.Tensor], - forward_batch: ForwardBatch, - layernorm: torch.nn.Module, - *, - cache=None, - scatters_residual: bool, - read: StageRead = NORM_READ, - update: StageUpdate = ADD, -): - hidden_states = _redistribute_to_attn_tp_shards(hidden_states).clone() - if scatters_residual and residual is not None: - residual = update.residual_to_attn_tp_shard(residual) - return read.update_and_read(update, hidden_states, residual, layernorm) - - -def _mlp_input_on_residual_shard( - hidden_states: torch.Tensor, - residual: torch.Tensor, - forward_batch: ForwardBatch, - layernorm: torch.nn.Module, - *, - cache=None, - reduces: bool = True, - read: StageRead, - update: StageUpdate, -): - """The residual stays on each rank's slice while the FFN takes the full - rows: reduce-scatter the attention output onto the slice, which completes - its sum, write it into the residual and read the FFN input there, then - gather the input back into the attention output's rows.""" - parallel = get_parallel() - if hidden_states.shape[0] == 0: - return hidden_states, hidden_states - shard = hidden_states.tensor_split(parallel.tp_size)[parallel.tp_rank] - if reduces: - parallel.tp_group.reduce_scatter_tensor(shard, hidden_states) - else: - shard = shard.clone() - shard, residual = read.update_and_read(update, shard, residual, layernorm) - attn_tp_all_gather_into_tensor(hidden_states, shard) - return hidden_states, residual - - -def _mlp_input_residual_into_sum( - hidden_states: torch.Tensor, - residual: torch.Tensor, - forward_batch: ForwardBatch, - layernorm: torch.nn.Module, - *, - cache=None, - read: StageRead = NORM_READ, - update: StageUpdate = ADD, -): - return _tp_all_reduce_with_scattered_residual( - hidden_states, residual, layernorm, read - ) - - -def _tp_all_reduce_with_scattered_residual( - hidden_states: torch.Tensor, - residual: torch.Tensor, - layernorm: torch.nn.Module, - read: StageRead = NORM_READ, -): - parallel = get_parallel() - if hidden_states.shape[0] == 0: - return hidden_states, hidden_states - - scattered_states = hidden_states.tensor_split(parallel.tp_size)[parallel.tp_rank] - scattered_states += residual - residual = tensor_model_parallel_all_reduce(hidden_states) - return read.read(residual, layernorm) - - -def _mlp_input_gather_attention_cp( - hidden_states: torch.Tensor, - residual: torch.Tensor, - forward_batch: ForwardBatch, - layernorm: torch.nn.Module, - *, - cache=None, - gather: Callable, - update: StageUpdate = ADD, -): - """DSA and MLA CP: complete this rank's shard, then gather the shards, of - equal length, over the attention-CP group. The residual stays on the - shard.""" - hidden_states, residual = gather( - hidden_states, residual, forward_batch, layernorm, update=update, cache=cache - ) - return dsa_cp_gather_hidden_states(hidden_states), residual - - -def _mlp_input_gather_moe_cp( - hidden_states: torch.Tensor, - residual: torch.Tensor, - forward_batch: ForwardBatch, - layernorm: torch.nn.Module, - *, - cache=None, - gather: Callable, - update: StageUpdate = ADD, -): - """Gather for the FFN, then over the MoE-CP group so each rank holds all - tokens of its MoE group (moe_dp_size < attn_cp_size). The residual stays at - TP_ATTN_FULL.""" - # Early return on empty tensor is safe for MOE_CP because: - # - During CP extend: zigzag split guarantees all CP ranks have non-zero tokens, - # so no rank hits this path while others proceed to the allgather. - # - During decode: moe_cp allgather is skipped (guarded by is_context_parallel_extend). - # - CUDA graph warmup: not applicable when --cuda-graph-backend-prefill=disabled is used. - if hidden_states.shape[0] == 0: - return hidden_states, residual - - hidden_states, residual = gather( - hidden_states, residual, forward_batch, layernorm, update=update, cache=cache - ) - - rows = moe_cp_gathered_rows(forward_batch) - if rows is not None and hidden_states.shape[0] > 0: - hidden_states = _redistribute_input_to_moe_cp( - hidden_states, rows, get_moe_cp_size() - ) - - return hidden_states, residual - - def _dp_scatter_group() -> GroupCoordinator: parallel = get_parallel() if parallel.tp_size == parallel.attn_dp_size: @@ -720,140 +290,134 @@ def _all_reduce_then_to_local_tokens( ) -class CommunicateSummableTensorPairFn: - """It is allowed to make (hidden_states, residual) := (hidden_states + residual, None) if needed.""" +def identity_output( + hidden_states: torch.Tensor, + residual: torch.Tensor, + forward_batch: ForwardBatch, + **kwargs, +): + return hidden_states, residual - @staticmethod - def _trivial( - hidden_states: torch.Tensor, - residual: torch.Tensor, - forward_batch: ForwardBatch, - **kwargs, - ): + +def update_and_gather( + hidden_states: torch.Tensor, + residual: torch.Tensor, + forward_batch: ForwardBatch, + update: StageUpdate = ADD, + **kwargs, +): + hidden_states = update.update(hidden_states, residual) + return _redistribute_from_attn_tp_shards(hidden_states), None + + +def output_on_residual_shard( + hidden_states: torch.Tensor, + residual: torch.Tensor, + forward_batch: ForwardBatch, + *, + sums: bool, + gathers_back: bool, + update: StageUpdate, + **kwargs, +): + """Bring the FFN output onto the slice of the rows the residual is on: + a reduce-scatter that also completes its sum when ``sums``, else this + rank's slice of the complete output. With ``gathers_back`` it is + written into the residual there and the full rows are gathered back.""" + parallel = get_parallel() + if sums: + hidden_states, _ = tp_reduce_scatter(hidden_states, None) + else: + hidden_states = hidden_states.tensor_split(parallel.tp_size)[parallel.tp_rank] + if not gathers_back: return hidden_states, residual + local_states = update.update(hidden_states, residual) + hidden_states = local_states.new_empty( + local_states.shape[0] * parallel.tp_size, *local_states.shape[1:] + ) + parallel.tp_group.all_gather_into_tensor(hidden_states, local_states) + return hidden_states, None - @staticmethod - def _gather( - hidden_states: torch.Tensor, - residual: torch.Tensor, - forward_batch: ForwardBatch, - update: StageUpdate = ADD, - **kwargs, - ): - hidden_states = update.update(hidden_states, residual) - return _redistribute_from_attn_tp_shards(hidden_states), None - @staticmethod - def _onto_residual_shard( - hidden_states: torch.Tensor, - residual: torch.Tensor, - forward_batch: ForwardBatch, - *, - sums: bool, - gathers_back: bool, - update: StageUpdate, - **kwargs, - ): - """Bring the FFN output onto the slice of the rows the residual is on: - a reduce-scatter that also completes its sum when ``sums``, else this - rank's slice of the complete output. With ``gathers_back`` it is - written into the residual there and the full rows are gathered back.""" - parallel = get_parallel() - if sums: - hidden_states, _ = tp_reduce_scatter(hidden_states, None) - else: - hidden_states = hidden_states.tensor_split(parallel.tp_size)[ - parallel.tp_rank - ] - if not gathers_back: - return hidden_states, residual - local_states = update.update(hidden_states, residual) - hidden_states = local_states.new_empty( - local_states.shape[0] * parallel.tp_size, *local_states.shape[1:] +def scatter_output( + hidden_states: torch.Tensor, + residual: torch.Tensor, + forward_batch: ForwardBatch, +): + assert residual is None, "not yet handled residual!=None" + return _redistribute_to_attn_tp_shards(hidden_states), None + + +def take_back_attention_cp_shard( + hidden_states: torch.Tensor, + residual: torch.Tensor, + forward_batch: ForwardBatch, + **kwargs, +): + """DSA and MLA CP: this rank's shard of a complete output gathered in + equal shards over the attention-CP group.""" + parallel = get_parallel() + shard = hidden_states.tensor_split(parallel.attn_cp_size)[parallel.attn_cp_rank] + return shard, residual + + +def reduce_scatter_over_cp( + hidden_states: torch.Tensor, + residual: torch.Tensor, + forward_batch: ForwardBatch, + **kwargs, +): + """DSA and MLA CP: sum the FFN output over the attention-CP group and + keep this rank's shard.""" + return dsa_cp_reduce_scatter_hidden_states(hidden_states), residual + + +def take_back_cp_shard( + hidden_states: torch.Tensor, + residual: torch.Tensor, + forward_batch: ForwardBatch, + **kwargs, +): + """This rank's CP shard of the rows the DP gather put in its DP group's + slot, at the shard's padded length with the padding zeroed.""" + held = forward_batch.attn_cp_metadata.per_rank_actual_token + local_hidden_states = get_local_dp_buffer(_dp_scatter_group())[ + : held[get_parallel().attn_cp_rank] + ] + dp_scatter( + local_hidden_states, + hidden_states, + forward_batch, + _cp_shard_token_rows(forward_batch), + ) + return local_hidden_states, residual + + +def scatter_moe_cp_output( + hidden_states: torch.Tensor, + residual: torch.Tensor, + forward_batch: ForwardBatch, + **kwargs, +): + """Scatter MoE output back to TP_ATTN_FULL after MOE_FULL computation. + + After moe_tensor_model_parallel_all_reduce (which runs unconditionally since + mlp_reduce_scatter=False for this path), all ranks in the moe_cp group hold the + full MoE result for all cp_per_moe token chunks. We simply slice out this rank's + CP-local portion. + + If DP>1, further scatter back to the local DP slice. + """ + # Only scatter back during prefill; decode was never allgathered so no-op. + # Safe w.r.t. empty tensors: same reasoning as _mlp_input_gather_moe_cp + # — CP extend always has non-zero tokens per rank, and decode skips this path. + rows = moe_cp_gathered_rows(forward_batch) + if rows is not None: + hidden_states = _redistribute_output_from_moe_cp(hidden_states, rows) + + if get_parallel().attn_dp_size > 1: + hidden_states = _to_local_tokens( + _redistribute_output, forward_batch, hidden_states ) - parallel.tp_group.all_gather_into_tensor(hidden_states, local_states) - return hidden_states, None - @staticmethod - def _scatter( - hidden_states: torch.Tensor, - residual: torch.Tensor, - forward_batch: ForwardBatch, - ): - assert residual is None, "not yet handled residual!=None" - return _redistribute_to_attn_tp_shards(hidden_states), None - - @staticmethod - def _take_back_attention_cp_shard( - hidden_states: torch.Tensor, - residual: torch.Tensor, - forward_batch: ForwardBatch, - **kwargs, - ): - """DSA and MLA CP: this rank's shard of a complete output gathered in - equal shards over the attention-CP group.""" - parallel = get_parallel() - shard = hidden_states.tensor_split(parallel.attn_cp_size)[parallel.attn_cp_rank] - return shard, residual - - @staticmethod - def _reduce_scatter_over_cp( - hidden_states: torch.Tensor, - residual: torch.Tensor, - forward_batch: ForwardBatch, - **kwargs, - ): - """DSA and MLA CP: sum the FFN output over the attention-CP group and - keep this rank's shard.""" - return dsa_cp_reduce_scatter_hidden_states(hidden_states), residual - - @staticmethod - def _take_back_cp_shard( - hidden_states: torch.Tensor, - residual: torch.Tensor, - forward_batch: ForwardBatch, - **kwargs, - ): - """This rank's CP shard of the rows the DP gather put in its DP group's - slot, at the shard's padded length with the padding zeroed.""" - held = forward_batch.attn_cp_metadata.per_rank_actual_token - local_hidden_states = get_local_dp_buffer(_dp_scatter_group())[ - : held[get_parallel().attn_cp_rank] - ] - dp_scatter( - local_hidden_states, - hidden_states, - forward_batch, - _cp_shard_token_rows(forward_batch), - ) - return local_hidden_states, residual - - @staticmethod - def _scatter_hidden_states_moe( - hidden_states: torch.Tensor, - residual: torch.Tensor, - forward_batch: ForwardBatch, - **kwargs, - ): - """Scatter MoE output back to TP_ATTN_FULL after MOE_FULL computation. - - After moe_tensor_model_parallel_all_reduce (which runs unconditionally since - mlp_reduce_scatter=False for this path), all ranks in the moe_cp group hold the - full MoE result for all cp_per_moe token chunks. We simply slice out this rank's - CP-local portion. - - If DP>1, further scatter back to the local DP slice. - """ - # Only scatter back during prefill; decode was never allgathered so no-op. - # Safe w.r.t. empty tensors: same reasoning as _mlp_input_gather_moe_cp - # — CP extend always has non-zero tokens per rank, and decode skips this path. - rows = moe_cp_gathered_rows(forward_batch) - if rows is not None: - hidden_states = _redistribute_output_from_moe_cp(hidden_states, rows) - - if get_parallel().attn_dp_size > 1: - hidden_states = _to_local_tokens( - _redistribute_output, forward_batch, hidden_states - ) - - return hidden_states, residual + return hidden_states, residual diff --git a/python/sglang/srt/layers/communicator/prepare.py b/python/sglang/srt/layers/communicator/prepare.py new file mode 100644 index 0000000000..9a452175c6 --- /dev/null +++ b/python/sglang/srt/layers/communicator/prepare.py @@ -0,0 +1,475 @@ +# Copyright 2023-2024 SGLang Team +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Complete pending output, update the residual, and read the consumer input.""" + +from typing import Callable, Optional, Tuple, Union + +import torch + +from sglang.srt.distributed import ( + attention_tensor_model_parallel_all_reduce, + tensor_model_parallel_all_reduce, +) +from sglang.srt.distributed.device_communicators.pynccl_allocator import ( + use_symmetric_memory, +) +from sglang.srt.layers.communicator.adapters.attention import ( + AttentionInputs, + _redistribute_from_attn_tp_shards, + _redistribute_to_attn_tp_shards, + get_attn_tp_context, +) +from sglang.srt.layers.communicator.adapters.context_parallel import ( + dsa_cp_gather_hidden_states, +) +from sglang.srt.layers.communicator.layout import ( + SumGroup, + _cp_shard_token_rows, + moe_cp_gathered_rows, +) +from sglang.srt.layers.communicator.output import ( + HandoffOutput, + UnreducedOutput, + reduce_output, +) +from sglang.srt.layers.communicator.residual import StageRead, StageUpdate +from sglang.srt.layers.communicator.residual.add_norm import ADD, NORM_READ +from sglang.srt.layers.communicator.residual.stream import DeclaredSum +from sglang.srt.layers.dp_attention import ( + attn_tp_all_gather_into_tensor, + dp_scatter, + get_moe_cp_size, + is_allocation_symmetric, +) +from sglang.srt.model_executor.forward_batch_info import ForwardBatch +from sglang.srt.runtime_context import get_parallel +from sglang.srt.utils import ( + is_npu, +) + +_is_npu = is_npu() + + +if _is_npu: + from sglang.srt.hardware_backend.npu.cmo import prepare_weight_cache + +from sglang.srt.layers.communicator.ops import ( + _mlp_input_reduce_output, + _redistribute_input_to_dp, + _redistribute_input_to_moe_cp, + _reduce_and_redistribute_output_to_attn_tp_shards, + _reduce_and_redistribute_output_to_dp, + _tp_all_gather_scattered_rows, +) + + +def _mlp_input_without_dp( + hidden_states: torch.Tensor, + residual: torch.Tensor, + forward_batch: ForwardBatch, + layernorm: torch.nn.Module, + *, + cache=None, + gathers_residual: bool, + fusions: Tuple[Callable, ...], + group: SumGroup = SumGroup.ATTN_TP, + read: StageRead = NORM_READ, + update: StageUpdate = ADD, +): + """Complete the sum the input owes over ``group`` on the rows it is on, + unless one of ``fusions`` does it with the residual add and the norm, then + write it into the residual and read the input.""" + if gathers_residual: + residual = update.residual_from_attn_tp_shards(residual) + for fused in fusions: + result = fused(hidden_states, residual, forward_batch) + if result is not None: + return result + if group is SumGroup.ATTN_TP: + # MHC sums its streams in full precision. + hidden_states = _mlp_input_reduce_output( + hidden_states, forward_batch, may_quantize=update.adds_plainly + ) + elif group is SumGroup.TP: + hidden_states = tensor_model_parallel_all_reduce(hidden_states) + if _is_npu and cache is not None: + _ = prepare_weight_cache(hidden_states, cache) + return read.update_and_read(update, hidden_states, residual, layernorm) + + +def _mlp_input_dp_replicate( + hidden_states: torch.Tensor, + residual: torch.Tensor, + forward_batch: ForwardBatch, + layernorm: torch.nn.Module, + *, + cache=None, + gathers_residual: bool, + reduces_attention_tp: bool, + places_cp_shards: bool = False, + read: StageRead = NORM_READ, + update: StageUpdate = ADD, +): + """Attention DP: complete the attention-TP sum if it is owed, write the + output into the residual and read the FFN input locally, then gather. With + ``places_cp_shards`` each CP rank puts its shard of the DP group's tokens + beside the others' in the group's slot.""" + if gathers_residual: + residual = update.residual_from_attn_tp_shards(residual) + if hidden_states.shape[0] != 0: + if reduces_attention_tp: + hidden_states = attention_tensor_model_parallel_all_reduce(hidden_states) + with use_symmetric_memory( + get_parallel().tp_group, + disabled=not is_allocation_symmetric(), + ): + hidden_states, residual = read.update_and_read( + update, hidden_states, residual, layernorm + ) + else: + hidden_states, residual = read.update_and_read( + update, hidden_states, residual, layernorm + ) + cp_shard_counts = _cp_shard_token_rows(forward_batch) if places_cp_shards else None + return ( + _redistribute_input_to_dp(hidden_states, forward_batch, cp_shard_counts), + residual, + ) + + +def _mlp_input_dp_partial( + hidden_states: torch.Tensor, + residual: torch.Tensor, + forward_batch: ForwardBatch, + layernorm: torch.nn.Module, + *, + cache=None, + gathers_residual: bool, + places_cp_shards: bool = False, + read: StageRead = NORM_READ, + update: StageUpdate = ADD, +): + """Attention DP: one rank adds the residual, the gather sums it, then read + the input from the sum. With ``places_cp_shards`` each CP rank puts its + shard of the DP group's tokens beside the others' in the group's slot.""" + if gathers_residual: + residual = _redistribute_from_attn_tp_shards(residual) + if get_parallel().attn_tp_rank == 0: + hidden_states += residual + cp_shard_counts = _cp_shard_token_rows(forward_batch) if places_cp_shards else None + hidden_states = _reduce_and_redistribute_output_to_dp( + hidden_states, forward_batch, cp_shard_counts + ) + dp_scatter(residual, hidden_states, forward_batch, cp_shard_counts) + hidden_states, _ = read.read(hidden_states, layernorm) + return hidden_states, residual + + +def _hand_qkv_hook_its_input( + hidden_states: torch.Tensor, + forward_batch: ForwardBatch, + qkv_latent_func: Optional[Callable], +) -> torch.Tensor: + """Give the attention's QKV hook the attention input.""" + if qkv_latent_func is not None: + get_attn_tp_context().set_attn_inputs( + AttentionInputs(hidden_states, forward_batch, qkv_latent_func) + ) + return hidden_states + + +def _hand_scattered_input_to_attention( + hidden_states: torch.Tensor, + forward_batch: ForwardBatch, + qkv_latent_func: Optional[Callable], +) -> torch.Tensor: + """Input-scattered attention takes each rank's slice, and its QKV hook + gathers the rows after the projection. DSA and attention without a hook + consume full hidden states, so those are gathered here, and the hook is + told they are.""" + ctx = get_attn_tp_context() + if ctx.is_dsa or qkv_latent_func is None: + hidden_states = _tp_all_gather_scattered_rows(hidden_states, forward_batch) + if qkv_latent_func is not None: + ctx.set_attn_inputs( + AttentionInputs( + hidden_states, + forward_batch, + qkv_latent_func, + is_pre_gathered=ctx.is_dsa, + ) + ) + return hidden_states + + +def _dispatch_consumer( + hidden_states, residual, forward_batch, norm, *, paths, update=ADD, **call +): + if residual is None: + # A missing residual cannot join a partial sum. The non-plain path + # performs enter/read without that optimization. + prepare = paths[False] + else: + try: + prepare = paths[update.adds_plainly] + except KeyError: + raise RuntimeError("producer update has no bound input path") from None + return prepare(hidden_states, residual, forward_batch, norm, update=update, **call) + + +def _consumer_step( + hidden_states: Union[torch.Tensor, UnreducedOutput, HandoffOutput], + residual: Optional[torch.Tensor], + forward_batch: ForwardBatch, + norm: torch.nn.Module, + *, + step: Callable, + adds_plainly: bool, + carried_fusions: Tuple[Callable, ...], + expected_sum: Optional[SumGroup] = None, + completed_step: Optional[Callable] = None, + pending=None, + written_step: Optional[Callable] = None, + written: bool = False, + update: StageUpdate = ADD, + **call, +): + """A boundary's half into a stage: complete what the value carries, a sum + or a handoff its producer left for this batch, then run ``step`` on the + complete value. Under attention DP the reduction back to this rank's tokens + comes first; otherwise one of ``carried_fusions`` may complete it with the + residual add and the read, and an empty batch has nothing to sum. An input + whose declared sum is already completed uses ``completed_step`` instead; + a pending declared sum must match ``expected_sum``. ``call`` is what the stage's read takes (the attention's + ``quant_format`` and ``post_residual_addition``).""" + if residual is not None and update.adds_plainly != adds_plainly: + raise RuntimeError("producer update does not match the boundary's capability") + if pending is not None: + if isinstance(pending.owed, DeclaredSum): + if pending.owed.group is not expected_sum: + raise RuntimeError("producer sum does not match the input declaration") + elif expected_sum is not None: + if pending.owed is not None: + raise RuntimeError("a declared input sum carried another completion") + step = completed_step + if written and written_step is not None: + step = written_step + if not isinstance(hidden_states, torch.Tensor): + owed = hidden_states + if residual is None: + raise RuntimeError(f"{type(owed).__name__} requires residual input") + if expected_sum is not None: + raise RuntimeError( + f"an input that owes its sum by construction arrived as " + f"{type(owed).__name__}" + ) + if ( + isinstance(owed, UnreducedOutput) + and owed.reduce_and_redistribute is not None + ): + hidden_states = reduce_output(owed) + else: + for fused in carried_fusions: + result = fused( + owed, residual, forward_batch, call.get("post_residual_addition") + ) + if result is not None: + return result + if isinstance(owed, HandoffOutput) or owed.partial.shape[0] != 0: + hidden_states = reduce_output(owed) + else: + hidden_states = owed.partial + return step(hidden_states, residual, forward_batch, norm, update=update, **call) + + +def _read_input( + hidden_states: torch.Tensor, + residual: Optional[torch.Tensor], + forward_batch: ForwardBatch, + norm: torch.nn.Module, + *, + cache=None, + layer_input: Optional[Callable], + enters_stack: bool, + read: StageRead, + update: StageUpdate = ADD, + quant_format: str = "", + post_residual_addition: Optional[torch.Tensor] = None, +): + """Complete what the input owes by construction (``layer_input``), then + write the previous stage's output into the residual and read the stage's + input with ``norm``. The layer stack's first stage (``enters_stack``) + starts its residual from its input.""" + enters = residual is None and enters_stack + if layer_input is not None: + hidden_states, residual = layer_input(hidden_states, residual) + if enters: + hidden_states, residual = read.enter(hidden_states), None + if residual is None: + # The previous layer already wrote its output into the residual. + return read.read(hidden_states, norm, quant_format) + return read.update_and_read( + update, + hidden_states, + residual, + norm, + quant_format=quant_format, + post_residual_addition=post_residual_addition, + ) + + +def _mlp_input_scatter( + hidden_states: torch.Tensor, + residual: torch.Tensor, + forward_batch: ForwardBatch, + layernorm: torch.nn.Module, + *, + cache=None, + scatters_residual: bool, + read: StageRead = NORM_READ, + update: StageUpdate = ADD, +): + hidden_states = _reduce_and_redistribute_output_to_attn_tp_shards(hidden_states) + if scatters_residual: + residual = update.residual_to_attn_tp_shard(residual) + return read.update_and_read(update, hidden_states, residual, layernorm) + + +def _mlp_input_slice( + hidden_states: torch.Tensor, + residual: Optional[torch.Tensor], + forward_batch: ForwardBatch, + layernorm: torch.nn.Module, + *, + cache=None, + scatters_residual: bool, + read: StageRead = NORM_READ, + update: StageUpdate = ADD, +): + hidden_states = _redistribute_to_attn_tp_shards(hidden_states).clone() + if scatters_residual and residual is not None: + residual = update.residual_to_attn_tp_shard(residual) + return read.update_and_read(update, hidden_states, residual, layernorm) + + +def _mlp_input_on_residual_shard( + hidden_states: torch.Tensor, + residual: torch.Tensor, + forward_batch: ForwardBatch, + layernorm: torch.nn.Module, + *, + cache=None, + reduces: bool = True, + read: StageRead, + update: StageUpdate, +): + """The residual stays on each rank's slice while the FFN takes the full + rows: reduce-scatter the attention output onto the slice, which completes + its sum, write it into the residual and read the FFN input there, then + gather the input back into the attention output's rows.""" + parallel = get_parallel() + if hidden_states.shape[0] == 0: + return hidden_states, hidden_states + shard = hidden_states.tensor_split(parallel.tp_size)[parallel.tp_rank] + if reduces: + parallel.tp_group.reduce_scatter_tensor(shard, hidden_states) + else: + shard = shard.clone() + shard, residual = read.update_and_read(update, shard, residual, layernorm) + attn_tp_all_gather_into_tensor(hidden_states, shard) + return hidden_states, residual + + +def _mlp_input_residual_into_sum( + hidden_states: torch.Tensor, + residual: torch.Tensor, + forward_batch: ForwardBatch, + layernorm: torch.nn.Module, + *, + cache=None, + read: StageRead = NORM_READ, + update: StageUpdate = ADD, +): + return _tp_all_reduce_with_scattered_residual( + hidden_states, residual, layernorm, read + ) + + +def _mlp_input_gather_attention_cp( + hidden_states: torch.Tensor, + residual: torch.Tensor, + forward_batch: ForwardBatch, + layernorm: torch.nn.Module, + *, + cache=None, + gather: Callable, + update: StageUpdate = ADD, +): + """DSA and MLA CP: complete this rank's shard, then gather the shards, of + equal length, over the attention-CP group. The residual stays on the + shard.""" + hidden_states, residual = gather( + hidden_states, residual, forward_batch, layernorm, update=update, cache=cache + ) + return dsa_cp_gather_hidden_states(hidden_states), residual + + +def _mlp_input_gather_moe_cp( + hidden_states: torch.Tensor, + residual: torch.Tensor, + forward_batch: ForwardBatch, + layernorm: torch.nn.Module, + *, + cache=None, + gather: Callable, + update: StageUpdate = ADD, +): + """Gather for the FFN, then over the MoE-CP group so each rank holds all + tokens of its MoE group (moe_dp_size < attn_cp_size). The residual stays at + TP_ATTN_FULL.""" + # Early return on empty tensor is safe for MOE_CP because: + # - During CP extend: zigzag split guarantees all CP ranks have non-zero tokens, + # so no rank hits this path while others proceed to the allgather. + # - During decode: moe_cp allgather is skipped (guarded by is_context_parallel_extend). + # - CUDA graph warmup: not applicable when --cuda-graph-backend-prefill=disabled is used. + if hidden_states.shape[0] == 0: + return hidden_states, residual + + hidden_states, residual = gather( + hidden_states, residual, forward_batch, layernorm, update=update, cache=cache + ) + + rows = moe_cp_gathered_rows(forward_batch) + if rows is not None and hidden_states.shape[0] > 0: + hidden_states = _redistribute_input_to_moe_cp( + hidden_states, rows, get_moe_cp_size() + ) + + return hidden_states, residual + + +def _tp_all_reduce_with_scattered_residual( + hidden_states: torch.Tensor, + residual: torch.Tensor, + layernorm: torch.nn.Module, + read: StageRead = NORM_READ, +): + parallel = get_parallel() + if hidden_states.shape[0] == 0: + return hidden_states, hidden_states + + scattered_states = hidden_states.tensor_split(parallel.tp_size)[parallel.tp_rank] + scattered_states += residual + residual = tensor_model_parallel_all_reduce(hidden_states) + return read.read(residual, layernorm) diff --git a/python/sglang/srt/layers/communicator/residual/__init__.py b/python/sglang/srt/layers/communicator/residual/__init__.py index 1a3965e820..3629c237f3 100644 --- a/python/sglang/srt/layers/communicator/residual/__init__.py +++ b/python/sglang/srt/layers/communicator/residual/__init__.py @@ -53,6 +53,8 @@ class StageRead(Protocol): # and the residual is left as it is: what a fused add + norm kernel computes, # and what may run on the rows a sum completes onto. norms_plainly: bool + # Preserve the read on the source rows before a DP gather. + before_gather: bool def enter(self, hidden_states) -> torch.Tensor: """The residual the layer stack starts from, given its input.""" diff --git a/python/sglang/srt/layers/communicator/residual/access.py b/python/sglang/srt/layers/communicator/residual/access.py index 9a35529c1a..1b00982143 100644 --- a/python/sglang/srt/layers/communicator/residual/access.py +++ b/python/sglang/srt/layers/communicator/residual/access.py @@ -59,6 +59,8 @@ def fold(hidden_states, residual): The following stage receives it with no outstanding residual addition.""" stream = residual if isinstance(residual, ResidualStream) else None if stream is not None: + if stream.pending is not None and not stream.pending.update.adds_plainly: + raise NotImplementedError("fold requires a plain residual update") hidden_states, residual = stream.finish(hidden_states) hidden_states = reduce_output(hidden_states) if residual is not None: @@ -70,10 +72,9 @@ def fold(hidden_states, residual): ) -def written(hidden_states, forward_batch): +def written(hidden_states): """Re-enter after a computation that already updated the full residual.""" - forward_batch.residual_stream = ResidualStream(hidden_states) - return hidden_states, forward_batch.residual_stream + return hidden_states, ResidualStream(hidden_states) def finish_layer_stack( @@ -90,16 +91,11 @@ def finish_layer_stack( that does a producer's handoff together with its own work (``final_norm_takes_handoff``) receives it as it is.""" if isinstance(residual, ResidualStream): - stream = forward_batch.residual_stream - if residual is not stream: - raise RuntimeError("residual alias belongs to a different invocation") - output = stream.finish( + return residual.finish( hidden_states, takes_handoff=final_norm_takes_handoff, preserve_declared=preserve_declared, ) - forward_batch.residual_stream = None - return output if final_norm_takes_handoff and isinstance(hidden_states, HandoffOutput): return hidden_states, residual return reduce_output(hidden_states), residual @@ -149,7 +145,9 @@ def snapshot( """Copy a complete output with its plain residual. A statically declared sum is reduced on a copy; the main output and residual remain unchanged.""" if isinstance(residual, ResidualStream): - return residual.snapshot(hidden_states, group=group) + if group is not None: + raise ValueError("a residual stream carries its own reduction") + return residual.snapshot(hidden_states) if group is not None: hidden_states = group.all_reduce(hidden_states.clone()) return hidden_states.clone() if residual is None else hidden_states + residual diff --git a/python/sglang/srt/layers/communicator/residual/add_norm.py b/python/sglang/srt/layers/communicator/residual/add_norm.py index 5cad111707..f3ae345066 100644 --- a/python/sglang/srt/layers/communicator/residual/add_norm.py +++ b/python/sglang/srt/layers/communicator/residual/add_norm.py @@ -14,6 +14,8 @@ """The plain residual: add and RMSNorm, and the fused all-reduce + add + norm kernels.""" +from dataclasses import dataclass +from enum import Enum, auto from typing import Optional import torch @@ -280,6 +282,14 @@ class Add: return _redistribute_from_attn_tp_shards(residual) +class Fp8Input(Enum): + """Additional input forms a projection can consume after a fused norm.""" + + TUPLE = auto() + TUPLE_AND_BF16 = auto() + + +@dataclass(frozen=True) class NormQuantRead: """The input is the residual's norm, fused with the quantization the consumer asks for (``quant_format``) and a post-residual addition; a plain @@ -287,6 +297,8 @@ class NormQuantRead: the norm.""" norms_plainly = True + before_gather: bool = False + fp8_input: Optional[Fp8Input] = None def enter(self, hidden_states): return hidden_states @@ -316,11 +328,13 @@ class NormQuantRead: ) +@dataclass(frozen=True) class NormRead: """The input is the residual's norm; a plain add of the previous output runs in the same kernel. An empty batch skips the norm.""" norms_plainly = True + before_gather: bool = False def enter(self, hidden_states): return hidden_states diff --git a/python/sglang/srt/layers/communicator/residual/batch.py b/python/sglang/srt/layers/communicator/residual/batch.py index 70cdb7dd90..9e93669619 100644 --- a/python/sglang/srt/layers/communicator/residual/batch.py +++ b/python/sglang/srt/layers/communicator/residual/batch.py @@ -19,9 +19,7 @@ from sglang.srt.model_executor.forward_batch_info import PPProxyTensors def start(forward_batch): - stream = ResidualStream() - forward_batch.residual_stream = stream - return stream + forward_batch.residual_stream = ResidualStream() def current(forward_batch): @@ -31,7 +29,8 @@ def current(forward_batch): return stream -def finish(hidden_states, forward_batch): +def complete_output(hidden_states, forward_batch): + """Complete the contribution while retaining its pending residual update.""" return current(forward_batch).complete(hidden_states) diff --git a/python/sglang/srt/layers/communicator/residual/mhc.py b/python/sglang/srt/layers/communicator/residual/mhc.py index 9ce90fbfc8..06f875fe09 100644 --- a/python/sglang/srt/layers/communicator/residual/mhc.py +++ b/python/sglang/srt/layers/communicator/residual/mhc.py @@ -123,6 +123,7 @@ class _AttentionRead: layer's output: an MHC layer takes its input written back.""" norms_plainly = False + before_gather = False def __init__(self, state: MHCState): self.state = state @@ -165,6 +166,7 @@ class _FfnRead: in hc_ffn_post_pre when it takes the batch.""" norms_plainly = False + before_gather = False def __init__(self, state: MHCState): self.state = state diff --git a/python/sglang/srt/layers/communicator/residual/stream.py b/python/sglang/srt/layers/communicator/residual/stream.py index a95da546c5..68eb449aba 100644 --- a/python/sglang/srt/layers/communicator/residual/stream.py +++ b/python/sglang/srt/layers/communicator/residual/stream.py @@ -158,22 +158,23 @@ class ResidualStream: self.check(hidden) return self.pending.complete() if self.pending is not None else hidden - def snapshot(self, hidden, *, group=None): + def snapshot(self, hidden): self.check(hidden) pending = self.pending if pending is None: return hidden.clone() + if not pending.update.adds_plainly: + raise NotImplementedError("snapshot requires a plain residual update") if pending.owed is None: value = pending.value elif isinstance(pending.owed, (CarriedSum, DeclaredSum)): value = pending.owed.complete(pending.value.clone()) else: raise NotImplementedError("a finalize handoff requires main-output capture") - if group is not None: - value = group.all_reduce(value.clone()) return value.clone() if self.residual is None else value + self.residual def finish(self, hidden, *, takes_handoff=False, preserve_declared=False): + """Export the contribution/residual pair; keep the stream available to readers.""" self.check(hidden) if self.pending is None: return hidden, None diff --git a/python/sglang/srt/layers/communicator/stage.py b/python/sglang/srt/layers/communicator/stage.py index 6e181cc1ee..b694fcad41 100644 --- a/python/sglang/srt/layers/communicator/stage.py +++ b/python/sglang/srt/layers/communicator/stage.py @@ -24,12 +24,13 @@ from sglang.srt.layers.communicator.adapters.lora import ( publish_attention, publish_ffn, ) -from sglang.srt.layers.communicator.boundary import ( - BoundarySteps, +from sglang.srt.layers.communicator.contracts import ( + BatchVariant, StageEntry, StageKind, + StageSteps, ) -from sglang.srt.layers.communicator.ops import CommunicateSimpleFn +from sglang.srt.layers.communicator.ops import gather_attention_tp from sglang.srt.layers.communicator.residual.access import buffer, from_pp from sglang.srt.layers.communicator.residual.add_norm import ( ADD, @@ -51,58 +52,40 @@ class StageCommunicator: def __init__( self, plan: "StagePlan", - kind: StageKind, - norm, *, - declaration: Optional["StageDeclaration"] = None, + declaration: "StageDeclaration", ): self.plan = plan - self.kind = kind - self.norm = norm - self._declaration = declaration + self.declaration = declaration @property - def declaration(self) -> "StageDeclaration": - """Immutable source contract for constructing another local sequence.""" - if self._declaration is None: - raise RuntimeError("this boundary was constructed from raw layout edges") - return self._declaration + def kind(self): + return self.declaration.kind + + @property + def norm(self): + return self.plan.norm def entry( - self, forward_batch: ForwardBatch, steps: Optional[BoundarySteps] = None + self, forward_batch: ForwardBatch, steps: Optional[StageSteps] = None ) -> StageEntry: if steps is None: steps = self.plan._batch_steps(forward_batch) - entry = steps.attention if self.kind is StageKind.ATTENTION else steps.ffn - if entry is None: - raise NotImplementedError( - f"a layer that is one stage has no {self.kind.name.lower()} stage" - ) - return entry + return steps.entry def _prepare( self, hidden_states, - residual, + stream: ResidualStream, forward_batch: ForwardBatch, - steps: Optional[BoundarySteps] = None, + steps: Optional[StageSteps] = None, **call, ): """The stage's input and the residual, from the previous stage's output (``call``: what the stage's read takes, e.g. the attention's ``quant_format``).""" entry = self.entry(forward_batch, steps) - stream = ( - residual - if isinstance(residual, ResidualStream) - else ResidualStream(residual) - ) - if not isinstance(residual, ResidualStream): - if residual is not None or entry.input_sum is not None: - hidden_states = stream.leave( - hidden_states, call.get("update", ADD), declared_sum=entry.input_sum - ) - elif ( + if ( stream.pending is None and stream.residual is None and entry.input_sum is not None @@ -136,8 +119,8 @@ class StageCommunicator: return self.plan.fusions @property - def input_rows(self): - return self.plan.input_rows + def incoming_residual_rows(self): + return self.plan.incoming_residual_rows @property def input_on_attention_tp_slices(self): @@ -158,11 +141,11 @@ class StageCommunicator: def finish(self, hidden_states, forward_batch): """Hand this attention's actual contribution to the following stage.""" - if self.kind is not StageKind.ATTENTION or not self.plan.fixed_output: + if self.kind is not StageKind.ATTENTION or not self.plan.direct_handoff: raise RuntimeError( "use the stage exit scope for an FFN or single-stage mixer" ) - produced = self.plan.produced(self.kind, forward_batch) + produced = self.plan.produced(forward_batch) return current(forward_batch).leave( hidden_states, produced.update, @@ -171,24 +154,24 @@ class StageCommunicator: def exit(self, forward_batch): result = ( - self.plan.output.ffn_exit(forward_batch) + self.plan.output.ffn_exit(forward_batch, stream=current(forward_batch)) if self.kind is StageKind.FFN - else self.plan.output.mixer_exit(forward_batch) + else self.plan.output.mixer_exit( + forward_batch, stream=current(forward_batch) + ) ) - result._stream = current(forward_batch) return result def from_pp(self, tensors, forward_batch, *, allow_missing_residual: bool = False): hidden_states, residual = from_pp( tensors, - residual_in_hidden=self.plan.residual_in_hidden, + residual_in_hidden=( + self.declaration.previous is not None + and self.declaration.previous.update.at_producer + ), allow_missing_residual=allow_missing_residual, ) - declared_sum = None - if forward_batch is not None: - steps = self.plan._batch_steps(forward_batch) - entry = steps.attention or steps.ffn - declared_sum = entry.input_sum + declared_sum = self.entry(forward_batch).input_sum hidden_states, forward_batch.residual_stream = ResidualStream.arrive( hidden_states, residual, ADD, declared_sum=declared_sum ) @@ -208,11 +191,9 @@ class StageCommunicator: return hidden_states, stream.snapshot(hidden_states) def postprocess(self, hidden_states, forward_batch): - hidden_states, stream = self.plan.output.postprocess_layer( + return self.plan.output.postprocess_layer( hidden_states, current(forward_batch), forward_batch ) - forward_batch.residual_stream = stream - return hidden_states def branch_input(self, source, hidden_states, forward_batch): hidden_states, stream = branch.branch_input( @@ -240,11 +221,14 @@ class StageCommunicator: forward_batch.residual_stream = stream return hidden_states - def _prepare_input(self, hidden_states, residual, forward_batch, **call): + def _prepare_input(self, hidden_states, stream, forward_batch, **call): plan = self.plan if self.kind is StageKind.ATTENTION: publish_attention(plan._publish_lora_layout) - if plan._sp_steps is not None and plan.is_first_layer: + if ( + plan._paths.get(BatchVariant.SEQUENCE_PARALLEL) is not None + and plan.enters_stack + ): get_forward().set( "sp_active", layernorm_sp.runs_sp(forward_batch.forward_mode) ) @@ -252,7 +236,7 @@ class StageCommunicator: hidden_states = layernorm_sp.sp_entry_scatter(hidden_states) else: publish_ffn(plan._publish_lora_layout, self.entry(forward_batch).input_rows) - return self._prepare(hidden_states, residual, forward_batch, **call) + return self._prepare(hidden_states, stream, forward_batch, **call) def _prepare_attention( self, @@ -315,11 +299,11 @@ class StageCommunicator: forward_batch=forward_batch, ) ) - # Input gathers allocate fresh DP buffers. + # Input gathers allocate fresh DP buffers (see gather_attention_tp). # Without a gather this is the mutable residual, so retention copies. captured_last_layer_outputs.capture( gathered_last_layer_output, - owned=move is CommunicateSimpleFn._scattered_to_tp_attn_full + owned=move is gather_attention_tp or ( move is None and entry.capture_preserves_residual is not None diff --git a/python/sglang/srt/models/bailing_moe.py b/python/sglang/srt/models/bailing_moe.py index a49ce6554c..14c007a446 100644 --- a/python/sglang/srt/models/bailing_moe.py +++ b/python/sglang/srt/models/bailing_moe.py @@ -844,7 +844,7 @@ class BailingMoEModel(nn.Module): if not self.pp_group.is_last_rank: return residual_batch.to_pp(hidden_states, forward_batch) else: - hidden_states = residual_batch.finish(hidden_states, forward_batch) + hidden_states = residual_batch.complete_output(hidden_states, forward_batch) if not forward_batch.forward_mode.is_idle(): hidden_states = residual_batch.norm( hidden_states, forward_batch, self.norm diff --git a/python/sglang/srt/models/bailing_moe_linear.py b/python/sglang/srt/models/bailing_moe_linear.py index 24d951c5a3..53504488a1 100644 --- a/python/sglang/srt/models/bailing_moe_linear.py +++ b/python/sglang/srt/models/bailing_moe_linear.py @@ -57,7 +57,7 @@ from sglang.srt.layers.vocab_parallel_embedding import ( from sglang.srt.model_executor.forward_batch_info import ForwardBatch, PPProxyTensors from sglang.srt.model_executor.runner import get_is_capture_mode from sglang.srt.model_loader.weight_utils import default_weight_loader -from sglang.srt.models.deepseek_v2 import DeepseekV2AttentionMLA, DeepseekV2MLP, _is_hip +from sglang.srt.models.deepseek_v2 import DeepseekV2AttentionMLA, DeepseekV2MLP from sglang.srt.models.utils import WeightsMapper from sglang.srt.runtime_context import ( get_device, @@ -1007,7 +1007,7 @@ class BailingMoELinearModel(nn.Module): if not self.pp_group.is_last_rank: return residual_batch.to_pp(hidden_states, forward_batch) else: - hidden_states = residual_batch.finish(hidden_states, forward_batch) + hidden_states = residual_batch.complete_output(hidden_states, forward_batch) if not forward_batch.forward_mode.is_idle(): hidden_states = residual_batch.norm( hidden_states, forward_batch, self.norm diff --git a/python/sglang/srt/models/bailing_moe_nextn.py b/python/sglang/srt/models/bailing_moe_nextn.py index 5d5963089f..2ea98da22c 100644 --- a/python/sglang/srt/models/bailing_moe_nextn.py +++ b/python/sglang/srt/models/bailing_moe_nextn.py @@ -181,7 +181,7 @@ class BailingMoEModelNextN(nn.Module): else: hidden_states = self.decoder(positions, hidden_states, forward_batch) - hidden_states = residual_batch.finish(hidden_states, forward_batch) + hidden_states = residual_batch.complete_output(hidden_states, forward_batch) if not forward_batch.forward_mode.is_idle(): hidden_states = residual_batch.norm( hidden_states, forward_batch, self.final_layernorm diff --git a/python/sglang/srt/models/bailing_moe_v3.py b/python/sglang/srt/models/bailing_moe_v3.py index b38a6202f1..10cd33a650 100644 --- a/python/sglang/srt/models/bailing_moe_v3.py +++ b/python/sglang/srt/models/bailing_moe_v3.py @@ -1395,7 +1395,7 @@ class BailingMoELinearModel(nn.Module): if not self.pp_group.is_last_rank: return residual_batch.to_pp(hidden_states, forward_batch) else: - hidden_states = residual_batch.finish(hidden_states, forward_batch) + hidden_states = residual_batch.complete_output(hidden_states, forward_batch) if not forward_batch.forward_mode.is_idle(): hidden_states = residual_batch.norm( hidden_states, diff --git a/python/sglang/srt/models/deepseek_nextn.py b/python/sglang/srt/models/deepseek_nextn.py index 833fffdd95..39d66bca73 100644 --- a/python/sglang/srt/models/deepseek_nextn.py +++ b/python/sglang/srt/models/deepseek_nextn.py @@ -226,7 +226,7 @@ class DeepseekModelNextN(nn.Module): zero_allocator, prev_topk_indices=index_topk_share.topk_indices, ) - hidden_states = residual_batch.finish(hidden_states, forward_batch) + hidden_states = residual_batch.complete_output(hidden_states, forward_batch) if not forward_batch.forward_mode.is_idle(): hidden_states = residual_batch.norm( hidden_states, forward_batch, self.shared_head.norm diff --git a/python/sglang/srt/models/deepseek_v2.py b/python/sglang/srt/models/deepseek_v2.py index 0b40c72ec9..963caa6329 100644 --- a/python/sglang/srt/models/deepseek_v2.py +++ b/python/sglang/srt/models/deepseek_v2.py @@ -2655,7 +2655,7 @@ class DeepseekV2DecoderLayer(nn.Module): declare_ffn( sparse=self.is_layer_sparse, next_sparse=self._stage_next_sparse, - output=output, + output_transform=output, ), post_attention_layernorm, {"fusions": fusions}, @@ -2998,12 +2998,6 @@ class DeepseekV2Model(nn.Module): and layer.mlp.num_fused_shared_experts == 0 and not layer.mlp._shared_expert_tp1 ), - # A TP1-replicated output must not ride into the next layer's reduction. - requires_local_reduction=lambda layer: ( - layer.mlp._shared_expert_tp1 - if isinstance(layer.mlp, DeepseekV2MoE) - else layer.mlp.tp_size == 1 - ), label="DeepSeek-V3/GLM", ) @@ -3160,7 +3154,7 @@ class DeepseekV2Model(nn.Module): proxy_tensors.tensors["topk_indices"] = topk_indices return proxy_tensors else: - hidden_states = residual_batch.finish(hidden_states, forward_batch) + hidden_states = residual_batch.complete_output(hidden_states, forward_batch) if not forward_batch.forward_mode.is_idle(): hidden_states = residual_batch.norm( hidden_states, forward_batch, self.norm diff --git a/python/sglang/srt/models/dots3_common/modeling.py b/python/sglang/srt/models/dots3_common/modeling.py index 4208e5de02..2030e3cff8 100644 --- a/python/sglang/srt/models/dots3_common/modeling.py +++ b/python/sglang/srt/models/dots3_common/modeling.py @@ -1795,7 +1795,7 @@ class Dots3Model(nn.Module): zero_allocator=zero_allocator, ) - hidden_states = residual_batch.finish(hidden_states, forward_batch) + hidden_states = residual_batch.complete_output(hidden_states, forward_batch) if not self.pp_group.is_last_rank: return residual_batch.to_pp(hidden_states, forward_batch) else: diff --git a/python/sglang/srt/models/dots3_common/nextn.py b/python/sglang/srt/models/dots3_common/nextn.py index 79da200858..a862b58c8f 100644 --- a/python/sglang/srt/models/dots3_common/nextn.py +++ b/python/sglang/srt/models/dots3_common/nextn.py @@ -119,7 +119,7 @@ class Dot3NoteModelNextN(nn.Module): hidden_states = head.decoder( positions, hidden_states, forward_batch, zero_allocator ) - hidden_states = residual_batch.finish(hidden_states, forward_batch) + hidden_states = residual_batch.complete_output(hidden_states, forward_batch) if not forward_batch.forward_mode.is_idle(): hidden_states = residual_batch.norm( diff --git a/python/sglang/srt/models/falcon_h1.py b/python/sglang/srt/models/falcon_h1.py index a79cb72d5f..500285bedc 100644 --- a/python/sglang/srt/models/falcon_h1.py +++ b/python/sglang/srt/models/falcon_h1.py @@ -226,7 +226,7 @@ class FalconH1HybridAttentionDecoderLayer(nn.Module): (declare_attn(), self.input_layernorm), ( declare_ffn( - output=OutputTransform( + output_transform=OutputTransform( self.feed_forward.scale_output, before_reduce_scatter=True ), sparse=self.is_layer_sparse, @@ -439,7 +439,7 @@ class FalconH1Model(nn.Module): forward_batch=forward_batch, ) - hidden_states = residual_batch.finish(hidden_states, forward_batch) + hidden_states = residual_batch.complete_output(hidden_states, forward_batch) if not forward_batch.forward_mode.is_idle(): hidden_states = residual_batch.norm( diff --git a/python/sglang/srt/models/gigachat35.py b/python/sglang/srt/models/gigachat35.py index d1fe5398b9..db74181dad 100644 --- a/python/sglang/srt/models/gigachat35.py +++ b/python/sglang/srt/models/gigachat35.py @@ -583,7 +583,7 @@ class GigaChat35Model(nn.Module): if not self.pp_group.is_last_rank: return residual_batch.to_pp(hidden_states, forward_batch) - hidden_states = residual_batch.finish(hidden_states, forward_batch) + hidden_states = residual_batch.complete_output(hidden_states, forward_batch) if hidden_states.shape[0] != 0: hidden_states = residual_batch.norm(hidden_states, forward_batch, self.norm) diff --git a/python/sglang/srt/models/gigachat35_mtp.py b/python/sglang/srt/models/gigachat35_mtp.py index 31dc0b037f..a78a804f9d 100644 --- a/python/sglang/srt/models/gigachat35_mtp.py +++ b/python/sglang/srt/models/gigachat35_mtp.py @@ -115,7 +115,7 @@ class GigaChat35ModelNextN(nn.Module): zero_allocator=zero_allocator, ) - hidden_states = residual_batch.finish(hidden_states, forward_batch) + hidden_states = residual_batch.complete_output(hidden_states, forward_batch) hidden_states_before_norm = None if not forward_batch.forward_mode.is_idle(): diff --git a/python/sglang/srt/models/glm4_moe.py b/python/sglang/srt/models/glm4_moe.py index 07d83d1dd9..473cc586a5 100644 --- a/python/sglang/srt/models/glm4_moe.py +++ b/python/sglang/srt/models/glm4_moe.py @@ -1121,7 +1121,7 @@ class Glm4MoeModel(nn.Module): if not self.pp_group.is_last_rank: return residual_batch.to_pp(hidden_states, forward_batch) else: - hidden_states = residual_batch.finish(hidden_states, forward_batch) + hidden_states = residual_batch.complete_output(hidden_states, forward_batch) if not forward_batch.forward_mode.is_idle(): hidden_states = residual_batch.norm( hidden_states, forward_batch, self.norm diff --git a/python/sglang/srt/models/glm4_moe_lite.py b/python/sglang/srt/models/glm4_moe_lite.py index bd9223ac67..5e46ec92b7 100644 --- a/python/sglang/srt/models/glm4_moe_lite.py +++ b/python/sglang/srt/models/glm4_moe_lite.py @@ -826,7 +826,7 @@ class Glm4MoeLiteModel(nn.Module): if not self.pp_group.is_last_rank: return residual_batch.to_pp(hidden_states, forward_batch) else: - hidden_states = residual_batch.finish(hidden_states, forward_batch) + hidden_states = residual_batch.complete_output(hidden_states, forward_batch) if not forward_batch.forward_mode.is_idle(): hidden_states = residual_batch.norm( hidden_states, forward_batch, self.norm diff --git a/python/sglang/srt/models/glm4_moe_lite_nextn.py b/python/sglang/srt/models/glm4_moe_lite_nextn.py index 867a0dcfea..6a65cefa8f 100644 --- a/python/sglang/srt/models/glm4_moe_lite_nextn.py +++ b/python/sglang/srt/models/glm4_moe_lite_nextn.py @@ -121,7 +121,7 @@ class Glm4MoeLiteModelNextN(nn.Module): positions, hidden_states, forward_batch, zero_allocator ) - hidden_states = residual_batch.finish(hidden_states, forward_batch) + hidden_states = residual_batch.complete_output(hidden_states, forward_batch) if not forward_batch.forward_mode.is_idle(): hidden_states = residual_batch.norm( hidden_states, forward_batch, self.shared_head.norm diff --git a/python/sglang/srt/models/glm4_moe_nextn.py b/python/sglang/srt/models/glm4_moe_nextn.py index a356943e9a..b806f86406 100644 --- a/python/sglang/srt/models/glm4_moe_nextn.py +++ b/python/sglang/srt/models/glm4_moe_nextn.py @@ -105,7 +105,7 @@ class Glm4MoeModelNextN(nn.Module): with get_global_expert_distribution_recorder().disable_this_region(): hidden_states = self.decoder(positions, hidden_states, forward_batch) - hidden_states = residual_batch.finish(hidden_states, forward_batch) + hidden_states = residual_batch.complete_output(hidden_states, forward_batch) if not forward_batch.forward_mode.is_idle(): hidden_states = residual_batch.norm( hidden_states, forward_batch, self.shared_head.norm diff --git a/python/sglang/srt/models/glm5_next.py b/python/sglang/srt/models/glm5_next.py index a5e4520e56..688f67a766 100644 --- a/python/sglang/srt/models/glm5_next.py +++ b/python/sglang/srt/models/glm5_next.py @@ -801,7 +801,6 @@ class Glm5NextDecoderLayer(nn.Module): ), self.input_layernorm, { - "residual_in_hidden": residual.ffn_update.at_producer, "qkv_latent_func": self.self_attn.prepare_qkv_latent if not self.is_linear_attn else None, @@ -1192,7 +1191,7 @@ class Glm5NextModel(nn.Module): return PPProxyTensors({"hidden_states": hidden_states}) return residual_batch.to_pp(hidden_states, forward_batch) else: - hidden_states = residual_batch.finish(hidden_states, forward_batch) + hidden_states = residual_batch.complete_output(hidden_states, forward_batch) if not forward_batch.forward_mode.is_idle(): hidden_states = residual_batch.norm( hidden_states, forward_batch, self.norm diff --git a/python/sglang/srt/models/gpt_oss.py b/python/sglang/srt/models/gpt_oss.py index 596cb5cf38..9f6f955fb1 100644 --- a/python/sglang/srt/models/gpt_oss.py +++ b/python/sglang/srt/models/gpt_oss.py @@ -728,7 +728,7 @@ class GptOssModel(nn.Module): if not self.pp_group.is_last_rank: return residual_batch.to_pp(hidden_states, forward_batch) else: - hidden_states = residual_batch.finish(hidden_states, forward_batch) + hidden_states = residual_batch.complete_output(hidden_states, forward_batch) if hidden_states.shape[0] != 0: hidden_states = residual_batch.norm( hidden_states, diff --git a/python/sglang/srt/models/interns2_mobius.py b/python/sglang/srt/models/interns2_mobius.py index 5f335d186f..a71f405065 100644 --- a/python/sglang/srt/models/interns2_mobius.py +++ b/python/sglang/srt/models/interns2_mobius.py @@ -20,6 +20,7 @@ from sglang.srt.layers.communicator import ( make_stages, ) from sglang.srt.layers.communicator.residual import batch as residual_batch +from sglang.srt.layers.communicator.residual.add_norm import Fp8Input, NormQuantRead from sglang.srt.layers.dp_attention import is_dp_attention_enabled from sglang.srt.layers.layernorm import GemmaRMSNorm from sglang.srt.layers.linear import ( @@ -49,7 +50,6 @@ from sglang.srt.models.qwen3_5 import ( Qwen3_5ForCausalLM, Qwen3_5ForConditionalGeneration, Qwen3_5GatedDeltaNet, - _enable_qwen35_fused_ar_quant, _linear_accepts_fp8_tuple, ) from sglang.srt.runtime_context import get_parallel, get_stream @@ -488,18 +488,15 @@ class InternS2MobiusLinearDecoderLayer(_InternS2MobiusDecoderMixin, nn.Module): self.post_attention_layernorm = GemmaRMSNorm( config.hidden_size, eps=config.rms_norm_eps ) - enable_fused_ar_quant = ( - _enable_qwen35_fused_ar_quant() - and _linear_accepts_fp8_tuple(self.linear_attn.in_proj_qkvz) - ) + accepts_fp8_input = _linear_accepts_fp8_tuple(self.linear_attn.in_proj_qkvz) self.attn_stage, self.ffn_stage = make_stages( ( - declare_attn(), + declare_attn( + read=NormQuantRead( + fp8_input=Fp8Input.TUPLE_AND_BF16 if accepts_fp8_input else None + ) + ), self.input_layernorm, - { - "enable_fused_ar_quant": enable_fused_ar_quant, - "fused_ar_quant_keep_bf16": enable_fused_ar_quant, - }, ), ( declare_ffn( @@ -623,17 +620,15 @@ class InternS2MobiusAttentionDecoderLayer( ) self.q_norm = GemmaRMSNorm(self.head_dim, eps=config.rms_norm_eps) self.k_norm = GemmaRMSNorm(self.head_dim, eps=config.rms_norm_eps) - enable_fused_ar_quant = ( - _enable_qwen35_fused_ar_quant() and _linear_accepts_fp8_tuple(self.qkv_proj) - ) + accepts_fp8_input = _linear_accepts_fp8_tuple(self.qkv_proj) self.attn_stage, self.ffn_stage = make_stages( ( - declare_attn(), + declare_attn( + read=NormQuantRead( + fp8_input=Fp8Input.TUPLE if accepts_fp8_input else None + ) + ), self.input_layernorm, - { - "enable_fused_ar_quant": enable_fused_ar_quant, - "fused_ar_quant_keep_bf16": False, - }, ), ( declare_ffn( @@ -793,7 +788,7 @@ class InternS2MobiusForCausalLM(Qwen3_5ForCausalLM): input_deepstack_embeds[:, start : start + self.hidden_size], ) - hidden_states = residual_batch.finish(hidden_states, forward_batch) + hidden_states = residual_batch.complete_output(hidden_states, forward_batch) if hidden_states.shape[0] != 0: hidden_states = residual_batch.norm(hidden_states, forward_batch, self.norm) diff --git a/python/sglang/srt/models/laguna.py b/python/sglang/srt/models/laguna.py index cd4c576fef..3689a55fc0 100644 --- a/python/sglang/srt/models/laguna.py +++ b/python/sglang/srt/models/laguna.py @@ -607,7 +607,7 @@ class LagunaModel(nn.Module): if not self.pp_group.is_last_rank: return residual_batch.to_pp(hidden_states, forward_batch) - hidden_states = residual_batch.finish(hidden_states, forward_batch) + hidden_states = residual_batch.complete_output(hidden_states, forward_batch) if hidden_states.shape[0] != 0: hidden_states = residual_batch.norm( diff --git a/python/sglang/srt/models/llada2.py b/python/sglang/srt/models/llada2.py index e69bd3efe8..a4526b0d7c 100644 --- a/python/sglang/srt/models/llada2.py +++ b/python/sglang/srt/models/llada2.py @@ -761,7 +761,7 @@ class LLaDA2MoeModel(nn.Module): if not self.pp_group.is_last_rank: return residual_batch.to_pp(hidden_states, forward_batch) else: - hidden_states = residual_batch.finish(hidden_states, forward_batch) + hidden_states = residual_batch.complete_output(hidden_states, forward_batch) if not forward_batch.forward_mode.is_idle(): hidden_states = residual_batch.norm( hidden_states, forward_batch, self.norm diff --git a/python/sglang/srt/models/llama4.py b/python/sglang/srt/models/llama4.py index dbfaf078cc..a4f22147e9 100644 --- a/python/sglang/srt/models/llama4.py +++ b/python/sglang/srt/models/llama4.py @@ -535,7 +535,7 @@ class Llama4Model(nn.Module): else None, ) - hidden_states = residual_batch.finish(hidden_states, forward_batch) + hidden_states = residual_batch.complete_output(hidden_states, forward_batch) if not forward_batch.forward_mode.is_idle(): hidden_states = residual_batch.norm(hidden_states, forward_batch, self.norm) diff --git a/python/sglang/srt/models/longcat_flash.py b/python/sglang/srt/models/longcat_flash.py index e68af8a056..a1bfc2b514 100644 --- a/python/sglang/srt/models/longcat_flash.py +++ b/python/sglang/srt/models/longcat_flash.py @@ -448,16 +448,16 @@ class LongcatFlashDecoderLayer(nn.Module): ) self.first_ffn, self.second_attn, self.second_ffn = make_stages( ( - declare_ffn(ordinary_only=True), + declare_ffn(), self.post_attention_layernorm[0], ), ( - declare_attn(ordinary_only=True), + declare_attn(), self.input_layernorm[1], {"qkv_latent_func": self.self_attn[1].prepare_qkv_latent}, ), ( - declare_ffn(ordinary_only=True), + declare_ffn(), self.post_attention_layernorm[1], ), prepared_from=self.moe_stage.declaration, diff --git a/python/sglang/srt/models/longcat_flash_nextn.py b/python/sglang/srt/models/longcat_flash_nextn.py index 3b2e642a76..9b8a6cc815 100644 --- a/python/sglang/srt/models/longcat_flash_nextn.py +++ b/python/sglang/srt/models/longcat_flash_nextn.py @@ -272,7 +272,7 @@ class LongcatFlashModelNextN(nn.Module): positions, hidden_states, forward_batch, zero_allocator ) - hidden_states = residual_batch.finish(hidden_states, forward_batch) + hidden_states = residual_batch.complete_output(hidden_states, forward_batch) if not forward_batch.forward_mode.is_idle(): hidden_states = residual_batch.norm( diff --git a/python/sglang/srt/models/mimo_v2.py b/python/sglang/srt/models/mimo_v2.py index dd706f142d..5cfac4be8c 100644 --- a/python/sglang/srt/models/mimo_v2.py +++ b/python/sglang/srt/models/mimo_v2.py @@ -1069,7 +1069,7 @@ class MiMoV2Model(nn.Module): else None, ) - hidden_states = residual_batch.finish(hidden_states, forward_batch) + hidden_states = residual_batch.complete_output(hidden_states, forward_batch) # A draft targeting the final layer ("after layer # num_hidden_layers-1") maps to capture index num_hidden_layers, diff --git a/python/sglang/srt/models/mimo_v2_nextn.py b/python/sglang/srt/models/mimo_v2_nextn.py index 02ee8535fb..d4d1e8a544 100644 --- a/python/sglang/srt/models/mimo_v2_nextn.py +++ b/python/sglang/srt/models/mimo_v2_nextn.py @@ -228,7 +228,7 @@ class MiMoV2ModelNextN(nn.Module): forward_batch=forward_batch, ) - hidden_states = residual_batch.finish(hidden_states, forward_batch) + hidden_states = residual_batch.complete_output(hidden_states, forward_batch) hidden_states_before_norm = None if not forward_batch.forward_mode.is_idle(): if forward_batch.return_hidden_states_before_norm: diff --git a/python/sglang/srt/models/minimax_m2.py b/python/sglang/srt/models/minimax_m2.py index 9414955ed4..db5b2078e8 100644 --- a/python/sglang/srt/models/minimax_m2.py +++ b/python/sglang/srt/models/minimax_m2.py @@ -1176,7 +1176,7 @@ class MiniMaxM2Model(nn.Module): if not self.pp_group.is_last_rank: return residual_batch.to_pp(hidden_states, forward_batch) - hidden_states = residual_batch.finish(hidden_states, forward_batch) + hidden_states = residual_batch.complete_output(hidden_states, forward_batch) if hidden_states.shape[0] != 0: hidden_states = residual_batch.norm(hidden_states, forward_batch, self.norm) diff --git a/python/sglang/srt/models/minimax_m3.py b/python/sglang/srt/models/minimax_m3.py index 7f541a56c3..7bded1282d 100644 --- a/python/sglang/srt/models/minimax_m3.py +++ b/python/sglang/srt/models/minimax_m3.py @@ -1517,7 +1517,7 @@ class MiniMaxM3Model(nn.Module): if not self.pp_group.is_last_rank: return residual_batch.to_pp(hidden_states, forward_batch) - hidden_states = residual_batch.finish(hidden_states, forward_batch) + hidden_states = residual_batch.complete_output(hidden_states, forward_batch) if hidden_states.shape[0] != 0: hidden_states = residual_batch.norm(hidden_states, forward_batch, self.norm) diff --git a/python/sglang/srt/models/nemotron_h.py b/python/sglang/srt/models/nemotron_h.py index 5adcd93367..3f6a7e39ce 100644 --- a/python/sglang/srt/models/nemotron_h.py +++ b/python/sglang/srt/models/nemotron_h.py @@ -753,7 +753,7 @@ class NemotronHModel(nn.Module): return residual_batch.to_pp( hidden_states, forward_batch, preserve_declared=True ) - hidden_states = residual_batch.finish(hidden_states, forward_batch) + hidden_states = residual_batch.complete_output(hidden_states, forward_batch) if self.end_layer in self.layers_to_capture: aux_hidden_states.append( residual_batch.snapshot(hidden_states, forward_batch) diff --git a/python/sglang/srt/models/nemotron_h_utils.py b/python/sglang/srt/models/nemotron_h_utils.py index 8516a4bc80..9bbcd5d37d 100644 --- a/python/sglang/srt/models/nemotron_h_utils.py +++ b/python/sglang/srt/models/nemotron_h_utils.py @@ -1,15 +1,16 @@ """Layer-communication helpers for the Nemotron-H model.""" -from typing import Optional - from sglang.srt.configs.nemotron_h import ATTENTION, MAMBA, MOE from sglang.srt.layers.communicator import ( - StageKind, + HandoffRows, + ProducerReduction, declare_attn, declare_ffn, make_stages, ) +from sglang.srt.layers.communicator.residual.add_norm import NormQuantRead, NormRead from sglang.srt.layers.layernorm import RMSNorm +from sglang.srt.runtime_context import get_parallel ATTN_LAYERS = (MAMBA, ATTENTION) @@ -18,33 +19,23 @@ def is_attn_layer(layer_type: str) -> bool: return layer_type in ATTN_LAYERS -def _stage_kind(pattern: str, layer_idx: int) -> Optional[StageKind]: - """Which stage of a decoder layer the stage at ``layer_idx`` stands for: a - Mamba or attention mixer the attention, an MLP or MoE the FFN; None past - either end.""" - if not 0 <= layer_idx < len(pattern): - return None - return StageKind.ATTENTION if is_attn_layer(pattern[layer_idx]) else StageKind.FFN - - def _declaration(pattern: str, layer_idx: int): - following = _stage_kind(pattern, layer_idx + 1) if is_attn_layer(pattern[layer_idx]): return declare_attn( - mixer_exit=True, - next_kind=following, - ordinary_only=True, + read=NormQuantRead(before_gather=True), + reduction=ProducerReduction.SCOPED, + gathers_tp_input=False, ) return declare_ffn( sparse=pattern[layer_idx] == MOE, - ordinary_only=True, - return_to_attention=True, + read=NormRead(before_gather=True), + dense_tp_size=get_parallel().tp_size if pattern[layer_idx] != MOE else None, + handoff_rows=HandoffRows.ATTENTION, ) def make_stage_boundary(layer_norm: RMSNorm, *, pattern: str, layer_idx: int): from sglang.srt.layers import layernorm_sp - from sglang.srt.runtime_context import get_parallel if get_parallel().attn_cp_size > 1 or layernorm_sp.layernorm_sp_enabled(): raise NotImplementedError("a Nemotron stage with attention CP or LayerNorm SP") @@ -53,9 +44,11 @@ def make_stage_boundary(layer_norm: RMSNorm, *, pattern: str, layer_idx: int): ( _declaration(pattern, layer_idx), layer_norm, - {"force_layernorm_before_dp_gather": True}, ), previous=previous, + following=_declaration(pattern, layer_idx + 1) + if layer_idx + 1 < len(pattern) + else None, terminal=layer_idx == len(pattern) - 1, ) return boundary diff --git a/python/sglang/srt/models/qwen2_moe.py b/python/sglang/srt/models/qwen2_moe.py index 0177eb237b..228c0b8b27 100644 --- a/python/sglang/srt/models/qwen2_moe.py +++ b/python/sglang/srt/models/qwen2_moe.py @@ -1214,7 +1214,7 @@ class Qwen2MoeModel(nn.Module): if not self.pp_group.is_last_rank: return residual_batch.to_pp(hidden_states, forward_batch) else: - hidden_states = residual_batch.finish(hidden_states, forward_batch) + hidden_states = residual_batch.complete_output(hidden_states, forward_batch) if hidden_states.shape[0] != 0: hidden_states = residual_batch.norm( hidden_states, forward_batch, self.norm @@ -1310,14 +1310,13 @@ class Qwen2MoeForCausalLM(nn.Module): ) if end == self.model.config.num_hidden_layers: - forward_batch.hidden_states = residual_batch.finish( + forward_batch.hidden_states = residual_batch.complete_output( forward_batch.hidden_states, forward_batch ) # norm hidden_states = residual_batch.norm( forward_batch.hidden_states, forward_batch, self.model.norm ) - forward_batch.residual = None forward_batch.hidden_states = hidden_states # logits process result = self.logits_processor( diff --git a/python/sglang/srt/models/qwen3.py b/python/sglang/srt/models/qwen3.py index 7f9ec3bd6c..515b5b2c16 100644 --- a/python/sglang/srt/models/qwen3.py +++ b/python/sglang/srt/models/qwen3.py @@ -477,7 +477,7 @@ class Qwen3Model(Qwen2Model): ) if not self.pp_group.is_last_rank: return residual_batch.to_pp(hidden_states, forward_batch) - hidden_states = residual_batch.finish(hidden_states, forward_batch) + hidden_states = residual_batch.complete_output(hidden_states, forward_batch) if hidden_states.shape[0] != 0: hidden_states = residual_batch.norm(hidden_states, forward_batch, self.norm) return ( diff --git a/python/sglang/srt/models/qwen3_5.py b/python/sglang/srt/models/qwen3_5.py index 6b808c90d7..966a4aeac4 100644 --- a/python/sglang/srt/models/qwen3_5.py +++ b/python/sglang/srt/models/qwen3_5.py @@ -29,6 +29,8 @@ from sglang.kernels.ops.attention.triton_gdn_fused_proj import ( qwen3_5_gdn_prefill_projection_views, ) from sglang.kernels.ops.elementwise.elementwise import fused_sigmoid_mul + +# Configs from sglang.srt.configs.qwen3_5 import ( Qwen3_5Config, Qwen3_5MoeConfig, @@ -52,12 +54,13 @@ from sglang.srt.layers.communicator import ( # Layers - Attention from sglang.srt.layers.communicator.residual import batch as residual_batch + +# Layers - Others +from sglang.srt.layers.communicator.residual.add_norm import Fp8Input, NormQuantRead from sglang.srt.layers.dp_attention import ( is_dp_attention_enabled, ) from sglang.srt.layers.flashinfer_comm_fusion import uses_cutedsl_ar_fusion - -# Layers - Others from sglang.srt.layers.layernorm import GemmaRMSNorm # Layers - Linear @@ -259,32 +262,6 @@ if _is_npu: from sgl_kernel_npu.fla.utils import fused_qkvzba_split_reshape_cat_contiguous -@lru_cache(maxsize=1) -def _enable_qwen35_fused_ar_quant() -> bool: - """Gate the fused AR+RMSNorm+per-group-FP8-quant path for Qwen3.5. - - The single-kernel backend is ROCm/aiter/gfx95-only. The model gate stays - tied to ROCm/aiter so non-gfx95 HIP can keep the existing 2-kernel fallback - behavior for tuple handoff when this branch is used. It replaces the - existing ``--enable-aiter-allreduce-fusion`` 3-kernel path - (AR → RMSNorm → per-group quant) with either a single fused kernel (when - the fully-fused variant is eligible) or a 2-kernel path - (fused AR+RMSNorm + separate per-group quant) that still saves one - kernel launch vs. baseline. The stage boundary gracefully falls back - to ``forward_with_allreduce_fusion`` (plain AR+RMSNorm) when the fused - quant helper returns ``None``, so turning this on never regresses the - AR+RMSNorm fusion itself. - - Opt-out: set ``SGLANG_DISABLE_FUSED_AR_QUANT=1`` to fall back to the - unmodified AR+RMSNorm fusion path. - """ - if not _use_aiter: - return False - if get_bool_env_var("SGLANG_DISABLE_FUSED_AR_QUANT", default="false"): - return False - return bool(get_exec().comm.enable_aiter_allreduce_fusion) - - def _linear_accepts_fp8_tuple(linear: nn.Module) -> bool: quant_method = getattr(linear, "quant_method", None) return quant_method.__class__.__name__ == "Fp8LinearMethod" and ( @@ -1115,18 +1092,17 @@ class Qwen3_5LinearDecoderLayer(nn.Module): # GDN layers need both bf16 (for the small in_proj_ba gating # projection) and a quantized tuple only when in_proj_qkvz can consume # it. Otherwise, stay on the plain AR+RMSNorm path. - enable_fused_ar_quant = ( - _enable_qwen35_fused_ar_quant() - and _linear_accepts_fp8_tuple(self.linear_attn.in_proj_qkvz) - ) + accepts_fp8_input = _linear_accepts_fp8_tuple(self.linear_attn.in_proj_qkvz) boundary_fusions = _layer_fusions(config, is_nextn) self.attn_stage, self.ffn_stage = make_stages( ( - declare_attn(), + declare_attn( + read=NormQuantRead( + fp8_input=Fp8Input.TUPLE_AND_BF16 if accepts_fp8_input else None + ) + ), self.input_layernorm, { - "enable_fused_ar_quant": enable_fused_ar_quant, - "fused_ar_quant_keep_bf16": enable_fused_ar_quant, "fusions": boundary_fusions, }, ), @@ -1322,17 +1298,17 @@ class Qwen3_5AttentionDecoderLayer(nn.Module): # Standard attention layers benefit from a fused quant epilogue only # when qkv_proj can consume the returned quantized tuple. - enable_fused_ar_quant = ( - _enable_qwen35_fused_ar_quant() and _linear_accepts_fp8_tuple(self.qkv_proj) - ) + accepts_fp8_input = _linear_accepts_fp8_tuple(self.qkv_proj) boundary_fusions = _layer_fusions(config, is_nextn) self.attn_stage, self.ffn_stage = make_stages( ( - declare_attn(), + declare_attn( + read=NormQuantRead( + fp8_input=Fp8Input.TUPLE if accepts_fp8_input else None + ) + ), self.input_layernorm, { - "enable_fused_ar_quant": enable_fused_ar_quant, - "fused_ar_quant_keep_bf16": False, "fusions": boundary_fusions, }, ), @@ -1752,6 +1728,7 @@ class Qwen3_5ForCausalLM(nn.Module): # Every layer was checked above. can_defer_finalize=lambda layer: True, label="Qwen3.5", + # The final GemmaRMSNorm consumes a finalize handoff. terminal_finalize=True, ) diff --git a/python/sglang/srt/models/qwen3_moe.py b/python/sglang/srt/models/qwen3_moe.py index 47eea51b15..758dfd82d5 100644 --- a/python/sglang/srt/models/qwen3_moe.py +++ b/python/sglang/srt/models/qwen3_moe.py @@ -1099,14 +1099,13 @@ class Qwen3MoeForCausalLM(nn.Module): ) if end == self.model.config.num_hidden_layers: - forward_batch.hidden_states = residual_batch.finish( + forward_batch.hidden_states = residual_batch.complete_output( forward_batch.hidden_states, forward_batch ) # norm hidden_states = residual_batch.norm( forward_batch.hidden_states, forward_batch, self.model.norm ) - forward_batch.residual = None forward_batch.hidden_states = hidden_states # logits process result = self.logits_processor( diff --git a/python/sglang/srt/models/qwen3_next.py b/python/sglang/srt/models/qwen3_next.py index 507dfdfa8e..c0294b683f 100644 --- a/python/sglang/srt/models/qwen3_next.py +++ b/python/sglang/srt/models/qwen3_next.py @@ -943,7 +943,7 @@ class Qwen3NextModel(nn.Module): else None, ) - hidden_states = residual_batch.finish(hidden_states, forward_batch) + hidden_states = residual_batch.complete_output(hidden_states, forward_batch) if not forward_batch.forward_mode.is_idle(): hidden_states = residual_batch.norm(hidden_states, forward_batch, self.norm) diff --git a/python/sglang/srt/models/qwen3_vl.py b/python/sglang/srt/models/qwen3_vl.py index 553add3fb0..4c4b543fd3 100644 --- a/python/sglang/srt/models/qwen3_vl.py +++ b/python/sglang/srt/models/qwen3_vl.py @@ -1267,7 +1267,7 @@ class Qwen3LLMModel(Qwen3Model): if not self.pp_group.is_last_rank: return residual_batch.to_pp(hidden_states, forward_batch) - hidden_states = residual_batch.finish(hidden_states, forward_batch) + hidden_states = residual_batch.complete_output(hidden_states, forward_batch) if hidden_states.shape[0] != 0: hidden_states = residual_batch.norm( hidden_states, diff --git a/python/sglang/srt/models/qwen3_vl_moe.py b/python/sglang/srt/models/qwen3_vl_moe.py index 1226b1684c..c8a44ae63f 100644 --- a/python/sglang/srt/models/qwen3_vl_moe.py +++ b/python/sglang/srt/models/qwen3_vl_moe.py @@ -157,7 +157,7 @@ class Qwen3MoeLLMModel(Qwen3MoeModel): if not self.pp_group.is_last_rank: return residual_batch.to_pp(hidden_states, forward_batch) else: - hidden_states = residual_batch.finish(hidden_states, forward_batch) + hidden_states = residual_batch.complete_output(hidden_states, forward_batch) if hidden_states.shape[0] != 0: hidden_states = residual_batch.norm( hidden_states, diff --git a/python/sglang/srt/models/sarvam_moe.py b/python/sglang/srt/models/sarvam_moe.py index 60c136ca51..c9818af7ca 100644 --- a/python/sglang/srt/models/sarvam_moe.py +++ b/python/sglang/srt/models/sarvam_moe.py @@ -1169,7 +1169,7 @@ class SarvamMLAModel(nn.Module): if not self.pp_group.is_last_rank: return residual_batch.to_pp(hidden_states, forward_batch) - hidden_states = residual_batch.finish(hidden_states, forward_batch) + hidden_states = residual_batch.complete_output(hidden_states, forward_batch) if hidden_states.shape[0] != 0: hidden_states = residual_batch.norm(hidden_states, forward_batch, self.norm) @@ -1272,7 +1272,7 @@ class SarvamMLAForCausalLM(nn.Module): ) if end == self.model.config.num_hidden_layers: - forward_batch.hidden_states = residual_batch.finish( + forward_batch.hidden_states = residual_batch.complete_output( forward_batch.hidden_states, forward_batch ) forward_batch.hidden_states = residual_batch.norm( @@ -1451,7 +1451,7 @@ class SarvamMoEForCausalLM(BailingMoEForCausalLM): ) if end == self.model.config.num_hidden_layers: - forward_batch.hidden_states = residual_batch.finish( + forward_batch.hidden_states = residual_batch.complete_output( forward_batch.hidden_states, forward_batch ) forward_batch.hidden_states = residual_batch.norm( diff --git a/python/sglang/srt/models/sdar.py b/python/sglang/srt/models/sdar.py index 5e3cb0a85e..1129caac12 100644 --- a/python/sglang/srt/models/sdar.py +++ b/python/sglang/srt/models/sdar.py @@ -412,7 +412,7 @@ class SDARModel(nn.Module): if not self.pp_group.is_last_rank: return residual_batch.to_pp(hidden_states, forward_batch) else: - hidden_states = residual_batch.finish(hidden_states, forward_batch) + hidden_states = residual_batch.complete_output(hidden_states, forward_batch) if not forward_batch.forward_mode.is_idle(): hidden_states = residual_batch.norm( hidden_states, forward_batch, self.norm diff --git a/python/sglang/srt/models/sdar_moe.py b/python/sglang/srt/models/sdar_moe.py index a690c14b43..102453fc1e 100644 --- a/python/sglang/srt/models/sdar_moe.py +++ b/python/sglang/srt/models/sdar_moe.py @@ -478,7 +478,7 @@ class SDARMoeModel(nn.Module): if not self.pp_group.is_last_rank: return residual_batch.to_pp(hidden_states, forward_batch) - hidden_states = residual_batch.finish(hidden_states, forward_batch) + hidden_states = residual_batch.complete_output(hidden_states, forward_batch) if not forward_batch.forward_mode.is_idle(): hidden_states = residual_batch.norm(hidden_states, forward_batch, self.norm) diff --git a/python/sglang/srt/models/step3_vl.py b/python/sglang/srt/models/step3_vl.py index 0706b50329..84e1202b60 100644 --- a/python/sglang/srt/models/step3_vl.py +++ b/python/sglang/srt/models/step3_vl.py @@ -478,7 +478,7 @@ class Step3TextModel(nn.Module): layer = self.layers[i] hidden_states = layer(positions, hidden_states, forward_batch) - hidden_states = residual_batch.finish(hidden_states, forward_batch) + hidden_states = residual_batch.complete_output(hidden_states, forward_batch) if hidden_states.shape[0] != 0: hidden_states = residual_batch.norm(hidden_states, forward_batch, self.norm) diff --git a/python/sglang/srt/models/step3p5.py b/python/sglang/srt/models/step3p5.py index 6915dc6853..5a58b3a02b 100644 --- a/python/sglang/srt/models/step3p5.py +++ b/python/sglang/srt/models/step3p5.py @@ -711,7 +711,7 @@ class Step3p5Model(nn.Module): if not self.pp_group.is_last_rank: return residual_batch.to_pp(hidden_states, forward_batch) else: - hidden_states = residual_batch.finish(hidden_states, forward_batch) + hidden_states = residual_batch.complete_output(hidden_states, forward_batch) hidden_states_before_norm = None if not self.pp_group.is_last_rank: return residual_batch.to_pp(hidden_states, forward_batch) diff --git a/python/sglang/srt/models/step3p5_mtp.py b/python/sglang/srt/models/step3p5_mtp.py index 08fd786fbe..b5b6327072 100644 --- a/python/sglang/srt/models/step3p5_mtp.py +++ b/python/sglang/srt/models/step3p5_mtp.py @@ -116,7 +116,7 @@ class Step3p5AMultiTokenPredictor(nn.Module): hidden_states=hidden_states, forward_batch=forward_batch, ) - hidden_states = residual_batch.finish(hidden_states, forward_batch) + hidden_states = residual_batch.complete_output(hidden_states, forward_batch) hidden_states_before_norm = None if not forward_batch.forward_mode.is_idle(): # if forward_batch.return_hidden_states_before_norm: diff --git a/python/sglang/srt/models/xllm.py b/python/sglang/srt/models/xllm.py index 7de43944bc..c1034f2397 100644 --- a/python/sglang/srt/models/xllm.py +++ b/python/sglang/srt/models/xllm.py @@ -1646,7 +1646,7 @@ class XllmModel(nn.Module): if not self.pp_group.is_last_rank: return residual_batch.to_pp(hidden_states, forward_batch) else: - hidden_states = residual_batch.finish(hidden_states, forward_batch) + hidden_states = residual_batch.complete_output(hidden_states, forward_batch) if hidden_states.shape[0] != 0: hidden_states = residual_batch.norm( hidden_states, forward_batch, self.norm diff --git a/python/sglang/test/boundary_fixtures.py b/python/sglang/test/boundary_fixtures.py index 997a9834b6..14ae2e9d20 100644 --- a/python/sglang/test/boundary_fixtures.py +++ b/python/sglang/test/boundary_fixtures.py @@ -2,12 +2,13 @@ from types import SimpleNamespace -from sglang.srt.layers.communicator.construction import BatchVariant, StagePlan +from sglang.srt.layers.communicator.construction import StagePlan from sglang.srt.layers.communicator.factories import ( declare_attn, declare_ffn, make_stages, ) +from sglang.srt.layers.communicator.ops import identity_output from sglang.srt.layers.communicator.residual.add_norm import PLAIN_RESIDUAL from sglang.srt.layers.communicator.stage import StageCommunicator @@ -40,20 +41,14 @@ def make_test_stages( next_sparse=next_sparse, read=residual.ffn_read, update=residual.ffn_update, - output=output, + output_transform=output, ) - common = { - key: options.pop(key) - for key in ("force_layernorm_before_dp_gather", "fusions") - if key in options - } + common = {key: options.pop(key) for key in ("fusions",) if key in options} attn, ffn = make_stages( ( attention, attention_norm, - dict( - residual_in_hidden=residual.ffn_update.at_producer, **common, **options - ), + dict(**common, **options), ), (ffn, ffn_norm, common), previous=previous, @@ -69,17 +64,8 @@ def stub_plan(): plan.fusions = None plan._publish_lora_layout = False plan._fusion_rows = None - plan._cp_steps = plan._sp_steps = plan._input_scattered_steps = None - plan.is_first_layer = False - plan.is_last_layer = False - plan.residual_in_hidden = False - # Keep production variant selection while tests replace individual paths. - plan._batch_steps = lambda batch: { - BatchVariant.ORDINARY: plan._steps, - BatchVariant.CONTEXT_PARALLEL: plan._cp_steps, - BatchVariant.INPUT_SCATTERED: plan._input_scattered_steps, - BatchVariant.SEQUENCE_PARALLEL: plan._sp_steps, - }[plan._variant(batch)] + plan._paths = {} + plan.enters_stack = False return plan @@ -87,7 +73,10 @@ def stub_stage(plan, kind): """Keep the same concrete boundary when a test patches its methods.""" stages = plan.__dict__.setdefault("_stub_stages", {}) if kind not in stages: - stages[kind] = StageCommunicator(plan, kind, plan.norm) + stages[kind] = StageCommunicator( + plan, + declaration=(declare_attn() if kind.name == "ATTENTION" else declare_ffn()), + ) return stages[kind] @@ -95,18 +84,16 @@ def sp_region_steps(): """Local SP rows with no owed sum, for tests of activation and exits.""" from sglang.srt.layers.communicator import ( NORM_QUANT_READ, - NORM_READ, - BoundarySteps, - CommunicateSummableTensorPairFn, EdgeDecl, Layout, StageEntry, StageInput, StageOutput, + StageSteps, TokenAxis, make_boundary, ) - from sglang.srt.layers.communicator.ops import _hand_qkv_hook_its_input + from sglang.srt.layers.communicator.prepare import _hand_qkv_hook_its_input rows = Layout(frozenset({TokenAxis.ATTN_TP_SCATTER})) output = StageOutput(rows) @@ -117,10 +104,62 @@ def sp_region_steps(): ) return StageEntry(selected.prepare, rows, handoff=handoff) - return BoundarySteps( - entry(NORM_QUANT_READ, _hand_qkv_hook_its_input), - entry(NORM_READ), - output, - CommunicateSummableTensorPairFn._trivial, - False, + return StageSteps( + entry(NORM_QUANT_READ, _hand_qkv_hook_its_input), output, identity_output ) + + +def prepare_input(stage, hidden, residual, forward_batch, **call): + return prepare_raw(stage, "_prepare_input", hidden, residual, forward_batch, **call) + + +def prepare_attention(stage, hidden, residual, forward_batch, *args, **call): + return prepare_raw( + stage, "_prepare_attention", hidden, residual, forward_batch, *args, **call + ) + + +def prepare_raw(stage, method, hidden, residual, forward_batch, *args, **call): + from sglang.srt.layers.communicator.residual.add_norm import ADD + from sglang.srt.layers.communicator.residual.stream import ResidualStream + + if not isinstance(residual, ResidualStream): + stream = ResidualStream(residual) + if residual is not None: + hidden = stream.leave( + hidden, + call.get("update", ADD), + declared_sum=stage.entry(forward_batch).input_sum, + ) + else: + stream = residual + return getattr(stage, method)(hidden, stream, forward_batch, *args, **call) + + +def finish_exit(scope, hidden, residual): + from sglang.srt.layers.communicator.residual.stream import ResidualStream + + if isinstance(residual, ResidualStream): + scope._stream = residual + return scope.finish(hidden), residual + scope._stream.residual = residual + output = scope.finish(hidden) + if scope._stream.pending is None: + return output, None + return scope._stream.input(output) + + +def postprocess_output(boundary, hidden, residual, forward_batch): + from sglang.srt.layers.communicator.residual.stream import ResidualStream + + stream = ( + residual if isinstance(residual, ResidualStream) else ResidualStream(residual) + ) + hidden = boundary.postprocess_layer(hidden, stream, forward_batch) + if isinstance(residual, ResidualStream): + return hidden, stream + return (hidden, None) if stream.pending is None else stream.input(hidden) + + +def identity_input(hidden_states, forward_batch): + return hidden_states diff --git a/python/sglang/test/communicator_patch.py b/python/sglang/test/communicator_patch.py index 0286b1d58f..9e0b3c3364 100644 --- a/python/sglang/test/communicator_patch.py +++ b/python/sglang/test/communicator_patch.py @@ -12,19 +12,19 @@ from sglang.srt.layers import communicator as communicator from sglang.srt.layers.communicator import ( boundary, construction, + contracts, exit, factories, - layer, layout, - legacy_stage, ops, output, + prepare, ) from sglang.srt.layers.communicator import residual as residual_contract from sglang.srt.layers.communicator import ( stage, ) -from sglang.srt.layers.communicator.adapters import attention, branch, lora +from sglang.srt.layers.communicator.adapters import attention, branch, lora, overlap from sglang.srt.layers.communicator.fusions import allreduce from sglang.srt.layers.communicator.residual import ( access, @@ -40,6 +40,7 @@ COMMUNICATOR_MODULES = ( exit, branch, lora, + overlap, layout, output, attention, @@ -50,10 +51,10 @@ COMMUNICATOR_MODULES = ( ops, boundary, construction, + contracts, + prepare, factories, stage, - layer, - legacy_stage, batch, stream, ) diff --git a/test/registered/ops/test_aiter_allreduce_fusion_amd.py b/test/registered/ops/test_aiter_allreduce_fusion_amd.py index 53d4412c4b..3e9ee6f56f 100755 --- a/test/registered/ops/test_aiter_allreduce_fusion_amd.py +++ b/test/registered/ops/test_aiter_allreduce_fusion_amd.py @@ -11,7 +11,7 @@ from unittest.mock import MagicMock, patch import torch from sglang.srt.layers.communicator import UnreducedOutput -from sglang.srt.layers.communicator.layer import LayerCommunicator +from sglang.srt.layers.communicator.fusions.allreduce import complete_attention_input from sglang.srt.layers.moe import utils as moe_utils from sglang.srt.model_executor.forward_batch_info import ForwardMode from sglang.srt.runtime_context import get_context, get_flags, get_parallel @@ -382,9 +382,7 @@ class TestAiterAllreduceFusionGate(CustomTestCase): norm = types.SimpleNamespace( forward_with_allreduce_fusion=MagicMock(return_value=("norm", "residual")) ) - plan = types.SimpleNamespace( - input_layernorm=norm, _attn_input_fuses_quant=False - ) + plan = types.SimpleNamespace(norm=norm) hidden = torch.ones(8, 32) owed = UnreducedOutput(hidden, group=group if matching_group else object()) with ( @@ -399,12 +397,14 @@ class TestAiterAllreduceFusionGate(CustomTestCase): ), get_flags().dp.override(enabled=dp_attention), ): - result = LayerCommunicator._reduce_output_and_update_and_read_residual( + result = complete_attention_input( plan, owed, hidden.clone(), _fake_forward_batch(forward_mode=forward_mode), None, + fuses_quant=False, + keep_bf16=False, ) self.assertEqual(norm.forward_with_allreduce_fusion.called, result is not None) return result is not None diff --git a/test/registered/unit/batch_overlap/test_tbo_unreduced_input.py b/test/registered/unit/batch_overlap/test_tbo_unreduced_input.py index a338254490..c699830886 100644 --- a/test/registered/unit/batch_overlap/test_tbo_unreduced_input.py +++ b/test/registered/unit/batch_overlap/test_tbo_unreduced_input.py @@ -32,15 +32,12 @@ class TestTboEntryReducesItsInput(CustomTestCase): dict( hidden_states=hidden_states[a:b], residual=residual[a:b], - forward_batch=SimpleNamespace( - tbo_parent_token_range=(a, b), residual_stream=None - ), + forward_batch=SimpleNamespace(tbo_parent_token_range=(a, b)), ) for a, b in ((0, 2), (2, 4)) ] def execute(inputs_arr, **kwargs): - self.assertIsNone(batch.residual_stream) parts_seen.extend(inputs_arr) self.assertIsNot( inputs_arr[0]["forward_batch"].residual_stream, @@ -81,7 +78,9 @@ class TestTboEntryReducesItsInput(CustomTestCase): merged = tbo.model_forward_stages( layers=[ SimpleNamespace( - attn_stage=SimpleNamespace(input_rows=Layout(frozenset())) + attn_stage=SimpleNamespace( + incoming_residual_rows=Layout(frozenset()) + ) ) ], enable_tbo=True, @@ -99,52 +98,6 @@ class TestTboEntryReducesItsInput(CustomTestCase): torch.testing.assert_close(merged, torch.full((4, 3), 10.0)) torch.testing.assert_close(output.residual, torch.full((4, 3), 3.0)) - def test_split_and_merge_see_the_reduced_tensor(self): - # Under attention DP the partial sum spans every DP rank's tokens; its - # reduction also brings it back to this rank's three. - reduced = torch.full((3, 4), 2.0) - hidden_states = UnreducedOutput( - torch.ones(6, 4), reduce_and_redistribute=lambda partial: reduced - ) - seen = {} - - def split(**kwargs): - seen["split"] = kwargs["hidden_states"] - return [{}, {}] - - def merge(output_a, output_b, original_len): - seen["original_len"] = original_len - return None, None - - with ( - patch.object(tbo, "_model_forward_tbo_split_inputs", split), - patch.object( - tbo, "execute_overlapped_operations", lambda **kwargs: [{}, {}] - ), - patch.object(tbo, "_model_forward_tbo_merge_outputs", merge), - patch.object( - tbo.deep_gemm_wrapper, - "configure_deep_gemm_num_sms", - lambda num_sms: empty_context(), - ), - ): - tbo._model_forward_tbo( - inputs=dict( - hidden_states=hidden_states, - residual=torch.zeros(3, 4), - positions=None, - forward_batch=SimpleNamespace(residual_stream=None), - zero_allocator=None, - ), - operations_strategy=SimpleNamespace( - deep_gemm_num_sms=None, operations=[], tbo_delta_stages=0 - ), - layer_input_rows=Layout(frozenset()), - ) - - self.assertIs(seen["split"], reduced) - self.assertEqual(seen["original_len"], 3) - if __name__ == "__main__": unittest.main() diff --git a/test/registered/unit/layers/moe/test_cutedsl_ar_fusion.py b/test/registered/unit/layers/moe/test_cutedsl_ar_fusion.py index 034e52acc8..729ac7dfc6 100644 --- a/test/registered/unit/layers/moe/test_cutedsl_ar_fusion.py +++ b/test/registered/unit/layers/moe/test_cutedsl_ar_fusion.py @@ -29,8 +29,7 @@ from sglang.srt.layers.communicator.fusions.cutedsl import ( install_cutedsl_fusion, prepare_cutedsl_fusion, ) -from sglang.srt.layers.communicator.ops import _consumer_step, _read_input -from sglang.srt.layers.communicator.residual import batch as residual_batch +from sglang.srt.layers.communicator.prepare import _consumer_step, _read_input from sglang.srt.layers.communicator.residual.access import finish_layer_stack from sglang.srt.layers.flashinfer_mnnvl_cutedsl import ( FlashInferMNNVLCuteDSLARFusion, @@ -42,7 +41,7 @@ from sglang.srt.model_executor.cuda_graph_config import CudaGraphConfig, PhaseCo from sglang.srt.model_executor.forward_batch_info import ForwardMode from sglang.srt.runtime_context import get_forward, publish, reset_context from sglang.srt.server_args import ServerArgs -from sglang.test.boundary_fixtures import stub_plan, stub_stage +from sglang.test.boundary_fixtures import prepare_input, stub_plan, stub_stage from sglang.test.ci.ci_register import register_cpu_ci from sglang.test.communicator_patch import patch_communicator @@ -52,22 +51,18 @@ _MODULE = "sglang.srt.layers.communicator.fusions.cutedsl" _DECODE = SimpleNamespace(forward_mode=ForwardMode.DECODE, input_ids=torch.zeros(8)) -def _owned_input(hidden, residual, batch=_DECODE): - stream = residual_batch.start(batch) - stream.write(residual) - return stream.leave(hidden, ADD), stream, batch - - def _communicator(): comm = stub_plan() + comm.terminal = False comm.fusions = CuteDSLFusion() comm.norm = RMSNorm(8, eps=1e-6) - comm.enable_fused_ar_quant = False comm._attn_input_fusions = attention_fusions(comm) # Only the ordinary batches' attention input half, with these entries. - comm._sp_steps = comm._input_scattered_steps = comm._cp_steps = None - comm._steps = SimpleNamespace( - attention=SimpleNamespace( + comm._paths[BatchVariant.SEQUENCE_PARALLEL] = comm._paths[ + BatchVariant.INPUT_SCATTERED + ] = comm._paths[BatchVariant.CONTEXT_PARALLEL] = None + comm._paths[BatchVariant.ORDINARY] = SimpleNamespace( + entry=SimpleNamespace( prepare=partial( _consumer_step, adds_plainly=True, @@ -143,7 +138,6 @@ def test_last_layer_consumes_but_does_not_skip_the_pending_all_reduce(eligible): ) ), hands_off_finalize=False, - output_is_replicated=False, ) hidden_states = UnreducedOutput(torch.zeros(8, 8)) @@ -153,8 +147,11 @@ def test_last_layer_consumes_but_does_not_skip_the_pending_all_reduce(eligible): lambda *a, **k: pytest.fail("fell through to the unfused path"), ), ): - out_hidden, _ = stub_stage(last, StageKind.ATTENTION)._prepare_input( - hidden_states, torch.zeros(8, 8), _DECODE + out_hidden, _ = prepare_input( + stub_stage(last, StageKind.ATTENTION), + hidden_states, + torch.zeros(8, 8), + _DECODE, ) assert torch.equal(out_hidden, torch.ones(8, 8)) @@ -229,7 +226,7 @@ def test_the_fusion_runs_only_on_the_ffn_full_rows(): (frozenset({TokenAxis.ATTN_TP_SCATTER}), False), ): comm._batch_steps = lambda fb, rows=Layout(sharded): SimpleNamespace( - attention=None, ffn=SimpleNamespace(input_rows=rows) + entry=SimpleNamespace(input_rows=rows) ) assert fusion._common_eligible(comm, _DECODE, 8) is eligible @@ -238,13 +235,9 @@ def test_finalize_handoff_is_a_producer_capability(eligible): comm = _communicator() fusion = comm.fusions assert not fusion.can_defer_finalize(comm, _DECODE) - fusion.install( - SimpleNamespace(), hands_off_finalize=True, output_is_replicated=False - ) + fusion.install(SimpleNamespace(), hands_off_finalize=True) with patch.object(CuteDSLFusion, "_should_use_finalize", return_value=True): assert fusion.can_defer_finalize(comm, _DECODE) - fusion.requires_local_reduction = True - assert not fusion.can_defer_finalize(comm, _DECODE) def test_install_does_not_require_a_fused_successor(): @@ -274,16 +267,6 @@ def test_install_does_not_require_a_fused_successor(): assert last.fusions.hands_off_finalize -def test_a_replicated_output_producer_keeps_its_own_all_reduce(eligible): - layers = [_test_layer(replicated=value) for value in (True, False)] - _install(layers, requires_local_reduction=lambda layer: layer.replicated) - replicated, plain = [layer.ffn_stage.plan for layer in layers] - assert replicated.fusions.requires_local_reduction - assert not plain.fusions.requires_local_reduction - # The restriction belongs to the producer; incoming sums can still fuse. - assert replicated.fusions._can_consume_post_moe_all_reduce(replicated, _DECODE, 8) - - def test_a_service_nested_under_a_wrapper_is_prepared(): """A VLM wrapper holds the model as a submodule and has no pre-capture hook; an unprepared service declines every M, leaving no fusion at all.""" @@ -436,8 +419,11 @@ def test_a_handoff_the_kernel_does_not_take_is_completed_then_normed(): patch.object(CuteDSLFusion, "_should_use_finalize", return_value=False), patch.object(CuteDSLFusion, "_common_eligible", return_value=False), ): - hidden, residual = stub_stage(comm, StageKind.ATTENTION)._prepare_input( - _handoff(finish), torch.full((2, 8), 2.0), _DECODE + hidden, residual = prepare_input( + stub_stage(comm, StageKind.ATTENTION), + _handoff(finish), + torch.full((2, 8), 2.0), + _DECODE, ) finish.assert_called_once_with() torch.testing.assert_close(residual.residual, torch.full((2, 8), 3.0)) diff --git a/test/registered/unit/layers/moe/test_moe_output_reduction.py b/test/registered/unit/layers/moe/test_moe_output_reduction.py index 3b9e29d45d..8dfb7d37f7 100644 --- a/test/registered/unit/layers/moe/test_moe_output_reduction.py +++ b/test/registered/unit/layers/moe/test_moe_output_reduction.py @@ -259,74 +259,5 @@ class TestModelsWithExplicitDpCompletion(CustomTestCase): torch.testing.assert_close(output, torch.full((2, 3), 2.0)) -def reads_the_tp1_shared_expert_switch(tree): - return any( - isinstance(node, ast.Attribute) and node.attr == "SGLANG_SHARED_EXPERT_TP1" - for node in ast.walk(tree) - ) - - -def calls(tree, name): - return any( - isinstance(node, ast.Call) - and name in (getattr(node.func, "id", None), getattr(node.func, "attr", None)) - for node in ast.walk(tree) - ) - - -class TestReplicatedSharedExpertsCountOnce(CustomTestCase): - def test_models_that_replicate_the_shared_expert_add_it_once(self): - # SGLANG_SHARED_EXPERT_TP1 replicates the shared expert without an a2a - # backend, so the MoE output can still owe a later TP sum. - subjects = {} - for path in sorted(MODELS_DIR.rglob("*.py")): - tree = ast.parse(path.read_text()) - if reads_the_tp1_shared_expert_switch(tree): - subjects[str(path.relative_to(MODELS_DIR))] = calls( - tree, "should_add_replicated_moe_output" - ) - self.assertTrue(subjects, "no model reads SGLANG_SHARED_EXPERT_TP1") - self.assertEqual( - [name for name, ok in subjects.items() if not ok], - [], - "add the replicated shared output under should_add_replicated_moe_output", - ) - - -def open_coded_reductions(tree): - """``if`` statements that test should_skip_post_experts_all_reduce for the TP - path and call tensor_model_parallel_all_reduce themselves.""" - for node in ast.walk(tree): - if not isinstance(node, ast.If): - continue - tests_skip = any( - isinstance(sub, ast.Call) - and getattr(sub.func, "id", None) == "should_skip_post_experts_all_reduce" - and any( - kw.arg == "is_tp_path" and getattr(kw.value, "value", None) is True - for kw in sub.keywords - ) - for sub in ast.walk(node.test) - ) - reduces = any( - isinstance(sub, ast.Call) - and getattr(sub.func, "id", None) == "tensor_model_parallel_all_reduce" - for stmt in node.body - for sub in ast.walk(stmt) - ) - if tests_skip and reduces: - yield node.lineno - - -class TestModelsUseTheSharedReduction(CustomTestCase): - def test_no_model_open_codes_the_tp_reduction(self): - found = [ - f"{path.relative_to(MODELS_DIR)}:{line}" - for path in sorted(MODELS_DIR.rglob("*.py")) - for line in open_coded_reductions(ast.parse(path.read_text())) - ] - self.assertEqual(found, [], "use reduce_moe_output") - - if __name__ == "__main__": unittest.main() diff --git a/test/registered/unit/layers/test_aux_hidden_states.py b/test/registered/unit/layers/test_aux_hidden_states.py index 27ff15d9ea..144e6201a2 100644 --- a/test/registered/unit/layers/test_aux_hidden_states.py +++ b/test/registered/unit/layers/test_aux_hidden_states.py @@ -1,6 +1,3 @@ -from sglang.srt.layers.communicator import StageKind -from sglang.test.boundary_fixtures import stub_plan, stub_stage - """Aux owns retained storage, including captures from reusable gather buffers.""" import unittest @@ -10,8 +7,10 @@ from unittest.mock import Mock, patch import torch from sglang.srt.layers.aux_hidden_states import AuxHiddenStateList, AuxHiddenStatePacker +from sglang.srt.layers.communicator import StageKind from sglang.srt.layers.communicator.residual.access import norm_output from sglang.srt.layers.communicator.residual.stream import ResidualStream +from sglang.test.boundary_fixtures import prepare_attention, stub_plan, stub_stage from sglang.test.ci.ci_register import register_cpu_ci from sglang.test.test_utils import CustomTestCase @@ -119,7 +118,7 @@ class TestAuxStorage(CustomTestCase): with patch.object( torch.Tensor, "clone", side_effect=AssertionError("extra clone") ): - stage._prepare_attention(value, stream, None, **kwargs) + prepare_attention(stage, value, stream, None, **kwargs) value.zero_() torch.testing.assert_close(outputs.finalize(), torch.full((2, 3), 4.0)) @@ -130,7 +129,7 @@ class TestAuxStorage(CustomTestCase): move = Mock(return_value=gathered) stage = self.boundary(stream, move=move) outputs = AuxHiddenStateList() - stage._prepare_attention(source, stream, None, outputs) + prepare_attention(stage, source, stream, None, outputs) gathered.zero_() source.zero_() torch.testing.assert_close(outputs[0], torch.full((4, 3), 7.0)) @@ -171,11 +170,131 @@ class TestAuxStorage(CustomTestCase): with patch.object( torch.Tensor, "clone", side_effect=AssertionError("extra clone") ): - stage._prepare_attention( - hidden, stream, None, capture_output=outputs.capture + prepare_attention( + stage, hidden, stream, None, capture_output=outputs.capture ) torch.testing.assert_close(outputs.finalize(), torch.full((2, 3), 4.0)) +class TestBoundCaptureOwnership(CustomTestCase): + def exercise(self, *, enabled, backend_accepts=True, custom=False, callback=False): + import test_declared_decoder_boundary as fixture + + from sglang.srt.layers import layernorm + from sglang.srt.layers.communicator import ( + ADD, + FusedMlpInput, + SumGroup, + declare_attn, + declare_ffn, + make_stages, + ) + from sglang.srt.model_executor.forward_batch_info import ForwardMode + from sglang.test.communicator_patch import patch_communicator + + parallel = fixture.parallel_of(attn_dp=1, attn_tp=2) + norm = layernorm.RMSNorm(4) + + def inplace_norm(value, residual, unused=None): + residual.add_(value) + return residual * 2, residual + + norm.forward = inplace_norm + + def fused_kernel(*, input_tensor, residual, **kwargs): + if not backend_accepts: + return None, None + updated = residual + input_tensor * 2 + return updated * 2, updated + + def custom_fused(value, residual, batch): + return inplace_norm(value * 2, residual) + + custom_fusions = ( + SimpleNamespace( + ffn_input=lambda plan: ( + FusedMlpInput(completes=SumGroup.ATTN_TP, run=custom_fused), + ), + attention_input=lambda plan: (), + ) + if custom + else None + ) + fb = SimpleNamespace( + forward_mode=ForwardMode.DECODE, + residual_stream=ResidualStream(torch.full((2, 4), 2.0)), + ) + hidden = fb.residual_stream.leave(torch.full((2, 4), 3.0), ADD) + outputs = AuxHiddenStateList() + with ( + fixture.planning(parallel), + patch_communicator("_use_aiter", False), + patch_communicator("apply_aiter_all_reduce_fusion", return_value=False), + patch_communicator( + "apply_flashinfer_allreduce_fusion", return_value=enabled + ), + patch_communicator( + "attention_tensor_model_parallel_all_reduce", + side_effect=lambda x: x * 2, + ), + patch.object(layernorm, "_use_aiter", False), + patch.object(layernorm, "get_parallel", return_value=parallel), + patch( + "sglang.srt.distributed.attention_tensor_model_parallel_all_reduce", + side_effect=lambda x: x * 2, + ), + patch( + "sglang.srt.layers.flashinfer_comm_fusion.flashinfer_allreduce_residual_rmsnorm", + side_effect=fused_kernel, + ), + ): + attn, ffn = make_stages( + (declare_attn(), fixture.Norm()), + (declare_ffn(), norm, {"fusions": custom_fusions}), + previous=declare_ffn(), + ) + predicate = attn.entry(fb).capture_preserves_residual + self.assertEqual( + predicate is not None and predicate(hidden, fb), enabled and not custom + ) + kwargs = ( + {"capture_output": outputs.capture} + if callback + else {"captured_last_layer_outputs": outputs} + ) + if enabled and not custom: + # Only capture is forbidden from cloning. The real FlashInfer + # wrapper's declined-kernel fallback may copy to preserve residual. + with patch.object( + torch.Tensor, "clone", side_effect=AssertionError("extra aux clone") + ): + attn.prepare(hidden, fb, **kwargs) + self.assertIs(outputs[0], fb.residual_stream.residual) + else: + attn.prepare(hidden, fb, **kwargs) + self.assertIsNot(outputs[0], fb.residual_stream.residual) + hidden = attn.finish(torch.ones(2, 4), fb) + result = ffn.prepare(hidden, fb) + torch.testing.assert_close(result, torch.full((2, 4), 14.0)) + torch.testing.assert_close(outputs[0], torch.full((2, 4), 5.0)) + torch.testing.assert_close( + fb.residual_stream.residual, torch.full((2, 4), 7.0) + ) + + def test_selected_flashinfer_wrapper_preserves_capture_including_fallback(self): + for accepts in (False, True): + for callback in (False, True): + with self.subTest(accepts=accepts, callback=callback): + self.exercise( + enabled=True, backend_accepts=accepts, callback=callback + ) + + def test_in_place_or_custom_read_keeps_capture_owned_by_aux(self): + for custom in (False, True): + for callback in (False, True): + with self.subTest(custom=custom, callback=callback): + self.exercise(enabled=custom, custom=custom, callback=callback) + + if __name__ == "__main__": unittest.main() diff --git a/test/registered/unit/layers/test_batch_owned_residual.py b/test/registered/unit/layers/test_batch_owned_residual.py index 5f061daf16..90b6229396 100644 --- a/test/registered/unit/layers/test_batch_owned_residual.py +++ b/test/registered/unit/layers/test_batch_owned_residual.py @@ -216,7 +216,8 @@ class TestBatchOwnedResidual(CustomTestCase): def test_take_output_rejects_pending_or_mismatched_outputs(self): fb = SimpleNamespace(residual_stream=None) - stream = batch.start(fb) + batch.start(fb) + stream = batch.current(fb) value = torch.ones(2, 4) stream.write(value) pending = stream.leave(value * 2, ADD) diff --git a/test/registered/unit/layers/test_boundary_edges.py b/test/registered/unit/layers/test_boundary_edges.py index cd60bdcf51..8703a868bf 100644 --- a/test/registered/unit/layers/test_boundary_edges.py +++ b/test/registered/unit/layers/test_boundary_edges.py @@ -19,7 +19,8 @@ from sglang.srt.layers.communicator import ( make_boundary, make_output_boundary, ) -from sglang.srt.layers.communicator import ops as comm_ops +from sglang.srt.layers.communicator import prepare as comm_ops +from sglang.srt.layers.communicator.ops import identity_output from sglang.srt.runtime_context import get_parallel from sglang.test.ci.ci_register import register_cpu_ci from sglang.test.communicator_patch import patch_communicator @@ -27,8 +28,6 @@ from sglang.test.test_utils import CustomTestCase register_cpu_ci(est_time=5, suite="base-a-test-cpu") -Pair = comm.CommunicateSummableTensorPairFn - def sizes(*, dp=1, cp=1, tp=1): return { @@ -105,7 +104,7 @@ class TestNonAlternatingEdges(CustomTestCase): residual_to=attention, ) producer = make_output_boundary(edge) - self.assertIs(producer.output_move, Pair._trivial) + self.assertIs(producer.output_move, identity_output) consumer = make_boundary(edge) self.assertIs(consumer.prepare.func, comm_ops._consumer_step) self.assertIsNone(consumer.prepare.keywords["step"].keywords["layer_input"]) @@ -162,7 +161,10 @@ class TestNonAlternatingEdges(CustomTestCase): completes=SumGroup.ATTN_TP, run=MagicMock() ) boundary = make_boundary(edge, fusions=(over_tp, over_attention_tp)) - self.assertEqual(boundary.fused, (over_tp,)) + self.assertEqual( + boundary.prepare.keywords["step"].keywords.get("fusions", ()), + (over_tp.run,), + ) step = make_boundary(edge).prepare self.assertIs(step.keywords["step"].func, comm_ops._mlp_input_without_dp) all_reduce = MagicMock(side_effect=lambda h: h * 2) @@ -277,24 +279,15 @@ class TestTheProducersUpdateChoosesTheOrder(CustomTestCase): boundary = make_boundary( edge, fusions=(fused,), carried_fusions=(carried,) ) - self.assertEqual(boundary.fused, (fused,) if offered else ()) + self.assertEqual( + boundary.prepare.keywords["step"].keywords.get("fusions", ()), + (fused.run,) if offered else (), + ) self.assertEqual( boundary.prepare.keywords["carried_fusions"], (carried,) if offered else (), ) - def test_the_consumer_reads_as_it_declares(self): - full = rows(sizes(tp=2)) - read = MagicMock() - edge = EdgeDecl( - produced=StageOutput(full), - need=StageInput(full, read=read), - residual=full, - residual_to=full, - ) - step = make_boundary(edge).prepare.keywords["step"] - self.assertIs(step.keywords["read"], read) - def test_the_next_ffn_completes_a_sum_left_for_it_once(self): full = rows(sizes(tp=2)) edge = EdgeDecl( @@ -323,18 +316,6 @@ class TestTheProducersUpdateChoosesTheOrder(CustomTestCase): group.all_reduce.assert_not_called() torch.testing.assert_close(hidden, torch.full((3, 4), 7.0)) - def test_the_producer_half_ends_on_the_rows_it_hands_on(self): - axis_sizes = sizes(dp=2, tp=2) - attention = rows(axis_sizes, TokenAxis.ATTN_DP) - edge = EdgeDecl( - produced=StageOutput(attention), - need=StageInput(rows(axis_sizes)), - residual=attention, - residual_to=attention, - ) - with self.assertRaises(NotImplementedError): - make_output_boundary(edge) - def test_cross_layer_update_requires_a_lifetime_guarantee(self): local = Layout(frozenset()) stateful_update = SimpleNamespace(adds_plainly=False, at_producer=False) @@ -358,6 +339,7 @@ class TestTheProducersUpdateChoosesTheOrder(CustomTestCase): class ProbeRead: + before_gather = False """A read that marks what it reads: the input is the residual plus 100. ``norms_plainly`` says whether it may stand in for a norm.""" @@ -411,7 +393,9 @@ class TestTheConsumerRunsItsDeclaredRead(CustomTestCase): ), fusions=fusions, ) - return boundary.prepare.keywords["step"], boundary.fused + return boundary.prepare.keywords["step"], boundary.prepare.keywords[ + "step" + ].keywords.get("fusions", ()) def test_the_dp_partial_reads_the_gathered_sum_with_the_declared_read(self): read = ProbeRead(norms_plainly=True) @@ -464,11 +448,9 @@ class TestTheConsumerRunsItsDeclaredRead(CustomTestCase): ProbeRead(norms_plainly), fusions=(fused,) ) self.assertIs(step.func, comm_ops._mlp_input_without_dp) - expected = (fused,) if norms_plainly else () + expected = (fused.run,) if norms_plainly else () self.assertEqual(tried, expected) - self.assertEqual( - step.keywords["fusions"], tuple(f.run for f in expected) - ) + self.assertEqual(step.keywords["fusions"], expected) def test_the_attention_input_tries_its_kernels_only_for_such_a_read(self): axis_sizes = sizes(tp=2) diff --git a/test/registered/unit/layers/test_boundary_output_contracts.py b/test/registered/unit/layers/test_boundary_output_contracts.py new file mode 100644 index 0000000000..a18c6be598 --- /dev/null +++ b/test/registered/unit/layers/test_boundary_output_contracts.py @@ -0,0 +1,242 @@ +"""Numerical and interface coverage for boundary/graph integration regressions.""" + +import unittest +from contextlib import nullcontext +from types import SimpleNamespace +from unittest.mock import Mock, patch + +import test_declared_decoder_boundary as fixture +import torch + +from sglang.srt.layers.communicator import ( + ADD, + ProducerReduction, + declare_attn, + declare_ffn, +) +from sglang.srt.layers.communicator import exit as exits +from sglang.srt.layers.communicator import ( + make_stages, +) +from sglang.srt.layers.communicator.contracts import BatchVariant +from sglang.srt.layers.communicator.fusions.cutedsl import CuteDSLFusion +from sglang.srt.layers.communicator.layout import SumGroup +from sglang.srt.layers.communicator.prepare import _dispatch_consumer +from sglang.srt.layers.communicator.residual import batch +from sglang.srt.layers.communicator.residual.stream import ResidualStream +from sglang.srt.model_executor.forward_batch_info import ForwardMode +from sglang.test.ci.ci_register import register_cpu_ci +from sglang.test.communicator_patch import patch_communicator + +register_cpu_ci(est_time=5, suite="base-a-test-cpu") + + +class TestBoundaryIntegrations(unittest.TestCase): + def test_dp_mixer_keeps_attention_group_reduction(self): + parallel = fixture.parallel_of(attn_dp=2, attn_tp=2) + rows = fixture.comm.Layout(frozenset()) + produced = fixture.comm.StageOutput( + rows, group=SumGroup.ATTN_TP, leaves_for_next_layer=True + ) + plan = SimpleNamespace(_batch_steps=lambda _: SimpleNamespace(output=produced)) + boundary = exits.OutputBoundary(plan) + boundary._ffn_sum_can_move_to_next_layer = lambda _: True + with ( + fixture.planning(parallel), + patch.object(exits, "is_dp_attention_enabled", return_value=True), + ): + stream = ResidualStream(torch.zeros(3, 4)) + result = exits.MixerExit(boundary, None, stream=stream) + self.assertFalse(result.skips_reduction) + value = torch.ones(3, 4) + self.assertIs(result.finish(value), value) + self.assertIsNone(stream.pending.owed) + + def test_pipeline_preserves_declared_sum_for_one_receiver_completion(self): + parallel = fixture.parallel_of( + attn_dp=1, attn_tp=2, enable_attn_tp_input_scattered=True + ) + with fixture.planning(parallel): + attn, _ = make_stages( + (declare_attn(), fixture.Norm()), + (declare_ffn(), fixture.Norm()), + previous=declare_ffn(), + ) + attn.plan._batch_steps = lambda _: attn.plan._paths[ + BatchVariant.INPUT_SCATTERED + ] + fb = SimpleNamespace(residual_stream=ResidualStream(torch.full((4, 4), 3.0))) + hidden = fb.residual_stream.leave( + torch.ones(4, 4), ADD, declared_sum=SumGroup.TP + ) + wire = batch.to_pp(hidden, fb) + self.assertIsNone(fb.residual_stream) + torch.testing.assert_close(wire["hidden_states"], torch.ones(4, 4)) + hidden = attn.from_pp(wire, fb) + reductions = [] + + def reduce_scatter(value, residual): + reductions.append("RS") + return value.chunk(2)[0] * 2, residual.chunk(2)[0] + + with ( + fixture.planning(parallel), + patch_communicator("tp_reduce_scatter", reduce_scatter), + ): + # Rebind the path with the numerical collective stand-in. + attn, _ = make_stages( + (declare_attn(), fixture.Norm()), + (declare_ffn(), fixture.Norm()), + previous=declare_ffn(), + ) + attn.plan._batch_steps = lambda _: attn.plan._paths[ + BatchVariant.INPUT_SCATTERED + ] + attn.plan.qkv_latent_func = None + entry = attn.entry(fb) + hidden, residual = fb.residual_stream.input(hidden) + output, _ = entry.prepare( + hidden, + residual, + fb, + fixture.Norm(), + pending=fb.residual_stream.pending, + update=ADD, + ) + self.assertEqual(reductions, ["RS"]) + torch.testing.assert_close(output, torch.full((2, 4), 10.0)) + + def test_terminal_finalize_requires_explicit_final_consumer(self): + parallel = fixture.parallel_of(attn_dp=1, attn_tp=2) + fb = SimpleNamespace( + input_ids=torch.zeros(2), + forward_mode=ForwardMode.DECODE, + dp_padding_mode=SimpleNamespace(is_max_len=lambda: False), + ) + for accepted in (False, True): + fusion = CuteDSLFusion() + fusion.install( + SimpleNamespace(), hands_off_finalize=True, terminal_finalize=accepted + ) + with ( + fixture.planning(parallel), + patch.object(fusion, "_should_use_finalize", return_value=True), + patch( + "sglang.srt.layers.communicator.fusions.cutedsl.get_parallel", + return_value=parallel, + ), + ): + _, ffn = make_stages( + (declare_attn(), fixture.Norm()), + (declare_ffn(sparse=True), fixture.Norm(), {"fusions": fusion}), + terminal=True, + ) + fb.residual_stream = ResidualStream(torch.ones(2, 4)) + with patch.object( + ffn.plan.output, "_postprocess_dp_step", return_value=None + ): + decision = ffn.exit(fb) + self.assertEqual(decision.defer_moe_finalize, accepted) + + def test_absent_residual_uses_non_partial_update_path(self): + plain = Mock(side_effect=AssertionError("cannot add a missing residual")) + generic = Mock(return_value=("input", None)) + self.assertEqual( + _dispatch_consumer( + None, None, None, None, paths={True: plain, False: generic} + ), + ("input", None), + ) + plain.assert_not_called() + + def test_lora_or_shared_tp1_defers_only_when_fusion_is_eligible(self): + group = object() + residual = torch.ones(2, 4) + fb = SimpleNamespace( + input_ids=torch.zeros(2), residual_stream=ResidualStream(residual) + ) + for lora, shared in ((True, False), (False, True), (False, False)): + for enabled in (False, True): + with ( + patch.object( + exits.envs.SGLANG_SHARED_EXPERT_TP1, "get", return_value=shared + ), + patch.object(exits, "_ffn_has_tokens", return_value=True), + patch.object( + exits, "post_experts_sum_is_one_all_reduce", return_value=False + ), + patch.object( + exits, + "get_lora", + return_value=SimpleNamespace(enable_lora=lora), + ), + patch.object( + exits, + "get_moe_a2a_backend", + return_value=SimpleNamespace(is_none=lambda: True), + ), + patch.object( + exits, + "get_exec", + return_value=SimpleNamespace( + comm=SimpleNamespace(enable_quant_communications=False) + ), + ), + patch.object( + exits, + "get_parallel", + return_value=SimpleNamespace(tp_group=group), + ), + patch.object( + exits, "post_experts_reduction_group", return_value=group + ), + patch.object( + exits, "apply_flashinfer_allreduce_fusion", return_value=enabled + ), + patch.object( + exits, "apply_aiter_all_reduce_fusion", return_value=False + ), + ): + self.assertEqual( + exits._can_defer_ffn_reduction(fb), enabled and (lora or shared) + ) + + def test_unsupported_producer_contracts_fail_at_declaration(self): + with self.assertRaises(ValueError): + declare_ffn(reduction=ProducerReduction.PARTIAL) + with self.assertRaises(ValueError): + declare_attn(reduction=ProducerReduction.LOCAL_TAIL) + + def test_dense_decoder_forwards_capture_callback(self): + from sglang.srt.models.llama4 import Llama4DecoderLayer + from sglang.srt.models.qwen3 import Qwen3DecoderLayer + + for cls in (Qwen3DecoderLayer, Llama4DecoderLayer): + captures = [] + + def prepare(hidden, fb, capture_output=None, **kwargs): + if capture_output is not None: + capture_output(hidden) + return hidden + + output = SimpleNamespace(finish=lambda hidden: hidden) + layer = SimpleNamespace( + attn_stage=SimpleNamespace( + prepare=prepare, finish=lambda hidden, fb: hidden + ), + ffn_stage=SimpleNamespace( + prepare=lambda hidden, fb, **kw: hidden, + exit=lambda fb: nullcontext(output), + ), + self_attn=lambda **kw: kw["hidden_states"], + mlp=lambda hidden, **kw: hidden, + feed_forward=lambda hidden, fb: hidden, + ) + value = torch.ones(2, 4) + cls.forward(layer, None, value, None, capture_output=captures.append) + self.assertEqual(len(captures), 1) + self.assertIs(captures[0], value) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/unit/layers/test_boundary_output_policy.py b/test/registered/unit/layers/test_boundary_output_policy.py index 8d21f864d5..c4511f9928 100644 --- a/test/registered/unit/layers/test_boundary_output_policy.py +++ b/test/registered/unit/layers/test_boundary_output_policy.py @@ -7,10 +7,12 @@ from unittest.mock import patch import test_declared_decoder_boundary as fixture import torch -from sglang.srt.arg_groups.fields.exec_ import ExecComm +from sglang.srt.layers.communicator.contracts import BatchVariant from sglang.srt.layers.communicator.output import OutputTransform +from sglang.srt.layers.communicator.residual.stream import ResidualStream from sglang.srt.layers.moe import utils as moe_utils from sglang.srt.runtime_context import get_forward +from sglang.test.boundary_fixtures import finish_exit from sglang.test.ci.ci_register import register_cpu_ci from sglang.test.communicator_patch import patch_communicator @@ -26,11 +28,8 @@ class TestBoundaryOutputPolicy(unittest.TestCase): ), ) - def test_policy_defaults_to_model_selection(self): - self.assertEqual(ExecComm().boundary_reduction, "auto") - def test_npu_weight_cache_belongs_to_one_prepare_call(self): - from sglang.srt.layers.communicator import ops + from sglang.srt.layers.communicator import prepare as ops from sglang.srt.layers.communicator.residual.stream import ResidualStream parallel = fixture.parallel_of(attn_dp=1, attn_tp=2) @@ -60,7 +59,6 @@ class TestBoundaryOutputPolicy(unittest.TestCase): result = layer.ffn.prepare(hidden, batch, cache=weights) torch.testing.assert_close(result, torch.full((2, 4), 4.0)) self.assertEqual(events, ["reduce", "cache", "reduce"]) - self.assertFalse(hasattr(layer.ffn.plan, "_context")) def test_token_slice_reads_the_current_parallel_rank(self): from sglang.srt.layers.communicator.ops import tp_slice @@ -162,15 +160,51 @@ class TestBoundaryOutputPolicy(unittest.TestCase): patch_communicator("can_use_dp_reduce_scatter", lambda: can_rs), patch_communicator("_to_local_tokens", move), ): - with layer.ffn.plan.output.ffn_exit(self.batch(max_len)) as output: + with layer.ffn.plan.output.ffn_exit( + self.batch(max_len), stream=ResidualStream() + ) as output: self.assertEqual( get_forward().mlp_reduce_scatter, selected != "_redistribute_output", ) - result, _ = output.finish(torch.ones(2, 4), torch.zeros(1, 4)) + result, _ = finish_exit(output, torch.ones(2, 4), torch.zeros(1, 4)) self.assertEqual(calls, [selected]) self.assertEqual(result.shape, (1, 4)) + def test_postprocess_only_scatters_an_already_reduced_output(self): + # The operation-scheduled API used by LongCat NextN receives an MLP + # output that was already summed. An enabled RSv must not sum it again. + for policy in ("ar", "rs", "rsv", "rs+rsv"): + for varlen in (False, True): + with self.subTest(policy=policy, varlen=varlen): + parallel = fixture.parallel_of(attn_dp=2, attn_tp=1) + layer = fixture.build( + fixture.layer_case(2, 3), parallel, boundary_reduction=policy + ) + calls = [] + hidden = torch.arange(8, dtype=torch.float32).reshape(2, 4) + residual = torch.ones(1, 4) + batch = self.batch() + batch.residual_stream = ResidualStream(residual) + + def move(step, batch, value): + calls.append(step.__name__) + return value[:1] + + with ( + fixture.planning(parallel), + patch_communicator( + "should_use_dp_reduce_scatterv", lambda: varlen + ), + patch_communicator("can_use_dp_reduce_scatter", lambda: True), + patch_communicator("_to_local_tokens", move), + ): + output = layer.ffn.postprocess(hidden, batch) + value, saved_residual = batch.residual_stream.finish(output) + self.assertEqual(calls, ["_redistribute_output"]) + torch.testing.assert_close(value, hidden[:1]) + self.assertIs(saved_residual, residual) + def test_moe_cannot_skip_from_topology_without_boundary_request(self): with ( patch.object(moe_utils, "should_use_dp_reduce_scatterv", return_value=True), @@ -220,12 +254,14 @@ class TestBoundaryOutputPolicy(unittest.TestCase): patch_communicator("can_use_dp_reduce_scatter", lambda: True), patch_communicator("_to_local_tokens", move), ): - with layer.ffn.plan.output.ffn_exit(self.batch()) as output: + with layer.ffn.plan.output.ffn_exit( + self.batch(), stream=ResidualStream() + ) as output: value = torch.ones(2, 4) if not get_forward().mlp_reduce_scatter: calls.append("AR") value = value * 2 - value, _ = output.finish(value, torch.zeros(1, 4)) + value, _ = finish_exit(output, value, torch.zeros(1, 4)) self.assertEqual( calls, ["AR", "multiply", "slice"] if disabled else ["multiply", "RS"], @@ -239,7 +275,7 @@ class TestBoundaryOutputPolicy(unittest.TestCase): parallel, output=OutputTransform(lambda x: x.square()), ) - output = layer.ffn.plan._steps.ffn_output + output = layer.ffn.plan._paths.get(BatchVariant.ORDINARY).output self.assertFalse(output.leaves_for_next_layer) self.assertFalse(output.leaves_for_reduce_scatter) self.assertFalse(output.leaves_for_reduce_scatterv) diff --git a/test/registered/unit/layers/test_communicator_ffn_exit.py b/test/registered/unit/layers/test_communicator_ffn_exit.py index 69aa130f71..4a3593ab63 100644 --- a/test/registered/unit/layers/test_communicator_ffn_exit.py +++ b/test/registered/unit/layers/test_communicator_ffn_exit.py @@ -1,13 +1,10 @@ -import ast import types import unittest from functools import partial -from pathlib import Path from unittest.mock import MagicMock import torch -import sglang from sglang.srt.layers import communicator as comm from sglang.srt.layers.communicator import ( Layout, @@ -15,13 +12,22 @@ from sglang.srt.layers.communicator import ( SumGroup, UnreducedOutput, ) -from sglang.srt.layers.communicator import ops as comm_ops +from sglang.srt.layers.communicator import ops as transport_ops +from sglang.srt.layers.communicator import prepare as comm_ops from sglang.srt.layers.communicator import ( reduce_output, ) +from sglang.srt.layers.communicator.contracts import BatchVariant +from sglang.srt.layers.communicator.ops import identity_output from sglang.srt.layers.communicator.residual.access import finish_layer_stack +from sglang.srt.layers.communicator.residual.stream import ResidualStream from sglang.srt.runtime_context import get_forward -from sglang.test.boundary_fixtures import sp_region_steps, stub_plan +from sglang.test.boundary_fixtures import ( + finish_exit, + identity_input, + sp_region_steps, + stub_plan, +) from sglang.test.ci.ci_register import register_cpu_ci from sglang.test.communicator_patch import patch_communicator from sglang.test.test_utils import CustomTestCase @@ -34,11 +40,11 @@ def make_group(scale=3): return types.SimpleNamespace(all_reduce=MagicMock(side_effect=lambda h: h * scale)) -def ordinary_steps(ffn_output, *, returns_over_dp=False): +def ordinary_steps(output, *, returns_over_dp=False): """A layer's ordinary steps, carrying the FFN output declaration the exit reads and whether the output goes back over attention DP.""" - return comm.BoundarySteps( - attention=comm.StageEntry( + return comm.StageSteps( + entry=comm.StageEntry( prepare=partial( comm_ops._consumer_step, step=partial( @@ -51,24 +57,12 @@ def ordinary_steps(ffn_output, *, returns_over_dp=False): carried_fusions=(), ), input_rows=comm.Layout(frozenset()), - input_move=comm.CommunicateSimpleFn._trivial, + input_move=identity_input, handoff=comm_ops._hand_qkv_hook_its_input, ), - ffn=comm.StageEntry( - prepare=partial( - comm_ops._read_input, - layer_input=None, - enters_stack=False, - read=comm.NORM_READ, - update=comm.ADD, - ), - input_rows=Layout(frozenset()), - ), - ffn_output=ffn_output, - ffn_output_move=None - if returns_over_dp - else comm.CommunicateSummableTensorPairFn._trivial, - ffn_sum_is_movable=True, + output=output, + returns_over_dp=returns_over_dp, + output_move=None if returns_over_dp else identity_output, ) @@ -80,17 +74,17 @@ def make_communicator( scatters_to_local_tokens=False, group=None, fusions=None, - allow_deferred=True, + leaves_sum=True, ): """A communicator whose decisions and postprocess are stubbed, built without the process-wide parallel state.""" communicator = stub_plan() communicator.fusions = fusions - communicator._steps = ordinary_steps( + communicator._paths[BatchVariant.ORDINARY] = ordinary_steps( StageOutput( Layout(frozenset()), group=SumGroup.TP, - leaves_for_next_layer=allow_deferred, + leaves_for_next_layer=leaves_sum, ), returns_over_dp=scatters_to_local_tokens, ) @@ -101,10 +95,10 @@ def make_communicator( communicator.output._ffn_leaves_sum_to_reduce_scatter = MagicMock( return_value=reduce_scatter ) - communicator.is_last_layer = False - communicator._sp_steps = None - communicator._input_scattered_steps = None - communicator._cp_steps = None + communicator.terminal = False + communicator._paths[BatchVariant.SEQUENCE_PARALLEL] = None + communicator._paths[BatchVariant.INPUT_SCATTERED] = None + communicator._paths[BatchVariant.CONTEXT_PARALLEL] = None communicator.output._postprocess_dp_step = MagicMock( return_value=reduce_scatter_step ) @@ -124,23 +118,20 @@ def published_flags(): class TestFfnExit(CustomTestCase): def setUp(self): + patches = patch_communicator("_batch_shards_over_cp", return_value=False) + patches.__enter__() + self.addCleanup(patches.__exit__, None, None, None) self.hidden_states = torch.ones(3, 4) self.residual = torch.zeros(3, 4) self.forward_batch = object() def run_exit(self, communicator): - with communicator.output.ffn_exit(self.forward_batch) as ffn_exit: + with communicator.output.ffn_exit( + self.forward_batch, stream=ResidualStream() + ) as ffn_exit: seen = published_flags() hidden_states = self.hidden_states * 2 - return seen, ffn_exit.finish(hidden_states, self.residual) - - def test_reduction_left_to_next_layer_is_unreduced(self): - communicator = make_communicator(fuse=True, reduce_scatter=False) - seen, (hidden_states, residual) = self.run_exit(communicator) - self.assertEqual(seen, (True, False)) - self.assertIsInstance(hidden_states, UnreducedOutput) - self.assertIs(residual, self.residual) - communicator.output._complete_ffn_output_now.assert_not_called() + return seen, finish_exit(ffn_exit, hidden_states, self.residual) def test_reduction_left_to_next_layer_declares_its_group(self): group = make_group() @@ -163,7 +154,7 @@ class TestFfnExit(CustomTestCase): self.assertEqual(seen, (False, True)) self.assertIsInstance(hidden_states, UnreducedOutput) bound = hidden_states.reduce_and_redistribute - self.assertIs(bound.func, comm_ops._to_local_tokens) + self.assertIs(bound.func, transport_ops._to_local_tokens) self.assertEqual(bound.args, (step, self.forward_batch)) self.assertIs(residual, self.residual) communicator.output._complete_ffn_output_now.assert_not_called() @@ -177,7 +168,7 @@ class TestFfnExit(CustomTestCase): _, (hidden_states, _) = self.run_exit(communicator) self.assertIsInstance(hidden_states, UnreducedOutput) bound = hidden_states.reduce_and_redistribute - self.assertIs(bound.func, comm_ops._all_reduce_then_to_local_tokens) + self.assertIs(bound.func, transport_ops._all_reduce_then_to_local_tokens) self.assertEqual(bound.args, (group, self.forward_batch)) communicator.output._complete_ffn_output_now.assert_not_called() group.all_reduce.assert_not_called() @@ -197,7 +188,7 @@ class TestFfnExit(CustomTestCase): def test_a_layer_that_declares_no_deferral_runs_postprocess(self): communicator = make_communicator( - fuse=True, reduce_scatter=True, allow_deferred=False + fuse=True, reduce_scatter=True, leaves_sum=False ) seen, (hidden_states, residual) = self.run_exit(communicator) @@ -212,30 +203,38 @@ class TestFfnExit(CustomTestCase): for fuse in (True, False): with self.subTest(fuse=fuse): communicator = make_communicator(fuse=fuse, reduce_scatter=False) - with communicator.output.ffn_exit(self.forward_batch) as ffn_exit: + with communicator.output.ffn_exit( + self.forward_batch, stream=ResidualStream() + ) as ffn_exit: seen = published_flags() decide = communicator.output._ffn_sum_moves_to_next_layer decide.return_value = not fuse hidden_states = self.hidden_states * 2 - hidden_states, _ = ffn_exit.finish(hidden_states, self.residual) + hidden_states, _ = finish_exit(ffn_exit, hidden_states, self.residual) self.assertEqual(seen, (fuse, False)) self.assertEqual(isinstance(hidden_states, UnreducedOutput), fuse) self.assertEqual( communicator.output._complete_ffn_output_now.called, not fuse ) + def test_exit_selects_its_batch_path_once(self): + communicator = make_communicator(fuse=False, reduce_scatter=False) + selected = communicator._paths.get(BatchVariant.ORDINARY) + lookup = MagicMock(return_value=selected) + communicator._batch_steps = lookup + self.run_exit(communicator) + lookup.assert_called_once_with(self.forward_batch) + self.assertIs( + communicator.output._complete_ffn_output_now.call_args.kwargs["steps"], + selected, + ) + def test_flags_are_restored_after_the_ffn(self): before = published_flags() - for fuse, reduce_scatter in ((True, False), (False, True)): - self.run_exit(make_communicator(fuse=fuse, reduce_scatter=reduce_scatter)) + for fuse, scatter in ((True, False), (False, True)): + self.run_exit(make_communicator(fuse=fuse, reduce_scatter=scatter)) self.assertEqual(published_flags(), before) - - def test_deferral_does_not_inspect_consumer_fusion(self): - for allows in (False, True): - communicator = make_communicator(fuse=allows, reduce_scatter=False) - seen, _ = self.run_exit(communicator) - self.assertEqual(seen, (allows, False)) - communicator.output._ffn_sum_moves_to_next_layer.assert_called_once() + self.assertEqual(published_flags(), before) def test_deferral_implies_fusion_and_passes_the_handoff_through(self): """A deferring communicator publishes both flags, and a non-tensor @@ -246,22 +245,32 @@ class TestFfnExit(CustomTestCase): fuse=False, reduce_scatter=False, fusions=fusions ) handoff = object() - with communicator.output.ffn_exit(self.forward_batch) as ffn_exit: + with communicator.output.ffn_exit( + self.forward_batch, stream=ResidualStream() + ) as ffn_exit: seen = published_flags() + (get_forward().defer_moe_finalize,) self.assertEqual(seen, (True, False, True)) self.assertEqual( - ffn_exit.finish(handoff, self.residual), (handoff, self.residual) + finish_exit(ffn_exit, handoff, self.residual), (handoff, self.residual) ) communicator.output._complete_ffn_output_now.assert_not_called() self.assertFalse(get_forward().defer_moe_finalize) # The MoE may decline per forward; a tensor then takes the fused exit. - hidden_states, _ = ffn_exit.finish(self.hidden_states * 2, self.residual) + with communicator.output.ffn_exit( + self.forward_batch, stream=ResidualStream() + ) as fallback_exit: + hidden_states, _ = finish_exit( + fallback_exit, self.hidden_states * 2, self.residual + ) self.assertIsInstance(hidden_states, UnreducedOutput) class TestReduceOutput(CustomTestCase): def setUp(self): + patches = patch_communicator("_batch_shards_over_cp", return_value=False) + patches.__enter__() + self.addCleanup(patches.__exit__, None, None, None) self.group = make_group() self.all_reduce = self.group.all_reduce @@ -270,9 +279,11 @@ class TestReduceOutput(CustomTestCase): def test_reduction_left_by_the_last_layer_runs_once(self): communicator = self.communicator(fuse=True) - with communicator.output.ffn_exit(object()) as ffn_exit: + with communicator.output.ffn_exit( + object(), stream=ResidualStream() + ) as ffn_exit: hidden_states = torch.ones(3, 4) - hidden_states, _ = ffn_exit.finish(hidden_states, torch.zeros(3, 4)) + hidden_states, _ = finish_exit(ffn_exit, hidden_states, torch.zeros(3, 4)) hidden_states = reduce_output(hidden_states) self.all_reduce.assert_called_once() @@ -293,25 +304,17 @@ class TestReduceOutput(CustomTestCase): self.assertIs(hidden_states, local) self.all_reduce.assert_not_called() - def test_complete_hidden_states_pass_through(self): - communicator = self.communicator(fuse=False) - with communicator.output.ffn_exit(object()) as ffn_exit: - hidden_states = torch.ones(3, 4) - hidden_states, _ = ffn_exit.finish(hidden_states, torch.zeros(3, 4)) - - self.assertIs(reduce_output(hidden_states), hidden_states) - self.assertIsNone(reduce_output(None)) - self.all_reduce.assert_not_called() - def test_finish_layer_stack_completes_the_last_layer(self): for fuse in (True, False): with self.subTest(fuse=fuse): self.all_reduce.reset_mock() communicator = self.communicator(fuse=fuse) residual = torch.zeros(3, 4) - with communicator.output.ffn_exit(object()) as ffn_exit: + with communicator.output.ffn_exit( + object(), stream=ResidualStream() + ) as ffn_exit: hidden_states = torch.ones(3, 4) - hidden_states, _ = ffn_exit.finish(hidden_states, residual) + hidden_states, _ = finish_exit(ffn_exit, hidden_states, residual) hidden_states, residual_out = finish_layer_stack( hidden_states, residual, object() @@ -326,15 +329,22 @@ class TestSelectFfnCompletion(CustomTestCase): """What the next layer's input runs in place of postprocess, chosen before the FFN runs.""" + def setUp(self): + patches = patch_communicator("_batch_shards_over_cp", return_value=False) + patches.__enter__() + self.addCleanup(patches.__exit__, None, None, None) + def communicator( self, *, fuse=False, is_last_layer=False, scatters=True, sp_region=False ): communicator = stub_plan() - communicator.is_last_layer = is_last_layer - communicator._sp_steps = sp_region_steps() if sp_region else None - communicator._input_scattered_steps = None - communicator._cp_steps = None - communicator._steps = ordinary_steps( + communicator.terminal = is_last_layer + communicator._paths[BatchVariant.SEQUENCE_PARALLEL] = ( + sp_region_steps() if sp_region else None + ) + communicator._paths[BatchVariant.INPUT_SCATTERED] = None + communicator._paths[BatchVariant.CONTEXT_PARALLEL] = None + communicator._paths[BatchVariant.ORDINARY] = ordinary_steps( StageOutput( Layout(frozenset()), group=SumGroup.MOE_OUTPUT, @@ -344,8 +354,8 @@ class TestSelectFfnCompletion(CustomTestCase): ), returns_over_dp=scatters, ) - communicator.output._ffn_sum_moves_to_next_layer = lambda forward_batch, **_: ( - fuse + communicator.output._ffn_sum_moves_to_next_layer = ( + lambda forward_batch, steps, **_: fuse ) communicator.output._ffn_leaves_sum_to_reduce_scatter = ( lambda forward_batch, dp_step: not fuse @@ -359,24 +369,26 @@ class TestSelectFfnCompletion(CustomTestCase): with patch_communicator( "_reduce_and_redistribute_output_step", return_value=step ): - completion = communicator.output._select_ffn_completion(forward_batch) + completion = communicator.output._select_ffn_completion( + forward_batch, communicator.output.plan._batch_steps(forward_batch) + ) hidden_states, _ = completion.complete(torch.ones(3, 4), None) return None if hidden_states == "now" else hidden_states def test_a_reduce_scatter_is_bound_for_the_next_layer(self): forward_batch = object() for step in ( - comm_ops._reduce_and_redistribute_output_varlen, - comm_ops._reduce_and_redistribute_output_max_len, + transport_ops._reduce_and_redistribute_output_varlen, + transport_ops._reduce_and_redistribute_output_max_len, ): with self.subTest(step=step.__name__): left = self.left(self.communicator(), step, forward_batch) bound = left.reduce_and_redistribute - self.assertIs(bound.func, comm_ops._to_local_tokens) + self.assertIs(bound.func, transport_ops._to_local_tokens) self.assertEqual(bound.args, (step, forward_batch)) def test_postprocess_keeps_everything_else(self): - reduce_scatter = comm_ops._reduce_and_redistribute_output_varlen + reduce_scatter = transport_ops._reduce_and_redistribute_output_varlen for name, communicator, step in ( ("scatter only", self.communicator(), None), ("last layer", self.communicator(is_last_layer=True), reduce_scatter), @@ -389,7 +401,9 @@ class TestSelectFfnCompletion(CustomTestCase): communicator = self.communicator(sp_region=True) with get_forward().scoped(sp_active=True): self.assertIsNone( - self.left(communicator, comm_ops._reduce_and_redistribute_output_varlen) + self.left( + communicator, transport_ops._reduce_and_redistribute_output_varlen + ) ) def test_a_deferred_sum_keeps_its_layout_or_scatters_back(self): @@ -400,7 +414,7 @@ class TestSelectFfnCompletion(CustomTestCase): moved = self.left(self.communicator(fuse=True), None, forward_batch) self.assertIsNone(moved.group) bound = moved.reduce_and_redistribute - self.assertIs(bound.func, comm_ops._all_reduce_then_to_local_tokens) + self.assertIs(bound.func, transport_ops._all_reduce_then_to_local_tokens) self.assertEqual(bound.args, (self.group, forward_batch)) def test_the_all_reduce_runs_before_the_scatter(self): @@ -421,195 +435,13 @@ class TestSelectFfnCompletion(CustomTestCase): side_effect=lambda out, full, fb: calls.append(("scatter", out)), ), ): - result = comm_ops._all_reduce_then_to_local_tokens(group, object(), partial) + result = transport_ops._all_reduce_then_to_local_tokens( + group, object(), partial + ) self.assertEqual([c[0] for c in calls], ["all_reduce", "buffer", "scatter"]) self.assertIs(calls[0][1], partial) self.assertIs(result, local) -SRT_DIR = Path(sglang.__file__).resolve().parent / "srt" -# Attributes a decoder layer holds its FFN, or parts of it, in. A layer that is -# one FFN stage holds it in ``mixer``; a layer with several dense FFNs, in -# ``mlps``. -FFN_ATTRS = { - "mlp", - "mlps", - "moe", - "shared_expert", - "shared_experts", - "share_expert", - "mixer", - "feed_forward", -} - - -def can_be_true(value, params): - """Whether a use_dp_attention_reduce value can turn it on: anything but - False or a function forwarding its own parameter of that name.""" - if isinstance(value, ast.Constant) and value.value is False: - return False - return not (isinstance(value, ast.Name) and value.id in params) - - -class SourceIndex: - def __init__(self): - self.paths = { - "sglang.srt." + str(p.relative_to(SRT_DIR))[:-3].replace("/", "."): p - for p in SRT_DIR.rglob("*.py") - } - self._trees = {} - - def tree(self, module): - if module not in self._trees: - self._trees[module] = ast.parse(self.paths[module].read_text()) - return self._trees[module] - - def resolve(self, module, name): - """The (module, ClassDef) that `name` refers to in `module`, if any.""" - for node in self.tree(module).body: - if isinstance(node, ast.ClassDef) and node.name == name: - return module, node - for node in ast.walk(self.tree(module)): - if isinstance(node, ast.ImportFrom) and node.module in self.paths: - for alias in node.names: - if (alias.asname or alias.name) == name: - return self.resolve(node.module, alias.name) - return None - - -def submodule_constructions(cls): - """(attribute, class name, call) for each `self.attr = Name(...)`, and for - each `Name(...)` in `self.attr = nn.ModuleList([...])`.""" - for node in ast.walk(cls): - if isinstance(node, ast.Assign) and isinstance(node.value, ast.Call): - for target in node.targets: - if ( - isinstance(target, ast.Attribute) - and getattr(target.value, "id", None) == "self" - ): - for call in constructed(node.value): - yield target.attr, call.func.id, call - - -def constructed(call): - """The `Name(...)` calls a call builds: itself, or the elements of the list - an `nn.ModuleList` holds.""" - if isinstance(call.func, ast.Name): - yield call - elif getattr(call.func, "attr", None) == "ModuleList" and call.args: - items = call.args[0] - elements = ( - [items.elt] - if isinstance(items, ast.ListComp) - else getattr(items, "elts", []) - ) - for element in elements: - if isinstance(element, ast.Call) and isinstance(element.func, ast.Name): - yield element - - -def attention_tp_reductions(cls): - """Lines in a class that can turn on use_dp_attention_reduce.""" - found = [] - for fn in ast.walk(cls): - if not isinstance(fn, (ast.FunctionDef, ast.AsyncFunctionDef)): - continue - params = {a.arg for a in fn.args.args + fn.args.kwonlyargs} - for node in ast.walk(fn): - if isinstance(node, ast.keyword) and node.arg == "use_dp_attention_reduce": - if can_be_true(node.value, params): - found.append(node.value.lineno) - elif isinstance(node, ast.Assign) and any( - isinstance(t, ast.Attribute) and t.attr == "use_dp_attention_reduce" - for t in node.targets - ): - if can_be_true(node.value, params): - found.append(node.lineno) - return found - - -def ffn_exit_classes(tree, source): - """Classes in a module that call ffn_exit, and the classes there that inherit - from them (whose constructors build the FFN the inherited forward runs).""" - classes = [node for node in tree.body if isinstance(node, ast.ClassDef)] - users = { - cls.name - for cls in classes - if ".output.ffn_exit(" in ast.get_source_segment(source, cls) - or any( - isinstance(node, ast.withitem) - and isinstance(node.optional_vars, ast.Name) - and node.optional_vars.id == "ffn_exit" - for node in ast.walk(cls) - ) - } - grew = True - while grew: - grew = False - for cls in classes: - if cls.name not in users and any( - getattr(base, "id", None) in users for base in cls.bases - ): - users.add(cls.name) - grew = True - return [cls for cls in classes if cls.name in users] - - -class TestFfnExitUsersOweATpSum(CustomTestCase): - """Under attention DP the next layer's input completes a deferred FFN sum - with an all-reduce over TP. An FFN whose linear layers reduce over the - attention-TP group only (use_dp_attention_reduce) owes a different sum, so - no FFN of a layer that uses ffn_exit may turn it on.""" - - def test_no_ffn_behind_ffn_exit_reduces_over_attention_tp(self): - index = SourceIndex() - layers, ffn_classes, offenders = [], set(), [] - for module, path in sorted(index.paths.items()): - source = path.read_text() - if not module.startswith("sglang.srt.models.") or ( - ".output.ffn_exit(" not in source and " as ffn_exit:" not in source - ): - continue - for cls in ffn_exit_classes(index.tree(module), source): - layers.append(f"{module}.{cls.name}") - todo = [] - for attr, name, call in submodule_constructions(cls): - if attr not in FFN_ATTRS: - continue - todo.append((module, name)) - offenders += [ - f"{module}.{cls.name}:{kw.value.lineno}" - for kw in call.keywords - if kw.arg == "use_dp_attention_reduce" - and can_be_true(kw.value, set()) - ] - # Everything an FFN class builds is part of the FFN. - while todo: - resolved = index.resolve(*todo.pop()) - if resolved is None: - continue - where, ffn = resolved - if (where, ffn.name) in ffn_classes: - continue - ffn_classes.add((where, ffn.name)) - offenders += [ - f"{where}.{ffn.name}:{line}" - for line in attention_tp_reductions(ffn) - ] - todo += [(where, n) for _, n, _ in submodule_constructions(ffn)] - self.assertTrue(layers and ffn_classes, "no FFN behind ffn_exit found") - # A layer that is one FFN stage inherits its forward and holds the FFN - # in ``mixer``. - nemotron = "sglang.srt.models.nemotron_h" - self.assertLessEqual( - {(nemotron, "NemotronHMLP"), (nemotron, "NemotronHMoE")}, ffn_classes - ) - # A layer with two dense FFNs holds them in an nn.ModuleList. - self.assertIn( - ("sglang.srt.models.longcat_flash", "LongcatFlashMLP"), ffn_classes - ) - self.assertEqual(sorted(set(offenders)), []) - - if __name__ == "__main__": unittest.main() diff --git a/test/registered/unit/layers/test_declared_attention_cp.py b/test/registered/unit/layers/test_declared_attention_cp.py index 507004bee0..830b72e9db 100644 --- a/test/registered/unit/layers/test_declared_attention_cp.py +++ b/test/registered/unit/layers/test_declared_attention_cp.py @@ -1,5 +1,3 @@ -from sglang.test.boundary_fixtures import make_test_stages - """A MoE layer on the TP group under DSA (and MLA) prefill CP, rank by rank. Two CP ranks build the layer's communicator for real and run prepare_mlp and @@ -21,6 +19,7 @@ from sglang.srt.layers import communicator as comm from sglang.srt.layers import layernorm_sp from sglang.srt.layers.communicator.adapters import context_parallel as dsa_cp from sglang.srt.layers.communicator.residual.stream import ResidualStream +from sglang.test.boundary_fixtures import finish_exit, make_test_stages, prepare_input from sglang.test.ci.ci_register import register_cpu_ci from sglang.test.communicator_patch import patch_communicator from sglang.test.test_utils import CustomTestCase @@ -210,16 +209,20 @@ class TestAttentionCpBoundary(CustomTestCase): with self.as_rank( cp, dict(gather=record_gather(cp), reduce_scatter=unused) ): - self.build(use_reduce_scatter).ffn._prepare_input( - self.values[cp], self.residuals[cp], self.cp_extend() + prepare_input( + self.build(use_reduce_scatter).ffn, + self.values[cp], + self.residuals[cp], + self.cp_extend(), ) gathered, residuals = {}, {} for cp in range(CP_SIZE): with self.as_rank(cp, dict(gather=fill_gather, reduce_scatter=unused)): - gathered[cp], residuals[cp] = self.build( - use_reduce_scatter - ).ffn._prepare_input( - self.values[cp], self.residuals[cp], self.cp_extend() + gathered[cp], residuals[cp] = prepare_input( + self.build(use_reduce_scatter).ffn, + self.values[cp], + self.residuals[cp], + self.cp_extend(), ) expected_rows = torch.cat( [self.values[cp] + self.residuals[cp] for cp in range(CP_SIZE)] @@ -227,7 +230,7 @@ class TestAttentionCpBoundary(CustomTestCase): for cp in range(CP_SIZE): torch.testing.assert_close(gathered[cp], expected_rows, rtol=0, atol=0) - def ffn_output(cp, leaves): + def compute_output(cp, leaves): return gathered[cp] * PARTIAL_WEIGHTS[cp] if leaves else gathered[cp] reduced = {} @@ -258,12 +261,12 @@ class TestAttentionCpBoundary(CustomTestCase): ) as rank: communicator = self.build(use_reduce_scatter) with communicator.ffn.plan.output.ffn_exit( - self.cp_extend() + self.cp_extend(), stream=ResidualStream() ) as exit_: published[cp] = rank.flags.mlp_reduce_scatter - output = ffn_output(cp, leaves=published[cp]) - back[cp], _ = exit_.finish( - output, ResidualStream(residuals[cp].residual) + output = compute_output(cp, leaves=published[cp]) + back[cp], _ = finish_exit( + exit_, output, ResidualStream(residuals[cp].residual) ) return published, back, reduced diff --git a/test/registered/unit/layers/test_declared_decoder_boundary.py b/test/registered/unit/layers/test_declared_decoder_boundary.py index c732c09895..9209e722a8 100644 --- a/test/registered/unit/layers/test_declared_decoder_boundary.py +++ b/test/registered/unit/layers/test_declared_decoder_boundary.py @@ -1,7 +1,3 @@ -from sglang.srt.layers.communicator.adapters import branch -from sglang.srt.layers.communicator.construction import BatchVariant -from sglang.test.boundary_fixtures import make_test_stages - """The boundaries of an attention and an FFN, chosen from both sides' declarations. The FFN runs on the TP group (a dense MLP, or a MoE not dispatched per DP shard) or on each attention-TP rank's slice of its DP shard's @@ -16,6 +12,7 @@ import itertools import threading import unittest from contextlib import ExitStack, contextmanager, nullcontext +from dataclasses import replace from functools import partial from types import SimpleNamespace from unittest.mock import MagicMock, patch @@ -32,13 +29,33 @@ from sglang.srt.layers.communicator import boundary as comm_boundary from sglang.srt.layers.communicator import construction as comm_layer from sglang.srt.layers.communicator import exit as comm_exit from sglang.srt.layers.communicator import layout as comm_layout -from sglang.srt.layers.communicator import ops as comm_ops +from sglang.srt.layers.communicator import ops as transport_ops +from sglang.srt.layers.communicator import prepare as comm_ops +from sglang.srt.layers.communicator.adapters import branch +from sglang.srt.layers.communicator.construction import BatchVariant from sglang.srt.layers.communicator.fusions.allreduce import complete_ffn_input +from sglang.srt.layers.communicator.ops import ( + gather_attention_tp, + identity_output, + output_on_residual_shard, + reduce_scatter_over_cp, + scatter_moe_cp_output, + scatter_output, + take_back_cp_shard, + update_and_gather, +) from sglang.srt.layers.communicator.residual import mhc as mhc_module from sglang.srt.layers.communicator.residual.access import finish_layer_stack -from sglang.srt.layers.communicator.residual.stream import OwedOutput +from sglang.srt.layers.communicator.residual.add_norm import NORM_READ, PLAIN_RESIDUAL +from sglang.srt.layers.communicator.residual.stream import OwedOutput, ResidualStream from sglang.srt.layers.moe import utils as moe_utils -from sglang.srt.runtime_context import LoRABatchLayout +from sglang.srt.model_executor.forward_batch_info import ForwardMode +from sglang.test.boundary_fixtures import ( + finish_exit, + make_test_stages, + postprocess_output, + prepare_input, +) from sglang.test.ci.ci_register import register_cpu_ci from sglang.test.communicator_patch import COMMUNICATOR_MODULES, patch_communicator from sglang.test.test_utils import CustomTestCase @@ -182,65 +199,9 @@ def build( ) -class TestWhichLayersUseDeclarations(CustomTestCase): +class TestStageLayoutSelection(CustomTestCase): """Production factories choose the input and output paths for each stage.""" - def declared(self, communicator): - return getattr( - communicator.ffn.plan._steps.ffn.prepare.keywords["step"], "func", None - ) in ( - comm_ops._mlp_input_dp_partial, - comm_ops._mlp_input_dp_replicate, - comm_ops._mlp_input_without_dp, - comm_ops._mlp_input_scatter, - ) - - def test_a_dense_model(self): - parallel = parallel_of(attn_dp=2, attn_tp=2) - for layer_id in range(3): - with self.subTest(layer_id=layer_id): - communicator = build(layer_case(layer_id, 3), parallel) - self.assertTrue(self.declared(communicator)) - self.assertIsNone(communicator.attn.plan._steps.attention.input_move) - self.assertTrue(communicator.ffn.plan._steps.returns_over_dp) - - def test_plain_tp(self): - for norm, fuses in ((Norm, False), (FusableNorm, True)): - with self.subTest(norm=norm.__name__): - with planning(parallel_of(attn_dp=1, attn_tp=2)): - communicator = make_test_stages( - **layer_case(1, 3), attention_norm=norm(), ffn_norm=norm() - ) - self.assertIs( - communicator.ffn.plan._steps.ffn.prepare.keywords["step"].func, - comm_ops._mlp_input_without_dp, - ) - entries = communicator.ffn.plan._steps.ffn.prepare.keywords[ - "step" - ].keywords["fusions"] - self.assertEqual(len(entries), int(fuses)) - if fuses: - self.assertIs(entries[0].func, complete_ffn_input) - self.assertEqual(entries[0].args, (communicator.ffn.plan,)) - self.assertIs( - bool(communicator.ffn.plan._steps.ffn.fused), - fuses, - ) - self.assertFalse(communicator.ffn.plan._steps.returns_over_dp) - self.assertIs( - communicator.ffn.plan._steps.ffn_output_move, - comm.CommunicateSummableTensorPairFn._trivial, - ) - # One rank: the attention output is complete, only the norm is left. - one_rank = parallel_of(attn_dp=1, attn_tp=1) - single = build(layer_case(1, 3), one_rank) - with planning(one_rank): - self.assertIsNotNone(single.attn.plan.variants) - self.assertIs( - single.ffn.plan._steps.ffn.prepare.keywords["step"].func, - comm_ops._read_input, - ) - def test_the_order_follows_the_attention_output(self): for attn_tp, force, order in ( (2, False, comm_ops._mlp_input_dp_partial), @@ -251,102 +212,17 @@ class TestWhichLayersUseDeclarations(CustomTestCase): communicator = build( layer_case(1, 3), parallel_of(attn_dp=2, attn_tp=attn_tp), - force_layernorm_before_dp_gather=force, + residual=PLAIN_RESIDUAL._replace( + ffn_read=replace(NORM_READ, before_gather=force) + ), ) self.assertIs( - communicator.ffn.plan._steps.ffn.prepare.keywords["step"].func, + communicator.ffn.plan._paths.get(BatchVariant.ORDINARY) + .entry.prepare.keywords["step"] + .func, order, ) - def test_a_dense_first_layer_before_sparse_ones(self): - parallel = parallel_of(attn_dp=2, attn_tp=2) - for a2a in (False, True): - with self.subTest(a2a=a2a): - layers = [ - build( - layer_case(i, 4, sparse=sparse, previous_sparse=previous), - parallel, - a2a=a2a, - ) - for i, (sparse, previous) in enumerate( - ((False, False), (True, False), (True, True), (False, True)) - ) - ] - self.assertEqual([self.declared(layer) for layer in layers], [True] * 4) - if not a2a: - moe = layers[1].ffn.plan._steps.ffn_output - self.assertIs(moe.group, SumGroup.MOE_OUTPUT) - self.assertTrue(moe.leaves_for_reduce_scatterv) - continue - # The first a2a layer slices the residual it takes from a - # dense layer; the next one takes it already sliced. - self.assertEqual( - [ - layers[i].ffn.plan._steps.ffn.prepare.keywords["step"].func - for i in (1, 2) - ], - [comm_ops._mlp_input_scatter] * 2, - ) - self.assertEqual( - [ - layers[i] - .ffn.plan._steps.ffn.prepare.keywords["step"] - .keywords["scatters_residual"] - for i in (1, 2) - ], - [True, False], - ) - self.assertIsNone(layers[1].ffn.plan._steps.ffn_output.group) - self.assertIs( - layers[2].attn.plan._steps.attention.input_move, - comm.CommunicateSimpleFn._scattered_to_tp_attn_full, - ) - self.assertIs( - layers[3].ffn.plan._steps.ffn.prepare.keywords["step"].func, - comm_ops._mlp_input_dp_partial, - ) - self.assertTrue( - layers[3] - .ffn.plan._steps.ffn.prepare.keywords["step"] - .keywords["gathers_residual"] - ) - - def test_a_dense_mlp_on_every_rank(self): - # moe_dense_tp_size 1: each rank runs the dense MLP on its own slice, as - # an a2a MoE does, and owes no sum. - parallel = parallel_of(attn_dp=2, attn_tp=2, moe_dense_tp_size=1) - no_overlap = SimpleNamespace( - comm=SimpleNamespace(boundary_reduction="rs+rsv"), - overlap=SimpleNamespace(enable_two_batch_overlap=False), - ) - for a2a in (False, True): - with ( - self.subTest(a2a=a2a), - patch_communicator("get_exec", lambda: no_overlap), - ): - layers = [ - build( - layer_case(i, 4, sparse=sparse, previous_sparse=previous), - parallel, - a2a=a2a, - ) - for i, (sparse, previous) in enumerate( - ((False, False), (False, False), (True, False), (True, True)) - ) - ] - self.assertEqual([self.declared(layer) for layer in layers], [True] * 4) - for dense in layers[:2]: - self.assertIs( - dense.ffn.plan._steps.ffn.prepare.keywords["step"].func, - comm_ops._mlp_input_scatter, - ) - self.assertIsNone(dense.ffn.plan._steps.ffn_output.group) - for after_dense in layers[1:3]: - self.assertIs( - after_dense.attn.plan._steps.attention.input_move, - comm.CommunicateSimpleFn._scattered_to_tp_attn_full, - ) - def test_the_last_a2a_layer_folds_the_residual_back(self): parallel = parallel_of(attn_dp=2, attn_tp=2) last = build( @@ -355,8 +231,8 @@ class TestWhichLayersUseDeclarations(CustomTestCase): a2a=True, ) self.assertIs( - last.ffn.plan._steps.ffn_output_move.func, - comm.CommunicateSummableTensorPairFn._gather, + last.ffn.plan._paths.get(BatchVariant.ORDINARY).output_move.func, + update_and_gather, ) def test_an_attention_that_gathers_its_slice_itself(self): @@ -364,7 +240,9 @@ class TestWhichLayersUseDeclarations(CustomTestCase): facts = layer_case(2, 4, sparse=True, previous_sparse=True) with patch_communicator("_use_ag_after_qlora", True): layer = build(facts, parallel, a2a=True) - self.assertIsNone(layer.attn.plan._steps.attention.input_move) + self.assertIsNone( + layer.attn.plan._paths.get(BatchVariant.ORDINARY).entry.input_move + ) def test_cp_without_prefill_cp(self): """No batch shards its tokens, so no CP steps. Under attention DP the @@ -372,17 +250,31 @@ class TestWhichLayersUseDeclarations(CustomTestCase): so the FFN completes its own sum; without attention DP it may leave it and input-scattered attention runs, as without CP.""" under_dp = build(layer_case(1, 3), parallel_of(attn_dp=2, attn_tp=1, attn_cp=2)) - self.assertIsNone(under_dp.ffn.plan._cp_steps) - self.assertFalse(under_dp.ffn.plan._steps.ffn_output.leaves_for_next_layer) - self.assertFalse(under_dp.ffn.plan._steps.ffn_output.leaves_for_reduce_scatterv) + self.assertIsNone(under_dp.ffn.plan._paths.get(BatchVariant.CONTEXT_PARALLEL)) + self.assertFalse( + under_dp.ffn.plan._paths.get( + BatchVariant.ORDINARY + ).output.leaves_for_next_layer + ) + self.assertFalse( + under_dp.ffn.plan._paths.get( + BatchVariant.ORDINARY + ).output.leaves_for_reduce_scatterv + ) without_dp = build( layer_case(1, 3), parallel_of( attn_dp=1, attn_tp=2, attn_cp=2, enable_attn_tp_input_scattered=True ), ) - self.assertTrue(without_dp.ffn.plan._steps.ffn_output.leaves_for_next_layer) - self.assertIsNotNone(without_dp.ffn.plan._input_scattered_steps) + self.assertTrue( + without_dp.ffn.plan._paths.get( + BatchVariant.ORDINARY + ).output.leaves_for_next_layer + ) + self.assertIsNotNone( + without_dp.ffn.plan._paths.get(BatchVariant.INPUT_SCATTERED) + ) def build_mhc( @@ -434,74 +326,62 @@ class TestMhcOnTheDeclarations(CustomTestCase): for key, value in keywords.items(): self.assertEqual(step.keywords[key], value, key) - def test_each_boundary_runs_the_shared_step(self): - for name, parallel, ffn_input, output_move in ( - ( - "attention TP 1", - parallel_of(attn_dp=1, attn_tp=1), - comm_ops._read_input, - comm.CommunicateSummableTensorPairFn._trivial, - ), - ( - "the attention-TP sum", - parallel_of(attn_dp=1, attn_tp=2), - comm_ops._mlp_input_without_dp, - comm.CommunicateSummableTensorPairFn._trivial, - ), - # The DP gather runs after hc_post, which is not a plain add. - ( - "a DP gather", - parallel_of(attn_dp=2, attn_tp=2), - comm_ops._mlp_input_dp_replicate, - None, - ), - ): - with self.subTest(name): - communicator = build_mhc(layer_case(1, 3), parallel) - self.assert_step( - communicator.ffn.plan._steps.ffn.prepare, ffn_input, communicator - ) - self.assertIs(communicator.ffn.plan._steps.ffn_output_move, output_move) - # The fused add + RMSNorm kernels do not write hc_post. - self.assertEqual(communicator.ffn.plan._steps.ffn.fused, ()) - self.assertEqual( - communicator.attn.plan._steps.attention.prepare.keywords[ - "carried_fusions" - ], - (), - ) - def test_the_move_back_over_dp_stays_with_the_layer(self): parallel = parallel_of(attn_dp=2, attn_tp=2) communicator = build_mhc(layer_case(1, 3), parallel) - self.assertTrue(communicator.ffn.plan._steps.returns_over_dp) - self.assertFalse(communicator.ffn.plan._steps.ffn_output.leaves_for_next_layer) + self.assertTrue( + communicator.ffn.plan._paths.get(BatchVariant.ORDINARY).returns_over_dp + ) + self.assertFalse( + communicator.ffn.plan._paths.get( + BatchVariant.ORDINARY + ).output.leaves_for_next_layer + ) # The move back also writes the FFN output into the streams. self.assertFalse( - communicator.ffn.plan.output._local_token_move_can_go_to_next_layer(None) + communicator.ffn.plan.output._local_token_move_can_go_to_next_layer( + communicator.ffn.plan.output.plan._batch_steps( + SimpleNamespace(forward_mode=ForwardMode.DECODE) + ) + ) ) max_len = SimpleNamespace( dp_padding_mode=SimpleNamespace(is_max_len=lambda: True), - forward_mode=SimpleNamespace(is_context_parallel_extend=lambda: False), + forward_mode=ForwardMode.DECODE, ) with ( planning(parallel), - patch_communicator("get_forward", lambda: SimpleNamespace(sp_active=False)), + patch_communicator( + "get_forward", + lambda: SimpleNamespace(sp_active=False, attn_input_scattered=False), + ), patch_communicator("should_use_dp_reduce_scatterv", lambda: False), patch_communicator("can_use_dp_reduce_scatter", lambda: True), ): - step = communicator.ffn.plan.output._postprocess_dp_step(max_len) + step = communicator.ffn.plan.output._postprocess_dp_step( + max_len, communicator.ffn.plan.output.plan._batch_steps(max_len) + ) self.assertTrue( communicator.ffn.plan.output._ffn_leaves_sum_to_reduce_scatter( - max_len, step + communicator.ffn.plan.output.plan._batch_steps(max_len), step ) ) def test_the_exit_runs_the_move_back_over_dp_it_chose(self): parallel = parallel_of(attn_dp=2, attn_tp=2) for name, reduce_scatterv, is_max_len, move in ( - ("SUM_LEN", True, False, comm_ops._reduce_and_redistribute_output_varlen), - ("MAX_LEN", False, True, comm_ops._reduce_and_redistribute_output_max_len), + ( + "SUM_LEN", + True, + False, + transport_ops._reduce_and_redistribute_output_varlen, + ), + ( + "MAX_LEN", + False, + True, + transport_ops._reduce_and_redistribute_output_max_len, + ), ): with self.subTest(name): hc_post = MagicMock(side_effect=lambda h, r, h_res, h_post: h + r) @@ -513,9 +393,7 @@ class TestMhcOnTheDeclarations(CustomTestCase): communicator.mhc.h_res = communicator.mhc.h_post = torch.zeros(2) forward_batch = SimpleNamespace( dp_padding_mode=SimpleNamespace(is_max_len=lambda: is_max_len), - forward_mode=SimpleNamespace( - is_context_parallel_extend=lambda: False - ), + forward_mode=ForwardMode.DECODE, ) choose = MagicMock(wraps=comm_exit._reduce_and_redistribute_output_step) to_local_tokens = MagicMock(side_effect=lambda step, fb, h: h[:2]) @@ -529,11 +407,11 @@ class TestMhcOnTheDeclarations(CustomTestCase): patch_communicator("_to_local_tokens", to_local_tokens), ): with communicator.ffn.plan.output.ffn_exit( - forward_batch + forward_batch, stream=ResidualStream() ) as ffn_exit: self.assertTrue(ffn_exit.mlp_reduce_scatter) - hidden, residual = ffn_exit.finish( - torch.ones(4, 4), torch.full((2, 4), 2.0) + hidden, residual = finish_exit( + ffn_exit, torch.ones(4, 4), torch.full((2, 4), 2.0) ) choose.assert_called_once() to_local_tokens.assert_called_once() @@ -556,20 +434,20 @@ class TestMhcOnTheDeclarations(CustomTestCase): ] first_a2a, last = layers[1], layers[2] self.assert_step( - first_a2a.ffn.plan._steps.ffn.prepare, + first_a2a.ffn.plan._paths.get(BatchVariant.ORDINARY).entry.prepare, comm_ops._mlp_input_scatter, first_a2a, scatters_residual=True, ) self.assert_step( - last.ffn.plan._steps.ffn.prepare, + last.ffn.plan._paths.get(BatchVariant.ORDINARY).entry.prepare, comm_ops._mlp_input_scatter, last, scatters_residual=False, ) self.assert_step( - last.ffn.plan._steps.ffn_output_move, - comm.CommunicateSummableTensorPairFn._gather, + last.ffn.plan._paths.get(BatchVariant.ORDINARY).output_move, + update_and_gather, last, ) state = first_a2a.mhc @@ -588,45 +466,49 @@ class TestMhcOnTheDeclarations(CustomTestCase): for layer_id in range(3): with self.subTest(layer_id=layer_id): communicator = build_mhc(layer_case(layer_id, 3), parallel) - steps = communicator.ffn.plan._input_scattered_steps + steps = communicator.ffn.plan._paths.get(BatchVariant.INPUT_SCATTERED) self.assert_step( - steps.ffn.prepare, + steps.entry.prepare, comm_ops._mlp_input_on_residual_shard, communicator, ) self.assert_step( - steps.ffn_output_move, - comm.CommunicateSummableTensorPairFn._onto_residual_shard, + steps.output_move, + output_on_residual_shard, communicator, sums=True, gathers_back=layer_id == 2, ) - self.assertTrue(steps.ffn_output_move_completes_sum) + self.assertTrue(steps.output_move_completes_sum) completes = build_mhc( layer_case(layer_id, 3), parallel, boundary_reduction="ar", - ).ffn.plan._input_scattered_steps - self.assertFalse(completes.ffn_output_move.keywords["sums"]) - self.assertFalse(completes.ffn_output_move_completes_sum) + ).ffn.plan._paths.get(BatchVariant.INPUT_SCATTERED) + self.assertFalse(completes.output_move.keywords["sums"]) + self.assertFalse(completes.output_move_completes_sum) # Only the first layer's input, the embedding's partial sum, # is completed onto the slice. self.assertIs( - communicator.attn.plan._input_scattered_steps.attention.prepare.keywords[ - "step" - ].keywords["layer_input"], + communicator.attn.plan._paths.get(BatchVariant.INPUT_SCATTERED) + .entry.prepare.keywords["step"] + .keywords["layer_input"], comm.tp_reduce_scatter if layer_id == 0 else None, ) self.assertIsNone( - communicator.attn.plan._input_scattered_steps.attention.input_move + communicator.attn.plan._paths.get( + BatchVariant.INPUT_SCATTERED + ).entry.input_move ) self.assertIs( - communicator.attn.plan._input_scattered_steps.attention.handoff, + communicator.attn.plan._paths.get( + BatchVariant.INPUT_SCATTERED + ).entry.handoff, comm_ops._hand_scattered_input_to_attention, ) self.assertIs( - communicator.ffn.plan._steps.ffn_output_move, - comm.CommunicateSummableTensorPairFn._trivial, + communicator.ffn.plan._paths.get(BatchVariant.ORDINARY).output_move, + identity_output, ) def test_two_batch_overlap_gathers_on_the_declarations(self): @@ -637,13 +519,13 @@ class TestMhcOnTheDeclarations(CustomTestCase): layer_case(1, 3, next_sparse=True), parallel, two_batch_overlap=True ) self.assert_step( - communicator.ffn.plan._steps.ffn.prepare, + communicator.ffn.plan._paths.get(BatchVariant.ORDINARY).entry.prepare, comm_ops._mlp_input_scatter, communicator, ) self.assert_step( - communicator.ffn.plan._steps.ffn_output_move, - comm.CommunicateSummableTensorPairFn._gather, + communicator.ffn.plan._paths.get(BatchVariant.ORDINARY).output_move, + update_and_gather, communicator, ) @@ -661,14 +543,20 @@ class TestMhcOnTheDeclarations(CustomTestCase): with ( planning(parallel), patch_communicator( - "get_forward", lambda: SimpleNamespace(sp_active=False) + "get_forward", + lambda: SimpleNamespace( + sp_active=False, attn_input_scattered=False + ), ), patch.object( mhc_module, "hc_contract", lambda h, hc_mult: h.sum(-1) ), ): - hidden, residual = communicator.ffn.plan.output.postprocess_layer( - torch.ones(2, 4), torch.full((2, 4), 2.0), None + hidden, residual = postprocess_output( + communicator.ffn.plan.output, + torch.ones(2, 4), + torch.full((2, 4), 2.0), + SimpleNamespace(forward_mode=ForwardMode.DECODE), ) self.assertIsNone(residual) expected = torch.full((2, 4), 3.0) @@ -757,7 +645,6 @@ class TestTwoBatchOverlap(CustomTestCase): return layers def test_the_dense_layer_before_the_split_hands_on_the_attention_rows(self): - pair = comm.CommunicateSummableTensorPairFn attention = comm.Layout(frozenset({TokenAxis.ATTN_DP})) local = comm.Layout(frozenset({TokenAxis.ATTN_DP, TokenAxis.ATTN_TP_SCATTER})) for tbo, gathered in ((True, True), (False, False)): @@ -765,130 +652,51 @@ class TestTwoBatchOverlap(CustomTestCase): first, before, after, last = self.layers(tbo=tbo) for layer in (first, before, after, last): self.assertIsNotNone(layer.attn.plan.variants) - self.assertIs(first.ffn.plan._steps.ffn_output_move, pair._trivial) + self.assertIs( + first.ffn.plan._paths.get(BatchVariant.ORDINARY).output_move, + identity_output, + ) if gathered: self.assertIs( - before.ffn.plan._steps.ffn_output_move.func, pair._gather + before.ffn.plan._paths.get( + BatchVariant.ORDINARY + ).output_move.func, + update_and_gather, + ) + self.assertEqual(after.attn.incoming_residual_rows, attention) + self.assertIsNone( + after.attn.plan._paths.get( + BatchVariant.ORDINARY + ).entry.input_move ) - self.assertEqual(after.attn.input_rows, attention) - self.assertIsNone(after.attn.plan._steps.attention.input_move) else: - self.assertIs(before.ffn.plan._steps.ffn_output_move, pair._trivial) - self.assertEqual(after.attn.input_rows, local) self.assertIs( - after.attn.plan._steps.attention.input_move, - comm.CommunicateSimpleFn._scattered_to_tp_attn_full, + before.ffn.plan._paths.get(BatchVariant.ORDINARY).output_move, + identity_output, + ) + self.assertEqual(after.attn.incoming_residual_rows, local) + self.assertIs( + after.attn.plan._paths.get( + BatchVariant.ORDINARY + ).entry.input_move, + gather_attention_tp, ) def test_the_split_moves_from_the_first_layer_s_rows(self): - pair = comm.CommunicateSummableTensorPairFn parallel = parallel_of(attn_dp=2, attn_tp=2) dp = frozenset({TokenAxis.ATTN_DP}) with planning(parallel): self.assertEqual( - comm.tbo_split_moves(comm.Layout(dp)), (pair._trivial, pair._trivial) + comm.tbo_split_moves(comm.Layout(dp)), + (identity_output, identity_output), ) self.assertEqual( comm.tbo_split_moves(comm.Layout(dp | {TokenAxis.ATTN_TP_SCATTER})), - (pair._gather, pair._scatter), + (update_and_gather, scatter_output), ) with self.assertRaises(NotImplementedError): comm.tbo_split_moves(comm.Layout(frozenset())) - def build(self, facts, parallel, *, a2a, two_batch_overlap): - overlap = SimpleNamespace( - comm=SimpleNamespace(boundary_reduction="rs+rsv"), - overlap=SimpleNamespace(enable_two_batch_overlap=two_batch_overlap), - ) - with ( - planning(parallel, a2a=a2a), - patch_communicator("get_exec", lambda: overlap), - ): - communicator = make_test_stages( - **facts, attention_norm=Norm(), ffn_norm=Norm() - ) - communicator.ffn.plan._publish_lora_layout = True - return communicator - - def test_the_rows_are_the_mlp_mode_layout(self): - for ( - attn_dp, - attn_tp, - layer_id, - sparse, - previous_sparse, - a2a, - dense_fully_dp, - two_batch_overlap, - ) in itertools.product( - (1, 2, 4), - (1, 2), - (0, 1, 3), - (False, True), - (False, True), - (False, True), - (False, True), - (False, True), - ): - if two_batch_overlap and attn_dp == 1: - continue - with self.subTest( - attn_dp=attn_dp, - attn_tp=attn_tp, - layer_id=layer_id, - sparse=sparse, - previous_sparse=previous_sparse, - a2a=a2a, - dense_fully_dp=dense_fully_dp, - two_batch_overlap=two_batch_overlap, - ): - parallel = parallel_of( - attn_dp=attn_dp, - attn_tp=attn_tp, - moe_dense_tp_size=1 if dense_fully_dp else None, - ) - facts = layer_case( - layer_id, 4, sparse=sparse, previous_sparse=previous_sparse - ) - communicator = self.build( - facts, - parallel, - a2a=a2a, - two_batch_overlap=two_batch_overlap, - ) - steps = communicator.ffn.plan._steps - self.assertEqual( - steps.ffn.input_rows, - communicator.ffn.plan.variants[ - BatchVariant.ORDINARY - ].incoming.need.layout, - ) - published = {} - with patch_communicator( - "get_forward", - lambda: SimpleNamespace(set=published.__setitem__), - ): - from sglang.srt.layers.communicator.adapters.lora import ( - publish_ffn, - ) - - publish_ffn( - communicator.ffn.plan._publish_lora_layout, steps.ffn.input_rows - ) - # LoRA admits attention DP only with attention TP 1, where the - # FFN takes every row unless it runs on each rank's own: a MoE - # dispatched by an a2a backend, or a dense MLP on every rank. - on_local_rows = a2a if sparse else dense_fully_dp - if attn_dp > 1 and attn_tp == 1: - self.assertIs( - published["lora_batch_layout"], - ( - LoRABatchLayout.DP_LOCAL - if on_local_rows - else LoRABatchLayout.TP_GLOBAL - ), - ) - class TestTheAttentionOutputDecidesItsSum(CustomTestCase): """prepare_mlp completes the attention-TP sum exactly when the attention @@ -897,11 +705,16 @@ class TestTheAttentionOutputDecidesItsSum(CustomTestCase): def test_the_attention_output_owes_the_attention_tp_sum(self): for attn_tp in (1, 2): stages = build(layer_case(1, 3), parallel_of(attn_dp=2, attn_tp=attn_tp)) - produced = stages.attn.plan._steps.ffn_output + produced = stages.attn.plan._paths.get(BatchVariant.ORDINARY).output self.assertIs(produced.group, SumGroup.ATTN_TP if attn_tp > 1 else None) self.assertIs(produced.always_leaves, attn_tp > 1) - self.assertIs(stages.ffn.plan._steps.ffn_output.group, SumGroup.TP) - self.assertFalse(stages.ffn.plan._steps.ffn_output.always_leaves) + self.assertIs( + stages.ffn.plan._paths.get(BatchVariant.ORDINARY).output.group, + SumGroup.TP, + ) + self.assertFalse( + stages.ffn.plan._paths.get(BatchVariant.ORDINARY).output.always_leaves + ) def run_steps(self, produced, *, force=False): sizes = { @@ -909,14 +722,15 @@ class TestTheAttentionOutputDecidesItsSum(CustomTestCase): TokenAxis.ATTN_CP: 1, TokenAxis.ATTN_TP_SCATTER: 2, } - steps, _, _ = comm_boundary._select_input_steps( + steps, _ = comm_boundary._select_input_steps( produced, residual=comm.Layout.sharded_over(TokenAxis.ATTN_DP, axis_sizes=sizes), residual_to=comm.Layout.sharded_over(TokenAxis.ATTN_DP, axis_sizes=sizes), - need=comm.StageInput(comm.Layout(frozenset())), + need=comm.StageInput( + comm.Layout(frozenset()), read=replace(NORM_READ, before_gather=force) + ), adds_plainly=True, fusions=(), - force_layernorm_before_gather=force, residual_joins_sum=False, cp_moves=None, enters_stack=False, @@ -973,20 +787,6 @@ class TestTheAttentionOutputDecidesItsSum(CustomTestCase): self.assertEqual(reductions, 0) torch.testing.assert_close(residual, torch.full((1, HIDDEN), 10.0)) - def test_declarations_the_steps_cannot_run_are_rejected(self): - layout = comm.Layout(frozenset({TokenAxis.ATTN_DP})) - for produced in ( - # Always leaves a sum over no group. - comm.StageOutput(layout, always_leaves=True), - # Owes a sum over a group the steps do not complete. - comm.StageOutput(layout, group=SumGroup.TP, always_leaves=True), - ): - with ( - self.subTest(produced=produced), - self.assertRaises(NotImplementedError), - ): - self.run_steps(produced) - class TestFusedKernelsTakeOnlyTheStepsTheyComplete(CustomTestCase): """A fused kernel replaces the attention-TP sum, the residual add and the @@ -1006,19 +806,18 @@ class TestFusedKernelsTakeOnlyTheStepsTheyComplete(CustomTestCase): TokenAxis.ATTN_TP_SCATTER: 2, } attention = comm.Layout.sharded_over(TokenAxis.ATTN_DP, axis_sizes=sizes) - steps, fused, _ = comm_boundary._select_input_steps( + steps, _ = comm_boundary._select_input_steps( comm.StageOutput(attention, group=SumGroup.ATTN_TP, always_leaves=True), residual=comm.Layout.sharded_over(TokenAxis.ATTN_DP, axis_sizes=sizes), residual_to=comm.Layout.sharded_over(TokenAxis.ATTN_DP, axis_sizes=sizes), need=comm.StageInput(comm.Layout(frozenset())), adds_plainly=True, fusions=fusions, - force_layernorm_before_gather=False, residual_joins_sum=False, cp_moves=None, enters_stack=False, ) - return steps, fused + return steps, steps.keywords.get("fusions", ()) def test_a_kernel_over_another_group_is_not_chosen(self): over_tp = self.fused(SumGroup.TP) @@ -1026,7 +825,7 @@ class TestFusedKernelsTakeOnlyTheStepsTheyComplete(CustomTestCase): steps, chosen = self.select(attn_dp=1, fusions=(over_tp, over_attention_tp)) self.assertIs(steps.func, comm_ops._mlp_input_without_dp) self.assertEqual(steps.keywords["fusions"], (over_attention_tp.run,)) - self.assertEqual(chosen, (over_attention_tp,)) + self.assertEqual(chosen, (over_attention_tp.run,)) def test_no_kernel_is_chosen_where_rows_are_gathered(self): steps, chosen = self.select(attn_dp=2, fusions=(self.fused(SumGroup.ATTN_TP),)) @@ -1042,29 +841,6 @@ class TestTheSequenceParallelRegion(CustomTestCase): SIZES = {TokenAxis.ATTN_DP: 1, TokenAxis.ATTN_CP: 1, TokenAxis.ATTN_TP_SCATTER: 2} - def test_a_layer_under_sp_takes_both_sets_of_steps(self): - parallel = parallel_of(attn_dp=1, attn_tp=2) - for layer_id in range(3): - with self.subTest(layer_id=layer_id): - with planning(parallel, sp=True): - communicator = make_test_stages( - **layer_case(layer_id, 3), - attention_norm=Norm(), - ffn_norm=Norm(), - ) - for boundary in (communicator.attn, communicator.ffn): - steps = boundary.plan._sp_steps - entry = steps.attention or steps.ffn - self.assertEqual( - entry.input_rows.sharded, frozenset({TokenAxis.ATTN_TP_SCATTER}) - ) - self.assertIsNone(entry.input_move) - self.assertIsNone(steps.ffn_output.group) - self.assertIs( - communicator.ffn.plan._steps.ffn.prepare.keywords["step"].func, - comm_ops._mlp_input_without_dp, - ) - def test_what_the_ffn_exit_reads_comes_from_the_batch_s_steps(self): # Inside the region the FFN output is complete: no sum to move. parallel = parallel_of(attn_dp=1, attn_tp=2) @@ -1074,7 +850,10 @@ class TestTheSequenceParallelRegion(CustomTestCase): self.subTest(sp_active=active), planning(parallel, sp=True), patch_communicator( - "get_forward", lambda: SimpleNamespace(sp_active=active) + "get_forward", + lambda: SimpleNamespace( + sp_active=active, attn_input_scattered=False + ), ), patch_communicator( "get_attn_tp_context", @@ -1083,17 +862,14 @@ class TestTheSequenceParallelRegion(CustomTestCase): patch_communicator("is_dp_attention_enabled", lambda: False), ): self.assertIs( - communicator.ffn.plan.output._ffn_sum_can_move_to_next_layer(None), + communicator.ffn.plan.output._ffn_sum_can_move_to_next_layer( + communicator.ffn.plan.output.plan._batch_steps( + SimpleNamespace(forward_mode=ForwardMode.DECODE) + ) + ), not active, ) - def test_without_sp_there_is_no_region(self): - with planning(parallel_of(attn_dp=1, attn_tp=2)): - communicator = make_test_stages( - **layer_case(1, 3), attention_norm=Norm(), ffn_norm=Norm() - ) - self.assertIsNone(communicator.ffn.plan._sp_steps) - class TestInputScatteredAttention(CustomTestCase): """On a batch whose attention input is scattered over attention TP, a layer @@ -1103,37 +879,6 @@ class TestInputScatteredAttention(CustomTestCase): SIZES = {TokenAxis.ATTN_DP: 1, TokenAxis.ATTN_CP: 1, TokenAxis.ATTN_TP_SCATTER: 2} - def test_the_declarations_choose_the_steps(self): - for hands_on in (False, True): - with self.subTest(hands_on_partial=hands_on): - communicator = build( - layer_case(1, 3), - boundary_reduction="rs+rsv" if hands_on else "ar", - parallel=parallel_of( - attn_dp=1, attn_tp=2, enable_attn_tp_input_scattered=True - ), - ) - steps = communicator.ffn.plan._input_scattered_steps - attention = communicator.attn.plan._input_scattered_steps.attention - self.assertIs( - attention.prepare.keywords["step"].keywords["layer_input"], - comm.tp_reduce_scatter, - ) - self.assertIsNone(attention.input_move) - slice_ = comm.Layout(frozenset({TokenAxis.ATTN_TP_SCATTER})) - self.assertEqual(attention.input_rows, slice_) - self.assertEqual(steps.ffn.input_rows, comm.Layout(frozenset())) - self.assertIs( - steps.ffn.prepare.keywords["step"].func, - comm_ops._mlp_input_residual_into_sum, - ) - self.assertIs( - steps.ffn_output_move, - comm.CommunicateSummableTensorPairFn._trivial, - ) - self.assertIs(steps.ffn_output.leaves_for_reduce_scatter, hands_on) - self.assertFalse(steps.ffn_output.leaves_for_next_layer) - def test_the_order_of_a_residual_on_the_slice_is_declared(self): # The same layouts take two orders: with a scattered input the residual # joins the attention output's sum; after a MoE on local rows it is @@ -1147,14 +892,13 @@ class TestInputScatteredAttention(CustomTestCase): (False, comm_ops._mlp_input_without_dp), ): with self.subTest(residual_joins_sum=joins): - steps, _, _ = comm_boundary._select_input_steps( + steps, _ = comm_boundary._select_input_steps( owed, residual=local, residual_to=attention, need=comm.StageInput(attention), adds_plainly=True, fusions=(), - force_layernorm_before_gather=False, residual_joins_sum=joins, cp_moves=None, enters_stack=False, @@ -1167,14 +911,13 @@ class TestInputScatteredAttention(CustomTestCase): sizes = self.SIZES attention = comm.Layout.sharded_over(axis_sizes=sizes) local = comm.Layout(frozenset({TokenAxis.ATTN_TP_SCATTER})) - step, _, _ = comm_boundary._select_input_steps( + step, _ = comm_boundary._select_input_steps( comm.StageOutput(attention), residual=attention, residual_to=local, need=comm.StageInput(local), adds_plainly=True, fusions=(), - force_layernorm_before_gather=False, residual_joins_sum=False, cp_moves=None, enters_stack=False, @@ -1213,7 +956,9 @@ class TestInputScatteredAttention(CustomTestCase): a2a=a2a, ) self.assertIs( - communicator.ffn.plan._input_scattered_steps is not None, expected + communicator.ffn.plan._paths.get(BatchVariant.INPUT_SCATTERED) + is not None, + expected, ) def test_a_batch_runs_them_only_while_its_input_is_scattered(self): @@ -1229,14 +974,19 @@ class TestInputScatteredAttention(CustomTestCase): lambda: SimpleNamespace(input_scattered=scattered), ), patch_communicator( - "get_forward", lambda: SimpleNamespace(sp_active=False) + "get_forward", + lambda: SimpleNamespace( + sp_active=False, attn_input_scattered=False + ), ), ): self.assertIs( - communicator.ffn.plan._batch_steps(None), - communicator.ffn.plan._input_scattered_steps + communicator.ffn.plan._batch_steps( + SimpleNamespace(forward_mode=ForwardMode.DECODE) + ), + communicator.ffn.plan._paths.get(BatchVariant.INPUT_SCATTERED) if scattered - else communicator.ffn.plan._steps, + else communicator.ffn.plan._paths.get(BatchVariant.ORDINARY), ) def test_prepare_attn_completes_the_scattered_input(self): @@ -1257,10 +1007,16 @@ class TestInputScatteredAttention(CustomTestCase): "get_attn_tp_context", lambda: SimpleNamespace(input_scattered=True), ), - patch_communicator("get_forward", lambda: SimpleNamespace(sp_active=False)), + patch_communicator( + "get_forward", + lambda: SimpleNamespace(sp_active=False, attn_input_scattered=False), + ), ): - hidden, residual = communicator.attn._prepare_input( - torch.ones(4, HIDDEN), torch.full((4, HIDDEN), 3.0), None + hidden, residual = prepare_input( + communicator.attn, + torch.ones(4, HIDDEN), + torch.full((4, HIDDEN), 3.0), + None, ) self.assertEqual(scattered, [4]) # Norm: (2 * (h + r), h + r) on the slice, with h the completed sum. @@ -1284,54 +1040,8 @@ class TestInputScatteredAttention(CustomTestCase): parallel, boundary_reduction="rs+rsv" if allow else "ar", ) - steps = communicator.ffn.plan._input_scattered_steps - self.assertIs(steps.ffn_output.leaves_for_reduce_scatter, hands_on) - - -class TestTheAttentionInputHalf(CustomTestCase): - """Every batch variant's half into the attention tries the fused entries - the layer chose, and starts the residual only on the stack's first - layer.""" - - def build_fusable(self, facts, parallel, **planning_kwargs): - with planning(parallel, **planning_kwargs): - return make_test_stages( - **facts, attention_norm=FusableNorm(), ffn_norm=Norm() - ) - - def test_every_variant_takes_the_layers_fused_entries(self): - cp = parallel_of(attn_dp=1, attn_tp=2, attn_cp=2, enable_prefill_cp=True) - for name, parallel, kwargs in ( - ( - "input-scattered", - parallel_of(attn_dp=1, attn_tp=2, enable_attn_tp_input_scattered=True), - {}, - ), - ("LayerNorm SP", parallel_of(attn_dp=1, attn_tp=2), {"sp": True}), - ("prefill CP", cp, {}), - ): - for layer_id in (0, 1): - with self.subTest(name, layer_id=layer_id): - facts = layer_case(layer_id, 3, sparse=False, previous_sparse=False) - communicator = self.build_fusable(facts, parallel, **kwargs) - fusions = communicator.attn.plan._carried_fusions - self.assertTrue(fusions) - variants = [ - steps - for steps in ( - communicator.attn.plan._steps, - communicator.attn.plan._sp_steps, - communicator.attn.plan._input_scattered_steps, - communicator.attn.plan._cp_steps, - ) - if steps is not None - ] - self.assertEqual(len(variants), 2) - for steps in variants: - prepare = steps.attention.prepare - self.assertIs(prepare.keywords["carried_fusions"], fusions) - bound = prepare.keywords["step"].keywords - self.assertIs(bound["enters_stack"], layer_id == 0) + steps = communicator.ffn.plan._paths.get(BatchVariant.INPUT_SCATTERED) + self.assertIs(steps.output.leaves_for_reduce_scatter, hands_on) class TestPrefillCP(CustomTestCase): @@ -1360,30 +1070,40 @@ class TestPrefillCP(CustomTestCase): parallel = self.dsa_parallel() facts = layer_case(1, 3, sparse=True, previous_sparse=False) communicator = build(facts, parallel, dsa_cp=True) - cp, ordinary = communicator.ffn.plan._cp_steps, communicator.ffn.plan._steps + cp, ordinary = ( + communicator.ffn.plan._paths.get(BatchVariant.CONTEXT_PARALLEL), + communicator.ffn.plan._paths.get(BatchVariant.ORDINARY), + ) self.assertIs( - cp.ffn.prepare.keywords["step"].func, + cp.entry.prepare.keywords["step"].func, comm_ops._mlp_input_gather_attention_cp, ) self.assertIs( - cp.ffn.prepare.keywords["step"].keywords["gather"].func, + cp.entry.prepare.keywords["step"].keywords["gather"].func, comm_ops._read_input, ) self.assertIs( - cp.ffn_output_move, - comm.CommunicateSummableTensorPairFn._reduce_scatter_over_cp, + cp.output_move, + reduce_scatter_over_cp, ) - self.assertTrue(cp.ffn_output.leaves_for_reduce_scatter) - self.assertFalse(cp.ffn_output.leaves_for_next_layer) + self.assertTrue(cp.output.leaves_for_reduce_scatter) + self.assertFalse(cp.output.leaves_for_next_layer) # The dense layer before it ran on the same shard: nothing to move. - self.assertIsNone(communicator.attn.plan._cp_steps.attention.input_move) - # Other batches hold every token on each CP rank: the MoE sums itself. - self.assertIs(ordinary.ffn.prepare.keywords["step"].func, comm_ops._read_input) - self.assertIs( - ordinary.ffn_output_move, comm.CommunicateSummableTensorPairFn._trivial + self.assertIsNone( + communicator.attn.plan._paths.get( + BatchVariant.CONTEXT_PARALLEL + ).entry.input_move ) + # Other batches hold every token on each CP rank: the MoE sums itself. + self.assertIs( + ordinary.entry.prepare.keywords["step"].func, comm_ops._read_input + ) + self.assertIs(ordinary.output_move, identity_output) with ( - patch_communicator("get_forward", lambda: SimpleNamespace(sp_active=False)), + patch_communicator( + "get_forward", + lambda: SimpleNamespace(sp_active=False, attn_input_scattered=False), + ), patch_communicator( "get_attn_tp_context", lambda: SimpleNamespace(input_scattered=False), @@ -1396,10 +1116,8 @@ class TestPrefillCP(CustomTestCase): ): self.assertIs( communicator.ffn.plan.output._ffn_leaves_sum_to_reduce_scatter( - SimpleNamespace( - forward_mode=SimpleNamespace( - is_context_parallel_extend=lambda: False - ) + communicator.ffn.plan.output.plan._batch_steps( + SimpleNamespace(forward_mode=ForwardMode.DECODE) ), None, ), @@ -1433,38 +1151,43 @@ class TestPrefillCP(CustomTestCase): parallel = self.dsa_parallel() facts = layer_case(1, 3, sparse=False, previous_sparse=False) communicator = build(facts, parallel, dsa_cp=True) - for steps in (communicator.ffn.plan._steps, communicator.ffn.plan._cp_steps): - self.assertIs(steps.ffn.prepare.keywords["step"].func, comm_ops._read_input) - self.assertIsNone(steps.ffn_output.group) + for steps in ( + communicator.ffn.plan._paths.get(BatchVariant.ORDINARY), + communicator.ffn.plan._paths.get(BatchVariant.CONTEXT_PARALLEL), + ): self.assertIs( - steps.ffn_output_move, comm.CommunicateSummableTensorPairFn._trivial + steps.entry.prepare.keywords["step"].func, comm_ops._read_input ) + self.assertIsNone(steps.output.group) + self.assertIs(steps.output_move, identity_output) def test_a_cp_extend_gathers_over_cp_and_takes_its_chunk_back(self): communicator = build(layer_case(1, 3), self.cp_parallel()) - cp = communicator.ffn.plan._cp_steps + cp = communicator.ffn.plan._paths.get(BatchVariant.CONTEXT_PARALLEL) self.assertIs( - cp.ffn.prepare.keywords["step"].func, comm_ops._mlp_input_gather_moe_cp + cp.entry.prepare.keywords["step"].func, comm_ops._mlp_input_gather_moe_cp ) # Each rank completes its own chunk before the gather. self.assertIs( - cp.ffn.prepare.keywords["step"].keywords["gather"].func, + cp.entry.prepare.keywords["step"].keywords["gather"].func, comm_ops._mlp_input_without_dp, ) self.assertIs( - cp.ffn_output_move, - comm.CommunicateSummableTensorPairFn._scatter_hidden_states_moe, + cp.output_move, + scatter_moe_cp_output, ) self.assertFalse(cp.returns_over_dp) - self.assertFalse(cp.ffn_output.leaves_for_next_layer) - self.assertFalse(cp.ffn_output.leaves_for_reduce_scatter) + self.assertFalse(cp.output.leaves_for_next_layer) + self.assertFalse(cp.output.leaves_for_reduce_scatter) self.assertIs( - communicator.ffn.plan._steps.ffn.prepare.keywords["step"].func, + communicator.ffn.plan._paths.get(BatchVariant.ORDINARY) + .entry.prepare.keywords["step"] + .func, comm_ops._mlp_input_without_dp, ) self.assertIs( - communicator.ffn.plan._steps.ffn_output_move, - comm.CommunicateSummableTensorPairFn._trivial, + communicator.ffn.plan._paths.get(BatchVariant.ORDINARY).output_move, + identity_output, ) def test_a_moe_on_its_own_cp_shard_hands_on_its_sum(self): @@ -1474,30 +1197,40 @@ class TestPrefillCP(CustomTestCase): layer_case(1, 3, sparse=True, previous_sparse=True), parallel, ) - for steps in (communicator.ffn.plan._cp_steps, communicator.ffn.plan._steps): + for steps in ( + communicator.ffn.plan._paths.get(BatchVariant.CONTEXT_PARALLEL), + communicator.ffn.plan._paths.get(BatchVariant.ORDINARY), + ): # The attention's rows are the MoE's: nothing to gather or take back. self.assertIs( - steps.ffn.prepare.keywords["step"].func, comm_ops._mlp_input_without_dp + steps.entry.prepare.keywords["step"].func, + comm_ops._mlp_input_without_dp, ) - self.assertIs( - steps.ffn_output_move, comm.CommunicateSummableTensorPairFn._trivial - ) - self.assertTrue(steps.ffn_output.leaves_for_next_layer) + self.assertIs(steps.output_move, identity_output) + self.assertTrue(steps.output.leaves_for_next_layer) dense = build(layer_case(1, 3), parallel) self.assertIs( - dense.ffn.plan._cp_steps.ffn.prepare.keywords["step"].func, + dense.ffn.plan._paths.get(BatchVariant.CONTEXT_PARALLEL) + .entry.prepare.keywords["step"] + .func, comm_ops._mlp_input_gather_moe_cp, ) - self.assertFalse(dense.ffn.plan._cp_steps.ffn_output.leaves_for_next_layer) + self.assertFalse( + dense.ffn.plan._paths.get( + BatchVariant.CONTEXT_PARALLEL + ).output.leaves_for_next_layer + ) def test_the_fused_kernels_run_on_each_chunk(self): with planning(self.cp_parallel()): communicator = make_test_stages( **layer_case(1, 3), attention_norm=FusableNorm(), ffn_norm=FusableNorm() ) - chunk = communicator.ffn.plan._cp_steps.ffn.prepare.keywords["step"].keywords[ - "gather" - ] + chunk = ( + communicator.ffn.plan._paths.get(BatchVariant.CONTEXT_PARALLEL) + .entry.prepare.keywords["step"] + .keywords["gather"] + ) self.assertEqual(len(chunk.keywords["fusions"]), 1) self.assertIs(chunk.keywords["fusions"][0].func, complete_ffn_input) self.assertEqual(chunk.keywords["fusions"][0].args, (communicator.ffn.plan,)) @@ -1510,7 +1243,11 @@ class TestPrefillCP(CustomTestCase): layer = build(facts, parallel, a2a=a2a, dsa_cp=dsa_cp) except NotImplementedError: return "refused" - return "ordinary" if layer.ffn.plan._cp_steps is None else "cp steps" + return ( + "ordinary" + if layer.ffn.plan._paths.get(BatchVariant.CONTEXT_PARALLEL) is None + else "cp steps" + ) def test_which_cp_the_steps_cover(self): dp_cp = dict(attn_dp=2, attn_tp=1, attn_cp=2, enable_prefill_cp=True) @@ -1587,25 +1324,26 @@ class TestPrefillCP(CustomTestCase): def test_under_attention_dp_one_dp_sum_gathers_both_axes(self): parallel = parallel_of(attn_dp=2, attn_tp=2, attn_cp=2, enable_prefill_cp=True) communicator = build(layer_case(1, 3), parallel) - cp, ordinary = communicator.ffn.plan._cp_steps, communicator.ffn.plan._steps - self.assertIs( - cp.ffn.prepare.keywords["step"].func, comm_ops._mlp_input_dp_partial - ) - self.assertTrue(cp.ffn.prepare.keywords["step"].keywords["places_cp_shards"]) - self.assertIs( - cp.ffn_output_move, comm.CommunicateSummableTensorPairFn._take_back_cp_shard + cp, ordinary = ( + communicator.ffn.plan._paths.get(BatchVariant.CONTEXT_PARALLEL), + communicator.ffn.plan._paths.get(BatchVariant.ORDINARY), ) self.assertIs( - ordinary.ffn.prepare.keywords["step"].func, comm_ops._mlp_input_dp_partial + cp.entry.prepare.keywords["step"].func, comm_ops._mlp_input_dp_partial + ) + self.assertTrue(cp.entry.prepare.keywords["step"].keywords["places_cp_shards"]) + self.assertIs(cp.output_move, take_back_cp_shard) + self.assertIs( + ordinary.entry.prepare.keywords["step"].func, comm_ops._mlp_input_dp_partial ) self.assertFalse( - ordinary.ffn.prepare.keywords["step"].keywords["places_cp_shards"] + ordinary.entry.prepare.keywords["step"].keywords["places_cp_shards"] ) self.assertTrue(ordinary.returns_over_dp) for steps in (cp, ordinary): - self.assertFalse(steps.ffn_output.leaves_for_next_layer) - self.assertFalse(steps.ffn_output.leaves_for_reduce_scatter) - self.assertFalse(steps.ffn_output.leaves_for_reduce_scatterv) + self.assertFalse(steps.output.leaves_for_next_layer) + self.assertFalse(steps.output.leaves_for_reduce_scatter) + self.assertFalse(steps.output.leaves_for_reduce_scatterv) def test_a_batch_runs_them_only_on_a_cp_extend(self): communicator = build(layer_case(1, 3), self.cp_parallel()) @@ -1621,16 +1359,34 @@ class TestPrefillCP(CustomTestCase): raise AssertionError("read for a batch that is not a CP extend") for fb, rows, expected in ( - (batch(False), unread, communicator.ffn.plan._steps), - (batch(True), lambda fb: None, communicator.ffn.plan._steps), - (batch(True), lambda fb: [2, 1], communicator.ffn.plan._cp_steps), + ( + batch(False), + unread, + communicator.ffn.plan._paths.get(BatchVariant.ORDINARY), + ), + ( + batch(True), + lambda fb: None, + communicator.ffn.plan._paths.get(BatchVariant.ORDINARY), + ), + ( + batch(True), + lambda fb: [2, 1], + communicator.ffn.plan._paths.get(BatchVariant.CONTEXT_PARALLEL), + ), ): with ( - self.subTest(expected=expected is communicator.ffn.plan._cp_steps), + self.subTest( + expected=expected + is communicator.ffn.plan._paths.get(BatchVariant.CONTEXT_PARALLEL) + ), planning(self.cp_parallel()), patch_communicator("moe_cp_gathered_rows", rows), patch_communicator( - "get_forward", lambda: SimpleNamespace(sp_active=False) + "get_forward", + lambda: SimpleNamespace( + sp_active=False, attn_input_scattered=False + ), ), patch_communicator( "get_attn_tp_context", @@ -1650,45 +1406,6 @@ class TestBranchRows(CustomTestCase): attention = comm.Layout(frozenset({TokenAxis.ATTN_DP})) full = comm.Layout(frozenset()) - def rows(self, communicator, parallel, cp_extend=False): - batch = SimpleNamespace( - forward_mode=SimpleNamespace(is_context_parallel_extend=lambda: cp_extend) - ) - with ( - planning(parallel), - patch_communicator("get_forward", lambda: SimpleNamespace(sp_active=False)), - patch_communicator( - "get_attn_tp_context", - lambda: SimpleNamespace(input_scattered=False), - ), - patch_communicator("moe_cp_gathered_rows", lambda fb: [2, 1]), - ): - return communicator.ffn.plan._branch_rows(batch) - - def test_each_branch_declares_its_rows(self): - parallel = parallel_of(attn_dp=2, attn_tp=2) - dense = build(layer_case(2, 6), parallel) - self.assertEqual( - self.rows(dense, parallel), (self.full, self.attention, self.attention) - ) - for a2a, expected in ( - (True, (self.local, self.local, self.local)), - (False, (self.full, self.attention, self.attention)), - ): - with self.subTest(a2a=a2a): - moe = build( - layer_case(1, 3, sparse=True, previous_sparse=True), - parallel, - a2a=a2a, - ) - self.assertEqual(self.rows(moe, parallel), expected) - - def test_a_batch_on_other_steps_has_no_branch_rows(self): - parallel = parallel_of(attn_dp=1, attn_tp=2, attn_cp=2, enable_prefill_cp=True) - communicator = build(layer_case(1, 3), parallel) - with self.assertRaises(NotImplementedError): - self.rows(communicator, parallel, cp_extend=True) - def test_branches_move_between_the_declared_rows(self): # An a2a MoE beside a dense FFN, under attention DP and TP. class Communicator(SimpleNamespace): @@ -1707,7 +1424,7 @@ class TestBranchRows(CustomTestCase): return value with patch_communicator("move_rows", recorded): - dense.branch_input(moe, "h0", "residual", None) + dense.branch_input(moe, "h0", ResidualStream("residual"), None) self.assertEqual( moves, [ @@ -1719,7 +1436,9 @@ class TestBranchRows(CustomTestCase): moe.branch_output("shortcut", None) self.assertEqual(moves, [("shortcut", self.local, self.local)]) moves.clear() - merged = moe.merge_branch(1, 2, "residual", dense, None) + stream = ResidualStream("residual") + hidden = stream.leave(2, comm.ADD) + merged = moe.merge_branch(1, hidden, stream, dense, None) self.assertEqual( moves, [ @@ -1728,7 +1447,9 @@ class TestBranchRows(CustomTestCase): ], ) # The contribution adds; the residual is the dense branch's. - self.assertEqual(merged, (3, "residual")) + self.assertEqual(merged[0], 3) + self.assertIs(merged[1], stream) + self.assertEqual(stream.residual, "residual") def test_a_value_is_gathered_then_cut_in_order(self): # Stand-ins that change the rows distinctly, so the order of the moves shows. @@ -2083,7 +1804,6 @@ class TestTwoLayers(CustomTestCase): ) def forward(rank): - batch = SimpleNamespace(**vars(forward_batch)) s = states[rank] layers = [ make_test_stages( @@ -2100,15 +1820,23 @@ class TestTwoLayers(CustomTestCase): residual = None handed_on = [] for layer_index, layer in enumerate(layers): - hidden, residual = layer.attn._prepare_input(hidden, residual, batch) + hidden, residual = prepare_input( + layer.attn, hidden, residual, forward_batch + ) hidden = attention(hidden, s) hidden = residual.leave( hidden, comm.ADD, - declared_sum=layer.ffn.plan._batch_steps(batch).ffn.input_sum, + declared_sum=layer.ffn.plan._batch_steps( + forward_batch + ).entry.input_sum, ) - hidden, residual = layer.ffn._prepare_input(hidden, residual, batch) - with layer.ffn.plan.output.ffn_exit(batch) as ffn_exit: + hidden, residual = prepare_input( + layer.ffn, hidden, residual, forward_batch + ) + with layer.ffn.plan.output.ffn_exit( + forward_batch, stream=ResidualStream() + ) as ffn_exit: if not sparse[layer_index]: hidden = dense_mlp(hidden, s) elif a2a: @@ -2116,10 +1844,9 @@ class TestTwoLayers(CustomTestCase): hidden = 5 * hidden else: hidden = moe(hidden, s) - hidden, residual = ffn_exit.finish(hidden, residual) + hidden, residual = finish_exit(ffn_exit, hidden, residual) handed_on.append(type(hidden)) - batch.residual_stream = residual - hidden, residual = finish_layer_stack(hidden, residual, batch) + hidden, residual = finish_layer_stack(hidden, residual, forward_batch) # The last a2a layer folds the residual into its output. if residual is not None: residual = residual[: s.rows] diff --git a/test/registered/unit/layers/test_dense_mlp_under_prefill_cp.py b/test/registered/unit/layers/test_dense_mlp_under_prefill_cp.py index 2bd60b96fb..666eeac93a 100644 --- a/test/registered/unit/layers/test_dense_mlp_under_prefill_cp.py +++ b/test/registered/unit/layers/test_dense_mlp_under_prefill_cp.py @@ -2,11 +2,12 @@ import unittest import test_declared_decoder_boundary as fixture -from sglang.srt.layers import communicator as comm from sglang.srt.layers.communicator import ( TokenAxis, ) -from sglang.srt.layers.communicator import ops as comm_ops +from sglang.srt.layers.communicator import prepare as comm_ops +from sglang.srt.layers.communicator.contracts import BatchVariant +from sglang.srt.layers.communicator.ops import scatter_moe_cp_output from sglang.test.boundary_fixtures import make_test_stages from sglang.test.ci.ci_register import register_cpu_ci from sglang.test.test_utils import CustomTestCase @@ -27,16 +28,16 @@ class TestDenseMlpUnderPrefillCP(CustomTestCase): stages = make_test_stages( attention_norm=fixture.Norm(), ffn_norm=fixture.Norm() ) - attention = stages.attn.plan._cp_steps.attention - steps = stages.ffn.plan._cp_steps + attention = stages.attn.plan._paths.get(BatchVariant.CONTEXT_PARALLEL).entry + steps = stages.ffn.plan._paths.get(BatchVariant.CONTEXT_PARALLEL) self.assertIn(TokenAxis.ATTN_CP, attention.input_rows.sharded) - self.assertNotIn(TokenAxis.ATTN_CP, steps.ffn.input_rows.sharded) + self.assertNotIn(TokenAxis.ATTN_CP, steps.entry.input_rows.sharded) self.assertIs( - steps.ffn.prepare.keywords["step"].func, comm_ops._mlp_input_gather_moe_cp + steps.entry.prepare.keywords["step"].func, comm_ops._mlp_input_gather_moe_cp ) self.assertIs( - steps.ffn_output_move, - comm.CommunicateSummableTensorPairFn._scatter_hidden_states_moe, + steps.output_move, + scatter_moe_cp_output, ) diff --git a/test/registered/unit/layers/test_dp_attention_cp_gather.py b/test/registered/unit/layers/test_dp_attention_cp_gather.py index e3a9741cd8..a8d97c72fb 100644 --- a/test/registered/unit/layers/test_dp_attention_cp_gather.py +++ b/test/registered/unit/layers/test_dp_attention_cp_gather.py @@ -1,5 +1,3 @@ -from sglang.test.boundary_fixtures import make_test_stages - """The dense-FFN gather and take-back under attention DP x CP x TP, rank by rank. Every rank builds its layer's communicator and runs its prepare_mlp and @@ -9,6 +7,7 @@ sum of the buffers the ranks hand to it. import unittest from contextlib import ExitStack, contextmanager, nullcontext +from dataclasses import replace from types import SimpleNamespace from unittest.mock import PropertyMock, patch @@ -16,12 +15,19 @@ import torch from sglang.srt.layers import communicator as comm from sglang.srt.layers import dp_attention, layernorm_sp -from sglang.srt.layers.communicator import ops as comm_ops +from sglang.srt.layers.communicator import prepare as comm_ops +from sglang.srt.layers.communicator.contracts import BatchVariant +from sglang.srt.layers.communicator.residual.add_norm import NORM_READ, PLAIN_RESIDUAL from sglang.srt.layers.cp import base as cp_base from sglang.srt.layers.cp import padding as cp_padding from sglang.srt.layers.cp.zigzag import ZigzagCPStrategy from sglang.srt.layers.dp_attention import DpPaddingMode from sglang.srt.model_executor.forward_batch_info import ForwardMode +from sglang.test.boundary_fixtures import ( + make_test_stages, + postprocess_output, + prepare_input, +) from sglang.test.ci.ci_register import register_cpu_ci from sglang.test.communicator_patch import patch_communicator from sglang.test.test_utils import CustomTestCase @@ -136,7 +142,7 @@ class TestDpCpGather(CustomTestCase): padding=DpPaddingMode.SUM_LEN, norm=layernorm, norm_rows=lambda rows: rows, - force_layernorm_before_dp_gather=False, + read=NORM_READ, ffn_input=None, sparse=False, ): @@ -263,7 +269,7 @@ class TestDpCpGather(CustomTestCase): next_sparse=False, attention_norm=norm, ffn_norm=norm, - force_layernorm_before_dp_gather=force_layernorm_before_dp_gather, + residual=PLAIN_RESIDUAL._replace(ffn_read=read), ) yield SimpleNamespace( communicator=communicator, @@ -283,14 +289,15 @@ class TestDpCpGather(CustomTestCase): def gather(rank, all_reduce): with as_rank(*rank, all_reduce) as r: self.assertIsNotNone( - r.communicator.ffn.plan._cp_steps, "declared under CP" + r.communicator.ffn.plan._paths.get(BatchVariant.CONTEXT_PARALLEL), + "declared under CP", ) if ffn_input is not None: steps = r.communicator.ffn.plan._batch_steps(r.forward_batch) - self.assertIs(steps.ffn.prepare.keywords["step"].func, ffn_input) + self.assertIs(steps.entry.prepare.keywords["step"].func, ffn_input) hidden_states, residual = rank_inputs(*rank) - return r.communicator.ffn._prepare_input( - hidden_states, residual, r.forward_batch + return prepare_input( + r.communicator.ffn, hidden_states, residual, r.forward_batch ) # The all-reduce is a sum over every rank: record what each rank hands @@ -330,8 +337,11 @@ class TestDpCpGather(CustomTestCase): torch.testing.assert_close(hidden_states, expected, rtol=0, atol=0) torch.testing.assert_close(residual.residual[:tokens], own, rtol=0, atol=0) with as_rank(*rank, None) as r: - back, _ = r.communicator.ffn.plan.output.postprocess_layer( - 3 * hidden_states, residual, r.forward_batch + back, _ = postprocess_output( + r.communicator.ffn.plan.output, + 3 * hidden_states, + residual, + r.forward_batch, ) expected_back = torch.zeros(group.held_rows[cp], HIDDEN).double() expected_back[:tokens] = 3 * norm_rows(own) @@ -399,7 +409,7 @@ class TestDpCpGather(CustomTestCase): attn_tp_size=2, norm=rms_norm, norm_rows=rms_rows, - force_layernorm_before_dp_gather=True, + read=replace(NORM_READ, before_gather=True), ffn_input=comm_ops._mlp_input_dp_replicate, ) diff --git a/test/registered/unit/layers/test_independent_stage_construction.py b/test/registered/unit/layers/test_independent_stage_construction.py index 93b9f0f00d..c2b18d2468 100644 --- a/test/registered/unit/layers/test_independent_stage_construction.py +++ b/test/registered/unit/layers/test_independent_stage_construction.py @@ -2,27 +2,25 @@ import itertools import unittest +from dataclasses import replace from types import SimpleNamespace +from unittest.mock import patch import test_declared_decoder_boundary as fixture import torch from sglang.srt.layers.communicator import ( - EdgeDecl, - Layout, - StageDecl, - StageInput, + ProducerReduction, StageKind, - StageOutput, declare_attn, declare_ffn, make_attn_stage, make_ffn_stage, - make_stage, make_stages, ) +from sglang.srt.layers.communicator.contracts import BatchVariant from sglang.srt.layers.communicator.residual import batch -from sglang.srt.layers.communicator.residual.add_norm import ADD +from sglang.srt.model_executor.forward_batch_info import ForwardMode from sglang.test.ci.ci_register import register_cpu_ci from sglang.test.test_utils import CustomTestCase @@ -30,169 +28,44 @@ register_cpu_ci(est_time=5, suite="base-a-test-cpu") class TestIndependentStageConstruction(CustomTestCase): - def make(self, kind, *, first=False, update=ADD, read=None): - rows = Layout(frozenset()) - need = StageInput(rows) if read is None else StageInput(rows, read=read) - produced = StageOutput(rows, update=update) - return make_stage( - StageDecl(need, produced), - kind=kind, - norm=fixture.Norm(), - incoming=EdgeDecl(StageOutput(rows), need, rows, rows), - outgoing=EdgeDecl(produced, StageInput(rows), rows, rows), - enters_stack=first, - terminal=True, - fixed_output=kind is StageKind.ATTENTION, - ) + def test_active_variant_without_a_bound_path_never_falls_back(self): + from sglang.srt.layers.communicator import construction + from sglang.test.boundary_fixtures import stub_plan - def test_any_two_stage_kinds_share_the_same_construction(self): - for first_kind in StageKind: - for second_kind in StageKind: - with ( - self.subTest(first=first_kind, second=second_kind), - fixture.planning(fixture.parallel_of(attn_dp=1, attn_tp=1)), - ): - first = self.make(first_kind, first=True) - second = self.make(second_kind) - self.assertIsNot(first.plan, second.plan) - self.assertFalse(hasattr(first.plan, "layer_facts")) - fb = SimpleNamespace() - batch.start(fb) - hidden = torch.ones(2, 4) - for stage in (first, second): - hidden = stage.prepare(hidden, fb) - if stage.kind is StageKind.ATTENTION: - hidden = stage.finish(hidden, fb) - else: - with stage.exit(fb) as output: - pass - hidden = output.finish(hidden) - torch.testing.assert_close( - batch.snapshot(hidden, fb), torch.full((2, 4), 9.0) - ) - - def test_binding_order_does_not_mutate_the_other_stage(self): - with fixture.planning(fixture.parallel_of(attn_dp=1, attn_tp=1)): - attn = declare_attn(terminal=True) - ffn = declare_ffn(previous=attn, terminal=True) - for reverse in (False, True): - a = lambda: make_attn_stage( - declaration=attn, norm=fixture.Norm(), following=ffn - ) - f = lambda: make_ffn_stage(declaration=ffn, norm=fixture.Norm()) - first, second = (f(), a()) if reverse else (a(), f()) - self.assertIsNot(first.plan, second.plan) - self.assertIsNot(first.norm, second.norm) - self.assertFalse(hasattr(first.plan, "layer_facts")) - self.assertFalse(hasattr(first.plan, "previous")) - with self.assertRaises(TypeError): - declare_ffn(previous=first) - with self.assertRaisesRegex(ValueError, "following"): - make_attn_stage(declaration=attn, norm=None, following=declare_ffn()) - - def test_public_factories_connect_all_stage_pairs(self): - for first_kind in StageKind: - for second_kind in StageKind: - with ( - self.subTest(first=first_kind, second=second_kind), - fixture.planning(fixture.parallel_of(attn_dp=1, attn_tp=1)), - ): - declare = { - StageKind.ATTENTION: declare_attn, - StageKind.FFN: declare_ffn, - } - build = { - StageKind.ATTENTION: make_attn_stage, - StageKind.FFN: make_ffn_stage, - } - first = declare[first_kind]() - second = declare[second_kind](previous=first) - stages = [ - build[first_kind]( - declaration=first, norm=fixture.Norm(), following=second - ), - build[second_kind](declaration=second, norm=fixture.Norm()), - ] - fb = SimpleNamespace() - batch.start(fb) - hidden = torch.ones(2, 4) - for stage in stages: - hidden = stage.prepare(hidden, fb) - if stage.kind is StageKind.ATTENTION: - hidden = stage.finish(hidden, fb) - else: - with stage.exit(fb) as output: - pass - hidden = output.finish(hidden) - torch.testing.assert_close( - batch.snapshot(hidden, fb), torch.full((2, 4), 9.0) - ) - - def test_prepared_branch_reuses_the_same_placements(self): - from sglang.srt.layers.communicator.factories import ( - _connect, - _connections, - _fork_input, - ) - - for dp in (1, 2): - for tp in (1, 2): - for local_dense in (False, True): - parallel = fixture.parallel_of( - attn_dp=dp, - attn_tp=tp, - moe_dense_tp_size=1 if local_dense else None, - ) - with ( - self.subTest(dp=dp, tp=tp, local_dense=local_dense), - fixture.planning(parallel), - ): - attn = declare_attn(previous=declare_ffn(sparse=True)) - moe = declare_ffn(previous=attn, sparse=True) - dense = declare_ffn( - prepared_from=moe, - ordinary_only=True, - ) - branch_attn = declare_attn(previous=dense, ordinary_only=True) - last_dense = declare_ffn( - previous=branch_attn, - ordinary_only=True, - ) - _, into_moe = _connections(attn, moe) - fork = _fork_input(into_moe, dense) - middle = _connect(dense, branch_attn) - last = _connect(branch_attn, last_dense, residual_from=middle) - for declaration, following, expected in ( - (dense, branch_attn, fork), - (branch_attn, last_dense, middle), - (last_dense, None, last), - ): - make = ( - make_attn_stage - if declaration.kind is StageKind.ATTENTION - else make_ffn_stage - ) - stage = make( - declaration=declaration, - norm=fixture.Norm(), - following=following, - ) - self.assertEqual( - { - v: edge.incoming - for v, edge in stage.plan.variants.items() - }, - expected.entries, - ) - self.assertFalse(stage.plan.is_first_layer) - if declaration is dense: - with self.assertRaisesRegex( - RuntimeError, "branch_input" - ): - stage.plan._steps.ffn.prepare(None, None, None) - - with self.assertRaisesRegex(ValueError, "choose"): - declare_ffn(previous=attn, prepared_from=moe) + for variant in BatchVariant: + plan = stub_plan() + ordinary = object() + plan._paths[BatchVariant.ORDINARY] = ordinary + with ( + self.subTest(variant=variant), + patch.object( + construction, + "get_forward", + return_value=SimpleNamespace( + sp_active=variant is BatchVariant.SEQUENCE_PARALLEL + ), + ), + patch.object( + construction, + "get_attn_tp_context", + return_value=SimpleNamespace( + input_scattered=variant is BatchVariant.INPUT_SCATTERED + ), + ), + patch.object( + construction, + "_batch_shards_over_cp", + return_value=variant is BatchVariant.CONTEXT_PARALLEL, + ), + ): + if variant is BatchVariant.ORDINARY: + self.assertIs(plan._batch_steps(None), ordinary) + else: + with self.assertRaisesRegex(NotImplementedError, variant.name): + plan._batch_steps(None) + selected = object() + plan._paths[variant] = selected + self.assertIs(plan._batch_steps(None), selected) def test_sequences_of_one_to_four_stages(self): for count in range(1, 5): @@ -209,36 +82,11 @@ class TestIndependentStageConstruction(CustomTestCase): *((d, fixture.Norm()) for d in declarations), terminal=True ) self.assertEqual(len(boundaries), count) - fb = SimpleNamespace() + fb = SimpleNamespace(forward_mode=ForwardMode.DECODE) batch.start(fb) hidden = torch.ones(2, 4) - # The direct factories provide an independent construction - # reference, including repeated kinds and terminal placement. - from dataclasses import replace - previous = None - linked = [] - for i, d in enumerate(declarations): - previous = replace( - d, previous=previous, terminal=i == count - 1 - ) - linked.append(previous) - for i, boundary in enumerate(boundaries): - build = ( - make_attn_stage - if kinds[i] is StageKind.ATTENTION - else make_ffn_stage - ) - reference = build( - declaration=linked[i], - norm=boundary.norm, - following=linked[i + 1] if i + 1 < count else None, - ) - self.assertEqual( - boundary.plan.variants, reference.plan.variants - ) - self.assertEqual(boundary.plan.is_last_layer, i == count - 1) - self.assertIsNone(declarations[i].previous) + for boundary in boundaries: hidden = boundary.prepare(hidden, fb) if boundary.kind is StageKind.ATTENTION: hidden = boundary.finish(hidden, fb) @@ -252,6 +100,60 @@ class TestIndependentStageConstruction(CustomTestCase): batch.snapshot(hidden, fb), torch.full((2, 4), float(3**count)) ) + def test_sequence_and_independent_factories_agree_on_terminal_placement(self): + for dp, tp, terminal in itertools.product((1, 2), (1, 2), (False, True)): + for kinds in itertools.product(StageKind, repeat=3): + with ( + self.subTest(dp=dp, tp=tp, terminal=terminal, kinds=kinds), + fixture.planning(fixture.parallel_of(attn_dp=dp, attn_tp=tp)), + ): + declarations = [ + declare_attn() if kind is StageKind.ATTENTION else declare_ffn() + for kind in kinds + ] + sequence = make_stages( + *( + (declaration, fixture.Norm()) + for declaration in declarations + ), + terminal=terminal, + ) + linked = [] + for index, declaration in enumerate(declarations): + linked.append( + replace( + declaration, + previous=linked[-1] if linked else None, + terminal=terminal and index == len(declarations) - 1, + ) + ) + for index, (declaration, combined) in enumerate( + zip(linked, sequence) + ): + make = ( + make_attn_stage + if declaration.kind is StageKind.ATTENTION + else make_ffn_stage + ) + independent = make( + declaration=declaration, + norm=fixture.Norm(), + following=linked[index + 1] + if index + 1 < len(linked) + else None, + ) + self.assertEqual(combined.plan.terminal, declaration.terminal) + self.assertEqual( + independent.plan.terminal, declaration.terminal + ) + self.assertEqual( + combined.plan.variants, independent.plan.variants + ) + for variant, steps in combined.plan._paths.items(): + self.assertEqual( + steps.output, independent.plan._paths[variant].output + ) + def test_sequence_branch_keeps_source_lineage(self): for dp, tp, local_dense in itertools.product((1, 2), (1, 2), (False, True)): with ( @@ -272,12 +174,12 @@ class TestIndependentStageConstruction(CustomTestCase): before = dict(source.plan.variants) first, attn, last = make_stages( ( - declare_ffn(ordinary_only=True), + declare_ffn(), fixture.Norm(), ), - (declare_attn(ordinary_only=True), fixture.Norm()), + (declare_attn(), fixture.Norm()), ( - declare_ffn(ordinary_only=True), + declare_ffn(), fixture.Norm(), ), prepared_from=source.declaration, @@ -292,21 +194,45 @@ class TestIndependentStageConstruction(CustomTestCase): edge.incoming.produced.layout, source_input.need.layout ) self.assertEqual(edge.incoming.residual, source_input.residual_to) - with self.assertRaisesRegex(RuntimeError, "branch_input"): - first.plan._steps.ffn.prepare(None, None, None) + + def test_a_replicated_tail_stays_in_compute_without_a_fusion_object(self): + for local_tail in (False, True): + with fixture.planning(fixture.parallel_of(attn_dp=1, attn_tp=2)): + boundary = make_ffn_stage( + declaration=declare_ffn( + previous=declare_attn(), + sparse=True, + reduction=ProducerReduction.LOCAL_TAIL + if local_tail + else ProducerReduction.SCOPED, + ), + norm=fixture.Norm(), + ) + self.assertIsNone(boundary.plan.fusions) + produced = boundary.plan._paths[BatchVariant.ORDINARY].output + self.assertEqual(produced.leaves_for_next_layer, not local_tail) + # A local compute tail does not change the existing RS alternative. + self.assertTrue(produced.leaves_for_reduce_scatter) def test_mixer_successor_comes_from_the_local_sequence(self): - for following in (declare_attn(mixer_exit=True), declare_ffn()): + for following in ( + declare_attn(reduction=ProducerReduction.SCOPED, gathers_tp_input=False), + declare_ffn(), + ): with ( self.subTest(kind=following.kind), fixture.planning(fixture.parallel_of(attn_dp=1, attn_tp=2)), ): mixer, consumer = make_stages( - (declare_attn(mixer_exit=True), fixture.Norm()), + ( + declare_attn( + reduction=ProducerReduction.SCOPED, gathers_tp_input=False + ), + fixture.Norm(), + ), (following, fixture.Norm()), terminal=True, ) - self.assertIs(mixer.declaration.next_kind, following.kind) for variant, edge in mixer.plan.variants.items(): output = edge.outgoing.produced self.assertEqual( @@ -319,33 +245,6 @@ class TestIndependentStageConstruction(CustomTestCase): incoming = consumer.plan.variants[variant].incoming.produced self.assertEqual(incoming.group, output.group) - def test_sequence_requires_one_source_and_external_terminal(self): - with self.assertRaisesRegex(ValueError, "at least one"): - make_stages() - with self.assertRaisesRegex(ValueError, "choose"): - make_stages( - (declare_attn(), None), - previous=declare_ffn(), - prepared_from=declare_ffn(), - ) - with self.assertRaisesRegex(ValueError, "input sources"): - make_stages((declare_attn(previous=declare_ffn()), None)) - with self.assertRaisesRegex(ValueError, "terminal"): - make_stages((declare_ffn(terminal=True), None)) - - def test_mismatched_declared_edge_is_rejected(self): - rows = Layout(frozenset()) - declaration = StageDecl(StageInput(rows), StageOutput(rows)) - wrong_input = StageInput(rows, read=object()) - with self.assertRaisesRegex(ValueError, "disagrees"): - make_stage( - declaration, - kind=StageKind.FFN, - norm=None, - incoming=EdgeDecl(StageOutput(rows), wrong_input, rows, rows), - outgoing=EdgeDecl(declaration.output, declaration.input, rows, rows), - ) - if __name__ == "__main__": unittest.main() diff --git a/test/registered/unit/layers/test_layer_communicator_fusion_gate.py b/test/registered/unit/layers/test_layer_communicator_fusion_gate.py index 32704ffd22..06bb9ecad7 100644 --- a/test/registered/unit/layers/test_layer_communicator_fusion_gate.py +++ b/test/registered/unit/layers/test_layer_communicator_fusion_gate.py @@ -12,16 +12,20 @@ from sglang.srt.layers.communicator import ( StageOutput, SumGroup, ) -from sglang.srt.layers.communicator import ops as comm_ops +from sglang.srt.layers.communicator import ops as transport_ops +from sglang.srt.layers.communicator import prepare as comm_ops +from sglang.srt.layers.communicator.contracts import BatchVariant from sglang.srt.layers.communicator.exit import OutputBoundary +from sglang.srt.layers.communicator.ops import identity_output from sglang.srt.layers.moe import ( can_merge_post_experts_all_reduce, deferred_post_experts_all_reduce, post_experts_all_reduce, ) from sglang.srt.layers.moe import utils as moe_utils +from sglang.srt.model_executor.forward_batch_info import ForwardMode from sglang.srt.runtime_context import get_forward, get_parallel -from sglang.test.boundary_fixtures import sp_region_steps, stub_plan +from sglang.test.boundary_fixtures import identity_input, sp_region_steps, stub_plan from sglang.test.ci.ci_register import register_cpu_ci from sglang.test.communicator_patch import patch_communicator from sglang.test.test_utils import CustomTestCase @@ -29,9 +33,9 @@ from sglang.test.test_utils import CustomTestCase register_cpu_ci(est_time=11, suite="base-a-test-cpu") -def _steps(*, ffn_output=None, returns_over_dp=False, ffn_sum_is_movable=True): - return comm.BoundarySteps( - attention=comm.StageEntry( +def _steps(*, output=None, returns_over_dp=False, ffn_sum_is_movable=True): + return comm.StageSteps( + entry=comm.StageEntry( prepare=partial( comm_ops._consumer_step, step=partial( @@ -44,24 +48,15 @@ def _steps(*, ffn_output=None, returns_over_dp=False, ffn_sum_is_movable=True): carried_fusions=(), ), input_rows=comm.Layout(frozenset()), - input_move=comm.CommunicateSimpleFn._trivial, + input_move=identity_input, handoff=comm_ops._hand_qkv_hook_its_input, ), - ffn=comm.StageEntry( - prepare=partial( - comm_ops._read_input, - layer_input=None, - enters_stack=False, - read=comm.NORM_READ, - update=comm.ADD, - ), - input_rows=Layout(frozenset()), + output=output + or StageOutput( + Layout(frozenset()), group=SumGroup.TP if ffn_sum_is_movable else None ), - ffn_output=ffn_output or StageOutput(Layout(frozenset()), group=SumGroup.TP), - ffn_output_move=None - if returns_over_dp - else comm.CommunicateSummableTensorPairFn._trivial, - ffn_sum_is_movable=ffn_sum_is_movable, + returns_over_dp=returns_over_dp, + output_move=None if returns_over_dp else identity_output, ) @@ -69,11 +64,13 @@ def _fake_communicator(ffn_sum_is_movable=True): communicator = stub_plan() communicator._speculative_algo = None # An FFN gathered over MoE-CP or on each rank's own rows has no movable sum. - communicator._steps = _steps(ffn_sum_is_movable=ffn_sum_is_movable) - communicator._sp_steps = None - communicator._input_scattered_steps = None - communicator._cp_steps = None - communicator.is_last_layer = False + communicator._paths[BatchVariant.ORDINARY] = _steps( + ffn_sum_is_movable=ffn_sum_is_movable + ) + communicator._paths[BatchVariant.SEQUENCE_PARALLEL] = None + communicator._paths[BatchVariant.INPUT_SCATTERED] = None + communicator._paths[BatchVariant.CONTEXT_PARALLEL] = None + communicator.terminal = False return communicator @@ -297,7 +294,7 @@ class TestFuseMlpAllReduceGate(CustomTestCase): ffn_sum_is_movable=True, ): forward_batch = types.SimpleNamespace( - input_ids=types.SimpleNamespace(shape=(8,)) + forward_mode=ForwardMode.DECODE, input_ids=types.SimpleNamespace(shape=(8,)) ) with ( patch_communicator("is_enable_moe_cp_allgather", return_value=False), @@ -315,7 +312,11 @@ class TestFuseMlpAllReduceGate(CustomTestCase): ): return OutputBoundary( _fake_communicator(ffn_sum_is_movable) - )._ffn_sum_can_move_to_next_layer(forward_batch) + )._ffn_sum_can_move_to_next_layer( + OutputBoundary( + _fake_communicator(ffn_sum_is_movable) + ).plan._batch_steps(forward_batch) + ) def test_hybrid_ep_tp_fuses_when_mergeable(self): self.assertTrue(self._should_fuse(moe_ep_size=2, moe_tp_size=2)) @@ -367,12 +368,14 @@ class TestDeferFfnReduction(CustomTestCase): ): tp_group_object = object() communicator = _fake_communicator() - communicator.is_last_layer = is_last_layer - communicator._sp_steps = sp_region_steps() if sp_active else None - communicator._input_scattered_steps = None - communicator._cp_steps = None - communicator._steps = _steps( - ffn_output=StageOutput( + communicator.terminal = is_last_layer + communicator._paths[BatchVariant.SEQUENCE_PARALLEL] = ( + sp_region_steps() if sp_active else None + ) + communicator._paths[BatchVariant.INPUT_SCATTERED] = None + communicator._paths[BatchVariant.CONTEXT_PARALLEL] = None + communicator._paths[BatchVariant.ORDINARY] = _steps( + output=StageOutput( Layout(frozenset()), group=SumGroup.MOE_OUTPUT, leaves_for_reduce_scatter=True, @@ -381,11 +384,10 @@ class TestDeferFfnReduction(CustomTestCase): returns_over_dp=scatters_to_local_tokens, ) forward_batch = types.SimpleNamespace( + forward_mode=ForwardMode.DECODE, input_ids=types.SimpleNamespace(shape=(batch_size,)), global_dp_buffer_len=global_tokens, - residual_stream=types.SimpleNamespace( - residual=torch.ones(max(batch_size, 1), 4) - ), + residual_stream=types.SimpleNamespace(residual=torch.ones(batch_size, 4)), ) with ( patch_communicator( @@ -395,6 +397,10 @@ class TestDeferFfnReduction(CustomTestCase): patch_communicator("is_enable_moe_cp_allgather", return_value=False), patch_communicator("apply_flashinfer_allreduce_fusion", return_value=fused), patch_communicator("_use_aiter", False), + patch_communicator( + "post_experts_reduction_group", + return_value=tp_group_object if tp_group else object(), + ), patch_communicator( "get_lora", return_value=types.SimpleNamespace(enable_lora=lora) ), @@ -406,10 +412,6 @@ class TestDeferFfnReduction(CustomTestCase): ) ), ), - patch_communicator( - "post_experts_reduction_group", - return_value=tp_group_object if tp_group else object(), - ), patch_communicator( "get_attn_tp_context", return_value=types.SimpleNamespace(input_scattered=False), @@ -470,7 +472,10 @@ class TestDeferFfnReduction(CustomTestCase): ), ): return communicator.output._ffn_sum_moves_to_next_layer( - forward_batch, mlp_reduce_scatter=reduce_scatter, dp_step=step + forward_batch, + communicator.output.plan._batch_steps(forward_batch), + mlp_reduce_scatter=reduce_scatter, + dp_step=step, ) def test_defers_whether_or_not_the_fused_kernel_takes_the_batch(self): @@ -512,11 +517,11 @@ class TestDeferFfnReductionUnderAttentionDp(CustomTestCase): for name, condition in ( ( "reduce-scatter", - dict(step=comm_ops._reduce_and_redistribute_output_varlen), + dict(step=transport_ops._reduce_and_redistribute_output_varlen), ), ( "MAX_LEN reduce-scatter", - dict(step=comm_ops._reduce_and_redistribute_output_max_len), + dict(step=transport_ops._reduce_and_redistribute_output_max_len), ), ("other postprocess", dict(scatters_to_local_tokens=False)), ("LayerNorm SP", dict(sp_active=True)), diff --git a/test/registered/unit/layers/test_layernorm_sp.py b/test/registered/unit/layers/test_layernorm_sp.py index 41331739f9..6eea228167 100644 --- a/test/registered/unit/layers/test_layernorm_sp.py +++ b/test/registered/unit/layers/test_layernorm_sp.py @@ -1,6 +1,3 @@ -from sglang.srt.layers.communicator import StageKind -from sglang.test.boundary_fixtures import sp_region_steps, stub_plan, stub_stage - """Unit tests for srt/layers/layernorm_sp (Megatron LayerNorm sequence parallelism). Covers the pure logic that gates SP -- the Qwen3 allowlist, the config guards, and @@ -20,10 +17,12 @@ from sglang.srt.layers import communicator as comm from sglang.srt.layers import layernorm_sp from sglang.srt.layers.communicator import ( Layout, + StageKind, StageOutput, SumGroup, ) -from sglang.srt.layers.communicator import ops as comm_ops +from sglang.srt.layers.communicator import prepare as comm_ops +from sglang.srt.layers.communicator.contracts import BatchVariant from sglang.srt.model_executor.forward_batch_info import ForwardMode from sglang.srt.runtime_context import ( get_flags, @@ -32,6 +31,13 @@ from sglang.srt.runtime_context import ( reset_context, ) from sglang.srt.server_args import ServerArgs +from sglang.test.boundary_fixtures import ( + postprocess_output, + prepare_input, + sp_region_steps, + stub_plan, + stub_stage, +) from sglang.test.ci.ci_register import register_cpu_ci from sglang.test.communicator_patch import patch_communicator from sglang.test.test_utils import CustomTestCase @@ -121,29 +127,6 @@ class TestLayerNormSPValidation(CustomTestCase): validate_layernorm_sp(**{**self.VALID, "speculative_algorithm": "EAGLE3"}) -class TestLayerNormSPActiveFlag(CustomTestCase): - def tearDown(self): - reset_context() - - def test_active_flag_is_registered_on_forward_flags(self): - """``sp_active`` must stay a registered ForwardFlags slot. - - ``set()`` rejects names missing from ``ForwardFlags._DEFAULTS``, so this - fails if the slot is dropped, and it must also be in ``_GRAPH_VISIBLE`` - because the participant linears read it under CUDA graph capture. - """ - from sglang.srt.runtime_context import ForwardFlags - - self.assertIn("sp_active", ForwardFlags._DEFAULTS) - self.assertIn("sp_active", ForwardFlags._GRAPH_VISIBLE) - - self.assertFalse(get_forward().sp_active) # default - get_forward().set("sp_active", True) - self.assertTrue(get_forward().sp_active) - get_forward().set("sp_active", False) - self.assertFalse(get_forward().sp_active) - - class _Norm: def __call__(self, x, residual=None, post_residual_addition=None): if residual is None: @@ -157,23 +140,34 @@ class TestSpRegionSteps(CustomTestCase): prefill forward and, inside it, keeps every boundary on this rank's sequence shard: add + norm, no move, no collective.""" + def test_sp_flag_remains_visible_to_traced_linears(self): + from sglang.srt.runtime_context import ForwardFlags + + self.assertIn("sp_active", ForwardFlags._GRAPH_VISIBLE) + before = get_forward().sp_active + with get_forward().scoped(sp_active=not before): + self.assertEqual(get_forward().sp_active, not before) + self.assertEqual(get_forward().sp_active, before) + def communicator(self, *, first_layer): c = stub_plan() - c._sp_steps = sp_region_steps() - c._input_scattered_steps = None - c._cp_steps = None - c.is_first_layer = first_layer + c._paths[BatchVariant.SEQUENCE_PARALLEL] = sp_region_steps() + c._paths[BatchVariant.INPUT_SCATTERED] = None + c._paths[BatchVariant.CONTEXT_PARALLEL] = None + c.enters_stack = first_layer c.is_sparse = False c._attn_input_fusions = () c.norm = _Norm() c.qkv_latent_func = None - c._steps = self.ordinary_steps(MagicMock(side_effect=AssertionError("moved"))) + c._paths[BatchVariant.ORDINARY] = self.ordinary_steps( + MagicMock(side_effect=AssertionError("moved")) + ) return c def ordinary_steps(self, attention_input): """The layer's steps outside the region, which must not run inside it.""" - return comm.BoundarySteps( - attention=comm.StageEntry( + return comm.StageSteps( + entry=comm.StageEntry( prepare=partial( comm_ops._consumer_step, step=partial( @@ -190,19 +184,15 @@ class TestSpRegionSteps(CustomTestCase): input_move=attention_input, handoff=comm_ops._hand_qkv_hook_its_input, ), - ffn=comm.StageEntry( - prepare=MagicMock(side_effect=AssertionError("ordinary FFN input ran")), - input_rows=Layout(frozenset()), - ), - ffn_output=StageOutput(Layout(frozenset()), group=SumGroup.TP), - ffn_output_move=MagicMock(side_effect=AssertionError("postprocess ran")), - ffn_sum_is_movable=True, + output=StageOutput(Layout(frozenset()), group=SumGroup.TP), + output_move=MagicMock(side_effect=AssertionError("postprocess ran")), ) def run_prepare_attn(self, communicator, mode, hidden, residual): batch = SimpleNamespace(forward_mode=mode) with ( get_flags().sp.override(enabled=True), + patch_communicator("_batch_shards_over_cp", return_value=False), get_forward().scoped(sp_active=False), patch_communicator( "get_attn_tp_context", @@ -212,8 +202,8 @@ class TestSpRegionSteps(CustomTestCase): layernorm_sp, "sp_entry_scatter", side_effect=lambda h: h[:1] ) as scatter, ): - out = stub_stage(communicator, StageKind.ATTENTION)._prepare_input( - hidden, residual, batch + out = prepare_input( + stub_stage(communicator, StageKind.ATTENTION), hidden, residual, batch ) return out, get_forward().sp_active, scatter @@ -230,7 +220,7 @@ class TestSpRegionSteps(CustomTestCase): def test_decode_leaves_the_region_closed(self): communicator = self.communicator(first_layer=True) move = MagicMock(side_effect=lambda **k: k["hidden_states"]) - communicator._steps = self.ordinary_steps(move) + communicator._paths[BatchVariant.ORDINARY] = self.ordinary_steps(move) (h, _), active, scatter = self.run_prepare_attn( communicator, ForwardMode.DECODE, torch.ones(2, 4), None ) @@ -240,7 +230,7 @@ class TestSpRegionSteps(CustomTestCase): def test_a_later_layer_does_not_reopen_the_region(self): communicator = self.communicator(first_layer=False) - communicator._steps = self.ordinary_steps( + communicator._paths[BatchVariant.ORDINARY] = self.ordinary_steps( MagicMock(side_effect=lambda **k: k["hidden_states"]) ) _, active, scatter = self.run_prepare_attn( @@ -254,24 +244,29 @@ class TestSpRegionSteps(CustomTestCase): hidden, residual = torch.ones(1, 4), torch.full((1, 4), 3.0) with ( get_flags().sp.override(enabled=True), + patch_communicator("_batch_shards_over_cp", return_value=False), get_forward().scoped(sp_active=True), ): with patch_communicator( "get_attn_tp_context", return_value=SimpleNamespace(input_scattered=False), ): - h, r = stub_stage(communicator, StageKind.ATTENTION)._prepare_input( + h, r = prepare_input( + stub_stage(communicator, StageKind.ATTENTION), hidden, residual.clone(), SimpleNamespace(forward_mode=ForwardMode.EXTEND), ) torch.testing.assert_close(h, torch.full((1, 4), 8.0)) - h, r = stub_stage(communicator, StageKind.FFN)._prepare_input( - hidden, residual.clone(), object() + h, r = prepare_input( + stub_stage(communicator, StageKind.FFN), + hidden, + residual.clone(), + object(), ) torch.testing.assert_close(h, torch.full((1, 4), 8.0)) torch.testing.assert_close(r.residual, torch.full((1, 4), 4.0)) - out = communicator.output.postprocess_layer(hidden, residual, object()) + out = postprocess_output(communicator.output, hidden, residual, object()) self.assertIs(out[0], hidden) self.assertIs(out[1], residual) diff --git a/test/registered/unit/layers/test_postprocess_reduce_scatterv.py b/test/registered/unit/layers/test_postprocess_reduce_scatterv.py deleted file mode 100644 index c0f766190c..0000000000 --- a/test/registered/unit/layers/test_postprocess_reduce_scatterv.py +++ /dev/null @@ -1,273 +0,0 @@ -from sglang.test.boundary_fixtures import stub_plan - -"""Under DP attention with reduce_scatterv, postprocess sums a FULL-mode layer's -output while scattering it back to the local tokens only when the FFN left its -all-reduce out.""" - -import contextlib -import itertools -import types -import unittest -from functools import partial -from unittest.mock import patch - -import torch - -from sglang.srt.layers import communicator as comm -from sglang.srt.layers.communicator import exit as comm_exit -from sglang.srt.layers.communicator import ops as comm_ops -from sglang.srt.layers.moe import utils as moe_utils -from sglang.srt.layers.moe.utils import ( - should_skip_mlp_all_reduce, - should_skip_post_experts_all_reduce, -) -from sglang.srt.runtime_context import get_forward -from sglang.test.ci.ci_register import register_cpu_ci -from sglang.test.communicator_patch import patch_communicator -from sglang.test.test_utils import CustomTestCase - -register_cpu_ci(est_time=5, suite="base-a-test-cpu") - - -def steps(*, ffn_output_move): - """A layer's steps with the given FFN output move; None sends the output - back over attention DP.""" - return comm.BoundarySteps( - attention=comm.StageEntry( - prepare=partial( - comm_ops._consumer_step, - step=partial( - comm_ops._read_input, - layer_input=None, - enters_stack=False, - read=comm.NORM_QUANT_READ, - update=comm.ADD, - ), - carried_fusions=(), - ), - input_rows=comm.Layout(frozenset()), - input_move=comm.CommunicateSimpleFn._trivial, - handoff=comm_ops._hand_qkv_hook_its_input, - ), - ffn=comm.StageEntry( - prepare=partial( - comm_ops._read_input, - layer_input=None, - enters_stack=False, - read=comm.NORM_READ, - update=comm.ADD, - ), - input_rows=comm.Layout(frozenset()), - ), - ffn_output=comm.StageOutput(comm.Layout(frozenset())), - ffn_output_move=ffn_output_move, - ffn_sum_is_movable=False, - ) - - -@contextlib.contextmanager -def reduce_scatterv_applies(): - with ( - patch_communicator("should_use_dp_reduce_scatterv", return_value=True), - patch.object(moe_utils, "should_use_dp_reduce_scatterv", return_value=True), - ): - yield - - -def postprocess_sums(*, reduce_scatter, is_layer_sparse): - calls = [] - parallel = types.SimpleNamespace( - tp_size=2, - attn_dp_size=2, - tp_group=types.SimpleNamespace( - reduce_scatterv=lambda *args, **kwargs: calls.append("reduce_scatterv") - ), - ) - forward_batch = types.SimpleNamespace( - dp_padding_mode=types.SimpleNamespace(is_max_len=lambda: False) - ) - with ( - reduce_scatterv_applies(), - patch_communicator("get_parallel", return_value=parallel), - patch_communicator("get_local_dp_buffer", return_value=torch.empty(1, 4)), - patch_communicator("get_dp_global_num_tokens", return_value=[1, 1]), - patch_communicator( - "dp_scatter", side_effect=lambda *args: calls.append("dp_scatter") - ), - ): - step = comm_exit._reduce_and_redistribute_output_step( - forward_batch, - leaves_for_reduce_scatter=reduce_scatter, - leaves_for_reduce_scatterv=reduce_scatter, - ) - comm_ops._to_local_tokens( - step or comm_ops._redistribute_output, forward_batch, torch.zeros(2, 4) - ) - assert len(calls) == 1, calls - return calls[0] == "reduce_scatterv" - - -def ffn_leaves_the_sum_out(*, reduce_scatter, is_layer_sparse): - # A layer that allows reduce-scatter publishes mlp_reduce_scatter while - # reduce_scatterv applies; a dense MLP reads only that flag. - with ( - reduce_scatterv_applies(), - patch.object( - moe_utils, - "get_parallel", - return_value=types.SimpleNamespace(tp_size=2, dwdp_size=1), - ), - patch.object( - moe_utils, - "get_moe_a2a_backend", - return_value=types.SimpleNamespace( - is_flashinfer=lambda: False, - is_pplx=lambda: False, - is_flashinfer_megamoe=lambda: False, - ), - ), - patch.object( - moe_utils, - "should_use_flashinfer_cutlass_moe_fp4_allgather", - return_value=False, - ), - get_forward().scoped(mlp_reduce_scatter=reduce_scatter), - ): - if is_layer_sparse: - return should_skip_post_experts_all_reduce(is_tp_path=True) - return should_skip_mlp_all_reduce() - - -class TestPostprocessReduceScatterv(CustomTestCase): - def test_sums_exactly_when_the_ffn_left_the_sum_out(self): - for allow, sparse in itertools.product((False, True), repeat=2): - case = dict(reduce_scatter=allow, is_layer_sparse=sparse) - with self.subTest(**case): - self.assertEqual( - postprocess_sums(**case), ffn_leaves_the_sum_out(**case) - ) - - def test_a_dense_layer_without_reduce_scatter_is_not_summed_again(self): - self.assertFalse(postprocess_sums(reduce_scatter=False, is_layer_sparse=False)) - - def test_postprocess_only_scatters_a_complete_output(self): - # The operation-scheduled path supplies a complete output; postprocess - # only scatters it, even when the exit could choose reduce-scatter. - for step in (object(), None): - with self.subTest(reduce_scatter=step is not None): - communicator = stub_plan() - communicator._sp_steps = None - communicator._input_scattered_steps = None - communicator._cp_steps = None - communicator._steps = steps(ffn_output_move=None) - communicator.output._postprocess_dp_step = lambda forward_batch: step - with patch_communicator( - "_to_local_tokens", side_effect=lambda s, fb, h: (s, h) - ): - hidden_states, residual = communicator.output.postprocess_layer( - "h", "r", "fb" - ) - self.assertEqual(hidden_states, (comm_ops._redistribute_output, "h")) - self.assertEqual(residual, "r") - - def test_legacy_postprocess_takes_the_output_back_with_the_exit_step(self): - # Under attention DP the base postprocess brings the FFN output back with - # the step the FFN exit chooses. - for step in (object(), None): - with self.subTest(reduce_scatter=step is not None): - communicator = comm.LayerCommunicator.__new__(comm.LayerCommunicator) - communicator._sp_steps = None - communicator._input_scattered_steps = None - communicator._cp_steps = None - communicator._steps = steps(ffn_output_move=None) - communicator._postprocess_dp_step = lambda forward_batch: step - with patch_communicator( - "_to_local_tokens", side_effect=lambda s, fb, h: (s, h) - ): - hidden_states, residual = communicator.postprocess_layer( - "h", "r", "fb" - ) - self.assertEqual( - hidden_states, (step or comm_ops._redistribute_output, "h") - ) - self.assertEqual(residual, "r") - - -class TestReduceAndRedistributeOutputStep(CustomTestCase): - """The reduce-scatter that brings a FULL-layout output back to this rank's - tokens, or None when only a scatter remains.""" - - def step(self, *, varlen, max_len, tiles, allow, sparse): - forward_batch = types.SimpleNamespace( - dp_padding_mode=types.SimpleNamespace(is_max_len=lambda: max_len) - ) - with ( - patch_communicator("should_use_dp_reduce_scatterv", return_value=varlen), - patch_communicator("can_use_dp_reduce_scatter", return_value=tiles), - ): - # Both dense and MoE producers follow the selected output step. - return comm_exit._reduce_and_redistribute_output_step( - forward_batch, - leaves_for_reduce_scatter=allow, - leaves_for_reduce_scatterv=allow, - ) - - def test_every_condition(self): - for varlen, max_len, tiles, allow, sparse in itertools.product( - (False, True), repeat=5 - ): - if varlen and allow: - expected = comm_ops._reduce_and_redistribute_output_varlen - elif allow and max_len and tiles: - expected = comm_ops._reduce_and_redistribute_output_max_len - else: - expected = None - case = dict( - varlen=varlen, max_len=max_len, tiles=tiles, allow=allow, sparse=sparse - ) - with self.subTest(**case): - self.assertIs(self.step(**case), expected) - - -class TestLongcatNextnReducesItsMlp(CustomTestCase): - """Its dense layer allows no reduce-scatter, so postprocess never sums the - MLP output; the MLP must all-reduce it itself.""" - - def test_the_dense_mlp_reduces_its_output(self): - from sglang.srt.models import longcat_flash_nextn as nextn - - mlp_kwargs = {} - config = types.SimpleNamespace( - hidden_size=8, - intermediate_size=16, - hidden_act="silu", - num_attention_heads=1, - qk_nope_head_dim=1, - qk_rope_head_dim=1, - v_head_dim=1, - q_lora_rank=None, - kv_lora_rank=1, - rope_theta=1.0, - max_position_embeddings=8, - rms_norm_eps=1e-6, - num_hidden_layers=1, - ) - with ( - patch.object(nextn, "DeepseekV2AttentionMLA"), - patch.object( - nextn, - "LongcatFlashMLP", - side_effect=lambda **kwargs: mlp_kwargs.update(kwargs), - ), - patch.object(nextn, "RMSNorm"), - patch.object(nextn, "get_parallel"), - patch.object(nextn, "declare_attn"), - patch.object(nextn, "declare_ffn"), - patch.object(nextn, "make_stages", return_value=(None, None)), - ): - nextn.LongcatFlashDenseDecoderLayer(config, layer_id=0) - self.assertIs(mlp_kwargs.get("reduce_results"), True) - - -if __name__ == "__main__": - unittest.main() diff --git a/test/registered/unit/layers/test_prepare_attn_steps.py b/test/registered/unit/layers/test_prepare_attn_steps.py index 595083efba..b0ebefe2ff 100644 --- a/test/registered/unit/layers/test_prepare_attn_steps.py +++ b/test/registered/unit/layers/test_prepare_attn_steps.py @@ -1,6 +1,3 @@ -from sglang.srt.layers.communicator import StageKind -from sglang.test.boundary_fixtures import stub_plan, stub_stage - """prepare_attn completes a sum the previous layer left, then adds the residual and applies the input norm in the form the attention's quant format wants.""" @@ -14,8 +11,12 @@ import msgspec import torch from sglang.srt.layers import communicator as comm -from sglang.srt.layers.communicator import ops as comm_ops +from sglang.srt.layers.communicator import StageKind +from sglang.srt.layers.communicator import prepare as comm_ops +from sglang.srt.layers.communicator.contracts import BatchVariant from sglang.srt.layers.communicator.fusions.allreduce import attention_fusions +from sglang.srt.layers.communicator.ops import identity_output +from sglang.test.boundary_fixtures import prepare_input, stub_plan, stub_stage from sglang.test.ci.ci_register import register_cpu_ci from sglang.test.communicator_patch import patch_communicator from sglang.test.test_utils import CustomTestCase @@ -51,18 +52,16 @@ def communicator(norm): c = stub_plan() c.norm = norm # A layer inside the stack: an absent residual was written back before. - c.is_first_layer = False - c._sp_steps = None - c._input_scattered_steps = None - c._cp_steps = None + c.enters_stack = False + c._paths[BatchVariant.SEQUENCE_PARALLEL] = None + c._paths[BatchVariant.INPUT_SCATTERED] = None + c._paths[BatchVariant.CONTEXT_PARALLEL] = None c.qkv_latent_func = None - c.enable_fused_ar_quant = False - c.fused_ar_quant_keep_bf16 = False # Construction picks the fused entries; call it under the platform patches. c._attn_input_fusions = attention_fusions(c) # A layer whose attention takes its input as it is and owes nothing on it. - c._steps = comm.BoundarySteps( - attention=comm.StageEntry( + c._paths[BatchVariant.ORDINARY] = comm.StageSteps( + entry=comm.StageEntry( prepare=partial( comm_ops._consumer_step, adds_plainly=True, @@ -79,19 +78,8 @@ def communicator(norm): input_move=lambda hidden_states, **_: hidden_states, handoff=comm_ops._hand_qkv_hook_its_input, ), - ffn=comm.StageEntry( - prepare=partial( - comm_ops._read_input, - layer_input=None, - enters_stack=False, - read=comm.NORM_READ, - update=comm.ADD, - ), - input_rows=comm.Layout(frozenset()), - ), - ffn_output=comm.StageOutput(comm.Layout(frozenset())), - ffn_output_move=comm.CommunicateSummableTensorPairFn._trivial, - ffn_sum_is_movable=False, + output=comm.StageOutput(comm.Layout(frozenset())), + output_move=identity_output, ) return c @@ -132,6 +120,9 @@ def platform(*, use_aiter=False, gfx95=False, fusion=False, kernel_group=True): return_value=SimpleNamespace(input_scattered=False, is_dsa=False), ) ) + stack.enter_context( + patch_communicator("_batch_shards_over_cp", return_value=False) + ) yield kernels, all_reduce, group @@ -158,8 +149,12 @@ class TestPrepareAttnSteps(CustomTestCase): platform(**flags) as (kernels, _, _), ): norm = Norm() - stub_stage(communicator(norm), StageKind.ATTENTION)._prepare_input( - torch.ones(2, 4), residual, None, quant_format=quant_format + prepare_input( + stub_stage(communicator(norm), StageKind.ATTENTION), + torch.ones(2, 4), + residual, + None, + quant_format=quant_format, ) for name, mock in kernels.items(): self.assertEqual(mock.called, name == kernel, name) @@ -174,9 +169,13 @@ class TestPrepareAttnSteps(CustomTestCase): ): with self.subTest(quant_format=quant_format), platform(**flags): hidden_states = torch.ones(2, 4) - _, residual = stub_stage( - communicator(Norm()), StageKind.ATTENTION - )._prepare_input(hidden_states, None, None, quant_format=quant_format) + _, residual = prepare_input( + stub_stage(communicator(Norm()), StageKind.ATTENTION), + hidden_states, + None, + None, + quant_format=quant_format, + ) self.assertIs(residual.residual, hidden_states) def test_a_pending_sum_is_completed_once(self): @@ -187,8 +186,11 @@ class TestPrepareAttnSteps(CustomTestCase): ): norm = Norm() hidden_states = comm.UnreducedOutput(torch.ones(2, 4), group=group) - stub_stage(communicator(norm), StageKind.ATTENTION)._prepare_input( - hidden_states, torch.zeros(2, 4), None + prepare_input( + stub_stage(communicator(norm), StageKind.ATTENTION), + hidden_states, + torch.zeros(2, 4), + None, ) self.assertEqual(all_reduce.call_count, 0 if fusion else 1) self.assertEqual( @@ -198,7 +200,8 @@ class TestPrepareAttnSteps(CustomTestCase): def test_a_pending_sum_keeps_the_quant_format_without_fusion(self): with platform(use_aiter=True) as (kernels, all_reduce, group): norm = Norm() - stub_stage(communicator(norm), StageKind.ATTENTION)._prepare_input( + prepare_input( + stub_stage(communicator(norm), StageKind.ATTENTION), comm.UnreducedOutput(torch.ones(2, 4), group=group), torch.zeros(2, 4), None, @@ -211,7 +214,8 @@ class TestPrepareAttnSteps(CustomTestCase): def test_a_pending_sum_over_another_group_skips_the_fused_kernel(self): with platform(fusion=True, kernel_group=False) as (_, all_reduce, group): norm = Norm() - stub_stage(communicator(norm), StageKind.ATTENTION)._prepare_input( + prepare_input( + stub_stage(communicator(norm), StageKind.ATTENTION), comm.UnreducedOutput(torch.ones(2, 4), group=group), torch.zeros(2, 4), None, @@ -222,7 +226,8 @@ class TestPrepareAttnSteps(CustomTestCase): def test_a_post_residual_addition_bypasses_the_fused_kernel(self): with platform(fusion=True) as (_, all_reduce, group): norm = Norm() - stub_stage(communicator(norm), StageKind.ATTENTION)._prepare_input( + prepare_input( + stub_stage(communicator(norm), StageKind.ATTENTION), comm.UnreducedOutput(torch.ones(2, 4), group=group), torch.zeros(2, 4), None, @@ -241,9 +246,8 @@ class TestPrepareAttnSteps(CustomTestCase): partial = torch.ones(3, 4) local = torch.full((local_rows, 4), 7.0) step = MagicMock(return_value=local) - hidden_states, residual = stub_stage( - communicator(norm), StageKind.ATTENTION - )._prepare_input( + hidden_states, residual = prepare_input( + stub_stage(communicator(norm), StageKind.ATTENTION), comm.UnreducedOutput(partial, reduce_and_redistribute=step), torch.zeros(local_rows, 4), None, @@ -261,10 +265,10 @@ class TestPrepareAttnSteps(CustomTestCase): with platform(fusion=True) as (_, all_reduce, _): c = communicator(Norm()) takes_anything = MagicMock(return_value=("fused", "fused")) - c._steps = msgspec.structs.replace( - c._steps, - attention=msgspec.structs.replace( - c._steps.attention, + c._paths[BatchVariant.ORDINARY] = msgspec.structs.replace( + c._paths.get(BatchVariant.ORDINARY), + entry=msgspec.structs.replace( + c._paths.get(BatchVariant.ORDINARY).entry, prepare=partial( comm_ops._consumer_step, adds_plainly=True, @@ -281,7 +285,8 @@ class TestPrepareAttnSteps(CustomTestCase): ) partial_sum = torch.ones(3, 4) step = MagicMock(return_value=torch.full((1, 4), 7.0)) - stub_stage(c, StageKind.ATTENTION)._prepare_input( + prepare_input( + stub_stage(c, StageKind.ATTENTION), comm.UnreducedOutput(partial_sum, reduce_and_redistribute=step), torch.zeros(1, 4), None, @@ -290,25 +295,44 @@ class TestPrepareAttnSteps(CustomTestCase): takes_anything.assert_not_called() self.assertEqual(all_reduce.call_count, 0) - def test_an_unreduced_output_without_a_residual_is_rejected(self): - for step in (None, MagicMock()): - with ( - self.subTest(redistributes=step is not None), - platform(fusion=True) as (_, all_reduce, _), - ): - norm = Norm() - with self.assertRaises(RuntimeError): - stub_stage(communicator(norm), StageKind.ATTENTION)._prepare_input( - comm.UnreducedOutput( - torch.ones(2, 4), reduce_and_redistribute=step + +class TestFusedReadForms(CustomTestCase): + def test_consumer_form_and_backend_policy_are_both_required(self): + from sglang.srt.layers.communicator.residual.add_norm import ( + Fp8Input, + NormQuantRead, + ) + + for form in (None, Fp8Input.TUPLE, Fp8Input.TUPLE_AND_BF16): + for enabled in (False, True): + for disabled_by_env in (False, True): + with ( + self.subTest( + form=form, enabled=enabled, disabled_by_env=disabled_by_env ), - None, - None, - ) - self.assertEqual(all_reduce.call_count, 0) - if step is not None: - step.assert_not_called() - self.assertEqual(norm.calls, []) + patch_communicator("_use_aiter", True), + patch_communicator( + "get_exec", + return_value=SimpleNamespace( + comm=SimpleNamespace( + enable_aiter_allreduce_fusion=enabled + ) + ), + ), + patch_communicator( + "get_bool_env_var", return_value=disabled_by_env + ), + ): + plan = SimpleNamespace(norm=MagicMock(), fusions=None) + (candidate,) = attention_fusions( + plan, NormQuantRead(fp8_input=form) + ) + expected = form is not None and enabled and not disabled_by_env + self.assertEqual(candidate.keywords["fuses_quant"], expected) + self.assertEqual( + candidate.keywords["keep_bf16"], + expected and form is Fp8Input.TUPLE_AND_BF16, + ) if __name__ == "__main__": diff --git a/test/registered/unit/layers/test_residual_stream.py b/test/registered/unit/layers/test_residual_stream.py index 56ef72f20c..073b58c8df 100644 --- a/test/registered/unit/layers/test_residual_stream.py +++ b/test/registered/unit/layers/test_residual_stream.py @@ -7,17 +7,16 @@ import torch from sglang.srt.layers.aux_hidden_states import AuxHiddenStateList from sglang.srt.layers.communicator import ( - BoundarySteps, EdgeDecl, Layout, StageEntry, StageInput, StageOutput, + StageSteps, SumGroup, make_boundary, ) -from sglang.srt.layers.communicator.boundary import StageKind -from sglang.srt.layers.communicator.construction import StageCommunicator +from sglang.srt.layers.communicator.contracts import BatchVariant, StageKind from sglang.srt.layers.communicator.output import UnreducedOutput from sglang.srt.layers.communicator.residual.access import add_to_output from sglang.srt.layers.communicator.residual.add_norm import ADD @@ -26,7 +25,9 @@ from sglang.srt.layers.communicator.residual.stream import ( OwedOutput, ResidualStream, ) -from sglang.test.boundary_fixtures import stub_plan, stub_stage +from sglang.srt.layers.communicator.stage import StageCommunicator +from sglang.srt.model_executor.forward_batch_info import ForwardMode +from sglang.test.boundary_fixtures import prepare_attention, stub_plan, stub_stage from sglang.test.ci.ci_register import register_cpu_ci from sglang.test.test_utils import CustomTestCase @@ -61,13 +62,10 @@ class TestResidualStream(CustomTestCase): EdgeDecl(StageOutput(rows), StageInput(rows, read=read), rows, rows), enters_stack=True, ) - steps = BoundarySteps( - StageEntry(boundary.prepare, rows), None, StageOutput(rows), None, False - ) + steps = StageSteps(StageEntry(boundary.prepare, rows), StageOutput(rows), None) stage = StageCommunicator( SimpleNamespace(norm=None), - StageKind.ATTENTION, - None, + declaration=SimpleNamespace(kind=StageKind.ATTENTION), ) value = torch.ones(2, 4) hidden, stream = stage._prepare(value, ResidualStream(), None, steps) @@ -107,13 +105,10 @@ class TestResidualStream(CustomTestCase): rows, ) ) - steps = BoundarySteps( - StageEntry(boundary.prepare, rows), None, StageOutput(rows), None, False - ) + steps = StageSteps(StageEntry(boundary.prepare, rows), StageOutput(rows), None) stage = StageCommunicator( SimpleNamespace(norm=None), - StageKind.ATTENTION, - None, + declaration=SimpleNamespace(kind=StageKind.ATTENTION), ) for increment in (2, 7): with self.subTest(increment=increment): @@ -211,6 +206,17 @@ class TestResidualStream(CustomTestCase): update=ADD, ) + def test_snapshot_rejects_stateful_update_without_touching_main_state(self): + update = SimpleNamespace(adds_plainly=False, update=Mock()) + stream = ResidualStream(self.residual) + hidden = stream.leave(UnreducedOutput(self.partial, group=self.group), update) + with self.assertRaisesRegex(NotImplementedError, "plain residual update"): + stream.snapshot(hidden) + self.group.all_reduce.assert_not_called() + update.update.assert_not_called() + self.assertIs(stream.pending.update, update) + self.assertIsNotNone(stream.pending.owed) + def test_snapshot_finishes_a_copy_without_consuming_main_work(self): snapshot = self.stream.snapshot(self.hidden) torch.testing.assert_close(snapshot, torch.full((2, 4), 5.0)) @@ -263,8 +269,12 @@ class TestResidualStream(CustomTestCase): capture_move_allocates=False, capture_preserves_residual=None, ) - output, stream = stub_stage(boundary, StageKind.ATTENTION)._prepare_attention( - self.hidden, self.stream, None, capture_output=capture + output, stream = prepare_attention( + stub_stage(boundary, StageKind.ATTENTION), + self.hidden, + self.stream, + None, + capture_output=capture, ) self.assertEqual(events, ["add_norm", "capture"]) self.group.all_reduce.assert_called_once() @@ -295,7 +305,8 @@ class TestResidualStream(CustomTestCase): capture_move_allocates=False, capture_preserves_residual=None, ) - output, _ = stub_stage(boundary, StageKind.ATTENTION)._prepare_attention( + output, _ = prepare_attention( + stub_stage(boundary, StageKind.ATTENTION), self.hidden, self.stream, None, @@ -431,12 +442,14 @@ class TestBatchStageOwnership(CustomTestCase): ) owner = stub_plan() owner.norm = None - owner._steps = BoundarySteps( - None, StageEntry(boundary.prepare, rows), StageOutput(rows), None, False + owner._paths[BatchVariant.ORDINARY] = StageSteps( + StageEntry(boundary.prepare, rows), StageOutput(rows), None ) - owner._sp_steps = owner._input_scattered_steps = owner._cp_steps = None + owner._paths[BatchVariant.SEQUENCE_PARALLEL] = owner._paths[ + BatchVariant.INPUT_SCATTERED + ] = owner._paths[BatchVariant.CONTEXT_PARALLEL] = None stage = stub_stage(owner, StageKind.FFN) - a, b = SimpleNamespace(), SimpleNamespace() + a, b = [SimpleNamespace(forward_mode=ForwardMode.DECODE) for _ in range(2)] batch.start(a) batch.start(b) first = stage.prepare(torch.ones(2, 4), a) diff --git a/test/registered/unit/lora/test_triton_dp_attention_unit.py b/test/registered/unit/lora/test_triton_dp_attention_unit.py index 766b5afddc..aa4aa89025 100644 --- a/test/registered/unit/lora/test_triton_dp_attention_unit.py +++ b/test/registered/unit/lora/test_triton_dp_attention_unit.py @@ -1,6 +1,3 @@ -from sglang.srt.layers.communicator import StageKind -from sglang.test.boundary_fixtures import stub_plan, stub_stage - """CPU-only metadata tests for explicit DP-attention LoRA routing.""" import sys @@ -16,10 +13,11 @@ from sglang.srt.layers.communicator import ( ADD, NORM_QUANT_READ, Layout, + StageKind, TokenAxis, ) -from sglang.srt.layers.communicator.boundary import StageEntry -from sglang.srt.layers.communicator.ops import _consumer_step, _read_input +from sglang.srt.layers.communicator.contracts import BatchVariant, StageEntry +from sglang.srt.layers.communicator.prepare import _consumer_step, _read_input from sglang.srt.layers.dp_attention import DpPaddingMode from sglang.srt.lora.backend.base_backend import BaseLoRABackend from sglang.srt.lora.backend.triton_backend import ( @@ -37,6 +35,7 @@ from sglang.srt.runtime_context import ( get_forward, get_parallel, ) +from sglang.test.boundary_fixtures import prepare_input, stub_plan, stub_stage from sglang.test.ci.ci_register import register_cpu_ci from sglang.test.communicator_patch import COMMUNICATOR_MODULES from sglang.test.test_utils import CustomTestCase @@ -233,51 +232,52 @@ def test_communicator_publishes_layout_at_each_transition( expected_mlp = expected_attn = initial communicator = stub_plan() communicator._publish_lora_layout = publish_lora_layout - communicator.is_first_layer = False - communicator._sp_steps = None + communicator.enters_stack = False + communicator._paths[BatchVariant.SEQUENCE_PARALLEL] = None communicator.norm = lambda x: x communicator.qkv_latent_func = None gathered, local = Layout(frozenset()), Layout(frozenset({TokenAxis.ATTN_DP})) # The rows of the steps the batch runs decide, not the ordinary steps'. - communicator._steps = SimpleNamespace( - ffn=SimpleNamespace(input_rows=local if gathered_over_dp else gathered) + communicator._paths[BatchVariant.ORDINARY] = SimpleNamespace( + entry=SimpleNamespace(input_rows=local if gathered_over_dp else gathered) ) - selected = SimpleNamespace( - attention=StageEntry( - input_rows=local, - prepare=partial( - _consumer_step, - step=partial( - _read_input, - layer_input=None, - enters_stack=False, - read=NORM_QUANT_READ, - update=ADD, - ), - carried_fusions=(), - adds_plainly=True, + attention_entry = StageEntry( + input_rows=local, + prepare=partial( + _consumer_step, + step=partial( + _read_input, + layer_input=None, + enters_stack=False, + read=NORM_QUANT_READ, + update=ADD, ), - input_move=lambda hidden_states, **kwargs: hidden_states, - handoff=lambda hidden_states, *args: hidden_states, - ), - ffn=StageEntry( - prepare=lambda hidden_states, residual, *args, **kwargs: ( - hidden_states, - residual, - ), - input_rows=gathered if gathered_over_dp else local, - input_move=None, - handoff=None, + carried_fusions=(), + adds_plainly=True, ), + input_move=lambda hidden_states, **kwargs: hidden_states, + handoff=lambda hidden_states, *args: hidden_states, ) + ffn_entry = StageEntry( + prepare=lambda hidden_states, residual, *args, **kwargs: ( + hidden_states, + residual, + ), + input_rows=gathered if gathered_over_dp else local, + input_move=None, + handoff=None, + ) + selected = SimpleNamespace(entry=ffn_entry) communicator._batch_steps = lambda forward_batch: selected hidden = torch.zeros(num_tokens, 4) with get_forward().scoped(lora_batch_layout=initial): for _ in range(2): - stub_stage(communicator, StageKind.FFN)._prepare_input(hidden, hidden, None) + selected.entry = ffn_entry + prepare_input(stub_stage(communicator, StageKind.FFN), hidden, hidden, None) assert get_forward().lora_batch_layout is expected_mlp - stub_stage(communicator, StageKind.ATTENTION)._prepare_input( - hidden, None, None + selected.entry = attention_entry + prepare_input( + stub_stage(communicator, StageKind.ATTENTION), hidden, None, None ) assert get_forward().lora_batch_layout is expected_attn diff --git a/test/registered/unit/models/test_aux_capture_deferred_allreduce.py b/test/registered/unit/models/test_aux_capture_deferred_allreduce.py index 3cdc3ec5b8..a7fd69c981 100644 --- a/test/registered/unit/models/test_aux_capture_deferred_allreduce.py +++ b/test/registered/unit/models/test_aux_capture_deferred_allreduce.py @@ -9,7 +9,9 @@ from torch import nn from sglang.srt.layers import communicator as comm from sglang.srt.layers.aux_hidden_states import AuxHiddenStateList from sglang.srt.layers.communicator import StageKind -from sglang.srt.layers.communicator import ops as comm_ops +from sglang.srt.layers.communicator import prepare as comm_ops +from sglang.srt.layers.communicator.contracts import BatchVariant +from sglang.srt.layers.communicator.ops import identity_output from sglang.srt.layers.communicator.residual.stream import ResidualStream from sglang.srt.model_executor.forward_batch_info import ForwardMode, PPProxyTensors from sglang.srt.models.bailing_moe import BailingMoEModel @@ -23,7 +25,12 @@ from sglang.srt.models.llama4 import Llama4Model from sglang.srt.models.qwen3 import Qwen3Model from sglang.srt.models.qwen3_vl import Qwen3LLMModel from sglang.srt.models.qwen3_vl_moe import Qwen3MoeLLMModel -from sglang.test.boundary_fixtures import stub_plan, stub_stage +from sglang.test.boundary_fixtures import ( + finish_exit, + identity_input, + stub_plan, + stub_stage, +) from sglang.test.ci.ci_register import register_cpu_ci from sglang.test.test_utils import CustomTestCase @@ -39,8 +46,10 @@ MODELS = ( GptOssModel, LagunaModel, Qwen3MoeLLMModel, + Qwen3Model, + Qwen3LLMModel, + Llama4Model, ) -CAPTURE_MODELS = (*MODELS, Llama4Model, Qwen3Model, Qwen3LLMModel) def all_reduce(hidden_states): @@ -59,8 +68,8 @@ class DeferringLayer(nn.Module): self.layer_communicator = stub_plan() self.layer_communicator.norm = None self.attn_stage = stub_stage(self.layer_communicator, StageKind.ATTENTION) - self.layer_communicator._steps = comm.BoundarySteps( - attention=comm.StageEntry( + self.layer_communicator._paths[BatchVariant.ORDINARY] = comm.StageSteps( + entry=comm.StageEntry( prepare=partial( comm_ops._consumer_step, step=partial( @@ -73,34 +82,23 @@ class DeferringLayer(nn.Module): carried_fusions=(), ), input_rows=comm.Layout(frozenset()), - input_move=comm.CommunicateSimpleFn._trivial, + input_move=identity_input, handoff=comm_ops._hand_qkv_hook_its_input, ), - ffn=comm.StageEntry( - prepare=partial( - comm_ops._read_input, - layer_input=None, - enters_stack=False, - read=comm.NORM_READ, - update=comm.ADD, - ), - input_rows=comm.Layout(frozenset()), - ), - ffn_output=comm.StageOutput( + output=comm.StageOutput( comm.Layout(frozenset()), group=comm.SumGroup.TP, leaves_for_next_layer=True, ), - ffn_output_move=comm.CommunicateSummableTensorPairFn._trivial, - ffn_sum_is_movable=True, + output_move=identity_output, ) self.layer_communicator.output._ffn_sum_moves_to_next_layer = ( - lambda batch, **_: defer + lambda batch, steps, **_: defer ) - self.layer_communicator.is_last_layer = False - self.layer_communicator._sp_steps = None - self.layer_communicator._input_scattered_steps = None - self.layer_communicator._cp_steps = None + self.layer_communicator.terminal = False + self.layer_communicator._paths[BatchVariant.SEQUENCE_PARALLEL] = None + self.layer_communicator._paths[BatchVariant.INPUT_SCATTERED] = None + self.layer_communicator._paths[BatchVariant.CONTEXT_PARALLEL] = None self.layer_communicator.output.ffn_reduction_group = lambda forward_batch: GROUP self.layer_communicator.output._ffn_leaves_sum_to_reduce_scatter = ( lambda batch, dp_step: False @@ -118,7 +116,7 @@ class DeferringLayer(nn.Module): *args, **kwargs, ): - stream = forward_batch.residual_stream + stream = forward_batch.residual_stream if self.stage_api else None if stream is not None: residual = stream if isinstance(residual, ResidualStream): @@ -132,12 +130,14 @@ class DeferringLayer(nn.Module): capture = kwargs.get("capture_output") if capture is not None: capture(residual.clone()) - with self.layer_communicator.output.ffn_exit(forward_batch) as ffn_exit: + with self.layer_communicator.output.ffn_exit( + forward_batch, stream=ResidualStream() + ) as ffn_exit: partial = torch.full_like(hidden_states, 0.5) if stream is not None: stream.write(residual) residual = stream - hidden_states, residual = ffn_exit.finish(partial, residual) + hidden_states, residual = finish_exit(ffn_exit, partial, residual) if self.stage_api: return (hidden_states, None) if self.return_topk else hidden_states if self.return_topk: @@ -164,7 +164,7 @@ def build_model(model_cls, *, defer, capture): model.use_hf_deepstack_order = False model.dflash_capture = capture model.enable_a2a_moe = False - model.config = SimpleNamespace(mhc=False) + model.config = SimpleNamespace(mhc=False, num_hidden_layers=NUM_LAYERS) if model_cls is BailingMoELinearModel: # Bailing-v3 captures after the layer; other models use boundary indices model.layers_to_capture = [0, 1, 2] if capture else [] @@ -177,13 +177,13 @@ def build_model(model_cls, *, defer, capture): BailingMoEModel, BailingMoELinearModel, GptOssModel, - Llama4Model, - Qwen3Model, - Qwen3LLMModel, Glm4MoeModel, Glm4MoeLiteModel, LagunaModel, Qwen3MoeLLMModel, + Qwen3Model, + Qwen3LLMModel, + Llama4Model, Glm5NextModel, ), ) @@ -223,7 +223,7 @@ class TestAuxCaptureDeferredAllreduce(CustomTestCase): forward_mode=ForwardMode.DECODE, capture_hidden_mode=SimpleNamespace(need_capture=lambda: True), ) - for model_cls in CAPTURE_MODELS: + for model_cls in MODELS: for defer in (False, True): for capture in (False, True): with ( @@ -316,6 +316,28 @@ class TestAuxCaptureDeferredAllreduce(CustomTestCase): torch.testing.assert_close(snapshots[0], inputs + NUM_LAYERS) torch.testing.assert_close(result, inputs + NUM_LAYERS) + def test_llama4_split_prefill_preserves_the_stream_between_segments(self): + from sglang.srt.models.llama4 import Llama4ForCausalLM, Llama4Model + + model = build_model(Llama4Model, defer=True, capture=False) + lm = SimpleNamespace( + model=model, + lm_head=None, + logits_processor=lambda ids, hidden, head, fb: hidden, + ) + fb = SimpleNamespace(forward_mode=ForwardMode.DECODE) + inputs = torch.ones(2, 4) + result = Llama4ForCausalLM.forward_split_prefill( + lm, None, None, fb, (0, 2), inputs.clone() + ) + self.assertIsNone(result) + self.assertIsNotNone(fb.residual_stream) + result = Llama4ForCausalLM.forward_split_prefill( + lm, None, None, fb, (2, NUM_LAYERS) + ) + torch.testing.assert_close(result, inputs + NUM_LAYERS) + self.assertIsNone(fb.residual_stream) + class TestPipelineResidualReception(CustomTestCase): def test_models_keep_the_received_residual_contribution(self): @@ -327,6 +349,8 @@ class TestPipelineResidualReception(CustomTestCase): capture_hidden_mode=SimpleNamespace(need_capture=lambda: False), ) for model_cls in MODELS: + if model_cls is Llama4Model: + continue # Llama4 has no pipeline model entry. with self.subTest(model=model_cls.__name__): model = build_model(model_cls, defer=True, capture=False) model.pp_group.is_first_rank = False @@ -370,14 +394,20 @@ class TestPipelineResidualReception(CustomTestCase): comm_instance = stub_plan() comm_instance.norm = None comm_instance._batch_steps = lambda batch: SimpleNamespace( - attention=SimpleNamespace(input_sum=None) + entry=SimpleNamespace(input_sum=None) ) batch = SimpleNamespace(residual_stream=None) - comm_instance.residual_in_hidden = True streams = torch.randn(2, 4, 3) - hidden = stub_stage(comm_instance, StageKind.ATTENTION).from_pp( - PPProxyTensors({"hidden_states": streams}), batch + from dataclasses import replace + + from sglang.srt.layers.communicator import declare_ffn + + stage = stub_stage(comm_instance, StageKind.ATTENTION) + stage.declaration = replace( + stage.declaration, + previous=declare_ffn(update=SimpleNamespace(at_producer=True)), ) + hidden = stage.from_pp(PPProxyTensors({"hidden_states": streams}), batch) residual = batch.residual_stream self.assertIs(hidden, streams) self.assertIsNone(residual.pending) @@ -387,7 +417,7 @@ class TestPipelineResidualReception(CustomTestCase): comm_instance = stub_plan() comm_instance.norm = None comm_instance._batch_steps = lambda batch: SimpleNamespace( - attention=SimpleNamespace(input_sum=None) + entry=SimpleNamespace(input_sum=None) ) batch = SimpleNamespace(residual_stream=None) partial = torch.randn(2, 4) diff --git a/test/registered/unit/models/test_last_layer_communicator.py b/test/registered/unit/models/test_last_layer_communicator.py index a1a5da934b..a6d13c80dd 100644 --- a/test/registered/unit/models/test_last_layer_communicator.py +++ b/test/registered/unit/models/test_last_layer_communicator.py @@ -8,7 +8,6 @@ from unittest.mock import MagicMock, patch import torch from sglang.test.ci.ci_register import register_cpu_ci -from sglang.test.communicator_patch import patch_communicator from sglang.test.test_utils import CustomTestCase register_cpu_ci(est_time=15, suite="base-a-test-cpu") @@ -271,32 +270,5 @@ class TestTerminalStages(CustomTestCase): self.assertTrue(planned["is_layer_sparse"]) -class TestLayerFactsLastLayer(CustomTestCase): - def test_the_plan_marks_the_last_layer(self): - from sglang.srt.layers import communicator as comm - - with ( - patch_communicator("enable_moe_dense_fully_dp", return_value=False), - patch_communicator("_generic_prefill_cp_shards_tokens", return_value=False), - patch_communicator("is_dsa_enable_prefill_cp", return_value=False), - patch_communicator("is_mla_cp_enabled", return_value=False), - ): - for num_layers, layer_id in ( - (NUM_LAYERS, 0), - (NUM_LAYERS, 2), - (NUM_LAYERS, 3), - (1, 0), - ): - with self.subTest(num_layers=num_layers, layer_id=layer_id): - modes = comm.LayerFacts.init_new( - layer_id=layer_id, - num_layers=num_layers, - is_layer_sparse=False, - is_previous_layer_sparse=False, - is_next_layer_sparse=False, - ) - self.assertEqual(modes.is_last_layer, layer_id == num_layers - 1) - - if __name__ == "__main__": unittest.main() diff --git a/test/registered/unit/models/test_layer_stack_exit.py b/test/registered/unit/models/test_layer_stack_exit.py deleted file mode 100644 index d799f05bbd..0000000000 --- a/test/registered/unit/models/test_layer_stack_exit.py +++ /dev/null @@ -1,451 +0,0 @@ -"""A decoder layer may leave its FFN all-reduce to the next layer's input norm. -The last layer on a rank has no next layer there, so a model built from such -layers must pass their output through ``finish_layer_stack`` before the final -norm or the send to the next pipeline rank. Between layers, an in-place write to -a layer's output needs the completed sum as well. Every set below is derived -from the model sources.""" - -import ast -import unittest -from pathlib import Path - -import sglang -from sglang.test.ci.ci_register import register_cpu_ci -from sglang.test.test_utils import CustomTestCase - -register_cpu_ci(est_time=10, suite="base-a-test-cpu") - -MODELS_DIR = Path(sglang.__file__).resolve().parent / "srt" / "models" -UNREDUCED = "UnreducedOutput" -EXIT = "finish_layer_stack" -COMPLETE = "reduce_output" -FINAL_NORMS = {"norm", "norm_f", "final_layernorm"} -IN_PLACE = {"add_", "sub_", "mul_", "copy_"} - - -def base_names(node): - return [ - base.id if isinstance(base, ast.Name) else base.attr - for base in node.bases - if isinstance(base, (ast.Name, ast.Attribute)) - ] - - -def method(node, name): - for item in node.body: - if isinstance(item, ast.FunctionDef) and item.name == name: - return item - return None - - -def called_name(call): - func = call.func - return func.id if isinstance(func, ast.Name) else getattr(func, "attr", None) - - -def calls(node, name): - for sub in ast.walk(node): - if isinstance(sub, ast.Call) and called_name(sub) == name: - yield sub - - -def delegates_to_super(forward): - return any( - isinstance(call.func, ast.Attribute) - and isinstance(call.func.value, ast.Call) - and getattr(call.func.value.func, "id", None) == "super" - for call in calls(forward, "forward") - ) - - -def defers(node): - """Asks whether to leave the reduction to the next layer (directly or - through ffn_exit), or wraps its output as unreduced itself.""" - return any(any(calls(node, name)) for name in ("ffn_exit", "exit", UNREDUCED)) - - -class Census: - def __init__(self, include_complete=False): - self.include_complete = include_complete - self.trees = {} - self.classes = {} - for path in sorted(MODELS_DIR.rglob("*.py")): - tree = ast.parse(path.read_text()) - self.trees[path] = tree - for node in ast.walk(tree): - if isinstance(node, ast.ClassDef): - self.classes.setdefault(node.name, []).append((path, node)) - self.deferring = self._deferring_layers() - self.family = self._layer_family() - self.stacks = self._layer_stacks() - - def _deferring_layers(self): - # Module-level helpers that defer on a layer's behalf. - helpers = { - path: { - st.name - for st in tree.body - if isinstance(st, ast.FunctionDef) and defers(st) - } - for path, tree in self.trees.items() - } - deferring = { - name - for name, defs in self.classes.items() - for path, node in defs - if defers(node) - or any(any(calls(node, h)) for h in helpers[path]) - or ( - self.include_complete - and any( - isinstance(call, ast.Call) - and isinstance(call.func, ast.Name) - and call.func.id - in { - "make_stages", - } - for call in ast.walk(node) - ) - ) - } - # A subclass defers through its base unless it replaces forward - # without delegating to it. - while True: - inherited = { - name - for name, defs in self.classes.items() - if name not in deferring - for _, node in defs - if set(base_names(node)) & deferring - and ( - method(node, "forward") is None - or delegates_to_super(method(node, "forward")) - ) - } - if not inherited: - return deferring - deferring |= inherited - - def _layer_family(self): - """Classes derived from a deferring layer, whether or not they still - defer (a subclass that replaces forward may not).""" - family = set(self.deferring) - while True: - derived = { - name - for name, defs in self.classes.items() - if name not in family - for _, node in defs - if set(base_names(node)) & family - } - if not derived: - return family - family |= derived - - def registries(self, path, names): - """Module-level names bound to a value that references ``names``, such - as a table of decoder layer types.""" - out = {} - for st in self.trees[path].body: - if isinstance(st, (ast.Assign, ast.AnnAssign)) and st.value is not None: - members = { - sub.id - for sub in ast.walk(st.value) - if isinstance(sub, ast.Name) and sub.id in names - } - targets = st.targets if isinstance(st, ast.Assign) else [st.target] - for target in targets: - if members and isinstance(target, ast.Name): - out[target.id] = members - return out - - def built_layers(self, path, node, names): - """The classes in ``names`` this class references other than as a base - or an isinstance argument, looking through module-level tables.""" - skip = {id(base) for base in node.bases} - for call in ast.walk(node): - if ( - isinstance(call, ast.Call) - and called_name(call) in ("isinstance", "issubclass") - and len(call.args) > 1 - ): - skip |= {id(sub) for sub in ast.walk(call.args[1])} - tables = self.registries(path, names) - found = set() - for sub in ast.walk(node): - if id(sub) in skip: - continue - name = sub.id if isinstance(sub, ast.Name) else getattr(sub, "attr", None) - if name in names: - found.add(name) - elif name in tables: - found |= tables[name] - return found - - def _layer_stacks(self): - """Classes whose instances run deferring layers: they build them, hold a - forward-less container that does, or subclass such a class.""" - stacks = {} - runners = set(self.deferring) - while True: - grown = False - for name, defs in self.classes.items(): - if name in self.deferring: - continue - for path, node in defs: - key = (path, node.name) - if key in stacks: - continue - layers = self.built_layers(path, node, runners) - parents = set(base_names(node)) & {n for _, n in stacks} - if not layers and not parents: - continue - if not layers and self.built_layers( - path, node, self.family - self.deferring - ): - continue # a subclass that builds layers which never defer - stacks[key] = (node, layers & self.deferring) - if self.used_forward(path, node) is None: - runners.add(name) - grown = True - if not grown: - return stacks - - def used_forward(self, path, node, seen=()): - forward = method(node, "forward") - if forward is not None: - return path, node, forward - for base in base_names(node): - for base_path, base_node in self.classes.get(base, ()): - if (base_path, base) not in seen: - found = self.used_forward( - base_path, base_node, seen + ((base_path, base),) - ) - if found: - return found - return None - - def subjects(self): - """forward methods the layer stacks run, with the deferring layers - each stack builds.""" - out = {} - for (path, _), (node, layers) in self.stacks.items(): - found = self.used_forward(path, node) - if found is None: - continue # a container, run by the class that holds it - fpath, fnode, forward = found - key = f"{fpath.relative_to(MODELS_DIR)}:{fnode.name}" - entry = out.setdefault(key, (forward, set())) - entry[1].update(layers) - return out - - def layer_calls_exit(self, name): - return any( - method(node, "forward") is not None - and any(calls(method(node, "forward"), EXIT)) - for _, node in self.classes.get(name, ()) - ) - - -def inside_loop(tree, target): - for node in ast.walk(tree): - if isinstance(node, (ast.For, ast.While)) and node is not tree: - if any(sub is target for sub in ast.walk(node)): - return True - return False - - -def is_final_norm(call): - func, chain = call.func, [] - while isinstance(func, ast.Attribute): - chain.append(func.attr) - func = func.value - return bool(FINAL_NORMS & set(chain)) - - -class TestLayerStackExit(CustomTestCase): - @classmethod - def setUpClass(cls): - cls.census = Census() - cls.subjects = cls.census.subjects() - cls.complete_census = Census(include_complete=True) - - def test_derivation_finds_each_kind_of_stack(self): - for name in ( - "glm4_moe_lite.py:Glm4MoeLiteModel", # builds its layers - "qwen2_moe.py:Qwen2MoeModel", # run by Qwen3-MoE, which passes them in - "qwen3_next.py:Qwen3NextModel", # layers defer through a helper - "nemotron_h.py:NemotronHModel", # layers come from a type table - "dots3_common/nextn.py:Dot3NoteModelNextN", # holds an MTP head - "mistral_large_3_eagle.py:MistralLarge3EagleModel", # subclass - ): - self.assertIn(name, self.subjects) - - def test_each_stack_ends_at_the_exit(self): - problems = [] - for name, (forward, layers) in sorted(self.complete_census.subjects().items()): - exits = list(calls(forward, EXIT)) + list(calls(forward, "take_output")) - if not exits: - batch_exits = [ - call - for call in ast.walk(forward) - if isinstance(call, ast.Call) - and isinstance(call.func, ast.Attribute) - and isinstance(call.func.value, ast.Name) - and call.func.value.id == "residual_batch" - and call.func.attr in {"norm", "to_pp", "finish"} - ] - if batch_exits and all( - not inside_loop(forward, call) for call in batch_exits - ): - continue - if delegates_to_super(forward): - continue - if layers and all( - self.complete_census.layer_calls_exit(n) for n in layers - ): - continue # the last layer ends the stack itself - problems.append(f"{name}: no {EXIT} call") - continue - if len(exits) > 1 or inside_loop(forward, exits[0]): - problems.append(f"{name}: {EXIT} must be called once, after the loop") - continue - line = exits[0].lineno - for call in ast.walk(forward): - if not isinstance(call, ast.Call) or call.lineno >= line: - continue - if called_name(call) == "PPProxyTensors" or is_final_norm(call): - problems.append( - f"{name} line {call.lineno}: {called_name(call)} before {EXIT}" - ) - self.assertEqual(problems, []) - - def test_pipeline_residual_is_received_through_the_boundary(self): - problems = [] - for name, (forward, _) in self.complete_census.subjects().items(): - for node in ast.walk(forward): - if ( - isinstance(node, ast.Subscript) - and isinstance(node.value, ast.Name) - and node.value.id == "pp_proxy_tensors" - and isinstance(node.slice, ast.Constant) - and node.slice.value == "residual" - ): - problems.append(f"{name}:{node.lineno}") - self.assertEqual(problems, []) - - def test_models_use_boundary_access_for_output_values(self): - problems = [] - forbidden = { - "UnreducedOutput", - "HandoffOutput", - "OwedOutput", - "Contribution", - "ResidualStream", - "reduce_output", - "layer_input_buffer", - } - for path, tree in self.complete_census.trees.items(): - for node in ast.walk(tree): - if isinstance(node, ast.Name) and node.id in forbidden: - problems.append(f"{path.name}:{node.lineno}: {node.id}") - for name, (forward, _) in self.complete_census.subjects().items(): - for node in ast.walk(forward): - if ( - isinstance(node, ast.BinOp) - and isinstance(node.op, ast.Add) - and isinstance(node.left, ast.Name) - and isinstance(node.right, ast.Name) - and {node.left.id, node.right.id} == {"hidden_states", "residual"} - ): - problems.append(f"{name}:{node.lineno}: residual add") - self.assertEqual(problems, []) - - def test_split_prefill_completes_only_at_the_final_segment(self): - census = self.complete_census - containers = {name for _, name in census.stacks} - while True: - grown = { - name - for name, definitions in census.classes.items() - for path, node in definitions - if set(base_names(node)) & containers - or census.built_layers(path, node, containers) - } - containers - if not grown: - break - containers |= grown - checked = 0 - for name in sorted(containers): - for path, node in census.classes[name]: - forward = method(node, "forward_split_prefill") - if forward is None: - continue - if any(calls(forward, "forward_split_prefill")): - continue # delegates to the inherited implementation - with self.subTest(model=name, file=path.name): - exits = list(calls(forward, EXIT)) + list( - calls(forward, "take_output") - ) - exits += [ - call - for call in calls(forward, "norm") - if isinstance(call.func, ast.Attribute) - and isinstance(call.func.value, ast.Name) - and call.func.value.id == "residual_batch" - ] - self.assertEqual(len(exits), 1) - exit_call = exits[0] - self.assertFalse(inside_loop(forward, exit_call)) - final_segments = [ - item - for item in ast.walk(forward) - if isinstance(item, ast.If) - and isinstance(item.test, ast.Compare) - and isinstance(item.test.left, ast.Name) - and item.test.left.id == "end" - and any(part is exit_call for part in ast.walk(item)) - ] - early_returns = [ - item - for item in forward.body - if isinstance(item, ast.If) - and isinstance(item.test, ast.Compare) - and isinstance(item.test.left, ast.Name) - and item.test.left.id == "end" - and any(isinstance(op, ast.NotEq) for op in item.test.ops) - and any(isinstance(part, ast.Return) for part in item.body) - and item.end_lineno < exit_call.lineno - ] - self.assertEqual(len(final_segments) + len(early_returns), 1) - for call in ast.walk(forward): - if ( - isinstance(call, ast.Call) - and call is not exit_call - and is_final_norm(call) - ): - self.assertLess(exit_call.lineno, call.lineno) - checked += 1 - self.assertGreater(checked, 0) - - def test_in_place_writes_between_layers_see_the_reduced_sum(self): - problems = [] - for name, (forward, _) in sorted(self.subjects.items()): - completes = [call.lineno for call in calls(forward, COMPLETE)] - for call in ast.walk(forward): - if ( - isinstance(call, ast.Call) - and isinstance(call.func, ast.Attribute) - and call.func.attr in IN_PLACE - and isinstance(call.func.value, ast.Name) - and call.func.value.id == "hidden_states" - and not any(line < call.lineno for line in completes) - ): - problems.append(f"{name} line {call.lineno}") - self.assertEqual( - problems, [], f"call {COMPLETE}(hidden_states) before writing in place" - ) - - -if __name__ == "__main__": - unittest.main() diff --git a/test/registered/unit/models/test_longcat_flash_shortcut.py b/test/registered/unit/models/test_longcat_flash_shortcut.py index 2bf9d71717..d535189d39 100644 --- a/test/registered/unit/models/test_longcat_flash_shortcut.py +++ b/test/registered/unit/models/test_longcat_flash_shortcut.py @@ -48,10 +48,12 @@ class TestLongcatShortcut(CustomTestCase): ) moe_communicator.post_attention_layernorm = None dense_communicator.post_attention_layernorm = None - layer.moe_stage = StageCommunicator(moe_communicator, comm.StageKind.FFN, None) + layer.moe_stage = StageCommunicator( + moe_communicator, declaration=comm.declare_ffn() + ) layer.moe_stage.prepare = lambda h, fb: fork_hidden layer.second_ffn = StageCommunicator( - dense_communicator, comm.StageKind.FFN, None + dense_communicator, declaration=comm.declare_ffn() ) layer.self_attn = [lambda **kw: kw["hidden_states"]] layer.mlp = nn.Identity() diff --git a/test/registered/unit/models/test_nemotron_h_stages.py b/test/registered/unit/models/test_nemotron_h_stages.py index fdb689c9e3..3f6c9c18a1 100644 --- a/test/registered/unit/models/test_nemotron_h_stages.py +++ b/test/registered/unit/models/test_nemotron_h_stages.py @@ -2,14 +2,11 @@ import itertools import unittest from dataclasses import replace from types import SimpleNamespace -from unittest.mock import MagicMock +from unittest.mock import MagicMock, patch import torch from sglang.srt.layers.communicator import ( - ADD, - NORM_QUANT_READ, - NORM_READ, Layout, MixerExit, StageOutput, @@ -17,6 +14,7 @@ from sglang.srt.layers.communicator import ( TokenAxis, UnreducedOutput, ) +from sglang.srt.layers.communicator.residual.stream import ResidualStream from sglang.srt.layers.moe.utils import should_skip_mlp_all_reduce from sglang.srt.models import nemotron_h_utils as utils from sglang.srt.runtime_context import get_parallel @@ -36,7 +34,12 @@ def layer_stage(pattern, index): previous=previous, terminal=index == len(pattern) - 1, ) - incoming, outgoing = _connections(declaration) + following = ( + replace(utils._declaration(pattern, index + 1), previous=declaration) + if index + 1 < len(pattern) + else None + ) + incoming, outgoing = _connections(declaration, following) return SimpleNamespace( kind=declaration.kind, edges=( @@ -65,8 +68,13 @@ def stages(pattern, *, dp=1, tp=1, a2a=False): tp_size=dp * tp, moe_dp_size=1, moe_dense_tp_size=None, + enable_attn_tp_input_scattered=False, + enable_prefill_cp=False, ) with ( + patch( + "sglang.srt.layers.layernorm_sp.layernorm_sp_enabled", return_value=False + ), patch_communicator( "get_exec", return_value=SimpleNamespace( @@ -76,6 +84,7 @@ def stages(pattern, *, dp=1, tp=1, a2a=False): ), patch_communicator("token_axis_sizes", return_value=sizes(dp=dp, tp=tp)), patch_communicator("get_parallel", return_value=parallel), + patch.object(utils, "get_parallel", return_value=parallel), patch_communicator("is_moe_input_scattered_across_dp_ranks", return_value=a2a), patch_communicator("enable_moe_dense_fully_dp", return_value=False), patch_communicator("_generic_prefill_cp_shards_tokens", return_value=False), @@ -155,17 +164,6 @@ class TestStageEdges(CustomTestCase): (SumGroup.TP, False, True), ) - def test_each_stage_reads_and_writes_as_a_layer_s_attention_or_ffn(self): - for pattern, tp in itertools.product(self.PATTERNS, (1, 2)): - with self.subTest(pattern=pattern, tp=tp): - for kind, layer in zip(pattern, stages(pattern, tp=tp)): - into, out_of = layer.edges - self.assertIs( - into.need.read, - NORM_QUANT_READ if utils.is_attn_layer(kind) else NORM_READ, - ) - self.assertIs(out_of.produced.update, ADD) - class TestMixerExit(CustomTestCase): """A mixer skips its output all-reduce when its output always leaves the sum @@ -189,15 +187,18 @@ class TestMixerExit(CustomTestCase): ) communicator = SimpleNamespace( plan=SimpleNamespace( - _batch_steps=lambda batch: SimpleNamespace(ffn_output=produced), + _batch_steps=lambda batch: SimpleNamespace(output=produced), ), _ffn_sum_can_move_to_next_layer=MagicMock(return_value=movable), ) hidden = torch.ones(2, 4) with get_parallel().override(tp_group=tp_group, tp_size=2): - with MixerExit(communicator, None) as mixer_exit: + with MixerExit( + communicator, None, stream=ResidualStream() + ) as mixer_exit: skipped = should_skip_mlp_all_reduce() output = mixer_exit.finish(hidden) + output, _ = mixer_exit._stream.input(output) self.assertFalse(should_skip_mlp_all_reduce()) hands_on = may and movable self.assertEqual(mixer_exit.skips_reduction, always or hands_on) diff --git a/test/registered/unit/models/test_step3p5_dense_reduce_scatter.py b/test/registered/unit/models/test_step3p5_dense_reduce_scatter.py index 9022c45ee9..672b7f73c1 100644 --- a/test/registered/unit/models/test_step3p5_dense_reduce_scatter.py +++ b/test/registered/unit/models/test_step3p5_dense_reduce_scatter.py @@ -1,6 +1,3 @@ -from sglang.srt.layers.communicator.residual import batch as residual_batch -from sglang.test.boundary_fixtures import stub_plan - """A Step-3.5 dense layer must leave its all-reduce out when postprocess reduce-scatters its output; otherwise the output is summed twice. The layer's FFN exit publishes the decision while the MLP runs, then completes it.""" @@ -15,6 +12,7 @@ import torch from sglang.srt.layers import communicator as comm from sglang.srt.layers.communicator.residual.stream import ResidualStream from sglang.srt.runtime_context import get_forward +from sglang.test.boundary_fixtures import stub_plan from sglang.test.ci.ci_register import register_cpu_ci from sglang.test.test_utils import CustomTestCase @@ -36,11 +34,11 @@ def dense_layer(*, reduce_scatter): complete = MagicMock(side_effect=lambda h, r: (h, r)) plan = stub_plan() plan._batch_steps = lambda fb: SimpleNamespace( - ffn_output=comm.StageOutput(comm.Layout(frozenset())), + output=comm.StageOutput(comm.Layout(frozenset())), returns_over_dp=False, - ffn_output_move_completes_sum=False, + output_move_completes_sum=False, ) - plan.output._select_ffn_completion = lambda fb: comm.FfnCompletion( + plan.output._select_ffn_completion = lambda fb, steps: comm.FfnCompletion( defer_moe_finalize=False, fuse_mlp_allreduce=False, mlp_reduce_scatter=reduce_scatter, @@ -48,9 +46,7 @@ def dense_layer(*, reduce_scatter): ) def stage_exit(fb): - result = plan.output.ffn_exit(fb) - result._stream = fb.residual_stream - return result + return plan.output.ffn_exit(fb, stream=fb.residual_stream) object.__setattr__( layer, @@ -79,9 +75,6 @@ class TestStep3p5DenseReduceScatter(CustomTestCase): for reduce_scatter in (False, True): with self.subTest(reduce_scatter=reduce_scatter): layer, seen, complete = dense_layer(reduce_scatter=reduce_scatter) - batch = SimpleNamespace(residual_stream=None) - stream = residual_batch.start(batch) - stream.write(torch.ones(2, 4)) layer.forward( positions=None, hidden_states=torch.ones(2, 4), diff --git a/test/registered/unit/server_args/test_server_args.py b/test/registered/unit/server_args/test_server_args.py index f2a57ae834..67010b1306 100644 --- a/test/registered/unit/server_args/test_server_args.py +++ b/test/registered/unit/server_args/test_server_args.py @@ -1465,6 +1465,7 @@ class TestContextParallelServerArgs(CustomTestCase): ) server_args._model_config = SimpleNamespace( hf_config=SimpleNamespace(architectures=["DeepseekV32ForCausalLM"]), + hf_text_config=SimpleNamespace(model_type="deepseek_v32"), is_multimodal=False, ) diff --git a/test/registered/unit/test_model_overrides.py b/test/registered/unit/test_model_overrides.py index 2be2dc164c..1ff4aa3f56 100644 --- a/test/registered/unit/test_model_overrides.py +++ b/test/registered/unit/test_model_overrides.py @@ -49,6 +49,100 @@ class _FakeArgs: metadata_but_not_overridable: A[bool, Arg(help="z")] = False +class TestBoundaryParallelismResolution(CustomTestCase): + def config(self, **options): + return SimpleNamespace( + **dict( + dict( + model_path="dummy", + attn_cp_size=1, + enable_prefill_cp=False, + enable_attn_tp_input_scattered=False, + moe_dp_size=1, + cp_strategy=None, + tp_size=4, + dp_size=1, + enable_aiter_allreduce_fusion=False, + ), + **options, + ) + ) + + def test_input_scattered_is_resolved_only_for_forwards_without_the_scope(self): + from sglang.srt.arg_groups.overrides import resolving_view + from sglang.srt.arg_groups.parallel_hook import handle_context_parallelism + + for model_type in ( + "nemotron_h", + "nemotron_h_puzzle", + "longcat_flash", + "deepseek_v3", + ): + with self.subTest(model_type=model_type): + cfg = self.config(enable_attn_tp_input_scattered=True) + model = SimpleNamespace( + hf_config=SimpleNamespace(architectures=["TestModel"]), + hf_text_config=SimpleNamespace(model_type=model_type), + ) + with ( + patch("sglang.srt.arg_groups.parallel_hook.run_hook"), + patch( + "sglang.srt.arg_groups.parallel_hook.model_config_of", + return_value=model, + ), + patch("sglang.srt.layers.cp.base.init_cp_strategy"), + ): + handle_context_parallelism(cfg) + self.assertTrue(cfg.enable_attn_tp_input_scattered) + self.assertEqual( + resolving_view(cfg).enable_attn_tp_input_scattered, + model_type == "deepseek_v3", + ) + + def test_context_parallel_model_support(self): + from sglang.srt.arg_groups.parallel_hook import handle_context_parallelism + + cases = ( + ("nemotron_h", 2, False, "Nemotron-H.*--attn-cp-size"), + ("nemotron_h_puzzle", 2, True, "Nemotron-H.*--attn-cp-size"), + ("longcat_flash", 2, True, "LongCat-Flash.*--enable-prefill-cp"), + ("nemotron_h", 1, False, None), + ("longcat_flash", 1, True, None), + ("longcat_flash", 2, False, None), + ("deepseek_v3", 2, True, None), + ) + for model_type, cp_size, prefill, error in cases: + with self.subTest(model_type=model_type, cp_size=cp_size, prefill=prefill): + cfg = self.config( + attn_cp_size=cp_size, + enable_prefill_cp=prefill, + cp_strategy="interleave" if prefill else None, + ) + model = SimpleNamespace( + hf_config=SimpleNamespace(architectures=["TestModel"]), + hf_text_config=SimpleNamespace(model_type=model_type), + ) + with ( + patch("sglang.srt.arg_groups.parallel_hook.run_hook"), + patch( + "sglang.srt.arg_groups.parallel_hook.model_config_of", + return_value=model, + ), + patch("sglang.srt.layers.cp.base.init_cp_strategy") as init_cp, + ): + if error is not None: + with self.assertRaisesRegex(ValueError, error): + handle_context_parallelism(cfg) + init_cp.assert_not_called() + else: + handle_context_parallelism(cfg) + init_cp.assert_called_once_with( + enable_prefill_cp=prefill, + cp_size=cp_size, + cp_strategy=cfg.cp_strategy, + ) + + class TestModelOverridableWhitelist(CustomTestCase): def test_whitelist_derivation_from_annotated_metadata(self): self.assertEqual( @@ -78,6 +172,7 @@ class TestModelOverridableWhitelist(CustomTestCase): "moe_runner_backend", "quantization", "enable_dp_attention", + "enable_attn_tp_input_scattered", "enable_dp_lm_head", "enable_tp_lm_head_all_to_all", "moe_a2a_backend",