diff --git a/docs/docs/advanced_features/server_arguments.mdx b/docs/docs/advanced_features/server_arguments.mdx
index 2a09a0c8c0..1dc5e89379 100644
--- a/docs/docs/advanced_features/server_arguments.mdx
+++ b/docs/docs/advanced_features/server_arguments.mdx
@@ -1551,6 +1551,12 @@ Combining `--enable-response-store` with `--disaggregation-mode=prefill` or `dec
The path of the draft model weights. This can be a local folder or a Hugging Face repo ID. |
`None` |
Type: str |
+
+
+ speculative_boundary_reduction Server args field; no CLI flag |
+ Resolved FFN boundary reduction policy for the speculative draft. Argument processing derives this field from boundary_reduction and the draft model configuration; an independent value is overwritten by resolution. See FFN boundary reduction. |
+ rs+rsv before resolution; model-dependent afterward |
+ ar, rs, rsv, rs+rsv |
| `--speculative-draft-model-revision` |
@@ -2520,6 +2526,12 @@ Combining `--enable-response-store` with `--disaggregation-mode=prefill` or `dec
`False` |
bool flag (set to enable) |
+
+ --boundary-reduction |
+ Select the optional FFN output reduction strategy. Unsupported reduce-scatter paths fall back to all-reduce followed by token redistribution. See FFN boundary reduction. |
+ auto |
+ auto, ar, rs, rsv, rs+rsv |
+
| `--disable-custom-all-reduce` |
Disable the custom all-reduce kernel and fall back to NCCL. |
@@ -2848,6 +2860,26 @@ Combining `--enable-response-store` with `--disaggregation-mode=prefill` or `dec
+### FFN boundary reduction
+
+Use `--boundary-reduction` to choose which optional FFN output reductions may combine the sum with token redistribution. The selected policy applies to both dense and MoE FFN layers. It currently controls models using independently constructed stage boundaries; models still using `LayerCommunicator` retain their existing reduction policy.
+
+| Value | Behavior |
+| --- | --- |
+| `auto` | Resolve the model default during server-argument processing, before model construction. |
+| `ar` | Use all-reduce followed by token redistribution where needed. |
+| `rs` | Permit fixed-size reduce-scatter when the layout and backend support it; otherwise use `ar`. |
+| `rsv` | Permit variable-size reduce-scatter for eligible attention-DP paths; otherwise use `ar`. |
+| `rs+rsv` | Permit both, trying the eligible variable-size path first, then fixed-size, then `ar`. |
+
+`auto` selects `ar` for dense Qwen3 language backbones (including Qwen3-VL wrappers) and Moss-VL; `rsv` for BailingMoELinear/BailingMoeV2_5, LongCat Flash and Step3-VL; and `rs+rsv` otherwise. Qwen3-MoE, Qwen3-Next and Qwen3.5 are not matched by the dense Qwen3 rule and resolve to `rs+rsv`.
+
+LongCat Flash and Moss-VL resolve these defaults, but still use the layer communicator at this revision and do not yet consume this option.
+
+With `auto`, an independent speculative draft resolves its own model default; same-model MTP inherits the target policy. An explicit value applies to both target and draft. The result is recorded in the `speculative_boundary_reduction` server args field (`no_cli=True`); it has no separate CLI flag or independent override during resolution.
+
+This option does not disable required attention or MoE collectives, including expert all-to-all, or select the all-reduce fusion backend. For example, `ar` can still use fused all-reduce plus residual normalization; use `--enforce-disable-flashinfer-allreduce-fusion` to disable that FlashInfer fusion.
+
## Dynamic batch tokenizer
diff --git a/python/sglang/srt/arg_groups/boundary_reduction.py b/python/sglang/srt/arg_groups/boundary_reduction.py
new file mode 100644
index 0000000000..d7ec8bc437
--- /dev/null
+++ b/python/sglang/srt/arg_groups/boundary_reduction.py
@@ -0,0 +1,88 @@
+"""Resolve collective preferences before constructing target and draft models."""
+
+import json
+
+from sglang.srt.arg_groups.model_override_base import model_config_of
+
+_RSV_ARCHITECTURES = {
+ "BailingMoELinearForCausalLM",
+ "BailingMoeV2_5ForCausalLM",
+ "LongcatFlashForCausalLM",
+ "LongcatFlashForCausalLMNextN",
+ "Step3VLForConditionalGeneration",
+}
+_AR_ARCHITECTURES = {
+ "Qwen3ForCausalLM",
+ "Qwen3ForSequenceClassification",
+ "Qwen3Model",
+ "Qwen3ForRewardModel",
+ "MossVLForCausalLM",
+ "MossVLForConditionalGeneration",
+}
+
+
+def model_boundary_reduction(config):
+ """One default for a model's dense and MoE layers, including its MTP."""
+ architectures = set(getattr(config, "architectures", None) or ())
+ if architectures & _RSV_ARCHITECTURES:
+ return "rsv"
+ if architectures & _AR_ARCHITECTURES:
+ return "ar"
+ # MiniCPM-V 4.5 constructs Qwen3 from its top-level config.
+ if (
+ getattr(config, "model_type", None) == "minicpmv"
+ and str(getattr(config, "version", "")) == "4.5"
+ ):
+ return "ar"
+ # Wrappers carry their actual language backbone in the nested HF config.
+ # Match Qwen3 specifically: Qwen3-MoE/Next/3.5 fall through to rs+rsv.
+ if getattr(config, "model_type", None) in ("qwen3", "qwen3_vl_text"):
+ return "ar"
+ get_text = getattr(config, "get_text_config", None)
+ if get_text is not None:
+ text = get_text()
+ if text is not config:
+ return model_boundary_reduction(text)
+ for name in ("text_config", "llm_config", "language_config"):
+ text = getattr(config, name, None)
+ if text is not None and text is not config:
+ return model_boundary_reduction(text)
+ return "rs+rsv"
+
+
+def resolve_boundary_reduction(view):
+ """Resolve auto once, after speculative paths/revisions have been resolved."""
+ requested = view.boundary_reduction
+ dummy = view.model_path.lower() in ("none", "dummy")
+ if requested != "auto":
+ target = draft = requested
+ else:
+ target = draft = (
+ "rs+rsv"
+ if dummy
+ else model_boundary_reduction(model_config_of(view).hf_config)
+ )
+ draft_path = view.speculative_draft_model_path
+ if (
+ not dummy
+ and view.speculative_algorithm is not None
+ and draft_path is not None
+ and draft_path != view.model_path
+ ):
+ from sglang.srt.utils.hf_transformers_utils import get_config
+
+ kwargs = {}
+ if view.decrypted_draft_config_file:
+ kwargs["_configuration_file"] = view.decrypted_draft_config_file
+ config = get_config(
+ draft_path,
+ trust_remote_code=view.trust_remote_code,
+ revision=view.speculative_draft_model_revision,
+ model_override_args=json.loads(view.json_model_override_args),
+ **kwargs,
+ )
+ draft = model_boundary_reduction(config)
+ return {
+ "boundary_reduction": target,
+ "speculative_boundary_reduction": draft,
+ }
diff --git a/python/sglang/srt/arg_groups/field_order.py b/python/sglang/srt/arg_groups/field_order.py
index 4074ce6b5e..20c35da680 100644
--- a/python/sglang/srt/arg_groups/field_order.py
+++ b/python/sglang/srt/arg_groups/field_order.py
@@ -250,6 +250,7 @@ POSITIONAL_FIELD_ORDER = (
"enable_torch_symm_mem",
"enable_scattered_sconv",
"pre_warm_nccl",
+ "boundary_reduction",
"enable_quant_communications",
"enable_flashinfer_allreduce_fusion",
"enforce_disable_flashinfer_allreduce_fusion",
diff --git a/python/sglang/srt/arg_groups/fields/exec_.py b/python/sglang/srt/arg_groups/fields/exec_.py
index bc3edcde58..9b98e144e2 100644
--- a/python/sglang/srt/arg_groups/fields/exec_.py
+++ b/python/sglang/srt/arg_groups/fields/exec_.py
@@ -620,6 +620,19 @@ class ExecComm(msgspec.Struct):
bool,
"Pre-warm NCCL/RCCL communicators during startup to reduce P99 TTFT cold-start latency. Default: enabled for AMD/HIP (RCCL), disabled for NVIDIA/CUDA (NCCL).",
] = False
+ boundary_reduction: A[
+ Literal["auto", "ar", "rs", "rsv", "rs+rsv"],
+ Arg(
+ help="Select FFN boundary reduction: ar uses all-reduce then token "
+ "redistribution; rs and rsv permit fixed-size and variable-size "
+ "reduce-scatter respectively; rs+rsv permits both (RSv first). "
+ "Unsupported paths fall back to ar. The option applies only to FFN "
+ "layers using stage boundaries, where auto resolves the model default. "
+ "Required attention and MoE collectives and "
+ "all-reduce fusion are unaffected.",
+ resolvable=True,
+ ),
+ ] = "auto"
enable_quant_communications: A[
Optional[bool],
"Enable INT8 quantization of TP communications (limited support).",
diff --git a/python/sglang/srt/arg_groups/fields/spec.py b/python/sglang/srt/arg_groups/fields/spec.py
index ac91102fdc..71154d8188 100644
--- a/python/sglang/srt/arg_groups/fields/spec.py
+++ b/python/sglang/srt/arg_groups/fields/spec.py
@@ -39,6 +39,10 @@ class Spec(msgspec.Struct):
aliases=["--speculative-draft-model"],
),
] = None
+ speculative_boundary_reduction: A[
+ Literal["ar", "rs", "rsv", "rs+rsv"],
+ Arg(no_cli=True, resolvable=True),
+ ] = "rs+rsv"
speculative_draft_model_revision: A[
Optional[str],
"The specific draft model version to use. It can be a branch name, a tag name, or a commit id. If unspecified, will use the default version.",
diff --git a/python/sglang/srt/arg_groups/pipeline.py b/python/sglang/srt/arg_groups/pipeline.py
index d7490eefe3..3963f548c1 100644
--- a/python/sglang/srt/arg_groups/pipeline.py
+++ b/python/sglang/srt/arg_groups/pipeline.py
@@ -92,6 +92,9 @@ def run_resolution_pipeline(server_args: Any) -> None:
run_hook(validate_prefill_cp_platform, server_args)
run_hook(handle_hardware_runtime_validation, server_args)
if cfg.model_path.lower() in ["none", "dummy"]:
+ from sglang.srt.arg_groups.boundary_reduction import resolve_boundary_reduction
+
+ run_post_process_pass(server_args, resolve_boundary_reduction)
return
from sglang.srt.arg_groups.model_path_hook import (
@@ -284,6 +287,10 @@ def run_resolution_pipeline(server_args: Any) -> None:
run_hook(handle_speculative_decoding, server_args)
+ from sglang.srt.arg_groups.boundary_reduction import resolve_boundary_reduction
+
+ run_post_process_pass(server_args, resolve_boundary_reduction)
+
# After the speculative hook so speculative_algorithm is final.
from sglang.srt.arg_groups.layernorm_sp_hook import handle_layernorm_sp
diff --git a/python/sglang/srt/batch_overlap/two_batch_overlap.py b/python/sglang/srt/batch_overlap/two_batch_overlap.py
index 32f3003729..6404bb7828 100644
--- a/python/sglang/srt/batch_overlap/two_batch_overlap.py
+++ b/python/sglang/srt/batch_overlap/two_batch_overlap.py
@@ -15,7 +15,6 @@ from sglang.srt.batch_overlap.operations import (
from sglang.srt.batch_overlap.operations_strategy import OperationsStrategy
from sglang.srt.layers import deep_gemm_wrapper
from sglang.srt.layers.communicator import (
- CommunicateContext,
Layout,
tbo_split_moves,
)
@@ -1022,14 +1021,12 @@ def _model_forward_tbo_split_inputs(
zero_allocator: Optional[BumpAllocator],
layer_input_rows: Layout,
) -> List[Dict]:
- context = CommunicateContext.init_new()
to_splitter, to_layer_input = tbo_split_moves(layer_input_rows)
hidden_states, residual = to_splitter(
hidden_states=hidden_states,
residual=residual,
forward_batch=forward_batch,
- context=context,
)
inputs_arr = _model_forward_tbo_split_inputs_raw(
@@ -1045,7 +1042,6 @@ def _model_forward_tbo_split_inputs(
hidden_states=hidden_states,
residual=residual,
forward_batch=forward_batch,
- context=context,
)
return dict(
hidden_states=hidden_states,
diff --git a/python/sglang/srt/layers/communicator/__init__.py b/python/sglang/srt/layers/communicator/__init__.py
index 6eddee083f..57e5e7a0a6 100644
--- a/python/sglang/srt/layers/communicator/__init__.py
+++ b/python/sglang/srt/layers/communicator/__init__.py
@@ -11,7 +11,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
-"""Communication between a decoder layer's stages."""
+"""Stage boundaries: layout transport, residual updates, reads, and fusion."""
from sglang.srt.layers.communicator.adapters.attention import (
AttentionInputs,
@@ -19,7 +19,6 @@ from sglang.srt.layers.communicator.adapters.attention import (
)
from sglang.srt.layers.communicator.boundary import (
BoundarySteps,
- DecoderLayerSides,
EdgeDecl,
FusedMlpInput,
LayerStage,
@@ -28,25 +27,25 @@ from sglang.srt.layers.communicator.boundary import (
StageInput,
StageKind,
StageOutput,
- decoder_layer_edges,
- decoder_layer_sides,
- input_scattered_layer_sides,
make_boundary,
make_output_boundary,
- sequence_parallel_layer_sides,
stage_edges,
tbo_split_moves,
)
-from sglang.srt.layers.communicator.layer import (
- FfnCompletion,
- FfnExit,
- LayerCommunicator,
- LayerFacts,
- MHCLayerCommunicator,
- MixerExit,
+from sglang.srt.layers.communicator.construction import (
+ BatchVariant,
+ StageEdges,
+ make_stage,
+)
+from sglang.srt.layers.communicator.exit import FfnCompletion, FfnExit, MixerExit
+from sglang.srt.layers.communicator.factories import (
+ declare_attn,
+ declare_ffn,
+ make_attn_stage,
+ make_ffn_stage,
+ make_stages,
)
from sglang.srt.layers.communicator.layout import (
- CommunicateContext,
Layout,
SumGroup,
TokenAxis,
@@ -79,25 +78,28 @@ from sglang.srt.layers.communicator.residual.mhc import (
)
__all__ = [
+ "LayerCommunicator",
+ "MHCLayerCommunicator",
+ "LayerFacts",
+ "declare_attn",
+ "declare_ffn",
+ "make_attn_stage",
+ "make_ffn_stage",
+ "make_stages",
"ADD",
"AttentionInputs",
"BoundarySteps",
- "CommunicateContext",
"CommunicateSimpleFn",
"CommunicateSummableTensorPairFn",
- "DecoderLayerSides",
"EdgeDecl",
"FUSE_ALLREDUCE_MAX_BATCH_SIZE",
"FfnCompletion",
"FfnExit",
"FusedMlpInput",
"HandoffOutput",
- "LayerCommunicator",
"LayerResidual",
- "LayerFacts",
"LayerStage",
"Layout",
- "MHCLayerCommunicator",
"MHCState",
"MixerExit",
"NORM_QUANT_READ",
@@ -111,20 +113,22 @@ __all__ = [
"SumGroup",
"TokenAxis",
"UnreducedOutput",
- "decoder_layer_edges",
- "decoder_layer_sides",
"enable_moe_dense_fully_dp",
"get_attn_tp_context",
- "input_scattered_layer_sides",
"make_boundary",
"make_output_boundary",
"moe_cp_gathers_sparse_moe_input",
"move_rows",
"reduce_output",
- "sequence_parallel_layer_sides",
"sparse_moe_gathers_over_moe_cp",
"stage_edges",
"tbo_split_moves",
"token_axis_sizes",
"tp_reduce_scatter",
]
+
+from sglang.srt.layers.communicator.layer import (
+ LayerCommunicator,
+ LayerFacts,
+ MHCLayerCommunicator,
+)
diff --git a/python/sglang/srt/layers/communicator/adapters/attention.py b/python/sglang/srt/layers/communicator/adapters/attention.py
index 5050a597f6..3027a45203 100644
--- a/python/sglang/srt/layers/communicator/adapters/attention.py
+++ b/python/sglang/srt/layers/communicator/adapters/attention.py
@@ -21,7 +21,6 @@ from typing import Callable
import torch
from sglang.srt.layers.communicator.layout import (
- CommunicateContext,
enable_moe_dense_fully_dp,
)
from sglang.srt.layers.dp_attention import (
@@ -182,7 +181,6 @@ def _redistribute_from_attn_tp_shards(tensor: torch.Tensor) -> torch.Tensor:
return gathered
-def _redistribute_to_attn_tp_shards(
- tensor: torch.Tensor, context: CommunicateContext
-) -> torch.Tensor:
- return tensor.tensor_split(context.attn_tp_size)[context.attn_tp_rank]
+def _redistribute_to_attn_tp_shards(tensor: torch.Tensor) -> torch.Tensor:
+ parallel = get_parallel()
+ return tensor.tensor_split(parallel.attn_tp_size)[parallel.attn_tp_rank]
diff --git a/python/sglang/srt/layers/communicator/adapters/branch.py b/python/sglang/srt/layers/communicator/adapters/branch.py
new file mode 100644
index 0000000000..1bef5817c5
--- /dev/null
+++ b/python/sglang/srt/layers/communicator/adapters/branch.py
@@ -0,0 +1,87 @@
+# Copyright 2023-2024 SGLang Team
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Move prepared branch inputs and merge complete branch contributions."""
+
+from __future__ import annotations
+
+from typing import Tuple
+
+import torch
+
+from sglang.srt.layers.communicator.ops import move_rows
+from sglang.srt.layers.communicator.residual.stream import ResidualStream
+from sglang.srt.model_executor.forward_batch_info import ForwardBatch
+
+
+def branch_input(
+ self,
+ source: object,
+ hidden_states: torch.Tensor,
+ residual: torch.Tensor,
+ forward_batch: ForwardBatch,
+) -> Tuple[torch.Tensor, torch.Tensor]:
+ """The FFN input and residual that ``source``'s boundary read for its
+ own FFN, for this layer's FFN, which branches from the same input: moved
+ to the rows this FFN needs and its residual's rows."""
+ rows, residual_rows, _ = source._branch_rows(forward_batch)
+ to, residual_to, _ = self._branch_rows(forward_batch)
+ stream = residual if isinstance(residual, ResidualStream) else None
+ if stream is not None:
+ if stream.pending is not None:
+ raise RuntimeError("a branch must start from a prepared stage input")
+ residual = stream.residual
+ hidden_states = move_rows(hidden_states, rows, to, forward_batch)
+ residual = move_rows(residual, residual_rows, residual_to, forward_batch)
+ return (
+ hidden_states,
+ ResidualStream(residual) if stream is not None else residual,
+ )
+
+
+def branch_output(
+ self, hidden_states: torch.Tensor, forward_batch: ForwardBatch
+) -> torch.Tensor:
+ """This layer's complete FFN output as a branch's contribution, which
+ adds to the layer's output without writing the residual: moved to the
+ rows the layer hands on."""
+ rows, _, to = self._branch_rows(forward_batch)
+ return move_rows(hidden_states, rows, to, forward_batch)
+
+
+def merge_branch(
+ self,
+ contribution: torch.Tensor,
+ hidden_states: torch.Tensor,
+ residual: torch.Tensor,
+ source: object,
+ forward_batch: ForwardBatch,
+) -> Tuple[torch.Tensor, torch.Tensor]:
+ """A contribution from ``branch_output`` summed with what ``source``'s
+ layer hands on, ``hidden_states`` and ``residual``, moved to the rows
+ this layer hands on."""
+ _, _, rows = source._branch_rows(forward_batch)
+ _, _, to = self._branch_rows(forward_batch)
+ stream = residual if isinstance(residual, ResidualStream) else None
+ update = None
+ if stream is not None:
+ stream.check(hidden_states)
+ update = stream.pending.update
+ hidden_states, residual = stream.finish(hidden_states)
+ hidden_states = move_rows(hidden_states, rows, to, forward_batch)
+ residual = move_rows(residual, rows, to, forward_batch)
+ output = contribution + hidden_states
+ if stream is not None:
+ stream.write(residual)
+ return stream.leave(output, update), stream
+ return output, residual
diff --git a/python/sglang/srt/layers/communicator/adapters/lora.py b/python/sglang/srt/layers/communicator/adapters/lora.py
new file mode 100644
index 0000000000..f971e31f19
--- /dev/null
+++ b/python/sglang/srt/layers/communicator/adapters/lora.py
@@ -0,0 +1,32 @@
+# Copyright 2023-2024 SGLang Team
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Publish the token rows consumed by the selected stage to LoRA kernels."""
+
+from sglang.srt.layers.communicator.layout import TokenAxis
+from sglang.srt.runtime_context import LoRABatchLayout, get_forward
+
+
+def publish_attention(enabled):
+ if enabled:
+ get_forward().set("lora_batch_layout", LoRABatchLayout.DP_LOCAL)
+
+
+def publish_ffn(enabled, input_rows):
+ if enabled:
+ get_forward().set(
+ "lora_batch_layout",
+ LoRABatchLayout.DP_LOCAL
+ if TokenAxis.ATTN_DP in input_rows.sharded
+ else LoRABatchLayout.TP_GLOBAL,
+ )
diff --git a/python/sglang/srt/layers/communicator/boundary.py b/python/sglang/srt/layers/communicator/boundary.py
index f866fce27e..3fb4b4438d 100644
--- a/python/sglang/srt/layers/communicator/boundary.py
+++ b/python/sglang/srt/layers/communicator/boundary.py
@@ -51,6 +51,7 @@ from sglang.srt.layers.communicator.ops import (
tp_reduce_scatter,
tp_slice,
)
+from sglang.srt.layers.communicator.output import OutputTransform
from sglang.srt.layers.communicator.residual import (
LayerResidual,
StageRead,
@@ -102,6 +103,7 @@ class StageOutput(msgspec.Struct, frozen=True):
# How the producer's output is written into the residual. An arrival
# description has no producer object; its edge declares capabilities only.
update: Optional[StageUpdate] = ADD
+ transform: Optional[OutputTransform] = None
class StageDecl(msgspec.Struct, frozen=True):
@@ -481,6 +483,7 @@ class FusedMlpInput(msgspec.Struct, frozen=True):
# The group whose sum it completes.
completes: SumGroup
run: Callable[..., Optional[Tuple[torch.Tensor, torch.Tensor]]]
+ preserves_residual: Optional[Callable] = None
def tbo_split_moves(layer_input_rows: Layout) -> Tuple[Callable, Callable]:
@@ -536,12 +539,12 @@ class StageEntry(msgspec.Struct, frozen=True):
# Completes what the input owes, writes the previous stage's output into
# the residual and reads this stage's input:
- # (hidden_states, residual, forward_batch, norm, context, **call).
+ # (hidden_states, residual, forward_batch, norm, **call).
prepare: Callable
# The rows prepare hands the stage.
input_rows: Layout
# Moves the input onto the stage's rows after prepare, when prepare does
- # not: (hidden_states, forward_batch, context) -> hidden_states.
+ # not: (hidden_states, forward_batch) -> hidden_states.
input_move: Optional[Callable] = None
# Hands the stage its input once it is on its rows:
# (hidden_states, forward_batch, qkv_latent_func) -> hidden_states.
@@ -551,6 +554,7 @@ class StageEntry(msgspec.Struct, frozen=True):
# Return the updated residual to the producer's rows for aux capture.
capture_move: Optional[Callable] = None
capture_move_allocates: bool = False
+ preserves_residual: Optional[Callable] = None
capture_preserves_residual: Optional[Callable] = None
input_sum: Optional[SumGroup] = None
@@ -619,6 +623,7 @@ class Boundary(msgspec.Struct, frozen=True):
output_move: Optional[Callable] = None
# Whether output_move also completes the sum the producer leaves.
output_move_completes_sum: bool = False
+ preserves_residual: Optional[Callable] = None
@property
def input_rows(self) -> Layout:
@@ -682,6 +687,7 @@ def make_boundary(
raise NotImplementedError("update capabilities require different input moves")
return msgspec.structs.replace(
first,
+ preserves_residual=None,
prepare=partial(
_dispatch_consumer,
paths={capability: path.prepare for capability, path in paths.items()},
@@ -756,8 +762,24 @@ def _bind_consumer(
cp_moves=cp_moves,
enters_stack=False,
)
+ preserves_residual = None
+ if (
+ edge.produced.always_leaves
+ and edge.produced.group is SumGroup.ATTN_TP
+ and isinstance(step, partial)
+ and step.func is _mlp_input_without_dp
+ and not step.keywords["gathers_residual"]
+ ):
+ selected = step.keywords["fusions"]
+ if selected:
+ # Only the first reachable candidate can certify ownership; an
+ # earlier custom candidate could otherwise mutate the residual.
+ preserves_residual = next(
+ f.preserves_residual for f in fusions if f.run == selected[0]
+ )
return Boundary(
edge,
+ preserves_residual=preserves_residual,
prepare=partial(
_consumer_step,
step=step,
diff --git a/python/sglang/srt/layers/communicator/construction.py b/python/sglang/srt/layers/communicator/construction.py
new file mode 100644
index 0000000000..91ca1f04d7
--- /dev/null
+++ b/python/sglang/srt/layers/communicator/construction.py
@@ -0,0 +1,307 @@
+# Copyright 2023-2024 SGLang Team
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Bind one stage to predeclared boundaries and reusable batch paths."""
+
+from dataclasses import dataclass
+from enum import Enum, auto
+from functools import cached_property
+from typing import Callable, Optional, Protocol, Tuple
+
+from sglang.srt.environ import envs
+from sglang.srt.layers.communicator.adapters.attention import get_attn_tp_context
+from sglang.srt.layers.communicator.boundary import (
+ BoundarySteps,
+ EdgeDecl,
+ FusedMlpInput,
+ StageDecl,
+ StageEntry,
+ StageKind,
+ make_boundary,
+ make_output_boundary,
+)
+from sglang.srt.layers.communicator.exit import OutputBoundary
+from sglang.srt.layers.communicator.fusions.allreduce import (
+ attention_fusions,
+ ffn_fusions,
+)
+from sglang.srt.layers.communicator.layout import (
+ TokenAxis,
+ _batch_shards_over_cp,
+ _gathers_over_attention_cp,
+ _generic_prefill_cp_shards_tokens,
+ enable_moe_dense_fully_dp,
+)
+from sglang.srt.layers.communicator.stage import StageCommunicator
+from sglang.srt.layers.moe import (
+ get_moe_a2a_backend,
+)
+from sglang.srt.model_executor.forward_batch_info import ForwardBatch
+from sglang.srt.runtime_context import (
+ get_forward,
+ get_lora,
+ get_parallel,
+ get_spec,
+)
+from sglang.srt.speculative.spec_info import SpeculativeAlgorithm
+
+_use_ag_after_qlora = envs.SGLANG_USE_AG_AFTER_QLORA.get()
+
+
+class BoundaryFusions(Protocol):
+ """The fused kernels a fusion backend gives a layer, bound to it and tried
+ before the layer's own: at its attention input (each takes what the previous
+ layer left, the residual, the batch and a post-residual addition), at its
+ FFN input, and at its FFN exit (each says what the exit does when its kernel
+ takes the batch)."""
+
+ def attention_input(self, layer: "StagePlan") -> Tuple[Callable, ...]: ...
+
+ def ffn_input(self, layer: "StagePlan") -> Tuple["FusedMlpInput", ...]: ...
+
+ requires_local_reduction: bool
+
+ def can_defer_finalize(
+ self, layer: "StagePlan", forward_batch: ForwardBatch
+ ) -> bool: ...
+
+
+def _refuse_uncovered_cp_moe(moe_on_local_rows: bool, cp_shards: bool) -> None:
+ """A MoE layer under attention CP whose tokens the steps cannot bring
+ to it: under a prefill CP, one dispatched per DP shard under attention DP
+ and GQA CP, and one on the TP group whose data-parallel groups are the CP
+ ranks under DSA or MLA CP; and under attention DP, one on the TP group
+ whose data-parallel groups are the CP ranks."""
+ parallel = get_parallel()
+ gqa = not _gathers_over_attention_cp()
+ if moe_on_local_rows:
+ if cp_shards and gqa and parallel.attn_dp_size > 1:
+ raise NotImplementedError(
+ "a MoE dispatched per DP shard under attention DP and GQA prefill CP"
+ )
+ elif parallel.moe_dp_size == parallel.attn_cp_size:
+ if cp_shards and not gqa:
+ raise NotImplementedError(
+ "a MoE on the TP group with moe_dp_size == attn_cp_size under "
+ "DSA or MLA prefill CP"
+ )
+ if parallel.attn_dp_size > 1:
+ raise NotImplementedError(
+ "a MoE on the TP group with moe_dp_size == attn_cp_size under "
+ "attention DP and attention CP"
+ )
+
+
+def _input_can_be_scattered() -> bool:
+ """Whether a batch may run this layer with input-scattered attention:
+ configured, on TP without attention DP, a prefill CP, an a2a backend or
+ a dense MLP on every rank. The rest of what ``AttnTpContext.init_context`` requires is only
+ known once the model is built."""
+ parallel = get_parallel()
+ return (
+ parallel.enable_attn_tp_input_scattered
+ and parallel.tp_size > 1
+ and parallel.attn_dp_size == 1
+ and not _generic_prefill_cp_shards_tokens()
+ and get_moe_a2a_backend().is_none()
+ and not enable_moe_dense_fully_dp()
+ )
+
+
+class BatchVariant(Enum):
+ ORDINARY = auto()
+ CONTEXT_PARALLEL = auto()
+ INPUT_SCATTERED = auto()
+ SEQUENCE_PARALLEL = auto()
+
+
+@dataclass(frozen=True)
+class StageEdges:
+ """One stage's incoming and outgoing declarations for one batch variant.
+
+ No adjacent stage object is required. Edges can come from a local model
+ assembler, pipeline contract or branch adapter.
+ """
+
+ incoming: EdgeDecl
+ outgoing: EdgeDecl
+ handoff: Optional[Callable] = None
+ cp_moves: object = None
+
+
+def _requires_branch_input(*args, **kwargs):
+ raise RuntimeError("a prepared branch must enter through branch_input, not prepare")
+
+
+class StagePlan:
+ """Precomputed paths for one stage; never owns the neighbouring norm."""
+
+ def __init__(
+ self,
+ kind,
+ norm,
+ variants,
+ *,
+ enters_stack=False,
+ prepared_input=False,
+ terminal=False,
+ fixed_output=False,
+ is_sparse=False,
+ qkv_latent_func=None,
+ force_layernorm_before_dp_gather=False,
+ enable_fused_ar_quant=False,
+ fused_ar_quant_keep_bf16=False,
+ residual_in_hidden=False,
+ fusions=None,
+ ):
+ self.kind = kind
+ self.norm = norm
+ self.variants = dict(variants)
+ self.is_first_layer = enters_stack
+ self.is_last_layer = terminal
+ self.fixed_output = fixed_output
+ self.is_sparse = is_sparse
+ self.qkv_latent_func = qkv_latent_func
+ self.enable_fused_ar_quant = enable_fused_ar_quant
+ self.fused_ar_quant_keep_bf16 = fused_ar_quant_keep_bf16
+ self.residual_in_hidden = residual_in_hidden
+ self.fusions = fusions
+ self._speculative_algo = SpeculativeAlgorithm.from_string(
+ get_spec().speculative_algorithm
+ )
+ self._publish_lora_layout = get_parallel().enable_dp_attention and bool(
+ get_lora().enable_lora
+ )
+ self._fusion_rows = None
+ carried = attention_fusions(self) if kind is StageKind.ATTENTION else ()
+ self._carried_fusions = carried
+ fused = ffn_fusions(self) if kind is StageKind.FFN else ()
+ self._paths = {}
+ for variant, edges in self.variants.items():
+ if prepared_input:
+ # The branch adapter moves an already-read input and forks its
+ # stream. Binding a normal prepare would add or norm it twice.
+ entry = StageEntry(
+ prepare=_requires_branch_input,
+ input_rows=edges.incoming.need.layout,
+ )
+ else:
+ into = make_boundary(
+ edges.incoming,
+ fusions=fused,
+ carried_fusions=carried,
+ force_layernorm_before_gather=force_layernorm_before_dp_gather,
+ cp_moves=edges.cp_moves,
+ enters_stack=enters_stack,
+ )
+ entry = StageEntry(
+ prepare=into.prepare,
+ input_rows=into.input_rows,
+ input_move=into.input_move,
+ handoff=edges.handoff,
+ fused=into.fused,
+ capture_move=into.capture_move,
+ capture_move_allocates=into.capture_move_allocates,
+ preserves_residual=into.preserves_residual,
+ input_sum=edges.incoming.produced.group
+ if edges.incoming.produced.always_leaves
+ else None,
+ )
+ out = (
+ None
+ if fixed_output
+ else make_output_boundary(edges.outgoing, cp_moves=edges.cp_moves)
+ )
+ self._paths[variant] = BoundarySteps(
+ attention=entry if kind is StageKind.ATTENTION else None,
+ ffn=entry if kind is StageKind.FFN else None,
+ ffn_output=edges.outgoing.produced,
+ ffn_output_move=None if out is None else out.output_move,
+ ffn_output_move_completes_sum=False
+ if out is None
+ else out.output_move_completes_sum,
+ ffn_sum_is_movable=edges.outgoing.produced.group is not None,
+ )
+ self._steps = self._paths[BatchVariant.ORDINARY]
+ self._cp_steps = self._paths.get(BatchVariant.CONTEXT_PARALLEL)
+ self._input_scattered_steps = self._paths.get(BatchVariant.INPUT_SCATTERED)
+ self._sp_steps = self._paths.get(BatchVariant.SEQUENCE_PARALLEL)
+
+ @property
+ def input_rows(self):
+ return self.variants[BatchVariant.ORDINARY].incoming.residual
+
+ @property
+ def input_on_attention_tp_slices(self):
+ return TokenAxis.ATTN_TP_SCATTER in self.input_rows.sharded
+
+ def _variant(self, forward_batch):
+ if self._sp_steps is not None and get_forward().sp_active:
+ return BatchVariant.SEQUENCE_PARALLEL
+ if (
+ self._input_scattered_steps is not None
+ and get_attn_tp_context().input_scattered
+ ):
+ return BatchVariant.INPUT_SCATTERED
+ if self._cp_steps is not None and _batch_shards_over_cp(forward_batch):
+ return BatchVariant.CONTEXT_PARALLEL
+ return BatchVariant.ORDINARY
+
+ def _batch_steps(self, forward_batch):
+ return self._paths[self._variant(forward_batch)]
+
+ def fusion_rows(self, forward_batch):
+ if self._fusion_rows is not None:
+ return self._fusion_rows[self._variant(forward_batch)]
+ entry = self._batch_steps(forward_batch)
+ return (entry.attention or entry.ffn).input_rows
+
+ def produced(self, kind, forward_batch):
+ return self._batch_steps(forward_batch).ffn_output
+
+ @cached_property
+ def output(self):
+ return OutputBoundary(self)
+
+ def _branch_rows(self, forward_batch):
+ if self._variant(forward_batch) is not BatchVariant.ORDINARY:
+ raise NotImplementedError("branch transport requires ordinary batch rows")
+ edges = self.variants[BatchVariant.ORDINARY]
+ return (
+ edges.incoming.need.layout,
+ edges.incoming.residual_to,
+ edges.outgoing.need.layout,
+ )
+
+
+def make_stage(
+ declaration: StageDecl,
+ *,
+ kind: StageKind,
+ norm,
+ incoming: EdgeDecl,
+ outgoing: EdgeDecl,
+ variants=None,
+ handoff=None,
+ **options,
+) -> StageCommunicator:
+ """Build one real stage from its own declaration and adjacent edges.
+
+ The optional variants supply the same stage's CP/SP/input-scattered
+ declarations. Neither a paired stage nor its layer facts/norm is needed.
+ """
+ if incoming.need != declaration.input or outgoing.produced != declaration.output:
+ raise ValueError("stage declaration disagrees with its boundaries")
+ paths = dict(variants or {})
+ paths[BatchVariant.ORDINARY] = StageEdges(incoming, outgoing, handoff)
+ return StageCommunicator(StagePlan(kind, norm, paths, **options), kind, norm)
diff --git a/python/sglang/srt/layers/communicator/exit.py b/python/sglang/srt/layers/communicator/exit.py
new file mode 100644
index 0000000000..4bb30f9343
--- /dev/null
+++ b/python/sglang/srt/layers/communicator/exit.py
@@ -0,0 +1,521 @@
+# Copyright 2023-2024 SGLang Team
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Producer output choices, reduction scopes, and completion."""
+
+from __future__ import annotations
+
+from functools import partial
+from typing import Callable, Optional, Tuple, Union
+
+import msgspec
+import torch
+
+from sglang.srt.distributed import GroupCoordinator
+from sglang.srt.environ import envs
+from sglang.srt.layers.communicator.adapters.attention import get_attn_tp_context
+from sglang.srt.layers.communicator.layout import (
+ SumGroup,
+ _ffn_has_tokens,
+ _sum_group,
+)
+from sglang.srt.layers.communicator.ops import (
+ _all_reduce_then_to_local_tokens,
+ _redistribute_output,
+ _reduce_and_redistribute_output_max_len,
+ _reduce_and_redistribute_output_varlen,
+ _to_local_tokens,
+)
+from sglang.srt.layers.communicator.output import (
+ UnreducedOutput,
+)
+from sglang.srt.layers.communicator.residual.add_norm import (
+ apply_aiter_all_reduce_fusion,
+ apply_flashinfer_allreduce_fusion,
+)
+from sglang.srt.layers.communicator.residual.stream import ResidualStream
+from sglang.srt.layers.dp_attention import (
+ can_use_dp_reduce_scatter,
+ is_dp_attention_enabled,
+ is_enable_moe_cp_allgather,
+)
+from sglang.srt.layers.moe import (
+ can_merge_post_experts_all_reduce,
+ get_moe_a2a_backend,
+ post_experts_reduction_group,
+ post_experts_sum_is_one_all_reduce,
+ should_use_dp_reduce_scatterv,
+)
+from sglang.srt.model_executor.forward_batch_info import ForwardBatch
+from sglang.srt.runtime_context import (
+ get_exec,
+ get_forward,
+ get_lora,
+ get_parallel,
+)
+
+
+def _reduce_and_redistribute_output_step(
+ forward_batch: ForwardBatch,
+ *,
+ leaves_for_reduce_scatter: bool,
+ leaves_for_reduce_scatterv: bool,
+) -> Optional[Callable[[torch.Tensor, torch.Tensor, ForwardBatch], None]]:
+ """The reduce-scatter that brings an FFN output gathered over attention
+ DP back to this rank's tokens when the FFN leaves its sum to it (see
+ StageOutput); None when the FFN reduces the output and only a scatter
+ remains."""
+ if should_use_dp_reduce_scatterv() and leaves_for_reduce_scatterv:
+ return _reduce_and_redistribute_output_varlen
+ if (
+ leaves_for_reduce_scatter
+ and forward_batch.dp_padding_mode.is_max_len()
+ and can_use_dp_reduce_scatter()
+ ):
+ return _reduce_and_redistribute_output_max_len
+ return None
+
+
+class OutputBoundary:
+ """Choose and finish one producer's output; the batch owns its residual."""
+
+ def __init__(self, plan):
+ self.plan = plan
+
+ def postprocess_layer(
+ self,
+ hidden_states: torch.Tensor,
+ residual: torch.Tensor,
+ forward_batch: ForwardBatch,
+ ):
+ """Move a complete output from the operation-scheduled producer path."""
+ stream = residual if isinstance(residual, ResidualStream) else None
+ hidden_states, residual = self._complete_ffn_output_now(
+ hidden_states,
+ stream.residual if stream is not None else residual,
+ forward_batch=forward_batch,
+ dp_step=None,
+ )
+ return self._leave_ffn_output(
+ hidden_states,
+ residual,
+ stream,
+ self.plan._batch_steps(forward_batch).ffn_output.update,
+ )
+
+ @staticmethod
+ def _leave_ffn_output(hidden_states, residual, stream, update, declared_sum=None):
+ if stream is None:
+ return hidden_states, residual
+ stream.residual = residual
+ if update.at_producer:
+ return stream.write(hidden_states), stream
+ return stream.leave(hidden_states, update, declared_sum=declared_sum), stream
+
+ @staticmethod
+ def _declared_ffn_sum(steps, skipped_reduction):
+ # The input-scattered path leaves the sum for the next input's TP
+ # reduce-scatter. Other skip paths complete it in the output move.
+ if (
+ skipped_reduction
+ and not steps.returns_over_dp
+ and not steps.ffn_output_move_completes_sum
+ ):
+ return SumGroup.TP
+ return None
+
+ def _local_token_move_can_go_to_next_layer(
+ self, forward_batch: ForwardBatch
+ ) -> bool:
+ """Whether the next layer's input can run this layer's move of its FFN
+ output back to this rank's tokens: the base postprocess scatter, when
+ the next layer's input also writes the output into the residual."""
+ steps = self.plan._batch_steps(forward_batch)
+ return steps.returns_over_dp and not steps.ffn_output.update.at_producer
+
+ def _postprocess_dp_step(
+ self, forward_batch: ForwardBatch
+ ) -> Optional[Callable[[torch.Tensor, torch.Tensor, ForwardBatch], None]]:
+ """The reduce-scatter that brings this layer's FFN output back to this
+ rank's tokens under attention DP; None when the base postprocess would
+ only scatter, or does not move tokens."""
+ steps = self.plan._batch_steps(forward_batch)
+ if not steps.returns_over_dp:
+ return None
+ return _reduce_and_redistribute_output_step(
+ forward_batch,
+ leaves_for_reduce_scatter=steps.ffn_output.leaves_for_reduce_scatter,
+ leaves_for_reduce_scatterv=steps.ffn_output.leaves_for_reduce_scatterv,
+ )
+
+ def _ffn_leaves_sum_to_reduce_scatter(
+ self, forward_batch: ForwardBatch, dp_step: Optional[Callable]
+ ) -> bool:
+ """Whether the FFN leaves its sum out because a reduce-scatter completes
+ it: the attention-DP one ``dp_step`` names, or the CP / input-scattered
+ one."""
+ steps = self.plan._batch_steps(forward_batch)
+ if dp_step is not None:
+ return True
+ if not steps.ffn_output.leaves_for_reduce_scatter:
+ return False
+ if steps.ffn_output_move_completes_sum:
+ return True
+ return get_attn_tp_context().input_scattered and not self.plan.is_last_layer
+
+ def ffn_reduction_group(self, forward_batch: ForwardBatch) -> GroupCoordinator:
+ """The group this layer's FFN output owes its sum over: the MoE output's
+ group on a sparse layer, the TP group a dense MLP reduces over."""
+ return _sum_group(self.plan._batch_steps(forward_batch).ffn_output.group)
+
+ def _select_ffn_completion(self, forward_batch: ForwardBatch) -> FfnCompletion:
+ """Decide once, before the FFN runs, what it skips and what completes its
+ output: the next layer's input, or this layer's postprocess step."""
+ dp_step = self._postprocess_dp_step(forward_batch)
+ mlp_reduce_scatter = self._ffn_leaves_sum_to_reduce_scatter(
+ forward_batch, dp_step
+ )
+ complete_now = partial(
+ self._complete_ffn_output_now,
+ forward_batch=forward_batch,
+ dp_step=dp_step,
+ )
+ steps = self.plan._batch_steps(forward_batch)
+ if not steps.ffn_output.leaves_for_next_layer:
+ return FfnCompletion(
+ defer_moe_finalize=False,
+ fuse_mlp_allreduce=False,
+ mlp_reduce_scatter=mlp_reduce_scatter,
+ complete=complete_now,
+ )
+ defer_moe_finalize = (
+ self.plan.fusions is not None
+ and self.plan.fusions.can_defer_finalize(self.plan, forward_batch)
+ )
+ # Producers declare remaining work independently of the kernel chosen
+ # by the consumer. Every handoff also carries an unfused completion.
+ fuse_mlp_allreduce = defer_moe_finalize or self._ffn_sum_moves_to_next_layer(
+ forward_batch, mlp_reduce_scatter=mlp_reduce_scatter, dp_step=dp_step
+ )
+ if fuse_mlp_allreduce:
+ group = self.ffn_reduction_group(forward_batch)
+ if steps.returns_over_dp:
+ # Under attention DP the next layer also brings the sum back to
+ # this rank's tokens.
+ wrap = partial(
+ UnreducedOutput,
+ reduce_and_redistribute=partial(
+ _all_reduce_then_to_local_tokens, group, forward_batch
+ ),
+ )
+ else:
+ wrap = partial(UnreducedOutput, group=group)
+ complete = partial(_leave_to_next_layer, wrap)
+ elif (
+ dp_step is not None
+ and not self.plan.is_last_layer
+ and self._local_token_move_can_go_to_next_layer(forward_batch)
+ ):
+ complete = partial(
+ _leave_to_next_layer,
+ partial(
+ UnreducedOutput,
+ reduce_and_redistribute=partial(
+ _to_local_tokens, dp_step, forward_batch
+ ),
+ ),
+ )
+ else:
+ complete = complete_now
+ return FfnCompletion(
+ defer_moe_finalize=defer_moe_finalize,
+ fuse_mlp_allreduce=fuse_mlp_allreduce,
+ mlp_reduce_scatter=mlp_reduce_scatter,
+ complete=complete,
+ )
+
+ def _complete_ffn_output_now(
+ self,
+ hidden_states: torch.Tensor,
+ residual: torch.Tensor,
+ *,
+ forward_batch: ForwardBatch,
+ dp_step: Optional[Callable],
+ ) -> Tuple[torch.Tensor, torch.Tensor]:
+ """This layer's postprocess, run with the attention-DP step already
+ chosen: the move back to where the next layer reads the FFN output, then
+ the write-back into the residual for a layer that does it itself."""
+ steps = self.plan._batch_steps(forward_batch)
+ if steps.ffn_output.transform is not None:
+ hidden_states = steps.ffn_output.transform.apply(hidden_states)
+ if steps.returns_over_dp:
+ hidden_states = _to_local_tokens(
+ dp_step or _redistribute_output, forward_batch, hidden_states
+ )
+ else:
+ hidden_states, residual = steps.ffn_output_move(
+ hidden_states=hidden_states,
+ residual=residual,
+ forward_batch=forward_batch,
+ )
+ update = steps.ffn_output.update
+ if residual is not None and update.at_producer:
+ hidden_states = update.update(hidden_states, residual)
+ residual = None
+ return hidden_states, residual
+
+ def mixer_exit(self, forward_batch: ForwardBatch) -> MixerExit:
+ """Decide once whether this stage's mixer (an attention-like stage)
+ skips its output all-reduce. Use the result as a context manager around
+ the mixer, then call ``finish``."""
+ return MixerExit(self, forward_batch)
+
+ def ffn_exit(self, forward_batch: ForwardBatch) -> FfnExit:
+ """Decide once how this layer's FFN output reduction completes. Use the
+ result as a context manager around the FFN call, then call ``finish``."""
+ return FfnExit(self, forward_batch)
+
+ def _ffn_sum_can_move_to_next_layer(self, forward_batch: ForwardBatch) -> bool:
+ # Under the MoE-CP all-gather the fusion path would skip postprocess_layer
+ # and its MoE-CP scatter, leaving hidden_states longer than the residual.
+ if (
+ (
+ self.plan.fusions is not None
+ and self.plan.fusions.requires_local_reduction
+ )
+ or is_enable_moe_cp_allgather()
+ or not self.plan._batch_steps(forward_batch).ffn_sum_is_movable
+ ):
+ return False
+
+ # The fused residual+LN reduces over a single group. Hybrid EP+TP spans
+ # two disjoint groups; post_experts_all_reduce() merges them into one
+ # _TP reduction when moe_dp_size == 1, which the fused kernel can absorb.
+ # When merging is blocked, no single group covers both, so fusion stays off.
+ parallel = get_parallel()
+ if (
+ parallel.moe_ep_size > 1
+ and parallel.moe_tp_size > 1
+ and not can_merge_post_experts_all_reduce()
+ ):
+ return False
+
+ if (
+ is_dp_attention_enabled()
+ and self.plan._speculative_algo is not None
+ and self.plan._speculative_algo.is_eagle()
+ ):
+ return False
+
+ return not get_attn_tp_context().input_scattered
+
+ def _ffn_sum_moves_to_next_layer(
+ self,
+ forward_batch: ForwardBatch,
+ *,
+ mlp_reduce_scatter: bool,
+ dp_step: Optional[Callable],
+ ) -> bool:
+ """Whether the FFN leaves its output's all-reduce to the next layer's
+ input when no fused kernel takes it: when the next layer would run the
+ same all-reduce the FFN itself would have."""
+ return (
+ get_parallel().tp_size > 1
+ and not self.plan.is_last_layer
+ and self._ffn_sum_can_move_to_next_layer(forward_batch)
+ and _can_defer_ffn_reduction(forward_batch, self.plan)
+ and not mlp_reduce_scatter
+ # Under attention DP the next layer must also run postprocess's
+ # scatter back to this rank's tokens, and nothing more.
+ and (
+ not is_dp_attention_enabled()
+ or (
+ self._local_token_move_can_go_to_next_layer(forward_batch)
+ and dp_step is None
+ )
+ )
+ )
+
+
+def _can_defer_ffn_reduction(forward_batch: ForwardBatch, boundary=None) -> bool:
+ """Admit ordinary single-sum outputs and the existing fused LoRA path.
+
+ A fused-kernel fallback completes the partial-output contract selected
+ before the producer ran, without recomputing the producer's LoRA path.
+ """
+ if not _ffn_has_tokens(forward_batch):
+ return False
+ if post_experts_sum_is_one_all_reduce():
+ return True
+ # LoRA-B is replicated and linear. TP1 shared experts add on rank zero
+ # when the sum is deferred. Preserve these fused paths, but keep their
+ # producer-side reduction order when no fused consumer is enabled.
+ if not (
+ (get_lora().enable_lora or envs.SGLANG_SHARED_EXPERT_TP1.get())
+ and get_moe_a2a_backend().is_none()
+ and not get_exec().comm.enable_quant_communications
+ and post_experts_reduction_group() is get_parallel().tp_group
+ ):
+ return False
+ if (
+ boundary is not None
+ and boundary.fusions is not None
+ and boundary.fusions.can_defer_all_reduce(boundary, forward_batch)
+ ):
+ return True
+ if apply_flashinfer_allreduce_fusion(forward_batch.input_ids.shape[0]):
+ return True
+ # Aiter also checks width and bytes, so use the actual residual storage.
+ residual = forward_batch.residual_stream.residual
+ return residual is not None and apply_aiter_all_reduce_fusion(
+ residual, forward_batch
+ )
+
+
+class FfnCompletion(msgspec.Struct, frozen=True):
+ """One FFN's reduction decision. The flags are published while the FFN runs;
+ ``complete(hidden_states, residual)`` then either wraps the output for the
+ next layer's input to complete, or runs this layer's postprocess step."""
+
+ defer_moe_finalize: bool
+ fuse_mlp_allreduce: bool
+ mlp_reduce_scatter: bool
+ complete: Callable[[torch.Tensor, torch.Tensor], Tuple]
+
+
+def _leave_to_next_layer(
+ wrap: Callable[[torch.Tensor], UnreducedOutput],
+ hidden_states: torch.Tensor,
+ residual: torch.Tensor,
+) -> Tuple[UnreducedOutput, torch.Tensor]:
+ return wrap(hidden_states), residual
+
+
+class MixerExit:
+ """The scope that publishes a mixer's decision while it runs: inside the
+ ``with`` block ``fuse_mlp_allreduce`` on ``get_forward()`` tells its
+ row-parallel output projection to skip the all-reduce. It skips when the
+ stage's output always leaves its sum (to an FFN stage, which completes it in
+ its input), and when its output declaration permits deferring the sum. The
+ consumer decides whether to fuse its completion with the input norm."""
+
+ __slots__ = (
+ "skips_reduction",
+ "_hands_on",
+ "_scope",
+ "_update",
+ "_declared_sum",
+ "_stream",
+ )
+
+ def __init__(self, communicator: OutputBoundary, forward_batch: ForwardBatch):
+ produced = communicator.plan._batch_steps(forward_batch).ffn_output
+ self._stream = None
+ self._update = produced.update
+ self._declared_sum = produced.group if produced.always_leaves else None
+ self._hands_on = (
+ produced.leaves_for_next_layer
+ and get_parallel().tp_size > 1
+ and not is_dp_attention_enabled()
+ and communicator._ffn_sum_can_move_to_next_layer(forward_batch)
+ )
+ self.skips_reduction = produced.always_leaves or self._hands_on
+ self._scope = get_forward().scoped(fuse_mlp_allreduce=self.skips_reduction)
+
+ def __enter__(self) -> MixerExit:
+ self._scope.__enter__()
+ return self
+
+ def __exit__(self, *exc_info):
+ return self._scope.__exit__(*exc_info)
+
+ def finish(self, hidden_states: torch.Tensor, residual=None):
+ """The mixer's output: its partial sum to the FFN stage after it, as a
+ value that owes the sum to a mixer after it, or complete."""
+ if self._stream is not None:
+ residual = self._stream
+ if self._hands_on:
+ hidden_states = UnreducedOutput(
+ hidden_states, group=get_parallel().tp_group
+ )
+ if isinstance(residual, ResidualStream):
+ output = residual.leave(
+ hidden_states, self._update, declared_sum=self._declared_sum
+ )
+ return output if self._stream is not None else (output, residual)
+ return hidden_states
+
+
+class FfnExit:
+ """The scope that publishes an FfnCompletion while the FFN runs: inside the
+ ``with`` block it is ``fuse_mlp_allreduce`` / ``mlp_reduce_scatter`` /
+ ``defer_moe_finalize`` on ``get_forward()``."""
+
+ __slots__ = (
+ "communicator",
+ "forward_batch",
+ "defer_moe_finalize",
+ "fuse_mlp_allreduce",
+ "mlp_reduce_scatter",
+ "_complete",
+ "_update",
+ "_declared_sum",
+ "_scope",
+ "_stream",
+ )
+
+ def __init__(self, communicator: OutputBoundary, forward_batch: ForwardBatch):
+ self._stream = None
+ self.communicator = communicator
+ self.forward_batch = forward_batch
+ completion = communicator._select_ffn_completion(forward_batch)
+ self.defer_moe_finalize = completion.defer_moe_finalize
+ self.fuse_mlp_allreduce = completion.fuse_mlp_allreduce
+ self.mlp_reduce_scatter = completion.mlp_reduce_scatter
+ self._complete = completion.complete
+ steps = communicator.plan._batch_steps(forward_batch)
+ self._update = steps.ffn_output.update
+ self._declared_sum = communicator._declared_ffn_sum(
+ steps, completion.mlp_reduce_scatter
+ )
+ self._scope = get_forward().scoped(
+ fuse_mlp_allreduce=self.fuse_mlp_allreduce,
+ mlp_reduce_scatter=self.mlp_reduce_scatter,
+ defer_moe_finalize=self.defer_moe_finalize,
+ )
+
+ def __enter__(self) -> FfnExit:
+ self._scope.__enter__()
+ return self
+
+ def __exit__(self, *exc_info):
+ return self._scope.__exit__(*exc_info)
+
+ def finish(
+ self, hidden_states: torch.Tensor, residual: torch.Tensor = None
+ ) -> Tuple[Union[torch.Tensor, UnreducedOutput], torch.Tensor]:
+ """Leave the reduction to the next layer's input, or postprocess."""
+ if self._stream is not None:
+ residual = self._stream
+ stream = residual if isinstance(residual, ResidualStream) else None
+ if stream is not None:
+ residual = stream.residual
+ if not isinstance(hidden_states, torch.Tensor):
+ # A deferred MoE finalize handoff, consumed by the next prepare_attn.
+ assert self.defer_moe_finalize, "unrequested deferred MoE handoff"
+ else:
+ hidden_states, residual = self._complete(hidden_states, residual)
+ result = self.communicator._leave_ffn_output(
+ hidden_states, residual, stream, self._update, self._declared_sum
+ )
+ return result[0] if self._stream is not None else result
diff --git a/python/sglang/srt/layers/communicator/factories.py b/python/sglang/srt/layers/communicator/factories.py
new file mode 100644
index 0000000000..03038f9772
--- /dev/null
+++ b/python/sglang/srt/layers/communicator/factories.py
@@ -0,0 +1,705 @@
+"""Construct one computing stage and its transport from a predecessor contract."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, replace
+from types import MappingProxyType
+from typing import Mapping, Optional
+
+import msgspec
+
+from sglang.srt.layers import layernorm_sp
+from sglang.srt.layers.communicator.boundary import (
+ EdgeDecl,
+ StageDecl,
+ StageInput,
+ StageKind,
+ StageOutput,
+ _cp_moves,
+)
+from sglang.srt.layers.communicator.construction import (
+ BatchVariant,
+ StageEdges,
+ StagePlan,
+ _input_can_be_scattered,
+ _refuse_uncovered_cp_moe,
+ _use_ag_after_qlora,
+)
+from sglang.srt.layers.communicator.layout import (
+ Layout,
+ SumGroup,
+ TokenAxis,
+ _gathers_over_attention_cp,
+ _generic_prefill_cp_shards_tokens,
+ enable_moe_dense_fully_dp,
+ sparse_moe_gathers_over_moe_cp,
+ token_axis_sizes,
+)
+from sglang.srt.layers.communicator.ops import (
+ _hand_qkv_hook_its_input,
+ _hand_scattered_input_to_attention,
+)
+from sglang.srt.layers.communicator.output import OutputTransform
+from sglang.srt.layers.communicator.residual import StageRead, StageUpdate
+from sglang.srt.layers.communicator.residual.add_norm import (
+ ADD,
+ NORM_QUANT_READ,
+ NORM_READ,
+)
+from sglang.srt.layers.communicator.stage import StageCommunicator
+from sglang.srt.layers.moe import is_moe_input_scattered_across_dp_ranks
+from sglang.srt.runtime_context import get_exec, get_parallel
+
+
+def _variants(*, ordinary_only=False):
+ yield BatchVariant.ORDINARY
+ if ordinary_only:
+ return
+ if _generic_prefill_cp_shards_tokens():
+ yield BatchVariant.CONTEXT_PARALLEL
+ if _input_can_be_scattered():
+ yield BatchVariant.INPUT_SCATTERED
+ if layernorm_sp.layernorm_sp_enabled():
+ yield BatchVariant.SEQUENCE_PARALLEL
+
+
+def _rows(variant):
+ axes = token_axis_sizes(cp_active=variant is BatchVariant.CONTEXT_PARALLEL)
+ attention = Layout.sharded_over(
+ TokenAxis.ATTN_DP, TokenAxis.ATTN_CP, axis_sizes=axes
+ )
+ local = Layout.sharded_over(
+ TokenAxis.ATTN_DP, TokenAxis.ATTN_CP, TokenAxis.ATTN_TP_SCATTER, axis_sizes=axes
+ )
+ return axes, attention, local, Layout.sharded_over(axis_sizes=axes)
+
+
+def _ffn_decl(
+ variant,
+ *,
+ sparse,
+ terminal=False,
+ next_sparse=False,
+ output=None,
+ read=NORM_READ,
+ update=ADD,
+ local_dense=True,
+):
+ parallel = get_parallel()
+ reduction = get_exec().comm.boundary_reduction
+ if reduction not in ("ar", "rs", "rsv", "rs+rsv"):
+ raise ValueError(
+ "boundary_reduction must be resolved before model construction"
+ )
+ can_move_output = output is None or output.before_reduce_scatter
+ use_reduce_scatter = reduction in ("rs", "rs+rsv") and can_move_output
+ use_reduce_scatterv = reduction in ("rsv", "rs+rsv") and can_move_output
+ cp_shards = _generic_prefill_cp_shards_tokens()
+ axes, attention, local, full = _rows(variant)
+ on_local = (
+ is_moe_input_scattered_across_dp_ranks()
+ if sparse
+ else (local_dense and enable_moe_dense_fully_dp())
+ )
+ if parallel.attn_cp_size > 1 and sparse:
+ _refuse_uncovered_cp_moe(on_local, cp_shards)
+ on_cp = (
+ sparse
+ and parallel.attn_cp_size > 1
+ and parallel.moe_dp_size == parallel.attn_cp_size
+ and not _gathers_over_attention_cp()
+ )
+ group = SumGroup.MOE_OUTPUT if sparse else SumGroup.TP
+ if variant is BatchVariant.SEQUENCE_PARALLEL:
+ return (
+ StageDecl(
+ StageInput(
+ full,
+ gathers_itself=frozenset({TokenAxis.ATTN_TP_SCATTER}),
+ read=read,
+ ),
+ StageOutput(local, update=update, transform=output),
+ ),
+ local,
+ local,
+ )
+ if variant is BatchVariant.INPUT_SCATTERED:
+ scattered_residual = update.at_producer
+ residual = local if scattered_residual else attention
+ returned = local if scattered_residual and not terminal else attention
+ return (
+ StageDecl(
+ StageInput(full, read=read),
+ StageOutput(
+ attention,
+ group=group,
+ leaves_for_reduce_scatter=use_reduce_scatter
+ and (scattered_residual or not terminal),
+ update=update,
+ transform=output,
+ ),
+ ),
+ residual,
+ returned,
+ )
+ if cp_shards and _cp_moves().reduce_scatter is not None:
+ may_leave = variant is not BatchVariant.CONTEXT_PARALLEL
+ may_scatter = True
+ elif cp_shards or parallel.attn_dp_size > 1:
+ may_leave = may_scatter = parallel.attn_cp_size == 1 or on_cp
+ else:
+ may_leave = may_scatter = True
+ rows = (
+ local
+ if on_local
+ else Layout.sharded_over(
+ *((TokenAxis.ATTN_CP,) if on_cp else ()), axis_sizes=axes
+ )
+ )
+ output = (
+ StageOutput(rows, update=update, transform=output)
+ if on_local
+ else StageOutput(
+ rows,
+ group=group,
+ leaves_for_next_layer=may_leave
+ and not terminal
+ and not update.at_producer
+ and update.can_defer_across_layers
+ and output is None,
+ leaves_for_reduce_scatter=use_reduce_scatter and may_scatter,
+ leaves_for_reduce_scatterv=use_reduce_scatterv and may_leave,
+ update=update,
+ transform=output,
+ )
+ )
+ gathers_for_tbo = (
+ enable_moe_dense_fully_dp()
+ and get_exec().overlap.enable_two_batch_overlap
+ and not sparse
+ and bool(next_sparse)
+ )
+ returned = local if on_local and not terminal and not gathers_for_tbo else attention
+ return (
+ StageDecl(StageInput(rows, read=read), output),
+ local if on_local else attention,
+ returned,
+ )
+
+
+@dataclass(frozen=True)
+class StageDeclaration:
+ """A computation and its input source, without an execution plan.
+
+ ``previous`` consumes a producer's output. ``prepared_from`` reuses an
+ already-read input for a branch; the two sources are mutually exclusive.
+ Sources may be reconstructed locally, including across pipeline ranks.
+ """
+
+ kind: StageKind
+ read: StageRead
+ update: StageUpdate
+ sparse: bool = False
+ terminal: bool = False
+ next_sparse: bool = False
+ output: Optional[OutputTransform] = None
+ ordinary_only: bool = False
+ # A mixer with its own exit publishes reduction decisions before compute.
+ mixer_exit: bool = False
+ next_kind: Optional[StageKind] = None
+ return_to_attention: bool = False
+ # Only declarations participate in construction, never executable stages.
+ previous: Optional[StageDeclaration] = None
+ prepared_from: Optional[StageDeclaration] = None
+
+ def __post_init__(self):
+ if self.previous is not None and self.prepared_from is not None:
+ raise ValueError("choose a previous output or a prepared branch input")
+ for source in (self.previous, self.prepared_from):
+ if source is not None and not isinstance(source, StageDeclaration):
+ raise TypeError("stage sources must be declarations")
+
+
+@dataclass(frozen=True)
+class StageConnection:
+ """The producer's exit and consumer's entry for each batch variant.
+
+ Immutable, norm-free declarations. Independently constructed equivalent
+ connections work across pipeline partitions; object identity is irrelevant.
+ """
+
+ producer: Optional[StageDeclaration]
+ consumer: Optional[StageDeclaration]
+ exits: Mapping[BatchVariant, EdgeDecl]
+ entries: Mapping[BatchVariant, EdgeDecl]
+
+ def __post_init__(self):
+ object.__setattr__(self, "exits", MappingProxyType(dict(self.exits)))
+ object.__setattr__(self, "entries", MappingProxyType(dict(self.entries)))
+
+
+def declare_attn(
+ *,
+ previous: Optional[StageDeclaration] = None,
+ prepared_from: Optional[StageDeclaration] = None,
+ read=NORM_QUANT_READ,
+ update=ADD,
+ terminal=False,
+ ordinary_only=False,
+ mixer_exit=False,
+ next_kind=None,
+):
+ return StageDeclaration(
+ StageKind.ATTENTION,
+ read,
+ update,
+ previous=previous,
+ prepared_from=prepared_from,
+ terminal=terminal,
+ ordinary_only=ordinary_only,
+ mixer_exit=mixer_exit,
+ next_kind=next_kind,
+ )
+
+
+def declare_ffn(
+ *,
+ previous: Optional[StageDeclaration] = None,
+ prepared_from: Optional[StageDeclaration] = None,
+ sparse=False,
+ read=NORM_READ,
+ update=ADD,
+ terminal=False,
+ next_sparse=False,
+ output=None,
+ ordinary_only=False,
+ return_to_attention=False,
+):
+ return StageDeclaration(
+ StageKind.FFN,
+ read,
+ update,
+ sparse=sparse,
+ previous=previous,
+ prepared_from=prepared_from,
+ terminal=terminal,
+ next_sparse=next_sparse,
+ output=output,
+ ordinary_only=ordinary_only,
+ return_to_attention=return_to_attention,
+ )
+
+
+def _resolve(stage, variant):
+ axes, attention, local, full = _rows(variant)
+ if stage.kind is StageKind.FFN:
+ if stage.update.at_producer:
+ if stage.sparse and sparse_moe_gathers_over_moe_cp():
+ raise NotImplementedError(
+ "MHC does not support a MoE gathered over the MoE-CP group"
+ )
+ if get_parallel().attn_cp_size > 1 and _input_can_be_scattered():
+ raise NotImplementedError(
+ "MHC with input-scattered attention under attention CP"
+ )
+ declaration, residual, returned = _ffn_decl(
+ variant,
+ sparse=stage.sparse,
+ terminal=stage.terminal,
+ next_sparse=stage.next_sparse,
+ output=stage.output,
+ read=stage.read,
+ update=stage.update,
+ local_dense=not stage.return_to_attention,
+ )
+ if stage.return_to_attention:
+ returned = attention
+ return declaration, residual, returned
+ sp = variant is BatchVariant.SEQUENCE_PARALLEL
+ scattered = variant is BatchVariant.INPUT_SCATTERED
+ gathers = (
+ frozenset({TokenAxis.ATTN_TP_SCATTER})
+ if not stage.mixer_exit and (sp or scattered or _use_ag_after_qlora)
+ else frozenset()
+ )
+ owes = not sp and axes[TokenAxis.ATTN_TP_SCATTER] > 1
+ declaration = StageDecl(
+ StageInput(attention, gathers_itself=gathers, read=stage.read),
+ StageOutput(
+ local if sp else attention,
+ group=SumGroup.ATTN_TP if owes else None,
+ always_leaves=owes
+ and (not stage.mixer_exit or stage.next_kind is StageKind.FFN),
+ leaves_for_next_layer=owes
+ and stage.mixer_exit
+ and stage.next_kind is StageKind.ATTENTION,
+ update=stage.update,
+ ),
+ )
+ return declaration, None, attention
+
+
+def _connect(producer, consumer, *, residual_from=None):
+ """Resolve producer and consumer declarations; None denotes an external endpoint.
+
+ A missing producer is the stack input; a missing consumer is a handoff
+ whose next read is bound separately (or the terminal output).
+ ``residual_from`` names the boundary that placed the producer's residual.
+ It is needed for attention, whose residual can stay on finer rows than its
+ compute input. No executable stage participates in this construction.
+ """
+ if producer is None and consumer is None:
+ raise ValueError("a boundary needs at least one declared side")
+ before, after = producer, consumer
+ ordinary_only = any(s.ordinary_only for s in (before, after) if s is not None)
+ exits, entries = {}, {}
+ for variant in _variants(ordinary_only=ordinary_only):
+ _, attention, local, _ = _rows(variant)
+ if before is None:
+ rows = local if variant is BatchVariant.SEQUENCE_PARALLEL else attention
+ owes = variant is BatchVariant.INPUT_SCATTERED
+ arrived = StageOutput(
+ rows,
+ group=SumGroup.TP if owes else None,
+ always_leaves=owes,
+ update=None,
+ )
+ residual = rows
+ capabilities = (True,)
+ else:
+ decl, during, returned = _resolve(before, variant)
+ if during is None:
+ if residual_from is not None:
+ if residual_from.consumer != producer:
+ raise ValueError("residual source must enter the producer")
+ during = residual_from.entries[variant].residual_to
+ elif before.mixer_exit:
+ during = attention
+ else:
+ raise ValueError(
+ "attention output needs its incoming residual placement"
+ )
+ exits[variant] = EdgeDecl(
+ decl.output, StageInput(returned), during, returned
+ )
+ if before.kind is StageKind.ATTENTION and not before.mixer_exit:
+ arrived, residual, capabilities = decl.output, during, ()
+ else:
+ owes = (
+ variant is BatchVariant.INPUT_SCATTERED
+ and not before.update.at_producer
+ )
+ carries = (before.mixer_exit or before.return_to_attention) and (
+ decl.output.always_leaves or decl.output.leaves_for_next_layer
+ )
+ arrived = StageOutput(
+ returned,
+ group=decl.output.group
+ if carries
+ else (SumGroup.TP if owes else None),
+ always_leaves=decl.output.always_leaves if carries else owes,
+ leaves_for_next_layer=decl.output.leaves_for_next_layer
+ if carries
+ else False,
+ update=None,
+ )
+ residual, capabilities = returned, (before.update.adds_plainly,)
+ if after is None:
+ continue
+ decl, during, _ = _resolve(after, variant)
+ if after.kind is StageKind.ATTENTION:
+ during = (
+ Layout(residual.sharded | {TokenAxis.ATTN_TP_SCATTER})
+ if variant is BatchVariant.INPUT_SCATTERED
+ and arrived.always_leaves
+ and arrived.update is None
+ and not after.mixer_exit
+ else residual
+ )
+ elif after.return_to_attention:
+ during = (
+ decl.input.layout
+ if residual.sharded <= decl.input.layout.sharded
+ else residual
+ )
+ joins = (
+ variant is BatchVariant.INPUT_SCATTERED
+ and arrived.update is not None
+ and arrived.update.adds_plainly
+ and after.kind is StageKind.FFN
+ )
+ edge = EdgeDecl(
+ arrived,
+ decl.input,
+ residual,
+ during,
+ residual_joins_sum=joins,
+ update_capabilities=capabilities,
+ )
+ entries[variant] = edge
+ if (
+ before is not None
+ and before.kind is StageKind.ATTENTION
+ and not before.mixer_exit
+ ):
+ exits[variant] = edge
+ return StageConnection(producer, consumer, exits, entries)
+
+
+def _fork_input(prepared, consumer):
+ """Place an already-read input onto a branch's computation rows.
+
+ The branch adapter performs the move and forks the stream. It does not
+ execute the branch's input norm again.
+ """
+ entries = {}
+ for variant in _variants(ordinary_only=consumer.ordinary_only):
+ source = prepared.entries[variant]
+ declaration, during, _ = _resolve(consumer, variant)
+ entries[variant] = EdgeDecl(
+ StageOutput(source.need.layout, update=None),
+ declaration.input,
+ source.residual_to,
+ during,
+ update_capabilities=(True,),
+ )
+ return StageConnection(prepared.consumer, consumer, {}, entries)
+
+
+def _incoming(stage):
+ if stage.prepared_from is not None:
+ return _fork_input(_incoming(stage.prepared_from), stage)
+ previous = stage.previous
+ # A normal attention may retain a finer residual than its output rows.
+ # FFN and mixer exits declare their returned residual placement directly.
+ source = (
+ _incoming(previous)
+ if previous is not None
+ and previous.kind is StageKind.ATTENTION
+ and not previous.mixer_exit
+ else None
+ )
+ return _connect(
+ previous,
+ stage,
+ residual_from=source,
+ )
+
+
+def _connections(stage, following=None):
+ if following is not None and following.previous != stage:
+ raise ValueError("the following declaration must consume this stage's output")
+ incoming = _incoming(stage)
+ outgoing = _connect(
+ stage,
+ following,
+ residual_from=incoming,
+ )
+ return incoming, outgoing
+
+
+def _bind_stage(declaration, norm, incoming, outgoing, **options):
+ if incoming.consumer != declaration or outgoing.producer != declaration:
+ raise ValueError("connections do not match the stage declaration")
+ if incoming.entries.keys() != outgoing.exits.keys():
+ raise ValueError("incoming and outgoing batch variants disagree")
+ if declaration.update.at_producer:
+ if declaration.sparse and sparse_moe_gathers_over_moe_cp():
+ raise NotImplementedError(
+ "MHC does not support a MoE gathered over the MoE-CP group"
+ )
+ if get_parallel().attn_cp_size > 1 and _input_can_be_scattered():
+ raise NotImplementedError(
+ "MHC with input-scattered attention under attention CP"
+ )
+ variants = {}
+ for variant, edge in incoming.entries.items():
+ if (
+ declaration.update.at_producer
+ and TokenAxis.ATTN_CP
+ in edge.produced.layout.sharded - edge.need.layout.sharded
+ ):
+ raise NotImplementedError("MHC with a gather over attention CP")
+ handoff = None
+ if declaration.kind is StageKind.ATTENTION:
+ handoff = (
+ _hand_scattered_input_to_attention
+ if variant is BatchVariant.INPUT_SCATTERED
+ and not declaration.update.adds_plainly
+ else _hand_qkv_hook_its_input
+ )
+ moves = _cp_moves() if variant is BatchVariant.CONTEXT_PARALLEL else None
+ variants[variant] = StageEdges(edge, outgoing.exits[variant], handoff, moves)
+ plan = StagePlan(
+ declaration.kind,
+ norm,
+ variants,
+ enters_stack=incoming.producer is None,
+ prepared_input=declaration.prepared_from is not None,
+ terminal=declaration.terminal,
+ fixed_output=declaration.kind is StageKind.ATTENTION
+ and not declaration.mixer_exit,
+ is_sparse=declaration.sparse,
+ **options,
+ )
+ if declaration.kind is StageKind.ATTENTION and outgoing.consumer is not None:
+ # Layout eligibility comes from the connected consumer, not a mutable
+ # link to its execution plan. Kernel binding remains consumer-owned.
+ from sglang.srt.layers.communicator.boundary import make_boundary
+
+ plan._fusion_rows = {
+ v: make_boundary(
+ edge,
+ cp_moves=_cp_moves() if v is BatchVariant.CONTEXT_PARALLEL else None,
+ force_layernorm_before_gather=options.get(
+ "force_layernorm_before_dp_gather", False
+ ),
+ ).input_rows
+ for v, edge in outgoing.entries.items()
+ }
+ return StageCommunicator(plan, declaration.kind, norm, declaration=declaration)
+
+
+def make_attn_stage(
+ *,
+ declaration,
+ norm,
+ following: Optional[StageDeclaration] = None,
+ qkv_latent_func=None,
+ force_layernorm_before_dp_gather=False,
+ enable_fused_ar_quant=False,
+ fused_ar_quant_keep_bf16=False,
+ residual_in_hidden=False,
+ fusions=None,
+):
+ """Resolve this attention's boundaries, then bind its norm and hooks.
+
+ ``following`` supplies the consumer declaration for a stage inside a
+ layer. With no following stage, the output is handed across a layer or
+ stack boundary; the receiver binds its own read independently.
+ """
+ if declaration.kind is not StageKind.ATTENTION:
+ raise TypeError("make_attn_stage requires an attention declaration")
+ incoming, outgoing = _connections(declaration, following)
+ return _bind_stage(
+ declaration,
+ norm,
+ incoming,
+ outgoing,
+ qkv_latent_func=qkv_latent_func,
+ force_layernorm_before_dp_gather=force_layernorm_before_dp_gather,
+ enable_fused_ar_quant=enable_fused_ar_quant,
+ fused_ar_quant_keep_bf16=fused_ar_quant_keep_bf16,
+ residual_in_hidden=residual_in_hidden,
+ fusions=fusions,
+ )
+
+
+def make_ffn_stage(
+ *,
+ declaration,
+ norm,
+ following: Optional[StageDeclaration] = None,
+ force_layernorm_before_dp_gather=False,
+ fusions=None,
+):
+ """Resolve this FFN's boundaries and bind its norm; EP stays inside compute."""
+ if declaration.kind is not StageKind.FFN:
+ raise TypeError("make_ffn_stage requires an FFN declaration")
+ incoming, outgoing = _connections(declaration, following)
+ return _bind_stage(
+ declaration,
+ norm,
+ incoming,
+ outgoing,
+ force_layernorm_before_dp_gather=force_layernorm_before_dp_gather,
+ residual_in_hidden=declaration.update.at_producer,
+ fusions=fusions,
+ )
+
+
+def make_stages(*stages, previous=None, prepared_from=None, terminal=False):
+ """Bind a local linear sequence and return its independent boundaries.
+
+ Each item is ``(declaration, norm)`` or ``(declaration, norm, options)``.
+ Options are the stage-specific keyword arguments of ``make_attn_stage``
+ or ``make_ffn_stage``; they are consumed only during construction.
+ Declarations in the sequence have no input source: this function connects
+ them in order. Only the final boundary is terminal when requested.
+
+ ``previous`` describes an external producer's output. ``prepared_from``
+ describes an already-read branch input, obtainable from the source
+ boundary's immutable ``declaration``. Neither takes an executable boundary.
+ No sequence object or runtime routing is retained.
+ """
+ if not stages:
+ raise ValueError("make_stages needs at least one stage")
+ if previous is not None and prepared_from is not None:
+ raise ValueError("choose a previous output or a prepared branch input")
+ declarations = []
+ bindings = []
+ for index, item in enumerate(stages):
+ if len(item) not in (2, 3):
+ raise ValueError("a stage needs (declaration, norm[, options])")
+ declaration, norm = item[:2]
+ if not isinstance(declaration, StageDeclaration):
+ raise TypeError("make_stages requires stage declarations")
+ if declaration.previous is not None or declaration.prepared_from is not None:
+ raise ValueError("pass input sources to make_stages, not its declarations")
+ if declaration.terminal:
+ raise ValueError("pass terminal to make_stages, not its declarations")
+ declaration = replace(
+ declaration,
+ previous=previous if index == 0 else declarations[-1],
+ prepared_from=prepared_from if index == 0 else None,
+ terminal=terminal and index == len(stages) - 1,
+ next_kind=stages[index + 1][0].kind
+ if declaration.mixer_exit and index + 1 < len(stages)
+ else declaration.next_kind,
+ )
+ declarations.append(declaration)
+ bindings.append((norm, item[2] if len(item) == 3 else {}))
+ boundaries = []
+ for index, (declaration, (norm, options)) in enumerate(zip(declarations, bindings)):
+ build = (
+ make_attn_stage
+ if declaration.kind is StageKind.ATTENTION
+ else make_ffn_stage
+ )
+ boundaries.append(
+ build(
+ declaration=declaration,
+ norm=norm,
+ following=declarations[index + 1]
+ if index + 1 < len(declarations)
+ else None,
+ **options,
+ )
+ )
+ for producer, consumer in zip(boundaries, boundaries[1:]):
+ if (
+ producer.kind is not StageKind.ATTENTION
+ or producer.declaration.mixer_exit
+ or consumer.kind is not StageKind.FFN
+ ):
+ continue
+ for variant, steps in producer.plan._paths.items():
+ predicate = consumer.plan._paths[variant].ffn.preserves_residual
+ if predicate is not None:
+ producer.plan._paths[variant] = msgspec.structs.replace(
+ steps,
+ attention=msgspec.structs.replace(
+ steps.attention, capture_preserves_residual=predicate
+ ),
+ )
+ producer.plan._steps = producer.plan._paths[BatchVariant.ORDINARY]
+ producer.plan._cp_steps = producer.plan._paths.get(
+ BatchVariant.CONTEXT_PARALLEL
+ )
+ producer.plan._input_scattered_steps = producer.plan._paths.get(
+ BatchVariant.INPUT_SCATTERED
+ )
+ producer.plan._sp_steps = producer.plan._paths.get(
+ BatchVariant.SEQUENCE_PARALLEL
+ )
+ return tuple(boundaries)
diff --git a/python/sglang/srt/layers/communicator/fusions/allreduce.py b/python/sglang/srt/layers/communicator/fusions/allreduce.py
new file mode 100644
index 0000000000..76bdb3b191
--- /dev/null
+++ b/python/sglang/srt/layers/communicator/fusions/allreduce.py
@@ -0,0 +1,148 @@
+# Copyright 2023-2024 SGLang Team
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+
+"""Select fused completion and read kernels for each consumer."""
+
+from functools import partial
+from typing import Callable, Optional, Tuple
+
+import torch
+
+from sglang.srt.layers.communicator.boundary import FusedMlpInput
+from sglang.srt.layers.communicator.layout import SumGroup
+from sglang.srt.layers.communicator.output import UnreducedOutput
+from sglang.srt.layers.communicator.residual.add_norm import (
+ apply_aiter_all_reduce_fusion,
+ apply_flashinfer_allreduce_fusion,
+)
+from sglang.srt.layers.layernorm import GemmaRMSNorm, RMSNorm
+from sglang.srt.layers.moe import post_experts_reduction_group
+from sglang.srt.model_executor.forward_batch_info import ForwardBatch
+from sglang.srt.runtime_context import get_parallel
+from sglang.srt.utils import get_bool_env_var, is_hip
+
+_use_aiter = get_bool_env_var("SGLANG_USE_AITER") and is_hip()
+
+
+def attention_fusions(plan) -> Tuple[Callable, ...]:
+ """The fused kernels that complete what the previous layer left together
+ with the residual update and the input norm, in the order they are tried.
+ Each takes (owed, residual, forward_batch, post_residual_addition) and
+ returns None when it does not take the batch. They add the residual
+ plainly; the boundary tries them only when the update it writes in is a
+ plain add. A backend's come first."""
+ given = plan.fusions.attention_input(plan) if plan.fusions else ()
+ if not hasattr(plan.norm, "forward_with_allreduce_fusion"):
+ return given
+ plan._attn_input_fuses_quant = (
+ plan.enable_fused_ar_quant
+ and _use_aiter
+ and hasattr(plan.norm, "forward_with_allreduce_fusion_quant_per_group")
+ )
+ return (*given, partial(complete_attention_input, plan))
+
+
+def complete_attention_input(
+ plan,
+ owed: UnreducedOutput,
+ residual: torch.Tensor,
+ forward_batch: ForwardBatch,
+ post_residual_addition: Optional[torch.Tensor],
+) -> Optional[Tuple[torch.Tensor, torch.Tensor]]:
+ """Complete the sum the previous layer left, add it to the residual and
+ apply the input norm in one aiter or flashinfer kernel; None when the
+ kernel does not take this batch. The result is not quantized for
+ ``quant_format``."""
+ if (
+ not isinstance(owed, UnreducedOutput)
+ # The kernel does not add it.
+ or post_residual_addition is not None
+ # The kernel reduces over the MoE output's group.
+ or owed.group is not post_experts_reduction_group()
+ ):
+ return None
+ hidden_states = owed.partial
+ if not (
+ apply_aiter_all_reduce_fusion(hidden_states, forward_batch)
+ or apply_flashinfer_allreduce_fusion(hidden_states.shape[0])
+ ):
+ return None
+ if plan._attn_input_fuses_quant:
+ # Falls back to AR+RMSNorm + separate quant internally when the
+ # fully-fused kernel cannot service the shape.
+ quant_result = plan.norm.forward_with_allreduce_fusion_quant_per_group(
+ hidden_states,
+ residual,
+ use_attn_tp_group=False,
+ keep_bf16=plan.fused_ar_quant_keep_bf16,
+ )
+ if quant_result is not None:
+ return quant_result
+ return plan.norm.forward_with_allreduce_fusion(
+ hidden_states, residual, use_attn_tp_group=False
+ )
+
+
+def ffn_fusions(plan) -> Tuple["FusedMlpInput", ...]:
+ """The fused kernels that can take the attention -> FFN steps, in the
+ order they are tried. They add the residual plainly; the boundary tries
+ them only when the update it writes in is a plain add. A backend's come
+ first."""
+ given = plan.fusions.ffn_input(plan) if plan.fusions else ()
+ if not hasattr(plan.norm, "forward_with_allreduce_fusion"):
+ return given
+ return (
+ *given,
+ FusedMlpInput(
+ completes=SumGroup.ATTN_TP,
+ run=partial(complete_ffn_input, plan),
+ preserves_residual=(
+ flashinfer_preserves_residual
+ if isinstance(plan.norm, (RMSNorm, GemmaRMSNorm))
+ and type(plan.norm).forward_with_allreduce_fusion
+ in (
+ RMSNorm.forward_with_allreduce_fusion,
+ GemmaRMSNorm.forward_with_allreduce_fusion,
+ )
+ else None
+ ),
+ ),
+ )
+
+
+def flashinfer_preserves_residual(value, forward_batch):
+ """The CUDA wrapper returns a fresh residual, including backend fallback."""
+ return (
+ not _use_aiter
+ and get_parallel().attn_tp_size > 1
+ and apply_flashinfer_allreduce_fusion(value.shape[0])
+ )
+
+
+def complete_ffn_input(
+ plan,
+ hidden_states: torch.Tensor,
+ residual: torch.Tensor,
+ forward_batch: ForwardBatch,
+) -> Optional[Tuple[torch.Tensor, torch.Tensor]]:
+ """The attention-TP all-reduce, residual add and norm in one aiter or
+ flashinfer kernel; None when neither takes the batch."""
+ if not (
+ apply_aiter_all_reduce_fusion(hidden_states, forward_batch)
+ or apply_flashinfer_allreduce_fusion(hidden_states.shape[0])
+ ):
+ return None
+ return plan.norm.forward_with_allreduce_fusion(
+ hidden_states, residual, use_attn_tp_group=True
+ )
diff --git a/python/sglang/srt/layers/communicator/layer.py b/python/sglang/srt/layers/communicator/layer.py
index 279eb29a18..91df3effbb 100644
--- a/python/sglang/srt/layers/communicator/layer.py
+++ b/python/sglang/srt/layers/communicator/layer.py
@@ -42,8 +42,8 @@ from sglang.srt.layers.communicator.boundary import (
sequence_parallel_layer_sides,
with_residual,
)
+from sglang.srt.layers.communicator.exit import _reduce_and_redistribute_output_step
from sglang.srt.layers.communicator.layout import (
- CommunicateContext,
Layout,
SumGroup,
TokenAxis,
@@ -56,6 +56,7 @@ from sglang.srt.layers.communicator.layout import (
sparse_moe_gathers_over_moe_cp,
token_axis_sizes,
)
+from sglang.srt.layers.communicator.legacy_stage import StageCommunicator
from sglang.srt.layers.communicator.ops import (
CommunicateSimpleFn,
_all_reduce_then_to_local_tokens,
@@ -63,7 +64,6 @@ from sglang.srt.layers.communicator.ops import (
_hand_scattered_input_to_attention,
_mlp_input_without_dp,
_redistribute_output,
- _reduce_and_redistribute_output_step,
_to_local_tokens,
move_rows,
)
@@ -189,79 +189,6 @@ def _can_defer_ffn_reduction(forward_batch: ForwardBatch, boundary=None) -> bool
)
-class StageCommunicator:
- """One stage of a layer, its attention or its FFN: the boundary into it,
- run with the stage's norm on the steps the layer chose for the batch."""
-
- def __init__(self, layer: "LayerCommunicator", name: str, norm_name: str):
- self._layer = layer
- self._name = name
- self._norm_name = norm_name
-
- @property
- def norm(self) -> torch.nn.Module:
- """The norm the stage reads its input with, one of the layer's."""
- return getattr(self._layer, self._norm_name)
-
- def entry(
- self, forward_batch: ForwardBatch, steps: Optional[BoundarySteps] = None
- ) -> StageEntry:
- if steps is None:
- steps = self._layer._batch_steps(forward_batch)
- entry = getattr(steps, self._name)
- if entry is None:
- raise NotImplementedError(
- f"a layer that is one stage has no {self._name} stage"
- )
- return entry
-
- def prepare(
- self,
- hidden_states,
- residual,
- forward_batch: ForwardBatch,
- steps: Optional[BoundarySteps] = None,
- **call,
- ):
- """The stage's input and the residual, from the previous stage's output
- (``call``: what the stage's read takes, e.g. the attention's
- ``quant_format``)."""
- entry = self.entry(forward_batch, steps)
- context = self._layer._context
- stream = residual_batch.current(forward_batch)
- if residual is not stream:
- raise RuntimeError("residual alias belongs to a different invocation")
- if (
- stream.pending is None
- and stream.residual is None
- and entry.input_sum is not None
- ):
- hidden_states = stream.leave(
- hidden_states, ADD, declared_sum=entry.input_sum
- )
- if stream.pending is not None:
- call["update"] = stream.pending.update
- call["pending"] = stream.pending
- if stream.pending is None and stream.residual is not None:
- call["written"] = True
- hidden_states, residual = stream.input(hidden_states)
- hidden_states, residual = entry.prepare(
- hidden_states, residual, forward_batch, self.norm, context, **call
- )
- if entry.input_move is not None:
- hidden_states = entry.input_move(
- hidden_states=hidden_states,
- forward_batch=forward_batch,
- context=context,
- )
- if entry.handoff is not None:
- hidden_states = entry.handoff(
- hidden_states, forward_batch, self._layer.qkv_latent_func
- )
- stream.write(residual)
- return hidden_states, stream
-
-
class LayerFusions(Protocol):
"""The fused kernels a fusion backend gives a layer, bound to it and tried
before the layer's own: at its attention input (each takes what the previous
@@ -336,10 +263,6 @@ class LayerCommunicator:
self._residual = residual
self.fusions = fusions
- self._context = CommunicateContext.init_new()
- self._context.force_layernorm_before_dp_gather = (
- force_layernorm_before_dp_gather
- )
# The fused kernels every batch's attention input tries first.
self._attn_input_fusions = self._select_attn_input_fusions()
self._speculative_algo = SpeculativeAlgorithm.from_string(
@@ -799,7 +722,6 @@ class LayerCommunicator:
else move(
hidden_states=residual_value,
forward_batch=forward_batch,
- context=self._context,
)
)
keeps = self.attn.entry(forward_batch).capture_preserves_residual
@@ -850,7 +772,7 @@ class LayerCommunicator:
)
if get_forward().sp_active:
hidden_states = layernorm_sp.sp_entry_scatter(hidden_states)
- return self.attn.prepare(
+ return self.attn._prepare(
hidden_states,
residual,
forward_batch,
@@ -979,8 +901,6 @@ class LayerCommunicator:
):
steps = self._batch_steps(forward_batch)
self.publish_mlp_lora_layout(steps)
- if cache is not None:
- self._context.cache = cache
if isinstance(residual, ResidualStream) and self.stage_edges is None:
# The paired boundary records the attention contribution once.
@@ -989,12 +909,13 @@ class LayerCommunicator:
self._residual.attention_update,
declared_sum=steps.ffn.input_sum,
)
- return self.ffn.prepare(
+ return self.ffn._prepare(
hidden_states,
residual,
forward_batch,
steps,
update=self._residual.attention_update,
+ cache=cache,
)
def maybe_prefetch_next_full_attention_kv(
@@ -1180,7 +1101,6 @@ class LayerCommunicator:
hidden_states=hidden_states,
residual=residual,
forward_batch=forward_batch,
- context=self._context,
allow_reduce_scatter=self.allow_reduce_scatter,
is_layer_sparse=self.layer_facts.is_layer_sparse,
)
@@ -1317,7 +1237,7 @@ class LayerCommunicator:
input when no fused kernel takes it: when the next layer would run the
same all-reduce the FFN itself would have."""
return (
- self._context.tp_size > 1
+ get_parallel().tp_size > 1
and not self.is_last_layer
and self._ffn_sum_can_move_to_next_layer(forward_batch)
and _can_defer_ffn_reduction(forward_batch, self)
@@ -1361,15 +1281,23 @@ class MixerExit:
its input), and when its output declaration permits deferring the sum. The
consumer decides whether to fuse its completion with the input norm."""
- __slots__ = ("skips_reduction", "_hands_on", "_scope", "_update", "_declared_sum")
+ __slots__ = (
+ "skips_reduction",
+ "_hands_on",
+ "_scope",
+ "_update",
+ "_declared_sum",
+ "_stream",
+ )
def __init__(self, communicator: LayerCommunicator, forward_batch: ForwardBatch):
produced = communicator._batch_steps(forward_batch).ffn_output
+ self._stream = None
self._update = produced.update
self._declared_sum = produced.group if produced.always_leaves else None
self._hands_on = (
produced.leaves_for_next_layer
- and communicator._context.tp_size > 1
+ and get_parallel().tp_size > 1
and not is_dp_attention_enabled()
and communicator._ffn_sum_can_move_to_next_layer(forward_batch)
)
@@ -1386,14 +1314,17 @@ class MixerExit:
def finish(self, hidden_states: torch.Tensor, residual=None):
"""The mixer's output: its partial sum to the FFN stage after it, as a
value that owes the sum to a mixer after it, or complete."""
+ if self._stream is not None:
+ residual = self._stream
if self._hands_on:
hidden_states = UnreducedOutput(
hidden_states, group=get_parallel().tp_group
)
if isinstance(residual, ResidualStream):
- return residual.leave(
+ output = residual.leave(
hidden_states, self._update, declared_sum=self._declared_sum
- ), residual
+ )
+ return output if self._stream is not None else (output, residual)
return hidden_states
@@ -1412,9 +1343,11 @@ class FfnExit:
"_update",
"_declared_sum",
"_scope",
+ "_stream",
)
def __init__(self, communicator: LayerCommunicator, forward_batch: ForwardBatch):
+ self._stream = None
self.communicator = communicator
self.forward_batch = forward_batch
completion = communicator._select_ffn_completion(forward_batch)
@@ -1441,7 +1374,7 @@ class FfnExit:
return self._scope.__exit__(*exc_info)
def finish(
- self, hidden_states: torch.Tensor, residual: torch.Tensor
+ self, hidden_states: torch.Tensor, residual: torch.Tensor = None
) -> Tuple[Union[torch.Tensor, UnreducedOutput], torch.Tensor]:
"""Leave the reduction to the next layer's input, or postprocess."""
stream = residual_batch.current(self.forward_batch)
@@ -1452,9 +1385,10 @@ class FfnExit:
assert self.defer_moe_finalize, "unrequested deferred MoE handoff"
else:
hidden_states, residual = self._complete(hidden_states, residual)
- return self.communicator._leave_ffn_output(
+ result = self.communicator._leave_ffn_output(
hidden_states, residual, stream, self._update, self._declared_sum
)
+ return result[0] if self._stream is not None else result
class MHCLayerCommunicator(LayerCommunicator):
@@ -1501,7 +1435,7 @@ class MHCLayerCommunicator(LayerCommunicator):
residual=self.mhc.layer_residual(),
)
# MHC has not been run with input-scattered attention under attention CP.
- if self._context.attn_cp_size > 1 and self._input_can_be_scattered():
+ if get_parallel().attn_cp_size > 1 and self._input_can_be_scattered():
raise NotImplementedError(
"MHCLayerCommunicator with input-scattered attention under attention CP"
)
diff --git a/python/sglang/srt/layers/communicator/layout.py b/python/sglang/srt/layers/communicator/layout.py
index 87a800e89b..afa617479d 100644
--- a/python/sglang/srt/layers/communicator/layout.py
+++ b/python/sglang/srt/layers/communicator/layout.py
@@ -14,7 +14,6 @@
"""Token axes, layouts and process groups of what a layer hands across its
boundaries, and the configuration and batch facts they depend on."""
-from dataclasses import dataclass
from enum import Enum, auto
from typing import Dict, FrozenSet, List, Mapping, Optional
@@ -94,38 +93,6 @@ def _ffn_has_tokens(forward_batch: ForwardBatch) -> bool:
return _batch_size(forward_batch) > 0
-@dataclass
-class CommunicateContext:
- attn_tp_rank: int
- attn_tp_size: int
- attn_dp_size: int
- attn_cp_rank: int
- attn_cp_size: int
- tp_size: int
- cache = None
- tp_rank: int
- force_layernorm_before_dp_gather: bool = False
-
- @classmethod
- def init_new(cls):
- attn_tp_rank = get_parallel().attn_tp_rank
- attn_tp_size = get_parallel().attn_tp_size
- attn_dp_size = get_parallel().attn_dp_size
- attn_cp_size = get_parallel().attn_cp_size
- attn_cp_rank = get_parallel().attn_cp_rank
- tp_size = get_parallel().tp_size
- tp_rank = get_parallel().tp_rank
- return cls(
- attn_tp_rank=attn_tp_rank,
- attn_tp_size=attn_tp_size,
- attn_dp_size=attn_dp_size,
- attn_cp_rank=attn_cp_rank,
- attn_cp_size=attn_cp_size,
- tp_size=tp_size,
- tp_rank=tp_rank,
- )
-
-
def moe_cp_gathered_rows(forward_batch: ForwardBatch) -> Optional[List[int]]:
"""The real rows each MoE-CP rank contributes when this batch's FFN input is
gathered across the MoE-CP group, or None when it is not: only a context
diff --git a/python/sglang/srt/layers/communicator/legacy_stage.py b/python/sglang/srt/layers/communicator/legacy_stage.py
new file mode 100644
index 0000000000..476b9bbc90
--- /dev/null
+++ b/python/sglang/srt/layers/communicator/legacy_stage.py
@@ -0,0 +1,160 @@
+# Copyright 2023-2024 SGLang Team
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""A stage's model-facing input and output boundary."""
+
+from typing import TYPE_CHECKING, Optional
+
+from sglang.srt.layers.communicator.boundary import BoundarySteps, StageEntry
+from sglang.srt.layers.communicator.residual.add_norm import ADD
+from sglang.srt.layers.communicator.residual.batch import current
+from sglang.srt.model_executor.forward_batch_info import ForwardBatch
+
+if TYPE_CHECKING:
+ from sglang.srt.layers.communicator.layer import LayerCommunicator
+
+
+class StageCommunicator:
+ """One stage of a layer, its attention or its FFN: the boundary into it,
+ run with the stage's norm on the steps the layer chose for the batch."""
+
+ def __init__(self, layer: "LayerCommunicator", name: str, norm_name: str):
+ self._layer = layer
+ self._name = name
+ self.norm = getattr(layer, norm_name)
+
+ def entry(
+ self, forward_batch: ForwardBatch, steps: Optional[BoundarySteps] = None
+ ) -> StageEntry:
+ if steps is None:
+ steps = self._layer._batch_steps(forward_batch)
+ entry = getattr(steps, self._name)
+ if entry is None:
+ raise NotImplementedError(
+ f"a layer that is one stage has no {self._name} stage"
+ )
+ return entry
+
+ def _prepare(
+ self,
+ hidden_states,
+ residual,
+ forward_batch: ForwardBatch,
+ steps: Optional[BoundarySteps] = None,
+ **call,
+ ):
+ """The stage's input and the residual, from the previous stage's output
+ (``call``: what the stage's read takes, e.g. the attention's
+ ``quant_format``)."""
+ entry = self.entry(forward_batch, steps)
+ stream = current(forward_batch)
+ if residual is not stream:
+ raise RuntimeError("residual alias belongs to a different invocation")
+ if (
+ stream.pending is None
+ and stream.residual is None
+ and entry.input_sum is not None
+ ):
+ hidden_states = stream.leave(
+ hidden_states, ADD, declared_sum=entry.input_sum
+ )
+ if stream.pending is not None:
+ call["update"] = stream.pending.update
+ call["pending"] = stream.pending
+ if stream.pending is None and stream.residual is not None:
+ call["written"] = True
+ hidden_states, residual = stream.input(hidden_states)
+ hidden_states, residual = entry.prepare(
+ hidden_states, residual, forward_batch, self.norm, **call
+ )
+ if entry.input_move is not None:
+ hidden_states = entry.input_move(
+ hidden_states=hidden_states,
+ forward_batch=forward_batch,
+ )
+ if entry.handoff is not None:
+ hidden_states = entry.handoff(
+ hidden_states, forward_batch, self._layer.qkv_latent_func
+ )
+ stream.write(residual)
+ return hidden_states, stream
+
+ @property
+ def input_rows(self):
+ return self._layer.input_rows
+
+ @property
+ def input_on_attention_tp_slices(self):
+ return self._layer.input_on_attention_tp_slices
+
+ def prepare(self, hidden_states, forward_batch, *, cache=None, **call):
+ stream = current(forward_batch)
+ if self._name == "attention":
+ hidden_states, stream = (
+ self._layer.prepare_attn_and_capture_last_layer_outputs(
+ hidden_states, stream, forward_batch, **call
+ )
+ )
+ else:
+ steps = self._layer._batch_steps(forward_batch)
+ self._layer.publish_mlp_lora_layout(steps)
+ if cache is not None:
+ call["cache"] = cache
+ hidden_states, stream = self._prepare(
+ hidden_states, stream, forward_batch, steps, **call
+ )
+ forward_batch.residual_stream = stream
+ return hidden_states
+
+ def finish(self, hidden_states, forward_batch):
+ """Hand this attention's actual contribution to the following stage."""
+ if self._name != "attention" or self._layer.stage_edges is not None:
+ raise RuntimeError(
+ "use the stage exit scope for an FFN or single-stage mixer"
+ )
+ return current(forward_batch).leave(
+ hidden_states,
+ self._layer._residual.attention_update,
+ declared_sum=self._layer._batch_steps(forward_batch).ffn.input_sum,
+ )
+
+ def exit(self, forward_batch):
+ result = (
+ self._layer.ffn_exit(forward_batch)
+ if self._name == "ffn"
+ else self._layer.mixer_exit(forward_batch)
+ )
+ result._stream = current(forward_batch)
+ return result
+
+ def from_pp(self, tensors, forward_batch, **kwargs):
+ hidden_states, stream = self._layer.from_pp(tensors, forward_batch, **kwargs)
+ forward_batch.residual_stream = stream
+ return hidden_states
+
+ def snapshot(self, hidden_states, forward_batch, *, at_input=False):
+ return self._layer.snapshot(
+ hidden_states, current(forward_batch), at_input=at_input
+ )
+
+ def capture_output(self, hidden_states, forward_batch, **kwargs):
+ return self._layer.capture_output(
+ hidden_states, current(forward_batch), **kwargs
+ )
+
+ def postprocess(self, hidden_states, forward_batch):
+ hidden_states, stream = self._layer.postprocess_layer(
+ hidden_states, current(forward_batch), forward_batch
+ )
+ forward_batch.residual_stream = stream
+ return hidden_states
diff --git a/python/sglang/srt/layers/communicator/ops.py b/python/sglang/srt/layers/communicator/ops.py
index e42c526fb2..a9a3810654 100644
--- a/python/sglang/srt/layers/communicator/ops.py
+++ b/python/sglang/srt/layers/communicator/ops.py
@@ -38,7 +38,6 @@ from sglang.srt.layers.communicator.adapters.context_parallel import (
dsa_cp_reduce_scatter_hidden_states,
)
from sglang.srt.layers.communicator.layout import (
- CommunicateContext,
Layout,
SumGroup,
TokenAxis,
@@ -56,7 +55,6 @@ from sglang.srt.layers.communicator.residual.stream import DeclaredSum
from sglang.srt.layers.dp_attention import (
attn_tp_all_gather_into_tensor,
attn_tp_reduce_scatter_tensor,
- can_use_dp_reduce_scatter,
dp_gather_partial,
dp_gather_replicate,
dp_reduce_scatter_tensor,
@@ -69,7 +67,6 @@ from sglang.srt.layers.dp_attention import (
is_allocation_symmetric,
moe_cp_all_gather_into_tensor,
)
-from sglang.srt.layers.moe import should_use_dp_reduce_scatterv
from sglang.srt.model_executor.forward_batch_info import ForwardBatch
from sglang.srt.runtime_context import get_exec, get_parallel
from sglang.srt.utils import (
@@ -86,28 +83,30 @@ if _is_npu:
def tp_reduce_scatter(
hidden_states: torch.Tensor,
residual: Optional[torch.Tensor],
- context: "CommunicateContext",
) -> Tuple[torch.Tensor, Optional[torch.Tensor]]:
- """Module-level so MHC communicators can reuse it without a
- ``LayerCommunicator`` instance."""
+ """Complete a TP sum onto each rank's token slice."""
+ parallel = get_parallel()
if hidden_states.shape[0] == 0:
return hidden_states, hidden_states
- assert hidden_states.shape[0] % context.tp_size == 0, (
- f"Expected total tokens {hidden_states.shape[0]} % tp_size {context.tp_size} to be 0"
+ assert hidden_states.shape[0] % parallel.tp_size == 0, (
+ f"Expected total tokens {hidden_states.shape[0]} % tp_size {parallel.tp_size} to be 0"
)
- local_tokens = hidden_states.shape[0] // context.tp_size
+ local_tokens = hidden_states.shape[0] // parallel.tp_size
output = hidden_states.new_empty(local_tokens, *hidden_states.shape[1:])
- get_parallel().tp_group.reduce_scatter_tensor(output, hidden_states)
+ parallel.tp_group.reduce_scatter_tensor(output, hidden_states)
if residual is not None:
- residual = residual.tensor_split(context.tp_size)[context.tp_rank]
+ residual = residual.tensor_split(parallel.tp_size)[parallel.tp_rank]
return output, residual
-def tp_slice(hidden_states, residual, context):
+def tp_slice(hidden_states, residual):
"""The rows reduce-scatter would return, when the sum is already complete."""
- hidden_states = hidden_states.tensor_split(context.tp_size)[context.tp_rank].clone()
+ parallel = get_parallel()
+ hidden_states = hidden_states.tensor_split(parallel.tp_size)[
+ parallel.tp_rank
+ ].clone()
if residual is not None:
- residual = residual.tensor_split(context.tp_size)[context.tp_rank]
+ residual = residual.tensor_split(parallel.tp_size)[parallel.tp_rank]
return hidden_states, residual
@@ -116,7 +115,6 @@ class CommunicateSimpleFn:
def _trivial(
hidden_states: torch.Tensor,
forward_batch: ForwardBatch,
- context: CommunicateContext,
) -> torch.Tensor:
return hidden_states
@@ -124,18 +122,18 @@ class CommunicateSimpleFn:
def _scattered_to_tp_attn_full(
hidden_states: Union[torch.Tensor, Tuple[torch.Tensor, torch.Tensor]],
forward_batch: ForwardBatch,
- context: CommunicateContext,
) -> Union[torch.Tensor, Tuple[torch.Tensor, torch.Tensor]]:
+ parallel = get_parallel()
if isinstance(hidden_states, tuple):
gathered_hidden_states = []
for local_hidden_states in hidden_states:
with use_symmetric_memory(
- get_parallel().tp_group,
+ parallel.tp_group,
disabled=not is_allocation_symmetric(),
):
output = torch.empty(
(
- local_hidden_states.shape[0] * context.attn_tp_size,
+ local_hidden_states.shape[0] * parallel.attn_tp_size,
*local_hidden_states.shape[1:],
),
dtype=local_hidden_states.dtype,
@@ -152,10 +150,11 @@ class CommunicateSimpleFn:
def _reduce_and_redistribute_output_to_attn_tp_shards(
- hidden_states: torch.Tensor, context: CommunicateContext
+ hidden_states: torch.Tensor,
) -> torch.Tensor:
- local_hidden_states = hidden_states.tensor_split(context.attn_tp_size)[
- context.attn_tp_rank
+ parallel = get_parallel()
+ local_hidden_states = hidden_states.tensor_split(parallel.attn_tp_size)[
+ parallel.attn_tp_rank
]
attn_tp_reduce_scatter_tensor(local_hidden_states, hidden_states)
return local_hidden_states
@@ -250,8 +249,8 @@ def _mlp_input_without_dp(
residual: torch.Tensor,
forward_batch: ForwardBatch,
layernorm: torch.nn.Module,
- context: CommunicateContext,
*,
+ cache=None,
gathers_residual: bool,
fusions: Tuple[Callable, ...],
group: SumGroup = SumGroup.ATTN_TP,
@@ -274,8 +273,8 @@ def _mlp_input_without_dp(
)
elif group is SumGroup.TP:
hidden_states = tensor_model_parallel_all_reduce(hidden_states)
- if _is_npu and context.cache is not None:
- _ = prepare_weight_cache(hidden_states, context.cache)
+ if _is_npu and cache is not None:
+ _ = prepare_weight_cache(hidden_states, cache)
return read.update_and_read(update, hidden_states, residual, layernorm)
@@ -284,8 +283,8 @@ def _mlp_input_dp_replicate(
residual: torch.Tensor,
forward_batch: ForwardBatch,
layernorm: torch.nn.Module,
- context: CommunicateContext,
*,
+ cache=None,
gathers_residual: bool,
reduces_attention_tp: bool,
places_cp_shards: bool = False,
@@ -324,8 +323,8 @@ def _mlp_input_dp_partial(
residual: torch.Tensor,
forward_batch: ForwardBatch,
layernorm: torch.nn.Module,
- context: CommunicateContext,
*,
+ cache=None,
gathers_residual: bool,
places_cp_shards: bool = False,
read: StageRead = NORM_READ,
@@ -336,7 +335,7 @@ def _mlp_input_dp_partial(
shard of the DP group's tokens beside the others' in the group's slot."""
if gathers_residual:
residual = _redistribute_from_attn_tp_shards(residual)
- if context.attn_tp_rank == 0:
+ if get_parallel().attn_tp_rank == 0:
hidden_states += residual
cp_shard_counts = _cp_shard_token_rows(forward_batch) if places_cp_shards else None
hidden_states = _reduce_and_redistribute_output_to_dp(
@@ -395,7 +394,7 @@ def _tp_all_gather_scattered_rows(
def _dispatch_consumer(
- hidden_states, residual, forward_batch, norm, context, *, paths, update=ADD, **call
+ hidden_states, residual, forward_batch, norm, *, paths, update=ADD, **call
):
if residual is None:
# Enter/read-only paths do not execute an update.
@@ -405,9 +404,7 @@ def _dispatch_consumer(
prepare = paths[update.adds_plainly]
except KeyError:
raise RuntimeError("producer update has no bound input path") from None
- return prepare(
- hidden_states, residual, forward_batch, norm, context, update=update, **call
- )
+ return prepare(hidden_states, residual, forward_batch, norm, update=update, **call)
def _consumer_step(
@@ -415,7 +412,6 @@ def _consumer_step(
residual: Optional[torch.Tensor],
forward_batch: ForwardBatch,
norm: torch.nn.Module,
- context: CommunicateContext,
*,
step: Callable,
adds_plainly: bool,
@@ -473,9 +469,7 @@ def _consumer_step(
hidden_states = reduce_output(owed)
else:
hidden_states = owed.partial
- return step(
- hidden_states, residual, forward_batch, norm, context, update=update, **call
- )
+ return step(hidden_states, residual, forward_batch, norm, update=update, **call)
def _read_input(
@@ -483,8 +477,8 @@ def _read_input(
residual: Optional[torch.Tensor],
forward_batch: ForwardBatch,
norm: torch.nn.Module,
- context: CommunicateContext,
*,
+ cache=None,
layer_input: Optional[Callable],
enters_stack: bool,
read: StageRead,
@@ -498,7 +492,7 @@ def _read_input(
starts its residual from its input."""
enters = residual is None and enters_stack
if layer_input is not None:
- hidden_states, residual = layer_input(hidden_states, residual, context)
+ hidden_states, residual = layer_input(hidden_states, residual)
if enters:
hidden_states, residual = read.enter(hidden_states), None
if residual is None:
@@ -519,17 +513,15 @@ def _mlp_input_scatter(
residual: torch.Tensor,
forward_batch: ForwardBatch,
layernorm: torch.nn.Module,
- context: CommunicateContext,
*,
+ cache=None,
scatters_residual: bool,
read: StageRead = NORM_READ,
update: StageUpdate = ADD,
):
- hidden_states = _reduce_and_redistribute_output_to_attn_tp_shards(
- hidden_states, context
- )
+ hidden_states = _reduce_and_redistribute_output_to_attn_tp_shards(hidden_states)
if scatters_residual:
- residual = update.residual_to_attn_tp_shard(residual, context)
+ residual = update.residual_to_attn_tp_shard(residual)
return read.update_and_read(update, hidden_states, residual, layernorm)
@@ -538,15 +530,15 @@ def _mlp_input_slice(
residual: Optional[torch.Tensor],
forward_batch: ForwardBatch,
layernorm: torch.nn.Module,
- context: CommunicateContext,
*,
+ cache=None,
scatters_residual: bool,
read: StageRead = NORM_READ,
update: StageUpdate = ADD,
):
- hidden_states = _redistribute_to_attn_tp_shards(hidden_states, context).clone()
+ hidden_states = _redistribute_to_attn_tp_shards(hidden_states).clone()
if scatters_residual and residual is not None:
- residual = update.residual_to_attn_tp_shard(residual, context)
+ residual = update.residual_to_attn_tp_shard(residual)
return read.update_and_read(update, hidden_states, residual, layernorm)
@@ -555,8 +547,8 @@ def _mlp_input_on_residual_shard(
residual: torch.Tensor,
forward_batch: ForwardBatch,
layernorm: torch.nn.Module,
- context: CommunicateContext,
*,
+ cache=None,
reduces: bool = True,
read: StageRead,
update: StageUpdate,
@@ -565,11 +557,12 @@ def _mlp_input_on_residual_shard(
rows: reduce-scatter the attention output onto the slice, which completes
its sum, write it into the residual and read the FFN input there, then
gather the input back into the attention output's rows."""
+ parallel = get_parallel()
if hidden_states.shape[0] == 0:
return hidden_states, hidden_states
- shard = hidden_states.tensor_split(context.tp_size)[context.tp_rank]
+ shard = hidden_states.tensor_split(parallel.tp_size)[parallel.tp_rank]
if reduces:
- get_parallel().tp_group.reduce_scatter_tensor(shard, hidden_states)
+ parallel.tp_group.reduce_scatter_tensor(shard, hidden_states)
else:
shard = shard.clone()
shard, residual = read.update_and_read(update, shard, residual, layernorm)
@@ -582,13 +575,13 @@ def _mlp_input_residual_into_sum(
residual: torch.Tensor,
forward_batch: ForwardBatch,
layernorm: torch.nn.Module,
- context: CommunicateContext,
*,
+ cache=None,
read: StageRead = NORM_READ,
update: StageUpdate = ADD,
):
return _tp_all_reduce_with_scattered_residual(
- hidden_states, residual, layernorm, context, read
+ hidden_states, residual, layernorm, read
)
@@ -596,13 +589,13 @@ def _tp_all_reduce_with_scattered_residual(
hidden_states: torch.Tensor,
residual: torch.Tensor,
layernorm: torch.nn.Module,
- context: CommunicateContext,
read: StageRead = NORM_READ,
):
+ parallel = get_parallel()
if hidden_states.shape[0] == 0:
return hidden_states, hidden_states
- scattered_states = hidden_states.tensor_split(context.tp_size)[context.tp_rank]
+ scattered_states = hidden_states.tensor_split(parallel.tp_size)[parallel.tp_rank]
scattered_states += residual
residual = tensor_model_parallel_all_reduce(hidden_states)
return read.read(residual, layernorm)
@@ -613,8 +606,8 @@ def _mlp_input_gather_attention_cp(
residual: torch.Tensor,
forward_batch: ForwardBatch,
layernorm: torch.nn.Module,
- context: CommunicateContext,
*,
+ cache=None,
gather: Callable,
update: StageUpdate = ADD,
):
@@ -622,7 +615,7 @@ def _mlp_input_gather_attention_cp(
equal length, over the attention-CP group. The residual stays on the
shard."""
hidden_states, residual = gather(
- hidden_states, residual, forward_batch, layernorm, context, update=update
+ hidden_states, residual, forward_batch, layernorm, update=update, cache=cache
)
return dsa_cp_gather_hidden_states(hidden_states), residual
@@ -632,8 +625,8 @@ def _mlp_input_gather_moe_cp(
residual: torch.Tensor,
forward_batch: ForwardBatch,
layernorm: torch.nn.Module,
- context: CommunicateContext,
*,
+ cache=None,
gather: Callable,
update: StageUpdate = ADD,
):
@@ -649,7 +642,7 @@ def _mlp_input_gather_moe_cp(
return hidden_states, residual
hidden_states, residual = gather(
- hidden_states, residual, forward_batch, layernorm, context, update=update
+ hidden_states, residual, forward_batch, layernorm, update=update, cache=cache
)
rows = moe_cp_gathered_rows(forward_batch)
@@ -709,27 +702,6 @@ def _redistribute_output_from_moe_cp(
).contiguous()
-def _reduce_and_redistribute_output_step(
- forward_batch: ForwardBatch,
- *,
- leaves_for_reduce_scatter: bool,
- leaves_for_reduce_scatterv: bool,
-) -> Optional[Callable[[torch.Tensor, torch.Tensor, ForwardBatch], None]]:
- """The reduce-scatter that brings an FFN output gathered over attention
- DP back to this rank's tokens when the FFN leaves its sum to it (see
- StageOutput); None when the FFN reduces the output and only a scatter
- remains."""
- if should_use_dp_reduce_scatterv() and leaves_for_reduce_scatterv:
- return _reduce_and_redistribute_output_varlen
- if (
- leaves_for_reduce_scatter
- and forward_batch.dp_padding_mode.is_max_len()
- and can_use_dp_reduce_scatter()
- ):
- return _reduce_and_redistribute_output_max_len
- return None
-
-
def _to_local_tokens(
step: Callable[[torch.Tensor, torch.Tensor, ForwardBatch], None],
forward_batch: ForwardBatch,
@@ -756,38 +728,15 @@ class CommunicateSummableTensorPairFn:
hidden_states: torch.Tensor,
residual: torch.Tensor,
forward_batch: ForwardBatch,
- context: CommunicateContext,
**kwargs,
):
return hidden_states, residual
- @staticmethod
- def _scatter_hidden_states(
- hidden_states: torch.Tensor,
- residual: torch.Tensor,
- forward_batch: ForwardBatch,
- context: CommunicateContext,
- allow_reduce_scatter: bool = False,
- is_layer_sparse: bool = False,
- ):
- step = (
- _reduce_and_redistribute_output_step(
- forward_batch,
- leaves_for_reduce_scatter=allow_reduce_scatter,
- # A MoE block leaves its sum to reduce_scatterv whenever it
- # applies (should_skip_post_experts_all_reduce).
- leaves_for_reduce_scatterv=allow_reduce_scatter or is_layer_sparse,
- )
- or _redistribute_output
- )
- return _to_local_tokens(step, forward_batch, hidden_states), residual
-
@staticmethod
def _gather(
hidden_states: torch.Tensor,
residual: torch.Tensor,
forward_batch: ForwardBatch,
- context: CommunicateContext,
update: StageUpdate = ADD,
**kwargs,
):
@@ -799,7 +748,6 @@ class CommunicateSummableTensorPairFn:
hidden_states: torch.Tensor,
residual: torch.Tensor,
forward_batch: ForwardBatch,
- context: CommunicateContext,
*,
sums: bool,
gathers_back: bool,
@@ -810,17 +758,20 @@ class CommunicateSummableTensorPairFn:
a reduce-scatter that also completes its sum when ``sums``, else this
rank's slice of the complete output. With ``gathers_back`` it is
written into the residual there and the full rows are gathered back."""
+ parallel = get_parallel()
if sums:
- hidden_states, _ = tp_reduce_scatter(hidden_states, None, context)
+ hidden_states, _ = tp_reduce_scatter(hidden_states, None)
else:
- hidden_states = hidden_states.tensor_split(context.tp_size)[context.tp_rank]
+ hidden_states = hidden_states.tensor_split(parallel.tp_size)[
+ parallel.tp_rank
+ ]
if not gathers_back:
return hidden_states, residual
local_states = update.update(hidden_states, residual)
hidden_states = local_states.new_empty(
- local_states.shape[0] * context.tp_size, *local_states.shape[1:]
+ local_states.shape[0] * parallel.tp_size, *local_states.shape[1:]
)
- get_parallel().tp_group.all_gather_into_tensor(hidden_states, local_states)
+ parallel.tp_group.all_gather_into_tensor(hidden_states, local_states)
return hidden_states, None
@staticmethod
@@ -828,17 +779,15 @@ class CommunicateSummableTensorPairFn:
hidden_states: torch.Tensor,
residual: torch.Tensor,
forward_batch: ForwardBatch,
- context: CommunicateContext,
):
assert residual is None, "not yet handled residual!=None"
- return _redistribute_to_attn_tp_shards(hidden_states, context), None
+ return _redistribute_to_attn_tp_shards(hidden_states), None
@staticmethod
def _take_back_attention_cp_shard(
hidden_states: torch.Tensor,
residual: torch.Tensor,
forward_batch: ForwardBatch,
- context: CommunicateContext,
**kwargs,
):
"""DSA and MLA CP: this rank's shard of a complete output gathered in
@@ -852,7 +801,6 @@ class CommunicateSummableTensorPairFn:
hidden_states: torch.Tensor,
residual: torch.Tensor,
forward_batch: ForwardBatch,
- context: CommunicateContext,
**kwargs,
):
"""DSA and MLA CP: sum the FFN output over the attention-CP group and
@@ -864,7 +812,6 @@ class CommunicateSummableTensorPairFn:
hidden_states: torch.Tensor,
residual: torch.Tensor,
forward_batch: ForwardBatch,
- context: CommunicateContext,
**kwargs,
):
"""This rank's CP shard of the rows the DP gather put in its DP group's
@@ -886,7 +833,6 @@ class CommunicateSummableTensorPairFn:
hidden_states: torch.Tensor,
residual: torch.Tensor,
forward_batch: ForwardBatch,
- context: CommunicateContext,
**kwargs,
):
"""Scatter MoE output back to TP_ATTN_FULL after MOE_FULL computation.
@@ -905,7 +851,7 @@ class CommunicateSummableTensorPairFn:
if rows is not None:
hidden_states = _redistribute_output_from_moe_cp(hidden_states, rows)
- if context.attn_dp_size > 1:
+ if get_parallel().attn_dp_size > 1:
hidden_states = _to_local_tokens(
_redistribute_output, forward_batch, hidden_states
)
diff --git a/python/sglang/srt/layers/communicator/output.py b/python/sglang/srt/layers/communicator/output.py
index c55b23f014..6e6c80b34c 100644
--- a/python/sglang/srt/layers/communicator/output.py
+++ b/python/sglang/srt/layers/communicator/output.py
@@ -21,6 +21,18 @@ import torch
from sglang.srt.distributed import GroupCoordinator
+class OutputTransform(msgspec.Struct, frozen=True):
+ """An operation on the producer contribution, before residual addition.
+
+ Some existing implementations apply it to a partial before reduce-scatter
+ but to a complete value after all-reduce. These are explicit alternatives,
+ not an assumption that an arbitrary operation commutes with reduction.
+ """
+
+ apply: Callable[[torch.Tensor], torch.Tensor]
+ before_reduce_scatter: bool = False
+
+
class UnreducedOutput(msgspec.Struct, frozen=True):
"""A layer output that still owes its sum, left for the next layer's input.
Hand it to the next layer, or pass it through reduce_output() before reading
diff --git a/python/sglang/srt/layers/communicator/residual/__init__.py b/python/sglang/srt/layers/communicator/residual/__init__.py
index c06f608f82..1a3965e820 100644
--- a/python/sglang/srt/layers/communicator/residual/__init__.py
+++ b/python/sglang/srt/layers/communicator/residual/__init__.py
@@ -38,7 +38,7 @@ class StageUpdate(Protocol):
def update(self, hidden_states, residual) -> torch.Tensor:
"""The residual with the output written into it."""
- def residual_to_attn_tp_shard(self, residual, context) -> torch.Tensor:
+ def residual_to_attn_tp_shard(self, residual) -> torch.Tensor:
"""This attention-TP rank's slice of the residual, with whatever the
update reads along with it."""
diff --git a/python/sglang/srt/layers/communicator/residual/access.py b/python/sglang/srt/layers/communicator/residual/access.py
index bd70243450..9a35529c1a 100644
--- a/python/sglang/srt/layers/communicator/residual/access.py
+++ b/python/sglang/srt/layers/communicator/residual/access.py
@@ -105,7 +105,7 @@ def finish_layer_stack(
return reduce_output(hidden_states), residual
-def norm_output(hidden_states, residual, norm, capture_output=None):
+def norm_output(hidden_states, residual, norm, capture_output=None, **read_kwargs):
"""The final norm and an optional capture of the same updated residual.
Keep add and norm together: normalizing a separately rounded residual is
@@ -116,7 +116,7 @@ def norm_output(hidden_states, residual, norm, capture_output=None):
if capture_output is not None:
capture_output(hidden_states)
return norm(hidden_states)
- hidden_states, residual = norm(hidden_states, residual)
+ hidden_states, residual = norm(hidden_states, residual, **read_kwargs)
if capture_output is not None:
capture_output(residual)
return hidden_states
diff --git a/python/sglang/srt/layers/communicator/residual/add_norm.py b/python/sglang/srt/layers/communicator/residual/add_norm.py
index a55b88f05e..5cad111707 100644
--- a/python/sglang/srt/layers/communicator/residual/add_norm.py
+++ b/python/sglang/srt/layers/communicator/residual/add_norm.py
@@ -273,8 +273,8 @@ class Add:
hidden_states += residual
return hidden_states
- def residual_to_attn_tp_shard(self, residual, context):
- return _redistribute_to_attn_tp_shards(residual, context)
+ def residual_to_attn_tp_shard(self, residual):
+ return _redistribute_to_attn_tp_shards(residual)
def residual_from_attn_tp_shards(self, residual):
return _redistribute_from_attn_tp_shards(residual)
diff --git a/python/sglang/srt/layers/communicator/residual/batch.py b/python/sglang/srt/layers/communicator/residual/batch.py
index 9057f9b240..c6d15293db 100644
--- a/python/sglang/srt/layers/communicator/residual/batch.py
+++ b/python/sglang/srt/layers/communicator/residual/batch.py
@@ -1,18 +1,82 @@
-"""Own one invocation's residual stream, including independent TBO children."""
+# Copyright 2023-2024 SGLang Team
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Residual access for one layer-stack invocation or TBO microbatch."""
+from sglang.srt.layers.communicator.residual import access
from sglang.srt.layers.communicator.residual.stream import ResidualStream
+from sglang.srt.model_executor.forward_batch_info import PPProxyTensors
def start(forward_batch):
- """Begin a fresh embedding-side invocation and return its stream."""
stream = ResidualStream()
forward_batch.residual_stream = stream
return stream
def current(forward_batch):
- """Return the invocation's authoritative stream while its stack is active."""
stream = forward_batch.residual_stream
if stream is None:
raise RuntimeError("start the layer stack before entering a stage")
return stream
+
+
+def finish(hidden_states, forward_batch):
+ return current(forward_batch).complete(hidden_states)
+
+
+def take_output(hidden_states, forward_batch):
+ """Release a terminal output whose residual update has already been written."""
+ stream = current(forward_batch)
+ stream.check(hidden_states)
+ if stream.pending is not None:
+ raise RuntimeError("write the residual update before taking the final output")
+ forward_batch.residual_stream = None
+ return hidden_states
+
+
+def norm(hidden_states, forward_batch, layernorm, capture_output=None, **read_kwargs):
+ hidden_states, residual = current(forward_batch).finish(hidden_states)
+ # The terminal consumer now owns the pair. Do not keep layer buffers alive
+ # through logits processing or the next forward on this batch.
+ forward_batch.residual_stream = None
+ return access.norm_output(
+ hidden_states, residual, layernorm, capture_output, **read_kwargs
+ )
+
+
+def to_pp(hidden_states, forward_batch, *, preserve_declared=True):
+ hidden_states, residual = current(forward_batch).finish(
+ hidden_states, preserve_declared=preserve_declared
+ )
+ forward_batch.residual_stream = None
+ return PPProxyTensors({"hidden_states": hidden_states, "residual": residual})
+
+
+def snapshot(hidden_states, forward_batch):
+ return current(forward_batch).snapshot(hidden_states)
+
+
+def add_to_output(hidden_states, forward_batch, extra):
+ output, _ = access.add_to_output(hidden_states, current(forward_batch), extra)
+ return output
+
+
+def fold(hidden_states, forward_batch):
+ output, _ = access.fold(hidden_states, current(forward_batch))
+ return output
+
+
+def written(hidden_states, forward_batch):
+ forward_batch.residual_stream = ResidualStream(hidden_states)
+ return hidden_states
diff --git a/python/sglang/srt/layers/communicator/residual/mhc.py b/python/sglang/srt/layers/communicator/residual/mhc.py
index 3a42c0bb59..9ce90fbfc8 100644
--- a/python/sglang/srt/layers/communicator/residual/mhc.py
+++ b/python/sglang/srt/layers/communicator/residual/mhc.py
@@ -20,6 +20,7 @@ import torch
from sglang.kernels.ops.layernorm.mhc import hc_contract, hc_expand
from sglang.srt.layers.communicator.residual import LayerResidual
+from sglang.srt.runtime_context import get_parallel
@dataclass
@@ -96,8 +97,9 @@ class MHCState:
self.h_res = None
self.h_post = None
- def residual_to_attn_tp_shard(self, residual, context):
- rank, size = context.attn_tp_rank, context.attn_tp_size
+ def residual_to_attn_tp_shard(self, residual):
+ parallel = get_parallel()
+ rank, size = parallel.attn_tp_rank, parallel.attn_tp_size
self.h_res = self.h_res.tensor_split(size)[rank]
self.h_post = self.h_post.tensor_split(size)[rank]
return residual.tensor_split(size)[rank]
@@ -151,8 +153,8 @@ class _AttentionUpdate:
def update(self, hidden_states, residual):
return self.state.mlp_combine(hidden_states, residual)
- def residual_to_attn_tp_shard(self, residual, context):
- return self.state.residual_to_attn_tp_shard(residual, context)
+ def residual_to_attn_tp_shard(self, residual):
+ return self.state.residual_to_attn_tp_shard(residual)
def residual_from_attn_tp_shards(self, residual):
return self.state.residual_from_attn_tp_shards(residual)
@@ -198,8 +200,8 @@ class _FfnUpdate:
hidden_states = hc_contract(hidden_states, self.state.hc_mult)
return hidden_states
- def residual_to_attn_tp_shard(self, residual, context):
- return self.state.residual_to_attn_tp_shard(residual, context)
+ def residual_to_attn_tp_shard(self, residual):
+ return self.state.residual_to_attn_tp_shard(residual)
def residual_from_attn_tp_shards(self, residual):
return self.state.residual_from_attn_tp_shards(residual)
diff --git a/python/sglang/srt/layers/communicator/stage.py b/python/sglang/srt/layers/communicator/stage.py
new file mode 100644
index 0000000000..6e181cc1ee
--- /dev/null
+++ b/python/sglang/srt/layers/communicator/stage.py
@@ -0,0 +1,329 @@
+# Copyright 2023-2024 SGLang Team
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""A stage's model-facing input and output boundary."""
+
+from typing import TYPE_CHECKING, Callable, Optional
+
+import torch
+
+from sglang.srt.layers import layernorm_sp
+from sglang.srt.layers.aux_hidden_states import AuxHiddenStateAccumulator
+from sglang.srt.layers.communicator.adapters import branch
+from sglang.srt.layers.communicator.adapters.lora import (
+ publish_attention,
+ publish_ffn,
+)
+from sglang.srt.layers.communicator.boundary import (
+ BoundarySteps,
+ StageEntry,
+ StageKind,
+)
+from sglang.srt.layers.communicator.ops import CommunicateSimpleFn
+from sglang.srt.layers.communicator.residual.access import buffer, from_pp
+from sglang.srt.layers.communicator.residual.add_norm import (
+ ADD,
+)
+from sglang.srt.layers.communicator.residual.batch import current
+from sglang.srt.layers.communicator.residual.stream import DeclaredSum, ResidualStream
+from sglang.srt.model_executor.forward_batch_info import ForwardBatch
+from sglang.srt.runtime_context import get_forward
+
+if TYPE_CHECKING:
+ from sglang.srt.layers.communicator.construction import StagePlan
+ from sglang.srt.layers.communicator.factories import StageDeclaration
+
+
+class StageCommunicator:
+ """One stage of a layer, its attention or its FFN: the boundary into it,
+ run with the stage's norm on the steps the layer chose for the batch."""
+
+ def __init__(
+ self,
+ plan: "StagePlan",
+ kind: StageKind,
+ norm,
+ *,
+ declaration: Optional["StageDeclaration"] = None,
+ ):
+ self.plan = plan
+ self.kind = kind
+ self.norm = norm
+ self._declaration = declaration
+
+ @property
+ def declaration(self) -> "StageDeclaration":
+ """Immutable source contract for constructing another local sequence."""
+ if self._declaration is None:
+ raise RuntimeError("this boundary was constructed from raw layout edges")
+ return self._declaration
+
+ def entry(
+ self, forward_batch: ForwardBatch, steps: Optional[BoundarySteps] = None
+ ) -> StageEntry:
+ if steps is None:
+ steps = self.plan._batch_steps(forward_batch)
+ entry = steps.attention if self.kind is StageKind.ATTENTION else steps.ffn
+ if entry is None:
+ raise NotImplementedError(
+ f"a layer that is one stage has no {self.kind.name.lower()} stage"
+ )
+ return entry
+
+ def _prepare(
+ self,
+ hidden_states,
+ residual,
+ forward_batch: ForwardBatch,
+ steps: Optional[BoundarySteps] = None,
+ **call,
+ ):
+ """The stage's input and the residual, from the previous stage's output
+ (``call``: what the stage's read takes, e.g. the attention's
+ ``quant_format``)."""
+ entry = self.entry(forward_batch, steps)
+ stream = (
+ residual
+ if isinstance(residual, ResidualStream)
+ else ResidualStream(residual)
+ )
+ if not isinstance(residual, ResidualStream):
+ if residual is not None or entry.input_sum is not None:
+ hidden_states = stream.leave(
+ hidden_states, call.get("update", ADD), declared_sum=entry.input_sum
+ )
+ elif (
+ stream.pending is None
+ and stream.residual is None
+ and entry.input_sum is not None
+ ):
+ hidden_states = stream.leave(
+ hidden_states, ADD, declared_sum=entry.input_sum
+ )
+ if stream.pending is not None:
+ call["update"] = stream.pending.update
+ call["pending"] = stream.pending
+ if stream.pending is None and stream.residual is not None:
+ call["written"] = True
+ hidden_states, residual = stream.input(hidden_states)
+ hidden_states, residual = entry.prepare(
+ hidden_states, residual, forward_batch, self.norm, **call
+ )
+ if entry.input_move is not None:
+ hidden_states = entry.input_move(
+ hidden_states=hidden_states,
+ forward_batch=forward_batch,
+ )
+ if entry.handoff is not None:
+ hidden_states = entry.handoff(
+ hidden_states, forward_batch, self.plan.qkv_latent_func
+ )
+ stream.write(residual)
+ return hidden_states, stream
+
+ @property
+ def fusions(self):
+ return self.plan.fusions
+
+ @property
+ def input_rows(self):
+ return self.plan.input_rows
+
+ @property
+ def input_on_attention_tp_slices(self):
+ return self.plan.input_on_attention_tp_slices
+
+ def prepare(self, hidden_states, forward_batch, *, cache=None, **call):
+ if cache is not None:
+ call["cache"] = cache
+ prepare = (
+ self._prepare_attention
+ if self.kind is StageKind.ATTENTION
+ else self._prepare_input
+ )
+ hidden_states, forward_batch.residual_stream = prepare(
+ hidden_states, current(forward_batch), forward_batch, **call
+ )
+ return hidden_states
+
+ def finish(self, hidden_states, forward_batch):
+ """Hand this attention's actual contribution to the following stage."""
+ if self.kind is not StageKind.ATTENTION or not self.plan.fixed_output:
+ raise RuntimeError(
+ "use the stage exit scope for an FFN or single-stage mixer"
+ )
+ produced = self.plan.produced(self.kind, forward_batch)
+ return current(forward_batch).leave(
+ hidden_states,
+ produced.update,
+ declared_sum=produced.group if produced.always_leaves else None,
+ )
+
+ def exit(self, forward_batch):
+ result = (
+ self.plan.output.ffn_exit(forward_batch)
+ if self.kind is StageKind.FFN
+ else self.plan.output.mixer_exit(forward_batch)
+ )
+ result._stream = current(forward_batch)
+ return result
+
+ def from_pp(self, tensors, forward_batch, *, allow_missing_residual: bool = False):
+ hidden_states, residual = from_pp(
+ tensors,
+ residual_in_hidden=self.plan.residual_in_hidden,
+ allow_missing_residual=allow_missing_residual,
+ )
+ declared_sum = None
+ if forward_batch is not None:
+ steps = self.plan._batch_steps(forward_batch)
+ entry = steps.attention or steps.ffn
+ declared_sum = entry.input_sum
+ hidden_states, forward_batch.residual_stream = ResidualStream.arrive(
+ hidden_states, residual, ADD, declared_sum=declared_sum
+ )
+ return hidden_states
+
+ def snapshot(self, hidden_states, forward_batch):
+ return current(forward_batch).snapshot(hidden_states)
+
+ def capture_output(self, hidden_states, forward_batch, *, skip_empty=False):
+ stream = current(forward_batch)
+ if skip_empty:
+ storage = buffer(hidden_states)
+ if storage is not None and storage.shape[0] == 0:
+ return hidden_states, None
+ if stream.pending is None or not isinstance(stream.pending.owed, DeclaredSum):
+ hidden_states = stream.complete(hidden_states)
+ return hidden_states, stream.snapshot(hidden_states)
+
+ def postprocess(self, hidden_states, forward_batch):
+ hidden_states, stream = self.plan.output.postprocess_layer(
+ hidden_states, current(forward_batch), forward_batch
+ )
+ forward_batch.residual_stream = stream
+ return hidden_states
+
+ def branch_input(self, source, hidden_states, forward_batch):
+ hidden_states, stream = branch.branch_input(
+ self.plan,
+ source.plan,
+ hidden_states,
+ current(forward_batch),
+ forward_batch,
+ )
+ forward_batch.residual_stream = stream
+ return hidden_states
+
+ def branch_output(self, hidden_states, forward_batch):
+ return branch.branch_output(self.plan, hidden_states, forward_batch)
+
+ def merge_branch(self, contribution, hidden_states, source, forward_batch):
+ hidden_states, stream = branch.merge_branch(
+ self.plan,
+ contribution,
+ hidden_states,
+ current(forward_batch),
+ source.plan,
+ forward_batch,
+ )
+ forward_batch.residual_stream = stream
+ return hidden_states
+
+ def _prepare_input(self, hidden_states, residual, forward_batch, **call):
+ plan = self.plan
+ if self.kind is StageKind.ATTENTION:
+ publish_attention(plan._publish_lora_layout)
+ if plan._sp_steps is not None and plan.is_first_layer:
+ get_forward().set(
+ "sp_active", layernorm_sp.runs_sp(forward_batch.forward_mode)
+ )
+ if get_forward().sp_active:
+ hidden_states = layernorm_sp.sp_entry_scatter(hidden_states)
+ else:
+ publish_ffn(plan._publish_lora_layout, self.entry(forward_batch).input_rows)
+ return self._prepare(hidden_states, residual, forward_batch, **call)
+
+ def _prepare_attention(
+ self,
+ hidden_states: torch.Tensor,
+ stream: ResidualStream,
+ forward_batch: ForwardBatch,
+ captured_last_layer_outputs: Optional[AuxHiddenStateAccumulator] = None,
+ post_residual_addition: Optional[torch.Tensor] = None,
+ quant_format: str = "",
+ capture_output: Optional[Callable] = None,
+ ):
+ # Aux consumers need a materialized output before the input norm.
+ # Complete communication here, preserving the existing add+norm kernel
+ # and its FP32 accumulation. Its residual result also supplies capture.
+ capture_before_read = capture_output is not None and (
+ (stream.residual is None and stream.pending is None)
+ or post_residual_addition is not None
+ )
+ if capture_output is not None:
+ hidden_states = stream.complete(hidden_states)
+ if capture_before_read:
+ # Embeddings precede enter; HF deepstack capture precedes the
+ # extra addition. Neither is the residual returned by the read.
+ value, previous = stream.finish(hidden_states)
+ if previous is None:
+ capture_output(value)
+ else:
+ capture_output(value + previous, owned=True)
+ hidden_states, stream = self._prepare_input(
+ hidden_states,
+ stream,
+ forward_batch,
+ quant_format=quant_format,
+ post_residual_addition=post_residual_addition,
+ )
+ entry = self.entry(forward_batch)
+ if capture_output is not None and not capture_before_read:
+ value = stream.residual
+ move = entry.capture_move
+ if move is not None:
+ value = move(value, forward_batch=forward_batch)
+ keeps_input = entry.capture_preserves_residual
+ capture_output(
+ value,
+ owned=entry.capture_move_allocates
+ or (
+ move is None
+ and keeps_input is not None
+ and keeps_input(value, forward_batch)
+ ),
+ )
+ if captured_last_layer_outputs is not None:
+ residual_value = stream.residual
+ move = entry.input_move
+ gathered_last_layer_output = (
+ residual_value
+ if move is None
+ else move(
+ hidden_states=residual_value,
+ forward_batch=forward_batch,
+ )
+ )
+ # Input gathers allocate fresh DP buffers.
+ # Without a gather this is the mutable residual, so retention copies.
+ captured_last_layer_outputs.capture(
+ gathered_last_layer_output,
+ owned=move is CommunicateSimpleFn._scattered_to_tp_attn_full
+ or (
+ move is None
+ and entry.capture_preserves_residual is not None
+ and entry.capture_preserves_residual(residual_value, forward_batch)
+ ),
+ )
+ return hidden_states, stream
diff --git a/python/sglang/srt/layers/moe/utils.py b/python/sglang/srt/layers/moe/utils.py
index 3fe41f9439..300d09390b 100644
--- a/python/sglang/srt/layers/moe/utils.py
+++ b/python/sglang/srt/layers/moe/utils.py
@@ -559,7 +559,12 @@ def draft_model_build_scope():
original_scope = moe.in_speculative_scope
try:
moe.in_speculative_scope = True
- yield
+ # Boundaries capture this resolved preference while the draft builds.
+ # Restoring it cannot change an already constructed target plan.
+ with get_exec().comm.override(
+ boundary_reduction=get_spec().speculative_boundary_reduction
+ ):
+ yield
finally:
moe.in_speculative_scope = original_scope
moe.disable_shared_experts_fusion = original_fusion
@@ -759,10 +764,8 @@ def should_skip_post_experts_all_reduce(*, is_tp_path: bool) -> bool:
Pass ``is_tp_path=True`` for the TP all-reduce, ``False`` for the EP one.
"""
- return (
- should_skip_mlp_all_reduce()
- or should_use_dp_reduce_scatterv()
- or post_experts_output_is_complete(is_tp_path=is_tp_path)
+ return should_skip_mlp_all_reduce() or post_experts_output_is_complete(
+ is_tp_path=is_tp_path
)
diff --git a/python/sglang/srt/model_executor/forward_batch_info.py b/python/sglang/srt/model_executor/forward_batch_info.py
index 4c5c9f2d8e..97cd785704 100644
--- a/python/sglang/srt/model_executor/forward_batch_info.py
+++ b/python/sglang/srt/model_executor/forward_batch_info.py
@@ -565,7 +565,7 @@ class ForwardBatch(ForwardBatchDeepSeekMHAMixin):
dp_prefill_cuda_graph_max_prefix_len: int = 0
global_forward_mode: Optional[ForwardMode] = None
- # Owned only while this invocation executes its local decoder stack.
+ # Current layer-stack invocation; each TBO child owns a separate stream.
residual_stream: Optional[ResidualStream] = None
# For two-batch overlap
diff --git a/python/sglang/srt/models/bailing_moe.py b/python/sglang/srt/models/bailing_moe.py
index 914189bb24..a49ce6554c 100644
--- a/python/sglang/srt/models/bailing_moe.py
+++ b/python/sglang/srt/models/bailing_moe.py
@@ -36,9 +36,10 @@ from sglang.srt.layers.aux_hidden_states import (
AuxHiddenStateList,
)
from sglang.srt.layers.communicator import (
- LayerCommunicator,
- LayerFacts,
+ declare_attn,
+ declare_ffn,
enable_moe_dense_fully_dp,
+ make_stages,
)
from sglang.srt.layers.communicator.residual import batch as residual_batch
from sglang.srt.layers.dp_attention import (
@@ -673,15 +674,6 @@ class BailingMoEBlock(nn.Module):
config, layer_id=layer_id + 1, is_nextn=False
)
- self.layer_facts = LayerFacts.init_new(
- layer_id=layer_id,
- # A NextN draft is a one-layer model.
- num_layers=1 if is_nextn else config.num_hidden_layers,
- is_layer_sparse=self.is_layer_sparse,
- is_previous_layer_sparse=is_previous_layer_sparse,
- is_next_layer_sparse=is_next_layer_sparse,
- )
-
if self.is_layer_sparse:
self.mlp = BailingMoESparseMoeBlock(
layer_id=layer_id,
@@ -706,11 +698,21 @@ class BailingMoEBlock(nn.Module):
self.post_attention_layernorm = RMSNorm(hidden_size, eps=config.rms_norm_eps)
- self.layer_communicator = LayerCommunicator(
- layer_facts=self.layer_facts,
- input_layernorm=self.input_layernorm,
- post_attention_layernorm=self.post_attention_layernorm,
- allow_reduce_scatter=True,
+ self.attn_stage, self.ffn_stage = make_stages(
+ (declare_attn(), self.input_layernorm),
+ (
+ declare_ffn(
+ sparse=self.is_layer_sparse,
+ next_sparse=is_next_layer_sparse,
+ ),
+ self.post_attention_layernorm,
+ ),
+ previous=declare_ffn(
+ sparse=is_previous_layer_sparse, next_sparse=self.is_layer_sparse
+ )
+ if layer_id != 0
+ else None,
+ terminal=layer_id == (1 if is_nextn else config.num_hidden_layers) - 1,
)
def _is_layer_sparse(
@@ -725,18 +727,14 @@ class BailingMoEBlock(nn.Module):
positions: torch.Tensor,
hidden_states: torch.Tensor,
forward_batch: ForwardBatch,
- residual: Optional[torch.Tensor],
captured_last_layer_outputs: Optional[AuxHiddenStateAccumulator] = None,
capture_output=None,
) -> torch.Tensor:
- hidden_states, residual = (
- self.layer_communicator.prepare_attn_and_capture_last_layer_outputs(
- hidden_states,
- residual,
- forward_batch,
- captured_last_layer_outputs=captured_last_layer_outputs,
- capture_output=capture_output,
- )
+ hidden_states = self.attn_stage.prepare(
+ hidden_states,
+ forward_batch,
+ captured_last_layer_outputs=captured_last_layer_outputs,
+ capture_output=capture_output,
)
if hidden_states.shape[0] != 0:
@@ -746,17 +744,17 @@ class BailingMoEBlock(nn.Module):
forward_batch=forward_batch,
)
- hidden_states, residual = self.layer_communicator.prepare_mlp(
+ hidden_states = self.attn_stage.finish(hidden_states, forward_batch)
+ hidden_states = self.ffn_stage.prepare(
hidden_states=hidden_states,
- residual=residual,
forward_batch=forward_batch,
)
- with self.layer_communicator.ffn_exit(forward_batch) as ffn_exit:
+ with self.ffn_stage.exit(forward_batch) as ffn_exit:
hidden_states = self.mlp(hidden_states, forward_batch)
- hidden_states, residual = ffn_exit.finish(hidden_states, residual)
+ hidden_states = ffn_exit.finish(hidden_states)
- return hidden_states, residual
+ return hidden_states
class BailingMoEModel(nn.Module):
@@ -821,51 +819,36 @@ class BailingMoEModel(nn.Module):
hidden_states = self.word_embeddings(input_ids)
else:
hidden_states = input_embeds
- residual = residual_batch.start(forward_batch)
+ residual_batch.start(forward_batch)
else:
assert pp_proxy_tensors is not None
- hidden_states, residual = self.layers[
- self.start_layer
- ].layer_communicator.from_pp(pp_proxy_tensors, forward_batch)
+ hidden_states = self.layers[self.start_layer].attn_stage.from_pp(
+ pp_proxy_tensors, forward_batch
+ )
aux_hidden_states = AuxHiddenStateList()
for i in range(self.start_layer, self.end_layer):
with get_global_expert_distribution_recorder().with_current_layer(i):
layer = self.layers[i]
- hidden_states, residual = layer(
+ hidden_states = layer(
positions,
hidden_states,
forward_batch,
- residual,
- captured_last_layer_outputs=(
- aux_hidden_states
- if getattr(layer, "_is_layer_to_capture", False)
- else None
- ),
+ captured_last_layer_outputs=aux_hidden_states
+ if getattr(layer, "_is_layer_to_capture", False)
+ else None,
capture_output=aux_hidden_states.capture
if i in self.layers_to_capture
else None,
)
- last_layer = self.layers[self.end_layer - 1]
- hidden_states, residual = last_layer.layer_communicator.finish_layer_stack(
- hidden_states,
- residual,
- forward_batch,
- preserve_declared=not self.pp_group.is_last_rank,
- )
if not self.pp_group.is_last_rank:
- return PPProxyTensors(
- {
- "hidden_states": hidden_states,
- "residual": residual,
- }
- )
+ return residual_batch.to_pp(hidden_states, forward_batch)
else:
+ hidden_states = residual_batch.finish(hidden_states, forward_batch)
if not forward_batch.forward_mode.is_idle():
- if residual is None:
- hidden_states = self.norm(hidden_states)
- else:
- hidden_states, _ = self.norm(hidden_states, residual)
+ hidden_states = residual_batch.norm(
+ hidden_states, forward_batch, self.norm
+ )
if len(aux_hidden_states) == 0:
return hidden_states
diff --git a/python/sglang/srt/models/bailing_moe_linear.py b/python/sglang/srt/models/bailing_moe_linear.py
index e1836d5992..24d951c5a3 100644
--- a/python/sglang/srt/models/bailing_moe_linear.py
+++ b/python/sglang/srt/models/bailing_moe_linear.py
@@ -16,8 +16,9 @@ from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_r
from sglang.srt.layers import deep_gemm_wrapper
from sglang.srt.layers.activation import SiluAndMul
from sglang.srt.layers.communicator import (
- LayerCommunicator,
- LayerFacts,
+ declare_attn,
+ declare_ffn,
+ make_stages,
)
from sglang.srt.layers.communicator.residual import batch as residual_batch
from sglang.srt.layers.dp_attention import (
@@ -814,26 +815,28 @@ class BailingMoELinearDecoderLayer(nn.Module):
self.input_layernorm = RMSNorm(self.hidden_size, eps=rms_norm_eps)
self.post_attention_layernorm = RMSNorm(self.hidden_size, eps=rms_norm_eps)
- self.layer_facts = LayerFacts.init_new(
- layer_id=layer_id,
- # A NextN draft is a one-layer model.
- num_layers=1 if is_nextn else config.num_hidden_layers,
- is_layer_sparse=is_moe_layer,
- is_previous_layer_sparse=is_previous_moe_layer,
- is_next_layer_sparse=is_next_layer_moe_layer,
- )
-
qkv_latent_func = (
self.attention.prepare_qkv_latent
if config.attention_type == 1 and self.use_mla
else None
)
- self.layer_communicator = LayerCommunicator(
- layer_facts=self.layer_facts,
- input_layernorm=self.input_layernorm,
- post_attention_layernorm=self.post_attention_layernorm,
- allow_reduce_scatter=False,
- qkv_latent_func=qkv_latent_func,
+ self.attn_stage, self.ffn_stage = make_stages(
+ (
+ declare_attn(),
+ self.input_layernorm,
+ {"qkv_latent_func": qkv_latent_func},
+ ),
+ (
+ declare_ffn(
+ sparse=is_moe_layer,
+ next_sparse=is_next_layer_moe_layer,
+ ),
+ self.post_attention_layernorm,
+ ),
+ previous=declare_ffn(sparse=is_previous_moe_layer, next_sparse=is_moe_layer)
+ if layer_id != 0
+ else None,
+ terminal=layer_id == (1 if is_nextn else config.num_hidden_layers) - 1,
)
def _is_layer_sparse(
@@ -848,13 +851,10 @@ class BailingMoELinearDecoderLayer(nn.Module):
hidden_states: torch.Tensor,
positions: torch.Tensor,
forward_batch: ForwardBatch,
- residual: Optional[torch.Tensor],
zero_allocator: BumpAllocator,
**kwargs,
- ) -> tuple[torch.Tensor, torch.Tensor]:
- hidden_states, residual = self.layer_communicator.prepare_attn(
- hidden_states, residual, forward_batch
- )
+ ) -> torch.Tensor:
+ hidden_states = self.attn_stage.prepare(hidden_states, forward_batch)
# logger.warning(
# f"===={self.layer_id=}, 1 shape= {hidden_states.shape}, {residual.shape}"
# )
@@ -875,16 +875,15 @@ class BailingMoELinearDecoderLayer(nn.Module):
# logger.warning(
# f"===={self.layer_id=}, 2 shape= {hidden_states.shape}, {residual.shape}"
# )
- hidden_states, residual = self.layer_communicator.prepare_mlp(
- hidden_states, residual, forward_batch
- )
+ hidden_states = self.attn_stage.finish(hidden_states, forward_batch)
+ hidden_states = self.ffn_stage.prepare(hidden_states, forward_batch)
# logger.warning(
# f"===={self.layer_id=}, 3 shape= {hidden_states.shape}, {residual.shape}"
# )
- with self.layer_communicator.ffn_exit(forward_batch) as ffn_exit:
+ with self.ffn_stage.exit(forward_batch) as ffn_exit:
hidden_states = self.mlp(hidden_states)
- hidden_states, residual = ffn_exit.finish(hidden_states, residual)
- return hidden_states, residual
+ hidden_states = ffn_exit.finish(hidden_states)
+ return hidden_states
@staticmethod
def shared_moe_coefficient_loader(
@@ -981,12 +980,12 @@ class BailingMoELinearModel(nn.Module):
hidden_states = self.word_embeddings(input_ids)
else:
hidden_states = inputs_embeds
- residual = residual_batch.start(forward_batch)
+ residual_batch.start(forward_batch)
else:
assert pp_proxy_tensors is not None
- hidden_states, residual = self.layers[
- self.start_layer
- ].layer_communicator.from_pp(pp_proxy_tensors, forward_batch)
+ hidden_states = self.layers[self.start_layer].attn_stage.from_pp(
+ pp_proxy_tensors, forward_batch
+ )
total_num_layers = self.end_layer - self.start_layer
device = inputs_embeds.device if inputs_embeds is not None else input_ids.device
@@ -999,30 +998,20 @@ class BailingMoELinearModel(nn.Module):
for i in range(self.start_layer, self.end_layer):
with get_global_expert_distribution_recorder().with_current_layer(i):
layer = self.layers[i]
- hidden_states, residual = layer(
+ hidden_states = layer(
hidden_states=hidden_states,
positions=positions,
forward_batch=forward_batch,
- residual=residual,
zero_allocator=zero_allocator,
)
- last_layer = self.layers[self.end_layer - 1]
- hidden_states, residual = last_layer.layer_communicator.finish_layer_stack(
- hidden_states,
- residual,
- forward_batch,
- preserve_declared=not self.pp_group.is_last_rank,
- )
if not self.pp_group.is_last_rank:
- return PPProxyTensors(
- {"hidden_states": hidden_states, "residual": residual}
- )
+ return residual_batch.to_pp(hidden_states, forward_batch)
else:
+ hidden_states = residual_batch.finish(hidden_states, forward_batch)
if not forward_batch.forward_mode.is_idle():
- if residual is None:
- hidden_states = self.norm(hidden_states)
- else:
- hidden_states, _ = self.norm(hidden_states, residual)
+ hidden_states = residual_batch.norm(
+ hidden_states, forward_batch, self.norm
+ )
return hidden_states
diff --git a/python/sglang/srt/models/bailing_moe_nextn.py b/python/sglang/srt/models/bailing_moe_nextn.py
index 6d2e4aacb4..5d5963089f 100644
--- a/python/sglang/srt/models/bailing_moe_nextn.py
+++ b/python/sglang/srt/models/bailing_moe_nextn.py
@@ -162,7 +162,7 @@ class BailingMoEModelNextN(nn.Module):
)
)
- residual = residual_batch.start(forward_batch)
+ residual_batch.start(forward_batch)
if self.is_hybrid:
device = input_ids.device
zero_allocator = BumpAllocator(
@@ -172,26 +172,20 @@ class BailingMoEModelNextN(nn.Module):
dtype=torch.float32,
device=device,
)
- hidden_states, residual = self.decoder(
+ hidden_states = self.decoder(
hidden_states=hidden_states,
positions=positions,
forward_batch=forward_batch,
- residual=residual,
zero_allocator=zero_allocator,
)
else:
- hidden_states, residual = self.decoder(
- positions, hidden_states, forward_batch, residual
- )
+ hidden_states = self.decoder(positions, hidden_states, forward_batch)
- hidden_states, residual = self.decoder.layer_communicator.finish_layer_stack(
- hidden_states, residual, forward_batch
- )
+ hidden_states = residual_batch.finish(hidden_states, forward_batch)
if not forward_batch.forward_mode.is_idle():
- if residual is not None:
- hidden_states, _ = self.final_layernorm(hidden_states, residual)
- else:
- hidden_states = self.final_layernorm(hidden_states)
+ hidden_states = residual_batch.norm(
+ hidden_states, forward_batch, self.final_layernorm
+ )
return hidden_states
diff --git a/python/sglang/srt/models/bailing_moe_v3.py b/python/sglang/srt/models/bailing_moe_v3.py
index 904194d4f4..b38a6202f1 100644
--- a/python/sglang/srt/models/bailing_moe_v3.py
+++ b/python/sglang/srt/models/bailing_moe_v3.py
@@ -30,9 +30,10 @@ from sglang.srt.layers import deep_gemm_wrapper
from sglang.srt.layers.activation import SiluAndMul
from sglang.srt.layers.aux_hidden_states import AuxHiddenStateList
from sglang.srt.layers.communicator import (
- LayerCommunicator,
- LayerFacts,
+ declare_attn,
+ declare_ffn,
enable_moe_dense_fully_dp,
+ make_stages,
)
from sglang.srt.layers.communicator.residual import batch as residual_batch
from sglang.srt.layers.dp_attention import is_dp_attention_enabled
@@ -1148,27 +1149,27 @@ class BailingMoELinearDecoderLayer(nn.Module):
self.input_layernorm = RMSNorm(self.hidden_size, eps=rms_norm_eps)
self.post_attention_layernorm = RMSNorm(self.hidden_size, eps=rms_norm_eps)
- self.layer_facts = LayerFacts.init_new(
- layer_id=layer_id,
- # NextN wraps a single decoder layer whose checkpoint prefix is the
- # post-model layer id. Treat it as a one-layer model for scatter-mode
- # planning so A2A/DeepEP outputs are gathered before logits.
- num_layers=1 if is_nextn else config.num_hidden_layers,
- is_layer_sparse=is_moe_layer,
- is_previous_layer_sparse=is_previous_moe_layer,
- is_next_layer_sparse=is_next_layer_sparse,
- )
-
- self.layer_communicator = LayerCommunicator(
- layer_facts=self.layer_facts,
- input_layernorm=self.input_layernorm,
- post_attention_layernorm=self.post_attention_layernorm,
- allow_reduce_scatter=True,
- qkv_latent_func=(
- self.attention.prepare_qkv_latent
- if self.attention_type == 1 and self.use_mla
- else None
+ self.attn_stage, self.ffn_stage = make_stages(
+ (
+ declare_attn(),
+ self.input_layernorm,
+ {
+ "qkv_latent_func": self.attention.prepare_qkv_latent
+ if self.attention_type == 1 and self.use_mla
+ else None
+ },
),
+ (
+ declare_ffn(
+ sparse=is_moe_layer,
+ next_sparse=is_next_layer_sparse,
+ ),
+ self.post_attention_layernorm,
+ ),
+ previous=declare_ffn(sparse=is_previous_moe_layer, next_sparse=is_moe_layer)
+ if layer_id != 0
+ else None,
+ terminal=layer_id == (1 if is_nextn else config.num_hidden_layers) - 1,
)
@torch.inference_mode()
@@ -1177,18 +1178,12 @@ class BailingMoELinearDecoderLayer(nn.Module):
hidden_states: torch.Tensor,
positions: torch.Tensor,
forward_batch: ForwardBatch,
- residual: Optional[torch.Tensor],
zero_allocator: BumpAllocator,
capture_output=None,
**kwargs,
- ) -> tuple[torch.Tensor, torch.Tensor]:
- hidden_states, residual = (
- self.layer_communicator.prepare_attn_and_capture_last_layer_outputs(
- hidden_states,
- residual,
- forward_batch,
- capture_output=capture_output,
- )
+ ) -> torch.Tensor:
+ hidden_states = self.attn_stage.prepare(
+ hidden_states, forward_batch, capture_output=capture_output
)
if not forward_batch.forward_mode.is_idle():
@@ -1213,11 +1208,10 @@ class BailingMoELinearDecoderLayer(nn.Module):
forward_batch=forward_batch,
)
- hidden_states, residual = self.layer_communicator.prepare_mlp(
- hidden_states, residual, forward_batch
- )
+ hidden_states = self.attn_stage.finish(hidden_states, forward_batch)
+ hidden_states = self.ffn_stage.prepare(hidden_states, forward_batch)
- with self.layer_communicator.ffn_exit(forward_batch) as ffn_exit:
+ with self.ffn_stage.exit(forward_batch) as ffn_exit:
if not (
enable_moe_dense_fully_dp()
and (not self.is_layer_sparse)
@@ -1227,9 +1221,9 @@ class BailingMoELinearDecoderLayer(nn.Module):
hidden_states,
forward_batch=forward_batch,
)
- hidden_states, residual = ffn_exit.finish(hidden_states, residual)
+ hidden_states = ffn_exit.finish(hidden_states)
- return hidden_states, residual
+ return hidden_states
@staticmethod
def shared_moe_coefficient_loader(
@@ -1347,12 +1341,12 @@ class BailingMoELinearModel(nn.Module):
hidden_states = self.word_embeddings(input_ids)
else:
hidden_states = input_embeds
- residual = residual_batch.start(forward_batch)
+ residual_batch.start(forward_batch)
else:
assert pp_proxy_tensors is not None
- hidden_states, residual = self.layers[
- self.start_layer
- ].layer_communicator.from_pp(pp_proxy_tensors, forward_batch)
+ hidden_states = self.layers[self.start_layer].attn_stage.from_pp(
+ pp_proxy_tensors, forward_batch
+ )
total_num_layers = self.end_layer - self.start_layer
device = hidden_states.device
@@ -1386,41 +1380,30 @@ class BailingMoELinearModel(nn.Module):
for i in range(self.start_layer, self.end_layer):
with get_global_expert_distribution_recorder().with_current_layer(i):
layer = self.layers[i]
- hidden_states, residual = layer(
+ hidden_states = layer(
hidden_states=hidden_states,
positions=positions,
forward_batch=forward_batch,
- residual=residual,
zero_allocator=zero_allocator,
capture_output=capture_output
if capture_aux
and i > self.start_layer
- and i - 1 in self.layers_to_capture
+ and (i - 1 in self.layers_to_capture)
else None,
)
- last_layer = self.layers[self.end_layer - 1]
- hidden_states, residual = last_layer.layer_communicator.finish_layer_stack(
- hidden_states,
- residual,
- forward_batch,
- preserve_declared=not self.pp_group.is_last_rank,
- )
if not self.pp_group.is_last_rank:
- return PPProxyTensors(
- {"hidden_states": hidden_states, "residual": residual}
- )
+ return residual_batch.to_pp(hidden_states, forward_batch)
else:
+ hidden_states = residual_batch.finish(hidden_states, forward_batch)
if not forward_batch.forward_mode.is_idle():
- hidden_states = last_layer.layer_communicator.norm_output(
+ hidden_states = residual_batch.norm(
hidden_states,
- residual,
+ forward_batch,
self.norm,
- capture_output=(
- capture_output
- if capture_aux and self.end_layer - 1 in self.layers_to_capture
- else None
- ),
+ capture_output=capture_output
+ if capture_aux and self.end_layer - 1 in self.layers_to_capture
+ else None,
)
if capture_aux and len(dspark_aux_hidden_states) > 0:
return hidden_states, dspark_aux_hidden_states
diff --git a/python/sglang/srt/models/deepseek_v4.py b/python/sglang/srt/models/deepseek_v4.py
index 4751bd2a84..30a622461c 100644
--- a/python/sglang/srt/models/deepseek_v4.py
+++ b/python/sglang/srt/models/deepseek_v4.py
@@ -3794,7 +3794,12 @@ class DeepseekV4DecoderLayer(nn.Module):
and forward_batch.dp_padding_mode.is_max_len()
and get_parallel().tp_size == get_parallel().attn_dp_size
)
- mlp_reduce_scatter = _use_cp or _use_reduce_scatterv or _use_reduce_scatter
+ mlp_reduce_scatter = (
+ _use_cp
+ or _use_reduce_scatterv
+ or _use_reduce_scatter
+ or (_use_tp_moe_gather and should_use_dp_reduce_scatterv())
+ )
# PoC (SGLANG_DP_SHARED_EXPERT_LOCAL): compute the replicated shared expert
# on LOCAL hidden before the gather and add it back after the combine
# (reduce_scatterv OR dp_scatter), instead of on the gathered global buffer.
diff --git a/python/sglang/srt/models/falcon_h1.py b/python/sglang/srt/models/falcon_h1.py
index 225476c58f..09086bf015 100644
--- a/python/sglang/srt/models/falcon_h1.py
+++ b/python/sglang/srt/models/falcon_h1.py
@@ -11,7 +11,12 @@ from sglang.srt.layers.attention.hybrid_linear_attn_backend import (
Mamba2AttnBackend,
)
from sglang.srt.layers.attention.mamba.mamba import MambaMixer2
-from sglang.srt.layers.communicator import LayerCommunicator, LayerFacts
+from sglang.srt.layers.communicator import (
+ declare_attn,
+ declare_ffn,
+ make_stages,
+)
+from sglang.srt.layers.communicator.output import OutputTransform
from sglang.srt.layers.communicator.residual import batch as residual_batch
from sglang.srt.layers.dp_attention import (
is_dp_attention_enabled,
@@ -90,9 +95,11 @@ class FalconH1MLP(nn.Module):
x = self.act_fn(gate_up)
x, _ = self.down_proj(x)
- x = x * self.down_multiplier
return x
+ def scale_output(self, x):
+ return x * self.down_multiplier
+
class FalconH1HybridAttentionDecoderLayer(nn.Module):
def __init__(
@@ -199,14 +206,6 @@ class FalconH1HybridAttentionDecoderLayer(nn.Module):
is_previous_layer_sparse = False
is_next_layer_sparse = False
- self.layer_facts = LayerFacts.init_new(
- layer_id=layer_id,
- num_layers=config.num_hidden_layers,
- is_layer_sparse=self.is_layer_sparse,
- is_previous_layer_sparse=is_previous_layer_sparse,
- is_next_layer_sparse=is_next_layer_sparse,
- )
-
self.feed_forward = FalconH1MLP(
hidden_size=self.hidden_size,
intermediate_size=config.intermediate_size,
@@ -223,12 +222,24 @@ class FalconH1HybridAttentionDecoderLayer(nn.Module):
self.q_norm = RMSNorm(self.head_dim, eps=config.rms_norm_eps)
self.k_norm = RMSNorm(self.head_dim, eps=config.rms_norm_eps)
- self.layer_communicator = LayerCommunicator(
- layer_facts=self.layer_facts,
- input_layernorm=self.input_layernorm,
- post_attention_layernorm=self.pre_ff_layernorm,
- allow_reduce_scatter=True,
- allow_deferred_ffn_reduction=False,
+ self.attn_stage, self.ffn_stage = make_stages(
+ (declare_attn(), self.input_layernorm),
+ (
+ declare_ffn(
+ output=OutputTransform(
+ self.feed_forward.scale_output, before_reduce_scatter=True
+ ),
+ sparse=self.is_layer_sparse,
+ next_sparse=is_next_layer_sparse,
+ ),
+ self.pre_ff_layernorm,
+ ),
+ previous=declare_ffn(
+ sparse=is_previous_layer_sparse, next_sparse=self.is_layer_sparse
+ )
+ if layer_id != 0
+ else None,
+ terminal=layer_id == config.num_hidden_layers - 1,
)
self.alt_stream = alt_stream
@@ -320,13 +331,10 @@ class FalconH1HybridAttentionDecoderLayer(nn.Module):
self,
positions: torch.Tensor,
hidden_states: torch.Tensor,
- residual: Optional[torch.Tensor],
forward_batch: ForwardBatch,
**kwargs: Any,
):
- hidden_states, residual = self.layer_communicator.prepare_attn(
- hidden_states, residual, forward_batch
- )
+ hidden_states = self.attn_stage.prepare(hidden_states, forward_batch)
if not forward_batch.forward_mode.is_idle():
# Attention block
@@ -355,12 +363,11 @@ class FalconH1HybridAttentionDecoderLayer(nn.Module):
hidden_states = attention_hidden_states + mamba_hidden_states
# Fully Connected
- hidden_states, residual = self.layer_communicator.prepare_mlp(
- hidden_states, residual, forward_batch
- )
- with self.layer_communicator.ffn_exit(forward_batch) as ffn_exit:
+ hidden_states = self.attn_stage.finish(hidden_states, forward_batch)
+ hidden_states = self.ffn_stage.prepare(hidden_states, forward_batch)
+ with self.ffn_stage.exit(forward_batch) as ffn_exit:
hidden_states = self.feed_forward(hidden_states, forward_batch)
- return ffn_exit.finish(hidden_states, residual)
+ return ffn_exit.finish(hidden_states)
ALL_DECODER_LAYER_TYPES = {
@@ -422,26 +429,22 @@ class FalconH1Model(nn.Module):
else:
hidden_states = self.embed_tokens(input_ids) * self.embedding_multiplier
- residual = residual_batch.start(forward_batch)
+ residual_batch.start(forward_batch)
for i in range(len(self.layers)):
layer = self.layers[i]
- hidden_states, residual = layer(
+ hidden_states = layer(
layer_id=i,
positions=positions,
hidden_states=hidden_states,
- residual=residual,
forward_batch=forward_batch,
)
- hidden_states, residual = self.layers[-1].layer_communicator.finish_layer_stack(
- hidden_states, residual, forward_batch
- )
+ hidden_states = residual_batch.finish(hidden_states, forward_batch)
if not forward_batch.forward_mode.is_idle():
- if residual is None:
- hidden_states = self.final_layernorm(hidden_states)
- else:
- hidden_states, _ = self.final_layernorm(hidden_states, residual)
+ hidden_states = residual_batch.norm(
+ hidden_states, forward_batch, self.final_layernorm
+ )
return hidden_states
diff --git a/python/sglang/srt/models/gpt_oss.py b/python/sglang/srt/models/gpt_oss.py
index 27e9a5ccf7..596cb5cf38 100644
--- a/python/sglang/srt/models/gpt_oss.py
+++ b/python/sglang/srt/models/gpt_oss.py
@@ -31,8 +31,9 @@ from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_r
from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation
from sglang.srt.layers.aux_hidden_states import AuxHiddenStateList
from sglang.srt.layers.communicator import (
- LayerCommunicator,
- LayerFacts,
+ declare_attn,
+ declare_ffn,
+ make_stages,
)
from sglang.srt.layers.communicator.residual import batch as residual_batch
from sglang.srt.layers.dp_attention import (
@@ -570,14 +571,6 @@ class GptOssDecoderLayer(nn.Module):
is_previous_layer_sparse = True
is_next_layer_sparse = True
- self.layer_facts = LayerFacts.init_new(
- layer_id=layer_id,
- num_layers=config.num_hidden_layers,
- is_layer_sparse=self.is_layer_sparse,
- is_previous_layer_sparse=is_previous_layer_sparse,
- is_next_layer_sparse=is_next_layer_sparse,
- )
-
if self.is_layer_sparse:
self.mlp = GptOssSparseMoeBlock(
layer_id=self.layer_id,
@@ -606,11 +599,21 @@ class GptOssDecoderLayer(nn.Module):
x_pad_to_multiple=post_attn_pad_multiple,
)
- self.layer_communicator = LayerCommunicator(
- layer_facts=self.layer_facts,
- input_layernorm=self.input_layernorm,
- post_attention_layernorm=self.post_attention_layernorm,
- allow_reduce_scatter=True,
+ self.attn_stage, self.ffn_stage = make_stages(
+ (declare_attn(), self.input_layernorm),
+ (
+ declare_ffn(
+ sparse=self.is_layer_sparse,
+ next_sparse=is_next_layer_sparse,
+ ),
+ self.post_attention_layernorm,
+ ),
+ previous=declare_ffn(
+ sparse=is_previous_layer_sparse, next_sparse=self.is_layer_sparse
+ )
+ if layer_id != 0
+ else None,
+ terminal=layer_id == config.num_hidden_layers - 1,
)
def forward(
@@ -618,16 +621,10 @@ class GptOssDecoderLayer(nn.Module):
positions: torch.Tensor,
hidden_states: torch.Tensor,
forward_batch: ForwardBatch,
- residual: Optional[torch.Tensor],
capture_output=None,
- ) -> Tuple[torch.Tensor, torch.Tensor]:
- hidden_states, residual = (
- self.layer_communicator.prepare_attn_and_capture_last_layer_outputs(
- hidden_states,
- residual,
- forward_batch,
- capture_output=capture_output,
- )
+ ) -> torch.Tensor:
+ hidden_states = self.attn_stage.prepare(
+ hidden_states, forward_batch, capture_output=capture_output
)
if hidden_states.shape[0] != 0:
@@ -637,15 +634,14 @@ class GptOssDecoderLayer(nn.Module):
forward_batch=forward_batch,
)
- hidden_states, residual = self.layer_communicator.prepare_mlp(
- hidden_states, residual, forward_batch
- )
+ hidden_states = self.attn_stage.finish(hidden_states, forward_batch)
+ hidden_states = self.ffn_stage.prepare(hidden_states, forward_batch)
- with self.layer_communicator.ffn_exit(forward_batch) as ffn_exit:
+ with self.ffn_stage.exit(forward_batch) as ffn_exit:
hidden_states = self.mlp(hidden_states, forward_batch)
- hidden_states, residual = ffn_exit.finish(hidden_states, residual)
+ hidden_states = ffn_exit.finish(hidden_states)
- return hidden_states, residual
+ return hidden_states
class GptOssModel(nn.Module):
@@ -708,12 +704,12 @@ class GptOssModel(nn.Module):
hidden_states = self.embed_tokens(input_ids)
else:
hidden_states = input_embeds
- residual = residual_batch.start(forward_batch)
+ residual_batch.start(forward_batch)
else:
assert pp_proxy_tensors is not None
- hidden_states, residual = self.layers[
- self.start_layer
- ].layer_communicator.from_pp(pp_proxy_tensors, forward_batch)
+ hidden_states = self.layers[self.start_layer].attn_stage.from_pp(
+ pp_proxy_tensors, forward_batch
+ )
# Capture hidden-state boundaries: boundary 0 is the embedding output,
# and boundary i + 1 is the output after transformer block i.
@@ -721,44 +717,30 @@ class GptOssModel(nn.Module):
for i in range(self.start_layer, self.end_layer):
with get_global_expert_distribution_recorder().with_current_layer(i):
layer = self.layers[i]
- hidden_states, residual = layer(
+ hidden_states = layer(
positions,
hidden_states,
forward_batch,
- residual,
capture_output=aux_hidden_states.capture
if i in self.layers_to_capture
else None,
)
- last_layer = self.layers[self.end_layer - 1]
- hidden_states, residual = last_layer.layer_communicator.finish_layer_stack(
- hidden_states,
- residual,
- forward_batch,
- preserve_declared=not self.pp_group.is_last_rank,
- )
if not self.pp_group.is_last_rank:
- return PPProxyTensors(
- {
- "hidden_states": hidden_states,
- "residual": residual,
- }
- )
+ return residual_batch.to_pp(hidden_states, forward_batch)
else:
+ hidden_states = residual_batch.finish(hidden_states, forward_batch)
if hidden_states.shape[0] != 0:
- hidden_states = last_layer.layer_communicator.norm_output(
+ hidden_states = residual_batch.norm(
hidden_states,
- residual,
+ forward_batch,
self.norm,
- capture_output=(
- aux_hidden_states.append
- if self.end_layer in self.layers_to_capture
- else None
- ),
+ capture_output=aux_hidden_states.capture
+ if self.end_layer in self.layers_to_capture
+ else None,
)
elif self.end_layer in self.layers_to_capture:
aux_hidden_states.append(
- last_layer.layer_communicator.snapshot(hidden_states, residual)
+ residual_batch.snapshot(hidden_states, forward_batch)
)
if len(aux_hidden_states) == 0:
return hidden_states
diff --git a/python/sglang/srt/models/llada2.py b/python/sglang/srt/models/llada2.py
index 0e503e67e9..e69bd3efe8 100644
--- a/python/sglang/srt/models/llada2.py
+++ b/python/sglang/srt/models/llada2.py
@@ -32,9 +32,10 @@ from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation
from sglang.srt.eplb.expert_location_dispatch import ExpertLocationDispatchInfo
from sglang.srt.layers.activation import SiluAndMul
from sglang.srt.layers.communicator import (
- LayerCommunicator,
- LayerFacts,
+ declare_attn,
+ declare_ffn,
enable_moe_dense_fully_dp,
+ make_stages,
)
from sglang.srt.layers.communicator.residual import batch as residual_batch
from sglang.srt.layers.dp_attention import (
@@ -613,14 +614,6 @@ class LLaDA2MoeBlock(nn.Module):
is_previous_layer_sparse = self._is_layer_sparse(config, layer_id=layer_id - 1)
is_next_layer_sparse = self._is_layer_sparse(config, layer_id=layer_id + 1)
- self.layer_facts = LayerFacts.init_new(
- layer_id=layer_id,
- num_layers=config.num_hidden_layers,
- is_layer_sparse=self.is_layer_sparse,
- is_previous_layer_sparse=is_previous_layer_sparse,
- is_next_layer_sparse=is_next_layer_sparse,
- )
-
if self.is_layer_sparse:
self.mlp = LLaDA2MoeSparseMoeBlock(
layer_id=layer_id,
@@ -645,12 +638,21 @@ class LLaDA2MoeBlock(nn.Module):
self.post_attention_layernorm = RMSNorm(hidden_size, eps=config.rms_norm_eps)
- self.layer_communicator = LayerCommunicator(
- layer_facts=self.layer_facts,
- input_layernorm=self.input_layernorm,
- post_attention_layernorm=self.post_attention_layernorm,
- allow_reduce_scatter=True,
- allow_deferred_ffn_reduction=False,
+ self.attn_stage, self.ffn_stage = make_stages(
+ (declare_attn(), self.input_layernorm),
+ (
+ declare_ffn(
+ sparse=self.is_layer_sparse,
+ next_sparse=is_next_layer_sparse,
+ ),
+ self.post_attention_layernorm,
+ ),
+ previous=declare_ffn(
+ sparse=is_previous_layer_sparse, next_sparse=self.is_layer_sparse
+ )
+ if layer_id != 0
+ else None,
+ terminal=layer_id == config.num_hidden_layers - 1,
)
def _is_layer_sparse(self, config: PretrainedConfig, layer_id: int) -> bool:
@@ -663,12 +665,9 @@ class LLaDA2MoeBlock(nn.Module):
positions: torch.Tensor,
hidden_states: torch.Tensor,
forward_batch: ForwardBatch,
- residual: Optional[torch.Tensor],
) -> torch.Tensor:
- hidden_states, residual = self.layer_communicator.prepare_attn(
- hidden_states=hidden_states,
- residual=residual,
- forward_batch=forward_batch,
+ hidden_states = self.attn_stage.prepare(
+ hidden_states=hidden_states, forward_batch=forward_batch
)
hidden_states = self.attention(
@@ -677,17 +676,17 @@ class LLaDA2MoeBlock(nn.Module):
forward_batch=forward_batch,
)
- hidden_states, residual = self.layer_communicator.prepare_mlp(
+ hidden_states = self.attn_stage.finish(hidden_states, forward_batch)
+ hidden_states = self.ffn_stage.prepare(
hidden_states=hidden_states,
- residual=residual,
forward_batch=forward_batch,
)
- with self.layer_communicator.ffn_exit(forward_batch) as ffn_exit:
+ with self.ffn_stage.exit(forward_batch) as ffn_exit:
hidden_states = self.mlp(hidden_states, forward_batch)
- hidden_states, residual = ffn_exit.finish(hidden_states, residual)
+ hidden_states = ffn_exit.finish(hidden_states)
- return hidden_states, residual
+ return hidden_states
class LLaDA2MoeModel(nn.Module):
@@ -747,44 +746,26 @@ class LLaDA2MoeModel(nn.Module):
hidden_states = self.word_embeddings(input_ids)
else:
hidden_states = input_embeds
- residual = residual_batch.start(forward_batch)
+ residual_batch.start(forward_batch)
else:
assert pp_proxy_tensors is not None
- hidden_states, residual = self.layers[
- self.start_layer
- ].layer_communicator.from_pp(pp_proxy_tensors, forward_batch)
+ hidden_states = self.layers[self.start_layer].attn_stage.from_pp(
+ pp_proxy_tensors, forward_batch
+ )
for i in range(self.start_layer, self.end_layer):
with get_global_expert_distribution_recorder().with_current_layer(i):
layer = self.layers[i]
- hidden_states, residual = layer(
- positions,
- hidden_states,
- forward_batch,
- residual,
- )
+ hidden_states = layer(positions, hidden_states, forward_batch)
- hidden_states, residual = self.layers[
- self.end_layer - 1
- ].layer_communicator.finish_layer_stack(
- hidden_states,
- residual,
- forward_batch,
- preserve_declared=not self.pp_group.is_last_rank,
- )
if not self.pp_group.is_last_rank:
- return PPProxyTensors(
- {
- "hidden_states": hidden_states,
- "residual": residual,
- }
- )
+ return residual_batch.to_pp(hidden_states, forward_batch)
else:
+ hidden_states = residual_batch.finish(hidden_states, forward_batch)
if not forward_batch.forward_mode.is_idle():
- if residual is None:
- hidden_states = self.norm(hidden_states)
- else:
- hidden_states, _ = self.norm(hidden_states, residual)
+ hidden_states = residual_batch.norm(
+ hidden_states, forward_batch, self.norm
+ )
return hidden_states
diff --git a/python/sglang/srt/models/llama4.py b/python/sglang/srt/models/llama4.py
index 12ccfdc81d..dbfaf078cc 100644
--- a/python/sglang/srt/models/llama4.py
+++ b/python/sglang/srt/models/llama4.py
@@ -25,7 +25,12 @@ import torch
from torch import nn
from transformers import Llama4TextConfig
-from sglang.srt.layers.communicator import LayerCommunicator, LayerFacts
+from sglang.srt.layers.aux_hidden_states import AuxHiddenStateList
+from sglang.srt.layers.communicator import (
+ declare_attn,
+ declare_ffn,
+ make_stages,
+)
from sglang.srt.layers.communicator.residual import batch as residual_batch
from sglang.srt.layers.dp_attention import (
is_dp_attention_enabled,
@@ -421,20 +426,19 @@ class Llama4DecoderLayer(nn.Module):
config.hidden_size, eps=config.rms_norm_eps
)
- self.layer_facts = LayerFacts.init_new(
- layer_id=layer_id,
- num_layers=config.num_hidden_layers,
- is_layer_sparse=is_moe_layer,
- is_previous_layer_sparse=is_previous_moe_layer,
- is_next_layer_sparse=is_next_moe_layer,
- )
-
- self.layer_communicator = LayerCommunicator(
- layer_facts=self.layer_facts,
- input_layernorm=self.input_layernorm,
- post_attention_layernorm=self.post_attention_layernorm,
- allow_reduce_scatter=True,
- allow_deferred_ffn_reduction=False,
+ self.attn_stage, self.ffn_stage = make_stages(
+ (declare_attn(), self.input_layernorm),
+ (
+ declare_ffn(
+ sparse=is_moe_layer,
+ next_sparse=is_next_moe_layer,
+ ),
+ self.post_attention_layernorm,
+ ),
+ previous=declare_ffn(sparse=is_previous_moe_layer, next_sparse=is_moe_layer)
+ if layer_id != 0
+ else None,
+ terminal=layer_id == config.num_hidden_layers - 1,
)
def _is_moe_layer(self, layer_id: int) -> bool:
@@ -453,10 +457,10 @@ class Llama4DecoderLayer(nn.Module):
positions: torch.Tensor,
hidden_states: torch.Tensor,
forward_batch: ForwardBatch,
- residual: Optional[torch.Tensor],
- ) -> Tuple[torch.Tensor, torch.Tensor]:
- hidden_states, residual = self.layer_communicator.prepare_attn(
- hidden_states, residual, forward_batch
+ capture_output=None,
+ ) -> torch.Tensor:
+ hidden_states = self.attn_stage.prepare(
+ hidden_states, forward_batch, capture_output=capture_output
)
if hidden_states.shape[0] != 0:
@@ -466,16 +470,15 @@ class Llama4DecoderLayer(nn.Module):
forward_batch=forward_batch,
)
- hidden_states, residual = self.layer_communicator.prepare_mlp(
- hidden_states, residual, forward_batch
- )
+ hidden_states = self.attn_stage.finish(hidden_states, forward_batch)
+ hidden_states = self.ffn_stage.prepare(hidden_states, forward_batch)
# Fully Connected
- with self.layer_communicator.ffn_exit(forward_batch) as ffn_exit:
+ with self.ffn_stage.exit(forward_batch) as ffn_exit:
hidden_states = self.feed_forward(hidden_states, forward_batch)
- hidden_states, residual = ffn_exit.finish(hidden_states, residual)
+ hidden_states = ffn_exit.finish(hidden_states)
- return hidden_states, residual
+ return hidden_states
class Llama4Model(nn.Module):
@@ -519,28 +522,22 @@ class Llama4Model(nn.Module):
hidden_states = self.embed_tokens(input_ids)
else:
hidden_states = input_embeds
- residual = residual_batch.start(forward_batch)
- aux_hidden_states = []
+ residual_batch.start(forward_batch)
+ aux_hidden_states = AuxHiddenStateList()
for i in range(len(self.layers)):
- if i in self.layers_to_capture:
- aux_hidden_states.append(
- self.layers[i].layer_communicator.snapshot(
- hidden_states, residual, at_input=True
- )
- )
layer = self.layers[i]
- hidden_states, residual = layer(
+ hidden_states = layer(
positions,
hidden_states,
forward_batch,
- residual,
+ capture_output=aux_hidden_states.capture
+ if i in self.layers_to_capture
+ else None,
)
- hidden_states, residual = self.layers[-1].layer_communicator.finish_layer_stack(
- hidden_states, residual, forward_batch
- )
+ hidden_states = residual_batch.finish(hidden_states, forward_batch)
if not forward_batch.forward_mode.is_idle():
- hidden_states, _ = self.norm(hidden_states, residual)
+ hidden_states = residual_batch.norm(hidden_states, forward_batch, self.norm)
if len(aux_hidden_states) == 0:
return hidden_states
@@ -568,29 +565,21 @@ class Llama4ForCausalLM(LlamaForCausalLM):
):
start, end = split_interval
if start == 0:
- forward_batch.residual = residual_batch.start(forward_batch)
+ residual_batch.start(forward_batch)
forward_batch.hidden_states = (
self.model.embed_tokens(input_ids)
if input_embeds is None
else input_embeds
)
for layer in self.model.layers[start:end]:
- forward_batch.hidden_states, forward_batch.residual = layer(
- positions,
- forward_batch.hidden_states,
- forward_batch,
- forward_batch.residual,
+ forward_batch.hidden_states = layer(
+ positions, forward_batch.hidden_states, forward_batch
)
if end != self.model.config.num_hidden_layers:
return None
- hidden, residual = LayerCommunicator.finish_layer_stack(
- forward_batch.hidden_states, forward_batch.residual, forward_batch
+ forward_batch.hidden_states = residual_batch.norm(
+ forward_batch.hidden_states, forward_batch, self.model.norm
)
- forward_batch.residual = None
- if residual is None:
- forward_batch.hidden_states = self.model.norm(hidden)
- else:
- forward_batch.hidden_states, _ = self.model.norm(hidden, residual)
return self.logits_processor(
input_ids, forward_batch.hidden_states, self.lm_head, forward_batch
)
diff --git a/python/sglang/srt/models/nemotron_h.py b/python/sglang/srt/models/nemotron_h.py
index 9540d79ce3..a6eba431c9 100644
--- a/python/sglang/srt/models/nemotron_h.py
+++ b/python/sglang/srt/models/nemotron_h.py
@@ -77,7 +77,7 @@ from sglang.srt.model_loader.weight_utils import (
replace_prefix,
replace_substrings,
)
-from sglang.srt.models.nemotron_h_utils import make_layer_communicator
+from sglang.srt.models.nemotron_h_utils import make_stage_boundary
from sglang.srt.models.utils import WeightsMapper
from sglang.srt.runtime_context import get_exec, get_forward, get_parallel
from sglang.srt.utils import (
@@ -375,8 +375,8 @@ class NemotronHMLPLikeDecoderLayer(nn.Module):
mixer's attention partial sum, or a value that is complete or carries the
sum an FFN before it left."""
- def _init_layer_communicator(self, config: NemotronHConfig, layer_idx: int):
- self.layer_communicator = make_layer_communicator(
+ def _init_stage_boundary(self, config: NemotronHConfig, layer_idx: int):
+ self.stage = make_stage_boundary(
self.norm, pattern=config.hybrid_override_pattern, layer_idx=layer_idx
)
@@ -384,15 +384,12 @@ class NemotronHMLPLikeDecoderLayer(nn.Module):
self,
*,
hidden_states,
- residual: torch.Tensor | None,
forward_batch: ForwardBatch,
):
- hidden_states, residual = self.layer_communicator.prepare_mlp(
- hidden_states, residual, forward_batch
- )
- with self.layer_communicator.ffn_exit(forward_batch) as ffn_exit:
+ hidden_states = self.stage.prepare(hidden_states, forward_batch)
+ with self.stage.exit(forward_batch) as ffn_exit:
hidden_states = self.mixer.forward(hidden_states)
- return ffn_exit.finish(hidden_states, residual)
+ return ffn_exit.finish(hidden_states)
class NemotronHMLPDecoderLayer(NemotronHMLPLikeDecoderLayer):
@@ -426,7 +423,7 @@ class NemotronHMLPDecoderLayer(NemotronHMLPLikeDecoderLayer):
)
self.norm = RMSNorm(config.hidden_size, eps=config.layer_norm_epsilon)
- self._init_layer_communicator(config, layer_idx)
+ self._init_stage_boundary(config, layer_idx)
class NemotronHMoEDecoderLayer(NemotronHMLPLikeDecoderLayer):
@@ -449,7 +446,7 @@ class NemotronHMoEDecoderLayer(NemotronHMLPLikeDecoderLayer):
)
self.norm = RMSNorm(config.hidden_size, eps=config.layer_norm_epsilon)
- self._init_layer_communicator(config, layer_idx)
+ self._init_stage_boundary(config, layer_idx)
class NemotronHAttnLikeDecoderLayer(nn.Module):
@@ -457,8 +454,8 @@ class NemotronHAttnLikeDecoderLayer(nn.Module):
output all-reduce to that stage's input; otherwise it reduces the output
itself unless the fused kernel takes the sum into the next input norm."""
- def _init_layer_communicator(self, config: NemotronHConfig, layer_idx: int):
- self.layer_communicator = make_layer_communicator(
+ def _init_stage_boundary(self, config: NemotronHConfig, layer_idx: int):
+ self.stage = make_stage_boundary(
self.norm, pattern=config.hybrid_override_pattern, layer_idx=layer_idx
)
@@ -466,20 +463,17 @@ class NemotronHAttnLikeDecoderLayer(nn.Module):
self,
*,
hidden_states,
- residual: torch.Tensor | None,
forward_batch: ForwardBatch,
):
- hidden_states, residual = self.layer_communicator.prepare_attn(
- hidden_states, residual, forward_batch
- )
+ hidden_states = self.stage.prepare(hidden_states, forward_batch)
if forward_batch.forward_mode.is_idle():
- return hidden_states, residual
+ return hidden_states
- with self.layer_communicator.mixer_exit(forward_batch) as mixer_exit:
+ with self.stage.exit(forward_batch) as mixer_exit:
hidden_states = self._forward_mixer(
hidden_states, forward_batch, mixer_exit.skips_reduction
)
- return mixer_exit.finish(hidden_states, residual)
+ return mixer_exit.finish(hidden_states)
class NemotronHMambaDecoderLayer(NemotronHAttnLikeDecoderLayer):
@@ -507,7 +501,7 @@ class NemotronHMambaDecoderLayer(NemotronHAttnLikeDecoderLayer):
)
self.norm = RMSNorm(config.hidden_size, eps=config.layer_norm_epsilon)
- self._init_layer_communicator(config, layer_idx)
+ self._init_stage_boundary(config, layer_idx)
def _forward_mamba(
self,
@@ -645,7 +639,7 @@ class NemotronHAttentionDecoderLayer(NemotronHAttnLikeDecoderLayer):
)
self.norm = RMSNorm(config.hidden_size, eps=config.layer_norm_epsilon)
- self._init_layer_communicator(config, layer_idx)
+ self._init_stage_boundary(config, layer_idx)
def _forward_mixer(
self,
@@ -733,50 +727,38 @@ class NemotronHModel(nn.Module):
hidden_states = inputs_embeds
else:
hidden_states = self.embed_tokens(input_ids)
- residual = residual_batch.start(forward_batch)
+ residual_batch.start(forward_batch)
else:
assert pp_proxy_tensors is not None
- hidden_states, residual = self.layers[
- self.start_layer
- ].layer_communicator.from_pp(pp_proxy_tensors, forward_batch)
+ hidden_states = self.layers[self.start_layer].stage.from_pp(
+ pp_proxy_tensors, forward_batch
+ )
aux_hidden_states = []
for i in range(self.start_layer, self.end_layer):
if i in self.layers_to_capture:
- hidden_states, snapshot = self.layers[
- i
- ].layer_communicator.capture_output(
- hidden_states, residual, at_input=True
+ hidden_states, snapshot = self.layers[i].stage.capture_output(
+ hidden_states, forward_batch, at_input=True
)
aux_hidden_states.append(snapshot)
layer = self.layers[i]
if not isinstance(layer, Layers):
raise ValueError(f"Unknown layer type: {type(layer)}")
- hidden_states, residual = layer.forward(
+ hidden_states = layer.forward(
hidden_states=hidden_states,
- residual=residual,
forward_batch=forward_batch,
)
- last_layer = self.layers[self.end_layer - 1]
- hidden_states, residual = last_layer.layer_communicator.finish_layer_stack(
- hidden_states,
- residual,
- forward_batch,
- preserve_declared=not self.pp_group.is_last_rank,
- )
if not self.pp_group.is_last_rank:
- return PPProxyTensors(
- {"hidden_states": hidden_states, "residual": residual}
+ return residual_batch.to_pp(
+ hidden_states, forward_batch, preserve_declared=True
)
+ hidden_states = residual_batch.finish(hidden_states, forward_batch)
if self.end_layer in self.layers_to_capture:
aux_hidden_states.append(
- last_layer.layer_communicator.snapshot(hidden_states, residual)
+ residual_batch.snapshot(hidden_states, forward_batch)
)
- if residual is None:
- hidden_states = self.norm_f(hidden_states)
- else:
- hidden_states, _ = self.norm_f(hidden_states, residual)
+ hidden_states = residual_batch.norm(hidden_states, forward_batch, self.norm_f)
if aux_hidden_states:
return hidden_states, aux_hidden_states
return hidden_states
diff --git a/python/sglang/srt/models/nemotron_h_mtp.py b/python/sglang/srt/models/nemotron_h_mtp.py
index 162b394689..c373bcd9c8 100644
--- a/python/sglang/srt/models/nemotron_h_mtp.py
+++ b/python/sglang/srt/models/nemotron_h_mtp.py
@@ -89,9 +89,8 @@ class NemotronHMTPAttentionDecoderLayer(NemotronHAttentionDecoderLayer):
*,
inputs_embeds: torch.Tensor,
hidden_states: torch.Tensor,
- residual: torch.Tensor | None = None,
forward_batch: ForwardBatch,
- ) -> tuple[torch.Tensor, torch.Tensor]:
+ ) -> torch.Tensor:
if self.has_start_projections:
inputs_embeds_normed = self.enorm(inputs_embeds)
previous_hidden_states_normed = self.hnorm(hidden_states)
@@ -105,23 +104,22 @@ class NemotronHMTPAttentionDecoderLayer(NemotronHAttentionDecoderLayer):
hidden_states, dim=-1
)
- hidden_states, residual = super().forward(
+ if self.has_start_projections:
+ hidden_states = residual_batch.written(hidden_states, forward_batch)
+
+ hidden_states = super().forward(
hidden_states=hidden_states,
- residual=residual,
forward_batch=forward_batch,
)
if self.has_end_norm:
- hidden_states, residual = self.layer_communicator.finish_layer_stack(
- hidden_states, residual, forward_batch
- )
- hidden_states, residual = self.layer_communicator.fold(
- hidden_states, residual
+ hidden_states = residual_batch.fold(hidden_states, forward_batch)
+
+ hidden_states = residual_batch.written(
+ self.final_layernorm(hidden_states), forward_batch
)
- hidden_states = self.final_layernorm(hidden_states)
-
- return hidden_states, residual
+ return hidden_states
class NemotronHMTPMoEDecoderLayer(NemotronHMoEDecoderLayer):
@@ -173,9 +171,8 @@ class NemotronHMTPMoEDecoderLayer(NemotronHMoEDecoderLayer):
*,
inputs_embeds: torch.Tensor,
hidden_states: torch.Tensor,
- residual: torch.Tensor | None = None,
forward_batch: ForwardBatch,
- ) -> tuple[torch.Tensor, torch.Tensor]:
+ ) -> torch.Tensor:
if self.has_start_projections:
inputs_embeds_normed = self.enorm(inputs_embeds)
previous_hidden_states_normed = self.hnorm(hidden_states)
@@ -189,23 +186,22 @@ class NemotronHMTPMoEDecoderLayer(NemotronHMoEDecoderLayer):
hidden_states, dim=-1
)
- hidden_states, residual = super().forward(
+ if self.has_start_projections:
+ hidden_states = residual_batch.written(hidden_states, forward_batch)
+
+ hidden_states = super().forward(
hidden_states=hidden_states,
- residual=residual,
forward_batch=forward_batch,
)
if self.has_end_norm:
- hidden_states, residual = self.layer_communicator.finish_layer_stack(
- hidden_states, residual, forward_batch
- )
- hidden_states, residual = self.layer_communicator.fold(
- hidden_states, residual
+ hidden_states = residual_batch.fold(hidden_states, forward_batch)
+
+ hidden_states = residual_batch.written(
+ self.final_layernorm(hidden_states), forward_batch
)
- hidden_states = self.final_layernorm(hidden_states)
-
- return hidden_states, residual
+ return hidden_states
class NemotronHMultiTokenPredictor(nn.Module):
@@ -302,18 +298,15 @@ class NemotronHMultiTokenPredictor(nn.Module):
inputs_embeds = self.get_input_embeddings(input_ids)
hidden_states = forward_batch.spec_info.hidden_states
- residual = None
+ residual_batch.start(forward_batch)
for i in range(self.pattern_len):
- if residual is None:
- residual = residual_batch.start(forward_batch)
- hidden_states, residual = self.layers[str(i)](
+ hidden_states = self.layers[str(i)](
inputs_embeds=inputs_embeds,
hidden_states=hidden_states,
- residual=residual,
forward_batch=forward_batch,
)
- return hidden_states
+ return residual_batch.take_output(hidden_states, forward_batch)
class NemotronHForCausalLMMTP(NemotronHForCausalLM):
diff --git a/python/sglang/srt/models/nemotron_h_utils.py b/python/sglang/srt/models/nemotron_h_utils.py
index b2270968d0..b06c9ac108 100644
--- a/python/sglang/srt/models/nemotron_h_utils.py
+++ b/python/sglang/srt/models/nemotron_h_utils.py
@@ -101,14 +101,12 @@ def layer_stage(pattern: str, layer_idx: int) -> LayerStage:
)
-def make_layer_communicator(
- layer_norm: RMSNorm, *, pattern: str, layer_idx: int
-) -> LayerCommunicator:
+def make_stage_boundary(layer_norm: RMSNorm, *, pattern: str, layer_idx: int):
"""The communicator of a layer that is one stage: only its own norm, and
boundaries built from the stages next to it in the pattern."""
stage = layer_stage(pattern, layer_idx)
for_attn = stage.kind is StageKind.ATTENTION
- return LayerCommunicator(
+ boundaries = LayerCommunicator(
layer_facts=LayerFacts(
is_layer_sparse=pattern[layer_idx] == MOE,
is_last_layer=layer_idx == len(pattern) - 1,
@@ -121,3 +119,5 @@ def make_layer_communicator(
allow_reduce_scatter=not for_attn,
stage=stage,
)
+
+ return boundaries.attn if for_attn else boundaries.ffn
diff --git a/python/sglang/srt/models/qwen3.py b/python/sglang/srt/models/qwen3.py
index 735ec82428..7f9ec3bd6c 100644
--- a/python/sglang/srt/models/qwen3.py
+++ b/python/sglang/srt/models/qwen3.py
@@ -1,12 +1,16 @@
# Adapted from qwen2.py
import logging
-from typing import Any, Dict, Iterable, List, Optional, Tuple, Union
+from typing import Any, Dict, Iterable, List, Optional, Tuple
import torch
from torch import nn
-from sglang.srt.layers.communicator import LayerCommunicator, LayerFacts
-from sglang.srt.layers.communicator.residual import access as residual_access
+from sglang.srt.layers.aux_hidden_states import AuxHiddenStateList
+from sglang.srt.layers.communicator import (
+ declare_attn,
+ declare_ffn,
+ make_stages,
+)
from sglang.srt.layers.communicator.residual import batch as residual_batch
from sglang.srt.layers.layernorm import RMSNorm
from sglang.srt.layers.linear import QKVParallelLinear, RowParallelLinear
@@ -372,18 +376,14 @@ class Qwen3DecoderLayer(nn.Module):
config.hidden_size, eps=config.rms_norm_eps, **norm_kwargs
)
- self.layer_facts = LayerFacts.init_new(
- layer_id=layer_id,
- num_layers=config.num_hidden_layers,
- is_layer_sparse=False,
- is_previous_layer_sparse=False,
- is_next_layer_sparse=False,
- )
- self.layer_communicator = LayerCommunicator(
- layer_facts=self.layer_facts,
- input_layernorm=self.input_layernorm,
- post_attention_layernorm=self.post_attention_layernorm,
- allow_deferred_ffn_reduction=False,
+ self.attn_stage, self.ffn_stage = make_stages(
+ (declare_attn(), self.input_layernorm),
+ (
+ declare_ffn(sparse=False, next_sparse=False),
+ self.post_attention_layernorm,
+ ),
+ previous=declare_ffn() if layer_id != 0 else None,
+ terminal=layer_id == config.num_hidden_layers - 1,
)
def forward(
@@ -391,15 +391,15 @@ class Qwen3DecoderLayer(nn.Module):
positions: torch.Tensor,
hidden_states: torch.Tensor,
forward_batch: ForwardBatch,
- residual: Optional[torch.Tensor],
post_residual_addition: Optional[torch.Tensor] = None,
- ) -> Tuple[torch.Tensor, torch.Tensor]:
+ capture_output=None,
+ ) -> torch.Tensor:
# Self Attention
- hidden_states, residual = self.layer_communicator.prepare_attn(
+ hidden_states = self.attn_stage.prepare(
hidden_states,
- residual,
forward_batch,
post_residual_addition=post_residual_addition,
+ capture_output=capture_output,
)
if hidden_states.shape[0] != 0:
hidden_states = self.self_attn(
@@ -409,9 +409,9 @@ class Qwen3DecoderLayer(nn.Module):
)
# Fully Connected
- hidden_states, residual = self.layer_communicator.prepare_mlp(
+ hidden_states = self.attn_stage.finish(hidden_states, forward_batch)
+ hidden_states = self.ffn_stage.prepare(
hidden_states,
- residual,
forward_batch,
cache=(
[self.mlp.gate_up_proj.weight, self.mlp.down_proj.weight]
@@ -424,12 +424,12 @@ class Qwen3DecoderLayer(nn.Module):
else None
),
)
- with self.layer_communicator.ffn_exit(forward_batch) as ffn_exit:
+ with self.ffn_stage.exit(forward_batch) as ffn_exit:
hidden_states = self.mlp(hidden_states, forward_batch=forward_batch)
if _is_npu and get_cmo_stream():
wait_cmo_stream()
- hidden_states, residual = ffn_exit.finish(hidden_states, residual)
- return hidden_states, residual
+ hidden_states = ffn_exit.finish(hidden_states)
+ return hidden_states
class Qwen3Model(Qwen2Model):
@@ -450,60 +450,39 @@ class Qwen3Model(Qwen2Model):
def forward(
self,
- input_ids: torch.Tensor,
- positions: torch.Tensor,
- forward_batch: ForwardBatch,
- input_embeds: torch.Tensor = None,
- pp_proxy_tensors: Optional[PPProxyTensors] = None,
- ) -> Union[torch.Tensor, PPProxyTensors]:
-
+ input_ids,
+ positions,
+ forward_batch,
+ input_embeds=None,
+ pp_proxy_tensors=None,
+ ):
if self.pp_group.is_first_rank:
- if input_embeds is None:
- hidden_states = self.embed_tokens(input_ids)
- else:
- hidden_states = input_embeds
- residual = residual_batch.start(forward_batch)
+ hidden_states = (
+ self.embed_tokens(input_ids) if input_embeds is None else input_embeds
+ )
+ residual_batch.start(forward_batch)
else:
- assert pp_proxy_tensors is not None
- hidden_states, residual = self.layers[
- self.start_layer
- ].layer_communicator.from_pp(pp_proxy_tensors, forward_batch)
-
- aux_hidden_states = []
+ hidden_states = self.layers[self.start_layer].attn_stage.from_pp(
+ pp_proxy_tensors, forward_batch
+ )
+ aux_hidden_states = AuxHiddenStateList()
for i in range(self.start_layer, self.end_layer):
- if i in self.layers_to_capture:
- aux_hidden_states.append(
- residual_access.snapshot(hidden_states, residual)
- )
- layer = self.layers[i]
- hidden_states, residual = layer(
+ hidden_states = self.layers[i](
positions,
hidden_states,
forward_batch,
- residual,
+ capture_output=aux_hidden_states.capture
+ if i in self.layers_to_capture
+ else None,
)
-
- hidden_states, residual = residual_access.finish_layer_stack(
- hidden_states, residual, forward_batch
- )
if not self.pp_group.is_last_rank:
- return PPProxyTensors(
- {
- "hidden_states": hidden_states,
- "residual": residual,
- }
- )
- else:
- if hidden_states.shape[0] != 0:
- if residual is None:
- hidden_states = self.norm(hidden_states)
- else:
- hidden_states, _ = self.norm(hidden_states, residual)
-
- if len(aux_hidden_states) == 0:
- return hidden_states
-
- return hidden_states, aux_hidden_states
+ return residual_batch.to_pp(hidden_states, forward_batch)
+ hidden_states = residual_batch.finish(hidden_states, forward_batch)
+ if hidden_states.shape[0] != 0:
+ hidden_states = residual_batch.norm(hidden_states, forward_batch, self.norm)
+ return (
+ (hidden_states, aux_hidden_states) if aux_hidden_states else hidden_states
+ )
class Qwen3ForCausalLM(nn.Module):
@@ -613,7 +592,7 @@ class Qwen3ForCausalLM(nn.Module):
start, end = split_interval
# embed
if start == 0:
- forward_batch.residual = residual_batch.start(forward_batch)
+ residual_batch.start(forward_batch)
if input_embeds is None:
forward_batch.hidden_states = self.model.embed_tokens(input_ids)
else:
@@ -621,25 +600,16 @@ class Qwen3ForCausalLM(nn.Module):
# decoder layer
for i in range(start, end):
layer = self.model.layers[i]
- forward_batch.hidden_states, forward_batch.residual = layer(
+ forward_batch.hidden_states = layer(
positions,
forward_batch.hidden_states,
forward_batch,
- forward_batch.residual,
)
if end == self.model.config.num_hidden_layers:
- forward_batch.hidden_states, forward_batch.residual = self.model.layers[
- end - 1
- ].layer_communicator.finish_layer_stack(
- forward_batch.hidden_states, forward_batch.residual, forward_batch
+ forward_batch.hidden_states = residual_batch.norm(
+ forward_batch.hidden_states, forward_batch, self.model.norm
)
- # norm
- hidden_states, _ = self.model.norm(
- forward_batch.hidden_states, forward_batch.residual
- )
- forward_batch.residual = None
- forward_batch.hidden_states = hidden_states
# logits process
result = self.logits_processor(
input_ids, forward_batch.hidden_states, self.lm_head, forward_batch
diff --git a/python/sglang/srt/models/qwen3_vl.py b/python/sglang/srt/models/qwen3_vl.py
index a644956def..553add3fb0 100644
--- a/python/sglang/srt/models/qwen3_vl.py
+++ b/python/sglang/srt/models/qwen3_vl.py
@@ -37,6 +37,7 @@ from sglang.srt.layers.attention.vision import (
VisionAttentionMetadata,
prepare_vision_attention_metadata,
)
+from sglang.srt.layers.aux_hidden_states import AuxHiddenStateList
from sglang.srt.layers.communicator.residual import batch as residual_batch
from sglang.srt.layers.conv import Conv3dLayer
from sglang.srt.layers.dp_attention import (
@@ -1205,24 +1206,23 @@ class Qwen3LLMModel(Qwen3Model):
hidden_states = self.embed_tokens(input_ids)
else:
hidden_states = input_embeds
- residual = residual_batch.start(forward_batch)
+ residual_batch.start(forward_batch)
else:
assert pp_proxy_tensors is not None
- hidden_states, residual = self.layers[
- self.start_layer
- ].layer_communicator.from_pp(pp_proxy_tensors, forward_batch)
+ hidden_states = self.layers[self.start_layer].attn_stage.from_pp(
+ pp_proxy_tensors, forward_batch
+ )
- aux_hidden_states = []
+ aux_hidden_states = AuxHiddenStateList()
for layer_idx, layer in enumerate(
self.layers[self.start_layer : self.end_layer]
):
layer_idx = layer_idx + self.start_layer
- if layer_idx in self.layers_to_capture:
- aux_hidden_states.append(
- layer.layer_communicator.snapshot(
- hidden_states, residual, at_input=True
- )
- )
+ capture_output = (
+ aux_hidden_states.capture
+ if layer_idx in self.layers_to_capture
+ else None
+ )
if self.use_hf_deepstack_order:
# HF-order path (RL on-policy / FSDP). SGLang applies residual at the START of the
@@ -1231,40 +1231,33 @@ class Qwen3LLMModel(Qwen3Model):
deepstack_embeds = self.get_deepstack_embeds(
layer_idx - 1, input_deepstack_embeds
)
- hidden_states, residual = layer(
+ hidden_states = layer(
positions,
hidden_states,
forward_batch,
- residual,
post_residual_addition=deepstack_embeds,
+ capture_output=capture_output,
)
else:
# Inference path: add deepstack directly to hidden_states at the end of the layer
# (original, grounding-correct order).
- hidden_states, residual = layer(
+ hidden_states = layer(
positions,
hidden_states,
forward_batch,
- residual,
+ capture_output=capture_output,
)
if (
input_deepstack_embeds is not None
and layer_idx in self.deepstack_embed_to_decoder_layer
):
sep = self.hidden_size * layer_idx
- hidden_states.add_(
- input_deepstack_embeds[:, sep : sep + self.hidden_size]
+ hidden_states = residual_batch.add_to_output(
+ hidden_states,
+ forward_batch,
+ input_deepstack_embeds[:, sep : sep + self.hidden_size],
)
- hidden_states, residual = self.layers[
- self.end_layer - 1
- ].layer_communicator.finish_layer_stack(
- hidden_states,
- residual,
- forward_batch,
- preserve_declared=not self.pp_group.is_last_rank,
- )
-
# Handle deepstack for the last processed layer (HF-order path only).
last_deepstack = (
self.get_deepstack_embeds(self.end_layer - 1, input_deepstack_embeds)
@@ -1273,20 +1266,15 @@ class Qwen3LLMModel(Qwen3Model):
)
if not self.pp_group.is_last_rank:
- return PPProxyTensors(
- {
- "hidden_states": hidden_states,
- "residual": residual,
- }
+ return residual_batch.to_pp(hidden_states, forward_batch)
+ hidden_states = residual_batch.finish(hidden_states, forward_batch)
+ if hidden_states.shape[0] != 0:
+ hidden_states = residual_batch.norm(
+ hidden_states,
+ forward_batch,
+ self.norm,
+ post_residual_addition=last_deepstack,
)
- else:
- if hidden_states.shape[0] != 0:
- if residual is None:
- hidden_states = self.norm(hidden_states)
- else:
- hidden_states, _ = self.norm(
- hidden_states, residual, post_residual_addition=last_deepstack
- )
if len(aux_hidden_states) == 0:
return hidden_states
diff --git a/python/sglang/srt/models/qwen4_exp.py b/python/sglang/srt/models/qwen4_exp.py
index 18623a1cc0..9779b6e62a 100644
--- a/python/sglang/srt/models/qwen4_exp.py
+++ b/python/sglang/srt/models/qwen4_exp.py
@@ -72,7 +72,7 @@ from sglang.srt.models.qwen4_exp_ple_table import (
make_ple_file_prefetcher,
make_ple_file_rss_trimmer,
)
-from sglang.srt.runtime_context import get_parallel
+from sglang.srt.runtime_context import get_forward, get_parallel
from sglang.srt.utils import get_bool_env_var, is_hip, logger
_use_aiter = get_bool_env_var("SGLANG_USE_AITER") and is_hip()
@@ -1439,14 +1439,16 @@ class Qwen4ExpLayerExtensionMixin:
attn_tp_chunks = list(hidden_states.tensor_split(attn_tp_size))
hidden_states = attn_tp_chunks[get_parallel().attn_tp_rank].contiguous()
- hidden_states = self.mlp(hidden_states, forward_batch)
+ use_reduce_scatterv = use_dp_moe_gather and should_use_dp_reduce_scatterv()
+ with get_forward().scoped(mlp_reduce_scatter=use_reduce_scatterv):
+ hidden_states = self.mlp(hidden_states, forward_batch)
if use_dp_moe_gather:
hidden_states, global_hidden_states = (
get_local_dp_buffer(get_parallel().tp_group),
hidden_states,
)
- if should_use_dp_reduce_scatterv():
+ if use_reduce_scatterv:
get_parallel().tp_group.reduce_scatterv(
global_hidden_states,
output=hidden_states,
diff --git a/python/sglang/srt/models/sarvam_moe.py b/python/sglang/srt/models/sarvam_moe.py
index 6de760f7df..beccb75323 100644
--- a/python/sglang/srt/models/sarvam_moe.py
+++ b/python/sglang/srt/models/sarvam_moe.py
@@ -1456,36 +1456,26 @@ class SarvamMoEForCausalLM(BailingMoEForCausalLM):
start, end = split_interval
if start == 0:
+ residual_batch.start(forward_batch)
if input_embeds is None:
forward_batch.hidden_states = self.model.word_embeddings(input_ids)
else:
forward_batch.hidden_states = input_embeds
- forward_batch.residual = residual_batch.start(forward_batch)
for i in range(start, end):
with get_global_expert_distribution_recorder().with_current_layer(i):
layer = self.model.layers[i]
- forward_batch.hidden_states, forward_batch.residual = layer(
- positions,
- forward_batch.hidden_states,
- forward_batch,
- forward_batch.residual,
+ forward_batch.hidden_states = layer(
+ positions, forward_batch.hidden_states, forward_batch
)
if end == self.model.config.num_hidden_layers:
- forward_batch.hidden_states, forward_batch.residual = self.model.layers[
- end - 1
- ].layer_communicator.finish_layer_stack(
- forward_batch.hidden_states, forward_batch.residual, forward_batch
+ forward_batch.hidden_states = residual_batch.finish(
+ forward_batch.hidden_states, forward_batch
+ )
+ forward_batch.hidden_states = residual_batch.norm(
+ forward_batch.hidden_states, forward_batch, self.model.norm
)
- if forward_batch.residual is None:
- hidden_states = self.model.norm(forward_batch.hidden_states)
- else:
- hidden_states, _ = self.model.norm(
- forward_batch.hidden_states, forward_batch.residual
- )
- forward_batch.residual = None
- forward_batch.hidden_states = hidden_states
return self.logits_processor(
input_ids, forward_batch.hidden_states, self.lm_head, forward_batch
diff --git a/python/sglang/srt/models/sdar.py b/python/sglang/srt/models/sdar.py
index 843046cdaf..5e3cb0a85e 100644
--- a/python/sglang/srt/models/sdar.py
+++ b/python/sglang/srt/models/sdar.py
@@ -11,7 +11,11 @@ from torch import nn
from transformers import PretrainedConfig
from sglang.srt.layers.activation import SiluAndMul
-from sglang.srt.layers.communicator import LayerCommunicator, LayerFacts
+from sglang.srt.layers.communicator import (
+ declare_attn,
+ declare_ffn,
+ make_stages,
+)
from sglang.srt.layers.communicator.residual import batch as residual_batch
from sglang.srt.layers.dp_attention import (
is_dp_attention_enabled,
@@ -291,19 +295,17 @@ class SDARBlock(nn.Module):
prefix=add_prefix("mlp", prefix),
)
- self.layer_facts = LayerFacts.init_new(
- layer_id=layer_id,
- num_layers=config.num_hidden_layers,
- is_layer_sparse=False,
- is_previous_layer_sparse=False,
- is_next_layer_sparse=False,
- )
- self.layer_communicator = LayerCommunicator(
- layer_facts=self.layer_facts,
- input_layernorm=self.input_layernorm,
- post_attention_layernorm=self.post_attention_layernorm,
- allow_reduce_scatter=True,
- allow_deferred_ffn_reduction=False,
+ self.attn_stage, self.ffn_stage = make_stages(
+ (declare_attn(), self.input_layernorm),
+ (
+ declare_ffn(
+ sparse=False,
+ next_sparse=False,
+ ),
+ self.post_attention_layernorm,
+ ),
+ previous=declare_ffn() if layer_id != 0 else None,
+ terminal=layer_id == config.num_hidden_layers - 1,
)
def forward(
@@ -311,13 +313,8 @@ class SDARBlock(nn.Module):
positions: torch.Tensor,
hidden_states: torch.Tensor,
forward_batch: ForwardBatch,
- residual: Optional[torch.Tensor],
- ) -> Tuple[torch.Tensor, torch.Tensor]:
- hidden_states, residual = self.layer_communicator.prepare_attn(
- hidden_states,
- residual,
- forward_batch,
- )
+ ) -> torch.Tensor:
+ hidden_states = self.attn_stage.prepare(hidden_states, forward_batch)
if hidden_states.shape[0] != 0:
hidden_states = self.self_attn(
@@ -326,17 +323,14 @@ class SDARBlock(nn.Module):
forward_batch=forward_batch,
)
- hidden_states, residual = self.layer_communicator.prepare_mlp(
- hidden_states,
- residual,
- forward_batch,
- )
+ hidden_states = self.attn_stage.finish(hidden_states, forward_batch)
+ hidden_states = self.ffn_stage.prepare(hidden_states, forward_batch)
- with self.layer_communicator.ffn_exit(forward_batch) as ffn_exit:
+ with self.ffn_stage.exit(forward_batch) as ffn_exit:
hidden_states = self.mlp(hidden_states)
- hidden_states, residual = ffn_exit.finish(hidden_states, residual)
+ hidden_states = ffn_exit.finish(hidden_states)
- return hidden_states, residual
+ return hidden_states
class SDARModel(nn.Module):
@@ -404,37 +398,25 @@ class SDARModel(nn.Module):
hidden_states = (
self.embed_tokens(input_ids) if input_embeds is None else input_embeds
)
- residual = residual_batch.start(forward_batch)
+ residual_batch.start(forward_batch)
else:
assert pp_proxy_tensors is not None
- hidden_states, residual = self.layers[
- self.start_layer
- ].layer_communicator.from_pp(
+ hidden_states = self.layers[self.start_layer].attn_stage.from_pp(
pp_proxy_tensors, forward_batch, allow_missing_residual=True
)
for i in range(self.start_layer, self.end_layer):
layer = self.layers[i]
- hidden_states, residual = layer(
- positions, hidden_states, forward_batch, residual
- )
-
- hidden_states, residual = self.layers[
- self.end_layer - 1
- ].layer_communicator.finish_layer_stack(
- hidden_states,
- residual,
- forward_batch,
- preserve_declared=not self.pp_group.is_last_rank,
- )
+ hidden_states = layer(positions, hidden_states, forward_batch)
if not self.pp_group.is_last_rank:
- return PPProxyTensors(
- {"hidden_states": hidden_states, "residual": residual}
- )
+ return residual_batch.to_pp(hidden_states, forward_batch)
else:
+ hidden_states = residual_batch.finish(hidden_states, forward_batch)
if not forward_batch.forward_mode.is_idle():
- hidden_states, residual = self.norm(hidden_states, residual)
+ hidden_states = residual_batch.norm(
+ hidden_states, forward_batch, self.norm
+ )
return hidden_states
diff --git a/python/sglang/srt/models/sdar_moe.py b/python/sglang/srt/models/sdar_moe.py
index a7cd5b6be8..a690c14b43 100644
--- a/python/sglang/srt/models/sdar_moe.py
+++ b/python/sglang/srt/models/sdar_moe.py
@@ -14,8 +14,9 @@ from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_r
from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation
from sglang.srt.eplb.expert_location_dispatch import ExpertLocationDispatchInfo
from sglang.srt.layers.communicator import (
- LayerCommunicator,
- LayerFacts,
+ declare_attn,
+ declare_ffn,
+ make_stages,
)
from sglang.srt.layers.communicator.residual import batch as residual_batch
from sglang.srt.layers.dp_attention import (
@@ -356,19 +357,16 @@ class SDARMoeBlock(nn.Module):
prefix=add_prefix("mlp", prefix),
)
- self.layer_facts = LayerFacts.init_new(
- layer_id=layer_id,
- num_layers=config.num_hidden_layers,
- is_layer_sparse=True,
- is_previous_layer_sparse=True,
- is_next_layer_sparse=True,
- )
-
- self.layer_communicator = LayerCommunicator(
- layer_facts=self.layer_facts,
- input_layernorm=self.input_layernorm,
- post_attention_layernorm=self.post_attention_layernorm,
- allow_reduce_scatter=True,
+ self.attn_stage, self.ffn_stage = make_stages(
+ (declare_attn(), self.input_layernorm),
+ (
+ declare_ffn(sparse=True, next_sparse=True),
+ self.post_attention_layernorm,
+ ),
+ previous=declare_ffn(sparse=True, next_sparse=True)
+ if layer_id != 0
+ else None,
+ terminal=layer_id == config.num_hidden_layers - 1,
)
def forward(
@@ -376,12 +374,9 @@ class SDARMoeBlock(nn.Module):
positions: torch.Tensor,
hidden_states: torch.Tensor,
forward_batch: ForwardBatch,
- residual: Optional[torch.Tensor],
- ) -> Tuple[torch.Tensor, torch.Tensor]:
+ ) -> torch.Tensor:
- hidden_states, residual = self.layer_communicator.prepare_attn(
- hidden_states, residual, forward_batch
- )
+ hidden_states = self.attn_stage.prepare(hidden_states, forward_batch)
if hidden_states.shape[0] != 0:
hidden_states = self.self_attn(
@@ -390,18 +385,17 @@ class SDARMoeBlock(nn.Module):
forward_batch=forward_batch,
)
- hidden_states, residual = self.layer_communicator.prepare_mlp(
- hidden_states, residual, forward_batch
- )
+ hidden_states = self.attn_stage.finish(hidden_states, forward_batch)
+ hidden_states = self.ffn_stage.prepare(hidden_states, forward_batch)
- with self.layer_communicator.ffn_exit(forward_batch) as ffn_exit:
+ with self.ffn_stage.exit(forward_batch) as ffn_exit:
hidden_states = self.mlp(
hidden_states,
forward_batch=forward_batch,
)
- hidden_states, residual = ffn_exit.finish(hidden_states, residual)
+ hidden_states = ffn_exit.finish(hidden_states)
- return hidden_states, residual
+ return hidden_states
class SDARMoeModel(nn.Module):
@@ -470,34 +464,24 @@ class SDARMoeModel(nn.Module):
hidden_states = (
self.embed_tokens(input_ids) if input_embeds is None else input_embeds
)
- residual = residual_batch.start(forward_batch)
+ residual_batch.start(forward_batch)
else:
assert pp_proxy_tensors is not None
- hidden_states, residual = self.layers[
- self.start_layer
- ].layer_communicator.from_pp(pp_proxy_tensors, forward_batch)
+ hidden_states = self.layers[self.start_layer].attn_stage.from_pp(
+ pp_proxy_tensors, forward_batch
+ )
for i in range(self.start_layer, self.end_layer):
layer = self.layers[i]
with get_global_expert_distribution_recorder().with_current_layer(i):
- hidden_states, residual = layer(
- positions, hidden_states, forward_batch, residual
- )
+ hidden_states = layer(positions, hidden_states, forward_batch)
- last_layer = self.layers[self.end_layer - 1]
- hidden_states, residual = last_layer.layer_communicator.finish_layer_stack(
- hidden_states,
- residual,
- forward_batch,
- preserve_declared=not self.pp_group.is_last_rank,
- )
if not self.pp_group.is_last_rank:
- return PPProxyTensors(
- {"hidden_states": hidden_states, "residual": residual}
- )
+ return residual_batch.to_pp(hidden_states, forward_batch)
+ hidden_states = residual_batch.finish(hidden_states, forward_batch)
if not forward_batch.forward_mode.is_idle():
- hidden_states, residual = self.norm(hidden_states, residual)
+ hidden_states = residual_batch.norm(hidden_states, forward_batch, self.norm)
return hidden_states
diff --git a/python/sglang/srt/models/step3_vl.py b/python/sglang/srt/models/step3_vl.py
index ead440abfc..0706b50329 100644
--- a/python/sglang/srt/models/step3_vl.py
+++ b/python/sglang/srt/models/step3_vl.py
@@ -16,13 +16,14 @@ from sglang.srt.configs.step3_vl import (
Step3VisionEncoderConfig,
Step3VLConfig,
)
-from sglang.srt.distributed import (
- tensor_model_parallel_all_reduce,
-)
from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation
from sglang.srt.layers.activation import SiluAndMul
from sglang.srt.layers.attention.vision import VisionAttention
-from sglang.srt.layers.communicator import LayerCommunicator, LayerFacts
+from sglang.srt.layers.communicator import (
+ declare_attn,
+ declare_ffn,
+ make_stages,
+)
from sglang.srt.layers.communicator.residual import batch as residual_batch
from sglang.srt.layers.conv import Conv2dLayer
from sglang.srt.layers.dp_attention import (
@@ -40,6 +41,7 @@ from sglang.srt.layers.moe import get_moe_a2a_backend
from sglang.srt.layers.moe.ep_moe.layer import get_moe_impl_class
from sglang.srt.layers.moe.fused_moe_triton import FusedMoE
from sglang.srt.layers.moe.topk import TopK
+from sglang.srt.layers.moe.utils import reduce_moe_output
from sglang.srt.layers.quantization.base_config import QuantizationConfig
from sglang.srt.layers.radix_attention import RadixAttention
from sglang.srt.layers.rotary_embedding import get_rope
@@ -78,6 +80,7 @@ class Step3TextMLP(nn.Module):
hidden_act: str,
quant_config: Optional[QuantizationConfig] = None,
prefix: str = "",
+ reduce_results: bool = True,
) -> None:
super().__init__()
self.gate_up_proj = MergedColumnParallelLinear(
@@ -93,6 +96,7 @@ class Step3TextMLP(nn.Module):
bias=False,
quant_config=quant_config,
prefix=add_prefix("down_proj", prefix),
+ reduce_results=reduce_results,
)
if hidden_act != "silu":
raise ValueError(
@@ -163,8 +167,6 @@ class Step3TextMoEMLP(nn.Module):
hidden_states=hidden_states, topk_output=topk_output
)
- if self.tp_size > 1:
- final_hidden_states = tensor_model_parallel_all_reduce(final_hidden_states)
return final_hidden_states.view(num_tokens, hidden_dim)
@@ -339,14 +341,6 @@ class Step3TextDecoderLayer(nn.Module):
)
self.is_next_layer_sparse = True if layer_id + 1 in moe_layers_idx else False
- self.layer_facts = LayerFacts.init_new(
- layer_id=layer_id,
- num_layers=config.num_hidden_layers,
- is_layer_sparse=self.is_layer_sparse,
- is_previous_layer_sparse=self.is_previous_layer_sparse,
- is_next_layer_sparse=self.is_next_layer_sparse,
- )
-
if not self.is_layer_sparse:
self.mlp = Step3TextMLP(
hidden_size=config.hidden_size,
@@ -370,6 +364,7 @@ class Step3TextDecoderLayer(nn.Module):
hidden_act="silu",
quant_config=quant_config,
prefix=add_prefix("share_expert", prefix),
+ reduce_results=False,
)
else:
self.moe = Step3TextMoEMLP(
@@ -379,11 +374,21 @@ class Step3TextDecoderLayer(nn.Module):
prefix=add_prefix("mlp", prefix),
)
- self.layer_communicator = LayerCommunicator(
- layer_facts=self.layer_facts,
- input_layernorm=self.input_layernorm,
- post_attention_layernorm=self.post_attention_layernorm,
- allow_deferred_ffn_reduction=False,
+ self.attn_stage, self.ffn_stage = make_stages(
+ (declare_attn(), self.input_layernorm),
+ (
+ declare_ffn(
+ sparse=self.is_layer_sparse,
+ next_sparse=self.is_next_layer_sparse,
+ ),
+ self.post_attention_layernorm,
+ ),
+ previous=declare_ffn(
+ sparse=self.is_previous_layer_sparse, next_sparse=self.is_layer_sparse
+ )
+ if layer_id != 0
+ else None,
+ terminal=layer_id == config.num_hidden_layers - 1,
)
def moe_mlp_forward(self, hidden_states):
@@ -393,19 +398,16 @@ class Step3TextDecoderLayer(nn.Module):
hidden_states += self.share_expert(h)
else:
hidden_states = self.moe(hidden_states)
- return hidden_states
+ return reduce_moe_output(hidden_states)
def forward(
self,
positions: torch.Tensor,
hidden_states: torch.Tensor,
forward_batch: ForwardBatch,
- residual: Optional[torch.Tensor],
- ) -> Tuple[torch.Tensor, torch.Tensor]:
+ ) -> torch.Tensor:
- hidden_states, residual = self.layer_communicator.prepare_attn(
- hidden_states, residual, forward_batch
- )
+ hidden_states = self.attn_stage.prepare(hidden_states, forward_batch)
if hidden_states.shape[0] != 0:
hidden_states = self.self_attn(
@@ -414,17 +416,16 @@ class Step3TextDecoderLayer(nn.Module):
forward_batch=forward_batch,
)
- hidden_states, residual = self.layer_communicator.prepare_mlp(
- hidden_states, residual, forward_batch
- )
- with self.layer_communicator.ffn_exit(forward_batch) as ffn_exit:
+ hidden_states = self.attn_stage.finish(hidden_states, forward_batch)
+ hidden_states = self.ffn_stage.prepare(hidden_states, forward_batch)
+ with self.ffn_stage.exit(forward_batch) as ffn_exit:
if self.use_moe:
hidden_states = self.moe_mlp_forward(hidden_states)
else:
hidden_states = self.mlp(hidden_states)
- hidden_states, residual = ffn_exit.finish(hidden_states, residual)
+ hidden_states = ffn_exit.finish(hidden_states)
- return hidden_states, residual
+ return hidden_states
class Step3TextModel(nn.Module):
@@ -472,22 +473,15 @@ class Step3TextModel(nn.Module):
else:
hidden_states = input_embeds
- residual = residual_batch.start(forward_batch)
+ residual_batch.start(forward_batch)
for i in range(len(self.layers)):
layer = self.layers[i]
- hidden_states, residual = layer(
- positions, hidden_states, forward_batch, residual
- )
+ hidden_states = layer(positions, hidden_states, forward_batch)
- hidden_states, residual = self.layers[-1].layer_communicator.finish_layer_stack(
- hidden_states, residual, forward_batch
- )
+ hidden_states = residual_batch.finish(hidden_states, forward_batch)
if hidden_states.shape[0] != 0:
- if residual is None:
- hidden_states = self.norm(hidden_states)
- else:
- hidden_states, _ = self.norm(hidden_states, residual)
+ hidden_states = residual_batch.norm(hidden_states, forward_batch, self.norm)
return hidden_states
diff --git a/python/sglang/test/boundary_fixtures.py b/python/sglang/test/boundary_fixtures.py
new file mode 100644
index 0000000000..5ae4de8887
--- /dev/null
+++ b/python/sglang/test/boundary_fixtures.py
@@ -0,0 +1,125 @@
+"""Small fixtures around production boundaries; no parallel layout recipes."""
+
+from types import SimpleNamespace
+
+from sglang.srt.layers.communicator.construction import BatchVariant, StagePlan
+from sglang.srt.layers.communicator.factories import (
+ declare_attn,
+ declare_ffn,
+ make_stages,
+)
+from sglang.srt.layers.communicator.residual.add_norm import PLAIN_RESIDUAL
+from sglang.srt.layers.communicator.stage import StageCommunicator
+
+
+def make_test_stages(
+ *,
+ attention_norm,
+ ffn_norm,
+ first=False,
+ last=False,
+ sparse=False,
+ previous_sparse=False,
+ next_sparse=False,
+ residual=PLAIN_RESIDUAL,
+ output=None,
+ **options,
+):
+ previous = (
+ None
+ if first
+ else declare_ffn(
+ sparse=previous_sparse, next_sparse=sparse, update=residual.ffn_update
+ )
+ )
+ attention = declare_attn(
+ read=residual.attention_read, update=residual.attention_update
+ )
+ ffn = declare_ffn(
+ sparse=sparse,
+ next_sparse=next_sparse,
+ read=residual.ffn_read,
+ update=residual.ffn_update,
+ output=output,
+ )
+ common = {
+ key: options.pop(key)
+ for key in ("force_layernorm_before_dp_gather", "fusions")
+ if key in options
+ }
+ attn, ffn = make_stages(
+ (
+ attention,
+ attention_norm,
+ dict(
+ residual_in_hidden=residual.ffn_update.at_producer, **common, **options
+ ),
+ ),
+ (ffn, ffn_norm, common),
+ previous=previous,
+ terminal=last,
+ )
+ return SimpleNamespace(attn=attn, ffn=ffn)
+
+
+def stub_plan():
+ """Supply only defaults for tests that inject preselected paths/kernels."""
+ plan = StagePlan.__new__(StagePlan)
+ plan.norm = None
+ plan.fusions = None
+ plan._publish_lora_layout = False
+ plan._fusion_rows = None
+ plan._cp_steps = plan._sp_steps = plan._input_scattered_steps = None
+ plan.is_first_layer = False
+ plan.residual_in_hidden = False
+ # Keep production variant selection while tests replace individual paths.
+ plan._batch_steps = lambda batch: {
+ BatchVariant.ORDINARY: plan._steps,
+ BatchVariant.CONTEXT_PARALLEL: plan._cp_steps,
+ BatchVariant.INPUT_SCATTERED: plan._input_scattered_steps,
+ BatchVariant.SEQUENCE_PARALLEL: plan._sp_steps,
+ }[plan._variant(batch)]
+ return plan
+
+
+def stub_stage(plan, kind):
+ """Keep the same concrete boundary when a test patches its methods."""
+ stages = plan.__dict__.setdefault("_stub_stages", {})
+ if kind not in stages:
+ stages[kind] = StageCommunicator(plan, kind, plan.norm)
+ return stages[kind]
+
+
+def sp_region_steps():
+ """Local SP rows with no owed sum, for tests of activation and exits."""
+ from sglang.srt.layers.communicator import (
+ NORM_QUANT_READ,
+ NORM_READ,
+ BoundarySteps,
+ CommunicateSummableTensorPairFn,
+ EdgeDecl,
+ Layout,
+ StageEntry,
+ StageInput,
+ StageOutput,
+ TokenAxis,
+ make_boundary,
+ )
+ from sglang.srt.layers.communicator.ops import _hand_qkv_hook_its_input
+
+ rows = Layout(frozenset({TokenAxis.ATTN_TP_SCATTER}))
+ output = StageOutput(rows)
+
+ def entry(read, handoff=None):
+ selected = make_boundary(
+ EdgeDecl(output, StageInput(rows, read=read), rows, rows)
+ )
+ return StageEntry(selected.prepare, rows, handoff=handoff)
+
+ return BoundarySteps(
+ entry(NORM_QUANT_READ, _hand_qkv_hook_its_input),
+ entry(NORM_READ),
+ output,
+ CommunicateSummableTensorPairFn._trivial,
+ False,
+ )
diff --git a/python/sglang/test/communicator_patch.py b/python/sglang/test/communicator_patch.py
index d082da4abb..521e40406c 100644
--- a/python/sglang/test/communicator_patch.py
+++ b/python/sglang/test/communicator_patch.py
@@ -8,14 +8,40 @@ the name, not only into the package.
import contextlib
from unittest import mock
-from sglang.srt.layers import communicator
-from sglang.srt.layers.communicator import boundary, layer, layout, ops, output
+from sglang.srt.layers import communicator as communicator
+from sglang.srt.layers.communicator import (
+ boundary,
+ construction,
+ exit,
+ factories,
+ layer,
+ layout,
+ legacy_stage,
+ ops,
+ output,
+)
from sglang.srt.layers.communicator import residual as residual_contract
-from sglang.srt.layers.communicator.adapters import attention
-from sglang.srt.layers.communicator.residual import access, add_norm, mhc
+from sglang.srt.layers.communicator import (
+ stage,
+)
+from sglang.srt.layers.communicator.adapters import attention, branch, lora
+from sglang.srt.layers.communicator.fusions import allreduce
+from sglang.srt.layers.communicator.residual import (
+ access,
+ add_norm,
+ batch,
+ mhc,
+ stream,
+)
COMMUNICATOR_MODULES = (
+ layer,
+ legacy_stage,
communicator,
+ allreduce,
+ exit,
+ branch,
+ lora,
layout,
output,
attention,
@@ -25,7 +51,11 @@ COMMUNICATOR_MODULES = (
mhc,
ops,
boundary,
- layer,
+ construction,
+ factories,
+ stage,
+ batch,
+ stream,
)
diff --git a/test/registered/unit/layers/moe/test_cutedsl_ar_fusion.py b/test/registered/unit/layers/moe/test_cutedsl_ar_fusion.py
index 94c7a160de..caa1e4d72e 100644
--- a/test/registered/unit/layers/moe/test_cutedsl_ar_fusion.py
+++ b/test/registered/unit/layers/moe/test_cutedsl_ar_fusion.py
@@ -57,7 +57,6 @@ def _communicator():
comm.enable_fused_ar_quant = False
comm._attn_input_fusions = comm._select_attn_input_fusions()
# Only the ordinary batches' attention input half, with these entries.
- comm._context = SimpleNamespace()
comm._sp_steps = comm._input_scattered_steps = comm._cp_steps = None
comm._steps = SimpleNamespace(
attention=SimpleNamespace(
@@ -548,7 +547,7 @@ class TestDeferredLoraAllReduce(unittest.TestCase):
from types import SimpleNamespace
from unittest.mock import patch
- from sglang.srt.layers.communicator import layer as exits
+ from sglang.srt.layers.communicator import exit as exits
from sglang.srt.layers.communicator.fusions.cutedsl import CuteDSLFusion
group = object()
diff --git a/test/registered/unit/layers/moe/test_moe_output_reduction.py b/test/registered/unit/layers/moe/test_moe_output_reduction.py
index 1c384f7638..3b9e29d45d 100644
--- a/test/registered/unit/layers/moe/test_moe_output_reduction.py
+++ b/test/registered/unit/layers/moe/test_moe_output_reduction.py
@@ -91,7 +91,7 @@ class TestReduceMoeOutput(CustomTestCase):
for flags, config in (
({"fuse_mlp_allreduce": True}, {}),
({"mlp_reduce_scatter": True}, {}),
- ({}, {"reduce_scatterv": True}),
+ ({"mlp_reduce_scatter": True}, {"reduce_scatterv": True}),
):
with self.subTest(flags=flags, config=config), moe_config(**config):
self.assertEqual(self.all_reduces(**flags)[0], 0)
@@ -138,14 +138,19 @@ class TestReplicatedMoeOutput(CustomTestCase):
for flags, config in (
({"fuse_mlp_allreduce": True}, {}),
({"mlp_reduce_scatter": True}, {}),
- ({}, {"reduce_scatterv": True}),
+ ({"mlp_reduce_scatter": True}, {"reduce_scatterv": True}),
):
with self.subTest(flags=flags, config=config):
outputs = self.summed_over_two_ranks(flags=flags, **config)
torch.testing.assert_close(sum(outputs), torch.full((2, 3), 13.0))
def test_every_rank_adds_it_to_a_reduced_or_complete_output(self):
- for config in ({}, {"backend": "flashinfer"}, {"fp4_allgather": True}):
+ for config in (
+ {},
+ {"backend": "flashinfer"},
+ {"fp4_allgather": True},
+ {"reduce_scatterv": True},
+ ):
with self.subTest(**config):
outputs = self.summed_over_two_ranks(**config)
torch.testing.assert_close(outputs[1], torch.full((2, 3), 12.0))
@@ -158,6 +163,102 @@ class TestReplicatedMoeOutput(CustomTestCase):
self.assertTrue(should_add_replicated_moe_output())
+class TestModelsWithExplicitDpCompletion(CustomTestCase):
+ def test_direct_dp_exits_publish_their_selected_sum(self):
+ # Execute the real orchestration methods without constructing weights.
+ # These models own their DP exit instead of using StageCommunicator.
+ import __future__
+
+ for filename, method in (
+ ("deepseek_v4.py", "_run_moe_ffn_dp_sync"),
+ ("qwen4_exp.py", "_run_qwen4_exp_mlp"),
+ ):
+ path = MODELS_DIR / filename
+ node = next(
+ n
+ for n in ast.walk(ast.parse(path.read_text()))
+ if isinstance(n, ast.FunctionDef) and n.name == method
+ )
+ for use_rsv in (False, True):
+ with self.subTest(model=filename, reduce_scatterv=use_rsv):
+ trace = []
+
+ def rsv(value, *, output, sizes):
+ trace.append("RSv")
+ output.copy_(value[:2] * 2)
+
+ def scatter(output, value, batch):
+ trace.append("slice")
+ output.copy_(value[:2])
+
+ group = types.SimpleNamespace(reduce_scatterv=rsv)
+ parallel = types.SimpleNamespace(
+ attn_dp_size=2, attn_tp_size=1, tp_size=2, tp_group=group
+ )
+ namespace = dict(
+ torch=torch,
+ get_forward=get_forward,
+ get_parallel=lambda: parallel,
+ get_moe_a2a_backend=lambda: a2a(),
+ should_use_dp_reduce_scatterv=lambda: use_rsv,
+ is_dp_gatherv_active=lambda: False,
+ envs=types.SimpleNamespace(
+ SGLANG_DP_USE_REDUCE_SCATTER=types.SimpleNamespace(
+ get=lambda: False
+ )
+ ),
+ _SHARED_EXPERT_LOCAL=False,
+ nullcontext=contextlib.nullcontext,
+ get_global_dp_buffer=lambda g: torch.empty(4, 3),
+ get_local_dp_buffer=lambda g: torch.empty(2, 3),
+ get_dp_global_num_tokens=lambda: [2, 2],
+ dp_gather_replicate=lambda output, value, batch: output.fill_(
+ 1
+ ),
+ dp_scatter=scatter,
+ )
+ exec(
+ compile(
+ ast.Module(body=[node], type_ignores=[]),
+ str(path),
+ "exec",
+ flags=__future__.annotations.compiler_flag,
+ ),
+ namespace,
+ )
+ model = types.SimpleNamespace(
+ dsa_enable_prefill_cp=False,
+ config=types.SimpleNamespace(num_experts=4),
+ _qwen4_exp_use_dp_moe_gather=lambda: True,
+ _qwen4_exp_use_attn_tp_a2a_scatter=lambda: False,
+ mlp=lambda value, batch, **kwargs: reduce_moe_output(value),
+ )
+ batch = types.SimpleNamespace(
+ dp_padding_mode=types.SimpleNamespace(is_max_len=lambda: True)
+ )
+ kwargs = (
+ dict(input_ids=None, input_ids_global=None)
+ if filename == "deepseek_v4.py"
+ else {}
+ )
+ with (
+ moe_config(),
+ get_forward().scoped(
+ fuse_mlp_allreduce=False, mlp_reduce_scatter=False
+ ),
+ patch(
+ "sglang.srt.distributed.communication_op.tensor_model_parallel_all_reduce",
+ side_effect=lambda value: trace.append("AR") or value * 2,
+ ),
+ ):
+ output = namespace[method](
+ model, torch.ones(2, 3), batch, **kwargs
+ )
+ self.assertFalse(get_forward().mlp_reduce_scatter)
+ self.assertEqual(trace, ["RSv"] if use_rsv else ["AR", "slice"])
+ torch.testing.assert_close(output, torch.full((2, 3), 2.0))
+
+
def reads_the_tp1_shared_expert_switch(tree):
return any(
isinstance(node, ast.Attribute) and node.attr == "SGLANG_SHARED_EXPERT_TP1"
diff --git a/test/registered/unit/layers/test_aux_hidden_states.py b/test/registered/unit/layers/test_aux_hidden_states.py
index 7d76daffe5..27ff15d9ea 100644
--- a/test/registered/unit/layers/test_aux_hidden_states.py
+++ b/test/registered/unit/layers/test_aux_hidden_states.py
@@ -1,3 +1,6 @@
+from sglang.srt.layers.communicator import StageKind
+from sglang.test.boundary_fixtures import stub_plan, stub_stage
+
"""Aux owns retained storage, including captures from reusable gather buffers."""
import unittest
@@ -7,7 +10,6 @@ from unittest.mock import Mock, patch
import torch
from sglang.srt.layers.aux_hidden_states import AuxHiddenStateList, AuxHiddenStatePacker
-from sglang.srt.layers.communicator import LayerCommunicator as BoundaryPlan
from sglang.srt.layers.communicator.residual.access import norm_output
from sglang.srt.layers.communicator.residual.stream import ResidualStream
from sglang.test.ci.ci_register import register_cpu_ci
@@ -17,28 +19,6 @@ register_cpu_ci(est_time=5, suite="base-a-test-cpu")
class TestAuxStorage(CustomTestCase):
- def test_constructed_layer_binds_read_only_residual_capture(self):
- import test_declared_decoder_boundary as fixture
-
- from sglang.srt.layers import communicator as comm
- from sglang.srt.layers import layernorm
- from sglang.test.communicator_patch import patch_communicator
-
- parallel = fixture.parallel_of(attn_dp=1, attn_tp=2)
- with (
- fixture.planning(parallel),
- patch_communicator("_use_aiter", False),
- patch_communicator("apply_flashinfer_allreduce_fusion", return_value=True),
- ):
- layer = comm.LayerCommunicator(
- layer_facts=fixture.layer_facts(1, 3),
- input_layernorm=layernorm.RMSNorm(4),
- post_attention_layernorm=layernorm.RMSNorm(4),
- )
- keeps = layer._steps.attention.capture_preserves_residual
- self.assertIsNotNone(keeps)
- self.assertTrue(keeps(torch.ones(2, 4), SimpleNamespace()))
-
def test_breakable_graph_copies_aux_collector_into_plain_list(self):
from sglang.srt.model_executor.runner_backend.breakable_cuda_graph_backend import (
BreakableCudaGraphBackend,
@@ -111,19 +91,18 @@ class TestAuxStorage(CustomTestCase):
self.assertIs(outputs[0], value)
def boundary(self, stream, *, move=None):
- boundary = BoundaryPlan.__new__(BoundaryPlan)
- boundary.input_layernorm = None
- boundary._context = None
- boundary.prepare_attn = Mock(return_value=(stream.residual, stream))
- boundary.attn = SimpleNamespace(
- entry=lambda _: SimpleNamespace(
- input_move=move,
- capture_move=None,
- capture_move_allocates=False,
- capture_preserves_residual=None,
- )
+ boundary = stub_plan()
+ boundary.norm = None
+ stub_stage(boundary, StageKind.ATTENTION)._prepare_input = Mock(
+ return_value=(stream.residual, stream)
)
- return boundary
+ stub_stage(boundary, StageKind.ATTENTION).entry = lambda _: SimpleNamespace(
+ input_move=move,
+ capture_move=None,
+ capture_move_allocates=False,
+ capture_preserves_residual=None,
+ )
+ return stub_stage(boundary, StageKind.ATTENTION)
def test_packer_receives_borrowed_boundary_value_without_intermediate_clone(self):
for callback in (False, True):
@@ -140,9 +119,7 @@ class TestAuxStorage(CustomTestCase):
with patch.object(
torch.Tensor, "clone", side_effect=AssertionError("extra clone")
):
- stage.prepare_attn_and_capture_last_layer_outputs(
- value, stream, None, **kwargs
- )
+ stage._prepare_attention(value, stream, None, **kwargs)
value.zero_()
torch.testing.assert_close(outputs.finalize(), torch.full((2, 3), 4.0))
@@ -153,7 +130,7 @@ class TestAuxStorage(CustomTestCase):
move = Mock(return_value=gathered)
stage = self.boundary(stream, move=move)
outputs = AuxHiddenStateList()
- stage.prepare_attn_and_capture_last_layer_outputs(source, stream, None, outputs)
+ stage._prepare_attention(source, stream, None, outputs)
gathered.zero_()
source.zero_()
torch.testing.assert_close(outputs[0], torch.full((4, 3), 7.0))
@@ -190,11 +167,11 @@ class TestAuxStorage(CustomTestCase):
stream.write(value)
return value, stream
- stage.prepare_attn = prepare
+ stage._prepare_input = prepare
with patch.object(
torch.Tensor, "clone", side_effect=AssertionError("extra clone")
):
- stage.prepare_attn_and_capture_last_layer_outputs(
+ stage._prepare_attention(
hidden, stream, None, capture_output=outputs.capture
)
torch.testing.assert_close(outputs.finalize(), torch.full((2, 3), 4.0))
diff --git a/test/registered/unit/layers/test_batch_owned_residual.py b/test/registered/unit/layers/test_batch_owned_residual.py
index b764298456..fc5967a354 100644
--- a/test/registered/unit/layers/test_batch_owned_residual.py
+++ b/test/registered/unit/layers/test_batch_owned_residual.py
@@ -21,7 +21,7 @@ from sglang.srt.layers.communicator import (
StageOutput,
make_boundary,
)
-from sglang.srt.layers.communicator.layer import StageCommunicator
+from sglang.srt.layers.communicator.legacy_stage import StageCommunicator
from sglang.srt.layers.communicator.output import UnreducedOutput
from sglang.srt.layers.communicator.residual import batch
from sglang.srt.model_executor.forward_batch_info import (
@@ -73,7 +73,7 @@ class TestBatchOwnedResidual(CustomTestCase):
def test_pp_receive_attaches_and_export_releases_the_same_stream(self):
communicator = LayerCommunicator.__new__(LayerCommunicator)
- communicator._batch_steps = lambda _: SimpleNamespace(
+ communicator._batch_steps = lambda fb: SimpleNamespace(
attention=SimpleNamespace(input_sum=None), ffn=None
)
fb = SimpleNamespace(residual_stream=None)
@@ -174,15 +174,17 @@ class TestBatchOwnedResidual(CustomTestCase):
class Layer:
layer_communicator = LayerCommunicator.__new__(LayerCommunicator)
- def __call__(self, positions, hidden, forward_batch, residual):
+ def __call__(self, positions, hidden, forward_batch, *legacy_residual):
stream = batch.current(forward_batch)
- self_outer.assertIs(residual, stream)
+ if legacy_residual:
+ self_outer.assertIs(legacy_residual[0], stream)
owner_ids.append(id(stream))
hidden, old = stream.finish(hidden)
stream.write(hidden if old is None else hidden + old)
- return stream.leave(
+ result = stream.leave(
UnreducedOutput(torch.ones_like(hidden), group=group), ADD
- ), stream
+ )
+ return (result, stream) if legacy_residual else result
self_outer = self
fb = SimpleNamespace(
@@ -214,6 +216,43 @@ class TestBatchOwnedResidual(CustomTestCase):
self.assertEqual(group.all_reduce.call_count, 4)
torch.testing.assert_close(result, torch.full((2, 4), 9.0))
+ def test_mtp_returns_written_normalized_output_and_releases_batch(self):
+ from sglang.srt.models.nemotron_h_mtp import NemotronHMultiTokenPredictor
+
+ def terminal_layer(*, inputs_embeds, hidden_states, forward_batch):
+ stream = batch.current(forward_batch)
+ stream.write(hidden_states)
+ hidden_states = stream.leave(torch.ones_like(hidden_states), ADD)
+ hidden_states = batch.fold(hidden_states, forward_batch)
+ normalized = hidden_states * 3
+ return batch.written(normalized, forward_batch)
+
+ predictor = SimpleNamespace(pattern_len=1, layers={"0": terminal_layer})
+ fb = SimpleNamespace(
+ residual_stream=None,
+ spec_info=SimpleNamespace(hidden_states=torch.ones(2, 4)),
+ )
+ result = NemotronHMultiTokenPredictor.forward(
+ predictor, None, None, fb, inputs_embeds=torch.zeros(2, 4)
+ )
+ torch.testing.assert_close(result, torch.full((2, 4), 6.0))
+ self.assertIsNone(fb.residual_stream)
+
+ def test_take_output_rejects_pending_or_mismatched_outputs(self):
+ fb = SimpleNamespace(residual_stream=None)
+ stream = batch.start(fb)
+ value = torch.ones(2, 4)
+ stream.write(value)
+ pending = stream.leave(value * 2, ADD)
+ with self.assertRaises(RuntimeError):
+ batch.take_output(pending, fb)
+ self.assertIs(batch.current(fb), stream)
+ stream.write(value)
+ with self.assertRaises(RuntimeError):
+ batch.take_output(value.clone(), fb)
+ self.assertIs(batch.take_output(value, fb), value)
+ self.assertIsNone(fb.residual_stream)
+
def test_interleaved_stages_share_only_their_own_batch_stream(self):
class Read:
norms_plainly = False
@@ -236,23 +275,23 @@ class TestBatchOwnedResidual(CustomTestCase):
StageEntry(boundary.prepare, rows), None, StageOutput(rows), None, False
)
stage = StageCommunicator(
- SimpleNamespace(input_layernorm=None, _context=None),
+ SimpleNamespace(input_layernorm=None),
"attention",
"input_layernorm",
)
a, b = [SimpleNamespace(residual_stream=None) for _ in range(2)]
sa, sb = batch.start(a), batch.start(b)
- first, first_alias = stage.prepare(torch.ones(2, 4), sa, a, steps)
- second, second_alias = stage.prepare(torch.full((2, 4), 10.0), sb, b, steps)
+ first, first_alias = stage._prepare(torch.ones(2, 4), sa, a, steps)
+ second, second_alias = stage._prepare(torch.full((2, 4), 10.0), sb, b, steps)
self.assertIs(first_alias, batch.current(a))
self.assertIs(second_alias, batch.current(b))
first = sa.leave(first * 3, ADD)
second = sb.leave(second * 5, ADD)
torch.testing.assert_close(
- stage.prepare(first, sa, a, steps)[0], torch.full((2, 4), 14.0)
+ stage._prepare(first, sa, a, steps)[0], torch.full((2, 4), 14.0)
)
torch.testing.assert_close(
- stage.prepare(second, sb, b, steps)[0], torch.full((2, 4), 220.0)
+ stage._prepare(second, sb, b, steps)[0], torch.full((2, 4), 220.0)
)
self.assertIsNot(batch.current(a), batch.current(b))
batch.start(a)
diff --git a/test/registered/unit/layers/test_boundary_edges.py b/test/registered/unit/layers/test_boundary_edges.py
index e0c4867f54..cd60bdcf51 100644
--- a/test/registered/unit/layers/test_boundary_edges.py
+++ b/test/registered/unit/layers/test_boundary_edges.py
@@ -16,12 +16,6 @@ from sglang.srt.layers.communicator import (
StageOutput,
SumGroup,
TokenAxis,
-)
-from sglang.srt.layers.communicator import boundary as comm_boundary
-from sglang.srt.layers.communicator import (
- decoder_layer_edges,
- decoder_layer_sides,
- input_scattered_layer_sides,
make_boundary,
make_output_boundary,
)
@@ -48,104 +42,6 @@ def rows(axis_sizes, *axes):
return Layout.sharded_over(*axes, axis_sizes=axis_sizes)
-class TestDecoderLayerEdges(CustomTestCase):
- """A decoder layer's three boundaries hand the residual on from one to the
- next: each edge starts on the rows the one before it ends on."""
-
- def test_each_edge_starts_where_the_one_before_ends(self):
- for dp, tp, ffn_local, previous_local, last in itertools.product(
- (1, 2), (1, 2), (False, True), (False, True), (False, True)
- ):
- with self.subTest(
- dp=dp, tp=tp, ffn_local=ffn_local, previous_local=previous_local
- ):
- sides = decoder_layer_sides(
- axis_sizes=sizes(dp=dp, tp=tp),
- ffn_on_local_rows=ffn_local,
- previous_on_local_rows=previous_local,
- is_last_layer=last,
- attention_gathers_local_rows=False,
- ffn_group=SumGroup.TP,
- leaves_for_next_layer=True,
- leaves_for_reduce_scatter=True,
- leaves_for_reduce_scatterv=True,
- )
- edges = decoder_layer_edges(sides)
- self.assertEqual(edges.into_attention.residual, sides.input_rows)
- self.assertEqual(
- edges.into_attention.residual_to, edges.into_ffn.residual
- )
- self.assertEqual(edges.into_ffn.residual_to, edges.out_of_ffn.residual)
- self.assertEqual(edges.out_of_ffn.residual_to, sides.output_rows)
- self.assertEqual(edges.out_of_ffn.need.layout, sides.output_rows)
- # Nothing owed by construction: a sum left for a batch comes
- # with the value.
- self.assertFalse(edges.into_attention.produced.always_leaves)
-
- def test_an_input_owed_by_construction_is_completed_onto_the_slice(self):
- axis_sizes = sizes(tp=2)
- sides = input_scattered_layer_sides(
- axis_sizes=axis_sizes, ffn_group=SumGroup.TP, hands_on_partial=True
- )
- edge = decoder_layer_edges(sides).into_attention
- self.assertIs(edge.produced.group, SumGroup.TP)
- self.assertTrue(edge.produced.always_leaves)
- self.assertEqual(edge.residual_to, rows(axis_sizes, TokenAxis.ATTN_TP_SCATTER))
- step = make_boundary(edge).prepare.keywords["step"]
- self.assertIs(step.keywords["layer_input"], comm.tp_reduce_scatter)
-
- def test_each_entry_keeps_the_move_after_its_read(self):
- sides = decoder_layer_sides(
- axis_sizes=sizes(dp=2, tp=2),
- ffn_on_local_rows=False,
- previous_on_local_rows=False,
- is_last_layer=False,
- attention_gathers_local_rows=False,
- ffn_group=SumGroup.TP,
- leaves_for_next_layer=True,
- leaves_for_reduce_scatter=True,
- leaves_for_reduce_scatterv=True,
- )
- edges = decoder_layer_edges(sides)
- moves = {edges.into_attention: MagicMock(), edges.into_ffn: MagicMock()}
- real = comm_boundary.make_boundary
-
- def with_move(edge, **kwargs):
- return msgspec.structs.replace(real(edge, **kwargs), input_move=moves[edge])
-
- with patch.object(comm_boundary, "make_boundary", with_move):
- steps = comm_boundary._select_boundary_steps(sides)
- self.assertIs(steps.attention.input_move, moves[edges.into_attention])
- self.assertIs(steps.ffn.input_move, moves[edges.into_ffn])
-
-
-class TestAMoeOnEachCpShard(CustomTestCase):
- """A MoE whose data-parallel groups are the CP ranks computes each CP shard
- on its own ranks: its input stays sharded over CP, and the attention output
- reaches it without a gather."""
-
- def test_the_ffn_rows_stay_sharded_over_cp(self):
- axis_sizes = sizes(cp=2, tp=2)
- sides = decoder_layer_sides(
- axis_sizes=axis_sizes,
- ffn_on_local_rows=False,
- previous_on_local_rows=False,
- is_last_layer=False,
- attention_gathers_local_rows=False,
- ffn_group=SumGroup.MOE_OUTPUT,
- leaves_for_next_layer=False,
- leaves_for_reduce_scatter=False,
- leaves_for_reduce_scatterv=False,
- ffn_shards_over_cp=True,
- )
- cp_rows = rows(axis_sizes, TokenAxis.ATTN_CP)
- self.assertEqual(sides.ffn.input.layout, cp_rows)
- self.assertEqual(sides.ffn.output.layout, cp_rows)
- self.assertIs(sides.ffn.output.group, SumGroup.MOE_OUTPUT)
- edges = decoder_layer_edges(sides)
- self.assertEqual(edges.into_ffn.need.layout, edges.into_ffn.produced.layout)
-
-
class TestTheConsumerHalfReadsOnlyItsOwnSide(CustomTestCase):
"""Two layers build the two halves of one boundary without sharing it; the
consumer's half never depends on what the producer may leave for a batch,
@@ -276,7 +172,6 @@ class TestNonAlternatingEdges(CustomTestCase):
torch.full((3, 4), 5.0),
None,
lambda hidden, residual: (hidden + residual, hidden + residual),
- None,
)
all_reduce.assert_called_once()
torch.testing.assert_close(hidden, torch.full((3, 4), 7.0))
@@ -292,7 +187,7 @@ class _WrittenIn:
def update(self, hidden_states, residual):
return 2 * hidden_states + residual
- def residual_to_attn_tp_shard(self, residual, context):
+ def residual_to_attn_tp_shard(self, residual):
return residual
def residual_from_attn_tp_shards(self, residual):
@@ -360,7 +255,6 @@ class TestTheProducersUpdateChoosesTheOrder(CustomTestCase):
torch.full((2, 4), 3.0),
None,
norm,
- None,
update=update,
)
torch.testing.assert_close(hidden, torch.full((2, 4), expected))
@@ -421,12 +315,11 @@ class TestTheProducersUpdateChoosesTheOrder(CustomTestCase):
residual,
None,
norm,
- None,
)
group.all_reduce.assert_called_once()
torch.testing.assert_close(hidden, torch.full((3, 4), 7.0))
group.all_reduce.reset_mock()
- hidden, _ = step(torch.full((3, 4), 2.0), residual, None, norm, None)
+ hidden, _ = step(torch.full((3, 4), 2.0), residual, None, norm)
group.all_reduce.assert_not_called()
torch.testing.assert_close(hidden, torch.full((3, 4), 7.0))
@@ -526,6 +419,7 @@ class TestTheConsumerRunsItsDeclaredRead(CustomTestCase):
self.assertIs(step.func, comm_ops._mlp_input_dp_partial)
hidden, residual = torch.ones(2, 4), torch.full((2, 4), 3.0)
with (
+ get_parallel().override(attn_tp_rank=0),
patch.object(
comm_ops,
"_reduce_and_redistribute_output_to_dp",
@@ -533,9 +427,7 @@ class TestTheConsumerRunsItsDeclaredRead(CustomTestCase):
),
patch.object(comm_ops, "dp_scatter", lambda *args: None),
):
- out, out_residual = step(
- hidden, residual, None, MagicMock(), SimpleNamespace(attn_tp_rank=0)
- )
+ out, out_residual = step(hidden, residual, None, MagicMock())
self.assertEqual(read.reads, 1)
torch.testing.assert_close(out, torch.full((2, 4), 108.0))
self.assertIs(out_residual, residual)
@@ -549,15 +441,15 @@ class TestTheConsumerRunsItsDeclaredRead(CustomTestCase):
step, _ = self._ffn_input(read, residual_joins_sum=True)
self.assertIs(step.func, comm_ops._mlp_input_residual_into_sum)
hidden, residual = torch.ones(4, 4), torch.full((2, 4), 3.0)
- with patch.object(
- comm_ops, "tensor_model_parallel_all_reduce", lambda h: h * 2
+ with (
+ get_parallel().override(tp_size=2, tp_rank=0),
+ patch.object(comm_ops, "tensor_model_parallel_all_reduce", lambda h: h * 2),
):
out, out_residual = step(
hidden,
residual,
None,
MagicMock(),
- SimpleNamespace(tp_size=2, tp_rank=0),
)
self.assertEqual(read.reads, 1)
expected = torch.tensor([8.0, 8.0, 2.0, 2.0])[:, None].expand(4, 4)
diff --git a/test/registered/unit/layers/test_boundary_output_policy.py b/test/registered/unit/layers/test_boundary_output_policy.py
new file mode 100644
index 0000000000..8d21f864d5
--- /dev/null
+++ b/test/registered/unit/layers/test_boundary_output_policy.py
@@ -0,0 +1,249 @@
+"""Output operations and collective fallbacks share one boundary decision."""
+
+import unittest
+from types import SimpleNamespace
+from unittest.mock import patch
+
+import test_declared_decoder_boundary as fixture
+import torch
+
+from sglang.srt.arg_groups.fields.exec_ import ExecComm
+from sglang.srt.layers.communicator.output import OutputTransform
+from sglang.srt.layers.moe import utils as moe_utils
+from sglang.srt.runtime_context import get_forward
+from sglang.test.ci.ci_register import register_cpu_ci
+from sglang.test.communicator_patch import patch_communicator
+
+register_cpu_ci(est_time=5, suite="base-a-test-cpu")
+
+
+class TestBoundaryOutputPolicy(unittest.TestCase):
+ def batch(self, max_len=True):
+ return SimpleNamespace(
+ dp_padding_mode=SimpleNamespace(is_max_len=lambda: max_len),
+ forward_mode=SimpleNamespace(
+ is_context_parallel_extend=lambda: False, is_decode_or_idle=lambda: True
+ ),
+ )
+
+ def test_policy_defaults_to_model_selection(self):
+ self.assertEqual(ExecComm().boundary_reduction, "auto")
+
+ def test_npu_weight_cache_belongs_to_one_prepare_call(self):
+ from sglang.srt.layers.communicator import ops
+ from sglang.srt.layers.communicator.residual.stream import ResidualStream
+
+ parallel = fixture.parallel_of(attn_dp=1, attn_tp=2)
+ layer = fixture.build(fixture.layer_case(0, 2), parallel)
+ batch = self.batch()
+ cache = object()
+ events = []
+
+ def reduce(value):
+ events.append("reduce")
+ return value * 2
+
+ def preload(value, weights):
+ events.append("cache")
+ self.assertIs(weights, cache)
+ torch.testing.assert_close(value, torch.full((2, 4), 2.0))
+
+ with (
+ fixture.planning(parallel),
+ patch.object(ops, "_is_npu", True),
+ patch.object(ops, "prepare_weight_cache", side_effect=preload, create=True),
+ patch_communicator("attention_tensor_model_parallel_all_reduce", reduce),
+ ):
+ for weights in (cache, None):
+ batch.residual_stream = ResidualStream(torch.zeros(2, 4))
+ hidden = layer.attn.finish(torch.ones(2, 4), batch)
+ result = layer.ffn.prepare(hidden, batch, cache=weights)
+ torch.testing.assert_close(result, torch.full((2, 4), 4.0))
+ self.assertEqual(events, ["reduce", "cache", "reduce"])
+ self.assertFalse(hasattr(layer.ffn.plan, "_context"))
+
+ def test_token_slice_reads_the_current_parallel_rank(self):
+ from sglang.srt.layers.communicator.ops import tp_slice
+
+ hidden = torch.arange(8).reshape(4, 2)
+ for rank in (0, 1):
+ with fixture.planning(
+ fixture.parallel_of(attn_dp=1, attn_tp=2, tp_rank=rank)
+ ):
+ value, residual = tp_slice(hidden, hidden)
+ torch.testing.assert_close(value, hidden.chunk(2)[rank])
+ torch.testing.assert_close(residual, hidden.chunk(2)[rank])
+
+ def test_cli_policy_reaches_execution_config(self):
+ import argparse
+
+ from sglang.srt.runtime_context import get_exec, publish, reset_context
+ from sglang.srt.server_args import ServerArgs
+
+ parser = argparse.ArgumentParser()
+ ServerArgs.add_cli_args(parser)
+ for policy in ("auto", "ar", "rs", "rsv", "rs+rsv"):
+ with self.subTest(policy=policy):
+ args = ["--model", "dummy", "--boundary-reduction", policy]
+ record = ServerArgs.from_cli_args(parser.parse_args(args))
+ reset_context()
+ try:
+ record.resolve_once()
+ publish(record, role="test")
+ self.assertEqual(
+ get_exec().comm.boundary_reduction,
+ "rs+rsv" if policy == "auto" else policy,
+ )
+ finally:
+ reset_context()
+
+ def test_step3_shared_and_routed_outputs_have_one_completion(self):
+ from sglang.srt.models.step3_vl import Step3TextDecoderLayer
+
+ parallel = SimpleNamespace(tp_size=2)
+ for fused_shared in (False, True):
+ for deferred in (False, True):
+ calls = []
+ layer = SimpleNamespace(
+ num_fused_shared_experts=int(fused_shared),
+ moe=lambda x: x * (0.75 if fused_shared else 0.25),
+ share_expert=lambda x: x * 0.5,
+ )
+ with (
+ patch.object(moe_utils, "get_parallel", return_value=parallel),
+ patch.object(
+ moe_utils, "post_experts_output_is_complete", return_value=False
+ ),
+ patch(
+ "sglang.srt.distributed.communication_op.tensor_model_parallel_all_reduce",
+ side_effect=lambda x: calls.append("AR") or x * 2,
+ ),
+ get_forward().scoped(
+ fuse_mlp_allreduce=deferred, mlp_reduce_scatter=False
+ ),
+ ):
+ result = Step3TextDecoderLayer.moe_mlp_forward(
+ layer, torch.ones(2, 4)
+ )
+ self.assertEqual(calls, [] if deferred else ["AR"])
+ torch.testing.assert_close(
+ result, torch.full((2, 4), 0.75 if deferred else 1.5)
+ )
+
+ def test_dp_skip_and_collective_are_selected_together(self):
+ import itertools
+
+ for policy, varlen, max_len, can_rs in itertools.product(
+ ("ar", "rs", "rsv", "rs+rsv"), (False, True), (False, True), (False, True)
+ ):
+ with self.subTest(
+ policy=policy, varlen=varlen, max_len=max_len, can_rs=can_rs
+ ):
+ parallel = fixture.parallel_of(attn_dp=2, attn_tp=1)
+ layer = fixture.build(
+ fixture.layer_case(2, 3), parallel, boundary_reduction=policy
+ )
+ calls = []
+
+ def move(step, batch, value):
+ calls.append(step.__name__)
+ return value[:1]
+
+ selected = (
+ "_reduce_and_redistribute_output_varlen"
+ if varlen and policy in ("rsv", "rs+rsv")
+ else "_reduce_and_redistribute_output_max_len"
+ if max_len and can_rs and policy in ("rs", "rs+rsv")
+ else "_redistribute_output"
+ )
+ with (
+ fixture.planning(parallel),
+ patch_communicator("should_use_dp_reduce_scatterv", lambda: varlen),
+ patch_communicator("can_use_dp_reduce_scatter", lambda: can_rs),
+ patch_communicator("_to_local_tokens", move),
+ ):
+ with layer.ffn.plan.output.ffn_exit(self.batch(max_len)) as output:
+ self.assertEqual(
+ get_forward().mlp_reduce_scatter,
+ selected != "_redistribute_output",
+ )
+ result, _ = output.finish(torch.ones(2, 4), torch.zeros(1, 4))
+ self.assertEqual(calls, [selected])
+ self.assertEqual(result.shape, (1, 4))
+
+ def test_moe_cannot_skip_from_topology_without_boundary_request(self):
+ with (
+ patch.object(moe_utils, "should_use_dp_reduce_scatterv", return_value=True),
+ patch.object(
+ moe_utils, "post_experts_output_is_complete", return_value=False
+ ),
+ ):
+ for skip in (False, True):
+ with get_forward().scoped(
+ fuse_mlp_allreduce=False, mlp_reduce_scatter=skip
+ ):
+ self.assertEqual(
+ moe_utils.should_skip_post_experts_all_reduce(is_tp_path=True),
+ skip,
+ )
+
+ def test_output_transform_stays_before_transport_and_after_producer_ar(self):
+ # The compute stub obeys the same skip flag as RowParallelLinear.
+ # Its trace distinguishes AR -> multiply from multiply -> RS.
+ for disabled in (False, True):
+ with self.subTest(disabled=disabled):
+ calls = []
+
+ def scale(value):
+ calls.append("multiply")
+ return value * 3
+
+ parallel = fixture.parallel_of(attn_dp=2, attn_tp=1)
+ layer = fixture.build(
+ fixture.layer_case(1, 3),
+ parallel,
+ output=OutputTransform(scale, before_reduce_scatter=True),
+ boundary_reduction="ar" if disabled else "rs+rsv",
+ )
+
+ def move(step, batch, value):
+ if "reduce" in step.__name__:
+ calls.append("RS")
+ value = value * 2
+ else:
+ calls.append("slice")
+ return value[:1]
+
+ with (
+ fixture.planning(parallel),
+ patch_communicator("should_use_dp_reduce_scatterv", lambda: False),
+ patch_communicator("can_use_dp_reduce_scatter", lambda: True),
+ patch_communicator("_to_local_tokens", move),
+ ):
+ with layer.ffn.plan.output.ffn_exit(self.batch()) as output:
+ value = torch.ones(2, 4)
+ if not get_forward().mlp_reduce_scatter:
+ calls.append("AR")
+ value = value * 2
+ value, _ = output.finish(value, torch.zeros(1, 4))
+ self.assertEqual(
+ calls,
+ ["AR", "multiply", "slice"] if disabled else ["multiply", "RS"],
+ )
+ torch.testing.assert_close(value, torch.full((1, 4), 6.0))
+
+ def test_transform_without_pre_rs_implementation_uses_ar(self):
+ parallel = fixture.parallel_of(attn_dp=2, attn_tp=1)
+ layer = fixture.build(
+ fixture.layer_case(1, 3),
+ parallel,
+ output=OutputTransform(lambda x: x.square()),
+ )
+ output = layer.ffn.plan._steps.ffn_output
+ self.assertFalse(output.leaves_for_next_layer)
+ self.assertFalse(output.leaves_for_reduce_scatter)
+ self.assertFalse(output.leaves_for_reduce_scatterv)
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/test/registered/unit/layers/test_communicator_ffn_exit.py b/test/registered/unit/layers/test_communicator_ffn_exit.py
index f00f7e92cb..69aa130f71 100644
--- a/test/registered/unit/layers/test_communicator_ffn_exit.py
+++ b/test/registered/unit/layers/test_communicator_ffn_exit.py
@@ -10,21 +10,18 @@ import torch
import sglang
from sglang.srt.layers import communicator as comm
from sglang.srt.layers.communicator import (
- LayerCommunicator,
Layout,
StageOutput,
SumGroup,
- TokenAxis,
UnreducedOutput,
)
-from sglang.srt.layers.communicator import boundary as comm_boundary
from sglang.srt.layers.communicator import ops as comm_ops
from sglang.srt.layers.communicator import (
reduce_output,
- sequence_parallel_layer_sides,
)
-from sglang.srt.layers.communicator.residual import batch as residual_batch
+from sglang.srt.layers.communicator.residual.access import finish_layer_stack
from sglang.srt.runtime_context import get_forward
+from sglang.test.boundary_fixtures import sp_region_steps, stub_plan
from sglang.test.ci.ci_register import register_cpu_ci
from sglang.test.communicator_patch import patch_communicator
from sglang.test.test_utils import CustomTestCase
@@ -32,19 +29,6 @@ from sglang.test.test_utils import CustomTestCase
register_cpu_ci(est_time=10, suite="base-a-test-cpu")
-def sp_region_steps():
- """The steps a layer runs while a LayerNorm SP region is active."""
- return comm_boundary._select_boundary_steps(
- sequence_parallel_layer_sides(
- axis_sizes={
- TokenAxis.ATTN_DP: 1,
- TokenAxis.ATTN_CP: 1,
- TokenAxis.ATTN_TP_SCATTER: 2,
- }
- )
- )
-
-
def make_group(scale=3):
"""The group a deferred sum is owed over, with a stubbed all-reduce."""
return types.SimpleNamespace(all_reduce=MagicMock(side_effect=lambda h: h * scale))
@@ -95,12 +79,13 @@ def make_communicator(
reduce_scatter_step=None,
scatters_to_local_tokens=False,
group=None,
- cls=LayerCommunicator,
+ fusions=None,
allow_deferred=True,
):
"""A communicator whose decisions and postprocess are stubbed, built
without the process-wide parallel state."""
- communicator = cls.__new__(cls)
+ communicator = stub_plan()
+ communicator.fusions = fusions
communicator._steps = ordinary_steps(
StageOutput(
Layout(frozenset()),
@@ -109,36 +94,29 @@ def make_communicator(
),
returns_over_dp=scatters_to_local_tokens,
)
- if cls is LayerCommunicator:
- communicator._ffn_sum_moves_to_next_layer = MagicMock(return_value=fuse)
+ if fusions is None:
+ communicator.output._ffn_sum_moves_to_next_layer = MagicMock(return_value=fuse)
else:
- # A subclass decides through its own fused kernels; no TP left to defer
- # over otherwise.
- communicator._context = types.SimpleNamespace(tp_size=1)
- communicator._ffn_leaves_sum_to_reduce_scatter = MagicMock(
+ communicator.output._ffn_sum_moves_to_next_layer = MagicMock(return_value=False)
+ communicator.output._ffn_leaves_sum_to_reduce_scatter = MagicMock(
return_value=reduce_scatter
)
communicator.is_last_layer = False
communicator._sp_steps = None
communicator._input_scattered_steps = None
communicator._cp_steps = None
- communicator._postprocess_dp_step = MagicMock(return_value=reduce_scatter_step)
- communicator.ffn_reduction_group = MagicMock(return_value=group or make_group())
- communicator._complete_ffn_output_now = MagicMock(
+ communicator.output._postprocess_dp_step = MagicMock(
+ return_value=reduce_scatter_step
+ )
+ communicator.output.ffn_reduction_group = MagicMock(
+ return_value=group or make_group()
+ )
+ communicator.output._complete_ffn_output_now = MagicMock(
side_effect=lambda hidden_states, residual, **_: (hidden_states + 1, residual)
)
return communicator
-def finish_exit(ffn_exit, hidden, residual):
- """Inspect the selected completion form inside a batch-owned stream."""
- stream = residual_batch.start(ffn_exit.forward_batch)
- stream.write(residual)
- hidden, output_stream = ffn_exit.finish(hidden, stream)
- assert output_stream is residual_batch.current(ffn_exit.forward_batch)
- return output_stream.input(hidden)
-
-
def published_flags():
forward = get_forward()
return forward.fuse_mlp_allreduce, forward.mlp_reduce_scatter
@@ -148,13 +126,13 @@ class TestFfnExit(CustomTestCase):
def setUp(self):
self.hidden_states = torch.ones(3, 4)
self.residual = torch.zeros(3, 4)
- self.forward_batch = types.SimpleNamespace(residual_stream=None)
+ self.forward_batch = object()
def run_exit(self, communicator):
- with communicator.ffn_exit(self.forward_batch) as ffn_exit:
+ with communicator.output.ffn_exit(self.forward_batch) as ffn_exit:
seen = published_flags()
hidden_states = self.hidden_states * 2
- return seen, finish_exit(ffn_exit, hidden_states, self.residual)
+ return seen, ffn_exit.finish(hidden_states, self.residual)
def test_reduction_left_to_next_layer_is_unreduced(self):
communicator = make_communicator(fuse=True, reduce_scatter=False)
@@ -162,7 +140,7 @@ class TestFfnExit(CustomTestCase):
self.assertEqual(seen, (True, False))
self.assertIsInstance(hidden_states, UnreducedOutput)
self.assertIs(residual, self.residual)
- communicator._complete_ffn_output_now.assert_not_called()
+ communicator.output._complete_ffn_output_now.assert_not_called()
def test_reduction_left_to_next_layer_declares_its_group(self):
group = make_group()
@@ -188,7 +166,7 @@ class TestFfnExit(CustomTestCase):
self.assertIs(bound.func, comm_ops._to_local_tokens)
self.assertEqual(bound.args, (step, self.forward_batch))
self.assertIs(residual, self.residual)
- communicator._complete_ffn_output_now.assert_not_called()
+ communicator.output._complete_ffn_output_now.assert_not_called()
step.assert_not_called()
def test_a_deferred_sum_carries_the_scatter_back_under_attention_dp(self):
@@ -201,7 +179,7 @@ class TestFfnExit(CustomTestCase):
bound = hidden_states.reduce_and_redistribute
self.assertIs(bound.func, comm_ops._all_reduce_then_to_local_tokens)
self.assertEqual(bound.args, (group, self.forward_batch))
- communicator._complete_ffn_output_now.assert_not_called()
+ communicator.output._complete_ffn_output_now.assert_not_called()
group.all_reduce.assert_not_called()
def test_postprocess_completes_other_exits(self):
@@ -213,7 +191,7 @@ class TestFfnExit(CustomTestCase):
seen, (hidden_states, residual) = self.run_exit(communicator)
self.assertEqual(seen, (False, reduce_scatter))
self.assertNotIsInstance(hidden_states, UnreducedOutput)
- communicator._complete_ffn_output_now.assert_called_once()
+ communicator.output._complete_ffn_output_now.assert_called_once()
torch.testing.assert_close(hidden_states, self.hidden_states * 2 + 1)
self.assertIs(residual, self.residual)
@@ -224,8 +202,8 @@ class TestFfnExit(CustomTestCase):
seen, (hidden_states, residual) = self.run_exit(communicator)
self.assertEqual(seen, (False, True))
- communicator._ffn_sum_moves_to_next_layer.assert_not_called()
- communicator._complete_ffn_output_now.assert_called_once()
+ communicator.output._ffn_sum_moves_to_next_layer.assert_not_called()
+ communicator.output._complete_ffn_output_now.assert_called_once()
torch.testing.assert_close(hidden_states, self.hidden_states * 2 + 1)
def test_finish_applies_the_decision_the_ffn_saw(self):
@@ -234,15 +212,17 @@ class TestFfnExit(CustomTestCase):
for fuse in (True, False):
with self.subTest(fuse=fuse):
communicator = make_communicator(fuse=fuse, reduce_scatter=False)
- with communicator.ffn_exit(self.forward_batch) as ffn_exit:
+ with communicator.output.ffn_exit(self.forward_batch) as ffn_exit:
seen = published_flags()
- decide = communicator._ffn_sum_moves_to_next_layer
+ decide = communicator.output._ffn_sum_moves_to_next_layer
decide.return_value = not fuse
hidden_states = self.hidden_states * 2
- hidden_states, _ = finish_exit(ffn_exit, hidden_states, self.residual)
+ hidden_states, _ = ffn_exit.finish(hidden_states, self.residual)
self.assertEqual(seen, (fuse, False))
self.assertEqual(isinstance(hidden_states, UnreducedOutput), fuse)
- self.assertEqual(communicator._complete_ffn_output_now.called, not fuse)
+ self.assertEqual(
+ communicator.output._complete_ffn_output_now.called, not fuse
+ )
def test_flags_are_restored_after_the_ffn(self):
before = published_flags()
@@ -255,28 +235,28 @@ class TestFfnExit(CustomTestCase):
communicator = make_communicator(fuse=allows, reduce_scatter=False)
seen, _ = self.run_exit(communicator)
self.assertEqual(seen, (allows, False))
- communicator._ffn_sum_moves_to_next_layer.assert_called_once()
+ communicator.output._ffn_sum_moves_to_next_layer.assert_called_once()
def test_deferral_implies_fusion_and_passes_the_handoff_through(self):
"""A deferring communicator publishes both flags, and a non-tensor
handoff leaves finish() untouched for the next layer's input norm."""
- class Defers(LayerCommunicator):
- fusions = types.SimpleNamespace(can_defer_finalize=lambda layer, fb: True)
-
- communicator = make_communicator(fuse=False, reduce_scatter=False, cls=Defers)
+ fusions = types.SimpleNamespace(can_defer_finalize=lambda layer, fb: True)
+ communicator = make_communicator(
+ fuse=False, reduce_scatter=False, fusions=fusions
+ )
handoff = object()
- with communicator.ffn_exit(self.forward_batch) as ffn_exit:
+ with communicator.output.ffn_exit(self.forward_batch) as ffn_exit:
seen = published_flags() + (get_forward().defer_moe_finalize,)
self.assertEqual(seen, (True, False, True))
self.assertEqual(
- finish_exit(ffn_exit, handoff, self.residual), (handoff, self.residual)
+ ffn_exit.finish(handoff, self.residual), (handoff, self.residual)
)
- communicator._complete_ffn_output_now.assert_not_called()
+ communicator.output._complete_ffn_output_now.assert_not_called()
self.assertFalse(get_forward().defer_moe_finalize)
# The MoE may decline per forward; a tensor then takes the fused exit.
- hidden_states, _ = finish_exit(ffn_exit, self.hidden_states * 2, self.residual)
+ hidden_states, _ = ffn_exit.finish(self.hidden_states * 2, self.residual)
self.assertIsInstance(hidden_states, UnreducedOutput)
@@ -290,11 +270,9 @@ class TestReduceOutput(CustomTestCase):
def test_reduction_left_by_the_last_layer_runs_once(self):
communicator = self.communicator(fuse=True)
- with communicator.ffn_exit(
- types.SimpleNamespace(residual_stream=None)
- ) as ffn_exit:
+ with communicator.output.ffn_exit(object()) as ffn_exit:
hidden_states = torch.ones(3, 4)
- hidden_states, _ = finish_exit(ffn_exit, hidden_states, torch.zeros(3, 4))
+ hidden_states, _ = ffn_exit.finish(hidden_states, torch.zeros(3, 4))
hidden_states = reduce_output(hidden_states)
self.all_reduce.assert_called_once()
@@ -317,11 +295,9 @@ class TestReduceOutput(CustomTestCase):
def test_complete_hidden_states_pass_through(self):
communicator = self.communicator(fuse=False)
- with communicator.ffn_exit(
- types.SimpleNamespace(residual_stream=None)
- ) as ffn_exit:
+ with communicator.output.ffn_exit(object()) as ffn_exit:
hidden_states = torch.ones(3, 4)
- hidden_states, _ = finish_exit(ffn_exit, hidden_states, torch.zeros(3, 4))
+ hidden_states, _ = ffn_exit.finish(hidden_states, torch.zeros(3, 4))
self.assertIs(reduce_output(hidden_states), hidden_states)
self.assertIsNone(reduce_output(None))
@@ -333,13 +309,11 @@ class TestReduceOutput(CustomTestCase):
self.all_reduce.reset_mock()
communicator = self.communicator(fuse=fuse)
residual = torch.zeros(3, 4)
- with communicator.ffn_exit(
- types.SimpleNamespace(residual_stream=None)
- ) as ffn_exit:
+ with communicator.output.ffn_exit(object()) as ffn_exit:
hidden_states = torch.ones(3, 4)
- hidden_states, _ = finish_exit(ffn_exit, hidden_states, residual)
+ hidden_states, _ = ffn_exit.finish(hidden_states, residual)
- hidden_states, residual_out = communicator.finish_layer_stack(
+ hidden_states, residual_out = finish_layer_stack(
hidden_states, residual, object()
)
self.assertEqual(self.all_reduce.call_count, int(fuse))
@@ -355,7 +329,7 @@ class TestSelectFfnCompletion(CustomTestCase):
def communicator(
self, *, fuse=False, is_last_layer=False, scatters=True, sp_region=False
):
- communicator = LayerCommunicator.__new__(LayerCommunicator)
+ communicator = stub_plan()
communicator.is_last_layer = is_last_layer
communicator._sp_steps = sp_region_steps() if sp_region else None
communicator._input_scattered_steps = None
@@ -370,20 +344,22 @@ class TestSelectFfnCompletion(CustomTestCase):
),
returns_over_dp=scatters,
)
- communicator._ffn_sum_moves_to_next_layer = lambda forward_batch, **_: fuse
- communicator._ffn_leaves_sum_to_reduce_scatter = lambda forward_batch, dp_step: (
- not fuse
+ communicator.output._ffn_sum_moves_to_next_layer = lambda forward_batch, **_: (
+ fuse
)
- communicator._complete_ffn_output_now = lambda h, r, **_: ("now", r)
+ communicator.output._ffn_leaves_sum_to_reduce_scatter = (
+ lambda forward_batch, dp_step: not fuse
+ )
+ communicator.output._complete_ffn_output_now = lambda h, r, **_: ("now", r)
self.group = make_group()
- communicator.ffn_reduction_group = lambda forward_batch: self.group
+ communicator.output.ffn_reduction_group = lambda forward_batch: self.group
return communicator
def left(self, communicator, step, forward_batch=None):
with patch_communicator(
"_reduce_and_redistribute_output_step", return_value=step
):
- completion = communicator._select_ffn_completion(forward_batch)
+ completion = communicator.output._select_ffn_completion(forward_batch)
hidden_states, _ = completion.complete(torch.ones(3, 4), None)
return None if hidden_states == "now" else hidden_states
@@ -559,7 +535,13 @@ def ffn_exit_classes(tree, source):
users = {
cls.name
for cls in classes
- if ".ffn_exit(" in ast.get_source_segment(source, cls)
+ if ".output.ffn_exit(" in ast.get_source_segment(source, cls)
+ or any(
+ isinstance(node, ast.withitem)
+ and isinstance(node.optional_vars, ast.Name)
+ and node.optional_vars.id == "ffn_exit"
+ for node in ast.walk(cls)
+ )
}
grew = True
while grew:
@@ -584,9 +566,8 @@ class TestFfnExitUsersOweATpSum(CustomTestCase):
layers, ffn_classes, offenders = [], set(), []
for module, path in sorted(index.paths.items()):
source = path.read_text()
- if (
- not module.startswith("sglang.srt.models.")
- or ".ffn_exit(" not in source
+ if not module.startswith("sglang.srt.models.") or (
+ ".output.ffn_exit(" not in source and " as ffn_exit:" not in source
):
continue
for cls in ffn_exit_classes(index.tree(module), source):
diff --git a/test/registered/unit/layers/test_declared_attention_cp.py b/test/registered/unit/layers/test_declared_attention_cp.py
index d96af6c7c3..507004bee0 100644
--- a/test/registered/unit/layers/test_declared_attention_cp.py
+++ b/test/registered/unit/layers/test_declared_attention_cp.py
@@ -1,3 +1,5 @@
+from sglang.test.boundary_fixtures import make_test_stages
+
"""A MoE layer on the TP group under DSA (and MLA) prefill CP, rank by rank.
Two CP ranks build the layer's communicator for real and run prepare_mlp and
@@ -62,13 +64,6 @@ def group(name, ranks):
return SimpleNamespace(name=name, ranks=list(ranks))
-def prepare_mlp(communicator, hidden, residual, forward_batch):
- forward_batch.residual_stream = ResidualStream(residual)
- return communicator.prepare_mlp(
- hidden, forward_batch.residual_stream, forward_batch
- )
-
-
class TestAttentionCpBoundary(CustomTestCase):
def setUp(self):
generator = torch.Generator().manual_seed(0)
@@ -166,27 +161,34 @@ class TestAttentionCpBoundary(CustomTestCase):
)
yield SimpleNamespace(parallel=parallel, flags=flags)
- def build(self, allow_reduce_scatter):
+ def build(self, use_reduce_scatter):
# A MoE layer after a dense one; dense layers run on every rank here.
- return comm.LayerCommunicator(
- layer_facts=SimpleNamespace(
- is_first_layer=False,
- is_last_layer=False,
- is_layer_sparse=True,
- is_previous_layer_sparse=False,
- is_next_layer_sparse=False,
+
+ with patch_communicator(
+ "get_exec",
+ lambda: SimpleNamespace(
+ comm=SimpleNamespace(
+ boundary_reduction="rs+rsv" if use_reduce_scatter else "ar"
+ ),
+ overlap=SimpleNamespace(enable_two_batch_overlap=False),
),
- input_layernorm=layernorm,
- post_attention_layernorm=layernorm,
- allow_reduce_scatter=allow_reduce_scatter,
- )
+ ):
+ return make_test_stages(
+ first=False,
+ last=False,
+ sparse=True,
+ previous_sparse=False,
+ next_sparse=False,
+ attention_norm=layernorm,
+ ffn_norm=layernorm,
+ )
def cp_extend(self):
return SimpleNamespace(
forward_mode=SimpleNamespace(is_context_parallel_extend=lambda: True)
)
- def run_ranks(self, allow_reduce_scatter):
+ def run_ranks(self, use_reduce_scatter):
"""Gather on each rank, run an FFN that leaves a partial sum or not,
finish the exit and return what each rank got back and published."""
handed = {}
@@ -208,20 +210,16 @@ class TestAttentionCpBoundary(CustomTestCase):
with self.as_rank(
cp, dict(gather=record_gather(cp), reduce_scatter=unused)
):
- prepare_mlp(
- self.build(allow_reduce_scatter),
- self.values[cp],
- self.residuals[cp],
- self.cp_extend(),
+ self.build(use_reduce_scatter).ffn._prepare_input(
+ self.values[cp], self.residuals[cp], self.cp_extend()
)
gathered, residuals = {}, {}
for cp in range(CP_SIZE):
with self.as_rank(cp, dict(gather=fill_gather, reduce_scatter=unused)):
- gathered[cp], residuals[cp] = prepare_mlp(
- self.build(allow_reduce_scatter),
- self.values[cp],
- self.residuals[cp],
- self.cp_extend(),
+ gathered[cp], residuals[cp] = self.build(
+ use_reduce_scatter
+ ).ffn._prepare_input(
+ self.values[cp], self.residuals[cp], self.cp_extend()
)
expected_rows = torch.cat(
[self.values[cp] + self.residuals[cp] for cp in range(CP_SIZE)]
@@ -258,17 +256,19 @@ class TestAttentionCpBoundary(CustomTestCase):
with self.as_rank(
cp, dict(gather=fill_gather, reduce_scatter=reduce_scatter)
) as rank:
- communicator = self.build(allow_reduce_scatter)
- batch = self.cp_extend()
- batch.residual_stream = ResidualStream(residuals[cp].residual)
- with communicator.ffn_exit(batch) as exit_:
+ communicator = self.build(use_reduce_scatter)
+ with communicator.ffn.plan.output.ffn_exit(
+ self.cp_extend()
+ ) as exit_:
published[cp] = rank.flags.mlp_reduce_scatter
output = ffn_output(cp, leaves=published[cp])
- back[cp], _ = exit_.finish(output, batch.residual_stream)
+ back[cp], _ = exit_.finish(
+ output, ResidualStream(residuals[cp].residual)
+ )
return published, back, reduced
def test_the_reduce_scatter_completes_the_sum_the_moe_leaves(self):
- published, back, reduced = self.run_ranks(allow_reduce_scatter=True)
+ published, back, reduced = self.run_ranks(use_reduce_scatter=True)
self.assertEqual(published, {0: True, 1: True}, "the MoE leaves its sum")
self.assertEqual(sorted(reduced), [0, 1], "every rank joins the reduce-scatter")
for cp in range(CP_SIZE):
@@ -277,7 +277,7 @@ class TestAttentionCpBoundary(CustomTestCase):
)
def test_a_complete_output_is_only_taken_back(self):
- published, back, reduced = self.run_ranks(allow_reduce_scatter=False)
+ published, back, reduced = self.run_ranks(use_reduce_scatter=False)
self.assertEqual(published, {0: False, 1: False}, "the MoE sums itself")
self.assertEqual(reduced, {}, "nothing is summed again")
for cp in range(CP_SIZE):
@@ -294,7 +294,7 @@ class TestAttentionCpBoundary(CustomTestCase):
0, dict(gather=unused, reduce_scatter=unused), moe_group=group("moe", [0])
):
with self.assertRaises(NotImplementedError):
- self.build(allow_reduce_scatter=True)
+ self.build(use_reduce_scatter=True)
if __name__ == "__main__":
diff --git a/test/registered/unit/layers/test_declared_decoder_boundary.py b/test/registered/unit/layers/test_declared_decoder_boundary.py
index 4a6a95d704..fe81ffd979 100644
--- a/test/registered/unit/layers/test_declared_decoder_boundary.py
+++ b/test/registered/unit/layers/test_declared_decoder_boundary.py
@@ -1,3 +1,7 @@
+from sglang.srt.layers.communicator.adapters import branch
+from sglang.srt.layers.communicator.construction import BatchVariant
+from sglang.test.boundary_fixtures import make_test_stages
+
"""The boundaries of an attention and an FFN, chosen from both sides'
declarations. The FFN runs on the TP group (a dense MLP, or a MoE not
dispatched per DP shard) or on each attention-TP rank's slice of its DP shard's
@@ -5,8 +9,7 @@ rows (a MoE dispatched per DP shard).
The numeric checks run every rank of a DP x attention-TP world as a thread over
fake collectives that wait for all members of their group, build the layers'
-communicators for real, and pass them layer facts that fail on any read of
-something else.
+boundaries through the production factories.
"""
import itertools
@@ -17,32 +20,22 @@ from functools import partial
from types import SimpleNamespace
from unittest.mock import MagicMock, patch
-import msgspec
import torch
from sglang.srt.layers import communicator as comm
from sglang.srt.layers import layernorm_sp
from sglang.srt.layers.communicator import (
- ADD,
- LayerCommunicator,
- LayerFacts,
- MHCLayerCommunicator,
SumGroup,
TokenAxis,
)
from sglang.srt.layers.communicator import boundary as comm_boundary
-from sglang.srt.layers.communicator import (
- decoder_layer_sides,
- input_scattered_layer_sides,
-)
-from sglang.srt.layers.communicator import layer as comm_layer
+from sglang.srt.layers.communicator import construction as comm_layer
+from sglang.srt.layers.communicator import exit as comm_exit
from sglang.srt.layers.communicator import layout as comm_layout
from sglang.srt.layers.communicator import ops as comm_ops
-from sglang.srt.layers.communicator import (
- sequence_parallel_layer_sides,
-)
-from sglang.srt.layers.communicator.residual import batch as residual_batch
+from sglang.srt.layers.communicator.fusions.allreduce import complete_ffn_input
from sglang.srt.layers.communicator.residual import mhc as mhc_module
+from sglang.srt.layers.communicator.residual.access import finish_layer_stack
from sglang.srt.layers.communicator.residual.stream import OwedOutput
from sglang.srt.layers.moe import utils as moe_utils
from sglang.srt.runtime_context import LoRABatchLayout
@@ -83,7 +76,14 @@ def parallel_of(*, attn_dp, attn_tp, attn_cp=1, **overrides):
@contextmanager
-def planning(parallel, *, sp=False, a2a=False, dsa_cp=False):
+def planning(
+ parallel,
+ *,
+ sp=False,
+ a2a=False,
+ dsa_cp=False,
+ boundary_reduction="rs+rsv",
+):
"""What layer planning and communicator construction read, without the
process-wide parallel state. ``parallel`` may be a callable, for a
per-thread parallel state. ``dsa_cp``: the prefill CP is DSA's (MLA's is
@@ -120,7 +120,8 @@ def planning(parallel, *, sp=False, a2a=False, dsa_cp=False):
patch_communicator(
"get_exec",
lambda: SimpleNamespace(
- overlap=SimpleNamespace(enable_two_batch_overlap=False)
+ comm=SimpleNamespace(boundary_reduction=boundary_reduction),
+ overlap=SimpleNamespace(enable_two_batch_overlap=False),
),
),
):
@@ -145,49 +146,15 @@ class FusableNorm(Norm):
raise AssertionError("not run here")
-class FactsOnly:
- """Layer facts that fail on any read of something else: construction reads
- only the layer facts."""
-
- def __init__(self, **facts):
- self.__dict__.update(facts)
-
- def __getattr__(self, name):
- raise AssertionError(f"read {name!r}, which is not a layer fact")
-
-
-def layer_facts(
+def layer_case(
layer_id, num_layers, *, sparse=False, previous_sparse=False, next_sparse=False
):
- return FactsOnly(
- is_layer_sparse=sparse,
- is_first_layer=layer_id == 0,
- is_last_layer=layer_id == num_layers - 1,
- is_previous_layer_sparse=None if layer_id == 0 else previous_sparse,
- is_next_layer_sparse=next_sparse,
- )
-
-
-def sides_of(
- axis_sizes,
- *,
- sparse=False,
- a2a=False,
- previous_a2a=False,
- last=False,
- leaves_next=False,
- leaves_rs=False,
-):
- return decoder_layer_sides(
- axis_sizes=axis_sizes,
- ffn_on_local_rows=sparse and a2a,
- previous_on_local_rows=previous_a2a,
- is_last_layer=last,
- attention_gathers_local_rows=False,
- ffn_group=SumGroup.MOE_OUTPUT if sparse else SumGroup.TP,
- leaves_for_next_layer=leaves_next,
- leaves_for_reduce_scatter=leaves_rs,
- leaves_for_reduce_scatterv=leaves_rs or sparse,
+ return dict(
+ sparse=sparse,
+ first=layer_id == 0,
+ last=layer_id == num_layers - 1,
+ previous_sparse=previous_sparse,
+ next_sparse=next_sparse,
)
@@ -195,48 +162,32 @@ def build(
facts,
parallel,
*,
- cls=LayerCommunicator,
sp=False,
a2a=False,
dsa_cp=False,
**kwargs,
):
- with planning(parallel, sp=sp, a2a=a2a, dsa_cp=dsa_cp):
- return cls(
- layer_facts=facts,
- input_layernorm=Norm(),
- post_attention_layernorm=Norm(),
+ with planning(
+ parallel,
+ sp=sp,
+ a2a=a2a,
+ dsa_cp=dsa_cp,
+ boundary_reduction=kwargs.pop("boundary_reduction", "rs+rsv"),
+ ):
+ return make_test_stages(
+ **facts,
+ attention_norm=Norm(),
+ ffn_norm=Norm(),
**kwargs,
)
-def planned_facts(
- layer_id,
- num_layers,
- *,
- sparse,
- previous_sparse,
- parallel,
- a2a=False,
- dsa_cp=False,
-):
- with planning(parallel, a2a=a2a, dsa_cp=dsa_cp):
- return LayerFacts.init_new(
- layer_id=layer_id,
- num_layers=num_layers,
- is_layer_sparse=sparse,
- is_previous_layer_sparse=previous_sparse,
- is_next_layer_sparse=False,
- )
-
-
class TestWhichLayersUseDeclarations(CustomTestCase):
- """The layer is the unit: all three boundary sides it owns follow the
- declarations, or none do."""
+ """Production factories choose the input and output paths for each stage."""
def declared(self, communicator):
return getattr(
- communicator._steps.ffn.prepare.keywords["step"], "func", None
+ communicator.ffn.plan._steps.ffn.prepare.keywords["step"], "func", None
) in (
comm_ops._mlp_input_dp_partial,
comm_ops._mlp_input_dp_replicate,
@@ -248,49 +199,46 @@ class TestWhichLayersUseDeclarations(CustomTestCase):
parallel = parallel_of(attn_dp=2, attn_tp=2)
for layer_id in range(3):
with self.subTest(layer_id=layer_id):
- communicator = build(layer_facts(layer_id, 3), parallel)
+ communicator = build(layer_case(layer_id, 3), parallel)
self.assertTrue(self.declared(communicator))
- self.assertIsNone(communicator._steps.attention.input_move)
- self.assertTrue(communicator._steps.returns_over_dp)
+ self.assertIsNone(communicator.attn.plan._steps.attention.input_move)
+ self.assertTrue(communicator.ffn.plan._steps.returns_over_dp)
def test_plain_tp(self):
for norm, fuses in ((Norm, False), (FusableNorm, True)):
with self.subTest(norm=norm.__name__):
with planning(parallel_of(attn_dp=1, attn_tp=2)):
- communicator = LayerCommunicator(
- layer_facts=layer_facts(1, 3),
- input_layernorm=norm(),
- post_attention_layernorm=norm(),
+ communicator = make_test_stages(
+ **layer_case(1, 3), attention_norm=norm(), ffn_norm=norm()
)
self.assertIs(
- communicator._steps.ffn.prepare.keywords["step"].func,
+ communicator.ffn.plan._steps.ffn.prepare.keywords["step"].func,
comm_ops._mlp_input_without_dp,
)
- entry = (
- communicator._mlp_input_reduce_output_and_update_and_read_residual
- )
- self.assertEqual(
- communicator._steps.ffn.prepare.keywords["step"].keywords[
- "fusions"
- ],
- (entry,) if fuses else (),
- )
+ entries = communicator.ffn.plan._steps.ffn.prepare.keywords[
+ "step"
+ ].keywords["fusions"]
+ self.assertEqual(len(entries), int(fuses))
+ if fuses:
+ self.assertIs(entries[0].func, complete_ffn_input)
+ self.assertEqual(entries[0].args, (communicator.ffn.plan,))
self.assertIs(
- bool(communicator._steps.ffn.fused),
+ bool(communicator.ffn.plan._steps.ffn.fused),
fuses,
)
- self.assertFalse(communicator._steps.returns_over_dp)
+ self.assertFalse(communicator.ffn.plan._steps.returns_over_dp)
self.assertIs(
- communicator._steps.ffn_output_move,
+ communicator.ffn.plan._steps.ffn_output_move,
comm.CommunicateSummableTensorPairFn._trivial,
)
# One rank: the attention output is complete, only the norm is left.
one_rank = parallel_of(attn_dp=1, attn_tp=1)
- single = build(layer_facts(1, 3), one_rank)
+ single = build(layer_case(1, 3), one_rank)
with planning(one_rank):
- self.assertIsNotNone(single._declared_sides())
+ self.assertIsNotNone(single.attn.plan.variants)
self.assertIs(
- single._steps.ffn.prepare.keywords["step"].func, comm_ops._read_input
+ single.ffn.plan._steps.ffn.prepare.keywords["step"].func,
+ comm_ops._read_input,
)
def test_the_order_follows_the_attention_output(self):
@@ -301,12 +249,13 @@ class TestWhichLayersUseDeclarations(CustomTestCase):
):
with self.subTest(attn_tp=attn_tp, force=force):
communicator = build(
- layer_facts(1, 3),
+ layer_case(1, 3),
parallel_of(attn_dp=2, attn_tp=attn_tp),
force_layernorm_before_dp_gather=force,
)
self.assertIs(
- communicator._steps.ffn.prepare.keywords["step"].func, order
+ communicator.ffn.plan._steps.ffn.prepare.keywords["step"].func,
+ order,
)
def test_a_dense_first_layer_before_sparse_ones(self):
@@ -315,17 +264,9 @@ class TestWhichLayersUseDeclarations(CustomTestCase):
with self.subTest(a2a=a2a):
layers = [
build(
- planned_facts(
- i,
- 4,
- sparse=sparse,
- previous_sparse=previous,
- parallel=parallel,
- a2a=a2a,
- ),
+ layer_case(i, 4, sparse=sparse, previous_sparse=previous),
parallel,
a2a=a2a,
- allow_reduce_scatter=True,
)
for i, (sparse, previous) in enumerate(
((False, False), (True, False), (True, True), (False, True))
@@ -333,7 +274,7 @@ class TestWhichLayersUseDeclarations(CustomTestCase):
]
self.assertEqual([self.declared(layer) for layer in layers], [True] * 4)
if not a2a:
- moe = layers[1]._steps.ffn_output
+ moe = layers[1].ffn.plan._steps.ffn_output
self.assertIs(moe.group, SumGroup.MOE_OUTPUT)
self.assertTrue(moe.leaves_for_reduce_scatterv)
continue
@@ -341,7 +282,7 @@ class TestWhichLayersUseDeclarations(CustomTestCase):
# dense layer; the next one takes it already sliced.
self.assertEqual(
[
- layers[i]._steps.ffn.prepare.keywords["step"].func
+ layers[i].ffn.plan._steps.ffn.prepare.keywords["step"].func
for i in (1, 2)
],
[comm_ops._mlp_input_scatter] * 2,
@@ -349,24 +290,24 @@ class TestWhichLayersUseDeclarations(CustomTestCase):
self.assertEqual(
[
layers[i]
- ._steps.ffn.prepare.keywords["step"]
+ .ffn.plan._steps.ffn.prepare.keywords["step"]
.keywords["scatters_residual"]
for i in (1, 2)
],
[True, False],
)
- self.assertIsNone(layers[1]._steps.ffn_output.group)
+ self.assertIsNone(layers[1].ffn.plan._steps.ffn_output.group)
self.assertIs(
- layers[2]._steps.attention.input_move,
+ layers[2].attn.plan._steps.attention.input_move,
comm.CommunicateSimpleFn._scattered_to_tp_attn_full,
)
self.assertIs(
- layers[3]._steps.ffn.prepare.keywords["step"].func,
+ layers[3].ffn.plan._steps.ffn.prepare.keywords["step"].func,
comm_ops._mlp_input_dp_partial,
)
self.assertTrue(
layers[3]
- ._steps.ffn.prepare.keywords["step"]
+ .ffn.plan._steps.ffn.prepare.keywords["step"]
.keywords["gathers_residual"]
)
@@ -375,7 +316,8 @@ class TestWhichLayersUseDeclarations(CustomTestCase):
# an a2a MoE does, and owes no sum.
parallel = parallel_of(attn_dp=2, attn_tp=2, moe_dense_tp_size=1)
no_overlap = SimpleNamespace(
- overlap=SimpleNamespace(enable_two_batch_overlap=False)
+ comm=SimpleNamespace(boundary_reduction="rs+rsv"),
+ overlap=SimpleNamespace(enable_two_batch_overlap=False),
)
for a2a in (False, True):
with (
@@ -384,17 +326,9 @@ class TestWhichLayersUseDeclarations(CustomTestCase):
):
layers = [
build(
- planned_facts(
- i,
- 4,
- sparse=sparse,
- previous_sparse=previous,
- parallel=parallel,
- a2a=a2a,
- ),
+ layer_case(i, 4, sparse=sparse, previous_sparse=previous),
parallel,
a2a=a2a,
- allow_reduce_scatter=True,
)
for i, (sparse, previous) in enumerate(
((False, False), (False, False), (True, False), (True, True))
@@ -403,84 +337,82 @@ class TestWhichLayersUseDeclarations(CustomTestCase):
self.assertEqual([self.declared(layer) for layer in layers], [True] * 4)
for dense in layers[:2]:
self.assertIs(
- dense._steps.ffn.prepare.keywords["step"].func,
+ dense.ffn.plan._steps.ffn.prepare.keywords["step"].func,
comm_ops._mlp_input_scatter,
)
- self.assertIsNone(dense._steps.ffn_output.group)
+ self.assertIsNone(dense.ffn.plan._steps.ffn_output.group)
for after_dense in layers[1:3]:
self.assertIs(
- after_dense._steps.attention.input_move,
+ after_dense.attn.plan._steps.attention.input_move,
comm.CommunicateSimpleFn._scattered_to_tp_attn_full,
)
def test_the_last_a2a_layer_folds_the_residual_back(self):
parallel = parallel_of(attn_dp=2, attn_tp=2)
last = build(
- planned_facts(
- 3, 4, sparse=True, previous_sparse=True, parallel=parallel, a2a=True
- ),
+ layer_case(3, 4, sparse=True, previous_sparse=True),
parallel,
a2a=True,
)
self.assertIs(
- last._steps.ffn_output_move.func,
+ last.ffn.plan._steps.ffn_output_move.func,
comm.CommunicateSummableTensorPairFn._gather,
)
def test_an_attention_that_gathers_its_slice_itself(self):
parallel = parallel_of(attn_dp=2, attn_tp=2)
- facts = planned_facts(
- 2, 4, sparse=True, previous_sparse=True, parallel=parallel, a2a=True
- )
+ facts = layer_case(2, 4, sparse=True, previous_sparse=True)
with patch_communicator("_use_ag_after_qlora", True):
layer = build(facts, parallel, a2a=True)
- self.assertIsNone(layer._steps.attention.input_move)
+ self.assertIsNone(layer.attn.plan._steps.attention.input_move)
def test_cp_without_prefill_cp(self):
"""No batch shards its tokens, so no CP steps. Under attention DP the
reduce-scatter back runs over the TP group, which spans the CP ranks,
so the FFN completes its own sum; without attention DP it may leave it
and input-scattered attention runs, as without CP."""
- under_dp = build(
- layer_facts(1, 3), parallel_of(attn_dp=2, attn_tp=1, attn_cp=2)
- )
- self.assertIsNone(under_dp._cp_steps)
- self.assertFalse(under_dp._steps.ffn_output.leaves_for_next_layer)
- self.assertFalse(under_dp._steps.ffn_output.leaves_for_reduce_scatterv)
+ under_dp = build(layer_case(1, 3), parallel_of(attn_dp=2, attn_tp=1, attn_cp=2))
+ self.assertIsNone(under_dp.ffn.plan._cp_steps)
+ self.assertFalse(under_dp.ffn.plan._steps.ffn_output.leaves_for_next_layer)
+ self.assertFalse(under_dp.ffn.plan._steps.ffn_output.leaves_for_reduce_scatterv)
without_dp = build(
- layer_facts(1, 3),
+ layer_case(1, 3),
parallel_of(
attn_dp=1, attn_tp=2, attn_cp=2, enable_attn_tp_input_scattered=True
),
)
- self.assertTrue(without_dp._steps.ffn_output.leaves_for_next_layer)
- self.assertIsNotNone(without_dp._input_scattered_steps)
-
- def test_a_layer_without_the_previous_layer_s_facts_is_refused(self):
- direct = LayerFacts()
- with self.assertRaises(NotImplementedError):
- build(direct, parallel_of(attn_dp=2, attn_tp=2))
+ self.assertTrue(without_dp.ffn.plan._steps.ffn_output.leaves_for_next_layer)
+ self.assertIsNotNone(without_dp.ffn.plan._input_scattered_steps)
def build_mhc(
facts, parallel, *, a2a=False, dsa_cp=False, two_batch_overlap=False, **kwargs
):
overlap = SimpleNamespace(
- overlap=SimpleNamespace(enable_two_batch_overlap=two_batch_overlap)
+ comm=SimpleNamespace(
+ boundary_reduction=kwargs.pop("boundary_reduction", "rs+rsv")
+ ),
+ overlap=SimpleNamespace(enable_two_batch_overlap=two_batch_overlap),
)
with (
planning(parallel, a2a=a2a, dsa_cp=dsa_cp),
patch_communicator("get_exec", lambda: overlap),
):
- return MHCLayerCommunicator(
- layer_facts=facts,
- input_layernorm=Norm(),
- post_attention_layernorm=Norm(),
+ mhc = mhc_module.MHCState(
+ is_last_layer=facts["last"],
hc_mult=2,
hc_attn_pre=lambda *a: None,
hc_ffn_pre=lambda *a: None,
**{"hc_post": lambda *a: None, **kwargs},
)
+ stages = make_test_stages(
+ **facts,
+ attention_norm=Norm(),
+ ffn_norm=Norm(),
+ residual=mhc.layer_residual(),
+ )
+ stages.mhc = mhc
+ return stages
class TestMhcOnTheDeclarations(CustomTestCase):
@@ -491,14 +423,14 @@ class TestMhcOnTheDeclarations(CustomTestCase):
if step.func is comm_ops._consumer_step:
step = step.keywords["step"]
self.assertIs(step.func, func)
- residual = communicator._residual
+ declaration = communicator.ffn.declaration
if "read" in step.keywords:
# The FFN input: the attention output's hc_post, then the FFN's read.
- self.assertIs(step.keywords["read"], residual.ffn_read)
+ self.assertIs(step.keywords["read"], declaration.read)
self.assertNotIn("update", step.keywords)
else:
# The FFN output's move: its hc_post.
- self.assertIs(step.keywords["update"], residual.ffn_update)
+ self.assertIs(step.keywords["update"], declaration.update)
for key, value in keywords.items():
self.assertEqual(step.keywords[key], value, key)
@@ -525,25 +457,29 @@ class TestMhcOnTheDeclarations(CustomTestCase):
),
):
with self.subTest(name):
- communicator = build_mhc(layer_facts(1, 3), parallel)
+ communicator = build_mhc(layer_case(1, 3), parallel)
self.assert_step(
- communicator._steps.ffn.prepare, ffn_input, communicator
+ communicator.ffn.plan._steps.ffn.prepare, ffn_input, communicator
)
- self.assertIs(communicator._steps.ffn_output_move, output_move)
+ self.assertIs(communicator.ffn.plan._steps.ffn_output_move, output_move)
# The fused add + RMSNorm kernels do not write hc_post.
- self.assertEqual(communicator._steps.ffn.fused, ())
+ self.assertEqual(communicator.ffn.plan._steps.ffn.fused, ())
self.assertEqual(
- communicator._steps.attention.prepare.keywords["carried_fusions"],
+ communicator.attn.plan._steps.attention.prepare.keywords[
+ "carried_fusions"
+ ],
(),
)
def test_the_move_back_over_dp_stays_with_the_layer(self):
parallel = parallel_of(attn_dp=2, attn_tp=2)
- communicator = build_mhc(layer_facts(1, 3), parallel, allow_reduce_scatter=True)
- self.assertTrue(communicator._steps.returns_over_dp)
- self.assertFalse(communicator._steps.ffn_output.leaves_for_next_layer)
+ communicator = build_mhc(layer_case(1, 3), parallel)
+ self.assertTrue(communicator.ffn.plan._steps.returns_over_dp)
+ self.assertFalse(communicator.ffn.plan._steps.ffn_output.leaves_for_next_layer)
# The move back also writes the FFN output into the streams.
- self.assertFalse(communicator._local_token_move_can_go_to_next_layer(None))
+ self.assertFalse(
+ communicator.ffn.plan.output._local_token_move_can_go_to_next_layer(None)
+ )
max_len = SimpleNamespace(
dp_padding_mode=SimpleNamespace(is_max_len=lambda: True),
forward_mode=SimpleNamespace(is_context_parallel_extend=lambda: False),
@@ -554,9 +490,11 @@ class TestMhcOnTheDeclarations(CustomTestCase):
patch_communicator("should_use_dp_reduce_scatterv", lambda: False),
patch_communicator("can_use_dp_reduce_scatter", lambda: True),
):
- step = communicator._postprocess_dp_step(max_len)
+ step = communicator.ffn.plan.output._postprocess_dp_step(max_len)
self.assertTrue(
- communicator._ffn_leaves_sum_to_reduce_scatter(max_len, step)
+ communicator.ffn.plan.output._ffn_leaves_sum_to_reduce_scatter(
+ max_len, step
+ )
)
def test_the_exit_runs_the_move_back_over_dp_it_chose(self):
@@ -568,9 +506,8 @@ class TestMhcOnTheDeclarations(CustomTestCase):
with self.subTest(name):
hc_post = MagicMock(side_effect=lambda h, r, h_res, h_post: h + r)
communicator = build_mhc(
- layer_facts(1, 3),
+ layer_case(1, 3),
parallel,
- allow_reduce_scatter=True,
hc_post=hc_post,
)
communicator.mhc.h_res = communicator.mhc.h_post = torch.zeros(2)
@@ -580,7 +517,7 @@ class TestMhcOnTheDeclarations(CustomTestCase):
is_context_parallel_extend=lambda: False
),
)
- choose = MagicMock(wraps=comm_ops._reduce_and_redistribute_output_step)
+ choose = MagicMock(wraps=comm_exit._reduce_and_redistribute_output_step)
to_local_tokens = MagicMock(side_effect=lambda step, fb, h: h[:2])
with (
planning(parallel),
@@ -591,12 +528,13 @@ class TestMhcOnTheDeclarations(CustomTestCase):
patch_communicator("_reduce_and_redistribute_output_step", choose),
patch_communicator("_to_local_tokens", to_local_tokens),
):
- stream = residual_batch.start(forward_batch)
- stream.write(torch.full((2, 4), 2.0))
- with communicator.ffn_exit(forward_batch) as ffn_exit:
+ with communicator.ffn.plan.output.ffn_exit(
+ forward_batch
+ ) as ffn_exit:
self.assertTrue(ffn_exit.mlp_reduce_scatter)
- hidden, stream = ffn_exit.finish(torch.ones(4, 4), stream)
- hidden, residual = stream.input(hidden)
+ hidden, residual = ffn_exit.finish(
+ torch.ones(4, 4), torch.full((2, 4), 2.0)
+ )
choose.assert_called_once()
to_local_tokens.assert_called_once()
self.assertIs(to_local_tokens.call_args.args[0], move)
@@ -608,14 +546,7 @@ class TestMhcOnTheDeclarations(CustomTestCase):
parallel = parallel_of(attn_dp=2, attn_tp=2)
layers = [
build_mhc(
- planned_facts(
- i,
- 3,
- sparse=sparse,
- previous_sparse=previous,
- parallel=parallel,
- a2a=True,
- ),
+ layer_case(i, 3, sparse=sparse, previous_sparse=previous),
parallel,
a2a=True,
)
@@ -625,26 +556,27 @@ class TestMhcOnTheDeclarations(CustomTestCase):
]
first_a2a, last = layers[1], layers[2]
self.assert_step(
- first_a2a._steps.ffn.prepare,
+ first_a2a.ffn.plan._steps.ffn.prepare,
comm_ops._mlp_input_scatter,
first_a2a,
scatters_residual=True,
)
self.assert_step(
- last._steps.ffn.prepare,
+ last.ffn.plan._steps.ffn.prepare,
comm_ops._mlp_input_scatter,
last,
scatters_residual=False,
)
self.assert_step(
- last._steps.ffn_output_move,
+ last.ffn.plan._steps.ffn_output_move,
comm.CommunicateSummableTensorPairFn._gather,
last,
)
state = first_a2a.mhc
state.h_res, state.h_post = torch.arange(4.0), torch.arange(4.0) + 10
context = SimpleNamespace(attn_tp_rank=1, attn_tp_size=2)
- residual = state.residual_to_attn_tp_shard(torch.arange(4.0) + 20, context)
+ with patch_communicator("get_parallel", lambda: context):
+ residual = state.residual_to_attn_tp_shard(torch.arange(4.0) + 20)
self.assertEqual(residual.tolist(), [22.0, 23.0])
self.assertEqual(state.h_res.tolist(), [2.0, 3.0])
self.assertEqual(state.h_post.tolist(), [12.0, 13.0])
@@ -655,10 +587,8 @@ class TestMhcOnTheDeclarations(CustomTestCase):
)
for layer_id in range(3):
with self.subTest(layer_id=layer_id):
- communicator = build_mhc(
- layer_facts(layer_id, 3), parallel, allow_reduce_scatter=True
- )
- steps = communicator._input_scattered_steps
+ communicator = build_mhc(layer_case(layer_id, 3), parallel)
+ steps = communicator.ffn.plan._input_scattered_steps
self.assert_step(
steps.ffn.prepare,
comm_ops._mlp_input_on_residual_shard,
@@ -673,22 +603,29 @@ class TestMhcOnTheDeclarations(CustomTestCase):
)
self.assertTrue(steps.ffn_output_move_completes_sum)
completes = build_mhc(
- layer_facts(layer_id, 3), parallel, allow_reduce_scatter=False
- )._input_scattered_steps
+ layer_case(layer_id, 3),
+ parallel,
+ boundary_reduction="ar",
+ ).ffn.plan._input_scattered_steps
self.assertFalse(completes.ffn_output_move.keywords["sums"])
self.assertFalse(completes.ffn_output_move_completes_sum)
# Only the first layer's input, the embedding's partial sum,
# is completed onto the slice.
self.assertIs(
- steps.attention.prepare.keywords["step"].keywords["layer_input"],
+ communicator.attn.plan._input_scattered_steps.attention.prepare.keywords[
+ "step"
+ ].keywords["layer_input"],
comm.tp_reduce_scatter if layer_id == 0 else None,
)
- self.assertIsNone(steps.attention.input_move)
- self.assertIs(
- steps.attention.handoff, comm_ops._hand_scattered_input_to_attention
+ self.assertIsNone(
+ communicator.attn.plan._input_scattered_steps.attention.input_move
)
self.assertIs(
- communicator._steps.ffn_output_move,
+ communicator.attn.plan._input_scattered_steps.attention.handoff,
+ comm_ops._hand_scattered_input_to_attention,
+ )
+ self.assertIs(
+ communicator.ffn.plan._steps.ffn_output_move,
comm.CommunicateSummableTensorPairFn._trivial,
)
@@ -697,13 +634,15 @@ class TestMhcOnTheDeclarations(CustomTestCase):
# rows: MHC writes its output into the streams, then gathers them.
parallel = parallel_of(attn_dp=2, attn_tp=2, moe_dense_tp_size=1)
communicator = build_mhc(
- layer_facts(1, 3, next_sparse=True), parallel, two_batch_overlap=True
+ layer_case(1, 3, next_sparse=True), parallel, two_batch_overlap=True
)
self.assert_step(
- communicator._steps.ffn.prepare, comm_ops._mlp_input_scatter, communicator
+ communicator.ffn.plan._steps.ffn.prepare,
+ comm_ops._mlp_input_scatter,
+ communicator,
)
self.assert_step(
- communicator._steps.ffn_output_move,
+ communicator.ffn.plan._steps.ffn_output_move,
comm.CommunicateSummableTensorPairFn._gather,
communicator,
)
@@ -714,7 +653,7 @@ class TestMhcOnTheDeclarations(CustomTestCase):
for layer_id, contracted in ((1, False), (2, True)):
with self.subTest(layer_id=layer_id):
communicator = build_mhc(
- layer_facts(layer_id, 3),
+ layer_case(layer_id, 3),
parallel,
hc_post=lambda h, r, h_res, h_post: h + r,
)
@@ -728,7 +667,7 @@ class TestMhcOnTheDeclarations(CustomTestCase):
mhc_module, "hc_contract", lambda h, hc_mult: h.sum(-1)
),
):
- hidden, residual = communicator.postprocess_layer(
+ hidden, residual = communicator.ffn.plan.output.postprocess_layer(
torch.ones(2, 4), torch.full((2, 4), 2.0), None
)
self.assertIsNone(residual)
@@ -745,30 +684,24 @@ class TestMhcOnTheDeclarations(CustomTestCase):
parallel = parallel_of(
attn_dp=1, attn_tp=1, attn_cp=2, enable_prefill_cp=True, moe_dense_tp_size=1
)
- facts = planned_facts(
- 1, 3, sparse=True, previous_sparse=False, parallel=parallel, dsa_cp=True
- )
+ facts = layer_case(1, 3, sparse=True, previous_sparse=False)
with self.assertRaises(NotImplementedError):
- build_mhc(facts, parallel, dsa_cp=True, allow_reduce_scatter=True)
+ build_mhc(facts, parallel, dsa_cp=True)
# A dense layer on every rank computes on its own shard: nothing moves.
- facts = planned_facts(
- 1, 3, sparse=False, previous_sparse=False, parallel=parallel, dsa_cp=True
- )
- build_mhc(facts, parallel, dsa_cp=True, allow_reduce_scatter=True)
+ facts = layer_case(1, 3, sparse=False, previous_sparse=False)
+ build_mhc(facts, parallel, dsa_cp=True)
def test_input_scattered_attention_under_attention_cp_is_rejected(self):
scattered = dict(enable_attn_tp_input_scattered=True)
parallel = parallel_of(attn_dp=1, attn_tp=2, attn_cp=2, **scattered)
with self.assertRaisesRegex(NotImplementedError, "input-scattered"):
build_mhc(
- planned_facts(
- 1, 3, sparse=False, previous_sparse=False, parallel=parallel
- ),
+ layer_case(1, 3, sparse=False, previous_sparse=False),
parallel,
)
parallel = parallel_of(attn_dp=1, attn_tp=2, **scattered)
build_mhc(
- planned_facts(1, 3, sparse=False, previous_sparse=False, parallel=parallel),
+ layer_case(1, 3, sparse=False, previous_sparse=False),
parallel,
)
@@ -779,22 +712,20 @@ class TestMhcOnTheDeclarations(CustomTestCase):
parallel = parallel_of(
attn_dp=1, attn_tp=2, attn_cp=2, enable_prefill_cp=prefill_cp
)
- facts = planned_facts(
- 1, 3, sparse=True, previous_sparse=True, parallel=parallel
- )
+ facts = layer_case(1, 3, sparse=True, previous_sparse=True)
with self.assertRaisesRegex(NotImplementedError, "MoE-CP group"):
build_mhc(facts, parallel)
# A dense layer builds, and so does a MoE on its own CP shard (MoE DP = CP).
parallel = parallel_of(attn_dp=1, attn_tp=2, attn_cp=2)
build_mhc(
- planned_facts(1, 3, sparse=False, previous_sparse=False, parallel=parallel),
+ layer_case(1, 3, sparse=False, previous_sparse=False),
parallel,
)
parallel = parallel_of(
attn_dp=1, attn_tp=2, attn_cp=2, moe_dp_size=2, moe_tp_size=2
)
build_mhc(
- planned_facts(1, 3, sparse=True, previous_sparse=True, parallel=parallel),
+ layer_case(1, 3, sparse=True, previous_sparse=True),
parallel,
)
@@ -806,23 +737,22 @@ class TestTwoBatchOverlap(CustomTestCase):
def layers(self, *, tbo):
parallel = parallel_of(attn_dp=2, attn_tp=2, moe_dense_tp_size=1)
- overlap = SimpleNamespace(overlap=SimpleNamespace(enable_two_batch_overlap=tbo))
+ overlap = SimpleNamespace(
+ comm=SimpleNamespace(boundary_reduction="rs+rsv"),
+ overlap=SimpleNamespace(enable_two_batch_overlap=tbo),
+ )
layers = []
with planning(parallel), patch_communicator("get_exec", lambda: overlap):
for i, sparse in enumerate((False, False, True, True)):
- facts = LayerFacts.init_new(
+ facts = layer_case(
layer_id=i,
num_layers=4,
- is_layer_sparse=sparse,
- is_previous_layer_sparse=i > 2,
- is_next_layer_sparse=i >= 1,
+ sparse=sparse,
+ previous_sparse=i > 2,
+ next_sparse=i >= 1,
)
layers.append(
- LayerCommunicator(
- layer_facts=facts,
- input_layernorm=Norm(),
- post_attention_layernorm=Norm(),
- )
+ make_test_stages(**facts, attention_norm=Norm(), ffn_norm=Norm())
)
return layers
@@ -834,17 +764,19 @@ class TestTwoBatchOverlap(CustomTestCase):
with self.subTest(two_batch_overlap=tbo):
first, before, after, last = self.layers(tbo=tbo)
for layer in (first, before, after, last):
- self.assertIsNotNone(layer._declared)
- self.assertIs(first._steps.ffn_output_move, pair._trivial)
+ self.assertIsNotNone(layer.attn.plan.variants)
+ self.assertIs(first.ffn.plan._steps.ffn_output_move, pair._trivial)
if gathered:
- self.assertIs(before._steps.ffn_output_move.func, pair._gather)
- self.assertEqual(after.input_rows, attention)
- self.assertIsNone(after._steps.attention.input_move)
- else:
- self.assertIs(before._steps.ffn_output_move, pair._trivial)
- self.assertEqual(after.input_rows, local)
self.assertIs(
- after._steps.attention.input_move,
+ before.ffn.plan._steps.ffn_output_move.func, pair._gather
+ )
+ self.assertEqual(after.attn.input_rows, attention)
+ self.assertIsNone(after.attn.plan._steps.attention.input_move)
+ else:
+ self.assertIs(before.ffn.plan._steps.ffn_output_move, pair._trivial)
+ self.assertEqual(after.attn.input_rows, local)
+ self.assertIs(
+ after.attn.plan._steps.attention.input_move,
comm.CommunicateSimpleFn._scattered_to_tp_attn_full,
)
@@ -865,18 +797,17 @@ class TestTwoBatchOverlap(CustomTestCase):
def build(self, facts, parallel, *, a2a, two_batch_overlap):
overlap = SimpleNamespace(
- overlap=SimpleNamespace(enable_two_batch_overlap=two_batch_overlap)
+ comm=SimpleNamespace(boundary_reduction="rs+rsv"),
+ overlap=SimpleNamespace(enable_two_batch_overlap=two_batch_overlap),
)
with (
planning(parallel, a2a=a2a),
patch_communicator("get_exec", lambda: overlap),
):
- communicator = LayerCommunicator(
- layer_facts=facts,
- input_layernorm=Norm(),
- post_attention_layernorm=Norm(),
+ communicator = make_test_stages(
+ **facts, attention_norm=Norm(), ffn_norm=Norm()
)
- communicator._publish_lora_layout = True
+ communicator.ffn.plan._publish_lora_layout = True
return communicator
def test_the_rows_are_the_mlp_mode_layout(self):
@@ -916,13 +847,8 @@ class TestTwoBatchOverlap(CustomTestCase):
attn_tp=attn_tp,
moe_dense_tp_size=1 if dense_fully_dp else None,
)
- facts = planned_facts(
- layer_id,
- 4,
- sparse=sparse,
- previous_sparse=previous_sparse,
- parallel=parallel,
- a2a=a2a,
+ facts = layer_case(
+ layer_id, 4, sparse=sparse, previous_sparse=previous_sparse
)
communicator = self.build(
facts,
@@ -930,16 +856,25 @@ class TestTwoBatchOverlap(CustomTestCase):
a2a=a2a,
two_batch_overlap=two_batch_overlap,
)
- steps = communicator._steps
+ steps = communicator.ffn.plan._steps
self.assertEqual(
- steps.ffn.input_rows, communicator._declared.ffn.input.layout
+ steps.ffn.input_rows,
+ communicator.ffn.plan.variants[
+ BatchVariant.ORDINARY
+ ].incoming.need.layout,
)
published = {}
with patch_communicator(
"get_forward",
lambda: SimpleNamespace(set=published.__setitem__),
):
- communicator.publish_mlp_lora_layout(steps)
+ from sglang.srt.layers.communicator.adapters.lora import (
+ publish_ffn,
+ )
+
+ publish_ffn(
+ communicator.ffn.plan._publish_lora_layout, steps.ffn.input_rows
+ )
# LoRA admits attention DP only with attention TP 1, where the
# FFN takes every row unless it runs on each rank's own: a MoE
# dispatched by an a2a backend, or a dense MLP on every rank.
@@ -961,20 +896,12 @@ class TestTheAttentionOutputDecidesItsSum(CustomTestCase):
def test_the_attention_output_owes_the_attention_tp_sum(self):
for attn_tp in (1, 2):
- sides = sides_of(
- {
- TokenAxis.ATTN_DP: 2,
- TokenAxis.ATTN_CP: 1,
- TokenAxis.ATTN_TP_SCATTER: attn_tp,
- }
- )
- owed = attn_tp > 1
- self.assertIs(
- sides.attention.output.group, SumGroup.ATTN_TP if owed else None
- )
- self.assertIs(sides.attention.output.always_leaves, owed)
- self.assertIs(sides.ffn.output.group, SumGroup.TP)
- self.assertFalse(sides.ffn.output.always_leaves)
+ stages = build(layer_case(1, 3), parallel_of(attn_dp=2, attn_tp=attn_tp))
+ produced = stages.attn.plan._steps.ffn_output
+ self.assertIs(produced.group, SumGroup.ATTN_TP if attn_tp > 1 else None)
+ self.assertIs(produced.always_leaves, attn_tp > 1)
+ self.assertIs(stages.ffn.plan._steps.ffn_output.group, SumGroup.TP)
+ self.assertFalse(stages.ffn.plan._steps.ffn_output.always_leaves)
def run_steps(self, produced, *, force=False):
sizes = {
@@ -982,12 +909,11 @@ class TestTheAttentionOutputDecidesItsSum(CustomTestCase):
TokenAxis.ATTN_CP: 1,
TokenAxis.ATTN_TP_SCATTER: 2,
}
- sides = sides_of(sizes)
steps, _, _ = comm_boundary._select_input_steps(
produced,
- residual=sides.input_rows,
- residual_to=sides.ffn_residual_rows,
- need=sides.ffn.input,
+ residual=comm.Layout.sharded_over(TokenAxis.ATTN_DP, axis_sizes=sizes),
+ residual_to=comm.Layout.sharded_over(TokenAxis.ATTN_DP, axis_sizes=sizes),
+ need=comm.StageInput(comm.Layout(frozenset())),
adds_plainly=True,
fusions=(),
force_layernorm_before_gather=force,
@@ -996,7 +922,6 @@ class TestTheAttentionOutputDecidesItsSum(CustomTestCase):
enters_stack=False,
)
reduced = []
- context = SimpleNamespace(attn_tp_size=2, attn_tp_rank=0)
with (
patch_communicator(
"attention_tensor_model_parallel_all_reduce",
@@ -1018,18 +943,11 @@ class TestTheAttentionOutputDecidesItsSum(CustomTestCase):
torch.full((1, HIDDEN), 3.0),
None,
Norm(),
- context,
)
return len(reduced), hidden, residual
def test_a_complete_output_is_not_summed_again(self):
- layout = sides_of(
- {
- TokenAxis.ATTN_DP: 2,
- TokenAxis.ATTN_CP: 1,
- TokenAxis.ATTN_TP_SCATTER: 2,
- }
- ).input_rows
+ layout = comm.Layout(frozenset({TokenAxis.ATTN_DP}))
complete = comm.StageOutput(layout)
for force in (False, True):
with self.subTest(force=force):
@@ -1039,13 +957,7 @@ class TestTheAttentionOutputDecidesItsSum(CustomTestCase):
torch.testing.assert_close(hidden, torch.full((1, HIDDEN), 20.0))
def test_an_owed_output_is_summed_once(self):
- layout = sides_of(
- {
- TokenAxis.ATTN_DP: 2,
- TokenAxis.ATTN_CP: 1,
- TokenAxis.ATTN_TP_SCATTER: 2,
- }
- ).input_rows
+ layout = comm.Layout(frozenset({TokenAxis.ATTN_DP}))
owed = comm.StageOutput(layout, group=SumGroup.ATTN_TP, always_leaves=True)
reductions, hidden, residual = self.run_steps(owed, force=True)
self.assertEqual(reductions, 1)
@@ -1055,26 +967,14 @@ class TestTheAttentionOutputDecidesItsSum(CustomTestCase):
def test_a_sum_left_only_for_some_batches_comes_with_the_value(self):
# Like a mixer exit: the steps take the output as complete, and a value
# that carries the sum is completed before them.
- layout = sides_of(
- {
- TokenAxis.ATTN_DP: 2,
- TokenAxis.ATTN_CP: 1,
- TokenAxis.ATTN_TP_SCATTER: 2,
- }
- ).input_rows
+ layout = comm.Layout(frozenset({TokenAxis.ATTN_DP}))
asked = comm.StageOutput(layout, group=SumGroup.ATTN_TP)
reductions, hidden, residual = self.run_steps(asked)
self.assertEqual(reductions, 0)
torch.testing.assert_close(residual, torch.full((1, HIDDEN), 10.0))
def test_declarations_the_steps_cannot_run_are_rejected(self):
- layout = sides_of(
- {
- TokenAxis.ATTN_DP: 2,
- TokenAxis.ATTN_CP: 1,
- TokenAxis.ATTN_TP_SCATTER: 2,
- }
- ).input_rows
+ layout = comm.Layout(frozenset({TokenAxis.ATTN_DP}))
for produced in (
# Always leaves a sum over no group.
comm.StageOutput(layout, always_leaves=True),
@@ -1100,18 +1000,17 @@ class TestFusedKernelsTakeOnlyTheStepsTheyComplete(CustomTestCase):
)
def select(self, *, attn_dp, fusions):
- sides = sides_of(
- {
- TokenAxis.ATTN_DP: attn_dp,
- TokenAxis.ATTN_CP: 1,
- TokenAxis.ATTN_TP_SCATTER: 2,
- }
- )
+ sizes = {
+ TokenAxis.ATTN_DP: attn_dp,
+ TokenAxis.ATTN_CP: 1,
+ TokenAxis.ATTN_TP_SCATTER: 2,
+ }
+ attention = comm.Layout.sharded_over(TokenAxis.ATTN_DP, axis_sizes=sizes)
steps, fused, _ = comm_boundary._select_input_steps(
- sides.attention.output,
- residual=sides.input_rows,
- residual_to=sides.ffn_residual_rows,
- need=sides.ffn.input,
+ comm.StageOutput(attention, group=SumGroup.ATTN_TP, always_leaves=True),
+ residual=comm.Layout.sharded_over(TokenAxis.ATTN_DP, axis_sizes=sizes),
+ residual_to=comm.Layout.sharded_over(TokenAxis.ATTN_DP, axis_sizes=sizes),
+ need=comm.StageInput(comm.Layout(frozenset())),
adds_plainly=True,
fusions=fusions,
force_layernorm_before_gather=False,
@@ -1142,87 +1041,34 @@ class TestTheSequenceParallelRegion(CustomTestCase):
take the layer's ordinary declared steps."""
SIZES = {TokenAxis.ATTN_DP: 1, TokenAxis.ATTN_CP: 1, TokenAxis.ATTN_TP_SCATTER: 2}
- SP_STEPS = comm.BoundarySteps(
- attention=comm.StageEntry(
- prepare=comm_ops._read_input,
- input_rows=comm.Layout(frozenset({TokenAxis.ATTN_TP_SCATTER})),
- handoff=comm_ops._hand_qkv_hook_its_input,
- ),
- ffn=comm.StageEntry(
- prepare=comm_ops._read_input,
- input_rows=comm.Layout(frozenset({TokenAxis.ATTN_TP_SCATTER})),
- ),
- ffn_output=sequence_parallel_layer_sides(axis_sizes=SIZES).ffn.output,
- ffn_output_move=comm.CommunicateSummableTensorPairFn._trivial,
- ffn_sum_is_movable=False,
- )
-
- def unbound(self, steps):
- """``steps`` with the plain residual's binding taken off the FFN input
- and the attention input, whose input owes nothing in the region."""
- ffn = steps.ffn.prepare.keywords["step"]
- attention = steps.attention.prepare.keywords["step"]
- for step, read in ((ffn, comm.NORM_READ), (attention, comm.NORM_QUANT_READ)):
- self.assertIsNone(step.keywords["layer_input"])
- self.assertIs(step.keywords["read"], read)
- self.assertNotIn("update", step.keywords)
- replace = msgspec.structs.replace
- return replace(
- steps,
- ffn=replace(steps.ffn, prepare=ffn.func),
- attention=replace(steps.attention, prepare=attention.func),
- )
-
- def test_the_region_declarations_choose_local_steps(self):
- for attn_tp in (2, 4):
- with self.subTest(attn_tp=attn_tp):
- sides = sequence_parallel_layer_sides(
- axis_sizes={
- TokenAxis.ATTN_DP: 1,
- TokenAxis.ATTN_CP: 1,
- TokenAxis.ATTN_TP_SCATTER: attn_tp,
- }
- )
- local = frozenset({TokenAxis.ATTN_TP_SCATTER})
- self.assertEqual(sides.input_rows.sharded, local)
- self.assertEqual(sides.attention.input.gathers_itself, local)
- self.assertEqual(sides.ffn.input.gathers_itself, local)
- self.assertIsNone(sides.attention.output.group)
- self.assertIsNone(sides.ffn.output.group)
- steps = self.unbound(comm_boundary._select_boundary_steps(sides))
- self.assertEqual(
- (
- steps.attention.input_move,
- steps.ffn.prepare,
- steps.ffn_output_move,
- ),
- (
- self.SP_STEPS.attention.input_move,
- self.SP_STEPS.ffn.prepare,
- self.SP_STEPS.ffn_output_move,
- ),
- )
def test_a_layer_under_sp_takes_both_sets_of_steps(self):
parallel = parallel_of(attn_dp=1, attn_tp=2)
for layer_id in range(3):
with self.subTest(layer_id=layer_id):
with planning(parallel, sp=True):
- communicator = LayerCommunicator(
- layer_facts=layer_facts(layer_id, 3),
- input_layernorm=Norm(),
- post_attention_layernorm=Norm(),
+ communicator = make_test_stages(
+ **layer_case(layer_id, 3),
+ attention_norm=Norm(),
+ ffn_norm=Norm(),
)
- self.assertEqual(self.unbound(communicator._sp_steps), self.SP_STEPS)
+ for boundary in (communicator.attn, communicator.ffn):
+ steps = boundary.plan._sp_steps
+ entry = steps.attention or steps.ffn
+ self.assertEqual(
+ entry.input_rows.sharded, frozenset({TokenAxis.ATTN_TP_SCATTER})
+ )
+ self.assertIsNone(entry.input_move)
+ self.assertIsNone(steps.ffn_output.group)
self.assertIs(
- communicator._steps.ffn.prepare.keywords["step"].func,
+ communicator.ffn.plan._steps.ffn.prepare.keywords["step"].func,
comm_ops._mlp_input_without_dp,
)
def test_what_the_ffn_exit_reads_comes_from_the_batch_s_steps(self):
# Inside the region the FFN output is complete: no sum to move.
parallel = parallel_of(attn_dp=1, attn_tp=2)
- communicator = build(layer_facts(1, 3), parallel, sp=True)
+ communicator = build(layer_case(1, 3), parallel, sp=True)
for active in (False, True):
with (
self.subTest(sp_active=active),
@@ -1237,33 +1083,16 @@ class TestTheSequenceParallelRegion(CustomTestCase):
patch_communicator("is_dp_attention_enabled", lambda: False),
):
self.assertIs(
- communicator._ffn_sum_can_move_to_next_layer(None), not active
+ communicator.ffn.plan.output._ffn_sum_can_move_to_next_layer(None),
+ not active,
)
def test_without_sp_there_is_no_region(self):
with planning(parallel_of(attn_dp=1, attn_tp=2)):
- communicator = LayerCommunicator(
- layer_facts=layer_facts(1, 3),
- input_layernorm=Norm(),
- post_attention_layernorm=Norm(),
+ communicator = make_test_stages(
+ **layer_case(1, 3), attention_norm=Norm(), ffn_norm=Norm()
)
- self.assertIsNone(communicator._sp_steps)
-
- def test_the_same_selector_chooses_a_layer_s_ordinary_steps(self):
- # Under attention DP the FFN exit chooses the move back per batch.
- kept = comm_boundary._select_boundary_steps(
- sides_of(self.SIZES, leaves_next=True)
- )
- self.assertTrue(kept.ffn_output.leaves_for_next_layer)
- self.assertTrue(kept.ffn_sum_is_movable)
- self.assertIs(
- kept.ffn_output_move, comm.CommunicateSummableTensorPairFn._trivial
- )
- returned = comm_boundary._select_boundary_steps(
- sides_of({**self.SIZES, TokenAxis.ATTN_DP: 2})
- )
- self.assertTrue(returned.returns_over_dp)
- self.assertIsNone(returned.ffn_output_move)
+ self.assertIsNone(communicator.ffn.plan._sp_steps)
class TestInputScatteredAttention(CustomTestCase):
@@ -1277,19 +1106,22 @@ class TestInputScatteredAttention(CustomTestCase):
def test_the_declarations_choose_the_steps(self):
for hands_on in (False, True):
with self.subTest(hands_on_partial=hands_on):
- sides = input_scattered_layer_sides(
- axis_sizes=self.SIZES,
- ffn_group=SumGroup.TP,
- hands_on_partial=hands_on,
+ communicator = build(
+ layer_case(1, 3),
+ boundary_reduction="rs+rsv" if hands_on else "ar",
+ parallel=parallel_of(
+ attn_dp=1, attn_tp=2, enable_attn_tp_input_scattered=True
+ ),
)
- steps = comm_boundary._select_boundary_steps(sides)
+ steps = communicator.ffn.plan._input_scattered_steps
+ attention = communicator.attn.plan._input_scattered_steps.attention
self.assertIs(
- steps.attention.prepare.keywords["step"].keywords["layer_input"],
+ attention.prepare.keywords["step"].keywords["layer_input"],
comm.tp_reduce_scatter,
)
- self.assertIsNone(steps.attention.input_move)
+ self.assertIsNone(attention.input_move)
slice_ = comm.Layout(frozenset({TokenAxis.ATTN_TP_SCATTER}))
- self.assertEqual(steps.attention.input_rows, slice_)
+ self.assertEqual(attention.input_rows, slice_)
self.assertEqual(steps.ffn.input_rows, comm.Layout(frozenset()))
self.assertIs(
steps.ffn.prepare.keywords["step"].func,
@@ -1351,11 +1183,13 @@ class TestInputScatteredAttention(CustomTestCase):
hidden = torch.arange(4.0)[:, None].expand(4, HIDDEN).clone()
residual = torch.ones(4, HIDDEN)
context = SimpleNamespace(attn_tp_size=2, attn_tp_rank=1)
- out, out_residual = step(hidden, residual, None, Norm(), context)
+ with patch_communicator("get_parallel", lambda: context):
+ out, out_residual = step(hidden, residual, None, Norm())
torch.testing.assert_close(out_residual, hidden[2:] + 1)
torch.testing.assert_close(out, 2 * (hidden[2:] + 1))
# At the start of the layer stack the input is the residual.
- out, out_residual = step(hidden, None, None, Norm(), context)
+ with patch_communicator("get_parallel", lambda: context):
+ out, out_residual = step(hidden, None, None, Norm())
torch.testing.assert_close(out_residual, hidden[2:])
torch.testing.assert_close(out, 2 * hidden[2:])
@@ -1374,18 +1208,19 @@ class TestInputScatteredAttention(CustomTestCase):
):
with self.subTest(name):
communicator = build(
- layer_facts(1, 3, sparse=a2a, previous_sparse=a2a),
+ layer_case(1, 3, sparse=a2a, previous_sparse=a2a),
parallel,
a2a=a2a,
- allow_reduce_scatter=True,
)
- self.assertIs(communicator._input_scattered_steps is not None, expected)
+ self.assertIs(
+ communicator.ffn.plan._input_scattered_steps is not None, expected
+ )
def test_a_batch_runs_them_only_while_its_input_is_scattered(self):
parallel = parallel_of(
attn_dp=1, attn_tp=2, enable_attn_tp_input_scattered=True
)
- communicator = build(layer_facts(1, 3), parallel, allow_reduce_scatter=True)
+ communicator = build(layer_case(1, 3), parallel)
for scattered in (False, True):
with (
self.subTest(input_scattered=scattered),
@@ -1398,10 +1233,10 @@ class TestInputScatteredAttention(CustomTestCase):
),
):
self.assertIs(
- communicator._batch_steps(None),
- communicator._input_scattered_steps
+ communicator.ffn.plan._batch_steps(None),
+ communicator.ffn.plan._input_scattered_steps
if scattered
- else communicator._steps,
+ else communicator.ffn.plan._steps,
)
def test_prepare_attn_completes_the_scattered_input(self):
@@ -1415,7 +1250,7 @@ class TestInputScatteredAttention(CustomTestCase):
parallel = parallel_of(
attn_dp=1, attn_tp=2, enable_attn_tp_input_scattered=True, tp_group=group
)
- communicator = build(layer_facts(1, 3), parallel, allow_reduce_scatter=True)
+ communicator = build(layer_case(1, 3), parallel)
with (
patch_communicator("get_parallel", lambda: parallel),
patch_communicator(
@@ -1424,13 +1259,9 @@ class TestInputScatteredAttention(CustomTestCase):
),
patch_communicator("get_forward", lambda: SimpleNamespace(sp_active=False)),
):
- batch = SimpleNamespace(residual_stream=None)
- stream = residual_batch.start(batch)
- stream.write(torch.full((4, HIDDEN), 3.0))
- contribution = stream.leave(
- torch.ones(4, HIDDEN), ADD, declared_sum=SumGroup.TP
+ hidden, residual = communicator.attn._prepare_input(
+ torch.ones(4, HIDDEN), torch.full((4, HIDDEN), 3.0), None
)
- hidden, residual = communicator.prepare_attn(contribution, stream, batch)
self.assertEqual(scattered, [4])
# Norm: (2 * (h + r), h + r) on the slice, with h the completed sum.
torch.testing.assert_close(residual.residual, torch.full((2, HIDDEN), 5.0))
@@ -1445,36 +1276,18 @@ class TestInputScatteredAttention(CustomTestCase):
(2, True, False),
(1, False, False),
):
- with self.subTest(layer_id=layer_id, allow_reduce_scatter=allow):
+ with self.subTest(
+ layer_id=layer_id, boundary_reduction="rs+rsv" if allow else "ar"
+ ):
communicator = build(
- layer_facts(layer_id, 3), parallel, allow_reduce_scatter=allow
+ layer_case(layer_id, 3),
+ parallel,
+ boundary_reduction="rs+rsv" if allow else "ar",
)
- steps = communicator._input_scattered_steps
+ steps = communicator.ffn.plan._input_scattered_steps
self.assertIs(steps.ffn_output.leaves_for_reduce_scatter, hands_on)
-class TestOneRepresentation(CustomTestCase):
- """Every layer runs from one BoundarySteps per batch, whichever entry chose
- its steps: none keeps the steps as separate attributes."""
-
- SEPARATE = (
- "_communicate_simple_fn",
- "_mlp_input",
- "_communicate_summable_tensor_pair_fn",
- "_ffn_output",
- "_postprocess_scatters_to_local_tokens",
- "_ffn_sum_is_movable",
- )
-
- def test_the_layer_holds_steps_only(self):
- communicator = build(layer_facts(1, 3), parallel_of(attn_dp=2, attn_tp=2))
- self.assertIsInstance(communicator._steps, comm.BoundarySteps)
- for attribute in self.SEPARATE:
- self.assertFalse(hasattr(communicator, attribute), attribute)
- self.assertIs(communicator._batch_steps(None), communicator._steps)
- self.assertTrue(communicator._steps.returns_over_dp)
-
-
class TestTheAttentionInputHalf(CustomTestCase):
"""Every batch variant's half into the attention tries the fused entries
the layer chose, and starts the residual only on the stack's first
@@ -1482,10 +1295,8 @@ class TestTheAttentionInputHalf(CustomTestCase):
def build_fusable(self, facts, parallel, **planning_kwargs):
with planning(parallel, **planning_kwargs):
- return LayerCommunicator(
- layer_facts=facts,
- input_layernorm=FusableNorm(),
- post_attention_layernorm=Norm(),
+ return make_test_stages(
+ **facts, attention_norm=FusableNorm(), ffn_norm=Norm()
)
def test_every_variant_takes_the_layers_fused_entries(self):
@@ -1501,23 +1312,17 @@ class TestTheAttentionInputHalf(CustomTestCase):
):
for layer_id in (0, 1):
with self.subTest(name, layer_id=layer_id):
- facts = planned_facts(
- layer_id,
- 3,
- sparse=False,
- previous_sparse=False,
- parallel=parallel,
- )
+ facts = layer_case(layer_id, 3, sparse=False, previous_sparse=False)
communicator = self.build_fusable(facts, parallel, **kwargs)
- fusions = communicator._attn_input_fusions
+ fusions = communicator.attn.plan._carried_fusions
self.assertTrue(fusions)
variants = [
steps
for steps in (
- communicator._steps,
- communicator._sp_steps,
- communicator._input_scattered_steps,
- communicator._cp_steps,
+ communicator.attn.plan._steps,
+ communicator.attn.plan._sp_steps,
+ communicator.attn.plan._input_scattered_steps,
+ communicator.attn.plan._cp_steps,
)
if steps is not None
]
@@ -1553,11 +1358,9 @@ class TestPrefillCP(CustomTestCase):
def test_a_dsa_cp_extend_leaves_its_sum_to_the_reduce_scatter(self):
# A MoE on the TP group, as under DSA interleave CP without a2a.
parallel = self.dsa_parallel()
- facts = planned_facts(
- 1, 3, sparse=True, previous_sparse=False, parallel=parallel, dsa_cp=True
- )
- communicator = build(facts, parallel, dsa_cp=True, allow_reduce_scatter=True)
- cp, ordinary = communicator._cp_steps, communicator._steps
+ facts = layer_case(1, 3, sparse=True, previous_sparse=False)
+ communicator = build(facts, parallel, dsa_cp=True)
+ cp, ordinary = communicator.ffn.plan._cp_steps, communicator.ffn.plan._steps
self.assertIs(
cp.ffn.prepare.keywords["step"].func,
comm_ops._mlp_input_gather_attention_cp,
@@ -1573,7 +1376,7 @@ class TestPrefillCP(CustomTestCase):
self.assertTrue(cp.ffn_output.leaves_for_reduce_scatter)
self.assertFalse(cp.ffn_output.leaves_for_next_layer)
# The dense layer before it ran on the same shard: nothing to move.
- self.assertIsNone(cp.attention.input_move)
+ self.assertIsNone(communicator.attn.plan._cp_steps.attention.input_move)
# Other batches hold every token on each CP rank: the MoE sums itself.
self.assertIs(ordinary.ffn.prepare.keywords["step"].func, comm_ops._read_input)
self.assertIs(
@@ -1592,7 +1395,7 @@ class TestPrefillCP(CustomTestCase):
patch_communicator("_batch_shards_over_cp", lambda fb: shards),
):
self.assertIs(
- communicator._ffn_leaves_sum_to_reduce_scatter(
+ communicator.ffn.plan.output._ffn_leaves_sum_to_reduce_scatter(
SimpleNamespace(
forward_mode=SimpleNamespace(
is_context_parallel_extend=lambda: False
@@ -1628,11 +1431,9 @@ class TestPrefillCP(CustomTestCase):
def test_dsa_cp_dense_layers_run_on_their_shard(self):
parallel = self.dsa_parallel()
- facts = planned_facts(
- 1, 3, sparse=False, previous_sparse=False, parallel=parallel, dsa_cp=True
- )
- communicator = build(facts, parallel, dsa_cp=True, allow_reduce_scatter=True)
- for steps in (communicator._steps, communicator._cp_steps):
+ facts = layer_case(1, 3, sparse=False, previous_sparse=False)
+ communicator = build(facts, parallel, dsa_cp=True)
+ for steps in (communicator.ffn.plan._steps, communicator.ffn.plan._cp_steps):
self.assertIs(steps.ffn.prepare.keywords["step"].func, comm_ops._read_input)
self.assertIsNone(steps.ffn_output.group)
self.assertIs(
@@ -1640,8 +1441,8 @@ class TestPrefillCP(CustomTestCase):
)
def test_a_cp_extend_gathers_over_cp_and_takes_its_chunk_back(self):
- communicator = build(layer_facts(1, 3), self.cp_parallel())
- cp = communicator._cp_steps
+ communicator = build(layer_case(1, 3), self.cp_parallel())
+ cp = communicator.ffn.plan._cp_steps
self.assertIs(
cp.ffn.prepare.keywords["step"].func, comm_ops._mlp_input_gather_moe_cp
)
@@ -1658,11 +1459,11 @@ class TestPrefillCP(CustomTestCase):
self.assertFalse(cp.ffn_output.leaves_for_next_layer)
self.assertFalse(cp.ffn_output.leaves_for_reduce_scatter)
self.assertIs(
- communicator._steps.ffn.prepare.keywords["step"].func,
+ communicator.ffn.plan._steps.ffn.prepare.keywords["step"].func,
comm_ops._mlp_input_without_dp,
)
self.assertIs(
- communicator._steps.ffn_output_move,
+ communicator.ffn.plan._steps.ffn_output_move,
comm.CommunicateSummableTensorPairFn._trivial,
)
@@ -1670,11 +1471,10 @@ class TestPrefillCP(CustomTestCase):
# MoE DP equal to CP: each CP rank's MoE runs on its own TP ranks.
parallel = self.cp_parallel(moe_dp_size=2, moe_tp_size=2)
communicator = build(
- layer_facts(1, 3, sparse=True, previous_sparse=True),
+ layer_case(1, 3, sparse=True, previous_sparse=True),
parallel,
- allow_deferred_ffn_reduction=True,
)
- for steps in (communicator._cp_steps, communicator._steps):
+ for steps in (communicator.ffn.plan._cp_steps, communicator.ffn.plan._steps):
# The attention's rows are the MoE's: nothing to gather or take back.
self.assertIs(
steps.ffn.prepare.keywords["step"].func, comm_ops._mlp_input_without_dp
@@ -1683,45 +1483,34 @@ class TestPrefillCP(CustomTestCase):
steps.ffn_output_move, comm.CommunicateSummableTensorPairFn._trivial
)
self.assertTrue(steps.ffn_output.leaves_for_next_layer)
- dense = build(layer_facts(1, 3), parallel, allow_deferred_ffn_reduction=True)
+ dense = build(layer_case(1, 3), parallel)
self.assertIs(
- dense._cp_steps.ffn.prepare.keywords["step"].func,
+ dense.ffn.plan._cp_steps.ffn.prepare.keywords["step"].func,
comm_ops._mlp_input_gather_moe_cp,
)
- self.assertFalse(dense._cp_steps.ffn_output.leaves_for_next_layer)
+ self.assertFalse(dense.ffn.plan._cp_steps.ffn_output.leaves_for_next_layer)
def test_the_fused_kernels_run_on_each_chunk(self):
with planning(self.cp_parallel()):
- communicator = LayerCommunicator(
- layer_facts=layer_facts(1, 3),
- input_layernorm=FusableNorm(),
- post_attention_layernorm=FusableNorm(),
+ communicator = make_test_stages(
+ **layer_case(1, 3), attention_norm=FusableNorm(), ffn_norm=FusableNorm()
)
- chunk = communicator._cp_steps.ffn.prepare.keywords["step"].keywords["gather"]
- self.assertEqual(
- chunk.keywords["fusions"],
- (communicator._mlp_input_reduce_output_and_update_and_read_residual,),
- )
+ chunk = communicator.ffn.plan._cp_steps.ffn.prepare.keywords["step"].keywords[
+ "gather"
+ ]
+ self.assertEqual(len(chunk.keywords["fusions"]), 1)
+ self.assertIs(chunk.keywords["fusions"][0].func, complete_ffn_input)
+ self.assertEqual(chunk.keywords["fusions"][0].args, (communicator.ffn.plan,))
def cp_outcome(self, parallel, *, sparse=True, a2a=False, dsa_cp=False):
"""A layer under attention CP: "cp steps" for the batches that shard
their tokens, "ordinary" when no batch does, or "refused"."""
- facts = planned_facts(
- 1,
- 3,
- sparse=sparse,
- previous_sparse=sparse,
- parallel=parallel,
- a2a=a2a,
- dsa_cp=dsa_cp,
- )
+ facts = layer_case(1, 3, sparse=sparse, previous_sparse=sparse)
try:
- layer = build(
- facts, parallel, a2a=a2a, dsa_cp=dsa_cp, allow_reduce_scatter=True
- )
+ layer = build(facts, parallel, a2a=a2a, dsa_cp=dsa_cp)
except NotImplementedError:
return "refused"
- return "ordinary" if layer._cp_steps is None else "cp steps"
+ return "ordinary" if layer.ffn.plan._cp_steps is None else "cp steps"
def test_which_cp_the_steps_cover(self):
dp_cp = dict(attn_dp=2, attn_tp=1, attn_cp=2, enable_prefill_cp=True)
@@ -1797,8 +1586,8 @@ class TestPrefillCP(CustomTestCase):
def test_under_attention_dp_one_dp_sum_gathers_both_axes(self):
parallel = parallel_of(attn_dp=2, attn_tp=2, attn_cp=2, enable_prefill_cp=True)
- communicator = build(layer_facts(1, 3), parallel)
- cp, ordinary = communicator._cp_steps, communicator._steps
+ communicator = build(layer_case(1, 3), parallel)
+ cp, ordinary = communicator.ffn.plan._cp_steps, communicator.ffn.plan._steps
self.assertIs(
cp.ffn.prepare.keywords["step"].func, comm_ops._mlp_input_dp_partial
)
@@ -1819,7 +1608,7 @@ class TestPrefillCP(CustomTestCase):
self.assertFalse(steps.ffn_output.leaves_for_reduce_scatterv)
def test_a_batch_runs_them_only_on_a_cp_extend(self):
- communicator = build(layer_facts(1, 3), self.cp_parallel())
+ communicator = build(layer_case(1, 3), self.cp_parallel())
def batch(cp_extend):
return SimpleNamespace(
@@ -1832,12 +1621,12 @@ class TestPrefillCP(CustomTestCase):
raise AssertionError("read for a batch that is not a CP extend")
for fb, rows, expected in (
- (batch(False), unread, communicator._steps),
- (batch(True), lambda fb: None, communicator._steps),
- (batch(True), lambda fb: [2, 1], communicator._cp_steps),
+ (batch(False), unread, communicator.ffn.plan._steps),
+ (batch(True), lambda fb: None, communicator.ffn.plan._steps),
+ (batch(True), lambda fb: [2, 1], communicator.ffn.plan._cp_steps),
):
with (
- self.subTest(expected=expected is communicator._cp_steps),
+ self.subTest(expected=expected is communicator.ffn.plan._cp_steps),
planning(self.cp_parallel()),
patch_communicator("moe_cp_gathered_rows", rows),
patch_communicator(
@@ -1848,7 +1637,7 @@ class TestPrefillCP(CustomTestCase):
lambda: SimpleNamespace(input_scattered=False),
),
):
- self.assertIs(communicator._batch_steps(fb), expected)
+ self.assertIs(communicator.ffn.plan._batch_steps(fb), expected)
class TestBranchRows(CustomTestCase):
@@ -1874,11 +1663,11 @@ class TestBranchRows(CustomTestCase):
),
patch_communicator("moe_cp_gathered_rows", lambda fb: [2, 1]),
):
- return communicator._branch_rows(batch)
+ return communicator.ffn.plan._branch_rows(batch)
def test_each_branch_declares_its_rows(self):
parallel = parallel_of(attn_dp=2, attn_tp=2)
- dense = build(layer_facts(2, 6), parallel)
+ dense = build(layer_case(2, 6), parallel)
self.assertEqual(
self.rows(dense, parallel), (self.full, self.attention, self.attention)
)
@@ -1888,7 +1677,7 @@ class TestBranchRows(CustomTestCase):
):
with self.subTest(a2a=a2a):
moe = build(
- layer_facts(1, 3, sparse=True, previous_sparse=True),
+ layer_case(1, 3, sparse=True, previous_sparse=True),
parallel,
a2a=a2a,
)
@@ -1896,16 +1685,16 @@ class TestBranchRows(CustomTestCase):
def test_a_batch_on_other_steps_has_no_branch_rows(self):
parallel = parallel_of(attn_dp=1, attn_tp=2, attn_cp=2, enable_prefill_cp=True)
- communicator = build(layer_facts(1, 3), parallel)
+ communicator = build(layer_case(1, 3), parallel)
with self.assertRaises(NotImplementedError):
self.rows(communicator, parallel, cp_extend=True)
def test_branches_move_between_the_declared_rows(self):
# An a2a MoE beside a dense FFN, under attention DP and TP.
class Communicator(SimpleNamespace):
- branch_input = LayerCommunicator.branch_input
- branch_output = LayerCommunicator.branch_output
- merge_branch = LayerCommunicator.merge_branch
+ branch_input = branch.branch_input
+ branch_output = branch.branch_output
+ merge_branch = branch.merge_branch
moe = Communicator(_branch_rows=lambda fb: (self.local, self.local, self.local))
dense = Communicator(
@@ -1918,9 +1707,7 @@ class TestBranchRows(CustomTestCase):
return value
with patch_communicator("move_rows", recorded):
- dense.branch_input(
- moe, "h0", "residual", SimpleNamespace(residual_stream=None)
- )
+ dense.branch_input(moe, "h0", "residual", None)
self.assertEqual(
moves,
[
@@ -2121,7 +1908,7 @@ def state():
@contextmanager
-def running(*, reduce_scatterv, a2a=False):
+def running(*, reduce_scatterv, a2a=False, use_reduce_scatter=True):
replaced = {
"get_forward": lambda: state().flags,
"attention_tensor_model_parallel_all_reduce": lambda x: (
@@ -2152,7 +1939,10 @@ def running(*, reduce_scatterv, a2a=False):
"post_experts_reduction_group": lambda: state().parallel.tp_group,
"get_lora": lambda: SimpleNamespace(enable_lora=False),
"get_exec": lambda: SimpleNamespace(
- comm=SimpleNamespace(enable_quant_communications=False)
+ comm=SimpleNamespace(
+ enable_quant_communications=False,
+ boundary_reduction="rs+rsv" if use_reduce_scatter else "ar",
+ )
),
"get_attn_tp_context": lambda: SimpleNamespace(
input_scattered=False, set_attn_inputs=lambda x: None
@@ -2201,14 +1991,9 @@ def dense_mlp(x, state):
def moe(x, state):
"""A MoE block not dispatched per DP shard: 5 * x, reduced over the MoE
- output's group unless a published flag asks it to leave the sum, or the
- attention-DP reduce_scatterv takes it, as the MoE blocks do."""
+ output's group unless a published flag asks it to leave the sum."""
partial_sum = 5 * x * WEIGHTS[state.parallel.tp_size][state.parallel.tp_rank]
- if (
- state.flags.fuse_mlp_allreduce
- or state.flags.mlp_reduce_scatter
- or comm_ops.should_use_dp_reduce_scatterv()
- ):
+ if state.flags.fuse_mlp_allreduce or state.flags.mlp_reduce_scatter:
return partial_sum
return comm_layout.post_experts_reduction_group().all_reduce(partial_sum)
@@ -2234,7 +2019,7 @@ class TestTwoLayers(CustomTestCase):
rows,
padding,
reduce_scatterv,
- leaves,
+ use_reduce_scatter,
sparse=(False, False),
a2a=False,
):
@@ -2301,29 +2086,29 @@ class TestTwoLayers(CustomTestCase):
batch = SimpleNamespace(**vars(forward_batch), residual_stream=None)
s = states[rank]
layers = [
- LayerCommunicator(
- layer_facts=layer_facts(
- i,
- 2,
- sparse=sparse[i],
- previous_sparse=i > 0 and sparse[i - 1],
+ make_test_stages(
+ **layer_case(
+ i, 2, sparse=sparse[i], previous_sparse=i > 0 and sparse[i - 1]
),
- input_layernorm=Norm(),
- post_attention_layernorm=Norm(),
- allow_reduce_scatter=leaves,
- allow_deferred_ffn_reduction=leaves,
+ attention_norm=Norm(),
+ ffn_norm=Norm(),
)
for i in range(2)
]
hidden = torch.zeros(s.local_rows, HIDDEN).double()
hidden[: s.rows] = embeddings[s.dp]
- residual = residual_batch.start(batch)
+ residual = None
handed_on = []
for layer_index, layer in enumerate(layers):
- hidden, residual = layer.prepare_attn(hidden, residual, batch)
+ hidden, residual = layer.attn._prepare_input(hidden, residual, batch)
hidden = attention(hidden, s)
- hidden, residual = layer.prepare_mlp(hidden, residual, batch)
- with layer.ffn_exit(batch) as ffn_exit:
+ hidden = residual.leave(
+ hidden,
+ comm.ADD,
+ declared_sum=layer.ffn.plan._batch_steps(batch).ffn.input_sum,
+ )
+ hidden, residual = layer.ffn._prepare_input(hidden, residual, batch)
+ with layer.ffn.plan.output.ffn_exit(batch) as ffn_exit:
if not sparse[layer_index]:
hidden = dense_mlp(hidden, s)
elif a2a:
@@ -2333,13 +2118,18 @@ class TestTwoLayers(CustomTestCase):
hidden = moe(hidden, s)
hidden, residual = ffn_exit.finish(hidden, residual)
handed_on.append(type(hidden))
- hidden, residual = layers[-1].finish_layer_stack(hidden, residual, batch)
+ batch.residual_stream = residual
+ hidden, residual = finish_layer_stack(hidden, residual, batch)
# The last a2a layer folds the residual into its output.
if residual is not None:
residual = residual[: s.rows]
return hidden[: s.rows], residual, handed_on
- with running(reduce_scatterv=reduce_scatterv, a2a=a2a):
+ with running(
+ reduce_scatterv=reduce_scatterv,
+ a2a=a2a,
+ use_reduce_scatter=use_reduce_scatter,
+ ):
results, errors = world.run(states, forward)
for rank, error in enumerate(errors):
if error is not None:
@@ -2357,7 +2147,10 @@ class TestTwoLayers(CustomTestCase):
return results
def test_rows_come_back_to_each_rank(self):
- for (attn_dp, attn_tp), rows, padding, leaves, sparse in itertools.product(
+ for (
+ attn_dp,
+ attn_tp,
+ ), rows, padding, use_reduce_scatter, sparse in itertools.product(
((2, 2), (4, 1), (2, 1), (1, 2), (1, 4)),
("ragged", "idle"),
("sum_len", "max_len"),
@@ -2383,7 +2176,7 @@ class TestTwoLayers(CustomTestCase):
attn_tp=attn_tp,
rows=token_rows,
padding=padding,
- leaves=leaves,
+ use_reduce_scatter=use_reduce_scatter,
reduce_scatterv=reduce_scatterv,
sparse=sparse,
a2a=a2a,
@@ -2394,25 +2187,26 @@ class TestTwoLayers(CustomTestCase):
rows=token_rows,
padding=padding,
reduce_scatterv=reduce_scatterv,
- leaves=leaves,
+ use_reduce_scatter=use_reduce_scatter,
sparse=sparse,
a2a=a2a,
)
first_layer_hands_on = results[0][2][0]
- # A producer that may leave its sum leaves it for the next
- # layer's input; one that may not hands on a complete value.
+ # Disabling optional RS still permits an ordinary AR at
+ # the next boundary. The actual owed work travels with the value.
# Without attention DP an empty batch completes its own sum.
# A MoE dispatched per DP shard hands on a complete value.
# Otherwise the reduce-scatter back to this rank's tokens
# can be left to the next layer; the all-reduce only
# without an a2a backend.
has_tokens = attn_dp > 1 or sum(token_rows) > 0
- reduce_scatter_left = attn_dp > 1 and (
- padding == "max_len" or reduce_scatterv
+ reduce_scatter_left = (
+ use_reduce_scatter
+ and attn_dp > 1
+ and (padding == "max_len" or reduce_scatterv)
)
owes = (
- leaves
- and has_tokens
+ has_tokens
and not (sparse[0] and a2a)
and (reduce_scatter_left or not a2a)
)
@@ -2427,7 +2221,11 @@ class TestTheFfnInputReduction(CustomTestCase):
for a plain residual; MHC sums its streams in full precision."""
def test_only_a_plain_residual_reduces_quantized(self):
- exec_ = SimpleNamespace(comm=SimpleNamespace(enable_quant_communications=True))
+ exec_ = SimpleNamespace(
+ comm=SimpleNamespace(
+ enable_quant_communications=True, boundary_reduction="rs+rsv"
+ )
+ )
batch = SimpleNamespace(
forward_mode=SimpleNamespace(is_decode_or_idle=lambda: False)
)
@@ -2454,7 +2252,6 @@ class TestTheFfnInputReduction(CustomTestCase):
torch.ones(2, HIDDEN),
batch,
None,
- SimpleNamespace(cache=None),
gathers_residual=False,
fusions=(),
read=read,
@@ -2463,83 +2260,5 @@ class TestTheFfnInputReduction(CustomTestCase):
self.assertEqual(calls, [reduction])
-class TestALayerThatIsOneStage(CustomTestCase):
- """A layer that is one stage reads its input and writes its output into the
- residual as its stage declares."""
-
- SIZES = {TokenAxis.ATTN_DP: 1, TokenAxis.ATTN_CP: 1, TokenAxis.ATTN_TP_SCATTER: 2}
-
- class ProbeRead:
- norms_plainly = True
-
- def __init__(self):
- self.reads = 0
-
- def read(self, residual, norm, quant_format="", post_residual_addition=None):
- self.reads += 1
- return residual + 100, residual
-
- def update_and_read(
- self,
- update,
- hidden_states,
- residual,
- norm,
- quant_format="",
- post_residual_addition=None,
- ):
- return self.read(update.update(hidden_states, residual), norm)
-
- def _stage(self, read, update):
- rows = comm.Layout.sharded_over(axis_sizes=self.SIZES)
- return comm.LayerStage(
- kind=comm.StageKind.FFN,
- edges=comm.stage_edges(
- previous=None,
- stage=comm.StageDecl(
- comm.StageInput(rows, read=read),
- comm.StageOutput(rows, update=update),
- ),
- rows=rows,
- ),
- )
-
- def test_its_steps_run_the_declared_read_and_update(self):
- read, update = (
- self.ProbeRead(),
- SimpleNamespace(adds_plainly=True, can_defer_across_layers=True),
- )
- communicator = build(
- layer_facts(1, 3),
- parallel_of(attn_dp=1, attn_tp=2),
- stage=self._stage(read, update),
- )
- into, out_of = communicator.stage_edges
- self.assertIs(into.need.read, read)
- self.assertIs(out_of.produced.update, update)
- hidden, residual = torch.ones(2, HIDDEN), torch.full((2, HIDDEN), 3.0)
- out, out_residual = communicator._steps.ffn.prepare(
- hidden, residual, None, Norm(), None
- )
- self.assertEqual(read.reads, 1)
- torch.testing.assert_close(out_residual, torch.full((2, HIDDEN), 4.0))
- torch.testing.assert_close(out, torch.full((2, HIDDEN), 104.0))
-
- def test_it_takes_no_residual_of_its_own(self):
- own = comm.LayerResidual(
- attention_read=self.ProbeRead(),
- attention_update=comm.ADD,
- ffn_read=comm.NORM_READ,
- ffn_update=comm.ADD,
- )
- with self.assertRaises(ValueError):
- build(
- layer_facts(1, 3),
- parallel_of(attn_dp=1, attn_tp=2),
- stage=self._stage(comm.NORM_READ, comm.ADD),
- residual=own,
- )
-
-
if __name__ == "__main__":
unittest.main()
diff --git a/test/registered/unit/layers/test_dense_mlp_under_prefill_cp.py b/test/registered/unit/layers/test_dense_mlp_under_prefill_cp.py
index 8fd8a596a2..2bd60b96fb 100644
--- a/test/registered/unit/layers/test_dense_mlp_under_prefill_cp.py
+++ b/test/registered/unit/layers/test_dense_mlp_under_prefill_cp.py
@@ -1,16 +1,13 @@
import unittest
+import test_declared_decoder_boundary as fixture
+
from sglang.srt.layers import communicator as comm
from sglang.srt.layers.communicator import (
- SumGroup,
TokenAxis,
- decoder_layer_edges,
- decoder_layer_sides,
- make_boundary,
- make_output_boundary,
)
from sglang.srt.layers.communicator import ops as comm_ops
-from sglang.srt.layers.communicator.boundary import CpMoves
+from sglang.test.boundary_fixtures import make_test_stages
from sglang.test.ci.ci_register import register_cpu_ci
from sglang.test.test_utils import CustomTestCase
@@ -23,33 +20,24 @@ class TestDenseMlpUnderPrefillCP(CustomTestCase):
(issue #38019: Qwen3-32B emitted garbage that never reached EOS)."""
def test_dense_mlp_gathers_across_cp(self):
- sides = decoder_layer_sides(
- axis_sizes={
- TokenAxis.ATTN_DP: 1,
- TokenAxis.ATTN_CP: 2,
- TokenAxis.ATTN_TP_SCATTER: 2,
- },
- ffn_on_local_rows=False,
- previous_on_local_rows=False,
- is_last_layer=False,
- attention_gathers_local_rows=False,
- ffn_group=SumGroup.TP,
- leaves_for_next_layer=False,
- leaves_for_reduce_scatter=False,
- leaves_for_reduce_scatterv=False,
+ parallel = fixture.parallel_of(
+ attn_dp=1, attn_tp=2, attn_cp=2, enable_prefill_cp=True
)
- # The TP group spans every CP rank, so the MLP takes every row.
- self.assertIn(TokenAxis.ATTN_CP, sides.attention.input.layout.sharded)
- self.assertNotIn(TokenAxis.ATTN_CP, sides.ffn.input.layout.sharded)
- moves = CpMoves(
- gather=comm_ops._mlp_input_gather_moe_cp,
- take_back=comm.CommunicateSummableTensorPairFn._scatter_hidden_states_moe,
+ with fixture.planning(parallel):
+ stages = make_test_stages(
+ attention_norm=fixture.Norm(), ffn_norm=fixture.Norm()
+ )
+ attention = stages.attn.plan._cp_steps.attention
+ steps = stages.ffn.plan._cp_steps
+ self.assertIn(TokenAxis.ATTN_CP, attention.input_rows.sharded)
+ self.assertNotIn(TokenAxis.ATTN_CP, steps.ffn.input_rows.sharded)
+ self.assertIs(
+ steps.ffn.prepare.keywords["step"].func, comm_ops._mlp_input_gather_moe_cp
+ )
+ self.assertIs(
+ steps.ffn_output_move,
+ comm.CommunicateSummableTensorPairFn._scatter_hidden_states_moe,
)
- edges = decoder_layer_edges(sides)
- step = make_boundary(edges.into_ffn, cp_moves=moves).prepare.keywords["step"]
- self.assertIs(step.func, comm_ops._mlp_input_gather_moe_cp)
- out = make_output_boundary(edges.out_of_ffn, cp_moves=moves)
- self.assertIs(out.output_move, moves.take_back)
if __name__ == "__main__":
diff --git a/test/registered/unit/layers/test_dp_attention_cp_gather.py b/test/registered/unit/layers/test_dp_attention_cp_gather.py
index e6b8f1f7bb..e3a9741cd8 100644
--- a/test/registered/unit/layers/test_dp_attention_cp_gather.py
+++ b/test/registered/unit/layers/test_dp_attention_cp_gather.py
@@ -1,3 +1,5 @@
+from sglang.test.boundary_fixtures import make_test_stages
+
"""The dense-FFN gather and take-back under attention DP x CP x TP, rank by rank.
Every rank builds its layer's communicator and runs its prepare_mlp and
@@ -15,7 +17,6 @@ import torch
from sglang.srt.layers import communicator as comm
from sglang.srt.layers import dp_attention, layernorm_sp
from sglang.srt.layers.communicator import ops as comm_ops
-from sglang.srt.layers.communicator.residual.stream import ResidualStream
from sglang.srt.layers.cp import base as cp_base
from sglang.srt.layers.cp import padding as cp_padding
from sglang.srt.layers.cp.zigzag import ZigzagCPStrategy
@@ -126,13 +127,6 @@ def cp_replicated(forward_mode, rows, seed):
return group
-def prepare_mlp(communicator, hidden, residual, forward_batch):
- forward_batch.residual_stream = ResidualStream(residual)
- return communicator.prepare_mlp(
- hidden, forward_batch.residual_stream, forward_batch
- )
-
-
class TestDpCpGather(CustomTestCase):
def run_ranks(
self,
@@ -252,16 +246,23 @@ class TestDpCpGather(CustomTestCase):
if target[0] is comm
else patch.object(*target, value)
)
- communicator = comm.LayerCommunicator(
- layer_facts=SimpleNamespace(
- is_first_layer=False,
- is_last_layer=False,
- is_layer_sparse=sparse,
- is_previous_layer_sparse=sparse,
- is_next_layer_sparse=False,
- ),
- input_layernorm=norm,
- post_attention_layernorm=norm,
+ stack.enter_context(
+ patch_communicator(
+ "get_exec",
+ lambda: SimpleNamespace(
+ comm=SimpleNamespace(boundary_reduction="rs+rsv"),
+ overlap=SimpleNamespace(enable_two_batch_overlap=False),
+ ),
+ )
+ )
+ communicator = make_test_stages(
+ first=False,
+ last=False,
+ sparse=sparse,
+ previous_sparse=sparse,
+ next_sparse=False,
+ attention_norm=norm,
+ ffn_norm=norm,
force_layernorm_before_dp_gather=force_layernorm_before_dp_gather,
)
yield SimpleNamespace(
@@ -281,13 +282,15 @@ class TestDpCpGather(CustomTestCase):
def gather(rank, all_reduce):
with as_rank(*rank, all_reduce) as r:
- self.assertIsNotNone(r.communicator._cp_steps, "declared under CP")
+ self.assertIsNotNone(
+ r.communicator.ffn.plan._cp_steps, "declared under CP"
+ )
if ffn_input is not None:
- steps = r.communicator._batch_steps(r.forward_batch)
+ steps = r.communicator.ffn.plan._batch_steps(r.forward_batch)
self.assertIs(steps.ffn.prepare.keywords["step"].func, ffn_input)
hidden_states, residual = rank_inputs(*rank)
- return prepare_mlp(
- r.communicator, hidden_states, residual, r.forward_batch
+ return r.communicator.ffn._prepare_input(
+ hidden_states, residual, r.forward_batch
)
# The all-reduce is a sum over every rank: record what each rank hands
@@ -327,8 +330,7 @@ class TestDpCpGather(CustomTestCase):
torch.testing.assert_close(hidden_states, expected, rtol=0, atol=0)
torch.testing.assert_close(residual.residual[:tokens], own, rtol=0, atol=0)
with as_rank(*rank, None) as r:
- r.forward_batch.residual_stream = residual
- back, _ = r.communicator.postprocess_layer(
+ back, _ = r.communicator.ffn.plan.output.postprocess_layer(
3 * hidden_states, residual, r.forward_batch
)
expected_back = torch.zeros(group.held_rows[cp], HIDDEN).double()
diff --git a/test/registered/unit/layers/test_independent_stage_construction.py b/test/registered/unit/layers/test_independent_stage_construction.py
new file mode 100644
index 0000000000..93b9f0f00d
--- /dev/null
+++ b/test/registered/unit/layers/test_independent_stage_construction.py
@@ -0,0 +1,351 @@
+"""Independent stages use adjacent declarations without a decoder-layer plan."""
+
+import itertools
+import unittest
+from types import SimpleNamespace
+
+import test_declared_decoder_boundary as fixture
+import torch
+
+from sglang.srt.layers.communicator import (
+ EdgeDecl,
+ Layout,
+ StageDecl,
+ StageInput,
+ StageKind,
+ StageOutput,
+ declare_attn,
+ declare_ffn,
+ make_attn_stage,
+ make_ffn_stage,
+ make_stage,
+ make_stages,
+)
+from sglang.srt.layers.communicator.residual import batch
+from sglang.srt.layers.communicator.residual.add_norm import ADD
+from sglang.test.ci.ci_register import register_cpu_ci
+from sglang.test.test_utils import CustomTestCase
+
+register_cpu_ci(est_time=5, suite="base-a-test-cpu")
+
+
+class TestIndependentStageConstruction(CustomTestCase):
+ def make(self, kind, *, first=False, update=ADD, read=None):
+ rows = Layout(frozenset())
+ need = StageInput(rows) if read is None else StageInput(rows, read=read)
+ produced = StageOutput(rows, update=update)
+ return make_stage(
+ StageDecl(need, produced),
+ kind=kind,
+ norm=fixture.Norm(),
+ incoming=EdgeDecl(StageOutput(rows), need, rows, rows),
+ outgoing=EdgeDecl(produced, StageInput(rows), rows, rows),
+ enters_stack=first,
+ terminal=True,
+ fixed_output=kind is StageKind.ATTENTION,
+ )
+
+ def test_any_two_stage_kinds_share_the_same_construction(self):
+ for first_kind in StageKind:
+ for second_kind in StageKind:
+ with (
+ self.subTest(first=first_kind, second=second_kind),
+ fixture.planning(fixture.parallel_of(attn_dp=1, attn_tp=1)),
+ ):
+ first = self.make(first_kind, first=True)
+ second = self.make(second_kind)
+ self.assertIsNot(first.plan, second.plan)
+ self.assertFalse(hasattr(first.plan, "layer_facts"))
+ fb = SimpleNamespace()
+ batch.start(fb)
+ hidden = torch.ones(2, 4)
+ for stage in (first, second):
+ hidden = stage.prepare(hidden, fb)
+ if stage.kind is StageKind.ATTENTION:
+ hidden = stage.finish(hidden, fb)
+ else:
+ with stage.exit(fb) as output:
+ pass
+ hidden = output.finish(hidden)
+ torch.testing.assert_close(
+ batch.snapshot(hidden, fb), torch.full((2, 4), 9.0)
+ )
+
+ def test_binding_order_does_not_mutate_the_other_stage(self):
+ with fixture.planning(fixture.parallel_of(attn_dp=1, attn_tp=1)):
+ attn = declare_attn(terminal=True)
+ ffn = declare_ffn(previous=attn, terminal=True)
+ for reverse in (False, True):
+ a = lambda: make_attn_stage(
+ declaration=attn, norm=fixture.Norm(), following=ffn
+ )
+ f = lambda: make_ffn_stage(declaration=ffn, norm=fixture.Norm())
+ first, second = (f(), a()) if reverse else (a(), f())
+ self.assertIsNot(first.plan, second.plan)
+ self.assertIsNot(first.norm, second.norm)
+ self.assertFalse(hasattr(first.plan, "layer_facts"))
+ self.assertFalse(hasattr(first.plan, "previous"))
+ with self.assertRaises(TypeError):
+ declare_ffn(previous=first)
+ with self.assertRaisesRegex(ValueError, "following"):
+ make_attn_stage(declaration=attn, norm=None, following=declare_ffn())
+
+ def test_public_factories_connect_all_stage_pairs(self):
+ for first_kind in StageKind:
+ for second_kind in StageKind:
+ with (
+ self.subTest(first=first_kind, second=second_kind),
+ fixture.planning(fixture.parallel_of(attn_dp=1, attn_tp=1)),
+ ):
+ declare = {
+ StageKind.ATTENTION: declare_attn,
+ StageKind.FFN: declare_ffn,
+ }
+ build = {
+ StageKind.ATTENTION: make_attn_stage,
+ StageKind.FFN: make_ffn_stage,
+ }
+ first = declare[first_kind]()
+ second = declare[second_kind](previous=first)
+ stages = [
+ build[first_kind](
+ declaration=first, norm=fixture.Norm(), following=second
+ ),
+ build[second_kind](declaration=second, norm=fixture.Norm()),
+ ]
+ fb = SimpleNamespace()
+ batch.start(fb)
+ hidden = torch.ones(2, 4)
+ for stage in stages:
+ hidden = stage.prepare(hidden, fb)
+ if stage.kind is StageKind.ATTENTION:
+ hidden = stage.finish(hidden, fb)
+ else:
+ with stage.exit(fb) as output:
+ pass
+ hidden = output.finish(hidden)
+ torch.testing.assert_close(
+ batch.snapshot(hidden, fb), torch.full((2, 4), 9.0)
+ )
+
+ def test_prepared_branch_reuses_the_same_placements(self):
+ from sglang.srt.layers.communicator.factories import (
+ _connect,
+ _connections,
+ _fork_input,
+ )
+
+ for dp in (1, 2):
+ for tp in (1, 2):
+ for local_dense in (False, True):
+ parallel = fixture.parallel_of(
+ attn_dp=dp,
+ attn_tp=tp,
+ moe_dense_tp_size=1 if local_dense else None,
+ )
+ with (
+ self.subTest(dp=dp, tp=tp, local_dense=local_dense),
+ fixture.planning(parallel),
+ ):
+ attn = declare_attn(previous=declare_ffn(sparse=True))
+ moe = declare_ffn(previous=attn, sparse=True)
+ dense = declare_ffn(
+ prepared_from=moe,
+ ordinary_only=True,
+ )
+ branch_attn = declare_attn(previous=dense, ordinary_only=True)
+ last_dense = declare_ffn(
+ previous=branch_attn,
+ ordinary_only=True,
+ )
+ _, into_moe = _connections(attn, moe)
+ fork = _fork_input(into_moe, dense)
+ middle = _connect(dense, branch_attn)
+ last = _connect(branch_attn, last_dense, residual_from=middle)
+ for declaration, following, expected in (
+ (dense, branch_attn, fork),
+ (branch_attn, last_dense, middle),
+ (last_dense, None, last),
+ ):
+ make = (
+ make_attn_stage
+ if declaration.kind is StageKind.ATTENTION
+ else make_ffn_stage
+ )
+ stage = make(
+ declaration=declaration,
+ norm=fixture.Norm(),
+ following=following,
+ )
+ self.assertEqual(
+ {
+ v: edge.incoming
+ for v, edge in stage.plan.variants.items()
+ },
+ expected.entries,
+ )
+ self.assertFalse(stage.plan.is_first_layer)
+ if declaration is dense:
+ with self.assertRaisesRegex(
+ RuntimeError, "branch_input"
+ ):
+ stage.plan._steps.ffn.prepare(None, None, None)
+
+ with self.assertRaisesRegex(ValueError, "choose"):
+ declare_ffn(previous=attn, prepared_from=moe)
+
+ def test_sequences_of_one_to_four_stages(self):
+ for count in range(1, 5):
+ for kinds in itertools.product(StageKind, repeat=count):
+ with (
+ self.subTest(kinds=kinds),
+ fixture.planning(fixture.parallel_of(attn_dp=1, attn_tp=1)),
+ ):
+ declarations = [
+ declare_attn() if kind is StageKind.ATTENTION else declare_ffn()
+ for kind in kinds
+ ]
+ boundaries = make_stages(
+ *((d, fixture.Norm()) for d in declarations), terminal=True
+ )
+ self.assertEqual(len(boundaries), count)
+ fb = SimpleNamespace()
+ batch.start(fb)
+ hidden = torch.ones(2, 4)
+ # The direct factories provide an independent construction
+ # reference, including repeated kinds and terminal placement.
+ from dataclasses import replace
+
+ previous = None
+ linked = []
+ for i, d in enumerate(declarations):
+ previous = replace(
+ d, previous=previous, terminal=i == count - 1
+ )
+ linked.append(previous)
+ for i, boundary in enumerate(boundaries):
+ build = (
+ make_attn_stage
+ if kinds[i] is StageKind.ATTENTION
+ else make_ffn_stage
+ )
+ reference = build(
+ declaration=linked[i],
+ norm=boundary.norm,
+ following=linked[i + 1] if i + 1 < count else None,
+ )
+ self.assertEqual(
+ boundary.plan.variants, reference.plan.variants
+ )
+ self.assertEqual(boundary.plan.is_last_layer, i == count - 1)
+ self.assertIsNone(declarations[i].previous)
+ hidden = boundary.prepare(hidden, fb)
+ if boundary.kind is StageKind.ATTENTION:
+ hidden = boundary.finish(hidden, fb)
+ else:
+ with boundary.exit(fb) as output:
+ pass
+ hidden = output.finish(hidden)
+ # Norm doubles its input, each later stage adds the prior
+ # residual. Snapshot adds the final contribution.
+ torch.testing.assert_close(
+ batch.snapshot(hidden, fb), torch.full((2, 4), float(3**count))
+ )
+
+ def test_sequence_branch_keeps_source_lineage(self):
+ for dp, tp, local_dense in itertools.product((1, 2), (1, 2), (False, True)):
+ with (
+ self.subTest(dp=dp, tp=tp, local_dense=local_dense),
+ fixture.planning(
+ fixture.parallel_of(
+ attn_dp=dp,
+ attn_tp=tp,
+ moe_dense_tp_size=1 if local_dense else None,
+ )
+ ),
+ ):
+ _, source = make_stages(
+ (declare_attn(), fixture.Norm()),
+ (declare_ffn(sparse=True), fixture.Norm()),
+ previous=declare_ffn(sparse=True),
+ )
+ before = dict(source.plan.variants)
+ first, attn, last = make_stages(
+ (
+ declare_ffn(ordinary_only=True),
+ fixture.Norm(),
+ ),
+ (declare_attn(ordinary_only=True), fixture.Norm()),
+ (
+ declare_ffn(ordinary_only=True),
+ fixture.Norm(),
+ ),
+ prepared_from=source.declaration,
+ )
+ self.assertEqual(source.plan.variants, before)
+ self.assertIs(first.declaration.prepared_from, source.declaration)
+ self.assertIs(attn.declaration.previous, first.declaration)
+ self.assertIs(last.declaration.previous, attn.declaration)
+ for v, edge in first.plan.variants.items():
+ source_input = source.plan.variants[v].incoming
+ self.assertEqual(
+ edge.incoming.produced.layout, source_input.need.layout
+ )
+ self.assertEqual(edge.incoming.residual, source_input.residual_to)
+ with self.assertRaisesRegex(RuntimeError, "branch_input"):
+ first.plan._steps.ffn.prepare(None, None, None)
+
+ def test_mixer_successor_comes_from_the_local_sequence(self):
+ for following in (declare_attn(mixer_exit=True), declare_ffn()):
+ with (
+ self.subTest(kind=following.kind),
+ fixture.planning(fixture.parallel_of(attn_dp=1, attn_tp=2)),
+ ):
+ mixer, consumer = make_stages(
+ (declare_attn(mixer_exit=True), fixture.Norm()),
+ (following, fixture.Norm()),
+ terminal=True,
+ )
+ self.assertIs(mixer.declaration.next_kind, following.kind)
+ for variant, edge in mixer.plan.variants.items():
+ output = edge.outgoing.produced
+ self.assertEqual(
+ output.always_leaves, following.kind is StageKind.FFN
+ )
+ self.assertEqual(
+ output.leaves_for_next_layer,
+ following.kind is StageKind.ATTENTION,
+ )
+ incoming = consumer.plan.variants[variant].incoming.produced
+ self.assertEqual(incoming.group, output.group)
+
+ def test_sequence_requires_one_source_and_external_terminal(self):
+ with self.assertRaisesRegex(ValueError, "at least one"):
+ make_stages()
+ with self.assertRaisesRegex(ValueError, "choose"):
+ make_stages(
+ (declare_attn(), None),
+ previous=declare_ffn(),
+ prepared_from=declare_ffn(),
+ )
+ with self.assertRaisesRegex(ValueError, "input sources"):
+ make_stages((declare_attn(previous=declare_ffn()), None))
+ with self.assertRaisesRegex(ValueError, "terminal"):
+ make_stages((declare_ffn(terminal=True), None))
+
+ def test_mismatched_declared_edge_is_rejected(self):
+ rows = Layout(frozenset())
+ declaration = StageDecl(StageInput(rows), StageOutput(rows))
+ wrong_input = StageInput(rows, read=object())
+ with self.assertRaisesRegex(ValueError, "disagrees"):
+ make_stage(
+ declaration,
+ kind=StageKind.FFN,
+ norm=None,
+ incoming=EdgeDecl(StageOutput(rows), wrong_input, rows, rows),
+ outgoing=EdgeDecl(declaration.output, declaration.input, rows, rows),
+ )
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/test/registered/unit/layers/test_layer_communicator_fusion_gate.py b/test/registered/unit/layers/test_layer_communicator_fusion_gate.py
index 215ac56a8b..32704ffd22 100644
--- a/test/registered/unit/layers/test_layer_communicator_fusion_gate.py
+++ b/test/registered/unit/layers/test_layer_communicator_fusion_gate.py
@@ -8,17 +8,12 @@ import torch
from sglang.srt.layers import communicator as comm
from sglang.srt.layers.communicator import (
- LayerCommunicator,
Layout,
StageOutput,
SumGroup,
- TokenAxis,
)
-from sglang.srt.layers.communicator import boundary as comm_boundary
from sglang.srt.layers.communicator import ops as comm_ops
-from sglang.srt.layers.communicator import (
- sequence_parallel_layer_sides,
-)
+from sglang.srt.layers.communicator.exit import OutputBoundary
from sglang.srt.layers.moe import (
can_merge_post_experts_all_reduce,
deferred_post_experts_all_reduce,
@@ -26,6 +21,7 @@ from sglang.srt.layers.moe import (
)
from sglang.srt.layers.moe import utils as moe_utils
from sglang.srt.runtime_context import get_forward, get_parallel
+from sglang.test.boundary_fixtures import sp_region_steps, stub_plan
from sglang.test.ci.ci_register import register_cpu_ci
from sglang.test.communicator_patch import patch_communicator
from sglang.test.test_utils import CustomTestCase
@@ -33,19 +29,6 @@ from sglang.test.test_utils import CustomTestCase
register_cpu_ci(est_time=11, suite="base-a-test-cpu")
-def sp_region_steps():
- """The steps a layer runs while a LayerNorm SP region is active."""
- return comm_boundary._select_boundary_steps(
- sequence_parallel_layer_sides(
- axis_sizes={
- TokenAxis.ATTN_DP: 1,
- TokenAxis.ATTN_CP: 1,
- TokenAxis.ATTN_TP_SCATTER: 2,
- }
- )
- )
-
-
def _steps(*, ffn_output=None, returns_over_dp=False, ffn_sum_is_movable=True):
return comm.BoundarySteps(
attention=comm.StageEntry(
@@ -83,7 +66,7 @@ def _steps(*, ffn_output=None, returns_over_dp=False, ffn_sum_is_movable=True):
def _fake_communicator(ffn_sum_is_movable=True):
- communicator = LayerCommunicator.__new__(LayerCommunicator)
+ communicator = stub_plan()
communicator._speculative_algo = None
# An FFN gathered over MoE-CP or on each rank's own rows has no movable sum.
communicator._steps = _steps(ffn_sum_is_movable=ffn_sum_is_movable)
@@ -91,7 +74,6 @@ def _fake_communicator(ffn_sum_is_movable=True):
communicator._input_scattered_steps = None
communicator._cp_steps = None
communicator.is_last_layer = False
- communicator._context = types.SimpleNamespace(tp_size=4)
return communicator
@@ -331,9 +313,9 @@ class TestFuseMlpAllReduceGate(CustomTestCase):
tp_size=moe_ep_size * moe_tp_size * moe_dp_size,
),
):
- return LayerCommunicator._ffn_sum_can_move_to_next_layer(
- _fake_communicator(ffn_sum_is_movable), forward_batch
- )
+ return OutputBoundary(
+ _fake_communicator(ffn_sum_is_movable)
+ )._ffn_sum_can_move_to_next_layer(forward_batch)
def test_hybrid_ep_tp_fuses_when_mergeable(self):
self.assertTrue(self._should_fuse(moe_ep_size=2, moe_tp_size=2))
@@ -487,7 +469,7 @@ class TestDeferFfnReduction(CustomTestCase):
return_value=tp_group_object if tp_group else object(),
),
):
- return communicator._ffn_sum_moves_to_next_layer(
+ return communicator.output._ffn_sum_moves_to_next_layer(
forward_batch, mlp_reduce_scatter=reduce_scatter, dp_step=step
)
diff --git a/test/registered/unit/layers/test_layernorm_sp.py b/test/registered/unit/layers/test_layernorm_sp.py
index 9bbdf7c5bc..41331739f9 100644
--- a/test/registered/unit/layers/test_layernorm_sp.py
+++ b/test/registered/unit/layers/test_layernorm_sp.py
@@ -1,3 +1,6 @@
+from sglang.srt.layers.communicator import StageKind
+from sglang.test.boundary_fixtures import sp_region_steps, stub_plan, stub_stage
+
"""Unit tests for srt/layers/layernorm_sp (Megatron LayerNorm sequence parallelism).
Covers the pure logic that gates SP -- the Qwen3 allowlist, the config guards, and
@@ -19,14 +22,8 @@ from sglang.srt.layers.communicator import (
Layout,
StageOutput,
SumGroup,
- TokenAxis,
)
-from sglang.srt.layers.communicator import boundary as comm_boundary
from sglang.srt.layers.communicator import ops as comm_ops
-from sglang.srt.layers.communicator import (
- sequence_parallel_layer_sides,
-)
-from sglang.srt.layers.communicator.residual import batch as residual_batch
from sglang.srt.model_executor.forward_batch_info import ForwardMode
from sglang.srt.runtime_context import (
get_flags,
@@ -42,19 +39,6 @@ from sglang.test.test_utils import CustomTestCase
register_cpu_ci(est_time=9, suite="base-a-test-cpu")
-def sp_region_steps():
- """The steps a layer runs while a LayerNorm SP region is active."""
- return comm_boundary._select_boundary_steps(
- sequence_parallel_layer_sides(
- axis_sizes={
- TokenAxis.ATTN_DP: 1,
- TokenAxis.ATTN_CP: 1,
- TokenAxis.ATTN_TP_SCATTER: 2,
- }
- )
- )
-
-
def _initialize(*, enable=True, arch="Qwen3ForCausalLM"):
publish(
ServerArgs(model_path="dummy", enable_layernorm_sp=enable), role="tokenizer"
@@ -174,19 +158,15 @@ class TestSpRegionSteps(CustomTestCase):
shard: add + norm, no move, no collective."""
def communicator(self, *, first_layer):
- c = comm.LayerCommunicator.__new__(comm.LayerCommunicator)
+ c = stub_plan()
c._sp_steps = sp_region_steps()
c._input_scattered_steps = None
c._cp_steps = None
- c.layer_facts = SimpleNamespace(
- is_first_layer=first_layer, is_layer_sparse=False
- )
- c.allow_reduce_scatter = False
+ c.is_first_layer = first_layer
+ c.is_sparse = False
c._attn_input_fusions = ()
- c.input_layernorm = _Norm()
- c.post_attention_layernorm = _Norm()
+ c.norm = _Norm()
c.qkv_latent_func = None
- c._context = SimpleNamespace(cache=None)
c._steps = self.ordinary_steps(MagicMock(side_effect=AssertionError("moved")))
return c
@@ -219,17 +199,8 @@ class TestSpRegionSteps(CustomTestCase):
ffn_sum_is_movable=True,
)
- def owned_input(self, hidden, residual, mode=ForwardMode.EXTEND, *, record=True):
- batch = SimpleNamespace(forward_mode=mode, residual_stream=None)
- stream = residual_batch.start(batch)
- if residual is not None:
- stream.write(residual)
- if record:
- hidden = stream.leave(hidden, comm.ADD)
- return hidden, stream, batch
-
def run_prepare_attn(self, communicator, mode, hidden, residual):
- hidden, residual, batch = self.owned_input(hidden, residual, mode)
+ batch = SimpleNamespace(forward_mode=mode)
with (
get_flags().sp.override(enabled=True),
get_forward().scoped(sp_active=False),
@@ -241,7 +212,9 @@ class TestSpRegionSteps(CustomTestCase):
layernorm_sp, "sp_entry_scatter", side_effect=lambda h: h[:1]
) as scatter,
):
- out = communicator.prepare_attn(hidden, residual, batch)
+ out = stub_stage(communicator, StageKind.ATTENTION)._prepare_input(
+ hidden, residual, batch
+ )
return out, get_forward().sp_active, scatter
def test_the_first_layer_opens_the_region_on_prefill(self):
@@ -287,21 +260,20 @@ class TestSpRegionSteps(CustomTestCase):
"get_attn_tp_context",
return_value=SimpleNamespace(input_scattered=False),
):
- h, r = communicator.prepare_attn(
- *self.owned_input(hidden, residual.clone())
+ h, r = stub_stage(communicator, StageKind.ATTENTION)._prepare_input(
+ hidden,
+ residual.clone(),
+ SimpleNamespace(forward_mode=ForwardMode.EXTEND),
)
torch.testing.assert_close(h, torch.full((1, 4), 8.0))
- h, r = communicator.prepare_mlp(
- *self.owned_input(hidden, residual.clone(), record=False)
+ h, r = stub_stage(communicator, StageKind.FFN)._prepare_input(
+ hidden, residual.clone(), object()
)
torch.testing.assert_close(h, torch.full((1, 4), 8.0))
torch.testing.assert_close(r.residual, torch.full((1, 4), 4.0))
- batch = SimpleNamespace(residual_stream=None)
- stream = residual_batch.start(batch)
- stream.write(residual)
- out = communicator.postprocess_layer(hidden, stream, batch)
+ out = communicator.output.postprocess_layer(hidden, residual, object())
self.assertIs(out[0], hidden)
- self.assertIs(out[1], stream)
+ self.assertIs(out[1], residual)
if __name__ == "__main__":
diff --git a/test/registered/unit/layers/test_postprocess_reduce_scatterv.py b/test/registered/unit/layers/test_postprocess_reduce_scatterv.py
index d73d2eac3a..4531203d2c 100644
--- a/test/registered/unit/layers/test_postprocess_reduce_scatterv.py
+++ b/test/registered/unit/layers/test_postprocess_reduce_scatterv.py
@@ -1,3 +1,5 @@
+from sglang.test.boundary_fixtures import stub_plan
+
"""Under DP attention with reduce_scatterv, postprocess sums a FULL-mode layer's
output while scattering it back to the local tokens only when the FFN left its
all-reduce out."""
@@ -12,11 +14,7 @@ from unittest.mock import patch
import torch
from sglang.srt.layers import communicator as comm
-from sglang.srt.layers.communicator import (
- CommunicateSummableTensorPairFn,
- LayerCommunicator,
- LayerFacts,
-)
+from sglang.srt.layers.communicator import exit as comm_exit
from sglang.srt.layers.communicator import ops as comm_ops
from sglang.srt.layers.moe import utils as moe_utils
from sglang.srt.layers.moe.utils import (
@@ -76,7 +74,7 @@ def reduce_scatterv_applies():
yield
-def postprocess_sums(*, allow_reduce_scatter, is_layer_sparse):
+def postprocess_sums(*, reduce_scatter, is_layer_sparse):
calls = []
parallel = types.SimpleNamespace(
tp_size=2,
@@ -97,19 +95,19 @@ def postprocess_sums(*, allow_reduce_scatter, is_layer_sparse):
"dp_scatter", side_effect=lambda *args: calls.append("dp_scatter")
),
):
- CommunicateSummableTensorPairFn._scatter_hidden_states(
- torch.zeros(2, 4),
- None,
+ step = comm_exit._reduce_and_redistribute_output_step(
forward_batch,
- context=None,
- allow_reduce_scatter=allow_reduce_scatter,
- is_layer_sparse=is_layer_sparse,
+ leaves_for_reduce_scatter=reduce_scatter,
+ leaves_for_reduce_scatterv=reduce_scatter,
+ )
+ comm_ops._to_local_tokens(
+ step or comm_ops._redistribute_output, forward_batch, torch.zeros(2, 4)
)
assert len(calls) == 1, calls
return calls[0] == "reduce_scatterv"
-def ffn_leaves_the_sum_out(*, allow_reduce_scatter, is_layer_sparse):
+def ffn_leaves_the_sum_out(*, reduce_scatter, is_layer_sparse):
# A layer that allows reduce-scatter publishes mlp_reduce_scatter while
# reduce_scatterv applies; a dense MLP reads only that flag.
with (
@@ -133,7 +131,7 @@ def ffn_leaves_the_sum_out(*, allow_reduce_scatter, is_layer_sparse):
"should_use_flashinfer_cutlass_moe_fp4_allgather",
return_value=False,
),
- get_forward().scoped(mlp_reduce_scatter=allow_reduce_scatter),
+ get_forward().scoped(mlp_reduce_scatter=reduce_scatter),
):
if is_layer_sparse:
return should_skip_post_experts_all_reduce(is_tp_path=True)
@@ -143,50 +141,41 @@ def ffn_leaves_the_sum_out(*, allow_reduce_scatter, is_layer_sparse):
class TestPostprocessReduceScatterv(CustomTestCase):
def test_sums_exactly_when_the_ffn_left_the_sum_out(self):
for allow, sparse in itertools.product((False, True), repeat=2):
- case = dict(allow_reduce_scatter=allow, is_layer_sparse=sparse)
+ case = dict(reduce_scatter=allow, is_layer_sparse=sparse)
with self.subTest(**case):
self.assertEqual(
postprocess_sums(**case), ffn_leaves_the_sum_out(**case)
)
def test_a_dense_layer_without_reduce_scatter_is_not_summed_again(self):
- self.assertFalse(
- postprocess_sums(allow_reduce_scatter=False, is_layer_sparse=False)
- )
+ self.assertFalse(postprocess_sums(reduce_scatter=False, is_layer_sparse=False))
- def test_layer_facts_record_whether_the_layer_is_sparse(self):
- for sparse in (False, True):
- with self.subTest(sparse=sparse):
- facts = LayerFacts.init_new(
- layer_id=1,
- num_layers=4,
- is_layer_sparse=sparse,
- is_previous_layer_sparse=sparse,
- is_next_layer_sparse=sparse,
- )
- self.assertEqual(facts.is_layer_sparse, sparse)
+ def test_postprocess_only_scatters_a_complete_output(self):
+ # The operation-scheduled path supplies a complete output; postprocess
+ # only scatters it, even when the exit could choose reduce-scatter.
+ for step in (object(), None):
+ with self.subTest(reduce_scatter=step is not None):
+ communicator = stub_plan()
+ communicator._sp_steps = None
+ communicator._input_scattered_steps = None
+ communicator._cp_steps = None
+ communicator._steps = steps(ffn_output_move=None)
+ communicator.output._postprocess_dp_step = lambda forward_batch: step
+ with patch_communicator(
+ "_to_local_tokens", side_effect=lambda s, fb, h: (s, h)
+ ):
+ hidden_states, residual = communicator.output.postprocess_layer(
+ "h", "r", "fb"
+ )
+ self.assertEqual(hidden_states, (comm_ops._redistribute_output, "h"))
+ self.assertEqual(residual, "r")
- def test_postprocess_passes_the_layer_sparsity(self):
- seen = {}
- communicator = LayerCommunicator.__new__(LayerCommunicator)
- communicator._sp_steps = None
- communicator._input_scattered_steps = None
- communicator._cp_steps = None
- communicator._context = None
- communicator.allow_reduce_scatter = False
- communicator.layer_facts = types.SimpleNamespace(is_layer_sparse=True)
- communicator._steps = steps(
- ffn_output_move=lambda **kwargs: seen.update(kwargs) or (None, None)
- )
- communicator.postprocess_layer(None, None, None)
- self.assertIs(seen["is_layer_sparse"], True)
-
- def test_postprocess_takes_the_output_back_with_the_exit_step(self):
+ def test_legacy_postprocess_takes_the_output_back_with_the_exit_step(self):
# Under attention DP the base postprocess brings the FFN output back with
# the step the FFN exit chooses.
for step in (object(), None):
with self.subTest(reduce_scatter=step is not None):
- communicator = LayerCommunicator.__new__(LayerCommunicator)
+ communicator = comm.LayerCommunicator.__new__(comm.LayerCommunicator)
communicator._sp_steps = None
communicator._input_scattered_steps = None
communicator._cp_steps = None
@@ -216,18 +205,18 @@ class TestReduceAndRedistributeOutputStep(CustomTestCase):
patch_communicator("should_use_dp_reduce_scatterv", return_value=varlen),
patch_communicator("can_use_dp_reduce_scatter", return_value=tiles),
):
- # A MoE block leaves its sum to reduce_scatterv whenever it applies.
- return comm_ops._reduce_and_redistribute_output_step(
+ # Both dense and MoE producers follow the selected output step.
+ return comm_exit._reduce_and_redistribute_output_step(
forward_batch,
leaves_for_reduce_scatter=allow,
- leaves_for_reduce_scatterv=allow or sparse,
+ leaves_for_reduce_scatterv=allow,
)
def test_every_condition(self):
for varlen, max_len, tiles, allow, sparse in itertools.product(
(False, True), repeat=5
):
- if varlen and (allow or sparse):
+ if varlen and allow:
expected = comm_ops._reduce_and_redistribute_output_varlen
elif allow and max_len and tiles:
expected = comm_ops._reduce_and_redistribute_output_max_len
diff --git a/test/registered/unit/layers/test_prepare_attn_steps.py b/test/registered/unit/layers/test_prepare_attn_steps.py
index a0d44d9ded..595083efba 100644
--- a/test/registered/unit/layers/test_prepare_attn_steps.py
+++ b/test/registered/unit/layers/test_prepare_attn_steps.py
@@ -1,3 +1,6 @@
+from sglang.srt.layers.communicator import StageKind
+from sglang.test.boundary_fixtures import stub_plan, stub_stage
+
"""prepare_attn completes a sum the previous layer left, then adds the residual
and applies the input norm in the form the attention's quant format wants."""
@@ -12,6 +15,7 @@ import torch
from sglang.srt.layers import communicator as comm
from sglang.srt.layers.communicator import ops as comm_ops
+from sglang.srt.layers.communicator.fusions.allreduce import attention_fusions
from sglang.test.ci.ci_register import register_cpu_ci
from sglang.test.communicator_patch import patch_communicator
from sglang.test.test_utils import CustomTestCase
@@ -44,19 +48,18 @@ class Norm:
def communicator(norm):
- c = comm.LayerCommunicator.__new__(comm.LayerCommunicator)
- c.input_layernorm = norm
+ c = stub_plan()
+ c.norm = norm
# A layer inside the stack: an absent residual was written back before.
- c.layer_facts = SimpleNamespace(is_first_layer=False)
+ c.is_first_layer = False
c._sp_steps = None
c._input_scattered_steps = None
c._cp_steps = None
c.qkv_latent_func = None
- c._context = None
c.enable_fused_ar_quant = False
c.fused_ar_quant_keep_bf16 = False
# Construction picks the fused entries; call it under the platform patches.
- c._attn_input_fusions = c._select_attn_input_fusions()
+ c._attn_input_fusions = attention_fusions(c)
# A layer whose attention takes its input as it is and owes nothing on it.
c._steps = comm.BoundarySteps(
attention=comm.StageEntry(
@@ -132,16 +135,6 @@ def platform(*, use_aiter=False, gfx95=False, fusion=False, kernel_group=True):
yield kernels, all_reduce, group
-def prepare_attn(communicator, hidden, residual, forward_batch=None, **kwargs):
- from sglang.srt.layers.communicator.residual.add_norm import ADD
- from sglang.srt.layers.communicator.residual.stream import ResidualStream
-
- batch = forward_batch or SimpleNamespace()
- batch.residual_stream = ResidualStream(residual)
- hidden = batch.residual_stream.leave(hidden, ADD)
- return communicator.prepare_attn(hidden, batch.residual_stream, batch, **kwargs)
-
-
class TestPrepareAttnSteps(CustomTestCase):
def test_each_quant_format_uses_its_kernel(self):
for quant_format, flags, kernel in (
@@ -165,12 +158,8 @@ class TestPrepareAttnSteps(CustomTestCase):
platform(**flags) as (kernels, _, _),
):
norm = Norm()
- prepare_attn(
- communicator(norm),
- torch.ones(2, 4),
- residual,
- None,
- quant_format=quant_format,
+ stub_stage(communicator(norm), StageKind.ATTENTION)._prepare_input(
+ torch.ones(2, 4), residual, None, quant_format=quant_format
)
for name, mock in kernels.items():
self.assertEqual(mock.called, name == kernel, name)
@@ -185,13 +174,9 @@ class TestPrepareAttnSteps(CustomTestCase):
):
with self.subTest(quant_format=quant_format), platform(**flags):
hidden_states = torch.ones(2, 4)
- _, residual = prepare_attn(
- communicator(Norm()),
- hidden_states,
- None,
- None,
- quant_format=quant_format,
- )
+ _, residual = stub_stage(
+ communicator(Norm()), StageKind.ATTENTION
+ )._prepare_input(hidden_states, None, None, quant_format=quant_format)
self.assertIs(residual.residual, hidden_states)
def test_a_pending_sum_is_completed_once(self):
@@ -202,7 +187,9 @@ class TestPrepareAttnSteps(CustomTestCase):
):
norm = Norm()
hidden_states = comm.UnreducedOutput(torch.ones(2, 4), group=group)
- prepare_attn(communicator(norm), hidden_states, torch.zeros(2, 4), None)
+ stub_stage(communicator(norm), StageKind.ATTENTION)._prepare_input(
+ hidden_states, torch.zeros(2, 4), None
+ )
self.assertEqual(all_reduce.call_count, 0 if fusion else 1)
self.assertEqual(
norm.calls, ["fused_all_reduce_norm"] if fusion else ["norm"]
@@ -211,8 +198,7 @@ class TestPrepareAttnSteps(CustomTestCase):
def test_a_pending_sum_keeps_the_quant_format_without_fusion(self):
with platform(use_aiter=True) as (kernels, all_reduce, group):
norm = Norm()
- prepare_attn(
- communicator(norm),
+ stub_stage(communicator(norm), StageKind.ATTENTION)._prepare_input(
comm.UnreducedOutput(torch.ones(2, 4), group=group),
torch.zeros(2, 4),
None,
@@ -225,8 +211,7 @@ class TestPrepareAttnSteps(CustomTestCase):
def test_a_pending_sum_over_another_group_skips_the_fused_kernel(self):
with platform(fusion=True, kernel_group=False) as (_, all_reduce, group):
norm = Norm()
- prepare_attn(
- communicator(norm),
+ stub_stage(communicator(norm), StageKind.ATTENTION)._prepare_input(
comm.UnreducedOutput(torch.ones(2, 4), group=group),
torch.zeros(2, 4),
None,
@@ -237,8 +222,7 @@ class TestPrepareAttnSteps(CustomTestCase):
def test_a_post_residual_addition_bypasses_the_fused_kernel(self):
with platform(fusion=True) as (_, all_reduce, group):
norm = Norm()
- prepare_attn(
- communicator(norm),
+ stub_stage(communicator(norm), StageKind.ATTENTION)._prepare_input(
comm.UnreducedOutput(torch.ones(2, 4), group=group),
torch.zeros(2, 4),
None,
@@ -257,8 +241,9 @@ class TestPrepareAttnSteps(CustomTestCase):
partial = torch.ones(3, 4)
local = torch.full((local_rows, 4), 7.0)
step = MagicMock(return_value=local)
- hidden_states, residual = prepare_attn(
- communicator(norm),
+ hidden_states, residual = stub_stage(
+ communicator(norm), StageKind.ATTENTION
+ )._prepare_input(
comm.UnreducedOutput(partial, reduce_and_redistribute=step),
torch.zeros(local_rows, 4),
None,
@@ -296,8 +281,7 @@ class TestPrepareAttnSteps(CustomTestCase):
)
partial_sum = torch.ones(3, 4)
step = MagicMock(return_value=torch.full((1, 4), 7.0))
- prepare_attn(
- c,
+ stub_stage(c, StageKind.ATTENTION)._prepare_input(
comm.UnreducedOutput(partial_sum, reduce_and_redistribute=step),
torch.zeros(1, 4),
None,
@@ -314,8 +298,7 @@ class TestPrepareAttnSteps(CustomTestCase):
):
norm = Norm()
with self.assertRaises(RuntimeError):
- prepare_attn(
- communicator(norm),
+ stub_stage(communicator(norm), StageKind.ATTENTION)._prepare_input(
comm.UnreducedOutput(
torch.ones(2, 4), reduce_and_redistribute=step
),
diff --git a/test/registered/unit/layers/test_residual_stream.py b/test/registered/unit/layers/test_residual_stream.py
index 44eb241df6..56ef72f20c 100644
--- a/test/registered/unit/layers/test_residual_stream.py
+++ b/test/registered/unit/layers/test_residual_stream.py
@@ -1,4 +1,5 @@
import unittest
+import weakref
from types import SimpleNamespace
from unittest.mock import Mock, patch
@@ -8,7 +9,6 @@ from sglang.srt.layers.aux_hidden_states import AuxHiddenStateList
from sglang.srt.layers.communicator import (
BoundarySteps,
EdgeDecl,
- LayerCommunicator,
Layout,
StageEntry,
StageInput,
@@ -16,7 +16,8 @@ from sglang.srt.layers.communicator import (
SumGroup,
make_boundary,
)
-from sglang.srt.layers.communicator.layer import StageCommunicator
+from sglang.srt.layers.communicator.boundary import StageKind
+from sglang.srt.layers.communicator.construction import StageCommunicator
from sglang.srt.layers.communicator.output import UnreducedOutput
from sglang.srt.layers.communicator.residual.access import add_to_output
from sglang.srt.layers.communicator.residual.add_norm import ADD
@@ -25,6 +26,7 @@ from sglang.srt.layers.communicator.residual.stream import (
OwedOutput,
ResidualStream,
)
+from sglang.test.boundary_fixtures import stub_plan, stub_stage
from sglang.test.ci.ci_register import register_cpu_ci
from sglang.test.test_utils import CustomTestCase
@@ -63,16 +65,15 @@ class TestResidualStream(CustomTestCase):
StageEntry(boundary.prepare, rows), None, StageOutput(rows), None, False
)
stage = StageCommunicator(
- SimpleNamespace(input_layernorm=None, _context=None),
- "attention",
- "input_layernorm",
+ SimpleNamespace(norm=None),
+ StageKind.ATTENTION,
+ None,
)
value = torch.ones(2, 4)
- batch = SimpleNamespace(residual_stream=ResidualStream())
- hidden, stream = stage.prepare(value, batch.residual_stream, batch, steps)
+ hidden, stream = stage._prepare(value, ResidualStream(), None, steps)
self.assertEqual(read.enters, 1)
torch.testing.assert_close(hidden, value * 6)
- hidden, stream = stage.prepare(stream.residual, stream, batch, steps)
+ hidden, stream = stage._prepare(stream.residual, stream, None, steps)
self.assertEqual(read.enters, 1)
torch.testing.assert_close(hidden, value * 6)
@@ -110,17 +111,16 @@ class TestResidualStream(CustomTestCase):
StageEntry(boundary.prepare, rows), None, StageOutput(rows), None, False
)
stage = StageCommunicator(
- SimpleNamespace(input_layernorm=None, _context=None),
- "attention",
- "input_layernorm",
+ SimpleNamespace(norm=None),
+ StageKind.ATTENTION,
+ None,
)
for increment in (2, 7):
with self.subTest(increment=increment):
actual = Update(increment)
stream = ResidualStream(torch.full((2, 4), 3.0))
hidden = stream.leave(torch.ones(2, 4), actual)
- batch = SimpleNamespace(residual_stream=stream)
- result, stream = stage.prepare(hidden, stream, batch, steps)
+ result, stream = stage._prepare(hidden, stream, None, steps)
torch.testing.assert_close(
result, torch.full((2, 4), 2.0 * (4 + increment))
)
@@ -139,7 +139,6 @@ class TestResidualStream(CustomTestCase):
self.residual,
None,
None,
- None,
update=SimpleNamespace(adds_plainly=False),
)
@@ -189,7 +188,6 @@ class TestResidualStream(CustomTestCase):
self.residual,
None,
norm,
- None,
pending=stream.pending,
update=stream.pending.update,
)
@@ -209,7 +207,6 @@ class TestResidualStream(CustomTestCase):
self.residual,
None,
None,
- None,
pending=stream.pending,
update=ADD,
)
@@ -241,7 +238,7 @@ class TestResidualStream(CustomTestCase):
self.stream.input(hidden.clone())
def test_capture_and_compute_share_the_fused_read(self):
- boundary = LayerCommunicator.__new__(LayerCommunicator)
+ boundary = stub_plan()
events = []
outputs = AuxHiddenStateList()
@@ -257,27 +254,28 @@ class TestResidualStream(CustomTestCase):
events.append("capture")
outputs.capture(value, owned=owned)
- boundary.prepare_attn = Mock(side_effect=prepare)
- boundary.attn = SimpleNamespace(
- entry=lambda batch: SimpleNamespace(
- capture_move=None,
- capture_move_allocates=False,
- capture_preserves_residual=None,
- )
+ boundary.norm = None
+ stub_stage(boundary, StageKind.ATTENTION)._prepare_input = Mock(
+ side_effect=prepare
)
- output, stream = boundary.prepare_attn_and_capture_last_layer_outputs(
+ stub_stage(boundary, StageKind.ATTENTION).entry = lambda batch: SimpleNamespace(
+ capture_move=None,
+ capture_move_allocates=False,
+ capture_preserves_residual=None,
+ )
+ output, stream = stub_stage(boundary, StageKind.ATTENTION)._prepare_attention(
self.hidden, self.stream, None, capture_output=capture
)
self.assertEqual(events, ["add_norm", "capture"])
self.group.all_reduce.assert_called_once()
- boundary.prepare_attn.assert_called_once()
+ stub_stage(boundary, StageKind.ATTENTION)._prepare_input.assert_called_once()
torch.testing.assert_close(output, torch.full((2, 4), 15.0))
torch.testing.assert_close(outputs[0], torch.full((2, 4), 5.0))
stream.residual.zero_()
torch.testing.assert_close(outputs[0], torch.full((2, 4), 5.0))
def test_deepstack_capture_precedes_the_extra_addition(self):
- boundary = LayerCommunicator.__new__(LayerCommunicator)
+ boundary = stub_plan()
extra = torch.full_like(self.partial, 7.0)
outputs = AuxHiddenStateList()
@@ -288,15 +286,16 @@ class TestResidualStream(CustomTestCase):
stream.write(updated)
return updated * 3, stream
- boundary.prepare_attn = Mock(side_effect=prepare)
- boundary.attn = SimpleNamespace(
- entry=lambda batch: SimpleNamespace(
- capture_move=None,
- capture_move_allocates=False,
- capture_preserves_residual=None,
- )
+ boundary.norm = None
+ stub_stage(boundary, StageKind.ATTENTION)._prepare_input = Mock(
+ side_effect=prepare
)
- output, _ = boundary.prepare_attn_and_capture_last_layer_outputs(
+ stub_stage(boundary, StageKind.ATTENTION).entry = lambda batch: SimpleNamespace(
+ capture_move=None,
+ capture_move_allocates=False,
+ capture_preserves_residual=None,
+ )
+ output, _ = stub_stage(boundary, StageKind.ATTENTION)._prepare_attention(
self.hidden,
self.stream,
None,
@@ -376,5 +375,81 @@ class TestResidualStream(CustomTestCase):
move.assert_called_once_with(self.partial)
+class TestBatchStageOwnership(CustomTestCase):
+ def test_terminal_norm_releases_layer_buffers(self):
+ from sglang.srt.layers.communicator.residual import batch
+
+ fb = SimpleNamespace(residual_stream=None)
+ batch.start(fb)
+ residual = torch.ones(2, 4)
+ contribution = torch.full_like(residual, 2)
+ refs = [weakref.ref(residual), weakref.ref(contribution)]
+ batch.current(fb).write(residual)
+ output = batch.current(fb).leave(contribution, ADD)
+ result = batch.norm(output, fb, lambda x, r: (x + r, r))
+ del output, residual, contribution
+ torch.testing.assert_close(result, torch.full((2, 4), 3.0))
+ self.assertTrue(all(ref() is None for ref in refs))
+ self.assertIsNone(fb.residual_stream)
+
+ def test_pp_export_transfers_buffer_ownership(self):
+ from sglang.srt.layers.communicator.residual import batch
+
+ fb = SimpleNamespace(residual_stream=None)
+ batch.start(fb)
+ residual = torch.ones(2, 4)
+ contribution = torch.full_like(residual, 2)
+ refs = [weakref.ref(residual), weakref.ref(contribution)]
+ batch.current(fb).write(residual)
+ output = batch.current(fb).leave(contribution, ADD)
+ proxy = batch.to_pp(output, fb)
+ del output, residual, contribution
+ self.assertTrue(all(ref() is not None for ref in refs))
+ self.assertIsNone(fb.residual_stream)
+ del proxy
+ self.assertTrue(all(ref() is None for ref in refs))
+
+ def test_interleaved_batches_keep_independent_streams(self):
+ from sglang.srt.layers.communicator.residual import batch
+
+ class Read:
+ norms_plainly = False
+
+ def enter(self, value):
+ return value
+
+ def read(self, value, norm, quant_format="", **kwargs):
+ return value * 2, value
+
+ def update_and_read(self, update, value, residual, norm, **kwargs):
+ updated = update.update(value, residual)
+ return updated * 2, updated
+
+ rows = Layout(frozenset())
+ boundary = make_boundary(
+ EdgeDecl(StageOutput(rows), StageInput(rows, read=Read()), rows, rows)
+ )
+ owner = stub_plan()
+ owner.norm = None
+ owner._steps = BoundarySteps(
+ None, StageEntry(boundary.prepare, rows), StageOutput(rows), None, False
+ )
+ owner._sp_steps = owner._input_scattered_steps = owner._cp_steps = None
+ stage = stub_stage(owner, StageKind.FFN)
+ a, b = SimpleNamespace(), SimpleNamespace()
+ batch.start(a)
+ batch.start(b)
+ first = stage.prepare(torch.ones(2, 4), a)
+ second = stage.prepare(torch.full((2, 4), 10.0), b)
+ first = batch.current(a).leave(first * 3, ADD)
+ second = batch.current(b).leave(second * 5, ADD)
+ torch.testing.assert_close(stage.prepare(first, a), torch.full((2, 4), 14.0))
+ torch.testing.assert_close(stage.prepare(second, b), torch.full((2, 4), 220.0))
+ self.assertIsNot(batch.current(a), batch.current(b))
+ batch.start(a)
+ self.assertIsNone(batch.current(a).pending)
+ self.assertIsNotNone(batch.current(b).residual)
+
+
if __name__ == "__main__":
unittest.main()
diff --git a/test/registered/unit/lora/test_triton_dp_attention_unit.py b/test/registered/unit/lora/test_triton_dp_attention_unit.py
index e887e0ed73..766b5afddc 100644
--- a/test/registered/unit/lora/test_triton_dp_attention_unit.py
+++ b/test/registered/unit/lora/test_triton_dp_attention_unit.py
@@ -1,3 +1,6 @@
+from sglang.srt.layers.communicator import StageKind
+from sglang.test.boundary_fixtures import stub_plan, stub_stage
+
"""CPU-only metadata tests for explicit DP-attention LoRA routing."""
import sys
@@ -12,12 +15,11 @@ import torch
from sglang.srt.layers.communicator import (
ADD,
NORM_QUANT_READ,
- LayerCommunicator,
Layout,
TokenAxis,
)
+from sglang.srt.layers.communicator.boundary import StageEntry
from sglang.srt.layers.communicator.ops import _consumer_step, _read_input
-from sglang.srt.layers.communicator.residual import batch as residual_batch
from sglang.srt.layers.dp_attention import DpPaddingMode
from sglang.srt.lora.backend.base_backend import BaseLoRABackend
from sglang.srt.lora.backend.triton_backend import (
@@ -229,13 +231,11 @@ def test_communicator_publishes_layout_at_each_transition(
if not publish_lora_layout:
# Without LoRA under DP attention, the communicator leaves the flag alone.
expected_mlp = expected_attn = initial
- communicator = LayerCommunicator.__new__(LayerCommunicator)
+ communicator = stub_plan()
communicator._publish_lora_layout = publish_lora_layout
- communicator.layer_facts = SimpleNamespace(is_first_layer=False)
- communicator._context = SimpleNamespace()
+ communicator.is_first_layer = False
communicator._sp_steps = None
- communicator.post_attention_layernorm = None
- communicator.input_layernorm = lambda x: x
+ communicator.norm = lambda x: x
communicator.qkv_latent_func = None
gathered, local = Layout(frozenset()), Layout(frozenset({TokenAxis.ATTN_DP}))
# The rows of the steps the batch runs decide, not the ordinary steps'.
@@ -243,8 +243,8 @@ def test_communicator_publishes_layout_at_each_transition(
ffn=SimpleNamespace(input_rows=local if gathered_over_dp else gathered)
)
selected = SimpleNamespace(
- attention=SimpleNamespace(
- input_sum=None,
+ attention=StageEntry(
+ input_rows=local,
prepare=partial(
_consumer_step,
step=partial(
@@ -260,8 +260,7 @@ def test_communicator_publishes_layout_at_each_transition(
input_move=lambda hidden_states, **kwargs: hidden_states,
handoff=lambda hidden_states, *args: hidden_states,
),
- ffn=SimpleNamespace(
- input_sum=None,
+ ffn=StageEntry(
prepare=lambda hidden_states, residual, *args, **kwargs: (
hidden_states,
residual,
@@ -275,13 +274,11 @@ def test_communicator_publishes_layout_at_each_transition(
hidden = torch.zeros(num_tokens, 4)
with get_forward().scoped(lora_batch_layout=initial):
for _ in range(2):
- batch = SimpleNamespace(residual_stream=None)
- stream = residual_batch.start(batch)
- stream.write(hidden)
- communicator.prepare_mlp(hidden, stream, batch)
+ stub_stage(communicator, StageKind.FFN)._prepare_input(hidden, hidden, None)
assert get_forward().lora_batch_layout is expected_mlp
- stream = residual_batch.start(batch)
- communicator.prepare_attn(hidden, stream, batch)
+ stub_stage(communicator, StageKind.ATTENTION)._prepare_input(
+ hidden, None, None
+ )
assert get_forward().lora_batch_layout is expected_attn
diff --git a/test/registered/unit/models/test_aux_capture_deferred_allreduce.py b/test/registered/unit/models/test_aux_capture_deferred_allreduce.py
index 056afaace1..64c2345259 100644
--- a/test/registered/unit/models/test_aux_capture_deferred_allreduce.py
+++ b/test/registered/unit/models/test_aux_capture_deferred_allreduce.py
@@ -8,6 +8,7 @@ from torch import nn
from sglang.srt.layers import communicator as comm
from sglang.srt.layers.communicator import ops as comm_ops
+from sglang.srt.layers.communicator.residual.stream import ResidualStream
from sglang.srt.model_executor.forward_batch_info import ForwardMode, PPProxyTensors
from sglang.srt.models.bailing_moe import BailingMoEModel
from sglang.srt.models.bailing_moe_v3 import BailingMoELinearModel
@@ -16,6 +17,9 @@ from sglang.srt.models.glm4_moe_lite import Glm4MoeLiteModel
from sglang.srt.models.glm5_next import Glm5NextModel
from sglang.srt.models.gpt_oss import GptOssModel
from sglang.srt.models.laguna import LagunaModel
+from sglang.srt.models.llama4 import Llama4Model
+from sglang.srt.models.qwen3 import Qwen3Model
+from sglang.srt.models.qwen3_vl import Qwen3LLMModel
from sglang.srt.models.qwen3_vl_moe import Qwen3MoeLLMModel
from sglang.test.ci.ci_register import register_cpu_ci
from sglang.test.test_utils import CustomTestCase
@@ -33,6 +37,7 @@ MODELS = (
LagunaModel,
Qwen3MoeLLMModel,
)
+CAPTURE_MODELS = (*MODELS, Llama4Model, Qwen3Model, Qwen3LLMModel)
def all_reduce(hidden_states):
@@ -44,10 +49,13 @@ GROUP = SimpleNamespace(all_reduce=all_reduce)
class DeferringLayer(nn.Module):
- def __init__(self, defer, return_topk=False):
+ def __init__(self, defer, return_topk=False, stage_api=False):
super().__init__()
self.return_topk = return_topk
+ self.stage_api = stage_api
self.layer_communicator = comm.LayerCommunicator.__new__(comm.LayerCommunicator)
+ self.layer_communicator.input_layernorm = None
+ self.attn_stage = self.layer_communicator.attn
self.layer_communicator._steps = comm.BoundarySteps(
attention=comm.StageEntry(
prepare=partial(
@@ -106,21 +114,27 @@ class DeferringLayer(nn.Module):
**kwargs,
):
stream = forward_batch.residual_stream
- assert residual is stream
- hidden_states, residual = stream.input(hidden_states)
+ if stream is not None:
+ residual = stream
+ if isinstance(residual, ResidualStream):
+ hidden_states, residual = residual.input(hidden_states)
hidden_states = comm.reduce_output(hidden_states)
if residual is None:
residual = hidden_states.clone()
else:
# later norms mutate the residual; captured snapshots must stay intact
residual.add_(hidden_states)
- stream.write(residual)
capture = kwargs.get("capture_output")
if capture is not None:
capture(residual.clone())
with self.layer_communicator.ffn_exit(forward_batch) as ffn_exit:
partial = torch.full_like(hidden_states, 0.5)
- hidden_states, residual = ffn_exit.finish(partial, stream)
+ if stream is not None:
+ stream.write(residual)
+ residual = stream
+ hidden_states, residual = ffn_exit.finish(partial, residual)
+ if self.stage_api:
+ return (hidden_states, None) if self.return_topk else hidden_states
if self.return_topk:
return hidden_states, residual, None
return hidden_states, residual
@@ -151,7 +165,17 @@ def build_model(model_cls, *, defer, capture):
model.layers_to_capture = [0, 1, 2] if capture else []
model.layers = nn.ModuleList(
DeferringLayer(
- defer and i < NUM_LAYERS - 1, return_topk=model_cls is Glm5NextModel
+ defer and i < NUM_LAYERS - 1,
+ return_topk=model_cls is Glm5NextModel,
+ stage_api=model_cls
+ in (
+ BailingMoEModel,
+ BailingMoELinearModel,
+ GptOssModel,
+ Llama4Model,
+ Qwen3Model,
+ Qwen3LLMModel,
+ ),
)
for i in range(NUM_LAYERS)
)
@@ -184,7 +208,7 @@ class TestAuxCaptureDeferredAllreduce(CustomTestCase):
forward_mode=ForwardMode.DECODE,
capture_hidden_mode=SimpleNamespace(need_capture=lambda: True),
)
- for model_cls in MODELS:
+ for model_cls in CAPTURE_MODELS:
for defer in (False, True):
for capture in (False, True):
with (
diff --git a/test/registered/unit/models/test_deepstack_deferred_allreduce.py b/test/registered/unit/models/test_deepstack_deferred_allreduce.py
index 8790f9f4ac..78340fa056 100644
--- a/test/registered/unit/models/test_deepstack_deferred_allreduce.py
+++ b/test/registered/unit/models/test_deepstack_deferred_allreduce.py
@@ -1,3 +1,5 @@
+from sglang.srt.layers.communicator.residual.add_norm import ADD
+
"""Deepstack visual embeddings are added to a decoder layer's output. When the
layer left its FFN all-reduce to the next layer, that output is one rank's
partial sum, and an embedding added to it is counted once per rank."""
@@ -9,7 +11,7 @@ import torch
from torch import nn
from sglang.srt.layers.communicator import LayerCommunicator, UnreducedOutput
-from sglang.srt.layers.communicator.residual.add_norm import ADD
+from sglang.srt.layers.communicator.residual import batch as residual_batch
from sglang.test.ci.ci_register import register_cpu_ci
from sglang.test.test_utils import CustomTestCase
@@ -56,6 +58,20 @@ class DeferringLayer(nn.Module):
return stream.leave(output, ADD), stream
+class DeferringStageLayer(DeferringLayer):
+ def forward(self, positions=None, hidden_states=None, forward_batch=None, **kwargs):
+ stream = residual_batch.current(forward_batch)
+ hidden_states, residual = stream.finish(hidden_states)
+ residual = hidden_states if residual is None else hidden_states + residual
+ stream.write(residual)
+ output = (
+ torch.ones_like(residual)
+ if self.is_last_layer
+ else UnreducedOutput(torch.full_like(residual, 1 / TP_SIZE), group=GROUP)
+ )
+ return stream.leave(output, ADD)
+
+
class SumNorm(nn.Module):
def forward(self, hidden_states, residual=None, post_residual_addition=None):
if residual is None:
@@ -158,9 +174,14 @@ class TestSplitPrefillCompletion(CustomTestCase):
with self.subTest(model=model_cls.__name__, tokens=tokens):
# Also leave the final output: the stack exit must handle it
# independently of the last layer's fusion decision.
+ stage_api = model_cls in (
+ Qwen3ForCausalLM,
+ SarvamMoEForCausalLM,
+ )
+ layer_type = DeferringStageLayer if stage_api else DeferringLayer
wrapper = SimpleNamespace(
model=SimpleNamespace(
- layers=[DeferringLayer(False) for _ in range(NUM_LAYERS)],
+ layers=[layer_type(False) for _ in range(NUM_LAYERS)],
config=SimpleNamespace(num_hidden_layers=NUM_LAYERS),
norm=SumNorm(),
),
diff --git a/test/registered/unit/models/test_last_layer_communicator.py b/test/registered/unit/models/test_last_layer_communicator.py
index 4f230a5b6b..2f213d28f3 100644
--- a/test/registered/unit/models/test_last_layer_communicator.py
+++ b/test/registered/unit/models/test_last_layer_communicator.py
@@ -160,13 +160,10 @@ CASES = {
def build(case, num_layers, layer_id, config=None, **kwargs):
- """Construct one decoder layer with its submodules stubbed. Returns the
- LayerCommunicator kwargs, the LayerFacts.init_new kwargs, and the
- names of the stubbed submodules it built."""
+ """Construct a decoder with stubbed compute and inspect its stage declarations."""
module_name, class_name, make_config, stubs, _ = CASES[case]
module = import_model(module_name)
- communicator = MagicMock()
- facts = MagicMock()
+ communicator = MagicMock(return_value=(MagicMock(), MagicMock()))
built = []
def recording_stub(name):
@@ -176,7 +173,17 @@ def build(case, num_layers, layer_id, config=None, **kwargs):
return make
- patches = dict(LayerCommunicator=communicator, LayerFacts=facts)
+ from sglang.srt.layers.communicator import declare_attn, declare_ffn
+
+ if hasattr(module, "make_stages"):
+ patches = dict(
+ declare_attn=declare_attn,
+ declare_ffn=declare_ffn,
+ make_stages=communicator,
+ )
+ else:
+ facts = MagicMock()
+ patches = dict(LayerCommunicator=communicator, LayerFacts=facts)
patches.update({name: recording_stub(name) for name in stubs})
if hasattr(module, "get_parallel"):
patches["get_parallel"] = lambda: PARALLEL
@@ -185,7 +192,29 @@ def build(case, num_layers, layer_id, config=None, **kwargs):
config or make_config(num_layers), layer_id=layer_id, **kwargs
)
communicator.assert_called_once()
- return communicator.call_args.kwargs, facts.init_new.call_args.kwargs, built
+ if not hasattr(module, "make_stages"):
+ passed = communicator.call_args.kwargs
+ planned = facts.init_new.call_args.kwargs
+ return (
+ passed,
+ {
+ "terminal": planned["layer_id"] == planned["num_layers"] - 1,
+ "enters_stack": planned["layer_id"] == 0,
+ "is_layer_sparse": planned["is_layer_sparse"],
+ },
+ built,
+ )
+ declaration = communicator.call_args.args[1][0]
+ sequence = communicator.call_args.kwargs
+ return (
+ {},
+ {
+ "terminal": sequence["terminal"],
+ "enters_stack": sequence["previous"] is None,
+ "is_layer_sparse": declaration.sparse,
+ },
+ built,
+ )
def planned_as_last(case, num_layers, layer_id, **kwargs):
@@ -193,10 +222,10 @@ def planned_as_last(case, num_layers, layer_id, **kwargs):
fact the communicator reads the last layer from."""
passed, planned, _ = build(case, num_layers, layer_id, **kwargs)
assert "is_last_layer" not in passed, "the plan is the only source"
- return planned["layer_id"] == planned["num_layers"] - 1
+ return planned["terminal"]
-class TestLastLayerCommunicator(CustomTestCase):
+class TestTerminalStages(CustomTestCase):
def test_only_the_last_layer_is_last(self):
"""With all-reduce fusion on, only the model's last layer is marked last,
so its FFN all-reduce is never left to a next layer that does not exist."""
@@ -241,7 +270,8 @@ class TestLastLayerCommunicator(CustomTestCase):
num_layers, layer_id = draft
with self.subTest(case=case):
_, planned, _ = build(case, num_layers, layer_id, is_nextn=True)
- self.assertEqual((planned["layer_id"], planned["num_layers"]), (0, 1))
+ self.assertTrue(planned["enters_stack"])
+ self.assertTrue(planned["terminal"])
def test_bailing_draft_layer_builds_an_moe(self):
"""The Bailing V2 NextN checkpoint holds expert weights, so the NextN
diff --git a/test/registered/unit/models/test_layer_stack_exit.py b/test/registered/unit/models/test_layer_stack_exit.py
index cd3258cd9d..cffcbaf825 100644
--- a/test/registered/unit/models/test_layer_stack_exit.py
+++ b/test/registered/unit/models/test_layer_stack_exit.py
@@ -58,25 +58,10 @@ def delegates_to_super(forward):
)
-def declares_no_deferral(node):
- """Builds its communicator with allow_deferred_ffn_reduction=False: its FFN
- completes its own reduction."""
- return any(
- keyword.arg == "allow_deferred_ffn_reduction"
- and isinstance(keyword.value, ast.Constant)
- and keyword.value.value is False
- for call in ast.walk(node)
- if isinstance(call, ast.Call)
- for keyword in call.keywords
- )
-
-
def defers(node):
"""Asks whether to leave the reduction to the next layer (directly or
through ffn_exit), or wraps its output as unreduced itself."""
- if declares_no_deferral(node):
- return False
- return any(any(calls(node, name)) for name in ("ffn_exit", UNREDUCED))
+ return any(any(calls(node, name)) for name in ("ffn_exit", "exit", UNREDUCED))
class Census:
@@ -115,7 +100,13 @@ class Census:
and any(
isinstance(call, ast.Call)
and isinstance(call.func, ast.Name)
- and call.func.id in {"LayerCommunicator", "MHCLayerCommunicator"}
+ and call.func.id
+ in {
+ "LayerCommunicator",
+ "MHCLayerCommunicator",
+ "make_decoder_stages",
+ "make_stages",
+ }
for call in ast.walk(node)
)
)
@@ -129,7 +120,6 @@ class Census:
if name not in deferring
for _, node in defs
if set(base_names(node)) & deferring
- and (self.include_complete or not declares_no_deferral(node))
and (
method(node, "forward") is None
or delegates_to_super(method(node, "forward"))
@@ -299,6 +289,19 @@ class TestLayerStackExit(CustomTestCase):
for name, (forward, layers) in sorted(self.complete_census.subjects().items()):
exits = list(calls(forward, EXIT)) + list(calls(forward, "take_output"))
if not exits:
+ batch_exits = [
+ call
+ for call in ast.walk(forward)
+ if isinstance(call, ast.Call)
+ and isinstance(call.func, ast.Attribute)
+ and isinstance(call.func.value, ast.Name)
+ and call.func.value.id == "residual_batch"
+ and call.func.attr in {"norm", "to_pp", "fold", "finish"}
+ ]
+ if batch_exits and all(
+ not inside_loop(forward, call) for call in batch_exits
+ ):
+ continue
if delegates_to_super(forward):
continue
if layers and all(
@@ -387,6 +390,13 @@ class TestLayerStackExit(CustomTestCase):
exits = list(calls(forward, EXIT)) + list(
calls(forward, "take_output")
)
+ exits += [
+ call
+ for call in calls(forward, "norm")
+ if isinstance(call.func, ast.Attribute)
+ and isinstance(call.func.value, ast.Name)
+ and call.func.value.id == "residual_batch"
+ ]
self.assertEqual(len(exits), 1)
exit_call = exits[0]
self.assertFalse(inside_loop(forward, exit_call))
@@ -412,7 +422,11 @@ class TestLayerStackExit(CustomTestCase):
]
self.assertEqual(len(final_segments) + len(early_returns), 1)
for call in ast.walk(forward):
- if isinstance(call, ast.Call) and is_final_norm(call):
+ if (
+ isinstance(call, ast.Call)
+ and call is not exit_call
+ and is_final_norm(call)
+ ):
self.assertLess(exit_call.lineno, call.lineno)
checked += 1
self.assertGreater(checked, 0)
diff --git a/test/registered/unit/models/test_nemotron_h_aux_capture.py b/test/registered/unit/models/test_nemotron_h_aux_capture.py
index e99f74ccce..23bc3b77db 100644
--- a/test/registered/unit/models/test_nemotron_h_aux_capture.py
+++ b/test/registered/unit/models/test_nemotron_h_aux_capture.py
@@ -55,7 +55,7 @@ def _build(pattern, tp, capture):
layer = cls.__new__(cls)
nn.Module.__init__(layer)
layer.norm = _Norm()
- layer._init_layer_communicator(config, i)
+ layer._init_stage_boundary(config, i)
layer.mixer = _Mixer(0.5 if kind in "M*" else 0.25, tp)
if kind == "M":
layer._forward_mamba = lambda h, batch, mixer=layer.mixer: mixer(h)
diff --git a/test/registered/unit/models/test_nemotron_h_mtp.py b/test/registered/unit/models/test_nemotron_h_mtp.py
index d6de48d818..ac58059b9d 100644
--- a/test/registered/unit/models/test_nemotron_h_mtp.py
+++ b/test/registered/unit/models/test_nemotron_h_mtp.py
@@ -26,9 +26,9 @@ class _RecordingLayer(nn.Module):
super().__init__()
self.inputs_embeds = None
- def forward(self, *, inputs_embeds, hidden_states, residual, forward_batch):
+ def forward(self, *, inputs_embeds, hidden_states, forward_batch):
self.inputs_embeds = inputs_embeds
- return hidden_states, residual
+ return hidden_states
class TestNemotronHMultiTokenPredictor(CustomTestCase):
diff --git a/test/registered/unit/models/test_nemotron_h_mtp_reduction.py b/test/registered/unit/models/test_nemotron_h_mtp_reduction.py
index 7cdb7b6f0e..23b4551181 100644
--- a/test/registered/unit/models/test_nemotron_h_mtp_reduction.py
+++ b/test/registered/unit/models/test_nemotron_h_mtp_reduction.py
@@ -5,9 +5,9 @@ from unittest.mock import Mock
import torch
from torch import nn
-from sglang.srt.layers.communicator import ADD
from sglang.srt.layers.communicator.layout import SumGroup
-from sglang.srt.layers.communicator.residual import batch as residual_batch
+from sglang.srt.layers.communicator.residual.add_norm import ADD
+from sglang.srt.layers.communicator.residual.stream import ResidualStream
from sglang.srt.model_executor.forward_batch_info import ForwardMode
from sglang.srt.models import nemotron_h_mtp
from sglang.srt.runtime_context import get_context, get_flags, get_parallel
@@ -72,30 +72,28 @@ class TestNemotronMTPReduction(CustomTestCase):
layer.has_end_norm = False
layer.mixer = nn.Identity()
layer.norm = _Norm()
- layer._init_layer_communicator(
+ layer._init_stage_boundary(
SimpleNamespace(hybrid_override_pattern="*E"), 1
)
partial = torch.tensor([[1.0, 2.0], [3.0, 4.0]])
residual = torch.tensor([[7.0, 3.0], [5.0, 9.0]])
expected = partial * tp + residual
+ stream = ResidualStream(residual)
+ hidden = stream.leave(
+ partial, ADD, declared_sum=SumGroup.ATTN_TP if tp > 1 else None
+ )
batch = SimpleNamespace(
forward_mode=ForwardMode.DECODE,
input_ids=torch.zeros(2, dtype=torch.long),
- residual_stream=None,
+ residual_stream=stream,
)
- stream = residual_batch.start(batch)
- stream.write(residual)
- partial = stream.leave(
- partial, ADD, declared_sum=SumGroup.ATTN_TP if tp > 1 else None
- )
- hidden, output_residual = layer(
- inputs_embeds=torch.zeros_like(residual),
- hidden_states=partial,
- residual=stream,
+ hidden = layer(
+ inputs_embeds=torch.zeros_like(partial),
+ hidden_states=hidden,
forward_batch=batch,
)
torch.testing.assert_close(hidden, expected)
- torch.testing.assert_close(output_residual.residual, expected)
+ torch.testing.assert_close(batch.residual_stream.residual, expected)
self.assertEqual(reduce.call_count, int(tp > 1))
diff --git a/test/registered/unit/models/test_nemotron_h_stages.py b/test/registered/unit/models/test_nemotron_h_stages.py
index 62c997ecf3..033cbe1f03 100644
--- a/test/registered/unit/models/test_nemotron_h_stages.py
+++ b/test/registered/unit/models/test_nemotron_h_stages.py
@@ -10,7 +10,6 @@ from sglang.srt.layers.communicator import (
NORM_QUANT_READ,
NORM_READ,
Layout,
- MixerExit,
StageDecl,
StageInput,
StageOutput,
@@ -19,6 +18,7 @@ from sglang.srt.layers.communicator import (
UnreducedOutput,
stage_edges,
)
+from sglang.srt.layers.communicator.layer import MixerExit
from sglang.srt.layers.moe.utils import should_skip_mlp_all_reduce
from sglang.srt.models import nemotron_h_utils as utils
from sglang.srt.runtime_context import get_parallel
@@ -164,11 +164,10 @@ class TestMixerExit(CustomTestCase):
)
communicator = SimpleNamespace(
_batch_steps=lambda batch: SimpleNamespace(ffn_output=produced),
- _context=SimpleNamespace(tp_size=2),
_ffn_sum_can_move_to_next_layer=MagicMock(return_value=movable),
)
hidden = torch.ones(2, 4)
- with get_parallel().override(tp_group=tp_group):
+ with get_parallel().override(tp_group=tp_group, tp_size=2):
with MixerExit(communicator, None) as mixer_exit:
skipped = should_skip_mlp_all_reduce()
output = mixer_exit.finish(hidden)
diff --git a/test/registered/unit/models/test_step3p5_dense_reduce_scatter.py b/test/registered/unit/models/test_step3p5_dense_reduce_scatter.py
index ed0f3eef67..e57e62e3c4 100644
--- a/test/registered/unit/models/test_step3p5_dense_reduce_scatter.py
+++ b/test/registered/unit/models/test_step3p5_dense_reduce_scatter.py
@@ -1,3 +1,5 @@
+from sglang.srt.layers.communicator.layer import FfnExit
+
"""A Step-3.5 dense layer must leave its all-reduce out when postprocess
reduce-scatters its output; otherwise the output is summed twice. The layer's
FFN exit publishes the decision while the MLP runs, then completes it."""
@@ -47,7 +49,7 @@ def dense_layer(*, reduce_scatter):
complete=complete,
),
)
- communicator.ffn_exit = lambda fb: comm.FfnExit(communicator, fb)
+ communicator.ffn_exit = lambda fb: FfnExit(communicator, fb)
object.__setattr__(layer, "layer_communicator", communicator)
object.__setattr__(layer, "use_moe", False)
object.__setattr__(layer, "mlp", mlp)
diff --git a/test/registered/unit/spec/test_draft_construction_isolation.py b/test/registered/unit/spec/test_draft_construction_isolation.py
index bf0aedd93e..85bf84b575 100644
--- a/test/registered/unit/spec/test_draft_construction_isolation.py
+++ b/test/registered/unit/spec/test_draft_construction_isolation.py
@@ -204,5 +204,39 @@ class TestDraftWeightUpdateRecord(CustomTestCase):
self.assertEqual(get_model().model_path, seeded.model_path)
+class TestDraftBoundaryPolicy(unittest.TestCase):
+ def test_scope_restores_target_policy_after_nested_failure(self):
+ from sglang.srt.runtime_context import (
+ get_exec,
+ get_spec,
+ publish,
+ reset_context,
+ )
+ from sglang.srt.server_args import ServerArgs
+
+ reset_context()
+ try:
+ publish(
+ ServerArgs(
+ model_path="dummy",
+ boundary_reduction="rs+rsv",
+ ),
+ role="test",
+ )
+ self.assertEqual(get_exec().comm.boundary_reduction, "rs+rsv")
+ with (
+ get_spec().override(speculative_boundary_reduction="ar"),
+ self.assertRaisesRegex(RuntimeError, "draft failed"),
+ ):
+ with draft_model_build_scope():
+ self.assertEqual(get_exec().comm.boundary_reduction, "ar")
+ with draft_model_build_scope():
+ self.assertEqual(get_exec().comm.boundary_reduction, "ar")
+ raise RuntimeError("draft failed")
+ self.assertEqual(get_exec().comm.boundary_reduction, "rs+rsv")
+ finally:
+ reset_context()
+
+
if __name__ == "__main__":
unittest.main()
diff --git a/test/registered/unit/test_model_overrides.py b/test/registered/unit/test_model_overrides.py
index 637a9cadcd..2be2dc164c 100644
--- a/test/registered/unit/test_model_overrides.py
+++ b/test/registered/unit/test_model_overrides.py
@@ -106,6 +106,8 @@ class TestModelOverridableWhitelist(CustomTestCase):
"fp8_gemm_runner_backend",
"fp4_gemm_runner_backend",
"disable_custom_all_reduce",
+ "boundary_reduction",
+ "speculative_boundary_reduction",
"enable_aiter_allreduce_fusion",
"disable_aiter_allreduce_fusion_in_prefill",
"disable_aiter_allreduce_fusion_in_decode",
@@ -121,6 +123,119 @@ class TestModelOverridableWhitelist(CustomTestCase):
)
+class TestBoundaryReductionDefaults(CustomTestCase):
+ def test_defaults_wrappers_and_explicit_policies_reach_runtime(self):
+ from unittest.mock import patch
+
+ from sglang.srt.arg_groups.boundary_reduction import resolve_boundary_reduction
+ from sglang.srt.arg_groups.overrides import run_post_process_pass
+ from sglang.srt.runtime_context import (
+ get_exec,
+ get_spec,
+ publish,
+ reset_context,
+ )
+ from sglang.srt.server_args import ServerArgs
+
+ cases = [
+ (SimpleNamespace(architectures=[architecture]), default)
+ for architecture, default in (
+ ("Qwen3ForCausalLM", "ar"),
+ ("Qwen3Model", "ar"),
+ ("MossVLForConditionalGeneration", "ar"),
+ ("BailingMoELinearForCausalLM", "rsv"),
+ ("BailingMoeV2_5ForCausalLM", "rsv"),
+ ("LongcatFlashForCausalLM", "rsv"),
+ ("LongcatFlashForCausalLMNextN", "rsv"),
+ ("Step3VLForConditionalGeneration", "rsv"),
+ ("MiMoV2ForCausalLM", "rs+rsv"),
+ ("Qwen3MoeForCausalLM", "rs+rsv"),
+ ("OtherModel", "rs+rsv"),
+ )
+ ]
+ for key in ("text_config", "llm_config"):
+ for backbone, expected in (("qwen3", "ar"), ("qwen3_moe", "rs+rsv")):
+ cases.append(
+ (
+ SimpleNamespace(**{key: SimpleNamespace(model_type=backbone)}),
+ expected,
+ )
+ )
+ cases.append(
+ (
+ SimpleNamespace(
+ get_text_config=lambda: SimpleNamespace(model_type="qwen3")
+ ),
+ "ar",
+ )
+ )
+ for config, default in cases:
+ for requested in ("auto", "ar", "rs", "rsv", "rs+rsv"):
+ with self.subTest(config=config, requested=requested):
+ reset_context()
+ try:
+ args = ServerArgs(
+ model_path="local-model", boundary_reduction=requested
+ )
+ args._model_config = SimpleNamespace(hf_config=config)
+ with patch(
+ "sglang.srt.arg_groups.pipeline.run_resolution_pipeline",
+ side_effect=lambda record: run_post_process_pass(
+ record, resolve_boundary_reduction
+ ),
+ ):
+ publish(args, role="test")
+ expected = default if requested == "auto" else requested
+ self.assertEqual(get_exec().comm.boundary_reduction, expected)
+ self.assertEqual(
+ get_spec().speculative_boundary_reduction, expected
+ )
+ self.assertEqual(args.boundary_reduction, requested)
+ finally:
+ reset_context()
+
+ def test_independent_draft_and_mtp_resolution(self):
+ from unittest.mock import patch
+
+ from sglang.srt.arg_groups.boundary_reduction import resolve_boundary_reduction
+ from sglang.srt.arg_groups.overrides import (
+ resolving_view,
+ run_post_process_pass,
+ )
+ from sglang.srt.server_args import ServerArgs
+
+ for same_model in (False, True):
+ for requested in ("auto", "ar", "rs", "rsv", "rs+rsv"):
+ args = ServerArgs(
+ model_path="target",
+ boundary_reduction=requested,
+ speculative_algorithm="EAGLE",
+ speculative_draft_model_path="target" if same_model else "draft",
+ )
+ args._model_config = SimpleNamespace(
+ hf_config=SimpleNamespace(architectures=["MiMoV2ForCausalLM"])
+ )
+ with patch(
+ "sglang.srt.utils.hf_transformers_utils.get_config",
+ return_value=SimpleNamespace(model_type="qwen3"),
+ ) as load:
+ run_post_process_pass(args, resolve_boundary_reduction)
+ resolved = resolving_view(args)
+ self.assertEqual(
+ resolved.boundary_reduction,
+ "rs+rsv" if requested == "auto" else requested,
+ )
+ self.assertEqual(
+ resolved.speculative_boundary_reduction,
+ ("rs+rsv" if same_model else "ar")
+ if requested == "auto"
+ else requested,
+ )
+ self.assertEqual(
+ load.call_count, int(requested == "auto" and not same_model)
+ )
+
+
class TestDSparkCheckpointConfig(CustomTestCase):
def test_sample_from_anchor_is_read_from_checkpoint_config(self):
from sglang.srt.speculative.dspark_components.dspark_config import (