diff --git a/docs/docs/references/nightly_precision_regression.mdx b/docs/docs/references/nightly_precision_regression.mdx index 70a4c8cde1..9371200c46 100644 --- a/docs/docs/references/nightly_precision_regression.mdx +++ b/docs/docs/references/nightly_precision_regression.mdx @@ -101,7 +101,7 @@ A `capture_signature` (SHA-1 hash of schema version, max_tokens, ignore_eos, TP The harness pins FlashInfer all-reduce fusion **on** (`--flashinfer-allreduce-fusion-backend trtllm`; the SM90 auto-enable was dropped in #23402). Two names are captured per layer: - `inputs.1` — hidden states entering the layer. The FFN boundary defers the cross-layer all-reduce, so with fusion active these are rank-local TP-partial sums. CPU-side reduction of them is not the kernel's semantic output and drifts across self-hosted runners, so they are **not** compared. `_assert_fused_tp_layout()` keeps them only as a guard: if they come back replicated, fusion silently fell back and the test fails. -- `self_attn.inputs.hidden_states` — the value attention consumes after `attn_stage.prepare()`, i.e. post fused all-reduce and residual RMSNorm. This is replicated across ranks and is what the comparator actually diffs. +- `self_attn.inputs.hidden_states` — the value attention consumes after `attn_boundary.prepare()`, i.e. post fused all-reduce and residual RMSNorm. This is replicated across ranks and is what the comparator actually diffs. This is why a stale baseline captured with fusion *not* initialized is incompatible: it holds replicated layer-entry tensors where the target holds TP-partial ones, which reads as an approximately `tp_size`x mismatch. Fusion initialization failures are tracked in [FlashInfer #3676](https://github.com/flashinfer-ai/flashinfer/issues/3676) and [SGLang #30875](https://github.com/sgl-project/sglang/issues/30875). diff --git a/python/sglang/srt/batch_overlap/two_batch_overlap.py b/python/sglang/srt/batch_overlap/two_batch_overlap.py index fc207eefb2..86c863307c 100644 --- a/python/sglang/srt/batch_overlap/two_batch_overlap.py +++ b/python/sglang/srt/batch_overlap/two_batch_overlap.py @@ -14,13 +14,13 @@ from sglang.srt.batch_overlap.operations import ( ) from sglang.srt.batch_overlap.operations_strategy import OperationsStrategy from sglang.srt.layers import deep_gemm_wrapper -from sglang.srt.layers.communicator import ( +from sglang.srt.layers.layer_boundary import ( Layout, tbo_split_moves, ) -from sglang.srt.layers.communicator.residual import batch as residual_batch -from sglang.srt.layers.communicator.residual.access import finish_layer_stack -from sglang.srt.layers.communicator.residual.stream import ResidualStream +from sglang.srt.layers.layer_boundary.residual import batch as residual_batch +from sglang.srt.layers.layer_boundary.residual.access import finish_layer_stack +from sglang.srt.layers.layer_boundary.residual.stream import ResidualStream from sglang.srt.layers.moe import ( get_deepep_mode, get_moe_a2a_backend, @@ -959,7 +959,7 @@ def model_forward_stages( parts = _model_forward_tbo_split_inputs( **inputs, residual=residual, - layer_input_rows=layers[0].attn_stage.incoming_residual_rows, + layer_input_rows=layers[0].attn_boundary.incoming_residual_rows, ) for part in parts: part["hidden_states"], child_stream = ResidualStream.arrive( diff --git a/python/sglang/srt/hardware_backend/npu/modules/deepseek_v2_attention_mla_npu.py b/python/sglang/srt/hardware_backend/npu/modules/deepseek_v2_attention_mla_npu.py index 9905cdb07a..768cec6d13 100644 --- a/python/sglang/srt/hardware_backend/npu/modules/deepseek_v2_attention_mla_npu.py +++ b/python/sglang/srt/hardware_backend/npu/modules/deepseek_v2_attention_mla_npu.py @@ -15,7 +15,7 @@ from sglang.srt.layers.attention.dsa.dsa_npu_indexer import scattered_to_tp_attn from sglang.srt.layers.attention.dsa.utils import ( dsa_use_prefill_cp, ) -from sglang.srt.layers.communicator import get_attn_tp_context +from sglang.srt.layers.layer_boundary import get_attn_tp_context from sglang.srt.model_executor.forward_context import get_token_to_kv_pool from sglang.srt.runtime_context import get_disagg from sglang.srt.state_capturer.indexer_topk import maybe_capture_indexer_topk diff --git a/python/sglang/srt/layers/communicator/__init__.py b/python/sglang/srt/layers/layer_boundary/__init__.py similarity index 77% rename from python/sglang/srt/layers/communicator/__init__.py rename to python/sglang/srt/layers/layer_boundary/__init__.py index 74c48c9389..d5d0ca7a85 100644 --- a/python/sglang/srt/layers/communicator/__init__.py +++ b/python/sglang/srt/layers/layer_boundary/__init__.py @@ -13,20 +13,20 @@ # ============================================================================== """Stage boundaries: layout transport, residual updates, reads, and fusion.""" -from sglang.srt.layers.communicator.adapters.attention import ( +from sglang.srt.layers.layer_boundary.adapters.attention import ( AttentionInputs, get_attn_tp_context, ) -from sglang.srt.layers.communicator.boundary import ( +from sglang.srt.layers.layer_boundary.boundary import ( make_boundary, make_output_boundary, tbo_split_moves, ) -from sglang.srt.layers.communicator.construction import ( +from sglang.srt.layers.layer_boundary.construction import ( BatchVariant, StageEdges, ) -from sglang.srt.layers.communicator.contracts import ( +from sglang.srt.layers.layer_boundary.contracts import ( EdgeDecl, FusedMlpInput, HandoffRows, @@ -38,15 +38,15 @@ from sglang.srt.layers.communicator.contracts import ( StageOutput, StageSteps, ) -from sglang.srt.layers.communicator.exit import FfnCompletion, FfnExit, MixerExit -from sglang.srt.layers.communicator.factories import ( +from sglang.srt.layers.layer_boundary.exit import FfnCompletion, FfnExit, MixerExit +from sglang.srt.layers.layer_boundary.factories import ( declare_attn, declare_ffn, make_attn_stage, make_ffn_stage, make_stages, ) -from sglang.srt.layers.communicator.layout import ( +from sglang.srt.layers.layer_boundary.layout import ( Layout, SumGroup, TokenAxis, @@ -55,24 +55,24 @@ from sglang.srt.layers.communicator.layout import ( sparse_moe_gathers_over_moe_cp, token_axis_sizes, ) -from sglang.srt.layers.communicator.ops import ( +from sglang.srt.layers.layer_boundary.ops import ( move_rows, tp_reduce_scatter, ) -from sglang.srt.layers.communicator.output import ( +from sglang.srt.layers.layer_boundary.output import ( HandoffOutput, UnreducedOutput, reduce_output, ) -from sglang.srt.layers.communicator.residual import LayerResidual -from sglang.srt.layers.communicator.residual.add_norm import ( +from sglang.srt.layers.layer_boundary.residual import LayerResidual +from sglang.srt.layers.layer_boundary.residual.add_norm import ( ADD, FUSE_ALLREDUCE_MAX_BATCH_SIZE, NORM_QUANT_READ, NORM_READ, PLAIN_RESIDUAL, ) -from sglang.srt.layers.communicator.residual.mhc import ( +from sglang.srt.layers.layer_boundary.residual.mhc import ( MHCState, ) diff --git a/python/sglang/srt/layers/communicator/adapters/__init__.py b/python/sglang/srt/layers/layer_boundary/adapters/__init__.py similarity index 100% rename from python/sglang/srt/layers/communicator/adapters/__init__.py rename to python/sglang/srt/layers/layer_boundary/adapters/__init__.py diff --git a/python/sglang/srt/layers/communicator/adapters/attention.py b/python/sglang/srt/layers/layer_boundary/adapters/attention.py similarity index 99% rename from python/sglang/srt/layers/communicator/adapters/attention.py rename to python/sglang/srt/layers/layer_boundary/adapters/attention.py index bc4c7098f8..cedbdf5d64 100644 --- a/python/sglang/srt/layers/communicator/adapters/attention.py +++ b/python/sglang/srt/layers/layer_boundary/adapters/attention.py @@ -20,14 +20,14 @@ from typing import Callable import torch -from sglang.srt.layers.communicator.layout import ( - enable_moe_dense_fully_dp, -) from sglang.srt.layers.dp_attention import ( attn_tp_all_gather_into_tensor, get_local_dp_buffer, is_dp_attention_enabled, ) +from sglang.srt.layers.layer_boundary.layout import ( + enable_moe_dense_fully_dp, +) from sglang.srt.layers.moe import get_moe_a2a_backend from sglang.srt.model_executor.cuda_graph_config import ( Backend, diff --git a/python/sglang/srt/layers/communicator/adapters/branch.py b/python/sglang/srt/layers/layer_boundary/adapters/branch.py similarity index 93% rename from python/sglang/srt/layers/communicator/adapters/branch.py rename to python/sglang/srt/layers/layer_boundary/adapters/branch.py index 5fa4fd724b..c1b13ec2b7 100644 --- a/python/sglang/srt/layers/communicator/adapters/branch.py +++ b/python/sglang/srt/layers/layer_boundary/adapters/branch.py @@ -19,12 +19,12 @@ from typing import TYPE_CHECKING, Tuple import torch -from sglang.srt.layers.communicator.ops import move_rows -from sglang.srt.layers.communicator.residual.stream import ResidualStream +from sglang.srt.layers.layer_boundary.ops import move_rows +from sglang.srt.layers.layer_boundary.residual.stream import ResidualStream from sglang.srt.model_executor.forward_batch_info import ForwardBatch if TYPE_CHECKING: - from sglang.srt.layers.communicator.construction import StagePlan + from sglang.srt.layers.layer_boundary.construction import StagePlan def branch_input( diff --git a/python/sglang/srt/layers/communicator/adapters/context_parallel.py b/python/sglang/srt/layers/layer_boundary/adapters/context_parallel.py similarity index 100% rename from python/sglang/srt/layers/communicator/adapters/context_parallel.py rename to python/sglang/srt/layers/layer_boundary/adapters/context_parallel.py diff --git a/python/sglang/srt/layers/communicator/adapters/lora.py b/python/sglang/srt/layers/layer_boundary/adapters/lora.py similarity index 95% rename from python/sglang/srt/layers/communicator/adapters/lora.py rename to python/sglang/srt/layers/layer_boundary/adapters/lora.py index f971e31f19..b2648ef2ec 100644 --- a/python/sglang/srt/layers/communicator/adapters/lora.py +++ b/python/sglang/srt/layers/layer_boundary/adapters/lora.py @@ -13,7 +13,7 @@ # ============================================================================== """Publish the token rows consumed by the selected stage to LoRA kernels.""" -from sglang.srt.layers.communicator.layout import TokenAxis +from sglang.srt.layers.layer_boundary.layout import TokenAxis from sglang.srt.runtime_context import LoRABatchLayout, get_forward diff --git a/python/sglang/srt/layers/communicator/adapters/overlap.py b/python/sglang/srt/layers/layer_boundary/adapters/overlap.py similarity index 85% rename from python/sglang/srt/layers/communicator/adapters/overlap.py rename to python/sglang/srt/layers/layer_boundary/adapters/overlap.py index 010547bedc..312fd53745 100644 --- a/python/sglang/srt/layers/communicator/adapters/overlap.py +++ b/python/sglang/srt/layers/layer_boundary/adapters/overlap.py @@ -1,7 +1,7 @@ """Row requirements at the entrance to two-batch overlap.""" -from sglang.srt.layers.communicator.contracts import HandoffRows -from sglang.srt.layers.communicator.layout import enable_moe_dense_fully_dp +from sglang.srt.layers.layer_boundary.contracts import HandoffRows +from sglang.srt.layers.layer_boundary.layout import enable_moe_dense_fully_dp from sglang.srt.runtime_context import get_exec diff --git a/python/sglang/srt/layers/communicator/boundary.py b/python/sglang/srt/layers/layer_boundary/boundary.py similarity index 99% rename from python/sglang/srt/layers/communicator/boundary.py rename to python/sglang/srt/layers/layer_boundary/boundary.py index 49df1be154..a60998a438 100644 --- a/python/sglang/srt/layers/communicator/boundary.py +++ b/python/sglang/srt/layers/layer_boundary/boundary.py @@ -19,14 +19,14 @@ from typing import Callable, Optional, Tuple import msgspec -from sglang.srt.layers.communicator.contracts import ( +from sglang.srt.layers.layer_boundary.contracts import ( CpMoves, EdgeDecl, FusedMlpInput, StageInput, StageOutput, ) -from sglang.srt.layers.communicator.layout import ( +from sglang.srt.layers.layer_boundary.layout import ( Layout, SumGroup, TokenAxis, @@ -35,7 +35,7 @@ from sglang.srt.layers.communicator.layout import ( _sum_group, token_axis_sizes, ) -from sglang.srt.layers.communicator.ops import ( +from sglang.srt.layers.layer_boundary.ops import ( gather_attention_tp, identity_output, move_rows, @@ -49,7 +49,7 @@ from sglang.srt.layers.communicator.ops import ( tp_slice, update_and_gather, ) -from sglang.srt.layers.communicator.prepare import ( +from sglang.srt.layers.layer_boundary.prepare import ( _consumer_step, _dispatch_consumer, _mlp_input_dp_partial, diff --git a/python/sglang/srt/layers/communicator/construction.py b/python/sglang/srt/layers/layer_boundary/construction.py similarity index 94% rename from python/sglang/srt/layers/communicator/construction.py rename to python/sglang/srt/layers/layer_boundary/construction.py index f740d7a336..22567df2b5 100644 --- a/python/sglang/srt/layers/communicator/construction.py +++ b/python/sglang/srt/layers/layer_boundary/construction.py @@ -18,13 +18,13 @@ from functools import cached_property from typing import Callable, Optional from sglang.srt.environ import envs -from sglang.srt.layers.communicator.adapters.attention import get_attn_tp_context -from sglang.srt.layers.communicator.boundary import ( +from sglang.srt.layers.layer_boundary.adapters.attention import get_attn_tp_context +from sglang.srt.layers.layer_boundary.boundary import ( _cp_moves, make_boundary, make_output_boundary, ) -from sglang.srt.layers.communicator.contracts import ( +from sglang.srt.layers.layer_boundary.contracts import ( BatchVariant, CpMoves, EdgeDecl, @@ -33,23 +33,23 @@ from sglang.srt.layers.communicator.contracts import ( StageKind, StageSteps, ) -from sglang.srt.layers.communicator.exit import OutputBoundary -from sglang.srt.layers.communicator.fusions.allreduce import ( +from sglang.srt.layers.layer_boundary.exit import OutputBoundary +from sglang.srt.layers.layer_boundary.fusions.allreduce import ( attention_fusions, ffn_fusions, ) -from sglang.srt.layers.communicator.layout import ( +from sglang.srt.layers.layer_boundary.layout import ( TokenAxis, _batch_shards_over_cp, _gathers_over_attention_cp, _generic_prefill_cp_shards_tokens, enable_moe_dense_fully_dp, ) -from sglang.srt.layers.communicator.prepare import ( +from sglang.srt.layers.layer_boundary.prepare import ( _hand_qkv_hook_its_input, _hand_scattered_input_to_attention, ) -from sglang.srt.layers.communicator.stage import StageCommunicator +from sglang.srt.layers.layer_boundary.stage import StageBoundary from sglang.srt.layers.moe import ( get_moe_a2a_backend, ) @@ -294,10 +294,10 @@ def _bind_stage(declaration, norm, incoming, outgoing, **options): if declaration.kind is StageKind.ATTENTION and outgoing.consumer is not None: # Layout eligibility comes from the connected consumer, not a mutable # link to its execution plan. Kernel binding remains consumer-owned. - from sglang.srt.layers.communicator.boundary import input_rows + from sglang.srt.layers.layer_boundary.boundary import input_rows plan._fusion_rows = { v: input_rows(edge) for v, edge in outgoing.entries.items() } - return StageCommunicator(plan, declaration=declaration) + return StageBoundary(plan, declaration=declaration) diff --git a/python/sglang/srt/layers/communicator/contracts.py b/python/sglang/srt/layers/layer_boundary/contracts.py similarity index 96% rename from python/sglang/srt/layers/communicator/contracts.py rename to python/sglang/srt/layers/layer_boundary/contracts.py index e9c1d3d23b..219b5708a9 100644 --- a/python/sglang/srt/layers/communicator/contracts.py +++ b/python/sglang/srt/layers/layer_boundary/contracts.py @@ -20,10 +20,10 @@ import msgspec import torch from sglang.srt.distributed import GroupCoordinator -from sglang.srt.layers.communicator.layout import Layout, SumGroup, TokenAxis -from sglang.srt.layers.communicator.output import OutputTransform -from sglang.srt.layers.communicator.residual import StageRead, StageUpdate -from sglang.srt.layers.communicator.residual.add_norm import ADD, NORM_READ +from sglang.srt.layers.layer_boundary.layout import Layout, SumGroup, TokenAxis +from sglang.srt.layers.layer_boundary.output import OutputTransform +from sglang.srt.layers.layer_boundary.residual import StageRead, StageUpdate +from sglang.srt.layers.layer_boundary.residual.add_norm import ADD, NORM_READ class ProducerReduction(Enum): diff --git a/python/sglang/srt/layers/communicator/exit.py b/python/sglang/srt/layers/layer_boundary/exit.py similarity index 98% rename from python/sglang/srt/layers/communicator/exit.py rename to python/sglang/srt/layers/layer_boundary/exit.py index a55b5cd0b9..fd4925c3cd 100644 --- a/python/sglang/srt/layers/communicator/exit.py +++ b/python/sglang/srt/layers/layer_boundary/exit.py @@ -23,32 +23,32 @@ import torch from sglang.srt.distributed import GroupCoordinator from sglang.srt.environ import envs -from sglang.srt.layers.communicator.adapters.attention import get_attn_tp_context -from sglang.srt.layers.communicator.layout import ( +from sglang.srt.layers.dp_attention import ( + can_use_dp_reduce_scatter, + is_dp_attention_enabled, + is_enable_moe_cp_allgather, +) +from sglang.srt.layers.layer_boundary.adapters.attention import get_attn_tp_context +from sglang.srt.layers.layer_boundary.layout import ( SumGroup, _ffn_has_tokens, _sum_group, ) -from sglang.srt.layers.communicator.ops import ( +from sglang.srt.layers.layer_boundary.ops import ( _all_reduce_then_to_local_tokens, _redistribute_output, _reduce_and_redistribute_output_max_len, _reduce_and_redistribute_output_varlen, _to_local_tokens, ) -from sglang.srt.layers.communicator.output import ( +from sglang.srt.layers.layer_boundary.output import ( UnreducedOutput, ) -from sglang.srt.layers.communicator.residual.add_norm import ( +from sglang.srt.layers.layer_boundary.residual.add_norm import ( apply_aiter_all_reduce_fusion, apply_flashinfer_allreduce_fusion, ) -from sglang.srt.layers.communicator.residual.stream import ResidualStream -from sglang.srt.layers.dp_attention import ( - can_use_dp_reduce_scatter, - is_dp_attention_enabled, - is_enable_moe_cp_allgather, -) +from sglang.srt.layers.layer_boundary.residual.stream import ResidualStream from sglang.srt.layers.moe import ( can_merge_post_experts_all_reduce, post_experts_sum_is_one_all_reduce, diff --git a/python/sglang/srt/layers/communicator/factories.py b/python/sglang/srt/layers/layer_boundary/factories.py similarity index 97% rename from python/sglang/srt/layers/communicator/factories.py rename to python/sglang/srt/layers/layer_boundary/factories.py index 1babf5e10d..d1fdc28085 100644 --- a/python/sglang/srt/layers/communicator/factories.py +++ b/python/sglang/srt/layers/layer_boundary/factories.py @@ -9,19 +9,19 @@ from typing import Mapping, Optional import msgspec from sglang.srt.layers import layernorm_sp -from sglang.srt.layers.communicator.adapters.overlap import ( +from sglang.srt.layers.layer_boundary.adapters.overlap import ( resolve_handoff_rows, tbo_handoff, ) -from sglang.srt.layers.communicator.boundary import _cp_moves -from sglang.srt.layers.communicator.construction import ( +from sglang.srt.layers.layer_boundary.boundary import _cp_moves +from sglang.srt.layers.layer_boundary.construction import ( BatchVariant, _bind_stage, _input_can_be_scattered, _refuse_uncovered_cp_moe, _use_ag_after_qlora, ) -from sglang.srt.layers.communicator.contracts import ( +from sglang.srt.layers.layer_boundary.contracts import ( EdgeDecl, HandoffRows, ProducerReduction, @@ -30,7 +30,7 @@ from sglang.srt.layers.communicator.contracts import ( StageKind, StageOutput, ) -from sglang.srt.layers.communicator.layout import ( +from sglang.srt.layers.layer_boundary.layout import ( Layout, SumGroup, TokenAxis, @@ -40,9 +40,9 @@ from sglang.srt.layers.communicator.layout import ( sparse_moe_gathers_over_moe_cp, token_axis_sizes, ) -from sglang.srt.layers.communicator.output import OutputTransform -from sglang.srt.layers.communicator.residual import StageRead, StageUpdate -from sglang.srt.layers.communicator.residual.add_norm import ( +from sglang.srt.layers.layer_boundary.output import OutputTransform +from sglang.srt.layers.layer_boundary.residual import StageRead, StageUpdate +from sglang.srt.layers.layer_boundary.residual.add_norm import ( ADD, NORM_QUANT_READ, NORM_READ, diff --git a/python/sglang/srt/layers/communicator/fusions/__init__.py b/python/sglang/srt/layers/layer_boundary/fusions/__init__.py similarity index 100% rename from python/sglang/srt/layers/communicator/fusions/__init__.py rename to python/sglang/srt/layers/layer_boundary/fusions/__init__.py diff --git a/python/sglang/srt/layers/communicator/fusions/allreduce.py b/python/sglang/srt/layers/layer_boundary/fusions/allreduce.py similarity index 95% rename from python/sglang/srt/layers/communicator/fusions/allreduce.py rename to python/sglang/srt/layers/layer_boundary/fusions/allreduce.py index e9e558b059..1b09a73abf 100644 --- a/python/sglang/srt/layers/communicator/fusions/allreduce.py +++ b/python/sglang/srt/layers/layer_boundary/fusions/allreduce.py @@ -19,10 +19,10 @@ from typing import Callable, Optional, Tuple import torch -from sglang.srt.layers.communicator.contracts import FusedMlpInput -from sglang.srt.layers.communicator.layout import SumGroup -from sglang.srt.layers.communicator.output import UnreducedOutput -from sglang.srt.layers.communicator.residual.add_norm import ( +from sglang.srt.layers.layer_boundary.contracts import FusedMlpInput +from sglang.srt.layers.layer_boundary.layout import SumGroup +from sglang.srt.layers.layer_boundary.output import UnreducedOutput +from sglang.srt.layers.layer_boundary.residual.add_norm import ( NORM_QUANT_READ, Fp8Input, NormQuantRead, diff --git a/python/sglang/srt/layers/communicator/fusions/cutedsl.py b/python/sglang/srt/layers/layer_boundary/fusions/cutedsl.py similarity index 98% rename from python/sglang/srt/layers/communicator/fusions/cutedsl.py rename to python/sglang/srt/layers/layer_boundary/fusions/cutedsl.py index 551c4453ea..cd7d5746fa 100644 --- a/python/sglang/srt/layers/communicator/fusions/cutedsl.py +++ b/python/sglang/srt/layers/layer_boundary/fusions/cutedsl.py @@ -13,7 +13,8 @@ from typing import Callable, Optional, Sequence import torch -from sglang.srt.layers.communicator import ( +from sglang.srt.layers.dp_attention import is_dp_attention_enabled +from sglang.srt.layers.layer_boundary import ( FusedMlpInput, HandoffOutput, SumGroup, @@ -21,7 +22,6 @@ from sglang.srt.layers.communicator import ( UnreducedOutput, get_attn_tp_context, ) -from sglang.srt.layers.dp_attention import is_dp_attention_enabled from sglang.srt.layers.layernorm import GemmaRMSNorm, RMSNorm from sglang.srt.layers.moe import get_moe_a2a_backend from sglang.srt.model_executor.forward_batch_info import ForwardBatch, ForwardMode @@ -359,7 +359,7 @@ class CuteDSLFusion: def _fusion_of(layer: torch.nn.Module) -> CuteDSLFusion | None: - fusions = getattr(layer.__dict__.get("ffn_stage"), "fusions", None) + fusions = getattr(layer.__dict__.get("ffn_boundary"), "fusions", None) return fusions if isinstance(fusions, CuteDSLFusion) else None @@ -388,8 +388,8 @@ def install_cutedsl_fusion( # The workspace compiles one epsilon and cannot see a per-layer one. for layer in fusion_layers: for norm in ( - layer.attn_stage.norm, - layer.ffn_stage.norm, + layer.attn_boundary.norm, + layer.ffn_boundary.norm, ): if _fused_norm_gamma(norm) is None: continue @@ -441,7 +441,7 @@ def prepare_cutedsl_fusion( for module in model.modules() # Only decoder modules carry an FFN stage. if isinstance( - fusion := getattr(module.__dict__.get("ffn_stage"), "fusions", None), + fusion := getattr(module.__dict__.get("ffn_boundary"), "fusions", None), CuteDSLFusion, ) ] diff --git a/python/sglang/srt/layers/communicator/layout.py b/python/sglang/srt/layers/layer_boundary/layout.py similarity index 100% rename from python/sglang/srt/layers/communicator/layout.py rename to python/sglang/srt/layers/layer_boundary/layout.py diff --git a/python/sglang/srt/layers/communicator/ops.py b/python/sglang/srt/layers/layer_boundary/ops.py similarity index 97% rename from python/sglang/srt/layers/communicator/ops.py rename to python/sglang/srt/layers/layer_boundary/ops.py index 1c6be8b3e9..171ae1bcf8 100644 --- a/python/sglang/srt/layers/communicator/ops.py +++ b/python/sglang/srt/layers/layer_boundary/ops.py @@ -26,21 +26,6 @@ from sglang.srt.distributed import ( from sglang.srt.distributed.device_communicators.pynccl_allocator import ( use_symmetric_memory, ) -from sglang.srt.layers.communicator.adapters.attention import ( - _redistribute_from_attn_tp_shards, - _redistribute_to_attn_tp_shards, -) -from sglang.srt.layers.communicator.adapters.context_parallel import ( - dsa_cp_reduce_scatter_hidden_states, -) -from sglang.srt.layers.communicator.layout import ( - Layout, - TokenAxis, - _cp_shard_token_rows, - moe_cp_gathered_rows, -) -from sglang.srt.layers.communicator.residual import StageUpdate -from sglang.srt.layers.communicator.residual.add_norm import ADD from sglang.srt.layers.dp_attention import ( attn_tp_all_gather_into_tensor, attn_tp_reduce_scatter_tensor, @@ -55,6 +40,21 @@ from sglang.srt.layers.dp_attention import ( is_allocation_symmetric, moe_cp_all_gather_into_tensor, ) +from sglang.srt.layers.layer_boundary.adapters.attention import ( + _redistribute_from_attn_tp_shards, + _redistribute_to_attn_tp_shards, +) +from sglang.srt.layers.layer_boundary.adapters.context_parallel import ( + dsa_cp_reduce_scatter_hidden_states, +) +from sglang.srt.layers.layer_boundary.layout import ( + Layout, + TokenAxis, + _cp_shard_token_rows, + moe_cp_gathered_rows, +) +from sglang.srt.layers.layer_boundary.residual import StageUpdate +from sglang.srt.layers.layer_boundary.residual.add_norm import ADD from sglang.srt.model_executor.forward_batch_info import ForwardBatch from sglang.srt.runtime_context import get_exec, get_parallel diff --git a/python/sglang/srt/layers/communicator/output.py b/python/sglang/srt/layers/layer_boundary/output.py similarity index 100% rename from python/sglang/srt/layers/communicator/output.py rename to python/sglang/srt/layers/layer_boundary/output.py diff --git a/python/sglang/srt/layers/communicator/prepare.py b/python/sglang/srt/layers/layer_boundary/prepare.py similarity index 96% rename from python/sglang/srt/layers/communicator/prepare.py rename to python/sglang/srt/layers/layer_boundary/prepare.py index 9a452175c6..36ad2f262f 100644 --- a/python/sglang/srt/layers/communicator/prepare.py +++ b/python/sglang/srt/layers/layer_boundary/prepare.py @@ -24,34 +24,34 @@ from sglang.srt.distributed import ( from sglang.srt.distributed.device_communicators.pynccl_allocator import ( use_symmetric_memory, ) -from sglang.srt.layers.communicator.adapters.attention import ( - AttentionInputs, - _redistribute_from_attn_tp_shards, - _redistribute_to_attn_tp_shards, - get_attn_tp_context, -) -from sglang.srt.layers.communicator.adapters.context_parallel import ( - dsa_cp_gather_hidden_states, -) -from sglang.srt.layers.communicator.layout import ( - SumGroup, - _cp_shard_token_rows, - moe_cp_gathered_rows, -) -from sglang.srt.layers.communicator.output import ( - HandoffOutput, - UnreducedOutput, - reduce_output, -) -from sglang.srt.layers.communicator.residual import StageRead, StageUpdate -from sglang.srt.layers.communicator.residual.add_norm import ADD, NORM_READ -from sglang.srt.layers.communicator.residual.stream import DeclaredSum from sglang.srt.layers.dp_attention import ( attn_tp_all_gather_into_tensor, dp_scatter, get_moe_cp_size, is_allocation_symmetric, ) +from sglang.srt.layers.layer_boundary.adapters.attention import ( + AttentionInputs, + _redistribute_from_attn_tp_shards, + _redistribute_to_attn_tp_shards, + get_attn_tp_context, +) +from sglang.srt.layers.layer_boundary.adapters.context_parallel import ( + dsa_cp_gather_hidden_states, +) +from sglang.srt.layers.layer_boundary.layout import ( + SumGroup, + _cp_shard_token_rows, + moe_cp_gathered_rows, +) +from sglang.srt.layers.layer_boundary.output import ( + HandoffOutput, + UnreducedOutput, + reduce_output, +) +from sglang.srt.layers.layer_boundary.residual import StageRead, StageUpdate +from sglang.srt.layers.layer_boundary.residual.add_norm import ADD, NORM_READ +from sglang.srt.layers.layer_boundary.residual.stream import DeclaredSum from sglang.srt.model_executor.forward_batch_info import ForwardBatch from sglang.srt.runtime_context import get_parallel from sglang.srt.utils import ( @@ -64,7 +64,7 @@ _is_npu = is_npu() if _is_npu: from sglang.srt.hardware_backend.npu.cmo import prepare_weight_cache -from sglang.srt.layers.communicator.ops import ( +from sglang.srt.layers.layer_boundary.ops import ( _mlp_input_reduce_output, _redistribute_input_to_dp, _redistribute_input_to_moe_cp, diff --git a/python/sglang/srt/layers/communicator/residual/__init__.py b/python/sglang/srt/layers/layer_boundary/residual/__init__.py similarity index 100% rename from python/sglang/srt/layers/communicator/residual/__init__.py rename to python/sglang/srt/layers/layer_boundary/residual/__init__.py diff --git a/python/sglang/srt/layers/communicator/residual/access.py b/python/sglang/srt/layers/layer_boundary/residual/access.py similarity index 97% rename from python/sglang/srt/layers/communicator/residual/access.py rename to python/sglang/srt/layers/layer_boundary/residual/access.py index 1b00982143..8f9f7b1b3a 100644 --- a/python/sglang/srt/layers/communicator/residual/access.py +++ b/python/sglang/srt/layers/layer_boundary/residual/access.py @@ -18,12 +18,12 @@ from typing import Optional, Tuple, Union import torch from sglang.srt.distributed import GroupCoordinator -from sglang.srt.layers.communicator.output import ( +from sglang.srt.layers.layer_boundary.output import ( HandoffOutput, UnreducedOutput, reduce_output, ) -from sglang.srt.layers.communicator.residual.stream import OwedOutput, ResidualStream +from sglang.srt.layers.layer_boundary.residual.stream import OwedOutput, ResidualStream from sglang.srt.model_executor.forward_batch_info import ForwardBatch, PPProxyTensors diff --git a/python/sglang/srt/layers/communicator/residual/add_norm.py b/python/sglang/srt/layers/layer_boundary/residual/add_norm.py similarity index 99% rename from python/sglang/srt/layers/communicator/residual/add_norm.py rename to python/sglang/srt/layers/layer_boundary/residual/add_norm.py index f3ae345066..d8857a9b46 100644 --- a/python/sglang/srt/layers/communicator/residual/add_norm.py +++ b/python/sglang/srt/layers/layer_boundary/residual/add_norm.py @@ -20,17 +20,17 @@ from typing import Optional import torch -from sglang.srt.layers.communicator.adapters.attention import ( - _redistribute_from_attn_tp_shards, - _redistribute_to_attn_tp_shards, - get_attn_tp_context, -) -from sglang.srt.layers.communicator.residual import LayerResidual from sglang.srt.layers.dp_attention import is_dp_attention_enabled from sglang.srt.layers.flashinfer_comm_fusion import ( is_flashinfer_allreduce_unavailable, uses_cutedsl_ar_fusion, ) +from sglang.srt.layers.layer_boundary.adapters.attention import ( + _redistribute_from_attn_tp_shards, + _redistribute_to_attn_tp_shards, + get_attn_tp_context, +) +from sglang.srt.layers.layer_boundary.residual import LayerResidual from sglang.srt.layers.quantization.fp8_utils import ( _use_aiter_bpreshuffle_gfx95, materialize_bpreshuffle_fp8_scale_tuple, diff --git a/python/sglang/srt/layers/communicator/residual/batch.py b/python/sglang/srt/layers/layer_boundary/residual/batch.py similarity index 94% rename from python/sglang/srt/layers/communicator/residual/batch.py rename to python/sglang/srt/layers/layer_boundary/residual/batch.py index 9e93669619..c65d23b497 100644 --- a/python/sglang/srt/layers/communicator/residual/batch.py +++ b/python/sglang/srt/layers/layer_boundary/residual/batch.py @@ -13,8 +13,8 @@ # ============================================================================== """Residual access for one layer-stack invocation or TBO microbatch.""" -from sglang.srt.layers.communicator.residual import access -from sglang.srt.layers.communicator.residual.stream import ResidualStream +from sglang.srt.layers.layer_boundary.residual import access +from sglang.srt.layers.layer_boundary.residual.stream import ResidualStream from sglang.srt.model_executor.forward_batch_info import PPProxyTensors @@ -60,7 +60,7 @@ def norm( # The terminal consumer now owns the pair. Do not keep layer buffers alive # through logits processing or the next forward on this batch. forward_batch.residual_stream = None - from sglang.srt.layers.communicator.output import HandoffOutput + from sglang.srt.layers.layer_boundary.output import HandoffOutput if isinstance(hidden_states, HandoffOutput): if residual is None: diff --git a/python/sglang/srt/layers/communicator/residual/mhc.py b/python/sglang/srt/layers/layer_boundary/residual/mhc.py similarity index 99% rename from python/sglang/srt/layers/communicator/residual/mhc.py rename to python/sglang/srt/layers/layer_boundary/residual/mhc.py index 06f875fe09..d6916c267d 100644 --- a/python/sglang/srt/layers/communicator/residual/mhc.py +++ b/python/sglang/srt/layers/layer_boundary/residual/mhc.py @@ -19,7 +19,7 @@ from typing import Callable, Optional import torch from sglang.kernels.ops.layernorm.mhc import hc_contract, hc_expand -from sglang.srt.layers.communicator.residual import LayerResidual +from sglang.srt.layers.layer_boundary.residual import LayerResidual from sglang.srt.runtime_context import get_parallel diff --git a/python/sglang/srt/layers/communicator/residual/stream.py b/python/sglang/srt/layers/layer_boundary/residual/stream.py similarity index 96% rename from python/sglang/srt/layers/communicator/residual/stream.py rename to python/sglang/srt/layers/layer_boundary/residual/stream.py index 68eb449aba..8d722c1acb 100644 --- a/python/sglang/srt/layers/communicator/residual/stream.py +++ b/python/sglang/srt/layers/layer_boundary/residual/stream.py @@ -19,9 +19,9 @@ import msgspec import torch from sglang.srt.distributed import GroupCoordinator -from sglang.srt.layers.communicator.layout import SumGroup, _sum_group -from sglang.srt.layers.communicator.output import HandoffOutput, UnreducedOutput -from sglang.srt.layers.communicator.residual import StageUpdate +from sglang.srt.layers.layer_boundary.layout import SumGroup, _sum_group +from sglang.srt.layers.layer_boundary.output import HandoffOutput, UnreducedOutput +from sglang.srt.layers.layer_boundary.residual import StageUpdate class CarriedSum(msgspec.Struct, frozen=True): diff --git a/python/sglang/srt/layers/communicator/stage.py b/python/sglang/srt/layers/layer_boundary/stage.py similarity index 94% rename from python/sglang/srt/layers/communicator/stage.py rename to python/sglang/srt/layers/layer_boundary/stage.py index b694fcad41..e5eac63aef 100644 --- a/python/sglang/srt/layers/communicator/stage.py +++ b/python/sglang/srt/layers/layer_boundary/stage.py @@ -19,33 +19,33 @@ import torch from sglang.srt.layers import layernorm_sp from sglang.srt.layers.aux_hidden_states import AuxHiddenStateAccumulator -from sglang.srt.layers.communicator.adapters import branch -from sglang.srt.layers.communicator.adapters.lora import ( +from sglang.srt.layers.layer_boundary.adapters import branch +from sglang.srt.layers.layer_boundary.adapters.lora import ( publish_attention, publish_ffn, ) -from sglang.srt.layers.communicator.contracts import ( +from sglang.srt.layers.layer_boundary.contracts import ( BatchVariant, StageEntry, StageKind, StageSteps, ) -from sglang.srt.layers.communicator.ops import gather_attention_tp -from sglang.srt.layers.communicator.residual.access import buffer, from_pp -from sglang.srt.layers.communicator.residual.add_norm import ( +from sglang.srt.layers.layer_boundary.ops import gather_attention_tp +from sglang.srt.layers.layer_boundary.residual.access import buffer, from_pp +from sglang.srt.layers.layer_boundary.residual.add_norm import ( ADD, ) -from sglang.srt.layers.communicator.residual.batch import current -from sglang.srt.layers.communicator.residual.stream import DeclaredSum, ResidualStream +from sglang.srt.layers.layer_boundary.residual.batch import current +from sglang.srt.layers.layer_boundary.residual.stream import DeclaredSum, ResidualStream from sglang.srt.model_executor.forward_batch_info import ForwardBatch from sglang.srt.runtime_context import get_forward if TYPE_CHECKING: - from sglang.srt.layers.communicator.construction import StagePlan - from sglang.srt.layers.communicator.factories import StageDeclaration + from sglang.srt.layers.layer_boundary.construction import StagePlan + from sglang.srt.layers.layer_boundary.factories import StageDeclaration -class StageCommunicator: +class StageBoundary: """One stage of a layer, its attention or its FFN: the boundary into it, run with the stage's norm on the steps the layer chose for the batch.""" diff --git a/python/sglang/srt/layers/vocab_parallel_embedding.py b/python/sglang/srt/layers/vocab_parallel_embedding.py index 18f4ac3c42..47ff41e897 100644 --- a/python/sglang/srt/layers/vocab_parallel_embedding.py +++ b/python/sglang/srt/layers/vocab_parallel_embedding.py @@ -21,12 +21,12 @@ from sglang.srt.distributed.device_communicators.pynccl_allocator import ( ) from sglang.srt.environ import envs from sglang.srt.layers.amx_utils import PackWeightMethod -from sglang.srt.layers.communicator import get_attn_tp_context from sglang.srt.layers.dp_attention import ( attn_tp_all_reduce, is_allocation_symmetric, is_dp_attention_enabled, ) +from sglang.srt.layers.layer_boundary import get_attn_tp_context from sglang.srt.layers.parameter import BasevLLMParameter from sglang.srt.layers.quantization.base_config import ( QuantizationConfig, diff --git a/python/sglang/srt/lora/layers.py b/python/sglang/srt/lora/layers.py index 400fd699ca..6ce0205d4c 100644 --- a/python/sglang/srt/lora/layers.py +++ b/python/sglang/srt/lora/layers.py @@ -119,7 +119,7 @@ class VocabParallelEmbeddingWithLoRA(BaseLayerWithLoRA): # all-reduce, making the unsharded LoRA approach mathematically # incorrect — a sharded LoRA kernel would be needed. if hasattr(base_layer, "tp_size") and base_layer.tp_size > 1: - from sglang.srt.layers.communicator import get_attn_tp_context + from sglang.srt.layers.layer_boundary import get_attn_tp_context assert not get_attn_tp_context().allow_input_scattered, ( "VocabParallelEmbeddingWithLoRA with TP > 1 under input_scattered mode (e.g., DeepSeek-v2 MLA with --enable-attn-tp-input-scattered) is not fully supported and may produce incorrect results. Consider disabling input_scattered or removing embed_tokens from LoRA target modules." @@ -295,7 +295,7 @@ class ParallelLMHeadWithLoRA(BaseLayerWithLoRA): # incompatible with input_scattered mode where the all-reduce is # skipped. if tp_size > 1: - from sglang.srt.layers.communicator import get_attn_tp_context + from sglang.srt.layers.layer_boundary import get_attn_tp_context if get_attn_tp_context().allow_input_scattered: raise ValueError( diff --git a/python/sglang/srt/model_executor/forward_batch_info.py b/python/sglang/srt/model_executor/forward_batch_info.py index 97cd785704..ee2e5d2bf0 100644 --- a/python/sglang/srt/model_executor/forward_batch_info.py +++ b/python/sglang/srt/model_executor/forward_batch_info.py @@ -71,9 +71,9 @@ from sglang.srt.utils.common import ceil_align, is_pin_memory_available if TYPE_CHECKING: from sglang.srt.configs.model_config import ModelConfig - from sglang.srt.layers.communicator.residual.stream import ResidualStream from sglang.srt.layers.cp.base import BaseContextParallelMetadata from sglang.srt.layers.dcp.metadata import DecodeContextParallelMetadata + from sglang.srt.layers.layer_boundary.residual.stream import ResidualStream from sglang.srt.layers.logits_processor import LogitsProcessorOutput from sglang.srt.managers.schedule_batch import MultimodalInputs, ScheduleBatch from sglang.srt.model_executor.model_runner import ModelRunner @@ -1257,7 +1257,7 @@ class ForwardBatch(ForwardBatchDeepSeekMHAMixin): def moe_num_token_non_padded(self) -> Optional[torch.Tensor]: """Bound for masking a sparse MoE's padded rows, or None when the MoE input is a gathered buffer whose real rows are not a prefix of it.""" - from sglang.srt.layers.communicator import moe_cp_gathers_sparse_moe_input + from sglang.srt.layers.layer_boundary import moe_cp_gathers_sparse_moe_input from sglang.srt.layers.moe.utils import is_moe_input_scattered_across_dp_ranks if self.num_token_non_padded is None: @@ -1916,7 +1916,7 @@ class ForwardBatch(ForwardBatchDeepSeekMHAMixin): return self.input_ids.shape[0] def prepare_attn_tp_scatter_input(self, model_runner: ModelRunner): - from sglang.srt.layers.communicator import get_attn_tp_context + from sglang.srt.layers.layer_boundary import get_attn_tp_context # Pure TP+SP has no MLP-sync pass, so stamp the decision here. self.attn_tp_sequence_sharded = model_runner.attn_tp_sequence_sharded( diff --git a/python/sglang/srt/model_executor/runner/base_runner.py b/python/sglang/srt/model_executor/runner/base_runner.py index fcef357fd7..455a4dfb58 100644 --- a/python/sglang/srt/model_executor/runner/base_runner.py +++ b/python/sglang/srt/model_executor/runner/base_runner.py @@ -288,12 +288,12 @@ class BaseRunner(ABC): with custom_all_reduce.register_graph_buffers). """ mr = self.model_runner - from sglang.srt.layers.communicator import FUSE_ALLREDUCE_MAX_BATCH_SIZE from sglang.srt.layers.flashinfer_comm_fusion import ( pre_initialize_workspaces, resolve_flashinfer_allreduce_fusion_backend, uses_cutedsl_ar_fusion, ) + from sglang.srt.layers.layer_boundary import FUSE_ALLREDUCE_MAX_BATCH_SIZE if get_exec().comm.flashinfer_allreduce_fusion_backend is None: return @@ -303,7 +303,7 @@ class BaseRunner(ABC): resolve_flashinfer_allreduce_fusion_backend() if not mr.is_draft_worker: # A draft installs no fusion communicator. - from sglang.srt.layers.communicator.fusions.cutedsl import ( + from sglang.srt.layers.layer_boundary.fusions.cutedsl import ( prepare_cutedsl_fusion, ) diff --git a/python/sglang/srt/models/bailing_moe.py b/python/sglang/srt/models/bailing_moe.py index 14c007a446..4b77564048 100644 --- a/python/sglang/srt/models/bailing_moe.py +++ b/python/sglang/srt/models/bailing_moe.py @@ -35,16 +35,16 @@ from sglang.srt.layers.aux_hidden_states import ( AuxHiddenStateAccumulator, AuxHiddenStateList, ) -from sglang.srt.layers.communicator import ( +from sglang.srt.layers.dp_attention import ( + is_dp_attention_enabled, +) +from sglang.srt.layers.layer_boundary import ( declare_attn, declare_ffn, enable_moe_dense_fully_dp, make_stages, ) -from sglang.srt.layers.communicator.residual import batch as residual_batch -from sglang.srt.layers.dp_attention import ( - is_dp_attention_enabled, -) +from sglang.srt.layers.layer_boundary.residual import batch as residual_batch from sglang.srt.layers.layernorm import RMSNorm from sglang.srt.layers.linear import ( MergedColumnParallelLinear, @@ -698,7 +698,7 @@ class BailingMoEBlock(nn.Module): self.post_attention_layernorm = RMSNorm(hidden_size, eps=config.rms_norm_eps) - self.attn_stage, self.ffn_stage = make_stages( + self.attn_boundary, self.ffn_boundary = make_stages( (declare_attn(), self.input_layernorm), ( declare_ffn( @@ -730,7 +730,7 @@ class BailingMoEBlock(nn.Module): captured_last_layer_outputs: Optional[AuxHiddenStateAccumulator] = None, capture_output=None, ) -> torch.Tensor: - hidden_states = self.attn_stage.prepare( + hidden_states = self.attn_boundary.prepare( hidden_states, forward_batch, captured_last_layer_outputs=captured_last_layer_outputs, @@ -744,13 +744,13 @@ class BailingMoEBlock(nn.Module): forward_batch=forward_batch, ) - hidden_states = self.attn_stage.finish(hidden_states, forward_batch) - hidden_states = self.ffn_stage.prepare( + hidden_states = self.attn_boundary.finish(hidden_states, forward_batch) + hidden_states = self.ffn_boundary.prepare( hidden_states=hidden_states, forward_batch=forward_batch, ) - with self.ffn_stage.exit(forward_batch) as ffn_exit: + with self.ffn_boundary.exit(forward_batch) as ffn_exit: hidden_states = self.mlp(hidden_states, forward_batch) hidden_states = ffn_exit.finish(hidden_states) @@ -822,7 +822,7 @@ class BailingMoEModel(nn.Module): residual_batch.start(forward_batch) else: assert pp_proxy_tensors is not None - hidden_states = self.layers[self.start_layer].attn_stage.from_pp( + hidden_states = self.layers[self.start_layer].attn_boundary.from_pp( pp_proxy_tensors, forward_batch ) diff --git a/python/sglang/srt/models/bailing_moe_linear.py b/python/sglang/srt/models/bailing_moe_linear.py index 53504488a1..7c6ee7a1f4 100644 --- a/python/sglang/srt/models/bailing_moe_linear.py +++ b/python/sglang/srt/models/bailing_moe_linear.py @@ -15,15 +15,15 @@ from sglang.kernels.ops.quantization.fp8_kernel import is_fp8_fnuz from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder from sglang.srt.layers import deep_gemm_wrapper from sglang.srt.layers.activation import SiluAndMul -from sglang.srt.layers.communicator import ( +from sglang.srt.layers.dp_attention import ( + is_dp_attention_enabled, +) +from sglang.srt.layers.layer_boundary import ( declare_attn, declare_ffn, make_stages, ) -from sglang.srt.layers.communicator.residual import batch as residual_batch -from sglang.srt.layers.dp_attention import ( - is_dp_attention_enabled, -) +from sglang.srt.layers.layer_boundary.residual import batch as residual_batch from sglang.srt.layers.layernorm import RMSNorm from sglang.srt.layers.linear import ( ColumnParallelLinear, @@ -820,7 +820,7 @@ class BailingMoELinearDecoderLayer(nn.Module): if config.attention_type == 1 and self.use_mla else None ) - self.attn_stage, self.ffn_stage = make_stages( + self.attn_boundary, self.ffn_boundary = make_stages( ( declare_attn(), self.input_layernorm, @@ -854,7 +854,7 @@ class BailingMoELinearDecoderLayer(nn.Module): zero_allocator: BumpAllocator, **kwargs, ) -> torch.Tensor: - hidden_states = self.attn_stage.prepare(hidden_states, forward_batch) + hidden_states = self.attn_boundary.prepare(hidden_states, forward_batch) # logger.warning( # f"===={self.layer_id=}, 1 shape= {hidden_states.shape}, {residual.shape}" # ) @@ -875,12 +875,12 @@ class BailingMoELinearDecoderLayer(nn.Module): # logger.warning( # f"===={self.layer_id=}, 2 shape= {hidden_states.shape}, {residual.shape}" # ) - hidden_states = self.attn_stage.finish(hidden_states, forward_batch) - hidden_states = self.ffn_stage.prepare(hidden_states, forward_batch) + hidden_states = self.attn_boundary.finish(hidden_states, forward_batch) + hidden_states = self.ffn_boundary.prepare(hidden_states, forward_batch) # logger.warning( # f"===={self.layer_id=}, 3 shape= {hidden_states.shape}, {residual.shape}" # ) - with self.ffn_stage.exit(forward_batch) as ffn_exit: + with self.ffn_boundary.exit(forward_batch) as ffn_exit: hidden_states = self.mlp(hidden_states) hidden_states = ffn_exit.finish(hidden_states) return hidden_states @@ -983,7 +983,7 @@ class BailingMoELinearModel(nn.Module): residual_batch.start(forward_batch) else: assert pp_proxy_tensors is not None - hidden_states = self.layers[self.start_layer].attn_stage.from_pp( + hidden_states = self.layers[self.start_layer].attn_boundary.from_pp( pp_proxy_tensors, forward_batch ) diff --git a/python/sglang/srt/models/bailing_moe_nextn.py b/python/sglang/srt/models/bailing_moe_nextn.py index 2ea98da22c..82191e3aed 100644 --- a/python/sglang/srt/models/bailing_moe_nextn.py +++ b/python/sglang/srt/models/bailing_moe_nextn.py @@ -26,8 +26,8 @@ import torch from torch import nn from transformers import PretrainedConfig -from sglang.srt.layers.communicator.residual import batch as residual_batch from sglang.srt.layers.dp_attention import is_dp_attention_enabled +from sglang.srt.layers.layer_boundary.residual import batch as residual_batch from sglang.srt.layers.layernorm import RMSNorm from sglang.srt.layers.linear import ReplicatedLinear from sglang.srt.layers.logits_processor import LogitsProcessor diff --git a/python/sglang/srt/models/bailing_moe_v3.py b/python/sglang/srt/models/bailing_moe_v3.py index 10cd33a650..e238082e49 100644 --- a/python/sglang/srt/models/bailing_moe_v3.py +++ b/python/sglang/srt/models/bailing_moe_v3.py @@ -29,14 +29,14 @@ from sglang.srt.eplb.expert_location_dispatch import ExpertLocationDispatchInfo from sglang.srt.layers import deep_gemm_wrapper from sglang.srt.layers.activation import SiluAndMul from sglang.srt.layers.aux_hidden_states import AuxHiddenStateList -from sglang.srt.layers.communicator import ( +from sglang.srt.layers.dp_attention import is_dp_attention_enabled +from sglang.srt.layers.layer_boundary import ( declare_attn, declare_ffn, enable_moe_dense_fully_dp, make_stages, ) -from sglang.srt.layers.communicator.residual import batch as residual_batch -from sglang.srt.layers.dp_attention import is_dp_attention_enabled +from sglang.srt.layers.layer_boundary.residual import batch as residual_batch from sglang.srt.layers.layernorm import RMSNorm from sglang.srt.layers.linear import ( ColumnParallelLinear, @@ -1149,7 +1149,7 @@ class BailingMoELinearDecoderLayer(nn.Module): self.input_layernorm = RMSNorm(self.hidden_size, eps=rms_norm_eps) self.post_attention_layernorm = RMSNorm(self.hidden_size, eps=rms_norm_eps) - self.attn_stage, self.ffn_stage = make_stages( + self.attn_boundary, self.ffn_boundary = make_stages( ( declare_attn(), self.input_layernorm, @@ -1182,7 +1182,7 @@ class BailingMoELinearDecoderLayer(nn.Module): capture_output=None, **kwargs, ) -> torch.Tensor: - hidden_states = self.attn_stage.prepare( + hidden_states = self.attn_boundary.prepare( hidden_states, forward_batch, capture_output=capture_output ) @@ -1208,10 +1208,10 @@ class BailingMoELinearDecoderLayer(nn.Module): forward_batch=forward_batch, ) - hidden_states = self.attn_stage.finish(hidden_states, forward_batch) - hidden_states = self.ffn_stage.prepare(hidden_states, forward_batch) + hidden_states = self.attn_boundary.finish(hidden_states, forward_batch) + hidden_states = self.ffn_boundary.prepare(hidden_states, forward_batch) - with self.ffn_stage.exit(forward_batch) as ffn_exit: + with self.ffn_boundary.exit(forward_batch) as ffn_exit: if not ( enable_moe_dense_fully_dp() and (not self.is_layer_sparse) @@ -1344,7 +1344,7 @@ class BailingMoELinearModel(nn.Module): residual_batch.start(forward_batch) else: assert pp_proxy_tensors is not None - hidden_states = self.layers[self.start_layer].attn_stage.from_pp( + hidden_states = self.layers[self.start_layer].attn_boundary.from_pp( pp_proxy_tensors, forward_batch ) diff --git a/python/sglang/srt/models/deepseek_common/attention_forward_methods/forward_mha.py b/python/sglang/srt/models/deepseek_common/attention_forward_methods/forward_mha.py index 62d407387f..e07b9ee46b 100644 --- a/python/sglang/srt/models/deepseek_common/attention_forward_methods/forward_mha.py +++ b/python/sglang/srt/models/deepseek_common/attention_forward_methods/forward_mha.py @@ -8,12 +8,12 @@ from sglang.kernels.ops.attention.dsa.dequant_k_cache import dequantize_k_cache_ from sglang.kernels.ops.attention.utils import concat_and_cast_mha_k_triton from sglang.srt.environ import envs from sglang.srt.layers.attention.tbo_backend import TboAttnBackend -from sglang.srt.layers.communicator import get_attn_tp_context from sglang.srt.layers.dcp import ( all_gather_kv_cache_for_mha_chunk_extend, all_gather_kv_cache_for_mha_extend, filter_dcp_local_kv_indices, ) +from sglang.srt.layers.layer_boundary import get_attn_tp_context from sglang.srt.model_executor.forward_batch_info import ForwardBatch from sglang.srt.model_executor.forward_context import ( get_attn_backend, diff --git a/python/sglang/srt/models/deepseek_common/attention_forward_methods/forward_mha_rocm.py b/python/sglang/srt/models/deepseek_common/attention_forward_methods/forward_mha_rocm.py index c369f9d32b..a28b3228a8 100644 --- a/python/sglang/srt/models/deepseek_common/attention_forward_methods/forward_mha_rocm.py +++ b/python/sglang/srt/models/deepseek_common/attention_forward_methods/forward_mha_rocm.py @@ -13,8 +13,8 @@ from typing import TYPE_CHECKING import torch from sglang.kernels.ops.attention.utils import concat_and_cast_mha_k_triton -from sglang.srt.layers.communicator import get_attn_tp_context from sglang.srt.layers.dcp import all_gather_kv_cache_for_mha_extend +from sglang.srt.layers.layer_boundary import get_attn_tp_context from sglang.srt.layers.quantization.fp8_utils import ( materialize_bpreshuffle_fp8_scale_tuple, ) diff --git a/python/sglang/srt/models/deepseek_common/attention_forward_methods/forward_mla.py b/python/sglang/srt/models/deepseek_common/attention_forward_methods/forward_mla.py index ca39935abb..8db5a0d519 100644 --- a/python/sglang/srt/models/deepseek_common/attention_forward_methods/forward_mla.py +++ b/python/sglang/srt/models/deepseek_common/attention_forward_methods/forward_mla.py @@ -16,13 +16,13 @@ from sglang.srt.environ import envs from sglang.srt.layers import deep_gemm_wrapper from sglang.srt.layers.attention.dsa.utils import is_graph_dsa_split_op_surface from sglang.srt.layers.attention.dsa_backend import prepare_kv_for_attention -from sglang.srt.layers.communicator import get_attn_tp_context from sglang.srt.layers.dcp import ( all_gather_kv_cache_for_mla_extend, all_gather_q_for_mla_decode, cp_lse_ag_out_rs_mla, dcp_a2a_lse_reduce, ) +from sglang.srt.layers.layer_boundary import get_attn_tp_context from sglang.srt.layers.logits_processor import get_in_autotune_dummy_run from sglang.srt.layers.radix_attention import unified_attention_with_output from sglang.srt.lora.deepseek_mla_correction import ( diff --git a/python/sglang/srt/models/deepseek_common/attention_forward_methods/forward_mla_rocm.py b/python/sglang/srt/models/deepseek_common/attention_forward_methods/forward_mla_rocm.py index 0f277b183e..d6e8728c16 100644 --- a/python/sglang/srt/models/deepseek_common/attention_forward_methods/forward_mla_rocm.py +++ b/python/sglang/srt/models/deepseek_common/attention_forward_methods/forward_mla_rocm.py @@ -20,13 +20,13 @@ from sglang.kernels.ops.quantization.fp8_kernel import ( ) from sglang.srt.environ import envs from sglang.srt.layers import deep_gemm_wrapper -from sglang.srt.layers.communicator import get_attn_tp_context from sglang.srt.layers.dcp import ( all_gather_kv_cache_for_mla_extend, all_gather_q_for_mla_decode, cp_lse_ag_out_rs_mla, dcp_a2a_lse_reduce, ) +from sglang.srt.layers.layer_boundary import get_attn_tp_context from sglang.srt.layers.logits_processor import get_in_autotune_dummy_run from sglang.srt.layers.quantization.fp8_utils import ( emit_transposed_bpreshuffle_scale, diff --git a/python/sglang/srt/models/deepseek_nextn.py b/python/sglang/srt/models/deepseek_nextn.py index 39d66bca73..7bc345ac2c 100644 --- a/python/sglang/srt/models/deepseek_nextn.py +++ b/python/sglang/srt/models/deepseek_nextn.py @@ -28,7 +28,7 @@ from sglang.kernels.ops.layernorm.fused_eh_norm import fused_eh_norm from sglang.srt.environ import envs from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder from sglang.srt.layers.attention.index_topk_share import IndexTopKShareState -from sglang.srt.layers.communicator.residual import batch as residual_batch +from sglang.srt.layers.layer_boundary.residual import batch as residual_batch from sglang.srt.layers.layernorm import RMSNorm from sglang.srt.layers.linear import ReplicatedLinear from sglang.srt.layers.logits_processor import LogitsProcessor diff --git a/python/sglang/srt/models/deepseek_v2.py b/python/sglang/srt/models/deepseek_v2.py index 963caa6329..76d85eedc1 100644 --- a/python/sglang/srt/models/deepseek_v2.py +++ b/python/sglang/srt/models/deepseek_v2.py @@ -67,23 +67,23 @@ from sglang.srt.layers.aux_hidden_states import ( AuxHiddenStateAccumulator, AuxHiddenStatePacker, ) -from sglang.srt.layers.communicator import ( +from sglang.srt.layers.cp.cp_decode_attn_tp import get_cp_decode_attn_tp_ctx +from sglang.srt.layers.dcp.planner import ( + prepare_decode_context_parallel_metadata, +) +from sglang.srt.layers.flashinfer_comm_fusion import uses_cutedsl_ar_fusion +from sglang.srt.layers.layer_boundary import ( declare_attn, declare_ffn, enable_moe_dense_fully_dp, get_attn_tp_context, make_stages, ) -from sglang.srt.layers.communicator.adapters.context_parallel import ( +from sglang.srt.layers.layer_boundary.adapters.context_parallel import ( maybe_prefetch_next_full_attention_kv, ) -from sglang.srt.layers.communicator.residual import access as residual_access -from sglang.srt.layers.communicator.residual import batch as residual_batch -from sglang.srt.layers.cp.cp_decode_attn_tp import get_cp_decode_attn_tp_ctx -from sglang.srt.layers.dcp.planner import ( - prepare_decode_context_parallel_metadata, -) -from sglang.srt.layers.flashinfer_comm_fusion import uses_cutedsl_ar_fusion +from sglang.srt.layers.layer_boundary.residual import access as residual_access +from sglang.srt.layers.layer_boundary.residual import batch as residual_batch from sglang.srt.layers.layernorm import RMSNorm from sglang.srt.layers.linear import ( ColumnParallelLinear, @@ -1109,7 +1109,7 @@ class DeepseekV2MoE(nn.Module): if deferred_finalize and get_forward().defer_moe_finalize: # deferred_finalize excludes _shared_expert_tp1, so the shared add folds in. assert shared_output is not None - from sglang.srt.layers.communicator.fusions.cutedsl import ( + from sglang.srt.layers.layer_boundary.fusions.cutedsl import ( MoeFinalizeHandoff, ) @@ -2619,7 +2619,7 @@ class DeepseekV2DecoderLayer(nn.Module): self._gfx95_quant_format = self._detect_gfx95_quant_format() - self.attn_stage, self.ffn_stage = self._build_stages( + self.attn_boundary, self.ffn_boundary = self._build_stages( input_layernorm=self.input_layernorm, post_attention_layernorm=self.post_attention_layernorm, qkv_latent_func=self.self_attn.prepare_qkv_latent, @@ -2642,10 +2642,10 @@ class DeepseekV2DecoderLayer(nn.Module): and _use_mnnvl_cutedsl_fusion() ): # Dense layers too: selecting cutedsl turns the legacy fusion off. - from sglang.srt.layers.communicator.fusions.cutedsl import CuteDSLFusion + from sglang.srt.layers.layer_boundary.fusions.cutedsl import CuteDSLFusion fusions = CuteDSLFusion() - attn_stage, ffn_stage = make_stages( + attn_boundary, ffn_boundary = make_stages( ( declare_attn(), input_layernorm, @@ -2667,7 +2667,7 @@ class DeepseekV2DecoderLayer(nn.Module): else None, terminal=self._stage_terminal, ) - return attn_stage, ffn_stage + return attn_boundary, ffn_boundary def _detect_gfx95_quant_format(self) -> str: if not _is_gfx95_supported: @@ -2721,7 +2721,7 @@ class DeepseekV2DecoderLayer(nn.Module): next_full_attention_layer_id: Optional[int] = None, ) -> torch.Tensor: hidden_states_orig = residual_access.buffer(hidden_states) - hidden_states = self.attn_stage.prepare( + hidden_states = self.attn_boundary.prepare( hidden_states, forward_batch, captured_last_layer_outputs=captured_last_layer_outputs, @@ -2736,7 +2736,7 @@ class DeepseekV2DecoderLayer(nn.Module): zero_allocator=zero_allocator, llama_4_scaling=llama_4_scaling, input_on_attention_tp_slices=( - self.attn_stage.input_on_attention_tp_slices + self.attn_boundary.input_on_attention_tp_slices ), prev_topk_indices=prev_topk_indices, ) @@ -2750,8 +2750,8 @@ class DeepseekV2DecoderLayer(nn.Module): forward_batch, next_full_attention_layer_id ) - hidden_states = self.attn_stage.finish(hidden_states, forward_batch) - hidden_states = self.ffn_stage.prepare(hidden_states, forward_batch) + hidden_states = self.attn_boundary.finish(hidden_states, forward_batch) + hidden_states = self.ffn_boundary.prepare(hidden_states, forward_batch) if isinstance(self.mlp, DeepseekV2MLP): gemm_output_zero_allocator = None @@ -2769,7 +2769,7 @@ class DeepseekV2DecoderLayer(nn.Module): else: _mlp_ctx = nullcontext() - with self.ffn_stage.exit(forward_batch) as ffn_exit, _mlp_ctx: + with self.ffn_boundary.exit(forward_batch) as ffn_exit, _mlp_ctx: hidden_states = self.mlp( hidden_states, forward_batch, @@ -2788,7 +2788,7 @@ class DeepseekV2DecoderLayer(nn.Module): zero_allocator: BumpAllocator, tbo_subbatch_index: Optional[int] = None, ): - state.hidden_states_after_comm_pre_attn = self.attn_stage.prepare( + state.hidden_states_after_comm_pre_attn = self.attn_boundary.prepare( hidden_states, forward_batch ) if get_moe_a2a_backend().is_mori(): @@ -2803,15 +2803,15 @@ class DeepseekV2DecoderLayer(nn.Module): ) def op_comm_prepare_mlp(self, state): - hidden_states = self.attn_stage.finish( + hidden_states = self.attn_boundary.finish( state.pop("hidden_states_after_attn"), state.forward_batch ) - state.hidden_states_mlp_input = self.ffn_stage.prepare( + state.hidden_states_mlp_input = self.ffn_boundary.prepare( hidden_states, state.forward_batch ) def op_comm_postprocess_layer(self, state): - hidden_states = self.ffn_stage.postprocess( + hidden_states = self.ffn_boundary.postprocess( state.pop("hidden_states_mlp_output"), state.forward_batch ) @@ -2978,7 +2978,7 @@ class DeepseekV2Model(nn.Module): # llama_4_scaling: for supporting Mistral-Large-3 model self.llama_4_scaling_config = getattr(config, "llama_4_scaling", None) - from sglang.srt.layers.communicator.fusions.cutedsl import ( + from sglang.srt.layers.layer_boundary.fusions.cutedsl import ( install_cutedsl_fusion, ) @@ -3030,7 +3030,7 @@ class DeepseekV2Model(nn.Module): initial_topk_indices = None else: assert pp_proxy_tensors is not None - hidden_states = self.layers[self.start_layer].attn_stage.from_pp( + hidden_states = self.layers[self.start_layer].attn_boundary.from_pp( pp_proxy_tensors, forward_batch ) initial_topk_indices = pp_proxy_tensors.tensors.get("topk_indices") diff --git a/python/sglang/srt/models/deepseek_v4.py b/python/sglang/srt/models/deepseek_v4.py index 30a622461c..c28eb599fe 100644 --- a/python/sglang/srt/models/deepseek_v4.py +++ b/python/sglang/srt/models/deepseek_v4.py @@ -70,11 +70,6 @@ from sglang.srt.layers.attention.dsv4.dsv41_sparse import ( DeepseekV41Indexer, ) from sglang.srt.layers.attention.dsv4.indexer import C4Indexer -from sglang.srt.layers.communicator import get_attn_tp_context -from sglang.srt.layers.communicator.adapters.context_parallel import ( - dsa_cp_gather_hidden_states, - dsa_cp_reduce_scatter_hidden_states, -) from sglang.srt.layers.cp.cp_decode_attn_tp import get_cp_decode_attn_tp_ctx from sglang.srt.layers.cp.utils import ( cp_gather_full_sequence_states, @@ -103,6 +98,11 @@ from sglang.srt.layers.dp_attention import ( is_dp_gatherv_active, ) from sglang.srt.layers.engram import Engram, EngramHasher, EngramLayout +from sglang.srt.layers.layer_boundary import get_attn_tp_context +from sglang.srt.layers.layer_boundary.adapters.context_parallel import ( + dsa_cp_gather_hidden_states, + dsa_cp_reduce_scatter_hidden_states, +) from sglang.srt.layers.layernorm import RMSNorm from sglang.srt.layers.linear import ColumnParallelLinear, RowParallelLinear from sglang.srt.layers.logits_processor import LogitsMetadata, LogitsProcessor diff --git a/python/sglang/srt/models/dots3_common/modeling.py b/python/sglang/srt/models/dots3_common/modeling.py index 2030e3cff8..ed4ff8ed92 100644 --- a/python/sglang/srt/models/dots3_common/modeling.py +++ b/python/sglang/srt/models/dots3_common/modeling.py @@ -57,14 +57,14 @@ from sglang.srt.eplb.expert_location_dispatch import ExpertLocationDispatchInfo from sglang.srt.layers import deep_gemm_wrapper from sglang.srt.layers.activation import SiluAndMul from sglang.srt.layers.attention.dsa.dsa_indexer import Indexer -from sglang.srt.layers.communicator import ( +from sglang.srt.layers.dp_attention import is_dp_attention_enabled +from sglang.srt.layers.layer_boundary import ( declare_attn, declare_ffn, enable_moe_dense_fully_dp, make_stages, ) -from sglang.srt.layers.communicator.residual import batch as residual_batch -from sglang.srt.layers.dp_attention import is_dp_attention_enabled +from sglang.srt.layers.layer_boundary.residual import batch as residual_batch from sglang.srt.layers.layernorm import RMSNorm from sglang.srt.layers.linear import ( ColumnParallelLinear, @@ -1572,7 +1572,7 @@ class Dots3DecoderLayer(nn.Module): config.hidden_size, eps=config.rms_norm_eps ) - self.attn_stage, self.ffn_stage = make_stages( + self.attn_boundary, self.ffn_boundary = make_stages( (declare_attn(), self.input_layernorm), ( declare_ffn( @@ -1607,7 +1607,7 @@ class Dots3DecoderLayer(nn.Module): zero_allocator: BumpAllocator, ) -> torch.Tensor: - hidden_states = self.attn_stage.prepare(hidden_states, forward_batch) + hidden_states = self.attn_boundary.prepare(hidden_states, forward_batch) hidden_states = self.self_attn( positions=positions, @@ -1616,10 +1616,10 @@ class Dots3DecoderLayer(nn.Module): zero_allocator=zero_allocator, ) - hidden_states = self.attn_stage.finish(hidden_states, forward_batch) - hidden_states = self.ffn_stage.prepare(hidden_states, forward_batch) + hidden_states = self.attn_boundary.finish(hidden_states, forward_batch) + hidden_states = self.ffn_boundary.prepare(hidden_states, forward_batch) - with self.ffn_stage.exit(forward_batch) as ffn_exit: + with self.ffn_boundary.exit(forward_batch) as ffn_exit: hidden_states = self.mlp( hidden_states, forward_batch, @@ -1637,7 +1637,7 @@ class Dots3DecoderLayer(nn.Module): zero_allocator: BumpAllocator, tbo_subbatch_index: Optional[int] = None, ): - state.hidden_states_after_comm_pre_attn = self.attn_stage.prepare( + state.hidden_states_after_comm_pre_attn = self.attn_boundary.prepare( hidden_states, forward_batch ) state.update( @@ -1650,10 +1650,10 @@ class Dots3DecoderLayer(nn.Module): ) def op_comm_prepare_mlp(self, state): - hidden_states = self.attn_stage.finish( + hidden_states = self.attn_boundary.finish( state.pop("hidden_states_after_attn"), state.forward_batch ) - state.hidden_states_mlp_input = self.ffn_stage.prepare( + state.hidden_states_mlp_input = self.ffn_boundary.prepare( hidden_states, state.forward_batch ) @@ -1671,7 +1671,7 @@ class Dots3DecoderLayer(nn.Module): state.hidden_states_mlp_output = hidden_states def op_comm_postprocess_layer(self, state): - hidden_states = self.ffn_stage.postprocess( + hidden_states = self.ffn_boundary.postprocess( state.pop("hidden_states_mlp_output"), state.forward_batch ) @@ -1763,7 +1763,7 @@ class Dots3Model(nn.Module): residual_batch.start(forward_batch) else: assert pp_proxy_tensors is not None - hidden_states = self.layers[self.start_layer].attn_stage.from_pp( + hidden_states = self.layers[self.start_layer].attn_boundary.from_pp( pp_proxy_tensors, forward_batch ) diff --git a/python/sglang/srt/models/dots3_common/nextn.py b/python/sglang/srt/models/dots3_common/nextn.py index a862b58c8f..47c6ecf1bb 100644 --- a/python/sglang/srt/models/dots3_common/nextn.py +++ b/python/sglang/srt/models/dots3_common/nextn.py @@ -8,8 +8,8 @@ from torch import nn from transformers import PretrainedConfig from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder -from sglang.srt.layers.communicator.residual import batch as residual_batch from sglang.srt.layers.dp_attention import is_dp_attention_enabled +from sglang.srt.layers.layer_boundary.residual import batch as residual_batch from sglang.srt.layers.layernorm import RMSNorm from sglang.srt.layers.linear import ReplicatedLinear from sglang.srt.layers.logits_processor import LogitsProcessor diff --git a/python/sglang/srt/models/ernie4.py b/python/sglang/srt/models/ernie4.py index 33cec5ea54..405b668355 100644 --- a/python/sglang/srt/models/ernie4.py +++ b/python/sglang/srt/models/ernie4.py @@ -26,7 +26,7 @@ from transformers.models.ernie4_5_moe.configuration_ernie4_5_moe import ( from sglang.srt.distributed import ( tensor_model_parallel_all_reduce, ) -from sglang.srt.layers.communicator import enable_moe_dense_fully_dp +from sglang.srt.layers.layer_boundary import enable_moe_dense_fully_dp from sglang.srt.layers.layernorm import RMSNorm from sglang.srt.layers.logits_processor import LogitsProcessor from sglang.srt.layers.moe.ep_moe.layer import get_moe_impl_class diff --git a/python/sglang/srt/models/falcon_h1.py b/python/sglang/srt/models/falcon_h1.py index 500285bedc..ecfbd97a40 100644 --- a/python/sglang/srt/models/falcon_h1.py +++ b/python/sglang/srt/models/falcon_h1.py @@ -11,16 +11,16 @@ from sglang.srt.layers.attention.hybrid_linear_attn_backend import ( Mamba2AttnBackend, ) from sglang.srt.layers.attention.mamba.mamba import MambaMixer2 -from sglang.srt.layers.communicator import ( +from sglang.srt.layers.dp_attention import ( + is_dp_attention_enabled, +) +from sglang.srt.layers.layer_boundary import ( declare_attn, declare_ffn, make_stages, ) -from sglang.srt.layers.communicator.output import OutputTransform -from sglang.srt.layers.communicator.residual import batch as residual_batch -from sglang.srt.layers.dp_attention import ( - is_dp_attention_enabled, -) +from sglang.srt.layers.layer_boundary.output import OutputTransform +from sglang.srt.layers.layer_boundary.residual import batch as residual_batch from sglang.srt.layers.layernorm import RMSNorm from sglang.srt.layers.linear import ( MergedColumnParallelLinear, @@ -222,7 +222,7 @@ class FalconH1HybridAttentionDecoderLayer(nn.Module): self.q_norm = RMSNorm(self.head_dim, eps=config.rms_norm_eps) self.k_norm = RMSNorm(self.head_dim, eps=config.rms_norm_eps) - self.attn_stage, self.ffn_stage = make_stages( + self.attn_boundary, self.ffn_boundary = make_stages( (declare_attn(), self.input_layernorm), ( declare_ffn( @@ -334,7 +334,7 @@ class FalconH1HybridAttentionDecoderLayer(nn.Module): forward_batch: ForwardBatch, **kwargs: Any, ): - hidden_states = self.attn_stage.prepare(hidden_states, forward_batch) + hidden_states = self.attn_boundary.prepare(hidden_states, forward_batch) if not forward_batch.forward_mode.is_idle(): # Attention block @@ -363,9 +363,9 @@ class FalconH1HybridAttentionDecoderLayer(nn.Module): hidden_states = attention_hidden_states + mamba_hidden_states # Fully Connected - hidden_states = self.attn_stage.finish(hidden_states, forward_batch) - hidden_states = self.ffn_stage.prepare(hidden_states, forward_batch) - with self.ffn_stage.exit(forward_batch) as ffn_exit: + hidden_states = self.attn_boundary.finish(hidden_states, forward_batch) + hidden_states = self.ffn_boundary.prepare(hidden_states, forward_batch) + with self.ffn_boundary.exit(forward_batch) as ffn_exit: hidden_states = self.feed_forward(hidden_states, forward_batch) return ffn_exit.finish(hidden_states) diff --git a/python/sglang/srt/models/gigachat35.py b/python/sglang/srt/models/gigachat35.py index db74181dad..d8d0420067 100644 --- a/python/sglang/srt/models/gigachat35.py +++ b/python/sglang/srt/models/gigachat35.py @@ -23,9 +23,9 @@ from transformers import PretrainedConfig import sglang.srt.models.deepseek_v2 as deepseek_v2 from sglang.srt.configs.gigachat35 import GigaChat35Config -from sglang.srt.layers.communicator import get_attn_tp_context -from sglang.srt.layers.communicator.output import OutputTransform -from sglang.srt.layers.communicator.residual import batch as residual_batch +from sglang.srt.layers.layer_boundary import get_attn_tp_context +from sglang.srt.layers.layer_boundary.output import OutputTransform +from sglang.srt.layers.layer_boundary.residual import batch as residual_batch from sglang.srt.layers.layernorm import GemmaRMSNorm, RMSNorm from sglang.srt.layers.linear import ColumnParallelLinear from sglang.srt.layers.logits_processor import LogitsProcessor @@ -441,7 +441,7 @@ class GigaChat35DecoderLayer(deepseek_v2.DeepseekV2DecoderLayer): post_layernorm=self.post_self_attn_layernorm, ) # The communicator chooses its fused steps from its norms when built. - self.attn_stage, self.ffn_stage = self._build_stages( + self.attn_boundary, self.ffn_boundary = self._build_stages( input_layernorm=attn_prepare_layernorm, post_attention_layernorm=mlp_prepare_layernorm, qkv_latent_func=qkv_latent_func, @@ -471,7 +471,7 @@ class GigaChat35DecoderLayer(deepseek_v2.DeepseekV2DecoderLayer): zero_allocator: BumpAllocator, **kwargs, ) -> torch.Tensor: - hidden_states = self.attn_stage.prepare(hidden_states, forward_batch) + hidden_states = self.attn_boundary.prepare(hidden_states, forward_batch) if self.use_linear_attn: hidden_states = self.self_attn( @@ -485,18 +485,18 @@ class GigaChat35DecoderLayer(deepseek_v2.DeepseekV2DecoderLayer): forward_batch=forward_batch, zero_allocator=zero_allocator, input_on_attention_tp_slices=( - self.attn_stage.input_on_attention_tp_slices + self.attn_boundary.input_on_attention_tp_slices ), ) get_attn_tp_context().clear_attn_inputs() - hidden_states = self.attn_stage.finish(hidden_states, forward_batch) - hidden_states = self.ffn_stage.prepare(hidden_states, forward_batch) + hidden_states = self.attn_boundary.finish(hidden_states, forward_batch) + hidden_states = self.ffn_boundary.prepare(hidden_states, forward_batch) # Unlike deepseek_v2, no moe_output_buffer_ctx here: non-inplace MoE # runners then allocate their output per forward instead of recycling # the layer-input buffer. The default (inplace) runners are unaffected. - with self.ffn_stage.exit(forward_batch) as ffn_exit: + with self.ffn_boundary.exit(forward_batch) as ffn_exit: hidden_states = self.mlp(hidden_states, forward_batch) hidden_states = ffn_exit.finish(hidden_states) @@ -562,7 +562,7 @@ class GigaChat35Model(nn.Module): residual_batch.start(forward_batch) else: assert pp_proxy_tensors is not None - hidden_states = self.layers[self.start_layer].attn_stage.from_pp( + hidden_states = self.layers[self.start_layer].attn_boundary.from_pp( pp_proxy_tensors, forward_batch ) diff --git a/python/sglang/srt/models/gigachat35_mtp.py b/python/sglang/srt/models/gigachat35_mtp.py index a78a804f9d..d7f22e2eb8 100644 --- a/python/sglang/srt/models/gigachat35_mtp.py +++ b/python/sglang/srt/models/gigachat35_mtp.py @@ -18,7 +18,7 @@ from typing import Iterable, Optional import torch from torch import nn -from sglang.srt.layers.communicator.residual import batch as residual_batch +from sglang.srt.layers.layer_boundary.residual import batch as residual_batch from sglang.srt.layers.logits_processor import LogitsProcessor from sglang.srt.layers.quantization.base_config import QuantizationConfig from sglang.srt.layers.vocab_parallel_embedding import ( diff --git a/python/sglang/srt/models/glm4_moe.py b/python/sglang/srt/models/glm4_moe.py index 473cc586a5..4ea7bc4936 100644 --- a/python/sglang/srt/models/glm4_moe.py +++ b/python/sglang/srt/models/glm4_moe.py @@ -39,17 +39,17 @@ from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation from sglang.srt.eplb.expert_location_dispatch import ExpertLocationDispatchInfo from sglang.srt.layers.activation import SiluAndMul from sglang.srt.layers.aux_hidden_states import AuxHiddenStateList -from sglang.srt.layers.communicator import ( +from sglang.srt.layers.dp_attention import ( + is_allocation_symmetric, + is_dp_attention_enabled, +) +from sglang.srt.layers.layer_boundary import ( declare_attn, declare_ffn, enable_moe_dense_fully_dp, make_stages, ) -from sglang.srt.layers.communicator.residual import batch as residual_batch -from sglang.srt.layers.dp_attention import ( - is_allocation_symmetric, - is_dp_attention_enabled, -) +from sglang.srt.layers.layer_boundary.residual import batch as residual_batch from sglang.srt.layers.layernorm import RMSNorm from sglang.srt.layers.linear import ( MergedColumnParallelLinear, @@ -866,7 +866,7 @@ class Glm4MoeDecoderLayer(nn.Module): config.hidden_size, eps=config.rms_norm_eps ) - self.attn_stage, self.ffn_stage = make_stages( + self.attn_boundary, self.ffn_boundary = make_stages( (declare_attn(), self.input_layernorm), ( declare_ffn( @@ -942,7 +942,7 @@ class Glm4MoeDecoderLayer(nn.Module): capture_output=None, ) -> torch.Tensor: - hidden_states = self.attn_stage.prepare( + hidden_states = self.attn_boundary.prepare( hidden_states, forward_batch, quant_format=self.attn_quant_format, @@ -955,10 +955,10 @@ class Glm4MoeDecoderLayer(nn.Module): forward_batch=forward_batch, ) - hidden_states = self.attn_stage.finish(hidden_states, forward_batch) - hidden_states = self.ffn_stage.prepare(hidden_states, forward_batch) + hidden_states = self.attn_boundary.finish(hidden_states, forward_batch) + hidden_states = self.ffn_boundary.prepare(hidden_states, forward_batch) - with self.ffn_stage.exit(forward_batch) as ffn_exit: + with self.ffn_boundary.exit(forward_batch) as ffn_exit: hidden_states = self.mlp(hidden_states, forward_batch) hidden_states = ffn_exit.finish(hidden_states) @@ -972,7 +972,7 @@ class Glm4MoeDecoderLayer(nn.Module): forward_batch: ForwardBatch, tbo_subbatch_index: Optional[int] = None, ): - state.hidden_states_after_comm_pre_attn = self.attn_stage.prepare( + state.hidden_states_after_comm_pre_attn = self.attn_boundary.prepare( hidden_states, forward_batch ) state.update( @@ -984,15 +984,15 @@ class Glm4MoeDecoderLayer(nn.Module): ) def op_comm_prepare_mlp(self, state): - hidden_states = self.attn_stage.finish( + hidden_states = self.attn_boundary.finish( state.pop("hidden_states_after_attn"), state.forward_batch ) - state.hidden_states_mlp_input = self.ffn_stage.prepare( + state.hidden_states_mlp_input = self.ffn_boundary.prepare( hidden_states, state.forward_batch ) def op_comm_postprocess_layer(self, state): - hidden_states = self.ffn_stage.postprocess( + hidden_states = self.ffn_boundary.postprocess( state.pop("hidden_states_mlp_output"), state.forward_batch ) @@ -1081,7 +1081,7 @@ class Glm4MoeModel(nn.Module): residual_batch.start(forward_batch) else: assert pp_proxy_tensors is not None - hidden_states = self.layers[self.start_layer].attn_stage.from_pp( + hidden_states = self.layers[self.start_layer].attn_boundary.from_pp( pp_proxy_tensors, forward_batch ) diff --git a/python/sglang/srt/models/glm4_moe_lite.py b/python/sglang/srt/models/glm4_moe_lite.py index 5e46ec92b7..e910e693da 100644 --- a/python/sglang/srt/models/glm4_moe_lite.py +++ b/python/sglang/srt/models/glm4_moe_lite.py @@ -33,18 +33,18 @@ from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation from sglang.srt.eplb.expert_location_dispatch import ExpertLocationDispatchInfo from sglang.srt.layers.activation import SiluAndMul from sglang.srt.layers.aux_hidden_states import AuxHiddenStateList -from sglang.srt.layers.communicator import ( +from sglang.srt.layers.dp_attention import ( + is_allocation_symmetric, + is_dp_attention_enabled, +) +from sglang.srt.layers.layer_boundary import ( declare_attn, declare_ffn, enable_moe_dense_fully_dp, get_attn_tp_context, make_stages, ) -from sglang.srt.layers.communicator.residual import batch as residual_batch -from sglang.srt.layers.dp_attention import ( - is_allocation_symmetric, - is_dp_attention_enabled, -) +from sglang.srt.layers.layer_boundary.residual import batch as residual_batch from sglang.srt.layers.layernorm import RMSNorm from sglang.srt.layers.linear import MergedColumnParallelLinear, RowParallelLinear from sglang.srt.layers.logits_processor import LogitsProcessor @@ -581,7 +581,7 @@ class Glm4MoeLiteDecoderLayer(nn.Module): self._gfx95_quant_format = self._detect_gfx95_quant_format() - self.attn_stage, self.ffn_stage = make_stages( + self.attn_boundary, self.ffn_boundary = make_stages( ( declare_attn(), self.input_layernorm, @@ -633,7 +633,7 @@ class Glm4MoeLiteDecoderLayer(nn.Module): zero_allocator: BumpAllocator, capture_output=None, ) -> torch.Tensor: - hidden_states = self.attn_stage.prepare( + hidden_states = self.attn_boundary.prepare( hidden_states, forward_batch, quant_format=getattr(self, "_gfx95_quant_format", ""), @@ -645,16 +645,18 @@ class Glm4MoeLiteDecoderLayer(nn.Module): hidden_states=hidden_states, forward_batch=forward_batch, zero_allocator=zero_allocator, - input_on_attention_tp_slices=(self.attn_stage.input_on_attention_tp_slices), + input_on_attention_tp_slices=( + self.attn_boundary.input_on_attention_tp_slices + ), ) if isinstance(hidden_states, tuple): hidden_states = hidden_states[0] get_attn_tp_context().clear_attn_inputs() - hidden_states = self.attn_stage.finish(hidden_states, forward_batch) - hidden_states = self.ffn_stage.prepare(hidden_states, forward_batch) + hidden_states = self.attn_boundary.finish(hidden_states, forward_batch) + hidden_states = self.ffn_boundary.prepare(hidden_states, forward_batch) - with self.ffn_stage.exit(forward_batch) as ffn_exit: + with self.ffn_boundary.exit(forward_batch) as ffn_exit: hidden_states = self.mlp(hidden_states, forward_batch) hidden_states = ffn_exit.finish(hidden_states) @@ -669,7 +671,7 @@ class Glm4MoeLiteDecoderLayer(nn.Module): zero_allocator: BumpAllocator, tbo_subbatch_index: Optional[int] = None, ): - state.hidden_states_after_comm_pre_attn = self.attn_stage.prepare( + state.hidden_states_after_comm_pre_attn = self.attn_boundary.prepare( hidden_states, forward_batch ) if get_moe_a2a_backend().is_mori(): @@ -684,15 +686,15 @@ class Glm4MoeLiteDecoderLayer(nn.Module): ) def op_comm_prepare_mlp(self, state): - hidden_states = self.attn_stage.finish( + hidden_states = self.attn_boundary.finish( state.pop("hidden_states_after_attn"), state.forward_batch ) - state.hidden_states_mlp_input = self.ffn_stage.prepare( + state.hidden_states_mlp_input = self.ffn_boundary.prepare( hidden_states, state.forward_batch ) def op_comm_postprocess_layer(self, state): - hidden_states = self.ffn_stage.postprocess( + hidden_states = self.ffn_boundary.postprocess( state.pop("hidden_states_mlp_output"), state.forward_batch ) @@ -779,7 +781,7 @@ class Glm4MoeLiteModel(nn.Module): residual_batch.start(forward_batch) else: assert pp_proxy_tensors is not None - hidden_states = self.layers[self.start_layer].attn_stage.from_pp( + hidden_states = self.layers[self.start_layer].attn_boundary.from_pp( pp_proxy_tensors, forward_batch ) device = hidden_states.device diff --git a/python/sglang/srt/models/glm4_moe_lite_nextn.py b/python/sglang/srt/models/glm4_moe_lite_nextn.py index 6a65cefa8f..25e0b7c93a 100644 --- a/python/sglang/srt/models/glm4_moe_lite_nextn.py +++ b/python/sglang/srt/models/glm4_moe_lite_nextn.py @@ -22,8 +22,8 @@ from torch import nn from transformers import PretrainedConfig from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder -from sglang.srt.layers.communicator.residual import batch as residual_batch from sglang.srt.layers.dp_attention import is_dp_attention_enabled +from sglang.srt.layers.layer_boundary.residual import batch as residual_batch from sglang.srt.layers.layernorm import RMSNorm from sglang.srt.layers.logits_processor import LogitsProcessor from sglang.srt.layers.quantization.base_config import QuantizationConfig diff --git a/python/sglang/srt/models/glm4_moe_nextn.py b/python/sglang/srt/models/glm4_moe_nextn.py index b806f86406..83750d2432 100644 --- a/python/sglang/srt/models/glm4_moe_nextn.py +++ b/python/sglang/srt/models/glm4_moe_nextn.py @@ -22,8 +22,8 @@ from torch import nn from transformers import PretrainedConfig from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder -from sglang.srt.layers.communicator.residual import batch as residual_batch from sglang.srt.layers.dp_attention import is_dp_attention_enabled +from sglang.srt.layers.layer_boundary.residual import batch as residual_batch from sglang.srt.layers.layernorm import RMSNorm from sglang.srt.layers.logits_processor import LogitsProcessor from sglang.srt.layers.quantization.base_config import QuantizationConfig diff --git a/python/sglang/srt/models/glm5_next.py b/python/sglang/srt/models/glm5_next.py index 688f67a766..5fc8c9bc5b 100644 --- a/python/sglang/srt/models/glm5_next.py +++ b/python/sglang/srt/models/glm5_next.py @@ -26,7 +26,7 @@ from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation from sglang.srt.layers.attention import vision_utils from sglang.srt.layers.attention.vision import VisionAttention from sglang.srt.layers.aux_hidden_states import AuxHiddenStateList -from sglang.srt.layers.communicator import ( +from sglang.srt.layers.layer_boundary import ( PLAIN_RESIDUAL, MHCState, declare_attn, @@ -35,8 +35,8 @@ from sglang.srt.layers.communicator import ( get_attn_tp_context, make_stages, ) -from sglang.srt.layers.communicator.residual import access as residual_access -from sglang.srt.layers.communicator.residual import batch as residual_batch +from sglang.srt.layers.layer_boundary.residual import access as residual_access +from sglang.srt.layers.layer_boundary.residual import batch as residual_batch from sglang.srt.layers.layernorm import RMSNorm from sglang.srt.layers.linear import ( ColumnParallelBatchedLinear, @@ -794,7 +794,7 @@ class Glm5NextDecoderLayer(nn.Module): ), is_last_layer=terminal, ).layer_residual() - self.attn_stage, self.ffn_stage = make_stages( + self.attn_boundary, self.ffn_boundary = make_stages( ( declare_attn( read=residual.attention_read, update=residual.attention_update @@ -932,7 +932,7 @@ class Glm5NextDecoderLayer(nn.Module): ): hidden_states_orig = residual_access.buffer(hidden_states) - hidden_states = self.attn_stage.prepare( + hidden_states = self.attn_boundary.prepare( hidden_states, forward_batch, capture_output=capture_output ) @@ -941,7 +941,9 @@ class Glm5NextDecoderLayer(nn.Module): hidden_states=hidden_states, forward_batch=forward_batch, zero_allocator=zero_allocator, - input_on_attention_tp_slices=(self.attn_stage.input_on_attention_tp_slices), + input_on_attention_tp_slices=( + self.attn_boundary.input_on_attention_tp_slices + ), prev_topk_indices=prev_topk_indices, ) if isinstance(hidden_states, tuple): @@ -950,8 +952,8 @@ class Glm5NextDecoderLayer(nn.Module): topk_indices = None get_attn_tp_context().clear_attn_inputs() - hidden_states = self.attn_stage.finish(hidden_states, forward_batch) - hidden_states = self.ffn_stage.prepare(hidden_states, forward_batch) + hidden_states = self.attn_boundary.finish(hidden_states, forward_batch) + hidden_states = self.ffn_boundary.prepare(hidden_states, forward_batch) if isinstance(self.mlp, Glm5NextMLP): gemm_output_zero_allocator = None @@ -967,7 +969,7 @@ class Glm5NextDecoderLayer(nn.Module): else: _mlp_ctx = nullcontext() - with self.ffn_stage.exit(forward_batch) as ffn_exit, _mlp_ctx: + with self.ffn_boundary.exit(forward_batch) as ffn_exit, _mlp_ctx: hidden_states = self.mlp( hidden_states, forward_batch, @@ -1108,7 +1110,7 @@ class Glm5NextModel(nn.Module): residual_batch.start(forward_batch) else: assert pp_proxy_tensors is not None - hidden_states = self.layers[self.start_layer].attn_stage.from_pp( + hidden_states = self.layers[self.start_layer].attn_boundary.from_pp( pp_proxy_tensors, forward_batch ) device = hidden_states.device diff --git a/python/sglang/srt/models/gpt_oss.py b/python/sglang/srt/models/gpt_oss.py index 9f6f955fb1..505f7afa75 100644 --- a/python/sglang/srt/models/gpt_oss.py +++ b/python/sglang/srt/models/gpt_oss.py @@ -30,15 +30,15 @@ from sglang.kernels.jit.utils import is_arch_support_pdl from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation from sglang.srt.layers.aux_hidden_states import AuxHiddenStateList -from sglang.srt.layers.communicator import ( +from sglang.srt.layers.dp_attention import ( + is_dp_attention_enabled, +) +from sglang.srt.layers.layer_boundary import ( declare_attn, declare_ffn, make_stages, ) -from sglang.srt.layers.communicator.residual import batch as residual_batch -from sglang.srt.layers.dp_attention import ( - is_dp_attention_enabled, -) +from sglang.srt.layers.layer_boundary.residual import batch as residual_batch from sglang.srt.layers.layernorm import RMSNorm from sglang.srt.layers.linear import ( QKVParallelLinear, @@ -599,7 +599,7 @@ class GptOssDecoderLayer(nn.Module): x_pad_to_multiple=post_attn_pad_multiple, ) - self.attn_stage, self.ffn_stage = make_stages( + self.attn_boundary, self.ffn_boundary = make_stages( (declare_attn(), self.input_layernorm), ( declare_ffn( @@ -623,7 +623,7 @@ class GptOssDecoderLayer(nn.Module): forward_batch: ForwardBatch, capture_output=None, ) -> torch.Tensor: - hidden_states = self.attn_stage.prepare( + hidden_states = self.attn_boundary.prepare( hidden_states, forward_batch, capture_output=capture_output ) @@ -634,10 +634,10 @@ class GptOssDecoderLayer(nn.Module): forward_batch=forward_batch, ) - hidden_states = self.attn_stage.finish(hidden_states, forward_batch) - hidden_states = self.ffn_stage.prepare(hidden_states, forward_batch) + hidden_states = self.attn_boundary.finish(hidden_states, forward_batch) + hidden_states = self.ffn_boundary.prepare(hidden_states, forward_batch) - with self.ffn_stage.exit(forward_batch) as ffn_exit: + with self.ffn_boundary.exit(forward_batch) as ffn_exit: hidden_states = self.mlp(hidden_states, forward_batch) hidden_states = ffn_exit.finish(hidden_states) @@ -707,7 +707,7 @@ class GptOssModel(nn.Module): residual_batch.start(forward_batch) else: assert pp_proxy_tensors is not None - hidden_states = self.layers[self.start_layer].attn_stage.from_pp( + hidden_states = self.layers[self.start_layer].attn_boundary.from_pp( pp_proxy_tensors, forward_batch ) diff --git a/python/sglang/srt/models/hunyuan_v4.py b/python/sglang/srt/models/hunyuan_v4.py index 8f59b56add..fce9541572 100644 --- a/python/sglang/srt/models/hunyuan_v4.py +++ b/python/sglang/srt/models/hunyuan_v4.py @@ -7,7 +7,7 @@ from torch import nn from transformers import PretrainedConfig from sglang.srt.layers.attention.index_topk_share import IndexTopKShareState -from sglang.srt.layers.communicator import AttentionInputs, get_attn_tp_context +from sglang.srt.layers.layer_boundary import AttentionInputs, get_attn_tp_context from sglang.srt.layers.layernorm import RMSNorm from sglang.srt.layers.linear import ColumnParallelLinear, ReplicatedLinear from sglang.srt.layers.logits_processor import LogitsProcessor diff --git a/python/sglang/srt/models/hunyuan_v4_nextn.py b/python/sglang/srt/models/hunyuan_v4_nextn.py index 60790f4ee7..93198d6842 100644 --- a/python/sglang/srt/models/hunyuan_v4_nextn.py +++ b/python/sglang/srt/models/hunyuan_v4_nextn.py @@ -5,7 +5,7 @@ import torch from torch import nn from sglang.srt.layers.attention.index_topk_share import IndexTopKShareState -from sglang.srt.layers.communicator import AttentionInputs, get_attn_tp_context +from sglang.srt.layers.layer_boundary import AttentionInputs, get_attn_tp_context from sglang.srt.layers.layernorm import RMSNorm from sglang.srt.layers.logits_processor import LogitsProcessor from sglang.srt.layers.vocab_parallel_embedding import ( diff --git a/python/sglang/srt/models/interns2_mobius.py b/python/sglang/srt/models/interns2_mobius.py index a71f405065..229daa8e80 100644 --- a/python/sglang/srt/models/interns2_mobius.py +++ b/python/sglang/srt/models/interns2_mobius.py @@ -14,14 +14,14 @@ from sglang.srt.layers.aux_hidden_states import ( AuxHiddenStateAccumulator, AuxHiddenStateList, ) -from sglang.srt.layers.communicator import ( +from sglang.srt.layers.dp_attention import is_dp_attention_enabled +from sglang.srt.layers.layer_boundary import ( declare_attn, declare_ffn, make_stages, ) -from sglang.srt.layers.communicator.residual import batch as residual_batch -from sglang.srt.layers.communicator.residual.add_norm import Fp8Input, NormQuantRead -from sglang.srt.layers.dp_attention import is_dp_attention_enabled +from sglang.srt.layers.layer_boundary.residual import batch as residual_batch +from sglang.srt.layers.layer_boundary.residual.add_norm import Fp8Input, NormQuantRead from sglang.srt.layers.layernorm import GemmaRMSNorm from sglang.srt.layers.linear import ( QKVParallelLinear, @@ -428,9 +428,9 @@ class _InternS2MobiusDecoderMixin: forward_batch: ForwardBatch, meta_mlp: nn.ModuleList, ) -> torch.Tensor: - hidden_states = self.attn_stage.finish(hidden_states, forward_batch) - hidden_states = self.ffn_stage.prepare(hidden_states, forward_batch) - with self.ffn_stage.exit(forward_batch) as ffn_exit: + hidden_states = self.attn_boundary.finish(hidden_states, forward_batch) + hidden_states = self.ffn_boundary.prepare(hidden_states, forward_batch) + with self.ffn_boundary.exit(forward_batch) as ffn_exit: hidden_states = self._forward_mobius_mlp( hidden_states, forward_batch, meta_mlp ) @@ -489,7 +489,7 @@ class InternS2MobiusLinearDecoderLayer(_InternS2MobiusDecoderMixin, nn.Module): config.hidden_size, eps=config.rms_norm_eps ) accepts_fp8_input = _linear_accepts_fp8_tuple(self.linear_attn.in_proj_qkvz) - self.attn_stage, self.ffn_stage = make_stages( + self.attn_boundary, self.ffn_boundary = make_stages( ( declare_attn( read=NormQuantRead( @@ -518,7 +518,7 @@ class InternS2MobiusLinearDecoderLayer(_InternS2MobiusDecoderMixin, nn.Module): **kwargs, ): forward_batch = kwargs["forward_batch"] - hidden_states = self.attn_stage.prepare( + hidden_states = self.attn_boundary.prepare( hidden_states, forward_batch, captured_last_layer_outputs=kwargs.get("captured_last_layer_outputs"), @@ -621,7 +621,7 @@ class InternS2MobiusAttentionDecoderLayer( self.q_norm = GemmaRMSNorm(self.head_dim, eps=config.rms_norm_eps) self.k_norm = GemmaRMSNorm(self.head_dim, eps=config.rms_norm_eps) accepts_fp8_input = _linear_accepts_fp8_tuple(self.qkv_proj) - self.attn_stage, self.ffn_stage = make_stages( + self.attn_boundary, self.ffn_boundary = make_stages( ( declare_attn( read=NormQuantRead( @@ -654,7 +654,7 @@ class InternS2MobiusAttentionDecoderLayer( **kwargs, ): del kwargs - hidden_states = self.attn_stage.prepare( + hidden_states = self.attn_boundary.prepare( hidden_states, forward_batch, captured_last_layer_outputs=captured_last_layer_outputs, diff --git a/python/sglang/srt/models/kimi_k25_eagle3.py b/python/sglang/srt/models/kimi_k25_eagle3.py index 24b34b21fa..d8285f7b22 100644 --- a/python/sglang/srt/models/kimi_k25_eagle3.py +++ b/python/sglang/srt/models/kimi_k25_eagle3.py @@ -25,7 +25,7 @@ from torch import nn from transformers import PretrainedConfig from sglang.srt.distributed.device_communicators import triton_symm_mem_ag -from sglang.srt.layers.communicator import AttentionInputs, get_attn_tp_context +from sglang.srt.layers.layer_boundary import AttentionInputs, get_attn_tp_context from sglang.srt.layers.layernorm import RMSNorm from sglang.srt.layers.linear import ColumnParallelLinear, ReplicatedLinear from sglang.srt.layers.logits_processor import LogitsProcessor diff --git a/python/sglang/srt/models/kimi_k3.py b/python/sglang/srt/models/kimi_k3.py index 4ae33981f7..b04ac8b40c 100644 --- a/python/sglang/srt/models/kimi_k3.py +++ b/python/sglang/srt/models/kimi_k3.py @@ -2615,7 +2615,7 @@ class KimiK3DecoderLayer(nn.Module): ) -> torch.Tensor: # For MLA layers with q_lora_rank, set up attn_inputs before the # forward call (normally done by the attention boundary). - from sglang.srt.layers.communicator import ( + from sglang.srt.layers.layer_boundary import ( AttentionInputs, get_attn_tp_context, ) diff --git a/python/sglang/srt/models/laguna.py b/python/sglang/srt/models/laguna.py index 3689a55fc0..22ce045c49 100644 --- a/python/sglang/srt/models/laguna.py +++ b/python/sglang/srt/models/laguna.py @@ -20,15 +20,15 @@ from sglang.srt.configs.laguna import LagunaConfig, normalize_gating from sglang.srt.environ import envs from sglang.srt.layers.activation import SiluAndMul from sglang.srt.layers.aux_hidden_states import AuxHiddenStateList -from sglang.srt.layers.communicator import ( +from sglang.srt.layers.dp_attention import ( + is_dp_attention_enabled, +) +from sglang.srt.layers.layer_boundary import ( declare_attn, declare_ffn, make_stages, ) -from sglang.srt.layers.communicator.residual import batch as residual_batch -from sglang.srt.layers.dp_attention import ( - is_dp_attention_enabled, -) +from sglang.srt.layers.layer_boundary.residual import batch as residual_batch from sglang.srt.layers.layernorm import RMSNorm from sglang.srt.layers.linear import ( ColumnParallelLinear, @@ -457,7 +457,7 @@ class LagunaDecoderLayer(nn.Module): config.hidden_size, eps=config.rms_norm_eps ) - self.attn_stage, self.ffn_stage = make_stages( + self.attn_boundary, self.ffn_boundary = make_stages( (declare_attn(), self.input_layernorm), ( declare_ffn( @@ -481,7 +481,7 @@ class LagunaDecoderLayer(nn.Module): forward_batch: ForwardBatch, capture_output=None, ) -> torch.Tensor: - hidden_states = self.attn_stage.prepare( + hidden_states = self.attn_boundary.prepare( hidden_states, forward_batch, capture_output=capture_output ) if hidden_states.shape[0] != 0: @@ -490,10 +490,10 @@ class LagunaDecoderLayer(nn.Module): hidden_states=hidden_states, forward_batch=forward_batch, ) - hidden_states = self.attn_stage.finish(hidden_states, forward_batch) - hidden_states = self.ffn_stage.prepare(hidden_states, forward_batch) + hidden_states = self.attn_boundary.finish(hidden_states, forward_batch) + hidden_states = self.ffn_boundary.prepare(hidden_states, forward_batch) - with self.ffn_stage.exit(forward_batch) as ffn_exit: + with self.ffn_boundary.exit(forward_batch) as ffn_exit: hidden_states = self.mlp( hidden_states, forward_batch=forward_batch, @@ -589,7 +589,7 @@ class LagunaModel(nn.Module): residual_batch.start(forward_batch) else: assert pp_proxy_tensors is not None - hidden_states = self.layers[self.start_layer].attn_stage.from_pp( + hidden_states = self.layers[self.start_layer].attn_boundary.from_pp( pp_proxy_tensors, forward_batch ) diff --git a/python/sglang/srt/models/llada2.py b/python/sglang/srt/models/llada2.py index a4526b0d7c..129e3b9525 100644 --- a/python/sglang/srt/models/llada2.py +++ b/python/sglang/srt/models/llada2.py @@ -31,16 +31,16 @@ from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_r from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation from sglang.srt.eplb.expert_location_dispatch import ExpertLocationDispatchInfo from sglang.srt.layers.activation import SiluAndMul -from sglang.srt.layers.communicator import ( +from sglang.srt.layers.dp_attention import ( + is_dp_attention_enabled, +) +from sglang.srt.layers.layer_boundary import ( declare_attn, declare_ffn, enable_moe_dense_fully_dp, make_stages, ) -from sglang.srt.layers.communicator.residual import batch as residual_batch -from sglang.srt.layers.dp_attention import ( - is_dp_attention_enabled, -) +from sglang.srt.layers.layer_boundary.residual import batch as residual_batch from sglang.srt.layers.layernorm import RMSNorm from sglang.srt.layers.linear import ( MergedColumnParallelLinear, @@ -638,7 +638,7 @@ class LLaDA2MoeBlock(nn.Module): self.post_attention_layernorm = RMSNorm(hidden_size, eps=config.rms_norm_eps) - self.attn_stage, self.ffn_stage = make_stages( + self.attn_boundary, self.ffn_boundary = make_stages( (declare_attn(), self.input_layernorm), ( declare_ffn( @@ -666,7 +666,7 @@ class LLaDA2MoeBlock(nn.Module): hidden_states: torch.Tensor, forward_batch: ForwardBatch, ) -> torch.Tensor: - hidden_states = self.attn_stage.prepare( + hidden_states = self.attn_boundary.prepare( hidden_states=hidden_states, forward_batch=forward_batch ) @@ -676,13 +676,13 @@ class LLaDA2MoeBlock(nn.Module): forward_batch=forward_batch, ) - hidden_states = self.attn_stage.finish(hidden_states, forward_batch) - hidden_states = self.ffn_stage.prepare( + hidden_states = self.attn_boundary.finish(hidden_states, forward_batch) + hidden_states = self.ffn_boundary.prepare( hidden_states=hidden_states, forward_batch=forward_batch, ) - with self.ffn_stage.exit(forward_batch) as ffn_exit: + with self.ffn_boundary.exit(forward_batch) as ffn_exit: hidden_states = self.mlp(hidden_states, forward_batch) hidden_states = ffn_exit.finish(hidden_states) @@ -749,7 +749,7 @@ class LLaDA2MoeModel(nn.Module): residual_batch.start(forward_batch) else: assert pp_proxy_tensors is not None - hidden_states = self.layers[self.start_layer].attn_stage.from_pp( + hidden_states = self.layers[self.start_layer].attn_boundary.from_pp( pp_proxy_tensors, forward_batch ) diff --git a/python/sglang/srt/models/llama4.py b/python/sglang/srt/models/llama4.py index a4f22147e9..81aeb414a2 100644 --- a/python/sglang/srt/models/llama4.py +++ b/python/sglang/srt/models/llama4.py @@ -26,15 +26,15 @@ from torch import nn from transformers import Llama4TextConfig from sglang.srt.layers.aux_hidden_states import AuxHiddenStateList -from sglang.srt.layers.communicator import ( +from sglang.srt.layers.dp_attention import ( + is_dp_attention_enabled, +) +from sglang.srt.layers.layer_boundary import ( declare_attn, declare_ffn, make_stages, ) -from sglang.srt.layers.communicator.residual import batch as residual_batch -from sglang.srt.layers.dp_attention import ( - is_dp_attention_enabled, -) +from sglang.srt.layers.layer_boundary.residual import batch as residual_batch from sglang.srt.layers.layernorm import RMSNorm from sglang.srt.layers.linear import ( QKVParallelLinear, @@ -426,7 +426,7 @@ class Llama4DecoderLayer(nn.Module): config.hidden_size, eps=config.rms_norm_eps ) - self.attn_stage, self.ffn_stage = make_stages( + self.attn_boundary, self.ffn_boundary = make_stages( (declare_attn(), self.input_layernorm), ( declare_ffn( @@ -459,7 +459,7 @@ class Llama4DecoderLayer(nn.Module): forward_batch: ForwardBatch, capture_output=None, ) -> torch.Tensor: - hidden_states = self.attn_stage.prepare( + hidden_states = self.attn_boundary.prepare( hidden_states, forward_batch, capture_output=capture_output ) @@ -470,11 +470,11 @@ class Llama4DecoderLayer(nn.Module): forward_batch=forward_batch, ) - hidden_states = self.attn_stage.finish(hidden_states, forward_batch) - hidden_states = self.ffn_stage.prepare(hidden_states, forward_batch) + hidden_states = self.attn_boundary.finish(hidden_states, forward_batch) + hidden_states = self.ffn_boundary.prepare(hidden_states, forward_batch) # Fully Connected - with self.ffn_stage.exit(forward_batch) as ffn_exit: + with self.ffn_boundary.exit(forward_batch) as ffn_exit: hidden_states = self.feed_forward(hidden_states, forward_batch) hidden_states = ffn_exit.finish(hidden_states) diff --git a/python/sglang/srt/models/longcat_flash.py b/python/sglang/srt/models/longcat_flash.py index a1bfc2b514..1ed7c296e4 100644 --- a/python/sglang/srt/models/longcat_flash.py +++ b/python/sglang/srt/models/longcat_flash.py @@ -51,15 +51,15 @@ from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_r from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation from sglang.srt.layers import deep_gemm_wrapper from sglang.srt.layers.activation import SiluAndMul -from sglang.srt.layers.communicator import ( +from sglang.srt.layers.dp_attention import ( + is_dp_attention_enabled, +) +from sglang.srt.layers.layer_boundary import ( declare_attn, declare_ffn, make_stages, ) -from sglang.srt.layers.communicator.residual import batch as residual_batch -from sglang.srt.layers.dp_attention import ( - is_dp_attention_enabled, -) +from sglang.srt.layers.layer_boundary.residual import batch as residual_batch from sglang.srt.layers.layernorm import RMSNorm from sglang.srt.layers.linear import ( MergedColumnParallelLinear, @@ -431,7 +431,7 @@ class LongcatFlashDecoderLayer(nn.Module): self.attn_tp_size = get_parallel().attn_tp_size self.attn_tp_rank = get_parallel().attn_tp_rank - self.attn_stage, self.moe_stage = make_stages( + self.attn_boundary, self.moe_boundary = make_stages( ( declare_attn(), self.input_layernorm[0], @@ -446,21 +446,23 @@ class LongcatFlashDecoderLayer(nn.Module): else None, terminal=self.layer_id == config.num_hidden_layers - 1, ) - self.first_ffn, self.second_attn, self.second_ffn = make_stages( - ( - declare_ffn(), - self.post_attention_layernorm[0], - ), - ( - declare_attn(), - self.input_layernorm[1], - {"qkv_latent_func": self.self_attn[1].prepare_qkv_latent}, - ), - ( - declare_ffn(), - self.post_attention_layernorm[1], - ), - prepared_from=self.moe_stage.declaration, + self.first_ffn_boundary, self.second_attn_boundary, self.second_ffn_boundary = ( + make_stages( + ( + declare_ffn(), + self.post_attention_layernorm[0], + ), + ( + declare_attn(), + self.input_layernorm[1], + {"qkv_latent_func": self.self_attn[1].prepare_qkv_latent}, + ), + ( + declare_ffn(), + self.post_attention_layernorm[1], + ), + prepared_from=self.moe_boundary.declaration, + ) ) def forward( @@ -472,7 +474,7 @@ class LongcatFlashDecoderLayer(nn.Module): prev_topk_indices: Optional[torch.Tensor], ) -> torch.Tensor: # first_attn - hidden_states = self.attn_stage.prepare(hidden_states, forward_batch) + hidden_states = self.attn_boundary.prepare(hidden_states, forward_batch) if hidden_states.shape[0] != 0: attn_out = self.self_attn[0]( positions=positions, @@ -487,9 +489,9 @@ class LongcatFlashDecoderLayer(nn.Module): hidden_states = attn_out # moe - hidden_states = self.attn_stage.finish(hidden_states, forward_batch) - hidden_states = self.moe_stage.prepare(hidden_states, forward_batch) - moe_hidden_states = self.moe_stage.branch_output( + hidden_states = self.attn_boundary.finish(hidden_states, forward_batch) + hidden_states = self.moe_boundary.prepare(hidden_states, forward_batch) + moe_hidden_states = self.moe_boundary.branch_output( self.mlp(hidden_states.clone()), forward_batch ) @@ -501,10 +503,10 @@ class LongcatFlashDecoderLayer(nn.Module): prev_topk_indices, ) - hidden_states = self.moe_stage.merge_branch( + hidden_states = self.moe_boundary.merge_branch( moe_hidden_states, hidden_states, - self.second_ffn, + self.second_ffn_boundary, forward_batch, ) return hidden_states, prev_topk_indices @@ -518,15 +520,15 @@ class LongcatFlashDecoderLayer(nn.Module): prev_topk_indices, ): # first_mlp, on the input the MoE's boundary read - hidden_states = self.first_ffn.branch_input( - self.moe_stage, hidden_states, forward_batch + hidden_states = self.first_ffn_boundary.branch_input( + self.moe_boundary, hidden_states, forward_batch ) - with self.first_ffn.exit(forward_batch) as ffn_exit: + with self.first_ffn_boundary.exit(forward_batch) as ffn_exit: hidden_states = self.mlps[0](hidden_states) hidden_states = ffn_exit.finish(hidden_states) - # second_attn - hidden_states = self.second_attn.prepare(hidden_states, forward_batch) + # second_attn_boundary + hidden_states = self.second_attn_boundary.prepare(hidden_states, forward_batch) if hidden_states.shape[0] != 0: attn_out = self.self_attn[1]( positions=positions, @@ -541,9 +543,9 @@ class LongcatFlashDecoderLayer(nn.Module): hidden_states = attn_out # second_mlp - hidden_states = self.second_attn.finish(hidden_states, forward_batch) - hidden_states = self.second_ffn.prepare(hidden_states, forward_batch) - with self.second_ffn.exit(forward_batch) as ffn_exit: + hidden_states = self.second_attn_boundary.finish(hidden_states, forward_batch) + hidden_states = self.second_ffn_boundary.prepare(hidden_states, forward_batch) + with self.second_ffn_boundary.exit(forward_batch) as ffn_exit: hidden_states = self.mlps[1](hidden_states) hidden_states = ffn_exit.finish(hidden_states) @@ -628,7 +630,7 @@ class LongcatFlashModel(nn.Module): for i in range(total_num_layers): if i in self.layers_to_capture: aux_hidden_states.append( - self.layers[i].attn_stage.snapshot(hidden_states, forward_batch) + self.layers[i].attn_boundary.snapshot(hidden_states, forward_batch) ) with get_global_expert_distribution_recorder().with_current_layer(i): layer = self.layers[i] diff --git a/python/sglang/srt/models/longcat_flash_nextn.py b/python/sglang/srt/models/longcat_flash_nextn.py index 9b8a6cc815..90f71e2647 100644 --- a/python/sglang/srt/models/longcat_flash_nextn.py +++ b/python/sglang/srt/models/longcat_flash_nextn.py @@ -41,15 +41,15 @@ from sglang.kernels.ops.quantization.fp8_kernel import is_fp8_fnuz from sglang.srt.configs import LongcatFlashConfig from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder from sglang.srt.layers import deep_gemm_wrapper -from sglang.srt.layers.communicator import ( +from sglang.srt.layers.dp_attention import ( + is_dp_attention_enabled, +) +from sglang.srt.layers.layer_boundary import ( declare_attn, declare_ffn, make_stages, ) -from sglang.srt.layers.communicator.residual import batch as residual_batch -from sglang.srt.layers.dp_attention import ( - is_dp_attention_enabled, -) +from sglang.srt.layers.layer_boundary.residual import batch as residual_batch from sglang.srt.layers.layernorm import RMSNorm from sglang.srt.layers.linear import ReplicatedLinear from sglang.srt.layers.logits_processor import LogitsProcessor @@ -165,7 +165,7 @@ class LongcatFlashDenseDecoderLayer(nn.Module): self.attn_tp_size = get_parallel().attn_tp_size self.attn_tp_rank = get_parallel().attn_tp_rank - self.attn_stage, self.ffn_stage = make_stages( + self.attn_boundary, self.ffn_boundary = make_stages( (declare_attn(), self.input_layernorm), ( declare_ffn(sparse=False, next_sparse=False), @@ -183,7 +183,7 @@ class LongcatFlashDenseDecoderLayer(nn.Module): zero_allocator: BumpAllocator, ) -> torch.Tensor: - hidden_states = self.attn_stage.prepare(hidden_states, forward_batch) + hidden_states = self.attn_boundary.prepare(hidden_states, forward_batch) if hidden_states.shape[0] != 0: hidden_states = self.self_attn( positions=positions, @@ -192,10 +192,10 @@ class LongcatFlashDenseDecoderLayer(nn.Module): zero_allocator=zero_allocator, ) - hidden_states = self.attn_stage.finish(hidden_states, forward_batch) - hidden_states = self.ffn_stage.prepare(hidden_states, forward_batch) + hidden_states = self.attn_boundary.finish(hidden_states, forward_batch) + hidden_states = self.ffn_boundary.prepare(hidden_states, forward_batch) hidden_states = self.mlp(hidden_states) - hidden_states = self.ffn_stage.postprocess(hidden_states, forward_batch) + hidden_states = self.ffn_boundary.postprocess(hidden_states, forward_batch) return hidden_states diff --git a/python/sglang/srt/models/mellum.py b/python/sglang/srt/models/mellum.py index 1915c728ba..5a7d394980 100644 --- a/python/sglang/srt/models/mellum.py +++ b/python/sglang/srt/models/mellum.py @@ -25,7 +25,7 @@ import torch from torch import nn from transformers import PretrainedConfig -from sglang.srt.layers.communicator import ( +from sglang.srt.layers.layer_boundary import ( declare_attn, declare_ffn, make_stages, @@ -449,7 +449,7 @@ class MellumDecoderLayer(Qwen3MoeDecoderLayer): self.input_layernorm = RMSNorm(cfg.hidden_size, eps=rms_norm_eps) self.post_attention_layernorm = RMSNorm(cfg.hidden_size, eps=rms_norm_eps) - self.attn_stage, self.ffn_stage = make_stages( + self.attn_boundary, self.ffn_boundary = make_stages( (declare_attn(), self.input_layernorm), ( declare_ffn( diff --git a/python/sglang/srt/models/mimo_v2.py b/python/sglang/srt/models/mimo_v2.py index 5cfac4be8c..de9bc0bef3 100644 --- a/python/sglang/srt/models/mimo_v2.py +++ b/python/sglang/srt/models/mimo_v2.py @@ -32,16 +32,16 @@ from sglang.srt.layers.aux_hidden_states import ( AuxHiddenStateAccumulator, AuxHiddenStatePacker, ) -from sglang.srt.layers.communicator import ( +from sglang.srt.layers.dp_attention import ( + is_dp_attention_enabled, +) +from sglang.srt.layers.layer_boundary import ( declare_attn, declare_ffn, enable_moe_dense_fully_dp, make_stages, ) -from sglang.srt.layers.communicator.residual import batch as residual_batch -from sglang.srt.layers.dp_attention import ( - is_dp_attention_enabled, -) +from sglang.srt.layers.layer_boundary.residual import batch as residual_batch from sglang.srt.layers.layernorm import RMSNorm from sglang.srt.layers.linear import ( MergedColumnParallelLinear, @@ -858,7 +858,7 @@ class MiMoV2DecoderLayer(nn.Module): config.hidden_size, eps=config.layernorm_epsilon ) - self.attn_stage, self.ffn_stage = make_stages( + self.attn_boundary, self.ffn_boundary = make_stages( (declare_attn(), self.input_layernorm), ( declare_ffn( @@ -883,7 +883,7 @@ class MiMoV2DecoderLayer(nn.Module): captured_last_layer_outputs: Optional[AuxHiddenStateAccumulator] = None, ) -> torch.Tensor: # Self Attention - hidden_states = self.attn_stage.prepare( + hidden_states = self.attn_boundary.prepare( hidden_states, forward_batch, captured_last_layer_outputs=captured_last_layer_outputs, @@ -896,10 +896,10 @@ class MiMoV2DecoderLayer(nn.Module): forward_batch=forward_batch, ) - hidden_states = self.attn_stage.finish(hidden_states, forward_batch) - hidden_states = self.ffn_stage.prepare(hidden_states, forward_batch) + hidden_states = self.attn_boundary.finish(hidden_states, forward_batch) + hidden_states = self.ffn_boundary.prepare(hidden_states, forward_batch) - with self.ffn_stage.exit(forward_batch) as ffn_exit: + with self.ffn_boundary.exit(forward_batch) as ffn_exit: hidden_states = self.mlp(hidden_states, forward_batch) hidden_states = ffn_exit.finish(hidden_states) @@ -924,7 +924,7 @@ class MiMoV2DecoderLayer(nn.Module): forward_batch: ForwardBatch, tbo_subbatch_index: Optional[int] = None, ): - state.hidden_states_after_comm_pre_attn = self.attn_stage.prepare( + state.hidden_states_after_comm_pre_attn = self.attn_boundary.prepare( hidden_states, forward_batch ) state.update( @@ -936,15 +936,15 @@ class MiMoV2DecoderLayer(nn.Module): ) def op_comm_prepare_mlp(self, state): - hidden_states = self.attn_stage.finish( + hidden_states = self.attn_boundary.finish( state.pop("hidden_states_after_attn"), state.forward_batch ) - state.hidden_states_mlp_input = self.ffn_stage.prepare( + state.hidden_states_mlp_input = self.ffn_boundary.prepare( hidden_states, state.forward_batch ) def op_comm_postprocess_layer(self, state): - hidden_states = self.ffn_stage.postprocess( + hidden_states = self.ffn_boundary.postprocess( state.pop("hidden_states_mlp_output"), state.forward_batch ) @@ -1035,7 +1035,7 @@ class MiMoV2Model(nn.Module): residual_batch.start(forward_batch) else: assert pp_proxy_tensors is not None - hidden_states = self.layers[self.start_layer].attn_stage.from_pp( + hidden_states = self.layers[self.start_layer].attn_boundary.from_pp( pp_proxy_tensors, forward_batch ) diff --git a/python/sglang/srt/models/mimo_v2_nextn.py b/python/sglang/srt/models/mimo_v2_nextn.py index d4d1e8a544..95f48457ac 100644 --- a/python/sglang/srt/models/mimo_v2_nextn.py +++ b/python/sglang/srt/models/mimo_v2_nextn.py @@ -21,16 +21,16 @@ from transformers import PretrainedConfig from sglang.srt.configs.model_config import get_mimo_v2_fused_qkv_expected_tp_size from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder -from sglang.srt.layers.communicator import ( +from sglang.srt.layers.dp_attention import ( + is_dp_attention_enabled, +) +from sglang.srt.layers.layer_boundary import ( declare_attn, declare_ffn, enable_moe_dense_fully_dp, make_stages, ) -from sglang.srt.layers.communicator.residual import batch as residual_batch -from sglang.srt.layers.dp_attention import ( - is_dp_attention_enabled, -) +from sglang.srt.layers.layer_boundary.residual import batch as residual_batch from sglang.srt.layers.layernorm import RMSNorm from sglang.srt.layers.logits_processor import LogitsProcessor from sglang.srt.layers.quantization.base_config import QuantizationConfig @@ -119,7 +119,7 @@ class MiMoV2MTPLayer(nn.Module): config.hidden_size, eps=config.layernorm_epsilon ) - self.attn_stage, self.ffn_stage = make_stages( + self.attn_boundary, self.ffn_boundary = make_stages( (declare_attn(), self.input_layernorm), ( declare_ffn( @@ -143,7 +143,7 @@ class MiMoV2MTPLayer(nn.Module): forward_batch: ForwardBatch, ) -> torch.Tensor: - hidden_states = self.attn_stage.prepare(hidden_states, forward_batch) + hidden_states = self.attn_boundary.prepare(hidden_states, forward_batch) if hidden_states.shape[0] != 0: hidden_states = self.self_attn( @@ -152,10 +152,10 @@ class MiMoV2MTPLayer(nn.Module): forward_batch=forward_batch, ) - hidden_states = self.attn_stage.finish(hidden_states, forward_batch) - hidden_states = self.ffn_stage.prepare(hidden_states, forward_batch) + hidden_states = self.attn_boundary.finish(hidden_states, forward_batch) + hidden_states = self.ffn_boundary.prepare(hidden_states, forward_batch) with ( - self.ffn_stage.exit(forward_batch) as ffn_exit, + self.ffn_boundary.exit(forward_batch) as ffn_exit, get_global_expert_distribution_recorder().disable_this_region(), ): hidden_states = self.mlp(hidden_states) diff --git a/python/sglang/srt/models/minimax_m2.py b/python/sglang/srt/models/minimax_m2.py index db5b2078e8..0beecf8eef 100644 --- a/python/sglang/srt/models/minimax_m2.py +++ b/python/sglang/srt/models/minimax_m2.py @@ -34,16 +34,16 @@ from sglang.srt.layers.aux_hidden_states import ( AuxHiddenStateAccumulator, AuxHiddenStateList, ) -from sglang.srt.layers.communicator import ( - declare_attn, - declare_ffn, - make_stages, -) -from sglang.srt.layers.communicator.residual import batch as residual_batch from sglang.srt.layers.dp_attention import ( attn_tp_all_reduce, is_dp_attention_enabled, ) +from sglang.srt.layers.layer_boundary import ( + declare_attn, + declare_ffn, + make_stages, +) +from sglang.srt.layers.layer_boundary.residual import batch as residual_batch from sglang.srt.layers.layernorm import RMSNorm from sglang.srt.layers.linear import ( QKVParallelLinear, @@ -982,7 +982,7 @@ class MiniMaxM2DecoderLayer(nn.Module): is_previous_layer_sparse = True is_next_layer_sparse = True - self.attn_stage, self.ffn_stage = make_stages( + self.attn_boundary, self.ffn_boundary = make_stages( (declare_attn(), self.input_layernorm), ( declare_ffn( @@ -1007,7 +1007,7 @@ class MiniMaxM2DecoderLayer(nn.Module): captured_last_layer_outputs: Optional[AuxHiddenStateAccumulator] = None, ) -> torch.Tensor: # Self Attention - hidden_states = self.attn_stage.prepare( + hidden_states = self.attn_boundary.prepare( hidden_states, forward_batch, captured_last_layer_outputs=captured_last_layer_outputs, @@ -1021,10 +1021,10 @@ class MiniMaxM2DecoderLayer(nn.Module): # Fully Connected (MLP or MoE) - hidden_states = self.attn_stage.finish(hidden_states, forward_batch) - hidden_states = self.ffn_stage.prepare(hidden_states, forward_batch) + hidden_states = self.attn_boundary.finish(hidden_states, forward_batch) + hidden_states = self.ffn_boundary.prepare(hidden_states, forward_batch) - with self.ffn_stage.exit(forward_batch) as ffn_exit: + with self.ffn_boundary.exit(forward_batch) as ffn_exit: hidden_states = self.block_sparse_moe(hidden_states, forward_batch) hidden_states = ffn_exit.finish(hidden_states) @@ -1041,7 +1041,7 @@ class MiniMaxM2DecoderLayer(nn.Module): tbo_subbatch_index: Optional[int] = None, ): """Communication prepare for attention - TBO operation""" - state.hidden_states_after_comm_pre_attn = self.attn_stage.prepare( + state.hidden_states_after_comm_pre_attn = self.attn_boundary.prepare( hidden_states, forward_batch ) state.update( @@ -1055,16 +1055,16 @@ class MiniMaxM2DecoderLayer(nn.Module): def op_comm_prepare_mlp(self, state): """Communication prepare for MLP - TBO operation""" - hidden_states = self.attn_stage.finish( + hidden_states = self.attn_boundary.finish( state.pop("hidden_states_after_attn"), state.forward_batch ) - state.hidden_states_mlp_input = self.ffn_stage.prepare( + state.hidden_states_mlp_input = self.ffn_boundary.prepare( hidden_states, state.forward_batch ) def op_comm_postprocess_layer(self, state): """Communication postprocess for layer - TBO operation""" - hidden_states = self.ffn_stage.postprocess( + hidden_states = self.ffn_boundary.postprocess( state.pop("hidden_states_mlp_output"), state.forward_batch ) @@ -1143,7 +1143,7 @@ class MiniMaxM2Model(nn.Module): residual_batch.start(forward_batch) else: assert pp_proxy_tensors is not None - hidden_states = self.layers[self.start_layer].attn_stage.from_pp( + hidden_states = self.layers[self.start_layer].attn_boundary.from_pp( pp_proxy_tensors, forward_batch ) diff --git a/python/sglang/srt/models/minimax_m3.py b/python/sglang/srt/models/minimax_m3.py index 7bded1282d..f08b2c1a8d 100644 --- a/python/sglang/srt/models/minimax_m3.py +++ b/python/sglang/srt/models/minimax_m3.py @@ -39,16 +39,16 @@ from sglang.srt.layers.aux_hidden_states import ( AuxHiddenStateAccumulator, AuxHiddenStateList, ) -from sglang.srt.layers.communicator import ( +from sglang.srt.layers.dp_attention import ( + is_dp_attention_enabled, +) +from sglang.srt.layers.layer_boundary import ( declare_attn, declare_ffn, enable_moe_dense_fully_dp, make_stages, ) -from sglang.srt.layers.communicator.residual import batch as residual_batch -from sglang.srt.layers.dp_attention import ( - is_dp_attention_enabled, -) +from sglang.srt.layers.layer_boundary.residual import batch as residual_batch from sglang.srt.layers.layernorm import GemmaRMSNorm, RMSNorm from sglang.srt.layers.linear import ( MergedColumnParallelLinear, @@ -1354,7 +1354,7 @@ class MiniMaxM3DecoderLayer(nn.Module): is_previous_layer_sparse = _is_layer_sparse(layer_id - 1) is_next_layer_sparse = _is_layer_sparse(layer_id + 1) - self.attn_stage, self.ffn_stage = make_stages( + self.attn_boundary, self.ffn_boundary = make_stages( (declare_attn(), self.input_layernorm), ( declare_ffn( @@ -1379,7 +1379,7 @@ class MiniMaxM3DecoderLayer(nn.Module): captured_last_layer_outputs: Optional[AuxHiddenStateAccumulator] = None, **kwargs, ) -> torch.Tensor: - hidden_states = self.attn_stage.prepare( + hidden_states = self.attn_boundary.prepare( hidden_states, forward_batch, captured_last_layer_outputs=captured_last_layer_outputs, @@ -1393,10 +1393,10 @@ class MiniMaxM3DecoderLayer(nn.Module): forward_batch=forward_batch, ) - hidden_states = self.attn_stage.finish(hidden_states, forward_batch) - hidden_states = self.ffn_stage.prepare(hidden_states, forward_batch) + hidden_states = self.attn_boundary.finish(hidden_states, forward_batch) + hidden_states = self.ffn_boundary.prepare(hidden_states, forward_batch) - with self.ffn_stage.exit(forward_batch) as ffn_exit: + with self.ffn_boundary.exit(forward_batch) as ffn_exit: if self.is_layer_sparse or hidden_states.shape[0] != 0: hidden_states = self.mlp( hidden_states, @@ -1483,7 +1483,7 @@ class MiniMaxM3Model(nn.Module): residual_batch.start(forward_batch) else: assert pp_proxy_tensors is not None - hidden_states = self.layers[self.start_layer].attn_stage.from_pp( + hidden_states = self.layers[self.start_layer].attn_boundary.from_pp( pp_proxy_tensors, forward_batch ) diff --git a/python/sglang/srt/models/moss_vl.py b/python/sglang/srt/models/moss_vl.py index 91a7d83ccb..a6b54f31b3 100644 --- a/python/sglang/srt/models/moss_vl.py +++ b/python/sglang/srt/models/moss_vl.py @@ -22,13 +22,13 @@ from sglang.srt.layers.attention.vision import ( VisionAttentionMetadata, prepare_vision_attention_metadata, ) -from sglang.srt.layers.communicator import ( +from sglang.srt.layers.conv import Conv3dLayer +from sglang.srt.layers.layer_boundary import ( declare_attn, declare_ffn, make_stages, ) -from sglang.srt.layers.communicator.residual import batch as residual_batch -from sglang.srt.layers.conv import Conv3dLayer +from sglang.srt.layers.layer_boundary.residual import batch as residual_batch from sglang.srt.layers.layernorm import RMSNorm from sglang.srt.layers.linear import ( ColumnParallelLinear, @@ -1017,7 +1017,7 @@ class MossVLSelfAttentionDecoderLayer(nn.Module): config.hidden_size, eps=config.rms_norm_eps, **norm_kwargs ) - self.attn_stage, self.ffn_stage = make_stages( + self.attn_boundary, self.ffn_boundary = make_stages( (declare_attn(), self.input_layernorm), ( declare_ffn(sparse=False, next_sparse=False), @@ -1034,7 +1034,7 @@ class MossVLSelfAttentionDecoderLayer(nn.Module): forward_batch: ForwardBatch, ) -> torch.Tensor: # Self Attention - hidden_states = self.attn_stage.prepare(hidden_states, forward_batch) + hidden_states = self.attn_boundary.prepare(hidden_states, forward_batch) if hidden_states.shape[0] != 0: hidden_states = self.self_attn( positions=positions, @@ -1043,9 +1043,9 @@ class MossVLSelfAttentionDecoderLayer(nn.Module): ) # MLP - hidden_states = self.attn_stage.finish(hidden_states, forward_batch) - hidden_states = self.ffn_stage.prepare(hidden_states, forward_batch) - with self.ffn_stage.exit(forward_batch) as ffn_exit: + hidden_states = self.attn_boundary.finish(hidden_states, forward_batch) + hidden_states = self.ffn_boundary.prepare(hidden_states, forward_batch) + with self.ffn_boundary.exit(forward_batch) as ffn_exit: hidden_states = self.mlp(hidden_states) hidden_states = ffn_exit.finish(hidden_states) return hidden_states diff --git a/python/sglang/srt/models/nemotron_h.py b/python/sglang/srt/models/nemotron_h.py index 3f6a7e39ce..4a0cc531c6 100644 --- a/python/sglang/srt/models/nemotron_h.py +++ b/python/sglang/srt/models/nemotron_h.py @@ -31,10 +31,10 @@ from sglang.srt.layers.attention.hybrid_linear_attn_backend import ( Mamba2AttnBackend, ) from sglang.srt.layers.attention.mamba.mamba import MambaMixer2 -from sglang.srt.layers.communicator.residual import batch as residual_batch from sglang.srt.layers.dp_attention import ( is_dp_attention_enabled, ) +from sglang.srt.layers.layer_boundary.residual import batch as residual_batch from sglang.srt.layers.layernorm import RMSNorm from sglang.srt.layers.linear import ( ColumnParallelLinear, @@ -376,7 +376,7 @@ class NemotronHMLPLikeDecoderLayer(nn.Module): sum an FFN before it left.""" def _init_stage_boundary(self, config: NemotronHConfig, layer_idx: int): - self.stage = make_stage_boundary( + self.boundary = make_stage_boundary( self.norm, pattern=config.hybrid_override_pattern, layer_idx=layer_idx ) @@ -386,8 +386,8 @@ class NemotronHMLPLikeDecoderLayer(nn.Module): hidden_states, forward_batch: ForwardBatch, ): - hidden_states = self.stage.prepare(hidden_states, forward_batch) - with self.stage.exit(forward_batch) as ffn_exit: + hidden_states = self.boundary.prepare(hidden_states, forward_batch) + with self.boundary.exit(forward_batch) as ffn_exit: hidden_states = self.mixer.forward(hidden_states) return ffn_exit.finish(hidden_states) @@ -455,7 +455,7 @@ class NemotronHAttnLikeDecoderLayer(nn.Module): itself unless the fused kernel takes the sum into the next input norm.""" def _init_stage_boundary(self, config: NemotronHConfig, layer_idx: int): - self.stage = make_stage_boundary( + self.boundary = make_stage_boundary( self.norm, pattern=config.hybrid_override_pattern, layer_idx=layer_idx ) @@ -465,11 +465,11 @@ class NemotronHAttnLikeDecoderLayer(nn.Module): hidden_states, forward_batch: ForwardBatch, ): - hidden_states = self.stage.prepare(hidden_states, forward_batch) + hidden_states = self.boundary.prepare(hidden_states, forward_batch) if forward_batch.forward_mode.is_idle(): return hidden_states - with self.stage.exit(forward_batch) as mixer_exit: + with self.boundary.exit(forward_batch) as mixer_exit: hidden_states = self._forward_mixer( hidden_states, forward_batch, mixer_exit.skips_reduction ) @@ -730,14 +730,14 @@ class NemotronHModel(nn.Module): residual_batch.start(forward_batch) else: assert pp_proxy_tensors is not None - hidden_states = self.layers[self.start_layer].stage.from_pp( + hidden_states = self.layers[self.start_layer].boundary.from_pp( pp_proxy_tensors, forward_batch ) aux_hidden_states = [] for i in range(self.start_layer, self.end_layer): if i in self.layers_to_capture: - hidden_states, snapshot = self.layers[i].stage.capture_output( + hidden_states, snapshot = self.layers[i].boundary.capture_output( hidden_states, forward_batch ) aux_hidden_states.append(snapshot) diff --git a/python/sglang/srt/models/nemotron_h_mtp.py b/python/sglang/srt/models/nemotron_h_mtp.py index c373bcd9c8..1392624d74 100644 --- a/python/sglang/srt/models/nemotron_h_mtp.py +++ b/python/sglang/srt/models/nemotron_h_mtp.py @@ -18,8 +18,8 @@ import torch from torch import nn from sglang.srt.configs import NemotronHConfig -from sglang.srt.layers.communicator.residual import batch as residual_batch from sglang.srt.layers.dp_attention import is_dp_attention_enabled +from sglang.srt.layers.layer_boundary.residual import batch as residual_batch from sglang.srt.layers.layernorm import RMSNorm from sglang.srt.layers.linear import ColumnParallelLinear from sglang.srt.layers.logits_processor import LogitsProcessor diff --git a/python/sglang/srt/models/nemotron_h_utils.py b/python/sglang/srt/models/nemotron_h_utils.py index 9bbcd5d37d..bce3b6ac00 100644 --- a/python/sglang/srt/models/nemotron_h_utils.py +++ b/python/sglang/srt/models/nemotron_h_utils.py @@ -1,14 +1,14 @@ """Layer-communication helpers for the Nemotron-H model.""" from sglang.srt.configs.nemotron_h import ATTENTION, MAMBA, MOE -from sglang.srt.layers.communicator import ( +from sglang.srt.layers.layer_boundary import ( HandoffRows, ProducerReduction, declare_attn, declare_ffn, make_stages, ) -from sglang.srt.layers.communicator.residual.add_norm import NormQuantRead, NormRead +from sglang.srt.layers.layer_boundary.residual.add_norm import NormQuantRead, NormRead from sglang.srt.layers.layernorm import RMSNorm from sglang.srt.runtime_context import get_parallel diff --git a/python/sglang/srt/models/qwen2.py b/python/sglang/srt/models/qwen2.py index 042621490e..588aecf793 100644 --- a/python/sglang/srt/models/qwen2.py +++ b/python/sglang/srt/models/qwen2.py @@ -26,12 +26,12 @@ from sglang.srt.distributed import ( get_pp_indices, ) from sglang.srt.layers.activation import SiluAndMul -from sglang.srt.layers.communicator.residual.access import ( +from sglang.srt.layers.dp_attention import is_dp_attention_enabled +from sglang.srt.layers.layer_boundary.residual.access import ( finish_layer_stack, from_pp, snapshot, ) -from sglang.srt.layers.dp_attention import is_dp_attention_enabled from sglang.srt.layers.layernorm import RMSNorm from sglang.srt.layers.linear import ( MergedColumnParallelLinear, diff --git a/python/sglang/srt/models/qwen2_moe.py b/python/sglang/srt/models/qwen2_moe.py index 228c0b8b27..8755df797e 100644 --- a/python/sglang/srt/models/qwen2_moe.py +++ b/python/sglang/srt/models/qwen2_moe.py @@ -44,16 +44,16 @@ from sglang.srt.layers.aux_hidden_states import ( AuxHiddenStateAccumulator, AuxHiddenStateList, ) -from sglang.srt.layers.communicator import ( - declare_attn, - declare_ffn, - make_stages, -) -from sglang.srt.layers.communicator.residual import batch as residual_batch from sglang.srt.layers.dp_attention import ( is_dp_attention_enabled, ) from sglang.srt.layers.flashinfer_comm_fusion import uses_cutedsl_ar_fusion +from sglang.srt.layers.layer_boundary import ( + declare_attn, + declare_ffn, + make_stages, +) +from sglang.srt.layers.layer_boundary.residual import batch as residual_batch from sglang.srt.layers.layernorm import RMSNorm from sglang.srt.layers.linear import ( MergedColumnParallelLinear, @@ -865,7 +865,7 @@ class Qwen2MoeSparseMoeBlock(nn.Module): if defer_finalize: if shared_output is None: raise RuntimeError("Qwen deferred finalize requires shared output") - from sglang.srt.layers.communicator.fusions.cutedsl import ( + from sglang.srt.layers.layer_boundary.fusions.cutedsl import ( MoeFinalizeHandoff, ) @@ -1053,7 +1053,7 @@ class Qwen2MoeDecoderLayer(nn.Module): self.post_attention_layernorm = RMSNorm( config.hidden_size, eps=config.rms_norm_eps ) - self.attn_stage, self.ffn_stage = make_stages( + self.attn_boundary, self.ffn_boundary = make_stages( (declare_attn(), self.input_layernorm), ( declare_ffn( @@ -1079,7 +1079,7 @@ class Qwen2MoeDecoderLayer(nn.Module): **kwargs, ) -> torch.Tensor: - hidden_states = self.attn_stage.prepare( + hidden_states = self.attn_boundary.prepare( hidden_states, forward_batch, captured_last_layer_outputs=captured_last_layer_outputs, @@ -1093,10 +1093,10 @@ class Qwen2MoeDecoderLayer(nn.Module): forward_batch=forward_batch, ) - hidden_states = self.attn_stage.finish(hidden_states, forward_batch) - hidden_states = self.ffn_stage.prepare(hidden_states, forward_batch) + hidden_states = self.attn_boundary.finish(hidden_states, forward_batch) + hidden_states = self.ffn_boundary.prepare(hidden_states, forward_batch) - with self.ffn_stage.exit(forward_batch) as ffn_exit: + with self.ffn_boundary.exit(forward_batch) as ffn_exit: hidden_states = self.mlp(hidden_states, forward_batch) hidden_states = ffn_exit.finish(hidden_states) @@ -1180,7 +1180,7 @@ class Qwen2MoeModel(nn.Module): residual_batch.start(forward_batch) else: assert pp_proxy_tensors is not None - hidden_states = self.layers[self.start_layer].attn_stage.from_pp( + hidden_states = self.layers[self.start_layer].attn_boundary.from_pp( pp_proxy_tensors, forward_batch ) diff --git a/python/sglang/srt/models/qwen3.py b/python/sglang/srt/models/qwen3.py index 515b5b2c16..2939fc608d 100644 --- a/python/sglang/srt/models/qwen3.py +++ b/python/sglang/srt/models/qwen3.py @@ -6,12 +6,12 @@ import torch from torch import nn from sglang.srt.layers.aux_hidden_states import AuxHiddenStateList -from sglang.srt.layers.communicator import ( +from sglang.srt.layers.layer_boundary import ( declare_attn, declare_ffn, make_stages, ) -from sglang.srt.layers.communicator.residual import batch as residual_batch +from sglang.srt.layers.layer_boundary.residual import batch as residual_batch from sglang.srt.layers.layernorm import RMSNorm from sglang.srt.layers.linear import QKVParallelLinear, RowParallelLinear from sglang.srt.layers.logits_processor import LogitsProcessor @@ -376,7 +376,7 @@ class Qwen3DecoderLayer(nn.Module): config.hidden_size, eps=config.rms_norm_eps, **norm_kwargs ) - self.attn_stage, self.ffn_stage = make_stages( + self.attn_boundary, self.ffn_boundary = make_stages( (declare_attn(), self.input_layernorm), ( declare_ffn(sparse=False, next_sparse=False), @@ -395,7 +395,7 @@ class Qwen3DecoderLayer(nn.Module): capture_output=None, ) -> torch.Tensor: # Self Attention - hidden_states = self.attn_stage.prepare( + hidden_states = self.attn_boundary.prepare( hidden_states, forward_batch, post_residual_addition=post_residual_addition, @@ -409,8 +409,8 @@ class Qwen3DecoderLayer(nn.Module): ) # Fully Connected - hidden_states = self.attn_stage.finish(hidden_states, forward_batch) - hidden_states = self.ffn_stage.prepare( + hidden_states = self.attn_boundary.finish(hidden_states, forward_batch) + hidden_states = self.ffn_boundary.prepare( hidden_states, forward_batch, cache=( @@ -424,7 +424,7 @@ class Qwen3DecoderLayer(nn.Module): else None ), ) - with self.ffn_stage.exit(forward_batch) as ffn_exit: + with self.ffn_boundary.exit(forward_batch) as ffn_exit: hidden_states = self.mlp(hidden_states, forward_batch=forward_batch) if _is_npu and get_cmo_stream(): wait_cmo_stream() @@ -462,7 +462,7 @@ class Qwen3Model(Qwen2Model): ) residual_batch.start(forward_batch) else: - hidden_states = self.layers[self.start_layer].attn_stage.from_pp( + hidden_states = self.layers[self.start_layer].attn_boundary.from_pp( pp_proxy_tensors, forward_batch ) aux_hidden_states = AuxHiddenStateList() diff --git a/python/sglang/srt/models/qwen3_5.py b/python/sglang/srt/models/qwen3_5.py index 966a4aeac4..fac16f30a3 100644 --- a/python/sglang/srt/models/qwen3_5.py +++ b/python/sglang/srt/models/qwen3_5.py @@ -46,21 +46,21 @@ from sglang.srt.layers.aux_hidden_states import ( AuxHiddenStateAccumulator, AuxHiddenStateList, ) -from sglang.srt.layers.communicator import ( +from sglang.srt.layers.dp_attention import ( + is_dp_attention_enabled, +) +from sglang.srt.layers.flashinfer_comm_fusion import uses_cutedsl_ar_fusion +from sglang.srt.layers.layer_boundary import ( declare_attn, declare_ffn, make_stages, ) # Layers - Attention -from sglang.srt.layers.communicator.residual import batch as residual_batch +from sglang.srt.layers.layer_boundary.residual import batch as residual_batch # Layers - Others -from sglang.srt.layers.communicator.residual.add_norm import Fp8Input, NormQuantRead -from sglang.srt.layers.dp_attention import ( - is_dp_attention_enabled, -) -from sglang.srt.layers.flashinfer_comm_fusion import uses_cutedsl_ar_fusion +from sglang.srt.layers.layer_boundary.residual.add_norm import Fp8Input, NormQuantRead from sglang.srt.layers.layernorm import GemmaRMSNorm # Layers - Linear @@ -224,7 +224,7 @@ def _use_mnnvl_cutedsl_fusion(config: Qwen3_5TextConfig, is_nextn: bool) -> bool def _layer_fusions(config: Qwen3_5TextConfig, is_nextn: bool): """The CuTe DSL kernels a layer tries before its own, when they are on.""" if _use_mnnvl_cutedsl_fusion(config, is_nextn): - from sglang.srt.layers.communicator.fusions.cutedsl import CuteDSLFusion + from sglang.srt.layers.layer_boundary.fusions.cutedsl import CuteDSLFusion return CuteDSLFusion() return None @@ -1094,7 +1094,7 @@ class Qwen3_5LinearDecoderLayer(nn.Module): # it. Otherwise, stay on the plain AR+RMSNorm path. accepts_fp8_input = _linear_accepts_fp8_tuple(self.linear_attn.in_proj_qkvz) boundary_fusions = _layer_fusions(config, is_nextn) - self.attn_stage, self.ffn_stage = make_stages( + self.attn_boundary, self.ffn_boundary = make_stages( ( declare_attn( read=NormQuantRead( @@ -1129,7 +1129,7 @@ class Qwen3_5LinearDecoderLayer(nn.Module): ): forward_batch = kwargs.get("forward_batch", None) - hidden_states = self.attn_stage.prepare( + hidden_states = self.attn_boundary.prepare( hidden_states, forward_batch, captured_last_layer_outputs=kwargs.get("captured_last_layer_outputs", None), @@ -1144,10 +1144,10 @@ class Qwen3_5LinearDecoderLayer(nn.Module): ) # Fully Connected - hidden_states = self.attn_stage.finish(hidden_states, forward_batch) - hidden_states = self.ffn_stage.prepare(hidden_states, forward_batch) + hidden_states = self.attn_boundary.finish(hidden_states, forward_batch) + hidden_states = self.ffn_boundary.prepare(hidden_states, forward_batch) - with self.ffn_stage.exit(forward_batch) as ffn_exit: + with self.ffn_boundary.exit(forward_batch) as ffn_exit: if isinstance(self.mlp, Qwen2MoeSparseMoeBlock): hidden_states = self.mlp( hidden_states, @@ -1300,7 +1300,7 @@ class Qwen3_5AttentionDecoderLayer(nn.Module): # when qkv_proj can consume the returned quantized tuple. accepts_fp8_input = _linear_accepts_fp8_tuple(self.qkv_proj) boundary_fusions = _layer_fusions(config, is_nextn) - self.attn_stage, self.ffn_stage = make_stages( + self.attn_boundary, self.ffn_boundary = make_stages( ( declare_attn( read=NormQuantRead( @@ -1533,7 +1533,7 @@ class Qwen3_5AttentionDecoderLayer(nn.Module): captured_last_layer_outputs: Optional[AuxHiddenStateAccumulator] = None, **kwargs, ): - hidden_states = self.attn_stage.prepare( + hidden_states = self.attn_boundary.prepare( hidden_states, forward_batch, captured_last_layer_outputs=captured_last_layer_outputs, @@ -1549,9 +1549,9 @@ class Qwen3_5AttentionDecoderLayer(nn.Module): ) # Fully Connected - hidden_states = self.attn_stage.finish(hidden_states, forward_batch) - hidden_states = self.ffn_stage.prepare(hidden_states, forward_batch) - with self.ffn_stage.exit(forward_batch) as ffn_exit: + hidden_states = self.attn_boundary.finish(hidden_states, forward_batch) + hidden_states = self.ffn_boundary.prepare(hidden_states, forward_batch) + with self.ffn_boundary.exit(forward_batch) as ffn_exit: if isinstance(self.mlp, Qwen2MoeSparseMoeBlock): hidden_states = self.mlp( hidden_states, @@ -1716,7 +1716,7 @@ class Qwen3_5ForCausalLM(nn.Module): "layers: " f"{unsupported_layers}" ) - from sglang.srt.layers.communicator.fusions.cutedsl import ( + from sglang.srt.layers.layer_boundary.fusions.cutedsl import ( install_cutedsl_fusion, ) @@ -1807,7 +1807,7 @@ class Qwen3_5ForCausalLM(nn.Module): residual_batch.start(forward_batch) else: assert pp_proxy_tensors is not None - hidden_states = self.layers[self.start_layer].attn_stage.from_pp( + hidden_states = self.layers[self.start_layer].attn_boundary.from_pp( pp_proxy_tensors, forward_batch ) diff --git a/python/sglang/srt/models/qwen3_moe.py b/python/sglang/srt/models/qwen3_moe.py index 758dfd82d5..d99f61687b 100644 --- a/python/sglang/srt/models/qwen3_moe.py +++ b/python/sglang/srt/models/qwen3_moe.py @@ -30,12 +30,12 @@ from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_r from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation from sglang.srt.eplb.expert_location_dispatch import ExpertLocationDispatchInfo from sglang.srt.layers.aux_hidden_states import AuxHiddenStateAccumulator -from sglang.srt.layers.communicator import ( +from sglang.srt.layers.layer_boundary import ( declare_attn, declare_ffn, make_stages, ) -from sglang.srt.layers.communicator.residual import batch as residual_batch +from sglang.srt.layers.layer_boundary.residual import batch as residual_batch from sglang.srt.layers.layernorm import RMSNorm from sglang.srt.layers.linear import ( QKVParallelLinear, @@ -866,7 +866,7 @@ class Qwen3MoeDecoderLayer(nn.Module): config.hidden_size, eps=config.rms_norm_eps ) - self.attn_stage, self.ffn_stage = make_stages( + self.attn_boundary, self.ffn_boundary = make_stages( (declare_attn(), self.input_layernorm), ( declare_ffn( @@ -893,7 +893,7 @@ class Qwen3MoeDecoderLayer(nn.Module): **kwargs, ) -> torch.Tensor: - hidden_states = self.attn_stage.prepare( + hidden_states = self.attn_boundary.prepare( hidden_states, forward_batch, captured_last_layer_outputs=captured_last_layer_outputs, @@ -908,10 +908,10 @@ class Qwen3MoeDecoderLayer(nn.Module): forward_batch=forward_batch, ) - hidden_states = self.attn_stage.finish(hidden_states, forward_batch) - hidden_states = self.ffn_stage.prepare(hidden_states, forward_batch) + hidden_states = self.attn_boundary.finish(hidden_states, forward_batch) + hidden_states = self.ffn_boundary.prepare(hidden_states, forward_batch) - with self.ffn_stage.exit(forward_batch) as ffn_exit: + with self.ffn_boundary.exit(forward_batch) as ffn_exit: hidden_states = self.mlp(hidden_states, forward_batch) hidden_states = ffn_exit.finish(hidden_states) @@ -925,7 +925,7 @@ class Qwen3MoeDecoderLayer(nn.Module): forward_batch: ForwardBatch, tbo_subbatch_index: Optional[int] = None, ): - state.hidden_states_after_comm_pre_attn = self.attn_stage.prepare( + state.hidden_states_after_comm_pre_attn = self.attn_boundary.prepare( hidden_states, forward_batch ) state.update( @@ -937,15 +937,15 @@ class Qwen3MoeDecoderLayer(nn.Module): ) def op_comm_prepare_mlp(self, state): - hidden_states = self.attn_stage.finish( + hidden_states = self.attn_boundary.finish( state.pop("hidden_states_after_attn"), state.forward_batch ) - state.hidden_states_mlp_input = self.ffn_stage.prepare( + state.hidden_states_mlp_input = self.ffn_boundary.prepare( hidden_states, state.forward_batch ) def op_comm_postprocess_layer(self, state): - hidden_states = self.ffn_stage.postprocess( + hidden_states = self.ffn_boundary.postprocess( state.pop("hidden_states_mlp_output"), state.forward_batch ) diff --git a/python/sglang/srt/models/qwen3_next.py b/python/sglang/srt/models/qwen3_next.py index c0294b683f..ccfd7c5656 100644 --- a/python/sglang/srt/models/qwen3_next.py +++ b/python/sglang/srt/models/qwen3_next.py @@ -19,15 +19,15 @@ from sglang.srt.layers.aux_hidden_states import ( AuxHiddenStateAccumulator, AuxHiddenStateList, ) -from sglang.srt.layers.communicator import ( +from sglang.srt.layers.dp_attention import ( + is_dp_attention_enabled, +) +from sglang.srt.layers.layer_boundary import ( declare_attn, declare_ffn, make_stages, ) -from sglang.srt.layers.communicator.residual import batch as residual_batch -from sglang.srt.layers.dp_attention import ( - is_dp_attention_enabled, -) +from sglang.srt.layers.layer_boundary.residual import batch as residual_batch from sglang.srt.layers.layernorm import GemmaRMSNorm from sglang.srt.layers.linear import ( ColumnParallelLinear, @@ -477,9 +477,9 @@ def _apply_qwen3_next_mlp( hidden_states: torch.Tensor, forward_batch: ForwardBatch, ) -> torch.Tensor: - hidden_states = layer.attn_stage.finish(hidden_states, forward_batch) - hidden_states = layer.ffn_stage.prepare(hidden_states, forward_batch) - with layer.ffn_stage.exit(forward_batch) as ffn_exit: + hidden_states = layer.attn_boundary.finish(hidden_states, forward_batch) + hidden_states = layer.ffn_boundary.prepare(hidden_states, forward_batch) + with layer.ffn_boundary.exit(forward_batch) as ffn_exit: if isinstance(layer.mlp, Qwen2MoeSparseMoeBlock): hidden_states = layer.mlp( hidden_states, @@ -537,7 +537,7 @@ class Qwen3HybridLinearDecoderLayer(nn.Module): self.post_attention_layernorm = GemmaRMSNorm( config.hidden_size, eps=config.rms_norm_eps ) - self.attn_stage, self.ffn_stage = make_stages( + self.attn_boundary, self.ffn_boundary = make_stages( (declare_attn(), self.input_layernorm), ( declare_ffn( @@ -562,7 +562,7 @@ class Qwen3HybridLinearDecoderLayer(nn.Module): ): forward_batch = kwargs.get("forward_batch", None) - hidden_states = self.attn_stage.prepare( + hidden_states = self.attn_boundary.prepare( hidden_states, forward_batch, captured_last_layer_outputs=captured_last_layer_outputs, @@ -705,7 +705,7 @@ class Qwen3HybridAttentionDecoderLayer(nn.Module): self.q_norm = GemmaRMSNorm(self.head_dim, eps=config.rms_norm_eps) self.k_norm = GemmaRMSNorm(self.head_dim, eps=config.rms_norm_eps) - self.attn_stage, self.ffn_stage = make_stages( + self.attn_boundary, self.ffn_boundary = make_stages( (declare_attn(), self.input_layernorm), ( declare_ffn( @@ -832,7 +832,7 @@ class Qwen3HybridAttentionDecoderLayer(nn.Module): captured_last_layer_outputs: Optional[AuxHiddenStateAccumulator] = None, **kwargs: Any, ): - hidden_states = self.attn_stage.prepare( + hidden_states = self.attn_boundary.prepare( hidden_states, forward_batch, captured_last_layer_outputs=captured_last_layer_outputs, diff --git a/python/sglang/srt/models/qwen3_vl.py b/python/sglang/srt/models/qwen3_vl.py index 4c4b543fd3..6f9faffd56 100644 --- a/python/sglang/srt/models/qwen3_vl.py +++ b/python/sglang/srt/models/qwen3_vl.py @@ -38,11 +38,11 @@ from sglang.srt.layers.attention.vision import ( prepare_vision_attention_metadata, ) from sglang.srt.layers.aux_hidden_states import AuxHiddenStateList -from sglang.srt.layers.communicator.residual import batch as residual_batch from sglang.srt.layers.conv import Conv3dLayer from sglang.srt.layers.dp_attention import ( is_dp_attention_enabled, ) +from sglang.srt.layers.layer_boundary.residual import batch as residual_batch from sglang.srt.layers.linear import ColumnParallelLinear, RowParallelLinear from sglang.srt.layers.logits_processor import LogitsProcessor from sglang.srt.layers.pooler import Pooler, PoolingType @@ -1209,7 +1209,7 @@ class Qwen3LLMModel(Qwen3Model): residual_batch.start(forward_batch) else: assert pp_proxy_tensors is not None - hidden_states = self.layers[self.start_layer].attn_stage.from_pp( + hidden_states = self.layers[self.start_layer].attn_boundary.from_pp( pp_proxy_tensors, forward_batch ) diff --git a/python/sglang/srt/models/qwen3_vl_moe.py b/python/sglang/srt/models/qwen3_vl_moe.py index c8a44ae63f..7f709e7d10 100644 --- a/python/sglang/srt/models/qwen3_vl_moe.py +++ b/python/sglang/srt/models/qwen3_vl_moe.py @@ -25,7 +25,7 @@ import torch.nn as nn from sglang.srt.configs.qwen3_vl import Qwen3VLMoeConfig, Qwen3VLMoeTextConfig from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation from sglang.srt.layers.aux_hidden_states import AuxHiddenStateList -from sglang.srt.layers.communicator.residual import batch as residual_batch +from sglang.srt.layers.layer_boundary.residual import batch as residual_batch from sglang.srt.layers.moe.fused_moe_triton.layer import FusedMoE from sglang.srt.layers.quantization.base_config import QuantizationConfig from sglang.srt.layers.utils import get_layer_id @@ -99,7 +99,7 @@ class Qwen3MoeLLMModel(Qwen3MoeModel): residual_batch.start(forward_batch) else: assert pp_proxy_tensors is not None - hidden_states = self.layers[self.start_layer].attn_stage.from_pp( + hidden_states = self.layers[self.start_layer].attn_boundary.from_pp( pp_proxy_tensors, forward_batch ) diff --git a/python/sglang/srt/models/qwen4_exp.py b/python/sglang/srt/models/qwen4_exp.py index 69397d5963..7bcc07ca5a 100644 --- a/python/sglang/srt/models/qwen4_exp.py +++ b/python/sglang/srt/models/qwen4_exp.py @@ -22,7 +22,6 @@ from sglang.srt.environ import envs from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation from sglang.srt.layers.aux_hidden_states import AuxHiddenStateList -from sglang.srt.layers.communicator import get_attn_tp_context from sglang.srt.layers.dp_attention import ( attn_tp_all_gather, attn_tp_all_reduce, @@ -38,6 +37,7 @@ from sglang.srt.layers.hyperconnection import ( GatedResidual, HyperConnectionConfig, ) +from sglang.srt.layers.layer_boundary import get_attn_tp_context from sglang.srt.layers.linear import ReplicatedLinear from sglang.srt.layers.logits_processor import LogitsProcessorOutput from sglang.srt.layers.moe import get_moe_a2a_backend, should_use_dp_reduce_scatterv @@ -1320,8 +1320,8 @@ class Qwen4ExpLayerExtensionMixin: for attr_name in ( "input_layernorm", "post_attention_layernorm", - "attn_stage", - "ffn_stage", + "attn_boundary", + "ffn_boundary", ): if hasattr(self, attr_name): delattr(self, attr_name) diff --git a/python/sglang/srt/models/sarvam_moe.py b/python/sglang/srt/models/sarvam_moe.py index c9818af7ca..a58b19ecdb 100644 --- a/python/sglang/srt/models/sarvam_moe.py +++ b/python/sglang/srt/models/sarvam_moe.py @@ -19,16 +19,16 @@ from sglang.srt.distributed import ( from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation from sglang.srt.layers.activation import SiluAndMul -from sglang.srt.layers.communicator import ( +from sglang.srt.layers.dp_attention import ( + is_dp_attention_enabled, +) +from sglang.srt.layers.layer_boundary import ( declare_attn, declare_ffn, enable_moe_dense_fully_dp, make_stages, ) -from sglang.srt.layers.communicator.residual import batch as residual_batch -from sglang.srt.layers.dp_attention import ( - is_dp_attention_enabled, -) +from sglang.srt.layers.layer_boundary.residual import batch as residual_batch from sglang.srt.layers.layernorm import RMSNorm from sglang.srt.layers.linear import ( ColumnParallelLinear, @@ -1050,7 +1050,7 @@ class SarvamMoEMLADecoderLayer(nn.Module): self.attn_tp_size = get_parallel().attn_tp_size - self.attn_stage, self.ffn_stage = make_stages( + self.attn_boundary, self.ffn_boundary = make_stages( ( declare_attn(), self.input_layernorm, @@ -1077,16 +1077,16 @@ class SarvamMoEMLADecoderLayer(nn.Module): hidden_states: torch.Tensor, forward_batch: ForwardBatch, ) -> torch.Tensor: - hidden_states = self.attn_stage.prepare(hidden_states, forward_batch) + hidden_states = self.attn_boundary.prepare(hidden_states, forward_batch) if hidden_states.shape[0] != 0: hidden_states = self.self_attn( positions=positions, hidden_states=hidden_states, forward_batch=forward_batch, ) - hidden_states = self.attn_stage.finish(hidden_states, forward_batch) - hidden_states = self.ffn_stage.prepare(hidden_states, forward_batch) - with self.ffn_stage.exit(forward_batch) as ffn_exit: + hidden_states = self.attn_boundary.finish(hidden_states, forward_batch) + hidden_states = self.ffn_boundary.prepare(hidden_states, forward_batch) + with self.ffn_boundary.exit(forward_batch) as ffn_exit: hidden_states = self.mlp(hidden_states, forward_batch) if ( not self.is_layer_sparse @@ -1159,7 +1159,7 @@ class SarvamMLAModel(nn.Module): residual_batch.start(forward_batch) else: assert pp_proxy_tensors is not None - hidden_states = self.layers[self.start_layer].attn_stage.from_pp( + hidden_states = self.layers[self.start_layer].attn_boundary.from_pp( pp_proxy_tensors, forward_batch ) diff --git a/python/sglang/srt/models/sdar.py b/python/sglang/srt/models/sdar.py index 1129caac12..6e8d510e0e 100644 --- a/python/sglang/srt/models/sdar.py +++ b/python/sglang/srt/models/sdar.py @@ -11,15 +11,15 @@ from torch import nn from transformers import PretrainedConfig from sglang.srt.layers.activation import SiluAndMul -from sglang.srt.layers.communicator import ( +from sglang.srt.layers.dp_attention import ( + is_dp_attention_enabled, +) +from sglang.srt.layers.layer_boundary import ( declare_attn, declare_ffn, make_stages, ) -from sglang.srt.layers.communicator.residual import batch as residual_batch -from sglang.srt.layers.dp_attention import ( - is_dp_attention_enabled, -) +from sglang.srt.layers.layer_boundary.residual import batch as residual_batch from sglang.srt.layers.layernorm import RMSNorm from sglang.srt.layers.linear import ( MergedColumnParallelLinear, @@ -295,7 +295,7 @@ class SDARBlock(nn.Module): prefix=add_prefix("mlp", prefix), ) - self.attn_stage, self.ffn_stage = make_stages( + self.attn_boundary, self.ffn_boundary = make_stages( (declare_attn(), self.input_layernorm), ( declare_ffn( @@ -314,7 +314,7 @@ class SDARBlock(nn.Module): hidden_states: torch.Tensor, forward_batch: ForwardBatch, ) -> torch.Tensor: - hidden_states = self.attn_stage.prepare(hidden_states, forward_batch) + hidden_states = self.attn_boundary.prepare(hidden_states, forward_batch) if hidden_states.shape[0] != 0: hidden_states = self.self_attn( @@ -323,10 +323,10 @@ class SDARBlock(nn.Module): forward_batch=forward_batch, ) - hidden_states = self.attn_stage.finish(hidden_states, forward_batch) - hidden_states = self.ffn_stage.prepare(hidden_states, forward_batch) + hidden_states = self.attn_boundary.finish(hidden_states, forward_batch) + hidden_states = self.ffn_boundary.prepare(hidden_states, forward_batch) - with self.ffn_stage.exit(forward_batch) as ffn_exit: + with self.ffn_boundary.exit(forward_batch) as ffn_exit: hidden_states = self.mlp(hidden_states) hidden_states = ffn_exit.finish(hidden_states) @@ -401,7 +401,7 @@ class SDARModel(nn.Module): residual_batch.start(forward_batch) else: assert pp_proxy_tensors is not None - hidden_states = self.layers[self.start_layer].attn_stage.from_pp( + hidden_states = self.layers[self.start_layer].attn_boundary.from_pp( pp_proxy_tensors, forward_batch, allow_missing_residual=True ) diff --git a/python/sglang/srt/models/sdar_moe.py b/python/sglang/srt/models/sdar_moe.py index 102453fc1e..f4cbd6a29b 100644 --- a/python/sglang/srt/models/sdar_moe.py +++ b/python/sglang/srt/models/sdar_moe.py @@ -13,15 +13,15 @@ from transformers import PretrainedConfig from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation from sglang.srt.eplb.expert_location_dispatch import ExpertLocationDispatchInfo -from sglang.srt.layers.communicator import ( +from sglang.srt.layers.dp_attention import ( + is_dp_attention_enabled, +) +from sglang.srt.layers.layer_boundary import ( declare_attn, declare_ffn, make_stages, ) -from sglang.srt.layers.communicator.residual import batch as residual_batch -from sglang.srt.layers.dp_attention import ( - is_dp_attention_enabled, -) +from sglang.srt.layers.layer_boundary.residual import batch as residual_batch from sglang.srt.layers.layernorm import RMSNorm from sglang.srt.layers.linear import ( QKVParallelLinear, @@ -357,7 +357,7 @@ class SDARMoeBlock(nn.Module): prefix=add_prefix("mlp", prefix), ) - self.attn_stage, self.ffn_stage = make_stages( + self.attn_boundary, self.ffn_boundary = make_stages( (declare_attn(), self.input_layernorm), ( declare_ffn(sparse=True, next_sparse=True), @@ -376,7 +376,7 @@ class SDARMoeBlock(nn.Module): forward_batch: ForwardBatch, ) -> torch.Tensor: - hidden_states = self.attn_stage.prepare(hidden_states, forward_batch) + hidden_states = self.attn_boundary.prepare(hidden_states, forward_batch) if hidden_states.shape[0] != 0: hidden_states = self.self_attn( @@ -385,10 +385,10 @@ class SDARMoeBlock(nn.Module): forward_batch=forward_batch, ) - hidden_states = self.attn_stage.finish(hidden_states, forward_batch) - hidden_states = self.ffn_stage.prepare(hidden_states, forward_batch) + hidden_states = self.attn_boundary.finish(hidden_states, forward_batch) + hidden_states = self.ffn_boundary.prepare(hidden_states, forward_batch) - with self.ffn_stage.exit(forward_batch) as ffn_exit: + with self.ffn_boundary.exit(forward_batch) as ffn_exit: hidden_states = self.mlp( hidden_states, forward_batch=forward_batch, @@ -467,7 +467,7 @@ class SDARMoeModel(nn.Module): residual_batch.start(forward_batch) else: assert pp_proxy_tensors is not None - hidden_states = self.layers[self.start_layer].attn_stage.from_pp( + hidden_states = self.layers[self.start_layer].attn_boundary.from_pp( pp_proxy_tensors, forward_batch ) diff --git a/python/sglang/srt/models/step3_vl.py b/python/sglang/srt/models/step3_vl.py index 84e1202b60..98ccda9cd5 100644 --- a/python/sglang/srt/models/step3_vl.py +++ b/python/sglang/srt/models/step3_vl.py @@ -19,16 +19,16 @@ from sglang.srt.configs.step3_vl import ( from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation from sglang.srt.layers.activation import SiluAndMul from sglang.srt.layers.attention.vision import VisionAttention -from sglang.srt.layers.communicator import ( - declare_attn, - declare_ffn, - make_stages, -) -from sglang.srt.layers.communicator.residual import batch as residual_batch from sglang.srt.layers.conv import Conv2dLayer from sglang.srt.layers.dp_attention import ( is_dp_attention_enabled, ) +from sglang.srt.layers.layer_boundary import ( + declare_attn, + declare_ffn, + make_stages, +) +from sglang.srt.layers.layer_boundary.residual import batch as residual_batch from sglang.srt.layers.layernorm import RMSNorm from sglang.srt.layers.linear import ( ColumnParallelLinear, @@ -374,7 +374,7 @@ class Step3TextDecoderLayer(nn.Module): prefix=add_prefix("mlp", prefix), ) - self.attn_stage, self.ffn_stage = make_stages( + self.attn_boundary, self.ffn_boundary = make_stages( (declare_attn(), self.input_layernorm), ( declare_ffn( @@ -407,7 +407,7 @@ class Step3TextDecoderLayer(nn.Module): forward_batch: ForwardBatch, ) -> torch.Tensor: - hidden_states = self.attn_stage.prepare(hidden_states, forward_batch) + hidden_states = self.attn_boundary.prepare(hidden_states, forward_batch) if hidden_states.shape[0] != 0: hidden_states = self.self_attn( @@ -416,9 +416,9 @@ class Step3TextDecoderLayer(nn.Module): forward_batch=forward_batch, ) - hidden_states = self.attn_stage.finish(hidden_states, forward_batch) - hidden_states = self.ffn_stage.prepare(hidden_states, forward_batch) - with self.ffn_stage.exit(forward_batch) as ffn_exit: + hidden_states = self.attn_boundary.finish(hidden_states, forward_batch) + hidden_states = self.ffn_boundary.prepare(hidden_states, forward_batch) + with self.ffn_boundary.exit(forward_batch) as ffn_exit: if self.use_moe: hidden_states = self.moe_mlp_forward(hidden_states) else: diff --git a/python/sglang/srt/models/step3p5.py b/python/sglang/srt/models/step3p5.py index 5a58b3a02b..c640e094dd 100644 --- a/python/sglang/srt/models/step3p5.py +++ b/python/sglang/srt/models/step3p5.py @@ -11,15 +11,15 @@ from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_r from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation from sglang.srt.eplb.expert_location_dispatch import ExpertLocationDispatchInfo from sglang.srt.layers.activation import SiluAndMul -from sglang.srt.layers.communicator import ( +from sglang.srt.layers.dp_attention import ( + is_dp_attention_enabled, +) +from sglang.srt.layers.layer_boundary import ( declare_attn, declare_ffn, make_stages, ) -from sglang.srt.layers.communicator.residual import batch as residual_batch -from sglang.srt.layers.dp_attention import ( - is_dp_attention_enabled, -) +from sglang.srt.layers.layer_boundary.residual import batch as residual_batch from sglang.srt.layers.layernorm import GemmaRMSNorm from sglang.srt.layers.linear import ( ColumnParallelLinear, @@ -566,7 +566,7 @@ class Step3p5DecoderLayer(nn.Module): # An MTP draft is a one-layer model; layer_id still indexes its config. - self.attn_stage, self.ffn_stage = make_stages( + self.attn_boundary, self.ffn_boundary = make_stages( (declare_attn(), self.input_layernorm), ( declare_ffn( @@ -594,7 +594,7 @@ class Step3p5DecoderLayer(nn.Module): post_residual_addition: Optional[torch.Tensor] = None, ) -> torch.Tensor: # Self Attention - hidden_states = self.attn_stage.prepare( + hidden_states = self.attn_boundary.prepare( hidden_states, forward_batch, post_residual_addition=post_residual_addition ) if hidden_states.shape[0] != 0: @@ -604,11 +604,11 @@ class Step3p5DecoderLayer(nn.Module): forward_batch=forward_batch, ) # Fully Connected - hidden_states = self.attn_stage.finish(hidden_states, forward_batch) - hidden_states = self.ffn_stage.prepare(hidden_states, forward_batch) + hidden_states = self.attn_boundary.finish(hidden_states, forward_batch) + hidden_states = self.ffn_boundary.prepare(hidden_states, forward_batch) if self.use_moe: - with self.ffn_stage.exit(forward_batch) as ffn_exit: + with self.ffn_boundary.exit(forward_batch) as ffn_exit: # Both share_expert and MoE return unreduced (TP-partial) outputs. # Combine them first, then do a single all-reduce — saving one # full-TP all-reduce per layer. @@ -621,7 +621,7 @@ class Step3p5DecoderLayer(nn.Module): hidden_states = tensor_model_parallel_all_reduce(hidden_states) return ffn_exit.finish(hidden_states) - with self.ffn_stage.exit(forward_batch) as ffn_exit: + with self.ffn_boundary.exit(forward_batch) as ffn_exit: hidden_states = self.mlp(hidden_states) return ffn_exit.finish(hidden_states) @@ -700,7 +700,7 @@ class Step3p5Model(nn.Module): residual_batch.start(forward_batch) else: assert pp_proxy_tensors is not None - hidden_states = self.layers[self.start_layer].attn_stage.from_pp( + hidden_states = self.layers[self.start_layer].attn_boundary.from_pp( pp_proxy_tensors, forward_batch ) diff --git a/python/sglang/srt/models/step3p5_mtp.py b/python/sglang/srt/models/step3p5_mtp.py index b5b6327072..fd4696eb6b 100644 --- a/python/sglang/srt/models/step3p5_mtp.py +++ b/python/sglang/srt/models/step3p5_mtp.py @@ -6,7 +6,7 @@ import torch import torch.nn as nn from transformers import PretrainedConfig -from sglang.srt.layers.communicator.residual import batch as residual_batch +from sglang.srt.layers.layer_boundary.residual import batch as residual_batch from sglang.srt.layers.layernorm import GemmaRMSNorm from sglang.srt.layers.logits_processor import LogitsProcessor from sglang.srt.layers.quantization.base_config import QuantizationConfig diff --git a/python/sglang/srt/models/xllm.py b/python/sglang/srt/models/xllm.py index c1034f2397..59a5e77a09 100644 --- a/python/sglang/srt/models/xllm.py +++ b/python/sglang/srt/models/xllm.py @@ -39,14 +39,14 @@ from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_r from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation from sglang.srt.eplb.expert_location_dispatch import ExpertLocationDispatchInfo from sglang.srt.layers.activation import SiluAndMul -from sglang.srt.layers.communicator import ( +from sglang.srt.layers.dp_attention import is_dp_attention_enabled +from sglang.srt.layers.layer_boundary import ( declare_attn, declare_ffn, enable_moe_dense_fully_dp, make_stages, ) -from sglang.srt.layers.communicator.residual import batch as residual_batch -from sglang.srt.layers.dp_attention import is_dp_attention_enabled +from sglang.srt.layers.layer_boundary.residual import batch as residual_batch from sglang.srt.layers.layernorm import RMSNorm from sglang.srt.layers.linear import ( ColumnParallelLinear, @@ -1528,7 +1528,7 @@ class XllmDecoderLayer(nn.Module): self.input_layernorm = _make_norm(config) self.post_attention_layernorm = _make_norm(config) - self.attn_stage, self.ffn_stage = make_stages( + self.attn_boundary, self.ffn_boundary = make_stages( (declare_attn(), self.input_layernorm), ( declare_ffn( @@ -1551,7 +1551,7 @@ class XllmDecoderLayer(nn.Module): hidden_states: torch.Tensor, forward_batch: ForwardBatch, ) -> torch.Tensor: - hidden_states = self.attn_stage.prepare(hidden_states, forward_batch) + hidden_states = self.attn_boundary.prepare(hidden_states, forward_batch) if hidden_states.shape[0] != 0: hidden_states = self.self_attn( @@ -1560,10 +1560,10 @@ class XllmDecoderLayer(nn.Module): forward_batch=forward_batch, ) - hidden_states = self.attn_stage.finish(hidden_states, forward_batch) - hidden_states = self.ffn_stage.prepare(hidden_states, forward_batch) + hidden_states = self.attn_boundary.finish(hidden_states, forward_batch) + hidden_states = self.ffn_boundary.prepare(hidden_states, forward_batch) - with self.ffn_stage.exit(forward_batch) as ffn_exit: + with self.ffn_boundary.exit(forward_batch) as ffn_exit: if isinstance(self.mlp, XllmMLP): hidden_states = self.mlp(hidden_states) else: @@ -1629,7 +1629,7 @@ class XllmModel(nn.Module): residual_batch.start(forward_batch) else: assert pp_proxy_tensors is not None - hidden_states = self.layers[self.start_layer].attn_stage.from_pp( + hidden_states = self.layers[self.start_layer].attn_boundary.from_pp( pp_proxy_tensors, forward_batch ) diff --git a/python/sglang/test/boundary_fixtures.py b/python/sglang/test/boundary_fixtures.py index 14ae2e9d20..bce61d7aba 100644 --- a/python/sglang/test/boundary_fixtures.py +++ b/python/sglang/test/boundary_fixtures.py @@ -2,15 +2,15 @@ from types import SimpleNamespace -from sglang.srt.layers.communicator.construction import StagePlan -from sglang.srt.layers.communicator.factories import ( +from sglang.srt.layers.layer_boundary.construction import StagePlan +from sglang.srt.layers.layer_boundary.factories import ( declare_attn, declare_ffn, make_stages, ) -from sglang.srt.layers.communicator.ops import identity_output -from sglang.srt.layers.communicator.residual.add_norm import PLAIN_RESIDUAL -from sglang.srt.layers.communicator.stage import StageCommunicator +from sglang.srt.layers.layer_boundary.ops import identity_output +from sglang.srt.layers.layer_boundary.residual.add_norm import PLAIN_RESIDUAL +from sglang.srt.layers.layer_boundary.stage import StageBoundary def make_test_stages( @@ -73,7 +73,7 @@ def stub_stage(plan, kind): """Keep the same concrete boundary when a test patches its methods.""" stages = plan.__dict__.setdefault("_stub_stages", {}) if kind not in stages: - stages[kind] = StageCommunicator( + stages[kind] = StageBoundary( plan, declaration=(declare_attn() if kind.name == "ATTENTION" else declare_ffn()), ) @@ -82,7 +82,7 @@ def stub_stage(plan, kind): def sp_region_steps(): """Local SP rows with no owed sum, for tests of activation and exits.""" - from sglang.srt.layers.communicator import ( + from sglang.srt.layers.layer_boundary import ( NORM_QUANT_READ, EdgeDecl, Layout, @@ -93,7 +93,7 @@ def sp_region_steps(): TokenAxis, make_boundary, ) - from sglang.srt.layers.communicator.prepare import _hand_qkv_hook_its_input + from sglang.srt.layers.layer_boundary.prepare import _hand_qkv_hook_its_input rows = Layout(frozenset({TokenAxis.ATTN_TP_SCATTER})) output = StageOutput(rows) @@ -120,8 +120,8 @@ def prepare_attention(stage, hidden, residual, forward_batch, *args, **call): def prepare_raw(stage, method, hidden, residual, forward_batch, *args, **call): - from sglang.srt.layers.communicator.residual.add_norm import ADD - from sglang.srt.layers.communicator.residual.stream import ResidualStream + from sglang.srt.layers.layer_boundary.residual.add_norm import ADD + from sglang.srt.layers.layer_boundary.residual.stream import ResidualStream if not isinstance(residual, ResidualStream): stream = ResidualStream(residual) @@ -137,7 +137,7 @@ def prepare_raw(stage, method, hidden, residual, forward_batch, *args, **call): def finish_exit(scope, hidden, residual): - from sglang.srt.layers.communicator.residual.stream import ResidualStream + from sglang.srt.layers.layer_boundary.residual.stream import ResidualStream if isinstance(residual, ResidualStream): scope._stream = residual @@ -150,7 +150,7 @@ def finish_exit(scope, hidden, residual): def postprocess_output(boundary, hidden, residual, forward_batch): - from sglang.srt.layers.communicator.residual.stream import ResidualStream + from sglang.srt.layers.layer_boundary.residual.stream import ResidualStream stream = ( residual if isinstance(residual, ResidualStream) else ResidualStream(residual) diff --git a/python/sglang/test/communicator_patch.py b/python/sglang/test/communicator_patch.py index 9e0b3c3364..fa5a36d3c0 100644 --- a/python/sglang/test/communicator_patch.py +++ b/python/sglang/test/communicator_patch.py @@ -1,6 +1,6 @@ """Patching a name the communicator package's modules look up. -Each module of ``sglang.srt.layers.communicator`` binds the names it imports +Each module of ``sglang.srt.layers.layer_boundary`` binds the names it imports in its own namespace, so a test double has to go into every module that reads the name, not only into the package. """ @@ -8,8 +8,8 @@ the name, not only into the package. import contextlib from unittest import mock -from sglang.srt.layers import communicator as communicator -from sglang.srt.layers.communicator import ( +from sglang.srt.layers import layer_boundary as communicator +from sglang.srt.layers.layer_boundary import ( boundary, construction, contracts, @@ -20,13 +20,13 @@ from sglang.srt.layers.communicator import ( output, prepare, ) -from sglang.srt.layers.communicator import residual as residual_contract -from sglang.srt.layers.communicator import ( +from sglang.srt.layers.layer_boundary import residual as residual_contract +from sglang.srt.layers.layer_boundary import ( stage, ) -from sglang.srt.layers.communicator.adapters import attention, branch, lora, overlap -from sglang.srt.layers.communicator.fusions import allreduce -from sglang.srt.layers.communicator.residual import ( +from sglang.srt.layers.layer_boundary.adapters import attention, branch, lora, overlap +from sglang.srt.layers.layer_boundary.fusions import allreduce +from sglang.srt.layers.layer_boundary.residual import ( access, add_norm, batch, diff --git a/test/registered/debug_utils/test_engine_dumper_comparator_e2e.py b/test/registered/debug_utils/test_engine_dumper_comparator_e2e.py index 22c9cfa4ab..7615d09574 100644 --- a/test/registered/debug_utils/test_engine_dumper_comparator_e2e.py +++ b/test/registered/debug_utils/test_engine_dumper_comparator_e2e.py @@ -68,7 +68,7 @@ patches: - target: sglang.srt.models.qwen3_moe.Qwen3MoeDecoderLayer.forward edits: - match: | - hidden_states = self.attn_stage.prepare( + hidden_states = self.attn_boundary.prepare( hidden_states, forward_batch, captured_last_layer_outputs=captured_last_layer_outputs, @@ -83,7 +83,7 @@ patches: forward_batch=forward_batch, ) append: "dumper.dump('attn_output', hidden_states, dims='t h[attn_tp:partial] # tp:replicated')" - - match: "hidden_states = self.ffn_stage.prepare(hidden_states, forward_batch)" + - match: "hidden_states = self.ffn_boundary.prepare(hidden_states, forward_batch)" append: "dumper.dump('pre_mlp_residual', hidden_states, dims='t h # tp:replicated')" - match: "hidden_states = self.mlp(hidden_states, forward_batch)" append: "dumper.dump('mlp_output', hidden_states, dims='t h[moe_tp:partial] # tp:replicated')" @@ -120,7 +120,7 @@ patches: - target: sglang.srt.models.qwen3_moe.Qwen3MoeDecoderLayer.forward edits: - match: | - hidden_states = self.attn_stage.prepare( + hidden_states = self.attn_boundary.prepare( hidden_states, forward_batch, captured_last_layer_outputs=captured_last_layer_outputs, @@ -135,7 +135,7 @@ patches: forward_batch=forward_batch, ) append: "dumper.dump('attn_output', hidden_states, dims='t h # tp:replicated')" - - match: "hidden_states = self.ffn_stage.prepare(hidden_states, forward_batch)" + - match: "hidden_states = self.ffn_boundary.prepare(hidden_states, forward_batch)" append: "dumper.dump('pre_mlp_residual', hidden_states, dims='t h # tp:replicated')" - match: "hidden_states = self.mlp(hidden_states, forward_batch)" append: "dumper.dump('mlp_output', hidden_states, dims='t h[moe_tp:partial] # tp:replicated')" diff --git a/test/registered/kernels/ops/layernorm/test_mhc_kernels.py b/test/registered/kernels/ops/layernorm/test_mhc_kernels.py index 5a7c936b5e..2a5c6fdf48 100644 --- a/test/registered/kernels/ops/layernorm/test_mhc_kernels.py +++ b/test/registered/kernels/ops/layernorm/test_mhc_kernels.py @@ -143,7 +143,7 @@ def test_mhc_fused_post_pre_matches_unfused( def _check_glm_boundary(x, residual, post, comb, fn, scale, base, *, use_norm): from sglang.srt.environ import envs - from sglang.srt.layers.communicator import MHCState + from sglang.srt.layers.layer_boundary import MHCState from sglang.srt.layers.layernorm import RMSNorm from sglang.srt.models.glm5_next import Glm5NextDecoderLayer diff --git a/test/registered/ops/test_aiter_allreduce_fusion_amd.py b/test/registered/ops/test_aiter_allreduce_fusion_amd.py index 3e9ee6f56f..727255d206 100755 --- a/test/registered/ops/test_aiter_allreduce_fusion_amd.py +++ b/test/registered/ops/test_aiter_allreduce_fusion_amd.py @@ -10,8 +10,8 @@ from unittest.mock import MagicMock, patch import torch -from sglang.srt.layers.communicator import UnreducedOutput -from sglang.srt.layers.communicator.fusions.allreduce import complete_attention_input +from sglang.srt.layers.layer_boundary import UnreducedOutput +from sglang.srt.layers.layer_boundary.fusions.allreduce import complete_attention_input from sglang.srt.layers.moe import utils as moe_utils from sglang.srt.model_executor.forward_batch_info import ForwardMode from sglang.srt.runtime_context import get_context, get_flags, get_parallel diff --git a/test/registered/unit/batch_overlap/test_tbo_unreduced_input.py b/test/registered/unit/batch_overlap/test_tbo_unreduced_input.py index c699830886..69d7903dce 100644 --- a/test/registered/unit/batch_overlap/test_tbo_unreduced_input.py +++ b/test/registered/unit/batch_overlap/test_tbo_unreduced_input.py @@ -9,8 +9,8 @@ from unittest.mock import patch import torch from sglang.srt.batch_overlap import two_batch_overlap as tbo -from sglang.srt.layers.communicator import ADD, Layout, UnreducedOutput -from sglang.srt.layers.communicator.residual.stream import ResidualStream +from sglang.srt.layers.layer_boundary import ADD, Layout, UnreducedOutput +from sglang.srt.layers.layer_boundary.residual.stream import ResidualStream from sglang.srt.utils import empty_context from sglang.test.ci.ci_register import register_cpu_ci from sglang.test.test_utils import CustomTestCase @@ -78,7 +78,7 @@ class TestTboEntryReducesItsInput(CustomTestCase): merged = tbo.model_forward_stages( layers=[ SimpleNamespace( - attn_stage=SimpleNamespace( + attn_boundary=SimpleNamespace( incoming_residual_rows=Layout(frozenset()) ) ) diff --git a/test/registered/unit/layers/moe/test_cutedsl_ar_fusion.py b/test/registered/unit/layers/moe/test_cutedsl_ar_fusion.py index 729ac7dfc6..6abfb5ae29 100644 --- a/test/registered/unit/layers/moe/test_cutedsl_ar_fusion.py +++ b/test/registered/unit/layers/moe/test_cutedsl_ar_fusion.py @@ -6,7 +6,12 @@ from unittest.mock import MagicMock, patch import pytest import torch -from sglang.srt.layers.communicator import ( +from sglang.srt.layers.flashinfer_mnnvl_cutedsl import ( + FlashInferMNNVLCuteDSLARFusion, + _retargeted_config, + _with_early_finalize_shared_load, +) +from sglang.srt.layers.layer_boundary import ( ADD, NORM_QUANT_READ, Layout, @@ -16,26 +21,21 @@ from sglang.srt.layers.communicator import ( UnreducedOutput, reduce_output, ) -from sglang.srt.layers.communicator.construction import BatchVariant -from sglang.srt.layers.communicator.fusions.allreduce import ( +from sglang.srt.layers.layer_boundary.construction import BatchVariant +from sglang.srt.layers.layer_boundary.fusions.allreduce import ( attention_fusions, complete_attention_input, complete_ffn_input, ffn_fusions, ) -from sglang.srt.layers.communicator.fusions.cutedsl import ( +from sglang.srt.layers.layer_boundary.fusions.cutedsl import ( CuteDSLFusion, MoeFinalizeHandoff, install_cutedsl_fusion, prepare_cutedsl_fusion, ) -from sglang.srt.layers.communicator.prepare import _consumer_step, _read_input -from sglang.srt.layers.communicator.residual.access import finish_layer_stack -from sglang.srt.layers.flashinfer_mnnvl_cutedsl import ( - FlashInferMNNVLCuteDSLARFusion, - _retargeted_config, - _with_early_finalize_shared_load, -) +from sglang.srt.layers.layer_boundary.prepare import _consumer_step, _read_input +from sglang.srt.layers.layer_boundary.residual.access import finish_layer_stack from sglang.srt.layers.layernorm import RMSNorm from sglang.srt.model_executor.cuda_graph_config import CudaGraphConfig, PhaseConfig from sglang.srt.model_executor.forward_batch_info import ForwardMode @@ -47,7 +47,7 @@ from sglang.test.communicator_patch import patch_communicator register_cpu_ci(est_time=9, suite="base-a-test-cpu") -_MODULE = "sglang.srt.layers.communicator.fusions.cutedsl" +_MODULE = "sglang.srt.layers.layer_boundary.fusions.cutedsl" _DECODE = SimpleNamespace(forward_mode=ForwardMode.DECODE, input_ids=torch.zeros(8)) @@ -91,8 +91,8 @@ def _bound(entry): def _test_layer(**kwargs): comm = _communicator() return SimpleNamespace( - attn_stage=stub_stage(comm, StageKind.ATTENTION), - ffn_stage=stub_stage(comm, StageKind.FFN), + attn_boundary=stub_stage(comm, StageKind.ATTENTION), + ffn_boundary=stub_stage(comm, StageKind.FFN), **kwargs, ) @@ -244,17 +244,17 @@ def test_install_does_not_require_a_fused_successor(): reset_context() publish(ServerArgs(model_path="dummy"), role="test") first, last = _communicator(), _communicator() - ordinary = SimpleNamespace(ffn_stage=SimpleNamespace(fusions=None)) + ordinary = SimpleNamespace(ffn_boundary=SimpleNamespace(fusions=None)) install_cutedsl_fusion( [ SimpleNamespace( - attn_stage=stub_stage(first, StageKind.ATTENTION), - ffn_stage=stub_stage(first, StageKind.FFN), + attn_boundary=stub_stage(first, StageKind.ATTENTION), + ffn_boundary=stub_stage(first, StageKind.FFN), ), ordinary, SimpleNamespace( - attn_stage=stub_stage(last, StageKind.ATTENTION), - ffn_stage=stub_stage(last, StageKind.FFN), + attn_boundary=stub_stage(last, StageKind.ATTENTION), + ffn_boundary=stub_stage(last, StageKind.FFN), ), ], hidden_size=8, @@ -273,14 +273,14 @@ def test_a_service_nested_under_a_wrapper_is_prepared(): prepared = [] layer = torch.nn.Linear(2, 2) comm = _communicator() - layer.attn_stage, layer.ffn_stage = ( + layer.attn_boundary, layer.ffn_boundary = ( stub_stage(comm, StageKind.ATTENTION), stub_stage(comm, StageKind.FFN), ) wrapper = torch.nn.Module() wrapper.language_model = torch.nn.Sequential(layer) _install([layer]) - layer.ffn_stage.fusions.service.prepare = lambda *, max_m: prepared.append(max_m) + layer.ffn_boundary.fusions.service.prepare = lambda *, max_m: prepared.append(max_m) # The workspace M bound is the largest of every framework source. reset_context() @@ -562,8 +562,8 @@ class TestDeferredLoraAllReduce(unittest.TestCase): from types import SimpleNamespace from unittest.mock import patch - from sglang.srt.layers.communicator import exit as exits - from sglang.srt.layers.communicator.fusions.cutedsl import CuteDSLFusion + from sglang.srt.layers.layer_boundary import exit as exits + from sglang.srt.layers.layer_boundary.fusions.cutedsl import CuteDSLFusion group = object() fb = SimpleNamespace( diff --git a/test/registered/unit/layers/moe/test_moe_output_reduction.py b/test/registered/unit/layers/moe/test_moe_output_reduction.py index 8dfb7d37f7..b2e9c78070 100644 --- a/test/registered/unit/layers/moe/test_moe_output_reduction.py +++ b/test/registered/unit/layers/moe/test_moe_output_reduction.py @@ -166,7 +166,7 @@ class TestReplicatedMoeOutput(CustomTestCase): class TestModelsWithExplicitDpCompletion(CustomTestCase): def test_direct_dp_exits_publish_their_selected_sum(self): # Execute the real orchestration methods without constructing weights. - # These models own their DP exit instead of using StageCommunicator. + # These models own their DP exit instead of using StageBoundary. import __future__ for filename, method in ( diff --git a/test/registered/unit/layers/test_aux_hidden_states.py b/test/registered/unit/layers/test_aux_hidden_states.py index 144e6201a2..b3e62306cb 100644 --- a/test/registered/unit/layers/test_aux_hidden_states.py +++ b/test/registered/unit/layers/test_aux_hidden_states.py @@ -7,9 +7,9 @@ from unittest.mock import Mock, patch import torch from sglang.srt.layers.aux_hidden_states import AuxHiddenStateList, AuxHiddenStatePacker -from sglang.srt.layers.communicator import StageKind -from sglang.srt.layers.communicator.residual.access import norm_output -from sglang.srt.layers.communicator.residual.stream import ResidualStream +from sglang.srt.layers.layer_boundary import StageKind +from sglang.srt.layers.layer_boundary.residual.access import norm_output +from sglang.srt.layers.layer_boundary.residual.stream import ResidualStream from sglang.test.boundary_fixtures import prepare_attention, stub_plan, stub_stage from sglang.test.ci.ci_register import register_cpu_ci from sglang.test.test_utils import CustomTestCase @@ -35,8 +35,8 @@ class TestAuxStorage(CustomTestCase): backend._copy_output_to_buffer(source, tuple(buffers), 2) def test_capture_move_owns_gather_but_not_slice(self): - from sglang.srt.layers.communicator.boundary import Boundary - from sglang.srt.layers.communicator.layout import Layout, TokenAxis + from sglang.srt.layers.layer_boundary.boundary import Boundary + from sglang.srt.layers.layer_boundary.layout import Layout, TokenAxis from sglang.test.communicator_patch import patch_communicator full = Layout(frozenset()) @@ -181,7 +181,7 @@ class TestBoundCaptureOwnership(CustomTestCase): import test_declared_decoder_boundary as fixture from sglang.srt.layers import layernorm - from sglang.srt.layers.communicator import ( + from sglang.srt.layers.layer_boundary import ( ADD, FusedMlpInput, SumGroup, diff --git a/test/registered/unit/layers/test_batch_owned_residual.py b/test/registered/unit/layers/test_batch_owned_residual.py index 90b6229396..47875bc33d 100644 --- a/test/registered/unit/layers/test_batch_owned_residual.py +++ b/test/registered/unit/layers/test_batch_owned_residual.py @@ -10,9 +10,9 @@ from unittest.mock import Mock, patch import torch from sglang.srt.batch_overlap.two_batch_overlap import TboForwardBatchPreparer -from sglang.srt.layers.communicator import ADD -from sglang.srt.layers.communicator.output import UnreducedOutput -from sglang.srt.layers.communicator.residual import batch +from sglang.srt.layers.layer_boundary import ADD +from sglang.srt.layers.layer_boundary.output import UnreducedOutput +from sglang.srt.layers.layer_boundary.residual import batch from sglang.srt.model_executor.forward_batch_info import ( ForwardBatch, ForwardMode, diff --git a/test/registered/unit/layers/test_boundary_edges.py b/test/registered/unit/layers/test_boundary_edges.py index 8703a868bf..dc07f20aa6 100644 --- a/test/registered/unit/layers/test_boundary_edges.py +++ b/test/registered/unit/layers/test_boundary_edges.py @@ -6,8 +6,8 @@ from unittest.mock import MagicMock, patch import msgspec import torch -from sglang.srt.layers import communicator as comm -from sglang.srt.layers.communicator import ( +from sglang.srt.layers import layer_boundary as comm +from sglang.srt.layers.layer_boundary import ( ADD, EdgeDecl, FusedMlpInput, @@ -19,8 +19,8 @@ from sglang.srt.layers.communicator import ( make_boundary, make_output_boundary, ) -from sglang.srt.layers.communicator import prepare as comm_ops -from sglang.srt.layers.communicator.ops import identity_output +from sglang.srt.layers.layer_boundary import prepare as comm_ops +from sglang.srt.layers.layer_boundary.ops import identity_output from sglang.srt.runtime_context import get_parallel from sglang.test.ci.ci_register import register_cpu_ci from sglang.test.communicator_patch import patch_communicator diff --git a/test/registered/unit/layers/test_boundary_output_contracts.py b/test/registered/unit/layers/test_boundary_output_contracts.py index a18c6be598..58958c8907 100644 --- a/test/registered/unit/layers/test_boundary_output_contracts.py +++ b/test/registered/unit/layers/test_boundary_output_contracts.py @@ -8,22 +8,22 @@ from unittest.mock import Mock, patch import test_declared_decoder_boundary as fixture import torch -from sglang.srt.layers.communicator import ( +from sglang.srt.layers.layer_boundary import ( ADD, ProducerReduction, declare_attn, declare_ffn, ) -from sglang.srt.layers.communicator import exit as exits -from sglang.srt.layers.communicator import ( +from sglang.srt.layers.layer_boundary import exit as exits +from sglang.srt.layers.layer_boundary import ( make_stages, ) -from sglang.srt.layers.communicator.contracts import BatchVariant -from sglang.srt.layers.communicator.fusions.cutedsl import CuteDSLFusion -from sglang.srt.layers.communicator.layout import SumGroup -from sglang.srt.layers.communicator.prepare import _dispatch_consumer -from sglang.srt.layers.communicator.residual import batch -from sglang.srt.layers.communicator.residual.stream import ResidualStream +from sglang.srt.layers.layer_boundary.contracts import BatchVariant +from sglang.srt.layers.layer_boundary.fusions.cutedsl import CuteDSLFusion +from sglang.srt.layers.layer_boundary.layout import SumGroup +from sglang.srt.layers.layer_boundary.prepare import _dispatch_consumer +from sglang.srt.layers.layer_boundary.residual import batch +from sglang.srt.layers.layer_boundary.residual.stream import ResidualStream from sglang.srt.model_executor.forward_batch_info import ForwardMode from sglang.test.ci.ci_register import register_cpu_ci from sglang.test.communicator_patch import patch_communicator @@ -122,7 +122,7 @@ class TestBoundaryIntegrations(unittest.TestCase): fixture.planning(parallel), patch.object(fusion, "_should_use_finalize", return_value=True), patch( - "sglang.srt.layers.communicator.fusions.cutedsl.get_parallel", + "sglang.srt.layers.layer_boundary.fusions.cutedsl.get_parallel", return_value=parallel, ), ): @@ -221,10 +221,10 @@ class TestBoundaryIntegrations(unittest.TestCase): output = SimpleNamespace(finish=lambda hidden: hidden) layer = SimpleNamespace( - attn_stage=SimpleNamespace( + attn_boundary=SimpleNamespace( prepare=prepare, finish=lambda hidden, fb: hidden ), - ffn_stage=SimpleNamespace( + ffn_boundary=SimpleNamespace( prepare=lambda hidden, fb, **kw: hidden, exit=lambda fb: nullcontext(output), ), diff --git a/test/registered/unit/layers/test_boundary_output_policy.py b/test/registered/unit/layers/test_boundary_output_policy.py index c4511f9928..cbfd7d5dca 100644 --- a/test/registered/unit/layers/test_boundary_output_policy.py +++ b/test/registered/unit/layers/test_boundary_output_policy.py @@ -7,9 +7,9 @@ from unittest.mock import patch import test_declared_decoder_boundary as fixture import torch -from sglang.srt.layers.communicator.contracts import BatchVariant -from sglang.srt.layers.communicator.output import OutputTransform -from sglang.srt.layers.communicator.residual.stream import ResidualStream +from sglang.srt.layers.layer_boundary.contracts import BatchVariant +from sglang.srt.layers.layer_boundary.output import OutputTransform +from sglang.srt.layers.layer_boundary.residual.stream import ResidualStream from sglang.srt.layers.moe import utils as moe_utils from sglang.srt.runtime_context import get_forward from sglang.test.boundary_fixtures import finish_exit @@ -29,8 +29,8 @@ class TestBoundaryOutputPolicy(unittest.TestCase): ) def test_npu_weight_cache_belongs_to_one_prepare_call(self): - from sglang.srt.layers.communicator import prepare as ops - from sglang.srt.layers.communicator.residual.stream import ResidualStream + from sglang.srt.layers.layer_boundary import prepare as ops + from sglang.srt.layers.layer_boundary.residual.stream import ResidualStream parallel = fixture.parallel_of(attn_dp=1, attn_tp=2) layer = fixture.build(fixture.layer_case(0, 2), parallel) @@ -61,7 +61,7 @@ class TestBoundaryOutputPolicy(unittest.TestCase): self.assertEqual(events, ["reduce", "cache", "reduce"]) def test_token_slice_reads_the_current_parallel_rank(self): - from sglang.srt.layers.communicator.ops import tp_slice + from sglang.srt.layers.layer_boundary.ops import tp_slice hidden = torch.arange(8).reshape(4, 2) for rank in (0, 1): diff --git a/test/registered/unit/layers/test_communicator_ffn_exit.py b/test/registered/unit/layers/test_communicator_ffn_exit.py index 4a3593ab63..9dbcf1dd0a 100644 --- a/test/registered/unit/layers/test_communicator_ffn_exit.py +++ b/test/registered/unit/layers/test_communicator_ffn_exit.py @@ -5,22 +5,22 @@ from unittest.mock import MagicMock import torch -from sglang.srt.layers import communicator as comm -from sglang.srt.layers.communicator import ( +from sglang.srt.layers import layer_boundary as comm +from sglang.srt.layers.layer_boundary import ( Layout, StageOutput, SumGroup, UnreducedOutput, ) -from sglang.srt.layers.communicator import ops as transport_ops -from sglang.srt.layers.communicator import prepare as comm_ops -from sglang.srt.layers.communicator import ( +from sglang.srt.layers.layer_boundary import ops as transport_ops +from sglang.srt.layers.layer_boundary import prepare as comm_ops +from sglang.srt.layers.layer_boundary import ( reduce_output, ) -from sglang.srt.layers.communicator.contracts import BatchVariant -from sglang.srt.layers.communicator.ops import identity_output -from sglang.srt.layers.communicator.residual.access import finish_layer_stack -from sglang.srt.layers.communicator.residual.stream import ResidualStream +from sglang.srt.layers.layer_boundary.contracts import BatchVariant +from sglang.srt.layers.layer_boundary.ops import identity_output +from sglang.srt.layers.layer_boundary.residual.access import finish_layer_stack +from sglang.srt.layers.layer_boundary.residual.stream import ResidualStream from sglang.srt.runtime_context import get_forward from sglang.test.boundary_fixtures import ( finish_exit, diff --git a/test/registered/unit/layers/test_declared_attention_cp.py b/test/registered/unit/layers/test_declared_attention_cp.py index 830b72e9db..1a6d945f27 100644 --- a/test/registered/unit/layers/test_declared_attention_cp.py +++ b/test/registered/unit/layers/test_declared_attention_cp.py @@ -15,10 +15,10 @@ from unittest.mock import patch import torch -from sglang.srt.layers import communicator as comm +from sglang.srt.layers import layer_boundary as comm from sglang.srt.layers import layernorm_sp -from sglang.srt.layers.communicator.adapters import context_parallel as dsa_cp -from sglang.srt.layers.communicator.residual.stream import ResidualStream +from sglang.srt.layers.layer_boundary.adapters import context_parallel as dsa_cp +from sglang.srt.layers.layer_boundary.residual.stream import ResidualStream from sglang.test.boundary_fixtures import finish_exit, make_test_stages, prepare_input from sglang.test.ci.ci_register import register_cpu_ci from sglang.test.communicator_patch import patch_communicator diff --git a/test/registered/unit/layers/test_declared_decoder_boundary.py b/test/registered/unit/layers/test_declared_decoder_boundary.py index 9209e722a8..384fb52777 100644 --- a/test/registered/unit/layers/test_declared_decoder_boundary.py +++ b/test/registered/unit/layers/test_declared_decoder_boundary.py @@ -19,22 +19,22 @@ from unittest.mock import MagicMock, patch import torch -from sglang.srt.layers import communicator as comm +from sglang.srt.layers import layer_boundary as comm from sglang.srt.layers import layernorm_sp -from sglang.srt.layers.communicator import ( +from sglang.srt.layers.layer_boundary import ( SumGroup, TokenAxis, ) -from sglang.srt.layers.communicator import boundary as comm_boundary -from sglang.srt.layers.communicator import construction as comm_layer -from sglang.srt.layers.communicator import exit as comm_exit -from sglang.srt.layers.communicator import layout as comm_layout -from sglang.srt.layers.communicator import ops as transport_ops -from sglang.srt.layers.communicator import prepare as comm_ops -from sglang.srt.layers.communicator.adapters import branch -from sglang.srt.layers.communicator.construction import BatchVariant -from sglang.srt.layers.communicator.fusions.allreduce import complete_ffn_input -from sglang.srt.layers.communicator.ops import ( +from sglang.srt.layers.layer_boundary import boundary as comm_boundary +from sglang.srt.layers.layer_boundary import construction as comm_layer +from sglang.srt.layers.layer_boundary import exit as comm_exit +from sglang.srt.layers.layer_boundary import layout as comm_layout +from sglang.srt.layers.layer_boundary import ops as transport_ops +from sglang.srt.layers.layer_boundary import prepare as comm_ops +from sglang.srt.layers.layer_boundary.adapters import branch +from sglang.srt.layers.layer_boundary.construction import BatchVariant +from sglang.srt.layers.layer_boundary.fusions.allreduce import complete_ffn_input +from sglang.srt.layers.layer_boundary.ops import ( gather_attention_tp, identity_output, output_on_residual_shard, @@ -44,10 +44,10 @@ from sglang.srt.layers.communicator.ops import ( take_back_cp_shard, update_and_gather, ) -from sglang.srt.layers.communicator.residual import mhc as mhc_module -from sglang.srt.layers.communicator.residual.access import finish_layer_stack -from sglang.srt.layers.communicator.residual.add_norm import NORM_READ, PLAIN_RESIDUAL -from sglang.srt.layers.communicator.residual.stream import OwedOutput, ResidualStream +from sglang.srt.layers.layer_boundary.residual import mhc as mhc_module +from sglang.srt.layers.layer_boundary.residual.access import finish_layer_stack +from sglang.srt.layers.layer_boundary.residual.add_norm import NORM_READ, PLAIN_RESIDUAL +from sglang.srt.layers.layer_boundary.residual.stream import OwedOutput, ResidualStream from sglang.srt.layers.moe import utils as moe_utils from sglang.srt.model_executor.forward_batch_info import ForwardMode from sglang.test.boundary_fixtures import ( diff --git a/test/registered/unit/layers/test_dense_mlp_under_prefill_cp.py b/test/registered/unit/layers/test_dense_mlp_under_prefill_cp.py index 666eeac93a..25306befca 100644 --- a/test/registered/unit/layers/test_dense_mlp_under_prefill_cp.py +++ b/test/registered/unit/layers/test_dense_mlp_under_prefill_cp.py @@ -2,12 +2,12 @@ import unittest import test_declared_decoder_boundary as fixture -from sglang.srt.layers.communicator import ( +from sglang.srt.layers.layer_boundary import ( TokenAxis, ) -from sglang.srt.layers.communicator import prepare as comm_ops -from sglang.srt.layers.communicator.contracts import BatchVariant -from sglang.srt.layers.communicator.ops import scatter_moe_cp_output +from sglang.srt.layers.layer_boundary import prepare as comm_ops +from sglang.srt.layers.layer_boundary.contracts import BatchVariant +from sglang.srt.layers.layer_boundary.ops import scatter_moe_cp_output from sglang.test.boundary_fixtures import make_test_stages from sglang.test.ci.ci_register import register_cpu_ci from sglang.test.test_utils import CustomTestCase diff --git a/test/registered/unit/layers/test_dp_attention_cp_gather.py b/test/registered/unit/layers/test_dp_attention_cp_gather.py index a8d97c72fb..7fa7f9685e 100644 --- a/test/registered/unit/layers/test_dp_attention_cp_gather.py +++ b/test/registered/unit/layers/test_dp_attention_cp_gather.py @@ -13,15 +13,16 @@ from unittest.mock import PropertyMock, patch import torch -from sglang.srt.layers import communicator as comm -from sglang.srt.layers import dp_attention, layernorm_sp -from sglang.srt.layers.communicator import prepare as comm_ops -from sglang.srt.layers.communicator.contracts import BatchVariant -from sglang.srt.layers.communicator.residual.add_norm import NORM_READ, PLAIN_RESIDUAL +from sglang.srt.layers import dp_attention +from sglang.srt.layers import layer_boundary as comm +from sglang.srt.layers import layernorm_sp from sglang.srt.layers.cp import base as cp_base from sglang.srt.layers.cp import padding as cp_padding from sglang.srt.layers.cp.zigzag import ZigzagCPStrategy from sglang.srt.layers.dp_attention import DpPaddingMode +from sglang.srt.layers.layer_boundary import prepare as comm_ops +from sglang.srt.layers.layer_boundary.contracts import BatchVariant +from sglang.srt.layers.layer_boundary.residual.add_norm import NORM_READ, PLAIN_RESIDUAL from sglang.srt.model_executor.forward_batch_info import ForwardMode from sglang.test.boundary_fixtures import ( make_test_stages, diff --git a/test/registered/unit/layers/test_independent_stage_construction.py b/test/registered/unit/layers/test_independent_stage_construction.py index c2b18d2468..9fc5316aca 100644 --- a/test/registered/unit/layers/test_independent_stage_construction.py +++ b/test/registered/unit/layers/test_independent_stage_construction.py @@ -9,7 +9,7 @@ from unittest.mock import patch import test_declared_decoder_boundary as fixture import torch -from sglang.srt.layers.communicator import ( +from sglang.srt.layers.layer_boundary import ( ProducerReduction, StageKind, declare_attn, @@ -18,8 +18,8 @@ from sglang.srt.layers.communicator import ( make_ffn_stage, make_stages, ) -from sglang.srt.layers.communicator.contracts import BatchVariant -from sglang.srt.layers.communicator.residual import batch +from sglang.srt.layers.layer_boundary.contracts import BatchVariant +from sglang.srt.layers.layer_boundary.residual import batch from sglang.srt.model_executor.forward_batch_info import ForwardMode from sglang.test.ci.ci_register import register_cpu_ci from sglang.test.test_utils import CustomTestCase @@ -29,7 +29,7 @@ register_cpu_ci(est_time=5, suite="base-a-test-cpu") class TestIndependentStageConstruction(CustomTestCase): def test_active_variant_without_a_bound_path_never_falls_back(self): - from sglang.srt.layers.communicator import construction + from sglang.srt.layers.layer_boundary import construction from sglang.test.boundary_fixtures import stub_plan for variant in BatchVariant: diff --git a/test/registered/unit/layers/test_layer_communicator_fusion_gate.py b/test/registered/unit/layers/test_layer_communicator_fusion_gate.py index 06bb9ecad7..8800b930f5 100644 --- a/test/registered/unit/layers/test_layer_communicator_fusion_gate.py +++ b/test/registered/unit/layers/test_layer_communicator_fusion_gate.py @@ -6,17 +6,17 @@ from unittest.mock import MagicMock, patch import torch -from sglang.srt.layers import communicator as comm -from sglang.srt.layers.communicator import ( +from sglang.srt.layers import layer_boundary as comm +from sglang.srt.layers.layer_boundary import ( Layout, StageOutput, SumGroup, ) -from sglang.srt.layers.communicator import ops as transport_ops -from sglang.srt.layers.communicator import prepare as comm_ops -from sglang.srt.layers.communicator.contracts import BatchVariant -from sglang.srt.layers.communicator.exit import OutputBoundary -from sglang.srt.layers.communicator.ops import identity_output +from sglang.srt.layers.layer_boundary import ops as transport_ops +from sglang.srt.layers.layer_boundary import prepare as comm_ops +from sglang.srt.layers.layer_boundary.contracts import BatchVariant +from sglang.srt.layers.layer_boundary.exit import OutputBoundary +from sglang.srt.layers.layer_boundary.ops import identity_output from sglang.srt.layers.moe import ( can_merge_post_experts_all_reduce, deferred_post_experts_all_reduce, diff --git a/test/registered/unit/layers/test_layernorm_sp.py b/test/registered/unit/layers/test_layernorm_sp.py index 6eea228167..244cf507d3 100644 --- a/test/registered/unit/layers/test_layernorm_sp.py +++ b/test/registered/unit/layers/test_layernorm_sp.py @@ -13,16 +13,16 @@ from unittest.mock import MagicMock, patch import torch from sglang.srt.arg_groups.layernorm_sp_hook import validate_layernorm_sp -from sglang.srt.layers import communicator as comm +from sglang.srt.layers import layer_boundary as comm from sglang.srt.layers import layernorm_sp -from sglang.srt.layers.communicator import ( +from sglang.srt.layers.layer_boundary import ( Layout, StageKind, StageOutput, SumGroup, ) -from sglang.srt.layers.communicator import prepare as comm_ops -from sglang.srt.layers.communicator.contracts import BatchVariant +from sglang.srt.layers.layer_boundary import prepare as comm_ops +from sglang.srt.layers.layer_boundary.contracts import BatchVariant from sglang.srt.model_executor.forward_batch_info import ForwardMode from sglang.srt.runtime_context import ( get_flags, diff --git a/test/registered/unit/layers/test_prepare_attn_steps.py b/test/registered/unit/layers/test_prepare_attn_steps.py index b0ebefe2ff..de9d3d59ea 100644 --- a/test/registered/unit/layers/test_prepare_attn_steps.py +++ b/test/registered/unit/layers/test_prepare_attn_steps.py @@ -10,12 +10,12 @@ from unittest.mock import MagicMock import msgspec import torch -from sglang.srt.layers import communicator as comm -from sglang.srt.layers.communicator import StageKind -from sglang.srt.layers.communicator import prepare as comm_ops -from sglang.srt.layers.communicator.contracts import BatchVariant -from sglang.srt.layers.communicator.fusions.allreduce import attention_fusions -from sglang.srt.layers.communicator.ops import identity_output +from sglang.srt.layers import layer_boundary as comm +from sglang.srt.layers.layer_boundary import StageKind +from sglang.srt.layers.layer_boundary import prepare as comm_ops +from sglang.srt.layers.layer_boundary.contracts import BatchVariant +from sglang.srt.layers.layer_boundary.fusions.allreduce import attention_fusions +from sglang.srt.layers.layer_boundary.ops import identity_output from sglang.test.boundary_fixtures import prepare_input, stub_plan, stub_stage from sglang.test.ci.ci_register import register_cpu_ci from sglang.test.communicator_patch import patch_communicator @@ -298,7 +298,7 @@ class TestPrepareAttnSteps(CustomTestCase): class TestFusedReadForms(CustomTestCase): def test_consumer_form_and_backend_policy_are_both_required(self): - from sglang.srt.layers.communicator.residual.add_norm import ( + from sglang.srt.layers.layer_boundary.residual.add_norm import ( Fp8Input, NormQuantRead, ) diff --git a/test/registered/unit/layers/test_residual_stream.py b/test/registered/unit/layers/test_residual_stream.py index 073b58c8df..dc8e8b31ba 100644 --- a/test/registered/unit/layers/test_residual_stream.py +++ b/test/registered/unit/layers/test_residual_stream.py @@ -6,7 +6,7 @@ from unittest.mock import Mock, patch import torch from sglang.srt.layers.aux_hidden_states import AuxHiddenStateList -from sglang.srt.layers.communicator import ( +from sglang.srt.layers.layer_boundary import ( EdgeDecl, Layout, StageEntry, @@ -16,16 +16,16 @@ from sglang.srt.layers.communicator import ( SumGroup, make_boundary, ) -from sglang.srt.layers.communicator.contracts import BatchVariant, StageKind -from sglang.srt.layers.communicator.output import UnreducedOutput -from sglang.srt.layers.communicator.residual.access import add_to_output -from sglang.srt.layers.communicator.residual.add_norm import ADD -from sglang.srt.layers.communicator.residual.stream import ( +from sglang.srt.layers.layer_boundary.contracts import BatchVariant, StageKind +from sglang.srt.layers.layer_boundary.output import UnreducedOutput +from sglang.srt.layers.layer_boundary.residual.access import add_to_output +from sglang.srt.layers.layer_boundary.residual.add_norm import ADD +from sglang.srt.layers.layer_boundary.residual.stream import ( DeclaredSum, OwedOutput, ResidualStream, ) -from sglang.srt.layers.communicator.stage import StageCommunicator +from sglang.srt.layers.layer_boundary.stage import StageBoundary from sglang.srt.model_executor.forward_batch_info import ForwardMode from sglang.test.boundary_fixtures import prepare_attention, stub_plan, stub_stage from sglang.test.ci.ci_register import register_cpu_ci @@ -63,7 +63,7 @@ class TestResidualStream(CustomTestCase): enters_stack=True, ) steps = StageSteps(StageEntry(boundary.prepare, rows), StageOutput(rows), None) - stage = StageCommunicator( + stage = StageBoundary( SimpleNamespace(norm=None), declaration=SimpleNamespace(kind=StageKind.ATTENTION), ) @@ -106,7 +106,7 @@ class TestResidualStream(CustomTestCase): ) ) steps = StageSteps(StageEntry(boundary.prepare, rows), StageOutput(rows), None) - stage = StageCommunicator( + stage = StageBoundary( SimpleNamespace(norm=None), declaration=SimpleNamespace(kind=StageKind.ATTENTION), ) @@ -142,7 +142,7 @@ class TestResidualStream(CustomTestCase): hidden = stream.leave(self.partial, ADD, declared_sum=SumGroup.TP) self.assertIsInstance(hidden, OwedOutput) with patch( - "sglang.srt.layers.communicator.residual.stream._sum_group", + "sglang.srt.layers.layer_boundary.residual.stream._sum_group", return_value=self.group, ): snapshot = stream.snapshot(hidden) @@ -170,7 +170,7 @@ class TestResidualStream(CustomTestCase): stream = ResidualStream(self.residual) hidden = stream.leave(self.partial, ADD, declared_sum=SumGroup.TP) with patch( - "sglang.srt.layers.communicator.residual.stream._sum_group", + "sglang.srt.layers.layer_boundary.residual.stream._sum_group", return_value=self.group, ): hidden = stream.complete(hidden) @@ -320,7 +320,7 @@ class TestResidualStream(CustomTestCase): def test_capture_restores_the_producers_rows_after_a_scattered_read(self): from unittest.mock import patch - from sglang.srt.layers.communicator import TokenAxis + from sglang.srt.layers.layer_boundary import TokenAxis full = Layout(frozenset()) local = Layout(frozenset({TokenAxis.ATTN_TP_SCATTER})) @@ -330,7 +330,7 @@ class TestResidualStream(CustomTestCase): shard = torch.full((2, 4), 3.0) expected = torch.cat([shard, torch.full_like(shard, 7.0)]) with patch( - "sglang.srt.layers.communicator.ops._redistribute_from_attn_tp_shards", + "sglang.srt.layers.layer_boundary.ops._redistribute_from_attn_tp_shards", return_value=expected, ) as gather: captured = boundary.capture_move(shard, forward_batch=None) @@ -339,7 +339,7 @@ class TestResidualStream(CustomTestCase): torch.testing.assert_close(shard, torch.full((2, 4), 3.0)) def test_final_capture_uses_the_norms_residual_result(self): - from sglang.srt.layers.communicator.residual.access import norm_output + from sglang.srt.layers.layer_boundary.residual.access import norm_output outputs = AuxHiddenStateList() updated = torch.full_like(self.partial, 5.0) @@ -388,7 +388,7 @@ class TestResidualStream(CustomTestCase): class TestBatchStageOwnership(CustomTestCase): def test_terminal_norm_releases_layer_buffers(self): - from sglang.srt.layers.communicator.residual import batch + from sglang.srt.layers.layer_boundary.residual import batch fb = SimpleNamespace(residual_stream=None) batch.start(fb) @@ -404,7 +404,7 @@ class TestBatchStageOwnership(CustomTestCase): self.assertIsNone(fb.residual_stream) def test_pp_export_transfers_buffer_ownership(self): - from sglang.srt.layers.communicator.residual import batch + from sglang.srt.layers.layer_boundary.residual import batch fb = SimpleNamespace(residual_stream=None) batch.start(fb) @@ -421,7 +421,7 @@ class TestBatchStageOwnership(CustomTestCase): self.assertTrue(all(ref() is None for ref in refs)) def test_interleaved_batches_keep_independent_streams(self): - from sglang.srt.layers.communicator.residual import batch + from sglang.srt.layers.layer_boundary.residual import batch class Read: norms_plainly = False diff --git a/test/registered/unit/lora/test_triton_dp_attention_unit.py b/test/registered/unit/lora/test_triton_dp_attention_unit.py index aa4aa89025..133e8125a3 100644 --- a/test/registered/unit/lora/test_triton_dp_attention_unit.py +++ b/test/registered/unit/lora/test_triton_dp_attention_unit.py @@ -9,16 +9,16 @@ from unittest.mock import patch import pytest import torch -from sglang.srt.layers.communicator import ( +from sglang.srt.layers.dp_attention import DpPaddingMode +from sglang.srt.layers.layer_boundary import ( ADD, NORM_QUANT_READ, Layout, StageKind, TokenAxis, ) -from sglang.srt.layers.communicator.contracts import BatchVariant, StageEntry -from sglang.srt.layers.communicator.prepare import _consumer_step, _read_input -from sglang.srt.layers.dp_attention import DpPaddingMode +from sglang.srt.layers.layer_boundary.contracts import BatchVariant, StageEntry +from sglang.srt.layers.layer_boundary.prepare import _consumer_step, _read_input from sglang.srt.lora.backend.base_backend import BaseLoRABackend from sglang.srt.lora.backend.triton_backend import ( TritonLoRABackend, diff --git a/test/registered/unit/models/test_aux_capture_deferred_allreduce.py b/test/registered/unit/models/test_aux_capture_deferred_allreduce.py index a7fd69c981..847e78022a 100644 --- a/test/registered/unit/models/test_aux_capture_deferred_allreduce.py +++ b/test/registered/unit/models/test_aux_capture_deferred_allreduce.py @@ -6,13 +6,13 @@ from unittest.mock import patch import torch from torch import nn -from sglang.srt.layers import communicator as comm +from sglang.srt.layers import layer_boundary as comm from sglang.srt.layers.aux_hidden_states import AuxHiddenStateList -from sglang.srt.layers.communicator import StageKind -from sglang.srt.layers.communicator import prepare as comm_ops -from sglang.srt.layers.communicator.contracts import BatchVariant -from sglang.srt.layers.communicator.ops import identity_output -from sglang.srt.layers.communicator.residual.stream import ResidualStream +from sglang.srt.layers.layer_boundary import StageKind +from sglang.srt.layers.layer_boundary import prepare as comm_ops +from sglang.srt.layers.layer_boundary.contracts import BatchVariant +from sglang.srt.layers.layer_boundary.ops import identity_output +from sglang.srt.layers.layer_boundary.residual.stream import ResidualStream from sglang.srt.model_executor.forward_batch_info import ForwardMode, PPProxyTensors from sglang.srt.models.bailing_moe import BailingMoEModel from sglang.srt.models.bailing_moe_v3 import BailingMoELinearModel @@ -67,7 +67,7 @@ class DeferringLayer(nn.Module): self.stage_api = stage_api self.layer_communicator = stub_plan() self.layer_communicator.norm = None - self.attn_stage = stub_stage(self.layer_communicator, StageKind.ATTENTION) + self.attn_boundary = stub_stage(self.layer_communicator, StageKind.ATTENTION) self.layer_communicator._paths[BatchVariant.ORDINARY] = comm.StageSteps( entry=comm.StageEntry( prepare=partial( @@ -400,7 +400,7 @@ class TestPipelineResidualReception(CustomTestCase): streams = torch.randn(2, 4, 3) from dataclasses import replace - from sglang.srt.layers.communicator import declare_ffn + from sglang.srt.layers.layer_boundary import declare_ffn stage = stub_stage(comm_instance, StageKind.ATTENTION) stage.declaration = replace( diff --git a/test/registered/unit/models/test_deepstack_deferred_allreduce.py b/test/registered/unit/models/test_deepstack_deferred_allreduce.py index f7fa0c57ed..7f51e3595d 100644 --- a/test/registered/unit/models/test_deepstack_deferred_allreduce.py +++ b/test/registered/unit/models/test_deepstack_deferred_allreduce.py @@ -8,10 +8,10 @@ from types import SimpleNamespace import torch from torch import nn -from sglang.srt.layers.communicator import UnreducedOutput -from sglang.srt.layers.communicator.residual import batch as residual_batch -from sglang.srt.layers.communicator.residual.add_norm import ADD -from sglang.srt.layers.communicator.residual.stream import OwedOutput +from sglang.srt.layers.layer_boundary import UnreducedOutput +from sglang.srt.layers.layer_boundary.residual import batch as residual_batch +from sglang.srt.layers.layer_boundary.residual.add_norm import ADD +from sglang.srt.layers.layer_boundary.residual.stream import OwedOutput from sglang.test.ci.ci_register import register_cpu_ci from sglang.test.test_utils import CustomTestCase diff --git a/test/registered/unit/models/test_glm5_next_dflash_capture.py b/test/registered/unit/models/test_glm5_next_dflash_capture.py index 0e6d9afdf9..06fefbd6ac 100644 --- a/test/registered/unit/models/test_glm5_next_dflash_capture.py +++ b/test/registered/unit/models/test_glm5_next_dflash_capture.py @@ -13,7 +13,7 @@ from types import SimpleNamespace import torch from torch import nn -from sglang.srt.layers.communicator.residual.stream import ResidualStream +from sglang.srt.layers.layer_boundary.residual.stream import ResidualStream from sglang.srt.models.glm5_next import Glm5NextModel from sglang.test.ci.ci_register import register_cpu_ci from sglang.test.test_utils import CustomTestCase diff --git a/test/registered/unit/models/test_last_layer_communicator.py b/test/registered/unit/models/test_last_layer_communicator.py index a6d13c80dd..b5115c850b 100644 --- a/test/registered/unit/models/test_last_layer_communicator.py +++ b/test/registered/unit/models/test_last_layer_communicator.py @@ -172,7 +172,7 @@ def build(case, num_layers, layer_id, config=None, **kwargs): return make - from sglang.srt.layers.communicator import declare_attn, declare_ffn + from sglang.srt.layers.layer_boundary import declare_attn, declare_ffn patches = dict( declare_attn=declare_attn, diff --git a/test/registered/unit/models/test_longcat_flash_shortcut.py b/test/registered/unit/models/test_longcat_flash_shortcut.py index d535189d39..1cea7bda7c 100644 --- a/test/registered/unit/models/test_longcat_flash_shortcut.py +++ b/test/registered/unit/models/test_longcat_flash_shortcut.py @@ -4,10 +4,10 @@ from types import SimpleNamespace import torch from torch import nn -from sglang.srt.layers import communicator as comm -from sglang.srt.layers.communicator.adapters import branch -from sglang.srt.layers.communicator.residual.stream import ResidualStream -from sglang.srt.layers.communicator.stage import StageCommunicator +from sglang.srt.layers import layer_boundary as comm +from sglang.srt.layers.layer_boundary.adapters import branch +from sglang.srt.layers.layer_boundary.residual.stream import ResidualStream +from sglang.srt.layers.layer_boundary.stage import StageBoundary from sglang.srt.models.longcat_flash import LongcatFlashDecoderLayer from sglang.test.ci.ci_register import register_cpu_ci from sglang.test.communicator_patch import patch_communicator @@ -43,16 +43,16 @@ class TestLongcatShortcut(CustomTestCase): layer = LongcatFlashDecoderLayer.__new__(LongcatFlashDecoderLayer) nn.Module.__init__(layer) batch = SimpleNamespace(residual_stream=ResidualStream()) - layer.attn_stage = SimpleNamespace( + layer.attn_boundary = SimpleNamespace( prepare=lambda h, fb: h, finish=lambda h, fb: h ) moe_communicator.post_attention_layernorm = None dense_communicator.post_attention_layernorm = None - layer.moe_stage = StageCommunicator( + layer.moe_boundary = StageBoundary( moe_communicator, declaration=comm.declare_ffn() ) - layer.moe_stage.prepare = lambda h, fb: fork_hidden - layer.second_ffn = StageCommunicator( + layer.moe_boundary.prepare = lambda h, fb: fork_hidden + layer.second_ffn_boundary = StageBoundary( dense_communicator, declaration=comm.declare_ffn() ) layer.self_attn = [lambda **kw: kw["hidden_states"]] diff --git a/test/registered/unit/models/test_nemotron_h_mtp_reduction.py b/test/registered/unit/models/test_nemotron_h_mtp_reduction.py index 23b4551181..b53704c03d 100644 --- a/test/registered/unit/models/test_nemotron_h_mtp_reduction.py +++ b/test/registered/unit/models/test_nemotron_h_mtp_reduction.py @@ -5,9 +5,9 @@ from unittest.mock import Mock import torch from torch import nn -from sglang.srt.layers.communicator.layout import SumGroup -from sglang.srt.layers.communicator.residual.add_norm import ADD -from sglang.srt.layers.communicator.residual.stream import ResidualStream +from sglang.srt.layers.layer_boundary.layout import SumGroup +from sglang.srt.layers.layer_boundary.residual.add_norm import ADD +from sglang.srt.layers.layer_boundary.residual.stream import ResidualStream from sglang.srt.model_executor.forward_batch_info import ForwardMode from sglang.srt.models import nemotron_h_mtp from sglang.srt.runtime_context import get_context, get_flags, get_parallel diff --git a/test/registered/unit/models/test_nemotron_h_stages.py b/test/registered/unit/models/test_nemotron_h_stages.py index 3f6c9c18a1..4bd676db3b 100644 --- a/test/registered/unit/models/test_nemotron_h_stages.py +++ b/test/registered/unit/models/test_nemotron_h_stages.py @@ -6,7 +6,7 @@ from unittest.mock import MagicMock, patch import torch -from sglang.srt.layers.communicator import ( +from sglang.srt.layers.layer_boundary import ( Layout, MixerExit, StageOutput, @@ -14,7 +14,7 @@ from sglang.srt.layers.communicator import ( TokenAxis, UnreducedOutput, ) -from sglang.srt.layers.communicator.residual.stream import ResidualStream +from sglang.srt.layers.layer_boundary.residual.stream import ResidualStream from sglang.srt.layers.moe.utils import should_skip_mlp_all_reduce from sglang.srt.models import nemotron_h_utils as utils from sglang.srt.runtime_context import get_parallel @@ -25,8 +25,8 @@ register_cpu_ci(est_time=3, suite="base-a-test-cpu") def layer_stage(pattern, index): - from sglang.srt.layers.communicator.construction import BatchVariant - from sglang.srt.layers.communicator.factories import _connections + from sglang.srt.layers.layer_boundary.construction import BatchVariant + from sglang.srt.layers.layer_boundary.factories import _connections previous = utils._declaration(pattern, index - 1) if index else None declaration = replace( diff --git a/test/registered/unit/models/test_step3p5_dense_reduce_scatter.py b/test/registered/unit/models/test_step3p5_dense_reduce_scatter.py index 672b7f73c1..62314a8eb7 100644 --- a/test/registered/unit/models/test_step3p5_dense_reduce_scatter.py +++ b/test/registered/unit/models/test_step3p5_dense_reduce_scatter.py @@ -9,8 +9,8 @@ from unittest.mock import MagicMock import torch -from sglang.srt.layers import communicator as comm -from sglang.srt.layers.communicator.residual.stream import ResidualStream +from sglang.srt.layers import layer_boundary as comm +from sglang.srt.layers.layer_boundary.residual.stream import ResidualStream from sglang.srt.runtime_context import get_forward from sglang.test.boundary_fixtures import stub_plan from sglang.test.ci.ci_register import register_cpu_ci @@ -50,7 +50,7 @@ def dense_layer(*, reduce_scatter): object.__setattr__( layer, - "attn_stage", + "attn_boundary", SimpleNamespace( prepare=lambda h, fb, **kwargs: h, finish=lambda h, fb: h, @@ -58,7 +58,7 @@ def dense_layer(*, reduce_scatter): ) object.__setattr__( layer, - "ffn_stage", + "ffn_boundary", SimpleNamespace( prepare=lambda h, fb: h, exit=stage_exit, diff --git a/test/registered/unit/test_runtime_context.py b/test/registered/unit/test_runtime_context.py index 7b429457a0..12e6287ae6 100644 --- a/test/registered/unit/test_runtime_context.py +++ b/test/registered/unit/test_runtime_context.py @@ -1503,7 +1503,7 @@ class TestForwardFlags(_IsolatedServerArgs): def test_attn_tp_context_per_forward_slots(self): from types import SimpleNamespace - from sglang.srt.layers.communicator import get_attn_tp_context + from sglang.srt.layers.layer_boundary import get_attn_tp_context from sglang.srt.runtime_context import get_forward reset_context()