[Refactor] Rename the module to layer_boundary (#41555)

This commit is contained in:
Cheng Wan
2026-09-28 20:17:50 -07:00
committed by GitHub
parent 701213c867
commit de9f850b36
124 changed files with 819 additions and 812 deletions
@@ -101,7 +101,7 @@ A `capture_signature` (SHA-1 hash of schema version, max_tokens, ignore_eos, TP
The harness pins FlashInfer all-reduce fusion **on** (`--flashinfer-allreduce-fusion-backend trtllm`; the SM90 auto-enable was dropped in #23402). Two names are captured per layer:
- `inputs.1` — hidden states entering the layer. The FFN boundary defers the cross-layer all-reduce, so with fusion active these are rank-local TP-partial sums. CPU-side reduction of them is not the kernel's semantic output and drifts across self-hosted runners, so they are **not** compared. `_assert_fused_tp_layout()` keeps them only as a guard: if they come back replicated, fusion silently fell back and the test fails.
- `self_attn.inputs.hidden_states` — the value attention consumes after `attn_stage.prepare()`, i.e. post fused all-reduce and residual RMSNorm. This is replicated across ranks and is what the comparator actually diffs.
- `self_attn.inputs.hidden_states` — the value attention consumes after `attn_boundary.prepare()`, i.e. post fused all-reduce and residual RMSNorm. This is replicated across ranks and is what the comparator actually diffs.
This is why a stale baseline captured with fusion *not* initialized is incompatible: it holds replicated layer-entry tensors where the target holds TP-partial ones, which reads as an approximately `tp_size`x mismatch. Fusion initialization failures are tracked in [FlashInfer #3676](https://github.com/flashinfer-ai/flashinfer/issues/3676) and [SGLang #30875](https://github.com/sgl-project/sglang/issues/30875).
@@ -14,13 +14,13 @@ from sglang.srt.batch_overlap.operations import (
)
from sglang.srt.batch_overlap.operations_strategy import OperationsStrategy
from sglang.srt.layers import deep_gemm_wrapper
from sglang.srt.layers.communicator import (
from sglang.srt.layers.layer_boundary import (
Layout,
tbo_split_moves,
)
from sglang.srt.layers.communicator.residual import batch as residual_batch
from sglang.srt.layers.communicator.residual.access import finish_layer_stack
from sglang.srt.layers.communicator.residual.stream import ResidualStream
from sglang.srt.layers.layer_boundary.residual import batch as residual_batch
from sglang.srt.layers.layer_boundary.residual.access import finish_layer_stack
from sglang.srt.layers.layer_boundary.residual.stream import ResidualStream
from sglang.srt.layers.moe import (
get_deepep_mode,
get_moe_a2a_backend,
@@ -959,7 +959,7 @@ def model_forward_stages(
parts = _model_forward_tbo_split_inputs(
**inputs,
residual=residual,
layer_input_rows=layers[0].attn_stage.incoming_residual_rows,
layer_input_rows=layers[0].attn_boundary.incoming_residual_rows,
)
for part in parts:
part["hidden_states"], child_stream = ResidualStream.arrive(
@@ -15,7 +15,7 @@ from sglang.srt.layers.attention.dsa.dsa_npu_indexer import scattered_to_tp_attn
from sglang.srt.layers.attention.dsa.utils import (
dsa_use_prefill_cp,
)
from sglang.srt.layers.communicator import get_attn_tp_context
from sglang.srt.layers.layer_boundary import get_attn_tp_context
from sglang.srt.model_executor.forward_context import get_token_to_kv_pool
from sglang.srt.runtime_context import get_disagg
from sglang.srt.state_capturer.indexer_topk import maybe_capture_indexer_topk
@@ -13,20 +13,20 @@
# ==============================================================================
"""Stage boundaries: layout transport, residual updates, reads, and fusion."""
from sglang.srt.layers.communicator.adapters.attention import (
from sglang.srt.layers.layer_boundary.adapters.attention import (
AttentionInputs,
get_attn_tp_context,
)
from sglang.srt.layers.communicator.boundary import (
from sglang.srt.layers.layer_boundary.boundary import (
make_boundary,
make_output_boundary,
tbo_split_moves,
)
from sglang.srt.layers.communicator.construction import (
from sglang.srt.layers.layer_boundary.construction import (
BatchVariant,
StageEdges,
)
from sglang.srt.layers.communicator.contracts import (
from sglang.srt.layers.layer_boundary.contracts import (
EdgeDecl,
FusedMlpInput,
HandoffRows,
@@ -38,15 +38,15 @@ from sglang.srt.layers.communicator.contracts import (
StageOutput,
StageSteps,
)
from sglang.srt.layers.communicator.exit import FfnCompletion, FfnExit, MixerExit
from sglang.srt.layers.communicator.factories import (
from sglang.srt.layers.layer_boundary.exit import FfnCompletion, FfnExit, MixerExit
from sglang.srt.layers.layer_boundary.factories import (
declare_attn,
declare_ffn,
make_attn_stage,
make_ffn_stage,
make_stages,
)
from sglang.srt.layers.communicator.layout import (
from sglang.srt.layers.layer_boundary.layout import (
Layout,
SumGroup,
TokenAxis,
@@ -55,24 +55,24 @@ from sglang.srt.layers.communicator.layout import (
sparse_moe_gathers_over_moe_cp,
token_axis_sizes,
)
from sglang.srt.layers.communicator.ops import (
from sglang.srt.layers.layer_boundary.ops import (
move_rows,
tp_reduce_scatter,
)
from sglang.srt.layers.communicator.output import (
from sglang.srt.layers.layer_boundary.output import (
HandoffOutput,
UnreducedOutput,
reduce_output,
)
from sglang.srt.layers.communicator.residual import LayerResidual
from sglang.srt.layers.communicator.residual.add_norm import (
from sglang.srt.layers.layer_boundary.residual import LayerResidual
from sglang.srt.layers.layer_boundary.residual.add_norm import (
ADD,
FUSE_ALLREDUCE_MAX_BATCH_SIZE,
NORM_QUANT_READ,
NORM_READ,
PLAIN_RESIDUAL,
)
from sglang.srt.layers.communicator.residual.mhc import (
from sglang.srt.layers.layer_boundary.residual.mhc import (
MHCState,
)
@@ -20,14 +20,14 @@ from typing import Callable
import torch
from sglang.srt.layers.communicator.layout import (
enable_moe_dense_fully_dp,
)
from sglang.srt.layers.dp_attention import (
attn_tp_all_gather_into_tensor,
get_local_dp_buffer,
is_dp_attention_enabled,
)
from sglang.srt.layers.layer_boundary.layout import (
enable_moe_dense_fully_dp,
)
from sglang.srt.layers.moe import get_moe_a2a_backend
from sglang.srt.model_executor.cuda_graph_config import (
Backend,
@@ -19,12 +19,12 @@ from typing import TYPE_CHECKING, Tuple
import torch
from sglang.srt.layers.communicator.ops import move_rows
from sglang.srt.layers.communicator.residual.stream import ResidualStream
from sglang.srt.layers.layer_boundary.ops import move_rows
from sglang.srt.layers.layer_boundary.residual.stream import ResidualStream
from sglang.srt.model_executor.forward_batch_info import ForwardBatch
if TYPE_CHECKING:
from sglang.srt.layers.communicator.construction import StagePlan
from sglang.srt.layers.layer_boundary.construction import StagePlan
def branch_input(
@@ -13,7 +13,7 @@
# ==============================================================================
"""Publish the token rows consumed by the selected stage to LoRA kernels."""
from sglang.srt.layers.communicator.layout import TokenAxis
from sglang.srt.layers.layer_boundary.layout import TokenAxis
from sglang.srt.runtime_context import LoRABatchLayout, get_forward
@@ -1,7 +1,7 @@
"""Row requirements at the entrance to two-batch overlap."""
from sglang.srt.layers.communicator.contracts import HandoffRows
from sglang.srt.layers.communicator.layout import enable_moe_dense_fully_dp
from sglang.srt.layers.layer_boundary.contracts import HandoffRows
from sglang.srt.layers.layer_boundary.layout import enable_moe_dense_fully_dp
from sglang.srt.runtime_context import get_exec
@@ -19,14 +19,14 @@ from typing import Callable, Optional, Tuple
import msgspec
from sglang.srt.layers.communicator.contracts import (
from sglang.srt.layers.layer_boundary.contracts import (
CpMoves,
EdgeDecl,
FusedMlpInput,
StageInput,
StageOutput,
)
from sglang.srt.layers.communicator.layout import (
from sglang.srt.layers.layer_boundary.layout import (
Layout,
SumGroup,
TokenAxis,
@@ -35,7 +35,7 @@ from sglang.srt.layers.communicator.layout import (
_sum_group,
token_axis_sizes,
)
from sglang.srt.layers.communicator.ops import (
from sglang.srt.layers.layer_boundary.ops import (
gather_attention_tp,
identity_output,
move_rows,
@@ -49,7 +49,7 @@ from sglang.srt.layers.communicator.ops import (
tp_slice,
update_and_gather,
)
from sglang.srt.layers.communicator.prepare import (
from sglang.srt.layers.layer_boundary.prepare import (
_consumer_step,
_dispatch_consumer,
_mlp_input_dp_partial,
@@ -18,13 +18,13 @@ from functools import cached_property
from typing import Callable, Optional
from sglang.srt.environ import envs
from sglang.srt.layers.communicator.adapters.attention import get_attn_tp_context
from sglang.srt.layers.communicator.boundary import (
from sglang.srt.layers.layer_boundary.adapters.attention import get_attn_tp_context
from sglang.srt.layers.layer_boundary.boundary import (
_cp_moves,
make_boundary,
make_output_boundary,
)
from sglang.srt.layers.communicator.contracts import (
from sglang.srt.layers.layer_boundary.contracts import (
BatchVariant,
CpMoves,
EdgeDecl,
@@ -33,23 +33,23 @@ from sglang.srt.layers.communicator.contracts import (
StageKind,
StageSteps,
)
from sglang.srt.layers.communicator.exit import OutputBoundary
from sglang.srt.layers.communicator.fusions.allreduce import (
from sglang.srt.layers.layer_boundary.exit import OutputBoundary
from sglang.srt.layers.layer_boundary.fusions.allreduce import (
attention_fusions,
ffn_fusions,
)
from sglang.srt.layers.communicator.layout import (
from sglang.srt.layers.layer_boundary.layout import (
TokenAxis,
_batch_shards_over_cp,
_gathers_over_attention_cp,
_generic_prefill_cp_shards_tokens,
enable_moe_dense_fully_dp,
)
from sglang.srt.layers.communicator.prepare import (
from sglang.srt.layers.layer_boundary.prepare import (
_hand_qkv_hook_its_input,
_hand_scattered_input_to_attention,
)
from sglang.srt.layers.communicator.stage import StageCommunicator
from sglang.srt.layers.layer_boundary.stage import StageBoundary
from sglang.srt.layers.moe import (
get_moe_a2a_backend,
)
@@ -294,10 +294,10 @@ def _bind_stage(declaration, norm, incoming, outgoing, **options):
if declaration.kind is StageKind.ATTENTION and outgoing.consumer is not None:
# Layout eligibility comes from the connected consumer, not a mutable
# link to its execution plan. Kernel binding remains consumer-owned.
from sglang.srt.layers.communicator.boundary import input_rows
from sglang.srt.layers.layer_boundary.boundary import input_rows
plan._fusion_rows = {
v: input_rows(edge) for v, edge in outgoing.entries.items()
}
return StageCommunicator(plan, declaration=declaration)
return StageBoundary(plan, declaration=declaration)
@@ -20,10 +20,10 @@ import msgspec
import torch
from sglang.srt.distributed import GroupCoordinator
from sglang.srt.layers.communicator.layout import Layout, SumGroup, TokenAxis
from sglang.srt.layers.communicator.output import OutputTransform
from sglang.srt.layers.communicator.residual import StageRead, StageUpdate
from sglang.srt.layers.communicator.residual.add_norm import ADD, NORM_READ
from sglang.srt.layers.layer_boundary.layout import Layout, SumGroup, TokenAxis
from sglang.srt.layers.layer_boundary.output import OutputTransform
from sglang.srt.layers.layer_boundary.residual import StageRead, StageUpdate
from sglang.srt.layers.layer_boundary.residual.add_norm import ADD, NORM_READ
class ProducerReduction(Enum):
@@ -23,32 +23,32 @@ import torch
from sglang.srt.distributed import GroupCoordinator
from sglang.srt.environ import envs
from sglang.srt.layers.communicator.adapters.attention import get_attn_tp_context
from sglang.srt.layers.communicator.layout import (
from sglang.srt.layers.dp_attention import (
can_use_dp_reduce_scatter,
is_dp_attention_enabled,
is_enable_moe_cp_allgather,
)
from sglang.srt.layers.layer_boundary.adapters.attention import get_attn_tp_context
from sglang.srt.layers.layer_boundary.layout import (
SumGroup,
_ffn_has_tokens,
_sum_group,
)
from sglang.srt.layers.communicator.ops import (
from sglang.srt.layers.layer_boundary.ops import (
_all_reduce_then_to_local_tokens,
_redistribute_output,
_reduce_and_redistribute_output_max_len,
_reduce_and_redistribute_output_varlen,
_to_local_tokens,
)
from sglang.srt.layers.communicator.output import (
from sglang.srt.layers.layer_boundary.output import (
UnreducedOutput,
)
from sglang.srt.layers.communicator.residual.add_norm import (
from sglang.srt.layers.layer_boundary.residual.add_norm import (
apply_aiter_all_reduce_fusion,
apply_flashinfer_allreduce_fusion,
)
from sglang.srt.layers.communicator.residual.stream import ResidualStream
from sglang.srt.layers.dp_attention import (
can_use_dp_reduce_scatter,
is_dp_attention_enabled,
is_enable_moe_cp_allgather,
)
from sglang.srt.layers.layer_boundary.residual.stream import ResidualStream
from sglang.srt.layers.moe import (
can_merge_post_experts_all_reduce,
post_experts_sum_is_one_all_reduce,
@@ -9,19 +9,19 @@ from typing import Mapping, Optional
import msgspec
from sglang.srt.layers import layernorm_sp
from sglang.srt.layers.communicator.adapters.overlap import (
from sglang.srt.layers.layer_boundary.adapters.overlap import (
resolve_handoff_rows,
tbo_handoff,
)
from sglang.srt.layers.communicator.boundary import _cp_moves
from sglang.srt.layers.communicator.construction import (
from sglang.srt.layers.layer_boundary.boundary import _cp_moves
from sglang.srt.layers.layer_boundary.construction import (
BatchVariant,
_bind_stage,
_input_can_be_scattered,
_refuse_uncovered_cp_moe,
_use_ag_after_qlora,
)
from sglang.srt.layers.communicator.contracts import (
from sglang.srt.layers.layer_boundary.contracts import (
EdgeDecl,
HandoffRows,
ProducerReduction,
@@ -30,7 +30,7 @@ from sglang.srt.layers.communicator.contracts import (
StageKind,
StageOutput,
)
from sglang.srt.layers.communicator.layout import (
from sglang.srt.layers.layer_boundary.layout import (
Layout,
SumGroup,
TokenAxis,
@@ -40,9 +40,9 @@ from sglang.srt.layers.communicator.layout import (
sparse_moe_gathers_over_moe_cp,
token_axis_sizes,
)
from sglang.srt.layers.communicator.output import OutputTransform
from sglang.srt.layers.communicator.residual import StageRead, StageUpdate
from sglang.srt.layers.communicator.residual.add_norm import (
from sglang.srt.layers.layer_boundary.output import OutputTransform
from sglang.srt.layers.layer_boundary.residual import StageRead, StageUpdate
from sglang.srt.layers.layer_boundary.residual.add_norm import (
ADD,
NORM_QUANT_READ,
NORM_READ,
@@ -19,10 +19,10 @@ from typing import Callable, Optional, Tuple
import torch
from sglang.srt.layers.communicator.contracts import FusedMlpInput
from sglang.srt.layers.communicator.layout import SumGroup
from sglang.srt.layers.communicator.output import UnreducedOutput
from sglang.srt.layers.communicator.residual.add_norm import (
from sglang.srt.layers.layer_boundary.contracts import FusedMlpInput
from sglang.srt.layers.layer_boundary.layout import SumGroup
from sglang.srt.layers.layer_boundary.output import UnreducedOutput
from sglang.srt.layers.layer_boundary.residual.add_norm import (
NORM_QUANT_READ,
Fp8Input,
NormQuantRead,
@@ -13,7 +13,8 @@ from typing import Callable, Optional, Sequence
import torch
from sglang.srt.layers.communicator import (
from sglang.srt.layers.dp_attention import is_dp_attention_enabled
from sglang.srt.layers.layer_boundary import (
FusedMlpInput,
HandoffOutput,
SumGroup,
@@ -21,7 +22,6 @@ from sglang.srt.layers.communicator import (
UnreducedOutput,
get_attn_tp_context,
)
from sglang.srt.layers.dp_attention import is_dp_attention_enabled
from sglang.srt.layers.layernorm import GemmaRMSNorm, RMSNorm
from sglang.srt.layers.moe import get_moe_a2a_backend
from sglang.srt.model_executor.forward_batch_info import ForwardBatch, ForwardMode
@@ -359,7 +359,7 @@ class CuteDSLFusion:
def _fusion_of(layer: torch.nn.Module) -> CuteDSLFusion | None:
fusions = getattr(layer.__dict__.get("ffn_stage"), "fusions", None)
fusions = getattr(layer.__dict__.get("ffn_boundary"), "fusions", None)
return fusions if isinstance(fusions, CuteDSLFusion) else None
@@ -388,8 +388,8 @@ def install_cutedsl_fusion(
# The workspace compiles one epsilon and cannot see a per-layer one.
for layer in fusion_layers:
for norm in (
layer.attn_stage.norm,
layer.ffn_stage.norm,
layer.attn_boundary.norm,
layer.ffn_boundary.norm,
):
if _fused_norm_gamma(norm) is None:
continue
@@ -441,7 +441,7 @@ def prepare_cutedsl_fusion(
for module in model.modules()
# Only decoder modules carry an FFN stage.
if isinstance(
fusion := getattr(module.__dict__.get("ffn_stage"), "fusions", None),
fusion := getattr(module.__dict__.get("ffn_boundary"), "fusions", None),
CuteDSLFusion,
)
]
@@ -26,21 +26,6 @@ from sglang.srt.distributed import (
from sglang.srt.distributed.device_communicators.pynccl_allocator import (
use_symmetric_memory,
)
from sglang.srt.layers.communicator.adapters.attention import (
_redistribute_from_attn_tp_shards,
_redistribute_to_attn_tp_shards,
)
from sglang.srt.layers.communicator.adapters.context_parallel import (
dsa_cp_reduce_scatter_hidden_states,
)
from sglang.srt.layers.communicator.layout import (
Layout,
TokenAxis,
_cp_shard_token_rows,
moe_cp_gathered_rows,
)
from sglang.srt.layers.communicator.residual import StageUpdate
from sglang.srt.layers.communicator.residual.add_norm import ADD
from sglang.srt.layers.dp_attention import (
attn_tp_all_gather_into_tensor,
attn_tp_reduce_scatter_tensor,
@@ -55,6 +40,21 @@ from sglang.srt.layers.dp_attention import (
is_allocation_symmetric,
moe_cp_all_gather_into_tensor,
)
from sglang.srt.layers.layer_boundary.adapters.attention import (
_redistribute_from_attn_tp_shards,
_redistribute_to_attn_tp_shards,
)
from sglang.srt.layers.layer_boundary.adapters.context_parallel import (
dsa_cp_reduce_scatter_hidden_states,
)
from sglang.srt.layers.layer_boundary.layout import (
Layout,
TokenAxis,
_cp_shard_token_rows,
moe_cp_gathered_rows,
)
from sglang.srt.layers.layer_boundary.residual import StageUpdate
from sglang.srt.layers.layer_boundary.residual.add_norm import ADD
from sglang.srt.model_executor.forward_batch_info import ForwardBatch
from sglang.srt.runtime_context import get_exec, get_parallel
@@ -24,34 +24,34 @@ from sglang.srt.distributed import (
from sglang.srt.distributed.device_communicators.pynccl_allocator import (
use_symmetric_memory,
)
from sglang.srt.layers.communicator.adapters.attention import (
AttentionInputs,
_redistribute_from_attn_tp_shards,
_redistribute_to_attn_tp_shards,
get_attn_tp_context,
)
from sglang.srt.layers.communicator.adapters.context_parallel import (
dsa_cp_gather_hidden_states,
)
from sglang.srt.layers.communicator.layout import (
SumGroup,
_cp_shard_token_rows,
moe_cp_gathered_rows,
)
from sglang.srt.layers.communicator.output import (
HandoffOutput,
UnreducedOutput,
reduce_output,
)
from sglang.srt.layers.communicator.residual import StageRead, StageUpdate
from sglang.srt.layers.communicator.residual.add_norm import ADD, NORM_READ
from sglang.srt.layers.communicator.residual.stream import DeclaredSum
from sglang.srt.layers.dp_attention import (
attn_tp_all_gather_into_tensor,
dp_scatter,
get_moe_cp_size,
is_allocation_symmetric,
)
from sglang.srt.layers.layer_boundary.adapters.attention import (
AttentionInputs,
_redistribute_from_attn_tp_shards,
_redistribute_to_attn_tp_shards,
get_attn_tp_context,
)
from sglang.srt.layers.layer_boundary.adapters.context_parallel import (
dsa_cp_gather_hidden_states,
)
from sglang.srt.layers.layer_boundary.layout import (
SumGroup,
_cp_shard_token_rows,
moe_cp_gathered_rows,
)
from sglang.srt.layers.layer_boundary.output import (
HandoffOutput,
UnreducedOutput,
reduce_output,
)
from sglang.srt.layers.layer_boundary.residual import StageRead, StageUpdate
from sglang.srt.layers.layer_boundary.residual.add_norm import ADD, NORM_READ
from sglang.srt.layers.layer_boundary.residual.stream import DeclaredSum
from sglang.srt.model_executor.forward_batch_info import ForwardBatch
from sglang.srt.runtime_context import get_parallel
from sglang.srt.utils import (
@@ -64,7 +64,7 @@ _is_npu = is_npu()
if _is_npu:
from sglang.srt.hardware_backend.npu.cmo import prepare_weight_cache
from sglang.srt.layers.communicator.ops import (
from sglang.srt.layers.layer_boundary.ops import (
_mlp_input_reduce_output,
_redistribute_input_to_dp,
_redistribute_input_to_moe_cp,
@@ -18,12 +18,12 @@ from typing import Optional, Tuple, Union
import torch
from sglang.srt.distributed import GroupCoordinator
from sglang.srt.layers.communicator.output import (
from sglang.srt.layers.layer_boundary.output import (
HandoffOutput,
UnreducedOutput,
reduce_output,
)
from sglang.srt.layers.communicator.residual.stream import OwedOutput, ResidualStream
from sglang.srt.layers.layer_boundary.residual.stream import OwedOutput, ResidualStream
from sglang.srt.model_executor.forward_batch_info import ForwardBatch, PPProxyTensors
@@ -20,17 +20,17 @@ from typing import Optional
import torch
from sglang.srt.layers.communicator.adapters.attention import (
_redistribute_from_attn_tp_shards,
_redistribute_to_attn_tp_shards,
get_attn_tp_context,
)
from sglang.srt.layers.communicator.residual import LayerResidual
from sglang.srt.layers.dp_attention import is_dp_attention_enabled
from sglang.srt.layers.flashinfer_comm_fusion import (
is_flashinfer_allreduce_unavailable,
uses_cutedsl_ar_fusion,
)
from sglang.srt.layers.layer_boundary.adapters.attention import (
_redistribute_from_attn_tp_shards,
_redistribute_to_attn_tp_shards,
get_attn_tp_context,
)
from sglang.srt.layers.layer_boundary.residual import LayerResidual
from sglang.srt.layers.quantization.fp8_utils import (
_use_aiter_bpreshuffle_gfx95,
materialize_bpreshuffle_fp8_scale_tuple,
@@ -13,8 +13,8 @@
# ==============================================================================
"""Residual access for one layer-stack invocation or TBO microbatch."""
from sglang.srt.layers.communicator.residual import access
from sglang.srt.layers.communicator.residual.stream import ResidualStream
from sglang.srt.layers.layer_boundary.residual import access
from sglang.srt.layers.layer_boundary.residual.stream import ResidualStream
from sglang.srt.model_executor.forward_batch_info import PPProxyTensors
@@ -60,7 +60,7 @@ def norm(
# The terminal consumer now owns the pair. Do not keep layer buffers alive
# through logits processing or the next forward on this batch.
forward_batch.residual_stream = None
from sglang.srt.layers.communicator.output import HandoffOutput
from sglang.srt.layers.layer_boundary.output import HandoffOutput
if isinstance(hidden_states, HandoffOutput):
if residual is None:
@@ -19,7 +19,7 @@ from typing import Callable, Optional
import torch
from sglang.kernels.ops.layernorm.mhc import hc_contract, hc_expand
from sglang.srt.layers.communicator.residual import LayerResidual
from sglang.srt.layers.layer_boundary.residual import LayerResidual
from sglang.srt.runtime_context import get_parallel
@@ -19,9 +19,9 @@ import msgspec
import torch
from sglang.srt.distributed import GroupCoordinator
from sglang.srt.layers.communicator.layout import SumGroup, _sum_group
from sglang.srt.layers.communicator.output import HandoffOutput, UnreducedOutput
from sglang.srt.layers.communicator.residual import StageUpdate
from sglang.srt.layers.layer_boundary.layout import SumGroup, _sum_group
from sglang.srt.layers.layer_boundary.output import HandoffOutput, UnreducedOutput
from sglang.srt.layers.layer_boundary.residual import StageUpdate
class CarriedSum(msgspec.Struct, frozen=True):
@@ -19,33 +19,33 @@ import torch
from sglang.srt.layers import layernorm_sp
from sglang.srt.layers.aux_hidden_states import AuxHiddenStateAccumulator
from sglang.srt.layers.communicator.adapters import branch
from sglang.srt.layers.communicator.adapters.lora import (
from sglang.srt.layers.layer_boundary.adapters import branch
from sglang.srt.layers.layer_boundary.adapters.lora import (
publish_attention,
publish_ffn,
)
from sglang.srt.layers.communicator.contracts import (
from sglang.srt.layers.layer_boundary.contracts import (
BatchVariant,
StageEntry,
StageKind,
StageSteps,
)
from sglang.srt.layers.communicator.ops import gather_attention_tp
from sglang.srt.layers.communicator.residual.access import buffer, from_pp
from sglang.srt.layers.communicator.residual.add_norm import (
from sglang.srt.layers.layer_boundary.ops import gather_attention_tp
from sglang.srt.layers.layer_boundary.residual.access import buffer, from_pp
from sglang.srt.layers.layer_boundary.residual.add_norm import (
ADD,
)
from sglang.srt.layers.communicator.residual.batch import current
from sglang.srt.layers.communicator.residual.stream import DeclaredSum, ResidualStream
from sglang.srt.layers.layer_boundary.residual.batch import current
from sglang.srt.layers.layer_boundary.residual.stream import DeclaredSum, ResidualStream
from sglang.srt.model_executor.forward_batch_info import ForwardBatch
from sglang.srt.runtime_context import get_forward
if TYPE_CHECKING:
from sglang.srt.layers.communicator.construction import StagePlan
from sglang.srt.layers.communicator.factories import StageDeclaration
from sglang.srt.layers.layer_boundary.construction import StagePlan
from sglang.srt.layers.layer_boundary.factories import StageDeclaration
class StageCommunicator:
class StageBoundary:
"""One stage of a layer, its attention or its FFN: the boundary into it,
run with the stage's norm on the steps the layer chose for the batch."""
@@ -21,12 +21,12 @@ from sglang.srt.distributed.device_communicators.pynccl_allocator import (
)
from sglang.srt.environ import envs
from sglang.srt.layers.amx_utils import PackWeightMethod
from sglang.srt.layers.communicator import get_attn_tp_context
from sglang.srt.layers.dp_attention import (
attn_tp_all_reduce,
is_allocation_symmetric,
is_dp_attention_enabled,
)
from sglang.srt.layers.layer_boundary import get_attn_tp_context
from sglang.srt.layers.parameter import BasevLLMParameter
from sglang.srt.layers.quantization.base_config import (
QuantizationConfig,
+2 -2
View File
@@ -119,7 +119,7 @@ class VocabParallelEmbeddingWithLoRA(BaseLayerWithLoRA):
# all-reduce, making the unsharded LoRA approach mathematically
# incorrect — a sharded LoRA kernel would be needed.
if hasattr(base_layer, "tp_size") and base_layer.tp_size > 1:
from sglang.srt.layers.communicator import get_attn_tp_context
from sglang.srt.layers.layer_boundary import get_attn_tp_context
assert not get_attn_tp_context().allow_input_scattered, (
"VocabParallelEmbeddingWithLoRA with TP > 1 under input_scattered mode (e.g., DeepSeek-v2 MLA with --enable-attn-tp-input-scattered) is not fully supported and may produce incorrect results. Consider disabling input_scattered or removing embed_tokens from LoRA target modules."
@@ -295,7 +295,7 @@ class ParallelLMHeadWithLoRA(BaseLayerWithLoRA):
# incompatible with input_scattered mode where the all-reduce is
# skipped.
if tp_size > 1:
from sglang.srt.layers.communicator import get_attn_tp_context
from sglang.srt.layers.layer_boundary import get_attn_tp_context
if get_attn_tp_context().allow_input_scattered:
raise ValueError(
@@ -71,9 +71,9 @@ from sglang.srt.utils.common import ceil_align, is_pin_memory_available
if TYPE_CHECKING:
from sglang.srt.configs.model_config import ModelConfig
from sglang.srt.layers.communicator.residual.stream import ResidualStream
from sglang.srt.layers.cp.base import BaseContextParallelMetadata
from sglang.srt.layers.dcp.metadata import DecodeContextParallelMetadata
from sglang.srt.layers.layer_boundary.residual.stream import ResidualStream
from sglang.srt.layers.logits_processor import LogitsProcessorOutput
from sglang.srt.managers.schedule_batch import MultimodalInputs, ScheduleBatch
from sglang.srt.model_executor.model_runner import ModelRunner
@@ -1257,7 +1257,7 @@ class ForwardBatch(ForwardBatchDeepSeekMHAMixin):
def moe_num_token_non_padded(self) -> Optional[torch.Tensor]:
"""Bound for masking a sparse MoE's padded rows, or None when the MoE
input is a gathered buffer whose real rows are not a prefix of it."""
from sglang.srt.layers.communicator import moe_cp_gathers_sparse_moe_input
from sglang.srt.layers.layer_boundary import moe_cp_gathers_sparse_moe_input
from sglang.srt.layers.moe.utils import is_moe_input_scattered_across_dp_ranks
if self.num_token_non_padded is None:
@@ -1916,7 +1916,7 @@ class ForwardBatch(ForwardBatchDeepSeekMHAMixin):
return self.input_ids.shape[0]
def prepare_attn_tp_scatter_input(self, model_runner: ModelRunner):
from sglang.srt.layers.communicator import get_attn_tp_context
from sglang.srt.layers.layer_boundary import get_attn_tp_context
# Pure TP+SP has no MLP-sync pass, so stamp the decision here.
self.attn_tp_sequence_sharded = model_runner.attn_tp_sequence_sharded(
@@ -288,12 +288,12 @@ class BaseRunner(ABC):
with custom_all_reduce.register_graph_buffers).
"""
mr = self.model_runner
from sglang.srt.layers.communicator import FUSE_ALLREDUCE_MAX_BATCH_SIZE
from sglang.srt.layers.flashinfer_comm_fusion import (
pre_initialize_workspaces,
resolve_flashinfer_allreduce_fusion_backend,
uses_cutedsl_ar_fusion,
)
from sglang.srt.layers.layer_boundary import FUSE_ALLREDUCE_MAX_BATCH_SIZE
if get_exec().comm.flashinfer_allreduce_fusion_backend is None:
return
@@ -303,7 +303,7 @@ class BaseRunner(ABC):
resolve_flashinfer_allreduce_fusion_backend()
if not mr.is_draft_worker:
# A draft installs no fusion communicator.
from sglang.srt.layers.communicator.fusions.cutedsl import (
from sglang.srt.layers.layer_boundary.fusions.cutedsl import (
prepare_cutedsl_fusion,
)
+11 -11
View File
@@ -35,16 +35,16 @@ from sglang.srt.layers.aux_hidden_states import (
AuxHiddenStateAccumulator,
AuxHiddenStateList,
)
from sglang.srt.layers.communicator import (
from sglang.srt.layers.dp_attention import (
is_dp_attention_enabled,
)
from sglang.srt.layers.layer_boundary import (
declare_attn,
declare_ffn,
enable_moe_dense_fully_dp,
make_stages,
)
from sglang.srt.layers.communicator.residual import batch as residual_batch
from sglang.srt.layers.dp_attention import (
is_dp_attention_enabled,
)
from sglang.srt.layers.layer_boundary.residual import batch as residual_batch
from sglang.srt.layers.layernorm import RMSNorm
from sglang.srt.layers.linear import (
MergedColumnParallelLinear,
@@ -698,7 +698,7 @@ class BailingMoEBlock(nn.Module):
self.post_attention_layernorm = RMSNorm(hidden_size, eps=config.rms_norm_eps)
self.attn_stage, self.ffn_stage = make_stages(
self.attn_boundary, self.ffn_boundary = make_stages(
(declare_attn(), self.input_layernorm),
(
declare_ffn(
@@ -730,7 +730,7 @@ class BailingMoEBlock(nn.Module):
captured_last_layer_outputs: Optional[AuxHiddenStateAccumulator] = None,
capture_output=None,
) -> torch.Tensor:
hidden_states = self.attn_stage.prepare(
hidden_states = self.attn_boundary.prepare(
hidden_states,
forward_batch,
captured_last_layer_outputs=captured_last_layer_outputs,
@@ -744,13 +744,13 @@ class BailingMoEBlock(nn.Module):
forward_batch=forward_batch,
)
hidden_states = self.attn_stage.finish(hidden_states, forward_batch)
hidden_states = self.ffn_stage.prepare(
hidden_states = self.attn_boundary.finish(hidden_states, forward_batch)
hidden_states = self.ffn_boundary.prepare(
hidden_states=hidden_states,
forward_batch=forward_batch,
)
with self.ffn_stage.exit(forward_batch) as ffn_exit:
with self.ffn_boundary.exit(forward_batch) as ffn_exit:
hidden_states = self.mlp(hidden_states, forward_batch)
hidden_states = ffn_exit.finish(hidden_states)
@@ -822,7 +822,7 @@ class BailingMoEModel(nn.Module):
residual_batch.start(forward_batch)
else:
assert pp_proxy_tensors is not None
hidden_states = self.layers[self.start_layer].attn_stage.from_pp(
hidden_states = self.layers[self.start_layer].attn_boundary.from_pp(
pp_proxy_tensors, forward_batch
)
+11 -11
View File
@@ -15,15 +15,15 @@ from sglang.kernels.ops.quantization.fp8_kernel import is_fp8_fnuz
from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder
from sglang.srt.layers import deep_gemm_wrapper
from sglang.srt.layers.activation import SiluAndMul
from sglang.srt.layers.communicator import (
from sglang.srt.layers.dp_attention import (
is_dp_attention_enabled,
)
from sglang.srt.layers.layer_boundary import (
declare_attn,
declare_ffn,
make_stages,
)
from sglang.srt.layers.communicator.residual import batch as residual_batch
from sglang.srt.layers.dp_attention import (
is_dp_attention_enabled,
)
from sglang.srt.layers.layer_boundary.residual import batch as residual_batch
from sglang.srt.layers.layernorm import RMSNorm
from sglang.srt.layers.linear import (
ColumnParallelLinear,
@@ -820,7 +820,7 @@ class BailingMoELinearDecoderLayer(nn.Module):
if config.attention_type == 1 and self.use_mla
else None
)
self.attn_stage, self.ffn_stage = make_stages(
self.attn_boundary, self.ffn_boundary = make_stages(
(
declare_attn(),
self.input_layernorm,
@@ -854,7 +854,7 @@ class BailingMoELinearDecoderLayer(nn.Module):
zero_allocator: BumpAllocator,
**kwargs,
) -> torch.Tensor:
hidden_states = self.attn_stage.prepare(hidden_states, forward_batch)
hidden_states = self.attn_boundary.prepare(hidden_states, forward_batch)
# logger.warning(
# f"===={self.layer_id=}, 1 shape= {hidden_states.shape}, {residual.shape}"
# )
@@ -875,12 +875,12 @@ class BailingMoELinearDecoderLayer(nn.Module):
# logger.warning(
# f"===={self.layer_id=}, 2 shape= {hidden_states.shape}, {residual.shape}"
# )
hidden_states = self.attn_stage.finish(hidden_states, forward_batch)
hidden_states = self.ffn_stage.prepare(hidden_states, forward_batch)
hidden_states = self.attn_boundary.finish(hidden_states, forward_batch)
hidden_states = self.ffn_boundary.prepare(hidden_states, forward_batch)
# logger.warning(
# f"===={self.layer_id=}, 3 shape= {hidden_states.shape}, {residual.shape}"
# )
with self.ffn_stage.exit(forward_batch) as ffn_exit:
with self.ffn_boundary.exit(forward_batch) as ffn_exit:
hidden_states = self.mlp(hidden_states)
hidden_states = ffn_exit.finish(hidden_states)
return hidden_states
@@ -983,7 +983,7 @@ class BailingMoELinearModel(nn.Module):
residual_batch.start(forward_batch)
else:
assert pp_proxy_tensors is not None
hidden_states = self.layers[self.start_layer].attn_stage.from_pp(
hidden_states = self.layers[self.start_layer].attn_boundary.from_pp(
pp_proxy_tensors, forward_batch
)
@@ -26,8 +26,8 @@ import torch
from torch import nn
from transformers import PretrainedConfig
from sglang.srt.layers.communicator.residual import batch as residual_batch
from sglang.srt.layers.dp_attention import is_dp_attention_enabled
from sglang.srt.layers.layer_boundary.residual import batch as residual_batch
from sglang.srt.layers.layernorm import RMSNorm
from sglang.srt.layers.linear import ReplicatedLinear
from sglang.srt.layers.logits_processor import LogitsProcessor
+9 -9
View File
@@ -29,14 +29,14 @@ from sglang.srt.eplb.expert_location_dispatch import ExpertLocationDispatchInfo
from sglang.srt.layers import deep_gemm_wrapper
from sglang.srt.layers.activation import SiluAndMul
from sglang.srt.layers.aux_hidden_states import AuxHiddenStateList
from sglang.srt.layers.communicator import (
from sglang.srt.layers.dp_attention import is_dp_attention_enabled
from sglang.srt.layers.layer_boundary import (
declare_attn,
declare_ffn,
enable_moe_dense_fully_dp,
make_stages,
)
from sglang.srt.layers.communicator.residual import batch as residual_batch
from sglang.srt.layers.dp_attention import is_dp_attention_enabled
from sglang.srt.layers.layer_boundary.residual import batch as residual_batch
from sglang.srt.layers.layernorm import RMSNorm
from sglang.srt.layers.linear import (
ColumnParallelLinear,
@@ -1149,7 +1149,7 @@ class BailingMoELinearDecoderLayer(nn.Module):
self.input_layernorm = RMSNorm(self.hidden_size, eps=rms_norm_eps)
self.post_attention_layernorm = RMSNorm(self.hidden_size, eps=rms_norm_eps)
self.attn_stage, self.ffn_stage = make_stages(
self.attn_boundary, self.ffn_boundary = make_stages(
(
declare_attn(),
self.input_layernorm,
@@ -1182,7 +1182,7 @@ class BailingMoELinearDecoderLayer(nn.Module):
capture_output=None,
**kwargs,
) -> torch.Tensor:
hidden_states = self.attn_stage.prepare(
hidden_states = self.attn_boundary.prepare(
hidden_states, forward_batch, capture_output=capture_output
)
@@ -1208,10 +1208,10 @@ class BailingMoELinearDecoderLayer(nn.Module):
forward_batch=forward_batch,
)
hidden_states = self.attn_stage.finish(hidden_states, forward_batch)
hidden_states = self.ffn_stage.prepare(hidden_states, forward_batch)
hidden_states = self.attn_boundary.finish(hidden_states, forward_batch)
hidden_states = self.ffn_boundary.prepare(hidden_states, forward_batch)
with self.ffn_stage.exit(forward_batch) as ffn_exit:
with self.ffn_boundary.exit(forward_batch) as ffn_exit:
if not (
enable_moe_dense_fully_dp()
and (not self.is_layer_sparse)
@@ -1344,7 +1344,7 @@ class BailingMoELinearModel(nn.Module):
residual_batch.start(forward_batch)
else:
assert pp_proxy_tensors is not None
hidden_states = self.layers[self.start_layer].attn_stage.from_pp(
hidden_states = self.layers[self.start_layer].attn_boundary.from_pp(
pp_proxy_tensors, forward_batch
)
@@ -8,12 +8,12 @@ from sglang.kernels.ops.attention.dsa.dequant_k_cache import dequantize_k_cache_
from sglang.kernels.ops.attention.utils import concat_and_cast_mha_k_triton
from sglang.srt.environ import envs
from sglang.srt.layers.attention.tbo_backend import TboAttnBackend
from sglang.srt.layers.communicator import get_attn_tp_context
from sglang.srt.layers.dcp import (
all_gather_kv_cache_for_mha_chunk_extend,
all_gather_kv_cache_for_mha_extend,
filter_dcp_local_kv_indices,
)
from sglang.srt.layers.layer_boundary import get_attn_tp_context
from sglang.srt.model_executor.forward_batch_info import ForwardBatch
from sglang.srt.model_executor.forward_context import (
get_attn_backend,
@@ -13,8 +13,8 @@ from typing import TYPE_CHECKING
import torch
from sglang.kernels.ops.attention.utils import concat_and_cast_mha_k_triton
from sglang.srt.layers.communicator import get_attn_tp_context
from sglang.srt.layers.dcp import all_gather_kv_cache_for_mha_extend
from sglang.srt.layers.layer_boundary import get_attn_tp_context
from sglang.srt.layers.quantization.fp8_utils import (
materialize_bpreshuffle_fp8_scale_tuple,
)
@@ -16,13 +16,13 @@ from sglang.srt.environ import envs
from sglang.srt.layers import deep_gemm_wrapper
from sglang.srt.layers.attention.dsa.utils import is_graph_dsa_split_op_surface
from sglang.srt.layers.attention.dsa_backend import prepare_kv_for_attention
from sglang.srt.layers.communicator import get_attn_tp_context
from sglang.srt.layers.dcp import (
all_gather_kv_cache_for_mla_extend,
all_gather_q_for_mla_decode,
cp_lse_ag_out_rs_mla,
dcp_a2a_lse_reduce,
)
from sglang.srt.layers.layer_boundary import get_attn_tp_context
from sglang.srt.layers.logits_processor import get_in_autotune_dummy_run
from sglang.srt.layers.radix_attention import unified_attention_with_output
from sglang.srt.lora.deepseek_mla_correction import (
@@ -20,13 +20,13 @@ from sglang.kernels.ops.quantization.fp8_kernel import (
)
from sglang.srt.environ import envs
from sglang.srt.layers import deep_gemm_wrapper
from sglang.srt.layers.communicator import get_attn_tp_context
from sglang.srt.layers.dcp import (
all_gather_kv_cache_for_mla_extend,
all_gather_q_for_mla_decode,
cp_lse_ag_out_rs_mla,
dcp_a2a_lse_reduce,
)
from sglang.srt.layers.layer_boundary import get_attn_tp_context
from sglang.srt.layers.logits_processor import get_in_autotune_dummy_run
from sglang.srt.layers.quantization.fp8_utils import (
emit_transposed_bpreshuffle_scale,
+1 -1
View File
@@ -28,7 +28,7 @@ from sglang.kernels.ops.layernorm.fused_eh_norm import fused_eh_norm
from sglang.srt.environ import envs
from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder
from sglang.srt.layers.attention.index_topk_share import IndexTopKShareState
from sglang.srt.layers.communicator.residual import batch as residual_batch
from sglang.srt.layers.layer_boundary.residual import batch as residual_batch
from sglang.srt.layers.layernorm import RMSNorm
from sglang.srt.layers.linear import ReplicatedLinear
from sglang.srt.layers.logits_processor import LogitsProcessor
+25 -25
View File
@@ -67,23 +67,23 @@ from sglang.srt.layers.aux_hidden_states import (
AuxHiddenStateAccumulator,
AuxHiddenStatePacker,
)
from sglang.srt.layers.communicator import (
from sglang.srt.layers.cp.cp_decode_attn_tp import get_cp_decode_attn_tp_ctx
from sglang.srt.layers.dcp.planner import (
prepare_decode_context_parallel_metadata,
)
from sglang.srt.layers.flashinfer_comm_fusion import uses_cutedsl_ar_fusion
from sglang.srt.layers.layer_boundary import (
declare_attn,
declare_ffn,
enable_moe_dense_fully_dp,
get_attn_tp_context,
make_stages,
)
from sglang.srt.layers.communicator.adapters.context_parallel import (
from sglang.srt.layers.layer_boundary.adapters.context_parallel import (
maybe_prefetch_next_full_attention_kv,
)
from sglang.srt.layers.communicator.residual import access as residual_access
from sglang.srt.layers.communicator.residual import batch as residual_batch
from sglang.srt.layers.cp.cp_decode_attn_tp import get_cp_decode_attn_tp_ctx
from sglang.srt.layers.dcp.planner import (
prepare_decode_context_parallel_metadata,
)
from sglang.srt.layers.flashinfer_comm_fusion import uses_cutedsl_ar_fusion
from sglang.srt.layers.layer_boundary.residual import access as residual_access
from sglang.srt.layers.layer_boundary.residual import batch as residual_batch
from sglang.srt.layers.layernorm import RMSNorm
from sglang.srt.layers.linear import (
ColumnParallelLinear,
@@ -1109,7 +1109,7 @@ class DeepseekV2MoE(nn.Module):
if deferred_finalize and get_forward().defer_moe_finalize:
# deferred_finalize excludes _shared_expert_tp1, so the shared add folds in.
assert shared_output is not None
from sglang.srt.layers.communicator.fusions.cutedsl import (
from sglang.srt.layers.layer_boundary.fusions.cutedsl import (
MoeFinalizeHandoff,
)
@@ -2619,7 +2619,7 @@ class DeepseekV2DecoderLayer(nn.Module):
self._gfx95_quant_format = self._detect_gfx95_quant_format()
self.attn_stage, self.ffn_stage = self._build_stages(
self.attn_boundary, self.ffn_boundary = self._build_stages(
input_layernorm=self.input_layernorm,
post_attention_layernorm=self.post_attention_layernorm,
qkv_latent_func=self.self_attn.prepare_qkv_latent,
@@ -2642,10 +2642,10 @@ class DeepseekV2DecoderLayer(nn.Module):
and _use_mnnvl_cutedsl_fusion()
):
# Dense layers too: selecting cutedsl turns the legacy fusion off.
from sglang.srt.layers.communicator.fusions.cutedsl import CuteDSLFusion
from sglang.srt.layers.layer_boundary.fusions.cutedsl import CuteDSLFusion
fusions = CuteDSLFusion()
attn_stage, ffn_stage = make_stages(
attn_boundary, ffn_boundary = make_stages(
(
declare_attn(),
input_layernorm,
@@ -2667,7 +2667,7 @@ class DeepseekV2DecoderLayer(nn.Module):
else None,
terminal=self._stage_terminal,
)
return attn_stage, ffn_stage
return attn_boundary, ffn_boundary
def _detect_gfx95_quant_format(self) -> str:
if not _is_gfx95_supported:
@@ -2721,7 +2721,7 @@ class DeepseekV2DecoderLayer(nn.Module):
next_full_attention_layer_id: Optional[int] = None,
) -> torch.Tensor:
hidden_states_orig = residual_access.buffer(hidden_states)
hidden_states = self.attn_stage.prepare(
hidden_states = self.attn_boundary.prepare(
hidden_states,
forward_batch,
captured_last_layer_outputs=captured_last_layer_outputs,
@@ -2736,7 +2736,7 @@ class DeepseekV2DecoderLayer(nn.Module):
zero_allocator=zero_allocator,
llama_4_scaling=llama_4_scaling,
input_on_attention_tp_slices=(
self.attn_stage.input_on_attention_tp_slices
self.attn_boundary.input_on_attention_tp_slices
),
prev_topk_indices=prev_topk_indices,
)
@@ -2750,8 +2750,8 @@ class DeepseekV2DecoderLayer(nn.Module):
forward_batch, next_full_attention_layer_id
)
hidden_states = self.attn_stage.finish(hidden_states, forward_batch)
hidden_states = self.ffn_stage.prepare(hidden_states, forward_batch)
hidden_states = self.attn_boundary.finish(hidden_states, forward_batch)
hidden_states = self.ffn_boundary.prepare(hidden_states, forward_batch)
if isinstance(self.mlp, DeepseekV2MLP):
gemm_output_zero_allocator = None
@@ -2769,7 +2769,7 @@ class DeepseekV2DecoderLayer(nn.Module):
else:
_mlp_ctx = nullcontext()
with self.ffn_stage.exit(forward_batch) as ffn_exit, _mlp_ctx:
with self.ffn_boundary.exit(forward_batch) as ffn_exit, _mlp_ctx:
hidden_states = self.mlp(
hidden_states,
forward_batch,
@@ -2788,7 +2788,7 @@ class DeepseekV2DecoderLayer(nn.Module):
zero_allocator: BumpAllocator,
tbo_subbatch_index: Optional[int] = None,
):
state.hidden_states_after_comm_pre_attn = self.attn_stage.prepare(
state.hidden_states_after_comm_pre_attn = self.attn_boundary.prepare(
hidden_states, forward_batch
)
if get_moe_a2a_backend().is_mori():
@@ -2803,15 +2803,15 @@ class DeepseekV2DecoderLayer(nn.Module):
)
def op_comm_prepare_mlp(self, state):
hidden_states = self.attn_stage.finish(
hidden_states = self.attn_boundary.finish(
state.pop("hidden_states_after_attn"), state.forward_batch
)
state.hidden_states_mlp_input = self.ffn_stage.prepare(
state.hidden_states_mlp_input = self.ffn_boundary.prepare(
hidden_states, state.forward_batch
)
def op_comm_postprocess_layer(self, state):
hidden_states = self.ffn_stage.postprocess(
hidden_states = self.ffn_boundary.postprocess(
state.pop("hidden_states_mlp_output"), state.forward_batch
)
@@ -2978,7 +2978,7 @@ class DeepseekV2Model(nn.Module):
# llama_4_scaling: for supporting Mistral-Large-3 model
self.llama_4_scaling_config = getattr(config, "llama_4_scaling", None)
from sglang.srt.layers.communicator.fusions.cutedsl import (
from sglang.srt.layers.layer_boundary.fusions.cutedsl import (
install_cutedsl_fusion,
)
@@ -3030,7 +3030,7 @@ class DeepseekV2Model(nn.Module):
initial_topk_indices = None
else:
assert pp_proxy_tensors is not None
hidden_states = self.layers[self.start_layer].attn_stage.from_pp(
hidden_states = self.layers[self.start_layer].attn_boundary.from_pp(
pp_proxy_tensors, forward_batch
)
initial_topk_indices = pp_proxy_tensors.tensors.get("topk_indices")
+5 -5
View File
@@ -70,11 +70,6 @@ from sglang.srt.layers.attention.dsv4.dsv41_sparse import (
DeepseekV41Indexer,
)
from sglang.srt.layers.attention.dsv4.indexer import C4Indexer
from sglang.srt.layers.communicator import get_attn_tp_context
from sglang.srt.layers.communicator.adapters.context_parallel import (
dsa_cp_gather_hidden_states,
dsa_cp_reduce_scatter_hidden_states,
)
from sglang.srt.layers.cp.cp_decode_attn_tp import get_cp_decode_attn_tp_ctx
from sglang.srt.layers.cp.utils import (
cp_gather_full_sequence_states,
@@ -103,6 +98,11 @@ from sglang.srt.layers.dp_attention import (
is_dp_gatherv_active,
)
from sglang.srt.layers.engram import Engram, EngramHasher, EngramLayout
from sglang.srt.layers.layer_boundary import get_attn_tp_context
from sglang.srt.layers.layer_boundary.adapters.context_parallel import (
dsa_cp_gather_hidden_states,
dsa_cp_reduce_scatter_hidden_states,
)
from sglang.srt.layers.layernorm import RMSNorm
from sglang.srt.layers.linear import ColumnParallelLinear, RowParallelLinear
from sglang.srt.layers.logits_processor import LogitsMetadata, LogitsProcessor
@@ -57,14 +57,14 @@ from sglang.srt.eplb.expert_location_dispatch import ExpertLocationDispatchInfo
from sglang.srt.layers import deep_gemm_wrapper
from sglang.srt.layers.activation import SiluAndMul
from sglang.srt.layers.attention.dsa.dsa_indexer import Indexer
from sglang.srt.layers.communicator import (
from sglang.srt.layers.dp_attention import is_dp_attention_enabled
from sglang.srt.layers.layer_boundary import (
declare_attn,
declare_ffn,
enable_moe_dense_fully_dp,
make_stages,
)
from sglang.srt.layers.communicator.residual import batch as residual_batch
from sglang.srt.layers.dp_attention import is_dp_attention_enabled
from sglang.srt.layers.layer_boundary.residual import batch as residual_batch
from sglang.srt.layers.layernorm import RMSNorm
from sglang.srt.layers.linear import (
ColumnParallelLinear,
@@ -1572,7 +1572,7 @@ class Dots3DecoderLayer(nn.Module):
config.hidden_size, eps=config.rms_norm_eps
)
self.attn_stage, self.ffn_stage = make_stages(
self.attn_boundary, self.ffn_boundary = make_stages(
(declare_attn(), self.input_layernorm),
(
declare_ffn(
@@ -1607,7 +1607,7 @@ class Dots3DecoderLayer(nn.Module):
zero_allocator: BumpAllocator,
) -> torch.Tensor:
hidden_states = self.attn_stage.prepare(hidden_states, forward_batch)
hidden_states = self.attn_boundary.prepare(hidden_states, forward_batch)
hidden_states = self.self_attn(
positions=positions,
@@ -1616,10 +1616,10 @@ class Dots3DecoderLayer(nn.Module):
zero_allocator=zero_allocator,
)
hidden_states = self.attn_stage.finish(hidden_states, forward_batch)
hidden_states = self.ffn_stage.prepare(hidden_states, forward_batch)
hidden_states = self.attn_boundary.finish(hidden_states, forward_batch)
hidden_states = self.ffn_boundary.prepare(hidden_states, forward_batch)
with self.ffn_stage.exit(forward_batch) as ffn_exit:
with self.ffn_boundary.exit(forward_batch) as ffn_exit:
hidden_states = self.mlp(
hidden_states,
forward_batch,
@@ -1637,7 +1637,7 @@ class Dots3DecoderLayer(nn.Module):
zero_allocator: BumpAllocator,
tbo_subbatch_index: Optional[int] = None,
):
state.hidden_states_after_comm_pre_attn = self.attn_stage.prepare(
state.hidden_states_after_comm_pre_attn = self.attn_boundary.prepare(
hidden_states, forward_batch
)
state.update(
@@ -1650,10 +1650,10 @@ class Dots3DecoderLayer(nn.Module):
)
def op_comm_prepare_mlp(self, state):
hidden_states = self.attn_stage.finish(
hidden_states = self.attn_boundary.finish(
state.pop("hidden_states_after_attn"), state.forward_batch
)
state.hidden_states_mlp_input = self.ffn_stage.prepare(
state.hidden_states_mlp_input = self.ffn_boundary.prepare(
hidden_states, state.forward_batch
)
@@ -1671,7 +1671,7 @@ class Dots3DecoderLayer(nn.Module):
state.hidden_states_mlp_output = hidden_states
def op_comm_postprocess_layer(self, state):
hidden_states = self.ffn_stage.postprocess(
hidden_states = self.ffn_boundary.postprocess(
state.pop("hidden_states_mlp_output"), state.forward_batch
)
@@ -1763,7 +1763,7 @@ class Dots3Model(nn.Module):
residual_batch.start(forward_batch)
else:
assert pp_proxy_tensors is not None
hidden_states = self.layers[self.start_layer].attn_stage.from_pp(
hidden_states = self.layers[self.start_layer].attn_boundary.from_pp(
pp_proxy_tensors, forward_batch
)
@@ -8,8 +8,8 @@ from torch import nn
from transformers import PretrainedConfig
from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder
from sglang.srt.layers.communicator.residual import batch as residual_batch
from sglang.srt.layers.dp_attention import is_dp_attention_enabled
from sglang.srt.layers.layer_boundary.residual import batch as residual_batch
from sglang.srt.layers.layernorm import RMSNorm
from sglang.srt.layers.linear import ReplicatedLinear
from sglang.srt.layers.logits_processor import LogitsProcessor
+1 -1
View File
@@ -26,7 +26,7 @@ from transformers.models.ernie4_5_moe.configuration_ernie4_5_moe import (
from sglang.srt.distributed import (
tensor_model_parallel_all_reduce,
)
from sglang.srt.layers.communicator import enable_moe_dense_fully_dp
from sglang.srt.layers.layer_boundary import enable_moe_dense_fully_dp
from sglang.srt.layers.layernorm import RMSNorm
from sglang.srt.layers.logits_processor import LogitsProcessor
from sglang.srt.layers.moe.ep_moe.layer import get_moe_impl_class
+11 -11
View File
@@ -11,16 +11,16 @@ from sglang.srt.layers.attention.hybrid_linear_attn_backend import (
Mamba2AttnBackend,
)
from sglang.srt.layers.attention.mamba.mamba import MambaMixer2
from sglang.srt.layers.communicator import (
from sglang.srt.layers.dp_attention import (
is_dp_attention_enabled,
)
from sglang.srt.layers.layer_boundary import (
declare_attn,
declare_ffn,
make_stages,
)
from sglang.srt.layers.communicator.output import OutputTransform
from sglang.srt.layers.communicator.residual import batch as residual_batch
from sglang.srt.layers.dp_attention import (
is_dp_attention_enabled,
)
from sglang.srt.layers.layer_boundary.output import OutputTransform
from sglang.srt.layers.layer_boundary.residual import batch as residual_batch
from sglang.srt.layers.layernorm import RMSNorm
from sglang.srt.layers.linear import (
MergedColumnParallelLinear,
@@ -222,7 +222,7 @@ class FalconH1HybridAttentionDecoderLayer(nn.Module):
self.q_norm = RMSNorm(self.head_dim, eps=config.rms_norm_eps)
self.k_norm = RMSNorm(self.head_dim, eps=config.rms_norm_eps)
self.attn_stage, self.ffn_stage = make_stages(
self.attn_boundary, self.ffn_boundary = make_stages(
(declare_attn(), self.input_layernorm),
(
declare_ffn(
@@ -334,7 +334,7 @@ class FalconH1HybridAttentionDecoderLayer(nn.Module):
forward_batch: ForwardBatch,
**kwargs: Any,
):
hidden_states = self.attn_stage.prepare(hidden_states, forward_batch)
hidden_states = self.attn_boundary.prepare(hidden_states, forward_batch)
if not forward_batch.forward_mode.is_idle():
# Attention block
@@ -363,9 +363,9 @@ class FalconH1HybridAttentionDecoderLayer(nn.Module):
hidden_states = attention_hidden_states + mamba_hidden_states
# Fully Connected
hidden_states = self.attn_stage.finish(hidden_states, forward_batch)
hidden_states = self.ffn_stage.prepare(hidden_states, forward_batch)
with self.ffn_stage.exit(forward_batch) as ffn_exit:
hidden_states = self.attn_boundary.finish(hidden_states, forward_batch)
hidden_states = self.ffn_boundary.prepare(hidden_states, forward_batch)
with self.ffn_boundary.exit(forward_batch) as ffn_exit:
hidden_states = self.feed_forward(hidden_states, forward_batch)
return ffn_exit.finish(hidden_states)
+10 -10
View File
@@ -23,9 +23,9 @@ from transformers import PretrainedConfig
import sglang.srt.models.deepseek_v2 as deepseek_v2
from sglang.srt.configs.gigachat35 import GigaChat35Config
from sglang.srt.layers.communicator import get_attn_tp_context
from sglang.srt.layers.communicator.output import OutputTransform
from sglang.srt.layers.communicator.residual import batch as residual_batch
from sglang.srt.layers.layer_boundary import get_attn_tp_context
from sglang.srt.layers.layer_boundary.output import OutputTransform
from sglang.srt.layers.layer_boundary.residual import batch as residual_batch
from sglang.srt.layers.layernorm import GemmaRMSNorm, RMSNorm
from sglang.srt.layers.linear import ColumnParallelLinear
from sglang.srt.layers.logits_processor import LogitsProcessor
@@ -441,7 +441,7 @@ class GigaChat35DecoderLayer(deepseek_v2.DeepseekV2DecoderLayer):
post_layernorm=self.post_self_attn_layernorm,
)
# The communicator chooses its fused steps from its norms when built.
self.attn_stage, self.ffn_stage = self._build_stages(
self.attn_boundary, self.ffn_boundary = self._build_stages(
input_layernorm=attn_prepare_layernorm,
post_attention_layernorm=mlp_prepare_layernorm,
qkv_latent_func=qkv_latent_func,
@@ -471,7 +471,7 @@ class GigaChat35DecoderLayer(deepseek_v2.DeepseekV2DecoderLayer):
zero_allocator: BumpAllocator,
**kwargs,
) -> torch.Tensor:
hidden_states = self.attn_stage.prepare(hidden_states, forward_batch)
hidden_states = self.attn_boundary.prepare(hidden_states, forward_batch)
if self.use_linear_attn:
hidden_states = self.self_attn(
@@ -485,18 +485,18 @@ class GigaChat35DecoderLayer(deepseek_v2.DeepseekV2DecoderLayer):
forward_batch=forward_batch,
zero_allocator=zero_allocator,
input_on_attention_tp_slices=(
self.attn_stage.input_on_attention_tp_slices
self.attn_boundary.input_on_attention_tp_slices
),
)
get_attn_tp_context().clear_attn_inputs()
hidden_states = self.attn_stage.finish(hidden_states, forward_batch)
hidden_states = self.ffn_stage.prepare(hidden_states, forward_batch)
hidden_states = self.attn_boundary.finish(hidden_states, forward_batch)
hidden_states = self.ffn_boundary.prepare(hidden_states, forward_batch)
# Unlike deepseek_v2, no moe_output_buffer_ctx here: non-inplace MoE
# runners then allocate their output per forward instead of recycling
# the layer-input buffer. The default (inplace) runners are unaffected.
with self.ffn_stage.exit(forward_batch) as ffn_exit:
with self.ffn_boundary.exit(forward_batch) as ffn_exit:
hidden_states = self.mlp(hidden_states, forward_batch)
hidden_states = ffn_exit.finish(hidden_states)
@@ -562,7 +562,7 @@ class GigaChat35Model(nn.Module):
residual_batch.start(forward_batch)
else:
assert pp_proxy_tensors is not None
hidden_states = self.layers[self.start_layer].attn_stage.from_pp(
hidden_states = self.layers[self.start_layer].attn_boundary.from_pp(
pp_proxy_tensors, forward_batch
)
+1 -1
View File
@@ -18,7 +18,7 @@ from typing import Iterable, Optional
import torch
from torch import nn
from sglang.srt.layers.communicator.residual import batch as residual_batch
from sglang.srt.layers.layer_boundary.residual import batch as residual_batch
from sglang.srt.layers.logits_processor import LogitsProcessor
from sglang.srt.layers.quantization.base_config import QuantizationConfig
from sglang.srt.layers.vocab_parallel_embedding import (
+16 -16
View File
@@ -39,17 +39,17 @@ from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation
from sglang.srt.eplb.expert_location_dispatch import ExpertLocationDispatchInfo
from sglang.srt.layers.activation import SiluAndMul
from sglang.srt.layers.aux_hidden_states import AuxHiddenStateList
from sglang.srt.layers.communicator import (
from sglang.srt.layers.dp_attention import (
is_allocation_symmetric,
is_dp_attention_enabled,
)
from sglang.srt.layers.layer_boundary import (
declare_attn,
declare_ffn,
enable_moe_dense_fully_dp,
make_stages,
)
from sglang.srt.layers.communicator.residual import batch as residual_batch
from sglang.srt.layers.dp_attention import (
is_allocation_symmetric,
is_dp_attention_enabled,
)
from sglang.srt.layers.layer_boundary.residual import batch as residual_batch
from sglang.srt.layers.layernorm import RMSNorm
from sglang.srt.layers.linear import (
MergedColumnParallelLinear,
@@ -866,7 +866,7 @@ class Glm4MoeDecoderLayer(nn.Module):
config.hidden_size, eps=config.rms_norm_eps
)
self.attn_stage, self.ffn_stage = make_stages(
self.attn_boundary, self.ffn_boundary = make_stages(
(declare_attn(), self.input_layernorm),
(
declare_ffn(
@@ -942,7 +942,7 @@ class Glm4MoeDecoderLayer(nn.Module):
capture_output=None,
) -> torch.Tensor:
hidden_states = self.attn_stage.prepare(
hidden_states = self.attn_boundary.prepare(
hidden_states,
forward_batch,
quant_format=self.attn_quant_format,
@@ -955,10 +955,10 @@ class Glm4MoeDecoderLayer(nn.Module):
forward_batch=forward_batch,
)
hidden_states = self.attn_stage.finish(hidden_states, forward_batch)
hidden_states = self.ffn_stage.prepare(hidden_states, forward_batch)
hidden_states = self.attn_boundary.finish(hidden_states, forward_batch)
hidden_states = self.ffn_boundary.prepare(hidden_states, forward_batch)
with self.ffn_stage.exit(forward_batch) as ffn_exit:
with self.ffn_boundary.exit(forward_batch) as ffn_exit:
hidden_states = self.mlp(hidden_states, forward_batch)
hidden_states = ffn_exit.finish(hidden_states)
@@ -972,7 +972,7 @@ class Glm4MoeDecoderLayer(nn.Module):
forward_batch: ForwardBatch,
tbo_subbatch_index: Optional[int] = None,
):
state.hidden_states_after_comm_pre_attn = self.attn_stage.prepare(
state.hidden_states_after_comm_pre_attn = self.attn_boundary.prepare(
hidden_states, forward_batch
)
state.update(
@@ -984,15 +984,15 @@ class Glm4MoeDecoderLayer(nn.Module):
)
def op_comm_prepare_mlp(self, state):
hidden_states = self.attn_stage.finish(
hidden_states = self.attn_boundary.finish(
state.pop("hidden_states_after_attn"), state.forward_batch
)
state.hidden_states_mlp_input = self.ffn_stage.prepare(
state.hidden_states_mlp_input = self.ffn_boundary.prepare(
hidden_states, state.forward_batch
)
def op_comm_postprocess_layer(self, state):
hidden_states = self.ffn_stage.postprocess(
hidden_states = self.ffn_boundary.postprocess(
state.pop("hidden_states_mlp_output"), state.forward_batch
)
@@ -1081,7 +1081,7 @@ class Glm4MoeModel(nn.Module):
residual_batch.start(forward_batch)
else:
assert pp_proxy_tensors is not None
hidden_states = self.layers[self.start_layer].attn_stage.from_pp(
hidden_states = self.layers[self.start_layer].attn_boundary.from_pp(
pp_proxy_tensors, forward_batch
)
+19 -17
View File
@@ -33,18 +33,18 @@ from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation
from sglang.srt.eplb.expert_location_dispatch import ExpertLocationDispatchInfo
from sglang.srt.layers.activation import SiluAndMul
from sglang.srt.layers.aux_hidden_states import AuxHiddenStateList
from sglang.srt.layers.communicator import (
from sglang.srt.layers.dp_attention import (
is_allocation_symmetric,
is_dp_attention_enabled,
)
from sglang.srt.layers.layer_boundary import (
declare_attn,
declare_ffn,
enable_moe_dense_fully_dp,
get_attn_tp_context,
make_stages,
)
from sglang.srt.layers.communicator.residual import batch as residual_batch
from sglang.srt.layers.dp_attention import (
is_allocation_symmetric,
is_dp_attention_enabled,
)
from sglang.srt.layers.layer_boundary.residual import batch as residual_batch
from sglang.srt.layers.layernorm import RMSNorm
from sglang.srt.layers.linear import MergedColumnParallelLinear, RowParallelLinear
from sglang.srt.layers.logits_processor import LogitsProcessor
@@ -581,7 +581,7 @@ class Glm4MoeLiteDecoderLayer(nn.Module):
self._gfx95_quant_format = self._detect_gfx95_quant_format()
self.attn_stage, self.ffn_stage = make_stages(
self.attn_boundary, self.ffn_boundary = make_stages(
(
declare_attn(),
self.input_layernorm,
@@ -633,7 +633,7 @@ class Glm4MoeLiteDecoderLayer(nn.Module):
zero_allocator: BumpAllocator,
capture_output=None,
) -> torch.Tensor:
hidden_states = self.attn_stage.prepare(
hidden_states = self.attn_boundary.prepare(
hidden_states,
forward_batch,
quant_format=getattr(self, "_gfx95_quant_format", ""),
@@ -645,16 +645,18 @@ class Glm4MoeLiteDecoderLayer(nn.Module):
hidden_states=hidden_states,
forward_batch=forward_batch,
zero_allocator=zero_allocator,
input_on_attention_tp_slices=(self.attn_stage.input_on_attention_tp_slices),
input_on_attention_tp_slices=(
self.attn_boundary.input_on_attention_tp_slices
),
)
if isinstance(hidden_states, tuple):
hidden_states = hidden_states[0]
get_attn_tp_context().clear_attn_inputs()
hidden_states = self.attn_stage.finish(hidden_states, forward_batch)
hidden_states = self.ffn_stage.prepare(hidden_states, forward_batch)
hidden_states = self.attn_boundary.finish(hidden_states, forward_batch)
hidden_states = self.ffn_boundary.prepare(hidden_states, forward_batch)
with self.ffn_stage.exit(forward_batch) as ffn_exit:
with self.ffn_boundary.exit(forward_batch) as ffn_exit:
hidden_states = self.mlp(hidden_states, forward_batch)
hidden_states = ffn_exit.finish(hidden_states)
@@ -669,7 +671,7 @@ class Glm4MoeLiteDecoderLayer(nn.Module):
zero_allocator: BumpAllocator,
tbo_subbatch_index: Optional[int] = None,
):
state.hidden_states_after_comm_pre_attn = self.attn_stage.prepare(
state.hidden_states_after_comm_pre_attn = self.attn_boundary.prepare(
hidden_states, forward_batch
)
if get_moe_a2a_backend().is_mori():
@@ -684,15 +686,15 @@ class Glm4MoeLiteDecoderLayer(nn.Module):
)
def op_comm_prepare_mlp(self, state):
hidden_states = self.attn_stage.finish(
hidden_states = self.attn_boundary.finish(
state.pop("hidden_states_after_attn"), state.forward_batch
)
state.hidden_states_mlp_input = self.ffn_stage.prepare(
state.hidden_states_mlp_input = self.ffn_boundary.prepare(
hidden_states, state.forward_batch
)
def op_comm_postprocess_layer(self, state):
hidden_states = self.ffn_stage.postprocess(
hidden_states = self.ffn_boundary.postprocess(
state.pop("hidden_states_mlp_output"), state.forward_batch
)
@@ -779,7 +781,7 @@ class Glm4MoeLiteModel(nn.Module):
residual_batch.start(forward_batch)
else:
assert pp_proxy_tensors is not None
hidden_states = self.layers[self.start_layer].attn_stage.from_pp(
hidden_states = self.layers[self.start_layer].attn_boundary.from_pp(
pp_proxy_tensors, forward_batch
)
device = hidden_states.device
@@ -22,8 +22,8 @@ from torch import nn
from transformers import PretrainedConfig
from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder
from sglang.srt.layers.communicator.residual import batch as residual_batch
from sglang.srt.layers.dp_attention import is_dp_attention_enabled
from sglang.srt.layers.layer_boundary.residual import batch as residual_batch
from sglang.srt.layers.layernorm import RMSNorm
from sglang.srt.layers.logits_processor import LogitsProcessor
from sglang.srt.layers.quantization.base_config import QuantizationConfig
+1 -1
View File
@@ -22,8 +22,8 @@ from torch import nn
from transformers import PretrainedConfig
from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder
from sglang.srt.layers.communicator.residual import batch as residual_batch
from sglang.srt.layers.dp_attention import is_dp_attention_enabled
from sglang.srt.layers.layer_boundary.residual import batch as residual_batch
from sglang.srt.layers.layernorm import RMSNorm
from sglang.srt.layers.logits_processor import LogitsProcessor
from sglang.srt.layers.quantization.base_config import QuantizationConfig
+12 -10
View File
@@ -26,7 +26,7 @@ from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation
from sglang.srt.layers.attention import vision_utils
from sglang.srt.layers.attention.vision import VisionAttention
from sglang.srt.layers.aux_hidden_states import AuxHiddenStateList
from sglang.srt.layers.communicator import (
from sglang.srt.layers.layer_boundary import (
PLAIN_RESIDUAL,
MHCState,
declare_attn,
@@ -35,8 +35,8 @@ from sglang.srt.layers.communicator import (
get_attn_tp_context,
make_stages,
)
from sglang.srt.layers.communicator.residual import access as residual_access
from sglang.srt.layers.communicator.residual import batch as residual_batch
from sglang.srt.layers.layer_boundary.residual import access as residual_access
from sglang.srt.layers.layer_boundary.residual import batch as residual_batch
from sglang.srt.layers.layernorm import RMSNorm
from sglang.srt.layers.linear import (
ColumnParallelBatchedLinear,
@@ -794,7 +794,7 @@ class Glm5NextDecoderLayer(nn.Module):
),
is_last_layer=terminal,
).layer_residual()
self.attn_stage, self.ffn_stage = make_stages(
self.attn_boundary, self.ffn_boundary = make_stages(
(
declare_attn(
read=residual.attention_read, update=residual.attention_update
@@ -932,7 +932,7 @@ class Glm5NextDecoderLayer(nn.Module):
):
hidden_states_orig = residual_access.buffer(hidden_states)
hidden_states = self.attn_stage.prepare(
hidden_states = self.attn_boundary.prepare(
hidden_states, forward_batch, capture_output=capture_output
)
@@ -941,7 +941,9 @@ class Glm5NextDecoderLayer(nn.Module):
hidden_states=hidden_states,
forward_batch=forward_batch,
zero_allocator=zero_allocator,
input_on_attention_tp_slices=(self.attn_stage.input_on_attention_tp_slices),
input_on_attention_tp_slices=(
self.attn_boundary.input_on_attention_tp_slices
),
prev_topk_indices=prev_topk_indices,
)
if isinstance(hidden_states, tuple):
@@ -950,8 +952,8 @@ class Glm5NextDecoderLayer(nn.Module):
topk_indices = None
get_attn_tp_context().clear_attn_inputs()
hidden_states = self.attn_stage.finish(hidden_states, forward_batch)
hidden_states = self.ffn_stage.prepare(hidden_states, forward_batch)
hidden_states = self.attn_boundary.finish(hidden_states, forward_batch)
hidden_states = self.ffn_boundary.prepare(hidden_states, forward_batch)
if isinstance(self.mlp, Glm5NextMLP):
gemm_output_zero_allocator = None
@@ -967,7 +969,7 @@ class Glm5NextDecoderLayer(nn.Module):
else:
_mlp_ctx = nullcontext()
with self.ffn_stage.exit(forward_batch) as ffn_exit, _mlp_ctx:
with self.ffn_boundary.exit(forward_batch) as ffn_exit, _mlp_ctx:
hidden_states = self.mlp(
hidden_states,
forward_batch,
@@ -1108,7 +1110,7 @@ class Glm5NextModel(nn.Module):
residual_batch.start(forward_batch)
else:
assert pp_proxy_tensors is not None
hidden_states = self.layers[self.start_layer].attn_stage.from_pp(
hidden_states = self.layers[self.start_layer].attn_boundary.from_pp(
pp_proxy_tensors, forward_batch
)
device = hidden_states.device
+11 -11
View File
@@ -30,15 +30,15 @@ from sglang.kernels.jit.utils import is_arch_support_pdl
from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder
from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation
from sglang.srt.layers.aux_hidden_states import AuxHiddenStateList
from sglang.srt.layers.communicator import (
from sglang.srt.layers.dp_attention import (
is_dp_attention_enabled,
)
from sglang.srt.layers.layer_boundary import (
declare_attn,
declare_ffn,
make_stages,
)
from sglang.srt.layers.communicator.residual import batch as residual_batch
from sglang.srt.layers.dp_attention import (
is_dp_attention_enabled,
)
from sglang.srt.layers.layer_boundary.residual import batch as residual_batch
from sglang.srt.layers.layernorm import RMSNorm
from sglang.srt.layers.linear import (
QKVParallelLinear,
@@ -599,7 +599,7 @@ class GptOssDecoderLayer(nn.Module):
x_pad_to_multiple=post_attn_pad_multiple,
)
self.attn_stage, self.ffn_stage = make_stages(
self.attn_boundary, self.ffn_boundary = make_stages(
(declare_attn(), self.input_layernorm),
(
declare_ffn(
@@ -623,7 +623,7 @@ class GptOssDecoderLayer(nn.Module):
forward_batch: ForwardBatch,
capture_output=None,
) -> torch.Tensor:
hidden_states = self.attn_stage.prepare(
hidden_states = self.attn_boundary.prepare(
hidden_states, forward_batch, capture_output=capture_output
)
@@ -634,10 +634,10 @@ class GptOssDecoderLayer(nn.Module):
forward_batch=forward_batch,
)
hidden_states = self.attn_stage.finish(hidden_states, forward_batch)
hidden_states = self.ffn_stage.prepare(hidden_states, forward_batch)
hidden_states = self.attn_boundary.finish(hidden_states, forward_batch)
hidden_states = self.ffn_boundary.prepare(hidden_states, forward_batch)
with self.ffn_stage.exit(forward_batch) as ffn_exit:
with self.ffn_boundary.exit(forward_batch) as ffn_exit:
hidden_states = self.mlp(hidden_states, forward_batch)
hidden_states = ffn_exit.finish(hidden_states)
@@ -707,7 +707,7 @@ class GptOssModel(nn.Module):
residual_batch.start(forward_batch)
else:
assert pp_proxy_tensors is not None
hidden_states = self.layers[self.start_layer].attn_stage.from_pp(
hidden_states = self.layers[self.start_layer].attn_boundary.from_pp(
pp_proxy_tensors, forward_batch
)
+1 -1
View File
@@ -7,7 +7,7 @@ from torch import nn
from transformers import PretrainedConfig
from sglang.srt.layers.attention.index_topk_share import IndexTopKShareState
from sglang.srt.layers.communicator import AttentionInputs, get_attn_tp_context
from sglang.srt.layers.layer_boundary import AttentionInputs, get_attn_tp_context
from sglang.srt.layers.layernorm import RMSNorm
from sglang.srt.layers.linear import ColumnParallelLinear, ReplicatedLinear
from sglang.srt.layers.logits_processor import LogitsProcessor
+1 -1
View File
@@ -5,7 +5,7 @@ import torch
from torch import nn
from sglang.srt.layers.attention.index_topk_share import IndexTopKShareState
from sglang.srt.layers.communicator import AttentionInputs, get_attn_tp_context
from sglang.srt.layers.layer_boundary import AttentionInputs, get_attn_tp_context
from sglang.srt.layers.layernorm import RMSNorm
from sglang.srt.layers.logits_processor import LogitsProcessor
from sglang.srt.layers.vocab_parallel_embedding import (
+11 -11
View File
@@ -14,14 +14,14 @@ from sglang.srt.layers.aux_hidden_states import (
AuxHiddenStateAccumulator,
AuxHiddenStateList,
)
from sglang.srt.layers.communicator import (
from sglang.srt.layers.dp_attention import is_dp_attention_enabled
from sglang.srt.layers.layer_boundary import (
declare_attn,
declare_ffn,
make_stages,
)
from sglang.srt.layers.communicator.residual import batch as residual_batch
from sglang.srt.layers.communicator.residual.add_norm import Fp8Input, NormQuantRead
from sglang.srt.layers.dp_attention import is_dp_attention_enabled
from sglang.srt.layers.layer_boundary.residual import batch as residual_batch
from sglang.srt.layers.layer_boundary.residual.add_norm import Fp8Input, NormQuantRead
from sglang.srt.layers.layernorm import GemmaRMSNorm
from sglang.srt.layers.linear import (
QKVParallelLinear,
@@ -428,9 +428,9 @@ class _InternS2MobiusDecoderMixin:
forward_batch: ForwardBatch,
meta_mlp: nn.ModuleList,
) -> torch.Tensor:
hidden_states = self.attn_stage.finish(hidden_states, forward_batch)
hidden_states = self.ffn_stage.prepare(hidden_states, forward_batch)
with self.ffn_stage.exit(forward_batch) as ffn_exit:
hidden_states = self.attn_boundary.finish(hidden_states, forward_batch)
hidden_states = self.ffn_boundary.prepare(hidden_states, forward_batch)
with self.ffn_boundary.exit(forward_batch) as ffn_exit:
hidden_states = self._forward_mobius_mlp(
hidden_states, forward_batch, meta_mlp
)
@@ -489,7 +489,7 @@ class InternS2MobiusLinearDecoderLayer(_InternS2MobiusDecoderMixin, nn.Module):
config.hidden_size, eps=config.rms_norm_eps
)
accepts_fp8_input = _linear_accepts_fp8_tuple(self.linear_attn.in_proj_qkvz)
self.attn_stage, self.ffn_stage = make_stages(
self.attn_boundary, self.ffn_boundary = make_stages(
(
declare_attn(
read=NormQuantRead(
@@ -518,7 +518,7 @@ class InternS2MobiusLinearDecoderLayer(_InternS2MobiusDecoderMixin, nn.Module):
**kwargs,
):
forward_batch = kwargs["forward_batch"]
hidden_states = self.attn_stage.prepare(
hidden_states = self.attn_boundary.prepare(
hidden_states,
forward_batch,
captured_last_layer_outputs=kwargs.get("captured_last_layer_outputs"),
@@ -621,7 +621,7 @@ class InternS2MobiusAttentionDecoderLayer(
self.q_norm = GemmaRMSNorm(self.head_dim, eps=config.rms_norm_eps)
self.k_norm = GemmaRMSNorm(self.head_dim, eps=config.rms_norm_eps)
accepts_fp8_input = _linear_accepts_fp8_tuple(self.qkv_proj)
self.attn_stage, self.ffn_stage = make_stages(
self.attn_boundary, self.ffn_boundary = make_stages(
(
declare_attn(
read=NormQuantRead(
@@ -654,7 +654,7 @@ class InternS2MobiusAttentionDecoderLayer(
**kwargs,
):
del kwargs
hidden_states = self.attn_stage.prepare(
hidden_states = self.attn_boundary.prepare(
hidden_states,
forward_batch,
captured_last_layer_outputs=captured_last_layer_outputs,
+1 -1
View File
@@ -25,7 +25,7 @@ from torch import nn
from transformers import PretrainedConfig
from sglang.srt.distributed.device_communicators import triton_symm_mem_ag
from sglang.srt.layers.communicator import AttentionInputs, get_attn_tp_context
from sglang.srt.layers.layer_boundary import AttentionInputs, get_attn_tp_context
from sglang.srt.layers.layernorm import RMSNorm
from sglang.srt.layers.linear import ColumnParallelLinear, ReplicatedLinear
from sglang.srt.layers.logits_processor import LogitsProcessor
+1 -1
View File
@@ -2615,7 +2615,7 @@ class KimiK3DecoderLayer(nn.Module):
) -> torch.Tensor:
# For MLA layers with q_lora_rank, set up attn_inputs before the
# forward call (normally done by the attention boundary).
from sglang.srt.layers.communicator import (
from sglang.srt.layers.layer_boundary import (
AttentionInputs,
get_attn_tp_context,
)
+11 -11
View File
@@ -20,15 +20,15 @@ from sglang.srt.configs.laguna import LagunaConfig, normalize_gating
from sglang.srt.environ import envs
from sglang.srt.layers.activation import SiluAndMul
from sglang.srt.layers.aux_hidden_states import AuxHiddenStateList
from sglang.srt.layers.communicator import (
from sglang.srt.layers.dp_attention import (
is_dp_attention_enabled,
)
from sglang.srt.layers.layer_boundary import (
declare_attn,
declare_ffn,
make_stages,
)
from sglang.srt.layers.communicator.residual import batch as residual_batch
from sglang.srt.layers.dp_attention import (
is_dp_attention_enabled,
)
from sglang.srt.layers.layer_boundary.residual import batch as residual_batch
from sglang.srt.layers.layernorm import RMSNorm
from sglang.srt.layers.linear import (
ColumnParallelLinear,
@@ -457,7 +457,7 @@ class LagunaDecoderLayer(nn.Module):
config.hidden_size, eps=config.rms_norm_eps
)
self.attn_stage, self.ffn_stage = make_stages(
self.attn_boundary, self.ffn_boundary = make_stages(
(declare_attn(), self.input_layernorm),
(
declare_ffn(
@@ -481,7 +481,7 @@ class LagunaDecoderLayer(nn.Module):
forward_batch: ForwardBatch,
capture_output=None,
) -> torch.Tensor:
hidden_states = self.attn_stage.prepare(
hidden_states = self.attn_boundary.prepare(
hidden_states, forward_batch, capture_output=capture_output
)
if hidden_states.shape[0] != 0:
@@ -490,10 +490,10 @@ class LagunaDecoderLayer(nn.Module):
hidden_states=hidden_states,
forward_batch=forward_batch,
)
hidden_states = self.attn_stage.finish(hidden_states, forward_batch)
hidden_states = self.ffn_stage.prepare(hidden_states, forward_batch)
hidden_states = self.attn_boundary.finish(hidden_states, forward_batch)
hidden_states = self.ffn_boundary.prepare(hidden_states, forward_batch)
with self.ffn_stage.exit(forward_batch) as ffn_exit:
with self.ffn_boundary.exit(forward_batch) as ffn_exit:
hidden_states = self.mlp(
hidden_states,
forward_batch=forward_batch,
@@ -589,7 +589,7 @@ class LagunaModel(nn.Module):
residual_batch.start(forward_batch)
else:
assert pp_proxy_tensors is not None
hidden_states = self.layers[self.start_layer].attn_stage.from_pp(
hidden_states = self.layers[self.start_layer].attn_boundary.from_pp(
pp_proxy_tensors, forward_batch
)
+11 -11
View File
@@ -31,16 +31,16 @@ from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_r
from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation
from sglang.srt.eplb.expert_location_dispatch import ExpertLocationDispatchInfo
from sglang.srt.layers.activation import SiluAndMul
from sglang.srt.layers.communicator import (
from sglang.srt.layers.dp_attention import (
is_dp_attention_enabled,
)
from sglang.srt.layers.layer_boundary import (
declare_attn,
declare_ffn,
enable_moe_dense_fully_dp,
make_stages,
)
from sglang.srt.layers.communicator.residual import batch as residual_batch
from sglang.srt.layers.dp_attention import (
is_dp_attention_enabled,
)
from sglang.srt.layers.layer_boundary.residual import batch as residual_batch
from sglang.srt.layers.layernorm import RMSNorm
from sglang.srt.layers.linear import (
MergedColumnParallelLinear,
@@ -638,7 +638,7 @@ class LLaDA2MoeBlock(nn.Module):
self.post_attention_layernorm = RMSNorm(hidden_size, eps=config.rms_norm_eps)
self.attn_stage, self.ffn_stage = make_stages(
self.attn_boundary, self.ffn_boundary = make_stages(
(declare_attn(), self.input_layernorm),
(
declare_ffn(
@@ -666,7 +666,7 @@ class LLaDA2MoeBlock(nn.Module):
hidden_states: torch.Tensor,
forward_batch: ForwardBatch,
) -> torch.Tensor:
hidden_states = self.attn_stage.prepare(
hidden_states = self.attn_boundary.prepare(
hidden_states=hidden_states, forward_batch=forward_batch
)
@@ -676,13 +676,13 @@ class LLaDA2MoeBlock(nn.Module):
forward_batch=forward_batch,
)
hidden_states = self.attn_stage.finish(hidden_states, forward_batch)
hidden_states = self.ffn_stage.prepare(
hidden_states = self.attn_boundary.finish(hidden_states, forward_batch)
hidden_states = self.ffn_boundary.prepare(
hidden_states=hidden_states,
forward_batch=forward_batch,
)
with self.ffn_stage.exit(forward_batch) as ffn_exit:
with self.ffn_boundary.exit(forward_batch) as ffn_exit:
hidden_states = self.mlp(hidden_states, forward_batch)
hidden_states = ffn_exit.finish(hidden_states)
@@ -749,7 +749,7 @@ class LLaDA2MoeModel(nn.Module):
residual_batch.start(forward_batch)
else:
assert pp_proxy_tensors is not None
hidden_states = self.layers[self.start_layer].attn_stage.from_pp(
hidden_states = self.layers[self.start_layer].attn_boundary.from_pp(
pp_proxy_tensors, forward_batch
)
+10 -10
View File
@@ -26,15 +26,15 @@ from torch import nn
from transformers import Llama4TextConfig
from sglang.srt.layers.aux_hidden_states import AuxHiddenStateList
from sglang.srt.layers.communicator import (
from sglang.srt.layers.dp_attention import (
is_dp_attention_enabled,
)
from sglang.srt.layers.layer_boundary import (
declare_attn,
declare_ffn,
make_stages,
)
from sglang.srt.layers.communicator.residual import batch as residual_batch
from sglang.srt.layers.dp_attention import (
is_dp_attention_enabled,
)
from sglang.srt.layers.layer_boundary.residual import batch as residual_batch
from sglang.srt.layers.layernorm import RMSNorm
from sglang.srt.layers.linear import (
QKVParallelLinear,
@@ -426,7 +426,7 @@ class Llama4DecoderLayer(nn.Module):
config.hidden_size, eps=config.rms_norm_eps
)
self.attn_stage, self.ffn_stage = make_stages(
self.attn_boundary, self.ffn_boundary = make_stages(
(declare_attn(), self.input_layernorm),
(
declare_ffn(
@@ -459,7 +459,7 @@ class Llama4DecoderLayer(nn.Module):
forward_batch: ForwardBatch,
capture_output=None,
) -> torch.Tensor:
hidden_states = self.attn_stage.prepare(
hidden_states = self.attn_boundary.prepare(
hidden_states, forward_batch, capture_output=capture_output
)
@@ -470,11 +470,11 @@ class Llama4DecoderLayer(nn.Module):
forward_batch=forward_batch,
)
hidden_states = self.attn_stage.finish(hidden_states, forward_batch)
hidden_states = self.ffn_stage.prepare(hidden_states, forward_batch)
hidden_states = self.attn_boundary.finish(hidden_states, forward_batch)
hidden_states = self.ffn_boundary.prepare(hidden_states, forward_batch)
# Fully Connected
with self.ffn_stage.exit(forward_batch) as ffn_exit:
with self.ffn_boundary.exit(forward_batch) as ffn_exit:
hidden_states = self.feed_forward(hidden_states, forward_batch)
hidden_states = ffn_exit.finish(hidden_states)
+38 -36
View File
@@ -51,15 +51,15 @@ from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_r
from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation
from sglang.srt.layers import deep_gemm_wrapper
from sglang.srt.layers.activation import SiluAndMul
from sglang.srt.layers.communicator import (
from sglang.srt.layers.dp_attention import (
is_dp_attention_enabled,
)
from sglang.srt.layers.layer_boundary import (
declare_attn,
declare_ffn,
make_stages,
)
from sglang.srt.layers.communicator.residual import batch as residual_batch
from sglang.srt.layers.dp_attention import (
is_dp_attention_enabled,
)
from sglang.srt.layers.layer_boundary.residual import batch as residual_batch
from sglang.srt.layers.layernorm import RMSNorm
from sglang.srt.layers.linear import (
MergedColumnParallelLinear,
@@ -431,7 +431,7 @@ class LongcatFlashDecoderLayer(nn.Module):
self.attn_tp_size = get_parallel().attn_tp_size
self.attn_tp_rank = get_parallel().attn_tp_rank
self.attn_stage, self.moe_stage = make_stages(
self.attn_boundary, self.moe_boundary = make_stages(
(
declare_attn(),
self.input_layernorm[0],
@@ -446,21 +446,23 @@ class LongcatFlashDecoderLayer(nn.Module):
else None,
terminal=self.layer_id == config.num_hidden_layers - 1,
)
self.first_ffn, self.second_attn, self.second_ffn = make_stages(
(
declare_ffn(),
self.post_attention_layernorm[0],
),
(
declare_attn(),
self.input_layernorm[1],
{"qkv_latent_func": self.self_attn[1].prepare_qkv_latent},
),
(
declare_ffn(),
self.post_attention_layernorm[1],
),
prepared_from=self.moe_stage.declaration,
self.first_ffn_boundary, self.second_attn_boundary, self.second_ffn_boundary = (
make_stages(
(
declare_ffn(),
self.post_attention_layernorm[0],
),
(
declare_attn(),
self.input_layernorm[1],
{"qkv_latent_func": self.self_attn[1].prepare_qkv_latent},
),
(
declare_ffn(),
self.post_attention_layernorm[1],
),
prepared_from=self.moe_boundary.declaration,
)
)
def forward(
@@ -472,7 +474,7 @@ class LongcatFlashDecoderLayer(nn.Module):
prev_topk_indices: Optional[torch.Tensor],
) -> torch.Tensor:
# first_attn
hidden_states = self.attn_stage.prepare(hidden_states, forward_batch)
hidden_states = self.attn_boundary.prepare(hidden_states, forward_batch)
if hidden_states.shape[0] != 0:
attn_out = self.self_attn[0](
positions=positions,
@@ -487,9 +489,9 @@ class LongcatFlashDecoderLayer(nn.Module):
hidden_states = attn_out
# moe
hidden_states = self.attn_stage.finish(hidden_states, forward_batch)
hidden_states = self.moe_stage.prepare(hidden_states, forward_batch)
moe_hidden_states = self.moe_stage.branch_output(
hidden_states = self.attn_boundary.finish(hidden_states, forward_batch)
hidden_states = self.moe_boundary.prepare(hidden_states, forward_batch)
moe_hidden_states = self.moe_boundary.branch_output(
self.mlp(hidden_states.clone()), forward_batch
)
@@ -501,10 +503,10 @@ class LongcatFlashDecoderLayer(nn.Module):
prev_topk_indices,
)
hidden_states = self.moe_stage.merge_branch(
hidden_states = self.moe_boundary.merge_branch(
moe_hidden_states,
hidden_states,
self.second_ffn,
self.second_ffn_boundary,
forward_batch,
)
return hidden_states, prev_topk_indices
@@ -518,15 +520,15 @@ class LongcatFlashDecoderLayer(nn.Module):
prev_topk_indices,
):
# first_mlp, on the input the MoE's boundary read
hidden_states = self.first_ffn.branch_input(
self.moe_stage, hidden_states, forward_batch
hidden_states = self.first_ffn_boundary.branch_input(
self.moe_boundary, hidden_states, forward_batch
)
with self.first_ffn.exit(forward_batch) as ffn_exit:
with self.first_ffn_boundary.exit(forward_batch) as ffn_exit:
hidden_states = self.mlps[0](hidden_states)
hidden_states = ffn_exit.finish(hidden_states)
# second_attn
hidden_states = self.second_attn.prepare(hidden_states, forward_batch)
# second_attn_boundary
hidden_states = self.second_attn_boundary.prepare(hidden_states, forward_batch)
if hidden_states.shape[0] != 0:
attn_out = self.self_attn[1](
positions=positions,
@@ -541,9 +543,9 @@ class LongcatFlashDecoderLayer(nn.Module):
hidden_states = attn_out
# second_mlp
hidden_states = self.second_attn.finish(hidden_states, forward_batch)
hidden_states = self.second_ffn.prepare(hidden_states, forward_batch)
with self.second_ffn.exit(forward_batch) as ffn_exit:
hidden_states = self.second_attn_boundary.finish(hidden_states, forward_batch)
hidden_states = self.second_ffn_boundary.prepare(hidden_states, forward_batch)
with self.second_ffn_boundary.exit(forward_batch) as ffn_exit:
hidden_states = self.mlps[1](hidden_states)
hidden_states = ffn_exit.finish(hidden_states)
@@ -628,7 +630,7 @@ class LongcatFlashModel(nn.Module):
for i in range(total_num_layers):
if i in self.layers_to_capture:
aux_hidden_states.append(
self.layers[i].attn_stage.snapshot(hidden_states, forward_batch)
self.layers[i].attn_boundary.snapshot(hidden_states, forward_batch)
)
with get_global_expert_distribution_recorder().with_current_layer(i):
layer = self.layers[i]
+10 -10
View File
@@ -41,15 +41,15 @@ from sglang.kernels.ops.quantization.fp8_kernel import is_fp8_fnuz
from sglang.srt.configs import LongcatFlashConfig
from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder
from sglang.srt.layers import deep_gemm_wrapper
from sglang.srt.layers.communicator import (
from sglang.srt.layers.dp_attention import (
is_dp_attention_enabled,
)
from sglang.srt.layers.layer_boundary import (
declare_attn,
declare_ffn,
make_stages,
)
from sglang.srt.layers.communicator.residual import batch as residual_batch
from sglang.srt.layers.dp_attention import (
is_dp_attention_enabled,
)
from sglang.srt.layers.layer_boundary.residual import batch as residual_batch
from sglang.srt.layers.layernorm import RMSNorm
from sglang.srt.layers.linear import ReplicatedLinear
from sglang.srt.layers.logits_processor import LogitsProcessor
@@ -165,7 +165,7 @@ class LongcatFlashDenseDecoderLayer(nn.Module):
self.attn_tp_size = get_parallel().attn_tp_size
self.attn_tp_rank = get_parallel().attn_tp_rank
self.attn_stage, self.ffn_stage = make_stages(
self.attn_boundary, self.ffn_boundary = make_stages(
(declare_attn(), self.input_layernorm),
(
declare_ffn(sparse=False, next_sparse=False),
@@ -183,7 +183,7 @@ class LongcatFlashDenseDecoderLayer(nn.Module):
zero_allocator: BumpAllocator,
) -> torch.Tensor:
hidden_states = self.attn_stage.prepare(hidden_states, forward_batch)
hidden_states = self.attn_boundary.prepare(hidden_states, forward_batch)
if hidden_states.shape[0] != 0:
hidden_states = self.self_attn(
positions=positions,
@@ -192,10 +192,10 @@ class LongcatFlashDenseDecoderLayer(nn.Module):
zero_allocator=zero_allocator,
)
hidden_states = self.attn_stage.finish(hidden_states, forward_batch)
hidden_states = self.ffn_stage.prepare(hidden_states, forward_batch)
hidden_states = self.attn_boundary.finish(hidden_states, forward_batch)
hidden_states = self.ffn_boundary.prepare(hidden_states, forward_batch)
hidden_states = self.mlp(hidden_states)
hidden_states = self.ffn_stage.postprocess(hidden_states, forward_batch)
hidden_states = self.ffn_boundary.postprocess(hidden_states, forward_batch)
return hidden_states
+2 -2
View File
@@ -25,7 +25,7 @@ import torch
from torch import nn
from transformers import PretrainedConfig
from sglang.srt.layers.communicator import (
from sglang.srt.layers.layer_boundary import (
declare_attn,
declare_ffn,
make_stages,
@@ -449,7 +449,7 @@ class MellumDecoderLayer(Qwen3MoeDecoderLayer):
self.input_layernorm = RMSNorm(cfg.hidden_size, eps=rms_norm_eps)
self.post_attention_layernorm = RMSNorm(cfg.hidden_size, eps=rms_norm_eps)
self.attn_stage, self.ffn_stage = make_stages(
self.attn_boundary, self.ffn_boundary = make_stages(
(declare_attn(), self.input_layernorm),
(
declare_ffn(
+15 -15
View File
@@ -32,16 +32,16 @@ from sglang.srt.layers.aux_hidden_states import (
AuxHiddenStateAccumulator,
AuxHiddenStatePacker,
)
from sglang.srt.layers.communicator import (
from sglang.srt.layers.dp_attention import (
is_dp_attention_enabled,
)
from sglang.srt.layers.layer_boundary import (
declare_attn,
declare_ffn,
enable_moe_dense_fully_dp,
make_stages,
)
from sglang.srt.layers.communicator.residual import batch as residual_batch
from sglang.srt.layers.dp_attention import (
is_dp_attention_enabled,
)
from sglang.srt.layers.layer_boundary.residual import batch as residual_batch
from sglang.srt.layers.layernorm import RMSNorm
from sglang.srt.layers.linear import (
MergedColumnParallelLinear,
@@ -858,7 +858,7 @@ class MiMoV2DecoderLayer(nn.Module):
config.hidden_size, eps=config.layernorm_epsilon
)
self.attn_stage, self.ffn_stage = make_stages(
self.attn_boundary, self.ffn_boundary = make_stages(
(declare_attn(), self.input_layernorm),
(
declare_ffn(
@@ -883,7 +883,7 @@ class MiMoV2DecoderLayer(nn.Module):
captured_last_layer_outputs: Optional[AuxHiddenStateAccumulator] = None,
) -> torch.Tensor:
# Self Attention
hidden_states = self.attn_stage.prepare(
hidden_states = self.attn_boundary.prepare(
hidden_states,
forward_batch,
captured_last_layer_outputs=captured_last_layer_outputs,
@@ -896,10 +896,10 @@ class MiMoV2DecoderLayer(nn.Module):
forward_batch=forward_batch,
)
hidden_states = self.attn_stage.finish(hidden_states, forward_batch)
hidden_states = self.ffn_stage.prepare(hidden_states, forward_batch)
hidden_states = self.attn_boundary.finish(hidden_states, forward_batch)
hidden_states = self.ffn_boundary.prepare(hidden_states, forward_batch)
with self.ffn_stage.exit(forward_batch) as ffn_exit:
with self.ffn_boundary.exit(forward_batch) as ffn_exit:
hidden_states = self.mlp(hidden_states, forward_batch)
hidden_states = ffn_exit.finish(hidden_states)
@@ -924,7 +924,7 @@ class MiMoV2DecoderLayer(nn.Module):
forward_batch: ForwardBatch,
tbo_subbatch_index: Optional[int] = None,
):
state.hidden_states_after_comm_pre_attn = self.attn_stage.prepare(
state.hidden_states_after_comm_pre_attn = self.attn_boundary.prepare(
hidden_states, forward_batch
)
state.update(
@@ -936,15 +936,15 @@ class MiMoV2DecoderLayer(nn.Module):
)
def op_comm_prepare_mlp(self, state):
hidden_states = self.attn_stage.finish(
hidden_states = self.attn_boundary.finish(
state.pop("hidden_states_after_attn"), state.forward_batch
)
state.hidden_states_mlp_input = self.ffn_stage.prepare(
state.hidden_states_mlp_input = self.ffn_boundary.prepare(
hidden_states, state.forward_batch
)
def op_comm_postprocess_layer(self, state):
hidden_states = self.ffn_stage.postprocess(
hidden_states = self.ffn_boundary.postprocess(
state.pop("hidden_states_mlp_output"), state.forward_batch
)
@@ -1035,7 +1035,7 @@ class MiMoV2Model(nn.Module):
residual_batch.start(forward_batch)
else:
assert pp_proxy_tensors is not None
hidden_states = self.layers[self.start_layer].attn_stage.from_pp(
hidden_states = self.layers[self.start_layer].attn_boundary.from_pp(
pp_proxy_tensors, forward_batch
)
+10 -10
View File
@@ -21,16 +21,16 @@ from transformers import PretrainedConfig
from sglang.srt.configs.model_config import get_mimo_v2_fused_qkv_expected_tp_size
from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder
from sglang.srt.layers.communicator import (
from sglang.srt.layers.dp_attention import (
is_dp_attention_enabled,
)
from sglang.srt.layers.layer_boundary import (
declare_attn,
declare_ffn,
enable_moe_dense_fully_dp,
make_stages,
)
from sglang.srt.layers.communicator.residual import batch as residual_batch
from sglang.srt.layers.dp_attention import (
is_dp_attention_enabled,
)
from sglang.srt.layers.layer_boundary.residual import batch as residual_batch
from sglang.srt.layers.layernorm import RMSNorm
from sglang.srt.layers.logits_processor import LogitsProcessor
from sglang.srt.layers.quantization.base_config import QuantizationConfig
@@ -119,7 +119,7 @@ class MiMoV2MTPLayer(nn.Module):
config.hidden_size, eps=config.layernorm_epsilon
)
self.attn_stage, self.ffn_stage = make_stages(
self.attn_boundary, self.ffn_boundary = make_stages(
(declare_attn(), self.input_layernorm),
(
declare_ffn(
@@ -143,7 +143,7 @@ class MiMoV2MTPLayer(nn.Module):
forward_batch: ForwardBatch,
) -> torch.Tensor:
hidden_states = self.attn_stage.prepare(hidden_states, forward_batch)
hidden_states = self.attn_boundary.prepare(hidden_states, forward_batch)
if hidden_states.shape[0] != 0:
hidden_states = self.self_attn(
@@ -152,10 +152,10 @@ class MiMoV2MTPLayer(nn.Module):
forward_batch=forward_batch,
)
hidden_states = self.attn_stage.finish(hidden_states, forward_batch)
hidden_states = self.ffn_stage.prepare(hidden_states, forward_batch)
hidden_states = self.attn_boundary.finish(hidden_states, forward_batch)
hidden_states = self.ffn_boundary.prepare(hidden_states, forward_batch)
with (
self.ffn_stage.exit(forward_batch) as ffn_exit,
self.ffn_boundary.exit(forward_batch) as ffn_exit,
get_global_expert_distribution_recorder().disable_this_region(),
):
hidden_states = self.mlp(hidden_states)
+16 -16
View File
@@ -34,16 +34,16 @@ from sglang.srt.layers.aux_hidden_states import (
AuxHiddenStateAccumulator,
AuxHiddenStateList,
)
from sglang.srt.layers.communicator import (
declare_attn,
declare_ffn,
make_stages,
)
from sglang.srt.layers.communicator.residual import batch as residual_batch
from sglang.srt.layers.dp_attention import (
attn_tp_all_reduce,
is_dp_attention_enabled,
)
from sglang.srt.layers.layer_boundary import (
declare_attn,
declare_ffn,
make_stages,
)
from sglang.srt.layers.layer_boundary.residual import batch as residual_batch
from sglang.srt.layers.layernorm import RMSNorm
from sglang.srt.layers.linear import (
QKVParallelLinear,
@@ -982,7 +982,7 @@ class MiniMaxM2DecoderLayer(nn.Module):
is_previous_layer_sparse = True
is_next_layer_sparse = True
self.attn_stage, self.ffn_stage = make_stages(
self.attn_boundary, self.ffn_boundary = make_stages(
(declare_attn(), self.input_layernorm),
(
declare_ffn(
@@ -1007,7 +1007,7 @@ class MiniMaxM2DecoderLayer(nn.Module):
captured_last_layer_outputs: Optional[AuxHiddenStateAccumulator] = None,
) -> torch.Tensor:
# Self Attention
hidden_states = self.attn_stage.prepare(
hidden_states = self.attn_boundary.prepare(
hidden_states,
forward_batch,
captured_last_layer_outputs=captured_last_layer_outputs,
@@ -1021,10 +1021,10 @@ class MiniMaxM2DecoderLayer(nn.Module):
# Fully Connected (MLP or MoE)
hidden_states = self.attn_stage.finish(hidden_states, forward_batch)
hidden_states = self.ffn_stage.prepare(hidden_states, forward_batch)
hidden_states = self.attn_boundary.finish(hidden_states, forward_batch)
hidden_states = self.ffn_boundary.prepare(hidden_states, forward_batch)
with self.ffn_stage.exit(forward_batch) as ffn_exit:
with self.ffn_boundary.exit(forward_batch) as ffn_exit:
hidden_states = self.block_sparse_moe(hidden_states, forward_batch)
hidden_states = ffn_exit.finish(hidden_states)
@@ -1041,7 +1041,7 @@ class MiniMaxM2DecoderLayer(nn.Module):
tbo_subbatch_index: Optional[int] = None,
):
"""Communication prepare for attention - TBO operation"""
state.hidden_states_after_comm_pre_attn = self.attn_stage.prepare(
state.hidden_states_after_comm_pre_attn = self.attn_boundary.prepare(
hidden_states, forward_batch
)
state.update(
@@ -1055,16 +1055,16 @@ class MiniMaxM2DecoderLayer(nn.Module):
def op_comm_prepare_mlp(self, state):
"""Communication prepare for MLP - TBO operation"""
hidden_states = self.attn_stage.finish(
hidden_states = self.attn_boundary.finish(
state.pop("hidden_states_after_attn"), state.forward_batch
)
state.hidden_states_mlp_input = self.ffn_stage.prepare(
state.hidden_states_mlp_input = self.ffn_boundary.prepare(
hidden_states, state.forward_batch
)
def op_comm_postprocess_layer(self, state):
"""Communication postprocess for layer - TBO operation"""
hidden_states = self.ffn_stage.postprocess(
hidden_states = self.ffn_boundary.postprocess(
state.pop("hidden_states_mlp_output"), state.forward_batch
)
@@ -1143,7 +1143,7 @@ class MiniMaxM2Model(nn.Module):
residual_batch.start(forward_batch)
else:
assert pp_proxy_tensors is not None
hidden_states = self.layers[self.start_layer].attn_stage.from_pp(
hidden_states = self.layers[self.start_layer].attn_boundary.from_pp(
pp_proxy_tensors, forward_batch
)
+11 -11
View File
@@ -39,16 +39,16 @@ from sglang.srt.layers.aux_hidden_states import (
AuxHiddenStateAccumulator,
AuxHiddenStateList,
)
from sglang.srt.layers.communicator import (
from sglang.srt.layers.dp_attention import (
is_dp_attention_enabled,
)
from sglang.srt.layers.layer_boundary import (
declare_attn,
declare_ffn,
enable_moe_dense_fully_dp,
make_stages,
)
from sglang.srt.layers.communicator.residual import batch as residual_batch
from sglang.srt.layers.dp_attention import (
is_dp_attention_enabled,
)
from sglang.srt.layers.layer_boundary.residual import batch as residual_batch
from sglang.srt.layers.layernorm import GemmaRMSNorm, RMSNorm
from sglang.srt.layers.linear import (
MergedColumnParallelLinear,
@@ -1354,7 +1354,7 @@ class MiniMaxM3DecoderLayer(nn.Module):
is_previous_layer_sparse = _is_layer_sparse(layer_id - 1)
is_next_layer_sparse = _is_layer_sparse(layer_id + 1)
self.attn_stage, self.ffn_stage = make_stages(
self.attn_boundary, self.ffn_boundary = make_stages(
(declare_attn(), self.input_layernorm),
(
declare_ffn(
@@ -1379,7 +1379,7 @@ class MiniMaxM3DecoderLayer(nn.Module):
captured_last_layer_outputs: Optional[AuxHiddenStateAccumulator] = None,
**kwargs,
) -> torch.Tensor:
hidden_states = self.attn_stage.prepare(
hidden_states = self.attn_boundary.prepare(
hidden_states,
forward_batch,
captured_last_layer_outputs=captured_last_layer_outputs,
@@ -1393,10 +1393,10 @@ class MiniMaxM3DecoderLayer(nn.Module):
forward_batch=forward_batch,
)
hidden_states = self.attn_stage.finish(hidden_states, forward_batch)
hidden_states = self.ffn_stage.prepare(hidden_states, forward_batch)
hidden_states = self.attn_boundary.finish(hidden_states, forward_batch)
hidden_states = self.ffn_boundary.prepare(hidden_states, forward_batch)
with self.ffn_stage.exit(forward_batch) as ffn_exit:
with self.ffn_boundary.exit(forward_batch) as ffn_exit:
if self.is_layer_sparse or hidden_states.shape[0] != 0:
hidden_states = self.mlp(
hidden_states,
@@ -1483,7 +1483,7 @@ class MiniMaxM3Model(nn.Module):
residual_batch.start(forward_batch)
else:
assert pp_proxy_tensors is not None
hidden_states = self.layers[self.start_layer].attn_stage.from_pp(
hidden_states = self.layers[self.start_layer].attn_boundary.from_pp(
pp_proxy_tensors, forward_batch
)
+8 -8
View File
@@ -22,13 +22,13 @@ from sglang.srt.layers.attention.vision import (
VisionAttentionMetadata,
prepare_vision_attention_metadata,
)
from sglang.srt.layers.communicator import (
from sglang.srt.layers.conv import Conv3dLayer
from sglang.srt.layers.layer_boundary import (
declare_attn,
declare_ffn,
make_stages,
)
from sglang.srt.layers.communicator.residual import batch as residual_batch
from sglang.srt.layers.conv import Conv3dLayer
from sglang.srt.layers.layer_boundary.residual import batch as residual_batch
from sglang.srt.layers.layernorm import RMSNorm
from sglang.srt.layers.linear import (
ColumnParallelLinear,
@@ -1017,7 +1017,7 @@ class MossVLSelfAttentionDecoderLayer(nn.Module):
config.hidden_size, eps=config.rms_norm_eps, **norm_kwargs
)
self.attn_stage, self.ffn_stage = make_stages(
self.attn_boundary, self.ffn_boundary = make_stages(
(declare_attn(), self.input_layernorm),
(
declare_ffn(sparse=False, next_sparse=False),
@@ -1034,7 +1034,7 @@ class MossVLSelfAttentionDecoderLayer(nn.Module):
forward_batch: ForwardBatch,
) -> torch.Tensor:
# Self Attention
hidden_states = self.attn_stage.prepare(hidden_states, forward_batch)
hidden_states = self.attn_boundary.prepare(hidden_states, forward_batch)
if hidden_states.shape[0] != 0:
hidden_states = self.self_attn(
positions=positions,
@@ -1043,9 +1043,9 @@ class MossVLSelfAttentionDecoderLayer(nn.Module):
)
# MLP
hidden_states = self.attn_stage.finish(hidden_states, forward_batch)
hidden_states = self.ffn_stage.prepare(hidden_states, forward_batch)
with self.ffn_stage.exit(forward_batch) as ffn_exit:
hidden_states = self.attn_boundary.finish(hidden_states, forward_batch)
hidden_states = self.ffn_boundary.prepare(hidden_states, forward_batch)
with self.ffn_boundary.exit(forward_batch) as ffn_exit:
hidden_states = self.mlp(hidden_states)
hidden_states = ffn_exit.finish(hidden_states)
return hidden_states
+9 -9
View File
@@ -31,10 +31,10 @@ from sglang.srt.layers.attention.hybrid_linear_attn_backend import (
Mamba2AttnBackend,
)
from sglang.srt.layers.attention.mamba.mamba import MambaMixer2
from sglang.srt.layers.communicator.residual import batch as residual_batch
from sglang.srt.layers.dp_attention import (
is_dp_attention_enabled,
)
from sglang.srt.layers.layer_boundary.residual import batch as residual_batch
from sglang.srt.layers.layernorm import RMSNorm
from sglang.srt.layers.linear import (
ColumnParallelLinear,
@@ -376,7 +376,7 @@ class NemotronHMLPLikeDecoderLayer(nn.Module):
sum an FFN before it left."""
def _init_stage_boundary(self, config: NemotronHConfig, layer_idx: int):
self.stage = make_stage_boundary(
self.boundary = make_stage_boundary(
self.norm, pattern=config.hybrid_override_pattern, layer_idx=layer_idx
)
@@ -386,8 +386,8 @@ class NemotronHMLPLikeDecoderLayer(nn.Module):
hidden_states,
forward_batch: ForwardBatch,
):
hidden_states = self.stage.prepare(hidden_states, forward_batch)
with self.stage.exit(forward_batch) as ffn_exit:
hidden_states = self.boundary.prepare(hidden_states, forward_batch)
with self.boundary.exit(forward_batch) as ffn_exit:
hidden_states = self.mixer.forward(hidden_states)
return ffn_exit.finish(hidden_states)
@@ -455,7 +455,7 @@ class NemotronHAttnLikeDecoderLayer(nn.Module):
itself unless the fused kernel takes the sum into the next input norm."""
def _init_stage_boundary(self, config: NemotronHConfig, layer_idx: int):
self.stage = make_stage_boundary(
self.boundary = make_stage_boundary(
self.norm, pattern=config.hybrid_override_pattern, layer_idx=layer_idx
)
@@ -465,11 +465,11 @@ class NemotronHAttnLikeDecoderLayer(nn.Module):
hidden_states,
forward_batch: ForwardBatch,
):
hidden_states = self.stage.prepare(hidden_states, forward_batch)
hidden_states = self.boundary.prepare(hidden_states, forward_batch)
if forward_batch.forward_mode.is_idle():
return hidden_states
with self.stage.exit(forward_batch) as mixer_exit:
with self.boundary.exit(forward_batch) as mixer_exit:
hidden_states = self._forward_mixer(
hidden_states, forward_batch, mixer_exit.skips_reduction
)
@@ -730,14 +730,14 @@ class NemotronHModel(nn.Module):
residual_batch.start(forward_batch)
else:
assert pp_proxy_tensors is not None
hidden_states = self.layers[self.start_layer].stage.from_pp(
hidden_states = self.layers[self.start_layer].boundary.from_pp(
pp_proxy_tensors, forward_batch
)
aux_hidden_states = []
for i in range(self.start_layer, self.end_layer):
if i in self.layers_to_capture:
hidden_states, snapshot = self.layers[i].stage.capture_output(
hidden_states, snapshot = self.layers[i].boundary.capture_output(
hidden_states, forward_batch
)
aux_hidden_states.append(snapshot)
+1 -1
View File
@@ -18,8 +18,8 @@ import torch
from torch import nn
from sglang.srt.configs import NemotronHConfig
from sglang.srt.layers.communicator.residual import batch as residual_batch
from sglang.srt.layers.dp_attention import is_dp_attention_enabled
from sglang.srt.layers.layer_boundary.residual import batch as residual_batch
from sglang.srt.layers.layernorm import RMSNorm
from sglang.srt.layers.linear import ColumnParallelLinear
from sglang.srt.layers.logits_processor import LogitsProcessor
+2 -2
View File
@@ -1,14 +1,14 @@
"""Layer-communication helpers for the Nemotron-H model."""
from sglang.srt.configs.nemotron_h import ATTENTION, MAMBA, MOE
from sglang.srt.layers.communicator import (
from sglang.srt.layers.layer_boundary import (
HandoffRows,
ProducerReduction,
declare_attn,
declare_ffn,
make_stages,
)
from sglang.srt.layers.communicator.residual.add_norm import NormQuantRead, NormRead
from sglang.srt.layers.layer_boundary.residual.add_norm import NormQuantRead, NormRead
from sglang.srt.layers.layernorm import RMSNorm
from sglang.srt.runtime_context import get_parallel
+2 -2
View File
@@ -26,12 +26,12 @@ from sglang.srt.distributed import (
get_pp_indices,
)
from sglang.srt.layers.activation import SiluAndMul
from sglang.srt.layers.communicator.residual.access import (
from sglang.srt.layers.dp_attention import is_dp_attention_enabled
from sglang.srt.layers.layer_boundary.residual.access import (
finish_layer_stack,
from_pp,
snapshot,
)
from sglang.srt.layers.dp_attention import is_dp_attention_enabled
from sglang.srt.layers.layernorm import RMSNorm
from sglang.srt.layers.linear import (
MergedColumnParallelLinear,
+13 -13
View File
@@ -44,16 +44,16 @@ from sglang.srt.layers.aux_hidden_states import (
AuxHiddenStateAccumulator,
AuxHiddenStateList,
)
from sglang.srt.layers.communicator import (
declare_attn,
declare_ffn,
make_stages,
)
from sglang.srt.layers.communicator.residual import batch as residual_batch
from sglang.srt.layers.dp_attention import (
is_dp_attention_enabled,
)
from sglang.srt.layers.flashinfer_comm_fusion import uses_cutedsl_ar_fusion
from sglang.srt.layers.layer_boundary import (
declare_attn,
declare_ffn,
make_stages,
)
from sglang.srt.layers.layer_boundary.residual import batch as residual_batch
from sglang.srt.layers.layernorm import RMSNorm
from sglang.srt.layers.linear import (
MergedColumnParallelLinear,
@@ -865,7 +865,7 @@ class Qwen2MoeSparseMoeBlock(nn.Module):
if defer_finalize:
if shared_output is None:
raise RuntimeError("Qwen deferred finalize requires shared output")
from sglang.srt.layers.communicator.fusions.cutedsl import (
from sglang.srt.layers.layer_boundary.fusions.cutedsl import (
MoeFinalizeHandoff,
)
@@ -1053,7 +1053,7 @@ class Qwen2MoeDecoderLayer(nn.Module):
self.post_attention_layernorm = RMSNorm(
config.hidden_size, eps=config.rms_norm_eps
)
self.attn_stage, self.ffn_stage = make_stages(
self.attn_boundary, self.ffn_boundary = make_stages(
(declare_attn(), self.input_layernorm),
(
declare_ffn(
@@ -1079,7 +1079,7 @@ class Qwen2MoeDecoderLayer(nn.Module):
**kwargs,
) -> torch.Tensor:
hidden_states = self.attn_stage.prepare(
hidden_states = self.attn_boundary.prepare(
hidden_states,
forward_batch,
captured_last_layer_outputs=captured_last_layer_outputs,
@@ -1093,10 +1093,10 @@ class Qwen2MoeDecoderLayer(nn.Module):
forward_batch=forward_batch,
)
hidden_states = self.attn_stage.finish(hidden_states, forward_batch)
hidden_states = self.ffn_stage.prepare(hidden_states, forward_batch)
hidden_states = self.attn_boundary.finish(hidden_states, forward_batch)
hidden_states = self.ffn_boundary.prepare(hidden_states, forward_batch)
with self.ffn_stage.exit(forward_batch) as ffn_exit:
with self.ffn_boundary.exit(forward_batch) as ffn_exit:
hidden_states = self.mlp(hidden_states, forward_batch)
hidden_states = ffn_exit.finish(hidden_states)
@@ -1180,7 +1180,7 @@ class Qwen2MoeModel(nn.Module):
residual_batch.start(forward_batch)
else:
assert pp_proxy_tensors is not None
hidden_states = self.layers[self.start_layer].attn_stage.from_pp(
hidden_states = self.layers[self.start_layer].attn_boundary.from_pp(
pp_proxy_tensors, forward_batch
)
+8 -8
View File
@@ -6,12 +6,12 @@ import torch
from torch import nn
from sglang.srt.layers.aux_hidden_states import AuxHiddenStateList
from sglang.srt.layers.communicator import (
from sglang.srt.layers.layer_boundary import (
declare_attn,
declare_ffn,
make_stages,
)
from sglang.srt.layers.communicator.residual import batch as residual_batch
from sglang.srt.layers.layer_boundary.residual import batch as residual_batch
from sglang.srt.layers.layernorm import RMSNorm
from sglang.srt.layers.linear import QKVParallelLinear, RowParallelLinear
from sglang.srt.layers.logits_processor import LogitsProcessor
@@ -376,7 +376,7 @@ class Qwen3DecoderLayer(nn.Module):
config.hidden_size, eps=config.rms_norm_eps, **norm_kwargs
)
self.attn_stage, self.ffn_stage = make_stages(
self.attn_boundary, self.ffn_boundary = make_stages(
(declare_attn(), self.input_layernorm),
(
declare_ffn(sparse=False, next_sparse=False),
@@ -395,7 +395,7 @@ class Qwen3DecoderLayer(nn.Module):
capture_output=None,
) -> torch.Tensor:
# Self Attention
hidden_states = self.attn_stage.prepare(
hidden_states = self.attn_boundary.prepare(
hidden_states,
forward_batch,
post_residual_addition=post_residual_addition,
@@ -409,8 +409,8 @@ class Qwen3DecoderLayer(nn.Module):
)
# Fully Connected
hidden_states = self.attn_stage.finish(hidden_states, forward_batch)
hidden_states = self.ffn_stage.prepare(
hidden_states = self.attn_boundary.finish(hidden_states, forward_batch)
hidden_states = self.ffn_boundary.prepare(
hidden_states,
forward_batch,
cache=(
@@ -424,7 +424,7 @@ class Qwen3DecoderLayer(nn.Module):
else None
),
)
with self.ffn_stage.exit(forward_batch) as ffn_exit:
with self.ffn_boundary.exit(forward_batch) as ffn_exit:
hidden_states = self.mlp(hidden_states, forward_batch=forward_batch)
if _is_npu and get_cmo_stream():
wait_cmo_stream()
@@ -462,7 +462,7 @@ class Qwen3Model(Qwen2Model):
)
residual_batch.start(forward_batch)
else:
hidden_states = self.layers[self.start_layer].attn_stage.from_pp(
hidden_states = self.layers[self.start_layer].attn_boundary.from_pp(
pp_proxy_tensors, forward_batch
)
aux_hidden_states = AuxHiddenStateList()
+20 -20
View File
@@ -46,21 +46,21 @@ from sglang.srt.layers.aux_hidden_states import (
AuxHiddenStateAccumulator,
AuxHiddenStateList,
)
from sglang.srt.layers.communicator import (
from sglang.srt.layers.dp_attention import (
is_dp_attention_enabled,
)
from sglang.srt.layers.flashinfer_comm_fusion import uses_cutedsl_ar_fusion
from sglang.srt.layers.layer_boundary import (
declare_attn,
declare_ffn,
make_stages,
)
# Layers - Attention
from sglang.srt.layers.communicator.residual import batch as residual_batch
from sglang.srt.layers.layer_boundary.residual import batch as residual_batch
# Layers - Others
from sglang.srt.layers.communicator.residual.add_norm import Fp8Input, NormQuantRead
from sglang.srt.layers.dp_attention import (
is_dp_attention_enabled,
)
from sglang.srt.layers.flashinfer_comm_fusion import uses_cutedsl_ar_fusion
from sglang.srt.layers.layer_boundary.residual.add_norm import Fp8Input, NormQuantRead
from sglang.srt.layers.layernorm import GemmaRMSNorm
# Layers - Linear
@@ -224,7 +224,7 @@ def _use_mnnvl_cutedsl_fusion(config: Qwen3_5TextConfig, is_nextn: bool) -> bool
def _layer_fusions(config: Qwen3_5TextConfig, is_nextn: bool):
"""The CuTe DSL kernels a layer tries before its own, when they are on."""
if _use_mnnvl_cutedsl_fusion(config, is_nextn):
from sglang.srt.layers.communicator.fusions.cutedsl import CuteDSLFusion
from sglang.srt.layers.layer_boundary.fusions.cutedsl import CuteDSLFusion
return CuteDSLFusion()
return None
@@ -1094,7 +1094,7 @@ class Qwen3_5LinearDecoderLayer(nn.Module):
# it. Otherwise, stay on the plain AR+RMSNorm path.
accepts_fp8_input = _linear_accepts_fp8_tuple(self.linear_attn.in_proj_qkvz)
boundary_fusions = _layer_fusions(config, is_nextn)
self.attn_stage, self.ffn_stage = make_stages(
self.attn_boundary, self.ffn_boundary = make_stages(
(
declare_attn(
read=NormQuantRead(
@@ -1129,7 +1129,7 @@ class Qwen3_5LinearDecoderLayer(nn.Module):
):
forward_batch = kwargs.get("forward_batch", None)
hidden_states = self.attn_stage.prepare(
hidden_states = self.attn_boundary.prepare(
hidden_states,
forward_batch,
captured_last_layer_outputs=kwargs.get("captured_last_layer_outputs", None),
@@ -1144,10 +1144,10 @@ class Qwen3_5LinearDecoderLayer(nn.Module):
)
# Fully Connected
hidden_states = self.attn_stage.finish(hidden_states, forward_batch)
hidden_states = self.ffn_stage.prepare(hidden_states, forward_batch)
hidden_states = self.attn_boundary.finish(hidden_states, forward_batch)
hidden_states = self.ffn_boundary.prepare(hidden_states, forward_batch)
with self.ffn_stage.exit(forward_batch) as ffn_exit:
with self.ffn_boundary.exit(forward_batch) as ffn_exit:
if isinstance(self.mlp, Qwen2MoeSparseMoeBlock):
hidden_states = self.mlp(
hidden_states,
@@ -1300,7 +1300,7 @@ class Qwen3_5AttentionDecoderLayer(nn.Module):
# when qkv_proj can consume the returned quantized tuple.
accepts_fp8_input = _linear_accepts_fp8_tuple(self.qkv_proj)
boundary_fusions = _layer_fusions(config, is_nextn)
self.attn_stage, self.ffn_stage = make_stages(
self.attn_boundary, self.ffn_boundary = make_stages(
(
declare_attn(
read=NormQuantRead(
@@ -1533,7 +1533,7 @@ class Qwen3_5AttentionDecoderLayer(nn.Module):
captured_last_layer_outputs: Optional[AuxHiddenStateAccumulator] = None,
**kwargs,
):
hidden_states = self.attn_stage.prepare(
hidden_states = self.attn_boundary.prepare(
hidden_states,
forward_batch,
captured_last_layer_outputs=captured_last_layer_outputs,
@@ -1549,9 +1549,9 @@ class Qwen3_5AttentionDecoderLayer(nn.Module):
)
# Fully Connected
hidden_states = self.attn_stage.finish(hidden_states, forward_batch)
hidden_states = self.ffn_stage.prepare(hidden_states, forward_batch)
with self.ffn_stage.exit(forward_batch) as ffn_exit:
hidden_states = self.attn_boundary.finish(hidden_states, forward_batch)
hidden_states = self.ffn_boundary.prepare(hidden_states, forward_batch)
with self.ffn_boundary.exit(forward_batch) as ffn_exit:
if isinstance(self.mlp, Qwen2MoeSparseMoeBlock):
hidden_states = self.mlp(
hidden_states,
@@ -1716,7 +1716,7 @@ class Qwen3_5ForCausalLM(nn.Module):
"layers: "
f"{unsupported_layers}"
)
from sglang.srt.layers.communicator.fusions.cutedsl import (
from sglang.srt.layers.layer_boundary.fusions.cutedsl import (
install_cutedsl_fusion,
)
@@ -1807,7 +1807,7 @@ class Qwen3_5ForCausalLM(nn.Module):
residual_batch.start(forward_batch)
else:
assert pp_proxy_tensors is not None
hidden_states = self.layers[self.start_layer].attn_stage.from_pp(
hidden_states = self.layers[self.start_layer].attn_boundary.from_pp(
pp_proxy_tensors, forward_batch
)
+11 -11
View File
@@ -30,12 +30,12 @@ from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_r
from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation
from sglang.srt.eplb.expert_location_dispatch import ExpertLocationDispatchInfo
from sglang.srt.layers.aux_hidden_states import AuxHiddenStateAccumulator
from sglang.srt.layers.communicator import (
from sglang.srt.layers.layer_boundary import (
declare_attn,
declare_ffn,
make_stages,
)
from sglang.srt.layers.communicator.residual import batch as residual_batch
from sglang.srt.layers.layer_boundary.residual import batch as residual_batch
from sglang.srt.layers.layernorm import RMSNorm
from sglang.srt.layers.linear import (
QKVParallelLinear,
@@ -866,7 +866,7 @@ class Qwen3MoeDecoderLayer(nn.Module):
config.hidden_size, eps=config.rms_norm_eps
)
self.attn_stage, self.ffn_stage = make_stages(
self.attn_boundary, self.ffn_boundary = make_stages(
(declare_attn(), self.input_layernorm),
(
declare_ffn(
@@ -893,7 +893,7 @@ class Qwen3MoeDecoderLayer(nn.Module):
**kwargs,
) -> torch.Tensor:
hidden_states = self.attn_stage.prepare(
hidden_states = self.attn_boundary.prepare(
hidden_states,
forward_batch,
captured_last_layer_outputs=captured_last_layer_outputs,
@@ -908,10 +908,10 @@ class Qwen3MoeDecoderLayer(nn.Module):
forward_batch=forward_batch,
)
hidden_states = self.attn_stage.finish(hidden_states, forward_batch)
hidden_states = self.ffn_stage.prepare(hidden_states, forward_batch)
hidden_states = self.attn_boundary.finish(hidden_states, forward_batch)
hidden_states = self.ffn_boundary.prepare(hidden_states, forward_batch)
with self.ffn_stage.exit(forward_batch) as ffn_exit:
with self.ffn_boundary.exit(forward_batch) as ffn_exit:
hidden_states = self.mlp(hidden_states, forward_batch)
hidden_states = ffn_exit.finish(hidden_states)
@@ -925,7 +925,7 @@ class Qwen3MoeDecoderLayer(nn.Module):
forward_batch: ForwardBatch,
tbo_subbatch_index: Optional[int] = None,
):
state.hidden_states_after_comm_pre_attn = self.attn_stage.prepare(
state.hidden_states_after_comm_pre_attn = self.attn_boundary.prepare(
hidden_states, forward_batch
)
state.update(
@@ -937,15 +937,15 @@ class Qwen3MoeDecoderLayer(nn.Module):
)
def op_comm_prepare_mlp(self, state):
hidden_states = self.attn_stage.finish(
hidden_states = self.attn_boundary.finish(
state.pop("hidden_states_after_attn"), state.forward_batch
)
state.hidden_states_mlp_input = self.ffn_stage.prepare(
state.hidden_states_mlp_input = self.ffn_boundary.prepare(
hidden_states, state.forward_batch
)
def op_comm_postprocess_layer(self, state):
hidden_states = self.ffn_stage.postprocess(
hidden_states = self.ffn_boundary.postprocess(
state.pop("hidden_states_mlp_output"), state.forward_batch
)
+12 -12
View File
@@ -19,15 +19,15 @@ from sglang.srt.layers.aux_hidden_states import (
AuxHiddenStateAccumulator,
AuxHiddenStateList,
)
from sglang.srt.layers.communicator import (
from sglang.srt.layers.dp_attention import (
is_dp_attention_enabled,
)
from sglang.srt.layers.layer_boundary import (
declare_attn,
declare_ffn,
make_stages,
)
from sglang.srt.layers.communicator.residual import batch as residual_batch
from sglang.srt.layers.dp_attention import (
is_dp_attention_enabled,
)
from sglang.srt.layers.layer_boundary.residual import batch as residual_batch
from sglang.srt.layers.layernorm import GemmaRMSNorm
from sglang.srt.layers.linear import (
ColumnParallelLinear,
@@ -477,9 +477,9 @@ def _apply_qwen3_next_mlp(
hidden_states: torch.Tensor,
forward_batch: ForwardBatch,
) -> torch.Tensor:
hidden_states = layer.attn_stage.finish(hidden_states, forward_batch)
hidden_states = layer.ffn_stage.prepare(hidden_states, forward_batch)
with layer.ffn_stage.exit(forward_batch) as ffn_exit:
hidden_states = layer.attn_boundary.finish(hidden_states, forward_batch)
hidden_states = layer.ffn_boundary.prepare(hidden_states, forward_batch)
with layer.ffn_boundary.exit(forward_batch) as ffn_exit:
if isinstance(layer.mlp, Qwen2MoeSparseMoeBlock):
hidden_states = layer.mlp(
hidden_states,
@@ -537,7 +537,7 @@ class Qwen3HybridLinearDecoderLayer(nn.Module):
self.post_attention_layernorm = GemmaRMSNorm(
config.hidden_size, eps=config.rms_norm_eps
)
self.attn_stage, self.ffn_stage = make_stages(
self.attn_boundary, self.ffn_boundary = make_stages(
(declare_attn(), self.input_layernorm),
(
declare_ffn(
@@ -562,7 +562,7 @@ class Qwen3HybridLinearDecoderLayer(nn.Module):
):
forward_batch = kwargs.get("forward_batch", None)
hidden_states = self.attn_stage.prepare(
hidden_states = self.attn_boundary.prepare(
hidden_states,
forward_batch,
captured_last_layer_outputs=captured_last_layer_outputs,
@@ -705,7 +705,7 @@ class Qwen3HybridAttentionDecoderLayer(nn.Module):
self.q_norm = GemmaRMSNorm(self.head_dim, eps=config.rms_norm_eps)
self.k_norm = GemmaRMSNorm(self.head_dim, eps=config.rms_norm_eps)
self.attn_stage, self.ffn_stage = make_stages(
self.attn_boundary, self.ffn_boundary = make_stages(
(declare_attn(), self.input_layernorm),
(
declare_ffn(
@@ -832,7 +832,7 @@ class Qwen3HybridAttentionDecoderLayer(nn.Module):
captured_last_layer_outputs: Optional[AuxHiddenStateAccumulator] = None,
**kwargs: Any,
):
hidden_states = self.attn_stage.prepare(
hidden_states = self.attn_boundary.prepare(
hidden_states,
forward_batch,
captured_last_layer_outputs=captured_last_layer_outputs,
+2 -2
View File
@@ -38,11 +38,11 @@ from sglang.srt.layers.attention.vision import (
prepare_vision_attention_metadata,
)
from sglang.srt.layers.aux_hidden_states import AuxHiddenStateList
from sglang.srt.layers.communicator.residual import batch as residual_batch
from sglang.srt.layers.conv import Conv3dLayer
from sglang.srt.layers.dp_attention import (
is_dp_attention_enabled,
)
from sglang.srt.layers.layer_boundary.residual import batch as residual_batch
from sglang.srt.layers.linear import ColumnParallelLinear, RowParallelLinear
from sglang.srt.layers.logits_processor import LogitsProcessor
from sglang.srt.layers.pooler import Pooler, PoolingType
@@ -1209,7 +1209,7 @@ class Qwen3LLMModel(Qwen3Model):
residual_batch.start(forward_batch)
else:
assert pp_proxy_tensors is not None
hidden_states = self.layers[self.start_layer].attn_stage.from_pp(
hidden_states = self.layers[self.start_layer].attn_boundary.from_pp(
pp_proxy_tensors, forward_batch
)
+2 -2
View File
@@ -25,7 +25,7 @@ import torch.nn as nn
from sglang.srt.configs.qwen3_vl import Qwen3VLMoeConfig, Qwen3VLMoeTextConfig
from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation
from sglang.srt.layers.aux_hidden_states import AuxHiddenStateList
from sglang.srt.layers.communicator.residual import batch as residual_batch
from sglang.srt.layers.layer_boundary.residual import batch as residual_batch
from sglang.srt.layers.moe.fused_moe_triton.layer import FusedMoE
from sglang.srt.layers.quantization.base_config import QuantizationConfig
from sglang.srt.layers.utils import get_layer_id
@@ -99,7 +99,7 @@ class Qwen3MoeLLMModel(Qwen3MoeModel):
residual_batch.start(forward_batch)
else:
assert pp_proxy_tensors is not None
hidden_states = self.layers[self.start_layer].attn_stage.from_pp(
hidden_states = self.layers[self.start_layer].attn_boundary.from_pp(
pp_proxy_tensors, forward_batch
)
+3 -3
View File
@@ -22,7 +22,6 @@ from sglang.srt.environ import envs
from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder
from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation
from sglang.srt.layers.aux_hidden_states import AuxHiddenStateList
from sglang.srt.layers.communicator import get_attn_tp_context
from sglang.srt.layers.dp_attention import (
attn_tp_all_gather,
attn_tp_all_reduce,
@@ -38,6 +37,7 @@ from sglang.srt.layers.hyperconnection import (
GatedResidual,
HyperConnectionConfig,
)
from sglang.srt.layers.layer_boundary import get_attn_tp_context
from sglang.srt.layers.linear import ReplicatedLinear
from sglang.srt.layers.logits_processor import LogitsProcessorOutput
from sglang.srt.layers.moe import get_moe_a2a_backend, should_use_dp_reduce_scatterv
@@ -1320,8 +1320,8 @@ class Qwen4ExpLayerExtensionMixin:
for attr_name in (
"input_layernorm",
"post_attention_layernorm",
"attn_stage",
"ffn_stage",
"attn_boundary",
"ffn_boundary",
):
if hasattr(self, attr_name):
delattr(self, attr_name)
+11 -11
View File
@@ -19,16 +19,16 @@ from sglang.srt.distributed import (
from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder
from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation
from sglang.srt.layers.activation import SiluAndMul
from sglang.srt.layers.communicator import (
from sglang.srt.layers.dp_attention import (
is_dp_attention_enabled,
)
from sglang.srt.layers.layer_boundary import (
declare_attn,
declare_ffn,
enable_moe_dense_fully_dp,
make_stages,
)
from sglang.srt.layers.communicator.residual import batch as residual_batch
from sglang.srt.layers.dp_attention import (
is_dp_attention_enabled,
)
from sglang.srt.layers.layer_boundary.residual import batch as residual_batch
from sglang.srt.layers.layernorm import RMSNorm
from sglang.srt.layers.linear import (
ColumnParallelLinear,
@@ -1050,7 +1050,7 @@ class SarvamMoEMLADecoderLayer(nn.Module):
self.attn_tp_size = get_parallel().attn_tp_size
self.attn_stage, self.ffn_stage = make_stages(
self.attn_boundary, self.ffn_boundary = make_stages(
(
declare_attn(),
self.input_layernorm,
@@ -1077,16 +1077,16 @@ class SarvamMoEMLADecoderLayer(nn.Module):
hidden_states: torch.Tensor,
forward_batch: ForwardBatch,
) -> torch.Tensor:
hidden_states = self.attn_stage.prepare(hidden_states, forward_batch)
hidden_states = self.attn_boundary.prepare(hidden_states, forward_batch)
if hidden_states.shape[0] != 0:
hidden_states = self.self_attn(
positions=positions,
hidden_states=hidden_states,
forward_batch=forward_batch,
)
hidden_states = self.attn_stage.finish(hidden_states, forward_batch)
hidden_states = self.ffn_stage.prepare(hidden_states, forward_batch)
with self.ffn_stage.exit(forward_batch) as ffn_exit:
hidden_states = self.attn_boundary.finish(hidden_states, forward_batch)
hidden_states = self.ffn_boundary.prepare(hidden_states, forward_batch)
with self.ffn_boundary.exit(forward_batch) as ffn_exit:
hidden_states = self.mlp(hidden_states, forward_batch)
if (
not self.is_layer_sparse
@@ -1159,7 +1159,7 @@ class SarvamMLAModel(nn.Module):
residual_batch.start(forward_batch)
else:
assert pp_proxy_tensors is not None
hidden_states = self.layers[self.start_layer].attn_stage.from_pp(
hidden_states = self.layers[self.start_layer].attn_boundary.from_pp(
pp_proxy_tensors, forward_batch
)
+11 -11
View File
@@ -11,15 +11,15 @@ from torch import nn
from transformers import PretrainedConfig
from sglang.srt.layers.activation import SiluAndMul
from sglang.srt.layers.communicator import (
from sglang.srt.layers.dp_attention import (
is_dp_attention_enabled,
)
from sglang.srt.layers.layer_boundary import (
declare_attn,
declare_ffn,
make_stages,
)
from sglang.srt.layers.communicator.residual import batch as residual_batch
from sglang.srt.layers.dp_attention import (
is_dp_attention_enabled,
)
from sglang.srt.layers.layer_boundary.residual import batch as residual_batch
from sglang.srt.layers.layernorm import RMSNorm
from sglang.srt.layers.linear import (
MergedColumnParallelLinear,
@@ -295,7 +295,7 @@ class SDARBlock(nn.Module):
prefix=add_prefix("mlp", prefix),
)
self.attn_stage, self.ffn_stage = make_stages(
self.attn_boundary, self.ffn_boundary = make_stages(
(declare_attn(), self.input_layernorm),
(
declare_ffn(
@@ -314,7 +314,7 @@ class SDARBlock(nn.Module):
hidden_states: torch.Tensor,
forward_batch: ForwardBatch,
) -> torch.Tensor:
hidden_states = self.attn_stage.prepare(hidden_states, forward_batch)
hidden_states = self.attn_boundary.prepare(hidden_states, forward_batch)
if hidden_states.shape[0] != 0:
hidden_states = self.self_attn(
@@ -323,10 +323,10 @@ class SDARBlock(nn.Module):
forward_batch=forward_batch,
)
hidden_states = self.attn_stage.finish(hidden_states, forward_batch)
hidden_states = self.ffn_stage.prepare(hidden_states, forward_batch)
hidden_states = self.attn_boundary.finish(hidden_states, forward_batch)
hidden_states = self.ffn_boundary.prepare(hidden_states, forward_batch)
with self.ffn_stage.exit(forward_batch) as ffn_exit:
with self.ffn_boundary.exit(forward_batch) as ffn_exit:
hidden_states = self.mlp(hidden_states)
hidden_states = ffn_exit.finish(hidden_states)
@@ -401,7 +401,7 @@ class SDARModel(nn.Module):
residual_batch.start(forward_batch)
else:
assert pp_proxy_tensors is not None
hidden_states = self.layers[self.start_layer].attn_stage.from_pp(
hidden_states = self.layers[self.start_layer].attn_boundary.from_pp(
pp_proxy_tensors, forward_batch, allow_missing_residual=True
)
+11 -11
View File
@@ -13,15 +13,15 @@ from transformers import PretrainedConfig
from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder
from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation
from sglang.srt.eplb.expert_location_dispatch import ExpertLocationDispatchInfo
from sglang.srt.layers.communicator import (
from sglang.srt.layers.dp_attention import (
is_dp_attention_enabled,
)
from sglang.srt.layers.layer_boundary import (
declare_attn,
declare_ffn,
make_stages,
)
from sglang.srt.layers.communicator.residual import batch as residual_batch
from sglang.srt.layers.dp_attention import (
is_dp_attention_enabled,
)
from sglang.srt.layers.layer_boundary.residual import batch as residual_batch
from sglang.srt.layers.layernorm import RMSNorm
from sglang.srt.layers.linear import (
QKVParallelLinear,
@@ -357,7 +357,7 @@ class SDARMoeBlock(nn.Module):
prefix=add_prefix("mlp", prefix),
)
self.attn_stage, self.ffn_stage = make_stages(
self.attn_boundary, self.ffn_boundary = make_stages(
(declare_attn(), self.input_layernorm),
(
declare_ffn(sparse=True, next_sparse=True),
@@ -376,7 +376,7 @@ class SDARMoeBlock(nn.Module):
forward_batch: ForwardBatch,
) -> torch.Tensor:
hidden_states = self.attn_stage.prepare(hidden_states, forward_batch)
hidden_states = self.attn_boundary.prepare(hidden_states, forward_batch)
if hidden_states.shape[0] != 0:
hidden_states = self.self_attn(
@@ -385,10 +385,10 @@ class SDARMoeBlock(nn.Module):
forward_batch=forward_batch,
)
hidden_states = self.attn_stage.finish(hidden_states, forward_batch)
hidden_states = self.ffn_stage.prepare(hidden_states, forward_batch)
hidden_states = self.attn_boundary.finish(hidden_states, forward_batch)
hidden_states = self.ffn_boundary.prepare(hidden_states, forward_batch)
with self.ffn_stage.exit(forward_batch) as ffn_exit:
with self.ffn_boundary.exit(forward_batch) as ffn_exit:
hidden_states = self.mlp(
hidden_states,
forward_batch=forward_batch,
@@ -467,7 +467,7 @@ class SDARMoeModel(nn.Module):
residual_batch.start(forward_batch)
else:
assert pp_proxy_tensors is not None
hidden_states = self.layers[self.start_layer].attn_stage.from_pp(
hidden_states = self.layers[self.start_layer].attn_boundary.from_pp(
pp_proxy_tensors, forward_batch
)
+11 -11
View File
@@ -19,16 +19,16 @@ from sglang.srt.configs.step3_vl import (
from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation
from sglang.srt.layers.activation import SiluAndMul
from sglang.srt.layers.attention.vision import VisionAttention
from sglang.srt.layers.communicator import (
declare_attn,
declare_ffn,
make_stages,
)
from sglang.srt.layers.communicator.residual import batch as residual_batch
from sglang.srt.layers.conv import Conv2dLayer
from sglang.srt.layers.dp_attention import (
is_dp_attention_enabled,
)
from sglang.srt.layers.layer_boundary import (
declare_attn,
declare_ffn,
make_stages,
)
from sglang.srt.layers.layer_boundary.residual import batch as residual_batch
from sglang.srt.layers.layernorm import RMSNorm
from sglang.srt.layers.linear import (
ColumnParallelLinear,
@@ -374,7 +374,7 @@ class Step3TextDecoderLayer(nn.Module):
prefix=add_prefix("mlp", prefix),
)
self.attn_stage, self.ffn_stage = make_stages(
self.attn_boundary, self.ffn_boundary = make_stages(
(declare_attn(), self.input_layernorm),
(
declare_ffn(
@@ -407,7 +407,7 @@ class Step3TextDecoderLayer(nn.Module):
forward_batch: ForwardBatch,
) -> torch.Tensor:
hidden_states = self.attn_stage.prepare(hidden_states, forward_batch)
hidden_states = self.attn_boundary.prepare(hidden_states, forward_batch)
if hidden_states.shape[0] != 0:
hidden_states = self.self_attn(
@@ -416,9 +416,9 @@ class Step3TextDecoderLayer(nn.Module):
forward_batch=forward_batch,
)
hidden_states = self.attn_stage.finish(hidden_states, forward_batch)
hidden_states = self.ffn_stage.prepare(hidden_states, forward_batch)
with self.ffn_stage.exit(forward_batch) as ffn_exit:
hidden_states = self.attn_boundary.finish(hidden_states, forward_batch)
hidden_states = self.ffn_boundary.prepare(hidden_states, forward_batch)
with self.ffn_boundary.exit(forward_batch) as ffn_exit:
if self.use_moe:
hidden_states = self.moe_mlp_forward(hidden_states)
else:
+12 -12
View File
@@ -11,15 +11,15 @@ from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_r
from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation
from sglang.srt.eplb.expert_location_dispatch import ExpertLocationDispatchInfo
from sglang.srt.layers.activation import SiluAndMul
from sglang.srt.layers.communicator import (
from sglang.srt.layers.dp_attention import (
is_dp_attention_enabled,
)
from sglang.srt.layers.layer_boundary import (
declare_attn,
declare_ffn,
make_stages,
)
from sglang.srt.layers.communicator.residual import batch as residual_batch
from sglang.srt.layers.dp_attention import (
is_dp_attention_enabled,
)
from sglang.srt.layers.layer_boundary.residual import batch as residual_batch
from sglang.srt.layers.layernorm import GemmaRMSNorm
from sglang.srt.layers.linear import (
ColumnParallelLinear,
@@ -566,7 +566,7 @@ class Step3p5DecoderLayer(nn.Module):
# An MTP draft is a one-layer model; layer_id still indexes its config.
self.attn_stage, self.ffn_stage = make_stages(
self.attn_boundary, self.ffn_boundary = make_stages(
(declare_attn(), self.input_layernorm),
(
declare_ffn(
@@ -594,7 +594,7 @@ class Step3p5DecoderLayer(nn.Module):
post_residual_addition: Optional[torch.Tensor] = None,
) -> torch.Tensor:
# Self Attention
hidden_states = self.attn_stage.prepare(
hidden_states = self.attn_boundary.prepare(
hidden_states, forward_batch, post_residual_addition=post_residual_addition
)
if hidden_states.shape[0] != 0:
@@ -604,11 +604,11 @@ class Step3p5DecoderLayer(nn.Module):
forward_batch=forward_batch,
)
# Fully Connected
hidden_states = self.attn_stage.finish(hidden_states, forward_batch)
hidden_states = self.ffn_stage.prepare(hidden_states, forward_batch)
hidden_states = self.attn_boundary.finish(hidden_states, forward_batch)
hidden_states = self.ffn_boundary.prepare(hidden_states, forward_batch)
if self.use_moe:
with self.ffn_stage.exit(forward_batch) as ffn_exit:
with self.ffn_boundary.exit(forward_batch) as ffn_exit:
# Both share_expert and MoE return unreduced (TP-partial) outputs.
# Combine them first, then do a single all-reduce — saving one
# full-TP all-reduce per layer.
@@ -621,7 +621,7 @@ class Step3p5DecoderLayer(nn.Module):
hidden_states = tensor_model_parallel_all_reduce(hidden_states)
return ffn_exit.finish(hidden_states)
with self.ffn_stage.exit(forward_batch) as ffn_exit:
with self.ffn_boundary.exit(forward_batch) as ffn_exit:
hidden_states = self.mlp(hidden_states)
return ffn_exit.finish(hidden_states)
@@ -700,7 +700,7 @@ class Step3p5Model(nn.Module):
residual_batch.start(forward_batch)
else:
assert pp_proxy_tensors is not None
hidden_states = self.layers[self.start_layer].attn_stage.from_pp(
hidden_states = self.layers[self.start_layer].attn_boundary.from_pp(
pp_proxy_tensors, forward_batch
)
+1 -1
View File
@@ -6,7 +6,7 @@ import torch
import torch.nn as nn
from transformers import PretrainedConfig
from sglang.srt.layers.communicator.residual import batch as residual_batch
from sglang.srt.layers.layer_boundary.residual import batch as residual_batch
from sglang.srt.layers.layernorm import GemmaRMSNorm
from sglang.srt.layers.logits_processor import LogitsProcessor
from sglang.srt.layers.quantization.base_config import QuantizationConfig
+9 -9
View File
@@ -39,14 +39,14 @@ from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_r
from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation
from sglang.srt.eplb.expert_location_dispatch import ExpertLocationDispatchInfo
from sglang.srt.layers.activation import SiluAndMul
from sglang.srt.layers.communicator import (
from sglang.srt.layers.dp_attention import is_dp_attention_enabled
from sglang.srt.layers.layer_boundary import (
declare_attn,
declare_ffn,
enable_moe_dense_fully_dp,
make_stages,
)
from sglang.srt.layers.communicator.residual import batch as residual_batch
from sglang.srt.layers.dp_attention import is_dp_attention_enabled
from sglang.srt.layers.layer_boundary.residual import batch as residual_batch
from sglang.srt.layers.layernorm import RMSNorm
from sglang.srt.layers.linear import (
ColumnParallelLinear,
@@ -1528,7 +1528,7 @@ class XllmDecoderLayer(nn.Module):
self.input_layernorm = _make_norm(config)
self.post_attention_layernorm = _make_norm(config)
self.attn_stage, self.ffn_stage = make_stages(
self.attn_boundary, self.ffn_boundary = make_stages(
(declare_attn(), self.input_layernorm),
(
declare_ffn(
@@ -1551,7 +1551,7 @@ class XllmDecoderLayer(nn.Module):
hidden_states: torch.Tensor,
forward_batch: ForwardBatch,
) -> torch.Tensor:
hidden_states = self.attn_stage.prepare(hidden_states, forward_batch)
hidden_states = self.attn_boundary.prepare(hidden_states, forward_batch)
if hidden_states.shape[0] != 0:
hidden_states = self.self_attn(
@@ -1560,10 +1560,10 @@ class XllmDecoderLayer(nn.Module):
forward_batch=forward_batch,
)
hidden_states = self.attn_stage.finish(hidden_states, forward_batch)
hidden_states = self.ffn_stage.prepare(hidden_states, forward_batch)
hidden_states = self.attn_boundary.finish(hidden_states, forward_batch)
hidden_states = self.ffn_boundary.prepare(hidden_states, forward_batch)
with self.ffn_stage.exit(forward_batch) as ffn_exit:
with self.ffn_boundary.exit(forward_batch) as ffn_exit:
if isinstance(self.mlp, XllmMLP):
hidden_states = self.mlp(hidden_states)
else:
@@ -1629,7 +1629,7 @@ class XllmModel(nn.Module):
residual_batch.start(forward_batch)
else:
assert pp_proxy_tensors is not None
hidden_states = self.layers[self.start_layer].attn_stage.from_pp(
hidden_states = self.layers[self.start_layer].attn_boundary.from_pp(
pp_proxy_tensors, forward_batch
)
+12 -12
View File
@@ -2,15 +2,15 @@
from types import SimpleNamespace
from sglang.srt.layers.communicator.construction import StagePlan
from sglang.srt.layers.communicator.factories import (
from sglang.srt.layers.layer_boundary.construction import StagePlan
from sglang.srt.layers.layer_boundary.factories import (
declare_attn,
declare_ffn,
make_stages,
)
from sglang.srt.layers.communicator.ops import identity_output
from sglang.srt.layers.communicator.residual.add_norm import PLAIN_RESIDUAL
from sglang.srt.layers.communicator.stage import StageCommunicator
from sglang.srt.layers.layer_boundary.ops import identity_output
from sglang.srt.layers.layer_boundary.residual.add_norm import PLAIN_RESIDUAL
from sglang.srt.layers.layer_boundary.stage import StageBoundary
def make_test_stages(
@@ -73,7 +73,7 @@ def stub_stage(plan, kind):
"""Keep the same concrete boundary when a test patches its methods."""
stages = plan.__dict__.setdefault("_stub_stages", {})
if kind not in stages:
stages[kind] = StageCommunicator(
stages[kind] = StageBoundary(
plan,
declaration=(declare_attn() if kind.name == "ATTENTION" else declare_ffn()),
)
@@ -82,7 +82,7 @@ def stub_stage(plan, kind):
def sp_region_steps():
"""Local SP rows with no owed sum, for tests of activation and exits."""
from sglang.srt.layers.communicator import (
from sglang.srt.layers.layer_boundary import (
NORM_QUANT_READ,
EdgeDecl,
Layout,
@@ -93,7 +93,7 @@ def sp_region_steps():
TokenAxis,
make_boundary,
)
from sglang.srt.layers.communicator.prepare import _hand_qkv_hook_its_input
from sglang.srt.layers.layer_boundary.prepare import _hand_qkv_hook_its_input
rows = Layout(frozenset({TokenAxis.ATTN_TP_SCATTER}))
output = StageOutput(rows)
@@ -120,8 +120,8 @@ def prepare_attention(stage, hidden, residual, forward_batch, *args, **call):
def prepare_raw(stage, method, hidden, residual, forward_batch, *args, **call):
from sglang.srt.layers.communicator.residual.add_norm import ADD
from sglang.srt.layers.communicator.residual.stream import ResidualStream
from sglang.srt.layers.layer_boundary.residual.add_norm import ADD
from sglang.srt.layers.layer_boundary.residual.stream import ResidualStream
if not isinstance(residual, ResidualStream):
stream = ResidualStream(residual)
@@ -137,7 +137,7 @@ def prepare_raw(stage, method, hidden, residual, forward_batch, *args, **call):
def finish_exit(scope, hidden, residual):
from sglang.srt.layers.communicator.residual.stream import ResidualStream
from sglang.srt.layers.layer_boundary.residual.stream import ResidualStream
if isinstance(residual, ResidualStream):
scope._stream = residual
@@ -150,7 +150,7 @@ def finish_exit(scope, hidden, residual):
def postprocess_output(boundary, hidden, residual, forward_batch):
from sglang.srt.layers.communicator.residual.stream import ResidualStream
from sglang.srt.layers.layer_boundary.residual.stream import ResidualStream
stream = (
residual if isinstance(residual, ResidualStream) else ResidualStream(residual)
+8 -8
View File
@@ -1,6 +1,6 @@
"""Patching a name the communicator package's modules look up.
Each module of ``sglang.srt.layers.communicator`` binds the names it imports
Each module of ``sglang.srt.layers.layer_boundary`` binds the names it imports
in its own namespace, so a test double has to go into every module that reads
the name, not only into the package.
"""
@@ -8,8 +8,8 @@ the name, not only into the package.
import contextlib
from unittest import mock
from sglang.srt.layers import communicator as communicator
from sglang.srt.layers.communicator import (
from sglang.srt.layers import layer_boundary as communicator
from sglang.srt.layers.layer_boundary import (
boundary,
construction,
contracts,
@@ -20,13 +20,13 @@ from sglang.srt.layers.communicator import (
output,
prepare,
)
from sglang.srt.layers.communicator import residual as residual_contract
from sglang.srt.layers.communicator import (
from sglang.srt.layers.layer_boundary import residual as residual_contract
from sglang.srt.layers.layer_boundary import (
stage,
)
from sglang.srt.layers.communicator.adapters import attention, branch, lora, overlap
from sglang.srt.layers.communicator.fusions import allreduce
from sglang.srt.layers.communicator.residual import (
from sglang.srt.layers.layer_boundary.adapters import attention, branch, lora, overlap
from sglang.srt.layers.layer_boundary.fusions import allreduce
from sglang.srt.layers.layer_boundary.residual import (
access,
add_norm,
batch,
@@ -68,7 +68,7 @@ patches:
- target: sglang.srt.models.qwen3_moe.Qwen3MoeDecoderLayer.forward
edits:
- match: |
hidden_states = self.attn_stage.prepare(
hidden_states = self.attn_boundary.prepare(
hidden_states,
forward_batch,
captured_last_layer_outputs=captured_last_layer_outputs,
@@ -83,7 +83,7 @@ patches:
forward_batch=forward_batch,
)
append: "dumper.dump('attn_output', hidden_states, dims='t h[attn_tp:partial] # tp:replicated')"
- match: "hidden_states = self.ffn_stage.prepare(hidden_states, forward_batch)"
- match: "hidden_states = self.ffn_boundary.prepare(hidden_states, forward_batch)"
append: "dumper.dump('pre_mlp_residual', hidden_states, dims='t h # tp:replicated')"
- match: "hidden_states = self.mlp(hidden_states, forward_batch)"
append: "dumper.dump('mlp_output', hidden_states, dims='t h[moe_tp:partial] # tp:replicated')"
@@ -120,7 +120,7 @@ patches:
- target: sglang.srt.models.qwen3_moe.Qwen3MoeDecoderLayer.forward
edits:
- match: |
hidden_states = self.attn_stage.prepare(
hidden_states = self.attn_boundary.prepare(
hidden_states,
forward_batch,
captured_last_layer_outputs=captured_last_layer_outputs,
@@ -135,7 +135,7 @@ patches:
forward_batch=forward_batch,
)
append: "dumper.dump('attn_output', hidden_states, dims='t h # tp:replicated')"
- match: "hidden_states = self.ffn_stage.prepare(hidden_states, forward_batch)"
- match: "hidden_states = self.ffn_boundary.prepare(hidden_states, forward_batch)"
append: "dumper.dump('pre_mlp_residual', hidden_states, dims='t h # tp:replicated')"
- match: "hidden_states = self.mlp(hidden_states, forward_batch)"
append: "dumper.dump('mlp_output', hidden_states, dims='t h[moe_tp:partial] # tp:replicated')"
@@ -143,7 +143,7 @@ def test_mhc_fused_post_pre_matches_unfused(
def _check_glm_boundary(x, residual, post, comb, fn, scale, base, *, use_norm):
from sglang.srt.environ import envs
from sglang.srt.layers.communicator import MHCState
from sglang.srt.layers.layer_boundary import MHCState
from sglang.srt.layers.layernorm import RMSNorm
from sglang.srt.models.glm5_next import Glm5NextDecoderLayer
@@ -10,8 +10,8 @@ from unittest.mock import MagicMock, patch
import torch
from sglang.srt.layers.communicator import UnreducedOutput
from sglang.srt.layers.communicator.fusions.allreduce import complete_attention_input
from sglang.srt.layers.layer_boundary import UnreducedOutput
from sglang.srt.layers.layer_boundary.fusions.allreduce import complete_attention_input
from sglang.srt.layers.moe import utils as moe_utils
from sglang.srt.model_executor.forward_batch_info import ForwardMode
from sglang.srt.runtime_context import get_context, get_flags, get_parallel
@@ -9,8 +9,8 @@ from unittest.mock import patch
import torch
from sglang.srt.batch_overlap import two_batch_overlap as tbo
from sglang.srt.layers.communicator import ADD, Layout, UnreducedOutput
from sglang.srt.layers.communicator.residual.stream import ResidualStream
from sglang.srt.layers.layer_boundary import ADD, Layout, UnreducedOutput
from sglang.srt.layers.layer_boundary.residual.stream import ResidualStream
from sglang.srt.utils import empty_context
from sglang.test.ci.ci_register import register_cpu_ci
from sglang.test.test_utils import CustomTestCase
@@ -78,7 +78,7 @@ class TestTboEntryReducesItsInput(CustomTestCase):
merged = tbo.model_forward_stages(
layers=[
SimpleNamespace(
attn_stage=SimpleNamespace(
attn_boundary=SimpleNamespace(
incoming_residual_rows=Layout(frozenset())
)
)
@@ -6,7 +6,12 @@ from unittest.mock import MagicMock, patch
import pytest
import torch
from sglang.srt.layers.communicator import (
from sglang.srt.layers.flashinfer_mnnvl_cutedsl import (
FlashInferMNNVLCuteDSLARFusion,
_retargeted_config,
_with_early_finalize_shared_load,
)
from sglang.srt.layers.layer_boundary import (
ADD,
NORM_QUANT_READ,
Layout,
@@ -16,26 +21,21 @@ from sglang.srt.layers.communicator import (
UnreducedOutput,
reduce_output,
)
from sglang.srt.layers.communicator.construction import BatchVariant
from sglang.srt.layers.communicator.fusions.allreduce import (
from sglang.srt.layers.layer_boundary.construction import BatchVariant
from sglang.srt.layers.layer_boundary.fusions.allreduce import (
attention_fusions,
complete_attention_input,
complete_ffn_input,
ffn_fusions,
)
from sglang.srt.layers.communicator.fusions.cutedsl import (
from sglang.srt.layers.layer_boundary.fusions.cutedsl import (
CuteDSLFusion,
MoeFinalizeHandoff,
install_cutedsl_fusion,
prepare_cutedsl_fusion,
)
from sglang.srt.layers.communicator.prepare import _consumer_step, _read_input
from sglang.srt.layers.communicator.residual.access import finish_layer_stack
from sglang.srt.layers.flashinfer_mnnvl_cutedsl import (
FlashInferMNNVLCuteDSLARFusion,
_retargeted_config,
_with_early_finalize_shared_load,
)
from sglang.srt.layers.layer_boundary.prepare import _consumer_step, _read_input
from sglang.srt.layers.layer_boundary.residual.access import finish_layer_stack
from sglang.srt.layers.layernorm import RMSNorm
from sglang.srt.model_executor.cuda_graph_config import CudaGraphConfig, PhaseConfig
from sglang.srt.model_executor.forward_batch_info import ForwardMode
@@ -47,7 +47,7 @@ from sglang.test.communicator_patch import patch_communicator
register_cpu_ci(est_time=9, suite="base-a-test-cpu")
_MODULE = "sglang.srt.layers.communicator.fusions.cutedsl"
_MODULE = "sglang.srt.layers.layer_boundary.fusions.cutedsl"
_DECODE = SimpleNamespace(forward_mode=ForwardMode.DECODE, input_ids=torch.zeros(8))
@@ -91,8 +91,8 @@ def _bound(entry):
def _test_layer(**kwargs):
comm = _communicator()
return SimpleNamespace(
attn_stage=stub_stage(comm, StageKind.ATTENTION),
ffn_stage=stub_stage(comm, StageKind.FFN),
attn_boundary=stub_stage(comm, StageKind.ATTENTION),
ffn_boundary=stub_stage(comm, StageKind.FFN),
**kwargs,
)
@@ -244,17 +244,17 @@ def test_install_does_not_require_a_fused_successor():
reset_context()
publish(ServerArgs(model_path="dummy"), role="test")
first, last = _communicator(), _communicator()
ordinary = SimpleNamespace(ffn_stage=SimpleNamespace(fusions=None))
ordinary = SimpleNamespace(ffn_boundary=SimpleNamespace(fusions=None))
install_cutedsl_fusion(
[
SimpleNamespace(
attn_stage=stub_stage(first, StageKind.ATTENTION),
ffn_stage=stub_stage(first, StageKind.FFN),
attn_boundary=stub_stage(first, StageKind.ATTENTION),
ffn_boundary=stub_stage(first, StageKind.FFN),
),
ordinary,
SimpleNamespace(
attn_stage=stub_stage(last, StageKind.ATTENTION),
ffn_stage=stub_stage(last, StageKind.FFN),
attn_boundary=stub_stage(last, StageKind.ATTENTION),
ffn_boundary=stub_stage(last, StageKind.FFN),
),
],
hidden_size=8,
@@ -273,14 +273,14 @@ def test_a_service_nested_under_a_wrapper_is_prepared():
prepared = []
layer = torch.nn.Linear(2, 2)
comm = _communicator()
layer.attn_stage, layer.ffn_stage = (
layer.attn_boundary, layer.ffn_boundary = (
stub_stage(comm, StageKind.ATTENTION),
stub_stage(comm, StageKind.FFN),
)
wrapper = torch.nn.Module()
wrapper.language_model = torch.nn.Sequential(layer)
_install([layer])
layer.ffn_stage.fusions.service.prepare = lambda *, max_m: prepared.append(max_m)
layer.ffn_boundary.fusions.service.prepare = lambda *, max_m: prepared.append(max_m)
# The workspace M bound is the largest of every framework source.
reset_context()
@@ -562,8 +562,8 @@ class TestDeferredLoraAllReduce(unittest.TestCase):
from types import SimpleNamespace
from unittest.mock import patch
from sglang.srt.layers.communicator import exit as exits
from sglang.srt.layers.communicator.fusions.cutedsl import CuteDSLFusion
from sglang.srt.layers.layer_boundary import exit as exits
from sglang.srt.layers.layer_boundary.fusions.cutedsl import CuteDSLFusion
group = object()
fb = SimpleNamespace(
@@ -166,7 +166,7 @@ class TestReplicatedMoeOutput(CustomTestCase):
class TestModelsWithExplicitDpCompletion(CustomTestCase):
def test_direct_dp_exits_publish_their_selected_sum(self):
# Execute the real orchestration methods without constructing weights.
# These models own their DP exit instead of using StageCommunicator.
# These models own their DP exit instead of using StageBoundary.
import __future__
for filename, method in (
@@ -7,9 +7,9 @@ from unittest.mock import Mock, patch
import torch
from sglang.srt.layers.aux_hidden_states import AuxHiddenStateList, AuxHiddenStatePacker
from sglang.srt.layers.communicator import StageKind
from sglang.srt.layers.communicator.residual.access import norm_output
from sglang.srt.layers.communicator.residual.stream import ResidualStream
from sglang.srt.layers.layer_boundary import StageKind
from sglang.srt.layers.layer_boundary.residual.access import norm_output
from sglang.srt.layers.layer_boundary.residual.stream import ResidualStream
from sglang.test.boundary_fixtures import prepare_attention, stub_plan, stub_stage
from sglang.test.ci.ci_register import register_cpu_ci
from sglang.test.test_utils import CustomTestCase
@@ -35,8 +35,8 @@ class TestAuxStorage(CustomTestCase):
backend._copy_output_to_buffer(source, tuple(buffers), 2)
def test_capture_move_owns_gather_but_not_slice(self):
from sglang.srt.layers.communicator.boundary import Boundary
from sglang.srt.layers.communicator.layout import Layout, TokenAxis
from sglang.srt.layers.layer_boundary.boundary import Boundary
from sglang.srt.layers.layer_boundary.layout import Layout, TokenAxis
from sglang.test.communicator_patch import patch_communicator
full = Layout(frozenset())
@@ -181,7 +181,7 @@ class TestBoundCaptureOwnership(CustomTestCase):
import test_declared_decoder_boundary as fixture
from sglang.srt.layers import layernorm
from sglang.srt.layers.communicator import (
from sglang.srt.layers.layer_boundary import (
ADD,
FusedMlpInput,
SumGroup,

Some files were not shown because too many files have changed in this diff Show More