[Refactor] Rename the module to layer_boundary (#41555)
This commit is contained in:
@@ -14,13 +14,13 @@ from sglang.srt.batch_overlap.operations import (
|
||||
)
|
||||
from sglang.srt.batch_overlap.operations_strategy import OperationsStrategy
|
||||
from sglang.srt.layers import deep_gemm_wrapper
|
||||
from sglang.srt.layers.communicator import (
|
||||
from sglang.srt.layers.layer_boundary import (
|
||||
Layout,
|
||||
tbo_split_moves,
|
||||
)
|
||||
from sglang.srt.layers.communicator.residual import batch as residual_batch
|
||||
from sglang.srt.layers.communicator.residual.access import finish_layer_stack
|
||||
from sglang.srt.layers.communicator.residual.stream import ResidualStream
|
||||
from sglang.srt.layers.layer_boundary.residual import batch as residual_batch
|
||||
from sglang.srt.layers.layer_boundary.residual.access import finish_layer_stack
|
||||
from sglang.srt.layers.layer_boundary.residual.stream import ResidualStream
|
||||
from sglang.srt.layers.moe import (
|
||||
get_deepep_mode,
|
||||
get_moe_a2a_backend,
|
||||
@@ -959,7 +959,7 @@ def model_forward_stages(
|
||||
parts = _model_forward_tbo_split_inputs(
|
||||
**inputs,
|
||||
residual=residual,
|
||||
layer_input_rows=layers[0].attn_stage.incoming_residual_rows,
|
||||
layer_input_rows=layers[0].attn_boundary.incoming_residual_rows,
|
||||
)
|
||||
for part in parts:
|
||||
part["hidden_states"], child_stream = ResidualStream.arrive(
|
||||
|
||||
@@ -15,7 +15,7 @@ from sglang.srt.layers.attention.dsa.dsa_npu_indexer import scattered_to_tp_attn
|
||||
from sglang.srt.layers.attention.dsa.utils import (
|
||||
dsa_use_prefill_cp,
|
||||
)
|
||||
from sglang.srt.layers.communicator import get_attn_tp_context
|
||||
from sglang.srt.layers.layer_boundary import get_attn_tp_context
|
||||
from sglang.srt.model_executor.forward_context import get_token_to_kv_pool
|
||||
from sglang.srt.runtime_context import get_disagg
|
||||
from sglang.srt.state_capturer.indexer_topk import maybe_capture_indexer_topk
|
||||
|
||||
+12
-12
@@ -13,20 +13,20 @@
|
||||
# ==============================================================================
|
||||
"""Stage boundaries: layout transport, residual updates, reads, and fusion."""
|
||||
|
||||
from sglang.srt.layers.communicator.adapters.attention import (
|
||||
from sglang.srt.layers.layer_boundary.adapters.attention import (
|
||||
AttentionInputs,
|
||||
get_attn_tp_context,
|
||||
)
|
||||
from sglang.srt.layers.communicator.boundary import (
|
||||
from sglang.srt.layers.layer_boundary.boundary import (
|
||||
make_boundary,
|
||||
make_output_boundary,
|
||||
tbo_split_moves,
|
||||
)
|
||||
from sglang.srt.layers.communicator.construction import (
|
||||
from sglang.srt.layers.layer_boundary.construction import (
|
||||
BatchVariant,
|
||||
StageEdges,
|
||||
)
|
||||
from sglang.srt.layers.communicator.contracts import (
|
||||
from sglang.srt.layers.layer_boundary.contracts import (
|
||||
EdgeDecl,
|
||||
FusedMlpInput,
|
||||
HandoffRows,
|
||||
@@ -38,15 +38,15 @@ from sglang.srt.layers.communicator.contracts import (
|
||||
StageOutput,
|
||||
StageSteps,
|
||||
)
|
||||
from sglang.srt.layers.communicator.exit import FfnCompletion, FfnExit, MixerExit
|
||||
from sglang.srt.layers.communicator.factories import (
|
||||
from sglang.srt.layers.layer_boundary.exit import FfnCompletion, FfnExit, MixerExit
|
||||
from sglang.srt.layers.layer_boundary.factories import (
|
||||
declare_attn,
|
||||
declare_ffn,
|
||||
make_attn_stage,
|
||||
make_ffn_stage,
|
||||
make_stages,
|
||||
)
|
||||
from sglang.srt.layers.communicator.layout import (
|
||||
from sglang.srt.layers.layer_boundary.layout import (
|
||||
Layout,
|
||||
SumGroup,
|
||||
TokenAxis,
|
||||
@@ -55,24 +55,24 @@ from sglang.srt.layers.communicator.layout import (
|
||||
sparse_moe_gathers_over_moe_cp,
|
||||
token_axis_sizes,
|
||||
)
|
||||
from sglang.srt.layers.communicator.ops import (
|
||||
from sglang.srt.layers.layer_boundary.ops import (
|
||||
move_rows,
|
||||
tp_reduce_scatter,
|
||||
)
|
||||
from sglang.srt.layers.communicator.output import (
|
||||
from sglang.srt.layers.layer_boundary.output import (
|
||||
HandoffOutput,
|
||||
UnreducedOutput,
|
||||
reduce_output,
|
||||
)
|
||||
from sglang.srt.layers.communicator.residual import LayerResidual
|
||||
from sglang.srt.layers.communicator.residual.add_norm import (
|
||||
from sglang.srt.layers.layer_boundary.residual import LayerResidual
|
||||
from sglang.srt.layers.layer_boundary.residual.add_norm import (
|
||||
ADD,
|
||||
FUSE_ALLREDUCE_MAX_BATCH_SIZE,
|
||||
NORM_QUANT_READ,
|
||||
NORM_READ,
|
||||
PLAIN_RESIDUAL,
|
||||
)
|
||||
from sglang.srt.layers.communicator.residual.mhc import (
|
||||
from sglang.srt.layers.layer_boundary.residual.mhc import (
|
||||
MHCState,
|
||||
)
|
||||
|
||||
+3
-3
@@ -20,14 +20,14 @@ from typing import Callable
|
||||
|
||||
import torch
|
||||
|
||||
from sglang.srt.layers.communicator.layout import (
|
||||
enable_moe_dense_fully_dp,
|
||||
)
|
||||
from sglang.srt.layers.dp_attention import (
|
||||
attn_tp_all_gather_into_tensor,
|
||||
get_local_dp_buffer,
|
||||
is_dp_attention_enabled,
|
||||
)
|
||||
from sglang.srt.layers.layer_boundary.layout import (
|
||||
enable_moe_dense_fully_dp,
|
||||
)
|
||||
from sglang.srt.layers.moe import get_moe_a2a_backend
|
||||
from sglang.srt.model_executor.cuda_graph_config import (
|
||||
Backend,
|
||||
+3
-3
@@ -19,12 +19,12 @@ from typing import TYPE_CHECKING, Tuple
|
||||
|
||||
import torch
|
||||
|
||||
from sglang.srt.layers.communicator.ops import move_rows
|
||||
from sglang.srt.layers.communicator.residual.stream import ResidualStream
|
||||
from sglang.srt.layers.layer_boundary.ops import move_rows
|
||||
from sglang.srt.layers.layer_boundary.residual.stream import ResidualStream
|
||||
from sglang.srt.model_executor.forward_batch_info import ForwardBatch
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from sglang.srt.layers.communicator.construction import StagePlan
|
||||
from sglang.srt.layers.layer_boundary.construction import StagePlan
|
||||
|
||||
|
||||
def branch_input(
|
||||
+1
-1
@@ -13,7 +13,7 @@
|
||||
# ==============================================================================
|
||||
"""Publish the token rows consumed by the selected stage to LoRA kernels."""
|
||||
|
||||
from sglang.srt.layers.communicator.layout import TokenAxis
|
||||
from sglang.srt.layers.layer_boundary.layout import TokenAxis
|
||||
from sglang.srt.runtime_context import LoRABatchLayout, get_forward
|
||||
|
||||
|
||||
+2
-2
@@ -1,7 +1,7 @@
|
||||
"""Row requirements at the entrance to two-batch overlap."""
|
||||
|
||||
from sglang.srt.layers.communicator.contracts import HandoffRows
|
||||
from sglang.srt.layers.communicator.layout import enable_moe_dense_fully_dp
|
||||
from sglang.srt.layers.layer_boundary.contracts import HandoffRows
|
||||
from sglang.srt.layers.layer_boundary.layout import enable_moe_dense_fully_dp
|
||||
from sglang.srt.runtime_context import get_exec
|
||||
|
||||
|
||||
+4
-4
@@ -19,14 +19,14 @@ from typing import Callable, Optional, Tuple
|
||||
|
||||
import msgspec
|
||||
|
||||
from sglang.srt.layers.communicator.contracts import (
|
||||
from sglang.srt.layers.layer_boundary.contracts import (
|
||||
CpMoves,
|
||||
EdgeDecl,
|
||||
FusedMlpInput,
|
||||
StageInput,
|
||||
StageOutput,
|
||||
)
|
||||
from sglang.srt.layers.communicator.layout import (
|
||||
from sglang.srt.layers.layer_boundary.layout import (
|
||||
Layout,
|
||||
SumGroup,
|
||||
TokenAxis,
|
||||
@@ -35,7 +35,7 @@ from sglang.srt.layers.communicator.layout import (
|
||||
_sum_group,
|
||||
token_axis_sizes,
|
||||
)
|
||||
from sglang.srt.layers.communicator.ops import (
|
||||
from sglang.srt.layers.layer_boundary.ops import (
|
||||
gather_attention_tp,
|
||||
identity_output,
|
||||
move_rows,
|
||||
@@ -49,7 +49,7 @@ from sglang.srt.layers.communicator.ops import (
|
||||
tp_slice,
|
||||
update_and_gather,
|
||||
)
|
||||
from sglang.srt.layers.communicator.prepare import (
|
||||
from sglang.srt.layers.layer_boundary.prepare import (
|
||||
_consumer_step,
|
||||
_dispatch_consumer,
|
||||
_mlp_input_dp_partial,
|
||||
+10
-10
@@ -18,13 +18,13 @@ from functools import cached_property
|
||||
from typing import Callable, Optional
|
||||
|
||||
from sglang.srt.environ import envs
|
||||
from sglang.srt.layers.communicator.adapters.attention import get_attn_tp_context
|
||||
from sglang.srt.layers.communicator.boundary import (
|
||||
from sglang.srt.layers.layer_boundary.adapters.attention import get_attn_tp_context
|
||||
from sglang.srt.layers.layer_boundary.boundary import (
|
||||
_cp_moves,
|
||||
make_boundary,
|
||||
make_output_boundary,
|
||||
)
|
||||
from sglang.srt.layers.communicator.contracts import (
|
||||
from sglang.srt.layers.layer_boundary.contracts import (
|
||||
BatchVariant,
|
||||
CpMoves,
|
||||
EdgeDecl,
|
||||
@@ -33,23 +33,23 @@ from sglang.srt.layers.communicator.contracts import (
|
||||
StageKind,
|
||||
StageSteps,
|
||||
)
|
||||
from sglang.srt.layers.communicator.exit import OutputBoundary
|
||||
from sglang.srt.layers.communicator.fusions.allreduce import (
|
||||
from sglang.srt.layers.layer_boundary.exit import OutputBoundary
|
||||
from sglang.srt.layers.layer_boundary.fusions.allreduce import (
|
||||
attention_fusions,
|
||||
ffn_fusions,
|
||||
)
|
||||
from sglang.srt.layers.communicator.layout import (
|
||||
from sglang.srt.layers.layer_boundary.layout import (
|
||||
TokenAxis,
|
||||
_batch_shards_over_cp,
|
||||
_gathers_over_attention_cp,
|
||||
_generic_prefill_cp_shards_tokens,
|
||||
enable_moe_dense_fully_dp,
|
||||
)
|
||||
from sglang.srt.layers.communicator.prepare import (
|
||||
from sglang.srt.layers.layer_boundary.prepare import (
|
||||
_hand_qkv_hook_its_input,
|
||||
_hand_scattered_input_to_attention,
|
||||
)
|
||||
from sglang.srt.layers.communicator.stage import StageCommunicator
|
||||
from sglang.srt.layers.layer_boundary.stage import StageBoundary
|
||||
from sglang.srt.layers.moe import (
|
||||
get_moe_a2a_backend,
|
||||
)
|
||||
@@ -294,10 +294,10 @@ def _bind_stage(declaration, norm, incoming, outgoing, **options):
|
||||
if declaration.kind is StageKind.ATTENTION and outgoing.consumer is not None:
|
||||
# Layout eligibility comes from the connected consumer, not a mutable
|
||||
# link to its execution plan. Kernel binding remains consumer-owned.
|
||||
from sglang.srt.layers.communicator.boundary import input_rows
|
||||
from sglang.srt.layers.layer_boundary.boundary import input_rows
|
||||
|
||||
plan._fusion_rows = {
|
||||
v: input_rows(edge) for v, edge in outgoing.entries.items()
|
||||
}
|
||||
|
||||
return StageCommunicator(plan, declaration=declaration)
|
||||
return StageBoundary(plan, declaration=declaration)
|
||||
+4
-4
@@ -20,10 +20,10 @@ import msgspec
|
||||
import torch
|
||||
|
||||
from sglang.srt.distributed import GroupCoordinator
|
||||
from sglang.srt.layers.communicator.layout import Layout, SumGroup, TokenAxis
|
||||
from sglang.srt.layers.communicator.output import OutputTransform
|
||||
from sglang.srt.layers.communicator.residual import StageRead, StageUpdate
|
||||
from sglang.srt.layers.communicator.residual.add_norm import ADD, NORM_READ
|
||||
from sglang.srt.layers.layer_boundary.layout import Layout, SumGroup, TokenAxis
|
||||
from sglang.srt.layers.layer_boundary.output import OutputTransform
|
||||
from sglang.srt.layers.layer_boundary.residual import StageRead, StageUpdate
|
||||
from sglang.srt.layers.layer_boundary.residual.add_norm import ADD, NORM_READ
|
||||
|
||||
|
||||
class ProducerReduction(Enum):
|
||||
+11
-11
@@ -23,32 +23,32 @@ import torch
|
||||
|
||||
from sglang.srt.distributed import GroupCoordinator
|
||||
from sglang.srt.environ import envs
|
||||
from sglang.srt.layers.communicator.adapters.attention import get_attn_tp_context
|
||||
from sglang.srt.layers.communicator.layout import (
|
||||
from sglang.srt.layers.dp_attention import (
|
||||
can_use_dp_reduce_scatter,
|
||||
is_dp_attention_enabled,
|
||||
is_enable_moe_cp_allgather,
|
||||
)
|
||||
from sglang.srt.layers.layer_boundary.adapters.attention import get_attn_tp_context
|
||||
from sglang.srt.layers.layer_boundary.layout import (
|
||||
SumGroup,
|
||||
_ffn_has_tokens,
|
||||
_sum_group,
|
||||
)
|
||||
from sglang.srt.layers.communicator.ops import (
|
||||
from sglang.srt.layers.layer_boundary.ops import (
|
||||
_all_reduce_then_to_local_tokens,
|
||||
_redistribute_output,
|
||||
_reduce_and_redistribute_output_max_len,
|
||||
_reduce_and_redistribute_output_varlen,
|
||||
_to_local_tokens,
|
||||
)
|
||||
from sglang.srt.layers.communicator.output import (
|
||||
from sglang.srt.layers.layer_boundary.output import (
|
||||
UnreducedOutput,
|
||||
)
|
||||
from sglang.srt.layers.communicator.residual.add_norm import (
|
||||
from sglang.srt.layers.layer_boundary.residual.add_norm import (
|
||||
apply_aiter_all_reduce_fusion,
|
||||
apply_flashinfer_allreduce_fusion,
|
||||
)
|
||||
from sglang.srt.layers.communicator.residual.stream import ResidualStream
|
||||
from sglang.srt.layers.dp_attention import (
|
||||
can_use_dp_reduce_scatter,
|
||||
is_dp_attention_enabled,
|
||||
is_enable_moe_cp_allgather,
|
||||
)
|
||||
from sglang.srt.layers.layer_boundary.residual.stream import ResidualStream
|
||||
from sglang.srt.layers.moe import (
|
||||
can_merge_post_experts_all_reduce,
|
||||
post_experts_sum_is_one_all_reduce,
|
||||
+8
-8
@@ -9,19 +9,19 @@ from typing import Mapping, Optional
|
||||
import msgspec
|
||||
|
||||
from sglang.srt.layers import layernorm_sp
|
||||
from sglang.srt.layers.communicator.adapters.overlap import (
|
||||
from sglang.srt.layers.layer_boundary.adapters.overlap import (
|
||||
resolve_handoff_rows,
|
||||
tbo_handoff,
|
||||
)
|
||||
from sglang.srt.layers.communicator.boundary import _cp_moves
|
||||
from sglang.srt.layers.communicator.construction import (
|
||||
from sglang.srt.layers.layer_boundary.boundary import _cp_moves
|
||||
from sglang.srt.layers.layer_boundary.construction import (
|
||||
BatchVariant,
|
||||
_bind_stage,
|
||||
_input_can_be_scattered,
|
||||
_refuse_uncovered_cp_moe,
|
||||
_use_ag_after_qlora,
|
||||
)
|
||||
from sglang.srt.layers.communicator.contracts import (
|
||||
from sglang.srt.layers.layer_boundary.contracts import (
|
||||
EdgeDecl,
|
||||
HandoffRows,
|
||||
ProducerReduction,
|
||||
@@ -30,7 +30,7 @@ from sglang.srt.layers.communicator.contracts import (
|
||||
StageKind,
|
||||
StageOutput,
|
||||
)
|
||||
from sglang.srt.layers.communicator.layout import (
|
||||
from sglang.srt.layers.layer_boundary.layout import (
|
||||
Layout,
|
||||
SumGroup,
|
||||
TokenAxis,
|
||||
@@ -40,9 +40,9 @@ from sglang.srt.layers.communicator.layout import (
|
||||
sparse_moe_gathers_over_moe_cp,
|
||||
token_axis_sizes,
|
||||
)
|
||||
from sglang.srt.layers.communicator.output import OutputTransform
|
||||
from sglang.srt.layers.communicator.residual import StageRead, StageUpdate
|
||||
from sglang.srt.layers.communicator.residual.add_norm import (
|
||||
from sglang.srt.layers.layer_boundary.output import OutputTransform
|
||||
from sglang.srt.layers.layer_boundary.residual import StageRead, StageUpdate
|
||||
from sglang.srt.layers.layer_boundary.residual.add_norm import (
|
||||
ADD,
|
||||
NORM_QUANT_READ,
|
||||
NORM_READ,
|
||||
+4
-4
@@ -19,10 +19,10 @@ from typing import Callable, Optional, Tuple
|
||||
|
||||
import torch
|
||||
|
||||
from sglang.srt.layers.communicator.contracts import FusedMlpInput
|
||||
from sglang.srt.layers.communicator.layout import SumGroup
|
||||
from sglang.srt.layers.communicator.output import UnreducedOutput
|
||||
from sglang.srt.layers.communicator.residual.add_norm import (
|
||||
from sglang.srt.layers.layer_boundary.contracts import FusedMlpInput
|
||||
from sglang.srt.layers.layer_boundary.layout import SumGroup
|
||||
from sglang.srt.layers.layer_boundary.output import UnreducedOutput
|
||||
from sglang.srt.layers.layer_boundary.residual.add_norm import (
|
||||
NORM_QUANT_READ,
|
||||
Fp8Input,
|
||||
NormQuantRead,
|
||||
+6
-6
@@ -13,7 +13,8 @@ from typing import Callable, Optional, Sequence
|
||||
|
||||
import torch
|
||||
|
||||
from sglang.srt.layers.communicator import (
|
||||
from sglang.srt.layers.dp_attention import is_dp_attention_enabled
|
||||
from sglang.srt.layers.layer_boundary import (
|
||||
FusedMlpInput,
|
||||
HandoffOutput,
|
||||
SumGroup,
|
||||
@@ -21,7 +22,6 @@ from sglang.srt.layers.communicator import (
|
||||
UnreducedOutput,
|
||||
get_attn_tp_context,
|
||||
)
|
||||
from sglang.srt.layers.dp_attention import is_dp_attention_enabled
|
||||
from sglang.srt.layers.layernorm import GemmaRMSNorm, RMSNorm
|
||||
from sglang.srt.layers.moe import get_moe_a2a_backend
|
||||
from sglang.srt.model_executor.forward_batch_info import ForwardBatch, ForwardMode
|
||||
@@ -359,7 +359,7 @@ class CuteDSLFusion:
|
||||
|
||||
|
||||
def _fusion_of(layer: torch.nn.Module) -> CuteDSLFusion | None:
|
||||
fusions = getattr(layer.__dict__.get("ffn_stage"), "fusions", None)
|
||||
fusions = getattr(layer.__dict__.get("ffn_boundary"), "fusions", None)
|
||||
return fusions if isinstance(fusions, CuteDSLFusion) else None
|
||||
|
||||
|
||||
@@ -388,8 +388,8 @@ def install_cutedsl_fusion(
|
||||
# The workspace compiles one epsilon and cannot see a per-layer one.
|
||||
for layer in fusion_layers:
|
||||
for norm in (
|
||||
layer.attn_stage.norm,
|
||||
layer.ffn_stage.norm,
|
||||
layer.attn_boundary.norm,
|
||||
layer.ffn_boundary.norm,
|
||||
):
|
||||
if _fused_norm_gamma(norm) is None:
|
||||
continue
|
||||
@@ -441,7 +441,7 @@ def prepare_cutedsl_fusion(
|
||||
for module in model.modules()
|
||||
# Only decoder modules carry an FFN stage.
|
||||
if isinstance(
|
||||
fusion := getattr(module.__dict__.get("ffn_stage"), "fusions", None),
|
||||
fusion := getattr(module.__dict__.get("ffn_boundary"), "fusions", None),
|
||||
CuteDSLFusion,
|
||||
)
|
||||
]
|
||||
+15
-15
@@ -26,21 +26,6 @@ from sglang.srt.distributed import (
|
||||
from sglang.srt.distributed.device_communicators.pynccl_allocator import (
|
||||
use_symmetric_memory,
|
||||
)
|
||||
from sglang.srt.layers.communicator.adapters.attention import (
|
||||
_redistribute_from_attn_tp_shards,
|
||||
_redistribute_to_attn_tp_shards,
|
||||
)
|
||||
from sglang.srt.layers.communicator.adapters.context_parallel import (
|
||||
dsa_cp_reduce_scatter_hidden_states,
|
||||
)
|
||||
from sglang.srt.layers.communicator.layout import (
|
||||
Layout,
|
||||
TokenAxis,
|
||||
_cp_shard_token_rows,
|
||||
moe_cp_gathered_rows,
|
||||
)
|
||||
from sglang.srt.layers.communicator.residual import StageUpdate
|
||||
from sglang.srt.layers.communicator.residual.add_norm import ADD
|
||||
from sglang.srt.layers.dp_attention import (
|
||||
attn_tp_all_gather_into_tensor,
|
||||
attn_tp_reduce_scatter_tensor,
|
||||
@@ -55,6 +40,21 @@ from sglang.srt.layers.dp_attention import (
|
||||
is_allocation_symmetric,
|
||||
moe_cp_all_gather_into_tensor,
|
||||
)
|
||||
from sglang.srt.layers.layer_boundary.adapters.attention import (
|
||||
_redistribute_from_attn_tp_shards,
|
||||
_redistribute_to_attn_tp_shards,
|
||||
)
|
||||
from sglang.srt.layers.layer_boundary.adapters.context_parallel import (
|
||||
dsa_cp_reduce_scatter_hidden_states,
|
||||
)
|
||||
from sglang.srt.layers.layer_boundary.layout import (
|
||||
Layout,
|
||||
TokenAxis,
|
||||
_cp_shard_token_rows,
|
||||
moe_cp_gathered_rows,
|
||||
)
|
||||
from sglang.srt.layers.layer_boundary.residual import StageUpdate
|
||||
from sglang.srt.layers.layer_boundary.residual.add_norm import ADD
|
||||
from sglang.srt.model_executor.forward_batch_info import ForwardBatch
|
||||
from sglang.srt.runtime_context import get_exec, get_parallel
|
||||
|
||||
+23
-23
@@ -24,34 +24,34 @@ from sglang.srt.distributed import (
|
||||
from sglang.srt.distributed.device_communicators.pynccl_allocator import (
|
||||
use_symmetric_memory,
|
||||
)
|
||||
from sglang.srt.layers.communicator.adapters.attention import (
|
||||
AttentionInputs,
|
||||
_redistribute_from_attn_tp_shards,
|
||||
_redistribute_to_attn_tp_shards,
|
||||
get_attn_tp_context,
|
||||
)
|
||||
from sglang.srt.layers.communicator.adapters.context_parallel import (
|
||||
dsa_cp_gather_hidden_states,
|
||||
)
|
||||
from sglang.srt.layers.communicator.layout import (
|
||||
SumGroup,
|
||||
_cp_shard_token_rows,
|
||||
moe_cp_gathered_rows,
|
||||
)
|
||||
from sglang.srt.layers.communicator.output import (
|
||||
HandoffOutput,
|
||||
UnreducedOutput,
|
||||
reduce_output,
|
||||
)
|
||||
from sglang.srt.layers.communicator.residual import StageRead, StageUpdate
|
||||
from sglang.srt.layers.communicator.residual.add_norm import ADD, NORM_READ
|
||||
from sglang.srt.layers.communicator.residual.stream import DeclaredSum
|
||||
from sglang.srt.layers.dp_attention import (
|
||||
attn_tp_all_gather_into_tensor,
|
||||
dp_scatter,
|
||||
get_moe_cp_size,
|
||||
is_allocation_symmetric,
|
||||
)
|
||||
from sglang.srt.layers.layer_boundary.adapters.attention import (
|
||||
AttentionInputs,
|
||||
_redistribute_from_attn_tp_shards,
|
||||
_redistribute_to_attn_tp_shards,
|
||||
get_attn_tp_context,
|
||||
)
|
||||
from sglang.srt.layers.layer_boundary.adapters.context_parallel import (
|
||||
dsa_cp_gather_hidden_states,
|
||||
)
|
||||
from sglang.srt.layers.layer_boundary.layout import (
|
||||
SumGroup,
|
||||
_cp_shard_token_rows,
|
||||
moe_cp_gathered_rows,
|
||||
)
|
||||
from sglang.srt.layers.layer_boundary.output import (
|
||||
HandoffOutput,
|
||||
UnreducedOutput,
|
||||
reduce_output,
|
||||
)
|
||||
from sglang.srt.layers.layer_boundary.residual import StageRead, StageUpdate
|
||||
from sglang.srt.layers.layer_boundary.residual.add_norm import ADD, NORM_READ
|
||||
from sglang.srt.layers.layer_boundary.residual.stream import DeclaredSum
|
||||
from sglang.srt.model_executor.forward_batch_info import ForwardBatch
|
||||
from sglang.srt.runtime_context import get_parallel
|
||||
from sglang.srt.utils import (
|
||||
@@ -64,7 +64,7 @@ _is_npu = is_npu()
|
||||
if _is_npu:
|
||||
from sglang.srt.hardware_backend.npu.cmo import prepare_weight_cache
|
||||
|
||||
from sglang.srt.layers.communicator.ops import (
|
||||
from sglang.srt.layers.layer_boundary.ops import (
|
||||
_mlp_input_reduce_output,
|
||||
_redistribute_input_to_dp,
|
||||
_redistribute_input_to_moe_cp,
|
||||
+2
-2
@@ -18,12 +18,12 @@ from typing import Optional, Tuple, Union
|
||||
import torch
|
||||
|
||||
from sglang.srt.distributed import GroupCoordinator
|
||||
from sglang.srt.layers.communicator.output import (
|
||||
from sglang.srt.layers.layer_boundary.output import (
|
||||
HandoffOutput,
|
||||
UnreducedOutput,
|
||||
reduce_output,
|
||||
)
|
||||
from sglang.srt.layers.communicator.residual.stream import OwedOutput, ResidualStream
|
||||
from sglang.srt.layers.layer_boundary.residual.stream import OwedOutput, ResidualStream
|
||||
from sglang.srt.model_executor.forward_batch_info import ForwardBatch, PPProxyTensors
|
||||
|
||||
|
||||
+6
-6
@@ -20,17 +20,17 @@ from typing import Optional
|
||||
|
||||
import torch
|
||||
|
||||
from sglang.srt.layers.communicator.adapters.attention import (
|
||||
_redistribute_from_attn_tp_shards,
|
||||
_redistribute_to_attn_tp_shards,
|
||||
get_attn_tp_context,
|
||||
)
|
||||
from sglang.srt.layers.communicator.residual import LayerResidual
|
||||
from sglang.srt.layers.dp_attention import is_dp_attention_enabled
|
||||
from sglang.srt.layers.flashinfer_comm_fusion import (
|
||||
is_flashinfer_allreduce_unavailable,
|
||||
uses_cutedsl_ar_fusion,
|
||||
)
|
||||
from sglang.srt.layers.layer_boundary.adapters.attention import (
|
||||
_redistribute_from_attn_tp_shards,
|
||||
_redistribute_to_attn_tp_shards,
|
||||
get_attn_tp_context,
|
||||
)
|
||||
from sglang.srt.layers.layer_boundary.residual import LayerResidual
|
||||
from sglang.srt.layers.quantization.fp8_utils import (
|
||||
_use_aiter_bpreshuffle_gfx95,
|
||||
materialize_bpreshuffle_fp8_scale_tuple,
|
||||
+3
-3
@@ -13,8 +13,8 @@
|
||||
# ==============================================================================
|
||||
"""Residual access for one layer-stack invocation or TBO microbatch."""
|
||||
|
||||
from sglang.srt.layers.communicator.residual import access
|
||||
from sglang.srt.layers.communicator.residual.stream import ResidualStream
|
||||
from sglang.srt.layers.layer_boundary.residual import access
|
||||
from sglang.srt.layers.layer_boundary.residual.stream import ResidualStream
|
||||
from sglang.srt.model_executor.forward_batch_info import PPProxyTensors
|
||||
|
||||
|
||||
@@ -60,7 +60,7 @@ def norm(
|
||||
# The terminal consumer now owns the pair. Do not keep layer buffers alive
|
||||
# through logits processing or the next forward on this batch.
|
||||
forward_batch.residual_stream = None
|
||||
from sglang.srt.layers.communicator.output import HandoffOutput
|
||||
from sglang.srt.layers.layer_boundary.output import HandoffOutput
|
||||
|
||||
if isinstance(hidden_states, HandoffOutput):
|
||||
if residual is None:
|
||||
+1
-1
@@ -19,7 +19,7 @@ from typing import Callable, Optional
|
||||
import torch
|
||||
|
||||
from sglang.kernels.ops.layernorm.mhc import hc_contract, hc_expand
|
||||
from sglang.srt.layers.communicator.residual import LayerResidual
|
||||
from sglang.srt.layers.layer_boundary.residual import LayerResidual
|
||||
from sglang.srt.runtime_context import get_parallel
|
||||
|
||||
|
||||
+3
-3
@@ -19,9 +19,9 @@ import msgspec
|
||||
import torch
|
||||
|
||||
from sglang.srt.distributed import GroupCoordinator
|
||||
from sglang.srt.layers.communicator.layout import SumGroup, _sum_group
|
||||
from sglang.srt.layers.communicator.output import HandoffOutput, UnreducedOutput
|
||||
from sglang.srt.layers.communicator.residual import StageUpdate
|
||||
from sglang.srt.layers.layer_boundary.layout import SumGroup, _sum_group
|
||||
from sglang.srt.layers.layer_boundary.output import HandoffOutput, UnreducedOutput
|
||||
from sglang.srt.layers.layer_boundary.residual import StageUpdate
|
||||
|
||||
|
||||
class CarriedSum(msgspec.Struct, frozen=True):
|
||||
+11
-11
@@ -19,33 +19,33 @@ import torch
|
||||
|
||||
from sglang.srt.layers import layernorm_sp
|
||||
from sglang.srt.layers.aux_hidden_states import AuxHiddenStateAccumulator
|
||||
from sglang.srt.layers.communicator.adapters import branch
|
||||
from sglang.srt.layers.communicator.adapters.lora import (
|
||||
from sglang.srt.layers.layer_boundary.adapters import branch
|
||||
from sglang.srt.layers.layer_boundary.adapters.lora import (
|
||||
publish_attention,
|
||||
publish_ffn,
|
||||
)
|
||||
from sglang.srt.layers.communicator.contracts import (
|
||||
from sglang.srt.layers.layer_boundary.contracts import (
|
||||
BatchVariant,
|
||||
StageEntry,
|
||||
StageKind,
|
||||
StageSteps,
|
||||
)
|
||||
from sglang.srt.layers.communicator.ops import gather_attention_tp
|
||||
from sglang.srt.layers.communicator.residual.access import buffer, from_pp
|
||||
from sglang.srt.layers.communicator.residual.add_norm import (
|
||||
from sglang.srt.layers.layer_boundary.ops import gather_attention_tp
|
||||
from sglang.srt.layers.layer_boundary.residual.access import buffer, from_pp
|
||||
from sglang.srt.layers.layer_boundary.residual.add_norm import (
|
||||
ADD,
|
||||
)
|
||||
from sglang.srt.layers.communicator.residual.batch import current
|
||||
from sglang.srt.layers.communicator.residual.stream import DeclaredSum, ResidualStream
|
||||
from sglang.srt.layers.layer_boundary.residual.batch import current
|
||||
from sglang.srt.layers.layer_boundary.residual.stream import DeclaredSum, ResidualStream
|
||||
from sglang.srt.model_executor.forward_batch_info import ForwardBatch
|
||||
from sglang.srt.runtime_context import get_forward
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from sglang.srt.layers.communicator.construction import StagePlan
|
||||
from sglang.srt.layers.communicator.factories import StageDeclaration
|
||||
from sglang.srt.layers.layer_boundary.construction import StagePlan
|
||||
from sglang.srt.layers.layer_boundary.factories import StageDeclaration
|
||||
|
||||
|
||||
class StageCommunicator:
|
||||
class StageBoundary:
|
||||
"""One stage of a layer, its attention or its FFN: the boundary into it,
|
||||
run with the stage's norm on the steps the layer chose for the batch."""
|
||||
|
||||
@@ -21,12 +21,12 @@ from sglang.srt.distributed.device_communicators.pynccl_allocator import (
|
||||
)
|
||||
from sglang.srt.environ import envs
|
||||
from sglang.srt.layers.amx_utils import PackWeightMethod
|
||||
from sglang.srt.layers.communicator import get_attn_tp_context
|
||||
from sglang.srt.layers.dp_attention import (
|
||||
attn_tp_all_reduce,
|
||||
is_allocation_symmetric,
|
||||
is_dp_attention_enabled,
|
||||
)
|
||||
from sglang.srt.layers.layer_boundary import get_attn_tp_context
|
||||
from sglang.srt.layers.parameter import BasevLLMParameter
|
||||
from sglang.srt.layers.quantization.base_config import (
|
||||
QuantizationConfig,
|
||||
|
||||
@@ -119,7 +119,7 @@ class VocabParallelEmbeddingWithLoRA(BaseLayerWithLoRA):
|
||||
# all-reduce, making the unsharded LoRA approach mathematically
|
||||
# incorrect — a sharded LoRA kernel would be needed.
|
||||
if hasattr(base_layer, "tp_size") and base_layer.tp_size > 1:
|
||||
from sglang.srt.layers.communicator import get_attn_tp_context
|
||||
from sglang.srt.layers.layer_boundary import get_attn_tp_context
|
||||
|
||||
assert not get_attn_tp_context().allow_input_scattered, (
|
||||
"VocabParallelEmbeddingWithLoRA with TP > 1 under input_scattered mode (e.g., DeepSeek-v2 MLA with --enable-attn-tp-input-scattered) is not fully supported and may produce incorrect results. Consider disabling input_scattered or removing embed_tokens from LoRA target modules."
|
||||
@@ -295,7 +295,7 @@ class ParallelLMHeadWithLoRA(BaseLayerWithLoRA):
|
||||
# incompatible with input_scattered mode where the all-reduce is
|
||||
# skipped.
|
||||
if tp_size > 1:
|
||||
from sglang.srt.layers.communicator import get_attn_tp_context
|
||||
from sglang.srt.layers.layer_boundary import get_attn_tp_context
|
||||
|
||||
if get_attn_tp_context().allow_input_scattered:
|
||||
raise ValueError(
|
||||
|
||||
@@ -71,9 +71,9 @@ from sglang.srt.utils.common import ceil_align, is_pin_memory_available
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from sglang.srt.configs.model_config import ModelConfig
|
||||
from sglang.srt.layers.communicator.residual.stream import ResidualStream
|
||||
from sglang.srt.layers.cp.base import BaseContextParallelMetadata
|
||||
from sglang.srt.layers.dcp.metadata import DecodeContextParallelMetadata
|
||||
from sglang.srt.layers.layer_boundary.residual.stream import ResidualStream
|
||||
from sglang.srt.layers.logits_processor import LogitsProcessorOutput
|
||||
from sglang.srt.managers.schedule_batch import MultimodalInputs, ScheduleBatch
|
||||
from sglang.srt.model_executor.model_runner import ModelRunner
|
||||
@@ -1257,7 +1257,7 @@ class ForwardBatch(ForwardBatchDeepSeekMHAMixin):
|
||||
def moe_num_token_non_padded(self) -> Optional[torch.Tensor]:
|
||||
"""Bound for masking a sparse MoE's padded rows, or None when the MoE
|
||||
input is a gathered buffer whose real rows are not a prefix of it."""
|
||||
from sglang.srt.layers.communicator import moe_cp_gathers_sparse_moe_input
|
||||
from sglang.srt.layers.layer_boundary import moe_cp_gathers_sparse_moe_input
|
||||
from sglang.srt.layers.moe.utils import is_moe_input_scattered_across_dp_ranks
|
||||
|
||||
if self.num_token_non_padded is None:
|
||||
@@ -1916,7 +1916,7 @@ class ForwardBatch(ForwardBatchDeepSeekMHAMixin):
|
||||
return self.input_ids.shape[0]
|
||||
|
||||
def prepare_attn_tp_scatter_input(self, model_runner: ModelRunner):
|
||||
from sglang.srt.layers.communicator import get_attn_tp_context
|
||||
from sglang.srt.layers.layer_boundary import get_attn_tp_context
|
||||
|
||||
# Pure TP+SP has no MLP-sync pass, so stamp the decision here.
|
||||
self.attn_tp_sequence_sharded = model_runner.attn_tp_sequence_sharded(
|
||||
|
||||
@@ -288,12 +288,12 @@ class BaseRunner(ABC):
|
||||
with custom_all_reduce.register_graph_buffers).
|
||||
"""
|
||||
mr = self.model_runner
|
||||
from sglang.srt.layers.communicator import FUSE_ALLREDUCE_MAX_BATCH_SIZE
|
||||
from sglang.srt.layers.flashinfer_comm_fusion import (
|
||||
pre_initialize_workspaces,
|
||||
resolve_flashinfer_allreduce_fusion_backend,
|
||||
uses_cutedsl_ar_fusion,
|
||||
)
|
||||
from sglang.srt.layers.layer_boundary import FUSE_ALLREDUCE_MAX_BATCH_SIZE
|
||||
|
||||
if get_exec().comm.flashinfer_allreduce_fusion_backend is None:
|
||||
return
|
||||
@@ -303,7 +303,7 @@ class BaseRunner(ABC):
|
||||
resolve_flashinfer_allreduce_fusion_backend()
|
||||
if not mr.is_draft_worker:
|
||||
# A draft installs no fusion communicator.
|
||||
from sglang.srt.layers.communicator.fusions.cutedsl import (
|
||||
from sglang.srt.layers.layer_boundary.fusions.cutedsl import (
|
||||
prepare_cutedsl_fusion,
|
||||
)
|
||||
|
||||
|
||||
@@ -35,16 +35,16 @@ from sglang.srt.layers.aux_hidden_states import (
|
||||
AuxHiddenStateAccumulator,
|
||||
AuxHiddenStateList,
|
||||
)
|
||||
from sglang.srt.layers.communicator import (
|
||||
from sglang.srt.layers.dp_attention import (
|
||||
is_dp_attention_enabled,
|
||||
)
|
||||
from sglang.srt.layers.layer_boundary import (
|
||||
declare_attn,
|
||||
declare_ffn,
|
||||
enable_moe_dense_fully_dp,
|
||||
make_stages,
|
||||
)
|
||||
from sglang.srt.layers.communicator.residual import batch as residual_batch
|
||||
from sglang.srt.layers.dp_attention import (
|
||||
is_dp_attention_enabled,
|
||||
)
|
||||
from sglang.srt.layers.layer_boundary.residual import batch as residual_batch
|
||||
from sglang.srt.layers.layernorm import RMSNorm
|
||||
from sglang.srt.layers.linear import (
|
||||
MergedColumnParallelLinear,
|
||||
@@ -698,7 +698,7 @@ class BailingMoEBlock(nn.Module):
|
||||
|
||||
self.post_attention_layernorm = RMSNorm(hidden_size, eps=config.rms_norm_eps)
|
||||
|
||||
self.attn_stage, self.ffn_stage = make_stages(
|
||||
self.attn_boundary, self.ffn_boundary = make_stages(
|
||||
(declare_attn(), self.input_layernorm),
|
||||
(
|
||||
declare_ffn(
|
||||
@@ -730,7 +730,7 @@ class BailingMoEBlock(nn.Module):
|
||||
captured_last_layer_outputs: Optional[AuxHiddenStateAccumulator] = None,
|
||||
capture_output=None,
|
||||
) -> torch.Tensor:
|
||||
hidden_states = self.attn_stage.prepare(
|
||||
hidden_states = self.attn_boundary.prepare(
|
||||
hidden_states,
|
||||
forward_batch,
|
||||
captured_last_layer_outputs=captured_last_layer_outputs,
|
||||
@@ -744,13 +744,13 @@ class BailingMoEBlock(nn.Module):
|
||||
forward_batch=forward_batch,
|
||||
)
|
||||
|
||||
hidden_states = self.attn_stage.finish(hidden_states, forward_batch)
|
||||
hidden_states = self.ffn_stage.prepare(
|
||||
hidden_states = self.attn_boundary.finish(hidden_states, forward_batch)
|
||||
hidden_states = self.ffn_boundary.prepare(
|
||||
hidden_states=hidden_states,
|
||||
forward_batch=forward_batch,
|
||||
)
|
||||
|
||||
with self.ffn_stage.exit(forward_batch) as ffn_exit:
|
||||
with self.ffn_boundary.exit(forward_batch) as ffn_exit:
|
||||
hidden_states = self.mlp(hidden_states, forward_batch)
|
||||
hidden_states = ffn_exit.finish(hidden_states)
|
||||
|
||||
@@ -822,7 +822,7 @@ class BailingMoEModel(nn.Module):
|
||||
residual_batch.start(forward_batch)
|
||||
else:
|
||||
assert pp_proxy_tensors is not None
|
||||
hidden_states = self.layers[self.start_layer].attn_stage.from_pp(
|
||||
hidden_states = self.layers[self.start_layer].attn_boundary.from_pp(
|
||||
pp_proxy_tensors, forward_batch
|
||||
)
|
||||
|
||||
|
||||
@@ -15,15 +15,15 @@ from sglang.kernels.ops.quantization.fp8_kernel import is_fp8_fnuz
|
||||
from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder
|
||||
from sglang.srt.layers import deep_gemm_wrapper
|
||||
from sglang.srt.layers.activation import SiluAndMul
|
||||
from sglang.srt.layers.communicator import (
|
||||
from sglang.srt.layers.dp_attention import (
|
||||
is_dp_attention_enabled,
|
||||
)
|
||||
from sglang.srt.layers.layer_boundary import (
|
||||
declare_attn,
|
||||
declare_ffn,
|
||||
make_stages,
|
||||
)
|
||||
from sglang.srt.layers.communicator.residual import batch as residual_batch
|
||||
from sglang.srt.layers.dp_attention import (
|
||||
is_dp_attention_enabled,
|
||||
)
|
||||
from sglang.srt.layers.layer_boundary.residual import batch as residual_batch
|
||||
from sglang.srt.layers.layernorm import RMSNorm
|
||||
from sglang.srt.layers.linear import (
|
||||
ColumnParallelLinear,
|
||||
@@ -820,7 +820,7 @@ class BailingMoELinearDecoderLayer(nn.Module):
|
||||
if config.attention_type == 1 and self.use_mla
|
||||
else None
|
||||
)
|
||||
self.attn_stage, self.ffn_stage = make_stages(
|
||||
self.attn_boundary, self.ffn_boundary = make_stages(
|
||||
(
|
||||
declare_attn(),
|
||||
self.input_layernorm,
|
||||
@@ -854,7 +854,7 @@ class BailingMoELinearDecoderLayer(nn.Module):
|
||||
zero_allocator: BumpAllocator,
|
||||
**kwargs,
|
||||
) -> torch.Tensor:
|
||||
hidden_states = self.attn_stage.prepare(hidden_states, forward_batch)
|
||||
hidden_states = self.attn_boundary.prepare(hidden_states, forward_batch)
|
||||
# logger.warning(
|
||||
# f"===={self.layer_id=}, 1 shape= {hidden_states.shape}, {residual.shape}"
|
||||
# )
|
||||
@@ -875,12 +875,12 @@ class BailingMoELinearDecoderLayer(nn.Module):
|
||||
# logger.warning(
|
||||
# f"===={self.layer_id=}, 2 shape= {hidden_states.shape}, {residual.shape}"
|
||||
# )
|
||||
hidden_states = self.attn_stage.finish(hidden_states, forward_batch)
|
||||
hidden_states = self.ffn_stage.prepare(hidden_states, forward_batch)
|
||||
hidden_states = self.attn_boundary.finish(hidden_states, forward_batch)
|
||||
hidden_states = self.ffn_boundary.prepare(hidden_states, forward_batch)
|
||||
# logger.warning(
|
||||
# f"===={self.layer_id=}, 3 shape= {hidden_states.shape}, {residual.shape}"
|
||||
# )
|
||||
with self.ffn_stage.exit(forward_batch) as ffn_exit:
|
||||
with self.ffn_boundary.exit(forward_batch) as ffn_exit:
|
||||
hidden_states = self.mlp(hidden_states)
|
||||
hidden_states = ffn_exit.finish(hidden_states)
|
||||
return hidden_states
|
||||
@@ -983,7 +983,7 @@ class BailingMoELinearModel(nn.Module):
|
||||
residual_batch.start(forward_batch)
|
||||
else:
|
||||
assert pp_proxy_tensors is not None
|
||||
hidden_states = self.layers[self.start_layer].attn_stage.from_pp(
|
||||
hidden_states = self.layers[self.start_layer].attn_boundary.from_pp(
|
||||
pp_proxy_tensors, forward_batch
|
||||
)
|
||||
|
||||
|
||||
@@ -26,8 +26,8 @@ import torch
|
||||
from torch import nn
|
||||
from transformers import PretrainedConfig
|
||||
|
||||
from sglang.srt.layers.communicator.residual import batch as residual_batch
|
||||
from sglang.srt.layers.dp_attention import is_dp_attention_enabled
|
||||
from sglang.srt.layers.layer_boundary.residual import batch as residual_batch
|
||||
from sglang.srt.layers.layernorm import RMSNorm
|
||||
from sglang.srt.layers.linear import ReplicatedLinear
|
||||
from sglang.srt.layers.logits_processor import LogitsProcessor
|
||||
|
||||
@@ -29,14 +29,14 @@ from sglang.srt.eplb.expert_location_dispatch import ExpertLocationDispatchInfo
|
||||
from sglang.srt.layers import deep_gemm_wrapper
|
||||
from sglang.srt.layers.activation import SiluAndMul
|
||||
from sglang.srt.layers.aux_hidden_states import AuxHiddenStateList
|
||||
from sglang.srt.layers.communicator import (
|
||||
from sglang.srt.layers.dp_attention import is_dp_attention_enabled
|
||||
from sglang.srt.layers.layer_boundary import (
|
||||
declare_attn,
|
||||
declare_ffn,
|
||||
enable_moe_dense_fully_dp,
|
||||
make_stages,
|
||||
)
|
||||
from sglang.srt.layers.communicator.residual import batch as residual_batch
|
||||
from sglang.srt.layers.dp_attention import is_dp_attention_enabled
|
||||
from sglang.srt.layers.layer_boundary.residual import batch as residual_batch
|
||||
from sglang.srt.layers.layernorm import RMSNorm
|
||||
from sglang.srt.layers.linear import (
|
||||
ColumnParallelLinear,
|
||||
@@ -1149,7 +1149,7 @@ class BailingMoELinearDecoderLayer(nn.Module):
|
||||
self.input_layernorm = RMSNorm(self.hidden_size, eps=rms_norm_eps)
|
||||
self.post_attention_layernorm = RMSNorm(self.hidden_size, eps=rms_norm_eps)
|
||||
|
||||
self.attn_stage, self.ffn_stage = make_stages(
|
||||
self.attn_boundary, self.ffn_boundary = make_stages(
|
||||
(
|
||||
declare_attn(),
|
||||
self.input_layernorm,
|
||||
@@ -1182,7 +1182,7 @@ class BailingMoELinearDecoderLayer(nn.Module):
|
||||
capture_output=None,
|
||||
**kwargs,
|
||||
) -> torch.Tensor:
|
||||
hidden_states = self.attn_stage.prepare(
|
||||
hidden_states = self.attn_boundary.prepare(
|
||||
hidden_states, forward_batch, capture_output=capture_output
|
||||
)
|
||||
|
||||
@@ -1208,10 +1208,10 @@ class BailingMoELinearDecoderLayer(nn.Module):
|
||||
forward_batch=forward_batch,
|
||||
)
|
||||
|
||||
hidden_states = self.attn_stage.finish(hidden_states, forward_batch)
|
||||
hidden_states = self.ffn_stage.prepare(hidden_states, forward_batch)
|
||||
hidden_states = self.attn_boundary.finish(hidden_states, forward_batch)
|
||||
hidden_states = self.ffn_boundary.prepare(hidden_states, forward_batch)
|
||||
|
||||
with self.ffn_stage.exit(forward_batch) as ffn_exit:
|
||||
with self.ffn_boundary.exit(forward_batch) as ffn_exit:
|
||||
if not (
|
||||
enable_moe_dense_fully_dp()
|
||||
and (not self.is_layer_sparse)
|
||||
@@ -1344,7 +1344,7 @@ class BailingMoELinearModel(nn.Module):
|
||||
residual_batch.start(forward_batch)
|
||||
else:
|
||||
assert pp_proxy_tensors is not None
|
||||
hidden_states = self.layers[self.start_layer].attn_stage.from_pp(
|
||||
hidden_states = self.layers[self.start_layer].attn_boundary.from_pp(
|
||||
pp_proxy_tensors, forward_batch
|
||||
)
|
||||
|
||||
|
||||
@@ -8,12 +8,12 @@ from sglang.kernels.ops.attention.dsa.dequant_k_cache import dequantize_k_cache_
|
||||
from sglang.kernels.ops.attention.utils import concat_and_cast_mha_k_triton
|
||||
from sglang.srt.environ import envs
|
||||
from sglang.srt.layers.attention.tbo_backend import TboAttnBackend
|
||||
from sglang.srt.layers.communicator import get_attn_tp_context
|
||||
from sglang.srt.layers.dcp import (
|
||||
all_gather_kv_cache_for_mha_chunk_extend,
|
||||
all_gather_kv_cache_for_mha_extend,
|
||||
filter_dcp_local_kv_indices,
|
||||
)
|
||||
from sglang.srt.layers.layer_boundary import get_attn_tp_context
|
||||
from sglang.srt.model_executor.forward_batch_info import ForwardBatch
|
||||
from sglang.srt.model_executor.forward_context import (
|
||||
get_attn_backend,
|
||||
|
||||
+1
-1
@@ -13,8 +13,8 @@ from typing import TYPE_CHECKING
|
||||
import torch
|
||||
|
||||
from sglang.kernels.ops.attention.utils import concat_and_cast_mha_k_triton
|
||||
from sglang.srt.layers.communicator import get_attn_tp_context
|
||||
from sglang.srt.layers.dcp import all_gather_kv_cache_for_mha_extend
|
||||
from sglang.srt.layers.layer_boundary import get_attn_tp_context
|
||||
from sglang.srt.layers.quantization.fp8_utils import (
|
||||
materialize_bpreshuffle_fp8_scale_tuple,
|
||||
)
|
||||
|
||||
@@ -16,13 +16,13 @@ from sglang.srt.environ import envs
|
||||
from sglang.srt.layers import deep_gemm_wrapper
|
||||
from sglang.srt.layers.attention.dsa.utils import is_graph_dsa_split_op_surface
|
||||
from sglang.srt.layers.attention.dsa_backend import prepare_kv_for_attention
|
||||
from sglang.srt.layers.communicator import get_attn_tp_context
|
||||
from sglang.srt.layers.dcp import (
|
||||
all_gather_kv_cache_for_mla_extend,
|
||||
all_gather_q_for_mla_decode,
|
||||
cp_lse_ag_out_rs_mla,
|
||||
dcp_a2a_lse_reduce,
|
||||
)
|
||||
from sglang.srt.layers.layer_boundary import get_attn_tp_context
|
||||
from sglang.srt.layers.logits_processor import get_in_autotune_dummy_run
|
||||
from sglang.srt.layers.radix_attention import unified_attention_with_output
|
||||
from sglang.srt.lora.deepseek_mla_correction import (
|
||||
|
||||
+1
-1
@@ -20,13 +20,13 @@ from sglang.kernels.ops.quantization.fp8_kernel import (
|
||||
)
|
||||
from sglang.srt.environ import envs
|
||||
from sglang.srt.layers import deep_gemm_wrapper
|
||||
from sglang.srt.layers.communicator import get_attn_tp_context
|
||||
from sglang.srt.layers.dcp import (
|
||||
all_gather_kv_cache_for_mla_extend,
|
||||
all_gather_q_for_mla_decode,
|
||||
cp_lse_ag_out_rs_mla,
|
||||
dcp_a2a_lse_reduce,
|
||||
)
|
||||
from sglang.srt.layers.layer_boundary import get_attn_tp_context
|
||||
from sglang.srt.layers.logits_processor import get_in_autotune_dummy_run
|
||||
from sglang.srt.layers.quantization.fp8_utils import (
|
||||
emit_transposed_bpreshuffle_scale,
|
||||
|
||||
@@ -28,7 +28,7 @@ from sglang.kernels.ops.layernorm.fused_eh_norm import fused_eh_norm
|
||||
from sglang.srt.environ import envs
|
||||
from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder
|
||||
from sglang.srt.layers.attention.index_topk_share import IndexTopKShareState
|
||||
from sglang.srt.layers.communicator.residual import batch as residual_batch
|
||||
from sglang.srt.layers.layer_boundary.residual import batch as residual_batch
|
||||
from sglang.srt.layers.layernorm import RMSNorm
|
||||
from sglang.srt.layers.linear import ReplicatedLinear
|
||||
from sglang.srt.layers.logits_processor import LogitsProcessor
|
||||
|
||||
@@ -67,23 +67,23 @@ from sglang.srt.layers.aux_hidden_states import (
|
||||
AuxHiddenStateAccumulator,
|
||||
AuxHiddenStatePacker,
|
||||
)
|
||||
from sglang.srt.layers.communicator import (
|
||||
from sglang.srt.layers.cp.cp_decode_attn_tp import get_cp_decode_attn_tp_ctx
|
||||
from sglang.srt.layers.dcp.planner import (
|
||||
prepare_decode_context_parallel_metadata,
|
||||
)
|
||||
from sglang.srt.layers.flashinfer_comm_fusion import uses_cutedsl_ar_fusion
|
||||
from sglang.srt.layers.layer_boundary import (
|
||||
declare_attn,
|
||||
declare_ffn,
|
||||
enable_moe_dense_fully_dp,
|
||||
get_attn_tp_context,
|
||||
make_stages,
|
||||
)
|
||||
from sglang.srt.layers.communicator.adapters.context_parallel import (
|
||||
from sglang.srt.layers.layer_boundary.adapters.context_parallel import (
|
||||
maybe_prefetch_next_full_attention_kv,
|
||||
)
|
||||
from sglang.srt.layers.communicator.residual import access as residual_access
|
||||
from sglang.srt.layers.communicator.residual import batch as residual_batch
|
||||
from sglang.srt.layers.cp.cp_decode_attn_tp import get_cp_decode_attn_tp_ctx
|
||||
from sglang.srt.layers.dcp.planner import (
|
||||
prepare_decode_context_parallel_metadata,
|
||||
)
|
||||
from sglang.srt.layers.flashinfer_comm_fusion import uses_cutedsl_ar_fusion
|
||||
from sglang.srt.layers.layer_boundary.residual import access as residual_access
|
||||
from sglang.srt.layers.layer_boundary.residual import batch as residual_batch
|
||||
from sglang.srt.layers.layernorm import RMSNorm
|
||||
from sglang.srt.layers.linear import (
|
||||
ColumnParallelLinear,
|
||||
@@ -1109,7 +1109,7 @@ class DeepseekV2MoE(nn.Module):
|
||||
if deferred_finalize and get_forward().defer_moe_finalize:
|
||||
# deferred_finalize excludes _shared_expert_tp1, so the shared add folds in.
|
||||
assert shared_output is not None
|
||||
from sglang.srt.layers.communicator.fusions.cutedsl import (
|
||||
from sglang.srt.layers.layer_boundary.fusions.cutedsl import (
|
||||
MoeFinalizeHandoff,
|
||||
)
|
||||
|
||||
@@ -2619,7 +2619,7 @@ class DeepseekV2DecoderLayer(nn.Module):
|
||||
|
||||
self._gfx95_quant_format = self._detect_gfx95_quant_format()
|
||||
|
||||
self.attn_stage, self.ffn_stage = self._build_stages(
|
||||
self.attn_boundary, self.ffn_boundary = self._build_stages(
|
||||
input_layernorm=self.input_layernorm,
|
||||
post_attention_layernorm=self.post_attention_layernorm,
|
||||
qkv_latent_func=self.self_attn.prepare_qkv_latent,
|
||||
@@ -2642,10 +2642,10 @@ class DeepseekV2DecoderLayer(nn.Module):
|
||||
and _use_mnnvl_cutedsl_fusion()
|
||||
):
|
||||
# Dense layers too: selecting cutedsl turns the legacy fusion off.
|
||||
from sglang.srt.layers.communicator.fusions.cutedsl import CuteDSLFusion
|
||||
from sglang.srt.layers.layer_boundary.fusions.cutedsl import CuteDSLFusion
|
||||
|
||||
fusions = CuteDSLFusion()
|
||||
attn_stage, ffn_stage = make_stages(
|
||||
attn_boundary, ffn_boundary = make_stages(
|
||||
(
|
||||
declare_attn(),
|
||||
input_layernorm,
|
||||
@@ -2667,7 +2667,7 @@ class DeepseekV2DecoderLayer(nn.Module):
|
||||
else None,
|
||||
terminal=self._stage_terminal,
|
||||
)
|
||||
return attn_stage, ffn_stage
|
||||
return attn_boundary, ffn_boundary
|
||||
|
||||
def _detect_gfx95_quant_format(self) -> str:
|
||||
if not _is_gfx95_supported:
|
||||
@@ -2721,7 +2721,7 @@ class DeepseekV2DecoderLayer(nn.Module):
|
||||
next_full_attention_layer_id: Optional[int] = None,
|
||||
) -> torch.Tensor:
|
||||
hidden_states_orig = residual_access.buffer(hidden_states)
|
||||
hidden_states = self.attn_stage.prepare(
|
||||
hidden_states = self.attn_boundary.prepare(
|
||||
hidden_states,
|
||||
forward_batch,
|
||||
captured_last_layer_outputs=captured_last_layer_outputs,
|
||||
@@ -2736,7 +2736,7 @@ class DeepseekV2DecoderLayer(nn.Module):
|
||||
zero_allocator=zero_allocator,
|
||||
llama_4_scaling=llama_4_scaling,
|
||||
input_on_attention_tp_slices=(
|
||||
self.attn_stage.input_on_attention_tp_slices
|
||||
self.attn_boundary.input_on_attention_tp_slices
|
||||
),
|
||||
prev_topk_indices=prev_topk_indices,
|
||||
)
|
||||
@@ -2750,8 +2750,8 @@ class DeepseekV2DecoderLayer(nn.Module):
|
||||
forward_batch, next_full_attention_layer_id
|
||||
)
|
||||
|
||||
hidden_states = self.attn_stage.finish(hidden_states, forward_batch)
|
||||
hidden_states = self.ffn_stage.prepare(hidden_states, forward_batch)
|
||||
hidden_states = self.attn_boundary.finish(hidden_states, forward_batch)
|
||||
hidden_states = self.ffn_boundary.prepare(hidden_states, forward_batch)
|
||||
|
||||
if isinstance(self.mlp, DeepseekV2MLP):
|
||||
gemm_output_zero_allocator = None
|
||||
@@ -2769,7 +2769,7 @@ class DeepseekV2DecoderLayer(nn.Module):
|
||||
else:
|
||||
_mlp_ctx = nullcontext()
|
||||
|
||||
with self.ffn_stage.exit(forward_batch) as ffn_exit, _mlp_ctx:
|
||||
with self.ffn_boundary.exit(forward_batch) as ffn_exit, _mlp_ctx:
|
||||
hidden_states = self.mlp(
|
||||
hidden_states,
|
||||
forward_batch,
|
||||
@@ -2788,7 +2788,7 @@ class DeepseekV2DecoderLayer(nn.Module):
|
||||
zero_allocator: BumpAllocator,
|
||||
tbo_subbatch_index: Optional[int] = None,
|
||||
):
|
||||
state.hidden_states_after_comm_pre_attn = self.attn_stage.prepare(
|
||||
state.hidden_states_after_comm_pre_attn = self.attn_boundary.prepare(
|
||||
hidden_states, forward_batch
|
||||
)
|
||||
if get_moe_a2a_backend().is_mori():
|
||||
@@ -2803,15 +2803,15 @@ class DeepseekV2DecoderLayer(nn.Module):
|
||||
)
|
||||
|
||||
def op_comm_prepare_mlp(self, state):
|
||||
hidden_states = self.attn_stage.finish(
|
||||
hidden_states = self.attn_boundary.finish(
|
||||
state.pop("hidden_states_after_attn"), state.forward_batch
|
||||
)
|
||||
state.hidden_states_mlp_input = self.ffn_stage.prepare(
|
||||
state.hidden_states_mlp_input = self.ffn_boundary.prepare(
|
||||
hidden_states, state.forward_batch
|
||||
)
|
||||
|
||||
def op_comm_postprocess_layer(self, state):
|
||||
hidden_states = self.ffn_stage.postprocess(
|
||||
hidden_states = self.ffn_boundary.postprocess(
|
||||
state.pop("hidden_states_mlp_output"), state.forward_batch
|
||||
)
|
||||
|
||||
@@ -2978,7 +2978,7 @@ class DeepseekV2Model(nn.Module):
|
||||
# llama_4_scaling: for supporting Mistral-Large-3 model
|
||||
self.llama_4_scaling_config = getattr(config, "llama_4_scaling", None)
|
||||
|
||||
from sglang.srt.layers.communicator.fusions.cutedsl import (
|
||||
from sglang.srt.layers.layer_boundary.fusions.cutedsl import (
|
||||
install_cutedsl_fusion,
|
||||
)
|
||||
|
||||
@@ -3030,7 +3030,7 @@ class DeepseekV2Model(nn.Module):
|
||||
initial_topk_indices = None
|
||||
else:
|
||||
assert pp_proxy_tensors is not None
|
||||
hidden_states = self.layers[self.start_layer].attn_stage.from_pp(
|
||||
hidden_states = self.layers[self.start_layer].attn_boundary.from_pp(
|
||||
pp_proxy_tensors, forward_batch
|
||||
)
|
||||
initial_topk_indices = pp_proxy_tensors.tensors.get("topk_indices")
|
||||
|
||||
@@ -70,11 +70,6 @@ from sglang.srt.layers.attention.dsv4.dsv41_sparse import (
|
||||
DeepseekV41Indexer,
|
||||
)
|
||||
from sglang.srt.layers.attention.dsv4.indexer import C4Indexer
|
||||
from sglang.srt.layers.communicator import get_attn_tp_context
|
||||
from sglang.srt.layers.communicator.adapters.context_parallel import (
|
||||
dsa_cp_gather_hidden_states,
|
||||
dsa_cp_reduce_scatter_hidden_states,
|
||||
)
|
||||
from sglang.srt.layers.cp.cp_decode_attn_tp import get_cp_decode_attn_tp_ctx
|
||||
from sglang.srt.layers.cp.utils import (
|
||||
cp_gather_full_sequence_states,
|
||||
@@ -103,6 +98,11 @@ from sglang.srt.layers.dp_attention import (
|
||||
is_dp_gatherv_active,
|
||||
)
|
||||
from sglang.srt.layers.engram import Engram, EngramHasher, EngramLayout
|
||||
from sglang.srt.layers.layer_boundary import get_attn_tp_context
|
||||
from sglang.srt.layers.layer_boundary.adapters.context_parallel import (
|
||||
dsa_cp_gather_hidden_states,
|
||||
dsa_cp_reduce_scatter_hidden_states,
|
||||
)
|
||||
from sglang.srt.layers.layernorm import RMSNorm
|
||||
from sglang.srt.layers.linear import ColumnParallelLinear, RowParallelLinear
|
||||
from sglang.srt.layers.logits_processor import LogitsMetadata, LogitsProcessor
|
||||
|
||||
@@ -57,14 +57,14 @@ from sglang.srt.eplb.expert_location_dispatch import ExpertLocationDispatchInfo
|
||||
from sglang.srt.layers import deep_gemm_wrapper
|
||||
from sglang.srt.layers.activation import SiluAndMul
|
||||
from sglang.srt.layers.attention.dsa.dsa_indexer import Indexer
|
||||
from sglang.srt.layers.communicator import (
|
||||
from sglang.srt.layers.dp_attention import is_dp_attention_enabled
|
||||
from sglang.srt.layers.layer_boundary import (
|
||||
declare_attn,
|
||||
declare_ffn,
|
||||
enable_moe_dense_fully_dp,
|
||||
make_stages,
|
||||
)
|
||||
from sglang.srt.layers.communicator.residual import batch as residual_batch
|
||||
from sglang.srt.layers.dp_attention import is_dp_attention_enabled
|
||||
from sglang.srt.layers.layer_boundary.residual import batch as residual_batch
|
||||
from sglang.srt.layers.layernorm import RMSNorm
|
||||
from sglang.srt.layers.linear import (
|
||||
ColumnParallelLinear,
|
||||
@@ -1572,7 +1572,7 @@ class Dots3DecoderLayer(nn.Module):
|
||||
config.hidden_size, eps=config.rms_norm_eps
|
||||
)
|
||||
|
||||
self.attn_stage, self.ffn_stage = make_stages(
|
||||
self.attn_boundary, self.ffn_boundary = make_stages(
|
||||
(declare_attn(), self.input_layernorm),
|
||||
(
|
||||
declare_ffn(
|
||||
@@ -1607,7 +1607,7 @@ class Dots3DecoderLayer(nn.Module):
|
||||
zero_allocator: BumpAllocator,
|
||||
) -> torch.Tensor:
|
||||
|
||||
hidden_states = self.attn_stage.prepare(hidden_states, forward_batch)
|
||||
hidden_states = self.attn_boundary.prepare(hidden_states, forward_batch)
|
||||
|
||||
hidden_states = self.self_attn(
|
||||
positions=positions,
|
||||
@@ -1616,10 +1616,10 @@ class Dots3DecoderLayer(nn.Module):
|
||||
zero_allocator=zero_allocator,
|
||||
)
|
||||
|
||||
hidden_states = self.attn_stage.finish(hidden_states, forward_batch)
|
||||
hidden_states = self.ffn_stage.prepare(hidden_states, forward_batch)
|
||||
hidden_states = self.attn_boundary.finish(hidden_states, forward_batch)
|
||||
hidden_states = self.ffn_boundary.prepare(hidden_states, forward_batch)
|
||||
|
||||
with self.ffn_stage.exit(forward_batch) as ffn_exit:
|
||||
with self.ffn_boundary.exit(forward_batch) as ffn_exit:
|
||||
hidden_states = self.mlp(
|
||||
hidden_states,
|
||||
forward_batch,
|
||||
@@ -1637,7 +1637,7 @@ class Dots3DecoderLayer(nn.Module):
|
||||
zero_allocator: BumpAllocator,
|
||||
tbo_subbatch_index: Optional[int] = None,
|
||||
):
|
||||
state.hidden_states_after_comm_pre_attn = self.attn_stage.prepare(
|
||||
state.hidden_states_after_comm_pre_attn = self.attn_boundary.prepare(
|
||||
hidden_states, forward_batch
|
||||
)
|
||||
state.update(
|
||||
@@ -1650,10 +1650,10 @@ class Dots3DecoderLayer(nn.Module):
|
||||
)
|
||||
|
||||
def op_comm_prepare_mlp(self, state):
|
||||
hidden_states = self.attn_stage.finish(
|
||||
hidden_states = self.attn_boundary.finish(
|
||||
state.pop("hidden_states_after_attn"), state.forward_batch
|
||||
)
|
||||
state.hidden_states_mlp_input = self.ffn_stage.prepare(
|
||||
state.hidden_states_mlp_input = self.ffn_boundary.prepare(
|
||||
hidden_states, state.forward_batch
|
||||
)
|
||||
|
||||
@@ -1671,7 +1671,7 @@ class Dots3DecoderLayer(nn.Module):
|
||||
state.hidden_states_mlp_output = hidden_states
|
||||
|
||||
def op_comm_postprocess_layer(self, state):
|
||||
hidden_states = self.ffn_stage.postprocess(
|
||||
hidden_states = self.ffn_boundary.postprocess(
|
||||
state.pop("hidden_states_mlp_output"), state.forward_batch
|
||||
)
|
||||
|
||||
@@ -1763,7 +1763,7 @@ class Dots3Model(nn.Module):
|
||||
residual_batch.start(forward_batch)
|
||||
else:
|
||||
assert pp_proxy_tensors is not None
|
||||
hidden_states = self.layers[self.start_layer].attn_stage.from_pp(
|
||||
hidden_states = self.layers[self.start_layer].attn_boundary.from_pp(
|
||||
pp_proxy_tensors, forward_batch
|
||||
)
|
||||
|
||||
|
||||
@@ -8,8 +8,8 @@ from torch import nn
|
||||
from transformers import PretrainedConfig
|
||||
|
||||
from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder
|
||||
from sglang.srt.layers.communicator.residual import batch as residual_batch
|
||||
from sglang.srt.layers.dp_attention import is_dp_attention_enabled
|
||||
from sglang.srt.layers.layer_boundary.residual import batch as residual_batch
|
||||
from sglang.srt.layers.layernorm import RMSNorm
|
||||
from sglang.srt.layers.linear import ReplicatedLinear
|
||||
from sglang.srt.layers.logits_processor import LogitsProcessor
|
||||
|
||||
@@ -26,7 +26,7 @@ from transformers.models.ernie4_5_moe.configuration_ernie4_5_moe import (
|
||||
from sglang.srt.distributed import (
|
||||
tensor_model_parallel_all_reduce,
|
||||
)
|
||||
from sglang.srt.layers.communicator import enable_moe_dense_fully_dp
|
||||
from sglang.srt.layers.layer_boundary import enable_moe_dense_fully_dp
|
||||
from sglang.srt.layers.layernorm import RMSNorm
|
||||
from sglang.srt.layers.logits_processor import LogitsProcessor
|
||||
from sglang.srt.layers.moe.ep_moe.layer import get_moe_impl_class
|
||||
|
||||
@@ -11,16 +11,16 @@ from sglang.srt.layers.attention.hybrid_linear_attn_backend import (
|
||||
Mamba2AttnBackend,
|
||||
)
|
||||
from sglang.srt.layers.attention.mamba.mamba import MambaMixer2
|
||||
from sglang.srt.layers.communicator import (
|
||||
from sglang.srt.layers.dp_attention import (
|
||||
is_dp_attention_enabled,
|
||||
)
|
||||
from sglang.srt.layers.layer_boundary import (
|
||||
declare_attn,
|
||||
declare_ffn,
|
||||
make_stages,
|
||||
)
|
||||
from sglang.srt.layers.communicator.output import OutputTransform
|
||||
from sglang.srt.layers.communicator.residual import batch as residual_batch
|
||||
from sglang.srt.layers.dp_attention import (
|
||||
is_dp_attention_enabled,
|
||||
)
|
||||
from sglang.srt.layers.layer_boundary.output import OutputTransform
|
||||
from sglang.srt.layers.layer_boundary.residual import batch as residual_batch
|
||||
from sglang.srt.layers.layernorm import RMSNorm
|
||||
from sglang.srt.layers.linear import (
|
||||
MergedColumnParallelLinear,
|
||||
@@ -222,7 +222,7 @@ class FalconH1HybridAttentionDecoderLayer(nn.Module):
|
||||
self.q_norm = RMSNorm(self.head_dim, eps=config.rms_norm_eps)
|
||||
self.k_norm = RMSNorm(self.head_dim, eps=config.rms_norm_eps)
|
||||
|
||||
self.attn_stage, self.ffn_stage = make_stages(
|
||||
self.attn_boundary, self.ffn_boundary = make_stages(
|
||||
(declare_attn(), self.input_layernorm),
|
||||
(
|
||||
declare_ffn(
|
||||
@@ -334,7 +334,7 @@ class FalconH1HybridAttentionDecoderLayer(nn.Module):
|
||||
forward_batch: ForwardBatch,
|
||||
**kwargs: Any,
|
||||
):
|
||||
hidden_states = self.attn_stage.prepare(hidden_states, forward_batch)
|
||||
hidden_states = self.attn_boundary.prepare(hidden_states, forward_batch)
|
||||
|
||||
if not forward_batch.forward_mode.is_idle():
|
||||
# Attention block
|
||||
@@ -363,9 +363,9 @@ class FalconH1HybridAttentionDecoderLayer(nn.Module):
|
||||
hidden_states = attention_hidden_states + mamba_hidden_states
|
||||
|
||||
# Fully Connected
|
||||
hidden_states = self.attn_stage.finish(hidden_states, forward_batch)
|
||||
hidden_states = self.ffn_stage.prepare(hidden_states, forward_batch)
|
||||
with self.ffn_stage.exit(forward_batch) as ffn_exit:
|
||||
hidden_states = self.attn_boundary.finish(hidden_states, forward_batch)
|
||||
hidden_states = self.ffn_boundary.prepare(hidden_states, forward_batch)
|
||||
with self.ffn_boundary.exit(forward_batch) as ffn_exit:
|
||||
hidden_states = self.feed_forward(hidden_states, forward_batch)
|
||||
return ffn_exit.finish(hidden_states)
|
||||
|
||||
|
||||
@@ -23,9 +23,9 @@ from transformers import PretrainedConfig
|
||||
|
||||
import sglang.srt.models.deepseek_v2 as deepseek_v2
|
||||
from sglang.srt.configs.gigachat35 import GigaChat35Config
|
||||
from sglang.srt.layers.communicator import get_attn_tp_context
|
||||
from sglang.srt.layers.communicator.output import OutputTransform
|
||||
from sglang.srt.layers.communicator.residual import batch as residual_batch
|
||||
from sglang.srt.layers.layer_boundary import get_attn_tp_context
|
||||
from sglang.srt.layers.layer_boundary.output import OutputTransform
|
||||
from sglang.srt.layers.layer_boundary.residual import batch as residual_batch
|
||||
from sglang.srt.layers.layernorm import GemmaRMSNorm, RMSNorm
|
||||
from sglang.srt.layers.linear import ColumnParallelLinear
|
||||
from sglang.srt.layers.logits_processor import LogitsProcessor
|
||||
@@ -441,7 +441,7 @@ class GigaChat35DecoderLayer(deepseek_v2.DeepseekV2DecoderLayer):
|
||||
post_layernorm=self.post_self_attn_layernorm,
|
||||
)
|
||||
# The communicator chooses its fused steps from its norms when built.
|
||||
self.attn_stage, self.ffn_stage = self._build_stages(
|
||||
self.attn_boundary, self.ffn_boundary = self._build_stages(
|
||||
input_layernorm=attn_prepare_layernorm,
|
||||
post_attention_layernorm=mlp_prepare_layernorm,
|
||||
qkv_latent_func=qkv_latent_func,
|
||||
@@ -471,7 +471,7 @@ class GigaChat35DecoderLayer(deepseek_v2.DeepseekV2DecoderLayer):
|
||||
zero_allocator: BumpAllocator,
|
||||
**kwargs,
|
||||
) -> torch.Tensor:
|
||||
hidden_states = self.attn_stage.prepare(hidden_states, forward_batch)
|
||||
hidden_states = self.attn_boundary.prepare(hidden_states, forward_batch)
|
||||
|
||||
if self.use_linear_attn:
|
||||
hidden_states = self.self_attn(
|
||||
@@ -485,18 +485,18 @@ class GigaChat35DecoderLayer(deepseek_v2.DeepseekV2DecoderLayer):
|
||||
forward_batch=forward_batch,
|
||||
zero_allocator=zero_allocator,
|
||||
input_on_attention_tp_slices=(
|
||||
self.attn_stage.input_on_attention_tp_slices
|
||||
self.attn_boundary.input_on_attention_tp_slices
|
||||
),
|
||||
)
|
||||
get_attn_tp_context().clear_attn_inputs()
|
||||
|
||||
hidden_states = self.attn_stage.finish(hidden_states, forward_batch)
|
||||
hidden_states = self.ffn_stage.prepare(hidden_states, forward_batch)
|
||||
hidden_states = self.attn_boundary.finish(hidden_states, forward_batch)
|
||||
hidden_states = self.ffn_boundary.prepare(hidden_states, forward_batch)
|
||||
|
||||
# Unlike deepseek_v2, no moe_output_buffer_ctx here: non-inplace MoE
|
||||
# runners then allocate their output per forward instead of recycling
|
||||
# the layer-input buffer. The default (inplace) runners are unaffected.
|
||||
with self.ffn_stage.exit(forward_batch) as ffn_exit:
|
||||
with self.ffn_boundary.exit(forward_batch) as ffn_exit:
|
||||
hidden_states = self.mlp(hidden_states, forward_batch)
|
||||
|
||||
hidden_states = ffn_exit.finish(hidden_states)
|
||||
@@ -562,7 +562,7 @@ class GigaChat35Model(nn.Module):
|
||||
residual_batch.start(forward_batch)
|
||||
else:
|
||||
assert pp_proxy_tensors is not None
|
||||
hidden_states = self.layers[self.start_layer].attn_stage.from_pp(
|
||||
hidden_states = self.layers[self.start_layer].attn_boundary.from_pp(
|
||||
pp_proxy_tensors, forward_batch
|
||||
)
|
||||
|
||||
|
||||
@@ -18,7 +18,7 @@ from typing import Iterable, Optional
|
||||
import torch
|
||||
from torch import nn
|
||||
|
||||
from sglang.srt.layers.communicator.residual import batch as residual_batch
|
||||
from sglang.srt.layers.layer_boundary.residual import batch as residual_batch
|
||||
from sglang.srt.layers.logits_processor import LogitsProcessor
|
||||
from sglang.srt.layers.quantization.base_config import QuantizationConfig
|
||||
from sglang.srt.layers.vocab_parallel_embedding import (
|
||||
|
||||
@@ -39,17 +39,17 @@ from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation
|
||||
from sglang.srt.eplb.expert_location_dispatch import ExpertLocationDispatchInfo
|
||||
from sglang.srt.layers.activation import SiluAndMul
|
||||
from sglang.srt.layers.aux_hidden_states import AuxHiddenStateList
|
||||
from sglang.srt.layers.communicator import (
|
||||
from sglang.srt.layers.dp_attention import (
|
||||
is_allocation_symmetric,
|
||||
is_dp_attention_enabled,
|
||||
)
|
||||
from sglang.srt.layers.layer_boundary import (
|
||||
declare_attn,
|
||||
declare_ffn,
|
||||
enable_moe_dense_fully_dp,
|
||||
make_stages,
|
||||
)
|
||||
from sglang.srt.layers.communicator.residual import batch as residual_batch
|
||||
from sglang.srt.layers.dp_attention import (
|
||||
is_allocation_symmetric,
|
||||
is_dp_attention_enabled,
|
||||
)
|
||||
from sglang.srt.layers.layer_boundary.residual import batch as residual_batch
|
||||
from sglang.srt.layers.layernorm import RMSNorm
|
||||
from sglang.srt.layers.linear import (
|
||||
MergedColumnParallelLinear,
|
||||
@@ -866,7 +866,7 @@ class Glm4MoeDecoderLayer(nn.Module):
|
||||
config.hidden_size, eps=config.rms_norm_eps
|
||||
)
|
||||
|
||||
self.attn_stage, self.ffn_stage = make_stages(
|
||||
self.attn_boundary, self.ffn_boundary = make_stages(
|
||||
(declare_attn(), self.input_layernorm),
|
||||
(
|
||||
declare_ffn(
|
||||
@@ -942,7 +942,7 @@ class Glm4MoeDecoderLayer(nn.Module):
|
||||
capture_output=None,
|
||||
) -> torch.Tensor:
|
||||
|
||||
hidden_states = self.attn_stage.prepare(
|
||||
hidden_states = self.attn_boundary.prepare(
|
||||
hidden_states,
|
||||
forward_batch,
|
||||
quant_format=self.attn_quant_format,
|
||||
@@ -955,10 +955,10 @@ class Glm4MoeDecoderLayer(nn.Module):
|
||||
forward_batch=forward_batch,
|
||||
)
|
||||
|
||||
hidden_states = self.attn_stage.finish(hidden_states, forward_batch)
|
||||
hidden_states = self.ffn_stage.prepare(hidden_states, forward_batch)
|
||||
hidden_states = self.attn_boundary.finish(hidden_states, forward_batch)
|
||||
hidden_states = self.ffn_boundary.prepare(hidden_states, forward_batch)
|
||||
|
||||
with self.ffn_stage.exit(forward_batch) as ffn_exit:
|
||||
with self.ffn_boundary.exit(forward_batch) as ffn_exit:
|
||||
hidden_states = self.mlp(hidden_states, forward_batch)
|
||||
hidden_states = ffn_exit.finish(hidden_states)
|
||||
|
||||
@@ -972,7 +972,7 @@ class Glm4MoeDecoderLayer(nn.Module):
|
||||
forward_batch: ForwardBatch,
|
||||
tbo_subbatch_index: Optional[int] = None,
|
||||
):
|
||||
state.hidden_states_after_comm_pre_attn = self.attn_stage.prepare(
|
||||
state.hidden_states_after_comm_pre_attn = self.attn_boundary.prepare(
|
||||
hidden_states, forward_batch
|
||||
)
|
||||
state.update(
|
||||
@@ -984,15 +984,15 @@ class Glm4MoeDecoderLayer(nn.Module):
|
||||
)
|
||||
|
||||
def op_comm_prepare_mlp(self, state):
|
||||
hidden_states = self.attn_stage.finish(
|
||||
hidden_states = self.attn_boundary.finish(
|
||||
state.pop("hidden_states_after_attn"), state.forward_batch
|
||||
)
|
||||
state.hidden_states_mlp_input = self.ffn_stage.prepare(
|
||||
state.hidden_states_mlp_input = self.ffn_boundary.prepare(
|
||||
hidden_states, state.forward_batch
|
||||
)
|
||||
|
||||
def op_comm_postprocess_layer(self, state):
|
||||
hidden_states = self.ffn_stage.postprocess(
|
||||
hidden_states = self.ffn_boundary.postprocess(
|
||||
state.pop("hidden_states_mlp_output"), state.forward_batch
|
||||
)
|
||||
|
||||
@@ -1081,7 +1081,7 @@ class Glm4MoeModel(nn.Module):
|
||||
residual_batch.start(forward_batch)
|
||||
else:
|
||||
assert pp_proxy_tensors is not None
|
||||
hidden_states = self.layers[self.start_layer].attn_stage.from_pp(
|
||||
hidden_states = self.layers[self.start_layer].attn_boundary.from_pp(
|
||||
pp_proxy_tensors, forward_batch
|
||||
)
|
||||
|
||||
|
||||
@@ -33,18 +33,18 @@ from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation
|
||||
from sglang.srt.eplb.expert_location_dispatch import ExpertLocationDispatchInfo
|
||||
from sglang.srt.layers.activation import SiluAndMul
|
||||
from sglang.srt.layers.aux_hidden_states import AuxHiddenStateList
|
||||
from sglang.srt.layers.communicator import (
|
||||
from sglang.srt.layers.dp_attention import (
|
||||
is_allocation_symmetric,
|
||||
is_dp_attention_enabled,
|
||||
)
|
||||
from sglang.srt.layers.layer_boundary import (
|
||||
declare_attn,
|
||||
declare_ffn,
|
||||
enable_moe_dense_fully_dp,
|
||||
get_attn_tp_context,
|
||||
make_stages,
|
||||
)
|
||||
from sglang.srt.layers.communicator.residual import batch as residual_batch
|
||||
from sglang.srt.layers.dp_attention import (
|
||||
is_allocation_symmetric,
|
||||
is_dp_attention_enabled,
|
||||
)
|
||||
from sglang.srt.layers.layer_boundary.residual import batch as residual_batch
|
||||
from sglang.srt.layers.layernorm import RMSNorm
|
||||
from sglang.srt.layers.linear import MergedColumnParallelLinear, RowParallelLinear
|
||||
from sglang.srt.layers.logits_processor import LogitsProcessor
|
||||
@@ -581,7 +581,7 @@ class Glm4MoeLiteDecoderLayer(nn.Module):
|
||||
|
||||
self._gfx95_quant_format = self._detect_gfx95_quant_format()
|
||||
|
||||
self.attn_stage, self.ffn_stage = make_stages(
|
||||
self.attn_boundary, self.ffn_boundary = make_stages(
|
||||
(
|
||||
declare_attn(),
|
||||
self.input_layernorm,
|
||||
@@ -633,7 +633,7 @@ class Glm4MoeLiteDecoderLayer(nn.Module):
|
||||
zero_allocator: BumpAllocator,
|
||||
capture_output=None,
|
||||
) -> torch.Tensor:
|
||||
hidden_states = self.attn_stage.prepare(
|
||||
hidden_states = self.attn_boundary.prepare(
|
||||
hidden_states,
|
||||
forward_batch,
|
||||
quant_format=getattr(self, "_gfx95_quant_format", ""),
|
||||
@@ -645,16 +645,18 @@ class Glm4MoeLiteDecoderLayer(nn.Module):
|
||||
hidden_states=hidden_states,
|
||||
forward_batch=forward_batch,
|
||||
zero_allocator=zero_allocator,
|
||||
input_on_attention_tp_slices=(self.attn_stage.input_on_attention_tp_slices),
|
||||
input_on_attention_tp_slices=(
|
||||
self.attn_boundary.input_on_attention_tp_slices
|
||||
),
|
||||
)
|
||||
if isinstance(hidden_states, tuple):
|
||||
hidden_states = hidden_states[0]
|
||||
get_attn_tp_context().clear_attn_inputs()
|
||||
|
||||
hidden_states = self.attn_stage.finish(hidden_states, forward_batch)
|
||||
hidden_states = self.ffn_stage.prepare(hidden_states, forward_batch)
|
||||
hidden_states = self.attn_boundary.finish(hidden_states, forward_batch)
|
||||
hidden_states = self.ffn_boundary.prepare(hidden_states, forward_batch)
|
||||
|
||||
with self.ffn_stage.exit(forward_batch) as ffn_exit:
|
||||
with self.ffn_boundary.exit(forward_batch) as ffn_exit:
|
||||
hidden_states = self.mlp(hidden_states, forward_batch)
|
||||
hidden_states = ffn_exit.finish(hidden_states)
|
||||
|
||||
@@ -669,7 +671,7 @@ class Glm4MoeLiteDecoderLayer(nn.Module):
|
||||
zero_allocator: BumpAllocator,
|
||||
tbo_subbatch_index: Optional[int] = None,
|
||||
):
|
||||
state.hidden_states_after_comm_pre_attn = self.attn_stage.prepare(
|
||||
state.hidden_states_after_comm_pre_attn = self.attn_boundary.prepare(
|
||||
hidden_states, forward_batch
|
||||
)
|
||||
if get_moe_a2a_backend().is_mori():
|
||||
@@ -684,15 +686,15 @@ class Glm4MoeLiteDecoderLayer(nn.Module):
|
||||
)
|
||||
|
||||
def op_comm_prepare_mlp(self, state):
|
||||
hidden_states = self.attn_stage.finish(
|
||||
hidden_states = self.attn_boundary.finish(
|
||||
state.pop("hidden_states_after_attn"), state.forward_batch
|
||||
)
|
||||
state.hidden_states_mlp_input = self.ffn_stage.prepare(
|
||||
state.hidden_states_mlp_input = self.ffn_boundary.prepare(
|
||||
hidden_states, state.forward_batch
|
||||
)
|
||||
|
||||
def op_comm_postprocess_layer(self, state):
|
||||
hidden_states = self.ffn_stage.postprocess(
|
||||
hidden_states = self.ffn_boundary.postprocess(
|
||||
state.pop("hidden_states_mlp_output"), state.forward_batch
|
||||
)
|
||||
|
||||
@@ -779,7 +781,7 @@ class Glm4MoeLiteModel(nn.Module):
|
||||
residual_batch.start(forward_batch)
|
||||
else:
|
||||
assert pp_proxy_tensors is not None
|
||||
hidden_states = self.layers[self.start_layer].attn_stage.from_pp(
|
||||
hidden_states = self.layers[self.start_layer].attn_boundary.from_pp(
|
||||
pp_proxy_tensors, forward_batch
|
||||
)
|
||||
device = hidden_states.device
|
||||
|
||||
@@ -22,8 +22,8 @@ from torch import nn
|
||||
from transformers import PretrainedConfig
|
||||
|
||||
from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder
|
||||
from sglang.srt.layers.communicator.residual import batch as residual_batch
|
||||
from sglang.srt.layers.dp_attention import is_dp_attention_enabled
|
||||
from sglang.srt.layers.layer_boundary.residual import batch as residual_batch
|
||||
from sglang.srt.layers.layernorm import RMSNorm
|
||||
from sglang.srt.layers.logits_processor import LogitsProcessor
|
||||
from sglang.srt.layers.quantization.base_config import QuantizationConfig
|
||||
|
||||
@@ -22,8 +22,8 @@ from torch import nn
|
||||
from transformers import PretrainedConfig
|
||||
|
||||
from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder
|
||||
from sglang.srt.layers.communicator.residual import batch as residual_batch
|
||||
from sglang.srt.layers.dp_attention import is_dp_attention_enabled
|
||||
from sglang.srt.layers.layer_boundary.residual import batch as residual_batch
|
||||
from sglang.srt.layers.layernorm import RMSNorm
|
||||
from sglang.srt.layers.logits_processor import LogitsProcessor
|
||||
from sglang.srt.layers.quantization.base_config import QuantizationConfig
|
||||
|
||||
@@ -26,7 +26,7 @@ from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation
|
||||
from sglang.srt.layers.attention import vision_utils
|
||||
from sglang.srt.layers.attention.vision import VisionAttention
|
||||
from sglang.srt.layers.aux_hidden_states import AuxHiddenStateList
|
||||
from sglang.srt.layers.communicator import (
|
||||
from sglang.srt.layers.layer_boundary import (
|
||||
PLAIN_RESIDUAL,
|
||||
MHCState,
|
||||
declare_attn,
|
||||
@@ -35,8 +35,8 @@ from sglang.srt.layers.communicator import (
|
||||
get_attn_tp_context,
|
||||
make_stages,
|
||||
)
|
||||
from sglang.srt.layers.communicator.residual import access as residual_access
|
||||
from sglang.srt.layers.communicator.residual import batch as residual_batch
|
||||
from sglang.srt.layers.layer_boundary.residual import access as residual_access
|
||||
from sglang.srt.layers.layer_boundary.residual import batch as residual_batch
|
||||
from sglang.srt.layers.layernorm import RMSNorm
|
||||
from sglang.srt.layers.linear import (
|
||||
ColumnParallelBatchedLinear,
|
||||
@@ -794,7 +794,7 @@ class Glm5NextDecoderLayer(nn.Module):
|
||||
),
|
||||
is_last_layer=terminal,
|
||||
).layer_residual()
|
||||
self.attn_stage, self.ffn_stage = make_stages(
|
||||
self.attn_boundary, self.ffn_boundary = make_stages(
|
||||
(
|
||||
declare_attn(
|
||||
read=residual.attention_read, update=residual.attention_update
|
||||
@@ -932,7 +932,7 @@ class Glm5NextDecoderLayer(nn.Module):
|
||||
):
|
||||
hidden_states_orig = residual_access.buffer(hidden_states)
|
||||
|
||||
hidden_states = self.attn_stage.prepare(
|
||||
hidden_states = self.attn_boundary.prepare(
|
||||
hidden_states, forward_batch, capture_output=capture_output
|
||||
)
|
||||
|
||||
@@ -941,7 +941,9 @@ class Glm5NextDecoderLayer(nn.Module):
|
||||
hidden_states=hidden_states,
|
||||
forward_batch=forward_batch,
|
||||
zero_allocator=zero_allocator,
|
||||
input_on_attention_tp_slices=(self.attn_stage.input_on_attention_tp_slices),
|
||||
input_on_attention_tp_slices=(
|
||||
self.attn_boundary.input_on_attention_tp_slices
|
||||
),
|
||||
prev_topk_indices=prev_topk_indices,
|
||||
)
|
||||
if isinstance(hidden_states, tuple):
|
||||
@@ -950,8 +952,8 @@ class Glm5NextDecoderLayer(nn.Module):
|
||||
topk_indices = None
|
||||
get_attn_tp_context().clear_attn_inputs()
|
||||
|
||||
hidden_states = self.attn_stage.finish(hidden_states, forward_batch)
|
||||
hidden_states = self.ffn_stage.prepare(hidden_states, forward_batch)
|
||||
hidden_states = self.attn_boundary.finish(hidden_states, forward_batch)
|
||||
hidden_states = self.ffn_boundary.prepare(hidden_states, forward_batch)
|
||||
|
||||
if isinstance(self.mlp, Glm5NextMLP):
|
||||
gemm_output_zero_allocator = None
|
||||
@@ -967,7 +969,7 @@ class Glm5NextDecoderLayer(nn.Module):
|
||||
else:
|
||||
_mlp_ctx = nullcontext()
|
||||
|
||||
with self.ffn_stage.exit(forward_batch) as ffn_exit, _mlp_ctx:
|
||||
with self.ffn_boundary.exit(forward_batch) as ffn_exit, _mlp_ctx:
|
||||
hidden_states = self.mlp(
|
||||
hidden_states,
|
||||
forward_batch,
|
||||
@@ -1108,7 +1110,7 @@ class Glm5NextModel(nn.Module):
|
||||
residual_batch.start(forward_batch)
|
||||
else:
|
||||
assert pp_proxy_tensors is not None
|
||||
hidden_states = self.layers[self.start_layer].attn_stage.from_pp(
|
||||
hidden_states = self.layers[self.start_layer].attn_boundary.from_pp(
|
||||
pp_proxy_tensors, forward_batch
|
||||
)
|
||||
device = hidden_states.device
|
||||
|
||||
@@ -30,15 +30,15 @@ from sglang.kernels.jit.utils import is_arch_support_pdl
|
||||
from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder
|
||||
from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation
|
||||
from sglang.srt.layers.aux_hidden_states import AuxHiddenStateList
|
||||
from sglang.srt.layers.communicator import (
|
||||
from sglang.srt.layers.dp_attention import (
|
||||
is_dp_attention_enabled,
|
||||
)
|
||||
from sglang.srt.layers.layer_boundary import (
|
||||
declare_attn,
|
||||
declare_ffn,
|
||||
make_stages,
|
||||
)
|
||||
from sglang.srt.layers.communicator.residual import batch as residual_batch
|
||||
from sglang.srt.layers.dp_attention import (
|
||||
is_dp_attention_enabled,
|
||||
)
|
||||
from sglang.srt.layers.layer_boundary.residual import batch as residual_batch
|
||||
from sglang.srt.layers.layernorm import RMSNorm
|
||||
from sglang.srt.layers.linear import (
|
||||
QKVParallelLinear,
|
||||
@@ -599,7 +599,7 @@ class GptOssDecoderLayer(nn.Module):
|
||||
x_pad_to_multiple=post_attn_pad_multiple,
|
||||
)
|
||||
|
||||
self.attn_stage, self.ffn_stage = make_stages(
|
||||
self.attn_boundary, self.ffn_boundary = make_stages(
|
||||
(declare_attn(), self.input_layernorm),
|
||||
(
|
||||
declare_ffn(
|
||||
@@ -623,7 +623,7 @@ class GptOssDecoderLayer(nn.Module):
|
||||
forward_batch: ForwardBatch,
|
||||
capture_output=None,
|
||||
) -> torch.Tensor:
|
||||
hidden_states = self.attn_stage.prepare(
|
||||
hidden_states = self.attn_boundary.prepare(
|
||||
hidden_states, forward_batch, capture_output=capture_output
|
||||
)
|
||||
|
||||
@@ -634,10 +634,10 @@ class GptOssDecoderLayer(nn.Module):
|
||||
forward_batch=forward_batch,
|
||||
)
|
||||
|
||||
hidden_states = self.attn_stage.finish(hidden_states, forward_batch)
|
||||
hidden_states = self.ffn_stage.prepare(hidden_states, forward_batch)
|
||||
hidden_states = self.attn_boundary.finish(hidden_states, forward_batch)
|
||||
hidden_states = self.ffn_boundary.prepare(hidden_states, forward_batch)
|
||||
|
||||
with self.ffn_stage.exit(forward_batch) as ffn_exit:
|
||||
with self.ffn_boundary.exit(forward_batch) as ffn_exit:
|
||||
hidden_states = self.mlp(hidden_states, forward_batch)
|
||||
hidden_states = ffn_exit.finish(hidden_states)
|
||||
|
||||
@@ -707,7 +707,7 @@ class GptOssModel(nn.Module):
|
||||
residual_batch.start(forward_batch)
|
||||
else:
|
||||
assert pp_proxy_tensors is not None
|
||||
hidden_states = self.layers[self.start_layer].attn_stage.from_pp(
|
||||
hidden_states = self.layers[self.start_layer].attn_boundary.from_pp(
|
||||
pp_proxy_tensors, forward_batch
|
||||
)
|
||||
|
||||
|
||||
@@ -7,7 +7,7 @@ from torch import nn
|
||||
from transformers import PretrainedConfig
|
||||
|
||||
from sglang.srt.layers.attention.index_topk_share import IndexTopKShareState
|
||||
from sglang.srt.layers.communicator import AttentionInputs, get_attn_tp_context
|
||||
from sglang.srt.layers.layer_boundary import AttentionInputs, get_attn_tp_context
|
||||
from sglang.srt.layers.layernorm import RMSNorm
|
||||
from sglang.srt.layers.linear import ColumnParallelLinear, ReplicatedLinear
|
||||
from sglang.srt.layers.logits_processor import LogitsProcessor
|
||||
|
||||
@@ -5,7 +5,7 @@ import torch
|
||||
from torch import nn
|
||||
|
||||
from sglang.srt.layers.attention.index_topk_share import IndexTopKShareState
|
||||
from sglang.srt.layers.communicator import AttentionInputs, get_attn_tp_context
|
||||
from sglang.srt.layers.layer_boundary import AttentionInputs, get_attn_tp_context
|
||||
from sglang.srt.layers.layernorm import RMSNorm
|
||||
from sglang.srt.layers.logits_processor import LogitsProcessor
|
||||
from sglang.srt.layers.vocab_parallel_embedding import (
|
||||
|
||||
@@ -14,14 +14,14 @@ from sglang.srt.layers.aux_hidden_states import (
|
||||
AuxHiddenStateAccumulator,
|
||||
AuxHiddenStateList,
|
||||
)
|
||||
from sglang.srt.layers.communicator import (
|
||||
from sglang.srt.layers.dp_attention import is_dp_attention_enabled
|
||||
from sglang.srt.layers.layer_boundary import (
|
||||
declare_attn,
|
||||
declare_ffn,
|
||||
make_stages,
|
||||
)
|
||||
from sglang.srt.layers.communicator.residual import batch as residual_batch
|
||||
from sglang.srt.layers.communicator.residual.add_norm import Fp8Input, NormQuantRead
|
||||
from sglang.srt.layers.dp_attention import is_dp_attention_enabled
|
||||
from sglang.srt.layers.layer_boundary.residual import batch as residual_batch
|
||||
from sglang.srt.layers.layer_boundary.residual.add_norm import Fp8Input, NormQuantRead
|
||||
from sglang.srt.layers.layernorm import GemmaRMSNorm
|
||||
from sglang.srt.layers.linear import (
|
||||
QKVParallelLinear,
|
||||
@@ -428,9 +428,9 @@ class _InternS2MobiusDecoderMixin:
|
||||
forward_batch: ForwardBatch,
|
||||
meta_mlp: nn.ModuleList,
|
||||
) -> torch.Tensor:
|
||||
hidden_states = self.attn_stage.finish(hidden_states, forward_batch)
|
||||
hidden_states = self.ffn_stage.prepare(hidden_states, forward_batch)
|
||||
with self.ffn_stage.exit(forward_batch) as ffn_exit:
|
||||
hidden_states = self.attn_boundary.finish(hidden_states, forward_batch)
|
||||
hidden_states = self.ffn_boundary.prepare(hidden_states, forward_batch)
|
||||
with self.ffn_boundary.exit(forward_batch) as ffn_exit:
|
||||
hidden_states = self._forward_mobius_mlp(
|
||||
hidden_states, forward_batch, meta_mlp
|
||||
)
|
||||
@@ -489,7 +489,7 @@ class InternS2MobiusLinearDecoderLayer(_InternS2MobiusDecoderMixin, nn.Module):
|
||||
config.hidden_size, eps=config.rms_norm_eps
|
||||
)
|
||||
accepts_fp8_input = _linear_accepts_fp8_tuple(self.linear_attn.in_proj_qkvz)
|
||||
self.attn_stage, self.ffn_stage = make_stages(
|
||||
self.attn_boundary, self.ffn_boundary = make_stages(
|
||||
(
|
||||
declare_attn(
|
||||
read=NormQuantRead(
|
||||
@@ -518,7 +518,7 @@ class InternS2MobiusLinearDecoderLayer(_InternS2MobiusDecoderMixin, nn.Module):
|
||||
**kwargs,
|
||||
):
|
||||
forward_batch = kwargs["forward_batch"]
|
||||
hidden_states = self.attn_stage.prepare(
|
||||
hidden_states = self.attn_boundary.prepare(
|
||||
hidden_states,
|
||||
forward_batch,
|
||||
captured_last_layer_outputs=kwargs.get("captured_last_layer_outputs"),
|
||||
@@ -621,7 +621,7 @@ class InternS2MobiusAttentionDecoderLayer(
|
||||
self.q_norm = GemmaRMSNorm(self.head_dim, eps=config.rms_norm_eps)
|
||||
self.k_norm = GemmaRMSNorm(self.head_dim, eps=config.rms_norm_eps)
|
||||
accepts_fp8_input = _linear_accepts_fp8_tuple(self.qkv_proj)
|
||||
self.attn_stage, self.ffn_stage = make_stages(
|
||||
self.attn_boundary, self.ffn_boundary = make_stages(
|
||||
(
|
||||
declare_attn(
|
||||
read=NormQuantRead(
|
||||
@@ -654,7 +654,7 @@ class InternS2MobiusAttentionDecoderLayer(
|
||||
**kwargs,
|
||||
):
|
||||
del kwargs
|
||||
hidden_states = self.attn_stage.prepare(
|
||||
hidden_states = self.attn_boundary.prepare(
|
||||
hidden_states,
|
||||
forward_batch,
|
||||
captured_last_layer_outputs=captured_last_layer_outputs,
|
||||
|
||||
@@ -25,7 +25,7 @@ from torch import nn
|
||||
from transformers import PretrainedConfig
|
||||
|
||||
from sglang.srt.distributed.device_communicators import triton_symm_mem_ag
|
||||
from sglang.srt.layers.communicator import AttentionInputs, get_attn_tp_context
|
||||
from sglang.srt.layers.layer_boundary import AttentionInputs, get_attn_tp_context
|
||||
from sglang.srt.layers.layernorm import RMSNorm
|
||||
from sglang.srt.layers.linear import ColumnParallelLinear, ReplicatedLinear
|
||||
from sglang.srt.layers.logits_processor import LogitsProcessor
|
||||
|
||||
@@ -2615,7 +2615,7 @@ class KimiK3DecoderLayer(nn.Module):
|
||||
) -> torch.Tensor:
|
||||
# For MLA layers with q_lora_rank, set up attn_inputs before the
|
||||
# forward call (normally done by the attention boundary).
|
||||
from sglang.srt.layers.communicator import (
|
||||
from sglang.srt.layers.layer_boundary import (
|
||||
AttentionInputs,
|
||||
get_attn_tp_context,
|
||||
)
|
||||
|
||||
@@ -20,15 +20,15 @@ from sglang.srt.configs.laguna import LagunaConfig, normalize_gating
|
||||
from sglang.srt.environ import envs
|
||||
from sglang.srt.layers.activation import SiluAndMul
|
||||
from sglang.srt.layers.aux_hidden_states import AuxHiddenStateList
|
||||
from sglang.srt.layers.communicator import (
|
||||
from sglang.srt.layers.dp_attention import (
|
||||
is_dp_attention_enabled,
|
||||
)
|
||||
from sglang.srt.layers.layer_boundary import (
|
||||
declare_attn,
|
||||
declare_ffn,
|
||||
make_stages,
|
||||
)
|
||||
from sglang.srt.layers.communicator.residual import batch as residual_batch
|
||||
from sglang.srt.layers.dp_attention import (
|
||||
is_dp_attention_enabled,
|
||||
)
|
||||
from sglang.srt.layers.layer_boundary.residual import batch as residual_batch
|
||||
from sglang.srt.layers.layernorm import RMSNorm
|
||||
from sglang.srt.layers.linear import (
|
||||
ColumnParallelLinear,
|
||||
@@ -457,7 +457,7 @@ class LagunaDecoderLayer(nn.Module):
|
||||
config.hidden_size, eps=config.rms_norm_eps
|
||||
)
|
||||
|
||||
self.attn_stage, self.ffn_stage = make_stages(
|
||||
self.attn_boundary, self.ffn_boundary = make_stages(
|
||||
(declare_attn(), self.input_layernorm),
|
||||
(
|
||||
declare_ffn(
|
||||
@@ -481,7 +481,7 @@ class LagunaDecoderLayer(nn.Module):
|
||||
forward_batch: ForwardBatch,
|
||||
capture_output=None,
|
||||
) -> torch.Tensor:
|
||||
hidden_states = self.attn_stage.prepare(
|
||||
hidden_states = self.attn_boundary.prepare(
|
||||
hidden_states, forward_batch, capture_output=capture_output
|
||||
)
|
||||
if hidden_states.shape[0] != 0:
|
||||
@@ -490,10 +490,10 @@ class LagunaDecoderLayer(nn.Module):
|
||||
hidden_states=hidden_states,
|
||||
forward_batch=forward_batch,
|
||||
)
|
||||
hidden_states = self.attn_stage.finish(hidden_states, forward_batch)
|
||||
hidden_states = self.ffn_stage.prepare(hidden_states, forward_batch)
|
||||
hidden_states = self.attn_boundary.finish(hidden_states, forward_batch)
|
||||
hidden_states = self.ffn_boundary.prepare(hidden_states, forward_batch)
|
||||
|
||||
with self.ffn_stage.exit(forward_batch) as ffn_exit:
|
||||
with self.ffn_boundary.exit(forward_batch) as ffn_exit:
|
||||
hidden_states = self.mlp(
|
||||
hidden_states,
|
||||
forward_batch=forward_batch,
|
||||
@@ -589,7 +589,7 @@ class LagunaModel(nn.Module):
|
||||
residual_batch.start(forward_batch)
|
||||
else:
|
||||
assert pp_proxy_tensors is not None
|
||||
hidden_states = self.layers[self.start_layer].attn_stage.from_pp(
|
||||
hidden_states = self.layers[self.start_layer].attn_boundary.from_pp(
|
||||
pp_proxy_tensors, forward_batch
|
||||
)
|
||||
|
||||
|
||||
@@ -31,16 +31,16 @@ from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_r
|
||||
from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation
|
||||
from sglang.srt.eplb.expert_location_dispatch import ExpertLocationDispatchInfo
|
||||
from sglang.srt.layers.activation import SiluAndMul
|
||||
from sglang.srt.layers.communicator import (
|
||||
from sglang.srt.layers.dp_attention import (
|
||||
is_dp_attention_enabled,
|
||||
)
|
||||
from sglang.srt.layers.layer_boundary import (
|
||||
declare_attn,
|
||||
declare_ffn,
|
||||
enable_moe_dense_fully_dp,
|
||||
make_stages,
|
||||
)
|
||||
from sglang.srt.layers.communicator.residual import batch as residual_batch
|
||||
from sglang.srt.layers.dp_attention import (
|
||||
is_dp_attention_enabled,
|
||||
)
|
||||
from sglang.srt.layers.layer_boundary.residual import batch as residual_batch
|
||||
from sglang.srt.layers.layernorm import RMSNorm
|
||||
from sglang.srt.layers.linear import (
|
||||
MergedColumnParallelLinear,
|
||||
@@ -638,7 +638,7 @@ class LLaDA2MoeBlock(nn.Module):
|
||||
|
||||
self.post_attention_layernorm = RMSNorm(hidden_size, eps=config.rms_norm_eps)
|
||||
|
||||
self.attn_stage, self.ffn_stage = make_stages(
|
||||
self.attn_boundary, self.ffn_boundary = make_stages(
|
||||
(declare_attn(), self.input_layernorm),
|
||||
(
|
||||
declare_ffn(
|
||||
@@ -666,7 +666,7 @@ class LLaDA2MoeBlock(nn.Module):
|
||||
hidden_states: torch.Tensor,
|
||||
forward_batch: ForwardBatch,
|
||||
) -> torch.Tensor:
|
||||
hidden_states = self.attn_stage.prepare(
|
||||
hidden_states = self.attn_boundary.prepare(
|
||||
hidden_states=hidden_states, forward_batch=forward_batch
|
||||
)
|
||||
|
||||
@@ -676,13 +676,13 @@ class LLaDA2MoeBlock(nn.Module):
|
||||
forward_batch=forward_batch,
|
||||
)
|
||||
|
||||
hidden_states = self.attn_stage.finish(hidden_states, forward_batch)
|
||||
hidden_states = self.ffn_stage.prepare(
|
||||
hidden_states = self.attn_boundary.finish(hidden_states, forward_batch)
|
||||
hidden_states = self.ffn_boundary.prepare(
|
||||
hidden_states=hidden_states,
|
||||
forward_batch=forward_batch,
|
||||
)
|
||||
|
||||
with self.ffn_stage.exit(forward_batch) as ffn_exit:
|
||||
with self.ffn_boundary.exit(forward_batch) as ffn_exit:
|
||||
hidden_states = self.mlp(hidden_states, forward_batch)
|
||||
hidden_states = ffn_exit.finish(hidden_states)
|
||||
|
||||
@@ -749,7 +749,7 @@ class LLaDA2MoeModel(nn.Module):
|
||||
residual_batch.start(forward_batch)
|
||||
else:
|
||||
assert pp_proxy_tensors is not None
|
||||
hidden_states = self.layers[self.start_layer].attn_stage.from_pp(
|
||||
hidden_states = self.layers[self.start_layer].attn_boundary.from_pp(
|
||||
pp_proxy_tensors, forward_batch
|
||||
)
|
||||
|
||||
|
||||
@@ -26,15 +26,15 @@ from torch import nn
|
||||
from transformers import Llama4TextConfig
|
||||
|
||||
from sglang.srt.layers.aux_hidden_states import AuxHiddenStateList
|
||||
from sglang.srt.layers.communicator import (
|
||||
from sglang.srt.layers.dp_attention import (
|
||||
is_dp_attention_enabled,
|
||||
)
|
||||
from sglang.srt.layers.layer_boundary import (
|
||||
declare_attn,
|
||||
declare_ffn,
|
||||
make_stages,
|
||||
)
|
||||
from sglang.srt.layers.communicator.residual import batch as residual_batch
|
||||
from sglang.srt.layers.dp_attention import (
|
||||
is_dp_attention_enabled,
|
||||
)
|
||||
from sglang.srt.layers.layer_boundary.residual import batch as residual_batch
|
||||
from sglang.srt.layers.layernorm import RMSNorm
|
||||
from sglang.srt.layers.linear import (
|
||||
QKVParallelLinear,
|
||||
@@ -426,7 +426,7 @@ class Llama4DecoderLayer(nn.Module):
|
||||
config.hidden_size, eps=config.rms_norm_eps
|
||||
)
|
||||
|
||||
self.attn_stage, self.ffn_stage = make_stages(
|
||||
self.attn_boundary, self.ffn_boundary = make_stages(
|
||||
(declare_attn(), self.input_layernorm),
|
||||
(
|
||||
declare_ffn(
|
||||
@@ -459,7 +459,7 @@ class Llama4DecoderLayer(nn.Module):
|
||||
forward_batch: ForwardBatch,
|
||||
capture_output=None,
|
||||
) -> torch.Tensor:
|
||||
hidden_states = self.attn_stage.prepare(
|
||||
hidden_states = self.attn_boundary.prepare(
|
||||
hidden_states, forward_batch, capture_output=capture_output
|
||||
)
|
||||
|
||||
@@ -470,11 +470,11 @@ class Llama4DecoderLayer(nn.Module):
|
||||
forward_batch=forward_batch,
|
||||
)
|
||||
|
||||
hidden_states = self.attn_stage.finish(hidden_states, forward_batch)
|
||||
hidden_states = self.ffn_stage.prepare(hidden_states, forward_batch)
|
||||
hidden_states = self.attn_boundary.finish(hidden_states, forward_batch)
|
||||
hidden_states = self.ffn_boundary.prepare(hidden_states, forward_batch)
|
||||
|
||||
# Fully Connected
|
||||
with self.ffn_stage.exit(forward_batch) as ffn_exit:
|
||||
with self.ffn_boundary.exit(forward_batch) as ffn_exit:
|
||||
hidden_states = self.feed_forward(hidden_states, forward_batch)
|
||||
hidden_states = ffn_exit.finish(hidden_states)
|
||||
|
||||
|
||||
@@ -51,15 +51,15 @@ from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_r
|
||||
from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation
|
||||
from sglang.srt.layers import deep_gemm_wrapper
|
||||
from sglang.srt.layers.activation import SiluAndMul
|
||||
from sglang.srt.layers.communicator import (
|
||||
from sglang.srt.layers.dp_attention import (
|
||||
is_dp_attention_enabled,
|
||||
)
|
||||
from sglang.srt.layers.layer_boundary import (
|
||||
declare_attn,
|
||||
declare_ffn,
|
||||
make_stages,
|
||||
)
|
||||
from sglang.srt.layers.communicator.residual import batch as residual_batch
|
||||
from sglang.srt.layers.dp_attention import (
|
||||
is_dp_attention_enabled,
|
||||
)
|
||||
from sglang.srt.layers.layer_boundary.residual import batch as residual_batch
|
||||
from sglang.srt.layers.layernorm import RMSNorm
|
||||
from sglang.srt.layers.linear import (
|
||||
MergedColumnParallelLinear,
|
||||
@@ -431,7 +431,7 @@ class LongcatFlashDecoderLayer(nn.Module):
|
||||
self.attn_tp_size = get_parallel().attn_tp_size
|
||||
self.attn_tp_rank = get_parallel().attn_tp_rank
|
||||
|
||||
self.attn_stage, self.moe_stage = make_stages(
|
||||
self.attn_boundary, self.moe_boundary = make_stages(
|
||||
(
|
||||
declare_attn(),
|
||||
self.input_layernorm[0],
|
||||
@@ -446,21 +446,23 @@ class LongcatFlashDecoderLayer(nn.Module):
|
||||
else None,
|
||||
terminal=self.layer_id == config.num_hidden_layers - 1,
|
||||
)
|
||||
self.first_ffn, self.second_attn, self.second_ffn = make_stages(
|
||||
(
|
||||
declare_ffn(),
|
||||
self.post_attention_layernorm[0],
|
||||
),
|
||||
(
|
||||
declare_attn(),
|
||||
self.input_layernorm[1],
|
||||
{"qkv_latent_func": self.self_attn[1].prepare_qkv_latent},
|
||||
),
|
||||
(
|
||||
declare_ffn(),
|
||||
self.post_attention_layernorm[1],
|
||||
),
|
||||
prepared_from=self.moe_stage.declaration,
|
||||
self.first_ffn_boundary, self.second_attn_boundary, self.second_ffn_boundary = (
|
||||
make_stages(
|
||||
(
|
||||
declare_ffn(),
|
||||
self.post_attention_layernorm[0],
|
||||
),
|
||||
(
|
||||
declare_attn(),
|
||||
self.input_layernorm[1],
|
||||
{"qkv_latent_func": self.self_attn[1].prepare_qkv_latent},
|
||||
),
|
||||
(
|
||||
declare_ffn(),
|
||||
self.post_attention_layernorm[1],
|
||||
),
|
||||
prepared_from=self.moe_boundary.declaration,
|
||||
)
|
||||
)
|
||||
|
||||
def forward(
|
||||
@@ -472,7 +474,7 @@ class LongcatFlashDecoderLayer(nn.Module):
|
||||
prev_topk_indices: Optional[torch.Tensor],
|
||||
) -> torch.Tensor:
|
||||
# first_attn
|
||||
hidden_states = self.attn_stage.prepare(hidden_states, forward_batch)
|
||||
hidden_states = self.attn_boundary.prepare(hidden_states, forward_batch)
|
||||
if hidden_states.shape[0] != 0:
|
||||
attn_out = self.self_attn[0](
|
||||
positions=positions,
|
||||
@@ -487,9 +489,9 @@ class LongcatFlashDecoderLayer(nn.Module):
|
||||
hidden_states = attn_out
|
||||
|
||||
# moe
|
||||
hidden_states = self.attn_stage.finish(hidden_states, forward_batch)
|
||||
hidden_states = self.moe_stage.prepare(hidden_states, forward_batch)
|
||||
moe_hidden_states = self.moe_stage.branch_output(
|
||||
hidden_states = self.attn_boundary.finish(hidden_states, forward_batch)
|
||||
hidden_states = self.moe_boundary.prepare(hidden_states, forward_batch)
|
||||
moe_hidden_states = self.moe_boundary.branch_output(
|
||||
self.mlp(hidden_states.clone()), forward_batch
|
||||
)
|
||||
|
||||
@@ -501,10 +503,10 @@ class LongcatFlashDecoderLayer(nn.Module):
|
||||
prev_topk_indices,
|
||||
)
|
||||
|
||||
hidden_states = self.moe_stage.merge_branch(
|
||||
hidden_states = self.moe_boundary.merge_branch(
|
||||
moe_hidden_states,
|
||||
hidden_states,
|
||||
self.second_ffn,
|
||||
self.second_ffn_boundary,
|
||||
forward_batch,
|
||||
)
|
||||
return hidden_states, prev_topk_indices
|
||||
@@ -518,15 +520,15 @@ class LongcatFlashDecoderLayer(nn.Module):
|
||||
prev_topk_indices,
|
||||
):
|
||||
# first_mlp, on the input the MoE's boundary read
|
||||
hidden_states = self.first_ffn.branch_input(
|
||||
self.moe_stage, hidden_states, forward_batch
|
||||
hidden_states = self.first_ffn_boundary.branch_input(
|
||||
self.moe_boundary, hidden_states, forward_batch
|
||||
)
|
||||
with self.first_ffn.exit(forward_batch) as ffn_exit:
|
||||
with self.first_ffn_boundary.exit(forward_batch) as ffn_exit:
|
||||
hidden_states = self.mlps[0](hidden_states)
|
||||
hidden_states = ffn_exit.finish(hidden_states)
|
||||
|
||||
# second_attn
|
||||
hidden_states = self.second_attn.prepare(hidden_states, forward_batch)
|
||||
# second_attn_boundary
|
||||
hidden_states = self.second_attn_boundary.prepare(hidden_states, forward_batch)
|
||||
if hidden_states.shape[0] != 0:
|
||||
attn_out = self.self_attn[1](
|
||||
positions=positions,
|
||||
@@ -541,9 +543,9 @@ class LongcatFlashDecoderLayer(nn.Module):
|
||||
hidden_states = attn_out
|
||||
|
||||
# second_mlp
|
||||
hidden_states = self.second_attn.finish(hidden_states, forward_batch)
|
||||
hidden_states = self.second_ffn.prepare(hidden_states, forward_batch)
|
||||
with self.second_ffn.exit(forward_batch) as ffn_exit:
|
||||
hidden_states = self.second_attn_boundary.finish(hidden_states, forward_batch)
|
||||
hidden_states = self.second_ffn_boundary.prepare(hidden_states, forward_batch)
|
||||
with self.second_ffn_boundary.exit(forward_batch) as ffn_exit:
|
||||
hidden_states = self.mlps[1](hidden_states)
|
||||
hidden_states = ffn_exit.finish(hidden_states)
|
||||
|
||||
@@ -628,7 +630,7 @@ class LongcatFlashModel(nn.Module):
|
||||
for i in range(total_num_layers):
|
||||
if i in self.layers_to_capture:
|
||||
aux_hidden_states.append(
|
||||
self.layers[i].attn_stage.snapshot(hidden_states, forward_batch)
|
||||
self.layers[i].attn_boundary.snapshot(hidden_states, forward_batch)
|
||||
)
|
||||
with get_global_expert_distribution_recorder().with_current_layer(i):
|
||||
layer = self.layers[i]
|
||||
|
||||
@@ -41,15 +41,15 @@ from sglang.kernels.ops.quantization.fp8_kernel import is_fp8_fnuz
|
||||
from sglang.srt.configs import LongcatFlashConfig
|
||||
from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder
|
||||
from sglang.srt.layers import deep_gemm_wrapper
|
||||
from sglang.srt.layers.communicator import (
|
||||
from sglang.srt.layers.dp_attention import (
|
||||
is_dp_attention_enabled,
|
||||
)
|
||||
from sglang.srt.layers.layer_boundary import (
|
||||
declare_attn,
|
||||
declare_ffn,
|
||||
make_stages,
|
||||
)
|
||||
from sglang.srt.layers.communicator.residual import batch as residual_batch
|
||||
from sglang.srt.layers.dp_attention import (
|
||||
is_dp_attention_enabled,
|
||||
)
|
||||
from sglang.srt.layers.layer_boundary.residual import batch as residual_batch
|
||||
from sglang.srt.layers.layernorm import RMSNorm
|
||||
from sglang.srt.layers.linear import ReplicatedLinear
|
||||
from sglang.srt.layers.logits_processor import LogitsProcessor
|
||||
@@ -165,7 +165,7 @@ class LongcatFlashDenseDecoderLayer(nn.Module):
|
||||
self.attn_tp_size = get_parallel().attn_tp_size
|
||||
self.attn_tp_rank = get_parallel().attn_tp_rank
|
||||
|
||||
self.attn_stage, self.ffn_stage = make_stages(
|
||||
self.attn_boundary, self.ffn_boundary = make_stages(
|
||||
(declare_attn(), self.input_layernorm),
|
||||
(
|
||||
declare_ffn(sparse=False, next_sparse=False),
|
||||
@@ -183,7 +183,7 @@ class LongcatFlashDenseDecoderLayer(nn.Module):
|
||||
zero_allocator: BumpAllocator,
|
||||
) -> torch.Tensor:
|
||||
|
||||
hidden_states = self.attn_stage.prepare(hidden_states, forward_batch)
|
||||
hidden_states = self.attn_boundary.prepare(hidden_states, forward_batch)
|
||||
if hidden_states.shape[0] != 0:
|
||||
hidden_states = self.self_attn(
|
||||
positions=positions,
|
||||
@@ -192,10 +192,10 @@ class LongcatFlashDenseDecoderLayer(nn.Module):
|
||||
zero_allocator=zero_allocator,
|
||||
)
|
||||
|
||||
hidden_states = self.attn_stage.finish(hidden_states, forward_batch)
|
||||
hidden_states = self.ffn_stage.prepare(hidden_states, forward_batch)
|
||||
hidden_states = self.attn_boundary.finish(hidden_states, forward_batch)
|
||||
hidden_states = self.ffn_boundary.prepare(hidden_states, forward_batch)
|
||||
hidden_states = self.mlp(hidden_states)
|
||||
hidden_states = self.ffn_stage.postprocess(hidden_states, forward_batch)
|
||||
hidden_states = self.ffn_boundary.postprocess(hidden_states, forward_batch)
|
||||
return hidden_states
|
||||
|
||||
|
||||
|
||||
@@ -25,7 +25,7 @@ import torch
|
||||
from torch import nn
|
||||
from transformers import PretrainedConfig
|
||||
|
||||
from sglang.srt.layers.communicator import (
|
||||
from sglang.srt.layers.layer_boundary import (
|
||||
declare_attn,
|
||||
declare_ffn,
|
||||
make_stages,
|
||||
@@ -449,7 +449,7 @@ class MellumDecoderLayer(Qwen3MoeDecoderLayer):
|
||||
self.input_layernorm = RMSNorm(cfg.hidden_size, eps=rms_norm_eps)
|
||||
self.post_attention_layernorm = RMSNorm(cfg.hidden_size, eps=rms_norm_eps)
|
||||
|
||||
self.attn_stage, self.ffn_stage = make_stages(
|
||||
self.attn_boundary, self.ffn_boundary = make_stages(
|
||||
(declare_attn(), self.input_layernorm),
|
||||
(
|
||||
declare_ffn(
|
||||
|
||||
@@ -32,16 +32,16 @@ from sglang.srt.layers.aux_hidden_states import (
|
||||
AuxHiddenStateAccumulator,
|
||||
AuxHiddenStatePacker,
|
||||
)
|
||||
from sglang.srt.layers.communicator import (
|
||||
from sglang.srt.layers.dp_attention import (
|
||||
is_dp_attention_enabled,
|
||||
)
|
||||
from sglang.srt.layers.layer_boundary import (
|
||||
declare_attn,
|
||||
declare_ffn,
|
||||
enable_moe_dense_fully_dp,
|
||||
make_stages,
|
||||
)
|
||||
from sglang.srt.layers.communicator.residual import batch as residual_batch
|
||||
from sglang.srt.layers.dp_attention import (
|
||||
is_dp_attention_enabled,
|
||||
)
|
||||
from sglang.srt.layers.layer_boundary.residual import batch as residual_batch
|
||||
from sglang.srt.layers.layernorm import RMSNorm
|
||||
from sglang.srt.layers.linear import (
|
||||
MergedColumnParallelLinear,
|
||||
@@ -858,7 +858,7 @@ class MiMoV2DecoderLayer(nn.Module):
|
||||
config.hidden_size, eps=config.layernorm_epsilon
|
||||
)
|
||||
|
||||
self.attn_stage, self.ffn_stage = make_stages(
|
||||
self.attn_boundary, self.ffn_boundary = make_stages(
|
||||
(declare_attn(), self.input_layernorm),
|
||||
(
|
||||
declare_ffn(
|
||||
@@ -883,7 +883,7 @@ class MiMoV2DecoderLayer(nn.Module):
|
||||
captured_last_layer_outputs: Optional[AuxHiddenStateAccumulator] = None,
|
||||
) -> torch.Tensor:
|
||||
# Self Attention
|
||||
hidden_states = self.attn_stage.prepare(
|
||||
hidden_states = self.attn_boundary.prepare(
|
||||
hidden_states,
|
||||
forward_batch,
|
||||
captured_last_layer_outputs=captured_last_layer_outputs,
|
||||
@@ -896,10 +896,10 @@ class MiMoV2DecoderLayer(nn.Module):
|
||||
forward_batch=forward_batch,
|
||||
)
|
||||
|
||||
hidden_states = self.attn_stage.finish(hidden_states, forward_batch)
|
||||
hidden_states = self.ffn_stage.prepare(hidden_states, forward_batch)
|
||||
hidden_states = self.attn_boundary.finish(hidden_states, forward_batch)
|
||||
hidden_states = self.ffn_boundary.prepare(hidden_states, forward_batch)
|
||||
|
||||
with self.ffn_stage.exit(forward_batch) as ffn_exit:
|
||||
with self.ffn_boundary.exit(forward_batch) as ffn_exit:
|
||||
hidden_states = self.mlp(hidden_states, forward_batch)
|
||||
hidden_states = ffn_exit.finish(hidden_states)
|
||||
|
||||
@@ -924,7 +924,7 @@ class MiMoV2DecoderLayer(nn.Module):
|
||||
forward_batch: ForwardBatch,
|
||||
tbo_subbatch_index: Optional[int] = None,
|
||||
):
|
||||
state.hidden_states_after_comm_pre_attn = self.attn_stage.prepare(
|
||||
state.hidden_states_after_comm_pre_attn = self.attn_boundary.prepare(
|
||||
hidden_states, forward_batch
|
||||
)
|
||||
state.update(
|
||||
@@ -936,15 +936,15 @@ class MiMoV2DecoderLayer(nn.Module):
|
||||
)
|
||||
|
||||
def op_comm_prepare_mlp(self, state):
|
||||
hidden_states = self.attn_stage.finish(
|
||||
hidden_states = self.attn_boundary.finish(
|
||||
state.pop("hidden_states_after_attn"), state.forward_batch
|
||||
)
|
||||
state.hidden_states_mlp_input = self.ffn_stage.prepare(
|
||||
state.hidden_states_mlp_input = self.ffn_boundary.prepare(
|
||||
hidden_states, state.forward_batch
|
||||
)
|
||||
|
||||
def op_comm_postprocess_layer(self, state):
|
||||
hidden_states = self.ffn_stage.postprocess(
|
||||
hidden_states = self.ffn_boundary.postprocess(
|
||||
state.pop("hidden_states_mlp_output"), state.forward_batch
|
||||
)
|
||||
|
||||
@@ -1035,7 +1035,7 @@ class MiMoV2Model(nn.Module):
|
||||
residual_batch.start(forward_batch)
|
||||
else:
|
||||
assert pp_proxy_tensors is not None
|
||||
hidden_states = self.layers[self.start_layer].attn_stage.from_pp(
|
||||
hidden_states = self.layers[self.start_layer].attn_boundary.from_pp(
|
||||
pp_proxy_tensors, forward_batch
|
||||
)
|
||||
|
||||
|
||||
@@ -21,16 +21,16 @@ from transformers import PretrainedConfig
|
||||
|
||||
from sglang.srt.configs.model_config import get_mimo_v2_fused_qkv_expected_tp_size
|
||||
from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder
|
||||
from sglang.srt.layers.communicator import (
|
||||
from sglang.srt.layers.dp_attention import (
|
||||
is_dp_attention_enabled,
|
||||
)
|
||||
from sglang.srt.layers.layer_boundary import (
|
||||
declare_attn,
|
||||
declare_ffn,
|
||||
enable_moe_dense_fully_dp,
|
||||
make_stages,
|
||||
)
|
||||
from sglang.srt.layers.communicator.residual import batch as residual_batch
|
||||
from sglang.srt.layers.dp_attention import (
|
||||
is_dp_attention_enabled,
|
||||
)
|
||||
from sglang.srt.layers.layer_boundary.residual import batch as residual_batch
|
||||
from sglang.srt.layers.layernorm import RMSNorm
|
||||
from sglang.srt.layers.logits_processor import LogitsProcessor
|
||||
from sglang.srt.layers.quantization.base_config import QuantizationConfig
|
||||
@@ -119,7 +119,7 @@ class MiMoV2MTPLayer(nn.Module):
|
||||
config.hidden_size, eps=config.layernorm_epsilon
|
||||
)
|
||||
|
||||
self.attn_stage, self.ffn_stage = make_stages(
|
||||
self.attn_boundary, self.ffn_boundary = make_stages(
|
||||
(declare_attn(), self.input_layernorm),
|
||||
(
|
||||
declare_ffn(
|
||||
@@ -143,7 +143,7 @@ class MiMoV2MTPLayer(nn.Module):
|
||||
forward_batch: ForwardBatch,
|
||||
) -> torch.Tensor:
|
||||
|
||||
hidden_states = self.attn_stage.prepare(hidden_states, forward_batch)
|
||||
hidden_states = self.attn_boundary.prepare(hidden_states, forward_batch)
|
||||
|
||||
if hidden_states.shape[0] != 0:
|
||||
hidden_states = self.self_attn(
|
||||
@@ -152,10 +152,10 @@ class MiMoV2MTPLayer(nn.Module):
|
||||
forward_batch=forward_batch,
|
||||
)
|
||||
|
||||
hidden_states = self.attn_stage.finish(hidden_states, forward_batch)
|
||||
hidden_states = self.ffn_stage.prepare(hidden_states, forward_batch)
|
||||
hidden_states = self.attn_boundary.finish(hidden_states, forward_batch)
|
||||
hidden_states = self.ffn_boundary.prepare(hidden_states, forward_batch)
|
||||
with (
|
||||
self.ffn_stage.exit(forward_batch) as ffn_exit,
|
||||
self.ffn_boundary.exit(forward_batch) as ffn_exit,
|
||||
get_global_expert_distribution_recorder().disable_this_region(),
|
||||
):
|
||||
hidden_states = self.mlp(hidden_states)
|
||||
|
||||
@@ -34,16 +34,16 @@ from sglang.srt.layers.aux_hidden_states import (
|
||||
AuxHiddenStateAccumulator,
|
||||
AuxHiddenStateList,
|
||||
)
|
||||
from sglang.srt.layers.communicator import (
|
||||
declare_attn,
|
||||
declare_ffn,
|
||||
make_stages,
|
||||
)
|
||||
from sglang.srt.layers.communicator.residual import batch as residual_batch
|
||||
from sglang.srt.layers.dp_attention import (
|
||||
attn_tp_all_reduce,
|
||||
is_dp_attention_enabled,
|
||||
)
|
||||
from sglang.srt.layers.layer_boundary import (
|
||||
declare_attn,
|
||||
declare_ffn,
|
||||
make_stages,
|
||||
)
|
||||
from sglang.srt.layers.layer_boundary.residual import batch as residual_batch
|
||||
from sglang.srt.layers.layernorm import RMSNorm
|
||||
from sglang.srt.layers.linear import (
|
||||
QKVParallelLinear,
|
||||
@@ -982,7 +982,7 @@ class MiniMaxM2DecoderLayer(nn.Module):
|
||||
is_previous_layer_sparse = True
|
||||
is_next_layer_sparse = True
|
||||
|
||||
self.attn_stage, self.ffn_stage = make_stages(
|
||||
self.attn_boundary, self.ffn_boundary = make_stages(
|
||||
(declare_attn(), self.input_layernorm),
|
||||
(
|
||||
declare_ffn(
|
||||
@@ -1007,7 +1007,7 @@ class MiniMaxM2DecoderLayer(nn.Module):
|
||||
captured_last_layer_outputs: Optional[AuxHiddenStateAccumulator] = None,
|
||||
) -> torch.Tensor:
|
||||
# Self Attention
|
||||
hidden_states = self.attn_stage.prepare(
|
||||
hidden_states = self.attn_boundary.prepare(
|
||||
hidden_states,
|
||||
forward_batch,
|
||||
captured_last_layer_outputs=captured_last_layer_outputs,
|
||||
@@ -1021,10 +1021,10 @@ class MiniMaxM2DecoderLayer(nn.Module):
|
||||
|
||||
# Fully Connected (MLP or MoE)
|
||||
|
||||
hidden_states = self.attn_stage.finish(hidden_states, forward_batch)
|
||||
hidden_states = self.ffn_stage.prepare(hidden_states, forward_batch)
|
||||
hidden_states = self.attn_boundary.finish(hidden_states, forward_batch)
|
||||
hidden_states = self.ffn_boundary.prepare(hidden_states, forward_batch)
|
||||
|
||||
with self.ffn_stage.exit(forward_batch) as ffn_exit:
|
||||
with self.ffn_boundary.exit(forward_batch) as ffn_exit:
|
||||
hidden_states = self.block_sparse_moe(hidden_states, forward_batch)
|
||||
hidden_states = ffn_exit.finish(hidden_states)
|
||||
|
||||
@@ -1041,7 +1041,7 @@ class MiniMaxM2DecoderLayer(nn.Module):
|
||||
tbo_subbatch_index: Optional[int] = None,
|
||||
):
|
||||
"""Communication prepare for attention - TBO operation"""
|
||||
state.hidden_states_after_comm_pre_attn = self.attn_stage.prepare(
|
||||
state.hidden_states_after_comm_pre_attn = self.attn_boundary.prepare(
|
||||
hidden_states, forward_batch
|
||||
)
|
||||
state.update(
|
||||
@@ -1055,16 +1055,16 @@ class MiniMaxM2DecoderLayer(nn.Module):
|
||||
|
||||
def op_comm_prepare_mlp(self, state):
|
||||
"""Communication prepare for MLP - TBO operation"""
|
||||
hidden_states = self.attn_stage.finish(
|
||||
hidden_states = self.attn_boundary.finish(
|
||||
state.pop("hidden_states_after_attn"), state.forward_batch
|
||||
)
|
||||
state.hidden_states_mlp_input = self.ffn_stage.prepare(
|
||||
state.hidden_states_mlp_input = self.ffn_boundary.prepare(
|
||||
hidden_states, state.forward_batch
|
||||
)
|
||||
|
||||
def op_comm_postprocess_layer(self, state):
|
||||
"""Communication postprocess for layer - TBO operation"""
|
||||
hidden_states = self.ffn_stage.postprocess(
|
||||
hidden_states = self.ffn_boundary.postprocess(
|
||||
state.pop("hidden_states_mlp_output"), state.forward_batch
|
||||
)
|
||||
|
||||
@@ -1143,7 +1143,7 @@ class MiniMaxM2Model(nn.Module):
|
||||
residual_batch.start(forward_batch)
|
||||
else:
|
||||
assert pp_proxy_tensors is not None
|
||||
hidden_states = self.layers[self.start_layer].attn_stage.from_pp(
|
||||
hidden_states = self.layers[self.start_layer].attn_boundary.from_pp(
|
||||
pp_proxy_tensors, forward_batch
|
||||
)
|
||||
|
||||
|
||||
@@ -39,16 +39,16 @@ from sglang.srt.layers.aux_hidden_states import (
|
||||
AuxHiddenStateAccumulator,
|
||||
AuxHiddenStateList,
|
||||
)
|
||||
from sglang.srt.layers.communicator import (
|
||||
from sglang.srt.layers.dp_attention import (
|
||||
is_dp_attention_enabled,
|
||||
)
|
||||
from sglang.srt.layers.layer_boundary import (
|
||||
declare_attn,
|
||||
declare_ffn,
|
||||
enable_moe_dense_fully_dp,
|
||||
make_stages,
|
||||
)
|
||||
from sglang.srt.layers.communicator.residual import batch as residual_batch
|
||||
from sglang.srt.layers.dp_attention import (
|
||||
is_dp_attention_enabled,
|
||||
)
|
||||
from sglang.srt.layers.layer_boundary.residual import batch as residual_batch
|
||||
from sglang.srt.layers.layernorm import GemmaRMSNorm, RMSNorm
|
||||
from sglang.srt.layers.linear import (
|
||||
MergedColumnParallelLinear,
|
||||
@@ -1354,7 +1354,7 @@ class MiniMaxM3DecoderLayer(nn.Module):
|
||||
is_previous_layer_sparse = _is_layer_sparse(layer_id - 1)
|
||||
is_next_layer_sparse = _is_layer_sparse(layer_id + 1)
|
||||
|
||||
self.attn_stage, self.ffn_stage = make_stages(
|
||||
self.attn_boundary, self.ffn_boundary = make_stages(
|
||||
(declare_attn(), self.input_layernorm),
|
||||
(
|
||||
declare_ffn(
|
||||
@@ -1379,7 +1379,7 @@ class MiniMaxM3DecoderLayer(nn.Module):
|
||||
captured_last_layer_outputs: Optional[AuxHiddenStateAccumulator] = None,
|
||||
**kwargs,
|
||||
) -> torch.Tensor:
|
||||
hidden_states = self.attn_stage.prepare(
|
||||
hidden_states = self.attn_boundary.prepare(
|
||||
hidden_states,
|
||||
forward_batch,
|
||||
captured_last_layer_outputs=captured_last_layer_outputs,
|
||||
@@ -1393,10 +1393,10 @@ class MiniMaxM3DecoderLayer(nn.Module):
|
||||
forward_batch=forward_batch,
|
||||
)
|
||||
|
||||
hidden_states = self.attn_stage.finish(hidden_states, forward_batch)
|
||||
hidden_states = self.ffn_stage.prepare(hidden_states, forward_batch)
|
||||
hidden_states = self.attn_boundary.finish(hidden_states, forward_batch)
|
||||
hidden_states = self.ffn_boundary.prepare(hidden_states, forward_batch)
|
||||
|
||||
with self.ffn_stage.exit(forward_batch) as ffn_exit:
|
||||
with self.ffn_boundary.exit(forward_batch) as ffn_exit:
|
||||
if self.is_layer_sparse or hidden_states.shape[0] != 0:
|
||||
hidden_states = self.mlp(
|
||||
hidden_states,
|
||||
@@ -1483,7 +1483,7 @@ class MiniMaxM3Model(nn.Module):
|
||||
residual_batch.start(forward_batch)
|
||||
else:
|
||||
assert pp_proxy_tensors is not None
|
||||
hidden_states = self.layers[self.start_layer].attn_stage.from_pp(
|
||||
hidden_states = self.layers[self.start_layer].attn_boundary.from_pp(
|
||||
pp_proxy_tensors, forward_batch
|
||||
)
|
||||
|
||||
|
||||
@@ -22,13 +22,13 @@ from sglang.srt.layers.attention.vision import (
|
||||
VisionAttentionMetadata,
|
||||
prepare_vision_attention_metadata,
|
||||
)
|
||||
from sglang.srt.layers.communicator import (
|
||||
from sglang.srt.layers.conv import Conv3dLayer
|
||||
from sglang.srt.layers.layer_boundary import (
|
||||
declare_attn,
|
||||
declare_ffn,
|
||||
make_stages,
|
||||
)
|
||||
from sglang.srt.layers.communicator.residual import batch as residual_batch
|
||||
from sglang.srt.layers.conv import Conv3dLayer
|
||||
from sglang.srt.layers.layer_boundary.residual import batch as residual_batch
|
||||
from sglang.srt.layers.layernorm import RMSNorm
|
||||
from sglang.srt.layers.linear import (
|
||||
ColumnParallelLinear,
|
||||
@@ -1017,7 +1017,7 @@ class MossVLSelfAttentionDecoderLayer(nn.Module):
|
||||
config.hidden_size, eps=config.rms_norm_eps, **norm_kwargs
|
||||
)
|
||||
|
||||
self.attn_stage, self.ffn_stage = make_stages(
|
||||
self.attn_boundary, self.ffn_boundary = make_stages(
|
||||
(declare_attn(), self.input_layernorm),
|
||||
(
|
||||
declare_ffn(sparse=False, next_sparse=False),
|
||||
@@ -1034,7 +1034,7 @@ class MossVLSelfAttentionDecoderLayer(nn.Module):
|
||||
forward_batch: ForwardBatch,
|
||||
) -> torch.Tensor:
|
||||
# Self Attention
|
||||
hidden_states = self.attn_stage.prepare(hidden_states, forward_batch)
|
||||
hidden_states = self.attn_boundary.prepare(hidden_states, forward_batch)
|
||||
if hidden_states.shape[0] != 0:
|
||||
hidden_states = self.self_attn(
|
||||
positions=positions,
|
||||
@@ -1043,9 +1043,9 @@ class MossVLSelfAttentionDecoderLayer(nn.Module):
|
||||
)
|
||||
|
||||
# MLP
|
||||
hidden_states = self.attn_stage.finish(hidden_states, forward_batch)
|
||||
hidden_states = self.ffn_stage.prepare(hidden_states, forward_batch)
|
||||
with self.ffn_stage.exit(forward_batch) as ffn_exit:
|
||||
hidden_states = self.attn_boundary.finish(hidden_states, forward_batch)
|
||||
hidden_states = self.ffn_boundary.prepare(hidden_states, forward_batch)
|
||||
with self.ffn_boundary.exit(forward_batch) as ffn_exit:
|
||||
hidden_states = self.mlp(hidden_states)
|
||||
hidden_states = ffn_exit.finish(hidden_states)
|
||||
return hidden_states
|
||||
|
||||
@@ -31,10 +31,10 @@ from sglang.srt.layers.attention.hybrid_linear_attn_backend import (
|
||||
Mamba2AttnBackend,
|
||||
)
|
||||
from sglang.srt.layers.attention.mamba.mamba import MambaMixer2
|
||||
from sglang.srt.layers.communicator.residual import batch as residual_batch
|
||||
from sglang.srt.layers.dp_attention import (
|
||||
is_dp_attention_enabled,
|
||||
)
|
||||
from sglang.srt.layers.layer_boundary.residual import batch as residual_batch
|
||||
from sglang.srt.layers.layernorm import RMSNorm
|
||||
from sglang.srt.layers.linear import (
|
||||
ColumnParallelLinear,
|
||||
@@ -376,7 +376,7 @@ class NemotronHMLPLikeDecoderLayer(nn.Module):
|
||||
sum an FFN before it left."""
|
||||
|
||||
def _init_stage_boundary(self, config: NemotronHConfig, layer_idx: int):
|
||||
self.stage = make_stage_boundary(
|
||||
self.boundary = make_stage_boundary(
|
||||
self.norm, pattern=config.hybrid_override_pattern, layer_idx=layer_idx
|
||||
)
|
||||
|
||||
@@ -386,8 +386,8 @@ class NemotronHMLPLikeDecoderLayer(nn.Module):
|
||||
hidden_states,
|
||||
forward_batch: ForwardBatch,
|
||||
):
|
||||
hidden_states = self.stage.prepare(hidden_states, forward_batch)
|
||||
with self.stage.exit(forward_batch) as ffn_exit:
|
||||
hidden_states = self.boundary.prepare(hidden_states, forward_batch)
|
||||
with self.boundary.exit(forward_batch) as ffn_exit:
|
||||
hidden_states = self.mixer.forward(hidden_states)
|
||||
return ffn_exit.finish(hidden_states)
|
||||
|
||||
@@ -455,7 +455,7 @@ class NemotronHAttnLikeDecoderLayer(nn.Module):
|
||||
itself unless the fused kernel takes the sum into the next input norm."""
|
||||
|
||||
def _init_stage_boundary(self, config: NemotronHConfig, layer_idx: int):
|
||||
self.stage = make_stage_boundary(
|
||||
self.boundary = make_stage_boundary(
|
||||
self.norm, pattern=config.hybrid_override_pattern, layer_idx=layer_idx
|
||||
)
|
||||
|
||||
@@ -465,11 +465,11 @@ class NemotronHAttnLikeDecoderLayer(nn.Module):
|
||||
hidden_states,
|
||||
forward_batch: ForwardBatch,
|
||||
):
|
||||
hidden_states = self.stage.prepare(hidden_states, forward_batch)
|
||||
hidden_states = self.boundary.prepare(hidden_states, forward_batch)
|
||||
if forward_batch.forward_mode.is_idle():
|
||||
return hidden_states
|
||||
|
||||
with self.stage.exit(forward_batch) as mixer_exit:
|
||||
with self.boundary.exit(forward_batch) as mixer_exit:
|
||||
hidden_states = self._forward_mixer(
|
||||
hidden_states, forward_batch, mixer_exit.skips_reduction
|
||||
)
|
||||
@@ -730,14 +730,14 @@ class NemotronHModel(nn.Module):
|
||||
residual_batch.start(forward_batch)
|
||||
else:
|
||||
assert pp_proxy_tensors is not None
|
||||
hidden_states = self.layers[self.start_layer].stage.from_pp(
|
||||
hidden_states = self.layers[self.start_layer].boundary.from_pp(
|
||||
pp_proxy_tensors, forward_batch
|
||||
)
|
||||
|
||||
aux_hidden_states = []
|
||||
for i in range(self.start_layer, self.end_layer):
|
||||
if i in self.layers_to_capture:
|
||||
hidden_states, snapshot = self.layers[i].stage.capture_output(
|
||||
hidden_states, snapshot = self.layers[i].boundary.capture_output(
|
||||
hidden_states, forward_batch
|
||||
)
|
||||
aux_hidden_states.append(snapshot)
|
||||
|
||||
@@ -18,8 +18,8 @@ import torch
|
||||
from torch import nn
|
||||
|
||||
from sglang.srt.configs import NemotronHConfig
|
||||
from sglang.srt.layers.communicator.residual import batch as residual_batch
|
||||
from sglang.srt.layers.dp_attention import is_dp_attention_enabled
|
||||
from sglang.srt.layers.layer_boundary.residual import batch as residual_batch
|
||||
from sglang.srt.layers.layernorm import RMSNorm
|
||||
from sglang.srt.layers.linear import ColumnParallelLinear
|
||||
from sglang.srt.layers.logits_processor import LogitsProcessor
|
||||
|
||||
@@ -1,14 +1,14 @@
|
||||
"""Layer-communication helpers for the Nemotron-H model."""
|
||||
|
||||
from sglang.srt.configs.nemotron_h import ATTENTION, MAMBA, MOE
|
||||
from sglang.srt.layers.communicator import (
|
||||
from sglang.srt.layers.layer_boundary import (
|
||||
HandoffRows,
|
||||
ProducerReduction,
|
||||
declare_attn,
|
||||
declare_ffn,
|
||||
make_stages,
|
||||
)
|
||||
from sglang.srt.layers.communicator.residual.add_norm import NormQuantRead, NormRead
|
||||
from sglang.srt.layers.layer_boundary.residual.add_norm import NormQuantRead, NormRead
|
||||
from sglang.srt.layers.layernorm import RMSNorm
|
||||
from sglang.srt.runtime_context import get_parallel
|
||||
|
||||
|
||||
@@ -26,12 +26,12 @@ from sglang.srt.distributed import (
|
||||
get_pp_indices,
|
||||
)
|
||||
from sglang.srt.layers.activation import SiluAndMul
|
||||
from sglang.srt.layers.communicator.residual.access import (
|
||||
from sglang.srt.layers.dp_attention import is_dp_attention_enabled
|
||||
from sglang.srt.layers.layer_boundary.residual.access import (
|
||||
finish_layer_stack,
|
||||
from_pp,
|
||||
snapshot,
|
||||
)
|
||||
from sglang.srt.layers.dp_attention import is_dp_attention_enabled
|
||||
from sglang.srt.layers.layernorm import RMSNorm
|
||||
from sglang.srt.layers.linear import (
|
||||
MergedColumnParallelLinear,
|
||||
|
||||
@@ -44,16 +44,16 @@ from sglang.srt.layers.aux_hidden_states import (
|
||||
AuxHiddenStateAccumulator,
|
||||
AuxHiddenStateList,
|
||||
)
|
||||
from sglang.srt.layers.communicator import (
|
||||
declare_attn,
|
||||
declare_ffn,
|
||||
make_stages,
|
||||
)
|
||||
from sglang.srt.layers.communicator.residual import batch as residual_batch
|
||||
from sglang.srt.layers.dp_attention import (
|
||||
is_dp_attention_enabled,
|
||||
)
|
||||
from sglang.srt.layers.flashinfer_comm_fusion import uses_cutedsl_ar_fusion
|
||||
from sglang.srt.layers.layer_boundary import (
|
||||
declare_attn,
|
||||
declare_ffn,
|
||||
make_stages,
|
||||
)
|
||||
from sglang.srt.layers.layer_boundary.residual import batch as residual_batch
|
||||
from sglang.srt.layers.layernorm import RMSNorm
|
||||
from sglang.srt.layers.linear import (
|
||||
MergedColumnParallelLinear,
|
||||
@@ -865,7 +865,7 @@ class Qwen2MoeSparseMoeBlock(nn.Module):
|
||||
if defer_finalize:
|
||||
if shared_output is None:
|
||||
raise RuntimeError("Qwen deferred finalize requires shared output")
|
||||
from sglang.srt.layers.communicator.fusions.cutedsl import (
|
||||
from sglang.srt.layers.layer_boundary.fusions.cutedsl import (
|
||||
MoeFinalizeHandoff,
|
||||
)
|
||||
|
||||
@@ -1053,7 +1053,7 @@ class Qwen2MoeDecoderLayer(nn.Module):
|
||||
self.post_attention_layernorm = RMSNorm(
|
||||
config.hidden_size, eps=config.rms_norm_eps
|
||||
)
|
||||
self.attn_stage, self.ffn_stage = make_stages(
|
||||
self.attn_boundary, self.ffn_boundary = make_stages(
|
||||
(declare_attn(), self.input_layernorm),
|
||||
(
|
||||
declare_ffn(
|
||||
@@ -1079,7 +1079,7 @@ class Qwen2MoeDecoderLayer(nn.Module):
|
||||
**kwargs,
|
||||
) -> torch.Tensor:
|
||||
|
||||
hidden_states = self.attn_stage.prepare(
|
||||
hidden_states = self.attn_boundary.prepare(
|
||||
hidden_states,
|
||||
forward_batch,
|
||||
captured_last_layer_outputs=captured_last_layer_outputs,
|
||||
@@ -1093,10 +1093,10 @@ class Qwen2MoeDecoderLayer(nn.Module):
|
||||
forward_batch=forward_batch,
|
||||
)
|
||||
|
||||
hidden_states = self.attn_stage.finish(hidden_states, forward_batch)
|
||||
hidden_states = self.ffn_stage.prepare(hidden_states, forward_batch)
|
||||
hidden_states = self.attn_boundary.finish(hidden_states, forward_batch)
|
||||
hidden_states = self.ffn_boundary.prepare(hidden_states, forward_batch)
|
||||
|
||||
with self.ffn_stage.exit(forward_batch) as ffn_exit:
|
||||
with self.ffn_boundary.exit(forward_batch) as ffn_exit:
|
||||
hidden_states = self.mlp(hidden_states, forward_batch)
|
||||
hidden_states = ffn_exit.finish(hidden_states)
|
||||
|
||||
@@ -1180,7 +1180,7 @@ class Qwen2MoeModel(nn.Module):
|
||||
residual_batch.start(forward_batch)
|
||||
else:
|
||||
assert pp_proxy_tensors is not None
|
||||
hidden_states = self.layers[self.start_layer].attn_stage.from_pp(
|
||||
hidden_states = self.layers[self.start_layer].attn_boundary.from_pp(
|
||||
pp_proxy_tensors, forward_batch
|
||||
)
|
||||
|
||||
|
||||
@@ -6,12 +6,12 @@ import torch
|
||||
from torch import nn
|
||||
|
||||
from sglang.srt.layers.aux_hidden_states import AuxHiddenStateList
|
||||
from sglang.srt.layers.communicator import (
|
||||
from sglang.srt.layers.layer_boundary import (
|
||||
declare_attn,
|
||||
declare_ffn,
|
||||
make_stages,
|
||||
)
|
||||
from sglang.srt.layers.communicator.residual import batch as residual_batch
|
||||
from sglang.srt.layers.layer_boundary.residual import batch as residual_batch
|
||||
from sglang.srt.layers.layernorm import RMSNorm
|
||||
from sglang.srt.layers.linear import QKVParallelLinear, RowParallelLinear
|
||||
from sglang.srt.layers.logits_processor import LogitsProcessor
|
||||
@@ -376,7 +376,7 @@ class Qwen3DecoderLayer(nn.Module):
|
||||
config.hidden_size, eps=config.rms_norm_eps, **norm_kwargs
|
||||
)
|
||||
|
||||
self.attn_stage, self.ffn_stage = make_stages(
|
||||
self.attn_boundary, self.ffn_boundary = make_stages(
|
||||
(declare_attn(), self.input_layernorm),
|
||||
(
|
||||
declare_ffn(sparse=False, next_sparse=False),
|
||||
@@ -395,7 +395,7 @@ class Qwen3DecoderLayer(nn.Module):
|
||||
capture_output=None,
|
||||
) -> torch.Tensor:
|
||||
# Self Attention
|
||||
hidden_states = self.attn_stage.prepare(
|
||||
hidden_states = self.attn_boundary.prepare(
|
||||
hidden_states,
|
||||
forward_batch,
|
||||
post_residual_addition=post_residual_addition,
|
||||
@@ -409,8 +409,8 @@ class Qwen3DecoderLayer(nn.Module):
|
||||
)
|
||||
|
||||
# Fully Connected
|
||||
hidden_states = self.attn_stage.finish(hidden_states, forward_batch)
|
||||
hidden_states = self.ffn_stage.prepare(
|
||||
hidden_states = self.attn_boundary.finish(hidden_states, forward_batch)
|
||||
hidden_states = self.ffn_boundary.prepare(
|
||||
hidden_states,
|
||||
forward_batch,
|
||||
cache=(
|
||||
@@ -424,7 +424,7 @@ class Qwen3DecoderLayer(nn.Module):
|
||||
else None
|
||||
),
|
||||
)
|
||||
with self.ffn_stage.exit(forward_batch) as ffn_exit:
|
||||
with self.ffn_boundary.exit(forward_batch) as ffn_exit:
|
||||
hidden_states = self.mlp(hidden_states, forward_batch=forward_batch)
|
||||
if _is_npu and get_cmo_stream():
|
||||
wait_cmo_stream()
|
||||
@@ -462,7 +462,7 @@ class Qwen3Model(Qwen2Model):
|
||||
)
|
||||
residual_batch.start(forward_batch)
|
||||
else:
|
||||
hidden_states = self.layers[self.start_layer].attn_stage.from_pp(
|
||||
hidden_states = self.layers[self.start_layer].attn_boundary.from_pp(
|
||||
pp_proxy_tensors, forward_batch
|
||||
)
|
||||
aux_hidden_states = AuxHiddenStateList()
|
||||
|
||||
@@ -46,21 +46,21 @@ from sglang.srt.layers.aux_hidden_states import (
|
||||
AuxHiddenStateAccumulator,
|
||||
AuxHiddenStateList,
|
||||
)
|
||||
from sglang.srt.layers.communicator import (
|
||||
from sglang.srt.layers.dp_attention import (
|
||||
is_dp_attention_enabled,
|
||||
)
|
||||
from sglang.srt.layers.flashinfer_comm_fusion import uses_cutedsl_ar_fusion
|
||||
from sglang.srt.layers.layer_boundary import (
|
||||
declare_attn,
|
||||
declare_ffn,
|
||||
make_stages,
|
||||
)
|
||||
|
||||
# Layers - Attention
|
||||
from sglang.srt.layers.communicator.residual import batch as residual_batch
|
||||
from sglang.srt.layers.layer_boundary.residual import batch as residual_batch
|
||||
|
||||
# Layers - Others
|
||||
from sglang.srt.layers.communicator.residual.add_norm import Fp8Input, NormQuantRead
|
||||
from sglang.srt.layers.dp_attention import (
|
||||
is_dp_attention_enabled,
|
||||
)
|
||||
from sglang.srt.layers.flashinfer_comm_fusion import uses_cutedsl_ar_fusion
|
||||
from sglang.srt.layers.layer_boundary.residual.add_norm import Fp8Input, NormQuantRead
|
||||
from sglang.srt.layers.layernorm import GemmaRMSNorm
|
||||
|
||||
# Layers - Linear
|
||||
@@ -224,7 +224,7 @@ def _use_mnnvl_cutedsl_fusion(config: Qwen3_5TextConfig, is_nextn: bool) -> bool
|
||||
def _layer_fusions(config: Qwen3_5TextConfig, is_nextn: bool):
|
||||
"""The CuTe DSL kernels a layer tries before its own, when they are on."""
|
||||
if _use_mnnvl_cutedsl_fusion(config, is_nextn):
|
||||
from sglang.srt.layers.communicator.fusions.cutedsl import CuteDSLFusion
|
||||
from sglang.srt.layers.layer_boundary.fusions.cutedsl import CuteDSLFusion
|
||||
|
||||
return CuteDSLFusion()
|
||||
return None
|
||||
@@ -1094,7 +1094,7 @@ class Qwen3_5LinearDecoderLayer(nn.Module):
|
||||
# it. Otherwise, stay on the plain AR+RMSNorm path.
|
||||
accepts_fp8_input = _linear_accepts_fp8_tuple(self.linear_attn.in_proj_qkvz)
|
||||
boundary_fusions = _layer_fusions(config, is_nextn)
|
||||
self.attn_stage, self.ffn_stage = make_stages(
|
||||
self.attn_boundary, self.ffn_boundary = make_stages(
|
||||
(
|
||||
declare_attn(
|
||||
read=NormQuantRead(
|
||||
@@ -1129,7 +1129,7 @@ class Qwen3_5LinearDecoderLayer(nn.Module):
|
||||
):
|
||||
forward_batch = kwargs.get("forward_batch", None)
|
||||
|
||||
hidden_states = self.attn_stage.prepare(
|
||||
hidden_states = self.attn_boundary.prepare(
|
||||
hidden_states,
|
||||
forward_batch,
|
||||
captured_last_layer_outputs=kwargs.get("captured_last_layer_outputs", None),
|
||||
@@ -1144,10 +1144,10 @@ class Qwen3_5LinearDecoderLayer(nn.Module):
|
||||
)
|
||||
|
||||
# Fully Connected
|
||||
hidden_states = self.attn_stage.finish(hidden_states, forward_batch)
|
||||
hidden_states = self.ffn_stage.prepare(hidden_states, forward_batch)
|
||||
hidden_states = self.attn_boundary.finish(hidden_states, forward_batch)
|
||||
hidden_states = self.ffn_boundary.prepare(hidden_states, forward_batch)
|
||||
|
||||
with self.ffn_stage.exit(forward_batch) as ffn_exit:
|
||||
with self.ffn_boundary.exit(forward_batch) as ffn_exit:
|
||||
if isinstance(self.mlp, Qwen2MoeSparseMoeBlock):
|
||||
hidden_states = self.mlp(
|
||||
hidden_states,
|
||||
@@ -1300,7 +1300,7 @@ class Qwen3_5AttentionDecoderLayer(nn.Module):
|
||||
# when qkv_proj can consume the returned quantized tuple.
|
||||
accepts_fp8_input = _linear_accepts_fp8_tuple(self.qkv_proj)
|
||||
boundary_fusions = _layer_fusions(config, is_nextn)
|
||||
self.attn_stage, self.ffn_stage = make_stages(
|
||||
self.attn_boundary, self.ffn_boundary = make_stages(
|
||||
(
|
||||
declare_attn(
|
||||
read=NormQuantRead(
|
||||
@@ -1533,7 +1533,7 @@ class Qwen3_5AttentionDecoderLayer(nn.Module):
|
||||
captured_last_layer_outputs: Optional[AuxHiddenStateAccumulator] = None,
|
||||
**kwargs,
|
||||
):
|
||||
hidden_states = self.attn_stage.prepare(
|
||||
hidden_states = self.attn_boundary.prepare(
|
||||
hidden_states,
|
||||
forward_batch,
|
||||
captured_last_layer_outputs=captured_last_layer_outputs,
|
||||
@@ -1549,9 +1549,9 @@ class Qwen3_5AttentionDecoderLayer(nn.Module):
|
||||
)
|
||||
|
||||
# Fully Connected
|
||||
hidden_states = self.attn_stage.finish(hidden_states, forward_batch)
|
||||
hidden_states = self.ffn_stage.prepare(hidden_states, forward_batch)
|
||||
with self.ffn_stage.exit(forward_batch) as ffn_exit:
|
||||
hidden_states = self.attn_boundary.finish(hidden_states, forward_batch)
|
||||
hidden_states = self.ffn_boundary.prepare(hidden_states, forward_batch)
|
||||
with self.ffn_boundary.exit(forward_batch) as ffn_exit:
|
||||
if isinstance(self.mlp, Qwen2MoeSparseMoeBlock):
|
||||
hidden_states = self.mlp(
|
||||
hidden_states,
|
||||
@@ -1716,7 +1716,7 @@ class Qwen3_5ForCausalLM(nn.Module):
|
||||
"layers: "
|
||||
f"{unsupported_layers}"
|
||||
)
|
||||
from sglang.srt.layers.communicator.fusions.cutedsl import (
|
||||
from sglang.srt.layers.layer_boundary.fusions.cutedsl import (
|
||||
install_cutedsl_fusion,
|
||||
)
|
||||
|
||||
@@ -1807,7 +1807,7 @@ class Qwen3_5ForCausalLM(nn.Module):
|
||||
residual_batch.start(forward_batch)
|
||||
else:
|
||||
assert pp_proxy_tensors is not None
|
||||
hidden_states = self.layers[self.start_layer].attn_stage.from_pp(
|
||||
hidden_states = self.layers[self.start_layer].attn_boundary.from_pp(
|
||||
pp_proxy_tensors, forward_batch
|
||||
)
|
||||
|
||||
|
||||
@@ -30,12 +30,12 @@ from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_r
|
||||
from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation
|
||||
from sglang.srt.eplb.expert_location_dispatch import ExpertLocationDispatchInfo
|
||||
from sglang.srt.layers.aux_hidden_states import AuxHiddenStateAccumulator
|
||||
from sglang.srt.layers.communicator import (
|
||||
from sglang.srt.layers.layer_boundary import (
|
||||
declare_attn,
|
||||
declare_ffn,
|
||||
make_stages,
|
||||
)
|
||||
from sglang.srt.layers.communicator.residual import batch as residual_batch
|
||||
from sglang.srt.layers.layer_boundary.residual import batch as residual_batch
|
||||
from sglang.srt.layers.layernorm import RMSNorm
|
||||
from sglang.srt.layers.linear import (
|
||||
QKVParallelLinear,
|
||||
@@ -866,7 +866,7 @@ class Qwen3MoeDecoderLayer(nn.Module):
|
||||
config.hidden_size, eps=config.rms_norm_eps
|
||||
)
|
||||
|
||||
self.attn_stage, self.ffn_stage = make_stages(
|
||||
self.attn_boundary, self.ffn_boundary = make_stages(
|
||||
(declare_attn(), self.input_layernorm),
|
||||
(
|
||||
declare_ffn(
|
||||
@@ -893,7 +893,7 @@ class Qwen3MoeDecoderLayer(nn.Module):
|
||||
**kwargs,
|
||||
) -> torch.Tensor:
|
||||
|
||||
hidden_states = self.attn_stage.prepare(
|
||||
hidden_states = self.attn_boundary.prepare(
|
||||
hidden_states,
|
||||
forward_batch,
|
||||
captured_last_layer_outputs=captured_last_layer_outputs,
|
||||
@@ -908,10 +908,10 @@ class Qwen3MoeDecoderLayer(nn.Module):
|
||||
forward_batch=forward_batch,
|
||||
)
|
||||
|
||||
hidden_states = self.attn_stage.finish(hidden_states, forward_batch)
|
||||
hidden_states = self.ffn_stage.prepare(hidden_states, forward_batch)
|
||||
hidden_states = self.attn_boundary.finish(hidden_states, forward_batch)
|
||||
hidden_states = self.ffn_boundary.prepare(hidden_states, forward_batch)
|
||||
|
||||
with self.ffn_stage.exit(forward_batch) as ffn_exit:
|
||||
with self.ffn_boundary.exit(forward_batch) as ffn_exit:
|
||||
hidden_states = self.mlp(hidden_states, forward_batch)
|
||||
hidden_states = ffn_exit.finish(hidden_states)
|
||||
|
||||
@@ -925,7 +925,7 @@ class Qwen3MoeDecoderLayer(nn.Module):
|
||||
forward_batch: ForwardBatch,
|
||||
tbo_subbatch_index: Optional[int] = None,
|
||||
):
|
||||
state.hidden_states_after_comm_pre_attn = self.attn_stage.prepare(
|
||||
state.hidden_states_after_comm_pre_attn = self.attn_boundary.prepare(
|
||||
hidden_states, forward_batch
|
||||
)
|
||||
state.update(
|
||||
@@ -937,15 +937,15 @@ class Qwen3MoeDecoderLayer(nn.Module):
|
||||
)
|
||||
|
||||
def op_comm_prepare_mlp(self, state):
|
||||
hidden_states = self.attn_stage.finish(
|
||||
hidden_states = self.attn_boundary.finish(
|
||||
state.pop("hidden_states_after_attn"), state.forward_batch
|
||||
)
|
||||
state.hidden_states_mlp_input = self.ffn_stage.prepare(
|
||||
state.hidden_states_mlp_input = self.ffn_boundary.prepare(
|
||||
hidden_states, state.forward_batch
|
||||
)
|
||||
|
||||
def op_comm_postprocess_layer(self, state):
|
||||
hidden_states = self.ffn_stage.postprocess(
|
||||
hidden_states = self.ffn_boundary.postprocess(
|
||||
state.pop("hidden_states_mlp_output"), state.forward_batch
|
||||
)
|
||||
|
||||
|
||||
@@ -19,15 +19,15 @@ from sglang.srt.layers.aux_hidden_states import (
|
||||
AuxHiddenStateAccumulator,
|
||||
AuxHiddenStateList,
|
||||
)
|
||||
from sglang.srt.layers.communicator import (
|
||||
from sglang.srt.layers.dp_attention import (
|
||||
is_dp_attention_enabled,
|
||||
)
|
||||
from sglang.srt.layers.layer_boundary import (
|
||||
declare_attn,
|
||||
declare_ffn,
|
||||
make_stages,
|
||||
)
|
||||
from sglang.srt.layers.communicator.residual import batch as residual_batch
|
||||
from sglang.srt.layers.dp_attention import (
|
||||
is_dp_attention_enabled,
|
||||
)
|
||||
from sglang.srt.layers.layer_boundary.residual import batch as residual_batch
|
||||
from sglang.srt.layers.layernorm import GemmaRMSNorm
|
||||
from sglang.srt.layers.linear import (
|
||||
ColumnParallelLinear,
|
||||
@@ -477,9 +477,9 @@ def _apply_qwen3_next_mlp(
|
||||
hidden_states: torch.Tensor,
|
||||
forward_batch: ForwardBatch,
|
||||
) -> torch.Tensor:
|
||||
hidden_states = layer.attn_stage.finish(hidden_states, forward_batch)
|
||||
hidden_states = layer.ffn_stage.prepare(hidden_states, forward_batch)
|
||||
with layer.ffn_stage.exit(forward_batch) as ffn_exit:
|
||||
hidden_states = layer.attn_boundary.finish(hidden_states, forward_batch)
|
||||
hidden_states = layer.ffn_boundary.prepare(hidden_states, forward_batch)
|
||||
with layer.ffn_boundary.exit(forward_batch) as ffn_exit:
|
||||
if isinstance(layer.mlp, Qwen2MoeSparseMoeBlock):
|
||||
hidden_states = layer.mlp(
|
||||
hidden_states,
|
||||
@@ -537,7 +537,7 @@ class Qwen3HybridLinearDecoderLayer(nn.Module):
|
||||
self.post_attention_layernorm = GemmaRMSNorm(
|
||||
config.hidden_size, eps=config.rms_norm_eps
|
||||
)
|
||||
self.attn_stage, self.ffn_stage = make_stages(
|
||||
self.attn_boundary, self.ffn_boundary = make_stages(
|
||||
(declare_attn(), self.input_layernorm),
|
||||
(
|
||||
declare_ffn(
|
||||
@@ -562,7 +562,7 @@ class Qwen3HybridLinearDecoderLayer(nn.Module):
|
||||
):
|
||||
forward_batch = kwargs.get("forward_batch", None)
|
||||
|
||||
hidden_states = self.attn_stage.prepare(
|
||||
hidden_states = self.attn_boundary.prepare(
|
||||
hidden_states,
|
||||
forward_batch,
|
||||
captured_last_layer_outputs=captured_last_layer_outputs,
|
||||
@@ -705,7 +705,7 @@ class Qwen3HybridAttentionDecoderLayer(nn.Module):
|
||||
self.q_norm = GemmaRMSNorm(self.head_dim, eps=config.rms_norm_eps)
|
||||
self.k_norm = GemmaRMSNorm(self.head_dim, eps=config.rms_norm_eps)
|
||||
|
||||
self.attn_stage, self.ffn_stage = make_stages(
|
||||
self.attn_boundary, self.ffn_boundary = make_stages(
|
||||
(declare_attn(), self.input_layernorm),
|
||||
(
|
||||
declare_ffn(
|
||||
@@ -832,7 +832,7 @@ class Qwen3HybridAttentionDecoderLayer(nn.Module):
|
||||
captured_last_layer_outputs: Optional[AuxHiddenStateAccumulator] = None,
|
||||
**kwargs: Any,
|
||||
):
|
||||
hidden_states = self.attn_stage.prepare(
|
||||
hidden_states = self.attn_boundary.prepare(
|
||||
hidden_states,
|
||||
forward_batch,
|
||||
captured_last_layer_outputs=captured_last_layer_outputs,
|
||||
|
||||
@@ -38,11 +38,11 @@ from sglang.srt.layers.attention.vision import (
|
||||
prepare_vision_attention_metadata,
|
||||
)
|
||||
from sglang.srt.layers.aux_hidden_states import AuxHiddenStateList
|
||||
from sglang.srt.layers.communicator.residual import batch as residual_batch
|
||||
from sglang.srt.layers.conv import Conv3dLayer
|
||||
from sglang.srt.layers.dp_attention import (
|
||||
is_dp_attention_enabled,
|
||||
)
|
||||
from sglang.srt.layers.layer_boundary.residual import batch as residual_batch
|
||||
from sglang.srt.layers.linear import ColumnParallelLinear, RowParallelLinear
|
||||
from sglang.srt.layers.logits_processor import LogitsProcessor
|
||||
from sglang.srt.layers.pooler import Pooler, PoolingType
|
||||
@@ -1209,7 +1209,7 @@ class Qwen3LLMModel(Qwen3Model):
|
||||
residual_batch.start(forward_batch)
|
||||
else:
|
||||
assert pp_proxy_tensors is not None
|
||||
hidden_states = self.layers[self.start_layer].attn_stage.from_pp(
|
||||
hidden_states = self.layers[self.start_layer].attn_boundary.from_pp(
|
||||
pp_proxy_tensors, forward_batch
|
||||
)
|
||||
|
||||
|
||||
@@ -25,7 +25,7 @@ import torch.nn as nn
|
||||
from sglang.srt.configs.qwen3_vl import Qwen3VLMoeConfig, Qwen3VLMoeTextConfig
|
||||
from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation
|
||||
from sglang.srt.layers.aux_hidden_states import AuxHiddenStateList
|
||||
from sglang.srt.layers.communicator.residual import batch as residual_batch
|
||||
from sglang.srt.layers.layer_boundary.residual import batch as residual_batch
|
||||
from sglang.srt.layers.moe.fused_moe_triton.layer import FusedMoE
|
||||
from sglang.srt.layers.quantization.base_config import QuantizationConfig
|
||||
from sglang.srt.layers.utils import get_layer_id
|
||||
@@ -99,7 +99,7 @@ class Qwen3MoeLLMModel(Qwen3MoeModel):
|
||||
residual_batch.start(forward_batch)
|
||||
else:
|
||||
assert pp_proxy_tensors is not None
|
||||
hidden_states = self.layers[self.start_layer].attn_stage.from_pp(
|
||||
hidden_states = self.layers[self.start_layer].attn_boundary.from_pp(
|
||||
pp_proxy_tensors, forward_batch
|
||||
)
|
||||
|
||||
|
||||
@@ -22,7 +22,6 @@ from sglang.srt.environ import envs
|
||||
from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder
|
||||
from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation
|
||||
from sglang.srt.layers.aux_hidden_states import AuxHiddenStateList
|
||||
from sglang.srt.layers.communicator import get_attn_tp_context
|
||||
from sglang.srt.layers.dp_attention import (
|
||||
attn_tp_all_gather,
|
||||
attn_tp_all_reduce,
|
||||
@@ -38,6 +37,7 @@ from sglang.srt.layers.hyperconnection import (
|
||||
GatedResidual,
|
||||
HyperConnectionConfig,
|
||||
)
|
||||
from sglang.srt.layers.layer_boundary import get_attn_tp_context
|
||||
from sglang.srt.layers.linear import ReplicatedLinear
|
||||
from sglang.srt.layers.logits_processor import LogitsProcessorOutput
|
||||
from sglang.srt.layers.moe import get_moe_a2a_backend, should_use_dp_reduce_scatterv
|
||||
@@ -1320,8 +1320,8 @@ class Qwen4ExpLayerExtensionMixin:
|
||||
for attr_name in (
|
||||
"input_layernorm",
|
||||
"post_attention_layernorm",
|
||||
"attn_stage",
|
||||
"ffn_stage",
|
||||
"attn_boundary",
|
||||
"ffn_boundary",
|
||||
):
|
||||
if hasattr(self, attr_name):
|
||||
delattr(self, attr_name)
|
||||
|
||||
@@ -19,16 +19,16 @@ from sglang.srt.distributed import (
|
||||
from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder
|
||||
from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation
|
||||
from sglang.srt.layers.activation import SiluAndMul
|
||||
from sglang.srt.layers.communicator import (
|
||||
from sglang.srt.layers.dp_attention import (
|
||||
is_dp_attention_enabled,
|
||||
)
|
||||
from sglang.srt.layers.layer_boundary import (
|
||||
declare_attn,
|
||||
declare_ffn,
|
||||
enable_moe_dense_fully_dp,
|
||||
make_stages,
|
||||
)
|
||||
from sglang.srt.layers.communicator.residual import batch as residual_batch
|
||||
from sglang.srt.layers.dp_attention import (
|
||||
is_dp_attention_enabled,
|
||||
)
|
||||
from sglang.srt.layers.layer_boundary.residual import batch as residual_batch
|
||||
from sglang.srt.layers.layernorm import RMSNorm
|
||||
from sglang.srt.layers.linear import (
|
||||
ColumnParallelLinear,
|
||||
@@ -1050,7 +1050,7 @@ class SarvamMoEMLADecoderLayer(nn.Module):
|
||||
|
||||
self.attn_tp_size = get_parallel().attn_tp_size
|
||||
|
||||
self.attn_stage, self.ffn_stage = make_stages(
|
||||
self.attn_boundary, self.ffn_boundary = make_stages(
|
||||
(
|
||||
declare_attn(),
|
||||
self.input_layernorm,
|
||||
@@ -1077,16 +1077,16 @@ class SarvamMoEMLADecoderLayer(nn.Module):
|
||||
hidden_states: torch.Tensor,
|
||||
forward_batch: ForwardBatch,
|
||||
) -> torch.Tensor:
|
||||
hidden_states = self.attn_stage.prepare(hidden_states, forward_batch)
|
||||
hidden_states = self.attn_boundary.prepare(hidden_states, forward_batch)
|
||||
if hidden_states.shape[0] != 0:
|
||||
hidden_states = self.self_attn(
|
||||
positions=positions,
|
||||
hidden_states=hidden_states,
|
||||
forward_batch=forward_batch,
|
||||
)
|
||||
hidden_states = self.attn_stage.finish(hidden_states, forward_batch)
|
||||
hidden_states = self.ffn_stage.prepare(hidden_states, forward_batch)
|
||||
with self.ffn_stage.exit(forward_batch) as ffn_exit:
|
||||
hidden_states = self.attn_boundary.finish(hidden_states, forward_batch)
|
||||
hidden_states = self.ffn_boundary.prepare(hidden_states, forward_batch)
|
||||
with self.ffn_boundary.exit(forward_batch) as ffn_exit:
|
||||
hidden_states = self.mlp(hidden_states, forward_batch)
|
||||
if (
|
||||
not self.is_layer_sparse
|
||||
@@ -1159,7 +1159,7 @@ class SarvamMLAModel(nn.Module):
|
||||
residual_batch.start(forward_batch)
|
||||
else:
|
||||
assert pp_proxy_tensors is not None
|
||||
hidden_states = self.layers[self.start_layer].attn_stage.from_pp(
|
||||
hidden_states = self.layers[self.start_layer].attn_boundary.from_pp(
|
||||
pp_proxy_tensors, forward_batch
|
||||
)
|
||||
|
||||
|
||||
@@ -11,15 +11,15 @@ from torch import nn
|
||||
from transformers import PretrainedConfig
|
||||
|
||||
from sglang.srt.layers.activation import SiluAndMul
|
||||
from sglang.srt.layers.communicator import (
|
||||
from sglang.srt.layers.dp_attention import (
|
||||
is_dp_attention_enabled,
|
||||
)
|
||||
from sglang.srt.layers.layer_boundary import (
|
||||
declare_attn,
|
||||
declare_ffn,
|
||||
make_stages,
|
||||
)
|
||||
from sglang.srt.layers.communicator.residual import batch as residual_batch
|
||||
from sglang.srt.layers.dp_attention import (
|
||||
is_dp_attention_enabled,
|
||||
)
|
||||
from sglang.srt.layers.layer_boundary.residual import batch as residual_batch
|
||||
from sglang.srt.layers.layernorm import RMSNorm
|
||||
from sglang.srt.layers.linear import (
|
||||
MergedColumnParallelLinear,
|
||||
@@ -295,7 +295,7 @@ class SDARBlock(nn.Module):
|
||||
prefix=add_prefix("mlp", prefix),
|
||||
)
|
||||
|
||||
self.attn_stage, self.ffn_stage = make_stages(
|
||||
self.attn_boundary, self.ffn_boundary = make_stages(
|
||||
(declare_attn(), self.input_layernorm),
|
||||
(
|
||||
declare_ffn(
|
||||
@@ -314,7 +314,7 @@ class SDARBlock(nn.Module):
|
||||
hidden_states: torch.Tensor,
|
||||
forward_batch: ForwardBatch,
|
||||
) -> torch.Tensor:
|
||||
hidden_states = self.attn_stage.prepare(hidden_states, forward_batch)
|
||||
hidden_states = self.attn_boundary.prepare(hidden_states, forward_batch)
|
||||
|
||||
if hidden_states.shape[0] != 0:
|
||||
hidden_states = self.self_attn(
|
||||
@@ -323,10 +323,10 @@ class SDARBlock(nn.Module):
|
||||
forward_batch=forward_batch,
|
||||
)
|
||||
|
||||
hidden_states = self.attn_stage.finish(hidden_states, forward_batch)
|
||||
hidden_states = self.ffn_stage.prepare(hidden_states, forward_batch)
|
||||
hidden_states = self.attn_boundary.finish(hidden_states, forward_batch)
|
||||
hidden_states = self.ffn_boundary.prepare(hidden_states, forward_batch)
|
||||
|
||||
with self.ffn_stage.exit(forward_batch) as ffn_exit:
|
||||
with self.ffn_boundary.exit(forward_batch) as ffn_exit:
|
||||
hidden_states = self.mlp(hidden_states)
|
||||
hidden_states = ffn_exit.finish(hidden_states)
|
||||
|
||||
@@ -401,7 +401,7 @@ class SDARModel(nn.Module):
|
||||
residual_batch.start(forward_batch)
|
||||
else:
|
||||
assert pp_proxy_tensors is not None
|
||||
hidden_states = self.layers[self.start_layer].attn_stage.from_pp(
|
||||
hidden_states = self.layers[self.start_layer].attn_boundary.from_pp(
|
||||
pp_proxy_tensors, forward_batch, allow_missing_residual=True
|
||||
)
|
||||
|
||||
|
||||
@@ -13,15 +13,15 @@ from transformers import PretrainedConfig
|
||||
from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder
|
||||
from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation
|
||||
from sglang.srt.eplb.expert_location_dispatch import ExpertLocationDispatchInfo
|
||||
from sglang.srt.layers.communicator import (
|
||||
from sglang.srt.layers.dp_attention import (
|
||||
is_dp_attention_enabled,
|
||||
)
|
||||
from sglang.srt.layers.layer_boundary import (
|
||||
declare_attn,
|
||||
declare_ffn,
|
||||
make_stages,
|
||||
)
|
||||
from sglang.srt.layers.communicator.residual import batch as residual_batch
|
||||
from sglang.srt.layers.dp_attention import (
|
||||
is_dp_attention_enabled,
|
||||
)
|
||||
from sglang.srt.layers.layer_boundary.residual import batch as residual_batch
|
||||
from sglang.srt.layers.layernorm import RMSNorm
|
||||
from sglang.srt.layers.linear import (
|
||||
QKVParallelLinear,
|
||||
@@ -357,7 +357,7 @@ class SDARMoeBlock(nn.Module):
|
||||
prefix=add_prefix("mlp", prefix),
|
||||
)
|
||||
|
||||
self.attn_stage, self.ffn_stage = make_stages(
|
||||
self.attn_boundary, self.ffn_boundary = make_stages(
|
||||
(declare_attn(), self.input_layernorm),
|
||||
(
|
||||
declare_ffn(sparse=True, next_sparse=True),
|
||||
@@ -376,7 +376,7 @@ class SDARMoeBlock(nn.Module):
|
||||
forward_batch: ForwardBatch,
|
||||
) -> torch.Tensor:
|
||||
|
||||
hidden_states = self.attn_stage.prepare(hidden_states, forward_batch)
|
||||
hidden_states = self.attn_boundary.prepare(hidden_states, forward_batch)
|
||||
|
||||
if hidden_states.shape[0] != 0:
|
||||
hidden_states = self.self_attn(
|
||||
@@ -385,10 +385,10 @@ class SDARMoeBlock(nn.Module):
|
||||
forward_batch=forward_batch,
|
||||
)
|
||||
|
||||
hidden_states = self.attn_stage.finish(hidden_states, forward_batch)
|
||||
hidden_states = self.ffn_stage.prepare(hidden_states, forward_batch)
|
||||
hidden_states = self.attn_boundary.finish(hidden_states, forward_batch)
|
||||
hidden_states = self.ffn_boundary.prepare(hidden_states, forward_batch)
|
||||
|
||||
with self.ffn_stage.exit(forward_batch) as ffn_exit:
|
||||
with self.ffn_boundary.exit(forward_batch) as ffn_exit:
|
||||
hidden_states = self.mlp(
|
||||
hidden_states,
|
||||
forward_batch=forward_batch,
|
||||
@@ -467,7 +467,7 @@ class SDARMoeModel(nn.Module):
|
||||
residual_batch.start(forward_batch)
|
||||
else:
|
||||
assert pp_proxy_tensors is not None
|
||||
hidden_states = self.layers[self.start_layer].attn_stage.from_pp(
|
||||
hidden_states = self.layers[self.start_layer].attn_boundary.from_pp(
|
||||
pp_proxy_tensors, forward_batch
|
||||
)
|
||||
|
||||
|
||||
@@ -19,16 +19,16 @@ from sglang.srt.configs.step3_vl import (
|
||||
from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation
|
||||
from sglang.srt.layers.activation import SiluAndMul
|
||||
from sglang.srt.layers.attention.vision import VisionAttention
|
||||
from sglang.srt.layers.communicator import (
|
||||
declare_attn,
|
||||
declare_ffn,
|
||||
make_stages,
|
||||
)
|
||||
from sglang.srt.layers.communicator.residual import batch as residual_batch
|
||||
from sglang.srt.layers.conv import Conv2dLayer
|
||||
from sglang.srt.layers.dp_attention import (
|
||||
is_dp_attention_enabled,
|
||||
)
|
||||
from sglang.srt.layers.layer_boundary import (
|
||||
declare_attn,
|
||||
declare_ffn,
|
||||
make_stages,
|
||||
)
|
||||
from sglang.srt.layers.layer_boundary.residual import batch as residual_batch
|
||||
from sglang.srt.layers.layernorm import RMSNorm
|
||||
from sglang.srt.layers.linear import (
|
||||
ColumnParallelLinear,
|
||||
@@ -374,7 +374,7 @@ class Step3TextDecoderLayer(nn.Module):
|
||||
prefix=add_prefix("mlp", prefix),
|
||||
)
|
||||
|
||||
self.attn_stage, self.ffn_stage = make_stages(
|
||||
self.attn_boundary, self.ffn_boundary = make_stages(
|
||||
(declare_attn(), self.input_layernorm),
|
||||
(
|
||||
declare_ffn(
|
||||
@@ -407,7 +407,7 @@ class Step3TextDecoderLayer(nn.Module):
|
||||
forward_batch: ForwardBatch,
|
||||
) -> torch.Tensor:
|
||||
|
||||
hidden_states = self.attn_stage.prepare(hidden_states, forward_batch)
|
||||
hidden_states = self.attn_boundary.prepare(hidden_states, forward_batch)
|
||||
|
||||
if hidden_states.shape[0] != 0:
|
||||
hidden_states = self.self_attn(
|
||||
@@ -416,9 +416,9 @@ class Step3TextDecoderLayer(nn.Module):
|
||||
forward_batch=forward_batch,
|
||||
)
|
||||
|
||||
hidden_states = self.attn_stage.finish(hidden_states, forward_batch)
|
||||
hidden_states = self.ffn_stage.prepare(hidden_states, forward_batch)
|
||||
with self.ffn_stage.exit(forward_batch) as ffn_exit:
|
||||
hidden_states = self.attn_boundary.finish(hidden_states, forward_batch)
|
||||
hidden_states = self.ffn_boundary.prepare(hidden_states, forward_batch)
|
||||
with self.ffn_boundary.exit(forward_batch) as ffn_exit:
|
||||
if self.use_moe:
|
||||
hidden_states = self.moe_mlp_forward(hidden_states)
|
||||
else:
|
||||
|
||||
@@ -11,15 +11,15 @@ from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_r
|
||||
from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation
|
||||
from sglang.srt.eplb.expert_location_dispatch import ExpertLocationDispatchInfo
|
||||
from sglang.srt.layers.activation import SiluAndMul
|
||||
from sglang.srt.layers.communicator import (
|
||||
from sglang.srt.layers.dp_attention import (
|
||||
is_dp_attention_enabled,
|
||||
)
|
||||
from sglang.srt.layers.layer_boundary import (
|
||||
declare_attn,
|
||||
declare_ffn,
|
||||
make_stages,
|
||||
)
|
||||
from sglang.srt.layers.communicator.residual import batch as residual_batch
|
||||
from sglang.srt.layers.dp_attention import (
|
||||
is_dp_attention_enabled,
|
||||
)
|
||||
from sglang.srt.layers.layer_boundary.residual import batch as residual_batch
|
||||
from sglang.srt.layers.layernorm import GemmaRMSNorm
|
||||
from sglang.srt.layers.linear import (
|
||||
ColumnParallelLinear,
|
||||
@@ -566,7 +566,7 @@ class Step3p5DecoderLayer(nn.Module):
|
||||
|
||||
# An MTP draft is a one-layer model; layer_id still indexes its config.
|
||||
|
||||
self.attn_stage, self.ffn_stage = make_stages(
|
||||
self.attn_boundary, self.ffn_boundary = make_stages(
|
||||
(declare_attn(), self.input_layernorm),
|
||||
(
|
||||
declare_ffn(
|
||||
@@ -594,7 +594,7 @@ class Step3p5DecoderLayer(nn.Module):
|
||||
post_residual_addition: Optional[torch.Tensor] = None,
|
||||
) -> torch.Tensor:
|
||||
# Self Attention
|
||||
hidden_states = self.attn_stage.prepare(
|
||||
hidden_states = self.attn_boundary.prepare(
|
||||
hidden_states, forward_batch, post_residual_addition=post_residual_addition
|
||||
)
|
||||
if hidden_states.shape[0] != 0:
|
||||
@@ -604,11 +604,11 @@ class Step3p5DecoderLayer(nn.Module):
|
||||
forward_batch=forward_batch,
|
||||
)
|
||||
# Fully Connected
|
||||
hidden_states = self.attn_stage.finish(hidden_states, forward_batch)
|
||||
hidden_states = self.ffn_stage.prepare(hidden_states, forward_batch)
|
||||
hidden_states = self.attn_boundary.finish(hidden_states, forward_batch)
|
||||
hidden_states = self.ffn_boundary.prepare(hidden_states, forward_batch)
|
||||
|
||||
if self.use_moe:
|
||||
with self.ffn_stage.exit(forward_batch) as ffn_exit:
|
||||
with self.ffn_boundary.exit(forward_batch) as ffn_exit:
|
||||
# Both share_expert and MoE return unreduced (TP-partial) outputs.
|
||||
# Combine them first, then do a single all-reduce — saving one
|
||||
# full-TP all-reduce per layer.
|
||||
@@ -621,7 +621,7 @@ class Step3p5DecoderLayer(nn.Module):
|
||||
hidden_states = tensor_model_parallel_all_reduce(hidden_states)
|
||||
return ffn_exit.finish(hidden_states)
|
||||
|
||||
with self.ffn_stage.exit(forward_batch) as ffn_exit:
|
||||
with self.ffn_boundary.exit(forward_batch) as ffn_exit:
|
||||
hidden_states = self.mlp(hidden_states)
|
||||
return ffn_exit.finish(hidden_states)
|
||||
|
||||
@@ -700,7 +700,7 @@ class Step3p5Model(nn.Module):
|
||||
residual_batch.start(forward_batch)
|
||||
else:
|
||||
assert pp_proxy_tensors is not None
|
||||
hidden_states = self.layers[self.start_layer].attn_stage.from_pp(
|
||||
hidden_states = self.layers[self.start_layer].attn_boundary.from_pp(
|
||||
pp_proxy_tensors, forward_batch
|
||||
)
|
||||
|
||||
|
||||
@@ -6,7 +6,7 @@ import torch
|
||||
import torch.nn as nn
|
||||
from transformers import PretrainedConfig
|
||||
|
||||
from sglang.srt.layers.communicator.residual import batch as residual_batch
|
||||
from sglang.srt.layers.layer_boundary.residual import batch as residual_batch
|
||||
from sglang.srt.layers.layernorm import GemmaRMSNorm
|
||||
from sglang.srt.layers.logits_processor import LogitsProcessor
|
||||
from sglang.srt.layers.quantization.base_config import QuantizationConfig
|
||||
|
||||
@@ -39,14 +39,14 @@ from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_r
|
||||
from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation
|
||||
from sglang.srt.eplb.expert_location_dispatch import ExpertLocationDispatchInfo
|
||||
from sglang.srt.layers.activation import SiluAndMul
|
||||
from sglang.srt.layers.communicator import (
|
||||
from sglang.srt.layers.dp_attention import is_dp_attention_enabled
|
||||
from sglang.srt.layers.layer_boundary import (
|
||||
declare_attn,
|
||||
declare_ffn,
|
||||
enable_moe_dense_fully_dp,
|
||||
make_stages,
|
||||
)
|
||||
from sglang.srt.layers.communicator.residual import batch as residual_batch
|
||||
from sglang.srt.layers.dp_attention import is_dp_attention_enabled
|
||||
from sglang.srt.layers.layer_boundary.residual import batch as residual_batch
|
||||
from sglang.srt.layers.layernorm import RMSNorm
|
||||
from sglang.srt.layers.linear import (
|
||||
ColumnParallelLinear,
|
||||
@@ -1528,7 +1528,7 @@ class XllmDecoderLayer(nn.Module):
|
||||
|
||||
self.input_layernorm = _make_norm(config)
|
||||
self.post_attention_layernorm = _make_norm(config)
|
||||
self.attn_stage, self.ffn_stage = make_stages(
|
||||
self.attn_boundary, self.ffn_boundary = make_stages(
|
||||
(declare_attn(), self.input_layernorm),
|
||||
(
|
||||
declare_ffn(
|
||||
@@ -1551,7 +1551,7 @@ class XllmDecoderLayer(nn.Module):
|
||||
hidden_states: torch.Tensor,
|
||||
forward_batch: ForwardBatch,
|
||||
) -> torch.Tensor:
|
||||
hidden_states = self.attn_stage.prepare(hidden_states, forward_batch)
|
||||
hidden_states = self.attn_boundary.prepare(hidden_states, forward_batch)
|
||||
|
||||
if hidden_states.shape[0] != 0:
|
||||
hidden_states = self.self_attn(
|
||||
@@ -1560,10 +1560,10 @@ class XllmDecoderLayer(nn.Module):
|
||||
forward_batch=forward_batch,
|
||||
)
|
||||
|
||||
hidden_states = self.attn_stage.finish(hidden_states, forward_batch)
|
||||
hidden_states = self.ffn_stage.prepare(hidden_states, forward_batch)
|
||||
hidden_states = self.attn_boundary.finish(hidden_states, forward_batch)
|
||||
hidden_states = self.ffn_boundary.prepare(hidden_states, forward_batch)
|
||||
|
||||
with self.ffn_stage.exit(forward_batch) as ffn_exit:
|
||||
with self.ffn_boundary.exit(forward_batch) as ffn_exit:
|
||||
if isinstance(self.mlp, XllmMLP):
|
||||
hidden_states = self.mlp(hidden_states)
|
||||
else:
|
||||
@@ -1629,7 +1629,7 @@ class XllmModel(nn.Module):
|
||||
residual_batch.start(forward_batch)
|
||||
else:
|
||||
assert pp_proxy_tensors is not None
|
||||
hidden_states = self.layers[self.start_layer].attn_stage.from_pp(
|
||||
hidden_states = self.layers[self.start_layer].attn_boundary.from_pp(
|
||||
pp_proxy_tensors, forward_batch
|
||||
)
|
||||
|
||||
|
||||
@@ -2,15 +2,15 @@
|
||||
|
||||
from types import SimpleNamespace
|
||||
|
||||
from sglang.srt.layers.communicator.construction import StagePlan
|
||||
from sglang.srt.layers.communicator.factories import (
|
||||
from sglang.srt.layers.layer_boundary.construction import StagePlan
|
||||
from sglang.srt.layers.layer_boundary.factories import (
|
||||
declare_attn,
|
||||
declare_ffn,
|
||||
make_stages,
|
||||
)
|
||||
from sglang.srt.layers.communicator.ops import identity_output
|
||||
from sglang.srt.layers.communicator.residual.add_norm import PLAIN_RESIDUAL
|
||||
from sglang.srt.layers.communicator.stage import StageCommunicator
|
||||
from sglang.srt.layers.layer_boundary.ops import identity_output
|
||||
from sglang.srt.layers.layer_boundary.residual.add_norm import PLAIN_RESIDUAL
|
||||
from sglang.srt.layers.layer_boundary.stage import StageBoundary
|
||||
|
||||
|
||||
def make_test_stages(
|
||||
@@ -73,7 +73,7 @@ def stub_stage(plan, kind):
|
||||
"""Keep the same concrete boundary when a test patches its methods."""
|
||||
stages = plan.__dict__.setdefault("_stub_stages", {})
|
||||
if kind not in stages:
|
||||
stages[kind] = StageCommunicator(
|
||||
stages[kind] = StageBoundary(
|
||||
plan,
|
||||
declaration=(declare_attn() if kind.name == "ATTENTION" else declare_ffn()),
|
||||
)
|
||||
@@ -82,7 +82,7 @@ def stub_stage(plan, kind):
|
||||
|
||||
def sp_region_steps():
|
||||
"""Local SP rows with no owed sum, for tests of activation and exits."""
|
||||
from sglang.srt.layers.communicator import (
|
||||
from sglang.srt.layers.layer_boundary import (
|
||||
NORM_QUANT_READ,
|
||||
EdgeDecl,
|
||||
Layout,
|
||||
@@ -93,7 +93,7 @@ def sp_region_steps():
|
||||
TokenAxis,
|
||||
make_boundary,
|
||||
)
|
||||
from sglang.srt.layers.communicator.prepare import _hand_qkv_hook_its_input
|
||||
from sglang.srt.layers.layer_boundary.prepare import _hand_qkv_hook_its_input
|
||||
|
||||
rows = Layout(frozenset({TokenAxis.ATTN_TP_SCATTER}))
|
||||
output = StageOutput(rows)
|
||||
@@ -120,8 +120,8 @@ def prepare_attention(stage, hidden, residual, forward_batch, *args, **call):
|
||||
|
||||
|
||||
def prepare_raw(stage, method, hidden, residual, forward_batch, *args, **call):
|
||||
from sglang.srt.layers.communicator.residual.add_norm import ADD
|
||||
from sglang.srt.layers.communicator.residual.stream import ResidualStream
|
||||
from sglang.srt.layers.layer_boundary.residual.add_norm import ADD
|
||||
from sglang.srt.layers.layer_boundary.residual.stream import ResidualStream
|
||||
|
||||
if not isinstance(residual, ResidualStream):
|
||||
stream = ResidualStream(residual)
|
||||
@@ -137,7 +137,7 @@ def prepare_raw(stage, method, hidden, residual, forward_batch, *args, **call):
|
||||
|
||||
|
||||
def finish_exit(scope, hidden, residual):
|
||||
from sglang.srt.layers.communicator.residual.stream import ResidualStream
|
||||
from sglang.srt.layers.layer_boundary.residual.stream import ResidualStream
|
||||
|
||||
if isinstance(residual, ResidualStream):
|
||||
scope._stream = residual
|
||||
@@ -150,7 +150,7 @@ def finish_exit(scope, hidden, residual):
|
||||
|
||||
|
||||
def postprocess_output(boundary, hidden, residual, forward_batch):
|
||||
from sglang.srt.layers.communicator.residual.stream import ResidualStream
|
||||
from sglang.srt.layers.layer_boundary.residual.stream import ResidualStream
|
||||
|
||||
stream = (
|
||||
residual if isinstance(residual, ResidualStream) else ResidualStream(residual)
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
"""Patching a name the communicator package's modules look up.
|
||||
|
||||
Each module of ``sglang.srt.layers.communicator`` binds the names it imports
|
||||
Each module of ``sglang.srt.layers.layer_boundary`` binds the names it imports
|
||||
in its own namespace, so a test double has to go into every module that reads
|
||||
the name, not only into the package.
|
||||
"""
|
||||
@@ -8,8 +8,8 @@ the name, not only into the package.
|
||||
import contextlib
|
||||
from unittest import mock
|
||||
|
||||
from sglang.srt.layers import communicator as communicator
|
||||
from sglang.srt.layers.communicator import (
|
||||
from sglang.srt.layers import layer_boundary as communicator
|
||||
from sglang.srt.layers.layer_boundary import (
|
||||
boundary,
|
||||
construction,
|
||||
contracts,
|
||||
@@ -20,13 +20,13 @@ from sglang.srt.layers.communicator import (
|
||||
output,
|
||||
prepare,
|
||||
)
|
||||
from sglang.srt.layers.communicator import residual as residual_contract
|
||||
from sglang.srt.layers.communicator import (
|
||||
from sglang.srt.layers.layer_boundary import residual as residual_contract
|
||||
from sglang.srt.layers.layer_boundary import (
|
||||
stage,
|
||||
)
|
||||
from sglang.srt.layers.communicator.adapters import attention, branch, lora, overlap
|
||||
from sglang.srt.layers.communicator.fusions import allreduce
|
||||
from sglang.srt.layers.communicator.residual import (
|
||||
from sglang.srt.layers.layer_boundary.adapters import attention, branch, lora, overlap
|
||||
from sglang.srt.layers.layer_boundary.fusions import allreduce
|
||||
from sglang.srt.layers.layer_boundary.residual import (
|
||||
access,
|
||||
add_norm,
|
||||
batch,
|
||||
|
||||
Reference in New Issue
Block a user