[Refactor] Migrate specialized decoder and overlap boundaries (#41553)

This commit is contained in:
Cheng Wan
2026-09-28 20:17:17 -07:00
committed by GitHub
parent 49b5e76e45
commit 9b89b96230
61 changed files with 1729 additions and 2037 deletions
+2 -3
View File
@@ -626,9 +626,8 @@ class ExecComm(msgspec.Struct):
help="Select FFN boundary reduction: ar uses all-reduce then token "
"redistribution; rs and rsv permit fixed-size and variable-size "
"reduce-scatter respectively; rs+rsv permits both (RSv first). "
"Unsupported paths fall back to ar. The option applies only to FFN "
"layers using stage boundaries, where auto resolves the model default. "
"Required attention and MoE collectives and "
"Unsupported paths fall back to ar. auto resolves the model default "
"for all FFN layers. Required attention and MoE collectives and "
"all-reduce fusion are unaffected.",
resolvable=True,
),
@@ -15,7 +15,7 @@ def handle_layernorm_sp(server_args: ServerArgs) -> None:
"""Validate --enable-layernorm-sp against the resolved parallelism config.
Runs in the resolution pipeline rather than in the layers so a model that
never builds a LayerCommunicator rejects the flag instead of ignoring it.
does not use layer boundaries rejects the flag instead of ignoring it.
"""
cfg = resolving_view(server_args)
if not cfg.enable_layernorm_sp:
@@ -18,6 +18,7 @@ from sglang.srt.layers.communicator import (
Layout,
tbo_split_moves,
)
from sglang.srt.layers.communicator.residual import batch as residual_batch
from sglang.srt.layers.communicator.residual.access import finish_layer_stack
from sglang.srt.layers.communicator.residual.stream import ResidualStream
from sglang.srt.layers.moe import (
@@ -930,6 +931,67 @@ def _compute_extend_num_tokens(input_ids, forward_mode: ForwardMode):
# -------------------------------- Execution ---------------------------------------
def model_forward_stages(
layers,
enable_tbo: bool,
positions: torch.Tensor,
forward_batch: ForwardBatch,
hidden_states: torch.Tensor,
zero_allocator: Optional[BumpAllocator] = None,
):
"""Run stage operations with an independent residual stream per microbatch."""
strategy = OperationsStrategy.init_new_tbo(
layers, forward_batch.global_forward_mode
)
inputs = dict(
positions=positions,
hidden_states=hidden_states,
forward_batch=forward_batch,
zero_allocator=zero_allocator,
)
if not enable_tbo:
return execute_operations(inputs, strategy.operations)["hidden_states"]
stream = residual_batch.current(forward_batch)
pending = stream.pending
hidden_states, residual = stream.finish(hidden_states)
inputs["hidden_states"] = hidden_states
parts = _model_forward_tbo_split_inputs(
**inputs, residual=residual, layer_input_rows=layers[0].attn_stage.input_rows
)
for part in parts:
part["hidden_states"], child_stream = ResidualStream.arrive(
part["hidden_states"],
part.pop("residual"),
pending.update if pending is not None else None,
)
part["forward_batch"].residual_stream = child_stream
original_len = hidden_states.shape[0]
forward_batch.residual_stream = None
del inputs, pending, stream, hidden_states, residual
context = (
empty_context()
if _is_hip
else deep_gemm_wrapper.configure_deep_gemm_num_sms(strategy.deep_gemm_num_sms)
)
with context:
outputs = execute_overlapped_operations(
inputs_arr=parts,
operations_arr=[strategy.operations] * 2,
delta_stages=[0, strategy.tbo_delta_stages],
)
for output in outputs:
output["residual"] = residual_batch.current(output["forward_batch"])
hidden_states, forward_batch.residual_stream = _model_forward_tbo_merge_outputs(
*outputs, original_len
)
for output in outputs:
output["forward_batch"].residual_stream = None
return hidden_states
def model_forward_maybe_tbo(
layers,
enable_tbo: bool,
@@ -79,8 +79,8 @@ from sglang.srt.layers.communicator.residual.mhc import (
__all__ = [
"LayerCommunicator",
"MHCLayerCommunicator",
"LayerFacts",
"MHCLayerCommunicator",
"declare_attn",
"declare_ffn",
"make_attn_stage",
@@ -191,17 +191,19 @@ class OutputBoundary:
dp_step=dp_step,
)
steps = self.plan._batch_steps(forward_batch)
if not steps.ffn_output.leaves_for_next_layer:
defer_moe_finalize = (
self.plan.fusions is not None
and self.plan.fusions.can_defer_finalize(self.plan, forward_batch)
)
if not steps.ffn_output.leaves_for_next_layer and not (
self.plan.is_last_layer and defer_moe_finalize
):
return FfnCompletion(
defer_moe_finalize=False,
fuse_mlp_allreduce=False,
mlp_reduce_scatter=mlp_reduce_scatter,
complete=complete_now,
)
defer_moe_finalize = (
self.plan.fusions is not None
and self.plan.fusions.can_defer_finalize(self.plan, forward_batch)
)
# Producers declare remaining work independently of the kernel chosen
# by the consumer. Every handoff also carries an unfused completion.
fuse_mlp_allreduce = defer_moe_finalize or self._ffn_sum_moves_to_next_layer(
@@ -16,7 +16,6 @@ import torch
from sglang.srt.layers.communicator import (
FusedMlpInput,
HandoffOutput,
LayerCommunicator,
SumGroup,
TokenAxis,
UnreducedOutput,
@@ -196,7 +195,7 @@ class CuteDSLFusionService:
class CuteDSLFusion:
"""One layer's CuTe DSL kernels, given to its LayerCommunicator at
"""One layer's CuTe DSL kernels, given to its stage at
construction. At the attention input: the MoE finalize + AR + add + norm of
a handoff the previous layer left, and the AR + add + norm of a sum it left.
At the FFN input: the attention output's AR + add + norm. At the FFN exit:
@@ -208,6 +207,7 @@ class CuteDSLFusion:
self.service: CuteDSLFusionService | None = None
# Producer capabilities; consumer kernel selection is independent.
self.hands_off_finalize = False
self.terminal_finalize = False
self.requires_local_reduction = False
def install(
@@ -216,25 +216,27 @@ class CuteDSLFusion:
*,
hands_off_finalize: bool,
output_is_replicated: bool,
terminal_finalize: bool = False,
) -> None:
"""Install the shared workspace and this producer's capabilities."""
self.service = service
self.hands_off_finalize = hands_off_finalize
self.terminal_finalize = terminal_finalize
self.requires_local_reduction = output_is_replicated
def attention_input(self, layer: LayerCommunicator) -> tuple:
def attention_input(self, layer) -> tuple:
return (
partial(self._finalize_output_and_update_and_read_residual, layer),
partial(self._reduce_output_and_update_and_read_residual, layer),
)
def ffn_input(self, layer: LayerCommunicator) -> tuple:
def ffn_input(self, layer) -> tuple:
parallel = get_parallel()
if (
TokenAxis.ATTN_TP_SCATTER not in layer.input_rows.sharded
# The workspace sums over TP, which is then the attention-TP group.
and parallel.attn_tp_size == parallel.tp_size
and _fused_norm_gamma(layer.post_attention_layernorm) is not None
and _fused_norm_gamma(layer.norm) is not None
):
return (
FusedMlpInput(
@@ -255,7 +257,7 @@ class CuteDSLFusion:
handoff is completed by its producer's own tail."""
if not isinstance(owed, MoeFinalizeHandoff):
return None
gamma = _fused_norm_gamma(layer.input_layernorm)
gamma = _fused_norm_gamma(layer.norm)
if gamma is None or not self._should_use_finalize(layer, forward_batch, owed.m):
return None
if post_residual_addition is not None:
@@ -280,7 +282,7 @@ class CuteDSLFusion:
return self.service.all_reduce_residual_rms_norm(
local_contribution=owed.partial,
residual=residual,
gamma=_fused_norm_gamma(layer.input_layernorm),
gamma=_fused_norm_gamma(layer.norm),
)
def _mlp_input_reduce_output_and_update_and_read_residual(
@@ -298,7 +300,7 @@ class CuteDSLFusion:
return self.service.all_reduce_residual_rms_norm(
local_contribution=hidden_states,
residual=residual,
gamma=_fused_norm_gamma(layer.post_attention_layernorm),
gamma=_fused_norm_gamma(layer.norm),
)
def _should_use_finalize(self, layer, forward_batch: ForwardBatch, m: int) -> bool:
@@ -313,7 +315,7 @@ class CuteDSLFusion:
"""Incoming, and independent of this layer's own successor."""
return (
self._common_eligible(layer, forward_batch, m)
and _fused_norm_gamma(layer.input_layernorm) is not None
and _fused_norm_gamma(layer.norm) is not None
and not get_exec().comm.enable_quant_communications
)
@@ -334,7 +336,11 @@ class CuteDSLFusion:
"""Whether this producer can emit a handoff with an unfused fallback."""
return (
self.hands_off_finalize
and not self.requires_local_reduction
and (not layer.is_last_layer or self.terminal_finalize)
and (
not self.requires_local_reduction
or (layer.is_last_layer and self.terminal_finalize)
)
and self._should_use_finalize(
layer, forward_batch, int(forward_batch.input_ids.shape[0])
)
@@ -355,12 +361,12 @@ class CuteDSLFusion:
and parallel.tp_size > 1
# The FFN runs on the full rows, not each rank's own slice.
and TokenAxis.ATTN_TP_SCATTER
not in layer.ffn.entry(forward_batch).input_rows.sharded
not in layer.fusion_rows(forward_batch).sharded
)
def _fusion_of(layer: torch.nn.Module) -> CuteDSLFusion | None:
fusions = getattr(layer.layer_communicator, "fusions", None)
fusions = getattr(layer.__dict__.get("ffn_stage"), "fusions", None)
return fusions if isinstance(fusions, CuteDSLFusion) else None
@@ -373,10 +379,11 @@ def install_cutedsl_fusion(
can_defer_finalize: _LayerPredicate,
requires_local_reduction: _LayerPredicate | None = None,
label: str,
terminal_finalize: bool = False,
) -> CuteDSLFusionService | None:
"""One shared workspace handle per fusion-enabled layer, or None.
Every entry of ``layers`` must carry a ``layer_communicator``.
Every entry of ``layers`` carries its attention and FFN stages.
"""
if get_flags().moe.in_speculative_scope:
# A draft shares the target's process, which holds one workspace.
@@ -389,8 +396,8 @@ def install_cutedsl_fusion(
# The workspace compiles one epsilon and cannot see a per-layer one.
for layer in fusion_layers:
for norm in (
layer.layer_communicator.input_layernorm,
layer.layer_communicator.post_attention_layernorm,
layer.attn_stage.norm,
layer.ffn_stage.norm,
):
if _fused_norm_gamma(norm) is None:
continue
@@ -416,6 +423,7 @@ def install_cutedsl_fusion(
fusion.install(
service,
hands_off_finalize=hands_off_finalize,
terminal_finalize=terminal_finalize,
output_is_replicated=requires_local_reduction is not None
and bool(requires_local_reduction(layer)),
)
@@ -441,11 +449,9 @@ def prepare_cutedsl_fusion(
fusions = [
fusion
for module in model.modules()
# Most modules carry no layer_communicator.
# Only decoder modules carry an FFN stage.
if isinstance(
fusion := getattr(
module.__dict__.get("layer_communicator"), "fusions", None
),
fusion := getattr(module.__dict__.get("ffn_stage"), "fusions", None),
CuteDSLFusion,
)
]
@@ -45,11 +45,37 @@ def take_output(hidden_states, forward_batch):
return hidden_states
def norm(hidden_states, forward_batch, layernorm, capture_output=None, **read_kwargs):
hidden_states, residual = current(forward_batch).finish(hidden_states)
def norm(
hidden_states,
forward_batch,
layernorm,
capture_output=None,
*,
handoff_norm=None,
skip_empty=False,
**read_kwargs,
):
hidden_states, residual = current(forward_batch).finish(
hidden_states, takes_handoff=handoff_norm is not None
)
# The terminal consumer now owns the pair. Do not keep layer buffers alive
# through logits processing or the next forward on this batch.
forward_batch.residual_stream = None
from sglang.srt.layers.communicator.output import HandoffOutput
if isinstance(hidden_states, HandoffOutput):
if residual is None:
raise RuntimeError("invalid final deferred MoE handoff")
if capture_output is not None:
raise RuntimeError(
"final handoff capture requires an explicit capture adapter"
)
hidden_states, _ = handoff_norm.finalize(
handoff=hidden_states, residual=residual, gamma=layernorm.gemma_weight
)
return hidden_states
if skip_empty and hidden_states.shape[0] == 0:
return hidden_states
return access.norm_output(
hidden_states, residual, layernorm, capture_output, **read_kwargs
)
+2 -2
View File
@@ -25,7 +25,7 @@ Everything SP lives here so the feature stays decoupled from model code and from
- which models opt in (the Qwen3-dense allowlist) and config validation,
- the per-forward ``sp_active`` flag (a ForwardFlags bool) read at depth by the
participant linears and the ``LayerCommunicator``,
participant linears and the layer-boundary batch selector,
- the entry-scatter / exit-gather collectives, and
- the fused matmul + collective fast-paths for the participant linears.
@@ -47,7 +47,7 @@ from sglang.srt.runtime_context import (
from sglang.srt.utils.common import ceil_align
# Architectures whose decoder layers route attention/MLP through
# ``LayerCommunicator`` with the standard participant linears, and for which SP
# layer boundaries with the standard participant linears, and for which SP
# has been validated. Other models reject --enable-layernorm-sp at construction.
# The mechanism is generic; extend the allowlist as families are validated.
SP_SUPPORTED_ARCHITECTURES = frozenset({"Qwen3ForCausalLM"})
+6 -12
View File
@@ -216,27 +216,21 @@ class DeepseekModelNextN(nn.Module):
else:
hidden_states = self.eh_proj(eh_input)
residual = residual_batch.start(forward_batch)
residual_batch.start(forward_batch)
index_topk_share = IndexTopKShareState.from_mtp_carry(forward_batch)
with get_global_expert_distribution_recorder().disable_this_region():
hidden_states, residual, topk_indices = self.decoder(
(hidden_states, topk_indices) = self.decoder(
positions,
hidden_states,
forward_batch,
residual,
zero_allocator,
prev_topk_indices=index_topk_share.topk_indices,
)
hidden_states, residual = (
self.decoder.layer_communicator.finish_layer_stack(
hidden_states, residual, forward_batch
)
)
hidden_states = residual_batch.finish(hidden_states, forward_batch)
if not forward_batch.forward_mode.is_idle():
if residual is not None:
hidden_states, _ = self.shared_head.norm(hidden_states, residual)
else:
hidden_states = self.shared_head.norm(hidden_states)
hidden_states = residual_batch.norm(
hidden_states, forward_batch, self.shared_head.norm
)
index_topk_share.update(topk_indices)
index_topk_share.publish()
+73 -80
View File
@@ -40,7 +40,7 @@ from sglang.kernels.ops.quantization.fp8_kernel import (
from sglang.srt.batch_overlap.single_batch_overlap import SboFlags, compute_overlap_args
from sglang.srt.batch_overlap.two_batch_overlap import (
MaybeTboDeepEPDispatcher,
model_forward_maybe_tbo,
model_forward_stages,
)
from sglang.srt.configs.model_config import (
compute_mla_mscale_scaling,
@@ -68,14 +68,16 @@ from sglang.srt.layers.aux_hidden_states import (
AuxHiddenStatePacker,
)
from sglang.srt.layers.communicator import (
LayerCommunicator,
LayerFacts,
declare_attn,
declare_ffn,
enable_moe_dense_fully_dp,
get_attn_tp_context,
make_stages,
)
from sglang.srt.layers.communicator.adapters.context_parallel import (
maybe_prefetch_next_full_attention_kv,
)
from sglang.srt.layers.communicator.residual import access as residual_access
from sglang.srt.layers.communicator.residual import batch as residual_batch
from sglang.srt.layers.cp.cp_decode_attn_tp import get_cp_decode_attn_tp_ctx
from sglang.srt.layers.dcp.planner import (
@@ -2578,13 +2580,12 @@ class DeepseekV2DecoderLayer(nn.Module):
is_previous_layer_sparse = self._is_layer_sparse(layer_id - 1, is_nextn=False)
is_next_layer_sparse = self._is_layer_sparse(layer_id + 1, is_nextn=False)
self.layer_facts = LayerFacts.init_new(
layer_id=layer_id,
num_layers=1 if is_nextn else config.num_hidden_layers,
is_layer_sparse=self.is_layer_sparse,
is_previous_layer_sparse=is_previous_layer_sparse,
is_next_layer_sparse=is_next_layer_sparse,
)
self._stage_enters_stack = (layer_id) == 0
self._stage_terminal = (layer_id) == (
1 if is_nextn else config.num_hidden_layers
) - 1
self._stage_previous_sparse = is_previous_layer_sparse
self._stage_next_sparse = is_next_layer_sparse
if self.is_layer_sparse:
self.mlp = DeepseekV2MoE(
@@ -2618,19 +2619,19 @@ class DeepseekV2DecoderLayer(nn.Module):
self._gfx95_quant_format = self._detect_gfx95_quant_format()
self.layer_communicator = self._build_layer_communicator(
self.attn_stage, self.ffn_stage = self._build_stages(
input_layernorm=self.input_layernorm,
post_attention_layernorm=self.post_attention_layernorm,
qkv_latent_func=self.self_attn.prepare_qkv_latent,
)
def _build_layer_communicator(
def _build_stages(
self,
*,
input_layernorm: nn.Module,
post_attention_layernorm: nn.Module,
qkv_latent_func: Optional[Callable],
allow_deferred_ffn_reduction: bool = True,
output=None,
):
"""The communicator for this layer's norms; it chooses its boundary
steps from them at construction."""
@@ -2644,15 +2645,29 @@ class DeepseekV2DecoderLayer(nn.Module):
from sglang.srt.layers.communicator.fusions.cutedsl import CuteDSLFusion
fusions = CuteDSLFusion()
return LayerCommunicator(
layer_facts=self.layer_facts,
input_layernorm=input_layernorm,
post_attention_layernorm=post_attention_layernorm,
allow_reduce_scatter=True,
qkv_latent_func=qkv_latent_func,
allow_deferred_ffn_reduction=allow_deferred_ffn_reduction,
fusions=fusions,
attn_stage, ffn_stage = make_stages(
(
declare_attn(),
input_layernorm,
{"qkv_latent_func": qkv_latent_func, "fusions": fusions},
),
(
declare_ffn(
sparse=self.is_layer_sparse,
next_sparse=self._stage_next_sparse,
output=output,
),
post_attention_layernorm,
{"fusions": fusions},
),
previous=declare_ffn(
sparse=self._stage_previous_sparse, next_sparse=self.is_layer_sparse
)
if not self._stage_enters_stack
else None,
terminal=self._stage_terminal,
)
return attn_stage, ffn_stage
def _detect_gfx95_quant_format(self) -> str:
if not _is_gfx95_supported:
@@ -2698,7 +2713,6 @@ class DeepseekV2DecoderLayer(nn.Module):
positions: torch.Tensor,
hidden_states: torch.Tensor,
forward_batch: ForwardBatch,
residual: Optional[torch.Tensor],
zero_allocator: BumpAllocator,
gemm_output_zero_allocator: BumpAllocator = None,
llama_4_scaling: Optional[torch.Tensor] = None,
@@ -2706,15 +2720,12 @@ class DeepseekV2DecoderLayer(nn.Module):
captured_last_layer_outputs: Optional[AuxHiddenStateAccumulator] = None,
next_full_attention_layer_id: Optional[int] = None,
) -> torch.Tensor:
hidden_states_orig = self.layer_communicator.buffer(hidden_states)
hidden_states, residual = (
self.layer_communicator.prepare_attn_and_capture_last_layer_outputs(
hidden_states,
residual,
forward_batch,
captured_last_layer_outputs=captured_last_layer_outputs,
quant_format=self._resolve_gfx95_quant_format(),
)
hidden_states_orig = residual_access.buffer(hidden_states)
hidden_states = self.attn_stage.prepare(
hidden_states,
forward_batch,
captured_last_layer_outputs=captured_last_layer_outputs,
quant_format=self._resolve_gfx95_quant_format(),
)
with self.self_attn.maybe_use_decode_attn_tp(forward_batch):
@@ -2725,7 +2736,7 @@ class DeepseekV2DecoderLayer(nn.Module):
zero_allocator=zero_allocator,
llama_4_scaling=llama_4_scaling,
input_on_attention_tp_slices=(
self.layer_communicator.input_on_attention_tp_slices
self.attn_stage.input_on_attention_tp_slices
),
prev_topk_indices=prev_topk_indices,
)
@@ -2739,9 +2750,8 @@ class DeepseekV2DecoderLayer(nn.Module):
forward_batch, next_full_attention_layer_id
)
hidden_states, residual = self.layer_communicator.prepare_mlp(
hidden_states, residual, forward_batch
)
hidden_states = self.attn_stage.finish(hidden_states, forward_batch)
hidden_states = self.ffn_stage.prepare(hidden_states, forward_batch)
if isinstance(self.mlp, DeepseekV2MLP):
gemm_output_zero_allocator = None
@@ -2759,15 +2769,15 @@ class DeepseekV2DecoderLayer(nn.Module):
else:
_mlp_ctx = nullcontext()
with self.layer_communicator.ffn_exit(forward_batch) as ffn_exit, _mlp_ctx:
with self.ffn_stage.exit(forward_batch) as ffn_exit, _mlp_ctx:
hidden_states = self.mlp(
hidden_states,
forward_batch,
gemm_output_zero_allocator,
)
hidden_states, residual = ffn_exit.finish(hidden_states, residual)
hidden_states = ffn_exit.finish(hidden_states)
return hidden_states, residual, topk_indices
return (hidden_states, topk_indices)
def op_comm_prepare_attn(
self,
@@ -2775,12 +2785,11 @@ class DeepseekV2DecoderLayer(nn.Module):
positions: torch.Tensor,
hidden_states: torch.Tensor,
forward_batch: ForwardBatch,
residual: Optional[torch.Tensor],
zero_allocator: BumpAllocator,
tbo_subbatch_index: Optional[int] = None,
):
state.hidden_states_after_comm_pre_attn, state.residual_after_input_ln = (
self.layer_communicator.prepare_attn(hidden_states, residual, forward_batch)
state.hidden_states_after_comm_pre_attn = self.attn_stage.prepare(
hidden_states, forward_batch
)
if get_moe_a2a_backend().is_mori():
state.num_tokens = hidden_states.shape[0]
@@ -2794,25 +2803,21 @@ class DeepseekV2DecoderLayer(nn.Module):
)
def op_comm_prepare_mlp(self, state):
state.hidden_states_mlp_input, state.residual_after_comm_pre_mlp = (
self.layer_communicator.prepare_mlp(
state.pop("hidden_states_after_attn"),
state.pop("residual_after_input_ln"),
state.forward_batch,
)
hidden_states = self.attn_stage.finish(
state.pop("hidden_states_after_attn"), state.forward_batch
)
state.hidden_states_mlp_input = self.ffn_stage.prepare(
hidden_states, state.forward_batch
)
def op_comm_postprocess_layer(self, state):
hidden_states, residual = self.layer_communicator.postprocess_layer(
state.pop("hidden_states_mlp_output"),
state.pop("residual_after_comm_pre_mlp"),
state.forward_batch,
hidden_states = self.ffn_stage.postprocess(
state.pop("hidden_states_mlp_output"), state.forward_batch
)
output = dict(
positions=state.positions,
hidden_states=hidden_states,
residual=residual,
forward_batch=state.forward_batch,
zero_allocator=state.zero_allocator,
tbo_subbatch_index=state.tbo_subbatch_index,
@@ -3027,13 +3032,13 @@ class DeepseekV2Model(nn.Module):
hidden_states = self.embed_tokens(input_ids)
else:
hidden_states = input_embeds
residual = residual_batch.start(forward_batch)
residual_batch.start(forward_batch)
initial_topk_indices = None
else:
assert pp_proxy_tensors is not None
hidden_states, residual = self.layers[
self.start_layer
].layer_communicator.from_pp(pp_proxy_tensors, forward_batch)
hidden_states = self.layers[self.start_layer].attn_stage.from_pp(
pp_proxy_tensors, forward_batch
)
initial_topk_indices = pp_proxy_tensors.tensors.get("topk_indices")
index_topk_share = IndexTopKShareState(forward_batch, initial_topk_indices)
if not self.pp_group.is_first_rank:
@@ -3103,18 +3108,17 @@ class DeepseekV2Model(nn.Module):
)
with ctx:
layer = self.layers[i]
hidden_states, residual, topk_indices = layer(
(hidden_states, topk_indices) = layer(
positions,
hidden_states,
forward_batch,
residual,
zero_allocator,
gemm_output_zero_allocator,
llama_4_scaling,
prev_topk_indices=index_topk_share.topk_indices,
captured_last_layer_outputs=(
aux_hidden_states if i in self.layers_to_capture else None
),
captured_last_layer_outputs=aux_hidden_states
if i in self.layers_to_capture
else None,
next_full_attention_layer_id=self.next_full_attention_layer_id.get(
i
),
@@ -3122,28 +3126,17 @@ class DeepseekV2Model(nn.Module):
index_topk_share.update(topk_indices)
if normal_end_layer != self.end_layer:
hidden_states, residual = model_forward_maybe_tbo(
hidden_states = model_forward_stages(
layers=self.layers[normal_end_layer : self.end_layer],
enable_tbo=True,
positions=positions,
forward_batch=forward_batch,
hidden_states=hidden_states,
residual=residual,
zero_allocator=zero_allocator,
)
last_layer = self.layers[self.end_layer - 1]
hidden_states, residual = last_layer.layer_communicator.finish_layer_stack(
hidden_states,
residual,
forward_batch,
preserve_declared=not self.pp_group.is_last_rank,
)
if not self.pp_group.is_last_rank:
proxy_tensors = {
"hidden_states": hidden_states,
"residual": residual,
}
proxy_tensors = residual_batch.to_pp(hidden_states, forward_batch)
if (
self.use_dsa
and dsa_forward_uses_topk
@@ -3164,14 +3157,14 @@ class DeepseekV2Model(nn.Module):
topk_indices = hidden_states.new_empty(
(0, get_dsa_index_topk(self.config)), dtype=torch.int32
)
proxy_tensors["topk_indices"] = topk_indices
return PPProxyTensors(proxy_tensors)
proxy_tensors.tensors["topk_indices"] = topk_indices
return proxy_tensors
else:
hidden_states = residual_batch.finish(hidden_states, forward_batch)
if not forward_batch.forward_mode.is_idle():
if residual is None:
hidden_states = self.norm(hidden_states)
else:
hidden_states, _ = self.norm(hidden_states, residual)
hidden_states = residual_batch.norm(
hidden_states, forward_batch, self.norm
)
if len(aux_hidden_states) == 0:
return hidden_states
@@ -42,7 +42,7 @@ from transformers import PretrainedConfig
from sglang.kernels.ops.quantization.fp8_kernel import is_fp8_fnuz
from sglang.srt.batch_overlap.two_batch_overlap import (
MaybeTboDeepEPDispatcher,
model_forward_maybe_tbo,
model_forward_stages,
)
from sglang.srt.configs.dots3 import Dots3Config
from sglang.srt.distributed import (
@@ -58,9 +58,10 @@ from sglang.srt.layers import deep_gemm_wrapper
from sglang.srt.layers.activation import SiluAndMul
from sglang.srt.layers.attention.dsa.dsa_indexer import Indexer
from sglang.srt.layers.communicator import (
LayerCommunicator,
LayerFacts,
declare_attn,
declare_ffn,
enable_moe_dense_fully_dp,
make_stages,
)
from sglang.srt.layers.communicator.residual import batch as residual_batch
from sglang.srt.layers.dp_attention import is_dp_attention_enabled
@@ -1542,14 +1543,6 @@ class Dots3DecoderLayer(nn.Module):
is_previous_layer_sparse = self._is_layer_sparse(layer_id - 1, is_nextn=False)
is_next_layer_sparse = self._is_layer_sparse(layer_id + 1, is_nextn=False)
self.layer_facts = LayerFacts.init_new(
layer_id=layer_id,
num_layers=1 if is_nextn else config.num_hidden_layers,
is_layer_sparse=self.is_layer_sparse,
is_previous_layer_sparse=is_previous_layer_sparse,
is_next_layer_sparse=is_next_layer_sparse,
)
if self.is_layer_sparse:
self.mlp = Dots3MoE(
config=config,
@@ -1579,11 +1572,21 @@ class Dots3DecoderLayer(nn.Module):
config.hidden_size, eps=config.rms_norm_eps
)
self.layer_communicator = LayerCommunicator(
layer_facts=self.layer_facts,
input_layernorm=self.input_layernorm,
post_attention_layernorm=self.post_attention_layernorm,
allow_reduce_scatter=True,
self.attn_stage, self.ffn_stage = make_stages(
(declare_attn(), self.input_layernorm),
(
declare_ffn(
sparse=self.is_layer_sparse,
next_sparse=is_next_layer_sparse,
),
self.post_attention_layernorm,
),
previous=declare_ffn(
sparse=is_previous_layer_sparse, next_sparse=self.is_layer_sparse
)
if layer_id != 0
else None,
terminal=layer_id == (1 if is_nextn else config.num_hidden_layers) - 1,
)
def _is_layer_sparse(self, layer_id: int, is_nextn: bool) -> bool:
@@ -1601,13 +1604,10 @@ class Dots3DecoderLayer(nn.Module):
positions: torch.Tensor,
hidden_states: torch.Tensor,
forward_batch: ForwardBatch,
residual: Optional[torch.Tensor],
zero_allocator: BumpAllocator,
) -> torch.Tensor:
hidden_states, residual = self.layer_communicator.prepare_attn(
hidden_states, residual, forward_batch
)
hidden_states = self.attn_stage.prepare(hidden_states, forward_batch)
hidden_states = self.self_attn(
positions=positions,
@@ -1616,18 +1616,17 @@ class Dots3DecoderLayer(nn.Module):
zero_allocator=zero_allocator,
)
hidden_states, residual = self.layer_communicator.prepare_mlp(
hidden_states, residual, forward_batch
)
hidden_states = self.attn_stage.finish(hidden_states, forward_batch)
hidden_states = self.ffn_stage.prepare(hidden_states, forward_batch)
with self.layer_communicator.ffn_exit(forward_batch) as ffn_exit:
with self.ffn_stage.exit(forward_batch) as ffn_exit:
hidden_states = self.mlp(
hidden_states,
forward_batch,
ffn_exit.fuse_mlp_allreduce,
ffn_exit.mlp_reduce_scatter,
)
return ffn_exit.finish(hidden_states, residual)
return ffn_exit.finish(hidden_states)
def op_comm_prepare_attn(
self,
@@ -1635,12 +1634,11 @@ class Dots3DecoderLayer(nn.Module):
positions: torch.Tensor,
hidden_states: torch.Tensor,
forward_batch: ForwardBatch,
residual: Optional[torch.Tensor],
zero_allocator: BumpAllocator,
tbo_subbatch_index: Optional[int] = None,
):
state.hidden_states_after_comm_pre_attn, state.residual_after_input_ln = (
self.layer_communicator.prepare_attn(hidden_states, residual, forward_batch)
state.hidden_states_after_comm_pre_attn = self.attn_stage.prepare(
hidden_states, forward_batch
)
state.update(
dict(
@@ -1652,12 +1650,11 @@ class Dots3DecoderLayer(nn.Module):
)
def op_comm_prepare_mlp(self, state):
state.hidden_states_mlp_input, state.residual_after_comm_pre_mlp = (
self.layer_communicator.prepare_mlp(
state.pop("hidden_states_after_attn"),
state.pop("residual_after_input_ln"),
state.forward_batch,
)
hidden_states = self.attn_stage.finish(
state.pop("hidden_states_after_attn"), state.forward_batch
)
state.hidden_states_mlp_input = self.ffn_stage.prepare(
hidden_states, state.forward_batch
)
def op_mlp(self, state):
@@ -1674,16 +1671,13 @@ class Dots3DecoderLayer(nn.Module):
state.hidden_states_mlp_output = hidden_states
def op_comm_postprocess_layer(self, state):
hidden_states, residual = self.layer_communicator.postprocess_layer(
state.pop("hidden_states_mlp_output"),
state.pop("residual_after_comm_pre_mlp"),
state.forward_batch,
hidden_states = self.ffn_stage.postprocess(
state.pop("hidden_states_mlp_output"), state.forward_batch
)
output = dict(
positions=state.positions,
hidden_states=hidden_states,
residual=residual,
forward_batch=state.forward_batch,
zero_allocator=state.zero_allocator,
tbo_subbatch_index=state.tbo_subbatch_index,
@@ -1766,12 +1760,12 @@ class Dots3Model(nn.Module):
hidden_states = self.embed_tokens(input_ids)
else:
hidden_states = input_embeds
residual = residual_batch.start(forward_batch)
residual_batch.start(forward_batch)
else:
assert pp_proxy_tensors is not None
hidden_states, residual = self.layers[
self.start_layer
].layer_communicator.from_pp(pp_proxy_tensors, forward_batch)
hidden_states = self.layers[self.start_layer].attn_stage.from_pp(
pp_proxy_tensors, forward_batch
)
normal_start_layer = self.start_layer
normal_end_layer = self.end_layer
@@ -1787,38 +1781,28 @@ class Dots3Model(nn.Module):
for i in range(normal_start_layer, normal_end_layer):
with get_global_expert_distribution_recorder().with_current_layer(i):
layer = self.layers[i]
hidden_states, residual = layer(
positions, hidden_states, forward_batch, residual, zero_allocator
hidden_states = layer(
positions, hidden_states, forward_batch, zero_allocator
)
if normal_end_layer != self.end_layer:
hidden_states, residual = model_forward_maybe_tbo(
hidden_states = model_forward_stages(
layers=self.layers[normal_end_layer : self.end_layer],
enable_tbo=True,
positions=positions,
forward_batch=forward_batch,
hidden_states=hidden_states,
residual=residual,
zero_allocator=zero_allocator,
)
last_layer = self.layers[self.end_layer - 1]
hidden_states, residual = last_layer.layer_communicator.finish_layer_stack(
hidden_states, residual, forward_batch
)
hidden_states = residual_batch.finish(hidden_states, forward_batch)
if not self.pp_group.is_last_rank:
return PPProxyTensors(
{
"hidden_states": hidden_states,
"residual": residual,
}
)
return residual_batch.to_pp(hidden_states, forward_batch)
else:
if not forward_batch.forward_mode.is_idle():
if residual is None:
hidden_states = self.norm(hidden_states)
else:
hidden_states, _ = self.norm(hidden_states, residual)
hidden_states = residual_batch.norm(
hidden_states, forward_batch, self.norm
)
return hidden_states
+7 -10
View File
@@ -114,20 +114,17 @@ class Dot3NoteModelNextN(nn.Module):
)
)
residual = residual_batch.start(forward_batch)
residual_batch.start(forward_batch)
with get_global_expert_distribution_recorder().disable_this_region():
hidden_states, residual = head.decoder(
positions, hidden_states, forward_batch, residual, zero_allocator
hidden_states = head.decoder(
positions, hidden_states, forward_batch, zero_allocator
)
hidden_states, residual = head.decoder.layer_communicator.finish_layer_stack(
hidden_states, residual, forward_batch
)
hidden_states = residual_batch.finish(hidden_states, forward_batch)
if not forward_batch.forward_mode.is_idle():
if residual is None:
hidden_states = head.shared_head.norm(hidden_states)
else:
hidden_states, _ = head.shared_head.norm(hidden_states, residual)
hidden_states = residual_batch.norm(
hidden_states, forward_batch, head.shared_head.norm
)
return hidden_states
def _embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor:
+1 -1
View File
@@ -195,7 +195,7 @@ class FalconH1HybridAttentionDecoderLayer(nn.Module):
rms_norm_eps=config.rms_norm_eps,
activation=config.hidden_act,
use_rms_norm=config.mamba_rms_norm,
# Like the attention's o_proj: prepare_mlp completes the sum of the
# Like the attention's o_proj: the FFN input completes the sum of the
# two mixers' partial outputs.
reduce_results=False,
prefix=f"{prefix}.mixer",
+31 -45
View File
@@ -24,6 +24,7 @@ from transformers import PretrainedConfig
import sglang.srt.models.deepseek_v2 as deepseek_v2
from sglang.srt.configs.gigachat35 import GigaChat35Config
from sglang.srt.layers.communicator import get_attn_tp_context
from sglang.srt.layers.communicator.output import OutputTransform
from sglang.srt.layers.communicator.residual import batch as residual_batch
from sglang.srt.layers.layernorm import GemmaRMSNorm, RMSNorm
from sglang.srt.layers.linear import ColumnParallelLinear
@@ -133,7 +134,7 @@ class GigaChat35PassthroughNorm(nn.Module):
Used as the ``input_layernorm`` slot when a layer has no pre-norm
(``layernorm_type="post"``); it only folds the residual so the
LayerCommunicator's prepare-step bookkeeping still works.
attention boundary receives the folded residual.
"""
def forward(
@@ -153,10 +154,10 @@ class GigaChat35PassthroughNorm(nn.Module):
class GigaChat35MlpPrepNorm(nn.Module):
"""Pre-MLP norm wrapper that folds the post-attention (sandwich) norm.
Replaces the LayerCommunicator's ``post_attention_layernorm`` slot so the
Serves as the FFN boundary's norm so the
optional ``post_self_attn_layernorm`` (applied to the attention output,
before the residual add) and the pre-MLP ``post_attention_layernorm`` are
both threaded through ``prepare_mlp`` -- matching the per-layer math of the
both run during FFN input preparation, matching the per-layer math of the
``pre_post`` sandwich exactly.
"""
@@ -389,7 +390,7 @@ class GigaChat35DecoderLayer(deepseek_v2.DeepseekV2DecoderLayer):
)
if hasattr(self.self_attn, "out_proj"):
self.self_attn.out_proj.reduce_results = False
self.layer_communicator.qkv_latent_func = None
qkv_latent_func = None
else:
self.self_attn = GigaChat35AttentionMLA(
config=config,
@@ -409,7 +410,7 @@ class GigaChat35DecoderLayer(deepseek_v2.DeepseekV2DecoderLayer):
prefix=add_prefix("self_attn", prefix),
alt_stream=alt_stream,
)
self.layer_communicator.qkv_latent_func = self.self_attn.prepare_qkv_latent
qkv_latent_func = self.self_attn.prepare_qkv_latent
self.is_sparse = self.is_layer_sparse
@@ -440,11 +441,17 @@ class GigaChat35DecoderLayer(deepseek_v2.DeepseekV2DecoderLayer):
post_layernorm=self.post_self_attn_layernorm,
)
# The communicator chooses its fused steps from its norms when built.
self.layer_communicator = self._build_layer_communicator(
self.attn_stage, self.ffn_stage = self._build_stages(
input_layernorm=attn_prepare_layernorm,
post_attention_layernorm=mlp_prepare_layernorm,
qkv_latent_func=self.layer_communicator.qkv_latent_func,
allow_deferred_ffn_reduction=False,
qkv_latent_func=qkv_latent_func,
output=(
OutputTransform(
self.post_feedforward_layernorm, before_reduce_scatter=True
)
if self.post_feedforward_layernorm is not None
else None
),
)
def _is_layer_sparse(self, layer_id: int, is_nextn: bool) -> bool:
@@ -461,13 +468,10 @@ class GigaChat35DecoderLayer(deepseek_v2.DeepseekV2DecoderLayer):
positions: torch.Tensor,
hidden_states: torch.Tensor,
forward_batch: ForwardBatch,
residual: Optional[torch.Tensor],
zero_allocator: BumpAllocator,
**kwargs,
) -> tuple[torch.Tensor, torch.Tensor]:
hidden_states, residual = self.layer_communicator.prepare_attn(
hidden_states, residual, forward_batch
)
) -> torch.Tensor:
hidden_states = self.attn_stage.prepare(hidden_states, forward_batch)
if self.use_linear_attn:
hidden_states = self.self_attn(
@@ -481,26 +485,22 @@ class GigaChat35DecoderLayer(deepseek_v2.DeepseekV2DecoderLayer):
forward_batch=forward_batch,
zero_allocator=zero_allocator,
input_on_attention_tp_slices=(
self.layer_communicator.input_on_attention_tp_slices
self.attn_stage.input_on_attention_tp_slices
),
)
get_attn_tp_context().clear_attn_inputs()
hidden_states, residual = self.layer_communicator.prepare_mlp(
hidden_states, residual, forward_batch
)
hidden_states = self.attn_stage.finish(hidden_states, forward_batch)
hidden_states = self.ffn_stage.prepare(hidden_states, forward_batch)
# Unlike deepseek_v2, no moe_output_buffer_ctx here: non-inplace MoE
# runners then allocate their output per forward instead of recycling
# the layer-input buffer. The default (inplace) runners are unaffected.
with self.layer_communicator.ffn_exit(forward_batch) as ffn_exit:
with self.ffn_stage.exit(forward_batch) as ffn_exit:
hidden_states = self.mlp(hidden_states, forward_batch)
if self.post_feedforward_layernorm is not None:
hidden_states = self.post_feedforward_layernorm(hidden_states)
hidden_states, residual = ffn_exit.finish(hidden_states, residual)
return hidden_states, residual
hidden_states = ffn_exit.finish(hidden_states)
return hidden_states
class GigaChat35Model(nn.Module):
@@ -559,12 +559,12 @@ class GigaChat35Model(nn.Module):
hidden_states = inputs_embeds
else:
hidden_states = self.embed_tokens(input_ids)
residual = residual_batch.start(forward_batch)
residual_batch.start(forward_batch)
else:
assert pp_proxy_tensors is not None
hidden_states, residual = self.layers[
self.start_layer
].layer_communicator.from_pp(pp_proxy_tensors, forward_batch)
hidden_states = self.layers[self.start_layer].attn_stage.from_pp(
pp_proxy_tensors, forward_batch
)
total_num_layers = self.end_layer - self.start_layer
zero_allocator = BumpAllocator(
@@ -574,33 +574,19 @@ class GigaChat35Model(nn.Module):
)
for i in range(self.start_layer, self.end_layer):
hidden_states, residual = self.layers[i](
hidden_states = self.layers[i](
positions=positions,
hidden_states=hidden_states,
forward_batch=forward_batch,
residual=residual,
zero_allocator=zero_allocator,
)
hidden_states, residual = self.layers[
self.end_layer - 1
].layer_communicator.finish_layer_stack(
hidden_states,
residual,
forward_batch,
preserve_declared=not self.pp_group.is_last_rank,
)
if not self.pp_group.is_last_rank:
return PPProxyTensors(
{"hidden_states": hidden_states, "residual": residual}
)
return residual_batch.to_pp(hidden_states, forward_batch)
hidden_states = residual_batch.finish(hidden_states, forward_batch)
if hidden_states.shape[0] != 0:
if residual is None:
hidden_states = self.norm(hidden_states)
else:
hidden_states, _ = self.norm(hidden_states, residual)
hidden_states = residual_batch.norm(hidden_states, forward_batch, self.norm)
return hidden_states
+9 -13
View File
@@ -82,7 +82,7 @@ class GigaChat35ModelNextN(nn.Module):
positions: torch.Tensor,
forward_batch: ForwardBatch,
input_embeds: Optional[torch.Tensor] = None,
) -> tuple[torch.Tensor, Optional[torch.Tensor]]:
) -> torch.Tensor:
zero_allocator = BumpAllocator(
buffer_size=2,
dtype=torch.float32,
@@ -107,28 +107,24 @@ class GigaChat35ModelNextN(nn.Module):
)
)
residual = residual_batch.start(forward_batch)
hidden_states, residual = self.decoder(
residual_batch.start(forward_batch)
hidden_states = self.decoder(
positions=positions,
hidden_states=hidden_states,
forward_batch=forward_batch,
residual=residual,
zero_allocator=zero_allocator,
)
hidden_states, residual = self.decoder.layer_communicator.finish_layer_stack(
hidden_states, residual, forward_batch
)
hidden_states = residual_batch.finish(hidden_states, forward_batch)
hidden_states_before_norm = None
if not forward_batch.forward_mode.is_idle():
hidden_states_before_norm = self.decoder.layer_communicator.snapshot(
hidden_states, residual
hidden_states_before_norm = residual_batch.snapshot(
hidden_states, forward_batch
)
hidden_states = residual_batch.norm(
hidden_states, forward_batch, self.shared_head.norm
)
if residual is not None:
hidden_states, _ = self.shared_head.norm(hidden_states, residual)
else:
hidden_states = self.shared_head.norm(hidden_states)
return hidden_states, hidden_states_before_norm
+49 -75
View File
@@ -26,7 +26,7 @@ from transformers import PretrainedConfig
from sglang.kernels.ops.quantization.fp8_kernel import is_fp8_fnuz
from sglang.srt.batch_overlap.single_batch_overlap import SboFlags
from sglang.srt.batch_overlap.two_batch_overlap import model_forward_maybe_tbo
from sglang.srt.batch_overlap.two_batch_overlap import model_forward_stages
from sglang.srt.distributed import (
get_pp_indices,
)
@@ -40,9 +40,10 @@ from sglang.srt.eplb.expert_location_dispatch import ExpertLocationDispatchInfo
from sglang.srt.layers.activation import SiluAndMul
from sglang.srt.layers.aux_hidden_states import AuxHiddenStateList
from sglang.srt.layers.communicator import (
LayerCommunicator,
LayerFacts,
declare_attn,
declare_ffn,
enable_moe_dense_fully_dp,
make_stages,
)
from sglang.srt.layers.communicator.residual import batch as residual_batch
from sglang.srt.layers.dp_attention import (
@@ -837,14 +838,6 @@ class Glm4MoeDecoderLayer(nn.Module):
is_previous_layer_sparse = self._is_layer_sparse(layer_id - 1, is_nextn=False)
is_next_layer_sparse = self._is_layer_sparse(layer_id + 1, is_nextn=False)
self.layer_facts = LayerFacts.init_new(
layer_id=layer_id,
num_layers=1 if is_nextn else config.num_hidden_layers,
is_layer_sparse=self.is_layer_sparse,
is_previous_layer_sparse=is_previous_layer_sparse,
is_next_layer_sparse=is_next_layer_sparse,
)
if self.is_layer_sparse:
self.mlp = Glm4MoeSparseMoeBlock(
config=config,
@@ -873,11 +866,21 @@ class Glm4MoeDecoderLayer(nn.Module):
config.hidden_size, eps=config.rms_norm_eps
)
self.layer_communicator = LayerCommunicator(
layer_facts=self.layer_facts,
input_layernorm=self.input_layernorm,
post_attention_layernorm=self.post_attention_layernorm,
allow_reduce_scatter=True,
self.attn_stage, self.ffn_stage = make_stages(
(declare_attn(), self.input_layernorm),
(
declare_ffn(
sparse=self.is_layer_sparse,
next_sparse=is_next_layer_sparse,
),
self.post_attention_layernorm,
),
previous=declare_ffn(
sparse=is_previous_layer_sparse, next_sparse=self.is_layer_sparse
)
if layer_id != 0
else None,
terminal=layer_id == (1 if is_nextn else config.num_hidden_layers) - 1,
)
# Detect if QKV uses aiter FP8 per-token quant so we can fuse
@@ -936,18 +939,14 @@ class Glm4MoeDecoderLayer(nn.Module):
positions: torch.Tensor,
hidden_states: torch.Tensor,
forward_batch: ForwardBatch,
residual: Optional[torch.Tensor],
capture_output=None,
) -> torch.Tensor:
hidden_states, residual = (
self.layer_communicator.prepare_attn_and_capture_last_layer_outputs(
hidden_states,
residual,
forward_batch,
quant_format=self.attn_quant_format,
capture_output=capture_output,
)
hidden_states = self.attn_stage.prepare(
hidden_states,
forward_batch,
quant_format=self.attn_quant_format,
capture_output=capture_output,
)
hidden_states = self.self_attn(
@@ -956,15 +955,14 @@ class Glm4MoeDecoderLayer(nn.Module):
forward_batch=forward_batch,
)
hidden_states, residual = self.layer_communicator.prepare_mlp(
hidden_states, residual, forward_batch
)
hidden_states = self.attn_stage.finish(hidden_states, forward_batch)
hidden_states = self.ffn_stage.prepare(hidden_states, forward_batch)
with self.layer_communicator.ffn_exit(forward_batch) as ffn_exit:
with self.ffn_stage.exit(forward_batch) as ffn_exit:
hidden_states = self.mlp(hidden_states, forward_batch)
hidden_states, residual = ffn_exit.finish(hidden_states, residual)
hidden_states = ffn_exit.finish(hidden_states)
return hidden_states, residual
return hidden_states
def op_comm_prepare_attn(
self,
@@ -972,16 +970,10 @@ class Glm4MoeDecoderLayer(nn.Module):
positions: torch.Tensor,
hidden_states: torch.Tensor,
forward_batch: ForwardBatch,
residual: Optional[torch.Tensor],
tbo_subbatch_index: Optional[int] = None,
):
state.hidden_states_after_comm_pre_attn, state.residual_after_input_ln = (
self.layer_communicator.prepare_attn(
hidden_states,
residual,
forward_batch,
quant_format=self.attn_quant_format,
)
state.hidden_states_after_comm_pre_attn = self.attn_stage.prepare(
hidden_states, forward_batch
)
state.update(
dict(
@@ -992,25 +984,21 @@ class Glm4MoeDecoderLayer(nn.Module):
)
def op_comm_prepare_mlp(self, state):
state.hidden_states_mlp_input, state.residual_after_comm_pre_mlp = (
self.layer_communicator.prepare_mlp(
state.pop("hidden_states_after_attn"),
state.pop("residual_after_input_ln"),
state.forward_batch,
)
hidden_states = self.attn_stage.finish(
state.pop("hidden_states_after_attn"), state.forward_batch
)
state.hidden_states_mlp_input = self.ffn_stage.prepare(
hidden_states, state.forward_batch
)
def op_comm_postprocess_layer(self, state):
hidden_states, residual = self.layer_communicator.postprocess_layer(
state.pop("hidden_states_mlp_output"),
state.pop("residual_after_comm_pre_mlp"),
state.forward_batch,
hidden_states = self.ffn_stage.postprocess(
state.pop("hidden_states_mlp_output"), state.forward_batch
)
output = dict(
positions=state.positions,
hidden_states=hidden_states,
residual=residual,
forward_batch=state.forward_batch,
tbo_subbatch_index=state.tbo_subbatch_index,
)
@@ -1090,12 +1078,12 @@ class Glm4MoeModel(nn.Module):
hidden_states = self.embed_tokens(input_ids)
else:
hidden_states = input_embeds
residual = residual_batch.start(forward_batch)
residual_batch.start(forward_batch)
else:
assert pp_proxy_tensors is not None
hidden_states, residual = self.layers[
self.start_layer
].layer_communicator.from_pp(pp_proxy_tensors, forward_batch)
hidden_states = self.layers[self.start_layer].attn_stage.from_pp(
pp_proxy_tensors, forward_batch
)
normal_start_layer = self.start_layer
normal_end_layer = self.end_layer
@@ -1112,46 +1100,32 @@ class Glm4MoeModel(nn.Module):
for i in range(normal_start_layer, normal_end_layer):
with get_global_expert_distribution_recorder().with_current_layer(i):
layer = self.layers[i]
hidden_states, residual = layer(
hidden_states = layer(
positions,
hidden_states,
forward_batch,
residual,
capture_output=aux_hidden_states.capture
if i in self.layers_to_capture
else None,
)
if normal_end_layer != self.end_layer:
hidden_states, residual = model_forward_maybe_tbo(
hidden_states = model_forward_stages(
layers=self.layers[normal_end_layer : self.end_layer],
enable_tbo=True,
positions=positions,
forward_batch=forward_batch,
hidden_states=hidden_states,
residual=residual,
)
last_layer = self.layers[self.end_layer - 1]
hidden_states, residual = last_layer.layer_communicator.finish_layer_stack(
hidden_states,
residual,
forward_batch,
preserve_declared=not self.pp_group.is_last_rank,
)
if not self.pp_group.is_last_rank:
return PPProxyTensors(
{
"hidden_states": hidden_states,
"residual": residual,
}
)
return residual_batch.to_pp(hidden_states, forward_batch)
else:
hidden_states = residual_batch.finish(hidden_states, forward_batch)
if not forward_batch.forward_mode.is_idle():
if residual is None:
hidden_states = self.norm(hidden_states)
else:
hidden_states, _ = self.norm(hidden_states, residual)
hidden_states = residual_batch.norm(
hidden_states, forward_batch, self.norm
)
if len(aux_hidden_states) == 0:
return hidden_states
return hidden_states, aux_hidden_states
+54 -74
View File
@@ -24,7 +24,7 @@ from torch import nn
from transformers import PretrainedConfig
from sglang.srt.batch_overlap.single_batch_overlap import SboFlags
from sglang.srt.batch_overlap.two_batch_overlap import model_forward_maybe_tbo
from sglang.srt.batch_overlap.two_batch_overlap import model_forward_stages
from sglang.srt.distributed.device_communicators.pynccl_allocator import (
use_symmetric_memory,
)
@@ -34,10 +34,11 @@ from sglang.srt.eplb.expert_location_dispatch import ExpertLocationDispatchInfo
from sglang.srt.layers.activation import SiluAndMul
from sglang.srt.layers.aux_hidden_states import AuxHiddenStateList
from sglang.srt.layers.communicator import (
LayerCommunicator,
LayerFacts,
declare_attn,
declare_ffn,
enable_moe_dense_fully_dp,
get_attn_tp_context,
make_stages,
)
from sglang.srt.layers.communicator.residual import batch as residual_batch
from sglang.srt.layers.dp_attention import (
@@ -549,14 +550,6 @@ class Glm4MoeLiteDecoderLayer(nn.Module):
is_previous_layer_sparse = self._is_layer_sparse(layer_id - 1, is_nextn=False)
is_next_layer_sparse = self._is_layer_sparse(layer_id + 1, is_nextn=False)
self.layer_facts = LayerFacts.init_new(
layer_id=layer_id,
num_layers=1 if is_nextn else config.num_hidden_layers,
is_layer_sparse=self.is_layer_sparse,
is_previous_layer_sparse=is_previous_layer_sparse,
is_next_layer_sparse=is_next_layer_sparse,
)
if self.is_layer_sparse:
self.mlp = Glm4MoeLiteSparseMoeBlock(
config=config,
@@ -588,12 +581,25 @@ class Glm4MoeLiteDecoderLayer(nn.Module):
self._gfx95_quant_format = self._detect_gfx95_quant_format()
self.layer_communicator = LayerCommunicator(
layer_facts=self.layer_facts,
input_layernorm=self.input_layernorm,
post_attention_layernorm=self.post_attention_layernorm,
allow_reduce_scatter=True,
qkv_latent_func=self.self_attn.prepare_qkv_latent,
self.attn_stage, self.ffn_stage = make_stages(
(
declare_attn(),
self.input_layernorm,
{"qkv_latent_func": self.self_attn.prepare_qkv_latent},
),
(
declare_ffn(
sparse=self.is_layer_sparse,
next_sparse=is_next_layer_sparse,
),
self.post_attention_layernorm,
),
previous=declare_ffn(
sparse=is_previous_layer_sparse, next_sparse=self.is_layer_sparse
)
if layer_id != 0
else None,
terminal=layer_id == (1 if is_nextn else config.num_hidden_layers) - 1,
)
def _detect_gfx95_quant_format(self) -> str:
@@ -624,18 +630,14 @@ class Glm4MoeLiteDecoderLayer(nn.Module):
positions: torch.Tensor,
hidden_states: torch.Tensor,
forward_batch: ForwardBatch,
residual: Optional[torch.Tensor],
zero_allocator: BumpAllocator,
capture_output=None,
) -> torch.Tensor:
hidden_states, residual = (
self.layer_communicator.prepare_attn_and_capture_last_layer_outputs(
hidden_states,
residual,
forward_batch,
quant_format=getattr(self, "_gfx95_quant_format", ""),
capture_output=capture_output,
)
hidden_states = self.attn_stage.prepare(
hidden_states,
forward_batch,
quant_format=getattr(self, "_gfx95_quant_format", ""),
capture_output=capture_output,
)
hidden_states = self.self_attn(
@@ -643,23 +645,20 @@ class Glm4MoeLiteDecoderLayer(nn.Module):
hidden_states=hidden_states,
forward_batch=forward_batch,
zero_allocator=zero_allocator,
input_on_attention_tp_slices=(
self.layer_communicator.input_on_attention_tp_slices
),
input_on_attention_tp_slices=(self.attn_stage.input_on_attention_tp_slices),
)
if isinstance(hidden_states, tuple):
hidden_states = hidden_states[0]
get_attn_tp_context().clear_attn_inputs()
hidden_states, residual = self.layer_communicator.prepare_mlp(
hidden_states, residual, forward_batch
)
hidden_states = self.attn_stage.finish(hidden_states, forward_batch)
hidden_states = self.ffn_stage.prepare(hidden_states, forward_batch)
with self.layer_communicator.ffn_exit(forward_batch) as ffn_exit:
with self.ffn_stage.exit(forward_batch) as ffn_exit:
hidden_states = self.mlp(hidden_states, forward_batch)
hidden_states, residual = ffn_exit.finish(hidden_states, residual)
hidden_states = ffn_exit.finish(hidden_states)
return hidden_states, residual
return hidden_states
def op_comm_prepare_attn(
self,
@@ -667,12 +666,11 @@ class Glm4MoeLiteDecoderLayer(nn.Module):
positions: torch.Tensor,
hidden_states: torch.Tensor,
forward_batch: ForwardBatch,
residual: Optional[torch.Tensor],
zero_allocator: BumpAllocator,
tbo_subbatch_index: Optional[int] = None,
):
state.hidden_states_after_comm_pre_attn, state.residual_after_input_ln = (
self.layer_communicator.prepare_attn(hidden_states, residual, forward_batch)
state.hidden_states_after_comm_pre_attn = self.attn_stage.prepare(
hidden_states, forward_batch
)
if get_moe_a2a_backend().is_mori():
state.num_tokens = hidden_states.shape[0]
@@ -686,25 +684,21 @@ class Glm4MoeLiteDecoderLayer(nn.Module):
)
def op_comm_prepare_mlp(self, state):
state.hidden_states_mlp_input, state.residual_after_comm_pre_mlp = (
self.layer_communicator.prepare_mlp(
state.pop("hidden_states_after_attn"),
state.pop("residual_after_input_ln"),
state.forward_batch,
)
hidden_states = self.attn_stage.finish(
state.pop("hidden_states_after_attn"), state.forward_batch
)
state.hidden_states_mlp_input = self.ffn_stage.prepare(
hidden_states, state.forward_batch
)
def op_comm_postprocess_layer(self, state):
hidden_states, residual = self.layer_communicator.postprocess_layer(
state.pop("hidden_states_mlp_output"),
state.pop("residual_after_comm_pre_mlp"),
state.forward_batch,
hidden_states = self.ffn_stage.postprocess(
state.pop("hidden_states_mlp_output"), state.forward_batch
)
output = dict(
positions=state.positions,
hidden_states=hidden_states,
residual=residual,
forward_batch=state.forward_batch,
zero_allocator=state.zero_allocator,
tbo_subbatch_index=state.tbo_subbatch_index,
@@ -782,12 +776,12 @@ class Glm4MoeLiteModel(nn.Module):
hidden_states = self.embed_tokens(input_ids)
else:
hidden_states = input_embeds
residual = residual_batch.start(forward_batch)
residual_batch.start(forward_batch)
else:
assert pp_proxy_tensors is not None
hidden_states, residual = self.layers[
self.start_layer
].layer_communicator.from_pp(pp_proxy_tensors, forward_batch)
hidden_states = self.layers[self.start_layer].attn_stage.from_pp(
pp_proxy_tensors, forward_batch
)
device = hidden_states.device
zero_allocator = BumpAllocator(
buffer_size=total_num_layers * 2 * (2 if forward_batch.can_run_tbo else 1),
@@ -809,11 +803,10 @@ class Glm4MoeLiteModel(nn.Module):
for i in range(normal_start_layer, normal_end_layer):
with get_global_expert_distribution_recorder().with_current_layer(i):
layer = self.layers[i]
hidden_states, residual = layer(
hidden_states = layer(
positions,
hidden_states,
forward_batch,
residual,
zero_allocator,
capture_output=aux_hidden_states.capture
if i in self.layers_to_capture
@@ -821,36 +814,23 @@ class Glm4MoeLiteModel(nn.Module):
)
if normal_end_layer != self.end_layer:
hidden_states, residual = model_forward_maybe_tbo(
hidden_states = model_forward_stages(
layers=self.layers[normal_end_layer : self.end_layer],
enable_tbo=True,
positions=positions,
forward_batch=forward_batch,
hidden_states=hidden_states,
residual=residual,
zero_allocator=zero_allocator,
)
last_layer = self.layers[self.end_layer - 1]
hidden_states, residual = last_layer.layer_communicator.finish_layer_stack(
hidden_states,
residual,
forward_batch,
preserve_declared=not self.pp_group.is_last_rank,
)
if not self.pp_group.is_last_rank:
return PPProxyTensors(
{
"hidden_states": hidden_states,
"residual": residual,
}
)
return residual_batch.to_pp(hidden_states, forward_batch)
else:
hidden_states = residual_batch.finish(hidden_states, forward_batch)
if not forward_batch.forward_mode.is_idle():
if residual is None:
hidden_states = self.norm(hidden_states)
else:
hidden_states, _ = self.norm(hidden_states, residual)
hidden_states = residual_batch.norm(
hidden_states, forward_batch, self.norm
)
if len(aux_hidden_states) == 0:
return hidden_states
@@ -115,20 +115,17 @@ class Glm4MoeLiteModelNextN(nn.Module):
)
)
residual = residual_batch.start(forward_batch)
residual_batch.start(forward_batch)
with get_global_expert_distribution_recorder().disable_this_region():
hidden_states, residual = self.decoder(
positions, hidden_states, forward_batch, residual, zero_allocator
hidden_states = self.decoder(
positions, hidden_states, forward_batch, zero_allocator
)
hidden_states, residual = self.decoder.layer_communicator.finish_layer_stack(
hidden_states, residual, forward_batch
)
hidden_states = residual_batch.finish(hidden_states, forward_batch)
if not forward_batch.forward_mode.is_idle():
if residual is not None:
hidden_states, _ = self.shared_head.norm(hidden_states, residual)
else:
hidden_states = self.shared_head.norm(hidden_states)
hidden_states = residual_batch.norm(
hidden_states, forward_batch, self.shared_head.norm
)
return hidden_states
+6 -11
View File
@@ -101,20 +101,15 @@ class Glm4MoeModelNextN(nn.Module):
)
)
residual = residual_batch.start(forward_batch)
residual_batch.start(forward_batch)
with get_global_expert_distribution_recorder().disable_this_region():
hidden_states, residual = self.decoder(
positions, hidden_states, forward_batch, residual
)
hidden_states = self.decoder(positions, hidden_states, forward_batch)
hidden_states, residual = self.decoder.layer_communicator.finish_layer_stack(
hidden_states, residual, forward_batch
)
hidden_states = residual_batch.finish(hidden_states, forward_batch)
if not forward_batch.forward_mode.is_idle():
if residual is not None:
hidden_states, _ = self.shared_head.norm(hidden_states, residual)
else:
hidden_states = self.shared_head.norm(hidden_states)
hidden_states = residual_batch.norm(
hidden_states, forward_batch, self.shared_head.norm
)
return hidden_states
+63 -77
View File
@@ -2,7 +2,7 @@ import logging
from array import array
from contextlib import nullcontext
from functools import partial
from typing import Any, Dict, Iterable, List, Optional, Tuple, Union
from typing import Iterable, List, Optional, Tuple, Union
import torch
from torch import nn
@@ -13,7 +13,7 @@ from sglang.kernels.ops.layernorm.mhc import hc_contract
from sglang.kernels.ops.layernorm.mhc import hc_post as _hc_post_fn
from sglang.kernels.ops.layernorm.mhc import hc_pre as _hc_pre_fn
from sglang.srt.batch_overlap.two_batch_overlap import (
model_forward_maybe_tbo,
model_forward_stages,
)
from sglang.srt.configs.glm5_next import Glm5NextConfig, Glm5NextTextConfig
from sglang.srt.configs.model_config import is_deepseek_dsa
@@ -27,12 +27,15 @@ from sglang.srt.layers.attention import vision_utils
from sglang.srt.layers.attention.vision import VisionAttention
from sglang.srt.layers.aux_hidden_states import AuxHiddenStateList
from sglang.srt.layers.communicator import (
LayerCommunicator,
LayerFacts,
MHCLayerCommunicator,
PLAIN_RESIDUAL,
MHCState,
declare_attn,
declare_ffn,
enable_moe_dense_fully_dp,
get_attn_tp_context,
make_stages,
)
from sglang.srt.layers.communicator.residual import access as residual_access
from sglang.srt.layers.communicator.residual import batch as residual_batch
from sglang.srt.layers.layernorm import RMSNorm
from sglang.srt.layers.linear import (
@@ -727,14 +730,6 @@ class Glm5NextDecoderLayer(nn.Module):
is_previous_layer_sparse = self._is_layer_sparse(layer_id - 1, is_nextn=False)
is_next_layer_sparse = self._is_layer_sparse(layer_id + 1, is_nextn=False)
self.layer_facts = LayerFacts.init_new(
layer_id=layer_id,
num_layers=1 if is_nextn else config.num_hidden_layers,
is_layer_sparse=self.is_layer_sparse,
is_previous_layer_sparse=is_previous_layer_sparse,
is_next_layer_sparse=is_next_layer_sparse,
)
if self.is_layer_sparse:
self.mlp = Glm5NextMoE(
config=config,
@@ -784,36 +779,52 @@ class Glm5NextDecoderLayer(nn.Module):
torch.empty(mix_hc, hc_dim, dtype=torch.float32)
)
shared_kwargs: Dict[str, Any] = dict(
layer_facts=self.layer_facts,
input_layernorm=self.input_layernorm,
post_attention_layernorm=self.post_attention_layernorm,
allow_reduce_scatter=True,
qkv_latent_func=(
self.self_attn.prepare_qkv_latent if not self.is_linear_attn else None
),
)
terminal = layer_id == (1 if is_nextn else config.num_hidden_layers) - 1
residual = PLAIN_RESIDUAL
if self.config.mhc:
mhc_kwargs: Dict[str, Any] = dict(
residual = MHCState(
hc_mult=config.hc_mult,
hc_attn_pre=self.hc_attn_pre,
hc_ffn_pre=self.hc_ffn_pre,
hc_post=self.hc_post,
# Resolved once: env and platform are frozen after startup,
# and None keeps the dispatch off the per-boundary path.
hc_ffn_post_pre=(
self.hc_ffn_post_pre
if is_cross_layer_mhc_fusion_enabled()
else None
),
is_last_layer=terminal,
).layer_residual()
self.attn_stage, self.ffn_stage = make_stages(
(
declare_attn(
read=residual.attention_read, update=residual.attention_update
),
self.input_layernorm,
{
"residual_in_hidden": residual.ffn_update.at_producer,
"qkv_latent_func": self.self_attn.prepare_qkv_latent
if not self.is_linear_attn
else None,
},
),
(
declare_ffn(
sparse=self.is_layer_sparse,
next_sparse=is_next_layer_sparse,
read=residual.ffn_read,
update=residual.ffn_update,
),
self.post_attention_layernorm,
),
previous=declare_ffn(
sparse=is_previous_layer_sparse,
next_sparse=self.is_layer_sparse,
update=residual.ffn_update,
)
self.layer_communicator = MHCLayerCommunicator(
**shared_kwargs,
**mhc_kwargs,
)
else:
self.layer_communicator = LayerCommunicator(**shared_kwargs)
if layer_id != 0
else None,
terminal=terminal,
)
def _hc_pre(
self, hc_fn, hc_scale, hc_base, hidden_states, out_norm_weight, out_norm_eps
@@ -915,21 +926,15 @@ class Glm5NextDecoderLayer(nn.Module):
positions: torch.Tensor,
hidden_states: torch.Tensor,
forward_batch: ForwardBatch,
residual: Optional[torch.Tensor],
zero_allocator: Optional[BumpAllocator] = None,
gemm_output_zero_allocator: BumpAllocator = None,
prev_topk_indices: Optional[torch.Tensor] = None,
capture_output=None,
):
hidden_states_orig = self.layer_communicator.buffer(hidden_states)
hidden_states_orig = residual_access.buffer(hidden_states)
hidden_states, residual = (
self.layer_communicator.prepare_attn_and_capture_last_layer_outputs(
hidden_states,
residual,
forward_batch,
capture_output=capture_output,
)
hidden_states = self.attn_stage.prepare(
hidden_states, forward_batch, capture_output=capture_output
)
hidden_states = self.self_attn(
@@ -937,9 +942,7 @@ class Glm5NextDecoderLayer(nn.Module):
hidden_states=hidden_states,
forward_batch=forward_batch,
zero_allocator=zero_allocator,
input_on_attention_tp_slices=(
self.layer_communicator.input_on_attention_tp_slices
),
input_on_attention_tp_slices=(self.attn_stage.input_on_attention_tp_slices),
prev_topk_indices=prev_topk_indices,
)
if isinstance(hidden_states, tuple):
@@ -948,11 +951,8 @@ class Glm5NextDecoderLayer(nn.Module):
topk_indices = None
get_attn_tp_context().clear_attn_inputs()
hidden_states, residual = self.layer_communicator.prepare_mlp(
hidden_states,
residual,
forward_batch,
)
hidden_states = self.attn_stage.finish(hidden_states, forward_batch)
hidden_states = self.ffn_stage.prepare(hidden_states, forward_batch)
if isinstance(self.mlp, Glm5NextMLP):
gemm_output_zero_allocator = None
@@ -968,15 +968,15 @@ class Glm5NextDecoderLayer(nn.Module):
else:
_mlp_ctx = nullcontext()
with self.layer_communicator.ffn_exit(forward_batch) as ffn_exit, _mlp_ctx:
with self.ffn_stage.exit(forward_batch) as ffn_exit, _mlp_ctx:
hidden_states = self.mlp(
hidden_states,
forward_batch,
gemm_output_zero_allocator,
)
hidden_states, residual = ffn_exit.finish(hidden_states, residual)
hidden_states = ffn_exit.finish(hidden_states)
return hidden_states, residual, topk_indices
return (hidden_states, topk_indices)
class Glm5NextModel(nn.Module):
@@ -1106,12 +1106,12 @@ class Glm5NextModel(nn.Module):
hidden_states = self.embed_tokens(input_ids)
else:
hidden_states = input_embeds
residual = residual_batch.start(forward_batch)
residual_batch.start(forward_batch)
else:
assert pp_proxy_tensors is not None
hidden_states, residual = self.layers[
self.start_layer
].layer_communicator.from_pp(pp_proxy_tensors, forward_batch)
hidden_states = self.layers[self.start_layer].attn_stage.from_pp(
pp_proxy_tensors, forward_batch
)
device = hidden_states.device
zero_allocator = BumpAllocator(
buffer_size=total_num_layers * 2 * (2 if forward_batch.can_run_tbo else 1),
@@ -1165,11 +1165,10 @@ class Glm5NextModel(nn.Module):
aux_hidden_states.capture(aux_hidden_state, owned=owned)
layer = self.layers[i]
hidden_states, residual, topk_indices = layer(
(hidden_states, topk_indices) = layer(
positions,
hidden_states,
forward_batch,
residual,
zero_allocator,
gemm_output_zero_allocator,
prev_topk_indices=topk_indices,
@@ -1179,38 +1178,25 @@ class Glm5NextModel(nn.Module):
)
if normal_end_layer != self.end_layer:
hidden_states, residual = model_forward_maybe_tbo(
hidden_states = model_forward_stages(
layers=self.layers[normal_end_layer : self.end_layer],
enable_tbo=True,
positions=positions,
forward_batch=forward_batch,
hidden_states=hidden_states,
residual=residual,
zero_allocator=zero_allocator,
)
last_layer = self.layers[self.end_layer - 1]
hidden_states, residual = last_layer.layer_communicator.finish_layer_stack(
hidden_states,
residual,
forward_batch,
preserve_declared=not self.pp_group.is_last_rank,
)
if not self.pp_group.is_last_rank:
if self.config.mhc:
return PPProxyTensors({"hidden_states": hidden_states})
return PPProxyTensors(
{
"hidden_states": hidden_states,
"residual": residual,
}
)
return residual_batch.to_pp(hidden_states, forward_batch)
else:
hidden_states = residual_batch.finish(hidden_states, forward_batch)
if not forward_batch.forward_mode.is_idle():
if residual is None:
hidden_states = self.norm(hidden_states)
else:
hidden_states, _ = self.norm(hidden_states, residual)
hidden_states = residual_batch.norm(
hidden_states, forward_batch, self.norm
)
if len(aux_hidden_states) == 0:
return hidden_states
+74 -78
View File
@@ -14,7 +14,11 @@ from sglang.srt.layers.aux_hidden_states import (
AuxHiddenStateAccumulator,
AuxHiddenStateList,
)
from sglang.srt.layers.communicator import LayerCommunicator, LayerFacts
from sglang.srt.layers.communicator import (
declare_attn,
declare_ffn,
make_stages,
)
from sglang.srt.layers.communicator.residual import batch as residual_batch
from sglang.srt.layers.dp_attention import is_dp_attention_enabled
from sglang.srt.layers.layernorm import GemmaRMSNorm
@@ -421,19 +425,17 @@ class _InternS2MobiusDecoderMixin:
def _forward_after_attention(
self,
hidden_states: torch.Tensor,
residual: torch.Tensor | None,
forward_batch: ForwardBatch,
meta_mlp: nn.ModuleList,
) -> tuple[torch.Tensor, torch.Tensor | None]:
hidden_states, residual = self.layer_communicator.prepare_mlp(
hidden_states, residual, forward_batch
)
with self.layer_communicator.ffn_exit(forward_batch) as ffn_exit:
) -> torch.Tensor:
hidden_states = self.attn_stage.finish(hidden_states, forward_batch)
hidden_states = self.ffn_stage.prepare(hidden_states, forward_batch)
with self.ffn_stage.exit(forward_batch) as ffn_exit:
hidden_states = self._forward_mobius_mlp(
hidden_states, forward_batch, meta_mlp
)
hidden_states, residual = ffn_exit.finish(hidden_states, residual)
return hidden_states, residual
hidden_states = ffn_exit.finish(hidden_states)
return hidden_states
class InternS2MobiusLayerMlp(nn.Module):
@@ -481,13 +483,7 @@ class InternS2MobiusLinearDecoderLayer(_InternS2MobiusDecoderMixin, nn.Module):
layer_prefix = prefix.removesuffix(".linear_attn")
self._init_mobius_mlp(config, quant_config, layer_prefix)
self.layer_facts = LayerFacts.init_new(
layer_id=layer_id,
num_layers=config.num_hidden_layers,
is_layer_sparse=True,
is_previous_layer_sparse=True,
is_next_layer_sparse=True,
)
self.input_layernorm = GemmaRMSNorm(config.hidden_size, eps=config.rms_norm_eps)
self.post_attention_layernorm = GemmaRMSNorm(
config.hidden_size, eps=config.rms_norm_eps
@@ -496,37 +492,43 @@ class InternS2MobiusLinearDecoderLayer(_InternS2MobiusDecoderMixin, nn.Module):
_enable_qwen35_fused_ar_quant()
and _linear_accepts_fp8_tuple(self.linear_attn.in_proj_qkvz)
)
self.layer_communicator = LayerCommunicator(
layer_facts=self.layer_facts,
input_layernorm=self.input_layernorm,
post_attention_layernorm=self.post_attention_layernorm,
allow_reduce_scatter=True,
enable_fused_ar_quant=enable_fused_ar_quant,
fused_ar_quant_keep_bf16=enable_fused_ar_quant,
allow_deferred_ffn_reduction=False,
self.attn_stage, self.ffn_stage = make_stages(
(
declare_attn(),
self.input_layernorm,
{
"enable_fused_ar_quant": enable_fused_ar_quant,
"fused_ar_quant_keep_bf16": enable_fused_ar_quant,
},
),
(
declare_ffn(
sparse=True,
next_sparse=True,
),
self.post_attention_layernorm,
),
previous=declare_ffn(sparse=True, next_sparse=True)
if layer_id != 0
else None,
terminal=layer_id == config.num_hidden_layers - 1,
)
def forward(
self,
hidden_states: torch.Tensor,
residual: torch.Tensor | None,
meta_mlp: nn.ModuleList,
**kwargs,
):
forward_batch = kwargs["forward_batch"]
hidden_states, residual = (
self.layer_communicator.prepare_attn_and_capture_last_layer_outputs(
hidden_states,
residual,
forward_batch,
captured_last_layer_outputs=kwargs.get("captured_last_layer_outputs"),
)
hidden_states = self.attn_stage.prepare(
hidden_states,
forward_batch,
captured_last_layer_outputs=kwargs.get("captured_last_layer_outputs"),
)
if not forward_batch.forward_mode.is_idle():
hidden_states = self.linear_attn(hidden_states, forward_batch)
return self._forward_after_attention(
hidden_states, residual, forward_batch, meta_mlp
)
return self._forward_after_attention(hidden_states, forward_batch, meta_mlp)
class InternS2MobiusAttentionDecoderLayer(
@@ -614,13 +616,7 @@ class InternS2MobiusAttentionDecoderLayer(
layer_prefix = prefix.removesuffix(".self_attn")
self._init_mobius_mlp(config, quant_config, layer_prefix)
self.layer_facts = LayerFacts.init_new(
layer_id=layer_id,
num_layers=config.num_hidden_layers,
is_layer_sparse=True,
is_previous_layer_sparse=True,
is_next_layer_sparse=True,
)
self.input_layernorm = GemmaRMSNorm(config.hidden_size, eps=config.rms_norm_eps)
self.post_attention_layernorm = GemmaRMSNorm(
config.hidden_size, eps=config.rms_norm_eps
@@ -630,14 +626,26 @@ class InternS2MobiusAttentionDecoderLayer(
enable_fused_ar_quant = (
_enable_qwen35_fused_ar_quant() and _linear_accepts_fp8_tuple(self.qkv_proj)
)
self.layer_communicator = LayerCommunicator(
layer_facts=self.layer_facts,
input_layernorm=self.input_layernorm,
post_attention_layernorm=self.post_attention_layernorm,
allow_reduce_scatter=True,
enable_fused_ar_quant=enable_fused_ar_quant,
fused_ar_quant_keep_bf16=False,
allow_deferred_ffn_reduction=False,
self.attn_stage, self.ffn_stage = make_stages(
(
declare_attn(),
self.input_layernorm,
{
"enable_fused_ar_quant": enable_fused_ar_quant,
"fused_ar_quant_keep_bf16": False,
},
),
(
declare_ffn(
sparse=True,
next_sparse=True,
),
self.post_attention_layernorm,
),
previous=declare_ffn(sparse=True, next_sparse=True)
if layer_id != 0
else None,
terminal=layer_id == config.num_hidden_layers - 1,
)
self.alt_stream = alt_stream
@@ -645,20 +653,16 @@ class InternS2MobiusAttentionDecoderLayer(
self,
positions: torch.Tensor,
hidden_states: torch.Tensor,
residual: torch.Tensor | None,
forward_batch: ForwardBatch,
meta_mlp: nn.ModuleList,
captured_last_layer_outputs: AuxHiddenStateAccumulator | None = None,
**kwargs,
):
del kwargs
hidden_states, residual = (
self.layer_communicator.prepare_attn_and_capture_last_layer_outputs(
hidden_states,
residual,
forward_batch,
captured_last_layer_outputs=captured_last_layer_outputs,
)
hidden_states = self.attn_stage.prepare(
hidden_states,
forward_batch,
captured_last_layer_outputs=captured_last_layer_outputs,
)
if not forward_batch.forward_mode.is_idle():
hidden_states = self.self_attention(
@@ -666,9 +670,7 @@ class InternS2MobiusAttentionDecoderLayer(
hidden_states=hidden_states,
forward_batch=forward_batch,
)
return self._forward_after_attention(
hidden_states, residual, forward_batch, meta_mlp
)
return self._forward_after_attention(hidden_states, forward_batch, meta_mlp)
class InternS2MobiusForCausalLM(Qwen3_5ForCausalLM):
@@ -767,20 +769,17 @@ class InternS2MobiusForCausalLM(Qwen3_5ForCausalLM):
hidden_states = (
self.embed_tokens(input_ids) if input_embeds is None else input_embeds
)
residual = residual_batch.start(forward_batch)
residual_batch.start(forward_batch)
aux_hidden_states = AuxHiddenStateList()
for layer_idx, layer in enumerate(self.layers):
hidden_states, residual = layer(
hidden_states = layer(
positions=positions,
hidden_states=hidden_states,
residual=residual,
forward_batch=forward_batch,
meta_mlp=self.meta_mlp,
captured_last_layer_outputs=(
aux_hidden_states
if getattr(layer, "_is_layer_to_capture", False)
else None
),
captured_last_layer_outputs=aux_hidden_states
if getattr(layer, "_is_layer_to_capture", False)
else None,
)
if (
input_deepstack_embeds is not None
@@ -788,19 +787,16 @@ class InternS2MobiusForCausalLM(Qwen3_5ForCausalLM):
and layer_idx < 3
):
start = self.hidden_size * layer_idx
hidden_states.add_(
input_deepstack_embeds[:, start : start + self.hidden_size]
hidden_states = residual_batch.add_to_output(
hidden_states,
forward_batch,
input_deepstack_embeds[:, start : start + self.hidden_size],
)
hidden_states, residual = self.layers[-1].layer_communicator.finish_layer_stack(
hidden_states, residual, forward_batch
)
hidden_states = residual_batch.finish(hidden_states, forward_batch)
if hidden_states.shape[0] != 0:
if residual is None:
hidden_states = self.norm(hidden_states)
else:
hidden_states, _ = self.norm(hidden_states, residual)
hidden_states = residual_batch.norm(hidden_states, forward_batch, self.norm)
return (
hidden_states
if not aux_hidden_states
+1 -1
View File
@@ -161,7 +161,7 @@ class Eagle3MLADecoderLayer(nn.Module):
attn_input = torch.cat([embeds, hidden_states], dim=-1)
# MLA's forward_absorb_prepare reads the qkv-down projection result
# from attn_tp_context. We bypass LayerCommunicator here (the eagle3
# from attn_tp_context. We bypass layer boundary preparation here (the eagle3
# draft layer is one isolated layer with custom pre-attention norms),
# so publish the attention input ourselves.
get_attn_tp_context().set_attn_inputs(
+8 -9
View File
@@ -243,15 +243,14 @@ def _merge_weights_as_views(
return merged, sizes
# K3 cannot use LayerCommunicator: the attn-res aggregation kernels replace
# input_layernorm / post_attention_layernorm, which the communicator expects
# to own. Instead the MLP/MoE modules gather/scatter around their own body:
# K3 manages its own boundaries: the attn-res aggregation kernels replace
# input_layernorm / post_attention_layernorm, and the MLP/MoE modules
# gather/scatter around their own body:
# attention and the attn-res buffers stay in local (per-DP-rank) token space,
# the MLP/MoE runs on the DP-gathered global batch, and the delayed prefix_sum
# add stays local, applied after the scatter back.
def _dp_local_buffer_group():
"""Symmetric-memory group for the local DP buffer (mirrors
CommunicateSummableTensorPairFn._scatter_hidden_states)."""
"""Symmetric-memory group holding this DP rank's local token buffer."""
parallel = get_parallel()
if parallel.tp_size == parallel.attn_dp_size:
return get_parallel().tp_group
@@ -2252,7 +2251,7 @@ class KimiK3MLAAttention(DeepseekV2AttentionMLA):
self.o_proj.forward = _symm_o_proj_forward
else:
# K3 has no LayerCommunicator, so o_proj must reduce within the
# K3 owns its output reduction, so o_proj must reduce within the
# attn-TP group itself: the default full-TP collective is the wrong
# group at attn_tp>1 and deadlocks idle DP ranks.
self.o_proj.use_dp_attention_reduce = True
@@ -2571,7 +2570,7 @@ class KimiK3DecoderLayer(nn.Module):
) -> torch.Tensor:
# DP attention: idle ranks (padded to the global shape) have no
# attention metadata; pass hidden_states through shape-preserving
# (same as the LayerCommunicator models' is_idle skip).
# (matching the idle skip in other DP-attention models).
if forward_batch.forward_mode.is_idle():
return hidden_states
@@ -2614,8 +2613,8 @@ class KimiK3DecoderLayer(nn.Module):
forward_batch: ForwardBatch,
zero_allocator: BumpAllocator,
) -> torch.Tensor:
# For MLA layers with q_lora_rank, set up communicator attn_inputs
# before the forward call (normally done by LayerCommunicator).
# For MLA layers with q_lora_rank, set up attn_inputs before the
# forward call (normally done by the attention boundary).
from sglang.srt.layers.communicator import (
AttentionInputs,
get_attn_tp_context,
+38 -52
View File
@@ -21,8 +21,9 @@ from sglang.srt.environ import envs
from sglang.srt.layers.activation import SiluAndMul
from sglang.srt.layers.aux_hidden_states import AuxHiddenStateList
from sglang.srt.layers.communicator import (
LayerCommunicator,
LayerFacts,
declare_attn,
declare_ffn,
make_stages,
)
from sglang.srt.layers.communicator.residual import batch as residual_batch
from sglang.srt.layers.dp_attention import (
@@ -456,18 +457,21 @@ class LagunaDecoderLayer(nn.Module):
config.hidden_size, eps=config.rms_norm_eps
)
self.layer_facts = LayerFacts.init_new(
layer_id=layer_id,
num_layers=config.num_hidden_layers,
is_layer_sparse=self.is_layer_sparse,
is_previous_layer_sparse=is_previous_layer_sparse,
is_next_layer_sparse=is_next_layer_sparse,
)
self.layer_communicator = LayerCommunicator(
layer_facts=self.layer_facts,
input_layernorm=self.input_layernorm,
post_attention_layernorm=self.post_attention_layernorm,
allow_reduce_scatter=True,
self.attn_stage, self.ffn_stage = make_stages(
(declare_attn(), self.input_layernorm),
(
declare_ffn(
sparse=self.is_layer_sparse,
next_sparse=is_next_layer_sparse,
),
self.post_attention_layernorm,
),
previous=declare_ffn(
sparse=is_previous_layer_sparse, next_sparse=self.is_layer_sparse
)
if layer_id != 0
else None,
terminal=layer_id == config.num_hidden_layers - 1,
)
def forward(
@@ -475,16 +479,10 @@ class LagunaDecoderLayer(nn.Module):
positions: torch.Tensor,
hidden_states: torch.Tensor,
forward_batch: ForwardBatch,
residual: Optional[torch.Tensor],
capture_output=None,
) -> Tuple[torch.Tensor, torch.Tensor]:
hidden_states, residual = (
self.layer_communicator.prepare_attn_and_capture_last_layer_outputs(
hidden_states,
residual,
forward_batch,
capture_output=capture_output,
)
) -> torch.Tensor:
hidden_states = self.attn_stage.prepare(
hidden_states, forward_batch, capture_output=capture_output
)
if hidden_states.shape[0] != 0:
hidden_states = self.self_attn(
@@ -492,17 +490,16 @@ class LagunaDecoderLayer(nn.Module):
hidden_states=hidden_states,
forward_batch=forward_batch,
)
hidden_states, residual = self.layer_communicator.prepare_mlp(
hidden_states, residual, forward_batch
)
hidden_states = self.attn_stage.finish(hidden_states, forward_batch)
hidden_states = self.ffn_stage.prepare(hidden_states, forward_batch)
with self.layer_communicator.ffn_exit(forward_batch) as ffn_exit:
with self.ffn_stage.exit(forward_batch) as ffn_exit:
hidden_states = self.mlp(
hidden_states,
forward_batch=forward_batch,
)
hidden_states, residual = ffn_exit.finish(hidden_states, residual)
return hidden_states, residual
hidden_states = ffn_exit.finish(hidden_states)
return hidden_states
class LagunaModel(nn.Module):
@@ -589,48 +586,37 @@ class LagunaModel(nn.Module):
hidden_states = self.embed_tokens(input_ids)
else:
hidden_states = input_embeds
residual = residual_batch.start(forward_batch)
residual_batch.start(forward_batch)
else:
assert pp_proxy_tensors is not None
hidden_states, residual = self.layers[
self.start_layer
].layer_communicator.from_pp(pp_proxy_tensors, forward_batch)
hidden_states = self.layers[self.start_layer].attn_stage.from_pp(
pp_proxy_tensors, forward_batch
)
aux_hidden_states = AuxHiddenStateList()
for i in range(self.start_layer, self.end_layer):
layer = self.layers[i]
hidden_states, residual = layer(
hidden_states = layer(
positions,
hidden_states,
forward_batch,
residual,
capture_output=aux_hidden_states.capture
if i in self.layers_to_capture
else None,
)
last_layer = self.layers[self.end_layer - 1]
hidden_states, residual = last_layer.layer_communicator.finish_layer_stack(
hidden_states,
residual,
forward_batch,
preserve_declared=not self.pp_group.is_last_rank,
)
if not self.pp_group.is_last_rank:
return PPProxyTensors(
{"hidden_states": hidden_states, "residual": residual}
)
return residual_batch.to_pp(hidden_states, forward_batch)
hidden_states = residual_batch.finish(hidden_states, forward_batch)
if hidden_states.shape[0] != 0:
hidden_states = last_layer.layer_communicator.norm_output(
hidden_states = residual_batch.norm(
hidden_states,
residual,
forward_batch,
self.norm,
capture_output=(
aux_hidden_states.append
if self.end_layer in self.layers_to_capture
else None
),
capture_output=aux_hidden_states.append
if self.end_layer in self.layers_to_capture
else None,
)
if len(aux_hidden_states) == 0:
return hidden_states
+66 -80
View File
@@ -51,7 +51,11 @@ from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_r
from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation
from sglang.srt.layers import deep_gemm_wrapper
from sglang.srt.layers.activation import SiluAndMul
from sglang.srt.layers.communicator import LayerCommunicator, LayerFacts
from sglang.srt.layers.communicator import (
declare_attn,
declare_ffn,
make_stages,
)
from sglang.srt.layers.communicator.residual import batch as residual_batch
from sglang.srt.layers.dp_attention import (
is_dp_attention_enabled,
@@ -66,7 +70,10 @@ from sglang.srt.layers.logits_processor import LogitsProcessor
from sglang.srt.layers.moe.ep_moe.layer import DeepEPMoE, get_moe_impl_class
from sglang.srt.layers.moe.fused_moe_triton.layer import FusedMoE
from sglang.srt.layers.moe.topk import StandardTopKOutput, TopK
from sglang.srt.layers.moe.utils import filter_moe_weight_param_global_expert
from sglang.srt.layers.moe.utils import (
filter_moe_weight_param_global_expert,
should_skip_mlp_all_reduce,
)
from sglang.srt.layers.n_gram_embedding import NgramEmbedding
from sglang.srt.layers.quantization.base_config import QuantizationConfig
from sglang.srt.layers.quantization.fp8_utils import (
@@ -323,7 +330,11 @@ class LongcatFlashMoE(nn.Module):
# expert outputs across EP ranks, so an extra all-reduce double-counts.
from sglang.srt.layers.moe.utils import get_moe_a2a_backend as _lc_gab
if self.tp_size > 1 and _lc_gab().is_none():
if (
self.tp_size > 1
and _lc_gab().is_none()
and not should_skip_mlp_all_reduce()
):
final_hidden_states = tensor_model_parallel_all_reduce(final_hidden_states)
return final_hidden_states.view(num_tokens, hidden_dim)
@@ -420,41 +431,36 @@ class LongcatFlashDecoderLayer(nn.Module):
self.attn_tp_size = get_parallel().attn_tp_size
self.attn_tp_rank = get_parallel().attn_tp_rank
self.mlp_layer_facts = [
LayerFacts.init_new(
layer_id=self.layer_id * 2 + i,
num_layers=config.num_hidden_layers,
is_layer_sparse=False,
is_previous_layer_sparse=False,
# TODO: Check if the following is correct.
is_next_layer_sparse=False,
)
for i in range(2)
]
self.mlp_layer_communicator = [
LayerCommunicator(
layer_facts=self.mlp_layer_facts[i],
input_layernorm=self.input_layernorm[i],
post_attention_layernorm=self.post_attention_layernorm[i],
qkv_latent_func=self.self_attn[i].prepare_qkv_latent,
allow_deferred_ffn_reduction=False,
)
for i in range(2)
]
self.moe_layer_facts = LayerFacts.init_new(
layer_id=self.layer_id,
num_layers=config.num_hidden_layers,
is_layer_sparse=True,
is_previous_layer_sparse=True,
# TODO: Check if the following is correct.
is_next_layer_sparse=True,
self.attn_stage, self.moe_stage = make_stages(
(
declare_attn(),
self.input_layernorm[0],
{"qkv_latent_func": self.self_attn[0].prepare_qkv_latent},
),
(
declare_ffn(sparse=True, next_sparse=True),
self.post_attention_layernorm[0],
),
previous=declare_ffn(sparse=True, next_sparse=True)
if self.layer_id != 0
else None,
terminal=self.layer_id == config.num_hidden_layers - 1,
)
self.moe_layer_communicator = LayerCommunicator(
layer_facts=self.moe_layer_facts,
input_layernorm=self.input_layernorm[0],
post_attention_layernorm=self.post_attention_layernorm[0],
qkv_latent_func=self.self_attn[0].prepare_qkv_latent,
self.first_ffn, self.second_attn, self.second_ffn = make_stages(
(
declare_ffn(ordinary_only=True),
self.post_attention_layernorm[0],
),
(
declare_attn(ordinary_only=True),
self.input_layernorm[1],
{"qkv_latent_func": self.self_attn[1].prepare_qkv_latent},
),
(
declare_ffn(ordinary_only=True),
self.post_attention_layernorm[1],
),
prepared_from=self.moe_stage.declaration,
)
def forward(
@@ -462,14 +468,11 @@ class LongcatFlashDecoderLayer(nn.Module):
positions: torch.Tensor,
hidden_states: torch.Tensor,
forward_batch: ForwardBatch,
residual: Optional[torch.Tensor],
zero_allocator: BumpAllocator,
prev_topk_indices: Optional[torch.Tensor],
) -> torch.Tensor:
# first_attn
hidden_states, residual = self.moe_layer_communicator.prepare_attn(
hidden_states, residual, forward_batch
)
hidden_states = self.attn_stage.prepare(hidden_states, forward_batch)
if hidden_states.shape[0] != 0:
attn_out = self.self_attn[0](
positions=positions,
@@ -484,52 +487,46 @@ class LongcatFlashDecoderLayer(nn.Module):
hidden_states = attn_out
# moe
hidden_states, residual = self.moe_layer_communicator.prepare_mlp(
hidden_states, residual, forward_batch
)
moe_hidden_states = self.moe_layer_communicator.branch_output(
hidden_states = self.attn_stage.finish(hidden_states, forward_batch)
hidden_states = self.moe_stage.prepare(hidden_states, forward_batch)
moe_hidden_states = self.moe_stage.branch_output(
self.mlp(hidden_states.clone()), forward_batch
)
hidden_states, residual, prev_topk_indices = self.forward_mlp(
hidden_states, prev_topk_indices = self.forward_mlp(
hidden_states,
positions,
residual,
forward_batch,
zero_allocator,
prev_topk_indices,
)
hidden_states, residual = self.moe_layer_communicator.merge_branch(
hidden_states = self.moe_stage.merge_branch(
moe_hidden_states,
hidden_states,
residual,
self.mlp_layer_communicator[1],
self.second_ffn,
forward_batch,
)
return hidden_states, residual, prev_topk_indices
return hidden_states, prev_topk_indices
def forward_mlp(
self,
hidden_states,
positions,
residual,
forward_batch,
zero_allocator,
prev_topk_indices,
):
# first_mlp, on the input the MoE's boundary read
hidden_states, residual = self.mlp_layer_communicator[0].branch_input(
self.moe_layer_communicator, hidden_states, residual, forward_batch
hidden_states = self.first_ffn.branch_input(
self.moe_stage, hidden_states, forward_batch
)
with self.mlp_layer_communicator[0].ffn_exit(forward_batch) as ffn_exit:
with self.first_ffn.exit(forward_batch) as ffn_exit:
hidden_states = self.mlps[0](hidden_states)
hidden_states, residual = ffn_exit.finish(hidden_states, residual)
hidden_states = ffn_exit.finish(hidden_states)
# second_attn
hidden_states, residual = self.mlp_layer_communicator[1].prepare_attn(
hidden_states, residual, forward_batch
)
hidden_states = self.second_attn.prepare(hidden_states, forward_batch)
if hidden_states.shape[0] != 0:
attn_out = self.self_attn[1](
positions=positions,
@@ -544,14 +541,13 @@ class LongcatFlashDecoderLayer(nn.Module):
hidden_states = attn_out
# second_mlp
hidden_states, residual = self.mlp_layer_communicator[1].prepare_mlp(
hidden_states, residual, forward_batch
)
with self.mlp_layer_communicator[1].ffn_exit(forward_batch) as ffn_exit:
hidden_states = self.second_attn.finish(hidden_states, forward_batch)
hidden_states = self.second_ffn.prepare(hidden_states, forward_batch)
with self.second_ffn.exit(forward_batch) as ffn_exit:
hidden_states = self.mlps[1](hidden_states)
hidden_states, residual = ffn_exit.finish(hidden_states, residual)
hidden_states = ffn_exit.finish(hidden_states)
return hidden_states, residual, prev_topk_indices
return hidden_states, prev_topk_indices
class LongcatFlashModel(nn.Module):
@@ -625,39 +621,29 @@ class LongcatFlashModel(nn.Module):
else:
hidden_states = input_embeds
residual = residual_batch.start(forward_batch)
residual_batch.start(forward_batch)
aux_hidden_states = []
topk_indices = None
for i in range(total_num_layers):
if i in self.layers_to_capture:
aux_hidden_states.append(
self.layers[i].moe_layer_communicator.snapshot(
hidden_states, residual, at_input=True
)
self.layers[i].attn_stage.snapshot(hidden_states, forward_batch)
)
with get_global_expert_distribution_recorder().with_current_layer(i):
layer = self.layers[i]
hidden_states, residual, topk_indices = layer(
hidden_states, topk_indices = layer(
positions,
hidden_states,
forward_batch,
residual,
zero_allocator,
topk_indices,
)
last_layer = self.layers[-1]
hidden_states, residual = last_layer.moe_layer_communicator.finish_layer_stack(
hidden_states, residual, forward_batch
hidden_states = residual_batch.norm(
hidden_states, forward_batch, self.norm, skip_empty=True
)
if hidden_states.shape[0] != 0:
if residual is None:
hidden_states = self.norm(hidden_states)
else:
hidden_states, _ = self.norm(hidden_states, residual)
if len(aux_hidden_states) == 0:
return hidden_states
+26 -33
View File
@@ -41,7 +41,11 @@ from sglang.kernels.ops.quantization.fp8_kernel import is_fp8_fnuz
from sglang.srt.configs import LongcatFlashConfig
from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder
from sglang.srt.layers import deep_gemm_wrapper
from sglang.srt.layers.communicator import LayerCommunicator, LayerFacts
from sglang.srt.layers.communicator import (
declare_attn,
declare_ffn,
make_stages,
)
from sglang.srt.layers.communicator.residual import batch as residual_batch
from sglang.srt.layers.dp_attention import (
is_dp_attention_enabled,
@@ -160,17 +164,15 @@ class LongcatFlashDenseDecoderLayer(nn.Module):
self.attn_tp_size = get_parallel().attn_tp_size
self.attn_tp_rank = get_parallel().attn_tp_rank
self.layer_facts = LayerFacts.init_new(
layer_id=self.layer_id,
num_layers=config.num_hidden_layers,
is_layer_sparse=False,
is_previous_layer_sparse=False,
is_next_layer_sparse=False,
)
self.layer_communicator = LayerCommunicator(
layer_facts=self.layer_facts,
input_layernorm=self.input_layernorm,
post_attention_layernorm=self.post_attention_layernorm,
self.attn_stage, self.ffn_stage = make_stages(
(declare_attn(), self.input_layernorm),
(
declare_ffn(sparse=False, next_sparse=False),
self.post_attention_layernorm,
),
previous=declare_ffn() if self.layer_id != 0 else None,
terminal=self.layer_id == config.num_hidden_layers - 1,
)
def forward(
@@ -178,13 +180,10 @@ class LongcatFlashDenseDecoderLayer(nn.Module):
positions: torch.Tensor,
hidden_states: torch.Tensor,
forward_batch: ForwardBatch,
residual: Optional[torch.Tensor],
zero_allocator: BumpAllocator,
) -> torch.Tensor:
hidden_states, residual = self.layer_communicator.prepare_attn(
hidden_states, residual, forward_batch
)
hidden_states = self.attn_stage.prepare(hidden_states, forward_batch)
if hidden_states.shape[0] != 0:
hidden_states = self.self_attn(
positions=positions,
@@ -193,14 +192,11 @@ class LongcatFlashDenseDecoderLayer(nn.Module):
zero_allocator=zero_allocator,
)
hidden_states, residual = self.layer_communicator.prepare_mlp(
hidden_states, residual, forward_batch
)
hidden_states = self.attn_stage.finish(hidden_states, forward_batch)
hidden_states = self.ffn_stage.prepare(hidden_states, forward_batch)
hidden_states = self.mlp(hidden_states)
hidden_states, residual = self.layer_communicator.postprocess_layer(
hidden_states, residual, forward_batch
)
return hidden_states, residual
hidden_states = self.ffn_stage.postprocess(hidden_states, forward_batch)
return hidden_states
class LongcatFlashModelNextN(nn.Module):
@@ -270,21 +266,18 @@ class LongcatFlashModelNextN(nn.Module):
)
)
residual = residual_batch.start(forward_batch)
residual_batch.start(forward_batch)
with get_global_expert_distribution_recorder().disable_this_region():
hidden_states, residual = self.decoder(
positions, hidden_states, forward_batch, residual, zero_allocator
hidden_states = self.decoder(
positions, hidden_states, forward_batch, zero_allocator
)
hidden_states, residual = self.decoder.layer_communicator.finish_layer_stack(
hidden_states, residual, forward_batch
)
hidden_states = residual_batch.finish(hidden_states, forward_batch)
if not forward_batch.forward_mode.is_idle():
if residual is not None:
hidden_states, _ = self.final_layernorm(hidden_states, residual)
else:
hidden_states = self.final_layernorm(hidden_states)
hidden_states = residual_batch.norm(
hidden_states, forward_batch, self.final_layernorm
)
return hidden_states
+20 -14
View File
@@ -25,7 +25,11 @@ import torch
from torch import nn
from transformers import PretrainedConfig
from sglang.srt.layers.communicator import LayerCommunicator, LayerFacts
from sglang.srt.layers.communicator import (
declare_attn,
declare_ffn,
make_stages,
)
from sglang.srt.layers.layernorm import RMSNorm
from sglang.srt.layers.linear import QKVParallelLinear, RowParallelLinear
from sglang.srt.layers.quantization.base_config import QuantizationConfig
@@ -442,22 +446,24 @@ class MellumDecoderLayer(Qwen3MoeDecoderLayer):
is_previous_layer_sparse = _is_sparse(layer_id - 1)
is_next_layer_sparse = _is_sparse(layer_id + 1)
self.layer_facts = LayerFacts.init_new(
layer_id=layer_id,
num_layers=cfg.num_hidden_layers,
is_layer_sparse=self.is_layer_sparse,
is_previous_layer_sparse=is_previous_layer_sparse,
is_next_layer_sparse=is_next_layer_sparse,
)
self.input_layernorm = RMSNorm(cfg.hidden_size, eps=rms_norm_eps)
self.post_attention_layernorm = RMSNorm(cfg.hidden_size, eps=rms_norm_eps)
self.layer_communicator = LayerCommunicator(
layer_facts=self.layer_facts,
input_layernorm=self.input_layernorm,
post_attention_layernorm=self.post_attention_layernorm,
allow_reduce_scatter=True,
self.attn_stage, self.ffn_stage = make_stages(
(declare_attn(), self.input_layernorm),
(
declare_ffn(
sparse=self.is_layer_sparse,
next_sparse=is_next_layer_sparse,
),
self.post_attention_layernorm,
),
previous=declare_ffn(
sparse=is_previous_layer_sparse, next_sparse=self.is_layer_sparse
)
if layer_id != 0
else None,
terminal=layer_id == cfg.num_hidden_layers - 1,
)
+56 -78
View File
@@ -22,7 +22,7 @@ import torch
import torch.nn.functional as F
from torch import nn
from sglang.srt.batch_overlap.two_batch_overlap import model_forward_maybe_tbo
from sglang.srt.batch_overlap.two_batch_overlap import model_forward_stages
from sglang.srt.configs.model_config import get_mimo_v2_fused_qkv_expected_tp_size
from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder
from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation
@@ -33,9 +33,10 @@ from sglang.srt.layers.aux_hidden_states import (
AuxHiddenStatePacker,
)
from sglang.srt.layers.communicator import (
LayerCommunicator,
LayerFacts,
declare_attn,
declare_ffn,
enable_moe_dense_fully_dp,
make_stages,
)
from sglang.srt.layers.communicator.residual import batch as residual_batch
from sglang.srt.layers.dp_attention import (
@@ -857,18 +858,21 @@ class MiMoV2DecoderLayer(nn.Module):
config.hidden_size, eps=config.layernorm_epsilon
)
self.layer_facts = LayerFacts.init_new(
layer_id=layer_id,
num_layers=config.num_hidden_layers,
is_layer_sparse=self.is_layer_sparse,
is_previous_layer_sparse=is_previous_layer_sparse,
is_next_layer_sparse=is_next_layer_sparse,
)
self.layer_communicator = LayerCommunicator(
layer_facts=self.layer_facts,
input_layernorm=self.input_layernorm,
post_attention_layernorm=self.post_attention_layernorm,
allow_reduce_scatter=True,
self.attn_stage, self.ffn_stage = make_stages(
(declare_attn(), self.input_layernorm),
(
declare_ffn(
sparse=self.is_layer_sparse,
next_sparse=is_next_layer_sparse,
),
self.post_attention_layernorm,
),
previous=declare_ffn(
sparse=is_previous_layer_sparse, next_sparse=self.is_layer_sparse
)
if layer_id != 0
else None,
terminal=layer_id == config.num_hidden_layers - 1,
)
def forward(
@@ -876,17 +880,13 @@ class MiMoV2DecoderLayer(nn.Module):
positions: torch.Tensor,
hidden_states: torch.Tensor,
forward_batch: ForwardBatch,
residual: Optional[torch.Tensor],
captured_last_layer_outputs: Optional[AuxHiddenStateAccumulator] = None,
) -> Tuple[torch.Tensor, torch.Tensor]:
) -> torch.Tensor:
# Self Attention
hidden_states, residual = (
self.layer_communicator.prepare_attn_and_capture_last_layer_outputs(
hidden_states,
residual,
forward_batch,
captured_last_layer_outputs=captured_last_layer_outputs,
)
hidden_states = self.attn_stage.prepare(
hidden_states,
forward_batch,
captured_last_layer_outputs=captured_last_layer_outputs,
)
if hidden_states.shape[0] != 0:
@@ -896,15 +896,14 @@ class MiMoV2DecoderLayer(nn.Module):
forward_batch=forward_batch,
)
hidden_states, residual = self.layer_communicator.prepare_mlp(
hidden_states, residual, forward_batch
)
hidden_states = self.attn_stage.finish(hidden_states, forward_batch)
hidden_states = self.ffn_stage.prepare(hidden_states, forward_batch)
with self.layer_communicator.ffn_exit(forward_batch) as ffn_exit:
with self.ffn_stage.exit(forward_batch) as ffn_exit:
hidden_states = self.mlp(hidden_states, forward_batch)
hidden_states, residual = ffn_exit.finish(hidden_states, residual)
hidden_states = ffn_exit.finish(hidden_states)
return hidden_states, residual
return hidden_states
def is_moe_layer(self, layer_idx: int) -> bool:
return (
@@ -923,11 +922,10 @@ class MiMoV2DecoderLayer(nn.Module):
positions: torch.Tensor,
hidden_states: torch.Tensor,
forward_batch: ForwardBatch,
residual: Optional[torch.Tensor],
tbo_subbatch_index: Optional[int] = None,
):
state.hidden_states_after_comm_pre_attn, state.residual_after_input_ln = (
self.layer_communicator.prepare_attn(hidden_states, residual, forward_batch)
state.hidden_states_after_comm_pre_attn = self.attn_stage.prepare(
hidden_states, forward_batch
)
state.update(
dict(
@@ -938,25 +936,21 @@ class MiMoV2DecoderLayer(nn.Module):
)
def op_comm_prepare_mlp(self, state):
state.hidden_states_mlp_input, state.residual_after_comm_pre_mlp = (
self.layer_communicator.prepare_mlp(
state.pop("hidden_states_after_attn"),
state.pop("residual_after_input_ln"),
state.forward_batch,
)
hidden_states = self.attn_stage.finish(
state.pop("hidden_states_after_attn"), state.forward_batch
)
state.hidden_states_mlp_input = self.ffn_stage.prepare(
hidden_states, state.forward_batch
)
def op_comm_postprocess_layer(self, state):
hidden_states, residual = self.layer_communicator.postprocess_layer(
state.pop("hidden_states_mlp_output"),
state.pop("residual_after_comm_pre_mlp"),
state.forward_batch,
hidden_states = self.ffn_stage.postprocess(
state.pop("hidden_states_mlp_output"), state.forward_batch
)
output = dict(
positions=state.positions,
hidden_states=hidden_states,
residual=residual,
forward_batch=state.forward_batch,
tbo_subbatch_index=state.tbo_subbatch_index,
)
@@ -1038,12 +1032,12 @@ class MiMoV2Model(nn.Module):
hidden_states = self.embed_tokens(input_ids)
else:
hidden_states = input_embeds
residual = residual_batch.start(forward_batch)
residual_batch.start(forward_batch)
else:
assert pp_proxy_tensors is not None
hidden_states, residual = self.layers[
self.start_layer
].layer_communicator.from_pp(pp_proxy_tensors, forward_batch)
hidden_states = self.layers[self.start_layer].attn_stage.from_pp(
pp_proxy_tensors, forward_batch
)
aux_hidden_states = AuxHiddenStatePacker(len(self.layers_to_capture))
if forward_batch.can_run_tbo and not self.layers_to_capture:
@@ -1053,39 +1047,29 @@ class MiMoV2Model(nn.Module):
# skip first layer for TBO when starting from layer 0
if self.start_layer == 0:
layer = self.layers[0]
hidden_states, residual = layer(
positions, hidden_states, forward_batch, residual
)
hidden_states = layer(positions, hidden_states, forward_batch)
tbo_start_layer = tbo_start_layer + 1
hidden_states, residual = model_forward_maybe_tbo(
hidden_states = model_forward_stages(
layers=self.layers[tbo_start_layer:tbo_end_layer],
enable_tbo=True,
positions=positions,
forward_batch=forward_batch,
hidden_states=hidden_states,
residual=residual,
)
else:
for i in range(self.start_layer, self.end_layer):
layer = self.layers[i]
hidden_states, residual = layer(
hidden_states = layer(
positions,
hidden_states,
forward_batch,
residual,
captured_last_layer_outputs=(
aux_hidden_states if i in self.layers_to_capture else None
),
captured_last_layer_outputs=aux_hidden_states
if i in self.layers_to_capture
else None,
)
last_layer = self.layers[self.end_layer - 1]
hidden_states, residual = last_layer.layer_communicator.finish_layer_stack(
hidden_states,
residual,
forward_batch,
preserve_declared=not self.pp_group.is_last_rank,
)
hidden_states = residual_batch.finish(hidden_states, forward_batch)
# A draft targeting the final layer ("after layer
# num_hidden_layers-1") maps to capture index num_hidden_layers,
@@ -1095,27 +1079,21 @@ class MiMoV2Model(nn.Module):
and self.config.num_hidden_layers in self.layers_to_capture
):
aux_hidden_states.append(
last_layer.layer_communicator.snapshot(hidden_states, residual)
residual_batch.snapshot(hidden_states, forward_batch)
)
hidden_states_before_norm = None
if not self.pp_group.is_last_rank:
return PPProxyTensors(
{
"hidden_states": hidden_states,
"residual": residual,
}
)
return residual_batch.to_pp(hidden_states, forward_batch)
else:
if hidden_states.shape[0] > 0:
if forward_batch.return_hidden_states_before_norm:
hidden_states_before_norm = last_layer.layer_communicator.snapshot(
hidden_states, residual
hidden_states_before_norm = residual_batch.snapshot(
hidden_states, forward_batch
)
if residual is None:
hidden_states = self.norm(hidden_states)
else:
hidden_states, _ = self.norm(hidden_states, residual)
hidden_states = residual_batch.norm(
hidden_states, forward_batch, self.norm
)
if len(aux_hidden_states) == 0:
return hidden_states, hidden_states_before_norm
+34 -36
View File
@@ -22,9 +22,10 @@ from transformers import PretrainedConfig
from sglang.srt.configs.model_config import get_mimo_v2_fused_qkv_expected_tp_size
from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder
from sglang.srt.layers.communicator import (
LayerCommunicator,
LayerFacts,
declare_attn,
declare_ffn,
enable_moe_dense_fully_dp,
make_stages,
)
from sglang.srt.layers.communicator.residual import batch as residual_batch
from sglang.srt.layers.dp_attention import (
@@ -117,18 +118,22 @@ class MiMoV2MTPLayer(nn.Module):
self.post_attention_layernorm = RMSNorm(
config.hidden_size, eps=config.layernorm_epsilon
)
self.layer_facts = LayerFacts.init_new(
layer_id=layer_id,
num_layers=1,
is_layer_sparse=self.is_layer_sparse,
is_previous_layer_sparse=is_previous_layer_sparse,
is_next_layer_sparse=is_next_layer_sparse,
)
self.layer_communicator = LayerCommunicator(
layer_facts=self.layer_facts,
input_layernorm=self.input_layernorm,
post_attention_layernorm=self.post_attention_layernorm,
allow_deferred_ffn_reduction=False,
self.attn_stage, self.ffn_stage = make_stages(
(declare_attn(), self.input_layernorm),
(
declare_ffn(
sparse=self.is_layer_sparse,
next_sparse=is_next_layer_sparse,
),
self.post_attention_layernorm,
),
previous=declare_ffn(
sparse=is_previous_layer_sparse, next_sparse=self.is_layer_sparse
)
if layer_id != 0
else None,
terminal=layer_id == 1 - 1,
)
def forward(
@@ -136,12 +141,9 @@ class MiMoV2MTPLayer(nn.Module):
positions: torch.Tensor,
hidden_states: torch.Tensor,
forward_batch: ForwardBatch,
residual: Optional[torch.Tensor],
) -> Tuple[torch.Tensor, torch.Tensor]:
) -> torch.Tensor:
hidden_states, residual = self.layer_communicator.prepare_attn(
hidden_states, residual, forward_batch
)
hidden_states = self.attn_stage.prepare(hidden_states, forward_batch)
if hidden_states.shape[0] != 0:
hidden_states = self.self_attn(
@@ -150,17 +152,16 @@ class MiMoV2MTPLayer(nn.Module):
forward_batch=forward_batch,
)
hidden_states, residual = self.layer_communicator.prepare_mlp(
hidden_states, residual, forward_batch
)
hidden_states = self.attn_stage.finish(hidden_states, forward_batch)
hidden_states = self.ffn_stage.prepare(hidden_states, forward_batch)
with (
self.layer_communicator.ffn_exit(forward_batch) as ffn_exit,
self.ffn_stage.exit(forward_batch) as ffn_exit,
get_global_expert_distribution_recorder().disable_this_region(),
):
hidden_states = self.mlp(hidden_states)
hidden_states, residual = ffn_exit.finish(hidden_states, residual)
hidden_states = ffn_exit.finish(hidden_states)
return hidden_states, residual
return hidden_states
class MiMoV2ModelNextN(nn.Module):
@@ -220,26 +221,23 @@ class MiMoV2ModelNextN(nn.Module):
dim=-1,
)
)
hidden_states, residual = self.mtp_block(
residual_batch.start(forward_batch)
hidden_states = self.mtp_block(
positions=positions,
hidden_states=hidden_states,
forward_batch=forward_batch,
residual=residual_batch.start(forward_batch),
)
hidden_states, residual = self.mtp_block.layer_communicator.finish_layer_stack(
hidden_states, residual, forward_batch
)
hidden_states = residual_batch.finish(hidden_states, forward_batch)
hidden_states_before_norm = None
if not forward_batch.forward_mode.is_idle():
if forward_batch.return_hidden_states_before_norm:
hidden_states_before_norm = self.mtp_block.layer_communicator.snapshot(
hidden_states, residual
hidden_states_before_norm = residual_batch.snapshot(
hidden_states, forward_batch
)
if residual is not None:
hidden_states, _ = self.final_layernorm(hidden_states, residual)
else:
hidden_states = self.final_layernorm(hidden_states)
hidden_states = residual_batch.norm(
hidden_states, forward_batch, self.final_layernorm
)
return hidden_states, hidden_states_before_norm
+49 -68
View File
@@ -27,7 +27,7 @@ from torch import nn
from transformers import PretrainedConfig
from sglang.kernels.kernel_api_logging import debug_kernel_api
from sglang.srt.batch_overlap.two_batch_overlap import model_forward_maybe_tbo
from sglang.srt.batch_overlap.two_batch_overlap import model_forward_stages
from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder
from sglang.srt.eplb.expert_location_dispatch import ExpertLocationDispatchInfo
from sglang.srt.layers.aux_hidden_states import (
@@ -35,8 +35,9 @@ from sglang.srt.layers.aux_hidden_states import (
AuxHiddenStateList,
)
from sglang.srt.layers.communicator import (
LayerCommunicator,
LayerFacts,
declare_attn,
declare_ffn,
make_stages,
)
from sglang.srt.layers.communicator.residual import batch as residual_batch
from sglang.srt.layers.dp_attention import (
@@ -980,19 +981,22 @@ class MiniMaxM2DecoderLayer(nn.Module):
is_previous_layer_sparse = True
is_next_layer_sparse = True
self.layer_facts = LayerFacts.init_new(
layer_id=layer_id,
num_layers=config.num_hidden_layers,
is_layer_sparse=self.is_layer_sparse,
is_previous_layer_sparse=is_previous_layer_sparse,
is_next_layer_sparse=is_next_layer_sparse,
)
self.layer_communicator = LayerCommunicator(
layer_facts=self.layer_facts,
input_layernorm=self.input_layernorm,
post_attention_layernorm=self.post_attention_layernorm,
allow_reduce_scatter=True,
self.attn_stage, self.ffn_stage = make_stages(
(declare_attn(), self.input_layernorm),
(
declare_ffn(
sparse=self.is_layer_sparse,
next_sparse=is_next_layer_sparse,
),
self.post_attention_layernorm,
),
previous=declare_ffn(
sparse=is_previous_layer_sparse, next_sparse=self.is_layer_sparse
)
if layer_id != 0
else None,
terminal=layer_id == config.num_hidden_layers - 1,
)
def forward(
@@ -1000,17 +1004,13 @@ class MiniMaxM2DecoderLayer(nn.Module):
positions: torch.Tensor,
hidden_states: torch.Tensor,
forward_batch: ForwardBatch,
residual: Optional[torch.Tensor],
captured_last_layer_outputs: Optional[AuxHiddenStateAccumulator] = None,
) -> torch.Tensor:
# Self Attention
hidden_states, residual = (
self.layer_communicator.prepare_attn_and_capture_last_layer_outputs(
hidden_states,
residual,
forward_batch,
captured_last_layer_outputs=captured_last_layer_outputs,
)
hidden_states = self.attn_stage.prepare(
hidden_states,
forward_batch,
captured_last_layer_outputs=captured_last_layer_outputs,
)
if not forward_batch.forward_mode.is_idle():
hidden_states = self.self_attn(
@@ -1021,15 +1021,14 @@ class MiniMaxM2DecoderLayer(nn.Module):
# Fully Connected (MLP or MoE)
hidden_states, residual = self.layer_communicator.prepare_mlp(
hidden_states, residual, forward_batch
)
hidden_states = self.attn_stage.finish(hidden_states, forward_batch)
hidden_states = self.ffn_stage.prepare(hidden_states, forward_batch)
with self.layer_communicator.ffn_exit(forward_batch) as ffn_exit:
with self.ffn_stage.exit(forward_batch) as ffn_exit:
hidden_states = self.block_sparse_moe(hidden_states, forward_batch)
hidden_states, residual = ffn_exit.finish(hidden_states, residual)
hidden_states = ffn_exit.finish(hidden_states)
return hidden_states, residual
return hidden_states
# TBO Operations for MiniMax Decoder Layer
def op_comm_prepare_attn(
@@ -1038,13 +1037,12 @@ class MiniMaxM2DecoderLayer(nn.Module):
positions: torch.Tensor,
hidden_states: torch.Tensor,
forward_batch: ForwardBatch,
residual: Optional[torch.Tensor],
zero_allocator: BumpAllocator,
tbo_subbatch_index: Optional[int] = None,
):
"""Communication prepare for attention - TBO operation"""
state.hidden_states_after_comm_pre_attn, state.residual_after_input_ln = (
self.layer_communicator.prepare_attn(hidden_states, residual, forward_batch)
state.hidden_states_after_comm_pre_attn = self.attn_stage.prepare(
hidden_states, forward_batch
)
state.update(
dict(
@@ -1057,26 +1055,22 @@ class MiniMaxM2DecoderLayer(nn.Module):
def op_comm_prepare_mlp(self, state):
"""Communication prepare for MLP - TBO operation"""
state.hidden_states_mlp_input, state.residual_after_comm_pre_mlp = (
self.layer_communicator.prepare_mlp(
state.pop("hidden_states_after_attn"),
state.pop("residual_after_input_ln"),
state.forward_batch,
)
hidden_states = self.attn_stage.finish(
state.pop("hidden_states_after_attn"), state.forward_batch
)
state.hidden_states_mlp_input = self.ffn_stage.prepare(
hidden_states, state.forward_batch
)
def op_comm_postprocess_layer(self, state):
"""Communication postprocess for layer - TBO operation"""
hidden_states, residual = self.layer_communicator.postprocess_layer(
state.pop("hidden_states_mlp_output"),
state.pop("residual_after_comm_pre_mlp"),
state.forward_batch,
hidden_states = self.ffn_stage.postprocess(
state.pop("hidden_states_mlp_output"), state.forward_batch
)
output = dict(
positions=state.positions,
hidden_states=hidden_states,
residual=residual,
forward_batch=state.forward_batch,
zero_allocator=state.zero_allocator,
tbo_subbatch_index=state.tbo_subbatch_index,
@@ -1146,22 +1140,21 @@ class MiniMaxM2Model(nn.Module):
hidden_states = self.get_input_embeddings(input_ids)
else:
hidden_states = input_embeds
residual = residual_batch.start(forward_batch)
residual_batch.start(forward_batch)
else:
assert pp_proxy_tensors is not None
hidden_states, residual = self.layers[
self.start_layer
].layer_communicator.from_pp(pp_proxy_tensors, forward_batch)
hidden_states = self.layers[self.start_layer].attn_stage.from_pp(
pp_proxy_tensors, forward_batch
)
aux_hidden_states = AuxHiddenStateList()
if forward_batch.can_run_tbo:
hidden_states, residual = model_forward_maybe_tbo(
hidden_states = model_forward_stages(
layers=self.layers,
enable_tbo=True,
positions=positions,
forward_batch=forward_batch,
hidden_states=hidden_states,
residual=residual,
)
else:
for i in range(self.start_layer, self.end_layer):
@@ -1172,33 +1165,21 @@ class MiniMaxM2Model(nn.Module):
)
with ctx:
layer = self.layers[i]
hidden_states, residual = layer(
hidden_states = layer(
positions=positions,
forward_batch=forward_batch,
hidden_states=hidden_states,
residual=residual,
captured_last_layer_outputs=(
aux_hidden_states if i in self.layers_to_capture else None
),
captured_last_layer_outputs=aux_hidden_states
if i in self.layers_to_capture
else None,
)
last_layer = self.layers[self.end_layer - 1]
hidden_states, residual = last_layer.layer_communicator.finish_layer_stack(
hidden_states,
residual,
forward_batch,
preserve_declared=not self.pp_group.is_last_rank,
)
if not self.pp_group.is_last_rank:
return PPProxyTensors(
{"hidden_states": hidden_states, "residual": residual}
)
return residual_batch.to_pp(hidden_states, forward_batch)
hidden_states = residual_batch.finish(hidden_states, forward_batch)
if hidden_states.shape[0] != 0:
if residual is not None:
hidden_states, _ = self.norm(hidden_states, residual)
else:
hidden_states = self.norm(hidden_states)
hidden_states = residual_batch.norm(hidden_states, forward_batch, self.norm)
if len(aux_hidden_states) == 0:
return hidden_states
+41 -57
View File
@@ -23,7 +23,7 @@ import torch
from torch import nn
from transformers import PretrainedConfig
from sglang.srt.batch_overlap.two_batch_overlap import model_forward_maybe_tbo
from sglang.srt.batch_overlap.two_batch_overlap import model_forward_stages
from sglang.srt.configs.model_config import (
get_minimax_sparse_disable_value_layer_ids,
get_minimax_sparse_layer_ids,
@@ -40,9 +40,10 @@ from sglang.srt.layers.aux_hidden_states import (
AuxHiddenStateList,
)
from sglang.srt.layers.communicator import (
LayerCommunicator,
LayerFacts,
declare_attn,
declare_ffn,
enable_moe_dense_fully_dp,
make_stages,
)
from sglang.srt.layers.communicator.residual import batch as residual_batch
from sglang.srt.layers.dp_attention import (
@@ -1302,7 +1303,7 @@ class MiniMaxM3DecoderLayer(nn.Module):
moe_layer_freq = getattr(config, "moe_layer_freq", None)
# Means "MLP is a sparse MoE", not attention sparsity. Kept as ``is_layer_sparse``
# because LayerCommunicator / LayerFacts / other models read this attr.
# because model construction and downstream integrations read this attr.
self.is_layer_sparse = (
moe_layer_freq[layer_id] != 0 if moe_layer_freq is not None else True
)
@@ -1352,19 +1353,22 @@ class MiniMaxM3DecoderLayer(nn.Module):
is_previous_layer_sparse = _is_layer_sparse(layer_id - 1)
is_next_layer_sparse = _is_layer_sparse(layer_id + 1)
self.layer_facts = LayerFacts.init_new(
layer_id=layer_id,
num_layers=config.num_hidden_layers,
is_layer_sparse=self.is_layer_sparse,
is_previous_layer_sparse=is_previous_layer_sparse,
is_next_layer_sparse=is_next_layer_sparse,
)
self.layer_communicator = LayerCommunicator(
layer_facts=self.layer_facts,
input_layernorm=self.input_layernorm,
post_attention_layernorm=self.post_attention_layernorm,
allow_reduce_scatter=True,
self.attn_stage, self.ffn_stage = make_stages(
(declare_attn(), self.input_layernorm),
(
declare_ffn(
sparse=self.is_layer_sparse,
next_sparse=is_next_layer_sparse,
),
self.post_attention_layernorm,
),
previous=declare_ffn(
sparse=is_previous_layer_sparse, next_sparse=self.is_layer_sparse
)
if layer_id != 0
else None,
terminal=layer_id == config.num_hidden_layers - 1,
)
def forward(
@@ -1372,18 +1376,14 @@ class MiniMaxM3DecoderLayer(nn.Module):
positions: torch.Tensor,
hidden_states: torch.Tensor,
forward_batch: ForwardBatch,
residual: Optional[torch.Tensor],
captured_last_layer_outputs: Optional[AuxHiddenStateAccumulator] = None,
**kwargs,
) -> torch.Tensor:
hidden_states, residual = (
self.layer_communicator.prepare_attn_and_capture_last_layer_outputs(
hidden_states,
residual,
forward_batch,
captured_last_layer_outputs=captured_last_layer_outputs,
**kwargs,
)
hidden_states = self.attn_stage.prepare(
hidden_states,
forward_batch,
captured_last_layer_outputs=captured_last_layer_outputs,
**kwargs,
)
if hidden_states.shape[0] != 0:
@@ -1393,11 +1393,10 @@ class MiniMaxM3DecoderLayer(nn.Module):
forward_batch=forward_batch,
)
hidden_states, residual = self.layer_communicator.prepare_mlp(
hidden_states, residual, forward_batch
)
hidden_states = self.attn_stage.finish(hidden_states, forward_batch)
hidden_states = self.ffn_stage.prepare(hidden_states, forward_batch)
with self.layer_communicator.ffn_exit(forward_batch) as ffn_exit:
with self.ffn_stage.exit(forward_batch) as ffn_exit:
if self.is_layer_sparse or hidden_states.shape[0] != 0:
hidden_states = self.mlp(
hidden_states,
@@ -1405,7 +1404,7 @@ class MiniMaxM3DecoderLayer(nn.Module):
should_allreduce_fusion=ffn_exit.fuse_mlp_allreduce,
use_reduce_scatter=ffn_exit.mlp_reduce_scatter,
)
return ffn_exit.finish(hidden_states, residual)
return ffn_exit.finish(hidden_states)
class MiniMaxM3Model(nn.Module):
@@ -1481,22 +1480,21 @@ class MiniMaxM3Model(nn.Module):
hidden_states = embeds(input_ids)
else:
hidden_states = input_embeds
residual = residual_batch.start(forward_batch)
residual_batch.start(forward_batch)
else:
assert pp_proxy_tensors is not None
hidden_states, residual = self.layers[
self.start_layer
].layer_communicator.from_pp(pp_proxy_tensors, forward_batch)
hidden_states = self.layers[self.start_layer].attn_stage.from_pp(
pp_proxy_tensors, forward_batch
)
aux_hidden_states = AuxHiddenStateList()
if forward_batch.can_run_tbo:
hidden_states, residual = model_forward_maybe_tbo(
hidden_states = model_forward_stages(
layers=self.layers,
enable_tbo=True,
positions=positions,
forward_batch=forward_batch,
hidden_states=hidden_states,
residual=residual,
)
else:
for i in range(self.start_layer, self.end_layer):
@@ -1508,34 +1506,20 @@ class MiniMaxM3Model(nn.Module):
)
with ctx:
layer = self.layers[i]
hidden_states, residual = layer(
hidden_states = layer(
positions=positions,
forward_batch=forward_batch,
hidden_states=hidden_states,
residual=residual,
captured_last_layer_outputs=(
aux_hidden_states
if getattr(layer, "_is_layer_to_capture", False)
else None
),
captured_last_layer_outputs=aux_hidden_states
if getattr(layer, "_is_layer_to_capture", False)
else None,
)
last_layer = self.layers[self.end_layer - 1]
hidden_states, residual = last_layer.layer_communicator.finish_layer_stack(
hidden_states,
residual,
forward_batch,
preserve_declared=not self.pp_group.is_last_rank,
)
if not self.pp_group.is_last_rank:
return PPProxyTensors(
{"hidden_states": hidden_states, "residual": residual}
)
return residual_batch.to_pp(hidden_states, forward_batch)
hidden_states = residual_batch.finish(hidden_states, forward_batch)
if hidden_states.shape[0] != 0:
if residual is not None:
hidden_states, _ = self.norm(hidden_states, residual)
else:
hidden_states = self.norm(hidden_states)
hidden_states = residual_batch.norm(hidden_states, forward_batch, self.norm)
if len(aux_hidden_states) == 0:
return hidden_states
+26 -43
View File
@@ -22,7 +22,11 @@ from sglang.srt.layers.attention.vision import (
VisionAttentionMetadata,
prepare_vision_attention_metadata,
)
from sglang.srt.layers.communicator import LayerCommunicator, LayerFacts
from sglang.srt.layers.communicator import (
declare_attn,
declare_ffn,
make_stages,
)
from sglang.srt.layers.communicator.residual import batch as residual_batch
from sglang.srt.layers.conv import Conv3dLayer
from sglang.srt.layers.layernorm import RMSNorm
@@ -1012,18 +1016,15 @@ class MossVLSelfAttentionDecoderLayer(nn.Module):
self.post_attention_layernorm = RMSNorm(
config.hidden_size, eps=config.rms_norm_eps, **norm_kwargs
)
self.layer_facts = LayerFacts.init_new(
layer_id=layer_id,
num_layers=config.num_hidden_layers,
is_layer_sparse=False,
is_previous_layer_sparse=False,
is_next_layer_sparse=False,
)
self.layer_communicator = LayerCommunicator(
layer_facts=self.layer_facts,
input_layernorm=self.input_layernorm,
post_attention_layernorm=self.post_attention_layernorm,
allow_deferred_ffn_reduction=False,
self.attn_stage, self.ffn_stage = make_stages(
(declare_attn(), self.input_layernorm),
(
declare_ffn(sparse=False, next_sparse=False),
self.post_attention_layernorm,
),
previous=declare_ffn() if layer_id != 0 else None,
terminal=layer_id == config.num_hidden_layers - 1,
)
def forward(
@@ -1031,12 +1032,9 @@ class MossVLSelfAttentionDecoderLayer(nn.Module):
positions: torch.Tensor,
hidden_states: torch.Tensor,
forward_batch: ForwardBatch,
residual: Optional[torch.Tensor],
) -> Tuple[torch.Tensor, torch.Tensor]:
) -> torch.Tensor:
# Self Attention
hidden_states, residual = self.layer_communicator.prepare_attn(
hidden_states, residual, forward_batch
)
hidden_states = self.attn_stage.prepare(hidden_states, forward_batch)
if hidden_states.shape[0] != 0:
hidden_states = self.self_attn(
positions=positions,
@@ -1045,15 +1043,12 @@ class MossVLSelfAttentionDecoderLayer(nn.Module):
)
# MLP
hidden_states, residual = self.layer_communicator.prepare_mlp(
hidden_states,
residual,
forward_batch,
)
with self.layer_communicator.ffn_exit(forward_batch) as ffn_exit:
hidden_states = self.attn_stage.finish(hidden_states, forward_batch)
hidden_states = self.ffn_stage.prepare(hidden_states, forward_batch)
with self.ffn_stage.exit(forward_batch) as ffn_exit:
hidden_states = self.mlp(hidden_states)
hidden_states, residual = ffn_exit.finish(hidden_states, residual)
return hidden_states, residual
hidden_states = ffn_exit.finish(hidden_states)
return hidden_states
# ==================== Text Model ====================
@@ -1110,14 +1105,12 @@ class MossVLTextModel(nn.Module):
vision_position_ids: Optional[torch.Tensor] = None,
) -> torch.Tensor:
hidden_states = self.embed_tokens(input_ids)
residual = residual_batch.start(forward_batch)
residual_batch.start(forward_batch)
for decoder_layer in self.layers:
if isinstance(decoder_layer, MossVLCrossAttentionDecoderLayer):
if not skip_cross_attention:
hidden_states, residual = LayerCommunicator.fold(
hidden_states, residual
)
hidden_states = residual_batch.fold(hidden_states, forward_batch)
hidden_states = decoder_layer(
hidden_states=hidden_states,
cross_attention_states=cross_attention_states,
@@ -1127,27 +1120,17 @@ class MossVLTextModel(nn.Module):
positions=positions,
vision_position_ids=vision_position_ids,
)
hidden_states, residual = LayerCommunicator.written(
hidden_states, forward_batch
)
hidden_states = residual_batch.written(hidden_states, forward_batch)
elif isinstance(decoder_layer, MossVLSelfAttentionDecoderLayer):
hidden_states, residual = decoder_layer(
hidden_states = decoder_layer(
positions=positions,
hidden_states=hidden_states,
forward_batch=forward_batch,
residual=residual,
)
else:
raise ValueError(f"Unknown decoder layer type {type(decoder_layer)}")
hidden_states, residual = LayerCommunicator.finish_layer_stack(
hidden_states, residual, forward_batch
)
if residual is not None:
hidden_states, _ = self.norm(hidden_states, residual)
else:
hidden_states = self.norm(hidden_states)
hidden_states = residual_batch.norm(hidden_states, forward_batch, self.norm)
return hidden_states
+1 -1
View File
@@ -738,7 +738,7 @@ class NemotronHModel(nn.Module):
for i in range(self.start_layer, self.end_layer):
if i in self.layers_to_capture:
hidden_states, snapshot = self.layers[i].stage.capture_output(
hidden_states, forward_batch, at_input=True
hidden_states, forward_batch
)
aux_hidden_states.append(snapshot)
layer = self.layers[i]
+27 -89
View File
@@ -4,22 +4,12 @@ from typing import Optional
from sglang.srt.configs.nemotron_h import ATTENTION, MAMBA, MOE
from sglang.srt.layers.communicator import (
NORM_QUANT_READ,
LayerCommunicator,
LayerFacts,
LayerStage,
Layout,
StageDecl,
StageInput,
StageKind,
StageOutput,
SumGroup,
TokenAxis,
stage_edges,
token_axis_sizes,
declare_attn,
declare_ffn,
make_stages,
)
from sglang.srt.layers.layernorm import RMSNorm
from sglang.srt.layers.moe.utils import get_moe_a2a_backend
ATTN_LAYERS = (MAMBA, ATTENTION)
@@ -37,87 +27,35 @@ def _stage_kind(pattern: str, layer_idx: int) -> Optional[StageKind]:
return StageKind.ATTENTION if is_attn_layer(pattern[layer_idx]) else StageKind.FFN
def _stage_decl(pattern: str, layer_idx: int) -> StageDecl:
"""What the stage at ``layer_idx`` declares. A mixer reads its input as
an attention does, computes on the attention's rows and leaves its
attention-TP sum to an FFN stage after it, or, when a fused kernel takes
it, to a mixer after it. An FFN computes on the TP group's rows, a MoE
dispatched by an a2a backend on this rank's own, and may leave its sum to a
mixer after it."""
axis_sizes = token_axis_sizes()
attention = Layout.sharded_over(
TokenAxis.ATTN_DP, TokenAxis.ATTN_CP, axis_sizes=axis_sizes
)
def _declaration(pattern: str, layer_idx: int):
following = _stage_kind(pattern, layer_idx + 1)
if is_attn_layer(pattern[layer_idx]):
owes = axis_sizes[TokenAxis.ATTN_TP_SCATTER] > 1
return StageDecl(
StageInput(attention, read=NORM_QUANT_READ),
StageOutput(
attention,
group=SumGroup.ATTN_TP if owes else None,
always_leaves=owes and following is StageKind.FFN,
leaves_for_next_layer=owes and following is StageKind.ATTENTION,
),
return declare_attn(
mixer_exit=True,
next_kind=following,
ordinary_only=True,
)
sparse = pattern[layer_idx] == MOE
if sparse and not get_moe_a2a_backend().is_none():
local = Layout.sharded_over(
TokenAxis.ATTN_DP,
TokenAxis.ATTN_CP,
TokenAxis.ATTN_TP_SCATTER,
axis_sizes=axis_sizes,
)
return StageDecl(StageInput(local), StageOutput(local))
full = Layout.sharded_over(axis_sizes=axis_sizes)
return StageDecl(
StageInput(full),
StageOutput(
full,
group=SumGroup.MOE_OUTPUT if sparse else SumGroup.TP,
leaves_for_next_layer=following is StageKind.ATTENTION,
leaves_for_reduce_scatter=True,
leaves_for_reduce_scatterv=True,
),
)
def layer_stage(pattern: str, layer_idx: int) -> LayerStage:
"""The layer's stage and its two boundaries, from its own declaration and
the previous stage's."""
rows = Layout.sharded_over(
TokenAxis.ATTN_DP, TokenAxis.ATTN_CP, axis_sizes=token_axis_sizes()
)
return LayerStage(
kind=_stage_kind(pattern, layer_idx),
edges=stage_edges(
previous=(
_stage_decl(pattern, layer_idx - 1).output if layer_idx > 0 else None
),
stage=_stage_decl(pattern, layer_idx),
rows=rows,
),
enters_stack=layer_idx == 0,
return declare_ffn(
sparse=pattern[layer_idx] == MOE,
ordinary_only=True,
return_to_attention=True,
)
def make_stage_boundary(layer_norm: RMSNorm, *, pattern: str, layer_idx: int):
"""The communicator of a layer that is one stage: only its own norm, and
boundaries built from the stages next to it in the pattern."""
stage = layer_stage(pattern, layer_idx)
for_attn = stage.kind is StageKind.ATTENTION
boundaries = LayerCommunicator(
layer_facts=LayerFacts(
is_layer_sparse=pattern[layer_idx] == MOE,
is_last_layer=layer_idx == len(pattern) - 1,
),
input_layernorm=layer_norm if for_attn else None,
post_attention_layernorm=None if for_attn else layer_norm,
# With attention TP > 1, the default gather adds the residual to one
# rank's partial in bf16 before the cross-rank sum.
force_layernorm_before_dp_gather=True,
allow_reduce_scatter=not for_attn,
stage=stage,
)
from sglang.srt.layers import layernorm_sp
from sglang.srt.runtime_context import get_parallel
return boundaries.attn if for_attn else boundaries.ffn
if get_parallel().attn_cp_size > 1 or layernorm_sp.layernorm_sp_enabled():
raise NotImplementedError("a Nemotron stage with attention CP or LayerNorm SP")
previous = _declaration(pattern, layer_idx - 1) if layer_idx > 0 else None
(boundary,) = make_stages(
(
_declaration(pattern, layer_idx),
layer_norm,
{"force_layernorm_before_dp_gather": True},
),
previous=previous,
terminal=layer_idx == len(pattern) - 1,
)
return boundary
+51 -75
View File
@@ -31,7 +31,7 @@ from sglang.kernels.ops.elementwise.elementwise import (
fused_gate_sigmoid_mul,
fused_gate_sigmoid_mul_add,
)
from sglang.srt.batch_overlap.two_batch_overlap import model_forward_maybe_tbo
from sglang.srt.batch_overlap.two_batch_overlap import model_forward_stages
from sglang.srt.distributed import (
get_pp_indices,
)
@@ -45,8 +45,9 @@ from sglang.srt.layers.aux_hidden_states import (
AuxHiddenStateList,
)
from sglang.srt.layers.communicator import (
LayerCommunicator,
LayerFacts,
declare_attn,
declare_ffn,
make_stages,
)
from sglang.srt.layers.communicator.residual import batch as residual_batch
from sglang.srt.layers.dp_attention import (
@@ -1032,14 +1033,6 @@ class Qwen2MoeDecoderLayer(nn.Module):
is_previous_layer_sparse = True
is_next_layer_sparse = True
self.layer_facts = LayerFacts.init_new(
layer_id=layer_id,
num_layers=config.num_hidden_layers,
is_layer_sparse=self.is_layer_sparse,
is_previous_layer_sparse=is_previous_layer_sparse,
is_next_layer_sparse=is_next_layer_sparse,
)
if self.is_layer_sparse:
self.mlp = Qwen2MoeSparseMoeBlock(
layer_id=layer_id,
@@ -1060,12 +1053,21 @@ class Qwen2MoeDecoderLayer(nn.Module):
self.post_attention_layernorm = RMSNorm(
config.hidden_size, eps=config.rms_norm_eps
)
self.layer_communicator = LayerCommunicator(
layer_facts=self.layer_facts,
input_layernorm=self.input_layernorm,
post_attention_layernorm=self.post_attention_layernorm,
allow_reduce_scatter=True,
allow_deferred_ffn_reduction=False,
self.attn_stage, self.ffn_stage = make_stages(
(declare_attn(), self.input_layernorm),
(
declare_ffn(
sparse=self.is_layer_sparse,
next_sparse=is_next_layer_sparse,
),
self.post_attention_layernorm,
),
previous=declare_ffn(
sparse=is_previous_layer_sparse, next_sparse=self.is_layer_sparse
)
if layer_id != 0
else None,
terminal=layer_id == config.num_hidden_layers - 1,
)
def forward(
@@ -1073,19 +1075,15 @@ class Qwen2MoeDecoderLayer(nn.Module):
positions: torch.Tensor,
hidden_states: torch.Tensor,
forward_batch: ForwardBatch,
residual: Optional[torch.Tensor],
captured_last_layer_outputs: Optional[AuxHiddenStateAccumulator] = None,
**kwargs,
) -> Tuple[torch.Tensor, torch.Tensor]:
) -> torch.Tensor:
hidden_states, residual = (
self.layer_communicator.prepare_attn_and_capture_last_layer_outputs(
hidden_states,
residual,
forward_batch,
captured_last_layer_outputs=captured_last_layer_outputs,
**kwargs,
)
hidden_states = self.attn_stage.prepare(
hidden_states,
forward_batch,
captured_last_layer_outputs=captured_last_layer_outputs,
**kwargs,
)
if hidden_states.shape[0] != 0:
@@ -1095,15 +1093,14 @@ class Qwen2MoeDecoderLayer(nn.Module):
forward_batch=forward_batch,
)
hidden_states, residual = self.layer_communicator.prepare_mlp(
hidden_states, residual, forward_batch
)
hidden_states = self.attn_stage.finish(hidden_states, forward_batch)
hidden_states = self.ffn_stage.prepare(hidden_states, forward_batch)
with self.layer_communicator.ffn_exit(forward_batch) as ffn_exit:
with self.ffn_stage.exit(forward_batch) as ffn_exit:
hidden_states = self.mlp(hidden_states, forward_batch)
hidden_states, residual = ffn_exit.finish(hidden_states, residual)
hidden_states = ffn_exit.finish(hidden_states)
return hidden_states, residual
return hidden_states
class Qwen2MoeModel(nn.Module):
@@ -1180,22 +1177,21 @@ class Qwen2MoeModel(nn.Module):
hidden_states = self.embed_tokens(input_ids)
else:
hidden_states = input_embeds
residual = residual_batch.start(forward_batch)
residual_batch.start(forward_batch)
else:
assert pp_proxy_tensors is not None
hidden_states, residual = self.layers[
self.start_layer
].layer_communicator.from_pp(pp_proxy_tensors, forward_batch)
hidden_states = self.layers[self.start_layer].attn_stage.from_pp(
pp_proxy_tensors, forward_batch
)
aux_hidden_states = AuxHiddenStateList()
if forward_batch.can_run_tbo:
hidden_states, residual = model_forward_maybe_tbo(
hidden_states = model_forward_stages(
layers=self.layers,
enable_tbo=True,
positions=positions,
forward_batch=forward_batch,
hidden_states=hidden_states,
residual=residual,
)
else:
for i in range(self.start_layer, self.end_layer):
@@ -1206,38 +1202,23 @@ class Qwen2MoeModel(nn.Module):
)
with ctx:
layer = self.layers[i]
hidden_states, residual = layer(
hidden_states = layer(
positions,
hidden_states,
forward_batch,
residual,
captured_last_layer_outputs=(
aux_hidden_states
if getattr(layer, "_is_layer_to_capture", False)
else None
),
captured_last_layer_outputs=aux_hidden_states
if getattr(layer, "_is_layer_to_capture", False)
else None,
)
last_layer = self.layers[self.end_layer - 1]
hidden_states, residual = last_layer.layer_communicator.finish_layer_stack(
hidden_states,
residual,
forward_batch,
preserve_declared=not self.pp_group.is_last_rank,
)
if not self.pp_group.is_last_rank:
return PPProxyTensors(
{
"hidden_states": hidden_states,
"residual": residual,
}
)
return residual_batch.to_pp(hidden_states, forward_batch)
else:
hidden_states = residual_batch.finish(hidden_states, forward_batch)
if hidden_states.shape[0] != 0:
if residual is None:
hidden_states = self.norm(hidden_states)
else:
hidden_states, _ = self.norm(hidden_states, residual)
hidden_states = residual_batch.norm(
hidden_states, forward_batch, self.norm
)
if len(aux_hidden_states) == 0:
return hidden_states
@@ -1314,7 +1295,7 @@ class Qwen2MoeForCausalLM(nn.Module):
start, end = split_interval
# embed
if start == 0:
forward_batch.residual = residual_batch.start(forward_batch)
residual_batch.start(forward_batch)
if input_embeds is None:
forward_batch.hidden_states = self.model.embed_tokens(input_ids)
else:
@@ -1324,22 +1305,17 @@ class Qwen2MoeForCausalLM(nn.Module):
for i in range(start, end):
with get_global_expert_distribution_recorder().with_current_layer(i):
layer = self.model.layers[i]
forward_batch.hidden_states, forward_batch.residual = layer(
positions,
forward_batch.hidden_states,
forward_batch,
forward_batch.residual,
forward_batch.hidden_states = layer(
positions, forward_batch.hidden_states, forward_batch
)
if end == self.model.config.num_hidden_layers:
forward_batch.hidden_states, forward_batch.residual = self.model.layers[
end - 1
].layer_communicator.finish_layer_stack(
forward_batch.hidden_states, forward_batch.residual, forward_batch
forward_batch.hidden_states = residual_batch.finish(
forward_batch.hidden_states, forward_batch
)
# norm
hidden_states, _ = self.model.norm(
forward_batch.hidden_states, forward_batch.residual
hidden_states = residual_batch.norm(
forward_batch.hidden_states, forward_batch, self.model.norm
)
forward_batch.residual = None
forward_batch.hidden_states = hidden_states
+90 -117
View File
@@ -40,18 +40,17 @@ from sglang.srt.environ import envs
from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder
from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation
from sglang.srt.layers.attention.mamba.mamba import mamba_v2_sharded_weight_loader
# Layers - Attention
from sglang.srt.layers.aux_hidden_states import (
AuxHiddenStateAccumulator,
AuxHiddenStateList,
)
from sglang.srt.layers.communicator import (
LayerCommunicator,
LayerFacts,
declare_attn,
declare_ffn,
make_stages,
)
# Configs
# Layers - Attention
from sglang.srt.layers.communicator.residual import batch as residual_batch
from sglang.srt.layers.dp_attention import (
is_dp_attention_enabled,
@@ -271,7 +270,7 @@ def _enable_qwen35_fused_ar_quant() -> bool:
(AR → RMSNorm → per-group quant) with either a single fused kernel (when
the fully-fused variant is eligible) or a 2-kernel path
(fused AR+RMSNorm + separate per-group quant) that still saves one
kernel launch vs. baseline. The LayerCommunicator gracefully falls back
kernel launch vs. baseline. The stage boundary gracefully falls back
to ``forward_with_allreduce_fusion`` (plain AR+RMSNorm) when the fused
quant helper returns ``None``, so turning this on never regresses the
AR+RMSNorm fusion itself.
@@ -1109,14 +1108,6 @@ class Qwen3_5LinearDecoderLayer(nn.Module):
else:
raise ValueError(f"Invalid model type: {config.model_type}")
self.layer_facts = LayerFacts.init_new(
layer_id=layer_id,
num_layers=config.num_hidden_layers,
is_layer_sparse=is_layer_sparse,
is_previous_layer_sparse=is_previous_layer_sparse,
is_next_layer_sparse=is_next_layer_sparse,
)
self.input_layernorm = GemmaRMSNorm(config.hidden_size, eps=config.rms_norm_eps)
self.post_attention_layernorm = GemmaRMSNorm(
config.hidden_size, eps=config.rms_norm_eps
@@ -1128,33 +1119,44 @@ class Qwen3_5LinearDecoderLayer(nn.Module):
_enable_qwen35_fused_ar_quant()
and _linear_accepts_fp8_tuple(self.linear_attn.in_proj_qkvz)
)
self.layer_communicator = LayerCommunicator(
layer_facts=self.layer_facts,
input_layernorm=self.input_layernorm,
post_attention_layernorm=self.post_attention_layernorm,
allow_reduce_scatter=True,
enable_fused_ar_quant=enable_fused_ar_quant,
fused_ar_quant_keep_bf16=enable_fused_ar_quant,
fusions=_layer_fusions(config, is_nextn),
boundary_fusions = _layer_fusions(config, is_nextn)
self.attn_stage, self.ffn_stage = make_stages(
(
declare_attn(),
self.input_layernorm,
{
"enable_fused_ar_quant": enable_fused_ar_quant,
"fused_ar_quant_keep_bf16": enable_fused_ar_quant,
"fusions": boundary_fusions,
},
),
(
declare_ffn(
sparse=is_layer_sparse,
next_sparse=is_next_layer_sparse,
),
self.post_attention_layernorm,
{"fusions": boundary_fusions},
),
previous=declare_ffn(
sparse=is_previous_layer_sparse, next_sparse=is_layer_sparse
)
if layer_id != 0
else None,
terminal=layer_id == config.num_hidden_layers - 1,
)
def forward(
self,
hidden_states: torch.Tensor,
residual: Optional[torch.Tensor],
**kwargs,
):
forward_batch = kwargs.get("forward_batch", None)
hidden_states, residual = (
self.layer_communicator.prepare_attn_and_capture_last_layer_outputs(
hidden_states,
residual,
forward_batch,
captured_last_layer_outputs=kwargs.get(
"captured_last_layer_outputs", None
),
)
hidden_states = self.attn_stage.prepare(
hidden_states,
forward_batch,
captured_last_layer_outputs=kwargs.get("captured_last_layer_outputs", None),
)
# fused AR+quant hands down a (fp8, scale) / (bf16, fp8, scale) tuple
@@ -1166,11 +1168,10 @@ class Qwen3_5LinearDecoderLayer(nn.Module):
)
# Fully Connected
hidden_states, residual = self.layer_communicator.prepare_mlp(
hidden_states, residual, forward_batch
)
hidden_states = self.attn_stage.finish(hidden_states, forward_batch)
hidden_states = self.ffn_stage.prepare(hidden_states, forward_batch)
with self.layer_communicator.ffn_exit(forward_batch) as ffn_exit:
with self.ffn_stage.exit(forward_batch) as ffn_exit:
if isinstance(self.mlp, Qwen2MoeSparseMoeBlock):
hidden_states = self.mlp(
hidden_states,
@@ -1179,7 +1180,7 @@ class Qwen3_5LinearDecoderLayer(nn.Module):
)
else:
hidden_states = self.mlp(hidden_states)
return ffn_exit.finish(hidden_states, residual)
return ffn_exit.finish(hidden_states)
class Qwen3_5AttentionDecoderLayer(nn.Module):
@@ -1311,14 +1312,6 @@ class Qwen3_5AttentionDecoderLayer(nn.Module):
else:
raise ValueError(f"Invalid model type: {config.model_type}")
self.layer_facts = LayerFacts.init_new(
layer_id=layer_id,
num_layers=config.num_hidden_layers,
is_layer_sparse=is_layer_sparse,
is_previous_layer_sparse=is_previous_layer_sparse,
is_next_layer_sparse=is_next_layer_sparse,
)
self.input_layernorm = GemmaRMSNorm(config.hidden_size, eps=config.rms_norm_eps)
self.post_attention_layernorm = GemmaRMSNorm(
config.hidden_size, eps=config.rms_norm_eps
@@ -1332,14 +1325,31 @@ class Qwen3_5AttentionDecoderLayer(nn.Module):
enable_fused_ar_quant = (
_enable_qwen35_fused_ar_quant() and _linear_accepts_fp8_tuple(self.qkv_proj)
)
self.layer_communicator = LayerCommunicator(
layer_facts=self.layer_facts,
input_layernorm=self.input_layernorm,
post_attention_layernorm=self.post_attention_layernorm,
allow_reduce_scatter=True,
enable_fused_ar_quant=enable_fused_ar_quant,
fused_ar_quant_keep_bf16=False,
fusions=_layer_fusions(config, is_nextn),
boundary_fusions = _layer_fusions(config, is_nextn)
self.attn_stage, self.ffn_stage = make_stages(
(
declare_attn(),
self.input_layernorm,
{
"enable_fused_ar_quant": enable_fused_ar_quant,
"fused_ar_quant_keep_bf16": False,
"fusions": boundary_fusions,
},
),
(
declare_ffn(
sparse=is_layer_sparse,
next_sparse=is_next_layer_sparse,
),
self.post_attention_layernorm,
{"fusions": boundary_fusions},
),
previous=declare_ffn(
sparse=is_previous_layer_sparse, next_sparse=is_layer_sparse
)
if layer_id != 0
else None,
terminal=layer_id == config.num_hidden_layers - 1,
)
self.alt_stream = alt_stream
@@ -1543,18 +1553,14 @@ class Qwen3_5AttentionDecoderLayer(nn.Module):
self,
positions: torch.Tensor,
hidden_states: torch.Tensor,
residual: Optional[torch.Tensor],
forward_batch: ForwardBatch,
captured_last_layer_outputs: Optional[AuxHiddenStateAccumulator] = None,
**kwargs,
):
hidden_states, residual = (
self.layer_communicator.prepare_attn_and_capture_last_layer_outputs(
hidden_states,
residual,
forward_batch,
captured_last_layer_outputs=captured_last_layer_outputs,
)
hidden_states = self.attn_stage.prepare(
hidden_states,
forward_batch,
captured_last_layer_outputs=captured_last_layer_outputs,
)
# fused AR+quant hands down a (fp8, scale) / (bf16, fp8, scale) tuple
@@ -1567,10 +1573,9 @@ class Qwen3_5AttentionDecoderLayer(nn.Module):
)
# Fully Connected
hidden_states, residual = self.layer_communicator.prepare_mlp(
hidden_states, residual, forward_batch
)
with self.layer_communicator.ffn_exit(forward_batch) as ffn_exit:
hidden_states = self.attn_stage.finish(hidden_states, forward_batch)
hidden_states = self.ffn_stage.prepare(hidden_states, forward_batch)
with self.ffn_stage.exit(forward_batch) as ffn_exit:
if isinstance(self.mlp, Qwen2MoeSparseMoeBlock):
hidden_states = self.mlp(
hidden_states,
@@ -1579,7 +1584,7 @@ class Qwen3_5AttentionDecoderLayer(nn.Module):
)
else:
hidden_states = self.mlp(hidden_states)
return ffn_exit.finish(hidden_states, residual)
return ffn_exit.finish(hidden_states)
ALL_DECODER_LAYER_TYPES = {
@@ -1747,6 +1752,7 @@ class Qwen3_5ForCausalLM(nn.Module):
# Every layer was checked above.
can_defer_finalize=lambda layer: True,
label="Qwen3.5",
terminal_finalize=True,
)
# Final normalization
@@ -1821,12 +1827,12 @@ class Qwen3_5ForCausalLM(nn.Module):
hidden_states = self.embed_tokens(input_ids)
else:
hidden_states = input_embeds
residual = residual_batch.start(forward_batch)
residual_batch.start(forward_batch)
else:
assert pp_proxy_tensors is not None
hidden_states, residual = self.layers[
self.start_layer
].layer_communicator.from_pp(pp_proxy_tensors, forward_batch)
hidden_states = self.layers[self.start_layer].attn_stage.from_pp(
pp_proxy_tensors, forward_batch
)
aux_hidden_states = AuxHiddenStateList()
# Pass through decoder layers
@@ -1835,16 +1841,13 @@ class Qwen3_5ForCausalLM(nn.Module):
with get_global_expert_distribution_recorder().with_current_layer(
layer_idx
):
hidden_states, residual = layer(
hidden_states = layer(
positions=positions,
hidden_states=hidden_states,
residual=residual,
forward_batch=forward_batch,
captured_last_layer_outputs=(
aux_hidden_states
if getattr(layer, "_is_layer_to_capture", False)
else None
),
captured_last_layer_outputs=aux_hidden_states
if getattr(layer, "_is_layer_to_capture", False)
else None,
)
# Process deepstack embeddings if provided
@@ -1854,52 +1857,22 @@ class Qwen3_5ForCausalLM(nn.Module):
and layer_idx < 3
):
sep = self.hidden_size * layer_idx
hidden_states, residual = layer.layer_communicator.add_to_output(
hidden_states = residual_batch.add_to_output(
hidden_states,
residual,
forward_batch,
input_deepstack_embeds[:, sep : sep + self.hidden_size],
)
last_layer = self.layers[self.end_layer - 1]
hidden_states, residual = last_layer.layer_communicator.finish_layer_stack(
hidden_states,
residual,
forward_batch,
# The final norm below finalizes a deferred MoE output in its kernel.
final_norm_takes_handoff=self.flashinfer_mnnvl_cutedsl_fusion is not None
and self.pp_group.is_last_rank,
preserve_declared=not self.pp_group.is_last_rank,
)
# Return intermediate tensors for pipeline parallelism
if not self.pp_group.is_last_rank:
return PPProxyTensors(
{
"hidden_states": hidden_states,
"residual": residual,
}
)
return residual_batch.to_pp(hidden_states, forward_batch)
# The final layer has no successor to consume its deferred MoE tail.
is_deferred_finalize = False
if self.flashinfer_mnnvl_cutedsl_fusion is not None:
from sglang.srt.layers.communicator.fusions.cutedsl import (
MoeFinalizeHandoff,
)
is_deferred_finalize = isinstance(hidden_states, MoeFinalizeHandoff)
if is_deferred_finalize:
if residual is None:
raise RuntimeError("invalid final deferred MoE handoff")
hidden_states, _ = self.flashinfer_mnnvl_cutedsl_fusion.finalize(
handoff=hidden_states, residual=residual, gamma=self.norm.gemma_weight
)
elif hidden_states.shape[0] != 0:
if residual is None:
hidden_states = self.norm(hidden_states)
else:
hidden_states, _ = self.norm(hidden_states, residual)
hidden_states = residual_batch.norm(
hidden_states,
forward_batch,
self.norm,
handoff_norm=self.flashinfer_mnnvl_cutedsl_fusion,
skip_empty=True,
)
if len(aux_hidden_states) == 0:
return hidden_states
+48 -57
View File
@@ -30,7 +30,11 @@ from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_r
from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation
from sglang.srt.eplb.expert_location_dispatch import ExpertLocationDispatchInfo
from sglang.srt.layers.aux_hidden_states import AuxHiddenStateAccumulator
from sglang.srt.layers.communicator import LayerCommunicator, LayerFacts
from sglang.srt.layers.communicator import (
declare_attn,
declare_ffn,
make_stages,
)
from sglang.srt.layers.communicator.residual import batch as residual_batch
from sglang.srt.layers.layernorm import RMSNorm
from sglang.srt.layers.linear import (
@@ -842,14 +846,6 @@ class Qwen3MoeDecoderLayer(nn.Module):
is_previous_layer_sparse = True
is_next_layer_sparse = True
self.layer_facts = LayerFacts.init_new(
layer_id=layer_id,
num_layers=config.num_hidden_layers,
is_layer_sparse=self.is_layer_sparse,
is_previous_layer_sparse=is_previous_layer_sparse,
is_next_layer_sparse=is_next_layer_sparse,
)
if self.is_layer_sparse:
self.mlp = Qwen3MoeSparseMoeBlock(
layer_id=self.layer_id,
@@ -870,11 +866,21 @@ class Qwen3MoeDecoderLayer(nn.Module):
config.hidden_size, eps=config.rms_norm_eps
)
self.layer_communicator = LayerCommunicator(
layer_facts=self.layer_facts,
input_layernorm=self.input_layernorm,
post_attention_layernorm=self.post_attention_layernorm,
allow_reduce_scatter=True,
self.attn_stage, self.ffn_stage = make_stages(
(declare_attn(), self.input_layernorm),
(
declare_ffn(
sparse=self.is_layer_sparse,
next_sparse=is_next_layer_sparse,
),
self.post_attention_layernorm,
),
previous=declare_ffn(
sparse=is_previous_layer_sparse, next_sparse=self.is_layer_sparse
)
if layer_id != 0
else None,
terminal=layer_id == config.num_hidden_layers - 1,
)
def forward(
@@ -882,21 +888,17 @@ class Qwen3MoeDecoderLayer(nn.Module):
positions: torch.Tensor,
hidden_states: torch.Tensor,
forward_batch: ForwardBatch,
residual: Optional[torch.Tensor],
captured_last_layer_outputs: Optional[AuxHiddenStateAccumulator] = None,
capture_output=None,
**kwargs,
) -> Tuple[torch.Tensor, torch.Tensor]:
) -> torch.Tensor:
hidden_states, residual = (
self.layer_communicator.prepare_attn_and_capture_last_layer_outputs(
hidden_states,
residual,
forward_batch,
captured_last_layer_outputs=captured_last_layer_outputs,
capture_output=capture_output,
**kwargs,
)
hidden_states = self.attn_stage.prepare(
hidden_states,
forward_batch,
captured_last_layer_outputs=captured_last_layer_outputs,
capture_output=capture_output,
**kwargs,
)
if hidden_states.shape[0] != 0:
@@ -906,15 +908,14 @@ class Qwen3MoeDecoderLayer(nn.Module):
forward_batch=forward_batch,
)
hidden_states, residual = self.layer_communicator.prepare_mlp(
hidden_states, residual, forward_batch
)
hidden_states = self.attn_stage.finish(hidden_states, forward_batch)
hidden_states = self.ffn_stage.prepare(hidden_states, forward_batch)
with self.layer_communicator.ffn_exit(forward_batch) as ffn_exit:
with self.ffn_stage.exit(forward_batch) as ffn_exit:
hidden_states = self.mlp(hidden_states, forward_batch)
hidden_states, residual = ffn_exit.finish(hidden_states, residual)
hidden_states = ffn_exit.finish(hidden_states)
return hidden_states, residual
return hidden_states
def op_comm_prepare_attn(
self,
@@ -922,11 +923,10 @@ class Qwen3MoeDecoderLayer(nn.Module):
positions: torch.Tensor,
hidden_states: torch.Tensor,
forward_batch: ForwardBatch,
residual: Optional[torch.Tensor],
tbo_subbatch_index: Optional[int] = None,
):
state.hidden_states_after_comm_pre_attn, state.residual_after_input_ln = (
self.layer_communicator.prepare_attn(hidden_states, residual, forward_batch)
state.hidden_states_after_comm_pre_attn = self.attn_stage.prepare(
hidden_states, forward_batch
)
state.update(
dict(
@@ -937,25 +937,21 @@ class Qwen3MoeDecoderLayer(nn.Module):
)
def op_comm_prepare_mlp(self, state):
state.hidden_states_mlp_input, state.residual_after_comm_pre_mlp = (
self.layer_communicator.prepare_mlp(
state.pop("hidden_states_after_attn"),
state.pop("residual_after_input_ln"),
state.forward_batch,
)
hidden_states = self.attn_stage.finish(
state.pop("hidden_states_after_attn"), state.forward_batch
)
state.hidden_states_mlp_input = self.ffn_stage.prepare(
hidden_states, state.forward_batch
)
def op_comm_postprocess_layer(self, state):
hidden_states, residual = self.layer_communicator.postprocess_layer(
state.pop("hidden_states_mlp_output"),
state.pop("residual_after_comm_pre_mlp"),
state.forward_batch,
hidden_states = self.ffn_stage.postprocess(
state.pop("hidden_states_mlp_output"), state.forward_batch
)
output = dict(
positions=state.positions,
hidden_states=hidden_states,
residual=residual,
forward_batch=state.forward_batch,
tbo_subbatch_index=state.tbo_subbatch_index,
)
@@ -1088,7 +1084,7 @@ class Qwen3MoeForCausalLM(nn.Module):
start, end = split_interval
# embed
if start == 0:
forward_batch.residual = residual_batch.start(forward_batch)
residual_batch.start(forward_batch)
if input_embeds is None:
forward_batch.hidden_states = self.model.embed_tokens(input_ids)
else:
@@ -1098,22 +1094,17 @@ class Qwen3MoeForCausalLM(nn.Module):
for i in range(start, end):
with get_global_expert_distribution_recorder().with_current_layer(i):
layer = self.model.layers[i]
forward_batch.hidden_states, forward_batch.residual = layer(
positions,
forward_batch.hidden_states,
forward_batch,
forward_batch.residual,
forward_batch.hidden_states = layer(
positions, forward_batch.hidden_states, forward_batch
)
if end == self.model.config.num_hidden_layers:
forward_batch.hidden_states, forward_batch.residual = self.model.layers[
end - 1
].layer_communicator.finish_layer_stack(
forward_batch.hidden_states, forward_batch.residual, forward_batch
forward_batch.hidden_states = residual_batch.finish(
forward_batch.hidden_states, forward_batch
)
# norm
hidden_states, _ = self.model.norm(
forward_batch.hidden_states, forward_batch.residual
hidden_states = residual_batch.norm(
forward_batch.hidden_states, forward_batch, self.model.norm
)
forward_batch.residual = None
forward_batch.hidden_states = hidden_states
+60 -75
View File
@@ -19,7 +19,11 @@ from sglang.srt.layers.aux_hidden_states import (
AuxHiddenStateAccumulator,
AuxHiddenStateList,
)
from sglang.srt.layers.communicator import LayerCommunicator, LayerFacts
from sglang.srt.layers.communicator import (
declare_attn,
declare_ffn,
make_stages,
)
from sglang.srt.layers.communicator.residual import batch as residual_batch
from sglang.srt.layers.dp_attention import (
is_dp_attention_enabled,
@@ -471,13 +475,11 @@ class Qwen3GatedDeltaNet(nn.Module):
def _apply_qwen3_next_mlp(
layer: nn.Module,
hidden_states: torch.Tensor,
residual: Optional[torch.Tensor],
forward_batch: ForwardBatch,
) -> Tuple[torch.Tensor, Optional[torch.Tensor]]:
hidden_states, residual = layer.layer_communicator.prepare_mlp(
hidden_states, residual, forward_batch
)
with layer.layer_communicator.ffn_exit(forward_batch) as ffn_exit:
) -> torch.Tensor:
hidden_states = layer.attn_stage.finish(hidden_states, forward_batch)
hidden_states = layer.ffn_stage.prepare(hidden_states, forward_batch)
with layer.ffn_stage.exit(forward_batch) as ffn_exit:
if isinstance(layer.mlp, Qwen2MoeSparseMoeBlock):
hidden_states = layer.mlp(
hidden_states,
@@ -485,9 +487,9 @@ def _apply_qwen3_next_mlp(
)
else:
hidden_states = layer.mlp(hidden_states)
hidden_states, residual = ffn_exit.finish(hidden_states, residual)
hidden_states = ffn_exit.finish(hidden_states)
return hidden_states, residual
return hidden_states
class Qwen3HybridLinearDecoderLayer(nn.Module):
@@ -512,14 +514,6 @@ class Qwen3HybridLinearDecoderLayer(nn.Module):
is_next_layer_sparse = True
self.layer_id = layer_id
self.layer_facts = LayerFacts.init_new(
layer_id=layer_id,
num_layers=config.num_hidden_layers,
is_layer_sparse=self.is_layer_sparse,
is_previous_layer_sparse=is_previous_layer_sparse,
is_next_layer_sparse=is_next_layer_sparse,
)
if self.is_layer_sparse:
self.mlp = Qwen2MoeSparseMoeBlock(
layer_id=layer_id,
@@ -543,29 +537,35 @@ class Qwen3HybridLinearDecoderLayer(nn.Module):
self.post_attention_layernorm = GemmaRMSNorm(
config.hidden_size, eps=config.rms_norm_eps
)
self.layer_communicator = LayerCommunicator(
layer_facts=self.layer_facts,
input_layernorm=self.input_layernorm,
post_attention_layernorm=self.post_attention_layernorm,
allow_reduce_scatter=True,
self.attn_stage, self.ffn_stage = make_stages(
(declare_attn(), self.input_layernorm),
(
declare_ffn(
sparse=self.is_layer_sparse,
next_sparse=is_next_layer_sparse,
),
self.post_attention_layernorm,
),
previous=declare_ffn(
sparse=is_previous_layer_sparse, next_sparse=self.is_layer_sparse
)
if layer_id != 0
else None,
terminal=layer_id == config.num_hidden_layers - 1,
)
def forward(
self,
hidden_states: torch.Tensor,
residual: Optional[torch.Tensor],
captured_last_layer_outputs: Optional[AuxHiddenStateAccumulator] = None,
**kwargs,
):
forward_batch = kwargs.get("forward_batch", None)
hidden_states, residual = (
self.layer_communicator.prepare_attn_and_capture_last_layer_outputs(
hidden_states,
residual,
forward_batch,
captured_last_layer_outputs=captured_last_layer_outputs,
)
hidden_states = self.attn_stage.prepare(
hidden_states,
forward_batch,
captured_last_layer_outputs=captured_last_layer_outputs,
)
if not forward_batch.forward_mode.is_idle():
@@ -573,11 +573,9 @@ class Qwen3HybridLinearDecoderLayer(nn.Module):
hidden_states,
forward_batch,
)
hidden_states, residual = _apply_qwen3_next_mlp(
self, hidden_states, residual, forward_batch
)
hidden_states = _apply_qwen3_next_mlp(self, hidden_states, forward_batch)
return hidden_states, residual
return hidden_states
class Qwen3HybridAttentionDecoderLayer(nn.Module):
@@ -680,14 +678,6 @@ class Qwen3HybridAttentionDecoderLayer(nn.Module):
is_previous_layer_sparse = True
is_next_layer_sparse = True
self.layer_facts = LayerFacts.init_new(
layer_id=layer_id,
num_layers=config.num_hidden_layers,
is_layer_sparse=self.is_layer_sparse,
is_previous_layer_sparse=is_previous_layer_sparse,
is_next_layer_sparse=is_next_layer_sparse,
)
if self.is_layer_sparse:
self.mlp = Qwen2MoeSparseMoeBlock(
layer_id=layer_id,
@@ -715,11 +705,21 @@ class Qwen3HybridAttentionDecoderLayer(nn.Module):
self.q_norm = GemmaRMSNorm(self.head_dim, eps=config.rms_norm_eps)
self.k_norm = GemmaRMSNorm(self.head_dim, eps=config.rms_norm_eps)
self.layer_communicator = LayerCommunicator(
layer_facts=self.layer_facts,
input_layernorm=self.input_layernorm,
post_attention_layernorm=self.post_attention_layernorm,
allow_reduce_scatter=True,
self.attn_stage, self.ffn_stage = make_stages(
(declare_attn(), self.input_layernorm),
(
declare_ffn(
sparse=self.is_layer_sparse,
next_sparse=is_next_layer_sparse,
),
self.post_attention_layernorm,
),
previous=declare_ffn(
sparse=is_previous_layer_sparse, next_sparse=self.is_layer_sparse
)
if layer_id != 0
else None,
terminal=layer_id == config.num_hidden_layers - 1,
)
self.alt_stream = alt_stream
@@ -828,18 +828,14 @@ class Qwen3HybridAttentionDecoderLayer(nn.Module):
self,
positions: torch.Tensor,
hidden_states: torch.Tensor,
residual: Optional[torch.Tensor],
forward_batch: ForwardBatch,
captured_last_layer_outputs: Optional[AuxHiddenStateAccumulator] = None,
**kwargs: Any,
):
hidden_states, residual = (
self.layer_communicator.prepare_attn_and_capture_last_layer_outputs(
hidden_states,
residual,
forward_batch,
captured_last_layer_outputs=captured_last_layer_outputs,
)
hidden_states = self.attn_stage.prepare(
hidden_states,
forward_batch,
captured_last_layer_outputs=captured_last_layer_outputs,
)
if not forward_batch.forward_mode.is_idle():
@@ -849,11 +845,9 @@ class Qwen3HybridAttentionDecoderLayer(nn.Module):
forward_batch=forward_batch,
)
hidden_states, residual = _apply_qwen3_next_mlp(
self, hidden_states, residual, forward_batch
)
hidden_states = _apply_qwen3_next_mlp(self, hidden_states, forward_batch)
return hidden_states, residual
return hidden_states
ALL_DECODER_LAYER_TYPES = {
@@ -934,33 +928,24 @@ class Qwen3NextModel(nn.Module):
else:
hidden_states = self.embed_tokens(input_ids)
residual = residual_batch.start(forward_batch)
residual_batch.start(forward_batch)
aux_hidden_states = AuxHiddenStateList()
for i in range(len(self.layers)):
layer = self.layers[i]
with get_global_expert_distribution_recorder().with_current_layer(i):
hidden_states, residual = layer(
hidden_states = layer(
layer_id=i,
positions=positions,
hidden_states=hidden_states,
residual=residual,
forward_batch=forward_batch,
captured_last_layer_outputs=(
aux_hidden_states
if getattr(layer, "_is_layer_to_capture", False)
else None
),
captured_last_layer_outputs=aux_hidden_states
if getattr(layer, "_is_layer_to_capture", False)
else None,
)
last_layer = self.layers[-1]
hidden_states, residual = last_layer.layer_communicator.finish_layer_stack(
hidden_states, residual, forward_batch
)
hidden_states = residual_batch.finish(hidden_states, forward_batch)
if not forward_batch.forward_mode.is_idle():
if residual is None:
hidden_states = self.norm(hidden_states)
else:
hidden_states, _ = self.norm(hidden_states, residual)
hidden_states = residual_batch.norm(hidden_states, forward_batch, self.norm)
if len(aux_hidden_states) == 0:
return hidden_states
+16 -29
View File
@@ -96,12 +96,12 @@ class Qwen3MoeLLMModel(Qwen3MoeModel):
hidden_states = self.embed_tokens(input_ids)
else:
hidden_states = input_embeds
residual = residual_batch.start(forward_batch)
residual_batch.start(forward_batch)
else:
assert pp_proxy_tensors is not None
hidden_states, residual = self.layers[
self.start_layer
].layer_communicator.from_pp(pp_proxy_tensors, forward_batch)
hidden_states = self.layers[self.start_layer].attn_stage.from_pp(
pp_proxy_tensors, forward_batch
)
aux_hidden_states = AuxHiddenStateList()
for layer_idx, layer in enumerate(
@@ -116,11 +116,10 @@ class Qwen3MoeLLMModel(Qwen3MoeModel):
deepstack_embeds = self.get_deepstack_embeds(
layer_idx - 1, input_deepstack_embeds
)
hidden_states, residual = layer(
hidden_states = layer(
positions,
hidden_states,
forward_batch,
residual,
post_residual_addition=deepstack_embeds,
capture_output=aux_hidden_states.capture
if layer_idx in self.layers_to_capture
@@ -129,11 +128,10 @@ class Qwen3MoeLLMModel(Qwen3MoeModel):
else:
# Inference path: add deepstack directly to hidden_states at the end of the layer
# (original, grounding-correct order).
hidden_states, residual = layer(
hidden_states = layer(
positions,
hidden_states,
forward_batch,
residual,
capture_output=aux_hidden_states.capture
if layer_idx in self.layers_to_capture
else None,
@@ -143,9 +141,9 @@ class Qwen3MoeLLMModel(Qwen3MoeModel):
and layer_idx in self.deepstack_embed_to_decoder_layer
):
sep = self.hidden_size * layer_idx
hidden_states, residual = layer.layer_communicator.add_to_output(
hidden_states = residual_batch.add_to_output(
hidden_states,
residual,
forward_batch,
input_deepstack_embeds[:, sep : sep + self.hidden_size],
)
@@ -156,28 +154,17 @@ class Qwen3MoeLLMModel(Qwen3MoeModel):
else None
)
last_layer = self.layers[self.end_layer - 1]
hidden_states, residual = last_layer.layer_communicator.finish_layer_stack(
hidden_states,
residual,
forward_batch,
preserve_declared=not self.pp_group.is_last_rank,
)
if not self.pp_group.is_last_rank:
return PPProxyTensors(
{
"hidden_states": hidden_states,
"residual": residual,
}
)
return residual_batch.to_pp(hidden_states, forward_batch)
else:
hidden_states = residual_batch.finish(hidden_states, forward_batch)
if hidden_states.shape[0] != 0:
if residual is None:
hidden_states = self.norm(hidden_states)
else:
hidden_states, _ = self.norm(
hidden_states, residual, post_residual_addition=last_deepstack
)
hidden_states = residual_batch.norm(
hidden_states,
forward_batch,
self.norm,
post_residual_addition=last_deepstack,
)
if len(aux_hidden_states) == 0:
return hidden_states
+2 -1
View File
@@ -1320,7 +1320,8 @@ class Qwen4ExpLayerExtensionMixin:
for attr_name in (
"input_layernorm",
"post_attention_layernorm",
"layer_communicator",
"attn_stage",
"ffn_stage",
):
if hasattr(self, attr_name):
delattr(self, attr_name)
+46 -65
View File
@@ -20,9 +20,10 @@ from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_r
from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation
from sglang.srt.layers.activation import SiluAndMul
from sglang.srt.layers.communicator import (
LayerCommunicator,
LayerFacts,
declare_attn,
declare_ffn,
enable_moe_dense_fully_dp,
make_stages,
)
from sglang.srt.layers.communicator.residual import batch as residual_batch
from sglang.srt.layers.dp_attention import (
@@ -1048,19 +1049,26 @@ class SarvamMoEMLADecoderLayer(nn.Module):
)
self.attn_tp_size = get_parallel().attn_tp_size
self.layer_facts = LayerFacts.init_new(
layer_id=layer_id,
num_layers=config.num_hidden_layers,
is_layer_sparse=self.is_layer_sparse,
is_previous_layer_sparse=is_previous_layer_sparse,
is_next_layer_sparse=is_next_layer_sparse,
)
self.layer_communicator = LayerCommunicator(
layer_facts=self.layer_facts,
input_layernorm=self.input_layernorm,
post_attention_layernorm=self.post_attention_layernorm,
qkv_latent_func=self.self_attn.prepare_qkv_latent,
allow_reduce_scatter=True,
self.attn_stage, self.ffn_stage = make_stages(
(
declare_attn(),
self.input_layernorm,
{"qkv_latent_func": self.self_attn.prepare_qkv_latent},
),
(
declare_ffn(
sparse=self.is_layer_sparse,
next_sparse=is_next_layer_sparse,
),
self.post_attention_layernorm,
),
previous=declare_ffn(
sparse=is_previous_layer_sparse, next_sparse=self.is_layer_sparse
)
if layer_id != 0
else None,
terminal=layer_id == config.num_hidden_layers - 1,
)
def forward(
@@ -1068,21 +1076,17 @@ class SarvamMoEMLADecoderLayer(nn.Module):
positions: torch.Tensor,
hidden_states: torch.Tensor,
forward_batch: ForwardBatch,
residual: Optional[torch.Tensor],
) -> Tuple[torch.Tensor, torch.Tensor]:
hidden_states, residual = self.layer_communicator.prepare_attn(
hidden_states, residual, forward_batch
)
) -> torch.Tensor:
hidden_states = self.attn_stage.prepare(hidden_states, forward_batch)
if hidden_states.shape[0] != 0:
hidden_states = self.self_attn(
positions=positions,
hidden_states=hidden_states,
forward_batch=forward_batch,
)
hidden_states, residual = self.layer_communicator.prepare_mlp(
hidden_states, residual, forward_batch
)
with self.layer_communicator.ffn_exit(forward_batch) as ffn_exit:
hidden_states = self.attn_stage.finish(hidden_states, forward_batch)
hidden_states = self.ffn_stage.prepare(hidden_states, forward_batch)
with self.ffn_stage.exit(forward_batch) as ffn_exit:
hidden_states = self.mlp(hidden_states, forward_batch)
if (
not self.is_layer_sparse
@@ -1091,8 +1095,8 @@ class SarvamMoEMLADecoderLayer(nn.Module):
and not ffn_exit.fuse_mlp_allreduce
):
hidden_states = tensor_model_parallel_all_reduce(hidden_states)
hidden_states, residual = ffn_exit.finish(hidden_states, residual)
return hidden_states, residual
hidden_states = ffn_exit.finish(hidden_states)
return hidden_states
class SarvamMLAModel(nn.Module):
@@ -1152,36 +1156,23 @@ class SarvamMLAModel(nn.Module):
hidden_states = self.embed_tokens(input_ids)
else:
hidden_states = input_embeds
residual = residual_batch.start(forward_batch)
residual_batch.start(forward_batch)
else:
assert pp_proxy_tensors is not None
hidden_states, residual = self.layers[
self.start_layer
].layer_communicator.from_pp(pp_proxy_tensors, forward_batch)
hidden_states = self.layers[self.start_layer].attn_stage.from_pp(
pp_proxy_tensors, forward_batch
)
for i in range(self.start_layer, self.end_layer):
layer = self.layers[i]
hidden_states, residual = layer(
positions, hidden_states, forward_batch, residual
)
hidden_states = layer(positions, hidden_states, forward_batch)
last_layer = self.layers[self.end_layer - 1]
hidden_states, residual = last_layer.layer_communicator.finish_layer_stack(
hidden_states,
residual,
forward_batch,
preserve_declared=not self.pp_group.is_last_rank,
)
if not self.pp_group.is_last_rank:
return PPProxyTensors(
{"hidden_states": hidden_states, "residual": residual}
)
return residual_batch.to_pp(hidden_states, forward_batch)
hidden_states = residual_batch.finish(hidden_states, forward_batch)
if hidden_states.shape[0] != 0:
if residual is None:
hidden_states = self.norm(hidden_states)
else:
hidden_states, _ = self.norm(hidden_states, residual)
hidden_states = residual_batch.norm(hidden_states, forward_batch, self.norm)
return hidden_states
@@ -1267,36 +1258,26 @@ class SarvamMLAForCausalLM(nn.Module):
) -> Optional[LogitsProcessorOutput]:
start, end = split_interval
if start == 0:
residual_batch.start(forward_batch)
if input_embeds is None:
forward_batch.hidden_states = self.model.embed_tokens(input_ids)
else:
forward_batch.hidden_states = input_embeds
forward_batch.residual = residual_batch.start(forward_batch)
for i in range(start, end):
with get_global_expert_distribution_recorder().with_current_layer(i):
layer = self.model.layers[i]
forward_batch.hidden_states, forward_batch.residual = layer(
positions,
forward_batch.hidden_states,
forward_batch,
forward_batch.residual,
forward_batch.hidden_states = layer(
positions, forward_batch.hidden_states, forward_batch
)
if end == self.model.config.num_hidden_layers:
forward_batch.hidden_states, forward_batch.residual = self.model.layers[
end - 1
].layer_communicator.finish_layer_stack(
forward_batch.hidden_states, forward_batch.residual, forward_batch
forward_batch.hidden_states = residual_batch.finish(
forward_batch.hidden_states, forward_batch
)
forward_batch.hidden_states = residual_batch.norm(
forward_batch.hidden_states, forward_batch, self.model.norm
)
if forward_batch.residual is None:
hidden_states = self.model.norm(forward_batch.hidden_states)
else:
hidden_states, _ = self.model.norm(
forward_batch.hidden_states, forward_batch.residual
)
forward_batch.residual = None
forward_batch.hidden_states = hidden_states
return self.logits_processor(
input_ids, forward_batch.hidden_states, self.lm_head, forward_batch
)
+44 -67
View File
@@ -11,7 +11,11 @@ from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_r
from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation
from sglang.srt.eplb.expert_location_dispatch import ExpertLocationDispatchInfo
from sglang.srt.layers.activation import SiluAndMul
from sglang.srt.layers.communicator import LayerCommunicator, LayerFacts
from sglang.srt.layers.communicator import (
declare_attn,
declare_ffn,
make_stages,
)
from sglang.srt.layers.communicator.residual import batch as residual_batch
from sglang.srt.layers.dp_attention import (
is_dp_attention_enabled,
@@ -561,21 +565,23 @@ class Step3p5DecoderLayer(nn.Module):
)
# An MTP draft is a one-layer model; layer_id still indexes its config.
self.layer_facts = LayerFacts.init_new(
layer_id=0 if is_nextn else layer_id,
num_layers=1 if is_nextn else config.num_hidden_layers,
is_layer_sparse=self.is_moe_layer,
is_previous_layer_sparse=self.is_previous_layer_sparse,
is_next_layer_sparse=self.is_next_layer_sparse,
)
self.layer_communicator = LayerCommunicator(
layer_facts=self.layer_facts,
input_layernorm=self.input_layernorm,
post_attention_layernorm=self.post_attention_layernorm,
allow_reduce_scatter=True,
# The dense MLP all-reduces its own output unless postprocess
# reduce-scatters it; it never leaves the sum to the next layer.
allow_deferred_ffn_reduction=self.use_moe,
self.attn_stage, self.ffn_stage = make_stages(
(declare_attn(), self.input_layernorm),
(
declare_ffn(
sparse=self.is_moe_layer,
next_sparse=self.is_next_layer_sparse,
),
self.post_attention_layernorm,
),
previous=declare_ffn(
sparse=self.is_previous_layer_sparse, next_sparse=self.is_moe_layer
)
if not (0 if is_nextn else layer_id) == 0
else None,
terminal=(0 if is_nextn else layer_id)
== (1 if is_nextn else config.num_hidden_layers) - 1,
)
self.layer_id = layer_id
@@ -585,15 +591,11 @@ class Step3p5DecoderLayer(nn.Module):
positions: torch.Tensor,
hidden_states: torch.Tensor,
forward_batch: ForwardBatch,
residual: Optional[torch.Tensor],
post_residual_addition: Optional[torch.Tensor] = None,
) -> Tuple[torch.Tensor, torch.Tensor]:
) -> torch.Tensor:
# Self Attention
hidden_states, residual = self.layer_communicator.prepare_attn(
hidden_states,
residual,
forward_batch,
post_residual_addition=post_residual_addition,
hidden_states = self.attn_stage.prepare(
hidden_states, forward_batch, post_residual_addition=post_residual_addition
)
if hidden_states.shape[0] != 0:
hidden_states = self.self_attn(
@@ -602,14 +604,11 @@ class Step3p5DecoderLayer(nn.Module):
forward_batch=forward_batch,
)
# Fully Connected
hidden_states, residual = self.layer_communicator.prepare_mlp(
hidden_states,
residual,
forward_batch,
)
hidden_states = self.attn_stage.finish(hidden_states, forward_batch)
hidden_states = self.ffn_stage.prepare(hidden_states, forward_batch)
if self.use_moe:
with self.layer_communicator.ffn_exit(forward_batch) as ffn_exit:
with self.ffn_stage.exit(forward_batch) as ffn_exit:
# Both share_expert and MoE return unreduced (TP-partial) outputs.
# Combine them first, then do a single all-reduce — saving one
# full-TP all-reduce per layer.
@@ -620,11 +619,11 @@ class Step3p5DecoderLayer(nn.Module):
hidden_states = moe_output + share_output
if not ffn_exit.fuse_mlp_allreduce and not ffn_exit.mlp_reduce_scatter:
hidden_states = tensor_model_parallel_all_reduce(hidden_states)
return ffn_exit.finish(hidden_states, residual)
return ffn_exit.finish(hidden_states)
with self.layer_communicator.ffn_exit(forward_batch) as ffn_exit:
with self.ffn_stage.exit(forward_batch) as ffn_exit:
hidden_states = self.mlp(hidden_states)
return ffn_exit.finish(hidden_states, residual)
return ffn_exit.finish(hidden_states)
class Step3p5Model(nn.Module):
@@ -698,55 +697,33 @@ class Step3p5Model(nn.Module):
hidden_states = self.embed_tokens(input_ids)
else:
hidden_states = input_embeds
residual = residual_batch.start(forward_batch)
residual_batch.start(forward_batch)
else:
assert pp_proxy_tensors is not None
hidden_states, residual = self.layers[
self.start_layer
].layer_communicator.from_pp(pp_proxy_tensors, forward_batch)
hidden_states = self.layers[self.start_layer].attn_stage.from_pp(
pp_proxy_tensors, forward_batch
)
for i in range(self.start_layer, self.end_layer):
layer = self.layers[i]
hidden_states, residual = layer(
positions,
hidden_states,
forward_batch,
residual,
)
hidden_states = layer(positions, hidden_states, forward_batch)
# break
last_layer = self.layers[self.end_layer - 1]
hidden_states, residual = last_layer.layer_communicator.finish_layer_stack(
hidden_states,
residual,
forward_batch,
preserve_declared=not self.pp_group.is_last_rank,
)
if not self.pp_group.is_last_rank:
return PPProxyTensors(
{
"hidden_states": hidden_states,
"residual": residual,
}
)
return residual_batch.to_pp(hidden_states, forward_batch)
else:
hidden_states = residual_batch.finish(hidden_states, forward_batch)
hidden_states_before_norm = None
if not self.pp_group.is_last_rank:
return PPProxyTensors(
{
"hidden_states": hidden_states,
"residual": residual,
}
)
return residual_batch.to_pp(hidden_states, forward_batch)
else:
if hidden_states.shape[0] > 0:
# if forward_batch.return_hidden_states_before_norm:
hidden_states_before_norm = last_layer.layer_communicator.snapshot(
hidden_states, residual
hidden_states_before_norm = residual_batch.snapshot(
hidden_states, forward_batch
)
hidden_states = residual_batch.norm(
hidden_states, forward_batch, self.norm
)
if residual is None:
hidden_states = self.norm(hidden_states)
else:
hidden_states, _ = self.norm(hidden_states, residual)
return hidden_states, hidden_states_before_norm
+8 -11
View File
@@ -110,25 +110,22 @@ class Step3p5AMultiTokenPredictor(nn.Module):
dim=-1,
)
)
hidden_states, residual = self.mtp_block(
residual_batch.start(forward_batch)
hidden_states = self.mtp_block(
positions=positions,
hidden_states=hidden_states,
forward_batch=forward_batch,
residual=residual_batch.start(forward_batch),
)
hidden_states, residual = self.mtp_block.layer_communicator.finish_layer_stack(
hidden_states, residual, forward_batch
)
hidden_states = residual_batch.finish(hidden_states, forward_batch)
hidden_states_before_norm = None
if not forward_batch.forward_mode.is_idle():
# if forward_batch.return_hidden_states_before_norm:
hidden_states_before_norm = self.mtp_block.layer_communicator.snapshot(
hidden_states, residual
hidden_states_before_norm = residual_batch.snapshot(
hidden_states, forward_batch
)
hidden_states = residual_batch.norm(
hidden_states, forward_batch, self.shared_head.norm
)
if residual is not None:
hidden_states, _ = self.shared_head.norm(hidden_states, residual)
else:
hidden_states = self.shared_head.norm(hidden_states)
return hidden_states, hidden_states_before_norm
+39 -69
View File
@@ -40,9 +40,10 @@ from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation
from sglang.srt.eplb.expert_location_dispatch import ExpertLocationDispatchInfo
from sglang.srt.layers.activation import SiluAndMul
from sglang.srt.layers.communicator import (
LayerCommunicator,
LayerFacts,
declare_attn,
declare_ffn,
enable_moe_dense_fully_dp,
make_stages,
)
from sglang.srt.layers.communicator.residual import batch as residual_batch
from sglang.srt.layers.dp_attention import is_dp_attention_enabled
@@ -860,11 +861,10 @@ class XllmMLP(nn.Module):
def forward(
self,
x,
use_reduce_scatter: bool = False,
):
gate_up, _ = self.gate_up_proj(x)
x = self.act_fn(gate_up)
x, _ = self.down_proj(x, skip_all_reduce=use_reduce_scatter)
x, _ = self.down_proj(x)
return x
@@ -1041,7 +1041,6 @@ class XllmSparseMoeBlock(nn.Module):
self,
hidden_states: torch.Tensor,
forward_batch: Optional[ForwardBatch] = None,
use_reduce_scatter: bool = False,
) -> torch.Tensor:
num_tokens, hidden_dim = hidden_states.shape
hidden_states = hidden_states.view(-1, hidden_dim)
@@ -1061,8 +1060,7 @@ class XllmSparseMoeBlock(nn.Module):
if shared_output is not None:
final_hidden_states += shared_output
if not use_reduce_scatter:
final_hidden_states = reduce_moe_output(final_hidden_states)
final_hidden_states = reduce_moe_output(final_hidden_states)
return final_hidden_states.view(num_tokens, hidden_dim)
@@ -1506,14 +1504,6 @@ class XllmDecoderLayer(nn.Module):
is_previous_layer_sparse = _is_sparse(layer_id - 1)
is_next_layer_sparse = _is_sparse(layer_id + 1)
self.layer_facts = LayerFacts.init_new(
layer_id=layer_id,
num_layers=config.num_hidden_layers,
is_layer_sparse=self.is_layer_sparse,
is_previous_layer_sparse=is_previous_layer_sparse,
is_next_layer_sparse=is_next_layer_sparse,
)
if self.is_layer_sparse:
self.mlp = XllmSparseMoeBlock(
layer_id=layer_id,
@@ -1538,12 +1528,21 @@ class XllmDecoderLayer(nn.Module):
self.input_layernorm = _make_norm(config)
self.post_attention_layernorm = _make_norm(config)
self.layer_communicator = LayerCommunicator(
layer_facts=self.layer_facts,
input_layernorm=self.input_layernorm,
post_attention_layernorm=self.post_attention_layernorm,
allow_reduce_scatter=True,
allow_deferred_ffn_reduction=False,
self.attn_stage, self.ffn_stage = make_stages(
(declare_attn(), self.input_layernorm),
(
declare_ffn(
sparse=self.is_layer_sparse,
next_sparse=is_next_layer_sparse,
),
self.post_attention_layernorm,
),
previous=declare_ffn(
sparse=is_previous_layer_sparse, next_sparse=self.is_layer_sparse
)
if layer_id != 0
else None,
terminal=layer_id == config.num_hidden_layers - 1,
)
def forward(
@@ -1551,13 +1550,8 @@ class XllmDecoderLayer(nn.Module):
positions: torch.Tensor,
hidden_states: torch.Tensor,
forward_batch: ForwardBatch,
residual: Optional[torch.Tensor],
) -> Tuple[torch.Tensor, torch.Tensor]:
hidden_states, residual = self.layer_communicator.prepare_attn(
hidden_states,
residual,
forward_batch,
)
) -> torch.Tensor:
hidden_states = self.attn_stage.prepare(hidden_states, forward_batch)
if hidden_states.shape[0] != 0:
hidden_states = self.self_attn(
@@ -1566,23 +1560,17 @@ class XllmDecoderLayer(nn.Module):
forward_batch=forward_batch,
)
hidden_states, residual = self.layer_communicator.prepare_mlp(
hidden_states, residual, forward_batch
)
hidden_states = self.attn_stage.finish(hidden_states, forward_batch)
hidden_states = self.ffn_stage.prepare(hidden_states, forward_batch)
with self.layer_communicator.ffn_exit(forward_batch) as ffn_exit:
use_reduce_scatter = ffn_exit.mlp_reduce_scatter
with self.ffn_stage.exit(forward_batch) as ffn_exit:
if isinstance(self.mlp, XllmMLP):
hidden_states = self.mlp(
hidden_states, use_reduce_scatter=use_reduce_scatter
)
hidden_states = self.mlp(hidden_states)
else:
hidden_states = self.mlp(
hidden_states, forward_batch, use_reduce_scatter
)
hidden_states, residual = ffn_exit.finish(hidden_states, residual)
hidden_states = self.mlp(hidden_states, forward_batch)
hidden_states = ffn_exit.finish(hidden_states)
return hidden_states, residual
return hidden_states
class XllmModel(nn.Module):
@@ -1638,12 +1626,12 @@ class XllmModel(nn.Module):
hidden_states = self.embed_tokens(input_ids)
else:
hidden_states = input_embeds
residual = residual_batch.start(forward_batch)
residual_batch.start(forward_batch)
else:
assert pp_proxy_tensors is not None
hidden_states, residual = self.layers[
self.start_layer
].layer_communicator.from_pp(pp_proxy_tensors, forward_batch)
hidden_states = self.layers[self.start_layer].attn_stage.from_pp(
pp_proxy_tensors, forward_batch
)
for i in range(self.start_layer, self.end_layer):
ctx = (
@@ -1653,34 +1641,16 @@ class XllmModel(nn.Module):
)
with ctx:
layer = self.layers[i]
hidden_states, residual = layer(
positions,
hidden_states,
forward_batch,
residual,
)
hidden_states = layer(positions, hidden_states, forward_batch)
hidden_states, residual = self.layers[
self.end_layer - 1
].layer_communicator.finish_layer_stack(
hidden_states,
residual,
forward_batch,
preserve_declared=not self.pp_group.is_last_rank,
)
if not self.pp_group.is_last_rank:
return PPProxyTensors(
{
"hidden_states": hidden_states,
"residual": residual,
}
)
return residual_batch.to_pp(hidden_states, forward_batch)
else:
hidden_states = residual_batch.finish(hidden_states, forward_batch)
if hidden_states.shape[0] != 0:
if residual is None:
hidden_states = self.norm(hidden_states)
else:
hidden_states, _ = self.norm(hidden_states, residual)
hidden_states = residual_batch.norm(
hidden_states, forward_batch, self.norm
)
return hidden_states
+1
View File
@@ -71,6 +71,7 @@ def stub_plan():
plan._fusion_rows = None
plan._cp_steps = plan._sp_steps = plan._input_scattered_steps = None
plan.is_first_layer = False
plan.is_last_layer = False
plan.residual_in_hidden = False
# Keep production variant selection while tests replace individual paths.
plan._batch_steps = lambda batch: {
+2 -2
View File
@@ -35,8 +35,6 @@ from sglang.srt.layers.communicator.residual import (
)
COMMUNICATOR_MODULES = (
layer,
legacy_stage,
communicator,
allreduce,
exit,
@@ -54,6 +52,8 @@ COMMUNICATOR_MODULES = (
construction,
factories,
stage,
layer,
legacy_stage,
batch,
stream,
)