[Refactor] Migrate specialized decoder and overlap boundaries (#41553)
This commit is contained in:
@@ -626,9 +626,8 @@ class ExecComm(msgspec.Struct):
|
||||
help="Select FFN boundary reduction: ar uses all-reduce then token "
|
||||
"redistribution; rs and rsv permit fixed-size and variable-size "
|
||||
"reduce-scatter respectively; rs+rsv permits both (RSv first). "
|
||||
"Unsupported paths fall back to ar. The option applies only to FFN "
|
||||
"layers using stage boundaries, where auto resolves the model default. "
|
||||
"Required attention and MoE collectives and "
|
||||
"Unsupported paths fall back to ar. auto resolves the model default "
|
||||
"for all FFN layers. Required attention and MoE collectives and "
|
||||
"all-reduce fusion are unaffected.",
|
||||
resolvable=True,
|
||||
),
|
||||
|
||||
@@ -15,7 +15,7 @@ def handle_layernorm_sp(server_args: ServerArgs) -> None:
|
||||
"""Validate --enable-layernorm-sp against the resolved parallelism config.
|
||||
|
||||
Runs in the resolution pipeline rather than in the layers so a model that
|
||||
never builds a LayerCommunicator rejects the flag instead of ignoring it.
|
||||
does not use layer boundaries rejects the flag instead of ignoring it.
|
||||
"""
|
||||
cfg = resolving_view(server_args)
|
||||
if not cfg.enable_layernorm_sp:
|
||||
|
||||
@@ -18,6 +18,7 @@ from sglang.srt.layers.communicator import (
|
||||
Layout,
|
||||
tbo_split_moves,
|
||||
)
|
||||
from sglang.srt.layers.communicator.residual import batch as residual_batch
|
||||
from sglang.srt.layers.communicator.residual.access import finish_layer_stack
|
||||
from sglang.srt.layers.communicator.residual.stream import ResidualStream
|
||||
from sglang.srt.layers.moe import (
|
||||
@@ -930,6 +931,67 @@ def _compute_extend_num_tokens(input_ids, forward_mode: ForwardMode):
|
||||
# -------------------------------- Execution ---------------------------------------
|
||||
|
||||
|
||||
def model_forward_stages(
|
||||
layers,
|
||||
enable_tbo: bool,
|
||||
positions: torch.Tensor,
|
||||
forward_batch: ForwardBatch,
|
||||
hidden_states: torch.Tensor,
|
||||
zero_allocator: Optional[BumpAllocator] = None,
|
||||
):
|
||||
"""Run stage operations with an independent residual stream per microbatch."""
|
||||
strategy = OperationsStrategy.init_new_tbo(
|
||||
layers, forward_batch.global_forward_mode
|
||||
)
|
||||
inputs = dict(
|
||||
positions=positions,
|
||||
hidden_states=hidden_states,
|
||||
forward_batch=forward_batch,
|
||||
zero_allocator=zero_allocator,
|
||||
)
|
||||
if not enable_tbo:
|
||||
return execute_operations(inputs, strategy.operations)["hidden_states"]
|
||||
|
||||
stream = residual_batch.current(forward_batch)
|
||||
pending = stream.pending
|
||||
hidden_states, residual = stream.finish(hidden_states)
|
||||
inputs["hidden_states"] = hidden_states
|
||||
parts = _model_forward_tbo_split_inputs(
|
||||
**inputs, residual=residual, layer_input_rows=layers[0].attn_stage.input_rows
|
||||
)
|
||||
for part in parts:
|
||||
part["hidden_states"], child_stream = ResidualStream.arrive(
|
||||
part["hidden_states"],
|
||||
part.pop("residual"),
|
||||
pending.update if pending is not None else None,
|
||||
)
|
||||
part["forward_batch"].residual_stream = child_stream
|
||||
|
||||
original_len = hidden_states.shape[0]
|
||||
forward_batch.residual_stream = None
|
||||
del inputs, pending, stream, hidden_states, residual
|
||||
|
||||
context = (
|
||||
empty_context()
|
||||
if _is_hip
|
||||
else deep_gemm_wrapper.configure_deep_gemm_num_sms(strategy.deep_gemm_num_sms)
|
||||
)
|
||||
with context:
|
||||
outputs = execute_overlapped_operations(
|
||||
inputs_arr=parts,
|
||||
operations_arr=[strategy.operations] * 2,
|
||||
delta_stages=[0, strategy.tbo_delta_stages],
|
||||
)
|
||||
for output in outputs:
|
||||
output["residual"] = residual_batch.current(output["forward_batch"])
|
||||
hidden_states, forward_batch.residual_stream = _model_forward_tbo_merge_outputs(
|
||||
*outputs, original_len
|
||||
)
|
||||
for output in outputs:
|
||||
output["forward_batch"].residual_stream = None
|
||||
return hidden_states
|
||||
|
||||
|
||||
def model_forward_maybe_tbo(
|
||||
layers,
|
||||
enable_tbo: bool,
|
||||
|
||||
@@ -79,8 +79,8 @@ from sglang.srt.layers.communicator.residual.mhc import (
|
||||
|
||||
__all__ = [
|
||||
"LayerCommunicator",
|
||||
"MHCLayerCommunicator",
|
||||
"LayerFacts",
|
||||
"MHCLayerCommunicator",
|
||||
"declare_attn",
|
||||
"declare_ffn",
|
||||
"make_attn_stage",
|
||||
|
||||
@@ -191,17 +191,19 @@ class OutputBoundary:
|
||||
dp_step=dp_step,
|
||||
)
|
||||
steps = self.plan._batch_steps(forward_batch)
|
||||
if not steps.ffn_output.leaves_for_next_layer:
|
||||
defer_moe_finalize = (
|
||||
self.plan.fusions is not None
|
||||
and self.plan.fusions.can_defer_finalize(self.plan, forward_batch)
|
||||
)
|
||||
if not steps.ffn_output.leaves_for_next_layer and not (
|
||||
self.plan.is_last_layer and defer_moe_finalize
|
||||
):
|
||||
return FfnCompletion(
|
||||
defer_moe_finalize=False,
|
||||
fuse_mlp_allreduce=False,
|
||||
mlp_reduce_scatter=mlp_reduce_scatter,
|
||||
complete=complete_now,
|
||||
)
|
||||
defer_moe_finalize = (
|
||||
self.plan.fusions is not None
|
||||
and self.plan.fusions.can_defer_finalize(self.plan, forward_batch)
|
||||
)
|
||||
# Producers declare remaining work independently of the kernel chosen
|
||||
# by the consumer. Every handoff also carries an unfused completion.
|
||||
fuse_mlp_allreduce = defer_moe_finalize or self._ffn_sum_moves_to_next_layer(
|
||||
|
||||
@@ -16,7 +16,6 @@ import torch
|
||||
from sglang.srt.layers.communicator import (
|
||||
FusedMlpInput,
|
||||
HandoffOutput,
|
||||
LayerCommunicator,
|
||||
SumGroup,
|
||||
TokenAxis,
|
||||
UnreducedOutput,
|
||||
@@ -196,7 +195,7 @@ class CuteDSLFusionService:
|
||||
|
||||
|
||||
class CuteDSLFusion:
|
||||
"""One layer's CuTe DSL kernels, given to its LayerCommunicator at
|
||||
"""One layer's CuTe DSL kernels, given to its stage at
|
||||
construction. At the attention input: the MoE finalize + AR + add + norm of
|
||||
a handoff the previous layer left, and the AR + add + norm of a sum it left.
|
||||
At the FFN input: the attention output's AR + add + norm. At the FFN exit:
|
||||
@@ -208,6 +207,7 @@ class CuteDSLFusion:
|
||||
self.service: CuteDSLFusionService | None = None
|
||||
# Producer capabilities; consumer kernel selection is independent.
|
||||
self.hands_off_finalize = False
|
||||
self.terminal_finalize = False
|
||||
self.requires_local_reduction = False
|
||||
|
||||
def install(
|
||||
@@ -216,25 +216,27 @@ class CuteDSLFusion:
|
||||
*,
|
||||
hands_off_finalize: bool,
|
||||
output_is_replicated: bool,
|
||||
terminal_finalize: bool = False,
|
||||
) -> None:
|
||||
"""Install the shared workspace and this producer's capabilities."""
|
||||
self.service = service
|
||||
self.hands_off_finalize = hands_off_finalize
|
||||
self.terminal_finalize = terminal_finalize
|
||||
self.requires_local_reduction = output_is_replicated
|
||||
|
||||
def attention_input(self, layer: LayerCommunicator) -> tuple:
|
||||
def attention_input(self, layer) -> tuple:
|
||||
return (
|
||||
partial(self._finalize_output_and_update_and_read_residual, layer),
|
||||
partial(self._reduce_output_and_update_and_read_residual, layer),
|
||||
)
|
||||
|
||||
def ffn_input(self, layer: LayerCommunicator) -> tuple:
|
||||
def ffn_input(self, layer) -> tuple:
|
||||
parallel = get_parallel()
|
||||
if (
|
||||
TokenAxis.ATTN_TP_SCATTER not in layer.input_rows.sharded
|
||||
# The workspace sums over TP, which is then the attention-TP group.
|
||||
and parallel.attn_tp_size == parallel.tp_size
|
||||
and _fused_norm_gamma(layer.post_attention_layernorm) is not None
|
||||
and _fused_norm_gamma(layer.norm) is not None
|
||||
):
|
||||
return (
|
||||
FusedMlpInput(
|
||||
@@ -255,7 +257,7 @@ class CuteDSLFusion:
|
||||
handoff is completed by its producer's own tail."""
|
||||
if not isinstance(owed, MoeFinalizeHandoff):
|
||||
return None
|
||||
gamma = _fused_norm_gamma(layer.input_layernorm)
|
||||
gamma = _fused_norm_gamma(layer.norm)
|
||||
if gamma is None or not self._should_use_finalize(layer, forward_batch, owed.m):
|
||||
return None
|
||||
if post_residual_addition is not None:
|
||||
@@ -280,7 +282,7 @@ class CuteDSLFusion:
|
||||
return self.service.all_reduce_residual_rms_norm(
|
||||
local_contribution=owed.partial,
|
||||
residual=residual,
|
||||
gamma=_fused_norm_gamma(layer.input_layernorm),
|
||||
gamma=_fused_norm_gamma(layer.norm),
|
||||
)
|
||||
|
||||
def _mlp_input_reduce_output_and_update_and_read_residual(
|
||||
@@ -298,7 +300,7 @@ class CuteDSLFusion:
|
||||
return self.service.all_reduce_residual_rms_norm(
|
||||
local_contribution=hidden_states,
|
||||
residual=residual,
|
||||
gamma=_fused_norm_gamma(layer.post_attention_layernorm),
|
||||
gamma=_fused_norm_gamma(layer.norm),
|
||||
)
|
||||
|
||||
def _should_use_finalize(self, layer, forward_batch: ForwardBatch, m: int) -> bool:
|
||||
@@ -313,7 +315,7 @@ class CuteDSLFusion:
|
||||
"""Incoming, and independent of this layer's own successor."""
|
||||
return (
|
||||
self._common_eligible(layer, forward_batch, m)
|
||||
and _fused_norm_gamma(layer.input_layernorm) is not None
|
||||
and _fused_norm_gamma(layer.norm) is not None
|
||||
and not get_exec().comm.enable_quant_communications
|
||||
)
|
||||
|
||||
@@ -334,7 +336,11 @@ class CuteDSLFusion:
|
||||
"""Whether this producer can emit a handoff with an unfused fallback."""
|
||||
return (
|
||||
self.hands_off_finalize
|
||||
and not self.requires_local_reduction
|
||||
and (not layer.is_last_layer or self.terminal_finalize)
|
||||
and (
|
||||
not self.requires_local_reduction
|
||||
or (layer.is_last_layer and self.terminal_finalize)
|
||||
)
|
||||
and self._should_use_finalize(
|
||||
layer, forward_batch, int(forward_batch.input_ids.shape[0])
|
||||
)
|
||||
@@ -355,12 +361,12 @@ class CuteDSLFusion:
|
||||
and parallel.tp_size > 1
|
||||
# The FFN runs on the full rows, not each rank's own slice.
|
||||
and TokenAxis.ATTN_TP_SCATTER
|
||||
not in layer.ffn.entry(forward_batch).input_rows.sharded
|
||||
not in layer.fusion_rows(forward_batch).sharded
|
||||
)
|
||||
|
||||
|
||||
def _fusion_of(layer: torch.nn.Module) -> CuteDSLFusion | None:
|
||||
fusions = getattr(layer.layer_communicator, "fusions", None)
|
||||
fusions = getattr(layer.__dict__.get("ffn_stage"), "fusions", None)
|
||||
return fusions if isinstance(fusions, CuteDSLFusion) else None
|
||||
|
||||
|
||||
@@ -373,10 +379,11 @@ def install_cutedsl_fusion(
|
||||
can_defer_finalize: _LayerPredicate,
|
||||
requires_local_reduction: _LayerPredicate | None = None,
|
||||
label: str,
|
||||
terminal_finalize: bool = False,
|
||||
) -> CuteDSLFusionService | None:
|
||||
"""One shared workspace handle per fusion-enabled layer, or None.
|
||||
|
||||
Every entry of ``layers`` must carry a ``layer_communicator``.
|
||||
Every entry of ``layers`` carries its attention and FFN stages.
|
||||
"""
|
||||
if get_flags().moe.in_speculative_scope:
|
||||
# A draft shares the target's process, which holds one workspace.
|
||||
@@ -389,8 +396,8 @@ def install_cutedsl_fusion(
|
||||
# The workspace compiles one epsilon and cannot see a per-layer one.
|
||||
for layer in fusion_layers:
|
||||
for norm in (
|
||||
layer.layer_communicator.input_layernorm,
|
||||
layer.layer_communicator.post_attention_layernorm,
|
||||
layer.attn_stage.norm,
|
||||
layer.ffn_stage.norm,
|
||||
):
|
||||
if _fused_norm_gamma(norm) is None:
|
||||
continue
|
||||
@@ -416,6 +423,7 @@ def install_cutedsl_fusion(
|
||||
fusion.install(
|
||||
service,
|
||||
hands_off_finalize=hands_off_finalize,
|
||||
terminal_finalize=terminal_finalize,
|
||||
output_is_replicated=requires_local_reduction is not None
|
||||
and bool(requires_local_reduction(layer)),
|
||||
)
|
||||
@@ -441,11 +449,9 @@ def prepare_cutedsl_fusion(
|
||||
fusions = [
|
||||
fusion
|
||||
for module in model.modules()
|
||||
# Most modules carry no layer_communicator.
|
||||
# Only decoder modules carry an FFN stage.
|
||||
if isinstance(
|
||||
fusion := getattr(
|
||||
module.__dict__.get("layer_communicator"), "fusions", None
|
||||
),
|
||||
fusion := getattr(module.__dict__.get("ffn_stage"), "fusions", None),
|
||||
CuteDSLFusion,
|
||||
)
|
||||
]
|
||||
|
||||
@@ -45,11 +45,37 @@ def take_output(hidden_states, forward_batch):
|
||||
return hidden_states
|
||||
|
||||
|
||||
def norm(hidden_states, forward_batch, layernorm, capture_output=None, **read_kwargs):
|
||||
hidden_states, residual = current(forward_batch).finish(hidden_states)
|
||||
def norm(
|
||||
hidden_states,
|
||||
forward_batch,
|
||||
layernorm,
|
||||
capture_output=None,
|
||||
*,
|
||||
handoff_norm=None,
|
||||
skip_empty=False,
|
||||
**read_kwargs,
|
||||
):
|
||||
hidden_states, residual = current(forward_batch).finish(
|
||||
hidden_states, takes_handoff=handoff_norm is not None
|
||||
)
|
||||
# The terminal consumer now owns the pair. Do not keep layer buffers alive
|
||||
# through logits processing or the next forward on this batch.
|
||||
forward_batch.residual_stream = None
|
||||
from sglang.srt.layers.communicator.output import HandoffOutput
|
||||
|
||||
if isinstance(hidden_states, HandoffOutput):
|
||||
if residual is None:
|
||||
raise RuntimeError("invalid final deferred MoE handoff")
|
||||
if capture_output is not None:
|
||||
raise RuntimeError(
|
||||
"final handoff capture requires an explicit capture adapter"
|
||||
)
|
||||
hidden_states, _ = handoff_norm.finalize(
|
||||
handoff=hidden_states, residual=residual, gamma=layernorm.gemma_weight
|
||||
)
|
||||
return hidden_states
|
||||
if skip_empty and hidden_states.shape[0] == 0:
|
||||
return hidden_states
|
||||
return access.norm_output(
|
||||
hidden_states, residual, layernorm, capture_output, **read_kwargs
|
||||
)
|
||||
|
||||
@@ -25,7 +25,7 @@ Everything SP lives here so the feature stays decoupled from model code and from
|
||||
|
||||
- which models opt in (the Qwen3-dense allowlist) and config validation,
|
||||
- the per-forward ``sp_active`` flag (a ForwardFlags bool) read at depth by the
|
||||
participant linears and the ``LayerCommunicator``,
|
||||
participant linears and the layer-boundary batch selector,
|
||||
- the entry-scatter / exit-gather collectives, and
|
||||
- the fused matmul + collective fast-paths for the participant linears.
|
||||
|
||||
@@ -47,7 +47,7 @@ from sglang.srt.runtime_context import (
|
||||
from sglang.srt.utils.common import ceil_align
|
||||
|
||||
# Architectures whose decoder layers route attention/MLP through
|
||||
# ``LayerCommunicator`` with the standard participant linears, and for which SP
|
||||
# layer boundaries with the standard participant linears, and for which SP
|
||||
# has been validated. Other models reject --enable-layernorm-sp at construction.
|
||||
# The mechanism is generic; extend the allowlist as families are validated.
|
||||
SP_SUPPORTED_ARCHITECTURES = frozenset({"Qwen3ForCausalLM"})
|
||||
|
||||
@@ -216,27 +216,21 @@ class DeepseekModelNextN(nn.Module):
|
||||
else:
|
||||
hidden_states = self.eh_proj(eh_input)
|
||||
|
||||
residual = residual_batch.start(forward_batch)
|
||||
residual_batch.start(forward_batch)
|
||||
index_topk_share = IndexTopKShareState.from_mtp_carry(forward_batch)
|
||||
with get_global_expert_distribution_recorder().disable_this_region():
|
||||
hidden_states, residual, topk_indices = self.decoder(
|
||||
(hidden_states, topk_indices) = self.decoder(
|
||||
positions,
|
||||
hidden_states,
|
||||
forward_batch,
|
||||
residual,
|
||||
zero_allocator,
|
||||
prev_topk_indices=index_topk_share.topk_indices,
|
||||
)
|
||||
hidden_states, residual = (
|
||||
self.decoder.layer_communicator.finish_layer_stack(
|
||||
hidden_states, residual, forward_batch
|
||||
)
|
||||
)
|
||||
hidden_states = residual_batch.finish(hidden_states, forward_batch)
|
||||
if not forward_batch.forward_mode.is_idle():
|
||||
if residual is not None:
|
||||
hidden_states, _ = self.shared_head.norm(hidden_states, residual)
|
||||
else:
|
||||
hidden_states = self.shared_head.norm(hidden_states)
|
||||
hidden_states = residual_batch.norm(
|
||||
hidden_states, forward_batch, self.shared_head.norm
|
||||
)
|
||||
|
||||
index_topk_share.update(topk_indices)
|
||||
index_topk_share.publish()
|
||||
|
||||
@@ -40,7 +40,7 @@ from sglang.kernels.ops.quantization.fp8_kernel import (
|
||||
from sglang.srt.batch_overlap.single_batch_overlap import SboFlags, compute_overlap_args
|
||||
from sglang.srt.batch_overlap.two_batch_overlap import (
|
||||
MaybeTboDeepEPDispatcher,
|
||||
model_forward_maybe_tbo,
|
||||
model_forward_stages,
|
||||
)
|
||||
from sglang.srt.configs.model_config import (
|
||||
compute_mla_mscale_scaling,
|
||||
@@ -68,14 +68,16 @@ from sglang.srt.layers.aux_hidden_states import (
|
||||
AuxHiddenStatePacker,
|
||||
)
|
||||
from sglang.srt.layers.communicator import (
|
||||
LayerCommunicator,
|
||||
LayerFacts,
|
||||
declare_attn,
|
||||
declare_ffn,
|
||||
enable_moe_dense_fully_dp,
|
||||
get_attn_tp_context,
|
||||
make_stages,
|
||||
)
|
||||
from sglang.srt.layers.communicator.adapters.context_parallel import (
|
||||
maybe_prefetch_next_full_attention_kv,
|
||||
)
|
||||
from sglang.srt.layers.communicator.residual import access as residual_access
|
||||
from sglang.srt.layers.communicator.residual import batch as residual_batch
|
||||
from sglang.srt.layers.cp.cp_decode_attn_tp import get_cp_decode_attn_tp_ctx
|
||||
from sglang.srt.layers.dcp.planner import (
|
||||
@@ -2578,13 +2580,12 @@ class DeepseekV2DecoderLayer(nn.Module):
|
||||
is_previous_layer_sparse = self._is_layer_sparse(layer_id - 1, is_nextn=False)
|
||||
is_next_layer_sparse = self._is_layer_sparse(layer_id + 1, is_nextn=False)
|
||||
|
||||
self.layer_facts = LayerFacts.init_new(
|
||||
layer_id=layer_id,
|
||||
num_layers=1 if is_nextn else config.num_hidden_layers,
|
||||
is_layer_sparse=self.is_layer_sparse,
|
||||
is_previous_layer_sparse=is_previous_layer_sparse,
|
||||
is_next_layer_sparse=is_next_layer_sparse,
|
||||
)
|
||||
self._stage_enters_stack = (layer_id) == 0
|
||||
self._stage_terminal = (layer_id) == (
|
||||
1 if is_nextn else config.num_hidden_layers
|
||||
) - 1
|
||||
self._stage_previous_sparse = is_previous_layer_sparse
|
||||
self._stage_next_sparse = is_next_layer_sparse
|
||||
|
||||
if self.is_layer_sparse:
|
||||
self.mlp = DeepseekV2MoE(
|
||||
@@ -2618,19 +2619,19 @@ class DeepseekV2DecoderLayer(nn.Module):
|
||||
|
||||
self._gfx95_quant_format = self._detect_gfx95_quant_format()
|
||||
|
||||
self.layer_communicator = self._build_layer_communicator(
|
||||
self.attn_stage, self.ffn_stage = self._build_stages(
|
||||
input_layernorm=self.input_layernorm,
|
||||
post_attention_layernorm=self.post_attention_layernorm,
|
||||
qkv_latent_func=self.self_attn.prepare_qkv_latent,
|
||||
)
|
||||
|
||||
def _build_layer_communicator(
|
||||
def _build_stages(
|
||||
self,
|
||||
*,
|
||||
input_layernorm: nn.Module,
|
||||
post_attention_layernorm: nn.Module,
|
||||
qkv_latent_func: Optional[Callable],
|
||||
allow_deferred_ffn_reduction: bool = True,
|
||||
output=None,
|
||||
):
|
||||
"""The communicator for this layer's norms; it chooses its boundary
|
||||
steps from them at construction."""
|
||||
@@ -2644,15 +2645,29 @@ class DeepseekV2DecoderLayer(nn.Module):
|
||||
from sglang.srt.layers.communicator.fusions.cutedsl import CuteDSLFusion
|
||||
|
||||
fusions = CuteDSLFusion()
|
||||
return LayerCommunicator(
|
||||
layer_facts=self.layer_facts,
|
||||
input_layernorm=input_layernorm,
|
||||
post_attention_layernorm=post_attention_layernorm,
|
||||
allow_reduce_scatter=True,
|
||||
qkv_latent_func=qkv_latent_func,
|
||||
allow_deferred_ffn_reduction=allow_deferred_ffn_reduction,
|
||||
fusions=fusions,
|
||||
attn_stage, ffn_stage = make_stages(
|
||||
(
|
||||
declare_attn(),
|
||||
input_layernorm,
|
||||
{"qkv_latent_func": qkv_latent_func, "fusions": fusions},
|
||||
),
|
||||
(
|
||||
declare_ffn(
|
||||
sparse=self.is_layer_sparse,
|
||||
next_sparse=self._stage_next_sparse,
|
||||
output=output,
|
||||
),
|
||||
post_attention_layernorm,
|
||||
{"fusions": fusions},
|
||||
),
|
||||
previous=declare_ffn(
|
||||
sparse=self._stage_previous_sparse, next_sparse=self.is_layer_sparse
|
||||
)
|
||||
if not self._stage_enters_stack
|
||||
else None,
|
||||
terminal=self._stage_terminal,
|
||||
)
|
||||
return attn_stage, ffn_stage
|
||||
|
||||
def _detect_gfx95_quant_format(self) -> str:
|
||||
if not _is_gfx95_supported:
|
||||
@@ -2698,7 +2713,6 @@ class DeepseekV2DecoderLayer(nn.Module):
|
||||
positions: torch.Tensor,
|
||||
hidden_states: torch.Tensor,
|
||||
forward_batch: ForwardBatch,
|
||||
residual: Optional[torch.Tensor],
|
||||
zero_allocator: BumpAllocator,
|
||||
gemm_output_zero_allocator: BumpAllocator = None,
|
||||
llama_4_scaling: Optional[torch.Tensor] = None,
|
||||
@@ -2706,15 +2720,12 @@ class DeepseekV2DecoderLayer(nn.Module):
|
||||
captured_last_layer_outputs: Optional[AuxHiddenStateAccumulator] = None,
|
||||
next_full_attention_layer_id: Optional[int] = None,
|
||||
) -> torch.Tensor:
|
||||
hidden_states_orig = self.layer_communicator.buffer(hidden_states)
|
||||
hidden_states, residual = (
|
||||
self.layer_communicator.prepare_attn_and_capture_last_layer_outputs(
|
||||
hidden_states,
|
||||
residual,
|
||||
forward_batch,
|
||||
captured_last_layer_outputs=captured_last_layer_outputs,
|
||||
quant_format=self._resolve_gfx95_quant_format(),
|
||||
)
|
||||
hidden_states_orig = residual_access.buffer(hidden_states)
|
||||
hidden_states = self.attn_stage.prepare(
|
||||
hidden_states,
|
||||
forward_batch,
|
||||
captured_last_layer_outputs=captured_last_layer_outputs,
|
||||
quant_format=self._resolve_gfx95_quant_format(),
|
||||
)
|
||||
|
||||
with self.self_attn.maybe_use_decode_attn_tp(forward_batch):
|
||||
@@ -2725,7 +2736,7 @@ class DeepseekV2DecoderLayer(nn.Module):
|
||||
zero_allocator=zero_allocator,
|
||||
llama_4_scaling=llama_4_scaling,
|
||||
input_on_attention_tp_slices=(
|
||||
self.layer_communicator.input_on_attention_tp_slices
|
||||
self.attn_stage.input_on_attention_tp_slices
|
||||
),
|
||||
prev_topk_indices=prev_topk_indices,
|
||||
)
|
||||
@@ -2739,9 +2750,8 @@ class DeepseekV2DecoderLayer(nn.Module):
|
||||
forward_batch, next_full_attention_layer_id
|
||||
)
|
||||
|
||||
hidden_states, residual = self.layer_communicator.prepare_mlp(
|
||||
hidden_states, residual, forward_batch
|
||||
)
|
||||
hidden_states = self.attn_stage.finish(hidden_states, forward_batch)
|
||||
hidden_states = self.ffn_stage.prepare(hidden_states, forward_batch)
|
||||
|
||||
if isinstance(self.mlp, DeepseekV2MLP):
|
||||
gemm_output_zero_allocator = None
|
||||
@@ -2759,15 +2769,15 @@ class DeepseekV2DecoderLayer(nn.Module):
|
||||
else:
|
||||
_mlp_ctx = nullcontext()
|
||||
|
||||
with self.layer_communicator.ffn_exit(forward_batch) as ffn_exit, _mlp_ctx:
|
||||
with self.ffn_stage.exit(forward_batch) as ffn_exit, _mlp_ctx:
|
||||
hidden_states = self.mlp(
|
||||
hidden_states,
|
||||
forward_batch,
|
||||
gemm_output_zero_allocator,
|
||||
)
|
||||
hidden_states, residual = ffn_exit.finish(hidden_states, residual)
|
||||
hidden_states = ffn_exit.finish(hidden_states)
|
||||
|
||||
return hidden_states, residual, topk_indices
|
||||
return (hidden_states, topk_indices)
|
||||
|
||||
def op_comm_prepare_attn(
|
||||
self,
|
||||
@@ -2775,12 +2785,11 @@ class DeepseekV2DecoderLayer(nn.Module):
|
||||
positions: torch.Tensor,
|
||||
hidden_states: torch.Tensor,
|
||||
forward_batch: ForwardBatch,
|
||||
residual: Optional[torch.Tensor],
|
||||
zero_allocator: BumpAllocator,
|
||||
tbo_subbatch_index: Optional[int] = None,
|
||||
):
|
||||
state.hidden_states_after_comm_pre_attn, state.residual_after_input_ln = (
|
||||
self.layer_communicator.prepare_attn(hidden_states, residual, forward_batch)
|
||||
state.hidden_states_after_comm_pre_attn = self.attn_stage.prepare(
|
||||
hidden_states, forward_batch
|
||||
)
|
||||
if get_moe_a2a_backend().is_mori():
|
||||
state.num_tokens = hidden_states.shape[0]
|
||||
@@ -2794,25 +2803,21 @@ class DeepseekV2DecoderLayer(nn.Module):
|
||||
)
|
||||
|
||||
def op_comm_prepare_mlp(self, state):
|
||||
state.hidden_states_mlp_input, state.residual_after_comm_pre_mlp = (
|
||||
self.layer_communicator.prepare_mlp(
|
||||
state.pop("hidden_states_after_attn"),
|
||||
state.pop("residual_after_input_ln"),
|
||||
state.forward_batch,
|
||||
)
|
||||
hidden_states = self.attn_stage.finish(
|
||||
state.pop("hidden_states_after_attn"), state.forward_batch
|
||||
)
|
||||
state.hidden_states_mlp_input = self.ffn_stage.prepare(
|
||||
hidden_states, state.forward_batch
|
||||
)
|
||||
|
||||
def op_comm_postprocess_layer(self, state):
|
||||
hidden_states, residual = self.layer_communicator.postprocess_layer(
|
||||
state.pop("hidden_states_mlp_output"),
|
||||
state.pop("residual_after_comm_pre_mlp"),
|
||||
state.forward_batch,
|
||||
hidden_states = self.ffn_stage.postprocess(
|
||||
state.pop("hidden_states_mlp_output"), state.forward_batch
|
||||
)
|
||||
|
||||
output = dict(
|
||||
positions=state.positions,
|
||||
hidden_states=hidden_states,
|
||||
residual=residual,
|
||||
forward_batch=state.forward_batch,
|
||||
zero_allocator=state.zero_allocator,
|
||||
tbo_subbatch_index=state.tbo_subbatch_index,
|
||||
@@ -3027,13 +3032,13 @@ class DeepseekV2Model(nn.Module):
|
||||
hidden_states = self.embed_tokens(input_ids)
|
||||
else:
|
||||
hidden_states = input_embeds
|
||||
residual = residual_batch.start(forward_batch)
|
||||
residual_batch.start(forward_batch)
|
||||
initial_topk_indices = None
|
||||
else:
|
||||
assert pp_proxy_tensors is not None
|
||||
hidden_states, residual = self.layers[
|
||||
self.start_layer
|
||||
].layer_communicator.from_pp(pp_proxy_tensors, forward_batch)
|
||||
hidden_states = self.layers[self.start_layer].attn_stage.from_pp(
|
||||
pp_proxy_tensors, forward_batch
|
||||
)
|
||||
initial_topk_indices = pp_proxy_tensors.tensors.get("topk_indices")
|
||||
index_topk_share = IndexTopKShareState(forward_batch, initial_topk_indices)
|
||||
if not self.pp_group.is_first_rank:
|
||||
@@ -3103,18 +3108,17 @@ class DeepseekV2Model(nn.Module):
|
||||
)
|
||||
with ctx:
|
||||
layer = self.layers[i]
|
||||
hidden_states, residual, topk_indices = layer(
|
||||
(hidden_states, topk_indices) = layer(
|
||||
positions,
|
||||
hidden_states,
|
||||
forward_batch,
|
||||
residual,
|
||||
zero_allocator,
|
||||
gemm_output_zero_allocator,
|
||||
llama_4_scaling,
|
||||
prev_topk_indices=index_topk_share.topk_indices,
|
||||
captured_last_layer_outputs=(
|
||||
aux_hidden_states if i in self.layers_to_capture else None
|
||||
),
|
||||
captured_last_layer_outputs=aux_hidden_states
|
||||
if i in self.layers_to_capture
|
||||
else None,
|
||||
next_full_attention_layer_id=self.next_full_attention_layer_id.get(
|
||||
i
|
||||
),
|
||||
@@ -3122,28 +3126,17 @@ class DeepseekV2Model(nn.Module):
|
||||
index_topk_share.update(topk_indices)
|
||||
|
||||
if normal_end_layer != self.end_layer:
|
||||
hidden_states, residual = model_forward_maybe_tbo(
|
||||
hidden_states = model_forward_stages(
|
||||
layers=self.layers[normal_end_layer : self.end_layer],
|
||||
enable_tbo=True,
|
||||
positions=positions,
|
||||
forward_batch=forward_batch,
|
||||
hidden_states=hidden_states,
|
||||
residual=residual,
|
||||
zero_allocator=zero_allocator,
|
||||
)
|
||||
|
||||
last_layer = self.layers[self.end_layer - 1]
|
||||
hidden_states, residual = last_layer.layer_communicator.finish_layer_stack(
|
||||
hidden_states,
|
||||
residual,
|
||||
forward_batch,
|
||||
preserve_declared=not self.pp_group.is_last_rank,
|
||||
)
|
||||
if not self.pp_group.is_last_rank:
|
||||
proxy_tensors = {
|
||||
"hidden_states": hidden_states,
|
||||
"residual": residual,
|
||||
}
|
||||
proxy_tensors = residual_batch.to_pp(hidden_states, forward_batch)
|
||||
if (
|
||||
self.use_dsa
|
||||
and dsa_forward_uses_topk
|
||||
@@ -3164,14 +3157,14 @@ class DeepseekV2Model(nn.Module):
|
||||
topk_indices = hidden_states.new_empty(
|
||||
(0, get_dsa_index_topk(self.config)), dtype=torch.int32
|
||||
)
|
||||
proxy_tensors["topk_indices"] = topk_indices
|
||||
return PPProxyTensors(proxy_tensors)
|
||||
proxy_tensors.tensors["topk_indices"] = topk_indices
|
||||
return proxy_tensors
|
||||
else:
|
||||
hidden_states = residual_batch.finish(hidden_states, forward_batch)
|
||||
if not forward_batch.forward_mode.is_idle():
|
||||
if residual is None:
|
||||
hidden_states = self.norm(hidden_states)
|
||||
else:
|
||||
hidden_states, _ = self.norm(hidden_states, residual)
|
||||
hidden_states = residual_batch.norm(
|
||||
hidden_states, forward_batch, self.norm
|
||||
)
|
||||
|
||||
if len(aux_hidden_states) == 0:
|
||||
return hidden_states
|
||||
|
||||
@@ -42,7 +42,7 @@ from transformers import PretrainedConfig
|
||||
from sglang.kernels.ops.quantization.fp8_kernel import is_fp8_fnuz
|
||||
from sglang.srt.batch_overlap.two_batch_overlap import (
|
||||
MaybeTboDeepEPDispatcher,
|
||||
model_forward_maybe_tbo,
|
||||
model_forward_stages,
|
||||
)
|
||||
from sglang.srt.configs.dots3 import Dots3Config
|
||||
from sglang.srt.distributed import (
|
||||
@@ -58,9 +58,10 @@ from sglang.srt.layers import deep_gemm_wrapper
|
||||
from sglang.srt.layers.activation import SiluAndMul
|
||||
from sglang.srt.layers.attention.dsa.dsa_indexer import Indexer
|
||||
from sglang.srt.layers.communicator import (
|
||||
LayerCommunicator,
|
||||
LayerFacts,
|
||||
declare_attn,
|
||||
declare_ffn,
|
||||
enable_moe_dense_fully_dp,
|
||||
make_stages,
|
||||
)
|
||||
from sglang.srt.layers.communicator.residual import batch as residual_batch
|
||||
from sglang.srt.layers.dp_attention import is_dp_attention_enabled
|
||||
@@ -1542,14 +1543,6 @@ class Dots3DecoderLayer(nn.Module):
|
||||
is_previous_layer_sparse = self._is_layer_sparse(layer_id - 1, is_nextn=False)
|
||||
is_next_layer_sparse = self._is_layer_sparse(layer_id + 1, is_nextn=False)
|
||||
|
||||
self.layer_facts = LayerFacts.init_new(
|
||||
layer_id=layer_id,
|
||||
num_layers=1 if is_nextn else config.num_hidden_layers,
|
||||
is_layer_sparse=self.is_layer_sparse,
|
||||
is_previous_layer_sparse=is_previous_layer_sparse,
|
||||
is_next_layer_sparse=is_next_layer_sparse,
|
||||
)
|
||||
|
||||
if self.is_layer_sparse:
|
||||
self.mlp = Dots3MoE(
|
||||
config=config,
|
||||
@@ -1579,11 +1572,21 @@ class Dots3DecoderLayer(nn.Module):
|
||||
config.hidden_size, eps=config.rms_norm_eps
|
||||
)
|
||||
|
||||
self.layer_communicator = LayerCommunicator(
|
||||
layer_facts=self.layer_facts,
|
||||
input_layernorm=self.input_layernorm,
|
||||
post_attention_layernorm=self.post_attention_layernorm,
|
||||
allow_reduce_scatter=True,
|
||||
self.attn_stage, self.ffn_stage = make_stages(
|
||||
(declare_attn(), self.input_layernorm),
|
||||
(
|
||||
declare_ffn(
|
||||
sparse=self.is_layer_sparse,
|
||||
next_sparse=is_next_layer_sparse,
|
||||
),
|
||||
self.post_attention_layernorm,
|
||||
),
|
||||
previous=declare_ffn(
|
||||
sparse=is_previous_layer_sparse, next_sparse=self.is_layer_sparse
|
||||
)
|
||||
if layer_id != 0
|
||||
else None,
|
||||
terminal=layer_id == (1 if is_nextn else config.num_hidden_layers) - 1,
|
||||
)
|
||||
|
||||
def _is_layer_sparse(self, layer_id: int, is_nextn: bool) -> bool:
|
||||
@@ -1601,13 +1604,10 @@ class Dots3DecoderLayer(nn.Module):
|
||||
positions: torch.Tensor,
|
||||
hidden_states: torch.Tensor,
|
||||
forward_batch: ForwardBatch,
|
||||
residual: Optional[torch.Tensor],
|
||||
zero_allocator: BumpAllocator,
|
||||
) -> torch.Tensor:
|
||||
|
||||
hidden_states, residual = self.layer_communicator.prepare_attn(
|
||||
hidden_states, residual, forward_batch
|
||||
)
|
||||
hidden_states = self.attn_stage.prepare(hidden_states, forward_batch)
|
||||
|
||||
hidden_states = self.self_attn(
|
||||
positions=positions,
|
||||
@@ -1616,18 +1616,17 @@ class Dots3DecoderLayer(nn.Module):
|
||||
zero_allocator=zero_allocator,
|
||||
)
|
||||
|
||||
hidden_states, residual = self.layer_communicator.prepare_mlp(
|
||||
hidden_states, residual, forward_batch
|
||||
)
|
||||
hidden_states = self.attn_stage.finish(hidden_states, forward_batch)
|
||||
hidden_states = self.ffn_stage.prepare(hidden_states, forward_batch)
|
||||
|
||||
with self.layer_communicator.ffn_exit(forward_batch) as ffn_exit:
|
||||
with self.ffn_stage.exit(forward_batch) as ffn_exit:
|
||||
hidden_states = self.mlp(
|
||||
hidden_states,
|
||||
forward_batch,
|
||||
ffn_exit.fuse_mlp_allreduce,
|
||||
ffn_exit.mlp_reduce_scatter,
|
||||
)
|
||||
return ffn_exit.finish(hidden_states, residual)
|
||||
return ffn_exit.finish(hidden_states)
|
||||
|
||||
def op_comm_prepare_attn(
|
||||
self,
|
||||
@@ -1635,12 +1634,11 @@ class Dots3DecoderLayer(nn.Module):
|
||||
positions: torch.Tensor,
|
||||
hidden_states: torch.Tensor,
|
||||
forward_batch: ForwardBatch,
|
||||
residual: Optional[torch.Tensor],
|
||||
zero_allocator: BumpAllocator,
|
||||
tbo_subbatch_index: Optional[int] = None,
|
||||
):
|
||||
state.hidden_states_after_comm_pre_attn, state.residual_after_input_ln = (
|
||||
self.layer_communicator.prepare_attn(hidden_states, residual, forward_batch)
|
||||
state.hidden_states_after_comm_pre_attn = self.attn_stage.prepare(
|
||||
hidden_states, forward_batch
|
||||
)
|
||||
state.update(
|
||||
dict(
|
||||
@@ -1652,12 +1650,11 @@ class Dots3DecoderLayer(nn.Module):
|
||||
)
|
||||
|
||||
def op_comm_prepare_mlp(self, state):
|
||||
state.hidden_states_mlp_input, state.residual_after_comm_pre_mlp = (
|
||||
self.layer_communicator.prepare_mlp(
|
||||
state.pop("hidden_states_after_attn"),
|
||||
state.pop("residual_after_input_ln"),
|
||||
state.forward_batch,
|
||||
)
|
||||
hidden_states = self.attn_stage.finish(
|
||||
state.pop("hidden_states_after_attn"), state.forward_batch
|
||||
)
|
||||
state.hidden_states_mlp_input = self.ffn_stage.prepare(
|
||||
hidden_states, state.forward_batch
|
||||
)
|
||||
|
||||
def op_mlp(self, state):
|
||||
@@ -1674,16 +1671,13 @@ class Dots3DecoderLayer(nn.Module):
|
||||
state.hidden_states_mlp_output = hidden_states
|
||||
|
||||
def op_comm_postprocess_layer(self, state):
|
||||
hidden_states, residual = self.layer_communicator.postprocess_layer(
|
||||
state.pop("hidden_states_mlp_output"),
|
||||
state.pop("residual_after_comm_pre_mlp"),
|
||||
state.forward_batch,
|
||||
hidden_states = self.ffn_stage.postprocess(
|
||||
state.pop("hidden_states_mlp_output"), state.forward_batch
|
||||
)
|
||||
|
||||
output = dict(
|
||||
positions=state.positions,
|
||||
hidden_states=hidden_states,
|
||||
residual=residual,
|
||||
forward_batch=state.forward_batch,
|
||||
zero_allocator=state.zero_allocator,
|
||||
tbo_subbatch_index=state.tbo_subbatch_index,
|
||||
@@ -1766,12 +1760,12 @@ class Dots3Model(nn.Module):
|
||||
hidden_states = self.embed_tokens(input_ids)
|
||||
else:
|
||||
hidden_states = input_embeds
|
||||
residual = residual_batch.start(forward_batch)
|
||||
residual_batch.start(forward_batch)
|
||||
else:
|
||||
assert pp_proxy_tensors is not None
|
||||
hidden_states, residual = self.layers[
|
||||
self.start_layer
|
||||
].layer_communicator.from_pp(pp_proxy_tensors, forward_batch)
|
||||
hidden_states = self.layers[self.start_layer].attn_stage.from_pp(
|
||||
pp_proxy_tensors, forward_batch
|
||||
)
|
||||
|
||||
normal_start_layer = self.start_layer
|
||||
normal_end_layer = self.end_layer
|
||||
@@ -1787,38 +1781,28 @@ class Dots3Model(nn.Module):
|
||||
for i in range(normal_start_layer, normal_end_layer):
|
||||
with get_global_expert_distribution_recorder().with_current_layer(i):
|
||||
layer = self.layers[i]
|
||||
hidden_states, residual = layer(
|
||||
positions, hidden_states, forward_batch, residual, zero_allocator
|
||||
hidden_states = layer(
|
||||
positions, hidden_states, forward_batch, zero_allocator
|
||||
)
|
||||
|
||||
if normal_end_layer != self.end_layer:
|
||||
hidden_states, residual = model_forward_maybe_tbo(
|
||||
hidden_states = model_forward_stages(
|
||||
layers=self.layers[normal_end_layer : self.end_layer],
|
||||
enable_tbo=True,
|
||||
positions=positions,
|
||||
forward_batch=forward_batch,
|
||||
hidden_states=hidden_states,
|
||||
residual=residual,
|
||||
zero_allocator=zero_allocator,
|
||||
)
|
||||
|
||||
last_layer = self.layers[self.end_layer - 1]
|
||||
hidden_states, residual = last_layer.layer_communicator.finish_layer_stack(
|
||||
hidden_states, residual, forward_batch
|
||||
)
|
||||
hidden_states = residual_batch.finish(hidden_states, forward_batch)
|
||||
if not self.pp_group.is_last_rank:
|
||||
return PPProxyTensors(
|
||||
{
|
||||
"hidden_states": hidden_states,
|
||||
"residual": residual,
|
||||
}
|
||||
)
|
||||
return residual_batch.to_pp(hidden_states, forward_batch)
|
||||
else:
|
||||
if not forward_batch.forward_mode.is_idle():
|
||||
if residual is None:
|
||||
hidden_states = self.norm(hidden_states)
|
||||
else:
|
||||
hidden_states, _ = self.norm(hidden_states, residual)
|
||||
hidden_states = residual_batch.norm(
|
||||
hidden_states, forward_batch, self.norm
|
||||
)
|
||||
return hidden_states
|
||||
|
||||
|
||||
|
||||
@@ -114,20 +114,17 @@ class Dot3NoteModelNextN(nn.Module):
|
||||
)
|
||||
)
|
||||
|
||||
residual = residual_batch.start(forward_batch)
|
||||
residual_batch.start(forward_batch)
|
||||
with get_global_expert_distribution_recorder().disable_this_region():
|
||||
hidden_states, residual = head.decoder(
|
||||
positions, hidden_states, forward_batch, residual, zero_allocator
|
||||
hidden_states = head.decoder(
|
||||
positions, hidden_states, forward_batch, zero_allocator
|
||||
)
|
||||
hidden_states, residual = head.decoder.layer_communicator.finish_layer_stack(
|
||||
hidden_states, residual, forward_batch
|
||||
)
|
||||
hidden_states = residual_batch.finish(hidden_states, forward_batch)
|
||||
|
||||
if not forward_batch.forward_mode.is_idle():
|
||||
if residual is None:
|
||||
hidden_states = head.shared_head.norm(hidden_states)
|
||||
else:
|
||||
hidden_states, _ = head.shared_head.norm(hidden_states, residual)
|
||||
hidden_states = residual_batch.norm(
|
||||
hidden_states, forward_batch, head.shared_head.norm
|
||||
)
|
||||
return hidden_states
|
||||
|
||||
def _embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor:
|
||||
|
||||
@@ -195,7 +195,7 @@ class FalconH1HybridAttentionDecoderLayer(nn.Module):
|
||||
rms_norm_eps=config.rms_norm_eps,
|
||||
activation=config.hidden_act,
|
||||
use_rms_norm=config.mamba_rms_norm,
|
||||
# Like the attention's o_proj: prepare_mlp completes the sum of the
|
||||
# Like the attention's o_proj: the FFN input completes the sum of the
|
||||
# two mixers' partial outputs.
|
||||
reduce_results=False,
|
||||
prefix=f"{prefix}.mixer",
|
||||
|
||||
@@ -24,6 +24,7 @@ from transformers import PretrainedConfig
|
||||
import sglang.srt.models.deepseek_v2 as deepseek_v2
|
||||
from sglang.srt.configs.gigachat35 import GigaChat35Config
|
||||
from sglang.srt.layers.communicator import get_attn_tp_context
|
||||
from sglang.srt.layers.communicator.output import OutputTransform
|
||||
from sglang.srt.layers.communicator.residual import batch as residual_batch
|
||||
from sglang.srt.layers.layernorm import GemmaRMSNorm, RMSNorm
|
||||
from sglang.srt.layers.linear import ColumnParallelLinear
|
||||
@@ -133,7 +134,7 @@ class GigaChat35PassthroughNorm(nn.Module):
|
||||
|
||||
Used as the ``input_layernorm`` slot when a layer has no pre-norm
|
||||
(``layernorm_type="post"``); it only folds the residual so the
|
||||
LayerCommunicator's prepare-step bookkeeping still works.
|
||||
attention boundary receives the folded residual.
|
||||
"""
|
||||
|
||||
def forward(
|
||||
@@ -153,10 +154,10 @@ class GigaChat35PassthroughNorm(nn.Module):
|
||||
class GigaChat35MlpPrepNorm(nn.Module):
|
||||
"""Pre-MLP norm wrapper that folds the post-attention (sandwich) norm.
|
||||
|
||||
Replaces the LayerCommunicator's ``post_attention_layernorm`` slot so the
|
||||
Serves as the FFN boundary's norm so the
|
||||
optional ``post_self_attn_layernorm`` (applied to the attention output,
|
||||
before the residual add) and the pre-MLP ``post_attention_layernorm`` are
|
||||
both threaded through ``prepare_mlp`` -- matching the per-layer math of the
|
||||
both run during FFN input preparation, matching the per-layer math of the
|
||||
``pre_post`` sandwich exactly.
|
||||
"""
|
||||
|
||||
@@ -389,7 +390,7 @@ class GigaChat35DecoderLayer(deepseek_v2.DeepseekV2DecoderLayer):
|
||||
)
|
||||
if hasattr(self.self_attn, "out_proj"):
|
||||
self.self_attn.out_proj.reduce_results = False
|
||||
self.layer_communicator.qkv_latent_func = None
|
||||
qkv_latent_func = None
|
||||
else:
|
||||
self.self_attn = GigaChat35AttentionMLA(
|
||||
config=config,
|
||||
@@ -409,7 +410,7 @@ class GigaChat35DecoderLayer(deepseek_v2.DeepseekV2DecoderLayer):
|
||||
prefix=add_prefix("self_attn", prefix),
|
||||
alt_stream=alt_stream,
|
||||
)
|
||||
self.layer_communicator.qkv_latent_func = self.self_attn.prepare_qkv_latent
|
||||
qkv_latent_func = self.self_attn.prepare_qkv_latent
|
||||
|
||||
self.is_sparse = self.is_layer_sparse
|
||||
|
||||
@@ -440,11 +441,17 @@ class GigaChat35DecoderLayer(deepseek_v2.DeepseekV2DecoderLayer):
|
||||
post_layernorm=self.post_self_attn_layernorm,
|
||||
)
|
||||
# The communicator chooses its fused steps from its norms when built.
|
||||
self.layer_communicator = self._build_layer_communicator(
|
||||
self.attn_stage, self.ffn_stage = self._build_stages(
|
||||
input_layernorm=attn_prepare_layernorm,
|
||||
post_attention_layernorm=mlp_prepare_layernorm,
|
||||
qkv_latent_func=self.layer_communicator.qkv_latent_func,
|
||||
allow_deferred_ffn_reduction=False,
|
||||
qkv_latent_func=qkv_latent_func,
|
||||
output=(
|
||||
OutputTransform(
|
||||
self.post_feedforward_layernorm, before_reduce_scatter=True
|
||||
)
|
||||
if self.post_feedforward_layernorm is not None
|
||||
else None
|
||||
),
|
||||
)
|
||||
|
||||
def _is_layer_sparse(self, layer_id: int, is_nextn: bool) -> bool:
|
||||
@@ -461,13 +468,10 @@ class GigaChat35DecoderLayer(deepseek_v2.DeepseekV2DecoderLayer):
|
||||
positions: torch.Tensor,
|
||||
hidden_states: torch.Tensor,
|
||||
forward_batch: ForwardBatch,
|
||||
residual: Optional[torch.Tensor],
|
||||
zero_allocator: BumpAllocator,
|
||||
**kwargs,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
hidden_states, residual = self.layer_communicator.prepare_attn(
|
||||
hidden_states, residual, forward_batch
|
||||
)
|
||||
) -> torch.Tensor:
|
||||
hidden_states = self.attn_stage.prepare(hidden_states, forward_batch)
|
||||
|
||||
if self.use_linear_attn:
|
||||
hidden_states = self.self_attn(
|
||||
@@ -481,26 +485,22 @@ class GigaChat35DecoderLayer(deepseek_v2.DeepseekV2DecoderLayer):
|
||||
forward_batch=forward_batch,
|
||||
zero_allocator=zero_allocator,
|
||||
input_on_attention_tp_slices=(
|
||||
self.layer_communicator.input_on_attention_tp_slices
|
||||
self.attn_stage.input_on_attention_tp_slices
|
||||
),
|
||||
)
|
||||
get_attn_tp_context().clear_attn_inputs()
|
||||
|
||||
hidden_states, residual = self.layer_communicator.prepare_mlp(
|
||||
hidden_states, residual, forward_batch
|
||||
)
|
||||
hidden_states = self.attn_stage.finish(hidden_states, forward_batch)
|
||||
hidden_states = self.ffn_stage.prepare(hidden_states, forward_batch)
|
||||
|
||||
# Unlike deepseek_v2, no moe_output_buffer_ctx here: non-inplace MoE
|
||||
# runners then allocate their output per forward instead of recycling
|
||||
# the layer-input buffer. The default (inplace) runners are unaffected.
|
||||
with self.layer_communicator.ffn_exit(forward_batch) as ffn_exit:
|
||||
with self.ffn_stage.exit(forward_batch) as ffn_exit:
|
||||
hidden_states = self.mlp(hidden_states, forward_batch)
|
||||
|
||||
if self.post_feedforward_layernorm is not None:
|
||||
hidden_states = self.post_feedforward_layernorm(hidden_states)
|
||||
|
||||
hidden_states, residual = ffn_exit.finish(hidden_states, residual)
|
||||
return hidden_states, residual
|
||||
hidden_states = ffn_exit.finish(hidden_states)
|
||||
return hidden_states
|
||||
|
||||
|
||||
class GigaChat35Model(nn.Module):
|
||||
@@ -559,12 +559,12 @@ class GigaChat35Model(nn.Module):
|
||||
hidden_states = inputs_embeds
|
||||
else:
|
||||
hidden_states = self.embed_tokens(input_ids)
|
||||
residual = residual_batch.start(forward_batch)
|
||||
residual_batch.start(forward_batch)
|
||||
else:
|
||||
assert pp_proxy_tensors is not None
|
||||
hidden_states, residual = self.layers[
|
||||
self.start_layer
|
||||
].layer_communicator.from_pp(pp_proxy_tensors, forward_batch)
|
||||
hidden_states = self.layers[self.start_layer].attn_stage.from_pp(
|
||||
pp_proxy_tensors, forward_batch
|
||||
)
|
||||
|
||||
total_num_layers = self.end_layer - self.start_layer
|
||||
zero_allocator = BumpAllocator(
|
||||
@@ -574,33 +574,19 @@ class GigaChat35Model(nn.Module):
|
||||
)
|
||||
|
||||
for i in range(self.start_layer, self.end_layer):
|
||||
hidden_states, residual = self.layers[i](
|
||||
hidden_states = self.layers[i](
|
||||
positions=positions,
|
||||
hidden_states=hidden_states,
|
||||
forward_batch=forward_batch,
|
||||
residual=residual,
|
||||
zero_allocator=zero_allocator,
|
||||
)
|
||||
|
||||
hidden_states, residual = self.layers[
|
||||
self.end_layer - 1
|
||||
].layer_communicator.finish_layer_stack(
|
||||
hidden_states,
|
||||
residual,
|
||||
forward_batch,
|
||||
preserve_declared=not self.pp_group.is_last_rank,
|
||||
)
|
||||
|
||||
if not self.pp_group.is_last_rank:
|
||||
return PPProxyTensors(
|
||||
{"hidden_states": hidden_states, "residual": residual}
|
||||
)
|
||||
return residual_batch.to_pp(hidden_states, forward_batch)
|
||||
hidden_states = residual_batch.finish(hidden_states, forward_batch)
|
||||
|
||||
if hidden_states.shape[0] != 0:
|
||||
if residual is None:
|
||||
hidden_states = self.norm(hidden_states)
|
||||
else:
|
||||
hidden_states, _ = self.norm(hidden_states, residual)
|
||||
hidden_states = residual_batch.norm(hidden_states, forward_batch, self.norm)
|
||||
return hidden_states
|
||||
|
||||
|
||||
|
||||
@@ -82,7 +82,7 @@ class GigaChat35ModelNextN(nn.Module):
|
||||
positions: torch.Tensor,
|
||||
forward_batch: ForwardBatch,
|
||||
input_embeds: Optional[torch.Tensor] = None,
|
||||
) -> tuple[torch.Tensor, Optional[torch.Tensor]]:
|
||||
) -> torch.Tensor:
|
||||
zero_allocator = BumpAllocator(
|
||||
buffer_size=2,
|
||||
dtype=torch.float32,
|
||||
@@ -107,28 +107,24 @@ class GigaChat35ModelNextN(nn.Module):
|
||||
)
|
||||
)
|
||||
|
||||
residual = residual_batch.start(forward_batch)
|
||||
hidden_states, residual = self.decoder(
|
||||
residual_batch.start(forward_batch)
|
||||
hidden_states = self.decoder(
|
||||
positions=positions,
|
||||
hidden_states=hidden_states,
|
||||
forward_batch=forward_batch,
|
||||
residual=residual,
|
||||
zero_allocator=zero_allocator,
|
||||
)
|
||||
|
||||
hidden_states, residual = self.decoder.layer_communicator.finish_layer_stack(
|
||||
hidden_states, residual, forward_batch
|
||||
)
|
||||
hidden_states = residual_batch.finish(hidden_states, forward_batch)
|
||||
|
||||
hidden_states_before_norm = None
|
||||
if not forward_batch.forward_mode.is_idle():
|
||||
hidden_states_before_norm = self.decoder.layer_communicator.snapshot(
|
||||
hidden_states, residual
|
||||
hidden_states_before_norm = residual_batch.snapshot(
|
||||
hidden_states, forward_batch
|
||||
)
|
||||
hidden_states = residual_batch.norm(
|
||||
hidden_states, forward_batch, self.shared_head.norm
|
||||
)
|
||||
if residual is not None:
|
||||
hidden_states, _ = self.shared_head.norm(hidden_states, residual)
|
||||
else:
|
||||
hidden_states = self.shared_head.norm(hidden_states)
|
||||
|
||||
return hidden_states, hidden_states_before_norm
|
||||
|
||||
|
||||
@@ -26,7 +26,7 @@ from transformers import PretrainedConfig
|
||||
|
||||
from sglang.kernels.ops.quantization.fp8_kernel import is_fp8_fnuz
|
||||
from sglang.srt.batch_overlap.single_batch_overlap import SboFlags
|
||||
from sglang.srt.batch_overlap.two_batch_overlap import model_forward_maybe_tbo
|
||||
from sglang.srt.batch_overlap.two_batch_overlap import model_forward_stages
|
||||
from sglang.srt.distributed import (
|
||||
get_pp_indices,
|
||||
)
|
||||
@@ -40,9 +40,10 @@ from sglang.srt.eplb.expert_location_dispatch import ExpertLocationDispatchInfo
|
||||
from sglang.srt.layers.activation import SiluAndMul
|
||||
from sglang.srt.layers.aux_hidden_states import AuxHiddenStateList
|
||||
from sglang.srt.layers.communicator import (
|
||||
LayerCommunicator,
|
||||
LayerFacts,
|
||||
declare_attn,
|
||||
declare_ffn,
|
||||
enable_moe_dense_fully_dp,
|
||||
make_stages,
|
||||
)
|
||||
from sglang.srt.layers.communicator.residual import batch as residual_batch
|
||||
from sglang.srt.layers.dp_attention import (
|
||||
@@ -837,14 +838,6 @@ class Glm4MoeDecoderLayer(nn.Module):
|
||||
is_previous_layer_sparse = self._is_layer_sparse(layer_id - 1, is_nextn=False)
|
||||
is_next_layer_sparse = self._is_layer_sparse(layer_id + 1, is_nextn=False)
|
||||
|
||||
self.layer_facts = LayerFacts.init_new(
|
||||
layer_id=layer_id,
|
||||
num_layers=1 if is_nextn else config.num_hidden_layers,
|
||||
is_layer_sparse=self.is_layer_sparse,
|
||||
is_previous_layer_sparse=is_previous_layer_sparse,
|
||||
is_next_layer_sparse=is_next_layer_sparse,
|
||||
)
|
||||
|
||||
if self.is_layer_sparse:
|
||||
self.mlp = Glm4MoeSparseMoeBlock(
|
||||
config=config,
|
||||
@@ -873,11 +866,21 @@ class Glm4MoeDecoderLayer(nn.Module):
|
||||
config.hidden_size, eps=config.rms_norm_eps
|
||||
)
|
||||
|
||||
self.layer_communicator = LayerCommunicator(
|
||||
layer_facts=self.layer_facts,
|
||||
input_layernorm=self.input_layernorm,
|
||||
post_attention_layernorm=self.post_attention_layernorm,
|
||||
allow_reduce_scatter=True,
|
||||
self.attn_stage, self.ffn_stage = make_stages(
|
||||
(declare_attn(), self.input_layernorm),
|
||||
(
|
||||
declare_ffn(
|
||||
sparse=self.is_layer_sparse,
|
||||
next_sparse=is_next_layer_sparse,
|
||||
),
|
||||
self.post_attention_layernorm,
|
||||
),
|
||||
previous=declare_ffn(
|
||||
sparse=is_previous_layer_sparse, next_sparse=self.is_layer_sparse
|
||||
)
|
||||
if layer_id != 0
|
||||
else None,
|
||||
terminal=layer_id == (1 if is_nextn else config.num_hidden_layers) - 1,
|
||||
)
|
||||
|
||||
# Detect if QKV uses aiter FP8 per-token quant so we can fuse
|
||||
@@ -936,18 +939,14 @@ class Glm4MoeDecoderLayer(nn.Module):
|
||||
positions: torch.Tensor,
|
||||
hidden_states: torch.Tensor,
|
||||
forward_batch: ForwardBatch,
|
||||
residual: Optional[torch.Tensor],
|
||||
capture_output=None,
|
||||
) -> torch.Tensor:
|
||||
|
||||
hidden_states, residual = (
|
||||
self.layer_communicator.prepare_attn_and_capture_last_layer_outputs(
|
||||
hidden_states,
|
||||
residual,
|
||||
forward_batch,
|
||||
quant_format=self.attn_quant_format,
|
||||
capture_output=capture_output,
|
||||
)
|
||||
hidden_states = self.attn_stage.prepare(
|
||||
hidden_states,
|
||||
forward_batch,
|
||||
quant_format=self.attn_quant_format,
|
||||
capture_output=capture_output,
|
||||
)
|
||||
|
||||
hidden_states = self.self_attn(
|
||||
@@ -956,15 +955,14 @@ class Glm4MoeDecoderLayer(nn.Module):
|
||||
forward_batch=forward_batch,
|
||||
)
|
||||
|
||||
hidden_states, residual = self.layer_communicator.prepare_mlp(
|
||||
hidden_states, residual, forward_batch
|
||||
)
|
||||
hidden_states = self.attn_stage.finish(hidden_states, forward_batch)
|
||||
hidden_states = self.ffn_stage.prepare(hidden_states, forward_batch)
|
||||
|
||||
with self.layer_communicator.ffn_exit(forward_batch) as ffn_exit:
|
||||
with self.ffn_stage.exit(forward_batch) as ffn_exit:
|
||||
hidden_states = self.mlp(hidden_states, forward_batch)
|
||||
hidden_states, residual = ffn_exit.finish(hidden_states, residual)
|
||||
hidden_states = ffn_exit.finish(hidden_states)
|
||||
|
||||
return hidden_states, residual
|
||||
return hidden_states
|
||||
|
||||
def op_comm_prepare_attn(
|
||||
self,
|
||||
@@ -972,16 +970,10 @@ class Glm4MoeDecoderLayer(nn.Module):
|
||||
positions: torch.Tensor,
|
||||
hidden_states: torch.Tensor,
|
||||
forward_batch: ForwardBatch,
|
||||
residual: Optional[torch.Tensor],
|
||||
tbo_subbatch_index: Optional[int] = None,
|
||||
):
|
||||
state.hidden_states_after_comm_pre_attn, state.residual_after_input_ln = (
|
||||
self.layer_communicator.prepare_attn(
|
||||
hidden_states,
|
||||
residual,
|
||||
forward_batch,
|
||||
quant_format=self.attn_quant_format,
|
||||
)
|
||||
state.hidden_states_after_comm_pre_attn = self.attn_stage.prepare(
|
||||
hidden_states, forward_batch
|
||||
)
|
||||
state.update(
|
||||
dict(
|
||||
@@ -992,25 +984,21 @@ class Glm4MoeDecoderLayer(nn.Module):
|
||||
)
|
||||
|
||||
def op_comm_prepare_mlp(self, state):
|
||||
state.hidden_states_mlp_input, state.residual_after_comm_pre_mlp = (
|
||||
self.layer_communicator.prepare_mlp(
|
||||
state.pop("hidden_states_after_attn"),
|
||||
state.pop("residual_after_input_ln"),
|
||||
state.forward_batch,
|
||||
)
|
||||
hidden_states = self.attn_stage.finish(
|
||||
state.pop("hidden_states_after_attn"), state.forward_batch
|
||||
)
|
||||
state.hidden_states_mlp_input = self.ffn_stage.prepare(
|
||||
hidden_states, state.forward_batch
|
||||
)
|
||||
|
||||
def op_comm_postprocess_layer(self, state):
|
||||
hidden_states, residual = self.layer_communicator.postprocess_layer(
|
||||
state.pop("hidden_states_mlp_output"),
|
||||
state.pop("residual_after_comm_pre_mlp"),
|
||||
state.forward_batch,
|
||||
hidden_states = self.ffn_stage.postprocess(
|
||||
state.pop("hidden_states_mlp_output"), state.forward_batch
|
||||
)
|
||||
|
||||
output = dict(
|
||||
positions=state.positions,
|
||||
hidden_states=hidden_states,
|
||||
residual=residual,
|
||||
forward_batch=state.forward_batch,
|
||||
tbo_subbatch_index=state.tbo_subbatch_index,
|
||||
)
|
||||
@@ -1090,12 +1078,12 @@ class Glm4MoeModel(nn.Module):
|
||||
hidden_states = self.embed_tokens(input_ids)
|
||||
else:
|
||||
hidden_states = input_embeds
|
||||
residual = residual_batch.start(forward_batch)
|
||||
residual_batch.start(forward_batch)
|
||||
else:
|
||||
assert pp_proxy_tensors is not None
|
||||
hidden_states, residual = self.layers[
|
||||
self.start_layer
|
||||
].layer_communicator.from_pp(pp_proxy_tensors, forward_batch)
|
||||
hidden_states = self.layers[self.start_layer].attn_stage.from_pp(
|
||||
pp_proxy_tensors, forward_batch
|
||||
)
|
||||
|
||||
normal_start_layer = self.start_layer
|
||||
normal_end_layer = self.end_layer
|
||||
@@ -1112,46 +1100,32 @@ class Glm4MoeModel(nn.Module):
|
||||
for i in range(normal_start_layer, normal_end_layer):
|
||||
with get_global_expert_distribution_recorder().with_current_layer(i):
|
||||
layer = self.layers[i]
|
||||
hidden_states, residual = layer(
|
||||
hidden_states = layer(
|
||||
positions,
|
||||
hidden_states,
|
||||
forward_batch,
|
||||
residual,
|
||||
capture_output=aux_hidden_states.capture
|
||||
if i in self.layers_to_capture
|
||||
else None,
|
||||
)
|
||||
|
||||
if normal_end_layer != self.end_layer:
|
||||
hidden_states, residual = model_forward_maybe_tbo(
|
||||
hidden_states = model_forward_stages(
|
||||
layers=self.layers[normal_end_layer : self.end_layer],
|
||||
enable_tbo=True,
|
||||
positions=positions,
|
||||
forward_batch=forward_batch,
|
||||
hidden_states=hidden_states,
|
||||
residual=residual,
|
||||
)
|
||||
|
||||
last_layer = self.layers[self.end_layer - 1]
|
||||
hidden_states, residual = last_layer.layer_communicator.finish_layer_stack(
|
||||
hidden_states,
|
||||
residual,
|
||||
forward_batch,
|
||||
preserve_declared=not self.pp_group.is_last_rank,
|
||||
)
|
||||
if not self.pp_group.is_last_rank:
|
||||
return PPProxyTensors(
|
||||
{
|
||||
"hidden_states": hidden_states,
|
||||
"residual": residual,
|
||||
}
|
||||
)
|
||||
return residual_batch.to_pp(hidden_states, forward_batch)
|
||||
else:
|
||||
hidden_states = residual_batch.finish(hidden_states, forward_batch)
|
||||
if not forward_batch.forward_mode.is_idle():
|
||||
if residual is None:
|
||||
hidden_states = self.norm(hidden_states)
|
||||
else:
|
||||
hidden_states, _ = self.norm(hidden_states, residual)
|
||||
hidden_states = residual_batch.norm(
|
||||
hidden_states, forward_batch, self.norm
|
||||
)
|
||||
if len(aux_hidden_states) == 0:
|
||||
return hidden_states
|
||||
return hidden_states, aux_hidden_states
|
||||
|
||||
@@ -24,7 +24,7 @@ from torch import nn
|
||||
from transformers import PretrainedConfig
|
||||
|
||||
from sglang.srt.batch_overlap.single_batch_overlap import SboFlags
|
||||
from sglang.srt.batch_overlap.two_batch_overlap import model_forward_maybe_tbo
|
||||
from sglang.srt.batch_overlap.two_batch_overlap import model_forward_stages
|
||||
from sglang.srt.distributed.device_communicators.pynccl_allocator import (
|
||||
use_symmetric_memory,
|
||||
)
|
||||
@@ -34,10 +34,11 @@ from sglang.srt.eplb.expert_location_dispatch import ExpertLocationDispatchInfo
|
||||
from sglang.srt.layers.activation import SiluAndMul
|
||||
from sglang.srt.layers.aux_hidden_states import AuxHiddenStateList
|
||||
from sglang.srt.layers.communicator import (
|
||||
LayerCommunicator,
|
||||
LayerFacts,
|
||||
declare_attn,
|
||||
declare_ffn,
|
||||
enable_moe_dense_fully_dp,
|
||||
get_attn_tp_context,
|
||||
make_stages,
|
||||
)
|
||||
from sglang.srt.layers.communicator.residual import batch as residual_batch
|
||||
from sglang.srt.layers.dp_attention import (
|
||||
@@ -549,14 +550,6 @@ class Glm4MoeLiteDecoderLayer(nn.Module):
|
||||
is_previous_layer_sparse = self._is_layer_sparse(layer_id - 1, is_nextn=False)
|
||||
is_next_layer_sparse = self._is_layer_sparse(layer_id + 1, is_nextn=False)
|
||||
|
||||
self.layer_facts = LayerFacts.init_new(
|
||||
layer_id=layer_id,
|
||||
num_layers=1 if is_nextn else config.num_hidden_layers,
|
||||
is_layer_sparse=self.is_layer_sparse,
|
||||
is_previous_layer_sparse=is_previous_layer_sparse,
|
||||
is_next_layer_sparse=is_next_layer_sparse,
|
||||
)
|
||||
|
||||
if self.is_layer_sparse:
|
||||
self.mlp = Glm4MoeLiteSparseMoeBlock(
|
||||
config=config,
|
||||
@@ -588,12 +581,25 @@ class Glm4MoeLiteDecoderLayer(nn.Module):
|
||||
|
||||
self._gfx95_quant_format = self._detect_gfx95_quant_format()
|
||||
|
||||
self.layer_communicator = LayerCommunicator(
|
||||
layer_facts=self.layer_facts,
|
||||
input_layernorm=self.input_layernorm,
|
||||
post_attention_layernorm=self.post_attention_layernorm,
|
||||
allow_reduce_scatter=True,
|
||||
qkv_latent_func=self.self_attn.prepare_qkv_latent,
|
||||
self.attn_stage, self.ffn_stage = make_stages(
|
||||
(
|
||||
declare_attn(),
|
||||
self.input_layernorm,
|
||||
{"qkv_latent_func": self.self_attn.prepare_qkv_latent},
|
||||
),
|
||||
(
|
||||
declare_ffn(
|
||||
sparse=self.is_layer_sparse,
|
||||
next_sparse=is_next_layer_sparse,
|
||||
),
|
||||
self.post_attention_layernorm,
|
||||
),
|
||||
previous=declare_ffn(
|
||||
sparse=is_previous_layer_sparse, next_sparse=self.is_layer_sparse
|
||||
)
|
||||
if layer_id != 0
|
||||
else None,
|
||||
terminal=layer_id == (1 if is_nextn else config.num_hidden_layers) - 1,
|
||||
)
|
||||
|
||||
def _detect_gfx95_quant_format(self) -> str:
|
||||
@@ -624,18 +630,14 @@ class Glm4MoeLiteDecoderLayer(nn.Module):
|
||||
positions: torch.Tensor,
|
||||
hidden_states: torch.Tensor,
|
||||
forward_batch: ForwardBatch,
|
||||
residual: Optional[torch.Tensor],
|
||||
zero_allocator: BumpAllocator,
|
||||
capture_output=None,
|
||||
) -> torch.Tensor:
|
||||
hidden_states, residual = (
|
||||
self.layer_communicator.prepare_attn_and_capture_last_layer_outputs(
|
||||
hidden_states,
|
||||
residual,
|
||||
forward_batch,
|
||||
quant_format=getattr(self, "_gfx95_quant_format", ""),
|
||||
capture_output=capture_output,
|
||||
)
|
||||
hidden_states = self.attn_stage.prepare(
|
||||
hidden_states,
|
||||
forward_batch,
|
||||
quant_format=getattr(self, "_gfx95_quant_format", ""),
|
||||
capture_output=capture_output,
|
||||
)
|
||||
|
||||
hidden_states = self.self_attn(
|
||||
@@ -643,23 +645,20 @@ class Glm4MoeLiteDecoderLayer(nn.Module):
|
||||
hidden_states=hidden_states,
|
||||
forward_batch=forward_batch,
|
||||
zero_allocator=zero_allocator,
|
||||
input_on_attention_tp_slices=(
|
||||
self.layer_communicator.input_on_attention_tp_slices
|
||||
),
|
||||
input_on_attention_tp_slices=(self.attn_stage.input_on_attention_tp_slices),
|
||||
)
|
||||
if isinstance(hidden_states, tuple):
|
||||
hidden_states = hidden_states[0]
|
||||
get_attn_tp_context().clear_attn_inputs()
|
||||
|
||||
hidden_states, residual = self.layer_communicator.prepare_mlp(
|
||||
hidden_states, residual, forward_batch
|
||||
)
|
||||
hidden_states = self.attn_stage.finish(hidden_states, forward_batch)
|
||||
hidden_states = self.ffn_stage.prepare(hidden_states, forward_batch)
|
||||
|
||||
with self.layer_communicator.ffn_exit(forward_batch) as ffn_exit:
|
||||
with self.ffn_stage.exit(forward_batch) as ffn_exit:
|
||||
hidden_states = self.mlp(hidden_states, forward_batch)
|
||||
hidden_states, residual = ffn_exit.finish(hidden_states, residual)
|
||||
hidden_states = ffn_exit.finish(hidden_states)
|
||||
|
||||
return hidden_states, residual
|
||||
return hidden_states
|
||||
|
||||
def op_comm_prepare_attn(
|
||||
self,
|
||||
@@ -667,12 +666,11 @@ class Glm4MoeLiteDecoderLayer(nn.Module):
|
||||
positions: torch.Tensor,
|
||||
hidden_states: torch.Tensor,
|
||||
forward_batch: ForwardBatch,
|
||||
residual: Optional[torch.Tensor],
|
||||
zero_allocator: BumpAllocator,
|
||||
tbo_subbatch_index: Optional[int] = None,
|
||||
):
|
||||
state.hidden_states_after_comm_pre_attn, state.residual_after_input_ln = (
|
||||
self.layer_communicator.prepare_attn(hidden_states, residual, forward_batch)
|
||||
state.hidden_states_after_comm_pre_attn = self.attn_stage.prepare(
|
||||
hidden_states, forward_batch
|
||||
)
|
||||
if get_moe_a2a_backend().is_mori():
|
||||
state.num_tokens = hidden_states.shape[0]
|
||||
@@ -686,25 +684,21 @@ class Glm4MoeLiteDecoderLayer(nn.Module):
|
||||
)
|
||||
|
||||
def op_comm_prepare_mlp(self, state):
|
||||
state.hidden_states_mlp_input, state.residual_after_comm_pre_mlp = (
|
||||
self.layer_communicator.prepare_mlp(
|
||||
state.pop("hidden_states_after_attn"),
|
||||
state.pop("residual_after_input_ln"),
|
||||
state.forward_batch,
|
||||
)
|
||||
hidden_states = self.attn_stage.finish(
|
||||
state.pop("hidden_states_after_attn"), state.forward_batch
|
||||
)
|
||||
state.hidden_states_mlp_input = self.ffn_stage.prepare(
|
||||
hidden_states, state.forward_batch
|
||||
)
|
||||
|
||||
def op_comm_postprocess_layer(self, state):
|
||||
hidden_states, residual = self.layer_communicator.postprocess_layer(
|
||||
state.pop("hidden_states_mlp_output"),
|
||||
state.pop("residual_after_comm_pre_mlp"),
|
||||
state.forward_batch,
|
||||
hidden_states = self.ffn_stage.postprocess(
|
||||
state.pop("hidden_states_mlp_output"), state.forward_batch
|
||||
)
|
||||
|
||||
output = dict(
|
||||
positions=state.positions,
|
||||
hidden_states=hidden_states,
|
||||
residual=residual,
|
||||
forward_batch=state.forward_batch,
|
||||
zero_allocator=state.zero_allocator,
|
||||
tbo_subbatch_index=state.tbo_subbatch_index,
|
||||
@@ -782,12 +776,12 @@ class Glm4MoeLiteModel(nn.Module):
|
||||
hidden_states = self.embed_tokens(input_ids)
|
||||
else:
|
||||
hidden_states = input_embeds
|
||||
residual = residual_batch.start(forward_batch)
|
||||
residual_batch.start(forward_batch)
|
||||
else:
|
||||
assert pp_proxy_tensors is not None
|
||||
hidden_states, residual = self.layers[
|
||||
self.start_layer
|
||||
].layer_communicator.from_pp(pp_proxy_tensors, forward_batch)
|
||||
hidden_states = self.layers[self.start_layer].attn_stage.from_pp(
|
||||
pp_proxy_tensors, forward_batch
|
||||
)
|
||||
device = hidden_states.device
|
||||
zero_allocator = BumpAllocator(
|
||||
buffer_size=total_num_layers * 2 * (2 if forward_batch.can_run_tbo else 1),
|
||||
@@ -809,11 +803,10 @@ class Glm4MoeLiteModel(nn.Module):
|
||||
for i in range(normal_start_layer, normal_end_layer):
|
||||
with get_global_expert_distribution_recorder().with_current_layer(i):
|
||||
layer = self.layers[i]
|
||||
hidden_states, residual = layer(
|
||||
hidden_states = layer(
|
||||
positions,
|
||||
hidden_states,
|
||||
forward_batch,
|
||||
residual,
|
||||
zero_allocator,
|
||||
capture_output=aux_hidden_states.capture
|
||||
if i in self.layers_to_capture
|
||||
@@ -821,36 +814,23 @@ class Glm4MoeLiteModel(nn.Module):
|
||||
)
|
||||
|
||||
if normal_end_layer != self.end_layer:
|
||||
hidden_states, residual = model_forward_maybe_tbo(
|
||||
hidden_states = model_forward_stages(
|
||||
layers=self.layers[normal_end_layer : self.end_layer],
|
||||
enable_tbo=True,
|
||||
positions=positions,
|
||||
forward_batch=forward_batch,
|
||||
hidden_states=hidden_states,
|
||||
residual=residual,
|
||||
zero_allocator=zero_allocator,
|
||||
)
|
||||
|
||||
last_layer = self.layers[self.end_layer - 1]
|
||||
hidden_states, residual = last_layer.layer_communicator.finish_layer_stack(
|
||||
hidden_states,
|
||||
residual,
|
||||
forward_batch,
|
||||
preserve_declared=not self.pp_group.is_last_rank,
|
||||
)
|
||||
if not self.pp_group.is_last_rank:
|
||||
return PPProxyTensors(
|
||||
{
|
||||
"hidden_states": hidden_states,
|
||||
"residual": residual,
|
||||
}
|
||||
)
|
||||
return residual_batch.to_pp(hidden_states, forward_batch)
|
||||
else:
|
||||
hidden_states = residual_batch.finish(hidden_states, forward_batch)
|
||||
if not forward_batch.forward_mode.is_idle():
|
||||
if residual is None:
|
||||
hidden_states = self.norm(hidden_states)
|
||||
else:
|
||||
hidden_states, _ = self.norm(hidden_states, residual)
|
||||
hidden_states = residual_batch.norm(
|
||||
hidden_states, forward_batch, self.norm
|
||||
)
|
||||
|
||||
if len(aux_hidden_states) == 0:
|
||||
return hidden_states
|
||||
|
||||
@@ -115,20 +115,17 @@ class Glm4MoeLiteModelNextN(nn.Module):
|
||||
)
|
||||
)
|
||||
|
||||
residual = residual_batch.start(forward_batch)
|
||||
residual_batch.start(forward_batch)
|
||||
with get_global_expert_distribution_recorder().disable_this_region():
|
||||
hidden_states, residual = self.decoder(
|
||||
positions, hidden_states, forward_batch, residual, zero_allocator
|
||||
hidden_states = self.decoder(
|
||||
positions, hidden_states, forward_batch, zero_allocator
|
||||
)
|
||||
|
||||
hidden_states, residual = self.decoder.layer_communicator.finish_layer_stack(
|
||||
hidden_states, residual, forward_batch
|
||||
)
|
||||
hidden_states = residual_batch.finish(hidden_states, forward_batch)
|
||||
if not forward_batch.forward_mode.is_idle():
|
||||
if residual is not None:
|
||||
hidden_states, _ = self.shared_head.norm(hidden_states, residual)
|
||||
else:
|
||||
hidden_states = self.shared_head.norm(hidden_states)
|
||||
hidden_states = residual_batch.norm(
|
||||
hidden_states, forward_batch, self.shared_head.norm
|
||||
)
|
||||
|
||||
return hidden_states
|
||||
|
||||
|
||||
@@ -101,20 +101,15 @@ class Glm4MoeModelNextN(nn.Module):
|
||||
)
|
||||
)
|
||||
|
||||
residual = residual_batch.start(forward_batch)
|
||||
residual_batch.start(forward_batch)
|
||||
with get_global_expert_distribution_recorder().disable_this_region():
|
||||
hidden_states, residual = self.decoder(
|
||||
positions, hidden_states, forward_batch, residual
|
||||
)
|
||||
hidden_states = self.decoder(positions, hidden_states, forward_batch)
|
||||
|
||||
hidden_states, residual = self.decoder.layer_communicator.finish_layer_stack(
|
||||
hidden_states, residual, forward_batch
|
||||
)
|
||||
hidden_states = residual_batch.finish(hidden_states, forward_batch)
|
||||
if not forward_batch.forward_mode.is_idle():
|
||||
if residual is not None:
|
||||
hidden_states, _ = self.shared_head.norm(hidden_states, residual)
|
||||
else:
|
||||
hidden_states = self.shared_head.norm(hidden_states)
|
||||
hidden_states = residual_batch.norm(
|
||||
hidden_states, forward_batch, self.shared_head.norm
|
||||
)
|
||||
|
||||
return hidden_states
|
||||
|
||||
|
||||
@@ -2,7 +2,7 @@ import logging
|
||||
from array import array
|
||||
from contextlib import nullcontext
|
||||
from functools import partial
|
||||
from typing import Any, Dict, Iterable, List, Optional, Tuple, Union
|
||||
from typing import Iterable, List, Optional, Tuple, Union
|
||||
|
||||
import torch
|
||||
from torch import nn
|
||||
@@ -13,7 +13,7 @@ from sglang.kernels.ops.layernorm.mhc import hc_contract
|
||||
from sglang.kernels.ops.layernorm.mhc import hc_post as _hc_post_fn
|
||||
from sglang.kernels.ops.layernorm.mhc import hc_pre as _hc_pre_fn
|
||||
from sglang.srt.batch_overlap.two_batch_overlap import (
|
||||
model_forward_maybe_tbo,
|
||||
model_forward_stages,
|
||||
)
|
||||
from sglang.srt.configs.glm5_next import Glm5NextConfig, Glm5NextTextConfig
|
||||
from sglang.srt.configs.model_config import is_deepseek_dsa
|
||||
@@ -27,12 +27,15 @@ from sglang.srt.layers.attention import vision_utils
|
||||
from sglang.srt.layers.attention.vision import VisionAttention
|
||||
from sglang.srt.layers.aux_hidden_states import AuxHiddenStateList
|
||||
from sglang.srt.layers.communicator import (
|
||||
LayerCommunicator,
|
||||
LayerFacts,
|
||||
MHCLayerCommunicator,
|
||||
PLAIN_RESIDUAL,
|
||||
MHCState,
|
||||
declare_attn,
|
||||
declare_ffn,
|
||||
enable_moe_dense_fully_dp,
|
||||
get_attn_tp_context,
|
||||
make_stages,
|
||||
)
|
||||
from sglang.srt.layers.communicator.residual import access as residual_access
|
||||
from sglang.srt.layers.communicator.residual import batch as residual_batch
|
||||
from sglang.srt.layers.layernorm import RMSNorm
|
||||
from sglang.srt.layers.linear import (
|
||||
@@ -727,14 +730,6 @@ class Glm5NextDecoderLayer(nn.Module):
|
||||
is_previous_layer_sparse = self._is_layer_sparse(layer_id - 1, is_nextn=False)
|
||||
is_next_layer_sparse = self._is_layer_sparse(layer_id + 1, is_nextn=False)
|
||||
|
||||
self.layer_facts = LayerFacts.init_new(
|
||||
layer_id=layer_id,
|
||||
num_layers=1 if is_nextn else config.num_hidden_layers,
|
||||
is_layer_sparse=self.is_layer_sparse,
|
||||
is_previous_layer_sparse=is_previous_layer_sparse,
|
||||
is_next_layer_sparse=is_next_layer_sparse,
|
||||
)
|
||||
|
||||
if self.is_layer_sparse:
|
||||
self.mlp = Glm5NextMoE(
|
||||
config=config,
|
||||
@@ -784,36 +779,52 @@ class Glm5NextDecoderLayer(nn.Module):
|
||||
torch.empty(mix_hc, hc_dim, dtype=torch.float32)
|
||||
)
|
||||
|
||||
shared_kwargs: Dict[str, Any] = dict(
|
||||
layer_facts=self.layer_facts,
|
||||
input_layernorm=self.input_layernorm,
|
||||
post_attention_layernorm=self.post_attention_layernorm,
|
||||
allow_reduce_scatter=True,
|
||||
qkv_latent_func=(
|
||||
self.self_attn.prepare_qkv_latent if not self.is_linear_attn else None
|
||||
),
|
||||
)
|
||||
|
||||
terminal = layer_id == (1 if is_nextn else config.num_hidden_layers) - 1
|
||||
residual = PLAIN_RESIDUAL
|
||||
if self.config.mhc:
|
||||
mhc_kwargs: Dict[str, Any] = dict(
|
||||
residual = MHCState(
|
||||
hc_mult=config.hc_mult,
|
||||
hc_attn_pre=self.hc_attn_pre,
|
||||
hc_ffn_pre=self.hc_ffn_pre,
|
||||
hc_post=self.hc_post,
|
||||
# Resolved once: env and platform are frozen after startup,
|
||||
# and None keeps the dispatch off the per-boundary path.
|
||||
hc_ffn_post_pre=(
|
||||
self.hc_ffn_post_pre
|
||||
if is_cross_layer_mhc_fusion_enabled()
|
||||
else None
|
||||
),
|
||||
is_last_layer=terminal,
|
||||
).layer_residual()
|
||||
self.attn_stage, self.ffn_stage = make_stages(
|
||||
(
|
||||
declare_attn(
|
||||
read=residual.attention_read, update=residual.attention_update
|
||||
),
|
||||
self.input_layernorm,
|
||||
{
|
||||
"residual_in_hidden": residual.ffn_update.at_producer,
|
||||
"qkv_latent_func": self.self_attn.prepare_qkv_latent
|
||||
if not self.is_linear_attn
|
||||
else None,
|
||||
},
|
||||
),
|
||||
(
|
||||
declare_ffn(
|
||||
sparse=self.is_layer_sparse,
|
||||
next_sparse=is_next_layer_sparse,
|
||||
read=residual.ffn_read,
|
||||
update=residual.ffn_update,
|
||||
),
|
||||
self.post_attention_layernorm,
|
||||
),
|
||||
previous=declare_ffn(
|
||||
sparse=is_previous_layer_sparse,
|
||||
next_sparse=self.is_layer_sparse,
|
||||
update=residual.ffn_update,
|
||||
)
|
||||
self.layer_communicator = MHCLayerCommunicator(
|
||||
**shared_kwargs,
|
||||
**mhc_kwargs,
|
||||
)
|
||||
else:
|
||||
self.layer_communicator = LayerCommunicator(**shared_kwargs)
|
||||
if layer_id != 0
|
||||
else None,
|
||||
terminal=terminal,
|
||||
)
|
||||
|
||||
def _hc_pre(
|
||||
self, hc_fn, hc_scale, hc_base, hidden_states, out_norm_weight, out_norm_eps
|
||||
@@ -915,21 +926,15 @@ class Glm5NextDecoderLayer(nn.Module):
|
||||
positions: torch.Tensor,
|
||||
hidden_states: torch.Tensor,
|
||||
forward_batch: ForwardBatch,
|
||||
residual: Optional[torch.Tensor],
|
||||
zero_allocator: Optional[BumpAllocator] = None,
|
||||
gemm_output_zero_allocator: BumpAllocator = None,
|
||||
prev_topk_indices: Optional[torch.Tensor] = None,
|
||||
capture_output=None,
|
||||
):
|
||||
hidden_states_orig = self.layer_communicator.buffer(hidden_states)
|
||||
hidden_states_orig = residual_access.buffer(hidden_states)
|
||||
|
||||
hidden_states, residual = (
|
||||
self.layer_communicator.prepare_attn_and_capture_last_layer_outputs(
|
||||
hidden_states,
|
||||
residual,
|
||||
forward_batch,
|
||||
capture_output=capture_output,
|
||||
)
|
||||
hidden_states = self.attn_stage.prepare(
|
||||
hidden_states, forward_batch, capture_output=capture_output
|
||||
)
|
||||
|
||||
hidden_states = self.self_attn(
|
||||
@@ -937,9 +942,7 @@ class Glm5NextDecoderLayer(nn.Module):
|
||||
hidden_states=hidden_states,
|
||||
forward_batch=forward_batch,
|
||||
zero_allocator=zero_allocator,
|
||||
input_on_attention_tp_slices=(
|
||||
self.layer_communicator.input_on_attention_tp_slices
|
||||
),
|
||||
input_on_attention_tp_slices=(self.attn_stage.input_on_attention_tp_slices),
|
||||
prev_topk_indices=prev_topk_indices,
|
||||
)
|
||||
if isinstance(hidden_states, tuple):
|
||||
@@ -948,11 +951,8 @@ class Glm5NextDecoderLayer(nn.Module):
|
||||
topk_indices = None
|
||||
get_attn_tp_context().clear_attn_inputs()
|
||||
|
||||
hidden_states, residual = self.layer_communicator.prepare_mlp(
|
||||
hidden_states,
|
||||
residual,
|
||||
forward_batch,
|
||||
)
|
||||
hidden_states = self.attn_stage.finish(hidden_states, forward_batch)
|
||||
hidden_states = self.ffn_stage.prepare(hidden_states, forward_batch)
|
||||
|
||||
if isinstance(self.mlp, Glm5NextMLP):
|
||||
gemm_output_zero_allocator = None
|
||||
@@ -968,15 +968,15 @@ class Glm5NextDecoderLayer(nn.Module):
|
||||
else:
|
||||
_mlp_ctx = nullcontext()
|
||||
|
||||
with self.layer_communicator.ffn_exit(forward_batch) as ffn_exit, _mlp_ctx:
|
||||
with self.ffn_stage.exit(forward_batch) as ffn_exit, _mlp_ctx:
|
||||
hidden_states = self.mlp(
|
||||
hidden_states,
|
||||
forward_batch,
|
||||
gemm_output_zero_allocator,
|
||||
)
|
||||
hidden_states, residual = ffn_exit.finish(hidden_states, residual)
|
||||
hidden_states = ffn_exit.finish(hidden_states)
|
||||
|
||||
return hidden_states, residual, topk_indices
|
||||
return (hidden_states, topk_indices)
|
||||
|
||||
|
||||
class Glm5NextModel(nn.Module):
|
||||
@@ -1106,12 +1106,12 @@ class Glm5NextModel(nn.Module):
|
||||
hidden_states = self.embed_tokens(input_ids)
|
||||
else:
|
||||
hidden_states = input_embeds
|
||||
residual = residual_batch.start(forward_batch)
|
||||
residual_batch.start(forward_batch)
|
||||
else:
|
||||
assert pp_proxy_tensors is not None
|
||||
hidden_states, residual = self.layers[
|
||||
self.start_layer
|
||||
].layer_communicator.from_pp(pp_proxy_tensors, forward_batch)
|
||||
hidden_states = self.layers[self.start_layer].attn_stage.from_pp(
|
||||
pp_proxy_tensors, forward_batch
|
||||
)
|
||||
device = hidden_states.device
|
||||
zero_allocator = BumpAllocator(
|
||||
buffer_size=total_num_layers * 2 * (2 if forward_batch.can_run_tbo else 1),
|
||||
@@ -1165,11 +1165,10 @@ class Glm5NextModel(nn.Module):
|
||||
aux_hidden_states.capture(aux_hidden_state, owned=owned)
|
||||
|
||||
layer = self.layers[i]
|
||||
hidden_states, residual, topk_indices = layer(
|
||||
(hidden_states, topk_indices) = layer(
|
||||
positions,
|
||||
hidden_states,
|
||||
forward_batch,
|
||||
residual,
|
||||
zero_allocator,
|
||||
gemm_output_zero_allocator,
|
||||
prev_topk_indices=topk_indices,
|
||||
@@ -1179,38 +1178,25 @@ class Glm5NextModel(nn.Module):
|
||||
)
|
||||
|
||||
if normal_end_layer != self.end_layer:
|
||||
hidden_states, residual = model_forward_maybe_tbo(
|
||||
hidden_states = model_forward_stages(
|
||||
layers=self.layers[normal_end_layer : self.end_layer],
|
||||
enable_tbo=True,
|
||||
positions=positions,
|
||||
forward_batch=forward_batch,
|
||||
hidden_states=hidden_states,
|
||||
residual=residual,
|
||||
zero_allocator=zero_allocator,
|
||||
)
|
||||
|
||||
last_layer = self.layers[self.end_layer - 1]
|
||||
hidden_states, residual = last_layer.layer_communicator.finish_layer_stack(
|
||||
hidden_states,
|
||||
residual,
|
||||
forward_batch,
|
||||
preserve_declared=not self.pp_group.is_last_rank,
|
||||
)
|
||||
if not self.pp_group.is_last_rank:
|
||||
if self.config.mhc:
|
||||
return PPProxyTensors({"hidden_states": hidden_states})
|
||||
return PPProxyTensors(
|
||||
{
|
||||
"hidden_states": hidden_states,
|
||||
"residual": residual,
|
||||
}
|
||||
)
|
||||
return residual_batch.to_pp(hidden_states, forward_batch)
|
||||
else:
|
||||
hidden_states = residual_batch.finish(hidden_states, forward_batch)
|
||||
if not forward_batch.forward_mode.is_idle():
|
||||
if residual is None:
|
||||
hidden_states = self.norm(hidden_states)
|
||||
else:
|
||||
hidden_states, _ = self.norm(hidden_states, residual)
|
||||
hidden_states = residual_batch.norm(
|
||||
hidden_states, forward_batch, self.norm
|
||||
)
|
||||
|
||||
if len(aux_hidden_states) == 0:
|
||||
return hidden_states
|
||||
|
||||
@@ -14,7 +14,11 @@ from sglang.srt.layers.aux_hidden_states import (
|
||||
AuxHiddenStateAccumulator,
|
||||
AuxHiddenStateList,
|
||||
)
|
||||
from sglang.srt.layers.communicator import LayerCommunicator, LayerFacts
|
||||
from sglang.srt.layers.communicator import (
|
||||
declare_attn,
|
||||
declare_ffn,
|
||||
make_stages,
|
||||
)
|
||||
from sglang.srt.layers.communicator.residual import batch as residual_batch
|
||||
from sglang.srt.layers.dp_attention import is_dp_attention_enabled
|
||||
from sglang.srt.layers.layernorm import GemmaRMSNorm
|
||||
@@ -421,19 +425,17 @@ class _InternS2MobiusDecoderMixin:
|
||||
def _forward_after_attention(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
residual: torch.Tensor | None,
|
||||
forward_batch: ForwardBatch,
|
||||
meta_mlp: nn.ModuleList,
|
||||
) -> tuple[torch.Tensor, torch.Tensor | None]:
|
||||
hidden_states, residual = self.layer_communicator.prepare_mlp(
|
||||
hidden_states, residual, forward_batch
|
||||
)
|
||||
with self.layer_communicator.ffn_exit(forward_batch) as ffn_exit:
|
||||
) -> torch.Tensor:
|
||||
hidden_states = self.attn_stage.finish(hidden_states, forward_batch)
|
||||
hidden_states = self.ffn_stage.prepare(hidden_states, forward_batch)
|
||||
with self.ffn_stage.exit(forward_batch) as ffn_exit:
|
||||
hidden_states = self._forward_mobius_mlp(
|
||||
hidden_states, forward_batch, meta_mlp
|
||||
)
|
||||
hidden_states, residual = ffn_exit.finish(hidden_states, residual)
|
||||
return hidden_states, residual
|
||||
hidden_states = ffn_exit.finish(hidden_states)
|
||||
return hidden_states
|
||||
|
||||
|
||||
class InternS2MobiusLayerMlp(nn.Module):
|
||||
@@ -481,13 +483,7 @@ class InternS2MobiusLinearDecoderLayer(_InternS2MobiusDecoderMixin, nn.Module):
|
||||
|
||||
layer_prefix = prefix.removesuffix(".linear_attn")
|
||||
self._init_mobius_mlp(config, quant_config, layer_prefix)
|
||||
self.layer_facts = LayerFacts.init_new(
|
||||
layer_id=layer_id,
|
||||
num_layers=config.num_hidden_layers,
|
||||
is_layer_sparse=True,
|
||||
is_previous_layer_sparse=True,
|
||||
is_next_layer_sparse=True,
|
||||
)
|
||||
|
||||
self.input_layernorm = GemmaRMSNorm(config.hidden_size, eps=config.rms_norm_eps)
|
||||
self.post_attention_layernorm = GemmaRMSNorm(
|
||||
config.hidden_size, eps=config.rms_norm_eps
|
||||
@@ -496,37 +492,43 @@ class InternS2MobiusLinearDecoderLayer(_InternS2MobiusDecoderMixin, nn.Module):
|
||||
_enable_qwen35_fused_ar_quant()
|
||||
and _linear_accepts_fp8_tuple(self.linear_attn.in_proj_qkvz)
|
||||
)
|
||||
self.layer_communicator = LayerCommunicator(
|
||||
layer_facts=self.layer_facts,
|
||||
input_layernorm=self.input_layernorm,
|
||||
post_attention_layernorm=self.post_attention_layernorm,
|
||||
allow_reduce_scatter=True,
|
||||
enable_fused_ar_quant=enable_fused_ar_quant,
|
||||
fused_ar_quant_keep_bf16=enable_fused_ar_quant,
|
||||
allow_deferred_ffn_reduction=False,
|
||||
self.attn_stage, self.ffn_stage = make_stages(
|
||||
(
|
||||
declare_attn(),
|
||||
self.input_layernorm,
|
||||
{
|
||||
"enable_fused_ar_quant": enable_fused_ar_quant,
|
||||
"fused_ar_quant_keep_bf16": enable_fused_ar_quant,
|
||||
},
|
||||
),
|
||||
(
|
||||
declare_ffn(
|
||||
sparse=True,
|
||||
next_sparse=True,
|
||||
),
|
||||
self.post_attention_layernorm,
|
||||
),
|
||||
previous=declare_ffn(sparse=True, next_sparse=True)
|
||||
if layer_id != 0
|
||||
else None,
|
||||
terminal=layer_id == config.num_hidden_layers - 1,
|
||||
)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
residual: torch.Tensor | None,
|
||||
meta_mlp: nn.ModuleList,
|
||||
**kwargs,
|
||||
):
|
||||
forward_batch = kwargs["forward_batch"]
|
||||
hidden_states, residual = (
|
||||
self.layer_communicator.prepare_attn_and_capture_last_layer_outputs(
|
||||
hidden_states,
|
||||
residual,
|
||||
forward_batch,
|
||||
captured_last_layer_outputs=kwargs.get("captured_last_layer_outputs"),
|
||||
)
|
||||
hidden_states = self.attn_stage.prepare(
|
||||
hidden_states,
|
||||
forward_batch,
|
||||
captured_last_layer_outputs=kwargs.get("captured_last_layer_outputs"),
|
||||
)
|
||||
if not forward_batch.forward_mode.is_idle():
|
||||
hidden_states = self.linear_attn(hidden_states, forward_batch)
|
||||
return self._forward_after_attention(
|
||||
hidden_states, residual, forward_batch, meta_mlp
|
||||
)
|
||||
return self._forward_after_attention(hidden_states, forward_batch, meta_mlp)
|
||||
|
||||
|
||||
class InternS2MobiusAttentionDecoderLayer(
|
||||
@@ -614,13 +616,7 @@ class InternS2MobiusAttentionDecoderLayer(
|
||||
|
||||
layer_prefix = prefix.removesuffix(".self_attn")
|
||||
self._init_mobius_mlp(config, quant_config, layer_prefix)
|
||||
self.layer_facts = LayerFacts.init_new(
|
||||
layer_id=layer_id,
|
||||
num_layers=config.num_hidden_layers,
|
||||
is_layer_sparse=True,
|
||||
is_previous_layer_sparse=True,
|
||||
is_next_layer_sparse=True,
|
||||
)
|
||||
|
||||
self.input_layernorm = GemmaRMSNorm(config.hidden_size, eps=config.rms_norm_eps)
|
||||
self.post_attention_layernorm = GemmaRMSNorm(
|
||||
config.hidden_size, eps=config.rms_norm_eps
|
||||
@@ -630,14 +626,26 @@ class InternS2MobiusAttentionDecoderLayer(
|
||||
enable_fused_ar_quant = (
|
||||
_enable_qwen35_fused_ar_quant() and _linear_accepts_fp8_tuple(self.qkv_proj)
|
||||
)
|
||||
self.layer_communicator = LayerCommunicator(
|
||||
layer_facts=self.layer_facts,
|
||||
input_layernorm=self.input_layernorm,
|
||||
post_attention_layernorm=self.post_attention_layernorm,
|
||||
allow_reduce_scatter=True,
|
||||
enable_fused_ar_quant=enable_fused_ar_quant,
|
||||
fused_ar_quant_keep_bf16=False,
|
||||
allow_deferred_ffn_reduction=False,
|
||||
self.attn_stage, self.ffn_stage = make_stages(
|
||||
(
|
||||
declare_attn(),
|
||||
self.input_layernorm,
|
||||
{
|
||||
"enable_fused_ar_quant": enable_fused_ar_quant,
|
||||
"fused_ar_quant_keep_bf16": False,
|
||||
},
|
||||
),
|
||||
(
|
||||
declare_ffn(
|
||||
sparse=True,
|
||||
next_sparse=True,
|
||||
),
|
||||
self.post_attention_layernorm,
|
||||
),
|
||||
previous=declare_ffn(sparse=True, next_sparse=True)
|
||||
if layer_id != 0
|
||||
else None,
|
||||
terminal=layer_id == config.num_hidden_layers - 1,
|
||||
)
|
||||
self.alt_stream = alt_stream
|
||||
|
||||
@@ -645,20 +653,16 @@ class InternS2MobiusAttentionDecoderLayer(
|
||||
self,
|
||||
positions: torch.Tensor,
|
||||
hidden_states: torch.Tensor,
|
||||
residual: torch.Tensor | None,
|
||||
forward_batch: ForwardBatch,
|
||||
meta_mlp: nn.ModuleList,
|
||||
captured_last_layer_outputs: AuxHiddenStateAccumulator | None = None,
|
||||
**kwargs,
|
||||
):
|
||||
del kwargs
|
||||
hidden_states, residual = (
|
||||
self.layer_communicator.prepare_attn_and_capture_last_layer_outputs(
|
||||
hidden_states,
|
||||
residual,
|
||||
forward_batch,
|
||||
captured_last_layer_outputs=captured_last_layer_outputs,
|
||||
)
|
||||
hidden_states = self.attn_stage.prepare(
|
||||
hidden_states,
|
||||
forward_batch,
|
||||
captured_last_layer_outputs=captured_last_layer_outputs,
|
||||
)
|
||||
if not forward_batch.forward_mode.is_idle():
|
||||
hidden_states = self.self_attention(
|
||||
@@ -666,9 +670,7 @@ class InternS2MobiusAttentionDecoderLayer(
|
||||
hidden_states=hidden_states,
|
||||
forward_batch=forward_batch,
|
||||
)
|
||||
return self._forward_after_attention(
|
||||
hidden_states, residual, forward_batch, meta_mlp
|
||||
)
|
||||
return self._forward_after_attention(hidden_states, forward_batch, meta_mlp)
|
||||
|
||||
|
||||
class InternS2MobiusForCausalLM(Qwen3_5ForCausalLM):
|
||||
@@ -767,20 +769,17 @@ class InternS2MobiusForCausalLM(Qwen3_5ForCausalLM):
|
||||
hidden_states = (
|
||||
self.embed_tokens(input_ids) if input_embeds is None else input_embeds
|
||||
)
|
||||
residual = residual_batch.start(forward_batch)
|
||||
residual_batch.start(forward_batch)
|
||||
aux_hidden_states = AuxHiddenStateList()
|
||||
for layer_idx, layer in enumerate(self.layers):
|
||||
hidden_states, residual = layer(
|
||||
hidden_states = layer(
|
||||
positions=positions,
|
||||
hidden_states=hidden_states,
|
||||
residual=residual,
|
||||
forward_batch=forward_batch,
|
||||
meta_mlp=self.meta_mlp,
|
||||
captured_last_layer_outputs=(
|
||||
aux_hidden_states
|
||||
if getattr(layer, "_is_layer_to_capture", False)
|
||||
else None
|
||||
),
|
||||
captured_last_layer_outputs=aux_hidden_states
|
||||
if getattr(layer, "_is_layer_to_capture", False)
|
||||
else None,
|
||||
)
|
||||
if (
|
||||
input_deepstack_embeds is not None
|
||||
@@ -788,19 +787,16 @@ class InternS2MobiusForCausalLM(Qwen3_5ForCausalLM):
|
||||
and layer_idx < 3
|
||||
):
|
||||
start = self.hidden_size * layer_idx
|
||||
hidden_states.add_(
|
||||
input_deepstack_embeds[:, start : start + self.hidden_size]
|
||||
hidden_states = residual_batch.add_to_output(
|
||||
hidden_states,
|
||||
forward_batch,
|
||||
input_deepstack_embeds[:, start : start + self.hidden_size],
|
||||
)
|
||||
|
||||
hidden_states, residual = self.layers[-1].layer_communicator.finish_layer_stack(
|
||||
hidden_states, residual, forward_batch
|
||||
)
|
||||
hidden_states = residual_batch.finish(hidden_states, forward_batch)
|
||||
|
||||
if hidden_states.shape[0] != 0:
|
||||
if residual is None:
|
||||
hidden_states = self.norm(hidden_states)
|
||||
else:
|
||||
hidden_states, _ = self.norm(hidden_states, residual)
|
||||
hidden_states = residual_batch.norm(hidden_states, forward_batch, self.norm)
|
||||
return (
|
||||
hidden_states
|
||||
if not aux_hidden_states
|
||||
|
||||
@@ -161,7 +161,7 @@ class Eagle3MLADecoderLayer(nn.Module):
|
||||
attn_input = torch.cat([embeds, hidden_states], dim=-1)
|
||||
|
||||
# MLA's forward_absorb_prepare reads the qkv-down projection result
|
||||
# from attn_tp_context. We bypass LayerCommunicator here (the eagle3
|
||||
# from attn_tp_context. We bypass layer boundary preparation here (the eagle3
|
||||
# draft layer is one isolated layer with custom pre-attention norms),
|
||||
# so publish the attention input ourselves.
|
||||
get_attn_tp_context().set_attn_inputs(
|
||||
|
||||
@@ -243,15 +243,14 @@ def _merge_weights_as_views(
|
||||
return merged, sizes
|
||||
|
||||
|
||||
# K3 cannot use LayerCommunicator: the attn-res aggregation kernels replace
|
||||
# input_layernorm / post_attention_layernorm, which the communicator expects
|
||||
# to own. Instead the MLP/MoE modules gather/scatter around their own body:
|
||||
# K3 manages its own boundaries: the attn-res aggregation kernels replace
|
||||
# input_layernorm / post_attention_layernorm, and the MLP/MoE modules
|
||||
# gather/scatter around their own body:
|
||||
# attention and the attn-res buffers stay in local (per-DP-rank) token space,
|
||||
# the MLP/MoE runs on the DP-gathered global batch, and the delayed prefix_sum
|
||||
# add stays local, applied after the scatter back.
|
||||
def _dp_local_buffer_group():
|
||||
"""Symmetric-memory group for the local DP buffer (mirrors
|
||||
CommunicateSummableTensorPairFn._scatter_hidden_states)."""
|
||||
"""Symmetric-memory group holding this DP rank's local token buffer."""
|
||||
parallel = get_parallel()
|
||||
if parallel.tp_size == parallel.attn_dp_size:
|
||||
return get_parallel().tp_group
|
||||
@@ -2252,7 +2251,7 @@ class KimiK3MLAAttention(DeepseekV2AttentionMLA):
|
||||
|
||||
self.o_proj.forward = _symm_o_proj_forward
|
||||
else:
|
||||
# K3 has no LayerCommunicator, so o_proj must reduce within the
|
||||
# K3 owns its output reduction, so o_proj must reduce within the
|
||||
# attn-TP group itself: the default full-TP collective is the wrong
|
||||
# group at attn_tp>1 and deadlocks idle DP ranks.
|
||||
self.o_proj.use_dp_attention_reduce = True
|
||||
@@ -2571,7 +2570,7 @@ class KimiK3DecoderLayer(nn.Module):
|
||||
) -> torch.Tensor:
|
||||
# DP attention: idle ranks (padded to the global shape) have no
|
||||
# attention metadata; pass hidden_states through shape-preserving
|
||||
# (same as the LayerCommunicator models' is_idle skip).
|
||||
# (matching the idle skip in other DP-attention models).
|
||||
if forward_batch.forward_mode.is_idle():
|
||||
return hidden_states
|
||||
|
||||
@@ -2614,8 +2613,8 @@ class KimiK3DecoderLayer(nn.Module):
|
||||
forward_batch: ForwardBatch,
|
||||
zero_allocator: BumpAllocator,
|
||||
) -> torch.Tensor:
|
||||
# For MLA layers with q_lora_rank, set up communicator attn_inputs
|
||||
# before the forward call (normally done by LayerCommunicator).
|
||||
# For MLA layers with q_lora_rank, set up attn_inputs before the
|
||||
# forward call (normally done by the attention boundary).
|
||||
from sglang.srt.layers.communicator import (
|
||||
AttentionInputs,
|
||||
get_attn_tp_context,
|
||||
|
||||
@@ -21,8 +21,9 @@ from sglang.srt.environ import envs
|
||||
from sglang.srt.layers.activation import SiluAndMul
|
||||
from sglang.srt.layers.aux_hidden_states import AuxHiddenStateList
|
||||
from sglang.srt.layers.communicator import (
|
||||
LayerCommunicator,
|
||||
LayerFacts,
|
||||
declare_attn,
|
||||
declare_ffn,
|
||||
make_stages,
|
||||
)
|
||||
from sglang.srt.layers.communicator.residual import batch as residual_batch
|
||||
from sglang.srt.layers.dp_attention import (
|
||||
@@ -456,18 +457,21 @@ class LagunaDecoderLayer(nn.Module):
|
||||
config.hidden_size, eps=config.rms_norm_eps
|
||||
)
|
||||
|
||||
self.layer_facts = LayerFacts.init_new(
|
||||
layer_id=layer_id,
|
||||
num_layers=config.num_hidden_layers,
|
||||
is_layer_sparse=self.is_layer_sparse,
|
||||
is_previous_layer_sparse=is_previous_layer_sparse,
|
||||
is_next_layer_sparse=is_next_layer_sparse,
|
||||
)
|
||||
self.layer_communicator = LayerCommunicator(
|
||||
layer_facts=self.layer_facts,
|
||||
input_layernorm=self.input_layernorm,
|
||||
post_attention_layernorm=self.post_attention_layernorm,
|
||||
allow_reduce_scatter=True,
|
||||
self.attn_stage, self.ffn_stage = make_stages(
|
||||
(declare_attn(), self.input_layernorm),
|
||||
(
|
||||
declare_ffn(
|
||||
sparse=self.is_layer_sparse,
|
||||
next_sparse=is_next_layer_sparse,
|
||||
),
|
||||
self.post_attention_layernorm,
|
||||
),
|
||||
previous=declare_ffn(
|
||||
sparse=is_previous_layer_sparse, next_sparse=self.is_layer_sparse
|
||||
)
|
||||
if layer_id != 0
|
||||
else None,
|
||||
terminal=layer_id == config.num_hidden_layers - 1,
|
||||
)
|
||||
|
||||
def forward(
|
||||
@@ -475,16 +479,10 @@ class LagunaDecoderLayer(nn.Module):
|
||||
positions: torch.Tensor,
|
||||
hidden_states: torch.Tensor,
|
||||
forward_batch: ForwardBatch,
|
||||
residual: Optional[torch.Tensor],
|
||||
capture_output=None,
|
||||
) -> Tuple[torch.Tensor, torch.Tensor]:
|
||||
hidden_states, residual = (
|
||||
self.layer_communicator.prepare_attn_and_capture_last_layer_outputs(
|
||||
hidden_states,
|
||||
residual,
|
||||
forward_batch,
|
||||
capture_output=capture_output,
|
||||
)
|
||||
) -> torch.Tensor:
|
||||
hidden_states = self.attn_stage.prepare(
|
||||
hidden_states, forward_batch, capture_output=capture_output
|
||||
)
|
||||
if hidden_states.shape[0] != 0:
|
||||
hidden_states = self.self_attn(
|
||||
@@ -492,17 +490,16 @@ class LagunaDecoderLayer(nn.Module):
|
||||
hidden_states=hidden_states,
|
||||
forward_batch=forward_batch,
|
||||
)
|
||||
hidden_states, residual = self.layer_communicator.prepare_mlp(
|
||||
hidden_states, residual, forward_batch
|
||||
)
|
||||
hidden_states = self.attn_stage.finish(hidden_states, forward_batch)
|
||||
hidden_states = self.ffn_stage.prepare(hidden_states, forward_batch)
|
||||
|
||||
with self.layer_communicator.ffn_exit(forward_batch) as ffn_exit:
|
||||
with self.ffn_stage.exit(forward_batch) as ffn_exit:
|
||||
hidden_states = self.mlp(
|
||||
hidden_states,
|
||||
forward_batch=forward_batch,
|
||||
)
|
||||
hidden_states, residual = ffn_exit.finish(hidden_states, residual)
|
||||
return hidden_states, residual
|
||||
hidden_states = ffn_exit.finish(hidden_states)
|
||||
return hidden_states
|
||||
|
||||
|
||||
class LagunaModel(nn.Module):
|
||||
@@ -589,48 +586,37 @@ class LagunaModel(nn.Module):
|
||||
hidden_states = self.embed_tokens(input_ids)
|
||||
else:
|
||||
hidden_states = input_embeds
|
||||
residual = residual_batch.start(forward_batch)
|
||||
residual_batch.start(forward_batch)
|
||||
else:
|
||||
assert pp_proxy_tensors is not None
|
||||
hidden_states, residual = self.layers[
|
||||
self.start_layer
|
||||
].layer_communicator.from_pp(pp_proxy_tensors, forward_batch)
|
||||
hidden_states = self.layers[self.start_layer].attn_stage.from_pp(
|
||||
pp_proxy_tensors, forward_batch
|
||||
)
|
||||
|
||||
aux_hidden_states = AuxHiddenStateList()
|
||||
for i in range(self.start_layer, self.end_layer):
|
||||
layer = self.layers[i]
|
||||
hidden_states, residual = layer(
|
||||
hidden_states = layer(
|
||||
positions,
|
||||
hidden_states,
|
||||
forward_batch,
|
||||
residual,
|
||||
capture_output=aux_hidden_states.capture
|
||||
if i in self.layers_to_capture
|
||||
else None,
|
||||
)
|
||||
|
||||
last_layer = self.layers[self.end_layer - 1]
|
||||
hidden_states, residual = last_layer.layer_communicator.finish_layer_stack(
|
||||
hidden_states,
|
||||
residual,
|
||||
forward_batch,
|
||||
preserve_declared=not self.pp_group.is_last_rank,
|
||||
)
|
||||
if not self.pp_group.is_last_rank:
|
||||
return PPProxyTensors(
|
||||
{"hidden_states": hidden_states, "residual": residual}
|
||||
)
|
||||
return residual_batch.to_pp(hidden_states, forward_batch)
|
||||
hidden_states = residual_batch.finish(hidden_states, forward_batch)
|
||||
|
||||
if hidden_states.shape[0] != 0:
|
||||
hidden_states = last_layer.layer_communicator.norm_output(
|
||||
hidden_states = residual_batch.norm(
|
||||
hidden_states,
|
||||
residual,
|
||||
forward_batch,
|
||||
self.norm,
|
||||
capture_output=(
|
||||
aux_hidden_states.append
|
||||
if self.end_layer in self.layers_to_capture
|
||||
else None
|
||||
),
|
||||
capture_output=aux_hidden_states.append
|
||||
if self.end_layer in self.layers_to_capture
|
||||
else None,
|
||||
)
|
||||
if len(aux_hidden_states) == 0:
|
||||
return hidden_states
|
||||
|
||||
@@ -51,7 +51,11 @@ from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_r
|
||||
from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation
|
||||
from sglang.srt.layers import deep_gemm_wrapper
|
||||
from sglang.srt.layers.activation import SiluAndMul
|
||||
from sglang.srt.layers.communicator import LayerCommunicator, LayerFacts
|
||||
from sglang.srt.layers.communicator import (
|
||||
declare_attn,
|
||||
declare_ffn,
|
||||
make_stages,
|
||||
)
|
||||
from sglang.srt.layers.communicator.residual import batch as residual_batch
|
||||
from sglang.srt.layers.dp_attention import (
|
||||
is_dp_attention_enabled,
|
||||
@@ -66,7 +70,10 @@ from sglang.srt.layers.logits_processor import LogitsProcessor
|
||||
from sglang.srt.layers.moe.ep_moe.layer import DeepEPMoE, get_moe_impl_class
|
||||
from sglang.srt.layers.moe.fused_moe_triton.layer import FusedMoE
|
||||
from sglang.srt.layers.moe.topk import StandardTopKOutput, TopK
|
||||
from sglang.srt.layers.moe.utils import filter_moe_weight_param_global_expert
|
||||
from sglang.srt.layers.moe.utils import (
|
||||
filter_moe_weight_param_global_expert,
|
||||
should_skip_mlp_all_reduce,
|
||||
)
|
||||
from sglang.srt.layers.n_gram_embedding import NgramEmbedding
|
||||
from sglang.srt.layers.quantization.base_config import QuantizationConfig
|
||||
from sglang.srt.layers.quantization.fp8_utils import (
|
||||
@@ -323,7 +330,11 @@ class LongcatFlashMoE(nn.Module):
|
||||
# expert outputs across EP ranks, so an extra all-reduce double-counts.
|
||||
from sglang.srt.layers.moe.utils import get_moe_a2a_backend as _lc_gab
|
||||
|
||||
if self.tp_size > 1 and _lc_gab().is_none():
|
||||
if (
|
||||
self.tp_size > 1
|
||||
and _lc_gab().is_none()
|
||||
and not should_skip_mlp_all_reduce()
|
||||
):
|
||||
final_hidden_states = tensor_model_parallel_all_reduce(final_hidden_states)
|
||||
|
||||
return final_hidden_states.view(num_tokens, hidden_dim)
|
||||
@@ -420,41 +431,36 @@ class LongcatFlashDecoderLayer(nn.Module):
|
||||
self.attn_tp_size = get_parallel().attn_tp_size
|
||||
self.attn_tp_rank = get_parallel().attn_tp_rank
|
||||
|
||||
self.mlp_layer_facts = [
|
||||
LayerFacts.init_new(
|
||||
layer_id=self.layer_id * 2 + i,
|
||||
num_layers=config.num_hidden_layers,
|
||||
is_layer_sparse=False,
|
||||
is_previous_layer_sparse=False,
|
||||
# TODO: Check if the following is correct.
|
||||
is_next_layer_sparse=False,
|
||||
)
|
||||
for i in range(2)
|
||||
]
|
||||
self.mlp_layer_communicator = [
|
||||
LayerCommunicator(
|
||||
layer_facts=self.mlp_layer_facts[i],
|
||||
input_layernorm=self.input_layernorm[i],
|
||||
post_attention_layernorm=self.post_attention_layernorm[i],
|
||||
qkv_latent_func=self.self_attn[i].prepare_qkv_latent,
|
||||
allow_deferred_ffn_reduction=False,
|
||||
)
|
||||
for i in range(2)
|
||||
]
|
||||
|
||||
self.moe_layer_facts = LayerFacts.init_new(
|
||||
layer_id=self.layer_id,
|
||||
num_layers=config.num_hidden_layers,
|
||||
is_layer_sparse=True,
|
||||
is_previous_layer_sparse=True,
|
||||
# TODO: Check if the following is correct.
|
||||
is_next_layer_sparse=True,
|
||||
self.attn_stage, self.moe_stage = make_stages(
|
||||
(
|
||||
declare_attn(),
|
||||
self.input_layernorm[0],
|
||||
{"qkv_latent_func": self.self_attn[0].prepare_qkv_latent},
|
||||
),
|
||||
(
|
||||
declare_ffn(sparse=True, next_sparse=True),
|
||||
self.post_attention_layernorm[0],
|
||||
),
|
||||
previous=declare_ffn(sparse=True, next_sparse=True)
|
||||
if self.layer_id != 0
|
||||
else None,
|
||||
terminal=self.layer_id == config.num_hidden_layers - 1,
|
||||
)
|
||||
self.moe_layer_communicator = LayerCommunicator(
|
||||
layer_facts=self.moe_layer_facts,
|
||||
input_layernorm=self.input_layernorm[0],
|
||||
post_attention_layernorm=self.post_attention_layernorm[0],
|
||||
qkv_latent_func=self.self_attn[0].prepare_qkv_latent,
|
||||
self.first_ffn, self.second_attn, self.second_ffn = make_stages(
|
||||
(
|
||||
declare_ffn(ordinary_only=True),
|
||||
self.post_attention_layernorm[0],
|
||||
),
|
||||
(
|
||||
declare_attn(ordinary_only=True),
|
||||
self.input_layernorm[1],
|
||||
{"qkv_latent_func": self.self_attn[1].prepare_qkv_latent},
|
||||
),
|
||||
(
|
||||
declare_ffn(ordinary_only=True),
|
||||
self.post_attention_layernorm[1],
|
||||
),
|
||||
prepared_from=self.moe_stage.declaration,
|
||||
)
|
||||
|
||||
def forward(
|
||||
@@ -462,14 +468,11 @@ class LongcatFlashDecoderLayer(nn.Module):
|
||||
positions: torch.Tensor,
|
||||
hidden_states: torch.Tensor,
|
||||
forward_batch: ForwardBatch,
|
||||
residual: Optional[torch.Tensor],
|
||||
zero_allocator: BumpAllocator,
|
||||
prev_topk_indices: Optional[torch.Tensor],
|
||||
) -> torch.Tensor:
|
||||
# first_attn
|
||||
hidden_states, residual = self.moe_layer_communicator.prepare_attn(
|
||||
hidden_states, residual, forward_batch
|
||||
)
|
||||
hidden_states = self.attn_stage.prepare(hidden_states, forward_batch)
|
||||
if hidden_states.shape[0] != 0:
|
||||
attn_out = self.self_attn[0](
|
||||
positions=positions,
|
||||
@@ -484,52 +487,46 @@ class LongcatFlashDecoderLayer(nn.Module):
|
||||
hidden_states = attn_out
|
||||
|
||||
# moe
|
||||
hidden_states, residual = self.moe_layer_communicator.prepare_mlp(
|
||||
hidden_states, residual, forward_batch
|
||||
)
|
||||
moe_hidden_states = self.moe_layer_communicator.branch_output(
|
||||
hidden_states = self.attn_stage.finish(hidden_states, forward_batch)
|
||||
hidden_states = self.moe_stage.prepare(hidden_states, forward_batch)
|
||||
moe_hidden_states = self.moe_stage.branch_output(
|
||||
self.mlp(hidden_states.clone()), forward_batch
|
||||
)
|
||||
|
||||
hidden_states, residual, prev_topk_indices = self.forward_mlp(
|
||||
hidden_states, prev_topk_indices = self.forward_mlp(
|
||||
hidden_states,
|
||||
positions,
|
||||
residual,
|
||||
forward_batch,
|
||||
zero_allocator,
|
||||
prev_topk_indices,
|
||||
)
|
||||
|
||||
hidden_states, residual = self.moe_layer_communicator.merge_branch(
|
||||
hidden_states = self.moe_stage.merge_branch(
|
||||
moe_hidden_states,
|
||||
hidden_states,
|
||||
residual,
|
||||
self.mlp_layer_communicator[1],
|
||||
self.second_ffn,
|
||||
forward_batch,
|
||||
)
|
||||
return hidden_states, residual, prev_topk_indices
|
||||
return hidden_states, prev_topk_indices
|
||||
|
||||
def forward_mlp(
|
||||
self,
|
||||
hidden_states,
|
||||
positions,
|
||||
residual,
|
||||
forward_batch,
|
||||
zero_allocator,
|
||||
prev_topk_indices,
|
||||
):
|
||||
# first_mlp, on the input the MoE's boundary read
|
||||
hidden_states, residual = self.mlp_layer_communicator[0].branch_input(
|
||||
self.moe_layer_communicator, hidden_states, residual, forward_batch
|
||||
hidden_states = self.first_ffn.branch_input(
|
||||
self.moe_stage, hidden_states, forward_batch
|
||||
)
|
||||
with self.mlp_layer_communicator[0].ffn_exit(forward_batch) as ffn_exit:
|
||||
with self.first_ffn.exit(forward_batch) as ffn_exit:
|
||||
hidden_states = self.mlps[0](hidden_states)
|
||||
hidden_states, residual = ffn_exit.finish(hidden_states, residual)
|
||||
hidden_states = ffn_exit.finish(hidden_states)
|
||||
|
||||
# second_attn
|
||||
hidden_states, residual = self.mlp_layer_communicator[1].prepare_attn(
|
||||
hidden_states, residual, forward_batch
|
||||
)
|
||||
hidden_states = self.second_attn.prepare(hidden_states, forward_batch)
|
||||
if hidden_states.shape[0] != 0:
|
||||
attn_out = self.self_attn[1](
|
||||
positions=positions,
|
||||
@@ -544,14 +541,13 @@ class LongcatFlashDecoderLayer(nn.Module):
|
||||
hidden_states = attn_out
|
||||
|
||||
# second_mlp
|
||||
hidden_states, residual = self.mlp_layer_communicator[1].prepare_mlp(
|
||||
hidden_states, residual, forward_batch
|
||||
)
|
||||
with self.mlp_layer_communicator[1].ffn_exit(forward_batch) as ffn_exit:
|
||||
hidden_states = self.second_attn.finish(hidden_states, forward_batch)
|
||||
hidden_states = self.second_ffn.prepare(hidden_states, forward_batch)
|
||||
with self.second_ffn.exit(forward_batch) as ffn_exit:
|
||||
hidden_states = self.mlps[1](hidden_states)
|
||||
hidden_states, residual = ffn_exit.finish(hidden_states, residual)
|
||||
hidden_states = ffn_exit.finish(hidden_states)
|
||||
|
||||
return hidden_states, residual, prev_topk_indices
|
||||
return hidden_states, prev_topk_indices
|
||||
|
||||
|
||||
class LongcatFlashModel(nn.Module):
|
||||
@@ -625,39 +621,29 @@ class LongcatFlashModel(nn.Module):
|
||||
else:
|
||||
hidden_states = input_embeds
|
||||
|
||||
residual = residual_batch.start(forward_batch)
|
||||
residual_batch.start(forward_batch)
|
||||
|
||||
aux_hidden_states = []
|
||||
topk_indices = None
|
||||
for i in range(total_num_layers):
|
||||
if i in self.layers_to_capture:
|
||||
aux_hidden_states.append(
|
||||
self.layers[i].moe_layer_communicator.snapshot(
|
||||
hidden_states, residual, at_input=True
|
||||
)
|
||||
self.layers[i].attn_stage.snapshot(hidden_states, forward_batch)
|
||||
)
|
||||
with get_global_expert_distribution_recorder().with_current_layer(i):
|
||||
layer = self.layers[i]
|
||||
hidden_states, residual, topk_indices = layer(
|
||||
hidden_states, topk_indices = layer(
|
||||
positions,
|
||||
hidden_states,
|
||||
forward_batch,
|
||||
residual,
|
||||
zero_allocator,
|
||||
topk_indices,
|
||||
)
|
||||
|
||||
last_layer = self.layers[-1]
|
||||
hidden_states, residual = last_layer.moe_layer_communicator.finish_layer_stack(
|
||||
hidden_states, residual, forward_batch
|
||||
hidden_states = residual_batch.norm(
|
||||
hidden_states, forward_batch, self.norm, skip_empty=True
|
||||
)
|
||||
|
||||
if hidden_states.shape[0] != 0:
|
||||
if residual is None:
|
||||
hidden_states = self.norm(hidden_states)
|
||||
else:
|
||||
hidden_states, _ = self.norm(hidden_states, residual)
|
||||
|
||||
if len(aux_hidden_states) == 0:
|
||||
return hidden_states
|
||||
|
||||
|
||||
@@ -41,7 +41,11 @@ from sglang.kernels.ops.quantization.fp8_kernel import is_fp8_fnuz
|
||||
from sglang.srt.configs import LongcatFlashConfig
|
||||
from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder
|
||||
from sglang.srt.layers import deep_gemm_wrapper
|
||||
from sglang.srt.layers.communicator import LayerCommunicator, LayerFacts
|
||||
from sglang.srt.layers.communicator import (
|
||||
declare_attn,
|
||||
declare_ffn,
|
||||
make_stages,
|
||||
)
|
||||
from sglang.srt.layers.communicator.residual import batch as residual_batch
|
||||
from sglang.srt.layers.dp_attention import (
|
||||
is_dp_attention_enabled,
|
||||
@@ -160,17 +164,15 @@ class LongcatFlashDenseDecoderLayer(nn.Module):
|
||||
|
||||
self.attn_tp_size = get_parallel().attn_tp_size
|
||||
self.attn_tp_rank = get_parallel().attn_tp_rank
|
||||
self.layer_facts = LayerFacts.init_new(
|
||||
layer_id=self.layer_id,
|
||||
num_layers=config.num_hidden_layers,
|
||||
is_layer_sparse=False,
|
||||
is_previous_layer_sparse=False,
|
||||
is_next_layer_sparse=False,
|
||||
)
|
||||
self.layer_communicator = LayerCommunicator(
|
||||
layer_facts=self.layer_facts,
|
||||
input_layernorm=self.input_layernorm,
|
||||
post_attention_layernorm=self.post_attention_layernorm,
|
||||
|
||||
self.attn_stage, self.ffn_stage = make_stages(
|
||||
(declare_attn(), self.input_layernorm),
|
||||
(
|
||||
declare_ffn(sparse=False, next_sparse=False),
|
||||
self.post_attention_layernorm,
|
||||
),
|
||||
previous=declare_ffn() if self.layer_id != 0 else None,
|
||||
terminal=self.layer_id == config.num_hidden_layers - 1,
|
||||
)
|
||||
|
||||
def forward(
|
||||
@@ -178,13 +180,10 @@ class LongcatFlashDenseDecoderLayer(nn.Module):
|
||||
positions: torch.Tensor,
|
||||
hidden_states: torch.Tensor,
|
||||
forward_batch: ForwardBatch,
|
||||
residual: Optional[torch.Tensor],
|
||||
zero_allocator: BumpAllocator,
|
||||
) -> torch.Tensor:
|
||||
|
||||
hidden_states, residual = self.layer_communicator.prepare_attn(
|
||||
hidden_states, residual, forward_batch
|
||||
)
|
||||
hidden_states = self.attn_stage.prepare(hidden_states, forward_batch)
|
||||
if hidden_states.shape[0] != 0:
|
||||
hidden_states = self.self_attn(
|
||||
positions=positions,
|
||||
@@ -193,14 +192,11 @@ class LongcatFlashDenseDecoderLayer(nn.Module):
|
||||
zero_allocator=zero_allocator,
|
||||
)
|
||||
|
||||
hidden_states, residual = self.layer_communicator.prepare_mlp(
|
||||
hidden_states, residual, forward_batch
|
||||
)
|
||||
hidden_states = self.attn_stage.finish(hidden_states, forward_batch)
|
||||
hidden_states = self.ffn_stage.prepare(hidden_states, forward_batch)
|
||||
hidden_states = self.mlp(hidden_states)
|
||||
hidden_states, residual = self.layer_communicator.postprocess_layer(
|
||||
hidden_states, residual, forward_batch
|
||||
)
|
||||
return hidden_states, residual
|
||||
hidden_states = self.ffn_stage.postprocess(hidden_states, forward_batch)
|
||||
return hidden_states
|
||||
|
||||
|
||||
class LongcatFlashModelNextN(nn.Module):
|
||||
@@ -270,21 +266,18 @@ class LongcatFlashModelNextN(nn.Module):
|
||||
)
|
||||
)
|
||||
|
||||
residual = residual_batch.start(forward_batch)
|
||||
residual_batch.start(forward_batch)
|
||||
with get_global_expert_distribution_recorder().disable_this_region():
|
||||
hidden_states, residual = self.decoder(
|
||||
positions, hidden_states, forward_batch, residual, zero_allocator
|
||||
hidden_states = self.decoder(
|
||||
positions, hidden_states, forward_batch, zero_allocator
|
||||
)
|
||||
|
||||
hidden_states, residual = self.decoder.layer_communicator.finish_layer_stack(
|
||||
hidden_states, residual, forward_batch
|
||||
)
|
||||
hidden_states = residual_batch.finish(hidden_states, forward_batch)
|
||||
|
||||
if not forward_batch.forward_mode.is_idle():
|
||||
if residual is not None:
|
||||
hidden_states, _ = self.final_layernorm(hidden_states, residual)
|
||||
else:
|
||||
hidden_states = self.final_layernorm(hidden_states)
|
||||
hidden_states = residual_batch.norm(
|
||||
hidden_states, forward_batch, self.final_layernorm
|
||||
)
|
||||
return hidden_states
|
||||
|
||||
|
||||
|
||||
@@ -25,7 +25,11 @@ import torch
|
||||
from torch import nn
|
||||
from transformers import PretrainedConfig
|
||||
|
||||
from sglang.srt.layers.communicator import LayerCommunicator, LayerFacts
|
||||
from sglang.srt.layers.communicator import (
|
||||
declare_attn,
|
||||
declare_ffn,
|
||||
make_stages,
|
||||
)
|
||||
from sglang.srt.layers.layernorm import RMSNorm
|
||||
from sglang.srt.layers.linear import QKVParallelLinear, RowParallelLinear
|
||||
from sglang.srt.layers.quantization.base_config import QuantizationConfig
|
||||
@@ -442,22 +446,24 @@ class MellumDecoderLayer(Qwen3MoeDecoderLayer):
|
||||
is_previous_layer_sparse = _is_sparse(layer_id - 1)
|
||||
is_next_layer_sparse = _is_sparse(layer_id + 1)
|
||||
|
||||
self.layer_facts = LayerFacts.init_new(
|
||||
layer_id=layer_id,
|
||||
num_layers=cfg.num_hidden_layers,
|
||||
is_layer_sparse=self.is_layer_sparse,
|
||||
is_previous_layer_sparse=is_previous_layer_sparse,
|
||||
is_next_layer_sparse=is_next_layer_sparse,
|
||||
)
|
||||
|
||||
self.input_layernorm = RMSNorm(cfg.hidden_size, eps=rms_norm_eps)
|
||||
self.post_attention_layernorm = RMSNorm(cfg.hidden_size, eps=rms_norm_eps)
|
||||
|
||||
self.layer_communicator = LayerCommunicator(
|
||||
layer_facts=self.layer_facts,
|
||||
input_layernorm=self.input_layernorm,
|
||||
post_attention_layernorm=self.post_attention_layernorm,
|
||||
allow_reduce_scatter=True,
|
||||
self.attn_stage, self.ffn_stage = make_stages(
|
||||
(declare_attn(), self.input_layernorm),
|
||||
(
|
||||
declare_ffn(
|
||||
sparse=self.is_layer_sparse,
|
||||
next_sparse=is_next_layer_sparse,
|
||||
),
|
||||
self.post_attention_layernorm,
|
||||
),
|
||||
previous=declare_ffn(
|
||||
sparse=is_previous_layer_sparse, next_sparse=self.is_layer_sparse
|
||||
)
|
||||
if layer_id != 0
|
||||
else None,
|
||||
terminal=layer_id == cfg.num_hidden_layers - 1,
|
||||
)
|
||||
|
||||
|
||||
|
||||
@@ -22,7 +22,7 @@ import torch
|
||||
import torch.nn.functional as F
|
||||
from torch import nn
|
||||
|
||||
from sglang.srt.batch_overlap.two_batch_overlap import model_forward_maybe_tbo
|
||||
from sglang.srt.batch_overlap.two_batch_overlap import model_forward_stages
|
||||
from sglang.srt.configs.model_config import get_mimo_v2_fused_qkv_expected_tp_size
|
||||
from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder
|
||||
from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation
|
||||
@@ -33,9 +33,10 @@ from sglang.srt.layers.aux_hidden_states import (
|
||||
AuxHiddenStatePacker,
|
||||
)
|
||||
from sglang.srt.layers.communicator import (
|
||||
LayerCommunicator,
|
||||
LayerFacts,
|
||||
declare_attn,
|
||||
declare_ffn,
|
||||
enable_moe_dense_fully_dp,
|
||||
make_stages,
|
||||
)
|
||||
from sglang.srt.layers.communicator.residual import batch as residual_batch
|
||||
from sglang.srt.layers.dp_attention import (
|
||||
@@ -857,18 +858,21 @@ class MiMoV2DecoderLayer(nn.Module):
|
||||
config.hidden_size, eps=config.layernorm_epsilon
|
||||
)
|
||||
|
||||
self.layer_facts = LayerFacts.init_new(
|
||||
layer_id=layer_id,
|
||||
num_layers=config.num_hidden_layers,
|
||||
is_layer_sparse=self.is_layer_sparse,
|
||||
is_previous_layer_sparse=is_previous_layer_sparse,
|
||||
is_next_layer_sparse=is_next_layer_sparse,
|
||||
)
|
||||
self.layer_communicator = LayerCommunicator(
|
||||
layer_facts=self.layer_facts,
|
||||
input_layernorm=self.input_layernorm,
|
||||
post_attention_layernorm=self.post_attention_layernorm,
|
||||
allow_reduce_scatter=True,
|
||||
self.attn_stage, self.ffn_stage = make_stages(
|
||||
(declare_attn(), self.input_layernorm),
|
||||
(
|
||||
declare_ffn(
|
||||
sparse=self.is_layer_sparse,
|
||||
next_sparse=is_next_layer_sparse,
|
||||
),
|
||||
self.post_attention_layernorm,
|
||||
),
|
||||
previous=declare_ffn(
|
||||
sparse=is_previous_layer_sparse, next_sparse=self.is_layer_sparse
|
||||
)
|
||||
if layer_id != 0
|
||||
else None,
|
||||
terminal=layer_id == config.num_hidden_layers - 1,
|
||||
)
|
||||
|
||||
def forward(
|
||||
@@ -876,17 +880,13 @@ class MiMoV2DecoderLayer(nn.Module):
|
||||
positions: torch.Tensor,
|
||||
hidden_states: torch.Tensor,
|
||||
forward_batch: ForwardBatch,
|
||||
residual: Optional[torch.Tensor],
|
||||
captured_last_layer_outputs: Optional[AuxHiddenStateAccumulator] = None,
|
||||
) -> Tuple[torch.Tensor, torch.Tensor]:
|
||||
) -> torch.Tensor:
|
||||
# Self Attention
|
||||
hidden_states, residual = (
|
||||
self.layer_communicator.prepare_attn_and_capture_last_layer_outputs(
|
||||
hidden_states,
|
||||
residual,
|
||||
forward_batch,
|
||||
captured_last_layer_outputs=captured_last_layer_outputs,
|
||||
)
|
||||
hidden_states = self.attn_stage.prepare(
|
||||
hidden_states,
|
||||
forward_batch,
|
||||
captured_last_layer_outputs=captured_last_layer_outputs,
|
||||
)
|
||||
|
||||
if hidden_states.shape[0] != 0:
|
||||
@@ -896,15 +896,14 @@ class MiMoV2DecoderLayer(nn.Module):
|
||||
forward_batch=forward_batch,
|
||||
)
|
||||
|
||||
hidden_states, residual = self.layer_communicator.prepare_mlp(
|
||||
hidden_states, residual, forward_batch
|
||||
)
|
||||
hidden_states = self.attn_stage.finish(hidden_states, forward_batch)
|
||||
hidden_states = self.ffn_stage.prepare(hidden_states, forward_batch)
|
||||
|
||||
with self.layer_communicator.ffn_exit(forward_batch) as ffn_exit:
|
||||
with self.ffn_stage.exit(forward_batch) as ffn_exit:
|
||||
hidden_states = self.mlp(hidden_states, forward_batch)
|
||||
hidden_states, residual = ffn_exit.finish(hidden_states, residual)
|
||||
hidden_states = ffn_exit.finish(hidden_states)
|
||||
|
||||
return hidden_states, residual
|
||||
return hidden_states
|
||||
|
||||
def is_moe_layer(self, layer_idx: int) -> bool:
|
||||
return (
|
||||
@@ -923,11 +922,10 @@ class MiMoV2DecoderLayer(nn.Module):
|
||||
positions: torch.Tensor,
|
||||
hidden_states: torch.Tensor,
|
||||
forward_batch: ForwardBatch,
|
||||
residual: Optional[torch.Tensor],
|
||||
tbo_subbatch_index: Optional[int] = None,
|
||||
):
|
||||
state.hidden_states_after_comm_pre_attn, state.residual_after_input_ln = (
|
||||
self.layer_communicator.prepare_attn(hidden_states, residual, forward_batch)
|
||||
state.hidden_states_after_comm_pre_attn = self.attn_stage.prepare(
|
||||
hidden_states, forward_batch
|
||||
)
|
||||
state.update(
|
||||
dict(
|
||||
@@ -938,25 +936,21 @@ class MiMoV2DecoderLayer(nn.Module):
|
||||
)
|
||||
|
||||
def op_comm_prepare_mlp(self, state):
|
||||
state.hidden_states_mlp_input, state.residual_after_comm_pre_mlp = (
|
||||
self.layer_communicator.prepare_mlp(
|
||||
state.pop("hidden_states_after_attn"),
|
||||
state.pop("residual_after_input_ln"),
|
||||
state.forward_batch,
|
||||
)
|
||||
hidden_states = self.attn_stage.finish(
|
||||
state.pop("hidden_states_after_attn"), state.forward_batch
|
||||
)
|
||||
state.hidden_states_mlp_input = self.ffn_stage.prepare(
|
||||
hidden_states, state.forward_batch
|
||||
)
|
||||
|
||||
def op_comm_postprocess_layer(self, state):
|
||||
hidden_states, residual = self.layer_communicator.postprocess_layer(
|
||||
state.pop("hidden_states_mlp_output"),
|
||||
state.pop("residual_after_comm_pre_mlp"),
|
||||
state.forward_batch,
|
||||
hidden_states = self.ffn_stage.postprocess(
|
||||
state.pop("hidden_states_mlp_output"), state.forward_batch
|
||||
)
|
||||
|
||||
output = dict(
|
||||
positions=state.positions,
|
||||
hidden_states=hidden_states,
|
||||
residual=residual,
|
||||
forward_batch=state.forward_batch,
|
||||
tbo_subbatch_index=state.tbo_subbatch_index,
|
||||
)
|
||||
@@ -1038,12 +1032,12 @@ class MiMoV2Model(nn.Module):
|
||||
hidden_states = self.embed_tokens(input_ids)
|
||||
else:
|
||||
hidden_states = input_embeds
|
||||
residual = residual_batch.start(forward_batch)
|
||||
residual_batch.start(forward_batch)
|
||||
else:
|
||||
assert pp_proxy_tensors is not None
|
||||
hidden_states, residual = self.layers[
|
||||
self.start_layer
|
||||
].layer_communicator.from_pp(pp_proxy_tensors, forward_batch)
|
||||
hidden_states = self.layers[self.start_layer].attn_stage.from_pp(
|
||||
pp_proxy_tensors, forward_batch
|
||||
)
|
||||
|
||||
aux_hidden_states = AuxHiddenStatePacker(len(self.layers_to_capture))
|
||||
if forward_batch.can_run_tbo and not self.layers_to_capture:
|
||||
@@ -1053,39 +1047,29 @@ class MiMoV2Model(nn.Module):
|
||||
# skip first layer for TBO when starting from layer 0
|
||||
if self.start_layer == 0:
|
||||
layer = self.layers[0]
|
||||
hidden_states, residual = layer(
|
||||
positions, hidden_states, forward_batch, residual
|
||||
)
|
||||
hidden_states = layer(positions, hidden_states, forward_batch)
|
||||
tbo_start_layer = tbo_start_layer + 1
|
||||
|
||||
hidden_states, residual = model_forward_maybe_tbo(
|
||||
hidden_states = model_forward_stages(
|
||||
layers=self.layers[tbo_start_layer:tbo_end_layer],
|
||||
enable_tbo=True,
|
||||
positions=positions,
|
||||
forward_batch=forward_batch,
|
||||
hidden_states=hidden_states,
|
||||
residual=residual,
|
||||
)
|
||||
else:
|
||||
for i in range(self.start_layer, self.end_layer):
|
||||
layer = self.layers[i]
|
||||
hidden_states, residual = layer(
|
||||
hidden_states = layer(
|
||||
positions,
|
||||
hidden_states,
|
||||
forward_batch,
|
||||
residual,
|
||||
captured_last_layer_outputs=(
|
||||
aux_hidden_states if i in self.layers_to_capture else None
|
||||
),
|
||||
captured_last_layer_outputs=aux_hidden_states
|
||||
if i in self.layers_to_capture
|
||||
else None,
|
||||
)
|
||||
|
||||
last_layer = self.layers[self.end_layer - 1]
|
||||
hidden_states, residual = last_layer.layer_communicator.finish_layer_stack(
|
||||
hidden_states,
|
||||
residual,
|
||||
forward_batch,
|
||||
preserve_declared=not self.pp_group.is_last_rank,
|
||||
)
|
||||
hidden_states = residual_batch.finish(hidden_states, forward_batch)
|
||||
|
||||
# A draft targeting the final layer ("after layer
|
||||
# num_hidden_layers-1") maps to capture index num_hidden_layers,
|
||||
@@ -1095,27 +1079,21 @@ class MiMoV2Model(nn.Module):
|
||||
and self.config.num_hidden_layers in self.layers_to_capture
|
||||
):
|
||||
aux_hidden_states.append(
|
||||
last_layer.layer_communicator.snapshot(hidden_states, residual)
|
||||
residual_batch.snapshot(hidden_states, forward_batch)
|
||||
)
|
||||
|
||||
hidden_states_before_norm = None
|
||||
if not self.pp_group.is_last_rank:
|
||||
return PPProxyTensors(
|
||||
{
|
||||
"hidden_states": hidden_states,
|
||||
"residual": residual,
|
||||
}
|
||||
)
|
||||
return residual_batch.to_pp(hidden_states, forward_batch)
|
||||
else:
|
||||
if hidden_states.shape[0] > 0:
|
||||
if forward_batch.return_hidden_states_before_norm:
|
||||
hidden_states_before_norm = last_layer.layer_communicator.snapshot(
|
||||
hidden_states, residual
|
||||
hidden_states_before_norm = residual_batch.snapshot(
|
||||
hidden_states, forward_batch
|
||||
)
|
||||
if residual is None:
|
||||
hidden_states = self.norm(hidden_states)
|
||||
else:
|
||||
hidden_states, _ = self.norm(hidden_states, residual)
|
||||
hidden_states = residual_batch.norm(
|
||||
hidden_states, forward_batch, self.norm
|
||||
)
|
||||
|
||||
if len(aux_hidden_states) == 0:
|
||||
return hidden_states, hidden_states_before_norm
|
||||
|
||||
@@ -22,9 +22,10 @@ from transformers import PretrainedConfig
|
||||
from sglang.srt.configs.model_config import get_mimo_v2_fused_qkv_expected_tp_size
|
||||
from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder
|
||||
from sglang.srt.layers.communicator import (
|
||||
LayerCommunicator,
|
||||
LayerFacts,
|
||||
declare_attn,
|
||||
declare_ffn,
|
||||
enable_moe_dense_fully_dp,
|
||||
make_stages,
|
||||
)
|
||||
from sglang.srt.layers.communicator.residual import batch as residual_batch
|
||||
from sglang.srt.layers.dp_attention import (
|
||||
@@ -117,18 +118,22 @@ class MiMoV2MTPLayer(nn.Module):
|
||||
self.post_attention_layernorm = RMSNorm(
|
||||
config.hidden_size, eps=config.layernorm_epsilon
|
||||
)
|
||||
self.layer_facts = LayerFacts.init_new(
|
||||
layer_id=layer_id,
|
||||
num_layers=1,
|
||||
is_layer_sparse=self.is_layer_sparse,
|
||||
is_previous_layer_sparse=is_previous_layer_sparse,
|
||||
is_next_layer_sparse=is_next_layer_sparse,
|
||||
)
|
||||
self.layer_communicator = LayerCommunicator(
|
||||
layer_facts=self.layer_facts,
|
||||
input_layernorm=self.input_layernorm,
|
||||
post_attention_layernorm=self.post_attention_layernorm,
|
||||
allow_deferred_ffn_reduction=False,
|
||||
|
||||
self.attn_stage, self.ffn_stage = make_stages(
|
||||
(declare_attn(), self.input_layernorm),
|
||||
(
|
||||
declare_ffn(
|
||||
sparse=self.is_layer_sparse,
|
||||
next_sparse=is_next_layer_sparse,
|
||||
),
|
||||
self.post_attention_layernorm,
|
||||
),
|
||||
previous=declare_ffn(
|
||||
sparse=is_previous_layer_sparse, next_sparse=self.is_layer_sparse
|
||||
)
|
||||
if layer_id != 0
|
||||
else None,
|
||||
terminal=layer_id == 1 - 1,
|
||||
)
|
||||
|
||||
def forward(
|
||||
@@ -136,12 +141,9 @@ class MiMoV2MTPLayer(nn.Module):
|
||||
positions: torch.Tensor,
|
||||
hidden_states: torch.Tensor,
|
||||
forward_batch: ForwardBatch,
|
||||
residual: Optional[torch.Tensor],
|
||||
) -> Tuple[torch.Tensor, torch.Tensor]:
|
||||
) -> torch.Tensor:
|
||||
|
||||
hidden_states, residual = self.layer_communicator.prepare_attn(
|
||||
hidden_states, residual, forward_batch
|
||||
)
|
||||
hidden_states = self.attn_stage.prepare(hidden_states, forward_batch)
|
||||
|
||||
if hidden_states.shape[0] != 0:
|
||||
hidden_states = self.self_attn(
|
||||
@@ -150,17 +152,16 @@ class MiMoV2MTPLayer(nn.Module):
|
||||
forward_batch=forward_batch,
|
||||
)
|
||||
|
||||
hidden_states, residual = self.layer_communicator.prepare_mlp(
|
||||
hidden_states, residual, forward_batch
|
||||
)
|
||||
hidden_states = self.attn_stage.finish(hidden_states, forward_batch)
|
||||
hidden_states = self.ffn_stage.prepare(hidden_states, forward_batch)
|
||||
with (
|
||||
self.layer_communicator.ffn_exit(forward_batch) as ffn_exit,
|
||||
self.ffn_stage.exit(forward_batch) as ffn_exit,
|
||||
get_global_expert_distribution_recorder().disable_this_region(),
|
||||
):
|
||||
hidden_states = self.mlp(hidden_states)
|
||||
hidden_states, residual = ffn_exit.finish(hidden_states, residual)
|
||||
hidden_states = ffn_exit.finish(hidden_states)
|
||||
|
||||
return hidden_states, residual
|
||||
return hidden_states
|
||||
|
||||
|
||||
class MiMoV2ModelNextN(nn.Module):
|
||||
@@ -220,26 +221,23 @@ class MiMoV2ModelNextN(nn.Module):
|
||||
dim=-1,
|
||||
)
|
||||
)
|
||||
hidden_states, residual = self.mtp_block(
|
||||
residual_batch.start(forward_batch)
|
||||
hidden_states = self.mtp_block(
|
||||
positions=positions,
|
||||
hidden_states=hidden_states,
|
||||
forward_batch=forward_batch,
|
||||
residual=residual_batch.start(forward_batch),
|
||||
)
|
||||
|
||||
hidden_states, residual = self.mtp_block.layer_communicator.finish_layer_stack(
|
||||
hidden_states, residual, forward_batch
|
||||
)
|
||||
hidden_states = residual_batch.finish(hidden_states, forward_batch)
|
||||
hidden_states_before_norm = None
|
||||
if not forward_batch.forward_mode.is_idle():
|
||||
if forward_batch.return_hidden_states_before_norm:
|
||||
hidden_states_before_norm = self.mtp_block.layer_communicator.snapshot(
|
||||
hidden_states, residual
|
||||
hidden_states_before_norm = residual_batch.snapshot(
|
||||
hidden_states, forward_batch
|
||||
)
|
||||
if residual is not None:
|
||||
hidden_states, _ = self.final_layernorm(hidden_states, residual)
|
||||
else:
|
||||
hidden_states = self.final_layernorm(hidden_states)
|
||||
hidden_states = residual_batch.norm(
|
||||
hidden_states, forward_batch, self.final_layernorm
|
||||
)
|
||||
|
||||
return hidden_states, hidden_states_before_norm
|
||||
|
||||
|
||||
@@ -27,7 +27,7 @@ from torch import nn
|
||||
from transformers import PretrainedConfig
|
||||
|
||||
from sglang.kernels.kernel_api_logging import debug_kernel_api
|
||||
from sglang.srt.batch_overlap.two_batch_overlap import model_forward_maybe_tbo
|
||||
from sglang.srt.batch_overlap.two_batch_overlap import model_forward_stages
|
||||
from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder
|
||||
from sglang.srt.eplb.expert_location_dispatch import ExpertLocationDispatchInfo
|
||||
from sglang.srt.layers.aux_hidden_states import (
|
||||
@@ -35,8 +35,9 @@ from sglang.srt.layers.aux_hidden_states import (
|
||||
AuxHiddenStateList,
|
||||
)
|
||||
from sglang.srt.layers.communicator import (
|
||||
LayerCommunicator,
|
||||
LayerFacts,
|
||||
declare_attn,
|
||||
declare_ffn,
|
||||
make_stages,
|
||||
)
|
||||
from sglang.srt.layers.communicator.residual import batch as residual_batch
|
||||
from sglang.srt.layers.dp_attention import (
|
||||
@@ -980,19 +981,22 @@ class MiniMaxM2DecoderLayer(nn.Module):
|
||||
|
||||
is_previous_layer_sparse = True
|
||||
is_next_layer_sparse = True
|
||||
self.layer_facts = LayerFacts.init_new(
|
||||
layer_id=layer_id,
|
||||
num_layers=config.num_hidden_layers,
|
||||
is_layer_sparse=self.is_layer_sparse,
|
||||
is_previous_layer_sparse=is_previous_layer_sparse,
|
||||
is_next_layer_sparse=is_next_layer_sparse,
|
||||
)
|
||||
|
||||
self.layer_communicator = LayerCommunicator(
|
||||
layer_facts=self.layer_facts,
|
||||
input_layernorm=self.input_layernorm,
|
||||
post_attention_layernorm=self.post_attention_layernorm,
|
||||
allow_reduce_scatter=True,
|
||||
self.attn_stage, self.ffn_stage = make_stages(
|
||||
(declare_attn(), self.input_layernorm),
|
||||
(
|
||||
declare_ffn(
|
||||
sparse=self.is_layer_sparse,
|
||||
next_sparse=is_next_layer_sparse,
|
||||
),
|
||||
self.post_attention_layernorm,
|
||||
),
|
||||
previous=declare_ffn(
|
||||
sparse=is_previous_layer_sparse, next_sparse=self.is_layer_sparse
|
||||
)
|
||||
if layer_id != 0
|
||||
else None,
|
||||
terminal=layer_id == config.num_hidden_layers - 1,
|
||||
)
|
||||
|
||||
def forward(
|
||||
@@ -1000,17 +1004,13 @@ class MiniMaxM2DecoderLayer(nn.Module):
|
||||
positions: torch.Tensor,
|
||||
hidden_states: torch.Tensor,
|
||||
forward_batch: ForwardBatch,
|
||||
residual: Optional[torch.Tensor],
|
||||
captured_last_layer_outputs: Optional[AuxHiddenStateAccumulator] = None,
|
||||
) -> torch.Tensor:
|
||||
# Self Attention
|
||||
hidden_states, residual = (
|
||||
self.layer_communicator.prepare_attn_and_capture_last_layer_outputs(
|
||||
hidden_states,
|
||||
residual,
|
||||
forward_batch,
|
||||
captured_last_layer_outputs=captured_last_layer_outputs,
|
||||
)
|
||||
hidden_states = self.attn_stage.prepare(
|
||||
hidden_states,
|
||||
forward_batch,
|
||||
captured_last_layer_outputs=captured_last_layer_outputs,
|
||||
)
|
||||
if not forward_batch.forward_mode.is_idle():
|
||||
hidden_states = self.self_attn(
|
||||
@@ -1021,15 +1021,14 @@ class MiniMaxM2DecoderLayer(nn.Module):
|
||||
|
||||
# Fully Connected (MLP or MoE)
|
||||
|
||||
hidden_states, residual = self.layer_communicator.prepare_mlp(
|
||||
hidden_states, residual, forward_batch
|
||||
)
|
||||
hidden_states = self.attn_stage.finish(hidden_states, forward_batch)
|
||||
hidden_states = self.ffn_stage.prepare(hidden_states, forward_batch)
|
||||
|
||||
with self.layer_communicator.ffn_exit(forward_batch) as ffn_exit:
|
||||
with self.ffn_stage.exit(forward_batch) as ffn_exit:
|
||||
hidden_states = self.block_sparse_moe(hidden_states, forward_batch)
|
||||
hidden_states, residual = ffn_exit.finish(hidden_states, residual)
|
||||
hidden_states = ffn_exit.finish(hidden_states)
|
||||
|
||||
return hidden_states, residual
|
||||
return hidden_states
|
||||
|
||||
# TBO Operations for MiniMax Decoder Layer
|
||||
def op_comm_prepare_attn(
|
||||
@@ -1038,13 +1037,12 @@ class MiniMaxM2DecoderLayer(nn.Module):
|
||||
positions: torch.Tensor,
|
||||
hidden_states: torch.Tensor,
|
||||
forward_batch: ForwardBatch,
|
||||
residual: Optional[torch.Tensor],
|
||||
zero_allocator: BumpAllocator,
|
||||
tbo_subbatch_index: Optional[int] = None,
|
||||
):
|
||||
"""Communication prepare for attention - TBO operation"""
|
||||
state.hidden_states_after_comm_pre_attn, state.residual_after_input_ln = (
|
||||
self.layer_communicator.prepare_attn(hidden_states, residual, forward_batch)
|
||||
state.hidden_states_after_comm_pre_attn = self.attn_stage.prepare(
|
||||
hidden_states, forward_batch
|
||||
)
|
||||
state.update(
|
||||
dict(
|
||||
@@ -1057,26 +1055,22 @@ class MiniMaxM2DecoderLayer(nn.Module):
|
||||
|
||||
def op_comm_prepare_mlp(self, state):
|
||||
"""Communication prepare for MLP - TBO operation"""
|
||||
state.hidden_states_mlp_input, state.residual_after_comm_pre_mlp = (
|
||||
self.layer_communicator.prepare_mlp(
|
||||
state.pop("hidden_states_after_attn"),
|
||||
state.pop("residual_after_input_ln"),
|
||||
state.forward_batch,
|
||||
)
|
||||
hidden_states = self.attn_stage.finish(
|
||||
state.pop("hidden_states_after_attn"), state.forward_batch
|
||||
)
|
||||
state.hidden_states_mlp_input = self.ffn_stage.prepare(
|
||||
hidden_states, state.forward_batch
|
||||
)
|
||||
|
||||
def op_comm_postprocess_layer(self, state):
|
||||
"""Communication postprocess for layer - TBO operation"""
|
||||
hidden_states, residual = self.layer_communicator.postprocess_layer(
|
||||
state.pop("hidden_states_mlp_output"),
|
||||
state.pop("residual_after_comm_pre_mlp"),
|
||||
state.forward_batch,
|
||||
hidden_states = self.ffn_stage.postprocess(
|
||||
state.pop("hidden_states_mlp_output"), state.forward_batch
|
||||
)
|
||||
|
||||
output = dict(
|
||||
positions=state.positions,
|
||||
hidden_states=hidden_states,
|
||||
residual=residual,
|
||||
forward_batch=state.forward_batch,
|
||||
zero_allocator=state.zero_allocator,
|
||||
tbo_subbatch_index=state.tbo_subbatch_index,
|
||||
@@ -1146,22 +1140,21 @@ class MiniMaxM2Model(nn.Module):
|
||||
hidden_states = self.get_input_embeddings(input_ids)
|
||||
else:
|
||||
hidden_states = input_embeds
|
||||
residual = residual_batch.start(forward_batch)
|
||||
residual_batch.start(forward_batch)
|
||||
else:
|
||||
assert pp_proxy_tensors is not None
|
||||
hidden_states, residual = self.layers[
|
||||
self.start_layer
|
||||
].layer_communicator.from_pp(pp_proxy_tensors, forward_batch)
|
||||
hidden_states = self.layers[self.start_layer].attn_stage.from_pp(
|
||||
pp_proxy_tensors, forward_batch
|
||||
)
|
||||
|
||||
aux_hidden_states = AuxHiddenStateList()
|
||||
if forward_batch.can_run_tbo:
|
||||
hidden_states, residual = model_forward_maybe_tbo(
|
||||
hidden_states = model_forward_stages(
|
||||
layers=self.layers,
|
||||
enable_tbo=True,
|
||||
positions=positions,
|
||||
forward_batch=forward_batch,
|
||||
hidden_states=hidden_states,
|
||||
residual=residual,
|
||||
)
|
||||
else:
|
||||
for i in range(self.start_layer, self.end_layer):
|
||||
@@ -1172,33 +1165,21 @@ class MiniMaxM2Model(nn.Module):
|
||||
)
|
||||
with ctx:
|
||||
layer = self.layers[i]
|
||||
hidden_states, residual = layer(
|
||||
hidden_states = layer(
|
||||
positions=positions,
|
||||
forward_batch=forward_batch,
|
||||
hidden_states=hidden_states,
|
||||
residual=residual,
|
||||
captured_last_layer_outputs=(
|
||||
aux_hidden_states if i in self.layers_to_capture else None
|
||||
),
|
||||
captured_last_layer_outputs=aux_hidden_states
|
||||
if i in self.layers_to_capture
|
||||
else None,
|
||||
)
|
||||
|
||||
last_layer = self.layers[self.end_layer - 1]
|
||||
hidden_states, residual = last_layer.layer_communicator.finish_layer_stack(
|
||||
hidden_states,
|
||||
residual,
|
||||
forward_batch,
|
||||
preserve_declared=not self.pp_group.is_last_rank,
|
||||
)
|
||||
if not self.pp_group.is_last_rank:
|
||||
return PPProxyTensors(
|
||||
{"hidden_states": hidden_states, "residual": residual}
|
||||
)
|
||||
return residual_batch.to_pp(hidden_states, forward_batch)
|
||||
hidden_states = residual_batch.finish(hidden_states, forward_batch)
|
||||
|
||||
if hidden_states.shape[0] != 0:
|
||||
if residual is not None:
|
||||
hidden_states, _ = self.norm(hidden_states, residual)
|
||||
else:
|
||||
hidden_states = self.norm(hidden_states)
|
||||
hidden_states = residual_batch.norm(hidden_states, forward_batch, self.norm)
|
||||
|
||||
if len(aux_hidden_states) == 0:
|
||||
return hidden_states
|
||||
|
||||
@@ -23,7 +23,7 @@ import torch
|
||||
from torch import nn
|
||||
from transformers import PretrainedConfig
|
||||
|
||||
from sglang.srt.batch_overlap.two_batch_overlap import model_forward_maybe_tbo
|
||||
from sglang.srt.batch_overlap.two_batch_overlap import model_forward_stages
|
||||
from sglang.srt.configs.model_config import (
|
||||
get_minimax_sparse_disable_value_layer_ids,
|
||||
get_minimax_sparse_layer_ids,
|
||||
@@ -40,9 +40,10 @@ from sglang.srt.layers.aux_hidden_states import (
|
||||
AuxHiddenStateList,
|
||||
)
|
||||
from sglang.srt.layers.communicator import (
|
||||
LayerCommunicator,
|
||||
LayerFacts,
|
||||
declare_attn,
|
||||
declare_ffn,
|
||||
enable_moe_dense_fully_dp,
|
||||
make_stages,
|
||||
)
|
||||
from sglang.srt.layers.communicator.residual import batch as residual_batch
|
||||
from sglang.srt.layers.dp_attention import (
|
||||
@@ -1302,7 +1303,7 @@ class MiniMaxM3DecoderLayer(nn.Module):
|
||||
|
||||
moe_layer_freq = getattr(config, "moe_layer_freq", None)
|
||||
# Means "MLP is a sparse MoE", not attention sparsity. Kept as ``is_layer_sparse``
|
||||
# because LayerCommunicator / LayerFacts / other models read this attr.
|
||||
# because model construction and downstream integrations read this attr.
|
||||
self.is_layer_sparse = (
|
||||
moe_layer_freq[layer_id] != 0 if moe_layer_freq is not None else True
|
||||
)
|
||||
@@ -1352,19 +1353,22 @@ class MiniMaxM3DecoderLayer(nn.Module):
|
||||
|
||||
is_previous_layer_sparse = _is_layer_sparse(layer_id - 1)
|
||||
is_next_layer_sparse = _is_layer_sparse(layer_id + 1)
|
||||
self.layer_facts = LayerFacts.init_new(
|
||||
layer_id=layer_id,
|
||||
num_layers=config.num_hidden_layers,
|
||||
is_layer_sparse=self.is_layer_sparse,
|
||||
is_previous_layer_sparse=is_previous_layer_sparse,
|
||||
is_next_layer_sparse=is_next_layer_sparse,
|
||||
)
|
||||
|
||||
self.layer_communicator = LayerCommunicator(
|
||||
layer_facts=self.layer_facts,
|
||||
input_layernorm=self.input_layernorm,
|
||||
post_attention_layernorm=self.post_attention_layernorm,
|
||||
allow_reduce_scatter=True,
|
||||
self.attn_stage, self.ffn_stage = make_stages(
|
||||
(declare_attn(), self.input_layernorm),
|
||||
(
|
||||
declare_ffn(
|
||||
sparse=self.is_layer_sparse,
|
||||
next_sparse=is_next_layer_sparse,
|
||||
),
|
||||
self.post_attention_layernorm,
|
||||
),
|
||||
previous=declare_ffn(
|
||||
sparse=is_previous_layer_sparse, next_sparse=self.is_layer_sparse
|
||||
)
|
||||
if layer_id != 0
|
||||
else None,
|
||||
terminal=layer_id == config.num_hidden_layers - 1,
|
||||
)
|
||||
|
||||
def forward(
|
||||
@@ -1372,18 +1376,14 @@ class MiniMaxM3DecoderLayer(nn.Module):
|
||||
positions: torch.Tensor,
|
||||
hidden_states: torch.Tensor,
|
||||
forward_batch: ForwardBatch,
|
||||
residual: Optional[torch.Tensor],
|
||||
captured_last_layer_outputs: Optional[AuxHiddenStateAccumulator] = None,
|
||||
**kwargs,
|
||||
) -> torch.Tensor:
|
||||
hidden_states, residual = (
|
||||
self.layer_communicator.prepare_attn_and_capture_last_layer_outputs(
|
||||
hidden_states,
|
||||
residual,
|
||||
forward_batch,
|
||||
captured_last_layer_outputs=captured_last_layer_outputs,
|
||||
**kwargs,
|
||||
)
|
||||
hidden_states = self.attn_stage.prepare(
|
||||
hidden_states,
|
||||
forward_batch,
|
||||
captured_last_layer_outputs=captured_last_layer_outputs,
|
||||
**kwargs,
|
||||
)
|
||||
|
||||
if hidden_states.shape[0] != 0:
|
||||
@@ -1393,11 +1393,10 @@ class MiniMaxM3DecoderLayer(nn.Module):
|
||||
forward_batch=forward_batch,
|
||||
)
|
||||
|
||||
hidden_states, residual = self.layer_communicator.prepare_mlp(
|
||||
hidden_states, residual, forward_batch
|
||||
)
|
||||
hidden_states = self.attn_stage.finish(hidden_states, forward_batch)
|
||||
hidden_states = self.ffn_stage.prepare(hidden_states, forward_batch)
|
||||
|
||||
with self.layer_communicator.ffn_exit(forward_batch) as ffn_exit:
|
||||
with self.ffn_stage.exit(forward_batch) as ffn_exit:
|
||||
if self.is_layer_sparse or hidden_states.shape[0] != 0:
|
||||
hidden_states = self.mlp(
|
||||
hidden_states,
|
||||
@@ -1405,7 +1404,7 @@ class MiniMaxM3DecoderLayer(nn.Module):
|
||||
should_allreduce_fusion=ffn_exit.fuse_mlp_allreduce,
|
||||
use_reduce_scatter=ffn_exit.mlp_reduce_scatter,
|
||||
)
|
||||
return ffn_exit.finish(hidden_states, residual)
|
||||
return ffn_exit.finish(hidden_states)
|
||||
|
||||
|
||||
class MiniMaxM3Model(nn.Module):
|
||||
@@ -1481,22 +1480,21 @@ class MiniMaxM3Model(nn.Module):
|
||||
hidden_states = embeds(input_ids)
|
||||
else:
|
||||
hidden_states = input_embeds
|
||||
residual = residual_batch.start(forward_batch)
|
||||
residual_batch.start(forward_batch)
|
||||
else:
|
||||
assert pp_proxy_tensors is not None
|
||||
hidden_states, residual = self.layers[
|
||||
self.start_layer
|
||||
].layer_communicator.from_pp(pp_proxy_tensors, forward_batch)
|
||||
hidden_states = self.layers[self.start_layer].attn_stage.from_pp(
|
||||
pp_proxy_tensors, forward_batch
|
||||
)
|
||||
|
||||
aux_hidden_states = AuxHiddenStateList()
|
||||
if forward_batch.can_run_tbo:
|
||||
hidden_states, residual = model_forward_maybe_tbo(
|
||||
hidden_states = model_forward_stages(
|
||||
layers=self.layers,
|
||||
enable_tbo=True,
|
||||
positions=positions,
|
||||
forward_batch=forward_batch,
|
||||
hidden_states=hidden_states,
|
||||
residual=residual,
|
||||
)
|
||||
else:
|
||||
for i in range(self.start_layer, self.end_layer):
|
||||
@@ -1508,34 +1506,20 @@ class MiniMaxM3Model(nn.Module):
|
||||
)
|
||||
with ctx:
|
||||
layer = self.layers[i]
|
||||
hidden_states, residual = layer(
|
||||
hidden_states = layer(
|
||||
positions=positions,
|
||||
forward_batch=forward_batch,
|
||||
hidden_states=hidden_states,
|
||||
residual=residual,
|
||||
captured_last_layer_outputs=(
|
||||
aux_hidden_states
|
||||
if getattr(layer, "_is_layer_to_capture", False)
|
||||
else None
|
||||
),
|
||||
captured_last_layer_outputs=aux_hidden_states
|
||||
if getattr(layer, "_is_layer_to_capture", False)
|
||||
else None,
|
||||
)
|
||||
|
||||
last_layer = self.layers[self.end_layer - 1]
|
||||
hidden_states, residual = last_layer.layer_communicator.finish_layer_stack(
|
||||
hidden_states,
|
||||
residual,
|
||||
forward_batch,
|
||||
preserve_declared=not self.pp_group.is_last_rank,
|
||||
)
|
||||
if not self.pp_group.is_last_rank:
|
||||
return PPProxyTensors(
|
||||
{"hidden_states": hidden_states, "residual": residual}
|
||||
)
|
||||
return residual_batch.to_pp(hidden_states, forward_batch)
|
||||
hidden_states = residual_batch.finish(hidden_states, forward_batch)
|
||||
if hidden_states.shape[0] != 0:
|
||||
if residual is not None:
|
||||
hidden_states, _ = self.norm(hidden_states, residual)
|
||||
else:
|
||||
hidden_states = self.norm(hidden_states)
|
||||
hidden_states = residual_batch.norm(hidden_states, forward_batch, self.norm)
|
||||
|
||||
if len(aux_hidden_states) == 0:
|
||||
return hidden_states
|
||||
|
||||
@@ -22,7 +22,11 @@ from sglang.srt.layers.attention.vision import (
|
||||
VisionAttentionMetadata,
|
||||
prepare_vision_attention_metadata,
|
||||
)
|
||||
from sglang.srt.layers.communicator import LayerCommunicator, LayerFacts
|
||||
from sglang.srt.layers.communicator import (
|
||||
declare_attn,
|
||||
declare_ffn,
|
||||
make_stages,
|
||||
)
|
||||
from sglang.srt.layers.communicator.residual import batch as residual_batch
|
||||
from sglang.srt.layers.conv import Conv3dLayer
|
||||
from sglang.srt.layers.layernorm import RMSNorm
|
||||
@@ -1012,18 +1016,15 @@ class MossVLSelfAttentionDecoderLayer(nn.Module):
|
||||
self.post_attention_layernorm = RMSNorm(
|
||||
config.hidden_size, eps=config.rms_norm_eps, **norm_kwargs
|
||||
)
|
||||
self.layer_facts = LayerFacts.init_new(
|
||||
layer_id=layer_id,
|
||||
num_layers=config.num_hidden_layers,
|
||||
is_layer_sparse=False,
|
||||
is_previous_layer_sparse=False,
|
||||
is_next_layer_sparse=False,
|
||||
)
|
||||
self.layer_communicator = LayerCommunicator(
|
||||
layer_facts=self.layer_facts,
|
||||
input_layernorm=self.input_layernorm,
|
||||
post_attention_layernorm=self.post_attention_layernorm,
|
||||
allow_deferred_ffn_reduction=False,
|
||||
|
||||
self.attn_stage, self.ffn_stage = make_stages(
|
||||
(declare_attn(), self.input_layernorm),
|
||||
(
|
||||
declare_ffn(sparse=False, next_sparse=False),
|
||||
self.post_attention_layernorm,
|
||||
),
|
||||
previous=declare_ffn() if layer_id != 0 else None,
|
||||
terminal=layer_id == config.num_hidden_layers - 1,
|
||||
)
|
||||
|
||||
def forward(
|
||||
@@ -1031,12 +1032,9 @@ class MossVLSelfAttentionDecoderLayer(nn.Module):
|
||||
positions: torch.Tensor,
|
||||
hidden_states: torch.Tensor,
|
||||
forward_batch: ForwardBatch,
|
||||
residual: Optional[torch.Tensor],
|
||||
) -> Tuple[torch.Tensor, torch.Tensor]:
|
||||
) -> torch.Tensor:
|
||||
# Self Attention
|
||||
hidden_states, residual = self.layer_communicator.prepare_attn(
|
||||
hidden_states, residual, forward_batch
|
||||
)
|
||||
hidden_states = self.attn_stage.prepare(hidden_states, forward_batch)
|
||||
if hidden_states.shape[0] != 0:
|
||||
hidden_states = self.self_attn(
|
||||
positions=positions,
|
||||
@@ -1045,15 +1043,12 @@ class MossVLSelfAttentionDecoderLayer(nn.Module):
|
||||
)
|
||||
|
||||
# MLP
|
||||
hidden_states, residual = self.layer_communicator.prepare_mlp(
|
||||
hidden_states,
|
||||
residual,
|
||||
forward_batch,
|
||||
)
|
||||
with self.layer_communicator.ffn_exit(forward_batch) as ffn_exit:
|
||||
hidden_states = self.attn_stage.finish(hidden_states, forward_batch)
|
||||
hidden_states = self.ffn_stage.prepare(hidden_states, forward_batch)
|
||||
with self.ffn_stage.exit(forward_batch) as ffn_exit:
|
||||
hidden_states = self.mlp(hidden_states)
|
||||
hidden_states, residual = ffn_exit.finish(hidden_states, residual)
|
||||
return hidden_states, residual
|
||||
hidden_states = ffn_exit.finish(hidden_states)
|
||||
return hidden_states
|
||||
|
||||
|
||||
# ==================== Text Model ====================
|
||||
@@ -1110,14 +1105,12 @@ class MossVLTextModel(nn.Module):
|
||||
vision_position_ids: Optional[torch.Tensor] = None,
|
||||
) -> torch.Tensor:
|
||||
hidden_states = self.embed_tokens(input_ids)
|
||||
residual = residual_batch.start(forward_batch)
|
||||
residual_batch.start(forward_batch)
|
||||
|
||||
for decoder_layer in self.layers:
|
||||
if isinstance(decoder_layer, MossVLCrossAttentionDecoderLayer):
|
||||
if not skip_cross_attention:
|
||||
hidden_states, residual = LayerCommunicator.fold(
|
||||
hidden_states, residual
|
||||
)
|
||||
hidden_states = residual_batch.fold(hidden_states, forward_batch)
|
||||
hidden_states = decoder_layer(
|
||||
hidden_states=hidden_states,
|
||||
cross_attention_states=cross_attention_states,
|
||||
@@ -1127,27 +1120,17 @@ class MossVLTextModel(nn.Module):
|
||||
positions=positions,
|
||||
vision_position_ids=vision_position_ids,
|
||||
)
|
||||
hidden_states, residual = LayerCommunicator.written(
|
||||
hidden_states, forward_batch
|
||||
)
|
||||
hidden_states = residual_batch.written(hidden_states, forward_batch)
|
||||
elif isinstance(decoder_layer, MossVLSelfAttentionDecoderLayer):
|
||||
hidden_states, residual = decoder_layer(
|
||||
hidden_states = decoder_layer(
|
||||
positions=positions,
|
||||
hidden_states=hidden_states,
|
||||
forward_batch=forward_batch,
|
||||
residual=residual,
|
||||
)
|
||||
else:
|
||||
raise ValueError(f"Unknown decoder layer type {type(decoder_layer)}")
|
||||
|
||||
hidden_states, residual = LayerCommunicator.finish_layer_stack(
|
||||
hidden_states, residual, forward_batch
|
||||
)
|
||||
|
||||
if residual is not None:
|
||||
hidden_states, _ = self.norm(hidden_states, residual)
|
||||
else:
|
||||
hidden_states = self.norm(hidden_states)
|
||||
hidden_states = residual_batch.norm(hidden_states, forward_batch, self.norm)
|
||||
return hidden_states
|
||||
|
||||
|
||||
|
||||
@@ -738,7 +738,7 @@ class NemotronHModel(nn.Module):
|
||||
for i in range(self.start_layer, self.end_layer):
|
||||
if i in self.layers_to_capture:
|
||||
hidden_states, snapshot = self.layers[i].stage.capture_output(
|
||||
hidden_states, forward_batch, at_input=True
|
||||
hidden_states, forward_batch
|
||||
)
|
||||
aux_hidden_states.append(snapshot)
|
||||
layer = self.layers[i]
|
||||
|
||||
@@ -4,22 +4,12 @@ from typing import Optional
|
||||
|
||||
from sglang.srt.configs.nemotron_h import ATTENTION, MAMBA, MOE
|
||||
from sglang.srt.layers.communicator import (
|
||||
NORM_QUANT_READ,
|
||||
LayerCommunicator,
|
||||
LayerFacts,
|
||||
LayerStage,
|
||||
Layout,
|
||||
StageDecl,
|
||||
StageInput,
|
||||
StageKind,
|
||||
StageOutput,
|
||||
SumGroup,
|
||||
TokenAxis,
|
||||
stage_edges,
|
||||
token_axis_sizes,
|
||||
declare_attn,
|
||||
declare_ffn,
|
||||
make_stages,
|
||||
)
|
||||
from sglang.srt.layers.layernorm import RMSNorm
|
||||
from sglang.srt.layers.moe.utils import get_moe_a2a_backend
|
||||
|
||||
ATTN_LAYERS = (MAMBA, ATTENTION)
|
||||
|
||||
@@ -37,87 +27,35 @@ def _stage_kind(pattern: str, layer_idx: int) -> Optional[StageKind]:
|
||||
return StageKind.ATTENTION if is_attn_layer(pattern[layer_idx]) else StageKind.FFN
|
||||
|
||||
|
||||
def _stage_decl(pattern: str, layer_idx: int) -> StageDecl:
|
||||
"""What the stage at ``layer_idx`` declares. A mixer reads its input as
|
||||
an attention does, computes on the attention's rows and leaves its
|
||||
attention-TP sum to an FFN stage after it, or, when a fused kernel takes
|
||||
it, to a mixer after it. An FFN computes on the TP group's rows, a MoE
|
||||
dispatched by an a2a backend on this rank's own, and may leave its sum to a
|
||||
mixer after it."""
|
||||
axis_sizes = token_axis_sizes()
|
||||
attention = Layout.sharded_over(
|
||||
TokenAxis.ATTN_DP, TokenAxis.ATTN_CP, axis_sizes=axis_sizes
|
||||
)
|
||||
def _declaration(pattern: str, layer_idx: int):
|
||||
following = _stage_kind(pattern, layer_idx + 1)
|
||||
if is_attn_layer(pattern[layer_idx]):
|
||||
owes = axis_sizes[TokenAxis.ATTN_TP_SCATTER] > 1
|
||||
return StageDecl(
|
||||
StageInput(attention, read=NORM_QUANT_READ),
|
||||
StageOutput(
|
||||
attention,
|
||||
group=SumGroup.ATTN_TP if owes else None,
|
||||
always_leaves=owes and following is StageKind.FFN,
|
||||
leaves_for_next_layer=owes and following is StageKind.ATTENTION,
|
||||
),
|
||||
return declare_attn(
|
||||
mixer_exit=True,
|
||||
next_kind=following,
|
||||
ordinary_only=True,
|
||||
)
|
||||
sparse = pattern[layer_idx] == MOE
|
||||
if sparse and not get_moe_a2a_backend().is_none():
|
||||
local = Layout.sharded_over(
|
||||
TokenAxis.ATTN_DP,
|
||||
TokenAxis.ATTN_CP,
|
||||
TokenAxis.ATTN_TP_SCATTER,
|
||||
axis_sizes=axis_sizes,
|
||||
)
|
||||
return StageDecl(StageInput(local), StageOutput(local))
|
||||
full = Layout.sharded_over(axis_sizes=axis_sizes)
|
||||
return StageDecl(
|
||||
StageInput(full),
|
||||
StageOutput(
|
||||
full,
|
||||
group=SumGroup.MOE_OUTPUT if sparse else SumGroup.TP,
|
||||
leaves_for_next_layer=following is StageKind.ATTENTION,
|
||||
leaves_for_reduce_scatter=True,
|
||||
leaves_for_reduce_scatterv=True,
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def layer_stage(pattern: str, layer_idx: int) -> LayerStage:
|
||||
"""The layer's stage and its two boundaries, from its own declaration and
|
||||
the previous stage's."""
|
||||
rows = Layout.sharded_over(
|
||||
TokenAxis.ATTN_DP, TokenAxis.ATTN_CP, axis_sizes=token_axis_sizes()
|
||||
)
|
||||
return LayerStage(
|
||||
kind=_stage_kind(pattern, layer_idx),
|
||||
edges=stage_edges(
|
||||
previous=(
|
||||
_stage_decl(pattern, layer_idx - 1).output if layer_idx > 0 else None
|
||||
),
|
||||
stage=_stage_decl(pattern, layer_idx),
|
||||
rows=rows,
|
||||
),
|
||||
enters_stack=layer_idx == 0,
|
||||
return declare_ffn(
|
||||
sparse=pattern[layer_idx] == MOE,
|
||||
ordinary_only=True,
|
||||
return_to_attention=True,
|
||||
)
|
||||
|
||||
|
||||
def make_stage_boundary(layer_norm: RMSNorm, *, pattern: str, layer_idx: int):
|
||||
"""The communicator of a layer that is one stage: only its own norm, and
|
||||
boundaries built from the stages next to it in the pattern."""
|
||||
stage = layer_stage(pattern, layer_idx)
|
||||
for_attn = stage.kind is StageKind.ATTENTION
|
||||
boundaries = LayerCommunicator(
|
||||
layer_facts=LayerFacts(
|
||||
is_layer_sparse=pattern[layer_idx] == MOE,
|
||||
is_last_layer=layer_idx == len(pattern) - 1,
|
||||
),
|
||||
input_layernorm=layer_norm if for_attn else None,
|
||||
post_attention_layernorm=None if for_attn else layer_norm,
|
||||
# With attention TP > 1, the default gather adds the residual to one
|
||||
# rank's partial in bf16 before the cross-rank sum.
|
||||
force_layernorm_before_dp_gather=True,
|
||||
allow_reduce_scatter=not for_attn,
|
||||
stage=stage,
|
||||
)
|
||||
from sglang.srt.layers import layernorm_sp
|
||||
from sglang.srt.runtime_context import get_parallel
|
||||
|
||||
return boundaries.attn if for_attn else boundaries.ffn
|
||||
if get_parallel().attn_cp_size > 1 or layernorm_sp.layernorm_sp_enabled():
|
||||
raise NotImplementedError("a Nemotron stage with attention CP or LayerNorm SP")
|
||||
previous = _declaration(pattern, layer_idx - 1) if layer_idx > 0 else None
|
||||
(boundary,) = make_stages(
|
||||
(
|
||||
_declaration(pattern, layer_idx),
|
||||
layer_norm,
|
||||
{"force_layernorm_before_dp_gather": True},
|
||||
),
|
||||
previous=previous,
|
||||
terminal=layer_idx == len(pattern) - 1,
|
||||
)
|
||||
return boundary
|
||||
|
||||
@@ -31,7 +31,7 @@ from sglang.kernels.ops.elementwise.elementwise import (
|
||||
fused_gate_sigmoid_mul,
|
||||
fused_gate_sigmoid_mul_add,
|
||||
)
|
||||
from sglang.srt.batch_overlap.two_batch_overlap import model_forward_maybe_tbo
|
||||
from sglang.srt.batch_overlap.two_batch_overlap import model_forward_stages
|
||||
from sglang.srt.distributed import (
|
||||
get_pp_indices,
|
||||
)
|
||||
@@ -45,8 +45,9 @@ from sglang.srt.layers.aux_hidden_states import (
|
||||
AuxHiddenStateList,
|
||||
)
|
||||
from sglang.srt.layers.communicator import (
|
||||
LayerCommunicator,
|
||||
LayerFacts,
|
||||
declare_attn,
|
||||
declare_ffn,
|
||||
make_stages,
|
||||
)
|
||||
from sglang.srt.layers.communicator.residual import batch as residual_batch
|
||||
from sglang.srt.layers.dp_attention import (
|
||||
@@ -1032,14 +1033,6 @@ class Qwen2MoeDecoderLayer(nn.Module):
|
||||
is_previous_layer_sparse = True
|
||||
is_next_layer_sparse = True
|
||||
|
||||
self.layer_facts = LayerFacts.init_new(
|
||||
layer_id=layer_id,
|
||||
num_layers=config.num_hidden_layers,
|
||||
is_layer_sparse=self.is_layer_sparse,
|
||||
is_previous_layer_sparse=is_previous_layer_sparse,
|
||||
is_next_layer_sparse=is_next_layer_sparse,
|
||||
)
|
||||
|
||||
if self.is_layer_sparse:
|
||||
self.mlp = Qwen2MoeSparseMoeBlock(
|
||||
layer_id=layer_id,
|
||||
@@ -1060,12 +1053,21 @@ class Qwen2MoeDecoderLayer(nn.Module):
|
||||
self.post_attention_layernorm = RMSNorm(
|
||||
config.hidden_size, eps=config.rms_norm_eps
|
||||
)
|
||||
self.layer_communicator = LayerCommunicator(
|
||||
layer_facts=self.layer_facts,
|
||||
input_layernorm=self.input_layernorm,
|
||||
post_attention_layernorm=self.post_attention_layernorm,
|
||||
allow_reduce_scatter=True,
|
||||
allow_deferred_ffn_reduction=False,
|
||||
self.attn_stage, self.ffn_stage = make_stages(
|
||||
(declare_attn(), self.input_layernorm),
|
||||
(
|
||||
declare_ffn(
|
||||
sparse=self.is_layer_sparse,
|
||||
next_sparse=is_next_layer_sparse,
|
||||
),
|
||||
self.post_attention_layernorm,
|
||||
),
|
||||
previous=declare_ffn(
|
||||
sparse=is_previous_layer_sparse, next_sparse=self.is_layer_sparse
|
||||
)
|
||||
if layer_id != 0
|
||||
else None,
|
||||
terminal=layer_id == config.num_hidden_layers - 1,
|
||||
)
|
||||
|
||||
def forward(
|
||||
@@ -1073,19 +1075,15 @@ class Qwen2MoeDecoderLayer(nn.Module):
|
||||
positions: torch.Tensor,
|
||||
hidden_states: torch.Tensor,
|
||||
forward_batch: ForwardBatch,
|
||||
residual: Optional[torch.Tensor],
|
||||
captured_last_layer_outputs: Optional[AuxHiddenStateAccumulator] = None,
|
||||
**kwargs,
|
||||
) -> Tuple[torch.Tensor, torch.Tensor]:
|
||||
) -> torch.Tensor:
|
||||
|
||||
hidden_states, residual = (
|
||||
self.layer_communicator.prepare_attn_and_capture_last_layer_outputs(
|
||||
hidden_states,
|
||||
residual,
|
||||
forward_batch,
|
||||
captured_last_layer_outputs=captured_last_layer_outputs,
|
||||
**kwargs,
|
||||
)
|
||||
hidden_states = self.attn_stage.prepare(
|
||||
hidden_states,
|
||||
forward_batch,
|
||||
captured_last_layer_outputs=captured_last_layer_outputs,
|
||||
**kwargs,
|
||||
)
|
||||
|
||||
if hidden_states.shape[0] != 0:
|
||||
@@ -1095,15 +1093,14 @@ class Qwen2MoeDecoderLayer(nn.Module):
|
||||
forward_batch=forward_batch,
|
||||
)
|
||||
|
||||
hidden_states, residual = self.layer_communicator.prepare_mlp(
|
||||
hidden_states, residual, forward_batch
|
||||
)
|
||||
hidden_states = self.attn_stage.finish(hidden_states, forward_batch)
|
||||
hidden_states = self.ffn_stage.prepare(hidden_states, forward_batch)
|
||||
|
||||
with self.layer_communicator.ffn_exit(forward_batch) as ffn_exit:
|
||||
with self.ffn_stage.exit(forward_batch) as ffn_exit:
|
||||
hidden_states = self.mlp(hidden_states, forward_batch)
|
||||
hidden_states, residual = ffn_exit.finish(hidden_states, residual)
|
||||
hidden_states = ffn_exit.finish(hidden_states)
|
||||
|
||||
return hidden_states, residual
|
||||
return hidden_states
|
||||
|
||||
|
||||
class Qwen2MoeModel(nn.Module):
|
||||
@@ -1180,22 +1177,21 @@ class Qwen2MoeModel(nn.Module):
|
||||
hidden_states = self.embed_tokens(input_ids)
|
||||
else:
|
||||
hidden_states = input_embeds
|
||||
residual = residual_batch.start(forward_batch)
|
||||
residual_batch.start(forward_batch)
|
||||
else:
|
||||
assert pp_proxy_tensors is not None
|
||||
hidden_states, residual = self.layers[
|
||||
self.start_layer
|
||||
].layer_communicator.from_pp(pp_proxy_tensors, forward_batch)
|
||||
hidden_states = self.layers[self.start_layer].attn_stage.from_pp(
|
||||
pp_proxy_tensors, forward_batch
|
||||
)
|
||||
|
||||
aux_hidden_states = AuxHiddenStateList()
|
||||
if forward_batch.can_run_tbo:
|
||||
hidden_states, residual = model_forward_maybe_tbo(
|
||||
hidden_states = model_forward_stages(
|
||||
layers=self.layers,
|
||||
enable_tbo=True,
|
||||
positions=positions,
|
||||
forward_batch=forward_batch,
|
||||
hidden_states=hidden_states,
|
||||
residual=residual,
|
||||
)
|
||||
else:
|
||||
for i in range(self.start_layer, self.end_layer):
|
||||
@@ -1206,38 +1202,23 @@ class Qwen2MoeModel(nn.Module):
|
||||
)
|
||||
with ctx:
|
||||
layer = self.layers[i]
|
||||
hidden_states, residual = layer(
|
||||
hidden_states = layer(
|
||||
positions,
|
||||
hidden_states,
|
||||
forward_batch,
|
||||
residual,
|
||||
captured_last_layer_outputs=(
|
||||
aux_hidden_states
|
||||
if getattr(layer, "_is_layer_to_capture", False)
|
||||
else None
|
||||
),
|
||||
captured_last_layer_outputs=aux_hidden_states
|
||||
if getattr(layer, "_is_layer_to_capture", False)
|
||||
else None,
|
||||
)
|
||||
|
||||
last_layer = self.layers[self.end_layer - 1]
|
||||
hidden_states, residual = last_layer.layer_communicator.finish_layer_stack(
|
||||
hidden_states,
|
||||
residual,
|
||||
forward_batch,
|
||||
preserve_declared=not self.pp_group.is_last_rank,
|
||||
)
|
||||
if not self.pp_group.is_last_rank:
|
||||
return PPProxyTensors(
|
||||
{
|
||||
"hidden_states": hidden_states,
|
||||
"residual": residual,
|
||||
}
|
||||
)
|
||||
return residual_batch.to_pp(hidden_states, forward_batch)
|
||||
else:
|
||||
hidden_states = residual_batch.finish(hidden_states, forward_batch)
|
||||
if hidden_states.shape[0] != 0:
|
||||
if residual is None:
|
||||
hidden_states = self.norm(hidden_states)
|
||||
else:
|
||||
hidden_states, _ = self.norm(hidden_states, residual)
|
||||
hidden_states = residual_batch.norm(
|
||||
hidden_states, forward_batch, self.norm
|
||||
)
|
||||
|
||||
if len(aux_hidden_states) == 0:
|
||||
return hidden_states
|
||||
@@ -1314,7 +1295,7 @@ class Qwen2MoeForCausalLM(nn.Module):
|
||||
start, end = split_interval
|
||||
# embed
|
||||
if start == 0:
|
||||
forward_batch.residual = residual_batch.start(forward_batch)
|
||||
residual_batch.start(forward_batch)
|
||||
if input_embeds is None:
|
||||
forward_batch.hidden_states = self.model.embed_tokens(input_ids)
|
||||
else:
|
||||
@@ -1324,22 +1305,17 @@ class Qwen2MoeForCausalLM(nn.Module):
|
||||
for i in range(start, end):
|
||||
with get_global_expert_distribution_recorder().with_current_layer(i):
|
||||
layer = self.model.layers[i]
|
||||
forward_batch.hidden_states, forward_batch.residual = layer(
|
||||
positions,
|
||||
forward_batch.hidden_states,
|
||||
forward_batch,
|
||||
forward_batch.residual,
|
||||
forward_batch.hidden_states = layer(
|
||||
positions, forward_batch.hidden_states, forward_batch
|
||||
)
|
||||
|
||||
if end == self.model.config.num_hidden_layers:
|
||||
forward_batch.hidden_states, forward_batch.residual = self.model.layers[
|
||||
end - 1
|
||||
].layer_communicator.finish_layer_stack(
|
||||
forward_batch.hidden_states, forward_batch.residual, forward_batch
|
||||
forward_batch.hidden_states = residual_batch.finish(
|
||||
forward_batch.hidden_states, forward_batch
|
||||
)
|
||||
# norm
|
||||
hidden_states, _ = self.model.norm(
|
||||
forward_batch.hidden_states, forward_batch.residual
|
||||
hidden_states = residual_batch.norm(
|
||||
forward_batch.hidden_states, forward_batch, self.model.norm
|
||||
)
|
||||
forward_batch.residual = None
|
||||
forward_batch.hidden_states = hidden_states
|
||||
|
||||
@@ -40,18 +40,17 @@ from sglang.srt.environ import envs
|
||||
from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder
|
||||
from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation
|
||||
from sglang.srt.layers.attention.mamba.mamba import mamba_v2_sharded_weight_loader
|
||||
|
||||
# Layers - Attention
|
||||
from sglang.srt.layers.aux_hidden_states import (
|
||||
AuxHiddenStateAccumulator,
|
||||
AuxHiddenStateList,
|
||||
)
|
||||
from sglang.srt.layers.communicator import (
|
||||
LayerCommunicator,
|
||||
LayerFacts,
|
||||
declare_attn,
|
||||
declare_ffn,
|
||||
make_stages,
|
||||
)
|
||||
|
||||
# Configs
|
||||
# Layers - Attention
|
||||
from sglang.srt.layers.communicator.residual import batch as residual_batch
|
||||
from sglang.srt.layers.dp_attention import (
|
||||
is_dp_attention_enabled,
|
||||
@@ -271,7 +270,7 @@ def _enable_qwen35_fused_ar_quant() -> bool:
|
||||
(AR → RMSNorm → per-group quant) with either a single fused kernel (when
|
||||
the fully-fused variant is eligible) or a 2-kernel path
|
||||
(fused AR+RMSNorm + separate per-group quant) that still saves one
|
||||
kernel launch vs. baseline. The LayerCommunicator gracefully falls back
|
||||
kernel launch vs. baseline. The stage boundary gracefully falls back
|
||||
to ``forward_with_allreduce_fusion`` (plain AR+RMSNorm) when the fused
|
||||
quant helper returns ``None``, so turning this on never regresses the
|
||||
AR+RMSNorm fusion itself.
|
||||
@@ -1109,14 +1108,6 @@ class Qwen3_5LinearDecoderLayer(nn.Module):
|
||||
else:
|
||||
raise ValueError(f"Invalid model type: {config.model_type}")
|
||||
|
||||
self.layer_facts = LayerFacts.init_new(
|
||||
layer_id=layer_id,
|
||||
num_layers=config.num_hidden_layers,
|
||||
is_layer_sparse=is_layer_sparse,
|
||||
is_previous_layer_sparse=is_previous_layer_sparse,
|
||||
is_next_layer_sparse=is_next_layer_sparse,
|
||||
)
|
||||
|
||||
self.input_layernorm = GemmaRMSNorm(config.hidden_size, eps=config.rms_norm_eps)
|
||||
self.post_attention_layernorm = GemmaRMSNorm(
|
||||
config.hidden_size, eps=config.rms_norm_eps
|
||||
@@ -1128,33 +1119,44 @@ class Qwen3_5LinearDecoderLayer(nn.Module):
|
||||
_enable_qwen35_fused_ar_quant()
|
||||
and _linear_accepts_fp8_tuple(self.linear_attn.in_proj_qkvz)
|
||||
)
|
||||
self.layer_communicator = LayerCommunicator(
|
||||
layer_facts=self.layer_facts,
|
||||
input_layernorm=self.input_layernorm,
|
||||
post_attention_layernorm=self.post_attention_layernorm,
|
||||
allow_reduce_scatter=True,
|
||||
enable_fused_ar_quant=enable_fused_ar_quant,
|
||||
fused_ar_quant_keep_bf16=enable_fused_ar_quant,
|
||||
fusions=_layer_fusions(config, is_nextn),
|
||||
boundary_fusions = _layer_fusions(config, is_nextn)
|
||||
self.attn_stage, self.ffn_stage = make_stages(
|
||||
(
|
||||
declare_attn(),
|
||||
self.input_layernorm,
|
||||
{
|
||||
"enable_fused_ar_quant": enable_fused_ar_quant,
|
||||
"fused_ar_quant_keep_bf16": enable_fused_ar_quant,
|
||||
"fusions": boundary_fusions,
|
||||
},
|
||||
),
|
||||
(
|
||||
declare_ffn(
|
||||
sparse=is_layer_sparse,
|
||||
next_sparse=is_next_layer_sparse,
|
||||
),
|
||||
self.post_attention_layernorm,
|
||||
{"fusions": boundary_fusions},
|
||||
),
|
||||
previous=declare_ffn(
|
||||
sparse=is_previous_layer_sparse, next_sparse=is_layer_sparse
|
||||
)
|
||||
if layer_id != 0
|
||||
else None,
|
||||
terminal=layer_id == config.num_hidden_layers - 1,
|
||||
)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
residual: Optional[torch.Tensor],
|
||||
**kwargs,
|
||||
):
|
||||
forward_batch = kwargs.get("forward_batch", None)
|
||||
|
||||
hidden_states, residual = (
|
||||
self.layer_communicator.prepare_attn_and_capture_last_layer_outputs(
|
||||
hidden_states,
|
||||
residual,
|
||||
forward_batch,
|
||||
captured_last_layer_outputs=kwargs.get(
|
||||
"captured_last_layer_outputs", None
|
||||
),
|
||||
)
|
||||
hidden_states = self.attn_stage.prepare(
|
||||
hidden_states,
|
||||
forward_batch,
|
||||
captured_last_layer_outputs=kwargs.get("captured_last_layer_outputs", None),
|
||||
)
|
||||
|
||||
# fused AR+quant hands down a (fp8, scale) / (bf16, fp8, scale) tuple
|
||||
@@ -1166,11 +1168,10 @@ class Qwen3_5LinearDecoderLayer(nn.Module):
|
||||
)
|
||||
|
||||
# Fully Connected
|
||||
hidden_states, residual = self.layer_communicator.prepare_mlp(
|
||||
hidden_states, residual, forward_batch
|
||||
)
|
||||
hidden_states = self.attn_stage.finish(hidden_states, forward_batch)
|
||||
hidden_states = self.ffn_stage.prepare(hidden_states, forward_batch)
|
||||
|
||||
with self.layer_communicator.ffn_exit(forward_batch) as ffn_exit:
|
||||
with self.ffn_stage.exit(forward_batch) as ffn_exit:
|
||||
if isinstance(self.mlp, Qwen2MoeSparseMoeBlock):
|
||||
hidden_states = self.mlp(
|
||||
hidden_states,
|
||||
@@ -1179,7 +1180,7 @@ class Qwen3_5LinearDecoderLayer(nn.Module):
|
||||
)
|
||||
else:
|
||||
hidden_states = self.mlp(hidden_states)
|
||||
return ffn_exit.finish(hidden_states, residual)
|
||||
return ffn_exit.finish(hidden_states)
|
||||
|
||||
|
||||
class Qwen3_5AttentionDecoderLayer(nn.Module):
|
||||
@@ -1311,14 +1312,6 @@ class Qwen3_5AttentionDecoderLayer(nn.Module):
|
||||
else:
|
||||
raise ValueError(f"Invalid model type: {config.model_type}")
|
||||
|
||||
self.layer_facts = LayerFacts.init_new(
|
||||
layer_id=layer_id,
|
||||
num_layers=config.num_hidden_layers,
|
||||
is_layer_sparse=is_layer_sparse,
|
||||
is_previous_layer_sparse=is_previous_layer_sparse,
|
||||
is_next_layer_sparse=is_next_layer_sparse,
|
||||
)
|
||||
|
||||
self.input_layernorm = GemmaRMSNorm(config.hidden_size, eps=config.rms_norm_eps)
|
||||
self.post_attention_layernorm = GemmaRMSNorm(
|
||||
config.hidden_size, eps=config.rms_norm_eps
|
||||
@@ -1332,14 +1325,31 @@ class Qwen3_5AttentionDecoderLayer(nn.Module):
|
||||
enable_fused_ar_quant = (
|
||||
_enable_qwen35_fused_ar_quant() and _linear_accepts_fp8_tuple(self.qkv_proj)
|
||||
)
|
||||
self.layer_communicator = LayerCommunicator(
|
||||
layer_facts=self.layer_facts,
|
||||
input_layernorm=self.input_layernorm,
|
||||
post_attention_layernorm=self.post_attention_layernorm,
|
||||
allow_reduce_scatter=True,
|
||||
enable_fused_ar_quant=enable_fused_ar_quant,
|
||||
fused_ar_quant_keep_bf16=False,
|
||||
fusions=_layer_fusions(config, is_nextn),
|
||||
boundary_fusions = _layer_fusions(config, is_nextn)
|
||||
self.attn_stage, self.ffn_stage = make_stages(
|
||||
(
|
||||
declare_attn(),
|
||||
self.input_layernorm,
|
||||
{
|
||||
"enable_fused_ar_quant": enable_fused_ar_quant,
|
||||
"fused_ar_quant_keep_bf16": False,
|
||||
"fusions": boundary_fusions,
|
||||
},
|
||||
),
|
||||
(
|
||||
declare_ffn(
|
||||
sparse=is_layer_sparse,
|
||||
next_sparse=is_next_layer_sparse,
|
||||
),
|
||||
self.post_attention_layernorm,
|
||||
{"fusions": boundary_fusions},
|
||||
),
|
||||
previous=declare_ffn(
|
||||
sparse=is_previous_layer_sparse, next_sparse=is_layer_sparse
|
||||
)
|
||||
if layer_id != 0
|
||||
else None,
|
||||
terminal=layer_id == config.num_hidden_layers - 1,
|
||||
)
|
||||
|
||||
self.alt_stream = alt_stream
|
||||
@@ -1543,18 +1553,14 @@ class Qwen3_5AttentionDecoderLayer(nn.Module):
|
||||
self,
|
||||
positions: torch.Tensor,
|
||||
hidden_states: torch.Tensor,
|
||||
residual: Optional[torch.Tensor],
|
||||
forward_batch: ForwardBatch,
|
||||
captured_last_layer_outputs: Optional[AuxHiddenStateAccumulator] = None,
|
||||
**kwargs,
|
||||
):
|
||||
hidden_states, residual = (
|
||||
self.layer_communicator.prepare_attn_and_capture_last_layer_outputs(
|
||||
hidden_states,
|
||||
residual,
|
||||
forward_batch,
|
||||
captured_last_layer_outputs=captured_last_layer_outputs,
|
||||
)
|
||||
hidden_states = self.attn_stage.prepare(
|
||||
hidden_states,
|
||||
forward_batch,
|
||||
captured_last_layer_outputs=captured_last_layer_outputs,
|
||||
)
|
||||
|
||||
# fused AR+quant hands down a (fp8, scale) / (bf16, fp8, scale) tuple
|
||||
@@ -1567,10 +1573,9 @@ class Qwen3_5AttentionDecoderLayer(nn.Module):
|
||||
)
|
||||
|
||||
# Fully Connected
|
||||
hidden_states, residual = self.layer_communicator.prepare_mlp(
|
||||
hidden_states, residual, forward_batch
|
||||
)
|
||||
with self.layer_communicator.ffn_exit(forward_batch) as ffn_exit:
|
||||
hidden_states = self.attn_stage.finish(hidden_states, forward_batch)
|
||||
hidden_states = self.ffn_stage.prepare(hidden_states, forward_batch)
|
||||
with self.ffn_stage.exit(forward_batch) as ffn_exit:
|
||||
if isinstance(self.mlp, Qwen2MoeSparseMoeBlock):
|
||||
hidden_states = self.mlp(
|
||||
hidden_states,
|
||||
@@ -1579,7 +1584,7 @@ class Qwen3_5AttentionDecoderLayer(nn.Module):
|
||||
)
|
||||
else:
|
||||
hidden_states = self.mlp(hidden_states)
|
||||
return ffn_exit.finish(hidden_states, residual)
|
||||
return ffn_exit.finish(hidden_states)
|
||||
|
||||
|
||||
ALL_DECODER_LAYER_TYPES = {
|
||||
@@ -1747,6 +1752,7 @@ class Qwen3_5ForCausalLM(nn.Module):
|
||||
# Every layer was checked above.
|
||||
can_defer_finalize=lambda layer: True,
|
||||
label="Qwen3.5",
|
||||
terminal_finalize=True,
|
||||
)
|
||||
|
||||
# Final normalization
|
||||
@@ -1821,12 +1827,12 @@ class Qwen3_5ForCausalLM(nn.Module):
|
||||
hidden_states = self.embed_tokens(input_ids)
|
||||
else:
|
||||
hidden_states = input_embeds
|
||||
residual = residual_batch.start(forward_batch)
|
||||
residual_batch.start(forward_batch)
|
||||
else:
|
||||
assert pp_proxy_tensors is not None
|
||||
hidden_states, residual = self.layers[
|
||||
self.start_layer
|
||||
].layer_communicator.from_pp(pp_proxy_tensors, forward_batch)
|
||||
hidden_states = self.layers[self.start_layer].attn_stage.from_pp(
|
||||
pp_proxy_tensors, forward_batch
|
||||
)
|
||||
|
||||
aux_hidden_states = AuxHiddenStateList()
|
||||
# Pass through decoder layers
|
||||
@@ -1835,16 +1841,13 @@ class Qwen3_5ForCausalLM(nn.Module):
|
||||
with get_global_expert_distribution_recorder().with_current_layer(
|
||||
layer_idx
|
||||
):
|
||||
hidden_states, residual = layer(
|
||||
hidden_states = layer(
|
||||
positions=positions,
|
||||
hidden_states=hidden_states,
|
||||
residual=residual,
|
||||
forward_batch=forward_batch,
|
||||
captured_last_layer_outputs=(
|
||||
aux_hidden_states
|
||||
if getattr(layer, "_is_layer_to_capture", False)
|
||||
else None
|
||||
),
|
||||
captured_last_layer_outputs=aux_hidden_states
|
||||
if getattr(layer, "_is_layer_to_capture", False)
|
||||
else None,
|
||||
)
|
||||
|
||||
# Process deepstack embeddings if provided
|
||||
@@ -1854,52 +1857,22 @@ class Qwen3_5ForCausalLM(nn.Module):
|
||||
and layer_idx < 3
|
||||
):
|
||||
sep = self.hidden_size * layer_idx
|
||||
hidden_states, residual = layer.layer_communicator.add_to_output(
|
||||
hidden_states = residual_batch.add_to_output(
|
||||
hidden_states,
|
||||
residual,
|
||||
forward_batch,
|
||||
input_deepstack_embeds[:, sep : sep + self.hidden_size],
|
||||
)
|
||||
|
||||
last_layer = self.layers[self.end_layer - 1]
|
||||
hidden_states, residual = last_layer.layer_communicator.finish_layer_stack(
|
||||
hidden_states,
|
||||
residual,
|
||||
forward_batch,
|
||||
# The final norm below finalizes a deferred MoE output in its kernel.
|
||||
final_norm_takes_handoff=self.flashinfer_mnnvl_cutedsl_fusion is not None
|
||||
and self.pp_group.is_last_rank,
|
||||
preserve_declared=not self.pp_group.is_last_rank,
|
||||
)
|
||||
|
||||
# Return intermediate tensors for pipeline parallelism
|
||||
if not self.pp_group.is_last_rank:
|
||||
return PPProxyTensors(
|
||||
{
|
||||
"hidden_states": hidden_states,
|
||||
"residual": residual,
|
||||
}
|
||||
)
|
||||
return residual_batch.to_pp(hidden_states, forward_batch)
|
||||
|
||||
# The final layer has no successor to consume its deferred MoE tail.
|
||||
is_deferred_finalize = False
|
||||
if self.flashinfer_mnnvl_cutedsl_fusion is not None:
|
||||
from sglang.srt.layers.communicator.fusions.cutedsl import (
|
||||
MoeFinalizeHandoff,
|
||||
)
|
||||
|
||||
is_deferred_finalize = isinstance(hidden_states, MoeFinalizeHandoff)
|
||||
|
||||
if is_deferred_finalize:
|
||||
if residual is None:
|
||||
raise RuntimeError("invalid final deferred MoE handoff")
|
||||
hidden_states, _ = self.flashinfer_mnnvl_cutedsl_fusion.finalize(
|
||||
handoff=hidden_states, residual=residual, gamma=self.norm.gemma_weight
|
||||
)
|
||||
elif hidden_states.shape[0] != 0:
|
||||
if residual is None:
|
||||
hidden_states = self.norm(hidden_states)
|
||||
else:
|
||||
hidden_states, _ = self.norm(hidden_states, residual)
|
||||
hidden_states = residual_batch.norm(
|
||||
hidden_states,
|
||||
forward_batch,
|
||||
self.norm,
|
||||
handoff_norm=self.flashinfer_mnnvl_cutedsl_fusion,
|
||||
skip_empty=True,
|
||||
)
|
||||
|
||||
if len(aux_hidden_states) == 0:
|
||||
return hidden_states
|
||||
|
||||
@@ -30,7 +30,11 @@ from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_r
|
||||
from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation
|
||||
from sglang.srt.eplb.expert_location_dispatch import ExpertLocationDispatchInfo
|
||||
from sglang.srt.layers.aux_hidden_states import AuxHiddenStateAccumulator
|
||||
from sglang.srt.layers.communicator import LayerCommunicator, LayerFacts
|
||||
from sglang.srt.layers.communicator import (
|
||||
declare_attn,
|
||||
declare_ffn,
|
||||
make_stages,
|
||||
)
|
||||
from sglang.srt.layers.communicator.residual import batch as residual_batch
|
||||
from sglang.srt.layers.layernorm import RMSNorm
|
||||
from sglang.srt.layers.linear import (
|
||||
@@ -842,14 +846,6 @@ class Qwen3MoeDecoderLayer(nn.Module):
|
||||
is_previous_layer_sparse = True
|
||||
is_next_layer_sparse = True
|
||||
|
||||
self.layer_facts = LayerFacts.init_new(
|
||||
layer_id=layer_id,
|
||||
num_layers=config.num_hidden_layers,
|
||||
is_layer_sparse=self.is_layer_sparse,
|
||||
is_previous_layer_sparse=is_previous_layer_sparse,
|
||||
is_next_layer_sparse=is_next_layer_sparse,
|
||||
)
|
||||
|
||||
if self.is_layer_sparse:
|
||||
self.mlp = Qwen3MoeSparseMoeBlock(
|
||||
layer_id=self.layer_id,
|
||||
@@ -870,11 +866,21 @@ class Qwen3MoeDecoderLayer(nn.Module):
|
||||
config.hidden_size, eps=config.rms_norm_eps
|
||||
)
|
||||
|
||||
self.layer_communicator = LayerCommunicator(
|
||||
layer_facts=self.layer_facts,
|
||||
input_layernorm=self.input_layernorm,
|
||||
post_attention_layernorm=self.post_attention_layernorm,
|
||||
allow_reduce_scatter=True,
|
||||
self.attn_stage, self.ffn_stage = make_stages(
|
||||
(declare_attn(), self.input_layernorm),
|
||||
(
|
||||
declare_ffn(
|
||||
sparse=self.is_layer_sparse,
|
||||
next_sparse=is_next_layer_sparse,
|
||||
),
|
||||
self.post_attention_layernorm,
|
||||
),
|
||||
previous=declare_ffn(
|
||||
sparse=is_previous_layer_sparse, next_sparse=self.is_layer_sparse
|
||||
)
|
||||
if layer_id != 0
|
||||
else None,
|
||||
terminal=layer_id == config.num_hidden_layers - 1,
|
||||
)
|
||||
|
||||
def forward(
|
||||
@@ -882,21 +888,17 @@ class Qwen3MoeDecoderLayer(nn.Module):
|
||||
positions: torch.Tensor,
|
||||
hidden_states: torch.Tensor,
|
||||
forward_batch: ForwardBatch,
|
||||
residual: Optional[torch.Tensor],
|
||||
captured_last_layer_outputs: Optional[AuxHiddenStateAccumulator] = None,
|
||||
capture_output=None,
|
||||
**kwargs,
|
||||
) -> Tuple[torch.Tensor, torch.Tensor]:
|
||||
) -> torch.Tensor:
|
||||
|
||||
hidden_states, residual = (
|
||||
self.layer_communicator.prepare_attn_and_capture_last_layer_outputs(
|
||||
hidden_states,
|
||||
residual,
|
||||
forward_batch,
|
||||
captured_last_layer_outputs=captured_last_layer_outputs,
|
||||
capture_output=capture_output,
|
||||
**kwargs,
|
||||
)
|
||||
hidden_states = self.attn_stage.prepare(
|
||||
hidden_states,
|
||||
forward_batch,
|
||||
captured_last_layer_outputs=captured_last_layer_outputs,
|
||||
capture_output=capture_output,
|
||||
**kwargs,
|
||||
)
|
||||
|
||||
if hidden_states.shape[0] != 0:
|
||||
@@ -906,15 +908,14 @@ class Qwen3MoeDecoderLayer(nn.Module):
|
||||
forward_batch=forward_batch,
|
||||
)
|
||||
|
||||
hidden_states, residual = self.layer_communicator.prepare_mlp(
|
||||
hidden_states, residual, forward_batch
|
||||
)
|
||||
hidden_states = self.attn_stage.finish(hidden_states, forward_batch)
|
||||
hidden_states = self.ffn_stage.prepare(hidden_states, forward_batch)
|
||||
|
||||
with self.layer_communicator.ffn_exit(forward_batch) as ffn_exit:
|
||||
with self.ffn_stage.exit(forward_batch) as ffn_exit:
|
||||
hidden_states = self.mlp(hidden_states, forward_batch)
|
||||
hidden_states, residual = ffn_exit.finish(hidden_states, residual)
|
||||
hidden_states = ffn_exit.finish(hidden_states)
|
||||
|
||||
return hidden_states, residual
|
||||
return hidden_states
|
||||
|
||||
def op_comm_prepare_attn(
|
||||
self,
|
||||
@@ -922,11 +923,10 @@ class Qwen3MoeDecoderLayer(nn.Module):
|
||||
positions: torch.Tensor,
|
||||
hidden_states: torch.Tensor,
|
||||
forward_batch: ForwardBatch,
|
||||
residual: Optional[torch.Tensor],
|
||||
tbo_subbatch_index: Optional[int] = None,
|
||||
):
|
||||
state.hidden_states_after_comm_pre_attn, state.residual_after_input_ln = (
|
||||
self.layer_communicator.prepare_attn(hidden_states, residual, forward_batch)
|
||||
state.hidden_states_after_comm_pre_attn = self.attn_stage.prepare(
|
||||
hidden_states, forward_batch
|
||||
)
|
||||
state.update(
|
||||
dict(
|
||||
@@ -937,25 +937,21 @@ class Qwen3MoeDecoderLayer(nn.Module):
|
||||
)
|
||||
|
||||
def op_comm_prepare_mlp(self, state):
|
||||
state.hidden_states_mlp_input, state.residual_after_comm_pre_mlp = (
|
||||
self.layer_communicator.prepare_mlp(
|
||||
state.pop("hidden_states_after_attn"),
|
||||
state.pop("residual_after_input_ln"),
|
||||
state.forward_batch,
|
||||
)
|
||||
hidden_states = self.attn_stage.finish(
|
||||
state.pop("hidden_states_after_attn"), state.forward_batch
|
||||
)
|
||||
state.hidden_states_mlp_input = self.ffn_stage.prepare(
|
||||
hidden_states, state.forward_batch
|
||||
)
|
||||
|
||||
def op_comm_postprocess_layer(self, state):
|
||||
hidden_states, residual = self.layer_communicator.postprocess_layer(
|
||||
state.pop("hidden_states_mlp_output"),
|
||||
state.pop("residual_after_comm_pre_mlp"),
|
||||
state.forward_batch,
|
||||
hidden_states = self.ffn_stage.postprocess(
|
||||
state.pop("hidden_states_mlp_output"), state.forward_batch
|
||||
)
|
||||
|
||||
output = dict(
|
||||
positions=state.positions,
|
||||
hidden_states=hidden_states,
|
||||
residual=residual,
|
||||
forward_batch=state.forward_batch,
|
||||
tbo_subbatch_index=state.tbo_subbatch_index,
|
||||
)
|
||||
@@ -1088,7 +1084,7 @@ class Qwen3MoeForCausalLM(nn.Module):
|
||||
start, end = split_interval
|
||||
# embed
|
||||
if start == 0:
|
||||
forward_batch.residual = residual_batch.start(forward_batch)
|
||||
residual_batch.start(forward_batch)
|
||||
if input_embeds is None:
|
||||
forward_batch.hidden_states = self.model.embed_tokens(input_ids)
|
||||
else:
|
||||
@@ -1098,22 +1094,17 @@ class Qwen3MoeForCausalLM(nn.Module):
|
||||
for i in range(start, end):
|
||||
with get_global_expert_distribution_recorder().with_current_layer(i):
|
||||
layer = self.model.layers[i]
|
||||
forward_batch.hidden_states, forward_batch.residual = layer(
|
||||
positions,
|
||||
forward_batch.hidden_states,
|
||||
forward_batch,
|
||||
forward_batch.residual,
|
||||
forward_batch.hidden_states = layer(
|
||||
positions, forward_batch.hidden_states, forward_batch
|
||||
)
|
||||
|
||||
if end == self.model.config.num_hidden_layers:
|
||||
forward_batch.hidden_states, forward_batch.residual = self.model.layers[
|
||||
end - 1
|
||||
].layer_communicator.finish_layer_stack(
|
||||
forward_batch.hidden_states, forward_batch.residual, forward_batch
|
||||
forward_batch.hidden_states = residual_batch.finish(
|
||||
forward_batch.hidden_states, forward_batch
|
||||
)
|
||||
# norm
|
||||
hidden_states, _ = self.model.norm(
|
||||
forward_batch.hidden_states, forward_batch.residual
|
||||
hidden_states = residual_batch.norm(
|
||||
forward_batch.hidden_states, forward_batch, self.model.norm
|
||||
)
|
||||
forward_batch.residual = None
|
||||
forward_batch.hidden_states = hidden_states
|
||||
|
||||
@@ -19,7 +19,11 @@ from sglang.srt.layers.aux_hidden_states import (
|
||||
AuxHiddenStateAccumulator,
|
||||
AuxHiddenStateList,
|
||||
)
|
||||
from sglang.srt.layers.communicator import LayerCommunicator, LayerFacts
|
||||
from sglang.srt.layers.communicator import (
|
||||
declare_attn,
|
||||
declare_ffn,
|
||||
make_stages,
|
||||
)
|
||||
from sglang.srt.layers.communicator.residual import batch as residual_batch
|
||||
from sglang.srt.layers.dp_attention import (
|
||||
is_dp_attention_enabled,
|
||||
@@ -471,13 +475,11 @@ class Qwen3GatedDeltaNet(nn.Module):
|
||||
def _apply_qwen3_next_mlp(
|
||||
layer: nn.Module,
|
||||
hidden_states: torch.Tensor,
|
||||
residual: Optional[torch.Tensor],
|
||||
forward_batch: ForwardBatch,
|
||||
) -> Tuple[torch.Tensor, Optional[torch.Tensor]]:
|
||||
hidden_states, residual = layer.layer_communicator.prepare_mlp(
|
||||
hidden_states, residual, forward_batch
|
||||
)
|
||||
with layer.layer_communicator.ffn_exit(forward_batch) as ffn_exit:
|
||||
) -> torch.Tensor:
|
||||
hidden_states = layer.attn_stage.finish(hidden_states, forward_batch)
|
||||
hidden_states = layer.ffn_stage.prepare(hidden_states, forward_batch)
|
||||
with layer.ffn_stage.exit(forward_batch) as ffn_exit:
|
||||
if isinstance(layer.mlp, Qwen2MoeSparseMoeBlock):
|
||||
hidden_states = layer.mlp(
|
||||
hidden_states,
|
||||
@@ -485,9 +487,9 @@ def _apply_qwen3_next_mlp(
|
||||
)
|
||||
else:
|
||||
hidden_states = layer.mlp(hidden_states)
|
||||
hidden_states, residual = ffn_exit.finish(hidden_states, residual)
|
||||
hidden_states = ffn_exit.finish(hidden_states)
|
||||
|
||||
return hidden_states, residual
|
||||
return hidden_states
|
||||
|
||||
|
||||
class Qwen3HybridLinearDecoderLayer(nn.Module):
|
||||
@@ -512,14 +514,6 @@ class Qwen3HybridLinearDecoderLayer(nn.Module):
|
||||
is_next_layer_sparse = True
|
||||
self.layer_id = layer_id
|
||||
|
||||
self.layer_facts = LayerFacts.init_new(
|
||||
layer_id=layer_id,
|
||||
num_layers=config.num_hidden_layers,
|
||||
is_layer_sparse=self.is_layer_sparse,
|
||||
is_previous_layer_sparse=is_previous_layer_sparse,
|
||||
is_next_layer_sparse=is_next_layer_sparse,
|
||||
)
|
||||
|
||||
if self.is_layer_sparse:
|
||||
self.mlp = Qwen2MoeSparseMoeBlock(
|
||||
layer_id=layer_id,
|
||||
@@ -543,29 +537,35 @@ class Qwen3HybridLinearDecoderLayer(nn.Module):
|
||||
self.post_attention_layernorm = GemmaRMSNorm(
|
||||
config.hidden_size, eps=config.rms_norm_eps
|
||||
)
|
||||
self.layer_communicator = LayerCommunicator(
|
||||
layer_facts=self.layer_facts,
|
||||
input_layernorm=self.input_layernorm,
|
||||
post_attention_layernorm=self.post_attention_layernorm,
|
||||
allow_reduce_scatter=True,
|
||||
self.attn_stage, self.ffn_stage = make_stages(
|
||||
(declare_attn(), self.input_layernorm),
|
||||
(
|
||||
declare_ffn(
|
||||
sparse=self.is_layer_sparse,
|
||||
next_sparse=is_next_layer_sparse,
|
||||
),
|
||||
self.post_attention_layernorm,
|
||||
),
|
||||
previous=declare_ffn(
|
||||
sparse=is_previous_layer_sparse, next_sparse=self.is_layer_sparse
|
||||
)
|
||||
if layer_id != 0
|
||||
else None,
|
||||
terminal=layer_id == config.num_hidden_layers - 1,
|
||||
)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
residual: Optional[torch.Tensor],
|
||||
captured_last_layer_outputs: Optional[AuxHiddenStateAccumulator] = None,
|
||||
**kwargs,
|
||||
):
|
||||
forward_batch = kwargs.get("forward_batch", None)
|
||||
|
||||
hidden_states, residual = (
|
||||
self.layer_communicator.prepare_attn_and_capture_last_layer_outputs(
|
||||
hidden_states,
|
||||
residual,
|
||||
forward_batch,
|
||||
captured_last_layer_outputs=captured_last_layer_outputs,
|
||||
)
|
||||
hidden_states = self.attn_stage.prepare(
|
||||
hidden_states,
|
||||
forward_batch,
|
||||
captured_last_layer_outputs=captured_last_layer_outputs,
|
||||
)
|
||||
|
||||
if not forward_batch.forward_mode.is_idle():
|
||||
@@ -573,11 +573,9 @@ class Qwen3HybridLinearDecoderLayer(nn.Module):
|
||||
hidden_states,
|
||||
forward_batch,
|
||||
)
|
||||
hidden_states, residual = _apply_qwen3_next_mlp(
|
||||
self, hidden_states, residual, forward_batch
|
||||
)
|
||||
hidden_states = _apply_qwen3_next_mlp(self, hidden_states, forward_batch)
|
||||
|
||||
return hidden_states, residual
|
||||
return hidden_states
|
||||
|
||||
|
||||
class Qwen3HybridAttentionDecoderLayer(nn.Module):
|
||||
@@ -680,14 +678,6 @@ class Qwen3HybridAttentionDecoderLayer(nn.Module):
|
||||
is_previous_layer_sparse = True
|
||||
is_next_layer_sparse = True
|
||||
|
||||
self.layer_facts = LayerFacts.init_new(
|
||||
layer_id=layer_id,
|
||||
num_layers=config.num_hidden_layers,
|
||||
is_layer_sparse=self.is_layer_sparse,
|
||||
is_previous_layer_sparse=is_previous_layer_sparse,
|
||||
is_next_layer_sparse=is_next_layer_sparse,
|
||||
)
|
||||
|
||||
if self.is_layer_sparse:
|
||||
self.mlp = Qwen2MoeSparseMoeBlock(
|
||||
layer_id=layer_id,
|
||||
@@ -715,11 +705,21 @@ class Qwen3HybridAttentionDecoderLayer(nn.Module):
|
||||
self.q_norm = GemmaRMSNorm(self.head_dim, eps=config.rms_norm_eps)
|
||||
self.k_norm = GemmaRMSNorm(self.head_dim, eps=config.rms_norm_eps)
|
||||
|
||||
self.layer_communicator = LayerCommunicator(
|
||||
layer_facts=self.layer_facts,
|
||||
input_layernorm=self.input_layernorm,
|
||||
post_attention_layernorm=self.post_attention_layernorm,
|
||||
allow_reduce_scatter=True,
|
||||
self.attn_stage, self.ffn_stage = make_stages(
|
||||
(declare_attn(), self.input_layernorm),
|
||||
(
|
||||
declare_ffn(
|
||||
sparse=self.is_layer_sparse,
|
||||
next_sparse=is_next_layer_sparse,
|
||||
),
|
||||
self.post_attention_layernorm,
|
||||
),
|
||||
previous=declare_ffn(
|
||||
sparse=is_previous_layer_sparse, next_sparse=self.is_layer_sparse
|
||||
)
|
||||
if layer_id != 0
|
||||
else None,
|
||||
terminal=layer_id == config.num_hidden_layers - 1,
|
||||
)
|
||||
|
||||
self.alt_stream = alt_stream
|
||||
@@ -828,18 +828,14 @@ class Qwen3HybridAttentionDecoderLayer(nn.Module):
|
||||
self,
|
||||
positions: torch.Tensor,
|
||||
hidden_states: torch.Tensor,
|
||||
residual: Optional[torch.Tensor],
|
||||
forward_batch: ForwardBatch,
|
||||
captured_last_layer_outputs: Optional[AuxHiddenStateAccumulator] = None,
|
||||
**kwargs: Any,
|
||||
):
|
||||
hidden_states, residual = (
|
||||
self.layer_communicator.prepare_attn_and_capture_last_layer_outputs(
|
||||
hidden_states,
|
||||
residual,
|
||||
forward_batch,
|
||||
captured_last_layer_outputs=captured_last_layer_outputs,
|
||||
)
|
||||
hidden_states = self.attn_stage.prepare(
|
||||
hidden_states,
|
||||
forward_batch,
|
||||
captured_last_layer_outputs=captured_last_layer_outputs,
|
||||
)
|
||||
|
||||
if not forward_batch.forward_mode.is_idle():
|
||||
@@ -849,11 +845,9 @@ class Qwen3HybridAttentionDecoderLayer(nn.Module):
|
||||
forward_batch=forward_batch,
|
||||
)
|
||||
|
||||
hidden_states, residual = _apply_qwen3_next_mlp(
|
||||
self, hidden_states, residual, forward_batch
|
||||
)
|
||||
hidden_states = _apply_qwen3_next_mlp(self, hidden_states, forward_batch)
|
||||
|
||||
return hidden_states, residual
|
||||
return hidden_states
|
||||
|
||||
|
||||
ALL_DECODER_LAYER_TYPES = {
|
||||
@@ -934,33 +928,24 @@ class Qwen3NextModel(nn.Module):
|
||||
else:
|
||||
hidden_states = self.embed_tokens(input_ids)
|
||||
|
||||
residual = residual_batch.start(forward_batch)
|
||||
residual_batch.start(forward_batch)
|
||||
aux_hidden_states = AuxHiddenStateList()
|
||||
for i in range(len(self.layers)):
|
||||
layer = self.layers[i]
|
||||
with get_global_expert_distribution_recorder().with_current_layer(i):
|
||||
hidden_states, residual = layer(
|
||||
hidden_states = layer(
|
||||
layer_id=i,
|
||||
positions=positions,
|
||||
hidden_states=hidden_states,
|
||||
residual=residual,
|
||||
forward_batch=forward_batch,
|
||||
captured_last_layer_outputs=(
|
||||
aux_hidden_states
|
||||
if getattr(layer, "_is_layer_to_capture", False)
|
||||
else None
|
||||
),
|
||||
captured_last_layer_outputs=aux_hidden_states
|
||||
if getattr(layer, "_is_layer_to_capture", False)
|
||||
else None,
|
||||
)
|
||||
|
||||
last_layer = self.layers[-1]
|
||||
hidden_states, residual = last_layer.layer_communicator.finish_layer_stack(
|
||||
hidden_states, residual, forward_batch
|
||||
)
|
||||
hidden_states = residual_batch.finish(hidden_states, forward_batch)
|
||||
if not forward_batch.forward_mode.is_idle():
|
||||
if residual is None:
|
||||
hidden_states = self.norm(hidden_states)
|
||||
else:
|
||||
hidden_states, _ = self.norm(hidden_states, residual)
|
||||
hidden_states = residual_batch.norm(hidden_states, forward_batch, self.norm)
|
||||
|
||||
if len(aux_hidden_states) == 0:
|
||||
return hidden_states
|
||||
|
||||
@@ -96,12 +96,12 @@ class Qwen3MoeLLMModel(Qwen3MoeModel):
|
||||
hidden_states = self.embed_tokens(input_ids)
|
||||
else:
|
||||
hidden_states = input_embeds
|
||||
residual = residual_batch.start(forward_batch)
|
||||
residual_batch.start(forward_batch)
|
||||
else:
|
||||
assert pp_proxy_tensors is not None
|
||||
hidden_states, residual = self.layers[
|
||||
self.start_layer
|
||||
].layer_communicator.from_pp(pp_proxy_tensors, forward_batch)
|
||||
hidden_states = self.layers[self.start_layer].attn_stage.from_pp(
|
||||
pp_proxy_tensors, forward_batch
|
||||
)
|
||||
|
||||
aux_hidden_states = AuxHiddenStateList()
|
||||
for layer_idx, layer in enumerate(
|
||||
@@ -116,11 +116,10 @@ class Qwen3MoeLLMModel(Qwen3MoeModel):
|
||||
deepstack_embeds = self.get_deepstack_embeds(
|
||||
layer_idx - 1, input_deepstack_embeds
|
||||
)
|
||||
hidden_states, residual = layer(
|
||||
hidden_states = layer(
|
||||
positions,
|
||||
hidden_states,
|
||||
forward_batch,
|
||||
residual,
|
||||
post_residual_addition=deepstack_embeds,
|
||||
capture_output=aux_hidden_states.capture
|
||||
if layer_idx in self.layers_to_capture
|
||||
@@ -129,11 +128,10 @@ class Qwen3MoeLLMModel(Qwen3MoeModel):
|
||||
else:
|
||||
# Inference path: add deepstack directly to hidden_states at the end of the layer
|
||||
# (original, grounding-correct order).
|
||||
hidden_states, residual = layer(
|
||||
hidden_states = layer(
|
||||
positions,
|
||||
hidden_states,
|
||||
forward_batch,
|
||||
residual,
|
||||
capture_output=aux_hidden_states.capture
|
||||
if layer_idx in self.layers_to_capture
|
||||
else None,
|
||||
@@ -143,9 +141,9 @@ class Qwen3MoeLLMModel(Qwen3MoeModel):
|
||||
and layer_idx in self.deepstack_embed_to_decoder_layer
|
||||
):
|
||||
sep = self.hidden_size * layer_idx
|
||||
hidden_states, residual = layer.layer_communicator.add_to_output(
|
||||
hidden_states = residual_batch.add_to_output(
|
||||
hidden_states,
|
||||
residual,
|
||||
forward_batch,
|
||||
input_deepstack_embeds[:, sep : sep + self.hidden_size],
|
||||
)
|
||||
|
||||
@@ -156,28 +154,17 @@ class Qwen3MoeLLMModel(Qwen3MoeModel):
|
||||
else None
|
||||
)
|
||||
|
||||
last_layer = self.layers[self.end_layer - 1]
|
||||
hidden_states, residual = last_layer.layer_communicator.finish_layer_stack(
|
||||
hidden_states,
|
||||
residual,
|
||||
forward_batch,
|
||||
preserve_declared=not self.pp_group.is_last_rank,
|
||||
)
|
||||
if not self.pp_group.is_last_rank:
|
||||
return PPProxyTensors(
|
||||
{
|
||||
"hidden_states": hidden_states,
|
||||
"residual": residual,
|
||||
}
|
||||
)
|
||||
return residual_batch.to_pp(hidden_states, forward_batch)
|
||||
else:
|
||||
hidden_states = residual_batch.finish(hidden_states, forward_batch)
|
||||
if hidden_states.shape[0] != 0:
|
||||
if residual is None:
|
||||
hidden_states = self.norm(hidden_states)
|
||||
else:
|
||||
hidden_states, _ = self.norm(
|
||||
hidden_states, residual, post_residual_addition=last_deepstack
|
||||
)
|
||||
hidden_states = residual_batch.norm(
|
||||
hidden_states,
|
||||
forward_batch,
|
||||
self.norm,
|
||||
post_residual_addition=last_deepstack,
|
||||
)
|
||||
|
||||
if len(aux_hidden_states) == 0:
|
||||
return hidden_states
|
||||
|
||||
@@ -1320,7 +1320,8 @@ class Qwen4ExpLayerExtensionMixin:
|
||||
for attr_name in (
|
||||
"input_layernorm",
|
||||
"post_attention_layernorm",
|
||||
"layer_communicator",
|
||||
"attn_stage",
|
||||
"ffn_stage",
|
||||
):
|
||||
if hasattr(self, attr_name):
|
||||
delattr(self, attr_name)
|
||||
|
||||
@@ -20,9 +20,10 @@ from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_r
|
||||
from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation
|
||||
from sglang.srt.layers.activation import SiluAndMul
|
||||
from sglang.srt.layers.communicator import (
|
||||
LayerCommunicator,
|
||||
LayerFacts,
|
||||
declare_attn,
|
||||
declare_ffn,
|
||||
enable_moe_dense_fully_dp,
|
||||
make_stages,
|
||||
)
|
||||
from sglang.srt.layers.communicator.residual import batch as residual_batch
|
||||
from sglang.srt.layers.dp_attention import (
|
||||
@@ -1048,19 +1049,26 @@ class SarvamMoEMLADecoderLayer(nn.Module):
|
||||
)
|
||||
|
||||
self.attn_tp_size = get_parallel().attn_tp_size
|
||||
self.layer_facts = LayerFacts.init_new(
|
||||
layer_id=layer_id,
|
||||
num_layers=config.num_hidden_layers,
|
||||
is_layer_sparse=self.is_layer_sparse,
|
||||
is_previous_layer_sparse=is_previous_layer_sparse,
|
||||
is_next_layer_sparse=is_next_layer_sparse,
|
||||
)
|
||||
self.layer_communicator = LayerCommunicator(
|
||||
layer_facts=self.layer_facts,
|
||||
input_layernorm=self.input_layernorm,
|
||||
post_attention_layernorm=self.post_attention_layernorm,
|
||||
qkv_latent_func=self.self_attn.prepare_qkv_latent,
|
||||
allow_reduce_scatter=True,
|
||||
|
||||
self.attn_stage, self.ffn_stage = make_stages(
|
||||
(
|
||||
declare_attn(),
|
||||
self.input_layernorm,
|
||||
{"qkv_latent_func": self.self_attn.prepare_qkv_latent},
|
||||
),
|
||||
(
|
||||
declare_ffn(
|
||||
sparse=self.is_layer_sparse,
|
||||
next_sparse=is_next_layer_sparse,
|
||||
),
|
||||
self.post_attention_layernorm,
|
||||
),
|
||||
previous=declare_ffn(
|
||||
sparse=is_previous_layer_sparse, next_sparse=self.is_layer_sparse
|
||||
)
|
||||
if layer_id != 0
|
||||
else None,
|
||||
terminal=layer_id == config.num_hidden_layers - 1,
|
||||
)
|
||||
|
||||
def forward(
|
||||
@@ -1068,21 +1076,17 @@ class SarvamMoEMLADecoderLayer(nn.Module):
|
||||
positions: torch.Tensor,
|
||||
hidden_states: torch.Tensor,
|
||||
forward_batch: ForwardBatch,
|
||||
residual: Optional[torch.Tensor],
|
||||
) -> Tuple[torch.Tensor, torch.Tensor]:
|
||||
hidden_states, residual = self.layer_communicator.prepare_attn(
|
||||
hidden_states, residual, forward_batch
|
||||
)
|
||||
) -> torch.Tensor:
|
||||
hidden_states = self.attn_stage.prepare(hidden_states, forward_batch)
|
||||
if hidden_states.shape[0] != 0:
|
||||
hidden_states = self.self_attn(
|
||||
positions=positions,
|
||||
hidden_states=hidden_states,
|
||||
forward_batch=forward_batch,
|
||||
)
|
||||
hidden_states, residual = self.layer_communicator.prepare_mlp(
|
||||
hidden_states, residual, forward_batch
|
||||
)
|
||||
with self.layer_communicator.ffn_exit(forward_batch) as ffn_exit:
|
||||
hidden_states = self.attn_stage.finish(hidden_states, forward_batch)
|
||||
hidden_states = self.ffn_stage.prepare(hidden_states, forward_batch)
|
||||
with self.ffn_stage.exit(forward_batch) as ffn_exit:
|
||||
hidden_states = self.mlp(hidden_states, forward_batch)
|
||||
if (
|
||||
not self.is_layer_sparse
|
||||
@@ -1091,8 +1095,8 @@ class SarvamMoEMLADecoderLayer(nn.Module):
|
||||
and not ffn_exit.fuse_mlp_allreduce
|
||||
):
|
||||
hidden_states = tensor_model_parallel_all_reduce(hidden_states)
|
||||
hidden_states, residual = ffn_exit.finish(hidden_states, residual)
|
||||
return hidden_states, residual
|
||||
hidden_states = ffn_exit.finish(hidden_states)
|
||||
return hidden_states
|
||||
|
||||
|
||||
class SarvamMLAModel(nn.Module):
|
||||
@@ -1152,36 +1156,23 @@ class SarvamMLAModel(nn.Module):
|
||||
hidden_states = self.embed_tokens(input_ids)
|
||||
else:
|
||||
hidden_states = input_embeds
|
||||
residual = residual_batch.start(forward_batch)
|
||||
residual_batch.start(forward_batch)
|
||||
else:
|
||||
assert pp_proxy_tensors is not None
|
||||
hidden_states, residual = self.layers[
|
||||
self.start_layer
|
||||
].layer_communicator.from_pp(pp_proxy_tensors, forward_batch)
|
||||
hidden_states = self.layers[self.start_layer].attn_stage.from_pp(
|
||||
pp_proxy_tensors, forward_batch
|
||||
)
|
||||
|
||||
for i in range(self.start_layer, self.end_layer):
|
||||
layer = self.layers[i]
|
||||
hidden_states, residual = layer(
|
||||
positions, hidden_states, forward_batch, residual
|
||||
)
|
||||
hidden_states = layer(positions, hidden_states, forward_batch)
|
||||
|
||||
last_layer = self.layers[self.end_layer - 1]
|
||||
hidden_states, residual = last_layer.layer_communicator.finish_layer_stack(
|
||||
hidden_states,
|
||||
residual,
|
||||
forward_batch,
|
||||
preserve_declared=not self.pp_group.is_last_rank,
|
||||
)
|
||||
if not self.pp_group.is_last_rank:
|
||||
return PPProxyTensors(
|
||||
{"hidden_states": hidden_states, "residual": residual}
|
||||
)
|
||||
return residual_batch.to_pp(hidden_states, forward_batch)
|
||||
hidden_states = residual_batch.finish(hidden_states, forward_batch)
|
||||
|
||||
if hidden_states.shape[0] != 0:
|
||||
if residual is None:
|
||||
hidden_states = self.norm(hidden_states)
|
||||
else:
|
||||
hidden_states, _ = self.norm(hidden_states, residual)
|
||||
hidden_states = residual_batch.norm(hidden_states, forward_batch, self.norm)
|
||||
|
||||
return hidden_states
|
||||
|
||||
@@ -1267,36 +1258,26 @@ class SarvamMLAForCausalLM(nn.Module):
|
||||
) -> Optional[LogitsProcessorOutput]:
|
||||
start, end = split_interval
|
||||
if start == 0:
|
||||
residual_batch.start(forward_batch)
|
||||
if input_embeds is None:
|
||||
forward_batch.hidden_states = self.model.embed_tokens(input_ids)
|
||||
else:
|
||||
forward_batch.hidden_states = input_embeds
|
||||
forward_batch.residual = residual_batch.start(forward_batch)
|
||||
|
||||
for i in range(start, end):
|
||||
with get_global_expert_distribution_recorder().with_current_layer(i):
|
||||
layer = self.model.layers[i]
|
||||
forward_batch.hidden_states, forward_batch.residual = layer(
|
||||
positions,
|
||||
forward_batch.hidden_states,
|
||||
forward_batch,
|
||||
forward_batch.residual,
|
||||
forward_batch.hidden_states = layer(
|
||||
positions, forward_batch.hidden_states, forward_batch
|
||||
)
|
||||
|
||||
if end == self.model.config.num_hidden_layers:
|
||||
forward_batch.hidden_states, forward_batch.residual = self.model.layers[
|
||||
end - 1
|
||||
].layer_communicator.finish_layer_stack(
|
||||
forward_batch.hidden_states, forward_batch.residual, forward_batch
|
||||
forward_batch.hidden_states = residual_batch.finish(
|
||||
forward_batch.hidden_states, forward_batch
|
||||
)
|
||||
forward_batch.hidden_states = residual_batch.norm(
|
||||
forward_batch.hidden_states, forward_batch, self.model.norm
|
||||
)
|
||||
if forward_batch.residual is None:
|
||||
hidden_states = self.model.norm(forward_batch.hidden_states)
|
||||
else:
|
||||
hidden_states, _ = self.model.norm(
|
||||
forward_batch.hidden_states, forward_batch.residual
|
||||
)
|
||||
forward_batch.residual = None
|
||||
forward_batch.hidden_states = hidden_states
|
||||
return self.logits_processor(
|
||||
input_ids, forward_batch.hidden_states, self.lm_head, forward_batch
|
||||
)
|
||||
|
||||
@@ -11,7 +11,11 @@ from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_r
|
||||
from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation
|
||||
from sglang.srt.eplb.expert_location_dispatch import ExpertLocationDispatchInfo
|
||||
from sglang.srt.layers.activation import SiluAndMul
|
||||
from sglang.srt.layers.communicator import LayerCommunicator, LayerFacts
|
||||
from sglang.srt.layers.communicator import (
|
||||
declare_attn,
|
||||
declare_ffn,
|
||||
make_stages,
|
||||
)
|
||||
from sglang.srt.layers.communicator.residual import batch as residual_batch
|
||||
from sglang.srt.layers.dp_attention import (
|
||||
is_dp_attention_enabled,
|
||||
@@ -561,21 +565,23 @@ class Step3p5DecoderLayer(nn.Module):
|
||||
)
|
||||
|
||||
# An MTP draft is a one-layer model; layer_id still indexes its config.
|
||||
self.layer_facts = LayerFacts.init_new(
|
||||
layer_id=0 if is_nextn else layer_id,
|
||||
num_layers=1 if is_nextn else config.num_hidden_layers,
|
||||
is_layer_sparse=self.is_moe_layer,
|
||||
is_previous_layer_sparse=self.is_previous_layer_sparse,
|
||||
is_next_layer_sparse=self.is_next_layer_sparse,
|
||||
)
|
||||
self.layer_communicator = LayerCommunicator(
|
||||
layer_facts=self.layer_facts,
|
||||
input_layernorm=self.input_layernorm,
|
||||
post_attention_layernorm=self.post_attention_layernorm,
|
||||
allow_reduce_scatter=True,
|
||||
# The dense MLP all-reduces its own output unless postprocess
|
||||
# reduce-scatters it; it never leaves the sum to the next layer.
|
||||
allow_deferred_ffn_reduction=self.use_moe,
|
||||
|
||||
self.attn_stage, self.ffn_stage = make_stages(
|
||||
(declare_attn(), self.input_layernorm),
|
||||
(
|
||||
declare_ffn(
|
||||
sparse=self.is_moe_layer,
|
||||
next_sparse=self.is_next_layer_sparse,
|
||||
),
|
||||
self.post_attention_layernorm,
|
||||
),
|
||||
previous=declare_ffn(
|
||||
sparse=self.is_previous_layer_sparse, next_sparse=self.is_moe_layer
|
||||
)
|
||||
if not (0 if is_nextn else layer_id) == 0
|
||||
else None,
|
||||
terminal=(0 if is_nextn else layer_id)
|
||||
== (1 if is_nextn else config.num_hidden_layers) - 1,
|
||||
)
|
||||
|
||||
self.layer_id = layer_id
|
||||
@@ -585,15 +591,11 @@ class Step3p5DecoderLayer(nn.Module):
|
||||
positions: torch.Tensor,
|
||||
hidden_states: torch.Tensor,
|
||||
forward_batch: ForwardBatch,
|
||||
residual: Optional[torch.Tensor],
|
||||
post_residual_addition: Optional[torch.Tensor] = None,
|
||||
) -> Tuple[torch.Tensor, torch.Tensor]:
|
||||
) -> torch.Tensor:
|
||||
# Self Attention
|
||||
hidden_states, residual = self.layer_communicator.prepare_attn(
|
||||
hidden_states,
|
||||
residual,
|
||||
forward_batch,
|
||||
post_residual_addition=post_residual_addition,
|
||||
hidden_states = self.attn_stage.prepare(
|
||||
hidden_states, forward_batch, post_residual_addition=post_residual_addition
|
||||
)
|
||||
if hidden_states.shape[0] != 0:
|
||||
hidden_states = self.self_attn(
|
||||
@@ -602,14 +604,11 @@ class Step3p5DecoderLayer(nn.Module):
|
||||
forward_batch=forward_batch,
|
||||
)
|
||||
# Fully Connected
|
||||
hidden_states, residual = self.layer_communicator.prepare_mlp(
|
||||
hidden_states,
|
||||
residual,
|
||||
forward_batch,
|
||||
)
|
||||
hidden_states = self.attn_stage.finish(hidden_states, forward_batch)
|
||||
hidden_states = self.ffn_stage.prepare(hidden_states, forward_batch)
|
||||
|
||||
if self.use_moe:
|
||||
with self.layer_communicator.ffn_exit(forward_batch) as ffn_exit:
|
||||
with self.ffn_stage.exit(forward_batch) as ffn_exit:
|
||||
# Both share_expert and MoE return unreduced (TP-partial) outputs.
|
||||
# Combine them first, then do a single all-reduce — saving one
|
||||
# full-TP all-reduce per layer.
|
||||
@@ -620,11 +619,11 @@ class Step3p5DecoderLayer(nn.Module):
|
||||
hidden_states = moe_output + share_output
|
||||
if not ffn_exit.fuse_mlp_allreduce and not ffn_exit.mlp_reduce_scatter:
|
||||
hidden_states = tensor_model_parallel_all_reduce(hidden_states)
|
||||
return ffn_exit.finish(hidden_states, residual)
|
||||
return ffn_exit.finish(hidden_states)
|
||||
|
||||
with self.layer_communicator.ffn_exit(forward_batch) as ffn_exit:
|
||||
with self.ffn_stage.exit(forward_batch) as ffn_exit:
|
||||
hidden_states = self.mlp(hidden_states)
|
||||
return ffn_exit.finish(hidden_states, residual)
|
||||
return ffn_exit.finish(hidden_states)
|
||||
|
||||
|
||||
class Step3p5Model(nn.Module):
|
||||
@@ -698,55 +697,33 @@ class Step3p5Model(nn.Module):
|
||||
hidden_states = self.embed_tokens(input_ids)
|
||||
else:
|
||||
hidden_states = input_embeds
|
||||
residual = residual_batch.start(forward_batch)
|
||||
residual_batch.start(forward_batch)
|
||||
else:
|
||||
assert pp_proxy_tensors is not None
|
||||
hidden_states, residual = self.layers[
|
||||
self.start_layer
|
||||
].layer_communicator.from_pp(pp_proxy_tensors, forward_batch)
|
||||
hidden_states = self.layers[self.start_layer].attn_stage.from_pp(
|
||||
pp_proxy_tensors, forward_batch
|
||||
)
|
||||
|
||||
for i in range(self.start_layer, self.end_layer):
|
||||
layer = self.layers[i]
|
||||
hidden_states, residual = layer(
|
||||
positions,
|
||||
hidden_states,
|
||||
forward_batch,
|
||||
residual,
|
||||
)
|
||||
hidden_states = layer(positions, hidden_states, forward_batch)
|
||||
# break
|
||||
last_layer = self.layers[self.end_layer - 1]
|
||||
hidden_states, residual = last_layer.layer_communicator.finish_layer_stack(
|
||||
hidden_states,
|
||||
residual,
|
||||
forward_batch,
|
||||
preserve_declared=not self.pp_group.is_last_rank,
|
||||
)
|
||||
if not self.pp_group.is_last_rank:
|
||||
return PPProxyTensors(
|
||||
{
|
||||
"hidden_states": hidden_states,
|
||||
"residual": residual,
|
||||
}
|
||||
)
|
||||
return residual_batch.to_pp(hidden_states, forward_batch)
|
||||
else:
|
||||
hidden_states = residual_batch.finish(hidden_states, forward_batch)
|
||||
hidden_states_before_norm = None
|
||||
if not self.pp_group.is_last_rank:
|
||||
return PPProxyTensors(
|
||||
{
|
||||
"hidden_states": hidden_states,
|
||||
"residual": residual,
|
||||
}
|
||||
)
|
||||
return residual_batch.to_pp(hidden_states, forward_batch)
|
||||
else:
|
||||
if hidden_states.shape[0] > 0:
|
||||
# if forward_batch.return_hidden_states_before_norm:
|
||||
hidden_states_before_norm = last_layer.layer_communicator.snapshot(
|
||||
hidden_states, residual
|
||||
hidden_states_before_norm = residual_batch.snapshot(
|
||||
hidden_states, forward_batch
|
||||
)
|
||||
hidden_states = residual_batch.norm(
|
||||
hidden_states, forward_batch, self.norm
|
||||
)
|
||||
if residual is None:
|
||||
hidden_states = self.norm(hidden_states)
|
||||
else:
|
||||
hidden_states, _ = self.norm(hidden_states, residual)
|
||||
return hidden_states, hidden_states_before_norm
|
||||
|
||||
|
||||
|
||||
@@ -110,25 +110,22 @@ class Step3p5AMultiTokenPredictor(nn.Module):
|
||||
dim=-1,
|
||||
)
|
||||
)
|
||||
hidden_states, residual = self.mtp_block(
|
||||
residual_batch.start(forward_batch)
|
||||
hidden_states = self.mtp_block(
|
||||
positions=positions,
|
||||
hidden_states=hidden_states,
|
||||
forward_batch=forward_batch,
|
||||
residual=residual_batch.start(forward_batch),
|
||||
)
|
||||
hidden_states, residual = self.mtp_block.layer_communicator.finish_layer_stack(
|
||||
hidden_states, residual, forward_batch
|
||||
)
|
||||
hidden_states = residual_batch.finish(hidden_states, forward_batch)
|
||||
hidden_states_before_norm = None
|
||||
if not forward_batch.forward_mode.is_idle():
|
||||
# if forward_batch.return_hidden_states_before_norm:
|
||||
hidden_states_before_norm = self.mtp_block.layer_communicator.snapshot(
|
||||
hidden_states, residual
|
||||
hidden_states_before_norm = residual_batch.snapshot(
|
||||
hidden_states, forward_batch
|
||||
)
|
||||
hidden_states = residual_batch.norm(
|
||||
hidden_states, forward_batch, self.shared_head.norm
|
||||
)
|
||||
if residual is not None:
|
||||
hidden_states, _ = self.shared_head.norm(hidden_states, residual)
|
||||
else:
|
||||
hidden_states = self.shared_head.norm(hidden_states)
|
||||
|
||||
return hidden_states, hidden_states_before_norm
|
||||
|
||||
|
||||
@@ -40,9 +40,10 @@ from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation
|
||||
from sglang.srt.eplb.expert_location_dispatch import ExpertLocationDispatchInfo
|
||||
from sglang.srt.layers.activation import SiluAndMul
|
||||
from sglang.srt.layers.communicator import (
|
||||
LayerCommunicator,
|
||||
LayerFacts,
|
||||
declare_attn,
|
||||
declare_ffn,
|
||||
enable_moe_dense_fully_dp,
|
||||
make_stages,
|
||||
)
|
||||
from sglang.srt.layers.communicator.residual import batch as residual_batch
|
||||
from sglang.srt.layers.dp_attention import is_dp_attention_enabled
|
||||
@@ -860,11 +861,10 @@ class XllmMLP(nn.Module):
|
||||
def forward(
|
||||
self,
|
||||
x,
|
||||
use_reduce_scatter: bool = False,
|
||||
):
|
||||
gate_up, _ = self.gate_up_proj(x)
|
||||
x = self.act_fn(gate_up)
|
||||
x, _ = self.down_proj(x, skip_all_reduce=use_reduce_scatter)
|
||||
x, _ = self.down_proj(x)
|
||||
return x
|
||||
|
||||
|
||||
@@ -1041,7 +1041,6 @@ class XllmSparseMoeBlock(nn.Module):
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
forward_batch: Optional[ForwardBatch] = None,
|
||||
use_reduce_scatter: bool = False,
|
||||
) -> torch.Tensor:
|
||||
num_tokens, hidden_dim = hidden_states.shape
|
||||
hidden_states = hidden_states.view(-1, hidden_dim)
|
||||
@@ -1061,8 +1060,7 @@ class XllmSparseMoeBlock(nn.Module):
|
||||
|
||||
if shared_output is not None:
|
||||
final_hidden_states += shared_output
|
||||
if not use_reduce_scatter:
|
||||
final_hidden_states = reduce_moe_output(final_hidden_states)
|
||||
final_hidden_states = reduce_moe_output(final_hidden_states)
|
||||
|
||||
return final_hidden_states.view(num_tokens, hidden_dim)
|
||||
|
||||
@@ -1506,14 +1504,6 @@ class XllmDecoderLayer(nn.Module):
|
||||
is_previous_layer_sparse = _is_sparse(layer_id - 1)
|
||||
is_next_layer_sparse = _is_sparse(layer_id + 1)
|
||||
|
||||
self.layer_facts = LayerFacts.init_new(
|
||||
layer_id=layer_id,
|
||||
num_layers=config.num_hidden_layers,
|
||||
is_layer_sparse=self.is_layer_sparse,
|
||||
is_previous_layer_sparse=is_previous_layer_sparse,
|
||||
is_next_layer_sparse=is_next_layer_sparse,
|
||||
)
|
||||
|
||||
if self.is_layer_sparse:
|
||||
self.mlp = XllmSparseMoeBlock(
|
||||
layer_id=layer_id,
|
||||
@@ -1538,12 +1528,21 @@ class XllmDecoderLayer(nn.Module):
|
||||
|
||||
self.input_layernorm = _make_norm(config)
|
||||
self.post_attention_layernorm = _make_norm(config)
|
||||
self.layer_communicator = LayerCommunicator(
|
||||
layer_facts=self.layer_facts,
|
||||
input_layernorm=self.input_layernorm,
|
||||
post_attention_layernorm=self.post_attention_layernorm,
|
||||
allow_reduce_scatter=True,
|
||||
allow_deferred_ffn_reduction=False,
|
||||
self.attn_stage, self.ffn_stage = make_stages(
|
||||
(declare_attn(), self.input_layernorm),
|
||||
(
|
||||
declare_ffn(
|
||||
sparse=self.is_layer_sparse,
|
||||
next_sparse=is_next_layer_sparse,
|
||||
),
|
||||
self.post_attention_layernorm,
|
||||
),
|
||||
previous=declare_ffn(
|
||||
sparse=is_previous_layer_sparse, next_sparse=self.is_layer_sparse
|
||||
)
|
||||
if layer_id != 0
|
||||
else None,
|
||||
terminal=layer_id == config.num_hidden_layers - 1,
|
||||
)
|
||||
|
||||
def forward(
|
||||
@@ -1551,13 +1550,8 @@ class XllmDecoderLayer(nn.Module):
|
||||
positions: torch.Tensor,
|
||||
hidden_states: torch.Tensor,
|
||||
forward_batch: ForwardBatch,
|
||||
residual: Optional[torch.Tensor],
|
||||
) -> Tuple[torch.Tensor, torch.Tensor]:
|
||||
hidden_states, residual = self.layer_communicator.prepare_attn(
|
||||
hidden_states,
|
||||
residual,
|
||||
forward_batch,
|
||||
)
|
||||
) -> torch.Tensor:
|
||||
hidden_states = self.attn_stage.prepare(hidden_states, forward_batch)
|
||||
|
||||
if hidden_states.shape[0] != 0:
|
||||
hidden_states = self.self_attn(
|
||||
@@ -1566,23 +1560,17 @@ class XllmDecoderLayer(nn.Module):
|
||||
forward_batch=forward_batch,
|
||||
)
|
||||
|
||||
hidden_states, residual = self.layer_communicator.prepare_mlp(
|
||||
hidden_states, residual, forward_batch
|
||||
)
|
||||
hidden_states = self.attn_stage.finish(hidden_states, forward_batch)
|
||||
hidden_states = self.ffn_stage.prepare(hidden_states, forward_batch)
|
||||
|
||||
with self.layer_communicator.ffn_exit(forward_batch) as ffn_exit:
|
||||
use_reduce_scatter = ffn_exit.mlp_reduce_scatter
|
||||
with self.ffn_stage.exit(forward_batch) as ffn_exit:
|
||||
if isinstance(self.mlp, XllmMLP):
|
||||
hidden_states = self.mlp(
|
||||
hidden_states, use_reduce_scatter=use_reduce_scatter
|
||||
)
|
||||
hidden_states = self.mlp(hidden_states)
|
||||
else:
|
||||
hidden_states = self.mlp(
|
||||
hidden_states, forward_batch, use_reduce_scatter
|
||||
)
|
||||
hidden_states, residual = ffn_exit.finish(hidden_states, residual)
|
||||
hidden_states = self.mlp(hidden_states, forward_batch)
|
||||
hidden_states = ffn_exit.finish(hidden_states)
|
||||
|
||||
return hidden_states, residual
|
||||
return hidden_states
|
||||
|
||||
|
||||
class XllmModel(nn.Module):
|
||||
@@ -1638,12 +1626,12 @@ class XllmModel(nn.Module):
|
||||
hidden_states = self.embed_tokens(input_ids)
|
||||
else:
|
||||
hidden_states = input_embeds
|
||||
residual = residual_batch.start(forward_batch)
|
||||
residual_batch.start(forward_batch)
|
||||
else:
|
||||
assert pp_proxy_tensors is not None
|
||||
hidden_states, residual = self.layers[
|
||||
self.start_layer
|
||||
].layer_communicator.from_pp(pp_proxy_tensors, forward_batch)
|
||||
hidden_states = self.layers[self.start_layer].attn_stage.from_pp(
|
||||
pp_proxy_tensors, forward_batch
|
||||
)
|
||||
|
||||
for i in range(self.start_layer, self.end_layer):
|
||||
ctx = (
|
||||
@@ -1653,34 +1641,16 @@ class XllmModel(nn.Module):
|
||||
)
|
||||
with ctx:
|
||||
layer = self.layers[i]
|
||||
hidden_states, residual = layer(
|
||||
positions,
|
||||
hidden_states,
|
||||
forward_batch,
|
||||
residual,
|
||||
)
|
||||
hidden_states = layer(positions, hidden_states, forward_batch)
|
||||
|
||||
hidden_states, residual = self.layers[
|
||||
self.end_layer - 1
|
||||
].layer_communicator.finish_layer_stack(
|
||||
hidden_states,
|
||||
residual,
|
||||
forward_batch,
|
||||
preserve_declared=not self.pp_group.is_last_rank,
|
||||
)
|
||||
if not self.pp_group.is_last_rank:
|
||||
return PPProxyTensors(
|
||||
{
|
||||
"hidden_states": hidden_states,
|
||||
"residual": residual,
|
||||
}
|
||||
)
|
||||
return residual_batch.to_pp(hidden_states, forward_batch)
|
||||
else:
|
||||
hidden_states = residual_batch.finish(hidden_states, forward_batch)
|
||||
if hidden_states.shape[0] != 0:
|
||||
if residual is None:
|
||||
hidden_states = self.norm(hidden_states)
|
||||
else:
|
||||
hidden_states, _ = self.norm(hidden_states, residual)
|
||||
hidden_states = residual_batch.norm(
|
||||
hidden_states, forward_batch, self.norm
|
||||
)
|
||||
|
||||
return hidden_states
|
||||
|
||||
|
||||
@@ -71,6 +71,7 @@ def stub_plan():
|
||||
plan._fusion_rows = None
|
||||
plan._cp_steps = plan._sp_steps = plan._input_scattered_steps = None
|
||||
plan.is_first_layer = False
|
||||
plan.is_last_layer = False
|
||||
plan.residual_in_hidden = False
|
||||
# Keep production variant selection while tests replace individual paths.
|
||||
plan._batch_steps = lambda batch: {
|
||||
|
||||
@@ -35,8 +35,6 @@ from sglang.srt.layers.communicator.residual import (
|
||||
)
|
||||
|
||||
COMMUNICATOR_MODULES = (
|
||||
layer,
|
||||
legacy_stage,
|
||||
communicator,
|
||||
allreduce,
|
||||
exit,
|
||||
@@ -54,6 +52,8 @@ COMMUNICATOR_MODULES = (
|
||||
construction,
|
||||
factories,
|
||||
stage,
|
||||
layer,
|
||||
legacy_stage,
|
||||
batch,
|
||||
stream,
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user