[Refactor] Choose every layer's boundaries from declarations and remove the scatter-mode selection (#41438)
This commit is contained in:
@@ -15,7 +15,7 @@ from sglang.srt.layers.attention.dsa.dsa_npu_indexer import scattered_to_tp_attn
|
||||
from sglang.srt.layers.attention.dsa.utils import (
|
||||
dsa_use_prefill_cp,
|
||||
)
|
||||
from sglang.srt.layers.communicator import ScatterMode, get_attn_tp_context
|
||||
from sglang.srt.layers.communicator import get_attn_tp_context
|
||||
from sglang.srt.model_executor.forward_context import get_token_to_kv_pool
|
||||
from sglang.srt.runtime_context import get_disagg
|
||||
|
||||
@@ -34,7 +34,7 @@ def forward_mha_prepare_npu(
|
||||
hidden_states: torch.Tensor,
|
||||
forward_batch: "ForwardBatch",
|
||||
zero_allocator: "BumpAllocator",
|
||||
layer_scatter_modes,
|
||||
input_on_attention_tp_slices: bool,
|
||||
):
|
||||
if m.q_lora_rank is not None:
|
||||
q, latent_cache = (
|
||||
@@ -62,11 +62,7 @@ def forward_mha_prepare_npu(
|
||||
|
||||
else:
|
||||
q = m.q_a_layernorm(q)
|
||||
if (
|
||||
_use_ag_after_qlora
|
||||
and layer_scatter_modes.layer_input_mode == ScatterMode.SCATTERED
|
||||
and layer_scatter_modes.attn_mode == ScatterMode.TP_ATTN_FULL
|
||||
):
|
||||
if _use_ag_after_qlora and input_on_attention_tp_slices:
|
||||
q = scattered_to_tp_attn_full(q, forward_batch)
|
||||
latent_cache = scattered_to_tp_attn_full(latent_cache, forward_batch)
|
||||
q = m.q_b_proj(q)[0].view(-1, m.num_local_heads, m.qk_head_dim)
|
||||
@@ -160,7 +156,7 @@ def forward_mla_prepare_npu(
|
||||
hidden_states: torch.Tensor,
|
||||
forward_batch: "ForwardBatch",
|
||||
zero_allocator: "BumpAllocator",
|
||||
layer_scatter_modes,
|
||||
input_on_attention_tp_slices: bool,
|
||||
):
|
||||
if is_mla_preprocess_enabled():
|
||||
if not hasattr(m, "mla_preprocess"):
|
||||
@@ -193,11 +189,7 @@ def forward_mla_prepare_npu(
|
||||
q_lora = None
|
||||
if m.q_lora_rank is not None:
|
||||
qkv_latent = get_attn_tp_context().fetch_qkv_latent()
|
||||
if (
|
||||
_use_ag_after_qlora
|
||||
and layer_scatter_modes.layer_input_mode == ScatterMode.SCATTERED
|
||||
and layer_scatter_modes.attn_mode == ScatterMode.TP_ATTN_FULL
|
||||
):
|
||||
if _use_ag_after_qlora and input_on_attention_tp_slices:
|
||||
q, latent_cache = qkv_latent.split(
|
||||
[m.q_lora_rank, m.kv_lora_rank + m.qk_rope_head_dim],
|
||||
dim=-1,
|
||||
@@ -369,7 +361,7 @@ def forward_dsa_prepare_npu(
|
||||
hidden_states: torch.Tensor,
|
||||
forward_batch: "ForwardBatch",
|
||||
zero_allocator: "BumpAllocator",
|
||||
layer_scatter_modes,
|
||||
input_on_attention_tp_slices: bool,
|
||||
prev_topk_indices: torch.Tensor = None,
|
||||
):
|
||||
dynamic_scale = None
|
||||
@@ -403,11 +395,7 @@ def forward_dsa_prepare_npu(
|
||||
)
|
||||
# overlap qk norm
|
||||
q = m.q_a_layernorm(q)
|
||||
if (
|
||||
_use_ag_after_qlora
|
||||
and layer_scatter_modes.layer_input_mode == ScatterMode.SCATTERED
|
||||
and layer_scatter_modes.attn_mode == ScatterMode.TP_ATTN_FULL
|
||||
):
|
||||
if _use_ag_after_qlora and input_on_attention_tp_slices:
|
||||
q = scattered_to_tp_attn_full(q, forward_batch)
|
||||
latent_cache = scattered_to_tp_attn_full(latent_cache, forward_batch)
|
||||
q_lora = q.clone() # required for topk_indices
|
||||
@@ -496,7 +484,7 @@ def forward_dsa_prepare_npu(
|
||||
positions,
|
||||
forward_batch,
|
||||
m.layer_id,
|
||||
layer_scatter_modes,
|
||||
input_on_attention_tp_slices,
|
||||
dynamic_scale,
|
||||
)
|
||||
else:
|
||||
|
||||
@@ -5,7 +5,6 @@ from functools import lru_cache
|
||||
import torch
|
||||
|
||||
from sglang.srt.environ import envs
|
||||
from sglang.srt.layers.communicator import ScatterMode
|
||||
from sglang.srt.layers.cp.utils import cp_gather_full_sequence_states
|
||||
from sglang.srt.layers.dp_attention import attn_tp_all_gather_into_tensor
|
||||
from sglang.srt.model_executor.forward_batch_info import ForwardBatch
|
||||
@@ -61,7 +60,7 @@ class DSANPUIndexerMixin:
|
||||
positions: torch.Tensor,
|
||||
forward_batch: ForwardBatch,
|
||||
layer_id: int,
|
||||
layer_scatter_modes=None,
|
||||
input_on_attention_tp_slices: bool = False,
|
||||
dynamic_scale: torch.Tensor = None,
|
||||
) -> torch.Tensor:
|
||||
if get_attn_backend().forward_metadata.seq_lens_cpu_int is None:
|
||||
@@ -141,11 +140,7 @@ class DSANPUIndexerMixin:
|
||||
|
||||
k_proj = self.wk(x)[0] # [b, s, 7168] @ [7168, 128] = [b, s, 128]
|
||||
k = self.k_norm(k_proj)
|
||||
if (
|
||||
_use_ag_after_qlora
|
||||
and layer_scatter_modes.layer_input_mode == ScatterMode.SCATTERED
|
||||
and layer_scatter_modes.attn_mode == ScatterMode.TP_ATTN_FULL
|
||||
):
|
||||
if _use_ag_after_qlora and input_on_attention_tp_slices:
|
||||
k = scattered_to_tp_attn_full(k, forward_batch)
|
||||
k_pe, k_nope = torch.split(
|
||||
k,
|
||||
@@ -289,11 +284,7 @@ class DSANPUIndexerMixin:
|
||||
torch.npu.current_stream().wait_event(q_rope_event)
|
||||
if envs.SGLANG_NPU_USE_MULTI_STREAM.get():
|
||||
torch.npu.current_stream().wait_event(weights_event)
|
||||
if (
|
||||
_use_ag_after_qlora
|
||||
and layer_scatter_modes.layer_input_mode == ScatterMode.SCATTERED
|
||||
and layer_scatter_modes.attn_mode == ScatterMode.TP_ATTN_FULL
|
||||
):
|
||||
if _use_ag_after_qlora and input_on_attention_tp_slices:
|
||||
weights = scattered_to_tp_attn_full(weights, forward_batch)
|
||||
block_table = get_attn_backend().forward_metadata.block_tables
|
||||
if (
|
||||
|
||||
@@ -15,7 +15,7 @@ import logging
|
||||
from contextlib import contextmanager
|
||||
from dataclasses import dataclass
|
||||
from enum import Enum, auto
|
||||
from functools import cached_property, partial
|
||||
from functools import partial
|
||||
from typing import Callable, Dict, List, Optional, Protocol, Tuple, Union
|
||||
|
||||
import msgspec
|
||||
@@ -877,17 +877,13 @@ class LayerCommunicator:
|
||||
# The steps the layer's ordinary batches run.
|
||||
sides = self._declared_sides()
|
||||
self._declared = sides
|
||||
self._steps = (
|
||||
self._steps_from_declarations(
|
||||
sides,
|
||||
fusions=self._select_mlp_input_fusions(),
|
||||
force_layernorm_before_gather=force_layernorm_before_dp_gather,
|
||||
)
|
||||
if sides is not None
|
||||
else self._select_boundaries_from_scatter_modes()
|
||||
self._steps = self._steps_from_declarations(
|
||||
sides,
|
||||
fusions=self._select_mlp_input_fusions(),
|
||||
force_layernorm_before_gather=force_layernorm_before_dp_gather,
|
||||
)
|
||||
# The steps a batch that shards its tokens over attention CP runs; None
|
||||
# without CP or where the steps come from the scatter modes.
|
||||
# when no batch does.
|
||||
self._cp_steps = (
|
||||
self._steps_from_declarations(
|
||||
self._declared_sides(cp_active=True),
|
||||
@@ -895,14 +891,14 @@ class LayerCommunicator:
|
||||
force_layernorm_before_gather=force_layernorm_before_dp_gather,
|
||||
cp_moves=_cp_moves(),
|
||||
)
|
||||
if sides is not None and get_parallel().attn_cp_size > 1
|
||||
if _generic_prefill_cp_shards_tokens()
|
||||
else None
|
||||
)
|
||||
# The steps a batch with input-scattered attention runs; None where it
|
||||
# cannot run or the steps come from the scatter modes.
|
||||
# cannot run.
|
||||
self._input_scattered_steps = (
|
||||
self._steps_for_input_scattered(sides)
|
||||
if sides is not None and self._input_can_be_scattered()
|
||||
if self._input_can_be_scattered()
|
||||
else None
|
||||
)
|
||||
|
||||
@@ -963,47 +959,52 @@ class LayerCommunicator:
|
||||
runs its ordinary steps."""
|
||||
if self._declared is not None:
|
||||
return self._declared.input_rows
|
||||
if self.stage_edges is not None:
|
||||
return self.stage_edges[0].residual
|
||||
# Such a batch holds every token on each CP rank.
|
||||
return scatter_mode_layouts(
|
||||
attn_dp_size=self._context.attn_dp_size,
|
||||
attn_cp_size=1,
|
||||
attn_tp_size=self._context.attn_tp_size,
|
||||
)[self.layer_scatter_modes.layer_input_mode]
|
||||
return self.stage_edges[0].residual
|
||||
|
||||
@property
|
||||
def input_on_attention_tp_slices(self) -> bool:
|
||||
"""Whether the layer's input arrives on each attention-TP rank's slice
|
||||
of its rows (after a layer on local rows), in a batch that runs its
|
||||
ordinary steps."""
|
||||
return TokenAxis.ATTN_TP_SCATTER in self.input_rows.sharded
|
||||
|
||||
def _input_can_be_scattered(self) -> bool:
|
||||
"""Whether a batch may run this layer with input-scattered attention:
|
||||
configured, on pure TP without an a2a backend or a dense MLP on every
|
||||
rank. The rest of what ``AttnTpContext.init_context`` requires is only
|
||||
configured, on TP without attention DP, a prefill CP, an a2a backend or
|
||||
a dense MLP on every rank. The rest of what ``AttnTpContext.init_context`` requires is only
|
||||
known once the model is built."""
|
||||
parallel = get_parallel()
|
||||
return (
|
||||
parallel.enable_attn_tp_input_scattered
|
||||
and parallel.tp_size > 1
|
||||
and parallel.attn_dp_size == 1
|
||||
and parallel.attn_cp_size == 1
|
||||
and not _generic_prefill_cp_shards_tokens()
|
||||
and get_moe_a2a_backend().is_none()
|
||||
and not enable_moe_dense_fully_dp()
|
||||
)
|
||||
|
||||
def _declared_sides(
|
||||
self, *, cp_active: bool = False
|
||||
) -> Optional[DecoderLayerSides]:
|
||||
def _declared_sides(self, *, cp_active: bool = False) -> DecoderLayerSides:
|
||||
"""The declarations this layer's boundaries are chosen from: an
|
||||
attention and an FFN, in a layer whose previous layer is known, with
|
||||
attention CP only for a prefill CP whose FFN gathers over the whole CP
|
||||
group. None when the steps come from the scatter modes. ``cp_active``
|
||||
gives a batch that shards its tokens over CP; the others hold every
|
||||
token on each CP rank. An active LayerNorm SP region and input-scattered
|
||||
attention have their own."""
|
||||
attention and an FFN, in a layer whose previous layer is known.
|
||||
``cp_active`` gives a batch that shards its tokens over CP; the others
|
||||
hold every token on each CP rank. An active LayerNorm SP region and
|
||||
input-scattered attention have their own. Raises NotImplementedError
|
||||
for the combinations the steps do not cover."""
|
||||
modes = self.layer_scatter_modes
|
||||
parallel = get_parallel()
|
||||
if not (modes.is_first_layer or modes.is_previous_layer_sparse is not None):
|
||||
raise NotImplementedError(
|
||||
"a layer built without the facts of the layer before it"
|
||||
)
|
||||
# A MoE dispatched per DP shard computes on this rank's local rows and
|
||||
# hands its layer's output on there; so does a dense MLP on every rank.
|
||||
moe_on_local_rows = is_moe_input_scattered_across_dp_ranks()
|
||||
dense_on_local_rows = enable_moe_dense_fully_dp()
|
||||
|
||||
# Some batch shards its tokens over attention CP.
|
||||
cp_shards = _generic_prefill_cp_shards_tokens()
|
||||
if parallel.attn_cp_size > 1 and modes.is_layer_sparse:
|
||||
self._refuse_uncovered_cp_moe(moe_on_local_rows, cp_shards)
|
||||
# A MoE whose data-parallel groups are the CP ranks computes each CP
|
||||
# shard of a GQA prefill CP on its own ranks.
|
||||
ffn_on_cp_shards = (
|
||||
@@ -1027,32 +1028,21 @@ class LayerCommunicator:
|
||||
and bool(next_sparse)
|
||||
)
|
||||
|
||||
if not (
|
||||
(parallel.attn_cp_size == 1 or _cp_on_declarations())
|
||||
# A MoE dispatched per DP shard under attention DP and GQA prefill
|
||||
# CP keeps the scatter-mode steps: it would dispatch each rank's CP
|
||||
# shard of the tokens, which is not covered.
|
||||
and not (
|
||||
parallel.attn_cp_size > 1
|
||||
and parallel.attn_dp_size > 1
|
||||
and modes.is_layer_sparse
|
||||
and moe_on_local_rows
|
||||
and not _gathers_over_attention_cp()
|
||||
)
|
||||
and (modes.is_first_layer or modes.is_previous_layer_sparse is not None)
|
||||
):
|
||||
return None
|
||||
if parallel.attn_cp_size > 1 and _cp_moves().reduce_scatter is not None:
|
||||
if cp_shards and _cp_moves().reduce_scatter is not None:
|
||||
# A CP extend's FFN may leave its sum to the reduce-scatter that
|
||||
# takes each rank's shard back (DSA and MLA CP).
|
||||
may_leave = not cp_active
|
||||
may_leave_to_reduce_scatter = True
|
||||
else:
|
||||
elif cp_shards or parallel.attn_dp_size > 1:
|
||||
# Otherwise under CP an FFN over every CP shard completes its own sum;
|
||||
# one on its own CP shard hands on its rows as without CP.
|
||||
may_leave = may_leave_to_reduce_scatter = (
|
||||
parallel.attn_cp_size == 1 or ffn_on_cp_shards
|
||||
)
|
||||
else:
|
||||
# No batch shards its tokens and there is no attention DP: as
|
||||
# without CP.
|
||||
may_leave = may_leave_to_reduce_scatter = True
|
||||
return decoder_layer_sides(
|
||||
axis_sizes=token_axis_sizes(cp_active=cp_active),
|
||||
ffn_on_local_rows=on_local_rows(modes.is_layer_sparse),
|
||||
@@ -1081,52 +1071,32 @@ class LayerCommunicator:
|
||||
and may_leave,
|
||||
)
|
||||
|
||||
def _select_boundaries_from_scatter_modes(self) -> "BoundarySteps":
|
||||
"""The layer's steps chosen from its scatter modes, for the layers the
|
||||
declarations do not cover yet."""
|
||||
attention_input, postprocess = self._post_init_communicate()
|
||||
ffn_input, fused = self._select_mlp_input()
|
||||
modes = self.layer_scatter_modes
|
||||
ops = self._residual_ops
|
||||
ffn_rows = self._context.layouts[modes.mlp_mode]
|
||||
ffn_output = StageOutput(
|
||||
ffn_rows,
|
||||
group=SumGroup.MOE_OUTPUT if modes.is_layer_sparse else SumGroup.TP,
|
||||
leaves_for_next_layer=self.allow_deferred_ffn_reduction,
|
||||
leaves_for_reduce_scatter=self.allow_reduce_scatter,
|
||||
# A MoE block leaves its sum to reduce_scatterv whenever it applies
|
||||
# (should_skip_post_experts_all_reduce); a dense MLP does only under
|
||||
# the published mlp_reduce_scatter.
|
||||
leaves_for_reduce_scatterv=(
|
||||
self.allow_reduce_scatter or modes.is_layer_sparse
|
||||
),
|
||||
)
|
||||
return BoundarySteps(
|
||||
attention_input=attention_input,
|
||||
ffn_input=ffn_input,
|
||||
ffn_input_rows=ffn_rows,
|
||||
ffn_output=ffn_output,
|
||||
# Under attention DP the base postprocess scatters the FFN output
|
||||
# back to this rank's tokens, which the next layer's input can run
|
||||
# instead; the MHC and DSA-CP postprocess do more.
|
||||
ffn_output_move=(
|
||||
None
|
||||
if postprocess is CommunicateSummableTensorPairFn._scatter_hidden_states
|
||||
else partial(postprocess, residual_ops=ops)
|
||||
),
|
||||
# Not when the way back is the MoE-CP scatter, nor when a SCATTERED
|
||||
# FFN computes whole tokens with nothing left to sum.
|
||||
ffn_sum_is_movable=modes.mlp_mode
|
||||
not in (ScatterMode.MOE_FULL, ScatterMode.SCATTERED),
|
||||
fused=fused,
|
||||
attention_prepare=partial(
|
||||
_attention_input_step,
|
||||
layer_input=_complete_scattered_input,
|
||||
fusions=self._attn_input_fusions,
|
||||
enters_stack=modes.is_first_layer,
|
||||
residual_ops=ops,
|
||||
),
|
||||
)
|
||||
@staticmethod
|
||||
def _refuse_uncovered_cp_moe(moe_on_local_rows: bool, cp_shards: bool) -> None:
|
||||
"""A MoE layer under attention CP whose tokens the steps cannot bring
|
||||
to it: under a prefill CP, one dispatched per DP shard under attention DP
|
||||
and GQA CP, and one on the TP group whose data-parallel groups are the CP
|
||||
ranks under DSA or MLA CP; and under attention DP, one on the TP group
|
||||
whose data-parallel groups are the CP ranks."""
|
||||
parallel = get_parallel()
|
||||
gqa = not _gathers_over_attention_cp()
|
||||
if moe_on_local_rows:
|
||||
if cp_shards and gqa and parallel.attn_dp_size > 1:
|
||||
raise NotImplementedError(
|
||||
"a MoE dispatched per DP shard under attention DP and GQA "
|
||||
"prefill CP"
|
||||
)
|
||||
elif parallel.moe_dp_size == parallel.attn_cp_size:
|
||||
if cp_shards and not gqa:
|
||||
raise NotImplementedError(
|
||||
"a MoE on the TP group with moe_dp_size == attn_cp_size under "
|
||||
"DSA or MLA prefill CP"
|
||||
)
|
||||
if parallel.attn_dp_size > 1:
|
||||
raise NotImplementedError(
|
||||
"a MoE on the TP group with moe_dp_size == attn_cp_size under "
|
||||
"attention DP and attention CP"
|
||||
)
|
||||
|
||||
def _steps_for_input_scattered(self, sides: DecoderLayerSides) -> "BoundarySteps":
|
||||
"""The steps a batch with input-scattered attention runs at this layer,
|
||||
@@ -1170,28 +1140,6 @@ class LayerCommunicator:
|
||||
**kwargs,
|
||||
)
|
||||
|
||||
def _post_init_communicate(self) -> Tuple[Callable, Callable]:
|
||||
"""The attention input move and the postprocess the scatter modes
|
||||
choose."""
|
||||
if _generic_prefill_cp_shards_tokens() and _gathers_over_attention_cp():
|
||||
# These tables have no attention-CP gather.
|
||||
raise NotImplementedError(
|
||||
"a DSA or MLA prefill CP layer outside the declarations"
|
||||
)
|
||||
return (
|
||||
CommunicateSimpleFn.get_fn(
|
||||
input_mode=self.layer_scatter_modes.layer_input_mode,
|
||||
output_mode=self.layer_scatter_modes.attn_mode,
|
||||
context=self._context,
|
||||
),
|
||||
CommunicateSummableTensorPairFn.get_fn(
|
||||
hidden_states_input_mode=self.layer_scatter_modes.mlp_mode,
|
||||
residual_input_mode=self.layer_scatter_modes.middle_residual_mode,
|
||||
output_mode=self.layer_scatter_modes.layer_output_mode,
|
||||
context=self._context,
|
||||
),
|
||||
)
|
||||
|
||||
def prepare_attn_and_capture_last_layer_outputs(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
@@ -1391,42 +1339,6 @@ class LayerCommunicator:
|
||||
hidden_states, residual, use_attn_tp_group=False
|
||||
)
|
||||
|
||||
def _select_mlp_input(self) -> Tuple[Callable, Tuple["FusedMlpInput", ...]]:
|
||||
"""The attention-TP -> FFN boundary's steps, chosen from the layouts, and
|
||||
the fused kernels they try first."""
|
||||
kind = mlp_input_kind(self.layer_scatter_modes, self._context)
|
||||
residual_input_mode = self.layer_scatter_modes.layer_input_mode
|
||||
ops = self._residual_ops
|
||||
if kind is MlpInputKind.NORM:
|
||||
return partial(_mlp_input_norm, residual_ops=ops), ()
|
||||
if kind is MlpInputKind.ATTN_TP_ALL_REDUCE:
|
||||
return partial(_mlp_input_attn_tp_all_reduce, residual_ops=ops), ()
|
||||
if kind is MlpInputKind.SCATTER:
|
||||
return (
|
||||
partial(
|
||||
_mlp_input_scatter,
|
||||
scatters_residual=residual_input_mode == ScatterMode.TP_ATTN_FULL,
|
||||
residual_ops=ops,
|
||||
),
|
||||
(),
|
||||
)
|
||||
# Neither fused kernel runs under attention DP.
|
||||
fusions = (
|
||||
self._select_mlp_input_fusions() if self._context.attn_dp_size == 1 else ()
|
||||
)
|
||||
steps = partial(
|
||||
_mlp_input_gather,
|
||||
order=_mlp_input_order(
|
||||
self._context,
|
||||
residual_input_mode,
|
||||
tuple(f.run for f in fusions),
|
||||
residual_ops=ops,
|
||||
),
|
||||
)
|
||||
if kind is MlpInputKind.GATHER_MOE_CP:
|
||||
steps = partial(_mlp_input_gather_moe_cp, gather=steps)
|
||||
return steps, fusions
|
||||
|
||||
def _select_mlp_input_fusions(self) -> Tuple["FusedMlpInput", ...]:
|
||||
"""The fused kernels that can take the attention -> FFN steps, in the
|
||||
order they are tried. They add the residual plainly."""
|
||||
@@ -1537,15 +1449,6 @@ class LayerCommunicator:
|
||||
return False
|
||||
if dp_step is not None or steps.ffn_output_move_completes_sum:
|
||||
return True
|
||||
# The scatter-mode steps of a DSA or MLA CP extend (the subclasses that
|
||||
# pick their own steps). Prefill CP predicates must stay out of decode
|
||||
# graph capture.
|
||||
if (
|
||||
self._cp_steps is None
|
||||
and forward_batch.forward_mode.is_context_parallel_extend()
|
||||
and (dsa_use_prefill_cp(forward_batch) or is_mla_cp_active(forward_batch))
|
||||
):
|
||||
return True
|
||||
return get_attn_tp_context().input_scattered and not self.is_last_layer
|
||||
|
||||
def ffn_reduction_group(self, forward_batch: ForwardBatch) -> GroupCoordinator:
|
||||
@@ -1672,15 +1575,10 @@ class LayerCommunicator:
|
||||
"a branch on a batch with attention CP, LayerNorm SP or "
|
||||
"input-scattered attention"
|
||||
)
|
||||
if self._declared is not None:
|
||||
sides = self._declared
|
||||
return sides.ffn.layout, sides.ffn_residual_rows, sides.output_rows
|
||||
layouts, modes = self._context.layouts, self.layer_scatter_modes
|
||||
return (
|
||||
layouts[modes.mlp_mode],
|
||||
layouts[modes.middle_residual_mode],
|
||||
layouts[modes.layer_output_mode],
|
||||
)
|
||||
if self._declared is None:
|
||||
raise NotImplementedError("a branch on a layer that is one stage")
|
||||
sides = self._declared
|
||||
return sides.ffn.layout, sides.ffn_residual_rows, sides.output_rows
|
||||
|
||||
def branch_input(
|
||||
self,
|
||||
@@ -1846,33 +1744,6 @@ class LayerCommunicator:
|
||||
)
|
||||
|
||||
|
||||
# MOE_FULL gathers across the MoE-CP group, which spans every CP rank when CP is on.
|
||||
_SCATTER_MODE_SHARDED_AXES = {
|
||||
ScatterMode.SCATTERED: (
|
||||
TokenAxis.ATTN_DP,
|
||||
TokenAxis.ATTN_CP,
|
||||
TokenAxis.ATTN_TP_SCATTER,
|
||||
),
|
||||
ScatterMode.TP_ATTN_FULL: (TokenAxis.ATTN_DP, TokenAxis.ATTN_CP),
|
||||
ScatterMode.FULL: (TokenAxis.ATTN_CP,),
|
||||
ScatterMode.MOE_FULL: (),
|
||||
}
|
||||
|
||||
|
||||
def scatter_mode_layouts(
|
||||
*, attn_dp_size: int, attn_cp_size: int, attn_tp_size: int
|
||||
) -> Dict[ScatterMode, Layout]:
|
||||
axis_sizes = {
|
||||
TokenAxis.ATTN_DP: attn_dp_size,
|
||||
TokenAxis.ATTN_CP: attn_cp_size,
|
||||
TokenAxis.ATTN_TP_SCATTER: attn_tp_size,
|
||||
}
|
||||
return {
|
||||
mode: Layout.sharded_over(*axes, axis_sizes=axis_sizes)
|
||||
for mode, axes in _SCATTER_MODE_SHARDED_AXES.items()
|
||||
}
|
||||
|
||||
|
||||
class FfnExitFusion(Enum):
|
||||
"""What an FFN exit does when a fused kernel of the next layer's input takes
|
||||
its sum."""
|
||||
@@ -1998,20 +1869,6 @@ class CommunicateContext:
|
||||
tp_rank: int
|
||||
force_layernorm_before_dp_gather: bool = False
|
||||
|
||||
def is_same_group_size(self, a: ScatterMode, b: ScatterMode):
|
||||
return self.process_group_sizes[a] == self.process_group_sizes[b]
|
||||
|
||||
@cached_property
|
||||
def layouts(self) -> Dict[ScatterMode, Layout]:
|
||||
return scatter_mode_layouts(
|
||||
attn_dp_size=self.attn_dp_size,
|
||||
attn_cp_size=self.attn_cp_size,
|
||||
attn_tp_size=self.attn_tp_size,
|
||||
)
|
||||
|
||||
def is_same_layout(self, a: ScatterMode, b: ScatterMode):
|
||||
return self.layouts[a] == self.layouts[b]
|
||||
|
||||
@classmethod
|
||||
def init_new(cls):
|
||||
attn_tp_rank = get_parallel().attn_tp_rank
|
||||
@@ -2044,24 +1901,6 @@ class CommunicateContext:
|
||||
|
||||
|
||||
class CommunicateSimpleFn:
|
||||
@staticmethod
|
||||
def get_fn(
|
||||
input_mode: ScatterMode,
|
||||
output_mode: ScatterMode,
|
||||
context: CommunicateContext,
|
||||
):
|
||||
if context.is_same_layout(input_mode, output_mode):
|
||||
return CommunicateSimpleFn._trivial
|
||||
|
||||
if (input_mode == ScatterMode.SCATTERED) and (
|
||||
output_mode == ScatterMode.TP_ATTN_FULL
|
||||
):
|
||||
if _use_ag_after_qlora:
|
||||
return CommunicateSimpleFn._trivial
|
||||
return CommunicateSimpleFn._scattered_to_tp_attn_full
|
||||
|
||||
raise NotImplementedError(f"{input_mode=} {output_mode=}")
|
||||
|
||||
@staticmethod
|
||||
def _trivial(
|
||||
hidden_states: torch.Tensor,
|
||||
@@ -2140,6 +1979,27 @@ def moe_cp_gathered_rows(forward_batch: ForwardBatch) -> Optional[List[int]]:
|
||||
return None
|
||||
|
||||
|
||||
def sparse_moe_gathers_over_moe_cp() -> bool:
|
||||
"""Whether a sparse MoE's input is gathered over the MoE-CP group on a CP
|
||||
extend: a MoE on the TP group under GQA CP whose MoE-CP group is wider than
|
||||
the MoE's data-parallel groups. DSA and MLA CP gather over attention CP
|
||||
instead."""
|
||||
return (
|
||||
not is_moe_input_scattered_across_dp_ranks()
|
||||
and is_enable_moe_cp_allgather()
|
||||
and not _gathers_over_attention_cp()
|
||||
)
|
||||
|
||||
|
||||
def moe_cp_gathers_sparse_moe_input(forward_batch: ForwardBatch) -> bool:
|
||||
"""Whether a sparse MoE's input is gathered over the MoE-CP group on this
|
||||
batch: a CP extend, for a MoE that gathers there."""
|
||||
return (
|
||||
sparse_moe_gathers_over_moe_cp()
|
||||
and moe_cp_gathered_rows(forward_batch) is not None
|
||||
)
|
||||
|
||||
|
||||
def _redistribute_input_to_moe_cp(
|
||||
hidden_states: torch.Tensor, rows: List[int], moe_cp_size: int
|
||||
) -> torch.Tensor:
|
||||
@@ -2338,38 +2198,6 @@ def _mlp_input_dp_partial(
|
||||
return hidden_states, residual
|
||||
|
||||
|
||||
def _mlp_input_order(
|
||||
context: CommunicateContext,
|
||||
residual_input_mode: ScatterMode,
|
||||
fusions: Tuple[Callable, ...],
|
||||
residual_ops: ResidualOps = ADD_AND_NORM,
|
||||
) -> Callable:
|
||||
"""The steps from the attention output to the FFN input, chosen from facts
|
||||
fixed at construction."""
|
||||
gathers_residual = (
|
||||
residual_input_mode == ScatterMode.SCATTERED and context.attn_tp_size > 1
|
||||
)
|
||||
if context.attn_dp_size == 1:
|
||||
return partial(
|
||||
_mlp_input_without_dp,
|
||||
gathers_residual=gathers_residual,
|
||||
fusions=fusions,
|
||||
residual_ops=residual_ops,
|
||||
)
|
||||
if (
|
||||
context.force_layernorm_before_dp_gather
|
||||
or context.attn_tp_size == 1
|
||||
or not residual_ops.adds_plainly
|
||||
):
|
||||
return partial(
|
||||
_mlp_input_dp_replicate,
|
||||
gathers_residual=gathers_residual,
|
||||
reduces_attention_tp=context.attn_tp_size > 1,
|
||||
residual_ops=residual_ops,
|
||||
)
|
||||
return partial(_mlp_input_dp_partial, gathers_residual=gathers_residual)
|
||||
|
||||
|
||||
def _sum_group(group: SumGroup) -> GroupCoordinator:
|
||||
parallel = get_parallel()
|
||||
if group is SumGroup.ATTN_TP:
|
||||
@@ -2407,22 +2235,6 @@ def tbo_split_moves(layer_input_rows: Layout) -> Tuple[Callable, Callable]:
|
||||
raise NotImplementedError(f"{layer_input_rows=}")
|
||||
|
||||
|
||||
def _cp_on_declarations() -> bool:
|
||||
"""Whether attention CP is one the declarations cover: a prefill CP that
|
||||
shards tokens, with DSA or MLA attention, with the FFN input gathered over a
|
||||
MoE-CP group that is the whole CP group, or, without attention DP, with the
|
||||
MoE's data-parallel groups the CP ranks (a MoE then computes each CP shard
|
||||
on its own ranks)."""
|
||||
parallel = get_parallel()
|
||||
return _generic_prefill_cp_shards_tokens() and (
|
||||
_gathers_over_attention_cp()
|
||||
or parallel.moe_dp_size == 1
|
||||
or (
|
||||
parallel.moe_dp_size == parallel.attn_cp_size and parallel.attn_dp_size == 1
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def _gathers_over_attention_cp() -> bool:
|
||||
"""Whether a CP extend gathers the FFN input over the attention-CP group in
|
||||
equal shards and takes the output back with a reduce-scatter there: DSA and
|
||||
@@ -2788,19 +2600,6 @@ def _select_boundary_steps(
|
||||
)
|
||||
|
||||
|
||||
def _complete_scattered_input(
|
||||
hidden_states: torch.Tensor,
|
||||
residual: Optional[torch.Tensor],
|
||||
context: "CommunicateContext",
|
||||
) -> Tuple[torch.Tensor, Optional[torch.Tensor]]:
|
||||
"""The scatter-mode path's input completion: with input-scattered
|
||||
attention the layer's input is a TP partial that a reduce-scatter completes
|
||||
onto this rank's slice."""
|
||||
if get_attn_tp_context().input_scattered:
|
||||
return tp_reduce_scatter(hidden_states, residual, context)
|
||||
return hidden_states, residual
|
||||
|
||||
|
||||
def _select_attention_input_move(rows: Layout, need: StageInput) -> Callable:
|
||||
"""How the rows a layer takes become its attention's input: as they are,
|
||||
or gathered over attention TP from each rank's slice, unless the attention
|
||||
@@ -3030,61 +2829,6 @@ def _select_ffn_output_move(
|
||||
return True, None, False
|
||||
|
||||
|
||||
class MlpInputKind(Enum):
|
||||
"""What the attention-TP -> FFN boundary does, given the two sides' layouts."""
|
||||
|
||||
# The layouts agree: add the residual and normalize.
|
||||
NORM = auto()
|
||||
# All-reduce over attention TP, normalize, and gather for the FFN group.
|
||||
GATHER = auto()
|
||||
# The same, then gather over the MoE-CP group (moe_dp_size < attn_cp_size).
|
||||
GATHER_MOE_CP = auto()
|
||||
# Reduce-scatter over attention TP to this rank's tokens, then normalize.
|
||||
SCATTER = auto()
|
||||
# All-reduce over attention TP for a dense MLP on that group.
|
||||
ATTN_TP_ALL_REDUCE = auto()
|
||||
|
||||
|
||||
def mlp_input_kind(
|
||||
modes: LayerScatterModes, context: CommunicateContext
|
||||
) -> MlpInputKind:
|
||||
hidden_in, residual_in = modes.attn_mode, modes.layer_input_mode
|
||||
hidden_out, residual_out = modes.mlp_mode, modes.middle_residual_mode
|
||||
if (
|
||||
context.is_same_layout(hidden_in, hidden_out)
|
||||
and context.is_same_layout(residual_in, residual_out)
|
||||
and context.attn_tp_size == 1
|
||||
):
|
||||
return MlpInputKind.NORM
|
||||
if hidden_in == ScatterMode.TP_ATTN_FULL and residual_in in (
|
||||
ScatterMode.SCATTERED,
|
||||
ScatterMode.TP_ATTN_FULL,
|
||||
):
|
||||
kind = {
|
||||
(ScatterMode.FULL, ScatterMode.TP_ATTN_FULL): MlpInputKind.GATHER,
|
||||
(
|
||||
ScatterMode.MOE_FULL,
|
||||
ScatterMode.TP_ATTN_FULL,
|
||||
): MlpInputKind.GATHER_MOE_CP,
|
||||
(ScatterMode.SCATTERED, ScatterMode.SCATTERED): MlpInputKind.SCATTER,
|
||||
}.get((hidden_out, residual_out))
|
||||
if kind is not None:
|
||||
return kind
|
||||
if (
|
||||
hidden_out == ScatterMode.TP_ATTN_FULL
|
||||
and residual_out == ScatterMode.TP_ATTN_FULL
|
||||
and context.attn_tp_size > 1
|
||||
):
|
||||
# Used when the dense MLP is tensor-parallelized along the
|
||||
# attention TP group (``moe_dense_tp_size > 1``): hidden states
|
||||
# need an all-reduce inside the attention TP group before the
|
||||
# next layernorm, while staying in TP_ATTN_FULL on both sides.
|
||||
return MlpInputKind.ATTN_TP_ALL_REDUCE
|
||||
raise NotImplementedError(
|
||||
f"{hidden_in=} {residual_in=} {hidden_out=} {residual_out=}"
|
||||
)
|
||||
|
||||
|
||||
def _mlp_input_completing_owed(
|
||||
hidden_states: Union[torch.Tensor, "UnreducedOutput"],
|
||||
residual: torch.Tensor,
|
||||
@@ -3113,25 +2857,6 @@ def _mlp_input_norm(
|
||||
return residual_ops.update_and_read_ffn_input(hidden_states, residual, layernorm)
|
||||
|
||||
|
||||
def _mlp_input_attn_tp_all_reduce(
|
||||
hidden_states: torch.Tensor,
|
||||
residual: torch.Tensor,
|
||||
forward_batch: ForwardBatch,
|
||||
layernorm: torch.nn.Module,
|
||||
context: CommunicateContext,
|
||||
*,
|
||||
residual_ops: ResidualOps = ADD_AND_NORM,
|
||||
):
|
||||
"""All-reduce hidden states inside the attention TP group, then layernorm.
|
||||
|
||||
Used when the dense MLP shares the attention TP group
|
||||
(``moe_dense_tp_size > 1``): both hidden states and residual stay in
|
||||
``TP_ATTN_FULL`` across the boundary.
|
||||
"""
|
||||
hidden_states = get_parallel().attn_tp_group.all_reduce(hidden_states)
|
||||
return residual_ops.update_and_read_ffn_input(hidden_states, residual, layernorm)
|
||||
|
||||
|
||||
def _mlp_input_scatter(
|
||||
hidden_states: torch.Tensor,
|
||||
residual: torch.Tensor,
|
||||
@@ -3216,24 +2941,6 @@ def _tp_all_reduce_with_scattered_residual(
|
||||
return hidden_states, residual
|
||||
|
||||
|
||||
def _mlp_input_gather(
|
||||
hidden_states: torch.Tensor,
|
||||
residual: torch.Tensor,
|
||||
forward_batch: ForwardBatch,
|
||||
layernorm: torch.nn.Module,
|
||||
context: CommunicateContext,
|
||||
*,
|
||||
order: Callable,
|
||||
):
|
||||
"""Run ``order``, the steps ``_mlp_input_order`` chose at construction,
|
||||
unless this batch's attention input is scattered."""
|
||||
if get_attn_tp_context().input_scattered:
|
||||
return _tp_all_reduce_with_scattered_residual(
|
||||
hidden_states, residual, layernorm, context
|
||||
)
|
||||
return order(hidden_states, residual, forward_batch, layernorm, context)
|
||||
|
||||
|
||||
def _mlp_input_gather_attention_cp(
|
||||
hidden_states: torch.Tensor,
|
||||
residual: torch.Tensor,
|
||||
@@ -3375,47 +3082,6 @@ def _all_reduce_then_to_local_tokens(
|
||||
class CommunicateSummableTensorPairFn:
|
||||
"""It is allowed to make (hidden_states, residual) := (hidden_states + residual, None) if needed."""
|
||||
|
||||
@staticmethod
|
||||
def get_fn(
|
||||
hidden_states_input_mode: ScatterMode,
|
||||
residual_input_mode: ScatterMode,
|
||||
output_mode: ScatterMode,
|
||||
context: CommunicateContext,
|
||||
):
|
||||
if context.is_same_layout(
|
||||
hidden_states_input_mode, output_mode
|
||||
) and context.is_same_layout(residual_input_mode, output_mode):
|
||||
return CommunicateSummableTensorPairFn._trivial
|
||||
|
||||
fn = {
|
||||
(
|
||||
ScatterMode.FULL,
|
||||
ScatterMode.TP_ATTN_FULL,
|
||||
ScatterMode.TP_ATTN_FULL,
|
||||
): CommunicateSummableTensorPairFn._scatter_hidden_states,
|
||||
(
|
||||
ScatterMode.SCATTERED,
|
||||
ScatterMode.SCATTERED,
|
||||
ScatterMode.TP_ATTN_FULL,
|
||||
): CommunicateSummableTensorPairFn._gather,
|
||||
(
|
||||
ScatterMode.TP_ATTN_FULL,
|
||||
ScatterMode.TP_ATTN_FULL,
|
||||
ScatterMode.SCATTERED,
|
||||
): CommunicateSummableTensorPairFn._scatter,
|
||||
(
|
||||
ScatterMode.MOE_FULL,
|
||||
ScatterMode.TP_ATTN_FULL,
|
||||
ScatterMode.TP_ATTN_FULL,
|
||||
): CommunicateSummableTensorPairFn._scatter_hidden_states_moe,
|
||||
}.get((hidden_states_input_mode, residual_input_mode, output_mode))
|
||||
if fn is not None:
|
||||
return fn
|
||||
|
||||
raise NotImplementedError(
|
||||
f"{hidden_states_input_mode=} {residual_input_mode=} {output_mode=}"
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _trivial(
|
||||
hidden_states: torch.Tensor,
|
||||
|
||||
@@ -22,7 +22,7 @@ from sglang.srt.layers.communicator import (
|
||||
BoundarySteps,
|
||||
LayerCommunicator,
|
||||
LayerScatterModes,
|
||||
ScatterMode,
|
||||
sparse_moe_gathers_over_moe_cp,
|
||||
)
|
||||
|
||||
|
||||
@@ -169,6 +169,12 @@ class MHCLayerCommunicator(LayerCommunicator):
|
||||
hc_ffn_post_pre=hc_ffn_post_pre,
|
||||
is_last_layer=layer_scatter_modes.is_last_layer,
|
||||
)
|
||||
if layer_scatter_modes.is_layer_sparse and sparse_moe_gathers_over_moe_cp():
|
||||
raise NotImplementedError(
|
||||
"MHCLayerCommunicator does not support a MoE gathered over the "
|
||||
"MoE-CP group (moe_dp_size < attention_context_parallel_size). "
|
||||
"Increase moe_dp_size to match attention_context_parallel_size."
|
||||
)
|
||||
# The postprocess writes the FFN output into the streams, so the FFN's
|
||||
# sum never waits for the next layer.
|
||||
super().__init__(
|
||||
@@ -180,6 +186,11 @@ class MHCLayerCommunicator(LayerCommunicator):
|
||||
allow_deferred_ffn_reduction=False,
|
||||
residual_ops=self.mhc,
|
||||
)
|
||||
# MHC has not been run with input-scattered attention under attention CP.
|
||||
if self._context.attn_cp_size > 1 and self._input_can_be_scattered():
|
||||
raise NotImplementedError(
|
||||
"MHCLayerCommunicator with input-scattered attention under attention CP"
|
||||
)
|
||||
|
||||
def _steps_from_declarations(
|
||||
self, sides: DecoderLayerSides, **kwargs
|
||||
@@ -193,12 +204,3 @@ class MHCLayerCommunicator(LayerCommunicator):
|
||||
f"MHCLayerCommunicator with a gather over attention CP: {sides=}"
|
||||
)
|
||||
return super()._steps_from_declarations(sides, **kwargs)
|
||||
|
||||
def _post_init_communicate(self):
|
||||
if self.layer_scatter_modes.mlp_mode == ScatterMode.MOE_FULL:
|
||||
raise NotImplementedError(
|
||||
"MHCLayerCommunicator does not support MOE_FULL "
|
||||
"(moe_dp_size < attention_context_parallel_size). Increase "
|
||||
"moe_dp_size to match attention_context_parallel_size."
|
||||
)
|
||||
return super()._post_init_communicate()
|
||||
|
||||
@@ -1230,23 +1230,19 @@ class ForwardBatch(ForwardBatchDeepSeekMHAMixin):
|
||||
def moe_num_token_non_padded(self) -> Optional[torch.Tensor]:
|
||||
"""Bound for masking a sparse MoE's padded rows, or None when the MoE
|
||||
input is a gathered buffer whose real rows are not a prefix of it."""
|
||||
from sglang.srt.layers.communicator import (
|
||||
ScatterMode,
|
||||
moe_cp_gathered_rows,
|
||||
sparse_mlp_scatter_mode,
|
||||
)
|
||||
from sglang.srt.layers.communicator import moe_cp_gathers_sparse_moe_input
|
||||
from sglang.srt.layers.moe.utils import is_moe_input_scattered_across_dp_ranks
|
||||
|
||||
if self.num_token_non_padded is None:
|
||||
return None
|
||||
|
||||
mode = sparse_mlp_scatter_mode()
|
||||
if mode == ScatterMode.SCATTERED:
|
||||
if is_moe_input_scattered_across_dp_ranks():
|
||||
# a2a dispatch, FP4 all-gather and dwdp all route the local shard.
|
||||
return self.num_token_non_padded
|
||||
if mode == ScatterMode.MOE_FULL and moe_cp_gathered_rows(self) is not None:
|
||||
if moe_cp_gathers_sparse_moe_input(self):
|
||||
return None
|
||||
# DSA / MLA CP take the FULL mode but all-gather across CP on a prefill,
|
||||
# which leaves the real rows zigzag-permuted rather than in a prefix.
|
||||
# DSA / MLA CP all-gather across attention CP on a prefill, which leaves
|
||||
# the real rows zigzag-permuted rather than in a prefix.
|
||||
if get_parallel().attn_cp_size > 1 and self._moe_input_gathered_across_cp():
|
||||
return None
|
||||
if get_parallel().attn_dp_size != 1:
|
||||
|
||||
@@ -2288,7 +2288,7 @@ class DeepseekV2AttentionMLA(
|
||||
hidden_states: torch.Tensor,
|
||||
forward_batch: ForwardBatch,
|
||||
zero_allocator: BumpAllocator,
|
||||
layer_scatter_modes: LayerScatterModes = None,
|
||||
input_on_attention_tp_slices: bool = False,
|
||||
llama_4_scaling: Optional[torch.Tensor] = None,
|
||||
prev_topk_indices: Optional[torch.Tensor] = None,
|
||||
):
|
||||
@@ -2297,7 +2297,7 @@ class DeepseekV2AttentionMLA(
|
||||
hidden_states=hidden_states,
|
||||
forward_batch=forward_batch,
|
||||
zero_allocator=zero_allocator,
|
||||
layer_scatter_modes=layer_scatter_modes,
|
||||
input_on_attention_tp_slices=input_on_attention_tp_slices,
|
||||
llama_4_scaling=llama_4_scaling,
|
||||
prev_topk_indices=prev_topk_indices,
|
||||
)
|
||||
@@ -2309,7 +2309,7 @@ class DeepseekV2AttentionMLA(
|
||||
hidden_states: torch.Tensor,
|
||||
forward_batch: ForwardBatch,
|
||||
zero_allocator: BumpAllocator,
|
||||
layer_scatter_modes: LayerScatterModes = None,
|
||||
input_on_attention_tp_slices: bool = False,
|
||||
llama_4_scaling: Optional[torch.Tensor] = None,
|
||||
prev_topk_indices: Optional[torch.Tensor] = None,
|
||||
):
|
||||
@@ -2394,7 +2394,7 @@ class DeepseekV2AttentionMLA(
|
||||
hidden_states,
|
||||
forward_batch,
|
||||
zero_allocator,
|
||||
layer_scatter_modes,
|
||||
input_on_attention_tp_slices,
|
||||
)
|
||||
elif attn_forward_method == AttnForwardMethod.MLA_NPU:
|
||||
inner_state = forward_mla_prepare_npu(
|
||||
@@ -2403,7 +2403,7 @@ class DeepseekV2AttentionMLA(
|
||||
hidden_states,
|
||||
forward_batch,
|
||||
zero_allocator,
|
||||
layer_scatter_modes,
|
||||
input_on_attention_tp_slices,
|
||||
)
|
||||
elif attn_forward_method == AttnForwardMethod.DSA_NPU:
|
||||
inner_state = forward_dsa_prepare_npu(
|
||||
@@ -2412,7 +2412,7 @@ class DeepseekV2AttentionMLA(
|
||||
hidden_states,
|
||||
forward_batch,
|
||||
zero_allocator,
|
||||
layer_scatter_modes,
|
||||
input_on_attention_tp_slices,
|
||||
prev_topk_indices,
|
||||
)
|
||||
else:
|
||||
@@ -2724,7 +2724,9 @@ class DeepseekV2DecoderLayer(nn.Module):
|
||||
forward_batch=forward_batch,
|
||||
zero_allocator=zero_allocator,
|
||||
llama_4_scaling=llama_4_scaling,
|
||||
layer_scatter_modes=self.layer_scatter_modes,
|
||||
input_on_attention_tp_slices=(
|
||||
self.layer_communicator.input_on_attention_tp_slices
|
||||
),
|
||||
prev_topk_indices=prev_topk_indices,
|
||||
)
|
||||
if isinstance(hidden_states, tuple):
|
||||
|
||||
@@ -479,7 +479,9 @@ class GigaChat35DecoderLayer(deepseek_v2.DeepseekV2DecoderLayer):
|
||||
hidden_states=hidden_states,
|
||||
forward_batch=forward_batch,
|
||||
zero_allocator=zero_allocator,
|
||||
layer_scatter_modes=self.layer_scatter_modes,
|
||||
input_on_attention_tp_slices=(
|
||||
self.layer_communicator.input_on_attention_tp_slices
|
||||
),
|
||||
)
|
||||
get_attn_tp_context().clear_attn_inputs()
|
||||
|
||||
|
||||
@@ -638,7 +638,9 @@ class Glm4MoeLiteDecoderLayer(nn.Module):
|
||||
hidden_states=hidden_states,
|
||||
forward_batch=forward_batch,
|
||||
zero_allocator=zero_allocator,
|
||||
layer_scatter_modes=self.layer_scatter_modes,
|
||||
input_on_attention_tp_slices=(
|
||||
self.layer_communicator.input_on_attention_tp_slices
|
||||
),
|
||||
)
|
||||
if isinstance(hidden_states, tuple):
|
||||
hidden_states = hidden_states[0]
|
||||
|
||||
@@ -932,7 +932,9 @@ class Glm5NextDecoderLayer(nn.Module):
|
||||
hidden_states=hidden_states,
|
||||
forward_batch=forward_batch,
|
||||
zero_allocator=zero_allocator,
|
||||
layer_scatter_modes=self.layer_scatter_modes,
|
||||
input_on_attention_tp_slices=(
|
||||
self.layer_communicator.input_on_attention_tp_slices
|
||||
),
|
||||
prev_topk_indices=prev_topk_indices,
|
||||
)
|
||||
if isinstance(hidden_states, tuple):
|
||||
|
||||
Reference in New Issue
Block a user