[Refactor] Replace LayerScatterModes with LayerFacts and remove ScatterMode (#41443)
This commit is contained in:
@@ -42,14 +42,13 @@ from sglang.srt.layers.communicator.layer import (
|
||||
FfnExit,
|
||||
FfnExitFusion,
|
||||
LayerCommunicator,
|
||||
LayerScatterModes,
|
||||
LayerFacts,
|
||||
MHCLayerCommunicator,
|
||||
MixerExit,
|
||||
)
|
||||
from sglang.srt.layers.communicator.layout import (
|
||||
CommunicateContext,
|
||||
Layout,
|
||||
ScatterMode,
|
||||
SumGroup,
|
||||
TokenAxis,
|
||||
enable_moe_dense_fully_dp,
|
||||
@@ -98,7 +97,7 @@ __all__ = [
|
||||
"HandoffOutput",
|
||||
"LayerCommunicator",
|
||||
"LayerResidual",
|
||||
"LayerScatterModes",
|
||||
"LayerFacts",
|
||||
"LayerStage",
|
||||
"Layout",
|
||||
"MHCLayerCommunicator",
|
||||
@@ -107,7 +106,6 @@ __all__ = [
|
||||
"NORM_QUANT_READ",
|
||||
"NORM_READ",
|
||||
"PLAIN_RESIDUAL",
|
||||
"ScatterMode",
|
||||
"StageDecl",
|
||||
"StageEntry",
|
||||
"StageInput",
|
||||
|
||||
@@ -24,7 +24,6 @@ import torch
|
||||
from sglang.srt.distributed import GroupCoordinator
|
||||
from sglang.srt.environ import envs
|
||||
from sglang.srt.layers import layernorm_sp
|
||||
from sglang.srt.layers.attention.dsa.utils import is_dsa_enable_prefill_cp
|
||||
from sglang.srt.layers.aux_hidden_states import AuxHiddenStateAccumulator
|
||||
from sglang.srt.layers.communicator.adapters.attention import get_attn_tp_context
|
||||
from sglang.srt.layers.communicator.boundary import (
|
||||
@@ -47,7 +46,6 @@ from sglang.srt.layers.communicator.boundary import (
|
||||
from sglang.srt.layers.communicator.layout import (
|
||||
CommunicateContext,
|
||||
Layout,
|
||||
ScatterMode,
|
||||
SumGroup,
|
||||
TokenAxis,
|
||||
_batch_shards_over_cp,
|
||||
@@ -56,7 +54,6 @@ from sglang.srt.layers.communicator.layout import (
|
||||
_gathers_over_attention_cp,
|
||||
_generic_prefill_cp_shards_tokens,
|
||||
_sum_group,
|
||||
enable_dwdp,
|
||||
enable_moe_dense_fully_dp,
|
||||
sparse_moe_gathers_over_moe_cp,
|
||||
token_axis_sizes,
|
||||
@@ -83,7 +80,6 @@ from sglang.srt.layers.communicator.residual.add_norm import (
|
||||
apply_flashinfer_allreduce_fusion,
|
||||
)
|
||||
from sglang.srt.layers.communicator.residual.mhc import MHCState
|
||||
from sglang.srt.layers.cp.utils import is_mla_cp_enabled
|
||||
from sglang.srt.layers.dp_attention import (
|
||||
is_dp_attention_enabled,
|
||||
is_enable_moe_cp_allgather,
|
||||
@@ -94,7 +90,6 @@ from sglang.srt.layers.moe import (
|
||||
is_moe_input_scattered_across_dp_ranks,
|
||||
post_experts_reduction_group,
|
||||
post_experts_sum_is_one_all_reduce,
|
||||
should_use_flashinfer_cutlass_moe_fp4_allgather,
|
||||
)
|
||||
from sglang.srt.model_executor.forward_batch_info import ForwardBatch
|
||||
from sglang.srt.runtime_context import (
|
||||
@@ -113,131 +108,39 @@ _use_ag_after_qlora = envs.SGLANG_USE_AG_AFTER_QLORA.get()
|
||||
|
||||
|
||||
@dataclass
|
||||
class _LayerModeComputationContext:
|
||||
num_layers: int
|
||||
layer_id: int
|
||||
is_layer_sparse: bool
|
||||
is_previous_layer_sparse: Optional[bool]
|
||||
is_next_layer_sparse: Optional[bool]
|
||||
class LayerFacts:
|
||||
"""Where a layer sits in the model and whether it and its neighbours have a
|
||||
sparse MLP: the facts its boundaries are declared from."""
|
||||
|
||||
def previous_layer(self):
|
||||
assert self.is_previous_layer_sparse is not None
|
||||
return _LayerModeComputationContext(
|
||||
num_layers=self.num_layers,
|
||||
layer_id=self.layer_id - 1,
|
||||
is_layer_sparse=self.is_previous_layer_sparse,
|
||||
is_previous_layer_sparse=None,
|
||||
is_next_layer_sparse=self.is_layer_sparse,
|
||||
)
|
||||
|
||||
|
||||
def sparse_mlp_scatter_mode() -> ScatterMode:
|
||||
"""SCATTERED hands a sparse MLP this rank's own token shard; FULL and
|
||||
MOE_FULL hand it a buffer gathered over the attn-TP or MoE-CP group."""
|
||||
if (
|
||||
# Token dispatch/combine will be handled outside of LayerCommunicator for these modes.
|
||||
not get_moe_a2a_backend().is_none()
|
||||
or should_use_flashinfer_cutlass_moe_fp4_allgather()
|
||||
or enable_dwdp()
|
||||
):
|
||||
return ScatterMode.SCATTERED
|
||||
# DSA CP and MLA CP both don't support MOE_FULL yet; fall back to FULL.
|
||||
if is_enable_moe_cp_allgather() and not (
|
||||
is_dsa_enable_prefill_cp() or is_mla_cp_enabled()
|
||||
):
|
||||
return ScatterMode.MOE_FULL
|
||||
return ScatterMode.FULL
|
||||
|
||||
|
||||
@dataclass
|
||||
class LayerScatterModes:
|
||||
layer_input_mode: ScatterMode
|
||||
attn_mode: ScatterMode
|
||||
# Can be further split into e.g. mlp_input_mode and mlp_output_mode if needed
|
||||
mlp_mode: ScatterMode
|
||||
middle_residual_mode: ScatterMode
|
||||
layer_output_mode: ScatterMode
|
||||
is_layer_sparse: bool = False
|
||||
# The model's first layer: its input is the embedding, not a layer output.
|
||||
is_first_layer: bool = False
|
||||
# The model's last layer: its output goes to the final norm, not a next layer.
|
||||
is_last_layer: bool = False
|
||||
# Whether the layer before this one has a sparse MLP; None when the modes
|
||||
# Whether the layer before this one has a sparse MLP; None when the facts
|
||||
# were given directly, not planned from the layer sequence.
|
||||
is_previous_layer_sparse: Optional[bool] = None
|
||||
# Whether the layer after this one has a sparse MLP; None likewise.
|
||||
is_next_layer_sparse: Optional[bool] = None
|
||||
|
||||
@classmethod
|
||||
def init_new(cls, **kwargs):
|
||||
context = _LayerModeComputationContext(**kwargs)
|
||||
def init_new(
|
||||
cls,
|
||||
*,
|
||||
layer_id: int,
|
||||
num_layers: int,
|
||||
is_layer_sparse: bool,
|
||||
is_previous_layer_sparse: Optional[bool],
|
||||
is_next_layer_sparse: Optional[bool],
|
||||
) -> "LayerFacts":
|
||||
return cls(
|
||||
layer_input_mode=cls._compute_layer_input_mode(context),
|
||||
attn_mode=ScatterMode.TP_ATTN_FULL,
|
||||
mlp_mode=cls._compute_mlp_mode(context),
|
||||
middle_residual_mode=cls._compute_middle_residual_mode(context),
|
||||
layer_output_mode=cls._compute_layer_output_mode(context),
|
||||
is_layer_sparse=context.is_layer_sparse,
|
||||
is_first_layer=context.layer_id == 0,
|
||||
is_last_layer=context.layer_id == context.num_layers - 1,
|
||||
is_previous_layer_sparse=context.is_previous_layer_sparse,
|
||||
is_next_layer_sparse=context.is_next_layer_sparse,
|
||||
is_layer_sparse=is_layer_sparse,
|
||||
is_first_layer=layer_id == 0,
|
||||
is_last_layer=layer_id == num_layers - 1,
|
||||
is_previous_layer_sparse=is_previous_layer_sparse,
|
||||
is_next_layer_sparse=is_next_layer_sparse,
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def _compute_layer_input_mode(cls, context: _LayerModeComputationContext):
|
||||
if context.layer_id == 0:
|
||||
return ScatterMode.model_input_output()
|
||||
return cls._compute_layer_output_mode(context.previous_layer())
|
||||
|
||||
@classmethod
|
||||
def _compute_mlp_mode(cls, context: _LayerModeComputationContext):
|
||||
if context.is_layer_sparse:
|
||||
return sparse_mlp_scatter_mode()
|
||||
else:
|
||||
if enable_moe_dense_fully_dp():
|
||||
return ScatterMode.SCATTERED
|
||||
# A TP-sharded dense MLP reduces over the whole TP group, which spans
|
||||
# every CP rank; a CP-sharded prefill must gather tokens across CP
|
||||
# first or the all-reduce sums different tokens' partial outputs.
|
||||
# MLA/DSA CP models gather over the attention-CP group instead.
|
||||
if _generic_prefill_cp_shards_tokens() and not (
|
||||
is_dsa_enable_prefill_cp() or is_mla_cp_enabled()
|
||||
):
|
||||
return ScatterMode.MOE_FULL
|
||||
return ScatterMode.FULL
|
||||
|
||||
@classmethod
|
||||
def _should_gather_for_tbo(cls, context: _LayerModeComputationContext):
|
||||
return (
|
||||
not context.is_layer_sparse
|
||||
and context.is_next_layer_sparse
|
||||
and enable_moe_dense_fully_dp()
|
||||
and get_exec().overlap.enable_two_batch_overlap
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def _compute_middle_residual_mode(cls, context: _LayerModeComputationContext):
|
||||
mlp_mode = cls._compute_mlp_mode(context)
|
||||
if mlp_mode == ScatterMode.SCATTERED:
|
||||
return ScatterMode.SCATTERED
|
||||
if mlp_mode in (ScatterMode.FULL, ScatterMode.MOE_FULL):
|
||||
return ScatterMode.TP_ATTN_FULL
|
||||
raise NotImplementedError
|
||||
|
||||
@classmethod
|
||||
def _compute_layer_output_mode(cls, context: _LayerModeComputationContext):
|
||||
mlp_mode = cls._compute_mlp_mode(context)
|
||||
if context.layer_id == context.num_layers - 1:
|
||||
return ScatterMode.model_input_output()
|
||||
if mlp_mode == ScatterMode.SCATTERED:
|
||||
if cls._should_gather_for_tbo(context):
|
||||
return ScatterMode.TP_ATTN_FULL
|
||||
return ScatterMode.SCATTERED
|
||||
if mlp_mode in (ScatterMode.FULL, ScatterMode.MOE_FULL):
|
||||
return ScatterMode.TP_ATTN_FULL
|
||||
raise NotImplementedError
|
||||
|
||||
|
||||
def _unfused_completion_matches_the_ffn(forward_batch: ForwardBatch) -> bool:
|
||||
"""Whether the next layer's all-reduce is the one the FFN would have run
|
||||
@@ -329,7 +232,7 @@ class LayerCommunicator:
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
layer_scatter_modes: LayerScatterModes,
|
||||
layer_facts: LayerFacts,
|
||||
input_layernorm: torch.nn.Module,
|
||||
post_attention_layernorm: torch.nn.Module,
|
||||
# Reduce scatter requires skipping all-reduce in model code after MoE/MLP, so only enable for models which have that implemented. Remove flag once done for all models that use LayerCommunicator.
|
||||
@@ -349,11 +252,11 @@ class LayerCommunicator:
|
||||
# The fused kernels a backend gives the layer, tried before its own.
|
||||
fusions: Optional[LayerFusions] = None,
|
||||
):
|
||||
self.layer_scatter_modes = layer_scatter_modes
|
||||
self.layer_facts = layer_facts
|
||||
self.input_layernorm = input_layernorm
|
||||
self.post_attention_layernorm = post_attention_layernorm
|
||||
self.allow_reduce_scatter = allow_reduce_scatter
|
||||
self.is_last_layer = layer_scatter_modes.is_last_layer
|
||||
self.is_last_layer = layer_facts.is_last_layer
|
||||
self.qkv_latent_func = qkv_latent_func
|
||||
self.force_layernorm_before_dp_gather = force_layernorm_before_dp_gather
|
||||
self.enable_fused_ar_quant = enable_fused_ar_quant
|
||||
@@ -425,7 +328,7 @@ class LayerCommunicator:
|
||||
residual,
|
||||
),
|
||||
attention_fusions=self._attn_input_fusions,
|
||||
enters_stack=self.layer_scatter_modes.is_first_layer,
|
||||
enters_stack=self.layer_facts.is_first_layer,
|
||||
)
|
||||
if layernorm_sp.layernorm_sp_enabled()
|
||||
else None
|
||||
@@ -514,7 +417,7 @@ class LayerCommunicator:
|
||||
hold every token on each CP rank. An active LayerNorm SP region and
|
||||
input-scattered attention have their own. Raises NotImplementedError
|
||||
for the combinations the steps do not cover."""
|
||||
modes = self.layer_scatter_modes
|
||||
modes = self.layer_facts
|
||||
parallel = get_parallel()
|
||||
if not (modes.is_first_layer or modes.is_previous_layer_sparse is not None):
|
||||
raise NotImplementedError(
|
||||
@@ -639,7 +542,7 @@ class LayerCommunicator:
|
||||
scattered = scattered_residual_layer_sides(
|
||||
axis_sizes=token_axis_sizes(),
|
||||
ffn_group=sides.ffn_output.group,
|
||||
is_first_layer=self.layer_scatter_modes.is_first_layer,
|
||||
is_first_layer=self.layer_facts.is_first_layer,
|
||||
is_last_layer=self.is_last_layer,
|
||||
leaves_for_reduce_scatter=self.allow_reduce_scatter,
|
||||
)
|
||||
@@ -649,7 +552,7 @@ class LayerCommunicator:
|
||||
with_residual(scattered, self._residual),
|
||||
attention_handoff=handoff,
|
||||
attention_fusions=self._attn_input_fusions,
|
||||
enters_stack=self.layer_scatter_modes.is_first_layer,
|
||||
enters_stack=self.layer_facts.is_first_layer,
|
||||
)
|
||||
|
||||
def _steps_from_declarations(
|
||||
@@ -659,7 +562,7 @@ class LayerCommunicator:
|
||||
return _select_boundary_steps(
|
||||
sides,
|
||||
attention_fusions=self._attn_input_fusions,
|
||||
enters_stack=self.layer_scatter_modes.is_first_layer,
|
||||
enters_stack=self.layer_facts.is_first_layer,
|
||||
**kwargs,
|
||||
)
|
||||
|
||||
@@ -748,7 +651,7 @@ class LayerCommunicator:
|
||||
# The SP region opens at the first layer, re-evaluated per forward so a
|
||||
# crash mid-loop cannot leak into the next one. It sets what the batch's
|
||||
# steps are chosen from, and the first layer's input owes nothing.
|
||||
if self._sp_steps is not None and self.layer_scatter_modes.is_first_layer:
|
||||
if self._sp_steps is not None and self.layer_facts.is_first_layer:
|
||||
get_forward().set(
|
||||
"sp_active", layernorm_sp.runs_sp(forward_batch.forward_mode)
|
||||
)
|
||||
@@ -1041,7 +944,7 @@ class LayerCommunicator:
|
||||
forward_batch=forward_batch,
|
||||
context=self._context,
|
||||
allow_reduce_scatter=self.allow_reduce_scatter,
|
||||
is_layer_sparse=self.layer_scatter_modes.is_layer_sparse,
|
||||
is_layer_sparse=self.layer_facts.is_layer_sparse,
|
||||
)
|
||||
update = steps.ffn_output.update
|
||||
if residual is not None and update.at_producer:
|
||||
@@ -1136,13 +1039,6 @@ class LayerCommunicator:
|
||||
return hidden_states, residual
|
||||
return reduce_output(hidden_states), residual
|
||||
|
||||
def should_use_reduce_scatter(self, forward_batch: ForwardBatch):
|
||||
"""Whether the FFN leaves its sum to a reduce-scatter, for layers that run
|
||||
their FFN outside ffn_exit."""
|
||||
return self._ffn_leaves_sum_to_reduce_scatter(
|
||||
forward_batch, self._postprocess_dp_step(forward_batch)
|
||||
)
|
||||
|
||||
def _select_ffn_exit_fusions(
|
||||
self,
|
||||
) -> Tuple[Callable[[ForwardBatch], Optional["FfnExitFusion"]], ...]:
|
||||
@@ -1359,7 +1255,7 @@ class MHCLayerCommunicator(LayerCommunicator):
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
layer_scatter_modes: LayerScatterModes,
|
||||
layer_facts: LayerFacts,
|
||||
input_layernorm: torch.nn.Module,
|
||||
post_attention_layernorm: torch.nn.Module,
|
||||
allow_reduce_scatter: bool = False,
|
||||
@@ -1377,9 +1273,9 @@ class MHCLayerCommunicator(LayerCommunicator):
|
||||
hc_ffn_pre=hc_ffn_pre,
|
||||
hc_post=hc_post,
|
||||
hc_ffn_post_pre=hc_ffn_post_pre,
|
||||
is_last_layer=layer_scatter_modes.is_last_layer,
|
||||
is_last_layer=layer_facts.is_last_layer,
|
||||
)
|
||||
if layer_scatter_modes.is_layer_sparse and sparse_moe_gathers_over_moe_cp():
|
||||
if layer_facts.is_layer_sparse and sparse_moe_gathers_over_moe_cp():
|
||||
raise NotImplementedError(
|
||||
"MHCLayerCommunicator does not support a MoE gathered over the "
|
||||
"MoE-CP group (moe_dp_size < attention_context_parallel_size). "
|
||||
@@ -1388,7 +1284,7 @@ class MHCLayerCommunicator(LayerCommunicator):
|
||||
# The postprocess writes the FFN output into the streams, so the FFN's
|
||||
# sum never waits for the next layer.
|
||||
super().__init__(
|
||||
layer_scatter_modes,
|
||||
layer_facts,
|
||||
input_layernorm,
|
||||
post_attention_layernorm,
|
||||
allow_reduce_scatter,
|
||||
|
||||
@@ -71,30 +71,6 @@ class SumGroup(Enum):
|
||||
MOE_OUTPUT = auto()
|
||||
|
||||
|
||||
class ScatterMode(Enum):
|
||||
"""
|
||||
Suppose we have TP=4, DP=2, enable-dp-attention, and the system handles seq a,b,c,d
|
||||
Model input/output: [ab, ab, cd, cd] for four ranks respectively
|
||||
SCATTERED: [a, b, c, d]
|
||||
TP_ATTN_FULL: [ab, ab, cd, cd], i.e. all ranks inside a TP attn group have full data of the group
|
||||
FULL: [abcd, abcd, abcd, abcd]
|
||||
MOE_FULL: full within the MoE group (cp_per_moe CP chunks), used when moe_dp_size < attn_cp_size
|
||||
"""
|
||||
|
||||
SCATTERED = auto()
|
||||
TP_ATTN_FULL = auto()
|
||||
FULL = auto()
|
||||
MOE_FULL = auto()
|
||||
|
||||
@staticmethod
|
||||
def model_input_output():
|
||||
"""The scatter mode for model forward pass input and output data"""
|
||||
if is_dsa_enable_prefill_cp() or is_mla_cp_enabled():
|
||||
return ScatterMode.SCATTERED
|
||||
|
||||
return ScatterMode.TP_ATTN_FULL
|
||||
|
||||
|
||||
def enable_moe_dense_fully_dp():
|
||||
return get_parallel().moe_dense_tp_size == 1
|
||||
|
||||
@@ -105,10 +81,6 @@ def _generic_prefill_cp_shards_tokens() -> bool:
|
||||
return parallel.attn_cp_size > 1 and parallel.enable_prefill_cp
|
||||
|
||||
|
||||
def enable_dwdp():
|
||||
return get_parallel().dwdp_size > 1
|
||||
|
||||
|
||||
def _batch_size(forward_batch: ForwardBatch) -> int:
|
||||
return (
|
||||
forward_batch.input_ids.shape[0] if hasattr(forward_batch, "input_ids") else 0
|
||||
@@ -124,7 +96,6 @@ def _ffn_has_tokens(forward_batch: ForwardBatch) -> bool:
|
||||
|
||||
@dataclass
|
||||
class CommunicateContext:
|
||||
process_group_sizes: Dict[ScatterMode, int]
|
||||
attn_tp_rank: int
|
||||
attn_tp_size: int
|
||||
attn_dp_size: int
|
||||
@@ -144,18 +115,7 @@ class CommunicateContext:
|
||||
attn_cp_rank = get_parallel().attn_cp_rank
|
||||
tp_size = get_parallel().tp_size
|
||||
tp_rank = get_parallel().tp_rank
|
||||
moe_cp_size = get_moe_cp_size()
|
||||
process_group_sizes = {
|
||||
ScatterMode.SCATTERED: 1,
|
||||
ScatterMode.TP_ATTN_FULL: attn_tp_size,
|
||||
# TODO: support --moe-dense-tp-size > 1
|
||||
# With context parallel enabled, we should exclude
|
||||
# the attn_cp_size from the total tp_size
|
||||
ScatterMode.FULL: tp_size // attn_cp_size,
|
||||
ScatterMode.MOE_FULL: tp_size // (attn_cp_size // moe_cp_size),
|
||||
}
|
||||
return cls(
|
||||
process_group_sizes=process_group_sizes,
|
||||
attn_tp_rank=attn_tp_rank,
|
||||
attn_tp_size=attn_tp_size,
|
||||
attn_dp_size=attn_dp_size,
|
||||
|
||||
@@ -33,7 +33,7 @@ from sglang.srt.eplb.expert_location_dispatch import ExpertLocationDispatchInfo
|
||||
from sglang.srt.layers.activation import SiluAndMul
|
||||
from sglang.srt.layers.communicator import (
|
||||
LayerCommunicator,
|
||||
LayerScatterModes,
|
||||
LayerFacts,
|
||||
enable_moe_dense_fully_dp,
|
||||
reduce_output,
|
||||
)
|
||||
@@ -669,7 +669,7 @@ class BailingMoEBlock(nn.Module):
|
||||
config, layer_id=layer_id + 1, is_nextn=False
|
||||
)
|
||||
|
||||
self.layer_scatter_modes = LayerScatterModes.init_new(
|
||||
self.layer_facts = LayerFacts.init_new(
|
||||
layer_id=layer_id,
|
||||
# A NextN draft is a one-layer model.
|
||||
num_layers=1 if is_nextn else config.num_hidden_layers,
|
||||
@@ -703,7 +703,7 @@ class BailingMoEBlock(nn.Module):
|
||||
self.post_attention_layernorm = RMSNorm(hidden_size, eps=config.rms_norm_eps)
|
||||
|
||||
self.layer_communicator = LayerCommunicator(
|
||||
layer_scatter_modes=self.layer_scatter_modes,
|
||||
layer_facts=self.layer_facts,
|
||||
input_layernorm=self.input_layernorm,
|
||||
post_attention_layernorm=self.post_attention_layernorm,
|
||||
allow_reduce_scatter=True,
|
||||
|
||||
@@ -17,7 +17,7 @@ from sglang.srt.layers import deep_gemm_wrapper
|
||||
from sglang.srt.layers.activation import SiluAndMul
|
||||
from sglang.srt.layers.communicator import (
|
||||
LayerCommunicator,
|
||||
LayerScatterModes,
|
||||
LayerFacts,
|
||||
)
|
||||
from sglang.srt.layers.dp_attention import (
|
||||
is_dp_attention_enabled,
|
||||
@@ -813,7 +813,7 @@ class BailingMoELinearDecoderLayer(nn.Module):
|
||||
self.input_layernorm = RMSNorm(self.hidden_size, eps=rms_norm_eps)
|
||||
self.post_attention_layernorm = RMSNorm(self.hidden_size, eps=rms_norm_eps)
|
||||
|
||||
self.layer_scatter_modes = LayerScatterModes.init_new(
|
||||
self.layer_facts = LayerFacts.init_new(
|
||||
layer_id=layer_id,
|
||||
# A NextN draft is a one-layer model.
|
||||
num_layers=1 if is_nextn else config.num_hidden_layers,
|
||||
@@ -828,7 +828,7 @@ class BailingMoELinearDecoderLayer(nn.Module):
|
||||
else None
|
||||
)
|
||||
self.layer_communicator = LayerCommunicator(
|
||||
layer_scatter_modes=self.layer_scatter_modes,
|
||||
layer_facts=self.layer_facts,
|
||||
input_layernorm=self.input_layernorm,
|
||||
post_attention_layernorm=self.post_attention_layernorm,
|
||||
allow_reduce_scatter=False,
|
||||
|
||||
@@ -30,7 +30,7 @@ from sglang.srt.layers import deep_gemm_wrapper
|
||||
from sglang.srt.layers.activation import SiluAndMul
|
||||
from sglang.srt.layers.communicator import (
|
||||
LayerCommunicator,
|
||||
LayerScatterModes,
|
||||
LayerFacts,
|
||||
enable_moe_dense_fully_dp,
|
||||
layer_input_buffer,
|
||||
reduce_output,
|
||||
@@ -1148,7 +1148,7 @@ class BailingMoELinearDecoderLayer(nn.Module):
|
||||
self.input_layernorm = RMSNorm(self.hidden_size, eps=rms_norm_eps)
|
||||
self.post_attention_layernorm = RMSNorm(self.hidden_size, eps=rms_norm_eps)
|
||||
|
||||
self.layer_scatter_modes = LayerScatterModes.init_new(
|
||||
self.layer_facts = LayerFacts.init_new(
|
||||
layer_id=layer_id,
|
||||
# NextN wraps a single decoder layer whose checkpoint prefix is the
|
||||
# post-model layer id. Treat it as a one-layer model for scatter-mode
|
||||
@@ -1160,7 +1160,7 @@ class BailingMoELinearDecoderLayer(nn.Module):
|
||||
)
|
||||
|
||||
self.layer_communicator = LayerCommunicator(
|
||||
layer_scatter_modes=self.layer_scatter_modes,
|
||||
layer_facts=self.layer_facts,
|
||||
input_layernorm=self.input_layernorm,
|
||||
post_attention_layernorm=self.post_attention_layernorm,
|
||||
allow_reduce_scatter=True,
|
||||
|
||||
@@ -69,7 +69,7 @@ from sglang.srt.layers.aux_hidden_states import (
|
||||
)
|
||||
from sglang.srt.layers.communicator import (
|
||||
LayerCommunicator,
|
||||
LayerScatterModes,
|
||||
LayerFacts,
|
||||
enable_moe_dense_fully_dp,
|
||||
get_attn_tp_context,
|
||||
layer_input_buffer,
|
||||
@@ -2576,7 +2576,7 @@ class DeepseekV2DecoderLayer(nn.Module):
|
||||
is_previous_layer_sparse = self._is_layer_sparse(layer_id - 1, is_nextn=False)
|
||||
is_next_layer_sparse = self._is_layer_sparse(layer_id + 1, is_nextn=False)
|
||||
|
||||
self.layer_scatter_modes = LayerScatterModes.init_new(
|
||||
self.layer_facts = LayerFacts.init_new(
|
||||
layer_id=layer_id,
|
||||
num_layers=1 if is_nextn else config.num_hidden_layers,
|
||||
is_layer_sparse=self.is_layer_sparse,
|
||||
@@ -2643,7 +2643,7 @@ class DeepseekV2DecoderLayer(nn.Module):
|
||||
|
||||
fusions = CuteDSLFusion()
|
||||
return LayerCommunicator(
|
||||
layer_scatter_modes=self.layer_scatter_modes,
|
||||
layer_facts=self.layer_facts,
|
||||
input_layernorm=input_layernorm,
|
||||
post_attention_layernorm=post_attention_layernorm,
|
||||
allow_reduce_scatter=True,
|
||||
|
||||
@@ -59,7 +59,7 @@ from sglang.srt.layers.activation import SiluAndMul
|
||||
from sglang.srt.layers.attention.dsa.dsa_indexer import Indexer
|
||||
from sglang.srt.layers.communicator import (
|
||||
LayerCommunicator,
|
||||
LayerScatterModes,
|
||||
LayerFacts,
|
||||
enable_moe_dense_fully_dp,
|
||||
)
|
||||
from sglang.srt.layers.dp_attention import is_dp_attention_enabled
|
||||
@@ -1541,7 +1541,7 @@ class Dots3DecoderLayer(nn.Module):
|
||||
is_previous_layer_sparse = self._is_layer_sparse(layer_id - 1, is_nextn=False)
|
||||
is_next_layer_sparse = self._is_layer_sparse(layer_id + 1, is_nextn=False)
|
||||
|
||||
self.layer_scatter_modes = LayerScatterModes.init_new(
|
||||
self.layer_facts = LayerFacts.init_new(
|
||||
layer_id=layer_id,
|
||||
num_layers=1 if is_nextn else config.num_hidden_layers,
|
||||
is_layer_sparse=self.is_layer_sparse,
|
||||
@@ -1579,7 +1579,7 @@ class Dots3DecoderLayer(nn.Module):
|
||||
)
|
||||
|
||||
self.layer_communicator = LayerCommunicator(
|
||||
layer_scatter_modes=self.layer_scatter_modes,
|
||||
layer_facts=self.layer_facts,
|
||||
input_layernorm=self.input_layernorm,
|
||||
post_attention_layernorm=self.post_attention_layernorm,
|
||||
allow_reduce_scatter=True,
|
||||
|
||||
@@ -11,7 +11,7 @@ from sglang.srt.layers.attention.hybrid_linear_attn_backend import (
|
||||
Mamba2AttnBackend,
|
||||
)
|
||||
from sglang.srt.layers.attention.mamba.mamba import MambaMixer2
|
||||
from sglang.srt.layers.communicator import LayerCommunicator, LayerScatterModes
|
||||
from sglang.srt.layers.communicator import LayerCommunicator, LayerFacts
|
||||
from sglang.srt.layers.dp_attention import (
|
||||
is_dp_attention_enabled,
|
||||
)
|
||||
@@ -198,7 +198,7 @@ class FalconH1HybridAttentionDecoderLayer(nn.Module):
|
||||
is_previous_layer_sparse = False
|
||||
is_next_layer_sparse = False
|
||||
|
||||
self.layer_scatter_modes = LayerScatterModes.init_new(
|
||||
self.layer_facts = LayerFacts.init_new(
|
||||
layer_id=layer_id,
|
||||
num_layers=config.num_hidden_layers,
|
||||
is_layer_sparse=self.is_layer_sparse,
|
||||
@@ -223,7 +223,7 @@ class FalconH1HybridAttentionDecoderLayer(nn.Module):
|
||||
self.k_norm = RMSNorm(self.head_dim, eps=config.rms_norm_eps)
|
||||
|
||||
self.layer_communicator = LayerCommunicator(
|
||||
layer_scatter_modes=self.layer_scatter_modes,
|
||||
layer_facts=self.layer_facts,
|
||||
input_layernorm=self.input_layernorm,
|
||||
post_attention_layernorm=self.pre_ff_layernorm,
|
||||
allow_reduce_scatter=True,
|
||||
|
||||
@@ -40,7 +40,7 @@ from sglang.srt.eplb.expert_location_dispatch import ExpertLocationDispatchInfo
|
||||
from sglang.srt.layers.activation import SiluAndMul
|
||||
from sglang.srt.layers.communicator import (
|
||||
LayerCommunicator,
|
||||
LayerScatterModes,
|
||||
LayerFacts,
|
||||
enable_moe_dense_fully_dp,
|
||||
reduce_output,
|
||||
)
|
||||
@@ -836,7 +836,7 @@ class Glm4MoeDecoderLayer(nn.Module):
|
||||
is_previous_layer_sparse = self._is_layer_sparse(layer_id - 1, is_nextn=False)
|
||||
is_next_layer_sparse = self._is_layer_sparse(layer_id + 1, is_nextn=False)
|
||||
|
||||
self.layer_scatter_modes = LayerScatterModes.init_new(
|
||||
self.layer_facts = LayerFacts.init_new(
|
||||
layer_id=layer_id,
|
||||
num_layers=1 if is_nextn else config.num_hidden_layers,
|
||||
is_layer_sparse=self.is_layer_sparse,
|
||||
@@ -873,7 +873,7 @@ class Glm4MoeDecoderLayer(nn.Module):
|
||||
)
|
||||
|
||||
self.layer_communicator = LayerCommunicator(
|
||||
layer_scatter_modes=self.layer_scatter_modes,
|
||||
layer_facts=self.layer_facts,
|
||||
input_layernorm=self.input_layernorm,
|
||||
post_attention_layernorm=self.post_attention_layernorm,
|
||||
allow_reduce_scatter=True,
|
||||
|
||||
@@ -34,7 +34,7 @@ from sglang.srt.eplb.expert_location_dispatch import ExpertLocationDispatchInfo
|
||||
from sglang.srt.layers.activation import SiluAndMul
|
||||
from sglang.srt.layers.communicator import (
|
||||
LayerCommunicator,
|
||||
LayerScatterModes,
|
||||
LayerFacts,
|
||||
enable_moe_dense_fully_dp,
|
||||
get_attn_tp_context,
|
||||
reduce_output,
|
||||
@@ -548,7 +548,7 @@ class Glm4MoeLiteDecoderLayer(nn.Module):
|
||||
is_previous_layer_sparse = self._is_layer_sparse(layer_id - 1, is_nextn=False)
|
||||
is_next_layer_sparse = self._is_layer_sparse(layer_id + 1, is_nextn=False)
|
||||
|
||||
self.layer_scatter_modes = LayerScatterModes.init_new(
|
||||
self.layer_facts = LayerFacts.init_new(
|
||||
layer_id=layer_id,
|
||||
num_layers=1 if is_nextn else config.num_hidden_layers,
|
||||
is_layer_sparse=self.is_layer_sparse,
|
||||
@@ -588,7 +588,7 @@ class Glm4MoeLiteDecoderLayer(nn.Module):
|
||||
self._gfx95_quant_format = self._detect_gfx95_quant_format()
|
||||
|
||||
self.layer_communicator = LayerCommunicator(
|
||||
layer_scatter_modes=self.layer_scatter_modes,
|
||||
layer_facts=self.layer_facts,
|
||||
input_layernorm=self.input_layernorm,
|
||||
post_attention_layernorm=self.post_attention_layernorm,
|
||||
allow_reduce_scatter=True,
|
||||
|
||||
@@ -27,7 +27,7 @@ from sglang.srt.layers.attention import vision_utils
|
||||
from sglang.srt.layers.attention.vision import VisionAttention
|
||||
from sglang.srt.layers.communicator import (
|
||||
LayerCommunicator,
|
||||
LayerScatterModes,
|
||||
LayerFacts,
|
||||
MHCLayerCommunicator,
|
||||
enable_moe_dense_fully_dp,
|
||||
get_attn_tp_context,
|
||||
@@ -726,7 +726,7 @@ class Glm5NextDecoderLayer(nn.Module):
|
||||
is_previous_layer_sparse = self._is_layer_sparse(layer_id - 1, is_nextn=False)
|
||||
is_next_layer_sparse = self._is_layer_sparse(layer_id + 1, is_nextn=False)
|
||||
|
||||
self.layer_scatter_modes = LayerScatterModes.init_new(
|
||||
self.layer_facts = LayerFacts.init_new(
|
||||
layer_id=layer_id,
|
||||
num_layers=1 if is_nextn else config.num_hidden_layers,
|
||||
is_layer_sparse=self.is_layer_sparse,
|
||||
@@ -784,7 +784,7 @@ class Glm5NextDecoderLayer(nn.Module):
|
||||
)
|
||||
|
||||
shared_kwargs: Dict[str, Any] = dict(
|
||||
layer_scatter_modes=self.layer_scatter_modes,
|
||||
layer_facts=self.layer_facts,
|
||||
input_layernorm=self.input_layernorm,
|
||||
post_attention_layernorm=self.post_attention_layernorm,
|
||||
allow_reduce_scatter=True,
|
||||
|
||||
@@ -31,7 +31,7 @@ from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_r
|
||||
from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation
|
||||
from sglang.srt.layers.communicator import (
|
||||
LayerCommunicator,
|
||||
LayerScatterModes,
|
||||
LayerFacts,
|
||||
reduce_output,
|
||||
)
|
||||
from sglang.srt.layers.dp_attention import (
|
||||
@@ -569,7 +569,7 @@ class GptOssDecoderLayer(nn.Module):
|
||||
is_previous_layer_sparse = True
|
||||
is_next_layer_sparse = True
|
||||
|
||||
self.layer_scatter_modes = LayerScatterModes.init_new(
|
||||
self.layer_facts = LayerFacts.init_new(
|
||||
layer_id=layer_id,
|
||||
num_layers=config.num_hidden_layers,
|
||||
is_layer_sparse=self.is_layer_sparse,
|
||||
@@ -606,7 +606,7 @@ class GptOssDecoderLayer(nn.Module):
|
||||
)
|
||||
|
||||
self.layer_communicator = LayerCommunicator(
|
||||
layer_scatter_modes=self.layer_scatter_modes,
|
||||
layer_facts=self.layer_facts,
|
||||
input_layernorm=self.input_layernorm,
|
||||
post_attention_layernorm=self.post_attention_layernorm,
|
||||
allow_reduce_scatter=True,
|
||||
|
||||
@@ -10,7 +10,7 @@ from sglang.srt.configs.interns2_mobius import (
|
||||
InternS2MobiusConfig,
|
||||
InternS2MobiusTextConfig,
|
||||
)
|
||||
from sglang.srt.layers.communicator import LayerCommunicator, LayerScatterModes
|
||||
from sglang.srt.layers.communicator import LayerCommunicator, LayerFacts
|
||||
from sglang.srt.layers.dp_attention import is_dp_attention_enabled
|
||||
from sglang.srt.layers.layernorm import GemmaRMSNorm
|
||||
from sglang.srt.layers.linear import (
|
||||
@@ -476,7 +476,7 @@ class InternS2MobiusLinearDecoderLayer(_InternS2MobiusDecoderMixin, nn.Module):
|
||||
|
||||
layer_prefix = prefix.removesuffix(".linear_attn")
|
||||
self._init_mobius_mlp(config, quant_config, layer_prefix)
|
||||
self.layer_scatter_modes = LayerScatterModes.init_new(
|
||||
self.layer_facts = LayerFacts.init_new(
|
||||
layer_id=layer_id,
|
||||
num_layers=config.num_hidden_layers,
|
||||
is_layer_sparse=True,
|
||||
@@ -492,7 +492,7 @@ class InternS2MobiusLinearDecoderLayer(_InternS2MobiusDecoderMixin, nn.Module):
|
||||
and _linear_accepts_fp8_tuple(self.linear_attn.in_proj_qkvz)
|
||||
)
|
||||
self.layer_communicator = LayerCommunicator(
|
||||
layer_scatter_modes=self.layer_scatter_modes,
|
||||
layer_facts=self.layer_facts,
|
||||
input_layernorm=self.input_layernorm,
|
||||
post_attention_layernorm=self.post_attention_layernorm,
|
||||
allow_reduce_scatter=True,
|
||||
@@ -609,7 +609,7 @@ class InternS2MobiusAttentionDecoderLayer(
|
||||
|
||||
layer_prefix = prefix.removesuffix(".self_attn")
|
||||
self._init_mobius_mlp(config, quant_config, layer_prefix)
|
||||
self.layer_scatter_modes = LayerScatterModes.init_new(
|
||||
self.layer_facts = LayerFacts.init_new(
|
||||
layer_id=layer_id,
|
||||
num_layers=config.num_hidden_layers,
|
||||
is_layer_sparse=True,
|
||||
@@ -626,7 +626,7 @@ class InternS2MobiusAttentionDecoderLayer(
|
||||
_enable_qwen35_fused_ar_quant() and _linear_accepts_fp8_tuple(self.qkv_proj)
|
||||
)
|
||||
self.layer_communicator = LayerCommunicator(
|
||||
layer_scatter_modes=self.layer_scatter_modes,
|
||||
layer_facts=self.layer_facts,
|
||||
input_layernorm=self.input_layernorm,
|
||||
post_attention_layernorm=self.post_attention_layernorm,
|
||||
allow_reduce_scatter=True,
|
||||
|
||||
@@ -21,7 +21,7 @@ from sglang.srt.environ import envs
|
||||
from sglang.srt.layers.activation import SiluAndMul
|
||||
from sglang.srt.layers.communicator import (
|
||||
LayerCommunicator,
|
||||
LayerScatterModes,
|
||||
LayerFacts,
|
||||
reduce_output,
|
||||
)
|
||||
from sglang.srt.layers.dp_attention import (
|
||||
@@ -455,7 +455,7 @@ class LagunaDecoderLayer(nn.Module):
|
||||
config.hidden_size, eps=config.rms_norm_eps
|
||||
)
|
||||
|
||||
self.layer_scatter_modes = LayerScatterModes.init_new(
|
||||
self.layer_facts = LayerFacts.init_new(
|
||||
layer_id=layer_id,
|
||||
num_layers=config.num_hidden_layers,
|
||||
is_layer_sparse=self.is_layer_sparse,
|
||||
@@ -463,7 +463,7 @@ class LagunaDecoderLayer(nn.Module):
|
||||
is_next_layer_sparse=is_next_layer_sparse,
|
||||
)
|
||||
self.layer_communicator = LayerCommunicator(
|
||||
layer_scatter_modes=self.layer_scatter_modes,
|
||||
layer_facts=self.layer_facts,
|
||||
input_layernorm=self.input_layernorm,
|
||||
post_attention_layernorm=self.post_attention_layernorm,
|
||||
allow_reduce_scatter=True,
|
||||
|
||||
@@ -33,7 +33,7 @@ from sglang.srt.eplb.expert_location_dispatch import ExpertLocationDispatchInfo
|
||||
from sglang.srt.layers.activation import SiluAndMul
|
||||
from sglang.srt.layers.communicator import (
|
||||
LayerCommunicator,
|
||||
LayerScatterModes,
|
||||
LayerFacts,
|
||||
enable_moe_dense_fully_dp,
|
||||
)
|
||||
from sglang.srt.layers.dp_attention import (
|
||||
@@ -612,7 +612,7 @@ class LLaDA2MoeBlock(nn.Module):
|
||||
is_previous_layer_sparse = self._is_layer_sparse(config, layer_id=layer_id - 1)
|
||||
is_next_layer_sparse = self._is_layer_sparse(config, layer_id=layer_id + 1)
|
||||
|
||||
self.layer_scatter_modes = LayerScatterModes.init_new(
|
||||
self.layer_facts = LayerFacts.init_new(
|
||||
layer_id=layer_id,
|
||||
num_layers=config.num_hidden_layers,
|
||||
is_layer_sparse=self.is_layer_sparse,
|
||||
@@ -645,7 +645,7 @@ class LLaDA2MoeBlock(nn.Module):
|
||||
self.post_attention_layernorm = RMSNorm(hidden_size, eps=config.rms_norm_eps)
|
||||
|
||||
self.layer_communicator = LayerCommunicator(
|
||||
layer_scatter_modes=self.layer_scatter_modes,
|
||||
layer_facts=self.layer_facts,
|
||||
input_layernorm=self.input_layernorm,
|
||||
post_attention_layernorm=self.post_attention_layernorm,
|
||||
allow_reduce_scatter=True,
|
||||
|
||||
@@ -25,7 +25,7 @@ import torch
|
||||
from torch import nn
|
||||
from transformers import Llama4TextConfig
|
||||
|
||||
from sglang.srt.layers.communicator import LayerCommunicator, LayerScatterModes
|
||||
from sglang.srt.layers.communicator import LayerCommunicator, LayerFacts
|
||||
from sglang.srt.layers.dp_attention import (
|
||||
is_dp_attention_enabled,
|
||||
)
|
||||
@@ -420,7 +420,7 @@ class Llama4DecoderLayer(nn.Module):
|
||||
config.hidden_size, eps=config.rms_norm_eps
|
||||
)
|
||||
|
||||
self.layer_scatter_modes = LayerScatterModes.init_new(
|
||||
self.layer_facts = LayerFacts.init_new(
|
||||
layer_id=layer_id,
|
||||
num_layers=config.num_hidden_layers,
|
||||
is_layer_sparse=is_moe_layer,
|
||||
@@ -429,7 +429,7 @@ class Llama4DecoderLayer(nn.Module):
|
||||
)
|
||||
|
||||
self.layer_communicator = LayerCommunicator(
|
||||
layer_scatter_modes=self.layer_scatter_modes,
|
||||
layer_facts=self.layer_facts,
|
||||
input_layernorm=self.input_layernorm,
|
||||
post_attention_layernorm=self.post_attention_layernorm,
|
||||
allow_reduce_scatter=True,
|
||||
|
||||
@@ -51,7 +51,7 @@ from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_r
|
||||
from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation
|
||||
from sglang.srt.layers import deep_gemm_wrapper
|
||||
from sglang.srt.layers.activation import SiluAndMul
|
||||
from sglang.srt.layers.communicator import LayerCommunicator, LayerScatterModes
|
||||
from sglang.srt.layers.communicator import LayerCommunicator, LayerFacts
|
||||
from sglang.srt.layers.dp_attention import (
|
||||
is_dp_attention_enabled,
|
||||
)
|
||||
@@ -419,8 +419,8 @@ class LongcatFlashDecoderLayer(nn.Module):
|
||||
self.attn_tp_size = get_parallel().attn_tp_size
|
||||
self.attn_tp_rank = get_parallel().attn_tp_rank
|
||||
|
||||
self.mlp_layer_scatter_modes = [
|
||||
LayerScatterModes.init_new(
|
||||
self.mlp_layer_facts = [
|
||||
LayerFacts.init_new(
|
||||
layer_id=self.layer_id * 2 + i,
|
||||
num_layers=config.num_hidden_layers,
|
||||
is_layer_sparse=False,
|
||||
@@ -432,7 +432,7 @@ class LongcatFlashDecoderLayer(nn.Module):
|
||||
]
|
||||
self.mlp_layer_communicator = [
|
||||
LayerCommunicator(
|
||||
layer_scatter_modes=self.mlp_layer_scatter_modes[i],
|
||||
layer_facts=self.mlp_layer_facts[i],
|
||||
input_layernorm=self.input_layernorm[i],
|
||||
post_attention_layernorm=self.post_attention_layernorm[i],
|
||||
qkv_latent_func=self.self_attn[i].prepare_qkv_latent,
|
||||
@@ -441,7 +441,7 @@ class LongcatFlashDecoderLayer(nn.Module):
|
||||
for i in range(2)
|
||||
]
|
||||
|
||||
self.moe_layer_scatter_modes = LayerScatterModes.init_new(
|
||||
self.moe_layer_facts = LayerFacts.init_new(
|
||||
layer_id=self.layer_id,
|
||||
num_layers=config.num_hidden_layers,
|
||||
is_layer_sparse=True,
|
||||
@@ -450,7 +450,7 @@ class LongcatFlashDecoderLayer(nn.Module):
|
||||
is_next_layer_sparse=True,
|
||||
)
|
||||
self.moe_layer_communicator = LayerCommunicator(
|
||||
layer_scatter_modes=self.moe_layer_scatter_modes,
|
||||
layer_facts=self.moe_layer_facts,
|
||||
input_layernorm=self.input_layernorm[0],
|
||||
post_attention_layernorm=self.post_attention_layernorm[0],
|
||||
qkv_latent_func=self.self_attn[0].prepare_qkv_latent,
|
||||
|
||||
@@ -41,7 +41,7 @@ from sglang.kernels.ops.quantization.fp8_kernel import is_fp8_fnuz
|
||||
from sglang.srt.configs import LongcatFlashConfig
|
||||
from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder
|
||||
from sglang.srt.layers import deep_gemm_wrapper
|
||||
from sglang.srt.layers.communicator import LayerCommunicator, LayerScatterModes
|
||||
from sglang.srt.layers.communicator import LayerCommunicator, LayerFacts
|
||||
from sglang.srt.layers.dp_attention import (
|
||||
is_dp_attention_enabled,
|
||||
)
|
||||
@@ -159,7 +159,7 @@ class LongcatFlashDenseDecoderLayer(nn.Module):
|
||||
|
||||
self.attn_tp_size = get_parallel().attn_tp_size
|
||||
self.attn_tp_rank = get_parallel().attn_tp_rank
|
||||
self.layer_scatter_modes = LayerScatterModes.init_new(
|
||||
self.layer_facts = LayerFacts.init_new(
|
||||
layer_id=self.layer_id,
|
||||
num_layers=config.num_hidden_layers,
|
||||
is_layer_sparse=False,
|
||||
@@ -167,7 +167,7 @@ class LongcatFlashDenseDecoderLayer(nn.Module):
|
||||
is_next_layer_sparse=False,
|
||||
)
|
||||
self.layer_communicator = LayerCommunicator(
|
||||
layer_scatter_modes=self.layer_scatter_modes,
|
||||
layer_facts=self.layer_facts,
|
||||
input_layernorm=self.input_layernorm,
|
||||
post_attention_layernorm=self.post_attention_layernorm,
|
||||
)
|
||||
|
||||
@@ -25,7 +25,7 @@ import torch
|
||||
from torch import nn
|
||||
from transformers import PretrainedConfig
|
||||
|
||||
from sglang.srt.layers.communicator import LayerCommunicator, LayerScatterModes
|
||||
from sglang.srt.layers.communicator import LayerCommunicator, LayerFacts
|
||||
from sglang.srt.layers.layernorm import RMSNorm
|
||||
from sglang.srt.layers.linear import QKVParallelLinear, RowParallelLinear
|
||||
from sglang.srt.layers.quantization.base_config import QuantizationConfig
|
||||
@@ -442,7 +442,7 @@ class MellumDecoderLayer(Qwen3MoeDecoderLayer):
|
||||
is_previous_layer_sparse = _is_sparse(layer_id - 1)
|
||||
is_next_layer_sparse = _is_sparse(layer_id + 1)
|
||||
|
||||
self.layer_scatter_modes = LayerScatterModes.init_new(
|
||||
self.layer_facts = LayerFacts.init_new(
|
||||
layer_id=layer_id,
|
||||
num_layers=cfg.num_hidden_layers,
|
||||
is_layer_sparse=self.is_layer_sparse,
|
||||
@@ -454,7 +454,7 @@ class MellumDecoderLayer(Qwen3MoeDecoderLayer):
|
||||
self.post_attention_layernorm = RMSNorm(cfg.hidden_size, eps=rms_norm_eps)
|
||||
|
||||
self.layer_communicator = LayerCommunicator(
|
||||
layer_scatter_modes=self.layer_scatter_modes,
|
||||
layer_facts=self.layer_facts,
|
||||
input_layernorm=self.input_layernorm,
|
||||
post_attention_layernorm=self.post_attention_layernorm,
|
||||
allow_reduce_scatter=True,
|
||||
|
||||
@@ -34,7 +34,7 @@ from sglang.srt.layers.aux_hidden_states import (
|
||||
)
|
||||
from sglang.srt.layers.communicator import (
|
||||
LayerCommunicator,
|
||||
LayerScatterModes,
|
||||
LayerFacts,
|
||||
enable_moe_dense_fully_dp,
|
||||
)
|
||||
from sglang.srt.layers.dp_attention import (
|
||||
@@ -856,7 +856,7 @@ class MiMoV2DecoderLayer(nn.Module):
|
||||
config.hidden_size, eps=config.layernorm_epsilon
|
||||
)
|
||||
|
||||
self.layer_scatter_modes = LayerScatterModes.init_new(
|
||||
self.layer_facts = LayerFacts.init_new(
|
||||
layer_id=layer_id,
|
||||
num_layers=config.num_hidden_layers,
|
||||
is_layer_sparse=self.is_layer_sparse,
|
||||
@@ -864,7 +864,7 @@ class MiMoV2DecoderLayer(nn.Module):
|
||||
is_next_layer_sparse=is_next_layer_sparse,
|
||||
)
|
||||
self.layer_communicator = LayerCommunicator(
|
||||
layer_scatter_modes=self.layer_scatter_modes,
|
||||
layer_facts=self.layer_facts,
|
||||
input_layernorm=self.input_layernorm,
|
||||
post_attention_layernorm=self.post_attention_layernorm,
|
||||
allow_reduce_scatter=True,
|
||||
|
||||
@@ -23,7 +23,7 @@ from sglang.srt.configs.model_config import get_mimo_v2_fused_qkv_expected_tp_si
|
||||
from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder
|
||||
from sglang.srt.layers.communicator import (
|
||||
LayerCommunicator,
|
||||
LayerScatterModes,
|
||||
LayerFacts,
|
||||
enable_moe_dense_fully_dp,
|
||||
)
|
||||
from sglang.srt.layers.dp_attention import (
|
||||
@@ -116,7 +116,7 @@ class MiMoV2MTPLayer(nn.Module):
|
||||
self.post_attention_layernorm = RMSNorm(
|
||||
config.hidden_size, eps=config.layernorm_epsilon
|
||||
)
|
||||
self.layer_scatter_modes = LayerScatterModes.init_new(
|
||||
self.layer_facts = LayerFacts.init_new(
|
||||
layer_id=layer_id,
|
||||
num_layers=1,
|
||||
is_layer_sparse=self.is_layer_sparse,
|
||||
@@ -124,7 +124,7 @@ class MiMoV2MTPLayer(nn.Module):
|
||||
is_next_layer_sparse=is_next_layer_sparse,
|
||||
)
|
||||
self.layer_communicator = LayerCommunicator(
|
||||
layer_scatter_modes=self.layer_scatter_modes,
|
||||
layer_facts=self.layer_facts,
|
||||
input_layernorm=self.input_layernorm,
|
||||
post_attention_layernorm=self.post_attention_layernorm,
|
||||
allow_deferred_ffn_reduction=False,
|
||||
|
||||
@@ -32,7 +32,7 @@ from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_r
|
||||
from sglang.srt.eplb.expert_location_dispatch import ExpertLocationDispatchInfo
|
||||
from sglang.srt.layers.communicator import (
|
||||
LayerCommunicator,
|
||||
LayerScatterModes,
|
||||
LayerFacts,
|
||||
)
|
||||
from sglang.srt.layers.dp_attention import (
|
||||
attn_tp_all_reduce,
|
||||
@@ -975,7 +975,7 @@ class MiniMaxM2DecoderLayer(nn.Module):
|
||||
|
||||
is_previous_layer_sparse = True
|
||||
is_next_layer_sparse = True
|
||||
self.layer_scatter_modes = LayerScatterModes.init_new(
|
||||
self.layer_facts = LayerFacts.init_new(
|
||||
layer_id=layer_id,
|
||||
num_layers=config.num_hidden_layers,
|
||||
is_layer_sparse=self.is_layer_sparse,
|
||||
@@ -984,7 +984,7 @@ class MiniMaxM2DecoderLayer(nn.Module):
|
||||
)
|
||||
|
||||
self.layer_communicator = LayerCommunicator(
|
||||
layer_scatter_modes=self.layer_scatter_modes,
|
||||
layer_facts=self.layer_facts,
|
||||
input_layernorm=self.input_layernorm,
|
||||
post_attention_layernorm=self.post_attention_layernorm,
|
||||
allow_reduce_scatter=True,
|
||||
|
||||
@@ -37,7 +37,7 @@ from sglang.srt.eplb.expert_location_dispatch import ExpertLocationDispatchInfo
|
||||
from sglang.srt.layers.activation import SiluAndMul
|
||||
from sglang.srt.layers.communicator import (
|
||||
LayerCommunicator,
|
||||
LayerScatterModes,
|
||||
LayerFacts,
|
||||
enable_moe_dense_fully_dp,
|
||||
)
|
||||
from sglang.srt.layers.dp_attention import (
|
||||
@@ -1297,7 +1297,7 @@ class MiniMaxM3DecoderLayer(nn.Module):
|
||||
|
||||
moe_layer_freq = getattr(config, "moe_layer_freq", None)
|
||||
# Means "MLP is a sparse MoE", not attention sparsity. Kept as ``is_layer_sparse``
|
||||
# because LayerCommunicator / LayerScatterModes / other models read this attr.
|
||||
# because LayerCommunicator / LayerFacts / other models read this attr.
|
||||
self.is_layer_sparse = (
|
||||
moe_layer_freq[layer_id] != 0 if moe_layer_freq is not None else True
|
||||
)
|
||||
@@ -1347,7 +1347,7 @@ class MiniMaxM3DecoderLayer(nn.Module):
|
||||
|
||||
is_previous_layer_sparse = _is_layer_sparse(layer_id - 1)
|
||||
is_next_layer_sparse = _is_layer_sparse(layer_id + 1)
|
||||
self.layer_scatter_modes = LayerScatterModes.init_new(
|
||||
self.layer_facts = LayerFacts.init_new(
|
||||
layer_id=layer_id,
|
||||
num_layers=config.num_hidden_layers,
|
||||
is_layer_sparse=self.is_layer_sparse,
|
||||
@@ -1356,7 +1356,7 @@ class MiniMaxM3DecoderLayer(nn.Module):
|
||||
)
|
||||
|
||||
self.layer_communicator = LayerCommunicator(
|
||||
layer_scatter_modes=self.layer_scatter_modes,
|
||||
layer_facts=self.layer_facts,
|
||||
input_layernorm=self.input_layernorm,
|
||||
post_attention_layernorm=self.post_attention_layernorm,
|
||||
allow_reduce_scatter=True,
|
||||
|
||||
@@ -22,7 +22,7 @@ from sglang.srt.layers.attention.vision import (
|
||||
VisionAttentionMetadata,
|
||||
prepare_vision_attention_metadata,
|
||||
)
|
||||
from sglang.srt.layers.communicator import LayerCommunicator, LayerScatterModes
|
||||
from sglang.srt.layers.communicator import LayerCommunicator, LayerFacts
|
||||
from sglang.srt.layers.conv import Conv3dLayer
|
||||
from sglang.srt.layers.layernorm import RMSNorm
|
||||
from sglang.srt.layers.linear import (
|
||||
@@ -1011,7 +1011,7 @@ class MossVLSelfAttentionDecoderLayer(nn.Module):
|
||||
self.post_attention_layernorm = RMSNorm(
|
||||
config.hidden_size, eps=config.rms_norm_eps, **norm_kwargs
|
||||
)
|
||||
self.layer_scatter_modes = LayerScatterModes.init_new(
|
||||
self.layer_facts = LayerFacts.init_new(
|
||||
layer_id=layer_id,
|
||||
num_layers=config.num_hidden_layers,
|
||||
is_layer_sparse=False,
|
||||
@@ -1019,7 +1019,7 @@ class MossVLSelfAttentionDecoderLayer(nn.Module):
|
||||
is_next_layer_sparse=False,
|
||||
)
|
||||
self.layer_communicator = LayerCommunicator(
|
||||
layer_scatter_modes=self.layer_scatter_modes,
|
||||
layer_facts=self.layer_facts,
|
||||
input_layernorm=self.input_layernorm,
|
||||
post_attention_layernorm=self.post_attention_layernorm,
|
||||
allow_deferred_ffn_reduction=False,
|
||||
|
||||
@@ -6,10 +6,9 @@ from sglang.srt.configs.nemotron_h import ATTENTION, MAMBA, MOE
|
||||
from sglang.srt.layers.communicator import (
|
||||
NORM_QUANT_READ,
|
||||
LayerCommunicator,
|
||||
LayerScatterModes,
|
||||
LayerFacts,
|
||||
LayerStage,
|
||||
Layout,
|
||||
ScatterMode,
|
||||
StageDecl,
|
||||
StageInput,
|
||||
StageKind,
|
||||
@@ -102,25 +101,6 @@ def layer_stage(pattern: str, layer_idx: int) -> LayerStage:
|
||||
)
|
||||
|
||||
|
||||
def _build_layer_scatter_modes(
|
||||
is_sparse: bool = False, is_last_layer: bool = False
|
||||
) -> LayerScatterModes:
|
||||
scatter_mlp = is_sparse and not get_moe_a2a_backend().is_none()
|
||||
mlp_mode = ScatterMode.SCATTERED if scatter_mlp else ScatterMode.FULL
|
||||
middle_residual_mode = (
|
||||
ScatterMode.SCATTERED if scatter_mlp else ScatterMode.TP_ATTN_FULL
|
||||
)
|
||||
return LayerScatterModes(
|
||||
layer_input_mode=ScatterMode.TP_ATTN_FULL,
|
||||
attn_mode=ScatterMode.TP_ATTN_FULL,
|
||||
mlp_mode=mlp_mode,
|
||||
middle_residual_mode=middle_residual_mode,
|
||||
layer_output_mode=ScatterMode.TP_ATTN_FULL,
|
||||
is_layer_sparse=is_sparse,
|
||||
is_last_layer=is_last_layer,
|
||||
)
|
||||
|
||||
|
||||
def make_layer_communicator(
|
||||
layer_norm: RMSNorm, *, pattern: str, layer_idx: int
|
||||
) -> LayerCommunicator:
|
||||
@@ -129,8 +109,9 @@ def make_layer_communicator(
|
||||
stage = layer_stage(pattern, layer_idx)
|
||||
for_attn = stage.kind is StageKind.ATTENTION
|
||||
return LayerCommunicator(
|
||||
layer_scatter_modes=_build_layer_scatter_modes(
|
||||
pattern[layer_idx] == MOE, is_last_layer=layer_idx == len(pattern) - 1
|
||||
layer_facts=LayerFacts(
|
||||
is_layer_sparse=pattern[layer_idx] == MOE,
|
||||
is_last_layer=layer_idx == len(pattern) - 1,
|
||||
),
|
||||
input_layernorm=layer_norm if for_attn else None,
|
||||
post_attention_layernorm=None if for_attn else layer_norm,
|
||||
|
||||
@@ -42,7 +42,7 @@ from sglang.srt.eplb.expert_location_dispatch import ExpertLocationDispatchInfo
|
||||
from sglang.srt.layers.activation import SiluAndMul
|
||||
from sglang.srt.layers.communicator import (
|
||||
LayerCommunicator,
|
||||
LayerScatterModes,
|
||||
LayerFacts,
|
||||
)
|
||||
from sglang.srt.layers.dp_attention import (
|
||||
is_dp_attention_enabled,
|
||||
@@ -1025,7 +1025,7 @@ class Qwen2MoeDecoderLayer(nn.Module):
|
||||
is_previous_layer_sparse = True
|
||||
is_next_layer_sparse = True
|
||||
|
||||
self.layer_scatter_modes = LayerScatterModes.init_new(
|
||||
self.layer_facts = LayerFacts.init_new(
|
||||
layer_id=layer_id,
|
||||
num_layers=config.num_hidden_layers,
|
||||
is_layer_sparse=self.is_layer_sparse,
|
||||
@@ -1054,7 +1054,7 @@ class Qwen2MoeDecoderLayer(nn.Module):
|
||||
config.hidden_size, eps=config.rms_norm_eps
|
||||
)
|
||||
self.layer_communicator = LayerCommunicator(
|
||||
layer_scatter_modes=self.layer_scatter_modes,
|
||||
layer_facts=self.layer_facts,
|
||||
input_layernorm=self.input_layernorm,
|
||||
post_attention_layernorm=self.post_attention_layernorm,
|
||||
allow_reduce_scatter=True,
|
||||
|
||||
@@ -5,7 +5,7 @@ from typing import Any, Dict, Iterable, List, Optional, Tuple
|
||||
import torch
|
||||
from torch import nn
|
||||
|
||||
from sglang.srt.layers.communicator import LayerCommunicator, LayerScatterModes
|
||||
from sglang.srt.layers.communicator import LayerCommunicator, LayerFacts
|
||||
from sglang.srt.layers.layernorm import RMSNorm
|
||||
from sglang.srt.layers.linear import QKVParallelLinear, RowParallelLinear
|
||||
from sglang.srt.layers.logits_processor import LogitsProcessor
|
||||
@@ -370,7 +370,7 @@ class Qwen3DecoderLayer(nn.Module):
|
||||
config.hidden_size, eps=config.rms_norm_eps, **norm_kwargs
|
||||
)
|
||||
|
||||
self.layer_scatter_modes = LayerScatterModes.init_new(
|
||||
self.layer_facts = LayerFacts.init_new(
|
||||
layer_id=layer_id,
|
||||
num_layers=config.num_hidden_layers,
|
||||
is_layer_sparse=False,
|
||||
@@ -378,7 +378,7 @@ class Qwen3DecoderLayer(nn.Module):
|
||||
is_next_layer_sparse=False,
|
||||
)
|
||||
self.layer_communicator = LayerCommunicator(
|
||||
layer_scatter_modes=self.layer_scatter_modes,
|
||||
layer_facts=self.layer_facts,
|
||||
input_layernorm=self.input_layernorm,
|
||||
post_attention_layernorm=self.post_attention_layernorm,
|
||||
allow_deferred_ffn_reduction=False,
|
||||
|
||||
@@ -45,7 +45,7 @@ from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation
|
||||
from sglang.srt.layers.attention.mamba.mamba import mamba_v2_sharded_weight_loader
|
||||
from sglang.srt.layers.communicator import (
|
||||
LayerCommunicator,
|
||||
LayerScatterModes,
|
||||
LayerFacts,
|
||||
reduce_output,
|
||||
)
|
||||
from sglang.srt.layers.dp_attention import (
|
||||
@@ -1104,7 +1104,7 @@ class Qwen3_5LinearDecoderLayer(nn.Module):
|
||||
else:
|
||||
raise ValueError(f"Invalid model type: {config.model_type}")
|
||||
|
||||
self.layer_scatter_modes = LayerScatterModes.init_new(
|
||||
self.layer_facts = LayerFacts.init_new(
|
||||
layer_id=layer_id,
|
||||
num_layers=config.num_hidden_layers,
|
||||
is_layer_sparse=is_layer_sparse,
|
||||
@@ -1124,7 +1124,7 @@ class Qwen3_5LinearDecoderLayer(nn.Module):
|
||||
and _linear_accepts_fp8_tuple(self.linear_attn.in_proj_qkvz)
|
||||
)
|
||||
self.layer_communicator = LayerCommunicator(
|
||||
layer_scatter_modes=self.layer_scatter_modes,
|
||||
layer_facts=self.layer_facts,
|
||||
input_layernorm=self.input_layernorm,
|
||||
post_attention_layernorm=self.post_attention_layernorm,
|
||||
allow_reduce_scatter=True,
|
||||
@@ -1306,7 +1306,7 @@ class Qwen3_5AttentionDecoderLayer(nn.Module):
|
||||
else:
|
||||
raise ValueError(f"Invalid model type: {config.model_type}")
|
||||
|
||||
self.layer_scatter_modes = LayerScatterModes.init_new(
|
||||
self.layer_facts = LayerFacts.init_new(
|
||||
layer_id=layer_id,
|
||||
num_layers=config.num_hidden_layers,
|
||||
is_layer_sparse=is_layer_sparse,
|
||||
@@ -1328,7 +1328,7 @@ class Qwen3_5AttentionDecoderLayer(nn.Module):
|
||||
_enable_qwen35_fused_ar_quant() and _linear_accepts_fp8_tuple(self.qkv_proj)
|
||||
)
|
||||
self.layer_communicator = LayerCommunicator(
|
||||
layer_scatter_modes=self.layer_scatter_modes,
|
||||
layer_facts=self.layer_facts,
|
||||
input_layernorm=self.input_layernorm,
|
||||
post_attention_layernorm=self.post_attention_layernorm,
|
||||
allow_reduce_scatter=True,
|
||||
|
||||
@@ -29,7 +29,7 @@ from transformers import PretrainedConfig
|
||||
from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder
|
||||
from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation
|
||||
from sglang.srt.eplb.expert_location_dispatch import ExpertLocationDispatchInfo
|
||||
from sglang.srt.layers.communicator import LayerCommunicator, LayerScatterModes
|
||||
from sglang.srt.layers.communicator import LayerCommunicator, LayerFacts
|
||||
from sglang.srt.layers.layernorm import RMSNorm
|
||||
from sglang.srt.layers.linear import (
|
||||
QKVParallelLinear,
|
||||
@@ -840,7 +840,7 @@ class Qwen3MoeDecoderLayer(nn.Module):
|
||||
is_previous_layer_sparse = True
|
||||
is_next_layer_sparse = True
|
||||
|
||||
self.layer_scatter_modes = LayerScatterModes.init_new(
|
||||
self.layer_facts = LayerFacts.init_new(
|
||||
layer_id=layer_id,
|
||||
num_layers=config.num_hidden_layers,
|
||||
is_layer_sparse=self.is_layer_sparse,
|
||||
@@ -869,7 +869,7 @@ class Qwen3MoeDecoderLayer(nn.Module):
|
||||
)
|
||||
|
||||
self.layer_communicator = LayerCommunicator(
|
||||
layer_scatter_modes=self.layer_scatter_modes,
|
||||
layer_facts=self.layer_facts,
|
||||
input_layernorm=self.input_layernorm,
|
||||
post_attention_layernorm=self.post_attention_layernorm,
|
||||
allow_reduce_scatter=True,
|
||||
|
||||
@@ -15,7 +15,7 @@ from sglang.srt.configs.qwen3_next import Qwen3NextConfig
|
||||
from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder
|
||||
from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation
|
||||
from sglang.srt.layers.attention.mamba.mamba import mamba_v2_sharded_weight_loader
|
||||
from sglang.srt.layers.communicator import LayerCommunicator, LayerScatterModes
|
||||
from sglang.srt.layers.communicator import LayerCommunicator, LayerFacts
|
||||
from sglang.srt.layers.dp_attention import (
|
||||
is_dp_attention_enabled,
|
||||
)
|
||||
@@ -507,7 +507,7 @@ class Qwen3HybridLinearDecoderLayer(nn.Module):
|
||||
is_next_layer_sparse = True
|
||||
self.layer_id = layer_id
|
||||
|
||||
self.layer_scatter_modes = LayerScatterModes.init_new(
|
||||
self.layer_facts = LayerFacts.init_new(
|
||||
layer_id=layer_id,
|
||||
num_layers=config.num_hidden_layers,
|
||||
is_layer_sparse=self.is_layer_sparse,
|
||||
@@ -539,7 +539,7 @@ class Qwen3HybridLinearDecoderLayer(nn.Module):
|
||||
config.hidden_size, eps=config.rms_norm_eps
|
||||
)
|
||||
self.layer_communicator = LayerCommunicator(
|
||||
layer_scatter_modes=self.layer_scatter_modes,
|
||||
layer_facts=self.layer_facts,
|
||||
input_layernorm=self.input_layernorm,
|
||||
post_attention_layernorm=self.post_attention_layernorm,
|
||||
allow_reduce_scatter=True,
|
||||
@@ -675,7 +675,7 @@ class Qwen3HybridAttentionDecoderLayer(nn.Module):
|
||||
is_previous_layer_sparse = True
|
||||
is_next_layer_sparse = True
|
||||
|
||||
self.layer_scatter_modes = LayerScatterModes.init_new(
|
||||
self.layer_facts = LayerFacts.init_new(
|
||||
layer_id=layer_id,
|
||||
num_layers=config.num_hidden_layers,
|
||||
is_layer_sparse=self.is_layer_sparse,
|
||||
@@ -711,7 +711,7 @@ class Qwen3HybridAttentionDecoderLayer(nn.Module):
|
||||
self.k_norm = GemmaRMSNorm(self.head_dim, eps=config.rms_norm_eps)
|
||||
|
||||
self.layer_communicator = LayerCommunicator(
|
||||
layer_scatter_modes=self.layer_scatter_modes,
|
||||
layer_facts=self.layer_facts,
|
||||
input_layernorm=self.input_layernorm,
|
||||
post_attention_layernorm=self.post_attention_layernorm,
|
||||
allow_reduce_scatter=True,
|
||||
|
||||
@@ -21,7 +21,7 @@ from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation
|
||||
from sglang.srt.layers.activation import SiluAndMul
|
||||
from sglang.srt.layers.communicator import (
|
||||
LayerCommunicator,
|
||||
LayerScatterModes,
|
||||
LayerFacts,
|
||||
enable_moe_dense_fully_dp,
|
||||
)
|
||||
from sglang.srt.layers.dp_attention import (
|
||||
@@ -1047,7 +1047,7 @@ class SarvamMoEMLADecoderLayer(nn.Module):
|
||||
)
|
||||
|
||||
self.attn_tp_size = get_parallel().attn_tp_size
|
||||
self.layer_scatter_modes = LayerScatterModes.init_new(
|
||||
self.layer_facts = LayerFacts.init_new(
|
||||
layer_id=layer_id,
|
||||
num_layers=config.num_hidden_layers,
|
||||
is_layer_sparse=self.is_layer_sparse,
|
||||
@@ -1055,7 +1055,7 @@ class SarvamMoEMLADecoderLayer(nn.Module):
|
||||
is_next_layer_sparse=is_next_layer_sparse,
|
||||
)
|
||||
self.layer_communicator = LayerCommunicator(
|
||||
layer_scatter_modes=self.layer_scatter_modes,
|
||||
layer_facts=self.layer_facts,
|
||||
input_layernorm=self.input_layernorm,
|
||||
post_attention_layernorm=self.post_attention_layernorm,
|
||||
qkv_latent_func=self.self_attn.prepare_qkv_latent,
|
||||
|
||||
@@ -11,7 +11,7 @@ from torch import nn
|
||||
from transformers import PretrainedConfig
|
||||
|
||||
from sglang.srt.layers.activation import SiluAndMul
|
||||
from sglang.srt.layers.communicator import LayerCommunicator, LayerScatterModes
|
||||
from sglang.srt.layers.communicator import LayerCommunicator, LayerFacts
|
||||
from sglang.srt.layers.dp_attention import (
|
||||
is_dp_attention_enabled,
|
||||
)
|
||||
@@ -290,7 +290,7 @@ class SDARBlock(nn.Module):
|
||||
prefix=add_prefix("mlp", prefix),
|
||||
)
|
||||
|
||||
self.layer_scatter_modes = LayerScatterModes.init_new(
|
||||
self.layer_facts = LayerFacts.init_new(
|
||||
layer_id=layer_id,
|
||||
num_layers=config.num_hidden_layers,
|
||||
is_layer_sparse=False,
|
||||
@@ -298,7 +298,7 @@ class SDARBlock(nn.Module):
|
||||
is_next_layer_sparse=False,
|
||||
)
|
||||
self.layer_communicator = LayerCommunicator(
|
||||
layer_scatter_modes=self.layer_scatter_modes,
|
||||
layer_facts=self.layer_facts,
|
||||
input_layernorm=self.input_layernorm,
|
||||
post_attention_layernorm=self.post_attention_layernorm,
|
||||
allow_reduce_scatter=True,
|
||||
|
||||
@@ -15,7 +15,7 @@ from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation
|
||||
from sglang.srt.eplb.expert_location_dispatch import ExpertLocationDispatchInfo
|
||||
from sglang.srt.layers.communicator import (
|
||||
LayerCommunicator,
|
||||
LayerScatterModes,
|
||||
LayerFacts,
|
||||
)
|
||||
from sglang.srt.layers.dp_attention import (
|
||||
is_dp_attention_enabled,
|
||||
@@ -355,7 +355,7 @@ class SDARMoeBlock(nn.Module):
|
||||
prefix=add_prefix("mlp", prefix),
|
||||
)
|
||||
|
||||
self.layer_scatter_modes = LayerScatterModes.init_new(
|
||||
self.layer_facts = LayerFacts.init_new(
|
||||
layer_id=layer_id,
|
||||
num_layers=config.num_hidden_layers,
|
||||
is_layer_sparse=True,
|
||||
@@ -364,7 +364,7 @@ class SDARMoeBlock(nn.Module):
|
||||
)
|
||||
|
||||
self.layer_communicator = LayerCommunicator(
|
||||
layer_scatter_modes=self.layer_scatter_modes,
|
||||
layer_facts=self.layer_facts,
|
||||
input_layernorm=self.input_layernorm,
|
||||
post_attention_layernorm=self.post_attention_layernorm,
|
||||
allow_reduce_scatter=True,
|
||||
|
||||
@@ -22,7 +22,7 @@ from sglang.srt.distributed import (
|
||||
from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation
|
||||
from sglang.srt.layers.activation import SiluAndMul
|
||||
from sglang.srt.layers.attention.vision import VisionAttention
|
||||
from sglang.srt.layers.communicator import LayerCommunicator, LayerScatterModes
|
||||
from sglang.srt.layers.communicator import LayerCommunicator, LayerFacts
|
||||
from sglang.srt.layers.conv import Conv2dLayer
|
||||
from sglang.srt.layers.dp_attention import (
|
||||
is_dp_attention_enabled,
|
||||
@@ -338,7 +338,7 @@ class Step3TextDecoderLayer(nn.Module):
|
||||
)
|
||||
self.is_next_layer_sparse = True if layer_id + 1 in moe_layers_idx else False
|
||||
|
||||
self.layer_scatter_modes = LayerScatterModes.init_new(
|
||||
self.layer_facts = LayerFacts.init_new(
|
||||
layer_id=layer_id,
|
||||
num_layers=config.num_hidden_layers,
|
||||
is_layer_sparse=self.is_layer_sparse,
|
||||
@@ -379,7 +379,7 @@ class Step3TextDecoderLayer(nn.Module):
|
||||
)
|
||||
|
||||
self.layer_communicator = LayerCommunicator(
|
||||
layer_scatter_modes=self.layer_scatter_modes,
|
||||
layer_facts=self.layer_facts,
|
||||
input_layernorm=self.input_layernorm,
|
||||
post_attention_layernorm=self.post_attention_layernorm,
|
||||
allow_deferred_ffn_reduction=False,
|
||||
|
||||
@@ -11,7 +11,7 @@ from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_r
|
||||
from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation
|
||||
from sglang.srt.eplb.expert_location_dispatch import ExpertLocationDispatchInfo
|
||||
from sglang.srt.layers.activation import SiluAndMul
|
||||
from sglang.srt.layers.communicator import LayerCommunicator, LayerScatterModes
|
||||
from sglang.srt.layers.communicator import LayerCommunicator, LayerFacts
|
||||
from sglang.srt.layers.dp_attention import (
|
||||
is_dp_attention_enabled,
|
||||
)
|
||||
@@ -560,7 +560,7 @@ class Step3p5DecoderLayer(nn.Module):
|
||||
)
|
||||
|
||||
# An MTP draft is a one-layer model; layer_id still indexes its config.
|
||||
self.layer_scatter_modes = LayerScatterModes.init_new(
|
||||
self.layer_facts = LayerFacts.init_new(
|
||||
layer_id=0 if is_nextn else layer_id,
|
||||
num_layers=1 if is_nextn else config.num_hidden_layers,
|
||||
is_layer_sparse=self.is_moe_layer,
|
||||
@@ -568,7 +568,7 @@ class Step3p5DecoderLayer(nn.Module):
|
||||
is_next_layer_sparse=self.is_next_layer_sparse,
|
||||
)
|
||||
self.layer_communicator = LayerCommunicator(
|
||||
layer_scatter_modes=self.layer_scatter_modes,
|
||||
layer_facts=self.layer_facts,
|
||||
input_layernorm=self.input_layernorm,
|
||||
post_attention_layernorm=self.post_attention_layernorm,
|
||||
allow_reduce_scatter=True,
|
||||
|
||||
@@ -41,7 +41,7 @@ from sglang.srt.eplb.expert_location_dispatch import ExpertLocationDispatchInfo
|
||||
from sglang.srt.layers.activation import SiluAndMul
|
||||
from sglang.srt.layers.communicator import (
|
||||
LayerCommunicator,
|
||||
LayerScatterModes,
|
||||
LayerFacts,
|
||||
enable_moe_dense_fully_dp,
|
||||
)
|
||||
from sglang.srt.layers.dp_attention import is_dp_attention_enabled
|
||||
@@ -1505,7 +1505,7 @@ class XllmDecoderLayer(nn.Module):
|
||||
is_previous_layer_sparse = _is_sparse(layer_id - 1)
|
||||
is_next_layer_sparse = _is_sparse(layer_id + 1)
|
||||
|
||||
self.layer_scatter_modes = LayerScatterModes.init_new(
|
||||
self.layer_facts = LayerFacts.init_new(
|
||||
layer_id=layer_id,
|
||||
num_layers=config.num_hidden_layers,
|
||||
is_layer_sparse=self.is_layer_sparse,
|
||||
@@ -1538,7 +1538,7 @@ class XllmDecoderLayer(nn.Module):
|
||||
self.input_layernorm = _make_norm(config)
|
||||
self.post_attention_layernorm = _make_norm(config)
|
||||
self.layer_communicator = LayerCommunicator(
|
||||
layer_scatter_modes=self.layer_scatter_modes,
|
||||
layer_facts=self.layer_facts,
|
||||
input_layernorm=self.input_layernorm,
|
||||
post_attention_layernorm=self.post_attention_layernorm,
|
||||
allow_reduce_scatter=True,
|
||||
|
||||
Reference in New Issue
Block a user