[Refactor] Let linear layers select their parallel group (#42426)

This commit is contained in:
Cheng Wan
2026-10-07 16:15:26 -07:00
committed by GitHub
parent 1066decce6
commit 4c5130ac5a
28 changed files with 832 additions and 177 deletions
+66 -18
View File
@@ -6,7 +6,7 @@ from __future__ import annotations
import itertools
import logging
from typing import TYPE_CHECKING, Dict, List, Optional, Tuple
from typing import TYPE_CHECKING, Dict, List, Literal, Optional, Tuple
import torch
from torch import nn
@@ -55,6 +55,40 @@ _disable_hip_linear_quant = _is_hip and get_bool_env_var(
logger = logging.getLogger(__name__)
LinearParallelGroup = Literal["tp", "attn_tp", "replicated"]
def _resolve_linear_partition(
parallel_group: Optional[LinearParallelGroup],
tp_rank: Optional[int],
tp_size: Optional[int],
) -> Tuple[int, int]:
"""Resolve a layer's partition in its construction scope.
Explicit integers remain available for layouts that do not correspond to a
process group. A group selection and explicit integers cannot be combined.
The returned values belong to the layer, including during later reloads.
"""
if parallel_group is not None:
if tp_rank is not None or tp_size is not None:
raise ValueError(
"parallel_group cannot be combined with tp_rank or tp_size"
)
if parallel_group == "replicated":
return 0, 1
if parallel_group not in ("tp", "attn_tp"):
raise ValueError(f"Unknown linear parallel_group: {parallel_group!r}")
parallel = get_parallel()
if parallel_group == "attn_tp":
return parallel.attn_tp_rank, parallel.attn_tp_size
return parallel.tp_rank, parallel.tp_size
if tp_rank is None:
tp_rank = get_parallel().tp_rank
if tp_size is None:
tp_size = get_parallel().tp_size
return tp_rank, tp_size
WEIGHT_LOADER_V2_SUPPORTED = [
"CompressedTensorsLinearMethod",
"AWQLinearMethod",
@@ -326,6 +360,8 @@ class ColumnParallelLinear(LinearBase):
quant_config: Quantization configure.
output_sizes: list of output sizes packed into one output, like for QKV
the list would be size 3.
parallel_group: Select TP, attention TP, or an unsharded partition.
Mutually exclusive with explicit tp_rank and tp_size.
prefix: The name of the layer in the state dict, including all parents
(e.g. model.layers.0.qkv_proj)
"""
@@ -345,6 +381,8 @@ class ColumnParallelLinear(LinearBase):
tp_size: Optional[int] = None,
use_presharded_weights: bool = False,
skip_block_quant_check: bool = False,
*,
parallel_group: Optional[LinearParallelGroup] = None,
):
super().__init__(
input_size, output_size, skip_bias_add, params_dtype, quant_config, prefix
@@ -355,10 +393,8 @@ class ColumnParallelLinear(LinearBase):
self.use_presharded_weights = use_presharded_weights
# Divide the weight matrix along the last dimension.
if tp_rank is None:
tp_rank = get_parallel().tp_rank
if tp_size is None:
tp_size = get_parallel().tp_size
tp_rank, tp_size = _resolve_linear_partition(parallel_group, tp_rank, tp_size)
self.parallel_group = parallel_group
self.tp_rank, self.tp_size = tp_rank, tp_size
assert self.quant_method is not None
self.output_size_per_partition = divide(self.output_size, tp_size)
@@ -504,7 +540,14 @@ class ColumnParallelLinear(LinearBase):
output_parallel = self.quant_method.apply(self, input_, bias)
if self.gather_output:
# All-gather across the partitions.
output = tensor_model_parallel_all_gather(output_parallel)
if self.parallel_group == "replicated":
output = output_parallel
elif self.parallel_group == "attn_tp":
output = get_parallel().attn_tp_group.all_gather(output_parallel, -1)
elif self.parallel_group == "tp":
output = get_parallel().tp_group.all_gather(output_parallel, -1)
else:
output = tensor_model_parallel_all_gather(output_parallel)
else:
output = output_parallel
output_bias = self.bias if self.skip_bias_add else None
@@ -555,13 +598,12 @@ class MergedColumnParallelLinear(ColumnParallelLinear):
tp_rank: Optional[int] = None,
tp_size: Optional[int] = None,
use_presharded_weights: bool = False,
*,
parallel_group: Optional[LinearParallelGroup] = None,
):
self.with_bias = bias
self.output_sizes = output_sizes
if tp_rank is None:
tp_rank = get_parallel().tp_rank
if tp_size is None:
tp_size = get_parallel().tp_size
tp_rank, tp_size = _resolve_linear_partition(parallel_group, tp_rank, tp_size)
self.tp_rank, self.tp_size = tp_rank, tp_size
assert all(output_size % tp_size == 0 for output_size in output_sizes)
self.use_presharded_weights = use_presharded_weights
@@ -578,6 +620,7 @@ class MergedColumnParallelLinear(ColumnParallelLinear):
tp_size=tp_size,
use_presharded_weights=use_presharded_weights,
)
self.parallel_group = parallel_group
self.prefix = prefix
def weight_loader(
@@ -988,6 +1031,8 @@ class QKVParallelLinear(ColumnParallelLinear):
skip_block_quant_check: bool = False,
kv_tp_rank: Optional[int] = None,
kv_tp_size: Optional[int] = None,
*,
parallel_group: Optional[LinearParallelGroup] = None,
):
self.with_bias = bias
self.hidden_size = hidden_size
@@ -998,10 +1043,7 @@ class QKVParallelLinear(ColumnParallelLinear):
total_num_kv_heads = total_num_heads
self.total_num_kv_heads = total_num_kv_heads
# Divide the weight matrix along the last dimension.
if tp_rank is None:
tp_rank = get_parallel().tp_rank
if tp_size is None:
tp_size = get_parallel().tp_size
tp_rank, tp_size = _resolve_linear_partition(parallel_group, tp_rank, tp_size)
self.tp_rank, self.tp_size = tp_rank, tp_size
if kv_tp_rank is None:
kv_tp_rank = tp_rank
@@ -1046,6 +1088,7 @@ class QKVParallelLinear(ColumnParallelLinear):
use_presharded_weights=self.use_presharded_weights,
skip_block_quant_check=skip_block_quant_check,
)
self.parallel_group = parallel_group
def _get_shard_offset_mapping(self, loaded_shard_id: str):
shard_offset_mapping = {
@@ -1439,6 +1482,9 @@ class RowParallelLinear(LinearBase):
We skip adding bias but instead return it.
params_dtype: Data type for the parameters.
quant_config: Quantization configure.
parallel_group: Select TP, attention TP, or an unsharded weight partition.
Mutually exclusive with explicit tp_rank and tp_size.
Reduction and allocation policies remain independent.
"""
def __init__(
@@ -1456,6 +1502,8 @@ class RowParallelLinear(LinearBase):
tp_size: Optional[int] = None,
use_presharded_weights: bool = False,
use_dp_attention_reduce: bool = False,
*,
parallel_group: Optional[LinearParallelGroup] = None,
):
quant_config = None if _disable_hip_linear_quant else quant_config
super().__init__(
@@ -1465,13 +1513,11 @@ class RowParallelLinear(LinearBase):
self.with_bias = bias
self.input_is_parallel = input_is_parallel
self.reduce_results = reduce_results
self.parallel_group = parallel_group
self.use_dp_attention_reduce = use_dp_attention_reduce
# Divide the weight matrix along the last dimension.
if tp_rank is None:
tp_rank = get_parallel().tp_rank
if tp_size is None:
tp_size = get_parallel().tp_size
tp_rank, tp_size = _resolve_linear_partition(parallel_group, tp_rank, tp_size)
self.tp_rank, self.tp_size = tp_rank, tp_size
self.input_size_per_partition = divide(input_size, self.tp_size)
assert self.quant_method is not None
@@ -1682,6 +1728,8 @@ class RowParallelLinear(LinearBase):
)
if quantize_communications:
output = tensor_model_parallel_quant_all_reduce(output_parallel)
elif self.parallel_group == "tp":
output = get_parallel().tp_group.all_reduce(output_parallel)
else:
output = tensor_model_parallel_all_reduce(output_parallel)
else:
+2 -5
View File
@@ -509,7 +509,6 @@ class BailingMoEAttention(nn.Module):
self.hidden_size = config.hidden_size
self.total_num_heads = config.num_attention_heads
self.total_kv_heads = config.num_key_value_heads
attn_tp_rank = get_parallel().attn_tp_rank
attn_tp_size = get_parallel().attn_tp_size
assert self.total_num_heads % attn_tp_size == 0
@@ -542,8 +541,7 @@ class BailingMoEAttention(nn.Module):
bias=(config.use_bias or config.use_qkv_bias),
quant_config=quant_config,
prefix=add_prefix("query_key_value", prefix),
tp_rank=attn_tp_rank,
tp_size=attn_tp_size,
parallel_group="attn_tp",
)
if self.use_qk_norm:
@@ -557,8 +555,7 @@ class BailingMoEAttention(nn.Module):
quant_config=quant_config,
reduce_results=reduce_results,
prefix=add_prefix("dense", prefix),
tp_rank=attn_tp_rank,
tp_size=attn_tp_size,
parallel_group="attn_tp",
)
if hasattr(config, "partial_rotary_factor"):
@@ -459,8 +459,7 @@ class BailingMoELinearAttention(nn.Module):
bias=(config.use_bias or config.use_qkv_bias),
quant_config=quant_config,
prefix=f"{prefix}.qkv_proj",
tp_rank=self.tp_rank,
tp_size=self.tp_size,
parallel_group="attn_tp",
)
if self.use_qk_norm:
@@ -473,8 +472,7 @@ class BailingMoELinearAttention(nn.Module):
bias=False,
quant_config=quant_config,
prefix=f"{prefix}.output_gate",
tp_rank=self.tp_rank,
tp_size=self.tp_size,
parallel_group="attn_tp",
)
self.dense = RowParallelLinear(
self.hidden_inner_size,
@@ -482,8 +480,7 @@ class BailingMoELinearAttention(nn.Module):
bias=config.use_bias,
quant_config=quant_config,
prefix=f"{prefix}.out_proj",
tp_rank=self.tp_rank,
tp_size=self.tp_size,
parallel_group="attn_tp",
reduce_results=False,
)
self.attn = RadixAttention(
+2 -4
View File
@@ -198,8 +198,7 @@ class CLIPAttention(nn.Module):
bias=True,
quant_config=quant_config,
prefix=add_prefix("qkv_proj", prefix),
tp_rank=parallel.attn_tp_rank,
tp_size=parallel.attn_tp_size,
parallel_group="attn_tp",
)
# TODO: this layer shards over attention TP but reduces over the full TP
# group; reduce over the attention-TP group so attention DP and attention
@@ -216,8 +215,7 @@ class CLIPAttention(nn.Module):
bias=True,
quant_config=quant_config,
prefix=add_prefix("proj", prefix),
tp_rank=parallel.attn_tp_rank,
tp_size=parallel.attn_tp_size,
parallel_group="attn_tp",
)
def forward(
+8 -12
View File
@@ -38,6 +38,7 @@ from sglang.srt.layers.layer_boundary import (
from sglang.srt.layers.layer_boundary.residual import batch as residual_batch
from sglang.srt.layers.layernorm import RMSNorm
from sglang.srt.layers.linear import (
LinearParallelGroup,
MergedColumnParallelLinear,
QKVParallelLinear,
ReplicatedLinear,
@@ -77,8 +78,8 @@ class ExaoneMoEMLP(nn.Module):
quant_config: Optional[QuantizationConfig] = None,
reduce_results: bool = True,
prefix: str = "",
tp_rank: Optional[int] = None,
tp_size: Optional[int] = None,
*,
parallel_group: LinearParallelGroup = "tp",
) -> None:
super().__init__()
gateup_quant_config = quant_config
@@ -95,8 +96,7 @@ class ExaoneMoEMLP(nn.Module):
bias=False,
quant_config=gateup_quant_config,
prefix=add_prefix("gate_up_proj", prefix),
tp_rank=tp_rank,
tp_size=tp_size,
parallel_group=parallel_group,
)
self.down_proj = RowParallelLinear(
intermediate_size,
@@ -105,8 +105,7 @@ class ExaoneMoEMLP(nn.Module):
quant_config=down_quant_config,
reduce_results=reduce_results,
prefix=add_prefix("down_proj", prefix),
tp_rank=tp_rank,
tp_size=tp_size,
parallel_group=parallel_group,
)
if hidden_act != "silu":
raise ValueError(
@@ -194,7 +193,7 @@ class ExaoneMoESparseMoEBlock(nn.Module):
reduce_results=False,
prefix=add_prefix("shared_experts", prefix),
**(
dict(tp_rank=0, tp_size=1)
dict(parallel_group="replicated")
if get_moe_a2a_backend().is_deepep()
else {}
),
@@ -310,7 +309,6 @@ class ExaoneMoEAttention(nn.Module):
) -> None:
super().__init__()
self.hidden_size = hidden_size
attn_tp_rank = get_parallel().attn_tp_rank
attn_tp_size = get_parallel().attn_tp_size
self.total_num_heads = num_heads
@@ -351,8 +349,7 @@ class ExaoneMoEAttention(nn.Module):
bias=bias,
quant_config=qkv_quant_config,
prefix=add_prefix("qkv_proj", prefix),
tp_rank=attn_tp_rank,
tp_size=attn_tp_size,
parallel_group="attn_tp",
)
self.o_proj = RowParallelLinear(
self.total_num_heads * self.head_dim,
@@ -361,8 +358,7 @@ class ExaoneMoEAttention(nn.Module):
quant_config=o_quant_config,
reduce_results=False,
prefix=add_prefix("o_proj", prefix),
tp_rank=attn_tp_rank,
tp_size=attn_tp_size,
parallel_group="attn_tp",
)
self.q_norm = RMSNorm(self.head_dim, eps=config.rms_norm_eps)
+2 -4
View File
@@ -157,8 +157,7 @@ class FalconH1HybridAttentionDecoderLayer(nn.Module):
self.total_num_kv_heads,
bias=False,
quant_config=quant_config,
tp_rank=self.attn_tp_rank,
tp_size=self.attn_tp_size,
parallel_group="attn_tp",
)
self.o_proj = RowParallelLinear(
@@ -167,8 +166,7 @@ class FalconH1HybridAttentionDecoderLayer(nn.Module):
bias=False,
quant_config=quant_config,
reduce_results=False,
tp_rank=self.attn_tp_rank,
tp_size=self.attn_tp_size,
parallel_group="attn_tp",
)
self.attn = RadixAttention(
+2 -5
View File
@@ -233,7 +233,6 @@ class Glm4MoeAttention(nn.Module):
self.hidden_size = hidden_size
self.start_layer = start_layer
attn_tp_rank = get_parallel().attn_tp_rank
attn_tp_size = get_parallel().attn_tp_size
self.total_num_heads = num_heads
@@ -264,8 +263,7 @@ class Glm4MoeAttention(nn.Module):
self.total_num_kv_heads,
bias=attention_bias,
quant_config=quant_config,
tp_rank=attn_tp_rank,
tp_size=attn_tp_size,
parallel_group="attn_tp",
prefix=add_prefix("qkv_proj", prefix),
)
@@ -274,8 +272,7 @@ class Glm4MoeAttention(nn.Module):
hidden_size,
bias=False,
quant_config=quant_config,
tp_rank=attn_tp_rank,
tp_size=attn_tp_size,
parallel_group="attn_tp",
reduce_results=False,
prefix=add_prefix("o_proj", prefix),
)
+2 -5
View File
@@ -382,7 +382,6 @@ class GptOssAttention(nn.Module):
self.hidden_size = hidden_size
self.sliding_window_size = sliding_window_size
attn_tp_rank = get_parallel().attn_tp_rank
attn_tp_size = get_parallel().attn_tp_size
self.total_num_heads = num_heads
@@ -413,8 +412,7 @@ class GptOssAttention(nn.Module):
bias=attention_bias,
params_dtype=params_dtype,
quant_config=quant_config,
tp_rank=attn_tp_rank,
tp_size=attn_tp_size,
parallel_group="attn_tp",
prefix=add_prefix("qkv_proj", prefix),
)
@@ -431,8 +429,7 @@ class GptOssAttention(nn.Module):
hidden_size,
bias=attention_bias,
quant_config=quant_config,
tp_rank=attn_tp_rank,
tp_size=attn_tp_size,
parallel_group="attn_tp",
reduce_results=False,
params_dtype=params_dtype,
prefix=add_prefix("o_proj", prefix),
@@ -353,10 +353,9 @@ class InklingAttention(nn.Module):
output_size=self.hidden_size,
bias=o_bias,
prefix=add_prefix("wo_ud", prefix),
tp_rank=attn_tp_rank,
tp_size=attn_tp_size,
reduce_results=False,
parallel_group="attn_tp",
use_dp_attention_reduce=True,
reduce_results=False,
quant_config=quant_config,
)
# --enable-scattered-sconv: the output reduction becomes a hidden-dim
+2 -4
View File
@@ -579,8 +579,7 @@ class InternS2MobiusAttentionDecoderLayer(
self.total_num_kv_heads,
bias=False,
quant_config=quant_config,
tp_rank=self.attn_tp_rank,
tp_size=self.attn_tp_size,
parallel_group="attn_tp",
prefix=add_prefix("qkv_proj", prefix),
)
self.o_proj = RowParallelLinear(
@@ -589,8 +588,7 @@ class InternS2MobiusAttentionDecoderLayer(
bias=False,
quant_config=quant_config,
reduce_results=False,
tp_rank=self.attn_tp_rank,
tp_size=self.attn_tp_size,
parallel_group="attn_tp",
prefix=add_prefix("o_proj", prefix),
)
self.attn = RadixAttention(
+2 -4
View File
@@ -310,8 +310,7 @@ class IQuestQ1Attention(nn.Module):
self.total_num_kv_heads,
bias=False,
quant_config=quant_config,
tp_rank=attn_tp_rank,
tp_size=attn_tp_size,
parallel_group="attn_tp",
prefix=add_prefix("qkv_proj", prefix),
)
# The decoder boundary reduces partial attention outputs. Only a
@@ -327,8 +326,7 @@ class IQuestQ1Attention(nn.Module):
config.hidden_size,
bias=False,
quant_config=quant_config,
tp_rank=attn_tp_rank,
tp_size=attn_tp_size,
parallel_group="attn_tp",
reduce_results=reduce_results,
prefix=add_prefix("o_proj", prefix),
)
+9 -14
View File
@@ -32,6 +32,7 @@ from sglang.srt.layers.layer_boundary.residual import batch as residual_batch
from sglang.srt.layers.layernorm import RMSNorm
from sglang.srt.layers.linear import (
ColumnParallelLinear,
LinearParallelGroup,
MergedColumnParallelLinear,
QKVParallelLinear,
RowParallelLinear,
@@ -68,8 +69,8 @@ class LagunaMLP(nn.Module):
quant_config: Optional[QuantizationConfig] = None,
reduce_results: bool = True,
prefix: str = "",
tp_rank: Optional[int] = None,
tp_size: Optional[int] = None,
*,
parallel_group: LinearParallelGroup = "tp",
) -> None:
super().__init__()
if hidden_act != "silu":
@@ -82,8 +83,7 @@ class LagunaMLP(nn.Module):
bias=False,
quant_config=quant_config,
prefix=add_prefix("gate_up_proj", prefix),
tp_rank=tp_rank,
tp_size=tp_size,
parallel_group=parallel_group,
)
self.down_proj = RowParallelLinear(
intermediate_size,
@@ -92,8 +92,7 @@ class LagunaMLP(nn.Module):
quant_config=quant_config,
reduce_results=reduce_results,
prefix=add_prefix("down_proj", prefix),
tp_rank=tp_rank,
tp_size=tp_size,
parallel_group=parallel_group,
)
self.act_fn = SiluAndMul()
@@ -188,7 +187,7 @@ class LagunaMoE(nn.Module):
quant_config=quant_config,
reduce_results=False,
prefix=add_prefix("shared_expert", prefix),
**(dict(tp_rank=0, tp_size=1) if self._shared_expert_tp1 else {}),
**(dict(parallel_group="replicated") if self._shared_expert_tp1 else {}),
)
def get_moe_weights(self):
@@ -256,7 +255,6 @@ class LagunaAttention(nn.Module):
self.gating = gating != "disabled"
self.gate_per_head = gating == "per-head"
attn_tp_rank = get_parallel().attn_tp_rank
attn_tp_size = get_parallel().attn_tp_size
self.total_num_heads = num_heads
@@ -279,8 +277,7 @@ class LagunaAttention(nn.Module):
self.total_num_kv_heads,
bias=attention_bias,
quant_config=quant_config,
tp_rank=attn_tp_rank,
tp_size=attn_tp_size,
parallel_group="attn_tp",
prefix=add_prefix("qkv_proj", prefix),
)
self.o_proj = RowParallelLinear(
@@ -288,8 +285,7 @@ class LagunaAttention(nn.Module):
hidden_size,
bias=attention_bias,
quant_config=quant_config,
tp_rank=attn_tp_rank,
tp_size=attn_tp_size,
parallel_group="attn_tp",
reduce_results=False,
prefix=add_prefix("o_proj", prefix),
)
@@ -306,8 +302,7 @@ class LagunaAttention(nn.Module):
bias=False,
gather_output=False,
quant_config=quant_config,
tp_rank=attn_tp_rank,
tp_size=attn_tp_size,
parallel_group="attn_tp",
prefix=add_prefix("g_proj", prefix),
)
else:
+2 -5
View File
@@ -427,7 +427,6 @@ class LLaDA2MoeAttention(nn.Module):
self.hidden_size = config.hidden_size
self.total_num_heads = config.num_attention_heads
self.total_kv_heads = config.num_key_value_heads
attn_tp_rank = get_parallel().attn_tp_rank
attn_tp_size = get_parallel().attn_tp_size
assert self.total_num_heads % attn_tp_size == 0
@@ -460,8 +459,7 @@ class LLaDA2MoeAttention(nn.Module):
bias=(config.use_bias or config.use_qkv_bias),
quant_config=quant_config,
prefix=add_prefix("query_key_value", prefix),
tp_rank=attn_tp_rank,
tp_size=attn_tp_size,
parallel_group="attn_tp",
)
if self.use_qk_norm:
@@ -475,8 +473,7 @@ class LLaDA2MoeAttention(nn.Module):
quant_config=quant_config,
reduce_results=reduce_results,
prefix=add_prefix("dense", prefix),
tp_rank=attn_tp_rank,
tp_size=attn_tp_size,
parallel_group="attn_tp",
)
if hasattr(config, "partial_rotary_factor"):
+2 -5
View File
@@ -212,7 +212,6 @@ class Llama4Attention(nn.Module):
self.use_rope = (layer_id + 1) % 4 != 0
self.use_qk_norm = config.use_qk_norm and self.use_rope
attn_tp_rank = get_parallel().attn_tp_rank
attn_tp_size = get_parallel().attn_tp_size
self.total_num_heads = num_heads
@@ -264,8 +263,7 @@ class Llama4Attention(nn.Module):
bias=bias,
quant_config=qkv_quant_config,
prefix=add_prefix("qkv_proj", prefix),
tp_rank=attn_tp_rank,
tp_size=attn_tp_size,
parallel_group="attn_tp",
)
self.o_proj = RowParallelLinear(
@@ -274,8 +272,7 @@ class Llama4Attention(nn.Module):
bias=bias_o_proj,
quant_config=o_quant_config,
prefix=add_prefix("o_proj", prefix),
tp_rank=attn_tp_rank,
tp_size=attn_tp_size,
parallel_group="attn_tp",
reduce_results=False,
)
is_neox_style = True
+2 -5
View File
@@ -164,7 +164,6 @@ class MellumAttention(Qwen3MoeAttention):
self.hidden_size = hidden_size
self.start_layer = start_layer
attn_tp_rank = get_parallel().attn_tp_rank
attn_tp_size = get_parallel().attn_tp_size
self.config = config
@@ -194,8 +193,7 @@ class MellumAttention(Qwen3MoeAttention):
self.total_num_kv_heads,
bias=attention_bias,
quant_config=quant_config,
tp_rank=attn_tp_rank,
tp_size=attn_tp_size,
parallel_group="attn_tp",
prefix=add_prefix("qkv_proj", prefix),
)
@@ -204,8 +202,7 @@ class MellumAttention(Qwen3MoeAttention):
hidden_size,
bias=attention_bias,
quant_config=quant_config,
tp_rank=attn_tp_rank,
tp_size=attn_tp_size,
parallel_group="attn_tp",
reduce_results=False,
prefix=add_prefix("o_proj", prefix),
)
+2 -5
View File
@@ -650,7 +650,6 @@ class MiniMaxM2Attention(nn.Module):
self.hidden_size = config.hidden_size
# Use attention TP rank/size for dp-attention support
attn_tp_rank = get_parallel().attn_tp_rank
attn_tp_size = get_parallel().attn_tp_size
# Get dimensions from config
@@ -696,8 +695,7 @@ class MiniMaxM2Attention(nn.Module):
self.total_num_kv_heads,
bias=False,
quant_config=quant_config,
tp_rank=attn_tp_rank,
tp_size=attn_tp_size,
parallel_group="attn_tp",
prefix=add_prefix("qkv_proj", prefix),
)
@@ -707,8 +705,7 @@ class MiniMaxM2Attention(nn.Module):
bias=False,
reduce_results=False,
quant_config=quant_config,
tp_rank=attn_tp_rank,
tp_size=attn_tp_size,
parallel_group="attn_tp",
prefix=add_prefix("o_proj", prefix),
)
+2 -5
View File
@@ -898,7 +898,6 @@ class MossVLSelfAttention(nn.Module):
super().__init__()
self.hidden_size = config.hidden_size
self.total_num_heads = config.num_attention_heads
attn_tp_rank = get_parallel().attn_tp_rank
attn_tp_size = get_parallel().attn_tp_size
assert self.total_num_heads % attn_tp_size == 0
@@ -928,8 +927,7 @@ class MossVLSelfAttention(nn.Module):
self.total_num_kv_heads,
bias=config.attention_bias,
quant_config=quant_config,
tp_rank=attn_tp_rank,
tp_size=attn_tp_size,
parallel_group="attn_tp",
prefix=add_prefix("qkv_proj", prefix),
)
self.o_proj = RowParallelLinear(
@@ -937,8 +935,7 @@ class MossVLSelfAttention(nn.Module):
config.hidden_size,
bias=config.attention_bias,
quant_config=quant_config,
tp_rank=attn_tp_rank,
tp_size=attn_tp_size,
parallel_group="attn_tp",
reduce_results=False,
prefix=add_prefix("o_proj", prefix),
)
+8 -12
View File
@@ -38,6 +38,7 @@ from sglang.srt.layers.layer_boundary.residual import batch as residual_batch
from sglang.srt.layers.layernorm import RMSNorm
from sglang.srt.layers.linear import (
ColumnParallelLinear,
LinearParallelGroup,
QKVParallelLinear,
ReplicatedLinear,
RowParallelLinear,
@@ -105,9 +106,9 @@ class NemotronHMLP(nn.Module):
quant_config: QuantizationConfig | None = None,
bias: bool = False,
reduce_results: bool = True,
tp_rank: int | None = None,
tp_size: int | None = None,
prefix: str = "",
*,
parallel_group: LinearParallelGroup = "tp",
) -> None:
super().__init__()
@@ -116,8 +117,7 @@ class NemotronHMLP(nn.Module):
output_size=intermediate_size,
bias=bias,
quant_config=quant_config,
tp_rank=tp_rank,
tp_size=tp_size,
parallel_group=parallel_group,
prefix=f"{prefix}.up_proj",
)
self.down_proj = RowParallelLinear(
@@ -126,8 +126,7 @@ class NemotronHMLP(nn.Module):
bias=bias,
quant_config=quant_config,
reduce_results=reduce_results,
tp_rank=tp_rank,
tp_size=tp_size,
parallel_group=parallel_group,
prefix=f"{prefix}.down_proj",
)
self.act_fn = ReLU2()
@@ -232,7 +231,7 @@ class NemotronHMoE(nn.Module):
quant_config=quant_config,
reduce_results=False,
**(
dict(tp_rank=0, tp_size=1)
dict(parallel_group="replicated")
if get_moe_a2a_backend().is_deepep()
or get_moe_a2a_backend().is_flashinfer()
or get_moe_a2a_backend().is_flashinfer_megamoe()
@@ -541,7 +540,6 @@ class NemotronHAttention(nn.Module):
) -> None:
super().__init__()
self.hidden_size = config.hidden_size
tp_rank = get_parallel().attn_tp_rank
tp_size = get_parallel().attn_tp_size
self.total_num_heads = config.num_attention_heads
assert self.total_num_heads % tp_size == 0
@@ -571,8 +569,7 @@ class NemotronHAttention(nn.Module):
self.total_num_kv_heads,
bias=False,
quant_config=quant_config,
tp_rank=tp_rank,
tp_size=tp_size,
parallel_group="attn_tp",
prefix=f"{prefix}.qkv_proj",
)
self.o_proj = RowParallelLinear(
@@ -580,8 +577,7 @@ class NemotronHAttention(nn.Module):
config.hidden_size,
bias=False,
quant_config=quant_config,
tp_rank=tp_rank,
tp_size=tp_size,
parallel_group="attn_tp",
reduce_results=False,
use_dp_attention_reduce=is_dp_attention_enabled(),
prefix=f"{prefix}.o_proj",
+2 -5
View File
@@ -246,7 +246,6 @@ class PhiMoEAttention(nn.Module):
super().__init__()
self.hidden_size = hidden_size
attn_tp_rank = get_parallel().attn_tp_rank
attn_tp_size = get_parallel().attn_tp_size
self.total_num_heads = num_heads
@@ -279,8 +278,7 @@ class PhiMoEAttention(nn.Module):
self.total_num_kv_heads,
bias=attention_bias,
quant_config=quant_config,
tp_rank=attn_tp_rank,
tp_size=attn_tp_size,
parallel_group="attn_tp",
prefix=add_prefix("qkv_proj", prefix),
)
# TODO: this layer shards over attention TP but reduces over the full TP
@@ -294,8 +292,7 @@ class PhiMoEAttention(nn.Module):
hidden_size,
bias=attention_bias,
quant_config=quant_config,
tp_rank=attn_tp_rank,
tp_size=attn_tp_size,
parallel_group="attn_tp",
prefix=add_prefix("o_proj", prefix),
)
self.rotary_emb = get_rope(
+8 -12
View File
@@ -56,6 +56,7 @@ from sglang.srt.layers.layer_boundary import (
from sglang.srt.layers.layer_boundary.residual import batch as residual_batch
from sglang.srt.layers.layernorm import RMSNorm
from sglang.srt.layers.linear import (
LinearParallelGroup,
MergedColumnParallelLinear,
QKVParallelLinear,
ReplicatedLinear,
@@ -176,8 +177,8 @@ class Qwen2MoeMLP(nn.Module):
quant_config: Optional[QuantizationConfig] = None,
reduce_results: bool = True,
prefix: str = "",
tp_rank: Optional[int] = None,
tp_size: Optional[int] = None,
*,
parallel_group: LinearParallelGroup = "tp",
) -> None:
super().__init__()
self.gate_up_proj = MergedColumnParallelLinear(
@@ -186,8 +187,7 @@ class Qwen2MoeMLP(nn.Module):
bias=False,
quant_config=quant_config,
prefix=add_prefix("gate_up_proj", prefix),
tp_rank=tp_rank,
tp_size=tp_size,
parallel_group=parallel_group,
)
self.down_proj = RowParallelLinear(
intermediate_size,
@@ -196,8 +196,7 @@ class Qwen2MoeMLP(nn.Module):
quant_config=quant_config,
reduce_results=reduce_results,
prefix=add_prefix("down_proj", prefix),
tp_rank=tp_rank,
tp_size=tp_size,
parallel_group=parallel_group,
)
if hidden_act != "silu":
raise ValueError(
@@ -365,7 +364,7 @@ class Qwen2MoeSparseMoeBlock(nn.Module):
reduce_results=False,
prefix=add_prefix("shared_expert", prefix),
**(
dict(tp_rank=0, tp_size=1)
dict(parallel_group="replicated")
if (
get_moe_a2a_backend().is_deepep()
or get_moe_a2a_backend().is_mori()
@@ -911,7 +910,6 @@ class Qwen2MoeAttention(nn.Module):
super().__init__()
self.hidden_size = hidden_size
attn_tp_rank = get_parallel().attn_tp_rank
attn_tp_size = get_parallel().attn_tp_size
self.total_num_heads = num_heads
@@ -941,8 +939,7 @@ class Qwen2MoeAttention(nn.Module):
self.total_num_kv_heads,
bias=qkv_bias,
quant_config=quant_config,
tp_rank=attn_tp_rank,
tp_size=attn_tp_size,
parallel_group="attn_tp",
prefix=add_prefix("qkv_proj", prefix),
)
@@ -951,8 +948,7 @@ class Qwen2MoeAttention(nn.Module):
hidden_size,
bias=False,
quant_config=quant_config,
tp_rank=attn_tp_rank,
tp_size=attn_tp_size,
parallel_group="attn_tp",
reduce_results=False,
prefix=add_prefix("o_proj", prefix),
)
+2 -5
View File
@@ -86,7 +86,6 @@ class Qwen3Attention(nn.Module):
self.hidden_size = hidden_size
self.start_layer = start_layer
self.total_num_heads = num_heads
attn_tp_rank = get_parallel().attn_tp_rank
attn_tp_size = get_parallel().attn_tp_size
assert self.total_num_heads % attn_tp_size == 0
@@ -126,8 +125,7 @@ class Qwen3Attention(nn.Module):
self.total_num_kv_heads,
bias=attention_bias,
quant_config=quant_config,
tp_rank=attn_tp_rank,
tp_size=attn_tp_size,
parallel_group="attn_tp",
prefix=add_prefix("qkv_proj", prefix),
)
self.o_proj = RowParallelLinear(
@@ -135,8 +133,7 @@ class Qwen3Attention(nn.Module):
hidden_size,
bias=attention_bias,
quant_config=quant_config,
tp_rank=attn_tp_rank,
tp_size=attn_tp_size,
parallel_group="attn_tp",
reduce_results=False,
prefix=add_prefix("o_proj", prefix),
)
+2 -5
View File
@@ -512,7 +512,6 @@ class Qwen3MoeAttention(nn.Module):
self.hidden_size = hidden_size
self.start_layer = start_layer
attn_tp_rank = get_parallel().attn_tp_rank
attn_tp_size = get_parallel().attn_tp_size
self.config = config
@@ -543,8 +542,7 @@ class Qwen3MoeAttention(nn.Module):
self.total_num_kv_heads,
bias=attention_bias,
quant_config=quant_config,
tp_rank=attn_tp_rank,
tp_size=attn_tp_size,
parallel_group="attn_tp",
prefix=add_prefix("qkv_proj", prefix),
)
@@ -553,8 +551,7 @@ class Qwen3MoeAttention(nn.Module):
hidden_size,
bias=attention_bias,
quant_config=quant_config,
tp_rank=attn_tp_rank,
tp_size=attn_tp_size,
parallel_group="attn_tp",
reduce_results=False,
prefix=add_prefix("o_proj", prefix),
)
+2 -4
View File
@@ -641,8 +641,7 @@ class Qwen3HybridAttentionDecoderLayer(nn.Module):
and quant_config.get_name() != "modelopt_fp4"
else None
),
tp_rank=self.attn_tp_rank,
tp_size=self.attn_tp_size,
parallel_group="attn_tp",
prefix=add_prefix("qkv_proj", prefix),
)
@@ -652,8 +651,7 @@ class Qwen3HybridAttentionDecoderLayer(nn.Module):
bias=False,
quant_config=quant_config,
reduce_results=False,
tp_rank=self.attn_tp_rank,
tp_size=self.attn_tp_size,
parallel_group="attn_tp",
prefix=add_prefix("o_proj", prefix),
)
+2 -5
View File
@@ -99,7 +99,6 @@ class SDARAttention(nn.Module):
self.layer_id = layer_id
self.hidden_size = config.hidden_size
self.total_num_heads = config.num_attention_heads
attn_tp_rank = get_parallel().attn_tp_rank
attn_tp_size = get_parallel().attn_tp_size
assert self.total_num_heads % attn_tp_size == 0
@@ -129,8 +128,7 @@ class SDARAttention(nn.Module):
self.total_num_kv_heads,
bias=getattr(config, "attention_bias", False),
quant_config=quant_config,
tp_rank=attn_tp_rank,
tp_size=attn_tp_size,
parallel_group="attn_tp",
prefix=add_prefix("qkv_proj", prefix),
)
self.o_proj = RowParallelLinear(
@@ -139,8 +137,7 @@ class SDARAttention(nn.Module):
bias=getattr(config, "attention_bias", False),
quant_config=quant_config,
reduce_results=reduce_results,
tp_rank=attn_tp_rank,
tp_size=attn_tp_size,
parallel_group="attn_tp",
prefix=add_prefix("o_proj", prefix),
)
+3 -7
View File
@@ -105,7 +105,6 @@ class Spark2_5Attention(nn.Module):
super().__init__()
self.hidden_size = hidden_size
self.total_num_heads = num_heads
attn_tp_rank = get_parallel().attn_tp_rank
attn_tp_size = get_parallel().attn_tp_size
assert self.total_num_heads % attn_tp_size == 0
@@ -139,8 +138,7 @@ class Spark2_5Attention(nn.Module):
self.total_num_kv_heads,
bias=False,
quant_config=quant_config,
tp_rank=attn_tp_rank,
tp_size=attn_tp_size,
parallel_group="attn_tp",
prefix=add_prefix("q_k_v_proj", prefix),
)
if self.headwise_attn_output_gate:
@@ -149,8 +147,7 @@ class Spark2_5Attention(nn.Module):
self.total_num_heads,
bias=False,
quant_config=None, # g_proj keeps bf16.
tp_rank=attn_tp_rank,
tp_size=attn_tp_size,
parallel_group="attn_tp",
prefix=add_prefix("g_proj", prefix),
)
@@ -159,8 +156,7 @@ class Spark2_5Attention(nn.Module):
hidden_size,
bias=False,
quant_config=quant_config,
tp_rank=attn_tp_rank,
tp_size=attn_tp_size,
parallel_group="attn_tp",
reduce_results=False,
prefix=add_prefix("out_proj", prefix),
)
+9 -8
View File
@@ -19,6 +19,7 @@ from sglang.srt.layers.layer_boundary.residual import batch as residual_batch
from sglang.srt.layers.layernorm import GemmaRMSNorm
from sglang.srt.layers.linear import (
ColumnParallelLinear,
LinearParallelGroup,
MergedColumnParallelLinear,
QKVParallelLinear,
ReplicatedLinear,
@@ -67,6 +68,8 @@ class Step3p5MLP(nn.Module):
quant_config: Optional[QuantizationConfig] = None,
prefix: str = "",
reduce_results: bool = True,
*,
parallel_group: LinearParallelGroup = "tp",
) -> None:
super().__init__()
self.hidden_size = hidden_size
@@ -76,6 +79,7 @@ class Step3p5MLP(nn.Module):
[intermediate_size] * 2,
bias=False,
quant_config=quant_config,
parallel_group=parallel_group,
prefix=add_prefix("gate_up_proj", prefix),
)
self.down_proj = RowParallelLinear(
@@ -83,6 +87,7 @@ class Step3p5MLP(nn.Module):
hidden_size,
bias=False,
quant_config=quant_config,
parallel_group=parallel_group,
prefix=add_prefix("down_proj", prefix),
reduce_results=reduce_results,
)
@@ -287,7 +292,6 @@ class Step3p5Attention(nn.Module):
super().__init__()
self.hidden_size = hidden_size
self.total_num_heads = num_heads
attn_tp_rank = get_parallel().attn_tp_rank
attn_tp_size = get_parallel().attn_tp_size
assert self.total_num_heads % attn_tp_size == 0
@@ -318,8 +322,7 @@ class Step3p5Attention(nn.Module):
self.total_num_kv_heads,
bias=False,
quant_config=quant_config,
tp_rank=attn_tp_rank,
tp_size=attn_tp_size,
parallel_group="attn_tp",
prefix=add_prefix("qkv_proj", prefix),
)
self.o_proj = RowParallelLinear(
@@ -327,8 +330,7 @@ class Step3p5Attention(nn.Module):
hidden_size,
bias=False,
quant_config=quant_config,
tp_rank=attn_tp_rank,
tp_size=attn_tp_size,
parallel_group="attn_tp",
reduce_results=False,
prefix=add_prefix("o_proj", prefix),
)
@@ -339,8 +341,7 @@ class Step3p5Attention(nn.Module):
hidden_size,
self.total_num_heads,
bias=False,
tp_rank=attn_tp_rank,
tp_size=attn_tp_size,
parallel_group="attn_tp",
prefix=add_prefix("g_proj", prefix),
)
@@ -490,7 +491,7 @@ class Step3p5DecoderLayer(nn.Module):
prefix=add_prefix("share_expert", prefix),
reduce_results=False,
**(
dict(tp_rank=0, tp_size=1)
dict(parallel_group="replicated")
if get_moe_a2a_backend().is_deepep()
or get_moe_a2a_backend().is_mooncake()
or get_moe_a2a_backend().is_nixl()
+2 -5
View File
@@ -1081,7 +1081,6 @@ class XllmAttention(nn.Module):
super().__init__()
self.hidden_size = hidden_size
attn_tp_rank = get_parallel().attn_tp_rank
attn_tp_size = get_parallel().attn_tp_size
self.total_num_heads = num_heads
@@ -1107,8 +1106,7 @@ class XllmAttention(nn.Module):
self.total_num_kv_heads,
bias=qkv_bias,
quant_config=quant_config,
tp_rank=attn_tp_rank,
tp_size=attn_tp_size,
parallel_group="attn_tp",
prefix=add_prefix("qkv_proj", prefix),
)
@@ -1117,8 +1115,7 @@ class XllmAttention(nn.Module):
hidden_size,
bias=qkv_bias,
quant_config=quant_config,
tp_rank=attn_tp_rank,
tp_size=attn_tp_size,
parallel_group="attn_tp",
reduce_results=False,
prefix=add_prefix("o_proj", prefix),
)