diff --git a/python/sglang/srt/layers/linear.py b/python/sglang/srt/layers/linear.py index 31ed3f6835..380ff4b941 100644 --- a/python/sglang/srt/layers/linear.py +++ b/python/sglang/srt/layers/linear.py @@ -6,7 +6,7 @@ from __future__ import annotations import itertools import logging -from typing import TYPE_CHECKING, Dict, List, Optional, Tuple +from typing import TYPE_CHECKING, Dict, List, Literal, Optional, Tuple import torch from torch import nn @@ -55,6 +55,40 @@ _disable_hip_linear_quant = _is_hip and get_bool_env_var( logger = logging.getLogger(__name__) +LinearParallelGroup = Literal["tp", "attn_tp", "replicated"] + + +def _resolve_linear_partition( + parallel_group: Optional[LinearParallelGroup], + tp_rank: Optional[int], + tp_size: Optional[int], +) -> Tuple[int, int]: + """Resolve a layer's partition in its construction scope. + + Explicit integers remain available for layouts that do not correspond to a + process group. A group selection and explicit integers cannot be combined. + The returned values belong to the layer, including during later reloads. + """ + if parallel_group is not None: + if tp_rank is not None or tp_size is not None: + raise ValueError( + "parallel_group cannot be combined with tp_rank or tp_size" + ) + if parallel_group == "replicated": + return 0, 1 + if parallel_group not in ("tp", "attn_tp"): + raise ValueError(f"Unknown linear parallel_group: {parallel_group!r}") + parallel = get_parallel() + if parallel_group == "attn_tp": + return parallel.attn_tp_rank, parallel.attn_tp_size + return parallel.tp_rank, parallel.tp_size + if tp_rank is None: + tp_rank = get_parallel().tp_rank + if tp_size is None: + tp_size = get_parallel().tp_size + return tp_rank, tp_size + + WEIGHT_LOADER_V2_SUPPORTED = [ "CompressedTensorsLinearMethod", "AWQLinearMethod", @@ -326,6 +360,8 @@ class ColumnParallelLinear(LinearBase): quant_config: Quantization configure. output_sizes: list of output sizes packed into one output, like for QKV the list would be size 3. + parallel_group: Select TP, attention TP, or an unsharded partition. + Mutually exclusive with explicit tp_rank and tp_size. prefix: The name of the layer in the state dict, including all parents (e.g. model.layers.0.qkv_proj) """ @@ -345,6 +381,8 @@ class ColumnParallelLinear(LinearBase): tp_size: Optional[int] = None, use_presharded_weights: bool = False, skip_block_quant_check: bool = False, + *, + parallel_group: Optional[LinearParallelGroup] = None, ): super().__init__( input_size, output_size, skip_bias_add, params_dtype, quant_config, prefix @@ -355,10 +393,8 @@ class ColumnParallelLinear(LinearBase): self.use_presharded_weights = use_presharded_weights # Divide the weight matrix along the last dimension. - if tp_rank is None: - tp_rank = get_parallel().tp_rank - if tp_size is None: - tp_size = get_parallel().tp_size + tp_rank, tp_size = _resolve_linear_partition(parallel_group, tp_rank, tp_size) + self.parallel_group = parallel_group self.tp_rank, self.tp_size = tp_rank, tp_size assert self.quant_method is not None self.output_size_per_partition = divide(self.output_size, tp_size) @@ -504,7 +540,14 @@ class ColumnParallelLinear(LinearBase): output_parallel = self.quant_method.apply(self, input_, bias) if self.gather_output: # All-gather across the partitions. - output = tensor_model_parallel_all_gather(output_parallel) + if self.parallel_group == "replicated": + output = output_parallel + elif self.parallel_group == "attn_tp": + output = get_parallel().attn_tp_group.all_gather(output_parallel, -1) + elif self.parallel_group == "tp": + output = get_parallel().tp_group.all_gather(output_parallel, -1) + else: + output = tensor_model_parallel_all_gather(output_parallel) else: output = output_parallel output_bias = self.bias if self.skip_bias_add else None @@ -555,13 +598,12 @@ class MergedColumnParallelLinear(ColumnParallelLinear): tp_rank: Optional[int] = None, tp_size: Optional[int] = None, use_presharded_weights: bool = False, + *, + parallel_group: Optional[LinearParallelGroup] = None, ): self.with_bias = bias self.output_sizes = output_sizes - if tp_rank is None: - tp_rank = get_parallel().tp_rank - if tp_size is None: - tp_size = get_parallel().tp_size + tp_rank, tp_size = _resolve_linear_partition(parallel_group, tp_rank, tp_size) self.tp_rank, self.tp_size = tp_rank, tp_size assert all(output_size % tp_size == 0 for output_size in output_sizes) self.use_presharded_weights = use_presharded_weights @@ -578,6 +620,7 @@ class MergedColumnParallelLinear(ColumnParallelLinear): tp_size=tp_size, use_presharded_weights=use_presharded_weights, ) + self.parallel_group = parallel_group self.prefix = prefix def weight_loader( @@ -988,6 +1031,8 @@ class QKVParallelLinear(ColumnParallelLinear): skip_block_quant_check: bool = False, kv_tp_rank: Optional[int] = None, kv_tp_size: Optional[int] = None, + *, + parallel_group: Optional[LinearParallelGroup] = None, ): self.with_bias = bias self.hidden_size = hidden_size @@ -998,10 +1043,7 @@ class QKVParallelLinear(ColumnParallelLinear): total_num_kv_heads = total_num_heads self.total_num_kv_heads = total_num_kv_heads # Divide the weight matrix along the last dimension. - if tp_rank is None: - tp_rank = get_parallel().tp_rank - if tp_size is None: - tp_size = get_parallel().tp_size + tp_rank, tp_size = _resolve_linear_partition(parallel_group, tp_rank, tp_size) self.tp_rank, self.tp_size = tp_rank, tp_size if kv_tp_rank is None: kv_tp_rank = tp_rank @@ -1046,6 +1088,7 @@ class QKVParallelLinear(ColumnParallelLinear): use_presharded_weights=self.use_presharded_weights, skip_block_quant_check=skip_block_quant_check, ) + self.parallel_group = parallel_group def _get_shard_offset_mapping(self, loaded_shard_id: str): shard_offset_mapping = { @@ -1439,6 +1482,9 @@ class RowParallelLinear(LinearBase): We skip adding bias but instead return it. params_dtype: Data type for the parameters. quant_config: Quantization configure. + parallel_group: Select TP, attention TP, or an unsharded weight partition. + Mutually exclusive with explicit tp_rank and tp_size. + Reduction and allocation policies remain independent. """ def __init__( @@ -1456,6 +1502,8 @@ class RowParallelLinear(LinearBase): tp_size: Optional[int] = None, use_presharded_weights: bool = False, use_dp_attention_reduce: bool = False, + *, + parallel_group: Optional[LinearParallelGroup] = None, ): quant_config = None if _disable_hip_linear_quant else quant_config super().__init__( @@ -1465,13 +1513,11 @@ class RowParallelLinear(LinearBase): self.with_bias = bias self.input_is_parallel = input_is_parallel self.reduce_results = reduce_results + self.parallel_group = parallel_group self.use_dp_attention_reduce = use_dp_attention_reduce # Divide the weight matrix along the last dimension. - if tp_rank is None: - tp_rank = get_parallel().tp_rank - if tp_size is None: - tp_size = get_parallel().tp_size + tp_rank, tp_size = _resolve_linear_partition(parallel_group, tp_rank, tp_size) self.tp_rank, self.tp_size = tp_rank, tp_size self.input_size_per_partition = divide(input_size, self.tp_size) assert self.quant_method is not None @@ -1682,6 +1728,8 @@ class RowParallelLinear(LinearBase): ) if quantize_communications: output = tensor_model_parallel_quant_all_reduce(output_parallel) + elif self.parallel_group == "tp": + output = get_parallel().tp_group.all_reduce(output_parallel) else: output = tensor_model_parallel_all_reduce(output_parallel) else: diff --git a/python/sglang/srt/models/bailing_moe.py b/python/sglang/srt/models/bailing_moe.py index ceda0c2a3a..8e01dc98df 100644 --- a/python/sglang/srt/models/bailing_moe.py +++ b/python/sglang/srt/models/bailing_moe.py @@ -509,7 +509,6 @@ class BailingMoEAttention(nn.Module): self.hidden_size = config.hidden_size self.total_num_heads = config.num_attention_heads self.total_kv_heads = config.num_key_value_heads - attn_tp_rank = get_parallel().attn_tp_rank attn_tp_size = get_parallel().attn_tp_size assert self.total_num_heads % attn_tp_size == 0 @@ -542,8 +541,7 @@ class BailingMoEAttention(nn.Module): bias=(config.use_bias or config.use_qkv_bias), quant_config=quant_config, prefix=add_prefix("query_key_value", prefix), - tp_rank=attn_tp_rank, - tp_size=attn_tp_size, + parallel_group="attn_tp", ) if self.use_qk_norm: @@ -557,8 +555,7 @@ class BailingMoEAttention(nn.Module): quant_config=quant_config, reduce_results=reduce_results, prefix=add_prefix("dense", prefix), - tp_rank=attn_tp_rank, - tp_size=attn_tp_size, + parallel_group="attn_tp", ) if hasattr(config, "partial_rotary_factor"): diff --git a/python/sglang/srt/models/bailing_moe_linear.py b/python/sglang/srt/models/bailing_moe_linear.py index ef27368c24..4e238bfa72 100644 --- a/python/sglang/srt/models/bailing_moe_linear.py +++ b/python/sglang/srt/models/bailing_moe_linear.py @@ -459,8 +459,7 @@ class BailingMoELinearAttention(nn.Module): bias=(config.use_bias or config.use_qkv_bias), quant_config=quant_config, prefix=f"{prefix}.qkv_proj", - tp_rank=self.tp_rank, - tp_size=self.tp_size, + parallel_group="attn_tp", ) if self.use_qk_norm: @@ -473,8 +472,7 @@ class BailingMoELinearAttention(nn.Module): bias=False, quant_config=quant_config, prefix=f"{prefix}.output_gate", - tp_rank=self.tp_rank, - tp_size=self.tp_size, + parallel_group="attn_tp", ) self.dense = RowParallelLinear( self.hidden_inner_size, @@ -482,8 +480,7 @@ class BailingMoELinearAttention(nn.Module): bias=config.use_bias, quant_config=quant_config, prefix=f"{prefix}.out_proj", - tp_rank=self.tp_rank, - tp_size=self.tp_size, + parallel_group="attn_tp", reduce_results=False, ) self.attn = RadixAttention( diff --git a/python/sglang/srt/models/clip.py b/python/sglang/srt/models/clip.py index 5addc3725a..f588e193b4 100644 --- a/python/sglang/srt/models/clip.py +++ b/python/sglang/srt/models/clip.py @@ -198,8 +198,7 @@ class CLIPAttention(nn.Module): bias=True, quant_config=quant_config, prefix=add_prefix("qkv_proj", prefix), - tp_rank=parallel.attn_tp_rank, - tp_size=parallel.attn_tp_size, + parallel_group="attn_tp", ) # TODO: this layer shards over attention TP but reduces over the full TP # group; reduce over the attention-TP group so attention DP and attention @@ -216,8 +215,7 @@ class CLIPAttention(nn.Module): bias=True, quant_config=quant_config, prefix=add_prefix("proj", prefix), - tp_rank=parallel.attn_tp_rank, - tp_size=parallel.attn_tp_size, + parallel_group="attn_tp", ) def forward( diff --git a/python/sglang/srt/models/exaone_moe.py b/python/sglang/srt/models/exaone_moe.py index 6b618d3e4c..503d1c6e9f 100755 --- a/python/sglang/srt/models/exaone_moe.py +++ b/python/sglang/srt/models/exaone_moe.py @@ -38,6 +38,7 @@ from sglang.srt.layers.layer_boundary import ( from sglang.srt.layers.layer_boundary.residual import batch as residual_batch from sglang.srt.layers.layernorm import RMSNorm from sglang.srt.layers.linear import ( + LinearParallelGroup, MergedColumnParallelLinear, QKVParallelLinear, ReplicatedLinear, @@ -77,8 +78,8 @@ class ExaoneMoEMLP(nn.Module): quant_config: Optional[QuantizationConfig] = None, reduce_results: bool = True, prefix: str = "", - tp_rank: Optional[int] = None, - tp_size: Optional[int] = None, + *, + parallel_group: LinearParallelGroup = "tp", ) -> None: super().__init__() gateup_quant_config = quant_config @@ -95,8 +96,7 @@ class ExaoneMoEMLP(nn.Module): bias=False, quant_config=gateup_quant_config, prefix=add_prefix("gate_up_proj", prefix), - tp_rank=tp_rank, - tp_size=tp_size, + parallel_group=parallel_group, ) self.down_proj = RowParallelLinear( intermediate_size, @@ -105,8 +105,7 @@ class ExaoneMoEMLP(nn.Module): quant_config=down_quant_config, reduce_results=reduce_results, prefix=add_prefix("down_proj", prefix), - tp_rank=tp_rank, - tp_size=tp_size, + parallel_group=parallel_group, ) if hidden_act != "silu": raise ValueError( @@ -194,7 +193,7 @@ class ExaoneMoESparseMoEBlock(nn.Module): reduce_results=False, prefix=add_prefix("shared_experts", prefix), **( - dict(tp_rank=0, tp_size=1) + dict(parallel_group="replicated") if get_moe_a2a_backend().is_deepep() else {} ), @@ -310,7 +309,6 @@ class ExaoneMoEAttention(nn.Module): ) -> None: super().__init__() self.hidden_size = hidden_size - attn_tp_rank = get_parallel().attn_tp_rank attn_tp_size = get_parallel().attn_tp_size self.total_num_heads = num_heads @@ -351,8 +349,7 @@ class ExaoneMoEAttention(nn.Module): bias=bias, quant_config=qkv_quant_config, prefix=add_prefix("qkv_proj", prefix), - tp_rank=attn_tp_rank, - tp_size=attn_tp_size, + parallel_group="attn_tp", ) self.o_proj = RowParallelLinear( self.total_num_heads * self.head_dim, @@ -361,8 +358,7 @@ class ExaoneMoEAttention(nn.Module): quant_config=o_quant_config, reduce_results=False, prefix=add_prefix("o_proj", prefix), - tp_rank=attn_tp_rank, - tp_size=attn_tp_size, + parallel_group="attn_tp", ) self.q_norm = RMSNorm(self.head_dim, eps=config.rms_norm_eps) diff --git a/python/sglang/srt/models/falcon_h1.py b/python/sglang/srt/models/falcon_h1.py index 8d2d70c096..86c1c1df02 100644 --- a/python/sglang/srt/models/falcon_h1.py +++ b/python/sglang/srt/models/falcon_h1.py @@ -157,8 +157,7 @@ class FalconH1HybridAttentionDecoderLayer(nn.Module): self.total_num_kv_heads, bias=False, quant_config=quant_config, - tp_rank=self.attn_tp_rank, - tp_size=self.attn_tp_size, + parallel_group="attn_tp", ) self.o_proj = RowParallelLinear( @@ -167,8 +166,7 @@ class FalconH1HybridAttentionDecoderLayer(nn.Module): bias=False, quant_config=quant_config, reduce_results=False, - tp_rank=self.attn_tp_rank, - tp_size=self.attn_tp_size, + parallel_group="attn_tp", ) self.attn = RadixAttention( diff --git a/python/sglang/srt/models/glm4_moe.py b/python/sglang/srt/models/glm4_moe.py index f8ca48d745..ae4b349d9b 100644 --- a/python/sglang/srt/models/glm4_moe.py +++ b/python/sglang/srt/models/glm4_moe.py @@ -233,7 +233,6 @@ class Glm4MoeAttention(nn.Module): self.hidden_size = hidden_size self.start_layer = start_layer - attn_tp_rank = get_parallel().attn_tp_rank attn_tp_size = get_parallel().attn_tp_size self.total_num_heads = num_heads @@ -264,8 +263,7 @@ class Glm4MoeAttention(nn.Module): self.total_num_kv_heads, bias=attention_bias, quant_config=quant_config, - tp_rank=attn_tp_rank, - tp_size=attn_tp_size, + parallel_group="attn_tp", prefix=add_prefix("qkv_proj", prefix), ) @@ -274,8 +272,7 @@ class Glm4MoeAttention(nn.Module): hidden_size, bias=False, quant_config=quant_config, - tp_rank=attn_tp_rank, - tp_size=attn_tp_size, + parallel_group="attn_tp", reduce_results=False, prefix=add_prefix("o_proj", prefix), ) diff --git a/python/sglang/srt/models/gpt_oss.py b/python/sglang/srt/models/gpt_oss.py index 335c537fbb..0befe23721 100644 --- a/python/sglang/srt/models/gpt_oss.py +++ b/python/sglang/srt/models/gpt_oss.py @@ -382,7 +382,6 @@ class GptOssAttention(nn.Module): self.hidden_size = hidden_size self.sliding_window_size = sliding_window_size - attn_tp_rank = get_parallel().attn_tp_rank attn_tp_size = get_parallel().attn_tp_size self.total_num_heads = num_heads @@ -413,8 +412,7 @@ class GptOssAttention(nn.Module): bias=attention_bias, params_dtype=params_dtype, quant_config=quant_config, - tp_rank=attn_tp_rank, - tp_size=attn_tp_size, + parallel_group="attn_tp", prefix=add_prefix("qkv_proj", prefix), ) @@ -431,8 +429,7 @@ class GptOssAttention(nn.Module): hidden_size, bias=attention_bias, quant_config=quant_config, - tp_rank=attn_tp_rank, - tp_size=attn_tp_size, + parallel_group="attn_tp", reduce_results=False, params_dtype=params_dtype, prefix=add_prefix("o_proj", prefix), diff --git a/python/sglang/srt/models/inkling_common/attn.py b/python/sglang/srt/models/inkling_common/attn.py index 76a623f5e8..74dfc3e6bf 100644 --- a/python/sglang/srt/models/inkling_common/attn.py +++ b/python/sglang/srt/models/inkling_common/attn.py @@ -353,10 +353,9 @@ class InklingAttention(nn.Module): output_size=self.hidden_size, bias=o_bias, prefix=add_prefix("wo_ud", prefix), - tp_rank=attn_tp_rank, - tp_size=attn_tp_size, - reduce_results=False, + parallel_group="attn_tp", use_dp_attention_reduce=True, + reduce_results=False, quant_config=quant_config, ) # --enable-scattered-sconv: the output reduction becomes a hidden-dim diff --git a/python/sglang/srt/models/interns2_mobius.py b/python/sglang/srt/models/interns2_mobius.py index 7a44063015..fadb815cb4 100644 --- a/python/sglang/srt/models/interns2_mobius.py +++ b/python/sglang/srt/models/interns2_mobius.py @@ -579,8 +579,7 @@ class InternS2MobiusAttentionDecoderLayer( self.total_num_kv_heads, bias=False, quant_config=quant_config, - tp_rank=self.attn_tp_rank, - tp_size=self.attn_tp_size, + parallel_group="attn_tp", prefix=add_prefix("qkv_proj", prefix), ) self.o_proj = RowParallelLinear( @@ -589,8 +588,7 @@ class InternS2MobiusAttentionDecoderLayer( bias=False, quant_config=quant_config, reduce_results=False, - tp_rank=self.attn_tp_rank, - tp_size=self.attn_tp_size, + parallel_group="attn_tp", prefix=add_prefix("o_proj", prefix), ) self.attn = RadixAttention( diff --git a/python/sglang/srt/models/iquest_q1.py b/python/sglang/srt/models/iquest_q1.py index 8bc8edff6f..5affe7ee57 100644 --- a/python/sglang/srt/models/iquest_q1.py +++ b/python/sglang/srt/models/iquest_q1.py @@ -310,8 +310,7 @@ class IQuestQ1Attention(nn.Module): self.total_num_kv_heads, bias=False, quant_config=quant_config, - tp_rank=attn_tp_rank, - tp_size=attn_tp_size, + parallel_group="attn_tp", prefix=add_prefix("qkv_proj", prefix), ) # The decoder boundary reduces partial attention outputs. Only a @@ -327,8 +326,7 @@ class IQuestQ1Attention(nn.Module): config.hidden_size, bias=False, quant_config=quant_config, - tp_rank=attn_tp_rank, - tp_size=attn_tp_size, + parallel_group="attn_tp", reduce_results=reduce_results, prefix=add_prefix("o_proj", prefix), ) diff --git a/python/sglang/srt/models/laguna.py b/python/sglang/srt/models/laguna.py index 450e44a680..99c3059aeb 100644 --- a/python/sglang/srt/models/laguna.py +++ b/python/sglang/srt/models/laguna.py @@ -32,6 +32,7 @@ from sglang.srt.layers.layer_boundary.residual import batch as residual_batch from sglang.srt.layers.layernorm import RMSNorm from sglang.srt.layers.linear import ( ColumnParallelLinear, + LinearParallelGroup, MergedColumnParallelLinear, QKVParallelLinear, RowParallelLinear, @@ -68,8 +69,8 @@ class LagunaMLP(nn.Module): quant_config: Optional[QuantizationConfig] = None, reduce_results: bool = True, prefix: str = "", - tp_rank: Optional[int] = None, - tp_size: Optional[int] = None, + *, + parallel_group: LinearParallelGroup = "tp", ) -> None: super().__init__() if hidden_act != "silu": @@ -82,8 +83,7 @@ class LagunaMLP(nn.Module): bias=False, quant_config=quant_config, prefix=add_prefix("gate_up_proj", prefix), - tp_rank=tp_rank, - tp_size=tp_size, + parallel_group=parallel_group, ) self.down_proj = RowParallelLinear( intermediate_size, @@ -92,8 +92,7 @@ class LagunaMLP(nn.Module): quant_config=quant_config, reduce_results=reduce_results, prefix=add_prefix("down_proj", prefix), - tp_rank=tp_rank, - tp_size=tp_size, + parallel_group=parallel_group, ) self.act_fn = SiluAndMul() @@ -188,7 +187,7 @@ class LagunaMoE(nn.Module): quant_config=quant_config, reduce_results=False, prefix=add_prefix("shared_expert", prefix), - **(dict(tp_rank=0, tp_size=1) if self._shared_expert_tp1 else {}), + **(dict(parallel_group="replicated") if self._shared_expert_tp1 else {}), ) def get_moe_weights(self): @@ -256,7 +255,6 @@ class LagunaAttention(nn.Module): self.gating = gating != "disabled" self.gate_per_head = gating == "per-head" - attn_tp_rank = get_parallel().attn_tp_rank attn_tp_size = get_parallel().attn_tp_size self.total_num_heads = num_heads @@ -279,8 +277,7 @@ class LagunaAttention(nn.Module): self.total_num_kv_heads, bias=attention_bias, quant_config=quant_config, - tp_rank=attn_tp_rank, - tp_size=attn_tp_size, + parallel_group="attn_tp", prefix=add_prefix("qkv_proj", prefix), ) self.o_proj = RowParallelLinear( @@ -288,8 +285,7 @@ class LagunaAttention(nn.Module): hidden_size, bias=attention_bias, quant_config=quant_config, - tp_rank=attn_tp_rank, - tp_size=attn_tp_size, + parallel_group="attn_tp", reduce_results=False, prefix=add_prefix("o_proj", prefix), ) @@ -306,8 +302,7 @@ class LagunaAttention(nn.Module): bias=False, gather_output=False, quant_config=quant_config, - tp_rank=attn_tp_rank, - tp_size=attn_tp_size, + parallel_group="attn_tp", prefix=add_prefix("g_proj", prefix), ) else: diff --git a/python/sglang/srt/models/llada2.py b/python/sglang/srt/models/llada2.py index 00fdd47dce..4fe4565764 100644 --- a/python/sglang/srt/models/llada2.py +++ b/python/sglang/srt/models/llada2.py @@ -427,7 +427,6 @@ class LLaDA2MoeAttention(nn.Module): self.hidden_size = config.hidden_size self.total_num_heads = config.num_attention_heads self.total_kv_heads = config.num_key_value_heads - attn_tp_rank = get_parallel().attn_tp_rank attn_tp_size = get_parallel().attn_tp_size assert self.total_num_heads % attn_tp_size == 0 @@ -460,8 +459,7 @@ class LLaDA2MoeAttention(nn.Module): bias=(config.use_bias or config.use_qkv_bias), quant_config=quant_config, prefix=add_prefix("query_key_value", prefix), - tp_rank=attn_tp_rank, - tp_size=attn_tp_size, + parallel_group="attn_tp", ) if self.use_qk_norm: @@ -475,8 +473,7 @@ class LLaDA2MoeAttention(nn.Module): quant_config=quant_config, reduce_results=reduce_results, prefix=add_prefix("dense", prefix), - tp_rank=attn_tp_rank, - tp_size=attn_tp_size, + parallel_group="attn_tp", ) if hasattr(config, "partial_rotary_factor"): diff --git a/python/sglang/srt/models/llama4.py b/python/sglang/srt/models/llama4.py index 7320b7e19c..0d14cebc14 100644 --- a/python/sglang/srt/models/llama4.py +++ b/python/sglang/srt/models/llama4.py @@ -212,7 +212,6 @@ class Llama4Attention(nn.Module): self.use_rope = (layer_id + 1) % 4 != 0 self.use_qk_norm = config.use_qk_norm and self.use_rope - attn_tp_rank = get_parallel().attn_tp_rank attn_tp_size = get_parallel().attn_tp_size self.total_num_heads = num_heads @@ -264,8 +263,7 @@ class Llama4Attention(nn.Module): bias=bias, quant_config=qkv_quant_config, prefix=add_prefix("qkv_proj", prefix), - tp_rank=attn_tp_rank, - tp_size=attn_tp_size, + parallel_group="attn_tp", ) self.o_proj = RowParallelLinear( @@ -274,8 +272,7 @@ class Llama4Attention(nn.Module): bias=bias_o_proj, quant_config=o_quant_config, prefix=add_prefix("o_proj", prefix), - tp_rank=attn_tp_rank, - tp_size=attn_tp_size, + parallel_group="attn_tp", reduce_results=False, ) is_neox_style = True diff --git a/python/sglang/srt/models/mellum.py b/python/sglang/srt/models/mellum.py index 2a87134ab2..bb9e7677ee 100644 --- a/python/sglang/srt/models/mellum.py +++ b/python/sglang/srt/models/mellum.py @@ -164,7 +164,6 @@ class MellumAttention(Qwen3MoeAttention): self.hidden_size = hidden_size self.start_layer = start_layer - attn_tp_rank = get_parallel().attn_tp_rank attn_tp_size = get_parallel().attn_tp_size self.config = config @@ -194,8 +193,7 @@ class MellumAttention(Qwen3MoeAttention): self.total_num_kv_heads, bias=attention_bias, quant_config=quant_config, - tp_rank=attn_tp_rank, - tp_size=attn_tp_size, + parallel_group="attn_tp", prefix=add_prefix("qkv_proj", prefix), ) @@ -204,8 +202,7 @@ class MellumAttention(Qwen3MoeAttention): hidden_size, bias=attention_bias, quant_config=quant_config, - tp_rank=attn_tp_rank, - tp_size=attn_tp_size, + parallel_group="attn_tp", reduce_results=False, prefix=add_prefix("o_proj", prefix), ) diff --git a/python/sglang/srt/models/minimax_m2.py b/python/sglang/srt/models/minimax_m2.py index ff9d3d7b41..1b04676ae9 100644 --- a/python/sglang/srt/models/minimax_m2.py +++ b/python/sglang/srt/models/minimax_m2.py @@ -650,7 +650,6 @@ class MiniMaxM2Attention(nn.Module): self.hidden_size = config.hidden_size # Use attention TP rank/size for dp-attention support - attn_tp_rank = get_parallel().attn_tp_rank attn_tp_size = get_parallel().attn_tp_size # Get dimensions from config @@ -696,8 +695,7 @@ class MiniMaxM2Attention(nn.Module): self.total_num_kv_heads, bias=False, quant_config=quant_config, - tp_rank=attn_tp_rank, - tp_size=attn_tp_size, + parallel_group="attn_tp", prefix=add_prefix("qkv_proj", prefix), ) @@ -707,8 +705,7 @@ class MiniMaxM2Attention(nn.Module): bias=False, reduce_results=False, quant_config=quant_config, - tp_rank=attn_tp_rank, - tp_size=attn_tp_size, + parallel_group="attn_tp", prefix=add_prefix("o_proj", prefix), ) diff --git a/python/sglang/srt/models/moss_vl.py b/python/sglang/srt/models/moss_vl.py index 94dd9d75b4..1b4be5fb32 100644 --- a/python/sglang/srt/models/moss_vl.py +++ b/python/sglang/srt/models/moss_vl.py @@ -898,7 +898,6 @@ class MossVLSelfAttention(nn.Module): super().__init__() self.hidden_size = config.hidden_size self.total_num_heads = config.num_attention_heads - attn_tp_rank = get_parallel().attn_tp_rank attn_tp_size = get_parallel().attn_tp_size assert self.total_num_heads % attn_tp_size == 0 @@ -928,8 +927,7 @@ class MossVLSelfAttention(nn.Module): self.total_num_kv_heads, bias=config.attention_bias, quant_config=quant_config, - tp_rank=attn_tp_rank, - tp_size=attn_tp_size, + parallel_group="attn_tp", prefix=add_prefix("qkv_proj", prefix), ) self.o_proj = RowParallelLinear( @@ -937,8 +935,7 @@ class MossVLSelfAttention(nn.Module): config.hidden_size, bias=config.attention_bias, quant_config=quant_config, - tp_rank=attn_tp_rank, - tp_size=attn_tp_size, + parallel_group="attn_tp", reduce_results=False, prefix=add_prefix("o_proj", prefix), ) diff --git a/python/sglang/srt/models/nemotron_h.py b/python/sglang/srt/models/nemotron_h.py index 6525e6f7d3..3b6460bc55 100644 --- a/python/sglang/srt/models/nemotron_h.py +++ b/python/sglang/srt/models/nemotron_h.py @@ -38,6 +38,7 @@ from sglang.srt.layers.layer_boundary.residual import batch as residual_batch from sglang.srt.layers.layernorm import RMSNorm from sglang.srt.layers.linear import ( ColumnParallelLinear, + LinearParallelGroup, QKVParallelLinear, ReplicatedLinear, RowParallelLinear, @@ -105,9 +106,9 @@ class NemotronHMLP(nn.Module): quant_config: QuantizationConfig | None = None, bias: bool = False, reduce_results: bool = True, - tp_rank: int | None = None, - tp_size: int | None = None, prefix: str = "", + *, + parallel_group: LinearParallelGroup = "tp", ) -> None: super().__init__() @@ -116,8 +117,7 @@ class NemotronHMLP(nn.Module): output_size=intermediate_size, bias=bias, quant_config=quant_config, - tp_rank=tp_rank, - tp_size=tp_size, + parallel_group=parallel_group, prefix=f"{prefix}.up_proj", ) self.down_proj = RowParallelLinear( @@ -126,8 +126,7 @@ class NemotronHMLP(nn.Module): bias=bias, quant_config=quant_config, reduce_results=reduce_results, - tp_rank=tp_rank, - tp_size=tp_size, + parallel_group=parallel_group, prefix=f"{prefix}.down_proj", ) self.act_fn = ReLU2() @@ -232,7 +231,7 @@ class NemotronHMoE(nn.Module): quant_config=quant_config, reduce_results=False, **( - dict(tp_rank=0, tp_size=1) + dict(parallel_group="replicated") if get_moe_a2a_backend().is_deepep() or get_moe_a2a_backend().is_flashinfer() or get_moe_a2a_backend().is_flashinfer_megamoe() @@ -541,7 +540,6 @@ class NemotronHAttention(nn.Module): ) -> None: super().__init__() self.hidden_size = config.hidden_size - tp_rank = get_parallel().attn_tp_rank tp_size = get_parallel().attn_tp_size self.total_num_heads = config.num_attention_heads assert self.total_num_heads % tp_size == 0 @@ -571,8 +569,7 @@ class NemotronHAttention(nn.Module): self.total_num_kv_heads, bias=False, quant_config=quant_config, - tp_rank=tp_rank, - tp_size=tp_size, + parallel_group="attn_tp", prefix=f"{prefix}.qkv_proj", ) self.o_proj = RowParallelLinear( @@ -580,8 +577,7 @@ class NemotronHAttention(nn.Module): config.hidden_size, bias=False, quant_config=quant_config, - tp_rank=tp_rank, - tp_size=tp_size, + parallel_group="attn_tp", reduce_results=False, use_dp_attention_reduce=is_dp_attention_enabled(), prefix=f"{prefix}.o_proj", diff --git a/python/sglang/srt/models/phimoe.py b/python/sglang/srt/models/phimoe.py index b0ef96b7f7..0a4ec8c469 100644 --- a/python/sglang/srt/models/phimoe.py +++ b/python/sglang/srt/models/phimoe.py @@ -246,7 +246,6 @@ class PhiMoEAttention(nn.Module): super().__init__() self.hidden_size = hidden_size - attn_tp_rank = get_parallel().attn_tp_rank attn_tp_size = get_parallel().attn_tp_size self.total_num_heads = num_heads @@ -279,8 +278,7 @@ class PhiMoEAttention(nn.Module): self.total_num_kv_heads, bias=attention_bias, quant_config=quant_config, - tp_rank=attn_tp_rank, - tp_size=attn_tp_size, + parallel_group="attn_tp", prefix=add_prefix("qkv_proj", prefix), ) # TODO: this layer shards over attention TP but reduces over the full TP @@ -294,8 +292,7 @@ class PhiMoEAttention(nn.Module): hidden_size, bias=attention_bias, quant_config=quant_config, - tp_rank=attn_tp_rank, - tp_size=attn_tp_size, + parallel_group="attn_tp", prefix=add_prefix("o_proj", prefix), ) self.rotary_emb = get_rope( diff --git a/python/sglang/srt/models/qwen2_moe.py b/python/sglang/srt/models/qwen2_moe.py index dbc5250375..2bbbbbf6c4 100644 --- a/python/sglang/srt/models/qwen2_moe.py +++ b/python/sglang/srt/models/qwen2_moe.py @@ -56,6 +56,7 @@ from sglang.srt.layers.layer_boundary import ( from sglang.srt.layers.layer_boundary.residual import batch as residual_batch from sglang.srt.layers.layernorm import RMSNorm from sglang.srt.layers.linear import ( + LinearParallelGroup, MergedColumnParallelLinear, QKVParallelLinear, ReplicatedLinear, @@ -176,8 +177,8 @@ class Qwen2MoeMLP(nn.Module): quant_config: Optional[QuantizationConfig] = None, reduce_results: bool = True, prefix: str = "", - tp_rank: Optional[int] = None, - tp_size: Optional[int] = None, + *, + parallel_group: LinearParallelGroup = "tp", ) -> None: super().__init__() self.gate_up_proj = MergedColumnParallelLinear( @@ -186,8 +187,7 @@ class Qwen2MoeMLP(nn.Module): bias=False, quant_config=quant_config, prefix=add_prefix("gate_up_proj", prefix), - tp_rank=tp_rank, - tp_size=tp_size, + parallel_group=parallel_group, ) self.down_proj = RowParallelLinear( intermediate_size, @@ -196,8 +196,7 @@ class Qwen2MoeMLP(nn.Module): quant_config=quant_config, reduce_results=reduce_results, prefix=add_prefix("down_proj", prefix), - tp_rank=tp_rank, - tp_size=tp_size, + parallel_group=parallel_group, ) if hidden_act != "silu": raise ValueError( @@ -365,7 +364,7 @@ class Qwen2MoeSparseMoeBlock(nn.Module): reduce_results=False, prefix=add_prefix("shared_expert", prefix), **( - dict(tp_rank=0, tp_size=1) + dict(parallel_group="replicated") if ( get_moe_a2a_backend().is_deepep() or get_moe_a2a_backend().is_mori() @@ -911,7 +910,6 @@ class Qwen2MoeAttention(nn.Module): super().__init__() self.hidden_size = hidden_size - attn_tp_rank = get_parallel().attn_tp_rank attn_tp_size = get_parallel().attn_tp_size self.total_num_heads = num_heads @@ -941,8 +939,7 @@ class Qwen2MoeAttention(nn.Module): self.total_num_kv_heads, bias=qkv_bias, quant_config=quant_config, - tp_rank=attn_tp_rank, - tp_size=attn_tp_size, + parallel_group="attn_tp", prefix=add_prefix("qkv_proj", prefix), ) @@ -951,8 +948,7 @@ class Qwen2MoeAttention(nn.Module): hidden_size, bias=False, quant_config=quant_config, - tp_rank=attn_tp_rank, - tp_size=attn_tp_size, + parallel_group="attn_tp", reduce_results=False, prefix=add_prefix("o_proj", prefix), ) diff --git a/python/sglang/srt/models/qwen3.py b/python/sglang/srt/models/qwen3.py index 1d9752e25a..f793c90bb5 100644 --- a/python/sglang/srt/models/qwen3.py +++ b/python/sglang/srt/models/qwen3.py @@ -86,7 +86,6 @@ class Qwen3Attention(nn.Module): self.hidden_size = hidden_size self.start_layer = start_layer self.total_num_heads = num_heads - attn_tp_rank = get_parallel().attn_tp_rank attn_tp_size = get_parallel().attn_tp_size assert self.total_num_heads % attn_tp_size == 0 @@ -126,8 +125,7 @@ class Qwen3Attention(nn.Module): self.total_num_kv_heads, bias=attention_bias, quant_config=quant_config, - tp_rank=attn_tp_rank, - tp_size=attn_tp_size, + parallel_group="attn_tp", prefix=add_prefix("qkv_proj", prefix), ) self.o_proj = RowParallelLinear( @@ -135,8 +133,7 @@ class Qwen3Attention(nn.Module): hidden_size, bias=attention_bias, quant_config=quant_config, - tp_rank=attn_tp_rank, - tp_size=attn_tp_size, + parallel_group="attn_tp", reduce_results=False, prefix=add_prefix("o_proj", prefix), ) diff --git a/python/sglang/srt/models/qwen3_moe.py b/python/sglang/srt/models/qwen3_moe.py index 0f9aa9bb39..a5698e177a 100644 --- a/python/sglang/srt/models/qwen3_moe.py +++ b/python/sglang/srt/models/qwen3_moe.py @@ -512,7 +512,6 @@ class Qwen3MoeAttention(nn.Module): self.hidden_size = hidden_size self.start_layer = start_layer - attn_tp_rank = get_parallel().attn_tp_rank attn_tp_size = get_parallel().attn_tp_size self.config = config @@ -543,8 +542,7 @@ class Qwen3MoeAttention(nn.Module): self.total_num_kv_heads, bias=attention_bias, quant_config=quant_config, - tp_rank=attn_tp_rank, - tp_size=attn_tp_size, + parallel_group="attn_tp", prefix=add_prefix("qkv_proj", prefix), ) @@ -553,8 +551,7 @@ class Qwen3MoeAttention(nn.Module): hidden_size, bias=attention_bias, quant_config=quant_config, - tp_rank=attn_tp_rank, - tp_size=attn_tp_size, + parallel_group="attn_tp", reduce_results=False, prefix=add_prefix("o_proj", prefix), ) diff --git a/python/sglang/srt/models/qwen3_next.py b/python/sglang/srt/models/qwen3_next.py index 463858940e..f0739fa1a6 100644 --- a/python/sglang/srt/models/qwen3_next.py +++ b/python/sglang/srt/models/qwen3_next.py @@ -641,8 +641,7 @@ class Qwen3HybridAttentionDecoderLayer(nn.Module): and quant_config.get_name() != "modelopt_fp4" else None ), - tp_rank=self.attn_tp_rank, - tp_size=self.attn_tp_size, + parallel_group="attn_tp", prefix=add_prefix("qkv_proj", prefix), ) @@ -652,8 +651,7 @@ class Qwen3HybridAttentionDecoderLayer(nn.Module): bias=False, quant_config=quant_config, reduce_results=False, - tp_rank=self.attn_tp_rank, - tp_size=self.attn_tp_size, + parallel_group="attn_tp", prefix=add_prefix("o_proj", prefix), ) diff --git a/python/sglang/srt/models/sdar.py b/python/sglang/srt/models/sdar.py index d96282736d..87d2c5f8d3 100644 --- a/python/sglang/srt/models/sdar.py +++ b/python/sglang/srt/models/sdar.py @@ -99,7 +99,6 @@ class SDARAttention(nn.Module): self.layer_id = layer_id self.hidden_size = config.hidden_size self.total_num_heads = config.num_attention_heads - attn_tp_rank = get_parallel().attn_tp_rank attn_tp_size = get_parallel().attn_tp_size assert self.total_num_heads % attn_tp_size == 0 @@ -129,8 +128,7 @@ class SDARAttention(nn.Module): self.total_num_kv_heads, bias=getattr(config, "attention_bias", False), quant_config=quant_config, - tp_rank=attn_tp_rank, - tp_size=attn_tp_size, + parallel_group="attn_tp", prefix=add_prefix("qkv_proj", prefix), ) self.o_proj = RowParallelLinear( @@ -139,8 +137,7 @@ class SDARAttention(nn.Module): bias=getattr(config, "attention_bias", False), quant_config=quant_config, reduce_results=reduce_results, - tp_rank=attn_tp_rank, - tp_size=attn_tp_size, + parallel_group="attn_tp", prefix=add_prefix("o_proj", prefix), ) diff --git a/python/sglang/srt/models/spark2_5.py b/python/sglang/srt/models/spark2_5.py index f0a90aad8f..c945a3a596 100644 --- a/python/sglang/srt/models/spark2_5.py +++ b/python/sglang/srt/models/spark2_5.py @@ -105,7 +105,6 @@ class Spark2_5Attention(nn.Module): super().__init__() self.hidden_size = hidden_size self.total_num_heads = num_heads - attn_tp_rank = get_parallel().attn_tp_rank attn_tp_size = get_parallel().attn_tp_size assert self.total_num_heads % attn_tp_size == 0 @@ -139,8 +138,7 @@ class Spark2_5Attention(nn.Module): self.total_num_kv_heads, bias=False, quant_config=quant_config, - tp_rank=attn_tp_rank, - tp_size=attn_tp_size, + parallel_group="attn_tp", prefix=add_prefix("q_k_v_proj", prefix), ) if self.headwise_attn_output_gate: @@ -149,8 +147,7 @@ class Spark2_5Attention(nn.Module): self.total_num_heads, bias=False, quant_config=None, # g_proj keeps bf16. - tp_rank=attn_tp_rank, - tp_size=attn_tp_size, + parallel_group="attn_tp", prefix=add_prefix("g_proj", prefix), ) @@ -159,8 +156,7 @@ class Spark2_5Attention(nn.Module): hidden_size, bias=False, quant_config=quant_config, - tp_rank=attn_tp_rank, - tp_size=attn_tp_size, + parallel_group="attn_tp", reduce_results=False, prefix=add_prefix("out_proj", prefix), ) diff --git a/python/sglang/srt/models/step3p5.py b/python/sglang/srt/models/step3p5.py index e5523a920f..43a95b772b 100644 --- a/python/sglang/srt/models/step3p5.py +++ b/python/sglang/srt/models/step3p5.py @@ -19,6 +19,7 @@ from sglang.srt.layers.layer_boundary.residual import batch as residual_batch from sglang.srt.layers.layernorm import GemmaRMSNorm from sglang.srt.layers.linear import ( ColumnParallelLinear, + LinearParallelGroup, MergedColumnParallelLinear, QKVParallelLinear, ReplicatedLinear, @@ -67,6 +68,8 @@ class Step3p5MLP(nn.Module): quant_config: Optional[QuantizationConfig] = None, prefix: str = "", reduce_results: bool = True, + *, + parallel_group: LinearParallelGroup = "tp", ) -> None: super().__init__() self.hidden_size = hidden_size @@ -76,6 +79,7 @@ class Step3p5MLP(nn.Module): [intermediate_size] * 2, bias=False, quant_config=quant_config, + parallel_group=parallel_group, prefix=add_prefix("gate_up_proj", prefix), ) self.down_proj = RowParallelLinear( @@ -83,6 +87,7 @@ class Step3p5MLP(nn.Module): hidden_size, bias=False, quant_config=quant_config, + parallel_group=parallel_group, prefix=add_prefix("down_proj", prefix), reduce_results=reduce_results, ) @@ -287,7 +292,6 @@ class Step3p5Attention(nn.Module): super().__init__() self.hidden_size = hidden_size self.total_num_heads = num_heads - attn_tp_rank = get_parallel().attn_tp_rank attn_tp_size = get_parallel().attn_tp_size assert self.total_num_heads % attn_tp_size == 0 @@ -318,8 +322,7 @@ class Step3p5Attention(nn.Module): self.total_num_kv_heads, bias=False, quant_config=quant_config, - tp_rank=attn_tp_rank, - tp_size=attn_tp_size, + parallel_group="attn_tp", prefix=add_prefix("qkv_proj", prefix), ) self.o_proj = RowParallelLinear( @@ -327,8 +330,7 @@ class Step3p5Attention(nn.Module): hidden_size, bias=False, quant_config=quant_config, - tp_rank=attn_tp_rank, - tp_size=attn_tp_size, + parallel_group="attn_tp", reduce_results=False, prefix=add_prefix("o_proj", prefix), ) @@ -339,8 +341,7 @@ class Step3p5Attention(nn.Module): hidden_size, self.total_num_heads, bias=False, - tp_rank=attn_tp_rank, - tp_size=attn_tp_size, + parallel_group="attn_tp", prefix=add_prefix("g_proj", prefix), ) @@ -490,7 +491,7 @@ class Step3p5DecoderLayer(nn.Module): prefix=add_prefix("share_expert", prefix), reduce_results=False, **( - dict(tp_rank=0, tp_size=1) + dict(parallel_group="replicated") if get_moe_a2a_backend().is_deepep() or get_moe_a2a_backend().is_mooncake() or get_moe_a2a_backend().is_nixl() diff --git a/python/sglang/srt/models/xllm.py b/python/sglang/srt/models/xllm.py index a75200a92f..093400a550 100644 --- a/python/sglang/srt/models/xllm.py +++ b/python/sglang/srt/models/xllm.py @@ -1081,7 +1081,6 @@ class XllmAttention(nn.Module): super().__init__() self.hidden_size = hidden_size - attn_tp_rank = get_parallel().attn_tp_rank attn_tp_size = get_parallel().attn_tp_size self.total_num_heads = num_heads @@ -1107,8 +1106,7 @@ class XllmAttention(nn.Module): self.total_num_kv_heads, bias=qkv_bias, quant_config=quant_config, - tp_rank=attn_tp_rank, - tp_size=attn_tp_size, + parallel_group="attn_tp", prefix=add_prefix("qkv_proj", prefix), ) @@ -1117,8 +1115,7 @@ class XllmAttention(nn.Module): hidden_size, bias=qkv_bias, quant_config=quant_config, - tp_rank=attn_tp_rank, - tp_size=attn_tp_size, + parallel_group="attn_tp", reduce_results=False, prefix=add_prefix("o_proj", prefix), ) diff --git a/test/registered/unit/layers/test_linear_parallel_groups.py b/test/registered/unit/layers/test_linear_parallel_groups.py new file mode 100644 index 0000000000..6482697b04 --- /dev/null +++ b/test/registered/unit/layers/test_linear_parallel_groups.py @@ -0,0 +1,680 @@ +"""Linear partitions, reloads, and communication follow the selected group.""" + +import unittest +from contextlib import nullcontext +from types import SimpleNamespace +from unittest.mock import Mock, patch + +import torch +import torch.nn.functional as F + +from sglang.srt.layers.linear import ( + ColumnParallelLinear, + MergedColumnParallelLinear, + QKVParallelLinear, + RowParallelLinear, +) +from sglang.srt.layers.parameter import ModelWeightParameter +from sglang.srt.runtime_context import SpawnRanks, get_parallel, publish, reset_context +from sglang.srt.server_args import ServerArgs +from sglang.test.ci.ci_register import register_cuda_ci +from sglang.test.test_utils import CustomTestCase + +register_cuda_ci(est_time=10, stage="base-b", runner_config="1-gpu-small") + + +class TestLinearParallelGroups(CustomTestCase): + def setUp(self): + reset_context() + self.addCleanup(reset_context) + publish( + ServerArgs(model_path="dummy", device="cpu", tp_size=4, attn_dp_size=2), + role="test", + ranks=SpawnRanks(world_rank=3), + ) + self.x = torch.arange(16, dtype=torch.float32).reshape(2, 8) / 16 + self.weight = torch.arange(64, dtype=torch.float32).reshape(8, 8) / 64 + + def test_column_partitions_and_reload_use_the_construction_scope(self): + for group, rank, size in ( + ("tp", 3, 4), + ("attn_tp", 1, 2), + ("replicated", 0, 1), + ): + with self.subTest(group=group): + layer = ColumnParallelLinear(8, 8, bias=False, parallel_group=group) + shard = self.weight.chunk(size, dim=0)[rank] + layer.weight.weight_loader(layer.weight, self.weight) + torch.testing.assert_close(layer(self.x)[0], F.linear(self.x, shard)) + with get_parallel().override(tp_rank=0, attn_dp_rank=0, attn_tp_rank=0): + layer.weight.weight_loader(layer.weight, self.weight + 1) + torch.testing.assert_close(layer.weight, shard + 1) + + with get_parallel().override(tp_rank=0, attn_dp_rank=0, attn_tp_rank=0): + with get_parallel().override(tp_rank=1, attn_tp_rank=1): + layer = ColumnParallelLinear(8, 8, bias=False, parallel_group="attn_tp") + layer.weight.weight_loader(layer.weight, self.weight) + torch.testing.assert_close(layer.weight, self.weight[4:]) + + def test_packed_loaders_partition_each_logical_weight(self): + merged = MergedColumnParallelLinear( + 8, [8, 4], bias=False, parallel_group="attn_tp" + ) + smaller = self.weight[:4] + 2 + merged.weight.weight_loader(merged.weight, self.weight, 0) + merged.weight.weight_loader(merged.weight, smaller, 1) + expected = torch.cat((self.weight[4:], smaller[2:])) + torch.testing.assert_close(merged(self.x)[0], F.linear(self.x, expected)) + + for kv_heads in (1, 4): + with self.subTest(kv_heads=kv_heads): + qkv = QKVParallelLinear( + 8, 2, 4, kv_heads, bias=False, parallel_group="attn_tp" + ) + q, k, v = ( + self.weight, + self.weight[: kv_heads * 2] + 2, + self.weight[: kv_heads * 2] + 4, + ) + for name, weight in (("q", q), ("k", k), ("v", v)): + qkv.weight.weight_loader(qkv.weight, weight, name) + kv_slice = ( + slice(None) if kv_heads == 1 else slice(k.shape[0] // 2, None) + ) + expected = torch.cat((q[4:], k[kv_slice], v[kv_slice])) + torch.testing.assert_close(qkv(self.x)[0], F.linear(self.x, expected)) + with get_parallel().override(tp_rank=0, attn_dp_rank=0, attn_tp_rank=0): + for name, weight in (("q", q), ("k", k), ("v", v)): + qkv.weight.weight_loader(qkv.weight, weight + 1, name) + torch.testing.assert_close(qkv.weight, expected + 1) + + def test_model_qkv_reload_keeps_attention_shards_and_replicated_kv(self): + from sglang.srt.models.qwen2_moe import Qwen2MoeAttention + + for kv_heads in (1, 4): + for bias in (False, True): + with self.subTest(kv_heads=kv_heads, bias=bias): + attention = Qwen2MoeAttention( + hidden_size=8, + num_heads=4, + num_kv_heads=kv_heads, + max_position_embeddings=16, + qkv_bias=bias, + ) + qkv = attention.qkv_proj + weights = { + "q": self.weight, + "k": self.weight[: kv_heads * 2] + 2, + "v": self.weight[: kv_heads * 2] + 4, + } + biases = { + name: torch.arange(weight.shape[0], dtype=torch.float32) + for name, weight in weights.items() + } + # Reload under another replica's scope. This model was + # constructed on attention rank 1, so Q stays on that rank + # and the one-head K/V layout remains replicated. + with get_parallel().override( + tp_rank=0, attn_dp_rank=0, attn_tp_rank=0 + ): + for name, weight in weights.items(): + qkv.weight.weight_loader(qkv.weight, weight, name) + if bias: + qkv.bias.weight_loader(qkv.bias, biases[name], name) + shards = { + name: value.chunk(2)[1] + if name == "q" or kv_heads >= 2 + else value + for name, value in weights.items() + } + bias_shards = { + name: value.chunk(2)[1] + if name == "q" or kv_heads >= 2 + else value + for name, value in biases.items() + } + expected_weight = torch.cat(tuple(shards.values())) + expected_bias = ( + torch.cat(tuple(bias_shards.values())) if bias else None + ) + torch.testing.assert_close( + qkv(self.x)[0], + F.linear(self.x, expected_weight, expected_bias), + ) + self.assertEqual(attention.q_size, shards["q"].shape[0]) + self.assertEqual(attention.kv_size, shards["k"].shape[0]) + + def test_exaone_attention_output_keeps_tp_allocation_and_partial_results(self): + from sglang.srt.models.exaone_moe import ExaoneMoEAttention + + tp = SimpleNamespace(world_size=4, all_reduce=Mock()) + attn = SimpleNamespace(world_size=2, all_reduce=Mock()) + for bias in (False, True): + for symmetric in (False, True): + with self.subTest(bias=bias, symmetric=symmetric): + attention = ExaoneMoEAttention( + config=SimpleNamespace( + rms_norm_eps=1e-6, layer_types=["full_attention"] + ), + hidden_size=8, + num_heads=4, + num_kv_heads=1, + max_position_embeddings=16, + bias=bias, + ) + row = attention.o_proj + with get_parallel().override( + tp_rank=0, attn_dp_rank=0, attn_tp_rank=0 + ): + row.weight.weight_loader(row.weight, self.weight) + if bias: + row.bias.weight_loader(row.bias, torch.arange(8).float()) + self.assertEqual((row.tp_rank, row.tp_size), (1, 2)) + self.assertEqual(attention.q_size, row.input_size_per_partition) + self.assertFalse(row.reduce_results) + # The boundary reduces this partial result. Its allocation + # still uses TP and the DP-padding condition. + with ( + get_parallel().override(tp_group=tp, attn_tp_group=attn), + patch( + "sglang.srt.layers.linear.is_allocation_symmetric", + return_value=symmetric, + ), + patch( + "sglang.srt.layers.linear.use_symmetric_memory", + return_value=nullcontext(), + ) as allocation, + ): + torch.testing.assert_close( + row(self.x[:, 4:])[0], + F.linear(self.x[:, 4:], self.weight[:, 4:]), + ) + allocation.assert_called_once_with(tp, disabled=not symmetric) + tp.all_reduce.assert_not_called() + attn.all_reduce.assert_not_called() + + def test_attention_rows_keep_their_existing_execution_policy(self): + from sglang.srt.layers.dp_attention import initialize_dp_attention_flags + from sglang.srt.models.nemotron_h import NemotronHAttention + from sglang.srt.models.qwen2_moe import Qwen2MoeAttention + + initialize_dp_attention_flags( + ServerArgs(model_path="dummy", device="cpu", tp_size=4, attn_dp_size=2) + ) + tp = SimpleNamespace(world_size=4, all_reduce=Mock()) + attn = SimpleNamespace(world_size=2, all_reduce=Mock()) + for use_attention_allocation in (False, True): + with self.subTest(use_attention_allocation=use_attention_allocation): + if use_attention_allocation: + attention = NemotronHAttention( + config=SimpleNamespace( + hidden_size=8, + num_attention_heads=4, + num_key_value_heads=1, + head_dim=2, + sliding_window=None, + ), + layer_idx=0, + ) + else: + attention = Qwen2MoeAttention( + hidden_size=8, + num_heads=4, + num_kv_heads=1, + max_position_embeddings=16, + ) + row = attention.o_proj + with get_parallel().override(tp_rank=0, attn_dp_rank=0, attn_tp_rank=0): + row.weight.weight_loader(row.weight, self.weight) + self.assertEqual((row.tp_rank, row.tp_size), (1, 2)) + self.assertFalse(row.reduce_results) + self.assertEqual(row.use_dp_attention_reduce, use_attention_allocation) + with ( + get_parallel().override(tp_group=tp, attn_tp_group=attn), + patch( + "sglang.srt.layers.linear.is_allocation_symmetric", + return_value=False, + ), + patch( + "sglang.srt.layers.linear.use_symmetric_memory", + return_value=nullcontext(), + ) as allocation, + ): + torch.testing.assert_close( + row(self.x[:, 4:])[0], + F.linear(self.x[:, 4:], self.weight[:, 4:]), + ) + if use_attention_allocation: + allocation.assert_called_once_with(attn) + else: + allocation.assert_called_once_with(tp, disabled=True) + tp.all_reduce.assert_not_called() + attn.all_reduce.assert_not_called() + + def test_model_gate_projections_reload_on_their_attention_shards(self): + from sglang.srt.models.laguna import LagunaAttention + from sglang.srt.models.step3p5 import Step3p5Attention + + for model, mode in ( + ("laguna", "disabled"), + ("laguna", "per-head"), + ("laguna", "per-element"), + ("step3p5", False), + ("step3p5", True), + ): + for kv_heads in (1, 4): + with self.subTest(model=model, mode=mode, kv_heads=kv_heads): + options = dict( + hidden_size=8, + num_heads=4, + num_kv_heads=kv_heads, + head_dim=2, + layer_id=0, + rms_norm_eps=1e-6, + rope_theta=10000, + rope_scaling=None, + partial_rotary_factor=1.0, + max_position_embeddings=16, + ) + if model == "laguna": + attention = LagunaAttention( + **options, + attention_bias=True, + sliding_window_size=-1, + layer_type="full_attention", + gating=mode, + ) + else: + attention = Step3p5Attention( + **options, use_head_wise_attn_gate=mode + ) + qkv = attention.qkv_proj + weights = { + "q": self.weight, + "k": self.weight[: kv_heads * 2] + 2, + "v": self.weight[: kv_heads * 2] + 4, + } + with get_parallel().override( + tp_rank=0, attn_dp_rank=0, attn_tp_rank=0 + ): + for name, weight in weights.items(): + qkv.weight.weight_loader(qkv.weight, weight, name) + if qkv.bias is not None: + qkv.bias.weight_loader( + qkv.bias, torch.zeros(weight.shape[0]), name + ) + shards = [ + weight.chunk(2)[1] if name == "q" or kv_heads >= 2 else weight + for name, weight in weights.items() + ] + torch.testing.assert_close( + qkv(self.x)[0], F.linear(self.x, torch.cat(shards)) + ) + gate = getattr(attention, "g_proj", None) + if mode in (False, "disabled"): + self.assertIsNone(gate) + continue + gate_weight = ( + self.weight if mode == "per-element" else self.weight[:4] + ) + with get_parallel().override( + tp_rank=0, attn_dp_rank=0, attn_tp_rank=0 + ): + gate.weight.weight_loader(gate.weight, gate_weight) + torch.testing.assert_close( + gate(self.x)[0], F.linear(self.x, gate_weight.chunk(2)[1]) + ) + + def test_v2_loaders_keep_their_partition_during_reload(self): + column = ColumnParallelLinear(8, 8, bias=False, parallel_group="attn_tp") + row = RowParallelLinear(8, 8, bias=False, parallel_group="attn_tp") + merged = MergedColumnParallelLinear( + 8, [8, 4], bias=False, parallel_group="attn_tp" + ) + qkv = QKVParallelLinear(8, 2, 4, 1, bias=False, parallel_group="attn_tp") + for layer in (column, row, merged, qkv): + layer.weight = ModelWeightParameter( + data=torch.empty_like(layer.weight), + input_dim=1, + output_dim=0, + weight_loader=layer.weight_loader_v2, + ) + # Reload while a different rank is current. The stored rank still owns + # the latter half of Q and each MLP matrix, while K/V stay replicated. + with get_parallel().override(tp_rank=0, attn_dp_rank=0, attn_tp_rank=0): + column.weight.weight_loader(column.weight, self.weight) + row.weight.weight_loader(row.weight, self.weight) + merged.weight.weight_loader(merged.weight, self.weight, 0) + merged.weight.weight_loader(merged.weight, self.weight[:4] + 2, 1) + for name, weight in ( + ("q", self.weight), + ("k", self.weight[:2] + 2), + ("v", self.weight[:2] + 4), + ): + qkv.weight.weight_loader(qkv.weight, weight, name) + torch.testing.assert_close(column.weight, self.weight[4:]) + torch.testing.assert_close(row.weight, self.weight[:, 4:]) + torch.testing.assert_close( + merged.weight, torch.cat((self.weight[4:], self.weight[2:4] + 2)) + ) + torch.testing.assert_close( + qkv.weight, + torch.cat((self.weight[4:], self.weight[:2] + 2, self.weight[:2] + 4)), + ) + + def test_gather_uses_the_selected_group(self): + tp = SimpleNamespace(world_size=4, all_gather=Mock()) + attn = SimpleNamespace(world_size=2, all_gather=Mock()) + with get_parallel().override(tp_group=tp, attn_tp_group=attn): + for group, rank, size, selected, other in ( + ("tp", 3, 4, tp, attn), + ("attn_tp", 1, 2, attn, tp), + ): + with self.subTest(group=group): + selected.all_gather.reset_mock() + other.all_gather.reset_mock() + expected = F.linear(self.x, self.weight) + selected.all_gather.return_value = expected + layer = ColumnParallelLinear( + 8, 8, bias=False, gather_output=True, parallel_group=group + ) + layer.weight.weight_loader(layer.weight, self.weight) + torch.testing.assert_close(layer(self.x)[0], expected) + payload, dim = selected.all_gather.call_args.args + torch.testing.assert_close( + payload, expected.chunk(size, dim=-1)[rank] + ) + self.assertEqual(dim, -1) + other.all_gather.assert_not_called() + + def test_row_uses_attention_group_and_preserves_reduction_overrides(self): + bias = torch.arange(8, dtype=torch.float32) + # Rank 2 contributes the bias once; rank 3 supplies the other input shard. + peer = F.linear(self.x[:, :4], self.weight[:, :4], bias) + partial = F.linear(self.x[:, 4:], self.weight[:, 4:]) + attn = SimpleNamespace( + world_size=2, all_reduce=Mock(side_effect=lambda x: x + peer) + ) + tp = SimpleNamespace( + world_size=4, all_reduce=Mock(side_effect=AssertionError("wrong group")) + ) + with ( + get_parallel().override(tp_group=tp, attn_tp_group=attn), + patch( + "sglang.srt.layers.linear.use_symmetric_memory", + return_value=nullcontext(), + ) as allocator, + ): + layer = RowParallelLinear( + 8, + 8, + input_is_parallel=False, + parallel_group="attn_tp", + use_dp_attention_reduce=True, + ) + layer.weight.weight_loader(layer.weight, self.weight) + layer.bias.weight_loader(layer.bias, bias) + torch.testing.assert_close( + layer(self.x)[0], F.linear(self.x, self.weight, bias) + ) + torch.testing.assert_close(attn.all_reduce.call_args.args[0], partial) + allocator.assert_called_with(attn) + tp.all_reduce.assert_not_called() + attn.all_reduce.reset_mock() + torch.testing.assert_close(layer(self.x, skip_all_reduce=True)[0], partial) + attn.all_reduce.assert_not_called() + layer.reduce_results = False + torch.testing.assert_close(layer(self.x)[0], partial) + attn.all_reduce.assert_not_called() + with get_parallel().override(tp_rank=0, attn_dp_rank=0, attn_tp_rank=0): + layer.weight.weight_loader(layer.weight, self.weight + 1) + torch.testing.assert_close(layer.weight, self.weight[:, 4:] + 1) + + def test_row_partition_keeps_independent_reduction_and_allocation_policies(self): + tp = SimpleNamespace(world_size=4, all_reduce=Mock(side_effect=lambda x: x * 2)) + attn = SimpleNamespace( + world_size=2, all_reduce=Mock(side_effect=lambda x: x * 3) + ) + for group, rank, size in ( + ("tp", 3, 4), + ("attn_tp", 1, 2), + ("replicated", 0, 1), + ): + default = RowParallelLinear(8, 8, parallel_group=group) + self.assertFalse(default.use_dp_attention_reduce) + for attention_reduce in (False, True): + for reduce_results in (False, True): + for symmetric in (False, True): + with self.subTest( + group=group, + attention_reduce=attention_reduce, + reduce_results=reduce_results, + symmetric=symmetric, + ): + layer = RowParallelLinear( + 8, + 8, + bias=False, + input_is_parallel=False, + parallel_group=group, + use_dp_attention_reduce=attention_reduce, + reduce_results=reduce_results, + ) + with get_parallel().override( + tp_rank=0, attn_dp_rank=0, attn_tp_rank=0 + ): + layer.weight.weight_loader(layer.weight, self.weight) + shard = self.weight.chunk(size, dim=1)[rank] + partial = F.linear(self.x.chunk(size, dim=1)[rank], shard) + with ( + get_parallel().override( + tp_group=tp, attn_tp_group=attn + ), + patch( + "sglang.srt.layers.linear.is_allocation_symmetric", + return_value=symmetric, + ), + patch( + "sglang.srt.layers.linear.use_symmetric_memory", + return_value=nullcontext(), + ) as allocator, + ): + # Existing runtime writers may switch execution + # policy without changing the stored partition. + for policy in (attention_reduce, not attention_reduce): + layer.use_dp_attention_reduce = policy + tp.all_reduce.reset_mock() + attn.all_reduce.reset_mock() + allocator.reset_mock() + selected, other = ( + (attn, tp) if policy else (tp, attn) + ) + reduces = reduce_results and size > 1 + expected = ( + partial * (3 if policy else 2) + if reduces + else partial + ) + torch.testing.assert_close( + layer(self.x)[0], expected + ) + if reduces: + selected.all_reduce.assert_called_once() + torch.testing.assert_close( + selected.all_reduce.call_args.args[0], + partial, + ) + else: + selected.all_reduce.assert_not_called() + other.all_reduce.assert_not_called() + if policy: + allocator.assert_called_once_with(attn) + else: + allocator.assert_called_once_with( + tp, disabled=not symmetric + ) + self.assertEqual( + (layer.tp_rank, layer.tp_size), (rank, size) + ) + + def test_replicated_model_mlps_keep_weights_and_tp_output_allocation(self): + from sglang.srt.models.exaone_moe import ExaoneMoEMLP + from sglang.srt.models.laguna import LagunaMLP + from sglang.srt.models.nemotron_h import NemotronHMLP + from sglang.srt.models.qwen2_moe import Qwen2MoeMLP + from sglang.srt.models.step3p5 import Step3p5MLP + + tp = SimpleNamespace( + world_size=4, all_reduce=Mock(side_effect=lambda tensor: tensor * 2) + ) + for cls in ( + ExaoneMoEMLP, + LagunaMLP, + NemotronHMLP, + Qwen2MoeMLP, + Step3p5MLP, + ): + groups = [(None, 3, 4), ("tp", 3, 4), ("replicated", 0, 1)] + for group, rank, size in groups: + with self.subTest(model=cls.__name__, group=group): + tp.all_reduce.reset_mock() + reduces = group is None + options = dict( + intermediate_size=8, + reduce_results=reduces, + parallel_group=group, + ) + if cls is NemotronHMLP: + options["config"] = SimpleNamespace(hidden_size=8) + else: + options["hidden_size"] = 8 + if cls is not Step3p5MLP: + options["hidden_act"] = "silu" + if group is None: + options.pop("parallel_group") + mlp = cls(**options) + # Use native activation for CPU weight and layout checks. + mlp.act_fn._forward_method = mlp.act_fn.forward_native + up = getattr(mlp, "gate_up_proj", getattr(mlp, "up_proj", None)) + down = mlp.down_proj + with get_parallel().override( + tp_rank=0, attn_dp_rank=0, attn_tp_rank=0 + ): + if cls is NemotronHMLP: + up.weight.weight_loader(up.weight, self.weight) + else: + up.weight.weight_loader(up.weight, self.weight, 0) + up.weight.weight_loader(up.weight, self.weight + 1, 1) + down.weight.weight_loader(down.weight, self.weight + 2) + local_weight = self.weight.chunk(size)[rank] + projected = F.linear(self.x, local_weight) + if cls is NemotronHMLP: + activated = projected.relu().square() + else: + activated = F.silu(projected) * F.linear( + self.x, (self.weight + 1).chunk(size)[rank] + ) + expected = F.linear( + activated, (self.weight + 2).chunk(size, dim=1)[rank] + ) + if reduces: + expected = expected * 2 + with ( + get_parallel().override(tp_group=tp), + patch( + "sglang.srt.layers.linear.is_allocation_symmetric", + return_value=True, + ), + patch( + "sglang.srt.layers.linear.use_symmetric_memory", + return_value=nullcontext(), + ) as allocator, + ): + torch.testing.assert_close(mlp(self.x), expected) + allocator.assert_called_once_with(tp, disabled=False) + if reduces: + tp.all_reduce.assert_called_once() + else: + tp.all_reduce.assert_not_called() + self.assertEqual((up.tp_rank, up.tp_size), (rank, size)) + self.assertEqual((down.tp_rank, down.tp_size), (rank, size)) + + def test_step_shared_expert_constructs_a_replicated_mlp(self): + from sglang.srt.models import step3p5 + + config = SimpleNamespace( + hidden_size=8, + layer_types=["full_attention"], + yarn_only_types=[], + rope_theta=[10000], + max_position_embeddings=16, + head_dim=2, + moe_layers_enum="0", + num_attention_heads=4, + num_attention_groups=1, + num_hidden_layers=1, + swiglu_limits_shared=None, + partial_rotary_factors=[1.0], + rms_norm_eps=1e-6, + use_head_wise_attn_gate=False, + share_expert_dim=8, + ) + backend = Mock() + backend.is_deepep.return_value = True + with ( + patch.object(step3p5, "get_moe_a2a_backend", return_value=backend), + patch.object(step3p5, "Step3p5MoEMLP", return_value=torch.nn.Identity()), + patch.object(step3p5, "append_stages", return_value=(Mock(), Mock())), + ): + layer = step3p5.Step3p5DecoderLayer(config) + self.assertEqual(layer.share_expert.gate_up_proj.tp_size, 1) + self.assertEqual(layer.share_expert.down_proj.tp_size, 1) + self.assertFalse(layer.share_expert.down_proj.reduce_results) + + def test_replicated_layers_need_no_group_handle(self): + with get_parallel().override(tp_group=None, attn_tp_group=None): + for cls, kwargs in ( + (ColumnParallelLinear, dict(gather_output=True)), + (RowParallelLinear, dict(input_is_parallel=False)), + ): + with self.subTest(layer=cls.__name__): + layer = cls(8, 8, bias=False, parallel_group="replicated", **kwargs) + layer.weight.weight_loader(layer.weight, self.weight) + torch.testing.assert_close( + layer(self.x)[0], F.linear(self.x, self.weight) + ) + + def test_legacy_arguments_remain_available_and_mixing_is_rejected(self): + constructors = ( + lambda **kwargs: ColumnParallelLinear(8, 8, bias=False, **kwargs), + lambda **kwargs: MergedColumnParallelLinear( + 8, [8, 4], bias=False, **kwargs + ), + lambda **kwargs: QKVParallelLinear(8, 2, 4, bias=False, **kwargs), + lambda **kwargs: RowParallelLinear(8, 8, bias=False, **kwargs), + ) + for build in constructors: + self.assertEqual((build().tp_rank, build().tp_size), (3, 4)) + explicit = build(tp_rank=0, tp_size=1) + self.assertEqual((explicit.tp_rank, explicit.tp_size), (0, 1)) + for kwargs in (dict(tp_rank=0), dict(tp_size=1)): + with self.assertRaisesRegex(ValueError, "cannot be combined"): + build(parallel_group="replicated", **kwargs) + with self.assertRaisesRegex(ValueError, "Unknown linear parallel_group"): + build(parallel_group="unknown") + for old_reduce in (False, True): + layer = RowParallelLinear( + 8, 8, parallel_group="attn_tp", use_dp_attention_reduce=old_reduce + ) + self.assertEqual((layer.tp_rank, layer.tp_size), (1, 2)) + self.assertEqual(layer.use_dp_attention_reduce, old_reduce) + legacy = RowParallelLinear( + 8, 8, tp_rank=1, tp_size=2, use_dp_attention_reduce=True + ) + self.assertTrue(legacy.use_dp_attention_reduce) + legacy.use_dp_attention_reduce = False + self.assertFalse(legacy.use_dp_attention_reduce) + + +if __name__ == "__main__": + unittest.main()