diff --git a/python/sglang/srt/layers/attention/mamba/mamba.py b/python/sglang/srt/layers/attention/mamba/mamba.py index fa17936ff0..e47ee73fdf 100644 --- a/python/sglang/srt/layers/attention/mamba/mamba.py +++ b/python/sglang/srt/layers/attention/mamba/mamba.py @@ -231,9 +231,11 @@ class MambaMixer2(torch.nn.Module): # - NOTE: currently for the world size DOES NOT divide groups # case, we only support the case when n_groups == 1 if is_dp_attention_enabled(): + parallel_group = "attn_tp" self.tp_size = get_parallel().attn_tp_size self.tp_rank = get_parallel().attn_tp_rank else: + parallel_group = "tp" self.tp_size = get_parallel().tp_size self.tp_rank = get_parallel().tp_rank @@ -284,8 +286,7 @@ class MambaMixer2(torch.nn.Module): bias=use_conv_bias, quant_config=None, prefix=f"{prefix}.conv1d", - tp_rank=self.tp_rank, - tp_size=self.tp_size, + parallel_group=parallel_group, ) self.in_proj = MergedColumnParallelLinear( @@ -300,8 +301,7 @@ class MambaMixer2(torch.nn.Module): bias=use_bias, quant_config=quant_config, prefix=f"{prefix}.in_proj", - tp_rank=self.tp_rank, - tp_size=self.tp_size, + parallel_group=parallel_group, ) else: # This is the n_groups == 1 case, @@ -313,8 +313,7 @@ class MambaMixer2(torch.nn.Module): bias=use_conv_bias, quant_config=None, prefix=f"{prefix}.conv1d", - tp_rank=self.tp_rank, - tp_size=self.tp_size, + parallel_group=parallel_group, ) self.in_proj = ColumnParallelLinear( @@ -323,8 +322,7 @@ class MambaMixer2(torch.nn.Module): bias=use_bias, quant_config=quant_config, prefix=f"{prefix}.in_proj", - tp_rank=self.tp_rank, - tp_size=self.tp_size, + parallel_group=parallel_group, ) # - because in_proj is a concatenation of 3 weights, we @@ -431,8 +429,7 @@ class MambaMixer2(torch.nn.Module): bias=use_bias, input_is_parallel=True, quant_config=quant_config, - tp_rank=self.tp_rank, - tp_size=self.tp_size, + parallel_group=parallel_group, reduce_results=reduce_results, use_dp_attention_reduce=reduce_results and is_dp_attention_enabled(), prefix=f"{prefix}.out_proj", diff --git a/python/sglang/srt/layers/linear.py b/python/sglang/srt/layers/linear.py index a112723707..102fcfb276 100644 --- a/python/sglang/srt/layers/linear.py +++ b/python/sglang/srt/layers/linear.py @@ -8,6 +8,7 @@ import itertools import logging from typing import TYPE_CHECKING, Dict, List, Literal, Optional, Tuple +import msgspec import torch from torch import nn from torch.nn.parameter import Parameter, UninitializedParameter @@ -55,15 +56,44 @@ _disable_hip_linear_quant = _is_hip and get_bool_env_var( logger = logging.getLogger(__name__) -LinearParallelGroup = Literal["tp", "attn_tp", "replicated"] + +class ReplicatedParallelGroup(msgspec.Struct, frozen=True): + """Share weight partitions across consecutive ranks; collectives stay separate.""" + + group: Literal["tp", "attn_tp"] + replica_size: int + + def __post_init__(self): + if self.group not in ("tp", "attn_tp"): + raise ValueError(f"Unknown replicated base group: {self.group!r}") + if type(self.replica_size) is not int or self.replica_size < 1: + raise ValueError("replica_size must be a positive integer") + + +LinearParallelGroup = ( + Literal["tp", "attn_tp", "replicated", "shared_experts_tp"] + | ReplicatedParallelGroup +) def resolve_linear_parallel_group( parallel_group: LinearParallelGroup, ) -> Tuple[int, int]: """Freeze a group's weight partition in the current construction scope.""" + if isinstance(parallel_group, ReplicatedParallelGroup): + rank, size = resolve_linear_parallel_group(parallel_group.group) + return rank // parallel_group.replica_size, divide( + size, parallel_group.replica_size + ) if parallel_group == "replicated": return 0, 1 + if parallel_group == "shared_experts_tp": + group = get_parallel().shared_experts_tp_group + if group is None: + raise ValueError( + "The shared-expert TP group must exist before construction" + ) + return group.rank_in_group, group.world_size if parallel_group not in ("tp", "attn_tp"): raise ValueError(f"Unknown linear parallel_group: {parallel_group!r}") parallel = get_parallel() @@ -1005,6 +1035,9 @@ class QKVParallelLinear(ColumnParallelLinear): be replicated while the query heads are partitioned. Args: + parallel_group: Group selecting the query weight partition. + kv_parallel_group: Optional independent key/value weight partition; + defaults to the query partition, including replicated head layouts. hidden_size: input hidden state size of the transformer. head_size: size of each attention head. total_num_heads: total number of attention query heads. @@ -1040,6 +1073,7 @@ class QKVParallelLinear(ColumnParallelLinear): kv_tp_size: Optional[int] = None, *, parallel_group: Optional[LinearParallelGroup] = None, + kv_parallel_group: Optional[LinearParallelGroup] = None, ): self.with_bias = bias self.hidden_size = hidden_size @@ -1052,10 +1086,15 @@ class QKVParallelLinear(ColumnParallelLinear): # Divide the weight matrix along the last dimension. tp_rank, tp_size = _resolve_linear_partition(parallel_group, tp_rank, tp_size) self.tp_rank, self.tp_size = tp_rank, tp_size - if kv_tp_rank is None: - kv_tp_rank = tp_rank - if kv_tp_size is None: - kv_tp_size = tp_size + if kv_parallel_group is not None: + kv_tp_rank, kv_tp_size = _resolve_linear_partition( + kv_parallel_group, kv_tp_rank, kv_tp_size + ) + else: + if kv_tp_rank is None: + kv_tp_rank = tp_rank + if kv_tp_size is None: + kv_tp_size = tp_size self.kv_tp_rank, self.kv_tp_size = kv_tp_rank, kv_tp_size self.num_heads = divide(self.total_num_heads, tp_size) if kv_tp_size >= self.total_num_kv_heads: @@ -1766,10 +1805,8 @@ class MergedColumnParallelRepeatedLinear(LinearBase): skip_bias_add: If true, skip adding bias but instead return it. params_dtype: Data type for the parameters. quant_config: Quantization configure. - tp_rank: Rank to shard the column-parallel part on. Defaults to the - global TP rank; pass the attention-TP rank to shard on attn-TP - instead (see KimiDeltaAttention's shard_on_attn_tp). - tp_size: World size matching ``tp_rank``. Defaults to global TP size. + parallel_group: Select TP, attention TP, or an unsharded partition for + the column-parallel part. Repeated weights remain unsharded. """ def __init__( @@ -1783,6 +1820,7 @@ class MergedColumnParallelRepeatedLinear(LinearBase): prefix: str = "", tp_rank: Optional[int] = None, tp_size: Optional[int] = None, + parallel_group: Optional[LinearParallelGroup] = None, ): output_size = sum(column_output_sizes) + sum(repeated_output_sizes) super().__init__( @@ -1794,11 +1832,9 @@ class MergedColumnParallelRepeatedLinear(LinearBase): prefix=prefix, ) self.num_column_parallel = len(column_output_sizes) - if tp_rank is None: - tp_rank = get_parallel().tp_rank - if tp_size is None: - tp_size = get_parallel().tp_size - self.tp_rank, self.tp_size = tp_rank, tp_size + self.tp_rank, self.tp_size = _resolve_linear_partition( + parallel_group, tp_rank, tp_size + ) self.output_partition_sizes = [ divide(x, self.tp_size) for x in column_output_sizes @@ -1843,10 +1879,8 @@ class ColumnParallelBatchedLinear(nn.Module): input_size: input dimension of the linear layer. output_size: output dimension of the linear layer. dtype: Data type for the parameters. - tp_rank: Rank to shard the output dimension on. Defaults to the global - TP rank; pass the attention-TP rank to shard on attn-TP instead - (see KimiDeltaAttention's shard_on_attn_tp). - tp_size: World size matching ``tp_rank``. Defaults to global TP size. + parallel_group: Select TP, attention TP, or an unsharded partition for + the output dimension. The batch dimension remains unsharded. """ def __init__( @@ -1857,13 +1891,12 @@ class ColumnParallelBatchedLinear(nn.Module): dtype: torch.dtype, tp_rank: Optional[int] = None, tp_size: Optional[int] = None, + parallel_group: Optional[LinearParallelGroup] = None, ): super().__init__() - if tp_rank is None: - tp_rank = get_parallel().tp_rank - if tp_size is None: - tp_size = get_parallel().tp_size - self.tp_rank, self.tp_size = tp_rank, tp_size + self.tp_rank, self.tp_size = _resolve_linear_partition( + parallel_group, tp_rank, tp_size + ) self.weight = nn.Parameter( torch.empty(batch, output_size // self.tp_size, input_size, dtype=dtype), requires_grad=False, diff --git a/python/sglang/srt/layers/mova.py b/python/sglang/srt/layers/mova.py index dc5ddb5c57..888a0bc69d 100644 --- a/python/sglang/srt/layers/mova.py +++ b/python/sglang/srt/layers/mova.py @@ -14,7 +14,7 @@ from __future__ import annotations -from typing import Optional, Tuple +from typing import TYPE_CHECKING, Optional, Tuple import torch import torch.nn.functional as F @@ -23,6 +23,9 @@ from torch import nn from sglang.srt.utils import set_weight_attrs from sglang.srt.utils.custom_op import register_custom_op +if TYPE_CHECKING: + from sglang.srt.layers.linear import LinearParallelGroup + _ROUTED_LINEAR_CHUNK_SIZE = 64 * 1024 @@ -245,10 +248,12 @@ class RoutedValueExperts(nn.Module): input_size: int, output_size: int, *, - tp_rank: int, - tp_size: int, + parallel_group: LinearParallelGroup = "attn_tp", ) -> None: + from sglang.srt.layers.linear import resolve_linear_parallel_group + super().__init__() + tp_rank, tp_size = resolve_linear_parallel_group(parallel_group) if output_size % tp_size: raise ValueError( f"MoVA value width {output_size} is not divisible by TP={tp_size}" diff --git a/python/sglang/srt/models/bailing_moe_v3.py b/python/sglang/srt/models/bailing_moe_v3.py index 6df9f75eca..474654475d 100644 --- a/python/sglang/srt/models/bailing_moe_v3.py +++ b/python/sglang/srt/models/bailing_moe_v3.py @@ -162,8 +162,6 @@ class DsV3MLA(DeepseekV2AttentionMLA): alt_stream, skip_rope, ) - attn_tp_rank = get_parallel().attn_tp_rank - attn_tp_size = get_parallel().attn_tp_size self.gated_attention_proj_granularity_type = getattr( config, "gated_attention_proj_granularity_type", None ) @@ -175,8 +173,7 @@ class DsV3MLA(DeepseekV2AttentionMLA): bias=False, prefix=f"{prefix}.output_gate", quant_config=None, - tp_rank=attn_tp_rank, - tp_size=attn_tp_size, + parallel_group="attn_tp", ) elif self.gated_attention_proj_granularity_type == "element_wise": self.g_proj = ColumnParallelLinear( @@ -184,8 +181,7 @@ class DsV3MLA(DeepseekV2AttentionMLA): self.num_heads * self.v_head_dim, bias=False, prefix=f"{prefix}.output_gate", - tp_rank=attn_tp_rank, - tp_size=attn_tp_size, + parallel_group="attn_tp", ) else: self.g_proj = None diff --git a/python/sglang/srt/models/exaone4.py b/python/sglang/srt/models/exaone4.py index c8c583b212..e0a1ed5935 100644 --- a/python/sglang/srt/models/exaone4.py +++ b/python/sglang/srt/models/exaone4.py @@ -112,9 +112,6 @@ class Exaone4Attention(nn.Module): self.hidden_size = hidden_size tp_size = get_parallel().tp_size - attn_tp_rank = get_parallel().attn_tp_rank - attn_tp_size = get_parallel().attn_tp_size - self.total_num_heads = num_heads assert self.total_num_heads % tp_size == 0 self.num_heads = self.total_num_heads // tp_size @@ -144,8 +141,7 @@ class Exaone4Attention(nn.Module): bias=bias, quant_config=quant_config, prefix=add_prefix("qkv_proj", prefix), - tp_rank=attn_tp_rank, - tp_size=attn_tp_size, + parallel_group="attn_tp", ) self.o_proj = RowParallelLinear( @@ -155,8 +151,7 @@ class Exaone4Attention(nn.Module): quant_config=quant_config, reduce_results=False, prefix=add_prefix("o_proj", prefix), - tp_rank=attn_tp_rank, - tp_size=attn_tp_size, + parallel_group="attn_tp", ) is_neox_style = True diff --git a/python/sglang/srt/models/gigachat35.py b/python/sglang/srt/models/gigachat35.py index 7b5645592b..13dbf0612d 100644 --- a/python/sglang/srt/models/gigachat35.py +++ b/python/sglang/srt/models/gigachat35.py @@ -296,8 +296,7 @@ class GigaChat35AttentionMLA(deepseek_v2.DeepseekV2AttentionMLA): bias=False, quant_config=quant_config, prefix=add_prefix("attn_gate", prefix), - tp_rank=get_parallel().attn_tp_rank, - tp_size=get_parallel().attn_tp_size, + parallel_group="attn_tp", ) self.o_proj.register_forward_pre_hook(self._o_proj_gate_hook) diff --git a/python/sglang/srt/models/glm5_next.py b/python/sglang/srt/models/glm5_next.py index 9bdbda858b..4da28bdf3d 100644 --- a/python/sglang/srt/models/glm5_next.py +++ b/python/sglang/srt/models/glm5_next.py @@ -396,7 +396,6 @@ class Glm5NextLinearAttention(nn.Module): ) -> None: super().__init__() head_shard_size = get_parallel().attn_tp_size - head_shard_rank = get_parallel().attn_tp_rank self.hidden_size = hidden_size self.config = config @@ -435,8 +434,7 @@ class Glm5NextLinearAttention(nn.Module): self.fg_sizes, quant_config=None, prefix=f"{prefix}.fused_qkvbfg_a_proj", - tp_rank=head_shard_rank, - tp_size=head_shard_size, + parallel_group="attn_tp", ) self.split_sizes = [ 3 * projection_size // head_shard_size, @@ -448,8 +446,7 @@ class Glm5NextLinearAttention(nn.Module): self.head_dim, projection_size, dtype=self.fused_qkvbfg_a_proj.params_dtype, - tp_rank=head_shard_rank, - tp_size=head_shard_size, + parallel_group="attn_tp", ) else: self.qkv_proj = QKVParallelLinear( @@ -459,8 +456,7 @@ class Glm5NextLinearAttention(nn.Module): self.num_k_heads, bias=False, quant_config=quant_config, - tp_rank=head_shard_rank, - tp_size=head_shard_size, + parallel_group="attn_tp", prefix=f"{prefix}.qkv_proj", ) @@ -471,8 +467,7 @@ class Glm5NextLinearAttention(nn.Module): [self.head_dim, self.head_dim], quant_config=None, prefix=f"{prefix}.fused_bfg_a_proj", - tp_rank=head_shard_rank, - tp_size=head_shard_size, + parallel_group="attn_tp", ) self.bfg_split_sizes = [self.local_num_heads, 2 * self.head_dim] self.fused_fg_b_proj = ColumnParallelBatchedLinear( @@ -480,8 +475,7 @@ class Glm5NextLinearAttention(nn.Module): self.head_dim, projection_size, dtype=self.fused_bfg_a_proj.params_dtype, - tp_rank=head_shard_rank, - tp_size=head_shard_size, + parallel_group="attn_tp", ) else: self.f_a_proj = ReplicatedLinear( @@ -498,8 +492,7 @@ class Glm5NextLinearAttention(nn.Module): bias=False, quant_config=quant_config, prefix=f"{prefix}.f_b_proj", - tp_rank=head_shard_rank, - tp_size=head_shard_size, + parallel_group="attn_tp", ) self.b_proj = ColumnParallelLinear( @@ -508,8 +501,7 @@ class Glm5NextLinearAttention(nn.Module): bias=False, quant_config=quant_config, prefix=f"{prefix}.b_proj", - tp_rank=head_shard_rank, - tp_size=head_shard_size, + parallel_group="attn_tp", ) self.g_a_proj = ReplicatedLinear( @@ -525,8 +517,7 @@ class Glm5NextLinearAttention(nn.Module): bias=False, quant_config=quant_config, prefix=f"{prefix}.g_b_proj", - tp_rank=head_shard_rank, - tp_size=head_shard_size, + parallel_group="attn_tp", ) self.dt_bias = nn.Parameter( @@ -544,8 +535,7 @@ class Glm5NextLinearAttention(nn.Module): bias=False, params_dtype=torch.float32, prefix=f"{prefix}.qkv_conv1d", - tp_rank=head_shard_rank, - tp_size=head_shard_size, + parallel_group="attn_tp", ) # ColumnParallelLinear's loader cannot reshape conv1d weights, so add the # singleton dimension after construction. @@ -569,8 +559,7 @@ class Glm5NextLinearAttention(nn.Module): quant_config=quant_config, prefix=f"{prefix}.o_proj", reduce_results=reduce_results, - tp_rank=head_shard_rank, - tp_size=head_shard_size, + parallel_group="attn_tp", ) conv_weights = self.qkv_conv1d.weight.squeeze(1) diff --git a/python/sglang/srt/models/hunyuan_v4.py b/python/sglang/srt/models/hunyuan_v4.py index 2fff1e3ff6..fef1c23237 100644 --- a/python/sglang/srt/models/hunyuan_v4.py +++ b/python/sglang/srt/models/hunyuan_v4.py @@ -357,14 +357,12 @@ class HYV4Attention(DeepseekV2AttentionMLA): alt_stream=alt_stream, is_nextn=is_nextn, ) - parallel = get_parallel() self.linear_gate = ColumnParallelLinear( config.hidden_size, config.num_attention_heads * config.v_head_dim, bias=False, quant_config=quant_config, - tp_rank=parallel.attn_tp_rank, - tp_size=parallel.attn_tp_size, + parallel_group="attn_tp", prefix=f"{prefix}.linear_gate", ) self.local_gate_width = self.num_local_heads * config.v_head_dim diff --git a/python/sglang/srt/models/inkling.py b/python/sglang/srt/models/inkling.py index e9eeb9e616..ef999fa50f 100644 --- a/python/sglang/srt/models/inkling.py +++ b/python/sglang/srt/models/inkling.py @@ -199,8 +199,7 @@ class InklingDecoderLayer(nn.Module): quant_config=quant_config, prefix=add_prefix("mlp", prefix), fused=True, - tp_rank=get_parallel().attn_tp_rank, - tp_size=get_parallel().attn_tp_size, + parallel_group="attn_tp", tp_group=get_parallel().attn_tp_group, use_dp_attention_reduce=True, ) diff --git a/python/sglang/srt/models/inkling_common/attn.py b/python/sglang/srt/models/inkling_common/attn.py index 74dfc3e6bf..f64c325a91 100644 --- a/python/sglang/srt/models/inkling_common/attn.py +++ b/python/sglang/srt/models/inkling_common/attn.py @@ -269,7 +269,6 @@ class InklingQKVRLinear(MergedColumnParallelLinear): inkling_head_dim: int, inkling_num_heads: int, inkling_d_rel: int, - inkling_tp_size: int, **kwargs, ) -> None: super().__init__(*args, **kwargs) @@ -277,7 +276,7 @@ class InklingQKVRLinear(MergedColumnParallelLinear): self.inkling_head_dim = inkling_head_dim self.inkling_num_heads = inkling_num_heads self.inkling_d_rel = inkling_d_rel - self.inkling_tp_size = inkling_tp_size + self.inkling_tp_size = self.tp_size class InklingAttention(nn.Module): @@ -305,7 +304,6 @@ class InklingAttention(nn.Module): self.hidden_size = hidden_size self.alt_stream = alt_stream - attn_tp_rank = get_parallel().attn_tp_rank attn_tp_size = get_parallel().attn_tp_size self.tp_size = attn_tp_size @@ -339,14 +337,12 @@ class InklingAttention(nn.Module): output_sizes=output_sizes, bias=q_bias, prefix=add_prefix("qkvr", prefix), - tp_rank=attn_tp_rank, - tp_size=attn_tp_size, + parallel_group="attn_tp", quant_config=quant_config, inkling_num_kv_heads=self.num_total_kv_heads, inkling_head_dim=self.head_dim, inkling_num_heads=self.num_total_heads, inkling_d_rel=self.d_rel, - inkling_tp_size=attn_tp_size, ) self.wo_ud = RowParallelLinear( input_size=self.head_dim * self.num_total_heads, diff --git a/python/sglang/srt/models/inkling_common/dense_mlp.py b/python/sglang/srt/models/inkling_common/dense_mlp.py index 8dd19a0813..2c269fb873 100644 --- a/python/sglang/srt/models/inkling_common/dense_mlp.py +++ b/python/sglang/srt/models/inkling_common/dense_mlp.py @@ -5,6 +5,7 @@ import torch from torch import nn from torch.nn import functional as F +from sglang.srt.layers.linear import LinearParallelGroup, resolve_linear_parallel_group from sglang.srt.layers.moe.moe_runner.base import MoeRunnerConfig from sglang.srt.layers.quantization.base_config import QuantizationConfig from sglang.srt.layers.quantization.unquant import UnquantizedFusedMoEMethod @@ -91,13 +92,10 @@ class InklingDenseMLP(LlamaMLP): quant_config: QuantizationConfig | None = None, prefix: str = "", fused: bool = False, - tp_rank: int = 0, - tp_size: int = 1, + parallel_group: LinearParallelGroup = "replicated", tp_group: torch.distributed.ProcessGroup | None = None, use_dp_attention_reduce: bool = False, ) -> None: - self.tp_rank = tp_rank - self.tp_size = tp_size self.tp_group = tp_group super().__init__( @@ -107,8 +105,7 @@ class InklingDenseMLP(LlamaMLP): quant_config=quant_config, prefix=prefix, reduce_results=False, - tp_rank=self.tp_rank, - tp_size=self.tp_size, + parallel_group=parallel_group, use_dp_attention_reduce=use_dp_attention_reduce, ) @@ -160,8 +157,7 @@ class InklingBatchDenseMLP(nn.Module, FusedMoELoadingMixin): prefix: str, quant_config: QuantizationConfig | None = None, inference_moe_w13_interleaved: bool = True, - tp_rank: int = 0, - tp_size: int = 1, + parallel_group: LinearParallelGroup = "replicated", tp_group: torch.distributed.ProcessGroup | None = None, linearized_bf16: bool = False, ): @@ -186,8 +182,9 @@ class InklingBatchDenseMLP(nn.Module, FusedMoELoadingMixin): self.hidden_size = d_model self.layer_id = layer_id - self.moe_tp_rank = tp_rank - self.moe_tp_size = tp_size + self.moe_tp_rank, self.moe_tp_size = resolve_linear_parallel_group( + parallel_group + ) self.tp_group = tp_group local_intermediate_size = shared_d_mlp // self.moe_tp_size diff --git a/python/sglang/srt/models/inkling_common/moe.py b/python/sglang/srt/models/inkling_common/moe.py index 6f095de4af..d776bfbe76 100644 --- a/python/sglang/srt/models/inkling_common/moe.py +++ b/python/sglang/srt/models/inkling_common/moe.py @@ -718,7 +718,6 @@ def _build_inkling_shared_experts( if n_shared_experts <= 0: return None # The shared dense MLP is reconstructed by the full-TP all-reduce in forward. - parallel = get_parallel() if shared_expert_sink: shared_prefix = add_prefix("shared_experts", prefix) shared_sink_serves_fp4 = InklingBatchDenseMLP._resolve_fp4_strategy( @@ -746,8 +745,7 @@ def _build_inkling_shared_experts( prefix=shared_prefix, quant_config=quant_config, inference_moe_w13_interleaved=inference_moe_w13_interleaved, - tp_rank=parallel.tp_rank, - tp_size=parallel.tp_size, + parallel_group="tp", tp_group=get_parallel().tp_group, ) return InklingBatchDenseMLP( @@ -765,8 +763,7 @@ def _build_inkling_shared_experts( layer_id=layer_id, prefix=add_prefix("shared_experts", prefix), quant_config=quant_config, - tp_rank=parallel.tp_rank, - tp_size=parallel.tp_size, + parallel_group="tp", tp_group=get_parallel().tp_group, ) diff --git a/python/sglang/srt/models/kimi_k3.py b/python/sglang/srt/models/kimi_k3.py index 6afa17e1d9..d1d31e6f42 100644 --- a/python/sglang/srt/models/kimi_k3.py +++ b/python/sglang/srt/models/kimi_k3.py @@ -51,6 +51,7 @@ from sglang.srt.layers.layernorm import RMSNorm from sglang.srt.layers.linear import ( ColumnParallelBatchedLinear, ColumnParallelLinear, + LinearParallelGroup, MergedColumnParallelLinear, MergedColumnParallelRepeatedLinear, QKVParallelLinear, @@ -305,8 +306,7 @@ class KimiK3MLP(nn.Module): prefix: str = "", activation_situ_beta: float | None = None, activation_situ_linear_beta: float | None = None, - tp_rank: Optional[int] = None, - tp_size: Optional[int] = None, + parallel_group: Optional[LinearParallelGroup] = None, ) -> None: super().__init__() # The Ascend path shards the dense MLP inside each attention-TP @@ -317,22 +317,17 @@ class KimiK3MLP(nn.Module): self._dense_attn_tp = ( get_parallel().enable_dense_mlp_attn_tp and is_dp_attention_enabled() - and tp_rank is None - and tp_size is None - ) - if self._dense_attn_tp: - tp_rank = get_parallel().attn_tp_rank - tp_size = get_parallel().attn_tp_size - _tp_kwargs = ( - dict(tp_rank=tp_rank, tp_size=tp_size) if tp_size is not None else {} + and parallel_group is None ) + if parallel_group is None: + parallel_group = "attn_tp" if self._dense_attn_tp else "tp" self.gate_up_proj = MergedColumnParallelLinear( hidden_size, [intermediate_size] * 2, bias=False, quant_config=quant_config, prefix=f"{prefix}.gate_up_proj", - **_tp_kwargs, + parallel_group=parallel_group, ) self.down_proj = RowParallelLinear( intermediate_size, @@ -342,7 +337,7 @@ class KimiK3MLP(nn.Module): reduce_results=reduce_results, use_dp_attention_reduce=self._dense_attn_tp, prefix=f"{prefix}.down_proj", - **_tp_kwargs, + parallel_group=parallel_group, ) if hidden_act == "silu": self.act_fn = SiluAndMul() @@ -586,9 +581,9 @@ class KimiK3MoE(nn.Module): self._ep_a2a and shared_tp is not None and shared_tp > 1 ) self._shared_experts_tp_group = None - shared_experts_tp_kwargs = {} + shared_experts_parallel_group = None if self._shared_experts_tp1: - shared_experts_tp_kwargs = dict(tp_rank=0, tp_size=1) + shared_experts_parallel_group = "replicated" elif self._shared_experts_tp_comm: group = ( parallel.shared_experts_tp_group @@ -597,8 +592,8 @@ class KimiK3MoE(nn.Module): ) assert group.world_size == shared_tp self._shared_experts_tp_group = group - shared_experts_tp_kwargs = dict( - tp_rank=group.rank_in_group, tp_size=group.world_size + shared_experts_parallel_group = ( + "shared_experts_tp" if requested_shared_tp is not None else "attn_tp" ) if self.num_shared_experts is not None and self.num_shared_experts > 0: shared_intermediate_size = moe_intermediate_size * self.num_shared_experts @@ -616,7 +611,7 @@ class KimiK3MoE(nn.Module): prefix=f"{prefix}.shared_experts", activation_situ_beta=config.activation_situ_beta, activation_situ_linear_beta=config.activation_situ_linear_beta, - **shared_experts_tp_kwargs, + parallel_group=shared_experts_parallel_group, ) else: self.shared_experts = None @@ -1556,7 +1551,7 @@ class KimiK3DeltaAttention(nn.Module): ) # The full-rank [q, k, v, g] merged projection is explicitly sharded - # with attn_tp_rank/attn_tp_size, so it also supports DP attention. + # using attention-TP placement, so it also supports DP attention. # The low-rank fused path still uses full-TP-only projection helpers. # For the full-rank gate (K3) the checkpoint quantizes only the MoE # experts; attention linears resolve to UnquantizedLinearMethod, so a @@ -1579,8 +1574,7 @@ class KimiK3DeltaAttention(nn.Module): ], bias=False, quant_config=quant_config, - tp_rank=self.attn_tp_rank, - tp_size=self.attn_tp_size, + parallel_group="attn_tp", prefix=f"{prefix}.fused_qkvg_proj", ) self.split_sizes = [ @@ -1592,8 +1586,7 @@ class KimiK3DeltaAttention(nn.Module): self.num_heads, bias=False, quant_config=quant_config, - tp_rank=self.attn_tp_rank, - tp_size=self.attn_tp_size, + parallel_group="attn_tp", prefix=f"{prefix}.b_proj", # TP8 shards K3's 96 beta rows below the 128-row FP8 block. skip_block_quant_check=self._bfa_uses_block_fp8, @@ -1610,8 +1603,7 @@ class KimiK3DeltaAttention(nn.Module): projection_size, bias=False, quant_config=quant_config, - tp_rank=self.attn_tp_rank, - tp_size=self.attn_tp_size, + parallel_group="attn_tp", prefix=f"{prefix}.f_b_proj", ) # Merged [f_a | b] weight, built after weight loading by @@ -1657,7 +1649,6 @@ class KimiK3DeltaAttention(nn.Module): 2, self.head_dim, projection_size, dtype=_dtype ) else: - attn_tp_rank = self.attn_tp_rank self.qkv_proj = QKVParallelLinear( self.hidden_size, self.head_dim, @@ -1665,8 +1656,7 @@ class KimiK3DeltaAttention(nn.Module): self.num_k_heads, bias=False, quant_config=quant_config, - tp_rank=attn_tp_rank, - tp_size=self.attn_tp_size, + parallel_group="attn_tp", v_head_size=self.head_v_dim, prefix=f"{prefix}.qkv_proj", ) @@ -1683,8 +1673,7 @@ class KimiK3DeltaAttention(nn.Module): projection_size, bias=False, quant_config=quant_config, - tp_rank=attn_tp_rank, - tp_size=self.attn_tp_size, + parallel_group="attn_tp", prefix=f"{prefix}.f_b_proj", ) self.b_proj = ColumnParallelLinear( @@ -1692,8 +1681,7 @@ class KimiK3DeltaAttention(nn.Module): self.num_heads, bias=False, quant_config=quant_config, - tp_rank=attn_tp_rank, - tp_size=self.attn_tp_size, + parallel_group="attn_tp", prefix=f"{prefix}.b_proj", ) @@ -1703,8 +1691,7 @@ class KimiK3DeltaAttention(nn.Module): projection_size, bias=False, quant_config=quant_config, - tp_rank=attn_tp_rank, - tp_size=self.attn_tp_size, + parallel_group="attn_tp", prefix=f"{prefix}.g_proj", ) else: @@ -1720,8 +1707,7 @@ class KimiK3DeltaAttention(nn.Module): projection_size, bias=False, quant_config=quant_config, - tp_rank=attn_tp_rank, - tp_size=self.attn_tp_size, + parallel_group="attn_tp", prefix=f"{prefix}.g_b_proj", ) @@ -1735,8 +1721,7 @@ class KimiK3DeltaAttention(nn.Module): output_sizes=[projection_size, projection_size, projection_size], bias=False, params_dtype=torch.float32, - tp_rank=self.attn_tp_rank, - tp_size=self.attn_tp_size, + parallel_group="attn_tp", prefix=f"{prefix}.qkv_conv1d", ) self.qkv_conv1d.weight.data = self.qkv_conv1d.weight.data.unsqueeze(1) @@ -1780,8 +1765,7 @@ class KimiK3DeltaAttention(nn.Module): # comm lives there). reduce_results=not self.all_reduce_fusion, quant_config=quant_config, - tp_rank=self.attn_tp_rank, - tp_size=self.attn_tp_size, + parallel_group="attn_tp", # Reduce within the attn-TP group: the default full-TP collective # is the wrong group at attn_tp>1 (sums across DP groups) and # deadlocks idle DP ranks. Off under all_reduce_fusion: the fused @@ -2290,8 +2274,7 @@ class KimiK3MLAAttention(DeepseekV2AttentionMLA): projection_size, bias=False, quant_config=quant_config, - tp_rank=get_parallel().attn_tp_rank, - tp_size=get_parallel().attn_tp_size, + parallel_group="attn_tp", prefix=f"{prefix}.g_proj", ) # Output gate multiplies the TP-local attention output right diff --git a/python/sglang/srt/models/kimi_linear.py b/python/sglang/srt/models/kimi_linear.py index dc68a6f689..b76c405b56 100644 --- a/python/sglang/srt/models/kimi_linear.py +++ b/python/sglang/srt/models/kimi_linear.py @@ -225,12 +225,11 @@ class KimiDeltaAttention(nn.Module): self.attn_tp_size = get_parallel().attn_tp_size # Group the weights are sharded over. Defaults to the global TP group, # which is what plain Kimi-Linear has always used. + parallel_group = "attn_tp" if shard_on_attn_tp else "tp" if shard_on_attn_tp: self.shard_tp_size = self.attn_tp_size - self.shard_tp_rank = get_parallel().attn_tp_rank else: self.shard_tp_size = self.tp_size - self.shard_tp_rank = get_parallel().tp_rank self.hidden_size = hidden_size self.config = config self.head_dim = config.linear_attn_config["head_dim"] @@ -277,8 +276,7 @@ class KimiDeltaAttention(nn.Module): bias=False, quant_config=quant_config, prefix=f"{prefix}.fused_qkvbfg_proj", - tp_rank=self.shard_tp_rank, - tp_size=self.shard_tp_size, + parallel_group=parallel_group, ) self.split_sizes = [3 * projection_size // self.shard_tp_size] if self.fuse_no_lora_beta: @@ -295,8 +293,7 @@ class KimiDeltaAttention(nn.Module): self.num_heads, bias=False, prefix=f"{prefix}.b_proj", - tp_rank=self.shard_tp_rank, - tp_size=self.shard_tp_size, + parallel_group=parallel_group, ) elif self.do_fuse_qkvbfg: # Fuse: q, k, v, beta (column parallel) + f_a, g_a (replicated) @@ -314,8 +311,7 @@ class KimiDeltaAttention(nn.Module): self.fg_sizes, quant_config=quant_config, prefix=f"{prefix}.fused_qkvbfg_a_proj", - tp_rank=self.shard_tp_rank, - tp_size=self.shard_tp_size, + parallel_group=parallel_group, ) self.split_sizes = [ 3 * projection_size // self.shard_tp_size, @@ -327,12 +323,10 @@ class KimiDeltaAttention(nn.Module): self.head_dim, projection_size, dtype=config.dtype, - tp_rank=self.shard_tp_rank, - tp_size=self.shard_tp_size, + parallel_group=parallel_group, ) else: # Unfused path: separate QKVParallelLinear - attn_tp_rank = get_parallel().attn_tp_rank self.qkv_proj = QKVParallelLinear( self.hidden_size, self.head_dim, @@ -340,8 +334,7 @@ class KimiDeltaAttention(nn.Module): self.num_k_heads, bias=False, quant_config=quant_config, - tp_rank=attn_tp_rank, - tp_size=self.attn_tp_size, + parallel_group="attn_tp", v_head_size=self.head_v_dim, prefix=f"{prefix}.qkv_proj", ) @@ -360,8 +353,7 @@ class KimiDeltaAttention(nn.Module): bias=False, quant_config=quant_config, prefix=f"{prefix}.f_b_proj", - tp_rank=self.shard_tp_rank, - tp_size=self.shard_tp_size, + parallel_group=parallel_group, ) self.b_proj = ColumnParallelLinear( @@ -370,8 +362,7 @@ class KimiDeltaAttention(nn.Module): bias=False, quant_config=quant_config, prefix=f"{prefix}.b_proj", - tp_rank=self.shard_tp_rank, - tp_size=self.shard_tp_size, + parallel_group=parallel_group, ) self.g_a_proj = ReplicatedLinear( @@ -387,8 +378,7 @@ class KimiDeltaAttention(nn.Module): bias=False, quant_config=quant_config, prefix=f"{prefix}.g_b_proj", - tp_rank=self.shard_tp_rank, - tp_size=self.shard_tp_size, + parallel_group=parallel_group, ) self.dt_bias = nn.Parameter( @@ -405,8 +395,7 @@ class KimiDeltaAttention(nn.Module): bias=False, params_dtype=torch.float32, prefix=f"{prefix}.qkv_conv1d", - tp_rank=self.shard_tp_rank, - tp_size=self.shard_tp_size, + parallel_group=parallel_group, ) # unsqueeze to fit conv1d weights shape into the linear weights shape. # Can't do this in `weight_loader` since it already exists in @@ -428,8 +417,7 @@ class KimiDeltaAttention(nn.Module): bias=False, quant_config=quant_config, prefix=f"{prefix}.o_proj", - tp_rank=self.shard_tp_rank, - tp_size=self.shard_tp_size, + parallel_group=parallel_group, reduce_results=reduce_results, ) diff --git a/python/sglang/srt/models/llama.py b/python/sglang/srt/models/llama.py index be0ff07b25..84569c13e0 100644 --- a/python/sglang/srt/models/llama.py +++ b/python/sglang/srt/models/llama.py @@ -39,6 +39,7 @@ from sglang.srt.layers.layer_boundary.residual import batch as residual_batch from sglang.srt.layers.layer_boundary.residual.add_norm import NORM_QUANT_READOUT from sglang.srt.layers.layernorm import RMSNorm from sglang.srt.layers.linear import ( + LinearParallelGroup, MergedColumnParallelLinear, QKVParallelLinear, RowParallelLinear, @@ -89,8 +90,7 @@ class LlamaMLP(nn.Module): quant_config: Optional[QuantizationConfig] = None, prefix: str = "", reduce_results: bool = True, - tp_rank: Optional[int] = None, - tp_size: Optional[int] = None, + parallel_group: LinearParallelGroup = "tp", use_dp_attention_reduce: bool = False, ) -> None: super().__init__() @@ -100,8 +100,7 @@ class LlamaMLP(nn.Module): bias=False, quant_config=quant_config, prefix=add_prefix("gate_up_proj", prefix), - tp_rank=tp_rank, - tp_size=tp_size, + parallel_group=parallel_group, ) self.down_proj = RowParallelLinear( intermediate_size, @@ -110,8 +109,7 @@ class LlamaMLP(nn.Module): quant_config=quant_config, prefix=add_prefix("down_proj", prefix), reduce_results=reduce_results, - tp_rank=tp_rank, - tp_size=tp_size, + parallel_group=parallel_group, use_dp_attention_reduce=use_dp_attention_reduce, ) if hidden_act != "silu": diff --git a/python/sglang/srt/models/mimo_v2.py b/python/sglang/srt/models/mimo_v2.py index 6476ec8695..9d9b69769d 100644 --- a/python/sglang/srt/models/mimo_v2.py +++ b/python/sglang/srt/models/mimo_v2.py @@ -606,7 +606,6 @@ class MiMoV2Attention(nn.Module): super().__init__() self.hidden_size = hidden_size - attn_tp_rank = get_parallel().attn_tp_rank attn_tp_size = get_parallel().attn_tp_size self.total_num_heads = num_heads @@ -641,8 +640,7 @@ class MiMoV2Attention(nn.Module): v_head_size=self.v_head_dim, bias=attention_bias, quant_config=quant_config, - tp_rank=attn_tp_rank, - tp_size=attn_tp_size, + parallel_group="attn_tp", prefix=add_prefix("qkv_proj", prefix), skip_block_quant_check=True, ) @@ -652,8 +650,7 @@ class MiMoV2Attention(nn.Module): hidden_size, bias=False, quant_config=quant_config, - tp_rank=attn_tp_rank, - tp_size=attn_tp_size, + parallel_group="attn_tp", reduce_results=False, prefix=add_prefix("o_proj", prefix), ) diff --git a/python/sglang/srt/models/minimax_m3.py b/python/sglang/srt/models/minimax_m3.py index cd0be2109d..7a1b4801d5 100644 --- a/python/sglang/srt/models/minimax_m3.py +++ b/python/sglang/srt/models/minimax_m3.py @@ -52,6 +52,7 @@ from sglang.srt.layers.linear import ( MergedColumnParallelLinear, QKVParallelLinear, ReplicatedLinear, + ReplicatedParallelGroup, RowParallelLinear, ) from sglang.srt.layers.logits_processor import LogitsProcessor @@ -601,6 +602,9 @@ class MiniMaxM3Attention(nn.Module): self.idx_replica_size = attn_tp_size // self.idx_head_tp_size self.idx_head_rank = attn_tp_rank // self.idx_replica_size self.num_idx_heads = self.total_idx_heads // self.idx_head_tp_size + index_parallel_group = ReplicatedParallelGroup( + "attn_tp", self.idx_replica_size + ) self.qkv_proj = QKVParallelLinear( self.hidden_size, @@ -609,8 +613,7 @@ class MiniMaxM3Attention(nn.Module): self.total_num_kv_heads, bias=False, quant_config=quant_config, - tp_rank=attn_tp_rank, - tp_size=attn_tp_size, + parallel_group="attn_tp", prefix=add_prefix("qkv_proj", prefix), ) @@ -620,8 +623,7 @@ class MiniMaxM3Attention(nn.Module): bias=False, reduce_results=False, quant_config=quant_config, - tp_rank=attn_tp_rank, - tp_size=attn_tp_size, + parallel_group="attn_tp", prefix=add_prefix("o_proj", prefix), ) @@ -642,8 +644,7 @@ class MiniMaxM3Attention(nn.Module): bias=False, quant_config=quant_config, v_head_size=(0 if self.disable_index_value else self.idx_head_dim), - tp_rank=self.idx_head_rank, - tp_size=self.idx_head_tp_size, + parallel_group=index_parallel_group, prefix=add_prefix("index_qkv_proj", prefix), ) @@ -658,8 +659,7 @@ class MiniMaxM3Attention(nn.Module): reduce_results=False, quant_config=quant_config, prefix=add_prefix("index_o_proj", prefix), - tp_rank=self.idx_head_rank, - tp_size=self.idx_head_tp_size, + parallel_group=index_parallel_group, ) self.index_rotary_emb = self.rotary_emb diff --git a/python/sglang/srt/models/nemotron_h_mtp.py b/python/sglang/srt/models/nemotron_h_mtp.py index c050745e0a..630fbd43ff 100644 --- a/python/sglang/srt/models/nemotron_h_mtp.py +++ b/python/sglang/srt/models/nemotron_h_mtp.py @@ -70,8 +70,7 @@ class NemotronHMTPAttentionDecoderLayer(NemotronHAttentionDecoderLayer): output_size=config.hidden_size, bias=False, gather_output=not _dp_attn, - tp_rank=get_parallel().attn_tp_rank if _dp_attn else None, - tp_size=get_parallel().attn_tp_size if _dp_attn else None, + parallel_group="attn_tp" if _dp_attn else "tp", params_dtype=( config.dtype if hasattr(config, "dtype") else torch.bfloat16 ), @@ -152,8 +151,7 @@ class NemotronHMTPMoEDecoderLayer(NemotronHMoEDecoderLayer): output_size=config.hidden_size, bias=False, gather_output=not _dp_attn, - tp_rank=get_parallel().attn_tp_rank if _dp_attn else None, - tp_size=get_parallel().attn_tp_size if _dp_attn else None, + parallel_group="attn_tp" if _dp_attn else "tp", params_dtype=( config.dtype if hasattr(config, "dtype") else torch.bfloat16 ), diff --git a/python/sglang/srt/models/qwen3_5.py b/python/sglang/srt/models/qwen3_5.py index 215f057960..5ebbe76c8f 100644 --- a/python/sglang/srt/models/qwen3_5.py +++ b/python/sglang/srt/models/qwen3_5.py @@ -72,9 +72,12 @@ from sglang.srt.layers.layernorm import GemmaRMSNorm # Layers - Linear from sglang.srt.layers.linear import ( ColumnParallelLinear, + LinearParallelGroup, MergedColumnParallelLinear, QKVParallelLinear, + ReplicatedParallelGroup, RowParallelLinear, + resolve_linear_parallel_group, ) from sglang.srt.layers.moe.fused_moe_triton.layer import FusedMoE from sglang.srt.layers.moe.utils import ( @@ -341,8 +344,7 @@ class Qwen3_5GatedDeltaNet(nn.Module): output_size=self.conv_dim, bias=False, quant_config=None, - tp_rank=self.attn_tp_rank, - tp_size=self.attn_tp_size, + parallel_group="attn_tp", prefix=add_prefix("conv1d", prefix), ) self.conv1d.weight.data = self.conv1d.weight.data.unsqueeze(1) @@ -354,8 +356,7 @@ class Qwen3_5GatedDeltaNet(nn.Module): value_dim=self.value_dim, quant_config=quant_config, prefix=add_prefix("in_proj_qkvz", prefix), - tp_rank=self.attn_tp_rank, - tp_size=self.attn_tp_size, + parallel_group="attn_tp", ) self.in_proj_ba = self.create_ba_proj( @@ -363,8 +364,7 @@ class Qwen3_5GatedDeltaNet(nn.Module): num_v_heads=self.num_v_heads, quant_config=quant_config, prefix=add_prefix("in_proj_ba", prefix), - tp_rank=self.attn_tp_rank, - tp_size=self.attn_tp_size, + parallel_group="attn_tp", ) # Override weight loaders for packed checkpoint format. @@ -462,8 +462,7 @@ class Qwen3_5GatedDeltaNet(nn.Module): input_is_parallel=True, reduce_results=False, quant_config=quant_config, - tp_rank=self.attn_tp_rank, - tp_size=self.attn_tp_size, + parallel_group="attn_tp", prefix=add_prefix("out_proj", prefix), ) @@ -578,8 +577,8 @@ class Qwen3_5GatedDeltaNet(nn.Module): value_dim: int, quant_config: QuantizationConfig | None, prefix: str, - tp_rank: Optional[int] = None, - tp_size: Optional[int] = None, + *, + parallel_group: LinearParallelGroup = "tp", ) -> MergedColumnParallelLinear: return MergedColumnParallelLinear( input_size=hidden_size, @@ -587,8 +586,7 @@ class Qwen3_5GatedDeltaNet(nn.Module): bias=False, quant_config=quant_config, prefix=prefix, - tp_rank=tp_rank, - tp_size=tp_size, + parallel_group=parallel_group, ) def create_ba_proj( @@ -597,8 +595,8 @@ class Qwen3_5GatedDeltaNet(nn.Module): num_v_heads: int, quant_config: QuantizationConfig | None, prefix: str, - tp_rank: Optional[int] = None, - tp_size: Optional[int] = None, + *, + parallel_group: LinearParallelGroup = "tp", ) -> MergedColumnParallelLinear: # Qwen3.5 has separate in_proj_b and in_proj_a weights in the # checkpoint, which are loaded into the fused in_proj_ba parameter @@ -609,8 +607,7 @@ class Qwen3_5GatedDeltaNet(nn.Module): bias=False, quant_config=quant_config, prefix=prefix, - tp_rank=tp_rank, - tp_size=tp_size, + parallel_group=parallel_group, ) def fix_query_key_value_ordering( @@ -1187,8 +1184,10 @@ class Qwen3_5AttentionDecoderLayer(nn.Module): # A Qwen3.5 draft is rewritten to the MTP arch (model_config._config_draft_model), # so is_nextn marks it. Drafts are TP-sharded and do not replicate KV under DCP. dcp_size = 1 if is_nextn else get_parallel().attn_dcp_size - self.kv_tp_size = self.attn_tp_size // dcp_size - self.kv_tp_rank = self.attn_tp_rank // dcp_size + kv_parallel_group = ReplicatedParallelGroup("attn_tp", dcp_size) + self.kv_tp_rank, self.kv_tp_size = resolve_linear_parallel_group( + kv_parallel_group + ) self.total_num_heads = config.num_attention_heads assert self.total_num_heads % self.attn_tp_size == 0 self.num_heads = self.total_num_heads // self.attn_tp_size @@ -1236,10 +1235,8 @@ class Qwen3_5AttentionDecoderLayer(nn.Module): self.total_num_kv_heads, bias=False, quant_config=quant_config, - tp_rank=self.attn_tp_rank, - tp_size=self.attn_tp_size, - kv_tp_rank=self.kv_tp_rank, - kv_tp_size=self.kv_tp_size, + parallel_group="attn_tp", + kv_parallel_group=kv_parallel_group, prefix=add_prefix("qkv_proj", prefix), ) @@ -1249,8 +1246,7 @@ class Qwen3_5AttentionDecoderLayer(nn.Module): bias=False, quant_config=quant_config, reduce_results=False, - tp_rank=self.attn_tp_rank, - tp_size=self.attn_tp_size, + parallel_group="attn_tp", prefix=add_prefix("o_proj", prefix), ) diff --git a/python/sglang/srt/models/qwen3_next.py b/python/sglang/srt/models/qwen3_next.py index f0739fa1a6..938f166eed 100644 --- a/python/sglang/srt/models/qwen3_next.py +++ b/python/sglang/srt/models/qwen3_next.py @@ -31,6 +31,7 @@ from sglang.srt.layers.layer_boundary.residual import batch as residual_batch from sglang.srt.layers.layernorm import GemmaRMSNorm from sglang.srt.layers.linear import ( ColumnParallelLinear, + LinearParallelGroup, MergedColumnParallelLinear, QKVParallelLinear, RowParallelLinear, @@ -132,8 +133,7 @@ class Qwen3GatedDeltaNet(nn.Module): output_size=self.conv_dim, bias=False, quant_config=None, - tp_rank=self.attn_tp_rank, - tp_size=self.attn_tp_size, + parallel_group="attn_tp", prefix=add_prefix("conv1d", prefix), ) self.conv1d.weight.data = self.conv1d.weight.data.unsqueeze(1) @@ -145,8 +145,7 @@ class Qwen3GatedDeltaNet(nn.Module): value_dim=self.value_dim, quant_config=quant_config, prefix=add_prefix("in_proj_qkvz", prefix), - tp_rank=self.attn_tp_rank, - tp_size=self.attn_tp_size, + parallel_group="attn_tp", ) self.in_proj_ba = MergedColumnParallelLinear( @@ -155,8 +154,7 @@ class Qwen3GatedDeltaNet(nn.Module): bias=False, quant_config=quant_config, prefix=add_prefix("in_proj_ba", prefix), - tp_rank=self.attn_tp_rank, - tp_size=self.attn_tp_size, + parallel_group="attn_tp", ) # Override weight_loader for packed checkpoint format. @@ -231,8 +229,7 @@ class Qwen3GatedDeltaNet(nn.Module): quant_config=quant_config, input_is_parallel=True, reduce_results=False, - tp_rank=self.attn_tp_rank, - tp_size=self.attn_tp_size, + parallel_group="attn_tp", prefix=add_prefix("out_proj", prefix), ) @@ -319,8 +316,8 @@ class Qwen3GatedDeltaNet(nn.Module): value_dim: int, quant_config: QuantizationConfig | None, prefix: str, - tp_rank: Optional[int] = None, - tp_size: Optional[int] = None, + *, + parallel_group: LinearParallelGroup = "tp", ) -> MergedColumnParallelLinear: return MergedColumnParallelLinear( input_size=hidden_size, @@ -328,8 +325,7 @@ class Qwen3GatedDeltaNet(nn.Module): bias=False, quant_config=quant_config, prefix=prefix, - tp_rank=tp_rank, - tp_size=tp_size, + parallel_group=parallel_group, ) def fix_query_key_value_ordering( diff --git a/python/sglang/srt/models/sdar_moe.py b/python/sglang/srt/models/sdar_moe.py index b5372e246a..90095df243 100644 --- a/python/sglang/srt/models/sdar_moe.py +++ b/python/sglang/srt/models/sdar_moe.py @@ -191,7 +191,6 @@ class SDARMoeAttention(nn.Module): self.hidden_size = config.hidden_size self.total_num_heads = config.num_attention_heads - attn_tp_rank = get_parallel().attn_tp_rank attn_tp_size = get_parallel().attn_tp_size assert self.total_num_heads % attn_tp_size == 0 @@ -226,8 +225,7 @@ class SDARMoeAttention(nn.Module): bias=getattr(config, "attention_bias", False), quant_config=quant_config, reduce_results=reduce_results, - tp_rank=attn_tp_rank, - tp_size=attn_tp_size, + parallel_group="attn_tp", prefix=add_prefix("o_proj", prefix), ) diff --git a/python/sglang/srt/models/step3_vl.py b/python/sglang/srt/models/step3_vl.py index e36dcfc26e..8c205d4f3d 100644 --- a/python/sglang/srt/models/step3_vl.py +++ b/python/sglang/srt/models/step3_vl.py @@ -189,7 +189,6 @@ class Step3TextAttention(nn.Module): super().__init__() self.hidden_size = hidden_size - attn_tp_rank = get_parallel().attn_tp_rank attn_tp_size = get_parallel().attn_tp_size self.total_num_heads = num_heads @@ -219,8 +218,7 @@ class Step3TextAttention(nn.Module): [self.q_size, self.kv_size, self.kv_size], bias=False, quant_config=quant_config, - tp_rank=0, # In fact, we need a MergedReplicatedLinear - tp_size=1, + parallel_group="replicated", prefix=add_prefix("qkv_proj", prefix), ) @@ -229,8 +227,7 @@ class Step3TextAttention(nn.Module): hidden_size, bias=False, quant_config=quant_config, - tp_rank=attn_tp_rank, - tp_size=attn_tp_size, + parallel_group="attn_tp", reduce_results=False, prefix=add_prefix("o_proj", prefix), ) @@ -242,8 +239,7 @@ class Step3TextAttention(nn.Module): self.head_dim * self.total_num_heads, bias=False, quant_config=quant_config, - tp_rank=attn_tp_rank, - tp_size=attn_tp_size, + parallel_group="attn_tp", prefix=add_prefix("wq", prefix), ) self.rotary_emb = get_rope( diff --git a/python/sglang/srt/models/xllm.py b/python/sglang/srt/models/xllm.py index 3f46f3433b..d8c9f411a0 100644 --- a/python/sglang/srt/models/xllm.py +++ b/python/sglang/srt/models/xllm.py @@ -1250,8 +1250,7 @@ class _XllmMoVAAttentionBase(nn.Module): self.total_num_heads * self.head_dim, bias=False, quant_config=None, - tp_rank=self.tp_rank, - tp_size=self.tp_size, + parallel_group="attn_tp", prefix=add_prefix("q_proj", prefix), ) self.k_proj = ColumnParallelLinear( @@ -1259,8 +1258,7 @@ class _XllmMoVAAttentionBase(nn.Module): self.total_num_kv_heads * self.head_dim, bias=False, quant_config=None, - tp_rank=self.tp_rank, - tp_size=self.tp_size, + parallel_group="attn_tp", prefix=add_prefix("k_proj", prefix), ) self.gate_proj = ColumnParallelLinear( @@ -1268,8 +1266,7 @@ class _XllmMoVAAttentionBase(nn.Module): self.total_num_heads * self.head_dim, bias=False, quant_config=None, - tp_rank=self.tp_rank, - tp_size=self.tp_size, + parallel_group="attn_tp", prefix=add_prefix("gate_proj", prefix), ) self.o_proj = RowParallelLinear( @@ -1277,8 +1274,7 @@ class _XllmMoVAAttentionBase(nn.Module): config.hidden_size, bias=False, quant_config=None, - tp_rank=self.tp_rank, - tp_size=self.tp_size, + parallel_group="attn_tp", reduce_results=False, prefix=add_prefix("o_proj", prefix), ) @@ -1346,8 +1342,7 @@ class XllmGatedAttention(_XllmMoVAAttentionBase): self.total_num_kv_heads * self.head_dim, bias=False, quant_config=None, - tp_rank=self.tp_rank, - tp_size=self.tp_size, + parallel_group="attn_tp", prefix=add_prefix("v_proj", prefix), ) @@ -1394,8 +1389,7 @@ class XllmMoVAAttention(_XllmMoVAAttentionBase): self.num_values, config.hidden_size, self.total_num_kv_heads * self.head_dim, - tp_rank=self.tp_rank, - tp_size=self.tp_size, + parallel_group="attn_tp", ) def _project_value(self, hidden_states: torch.Tensor) -> torch.Tensor: diff --git a/python/sglang/srt/models/zaya.py b/python/sglang/srt/models/zaya.py index cf4597b0bc..2c89f5ce42 100644 --- a/python/sglang/srt/models/zaya.py +++ b/python/sglang/srt/models/zaya.py @@ -62,8 +62,10 @@ from sglang.srt.layers.layer_boundary.residual.add_norm import PLAIN_ADD from sglang.srt.layers.layernorm import RMSNorm from sglang.srt.layers.linear import ( ColumnParallelLinear, + LinearParallelGroup, ReplicatedLinear, RowParallelLinear, + resolve_linear_parallel_group, ) from sglang.srt.layers.logits_processor import LogitsProcessor from sglang.srt.layers.moe.ep_moe.layer import get_moe_impl_class @@ -432,8 +434,7 @@ class CCA(nn.Module): layer_id: int, quant_config: Optional[QuantizationConfig] = None, prefix: str = "", - tp_rank: Optional[int] = None, - tp_size: Optional[int] = None, + parallel_group: LinearParallelGroup = "tp", ) -> None: super().__init__() self.config = config @@ -446,12 +447,7 @@ class CCA(nn.Module): self.padding1 = self.cca_time1 - 1 self.total_padding = self.padding0 + self.padding1 - if tp_rank is None: - tp_rank = get_parallel().tp_rank - if tp_size is None: - tp_size = get_parallel().tp_size - self.tp_rank = int(tp_rank) - self.tp_size = int(tp_size) + self.tp_rank, self.tp_size = resolve_linear_parallel_group(parallel_group) # Full (global) head counts retained for weight loading and shape asserts. self.num_q_heads_full = int(cca_num_q_heads) @@ -506,8 +502,7 @@ class CCA(nn.Module): gather_output=False, quant_config=quant_config, prefix=add_prefix("linear_q", prefix), - tp_rank=self.tp_rank, - tp_size=self.tp_size, + parallel_group=parallel_group, ) self.linear_k = ColumnParallelLinear( self.hidden_size, @@ -516,8 +511,7 @@ class CCA(nn.Module): gather_output=False, quant_config=quant_config, prefix=add_prefix("linear_k", prefix), - tp_rank=self.tp_rank, - tp_size=self.tp_size, + parallel_group=parallel_group, ) else: self.linear_q = ReplicatedLinear( diff --git a/test/registered/unit/layers/test_attention_projection_parallel_groups.py b/test/registered/unit/layers/test_attention_projection_parallel_groups.py new file mode 100644 index 0000000000..5582041c1d --- /dev/null +++ b/test/registered/unit/layers/test_attention_projection_parallel_groups.py @@ -0,0 +1,373 @@ +"""Attention projections preserve checkpoint placement and collective policy.""" + +import unittest +from contextlib import nullcontext +from types import SimpleNamespace +from unittest.mock import Mock, patch + +import torch +import torch.nn.functional as F + +from sglang.srt.layers.dp_attention import initialize_dp_attention_flags +from sglang.srt.layers.linear import QKVParallelLinear, RowParallelLinear +from sglang.srt.runtime_context import SpawnRanks, get_parallel, publish, reset_context +from sglang.srt.server_args import ServerArgs +from sglang.test.ci.ci_register import register_cuda_ci +from sglang.test.test_utils import CustomTestCase + +register_cuda_ci(est_time=15, stage="base-b", runner_config="1-gpu-small") + +ATTENTION_CASES = ( + ("bailing", "head_wise"), + ("bailing", "element_wise"), + ("bailing", None), + ("gigachat", False), + ("gigachat", True), + ("hunyuan", None), + ("hunyuan", "elementwise"), + ("exaone4", False), + ("exaone4", True), + ("mimo", False), + ("mimo", True), + ("minimax", "dense"), + ("minimax", "index_replicated"), + ("minimax", "index_sharded"), + ("minimax", "index_no_value"), + ("mtp", "attention"), + ("mtp", "moe"), + ("sdar", False), + ("sdar", True), + ("step3", False), + ("step3", True), + ("xllm", "gated"), + ("xllm", "mova"), +) + + +def build_attention( + model, variant, width=32, head_dim=8, quant_config=None, start=True +): + config = SimpleNamespace( + hidden_size=width, + num_attention_heads=8, + num_key_value_heads=4, + head_dim=head_dim, + rms_norm_eps=1e-6, + layer_norm_epsilon=1e-6, + max_position_embeddings=16, + rope_theta=10000, + rope_scaling=None, + qk_nope_head_dim=head_dim - 4, + qk_rope_head_dim=4, + v_head_dim=head_dim, + q_lora_rank=width, + kv_lora_rank=width, + attention_bias=True, + rope_parameters={ + "rope_theta": 10000, + "rope_type": "deepseek_yarn", + "factor": 1.0, + "original_max_position_embeddings": 16, + }, + gated_attention_proj_granularity_type=variant, + gated_attention=bool(variant), + gating_type=variant, + use_qk_norm=True, + qk_norm_type="per_head", + use_gemma_norm=variant == "index_sharded", + dtype=torch.get_default_dtype(), + num_values=4, + num_values_per_tok=2, + sparse_attention_config={ + "sparse_num_index_heads": 8 if variant == "index_sharded" else 1, + "sparse_index_dim": head_dim, + }, + ) + if model in ("bailing", "gigachat", "hunyuan"): + if model == "hunyuan": + from sglang.srt.models.hunyuan_v4 import HYV4Attention + + module = HYV4Attention(config, 0, quant_config=quant_config) + names = ("linear_gate",) + else: + if model == "bailing": + from sglang.srt.models.bailing_moe_v3 import DsV3MLA + + constructor, name = DsV3MLA, "g_proj" + else: + from sglang.srt.models.gigachat35 import GigaChat35AttentionMLA + + constructor, name = GigaChat35AttentionMLA, "attn_gate" + module = constructor( + config, + width, + 8, + head_dim - 4, + 4, + head_dim, + None if model == "gigachat" and variant else width, + width, + rope_scaling=config.rope_parameters, + max_position_embeddings=16, + quant_config=quant_config, + reduce_results=False, + layer_id=0, + ) + names = (name,) if getattr(module, name, None) is not None else () + elif model == "exaone4": + from sglang.srt.models.exaone4 import Exaone4Attention + + module = Exaone4Attention( + config, + width, + 8, + 1 if variant else 4, + head_dim=head_dim, + max_position_embeddings=16, + quant_config=quant_config, + bias=True, + bias_o_proj=True, + ) + # Preserve the existing full-TP head counts independently of projection placement. + assert module.num_heads == 8 // get_parallel().tp_size + names = ("qkv_proj", "o_proj") + elif model == "mimo": + from sglang.srt.models.mimo_v2 import MiMoV2Attention + + module = MiMoV2Attention( + width, + 8, + 1 if variant else 4, + head_dim=head_dim, + v_head_dim=2 * head_dim if variant else head_dim, + attention_bias=True, + max_position_embeddings=16, + quant_config=quant_config, + ) + names = ("qkv_proj", "o_proj") + elif model == "minimax": + from sglang.srt.models.minimax_m3 import MiniMaxM3Attention + + module = MiniMaxM3Attention( + config, + quant_config=quant_config, + is_sparse_attention_layer=variant != "dense", + disable_index_value=variant == "index_no_value", + ) + names = ("qkv_proj", "o_proj") + elif model == "mtp": + from sglang.srt.models.nemotron_h import ( + NemotronHAttentionDecoderLayer, + NemotronHMoEDecoderLayer, + ) + from sglang.srt.models.nemotron_h_mtp import ( + NemotronHMTPAttentionDecoderLayer, + NemotronHMTPMoEDecoderLayer, + ) + + constructor, parent = ( + (NemotronHMTPAttentionDecoderLayer, NemotronHAttentionDecoderLayer) + if variant == "attention" + else (NemotronHMTPMoEDecoderLayer, NemotronHMoEDecoderLayer) + ) + # Exercise the MTP constructor, leaving the unrelated parent mixer outside this fixture. + with patch.object( + parent, "__init__", lambda self, **kwargs: torch.nn.Module.__init__(self) + ): + module = constructor( + config, + 0, + quant_config=quant_config, + has_start_projections=start, + has_end_norm=True, + ) + names = ("eh_proj",) if start else () + elif model == "sdar": + from sglang.srt.models.sdar_moe import SDARMoeAttention + + module = SDARMoeAttention( + config, 0, quant_config=quant_config, reduce_results=variant + ) + names = ("o_proj",) + elif model == "step3": + from sglang.srt.models.step3_vl import Step3TextAttention + + module = Step3TextAttention( + width, + 8, + 4, + head_dim, + width // 2 if variant else 0, + rms_norm_eps=1e-6, + max_position_embeddings=16, + quant_config=quant_config, + ) + names = ("qkv_proj", "wq", "o_proj") + else: + from sglang.srt.models.xllm import XllmGatedAttention, XllmMoVAAttention + + constructor = XllmGatedAttention if variant == "gated" else XllmMoVAAttention + module = constructor(config, 0, quant_config=quant_config) + names = ("q_proj", "k_proj", "gate_proj", "o_proj") + if variant == "gated": + names += ("v_proj",) + return module, names + + +def values(rows, columns, layer, offset=0): + return ( + (torch.arange(rows * columns, device=layer.weight.device) + offset) % 23 - 11 + ).reshape(rows, columns).to(layer.weight.dtype) / 128 + + +def load_projection(layer): + """Load at rank zero and return the checkpoint shard owned at construction.""" + rank, size = layer.tp_rank, layer.tp_size + row = isinstance(layer, RowParallelLinear) + bias_shard = None + with get_parallel().override(tp_rank=0, attn_tp_rank=0, attn_dp_rank=0): + if isinstance(layer, QKVParallelLinear): + weights, biases = [], [] + for i, shard_id in enumerate(("q", "k", "v")): + heads = ( + layer.total_num_heads + if shard_id == "q" + else layer.total_num_kv_heads + ) + dim = layer.v_head_size if shard_id == "v" else layer.head_size + weight = values(heads * dim, layer.input_size, layer, i) + partitions = size if shard_id == "q" else min(heads, size) + index = rank if shard_id == "q" else rank // max(size // heads, 1) + layer.weight.weight_loader(layer.weight, weight, shard_id) + weights.append(weight.chunk(partitions)[index]) + if layer.bias is not None: + bias = values(heads * dim, 1, layer, i + 4).flatten() + layer.bias.weight_loader(layer.bias, bias, shard_id) + biases.append(bias.chunk(partitions)[index]) + shard = torch.cat(weights) + if biases: + bias_shard = torch.cat(biases) + else: + weight = values(layer.output_size, layer.input_size, layer) + layer.weight.weight_loader(layer.weight, weight) + pieces = weight.split(getattr(layer, "output_sizes", [layer.output_size])) + shard = ( + weight.chunk(size, dim=1)[rank] + if row + else torch.cat([p.chunk(size)[rank] for p in pieces]) + ) + if layer.bias is not None: + bias = values(layer.output_size, 1, layer, 5).flatten() + layer.bias.weight_loader(layer.bias, bias) + bias_shard = bias if row else bias.chunk(size)[rank] + return shard, bias_shard + + +class TestAttentionProjectionParallelGroups(CustomTestCase): + def setUp(self): + reset_context() + self.addCleanup(reset_context) + + def test_projection_reload_math_and_collective_policy(self): + for dp_size in (1, 2): + for rank in (0, 3): + reset_context() + server = ServerArgs( + model_path="dummy", device="cpu", tp_size=4, attn_dp_size=dp_size + ) + publish(server, role="test", ranks=SpawnRanks(world_rank=rank)) + initialize_dp_attention_flags(server) + tp, attn = Mock(), Mock() + tp.all_reduce.side_effect = lambda x: x * 4 + attn.all_reduce.side_effect = lambda x: x * (4 // dp_size) + tp.all_gather.side_effect = lambda x, dim: torch.cat([x] * 4, dim=dim) + attn.all_gather.side_effect = lambda x, dim: torch.cat( + [x] * (4 // dp_size), dim=dim + ) + for model, variant in ATTENTION_CASES: + with self.subTest( + dp=dp_size, rank=rank, model=model, variant=variant + ): + module, names = build_attention(model, variant) + for name in names: + layer = getattr(module, name) + shard, bias = load_projection(layer) + torch.testing.assert_close(layer.weight, shard) + if bias is not None: + torch.testing.assert_close(layer.bias, bias) + row = isinstance(layer, RowParallelLinear) + size = ( + 1 + if model == "step3" and name == "qkv_proj" + else 4 // dp_size + ) + if model == "mtp" and dp_size == 1: + size = 4 + self.assertEqual( + (layer.tp_rank, layer.tp_size), (rank % size, size) + ) + inputs = values(2, shard.shape[1], layer, 7) + used_bias = bias if not row or layer.tp_rank == 0 else None + expected = F.linear(inputs, shard, used_bias) + if row and layer.reduce_results: + expected *= 4 + if not row and layer.gather_output: + expected = torch.cat([expected] * 4, dim=-1) + for group in (tp, attn): + group.all_reduce.reset_mock() + group.all_gather.reset_mock() + with ( + get_parallel().override( + tp_group=tp, attn_tp_group=attn + ), + patch( + "sglang.srt.layers.linear.is_allocation_symmetric", + return_value=True, + ), + patch( + "sglang.srt.layers.linear.use_symmetric_memory", + return_value=nullcontext(), + ) as allocator, + ): + actual = layer(inputs)[0] + torch.testing.assert_close(actual, expected) + if row: + allocator.assert_called_once_with(tp, disabled=False) + self.assertFalse(layer.use_dp_attention_reduce) + self.assertEqual( + layer.reduce_results, model == "sdar" and variant + ) + else: + allocator.assert_not_called() + self.assertEqual( + tp.all_reduce.call_count, + int(row and layer.reduce_results), + ) + self.assertEqual( + tp.all_gather.call_count, + int(not row and layer.gather_output), + ) + self.assertEqual( + attn.all_reduce.call_count + attn.all_gather.call_count, + 0, + ) + + def test_disabled_mtp_start_projections(self): + publish(ServerArgs(model_path="dummy", device="cpu"), role="test") + for variant in ("attention", "moe"): + module, names = build_attention("mtp", variant, start=False) + self.assertFalse(names) + self.assertFalse(hasattr(module, "eh_proj")) + self.assertTrue(hasattr(module, "final_layernorm")) + + def test_xllm_keeps_quantization_rejection(self): + from sglang.srt.layers.quantization.fp8 import Fp8Config + + publish(ServerArgs(model_path="dummy", device="cpu"), role="test") + for variant in ("gated", "mova"): + with self.assertRaisesRegex(ValueError, "unquantized bf16/fp16"): + build_attention("xllm", variant, quant_config=Fp8Config()) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/unit/layers/test_custom_projection_parallel_groups.py b/test/registered/unit/layers/test_custom_projection_parallel_groups.py new file mode 100644 index 0000000000..54caa754d8 --- /dev/null +++ b/test/registered/unit/layers/test_custom_projection_parallel_groups.py @@ -0,0 +1,337 @@ +"""Custom projections retain frozen checkpoint shards and native math.""" + +import inspect +import unittest +from contextlib import nullcontext +from unittest.mock import Mock, patch + +import torch +import torch.nn.functional as F + +from sglang.srt.layers import linear, mova +from sglang.srt.models.inkling_common import dense_mlp +from sglang.srt.models.inkling_common.attn import InklingAttention +from sglang.srt.models.inkling_common.moe import _build_inkling_shared_experts +from sglang.srt.runtime_context import SpawnRanks, get_parallel, publish, reset_context +from sglang.srt.server_args import ServerArgs +from sglang.test.ci.ci_register import register_cpu_ci +from sglang.test.test_utils import CustomTestCase + +register_cpu_ci(est_time=15, suite="base-a-test-cpu") + + +def values(shape, offset=0, *, device="cpu", dtype=torch.float32): + count = 1 + for size in shape: + count *= size + return ( + (((torch.arange(count, device=device) + offset) % 29 - 14) / 128) + .reshape(shape) + .to(dtype) + ) + + +def build_attention(kv_heads, *, width=32, head_dim=8, quant_config=None, bias=False): + return InklingAttention( + width, + 8, + kv_heads, + head_dim, + head_dim, + 8, + 4, + 1e-6, + False, + 0, + q_bias=bias, + o_bias=bias, + quant_config=quant_config, + ) + + +def load_projection(layer): + weights, biases = [], [] + r, s = layer.tp_rank, layer.tp_size + row = isinstance(layer, linear.RowParallelLinear) + with get_parallel().override(tp_rank=0, attn_tp_rank=0, attn_dp_rank=0): + if row: + full = values( + (layer.output_size, layer.input_size), + device=layer.weight.device, + dtype=layer.weight.dtype, + ) + layer.weight.weight_loader(layer.weight, full) + weights.append(full.chunk(s, dim=1)[r]) + if layer.bias is not None: + full = values( + (layer.output_size,), + 4, + device=layer.bias.device, + dtype=layer.bias.dtype, + ) + layer.bias.weight_loader(layer.bias, full) + biases.append(full) + else: + for i, size in enumerate(layer.output_sizes): + full = values( + (size, layer.input_size), + i, + device=layer.weight.device, + dtype=layer.weight.dtype, + ) + layer.weight.weight_loader(layer.weight, full, i) + weights.append(full.chunk(s)[r]) + if layer.bias is not None: + full = values( + (size,), i + 4, device=layer.bias.device, dtype=layer.bias.dtype + ) + layer.bias.weight_loader(layer.bias, full, i) + biases.append(full.chunk(s)[r]) + return weights[0] if row else torch.cat(weights), ( + biases[0] if row else torch.cat(biases) + ) if biases else None + + +def build_batch(group=None, *, width=16, linearized=False, execution_group=None): + kwargs = {} + if group is not None: + if ( + "parallel_group" + in inspect.signature(dense_mlp.InklingBatchDenseMLP).parameters + ): + kwargs["parallel_group"] = group + else: + parallel = get_parallel() + rank, size = ( + (0, 1) + if group == "replicated" + else (parallel.attn_tp_rank, parallel.attn_tp_size) + if group == "attn_tp" + else (parallel.tp_rank, parallel.tp_size) + ) + kwargs.update(tp_rank=rank, tp_size=size) + return dense_mlp.InklingBatchDenseMLP( + 2, + width, + 2 * width, + 0, + "shared", + tp_group=execution_group, + linearized_bf16=linearized, + **kwargs, + ) + + +def load_batch(module): + device, dtype = module.w13_weight.device, module.w13_weight.dtype + full_f = module.intermediate_size_per_partition * module.moe_tp_size + full = { + "w1": values((2, full_f, module.hidden_size), 0, device=device, dtype=dtype), + "w3": values((2, full_f, module.hidden_size), 1, device=device, dtype=dtype), + "w2": values((2, module.hidden_size, full_f), 2, device=device, dtype=dtype), + } + full13 = torch.stack([full["w1"], full["w3"]], dim=2).flatten(1, 2) + with get_parallel().override(tp_rank=0, attn_tp_rank=0, attn_dp_rank=0): + module.weight_loader_fused(module.w13_weight, full13, "w13.weight", "w13") + module.weight_loader_fused(module.w2_weight, full["w2"], "w2.weight", "w2") + shards = { + name: weight.chunk(module.moe_tp_size, dim=2 if name == "w2" else 1)[ + module.moe_tp_rank + ] + for name, weight in full.items() + } + expected13 = torch.stack([shards["w1"], shards["w3"]], dim=2).flatten(1, 2) + torch.testing.assert_close(module.w13_weight, expected13) + torch.testing.assert_close(module.w2_weight, shards["w2"]) + return shards + + +def batch_reference(x, gamma, shards): + outputs = [] + for i in range(gamma.shape[-1]): + gate = F.linear(x, shards["w1"][i]) + up = F.linear(x, shards["w3"][i]) + activation = ( + F.silu(gate.float()) * up.float() * gamma[:, i : i + 1].float() + ).to(x.dtype) + outputs.append(F.linear(activation, shards["w2"][i])) + return torch.stack(outputs).float().sum(0).to(x.dtype) + + +def build_values(group="attn_tp", *, width=16): + kwargs = dict(parallel_group=group) + if "parallel_group" not in inspect.signature(mova.RoutedValueExperts).parameters: + parallel = get_parallel() + rank, size = ( + (0, 1) + if group == "replicated" + else (parallel.attn_tp_rank, parallel.attn_tp_size) + if group == "attn_tp" + else (parallel.tp_rank, parallel.tp_size) + ) + kwargs = dict(tp_rank=rank, tp_size=size) + return mova.RoutedValueExperts(2, width, width, **kwargs) + + +def load_values(module, rank, size): + full = values( + (2, module.output_size, module.input_size), + device=module.weight.device, + dtype=module.weight.dtype, + ) + with get_parallel().override(tp_rank=0, attn_tp_rank=0, attn_dp_rank=0): + module.weight_loader(module.weight, full) + expected = full.chunk(size, dim=1)[rank] + torch.testing.assert_close(module.weight, expected) + return expected + + +class TestCustomProjectionParallelGroups(CustomTestCase): + def setUp(self): + reset_context() + self.addCleanup(reset_context) + + def test_native_custom_loaders_and_math_after_construction_scope(self): + for dp in (1, 2): + for rank in range(4): + reset_context() + publish( + ServerArgs( + model_path="dummy", device="cpu", tp_size=4, attn_dp_size=dp + ), + role="test", + ranks=SpawnRanks(world_rank=rank), + ) + for kv_heads in (1, 4): + module = build_attention(kv_heads, bias=True) + self.assertEqual(module.qkvr.inkling_tp_size, 4 // dp) + self.assertEqual(module.qkvr.inkling_num_kv_heads, kv_heads) + for name in ("qkvr", "wo_ud"): + layer = getattr(module, name) + expected, bias = load_projection(layer) + torch.testing.assert_close(layer.weight, expected) + x = values((2, expected.shape[1])) + used_bias = ( + bias if name == "qkvr" or layer.tp_rank == 0 else None + ) + with ( + get_parallel().override( + tp_group=Mock(), attn_tp_group=Mock() + ), + patch.object( + linear, + "use_symmetric_memory", + return_value=nullcontext(), + ), + ): + actual = layer(x)[0] + torch.testing.assert_close( + actual, F.linear(x, expected, used_bias) + ) + for group, r, s in ( + (None, 0, 1), + ("replicated", 0, 1), + ("tp", rank, 4), + ("attn_tp", rank % (4 // dp), 4 // dp), + ): + for linearized in (False, True): + module = build_batch(group, linearized=linearized) + self.assertEqual( + (module.moe_tp_rank, module.moe_tp_size), (r, s) + ) + shards = load_batch(module) + x, gamma = values((2, 16)), values((2, 2), 8) + 1 + eager_sum = getattr( + dense_mlp._sum_dim0, + "_torchdynamo_orig_callable", + dense_mlp._sum_dim0, + ) + + def swiglu(y, g): + return ( + F.silu(y[..., ::2].float()) + * y[..., 1::2].float() + * g.unsqueeze(-1).float() + ).to(y.dtype) + + with ( + patch.object(module, "_swiglu", swiglu), + patch.object(dense_mlp, "_sum_dim0", eager_sum), + ): + actual = module(x, gamma) + torch.testing.assert_close( + actual, batch_reference(x, gamma, shards) + ) + self.assertEqual(module.moe_runner_config.moe_tp_rank, r) + self.assertEqual(module.helper.moe_tp_rank, r) + self.assertEqual(module.helper.moe_tp_size, s) + value_module = build_values(group or "replicated") + expected = load_values(value_module, r, s) + selected = torch.tensor([[0, 1], [1, 0]], dtype=torch.int32) + gamma = values((2, 2), 9) + 1 + x = values((2, 16)) + actual = value_module(x, gamma, selected) + reference = torch.stack( + [ + sum( + F.silu(F.linear(x[t], expected[e])) * gamma[t, i] + for i, e in enumerate(selected[t].tolist()) + ) + for t in range(2) + ] + ) + torch.testing.assert_close(actual, reference) + + def test_shared_factory_keeps_full_tp_and_execution_handle(self): + publish( + ServerArgs(model_path="dummy", device="cpu", tp_size=4, attn_dp_size=2), + role="test", + ranks=SpawnRanks(world_rank=3), + ) + execution = Mock() + with ( + get_parallel().override(tp_group=execution), + patch( + "sglang.srt.models.inkling_common.moe.use_inkling_shared_fused_moe", + return_value=False, + ), + ): + module = _build_inkling_shared_experts( + n_shared_experts=2, + shared_expert_sink=True, + shared_experts_size=1, + inference_moe_w13_interleaved=True, + hidden_size=16, + intermediate_size=32, + layer_id=0, + prefix="model.layers.0", + ) + self.assertEqual((module.moe_tp_rank, module.moe_tp_size), (3, 4)) + self.assertIs(module.tp_group, execution) + shards = load_batch(module) + x, gamma = values((2, 16)), values((2, 2), 8) + 1 + eager_sum = getattr( + dense_mlp._sum_dim0, "_torchdynamo_orig_callable", dense_mlp._sum_dim0 + ) + + def swiglu(y, g): + return ( + F.silu(y[..., ::2].float()) + * y[..., 1::2].float() + * g.unsqueeze(-1).float() + ).to(y.dtype) + + with ( + patch.object(module, "_swiglu", swiglu), + patch.object(dense_mlp, "_sum_dim0", eager_sum), + patch.object( + dense_mlp, "symm_mem_all_reduce", side_effect=lambda v, group: v * 4 + ) as reduction, + ): + actual = module(x, gamma) + torch.testing.assert_close(actual, batch_reference(x, gamma, shards) * 4) + self.assertIs(reduction.call_args.args[1], execution) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/unit/layers/test_kimi_attention_parallel_groups.py b/test/registered/unit/layers/test_kimi_attention_parallel_groups.py new file mode 100644 index 0000000000..85c3d0b1da --- /dev/null +++ b/test/registered/unit/layers/test_kimi_attention_parallel_groups.py @@ -0,0 +1,227 @@ +"""Kimi attention projections retain checkpoint layout and reduction policy.""" + +import unittest +from contextlib import nullcontext +from types import SimpleNamespace +from unittest.mock import Mock, patch + +import torch +import torch.nn.functional as F + +from sglang.srt.layers.dp_attention import initialize_dp_attention_flags +from sglang.srt.layers.linear import QKVParallelLinear, RowParallelLinear +from sglang.srt.models.kimi_k3 import KimiK3DeltaAttention, KimiK3MLAAttention +from sglang.srt.runtime_context import SpawnRanks, get_parallel, publish, reset_context +from sglang.srt.server_args import ServerArgs +from sglang.test.ci.ci_register import register_cpu_ci +from sglang.test.test_utils import CustomTestCase + +register_cpu_ci(est_time=15, suite="base-a-test-cpu") + +KIMI_CASES = tuple((kind, gate) for kind in ("delta", "mla") for gate in (False, True)) + + +def build_kimi( + kind, + gate, + fusion=False, + fallback=False, + width=32, + head_dim=8, + heads=8, + quant_config=None, +): + config = SimpleNamespace( + hidden_size=width, + num_attention_heads=heads, + qk_nope_head_dim=head_dim - 4, + qk_rope_head_dim=4, + v_head_dim=head_dim, + q_lora_rank=width, + kv_lora_rank=width, + rms_norm_eps=1e-6, + max_position_embeddings=16, + dtype=torch.get_default_dtype(), + mla_use_output_gate=gate, + linear_attn_config={ + "head_dim": head_dim, + "num_heads": heads, + "short_conv_kernel_size": 3, + "use_full_rank_gate": gate, + }, + ) + with ( + patch("sglang.srt.models.kimi_k3._o_proj_takes_output", return_value=False) + if fallback + else nullcontext() + ): + if kind == "mla": + module = KimiK3MLAAttention( + config, 0, quant_config=quant_config, all_reduce_fusion=fusion + ) + names = ("g_proj",) if gate else () + else: + module = KimiK3DeltaAttention( + 0, width, config, quant_config=quant_config, all_reduce_fusion=fusion + ) + if gate: + names = ( + "fused_qkvg_proj", + "b_proj", + "f_b_proj", + "qkv_conv1d", + "o_proj", + ) + elif module.do_fuse_qkvbfg: + # The full-TP repeated/batched helpers keep their existing interface. + names = ("qkv_conv1d", "o_proj") + else: + names = ( + "qkv_proj", + "b_proj", + "f_b_proj", + "g_b_proj", + "qkv_conv1d", + "o_proj", + ) + return module, names + + +def values(rows, columns, layer, offset=0): + return ( + (torch.arange(rows * columns, device=layer.weight.device) + offset) % 23 - 11 + ).reshape(rows, columns).to(layer.weight.dtype) / 128 + + +def load_projection(layer): + rank, size = layer.tp_rank, layer.tp_size + row = isinstance(layer, RowParallelLinear) + with get_parallel().override(tp_rank=0, attn_tp_rank=0, attn_dp_rank=0): + if isinstance(layer, QKVParallelLinear): + shards = [] + for i, shard_id in enumerate(("q", "k", "v")): + heads = ( + layer.total_num_heads + if shard_id == "q" + else layer.total_num_kv_heads + ) + dim = layer.v_head_size if shard_id == "v" else layer.head_size + weight = values(heads * dim, layer.input_size, layer, i) + layer.weight.weight_loader(layer.weight, weight, shard_id) + shards.append(weight.chunk(size)[rank]) + shard = torch.cat(shards) + else: + weight = values(layer.output_size, layer.input_size, layer) + if layer.weight.ndim == 3: + weight = weight.unsqueeze(1) + layer.weight.weight_loader(layer.weight, weight) + pieces = weight.split(getattr(layer, "output_sizes", [layer.output_size])) + shard = ( + weight.chunk(size, dim=1)[rank] + if row + else torch.cat([p.chunk(size)[rank] for p in pieces]) + ) + return shard + + +class TestKimiAttentionParallelGroups(CustomTestCase): + def setUp(self): + reset_context() + self.addCleanup(reset_context) + + def test_projection_reload_math_and_policy(self): + for dp in (1, 2): + for rank in (0, 3): + reset_context() + server = ServerArgs( + model_path="dummy", device="cpu", tp_size=4, attn_dp_size=dp + ) + publish(server, role="test", ranks=SpawnRanks(world_rank=rank)) + initialize_dp_attention_flags(server) + tp, attn = Mock(), Mock() + tp.all_reduce.side_effect = lambda x: x * 4 + attn.all_reduce.side_effect = lambda x: x * (4 // dp) + for kind, gate in KIMI_CASES: + for fusion in (False, True): + for fallback in (False, True) if fusion else (False,): + with self.subTest( + dp=dp, + rank=rank, + kind=kind, + gate=gate, + fusion=fusion, + fallback=fallback, + ): + module, names = build_kimi(kind, gate, fusion, fallback) + active_fusion = fusion and not fallback + self.assertEqual( + module.all_reduce_fusion, active_fusion + ) + self.assertEqual( + module.o_proj.reduce_results, not active_fusion + ) + self.assertEqual( + module.o_proj.use_dp_attention_reduce, + not active_fusion, + ) + for name in names: + layer = getattr(module, name) + shard = load_projection(layer) + torch.testing.assert_close(layer.weight, shard) + self.assertEqual( + (layer.tp_rank, layer.tp_size), + (rank % (4 // dp), 4 // dp), + ) + if name == "qkv_conv1d": + inputs = values( + 2 * shard.shape[0], 6, layer, 7 + ).reshape(2, shard.shape[0], 6) + actual = F.conv1d( + inputs, layer.weight, groups=shard.shape[0] + ) + expected = ( + inputs.unfold(-1, 3, 1) + * shard.squeeze(1)[None, :, None, :] + ).sum(-1) + torch.testing.assert_close(actual, expected) + continue + row = isinstance(layer, RowParallelLinear) + inputs = values(2, shard.shape[1], layer, 7) + expected = F.linear(inputs, shard) + if row and layer.reduce_results: + expected *= 4 // dp + for group in (tp, attn): + group.all_reduce.reset_mock() + with ( + get_parallel().override( + tp_group=tp, attn_tp_group=attn + ), + patch( + "sglang.srt.layers.linear.is_allocation_symmetric", + return_value=True, + ), + patch( + "sglang.srt.layers.linear.use_symmetric_memory", + return_value=nullcontext(), + ) as allocation, + ): + actual = layer(inputs)[0] + torch.testing.assert_close(actual, expected) + if row: + if layer.use_dp_attention_reduce: + allocation.assert_called_once_with(attn) + else: + allocation.assert_called_once_with( + tp, disabled=False + ) + else: + allocation.assert_not_called() + self.assertEqual(tp.all_reduce.call_count, 0) + self.assertEqual( + attn.all_reduce.call_count, + int(row and layer.reduce_results), + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/unit/layers/test_kimi_mlp_parallel_groups.py b/test/registered/unit/layers/test_kimi_mlp_parallel_groups.py new file mode 100644 index 0000000000..a7f8d5cd72 --- /dev/null +++ b/test/registered/unit/layers/test_kimi_mlp_parallel_groups.py @@ -0,0 +1,279 @@ +"""Kimi dense and shared MLPs freeze placement without changing row policy.""" + +import inspect +import unittest +from contextlib import nullcontext +from types import SimpleNamespace +from unittest.mock import Mock, patch + +import torch +import torch.nn.functional as F + +from sglang.srt.layers import linear +from sglang.srt.layers.dp_attention import initialize_dp_attention_flags +from sglang.srt.models import kimi_k3 +from sglang.srt.runtime_context import SpawnRanks, get_parallel, publish, reset_context +from sglang.srt.server_args import ServerArgs +from sglang.test.ci.ci_register import register_cpu_ci +from sglang.test.test_utils import CustomTestCase + +register_cpu_ci(est_time=15, suite="base-a-test-cpu") + + +def build_mlp(group=None, *, width=16, quant_config=None, reduce=True): + kwargs = {} + if group is not None: + if "parallel_group" in inspect.signature(kimi_k3.KimiK3MLP).parameters: + kwargs["parallel_group"] = group + else: + parallel = get_parallel() + if group == "replicated": + rank, size = 0, 1 + elif group == "shared_experts_tp": + rank = parallel.shared_experts_tp_group.rank_in_group + size = parallel.shared_experts_tp_group.world_size + elif group == "attn_tp": + rank, size = parallel.attn_tp_rank, parallel.attn_tp_size + else: + rank, size = parallel.tp_rank, parallel.tp_size + kwargs.update(tp_rank=rank, tp_size=size) + return kimi_k3.KimiK3MLP( + width, + 2 * width, + "silu", + quant_config=quant_config, + reduce_results=reduce, + **kwargs, + ) + + +class _RoutedExperts(torch.nn.Module): + should_fuse_routed_scaling_factor_in_topk = False + + def __init__(self, **kwargs): + super().__init__() + + +class _Gate(torch.nn.Module): + def __init__(self, *args, **kwargs): + super().__init__() + self.e_score_correction_bias = None + + +def build_shared_experts(*, ep_a2a=True, width=16, shared=1, quant_config=None): + # Routed experts and their router do not participate in this placement + # test. The real MoE constructor still builds its real shared MLP. + backend = SimpleNamespace( + is_megamoe=lambda: False, + is_deepep=lambda: ep_a2a, + is_mooncake=lambda: False, + is_ascend_fuseep=lambda: False, + is_mori=lambda: False, + ) + config = SimpleNamespace( + hidden_size=width, + moe_intermediate_size=2 * width, + moe_renormalize=True, + routed_scaling_factor=1.0, + num_shared_experts=shared, + routed_expert_hidden_size=None, + num_experts=4, + num_experts_per_token=1, + hidden_act="silu", + activation_situ_beta=None, + activation_situ_linear_beta=None, + num_expert_group=1, + topk_group=1, + moe_router_activation_func="sigmoid", + ) + with ( + patch.object(kimi_k3, "MoEGate", _Gate), + patch.object(kimi_k3, "get_moe_impl_class", return_value=_RoutedExperts), + patch.object(kimi_k3, "get_moe_a2a_backend", return_value=backend), + ): + module = kimi_k3.KimiK3MoE(config, quant_config=quant_config) + return module, module.shared_experts + + +def load_projection(layer): + full = ( + torch.arange(layer.output_size * layer.input_size, device=layer.weight.device) + % 29 + - 14 + ).reshape(layer.output_size, layer.input_size).to(layer.weight.dtype) / 128 + rank, size = layer.tp_rank, layer.tp_size + with get_parallel().override( + tp_rank=0, attn_tp_rank=0, attn_dp_rank=0, shared_experts_tp_group=None + ): + layer.weight.weight_loader(layer.weight, full) + if isinstance(layer, linear.RowParallelLinear): + expected = full.chunk(size, dim=1)[rank] + else: + expected = torch.cat( + [v.chunk(size)[rank] for v in full.split(layer.output_sizes)] + ) + return expected, None + + +class TestKimiMLPParallelGroups(CustomTestCase): + def setUp(self): + reset_context() + self.addCleanup(reset_context) + + def test_dense_and_explicit_group_layout_and_row_policy(self): + for dp in (1, 2): + for dense_attn in (False, True): + for rank in range(4): + reset_context() + server = ServerArgs( + model_path="dummy", + device="cpu", + tp_size=4, + attn_dp_size=dp, + enable_dense_mlp_attn_tp=dense_attn, + ) + publish(server, role="test", ranks=SpawnRanks(world_rank=rank)) + initialize_dp_attention_flags(server) + shared = SimpleNamespace(rank_in_group=rank % 2, world_size=2) + with get_parallel().override(shared_experts_tp_group=shared): + for group in ( + None, + "tp", + "attn_tp", + "replicated", + "shared_experts_tp", + ): + for reduce in (False, True): + module = build_mlp(group, reduce=reduce) + auto_attn = group is None and dense_attn and dp > 1 + r, s = ( + (0, 1) + if group == "replicated" + else (rank % 2, 2) + if group == "shared_experts_tp" + else (rank % (4 // dp), 4 // dp) + if group == "attn_tp" or auto_attn + else (rank, 4) + ) + self.assertEqual(module._dense_attn_tp, auto_attn) + for name in ("gate_up_proj", "down_proj"): + layer = getattr(module, name) + self.assertEqual( + (layer.tp_rank, layer.tp_size), (r, s) + ) + expected, _ = load_projection(layer) + torch.testing.assert_close(layer.weight, expected) + x = ( + ( + torch.arange(2 * expected.shape[1]).reshape( + 2, -1 + ) + % 17 + - 8 + ) + / 128 + ).to(layer.weight.dtype) + tp, attn = Mock(), Mock() + tp.all_reduce.side_effect = lambda v: v * 4 + attn.all_reduce.side_effect = lambda v: ( + v * (4 // dp) + ) + row = name == "down_proj" + reference = F.linear(x, expected) + if row and reduce and s > 1: + reference *= (4 // dp) if auto_attn else 4 + with ( + get_parallel().override( + tp_group=tp, attn_tp_group=attn + ), + patch.object( + linear, + "use_symmetric_memory", + return_value=nullcontext(), + ) as allocation, + ): + actual = layer(x)[0] + torch.testing.assert_close(actual, reference) + if row: + self.assertEqual(layer.reduce_results, reduce) + self.assertEqual( + layer.use_dp_attention_reduce, auto_attn + ) + self.assertIs( + allocation.call_args.args[0], + attn if auto_attn else tp, + ) + else: + allocation.assert_not_called() + + def test_moe_shared_consumer_preserves_selection_and_execution_handle(self): + for rank in range(4): + reset_context() + server = ServerArgs( + model_path="dummy", device="cpu", tp_size=4, attn_dp_size=1 + ) + publish(server, role="test", ranks=SpawnRanks(world_rank=rank)) + initialize_dp_attention_flags(server) + attn = SimpleNamespace(world_size=4, rank_in_group=rank) + shared = SimpleNamespace(world_size=2, rank_in_group=rank % 2) + for ep, requested, enabled, expected_size in ( + (False, None, False, 4), + (True, None, False, 1), + (True, 1, False, 1), + (True, 2, False, 2), + (True, None, True, 4), + ): + with get_parallel().override( + shared_experts_tp_size=requested, + enable_shared_experts_attn_tp=enabled, + shared_experts_tp_group=shared, + attn_tp_group=attn, + ): + owner, module = build_shared_experts(ep_a2a=ep) + comm = ep and expected_size > 1 + self.assertEqual(owner._shared_experts_tp_comm, comm) + self.assertEqual( + owner._shared_experts_tp1, ep and expected_size == 1 + ) + self.assertIs( + owner._shared_experts_tp_group, + shared if requested == 2 else attn if comm else None, + ) + self.assertFalse(module._dense_attn_tp) + for name in ("gate_up_proj", "down_proj"): + layer = getattr(module, name) + self.assertEqual( + (layer.tp_rank, layer.tp_size), + (rank % expected_size, expected_size), + ) + expected, _ = load_projection(layer) + torch.testing.assert_close(layer.weight, expected) + self.assertFalse(module.down_proj.reduce_results) + self.assertFalse(module.down_proj.use_dp_attention_reduce) + with get_parallel().override(shared_experts_tp_size=2): + with self.assertRaisesRegex(ValueError, "requires an EP a2a"): + build_shared_experts(ep_a2a=False) + with get_parallel().override( + shared_experts_tp_size=3, + shared_experts_tp_group=SimpleNamespace( + world_size=3, rank_in_group=rank % 3 + ), + ): + with self.assertRaisesRegex(ValueError, "must be divisible"): + build_shared_experts() + _, disabled = build_shared_experts(shared=0) + self.assertIsNone(disabled) + + def test_shared_selector_requires_a_built_group(self): + publish( + ServerArgs(model_path="dummy", device="cpu"), + role="test", + ranks=SpawnRanks(world_rank=0), + ) + with get_parallel().override(shared_experts_tp_group=None): + with self.assertRaisesRegex(ValueError, "must exist before construction"): + linear.resolve_linear_parallel_group("shared_experts_tp") + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/unit/layers/test_linear_attention_parallel_groups.py b/test/registered/unit/layers/test_linear_attention_parallel_groups.py new file mode 100644 index 0000000000..f2ccebbbb3 --- /dev/null +++ b/test/registered/unit/layers/test_linear_attention_parallel_groups.py @@ -0,0 +1,329 @@ +"""Linear attention projections retain checkpoint layout across reload scopes.""" + +import unittest +from contextlib import nullcontext +from types import SimpleNamespace +from unittest.mock import Mock, patch + +import torch +import torch.nn.functional as F + +from sglang.srt.layers.dp_attention import initialize_dp_attention_flags +from sglang.srt.layers.linear import RowParallelLinear +from sglang.srt.runtime_context import SpawnRanks, get_parallel, publish, reset_context +from sglang.srt.server_args import ServerArgs +from sglang.srt.utils import is_cpu +from sglang.test.ci.ci_register import register_cpu_ci +from sglang.test.test_utils import CustomTestCase + +register_cpu_ci(est_time=15, suite="base-a-test-cpu") + + +def build_linear_attention( + model, + n_groups=8, + reduce_results=None, + width=32, + head_dim=8, + quant_config=None, + heads=8, + device="cpu", +): + if model == "mamba": + from sglang.srt.configs.mamba_utils import Mamba2StateShape + from sglang.srt.layers.attention.mamba.mamba import MambaMixer2 + + size = ( + get_parallel().attn_tp_size + if get_parallel().attn_dp_size > 1 + else get_parallel().tp_size + ) + shape = Mamba2StateShape.create( + tp_world_size=size, + intermediate_size=heads * head_dim, + n_groups=n_groups, + num_heads=heads, + head_dim=head_dim, + state_size=head_dim, + conv_kernel=3, + ) + module = MambaMixer2( + SimpleNamespace(shape=shape), + width, + use_conv_bias=True, + use_bias=True, + n_groups=n_groups, + quant_config=quant_config, + reduce_results=reduce_results, + ) + names = ("conv1d", "in_proj", "out_proj") + else: + config = SimpleNamespace( + hidden_size=width, + linear_num_value_heads=heads, + linear_num_key_heads=heads, + linear_num_value_heads_cpu=heads, + linear_num_key_heads_cpu=heads, + linear_key_head_dim=head_dim, + linear_value_head_dim=head_dim, + linear_conv_kernel_dim=3, + hidden_act="silu", + output_gate_type=None, + rms_norm_eps=1e-6, + torch_dtype=torch.get_default_dtype(), + ) + if model in ("qwen_next", "gigachat"): + from sglang.srt.models.qwen3_next import Qwen3GatedDeltaNet + + constructor = Qwen3GatedDeltaNet + if model == "gigachat": + from sglang.srt.models.gigachat35 import GigaChat35GatedDeltaNet + + constructor = GigaChat35GatedDeltaNet + else: + from sglang.srt.models.qwen3_5 import Qwen3_5GatedDeltaNet + + constructor = Qwen3_5GatedDeltaNet + with patch( + "torch.get_device_module", + return_value=SimpleNamespace(current_device=lambda: device), + ): + module = constructor(config, 0, quant_config=quant_config) + names = ("conv1d", "in_proj_qkvz", "in_proj_ba", "out_proj") + return module, names + + +def checkpoint_values(rows, columns, layer, offset=0): + return ( + (torch.arange(rows * columns, device=layer.weight.device) + offset) % 23 - 11 + ).reshape(rows, columns).to(layer.weight.dtype) / 128 + + +def load_projection(module, name, n_groups=8, checkpoint="fused"): + """Return an independently sliced checkpoint reference, including copied groups.""" + layer = getattr(module, name) + rank, size = layer.tp_rank, layer.tp_size + row = isinstance(layer, RowParallelLinear) + mamba = hasattr(module, "intermediate_size") + duplicate = mamba and n_groups == 1 and size > 1 + if name == "conv1d": + sizes = ( + [ + module.intermediate_size, + n_groups * module.ssm_state_size, + n_groups * module.ssm_state_size, + ] + if mamba + else [module.key_dim, module.key_dim, module.value_dim] + ) + elif mamba and name == "in_proj": + sizes = [ + module.intermediate_size, + module.intermediate_size, + n_groups * module.ssm_state_size, + n_groups * module.ssm_state_size, + module.num_heads, + ] + else: + sizes = getattr(layer, "output_sizes", [layer.output_size]) + weight = checkpoint_values(sum(sizes), layer.input_size, layer) + pieces = weight.split(sizes) + if row: + shard = weight.chunk(size, dim=1)[rank] + elif duplicate: + copied = (1, 2) if name == "conv1d" else (2, 3) + shard = torch.cat( + [ + piece if i in copied else piece.chunk(size)[rank] + for i, piece in enumerate(pieces) + ] + ) + elif ( + module.__class__.__name__ in ("Qwen3GatedDeltaNet", "GigaChat35GatedDeltaNet") + and name.startswith("in_proj") + and checkpoint == "fused" + ): + shard = weight.chunk(size)[rank] + else: + shard = torch.cat([piece.chunk(size)[rank] for piece in pieces]) + bias_shard = None + with get_parallel().override(tp_rank=0, attn_dp_rank=0, attn_tp_rank=0): + if name.startswith("in_proj") and not mamba and checkpoint != "fused": + if checkpoint == "tuple": + indices = tuple(range(len(sizes) - 1)) + layer.weight.weight_loader( + layer.weight, torch.cat(pieces[:-1]), indices + ) + layer.weight.weight_loader(layer.weight, pieces[-1], len(sizes) - 1) + else: + for i, piece in enumerate(pieces): + layer.weight.weight_loader(layer.weight, piece, i) + else: + loaded = weight.unsqueeze(1) if name == "conv1d" else weight + layer.weight.weight_loader(layer.weight, loaded) + if layer.bias is not None: + rows = sum(sizes) if name == "conv1d" else layer.output_size + bias = checkpoint_values(rows, 1, layer, 3).flatten() + layer.bias.weight_loader(layer.bias, bias) + if row: + bias_shard = bias + elif duplicate and name == "conv1d": + bias_shard = torch.cat( + [ + p if i in (1, 2) else p.chunk(size)[rank] + for i, p in enumerate(bias.split(sizes)) + ] + ) + elif duplicate: + bias_shard = bias.chunk(size)[rank] + else: + bias_shard = torch.cat([p.chunk(size)[rank] for p in bias.split(sizes)]) + return shard.unsqueeze(1) if name == "conv1d" else shard, bias_shard + + +class TestLinearAttentionParallelGroups(CustomTestCase): + def setUp(self): + reset_context() + self.addCleanup(reset_context) + + def test_checkpoint_reload_projection_math_and_row_policy(self): + # The existing CPU bias-padding loader cannot load duplicated Mamba groups. + variants = [ + (model, 8, None) for model in ("qwen_next", "qwen35", "gigachat") + ] + [ + ("mamba", groups, reduce) + for groups in ((8,) if is_cpu() else (1, 8)) + for reduce in (None, False, True) + ] + for dp_size in (1, 2): + for rank in (0, 3): + server = ServerArgs( + model_path="dummy", device="cpu", tp_size=4, attn_dp_size=dp_size + ) + reset_context() + publish(server, role="test", ranks=SpawnRanks(world_rank=rank)) + initialize_dp_attention_flags(server) + size = 4 // dp_size + tp, attn = Mock(), Mock() + tp.all_reduce.side_effect = lambda x: x * 4 + attn.all_reduce.side_effect = lambda x: x * size + for model, groups, reduce in variants: + with self.subTest( + dp=dp_size, rank=rank, model=model, groups=groups, reduce=reduce + ): + module, names = build_linear_attention(model, groups, reduce) + for name in names: + layer = getattr(module, name) + modes = ( + ( + ("fused", "split", "tuple") + if model == "qwen35" + else ("fused", "split") + ) + if model != "mamba" and name.startswith("in_proj") + else ("fused",) + ) + for checkpoint in modes: + shard, bias = load_projection( + module, name, groups, checkpoint + ) + torch.testing.assert_close(layer.weight, shard) + if bias is not None: + torch.testing.assert_close(layer.bias, bias) + self.assertEqual( + (layer.tp_rank, layer.tp_size), (rank % size, size) + ) + row = isinstance(layer, RowParallelLinear) + if name == "conv1d": + inputs = checkpoint_values( + 2 * shard.shape[0], 5, layer, 7 + ).reshape(2, shard.shape[0], 5) + actual = F.conv1d( + inputs, + layer.weight, + layer.bias, + groups=shard.shape[0], + ) + expected = ( + inputs.unfold(-1, 3, 1) + * shard.squeeze(1)[None, :, None, :] + ).sum(-1) + if bias is not None: + expected += bias[None, :, None] + torch.testing.assert_close(actual, expected) + continue + inputs = checkpoint_values(2, shard.shape[1], layer, 7) + used_bias = ( + bias if not row or layer.tp_rank == 0 else None + ) + expected = F.linear(inputs, shard, used_bias) + if row and layer.reduce_results: + expected *= ( + size if layer.use_dp_attention_reduce else 4 + ) + tp.all_reduce.reset_mock() + attn.all_reduce.reset_mock() + with ( + get_parallel().override( + tp_group=tp, attn_tp_group=attn + ), + patch( + "sglang.srt.layers.linear.is_allocation_symmetric", + return_value=True, + ), + patch( + "sglang.srt.layers.linear.use_symmetric_memory", + return_value=nullcontext(), + ) as allocator, + ): + actual = layer(inputs)[0] + torch.testing.assert_close(actual, expected) + if row: + group = ( + attn if layer.use_dp_attention_reduce else tp + ) + expected_kwargs = ( + {} + if layer.use_dp_attention_reduce + else {"disabled": False} + ) + allocator.assert_called_once_with( + group, **expected_kwargs + ) + self.assertEqual( + tp.all_reduce.call_count + + attn.all_reduce.call_count, + int(layer.reduce_results), + ) + self.assertEqual( + layer.reduce_results, + (dp_size == 1 if reduce is None else reduce) + if model == "mamba" + else False, + ) + self.assertEqual( + layer.use_dp_attention_reduce, + model == "mamba" + and layer.reduce_results + and dp_size > 1, + ) + else: + allocator.assert_not_called() + + def test_projection_helpers_default_to_full_tp(self): + publish( + ServerArgs(model_path="dummy", device="cpu", tp_size=4, attn_dp_size=2), + role="test", + ranks=SpawnRanks(world_rank=3), + ) + for model in ("qwen_next", "qwen35"): + module, _ = build_linear_attention(model) + projections = [module.create_qkvz_proj(32, 64, 64, None, "")] + if model == "qwen35": + projections.append(module.create_ba_proj(32, 8, None, "")) + for layer in projections: + self.assertEqual((layer.tp_rank, layer.tp_size), (3, 4)) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/unit/layers/test_llama_inkling_parallel_groups.py b/test/registered/unit/layers/test_llama_inkling_parallel_groups.py new file mode 100644 index 0000000000..a0e541b410 --- /dev/null +++ b/test/registered/unit/layers/test_llama_inkling_parallel_groups.py @@ -0,0 +1,222 @@ +"""Dense MLP placement stays frozen while reduction uses its own policy.""" + +import inspect +import unittest +from contextlib import nullcontext +from unittest.mock import Mock, patch + +import torch +import torch.nn.functional as F + +from sglang.srt.models.inkling_common import dense_mlp +from sglang.srt.models.llama import LlamaMLP +from sglang.srt.runtime_context import SpawnRanks, get_parallel, publish, reset_context +from sglang.srt.server_args import ServerArgs +from sglang.test.ci.ci_register import register_cpu_ci +from sglang.test.test_utils import CustomTestCase + +register_cpu_ci(est_time=10, suite="base-a-test-cpu") + + +def build_mlp( + model, + group=None, + *, + width=8, + reduce=True, + dp_reduce=False, + quant_config=None, + fused=False, + lora=False, + execution_group=None, +): + cls = LlamaMLP if model == "llama" else dense_mlp.InklingDenseMLP + kwargs = dict( + hidden_size=width, + intermediate_size=width, + quant_config=quant_config, + use_dp_attention_reduce=dp_reduce, + ) + if model == "llama": + kwargs.update(hidden_act="silu", reduce_results=reduce) + else: + kwargs.update( + use_global_scale=True, layer_id=0, fused=fused, tp_group=execution_group + ) + if group is not None: + if "parallel_group" in inspect.signature(cls).parameters: + kwargs["parallel_group"] = group + else: + parallel = get_parallel() + rank, size = ( + (0, 1) + if group == "replicated" + else (parallel.attn_tp_rank, parallel.attn_tp_size) + if group == "attn_tp" + else (parallel.tp_rank, parallel.tp_size) + ) + kwargs.update(tp_rank=rank, tp_size=size) + with patch.object(dense_mlp, "lora_compatible_layout_enabled", return_value=lora): + module = cls(**kwargs) + return module + + +def load_projection(layer): + row = hasattr(layer, "reduce_results") + shape = (layer.output_size, layer.input_size) + weight = ( + torch.arange(shape[0] * shape[1], device=layer.weight.device).reshape(shape) + % 23 + - 11 + ).to(layer.weight.dtype) / 64 + layer.weight.weight_loader(layer.weight, weight) + if row: + expected = weight.chunk(layer.tp_size, dim=1)[layer.tp_rank] + else: + expected = torch.cat( + [ + part.chunk(layer.tp_size)[layer.tp_rank] + for part in weight.split(layer.output_sizes) + ] + ) + torch.testing.assert_close(layer.weight, expected) + return expected + + +class TestLlamaInklingParallelGroups(CustomTestCase): + def setUp(self): + reset_context() + self.addCleanup(reset_context) + publish( + ServerArgs(model_path="dummy", device="cpu", tp_size=4, attn_dp_size=2), + role="test", + ranks=SpawnRanks(world_rank=3), + ) + + def test_frozen_loaders_forward_and_execution_policy(self): + tp, attn, execution = Mock(), Mock(), Mock() + tp.all_reduce.side_effect = lambda x: x * 2 + attn.all_reduce.side_effect = lambda x: x * 3 + x = torch.arange(16, dtype=torch.float32).reshape(2, 8) / 32 + for model in ("llama", "inkling"): + for group in (None, "tp", "attn_tp", "replicated"): + for reduce in (False, True): + for dp_reduce in (False, True): + with self.subTest( + model=model, group=group, reduce=reduce, dp_reduce=dp_reduce + ): + module = build_mlp( + model, + group, + reduce=reduce, + dp_reduce=dp_reduce, + execution_group=execution, + ) + up, down = module.gate_up_proj, module.down_proj + expected_group = group or ( + "tp" if model == "llama" else "replicated" + ) + rank, size = { + "tp": (3, 4), + "attn_tp": (1, 2), + "replicated": (0, 1), + }[expected_group] + with get_parallel().override( + tp_rank=0, attn_dp_rank=0, attn_tp_rank=0 + ): + a, b = load_projection(up), load_projection(down) + gate, other = F.linear(x, a).chunk(2, dim=-1) + expected = F.linear(F.silu(gate) * other, b) + row_reduce = reduce and model == "llama" + if row_reduce and size > 1: + expected *= 3 if dp_reduce else 2 + if model == "llama": + module.act_fn._forward_method = ( + module.act_fn.forward_native + ) + else: + module.global_scale.data.fill_(2) + expected *= 2 * 5 + module.act_fn.forward = dense_mlp.swiglu_contiguous._torchdynamo_orig_callable + tp.all_reduce.reset_mock() + attn.all_reduce.reset_mock() + with ( + get_parallel().override( + tp_group=tp, attn_tp_group=attn + ), + patch( + "sglang.srt.layers.linear.is_allocation_symmetric", + return_value=False, + ), + patch( + "sglang.srt.layers.linear.use_symmetric_memory", + return_value=nullcontext(), + ) as allocator, + patch.object( + dense_mlp, + "symm_mem_all_reduce", + side_effect=lambda y, g: y * 5, + ) as comm, + ): + actual = module(x) + torch.testing.assert_close(actual, expected) + allocator.assert_called_once_with( + attn + ) if dp_reduce else allocator.assert_called_once_with( + tp, disabled=True + ) + self.assertEqual( + tp.all_reduce.call_count, + int(row_reduce and size > 1 and not dp_reduce), + ) + self.assertEqual( + attn.all_reduce.call_count, + int(row_reduce and size > 1 and dp_reduce), + ) + if model == "inkling": + comm.assert_called_once_with( + unittest.mock.ANY, execution + ) + else: + comm.assert_not_called() + self.assertEqual((up.tp_rank, up.tp_size), (rank, size)) + self.assertEqual((down.tp_rank, down.tp_size), (rank, size)) + self.assertEqual(down.reduce_results, row_reduce) + self.assertEqual(down.use_dp_attention_reduce, dp_reduce) + + def test_inkling_lora_layout_and_scattered_execution(self): + x = torch.arange(16, dtype=torch.float32).reshape(2, 8) / 32 + execution = Mock() + for lora in (False, True): + module = build_mlp( + "inkling", "attn_tp", fused=True, lora=lora, execution_group=execution + ) + self.assertEqual(module.act_fn.interleaved, not lora) + up, down = module.gate_up_proj, module.down_proj + a, b = load_projection(up), load_projection(down) + z = F.linear(x, a) + gate, other = z.chunk(2, dim=-1) if lora else (z[..., ::2], z[..., 1::2]) + expected = F.linear(F.silu(gate) * other, b) + module.global_scale.data.fill_(1) + function = dense_mlp.swiglu_contiguous if lora else dense_mlp.swiglu + module.act_fn.forward = function._torchdynamo_orig_callable + module.scattered_sconv = True + with ( + get_parallel().override(tp_group=Mock()), + patch( + "sglang.srt.layers.linear.use_symmetric_memory", + return_value=nullcontext(), + ), + patch.object( + dense_mlp, "reduce_scatter_hidden", side_effect=lambda y, g: y + ) as scatter, + ): + torch.testing.assert_close(module(x), expected) + scatter.assert_called_once_with(unittest.mock.ANY, execution) + scatter.reset_mock() + torch.testing.assert_close(module(x, use_reduce_scatter=True), expected) + scatter.assert_not_called() + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/unit/layers/test_mova.py b/test/registered/unit/layers/test_mova.py index 72d6b35848..802d80bdde 100644 --- a/test/registered/unit/layers/test_mova.py +++ b/test/registered/unit/layers/test_mova.py @@ -19,6 +19,7 @@ from sglang.srt.layers.mova import ( mova_router_topk, routed_linear, ) +from sglang.srt.runtime_context import get_parallel from sglang.test.ci.ci_register import register_cpu_ci register_cpu_ci(est_time=4, suite="base-a-test-cpu") @@ -97,13 +98,28 @@ def test_routed_linear_cpu_matches_independent_expected_math(): def test_value_expert_loader_shards_output_dimension(): - experts = RoutedValueExperts( - num_experts=2, - input_size=3, - output_size=4, + with get_parallel().override( tp_rank=1, tp_size=2, - ) + attn_tp_rank=1, + attn_tp_size=2, + attn_dp_size=1, + attn_dp_rank=0, + attn_cp_size=1, + attn_cp_rank=0, + moe_tp_size=2, + moe_ep_size=1, + moe_dp_size=1, + tp_group=None, + attn_tp_group=None, + moe_ep_group=None, + ): + experts = RoutedValueExperts( + num_experts=2, + input_size=3, + output_size=4, + parallel_group="tp", + ) packed = torch.arange(2 * 4 * 3, dtype=torch.float32).reshape(2, 4, 3) experts.weight_loader(experts.weight, packed) torch.testing.assert_close(experts.weight, packed[:, 2:]) diff --git a/test/registered/unit/layers/test_repeated_batched_parallel_groups.py b/test/registered/unit/layers/test_repeated_batched_parallel_groups.py new file mode 100644 index 0000000000..086a7d20c2 --- /dev/null +++ b/test/registered/unit/layers/test_repeated_batched_parallel_groups.py @@ -0,0 +1,329 @@ +"""Repeated and batched projections retain native checkpoint partitions.""" + +import inspect +import unittest +from contextlib import nullcontext +from types import SimpleNamespace +from unittest.mock import Mock, patch + +import torch +import torch.nn.functional as F + +from sglang.srt.layers import linear +from sglang.srt.layers.quantization.fp8 import Fp8Config +from sglang.srt.runtime_context import SpawnRanks, get_parallel, publish, reset_context +from sglang.srt.server_args import ServerArgs +from sglang.test.ci.ci_register import register_cpu_ci +from sglang.test.test_utils import CustomTestCase + +register_cpu_ci(est_time=15, suite="base-a-test-cpu") + +CASES = ( + ("glm", "fused"), + ("glm", "bfg"), + ("glm", "split"), + ("kimi", "fused"), + ("kimi", "split"), + ("kimi", "full"), + ("kimi", "full_fp8"), +) + + +def build_projections( + kind, route, shard_attn=False, *, width=8, heads=4, head_dim=4, reduce=False +): + from sglang.srt.models.glm5_next import Glm5NextLinearAttention + from sglang.srt.models.kimi_linear import KimiDeltaAttention + + prefix = "model.layers.0.self_attn" + config = SimpleNamespace( + dtype=torch.get_default_dtype(), + linear_attn_config=dict( + head_dim=head_dim, num_heads=heads, short_conv_kernel_size=3 + ), + ) + quant = None + if route in ("bfg", "split", "full_fp8"): + ignored = ( + [ + prefix + "." + n + for n in ("b_proj", "f_a_proj", "g_a_proj", "f_b_proj", "g_b_proj") + ] + if route == "bfg" + else [] + ) + quant = Fp8Config( + ignored_layers=ignored, + packed_modules_mapping=Glm5NextLinearAttention._PACKED_MODULES_MAPPING + if kind == "glm" + else None, + ) + cls = Glm5NextLinearAttention if kind == "glm" else KimiDeltaAttention + kwargs = ( + {} + if kind == "glm" + else dict(no_kda_lora=route.startswith("full"), shard_on_attn_tp=shard_attn) + ) + module = cls( + 0, + width, + config, + quant_config=quant, + prefix=prefix, + reduce_results=reduce, + **kwargs, + ) + classes = ( + linear.ColumnParallelLinear, + linear.RowParallelLinear, + linear.MergedColumnParallelRepeatedLinear, + linear.ColumnParallelBatchedLinear, + linear.ReplicatedLinear, + ) + names = tuple( + name for name, layer in module.named_children() if isinstance(layer, classes) + ) + return module, names + + +def values(rows, columns, layer, offset=0): + return ( + (torch.arange(rows * columns, device=layer.weight.device) + offset) % 23 - 11 + ).reshape(rows, columns).to(layer.weight.dtype) / 128 + + +def load_projection(layer): + rank, size = getattr(layer, "tp_rank", 0), getattr(layer, "tp_size", 1) + with get_parallel().override(tp_rank=0, attn_tp_rank=0, attn_dp_rank=0): + if isinstance(layer, linear.MergedColumnParallelRepeatedLinear): + shards = [] + for i, local_size in enumerate(layer.output_partition_sizes): + column = i < layer.num_column_parallel + full = values( + local_size * size if column else local_size, + layer.input_size, + layer, + i, + ) + layer.weight.weight_loader(layer.weight, full, i) + shards.append(full.chunk(size)[rank] if column else full) + return torch.cat(shards) + if isinstance(layer, linear.ColumnParallelBatchedLinear): + shards = [] + for i in range(layer.weight.shape[0]): + full = values( + layer.weight.shape[1] * size, layer.weight.shape[2], layer, i + ) + layer.weight.weight_loader(layer.weight, full, i) + shards.append(full.chunk(size)[rank]) + return torch.stack(shards) + if isinstance(layer, linear.QKVParallelLinear): + shards = [] + for i, name in enumerate(("q", "k", "v")): + heads = ( + layer.total_num_heads if name == "q" else layer.total_num_kv_heads + ) + dim = layer.v_head_size if name == "v" else layer.head_size + full = values(heads * dim, layer.input_size, layer, i) + layer.weight.weight_loader(layer.weight, full, name) + shards.append(full.chunk(size)[rank]) + return torch.cat(shards) + full = values(layer.output_size, layer.input_size, layer) + if layer.weight.ndim == 3: + full = full.unsqueeze(1) + layer.weight.weight_loader(layer.weight, full) + if isinstance(layer, linear.RowParallelLinear): + return full.chunk(size, dim=1)[rank] + return torch.cat( + [ + part.chunk(size)[rank] + for part in full.split( + getattr(layer, "output_sizes", [layer.output_size]) + ) + ] + ) + + +class TestRepeatedBatchedParallelGroups(CustomTestCase): + def setUp(self): + reset_context() + self.addCleanup(reset_context) + self.addCleanup(torch.set_default_dtype, torch.get_default_dtype()) + torch.set_default_dtype(torch.float32) + + def test_helper_group_selection_frozen_reload_and_native_math(self): + publish( + ServerArgs(model_path="dummy", device="cpu", tp_size=4, attn_dp_size=2), + role="test", + ranks=SpawnRanks(world_rank=3), + ) + for group, rank, size in ( + (None, 3, 4), + ("tp", 3, 4), + ("attn_tp", 1, 2), + ("replicated", 0, 1), + ): + for cls in ( + linear.MergedColumnParallelRepeatedLinear, + linear.ColumnParallelBatchedLinear, + ): + kwargs = {} + if group is not None: + kwargs = ( + dict(parallel_group=group) + if "parallel_group" in inspect.signature(cls).parameters + else dict(tp_rank=rank, tp_size=size) + ) + layer = ( + cls(8, [16, 8], [4], **kwargs) + if cls is linear.MergedColumnParallelRepeatedLinear + else cls(2, 8, 16, dtype=torch.float32, **kwargs) + ) + expected = load_projection(layer) + torch.testing.assert_close(layer.weight, expected) + self.assertEqual((layer.tp_rank, layer.tp_size), (rank, size)) + if cls is linear.MergedColumnParallelRepeatedLinear: + x = values(2, 8, layer, 3) + actual = layer(x) + reference = F.linear(x, expected) + else: + x = values(4, 8, layer, 3).reshape(2, 2, 8) + actual = layer(x) + reference = torch.stack( + [F.linear(x[i], expected[i]) for i in range(2)] + ) + torch.testing.assert_close(actual, reference) + + def test_native_attention_projection_routes_loaders_and_row_policy(self): + for dp in (1, 2): + for rank in (0, 3): + reset_context() + publish( + ServerArgs( + model_path="dummy", device="cpu", tp_size=4, attn_dp_size=dp + ), + role="test", + ranks=SpawnRanks(world_rank=rank), + ) + tp, attn = Mock(), Mock() + tp.all_reduce.side_effect = lambda x: x * 4 + for kind, route in CASES: + for shard_attn in (False, True) if kind == "kimi" else (True,): + for reduce in (False, True): + with self.subTest( + dp=dp, + rank=rank, + kind=kind, + route=route, + shard_attn=shard_attn, + reduce=reduce, + ): + module, names = build_projections( + kind, route, shard_attn, reduce=reduce + ) + for name in names: + layer = getattr(module, name) + expected = load_projection(layer) + torch.testing.assert_close(layer.weight, expected) + conv = name == "qkv_conv1d" + batch = isinstance( + layer, linear.ColumnParallelBatchedLinear + ) + row = isinstance(layer, linear.RowParallelLinear) + size = ( + 4 // dp + if kind == "glm" + or shard_attn + or isinstance(layer, linear.QKVParallelLinear) + else 4 + ) + if isinstance(layer, linear.ReplicatedLinear): + size = 1 + local_rank = rank % size + self.assertEqual( + ( + getattr(layer, "tp_rank", 0), + getattr(layer, "tp_size", 1), + ), + (local_rank, size), + ) + if conv: + x = values( + 2 * expected.shape[0], 6, layer, 3 + ).reshape(2, expected.shape[0], 6) + reference = ( + x.unfold(-1, 3, 1) + * expected.squeeze(1)[None, :, None, :] + ).sum(-1) + actual = F.conv1d( + x, layer.weight, groups=expected.shape[0] + ) + elif batch: + x = values( + 2 * expected.shape[0], + expected.shape[2], + layer, + 3, + ).reshape(expected.shape[0], 2, -1) + reference = torch.stack( + [ + F.linear(x[i], expected[i]) + for i in range(expected.shape[0]) + ] + ) + actual = layer(x) + else: + x = values(2, expected.shape[1], layer, 3) + reference = F.linear(x, expected) + if row and reduce and size > 1: + reference *= 4 + tp.all_reduce.reset_mock() + attn.all_reduce.reset_mock() + dense = ( + patch.object( + layer.quant_method, + "apply", + side_effect=lambda l, x, bias=None: ( + F.linear(x, l.weight, bias) + ), + ) + if layer.quant_method.__class__.__name__ + == "Fp8LinearMethod" + else nullcontext() + ) + with ( + get_parallel().override( + tp_group=tp, attn_tp_group=attn + ), + patch( + "sglang.srt.layers.linear.use_symmetric_memory", + return_value=nullcontext(), + ) as allocation, + patch( + "sglang.srt.layers.linear.is_allocation_symmetric", + return_value=False, + ), + dense, + ): + out = layer(x) + actual = ( + out[0] + if isinstance(out, tuple) + else out + ) + if row: + allocation.assert_called_once_with( + tp, disabled=True + ) + else: + allocation.assert_not_called() + self.assertEqual( + tp.all_reduce.call_count, + int(row and reduce and size > 1), + ) + attn.all_reduce.assert_not_called() + torch.testing.assert_close(actual, reference) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/unit/layers/test_replicated_attention_parallel_groups.py b/test/registered/unit/layers/test_replicated_attention_parallel_groups.py new file mode 100644 index 0000000000..96b187b509 --- /dev/null +++ b/test/registered/unit/layers/test_replicated_attention_parallel_groups.py @@ -0,0 +1,244 @@ +"""Replicated attention partitions preserve native projection and loader math.""" + +import unittest +from contextlib import nullcontext +from types import SimpleNamespace +from unittest.mock import Mock, patch + +import torch +import torch.nn.functional as F + +from sglang.srt.layers import linear +from sglang.srt.layers.layer_boundary.factories import layer_stack +from sglang.srt.runtime_context import SpawnRanks, get_parallel, publish, reset_context +from sglang.srt.server_args import ServerArgs +from sglang.test.ci.ci_register import register_cuda_ci +from sglang.test.test_utils import CustomTestCase + +register_cuda_ci(est_time=20, stage="base-b", runner_config="1-gpu-small") + + +def build_attention(model, variant, *, width=32, head_dim=8, quant_config=None): + config = SimpleNamespace( + hidden_size=width, + num_attention_heads=8, + num_key_value_heads=variant[2] if model == "qwen" else 4, + head_dim=head_dim, + rms_norm_eps=1e-6, + max_position_embeddings=64, + rope_theta=10000, + rope_scaling=None, + partial_rotary_factor=1.0, + model_type="qwen3_5_text", + intermediate_size=2 * width, + hidden_act="silu", + num_hidden_layers=2, + attn_output_gate=variant[1] if model == "qwen" else False, + qk_norm_type="per_head", + sparse_attention_config=dict( + sparse_num_index_heads=variant[0], sparse_index_dim=head_dim + ), + ) + if model == "qwen": + from sglang.srt.models.qwen3_5 import Qwen3_5AttentionDecoderLayer + + with layer_stack(): + module = Qwen3_5AttentionDecoderLayer( + config, 0, quant_config=quant_config, is_nextn=variant[0] + ) + names = ("qkv_proj", "o_proj") + else: + from sglang.srt.models.minimax_m3 import MiniMaxM3Attention + + module = MiniMaxM3Attention( + config, + quant_config=quant_config, + is_sparse_attention_layer=True, + disable_index_value=variant[1], + ) + names = ("qkv_proj", "o_proj", "index_qkv_proj") + if not variant[1]: + names += ("index_o_proj",) + return module, names + + +def cases(): + for nextn in (False, True): + for gate in (False, True): + for kv_heads in (1, 4): + yield "qwen", (nextn, gate, kv_heads) + for heads in (1, 2, 8): + for disable_value in (False, True): + yield "minimax", (heads, disable_value) + + +def values(rows, columns, layer, offset=0): + return ( + (torch.arange(rows * columns, device=layer.weight.device) + offset) % 29 - 14 + ).reshape(rows, columns).to(layer.weight.dtype) / 128 + + +def load_projection(layer): + rank, size = layer.tp_rank, layer.tp_size + with get_parallel().override(tp_rank=0, attn_tp_rank=0, attn_dp_rank=0): + if isinstance(layer, linear.QKVParallelLinear): + shards = [] + for i, name in enumerate(("q", "k", "v")): + heads = ( + layer.total_num_heads if name == "q" else layer.total_num_kv_heads + ) + dim = layer.v_head_size if name == "v" else layer.head_size + full = values(heads * dim, layer.input_size, layer, i) + r, s = ( + (rank, size) + if name == "q" + else (layer.kv_tp_rank, layer.kv_tp_size) + ) + pieces = s if name == "q" else min(heads, s) + index = r if name == "q" else r // max(s // heads, 1) + layer.weight.weight_loader(layer.weight, full, name) + shards.append(full.chunk(pieces)[index]) + return torch.cat(shards), None + full = values(layer.output_size, layer.input_size, layer) + layer.weight.weight_loader(layer.weight, full) + return full.chunk(size, dim=1)[rank], None + + +class TestReplicatedAttentionParallelGroups(CustomTestCase): + def setUp(self): + reset_context() + self.addCleanup(reset_context) + + def test_native_models_keep_independent_kv_and_index_partitions(self): + for dp in (1, 2): + for dcp in (1, 2, 4): + if dcp > 4 // dp: + continue + for rank in range(4): + reset_context() + publish( + ServerArgs( + model_path="dummy", + device="cpu", + tp_size=4, + attn_dp_size=dp, + dcp_size=dcp, + ), + role="test", + ranks=SpawnRanks(world_rank=rank), + ) + for model, variant in cases(): + if model == "minimax" and dcp != 1: + continue + with self.subTest( + dp=dp, dcp=dcp, rank=rank, model=model, variant=variant + ): + module, names = build_attention(model, variant) + attn_rank, attn_size = rank % (4 // dp), 4 // dp + for name in names: + layer = getattr(module, name) + if name.startswith("index_"): + size = min(attn_size, variant[0]) + expected_rank = attn_rank // (attn_size // size) + else: + size, expected_rank = attn_size, attn_rank + self.assertEqual( + (layer.tp_rank, layer.tp_size), + (expected_rank, size), + ) + if isinstance(layer, linear.QKVParallelLinear): + factor = ( + dcp if model == "qwen" and not variant[0] else 1 + ) + self.assertEqual( + (layer.kv_tp_rank, layer.kv_tp_size), + (expected_rank // factor, size // factor), + ) + shard, _ = load_projection(layer) + torch.testing.assert_close(layer.weight, shard) + x = values(2, shard.shape[1], layer, 7) + tp, attn = Mock(), Mock() + with ( + get_parallel().override( + tp_group=tp, attn_tp_group=attn + ), + patch.object( + linear, + "use_symmetric_memory", + return_value=nullcontext(), + ) as allocation, + ): + actual = layer(x)[0] + torch.testing.assert_close(actual, F.linear(x, shard)) + row = isinstance(layer, linear.RowParallelLinear) + if row: + self.assertFalse(layer.reduce_results) + self.assertFalse(layer.use_dp_attention_reduce) + self.assertIs(allocation.call_args.args[0], tp) + else: + allocation.assert_not_called() + self.assertEqual( + tp.all_reduce.call_count + + attn.all_reduce.call_count, + 0, + ) + if name.startswith("index_"): + self.assertEqual( + module.idx_replica_size, attn_size // size + ) + + def test_group_recipes_freeze_independent_query_and_kv_partitions(self): + from sglang.srt.layers.linear import ReplicatedParallelGroup + + publish( + ServerArgs(model_path="dummy", device="cpu", tp_size=8, attn_dp_size=2), + role="test", + ranks=SpawnRanks(world_rank=7), + ) + for group, factor, rank, size in ( + ("tp", 2, 3, 4), + ("tp", 4, 1, 2), + ("attn_tp", 2, 1, 2), + ("attn_tp", 4, 0, 1), + ): + selection = ReplicatedParallelGroup(group, factor) + self.assertEqual( + linear.resolve_linear_parallel_group(selection), (rank, size) + ) + qkv = linear.QKVParallelLinear( + 8, + 2, + 8, + 4, + bias=False, + parallel_group="attn_tp", + kv_parallel_group=selection, + ) + self.assertEqual((qkv.tp_rank, qkv.tp_size), (3, 4)) + self.assertEqual((qkv.kv_tp_rank, qkv.kv_tp_size), (rank, size)) + expected, _ = load_projection(qkv) + torch.testing.assert_close(qkv.weight, expected) + x = values(2, 8, qkv, 3) + torch.testing.assert_close(qkv(x)[0], F.linear(x, expected)) + + def test_invalid_replica_factors_and_conflicting_kv_placement(self): + from sglang.srt.layers.linear import ReplicatedParallelGroup + + for factor in (0, -1, True, 1.5): + with self.assertRaises(ValueError): + ReplicatedParallelGroup("attn_tp", factor) + with self.assertRaises(ValueError): + ReplicatedParallelGroup("invalid", 1) + publish( + ServerArgs(model_path="dummy", device="cpu", tp_size=4), + role="test", + ranks=SpawnRanks(world_rank=0), + ) + with self.assertRaises(AssertionError): + linear.resolve_linear_parallel_group(ReplicatedParallelGroup("tp", 3)) + with self.assertRaisesRegex(ValueError, "cannot be combined"): + linear.QKVParallelLinear(8, 2, 8, 4, kv_parallel_group="tp", kv_tp_size=4) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/unit/models/test_hunyuan_v4.py b/test/registered/unit/models/test_hunyuan_v4.py index a9069ee81e..d6e3b00185 100644 --- a/test/registered/unit/models/test_hunyuan_v4.py +++ b/test/registered/unit/models/test_hunyuan_v4.py @@ -28,7 +28,7 @@ def test_attention_gate_uses_attention_tp(monkeypatch): def __init__(self, input_size, output_size, **kwargs): super().__init__() captured.update(kwargs) - self.output_size_per_partition = output_size // kwargs["tp_size"] + self.output_size_per_partition = output_size // parallel.attn_tp_size monkeypatch.setattr( hunyuan_v4.DeepseekV2AttentionMLA, "__init__", fake_attention_init @@ -56,8 +56,7 @@ def test_attention_gate_uses_attention_tp(monkeypatch): attention = hunyuan_v4.HYV4Attention(config, layer_id=0) - assert captured["tp_rank"] == parallel.attn_tp_rank - assert captured["tp_size"] == parallel.attn_tp_size + assert captured["parallel_group"] == "attn_tp" assert attention.local_gate_width == (64 // attn_tp_size) * 256 assert attention.linear_gate.output_size_per_partition == attention.local_gate_width diff --git a/test/registered/unit/models/test_zaya_cca.py b/test/registered/unit/models/test_zaya_cca.py index c4d2c74983..1979f0ed5d 100644 --- a/test/registered/unit/models/test_zaya_cca.py +++ b/test/registered/unit/models/test_zaya_cca.py @@ -29,7 +29,7 @@ from typing import List, Optional import torch -from sglang.srt.runtime_context import get_context +from sglang.srt.runtime_context import derive_parallel_widths, get_context, get_parallel from sglang.test.ci.ci_register import register_cpu_ci from sglang.test.layer_ut_utils import init_single_process_dist from sglang.test.test_utils import CustomTestCase @@ -38,11 +38,7 @@ register_cpu_ci(est_time=11, suite="base-a-test-cpu") def _ensure_dist_initialized(cls) -> None: - """CCA reads the TP rank / world size inside ``__init__`` to size its - head-parallel projections. The rank is the live group's, so the groups must - exist before construction; the size answers from the published ``parallel`` - bag, so the case has to publish a context as well. - """ + """Publish the single-process construction context used by the tests.""" init_single_process_dist() override = get_context().override_server_args(tp_size=1) override.install() @@ -213,6 +209,43 @@ def _make_tiny_config(num_hidden_layers: int = 2): ) +def _build_cca(tp_rank=None, tp_size=None, **kwargs): + from sglang.srt.models.zaya import CCA + + parallel = get_parallel() + rank = parallel.tp_rank if tp_rank is None else tp_rank + size = parallel.tp_size if tp_size is None else tp_size + widths = derive_parallel_widths( + tp_size=size, + attn_cp_size=1, + attn_dp_size=1, + moe_ep_size=1, + moe_dp_size=1, + dcp_size=1, + dcp_enabled=False, + ) + group = SimpleNamespace(world_size=size, rank_in_group=rank) + # Only construction uses the virtual group. Reload and forward run after + # this scope closes, so they must use the module's frozen head partition. + with parallel.override( + **widths, + tp_size=size, + attn_cp_size=1, + moe_dp_size=1, + tp_rank=rank, + attn_tp_rank=rank, + attn_dp_rank=0, + attn_cp_rank=0, + moe_tp_rank=rank, + moe_ep_rank=0, + moe_dp_rank=0, + tp_group=group, + attn_tp_group=group, + moe_tp_group=group, + ): + return CCA(**kwargs) + + def _make_tiny_cca( seed: int = 0, tp_rank: Optional[int] = None, @@ -220,12 +253,10 @@ def _make_tiny_cca( layer_id: int = 0, config=None, ): - from sglang.srt.models.zaya import CCA - if config is None: config = _make_tiny_config() torch.manual_seed(seed) - cca = CCA( + cca = _build_cca( config=config, cca_num_k_heads=config.num_query_groups, cca_num_q_heads=config.num_attention_heads, @@ -767,12 +798,10 @@ class TestZayaCCATensorParallel(CustomTestCase): both num_q_heads and num_k_heads, since both grouped-mean and conv_qk.1 require each rank to hold whole K-head groups. """ - from sglang.srt.models.zaya import CCA - cfg = _make_tiny_config() # tiny config has num_query_groups=2; TP=4 cannot divide it cleanly. with self.assertRaises(AssertionError): - CCA( + _build_cca( config=cfg, cca_num_k_heads=cfg.num_query_groups, cca_num_q_heads=cfg.num_attention_heads,