From c7fa37a5ca8898d45bd2a92272a35fee6c213c73 Mon Sep 17 00:00:00 2001 From: Cheng Wan <54331508+ch-wan@users.noreply.github.com> Date: Wed, 30 Sep 2026 13:27:58 -0700 Subject: [PATCH] [Refactor] Let FusedMoE's weight-loading helpers read the layer's MoE-TP rank (#41814) --- .../srt/layers/moe/fused_moe_triton/layer.py | 39 +++---------------- 1 file changed, 6 insertions(+), 33 deletions(-) diff --git a/python/sglang/srt/layers/moe/fused_moe_triton/layer.py b/python/sglang/srt/layers/moe/fused_moe_triton/layer.py index 739ad79583..32881b37a9 100644 --- a/python/sglang/srt/layers/moe/fused_moe_triton/layer.py +++ b/python/sglang/srt/layers/moe/fused_moe_triton/layer.py @@ -673,7 +673,6 @@ class FusedMoE(torch.nn.Module): expert_data: torch.Tensor, shard_id: str, loaded_weight: torch.Tensor, - tp_rank: int, is_bias: bool = False, ): # Load grouped weight scales for group quantization @@ -684,7 +683,6 @@ class FusedMoE(torch.nn.Module): shard_dim=shard_dim, loaded_weight=loaded_weight, expert_data=expert_data, - tp_rank=tp_rank, is_bias=is_bias, ) elif shard_id in ("w1", "w3", "w13"): @@ -693,7 +691,6 @@ class FusedMoE(torch.nn.Module): shard_dim=shard_dim, loaded_weight=loaded_weight, expert_data=expert_data, - tp_rank=tp_rank, is_bias=is_bias, ) @@ -703,7 +700,6 @@ class FusedMoE(torch.nn.Module): shard_dim: int, shard_id: str, loaded_weight: torch.Tensor, - tp_rank: int, ): # for per channel weight quantization if shard_id == "w2": @@ -715,7 +711,6 @@ class FusedMoE(torch.nn.Module): shard_dim=shard_dim, loaded_weight=loaded_weight, expert_data=expert_data, - tp_rank=tp_rank, ) def _load_w13( @@ -724,7 +719,6 @@ class FusedMoE(torch.nn.Module): shard_dim: int, shard_id: str, loaded_weight: torch.Tensor, - tp_rank: int, is_bias: bool = False, ): # Index the loaded weight for tp sharding. @@ -765,7 +759,7 @@ class FusedMoE(torch.nn.Module): expert_data, loaded_weight, start, - shard_size * tp_rank, + shard_size * self.moe_tp_rank, shard_dim, shard_size, not self.use_presharded_weights, @@ -781,7 +775,7 @@ class FusedMoE(torch.nn.Module): # shard size from the loaded weight so we index correctly. loaded_shard_size = loaded_weight.shape[shard_dim] // self.moe_tp_size loaded_weight = loaded_weight.narrow( - shard_dim, loaded_shard_size * tp_rank, loaded_shard_size + shard_dim, loaded_shard_size * self.moe_tp_rank, loaded_shard_size ) expert_data = expert_data.narrow(shard_dim, start, shard_size) @@ -808,7 +802,6 @@ class FusedMoE(torch.nn.Module): shard_dim: int, shard_id: str, loaded_weight: torch.Tensor, - tp_rank: int, is_bias: bool = False, ): """Load w2 weights for down projection. @@ -818,7 +811,6 @@ class FusedMoE(torch.nn.Module): shard_dim: The dimension to shard along shard_id: The shard ID (must be "w2") loaded_weight: The weight tensor to load from - tp_rank: The tensor parallel rank """ if not isinstance(expert_data, torch.Tensor) or not isinstance( loaded_weight, torch.Tensor @@ -856,7 +848,7 @@ class FusedMoE(torch.nn.Module): expert_data, loaded_weight, 0, # param_data_start - shard_size * tp_rank, + shard_size * self.moe_tp_rank, shard_dim, shard_size, not self.use_presharded_weights, @@ -869,7 +861,7 @@ class FusedMoE(torch.nn.Module): # do not cause out-of-bounds indexing into the checkpoint. loaded_shard_size = loaded_weight.shape[shard_dim] // self.moe_tp_size loaded_weight = loaded_weight.narrow( - shard_dim, loaded_shard_size * tp_rank, loaded_shard_size + shard_dim, loaded_shard_size * self.moe_tp_rank, loaded_shard_size ) # w2, down_proj: Load into only logical weight of w2. @@ -896,7 +888,6 @@ class FusedMoE(torch.nn.Module): shard_id: str, expert_id: int, shard_dim: int, - tp_rank: int, ) -> bool: if ( not self._has_fused_shared @@ -964,14 +955,12 @@ class FusedMoE(torch.nn.Module): expert_data=weight_data, shard_id=shard_id, loaded_weight=fp4_weight, - tp_rank=tp_rank, ) self._load_model_weight_or_group_weight_scale( shard_dim=shard_dim, expert_data=scale_data, shard_id=shard_id, loaded_weight=fp4_scale, - tp_rank=tp_rank, ) return True @@ -989,7 +978,6 @@ class FusedMoE(torch.nn.Module): expert_data: torch.Tensor, shard_dim: int, loaded_weight: torch.Tensor, - tp_rank: int, ): if shard_id == "w2": self._load_w2( @@ -997,7 +985,6 @@ class FusedMoE(torch.nn.Module): shard_dim=shard_dim, loaded_weight=loaded_weight, expert_data=expert_data, - tp_rank=tp_rank, ) else: assert shard_id in ("w1", "w3") @@ -1124,7 +1111,6 @@ class FusedMoE(torch.nn.Module): loaded_weight: torch.Tensor, shard_id: str, expert_id: int, - tp_rank: int, ) -> bool: """Handle GGUF weight loading. @@ -1133,7 +1119,6 @@ class FusedMoE(torch.nn.Module): loaded_weight: The weight tensor to load. shard_id: The shard ID (w1, w2, or w3). expert_id: The expert ID. - tp_rank: The tensor parallel rank. Returns: True if the weight was handled as a GGUF weight, False otherwise. @@ -1151,7 +1136,7 @@ class FusedMoE(torch.nn.Module): if self.moe_tp_size > 1: if shard_id in ["w1", "w3", "w2"] and output_dim == 0: shard_size = loaded_weight.size(0) // self.moe_tp_size - start_idx = tp_rank * shard_size + start_idx = self.moe_tp_rank * shard_size loaded_weight = loaded_weight.narrow( 0, start_idx, shard_size ).clone() @@ -1175,10 +1160,8 @@ class FusedMoE(torch.nn.Module): shard_id: str, expert_id: int, ) -> None: - tp_rank = self.moe_tp_rank - # Special case for GGUF weights - if self._load_gguf_weight(param, loaded_weight, shard_id, expert_id, tp_rank): + if self._load_gguf_weight(param, loaded_weight, shard_id, expert_id): return # compressed-tensors checkpoints with packed weights are stored flipped @@ -1261,7 +1244,6 @@ class FusedMoE(torch.nn.Module): shard_id=shard_id, expert_id=expert_id, shard_dim=shard_dim, - tp_rank=tp_rank, ): return @@ -1300,7 +1282,6 @@ class FusedMoE(torch.nn.Module): shard_id=shard_id, loaded_weight=loaded_weight, expert_data=expert_data, - tp_rank=tp_rank, ) return @@ -1328,7 +1309,6 @@ class FusedMoE(torch.nn.Module): shard_dim=shard_dim, loaded_weight=loaded_weight, expert_data=expert_data, - tp_rank=tp_rank, ) return @@ -1350,7 +1330,6 @@ class FusedMoE(torch.nn.Module): shard_dim=shard_dim, loaded_weight=loaded_weight, expert_data=expert_data, - tp_rank=tp_rank, ) elif quant_method in [ FusedMoeWeightScaleSupported.GROUP.value, @@ -1361,7 +1340,6 @@ class FusedMoE(torch.nn.Module): shard_dim=shard_dim, loaded_weight=loaded_weight, expert_data=expert_data, - tp_rank=tp_rank, ) elif quant_method == FusedMoeWeightScaleSupported.TENSOR.value: # INT4-FP8 (INT4 MoE Weight, FP8 Compute): Adjust FP8 per-tensor scaling number for e4m3fnuz (AMD) @@ -1395,7 +1373,6 @@ class FusedMoE(torch.nn.Module): shard_dim=shard_dim, loaded_weight=loaded_weight, expert_data=expert_data, - tp_rank=tp_rank, ) return @@ -1408,7 +1385,6 @@ class FusedMoE(torch.nn.Module): shard_dim=shard_dim, loaded_weight=loaded_weight, expert_data=expert_data, - tp_rank=tp_rank, ) def weight_loader_fused( @@ -1418,8 +1394,6 @@ class FusedMoE(torch.nn.Module): weight_name: str, shard_id: str, ) -> None: - tp_rank = self.moe_tp_rank - # Mirror _weight_loader_impl: the trtllm bf16 prep reshapes expert weights # into block layout; hot weight updates must restore canonical shapes first. method = self.quant_method @@ -1509,7 +1483,6 @@ class FusedMoE(torch.nn.Module): shard_dim=shard_dim, loaded_weight=loaded_weight, expert_data=expert_data, - tp_rank=tp_rank, is_bias=is_bias, ) return