[Refactor] Let FusedMoE's weight-loading helpers read the layer's MoE-TP rank (#41814)

This commit is contained in:
Cheng Wan
2026-09-30 13:27:58 -07:00
committed by GitHub
parent 499e86db98
commit c7fa37a5ca
@@ -673,7 +673,6 @@ class FusedMoE(torch.nn.Module):
expert_data: torch.Tensor,
shard_id: str,
loaded_weight: torch.Tensor,
tp_rank: int,
is_bias: bool = False,
):
# Load grouped weight scales for group quantization
@@ -684,7 +683,6 @@ class FusedMoE(torch.nn.Module):
shard_dim=shard_dim,
loaded_weight=loaded_weight,
expert_data=expert_data,
tp_rank=tp_rank,
is_bias=is_bias,
)
elif shard_id in ("w1", "w3", "w13"):
@@ -693,7 +691,6 @@ class FusedMoE(torch.nn.Module):
shard_dim=shard_dim,
loaded_weight=loaded_weight,
expert_data=expert_data,
tp_rank=tp_rank,
is_bias=is_bias,
)
@@ -703,7 +700,6 @@ class FusedMoE(torch.nn.Module):
shard_dim: int,
shard_id: str,
loaded_weight: torch.Tensor,
tp_rank: int,
):
# for per channel weight quantization
if shard_id == "w2":
@@ -715,7 +711,6 @@ class FusedMoE(torch.nn.Module):
shard_dim=shard_dim,
loaded_weight=loaded_weight,
expert_data=expert_data,
tp_rank=tp_rank,
)
def _load_w13(
@@ -724,7 +719,6 @@ class FusedMoE(torch.nn.Module):
shard_dim: int,
shard_id: str,
loaded_weight: torch.Tensor,
tp_rank: int,
is_bias: bool = False,
):
# Index the loaded weight for tp sharding.
@@ -765,7 +759,7 @@ class FusedMoE(torch.nn.Module):
expert_data,
loaded_weight,
start,
shard_size * tp_rank,
shard_size * self.moe_tp_rank,
shard_dim,
shard_size,
not self.use_presharded_weights,
@@ -781,7 +775,7 @@ class FusedMoE(torch.nn.Module):
# shard size from the loaded weight so we index correctly.
loaded_shard_size = loaded_weight.shape[shard_dim] // self.moe_tp_size
loaded_weight = loaded_weight.narrow(
shard_dim, loaded_shard_size * tp_rank, loaded_shard_size
shard_dim, loaded_shard_size * self.moe_tp_rank, loaded_shard_size
)
expert_data = expert_data.narrow(shard_dim, start, shard_size)
@@ -808,7 +802,6 @@ class FusedMoE(torch.nn.Module):
shard_dim: int,
shard_id: str,
loaded_weight: torch.Tensor,
tp_rank: int,
is_bias: bool = False,
):
"""Load w2 weights for down projection.
@@ -818,7 +811,6 @@ class FusedMoE(torch.nn.Module):
shard_dim: The dimension to shard along
shard_id: The shard ID (must be "w2")
loaded_weight: The weight tensor to load from
tp_rank: The tensor parallel rank
"""
if not isinstance(expert_data, torch.Tensor) or not isinstance(
loaded_weight, torch.Tensor
@@ -856,7 +848,7 @@ class FusedMoE(torch.nn.Module):
expert_data,
loaded_weight,
0, # param_data_start
shard_size * tp_rank,
shard_size * self.moe_tp_rank,
shard_dim,
shard_size,
not self.use_presharded_weights,
@@ -869,7 +861,7 @@ class FusedMoE(torch.nn.Module):
# do not cause out-of-bounds indexing into the checkpoint.
loaded_shard_size = loaded_weight.shape[shard_dim] // self.moe_tp_size
loaded_weight = loaded_weight.narrow(
shard_dim, loaded_shard_size * tp_rank, loaded_shard_size
shard_dim, loaded_shard_size * self.moe_tp_rank, loaded_shard_size
)
# w2, down_proj: Load into only logical weight of w2.
@@ -896,7 +888,6 @@ class FusedMoE(torch.nn.Module):
shard_id: str,
expert_id: int,
shard_dim: int,
tp_rank: int,
) -> bool:
if (
not self._has_fused_shared
@@ -964,14 +955,12 @@ class FusedMoE(torch.nn.Module):
expert_data=weight_data,
shard_id=shard_id,
loaded_weight=fp4_weight,
tp_rank=tp_rank,
)
self._load_model_weight_or_group_weight_scale(
shard_dim=shard_dim,
expert_data=scale_data,
shard_id=shard_id,
loaded_weight=fp4_scale,
tp_rank=tp_rank,
)
return True
@@ -989,7 +978,6 @@ class FusedMoE(torch.nn.Module):
expert_data: torch.Tensor,
shard_dim: int,
loaded_weight: torch.Tensor,
tp_rank: int,
):
if shard_id == "w2":
self._load_w2(
@@ -997,7 +985,6 @@ class FusedMoE(torch.nn.Module):
shard_dim=shard_dim,
loaded_weight=loaded_weight,
expert_data=expert_data,
tp_rank=tp_rank,
)
else:
assert shard_id in ("w1", "w3")
@@ -1124,7 +1111,6 @@ class FusedMoE(torch.nn.Module):
loaded_weight: torch.Tensor,
shard_id: str,
expert_id: int,
tp_rank: int,
) -> bool:
"""Handle GGUF weight loading.
@@ -1133,7 +1119,6 @@ class FusedMoE(torch.nn.Module):
loaded_weight: The weight tensor to load.
shard_id: The shard ID (w1, w2, or w3).
expert_id: The expert ID.
tp_rank: The tensor parallel rank.
Returns:
True if the weight was handled as a GGUF weight, False otherwise.
@@ -1151,7 +1136,7 @@ class FusedMoE(torch.nn.Module):
if self.moe_tp_size > 1:
if shard_id in ["w1", "w3", "w2"] and output_dim == 0:
shard_size = loaded_weight.size(0) // self.moe_tp_size
start_idx = tp_rank * shard_size
start_idx = self.moe_tp_rank * shard_size
loaded_weight = loaded_weight.narrow(
0, start_idx, shard_size
).clone()
@@ -1175,10 +1160,8 @@ class FusedMoE(torch.nn.Module):
shard_id: str,
expert_id: int,
) -> None:
tp_rank = self.moe_tp_rank
# Special case for GGUF weights
if self._load_gguf_weight(param, loaded_weight, shard_id, expert_id, tp_rank):
if self._load_gguf_weight(param, loaded_weight, shard_id, expert_id):
return
# compressed-tensors checkpoints with packed weights are stored flipped
@@ -1261,7 +1244,6 @@ class FusedMoE(torch.nn.Module):
shard_id=shard_id,
expert_id=expert_id,
shard_dim=shard_dim,
tp_rank=tp_rank,
):
return
@@ -1300,7 +1282,6 @@ class FusedMoE(torch.nn.Module):
shard_id=shard_id,
loaded_weight=loaded_weight,
expert_data=expert_data,
tp_rank=tp_rank,
)
return
@@ -1328,7 +1309,6 @@ class FusedMoE(torch.nn.Module):
shard_dim=shard_dim,
loaded_weight=loaded_weight,
expert_data=expert_data,
tp_rank=tp_rank,
)
return
@@ -1350,7 +1330,6 @@ class FusedMoE(torch.nn.Module):
shard_dim=shard_dim,
loaded_weight=loaded_weight,
expert_data=expert_data,
tp_rank=tp_rank,
)
elif quant_method in [
FusedMoeWeightScaleSupported.GROUP.value,
@@ -1361,7 +1340,6 @@ class FusedMoE(torch.nn.Module):
shard_dim=shard_dim,
loaded_weight=loaded_weight,
expert_data=expert_data,
tp_rank=tp_rank,
)
elif quant_method == FusedMoeWeightScaleSupported.TENSOR.value:
# INT4-FP8 (INT4 MoE Weight, FP8 Compute): Adjust FP8 per-tensor scaling number for e4m3fnuz (AMD)
@@ -1395,7 +1373,6 @@ class FusedMoE(torch.nn.Module):
shard_dim=shard_dim,
loaded_weight=loaded_weight,
expert_data=expert_data,
tp_rank=tp_rank,
)
return
@@ -1408,7 +1385,6 @@ class FusedMoE(torch.nn.Module):
shard_dim=shard_dim,
loaded_weight=loaded_weight,
expert_data=expert_data,
tp_rank=tp_rank,
)
def weight_loader_fused(
@@ -1418,8 +1394,6 @@ class FusedMoE(torch.nn.Module):
weight_name: str,
shard_id: str,
) -> None:
tp_rank = self.moe_tp_rank
# Mirror _weight_loader_impl: the trtllm bf16 prep reshapes expert weights
# into block layout; hot weight updates must restore canonical shapes first.
method = self.quant_method
@@ -1509,7 +1483,6 @@ class FusedMoE(torch.nn.Module):
shard_dim=shard_dim,
loaded_weight=loaded_weight,
expert_data=expert_data,
tp_rank=tp_rank,
is_bias=is_bias,
)
return