[Refactor] Let FusedMoE's weight-loading helpers read the layer's MoE-TP rank (#41814)
This commit is contained in:
@@ -673,7 +673,6 @@ class FusedMoE(torch.nn.Module):
|
||||
expert_data: torch.Tensor,
|
||||
shard_id: str,
|
||||
loaded_weight: torch.Tensor,
|
||||
tp_rank: int,
|
||||
is_bias: bool = False,
|
||||
):
|
||||
# Load grouped weight scales for group quantization
|
||||
@@ -684,7 +683,6 @@ class FusedMoE(torch.nn.Module):
|
||||
shard_dim=shard_dim,
|
||||
loaded_weight=loaded_weight,
|
||||
expert_data=expert_data,
|
||||
tp_rank=tp_rank,
|
||||
is_bias=is_bias,
|
||||
)
|
||||
elif shard_id in ("w1", "w3", "w13"):
|
||||
@@ -693,7 +691,6 @@ class FusedMoE(torch.nn.Module):
|
||||
shard_dim=shard_dim,
|
||||
loaded_weight=loaded_weight,
|
||||
expert_data=expert_data,
|
||||
tp_rank=tp_rank,
|
||||
is_bias=is_bias,
|
||||
)
|
||||
|
||||
@@ -703,7 +700,6 @@ class FusedMoE(torch.nn.Module):
|
||||
shard_dim: int,
|
||||
shard_id: str,
|
||||
loaded_weight: torch.Tensor,
|
||||
tp_rank: int,
|
||||
):
|
||||
# for per channel weight quantization
|
||||
if shard_id == "w2":
|
||||
@@ -715,7 +711,6 @@ class FusedMoE(torch.nn.Module):
|
||||
shard_dim=shard_dim,
|
||||
loaded_weight=loaded_weight,
|
||||
expert_data=expert_data,
|
||||
tp_rank=tp_rank,
|
||||
)
|
||||
|
||||
def _load_w13(
|
||||
@@ -724,7 +719,6 @@ class FusedMoE(torch.nn.Module):
|
||||
shard_dim: int,
|
||||
shard_id: str,
|
||||
loaded_weight: torch.Tensor,
|
||||
tp_rank: int,
|
||||
is_bias: bool = False,
|
||||
):
|
||||
# Index the loaded weight for tp sharding.
|
||||
@@ -765,7 +759,7 @@ class FusedMoE(torch.nn.Module):
|
||||
expert_data,
|
||||
loaded_weight,
|
||||
start,
|
||||
shard_size * tp_rank,
|
||||
shard_size * self.moe_tp_rank,
|
||||
shard_dim,
|
||||
shard_size,
|
||||
not self.use_presharded_weights,
|
||||
@@ -781,7 +775,7 @@ class FusedMoE(torch.nn.Module):
|
||||
# shard size from the loaded weight so we index correctly.
|
||||
loaded_shard_size = loaded_weight.shape[shard_dim] // self.moe_tp_size
|
||||
loaded_weight = loaded_weight.narrow(
|
||||
shard_dim, loaded_shard_size * tp_rank, loaded_shard_size
|
||||
shard_dim, loaded_shard_size * self.moe_tp_rank, loaded_shard_size
|
||||
)
|
||||
|
||||
expert_data = expert_data.narrow(shard_dim, start, shard_size)
|
||||
@@ -808,7 +802,6 @@ class FusedMoE(torch.nn.Module):
|
||||
shard_dim: int,
|
||||
shard_id: str,
|
||||
loaded_weight: torch.Tensor,
|
||||
tp_rank: int,
|
||||
is_bias: bool = False,
|
||||
):
|
||||
"""Load w2 weights for down projection.
|
||||
@@ -818,7 +811,6 @@ class FusedMoE(torch.nn.Module):
|
||||
shard_dim: The dimension to shard along
|
||||
shard_id: The shard ID (must be "w2")
|
||||
loaded_weight: The weight tensor to load from
|
||||
tp_rank: The tensor parallel rank
|
||||
"""
|
||||
if not isinstance(expert_data, torch.Tensor) or not isinstance(
|
||||
loaded_weight, torch.Tensor
|
||||
@@ -856,7 +848,7 @@ class FusedMoE(torch.nn.Module):
|
||||
expert_data,
|
||||
loaded_weight,
|
||||
0, # param_data_start
|
||||
shard_size * tp_rank,
|
||||
shard_size * self.moe_tp_rank,
|
||||
shard_dim,
|
||||
shard_size,
|
||||
not self.use_presharded_weights,
|
||||
@@ -869,7 +861,7 @@ class FusedMoE(torch.nn.Module):
|
||||
# do not cause out-of-bounds indexing into the checkpoint.
|
||||
loaded_shard_size = loaded_weight.shape[shard_dim] // self.moe_tp_size
|
||||
loaded_weight = loaded_weight.narrow(
|
||||
shard_dim, loaded_shard_size * tp_rank, loaded_shard_size
|
||||
shard_dim, loaded_shard_size * self.moe_tp_rank, loaded_shard_size
|
||||
)
|
||||
|
||||
# w2, down_proj: Load into only logical weight of w2.
|
||||
@@ -896,7 +888,6 @@ class FusedMoE(torch.nn.Module):
|
||||
shard_id: str,
|
||||
expert_id: int,
|
||||
shard_dim: int,
|
||||
tp_rank: int,
|
||||
) -> bool:
|
||||
if (
|
||||
not self._has_fused_shared
|
||||
@@ -964,14 +955,12 @@ class FusedMoE(torch.nn.Module):
|
||||
expert_data=weight_data,
|
||||
shard_id=shard_id,
|
||||
loaded_weight=fp4_weight,
|
||||
tp_rank=tp_rank,
|
||||
)
|
||||
self._load_model_weight_or_group_weight_scale(
|
||||
shard_dim=shard_dim,
|
||||
expert_data=scale_data,
|
||||
shard_id=shard_id,
|
||||
loaded_weight=fp4_scale,
|
||||
tp_rank=tp_rank,
|
||||
)
|
||||
return True
|
||||
|
||||
@@ -989,7 +978,6 @@ class FusedMoE(torch.nn.Module):
|
||||
expert_data: torch.Tensor,
|
||||
shard_dim: int,
|
||||
loaded_weight: torch.Tensor,
|
||||
tp_rank: int,
|
||||
):
|
||||
if shard_id == "w2":
|
||||
self._load_w2(
|
||||
@@ -997,7 +985,6 @@ class FusedMoE(torch.nn.Module):
|
||||
shard_dim=shard_dim,
|
||||
loaded_weight=loaded_weight,
|
||||
expert_data=expert_data,
|
||||
tp_rank=tp_rank,
|
||||
)
|
||||
else:
|
||||
assert shard_id in ("w1", "w3")
|
||||
@@ -1124,7 +1111,6 @@ class FusedMoE(torch.nn.Module):
|
||||
loaded_weight: torch.Tensor,
|
||||
shard_id: str,
|
||||
expert_id: int,
|
||||
tp_rank: int,
|
||||
) -> bool:
|
||||
"""Handle GGUF weight loading.
|
||||
|
||||
@@ -1133,7 +1119,6 @@ class FusedMoE(torch.nn.Module):
|
||||
loaded_weight: The weight tensor to load.
|
||||
shard_id: The shard ID (w1, w2, or w3).
|
||||
expert_id: The expert ID.
|
||||
tp_rank: The tensor parallel rank.
|
||||
|
||||
Returns:
|
||||
True if the weight was handled as a GGUF weight, False otherwise.
|
||||
@@ -1151,7 +1136,7 @@ class FusedMoE(torch.nn.Module):
|
||||
if self.moe_tp_size > 1:
|
||||
if shard_id in ["w1", "w3", "w2"] and output_dim == 0:
|
||||
shard_size = loaded_weight.size(0) // self.moe_tp_size
|
||||
start_idx = tp_rank * shard_size
|
||||
start_idx = self.moe_tp_rank * shard_size
|
||||
loaded_weight = loaded_weight.narrow(
|
||||
0, start_idx, shard_size
|
||||
).clone()
|
||||
@@ -1175,10 +1160,8 @@ class FusedMoE(torch.nn.Module):
|
||||
shard_id: str,
|
||||
expert_id: int,
|
||||
) -> None:
|
||||
tp_rank = self.moe_tp_rank
|
||||
|
||||
# Special case for GGUF weights
|
||||
if self._load_gguf_weight(param, loaded_weight, shard_id, expert_id, tp_rank):
|
||||
if self._load_gguf_weight(param, loaded_weight, shard_id, expert_id):
|
||||
return
|
||||
|
||||
# compressed-tensors checkpoints with packed weights are stored flipped
|
||||
@@ -1261,7 +1244,6 @@ class FusedMoE(torch.nn.Module):
|
||||
shard_id=shard_id,
|
||||
expert_id=expert_id,
|
||||
shard_dim=shard_dim,
|
||||
tp_rank=tp_rank,
|
||||
):
|
||||
return
|
||||
|
||||
@@ -1300,7 +1282,6 @@ class FusedMoE(torch.nn.Module):
|
||||
shard_id=shard_id,
|
||||
loaded_weight=loaded_weight,
|
||||
expert_data=expert_data,
|
||||
tp_rank=tp_rank,
|
||||
)
|
||||
return
|
||||
|
||||
@@ -1328,7 +1309,6 @@ class FusedMoE(torch.nn.Module):
|
||||
shard_dim=shard_dim,
|
||||
loaded_weight=loaded_weight,
|
||||
expert_data=expert_data,
|
||||
tp_rank=tp_rank,
|
||||
)
|
||||
return
|
||||
|
||||
@@ -1350,7 +1330,6 @@ class FusedMoE(torch.nn.Module):
|
||||
shard_dim=shard_dim,
|
||||
loaded_weight=loaded_weight,
|
||||
expert_data=expert_data,
|
||||
tp_rank=tp_rank,
|
||||
)
|
||||
elif quant_method in [
|
||||
FusedMoeWeightScaleSupported.GROUP.value,
|
||||
@@ -1361,7 +1340,6 @@ class FusedMoE(torch.nn.Module):
|
||||
shard_dim=shard_dim,
|
||||
loaded_weight=loaded_weight,
|
||||
expert_data=expert_data,
|
||||
tp_rank=tp_rank,
|
||||
)
|
||||
elif quant_method == FusedMoeWeightScaleSupported.TENSOR.value:
|
||||
# INT4-FP8 (INT4 MoE Weight, FP8 Compute): Adjust FP8 per-tensor scaling number for e4m3fnuz (AMD)
|
||||
@@ -1395,7 +1373,6 @@ class FusedMoE(torch.nn.Module):
|
||||
shard_dim=shard_dim,
|
||||
loaded_weight=loaded_weight,
|
||||
expert_data=expert_data,
|
||||
tp_rank=tp_rank,
|
||||
)
|
||||
return
|
||||
|
||||
@@ -1408,7 +1385,6 @@ class FusedMoE(torch.nn.Module):
|
||||
shard_dim=shard_dim,
|
||||
loaded_weight=loaded_weight,
|
||||
expert_data=expert_data,
|
||||
tp_rank=tp_rank,
|
||||
)
|
||||
|
||||
def weight_loader_fused(
|
||||
@@ -1418,8 +1394,6 @@ class FusedMoE(torch.nn.Module):
|
||||
weight_name: str,
|
||||
shard_id: str,
|
||||
) -> None:
|
||||
tp_rank = self.moe_tp_rank
|
||||
|
||||
# Mirror _weight_loader_impl: the trtllm bf16 prep reshapes expert weights
|
||||
# into block layout; hot weight updates must restore canonical shapes first.
|
||||
method = self.quant_method
|
||||
@@ -1509,7 +1483,6 @@ class FusedMoE(torch.nn.Module):
|
||||
shard_dim=shard_dim,
|
||||
loaded_weight=loaded_weight,
|
||||
expert_data=expert_data,
|
||||
tp_rank=tp_rank,
|
||||
is_bias=is_bias,
|
||||
)
|
||||
return
|
||||
|
||||
Reference in New Issue
Block a user