[Fix] Shard MoE WNA16 and Quark INT4-FP8 weights by the MoE placement (#41807)
This commit is contained in:
@@ -9,6 +9,7 @@ from typing import TYPE_CHECKING, Any, Dict, List, Optional
|
||||
import numpy as np
|
||||
import torch
|
||||
|
||||
from sglang.srt.eplb.expert_location import get_global_expert_location_metadata
|
||||
from sglang.srt.layers.moe import MoeRunner, MoeRunnerBackend, MoeRunnerConfig
|
||||
from sglang.srt.layers.moe.moe_runner.triton import TritonMoeQuantInfo
|
||||
from sglang.srt.layers.quantization.awq import AWQConfig
|
||||
@@ -226,6 +227,23 @@ def is_layer_skipped_quant(prefix: str, modules_to_not_convert: List[str]):
|
||||
return any(module_name in prefix for module_name in modules_to_not_convert)
|
||||
|
||||
|
||||
def _local_expert_ids(layer, expert_id: int) -> List[int]:
|
||||
"""This rank's slots for checkpoint expert ``expert_id``, as the layer maps it.
|
||||
|
||||
The qzeros are written here rather than through the layer's weight loader,
|
||||
so they need the same expert placement: EPLB's physical replicas, then the
|
||||
expert-parallel slice this rank stores.
|
||||
"""
|
||||
metadata = get_global_expert_location_metadata()
|
||||
physical = (
|
||||
[expert_id]
|
||||
if metadata is None
|
||||
else metadata.logical_to_all_physical(layer.layer_id, expert_id)
|
||||
)
|
||||
local = (layer._map_global_expert_id_to_local_expert_id(p) for p in physical)
|
||||
return [i for i in local if 0 <= i < layer.num_local_experts]
|
||||
|
||||
|
||||
class MoeWNA16Method(FusedMoEMethodBase):
|
||||
"""Linear method for MOE WNA16 (W8A16/W4A16) quantization.
|
||||
|
||||
@@ -454,7 +472,8 @@ class MoeWNA16Method(FusedMoEMethodBase):
|
||||
|
||||
tp_group = get_parallel().tp_group
|
||||
device = tp_group.device
|
||||
tp_rank = get_parallel().tp_rank
|
||||
# The qzeros are split into moe_tp_size shards, one per MoE-TP rank.
|
||||
moe_tp_rank = layer.moe_tp_rank
|
||||
loaded_weight = loaded_weight.to(device)
|
||||
shard_size = layer.intermediate_size_per_partition
|
||||
|
||||
@@ -494,15 +513,18 @@ class MoeWNA16Method(FusedMoEMethodBase):
|
||||
if "w13_qzeros" in weight_name:
|
||||
tensor = loaded_weight.view(
|
||||
layer.moe_tp_size, -1, loaded_weight.size(1)
|
||||
)[tp_rank]
|
||||
if shard_id == "w1":
|
||||
param.data[expert_id, : shard_size // 2] = tensor
|
||||
else:
|
||||
param.data[expert_id, shard_size // 2 :] = tensor
|
||||
)[moe_tp_rank]
|
||||
for local_expert_id in _local_expert_ids(layer, expert_id):
|
||||
if shard_id == "w1":
|
||||
param.data[local_expert_id, : shard_size // 2] = tensor
|
||||
else:
|
||||
param.data[local_expert_id, shard_size // 2 :] = tensor
|
||||
elif "w2_qzeros" in weight_name:
|
||||
param.data[expert_id] = loaded_weight.view(
|
||||
tensor = loaded_weight.view(
|
||||
loaded_weight.size(0), layer.moe_tp_size, -1
|
||||
)[:, tp_rank]
|
||||
)[:, moe_tp_rank]
|
||||
for local_expert_id in _local_expert_ids(layer, expert_id):
|
||||
param.data[local_expert_id] = tensor
|
||||
else:
|
||||
weight_loader(param, loaded_weight, weight_name, shard_id, expert_id)
|
||||
|
||||
|
||||
@@ -138,8 +138,6 @@ class QuarkInt4Fp8MoEMethod(FusedMoEMethodBase):
|
||||
|
||||
self.online_quant_progress_bar = self.quant_config.online_quant_progress_bar
|
||||
|
||||
self.tp_rank = get_parallel().tp_rank
|
||||
|
||||
if not _is_hip:
|
||||
raise NotImplementedError(
|
||||
"The quark_int4fp8_moe online quantization scheme is only supported on AMD GPUs."
|
||||
@@ -170,12 +168,12 @@ class QuarkInt4Fp8MoEMethod(FusedMoEMethodBase):
|
||||
if shard_id in ["w1", "w3"]:
|
||||
shard_dim = 0
|
||||
loaded_weight = loaded_weight.narrow(
|
||||
shard_dim, shard_size * self.tp_rank, shard_size
|
||||
shard_dim, shard_size * layer.moe_tp_rank, shard_size
|
||||
)
|
||||
else:
|
||||
shard_dim = 1
|
||||
loaded_weight = loaded_weight.narrow(
|
||||
shard_dim, shard_size * self.tp_rank, shard_size
|
||||
shard_dim, shard_size * layer.moe_tp_rank, shard_size
|
||||
)
|
||||
|
||||
# We want to run online quantization on-device for speed purposes.
|
||||
|
||||
Reference in New Issue
Block a user