[Fix] Shard MoE WNA16 and Quark INT4-FP8 weights by the MoE placement (#41807)

This commit is contained in:
Cheng Wan
2026-09-30 13:25:38 -07:00
committed by GitHub
parent 3f20738846
commit e289989933
3 changed files with 148 additions and 12 deletions
@@ -9,6 +9,7 @@ from typing import TYPE_CHECKING, Any, Dict, List, Optional
import numpy as np
import torch
from sglang.srt.eplb.expert_location import get_global_expert_location_metadata
from sglang.srt.layers.moe import MoeRunner, MoeRunnerBackend, MoeRunnerConfig
from sglang.srt.layers.moe.moe_runner.triton import TritonMoeQuantInfo
from sglang.srt.layers.quantization.awq import AWQConfig
@@ -226,6 +227,23 @@ def is_layer_skipped_quant(prefix: str, modules_to_not_convert: List[str]):
return any(module_name in prefix for module_name in modules_to_not_convert)
def _local_expert_ids(layer, expert_id: int) -> List[int]:
"""This rank's slots for checkpoint expert ``expert_id``, as the layer maps it.
The qzeros are written here rather than through the layer's weight loader,
so they need the same expert placement: EPLB's physical replicas, then the
expert-parallel slice this rank stores.
"""
metadata = get_global_expert_location_metadata()
physical = (
[expert_id]
if metadata is None
else metadata.logical_to_all_physical(layer.layer_id, expert_id)
)
local = (layer._map_global_expert_id_to_local_expert_id(p) for p in physical)
return [i for i in local if 0 <= i < layer.num_local_experts]
class MoeWNA16Method(FusedMoEMethodBase):
"""Linear method for MOE WNA16 (W8A16/W4A16) quantization.
@@ -454,7 +472,8 @@ class MoeWNA16Method(FusedMoEMethodBase):
tp_group = get_parallel().tp_group
device = tp_group.device
tp_rank = get_parallel().tp_rank
# The qzeros are split into moe_tp_size shards, one per MoE-TP rank.
moe_tp_rank = layer.moe_tp_rank
loaded_weight = loaded_weight.to(device)
shard_size = layer.intermediate_size_per_partition
@@ -494,15 +513,18 @@ class MoeWNA16Method(FusedMoEMethodBase):
if "w13_qzeros" in weight_name:
tensor = loaded_weight.view(
layer.moe_tp_size, -1, loaded_weight.size(1)
)[tp_rank]
if shard_id == "w1":
param.data[expert_id, : shard_size // 2] = tensor
else:
param.data[expert_id, shard_size // 2 :] = tensor
)[moe_tp_rank]
for local_expert_id in _local_expert_ids(layer, expert_id):
if shard_id == "w1":
param.data[local_expert_id, : shard_size // 2] = tensor
else:
param.data[local_expert_id, shard_size // 2 :] = tensor
elif "w2_qzeros" in weight_name:
param.data[expert_id] = loaded_weight.view(
tensor = loaded_weight.view(
loaded_weight.size(0), layer.moe_tp_size, -1
)[:, tp_rank]
)[:, moe_tp_rank]
for local_expert_id in _local_expert_ids(layer, expert_id):
param.data[local_expert_id] = tensor
else:
weight_loader(param, loaded_weight, weight_name, shard_id, expert_id)
@@ -138,8 +138,6 @@ class QuarkInt4Fp8MoEMethod(FusedMoEMethodBase):
self.online_quant_progress_bar = self.quant_config.online_quant_progress_bar
self.tp_rank = get_parallel().tp_rank
if not _is_hip:
raise NotImplementedError(
"The quark_int4fp8_moe online quantization scheme is only supported on AMD GPUs."
@@ -170,12 +168,12 @@ class QuarkInt4Fp8MoEMethod(FusedMoEMethodBase):
if shard_id in ["w1", "w3"]:
shard_dim = 0
loaded_weight = loaded_weight.narrow(
shard_dim, shard_size * self.tp_rank, shard_size
shard_dim, shard_size * layer.moe_tp_rank, shard_size
)
else:
shard_dim = 1
loaded_weight = loaded_weight.narrow(
shard_dim, shard_size * self.tp_rank, shard_size
shard_dim, shard_size * layer.moe_tp_rank, shard_size
)
# We want to run online quantization on-device for speed purposes.