[AMD][V4.1][*/N] Fix shared-expert fusion accuracy and speed up MoE routing on ROCm (#42011)
Co-authored-by: wunhuang <wunhuang@amd.com>
This commit is contained in:
@@ -224,11 +224,13 @@ def _router_gate_kernel(
|
||||
stride_pm,
|
||||
stride_om,
|
||||
routed_scaling_factor,
|
||||
shared_id_base,
|
||||
SPLIT_K: tl.constexpr,
|
||||
WRITE_LOGITS: tl.constexpr,
|
||||
HAS_BIAS: tl.constexpr,
|
||||
RENORM: tl.constexpr,
|
||||
TOPK: tl.constexpr,
|
||||
NUM_SHARED: tl.constexpr,
|
||||
):
|
||||
row = tl.program_id(0)
|
||||
weights, ids = _gate_row(
|
||||
@@ -247,7 +249,12 @@ def _router_gate_kernel(
|
||||
TOPK,
|
||||
)
|
||||
lane = tl.arange(0, 64)
|
||||
out_mask = lane < TOPK
|
||||
if NUM_SHARED > 0:
|
||||
# fused_append_shared_experts: shared slot s holds id shared_id_base + s, weight 1.0
|
||||
is_shared = lane >= TOPK
|
||||
weights = tl.where(is_shared, 1.0, weights)
|
||||
ids = tl.where(is_shared, shared_id_base + lane - TOPK, ids)
|
||||
out_mask = lane < TOPK + NUM_SHARED
|
||||
tl.store(weights_ptr + row * stride_om + lane, weights, mask=out_mask)
|
||||
tl.store(ids_ptr + row * stride_om + lane, ids, mask=out_mask)
|
||||
|
||||
@@ -260,12 +267,15 @@ def rocm_router_gate(
|
||||
routed_scaling_factor: Optional[float],
|
||||
*,
|
||||
partials: Optional[torch.Tensor] = None,
|
||||
num_shared: int = 0,
|
||||
) -> Tuple[torch.Tensor, torch.Tensor]:
|
||||
"""aiter topk_gating(..., score_func="sqrtsoftplus") for 384 experts: fp32 weights and int32
|
||||
ids [M, topk]; with partials their fixed-order sum is gated and written into
|
||||
gating_output."""
|
||||
gating_output. num_shared > 0 appends fused_append_shared_experts(..., scale_factor=1.0,
|
||||
N=384) columns: ids and weights become [M, topk + num_shared]."""
|
||||
M, num_experts = gating_output.shape
|
||||
assert num_experts == _GATE_NUM_EXPERTS and 0 < topk <= _MAX_TOPK
|
||||
assert 0 <= num_shared <= _WARP - topk
|
||||
assert gating_output.stride(1) == 1
|
||||
if partials is not None:
|
||||
assert (
|
||||
@@ -282,8 +292,9 @@ def rocm_router_gate(
|
||||
assert (
|
||||
correction_bias.shape == (num_experts,) and correction_bias.is_contiguous()
|
||||
)
|
||||
weights = torch.empty((M, topk), dtype=torch.float32, device=gating_output.device)
|
||||
ids = torch.empty((M, topk), dtype=torch.int32, device=gating_output.device)
|
||||
width = topk + num_shared
|
||||
weights = torch.empty((M, width), dtype=torch.float32, device=gating_output.device)
|
||||
ids = torch.empty((M, width), dtype=torch.int32, device=gating_output.device)
|
||||
if M == 0:
|
||||
return weights, ids
|
||||
_router_gate_kernel[(M,)](
|
||||
@@ -297,11 +308,13 @@ def rocm_router_gate(
|
||||
stride_pm,
|
||||
weights.stride(0),
|
||||
float(1.0 if routed_scaling_factor is None else routed_scaling_factor),
|
||||
num_experts,
|
||||
SPLIT_K=split_k,
|
||||
WRITE_LOGITS=split_k > 0,
|
||||
HAS_BIAS=correction_bias is not None,
|
||||
RENORM=bool(renormalize),
|
||||
TOPK=topk,
|
||||
NUM_SHARED=num_shared,
|
||||
num_warps=1,
|
||||
)
|
||||
return weights, ids
|
||||
|
||||
@@ -887,6 +887,10 @@ class Envs:
|
||||
# (matches `gate_mode="separated"`, the layout used by gptoss_fp4 tuned
|
||||
# configs and by Mxfp4MoEMethod's post-fix weight shuffle).
|
||||
SGLANG_USE_AITER_MOE_GU_ITLV = EnvBool(True)
|
||||
# aiter opus moe_sorting dispatch policy (0 auto, 1 oneshot, 2 multi-phase). Auto picks
|
||||
# oneshot below ~24 tokens, which on gfx950 costs 11-16 us vs 6-7 us for multi-phase at
|
||||
# E=385 / 129; outputs are identical and multi-phase is never slower up to 16384 tokens.
|
||||
SGLANG_AITER_MOE_SORTING_DISPATCH_POLICY = EnvInt(2)
|
||||
# Fold `silu(gate) * up` into the triton MoE up-GEMM epilogue. W13 rows are
|
||||
# permuted in place at load so gate/up land in adjacent columns of the same
|
||||
# output tile, which removes intermediate_cache1 and the standalone
|
||||
|
||||
@@ -12,6 +12,7 @@ import torch
|
||||
|
||||
_SMALLM_MOE_ON = os.environ.get("SGLANG_ROCM_SMALLM_MOE", "1") != "0"
|
||||
|
||||
from sglang.srt.environ import envs
|
||||
from sglang.srt.layers.moe.moe_runner.base import (
|
||||
MoeQuantInfo,
|
||||
MoeRunnerConfig,
|
||||
@@ -43,6 +44,8 @@ if TYPE_CHECKING:
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_MOE_SORTING_DISPATCH_POLICY = envs.SGLANG_AITER_MOE_SORTING_DISPATCH_POLICY.get()
|
||||
|
||||
|
||||
class AiterQuantType(str, Enum):
|
||||
NONE = "No"
|
||||
@@ -327,6 +330,7 @@ class AiterRunnerCore(MoeRunnerCore):
|
||||
extra["swiglu_limit"] = quant_info.swiglu_limit
|
||||
if self.config.no_combine:
|
||||
extra["no_combine"] = True
|
||||
extra["moe_sorting_dispatch_policy"] = _MOE_SORTING_DISPATCH_POLICY
|
||||
|
||||
# gfx950 small-M MXFP4 kernel (on by default, SGLANG_ROCM_SMALLM_MOE=0 disables): same layouts as aiter, bf16 activations.
|
||||
if _SMALLM_MOE_ON and quant_info.w13_weight.element_size() == 1:
|
||||
|
||||
@@ -1441,8 +1441,12 @@ def biased_topk_jit_kernel_impl(
|
||||
packed_out: Optional[torch.Tensor] = None,
|
||||
sqrtsoftplus_log1p: bool = False,
|
||||
router_logits_partials: Optional[torch.Tensor] = None,
|
||||
num_shared_append: int = 0,
|
||||
) -> Tuple[torch.Tensor, torch.Tensor]:
|
||||
"""num_shared_append is only workable when router_logits_partials is not None
|
||||
and it is only used by rocm_router_gate"""
|
||||
assert hidden_states.shape[0] == gating_output.shape[0], "Number of tokens mismatch"
|
||||
assert num_shared_append == 0 or router_logits_partials is not None
|
||||
|
||||
if _use_aiter and scoring_func == "sqrtsoftplus" and num_fused_shared_experts == 0:
|
||||
assert packed_out is None, "aiter topk_gating cannot emit packed ids"
|
||||
@@ -1455,6 +1459,7 @@ def biased_topk_jit_kernel_impl(
|
||||
renormalize,
|
||||
routed_scaling_factor,
|
||||
partials=router_logits_partials,
|
||||
num_shared=num_shared_append,
|
||||
)
|
||||
|
||||
from aiter import topk_gating
|
||||
@@ -2274,7 +2279,17 @@ def _post_process_topk_ids(
|
||||
fused_shared_experts_scaling_factor = (
|
||||
topk_config.fused_shared_experts_scaling_factor
|
||||
)
|
||||
capture_routed_experts_if_allowed(topk_config, layer_id, topk_ids)
|
||||
# The router already wrote the aiter shared columns (see _aiter_append below).
|
||||
_gate_appended = (
|
||||
num_fused_shared_experts > 0
|
||||
and _use_aiter
|
||||
and not use_per_rank_shared_slots
|
||||
and topk_ids.shape[-1] == topk_config.top_k
|
||||
)
|
||||
routed_topk_ids = (
|
||||
topk_ids[:, :-num_fused_shared_experts] if _gate_appended else topk_ids
|
||||
)
|
||||
capture_routed_experts_if_allowed(topk_config, layer_id, routed_topk_ids)
|
||||
recorder_topk_ids = None
|
||||
_fold_pad_into_append = False
|
||||
recorder_was_fused = False
|
||||
@@ -2378,13 +2393,17 @@ def _post_process_topk_ids(
|
||||
# second zeroing here would be redundant (zeroing is idempotent).
|
||||
|
||||
if recorder_topk_ids is None and not recorder_was_fused:
|
||||
recorder_topk_ids = topk_ids
|
||||
recorder_topk_ids = (
|
||||
topk_ids[:, :-num_fused_shared_experts] if _gate_appended else topk_ids
|
||||
)
|
||||
|
||||
_aiter_append = num_fused_shared_experts > 0 and _use_aiter
|
||||
if _aiter_append and envs.SGLANG_OPT_USE_JIT_KERNEL_GROUPED_TOPK.get():
|
||||
# That router emits the shared slots itself; appending again would write
|
||||
# the shared id twice and evict a real routed expert.
|
||||
_aiter_append = topk_ids.shape[-1] < topk_config.top_k
|
||||
# The JIT grouped router and the ROCm decode gate emit the shared slots themselves;
|
||||
# appending again would write the shared id twice and evict a real routed expert.
|
||||
_aiter_append = (
|
||||
num_fused_shared_experts > 0
|
||||
and _use_aiter
|
||||
and topk_ids.shape[-1] < topk_config.top_k
|
||||
)
|
||||
|
||||
if _aiter_append and use_per_rank_shared_slots:
|
||||
# Fused path: append shared experts AND apply the per-rank shared-slot
|
||||
@@ -2536,6 +2555,7 @@ def select_experts(
|
||||
and (
|
||||
num_fused_shared_experts == 0
|
||||
or has_per_rank_fused_shared_slots(num_fused_shared_experts)
|
||||
or not _eplb_remap_enabled()
|
||||
)
|
||||
):
|
||||
# only the aiter sqrtsoftplus gate takes the partials; every other route reads router_logits
|
||||
@@ -2668,6 +2688,13 @@ def select_experts(
|
||||
_packed_kwargs["sqrtsoftplus_log1p"] = True
|
||||
if router_logits_partials is not None:
|
||||
_packed_kwargs["router_logits_partials"] = router_logits_partials
|
||||
if (
|
||||
_use_aiter
|
||||
and num_fused_shared_experts > 0
|
||||
and not has_per_rank_fused_shared_slots(num_fused_shared_experts)
|
||||
):
|
||||
# the ROCm decode gate writes the aiter shared columns itself
|
||||
_packed_kwargs["num_shared_append"] = num_fused_shared_experts
|
||||
topk_weights, topk_ids = _biased_topk(
|
||||
hidden_states=hidden_states,
|
||||
gating_output=router_logits,
|
||||
|
||||
@@ -1729,6 +1729,10 @@ def quantize_block_fp8_weight_to_mxfp4(
|
||||
weight_block_size: List[int],
|
||||
mxfp4_block_size: int = 32,
|
||||
) -> Tuple[torch.Tensor, torch.Tensor]:
|
||||
if _is_hip:
|
||||
return _quantize_block_fp8_weight_to_mxfp4_rne(
|
||||
fp8_weight, fp8_scale, weight_block_size, mxfp4_block_size
|
||||
)
|
||||
fp8_weight_dequant = block_quant_dequant(
|
||||
fp8_weight,
|
||||
fp8_scale.to(torch.float32),
|
||||
@@ -1747,6 +1751,44 @@ def quantize_block_fp8_weight_to_mxfp4(
|
||||
return fp4_weight, fp4_scale.contiguous().view(torch.float8_e8m0fnu)
|
||||
|
||||
|
||||
def _quantize_block_fp8_weight_to_mxfp4_rne(
|
||||
fp8_weight: torch.Tensor,
|
||||
fp8_scale: torch.Tensor,
|
||||
weight_block_size: List[int],
|
||||
mxfp4_block_size: int,
|
||||
) -> Tuple[torch.Tensor, torch.Tensor]:
|
||||
fp8_weight_dequant = block_quant_dequant(
|
||||
fp8_weight,
|
||||
fp8_scale.to(torch.float32),
|
||||
weight_block_size,
|
||||
torch.float32,
|
||||
)
|
||||
*lead, k = fp8_weight_dequant.shape
|
||||
x = fp8_weight_dequant.reshape(-1, mxfp4_block_size)
|
||||
|
||||
# Scale as in aiter dynamic_mxfp4_quant: round amax's mantissa at 1.75, then
|
||||
# floor(log2) - 2 (e2m1 emax), saturating the few elements above 6 * scale.
|
||||
amax = x.abs().amax(dim=-1, keepdim=True).clamp_min(torch.finfo(torch.float32).tiny)
|
||||
amax = ((amax.view(torch.int32) + 0x200000) & 0x7F800000).view(torch.float32)
|
||||
exp = (torch.floor(torch.log2(amax)) - 2).clamp(-127, 127)
|
||||
y = (x.abs() / torch.exp2(exp)).clamp(max=6.0)
|
||||
|
||||
# Round to nearest, ties to even mantissa. The FP8 source puts many elements
|
||||
# exactly on e2m1 midpoints; ties toward zero shrink each matrix by ~5%.
|
||||
mag = torch.where(
|
||||
y < 2.0,
|
||||
torch.round(y * 2.0),
|
||||
torch.where(y < 4.0, torch.round(y) + 2.0, torch.round(y * 0.5) + 4.0),
|
||||
).to(torch.uint8)
|
||||
codes = mag | ((x < 0) & (mag != 0)).to(torch.uint8) << 3
|
||||
codes = codes.reshape(*lead, k)
|
||||
fp4_weight = (codes[..., 0::2] | (codes[..., 1::2] << 4)).contiguous()
|
||||
fp4_scale = (exp + 127).to(torch.uint8).reshape(*lead, k // mxfp4_block_size)
|
||||
return fp4_weight.view(torch.int8), fp4_scale.contiguous().view(
|
||||
torch.float8_e8m0fnu
|
||||
)
|
||||
|
||||
|
||||
def requant_weight_ue8m0_inplace(weight, weight_scale_inv, weight_block_size):
|
||||
assert isinstance(weight, torch.nn.Parameter)
|
||||
assert isinstance(weight_scale_inv, torch.nn.Parameter)
|
||||
|
||||
Reference in New Issue
Block a user