diff --git a/python/sglang/kernels/ops/moe/rocm_router_gate.py b/python/sglang/kernels/ops/moe/rocm_router_gate.py index ac747c85c8..f30c533007 100644 --- a/python/sglang/kernels/ops/moe/rocm_router_gate.py +++ b/python/sglang/kernels/ops/moe/rocm_router_gate.py @@ -224,11 +224,13 @@ def _router_gate_kernel( stride_pm, stride_om, routed_scaling_factor, + shared_id_base, SPLIT_K: tl.constexpr, WRITE_LOGITS: tl.constexpr, HAS_BIAS: tl.constexpr, RENORM: tl.constexpr, TOPK: tl.constexpr, + NUM_SHARED: tl.constexpr, ): row = tl.program_id(0) weights, ids = _gate_row( @@ -247,7 +249,12 @@ def _router_gate_kernel( TOPK, ) lane = tl.arange(0, 64) - out_mask = lane < TOPK + if NUM_SHARED > 0: + # fused_append_shared_experts: shared slot s holds id shared_id_base + s, weight 1.0 + is_shared = lane >= TOPK + weights = tl.where(is_shared, 1.0, weights) + ids = tl.where(is_shared, shared_id_base + lane - TOPK, ids) + out_mask = lane < TOPK + NUM_SHARED tl.store(weights_ptr + row * stride_om + lane, weights, mask=out_mask) tl.store(ids_ptr + row * stride_om + lane, ids, mask=out_mask) @@ -260,12 +267,15 @@ def rocm_router_gate( routed_scaling_factor: Optional[float], *, partials: Optional[torch.Tensor] = None, + num_shared: int = 0, ) -> Tuple[torch.Tensor, torch.Tensor]: """aiter topk_gating(..., score_func="sqrtsoftplus") for 384 experts: fp32 weights and int32 ids [M, topk]; with partials their fixed-order sum is gated and written into - gating_output.""" + gating_output. num_shared > 0 appends fused_append_shared_experts(..., scale_factor=1.0, + N=384) columns: ids and weights become [M, topk + num_shared].""" M, num_experts = gating_output.shape assert num_experts == _GATE_NUM_EXPERTS and 0 < topk <= _MAX_TOPK + assert 0 <= num_shared <= _WARP - topk assert gating_output.stride(1) == 1 if partials is not None: assert ( @@ -282,8 +292,9 @@ def rocm_router_gate( assert ( correction_bias.shape == (num_experts,) and correction_bias.is_contiguous() ) - weights = torch.empty((M, topk), dtype=torch.float32, device=gating_output.device) - ids = torch.empty((M, topk), dtype=torch.int32, device=gating_output.device) + width = topk + num_shared + weights = torch.empty((M, width), dtype=torch.float32, device=gating_output.device) + ids = torch.empty((M, width), dtype=torch.int32, device=gating_output.device) if M == 0: return weights, ids _router_gate_kernel[(M,)]( @@ -297,11 +308,13 @@ def rocm_router_gate( stride_pm, weights.stride(0), float(1.0 if routed_scaling_factor is None else routed_scaling_factor), + num_experts, SPLIT_K=split_k, WRITE_LOGITS=split_k > 0, HAS_BIAS=correction_bias is not None, RENORM=bool(renormalize), TOPK=topk, + NUM_SHARED=num_shared, num_warps=1, ) return weights, ids diff --git a/python/sglang/srt/environ.py b/python/sglang/srt/environ.py index 5a9a98b184..483771531c 100644 --- a/python/sglang/srt/environ.py +++ b/python/sglang/srt/environ.py @@ -887,6 +887,10 @@ class Envs: # (matches `gate_mode="separated"`, the layout used by gptoss_fp4 tuned # configs and by Mxfp4MoEMethod's post-fix weight shuffle). SGLANG_USE_AITER_MOE_GU_ITLV = EnvBool(True) + # aiter opus moe_sorting dispatch policy (0 auto, 1 oneshot, 2 multi-phase). Auto picks + # oneshot below ~24 tokens, which on gfx950 costs 11-16 us vs 6-7 us for multi-phase at + # E=385 / 129; outputs are identical and multi-phase is never slower up to 16384 tokens. + SGLANG_AITER_MOE_SORTING_DISPATCH_POLICY = EnvInt(2) # Fold `silu(gate) * up` into the triton MoE up-GEMM epilogue. W13 rows are # permuted in place at load so gate/up land in adjacent columns of the same # output tile, which removes intermediate_cache1 and the standalone diff --git a/python/sglang/srt/layers/moe/moe_runner/aiter.py b/python/sglang/srt/layers/moe/moe_runner/aiter.py index 44ec32f574..34d54a8a69 100644 --- a/python/sglang/srt/layers/moe/moe_runner/aiter.py +++ b/python/sglang/srt/layers/moe/moe_runner/aiter.py @@ -12,6 +12,7 @@ import torch _SMALLM_MOE_ON = os.environ.get("SGLANG_ROCM_SMALLM_MOE", "1") != "0" +from sglang.srt.environ import envs from sglang.srt.layers.moe.moe_runner.base import ( MoeQuantInfo, MoeRunnerConfig, @@ -43,6 +44,8 @@ if TYPE_CHECKING: logger = logging.getLogger(__name__) +_MOE_SORTING_DISPATCH_POLICY = envs.SGLANG_AITER_MOE_SORTING_DISPATCH_POLICY.get() + class AiterQuantType(str, Enum): NONE = "No" @@ -327,6 +330,7 @@ class AiterRunnerCore(MoeRunnerCore): extra["swiglu_limit"] = quant_info.swiglu_limit if self.config.no_combine: extra["no_combine"] = True + extra["moe_sorting_dispatch_policy"] = _MOE_SORTING_DISPATCH_POLICY # gfx950 small-M MXFP4 kernel (on by default, SGLANG_ROCM_SMALLM_MOE=0 disables): same layouts as aiter, bf16 activations. if _SMALLM_MOE_ON and quant_info.w13_weight.element_size() == 1: diff --git a/python/sglang/srt/layers/moe/topk.py b/python/sglang/srt/layers/moe/topk.py index 25c6a34a07..c8a1eceb4d 100644 --- a/python/sglang/srt/layers/moe/topk.py +++ b/python/sglang/srt/layers/moe/topk.py @@ -1441,8 +1441,12 @@ def biased_topk_jit_kernel_impl( packed_out: Optional[torch.Tensor] = None, sqrtsoftplus_log1p: bool = False, router_logits_partials: Optional[torch.Tensor] = None, + num_shared_append: int = 0, ) -> Tuple[torch.Tensor, torch.Tensor]: + """num_shared_append is only workable when router_logits_partials is not None + and it is only used by rocm_router_gate""" assert hidden_states.shape[0] == gating_output.shape[0], "Number of tokens mismatch" + assert num_shared_append == 0 or router_logits_partials is not None if _use_aiter and scoring_func == "sqrtsoftplus" and num_fused_shared_experts == 0: assert packed_out is None, "aiter topk_gating cannot emit packed ids" @@ -1455,6 +1459,7 @@ def biased_topk_jit_kernel_impl( renormalize, routed_scaling_factor, partials=router_logits_partials, + num_shared=num_shared_append, ) from aiter import topk_gating @@ -2274,7 +2279,17 @@ def _post_process_topk_ids( fused_shared_experts_scaling_factor = ( topk_config.fused_shared_experts_scaling_factor ) - capture_routed_experts_if_allowed(topk_config, layer_id, topk_ids) + # The router already wrote the aiter shared columns (see _aiter_append below). + _gate_appended = ( + num_fused_shared_experts > 0 + and _use_aiter + and not use_per_rank_shared_slots + and topk_ids.shape[-1] == topk_config.top_k + ) + routed_topk_ids = ( + topk_ids[:, :-num_fused_shared_experts] if _gate_appended else topk_ids + ) + capture_routed_experts_if_allowed(topk_config, layer_id, routed_topk_ids) recorder_topk_ids = None _fold_pad_into_append = False recorder_was_fused = False @@ -2378,13 +2393,17 @@ def _post_process_topk_ids( # second zeroing here would be redundant (zeroing is idempotent). if recorder_topk_ids is None and not recorder_was_fused: - recorder_topk_ids = topk_ids + recorder_topk_ids = ( + topk_ids[:, :-num_fused_shared_experts] if _gate_appended else topk_ids + ) - _aiter_append = num_fused_shared_experts > 0 and _use_aiter - if _aiter_append and envs.SGLANG_OPT_USE_JIT_KERNEL_GROUPED_TOPK.get(): - # That router emits the shared slots itself; appending again would write - # the shared id twice and evict a real routed expert. - _aiter_append = topk_ids.shape[-1] < topk_config.top_k + # The JIT grouped router and the ROCm decode gate emit the shared slots themselves; + # appending again would write the shared id twice and evict a real routed expert. + _aiter_append = ( + num_fused_shared_experts > 0 + and _use_aiter + and topk_ids.shape[-1] < topk_config.top_k + ) if _aiter_append and use_per_rank_shared_slots: # Fused path: append shared experts AND apply the per-rank shared-slot @@ -2536,6 +2555,7 @@ def select_experts( and ( num_fused_shared_experts == 0 or has_per_rank_fused_shared_slots(num_fused_shared_experts) + or not _eplb_remap_enabled() ) ): # only the aiter sqrtsoftplus gate takes the partials; every other route reads router_logits @@ -2668,6 +2688,13 @@ def select_experts( _packed_kwargs["sqrtsoftplus_log1p"] = True if router_logits_partials is not None: _packed_kwargs["router_logits_partials"] = router_logits_partials + if ( + _use_aiter + and num_fused_shared_experts > 0 + and not has_per_rank_fused_shared_slots(num_fused_shared_experts) + ): + # the ROCm decode gate writes the aiter shared columns itself + _packed_kwargs["num_shared_append"] = num_fused_shared_experts topk_weights, topk_ids = _biased_topk( hidden_states=hidden_states, gating_output=router_logits, diff --git a/python/sglang/srt/layers/quantization/fp8_utils.py b/python/sglang/srt/layers/quantization/fp8_utils.py index d69e691b84..e7d10b0a8c 100755 --- a/python/sglang/srt/layers/quantization/fp8_utils.py +++ b/python/sglang/srt/layers/quantization/fp8_utils.py @@ -1729,6 +1729,10 @@ def quantize_block_fp8_weight_to_mxfp4( weight_block_size: List[int], mxfp4_block_size: int = 32, ) -> Tuple[torch.Tensor, torch.Tensor]: + if _is_hip: + return _quantize_block_fp8_weight_to_mxfp4_rne( + fp8_weight, fp8_scale, weight_block_size, mxfp4_block_size + ) fp8_weight_dequant = block_quant_dequant( fp8_weight, fp8_scale.to(torch.float32), @@ -1747,6 +1751,44 @@ def quantize_block_fp8_weight_to_mxfp4( return fp4_weight, fp4_scale.contiguous().view(torch.float8_e8m0fnu) +def _quantize_block_fp8_weight_to_mxfp4_rne( + fp8_weight: torch.Tensor, + fp8_scale: torch.Tensor, + weight_block_size: List[int], + mxfp4_block_size: int, +) -> Tuple[torch.Tensor, torch.Tensor]: + fp8_weight_dequant = block_quant_dequant( + fp8_weight, + fp8_scale.to(torch.float32), + weight_block_size, + torch.float32, + ) + *lead, k = fp8_weight_dequant.shape + x = fp8_weight_dequant.reshape(-1, mxfp4_block_size) + + # Scale as in aiter dynamic_mxfp4_quant: round amax's mantissa at 1.75, then + # floor(log2) - 2 (e2m1 emax), saturating the few elements above 6 * scale. + amax = x.abs().amax(dim=-1, keepdim=True).clamp_min(torch.finfo(torch.float32).tiny) + amax = ((amax.view(torch.int32) + 0x200000) & 0x7F800000).view(torch.float32) + exp = (torch.floor(torch.log2(amax)) - 2).clamp(-127, 127) + y = (x.abs() / torch.exp2(exp)).clamp(max=6.0) + + # Round to nearest, ties to even mantissa. The FP8 source puts many elements + # exactly on e2m1 midpoints; ties toward zero shrink each matrix by ~5%. + mag = torch.where( + y < 2.0, + torch.round(y * 2.0), + torch.where(y < 4.0, torch.round(y) + 2.0, torch.round(y * 0.5) + 4.0), + ).to(torch.uint8) + codes = mag | ((x < 0) & (mag != 0)).to(torch.uint8) << 3 + codes = codes.reshape(*lead, k) + fp4_weight = (codes[..., 0::2] | (codes[..., 1::2] << 4)).contiguous() + fp4_scale = (exp + 127).to(torch.uint8).reshape(*lead, k // mxfp4_block_size) + return fp4_weight.view(torch.int8), fp4_scale.contiguous().view( + torch.float8_e8m0fnu + ) + + def requant_weight_ue8m0_inplace(weight, weight_scale_inv, weight_block_size): assert isinstance(weight, torch.nn.Parameter) assert isinstance(weight_scale_inv, torch.nn.Parameter) diff --git a/test/registered/kernels/ops/moe/test_rocm_router_gate.py b/test/registered/kernels/ops/moe/test_rocm_router_gate.py index a252b02dfa..ea89ae2fdc 100644 --- a/test/registered/kernels/ops/moe/test_rocm_router_gate.py +++ b/test/registered/kernels/ops/moe/test_rocm_router_gate.py @@ -98,6 +98,135 @@ class TestRocmRouterGate(CustomTestCase): self.assertTrue(torch.equal(out_i, ref_i)) self.assertTrue(torch.equal(out_w, ref_w)) + def test_gate_shared_append_matches_aiter_plus_append(self): + from sglang.kernels.ops.moe.fused_moe_triton_kernels import ( + fused_append_shared_experts, + ) + + weight = (self._randn(NUM_EXPERTS, HIDDEN) * 0.02).to(torch.bfloat16) + for num_tokens in (1, 6, 64): + x = self._randn(num_tokens, HIDDEN).to(torch.bfloat16) + partials = rocm_router_gemv_split_k(x, weight) + logits = torch.empty(num_tokens, NUM_EXPERTS, device=self.device) + rocm_router_reduce_partials(partials, logits) + ref_w, ref_i = _aiter_gate(logits, self.bias_bf16, TOPK, True, 1.5) + ref_i, ref_w = fused_append_shared_experts( + ref_i, ref_w, 1, 1.0, NUM_EXPERTS + ) + fused_logits = torch.empty_like(logits) + out_w, out_i = rocm_router_gate( + fused_logits, + self.bias_bf16, + TOPK, + True, + 1.5, + partials=partials, + num_shared=1, + ) + self.assertTrue(torch.equal(ref_i, out_i), f"ids M={num_tokens}") + self.assertTrue(torch.equal(ref_w, out_w), f"weights M={num_tokens}") + self.assertTrue(torch.equal(fused_logits, logits), f"logits M={num_tokens}") + + def test_select_experts_partials_with_fused_shared_expert(self): + from sglang.srt.layers.moe.topk import TopKConfig, select_experts + + weight = (self._randn(NUM_EXPERTS, HIDDEN) * 0.02).to(torch.bfloat16) + for scaling in (None, 0.5): + config = TopKConfig( + top_k=TOPK + 1, + renormalize=True, + num_fused_shared_experts=1, + correction_bias=self.bias_bf16, + routed_scaling_factor=1.5, + fused_shared_experts_scaling_factor=scaling, + scoring_func="sqrtsoftplus", + ) + for num_tokens in (1, 6, 64): + x = self._randn(num_tokens, HIDDEN).to(torch.bfloat16) + partials = rocm_router_gemv_split_k(x, weight) + logits = torch.empty(num_tokens, NUM_EXPERTS, device=self.device) + rocm_router_reduce_partials(partials, logits) + ref = select_experts(x, logits.clone(), config) + out = select_experts( + x, + torch.empty_like(logits), + config, + router_logits_partials=partials, + ) + msg = f"M={num_tokens} scaling={scaling}" + self.assertEqual(out.topk_ids.shape, (num_tokens, TOPK + 1), msg) + self.assertTrue(torch.equal(ref.topk_ids, out.topk_ids), msg) + self.assertTrue(torch.equal(ref.topk_weights, out.topk_weights), msg) + self.assertTrue(torch.equal(ref.router_logits, out.router_logits), msg) + + +@unittest.skipUnless( + is_hip() and is_gfx95_supported() and aiter_topk_gating is not None, + "ROCm gfx95 + aiter only", +) +class TestAiterMoeSortingDispatchPolicy(CustomTestCase): + """SGLANG_AITER_MOE_SORTING_DISPATCH_POLICY=2 must sort exactly like aiter's auto policy.""" + + def _sort(self, ids, weights, num_experts, policy, block_m=32, hidden=HIDDEN): + import aiter + + m, topk = ids.shape + padded = ids.numel() + num_experts * block_m - topk + blocks = (padded + block_m - 1) // block_m + sorted_ids = torch.empty(blocks * block_m, dtype=torch.int32, device="cuda") + sorted_weights = torch.empty(blocks * block_m, device="cuda") + sorted_experts = torch.empty(blocks, dtype=torch.int32, device="cuda") + num_valid = torch.empty(2, dtype=torch.int32, device="cuda") + moe_buf = torch.ones(m, hidden, dtype=torch.bfloat16, device="cuda") + ws = aiter.moe_sorting_opus_get_workspace_size(m, num_experts, topk, policy) + workspace = ( + torch.empty(ws, dtype=torch.uint8, device="cuda") if ws > 0 else None + ) + aiter.moe_sorting_opus_fwd( + ids, + weights, + sorted_ids, + sorted_weights, + sorted_experts, + num_valid, + moe_buf, + num_experts, + block_m, + None, + None, + workspace, + policy, + None, + None, + None, + ) + n = int(num_valid[0]) + return ( + num_valid.clone(), + sorted_ids[:n].clone(), + sorted_weights[:n].clone(), + sorted_experts[: n // block_m].clone(), + bool((moe_buf == 0).all()), + ) + + def test_multi_phase_matches_auto(self): + gen = torch.Generator(device="cuda").manual_seed(0) + for num_experts, topk in ((385, 7), (129, 4)): + routed = num_experts - 1 + for m in (1, 6, 16, 64, 1024): + ids = torch.argsort( + torch.rand(m, routed, device="cuda", generator=gen), dim=1 + )[:, : topk - 1].int() + shared = torch.full((m, 1), routed, dtype=torch.int32, device="cuda") + ids = torch.cat([ids, shared], dim=1).contiguous() + weights = torch.rand(m, topk, device="cuda", generator=gen) + auto = self._sort(ids, weights, num_experts, policy=0) + multi_phase = self._sort(ids, weights, num_experts, policy=2) + msg = f"E={num_experts} topk={topk} M={m}" + self.assertTrue(auto[4] and multi_phase[4], f"moe_buf zeroed {msg}") + for a, b in zip(auto[:4], multi_phase[:4]): + self.assertTrue(torch.equal(a, b), msg) + if __name__ == "__main__": unittest.main() diff --git a/test/registered/unit/models/test_deepseek_v4_mxfp4_shared_expert_requant.py b/test/registered/unit/models/test_deepseek_v4_mxfp4_shared_expert_requant.py index 88ecb9e9a6..db9315fa45 100644 --- a/test/registered/unit/models/test_deepseek_v4_mxfp4_shared_expert_requant.py +++ b/test/registered/unit/models/test_deepseek_v4_mxfp4_shared_expert_requant.py @@ -1,7 +1,9 @@ import unittest +from unittest import mock import torch +from sglang.srt.layers.quantization import fp8_utils from sglang.srt.layers.quantization.fp8_utils import quantize_block_fp8_weight_to_mxfp4 from sglang.test.ci.ci_register import register_cpu_ci from sglang.test.test_utils import CustomTestCase @@ -64,12 +66,25 @@ class TestQuantizeBlockFp8WeightToMxfp4(CustomTestCase): self.assertEqual(as_u8[0, 0].item(), 0x01 | (0x0D << 4)) # byte 1 = code(6.0) | code(1.5) << 4 self.assertEqual(as_u8[0, 1].item(), 0x07 | (0x03 << 4)) - # Zero padding is not asserted byte-exactly: the quantizer encodes 0.0 - # as -0.0 (code 8), which the kernel decodes back to zero. Check the - # padding dequantizes to zero without pinning its sign bit. + # Zero padding dequantizes to zero; its sign bit is not part of the contract. deq = _dequant_mxfp4(packed, scales) self.assertTrue((deq[0, 4:] == 0).all()) + @mock.patch.object(fp8_utils, "_is_hip", True) + def test_ties_round_to_even(self): + # FP8 sources land exactly on e2m1 midpoints; rounding them toward zero + # biases the whole matrix low, so ROCm rounds ties to the even mantissa. + w = torch.zeros(1, 32, dtype=torch.bfloat16) + w[0, 0] = 6.0 # pins scale 2**0 + ties = [0.25, 0.75, 1.25, 1.75, 2.5, 3.5, 5.0] + expected = [0.0, 1.0, 1.0, 2.0, 2.0, 4.0, 4.0] + w[0, 1 : 1 + len(ties)] = torch.tensor(ties) + w[0, 8 : 8 + len(ties)] = -torch.tensor(ties) + packed, scales = self._requant(w) + deq = _dequant_mxfp4(packed, scales) + torch.testing.assert_close(deq[0, 1 : 1 + len(ties)], torch.tensor(expected)) + torch.testing.assert_close(deq[0, 8 : 8 + len(ties)], -torch.tensor(expected)) + def test_roundtrip_error_is_mxfp4_sized(self): torch.manual_seed(0) w = (torch.randn(64, 128) * 0.1).to(torch.bfloat16)