From 9fde8bcc466f5f8b5c8222d266feff95551c71da Mon Sep 17 00:00:00 2001 From: kk <43161300+kkHuang-amd@users.noreply.github.com> Date: Fri, 16 Oct 2026 17:09:32 +0800 Subject: [PATCH] [Perf] [AMD] moe: small-batch sort with histogram + bitonic sort for 32 < P <= 256 (3/8) (#44004) Co-authored-by: Duyi-Wang --- .../kernels/ops/moe/moe_sorting_small.py | 108 +++++++++++++++++- test/registered/amd/test_moe_sorting_small.py | 58 +++++++++- 2 files changed, 162 insertions(+), 4 deletions(-) diff --git a/python/sglang/kernels/ops/moe/moe_sorting_small.py b/python/sglang/kernels/ops/moe/moe_sorting_small.py index 386873a421..2a2389bec9 100644 --- a/python/sglang/kernels/ops/moe/moe_sorting_small.py +++ b/python/sglang/kernels/ops/moe/moe_sorting_small.py @@ -294,8 +294,81 @@ def _moe_sorting_small_kernel_distributed( tl.store(qscale_ptr + sw, exp.to(tl.uint8)) +# no-quant path: histogram + bitonic sort keep every pair in registers of one CTA +@triton.jit(do_not_specialize=["M", "moe_buf_numel", "num_buf"]) +def _moe_sorting_small_kernel_sorted( + topk_ids_ptr, # [M, topk] i32 + topk_weights_ptr, # [M, topk] fp32 + sorted_ids_ptr, # [max_padded] i32 + sorted_weights_ptr, # [max_padded] fp32 + sorted_expert_ids_ptr, # [max_blocks] i32 + num_valid_ids_ptr, # [2] i32 + moe_buf_ptr, + moe_buf_numel, + M, + TOPK: tl.constexpr, + BLOCK_SIZE: tl.constexpr, + P_POW2: tl.constexpr, # >= M * topk + E_POW2: tl.constexpr, # > num_experts; bin E_POW2 - 1 holds the inactive lanes + PAD_POW2: tl.constexpr, # >= (M * topk) * BLOCK_SIZE (worst-case padded len) + BUF_BLOCK: tl.constexpr, + num_buf, # buf-zero CTAs occupy pids [1, num_buf] +): + pid = tl.program_id(0) + if pid > 0: + offs = (pid - 1) * BUF_BLOCK + tl.arange(0, BUF_BLOCK) + tl.store( + moe_buf_ptr + offs, + tl.zeros((BUF_BLOCK,), moe_buf_ptr.dtype.element_ty), + mask=offs < moe_buf_numel, + ) + return + + P = M * TOPK + offs_p = tl.arange(0, P_POW2) + e = tl.load(topk_ids_ptr + offs_p, mask=offs_p < P, other=E_POW2 - 1) + offs_e = tl.arange(0, E_POW2) + cnt = tl.where(offs_e < E_POW2 - 1, tl.histogram(e, E_POW2), 0) + blocks = (cnt + BLOCK_SIZE - 1) // BLOCK_SIZE + # sorted position of a pair -> padded dest: add the block-tail padding of smaller experts + shift = (tl.cumsum(blocks, 0) - blocks) * BLOCK_SIZE - (tl.cumsum(cnt, 0) - cnt) + num_valid = tl.sum(blocks, 0) * BLOCK_SIZE + + # pad the whole used region first, then scatter the real pairs over it + offs_pad = tl.arange(0, PAD_POW2) + pad_mask = offs_pad < num_valid + tl.store( + sorted_ids_ptr + offs_pad, + tl.full((PAD_POW2,), (TOPK << 24) | M, tl.int32), + mask=pad_mask, + ) + tl.store( + sorted_weights_ptr + offs_pad, tl.zeros((PAD_POW2,), tl.float32), mask=pad_mask + ) + tl.debug_barrier() + + # stable (expert, pair) order; inactive lanes sort last + key = tl.sort(e * P_POW2 + offs_p) + es = key // P_POW2 + ps = key % P_POW2 + valid = offs_p < P + dest = offs_p + tl.gather(shift, tl.where(valid, es, 0), 0) + tl.store(sorted_ids_ptr + dest, ((ps % TOPK) << 24) | (ps // TOPK), mask=valid) + tl.store( + sorted_weights_ptr + dest, + tl.load(topk_weights_ptr + ps, mask=valid, other=0.0), + mask=valid, + ) + # every block holds a real pair; its pairs all write the same expert id + tl.store(sorted_expert_ids_ptr + dest // BLOCK_SIZE, es, mask=valid) + tl.store( + num_valid_ids_ptr + tl.arange(0, 2), + tl.where(tl.arange(0, 2) == 0, num_valid, M), + ) + + def _small_sort_supported( - topk_ids, block_size, expert_mask, num_local_tokens, num_experts + topk_ids, block_size, expert_mask, num_local_tokens, num_experts, emit_mx=True ): m, topk = topk_ids.shape p = m * topk @@ -304,8 +377,14 @@ def _small_sort_supported( and num_local_tokens is None and p <= 256 # ponytail: the distributed variant scans every 64-expert chunk per pair, so it beats - # aiter at E=129 but loses at E=513; cutoff untuned in between - and ((p <= 64 and p <= 2 * block_size) or num_experts <= 256) + # aiter at E=129 but loses at E=513; cutoff untuned in between. The no-quant + # path past 32 pairs takes the histogram + bitonic sort kernel instead, which + # does not scan per expert chunk (measured up to E=385, i.e. E_POW2 <= 512). + and ( + (p <= 64 and p <= 2 * block_size) + or num_experts <= 256 + or (not emit_mx and p > 32 and num_experts < 512) + ) and topk < 128 and topk_ids.dtype == torch.int32 and topk_ids.is_contiguous() @@ -343,6 +422,28 @@ def _run_small_sort( n_cols, scalen_pad = 32, 8 qout = qscale = moe_buf # unused placeholder pointers num_buf = triton.cdiv(max(moe_buf.numel(), 1), buf_block) + if not emit_mx and p > 32: + # beyond one 32-lane tile the sort variant beats both P x P compare kernels + _moe_sorting_small_kernel_sorted[(1 + num_buf,)]( + topk_ids, + topk_weights, + sorted_ids, + sorted_weights, + sorted_expert_ids, + num_valid_ids, + moe_buf, + moe_buf.numel(), + m, + TOPK=topk, + BLOCK_SIZE=block_size, + P_POW2=triton.next_power_of_2(p), + E_POW2=triton.next_power_of_2(num_experts + 1), + PAD_POW2=triton.next_power_of_2(p * block_size), + BUF_BLOCK=buf_block, + num_buf=num_buf, + num_warps=4, + ) + return None if p <= 64 and p <= 2 * block_size: # compact variant: one sort CTA does the P x P rank compare num_quant = (p * (n_cols // min(2048, n_cols))) if emit_mx else 0 @@ -491,6 +592,7 @@ def apply_aiter_small_moe_sort_patch() -> None: expert_mask, num_local_tokens, int(num_experts), + emit_mx=_pending_quant_input.get() is not None, ) ): device = topk_ids.device diff --git a/test/registered/amd/test_moe_sorting_small.py b/test/registered/amd/test_moe_sorting_small.py index 5afcfe6af2..7fcbb2c654 100644 --- a/test/registered/amd/test_moe_sorting_small.py +++ b/test/registered/amd/test_moe_sorting_small.py @@ -7,7 +7,7 @@ import torch from sglang.test.ci.ci_register import register_amd_ci from sglang.test.test_utils import CustomTestCase -register_amd_ci(est_time=20, suite="stage-b-test-1-gpu-small-amd-mi35x") +register_amd_ci(est_time=80, suite="stage-b-test-1-gpu-small-amd-mi35x") def _gfx950(): @@ -126,5 +126,61 @@ class TestMoeSortingSmall(CustomTestCase): self.assertFalse(self.S._small_sort_supported(ids, 32, None, None, self.E)) +def _reference(ids, w, num_experts, block_size): + m, topk = ids.shape + flat = ids.flatten().tolist() + sorted_ids, sorted_w, expert_ids = [], [], [] + for e in range(num_experts): + pairs = [p for p, x in enumerate(flat) if x == e] + if not pairs: + continue + n = -(-len(pairs) // block_size) * block_size + sorted_ids += [((p % topk) << 24) | (p // topk) for p in pairs] + sorted_ids += [(topk << 24) | m] * (n - len(pairs)) + sorted_w += [w.flatten()[p].item() for p in pairs] + [0.0] * (n - len(pairs)) + expert_ids += [e] * (n // block_size) + return sorted_ids, sorted_w, expert_ids + + +@unittest.skipUnless(torch.cuda.is_available() and torch.version.hip, "ROCm only") +class TestMoeSortingSmallNoQuant(CustomTestCase): + """No-quant path of _run_small_sort (incl. the histogram + bitonic kernel for + 32 < P <= 256) vs a torch reference of the aiter moe_sorting layout.""" + + def _check(self, m, topk, num_experts, block_size, hot): + from sglang.kernels.ops.moe.moe_sorting_small import _run_small_sort + + g = torch.Generator().manual_seed(m * 1000 + topk) + ids = torch.stack( + [torch.randperm(hot, generator=g)[:topk] for _ in range(m)] + ).to(torch.int32) + ids = (ids + num_experts - hot).cuda() # hot = the top `hot` expert ids + w = torch.rand(m, topk, generator=g).cuda() + max_padded = m * topk + num_experts * block_size - topk + sorted_ids = torch.full((max_padded,), -7, dtype=torch.int32, device="cuda") + sorted_w = torch.full((max_padded,), -7.0, device="cuda") + expert_ids = torch.full( + (-(-max_padded // block_size),), -7, dtype=torch.int32, device="cuda" + ) + num_valid = torch.full((2,), -7, dtype=torch.int32, device="cuda") + moe_buf = torch.ones(m, 7168, dtype=torch.bfloat16, device="cuda") + args = (sorted_ids, sorted_w, expert_ids, num_valid, moe_buf, block_size) + _run_small_sort(ids, w, *args, None, num_experts) + ref_ids, ref_w, ref_e = _reference(ids.cpu(), w.cpu(), num_experts, block_size) + n = len(ref_ids) + self.assertEqual(num_valid.tolist(), [n, m]) + self.assertEqual(sorted_ids[:n].tolist(), ref_ids) + self.assertEqual(sorted_w[:n].tolist(), ref_w) + self.assertEqual(expert_ids[: n // block_size].tolist(), ref_e) + self.assertTrue(bool((moe_buf == 0).all())) + + def test_no_quant(self): + for num_experts, topk, block_size in ((385, 7, 32), (257, 9, 32), (129, 8, 16)): + for m in range(1, 256 // topk + 1): + for hot in (num_experts, 12): # uniform and skewed routing + with self.subTest(e=num_experts, topk=topk, m=m, hot=hot): + self._check(m, topk, num_experts, block_size, hot) + + if __name__ == "__main__": unittest.main()