From 0b658c12ff1930d0fe75909fe68f6efda449f31d Mon Sep 17 00:00:00 2001 From: metamergebot Date: Thu, 8 Oct 2026 15:01:16 -0700 Subject: [PATCH] [Bench][MoE] Deal the simulated round-robin experts evenly across EP ranks (#43188) Co-authored-by: Hanming Lu <69857889+hanming-lu@users.noreply.github.com> --- python/sglang/srt/layers/moe/topk.py | 95 ++++-- python/sglang/srt/runtime_context.py | 5 + .../moe/test_simulate_balanced_routing.py | 41 +-- .../ops/moe/test_round_robin_ep_balance.py | 289 ++++++++++++++++++ 4 files changed, 387 insertions(+), 43 deletions(-) create mode 100644 test/registered/kernels/ops/moe/test_round_robin_ep_balance.py diff --git a/python/sglang/srt/layers/moe/topk.py b/python/sglang/srt/layers/moe/topk.py index 63b689d86e..6d8db4e2c3 100644 --- a/python/sglang/srt/layers/moe/topk.py +++ b/python/sglang/srt/layers/moe/topk.py @@ -40,6 +40,7 @@ if TYPE_CHECKING: from sglang.srt.runtime_context import ( get_exec, + get_forward, get_lora, get_parallel, get_server_args, @@ -426,9 +427,11 @@ def _simulate_balanced_routing_kernel( topk_weights_ptr, num_experts, step, + num_ranks, + experts_per_rank, + deal_offset, inv_k, seed, - layer_offset, token_shard_rank, num_token_shards, stride_im, @@ -450,22 +453,29 @@ def _simulate_balanced_routing_kernel( - ``topk_weights_ptr``: ``[num_tokens, K]`` (row-major; strides passed in), overwritten in place - ``RANDOM=False`` is the deterministic round-robin base ``token + layer_offset``; - ``RANDOM=True`` is a random per-token base (uniform, balanced in expectation; - ``seed`` is a kernel arg, so it is baked at CUDA-graph capture and replays stay - balanced). Both spread the k experts by ``step`` and emit global expert ids - (any EP logical->physical remap happens later in ``_post_process_topk_ids``). - ``token_shard_rank`` and ``num_token_shards`` ensure scattered DP ranks generate - different expert assignments for their local tokens when DP > 1.""" + ``RANDOM=False`` is the deterministic round-robin: slot ``d = t * K + j + + deal_offset`` goes to EP rank ``d % num_ranks`` and to expert ``(d // + num_ranks) % experts_per_rank`` within that rank, so consecutive slots land + on consecutive ranks. ``deal_offset`` folds in the layer offset and the + per-shard rotation computed by the wrapper. ``RANDOM=True`` is a random + per-token base (uniform, balanced in expectation; ``seed`` is a kernel arg, + so it is baked at CUDA-graph capture and replays stay balanced) spread by + ``step``; it is the only path that reads ``token_shard_rank`` / + ``num_token_shards``, to seed scattered DP shards differently. Both emit + global expert ids (any EP logical->physical remap happens later in + ``_post_process_topk_ids``).""" t = tl.program_id(0) - global_t = t * num_token_shards + token_shard_rank j = tl.arange(0, BLOCK_K) mask = j < K if RANDOM: + global_t = t * num_token_shards + token_shard_rank base = (tl.rand(seed, global_t) * num_experts).to(tl.int32) + gid = (base + j * step) % num_experts else: - base = global_t + layer_offset - gid = (base + j * step) % num_experts + deal = t * K + j + deal_offset + gid = (deal % num_ranks) * experts_per_rank + ( + deal // num_ranks + ) % experts_per_rank tl.store(topk_ids_ptr + t * stride_im + j * stride_ik, gid, mask=mask) tl.store( topk_weights_ptr + t * stride_wm + j * stride_wk, @@ -485,6 +495,7 @@ def _simulate_balanced_routing( num_experts: int, *, random: bool, + num_ranks: int = 1, layer_id: Optional[int] = None, token_shard_rank: int = 0, num_token_shards: int = 1, @@ -498,15 +509,44 @@ def _simulate_balanced_routing( - ``topk_ids``: ``[num_tokens, k]``, overwritten in place - ``topk_weights``: ``[num_tokens, k]``, overwritten in place - ``token_shard_rank`` and ``num_token_shards`` describe scattered DP input. - Their defaults describe a gathered token buffer (effective DP=1). ``seed`` - is exposed for deterministic tests; production calls use a per-launch seed. + ``token_shard_rank`` / ``num_token_shards`` identify this call's slice of + the global token batch (scattered DP, and attention-TP slices under + sequence sharding); the defaults describe one gathered buffer. + + Round-robin deals each shard's slots ``t * k + j`` over the ``num_ranks`` + EP ranks in order, with shard ``r`` starting at rank ``r`` (spread evenly + when there are fewer shards than ranks) and at expert ``r`` within every + rank. Nothing cross-shard is read, so the result is the same under eager, + CUDA-graph replay and two-batch overlap. Per-EP-rank load: a shard's own + load differs by at most one slot across ranks, and is exact when its + ``tokens * k`` is a multiple of ``num_ranks``; the total over shards is + exact when every shard is, or when all shards hold equal counts and the + shard count is a multiple of ``num_ranks``, and otherwise differs by at + most one slot per shard whose ``tokens * k`` leaves a remainder. Balance is + over logical expert blocks ``num_experts // num_ranks``: a non-trivial + expert placement (EPLB, redundant experts) remaps it afterwards. + + ``seed`` is exposed for deterministic tests; production calls use a + per-launch seed. """ global _simulate_uniform_seed num_tokens, k = topk_ids.shape if num_tokens == 0 or k == 0: return assert 0 <= token_shard_rank < num_token_shards + assert num_ranks >= 1 and num_experts % num_ranks == 0, (num_experts, num_ranks) + experts_per_rank = num_experts // num_ranks + # Rank rotation keeps different shards' partial tail blocks off the same + # ranks; the expert rotation (a multiple of num_ranks, so it leaves the + # rank untouched) keeps small batches from piling every shard's slots onto + # the same one or two experts of each rank. + rank_rotation = ( + token_shard_rank * max(num_ranks, num_token_shards) + ) // num_token_shards + expert_rotation = token_shard_rank * num_ranks + deal_offset = ( + (0 if layer_id is None else layer_id) + rank_rotation + expert_rotation + ) if random and seed is None: seed = _simulate_uniform_seed _simulate_uniform_seed += 1 @@ -517,9 +557,11 @@ def _simulate_balanced_routing( topk_weights, num_experts, max(num_experts // k, 1), + num_ranks, + experts_per_rank, + deal_offset, 1.0 / k, seed, - 0 if layer_id is None else layer_id, token_shard_rank, num_token_shards, topk_ids.stride(0), @@ -2789,20 +2831,37 @@ def select_experts( # dummy/random benchmark tokens don't skew MoE load) via a single fused # Triton kernel — one launch instead of the ~5-7 small elementwise ops it # replaces, to minimize timing perturbation. Do NOT use in production. + parallel = get_parallel() if is_moe_input_scattered_across_dp_ranks(): - parallel = get_parallel() token_shard_rank = parallel.attn_dp_rank num_token_shards = parallel.attn_dp_size + forward = get_forward() + if forward.attn_tp_sequence_sharded or forward.attn_input_scattered: + # This forward routes attention-TP-local rows (sequence-sharded + # attention TP, or the layer-boundary attention input scatter): + # each attention-TP rank holds its own slice of the DP shard, so + # it is a shard of its own rather than a replica of its peers. + token_shard_rank = ( + token_shard_rank * parallel.attn_tp_size + parallel.attn_tp_rank + ) + num_token_shards *= parallel.attn_tp_size else: # Gathered MoE presents one global token buffer to every rank, so # its routing must remain identical across those replicas. token_shard_rank, num_token_shards = 0, 1 + override_topk_ids, override_topk_weights = topk_ids, topk_weights + if num_routed_topk < top_k: + # Only the routed columns are simulated; a fused shared-expert + # column keeps the id and weight the router gave it. + override_topk_ids = topk_ids[:, :num_routed_topk] + override_topk_weights = topk_weights[:, :num_routed_topk] _simulate_balanced_routing( - topk_ids, - topk_weights, + override_topk_ids, + override_topk_weights, router_logits.shape[1], random=simulate_uniform_experts, + num_ranks=parallel.moe_ep_size, layer_id=layer_id, token_shard_rank=token_shard_rank, num_token_shards=num_token_shards, diff --git a/python/sglang/srt/runtime_context.py b/python/sglang/srt/runtime_context.py index 79832b28b5..f6792cf9e7 100644 --- a/python/sglang/srt/runtime_context.py +++ b/python/sglang/srt/runtime_context.py @@ -693,6 +693,10 @@ class ForwardFlags: "lora_batch_layout": LoRABatchLayout.DP_LOCAL, # LayerNorm sequence parallelism region; see layers/layernorm_sp.py. "sp_active": False, + # This forward's MoE runs on attention-TP-local token slices (the + # ForwardBatch.attn_tp_sequence_sharded decision, stamped by the model + # that slices); read by the benchmark routing override. + "attn_tp_sequence_sharded": False, } # Read/written inside compiled graphs (vocab embedding, layer boundaries, @@ -709,6 +713,7 @@ class ForwardFlags: "defer_moe_finalize", "lora_batch_layout", "sp_active", + "attn_tp_sequence_sharded", } ) diff --git a/test/manual/layers/moe/test_simulate_balanced_routing.py b/test/manual/layers/moe/test_simulate_balanced_routing.py index 314e44525c..caf48c7abe 100644 --- a/test/manual/layers/moe/test_simulate_balanced_routing.py +++ b/test/manual/layers/moe/test_simulate_balanced_routing.py @@ -33,21 +33,16 @@ def _make_round_robin_expert_ids( dtype: torch.dtype, layer_id: Optional[int] = None, ) -> torch.Tensor: - # Deterministic, perfectly balanced expert assignment: each token's top-k is - # spread by num_experts//topk. Returns global expert ids of shape - # [num_tokens, topk]. + # Deterministic, perfectly balanced expert assignment: the slots + # ``t * topk + j`` are dealt over the experts in order (single EP rank). + # Returns global expert ids of shape [num_tokens, topk]. if topk == 0: return torch.empty((num_tokens, 0), device=device, dtype=dtype) - step = max(num_experts // topk, 1) layer_offset = 0 if layer_id is None else layer_id - offsets = torch.arange(num_tokens, device=device, dtype=dtype).unsqueeze( - 1 - ) # [num_tokens, 1] - steps = ( - torch.arange(topk, device=device, dtype=dtype).unsqueeze(0) * step - ) # [1, topk] - return (offsets + layer_offset + steps) % num_experts # [num_tokens, topk] + tokens = torch.arange(num_tokens, device=device, dtype=dtype).unsqueeze(1) + slots = torch.arange(topk, device=device, dtype=dtype).unsqueeze(0) + return (tokens * topk + slots + layer_offset) % num_experts def _alloc( @@ -107,21 +102,17 @@ class TestSimulateBalancedRouting(CustomTestCase): for row in ids[:64]: self.assertEqual(row.unique().numel(), K) - @parameterized.expand( - [ - ("round_robin_dp2", False, 2), - ("round_robin_dp4", False, 4), - ("uniform_dp2", True, 2), - ("uniform_dp4", True, 4), - ] - ) - def test_interleaved_dp_assignments_match_dp1( - self, _name: str, random: bool, dp_size: int + @parameterized.expand([("dp2", 2), ("dp4", 4)]) + def test_uniform_interleaved_dp_assignments_match_dp1( + self, _name: str, dp_size: int ) -> None: - # Interleaving the DP-local outputs must exactly reproduce the expert - # assignments for the equivalent DP=1 input. The fixed seed models - # independent processes entering the same uniform-routing call with - # the same initial seed. + random = True + # Interleaving the DP-local uniform outputs must exactly reproduce the + # expert assignments for the equivalent DP=1 input. The fixed seed + # models independent processes entering the same uniform-routing call + # with the same initial seed. (Round-robin deals each shard its own + # rotated stream instead; its DP behaviour is covered by + # test/registered/kernels/ops/moe/test_round_robin_ep_balance.py.) T = 16 seed = 17 layer_id = 3 diff --git a/test/registered/kernels/ops/moe/test_round_robin_ep_balance.py b/test/registered/kernels/ops/moe/test_round_robin_ep_balance.py new file mode 100644 index 0000000000..08060747a3 --- /dev/null +++ b/test/registered/kernels/ops/moe/test_round_robin_ep_balance.py @@ -0,0 +1,289 @@ +"""GPU coverage for the round-robin benchmark routing override's EP-rank balance.""" + +import sys +from unittest.mock import patch + +import pytest +import torch + +from sglang.srt.environ import envs +from sglang.srt.layers.moe.topk import ( + TopKConfig, + _simulate_balanced_routing, + select_experts, +) +from sglang.srt.runtime_context import get_forward, get_parallel +from sglang.test.ci.ci_register import register_cuda_ci + +register_cuda_ci(est_time=20, stage="base-b-kernel-unit", runner_config="1-gpu-large") + +NUM_EXPERTS = 256 +TOPK = 8 +EP_SIZE = 16 + + +def _reference( + num_tokens: int, + *, + num_ranks: int, + token_shard_rank: int = 0, + num_token_shards: int = 1, + layer_id: int = 0, + topk: int = TOPK, + num_experts: int = NUM_EXPERTS, +) -> torch.Tensor: + """Plain-torch statement of the dealing: slot ``t*k + j`` of shard ``r`` + goes to rank ``d % P`` and expert ``(d // P) % (E // P)`` in that rank, + with ``d`` offset by the layer, a rank rotation of ``r`` (spread evenly + when there are fewer shards than ranks) and an expert rotation of ``r``.""" + experts_per_rank = num_experts // num_ranks + rank_rotation = ( + token_shard_rank * max(num_ranks, num_token_shards) // num_token_shards + ) + offset = layer_id + rank_rotation + token_shard_rank * num_ranks + tokens = torch.arange(num_tokens, device="cuda").unsqueeze(1) + slots = torch.arange(topk, device="cuda").unsqueeze(0) + deal = tokens * topk + slots + offset + return (deal % num_ranks) * experts_per_rank + ( + deal // num_ranks + ) % experts_per_rank + + +def _route( + num_tokens: int, + *, + num_ranks: int = EP_SIZE, + token_shard_rank: int = 0, + num_token_shards: int = 1, + layer_id: int = 0, + topk: int = TOPK, + num_experts: int = NUM_EXPERTS, + dtype: torch.dtype = torch.int32, +): + ids = torch.full((num_tokens, topk), -1, dtype=dtype, device="cuda") + weights = torch.zeros((num_tokens, topk), dtype=torch.float32, device="cuda") + _simulate_balanced_routing( + ids, + weights, + num_experts, + random=False, + num_ranks=num_ranks, + layer_id=layer_id, + token_shard_rank=token_shard_rank, + num_token_shards=num_token_shards, + ) + return ids, weights + + +def _rank_loads(ids: torch.Tensor, num_ranks: int = EP_SIZE) -> torch.Tensor: + return torch.bincount( + ids.flatten().long() // (NUM_EXPERTS // num_ranks), minlength=num_ranks + ) + + +def _sharded_ids(counts: list[int], *, layer_id: int = 0) -> torch.Tensor: + return torch.cat( + [ + _route( + count, + token_shard_rank=rank, + num_token_shards=len(counts), + layer_id=layer_id, + )[0] + for rank, count in enumerate(counts) + ] + ) + + +@pytest.mark.parametrize( + "num_ranks, num_shards, tokens, topk, layer_id, dtype", + [ + (16, 1, 300, 8, 0, torch.int32), # one shard + (1, 1, 17, 8, 3, torch.int32), # single rank: sequential experts + (16, 16, 3, 8, 5, torch.int32), # as many shards as ranks + (16, 4, 2, 8, 0, torch.int64), # fewer shards than ranks, int64 ids + (16, 24, 1, 8, 0, torch.int32), # more shards than ranks + (16, 2, 5, 6, 3, torch.int32), # non-power-of-two top-k + ], +) +def test_kernel_matches_reference_on_every_shard( + num_ranks, num_shards, tokens, topk, layer_id, dtype +): + for shard in range(num_shards): + ids, weights = _route( + tokens, + num_ranks=num_ranks, + token_shard_rank=shard, + num_token_shards=num_shards, + layer_id=layer_id, + topk=topk, + dtype=dtype, + ) + expected = _reference( + tokens, + num_ranks=num_ranks, + token_shard_rank=shard, + num_token_shards=num_shards, + layer_id=layer_id, + topk=topk, + ) + assert torch.equal(ids.long(), expected), (shard, ids, expected) + assert torch.equal(weights, torch.full_like(weights, 1.0 / topk)) + + +@pytest.mark.parametrize("num_tokens", [1, 2, 4, 300]) +@pytest.mark.parametrize("num_ranks", [4, 8, 16]) +def test_single_shard_balances_within_one_slot(num_tokens, num_ranks): + ids, _ = _route(num_tokens, num_ranks=num_ranks) + loads = _rank_loads(ids, num_ranks) + assert int(loads.max() - loads.min()) <= 1 + assert all(row.unique().numel() == TOPK for row in ids) + + +@pytest.mark.parametrize("tokens_per_shard", [1, 2, 64]) +@pytest.mark.parametrize("layer_id", [0, 5]) +def test_equal_shards_balance_exactly(tokens_per_shard, layer_id): + # Every CUDA-graph decode step with equal per-rank batch sizes: the + # rotated per-shard streams tile the EP ranks with no remainder. + loads = _rank_loads(_sharded_ids([tokens_per_shard] * EP_SIZE, layer_id=layer_id)) + assert torch.equal(loads, torch.full_like(loads, tokens_per_shard * TOPK)) + + +@pytest.mark.parametrize( + "counts", + [ + [2] + [1] * 15, + [2, 1] * 8, + [65] * 8 + [64] * 8, + [1, 3, 5, 7] + [0] * 12, + # Synthetic uneven prefill: 1001, 2002, 3003, 4004, 1005, 2006, 3007, 4008. + [1000 * (r % 4 + 1) + r + 1 for r in range(8)], + ], +) +def test_uneven_shards_spread_is_at_most_one_slot_per_partial_shard(counts): + # Without cross-shard token counts, a shard whose ``tokens * k`` is not a + # multiple of the rank count leaves one partial block, worth at most one + # slot of imbalance; shards with whole blocks contribute none. + loads = _rank_loads(_sharded_ids(counts)) + partial_shards = sum(1 for c in counts if (c * TOPK) % EP_SIZE) + assert int(loads.max() - loads.min()) <= partial_shards + assert int(loads.sum()) == sum(counts) * TOPK + + +def test_small_batches_spread_over_experts_within_each_rank(): + # Sixteen 1-token shards: the expert rotation keeps them from all landing on + # the first expert of every rank (which would make weight-bandwidth-bound + # decode measurements optimistic). + ids = _sharded_ids([1] * EP_SIZE) + experts_per_rank = NUM_EXPERTS // EP_SIZE + for rank in range(EP_SIZE): + on_rank = ids[ids // experts_per_rank == rank] # 8 slots from 8 shards + assert on_rank.unique().numel() >= on_rank.numel() * 3 // 4, (rank, on_rank) + + +def _fixed_router(hidden_states, gating_output, topk, renormalize): + del hidden_states, renormalize + ids = torch.full( + (gating_output.shape[0], topk), 77, dtype=torch.int32, device="cuda" + ) + weights = torch.full( + (gating_output.shape[0], topk), 0.75, dtype=torch.float32, device="cuda" + ) + return weights, ids + + +def _select( + top_k: int = TOPK, + num_fused_shared_experts: int = 0, + *, + num_tokens: int = 2, + dp_rank: int = 0, + dp_size: int = 1, + tp_rank: int = 0, + tp_size: int = 1, + ep_size: int = EP_SIZE, + scattered: bool = True, + uniform: bool = False, + **forward_flags, +): + with ( + envs.SGLANG_SIMULATE_UNIFORM_EXPERTS.override(uniform), + envs.SGLANG_SIMULATE_ROUND_ROBIN_EXPERTS.override(not uniform), + get_parallel().override( + attn_dp_rank=dp_rank, + attn_dp_size=dp_size, + attn_tp_rank=tp_rank, + attn_tp_size=tp_size, + moe_ep_size=ep_size, + ), + get_forward().scoped(**forward_flags), + patch( + "sglang.srt.layers.moe.topk.is_moe_input_scattered_across_dp_ranks", + return_value=scattered, + ), + ): + return select_experts( + hidden_states=torch.zeros((num_tokens, 4), device="cuda"), + router_logits=torch.zeros((num_tokens, NUM_EXPERTS), device="cuda"), + topk_config=TopKConfig( + top_k=top_k, + num_fused_shared_experts=num_fused_shared_experts, + custom_routing_function=_fixed_router, + ), + layer_id=0, + ) + + +def test_select_experts_gathered_single_rank_deals_experts_in_order(): + output = _select(num_tokens=4, ep_size=1, scattered=False) + expected = _reference(4, num_ranks=1) + assert torch.equal(output.topk_ids.long(), expected) + + +def test_select_experts_scattered_dp_shards_tile_the_ranks(): + ids = torch.cat( + [ + _select(num_tokens=1, dp_rank=r, dp_size=EP_SIZE).topk_ids + for r in range(EP_SIZE) + ] + ) + loads = _rank_loads(ids) + assert torch.equal(loads, torch.full_like(loads, TOPK)) + + +@pytest.mark.parametrize("flag", ["attn_tp_sequence_sharded", "attn_input_scattered"]) +def test_select_experts_sharded_attention_tp_ranks_are_distinct_shards(flag): + # DP1 x attention-TP2 with attention-TP-local rows: each attention-TP rank + # deals its own shard, and the two 1-token slices cover the ranks exactly. + outputs = [ + _select(num_tokens=1, tp_rank=tp_rank, tp_size=2, **{flag: True}).topk_ids + for tp_rank in range(2) + ] + assert not torch.equal(outputs[0], outputs[1]) + loads = _rank_loads(torch.cat(outputs)) + assert torch.equal(loads, torch.ones_like(loads)) + + +def test_select_experts_replicated_attention_tp_ranks_agree(): + # Without attention-TP-local rows the attention-TP ranks hold the same + # tokens and must route them identically. + outputs = [_select(tp_rank=tp_rank, tp_size=2).topk_ids for tp_rank in range(2)] + assert torch.equal(outputs[0], outputs[1]) + + +@pytest.mark.parametrize("uniform", [False, True]) +def test_select_experts_preserves_fused_shared_expert_column(uniform): + output = _select(TOPK + 1, num_fused_shared_experts=1, uniform=uniform) + assert torch.equal(output.topk_ids[:, -1], torch.full((2,), 77, device="cuda")) + assert torch.equal( + output.topk_weights[:, -1], torch.full((2,), 0.75, device="cuda") + ) + routed_weights = output.topk_weights[:, :TOPK] + assert torch.equal(routed_weights, torch.full_like(routed_weights, 1.0 / TOPK)) + if not uniform: + loads = _rank_loads(output.topk_ids[:, :TOPK]) + assert torch.equal(loads, torch.ones_like(loads)) + + +if __name__ == "__main__": + sys.exit(pytest.main([__file__]))