[Bench][MoE] Deal the simulated round-robin experts evenly across EP ranks (#43188)

Co-authored-by: Hanming Lu <69857889+hanming-lu@users.noreply.github.com>
This commit is contained in:
metamergebot
2026-10-08 15:01:16 -07:00
committed by GitHub
co-authored by Hanming Lu
parent b0e06359ac
commit 0b658c12ff
4 changed files with 387 additions and 43 deletions
+77 -18
View File
@@ -40,6 +40,7 @@ if TYPE_CHECKING:
from sglang.srt.runtime_context import (
get_exec,
get_forward,
get_lora,
get_parallel,
get_server_args,
@@ -426,9 +427,11 @@ def _simulate_balanced_routing_kernel(
topk_weights_ptr,
num_experts,
step,
num_ranks,
experts_per_rank,
deal_offset,
inv_k,
seed,
layer_offset,
token_shard_rank,
num_token_shards,
stride_im,
@@ -450,22 +453,29 @@ def _simulate_balanced_routing_kernel(
- ``topk_weights_ptr``: ``[num_tokens, K]`` (row-major; strides passed in),
overwritten in place
``RANDOM=False`` is the deterministic round-robin base ``token + layer_offset``;
``RANDOM=True`` is a random per-token base (uniform, balanced in expectation;
``seed`` is a kernel arg, so it is baked at CUDA-graph capture and replays stay
balanced). Both spread the k experts by ``step`` and emit global expert ids
(any EP logical->physical remap happens later in ``_post_process_topk_ids``).
``token_shard_rank`` and ``num_token_shards`` ensure scattered DP ranks generate
different expert assignments for their local tokens when DP > 1."""
``RANDOM=False`` is the deterministic round-robin: slot ``d = t * K + j +
deal_offset`` goes to EP rank ``d % num_ranks`` and to expert ``(d //
num_ranks) % experts_per_rank`` within that rank, so consecutive slots land
on consecutive ranks. ``deal_offset`` folds in the layer offset and the
per-shard rotation computed by the wrapper. ``RANDOM=True`` is a random
per-token base (uniform, balanced in expectation; ``seed`` is a kernel arg,
so it is baked at CUDA-graph capture and replays stay balanced) spread by
``step``; it is the only path that reads ``token_shard_rank`` /
``num_token_shards``, to seed scattered DP shards differently. Both emit
global expert ids (any EP logical->physical remap happens later in
``_post_process_topk_ids``)."""
t = tl.program_id(0)
global_t = t * num_token_shards + token_shard_rank
j = tl.arange(0, BLOCK_K)
mask = j < K
if RANDOM:
global_t = t * num_token_shards + token_shard_rank
base = (tl.rand(seed, global_t) * num_experts).to(tl.int32)
gid = (base + j * step) % num_experts
else:
base = global_t + layer_offset
gid = (base + j * step) % num_experts
deal = t * K + j + deal_offset
gid = (deal % num_ranks) * experts_per_rank + (
deal // num_ranks
) % experts_per_rank
tl.store(topk_ids_ptr + t * stride_im + j * stride_ik, gid, mask=mask)
tl.store(
topk_weights_ptr + t * stride_wm + j * stride_wk,
@@ -485,6 +495,7 @@ def _simulate_balanced_routing(
num_experts: int,
*,
random: bool,
num_ranks: int = 1,
layer_id: Optional[int] = None,
token_shard_rank: int = 0,
num_token_shards: int = 1,
@@ -498,15 +509,44 @@ def _simulate_balanced_routing(
- ``topk_ids``: ``[num_tokens, k]``, overwritten in place
- ``topk_weights``: ``[num_tokens, k]``, overwritten in place
``token_shard_rank`` and ``num_token_shards`` describe scattered DP input.
Their defaults describe a gathered token buffer (effective DP=1). ``seed``
is exposed for deterministic tests; production calls use a per-launch seed.
``token_shard_rank`` / ``num_token_shards`` identify this call's slice of
the global token batch (scattered DP, and attention-TP slices under
sequence sharding); the defaults describe one gathered buffer.
Round-robin deals each shard's slots ``t * k + j`` over the ``num_ranks``
EP ranks in order, with shard ``r`` starting at rank ``r`` (spread evenly
when there are fewer shards than ranks) and at expert ``r`` within every
rank. Nothing cross-shard is read, so the result is the same under eager,
CUDA-graph replay and two-batch overlap. Per-EP-rank load: a shard's own
load differs by at most one slot across ranks, and is exact when its
``tokens * k`` is a multiple of ``num_ranks``; the total over shards is
exact when every shard is, or when all shards hold equal counts and the
shard count is a multiple of ``num_ranks``, and otherwise differs by at
most one slot per shard whose ``tokens * k`` leaves a remainder. Balance is
over logical expert blocks ``num_experts // num_ranks``: a non-trivial
expert placement (EPLB, redundant experts) remaps it afterwards.
``seed`` is exposed for deterministic tests; production calls use a
per-launch seed.
"""
global _simulate_uniform_seed
num_tokens, k = topk_ids.shape
if num_tokens == 0 or k == 0:
return
assert 0 <= token_shard_rank < num_token_shards
assert num_ranks >= 1 and num_experts % num_ranks == 0, (num_experts, num_ranks)
experts_per_rank = num_experts // num_ranks
# Rank rotation keeps different shards' partial tail blocks off the same
# ranks; the expert rotation (a multiple of num_ranks, so it leaves the
# rank untouched) keeps small batches from piling every shard's slots onto
# the same one or two experts of each rank.
rank_rotation = (
token_shard_rank * max(num_ranks, num_token_shards)
) // num_token_shards
expert_rotation = token_shard_rank * num_ranks
deal_offset = (
(0 if layer_id is None else layer_id) + rank_rotation + expert_rotation
)
if random and seed is None:
seed = _simulate_uniform_seed
_simulate_uniform_seed += 1
@@ -517,9 +557,11 @@ def _simulate_balanced_routing(
topk_weights,
num_experts,
max(num_experts // k, 1),
num_ranks,
experts_per_rank,
deal_offset,
1.0 / k,
seed,
0 if layer_id is None else layer_id,
token_shard_rank,
num_token_shards,
topk_ids.stride(0),
@@ -2789,20 +2831,37 @@ def select_experts(
# dummy/random benchmark tokens don't skew MoE load) via a single fused
# Triton kernel — one launch instead of the ~5-7 small elementwise ops it
# replaces, to minimize timing perturbation. Do NOT use in production.
parallel = get_parallel()
if is_moe_input_scattered_across_dp_ranks():
parallel = get_parallel()
token_shard_rank = parallel.attn_dp_rank
num_token_shards = parallel.attn_dp_size
forward = get_forward()
if forward.attn_tp_sequence_sharded or forward.attn_input_scattered:
# This forward routes attention-TP-local rows (sequence-sharded
# attention TP, or the layer-boundary attention input scatter):
# each attention-TP rank holds its own slice of the DP shard, so
# it is a shard of its own rather than a replica of its peers.
token_shard_rank = (
token_shard_rank * parallel.attn_tp_size + parallel.attn_tp_rank
)
num_token_shards *= parallel.attn_tp_size
else:
# Gathered MoE presents one global token buffer to every rank, so
# its routing must remain identical across those replicas.
token_shard_rank, num_token_shards = 0, 1
override_topk_ids, override_topk_weights = topk_ids, topk_weights
if num_routed_topk < top_k:
# Only the routed columns are simulated; a fused shared-expert
# column keeps the id and weight the router gave it.
override_topk_ids = topk_ids[:, :num_routed_topk]
override_topk_weights = topk_weights[:, :num_routed_topk]
_simulate_balanced_routing(
topk_ids,
topk_weights,
override_topk_ids,
override_topk_weights,
router_logits.shape[1],
random=simulate_uniform_experts,
num_ranks=parallel.moe_ep_size,
layer_id=layer_id,
token_shard_rank=token_shard_rank,
num_token_shards=num_token_shards,
+5
View File
@@ -693,6 +693,10 @@ class ForwardFlags:
"lora_batch_layout": LoRABatchLayout.DP_LOCAL,
# LayerNorm sequence parallelism region; see layers/layernorm_sp.py.
"sp_active": False,
# This forward's MoE runs on attention-TP-local token slices (the
# ForwardBatch.attn_tp_sequence_sharded decision, stamped by the model
# that slices); read by the benchmark routing override.
"attn_tp_sequence_sharded": False,
}
# Read/written inside compiled graphs (vocab embedding, layer boundaries,
@@ -709,6 +713,7 @@ class ForwardFlags:
"defer_moe_finalize",
"lora_batch_layout",
"sp_active",
"attn_tp_sequence_sharded",
}
)