[Fix] moe: Apply SGLANG_SIMULATE_* routing on DeepSeek-V4.1 vision-biased gates (#45628)

This commit is contained in:
Khoa Pham
2026-10-21 10:00:15 -07:00
committed by GitHub
parent 6e98c84c60
commit ed268dd320
2 changed files with 75 additions and 36 deletions
+63 -36
View File
@@ -2386,6 +2386,64 @@ else:
fused_topk_native = fused_topk_torch_native
def routing_simulation_enabled() -> bool:
return (
envs.SGLANG_SIMULATE_UNIFORM_EXPERTS.get()
or envs.SGLANG_SIMULATE_ROUND_ROBIN_EXPERTS.get()
)
def apply_routing_simulation(
topk_ids: torch.Tensor,
topk_weights: torch.Tensor,
num_experts: int,
*,
num_routed_topk: int,
top_k: int,
layer_id: Optional[int],
) -> None:
simulate_uniform_experts = envs.SGLANG_SIMULATE_UNIFORM_EXPERTS.get()
# Benchmark-only: override gating with a balanced expert assignment (so
# dummy/random benchmark tokens don't skew MoE load) via a single fused
# Triton kernel — one launch instead of the ~5-7 small elementwise ops it
# replaces, to minimize timing perturbation. Do NOT use in production.
parallel = get_parallel()
if is_moe_input_scattered_across_dp_ranks():
token_shard_rank = parallel.attn_dp_rank
num_token_shards = parallel.attn_dp_size
forward = get_forward()
if forward.attn_tp_sequence_sharded or forward.attn_input_scattered:
# This forward routes attention-TP-local rows (sequence-sharded
# attention TP, or the layer-boundary attention input scatter):
# each attention-TP rank holds its own slice of the DP shard, so
# it is a shard of its own rather than a replica of its peers.
token_shard_rank = (
token_shard_rank * parallel.attn_tp_size + parallel.attn_tp_rank
)
num_token_shards *= parallel.attn_tp_size
else:
# Gathered MoE presents one global token buffer to every rank, so
# its routing must remain identical across those replicas.
token_shard_rank, num_token_shards = 0, 1
override_topk_ids, override_topk_weights = topk_ids, topk_weights
if num_routed_topk < top_k:
# Only the routed columns are simulated; a fused shared-expert
# column keeps the id and weight the router gave it.
override_topk_ids = topk_ids[:, :num_routed_topk]
override_topk_weights = topk_weights[:, :num_routed_topk]
_simulate_balanced_routing(
override_topk_ids,
override_topk_weights,
num_experts,
random=simulate_uniform_experts,
num_ranks=parallel.moe_ep_size,
layer_id=layer_id,
token_shard_rank=token_shard_rank,
num_token_shards=num_token_shards,
)
def is_routed_scaling_factor_fused_in_topk(topk_config: TopKConfig) -> bool:
"""Whether routing already applied the scale that the MoE epilogue must skip."""
if topk_config.apply_routed_scaling_factor_on_output:
@@ -3043,44 +3101,13 @@ def select_experts(
topk_weights[:, :num_routed_for_gate] *= routed_scaling_factor
if routing_overridden:
# Benchmark-only: override gating with a balanced expert assignment (so
# dummy/random benchmark tokens don't skew MoE load) via a single fused
# Triton kernel — one launch instead of the ~5-7 small elementwise ops it
# replaces, to minimize timing perturbation. Do NOT use in production.
parallel = get_parallel()
if is_moe_input_scattered_across_dp_ranks():
token_shard_rank = parallel.attn_dp_rank
num_token_shards = parallel.attn_dp_size
forward = get_forward()
if forward.attn_tp_sequence_sharded or forward.attn_input_scattered:
# This forward routes attention-TP-local rows (sequence-sharded
# attention TP, or the layer-boundary attention input scatter):
# each attention-TP rank holds its own slice of the DP shard, so
# it is a shard of its own rather than a replica of its peers.
token_shard_rank = (
token_shard_rank * parallel.attn_tp_size + parallel.attn_tp_rank
)
num_token_shards *= parallel.attn_tp_size
else:
# Gathered MoE presents one global token buffer to every rank, so
# its routing must remain identical across those replicas.
token_shard_rank, num_token_shards = 0, 1
override_topk_ids, override_topk_weights = topk_ids, topk_weights
if num_routed_topk < top_k:
# Only the routed columns are simulated; a fused shared-expert
# column keeps the id and weight the router gave it.
override_topk_ids = topk_ids[:, :num_routed_topk]
override_topk_weights = topk_weights[:, :num_routed_topk]
_simulate_balanced_routing(
override_topk_ids,
override_topk_weights,
apply_routing_simulation(
topk_ids,
topk_weights,
router_logits.shape[1],
random=simulate_uniform_experts,
num_ranks=parallel.moe_ep_size,
num_routed_topk=num_routed_topk,
top_k=top_k,
layer_id=layer_id,
token_shard_rank=token_shard_rank,
num_token_shards=num_token_shards,
)
# The override rewrote every row, including the router-masked ones.
padded_rows_masked = False
@@ -10,6 +10,8 @@ from sglang.srt.layers.moe.topk import (
_mask_topk_ids_padded_region,
_post_process_topk_ids,
_zero_topk_weights_padded_region,
apply_routing_simulation,
routing_simulation_enabled,
)
from sglang.srt.layers.moe.utils import (
get_moe_a2a_backend,
@@ -138,6 +140,16 @@ def _finish_vision_topk(
):
config = moe.topk.topk_config
use_mega_moe = get_moe_a2a_backend().is_megamoe()
if routing_simulation_enabled():
apply_routing_simulation(
indices,
weights,
logits.shape[1],
num_routed_topk=config.top_k - config.num_shared_topk_slots,
top_k=config.top_k,
layer_id=moe.layer_id,
)
padded_rows_masked = False
indices, weights, recorder_ids = _post_process_topk_ids(
indices,
weights,