[Fix] moe: Apply SGLANG_SIMULATE_* routing on DeepSeek-V4.1 vision-biased gates (#45628)
This commit is contained in:
@@ -2386,6 +2386,64 @@ else:
|
||||
fused_topk_native = fused_topk_torch_native
|
||||
|
||||
|
||||
def routing_simulation_enabled() -> bool:
|
||||
return (
|
||||
envs.SGLANG_SIMULATE_UNIFORM_EXPERTS.get()
|
||||
or envs.SGLANG_SIMULATE_ROUND_ROBIN_EXPERTS.get()
|
||||
)
|
||||
|
||||
|
||||
def apply_routing_simulation(
|
||||
topk_ids: torch.Tensor,
|
||||
topk_weights: torch.Tensor,
|
||||
num_experts: int,
|
||||
*,
|
||||
num_routed_topk: int,
|
||||
top_k: int,
|
||||
layer_id: Optional[int],
|
||||
) -> None:
|
||||
simulate_uniform_experts = envs.SGLANG_SIMULATE_UNIFORM_EXPERTS.get()
|
||||
# Benchmark-only: override gating with a balanced expert assignment (so
|
||||
# dummy/random benchmark tokens don't skew MoE load) via a single fused
|
||||
# Triton kernel — one launch instead of the ~5-7 small elementwise ops it
|
||||
# replaces, to minimize timing perturbation. Do NOT use in production.
|
||||
parallel = get_parallel()
|
||||
if is_moe_input_scattered_across_dp_ranks():
|
||||
token_shard_rank = parallel.attn_dp_rank
|
||||
num_token_shards = parallel.attn_dp_size
|
||||
forward = get_forward()
|
||||
if forward.attn_tp_sequence_sharded or forward.attn_input_scattered:
|
||||
# This forward routes attention-TP-local rows (sequence-sharded
|
||||
# attention TP, or the layer-boundary attention input scatter):
|
||||
# each attention-TP rank holds its own slice of the DP shard, so
|
||||
# it is a shard of its own rather than a replica of its peers.
|
||||
token_shard_rank = (
|
||||
token_shard_rank * parallel.attn_tp_size + parallel.attn_tp_rank
|
||||
)
|
||||
num_token_shards *= parallel.attn_tp_size
|
||||
else:
|
||||
# Gathered MoE presents one global token buffer to every rank, so
|
||||
# its routing must remain identical across those replicas.
|
||||
token_shard_rank, num_token_shards = 0, 1
|
||||
|
||||
override_topk_ids, override_topk_weights = topk_ids, topk_weights
|
||||
if num_routed_topk < top_k:
|
||||
# Only the routed columns are simulated; a fused shared-expert
|
||||
# column keeps the id and weight the router gave it.
|
||||
override_topk_ids = topk_ids[:, :num_routed_topk]
|
||||
override_topk_weights = topk_weights[:, :num_routed_topk]
|
||||
_simulate_balanced_routing(
|
||||
override_topk_ids,
|
||||
override_topk_weights,
|
||||
num_experts,
|
||||
random=simulate_uniform_experts,
|
||||
num_ranks=parallel.moe_ep_size,
|
||||
layer_id=layer_id,
|
||||
token_shard_rank=token_shard_rank,
|
||||
num_token_shards=num_token_shards,
|
||||
)
|
||||
|
||||
|
||||
def is_routed_scaling_factor_fused_in_topk(topk_config: TopKConfig) -> bool:
|
||||
"""Whether routing already applied the scale that the MoE epilogue must skip."""
|
||||
if topk_config.apply_routed_scaling_factor_on_output:
|
||||
@@ -3043,44 +3101,13 @@ def select_experts(
|
||||
topk_weights[:, :num_routed_for_gate] *= routed_scaling_factor
|
||||
|
||||
if routing_overridden:
|
||||
# Benchmark-only: override gating with a balanced expert assignment (so
|
||||
# dummy/random benchmark tokens don't skew MoE load) via a single fused
|
||||
# Triton kernel — one launch instead of the ~5-7 small elementwise ops it
|
||||
# replaces, to minimize timing perturbation. Do NOT use in production.
|
||||
parallel = get_parallel()
|
||||
if is_moe_input_scattered_across_dp_ranks():
|
||||
token_shard_rank = parallel.attn_dp_rank
|
||||
num_token_shards = parallel.attn_dp_size
|
||||
forward = get_forward()
|
||||
if forward.attn_tp_sequence_sharded or forward.attn_input_scattered:
|
||||
# This forward routes attention-TP-local rows (sequence-sharded
|
||||
# attention TP, or the layer-boundary attention input scatter):
|
||||
# each attention-TP rank holds its own slice of the DP shard, so
|
||||
# it is a shard of its own rather than a replica of its peers.
|
||||
token_shard_rank = (
|
||||
token_shard_rank * parallel.attn_tp_size + parallel.attn_tp_rank
|
||||
)
|
||||
num_token_shards *= parallel.attn_tp_size
|
||||
else:
|
||||
# Gathered MoE presents one global token buffer to every rank, so
|
||||
# its routing must remain identical across those replicas.
|
||||
token_shard_rank, num_token_shards = 0, 1
|
||||
|
||||
override_topk_ids, override_topk_weights = topk_ids, topk_weights
|
||||
if num_routed_topk < top_k:
|
||||
# Only the routed columns are simulated; a fused shared-expert
|
||||
# column keeps the id and weight the router gave it.
|
||||
override_topk_ids = topk_ids[:, :num_routed_topk]
|
||||
override_topk_weights = topk_weights[:, :num_routed_topk]
|
||||
_simulate_balanced_routing(
|
||||
override_topk_ids,
|
||||
override_topk_weights,
|
||||
apply_routing_simulation(
|
||||
topk_ids,
|
||||
topk_weights,
|
||||
router_logits.shape[1],
|
||||
random=simulate_uniform_experts,
|
||||
num_ranks=parallel.moe_ep_size,
|
||||
num_routed_topk=num_routed_topk,
|
||||
top_k=top_k,
|
||||
layer_id=layer_id,
|
||||
token_shard_rank=token_shard_rank,
|
||||
num_token_shards=num_token_shards,
|
||||
)
|
||||
# The override rewrote every row, including the router-masked ones.
|
||||
padded_rows_masked = False
|
||||
|
||||
@@ -10,6 +10,8 @@ from sglang.srt.layers.moe.topk import (
|
||||
_mask_topk_ids_padded_region,
|
||||
_post_process_topk_ids,
|
||||
_zero_topk_weights_padded_region,
|
||||
apply_routing_simulation,
|
||||
routing_simulation_enabled,
|
||||
)
|
||||
from sglang.srt.layers.moe.utils import (
|
||||
get_moe_a2a_backend,
|
||||
@@ -138,6 +140,16 @@ def _finish_vision_topk(
|
||||
):
|
||||
config = moe.topk.topk_config
|
||||
use_mega_moe = get_moe_a2a_backend().is_megamoe()
|
||||
if routing_simulation_enabled():
|
||||
apply_routing_simulation(
|
||||
indices,
|
||||
weights,
|
||||
logits.shape[1],
|
||||
num_routed_topk=config.top_k - config.num_shared_topk_slots,
|
||||
top_k=config.top_k,
|
||||
layer_id=moe.layer_id,
|
||||
)
|
||||
padded_rows_masked = False
|
||||
indices, weights, recorder_ids = _post_process_topk_ids(
|
||||
indices,
|
||||
weights,
|
||||
|
||||
Reference in New Issue
Block a user