[Fix] moe: Preserve negative routing sentinel in EP mapping (#40498)

This commit is contained in:
William Hu
2026-10-15 15:05:08 -07:00
committed by GitHub
parent 91f6ef2322
commit 4e0609ea27
3 changed files with 63 additions and 9 deletions
+3 -4
View File
@@ -799,10 +799,9 @@ def mask_dp_pad_moe_topk_ids(topk_ids: torch.Tensor) -> None:
outputs are then discarded by the post-reorder scatter — pure wasted
compute, and a masked-grouped-GEMM workspace blow-up when they collide
on the same top-k. -1 is the drop sentinel both the triton fused_moe
(filter_expert) and the DeepGEMM EP preprocess honor; it must be applied
AFTER the local_expert_mapping gather (a pre-translation -1 aliases to
the mapping table's last entry). Capture-safe: per-batch state is read
only from the replay-updated global_num_tokens_gpu tensor.
(filter_expert) and the DeepGEMM EP preprocess honor. Capture-safe:
per-batch state is read only from the replay-updated
global_num_tokens_gpu tensor.
"""
counts = _DpGatheredBufferWrapper.get_dp_global_num_tokens_gpu()
if counts is None:
@@ -91,6 +91,17 @@ class StandardCombineInput(NamedTuple):
assert isinstance(StandardCombineInput, CombineInput)
def _map_topk_ids_to_local_experts(
topk_ids: torch.Tensor, local_expert_mapping: torch.Tensor
) -> torch.Tensor:
"""Map global expert ids while preserving the ``-1`` drop sentinel."""
return torch.where(
topk_ids < 0,
-1,
local_expert_mapping[topk_ids.clamp_min(0)],
)
class StandardDispatcher(BaseDispatcher):
def __init__(self, moe_runner_config: MoeRunnerConfig):
super().__init__()
@@ -223,15 +234,15 @@ class StandardDispatcher(BaseDispatcher):
)
elif not self.use_aiter_moe_runner:
if TopKOutputChecker.format_is_standard(topk_output):
topk_ids_local = self.local_expert_mapping[topk_output.topk_ids]
topk_ids_local = _map_topk_ids_to_local_experts(
topk_output.topk_ids, self.local_expert_mapping
)
# Drop dp-attention MAX_LEN pad rows from the dispatch:
# pad rows carry stale hidden through the router and
# their expert outputs are discarded downstream — pure
# wasted compute (and a masked-grouped-GEMM workspace
# blow-up when they collide on the same top-k). Must
# run POST-translation (a pre-translation -1 aliases to
# the mapping table's last entry); -1 is the drop
# sentinel both the triton and deep_gemm runners honor.
# blow-up when they collide on the same top-k). -1 is the
# drop sentinel both the triton and deep_gemm runners honor.
if _MASK_DP_PAD_MOE and is_dp_max_padding():
mask_dp_pad_moe_topk_ids(topk_ids_local)
topk_output = topk_output._replace(topk_ids=topk_ids_local)
@@ -0,0 +1,44 @@
import sys
from unittest.mock import patch
import torch
from sglang.srt.layers.moe.token_dispatcher.standard import (
StandardDispatcher,
)
from sglang.srt.layers.moe.topk import StandardTopKOutput
from sglang.test.ci.ci_register import register_cpu_ci
register_cpu_ci(est_time=1, suite="base-a-test-cpu")
def test_dispatch_preserves_negative_routing_sentinel():
dispatcher = object.__new__(StandardDispatcher)
dispatcher.moe_ep_size = 2
dispatcher.skip_local_expert_mapping = False
dispatcher.local_expert_mapping = torch.tensor([-1, -1, 0, 1], dtype=torch.int32)
dispatcher.use_aiter_moe_runner = False
dispatcher.expert_mask_gpu = None
topk_ids = torch.tensor([[-1, 3], [2, 0]], dtype=torch.int32)
topk_output = StandardTopKOutput(
topk_weights=torch.ones_like(topk_ids, dtype=torch.float32),
topk_ids=topk_ids,
router_logits=None,
)
with patch(
"sglang.srt.layers.moe.token_dispatcher.standard."
"should_use_flashinfer_cutlass_moe_fp4_allgather",
return_value=False,
):
output = dispatcher.dispatch(torch.ones((2, 4)), topk_output)
expected = torch.tensor([[-1, 1], [0, -1]], dtype=torch.int32)
assert torch.equal(output.topk_output.topk_ids, expected)
if __name__ == "__main__":
import pytest
sys.exit(pytest.main([__file__, "-v"]))