[Fix] moe: Preserve negative routing sentinel in EP mapping (#40498)
This commit is contained in:
@@ -799,10 +799,9 @@ def mask_dp_pad_moe_topk_ids(topk_ids: torch.Tensor) -> None:
|
||||
outputs are then discarded by the post-reorder scatter — pure wasted
|
||||
compute, and a masked-grouped-GEMM workspace blow-up when they collide
|
||||
on the same top-k. -1 is the drop sentinel both the triton fused_moe
|
||||
(filter_expert) and the DeepGEMM EP preprocess honor; it must be applied
|
||||
AFTER the local_expert_mapping gather (a pre-translation -1 aliases to
|
||||
the mapping table's last entry). Capture-safe: per-batch state is read
|
||||
only from the replay-updated global_num_tokens_gpu tensor.
|
||||
(filter_expert) and the DeepGEMM EP preprocess honor. Capture-safe:
|
||||
per-batch state is read only from the replay-updated
|
||||
global_num_tokens_gpu tensor.
|
||||
"""
|
||||
counts = _DpGatheredBufferWrapper.get_dp_global_num_tokens_gpu()
|
||||
if counts is None:
|
||||
|
||||
@@ -91,6 +91,17 @@ class StandardCombineInput(NamedTuple):
|
||||
assert isinstance(StandardCombineInput, CombineInput)
|
||||
|
||||
|
||||
def _map_topk_ids_to_local_experts(
|
||||
topk_ids: torch.Tensor, local_expert_mapping: torch.Tensor
|
||||
) -> torch.Tensor:
|
||||
"""Map global expert ids while preserving the ``-1`` drop sentinel."""
|
||||
return torch.where(
|
||||
topk_ids < 0,
|
||||
-1,
|
||||
local_expert_mapping[topk_ids.clamp_min(0)],
|
||||
)
|
||||
|
||||
|
||||
class StandardDispatcher(BaseDispatcher):
|
||||
def __init__(self, moe_runner_config: MoeRunnerConfig):
|
||||
super().__init__()
|
||||
@@ -223,15 +234,15 @@ class StandardDispatcher(BaseDispatcher):
|
||||
)
|
||||
elif not self.use_aiter_moe_runner:
|
||||
if TopKOutputChecker.format_is_standard(topk_output):
|
||||
topk_ids_local = self.local_expert_mapping[topk_output.topk_ids]
|
||||
topk_ids_local = _map_topk_ids_to_local_experts(
|
||||
topk_output.topk_ids, self.local_expert_mapping
|
||||
)
|
||||
# Drop dp-attention MAX_LEN pad rows from the dispatch:
|
||||
# pad rows carry stale hidden through the router and
|
||||
# their expert outputs are discarded downstream — pure
|
||||
# wasted compute (and a masked-grouped-GEMM workspace
|
||||
# blow-up when they collide on the same top-k). Must
|
||||
# run POST-translation (a pre-translation -1 aliases to
|
||||
# the mapping table's last entry); -1 is the drop
|
||||
# sentinel both the triton and deep_gemm runners honor.
|
||||
# blow-up when they collide on the same top-k). -1 is the
|
||||
# drop sentinel both the triton and deep_gemm runners honor.
|
||||
if _MASK_DP_PAD_MOE and is_dp_max_padding():
|
||||
mask_dp_pad_moe_topk_ids(topk_ids_local)
|
||||
topk_output = topk_output._replace(topk_ids=topk_ids_local)
|
||||
|
||||
@@ -0,0 +1,44 @@
|
||||
import sys
|
||||
from unittest.mock import patch
|
||||
|
||||
import torch
|
||||
|
||||
from sglang.srt.layers.moe.token_dispatcher.standard import (
|
||||
StandardDispatcher,
|
||||
)
|
||||
from sglang.srt.layers.moe.topk import StandardTopKOutput
|
||||
from sglang.test.ci.ci_register import register_cpu_ci
|
||||
|
||||
register_cpu_ci(est_time=1, suite="base-a-test-cpu")
|
||||
|
||||
|
||||
def test_dispatch_preserves_negative_routing_sentinel():
|
||||
dispatcher = object.__new__(StandardDispatcher)
|
||||
dispatcher.moe_ep_size = 2
|
||||
dispatcher.skip_local_expert_mapping = False
|
||||
dispatcher.local_expert_mapping = torch.tensor([-1, -1, 0, 1], dtype=torch.int32)
|
||||
dispatcher.use_aiter_moe_runner = False
|
||||
dispatcher.expert_mask_gpu = None
|
||||
|
||||
topk_ids = torch.tensor([[-1, 3], [2, 0]], dtype=torch.int32)
|
||||
topk_output = StandardTopKOutput(
|
||||
topk_weights=torch.ones_like(topk_ids, dtype=torch.float32),
|
||||
topk_ids=topk_ids,
|
||||
router_logits=None,
|
||||
)
|
||||
|
||||
with patch(
|
||||
"sglang.srt.layers.moe.token_dispatcher.standard."
|
||||
"should_use_flashinfer_cutlass_moe_fp4_allgather",
|
||||
return_value=False,
|
||||
):
|
||||
output = dispatcher.dispatch(torch.ones((2, 4)), topk_output)
|
||||
|
||||
expected = torch.tensor([[-1, 1], [0, -1]], dtype=torch.int32)
|
||||
assert torch.equal(output.topk_output.topk_ids, expected)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
import pytest
|
||||
|
||||
sys.exit(pytest.main([__file__, "-v"]))
|
||||
Reference in New Issue
Block a user