diff --git a/python/sglang/srt/layers/dp_attention.py b/python/sglang/srt/layers/dp_attention.py index 2f67a25043..022f29efd1 100644 --- a/python/sglang/srt/layers/dp_attention.py +++ b/python/sglang/srt/layers/dp_attention.py @@ -799,10 +799,9 @@ def mask_dp_pad_moe_topk_ids(topk_ids: torch.Tensor) -> None: outputs are then discarded by the post-reorder scatter — pure wasted compute, and a masked-grouped-GEMM workspace blow-up when they collide on the same top-k. -1 is the drop sentinel both the triton fused_moe - (filter_expert) and the DeepGEMM EP preprocess honor; it must be applied - AFTER the local_expert_mapping gather (a pre-translation -1 aliases to - the mapping table's last entry). Capture-safe: per-batch state is read - only from the replay-updated global_num_tokens_gpu tensor. + (filter_expert) and the DeepGEMM EP preprocess honor. Capture-safe: + per-batch state is read only from the replay-updated + global_num_tokens_gpu tensor. """ counts = _DpGatheredBufferWrapper.get_dp_global_num_tokens_gpu() if counts is None: diff --git a/python/sglang/srt/layers/moe/token_dispatcher/standard.py b/python/sglang/srt/layers/moe/token_dispatcher/standard.py index e2bf176816..0bb57e62f4 100644 --- a/python/sglang/srt/layers/moe/token_dispatcher/standard.py +++ b/python/sglang/srt/layers/moe/token_dispatcher/standard.py @@ -91,6 +91,17 @@ class StandardCombineInput(NamedTuple): assert isinstance(StandardCombineInput, CombineInput) +def _map_topk_ids_to_local_experts( + topk_ids: torch.Tensor, local_expert_mapping: torch.Tensor +) -> torch.Tensor: + """Map global expert ids while preserving the ``-1`` drop sentinel.""" + return torch.where( + topk_ids < 0, + -1, + local_expert_mapping[topk_ids.clamp_min(0)], + ) + + class StandardDispatcher(BaseDispatcher): def __init__(self, moe_runner_config: MoeRunnerConfig): super().__init__() @@ -223,15 +234,15 @@ class StandardDispatcher(BaseDispatcher): ) elif not self.use_aiter_moe_runner: if TopKOutputChecker.format_is_standard(topk_output): - topk_ids_local = self.local_expert_mapping[topk_output.topk_ids] + topk_ids_local = _map_topk_ids_to_local_experts( + topk_output.topk_ids, self.local_expert_mapping + ) # Drop dp-attention MAX_LEN pad rows from the dispatch: # pad rows carry stale hidden through the router and # their expert outputs are discarded downstream — pure # wasted compute (and a masked-grouped-GEMM workspace - # blow-up when they collide on the same top-k). Must - # run POST-translation (a pre-translation -1 aliases to - # the mapping table's last entry); -1 is the drop - # sentinel both the triton and deep_gemm runners honor. + # blow-up when they collide on the same top-k). -1 is the + # drop sentinel both the triton and deep_gemm runners honor. if _MASK_DP_PAD_MOE and is_dp_max_padding(): mask_dp_pad_moe_topk_ids(topk_ids_local) topk_output = topk_output._replace(topk_ids=topk_ids_local) diff --git a/test/registered/unit/layers/moe/test_standard_dispatcher.py b/test/registered/unit/layers/moe/test_standard_dispatcher.py new file mode 100644 index 0000000000..2e0e42772e --- /dev/null +++ b/test/registered/unit/layers/moe/test_standard_dispatcher.py @@ -0,0 +1,44 @@ +import sys +from unittest.mock import patch + +import torch + +from sglang.srt.layers.moe.token_dispatcher.standard import ( + StandardDispatcher, +) +from sglang.srt.layers.moe.topk import StandardTopKOutput +from sglang.test.ci.ci_register import register_cpu_ci + +register_cpu_ci(est_time=1, suite="base-a-test-cpu") + + +def test_dispatch_preserves_negative_routing_sentinel(): + dispatcher = object.__new__(StandardDispatcher) + dispatcher.moe_ep_size = 2 + dispatcher.skip_local_expert_mapping = False + dispatcher.local_expert_mapping = torch.tensor([-1, -1, 0, 1], dtype=torch.int32) + dispatcher.use_aiter_moe_runner = False + dispatcher.expert_mask_gpu = None + + topk_ids = torch.tensor([[-1, 3], [2, 0]], dtype=torch.int32) + topk_output = StandardTopKOutput( + topk_weights=torch.ones_like(topk_ids, dtype=torch.float32), + topk_ids=topk_ids, + router_logits=None, + ) + + with patch( + "sglang.srt.layers.moe.token_dispatcher.standard." + "should_use_flashinfer_cutlass_moe_fp4_allgather", + return_value=False, + ): + output = dispatcher.dispatch(torch.ones((2, 4)), topk_output) + + expected = torch.tensor([[-1, 1], [0, -1]], dtype=torch.int32) + assert torch.equal(output.topk_output.topk_ids, expected) + + +if __name__ == "__main__": + import pytest + + sys.exit(pytest.main([__file__, "-v"]))