From e2e1a97fc3e057f00ba464d49b46e723403811e6 Mon Sep 17 00:00:00 2001 From: Aditya Suresh <152752311+adi-suresh01@users.noreply.github.com> Date: Tue, 20 Oct 2026 19:07:48 -0400 Subject: [PATCH] [Fix] moe: Map expert ids for layers whose runner needs local ids under EP (#44480) Co-authored-by: Cheng Wan --- .../srt/layers/moe/fused_moe_triton/layer.py | 13 +- .../layers/moe/token_dispatcher/standard.py | 62 ++++++-- ...test_standard_dispatcher_runner_backend.py | 149 ++++++++++++++++++ .../unit/lora/test_mem_pool_ep_unit.py | 12 +- 4 files changed, 217 insertions(+), 19 deletions(-) create mode 100644 test/registered/unit/layers/moe/test_standard_dispatcher_runner_backend.py diff --git a/python/sglang/srt/layers/moe/fused_moe_triton/layer.py b/python/sglang/srt/layers/moe/fused_moe_triton/layer.py index af80737145..40fdec171c 100644 --- a/python/sglang/srt/layers/moe/fused_moe_triton/layer.py +++ b/python/sglang/srt/layers/moe/fused_moe_triton/layer.py @@ -165,6 +165,15 @@ def _get_deepep_comm_group(a2a_backend): return group +def _built_runner_backend(quant_method: FusedMoEMethodBase): + # The layer calls create_moe_runner before building its dispatcher, so the + # runner the quant method actually picked is known here. Methods that keep + # no runner of their own (including wrappers that build one on an inner + # scheme) return None and the dispatcher follows --moe-runner-backend. + runner = quant_method.runner + return None if runner is None else runner.runner_backend + + def create_moe_dispatcher( moe_runner_config: MoeRunnerConfig, quant_method: FusedMoEMethodBase, @@ -183,7 +192,9 @@ def create_moe_dispatcher( # StandardDispatcher is created but never invoked. # flashinfer_megamoe does its EP all-to-all inside the kernel, so the # dispatcher stays a pure noop passthrough. - return StandardDispatcher(moe_runner_config) + return StandardDispatcher( + moe_runner_config, runner_backend=_built_runner_backend(quant_method) + ) elif ( a2a_backend.is_deepep() or a2a_backend.is_mooncake() diff --git a/python/sglang/srt/layers/moe/token_dispatcher/standard.py b/python/sglang/srt/layers/moe/token_dispatcher/standard.py index 0bb57e62f4..7ae47fa9d2 100644 --- a/python/sglang/srt/layers/moe/token_dispatcher/standard.py +++ b/python/sglang/srt/layers/moe/token_dispatcher/standard.py @@ -28,7 +28,7 @@ from sglang.srt.layers.moe.utils import ( get_moe_runner_backend, should_use_flashinfer_cutlass_moe_fp4_allgather, ) -from sglang.srt.runtime_context import get_parallel +from sglang.srt.runtime_context import get_lora, get_parallel from sglang.srt.utils.common import ( get_bool_env_var, get_device, @@ -44,6 +44,7 @@ _MASK_DP_PAD_MOE = _envs.SGLANG_OPT_MASK_DP_PAD_MOE.get() if TYPE_CHECKING: from sglang.srt.layers.moe.topk import TopKOutput + from sglang.srt.layers.moe.utils import MoeRunnerBackendLike try: @@ -102,8 +103,30 @@ def _map_topk_ids_to_local_experts( ) +def _keeps_global_expert_ids(backend: MoeRunnerBackendLike) -> bool: + # These runners handle EP with global expert IDs: + # - cutlass / cutedsl / trtllm_routed handle EP internally + # - mxfp4 dispatcher mapping is already global + # - hpc_ops consumes global ids together with rank_ep / num_expert_total + # - flashinfer_megamoe routes by global expert ID inside the mega kernel + return ( + backend.is_flashinfer_cutlass() + or backend.is_flashinfer_cutedsl() + or backend.is_flashinfer_trtllm() + or backend.is_experimental_sgl_trtllm() + or backend.is_flashinfer_trtllm_routed() + or backend.is_hpc_ops() + or backend.is_flashinfer_megamoe() + or backend.is_flashinfer_mxfp4() + ) + + class StandardDispatcher(BaseDispatcher): - def __init__(self, moe_runner_config: MoeRunnerConfig): + def __init__( + self, + moe_runner_config: MoeRunnerConfig, + runner_backend: Optional[MoeRunnerBackendLike] = None, + ): super().__init__() self.moe_ep_size = get_parallel().moe_ep_size backend = get_moe_runner_backend() @@ -115,21 +138,26 @@ class StandardDispatcher(BaseDispatcher): self.use_aiter_moe_runner = backend.is_aiter() or ( backend.is_auto() and _use_aiter and get_moe_a2a_backend().supports_aiter() ) - # Skip local expert mapping when the backend handles EP with global expert IDs: - # - cutlass / cutedsl / trtllm_routed handle EP internally - # - mxfp4 dispatcher mapping is already global - # - hpc_ops consumes global ids together with rank_ep / num_expert_total - # - flashinfer_megamoe routes by global expert ID inside the mega kernel - self.skip_local_expert_mapping = ( - backend.is_flashinfer_cutlass() - or backend.is_flashinfer_cutedsl() - or backend.is_flashinfer_trtllm() - or backend.is_experimental_sgl_trtllm() - or backend.is_flashinfer_trtllm_routed() - or backend.is_hpc_ops() - or backend.is_flashinfer_megamoe() - or self.enable_flashinfer_mxfp4_moe - ) + self.skip_local_expert_mapping = _keeps_global_expert_ids(backend) + # A built runner may enable local-ID mapping; all other flags above + # follow the requested backend. + if ( + self.skip_local_expert_mapping + and runner_backend is not None + and not _keeps_global_expert_ids(runner_backend) + ): + # The LoRA pool keys experts by the global backend's convention, so + # it would read the wrong adapters for this layer under EP. + if self.moe_ep_size > 1 and ( + get_lora().enable_lora or get_lora().lora_paths + ): + raise ValueError( + "LoRA with expert parallelism is not supported on this MoE " + f"layer: --moe-runner-backend {backend.value} keeps global " + f"expert ids, but the layer's quantization method runs the " + f"'{runner_backend.value}' runner, which needs local ids." + ) + self.skip_local_expert_mapping = False self.num_experts = moe_runner_config.num_experts self.num_local_experts = moe_runner_config.num_local_experts self.num_local_shared_experts = moe_runner_config.num_fused_shared_experts diff --git a/test/registered/unit/layers/moe/test_standard_dispatcher_runner_backend.py b/test/registered/unit/layers/moe/test_standard_dispatcher_runner_backend.py new file mode 100644 index 0000000000..e0ea1e401d --- /dev/null +++ b/test/registered/unit/layers/moe/test_standard_dispatcher_runner_backend.py @@ -0,0 +1,149 @@ +"""StandardDispatcher must give local expert ids to a runner that needs them. + +Fp8MoEMethod runs its fp8 experts on triton when --moe-runner-backend is +flashinfer_cutlass, which NVFP4 checkpoints with fp8 MTP experts pin on sm120. +A dispatcher that only reads the global backend keeps global expert ids for +that layer, and under EP the triton kernel indexes experts the rank does not +hold (an illegal memory access on the draft extend of +nvidia/Qwen3.8-Flash-Next-NVFP4 at --tp 2 --ep 2). +""" + +from types import SimpleNamespace + +import pytest +import torch + +from sglang.srt.layers.moe.moe_runner.base import MoeRunnerConfig +from sglang.srt.layers.moe.token_dispatcher import standard +from sglang.srt.layers.moe.topk import StandardTopKOutput +from sglang.srt.layers.moe.utils import MoeA2ABackend, MoeRunnerBackend +from sglang.srt.runtime_context import get_flags +from sglang.test.ci.ci_register import register_cpu_ci + +register_cpu_ci(est_time=8, suite="stage-b-test-cpu-intel") + +NUM_EXPERTS = 8 +NUM_LOCAL_EXPERTS = 4 + + +@pytest.fixture +def ep2_rank0(monkeypatch): + moe = get_flags().moe + saved = (moe.runner_backend, moe.a2a_backend) + moe.a2a_backend = MoeA2ABackend.NONE + monkeypatch.setattr(standard, "get_device", lambda: "cpu") + monkeypatch.setattr( + standard, + "get_parallel", + lambda: SimpleNamespace(moe_ep_size=2, moe_ep_rank=0), + ) + monkeypatch.setattr( + standard, + "get_lora", + lambda: SimpleNamespace(enable_lora=False, lora_paths=None), + raising=False, + ) + yield moe + moe.runner_backend, moe.a2a_backend = saved + + +@pytest.fixture +def cutlass_pin(ep2_rank0): + ep2_rank0.runner_backend = MoeRunnerBackend.FLASHINFER_CUTLASS + return ep2_rank0 + + +def _config(): + return MoeRunnerConfig( + num_experts=NUM_EXPERTS, + num_local_experts=NUM_LOCAL_EXPERTS, + num_fused_shared_experts=0, + top_k=2, + ) + + +def _dispatch_ids(dispatcher, topk_ids): + topk_output = StandardTopKOutput( + topk_weights=torch.ones(topk_ids.shape, device=topk_ids.device), + topk_ids=topk_ids, + router_logits=torch.zeros( + topk_ids.shape[0], NUM_EXPERTS, device=topk_ids.device + ), + ) + hidden_states = torch.zeros(topk_ids.shape[0], 16, device=topk_ids.device) + return dispatcher.dispatch(hidden_states, topk_output).topk_output.topk_ids + + +def test_global_backend_still_decides_without_a_runner(cutlass_pin): + dispatcher = standard.StandardDispatcher(_config()) + assert dispatcher.skip_local_expert_mapping + + +def test_triton_runner_gets_local_ids_under_cutlass_pin(cutlass_pin): + dispatcher = standard.StandardDispatcher( + _config(), runner_backend=MoeRunnerBackend.TRITON + ) + assert not dispatcher.skip_local_expert_mapping + + topk_ids = torch.tensor([[0, 5], [3, 7]], dtype=torch.int32, device="cpu") + local_ids = _dispatch_ids(dispatcher, topk_ids) + # Rank 0 holds global experts 0-3; experts on the other rank become the + # -1 drop sentinel instead of indexing past the local weights. + assert local_ids.tolist() == [[0, -1], [3, -1]] + + +def test_runner_that_keeps_global_ids_changes_nothing(cutlass_pin): + dispatcher = standard.StandardDispatcher( + _config(), runner_backend=MoeRunnerBackend.FLASHINFER_CUTLASS + ) + assert dispatcher.skip_local_expert_mapping + + +def test_runner_never_turns_mapping_off(ep2_rank0): + # Under auto a built runner is never AUTO, e.g. ModelOpt NVFP4 resolving + # to flashinfer_trtllm on sm100; the global layout must stay in charge. + ep2_rank0.runner_backend = MoeRunnerBackend.AUTO + dispatcher = standard.StandardDispatcher( + _config(), runner_backend=MoeRunnerBackend.FLASHINFER_TRTLLM + ) + assert not dispatcher.skip_local_expert_mapping + + +def test_aiter_flag_follows_global_backend(ep2_rank0, monkeypatch): + # Unquantized MoE on ROCm builds a triton runner next to its AITER one; + # the AITER expert mask must still be built from the global backend. + ep2_rank0.runner_backend = MoeRunnerBackend.AUTO + monkeypatch.setattr(standard, "_use_aiter", True) + dispatcher = standard.StandardDispatcher( + _config(), runner_backend=MoeRunnerBackend.TRITON + ) + assert dispatcher.use_aiter_moe_runner + + +def test_lora_with_ep_is_rejected_on_a_remapped_layer(cutlass_pin, monkeypatch): + monkeypatch.setattr( + standard, + "get_lora", + lambda: SimpleNamespace(enable_lora=True, lora_paths=None), + ) + with pytest.raises(ValueError, match="LoRA"): + standard.StandardDispatcher(_config(), runner_backend=MoeRunnerBackend.TRITON) + + +def test_fp8_triton_fallback_reaches_the_dispatcher(cutlass_pin): + from sglang.srt.layers.moe.fused_moe_triton.layer import create_moe_dispatcher + from sglang.srt.layers.quantization.fp8 import Fp8MoEMethod + + method = object.__new__(Fp8MoEMethod) + method.is_fp4_expert = False + method.create_moe_runner(SimpleNamespace(), _config()) + assert method.runner.runner_backend.is_triton() + + dispatcher = create_moe_dispatcher(_config(), method) + assert not dispatcher.skip_local_expert_mapping + + +if __name__ == "__main__": + import sys + + sys.exit(pytest.main([__file__, "-v"])) diff --git a/test/registered/unit/lora/test_mem_pool_ep_unit.py b/test/registered/unit/lora/test_mem_pool_ep_unit.py index c36cf338ff..4d1136f079 100644 --- a/test/registered/unit/lora/test_mem_pool_ep_unit.py +++ b/test/registered/unit/lora/test_mem_pool_ep_unit.py @@ -20,6 +20,7 @@ register_cpu_ci(est_time=8, suite="base-a-test-cpu") import ast import types +import typing import unittest import unittest.mock as mock from enum import Enum @@ -156,6 +157,11 @@ def _load_standard_dispatcher(get_parallel, get_backend): def __init__(self): pass + keeps_global_ids = next( + node + for node in tree.body + if isinstance(node, ast.FunctionDef) and node.name == "_keeps_global_expert_ids" + ) test_class = ast.ClassDef( name="_StandardDispatcherUnderTest", bases=[ast.Name(id="_DispatcherBase", ctx=ast.Load())], @@ -166,6 +172,8 @@ def _load_standard_dispatcher(get_parallel, get_backend): namespace = { "_DispatcherBase": _DispatcherBase, "MoeRunnerConfig": object, + "MoeRunnerBackendLike": object, + "Optional": typing.Optional, "get_parallel": get_parallel, "get_moe_runner_backend": get_backend, "_use_aiter": False, @@ -175,7 +183,9 @@ def _load_standard_dispatcher(get_parallel, get_backend): } exec( compile( - ast.fix_missing_locations(ast.Module(body=[test_class], type_ignores=[])), + ast.fix_missing_locations( + ast.Module(body=[keeps_global_ids, test_class], type_ignores=[]) + ), str(STANDARD_DISPATCHER_PATH), "exec", ),