From 0d0be8fc9826405c49be28fafe53399687555dce Mon Sep 17 00:00:00 2001 From: ChengYM Date: Mon, 12 Oct 2026 11:54:13 +0800 Subject: [PATCH] [AMD][MORI] Add MORI EP V2 support (#40666) --- .../docs/references/environment_variables.mdx | 12 +- python/sglang/srt/arg_groups/moe_hook.py | 75 +- python/sglang/srt/environ.py | 11 + python/sglang/srt/eplb/expert_distribution.py | 11 +- python/sglang/srt/layers/moe/ep_moe/layer.py | 2 + .../sglang/srt/layers/moe/moe_runner/aiter.py | 130 +- .../srt/layers/moe/token_dispatcher/moriep.py | 1162 ++++++++++++----- python/sglang/srt/layers/moe/utils.py | 3 + .../srt/layers/quantization/fp8_utils.py | 11 +- .../sglang/srt/layers/quantization/mxfp4.py | 10 + .../sglang/srt/layers/quantization/unquant.py | 4 + python/sglang/srt/models/deepseek_v4.py | 34 +- .../sglang/srt/models/deepseek_v4_dspark.py | 7 +- python/sglang/srt/utils/common.py | 8 +- test/manual/dsv4/test_mori_epv2_dispatcher.py | 516 ++++++++ .../eplb/test_megamoe_prefill_recorder.py | 13 + .../unit/layers/moe/test_aiter_runner.py | 83 ++ 17 files changed, 1661 insertions(+), 431 deletions(-) create mode 100644 test/manual/dsv4/test_mori_epv2_dispatcher.py diff --git a/docs/docs/references/environment_variables.mdx b/docs/docs/references/environment_variables.mdx index 89122c3da7..b7aa1e550b 100644 --- a/docs/docs/references/environment_variables.mdx +++ b/docs/docs/references/environment_variables.mdx @@ -517,9 +517,19 @@ SGLang supports various environment variables that can be used to configure its SGLANG_MORI_MOE_MAX_INPUT_TOKENS - Truncate the dispatch buffer to this many rows before MoE computation, reducing kernel overhead on padding tokens. The value must be >= the actual number of received tokens (totalRecvTokenNum); setting it too small causes incorrect results. 0 disables truncation (use full buffer). + Manual MoE input row limit for MORI EPv1 and EPv2. A positive value overrides SGLANG_MORI_RECV_BOUND and is used unrounded, capped at the receive capacity; it only narrows the MoE input view, not the communication buffer. It must be >= the actual received tokens, or results are incorrect. 0 leaves it to SGLANG_MORI_RECV_BOUND. 0 + + SGLANG_MORI_EP_V2 + Select MORI EPv2 for --moe-a2a-backend mori. When unset, EPv2 is used only for DeepSeek-V4 models on AITER with ep_size <= 8 (on gfx1250, any ep_size that fits one UALink vPOD), a bfloat16 model dtype and MORI_ENABLE_SDMA off, and MORI EPv1 otherwise. Set to true to use EPv2 even when these checks fail (a warning names the failed check), or false to always use EPv1. + true + + + SGLANG_MORI_RECV_BOUND + Hand the MoE only the receive rows the per-rank DP token counts can fill (rounded up) instead of the full MORI receive buffer. Unverified layouts, missing metadata and two-batch overlap keep the full view. + false + SGLANG_PPLX_NUM_MAX_DISPATCH_TOKENS_PER_RANK Maximum number of dispatched tokens per rank for the PPLX (--moe-a2a-backend pplx) NVSHMEM buffer allocation. Must be ≥ the worst-case per-rank forward tokens (the per-rank prefill chunk chunked_prefill_size / dp_size, or the decode CUDA-graph batch size); the server refuses to start otherwise. diff --git a/python/sglang/srt/arg_groups/moe_hook.py b/python/sglang/srt/arg_groups/moe_hook.py index 7760431cf9..946a1f875d 100644 --- a/python/sglang/srt/arg_groups/moe_hook.py +++ b/python/sglang/srt/arg_groups/moe_hook.py @@ -30,7 +30,12 @@ from sglang.srt.connector import ConnectorType from sglang.srt.environ import envs from sglang.srt.model_executor.cuda_graph_config import Backend, Phase, with_phase from sglang.srt.runtime_context import attn_dp_enabled_of, get_platform -from sglang.srt.utils.common import is_sm100_supported, parse_connector_type +from sglang.srt.utils.common import ( + get_bool_env_var, + is_gfx1250_supported, + is_sm100_supported, + parse_connector_type, +) logger = logging.getLogger(__name__) @@ -460,6 +465,7 @@ def handle_a2a_moe(server_args: Any): ) if a2a_backend == "mori": + is_epv2 = _resolve_mori_ep_v2(server_args) if cfg.deepep_mode == "auto": declare_resolution( server_args, @@ -467,6 +473,16 @@ def handle_a2a_moe(server_args: Any): deepep_mode="normal", ) logger.warning("auto set deepep_mode=`normal` for MORI EP") + elif is_epv2 and cfg.deepep_mode == "low_latency": + declare_resolution( + server_args, + "_handle_a2a_moe", + deepep_mode="normal", + ) + logger.warning( + "MORI EPv2 has no separate low-latency kernel; " + "using deepep_mode=`normal` for deepep_mode=`low_latency`" + ) # Check chunked prefill for mori # Skip validation if chunked prefill is disabled (i.e., size <= 0). @@ -682,6 +698,63 @@ def validate_flashinfer_a2a_token_budget(server_args: Any) -> None: ) +def _resolve_mori_ep_v2(server_args: Any) -> bool: + """Whether to use MORI EPv2. + + Unset prefers EPv2 and falls back to EPv1 where it is unsupported; an explicit + true keeps EPv2 regardless, and false selects EPv1. + """ + if not envs.SGLANG_MORI_EP_V2.get(): + return False + + reason = _mori_epv2_unsupported_reason(server_args) + if reason is None: + return True + if envs.SGLANG_MORI_EP_V2.is_set(): + logger.warning( + "SGLANG_MORI_EP_V2=true is set explicitly, so MORI EPv2 is used even " + "though it does not support %s.", + reason, + ) + return True + + logger.warning( + "MORI EPv2 does not support %s; falling back to MORI EPv1. " + "Set SGLANG_MORI_EP_V2=true to force EPv2, or false to select EPv1.", + reason, + ) + # Worker processes inherit the env, so every reader agrees on EPv1. + envs.SGLANG_MORI_EP_V2.set(False) + return False + + +def _mori_epv2_unsupported_reason(server_args: Any) -> str | None: + # The dispatcher falls back from unsupported dispatch/combine dtypes once it + # knows the weight dtype. + import torch + + from sglang.srt.configs.model_config import is_deepseek_v4 + + cfg = resolved_view(server_args) + # gfx1250 forms its LSA team from the UALink vPOD, which can span hosts and is + # only known once cco is up, so init_mori_epv2_op checks that EP fits in it. + if cfg.tp_size > 8 and not is_gfx1250_supported(): + return f"ep_size={cfg.tp_size} across hosts (EPv2 is validated within one host)" + if not envs.SGLANG_USE_AITER.get(): + return "SGLANG_USE_AITER=0 (EPv2 experts run on the aiter MoE runner)" + if get_bool_env_var("MORI_ENABLE_SDMA", "false"): + return "MORI_ENABLE_SDMA=1 (SDMA is EPv1-only)" + if parse_connector_type(cfg.model_path) == ConnectorType.INSTANCE: + return "a model loaded through an instance connector (cannot validate it)" + model_config = model_config_of(server_args) + if model_config.dtype != torch.bfloat16: + return f"dtype={model_config.dtype} (EPv2 requires bfloat16)" + if not is_deepseek_v4(model_config.hf_config): + architecture = (model_config.hf_config.architectures or [None])[0] + return f"{architecture} (EPv2 is validated on DeepSeek-V4 only)" + return None + + def required_mori_dispatch_tokens_per_rank(server_args: Any) -> int: """Max tokens a single rank dispatches through MoRI in one forward.""" cfg = resolving_view(server_args) diff --git a/python/sglang/srt/environ.py b/python/sglang/srt/environ.py index 150648516f..c1f436eee5 100644 --- a/python/sglang/srt/environ.py +++ b/python/sglang/srt/environ.py @@ -864,6 +864,17 @@ class Envs: # and relies on the RDMA retry-exceeded timeout only. SGLANG_MORI_TRANSFER_TIMEOUT_MS = EnvInt(0) SGLANG_MORI_NUM_MAX_DISPATCH_TOKENS_PER_RANK = EnvInt(4096) + # Unset: EPv2 where moe_hook finds it supported (DeepSeek-V4 only), else EPv1. + # true forces EPv2; false forces EPv1. + SGLANG_MORI_EP_V2 = EnvBool(True) + SGLANG_MORI_DISPATCH_DTYPE = EnvStr("auto") + SGLANG_MORI_COMBINE_DTYPE = EnvStr("auto") + SGLANG_MORI_PREALLOC_MAX_RECV_TOKENS = EnvInt(0) + # Hand AITER only ceil32(sum of per-rank DP tokens) rows of the MORI EP receive + # buffer; unproved layouts or metadata keep the full view. Forces DP-synced buckets. + SGLANG_MORI_RECV_BOUND = EnvBool(False) + # Manual MoE input row limit (EPv1 and EPv2); 0 leaves it to SGLANG_MORI_RECV_BOUND. + SGLANG_MORI_MOE_MAX_INPUT_TOKENS = EnvInt(0) # =================================================================== # AMD, ROCm, and AITER diff --git a/python/sglang/srt/eplb/expert_distribution.py b/python/sglang/srt/eplb/expert_distribution.py index 4f99f17b79..070f08dea9 100644 --- a/python/sglang/srt/eplb/expert_distribution.py +++ b/python/sglang/srt/eplb/expert_distribution.py @@ -1092,9 +1092,14 @@ class _InfiniteBuffer(_Buffer): device = self._buffer.device if self._size == curr_buffer_size: - new_buffer = torch.zeros( - (2 * curr_buffer_size, *self._item_shape), dtype=dtype, device=device - ) + # Growth can happen inside a caller's inference_mode (e.g. DSpark draft); + # an inference tensor buffer would reject later appends made outside it. + with torch.inference_mode(False): + new_buffer = torch.zeros( + (2 * curr_buffer_size, *self._item_shape), + dtype=dtype, + device=device, + ) new_buffer[:curr_buffer_size] = self._buffer self._buffer = new_buffer diff --git a/python/sglang/srt/layers/moe/ep_moe/layer.py b/python/sglang/srt/layers/moe/ep_moe/layer.py index d445bfc71e..cd5164c3be 100644 --- a/python/sglang/srt/layers/moe/ep_moe/layer.py +++ b/python/sglang/srt/layers/moe/ep_moe/layer.py @@ -352,6 +352,8 @@ class DeepEPMoE(FusedMoE): def get_moe_impl_class(quant_config: Optional[QuantizationConfig]): + if get_moe_a2a_backend().is_mori_epv2(): + return FusedMoE # [TODO] kk, temporary solution if ( get_moe_a2a_backend().is_mori() diff --git a/python/sglang/srt/layers/moe/moe_runner/aiter.py b/python/sglang/srt/layers/moe/moe_runner/aiter.py index 18dc7c7734..5548c8bd2e 100644 --- a/python/sglang/srt/layers/moe/moe_runner/aiter.py +++ b/python/sglang/srt/layers/moe/moe_runner/aiter.py @@ -23,8 +23,7 @@ from sglang.srt.layers.moe.moe_runner.base import ( register_pre_permute, ) from sglang.srt.layers.moe.utils import MoeRunnerBackend -from sglang.srt.runtime_context import get_parallel -from sglang.srt.utils import get_bool_env_var, get_int_env_var, is_gfx95_supported +from sglang.srt.utils import get_bool_env_var, is_gfx95_supported if TYPE_CHECKING: from sglang.srt.layers.moe.token_dispatcher.base import CombineInput @@ -87,6 +86,8 @@ class AiterRunnerInput(RunnerInput): # Mori-only fused_moe kwargs. num_local_tokens: Optional[torch.Tensor] = None output_dtype: Optional[torch.dtype] = None + # Optional MORI EPv2 combine staging view for fused_moe output. + output: Optional[torch.Tensor] = None @property def runner_backend(self) -> MoeRunnerBackend: @@ -156,95 +157,11 @@ def _aiter_fused_moe_supports_no_combine() -> bool: return "no_combine" in inspect.signature(fused_moe).parameters -_RECV_BOUND_LOGGED: set[int] = set() -_RECV_BOUND_WARNED = False +@functools.cache +def _aiter_fused_moe_supports_output() -> bool: + from aiter.fused_moe import fused_moe - -def _warn_recv_bound_unavailable() -> None: - global _RECV_BOUND_WARNED - if not _RECV_BOUND_WARNED: - _RECV_BOUND_WARNED = True - logger.warning( - "SGLANG_MORI_RECV_BOUND is set but the per-rank DP token counts do " - "not cover every mori sender, so the receive fan-in is unknown; " - "leaving the receive buffer unbounded." - ) - - -def _mori_decode_recv_bound(recv_rows: int, topk: int) -> int: - """Live rows mori's receive buffer can hold in decode, or 0 for "do not bound". - - Worst case fan-in is every rank routing all of its tokens to this one, so - `sum(per-rank tokens) * topk`, where topk already includes the fused shared - expert. The per-rank counts come from the DP sync, so this is the fan-in for - the batch actually being run rather than an upper bound over all batches. - - That is only sound because enabling this gate also makes - `require_mlp_tp_gather()` true for mori, which gives every rank the same - cuda-graph bucket. The value is baked into a captured graph and has to hold - for every later replay; with per-rank buckets a rank on a narrow tier could - be handed rows by a peer on a wider one, and the only bound valid under that - is the widest tier's -- 4-16x looser than the batch being run, which costs - more in expert-GEMM tiles (M 32/64 -> 128) than the trim saves. - - Two cases stay unbounded, because a bound below the real fan-in silently - drops rows from the all-to-all -- wrong output rather than an error: - - * Prefill, whose per-rank counts are uneven and not knowable here. - * Anything that leaves the per-rank counts unpopulated, or where the EP world - is wider than the DP world so the counts do not cover every sender. - """ - if not get_bool_env_var("SGLANG_MORI_RECV_BOUND", "false"): - return 0 - - from sglang.srt.layers.dp_attention import ( - get_dp_global_num_tokens, - get_is_extend_in_batch, - ) - - if get_is_extend_in_batch(): - return 0 - - per_rank_tokens = get_dp_global_num_tokens() - ep_size = get_parallel().moe_ep_size - if not per_rank_tokens or len(per_rank_tokens) < ep_size: - # Either the DP sync did not publish counts, or they do not cover every - # mori sender. Both mean the fan-in is unknown here. - _warn_recv_bound_unavailable() - return 0 - - max_tokens = sum(per_rank_tokens) - bound = max_tokens * topk - # Never grow the tensor, and nothing to do when there is nothing to trim. - if not 0 < bound < recv_rows: - return 0 - - # One INFO line the first time it engages, so an inert bound is not mistaken - # for an active one in the results. Per-tier values go to DEBUG: capture - # visits every tier, and at INFO on every rank that is dozens of lines. - if get_parallel().tp_rank == 0 and bound not in _RECV_BOUND_LOGGED: - first = not _RECV_BOUND_LOGGED - _RECV_BOUND_LOGGED.add(bound) - if first: - logger.info( - "mori recv bound active: %d rows -> %d for this tier " - "(dp_tokens=%d ep=%d topk=%d); per-tier values at DEBUG", - recv_rows, - bound, - max_tokens, - get_parallel().moe_ep_size, - topk, - ) - else: - logger.debug( - "mori recv bound: %d rows -> %d (dp_tokens=%d ep=%d topk=%d)", - recv_rows, - bound, - max_tokens, - get_parallel().moe_ep_size, - topk, - ) - return bound + return "output" in inspect.signature(fused_moe).parameters class AiterRunnerCore(MoeRunnerCore): @@ -283,8 +200,6 @@ class AiterRunnerCore(MoeRunnerCore): from aiter.fused_moe import fused_moe - from sglang.srt.environ import envs - a1_scale = ( runner_input.a1_scale if runner_input.a1_scale is not None @@ -330,6 +245,15 @@ class AiterRunnerCore(MoeRunnerCore): extra["swiglu_limit"] = quant_info.swiglu_limit if self.config.no_combine: extra["no_combine"] = True + elif ( + runner_input.output is not None + and "output" not in extra + and extra.get("stage2_scatter") is None + and _aiter_fused_moe_supports_output() + ): + # MORI EPv2 skips staging when this exact arena view is returned. Older + # AITER versions, no_combine and stage2_scatter keep their old path. + extra["output"] = runner_input.output extra["moe_sorting_dispatch_policy"] = _MOE_SORTING_DISPATCH_POLICY # gfx950 small-M MXFP4 kernel (on by default, SGLANG_ROCM_SMALLM_MOE=0 disables): same layouts as aiter, bf16 activations. @@ -432,8 +356,8 @@ def pre_permute_standard_to_aiter( def _is_mori_dispatch_output(dispatch_output: Any) -> bool: - # MoriEP{Normal,LL}DispatchOutput carry the post-mori-permute origin_topk_* - # tensors that the standard DeepEP outputs lack. + # MORI EP dispatch outputs carry the origin_topk_* tensors for combine, + # which the standard DeepEP outputs lack. return hasattr(dispatch_output, "origin_topk_ids") @@ -486,6 +410,7 @@ def _pre_permute_deepep_to_aiter( a1_scale: Optional[torch.Tensor] = None num_local_tokens: Optional[torch.Tensor] = None output_dtype: Optional[torch.dtype] = None + output: Optional[torch.Tensor] = None quant_type = quant_info.quant_type if is_mori: @@ -494,21 +419,21 @@ def _pre_permute_deepep_to_aiter( a1_scale = dispatch_output.hidden_states_scale num_local_tokens = dispatch_output.num_recv_tokens_per_expert output_dtype = dispatch_output.out_dtype + output = getattr(dispatch_output, "expert_output", None) + mori_max = dispatch_output.recv_cap - # Truncate dispatch tensors to the configured cap; mori combine only + # Truncate dispatch tensors to the configured cap; MORI EP combine only # reads [0, totalRecvTokenNum), so the truncated result needs no - # padding back. - mori_max = get_int_env_var("SGLANG_MORI_MOE_MAX_INPUT_TOKENS", 0) - if mori_max <= 0: - mori_max = _mori_decode_recv_bound( - hidden_states.shape[0], topk_ids.shape[-1] - ) - if mori_max > 0: + # padding back. Slice the optional EPv2 output to the same row count, + # retaining the physical arena and routing stride. + if 0 < mori_max < hidden_states.shape[0]: hidden_states = hidden_states[:mori_max] if a1_scale is not None: a1_scale = a1_scale[:mori_max] topk_ids = topk_ids[:mori_max] topk_weights = topk_weights[:mori_max] + if output is not None: + output = output[:mori_max] # Upscale dispatched activations when there is no AITER kernel for the # weight/activation dtype pair. @@ -595,6 +520,7 @@ def _pre_permute_deepep_to_aiter( a1_scale=a1_scale, num_local_tokens=num_local_tokens, output_dtype=output_dtype, + output=output, ) diff --git a/python/sglang/srt/layers/moe/token_dispatcher/moriep.py b/python/sglang/srt/layers/moe/token_dispatcher/moriep.py index cf7b32747e..1d14cbe6f4 100644 --- a/python/sglang/srt/layers/moe/token_dispatcher/moriep.py +++ b/python/sglang/srt/layers/moe/token_dispatcher/moriep.py @@ -4,12 +4,14 @@ import functools import logging import os from dataclasses import dataclass -from typing import TYPE_CHECKING, List, NamedTuple, Optional, Tuple +from typing import TYPE_CHECKING, NamedTuple, Optional +from sglang.srt.environ import envs from sglang.srt.eplb.expert_distribution import ( _ExpertDistributionRecorderNoop, get_global_expert_distribution_recorder, ) +from sglang.srt.eplb.expert_location import get_global_expert_location_metadata from sglang.srt.layers.dp_attention import get_is_extend_in_batch from sglang.srt.layers.moe.token_dispatcher.base import ( BaseDispatcher, @@ -46,6 +48,9 @@ from sglang.kernels.ops.quantization.fp8_kernel import fp8_dtype FP8_BLOCK_SIZE = 128 MXFP4_BLOCK_SIZE = 32 +MORI_MIN_LOGICAL_RECV_ROWS = 32 +MORI_LOGICAL_RECV_ROW_ALIGN = 32 + _is_hip = is_hip() _use_aiter = get_bool_env_var("SGLANG_USE_AITER") and _is_hip @@ -55,6 +60,50 @@ if _use_aiter: logger = logging.getLogger(__name__) +def round_logical_recv_rows(cluster_rows: int, pow2_buckets: bool) -> int: + # Measured on MI355X with DSV4 FP4 AITER fused MoE: latency tracks the logical + # M and a new M costs no JIT, so power-of-two padding only pays off for a + # backend that pre-builds power-of-two receive caps. + if pow2_buckets: + return max(MORI_MIN_LOGICAL_RECV_ROWS, 1 << (cluster_rows - 1).bit_length()) + align = MORI_LOGICAL_RECV_ROW_ALIGN + return max(MORI_MIN_LOGICAL_RECV_ROWS, -(-cluster_rows // align) * align) + + +def mori_recv_bound(local_rows: int, sender_rows: Optional[tuple[int, ...]]) -> int: + """Return the unrounded receive-row bound, or 0 to keep the full view. + + Verified MORI kernels deduplicate rows per destination rank, so the bound + comes from sender rows. The caller checks backend/TBO eligibility and + applies rounding and the physical capacity. + """ + parallel = get_parallel() + ep_size = parallel.moe_ep_size + # ParallelContext already validates rank ranges and topology relationships. + if ep_size <= 1 or parallel.tp_size != ep_size or parallel.attn_cp_size != 1: + return 0 + + attn_dp_size = parallel.attn_dp_size + attn_dp_rank = parallel.attn_dp_rank + attn_tp_size = parallel.attn_tp_size + attn_tp_rank = parallel.attn_tp_rank + + if attn_dp_size == 1: + # Use the local tensor_split chunk: DSpark can leave DP token counts stale. + return attn_tp_size * local_rows + attn_tp_rank + if ( + sender_rows is None + or len(sender_rows) != attn_dp_size + or any(rows < 0 for rows in sender_rows) + ): + return 0 + base, extra = divmod(sender_rows[attn_dp_rank], attn_tp_size) + expected_local_rows = base + int(attn_tp_rank < extra) + if local_rows != expected_local_rows: + return 0 + return sum(sender_rows) + + def _should_record_expert_distribution() -> bool: recorder = get_global_expert_distribution_recorder() if recorder.recording: @@ -68,6 +117,15 @@ def _should_record_expert_distribution() -> bool: return False +def _record_local_expert_count(local_expert_count: torch.Tensor) -> None: + # The recorder tracks routed experts only; drop the per-rank fused shared + # expert slot that follows them. + num_routed = get_global_expert_location_metadata().num_local_physical_experts + get_global_expert_distribution_recorder().on_deepep_dispatch_low_latency( + local_expert_count[:num_routed] + ) + + @functools.lru_cache(maxsize=1) def _aiter_supports_mxfp8_dispatch() -> bool: """Whether this aiter can consume fp8 activations with group-32 e8m0 scales. @@ -91,6 +149,17 @@ def _aiter_supports_mxfp8_dispatch() -> bool: return False +@functools.lru_cache(maxsize=1) +def _mori_epv2_fp8_direct_cast_saturates() -> bool: + # Older MORI EPv2 fp8_direct_cast combine emits NaN past the fp8 max; the fix + # added this helper. Drop once the minimum MORI carries the fix. + try: + from mori.ops.dispatch_combine_v2 import intranode_kernels + except ImportError: + return False + return hasattr(intranode_kernels, "_direct_cast_fp8_max") + + class MoriEPPDispatchHooks(DeepEPPDispatchHooks): def __call__(self, dispatcher: BaseDispatcher): for hook_fun in self.hook_dict.values(): @@ -104,10 +173,13 @@ class MoriEPNormalDispatchOutput(NamedTuple): hidden_states_scale: Optional[torch.Tensor] topk_ids: torch.Tensor topk_weights: torch.Tensor - num_recv_tokens_per_expert: List[int] + num_recv_tokens_per_expert: torch.Tensor origin_topk_ids: torch.Tensor origin_topk_weights: torch.Tensor out_dtype: torch.dtype + expert_output: Optional[torch.Tensor] = None + # Logical MoE input cap selected by the dispatcher; 0 disables trimming. + recv_cap: int = 0 @property def format(self) -> DispatchOutputFormat: @@ -121,10 +193,12 @@ class MoriEPLLDispatchOutput(NamedTuple): hidden_states_scale: Optional[torch.Tensor] topk_ids: torch.Tensor topk_weights: torch.Tensor - num_recv_tokens_per_expert: List[int] + num_recv_tokens_per_expert: torch.Tensor origin_topk_ids: torch.Tensor origin_topk_weights: torch.Tensor out_dtype: torch.dtype + # Logical MoE input cap selected by the dispatcher; 0 disables trimming. + recv_cap: int = 0 @property def format(self) -> DispatchOutputFormat: @@ -184,14 +258,24 @@ class CombineDtype(Enum): @dataclass(frozen=True) -class EpDispatchConfig: +class _MoriEPv1LaunchConfig: kernel_type: mori.ops.EpDispatchCombineKernelType warp_num_per_block: int block_num: int rdma_block_num: int -def get_ep_dispatch_configs(num_max_dispatch_tokens_per_rank: int = 4096): +@dataclass(frozen=True) +class _MoriEPv2LaunchConfig: + dispatch_block_num: Optional[int] + dispatch_warp_num_per_block: Optional[int] + combine_block_num: Optional[int] + combine_warp_num_per_block: Optional[int] + + +def _get_epv1_launch_configs( + num_max_dispatch_tokens_per_rank: int = 4096, +) -> dict[EpMode, _MoriEPv1LaunchConfig]: import mori # Selects the inter-node kernel. `InterNodeV1LL` is used if `num_max_dispatch_tokens_per_rank` @@ -209,19 +293,19 @@ def get_ep_dispatch_configs(num_max_dispatch_tokens_per_rank: int = 4096): return { # TODO(billishyahao): need to tune different configs for intra node async # Also could be tuned for different AMD platform - EpMode.INTRA_NODE: EpDispatchConfig( + EpMode.INTRA_NODE: _MoriEPv1LaunchConfig( kernel_type=mori.ops.EpDispatchCombineKernelType.IntraNode, warp_num_per_block=16, block_num=80, rdma_block_num=0, ), - EpMode.INTER_NODE: EpDispatchConfig( + EpMode.INTER_NODE: _MoriEPv1LaunchConfig( kernel_type=inter_kernel_type, warp_num_per_block=8, block_num=64, rdma_block_num=32, ), - EpMode.LOW_LATENCY: EpDispatchConfig( + EpMode.LOW_LATENCY: _MoriEPv1LaunchConfig( kernel_type=mori.ops.EpDispatchCombineKernelType.AsyncLL, warp_num_per_block=8, block_num=64, @@ -230,6 +314,20 @@ def get_ep_dispatch_configs(num_max_dispatch_tokens_per_rank: int = 4096): } +def _get_epv2_launch_config(tbo_enabled: bool) -> _MoriEPv2LaunchConfig: + if not tbo_enabled: + # MORI picks its own schedule. + return _MoriEPv2LaunchConfig(None, None, None, None) + # Under TBO a sibling batch computes while this one communicates, so cap the + # grid to leave CUs for it. + return _MoriEPv2LaunchConfig( + dispatch_block_num=32, + dispatch_warp_num_per_block=4, + combine_block_num=48, + combine_warp_num_per_block=4, + ) + + # init_mori_op only needs do once in model initial stage # use lru_cache to reuse the same mori_op instance to avoid the init overhead for mori @lru_cache(maxsize=4) @@ -279,12 +377,12 @@ def init_mori_op( if async_mode: mode = EpMode.LOW_LATENCY - cfg = get_ep_dispatch_configs(num_max_dispatch_tokens_per_rank)[mode] + launch_config = _get_epv1_launch_configs(num_max_dispatch_tokens_per_rank)[mode] - kernel_type = cfg.kernel_type - warp_num_per_block = cfg.warp_num_per_block - block_num = cfg.block_num - rdma_block_num = cfg.rdma_block_num + kernel_type = launch_config.kernel_type + warp_num_per_block = launch_config.warp_num_per_block + block_num = launch_config.block_num + rdma_block_num = launch_config.rdma_block_num hidden_dim = hidden_size scale_dim = 1 @@ -412,7 +510,208 @@ class CommStreamPool: cls._streams.pop(key, None) +def _init_cco_communicator(group, instance_id: int, per_rank_vmm_gb: int): + from mori.cco import Communicator + + parallel = get_parallel() + world_size = parallel.moe_ep_size + rank = parallel.moe_ep_rank + + uid = Communicator.get_unique_id() if rank == 0 else None + uid = group.broadcast_object(uid, src=0) + comm = Communicator.init( + world_size, rank, uid, per_rank_vmm=per_rank_vmm_gb * (1 << 30) + ) + logger.info( + f"[MORI EPv2 init] cco communicator world={world_size} rank={rank} " + f"instance={instance_id} per_rank_vmm_gb={per_rank_vmm_gb}" + ) + return comm + + +def _cco_lsa_size(comm) -> int: + """Ranks in this rank's cco LSA team, i.e. the ones sharing its flat VA.""" + import mori.cco.cco as _cco + from mori.cco.communicator import DevCommHandle + + # The Communicator does not expose the team; a device comm without GDA QPs + # reads it cheaply. + requirements = _cco.DevCommRequirements() + requirements.gda_connection_type = _cco.GDA_CONNECTION_NONE + handle = DevCommHandle(comm, requirements=requirements) + try: + return handle.lsa_size + finally: + handle.close() + + +def _epv2_arena_bytes(cfg) -> int: + """Upper bound of the MORI EPv2 intranode symmetric arena for `cfg`.""" + cap = cfg.effective_max_recv + topk = cfg.num_experts_per_token + # The hip backend pads scale rows to 128 B; flydsl uses the unpadded row. + scale_row = -(-cfg.scale_dim * cfg.scale_type_size // 128) * 128 + per_recv_row = ( + cfg.token_nbytes + cfg.combine_token_nbytes + scale_row + topk * 8 + 4 + ) + total = cap * per_recv_row + cfg.world_size * 12 + if cfg.is_scatter: + scatter_rows = cfg.world_size * cfg.max_num_inp_token_per_rank + scatter_row = cfg.hidden_dim * cfg.wire_elem_size + topk * 4 + if cfg.fp8_blockwise: + scatter_row += cfg.combine_scale_dim * 4 + total += scatter_rows * scatter_row + # SymmArena aligns each of its (at most 12) regions to 256 B. + return total + 12 * 256 + + +# Arbitrary floor: the former fixed per-rank cco window. +_EPV2_MIN_PER_RANK_VMM_GB = 4 + + +def _epv2_per_rank_vmm_gb(cfg) -> int: + # The window only reserves virtual address space, so size it to the arena. + required_gb = -(-_epv2_arena_bytes(cfg) // (1 << 30)) + return max(_EPV2_MIN_PER_RANK_VMM_GB, required_gb) + + +# MORI EPv2 per-token scale layout: (scale_dim, bytes per scale) for each dispatch dtype. +_EPV2_DISPATCH_SCALES = { + DispatchDtype.bf16: (0, 0), + DispatchDtype.fp8: (FP8_BLOCK_SIZE, torch.float32.itemsize), + DispatchDtype.fp4: (MXFP4_BLOCK_SIZE, torch.float8_e8m0fnu.itemsize), + DispatchDtype.mxfp8: (MXFP4_BLOCK_SIZE, torch.float8_e8m0fnu.itemsize), +} +_EPV2_COMBINE_QUANT = { + CombineDtype.bf16: "none", + CombineDtype.fp8: "fp8_blockwise", + CombineDtype.fp8_direct_cast: "fp8_direct_cast", +} + + +def _epv2_dispatch_torch_dtype(dispatch_dtype: DispatchDtype) -> torch.dtype: + if dispatch_dtype == DispatchDtype.bf16: + return torch.bfloat16 + if dispatch_dtype == DispatchDtype.fp4: + return torch.float4_e2m1fn_x2 + return fp8_dtype + + +def _supports_dynamic_recv_cap(op) -> bool: + # MORI quantized combines run in scatter mode, which only takes the full cap. + return hasattr(op, "prepare_recv_cap") and not op.cfg.is_scatter + + +# One op (and its own cco window, sized for its arena) per distinct config; +# never evicted, since MoE layers keep using the op they were built with. +@lru_cache(maxsize=None) +def init_mori_epv2_op( + group, + router_topk: int, + num_experts: int, + num_local_experts: int, + hidden_size: int, + params_dtype: torch.dtype, + max_tokens_per_rank: int, + instance_id: int = 0, + max_total_recv_tokens: int = 0, + dispatch_dtype: DispatchDtype = DispatchDtype.bf16, + combine_dtype: CombineDtype = CombineDtype.bf16, + launch_config: _MoriEPv2LaunchConfig = _MoriEPv2LaunchConfig( + None, None, None, None + ), +): + from mori.ops.dispatch_combine_v2 import ( + EpDispatchCombineConfig, + EpDispatchCombineOp, + ) + + if params_dtype != torch.bfloat16: + raise ValueError( + "MORI EPv2 requires BF16 expert/combine output; " + f"got params_dtype={params_dtype}" + ) + if combine_dtype not in _EPV2_COMBINE_QUANT: + raise ValueError(f"unsupported MORI EPv2 combine dtype: {combine_dtype}") + # MORI EPv2 quantizes the combine wire only on a symmetric bf16 op (scatter mode). + if combine_dtype != CombineDtype.bf16 and dispatch_dtype != DispatchDtype.bf16: + raise ValueError( + f"MORI EPv2 combine dtype {combine_dtype} requires bf16 dispatch; " + f"got {dispatch_dtype}" + ) + transport_dtype = _epv2_dispatch_torch_dtype(dispatch_dtype) + is_asymmetric = transport_dtype != torch.bfloat16 + scale_block, scale_type_size = _EPV2_DISPATCH_SCALES[dispatch_dtype] + + parallel = get_parallel() + world_size = parallel.moe_ep_size + rank = parallel.moe_ep_rank + cfg_kwargs = dict( + rank=rank, + world_size=world_size, + hidden_dim=hidden_size, + max_num_inp_token_per_rank=max_tokens_per_rank, + num_experts_per_rank=num_local_experts, + num_experts_per_token=router_topk, + data_type=torch.bfloat16, + dispatch_data_type=transport_dtype if is_asymmetric else None, + combine_data_type=torch.bfloat16 if is_asymmetric else None, + scale_dim=hidden_size // scale_block if scale_block else 0, + scale_type_size=scale_type_size, + combine_mode="gather", + quant_type=_EPV2_COMBINE_QUANT[combine_dtype], + max_total_recv_tokens=max_total_recv_tokens, + ) + cfg = EpDispatchCombineConfig( + **cfg_kwargs, + dispatch_block_num=launch_config.dispatch_block_num, + combine_block_num=launch_config.combine_block_num, + warp_num_per_block=launch_config.dispatch_warp_num_per_block, + combine_warp_num_per_block=launch_config.combine_warp_num_per_block, + ) + comm = _init_cco_communicator( + group=group, instance_id=instance_id, per_rank_vmm_gb=_epv2_per_rank_vmm_gb(cfg) + ) + # Ranks share the flat VA only within the cco LSA team (a host, or a UALink + # vPOD on gfx1250); a wider EP group needs MORI's internode path, which only + # the hip backend has. MORI rejects the configs that path cannot run. + lsa_size = _cco_lsa_size(comm) + if lsa_size < world_size: + if "gpu_per_node" not in EpDispatchCombineConfig.__dataclass_fields__: + raise ValueError( + f"MORI EPv2 ep_size={world_size} spans more than its cco LSA team " + f"({lsa_size} GPUs), and this MORI has no internode EPv2; " + "set SGLANG_MORI_EP_V2=false to use EPv1" + ) + # The launch config is tuned for the intranode kernels; let MORI's + # internode tables pick the geometry. + cfg = EpDispatchCombineConfig( + **cfg_kwargs, gpu_per_node=lsa_size, kernel_backend="hip" + ) + op = EpDispatchCombineOp(cfg, comm) + comm.barrier() + if _supports_dynamic_recv_cap(op): + # Pre-compile every power-of-two receive cap _select_recv_cap can pick, so + # CUDA graph capture never compiles a new tier. + cap = MORI_MIN_LOGICAL_RECV_ROWS + while cap <= cfg.effective_max_recv: + op.prepare_recv_cap(cap) + cap *= 2 + logger.info( + f"[MORI EPv2 init] world={world_size} rank={rank} lsa_size={lsa_size} " + f"hidden={hidden_size} experts={num_experts} local_experts={num_local_experts} " + f"topk={router_topk} max_tokens={max_tokens_per_rank} " + f"recv_cap={cfg.effective_max_recv} " + f"dispatch_dtype={dispatch_dtype} combine_dtype={combine_dtype} " + f"combine_mode={cfg.combine_mode} launch_config={launch_config} " + f"schedule={cfg.schedule}" + ) + return op + + class _MoriEPDispatcherImplBase: + _is_epv2 = False + def __init__( self, group: torch.distributed.ProcessGroup, @@ -425,10 +724,6 @@ class _MoriEPDispatcherImplBase: deepep_mode: DeepEPMode, instance_id: int = 0, ): - try: - import mori # noqa: F401 - except ImportError: - raise ImportError("Mori EP is not installed. Please install.") self.group = group self.router_topk = router_topk self.permute_fusion = permute_fusion @@ -438,22 +733,241 @@ class _MoriEPDispatcherImplBase: self.params_dtype = params_dtype self.deepep_mode = deepep_mode self.instance_id = instance_id - - self.num_max_dispatch_tokens_per_rank = get_int_env_var( - "SGLANG_MORI_NUM_MAX_DISPATCH_TOKENS_PER_RANK", 4096 + self.num_max_dispatch_tokens_per_rank = ( + envs.SGLANG_MORI_NUM_MAX_DISPATCH_TOKENS_PER_RANK.get() ) - - self.enable_sdma = get_bool_env_var("MORI_ENABLE_SDMA", "false") - self.use_external_inp_buf = True - - self._mori_op = None self.dispatch_dtype = DispatchDtype.bf16 self.combine_dtype = CombineDtype.bf16 + self.quant_config = {} + self.overlap_args = None + self.meta_overlap_args = None + self._comm_stream = None + self.async_finish = False + self._tbo_enabled = is_tbo_enabled() + self._trim_recv = envs.SGLANG_MORI_RECV_BOUND.get() + self._manual_recv_cap = envs.SGLANG_MORI_MOE_MAX_INPUT_TOKENS.get() + self._recv_cap_pow2_buckets = False - self.quant_config: Optional[dict] = None + @staticmethod + def _snapshot_sender_rows() -> tuple[int, ...] | None: + """Copy CPU scheduler metadata; non-int rows would need a device sync.""" + from sglang.srt.layers.dp_attention import get_dp_global_num_tokens - self.overlap_args: Optional[CombineOverlapArgs] = None - self.meta_overlap_args: Optional[dict] = None + rows = get_dp_global_num_tokens() + if not isinstance(rows, (list, tuple)) or not all( + isinstance(r, int) for r in rows + ): + return None + return tuple(rows) + + def _get_physical_recv_rows(self) -> int: + raise NotImplementedError + + def _is_recv_layout_verified(self) -> bool: + raise NotImplementedError + + def _select_recv_cap(self) -> int: + """Select the logical MoE input cap before dispatch_b; 0 keeps the full view.""" + self._recv_cap = 0 + if self._manual_recv_cap > 0: + # The caller is responsible for covering all received tokens, as + # with the original EPv1 override. Do not round an explicit cap. + self._recv_cap = min(self._manual_recv_cap, self._get_physical_recv_rows()) + elif ( + self._trim_recv + and not self._tbo_enabled + and self._is_recv_layout_verified() + ): + bound = mori_recv_bound(self._num_tokens, self._dispatch_sender_rows) + # Empty or unproved bounds keep the full view. + if bound > 0: + cap = round_logical_recv_rows( + bound, pow2_buckets=self._recv_cap_pow2_buckets + ) + if cap < self._get_physical_recv_rows(): + self._recv_cap = cap + return self._recv_cap + + def _quantize_dispatch_input(self, hidden_states): + num_tokens = hidden_states.shape[0] + scale = None + if self.dispatch_dtype == DispatchDtype.mxfp8: + # MXFP8 quant on live tokens, before the wire. + if num_tokens > 0: + # scale_type must be set explicitly: per_1x32_mx_quant_hip still + # defaults fp8 output to continuous fp32 scales for backward + # compatibility, but the MoE kernels (and the 1-byte scale_dim + # configured above) need the e8m0 byte layout. + hidden_states, scale = self.mxfp8_quant_func( + hidden_states, + quant_dtype=fp8_dtype, + scale_type=torch.float8_e8m0fnu, + ) + else: + hidden_states = torch.empty( + hidden_states.shape, dtype=fp8_dtype, device=hidden_states.device + ) + scale = torch.empty( + (0, self.hidden_size // MXFP4_BLOCK_SIZE), + dtype=torch.float8_e8m0fnu, + device=hidden_states.device, + ) + elif self.dispatch_dtype == DispatchDtype.fp8: + # FP8 quant + if num_tokens > 0: + # NOTE: aiter is able to handle token=0 case in UT. But for some + # reason it failed at e2e case. Root cause TBD. + hidden_states, scale = self.fp8_quant_func( + hidden_states, quant_dtype=fp8_dtype + ) + else: + hidden_states = torch.empty( + hidden_states.shape, dtype=fp8_dtype, device=hidden_states.device + ) + scale = torch.empty( + (0, self.hidden_size // FP8_BLOCK_SIZE), + dtype=torch.float32, + device=hidden_states.device, + ) + + elif self.dispatch_dtype == DispatchDtype.fp4: + # FP4 quant + if num_tokens > 0: + hidden_states, scale = self.fp4_quant_func(hidden_states, shuffle=False) + else: + hidden_states = torch.empty( + (0, self.hidden_size // 2), + dtype=torch.float4_e2m1fn_x2, + device=hidden_states.device, + ) + scale = torch.empty( + (0, self.hidden_size // MXFP4_BLOCK_SIZE), + dtype=torch.float8_e8m0fnu, + device=hidden_states.device, + ) + + return hidden_states, scale + + def _prepare_topk(self, topk_output): + return topk_output.topk_weights, topk_output.topk_ids + + def _capture_event_if_async(self): + if self._comm_stream is None or not self.async_finish: + return None + event = torch.cuda.Event(blocking=False, interprocess=False) + event.record(torch.cuda.current_stream()) + return event + + def dispatch_a( + self, + hidden_states: torch.Tensor, + topk_output: TopKOutput, + ): + self._num_tokens = hidden_states.shape[0] + # Keep metadata attached to this dispatch when a/b are separated or + # two TBO children interleave. + self._dispatch_sender_rows = self._snapshot_sender_rows() + output_dtype = hidden_states.dtype + hidden_states, scale = self._quantize_dispatch_input(hidden_states) + topk_weights, topk_ids = self._prepare_topk(topk_output) + return ( + hidden_states, + topk_weights, + topk_ids, + scale, + output_dtype, + self._capture_event_if_async(), + ) + + def dispatch_b(self, *args, **kwargs): + raise NotImplementedError + + def combine_a( + self, + hidden_states: torch.Tensor, + topk_ids: torch.Tensor, + topk_weights: torch.Tensor, + ): + return hidden_states, topk_ids, topk_weights, self._capture_event_if_async() + + def combine_b(self, *args, **kwargs): + raise NotImplementedError + + def set_quant_config(self, quant_config: dict) -> None: + self.quant_config = quant_config + weight_dtype = quant_config.get("weight_dtype") + self.dispatch_dtype = DispatchDtype.bf16 + if weight_dtype == torch.float4_e2m1fn_x2: + self.dispatch_dtype = DispatchDtype.fp4 + elif weight_dtype in (torch.float8_e4m3fn, torch.float8_e4m3fnuz): + self.dispatch_dtype = DispatchDtype.fp8 + self.combine_dtype = self._default_combine_dtype(weight_dtype) + # Combine overrides read the final dispatch dtype, so apply dispatch first. + self._apply_dispatch_dtype_override() + self._apply_combine_dtype_override() + + def _default_combine_dtype(self, weight_dtype) -> CombineDtype: + raise NotImplementedError + + def _apply_dispatch_dtype_override(self): + """Apply the SGLANG_MORI_DISPATCH_DTYPE override.""" + dispatch_dtype = envs.SGLANG_MORI_DISPATCH_DTYPE.get().lower() + epv2_dtypes = ("", "auto", "bf16", "fp8", "fp4", "mxfp8") + if self._is_epv2 and dispatch_dtype not in epv2_dtypes: + raise ValueError( + "SGLANG_MORI_DISPATCH_DTYPE must be auto, bf16, fp8, fp4 or mxfp8 " + f"for EPv2; got {dispatch_dtype!r}" + ) + if envs.SGLANG_MORI_DISPATCH_DTYPE.is_set(): + if dispatch_dtype != "auto": + if dispatch_dtype == "bf16": + self.dispatch_dtype = DispatchDtype.bf16 + elif dispatch_dtype == "fp8": + self.dispatch_dtype = DispatchDtype.fp8 + elif dispatch_dtype == "fp4": + self.dispatch_dtype = DispatchDtype.fp4 + elif dispatch_dtype == "mxfp8": + if _aiter_supports_mxfp8_dispatch(): + self.dispatch_dtype = DispatchDtype.mxfp8 + else: + logger.warning_once( + "SGLANG_MORI_DISPATCH_DTYPE=mxfp8 requires an aiter " + "build whose per_1x32 quant accepts scale_type " + "(for the group-32 e8m0 byte layout the MoE kernels " + "consume). This aiter does not, so the send-side " + "quant would emit continuous fp32 scales and the " + "MoE would read them as e8m0 bytes. Falling back to " + "bf16 dispatch." + ) + + def _apply_combine_dtype_override(self): + raise NotImplementedError + + def set_overlap_args( + self, combine_overlap_args: CombineOverlapArgs, meta_overlap_args: dict + ) -> None: + self.overlap_args = combine_overlap_args + self.meta_overlap_args = meta_overlap_args + + def clear_overlap_args(self) -> None: + self.overlap_args = None + self.meta_overlap_args = None + + +class _MoriEPv1DispatcherImplBase(_MoriEPDispatcherImplBase): + def __init__(self, **kwargs): + try: + import mori # noqa: F401 + except ImportError: + raise ImportError("Mori EP is not installed. Please install.") + super().__init__(**kwargs) + self.enable_sdma = get_bool_env_var("MORI_ENABLE_SDMA", "false") + self.use_external_inp_buf = True + self._mori_op = None + self.fp8_quant_func = get_hip_quant(QuantType.per_1x128) + self.fp4_quant_func = get_hip_quant(QuantType.per_1x32) + # Same MX entry point; quant_dtype selects fp4x2 vs fp8. + self.mxfp8_quant_func = get_hip_quant(QuantType.per_1x32) @property def mori_op(self): @@ -478,31 +992,12 @@ class _MoriEPDispatcherImplBase: ) return self._mori_op - def _apply_dispatch_dtype_override(self): - """Apply env var override to fp8_dispatch/fp4_dispatch/fp8_combine flags.""" - if "SGLANG_MORI_DISPATCH_DTYPE" in os.environ: - dispatch_dtype = os.environ["SGLANG_MORI_DISPATCH_DTYPE"].lower() - if dispatch_dtype != "auto": - if dispatch_dtype == "bf16": - self.dispatch_dtype = DispatchDtype.bf16 - elif dispatch_dtype == "fp8": - self.dispatch_dtype = DispatchDtype.fp8 - elif dispatch_dtype == "fp4": - self.dispatch_dtype = DispatchDtype.fp4 - elif dispatch_dtype == "mxfp8": - if _aiter_supports_mxfp8_dispatch(): - self.dispatch_dtype = DispatchDtype.mxfp8 - else: - logger.warning_once( - "SGLANG_MORI_DISPATCH_DTYPE=mxfp8 requires an aiter " - "build whose per_1x32 quant accepts scale_type " - "(for the group-32 e8m0 byte layout the MoE kernels " - "consume). This aiter does not, so the send-side " - "quant would emit continuous fp32 scales and the " - "MoE would read them as e8m0 bytes. Falling back to " - "bf16 dispatch." - ) + def _default_combine_dtype(self, weight_dtype) -> CombineDtype: + if weight_dtype == torch.float4_e2m1fn_x2: + return CombineDtype.fp8 + return CombineDtype.bf16 + def _apply_combine_dtype_override(self): if "SGLANG_MORI_COMBINE_DTYPE" in os.environ: combine_dtype = os.environ["SGLANG_MORI_COMBINE_DTYPE"].lower() if combine_dtype != "auto": @@ -515,157 +1010,27 @@ class _MoriEPDispatcherImplBase: elif combine_dtype == "fp4": self.combine_dtype = CombineDtype.fp4 - def dispatch_a( - self, - hidden_states: torch.Tensor, - topk_output: TopKOutput, - ): - raise NotImplementedError + def _get_physical_recv_rows(self) -> int: + # MORI uses this same capacity to construct its dispatch output views. + return self.mori_op.max_num_tokens_to_recv() - def dispatch_b(self, *args, **kwargs): - raise NotImplementedError + def _is_recv_layout_verified(self) -> bool: + import mori - def combine_a( - self, - hidden_states: torch.Tensor, - topk_ids: torch.Tensor, - topk_weights: torch.Tensor, - ): - raise NotImplementedError - - def combine_b(self, *args, **kwargs): - raise NotImplementedError - - def set_quant_config(self, quant_config: dict) -> None: - self.quant_config = quant_config - # Auto-detect dispatch quantization from weight dtype - weight_dtype = quant_config.get("weight_dtype", None) - if weight_dtype in (torch.float8_e4m3fn, torch.float8_e4m3fnuz): - self.dispatch_dtype = DispatchDtype.fp8 - self.combine_dtype = CombineDtype.bf16 - elif weight_dtype == torch.float4_e2m1fn_x2: - self.dispatch_dtype = DispatchDtype.fp4 - self.combine_dtype = CombineDtype.fp8 - else: - self.dispatch_dtype = DispatchDtype.bf16 - self.combine_dtype = CombineDtype.bf16 - # Apply env var override immediately so dispatch_a sees correct flags - self._apply_dispatch_dtype_override() - - def set_overlap_args( - self, combine_overlap_args: CombineOverlapArgs, meta_overlap_args: dict - ) -> None: - self.overlap_args = combine_overlap_args - self.meta_overlap_args = meta_overlap_args - - def clear_overlap_args(self) -> None: - self.overlap_args = None - self.meta_overlap_args = None - - def _combine_kwargs(self, hidden_states: torch.Tensor) -> dict: - return {} + return self.mori_op.config.kernel_type in ( + mori.ops.EpDispatchCombineKernelType.IntraNode, + mori.ops.EpDispatchCombineKernelType.AsyncLL, + ) -class _MoriEPDispatcherImplNormal(_MoriEPDispatcherImplBase): +class _MoriEPDispatcherImplNormal(_MoriEPv1DispatcherImplBase): def __init__(self, async_finish: bool, **kwargs): super().__init__(**kwargs) - self.async_finish = async_finish - self.quant_config = {} - self.fp8_quant_func = get_hip_quant(QuantType.per_1x128) - self.fp4_quant_func = get_hip_quant(QuantType.per_1x32) - # Same MX entry point; quant_dtype selects fp4x2 vs fp8. - self.mxfp8_quant_func = get_hip_quant(QuantType.per_1x32) self.enable_dual_stream = is_tbo_enabled() - self._comm_stream = None if self.enable_dual_stream: self._comm_stream = CommStreamPool.get_stream_from_pool(self.group) - def _capture_event_if_async(self) -> Optional[torch.cuda.Event]: - assert self.enable_dual_stream, "dual stream must be enabled" - if not self.async_finish: - return None - ev = torch.cuda.Event(blocking=False, interprocess=False) - ev.record(torch.cuda.current_stream()) - return ev - - def dispatch_a( - self, - hidden_states: torch.Tensor, - topk_output: TopKOutput, - ): - topk_weights, topk_ids = topk_output.topk_weights, topk_output.topk_ids - - num_token = hidden_states.shape[0] - output_dtype = hidden_states.dtype - scale = None - - if self.dispatch_dtype == DispatchDtype.mxfp8: - # MXFP8 quant on live tokens, before the wire. - if num_token > 0: - # scale_type must be set explicitly: per_1x32_mx_quant_hip still - # defaults fp8 output to continuous fp32 scales for backward - # compatibility, but the MoE kernels (and the 1-byte scale_dim - # configured above) need the e8m0 byte layout. - hidden_states, scale = self.mxfp8_quant_func( - hidden_states, - quant_dtype=fp8_dtype, - scale_type=torch.float8_e8m0fnu, - ) - else: - hidden_states = torch.empty( - hidden_states.shape, dtype=fp8_dtype, device=hidden_states.device - ) - scale = torch.empty( - (0, self.hidden_size // MXFP4_BLOCK_SIZE), - dtype=torch.float8_e8m0fnu, - device=hidden_states.device, - ) - elif self.dispatch_dtype == DispatchDtype.fp8: - # FP8 quant - if num_token > 0: - # NOTE: aiter is able to handle token=0 case in UT. But for some - # reason it failed at e2e case. Root cause TBD. - hidden_states, scale = self.fp8_quant_func( - hidden_states, quant_dtype=fp8_dtype - ) - else: - hidden_states = torch.empty( - hidden_states.shape, dtype=fp8_dtype, device=hidden_states.device - ) - scale = torch.empty( - (0, self.hidden_size // FP8_BLOCK_SIZE), - dtype=torch.float32, - device=hidden_states.device, - ) - - elif self.dispatch_dtype == DispatchDtype.fp4: - # FP4 quant - if num_token > 0: - hidden_states, scale = self.fp4_quant_func(hidden_states, shuffle=False) - else: - hidden_states = torch.empty( - (0, self.hidden_size // 2), - dtype=torch.float4_e2m1fn_x2, - device=hidden_states.device, - ) - scale = torch.empty( - (0, self.hidden_size // MXFP4_BLOCK_SIZE), - dtype=torch.float8_e8m0fnu, - device=hidden_states.device, - ) - - previous_event = self._capture_event_if_async() if self._comm_stream else None - - return ( - hidden_states, - topk_weights, - topk_ids, - scale, - output_dtype, - previous_event, - ) - def dispatch_b( self, hidden_states, @@ -703,6 +1068,7 @@ class _MoriEPDispatcherImplNormal(_MoriEPDispatcherImplBase): origin_topk_ids=topk_ids, origin_topk_weights=topk_weights, out_dtype=output_dtype, + recv_cap=self._recv_cap, ) def _dispatch_core( @@ -785,9 +1151,7 @@ class _MoriEPDispatcherImplNormal(_MoriEPDispatcherImplBase): # mori local_expert_count is a GPU tensor; route it through the # low_latency hook only when the recorder is actually active. if record: - get_global_expert_distribution_recorder().on_deepep_dispatch_low_latency( - self.mori_op.local_expert_count - ) + _record_local_expert_count(self.mori_op.local_expert_count) return ( packed_recv_hidden, @@ -798,21 +1162,10 @@ class _MoriEPDispatcherImplNormal(_MoriEPDispatcherImplBase): done_event, ) - def combine_a( - self, - hidden_states: torch.Tensor, - topk_ids: torch.Tensor, - topk_weights: torch.Tensor, - ): - previous_event = self._capture_event_if_async() if self._comm_stream else None - return hidden_states, topk_ids, topk_weights, previous_event - def combine_b(self, hidden_states, topk_ids, topk_weights, previous_event): - hidden_states, done_event = self._combine_core( hidden_states, topk_ids, topk_weights, previous_event ) - if self._comm_stream and self.async_finish and done_event is not None: torch.cuda.current_stream().wait_event(done_event) @@ -844,10 +1197,7 @@ class _MoriEPDispatcherImplNormal(_MoriEPDispatcherImplBase): if self.enable_sdma else self.mori_op.combine ) - combine_kwargs = self._combine_kwargs(hidden_states) - combined_hidden_states = combine_fn( - hidden_states, None, topk_ids, **combine_kwargs - )[0] + combined_hidden_states = combine_fn(hidden_states, None, topk_ids)[0] if self.enable_sdma: self.mori_op.combine_recv() @@ -858,26 +1208,14 @@ class _MoriEPDispatcherImplNormal(_MoriEPDispatcherImplBase): compute_stream.wait_stream(comm_stream) else: - combine_kwargs = self._combine_kwargs(hidden_states) combined_hidden_states = self.mori_op.combine( - hidden_states, None, topk_ids, **combine_kwargs + hidden_states, None, topk_ids )[0] return combined_hidden_states, done_event - def set_quant_config(self, quant_config: dict): - super().set_quant_config(quant_config) - - -class _MoriEPDispatcherImplLowLatency(_MoriEPDispatcherImplBase): - def __init__(self, **kwargs): - super().__init__(**kwargs) - self.quant_config = {} - self.fp8_quant_func = get_hip_quant(QuantType.per_1x128) - self.fp4_quant_func = get_hip_quant(QuantType.per_1x32) - # Same MX entry point; quant_dtype selects fp4x2 vs fp8. - self.mxfp8_quant_func = get_hip_quant(QuantType.per_1x32) +class _MoriEPDispatcherImplLowLatency(_MoriEPv1DispatcherImplBase): def dispatch_a( self, hidden_states: torch.Tensor, @@ -890,66 +1228,14 @@ class _MoriEPDispatcherImplLowLatency(_MoriEPDispatcherImplBase): is mori.ops.EpDispatchCombineKernelType.AsyncLL ), "mori asyncll mismatch" - num_tokens = hidden_states.shape[0] - output_dtype = hidden_states.dtype - scale = None - - if self.dispatch_dtype == DispatchDtype.mxfp8: - # MXFP8 quant on live tokens, before the wire. - if num_tokens > 0: - # scale_type must be set explicitly: per_1x32_mx_quant_hip still - # defaults fp8 output to continuous fp32 scales for backward - # compatibility, but the MoE kernels (and the 1-byte scale_dim - # configured above) need the e8m0 byte layout. - hidden_states, scale = self.mxfp8_quant_func( - hidden_states, - quant_dtype=fp8_dtype, - scale_type=torch.float8_e8m0fnu, - ) - else: - hidden_states = torch.empty( - hidden_states.shape, dtype=fp8_dtype, device=hidden_states.device - ) - scale = torch.empty( - (0, self.hidden_size // MXFP4_BLOCK_SIZE), - dtype=torch.float8_e8m0fnu, - device=hidden_states.device, - ) - elif self.dispatch_dtype == DispatchDtype.fp8: - # FP8 quant - if num_tokens > 0: - # NOTE: aiter is able to handle token=0 case in UT. But for some - # reason it failed at e2e case. Root cause TBD. - hidden_states, scale = self.fp8_quant_func( - hidden_states, quant_dtype=fp8_dtype - ) - else: - hidden_states = torch.empty( - hidden_states.shape, dtype=fp8_dtype, device=hidden_states.device - ) - scale = torch.empty( - (0, self.hidden_size // FP8_BLOCK_SIZE), - dtype=torch.float32, - device=hidden_states.device, - ) - - elif self.dispatch_dtype == DispatchDtype.fp4: - # FP4 quant - if num_tokens > 0: - hidden_states, scale = self.fp4_quant_func(hidden_states, shuffle=False) - else: - hidden_states = torch.empty( - (0, self.hidden_size // 2), - dtype=torch.float4_e2m1fn_x2, - device=hidden_states.device, - ) - scale = torch.empty( - (0, self.hidden_size // MXFP4_BLOCK_SIZE), - dtype=torch.float8_e8m0fnu, - device=hidden_states.device, - ) - - topk_weights, topk_ids = topk_output.topk_weights, topk_output.topk_ids + ( + hidden_states, + topk_weights, + topk_ids, + scale, + output_dtype, + _, + ) = super().dispatch_a(hidden_states, topk_output) ( packed_recv_hidden, @@ -994,9 +1280,7 @@ class _MoriEPDispatcherImplLowLatency(_MoriEPDispatcherImplBase): self.mori_op.dispatch_recv(call_local_expert_count=record) if record: - get_global_expert_distribution_recorder().on_deepep_dispatch_low_latency( - self.mori_op.local_expert_count - ) + _record_local_expert_count(self.mori_op.local_expert_count) return MoriEPLLDispatchOutput( hidden_states=hidden_states, @@ -1007,6 +1291,7 @@ class _MoriEPDispatcherImplLowLatency(_MoriEPDispatcherImplBase): origin_topk_ids=topk_ids, origin_topk_weights=topk_weights, out_dtype=output_dtype, + recv_cap=self._recv_cap, ) def _dispatch_core( @@ -1050,9 +1335,7 @@ class _MoriEPDispatcherImplLowLatency(_MoriEPDispatcherImplBase): return hidden_states, topk_ids, topk_weights, overlap_args def combine_b(self, hidden_states, topk_ids, topk_weights, previous_event): - self.mori_op.combine_recv() - return hidden_states[0] def _combine_core( @@ -1068,11 +1351,253 @@ class _MoriEPDispatcherImplLowLatency(_MoriEPDispatcherImplBase): return combined_hidden_states - def set_quant_config(self, quant_config: dict): + +class _MoriEPv2DispatcherImplNormal(_MoriEPDispatcherImplBase): + _is_epv2 = True + + def __init__(self, async_finish: bool, **kwargs): + super().__init__(**kwargs) + if ( + not self.deepep_mode.enable_normal() + or self.deepep_mode.enable_low_latency() + ): + raise ValueError("MORI EPv2 currently supports normal mode only") + try: + from mori.cco import Communicator # noqa: F401 + from mori.ops.dispatch_combine_v2 import EpDispatchCombineOp # noqa: F401 + except ImportError as exc: + raise ImportError( + "MORI EPv2 requires a MORI build with cco and dispatch_combine_v2" + ) from exc + + # EPv2 forwards exactly top-k routed experts and has no fake expert slot. + os.environ.setdefault("AITER_FLYDSL_EP_NO_FAKE_EXPERT", "1") + self.async_finish = async_finish + # Same dual-stream setup as EPv1 normal; dispatch_b/combine_b order the + # comm stream with the events async_finish records. + if self._tbo_enabled and async_finish: + self._comm_stream = CommStreamPool.get_stream_from_pool(self.group) + self._launch_config = _get_epv2_launch_config(self._tbo_enabled) + self._mori_op = None + self.fp8_quant_func = None + self.fp4_quant_func = None + self.mxfp8_quant_func = None + if _use_aiter: + self.fp8_quant_func = get_hip_quant(QuantType.per_1x128) + self.fp4_quant_func = get_hip_quant(QuantType.per_1x32) + self.mxfp8_quant_func = get_hip_quant(QuantType.per_1x32) + + def set_quant_config(self, quant_config: dict) -> None: super().set_quant_config(quant_config) + # All EPv2 quant funcs are bound together, only when Aiter is available. + if self.dispatch_dtype != DispatchDtype.bf16 and self.fp4_quant_func is None: + raise RuntimeError( + f"MORI EPv2 {self.dispatch_dtype} dispatch requires Aiter quantization" + ) + # Build the op once the dtypes are final; see _initialize_op for the timing. + if self._mori_op is None: + self._initialize_op() + + def _default_combine_dtype(self, weight_dtype) -> CombineDtype: + return CombineDtype.bf16 + + def _apply_combine_dtype_override(self): + if envs.SGLANG_MORI_COMBINE_DTYPE.is_set(): + combine_dtype = envs.SGLANG_MORI_COMBINE_DTYPE.get().lower() + if combine_dtype == "fp8": + self.combine_dtype = CombineDtype.fp8 + elif combine_dtype == "fp8_direct_cast": + self.combine_dtype = CombineDtype.fp8_direct_cast + elif combine_dtype == "fp4": + logger.warning_once( + "SGLANG_MORI_COMBINE_DTYPE=fp4 is not supported by MORI EPv2; " + "falling back to bf16." + ) + elif combine_dtype not in ("", "auto", "bf16"): + raise ValueError( + "SGLANG_MORI_COMBINE_DTYPE must be auto, bf16, fp8, " + f"fp8_direct_cast or fp4; got {combine_dtype!r}" + ) + if ( + self.combine_dtype != CombineDtype.bf16 + and self.dispatch_dtype != DispatchDtype.bf16 + ): + # MORI EPv2 rejects a quantized combine on an asymmetric (quantized-dispatch) op. + logger.warning_once( + f"MORI EPv2 {self.combine_dtype} combine requires bf16 dispatch, " + f"got {self.dispatch_dtype}; falling back to bf16 combine." + ) + self.combine_dtype = CombineDtype.bf16 + if ( + self.combine_dtype == CombineDtype.fp8_direct_cast + and not _mori_epv2_fp8_direct_cast_saturates() + ): + logger.warning_once( + "This MORI build's EPv2 fp8_direct_cast combine emits NaN instead " + "of saturating values past the fp8 max, which corrupts models with " + "large expert outputs such as DeepSeek-V4; falling back to fp8 " + "(blockwise) combine. Upgrade MORI to use fp8_direct_cast." + ) + self.combine_dtype = CombineDtype.fp8 + + def _initialize_op(self): + # set_quant_config runs during model loading, before CUDA graph capture, + # so collective CCO window creation and graph-tier JIT happen safely. + self._mori_op = init_mori_epv2_op( + self.group, + self.router_topk, + self.num_experts, + self.num_local_experts, + self.hidden_size, + self.params_dtype, + self.num_max_dispatch_tokens_per_rank, + self.instance_id, + envs.SGLANG_MORI_PREALLOC_MAX_RECV_TOKENS.get(), + self.dispatch_dtype, + self.combine_dtype, + self._launch_config, + ) + # Keep logical caps on the power-of-two ladder init_mori_epv2_op pre-compiled. + self._recv_cap_pow2_buckets = _supports_dynamic_recv_cap(self._mori_op) + + @property + def mori_op(self): + if self._mori_op is None: + logger.warning_once( + "MORI EPv2 op is built lazily because set_quant_config was not " + "called: dtype env overrides are ignored and its CCO window is " + "allocated after KV cache sizing." + ) + self._initialize_op() + return self._mori_op + + def _get_physical_recv_rows(self) -> int: + return self.mori_op.cfg.effective_max_recv + + def _is_recv_layout_verified(self) -> bool: + # Older MORI builds lack these fields; keep their full receive view. + return not getattr(self.mori_op.cfg, "is_internode", True) and getattr( + self.mori_op, "backend_name", "unknown" + ) in ("flydsl", "hip") + + def dispatch_a(self, hidden_states: torch.Tensor, topk_output: TopKOutput): + if hidden_states.dtype != torch.bfloat16: + raise ValueError( + "MORI EPv2 expects BF16 activations before transport quantization; " + f"hidden_states.dtype={hidden_states.dtype}" + ) + return super().dispatch_a(hidden_states, topk_output) + + def _prepare_topk(self, topk_output): + return ( + topk_output.topk_weights.to(torch.float32), + topk_output.topk_ids.to(torch.int32), + ) + + def dispatch_b( + self, hidden_states, topk_weights, topk_ids, scale, output_dtype, ready_event + ) -> DispatchOutput: + kwargs = {"return_routing": True} + if _supports_dynamic_recv_cap(self.mori_op): + # A manual cap is an exact MoE input limit, not a MORI graph tier. + # Keep the full transport view so arbitrary caps need no new JIT + # specialization during capture; AITER applies the logical slice. + dispatch_cap = ( + self.mori_op.cfg.effective_max_recv + if self._manual_recv_cap > 0 or self._recv_cap == 0 + else self._recv_cap + ) + kwargs.update(recv_cap=dispatch_cap, clone_routing=False) + if self._comm_stream is None: + result = self.mori_op.dispatch( + hidden_states, topk_weights, scale, topk_ids, **kwargs + ) + else: + compute_stream = torch.cuda.current_stream() + keepalive = (hidden_states, topk_weights, topk_ids, scale) + with torch.cuda.stream(self._comm_stream): + assert ready_event is not None + self._comm_stream.wait_event(ready_event) + result = self.mori_op.dispatch( + hidden_states, topk_weights, scale, topk_ids, **kwargs + ) + done_event = torch.cuda.Event(blocking=False, interprocess=False) + done_event.record(self._comm_stream) + compute_stream.wait_event(done_event) + del keepalive + ( + recv_hidden, + recv_weights, + recv_scales, + recv_indices, + total_recv, + routing, + ) = result + if recv_scales is not None: + # MORI forwards scales as opaque i32 dwords; restore the quantizer layout. + if self.dispatch_dtype == DispatchDtype.fp8: + recv_scales = recv_scales.view(torch.float32)[ + :, : self.hidden_size // FP8_BLOCK_SIZE + ] + else: + recv_scales = recv_scales.view(torch.float8_e8m0fnu)[ + :, : self.hidden_size // MXFP4_BLOCK_SIZE + ] + self._routing = routing + self._recv_topk_ids = recv_indices + # Count after dispatch and before combine (combine resets total_recv), on + # the compute stream that already waited for the dispatch. + if _should_record_expert_distribution(): + _record_local_expert_count(self.mori_op.local_expert_count()) + # Let AITER write the expert output straight into combine's staging view so + # combine skips its copy. TBO children own separate arenas (instance_id); + # combine waits on the compute event recorded after the expert writes. + expert_output = None + combine_in_view = getattr(self.mori_op, "combine_in_view", None) + if combine_in_view is not None: + expert_output = combine_in_view()[: recv_hidden.shape[0]] + return MoriEPNormalDispatchOutput( + hidden_states=recv_hidden, + hidden_states_scale=recv_scales, + topk_ids=recv_indices, + topk_weights=recv_weights, + num_recv_tokens_per_expert=total_recv, + origin_topk_ids=topk_ids, + origin_topk_weights=topk_weights, + out_dtype=output_dtype, + expert_output=expert_output, + recv_cap=self._recv_cap, + ) + + def combine_b(self, hidden_states, topk_ids, topk_weights, ready_event): + if self._comm_stream is None: + out, _ = self.mori_op.combine( + hidden_states, None, self._recv_topk_ids, routing=self._routing + ) + else: + compute_stream = torch.cuda.current_stream() + keepalive = ( + hidden_states, + topk_ids, + topk_weights, + self._recv_topk_ids, + self._routing, + ) + with torch.cuda.stream(self._comm_stream): + assert ready_event is not None + self._comm_stream.wait_event(ready_event) + out, _ = self.mori_op.combine( + hidden_states, None, self._recv_topk_ids, routing=self._routing + ) + done_event = torch.cuda.Event(blocking=False, interprocess=False) + done_event.record(self._comm_stream) + compute_stream.wait_event(done_event) + del keepalive + if not torch.cuda.is_current_stream_capturing(): + del self._routing + return out -@dataclass class _Stage(Enum): INITIAL = auto() AFTER_DISPATCH_A = auto() @@ -1097,10 +1622,17 @@ class MoriEPDispatcher(BaseDispatcher): ): super().__init__() + self._is_epv2 = envs.SGLANG_MORI_EP_V2.get() + if self._is_epv2 and deepep_mode == DeepEPMode.AUTO: + deepep_mode = DeepEPMode.NORMAL self.deepep_mode = deepep_mode async_mode = self.deepep_mode.enable_low_latency() - if get_bool_env_var("SGLANG_ROCM_USE_MULTI_STREAM") and not async_mode: + if ( + not self._is_epv2 + and get_bool_env_var("SGLANG_ROCM_USE_MULTI_STREAM") + and not async_mode + ): logger.warning_once( "SGLANG_ROCM_USE_MULTI_STREAM=1 is set but Mori AsyncLL is " "not enabled (--deepep-mode=%s). The alt-stream overlap only " @@ -1123,16 +1655,21 @@ class MoriEPDispatcher(BaseDispatcher): instance_id=instance_id, ) - if self.deepep_mode.enable_low_latency(): - self._low_latency_dispatcher = _MoriEPDispatcherImplLowLatency( - **common_kwargs, + if self._is_epv2: + self._normal_dispatcher = _MoriEPv2DispatcherImplNormal( + async_finish=async_finish, **common_kwargs ) + else: + if self.deepep_mode.enable_low_latency(): + self._low_latency_dispatcher = _MoriEPDispatcherImplLowLatency( + **common_kwargs, + ) - if self.deepep_mode.enable_normal(): - self._normal_dispatcher = _MoriEPDispatcherImplNormal( - async_finish=async_finish, - **common_kwargs, - ) + if self.deepep_mode.enable_normal(): + self._normal_dispatcher = _MoriEPDispatcherImplNormal( + async_finish=async_finish, + **common_kwargs, + ) self._stage = _Stage.INITIAL self._deepep_dispatch_hooks = MoriEPPDispatchHooks() @@ -1140,7 +1677,11 @@ class MoriEPDispatcher(BaseDispatcher): # Mori dispatch produces global topk_ids in [0, num_experts); mask out # experts that are not local to this rank. self.expert_mask_gpu = None - if _use_aiter and num_experts is not None and num_local_experts is not None: + if ( + (_use_aiter or self._is_epv2) + and num_experts is not None + and num_local_experts is not None + ): ep_rank = get_parallel().moe_ep_rank expert_mask = torch.zeros( num_experts, @@ -1156,12 +1697,10 @@ class MoriEPDispatcher(BaseDispatcher): hidden_states: torch.Tensor, topk_output: TopKOutput, ) -> DispatchOutput: - self._num_tokens = hidden_states.shape[0] self.dispatch_a(hidden_states, topk_output) if self._deepep_dispatch_hooks is not None: self._deepep_dispatch_hooks(self) - ret = self.dispatch_b() - return ret + return self.dispatch_b() def dispatch_a( self, @@ -1169,22 +1708,23 @@ class MoriEPDispatcher(BaseDispatcher): topk_output: TopKOutput, ): self._update_stage(_Stage.INITIAL, _Stage.AFTER_DISPATCH_A) - inner_state = self._get_impl().dispatch_a( - hidden_states=hidden_states, - topk_output=topk_output, + self._num_tokens = hidden_states.shape[0] + self._dispatch_intermediate_state = self._get_impl().dispatch_a( + hidden_states=hidden_states, topk_output=topk_output ) - self._dispatch_intermediate_state = inner_state def dispatch_b(self): self._update_stage(_Stage.AFTER_DISPATCH_A, _Stage.AFTER_DISPATCH_B) inner_state = self._dispatch_intermediate_state del self._dispatch_intermediate_state - return self._get_impl().dispatch_b(*inner_state) + impl = self._get_impl() + impl._select_recv_cap() + return impl.dispatch_b(*inner_state) def combine( self, combine_input: CombineInput, - ) -> Tuple: + ) -> torch.Tensor: self.combine_a(combine_input) hidden_states = self.combine_b() return hidden_states[: self._num_tokens] @@ -1209,6 +1749,8 @@ class MoriEPDispatcher(BaseDispatcher): return self._get_impl().combine_b(*inner_state) def _get_impl(self) -> _MoriEPDispatcherImplBase: + if self._is_epv2: + return self._normal_dispatcher is_extend_in_batch = get_is_extend_in_batch() resolved_deepep_mode = self.deepep_mode.resolve(is_extend_in_batch) if resolved_deepep_mode == DeepEPMode.NORMAL: diff --git a/python/sglang/srt/layers/moe/utils.py b/python/sglang/srt/layers/moe/utils.py index 8eb3efe550..989994a946 100644 --- a/python/sglang/srt/layers/moe/utils.py +++ b/python/sglang/srt/layers/moe/utils.py @@ -80,6 +80,9 @@ class MoeA2ABackend(Enum): def is_mori(self): return self == MoeA2ABackend.MORI + def is_mori_epv2(self): + return self.is_mori() and envs.SGLANG_MORI_EP_V2.get() + def is_megamoe(self): return self == MoeA2ABackend.MEGAMOE diff --git a/python/sglang/srt/layers/quantization/fp8_utils.py b/python/sglang/srt/layers/quantization/fp8_utils.py index 92adc1b281..6fa480b3b8 100755 --- a/python/sglang/srt/layers/quantization/fp8_utils.py +++ b/python/sglang/srt/layers/quantization/fp8_utils.py @@ -1449,6 +1449,11 @@ def aiter_w8a8_block_fp8_linear( # assert input_scale is None input_2d = input.view(-1, input.shape[-1]) output_shape = [*input.shape[:-1], weight.shape[0]] + output_dtype = torch.bfloat16 if input_scale is not None else input.dtype + # DP-attention idle ranks run zero-token forwards (eager has no padded graph + # bucket); CK blockscale GEMM fails M == 0 with invalid configuration argument. + if input_2d.shape[0] == 0: + return input.new_empty(output_shape, dtype=output_dtype) # dense linears converted from MXFP8 carry a rowwise-fp8 copy; its ptpc GEMM # beats the block-fp8 GEMMs at decode-sized M @@ -1524,15 +1529,13 @@ def aiter_w8a8_block_fp8_linear( weight, x_scale, weight_scale, - dtype=torch.bfloat16 if input_scale is not None else input.dtype, + dtype=output_dtype, ) if bias is not None: output += bias - return output.to( - dtype=torch.bfloat16 if input_scale is not None else input_2d.dtype - ).view(*output_shape) + return output.to(dtype=output_dtype).view(*output_shape) def triton_w8a8_block_fp8_linear( diff --git a/python/sglang/srt/layers/quantization/mxfp4.py b/python/sglang/srt/layers/quantization/mxfp4.py index 9c5e2fee1b..57da3162ef 100644 --- a/python/sglang/srt/layers/quantization/mxfp4.py +++ b/python/sglang/srt/layers/quantization/mxfp4.py @@ -633,6 +633,11 @@ class Mxfp4MoEMethod(FusedMoEMethodBase): set_weight_attrs(w2_weight_bias, extra_weight_attrs) def process_weights_after_loading(self, layer): + if get_moe_a2a_backend().is_mori() and hasattr(layer, "dispatcher"): + # Resolve MORI dtype overrides and build the EPv2 op before KV cache + # sizing. No weight_dtype keeps bf16 dispatch, the validated default here. + layer.dispatcher.set_quant_config({}) + if self.use_marlin and not self.use_mega_moe: from sglang.srt.layers.quantization.marlin_utils import ( check_moe_marlin_supports_layer, @@ -1879,6 +1884,11 @@ class Mxfp4DynamicQuantMoEMethod(FusedMoEMethodBase): return w, mx_scales def process_weights_after_loading(self, layer: torch.nn.Module) -> None: + if get_moe_a2a_backend().is_mori() and hasattr(layer, "dispatcher"): + # Resolve MORI dtype overrides and build the EPv2 op before KV cache + # sizing. No weight_dtype keeps bf16 dispatch, the validated default here. + layer.dispatcher.set_quant_config({}) + w13, w13_mx_scales = self.mxfp4_quantize(layer.w13_weight.data) w2, w2_mx_scales = self.mxfp4_quantize(layer.w2_weight.data) diff --git a/python/sglang/srt/layers/quantization/unquant.py b/python/sglang/srt/layers/quantization/unquant.py index ac239656cd..a133eb5704 100644 --- a/python/sglang/srt/layers/quantization/unquant.py +++ b/python/sglang/srt/layers/quantization/unquant.py @@ -796,6 +796,10 @@ class UnquantizedFusedMoEMethod(FusedMoEMethodBase, BaseFusedOp): ): layer.dispatcher.set_quant_config({"dispatcher_output_dtype": "bf16"}) + if get_moe_a2a_backend().is_mori() and hasattr(layer, "dispatcher"): + # Resolve MORI dtype overrides and build the EPv2 op before KV cache sizing. + layer.dispatcher.set_quant_config({"weight_dtype": layer.w13_weight.dtype}) + if self.use_flashinfer_cutlass: from sglang.srt.layers.moe.moe_runner.flashinfer_cutlass import ( materialize_swiglu_params_for_cutlass, diff --git a/python/sglang/srt/models/deepseek_v4.py b/python/sglang/srt/models/deepseek_v4.py index 85788b7ebf..dd6bf85ae0 100644 --- a/python/sglang/srt/models/deepseek_v4.py +++ b/python/sglang/srt/models/deepseek_v4.py @@ -79,6 +79,7 @@ from sglang.srt.layers.dp_attention import ( DpPaddingMode, _tbo_event, attn_tp_all_gather, + attn_tp_all_gather_into_tensor, attn_tp_all_reduce, dp_gather_partial, dp_gather_replicate, @@ -2676,6 +2677,28 @@ def _every_row_routed(forward_batch: ForwardBatch, num_rows: int): ) = saved +def _attn_tp_all_gather_chunks( + local: torch.Tensor, chunk_rows: List[int] +) -> torch.Tensor: + # Tensor all_gather stays on aiter AG / pynccl under graph capture; the list + # all_gather falls back to ProcessGroupNCCL, which invalidates HIP capture. + max_rows = max(chunk_rows) + padded = local + if local.shape[0] != max_rows: + padded = local.new_zeros((max_rows, *local.shape[1:])) + padded[: local.shape[0]].copy_(local) + gathered = local.new_empty((len(chunk_rows) * max_rows, *local.shape[1:])) + attn_tp_all_gather_into_tensor(gathered, padded.contiguous()) + if all(rows == max_rows for rows in chunk_rows): + return gathered + return torch.cat( + [ + gathered[i * max_rows : i * max_rows + rows] + for i, rows in enumerate(chunk_rows) + ] + ) + + class DeepseekV4DecoderLayer(nn.Module): def __init__( self, @@ -3556,9 +3579,14 @@ class DeepseekV4DecoderLayer(nn.Module): hidden_states = hidden_states + _shared_local[:n] if _use_tp_attn_a2a_scatter: assert _a2a_scatter_chunks is not None - gathered = [torch.empty_like(t) for t in _a2a_scatter_chunks] - attn_tp_all_gather(gathered, hidden_states.contiguous()) - hidden_states = torch.cat(gathered) + if _is_hip: + hidden_states = _attn_tp_all_gather_chunks( + hidden_states, chunk_rows=[t.shape[0] for t in _a2a_scatter_chunks] + ) + else: + gathered = [torch.empty_like(t) for t in _a2a_scatter_chunks] + attn_tp_all_gather(gathered, hidden_states.contiguous()) + hidden_states = torch.cat(gathered) return hidden_states # ------------------------------------------------------------------ diff --git a/python/sglang/srt/models/deepseek_v4_dspark.py b/python/sglang/srt/models/deepseek_v4_dspark.py index a77485cfae..318c52d9ae 100644 --- a/python/sglang/srt/models/deepseek_v4_dspark.py +++ b/python/sglang/srt/models/deepseek_v4_dspark.py @@ -17,6 +17,7 @@ from sglang.kernels.ops.speculative.dspark.dspark_draft_model import CommitKvPro from sglang.srt.configs.deepseek_v4 import DeepSeekV4Config from sglang.srt.distributed.device_communicators.vocab_gather import make_vocab_gather from sglang.srt.environ import envs +from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder from sglang.srt.layers.dp_attention import is_dp_attention_enabled from sglang.srt.layers.layernorm import RMSNorm from sglang.srt.layers.logits_processor import LogitsProcessorOutput @@ -951,8 +952,10 @@ class DeepseekV4ForCausalLMDSpark(nn.Module): input_embeds = self.forward_embed(input_ids) x = input_embeds pre = None - for stage in self.stages: - x, pre = stage(positions, x, forward_batch, pre) + # Draft MoE layers have no recorder layer index, so keep them out of the stats. + with get_global_expert_distribution_recorder().disable_this_region(): + for stage in self.stages: + x, pre = stage(positions, x, forward_batch, pre) if self.hc_pre_from_prev_sublayer: from sglang.kernels.ops.layernorm.mhc import hc_combine diff --git a/python/sglang/srt/utils/common.py b/python/sglang/srt/utils/common.py index 31e7e9f8a1..b935de39aa 100644 --- a/python/sglang/srt/utils/common.py +++ b/python/sglang/srt/utils/common.py @@ -4145,8 +4145,8 @@ def require_mlp_tp_gather(*, moe_a2a_backend=None): return True elif moe_a2a_backend.is_none(): return True - elif moe_a2a_backend.is_flashinfer(): - # FlashInfer MoE A2A needs a rank-invariant, DP-synchronized per-rank + elif moe_a2a_backend.is_flashinfer() or moe_a2a_backend.is_mori_epv2(): + # FlashInfer/MORI EPv2 A2A need a rank-invariant, DP-synchronized per-rank # token count: MoeAlltoAll uses fixed-geometry buffers and the decode # cuda-graph bucket must be identical across EP ranks, otherwise ranks # replay different-sized graphs -> geometry mismatch -> illegal memory @@ -4155,9 +4155,7 @@ def require_mlp_tp_gather(*, moe_a2a_backend=None): # reuse this flag's DP-sync bookkeeping (uniform global_num_tokens + # max-based graph bucket). See #30432 re: the misleading flag name. return True - elif moe_a2a_backend.is_mori() and get_bool_env_var( - "SGLANG_MORI_RECV_BOUND", "false" - ): + elif moe_a2a_backend.is_mori() and envs.SGLANG_MORI_RECV_BOUND.get(): # Same bookkeeping, for the same reason. Bounding mori's receive # buffer means baking a fan-in size into a captured graph, and the # fan-in depends on what the *peers* send. Without a DP-synchronized diff --git a/test/manual/dsv4/test_mori_epv2_dispatcher.py b/test/manual/dsv4/test_mori_epv2_dispatcher.py new file mode 100644 index 0000000000..2716bd6c67 --- /dev/null +++ b/test/manual/dsv4/test_mori_epv2_dispatcher.py @@ -0,0 +1,516 @@ +"""Eight-GPU identity round trip through the SGLang MORI EPv2 adapter. + +TBO=1 runs the FP4-asymmetric two-child TBO path instead; there TEST_CUDA_GRAPH=1 +captures and replays REPLAY_ITERS times (default 3), with each child writing its +expert output directly into its own combine staging view. +The identity expert checks buffer correctness, not AITER performance. +""" + +import json +import os +from pathlib import Path +from types import SimpleNamespace + +import torch +import torch.distributed as dist + +import sglang.srt.layers.dp_attention as dp_attention +import sglang.srt.layers.moe.token_dispatcher.moriep as adapter +from sglang.srt.batch_overlap.two_batch_overlap import MaybeTboDeepEPDispatcher +from sglang.srt.environ import envs +from sglang.srt.layers.moe.token_dispatcher.moriep import round_logical_recv_rows +from sglang.srt.layers.moe.topk import StandardTopKOutput +from sglang.srt.layers.moe.utils import DeepEPMode, MoeA2ABackend +from sglang.srt.runtime_context import get_flags + + +def _fp4_lookup(): + # e2m1 code points, low nibble first. + values = [0.0, 0.5, 1.0, 1.5, 2.0, 3.0, 4.0, 6.0] + return torch.tensor( + values + [-v for v in values], dtype=torch.float32, device="cuda" + ) + + +def _expert_output(dispatched, dispatch: str, fp4_lookup=None): + if dispatch == "bf16": + output = dispatched.hidden_states + return output[: dispatched.recv_cap] if dispatched.recv_cap > 0 else output + + if dispatch == "fp4": + packed = dispatched.hidden_states.view(torch.uint8) + nibbles = torch.stack((packed & 0xF, packed >> 4), dim=-1).flatten(-2) + values = fp4_lookup[nibbles.long()] + else: + values = dispatched.hidden_states.float() + # fp8 uses fp32 scales per 128 columns; fp4 and mxfp8 use e8m0 per 32. + block = 128 if dispatch == "fp8" else 32 + scales = dispatched.hidden_states_scale.float().repeat_interleave(block, dim=1) + output = (values * scales[:, : values.shape[1]]).to(torch.bfloat16) + valid_rows = ( + torch.arange(output.shape[0], device=output.device) + < dispatched.num_recv_tokens_per_expert.reshape(-1)[0] + ) + output = torch.where(valid_rows[:, None], output, torch.zeros_like(output)) + return output[: dispatched.recv_cap] if dispatched.recv_cap > 0 else output + + +class _Group: + def __init__(self, process_group): + self.cpu_group = process_group + self.world_size = dist.get_world_size(process_group) + self.rank_in_group = dist.get_rank(process_group) + + def broadcast_object(self, obj, src=0): + values = [obj if self.rank_in_group == src else None] + dist.broadcast_object_list(values, src=src, group=self.cpu_group) + return values[0] + + +def _expected_unique_destinations(topk_ids: torch.Tensor, experts_per_rank: int): + return torch.tensor( + [ + len(set(row.tolist())) + for row in (topk_ids.cpu().to(torch.int64) // experts_per_rank) + ], + dtype=torch.float32, + ).view(-1, 1) + + +def main(): + dist.init_process_group("gloo") + rank = dist.get_rank() + world_size = dist.get_world_size() + local_rank = int(os.environ["LOCAL_RANK"]) + assert world_size == 8 + torch.cuda.set_device(local_rank) + if os.environ.get("TBO", "0") == "1": + _run_tbo(rank, world_size) + + hidden_size = int(os.environ.get("HIDDEN", "7168")) + topk = int(os.environ.get("TOPK", "6")) + experts_per_rank = int(os.environ.get("EPR", "48")) + num_experts = world_size * experts_per_rank + tokens = int(os.environ.get("TOKENS", "64")) + # DISPATCH / COMBINE map to SGLANG_MORI_DISPATCH_DTYPE / SGLANG_MORI_COMBINE_DTYPE. + dispatch = os.environ.get( + "DISPATCH", "fp4" if os.environ.get("FP4", "0") == "1" else "bf16" + ) + combine = os.environ.get("COMBINE", "bf16") + os.environ["SGLANG_MORI_DISPATCH_DTYPE"] = dispatch + os.environ["SGLANG_MORI_COMBINE_DTYPE"] = combine + fp4_enabled = dispatch == "fp4" + fp4_lookup = _fp4_lookup() if fp4_enabled else None + if os.environ.get("EMPTY_LAST_RANK", "0") == "1" and rank == world_size - 1: + tokens = 0 + + # ATTN_TP > 1 models attention TP: each DP group's TOKENS * (group + 1) rows + # are tensor_split over its attention-TP ranks, as the model does before MoE. + attn_tp_size = int(os.environ.get("ATTN_TP", "1")) + attn_dp_size = world_size // attn_tp_size + attn_dp_rank, attn_tp_rank = divmod(rank, attn_tp_size) + if attn_tp_size == 1: + sender_rows = [None] * world_size + dist.all_gather_object(sender_rows, tokens) + else: + sender_rows = [tokens * (group + 1) for group in range(attn_dp_size)] + tokens = ( + torch.empty(sender_rows[attn_dp_rank], device="cpu") + .tensor_split(attn_tp_size)[attn_tp_rank] + .numel() + ) + # STALE_ROWS models DSpark draft forwards that keep the target step's metadata. + stale_rows = os.environ.get("STALE_ROWS") + metadata_rows = [int(stale_rows)] if stale_rows else sender_rows + dp_attention.set_dp_buffer_len(sum(sender_rows), tokens, False, metadata_rows) + + adapter.is_tbo_enabled = lambda: False + adapter.get_parallel = lambda: SimpleNamespace( + moe_ep_size=world_size, + moe_ep_rank=rank, + tp_size=world_size, + attn_dp_size=attn_dp_size, + attn_dp_rank=attn_dp_rank, + attn_tp_size=attn_tp_size, + attn_tp_rank=attn_tp_rank, + attn_cp_size=1, + moe_tp_size=1, + moe_dp_size=1, + launch_world_rank=rank, + ) + group = _Group(dist.group.WORLD) + envs.SGLANG_MORI_EP_V2.set(True) + dispatcher = adapter.MoriEPDispatcher( + group=group, + router_topk=topk, + num_experts=num_experts, + num_local_experts=experts_per_rank, + hidden_size=hidden_size, + params_dtype=torch.bfloat16, + deepep_mode=DeepEPMode.NORMAL, + ) + dispatcher.set_quant_config( + {"weight_dtype": (torch.float4_e2m1fn_x2 if fp4_enabled else torch.bfloat16)} + ) + + generator = torch.Generator(device="cpu").manual_seed(20260803 + rank) + hidden = torch.randn( + tokens, hidden_size, generator=generator, dtype=torch.bfloat16 + ).cuda() + if os.environ.get("SKEWED", "0") == "1": + first_expert = int(os.environ.get("SKEW_RANK", "0")) * experts_per_rank + topk_ids = ( + torch.arange(first_expert, first_expert + topk, dtype=torch.int32) + .repeat(tokens, 1) + .cuda() + ) + else: + topk_ids = torch.randint( + 0, + num_experts, + (tokens, topk), + generator=generator, + dtype=torch.int32, + ).cuda() + topk_weights = torch.rand( + tokens, topk, generator=generator, dtype=torch.float32 + ).cuda() + topk_output = StandardTopKOutput(topk_weights, topk_ids, None) + + dispatched = dispatcher.dispatch(hidden, topk_output) + impl = dispatcher._get_impl() + assert impl.dispatch_dtype.name == dispatch, (impl.dispatch_dtype, dispatch) + # MORI builds without the saturation fix fall back to blockwise fp8. + expected_combine = combine + saturates = adapter._mori_epv2_fp8_direct_cast_saturates() + if combine == "fp8_direct_cast" and not saturates: + expected_combine = "fp8" + assert impl.combine_dtype.name == expected_combine, (impl.combine_dtype, combine) + expected_cap = 0 + if impl._manual_recv_cap > 0: + expected_cap = min(impl._manual_recv_cap, impl.mori_op.cfg.effective_max_recv) + elif impl._trim_recv and impl._is_recv_layout_verified(): + cluster_rows = ( + sum(sender_rows) + if attn_dp_size > 1 + else attn_tp_size * tokens + attn_tp_rank + ) + if cluster_rows > 0: + rounded = round_logical_recv_rows( + cluster_rows, pow2_buckets=impl._recv_cap_pow2_buckets + ) + # A bound that covers the whole receive buffer keeps the full view. + if rounded < impl.mori_op.cfg.effective_max_recv: + expected_cap = rounded + assert dispatched.recv_cap == expected_cap + recv_rows = int(dispatched.num_recv_tokens_per_expert.reshape(-1)[0].item()) + assert recv_rows <= sum(sender_rows), (recv_rows, sender_rows) + assert expected_cap == 0 or recv_rows <= expected_cap, (recv_rows, expected_cap) + combined = dispatcher.combine( + ( + _expert_output(dispatched, dispatch, fp4_lookup), + dispatched.topk_ids, + dispatched.topk_weights, + ) + ) + torch.cuda.synchronize() + dispatcher._get_impl().mori_op.comm.barrier() + + expected = ( + _expected_unique_destinations(topk_ids, experts_per_rank) * hidden.float().cpu() + ).to(torch.bfloat16) + # Arbitrary bounds: fp4 keeps ~1 mantissa bit, e4m3 keeps 3 (per quantization hop). + if fp4_enabled: + tolerance = 6e-1 + elif dispatch != "bf16" or combine != "bf16": + tolerance = 1.5e-1 + else: + tolerance = 2e-2 + error = (combined.float().cpu() - expected.float()).abs() + ok = torch.allclose( + combined.float().cpu(), expected.float(), atol=tolerance, rtol=tolerance + ) + failures = torch.tensor([not ok], dtype=torch.int32) + dist.all_reduce(failures) + if rank == 0: + print( + "# MORI-EPV2-SGLANG-IDENTITY: " + f"{'PASS' if failures.item() == 0 else 'FAIL'} " + f"tokens={tokens} hidden={hidden_size} topk={topk} " + f"skewed={os.environ.get('SKEWED', '0')} " + f"empty_last_rank={os.environ.get('EMPTY_LAST_RANK', '0')} " + f"attn_tp={attn_tp_size} sender_rows={sender_rows} stale_rows={stale_rows} " + f"recv_cap={dispatched.recv_cap}", + f"dispatch={dispatch} combine={combine} " + f"combine_mode={impl.mori_op.cfg.combine_mode}", + f"mean_abs_error={error.mean().item():.6f}", + f"max_abs_error={error.max().item() if error.numel() else 0:.6f}", + flush=True, + ) + + graph_replays = int(os.environ.get("GRAPH_REPLAYS", "0")) + if graph_replays: + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(graph): + graph_dispatched = dispatcher.dispatch(hidden, topk_output) + graph_combined = dispatcher.combine( + ( + _expert_output(graph_dispatched, dispatch, fp4_lookup), + graph_dispatched.topk_ids, + graph_dispatched.topk_weights, + ) + ) + torch.cuda.synchronize() + dispatcher._get_impl().mori_op.comm.barrier() + for _ in range(graph_replays): + graph.replay() + torch.cuda.synchronize() + dispatcher._get_impl().mori_op.comm.barrier() + graph_ok = torch.allclose( + graph_combined.float().cpu(), + expected.float(), + atol=tolerance, + rtol=tolerance, + ) + graph_failures = torch.tensor([not graph_ok], dtype=torch.int32) + dist.all_reduce(graph_failures) + + eager_dispatched = dispatcher.dispatch(hidden, topk_output) + eager_after_graph = dispatcher.combine( + ( + _expert_output(eager_dispatched, dispatch, fp4_lookup), + eager_dispatched.topk_ids, + eager_dispatched.topk_weights, + ) + ) + torch.cuda.synchronize() + dispatcher._get_impl().mori_op.comm.barrier() + eager_after_graph_ok = torch.allclose( + eager_after_graph.float().cpu(), + expected.float(), + atol=tolerance, + rtol=tolerance, + ) + eager_after_graph_failures = torch.tensor( + [not eager_after_graph_ok], dtype=torch.int32 + ) + dist.all_reduce(eager_after_graph_failures) + failures += graph_failures + eager_after_graph_failures + if rank == 0: + print( + "# MORI-EPV2-SGLANG-GRAPH: " + f"{'PASS' if graph_failures.item() == 0 else 'FAIL'} " + f"replays={graph_replays}; " + "post_graph_eager=" + f"{'PASS' if eager_after_graph_failures.item() == 0 else 'FAIL'}", + flush=True, + ) + + dispatcher._get_impl().mori_op.close() + dispatcher._get_impl().mori_op.comm.destroy() + dist.destroy_process_group() + raise SystemExit(int(failures.item() != 0)) + + +def _run_tbo(rank, world_size): + hidden_size, topk, experts_per_rank = 7168, 6, 48 + num_experts = world_size * experts_per_rank + adapter.get_parallel = lambda: SimpleNamespace( + moe_ep_size=world_size, + moe_ep_rank=rank, + tp_size=world_size, + attn_dp_size=world_size, + attn_dp_rank=rank, + attn_tp_size=1, + attn_tp_rank=0, + attn_cp_size=1, + moe_tp_size=1, + moe_dp_size=1, + launch_world_rank=rank, + world_rank=rank, + ) + group = _Group(dist.group.WORLD) + kwargs = dict( + group=group, + router_topk=topk, + num_experts=num_experts, + num_local_experts=experts_per_rank, + hidden_size=hidden_size, + params_dtype=torch.bfloat16, + async_finish=True, + ) + with ( + envs.SGLANG_MORI_EP_V2.override(True), + get_flags().moe.override(tbo_enabled=True, a2a_backend=MoeA2ABackend.MORI), + ): + dispatcher = MaybeTboDeepEPDispatcher(**kwargs) + assert len(dispatcher._inners) == 2 + for child in dispatcher._inners: + child.set_quant_config({"weight_dtype": torch.float4_e2m1fn_x2}) + children = [child._get_impl() for child in dispatcher._inners] + assert children[0]._comm_stream is children[1]._comm_stream + assert children[0]._launch_config == adapter._MoriEPv2LaunchConfig(32, 4, 48, 4) + assert children[0].mori_op is not children[1].mori_op + probe_trim = os.environ.get("PROBE_TBO_TRIM", "0") == "1" + if probe_trim: + # Validation-only probe: keep the real shared TBO stream/dispatchers, + # but allow each child to consume metadata snapshotted immediately + # before its dispatch_a. Production remains conservatively gated off. + for child in children: + child._tbo_enabled = False + + fp4_lookup = _fp4_lookup() + failures = torch.zeros(1, dtype=torch.int32) + inputs = [] + child_token_counts = ( + (12, 0, 7, 2, 9, 0, 4, 1), + (0, 11, 3, 0, 5, 8, 1, 6), + ) + for child_id, token_counts in enumerate(child_token_counts): + tokens = token_counts[rank] + generator = torch.Generator(device="cpu").manual_seed( + 20260805 + child_id * 100 + rank + ) + hidden = torch.randn( + tokens, hidden_size, dtype=torch.bfloat16, generator=generator + ).cuda() + ids = torch.randint( + 0, + num_experts, + (tokens, topk), + dtype=torch.int32, + generator=generator, + ).cuda() + weights = torch.rand( + tokens, topk, dtype=torch.float32, generator=generator + ).cuda() + inputs.append((hidden, ids, StandardTopKOutput(weights, ids, None))) + + def run_step(): + for child_id, (hidden, _ids, topk_output) in enumerate(inputs): + token_counts = child_token_counts[child_id] + dp_attention.set_dp_buffer_len( + sum(token_counts), token_counts[rank], False, list(token_counts) + ) + dispatcher.dispatch_a( + tbo_subbatch_index=child_id, + hidden_states=hidden, + topk_output=topk_output, + ) + outputs = [ + dispatcher.dispatch_b(tbo_subbatch_index=child_id) for child_id in range(2) + ] + direct_views = [] + for child_id, (child, output) in enumerate(zip(children, outputs)): + if child._manual_recv_cap > 0: + assert output.recv_cap == min( + child._manual_recv_cap, child.mori_op.cfg.effective_max_recv + ) + elif child._trim_recv and probe_trim and child._is_recv_layout_verified(): + expected_cap = round_logical_recv_rows( + sum(child_token_counts[child_id]), + pow2_buckets=child._recv_cap_pow2_buckets, + ) + assert output.recv_cap == expected_cap + else: + assert output.recv_cap == 0 + expected_direct = callable(getattr(child.mori_op, "combine_in_view", None)) + assert (output.expert_output is not None) == expected_direct + if output.expert_output is not None: + assert ( + output.expert_output.data_ptr() + == child.mori_op.combine_in_view().data_ptr() + ) + direct_views.append(output.expert_output.data_ptr()) + assert len(direct_views) == len(set(direct_views)), ( + "TBO children share an output buffer" + ) + for child_id, output in enumerate(outputs): + expert_out = _expert_output(output, "fp4", fp4_lookup) + if output.expert_output is not None: + direct_out = output.expert_output[: expert_out.shape[0]] + direct_out.copy_(expert_out) + expert_out = direct_out + dispatcher.combine_a( + tbo_subbatch_index=child_id, + combine_input=(expert_out, output.topk_ids, output.topk_weights), + ) + actual = [ + dispatcher.combine_b(tbo_subbatch_index=child_id)[ + : inputs[child_id][0].shape[0] + ] + for child_id in range(2) + ] + return actual, outputs + + def check(actual, atol=0.6, rtol=0.6): + torch.cuda.synchronize() + for child_id, (hidden, ids, _topk_output) in enumerate(inputs): + if not torch.allclose( + actual[child_id].float().cpu(), + ( + _expected_unique_destinations(ids, experts_per_rank) + * hidden.float().cpu() + ) + .to(torch.bfloat16) + .float(), + atol=atol, + rtol=rtol, + ): + failures.add_(1) + + replay_iters = int(os.environ.get("REPLAY_ITERS", "3")) + assert replay_iters > 0 + use_graph = os.environ.get("TEST_CUDA_GRAPH", "0") == "1" + graph = None + # Exercise random FP4 inputs and warm up both instances before graph capture. + for _ in range(2 if use_graph else 1): + actual, outputs = run_step() + check(actual) + if use_graph: + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(graph): + actual, outputs = run_step() + graph.replay() + check(actual) + for iteration in range(replay_iters): + # Change values and routing while retaining the captured input addresses. + for child_id, (hidden, ids, _topk_output) in enumerate(inputs): + # Powers of two are exact in FP4; opposite signs expose child aliasing. + value = 2.0 ** (iteration % 4 - 3) + hidden.fill_(value if child_id == 0 else -2 * value) + ids.copy_((ids + 13) % num_experts) + if graph is None: + actual, outputs = run_step() + else: + graph.replay() + check(actual, atol=2e-2, rtol=2e-2) + graph = None + torch.cuda.synchronize() + dist.all_reduce(failures) + if rank == 0: + summary = { + "status": "PASS" if failures.item() == 0 else "FAIL", + "probe_trim": probe_trim, + "direct_output": [output.expert_output is not None for output in outputs], + "cuda_graph": use_graph, + "replay_iters": replay_iters, + "recv_caps": [output.recv_cap for output in outputs], + "child_sender_rows": child_token_counts, + "failures": failures.item(), + } + print( + "# MORI-EPV2-FP4-TBO: " + json.dumps(summary, sort_keys=True), + flush=True, + ) + if result_json := os.environ.get("RESULT_JSON"): + Path(result_json).write_text(json.dumps(summary, indent=2) + "\n") + for child in children: + child.mori_op.close() + child.mori_op.comm.destroy() + dist.destroy_process_group() + raise SystemExit(int(failures.item() != 0)) + + +if __name__ == "__main__": + main() diff --git a/test/registered/unit/eplb/test_megamoe_prefill_recorder.py b/test/registered/unit/eplb/test_megamoe_prefill_recorder.py index 40207ec671..025c5de89a 100644 --- a/test/registered/unit/eplb/test_megamoe_prefill_recorder.py +++ b/test/registered/unit/eplb/test_megamoe_prefill_recorder.py @@ -8,6 +8,7 @@ from sglang.srt.eplb.eplb_manager import EPLBManager from sglang.srt.eplb.eplb_map_record_fused import eplb_map_and_record_fused from sglang.srt.eplb.expert_distribution import ( _ExpertDistributionRecorderReal, + _InfiniteBuffer, _SelectExpertsSinglePassGatherer, should_advance_eplb_counter, should_record_forward_pass, @@ -179,5 +180,17 @@ class TestMegaMoEPrefillRecorder(unittest.TestCase): return recorder +class TestInfiniteBuffer(unittest.TestCase): + def test_growth_inside_inference_mode_accepts_later_appends(self): + """A buffer grown under inference_mode must still accept appends outside it.""" + buffer = _InfiniteBuffer(item_shape=(2,), dtype=torch.int32, device="cpu") + with torch.inference_mode(): + for i in range(129): + buffer.append(torch.full((2,), i, dtype=torch.int32)) + with torch.no_grad(): + buffer.append(torch.full((2,), 129, dtype=torch.int32)) + self.assertEqual(buffer.get_all()[:, 0].tolist(), list(range(130))) + + if __name__ == "__main__": unittest.main() diff --git a/test/registered/unit/layers/moe/test_aiter_runner.py b/test/registered/unit/layers/moe/test_aiter_runner.py index 60faa2369f..aed8b86aaa 100644 --- a/test/registered/unit/layers/moe/test_aiter_runner.py +++ b/test/registered/unit/layers/moe/test_aiter_runner.py @@ -5,6 +5,7 @@ import pytest import torch import sglang.srt.layers.moe.moe_runner.aiter as aiter_runner +from sglang.srt.environ import envs from sglang.srt.layers.moe.moe_runner.aiter import ( AiterMoeQuantInfo, AiterQuantType, @@ -115,5 +116,87 @@ def test_aiter_runner_preserves_no_combine_rank_for_empty_input(monkeypatch): assert output.hidden_states.shape == (0, 2, 4) +@pytest.mark.parametrize("supports_output", [False, True]) +def test_aiter_runner_uses_epv2_output_when_kernel_supports_it( + monkeypatch, supports_output +): + def fused_moe(hidden_states, output=None, **kwargs): + result = hidden_states + 2 + if output is not None: + output.copy_(result) + return output + return result + + _install_fake_aiter(monkeypatch, fused_moe) + monkeypatch.setattr( + aiter_runner, "_aiter_fused_moe_supports_output", lambda: supports_output + ) + runner_input = _runner_input() + runner_input.hidden_states.fill_(1) + runner_input.output = torch.zeros_like(runner_input.hidden_states) + runner = AiterRunnerCore(MoeRunnerConfig(activation="silu")) + + result = runner.run(runner_input, _quant_info(), running_state={}).hidden_states + + torch.testing.assert_close(result, torch.full_like(result, 3)) + assert (result.data_ptr() == runner_input.output.data_ptr()) == supports_output + torch.testing.assert_close( + runner_input.output, + torch.full_like(result, 3 if supports_output else 0), + ) + + +@pytest.mark.parametrize( + "low_latency,recv_cap,expected_rows", + [(False, 0, 16), (False, 32, 16), (True, 5, 5)], +) +def test_mori_pre_permute_consumes_dispatcher_cap( + monkeypatch, low_latency, recv_cap, expected_rows +): + from sglang.srt.layers.moe.token_dispatcher.moriep import ( + MoriEPLLDispatchOutput, + MoriEPNormalDispatchOutput, + ) + + fake_utils = ModuleType("sglang.kernels.ops.moe.rocm_moe_utils") + fake_utils.upscale = None + fake_utils.upscale_mxfp4 = None + monkeypatch.setitem(sys.modules, fake_utils.__name__, fake_utils) + output_type = MoriEPLLDispatchOutput if low_latency else MoriEPNormalDispatchOutput + hidden = torch.zeros((16, 4), dtype=torch.bfloat16) + scales = torch.ones((16, 1)) + ids = torch.zeros((16, 2), dtype=torch.int32) + weights = torch.ones((16, 2)) + kwargs = {} if low_latency else {"expert_output": torch.empty_like(hidden)} + dispatched = output_type( + hidden_states=hidden, + hidden_states_scale=scales, + topk_ids=ids, + topk_weights=weights, + num_recv_tokens_per_expert=torch.tensor([5]), + origin_topk_ids=ids[:2], + origin_topk_weights=weights[:2], + out_dtype=torch.bfloat16, + recv_cap=recv_cap, + **kwargs, + ) + quant_info = _quant_info(quant_type=AiterQuantType.NONE) + state = {} + # The dispatcher owns the override. The runner must not read it again. + with envs.SGLANG_MORI_MOE_MAX_INPUT_TOKENS.override(3): + result = aiter_runner._pre_permute_deepep_to_aiter( + dispatched, quant_info, MoeRunnerConfig(), state + ) + assert result.hidden_states.shape[0] == expected_rows + assert result.topk_ids.shape[0] == expected_rows + assert result.topk_weights.shape[0] == expected_rows + assert result.a1_scale.shape[0] == expected_rows + assert result.hidden_states.data_ptr() == hidden.data_ptr() + assert state["aiter_combine_topk_ids"] is dispatched.origin_topk_ids + if not low_latency: + assert result.output.shape[0] == expected_rows + assert result.output.data_ptr() == dispatched.expert_output.data_ptr() + + if __name__ == "__main__": sys.exit(pytest.main([__file__, "-v"]))