diff --git a/docs/docs/references/environment_variables.mdx b/docs/docs/references/environment_variables.mdx
index 89122c3da7..b7aa1e550b 100644
--- a/docs/docs/references/environment_variables.mdx
+++ b/docs/docs/references/environment_variables.mdx
@@ -517,9 +517,19 @@ SGLang supports various environment variables that can be used to configure its
SGLANG_MORI_MOE_MAX_INPUT_TOKENS |
- Truncate the dispatch buffer to this many rows before MoE computation, reducing kernel overhead on padding tokens. The value must be >= the actual number of received tokens (totalRecvTokenNum); setting it too small causes incorrect results. 0 disables truncation (use full buffer). |
+ Manual MoE input row limit for MORI EPv1 and EPv2. A positive value overrides SGLANG_MORI_RECV_BOUND and is used unrounded, capped at the receive capacity; it only narrows the MoE input view, not the communication buffer. It must be >= the actual received tokens, or results are incorrect. 0 leaves it to SGLANG_MORI_RECV_BOUND. |
0 |
+
+ SGLANG_MORI_EP_V2 |
+ Select MORI EPv2 for --moe-a2a-backend mori. When unset, EPv2 is used only for DeepSeek-V4 models on AITER with ep_size <= 8 (on gfx1250, any ep_size that fits one UALink vPOD), a bfloat16 model dtype and MORI_ENABLE_SDMA off, and MORI EPv1 otherwise. Set to true to use EPv2 even when these checks fail (a warning names the failed check), or false to always use EPv1. |
+ true |
+
+
+ SGLANG_MORI_RECV_BOUND |
+ Hand the MoE only the receive rows the per-rank DP token counts can fill (rounded up) instead of the full MORI receive buffer. Unverified layouts, missing metadata and two-batch overlap keep the full view. |
+ false |
+
SGLANG_PPLX_NUM_MAX_DISPATCH_TOKENS_PER_RANK |
Maximum number of dispatched tokens per rank for the PPLX (--moe-a2a-backend pplx) NVSHMEM buffer allocation. Must be ≥ the worst-case per-rank forward tokens (the per-rank prefill chunk chunked_prefill_size / dp_size, or the decode CUDA-graph batch size); the server refuses to start otherwise. |
diff --git a/python/sglang/srt/arg_groups/moe_hook.py b/python/sglang/srt/arg_groups/moe_hook.py
index 7760431cf9..946a1f875d 100644
--- a/python/sglang/srt/arg_groups/moe_hook.py
+++ b/python/sglang/srt/arg_groups/moe_hook.py
@@ -30,7 +30,12 @@ from sglang.srt.connector import ConnectorType
from sglang.srt.environ import envs
from sglang.srt.model_executor.cuda_graph_config import Backend, Phase, with_phase
from sglang.srt.runtime_context import attn_dp_enabled_of, get_platform
-from sglang.srt.utils.common import is_sm100_supported, parse_connector_type
+from sglang.srt.utils.common import (
+ get_bool_env_var,
+ is_gfx1250_supported,
+ is_sm100_supported,
+ parse_connector_type,
+)
logger = logging.getLogger(__name__)
@@ -460,6 +465,7 @@ def handle_a2a_moe(server_args: Any):
)
if a2a_backend == "mori":
+ is_epv2 = _resolve_mori_ep_v2(server_args)
if cfg.deepep_mode == "auto":
declare_resolution(
server_args,
@@ -467,6 +473,16 @@ def handle_a2a_moe(server_args: Any):
deepep_mode="normal",
)
logger.warning("auto set deepep_mode=`normal` for MORI EP")
+ elif is_epv2 and cfg.deepep_mode == "low_latency":
+ declare_resolution(
+ server_args,
+ "_handle_a2a_moe",
+ deepep_mode="normal",
+ )
+ logger.warning(
+ "MORI EPv2 has no separate low-latency kernel; "
+ "using deepep_mode=`normal` for deepep_mode=`low_latency`"
+ )
# Check chunked prefill for mori
# Skip validation if chunked prefill is disabled (i.e., size <= 0).
@@ -682,6 +698,63 @@ def validate_flashinfer_a2a_token_budget(server_args: Any) -> None:
)
+def _resolve_mori_ep_v2(server_args: Any) -> bool:
+ """Whether to use MORI EPv2.
+
+ Unset prefers EPv2 and falls back to EPv1 where it is unsupported; an explicit
+ true keeps EPv2 regardless, and false selects EPv1.
+ """
+ if not envs.SGLANG_MORI_EP_V2.get():
+ return False
+
+ reason = _mori_epv2_unsupported_reason(server_args)
+ if reason is None:
+ return True
+ if envs.SGLANG_MORI_EP_V2.is_set():
+ logger.warning(
+ "SGLANG_MORI_EP_V2=true is set explicitly, so MORI EPv2 is used even "
+ "though it does not support %s.",
+ reason,
+ )
+ return True
+
+ logger.warning(
+ "MORI EPv2 does not support %s; falling back to MORI EPv1. "
+ "Set SGLANG_MORI_EP_V2=true to force EPv2, or false to select EPv1.",
+ reason,
+ )
+ # Worker processes inherit the env, so every reader agrees on EPv1.
+ envs.SGLANG_MORI_EP_V2.set(False)
+ return False
+
+
+def _mori_epv2_unsupported_reason(server_args: Any) -> str | None:
+ # The dispatcher falls back from unsupported dispatch/combine dtypes once it
+ # knows the weight dtype.
+ import torch
+
+ from sglang.srt.configs.model_config import is_deepseek_v4
+
+ cfg = resolved_view(server_args)
+ # gfx1250 forms its LSA team from the UALink vPOD, which can span hosts and is
+ # only known once cco is up, so init_mori_epv2_op checks that EP fits in it.
+ if cfg.tp_size > 8 and not is_gfx1250_supported():
+ return f"ep_size={cfg.tp_size} across hosts (EPv2 is validated within one host)"
+ if not envs.SGLANG_USE_AITER.get():
+ return "SGLANG_USE_AITER=0 (EPv2 experts run on the aiter MoE runner)"
+ if get_bool_env_var("MORI_ENABLE_SDMA", "false"):
+ return "MORI_ENABLE_SDMA=1 (SDMA is EPv1-only)"
+ if parse_connector_type(cfg.model_path) == ConnectorType.INSTANCE:
+ return "a model loaded through an instance connector (cannot validate it)"
+ model_config = model_config_of(server_args)
+ if model_config.dtype != torch.bfloat16:
+ return f"dtype={model_config.dtype} (EPv2 requires bfloat16)"
+ if not is_deepseek_v4(model_config.hf_config):
+ architecture = (model_config.hf_config.architectures or [None])[0]
+ return f"{architecture} (EPv2 is validated on DeepSeek-V4 only)"
+ return None
+
+
def required_mori_dispatch_tokens_per_rank(server_args: Any) -> int:
"""Max tokens a single rank dispatches through MoRI in one forward."""
cfg = resolving_view(server_args)
diff --git a/python/sglang/srt/environ.py b/python/sglang/srt/environ.py
index 150648516f..c1f436eee5 100644
--- a/python/sglang/srt/environ.py
+++ b/python/sglang/srt/environ.py
@@ -864,6 +864,17 @@ class Envs:
# and relies on the RDMA retry-exceeded timeout only.
SGLANG_MORI_TRANSFER_TIMEOUT_MS = EnvInt(0)
SGLANG_MORI_NUM_MAX_DISPATCH_TOKENS_PER_RANK = EnvInt(4096)
+ # Unset: EPv2 where moe_hook finds it supported (DeepSeek-V4 only), else EPv1.
+ # true forces EPv2; false forces EPv1.
+ SGLANG_MORI_EP_V2 = EnvBool(True)
+ SGLANG_MORI_DISPATCH_DTYPE = EnvStr("auto")
+ SGLANG_MORI_COMBINE_DTYPE = EnvStr("auto")
+ SGLANG_MORI_PREALLOC_MAX_RECV_TOKENS = EnvInt(0)
+ # Hand AITER only ceil32(sum of per-rank DP tokens) rows of the MORI EP receive
+ # buffer; unproved layouts or metadata keep the full view. Forces DP-synced buckets.
+ SGLANG_MORI_RECV_BOUND = EnvBool(False)
+ # Manual MoE input row limit (EPv1 and EPv2); 0 leaves it to SGLANG_MORI_RECV_BOUND.
+ SGLANG_MORI_MOE_MAX_INPUT_TOKENS = EnvInt(0)
# ===================================================================
# AMD, ROCm, and AITER
diff --git a/python/sglang/srt/eplb/expert_distribution.py b/python/sglang/srt/eplb/expert_distribution.py
index 4f99f17b79..070f08dea9 100644
--- a/python/sglang/srt/eplb/expert_distribution.py
+++ b/python/sglang/srt/eplb/expert_distribution.py
@@ -1092,9 +1092,14 @@ class _InfiniteBuffer(_Buffer):
device = self._buffer.device
if self._size == curr_buffer_size:
- new_buffer = torch.zeros(
- (2 * curr_buffer_size, *self._item_shape), dtype=dtype, device=device
- )
+ # Growth can happen inside a caller's inference_mode (e.g. DSpark draft);
+ # an inference tensor buffer would reject later appends made outside it.
+ with torch.inference_mode(False):
+ new_buffer = torch.zeros(
+ (2 * curr_buffer_size, *self._item_shape),
+ dtype=dtype,
+ device=device,
+ )
new_buffer[:curr_buffer_size] = self._buffer
self._buffer = new_buffer
diff --git a/python/sglang/srt/layers/moe/ep_moe/layer.py b/python/sglang/srt/layers/moe/ep_moe/layer.py
index d445bfc71e..cd5164c3be 100644
--- a/python/sglang/srt/layers/moe/ep_moe/layer.py
+++ b/python/sglang/srt/layers/moe/ep_moe/layer.py
@@ -352,6 +352,8 @@ class DeepEPMoE(FusedMoE):
def get_moe_impl_class(quant_config: Optional[QuantizationConfig]):
+ if get_moe_a2a_backend().is_mori_epv2():
+ return FusedMoE
# [TODO] kk, temporary solution
if (
get_moe_a2a_backend().is_mori()
diff --git a/python/sglang/srt/layers/moe/moe_runner/aiter.py b/python/sglang/srt/layers/moe/moe_runner/aiter.py
index 18dc7c7734..5548c8bd2e 100644
--- a/python/sglang/srt/layers/moe/moe_runner/aiter.py
+++ b/python/sglang/srt/layers/moe/moe_runner/aiter.py
@@ -23,8 +23,7 @@ from sglang.srt.layers.moe.moe_runner.base import (
register_pre_permute,
)
from sglang.srt.layers.moe.utils import MoeRunnerBackend
-from sglang.srt.runtime_context import get_parallel
-from sglang.srt.utils import get_bool_env_var, get_int_env_var, is_gfx95_supported
+from sglang.srt.utils import get_bool_env_var, is_gfx95_supported
if TYPE_CHECKING:
from sglang.srt.layers.moe.token_dispatcher.base import CombineInput
@@ -87,6 +86,8 @@ class AiterRunnerInput(RunnerInput):
# Mori-only fused_moe kwargs.
num_local_tokens: Optional[torch.Tensor] = None
output_dtype: Optional[torch.dtype] = None
+ # Optional MORI EPv2 combine staging view for fused_moe output.
+ output: Optional[torch.Tensor] = None
@property
def runner_backend(self) -> MoeRunnerBackend:
@@ -156,95 +157,11 @@ def _aiter_fused_moe_supports_no_combine() -> bool:
return "no_combine" in inspect.signature(fused_moe).parameters
-_RECV_BOUND_LOGGED: set[int] = set()
-_RECV_BOUND_WARNED = False
+@functools.cache
+def _aiter_fused_moe_supports_output() -> bool:
+ from aiter.fused_moe import fused_moe
-
-def _warn_recv_bound_unavailable() -> None:
- global _RECV_BOUND_WARNED
- if not _RECV_BOUND_WARNED:
- _RECV_BOUND_WARNED = True
- logger.warning(
- "SGLANG_MORI_RECV_BOUND is set but the per-rank DP token counts do "
- "not cover every mori sender, so the receive fan-in is unknown; "
- "leaving the receive buffer unbounded."
- )
-
-
-def _mori_decode_recv_bound(recv_rows: int, topk: int) -> int:
- """Live rows mori's receive buffer can hold in decode, or 0 for "do not bound".
-
- Worst case fan-in is every rank routing all of its tokens to this one, so
- `sum(per-rank tokens) * topk`, where topk already includes the fused shared
- expert. The per-rank counts come from the DP sync, so this is the fan-in for
- the batch actually being run rather than an upper bound over all batches.
-
- That is only sound because enabling this gate also makes
- `require_mlp_tp_gather()` true for mori, which gives every rank the same
- cuda-graph bucket. The value is baked into a captured graph and has to hold
- for every later replay; with per-rank buckets a rank on a narrow tier could
- be handed rows by a peer on a wider one, and the only bound valid under that
- is the widest tier's -- 4-16x looser than the batch being run, which costs
- more in expert-GEMM tiles (M 32/64 -> 128) than the trim saves.
-
- Two cases stay unbounded, because a bound below the real fan-in silently
- drops rows from the all-to-all -- wrong output rather than an error:
-
- * Prefill, whose per-rank counts are uneven and not knowable here.
- * Anything that leaves the per-rank counts unpopulated, or where the EP world
- is wider than the DP world so the counts do not cover every sender.
- """
- if not get_bool_env_var("SGLANG_MORI_RECV_BOUND", "false"):
- return 0
-
- from sglang.srt.layers.dp_attention import (
- get_dp_global_num_tokens,
- get_is_extend_in_batch,
- )
-
- if get_is_extend_in_batch():
- return 0
-
- per_rank_tokens = get_dp_global_num_tokens()
- ep_size = get_parallel().moe_ep_size
- if not per_rank_tokens or len(per_rank_tokens) < ep_size:
- # Either the DP sync did not publish counts, or they do not cover every
- # mori sender. Both mean the fan-in is unknown here.
- _warn_recv_bound_unavailable()
- return 0
-
- max_tokens = sum(per_rank_tokens)
- bound = max_tokens * topk
- # Never grow the tensor, and nothing to do when there is nothing to trim.
- if not 0 < bound < recv_rows:
- return 0
-
- # One INFO line the first time it engages, so an inert bound is not mistaken
- # for an active one in the results. Per-tier values go to DEBUG: capture
- # visits every tier, and at INFO on every rank that is dozens of lines.
- if get_parallel().tp_rank == 0 and bound not in _RECV_BOUND_LOGGED:
- first = not _RECV_BOUND_LOGGED
- _RECV_BOUND_LOGGED.add(bound)
- if first:
- logger.info(
- "mori recv bound active: %d rows -> %d for this tier "
- "(dp_tokens=%d ep=%d topk=%d); per-tier values at DEBUG",
- recv_rows,
- bound,
- max_tokens,
- get_parallel().moe_ep_size,
- topk,
- )
- else:
- logger.debug(
- "mori recv bound: %d rows -> %d (dp_tokens=%d ep=%d topk=%d)",
- recv_rows,
- bound,
- max_tokens,
- get_parallel().moe_ep_size,
- topk,
- )
- return bound
+ return "output" in inspect.signature(fused_moe).parameters
class AiterRunnerCore(MoeRunnerCore):
@@ -283,8 +200,6 @@ class AiterRunnerCore(MoeRunnerCore):
from aiter.fused_moe import fused_moe
- from sglang.srt.environ import envs
-
a1_scale = (
runner_input.a1_scale
if runner_input.a1_scale is not None
@@ -330,6 +245,15 @@ class AiterRunnerCore(MoeRunnerCore):
extra["swiglu_limit"] = quant_info.swiglu_limit
if self.config.no_combine:
extra["no_combine"] = True
+ elif (
+ runner_input.output is not None
+ and "output" not in extra
+ and extra.get("stage2_scatter") is None
+ and _aiter_fused_moe_supports_output()
+ ):
+ # MORI EPv2 skips staging when this exact arena view is returned. Older
+ # AITER versions, no_combine and stage2_scatter keep their old path.
+ extra["output"] = runner_input.output
extra["moe_sorting_dispatch_policy"] = _MOE_SORTING_DISPATCH_POLICY
# gfx950 small-M MXFP4 kernel (on by default, SGLANG_ROCM_SMALLM_MOE=0 disables): same layouts as aiter, bf16 activations.
@@ -432,8 +356,8 @@ def pre_permute_standard_to_aiter(
def _is_mori_dispatch_output(dispatch_output: Any) -> bool:
- # MoriEP{Normal,LL}DispatchOutput carry the post-mori-permute origin_topk_*
- # tensors that the standard DeepEP outputs lack.
+ # MORI EP dispatch outputs carry the origin_topk_* tensors for combine,
+ # which the standard DeepEP outputs lack.
return hasattr(dispatch_output, "origin_topk_ids")
@@ -486,6 +410,7 @@ def _pre_permute_deepep_to_aiter(
a1_scale: Optional[torch.Tensor] = None
num_local_tokens: Optional[torch.Tensor] = None
output_dtype: Optional[torch.dtype] = None
+ output: Optional[torch.Tensor] = None
quant_type = quant_info.quant_type
if is_mori:
@@ -494,21 +419,21 @@ def _pre_permute_deepep_to_aiter(
a1_scale = dispatch_output.hidden_states_scale
num_local_tokens = dispatch_output.num_recv_tokens_per_expert
output_dtype = dispatch_output.out_dtype
+ output = getattr(dispatch_output, "expert_output", None)
+ mori_max = dispatch_output.recv_cap
- # Truncate dispatch tensors to the configured cap; mori combine only
+ # Truncate dispatch tensors to the configured cap; MORI EP combine only
# reads [0, totalRecvTokenNum), so the truncated result needs no
- # padding back.
- mori_max = get_int_env_var("SGLANG_MORI_MOE_MAX_INPUT_TOKENS", 0)
- if mori_max <= 0:
- mori_max = _mori_decode_recv_bound(
- hidden_states.shape[0], topk_ids.shape[-1]
- )
- if mori_max > 0:
+ # padding back. Slice the optional EPv2 output to the same row count,
+ # retaining the physical arena and routing stride.
+ if 0 < mori_max < hidden_states.shape[0]:
hidden_states = hidden_states[:mori_max]
if a1_scale is not None:
a1_scale = a1_scale[:mori_max]
topk_ids = topk_ids[:mori_max]
topk_weights = topk_weights[:mori_max]
+ if output is not None:
+ output = output[:mori_max]
# Upscale dispatched activations when there is no AITER kernel for the
# weight/activation dtype pair.
@@ -595,6 +520,7 @@ def _pre_permute_deepep_to_aiter(
a1_scale=a1_scale,
num_local_tokens=num_local_tokens,
output_dtype=output_dtype,
+ output=output,
)
diff --git a/python/sglang/srt/layers/moe/token_dispatcher/moriep.py b/python/sglang/srt/layers/moe/token_dispatcher/moriep.py
index cf7b32747e..1d14cbe6f4 100644
--- a/python/sglang/srt/layers/moe/token_dispatcher/moriep.py
+++ b/python/sglang/srt/layers/moe/token_dispatcher/moriep.py
@@ -4,12 +4,14 @@ import functools
import logging
import os
from dataclasses import dataclass
-from typing import TYPE_CHECKING, List, NamedTuple, Optional, Tuple
+from typing import TYPE_CHECKING, NamedTuple, Optional
+from sglang.srt.environ import envs
from sglang.srt.eplb.expert_distribution import (
_ExpertDistributionRecorderNoop,
get_global_expert_distribution_recorder,
)
+from sglang.srt.eplb.expert_location import get_global_expert_location_metadata
from sglang.srt.layers.dp_attention import get_is_extend_in_batch
from sglang.srt.layers.moe.token_dispatcher.base import (
BaseDispatcher,
@@ -46,6 +48,9 @@ from sglang.kernels.ops.quantization.fp8_kernel import fp8_dtype
FP8_BLOCK_SIZE = 128
MXFP4_BLOCK_SIZE = 32
+MORI_MIN_LOGICAL_RECV_ROWS = 32
+MORI_LOGICAL_RECV_ROW_ALIGN = 32
+
_is_hip = is_hip()
_use_aiter = get_bool_env_var("SGLANG_USE_AITER") and _is_hip
@@ -55,6 +60,50 @@ if _use_aiter:
logger = logging.getLogger(__name__)
+def round_logical_recv_rows(cluster_rows: int, pow2_buckets: bool) -> int:
+ # Measured on MI355X with DSV4 FP4 AITER fused MoE: latency tracks the logical
+ # M and a new M costs no JIT, so power-of-two padding only pays off for a
+ # backend that pre-builds power-of-two receive caps.
+ if pow2_buckets:
+ return max(MORI_MIN_LOGICAL_RECV_ROWS, 1 << (cluster_rows - 1).bit_length())
+ align = MORI_LOGICAL_RECV_ROW_ALIGN
+ return max(MORI_MIN_LOGICAL_RECV_ROWS, -(-cluster_rows // align) * align)
+
+
+def mori_recv_bound(local_rows: int, sender_rows: Optional[tuple[int, ...]]) -> int:
+ """Return the unrounded receive-row bound, or 0 to keep the full view.
+
+ Verified MORI kernels deduplicate rows per destination rank, so the bound
+ comes from sender rows. The caller checks backend/TBO eligibility and
+ applies rounding and the physical capacity.
+ """
+ parallel = get_parallel()
+ ep_size = parallel.moe_ep_size
+ # ParallelContext already validates rank ranges and topology relationships.
+ if ep_size <= 1 or parallel.tp_size != ep_size or parallel.attn_cp_size != 1:
+ return 0
+
+ attn_dp_size = parallel.attn_dp_size
+ attn_dp_rank = parallel.attn_dp_rank
+ attn_tp_size = parallel.attn_tp_size
+ attn_tp_rank = parallel.attn_tp_rank
+
+ if attn_dp_size == 1:
+ # Use the local tensor_split chunk: DSpark can leave DP token counts stale.
+ return attn_tp_size * local_rows + attn_tp_rank
+ if (
+ sender_rows is None
+ or len(sender_rows) != attn_dp_size
+ or any(rows < 0 for rows in sender_rows)
+ ):
+ return 0
+ base, extra = divmod(sender_rows[attn_dp_rank], attn_tp_size)
+ expected_local_rows = base + int(attn_tp_rank < extra)
+ if local_rows != expected_local_rows:
+ return 0
+ return sum(sender_rows)
+
+
def _should_record_expert_distribution() -> bool:
recorder = get_global_expert_distribution_recorder()
if recorder.recording:
@@ -68,6 +117,15 @@ def _should_record_expert_distribution() -> bool:
return False
+def _record_local_expert_count(local_expert_count: torch.Tensor) -> None:
+ # The recorder tracks routed experts only; drop the per-rank fused shared
+ # expert slot that follows them.
+ num_routed = get_global_expert_location_metadata().num_local_physical_experts
+ get_global_expert_distribution_recorder().on_deepep_dispatch_low_latency(
+ local_expert_count[:num_routed]
+ )
+
+
@functools.lru_cache(maxsize=1)
def _aiter_supports_mxfp8_dispatch() -> bool:
"""Whether this aiter can consume fp8 activations with group-32 e8m0 scales.
@@ -91,6 +149,17 @@ def _aiter_supports_mxfp8_dispatch() -> bool:
return False
+@functools.lru_cache(maxsize=1)
+def _mori_epv2_fp8_direct_cast_saturates() -> bool:
+ # Older MORI EPv2 fp8_direct_cast combine emits NaN past the fp8 max; the fix
+ # added this helper. Drop once the minimum MORI carries the fix.
+ try:
+ from mori.ops.dispatch_combine_v2 import intranode_kernels
+ except ImportError:
+ return False
+ return hasattr(intranode_kernels, "_direct_cast_fp8_max")
+
+
class MoriEPPDispatchHooks(DeepEPPDispatchHooks):
def __call__(self, dispatcher: BaseDispatcher):
for hook_fun in self.hook_dict.values():
@@ -104,10 +173,13 @@ class MoriEPNormalDispatchOutput(NamedTuple):
hidden_states_scale: Optional[torch.Tensor]
topk_ids: torch.Tensor
topk_weights: torch.Tensor
- num_recv_tokens_per_expert: List[int]
+ num_recv_tokens_per_expert: torch.Tensor
origin_topk_ids: torch.Tensor
origin_topk_weights: torch.Tensor
out_dtype: torch.dtype
+ expert_output: Optional[torch.Tensor] = None
+ # Logical MoE input cap selected by the dispatcher; 0 disables trimming.
+ recv_cap: int = 0
@property
def format(self) -> DispatchOutputFormat:
@@ -121,10 +193,12 @@ class MoriEPLLDispatchOutput(NamedTuple):
hidden_states_scale: Optional[torch.Tensor]
topk_ids: torch.Tensor
topk_weights: torch.Tensor
- num_recv_tokens_per_expert: List[int]
+ num_recv_tokens_per_expert: torch.Tensor
origin_topk_ids: torch.Tensor
origin_topk_weights: torch.Tensor
out_dtype: torch.dtype
+ # Logical MoE input cap selected by the dispatcher; 0 disables trimming.
+ recv_cap: int = 0
@property
def format(self) -> DispatchOutputFormat:
@@ -184,14 +258,24 @@ class CombineDtype(Enum):
@dataclass(frozen=True)
-class EpDispatchConfig:
+class _MoriEPv1LaunchConfig:
kernel_type: mori.ops.EpDispatchCombineKernelType
warp_num_per_block: int
block_num: int
rdma_block_num: int
-def get_ep_dispatch_configs(num_max_dispatch_tokens_per_rank: int = 4096):
+@dataclass(frozen=True)
+class _MoriEPv2LaunchConfig:
+ dispatch_block_num: Optional[int]
+ dispatch_warp_num_per_block: Optional[int]
+ combine_block_num: Optional[int]
+ combine_warp_num_per_block: Optional[int]
+
+
+def _get_epv1_launch_configs(
+ num_max_dispatch_tokens_per_rank: int = 4096,
+) -> dict[EpMode, _MoriEPv1LaunchConfig]:
import mori
# Selects the inter-node kernel. `InterNodeV1LL` is used if `num_max_dispatch_tokens_per_rank`
@@ -209,19 +293,19 @@ def get_ep_dispatch_configs(num_max_dispatch_tokens_per_rank: int = 4096):
return {
# TODO(billishyahao): need to tune different configs for intra node async
# Also could be tuned for different AMD platform
- EpMode.INTRA_NODE: EpDispatchConfig(
+ EpMode.INTRA_NODE: _MoriEPv1LaunchConfig(
kernel_type=mori.ops.EpDispatchCombineKernelType.IntraNode,
warp_num_per_block=16,
block_num=80,
rdma_block_num=0,
),
- EpMode.INTER_NODE: EpDispatchConfig(
+ EpMode.INTER_NODE: _MoriEPv1LaunchConfig(
kernel_type=inter_kernel_type,
warp_num_per_block=8,
block_num=64,
rdma_block_num=32,
),
- EpMode.LOW_LATENCY: EpDispatchConfig(
+ EpMode.LOW_LATENCY: _MoriEPv1LaunchConfig(
kernel_type=mori.ops.EpDispatchCombineKernelType.AsyncLL,
warp_num_per_block=8,
block_num=64,
@@ -230,6 +314,20 @@ def get_ep_dispatch_configs(num_max_dispatch_tokens_per_rank: int = 4096):
}
+def _get_epv2_launch_config(tbo_enabled: bool) -> _MoriEPv2LaunchConfig:
+ if not tbo_enabled:
+ # MORI picks its own schedule.
+ return _MoriEPv2LaunchConfig(None, None, None, None)
+ # Under TBO a sibling batch computes while this one communicates, so cap the
+ # grid to leave CUs for it.
+ return _MoriEPv2LaunchConfig(
+ dispatch_block_num=32,
+ dispatch_warp_num_per_block=4,
+ combine_block_num=48,
+ combine_warp_num_per_block=4,
+ )
+
+
# init_mori_op only needs do once in model initial stage
# use lru_cache to reuse the same mori_op instance to avoid the init overhead for mori
@lru_cache(maxsize=4)
@@ -279,12 +377,12 @@ def init_mori_op(
if async_mode:
mode = EpMode.LOW_LATENCY
- cfg = get_ep_dispatch_configs(num_max_dispatch_tokens_per_rank)[mode]
+ launch_config = _get_epv1_launch_configs(num_max_dispatch_tokens_per_rank)[mode]
- kernel_type = cfg.kernel_type
- warp_num_per_block = cfg.warp_num_per_block
- block_num = cfg.block_num
- rdma_block_num = cfg.rdma_block_num
+ kernel_type = launch_config.kernel_type
+ warp_num_per_block = launch_config.warp_num_per_block
+ block_num = launch_config.block_num
+ rdma_block_num = launch_config.rdma_block_num
hidden_dim = hidden_size
scale_dim = 1
@@ -412,7 +510,208 @@ class CommStreamPool:
cls._streams.pop(key, None)
+def _init_cco_communicator(group, instance_id: int, per_rank_vmm_gb: int):
+ from mori.cco import Communicator
+
+ parallel = get_parallel()
+ world_size = parallel.moe_ep_size
+ rank = parallel.moe_ep_rank
+
+ uid = Communicator.get_unique_id() if rank == 0 else None
+ uid = group.broadcast_object(uid, src=0)
+ comm = Communicator.init(
+ world_size, rank, uid, per_rank_vmm=per_rank_vmm_gb * (1 << 30)
+ )
+ logger.info(
+ f"[MORI EPv2 init] cco communicator world={world_size} rank={rank} "
+ f"instance={instance_id} per_rank_vmm_gb={per_rank_vmm_gb}"
+ )
+ return comm
+
+
+def _cco_lsa_size(comm) -> int:
+ """Ranks in this rank's cco LSA team, i.e. the ones sharing its flat VA."""
+ import mori.cco.cco as _cco
+ from mori.cco.communicator import DevCommHandle
+
+ # The Communicator does not expose the team; a device comm without GDA QPs
+ # reads it cheaply.
+ requirements = _cco.DevCommRequirements()
+ requirements.gda_connection_type = _cco.GDA_CONNECTION_NONE
+ handle = DevCommHandle(comm, requirements=requirements)
+ try:
+ return handle.lsa_size
+ finally:
+ handle.close()
+
+
+def _epv2_arena_bytes(cfg) -> int:
+ """Upper bound of the MORI EPv2 intranode symmetric arena for `cfg`."""
+ cap = cfg.effective_max_recv
+ topk = cfg.num_experts_per_token
+ # The hip backend pads scale rows to 128 B; flydsl uses the unpadded row.
+ scale_row = -(-cfg.scale_dim * cfg.scale_type_size // 128) * 128
+ per_recv_row = (
+ cfg.token_nbytes + cfg.combine_token_nbytes + scale_row + topk * 8 + 4
+ )
+ total = cap * per_recv_row + cfg.world_size * 12
+ if cfg.is_scatter:
+ scatter_rows = cfg.world_size * cfg.max_num_inp_token_per_rank
+ scatter_row = cfg.hidden_dim * cfg.wire_elem_size + topk * 4
+ if cfg.fp8_blockwise:
+ scatter_row += cfg.combine_scale_dim * 4
+ total += scatter_rows * scatter_row
+ # SymmArena aligns each of its (at most 12) regions to 256 B.
+ return total + 12 * 256
+
+
+# Arbitrary floor: the former fixed per-rank cco window.
+_EPV2_MIN_PER_RANK_VMM_GB = 4
+
+
+def _epv2_per_rank_vmm_gb(cfg) -> int:
+ # The window only reserves virtual address space, so size it to the arena.
+ required_gb = -(-_epv2_arena_bytes(cfg) // (1 << 30))
+ return max(_EPV2_MIN_PER_RANK_VMM_GB, required_gb)
+
+
+# MORI EPv2 per-token scale layout: (scale_dim, bytes per scale) for each dispatch dtype.
+_EPV2_DISPATCH_SCALES = {
+ DispatchDtype.bf16: (0, 0),
+ DispatchDtype.fp8: (FP8_BLOCK_SIZE, torch.float32.itemsize),
+ DispatchDtype.fp4: (MXFP4_BLOCK_SIZE, torch.float8_e8m0fnu.itemsize),
+ DispatchDtype.mxfp8: (MXFP4_BLOCK_SIZE, torch.float8_e8m0fnu.itemsize),
+}
+_EPV2_COMBINE_QUANT = {
+ CombineDtype.bf16: "none",
+ CombineDtype.fp8: "fp8_blockwise",
+ CombineDtype.fp8_direct_cast: "fp8_direct_cast",
+}
+
+
+def _epv2_dispatch_torch_dtype(dispatch_dtype: DispatchDtype) -> torch.dtype:
+ if dispatch_dtype == DispatchDtype.bf16:
+ return torch.bfloat16
+ if dispatch_dtype == DispatchDtype.fp4:
+ return torch.float4_e2m1fn_x2
+ return fp8_dtype
+
+
+def _supports_dynamic_recv_cap(op) -> bool:
+ # MORI quantized combines run in scatter mode, which only takes the full cap.
+ return hasattr(op, "prepare_recv_cap") and not op.cfg.is_scatter
+
+
+# One op (and its own cco window, sized for its arena) per distinct config;
+# never evicted, since MoE layers keep using the op they were built with.
+@lru_cache(maxsize=None)
+def init_mori_epv2_op(
+ group,
+ router_topk: int,
+ num_experts: int,
+ num_local_experts: int,
+ hidden_size: int,
+ params_dtype: torch.dtype,
+ max_tokens_per_rank: int,
+ instance_id: int = 0,
+ max_total_recv_tokens: int = 0,
+ dispatch_dtype: DispatchDtype = DispatchDtype.bf16,
+ combine_dtype: CombineDtype = CombineDtype.bf16,
+ launch_config: _MoriEPv2LaunchConfig = _MoriEPv2LaunchConfig(
+ None, None, None, None
+ ),
+):
+ from mori.ops.dispatch_combine_v2 import (
+ EpDispatchCombineConfig,
+ EpDispatchCombineOp,
+ )
+
+ if params_dtype != torch.bfloat16:
+ raise ValueError(
+ "MORI EPv2 requires BF16 expert/combine output; "
+ f"got params_dtype={params_dtype}"
+ )
+ if combine_dtype not in _EPV2_COMBINE_QUANT:
+ raise ValueError(f"unsupported MORI EPv2 combine dtype: {combine_dtype}")
+ # MORI EPv2 quantizes the combine wire only on a symmetric bf16 op (scatter mode).
+ if combine_dtype != CombineDtype.bf16 and dispatch_dtype != DispatchDtype.bf16:
+ raise ValueError(
+ f"MORI EPv2 combine dtype {combine_dtype} requires bf16 dispatch; "
+ f"got {dispatch_dtype}"
+ )
+ transport_dtype = _epv2_dispatch_torch_dtype(dispatch_dtype)
+ is_asymmetric = transport_dtype != torch.bfloat16
+ scale_block, scale_type_size = _EPV2_DISPATCH_SCALES[dispatch_dtype]
+
+ parallel = get_parallel()
+ world_size = parallel.moe_ep_size
+ rank = parallel.moe_ep_rank
+ cfg_kwargs = dict(
+ rank=rank,
+ world_size=world_size,
+ hidden_dim=hidden_size,
+ max_num_inp_token_per_rank=max_tokens_per_rank,
+ num_experts_per_rank=num_local_experts,
+ num_experts_per_token=router_topk,
+ data_type=torch.bfloat16,
+ dispatch_data_type=transport_dtype if is_asymmetric else None,
+ combine_data_type=torch.bfloat16 if is_asymmetric else None,
+ scale_dim=hidden_size // scale_block if scale_block else 0,
+ scale_type_size=scale_type_size,
+ combine_mode="gather",
+ quant_type=_EPV2_COMBINE_QUANT[combine_dtype],
+ max_total_recv_tokens=max_total_recv_tokens,
+ )
+ cfg = EpDispatchCombineConfig(
+ **cfg_kwargs,
+ dispatch_block_num=launch_config.dispatch_block_num,
+ combine_block_num=launch_config.combine_block_num,
+ warp_num_per_block=launch_config.dispatch_warp_num_per_block,
+ combine_warp_num_per_block=launch_config.combine_warp_num_per_block,
+ )
+ comm = _init_cco_communicator(
+ group=group, instance_id=instance_id, per_rank_vmm_gb=_epv2_per_rank_vmm_gb(cfg)
+ )
+ # Ranks share the flat VA only within the cco LSA team (a host, or a UALink
+ # vPOD on gfx1250); a wider EP group needs MORI's internode path, which only
+ # the hip backend has. MORI rejects the configs that path cannot run.
+ lsa_size = _cco_lsa_size(comm)
+ if lsa_size < world_size:
+ if "gpu_per_node" not in EpDispatchCombineConfig.__dataclass_fields__:
+ raise ValueError(
+ f"MORI EPv2 ep_size={world_size} spans more than its cco LSA team "
+ f"({lsa_size} GPUs), and this MORI has no internode EPv2; "
+ "set SGLANG_MORI_EP_V2=false to use EPv1"
+ )
+ # The launch config is tuned for the intranode kernels; let MORI's
+ # internode tables pick the geometry.
+ cfg = EpDispatchCombineConfig(
+ **cfg_kwargs, gpu_per_node=lsa_size, kernel_backend="hip"
+ )
+ op = EpDispatchCombineOp(cfg, comm)
+ comm.barrier()
+ if _supports_dynamic_recv_cap(op):
+ # Pre-compile every power-of-two receive cap _select_recv_cap can pick, so
+ # CUDA graph capture never compiles a new tier.
+ cap = MORI_MIN_LOGICAL_RECV_ROWS
+ while cap <= cfg.effective_max_recv:
+ op.prepare_recv_cap(cap)
+ cap *= 2
+ logger.info(
+ f"[MORI EPv2 init] world={world_size} rank={rank} lsa_size={lsa_size} "
+ f"hidden={hidden_size} experts={num_experts} local_experts={num_local_experts} "
+ f"topk={router_topk} max_tokens={max_tokens_per_rank} "
+ f"recv_cap={cfg.effective_max_recv} "
+ f"dispatch_dtype={dispatch_dtype} combine_dtype={combine_dtype} "
+ f"combine_mode={cfg.combine_mode} launch_config={launch_config} "
+ f"schedule={cfg.schedule}"
+ )
+ return op
+
+
class _MoriEPDispatcherImplBase:
+ _is_epv2 = False
+
def __init__(
self,
group: torch.distributed.ProcessGroup,
@@ -425,10 +724,6 @@ class _MoriEPDispatcherImplBase:
deepep_mode: DeepEPMode,
instance_id: int = 0,
):
- try:
- import mori # noqa: F401
- except ImportError:
- raise ImportError("Mori EP is not installed. Please install.")
self.group = group
self.router_topk = router_topk
self.permute_fusion = permute_fusion
@@ -438,22 +733,241 @@ class _MoriEPDispatcherImplBase:
self.params_dtype = params_dtype
self.deepep_mode = deepep_mode
self.instance_id = instance_id
-
- self.num_max_dispatch_tokens_per_rank = get_int_env_var(
- "SGLANG_MORI_NUM_MAX_DISPATCH_TOKENS_PER_RANK", 4096
+ self.num_max_dispatch_tokens_per_rank = (
+ envs.SGLANG_MORI_NUM_MAX_DISPATCH_TOKENS_PER_RANK.get()
)
-
- self.enable_sdma = get_bool_env_var("MORI_ENABLE_SDMA", "false")
- self.use_external_inp_buf = True
-
- self._mori_op = None
self.dispatch_dtype = DispatchDtype.bf16
self.combine_dtype = CombineDtype.bf16
+ self.quant_config = {}
+ self.overlap_args = None
+ self.meta_overlap_args = None
+ self._comm_stream = None
+ self.async_finish = False
+ self._tbo_enabled = is_tbo_enabled()
+ self._trim_recv = envs.SGLANG_MORI_RECV_BOUND.get()
+ self._manual_recv_cap = envs.SGLANG_MORI_MOE_MAX_INPUT_TOKENS.get()
+ self._recv_cap_pow2_buckets = False
- self.quant_config: Optional[dict] = None
+ @staticmethod
+ def _snapshot_sender_rows() -> tuple[int, ...] | None:
+ """Copy CPU scheduler metadata; non-int rows would need a device sync."""
+ from sglang.srt.layers.dp_attention import get_dp_global_num_tokens
- self.overlap_args: Optional[CombineOverlapArgs] = None
- self.meta_overlap_args: Optional[dict] = None
+ rows = get_dp_global_num_tokens()
+ if not isinstance(rows, (list, tuple)) or not all(
+ isinstance(r, int) for r in rows
+ ):
+ return None
+ return tuple(rows)
+
+ def _get_physical_recv_rows(self) -> int:
+ raise NotImplementedError
+
+ def _is_recv_layout_verified(self) -> bool:
+ raise NotImplementedError
+
+ def _select_recv_cap(self) -> int:
+ """Select the logical MoE input cap before dispatch_b; 0 keeps the full view."""
+ self._recv_cap = 0
+ if self._manual_recv_cap > 0:
+ # The caller is responsible for covering all received tokens, as
+ # with the original EPv1 override. Do not round an explicit cap.
+ self._recv_cap = min(self._manual_recv_cap, self._get_physical_recv_rows())
+ elif (
+ self._trim_recv
+ and not self._tbo_enabled
+ and self._is_recv_layout_verified()
+ ):
+ bound = mori_recv_bound(self._num_tokens, self._dispatch_sender_rows)
+ # Empty or unproved bounds keep the full view.
+ if bound > 0:
+ cap = round_logical_recv_rows(
+ bound, pow2_buckets=self._recv_cap_pow2_buckets
+ )
+ if cap < self._get_physical_recv_rows():
+ self._recv_cap = cap
+ return self._recv_cap
+
+ def _quantize_dispatch_input(self, hidden_states):
+ num_tokens = hidden_states.shape[0]
+ scale = None
+ if self.dispatch_dtype == DispatchDtype.mxfp8:
+ # MXFP8 quant on live tokens, before the wire.
+ if num_tokens > 0:
+ # scale_type must be set explicitly: per_1x32_mx_quant_hip still
+ # defaults fp8 output to continuous fp32 scales for backward
+ # compatibility, but the MoE kernels (and the 1-byte scale_dim
+ # configured above) need the e8m0 byte layout.
+ hidden_states, scale = self.mxfp8_quant_func(
+ hidden_states,
+ quant_dtype=fp8_dtype,
+ scale_type=torch.float8_e8m0fnu,
+ )
+ else:
+ hidden_states = torch.empty(
+ hidden_states.shape, dtype=fp8_dtype, device=hidden_states.device
+ )
+ scale = torch.empty(
+ (0, self.hidden_size // MXFP4_BLOCK_SIZE),
+ dtype=torch.float8_e8m0fnu,
+ device=hidden_states.device,
+ )
+ elif self.dispatch_dtype == DispatchDtype.fp8:
+ # FP8 quant
+ if num_tokens > 0:
+ # NOTE: aiter is able to handle token=0 case in UT. But for some
+ # reason it failed at e2e case. Root cause TBD.
+ hidden_states, scale = self.fp8_quant_func(
+ hidden_states, quant_dtype=fp8_dtype
+ )
+ else:
+ hidden_states = torch.empty(
+ hidden_states.shape, dtype=fp8_dtype, device=hidden_states.device
+ )
+ scale = torch.empty(
+ (0, self.hidden_size // FP8_BLOCK_SIZE),
+ dtype=torch.float32,
+ device=hidden_states.device,
+ )
+
+ elif self.dispatch_dtype == DispatchDtype.fp4:
+ # FP4 quant
+ if num_tokens > 0:
+ hidden_states, scale = self.fp4_quant_func(hidden_states, shuffle=False)
+ else:
+ hidden_states = torch.empty(
+ (0, self.hidden_size // 2),
+ dtype=torch.float4_e2m1fn_x2,
+ device=hidden_states.device,
+ )
+ scale = torch.empty(
+ (0, self.hidden_size // MXFP4_BLOCK_SIZE),
+ dtype=torch.float8_e8m0fnu,
+ device=hidden_states.device,
+ )
+
+ return hidden_states, scale
+
+ def _prepare_topk(self, topk_output):
+ return topk_output.topk_weights, topk_output.topk_ids
+
+ def _capture_event_if_async(self):
+ if self._comm_stream is None or not self.async_finish:
+ return None
+ event = torch.cuda.Event(blocking=False, interprocess=False)
+ event.record(torch.cuda.current_stream())
+ return event
+
+ def dispatch_a(
+ self,
+ hidden_states: torch.Tensor,
+ topk_output: TopKOutput,
+ ):
+ self._num_tokens = hidden_states.shape[0]
+ # Keep metadata attached to this dispatch when a/b are separated or
+ # two TBO children interleave.
+ self._dispatch_sender_rows = self._snapshot_sender_rows()
+ output_dtype = hidden_states.dtype
+ hidden_states, scale = self._quantize_dispatch_input(hidden_states)
+ topk_weights, topk_ids = self._prepare_topk(topk_output)
+ return (
+ hidden_states,
+ topk_weights,
+ topk_ids,
+ scale,
+ output_dtype,
+ self._capture_event_if_async(),
+ )
+
+ def dispatch_b(self, *args, **kwargs):
+ raise NotImplementedError
+
+ def combine_a(
+ self,
+ hidden_states: torch.Tensor,
+ topk_ids: torch.Tensor,
+ topk_weights: torch.Tensor,
+ ):
+ return hidden_states, topk_ids, topk_weights, self._capture_event_if_async()
+
+ def combine_b(self, *args, **kwargs):
+ raise NotImplementedError
+
+ def set_quant_config(self, quant_config: dict) -> None:
+ self.quant_config = quant_config
+ weight_dtype = quant_config.get("weight_dtype")
+ self.dispatch_dtype = DispatchDtype.bf16
+ if weight_dtype == torch.float4_e2m1fn_x2:
+ self.dispatch_dtype = DispatchDtype.fp4
+ elif weight_dtype in (torch.float8_e4m3fn, torch.float8_e4m3fnuz):
+ self.dispatch_dtype = DispatchDtype.fp8
+ self.combine_dtype = self._default_combine_dtype(weight_dtype)
+ # Combine overrides read the final dispatch dtype, so apply dispatch first.
+ self._apply_dispatch_dtype_override()
+ self._apply_combine_dtype_override()
+
+ def _default_combine_dtype(self, weight_dtype) -> CombineDtype:
+ raise NotImplementedError
+
+ def _apply_dispatch_dtype_override(self):
+ """Apply the SGLANG_MORI_DISPATCH_DTYPE override."""
+ dispatch_dtype = envs.SGLANG_MORI_DISPATCH_DTYPE.get().lower()
+ epv2_dtypes = ("", "auto", "bf16", "fp8", "fp4", "mxfp8")
+ if self._is_epv2 and dispatch_dtype not in epv2_dtypes:
+ raise ValueError(
+ "SGLANG_MORI_DISPATCH_DTYPE must be auto, bf16, fp8, fp4 or mxfp8 "
+ f"for EPv2; got {dispatch_dtype!r}"
+ )
+ if envs.SGLANG_MORI_DISPATCH_DTYPE.is_set():
+ if dispatch_dtype != "auto":
+ if dispatch_dtype == "bf16":
+ self.dispatch_dtype = DispatchDtype.bf16
+ elif dispatch_dtype == "fp8":
+ self.dispatch_dtype = DispatchDtype.fp8
+ elif dispatch_dtype == "fp4":
+ self.dispatch_dtype = DispatchDtype.fp4
+ elif dispatch_dtype == "mxfp8":
+ if _aiter_supports_mxfp8_dispatch():
+ self.dispatch_dtype = DispatchDtype.mxfp8
+ else:
+ logger.warning_once(
+ "SGLANG_MORI_DISPATCH_DTYPE=mxfp8 requires an aiter "
+ "build whose per_1x32 quant accepts scale_type "
+ "(for the group-32 e8m0 byte layout the MoE kernels "
+ "consume). This aiter does not, so the send-side "
+ "quant would emit continuous fp32 scales and the "
+ "MoE would read them as e8m0 bytes. Falling back to "
+ "bf16 dispatch."
+ )
+
+ def _apply_combine_dtype_override(self):
+ raise NotImplementedError
+
+ def set_overlap_args(
+ self, combine_overlap_args: CombineOverlapArgs, meta_overlap_args: dict
+ ) -> None:
+ self.overlap_args = combine_overlap_args
+ self.meta_overlap_args = meta_overlap_args
+
+ def clear_overlap_args(self) -> None:
+ self.overlap_args = None
+ self.meta_overlap_args = None
+
+
+class _MoriEPv1DispatcherImplBase(_MoriEPDispatcherImplBase):
+ def __init__(self, **kwargs):
+ try:
+ import mori # noqa: F401
+ except ImportError:
+ raise ImportError("Mori EP is not installed. Please install.")
+ super().__init__(**kwargs)
+ self.enable_sdma = get_bool_env_var("MORI_ENABLE_SDMA", "false")
+ self.use_external_inp_buf = True
+ self._mori_op = None
+ self.fp8_quant_func = get_hip_quant(QuantType.per_1x128)
+ self.fp4_quant_func = get_hip_quant(QuantType.per_1x32)
+ # Same MX entry point; quant_dtype selects fp4x2 vs fp8.
+ self.mxfp8_quant_func = get_hip_quant(QuantType.per_1x32)
@property
def mori_op(self):
@@ -478,31 +992,12 @@ class _MoriEPDispatcherImplBase:
)
return self._mori_op
- def _apply_dispatch_dtype_override(self):
- """Apply env var override to fp8_dispatch/fp4_dispatch/fp8_combine flags."""
- if "SGLANG_MORI_DISPATCH_DTYPE" in os.environ:
- dispatch_dtype = os.environ["SGLANG_MORI_DISPATCH_DTYPE"].lower()
- if dispatch_dtype != "auto":
- if dispatch_dtype == "bf16":
- self.dispatch_dtype = DispatchDtype.bf16
- elif dispatch_dtype == "fp8":
- self.dispatch_dtype = DispatchDtype.fp8
- elif dispatch_dtype == "fp4":
- self.dispatch_dtype = DispatchDtype.fp4
- elif dispatch_dtype == "mxfp8":
- if _aiter_supports_mxfp8_dispatch():
- self.dispatch_dtype = DispatchDtype.mxfp8
- else:
- logger.warning_once(
- "SGLANG_MORI_DISPATCH_DTYPE=mxfp8 requires an aiter "
- "build whose per_1x32 quant accepts scale_type "
- "(for the group-32 e8m0 byte layout the MoE kernels "
- "consume). This aiter does not, so the send-side "
- "quant would emit continuous fp32 scales and the "
- "MoE would read them as e8m0 bytes. Falling back to "
- "bf16 dispatch."
- )
+ def _default_combine_dtype(self, weight_dtype) -> CombineDtype:
+ if weight_dtype == torch.float4_e2m1fn_x2:
+ return CombineDtype.fp8
+ return CombineDtype.bf16
+ def _apply_combine_dtype_override(self):
if "SGLANG_MORI_COMBINE_DTYPE" in os.environ:
combine_dtype = os.environ["SGLANG_MORI_COMBINE_DTYPE"].lower()
if combine_dtype != "auto":
@@ -515,157 +1010,27 @@ class _MoriEPDispatcherImplBase:
elif combine_dtype == "fp4":
self.combine_dtype = CombineDtype.fp4
- def dispatch_a(
- self,
- hidden_states: torch.Tensor,
- topk_output: TopKOutput,
- ):
- raise NotImplementedError
+ def _get_physical_recv_rows(self) -> int:
+ # MORI uses this same capacity to construct its dispatch output views.
+ return self.mori_op.max_num_tokens_to_recv()
- def dispatch_b(self, *args, **kwargs):
- raise NotImplementedError
+ def _is_recv_layout_verified(self) -> bool:
+ import mori
- def combine_a(
- self,
- hidden_states: torch.Tensor,
- topk_ids: torch.Tensor,
- topk_weights: torch.Tensor,
- ):
- raise NotImplementedError
-
- def combine_b(self, *args, **kwargs):
- raise NotImplementedError
-
- def set_quant_config(self, quant_config: dict) -> None:
- self.quant_config = quant_config
- # Auto-detect dispatch quantization from weight dtype
- weight_dtype = quant_config.get("weight_dtype", None)
- if weight_dtype in (torch.float8_e4m3fn, torch.float8_e4m3fnuz):
- self.dispatch_dtype = DispatchDtype.fp8
- self.combine_dtype = CombineDtype.bf16
- elif weight_dtype == torch.float4_e2m1fn_x2:
- self.dispatch_dtype = DispatchDtype.fp4
- self.combine_dtype = CombineDtype.fp8
- else:
- self.dispatch_dtype = DispatchDtype.bf16
- self.combine_dtype = CombineDtype.bf16
- # Apply env var override immediately so dispatch_a sees correct flags
- self._apply_dispatch_dtype_override()
-
- def set_overlap_args(
- self, combine_overlap_args: CombineOverlapArgs, meta_overlap_args: dict
- ) -> None:
- self.overlap_args = combine_overlap_args
- self.meta_overlap_args = meta_overlap_args
-
- def clear_overlap_args(self) -> None:
- self.overlap_args = None
- self.meta_overlap_args = None
-
- def _combine_kwargs(self, hidden_states: torch.Tensor) -> dict:
- return {}
+ return self.mori_op.config.kernel_type in (
+ mori.ops.EpDispatchCombineKernelType.IntraNode,
+ mori.ops.EpDispatchCombineKernelType.AsyncLL,
+ )
-class _MoriEPDispatcherImplNormal(_MoriEPDispatcherImplBase):
+class _MoriEPDispatcherImplNormal(_MoriEPv1DispatcherImplBase):
def __init__(self, async_finish: bool, **kwargs):
super().__init__(**kwargs)
-
self.async_finish = async_finish
- self.quant_config = {}
- self.fp8_quant_func = get_hip_quant(QuantType.per_1x128)
- self.fp4_quant_func = get_hip_quant(QuantType.per_1x32)
- # Same MX entry point; quant_dtype selects fp4x2 vs fp8.
- self.mxfp8_quant_func = get_hip_quant(QuantType.per_1x32)
self.enable_dual_stream = is_tbo_enabled()
- self._comm_stream = None
if self.enable_dual_stream:
self._comm_stream = CommStreamPool.get_stream_from_pool(self.group)
- def _capture_event_if_async(self) -> Optional[torch.cuda.Event]:
- assert self.enable_dual_stream, "dual stream must be enabled"
- if not self.async_finish:
- return None
- ev = torch.cuda.Event(blocking=False, interprocess=False)
- ev.record(torch.cuda.current_stream())
- return ev
-
- def dispatch_a(
- self,
- hidden_states: torch.Tensor,
- topk_output: TopKOutput,
- ):
- topk_weights, topk_ids = topk_output.topk_weights, topk_output.topk_ids
-
- num_token = hidden_states.shape[0]
- output_dtype = hidden_states.dtype
- scale = None
-
- if self.dispatch_dtype == DispatchDtype.mxfp8:
- # MXFP8 quant on live tokens, before the wire.
- if num_token > 0:
- # scale_type must be set explicitly: per_1x32_mx_quant_hip still
- # defaults fp8 output to continuous fp32 scales for backward
- # compatibility, but the MoE kernels (and the 1-byte scale_dim
- # configured above) need the e8m0 byte layout.
- hidden_states, scale = self.mxfp8_quant_func(
- hidden_states,
- quant_dtype=fp8_dtype,
- scale_type=torch.float8_e8m0fnu,
- )
- else:
- hidden_states = torch.empty(
- hidden_states.shape, dtype=fp8_dtype, device=hidden_states.device
- )
- scale = torch.empty(
- (0, self.hidden_size // MXFP4_BLOCK_SIZE),
- dtype=torch.float8_e8m0fnu,
- device=hidden_states.device,
- )
- elif self.dispatch_dtype == DispatchDtype.fp8:
- # FP8 quant
- if num_token > 0:
- # NOTE: aiter is able to handle token=0 case in UT. But for some
- # reason it failed at e2e case. Root cause TBD.
- hidden_states, scale = self.fp8_quant_func(
- hidden_states, quant_dtype=fp8_dtype
- )
- else:
- hidden_states = torch.empty(
- hidden_states.shape, dtype=fp8_dtype, device=hidden_states.device
- )
- scale = torch.empty(
- (0, self.hidden_size // FP8_BLOCK_SIZE),
- dtype=torch.float32,
- device=hidden_states.device,
- )
-
- elif self.dispatch_dtype == DispatchDtype.fp4:
- # FP4 quant
- if num_token > 0:
- hidden_states, scale = self.fp4_quant_func(hidden_states, shuffle=False)
- else:
- hidden_states = torch.empty(
- (0, self.hidden_size // 2),
- dtype=torch.float4_e2m1fn_x2,
- device=hidden_states.device,
- )
- scale = torch.empty(
- (0, self.hidden_size // MXFP4_BLOCK_SIZE),
- dtype=torch.float8_e8m0fnu,
- device=hidden_states.device,
- )
-
- previous_event = self._capture_event_if_async() if self._comm_stream else None
-
- return (
- hidden_states,
- topk_weights,
- topk_ids,
- scale,
- output_dtype,
- previous_event,
- )
-
def dispatch_b(
self,
hidden_states,
@@ -703,6 +1068,7 @@ class _MoriEPDispatcherImplNormal(_MoriEPDispatcherImplBase):
origin_topk_ids=topk_ids,
origin_topk_weights=topk_weights,
out_dtype=output_dtype,
+ recv_cap=self._recv_cap,
)
def _dispatch_core(
@@ -785,9 +1151,7 @@ class _MoriEPDispatcherImplNormal(_MoriEPDispatcherImplBase):
# mori local_expert_count is a GPU tensor; route it through the
# low_latency hook only when the recorder is actually active.
if record:
- get_global_expert_distribution_recorder().on_deepep_dispatch_low_latency(
- self.mori_op.local_expert_count
- )
+ _record_local_expert_count(self.mori_op.local_expert_count)
return (
packed_recv_hidden,
@@ -798,21 +1162,10 @@ class _MoriEPDispatcherImplNormal(_MoriEPDispatcherImplBase):
done_event,
)
- def combine_a(
- self,
- hidden_states: torch.Tensor,
- topk_ids: torch.Tensor,
- topk_weights: torch.Tensor,
- ):
- previous_event = self._capture_event_if_async() if self._comm_stream else None
- return hidden_states, topk_ids, topk_weights, previous_event
-
def combine_b(self, hidden_states, topk_ids, topk_weights, previous_event):
-
hidden_states, done_event = self._combine_core(
hidden_states, topk_ids, topk_weights, previous_event
)
-
if self._comm_stream and self.async_finish and done_event is not None:
torch.cuda.current_stream().wait_event(done_event)
@@ -844,10 +1197,7 @@ class _MoriEPDispatcherImplNormal(_MoriEPDispatcherImplBase):
if self.enable_sdma
else self.mori_op.combine
)
- combine_kwargs = self._combine_kwargs(hidden_states)
- combined_hidden_states = combine_fn(
- hidden_states, None, topk_ids, **combine_kwargs
- )[0]
+ combined_hidden_states = combine_fn(hidden_states, None, topk_ids)[0]
if self.enable_sdma:
self.mori_op.combine_recv()
@@ -858,26 +1208,14 @@ class _MoriEPDispatcherImplNormal(_MoriEPDispatcherImplBase):
compute_stream.wait_stream(comm_stream)
else:
- combine_kwargs = self._combine_kwargs(hidden_states)
combined_hidden_states = self.mori_op.combine(
- hidden_states, None, topk_ids, **combine_kwargs
+ hidden_states, None, topk_ids
)[0]
return combined_hidden_states, done_event
- def set_quant_config(self, quant_config: dict):
- super().set_quant_config(quant_config)
-
-
-class _MoriEPDispatcherImplLowLatency(_MoriEPDispatcherImplBase):
- def __init__(self, **kwargs):
- super().__init__(**kwargs)
- self.quant_config = {}
- self.fp8_quant_func = get_hip_quant(QuantType.per_1x128)
- self.fp4_quant_func = get_hip_quant(QuantType.per_1x32)
- # Same MX entry point; quant_dtype selects fp4x2 vs fp8.
- self.mxfp8_quant_func = get_hip_quant(QuantType.per_1x32)
+class _MoriEPDispatcherImplLowLatency(_MoriEPv1DispatcherImplBase):
def dispatch_a(
self,
hidden_states: torch.Tensor,
@@ -890,66 +1228,14 @@ class _MoriEPDispatcherImplLowLatency(_MoriEPDispatcherImplBase):
is mori.ops.EpDispatchCombineKernelType.AsyncLL
), "mori asyncll mismatch"
- num_tokens = hidden_states.shape[0]
- output_dtype = hidden_states.dtype
- scale = None
-
- if self.dispatch_dtype == DispatchDtype.mxfp8:
- # MXFP8 quant on live tokens, before the wire.
- if num_tokens > 0:
- # scale_type must be set explicitly: per_1x32_mx_quant_hip still
- # defaults fp8 output to continuous fp32 scales for backward
- # compatibility, but the MoE kernels (and the 1-byte scale_dim
- # configured above) need the e8m0 byte layout.
- hidden_states, scale = self.mxfp8_quant_func(
- hidden_states,
- quant_dtype=fp8_dtype,
- scale_type=torch.float8_e8m0fnu,
- )
- else:
- hidden_states = torch.empty(
- hidden_states.shape, dtype=fp8_dtype, device=hidden_states.device
- )
- scale = torch.empty(
- (0, self.hidden_size // MXFP4_BLOCK_SIZE),
- dtype=torch.float8_e8m0fnu,
- device=hidden_states.device,
- )
- elif self.dispatch_dtype == DispatchDtype.fp8:
- # FP8 quant
- if num_tokens > 0:
- # NOTE: aiter is able to handle token=0 case in UT. But for some
- # reason it failed at e2e case. Root cause TBD.
- hidden_states, scale = self.fp8_quant_func(
- hidden_states, quant_dtype=fp8_dtype
- )
- else:
- hidden_states = torch.empty(
- hidden_states.shape, dtype=fp8_dtype, device=hidden_states.device
- )
- scale = torch.empty(
- (0, self.hidden_size // FP8_BLOCK_SIZE),
- dtype=torch.float32,
- device=hidden_states.device,
- )
-
- elif self.dispatch_dtype == DispatchDtype.fp4:
- # FP4 quant
- if num_tokens > 0:
- hidden_states, scale = self.fp4_quant_func(hidden_states, shuffle=False)
- else:
- hidden_states = torch.empty(
- (0, self.hidden_size // 2),
- dtype=torch.float4_e2m1fn_x2,
- device=hidden_states.device,
- )
- scale = torch.empty(
- (0, self.hidden_size // MXFP4_BLOCK_SIZE),
- dtype=torch.float8_e8m0fnu,
- device=hidden_states.device,
- )
-
- topk_weights, topk_ids = topk_output.topk_weights, topk_output.topk_ids
+ (
+ hidden_states,
+ topk_weights,
+ topk_ids,
+ scale,
+ output_dtype,
+ _,
+ ) = super().dispatch_a(hidden_states, topk_output)
(
packed_recv_hidden,
@@ -994,9 +1280,7 @@ class _MoriEPDispatcherImplLowLatency(_MoriEPDispatcherImplBase):
self.mori_op.dispatch_recv(call_local_expert_count=record)
if record:
- get_global_expert_distribution_recorder().on_deepep_dispatch_low_latency(
- self.mori_op.local_expert_count
- )
+ _record_local_expert_count(self.mori_op.local_expert_count)
return MoriEPLLDispatchOutput(
hidden_states=hidden_states,
@@ -1007,6 +1291,7 @@ class _MoriEPDispatcherImplLowLatency(_MoriEPDispatcherImplBase):
origin_topk_ids=topk_ids,
origin_topk_weights=topk_weights,
out_dtype=output_dtype,
+ recv_cap=self._recv_cap,
)
def _dispatch_core(
@@ -1050,9 +1335,7 @@ class _MoriEPDispatcherImplLowLatency(_MoriEPDispatcherImplBase):
return hidden_states, topk_ids, topk_weights, overlap_args
def combine_b(self, hidden_states, topk_ids, topk_weights, previous_event):
-
self.mori_op.combine_recv()
-
return hidden_states[0]
def _combine_core(
@@ -1068,11 +1351,253 @@ class _MoriEPDispatcherImplLowLatency(_MoriEPDispatcherImplBase):
return combined_hidden_states
- def set_quant_config(self, quant_config: dict):
+
+class _MoriEPv2DispatcherImplNormal(_MoriEPDispatcherImplBase):
+ _is_epv2 = True
+
+ def __init__(self, async_finish: bool, **kwargs):
+ super().__init__(**kwargs)
+ if (
+ not self.deepep_mode.enable_normal()
+ or self.deepep_mode.enable_low_latency()
+ ):
+ raise ValueError("MORI EPv2 currently supports normal mode only")
+ try:
+ from mori.cco import Communicator # noqa: F401
+ from mori.ops.dispatch_combine_v2 import EpDispatchCombineOp # noqa: F401
+ except ImportError as exc:
+ raise ImportError(
+ "MORI EPv2 requires a MORI build with cco and dispatch_combine_v2"
+ ) from exc
+
+ # EPv2 forwards exactly top-k routed experts and has no fake expert slot.
+ os.environ.setdefault("AITER_FLYDSL_EP_NO_FAKE_EXPERT", "1")
+ self.async_finish = async_finish
+ # Same dual-stream setup as EPv1 normal; dispatch_b/combine_b order the
+ # comm stream with the events async_finish records.
+ if self._tbo_enabled and async_finish:
+ self._comm_stream = CommStreamPool.get_stream_from_pool(self.group)
+ self._launch_config = _get_epv2_launch_config(self._tbo_enabled)
+ self._mori_op = None
+ self.fp8_quant_func = None
+ self.fp4_quant_func = None
+ self.mxfp8_quant_func = None
+ if _use_aiter:
+ self.fp8_quant_func = get_hip_quant(QuantType.per_1x128)
+ self.fp4_quant_func = get_hip_quant(QuantType.per_1x32)
+ self.mxfp8_quant_func = get_hip_quant(QuantType.per_1x32)
+
+ def set_quant_config(self, quant_config: dict) -> None:
super().set_quant_config(quant_config)
+ # All EPv2 quant funcs are bound together, only when Aiter is available.
+ if self.dispatch_dtype != DispatchDtype.bf16 and self.fp4_quant_func is None:
+ raise RuntimeError(
+ f"MORI EPv2 {self.dispatch_dtype} dispatch requires Aiter quantization"
+ )
+ # Build the op once the dtypes are final; see _initialize_op for the timing.
+ if self._mori_op is None:
+ self._initialize_op()
+
+ def _default_combine_dtype(self, weight_dtype) -> CombineDtype:
+ return CombineDtype.bf16
+
+ def _apply_combine_dtype_override(self):
+ if envs.SGLANG_MORI_COMBINE_DTYPE.is_set():
+ combine_dtype = envs.SGLANG_MORI_COMBINE_DTYPE.get().lower()
+ if combine_dtype == "fp8":
+ self.combine_dtype = CombineDtype.fp8
+ elif combine_dtype == "fp8_direct_cast":
+ self.combine_dtype = CombineDtype.fp8_direct_cast
+ elif combine_dtype == "fp4":
+ logger.warning_once(
+ "SGLANG_MORI_COMBINE_DTYPE=fp4 is not supported by MORI EPv2; "
+ "falling back to bf16."
+ )
+ elif combine_dtype not in ("", "auto", "bf16"):
+ raise ValueError(
+ "SGLANG_MORI_COMBINE_DTYPE must be auto, bf16, fp8, "
+ f"fp8_direct_cast or fp4; got {combine_dtype!r}"
+ )
+ if (
+ self.combine_dtype != CombineDtype.bf16
+ and self.dispatch_dtype != DispatchDtype.bf16
+ ):
+ # MORI EPv2 rejects a quantized combine on an asymmetric (quantized-dispatch) op.
+ logger.warning_once(
+ f"MORI EPv2 {self.combine_dtype} combine requires bf16 dispatch, "
+ f"got {self.dispatch_dtype}; falling back to bf16 combine."
+ )
+ self.combine_dtype = CombineDtype.bf16
+ if (
+ self.combine_dtype == CombineDtype.fp8_direct_cast
+ and not _mori_epv2_fp8_direct_cast_saturates()
+ ):
+ logger.warning_once(
+ "This MORI build's EPv2 fp8_direct_cast combine emits NaN instead "
+ "of saturating values past the fp8 max, which corrupts models with "
+ "large expert outputs such as DeepSeek-V4; falling back to fp8 "
+ "(blockwise) combine. Upgrade MORI to use fp8_direct_cast."
+ )
+ self.combine_dtype = CombineDtype.fp8
+
+ def _initialize_op(self):
+ # set_quant_config runs during model loading, before CUDA graph capture,
+ # so collective CCO window creation and graph-tier JIT happen safely.
+ self._mori_op = init_mori_epv2_op(
+ self.group,
+ self.router_topk,
+ self.num_experts,
+ self.num_local_experts,
+ self.hidden_size,
+ self.params_dtype,
+ self.num_max_dispatch_tokens_per_rank,
+ self.instance_id,
+ envs.SGLANG_MORI_PREALLOC_MAX_RECV_TOKENS.get(),
+ self.dispatch_dtype,
+ self.combine_dtype,
+ self._launch_config,
+ )
+ # Keep logical caps on the power-of-two ladder init_mori_epv2_op pre-compiled.
+ self._recv_cap_pow2_buckets = _supports_dynamic_recv_cap(self._mori_op)
+
+ @property
+ def mori_op(self):
+ if self._mori_op is None:
+ logger.warning_once(
+ "MORI EPv2 op is built lazily because set_quant_config was not "
+ "called: dtype env overrides are ignored and its CCO window is "
+ "allocated after KV cache sizing."
+ )
+ self._initialize_op()
+ return self._mori_op
+
+ def _get_physical_recv_rows(self) -> int:
+ return self.mori_op.cfg.effective_max_recv
+
+ def _is_recv_layout_verified(self) -> bool:
+ # Older MORI builds lack these fields; keep their full receive view.
+ return not getattr(self.mori_op.cfg, "is_internode", True) and getattr(
+ self.mori_op, "backend_name", "unknown"
+ ) in ("flydsl", "hip")
+
+ def dispatch_a(self, hidden_states: torch.Tensor, topk_output: TopKOutput):
+ if hidden_states.dtype != torch.bfloat16:
+ raise ValueError(
+ "MORI EPv2 expects BF16 activations before transport quantization; "
+ f"hidden_states.dtype={hidden_states.dtype}"
+ )
+ return super().dispatch_a(hidden_states, topk_output)
+
+ def _prepare_topk(self, topk_output):
+ return (
+ topk_output.topk_weights.to(torch.float32),
+ topk_output.topk_ids.to(torch.int32),
+ )
+
+ def dispatch_b(
+ self, hidden_states, topk_weights, topk_ids, scale, output_dtype, ready_event
+ ) -> DispatchOutput:
+ kwargs = {"return_routing": True}
+ if _supports_dynamic_recv_cap(self.mori_op):
+ # A manual cap is an exact MoE input limit, not a MORI graph tier.
+ # Keep the full transport view so arbitrary caps need no new JIT
+ # specialization during capture; AITER applies the logical slice.
+ dispatch_cap = (
+ self.mori_op.cfg.effective_max_recv
+ if self._manual_recv_cap > 0 or self._recv_cap == 0
+ else self._recv_cap
+ )
+ kwargs.update(recv_cap=dispatch_cap, clone_routing=False)
+ if self._comm_stream is None:
+ result = self.mori_op.dispatch(
+ hidden_states, topk_weights, scale, topk_ids, **kwargs
+ )
+ else:
+ compute_stream = torch.cuda.current_stream()
+ keepalive = (hidden_states, topk_weights, topk_ids, scale)
+ with torch.cuda.stream(self._comm_stream):
+ assert ready_event is not None
+ self._comm_stream.wait_event(ready_event)
+ result = self.mori_op.dispatch(
+ hidden_states, topk_weights, scale, topk_ids, **kwargs
+ )
+ done_event = torch.cuda.Event(blocking=False, interprocess=False)
+ done_event.record(self._comm_stream)
+ compute_stream.wait_event(done_event)
+ del keepalive
+ (
+ recv_hidden,
+ recv_weights,
+ recv_scales,
+ recv_indices,
+ total_recv,
+ routing,
+ ) = result
+ if recv_scales is not None:
+ # MORI forwards scales as opaque i32 dwords; restore the quantizer layout.
+ if self.dispatch_dtype == DispatchDtype.fp8:
+ recv_scales = recv_scales.view(torch.float32)[
+ :, : self.hidden_size // FP8_BLOCK_SIZE
+ ]
+ else:
+ recv_scales = recv_scales.view(torch.float8_e8m0fnu)[
+ :, : self.hidden_size // MXFP4_BLOCK_SIZE
+ ]
+ self._routing = routing
+ self._recv_topk_ids = recv_indices
+ # Count after dispatch and before combine (combine resets total_recv), on
+ # the compute stream that already waited for the dispatch.
+ if _should_record_expert_distribution():
+ _record_local_expert_count(self.mori_op.local_expert_count())
+ # Let AITER write the expert output straight into combine's staging view so
+ # combine skips its copy. TBO children own separate arenas (instance_id);
+ # combine waits on the compute event recorded after the expert writes.
+ expert_output = None
+ combine_in_view = getattr(self.mori_op, "combine_in_view", None)
+ if combine_in_view is not None:
+ expert_output = combine_in_view()[: recv_hidden.shape[0]]
+ return MoriEPNormalDispatchOutput(
+ hidden_states=recv_hidden,
+ hidden_states_scale=recv_scales,
+ topk_ids=recv_indices,
+ topk_weights=recv_weights,
+ num_recv_tokens_per_expert=total_recv,
+ origin_topk_ids=topk_ids,
+ origin_topk_weights=topk_weights,
+ out_dtype=output_dtype,
+ expert_output=expert_output,
+ recv_cap=self._recv_cap,
+ )
+
+ def combine_b(self, hidden_states, topk_ids, topk_weights, ready_event):
+ if self._comm_stream is None:
+ out, _ = self.mori_op.combine(
+ hidden_states, None, self._recv_topk_ids, routing=self._routing
+ )
+ else:
+ compute_stream = torch.cuda.current_stream()
+ keepalive = (
+ hidden_states,
+ topk_ids,
+ topk_weights,
+ self._recv_topk_ids,
+ self._routing,
+ )
+ with torch.cuda.stream(self._comm_stream):
+ assert ready_event is not None
+ self._comm_stream.wait_event(ready_event)
+ out, _ = self.mori_op.combine(
+ hidden_states, None, self._recv_topk_ids, routing=self._routing
+ )
+ done_event = torch.cuda.Event(blocking=False, interprocess=False)
+ done_event.record(self._comm_stream)
+ compute_stream.wait_event(done_event)
+ del keepalive
+ if not torch.cuda.is_current_stream_capturing():
+ del self._routing
+ return out
-@dataclass
class _Stage(Enum):
INITIAL = auto()
AFTER_DISPATCH_A = auto()
@@ -1097,10 +1622,17 @@ class MoriEPDispatcher(BaseDispatcher):
):
super().__init__()
+ self._is_epv2 = envs.SGLANG_MORI_EP_V2.get()
+ if self._is_epv2 and deepep_mode == DeepEPMode.AUTO:
+ deepep_mode = DeepEPMode.NORMAL
self.deepep_mode = deepep_mode
async_mode = self.deepep_mode.enable_low_latency()
- if get_bool_env_var("SGLANG_ROCM_USE_MULTI_STREAM") and not async_mode:
+ if (
+ not self._is_epv2
+ and get_bool_env_var("SGLANG_ROCM_USE_MULTI_STREAM")
+ and not async_mode
+ ):
logger.warning_once(
"SGLANG_ROCM_USE_MULTI_STREAM=1 is set but Mori AsyncLL is "
"not enabled (--deepep-mode=%s). The alt-stream overlap only "
@@ -1123,16 +1655,21 @@ class MoriEPDispatcher(BaseDispatcher):
instance_id=instance_id,
)
- if self.deepep_mode.enable_low_latency():
- self._low_latency_dispatcher = _MoriEPDispatcherImplLowLatency(
- **common_kwargs,
+ if self._is_epv2:
+ self._normal_dispatcher = _MoriEPv2DispatcherImplNormal(
+ async_finish=async_finish, **common_kwargs
)
+ else:
+ if self.deepep_mode.enable_low_latency():
+ self._low_latency_dispatcher = _MoriEPDispatcherImplLowLatency(
+ **common_kwargs,
+ )
- if self.deepep_mode.enable_normal():
- self._normal_dispatcher = _MoriEPDispatcherImplNormal(
- async_finish=async_finish,
- **common_kwargs,
- )
+ if self.deepep_mode.enable_normal():
+ self._normal_dispatcher = _MoriEPDispatcherImplNormal(
+ async_finish=async_finish,
+ **common_kwargs,
+ )
self._stage = _Stage.INITIAL
self._deepep_dispatch_hooks = MoriEPPDispatchHooks()
@@ -1140,7 +1677,11 @@ class MoriEPDispatcher(BaseDispatcher):
# Mori dispatch produces global topk_ids in [0, num_experts); mask out
# experts that are not local to this rank.
self.expert_mask_gpu = None
- if _use_aiter and num_experts is not None and num_local_experts is not None:
+ if (
+ (_use_aiter or self._is_epv2)
+ and num_experts is not None
+ and num_local_experts is not None
+ ):
ep_rank = get_parallel().moe_ep_rank
expert_mask = torch.zeros(
num_experts,
@@ -1156,12 +1697,10 @@ class MoriEPDispatcher(BaseDispatcher):
hidden_states: torch.Tensor,
topk_output: TopKOutput,
) -> DispatchOutput:
- self._num_tokens = hidden_states.shape[0]
self.dispatch_a(hidden_states, topk_output)
if self._deepep_dispatch_hooks is not None:
self._deepep_dispatch_hooks(self)
- ret = self.dispatch_b()
- return ret
+ return self.dispatch_b()
def dispatch_a(
self,
@@ -1169,22 +1708,23 @@ class MoriEPDispatcher(BaseDispatcher):
topk_output: TopKOutput,
):
self._update_stage(_Stage.INITIAL, _Stage.AFTER_DISPATCH_A)
- inner_state = self._get_impl().dispatch_a(
- hidden_states=hidden_states,
- topk_output=topk_output,
+ self._num_tokens = hidden_states.shape[0]
+ self._dispatch_intermediate_state = self._get_impl().dispatch_a(
+ hidden_states=hidden_states, topk_output=topk_output
)
- self._dispatch_intermediate_state = inner_state
def dispatch_b(self):
self._update_stage(_Stage.AFTER_DISPATCH_A, _Stage.AFTER_DISPATCH_B)
inner_state = self._dispatch_intermediate_state
del self._dispatch_intermediate_state
- return self._get_impl().dispatch_b(*inner_state)
+ impl = self._get_impl()
+ impl._select_recv_cap()
+ return impl.dispatch_b(*inner_state)
def combine(
self,
combine_input: CombineInput,
- ) -> Tuple:
+ ) -> torch.Tensor:
self.combine_a(combine_input)
hidden_states = self.combine_b()
return hidden_states[: self._num_tokens]
@@ -1209,6 +1749,8 @@ class MoriEPDispatcher(BaseDispatcher):
return self._get_impl().combine_b(*inner_state)
def _get_impl(self) -> _MoriEPDispatcherImplBase:
+ if self._is_epv2:
+ return self._normal_dispatcher
is_extend_in_batch = get_is_extend_in_batch()
resolved_deepep_mode = self.deepep_mode.resolve(is_extend_in_batch)
if resolved_deepep_mode == DeepEPMode.NORMAL:
diff --git a/python/sglang/srt/layers/moe/utils.py b/python/sglang/srt/layers/moe/utils.py
index 8eb3efe550..989994a946 100644
--- a/python/sglang/srt/layers/moe/utils.py
+++ b/python/sglang/srt/layers/moe/utils.py
@@ -80,6 +80,9 @@ class MoeA2ABackend(Enum):
def is_mori(self):
return self == MoeA2ABackend.MORI
+ def is_mori_epv2(self):
+ return self.is_mori() and envs.SGLANG_MORI_EP_V2.get()
+
def is_megamoe(self):
return self == MoeA2ABackend.MEGAMOE
diff --git a/python/sglang/srt/layers/quantization/fp8_utils.py b/python/sglang/srt/layers/quantization/fp8_utils.py
index 92adc1b281..6fa480b3b8 100755
--- a/python/sglang/srt/layers/quantization/fp8_utils.py
+++ b/python/sglang/srt/layers/quantization/fp8_utils.py
@@ -1449,6 +1449,11 @@ def aiter_w8a8_block_fp8_linear(
# assert input_scale is None
input_2d = input.view(-1, input.shape[-1])
output_shape = [*input.shape[:-1], weight.shape[0]]
+ output_dtype = torch.bfloat16 if input_scale is not None else input.dtype
+ # DP-attention idle ranks run zero-token forwards (eager has no padded graph
+ # bucket); CK blockscale GEMM fails M == 0 with invalid configuration argument.
+ if input_2d.shape[0] == 0:
+ return input.new_empty(output_shape, dtype=output_dtype)
# dense linears converted from MXFP8 carry a rowwise-fp8 copy; its ptpc GEMM
# beats the block-fp8 GEMMs at decode-sized M
@@ -1524,15 +1529,13 @@ def aiter_w8a8_block_fp8_linear(
weight,
x_scale,
weight_scale,
- dtype=torch.bfloat16 if input_scale is not None else input.dtype,
+ dtype=output_dtype,
)
if bias is not None:
output += bias
- return output.to(
- dtype=torch.bfloat16 if input_scale is not None else input_2d.dtype
- ).view(*output_shape)
+ return output.to(dtype=output_dtype).view(*output_shape)
def triton_w8a8_block_fp8_linear(
diff --git a/python/sglang/srt/layers/quantization/mxfp4.py b/python/sglang/srt/layers/quantization/mxfp4.py
index 9c5e2fee1b..57da3162ef 100644
--- a/python/sglang/srt/layers/quantization/mxfp4.py
+++ b/python/sglang/srt/layers/quantization/mxfp4.py
@@ -633,6 +633,11 @@ class Mxfp4MoEMethod(FusedMoEMethodBase):
set_weight_attrs(w2_weight_bias, extra_weight_attrs)
def process_weights_after_loading(self, layer):
+ if get_moe_a2a_backend().is_mori() and hasattr(layer, "dispatcher"):
+ # Resolve MORI dtype overrides and build the EPv2 op before KV cache
+ # sizing. No weight_dtype keeps bf16 dispatch, the validated default here.
+ layer.dispatcher.set_quant_config({})
+
if self.use_marlin and not self.use_mega_moe:
from sglang.srt.layers.quantization.marlin_utils import (
check_moe_marlin_supports_layer,
@@ -1879,6 +1884,11 @@ class Mxfp4DynamicQuantMoEMethod(FusedMoEMethodBase):
return w, mx_scales
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
+ if get_moe_a2a_backend().is_mori() and hasattr(layer, "dispatcher"):
+ # Resolve MORI dtype overrides and build the EPv2 op before KV cache
+ # sizing. No weight_dtype keeps bf16 dispatch, the validated default here.
+ layer.dispatcher.set_quant_config({})
+
w13, w13_mx_scales = self.mxfp4_quantize(layer.w13_weight.data)
w2, w2_mx_scales = self.mxfp4_quantize(layer.w2_weight.data)
diff --git a/python/sglang/srt/layers/quantization/unquant.py b/python/sglang/srt/layers/quantization/unquant.py
index ac239656cd..a133eb5704 100644
--- a/python/sglang/srt/layers/quantization/unquant.py
+++ b/python/sglang/srt/layers/quantization/unquant.py
@@ -796,6 +796,10 @@ class UnquantizedFusedMoEMethod(FusedMoEMethodBase, BaseFusedOp):
):
layer.dispatcher.set_quant_config({"dispatcher_output_dtype": "bf16"})
+ if get_moe_a2a_backend().is_mori() and hasattr(layer, "dispatcher"):
+ # Resolve MORI dtype overrides and build the EPv2 op before KV cache sizing.
+ layer.dispatcher.set_quant_config({"weight_dtype": layer.w13_weight.dtype})
+
if self.use_flashinfer_cutlass:
from sglang.srt.layers.moe.moe_runner.flashinfer_cutlass import (
materialize_swiglu_params_for_cutlass,
diff --git a/python/sglang/srt/models/deepseek_v4.py b/python/sglang/srt/models/deepseek_v4.py
index 85788b7ebf..dd6bf85ae0 100644
--- a/python/sglang/srt/models/deepseek_v4.py
+++ b/python/sglang/srt/models/deepseek_v4.py
@@ -79,6 +79,7 @@ from sglang.srt.layers.dp_attention import (
DpPaddingMode,
_tbo_event,
attn_tp_all_gather,
+ attn_tp_all_gather_into_tensor,
attn_tp_all_reduce,
dp_gather_partial,
dp_gather_replicate,
@@ -2676,6 +2677,28 @@ def _every_row_routed(forward_batch: ForwardBatch, num_rows: int):
) = saved
+def _attn_tp_all_gather_chunks(
+ local: torch.Tensor, chunk_rows: List[int]
+) -> torch.Tensor:
+ # Tensor all_gather stays on aiter AG / pynccl under graph capture; the list
+ # all_gather falls back to ProcessGroupNCCL, which invalidates HIP capture.
+ max_rows = max(chunk_rows)
+ padded = local
+ if local.shape[0] != max_rows:
+ padded = local.new_zeros((max_rows, *local.shape[1:]))
+ padded[: local.shape[0]].copy_(local)
+ gathered = local.new_empty((len(chunk_rows) * max_rows, *local.shape[1:]))
+ attn_tp_all_gather_into_tensor(gathered, padded.contiguous())
+ if all(rows == max_rows for rows in chunk_rows):
+ return gathered
+ return torch.cat(
+ [
+ gathered[i * max_rows : i * max_rows + rows]
+ for i, rows in enumerate(chunk_rows)
+ ]
+ )
+
+
class DeepseekV4DecoderLayer(nn.Module):
def __init__(
self,
@@ -3556,9 +3579,14 @@ class DeepseekV4DecoderLayer(nn.Module):
hidden_states = hidden_states + _shared_local[:n]
if _use_tp_attn_a2a_scatter:
assert _a2a_scatter_chunks is not None
- gathered = [torch.empty_like(t) for t in _a2a_scatter_chunks]
- attn_tp_all_gather(gathered, hidden_states.contiguous())
- hidden_states = torch.cat(gathered)
+ if _is_hip:
+ hidden_states = _attn_tp_all_gather_chunks(
+ hidden_states, chunk_rows=[t.shape[0] for t in _a2a_scatter_chunks]
+ )
+ else:
+ gathered = [torch.empty_like(t) for t in _a2a_scatter_chunks]
+ attn_tp_all_gather(gathered, hidden_states.contiguous())
+ hidden_states = torch.cat(gathered)
return hidden_states
# ------------------------------------------------------------------
diff --git a/python/sglang/srt/models/deepseek_v4_dspark.py b/python/sglang/srt/models/deepseek_v4_dspark.py
index a77485cfae..318c52d9ae 100644
--- a/python/sglang/srt/models/deepseek_v4_dspark.py
+++ b/python/sglang/srt/models/deepseek_v4_dspark.py
@@ -17,6 +17,7 @@ from sglang.kernels.ops.speculative.dspark.dspark_draft_model import CommitKvPro
from sglang.srt.configs.deepseek_v4 import DeepSeekV4Config
from sglang.srt.distributed.device_communicators.vocab_gather import make_vocab_gather
from sglang.srt.environ import envs
+from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder
from sglang.srt.layers.dp_attention import is_dp_attention_enabled
from sglang.srt.layers.layernorm import RMSNorm
from sglang.srt.layers.logits_processor import LogitsProcessorOutput
@@ -951,8 +952,10 @@ class DeepseekV4ForCausalLMDSpark(nn.Module):
input_embeds = self.forward_embed(input_ids)
x = input_embeds
pre = None
- for stage in self.stages:
- x, pre = stage(positions, x, forward_batch, pre)
+ # Draft MoE layers have no recorder layer index, so keep them out of the stats.
+ with get_global_expert_distribution_recorder().disable_this_region():
+ for stage in self.stages:
+ x, pre = stage(positions, x, forward_batch, pre)
if self.hc_pre_from_prev_sublayer:
from sglang.kernels.ops.layernorm.mhc import hc_combine
diff --git a/python/sglang/srt/utils/common.py b/python/sglang/srt/utils/common.py
index 31e7e9f8a1..b935de39aa 100644
--- a/python/sglang/srt/utils/common.py
+++ b/python/sglang/srt/utils/common.py
@@ -4145,8 +4145,8 @@ def require_mlp_tp_gather(*, moe_a2a_backend=None):
return True
elif moe_a2a_backend.is_none():
return True
- elif moe_a2a_backend.is_flashinfer():
- # FlashInfer MoE A2A needs a rank-invariant, DP-synchronized per-rank
+ elif moe_a2a_backend.is_flashinfer() or moe_a2a_backend.is_mori_epv2():
+ # FlashInfer/MORI EPv2 A2A need a rank-invariant, DP-synchronized per-rank
# token count: MoeAlltoAll uses fixed-geometry buffers and the decode
# cuda-graph bucket must be identical across EP ranks, otherwise ranks
# replay different-sized graphs -> geometry mismatch -> illegal memory
@@ -4155,9 +4155,7 @@ def require_mlp_tp_gather(*, moe_a2a_backend=None):
# reuse this flag's DP-sync bookkeeping (uniform global_num_tokens +
# max-based graph bucket). See #30432 re: the misleading flag name.
return True
- elif moe_a2a_backend.is_mori() and get_bool_env_var(
- "SGLANG_MORI_RECV_BOUND", "false"
- ):
+ elif moe_a2a_backend.is_mori() and envs.SGLANG_MORI_RECV_BOUND.get():
# Same bookkeeping, for the same reason. Bounding mori's receive
# buffer means baking a fan-in size into a captured graph, and the
# fan-in depends on what the *peers* send. Without a DP-synchronized
diff --git a/test/manual/dsv4/test_mori_epv2_dispatcher.py b/test/manual/dsv4/test_mori_epv2_dispatcher.py
new file mode 100644
index 0000000000..2716bd6c67
--- /dev/null
+++ b/test/manual/dsv4/test_mori_epv2_dispatcher.py
@@ -0,0 +1,516 @@
+"""Eight-GPU identity round trip through the SGLang MORI EPv2 adapter.
+
+TBO=1 runs the FP4-asymmetric two-child TBO path instead; there TEST_CUDA_GRAPH=1
+captures and replays REPLAY_ITERS times (default 3), with each child writing its
+expert output directly into its own combine staging view.
+The identity expert checks buffer correctness, not AITER performance.
+"""
+
+import json
+import os
+from pathlib import Path
+from types import SimpleNamespace
+
+import torch
+import torch.distributed as dist
+
+import sglang.srt.layers.dp_attention as dp_attention
+import sglang.srt.layers.moe.token_dispatcher.moriep as adapter
+from sglang.srt.batch_overlap.two_batch_overlap import MaybeTboDeepEPDispatcher
+from sglang.srt.environ import envs
+from sglang.srt.layers.moe.token_dispatcher.moriep import round_logical_recv_rows
+from sglang.srt.layers.moe.topk import StandardTopKOutput
+from sglang.srt.layers.moe.utils import DeepEPMode, MoeA2ABackend
+from sglang.srt.runtime_context import get_flags
+
+
+def _fp4_lookup():
+ # e2m1 code points, low nibble first.
+ values = [0.0, 0.5, 1.0, 1.5, 2.0, 3.0, 4.0, 6.0]
+ return torch.tensor(
+ values + [-v for v in values], dtype=torch.float32, device="cuda"
+ )
+
+
+def _expert_output(dispatched, dispatch: str, fp4_lookup=None):
+ if dispatch == "bf16":
+ output = dispatched.hidden_states
+ return output[: dispatched.recv_cap] if dispatched.recv_cap > 0 else output
+
+ if dispatch == "fp4":
+ packed = dispatched.hidden_states.view(torch.uint8)
+ nibbles = torch.stack((packed & 0xF, packed >> 4), dim=-1).flatten(-2)
+ values = fp4_lookup[nibbles.long()]
+ else:
+ values = dispatched.hidden_states.float()
+ # fp8 uses fp32 scales per 128 columns; fp4 and mxfp8 use e8m0 per 32.
+ block = 128 if dispatch == "fp8" else 32
+ scales = dispatched.hidden_states_scale.float().repeat_interleave(block, dim=1)
+ output = (values * scales[:, : values.shape[1]]).to(torch.bfloat16)
+ valid_rows = (
+ torch.arange(output.shape[0], device=output.device)
+ < dispatched.num_recv_tokens_per_expert.reshape(-1)[0]
+ )
+ output = torch.where(valid_rows[:, None], output, torch.zeros_like(output))
+ return output[: dispatched.recv_cap] if dispatched.recv_cap > 0 else output
+
+
+class _Group:
+ def __init__(self, process_group):
+ self.cpu_group = process_group
+ self.world_size = dist.get_world_size(process_group)
+ self.rank_in_group = dist.get_rank(process_group)
+
+ def broadcast_object(self, obj, src=0):
+ values = [obj if self.rank_in_group == src else None]
+ dist.broadcast_object_list(values, src=src, group=self.cpu_group)
+ return values[0]
+
+
+def _expected_unique_destinations(topk_ids: torch.Tensor, experts_per_rank: int):
+ return torch.tensor(
+ [
+ len(set(row.tolist()))
+ for row in (topk_ids.cpu().to(torch.int64) // experts_per_rank)
+ ],
+ dtype=torch.float32,
+ ).view(-1, 1)
+
+
+def main():
+ dist.init_process_group("gloo")
+ rank = dist.get_rank()
+ world_size = dist.get_world_size()
+ local_rank = int(os.environ["LOCAL_RANK"])
+ assert world_size == 8
+ torch.cuda.set_device(local_rank)
+ if os.environ.get("TBO", "0") == "1":
+ _run_tbo(rank, world_size)
+
+ hidden_size = int(os.environ.get("HIDDEN", "7168"))
+ topk = int(os.environ.get("TOPK", "6"))
+ experts_per_rank = int(os.environ.get("EPR", "48"))
+ num_experts = world_size * experts_per_rank
+ tokens = int(os.environ.get("TOKENS", "64"))
+ # DISPATCH / COMBINE map to SGLANG_MORI_DISPATCH_DTYPE / SGLANG_MORI_COMBINE_DTYPE.
+ dispatch = os.environ.get(
+ "DISPATCH", "fp4" if os.environ.get("FP4", "0") == "1" else "bf16"
+ )
+ combine = os.environ.get("COMBINE", "bf16")
+ os.environ["SGLANG_MORI_DISPATCH_DTYPE"] = dispatch
+ os.environ["SGLANG_MORI_COMBINE_DTYPE"] = combine
+ fp4_enabled = dispatch == "fp4"
+ fp4_lookup = _fp4_lookup() if fp4_enabled else None
+ if os.environ.get("EMPTY_LAST_RANK", "0") == "1" and rank == world_size - 1:
+ tokens = 0
+
+ # ATTN_TP > 1 models attention TP: each DP group's TOKENS * (group + 1) rows
+ # are tensor_split over its attention-TP ranks, as the model does before MoE.
+ attn_tp_size = int(os.environ.get("ATTN_TP", "1"))
+ attn_dp_size = world_size // attn_tp_size
+ attn_dp_rank, attn_tp_rank = divmod(rank, attn_tp_size)
+ if attn_tp_size == 1:
+ sender_rows = [None] * world_size
+ dist.all_gather_object(sender_rows, tokens)
+ else:
+ sender_rows = [tokens * (group + 1) for group in range(attn_dp_size)]
+ tokens = (
+ torch.empty(sender_rows[attn_dp_rank], device="cpu")
+ .tensor_split(attn_tp_size)[attn_tp_rank]
+ .numel()
+ )
+ # STALE_ROWS models DSpark draft forwards that keep the target step's metadata.
+ stale_rows = os.environ.get("STALE_ROWS")
+ metadata_rows = [int(stale_rows)] if stale_rows else sender_rows
+ dp_attention.set_dp_buffer_len(sum(sender_rows), tokens, False, metadata_rows)
+
+ adapter.is_tbo_enabled = lambda: False
+ adapter.get_parallel = lambda: SimpleNamespace(
+ moe_ep_size=world_size,
+ moe_ep_rank=rank,
+ tp_size=world_size,
+ attn_dp_size=attn_dp_size,
+ attn_dp_rank=attn_dp_rank,
+ attn_tp_size=attn_tp_size,
+ attn_tp_rank=attn_tp_rank,
+ attn_cp_size=1,
+ moe_tp_size=1,
+ moe_dp_size=1,
+ launch_world_rank=rank,
+ )
+ group = _Group(dist.group.WORLD)
+ envs.SGLANG_MORI_EP_V2.set(True)
+ dispatcher = adapter.MoriEPDispatcher(
+ group=group,
+ router_topk=topk,
+ num_experts=num_experts,
+ num_local_experts=experts_per_rank,
+ hidden_size=hidden_size,
+ params_dtype=torch.bfloat16,
+ deepep_mode=DeepEPMode.NORMAL,
+ )
+ dispatcher.set_quant_config(
+ {"weight_dtype": (torch.float4_e2m1fn_x2 if fp4_enabled else torch.bfloat16)}
+ )
+
+ generator = torch.Generator(device="cpu").manual_seed(20260803 + rank)
+ hidden = torch.randn(
+ tokens, hidden_size, generator=generator, dtype=torch.bfloat16
+ ).cuda()
+ if os.environ.get("SKEWED", "0") == "1":
+ first_expert = int(os.environ.get("SKEW_RANK", "0")) * experts_per_rank
+ topk_ids = (
+ torch.arange(first_expert, first_expert + topk, dtype=torch.int32)
+ .repeat(tokens, 1)
+ .cuda()
+ )
+ else:
+ topk_ids = torch.randint(
+ 0,
+ num_experts,
+ (tokens, topk),
+ generator=generator,
+ dtype=torch.int32,
+ ).cuda()
+ topk_weights = torch.rand(
+ tokens, topk, generator=generator, dtype=torch.float32
+ ).cuda()
+ topk_output = StandardTopKOutput(topk_weights, topk_ids, None)
+
+ dispatched = dispatcher.dispatch(hidden, topk_output)
+ impl = dispatcher._get_impl()
+ assert impl.dispatch_dtype.name == dispatch, (impl.dispatch_dtype, dispatch)
+ # MORI builds without the saturation fix fall back to blockwise fp8.
+ expected_combine = combine
+ saturates = adapter._mori_epv2_fp8_direct_cast_saturates()
+ if combine == "fp8_direct_cast" and not saturates:
+ expected_combine = "fp8"
+ assert impl.combine_dtype.name == expected_combine, (impl.combine_dtype, combine)
+ expected_cap = 0
+ if impl._manual_recv_cap > 0:
+ expected_cap = min(impl._manual_recv_cap, impl.mori_op.cfg.effective_max_recv)
+ elif impl._trim_recv and impl._is_recv_layout_verified():
+ cluster_rows = (
+ sum(sender_rows)
+ if attn_dp_size > 1
+ else attn_tp_size * tokens + attn_tp_rank
+ )
+ if cluster_rows > 0:
+ rounded = round_logical_recv_rows(
+ cluster_rows, pow2_buckets=impl._recv_cap_pow2_buckets
+ )
+ # A bound that covers the whole receive buffer keeps the full view.
+ if rounded < impl.mori_op.cfg.effective_max_recv:
+ expected_cap = rounded
+ assert dispatched.recv_cap == expected_cap
+ recv_rows = int(dispatched.num_recv_tokens_per_expert.reshape(-1)[0].item())
+ assert recv_rows <= sum(sender_rows), (recv_rows, sender_rows)
+ assert expected_cap == 0 or recv_rows <= expected_cap, (recv_rows, expected_cap)
+ combined = dispatcher.combine(
+ (
+ _expert_output(dispatched, dispatch, fp4_lookup),
+ dispatched.topk_ids,
+ dispatched.topk_weights,
+ )
+ )
+ torch.cuda.synchronize()
+ dispatcher._get_impl().mori_op.comm.barrier()
+
+ expected = (
+ _expected_unique_destinations(topk_ids, experts_per_rank) * hidden.float().cpu()
+ ).to(torch.bfloat16)
+ # Arbitrary bounds: fp4 keeps ~1 mantissa bit, e4m3 keeps 3 (per quantization hop).
+ if fp4_enabled:
+ tolerance = 6e-1
+ elif dispatch != "bf16" or combine != "bf16":
+ tolerance = 1.5e-1
+ else:
+ tolerance = 2e-2
+ error = (combined.float().cpu() - expected.float()).abs()
+ ok = torch.allclose(
+ combined.float().cpu(), expected.float(), atol=tolerance, rtol=tolerance
+ )
+ failures = torch.tensor([not ok], dtype=torch.int32)
+ dist.all_reduce(failures)
+ if rank == 0:
+ print(
+ "# MORI-EPV2-SGLANG-IDENTITY: "
+ f"{'PASS' if failures.item() == 0 else 'FAIL'} "
+ f"tokens={tokens} hidden={hidden_size} topk={topk} "
+ f"skewed={os.environ.get('SKEWED', '0')} "
+ f"empty_last_rank={os.environ.get('EMPTY_LAST_RANK', '0')} "
+ f"attn_tp={attn_tp_size} sender_rows={sender_rows} stale_rows={stale_rows} "
+ f"recv_cap={dispatched.recv_cap}",
+ f"dispatch={dispatch} combine={combine} "
+ f"combine_mode={impl.mori_op.cfg.combine_mode}",
+ f"mean_abs_error={error.mean().item():.6f}",
+ f"max_abs_error={error.max().item() if error.numel() else 0:.6f}",
+ flush=True,
+ )
+
+ graph_replays = int(os.environ.get("GRAPH_REPLAYS", "0"))
+ if graph_replays:
+ graph = torch.cuda.CUDAGraph()
+ with torch.cuda.graph(graph):
+ graph_dispatched = dispatcher.dispatch(hidden, topk_output)
+ graph_combined = dispatcher.combine(
+ (
+ _expert_output(graph_dispatched, dispatch, fp4_lookup),
+ graph_dispatched.topk_ids,
+ graph_dispatched.topk_weights,
+ )
+ )
+ torch.cuda.synchronize()
+ dispatcher._get_impl().mori_op.comm.barrier()
+ for _ in range(graph_replays):
+ graph.replay()
+ torch.cuda.synchronize()
+ dispatcher._get_impl().mori_op.comm.barrier()
+ graph_ok = torch.allclose(
+ graph_combined.float().cpu(),
+ expected.float(),
+ atol=tolerance,
+ rtol=tolerance,
+ )
+ graph_failures = torch.tensor([not graph_ok], dtype=torch.int32)
+ dist.all_reduce(graph_failures)
+
+ eager_dispatched = dispatcher.dispatch(hidden, topk_output)
+ eager_after_graph = dispatcher.combine(
+ (
+ _expert_output(eager_dispatched, dispatch, fp4_lookup),
+ eager_dispatched.topk_ids,
+ eager_dispatched.topk_weights,
+ )
+ )
+ torch.cuda.synchronize()
+ dispatcher._get_impl().mori_op.comm.barrier()
+ eager_after_graph_ok = torch.allclose(
+ eager_after_graph.float().cpu(),
+ expected.float(),
+ atol=tolerance,
+ rtol=tolerance,
+ )
+ eager_after_graph_failures = torch.tensor(
+ [not eager_after_graph_ok], dtype=torch.int32
+ )
+ dist.all_reduce(eager_after_graph_failures)
+ failures += graph_failures + eager_after_graph_failures
+ if rank == 0:
+ print(
+ "# MORI-EPV2-SGLANG-GRAPH: "
+ f"{'PASS' if graph_failures.item() == 0 else 'FAIL'} "
+ f"replays={graph_replays}; "
+ "post_graph_eager="
+ f"{'PASS' if eager_after_graph_failures.item() == 0 else 'FAIL'}",
+ flush=True,
+ )
+
+ dispatcher._get_impl().mori_op.close()
+ dispatcher._get_impl().mori_op.comm.destroy()
+ dist.destroy_process_group()
+ raise SystemExit(int(failures.item() != 0))
+
+
+def _run_tbo(rank, world_size):
+ hidden_size, topk, experts_per_rank = 7168, 6, 48
+ num_experts = world_size * experts_per_rank
+ adapter.get_parallel = lambda: SimpleNamespace(
+ moe_ep_size=world_size,
+ moe_ep_rank=rank,
+ tp_size=world_size,
+ attn_dp_size=world_size,
+ attn_dp_rank=rank,
+ attn_tp_size=1,
+ attn_tp_rank=0,
+ attn_cp_size=1,
+ moe_tp_size=1,
+ moe_dp_size=1,
+ launch_world_rank=rank,
+ world_rank=rank,
+ )
+ group = _Group(dist.group.WORLD)
+ kwargs = dict(
+ group=group,
+ router_topk=topk,
+ num_experts=num_experts,
+ num_local_experts=experts_per_rank,
+ hidden_size=hidden_size,
+ params_dtype=torch.bfloat16,
+ async_finish=True,
+ )
+ with (
+ envs.SGLANG_MORI_EP_V2.override(True),
+ get_flags().moe.override(tbo_enabled=True, a2a_backend=MoeA2ABackend.MORI),
+ ):
+ dispatcher = MaybeTboDeepEPDispatcher(**kwargs)
+ assert len(dispatcher._inners) == 2
+ for child in dispatcher._inners:
+ child.set_quant_config({"weight_dtype": torch.float4_e2m1fn_x2})
+ children = [child._get_impl() for child in dispatcher._inners]
+ assert children[0]._comm_stream is children[1]._comm_stream
+ assert children[0]._launch_config == adapter._MoriEPv2LaunchConfig(32, 4, 48, 4)
+ assert children[0].mori_op is not children[1].mori_op
+ probe_trim = os.environ.get("PROBE_TBO_TRIM", "0") == "1"
+ if probe_trim:
+ # Validation-only probe: keep the real shared TBO stream/dispatchers,
+ # but allow each child to consume metadata snapshotted immediately
+ # before its dispatch_a. Production remains conservatively gated off.
+ for child in children:
+ child._tbo_enabled = False
+
+ fp4_lookup = _fp4_lookup()
+ failures = torch.zeros(1, dtype=torch.int32)
+ inputs = []
+ child_token_counts = (
+ (12, 0, 7, 2, 9, 0, 4, 1),
+ (0, 11, 3, 0, 5, 8, 1, 6),
+ )
+ for child_id, token_counts in enumerate(child_token_counts):
+ tokens = token_counts[rank]
+ generator = torch.Generator(device="cpu").manual_seed(
+ 20260805 + child_id * 100 + rank
+ )
+ hidden = torch.randn(
+ tokens, hidden_size, dtype=torch.bfloat16, generator=generator
+ ).cuda()
+ ids = torch.randint(
+ 0,
+ num_experts,
+ (tokens, topk),
+ dtype=torch.int32,
+ generator=generator,
+ ).cuda()
+ weights = torch.rand(
+ tokens, topk, dtype=torch.float32, generator=generator
+ ).cuda()
+ inputs.append((hidden, ids, StandardTopKOutput(weights, ids, None)))
+
+ def run_step():
+ for child_id, (hidden, _ids, topk_output) in enumerate(inputs):
+ token_counts = child_token_counts[child_id]
+ dp_attention.set_dp_buffer_len(
+ sum(token_counts), token_counts[rank], False, list(token_counts)
+ )
+ dispatcher.dispatch_a(
+ tbo_subbatch_index=child_id,
+ hidden_states=hidden,
+ topk_output=topk_output,
+ )
+ outputs = [
+ dispatcher.dispatch_b(tbo_subbatch_index=child_id) for child_id in range(2)
+ ]
+ direct_views = []
+ for child_id, (child, output) in enumerate(zip(children, outputs)):
+ if child._manual_recv_cap > 0:
+ assert output.recv_cap == min(
+ child._manual_recv_cap, child.mori_op.cfg.effective_max_recv
+ )
+ elif child._trim_recv and probe_trim and child._is_recv_layout_verified():
+ expected_cap = round_logical_recv_rows(
+ sum(child_token_counts[child_id]),
+ pow2_buckets=child._recv_cap_pow2_buckets,
+ )
+ assert output.recv_cap == expected_cap
+ else:
+ assert output.recv_cap == 0
+ expected_direct = callable(getattr(child.mori_op, "combine_in_view", None))
+ assert (output.expert_output is not None) == expected_direct
+ if output.expert_output is not None:
+ assert (
+ output.expert_output.data_ptr()
+ == child.mori_op.combine_in_view().data_ptr()
+ )
+ direct_views.append(output.expert_output.data_ptr())
+ assert len(direct_views) == len(set(direct_views)), (
+ "TBO children share an output buffer"
+ )
+ for child_id, output in enumerate(outputs):
+ expert_out = _expert_output(output, "fp4", fp4_lookup)
+ if output.expert_output is not None:
+ direct_out = output.expert_output[: expert_out.shape[0]]
+ direct_out.copy_(expert_out)
+ expert_out = direct_out
+ dispatcher.combine_a(
+ tbo_subbatch_index=child_id,
+ combine_input=(expert_out, output.topk_ids, output.topk_weights),
+ )
+ actual = [
+ dispatcher.combine_b(tbo_subbatch_index=child_id)[
+ : inputs[child_id][0].shape[0]
+ ]
+ for child_id in range(2)
+ ]
+ return actual, outputs
+
+ def check(actual, atol=0.6, rtol=0.6):
+ torch.cuda.synchronize()
+ for child_id, (hidden, ids, _topk_output) in enumerate(inputs):
+ if not torch.allclose(
+ actual[child_id].float().cpu(),
+ (
+ _expected_unique_destinations(ids, experts_per_rank)
+ * hidden.float().cpu()
+ )
+ .to(torch.bfloat16)
+ .float(),
+ atol=atol,
+ rtol=rtol,
+ ):
+ failures.add_(1)
+
+ replay_iters = int(os.environ.get("REPLAY_ITERS", "3"))
+ assert replay_iters > 0
+ use_graph = os.environ.get("TEST_CUDA_GRAPH", "0") == "1"
+ graph = None
+ # Exercise random FP4 inputs and warm up both instances before graph capture.
+ for _ in range(2 if use_graph else 1):
+ actual, outputs = run_step()
+ check(actual)
+ if use_graph:
+ graph = torch.cuda.CUDAGraph()
+ with torch.cuda.graph(graph):
+ actual, outputs = run_step()
+ graph.replay()
+ check(actual)
+ for iteration in range(replay_iters):
+ # Change values and routing while retaining the captured input addresses.
+ for child_id, (hidden, ids, _topk_output) in enumerate(inputs):
+ # Powers of two are exact in FP4; opposite signs expose child aliasing.
+ value = 2.0 ** (iteration % 4 - 3)
+ hidden.fill_(value if child_id == 0 else -2 * value)
+ ids.copy_((ids + 13) % num_experts)
+ if graph is None:
+ actual, outputs = run_step()
+ else:
+ graph.replay()
+ check(actual, atol=2e-2, rtol=2e-2)
+ graph = None
+ torch.cuda.synchronize()
+ dist.all_reduce(failures)
+ if rank == 0:
+ summary = {
+ "status": "PASS" if failures.item() == 0 else "FAIL",
+ "probe_trim": probe_trim,
+ "direct_output": [output.expert_output is not None for output in outputs],
+ "cuda_graph": use_graph,
+ "replay_iters": replay_iters,
+ "recv_caps": [output.recv_cap for output in outputs],
+ "child_sender_rows": child_token_counts,
+ "failures": failures.item(),
+ }
+ print(
+ "# MORI-EPV2-FP4-TBO: " + json.dumps(summary, sort_keys=True),
+ flush=True,
+ )
+ if result_json := os.environ.get("RESULT_JSON"):
+ Path(result_json).write_text(json.dumps(summary, indent=2) + "\n")
+ for child in children:
+ child.mori_op.close()
+ child.mori_op.comm.destroy()
+ dist.destroy_process_group()
+ raise SystemExit(int(failures.item() != 0))
+
+
+if __name__ == "__main__":
+ main()
diff --git a/test/registered/unit/eplb/test_megamoe_prefill_recorder.py b/test/registered/unit/eplb/test_megamoe_prefill_recorder.py
index 40207ec671..025c5de89a 100644
--- a/test/registered/unit/eplb/test_megamoe_prefill_recorder.py
+++ b/test/registered/unit/eplb/test_megamoe_prefill_recorder.py
@@ -8,6 +8,7 @@ from sglang.srt.eplb.eplb_manager import EPLBManager
from sglang.srt.eplb.eplb_map_record_fused import eplb_map_and_record_fused
from sglang.srt.eplb.expert_distribution import (
_ExpertDistributionRecorderReal,
+ _InfiniteBuffer,
_SelectExpertsSinglePassGatherer,
should_advance_eplb_counter,
should_record_forward_pass,
@@ -179,5 +180,17 @@ class TestMegaMoEPrefillRecorder(unittest.TestCase):
return recorder
+class TestInfiniteBuffer(unittest.TestCase):
+ def test_growth_inside_inference_mode_accepts_later_appends(self):
+ """A buffer grown under inference_mode must still accept appends outside it."""
+ buffer = _InfiniteBuffer(item_shape=(2,), dtype=torch.int32, device="cpu")
+ with torch.inference_mode():
+ for i in range(129):
+ buffer.append(torch.full((2,), i, dtype=torch.int32))
+ with torch.no_grad():
+ buffer.append(torch.full((2,), 129, dtype=torch.int32))
+ self.assertEqual(buffer.get_all()[:, 0].tolist(), list(range(130)))
+
+
if __name__ == "__main__":
unittest.main()
diff --git a/test/registered/unit/layers/moe/test_aiter_runner.py b/test/registered/unit/layers/moe/test_aiter_runner.py
index 60faa2369f..aed8b86aaa 100644
--- a/test/registered/unit/layers/moe/test_aiter_runner.py
+++ b/test/registered/unit/layers/moe/test_aiter_runner.py
@@ -5,6 +5,7 @@ import pytest
import torch
import sglang.srt.layers.moe.moe_runner.aiter as aiter_runner
+from sglang.srt.environ import envs
from sglang.srt.layers.moe.moe_runner.aiter import (
AiterMoeQuantInfo,
AiterQuantType,
@@ -115,5 +116,87 @@ def test_aiter_runner_preserves_no_combine_rank_for_empty_input(monkeypatch):
assert output.hidden_states.shape == (0, 2, 4)
+@pytest.mark.parametrize("supports_output", [False, True])
+def test_aiter_runner_uses_epv2_output_when_kernel_supports_it(
+ monkeypatch, supports_output
+):
+ def fused_moe(hidden_states, output=None, **kwargs):
+ result = hidden_states + 2
+ if output is not None:
+ output.copy_(result)
+ return output
+ return result
+
+ _install_fake_aiter(monkeypatch, fused_moe)
+ monkeypatch.setattr(
+ aiter_runner, "_aiter_fused_moe_supports_output", lambda: supports_output
+ )
+ runner_input = _runner_input()
+ runner_input.hidden_states.fill_(1)
+ runner_input.output = torch.zeros_like(runner_input.hidden_states)
+ runner = AiterRunnerCore(MoeRunnerConfig(activation="silu"))
+
+ result = runner.run(runner_input, _quant_info(), running_state={}).hidden_states
+
+ torch.testing.assert_close(result, torch.full_like(result, 3))
+ assert (result.data_ptr() == runner_input.output.data_ptr()) == supports_output
+ torch.testing.assert_close(
+ runner_input.output,
+ torch.full_like(result, 3 if supports_output else 0),
+ )
+
+
+@pytest.mark.parametrize(
+ "low_latency,recv_cap,expected_rows",
+ [(False, 0, 16), (False, 32, 16), (True, 5, 5)],
+)
+def test_mori_pre_permute_consumes_dispatcher_cap(
+ monkeypatch, low_latency, recv_cap, expected_rows
+):
+ from sglang.srt.layers.moe.token_dispatcher.moriep import (
+ MoriEPLLDispatchOutput,
+ MoriEPNormalDispatchOutput,
+ )
+
+ fake_utils = ModuleType("sglang.kernels.ops.moe.rocm_moe_utils")
+ fake_utils.upscale = None
+ fake_utils.upscale_mxfp4 = None
+ monkeypatch.setitem(sys.modules, fake_utils.__name__, fake_utils)
+ output_type = MoriEPLLDispatchOutput if low_latency else MoriEPNormalDispatchOutput
+ hidden = torch.zeros((16, 4), dtype=torch.bfloat16)
+ scales = torch.ones((16, 1))
+ ids = torch.zeros((16, 2), dtype=torch.int32)
+ weights = torch.ones((16, 2))
+ kwargs = {} if low_latency else {"expert_output": torch.empty_like(hidden)}
+ dispatched = output_type(
+ hidden_states=hidden,
+ hidden_states_scale=scales,
+ topk_ids=ids,
+ topk_weights=weights,
+ num_recv_tokens_per_expert=torch.tensor([5]),
+ origin_topk_ids=ids[:2],
+ origin_topk_weights=weights[:2],
+ out_dtype=torch.bfloat16,
+ recv_cap=recv_cap,
+ **kwargs,
+ )
+ quant_info = _quant_info(quant_type=AiterQuantType.NONE)
+ state = {}
+ # The dispatcher owns the override. The runner must not read it again.
+ with envs.SGLANG_MORI_MOE_MAX_INPUT_TOKENS.override(3):
+ result = aiter_runner._pre_permute_deepep_to_aiter(
+ dispatched, quant_info, MoeRunnerConfig(), state
+ )
+ assert result.hidden_states.shape[0] == expected_rows
+ assert result.topk_ids.shape[0] == expected_rows
+ assert result.topk_weights.shape[0] == expected_rows
+ assert result.a1_scale.shape[0] == expected_rows
+ assert result.hidden_states.data_ptr() == hidden.data_ptr()
+ assert state["aiter_combine_topk_ids"] is dispatched.origin_topk_ids
+ if not low_latency:
+ assert result.output.shape[0] == expected_rows
+ assert result.output.data_ptr() == dispatched.expert_output.data_ptr()
+
+
if __name__ == "__main__":
sys.exit(pytest.main([__file__, "-v"]))