[AMD][MORI] Add MORI EP V2 support (#40666)

This commit is contained in:
ChengYM
2026-10-11 20:54:13 -07:00
committed by GitHub
parent 20c49f12c9
commit 0d0be8fc98
17 changed files with 1661 additions and 431 deletions
+74 -1
View File
@@ -30,7 +30,12 @@ from sglang.srt.connector import ConnectorType
from sglang.srt.environ import envs
from sglang.srt.model_executor.cuda_graph_config import Backend, Phase, with_phase
from sglang.srt.runtime_context import attn_dp_enabled_of, get_platform
from sglang.srt.utils.common import is_sm100_supported, parse_connector_type
from sglang.srt.utils.common import (
get_bool_env_var,
is_gfx1250_supported,
is_sm100_supported,
parse_connector_type,
)
logger = logging.getLogger(__name__)
@@ -460,6 +465,7 @@ def handle_a2a_moe(server_args: Any):
)
if a2a_backend == "mori":
is_epv2 = _resolve_mori_ep_v2(server_args)
if cfg.deepep_mode == "auto":
declare_resolution(
server_args,
@@ -467,6 +473,16 @@ def handle_a2a_moe(server_args: Any):
deepep_mode="normal",
)
logger.warning("auto set deepep_mode=`normal` for MORI EP")
elif is_epv2 and cfg.deepep_mode == "low_latency":
declare_resolution(
server_args,
"_handle_a2a_moe",
deepep_mode="normal",
)
logger.warning(
"MORI EPv2 has no separate low-latency kernel; "
"using deepep_mode=`normal` for deepep_mode=`low_latency`"
)
# Check chunked prefill for mori
# Skip validation if chunked prefill is disabled (i.e., size <= 0).
@@ -682,6 +698,63 @@ def validate_flashinfer_a2a_token_budget(server_args: Any) -> None:
)
def _resolve_mori_ep_v2(server_args: Any) -> bool:
"""Whether to use MORI EPv2.
Unset prefers EPv2 and falls back to EPv1 where it is unsupported; an explicit
true keeps EPv2 regardless, and false selects EPv1.
"""
if not envs.SGLANG_MORI_EP_V2.get():
return False
reason = _mori_epv2_unsupported_reason(server_args)
if reason is None:
return True
if envs.SGLANG_MORI_EP_V2.is_set():
logger.warning(
"SGLANG_MORI_EP_V2=true is set explicitly, so MORI EPv2 is used even "
"though it does not support %s.",
reason,
)
return True
logger.warning(
"MORI EPv2 does not support %s; falling back to MORI EPv1. "
"Set SGLANG_MORI_EP_V2=true to force EPv2, or false to select EPv1.",
reason,
)
# Worker processes inherit the env, so every reader agrees on EPv1.
envs.SGLANG_MORI_EP_V2.set(False)
return False
def _mori_epv2_unsupported_reason(server_args: Any) -> str | None:
# The dispatcher falls back from unsupported dispatch/combine dtypes once it
# knows the weight dtype.
import torch
from sglang.srt.configs.model_config import is_deepseek_v4
cfg = resolved_view(server_args)
# gfx1250 forms its LSA team from the UALink vPOD, which can span hosts and is
# only known once cco is up, so init_mori_epv2_op checks that EP fits in it.
if cfg.tp_size > 8 and not is_gfx1250_supported():
return f"ep_size={cfg.tp_size} across hosts (EPv2 is validated within one host)"
if not envs.SGLANG_USE_AITER.get():
return "SGLANG_USE_AITER=0 (EPv2 experts run on the aiter MoE runner)"
if get_bool_env_var("MORI_ENABLE_SDMA", "false"):
return "MORI_ENABLE_SDMA=1 (SDMA is EPv1-only)"
if parse_connector_type(cfg.model_path) == ConnectorType.INSTANCE:
return "a model loaded through an instance connector (cannot validate it)"
model_config = model_config_of(server_args)
if model_config.dtype != torch.bfloat16:
return f"dtype={model_config.dtype} (EPv2 requires bfloat16)"
if not is_deepseek_v4(model_config.hf_config):
architecture = (model_config.hf_config.architectures or [None])[0]
return f"{architecture} (EPv2 is validated on DeepSeek-V4 only)"
return None
def required_mori_dispatch_tokens_per_rank(server_args: Any) -> int:
"""Max tokens a single rank dispatches through MoRI in one forward."""
cfg = resolving_view(server_args)
+11
View File
@@ -864,6 +864,17 @@ class Envs:
# and relies on the RDMA retry-exceeded timeout only.
SGLANG_MORI_TRANSFER_TIMEOUT_MS = EnvInt(0)
SGLANG_MORI_NUM_MAX_DISPATCH_TOKENS_PER_RANK = EnvInt(4096)
# Unset: EPv2 where moe_hook finds it supported (DeepSeek-V4 only), else EPv1.
# true forces EPv2; false forces EPv1.
SGLANG_MORI_EP_V2 = EnvBool(True)
SGLANG_MORI_DISPATCH_DTYPE = EnvStr("auto")
SGLANG_MORI_COMBINE_DTYPE = EnvStr("auto")
SGLANG_MORI_PREALLOC_MAX_RECV_TOKENS = EnvInt(0)
# Hand AITER only ceil32(sum of per-rank DP tokens) rows of the MORI EP receive
# buffer; unproved layouts or metadata keep the full view. Forces DP-synced buckets.
SGLANG_MORI_RECV_BOUND = EnvBool(False)
# Manual MoE input row limit (EPv1 and EPv2); 0 leaves it to SGLANG_MORI_RECV_BOUND.
SGLANG_MORI_MOE_MAX_INPUT_TOKENS = EnvInt(0)
# ===================================================================
# AMD, ROCm, and AITER
@@ -1092,9 +1092,14 @@ class _InfiniteBuffer(_Buffer):
device = self._buffer.device
if self._size == curr_buffer_size:
new_buffer = torch.zeros(
(2 * curr_buffer_size, *self._item_shape), dtype=dtype, device=device
)
# Growth can happen inside a caller's inference_mode (e.g. DSpark draft);
# an inference tensor buffer would reject later appends made outside it.
with torch.inference_mode(False):
new_buffer = torch.zeros(
(2 * curr_buffer_size, *self._item_shape),
dtype=dtype,
device=device,
)
new_buffer[:curr_buffer_size] = self._buffer
self._buffer = new_buffer
@@ -352,6 +352,8 @@ class DeepEPMoE(FusedMoE):
def get_moe_impl_class(quant_config: Optional[QuantizationConfig]):
if get_moe_a2a_backend().is_mori_epv2():
return FusedMoE
# [TODO] kk, temporary solution
if (
get_moe_a2a_backend().is_mori()
+28 -102
View File
@@ -23,8 +23,7 @@ from sglang.srt.layers.moe.moe_runner.base import (
register_pre_permute,
)
from sglang.srt.layers.moe.utils import MoeRunnerBackend
from sglang.srt.runtime_context import get_parallel
from sglang.srt.utils import get_bool_env_var, get_int_env_var, is_gfx95_supported
from sglang.srt.utils import get_bool_env_var, is_gfx95_supported
if TYPE_CHECKING:
from sglang.srt.layers.moe.token_dispatcher.base import CombineInput
@@ -87,6 +86,8 @@ class AiterRunnerInput(RunnerInput):
# Mori-only fused_moe kwargs.
num_local_tokens: Optional[torch.Tensor] = None
output_dtype: Optional[torch.dtype] = None
# Optional MORI EPv2 combine staging view for fused_moe output.
output: Optional[torch.Tensor] = None
@property
def runner_backend(self) -> MoeRunnerBackend:
@@ -156,95 +157,11 @@ def _aiter_fused_moe_supports_no_combine() -> bool:
return "no_combine" in inspect.signature(fused_moe).parameters
_RECV_BOUND_LOGGED: set[int] = set()
_RECV_BOUND_WARNED = False
@functools.cache
def _aiter_fused_moe_supports_output() -> bool:
from aiter.fused_moe import fused_moe
def _warn_recv_bound_unavailable() -> None:
global _RECV_BOUND_WARNED
if not _RECV_BOUND_WARNED:
_RECV_BOUND_WARNED = True
logger.warning(
"SGLANG_MORI_RECV_BOUND is set but the per-rank DP token counts do "
"not cover every mori sender, so the receive fan-in is unknown; "
"leaving the receive buffer unbounded."
)
def _mori_decode_recv_bound(recv_rows: int, topk: int) -> int:
"""Live rows mori's receive buffer can hold in decode, or 0 for "do not bound".
Worst case fan-in is every rank routing all of its tokens to this one, so
`sum(per-rank tokens) * topk`, where topk already includes the fused shared
expert. The per-rank counts come from the DP sync, so this is the fan-in for
the batch actually being run rather than an upper bound over all batches.
That is only sound because enabling this gate also makes
`require_mlp_tp_gather()` true for mori, which gives every rank the same
cuda-graph bucket. The value is baked into a captured graph and has to hold
for every later replay; with per-rank buckets a rank on a narrow tier could
be handed rows by a peer on a wider one, and the only bound valid under that
is the widest tier's -- 4-16x looser than the batch being run, which costs
more in expert-GEMM tiles (M 32/64 -> 128) than the trim saves.
Two cases stay unbounded, because a bound below the real fan-in silently
drops rows from the all-to-all -- wrong output rather than an error:
* Prefill, whose per-rank counts are uneven and not knowable here.
* Anything that leaves the per-rank counts unpopulated, or where the EP world
is wider than the DP world so the counts do not cover every sender.
"""
if not get_bool_env_var("SGLANG_MORI_RECV_BOUND", "false"):
return 0
from sglang.srt.layers.dp_attention import (
get_dp_global_num_tokens,
get_is_extend_in_batch,
)
if get_is_extend_in_batch():
return 0
per_rank_tokens = get_dp_global_num_tokens()
ep_size = get_parallel().moe_ep_size
if not per_rank_tokens or len(per_rank_tokens) < ep_size:
# Either the DP sync did not publish counts, or they do not cover every
# mori sender. Both mean the fan-in is unknown here.
_warn_recv_bound_unavailable()
return 0
max_tokens = sum(per_rank_tokens)
bound = max_tokens * topk
# Never grow the tensor, and nothing to do when there is nothing to trim.
if not 0 < bound < recv_rows:
return 0
# One INFO line the first time it engages, so an inert bound is not mistaken
# for an active one in the results. Per-tier values go to DEBUG: capture
# visits every tier, and at INFO on every rank that is dozens of lines.
if get_parallel().tp_rank == 0 and bound not in _RECV_BOUND_LOGGED:
first = not _RECV_BOUND_LOGGED
_RECV_BOUND_LOGGED.add(bound)
if first:
logger.info(
"mori recv bound active: %d rows -> %d for this tier "
"(dp_tokens=%d ep=%d topk=%d); per-tier values at DEBUG",
recv_rows,
bound,
max_tokens,
get_parallel().moe_ep_size,
topk,
)
else:
logger.debug(
"mori recv bound: %d rows -> %d (dp_tokens=%d ep=%d topk=%d)",
recv_rows,
bound,
max_tokens,
get_parallel().moe_ep_size,
topk,
)
return bound
return "output" in inspect.signature(fused_moe).parameters
class AiterRunnerCore(MoeRunnerCore):
@@ -283,8 +200,6 @@ class AiterRunnerCore(MoeRunnerCore):
from aiter.fused_moe import fused_moe
from sglang.srt.environ import envs
a1_scale = (
runner_input.a1_scale
if runner_input.a1_scale is not None
@@ -330,6 +245,15 @@ class AiterRunnerCore(MoeRunnerCore):
extra["swiglu_limit"] = quant_info.swiglu_limit
if self.config.no_combine:
extra["no_combine"] = True
elif (
runner_input.output is not None
and "output" not in extra
and extra.get("stage2_scatter") is None
and _aiter_fused_moe_supports_output()
):
# MORI EPv2 skips staging when this exact arena view is returned. Older
# AITER versions, no_combine and stage2_scatter keep their old path.
extra["output"] = runner_input.output
extra["moe_sorting_dispatch_policy"] = _MOE_SORTING_DISPATCH_POLICY
# gfx950 small-M MXFP4 kernel (on by default, SGLANG_ROCM_SMALLM_MOE=0 disables): same layouts as aiter, bf16 activations.
@@ -432,8 +356,8 @@ def pre_permute_standard_to_aiter(
def _is_mori_dispatch_output(dispatch_output: Any) -> bool:
# MoriEP{Normal,LL}DispatchOutput carry the post-mori-permute origin_topk_*
# tensors that the standard DeepEP outputs lack.
# MORI EP dispatch outputs carry the origin_topk_* tensors for combine,
# which the standard DeepEP outputs lack.
return hasattr(dispatch_output, "origin_topk_ids")
@@ -486,6 +410,7 @@ def _pre_permute_deepep_to_aiter(
a1_scale: Optional[torch.Tensor] = None
num_local_tokens: Optional[torch.Tensor] = None
output_dtype: Optional[torch.dtype] = None
output: Optional[torch.Tensor] = None
quant_type = quant_info.quant_type
if is_mori:
@@ -494,21 +419,21 @@ def _pre_permute_deepep_to_aiter(
a1_scale = dispatch_output.hidden_states_scale
num_local_tokens = dispatch_output.num_recv_tokens_per_expert
output_dtype = dispatch_output.out_dtype
output = getattr(dispatch_output, "expert_output", None)
mori_max = dispatch_output.recv_cap
# Truncate dispatch tensors to the configured cap; mori combine only
# Truncate dispatch tensors to the configured cap; MORI EP combine only
# reads [0, totalRecvTokenNum), so the truncated result needs no
# padding back.
mori_max = get_int_env_var("SGLANG_MORI_MOE_MAX_INPUT_TOKENS", 0)
if mori_max <= 0:
mori_max = _mori_decode_recv_bound(
hidden_states.shape[0], topk_ids.shape[-1]
)
if mori_max > 0:
# padding back. Slice the optional EPv2 output to the same row count,
# retaining the physical arena and routing stride.
if 0 < mori_max < hidden_states.shape[0]:
hidden_states = hidden_states[:mori_max]
if a1_scale is not None:
a1_scale = a1_scale[:mori_max]
topk_ids = topk_ids[:mori_max]
topk_weights = topk_weights[:mori_max]
if output is not None:
output = output[:mori_max]
# Upscale dispatched activations when there is no AITER kernel for the
# weight/activation dtype pair.
@@ -595,6 +520,7 @@ def _pre_permute_deepep_to_aiter(
a1_scale=a1_scale,
num_local_tokens=num_local_tokens,
output_dtype=output_dtype,
output=output,
)
File diff suppressed because it is too large Load Diff
+3
View File
@@ -80,6 +80,9 @@ class MoeA2ABackend(Enum):
def is_mori(self):
return self == MoeA2ABackend.MORI
def is_mori_epv2(self):
return self.is_mori() and envs.SGLANG_MORI_EP_V2.get()
def is_megamoe(self):
return self == MoeA2ABackend.MEGAMOE
@@ -1449,6 +1449,11 @@ def aiter_w8a8_block_fp8_linear(
# assert input_scale is None
input_2d = input.view(-1, input.shape[-1])
output_shape = [*input.shape[:-1], weight.shape[0]]
output_dtype = torch.bfloat16 if input_scale is not None else input.dtype
# DP-attention idle ranks run zero-token forwards (eager has no padded graph
# bucket); CK blockscale GEMM fails M == 0 with invalid configuration argument.
if input_2d.shape[0] == 0:
return input.new_empty(output_shape, dtype=output_dtype)
# dense linears converted from MXFP8 carry a rowwise-fp8 copy; its ptpc GEMM
# beats the block-fp8 GEMMs at decode-sized M
@@ -1524,15 +1529,13 @@ def aiter_w8a8_block_fp8_linear(
weight,
x_scale,
weight_scale,
dtype=torch.bfloat16 if input_scale is not None else input.dtype,
dtype=output_dtype,
)
if bias is not None:
output += bias
return output.to(
dtype=torch.bfloat16 if input_scale is not None else input_2d.dtype
).view(*output_shape)
return output.to(dtype=output_dtype).view(*output_shape)
def triton_w8a8_block_fp8_linear(
@@ -633,6 +633,11 @@ class Mxfp4MoEMethod(FusedMoEMethodBase):
set_weight_attrs(w2_weight_bias, extra_weight_attrs)
def process_weights_after_loading(self, layer):
if get_moe_a2a_backend().is_mori() and hasattr(layer, "dispatcher"):
# Resolve MORI dtype overrides and build the EPv2 op before KV cache
# sizing. No weight_dtype keeps bf16 dispatch, the validated default here.
layer.dispatcher.set_quant_config({})
if self.use_marlin and not self.use_mega_moe:
from sglang.srt.layers.quantization.marlin_utils import (
check_moe_marlin_supports_layer,
@@ -1879,6 +1884,11 @@ class Mxfp4DynamicQuantMoEMethod(FusedMoEMethodBase):
return w, mx_scales
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
if get_moe_a2a_backend().is_mori() and hasattr(layer, "dispatcher"):
# Resolve MORI dtype overrides and build the EPv2 op before KV cache
# sizing. No weight_dtype keeps bf16 dispatch, the validated default here.
layer.dispatcher.set_quant_config({})
w13, w13_mx_scales = self.mxfp4_quantize(layer.w13_weight.data)
w2, w2_mx_scales = self.mxfp4_quantize(layer.w2_weight.data)
@@ -796,6 +796,10 @@ class UnquantizedFusedMoEMethod(FusedMoEMethodBase, BaseFusedOp):
):
layer.dispatcher.set_quant_config({"dispatcher_output_dtype": "bf16"})
if get_moe_a2a_backend().is_mori() and hasattr(layer, "dispatcher"):
# Resolve MORI dtype overrides and build the EPv2 op before KV cache sizing.
layer.dispatcher.set_quant_config({"weight_dtype": layer.w13_weight.dtype})
if self.use_flashinfer_cutlass:
from sglang.srt.layers.moe.moe_runner.flashinfer_cutlass import (
materialize_swiglu_params_for_cutlass,
+31 -3
View File
@@ -79,6 +79,7 @@ from sglang.srt.layers.dp_attention import (
DpPaddingMode,
_tbo_event,
attn_tp_all_gather,
attn_tp_all_gather_into_tensor,
attn_tp_all_reduce,
dp_gather_partial,
dp_gather_replicate,
@@ -2676,6 +2677,28 @@ def _every_row_routed(forward_batch: ForwardBatch, num_rows: int):
) = saved
def _attn_tp_all_gather_chunks(
local: torch.Tensor, chunk_rows: List[int]
) -> torch.Tensor:
# Tensor all_gather stays on aiter AG / pynccl under graph capture; the list
# all_gather falls back to ProcessGroupNCCL, which invalidates HIP capture.
max_rows = max(chunk_rows)
padded = local
if local.shape[0] != max_rows:
padded = local.new_zeros((max_rows, *local.shape[1:]))
padded[: local.shape[0]].copy_(local)
gathered = local.new_empty((len(chunk_rows) * max_rows, *local.shape[1:]))
attn_tp_all_gather_into_tensor(gathered, padded.contiguous())
if all(rows == max_rows for rows in chunk_rows):
return gathered
return torch.cat(
[
gathered[i * max_rows : i * max_rows + rows]
for i, rows in enumerate(chunk_rows)
]
)
class DeepseekV4DecoderLayer(nn.Module):
def __init__(
self,
@@ -3556,9 +3579,14 @@ class DeepseekV4DecoderLayer(nn.Module):
hidden_states = hidden_states + _shared_local[:n]
if _use_tp_attn_a2a_scatter:
assert _a2a_scatter_chunks is not None
gathered = [torch.empty_like(t) for t in _a2a_scatter_chunks]
attn_tp_all_gather(gathered, hidden_states.contiguous())
hidden_states = torch.cat(gathered)
if _is_hip:
hidden_states = _attn_tp_all_gather_chunks(
hidden_states, chunk_rows=[t.shape[0] for t in _a2a_scatter_chunks]
)
else:
gathered = [torch.empty_like(t) for t in _a2a_scatter_chunks]
attn_tp_all_gather(gathered, hidden_states.contiguous())
hidden_states = torch.cat(gathered)
return hidden_states
# ------------------------------------------------------------------
@@ -17,6 +17,7 @@ from sglang.kernels.ops.speculative.dspark.dspark_draft_model import CommitKvPro
from sglang.srt.configs.deepseek_v4 import DeepSeekV4Config
from sglang.srt.distributed.device_communicators.vocab_gather import make_vocab_gather
from sglang.srt.environ import envs
from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder
from sglang.srt.layers.dp_attention import is_dp_attention_enabled
from sglang.srt.layers.layernorm import RMSNorm
from sglang.srt.layers.logits_processor import LogitsProcessorOutput
@@ -951,8 +952,10 @@ class DeepseekV4ForCausalLMDSpark(nn.Module):
input_embeds = self.forward_embed(input_ids)
x = input_embeds
pre = None
for stage in self.stages:
x, pre = stage(positions, x, forward_batch, pre)
# Draft MoE layers have no recorder layer index, so keep them out of the stats.
with get_global_expert_distribution_recorder().disable_this_region():
for stage in self.stages:
x, pre = stage(positions, x, forward_batch, pre)
if self.hc_pre_from_prev_sublayer:
from sglang.kernels.ops.layernorm.mhc import hc_combine
+3 -5
View File
@@ -4145,8 +4145,8 @@ def require_mlp_tp_gather(*, moe_a2a_backend=None):
return True
elif moe_a2a_backend.is_none():
return True
elif moe_a2a_backend.is_flashinfer():
# FlashInfer MoE A2A needs a rank-invariant, DP-synchronized per-rank
elif moe_a2a_backend.is_flashinfer() or moe_a2a_backend.is_mori_epv2():
# FlashInfer/MORI EPv2 A2A need a rank-invariant, DP-synchronized per-rank
# token count: MoeAlltoAll uses fixed-geometry buffers and the decode
# cuda-graph bucket must be identical across EP ranks, otherwise ranks
# replay different-sized graphs -> geometry mismatch -> illegal memory
@@ -4155,9 +4155,7 @@ def require_mlp_tp_gather(*, moe_a2a_backend=None):
# reuse this flag's DP-sync bookkeeping (uniform global_num_tokens +
# max-based graph bucket). See #30432 re: the misleading flag name.
return True
elif moe_a2a_backend.is_mori() and get_bool_env_var(
"SGLANG_MORI_RECV_BOUND", "false"
):
elif moe_a2a_backend.is_mori() and envs.SGLANG_MORI_RECV_BOUND.get():
# Same bookkeeping, for the same reason. Bounding mori's receive
# buffer means baking a fan-in size into a captured graph, and the
# fan-in depends on what the *peers* send. Without a DP-synchronized