[Perf] [AMD] dsv4: Add a separate MegaMoE decode instance (#44657)

Co-authored-by: HAI <hixiao@gmail.com>
This commit is contained in:
Xinyi Song
2026-10-16 03:11:02 -07:00
committed by GitHub
co-authored by HAI
parent 2adc76ad61
commit 4afae5568b
4 changed files with 530 additions and 20 deletions
@@ -79,6 +79,7 @@ def validate_mega_moe_token_budget_for_model(server_args: ServerArgs) -> None:
cfg = resolving_view(server_args)
if cfg.moe_a2a_backend != "megamoe":
return
validate_flydsl_decode_capacity(server_args)
if parse_connector_type(cfg.model_path) == ConnectorType.INSTANCE:
return
model_config = model_config_of(server_args)
@@ -98,6 +99,58 @@ def mega_moe_decode_tokens_per_rank(cfg) -> int:
return decode_max_bs * num_tokens_per_req
def flydsl_decode_tokens_per_rank(cfg) -> int | None:
"""Rows of the largest decode-step forward on one rank, or None while the
request cap is still to be sized from memory.
Eager batches and decode graph buckets are both capped at the per-rank
request pool; a request runs at most max(draft tokens, top-k) rows
(verify and draft extend, draft decode).
"""
if cfg.max_running_requests is None:
return None
requests = cfg.max_running_requests
if attn_dp_enabled_of(cfg):
requests //= cfg.attn_dp_size
tokens_per_req = 1
if cfg.speculative_algorithm:
tokens_per_req = max(
cfg.speculative_num_draft_tokens or 1, cfg.speculative_eagle_topk or 1
)
return requests * tokens_per_req
def validate_flydsl_decode_capacity(server_args: ServerArgs) -> None:
if not (get_platform().is_hip and envs.SGLANG_AMD_USE_FLYDSL_MEGA_MOE.get()):
return
decode_mtpr = envs.SGLANG_AMD_FLYDSL_MEGA_MOE_DECODE_MTPR.get()
if decode_mtpr <= 0 or decode_mtpr >= envs.SGLANG_AMD_FLYDSL_MEGA_MOE_MTPR.get():
# Off, one merged capacity, or rejected when the capacities are built.
return
cfg = resolving_view(server_args)
if cfg.disaggregation_mode == "prefill":
return
if cfg.attn_cp_size > 1:
raise ValueError(
"SGLANG_AMD_FLYDSL_MEGA_MOE_DECODE_MTPR is not supported with "
"attention context parallelism (--attn-cp-size > 1); set it to 0."
)
decode_tokens = flydsl_decode_tokens_per_rank(cfg)
if decode_tokens is None:
raise ValueError(
"SGLANG_AMD_FLYDSL_MEGA_MOE_DECODE_MTPR needs --max-running-requests "
"to bound the largest decode / verify forward on one rank."
)
if decode_tokens > decode_mtpr:
raise ValueError(
f"SGLANG_AMD_FLYDSL_MEGA_MOE_DECODE_MTPR={decode_mtpr} must cover the "
"largest decode / verify forward on one rank: max running requests "
f"per DP rank x tokens per request = {decode_tokens}. Raise it to a "
f"power of two of at least {decode_tokens} or lower "
"--max-running-requests."
)
def validate_mega_moe_token_budget(server_args: ServerArgs, model_label: str) -> None:
cfg = resolving_view(server_args)
if cfg.moe_a2a_backend != "megamoe":
+5
View File
@@ -972,6 +972,11 @@ class Envs:
SGLANG_USE_AITER_FP8_PER_TOKEN = EnvBool(False)
SGLANG_AMD_USE_FLYDSL_MEGA_MOE = EnvBool(False)
SGLANG_AMD_FLYDSL_MEGA_MOE_MTPR = EnvInt(8192)
# Smaller MegaMoEV2 capacity for decode steps; a step with an extend on any
# DP rank keeps SGLANG_AMD_FLYDSL_MEGA_MOE_MTPR. Must hold one rank's
# largest decode forward (checked at startup). 0 disables it; otherwise a
# power of two no larger than SGLANG_AMD_FLYDSL_MEGA_MOE_MTPR.
SGLANG_AMD_FLYDSL_MEGA_MOE_DECODE_MTPR = EnvInt(0)
SGLANG_AMD_FLYDSL_MEGA_QUANT = EnvStr("")
SGLANG_AITER_MEGA_RANK_SYNC = EnvBool(False)
SGLANG_AITER_MEGA_EPLB_PREFILL_ONLY = EnvBool(False)
+84 -20
View File
@@ -4,6 +4,8 @@
from __future__ import annotations
import functools
import logging
from contextlib import nullcontext
from types import SimpleNamespace
from typing import TYPE_CHECKING
@@ -14,7 +16,10 @@ from sglang.kernels.ops.moe.fill_padded_rows import _fill_padded_rows
from sglang.srt.environ import envs
from sglang.srt.eplb.expert_location_dispatch import ExpertLocationDispatchInfo
from sglang.srt.layers.attention.dsa.utils import is_dsa_enable_prefill_cp
from sglang.srt.layers.dp_attention import get_dp_global_num_tokens
from sglang.srt.layers.dp_attention import (
get_dp_global_num_tokens,
get_is_extend_in_batch,
)
from sglang.srt.layers.moe.utils import get_moe_a2a_backend
from sglang.srt.model_executor.runner import get_is_capture_mode
from sglang.srt.runtime_context import get_parallel
@@ -23,6 +28,7 @@ if TYPE_CHECKING:
from sglang.srt.model_executor.forward_batch_info import ForwardBatch
from sglang.srt.models.deepseek_v2 import DeepseekV2MoE
logger = logging.getLogger(__name__)
_MORI_SHMEM_READY = False
_MEGA_MOE_INSTANCE: dict = {}
@@ -54,6 +60,66 @@ def _mtpr() -> int:
return mtpr
def _decode_mtpr() -> int:
decode_mtpr = int(envs.SGLANG_AMD_FLYDSL_MEGA_MOE_DECODE_MTPR.get())
if decode_mtpr == 0:
return 0
mtpr = _mtpr()
if decode_mtpr < 0 or decode_mtpr > mtpr or decode_mtpr & (decode_mtpr - 1):
raise ValueError(
f"SGLANG_AMD_FLYDSL_MEGA_MOE_DECODE_MTPR={decode_mtpr} must be 0 (off) "
"or a power of two no larger than "
f"SGLANG_AMD_FLYDSL_MEGA_MOE_MTPR={mtpr}"
)
if decode_mtpr == mtpr:
_warn_decode_mtpr_merged(mtpr)
return 0
return decode_mtpr
@functools.cache
def _warn_decode_mtpr_merged(mtpr: int) -> None:
logger.warning(
"SGLANG_AMD_FLYDSL_MEGA_MOE_DECODE_MTPR equals "
"SGLANG_AMD_FLYDSL_MEGA_MOE_MTPR=%d; running one MegaMoE capacity",
mtpr,
)
def _capacities() -> tuple[int, ...]:
decode_mtpr = _decode_mtpr()
return (decode_mtpr, _mtpr()) if decode_mtpr else (_mtpr(),)
def _is_decode_step_mode(forward_batch: ForwardBatch | None) -> bool:
# IDLE also runs in prefill steps; _select_mtpr tells the two apart.
if forward_batch is None:
return False
mode = forward_batch.forward_mode
return (
mode.is_decode_or_idle() or mode.is_target_verify() or mode.is_draft_extend_v2()
)
def _select_mtpr(forward_batch: ForwardBatch | None) -> int:
"""The capacity this forward runs with; every EP rank must pick the same one.
Local token counts, graph buckets and graph-vs-eager can differ across
ranks (DP counts are gathered only under require_mlp_tp_gather), so only
the step kind decides.
"""
mtpr = _mtpr()
decode_mtpr = _decode_mtpr()
if not decode_mtpr or not _is_decode_step_mode(forward_batch):
return mtpr
if get_is_capture_mode():
# Decode-step graphs replay only when no DP rank extends
# (can_run_decode_cuda_graph is min-reduced over the group).
return decode_mtpr
# is_extend_in_batch is max-reduced over the DP group.
return mtpr if get_is_extend_in_batch() else decode_mtpr
def _ep_rank_world():
parallel = get_parallel()
return parallel.moe_ep_rank, parallel.moe_ep_size
@@ -130,23 +196,21 @@ def _get_or_build_mega_moe(
swiglu_limit: float,
mtpr: int,
):
rank, world = _ep_rank_world()
shape_key = (rank, world, model_dim, inter_dim, experts, topk, quant, swiglu_limit)
mega = _MEGA_MOE_INSTANCE.get((*shape_key, mtpr))
if mega is not None:
return mega
_ensure_mori_shmem()
backend = _import_aiter_megamoe()
rank, world = _ep_rank_world()
key = (
rank,
world,
model_dim,
inter_dim,
experts,
topk,
quant,
swiglu_limit,
mtpr,
)
mega = _MEGA_MOE_INSTANCE.get(key)
if mega is None:
mega = backend.op(
# Construction synchronizes the device and barriers every EP rank, so all
# capacities are built together on the first (eager) call, in the same order
# on every rank: a capacity first used later could be inside graph capture.
for capacity in _capacities():
key = (*shape_key, capacity)
if key in _MEGA_MOE_INSTANCE:
continue
_MEGA_MOE_INSTANCE[key] = backend.op(
rank=rank,
world_size=world,
model_dim=model_dim,
@@ -158,11 +222,11 @@ def _get_or_build_mega_moe(
w1_scale=layer._mega_w1_scale,
w2=layer._mega_w2,
w2_scale=layer._mega_w2_scale,
max_tok_per_rank=mtpr,
max_tok_per_rank=capacity,
swiglu_limit=swiglu_limit,
)
_MEGA_MOE_INSTANCE[key] = mega
return mega
logger.info("Aiter MegaMoE: built capacity max_tok_per_rank=%d", capacity)
return _MEGA_MOE_INSTANCE[(*shape_key, mtpr)]
def _swap_layer_weights(mega, layer) -> None:
@@ -288,7 +352,7 @@ def _run_mega_routed(
)
topk_weights = hidden_states.new_zeros((1, topk), dtype=torch.float32)
selected_mtpr = _mtpr()
selected_mtpr = _select_mtpr(forward_batch)
x_in = x_in.contiguous()
topk_ids = topk_ids.contiguous()
topk_weights = topk_weights.contiguous()