[Perf] [AMD] dsv4: Add a separate MegaMoE decode instance (#44657)
Co-authored-by: HAI <hixiao@gmail.com>
This commit is contained in:
@@ -79,6 +79,7 @@ def validate_mega_moe_token_budget_for_model(server_args: ServerArgs) -> None:
|
||||
cfg = resolving_view(server_args)
|
||||
if cfg.moe_a2a_backend != "megamoe":
|
||||
return
|
||||
validate_flydsl_decode_capacity(server_args)
|
||||
if parse_connector_type(cfg.model_path) == ConnectorType.INSTANCE:
|
||||
return
|
||||
model_config = model_config_of(server_args)
|
||||
@@ -98,6 +99,58 @@ def mega_moe_decode_tokens_per_rank(cfg) -> int:
|
||||
return decode_max_bs * num_tokens_per_req
|
||||
|
||||
|
||||
def flydsl_decode_tokens_per_rank(cfg) -> int | None:
|
||||
"""Rows of the largest decode-step forward on one rank, or None while the
|
||||
request cap is still to be sized from memory.
|
||||
|
||||
Eager batches and decode graph buckets are both capped at the per-rank
|
||||
request pool; a request runs at most max(draft tokens, top-k) rows
|
||||
(verify and draft extend, draft decode).
|
||||
"""
|
||||
if cfg.max_running_requests is None:
|
||||
return None
|
||||
requests = cfg.max_running_requests
|
||||
if attn_dp_enabled_of(cfg):
|
||||
requests //= cfg.attn_dp_size
|
||||
tokens_per_req = 1
|
||||
if cfg.speculative_algorithm:
|
||||
tokens_per_req = max(
|
||||
cfg.speculative_num_draft_tokens or 1, cfg.speculative_eagle_topk or 1
|
||||
)
|
||||
return requests * tokens_per_req
|
||||
|
||||
|
||||
def validate_flydsl_decode_capacity(server_args: ServerArgs) -> None:
|
||||
if not (get_platform().is_hip and envs.SGLANG_AMD_USE_FLYDSL_MEGA_MOE.get()):
|
||||
return
|
||||
decode_mtpr = envs.SGLANG_AMD_FLYDSL_MEGA_MOE_DECODE_MTPR.get()
|
||||
if decode_mtpr <= 0 or decode_mtpr >= envs.SGLANG_AMD_FLYDSL_MEGA_MOE_MTPR.get():
|
||||
# Off, one merged capacity, or rejected when the capacities are built.
|
||||
return
|
||||
cfg = resolving_view(server_args)
|
||||
if cfg.disaggregation_mode == "prefill":
|
||||
return
|
||||
if cfg.attn_cp_size > 1:
|
||||
raise ValueError(
|
||||
"SGLANG_AMD_FLYDSL_MEGA_MOE_DECODE_MTPR is not supported with "
|
||||
"attention context parallelism (--attn-cp-size > 1); set it to 0."
|
||||
)
|
||||
decode_tokens = flydsl_decode_tokens_per_rank(cfg)
|
||||
if decode_tokens is None:
|
||||
raise ValueError(
|
||||
"SGLANG_AMD_FLYDSL_MEGA_MOE_DECODE_MTPR needs --max-running-requests "
|
||||
"to bound the largest decode / verify forward on one rank."
|
||||
)
|
||||
if decode_tokens > decode_mtpr:
|
||||
raise ValueError(
|
||||
f"SGLANG_AMD_FLYDSL_MEGA_MOE_DECODE_MTPR={decode_mtpr} must cover the "
|
||||
"largest decode / verify forward on one rank: max running requests "
|
||||
f"per DP rank x tokens per request = {decode_tokens}. Raise it to a "
|
||||
f"power of two of at least {decode_tokens} or lower "
|
||||
"--max-running-requests."
|
||||
)
|
||||
|
||||
|
||||
def validate_mega_moe_token_budget(server_args: ServerArgs, model_label: str) -> None:
|
||||
cfg = resolving_view(server_args)
|
||||
if cfg.moe_a2a_backend != "megamoe":
|
||||
|
||||
@@ -972,6 +972,11 @@ class Envs:
|
||||
SGLANG_USE_AITER_FP8_PER_TOKEN = EnvBool(False)
|
||||
SGLANG_AMD_USE_FLYDSL_MEGA_MOE = EnvBool(False)
|
||||
SGLANG_AMD_FLYDSL_MEGA_MOE_MTPR = EnvInt(8192)
|
||||
# Smaller MegaMoEV2 capacity for decode steps; a step with an extend on any
|
||||
# DP rank keeps SGLANG_AMD_FLYDSL_MEGA_MOE_MTPR. Must hold one rank's
|
||||
# largest decode forward (checked at startup). 0 disables it; otherwise a
|
||||
# power of two no larger than SGLANG_AMD_FLYDSL_MEGA_MOE_MTPR.
|
||||
SGLANG_AMD_FLYDSL_MEGA_MOE_DECODE_MTPR = EnvInt(0)
|
||||
SGLANG_AMD_FLYDSL_MEGA_QUANT = EnvStr("")
|
||||
SGLANG_AITER_MEGA_RANK_SYNC = EnvBool(False)
|
||||
SGLANG_AITER_MEGA_EPLB_PREFILL_ONLY = EnvBool(False)
|
||||
|
||||
@@ -4,6 +4,8 @@
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import functools
|
||||
import logging
|
||||
from contextlib import nullcontext
|
||||
from types import SimpleNamespace
|
||||
from typing import TYPE_CHECKING
|
||||
@@ -14,7 +16,10 @@ from sglang.kernels.ops.moe.fill_padded_rows import _fill_padded_rows
|
||||
from sglang.srt.environ import envs
|
||||
from sglang.srt.eplb.expert_location_dispatch import ExpertLocationDispatchInfo
|
||||
from sglang.srt.layers.attention.dsa.utils import is_dsa_enable_prefill_cp
|
||||
from sglang.srt.layers.dp_attention import get_dp_global_num_tokens
|
||||
from sglang.srt.layers.dp_attention import (
|
||||
get_dp_global_num_tokens,
|
||||
get_is_extend_in_batch,
|
||||
)
|
||||
from sglang.srt.layers.moe.utils import get_moe_a2a_backend
|
||||
from sglang.srt.model_executor.runner import get_is_capture_mode
|
||||
from sglang.srt.runtime_context import get_parallel
|
||||
@@ -23,6 +28,7 @@ if TYPE_CHECKING:
|
||||
from sglang.srt.model_executor.forward_batch_info import ForwardBatch
|
||||
from sglang.srt.models.deepseek_v2 import DeepseekV2MoE
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_MORI_SHMEM_READY = False
|
||||
_MEGA_MOE_INSTANCE: dict = {}
|
||||
@@ -54,6 +60,66 @@ def _mtpr() -> int:
|
||||
return mtpr
|
||||
|
||||
|
||||
def _decode_mtpr() -> int:
|
||||
decode_mtpr = int(envs.SGLANG_AMD_FLYDSL_MEGA_MOE_DECODE_MTPR.get())
|
||||
if decode_mtpr == 0:
|
||||
return 0
|
||||
mtpr = _mtpr()
|
||||
if decode_mtpr < 0 or decode_mtpr > mtpr or decode_mtpr & (decode_mtpr - 1):
|
||||
raise ValueError(
|
||||
f"SGLANG_AMD_FLYDSL_MEGA_MOE_DECODE_MTPR={decode_mtpr} must be 0 (off) "
|
||||
"or a power of two no larger than "
|
||||
f"SGLANG_AMD_FLYDSL_MEGA_MOE_MTPR={mtpr}"
|
||||
)
|
||||
if decode_mtpr == mtpr:
|
||||
_warn_decode_mtpr_merged(mtpr)
|
||||
return 0
|
||||
return decode_mtpr
|
||||
|
||||
|
||||
@functools.cache
|
||||
def _warn_decode_mtpr_merged(mtpr: int) -> None:
|
||||
logger.warning(
|
||||
"SGLANG_AMD_FLYDSL_MEGA_MOE_DECODE_MTPR equals "
|
||||
"SGLANG_AMD_FLYDSL_MEGA_MOE_MTPR=%d; running one MegaMoE capacity",
|
||||
mtpr,
|
||||
)
|
||||
|
||||
|
||||
def _capacities() -> tuple[int, ...]:
|
||||
decode_mtpr = _decode_mtpr()
|
||||
return (decode_mtpr, _mtpr()) if decode_mtpr else (_mtpr(),)
|
||||
|
||||
|
||||
def _is_decode_step_mode(forward_batch: ForwardBatch | None) -> bool:
|
||||
# IDLE also runs in prefill steps; _select_mtpr tells the two apart.
|
||||
if forward_batch is None:
|
||||
return False
|
||||
mode = forward_batch.forward_mode
|
||||
return (
|
||||
mode.is_decode_or_idle() or mode.is_target_verify() or mode.is_draft_extend_v2()
|
||||
)
|
||||
|
||||
|
||||
def _select_mtpr(forward_batch: ForwardBatch | None) -> int:
|
||||
"""The capacity this forward runs with; every EP rank must pick the same one.
|
||||
|
||||
Local token counts, graph buckets and graph-vs-eager can differ across
|
||||
ranks (DP counts are gathered only under require_mlp_tp_gather), so only
|
||||
the step kind decides.
|
||||
"""
|
||||
mtpr = _mtpr()
|
||||
decode_mtpr = _decode_mtpr()
|
||||
if not decode_mtpr or not _is_decode_step_mode(forward_batch):
|
||||
return mtpr
|
||||
if get_is_capture_mode():
|
||||
# Decode-step graphs replay only when no DP rank extends
|
||||
# (can_run_decode_cuda_graph is min-reduced over the group).
|
||||
return decode_mtpr
|
||||
# is_extend_in_batch is max-reduced over the DP group.
|
||||
return mtpr if get_is_extend_in_batch() else decode_mtpr
|
||||
|
||||
|
||||
def _ep_rank_world():
|
||||
parallel = get_parallel()
|
||||
return parallel.moe_ep_rank, parallel.moe_ep_size
|
||||
@@ -130,23 +196,21 @@ def _get_or_build_mega_moe(
|
||||
swiglu_limit: float,
|
||||
mtpr: int,
|
||||
):
|
||||
rank, world = _ep_rank_world()
|
||||
shape_key = (rank, world, model_dim, inter_dim, experts, topk, quant, swiglu_limit)
|
||||
mega = _MEGA_MOE_INSTANCE.get((*shape_key, mtpr))
|
||||
if mega is not None:
|
||||
return mega
|
||||
_ensure_mori_shmem()
|
||||
backend = _import_aiter_megamoe()
|
||||
rank, world = _ep_rank_world()
|
||||
key = (
|
||||
rank,
|
||||
world,
|
||||
model_dim,
|
||||
inter_dim,
|
||||
experts,
|
||||
topk,
|
||||
quant,
|
||||
swiglu_limit,
|
||||
mtpr,
|
||||
)
|
||||
mega = _MEGA_MOE_INSTANCE.get(key)
|
||||
if mega is None:
|
||||
mega = backend.op(
|
||||
# Construction synchronizes the device and barriers every EP rank, so all
|
||||
# capacities are built together on the first (eager) call, in the same order
|
||||
# on every rank: a capacity first used later could be inside graph capture.
|
||||
for capacity in _capacities():
|
||||
key = (*shape_key, capacity)
|
||||
if key in _MEGA_MOE_INSTANCE:
|
||||
continue
|
||||
_MEGA_MOE_INSTANCE[key] = backend.op(
|
||||
rank=rank,
|
||||
world_size=world,
|
||||
model_dim=model_dim,
|
||||
@@ -158,11 +222,11 @@ def _get_or_build_mega_moe(
|
||||
w1_scale=layer._mega_w1_scale,
|
||||
w2=layer._mega_w2,
|
||||
w2_scale=layer._mega_w2_scale,
|
||||
max_tok_per_rank=mtpr,
|
||||
max_tok_per_rank=capacity,
|
||||
swiglu_limit=swiglu_limit,
|
||||
)
|
||||
_MEGA_MOE_INSTANCE[key] = mega
|
||||
return mega
|
||||
logger.info("Aiter MegaMoE: built capacity max_tok_per_rank=%d", capacity)
|
||||
return _MEGA_MOE_INSTANCE[(*shape_key, mtpr)]
|
||||
|
||||
|
||||
def _swap_layer_weights(mega, layer) -> None:
|
||||
@@ -288,7 +352,7 @@ def _run_mega_routed(
|
||||
)
|
||||
topk_weights = hidden_states.new_zeros((1, topk), dtype=torch.float32)
|
||||
|
||||
selected_mtpr = _mtpr()
|
||||
selected_mtpr = _select_mtpr(forward_batch)
|
||||
x_in = x_in.contiguous()
|
||||
topk_ids = topk_ids.contiguous()
|
||||
topk_weights = topk_weights.contiguous()
|
||||
|
||||
Reference in New Issue
Block a user