From 4afae5568bdc72487db44f40f619780b1ba0efee Mon Sep 17 00:00:00 2001 From: Xinyi Song Date: Fri, 16 Oct 2026 03:11:02 -0700 Subject: [PATCH] [Perf] [AMD] dsv4: Add a separate MegaMoE decode instance (#44657) Co-authored-by: HAI --- python/sglang/srt/arg_groups/mega_moe_hook.py | 53 +++ python/sglang/srt/environ.py | 5 + .../sglang/srt/layers/moe/mega_moe_flydsl.py | 104 ++++- .../moe/test_mega_moe_decode_capacity.py | 388 ++++++++++++++++++ 4 files changed, 530 insertions(+), 20 deletions(-) create mode 100644 test/registered/unit/layers/moe/test_mega_moe_decode_capacity.py diff --git a/python/sglang/srt/arg_groups/mega_moe_hook.py b/python/sglang/srt/arg_groups/mega_moe_hook.py index 654c956bf3..12a903b52d 100644 --- a/python/sglang/srt/arg_groups/mega_moe_hook.py +++ b/python/sglang/srt/arg_groups/mega_moe_hook.py @@ -79,6 +79,7 @@ def validate_mega_moe_token_budget_for_model(server_args: ServerArgs) -> None: cfg = resolving_view(server_args) if cfg.moe_a2a_backend != "megamoe": return + validate_flydsl_decode_capacity(server_args) if parse_connector_type(cfg.model_path) == ConnectorType.INSTANCE: return model_config = model_config_of(server_args) @@ -98,6 +99,58 @@ def mega_moe_decode_tokens_per_rank(cfg) -> int: return decode_max_bs * num_tokens_per_req +def flydsl_decode_tokens_per_rank(cfg) -> int | None: + """Rows of the largest decode-step forward on one rank, or None while the + request cap is still to be sized from memory. + + Eager batches and decode graph buckets are both capped at the per-rank + request pool; a request runs at most max(draft tokens, top-k) rows + (verify and draft extend, draft decode). + """ + if cfg.max_running_requests is None: + return None + requests = cfg.max_running_requests + if attn_dp_enabled_of(cfg): + requests //= cfg.attn_dp_size + tokens_per_req = 1 + if cfg.speculative_algorithm: + tokens_per_req = max( + cfg.speculative_num_draft_tokens or 1, cfg.speculative_eagle_topk or 1 + ) + return requests * tokens_per_req + + +def validate_flydsl_decode_capacity(server_args: ServerArgs) -> None: + if not (get_platform().is_hip and envs.SGLANG_AMD_USE_FLYDSL_MEGA_MOE.get()): + return + decode_mtpr = envs.SGLANG_AMD_FLYDSL_MEGA_MOE_DECODE_MTPR.get() + if decode_mtpr <= 0 or decode_mtpr >= envs.SGLANG_AMD_FLYDSL_MEGA_MOE_MTPR.get(): + # Off, one merged capacity, or rejected when the capacities are built. + return + cfg = resolving_view(server_args) + if cfg.disaggregation_mode == "prefill": + return + if cfg.attn_cp_size > 1: + raise ValueError( + "SGLANG_AMD_FLYDSL_MEGA_MOE_DECODE_MTPR is not supported with " + "attention context parallelism (--attn-cp-size > 1); set it to 0." + ) + decode_tokens = flydsl_decode_tokens_per_rank(cfg) + if decode_tokens is None: + raise ValueError( + "SGLANG_AMD_FLYDSL_MEGA_MOE_DECODE_MTPR needs --max-running-requests " + "to bound the largest decode / verify forward on one rank." + ) + if decode_tokens > decode_mtpr: + raise ValueError( + f"SGLANG_AMD_FLYDSL_MEGA_MOE_DECODE_MTPR={decode_mtpr} must cover the " + "largest decode / verify forward on one rank: max running requests " + f"per DP rank x tokens per request = {decode_tokens}. Raise it to a " + f"power of two of at least {decode_tokens} or lower " + "--max-running-requests." + ) + + def validate_mega_moe_token_budget(server_args: ServerArgs, model_label: str) -> None: cfg = resolving_view(server_args) if cfg.moe_a2a_backend != "megamoe": diff --git a/python/sglang/srt/environ.py b/python/sglang/srt/environ.py index fb4e6fe610..05dc9ec0e6 100644 --- a/python/sglang/srt/environ.py +++ b/python/sglang/srt/environ.py @@ -972,6 +972,11 @@ class Envs: SGLANG_USE_AITER_FP8_PER_TOKEN = EnvBool(False) SGLANG_AMD_USE_FLYDSL_MEGA_MOE = EnvBool(False) SGLANG_AMD_FLYDSL_MEGA_MOE_MTPR = EnvInt(8192) + # Smaller MegaMoEV2 capacity for decode steps; a step with an extend on any + # DP rank keeps SGLANG_AMD_FLYDSL_MEGA_MOE_MTPR. Must hold one rank's + # largest decode forward (checked at startup). 0 disables it; otherwise a + # power of two no larger than SGLANG_AMD_FLYDSL_MEGA_MOE_MTPR. + SGLANG_AMD_FLYDSL_MEGA_MOE_DECODE_MTPR = EnvInt(0) SGLANG_AMD_FLYDSL_MEGA_QUANT = EnvStr("") SGLANG_AITER_MEGA_RANK_SYNC = EnvBool(False) SGLANG_AITER_MEGA_EPLB_PREFILL_ONLY = EnvBool(False) diff --git a/python/sglang/srt/layers/moe/mega_moe_flydsl.py b/python/sglang/srt/layers/moe/mega_moe_flydsl.py index 0990258cee..96f37d4510 100644 --- a/python/sglang/srt/layers/moe/mega_moe_flydsl.py +++ b/python/sglang/srt/layers/moe/mega_moe_flydsl.py @@ -4,6 +4,8 @@ from __future__ import annotations +import functools +import logging from contextlib import nullcontext from types import SimpleNamespace from typing import TYPE_CHECKING @@ -14,7 +16,10 @@ from sglang.kernels.ops.moe.fill_padded_rows import _fill_padded_rows from sglang.srt.environ import envs from sglang.srt.eplb.expert_location_dispatch import ExpertLocationDispatchInfo from sglang.srt.layers.attention.dsa.utils import is_dsa_enable_prefill_cp -from sglang.srt.layers.dp_attention import get_dp_global_num_tokens +from sglang.srt.layers.dp_attention import ( + get_dp_global_num_tokens, + get_is_extend_in_batch, +) from sglang.srt.layers.moe.utils import get_moe_a2a_backend from sglang.srt.model_executor.runner import get_is_capture_mode from sglang.srt.runtime_context import get_parallel @@ -23,6 +28,7 @@ if TYPE_CHECKING: from sglang.srt.model_executor.forward_batch_info import ForwardBatch from sglang.srt.models.deepseek_v2 import DeepseekV2MoE +logger = logging.getLogger(__name__) _MORI_SHMEM_READY = False _MEGA_MOE_INSTANCE: dict = {} @@ -54,6 +60,66 @@ def _mtpr() -> int: return mtpr +def _decode_mtpr() -> int: + decode_mtpr = int(envs.SGLANG_AMD_FLYDSL_MEGA_MOE_DECODE_MTPR.get()) + if decode_mtpr == 0: + return 0 + mtpr = _mtpr() + if decode_mtpr < 0 or decode_mtpr > mtpr or decode_mtpr & (decode_mtpr - 1): + raise ValueError( + f"SGLANG_AMD_FLYDSL_MEGA_MOE_DECODE_MTPR={decode_mtpr} must be 0 (off) " + "or a power of two no larger than " + f"SGLANG_AMD_FLYDSL_MEGA_MOE_MTPR={mtpr}" + ) + if decode_mtpr == mtpr: + _warn_decode_mtpr_merged(mtpr) + return 0 + return decode_mtpr + + +@functools.cache +def _warn_decode_mtpr_merged(mtpr: int) -> None: + logger.warning( + "SGLANG_AMD_FLYDSL_MEGA_MOE_DECODE_MTPR equals " + "SGLANG_AMD_FLYDSL_MEGA_MOE_MTPR=%d; running one MegaMoE capacity", + mtpr, + ) + + +def _capacities() -> tuple[int, ...]: + decode_mtpr = _decode_mtpr() + return (decode_mtpr, _mtpr()) if decode_mtpr else (_mtpr(),) + + +def _is_decode_step_mode(forward_batch: ForwardBatch | None) -> bool: + # IDLE also runs in prefill steps; _select_mtpr tells the two apart. + if forward_batch is None: + return False + mode = forward_batch.forward_mode + return ( + mode.is_decode_or_idle() or mode.is_target_verify() or mode.is_draft_extend_v2() + ) + + +def _select_mtpr(forward_batch: ForwardBatch | None) -> int: + """The capacity this forward runs with; every EP rank must pick the same one. + + Local token counts, graph buckets and graph-vs-eager can differ across + ranks (DP counts are gathered only under require_mlp_tp_gather), so only + the step kind decides. + """ + mtpr = _mtpr() + decode_mtpr = _decode_mtpr() + if not decode_mtpr or not _is_decode_step_mode(forward_batch): + return mtpr + if get_is_capture_mode(): + # Decode-step graphs replay only when no DP rank extends + # (can_run_decode_cuda_graph is min-reduced over the group). + return decode_mtpr + # is_extend_in_batch is max-reduced over the DP group. + return mtpr if get_is_extend_in_batch() else decode_mtpr + + def _ep_rank_world(): parallel = get_parallel() return parallel.moe_ep_rank, parallel.moe_ep_size @@ -130,23 +196,21 @@ def _get_or_build_mega_moe( swiglu_limit: float, mtpr: int, ): + rank, world = _ep_rank_world() + shape_key = (rank, world, model_dim, inter_dim, experts, topk, quant, swiglu_limit) + mega = _MEGA_MOE_INSTANCE.get((*shape_key, mtpr)) + if mega is not None: + return mega _ensure_mori_shmem() backend = _import_aiter_megamoe() - rank, world = _ep_rank_world() - key = ( - rank, - world, - model_dim, - inter_dim, - experts, - topk, - quant, - swiglu_limit, - mtpr, - ) - mega = _MEGA_MOE_INSTANCE.get(key) - if mega is None: - mega = backend.op( + # Construction synchronizes the device and barriers every EP rank, so all + # capacities are built together on the first (eager) call, in the same order + # on every rank: a capacity first used later could be inside graph capture. + for capacity in _capacities(): + key = (*shape_key, capacity) + if key in _MEGA_MOE_INSTANCE: + continue + _MEGA_MOE_INSTANCE[key] = backend.op( rank=rank, world_size=world, model_dim=model_dim, @@ -158,11 +222,11 @@ def _get_or_build_mega_moe( w1_scale=layer._mega_w1_scale, w2=layer._mega_w2, w2_scale=layer._mega_w2_scale, - max_tok_per_rank=mtpr, + max_tok_per_rank=capacity, swiglu_limit=swiglu_limit, ) - _MEGA_MOE_INSTANCE[key] = mega - return mega + logger.info("Aiter MegaMoE: built capacity max_tok_per_rank=%d", capacity) + return _MEGA_MOE_INSTANCE[(*shape_key, mtpr)] def _swap_layer_weights(mega, layer) -> None: @@ -288,7 +352,7 @@ def _run_mega_routed( ) topk_weights = hidden_states.new_zeros((1, topk), dtype=torch.float32) - selected_mtpr = _mtpr() + selected_mtpr = _select_mtpr(forward_batch) x_in = x_in.contiguous() topk_ids = topk_ids.contiguous() topk_weights = topk_weights.contiguous() diff --git a/test/registered/unit/layers/moe/test_mega_moe_decode_capacity.py b/test/registered/unit/layers/moe/test_mega_moe_decode_capacity.py new file mode 100644 index 0000000000..aa6db48fcd --- /dev/null +++ b/test/registered/unit/layers/moe/test_mega_moe_decode_capacity.py @@ -0,0 +1,388 @@ +"""MegaMoE capacity selection must keep one cross-rank wire protocol. + +Exercises the real host dispatch and instance cache in ``mega_moe_flydsl`` with +CPU tensors, and the startup capacity check in ``mega_moe_hook``. Only the +AITER constructor/kernel and the per-forward DP context are replaced; GPU +collectives and graph replay are out of scope. +""" + +import sys +from types import ModuleType, SimpleNamespace + +import pytest +import torch + +import sglang.srt.arg_groups.mega_moe_hook as hook +import sglang.srt.layers.moe.mega_moe_flydsl as mega +from sglang.srt.model_executor.forward_batch_info import ForwardMode +from sglang.test.ci.ci_register import register_cpu_ci + +register_cpu_ci(est_time=2, suite="base-a-test-cpu") + +LARGE = 16384 +DECODE = 512 +EXPERTS = 384 +TOPK = 6 +HIDDEN = 8 + + +def _layer(value=0): + return SimpleNamespace( + num_experts=EXPERTS, + should_fuse_routed_scaling_factor_in_topk=True, + **{ + name: torch.full((48, 4), value, dtype=torch.uint8) + for name in ("_mega_w1", "_mega_w1_scale", "_mega_w2", "_mega_w2_scale") + }, + ) + + +@pytest.fixture +def runtime(monkeypatch): + state = SimpleNamespace( + rank=0, + capture=False, + extend=False, + global_tokens=None, + events=[], + instances={}, + ) + monkeypatch.setenv("SGLANG_AMD_FLYDSL_MEGA_MOE_MTPR", str(LARGE)) + monkeypatch.setenv("SGLANG_AMD_FLYDSL_MEGA_MOE_DECODE_MTPR", str(DECODE)) + monkeypatch.setenv("SGLANG_AITER_MEGA_RANK_SYNC", "1") + monkeypatch.setenv("SGLANG_AMD_FLYDSL_MEGA_QUANT", "a8w4") + monkeypatch.setattr(mega, "_MEGA_MOE_INSTANCE", {}) + mega._warn_decode_mtpr_merged.cache_clear() + # CPU-only PyTorch raises from the real query. + monkeypatch.setattr(torch.cuda, "is_current_stream_capturing", lambda: False) + monkeypatch.setattr(mega, "get_is_capture_mode", lambda: state.capture) + monkeypatch.setattr(mega, "get_is_extend_in_batch", lambda: state.extend) + monkeypatch.setattr(mega, "get_dp_global_num_tokens", lambda: state.global_tokens) + monkeypatch.setattr(mega, "_ensure_mori_shmem", lambda: None) + monkeypatch.setattr(mega, "_ep_rank_world", lambda: (state.rank, 8)) + monkeypatch.setattr( + mega, + "ExpertLocationDispatchInfo", + SimpleNamespace(init_new=lambda layer_id: None), + ) + utils = ModuleType("sglang.srt.models.deepseek_common.utils") + utils._use_aiter = True + monkeypatch.setitem(sys.modules, utils.__name__, utils) + + class FakeMegaMoEV2: + def __init__(self, **kwargs): + assert not torch.cuda.is_current_stream_capturing() + self.rank = kwargs["rank"] + self.capacity = kwargs["max_tok_per_rank"] + state.events.append(("build", self.rank, self.capacity)) + state.instances[self.rank, self.capacity] = self + + def forward(self, x, weights, ids, slice_output=False): + state.events.append(("forward", self.rank, self.capacity)) + return torch.full_like(x, self.capacity) + + monkeypatch.setattr( + mega, "_import_aiter_megamoe", lambda: SimpleNamespace(op=FakeMegaMoEV2) + ) + + def run( + local_rows, + global_tokens=None, + *, + rank=0, + mode=ForwardMode.DECODE, + extend=False, + capture=False, + layer=None, + ): + state.rank = rank + state.extend = extend + state.capture = capture + state.global_tokens = global_tokens + moe = SimpleNamespace( + config=SimpleNamespace( + hidden_size=HIDDEN, + num_experts_per_tok=TOPK, + moe_intermediate_size=HIDDEN, + swiglu_limit=0.0, + ), + num_fused_shared_experts=0, + experts=state.layer if layer is None else layer, + gate=lambda h, forward_batch=None: torch.zeros(h.shape[0], EXPERTS), + topk=lambda h, logits, **kwargs: SimpleNamespace( + topk_ids=torch.zeros(h.shape[0], TOPK, dtype=torch.int64), + topk_weights=torch.ones(h.shape[0], TOPK), + ), + is_hash=False, + layer_id=0, + routed_scaling_factor=1.0, + ) + forward_batch = ( + None + if mode is None + else SimpleNamespace(forward_mode=mode, num_token_non_padded=None) + ) + hidden = torch.zeros(local_rows, HIDDEN, dtype=torch.float32) + return mega._run_mega_routed(moe, hidden, forward_batch, None) + + state.layer = _layer() + state.run = run + return state + + +def _built(runtime, rank=0): + return [c for event, pe, c in runtime.events if event == "build" and pe == rank] + + +def _forwarded(runtime): + return [c for event, _, c in runtime.events if event == "forward"] + + +def _run_step(runtime, ranks): + for rank, kwargs in enumerate(ranks): + runtime.run(rank=rank, **kwargs) + return _forwarded(runtime) + + +@pytest.mark.parametrize("capture", [False, True]) +def test_rank_local_rows_across_the_decode_capacity_fail_instead_of_splitting( + runtime, capture +): + # EP8 / DP8 with dense TP1 and the DP LM head gathers neither DP token + # counts nor a DP-max graph bucket, so one rank can decode 128 rows while + # another decodes 1024. Startup rejects such configs; if one slips through, + # the overflowing rank must fail rather than run the large capacity alone. + mode = ForwardMode.TARGET_VERIFY + runtime.run(128, [128], rank=0, mode=mode, capture=capture) + with pytest.raises(AssertionError, match=f"exceed MTPR {DECODE}"): + runtime.run(1024, [1024], rank=1, mode=mode, capture=capture) + assert _forwarded(runtime) == [DECODE] + + +def test_decode_step_ignores_rank_local_counts_and_graph_buckets(runtime): + # Within the decode capacity: each rank sees only its own count, replays + # the bucket of its own batch, or runs eager beside the graph replays. + step = [ + dict(local_rows=7, global_tokens=[7]), + dict(local_rows=480, global_tokens=[480]), + dict(local_rows=56, mode=ForwardMode.TARGET_VERIFY, capture=True), + dict(local_rows=448, mode=ForwardMode.TARGET_VERIFY, capture=True), + dict(local_rows=448, global_tokens=[448], mode=ForwardMode.TARGET_VERIFY), + dict(local_rows=0, global_tokens=[0], mode=ForwardMode.IDLE), + dict(local_rows=0, mode=ForwardMode.IDLE, capture=True), + dict(local_rows=7, global_tokens=[7, 480, 56, 448, 448, 0, 0, 7]), + ] + assert _run_step(runtime, step) == [DECODE] * len(step) + + +def test_draft_extend_and_idle_ranks_share_the_decode_capacity(runtime): + # The draft extend after a verify runs DRAFT_EXTEND_V2 on ranks with + # requests, eager or from its graph, and IDLE on the rest. + step = [ + dict(local_rows=56, mode=ForwardMode.DRAFT_EXTEND_V2, capture=True), + dict(local_rows=448, global_tokens=[448], mode=ForwardMode.DRAFT_EXTEND_V2), + dict(local_rows=0, global_tokens=[0], mode=ForwardMode.IDLE), + ] + assert _run_step(runtime, step) == [DECODE] * len(step) + + +def test_an_extend_on_any_rank_moves_the_step_to_the_large_capacity(runtime): + # is_extend_in_batch is max-reduced, and decode graphs do not replay in + # this step; a prefill graph does, captured with the flag cleared. + prefill_graph = dict(mode=ForwardMode.EXTEND, capture=True) + step = [ + dict(local_rows=128, global_tokens=[128] * 8, **prefill_graph), + dict(local_rows=4096, global_tokens=[4096], mode=ForwardMode.EXTEND), + dict(local_rows=7, global_tokens=[7]), + dict(local_rows=448, global_tokens=[448], mode=ForwardMode.TARGET_VERIFY), + dict(local_rows=448, global_tokens=[448], mode=ForwardMode.DRAFT_EXTEND_V2), + dict(local_rows=0, global_tokens=[0], mode=ForwardMode.IDLE), + ] + for kwargs in step[1:]: + kwargs["extend"] = True + assert _run_step(runtime, step) == [LARGE] * len(step) + + +@pytest.mark.parametrize("capture", [False, True]) +@pytest.mark.parametrize( + "mode", + [ + ForwardMode.EXTEND, + ForwardMode.MIXED, + ForwardMode.SPLIT_PREFILL, + ForwardMode.DLLM_EXTEND, + None, + ], +) +def test_prefill_forwards_keep_the_large_capacity(runtime, mode, capture): + runtime.run(128, [128] * 8, mode=mode, capture=capture) + assert _forwarded(runtime) == [LARGE] + + +def test_idle_rank_enters_with_the_decode_capacity(runtime): + output = runtime.run(0, [0], rank=7, mode=ForwardMode.IDLE) + + assert _forwarded(runtime) == [DECODE] + assert output.shape == (0, HIDDEN) + + +@pytest.mark.parametrize("mode", [ForwardMode.DECODE, ForwardMode.EXTEND]) +def test_first_call_builds_every_capacity_in_one_order(runtime, mode): + for rank in range(2): + runtime.run(1, [1], rank=rank, mode=mode, extend=mode.is_extend()) + assert _built(runtime, 0) == _built(runtime, 1) == [DECODE, LARGE] + + runtime.run(1, capture=True) + runtime.run(1, [1], mode=ForwardMode.EXTEND, extend=True) + assert _built(runtime, 0) == [DECODE, LARGE] + + +def test_each_use_rebinds_the_layer_weights_without_copying(runtime): + runtime.run(1, [1], mode=ForwardMode.EXTEND, extend=True) + next_layer = _layer(value=7) + runtime.run(1, [1], layer=next_layer) + + decode = runtime.instances[0, DECODE] + assert decode._s1_w1.data_ptr() == next_layer._mega_w1.data_ptr() + assert decode._s1_w1_scale.data_ptr() == next_layer._mega_w1_scale.data_ptr() + assert decode.w2 is next_layer._mega_w2 + assert decode.w2_scale is next_layer._mega_w2_scale + + # EPLB migrates expert rows in place; the bound views must observe it. + next_layer._mega_w1[0].fill_(9) + assert torch.all(decode._s1_w1[0] == 9) + + +@pytest.mark.parametrize("decode_mtpr", ["0", str(LARGE)]) +def test_disabled_decode_capacity_keeps_one_instance(runtime, monkeypatch, decode_mtpr): + monkeypatch.setenv("SGLANG_AMD_FLYDSL_MEGA_MOE_DECODE_MTPR", decode_mtpr) + runtime.run(1, [1] * 8) + + assert _built(runtime) == [LARGE] + assert _forwarded(runtime) == [LARGE] + + +def test_decode_capacity_equal_to_the_large_one_warns_once( + runtime, monkeypatch, caplog +): + monkeypatch.setenv("SGLANG_AMD_FLYDSL_MEGA_MOE_DECODE_MTPR", str(LARGE)) + with caplog.at_level("WARNING", logger=mega.logger.name): + runtime.run(1, [1] * 8) + runtime.run(1, [1] * 8) + assert [r.levelname for r in caplog.records].count("WARNING") == 1 + + +@pytest.mark.parametrize("decode_mtpr", ["-1", "384", str(2 * LARGE)]) +def test_invalid_decode_capacity_is_rejected(runtime, monkeypatch, decode_mtpr): + monkeypatch.setenv("SGLANG_AMD_FLYDSL_MEGA_MOE_DECODE_MTPR", decode_mtpr) + with pytest.raises(ValueError, match="power of two no larger than"): + runtime.run(1, [1] * 8) + + +@pytest.mark.xfail( + strict=True, + reason="no guard yet: a capacity first built inside graph capture is not rejected", +) +def test_capture_cache_miss_is_rejected_before_symmetric_allocation( + runtime, monkeypatch +): + monkeypatch.setattr(torch.cuda, "is_current_stream_capturing", lambda: True) + with pytest.raises(RuntimeError, match="before graph capture"): + runtime.run(1, None, capture=True) + assert runtime.events == [] + + +def _cfg(**overrides): + # Recipe c128 EP8 arm: DP8, DSpark with 7 draft tokens. + cfg = dict( + disaggregation_mode="null", + attn_cp_size=1, + attn_dp_size=8, + ep_join_mode=None, + max_running_requests=256, + speculative_algorithm="DSPARK", + speculative_num_draft_tokens=7, + speculative_eagle_topk=1, + ) + cfg.update(overrides) + return SimpleNamespace(**cfg) + + +@pytest.fixture +def startup(monkeypatch): + monkeypatch.setenv("SGLANG_AMD_USE_FLYDSL_MEGA_MOE", "1") + monkeypatch.setenv("SGLANG_AMD_FLYDSL_MEGA_MOE_MTPR", str(LARGE)) + monkeypatch.setenv("SGLANG_AMD_FLYDSL_MEGA_MOE_DECODE_MTPR", str(DECODE)) + monkeypatch.setattr(hook, "get_platform", lambda: SimpleNamespace(is_hip=True)) + monkeypatch.setattr(hook, "resolving_view", lambda server_args: server_args) + return hook.validate_flydsl_decode_capacity + + +@pytest.mark.parametrize( + ("overrides", "rows"), + [ + ({}, 32 * 7), + (dict(attn_dp_size=1), 256 * 7), + (dict(speculative_algorithm=None), 32), + ( + dict( + speculative_algorithm="EAGLE", + speculative_num_draft_tokens=4, + speculative_eagle_topk=8, + ), + 32 * 8, + ), + (dict(max_running_requests=None), None), + ], +) +def test_decode_rows_per_rank(overrides, rows): + assert hook.flydsl_decode_tokens_per_rank(_cfg(**overrides)) == rows + + +@pytest.mark.parametrize( + ("max_running_requests", "decode_mtpr"), + # Recipe c128 / c256 / c384 arms: 224 / 448 / 672 rows per rank. + [(256, 512), (512, 512), (768, 1024)], +) +def test_startup_accepts_a_decode_capacity_that_holds_every_rank( + startup, monkeypatch, max_running_requests, decode_mtpr +): + monkeypatch.setenv("SGLANG_AMD_FLYDSL_MEGA_MOE_DECODE_MTPR", str(decode_mtpr)) + startup(_cfg(max_running_requests=max_running_requests)) + + +def test_startup_rejects_a_decode_capacity_one_rank_can_overflow(startup): + # One rank may then decode 1024 rows beside another's 128. + with pytest.raises(ValueError, match="at least 1024"): + startup(_cfg(max_running_requests=8 * 1024, speculative_algorithm=None)) + + +def test_startup_needs_a_request_cap(startup): + with pytest.raises(ValueError, match="--max-running-requests"): + startup(_cfg(max_running_requests=None)) + + +def test_startup_rejects_attention_context_parallelism(startup): + with pytest.raises(ValueError, match="context parallelism"): + startup(_cfg(attn_cp_size=2)) + + +@pytest.mark.parametrize( + ("env", "overrides"), + [ + ({}, dict(disaggregation_mode="prefill")), + ({"SGLANG_AMD_FLYDSL_MEGA_MOE_DECODE_MTPR": "0"}, {}), + ({"SGLANG_AMD_FLYDSL_MEGA_MOE_DECODE_MTPR": str(LARGE)}, {}), + ({"SGLANG_AMD_USE_FLYDSL_MEGA_MOE": "0"}, {}), + ], +) +def test_startup_skips_runs_without_a_decode_capacity( + startup, monkeypatch, env, overrides +): + for name, value in env.items(): + monkeypatch.setenv(name, value) + startup(_cfg(max_running_requests=None, **overrides)) + + +if __name__ == "__main__": + sys.exit(pytest.main([__file__, "-v"]))