diff --git a/python/sglang/srt/arg_groups/mega_moe_hook.py b/python/sglang/srt/arg_groups/mega_moe_hook.py index 12a903b52d..5475e1e206 100644 --- a/python/sglang/srt/arg_groups/mega_moe_hook.py +++ b/python/sglang/srt/arg_groups/mega_moe_hook.py @@ -14,7 +14,7 @@ from sglang.srt.arg_groups.overrides import ( from sglang.srt.connector import ConnectorType from sglang.srt.environ import envs from sglang.srt.runtime_context import attn_dp_enabled_of, get_platform -from sglang.srt.utils.common import parse_connector_type +from sglang.srt.utils.common import ceil_align, parse_connector_type logger = logging.getLogger(__name__) @@ -93,6 +93,12 @@ def validate_mega_moe_token_budget_for_model(server_args: ServerArgs) -> None: def mega_moe_decode_tokens_per_rank(cfg) -> int: cg_config = cfg.cuda_graph_config decode_max_bs = (cg_config.decode.max_bs if cg_config is not None else 0) or 0 + if cfg.max_running_requests is not None: + per_rank_requests = ceil_align( + cfg.max_running_requests // cfg.attn_dp_size, + cfg.tp_size // cfg.attn_dp_size, + ) + decode_max_bs = min(decode_max_bs, per_rank_requests) num_tokens_per_req = ( (cfg.speculative_num_draft_tokens or 1) if cfg.speculative_algorithm else 1 ) @@ -168,11 +174,11 @@ def validate_mega_moe_token_budget(server_args: ServerArgs, model_label: str) -> f"{model_label} with MegaMoE requires " "SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK to cover the " "largest decode / verify forward on one rank. Current values: " - f"decode cuda graph max bs x tokens per request = {decode_tokens}, " + f"largest decode graph bucket x tokens per request = {decode_tokens}, " f"SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK=" f"{max_tokens_per_rank}. Raise the env var to at least " f"{decode_tokens} or lower --cuda-graph-max-bs / " - "--speculative-num-draft-tokens." + "--max-running-requests / --speculative-num-draft-tokens." ) if cfg.disaggregation_mode == "decode": # decode node will skip the check because decode bs is not relevant with --chunk-prefill-size diff --git a/test/registered/unit/server_args/test_server_args.py b/test/registered/unit/server_args/test_server_args.py index 2265eb617d..7841229d32 100644 --- a/test/registered/unit/server_args/test_server_args.py +++ b/test/registered/unit/server_args/test_server_args.py @@ -355,6 +355,7 @@ class TestPrepareServerArgs(CustomTestCase): ), speculative_algorithm=algorithm, speculative_num_draft_tokens=draft_tokens, + max_running_requests=None, ) self.assertEqual(mega_moe_decode_tokens_per_rank(view(16384)), 16384) @@ -364,6 +365,54 @@ class TestPrepareServerArgs(CustomTestCase): self.assertEqual(mega_moe_decode_tokens_per_rank(view(None)), 0) self.assertEqual(mega_moe_decode_tokens_per_rank(view(0, cg=False)), 0) + def test_megamoe_decode_budget_clamps_to_request_pool(self): + """A DP16 pool of 256 requests per rank was validated as 4096 graph requests.""" + from sglang.srt.arg_groups.mega_moe_hook import ( + mega_moe_decode_tokens_per_rank, + validate_mega_moe_token_budget, + ) + + def view(tp_size, dp_size, requests, graph_bs): + return SimpleNamespace( + cuda_graph_config=SimpleNamespace( + decode=SimpleNamespace(max_bs=graph_bs) + ), + max_running_requests=requests, + attn_dp_size=dp_size, + tp_size=tp_size, + speculative_algorithm="DSPARK", + speculative_num_draft_tokens=6, + moe_a2a_backend="megamoe", + disaggregation_mode="decode", + ) + + self.assertEqual( + mega_moe_decode_tokens_per_rank(view(16, 16, 4096, 4096)), 1536 + ) + self.assertEqual(mega_moe_decode_tokens_per_rank(view(16, 16, 4096, 128)), 768) + # The pool is padded to attn TP after the DP divide: 1 -> 8, 36 // 4 = 9 -> 12. + self.assertEqual(mega_moe_decode_tokens_per_rank(view(8, 1, 1, 4096)), 48) + self.assertEqual(mega_moe_decode_tokens_per_rank(view(16, 4, 36, 4096)), 72) + + with ( + override_platform(is_hip=False), + patch( + "sglang.srt.arg_groups.mega_moe_hook.resolving_view", + return_value=view(16, 16, 4096, 4096), + ), + ): + with envs.SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK.override( + 1536 + ): + validate_mega_moe_token_budget(None, "DeepseekV4ForCausalLM") + with ( + envs.SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK.override( + 1535 + ), + self.assertRaisesRegex(ValueError, "largest decode / verify"), + ): + validate_mega_moe_token_budget(None, "DeepseekV4ForCausalLM") + def test_w4a4_mxfp4_megamoe_disabled_preserves_deepgemm_env(self): deepgemm_env = { "DG_USE_FP4_ACTS": "0",