[Fix] megamoe: Respect request-pool limits in decode validation (#44816)

This commit is contained in:
Khoa Pham
2026-10-16 18:56:38 -07:00
committed by GitHub
parent 5cbe2371a5
commit ff979e0816
2 changed files with 58 additions and 3 deletions
@@ -14,7 +14,7 @@ from sglang.srt.arg_groups.overrides import (
from sglang.srt.connector import ConnectorType
from sglang.srt.environ import envs
from sglang.srt.runtime_context import attn_dp_enabled_of, get_platform
from sglang.srt.utils.common import parse_connector_type
from sglang.srt.utils.common import ceil_align, parse_connector_type
logger = logging.getLogger(__name__)
@@ -93,6 +93,12 @@ def validate_mega_moe_token_budget_for_model(server_args: ServerArgs) -> None:
def mega_moe_decode_tokens_per_rank(cfg) -> int:
cg_config = cfg.cuda_graph_config
decode_max_bs = (cg_config.decode.max_bs if cg_config is not None else 0) or 0
if cfg.max_running_requests is not None:
per_rank_requests = ceil_align(
cfg.max_running_requests // cfg.attn_dp_size,
cfg.tp_size // cfg.attn_dp_size,
)
decode_max_bs = min(decode_max_bs, per_rank_requests)
num_tokens_per_req = (
(cfg.speculative_num_draft_tokens or 1) if cfg.speculative_algorithm else 1
)
@@ -168,11 +174,11 @@ def validate_mega_moe_token_budget(server_args: ServerArgs, model_label: str) ->
f"{model_label} with MegaMoE requires "
"SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK to cover the "
"largest decode / verify forward on one rank. Current values: "
f"decode cuda graph max bs x tokens per request = {decode_tokens}, "
f"largest decode graph bucket x tokens per request = {decode_tokens}, "
f"SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK="
f"{max_tokens_per_rank}. Raise the env var to at least "
f"{decode_tokens} or lower --cuda-graph-max-bs / "
"--speculative-num-draft-tokens."
"--max-running-requests / --speculative-num-draft-tokens."
)
if cfg.disaggregation_mode == "decode":
# decode node will skip the check because decode bs is not relevant with --chunk-prefill-size