[NVIDIA][MegaMoE] Fix the default spec decoding moe backend when the target model uses flashinfer megamoe (#40772)
Co-authored-by: Po-Han Huang (NVIDIA) <53919306+nvpohanh@users.noreply.github.com>
This commit is contained in:
@@ -172,7 +172,7 @@ class Spec(msgspec.Struct):
|
||||
speculative_moe_runner_backend: A[
|
||||
Optional[str],
|
||||
Arg(
|
||||
help="Choose the runner backend for MoE in speculative decoding.",
|
||||
help="Choose the runner backend for MoE in speculative decoding. Defaults to --moe-runner-backend, except flashinfer_megamoe defaults the draft to flashinfer_trtllm (draft layers are typically unquantized, which flashinfer_megamoe's global-expert-id dispatch doesn't support).",
|
||||
choices=MOE_RUNNER_BACKEND_CHOICES,
|
||||
resolvable=True,
|
||||
),
|
||||
@@ -180,7 +180,7 @@ class Spec(msgspec.Struct):
|
||||
speculative_moe_a2a_backend: A[
|
||||
Optional[str],
|
||||
Arg(
|
||||
help="Choose the backend for MoE A2A in speculative decoding",
|
||||
help="Choose the backend for MoE A2A in speculative decoding. Defaults to --moe-a2a-backend, except flashinfer_megamoe defaults the draft to none (draft layers are typically unquantized, which flashinfer_megamoe's global-expert-id dispatch doesn't support).",
|
||||
choices=[
|
||||
"none",
|
||||
"deepep",
|
||||
|
||||
@@ -1584,9 +1584,22 @@ def _pipeline_parallel_overlap_disable(view: Any) -> dict:
|
||||
def _speculative_moe_runner_default(view: Any) -> dict:
|
||||
"""Default the speculative (draft) MoE runner backend to the resolved
|
||||
target-model backend. Invoked at the head of the speculative-decoding
|
||||
hook, after the MoE kernel chain has resolved."""
|
||||
hook, after the MoE kernel chain has resolved.
|
||||
|
||||
flashinfer_megamoe routes by global expert id inside the mega kernel and
|
||||
only supports the quantized MoE layers it prepared weights for. Draft
|
||||
(MTP/NextN) layers are typically unquantized and run through a regular
|
||||
runner, so inheriting it would mismatch the draft's runner and corrupt
|
||||
expert-id dispatch. Default the draft to flashinfer_trtllm instead; users
|
||||
with a quantized draft can set --speculative-moe-runner-backend explicitly.
|
||||
(DeepGEMM megamoe is selected via the a2a backend and falls back per-layer
|
||||
when its weights are not prepared, so it needs no override.)
|
||||
"""
|
||||
if view.speculative_moe_runner_backend is None:
|
||||
return {"speculative_moe_runner_backend": view.moe_runner_backend}
|
||||
default = view.moe_runner_backend
|
||||
if default == "flashinfer_megamoe":
|
||||
default = "flashinfer_trtllm"
|
||||
return {"speculative_moe_runner_backend": default}
|
||||
return {}
|
||||
|
||||
|
||||
|
||||
@@ -443,6 +443,27 @@ def get_deepep_v2_fp8_scale_format() -> DeepEPv2Fp8ScaleFormat:
|
||||
)
|
||||
|
||||
|
||||
# Only flashinfer_megamoe is special-cased: its global-expert-id dispatch
|
||||
# assumes quantized layers, which draft (MTP/NextN) layers typically are not.
|
||||
# DeepGEMM megamoe (a2a "megamoe") falls back per-layer when its weights
|
||||
# are not prepared, so drafts can inherit it. A quantized draft that wants
|
||||
# flashinfer_megamoe must set the --speculative-moe-* flags explicitly.
|
||||
def _default_speculative_a2a_backend(
|
||||
target_a2a_backend: MoeA2ABackend,
|
||||
) -> MoeA2ABackend:
|
||||
if target_a2a_backend.is_flashinfer_megamoe():
|
||||
return MoeA2ABackend.NONE
|
||||
return target_a2a_backend
|
||||
|
||||
|
||||
def _default_speculative_moe_runner_backend(
|
||||
target_runner_backend: MoeRunnerBackendLike,
|
||||
) -> MoeRunnerBackendLike:
|
||||
if target_runner_backend.is_flashinfer_megamoe():
|
||||
return MoeRunnerBackend.FLASHINFER_TRTLLM
|
||||
return target_runner_backend
|
||||
|
||||
|
||||
def initialize_moe_config():
|
||||
"""Seed the MoE runtime flags from the published configuration.
|
||||
|
||||
@@ -460,12 +481,12 @@ def initialize_moe_config():
|
||||
moe.speculative_runner_backend = (
|
||||
resolve_moe_runner_backend(spec.speculative_moe_runner_backend)
|
||||
if spec.speculative_moe_runner_backend is not None
|
||||
else moe.runner_backend
|
||||
else _default_speculative_moe_runner_backend(moe.runner_backend)
|
||||
)
|
||||
moe.speculative_a2a_backend = (
|
||||
MoeA2ABackend(spec.speculative_moe_a2a_backend)
|
||||
if spec.speculative_moe_a2a_backend is not None
|
||||
else moe.a2a_backend
|
||||
else _default_speculative_a2a_backend(moe.a2a_backend)
|
||||
)
|
||||
moe.deepep_mode = DeepEPMode(exec_moe.deepep_mode)
|
||||
moe.deepep_config = exec_moe.deepep_config or ""
|
||||
|
||||
Reference in New Issue
Block a user