[Spec][MegaMoE] Let the speculative draft choose its own W4A4 MXFP4 MegaMoE MMA type (#42022)

Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Po-Han Huang (NVIDIA)
2026-10-06 14:09:57 -07:00
committed by GitHub
co-authored by Claude Opus 5.5
parent 12004043b8
commit 3eec178e48
10 changed files with 172 additions and 39 deletions
@@ -284,6 +284,7 @@ POSITIONAL_FIELD_ORDER = (
"speculative_draft_window_size",
"speculative_moe_runner_backend",
"speculative_moe_a2a_backend",
"speculative_enable_w4a4_mxfp4_megamoe",
"speculative_draft_model_quantization",
"_speculative_draft_quantization_explicitly_set",
"speculative_skip_dp_mlp_sync",
@@ -2,6 +2,7 @@
from __future__ import annotations
import argparse
from typing import (
Literal,
Optional,
@@ -196,6 +197,21 @@ class Spec(msgspec.Struct):
resolvable=True,
),
] = None
speculative_enable_w4a4_mxfp4_megamoe: A[
Optional[bool],
Arg(
help="Whether the draft model's MXFP4 MegaMoE layers use the W4A4 "
"mxf4xmxf4 MMA type (see --enable-w4a4-mxfp4-megamoe). Pass "
"--no-speculative-enable-w4a4-mxfp4-megamoe to keep the draft on "
"fp8xfp4 (W4A8) while the target runs W4A4. Same as "
"--enable-w4a4-mxfp4-megamoe if unset. A draft that runs MXFP4 "
"MegaMoE with a different MMA type from the target may allocate an "
"additional MegaMoE symmetric buffer after the KV pool is sized; "
"lower --mem-fraction-static if CUDA-graph capture runs out of "
"memory.",
action=argparse.BooleanOptionalAction,
),
] = None
speculative_draft_model_quantization: A[
Optional[str],
Arg(
@@ -403,6 +403,8 @@ class FusedMoE(torch.nn.Module):
# Set by the quant method when it repacks experts for MegaMoE.
self._mega_moe_weights_built = False
self._mega_moe_nvfp4 = False
# Read at construction: draft_model_build_scope applies the draft's own value.
self._mega_moe_w4a4 = get_exec().moe.enable_w4a4_mxfp4_megamoe
self._pending_fp8_shared_weights: dict[tuple[int, str], torch.Tensor] = {}
self._pending_fp8_shared_scales: dict[tuple[int, str], torch.Tensor] = {}
+7 -8
View File
@@ -33,7 +33,6 @@ from sglang.srt.layers.moe.mega_moe_sm90 import (
from sglang.srt.layers.moe.utils import get_moe_a2a_backend
from sglang.srt.model_executor.runner import get_is_capture_mode
from sglang.srt.models.deepseek_common.utils import _device_sm
from sglang.srt.runtime_context import get_exec
from sglang.srt.utils import is_hip, is_sm100_supported
if TYPE_CHECKING:
@@ -53,10 +52,11 @@ def _use_amd_flydsl_mega_moe() -> bool:
return _is_hip and envs.SGLANG_AMD_USE_FLYDSL_MEGA_MOE.get()
def _mega_moe_mma_type(experts=None) -> str:
if experts is not None and experts._mega_moe_nvfp4:
def _mega_moe_mma_type(experts) -> str:
if experts._mega_moe_nvfp4:
return "nvfp4xnvfp4"
return "mxf4xmxf4" if get_exec().moe.enable_w4a4_mxfp4_megamoe else "fp8xfp4"
# Per layer: a draft may differ from the target (draft_model_build_scope).
return "mxf4xmxf4" if experts._mega_moe_w4a4 else "fp8xfp4"
@functools.lru_cache(maxsize=1)
@@ -118,12 +118,11 @@ def _get_mega_moe_symm_buffer(
hidden: int,
intermediate_hidden: int,
num_shared_experts: int = 0,
mma_type: Optional[str] = None,
*,
mma_type: str,
) -> SymmBuffer:
import deep_gemm
if mma_type is None:
mma_type = _mega_moe_mma_type()
with _configure_mega_moe_deep_gemm_num_sms(deep_gemm):
key = (
id(group),
@@ -513,7 +512,7 @@ def build_mega_moe_experts_weights(experts) -> None:
if getattr(experts, "_mega_moe_weights_built", False):
return
mma_type = _mega_moe_mma_type()
mma_type = _mega_moe_mma_type(experts)
w13 = experts.w13_weight.data
w13_sf_fp32 = experts.w13_weight_scale_inv.data
w2 = experts.w2_weight.data
+27 -6
View File
@@ -2,7 +2,7 @@ from __future__ import annotations
import logging
import os
from contextlib import contextmanager
from contextlib import AbstractContextManager, contextmanager, nullcontext
from dataclasses import dataclass
from enum import Enum, IntEnum
from typing import NamedTuple
@@ -546,9 +546,18 @@ def is_shared_experts_fusion_disabled() -> bool:
@contextmanager
def draft_model_build_scope():
"""Brackets a draft model's CONSTRUCTION: the gates it runs record their
fusion decision on the speculative leaf as well, and the target's ACTIVE
value returns on exit.
"""Brackets a draft model's CONSTRUCTION with the draft's construction-time
settings; every value below returns to the target's on exit, including on
error.
- Shared-experts fusion: the gates the draft runs also record their
decision on the speculative leaf; the target's ACTIVE
``disable_shared_experts_fusion`` is restored.
- ``boundary_reduction`` is ``--speculative-boundary-reduction``.
Boundaries built here keep the resolved value after exit.
- ``enable_w4a4_mxfp4_megamoe`` is ``--speculative-enable-w4a4-mxfp4-megamoe``
when set, else inherited. ``FusedMoE`` layers built here keep the
draft's MegaMoE MMA type after exit.
Deliberately does not touch ``runner_backend`` — swapping that is
``speculative_moe_backend_context``'s job and has to bracket the draft's
@@ -561,8 +570,11 @@ def draft_model_build_scope():
moe.in_speculative_scope = True
# Boundaries capture this resolved preference while the draft builds.
# Restoring it cannot change an already constructed target plan.
with get_exec().comm.override(
boundary_reduction=get_spec().speculative_boundary_reduction
with (
get_exec().comm.override(
boundary_reduction=get_spec().speculative_boundary_reduction
),
_draft_w4a4_mxfp4_megamoe_override(),
):
yield
finally:
@@ -570,6 +582,15 @@ def draft_model_build_scope():
moe.disable_shared_experts_fusion = original_fusion
def _draft_w4a4_mxfp4_megamoe_override() -> AbstractContextManager:
# FusedMoE pins its MegaMoE MMA type at construction, so the draft's layers
# keep this value after the scope exits.
draft_w4a4 = get_spec().speculative_enable_w4a4_mxfp4_megamoe
if draft_w4a4 is None:
return nullcontext()
return get_exec().moe.override(enable_w4a4_mxfp4_megamoe=draft_w4a4)
def install_shared_experts_fusion_decision(
model_class, hf_config, quant_config
) -> None:
@@ -700,7 +700,7 @@ class Mxfp4MoEMethod(FusedMoEMethodBase):
check_mega_moe_shapes,
)
mma_type = _mega_moe_mma_type()
mma_type = _mega_moe_mma_type(layer)
check_mega_moe_shapes(
hidden=layer.w13_weight.shape[2] * 2,
intermediate=layer.w13_weight.shape[1] // 2,
+2
View File
@@ -788,6 +788,7 @@ class KimiK3MoE(nn.Module):
from sglang.srt.layers.moe.mega_moe import (
_configure_mega_moe_deep_gemm_num_sms,
_get_mega_moe_symm_buffer,
_mega_moe_mma_type,
)
from sglang.srt.runtime_context import get_parallel
@@ -813,6 +814,7 @@ class KimiK3MoE(nn.Module):
num_topk=self._mega_top_k,
hidden=self.moe_hidden_size,
intermediate_hidden=self._mega_intermediate_size,
mma_type=_mega_moe_mma_type(self.experts),
)
if num_tokens > 0: