[Spec][MegaMoE] Let the speculative draft choose its own W4A4 MXFP4 MegaMoE MMA type (#42022)
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
parent
12004043b8
commit
3eec178e48
@@ -284,6 +284,7 @@ POSITIONAL_FIELD_ORDER = (
|
||||
"speculative_draft_window_size",
|
||||
"speculative_moe_runner_backend",
|
||||
"speculative_moe_a2a_backend",
|
||||
"speculative_enable_w4a4_mxfp4_megamoe",
|
||||
"speculative_draft_model_quantization",
|
||||
"_speculative_draft_quantization_explicitly_set",
|
||||
"speculative_skip_dp_mlp_sync",
|
||||
|
||||
@@ -2,6 +2,7 @@
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
from typing import (
|
||||
Literal,
|
||||
Optional,
|
||||
@@ -196,6 +197,21 @@ class Spec(msgspec.Struct):
|
||||
resolvable=True,
|
||||
),
|
||||
] = None
|
||||
speculative_enable_w4a4_mxfp4_megamoe: A[
|
||||
Optional[bool],
|
||||
Arg(
|
||||
help="Whether the draft model's MXFP4 MegaMoE layers use the W4A4 "
|
||||
"mxf4xmxf4 MMA type (see --enable-w4a4-mxfp4-megamoe). Pass "
|
||||
"--no-speculative-enable-w4a4-mxfp4-megamoe to keep the draft on "
|
||||
"fp8xfp4 (W4A8) while the target runs W4A4. Same as "
|
||||
"--enable-w4a4-mxfp4-megamoe if unset. A draft that runs MXFP4 "
|
||||
"MegaMoE with a different MMA type from the target may allocate an "
|
||||
"additional MegaMoE symmetric buffer after the KV pool is sized; "
|
||||
"lower --mem-fraction-static if CUDA-graph capture runs out of "
|
||||
"memory.",
|
||||
action=argparse.BooleanOptionalAction,
|
||||
),
|
||||
] = None
|
||||
speculative_draft_model_quantization: A[
|
||||
Optional[str],
|
||||
Arg(
|
||||
|
||||
@@ -403,6 +403,8 @@ class FusedMoE(torch.nn.Module):
|
||||
# Set by the quant method when it repacks experts for MegaMoE.
|
||||
self._mega_moe_weights_built = False
|
||||
self._mega_moe_nvfp4 = False
|
||||
# Read at construction: draft_model_build_scope applies the draft's own value.
|
||||
self._mega_moe_w4a4 = get_exec().moe.enable_w4a4_mxfp4_megamoe
|
||||
self._pending_fp8_shared_weights: dict[tuple[int, str], torch.Tensor] = {}
|
||||
self._pending_fp8_shared_scales: dict[tuple[int, str], torch.Tensor] = {}
|
||||
|
||||
|
||||
@@ -33,7 +33,6 @@ from sglang.srt.layers.moe.mega_moe_sm90 import (
|
||||
from sglang.srt.layers.moe.utils import get_moe_a2a_backend
|
||||
from sglang.srt.model_executor.runner import get_is_capture_mode
|
||||
from sglang.srt.models.deepseek_common.utils import _device_sm
|
||||
from sglang.srt.runtime_context import get_exec
|
||||
from sglang.srt.utils import is_hip, is_sm100_supported
|
||||
|
||||
if TYPE_CHECKING:
|
||||
@@ -53,10 +52,11 @@ def _use_amd_flydsl_mega_moe() -> bool:
|
||||
return _is_hip and envs.SGLANG_AMD_USE_FLYDSL_MEGA_MOE.get()
|
||||
|
||||
|
||||
def _mega_moe_mma_type(experts=None) -> str:
|
||||
if experts is not None and experts._mega_moe_nvfp4:
|
||||
def _mega_moe_mma_type(experts) -> str:
|
||||
if experts._mega_moe_nvfp4:
|
||||
return "nvfp4xnvfp4"
|
||||
return "mxf4xmxf4" if get_exec().moe.enable_w4a4_mxfp4_megamoe else "fp8xfp4"
|
||||
# Per layer: a draft may differ from the target (draft_model_build_scope).
|
||||
return "mxf4xmxf4" if experts._mega_moe_w4a4 else "fp8xfp4"
|
||||
|
||||
|
||||
@functools.lru_cache(maxsize=1)
|
||||
@@ -118,12 +118,11 @@ def _get_mega_moe_symm_buffer(
|
||||
hidden: int,
|
||||
intermediate_hidden: int,
|
||||
num_shared_experts: int = 0,
|
||||
mma_type: Optional[str] = None,
|
||||
*,
|
||||
mma_type: str,
|
||||
) -> SymmBuffer:
|
||||
import deep_gemm
|
||||
|
||||
if mma_type is None:
|
||||
mma_type = _mega_moe_mma_type()
|
||||
with _configure_mega_moe_deep_gemm_num_sms(deep_gemm):
|
||||
key = (
|
||||
id(group),
|
||||
@@ -513,7 +512,7 @@ def build_mega_moe_experts_weights(experts) -> None:
|
||||
if getattr(experts, "_mega_moe_weights_built", False):
|
||||
return
|
||||
|
||||
mma_type = _mega_moe_mma_type()
|
||||
mma_type = _mega_moe_mma_type(experts)
|
||||
w13 = experts.w13_weight.data
|
||||
w13_sf_fp32 = experts.w13_weight_scale_inv.data
|
||||
w2 = experts.w2_weight.data
|
||||
|
||||
@@ -2,7 +2,7 @@ from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import os
|
||||
from contextlib import contextmanager
|
||||
from contextlib import AbstractContextManager, contextmanager, nullcontext
|
||||
from dataclasses import dataclass
|
||||
from enum import Enum, IntEnum
|
||||
from typing import NamedTuple
|
||||
@@ -546,9 +546,18 @@ def is_shared_experts_fusion_disabled() -> bool:
|
||||
|
||||
@contextmanager
|
||||
def draft_model_build_scope():
|
||||
"""Brackets a draft model's CONSTRUCTION: the gates it runs record their
|
||||
fusion decision on the speculative leaf as well, and the target's ACTIVE
|
||||
value returns on exit.
|
||||
"""Brackets a draft model's CONSTRUCTION with the draft's construction-time
|
||||
settings; every value below returns to the target's on exit, including on
|
||||
error.
|
||||
|
||||
- Shared-experts fusion: the gates the draft runs also record their
|
||||
decision on the speculative leaf; the target's ACTIVE
|
||||
``disable_shared_experts_fusion`` is restored.
|
||||
- ``boundary_reduction`` is ``--speculative-boundary-reduction``.
|
||||
Boundaries built here keep the resolved value after exit.
|
||||
- ``enable_w4a4_mxfp4_megamoe`` is ``--speculative-enable-w4a4-mxfp4-megamoe``
|
||||
when set, else inherited. ``FusedMoE`` layers built here keep the
|
||||
draft's MegaMoE MMA type after exit.
|
||||
|
||||
Deliberately does not touch ``runner_backend`` — swapping that is
|
||||
``speculative_moe_backend_context``'s job and has to bracket the draft's
|
||||
@@ -561,8 +570,11 @@ def draft_model_build_scope():
|
||||
moe.in_speculative_scope = True
|
||||
# Boundaries capture this resolved preference while the draft builds.
|
||||
# Restoring it cannot change an already constructed target plan.
|
||||
with get_exec().comm.override(
|
||||
boundary_reduction=get_spec().speculative_boundary_reduction
|
||||
with (
|
||||
get_exec().comm.override(
|
||||
boundary_reduction=get_spec().speculative_boundary_reduction
|
||||
),
|
||||
_draft_w4a4_mxfp4_megamoe_override(),
|
||||
):
|
||||
yield
|
||||
finally:
|
||||
@@ -570,6 +582,15 @@ def draft_model_build_scope():
|
||||
moe.disable_shared_experts_fusion = original_fusion
|
||||
|
||||
|
||||
def _draft_w4a4_mxfp4_megamoe_override() -> AbstractContextManager:
|
||||
# FusedMoE pins its MegaMoE MMA type at construction, so the draft's layers
|
||||
# keep this value after the scope exits.
|
||||
draft_w4a4 = get_spec().speculative_enable_w4a4_mxfp4_megamoe
|
||||
if draft_w4a4 is None:
|
||||
return nullcontext()
|
||||
return get_exec().moe.override(enable_w4a4_mxfp4_megamoe=draft_w4a4)
|
||||
|
||||
|
||||
def install_shared_experts_fusion_decision(
|
||||
model_class, hf_config, quant_config
|
||||
) -> None:
|
||||
|
||||
@@ -700,7 +700,7 @@ class Mxfp4MoEMethod(FusedMoEMethodBase):
|
||||
check_mega_moe_shapes,
|
||||
)
|
||||
|
||||
mma_type = _mega_moe_mma_type()
|
||||
mma_type = _mega_moe_mma_type(layer)
|
||||
check_mega_moe_shapes(
|
||||
hidden=layer.w13_weight.shape[2] * 2,
|
||||
intermediate=layer.w13_weight.shape[1] // 2,
|
||||
|
||||
@@ -788,6 +788,7 @@ class KimiK3MoE(nn.Module):
|
||||
from sglang.srt.layers.moe.mega_moe import (
|
||||
_configure_mega_moe_deep_gemm_num_sms,
|
||||
_get_mega_moe_symm_buffer,
|
||||
_mega_moe_mma_type,
|
||||
)
|
||||
from sglang.srt.runtime_context import get_parallel
|
||||
|
||||
@@ -813,6 +814,7 @@ class KimiK3MoE(nn.Module):
|
||||
num_topk=self._mega_top_k,
|
||||
hidden=self.moe_hidden_size,
|
||||
intermediate_hidden=self._mega_intermediate_size,
|
||||
mma_type=_mega_moe_mma_type(self.experts),
|
||||
)
|
||||
|
||||
if num_tokens > 0:
|
||||
|
||||
Reference in New Issue
Block a user