diff --git a/docs/docs/advanced_features/server_arguments.mdx b/docs/docs/advanced_features/server_arguments.mdx
index 466b858bce..cb3fea10a5 100644
--- a/docs/docs/advanced_features/server_arguments.mdx
+++ b/docs/docs/advanced_features/server_arguments.mdx
@@ -1647,6 +1647,12 @@ Combining `--enable-response-store` with `--disaggregation-mode=prefill` or `dec
MOE A2A backend for EAGLE speculative decoding, see `--moe-a2a-backend` for options. Same as moe a2a backend if unset. |
`None` |
none, deepep, mooncake, nixl, mori, ascend_fuseep, flashinfer, megamoe, pplx |
+
+
+ | `--speculative-enable-w4a4-mxfp4-megamoe` |
+ Whether the draft model's MXFP4 MegaMoE layers use the W4A4 `mxf4xmxf4` MMA type. Pass `--no-speculative-enable-w4a4-mxfp4-megamoe` to keep the draft on `fp8xfp4` (W4A8) while the target runs W4A4. Same as `--enable-w4a4-mxfp4-megamoe` if unset. A draft that runs MXFP4 MegaMoE with a different MMA type from the target may allocate an additional MegaMoE symmetric buffer after the KV pool is sized; lower `--mem-fraction-static` if CUDA-graph capture runs out of memory. |
+ `None` |
+ Type: bool |
| `--speculative-draft-model-quantization` |
diff --git a/python/sglang/srt/arg_groups/field_order.py b/python/sglang/srt/arg_groups/field_order.py
index d38e398395..cd77b9ebb3 100644
--- a/python/sglang/srt/arg_groups/field_order.py
+++ b/python/sglang/srt/arg_groups/field_order.py
@@ -284,6 +284,7 @@ POSITIONAL_FIELD_ORDER = (
"speculative_draft_window_size",
"speculative_moe_runner_backend",
"speculative_moe_a2a_backend",
+ "speculative_enable_w4a4_mxfp4_megamoe",
"speculative_draft_model_quantization",
"_speculative_draft_quantization_explicitly_set",
"speculative_skip_dp_mlp_sync",
diff --git a/python/sglang/srt/arg_groups/fields/spec.py b/python/sglang/srt/arg_groups/fields/spec.py
index fc35e8bf77..199e865ef1 100644
--- a/python/sglang/srt/arg_groups/fields/spec.py
+++ b/python/sglang/srt/arg_groups/fields/spec.py
@@ -2,6 +2,7 @@
from __future__ import annotations
+import argparse
from typing import (
Literal,
Optional,
@@ -196,6 +197,21 @@ class Spec(msgspec.Struct):
resolvable=True,
),
] = None
+ speculative_enable_w4a4_mxfp4_megamoe: A[
+ Optional[bool],
+ Arg(
+ help="Whether the draft model's MXFP4 MegaMoE layers use the W4A4 "
+ "mxf4xmxf4 MMA type (see --enable-w4a4-mxfp4-megamoe). Pass "
+ "--no-speculative-enable-w4a4-mxfp4-megamoe to keep the draft on "
+ "fp8xfp4 (W4A8) while the target runs W4A4. Same as "
+ "--enable-w4a4-mxfp4-megamoe if unset. A draft that runs MXFP4 "
+ "MegaMoE with a different MMA type from the target may allocate an "
+ "additional MegaMoE symmetric buffer after the KV pool is sized; "
+ "lower --mem-fraction-static if CUDA-graph capture runs out of "
+ "memory.",
+ action=argparse.BooleanOptionalAction,
+ ),
+ ] = None
speculative_draft_model_quantization: A[
Optional[str],
Arg(
diff --git a/python/sglang/srt/layers/moe/fused_moe_triton/layer.py b/python/sglang/srt/layers/moe/fused_moe_triton/layer.py
index da18b9ac18..5876b7c1d0 100644
--- a/python/sglang/srt/layers/moe/fused_moe_triton/layer.py
+++ b/python/sglang/srt/layers/moe/fused_moe_triton/layer.py
@@ -403,6 +403,8 @@ class FusedMoE(torch.nn.Module):
# Set by the quant method when it repacks experts for MegaMoE.
self._mega_moe_weights_built = False
self._mega_moe_nvfp4 = False
+ # Read at construction: draft_model_build_scope applies the draft's own value.
+ self._mega_moe_w4a4 = get_exec().moe.enable_w4a4_mxfp4_megamoe
self._pending_fp8_shared_weights: dict[tuple[int, str], torch.Tensor] = {}
self._pending_fp8_shared_scales: dict[tuple[int, str], torch.Tensor] = {}
diff --git a/python/sglang/srt/layers/moe/mega_moe.py b/python/sglang/srt/layers/moe/mega_moe.py
index 324594cb10..bb1b8f624d 100644
--- a/python/sglang/srt/layers/moe/mega_moe.py
+++ b/python/sglang/srt/layers/moe/mega_moe.py
@@ -33,7 +33,6 @@ from sglang.srt.layers.moe.mega_moe_sm90 import (
from sglang.srt.layers.moe.utils import get_moe_a2a_backend
from sglang.srt.model_executor.runner import get_is_capture_mode
from sglang.srt.models.deepseek_common.utils import _device_sm
-from sglang.srt.runtime_context import get_exec
from sglang.srt.utils import is_hip, is_sm100_supported
if TYPE_CHECKING:
@@ -53,10 +52,11 @@ def _use_amd_flydsl_mega_moe() -> bool:
return _is_hip and envs.SGLANG_AMD_USE_FLYDSL_MEGA_MOE.get()
-def _mega_moe_mma_type(experts=None) -> str:
- if experts is not None and experts._mega_moe_nvfp4:
+def _mega_moe_mma_type(experts) -> str:
+ if experts._mega_moe_nvfp4:
return "nvfp4xnvfp4"
- return "mxf4xmxf4" if get_exec().moe.enable_w4a4_mxfp4_megamoe else "fp8xfp4"
+ # Per layer: a draft may differ from the target (draft_model_build_scope).
+ return "mxf4xmxf4" if experts._mega_moe_w4a4 else "fp8xfp4"
@functools.lru_cache(maxsize=1)
@@ -118,12 +118,11 @@ def _get_mega_moe_symm_buffer(
hidden: int,
intermediate_hidden: int,
num_shared_experts: int = 0,
- mma_type: Optional[str] = None,
+ *,
+ mma_type: str,
) -> SymmBuffer:
import deep_gemm
- if mma_type is None:
- mma_type = _mega_moe_mma_type()
with _configure_mega_moe_deep_gemm_num_sms(deep_gemm):
key = (
id(group),
@@ -513,7 +512,7 @@ def build_mega_moe_experts_weights(experts) -> None:
if getattr(experts, "_mega_moe_weights_built", False):
return
- mma_type = _mega_moe_mma_type()
+ mma_type = _mega_moe_mma_type(experts)
w13 = experts.w13_weight.data
w13_sf_fp32 = experts.w13_weight_scale_inv.data
w2 = experts.w2_weight.data
diff --git a/python/sglang/srt/layers/moe/utils.py b/python/sglang/srt/layers/moe/utils.py
index eff60f3730..25f365694c 100644
--- a/python/sglang/srt/layers/moe/utils.py
+++ b/python/sglang/srt/layers/moe/utils.py
@@ -2,7 +2,7 @@ from __future__ import annotations
import logging
import os
-from contextlib import contextmanager
+from contextlib import AbstractContextManager, contextmanager, nullcontext
from dataclasses import dataclass
from enum import Enum, IntEnum
from typing import NamedTuple
@@ -546,9 +546,18 @@ def is_shared_experts_fusion_disabled() -> bool:
@contextmanager
def draft_model_build_scope():
- """Brackets a draft model's CONSTRUCTION: the gates it runs record their
- fusion decision on the speculative leaf as well, and the target's ACTIVE
- value returns on exit.
+ """Brackets a draft model's CONSTRUCTION with the draft's construction-time
+ settings; every value below returns to the target's on exit, including on
+ error.
+
+ - Shared-experts fusion: the gates the draft runs also record their
+ decision on the speculative leaf; the target's ACTIVE
+ ``disable_shared_experts_fusion`` is restored.
+ - ``boundary_reduction`` is ``--speculative-boundary-reduction``.
+ Boundaries built here keep the resolved value after exit.
+ - ``enable_w4a4_mxfp4_megamoe`` is ``--speculative-enable-w4a4-mxfp4-megamoe``
+ when set, else inherited. ``FusedMoE`` layers built here keep the
+ draft's MegaMoE MMA type after exit.
Deliberately does not touch ``runner_backend`` — swapping that is
``speculative_moe_backend_context``'s job and has to bracket the draft's
@@ -561,8 +570,11 @@ def draft_model_build_scope():
moe.in_speculative_scope = True
# Boundaries capture this resolved preference while the draft builds.
# Restoring it cannot change an already constructed target plan.
- with get_exec().comm.override(
- boundary_reduction=get_spec().speculative_boundary_reduction
+ with (
+ get_exec().comm.override(
+ boundary_reduction=get_spec().speculative_boundary_reduction
+ ),
+ _draft_w4a4_mxfp4_megamoe_override(),
):
yield
finally:
@@ -570,6 +582,15 @@ def draft_model_build_scope():
moe.disable_shared_experts_fusion = original_fusion
+def _draft_w4a4_mxfp4_megamoe_override() -> AbstractContextManager:
+ # FusedMoE pins its MegaMoE MMA type at construction, so the draft's layers
+ # keep this value after the scope exits.
+ draft_w4a4 = get_spec().speculative_enable_w4a4_mxfp4_megamoe
+ if draft_w4a4 is None:
+ return nullcontext()
+ return get_exec().moe.override(enable_w4a4_mxfp4_megamoe=draft_w4a4)
+
+
def install_shared_experts_fusion_decision(
model_class, hf_config, quant_config
) -> None:
diff --git a/python/sglang/srt/layers/quantization/mxfp4.py b/python/sglang/srt/layers/quantization/mxfp4.py
index 81883eb084..9c5e2fee1b 100644
--- a/python/sglang/srt/layers/quantization/mxfp4.py
+++ b/python/sglang/srt/layers/quantization/mxfp4.py
@@ -700,7 +700,7 @@ class Mxfp4MoEMethod(FusedMoEMethodBase):
check_mega_moe_shapes,
)
- mma_type = _mega_moe_mma_type()
+ mma_type = _mega_moe_mma_type(layer)
check_mega_moe_shapes(
hidden=layer.w13_weight.shape[2] * 2,
intermediate=layer.w13_weight.shape[1] // 2,
diff --git a/python/sglang/srt/models/kimi_k3.py b/python/sglang/srt/models/kimi_k3.py
index be3c49e418..6afa17e1d9 100644
--- a/python/sglang/srt/models/kimi_k3.py
+++ b/python/sglang/srt/models/kimi_k3.py
@@ -788,6 +788,7 @@ class KimiK3MoE(nn.Module):
from sglang.srt.layers.moe.mega_moe import (
_configure_mega_moe_deep_gemm_num_sms,
_get_mega_moe_symm_buffer,
+ _mega_moe_mma_type,
)
from sglang.srt.runtime_context import get_parallel
@@ -813,6 +814,7 @@ class KimiK3MoE(nn.Module):
num_topk=self._mega_top_k,
hidden=self.moe_hidden_size,
intermediate_hidden=self._mega_intermediate_size,
+ mma_type=_mega_moe_mma_type(self.experts),
)
if num_tokens > 0:
diff --git a/test/registered/unit/layers/moe/test_mega_moe_deepgemm_api.py b/test/registered/unit/layers/moe/test_mega_moe_deepgemm_api.py
index 8a706bf962..6bb7736a82 100644
--- a/test/registered/unit/layers/moe/test_mega_moe_deepgemm_api.py
+++ b/test/registered/unit/layers/moe/test_mega_moe_deepgemm_api.py
@@ -13,8 +13,11 @@ from sglang.test.test_utils import CustomTestCase, maybe_stub_sgl_kernel
maybe_stub_sgl_kernel()
-from sglang.srt.layers.moe import mega_moe
-from sglang.srt.runtime_context import get_parallel
+from sglang.srt.layers.moe import MoeA2ABackend, MoeRunnerBackend, mega_moe
+from sglang.srt.layers.moe.fused_moe_triton import layer as fused_moe_layer_module
+from sglang.srt.layers.moe.utils import draft_model_build_scope
+from sglang.srt.layers.quantization.unquant import UnquantizedFusedMoEMethod
+from sglang.srt.runtime_context import get_context, get_exec, get_flags, get_parallel
register_cpu_ci(est_time=8, suite="base-a-test-cpu")
@@ -45,15 +48,7 @@ class TestDeepGemmMegaMoeApi(CustomTestCase):
deep_gemm.get_symm_buffer_for_mega_moe = MagicMock(return_value=expected_buffer)
group = object()
- with (
- patch.dict(sys.modules, {"deep_gemm": deep_gemm}),
- patch.object(
- mega_moe,
- "_mega_moe_mma_type",
- return_value="mxf4xmxf4",
- create=True,
- ),
- ):
+ with patch.dict(sys.modules, {"deep_gemm": deep_gemm}):
actual_buffer = mega_moe._get_mega_moe_symm_buffer(
group,
num_experts=8,
@@ -61,6 +56,7 @@ class TestDeepGemmMegaMoeApi(CustomTestCase):
num_topk=2,
hidden=128,
intermediate_hidden=256,
+ mma_type="mxf4xmxf4",
)
self.assertIs(actual_buffer, expected_buffer)
@@ -74,14 +70,7 @@ class TestDeepGemmMegaMoeApi(CustomTestCase):
deep_gemm.get_symm_buffer_for_mega_moe = MagicMock(side_effect=expected_buffers)
group = object()
- with (
- patch.dict(sys.modules, {"deep_gemm": deep_gemm}),
- patch.object(
- mega_moe,
- "_mega_moe_mma_type",
- side_effect=("fp8xfp4", "mxf4xmxf4"),
- ),
- ):
+ with patch.dict(sys.modules, {"deep_gemm": deep_gemm}):
actual_buffers = tuple(
mega_moe._get_mega_moe_symm_buffer(
group,
@@ -90,8 +79,9 @@ class TestDeepGemmMegaMoeApi(CustomTestCase):
num_topk=2,
hidden=128,
intermediate_hidden=256,
+ mma_type=mma_type,
)
- for _ in range(2)
+ for mma_type in ("fp8xfp4", "mxf4xmxf4")
)
self.assertEqual(actual_buffers, expected_buffers)
@@ -406,11 +396,88 @@ class TestDeepGemmMegaMoeApi(CustomTestCase):
torch.testing.assert_close(actual, expected)
- def _get_test_buffer(self, group):
- with (
- patch.dict(sys.modules, {"deep_gemm": self.deep_gemm}),
- patch.object(mega_moe, "_mega_moe_mma_type", return_value="fp8xfp4"),
+ def test_draft_layers_keep_their_own_w4a4_choice(self):
+ """Draft MegaMoE layers must not follow the target's MMA type once built.
+
+ The draft forward runs outside draft_model_build_scope, so a layer that
+ re-read the global flag would pair draft weights with the wrong kernel.
+ """
+ for target_flag, draft_flag, expected_target, expected_draft in (
+ (True, False, "mxf4xmxf4", "fp8xfp4"),
+ (True, None, "mxf4xmxf4", "mxf4xmxf4"),
+ (False, True, "fp8xfp4", "mxf4xmxf4"),
+ (False, None, "fp8xfp4", "fp8xfp4"),
):
+ with self.subTest(
+ enable_w4a4_mxfp4_megamoe=target_flag,
+ speculative_enable_w4a4_mxfp4_megamoe=draft_flag,
+ ):
+ with get_context().override_server_args(
+ model_path="dummy",
+ enable_w4a4_mxfp4_megamoe=target_flag,
+ speculative_enable_w4a4_mxfp4_megamoe=draft_flag,
+ ):
+ target = self._build_fused_moe()
+ with draft_model_build_scope():
+ draft = self._build_fused_moe()
+
+ self.assertEqual(
+ get_exec().moe.enable_w4a4_mxfp4_megamoe, target_flag
+ )
+ self.assertEqual(
+ mega_moe._mega_moe_mma_type(target), expected_target
+ )
+ self.assertEqual(mega_moe._mega_moe_mma_type(draft), expected_draft)
+
+ def test_draft_build_scope_restores_w4a4_on_exception(self):
+ """A failed draft build must not leave the target on the draft's MMA type."""
+ with get_context().override_server_args(
+ model_path="dummy",
+ enable_w4a4_mxfp4_megamoe=False,
+ speculative_enable_w4a4_mxfp4_megamoe=True,
+ ):
+ with self.assertRaisesRegex(RuntimeError, "draft build failed"):
+ with draft_model_build_scope():
+ self.assertTrue(get_exec().moe.enable_w4a4_mxfp4_megamoe)
+ raise RuntimeError("draft build failed")
+
+ self.assertFalse(get_exec().moe.enable_w4a4_mxfp4_megamoe)
+ target = self._build_fused_moe()
+ self.assertEqual(mega_moe._mega_moe_mma_type(target), "fp8xfp4")
+
+ def _build_fused_moe(self):
+ method = UnquantizedFusedMoEMethod()
+ with (
+ patch.object(method, "create_weights"),
+ patch.object(method, "create_moe_runner"),
+ patch.object(
+ fused_moe_layer_module,
+ "create_moe_dispatcher",
+ return_value=SimpleNamespace(),
+ ),
+ get_flags().moe.override(
+ runner_backend=MoeRunnerBackend.AUTO,
+ a2a_backend=MoeA2ABackend.MEGAMOE,
+ ),
+ get_parallel().override(
+ moe_ep_size=1,
+ moe_ep_rank=0,
+ moe_tp_size=1,
+ moe_tp_rank=0,
+ tp_size=1,
+ tp_rank=0,
+ ),
+ ):
+ return fused_moe_layer_module.FusedMoE(
+ num_experts=2,
+ hidden_size=4,
+ intermediate_size=8,
+ layer_id=0,
+ quant_method=method,
+ )
+
+ def _get_test_buffer(self, group):
+ with patch.dict(sys.modules, {"deep_gemm": self.deep_gemm}):
return mega_moe._get_mega_moe_symm_buffer(
group,
num_experts=8,
@@ -418,6 +485,7 @@ class TestDeepGemmMegaMoeApi(CustomTestCase):
num_topk=2,
hidden=128,
intermediate_hidden=256,
+ mma_type="fp8xfp4",
)
diff --git a/test/registered/unit/server_args/test_server_args.py b/test/registered/unit/server_args/test_server_args.py
index d94dbf3f68..0a4be745f2 100644
--- a/test/registered/unit/server_args/test_server_args.py
+++ b/test/registered/unit/server_args/test_server_args.py
@@ -208,6 +208,24 @@ class TestPrepareServerArgs(CustomTestCase):
self.assertEqual(os.environ["DG_USE_FP4_ACTS"], "0")
self.assertEqual(os.environ["DG_USE_MXF4_KIND"], "0")
+ def test_speculative_w4a4_mxfp4_megamoe_is_tri_state(self):
+ # Unset must stay None, not False: None is what lets the draft inherit
+ # --enable-w4a4-mxfp4-megamoe.
+ for flag, expected in (
+ ([], None),
+ (["--speculative-enable-w4a4-mxfp4-megamoe"], True),
+ (["--no-speculative-enable-w4a4-mxfp4-megamoe"], False),
+ ):
+ with self.subTest(flag=flag):
+ args = prepare_server_args(
+ ["--model-path", "dummy", "--enable-w4a4-mxfp4-megamoe", *flag]
+ )
+ args.resolve_once()
+ self.assertIs(
+ resolution_result(args, "speculative_enable_w4a4_mxfp4_megamoe"),
+ expected,
+ )
+
def test_megamoe_rejects_two_batch_overlap(self):
# The fused kernel has no dispatch/combine split for the TBO ops to call.
with override_platform(is_cuda=True, is_sm90=False, is_sm100=True):