diff --git a/docs/docs/advanced_features/server_arguments.mdx b/docs/docs/advanced_features/server_arguments.mdx index 466b858bce..cb3fea10a5 100644 --- a/docs/docs/advanced_features/server_arguments.mdx +++ b/docs/docs/advanced_features/server_arguments.mdx @@ -1647,6 +1647,12 @@ Combining `--enable-response-store` with `--disaggregation-mode=prefill` or `dec MOE A2A backend for EAGLE speculative decoding, see `--moe-a2a-backend` for options. Same as moe a2a backend if unset. `None` none, deepep, mooncake, nixl, mori, ascend_fuseep, flashinfer, megamoe, pplx + + + `--speculative-enable-w4a4-mxfp4-megamoe` + Whether the draft model's MXFP4 MegaMoE layers use the W4A4 `mxf4xmxf4` MMA type. Pass `--no-speculative-enable-w4a4-mxfp4-megamoe` to keep the draft on `fp8xfp4` (W4A8) while the target runs W4A4. Same as `--enable-w4a4-mxfp4-megamoe` if unset. A draft that runs MXFP4 MegaMoE with a different MMA type from the target may allocate an additional MegaMoE symmetric buffer after the KV pool is sized; lower `--mem-fraction-static` if CUDA-graph capture runs out of memory. + `None` + Type: bool `--speculative-draft-model-quantization` diff --git a/python/sglang/srt/arg_groups/field_order.py b/python/sglang/srt/arg_groups/field_order.py index d38e398395..cd77b9ebb3 100644 --- a/python/sglang/srt/arg_groups/field_order.py +++ b/python/sglang/srt/arg_groups/field_order.py @@ -284,6 +284,7 @@ POSITIONAL_FIELD_ORDER = ( "speculative_draft_window_size", "speculative_moe_runner_backend", "speculative_moe_a2a_backend", + "speculative_enable_w4a4_mxfp4_megamoe", "speculative_draft_model_quantization", "_speculative_draft_quantization_explicitly_set", "speculative_skip_dp_mlp_sync", diff --git a/python/sglang/srt/arg_groups/fields/spec.py b/python/sglang/srt/arg_groups/fields/spec.py index fc35e8bf77..199e865ef1 100644 --- a/python/sglang/srt/arg_groups/fields/spec.py +++ b/python/sglang/srt/arg_groups/fields/spec.py @@ -2,6 +2,7 @@ from __future__ import annotations +import argparse from typing import ( Literal, Optional, @@ -196,6 +197,21 @@ class Spec(msgspec.Struct): resolvable=True, ), ] = None + speculative_enable_w4a4_mxfp4_megamoe: A[ + Optional[bool], + Arg( + help="Whether the draft model's MXFP4 MegaMoE layers use the W4A4 " + "mxf4xmxf4 MMA type (see --enable-w4a4-mxfp4-megamoe). Pass " + "--no-speculative-enable-w4a4-mxfp4-megamoe to keep the draft on " + "fp8xfp4 (W4A8) while the target runs W4A4. Same as " + "--enable-w4a4-mxfp4-megamoe if unset. A draft that runs MXFP4 " + "MegaMoE with a different MMA type from the target may allocate an " + "additional MegaMoE symmetric buffer after the KV pool is sized; " + "lower --mem-fraction-static if CUDA-graph capture runs out of " + "memory.", + action=argparse.BooleanOptionalAction, + ), + ] = None speculative_draft_model_quantization: A[ Optional[str], Arg( diff --git a/python/sglang/srt/layers/moe/fused_moe_triton/layer.py b/python/sglang/srt/layers/moe/fused_moe_triton/layer.py index da18b9ac18..5876b7c1d0 100644 --- a/python/sglang/srt/layers/moe/fused_moe_triton/layer.py +++ b/python/sglang/srt/layers/moe/fused_moe_triton/layer.py @@ -403,6 +403,8 @@ class FusedMoE(torch.nn.Module): # Set by the quant method when it repacks experts for MegaMoE. self._mega_moe_weights_built = False self._mega_moe_nvfp4 = False + # Read at construction: draft_model_build_scope applies the draft's own value. + self._mega_moe_w4a4 = get_exec().moe.enable_w4a4_mxfp4_megamoe self._pending_fp8_shared_weights: dict[tuple[int, str], torch.Tensor] = {} self._pending_fp8_shared_scales: dict[tuple[int, str], torch.Tensor] = {} diff --git a/python/sglang/srt/layers/moe/mega_moe.py b/python/sglang/srt/layers/moe/mega_moe.py index 324594cb10..bb1b8f624d 100644 --- a/python/sglang/srt/layers/moe/mega_moe.py +++ b/python/sglang/srt/layers/moe/mega_moe.py @@ -33,7 +33,6 @@ from sglang.srt.layers.moe.mega_moe_sm90 import ( from sglang.srt.layers.moe.utils import get_moe_a2a_backend from sglang.srt.model_executor.runner import get_is_capture_mode from sglang.srt.models.deepseek_common.utils import _device_sm -from sglang.srt.runtime_context import get_exec from sglang.srt.utils import is_hip, is_sm100_supported if TYPE_CHECKING: @@ -53,10 +52,11 @@ def _use_amd_flydsl_mega_moe() -> bool: return _is_hip and envs.SGLANG_AMD_USE_FLYDSL_MEGA_MOE.get() -def _mega_moe_mma_type(experts=None) -> str: - if experts is not None and experts._mega_moe_nvfp4: +def _mega_moe_mma_type(experts) -> str: + if experts._mega_moe_nvfp4: return "nvfp4xnvfp4" - return "mxf4xmxf4" if get_exec().moe.enable_w4a4_mxfp4_megamoe else "fp8xfp4" + # Per layer: a draft may differ from the target (draft_model_build_scope). + return "mxf4xmxf4" if experts._mega_moe_w4a4 else "fp8xfp4" @functools.lru_cache(maxsize=1) @@ -118,12 +118,11 @@ def _get_mega_moe_symm_buffer( hidden: int, intermediate_hidden: int, num_shared_experts: int = 0, - mma_type: Optional[str] = None, + *, + mma_type: str, ) -> SymmBuffer: import deep_gemm - if mma_type is None: - mma_type = _mega_moe_mma_type() with _configure_mega_moe_deep_gemm_num_sms(deep_gemm): key = ( id(group), @@ -513,7 +512,7 @@ def build_mega_moe_experts_weights(experts) -> None: if getattr(experts, "_mega_moe_weights_built", False): return - mma_type = _mega_moe_mma_type() + mma_type = _mega_moe_mma_type(experts) w13 = experts.w13_weight.data w13_sf_fp32 = experts.w13_weight_scale_inv.data w2 = experts.w2_weight.data diff --git a/python/sglang/srt/layers/moe/utils.py b/python/sglang/srt/layers/moe/utils.py index eff60f3730..25f365694c 100644 --- a/python/sglang/srt/layers/moe/utils.py +++ b/python/sglang/srt/layers/moe/utils.py @@ -2,7 +2,7 @@ from __future__ import annotations import logging import os -from contextlib import contextmanager +from contextlib import AbstractContextManager, contextmanager, nullcontext from dataclasses import dataclass from enum import Enum, IntEnum from typing import NamedTuple @@ -546,9 +546,18 @@ def is_shared_experts_fusion_disabled() -> bool: @contextmanager def draft_model_build_scope(): - """Brackets a draft model's CONSTRUCTION: the gates it runs record their - fusion decision on the speculative leaf as well, and the target's ACTIVE - value returns on exit. + """Brackets a draft model's CONSTRUCTION with the draft's construction-time + settings; every value below returns to the target's on exit, including on + error. + + - Shared-experts fusion: the gates the draft runs also record their + decision on the speculative leaf; the target's ACTIVE + ``disable_shared_experts_fusion`` is restored. + - ``boundary_reduction`` is ``--speculative-boundary-reduction``. + Boundaries built here keep the resolved value after exit. + - ``enable_w4a4_mxfp4_megamoe`` is ``--speculative-enable-w4a4-mxfp4-megamoe`` + when set, else inherited. ``FusedMoE`` layers built here keep the + draft's MegaMoE MMA type after exit. Deliberately does not touch ``runner_backend`` — swapping that is ``speculative_moe_backend_context``'s job and has to bracket the draft's @@ -561,8 +570,11 @@ def draft_model_build_scope(): moe.in_speculative_scope = True # Boundaries capture this resolved preference while the draft builds. # Restoring it cannot change an already constructed target plan. - with get_exec().comm.override( - boundary_reduction=get_spec().speculative_boundary_reduction + with ( + get_exec().comm.override( + boundary_reduction=get_spec().speculative_boundary_reduction + ), + _draft_w4a4_mxfp4_megamoe_override(), ): yield finally: @@ -570,6 +582,15 @@ def draft_model_build_scope(): moe.disable_shared_experts_fusion = original_fusion +def _draft_w4a4_mxfp4_megamoe_override() -> AbstractContextManager: + # FusedMoE pins its MegaMoE MMA type at construction, so the draft's layers + # keep this value after the scope exits. + draft_w4a4 = get_spec().speculative_enable_w4a4_mxfp4_megamoe + if draft_w4a4 is None: + return nullcontext() + return get_exec().moe.override(enable_w4a4_mxfp4_megamoe=draft_w4a4) + + def install_shared_experts_fusion_decision( model_class, hf_config, quant_config ) -> None: diff --git a/python/sglang/srt/layers/quantization/mxfp4.py b/python/sglang/srt/layers/quantization/mxfp4.py index 81883eb084..9c5e2fee1b 100644 --- a/python/sglang/srt/layers/quantization/mxfp4.py +++ b/python/sglang/srt/layers/quantization/mxfp4.py @@ -700,7 +700,7 @@ class Mxfp4MoEMethod(FusedMoEMethodBase): check_mega_moe_shapes, ) - mma_type = _mega_moe_mma_type() + mma_type = _mega_moe_mma_type(layer) check_mega_moe_shapes( hidden=layer.w13_weight.shape[2] * 2, intermediate=layer.w13_weight.shape[1] // 2, diff --git a/python/sglang/srt/models/kimi_k3.py b/python/sglang/srt/models/kimi_k3.py index be3c49e418..6afa17e1d9 100644 --- a/python/sglang/srt/models/kimi_k3.py +++ b/python/sglang/srt/models/kimi_k3.py @@ -788,6 +788,7 @@ class KimiK3MoE(nn.Module): from sglang.srt.layers.moe.mega_moe import ( _configure_mega_moe_deep_gemm_num_sms, _get_mega_moe_symm_buffer, + _mega_moe_mma_type, ) from sglang.srt.runtime_context import get_parallel @@ -813,6 +814,7 @@ class KimiK3MoE(nn.Module): num_topk=self._mega_top_k, hidden=self.moe_hidden_size, intermediate_hidden=self._mega_intermediate_size, + mma_type=_mega_moe_mma_type(self.experts), ) if num_tokens > 0: diff --git a/test/registered/unit/layers/moe/test_mega_moe_deepgemm_api.py b/test/registered/unit/layers/moe/test_mega_moe_deepgemm_api.py index 8a706bf962..6bb7736a82 100644 --- a/test/registered/unit/layers/moe/test_mega_moe_deepgemm_api.py +++ b/test/registered/unit/layers/moe/test_mega_moe_deepgemm_api.py @@ -13,8 +13,11 @@ from sglang.test.test_utils import CustomTestCase, maybe_stub_sgl_kernel maybe_stub_sgl_kernel() -from sglang.srt.layers.moe import mega_moe -from sglang.srt.runtime_context import get_parallel +from sglang.srt.layers.moe import MoeA2ABackend, MoeRunnerBackend, mega_moe +from sglang.srt.layers.moe.fused_moe_triton import layer as fused_moe_layer_module +from sglang.srt.layers.moe.utils import draft_model_build_scope +from sglang.srt.layers.quantization.unquant import UnquantizedFusedMoEMethod +from sglang.srt.runtime_context import get_context, get_exec, get_flags, get_parallel register_cpu_ci(est_time=8, suite="base-a-test-cpu") @@ -45,15 +48,7 @@ class TestDeepGemmMegaMoeApi(CustomTestCase): deep_gemm.get_symm_buffer_for_mega_moe = MagicMock(return_value=expected_buffer) group = object() - with ( - patch.dict(sys.modules, {"deep_gemm": deep_gemm}), - patch.object( - mega_moe, - "_mega_moe_mma_type", - return_value="mxf4xmxf4", - create=True, - ), - ): + with patch.dict(sys.modules, {"deep_gemm": deep_gemm}): actual_buffer = mega_moe._get_mega_moe_symm_buffer( group, num_experts=8, @@ -61,6 +56,7 @@ class TestDeepGemmMegaMoeApi(CustomTestCase): num_topk=2, hidden=128, intermediate_hidden=256, + mma_type="mxf4xmxf4", ) self.assertIs(actual_buffer, expected_buffer) @@ -74,14 +70,7 @@ class TestDeepGemmMegaMoeApi(CustomTestCase): deep_gemm.get_symm_buffer_for_mega_moe = MagicMock(side_effect=expected_buffers) group = object() - with ( - patch.dict(sys.modules, {"deep_gemm": deep_gemm}), - patch.object( - mega_moe, - "_mega_moe_mma_type", - side_effect=("fp8xfp4", "mxf4xmxf4"), - ), - ): + with patch.dict(sys.modules, {"deep_gemm": deep_gemm}): actual_buffers = tuple( mega_moe._get_mega_moe_symm_buffer( group, @@ -90,8 +79,9 @@ class TestDeepGemmMegaMoeApi(CustomTestCase): num_topk=2, hidden=128, intermediate_hidden=256, + mma_type=mma_type, ) - for _ in range(2) + for mma_type in ("fp8xfp4", "mxf4xmxf4") ) self.assertEqual(actual_buffers, expected_buffers) @@ -406,11 +396,88 @@ class TestDeepGemmMegaMoeApi(CustomTestCase): torch.testing.assert_close(actual, expected) - def _get_test_buffer(self, group): - with ( - patch.dict(sys.modules, {"deep_gemm": self.deep_gemm}), - patch.object(mega_moe, "_mega_moe_mma_type", return_value="fp8xfp4"), + def test_draft_layers_keep_their_own_w4a4_choice(self): + """Draft MegaMoE layers must not follow the target's MMA type once built. + + The draft forward runs outside draft_model_build_scope, so a layer that + re-read the global flag would pair draft weights with the wrong kernel. + """ + for target_flag, draft_flag, expected_target, expected_draft in ( + (True, False, "mxf4xmxf4", "fp8xfp4"), + (True, None, "mxf4xmxf4", "mxf4xmxf4"), + (False, True, "fp8xfp4", "mxf4xmxf4"), + (False, None, "fp8xfp4", "fp8xfp4"), ): + with self.subTest( + enable_w4a4_mxfp4_megamoe=target_flag, + speculative_enable_w4a4_mxfp4_megamoe=draft_flag, + ): + with get_context().override_server_args( + model_path="dummy", + enable_w4a4_mxfp4_megamoe=target_flag, + speculative_enable_w4a4_mxfp4_megamoe=draft_flag, + ): + target = self._build_fused_moe() + with draft_model_build_scope(): + draft = self._build_fused_moe() + + self.assertEqual( + get_exec().moe.enable_w4a4_mxfp4_megamoe, target_flag + ) + self.assertEqual( + mega_moe._mega_moe_mma_type(target), expected_target + ) + self.assertEqual(mega_moe._mega_moe_mma_type(draft), expected_draft) + + def test_draft_build_scope_restores_w4a4_on_exception(self): + """A failed draft build must not leave the target on the draft's MMA type.""" + with get_context().override_server_args( + model_path="dummy", + enable_w4a4_mxfp4_megamoe=False, + speculative_enable_w4a4_mxfp4_megamoe=True, + ): + with self.assertRaisesRegex(RuntimeError, "draft build failed"): + with draft_model_build_scope(): + self.assertTrue(get_exec().moe.enable_w4a4_mxfp4_megamoe) + raise RuntimeError("draft build failed") + + self.assertFalse(get_exec().moe.enable_w4a4_mxfp4_megamoe) + target = self._build_fused_moe() + self.assertEqual(mega_moe._mega_moe_mma_type(target), "fp8xfp4") + + def _build_fused_moe(self): + method = UnquantizedFusedMoEMethod() + with ( + patch.object(method, "create_weights"), + patch.object(method, "create_moe_runner"), + patch.object( + fused_moe_layer_module, + "create_moe_dispatcher", + return_value=SimpleNamespace(), + ), + get_flags().moe.override( + runner_backend=MoeRunnerBackend.AUTO, + a2a_backend=MoeA2ABackend.MEGAMOE, + ), + get_parallel().override( + moe_ep_size=1, + moe_ep_rank=0, + moe_tp_size=1, + moe_tp_rank=0, + tp_size=1, + tp_rank=0, + ), + ): + return fused_moe_layer_module.FusedMoE( + num_experts=2, + hidden_size=4, + intermediate_size=8, + layer_id=0, + quant_method=method, + ) + + def _get_test_buffer(self, group): + with patch.dict(sys.modules, {"deep_gemm": self.deep_gemm}): return mega_moe._get_mega_moe_symm_buffer( group, num_experts=8, @@ -418,6 +485,7 @@ class TestDeepGemmMegaMoeApi(CustomTestCase): num_topk=2, hidden=128, intermediate_hidden=256, + mma_type="fp8xfp4", ) diff --git a/test/registered/unit/server_args/test_server_args.py b/test/registered/unit/server_args/test_server_args.py index d94dbf3f68..0a4be745f2 100644 --- a/test/registered/unit/server_args/test_server_args.py +++ b/test/registered/unit/server_args/test_server_args.py @@ -208,6 +208,24 @@ class TestPrepareServerArgs(CustomTestCase): self.assertEqual(os.environ["DG_USE_FP4_ACTS"], "0") self.assertEqual(os.environ["DG_USE_MXF4_KIND"], "0") + def test_speculative_w4a4_mxfp4_megamoe_is_tri_state(self): + # Unset must stay None, not False: None is what lets the draft inherit + # --enable-w4a4-mxfp4-megamoe. + for flag, expected in ( + ([], None), + (["--speculative-enable-w4a4-mxfp4-megamoe"], True), + (["--no-speculative-enable-w4a4-mxfp4-megamoe"], False), + ): + with self.subTest(flag=flag): + args = prepare_server_args( + ["--model-path", "dummy", "--enable-w4a4-mxfp4-megamoe", *flag] + ) + args.resolve_once() + self.assertIs( + resolution_result(args, "speculative_enable_w4a4_mxfp4_megamoe"), + expected, + ) + def test_megamoe_rejects_two_batch_overlap(self): # The fused kernel has no dispatch/combine split for the TBO ops to call. with override_platform(is_cuda=True, is_sm90=False, is_sm100=True):