diff --git a/docs/docs/references/environment_variables.mdx b/docs/docs/references/environment_variables.mdx
index 4e25027e80..947d091589 100644
--- a/docs/docs/references/environment_variables.mdx
+++ b/docs/docs/references/environment_variables.mdx
@@ -363,7 +363,7 @@ SGLang supports various environment variables that can be used to configure its
SGLANG_FLASHINFER_MEGAMOE_IN_KERNEL_FC2_REDUCE |
- Permit CuTe DSL MegaMOE in-kernel FC2 reduction for NVFP4 and MXFP8. FlashInfer's tuning profile may still select external reduction. BF16 atomic accumulation is nondeterministic; leave disabled for reproducibility. Requires BF16 combine. |
+ Permit CuTe DSL MegaMOE in-kernel FC2 reduction for NVFP4 W4A4, NVFP4 W4A16, and MXFP8. FlashInfer's tuning profile may still select external reduction. BF16 atomic accumulation is nondeterministic; leave disabled for reproducibility. Requires BF16 combine. |
"false" |
@@ -821,7 +821,7 @@ SGLang supports various environment variables that can be used to configure its
SGLANG_FLASHINFER_CUTEDSL_NVFP4_W4A16 |
- Use BF16 activations and outputs with FlashInfer CuTe DSL NVFP4 weights on NVIDIA SM100-family GPUs (SM100/SM103). Serialized dense linear layers use this mode with --fp4-gemm-backend flashinfer_cutedsl. MoE layers use it with --moe-runner-backend flashinfer_cutedsl; the MoE path supports online weight quantization and serialized ModelOpt NVFP4 weights with either no A2A or FlashInfer A2A, and honors SGLANG_FLASHINFER_MOE_FUSED_FINALIZE. |
+ Use BF16 activations and outputs with FlashInfer CuTe DSL NVFP4 weights on NVIDIA SM100-family GPUs (SM100/SM103). Serialized dense linear layers use this mode with --fp4-gemm-backend flashinfer_cutedsl. With --moe-runner-backend flashinfer_cutedsl, MoE layers support online weight quantization and serialized ModelOpt NVFP4 weights with either no A2A or FlashInfer A2A, and honor SGLANG_FLASHINFER_MOE_FUSED_FINALIZE. With --moe-runner-backend flashinfer_megamoe --moe-a2a-backend flashinfer_megamoe, MoE layers support serialized ModelOpt NVFP4 weights and --quantization nvfp4_online for gated SiLU/SwiGLU or SiTU. MegaMOE uses BF16 activations and combine, with equal TP, attention-DP, and EP sizes. |
false |
diff --git a/python/sglang/srt/arg_groups/moe_hook.py b/python/sglang/srt/arg_groups/moe_hook.py
index 946a1f875d..8db87a3e85 100644
--- a/python/sglang/srt/arg_groups/moe_hook.py
+++ b/python/sglang/srt/arg_groups/moe_hook.py
@@ -185,6 +185,11 @@ def handle_flashinfer_a2a_dispatch_type(server_args: Any):
def validate_flashinfer_megamoe_envs() -> None:
combine_dtype = envs.SGLANG_FLASHINFER_MEGAMOE_COMBINE_DTYPE.get().strip().lower()
+ if envs.SGLANG_FLASHINFER_CUTEDSL_NVFP4_W4A16.get() and combine_dtype != "bf16":
+ raise ValueError(
+ "FlashInfer MegaMOE NVFP4 W4A16 requires "
+ "SGLANG_FLASHINFER_MEGAMOE_COMBINE_DTYPE=bf16."
+ )
if combine_dtype not in ("bf16", "mxfp8", "nvfp4"):
raise ValueError(
"SGLANG_FLASHINFER_MEGAMOE_COMBINE_DTYPE must be one of "
@@ -215,6 +220,8 @@ def validate_flashinfer_megamoe_model(server_args: Any) -> None:
"NemotronHPuzzleForCausalLM",
"Qwen2MoeForCausalLM",
"Qwen3MoeForCausalLM",
+ "KimiK3ForConditionalGeneration",
+ "KimiK3LinearForCausalLM",
)
if not any(
architecture in validated_architectures for architecture in architectures
@@ -228,13 +235,18 @@ def validate_flashinfer_megamoe_model(server_args: Any) -> None:
quantization = resolved_view(server_args).quantization
supports_megamoe_quantization = (
quantization in ("mxfp8", "modelopt_fp4")
+ or (
+ quantization == "nvfp4_online"
+ and envs.SGLANG_FLASHINFER_CUTEDSL_NVFP4_W4A16.get()
+ )
or model_config.is_fp4_experts
or model_config.nvfp4_moe_meta is not None
)
if not supports_megamoe_quantization:
raise ValueError(
"FlashInfer MegaMOE currently supports only MXFP8, ModelOpt "
- "NVFP4, FP4-expert, or hybrid NVFP4 MoE checkpoints; got "
+ "NVFP4, FP4-expert, hybrid NVFP4 MoE checkpoints, or nvfp4_online "
+ "with SGLANG_FLASHINFER_CUTEDSL_NVFP4_W4A16=1; got "
f"quantization={quantization!r}. Standard FP8 MoE checkpoints "
"are not supported."
)
@@ -274,6 +286,13 @@ def handle_a2a_moe(server_args: Any):
if a2a_backend == "flashinfer_megamoe":
validate_flashinfer_megamoe_model(server_args)
validate_flashinfer_megamoe_envs()
+ if envs.SGLANG_FLASHINFER_CUTEDSL_NVFP4_W4A16.get():
+ import torch
+
+ if model_config_of(server_args).dtype != torch.bfloat16:
+ raise ValueError(
+ "FlashInfer MegaMOE NVFP4 W4A16 requires --dtype bfloat16."
+ )
assert attn_dp_enabled_of(cfg) and cfg.attn_dp_size == cfg.tp_size, (
"FlashInfer MegaMOE is only supported with --attn-dp-size equal to --tp-size"
)
@@ -289,6 +308,22 @@ def handle_a2a_moe(server_args: Any):
"FlashInfer MegaMOE currently requires an SM100-family "
"CUDA device for all supported quantization formats."
)
+ if cfg.nnodes == 1 and (
+ cfg.speculative_algorithm is None
+ or cfg.speculative_moe_a2a_backend in (None, "none")
+ ):
+ # An unset draft A2A backend defaults to none for this target.
+ # MegaMoE kernels use directly mapped peer memory. Avoid NVSHMEM's
+ # uninitialized proxy global-exit request; see
+ # https://github.com/NVIDIA/nvshmem/pull/110.
+ # Disable remote transports too: the local-only flag cannot disable
+ # a full transport proxy.
+ # Device-side global_exit and proxy timeout polling are unavailable
+ # with these defaults. Preserve explicit NVSHMEM configuration and
+ # exclude drafts with their own A2A backend: these are process-wide.
+ os.environ.setdefault("NVSHMEM_REMOTE_TRANSPORT", "none")
+ os.environ.setdefault("NVSHMEM_IB_ENABLE_IBGDA", "0")
+ os.environ.setdefault("NVSHMEM_DISABLE_LOCAL_ONLY_PROXY", "1")
logger.info(
"FlashInfer MegaMOE is enabled. The expert parallel size is "
"adjusted to be the same as the tensor parallel size[%s].",
diff --git a/python/sglang/srt/arg_groups/overrides.py b/python/sglang/srt/arg_groups/overrides.py
index b30360d623..a20194b744 100644
--- a/python/sglang/srt/arg_groups/overrides.py
+++ b/python/sglang/srt/arg_groups/overrides.py
@@ -1432,22 +1432,30 @@ def _moe_runner_backend_quant_constraints(view: Any) -> dict:
"""Resolve MoE runner backends from quantization."""
moe_runner_backend = view.moe_runner_backend
if view.quantization == "nvfp4_online":
+ use_megamoe_w4a16 = (
+ view.moe_a2a_backend == "flashinfer_megamoe"
+ and envs.SGLANG_FLASHINFER_CUTEDSL_NVFP4_W4A16.get()
+ )
if not get_platform().is_sm100:
raise ValueError(
"--quantization nvfp4_online is supported only on "
"NVIDIA Blackwell SM100/SM103 GPUs."
)
if moe_runner_backend == "auto":
- moe_runner_backend = "flashinfer_trtllm"
+ moe_runner_backend = (
+ "flashinfer_megamoe" if use_megamoe_w4a16 else "flashinfer_trtllm"
+ )
elif moe_runner_backend not in [
"flashinfer_trtllm",
"flashinfer_trtllm_routed",
"flashinfer_cutedsl",
- ]:
+ ] and not (moe_runner_backend == "flashinfer_megamoe" and use_megamoe_w4a16):
raise ValueError(
"--quantization nvfp4_online supports only "
"--moe-runner-backend flashinfer_trtllm or "
- "flashinfer_trtllm_routed, or flashinfer_cutedsl."
+ "flashinfer_trtllm_routed, or flashinfer_cutedsl; "
+ "flashinfer_megamoe requires --moe-a2a-backend flashinfer_megamoe "
+ "and SGLANG_FLASHINFER_CUTEDSL_NVFP4_W4A16=1."
)
# Ascend runs MXFP8 MoE on the Ascend runner; every backend selected below is
# CUDA/ROCm-only. Forcing one here would not merely pick the wrong runner:
diff --git a/python/sglang/srt/environ.py b/python/sglang/srt/environ.py
index a37a5c09ca..284df9b3be 100644
--- a/python/sglang/srt/environ.py
+++ b/python/sglang/srt/environ.py
@@ -1104,8 +1104,8 @@ class Envs:
# (derived from cuda_graph_max_bs / chunked_prefill_size when unset).
SGLANG_FLASHINFER_MEGAMOE_MAX_TOKENS_PER_RANK = EnvInt(0)
# Opt-in in-kernel FC2 top-k reduce (cross-rank REDG atomic-add) for the
- # cutedsl mega kernels (NVFP4 / MXFP8). Permits in-kernel reduction;
- # FlashInfer's tuning profile may still select external reduction.
+ # cutedsl mega kernels (NVFP4 W4A4 / W4A16 and MXFP8). Permits in-kernel
+ # reduction; FlashInfer's tuning profile may still select external reduction.
# BF16 atomic accumulation is nondeterministic -- keep off for
# bit-reproducibility.
# No effect on the DeepGEMM (block-FP8) mega path, which lacks the knob.
@@ -1118,7 +1118,8 @@ class Envs:
# Enable per-token FP32 activation scaling for serialized ModelOpt FP4 with
# FlashInfer TRT-LLM or CuTe DSL v2 MoE.
SGLANG_FLASHINFER_NVFP4_PER_TOKEN_ACTIVATION = EnvBool(False)
- # Use BF16 activations with FlashInfer CuTe DSL NVFP4 dense and MoE weights.
+ # Use BF16 activations with FlashInfer CuTe DSL NVFP4 dense and MoE weights,
+ # including the flashinfer_megamoe runner (BF16 combine).
SGLANG_FLASHINFER_CUTEDSL_NVFP4_W4A16 = EnvBool(False)
# Launch the TRT-LLM MoE grouped GEMMs with PDL only at or below this
# token count.
diff --git a/python/sglang/srt/layers/moe/flashinfer_megamoe.py b/python/sglang/srt/layers/moe/flashinfer_megamoe.py
index 1726338398..1c32bb29e9 100644
--- a/python/sglang/srt/layers/moe/flashinfer_megamoe.py
+++ b/python/sglang/srt/layers/moe/flashinfer_megamoe.py
@@ -347,41 +347,69 @@ def ensure_fp4_moe_layer_for_flashinfer_megamoe(layer: FusedMoE) -> Any:
)
+def _nvfp4_megamoe_activation_kwargs(config: MoeRunnerConfig) -> dict[str, Any]:
+ if config.is_gated and config.activation == "silu":
+ return {"activation": "swiglu", "gate_up_clamp": config.swiglu_limit}
+ if config.is_gated and config.activation == "situ":
+ # Kimi K3 stores SiTU's tanh constants in the existing GEMM1 fields.
+ # The up-branch tanh scale is not a SwiGLU hard-clamp limit.
+ return {
+ "activation": "situ",
+ "situ_beta": config.gemm1_alpha,
+ "situ_linear_beta": config.gemm1_clamp_limit,
+ "gate_up_clamp": None,
+ }
+ raise ValueError("FlashInfer NVFP4 MegaMOE only supports gated SiLU or SiTU.")
+
+
def ensure_nvfp4_moe_layer_for_flashinfer_megamoe(layer: FusedMoE) -> Any:
mega = _get_or_init_flashinfer_megamoe_layer_state(layer)
if mega is not None:
return mega
- from flashinfer.moe_ep import Nvfp4CutedslMegaMoeConfig
+ activation_kwargs = _nvfp4_megamoe_activation_kwargs(layer.moe_runner_config)
+ if envs.SGLANG_FLASHINFER_CUTEDSL_NVFP4_W4A16.get():
+ from flashinfer.moe_ep import Sm100_Bf16_Nvfp4_Bf16_Cutedsl_MegaMoeConfig
- input_norm_const = layer._flashinfer_megamoe_input_norm_const
- if input_norm_const is None:
- if torch.cuda.is_available() and torch.cuda.is_current_stream_capturing():
- raise RuntimeError(
- "FlashInfer NVFP4 MegaMOE layer must be initialized before "
- "CUDA graph capture."
+ megakernel_config = Sm100_Bf16_Nvfp4_Bf16_Cutedsl_MegaMoeConfig(
+ intermediate_size=layer.intermediate_size_per_partition,
+ top_k=layer.top_k,
+ enable_in_kernel_fc2_reduce=envs.SGLANG_FLASHINFER_MEGAMOE_IN_KERNEL_FC2_REDUCE.get(),
+ **activation_kwargs,
+ )
+ else:
+ from flashinfer.moe_ep import Nvfp4CutedslMegaMoeConfig
+
+ input_norm_const = layer._flashinfer_megamoe_input_norm_const
+ if input_norm_const is None:
+ if torch.cuda.is_available() and torch.cuda.is_current_stream_capturing():
+ raise RuntimeError(
+ "FlashInfer NVFP4 MegaMOE layer must be initialized before "
+ "CUDA graph capture."
+ )
+ logger.warning(
+ "FlashInfer NVFP4 MegaMOE layer[%s]: input_norm_const was not "
+ "precomputed at weight-load time; computing it lazily now via a "
+ "blocking device sync. This should only happen once per layer, "
+ "but if it happens during warmup it will look like a stall.",
+ layer.layer_id,
)
- logger.warning(
- "FlashInfer NVFP4 MegaMOE layer[%s]: input_norm_const was not "
- "precomputed at weight-load time; computing it lazily now via a "
- "blocking device sync. This should only happen once per layer, "
- "but if it happens during warmup it will look like a stall.",
- layer.layer_id,
- )
- input_norm_const = _scalar_float(layer.w13_input_scale_quant)
- layer._flashinfer_megamoe_input_norm_const = input_norm_const
+ input_norm_const = _scalar_float(layer.w13_input_scale_quant)
+ layer._flashinfer_megamoe_input_norm_const = input_norm_const
- return _ensure_flashinfer_megamoe_layer(
- layer,
- megakernel_config=Nvfp4CutedslMegaMoeConfig(
+ megakernel_config = Nvfp4CutedslMegaMoeConfig(
intermediate_size=layer.intermediate_size_per_partition,
top_k=layer.top_k,
- gate_up_clamp=layer.moe_runner_config.swiglu_limit,
apply_topk_in_fc1=True,
enable_in_kernel_fc2_reduce=envs.SGLANG_FLASHINFER_MEGAMOE_IN_KERNEL_FC2_REDUCE.get(),
combine_dtype=resolve_flashinfer_megamoe_combine_dtype(),
input_norm_const=input_norm_const,
- ),
+ **activation_kwargs,
+ )
+
+ return _ensure_flashinfer_megamoe_layer(
+ layer,
+ megakernel_config=megakernel_config,
w13_scale=layer.w13_weight_scale,
w2_scale=layer.w2_weight_scale,
)
@@ -417,6 +445,11 @@ def prepare_fp4_moe_weights_for_flashinfer_megamoe(
current moe_ep API owns backend-specific weight preprocessing, including
DeepGEMM scale layout transforms.
"""
+ if (
+ not layer.moe_runner_config.is_gated
+ or layer.moe_runner_config.activation != "silu"
+ ):
+ raise ValueError("FlashInfer DeepGEMM MegaMOE only supports gated SiLU.")
_init_flashinfer_megamoe_layer_state(layer)
from flashinfer.moe_ep import (
@@ -443,31 +476,56 @@ def prepare_fp4_moe_weights_for_flashinfer_megamoe(
)
+def make_nvfp4_megamoe_weight_loader(
+ layer: FusedMoE, weight_loader: Callable
+) -> Callable:
+ def load_weights(*args, **kwargs):
+ # Reload into canonical views of the existing prepared allocations.
+ # The online loader also writes block scales through its inner loader,
+ # so restore all four planes before handing it the first source tensor.
+ for name, (load_view, _) in getattr(
+ layer, "_flashinfer_megamoe_weight_views", {}
+ ).items():
+ getattr(layer, name).data = load_view
+ return weight_loader(*args, **kwargs)
+
+ return load_weights
+
+
def prepare_nvfp4_moe_weights_for_flashinfer_megamoe(
layer: FusedMoE,
) -> None:
- _init_flashinfer_megamoe_layer_state(layer)
+ use_w4a16 = envs.SGLANG_FLASHINFER_CUTEDSL_NVFP4_W4A16.get()
+ # Both NVFP4 modes reload into the same prepared allocations. Preserve the
+ # live layer, tuned profiles and pooled workspace owner for CUDA graphs.
+ mega = _get_or_init_flashinfer_megamoe_layer_state(layer)
- from flashinfer.moe_ep import (
- MoEWeightPack,
- preprocess_nvfp4_cutedsl_mega_weights,
- )
+ from flashinfer.moe_ep import MoEWeightPack
- if layer.hidden_size % 128 != 0:
+ activation_kwargs = _nvfp4_megamoe_activation_kwargs(layer.moe_runner_config)
+ if layer.moe_runner_config.apply_router_weight_on_input:
+ raise ValueError(
+ "FlashInfer NVFP4 MegaMOE does not support routing weights on input."
+ )
+
+ hidden_alignment = 32 if use_w4a16 else 128
+ intermediate_alignment = 64 if use_w4a16 else 128
+ if layer.hidden_size % hidden_alignment != 0:
raise ValueError(
"FlashInfer NVFP4 MegaMOE requires hidden_size to be a multiple "
- f"of 128, got {layer.hidden_size}."
+ f"of {hidden_alignment}, got {layer.hidden_size}."
)
- if layer.quant_config.use_per_token_activation:
+ if layer.quant_config.use_per_token_activation and not use_w4a16:
raise ValueError(
"FlashInfer NVFP4 MegaMOE does not support per-token activation "
"scaling. Use flashinfer_trtllm/flashinfer_trtllm_routed for "
"ModelOpt NVFP4 per-token activation."
)
- if layer.intermediate_size_per_partition % 128 != 0:
+ if layer.intermediate_size_per_partition % intermediate_alignment != 0:
raise ValueError(
"FlashInfer NVFP4 MegaMOE requires intermediate_size_per_partition "
- f"to be a multiple of 128, got {layer.intermediate_size_per_partition}."
+ f"to be a multiple of {intermediate_alignment}, "
+ f"got {layer.intermediate_size_per_partition}."
)
if layer.num_experts % layer.moe_ep_size != 0:
raise ValueError(
@@ -476,36 +534,75 @@ def prepare_nvfp4_moe_weights_for_flashinfer_megamoe(
)
_validate_nvfp4_fc1_alpha(layer)
- layer._flashinfer_megamoe_input_norm_const = _scalar_float(
- layer.w13_input_scale_quant
- )
-
- gate_up_clamp = layer.moe_runner_config.swiglu_limit
-
+ weight_views = getattr(layer, "_flashinfer_megamoe_weight_views", None)
+ if weight_views is None:
+ load_layouts = {}
+ for name in ("w13_weight", "w13_weight_scale", "w2_weight", "w2_weight_scale"):
+ param = getattr(layer, name)
+ load_layouts[name] = (param.shape, param.stride(), param.dtype)
weights = MoEWeightPack(
w13=layer.w13_weight.data,
w2=layer.w2_weight.data,
w13_scale=layer.w13_weight_scale.data,
w2_scale=layer.w2_weight_scale.data,
)
- transformed_weights = preprocess_nvfp4_cutedsl_mega_weights(
- weights,
- intermediate_size=layer.intermediate_size_per_partition,
- hidden_size=layer.hidden_size,
- gate_up_clamp=gate_up_clamp,
- activation_clamp=None,
- )
+ if use_w4a16:
+ from flashinfer.moe_ep import preprocess_bf16_nvfp4_cutedsl_mega_weights
+
+ transformed_weights = preprocess_bf16_nvfp4_cutedsl_mega_weights(
+ weights,
+ intermediate_size=layer.intermediate_size_per_partition,
+ hidden_size=layer.hidden_size,
+ )
+ else:
+ from flashinfer.moe_ep import preprocess_nvfp4_cutedsl_mega_weights
+
+ input_norm_const = _scalar_float(layer.w13_input_scale_quant)
+ if (
+ mega is not None
+ and input_norm_const != layer._flashinfer_megamoe_input_norm_const
+ ):
+ # FlashInfer captures this scalar by value; updating Python config
+ # would leave existing CUDA graphs using the old normalization.
+ raise ValueError(
+ "FlashInfer NVFP4 W4A4 MegaMOE weight reload requires an unchanged "
+ "input activation scale. Restart the server to change it."
+ )
+ layer._flashinfer_megamoe_input_norm_const = input_norm_const
+ transformed_weights = preprocess_nvfp4_cutedsl_mega_weights(
+ weights,
+ intermediate_size=layer.intermediate_size_per_partition,
+ hidden_size=layer.hidden_size,
+ gate_up_clamp=activation_kwargs["gate_up_clamp"],
+ activation_clamp=None,
+ )
+ if weight_views is not None:
+ for name, (_, prepared_view) in weight_views.items():
+ getattr(layer, name).data = prepared_view
_bind_transformed_weights(
layer,
transformed_weights,
w13_scale_name="w13_weight_scale",
w2_scale_name="w2_weight_scale",
)
+ if weight_views is None:
+ # Padded scale storage can exceed the canonical load shape. as_strided
+ # exposes only the original extent without allocating another copy.
+ layer._flashinfer_megamoe_weight_views = {}
+ for name, (shape, stride, dtype) in load_layouts.items():
+ prepared_view = getattr(layer, name).data
+ load_view = torch.as_strided(prepared_view.view(dtype), shape, stride)
+ layer._flashinfer_megamoe_weight_views[name] = (load_view, prepared_view)
def prepare_mxfp8_moe_weights_for_flashinfer_megamoe(
layer: FusedMoE,
) -> None:
+ if (
+ not layer.moe_runner_config.is_gated
+ or layer.moe_runner_config.activation != "silu"
+ ):
+ raise ValueError("FlashInfer MXFP8 MegaMOE only supports gated SiLU.")
_init_flashinfer_megamoe_layer_state(layer)
from flashinfer.moe_ep import (
@@ -578,7 +675,8 @@ def run_flashinfer_megamoe(
# directly into its final int64 workspace buffer. Keep this path copy-free.
topk_ids=topk_ids,
topk_weights=topk_weights.to(torch.float32),
- # Stage per-layer scales on every call so FlashInfer can pool workspaces.
+ # Stage per-layer scales on every call so FlashInfer can share its
+ # reference-counted workspace across layers and observe weight reloads.
fc1_alpha=quant_info.fc1_alpha,
fc2_alpha=quant_info.fc2_alpha,
fc1_norm_const=quant_info.fc1_norm_const,
diff --git a/python/sglang/srt/layers/quantization/modelopt_quant.py b/python/sglang/srt/layers/quantization/modelopt_quant.py
index a91d928ef8..af7f8bb039 100755
--- a/python/sglang/srt/layers/quantization/modelopt_quant.py
+++ b/python/sglang/srt/layers/quantization/modelopt_quant.py
@@ -2439,6 +2439,12 @@ class ModelOptNvFp4FusedMoEMethod(FusedMoEMethodBase):
weight_loader = self.prepare_weight_loader(
layer, extra_weight_attrs.get("weight_loader")
)
+ if get_moe_runner_backend().is_flashinfer_megamoe():
+ from sglang.srt.layers.moe.flashinfer_megamoe import (
+ make_nvfp4_megamoe_weight_loader,
+ )
+
+ weight_loader = make_nvfp4_megamoe_weight_loader(layer, weight_loader)
# GEMM 1
num_shards = 2 if layer.moe_runner_config.is_gated else 1
@@ -2738,11 +2744,10 @@ class ModelOptNvFp4FusedMoEMethod(FusedMoEMethodBase):
w13_input_scale = layer.w13_input_scale.max(dim=-1).values.to(torch.float32)
w2_input_scale = layer.w2_input_scale
- use_cutedsl_w4a16 = (
- self._is_cutedsl_v2_standard
- and envs.SGLANG_FLASHINFER_CUTEDSL_NVFP4_W4A16.get()
- )
- if self.quant_config.use_per_token_activation or use_cutedsl_w4a16:
+ use_w4a16 = (
+ self._is_cutedsl_v2_standard or moe_runner_backend.is_flashinfer_megamoe()
+ ) and envs.SGLANG_FLASHINFER_CUTEDSL_NVFP4_W4A16.get()
+ if self.quant_config.use_per_token_activation or use_w4a16:
# FlashInfer computes activation scales dynamically per token, so
# the static checkpoint activation scale is intentionally neutral.
# CuTe DSL W4A16 keeps activations in BF16, so its GEMM alphas must
@@ -2825,7 +2830,7 @@ class ModelOptNvFp4FusedMoEMethod(FusedMoEMethodBase):
# TODO: for flashinfer always do MOE_NVFP4_DISPATCH
use_dispatch_fp4 = (
not self.quant_config.use_per_token_activation
- and not use_cutedsl_w4a16
+ and not use_w4a16
and (
use_nvfp4_dispatch or should_use_flashinfer_cutlass_moe_fp4_allgather()
)
@@ -3117,7 +3122,11 @@ class ModelOptNvFp4FusedMoEMethod(FusedMoEMethodBase):
)
assert activation in _SUPPORTED_ACT_STRS or (
- activation == "situ" and moe_runner_backend.is_flashinfer_trtllm()
+ activation == "situ"
+ and (
+ moe_runner_backend.is_flashinfer_trtllm()
+ or moe_runner_backend.is_flashinfer_megamoe()
+ )
), f"{activation=} is unsupported by {moe_runner_backend}"
if moe_runner_backend.is_flashinfer_megamoe():
@@ -3127,12 +3136,13 @@ class ModelOptNvFp4FusedMoEMethod(FusedMoEMethodBase):
)
mega = ensure_nvfp4_moe_layer_for_flashinfer_megamoe(layer)
+ use_w4a16 = envs.SGLANG_FLASHINFER_CUTEDSL_NVFP4_W4A16.get()
quant_info = FlashInferMegaMoeQuantInfo(
mega=mega,
mega_forward=layer._flashinfer_megamoe_forward,
fc1_alpha=layer.g1_alphas,
fc2_alpha=layer.g2_alphas,
- fc1_norm_const=layer.w2_input_scale_quant,
+ fc1_norm_const=None if use_w4a16 else layer.w2_input_scale_quant,
apply_routed_scaling_factor=(
not layer.should_fuse_routed_scaling_factor_in_topk
),
diff --git a/python/sglang/srt/layers/quantization/nvfp4_online.py b/python/sglang/srt/layers/quantization/nvfp4_online.py
index 30d9fead8e..da44a3dabe 100644
--- a/python/sglang/srt/layers/quantization/nvfp4_online.py
+++ b/python/sglang/srt/layers/quantization/nvfp4_online.py
@@ -12,6 +12,7 @@ from typing import Any, Callable, Dict, List, Optional
import torch
from sglang.srt.environ import envs
+from sglang.srt.layers.moe import get_moe_runner_backend
from sglang.srt.layers.quantization.fp8_utils import (
block_quant_dequant,
inverse_transform_scale_ue8m0,
@@ -31,13 +32,14 @@ logger = logging.getLogger(__name__)
class NvFp4OnlineConfig(ModelOptQuantConfig):
- """Load-time NVFP4 with online per-token FP32 activation scales.
+ """Load-time NVFP4 expert weights with online activation handling.
- `--quantization nvfp4_online` exclusively means online per-token FP32
- activation scaling. Use `modelopt_fp4` for per-tensor FP32 activation scales
- or serialized NVFP4 checkpoints. This path converts BF16/FP16/FP8 MoE expert
- weights as they load; dense layers retain their source precision or
- quantization.
+ `--quantization nvfp4_online` uses per-token FP32 activation scales by
+ default. With `SGLANG_FLASHINFER_CUTEDSL_NVFP4_W4A16=1`, the flashinfer_cutedsl
+ and flashinfer_megamoe backends keep activations in BF16. Use `modelopt_fp4`
+ for per-tensor FP32 activation scales or serialized NVFP4 checkpoints.
+ This path converts BF16/FP16/FP8 MoE expert weights as they load; dense
+ layers retain their source precision or quantization.
"""
# Marker consumed by the ModelOpt FP4 layout and the model loader. Serialized
@@ -208,13 +210,19 @@ class ModelOptNvFp4OnlineFusedMoEMethod(ModelOptNvFp4FusedMoEMethod):
else layer_prefix
)
if quant_config.use_per_token_activation and not (
- self.enable_flashinfer_trtllm_moe or self._is_cutedsl_v2_standard
+ self.enable_flashinfer_trtllm_moe
+ or self._is_cutedsl_v2_standard
+ or (
+ get_moe_runner_backend().is_flashinfer_megamoe()
+ and envs.SGLANG_FLASHINFER_CUTEDSL_NVFP4_W4A16.get()
+ )
):
raise ValueError(
- "--quantization nvfp4_online requires online per-token FP32 "
- "activation scales and supports flashinfer_trtllm, "
+ "--quantization nvfp4_online supports flashinfer_trtllm, "
"flashinfer_trtllm_routed, or flashinfer_cutedsl with no A2A "
- "or FlashInfer A2A. Use --quantization modelopt_fp4 for "
+ "or FlashInfer A2A, or flashinfer_megamoe with "
+ "SGLANG_FLASHINFER_CUTEDSL_NVFP4_W4A16=1. "
+ "Use --quantization modelopt_fp4 for "
"per-tensor FP32 activation scales."
)
diff --git a/python/sglang/srt/model_loader/weight_utils.py b/python/sglang/srt/model_loader/weight_utils.py
index 9fc65d822b..a77aecb388 100644
--- a/python/sglang/srt/model_loader/weight_utils.py
+++ b/python/sglang/srt/model_loader/weight_utils.py
@@ -1770,7 +1770,8 @@ def initialize_dummy_weights(
"""
for name, param in model.state_dict().items():
if torch.is_floating_point(param):
- if name.endswith("weight_scale_inv"):
+ # Neutral FP4 global scales keep fused gate/up projections compatible.
+ if name.endswith(("weight_scale_inv", "weight_scale_2")):
param.fill_(1.0)
continue
generator = torch.Generator(device=param.data.device)
diff --git a/python/sglang/srt/models/kimi_k3.py b/python/sglang/srt/models/kimi_k3.py
index 9158dd950b..bacea48aa2 100644
--- a/python/sglang/srt/models/kimi_k3.py
+++ b/python/sglang/srt/models/kimi_k3.py
@@ -438,6 +438,7 @@ def _uses_ep_a2a() -> bool:
_a2a_backend = get_moe_a2a_backend()
return (
_a2a_backend.is_megamoe()
+ or _a2a_backend.is_flashinfer_megamoe()
or _a2a_backend.is_deepep()
or _a2a_backend.is_mooncake()
or _a2a_backend.is_ascend_fuseep()
@@ -546,6 +547,7 @@ class KimiK3MoE(nn.Module):
)
# mega pre-dispatch consumes raw topk_ids/topk_weights
or get_moe_a2a_backend().is_megamoe()
+ or get_moe_a2a_backend().is_flashinfer_megamoe()
else None
),
)
diff --git a/test/registered/backends/test_flashinfer_nvfp4_online_moe_backend.py b/test/registered/backends/test_flashinfer_nvfp4_online_moe_backend.py
index f6bb9d0459..e3f6914d91 100644
--- a/test/registered/backends/test_flashinfer_nvfp4_online_moe_backend.py
+++ b/test/registered/backends/test_flashinfer_nvfp4_online_moe_backend.py
@@ -165,5 +165,36 @@ class TestFlashinferCuteDSLMoeBackendNvFp4OnlineW4A16(
}
+class TestFlashinferMegaMoeBackendNvFp4OnlineW4A16(
+ FlashinferNvFp4OnlineMoeBackendBase, CustomTestCase
+):
+ backend = "flashinfer_megamoe"
+ # Use Qwen's gated activation; Nemotron's ReLU-squared is unsupported.
+ model = "Qwen/Qwen3-30B-A3B-Instruct-2507-FP8"
+ extra_args = [
+ "--dtype",
+ "bfloat16",
+ "--attn-dp-size",
+ "4",
+ "--enable-dp-lm-head",
+ "--moe-a2a-backend",
+ "flashinfer_megamoe",
+ "--cuda-graph-backend-prefill",
+ "disabled",
+ ]
+ eval_args = {"api": "completion", "max_tokens": 512}
+ extra_env = {
+ "FLASHINFER_NVFP4_4OVER6": "1",
+ "FLASHINFER_NVFP4_4OVER6_ERR_MODE": "MSE",
+ "FLASHINFER_NVFP4_4OVER6_ERR_USE_FAST_MATH": "1",
+ "FLASHINFER_NVFP4_4OVER6_E4M3_USE_256": "1",
+ "SGLANG_FLASHINFER_CUTEDSL_NVFP4_W4A16": "1",
+ "SGLANG_FLASHINFER_MEGAMOE_IN_KERNEL_FC2_REDUCE": "0",
+ "SGLANG_FLASHINFER_NVFP4_PER_TOKEN_ACTIVATION": "0",
+ # Convert every routed expert: MegaMoE cannot run excluded FP8 experts.
+ "SGLANG_FP4_IGNORED_LAYERS": "",
+ }
+
+
if __name__ == "__main__":
unittest.main()
diff --git a/test/registered/rl/test_update_weights_from_disk_blackwell.py b/test/registered/rl/test_update_weights_from_disk_blackwell.py
index 34463dd50f..2ff87fd2f7 100644
--- a/test/registered/rl/test_update_weights_from_disk_blackwell.py
+++ b/test/registered/rl/test_update_weights_from_disk_blackwell.py
@@ -299,5 +299,38 @@ class TestServerUpdateWeightsFromDiskNVFP4W4A16CuteDSL(
)
+class TestServerUpdateWeightsFromDiskNVFP4W4A16MegaMoE(
+ UpdateWeightsFromDiskBase, CustomTestCase
+):
+ model = "nvidia/Qwen3-30B-A3B-NVFP4"
+ decode_payload = {**UpdateWeightsFromDiskBase.decode_payload, "routed_dp_rank": 0}
+ launch_env = {
+ "SGLANG_FLASHINFER_CUTEDSL_NVFP4_W4A16": "1",
+ "SGLANG_FLASHINFER_MEGAMOE_IN_KERNEL_FC2_REDUCE": "0",
+ "SGLANG_FLASHINFER_NVFP4_PER_TOKEN_ACTIVATION": "0",
+ }
+ backend_test_suites = (
+ {
+ "name": "flashinfer_megamoe_nvfp4_w4a16",
+ "other_args": (
+ "--dtype",
+ "bfloat16",
+ "--tp-size",
+ "4",
+ "--attn-dp-size",
+ "4",
+ "--ep-size",
+ "4",
+ "--fp4-gemm-backend",
+ "flashinfer_cutedsl",
+ "--moe-runner-backend",
+ "flashinfer_megamoe",
+ "--moe-a2a-backend",
+ "flashinfer_megamoe",
+ ),
+ },
+ )
+
+
if __name__ == "__main__":
unittest.main()
diff --git a/test/registered/unit/layers/moe/test_flashinfer_megamoe.py b/test/registered/unit/layers/moe/test_flashinfer_megamoe.py
index 640125fc18..b3c00fce8a 100644
--- a/test/registered/unit/layers/moe/test_flashinfer_megamoe.py
+++ b/test/registered/unit/layers/moe/test_flashinfer_megamoe.py
@@ -44,6 +44,7 @@ def _load_megamoe_module(monkeypatch):
"deep_gemm.utils.math": types.ModuleType("deep_gemm.utils.math"),
}
fake_modules["sglang.srt.environ"].envs = types.SimpleNamespace(
+ SGLANG_FLASHINFER_CUTEDSL_NVFP4_W4A16=types.SimpleNamespace(get=lambda: False),
SGLANG_FLASHINFER_MEGAMOE_MAX_TOKENS_PER_RANK=types.SimpleNamespace(
get=lambda: 0
),
@@ -478,5 +479,314 @@ def test_capture_safe_ue8m0_pack_is_scoped(monkeypatch):
assert dgm.pack_ue8m0_to_int is original
+@pytest.mark.parametrize("in_kernel_reduce", [False, True])
+@pytest.mark.parametrize("use_w4a16", [False, True])
+@pytest.mark.parametrize("activation", ["silu", "situ"])
+def test_nvfp4_keeps_weight_scale_storage_and_activation_contract(
+ monkeypatch, in_kernel_reduce, use_w4a16, activation
+):
+ module = _load_megamoe_module(monkeypatch)
+ monkeypatch.setattr(
+ module.envs.SGLANG_FLASHINFER_CUTEDSL_NVFP4_W4A16, "get", lambda: use_w4a16
+ )
+ monkeypatch.setattr(
+ module.envs.SGLANG_FLASHINFER_MEGAMOE_IN_KERNEL_FC2_REDUCE,
+ "get",
+ lambda: in_kernel_reduce,
+ )
+ fake_moe_ep = types.ModuleType("flashinfer.moe_ep")
+
+ def make_config(
+ *,
+ intermediate_size,
+ top_k,
+ activation,
+ gate_up_clamp,
+ enable_in_kernel_fc2_reduce,
+ situ_beta=None,
+ situ_linear_beta=None,
+ apply_topk_in_fc1=False,
+ combine_dtype="bf16",
+ input_norm_const=None,
+ ):
+ return types.SimpleNamespace(
+ intermediate_size=intermediate_size,
+ top_k=top_k,
+ activation=activation,
+ gate_up_clamp=gate_up_clamp,
+ situ_beta=situ_beta,
+ situ_linear_beta=situ_linear_beta,
+ apply_topk_in_fc1=apply_topk_in_fc1,
+ combine_dtype=combine_dtype,
+ input_norm_const=input_norm_const,
+ enable_in_kernel_fc2_reduce=enable_in_kernel_fc2_reduce,
+ kernel_name=(
+ "sm100_bf16_nvfp4_bf16_cutedsl"
+ if use_w4a16
+ else "sm100_nvfp4_nvfp4_bf16_cutedsl"
+ ),
+ knobs=None,
+ )
+
+ class Mega:
+ supports_output_view = False
+
+ def __init__(self, **kwargs):
+ self.__dict__.update(kwargs)
+
+ def forward(self, tensors, workspace=None):
+ self.tensors = tensors
+ return tensors.hidden_states
+
+ fake_moe_ep.Sm100_Bf16_Nvfp4_Bf16_Cutedsl_MegaMoeConfig = make_config
+ fake_moe_ep.Nvfp4CutedslMegaMoeConfig = make_config
+ fake_moe_ep.BootstrapConfig = types.SimpleNamespace
+ fake_moe_ep.FleetParams = types.SimpleNamespace
+ fake_moe_ep.MegaConfig = types.SimpleNamespace
+ fake_moe_ep.MoEEpTensors = types.SimpleNamespace
+ fake_moe_ep.MoEEpMegaLayer = Mega
+ monkeypatch.setitem(sys.modules, "flashinfer.moe_ep", fake_moe_ep)
+ monkeypatch.setattr(torch.cuda, "current_device", lambda: 0)
+ monkeypatch.setattr(torch.cuda, "is_current_stream_capturing", lambda: False)
+ hidden, intermediate = (32, 64) if use_w4a16 else (128, 128)
+ # W4A16 consumes only weight decode scales; activation scales exist only
+ # in the W4A4 case below.
+ layer = types.SimpleNamespace(
+ layer_id=0,
+ hidden_size=hidden,
+ num_experts=2,
+ moe_ep_size=1,
+ moe_ep_rank=0,
+ intermediate_size_per_partition=intermediate,
+ top_k=2,
+ moe_runner_config=types.SimpleNamespace(
+ is_gated=True,
+ activation=activation,
+ swiglu_limit=8.0,
+ gemm1_alpha=4.0 if activation == "situ" else None,
+ gemm1_clamp_limit=25.0 if activation == "situ" else None,
+ ),
+ w13_weight=torch.zeros(
+ 2, 2 * intermediate, hidden // 2, dtype=torch.uint8
+ ).transpose(1, 2),
+ w2_weight=torch.zeros(
+ 2, hidden, intermediate // 2, dtype=torch.uint8
+ ).transpose(1, 2),
+ w13_weight_scale=torch.zeros(2, 512, dtype=torch.float8_e4m3fn),
+ w2_weight_scale=torch.zeros(2, 512, dtype=torch.float8_e4m3fn),
+ g1_alphas=torch.tensor([0.25, 0.5], dtype=torch.float32),
+ g2_alphas=torch.tensor([0.75, 1.0], dtype=torch.float32),
+ )
+ if not use_w4a16:
+ layer.w13_input_scale_quant = torch.tensor(2.0)
+ layer.w2_input_scale_quant = torch.tensor([1.25, 1.5])
+
+ mega = module.ensure_nvfp4_moe_layer_for_flashinfer_megamoe(layer)
+ config = mega.backend.megakernel
+ assert config.enable_in_kernel_fc2_reduce is in_kernel_reduce
+ assert config.activation == ("situ" if activation == "situ" else "swiglu")
+ assert config.gate_up_clamp == (None if activation == "situ" else 8.0)
+ assert config.situ_beta == (4.0 if activation == "situ" else None)
+ assert config.situ_linear_beta == (25.0 if activation == "situ" else None)
+ assert config.apply_topk_in_fc1 is (not use_w4a16)
+ assert config.input_norm_const == (None if use_w4a16 else 2.0)
+ fc1, fc2 = mega.backend.transformed_weights
+ assert len(fc1) == len(fc2) == 2
+ for actual, expected in zip(
+ (*fc1, *fc2),
+ (
+ layer.w13_weight,
+ layer.w13_weight_scale,
+ layer.w2_weight,
+ layer.w2_weight_scale,
+ ),
+ strict=True,
+ ):
+ assert actual.data_ptr() == expected.data_ptr()
+
+ # Scales are staged on each forward, so a shared workspace sees this
+ # layer's current decode scales after a weight update.
+ dispatch_output = types.SimpleNamespace(
+ hidden_states=torch.zeros(1, hidden, dtype=torch.bfloat16),
+ topk_output=types.SimpleNamespace(
+ topk_ids=torch.tensor([[0, 1]], dtype=torch.int32),
+ topk_weights=torch.tensor([[0.25, 0.75]], dtype=torch.float32),
+ ),
+ )
+ quant_info = module.FlashInferMegaMoeQuantInfo(
+ mega=mega,
+ mega_forward=layer._flashinfer_megamoe_forward,
+ fc1_alpha=layer.g1_alphas,
+ fc2_alpha=layer.g2_alphas,
+ fc1_norm_const=None if use_w4a16 else layer.w2_input_scale_quant,
+ )
+ for scale in (2.0, 3.0):
+ layer.g1_alphas.fill_(scale)
+ layer.g2_alphas.fill_(scale + 1)
+ if not use_w4a16:
+ layer.w2_input_scale_quant.fill_(scale + 2)
+ module.run_flashinfer_megamoe(
+ dispatch_output,
+ quant_info,
+ types.SimpleNamespace(routed_scaling_factor=1.0),
+ )
+ assert mega.tensors.fc1_alpha.data_ptr() == layer.g1_alphas.data_ptr()
+ assert mega.tensors.fc2_alpha.data_ptr() == layer.g2_alphas.data_ptr()
+ if use_w4a16:
+ assert mega.tensors.fc1_norm_const is None
+ else:
+ assert mega.tensors.fc1_norm_const is layer.w2_input_scale_quant
+ torch.testing.assert_close(
+ mega.tensors.fc1_norm_const, torch.full((2,), scale + 2)
+ )
+ assert mega.tensors.topk_weights is dispatch_output.topk_output.topk_weights
+ torch.testing.assert_close(mega.tensors.fc1_alpha, torch.full((2,), scale))
+ torch.testing.assert_close(mega.tensors.fc2_alpha, torch.full((2,), scale + 1))
+
+
+@pytest.mark.parametrize(
+ "activation,situ_linear_beta", [("silu", None), ("situ", 25.0), ("situ", None)]
+)
+@pytest.mark.parametrize("use_w4a16", [False, True])
+def test_nvfp4_reload_preserves_layer_and_prepared_weight_storage(
+ monkeypatch, activation, situ_linear_beta, use_w4a16
+):
+ module = _load_megamoe_module(monkeypatch)
+ monkeypatch.setattr(
+ module.envs.SGLANG_FLASHINFER_CUTEDSL_NVFP4_W4A16, "get", lambda: use_w4a16
+ )
+ # Import the real parameter-binding helper without the full runtime.
+ spec = importlib.util.spec_from_file_location(
+ "sglang.srt.layers.utils.common",
+ Path(module.__file__).parents[1] / "utils/common.py",
+ )
+ common = importlib.util.module_from_spec(spec)
+ monkeypatch.setitem(sys.modules, spec.name, common)
+ spec.loader.exec_module(common)
+ hidden, intermediate = (32, 64) if use_w4a16 else (128, 128)
+ layer = torch.nn.Module()
+ layer.hidden_size = hidden
+ layer.intermediate_size_per_partition = intermediate
+ layer.num_local_experts = layer.num_experts = 2
+ layer.moe_ep_size = 1
+ layer.quant_config = types.SimpleNamespace(use_per_token_activation=use_w4a16)
+ layer.moe_runner_config = types.SimpleNamespace(
+ is_gated=True,
+ activation=activation,
+ swiglu_limit=None,
+ gemm1_alpha=4.0 if activation == "situ" else None,
+ gemm1_clamp_limit=situ_linear_beta,
+ apply_router_weight_on_input=False,
+ )
+ layer.g1_alphas = layer.g1_alphas_up = torch.ones(2)
+ layer.g2_alphas = torch.ones(2)
+ layer.w13_input_scale_quant = torch.tensor(1.0)
+ layer.w2_input_scale_quant = torch.ones(2)
+ for prefix, rows, columns in (
+ ("w13", 2 * intermediate, hidden),
+ ("w2", hidden, intermediate),
+ ):
+ for suffix, divisor, dtype in (
+ ("weight", 2, torch.uint8),
+ ("weight_scale", 16, torch.float8_e4m3fn),
+ ):
+ layer.register_parameter(
+ f"{prefix}_{suffix}",
+ torch.nn.Parameter(
+ torch.zeros(2, rows, columns // divisor, dtype=dtype),
+ requires_grad=False,
+ ),
+ )
+ canonical = {name: (p.shape, p.dtype) for name, p in layer.named_parameters()}
+
+ def preprocess(
+ weights,
+ *,
+ intermediate_size,
+ hidden_size,
+ gate_up_clamp=None,
+ activation_clamp=None,
+ ):
+ assert (intermediate_size, hidden_size) == (intermediate, hidden)
+ result = []
+ for weight, scale in (
+ (weights.w13, weights.w13_scale),
+ (weights.w2, weights.w2_scale),
+ ):
+ # Both modes expose FP4 weights with their typed view. W4A4
+ # exposes prepared FP8 scales as bytes; W4A16 keeps their FP8 view.
+ padded = torch.zeros(
+ 2, ((scale[0].numel() + 511) // 512) * 512, dtype=scale.dtype
+ )
+ padded[:, : scale[0].numel()].copy_(scale.flatten(1))
+ scale_dtype = torch.float8_e4m3fn if use_w4a16 else torch.uint8
+ # W4A4 FC2 preprocessing returns a view of the canonical bytes.
+ prepared_weight = (
+ weight if not use_w4a16 and weight is weights.w2 else weight.clone()
+ )
+ result.append(
+ (
+ prepared_weight.view(torch.float4_e2m1fn_x2).transpose(1, 2),
+ padded.view(scale_dtype),
+ )
+ )
+ return tuple(result)
+
+ def make_weight_pack(*, w13, w2, w13_scale, w2_scale):
+ return types.SimpleNamespace(
+ w13=w13, w2=w2, w13_scale=w13_scale, w2_scale=w2_scale
+ )
+
+ fake_moe_ep = types.ModuleType("flashinfer.moe_ep")
+ fake_moe_ep.MoEWeightPack = make_weight_pack
+ fake_moe_ep.preprocess_bf16_nvfp4_cutedsl_mega_weights = preprocess
+ fake_moe_ep.preprocess_nvfp4_cutedsl_mega_weights = preprocess
+ monkeypatch.setitem(sys.modules, "flashinfer.moe_ep", fake_moe_ep)
+ module.prepare_nvfp4_moe_weights_for_flashinfer_megamoe(layer)
+ prepared = {
+ name: (p.shape, p.stride(), p.dtype, p.data_ptr())
+ for name, p in layer.named_parameters()
+ }
+ mega = types.SimpleNamespace(
+ _workspace=object(),
+ _transformed_weights={name: p.data for name, p in layer.named_parameters()},
+ )
+ forward = object()
+ layer._flashinfer_megamoe_layer = mega
+ layer._flashinfer_megamoe_forward = forward
+
+ def load(param, value):
+ for name, p in layer.named_parameters():
+ assert (p.shape, p.dtype) == canonical[name]
+ param.data.fill_(value)
+
+ loader = module.make_nvfp4_megamoe_weight_loader(layer, load)
+ for value in (1, 2):
+ for param in layer.parameters():
+ loader(param, value)
+ module.prepare_nvfp4_moe_weights_for_flashinfer_megamoe(layer)
+ assert layer._flashinfer_megamoe_layer is mega
+ assert layer._flashinfer_megamoe_forward is forward
+ for name, p in layer.named_parameters():
+ assert (p.shape, p.stride(), p.dtype, p.data_ptr()) == prepared[name]
+ # Views held by the live backend and captured graphs see new bytes.
+ p = mega._transformed_weights[name]
+ if name.endswith("_weight"):
+ assert (p.view(torch.uint8) == value).all()
+ else:
+ scale = p.view(torch.float8_e4m3fn).float()
+ assert (scale[:, : canonical[name][0].numel() // 2] == value).all()
+
+ if not use_w4a16:
+ # FlashInfer 0.7.1 embeds this scalar in the live kernel configuration;
+ # accepting a changed scalar would keep a stale captured quantizer.
+ layer.w13_input_scale_quant.fill_(2.0)
+ with pytest.raises(ValueError, match="input activation scale"):
+ module.prepare_nvfp4_moe_weights_for_flashinfer_megamoe(layer)
+ assert layer._flashinfer_megamoe_layer is mega
+ assert layer._flashinfer_megamoe_forward is forward
+ assert layer._flashinfer_megamoe_input_norm_const == 1.0
+
+
if __name__ == "__main__":
sys.exit(pytest.main([__file__, "-v"]))
diff --git a/test/registered/unit/layers/test_kimi_mlp_parallel_groups.py b/test/registered/unit/layers/test_kimi_mlp_parallel_groups.py
index 56b1eb23c4..57915f8eac 100644
--- a/test/registered/unit/layers/test_kimi_mlp_parallel_groups.py
+++ b/test/registered/unit/layers/test_kimi_mlp_parallel_groups.py
@@ -49,6 +49,7 @@ def build_shared_experts(*, ep_a2a=True, width=16, shared=1, quant_config=None):
# test. The real MoE constructor still builds its real shared MLP.
backend = SimpleNamespace(
is_megamoe=lambda: False,
+ is_flashinfer_megamoe=lambda: False,
is_deepep=lambda: ep_a2a,
is_mooncake=lambda: False,
is_ascend_fuseep=lambda: False,
diff --git a/test/registered/unit/model_loader/test_weight_utils.py b/test/registered/unit/model_loader/test_weight_utils.py
index 9ebaa72037..b7b3e5aff6 100644
--- a/test/registered/unit/model_loader/test_weight_utils.py
+++ b/test/registered/unit/model_loader/test_weight_utils.py
@@ -7,8 +7,11 @@ import unittest
from types import SimpleNamespace
from unittest.mock import patch
+import torch
+
from sglang.srt.model_loader.weight_utils import (
filter_duplicate_safetensors_files,
+ initialize_dummy_weights,
maybe_add_mtp_safetensors,
)
from sglang.srt.utils import runai_utils
@@ -199,5 +202,23 @@ class TestMaybeAddMtpSafetensors(CustomTestCase):
listing.assert_not_called()
+class TestInitializeDummyWeights(CustomTestCase):
+ def test_nvfp4_dummy_scales_are_neutral(self):
+ model = torch.nn.Linear(8, 4, bias=False)
+ for name, shape in (
+ ("w13_weight_scale_2", (4, 2)),
+ ("w2_weight_scale_2", (4,)),
+ ):
+ model.register_parameter(
+ name, torch.nn.Parameter(torch.empty(shape), requires_grad=False)
+ )
+
+ initialize_dummy_weights(model, low=0.25, high=0.75)
+
+ self.assertTrue(torch.equal(model.w13_weight_scale_2, torch.ones(4, 2)))
+ self.assertTrue(torch.equal(model.w2_weight_scale_2, torch.ones(4)))
+ self.assertTrue(torch.all((model.weight >= 0.25) & (model.weight <= 0.75)))
+
+
if __name__ == "__main__":
unittest.main()
diff --git a/test/registered/unit/server_args/test_server_args.py b/test/registered/unit/server_args/test_server_args.py
index b9ec877344..2265eb617d 100644
--- a/test/registered/unit/server_args/test_server_args.py
+++ b/test/registered/unit/server_args/test_server_args.py
@@ -1756,6 +1756,9 @@ class TestFlashinferMegaMoeConfig(CustomTestCase):
"DeepseekV32ForCausalLM",
"DeepseekV4ForCausalLM",
"Glm4MoeForCausalLM",
+ "GlmMoeDsaForCausalLM",
+ "KimiK3ForConditionalGeneration",
+ "KimiK3LinearForCausalLM",
"NemotronHForCausalLM",
"NemotronHPuzzleForCausalLM",
"Qwen2MoeForCausalLM",