diff --git a/docs/docs/references/environment_variables.mdx b/docs/docs/references/environment_variables.mdx index 4e25027e80..947d091589 100644 --- a/docs/docs/references/environment_variables.mdx +++ b/docs/docs/references/environment_variables.mdx @@ -363,7 +363,7 @@ SGLang supports various environment variables that can be used to configure its SGLANG_FLASHINFER_MEGAMOE_IN_KERNEL_FC2_REDUCE - Permit CuTe DSL MegaMOE in-kernel FC2 reduction for NVFP4 and MXFP8. FlashInfer's tuning profile may still select external reduction. BF16 atomic accumulation is nondeterministic; leave disabled for reproducibility. Requires BF16 combine. + Permit CuTe DSL MegaMOE in-kernel FC2 reduction for NVFP4 W4A4, NVFP4 W4A16, and MXFP8. FlashInfer's tuning profile may still select external reduction. BF16 atomic accumulation is nondeterministic; leave disabled for reproducibility. Requires BF16 combine. "false" @@ -821,7 +821,7 @@ SGLang supports various environment variables that can be used to configure its SGLANG_FLASHINFER_CUTEDSL_NVFP4_W4A16 - Use BF16 activations and outputs with FlashInfer CuTe DSL NVFP4 weights on NVIDIA SM100-family GPUs (SM100/SM103). Serialized dense linear layers use this mode with --fp4-gemm-backend flashinfer_cutedsl. MoE layers use it with --moe-runner-backend flashinfer_cutedsl; the MoE path supports online weight quantization and serialized ModelOpt NVFP4 weights with either no A2A or FlashInfer A2A, and honors SGLANG_FLASHINFER_MOE_FUSED_FINALIZE. + Use BF16 activations and outputs with FlashInfer CuTe DSL NVFP4 weights on NVIDIA SM100-family GPUs (SM100/SM103). Serialized dense linear layers use this mode with --fp4-gemm-backend flashinfer_cutedsl. With --moe-runner-backend flashinfer_cutedsl, MoE layers support online weight quantization and serialized ModelOpt NVFP4 weights with either no A2A or FlashInfer A2A, and honor SGLANG_FLASHINFER_MOE_FUSED_FINALIZE. With --moe-runner-backend flashinfer_megamoe --moe-a2a-backend flashinfer_megamoe, MoE layers support serialized ModelOpt NVFP4 weights and --quantization nvfp4_online for gated SiLU/SwiGLU or SiTU. MegaMOE uses BF16 activations and combine, with equal TP, attention-DP, and EP sizes. false diff --git a/python/sglang/srt/arg_groups/moe_hook.py b/python/sglang/srt/arg_groups/moe_hook.py index 946a1f875d..8db87a3e85 100644 --- a/python/sglang/srt/arg_groups/moe_hook.py +++ b/python/sglang/srt/arg_groups/moe_hook.py @@ -185,6 +185,11 @@ def handle_flashinfer_a2a_dispatch_type(server_args: Any): def validate_flashinfer_megamoe_envs() -> None: combine_dtype = envs.SGLANG_FLASHINFER_MEGAMOE_COMBINE_DTYPE.get().strip().lower() + if envs.SGLANG_FLASHINFER_CUTEDSL_NVFP4_W4A16.get() and combine_dtype != "bf16": + raise ValueError( + "FlashInfer MegaMOE NVFP4 W4A16 requires " + "SGLANG_FLASHINFER_MEGAMOE_COMBINE_DTYPE=bf16." + ) if combine_dtype not in ("bf16", "mxfp8", "nvfp4"): raise ValueError( "SGLANG_FLASHINFER_MEGAMOE_COMBINE_DTYPE must be one of " @@ -215,6 +220,8 @@ def validate_flashinfer_megamoe_model(server_args: Any) -> None: "NemotronHPuzzleForCausalLM", "Qwen2MoeForCausalLM", "Qwen3MoeForCausalLM", + "KimiK3ForConditionalGeneration", + "KimiK3LinearForCausalLM", ) if not any( architecture in validated_architectures for architecture in architectures @@ -228,13 +235,18 @@ def validate_flashinfer_megamoe_model(server_args: Any) -> None: quantization = resolved_view(server_args).quantization supports_megamoe_quantization = ( quantization in ("mxfp8", "modelopt_fp4") + or ( + quantization == "nvfp4_online" + and envs.SGLANG_FLASHINFER_CUTEDSL_NVFP4_W4A16.get() + ) or model_config.is_fp4_experts or model_config.nvfp4_moe_meta is not None ) if not supports_megamoe_quantization: raise ValueError( "FlashInfer MegaMOE currently supports only MXFP8, ModelOpt " - "NVFP4, FP4-expert, or hybrid NVFP4 MoE checkpoints; got " + "NVFP4, FP4-expert, hybrid NVFP4 MoE checkpoints, or nvfp4_online " + "with SGLANG_FLASHINFER_CUTEDSL_NVFP4_W4A16=1; got " f"quantization={quantization!r}. Standard FP8 MoE checkpoints " "are not supported." ) @@ -274,6 +286,13 @@ def handle_a2a_moe(server_args: Any): if a2a_backend == "flashinfer_megamoe": validate_flashinfer_megamoe_model(server_args) validate_flashinfer_megamoe_envs() + if envs.SGLANG_FLASHINFER_CUTEDSL_NVFP4_W4A16.get(): + import torch + + if model_config_of(server_args).dtype != torch.bfloat16: + raise ValueError( + "FlashInfer MegaMOE NVFP4 W4A16 requires --dtype bfloat16." + ) assert attn_dp_enabled_of(cfg) and cfg.attn_dp_size == cfg.tp_size, ( "FlashInfer MegaMOE is only supported with --attn-dp-size equal to --tp-size" ) @@ -289,6 +308,22 @@ def handle_a2a_moe(server_args: Any): "FlashInfer MegaMOE currently requires an SM100-family " "CUDA device for all supported quantization formats." ) + if cfg.nnodes == 1 and ( + cfg.speculative_algorithm is None + or cfg.speculative_moe_a2a_backend in (None, "none") + ): + # An unset draft A2A backend defaults to none for this target. + # MegaMoE kernels use directly mapped peer memory. Avoid NVSHMEM's + # uninitialized proxy global-exit request; see + # https://github.com/NVIDIA/nvshmem/pull/110. + # Disable remote transports too: the local-only flag cannot disable + # a full transport proxy. + # Device-side global_exit and proxy timeout polling are unavailable + # with these defaults. Preserve explicit NVSHMEM configuration and + # exclude drafts with their own A2A backend: these are process-wide. + os.environ.setdefault("NVSHMEM_REMOTE_TRANSPORT", "none") + os.environ.setdefault("NVSHMEM_IB_ENABLE_IBGDA", "0") + os.environ.setdefault("NVSHMEM_DISABLE_LOCAL_ONLY_PROXY", "1") logger.info( "FlashInfer MegaMOE is enabled. The expert parallel size is " "adjusted to be the same as the tensor parallel size[%s].", diff --git a/python/sglang/srt/arg_groups/overrides.py b/python/sglang/srt/arg_groups/overrides.py index b30360d623..a20194b744 100644 --- a/python/sglang/srt/arg_groups/overrides.py +++ b/python/sglang/srt/arg_groups/overrides.py @@ -1432,22 +1432,30 @@ def _moe_runner_backend_quant_constraints(view: Any) -> dict: """Resolve MoE runner backends from quantization.""" moe_runner_backend = view.moe_runner_backend if view.quantization == "nvfp4_online": + use_megamoe_w4a16 = ( + view.moe_a2a_backend == "flashinfer_megamoe" + and envs.SGLANG_FLASHINFER_CUTEDSL_NVFP4_W4A16.get() + ) if not get_platform().is_sm100: raise ValueError( "--quantization nvfp4_online is supported only on " "NVIDIA Blackwell SM100/SM103 GPUs." ) if moe_runner_backend == "auto": - moe_runner_backend = "flashinfer_trtllm" + moe_runner_backend = ( + "flashinfer_megamoe" if use_megamoe_w4a16 else "flashinfer_trtllm" + ) elif moe_runner_backend not in [ "flashinfer_trtllm", "flashinfer_trtllm_routed", "flashinfer_cutedsl", - ]: + ] and not (moe_runner_backend == "flashinfer_megamoe" and use_megamoe_w4a16): raise ValueError( "--quantization nvfp4_online supports only " "--moe-runner-backend flashinfer_trtllm or " - "flashinfer_trtllm_routed, or flashinfer_cutedsl." + "flashinfer_trtllm_routed, or flashinfer_cutedsl; " + "flashinfer_megamoe requires --moe-a2a-backend flashinfer_megamoe " + "and SGLANG_FLASHINFER_CUTEDSL_NVFP4_W4A16=1." ) # Ascend runs MXFP8 MoE on the Ascend runner; every backend selected below is # CUDA/ROCm-only. Forcing one here would not merely pick the wrong runner: diff --git a/python/sglang/srt/environ.py b/python/sglang/srt/environ.py index a37a5c09ca..284df9b3be 100644 --- a/python/sglang/srt/environ.py +++ b/python/sglang/srt/environ.py @@ -1104,8 +1104,8 @@ class Envs: # (derived from cuda_graph_max_bs / chunked_prefill_size when unset). SGLANG_FLASHINFER_MEGAMOE_MAX_TOKENS_PER_RANK = EnvInt(0) # Opt-in in-kernel FC2 top-k reduce (cross-rank REDG atomic-add) for the - # cutedsl mega kernels (NVFP4 / MXFP8). Permits in-kernel reduction; - # FlashInfer's tuning profile may still select external reduction. + # cutedsl mega kernels (NVFP4 W4A4 / W4A16 and MXFP8). Permits in-kernel + # reduction; FlashInfer's tuning profile may still select external reduction. # BF16 atomic accumulation is nondeterministic -- keep off for # bit-reproducibility. # No effect on the DeepGEMM (block-FP8) mega path, which lacks the knob. @@ -1118,7 +1118,8 @@ class Envs: # Enable per-token FP32 activation scaling for serialized ModelOpt FP4 with # FlashInfer TRT-LLM or CuTe DSL v2 MoE. SGLANG_FLASHINFER_NVFP4_PER_TOKEN_ACTIVATION = EnvBool(False) - # Use BF16 activations with FlashInfer CuTe DSL NVFP4 dense and MoE weights. + # Use BF16 activations with FlashInfer CuTe DSL NVFP4 dense and MoE weights, + # including the flashinfer_megamoe runner (BF16 combine). SGLANG_FLASHINFER_CUTEDSL_NVFP4_W4A16 = EnvBool(False) # Launch the TRT-LLM MoE grouped GEMMs with PDL only at or below this # token count. diff --git a/python/sglang/srt/layers/moe/flashinfer_megamoe.py b/python/sglang/srt/layers/moe/flashinfer_megamoe.py index 1726338398..1c32bb29e9 100644 --- a/python/sglang/srt/layers/moe/flashinfer_megamoe.py +++ b/python/sglang/srt/layers/moe/flashinfer_megamoe.py @@ -347,41 +347,69 @@ def ensure_fp4_moe_layer_for_flashinfer_megamoe(layer: FusedMoE) -> Any: ) +def _nvfp4_megamoe_activation_kwargs(config: MoeRunnerConfig) -> dict[str, Any]: + if config.is_gated and config.activation == "silu": + return {"activation": "swiglu", "gate_up_clamp": config.swiglu_limit} + if config.is_gated and config.activation == "situ": + # Kimi K3 stores SiTU's tanh constants in the existing GEMM1 fields. + # The up-branch tanh scale is not a SwiGLU hard-clamp limit. + return { + "activation": "situ", + "situ_beta": config.gemm1_alpha, + "situ_linear_beta": config.gemm1_clamp_limit, + "gate_up_clamp": None, + } + raise ValueError("FlashInfer NVFP4 MegaMOE only supports gated SiLU or SiTU.") + + def ensure_nvfp4_moe_layer_for_flashinfer_megamoe(layer: FusedMoE) -> Any: mega = _get_or_init_flashinfer_megamoe_layer_state(layer) if mega is not None: return mega - from flashinfer.moe_ep import Nvfp4CutedslMegaMoeConfig + activation_kwargs = _nvfp4_megamoe_activation_kwargs(layer.moe_runner_config) + if envs.SGLANG_FLASHINFER_CUTEDSL_NVFP4_W4A16.get(): + from flashinfer.moe_ep import Sm100_Bf16_Nvfp4_Bf16_Cutedsl_MegaMoeConfig - input_norm_const = layer._flashinfer_megamoe_input_norm_const - if input_norm_const is None: - if torch.cuda.is_available() and torch.cuda.is_current_stream_capturing(): - raise RuntimeError( - "FlashInfer NVFP4 MegaMOE layer must be initialized before " - "CUDA graph capture." + megakernel_config = Sm100_Bf16_Nvfp4_Bf16_Cutedsl_MegaMoeConfig( + intermediate_size=layer.intermediate_size_per_partition, + top_k=layer.top_k, + enable_in_kernel_fc2_reduce=envs.SGLANG_FLASHINFER_MEGAMOE_IN_KERNEL_FC2_REDUCE.get(), + **activation_kwargs, + ) + else: + from flashinfer.moe_ep import Nvfp4CutedslMegaMoeConfig + + input_norm_const = layer._flashinfer_megamoe_input_norm_const + if input_norm_const is None: + if torch.cuda.is_available() and torch.cuda.is_current_stream_capturing(): + raise RuntimeError( + "FlashInfer NVFP4 MegaMOE layer must be initialized before " + "CUDA graph capture." + ) + logger.warning( + "FlashInfer NVFP4 MegaMOE layer[%s]: input_norm_const was not " + "precomputed at weight-load time; computing it lazily now via a " + "blocking device sync. This should only happen once per layer, " + "but if it happens during warmup it will look like a stall.", + layer.layer_id, ) - logger.warning( - "FlashInfer NVFP4 MegaMOE layer[%s]: input_norm_const was not " - "precomputed at weight-load time; computing it lazily now via a " - "blocking device sync. This should only happen once per layer, " - "but if it happens during warmup it will look like a stall.", - layer.layer_id, - ) - input_norm_const = _scalar_float(layer.w13_input_scale_quant) - layer._flashinfer_megamoe_input_norm_const = input_norm_const + input_norm_const = _scalar_float(layer.w13_input_scale_quant) + layer._flashinfer_megamoe_input_norm_const = input_norm_const - return _ensure_flashinfer_megamoe_layer( - layer, - megakernel_config=Nvfp4CutedslMegaMoeConfig( + megakernel_config = Nvfp4CutedslMegaMoeConfig( intermediate_size=layer.intermediate_size_per_partition, top_k=layer.top_k, - gate_up_clamp=layer.moe_runner_config.swiglu_limit, apply_topk_in_fc1=True, enable_in_kernel_fc2_reduce=envs.SGLANG_FLASHINFER_MEGAMOE_IN_KERNEL_FC2_REDUCE.get(), combine_dtype=resolve_flashinfer_megamoe_combine_dtype(), input_norm_const=input_norm_const, - ), + **activation_kwargs, + ) + + return _ensure_flashinfer_megamoe_layer( + layer, + megakernel_config=megakernel_config, w13_scale=layer.w13_weight_scale, w2_scale=layer.w2_weight_scale, ) @@ -417,6 +445,11 @@ def prepare_fp4_moe_weights_for_flashinfer_megamoe( current moe_ep API owns backend-specific weight preprocessing, including DeepGEMM scale layout transforms. """ + if ( + not layer.moe_runner_config.is_gated + or layer.moe_runner_config.activation != "silu" + ): + raise ValueError("FlashInfer DeepGEMM MegaMOE only supports gated SiLU.") _init_flashinfer_megamoe_layer_state(layer) from flashinfer.moe_ep import ( @@ -443,31 +476,56 @@ def prepare_fp4_moe_weights_for_flashinfer_megamoe( ) +def make_nvfp4_megamoe_weight_loader( + layer: FusedMoE, weight_loader: Callable +) -> Callable: + def load_weights(*args, **kwargs): + # Reload into canonical views of the existing prepared allocations. + # The online loader also writes block scales through its inner loader, + # so restore all four planes before handing it the first source tensor. + for name, (load_view, _) in getattr( + layer, "_flashinfer_megamoe_weight_views", {} + ).items(): + getattr(layer, name).data = load_view + return weight_loader(*args, **kwargs) + + return load_weights + + def prepare_nvfp4_moe_weights_for_flashinfer_megamoe( layer: FusedMoE, ) -> None: - _init_flashinfer_megamoe_layer_state(layer) + use_w4a16 = envs.SGLANG_FLASHINFER_CUTEDSL_NVFP4_W4A16.get() + # Both NVFP4 modes reload into the same prepared allocations. Preserve the + # live layer, tuned profiles and pooled workspace owner for CUDA graphs. + mega = _get_or_init_flashinfer_megamoe_layer_state(layer) - from flashinfer.moe_ep import ( - MoEWeightPack, - preprocess_nvfp4_cutedsl_mega_weights, - ) + from flashinfer.moe_ep import MoEWeightPack - if layer.hidden_size % 128 != 0: + activation_kwargs = _nvfp4_megamoe_activation_kwargs(layer.moe_runner_config) + if layer.moe_runner_config.apply_router_weight_on_input: + raise ValueError( + "FlashInfer NVFP4 MegaMOE does not support routing weights on input." + ) + + hidden_alignment = 32 if use_w4a16 else 128 + intermediate_alignment = 64 if use_w4a16 else 128 + if layer.hidden_size % hidden_alignment != 0: raise ValueError( "FlashInfer NVFP4 MegaMOE requires hidden_size to be a multiple " - f"of 128, got {layer.hidden_size}." + f"of {hidden_alignment}, got {layer.hidden_size}." ) - if layer.quant_config.use_per_token_activation: + if layer.quant_config.use_per_token_activation and not use_w4a16: raise ValueError( "FlashInfer NVFP4 MegaMOE does not support per-token activation " "scaling. Use flashinfer_trtllm/flashinfer_trtllm_routed for " "ModelOpt NVFP4 per-token activation." ) - if layer.intermediate_size_per_partition % 128 != 0: + if layer.intermediate_size_per_partition % intermediate_alignment != 0: raise ValueError( "FlashInfer NVFP4 MegaMOE requires intermediate_size_per_partition " - f"to be a multiple of 128, got {layer.intermediate_size_per_partition}." + f"to be a multiple of {intermediate_alignment}, " + f"got {layer.intermediate_size_per_partition}." ) if layer.num_experts % layer.moe_ep_size != 0: raise ValueError( @@ -476,36 +534,75 @@ def prepare_nvfp4_moe_weights_for_flashinfer_megamoe( ) _validate_nvfp4_fc1_alpha(layer) - layer._flashinfer_megamoe_input_norm_const = _scalar_float( - layer.w13_input_scale_quant - ) - - gate_up_clamp = layer.moe_runner_config.swiglu_limit - + weight_views = getattr(layer, "_flashinfer_megamoe_weight_views", None) + if weight_views is None: + load_layouts = {} + for name in ("w13_weight", "w13_weight_scale", "w2_weight", "w2_weight_scale"): + param = getattr(layer, name) + load_layouts[name] = (param.shape, param.stride(), param.dtype) weights = MoEWeightPack( w13=layer.w13_weight.data, w2=layer.w2_weight.data, w13_scale=layer.w13_weight_scale.data, w2_scale=layer.w2_weight_scale.data, ) - transformed_weights = preprocess_nvfp4_cutedsl_mega_weights( - weights, - intermediate_size=layer.intermediate_size_per_partition, - hidden_size=layer.hidden_size, - gate_up_clamp=gate_up_clamp, - activation_clamp=None, - ) + if use_w4a16: + from flashinfer.moe_ep import preprocess_bf16_nvfp4_cutedsl_mega_weights + + transformed_weights = preprocess_bf16_nvfp4_cutedsl_mega_weights( + weights, + intermediate_size=layer.intermediate_size_per_partition, + hidden_size=layer.hidden_size, + ) + else: + from flashinfer.moe_ep import preprocess_nvfp4_cutedsl_mega_weights + + input_norm_const = _scalar_float(layer.w13_input_scale_quant) + if ( + mega is not None + and input_norm_const != layer._flashinfer_megamoe_input_norm_const + ): + # FlashInfer captures this scalar by value; updating Python config + # would leave existing CUDA graphs using the old normalization. + raise ValueError( + "FlashInfer NVFP4 W4A4 MegaMOE weight reload requires an unchanged " + "input activation scale. Restart the server to change it." + ) + layer._flashinfer_megamoe_input_norm_const = input_norm_const + transformed_weights = preprocess_nvfp4_cutedsl_mega_weights( + weights, + intermediate_size=layer.intermediate_size_per_partition, + hidden_size=layer.hidden_size, + gate_up_clamp=activation_kwargs["gate_up_clamp"], + activation_clamp=None, + ) + if weight_views is not None: + for name, (_, prepared_view) in weight_views.items(): + getattr(layer, name).data = prepared_view _bind_transformed_weights( layer, transformed_weights, w13_scale_name="w13_weight_scale", w2_scale_name="w2_weight_scale", ) + if weight_views is None: + # Padded scale storage can exceed the canonical load shape. as_strided + # exposes only the original extent without allocating another copy. + layer._flashinfer_megamoe_weight_views = {} + for name, (shape, stride, dtype) in load_layouts.items(): + prepared_view = getattr(layer, name).data + load_view = torch.as_strided(prepared_view.view(dtype), shape, stride) + layer._flashinfer_megamoe_weight_views[name] = (load_view, prepared_view) def prepare_mxfp8_moe_weights_for_flashinfer_megamoe( layer: FusedMoE, ) -> None: + if ( + not layer.moe_runner_config.is_gated + or layer.moe_runner_config.activation != "silu" + ): + raise ValueError("FlashInfer MXFP8 MegaMOE only supports gated SiLU.") _init_flashinfer_megamoe_layer_state(layer) from flashinfer.moe_ep import ( @@ -578,7 +675,8 @@ def run_flashinfer_megamoe( # directly into its final int64 workspace buffer. Keep this path copy-free. topk_ids=topk_ids, topk_weights=topk_weights.to(torch.float32), - # Stage per-layer scales on every call so FlashInfer can pool workspaces. + # Stage per-layer scales on every call so FlashInfer can share its + # reference-counted workspace across layers and observe weight reloads. fc1_alpha=quant_info.fc1_alpha, fc2_alpha=quant_info.fc2_alpha, fc1_norm_const=quant_info.fc1_norm_const, diff --git a/python/sglang/srt/layers/quantization/modelopt_quant.py b/python/sglang/srt/layers/quantization/modelopt_quant.py index a91d928ef8..af7f8bb039 100755 --- a/python/sglang/srt/layers/quantization/modelopt_quant.py +++ b/python/sglang/srt/layers/quantization/modelopt_quant.py @@ -2439,6 +2439,12 @@ class ModelOptNvFp4FusedMoEMethod(FusedMoEMethodBase): weight_loader = self.prepare_weight_loader( layer, extra_weight_attrs.get("weight_loader") ) + if get_moe_runner_backend().is_flashinfer_megamoe(): + from sglang.srt.layers.moe.flashinfer_megamoe import ( + make_nvfp4_megamoe_weight_loader, + ) + + weight_loader = make_nvfp4_megamoe_weight_loader(layer, weight_loader) # GEMM 1 num_shards = 2 if layer.moe_runner_config.is_gated else 1 @@ -2738,11 +2744,10 @@ class ModelOptNvFp4FusedMoEMethod(FusedMoEMethodBase): w13_input_scale = layer.w13_input_scale.max(dim=-1).values.to(torch.float32) w2_input_scale = layer.w2_input_scale - use_cutedsl_w4a16 = ( - self._is_cutedsl_v2_standard - and envs.SGLANG_FLASHINFER_CUTEDSL_NVFP4_W4A16.get() - ) - if self.quant_config.use_per_token_activation or use_cutedsl_w4a16: + use_w4a16 = ( + self._is_cutedsl_v2_standard or moe_runner_backend.is_flashinfer_megamoe() + ) and envs.SGLANG_FLASHINFER_CUTEDSL_NVFP4_W4A16.get() + if self.quant_config.use_per_token_activation or use_w4a16: # FlashInfer computes activation scales dynamically per token, so # the static checkpoint activation scale is intentionally neutral. # CuTe DSL W4A16 keeps activations in BF16, so its GEMM alphas must @@ -2825,7 +2830,7 @@ class ModelOptNvFp4FusedMoEMethod(FusedMoEMethodBase): # TODO: for flashinfer always do MOE_NVFP4_DISPATCH use_dispatch_fp4 = ( not self.quant_config.use_per_token_activation - and not use_cutedsl_w4a16 + and not use_w4a16 and ( use_nvfp4_dispatch or should_use_flashinfer_cutlass_moe_fp4_allgather() ) @@ -3117,7 +3122,11 @@ class ModelOptNvFp4FusedMoEMethod(FusedMoEMethodBase): ) assert activation in _SUPPORTED_ACT_STRS or ( - activation == "situ" and moe_runner_backend.is_flashinfer_trtllm() + activation == "situ" + and ( + moe_runner_backend.is_flashinfer_trtllm() + or moe_runner_backend.is_flashinfer_megamoe() + ) ), f"{activation=} is unsupported by {moe_runner_backend}" if moe_runner_backend.is_flashinfer_megamoe(): @@ -3127,12 +3136,13 @@ class ModelOptNvFp4FusedMoEMethod(FusedMoEMethodBase): ) mega = ensure_nvfp4_moe_layer_for_flashinfer_megamoe(layer) + use_w4a16 = envs.SGLANG_FLASHINFER_CUTEDSL_NVFP4_W4A16.get() quant_info = FlashInferMegaMoeQuantInfo( mega=mega, mega_forward=layer._flashinfer_megamoe_forward, fc1_alpha=layer.g1_alphas, fc2_alpha=layer.g2_alphas, - fc1_norm_const=layer.w2_input_scale_quant, + fc1_norm_const=None if use_w4a16 else layer.w2_input_scale_quant, apply_routed_scaling_factor=( not layer.should_fuse_routed_scaling_factor_in_topk ), diff --git a/python/sglang/srt/layers/quantization/nvfp4_online.py b/python/sglang/srt/layers/quantization/nvfp4_online.py index 30d9fead8e..da44a3dabe 100644 --- a/python/sglang/srt/layers/quantization/nvfp4_online.py +++ b/python/sglang/srt/layers/quantization/nvfp4_online.py @@ -12,6 +12,7 @@ from typing import Any, Callable, Dict, List, Optional import torch from sglang.srt.environ import envs +from sglang.srt.layers.moe import get_moe_runner_backend from sglang.srt.layers.quantization.fp8_utils import ( block_quant_dequant, inverse_transform_scale_ue8m0, @@ -31,13 +32,14 @@ logger = logging.getLogger(__name__) class NvFp4OnlineConfig(ModelOptQuantConfig): - """Load-time NVFP4 with online per-token FP32 activation scales. + """Load-time NVFP4 expert weights with online activation handling. - `--quantization nvfp4_online` exclusively means online per-token FP32 - activation scaling. Use `modelopt_fp4` for per-tensor FP32 activation scales - or serialized NVFP4 checkpoints. This path converts BF16/FP16/FP8 MoE expert - weights as they load; dense layers retain their source precision or - quantization. + `--quantization nvfp4_online` uses per-token FP32 activation scales by + default. With `SGLANG_FLASHINFER_CUTEDSL_NVFP4_W4A16=1`, the flashinfer_cutedsl + and flashinfer_megamoe backends keep activations in BF16. Use `modelopt_fp4` + for per-tensor FP32 activation scales or serialized NVFP4 checkpoints. + This path converts BF16/FP16/FP8 MoE expert weights as they load; dense + layers retain their source precision or quantization. """ # Marker consumed by the ModelOpt FP4 layout and the model loader. Serialized @@ -208,13 +210,19 @@ class ModelOptNvFp4OnlineFusedMoEMethod(ModelOptNvFp4FusedMoEMethod): else layer_prefix ) if quant_config.use_per_token_activation and not ( - self.enable_flashinfer_trtllm_moe or self._is_cutedsl_v2_standard + self.enable_flashinfer_trtllm_moe + or self._is_cutedsl_v2_standard + or ( + get_moe_runner_backend().is_flashinfer_megamoe() + and envs.SGLANG_FLASHINFER_CUTEDSL_NVFP4_W4A16.get() + ) ): raise ValueError( - "--quantization nvfp4_online requires online per-token FP32 " - "activation scales and supports flashinfer_trtllm, " + "--quantization nvfp4_online supports flashinfer_trtllm, " "flashinfer_trtllm_routed, or flashinfer_cutedsl with no A2A " - "or FlashInfer A2A. Use --quantization modelopt_fp4 for " + "or FlashInfer A2A, or flashinfer_megamoe with " + "SGLANG_FLASHINFER_CUTEDSL_NVFP4_W4A16=1. " + "Use --quantization modelopt_fp4 for " "per-tensor FP32 activation scales." ) diff --git a/python/sglang/srt/model_loader/weight_utils.py b/python/sglang/srt/model_loader/weight_utils.py index 9fc65d822b..a77aecb388 100644 --- a/python/sglang/srt/model_loader/weight_utils.py +++ b/python/sglang/srt/model_loader/weight_utils.py @@ -1770,7 +1770,8 @@ def initialize_dummy_weights( """ for name, param in model.state_dict().items(): if torch.is_floating_point(param): - if name.endswith("weight_scale_inv"): + # Neutral FP4 global scales keep fused gate/up projections compatible. + if name.endswith(("weight_scale_inv", "weight_scale_2")): param.fill_(1.0) continue generator = torch.Generator(device=param.data.device) diff --git a/python/sglang/srt/models/kimi_k3.py b/python/sglang/srt/models/kimi_k3.py index 9158dd950b..bacea48aa2 100644 --- a/python/sglang/srt/models/kimi_k3.py +++ b/python/sglang/srt/models/kimi_k3.py @@ -438,6 +438,7 @@ def _uses_ep_a2a() -> bool: _a2a_backend = get_moe_a2a_backend() return ( _a2a_backend.is_megamoe() + or _a2a_backend.is_flashinfer_megamoe() or _a2a_backend.is_deepep() or _a2a_backend.is_mooncake() or _a2a_backend.is_ascend_fuseep() @@ -546,6 +547,7 @@ class KimiK3MoE(nn.Module): ) # mega pre-dispatch consumes raw topk_ids/topk_weights or get_moe_a2a_backend().is_megamoe() + or get_moe_a2a_backend().is_flashinfer_megamoe() else None ), ) diff --git a/test/registered/backends/test_flashinfer_nvfp4_online_moe_backend.py b/test/registered/backends/test_flashinfer_nvfp4_online_moe_backend.py index f6bb9d0459..e3f6914d91 100644 --- a/test/registered/backends/test_flashinfer_nvfp4_online_moe_backend.py +++ b/test/registered/backends/test_flashinfer_nvfp4_online_moe_backend.py @@ -165,5 +165,36 @@ class TestFlashinferCuteDSLMoeBackendNvFp4OnlineW4A16( } +class TestFlashinferMegaMoeBackendNvFp4OnlineW4A16( + FlashinferNvFp4OnlineMoeBackendBase, CustomTestCase +): + backend = "flashinfer_megamoe" + # Use Qwen's gated activation; Nemotron's ReLU-squared is unsupported. + model = "Qwen/Qwen3-30B-A3B-Instruct-2507-FP8" + extra_args = [ + "--dtype", + "bfloat16", + "--attn-dp-size", + "4", + "--enable-dp-lm-head", + "--moe-a2a-backend", + "flashinfer_megamoe", + "--cuda-graph-backend-prefill", + "disabled", + ] + eval_args = {"api": "completion", "max_tokens": 512} + extra_env = { + "FLASHINFER_NVFP4_4OVER6": "1", + "FLASHINFER_NVFP4_4OVER6_ERR_MODE": "MSE", + "FLASHINFER_NVFP4_4OVER6_ERR_USE_FAST_MATH": "1", + "FLASHINFER_NVFP4_4OVER6_E4M3_USE_256": "1", + "SGLANG_FLASHINFER_CUTEDSL_NVFP4_W4A16": "1", + "SGLANG_FLASHINFER_MEGAMOE_IN_KERNEL_FC2_REDUCE": "0", + "SGLANG_FLASHINFER_NVFP4_PER_TOKEN_ACTIVATION": "0", + # Convert every routed expert: MegaMoE cannot run excluded FP8 experts. + "SGLANG_FP4_IGNORED_LAYERS": "", + } + + if __name__ == "__main__": unittest.main() diff --git a/test/registered/rl/test_update_weights_from_disk_blackwell.py b/test/registered/rl/test_update_weights_from_disk_blackwell.py index 34463dd50f..2ff87fd2f7 100644 --- a/test/registered/rl/test_update_weights_from_disk_blackwell.py +++ b/test/registered/rl/test_update_weights_from_disk_blackwell.py @@ -299,5 +299,38 @@ class TestServerUpdateWeightsFromDiskNVFP4W4A16CuteDSL( ) +class TestServerUpdateWeightsFromDiskNVFP4W4A16MegaMoE( + UpdateWeightsFromDiskBase, CustomTestCase +): + model = "nvidia/Qwen3-30B-A3B-NVFP4" + decode_payload = {**UpdateWeightsFromDiskBase.decode_payload, "routed_dp_rank": 0} + launch_env = { + "SGLANG_FLASHINFER_CUTEDSL_NVFP4_W4A16": "1", + "SGLANG_FLASHINFER_MEGAMOE_IN_KERNEL_FC2_REDUCE": "0", + "SGLANG_FLASHINFER_NVFP4_PER_TOKEN_ACTIVATION": "0", + } + backend_test_suites = ( + { + "name": "flashinfer_megamoe_nvfp4_w4a16", + "other_args": ( + "--dtype", + "bfloat16", + "--tp-size", + "4", + "--attn-dp-size", + "4", + "--ep-size", + "4", + "--fp4-gemm-backend", + "flashinfer_cutedsl", + "--moe-runner-backend", + "flashinfer_megamoe", + "--moe-a2a-backend", + "flashinfer_megamoe", + ), + }, + ) + + if __name__ == "__main__": unittest.main() diff --git a/test/registered/unit/layers/moe/test_flashinfer_megamoe.py b/test/registered/unit/layers/moe/test_flashinfer_megamoe.py index 640125fc18..b3c00fce8a 100644 --- a/test/registered/unit/layers/moe/test_flashinfer_megamoe.py +++ b/test/registered/unit/layers/moe/test_flashinfer_megamoe.py @@ -44,6 +44,7 @@ def _load_megamoe_module(monkeypatch): "deep_gemm.utils.math": types.ModuleType("deep_gemm.utils.math"), } fake_modules["sglang.srt.environ"].envs = types.SimpleNamespace( + SGLANG_FLASHINFER_CUTEDSL_NVFP4_W4A16=types.SimpleNamespace(get=lambda: False), SGLANG_FLASHINFER_MEGAMOE_MAX_TOKENS_PER_RANK=types.SimpleNamespace( get=lambda: 0 ), @@ -478,5 +479,314 @@ def test_capture_safe_ue8m0_pack_is_scoped(monkeypatch): assert dgm.pack_ue8m0_to_int is original +@pytest.mark.parametrize("in_kernel_reduce", [False, True]) +@pytest.mark.parametrize("use_w4a16", [False, True]) +@pytest.mark.parametrize("activation", ["silu", "situ"]) +def test_nvfp4_keeps_weight_scale_storage_and_activation_contract( + monkeypatch, in_kernel_reduce, use_w4a16, activation +): + module = _load_megamoe_module(monkeypatch) + monkeypatch.setattr( + module.envs.SGLANG_FLASHINFER_CUTEDSL_NVFP4_W4A16, "get", lambda: use_w4a16 + ) + monkeypatch.setattr( + module.envs.SGLANG_FLASHINFER_MEGAMOE_IN_KERNEL_FC2_REDUCE, + "get", + lambda: in_kernel_reduce, + ) + fake_moe_ep = types.ModuleType("flashinfer.moe_ep") + + def make_config( + *, + intermediate_size, + top_k, + activation, + gate_up_clamp, + enable_in_kernel_fc2_reduce, + situ_beta=None, + situ_linear_beta=None, + apply_topk_in_fc1=False, + combine_dtype="bf16", + input_norm_const=None, + ): + return types.SimpleNamespace( + intermediate_size=intermediate_size, + top_k=top_k, + activation=activation, + gate_up_clamp=gate_up_clamp, + situ_beta=situ_beta, + situ_linear_beta=situ_linear_beta, + apply_topk_in_fc1=apply_topk_in_fc1, + combine_dtype=combine_dtype, + input_norm_const=input_norm_const, + enable_in_kernel_fc2_reduce=enable_in_kernel_fc2_reduce, + kernel_name=( + "sm100_bf16_nvfp4_bf16_cutedsl" + if use_w4a16 + else "sm100_nvfp4_nvfp4_bf16_cutedsl" + ), + knobs=None, + ) + + class Mega: + supports_output_view = False + + def __init__(self, **kwargs): + self.__dict__.update(kwargs) + + def forward(self, tensors, workspace=None): + self.tensors = tensors + return tensors.hidden_states + + fake_moe_ep.Sm100_Bf16_Nvfp4_Bf16_Cutedsl_MegaMoeConfig = make_config + fake_moe_ep.Nvfp4CutedslMegaMoeConfig = make_config + fake_moe_ep.BootstrapConfig = types.SimpleNamespace + fake_moe_ep.FleetParams = types.SimpleNamespace + fake_moe_ep.MegaConfig = types.SimpleNamespace + fake_moe_ep.MoEEpTensors = types.SimpleNamespace + fake_moe_ep.MoEEpMegaLayer = Mega + monkeypatch.setitem(sys.modules, "flashinfer.moe_ep", fake_moe_ep) + monkeypatch.setattr(torch.cuda, "current_device", lambda: 0) + monkeypatch.setattr(torch.cuda, "is_current_stream_capturing", lambda: False) + hidden, intermediate = (32, 64) if use_w4a16 else (128, 128) + # W4A16 consumes only weight decode scales; activation scales exist only + # in the W4A4 case below. + layer = types.SimpleNamespace( + layer_id=0, + hidden_size=hidden, + num_experts=2, + moe_ep_size=1, + moe_ep_rank=0, + intermediate_size_per_partition=intermediate, + top_k=2, + moe_runner_config=types.SimpleNamespace( + is_gated=True, + activation=activation, + swiglu_limit=8.0, + gemm1_alpha=4.0 if activation == "situ" else None, + gemm1_clamp_limit=25.0 if activation == "situ" else None, + ), + w13_weight=torch.zeros( + 2, 2 * intermediate, hidden // 2, dtype=torch.uint8 + ).transpose(1, 2), + w2_weight=torch.zeros( + 2, hidden, intermediate // 2, dtype=torch.uint8 + ).transpose(1, 2), + w13_weight_scale=torch.zeros(2, 512, dtype=torch.float8_e4m3fn), + w2_weight_scale=torch.zeros(2, 512, dtype=torch.float8_e4m3fn), + g1_alphas=torch.tensor([0.25, 0.5], dtype=torch.float32), + g2_alphas=torch.tensor([0.75, 1.0], dtype=torch.float32), + ) + if not use_w4a16: + layer.w13_input_scale_quant = torch.tensor(2.0) + layer.w2_input_scale_quant = torch.tensor([1.25, 1.5]) + + mega = module.ensure_nvfp4_moe_layer_for_flashinfer_megamoe(layer) + config = mega.backend.megakernel + assert config.enable_in_kernel_fc2_reduce is in_kernel_reduce + assert config.activation == ("situ" if activation == "situ" else "swiglu") + assert config.gate_up_clamp == (None if activation == "situ" else 8.0) + assert config.situ_beta == (4.0 if activation == "situ" else None) + assert config.situ_linear_beta == (25.0 if activation == "situ" else None) + assert config.apply_topk_in_fc1 is (not use_w4a16) + assert config.input_norm_const == (None if use_w4a16 else 2.0) + fc1, fc2 = mega.backend.transformed_weights + assert len(fc1) == len(fc2) == 2 + for actual, expected in zip( + (*fc1, *fc2), + ( + layer.w13_weight, + layer.w13_weight_scale, + layer.w2_weight, + layer.w2_weight_scale, + ), + strict=True, + ): + assert actual.data_ptr() == expected.data_ptr() + + # Scales are staged on each forward, so a shared workspace sees this + # layer's current decode scales after a weight update. + dispatch_output = types.SimpleNamespace( + hidden_states=torch.zeros(1, hidden, dtype=torch.bfloat16), + topk_output=types.SimpleNamespace( + topk_ids=torch.tensor([[0, 1]], dtype=torch.int32), + topk_weights=torch.tensor([[0.25, 0.75]], dtype=torch.float32), + ), + ) + quant_info = module.FlashInferMegaMoeQuantInfo( + mega=mega, + mega_forward=layer._flashinfer_megamoe_forward, + fc1_alpha=layer.g1_alphas, + fc2_alpha=layer.g2_alphas, + fc1_norm_const=None if use_w4a16 else layer.w2_input_scale_quant, + ) + for scale in (2.0, 3.0): + layer.g1_alphas.fill_(scale) + layer.g2_alphas.fill_(scale + 1) + if not use_w4a16: + layer.w2_input_scale_quant.fill_(scale + 2) + module.run_flashinfer_megamoe( + dispatch_output, + quant_info, + types.SimpleNamespace(routed_scaling_factor=1.0), + ) + assert mega.tensors.fc1_alpha.data_ptr() == layer.g1_alphas.data_ptr() + assert mega.tensors.fc2_alpha.data_ptr() == layer.g2_alphas.data_ptr() + if use_w4a16: + assert mega.tensors.fc1_norm_const is None + else: + assert mega.tensors.fc1_norm_const is layer.w2_input_scale_quant + torch.testing.assert_close( + mega.tensors.fc1_norm_const, torch.full((2,), scale + 2) + ) + assert mega.tensors.topk_weights is dispatch_output.topk_output.topk_weights + torch.testing.assert_close(mega.tensors.fc1_alpha, torch.full((2,), scale)) + torch.testing.assert_close(mega.tensors.fc2_alpha, torch.full((2,), scale + 1)) + + +@pytest.mark.parametrize( + "activation,situ_linear_beta", [("silu", None), ("situ", 25.0), ("situ", None)] +) +@pytest.mark.parametrize("use_w4a16", [False, True]) +def test_nvfp4_reload_preserves_layer_and_prepared_weight_storage( + monkeypatch, activation, situ_linear_beta, use_w4a16 +): + module = _load_megamoe_module(monkeypatch) + monkeypatch.setattr( + module.envs.SGLANG_FLASHINFER_CUTEDSL_NVFP4_W4A16, "get", lambda: use_w4a16 + ) + # Import the real parameter-binding helper without the full runtime. + spec = importlib.util.spec_from_file_location( + "sglang.srt.layers.utils.common", + Path(module.__file__).parents[1] / "utils/common.py", + ) + common = importlib.util.module_from_spec(spec) + monkeypatch.setitem(sys.modules, spec.name, common) + spec.loader.exec_module(common) + hidden, intermediate = (32, 64) if use_w4a16 else (128, 128) + layer = torch.nn.Module() + layer.hidden_size = hidden + layer.intermediate_size_per_partition = intermediate + layer.num_local_experts = layer.num_experts = 2 + layer.moe_ep_size = 1 + layer.quant_config = types.SimpleNamespace(use_per_token_activation=use_w4a16) + layer.moe_runner_config = types.SimpleNamespace( + is_gated=True, + activation=activation, + swiglu_limit=None, + gemm1_alpha=4.0 if activation == "situ" else None, + gemm1_clamp_limit=situ_linear_beta, + apply_router_weight_on_input=False, + ) + layer.g1_alphas = layer.g1_alphas_up = torch.ones(2) + layer.g2_alphas = torch.ones(2) + layer.w13_input_scale_quant = torch.tensor(1.0) + layer.w2_input_scale_quant = torch.ones(2) + for prefix, rows, columns in ( + ("w13", 2 * intermediate, hidden), + ("w2", hidden, intermediate), + ): + for suffix, divisor, dtype in ( + ("weight", 2, torch.uint8), + ("weight_scale", 16, torch.float8_e4m3fn), + ): + layer.register_parameter( + f"{prefix}_{suffix}", + torch.nn.Parameter( + torch.zeros(2, rows, columns // divisor, dtype=dtype), + requires_grad=False, + ), + ) + canonical = {name: (p.shape, p.dtype) for name, p in layer.named_parameters()} + + def preprocess( + weights, + *, + intermediate_size, + hidden_size, + gate_up_clamp=None, + activation_clamp=None, + ): + assert (intermediate_size, hidden_size) == (intermediate, hidden) + result = [] + for weight, scale in ( + (weights.w13, weights.w13_scale), + (weights.w2, weights.w2_scale), + ): + # Both modes expose FP4 weights with their typed view. W4A4 + # exposes prepared FP8 scales as bytes; W4A16 keeps their FP8 view. + padded = torch.zeros( + 2, ((scale[0].numel() + 511) // 512) * 512, dtype=scale.dtype + ) + padded[:, : scale[0].numel()].copy_(scale.flatten(1)) + scale_dtype = torch.float8_e4m3fn if use_w4a16 else torch.uint8 + # W4A4 FC2 preprocessing returns a view of the canonical bytes. + prepared_weight = ( + weight if not use_w4a16 and weight is weights.w2 else weight.clone() + ) + result.append( + ( + prepared_weight.view(torch.float4_e2m1fn_x2).transpose(1, 2), + padded.view(scale_dtype), + ) + ) + return tuple(result) + + def make_weight_pack(*, w13, w2, w13_scale, w2_scale): + return types.SimpleNamespace( + w13=w13, w2=w2, w13_scale=w13_scale, w2_scale=w2_scale + ) + + fake_moe_ep = types.ModuleType("flashinfer.moe_ep") + fake_moe_ep.MoEWeightPack = make_weight_pack + fake_moe_ep.preprocess_bf16_nvfp4_cutedsl_mega_weights = preprocess + fake_moe_ep.preprocess_nvfp4_cutedsl_mega_weights = preprocess + monkeypatch.setitem(sys.modules, "flashinfer.moe_ep", fake_moe_ep) + module.prepare_nvfp4_moe_weights_for_flashinfer_megamoe(layer) + prepared = { + name: (p.shape, p.stride(), p.dtype, p.data_ptr()) + for name, p in layer.named_parameters() + } + mega = types.SimpleNamespace( + _workspace=object(), + _transformed_weights={name: p.data for name, p in layer.named_parameters()}, + ) + forward = object() + layer._flashinfer_megamoe_layer = mega + layer._flashinfer_megamoe_forward = forward + + def load(param, value): + for name, p in layer.named_parameters(): + assert (p.shape, p.dtype) == canonical[name] + param.data.fill_(value) + + loader = module.make_nvfp4_megamoe_weight_loader(layer, load) + for value in (1, 2): + for param in layer.parameters(): + loader(param, value) + module.prepare_nvfp4_moe_weights_for_flashinfer_megamoe(layer) + assert layer._flashinfer_megamoe_layer is mega + assert layer._flashinfer_megamoe_forward is forward + for name, p in layer.named_parameters(): + assert (p.shape, p.stride(), p.dtype, p.data_ptr()) == prepared[name] + # Views held by the live backend and captured graphs see new bytes. + p = mega._transformed_weights[name] + if name.endswith("_weight"): + assert (p.view(torch.uint8) == value).all() + else: + scale = p.view(torch.float8_e4m3fn).float() + assert (scale[:, : canonical[name][0].numel() // 2] == value).all() + + if not use_w4a16: + # FlashInfer 0.7.1 embeds this scalar in the live kernel configuration; + # accepting a changed scalar would keep a stale captured quantizer. + layer.w13_input_scale_quant.fill_(2.0) + with pytest.raises(ValueError, match="input activation scale"): + module.prepare_nvfp4_moe_weights_for_flashinfer_megamoe(layer) + assert layer._flashinfer_megamoe_layer is mega + assert layer._flashinfer_megamoe_forward is forward + assert layer._flashinfer_megamoe_input_norm_const == 1.0 + + if __name__ == "__main__": sys.exit(pytest.main([__file__, "-v"])) diff --git a/test/registered/unit/layers/test_kimi_mlp_parallel_groups.py b/test/registered/unit/layers/test_kimi_mlp_parallel_groups.py index 56b1eb23c4..57915f8eac 100644 --- a/test/registered/unit/layers/test_kimi_mlp_parallel_groups.py +++ b/test/registered/unit/layers/test_kimi_mlp_parallel_groups.py @@ -49,6 +49,7 @@ def build_shared_experts(*, ep_a2a=True, width=16, shared=1, quant_config=None): # test. The real MoE constructor still builds its real shared MLP. backend = SimpleNamespace( is_megamoe=lambda: False, + is_flashinfer_megamoe=lambda: False, is_deepep=lambda: ep_a2a, is_mooncake=lambda: False, is_ascend_fuseep=lambda: False, diff --git a/test/registered/unit/model_loader/test_weight_utils.py b/test/registered/unit/model_loader/test_weight_utils.py index 9ebaa72037..b7b3e5aff6 100644 --- a/test/registered/unit/model_loader/test_weight_utils.py +++ b/test/registered/unit/model_loader/test_weight_utils.py @@ -7,8 +7,11 @@ import unittest from types import SimpleNamespace from unittest.mock import patch +import torch + from sglang.srt.model_loader.weight_utils import ( filter_duplicate_safetensors_files, + initialize_dummy_weights, maybe_add_mtp_safetensors, ) from sglang.srt.utils import runai_utils @@ -199,5 +202,23 @@ class TestMaybeAddMtpSafetensors(CustomTestCase): listing.assert_not_called() +class TestInitializeDummyWeights(CustomTestCase): + def test_nvfp4_dummy_scales_are_neutral(self): + model = torch.nn.Linear(8, 4, bias=False) + for name, shape in ( + ("w13_weight_scale_2", (4, 2)), + ("w2_weight_scale_2", (4,)), + ): + model.register_parameter( + name, torch.nn.Parameter(torch.empty(shape), requires_grad=False) + ) + + initialize_dummy_weights(model, low=0.25, high=0.75) + + self.assertTrue(torch.equal(model.w13_weight_scale_2, torch.ones(4, 2))) + self.assertTrue(torch.equal(model.w2_weight_scale_2, torch.ones(4))) + self.assertTrue(torch.all((model.weight >= 0.25) & (model.weight <= 0.75))) + + if __name__ == "__main__": unittest.main() diff --git a/test/registered/unit/server_args/test_server_args.py b/test/registered/unit/server_args/test_server_args.py index b9ec877344..2265eb617d 100644 --- a/test/registered/unit/server_args/test_server_args.py +++ b/test/registered/unit/server_args/test_server_args.py @@ -1756,6 +1756,9 @@ class TestFlashinferMegaMoeConfig(CustomTestCase): "DeepseekV32ForCausalLM", "DeepseekV4ForCausalLM", "Glm4MoeForCausalLM", + "GlmMoeDsaForCausalLM", + "KimiK3ForConditionalGeneration", + "KimiK3LinearForCausalLM", "NemotronHForCausalLM", "NemotronHPuzzleForCausalLM", "Qwen2MoeForCausalLM",