[Feat] FlashInfer: Add NVFP4 W4A16 and SiTU support to MegaMoE (#44036)

This commit is contained in:
Ziang Li
2026-10-16 00:38:50 -07:00
committed by GitHub
parent 3a5d46e946
commit 7098ec387c
15 changed files with 635 additions and 73 deletions
+36 -1
View File
@@ -185,6 +185,11 @@ def handle_flashinfer_a2a_dispatch_type(server_args: Any):
def validate_flashinfer_megamoe_envs() -> None:
combine_dtype = envs.SGLANG_FLASHINFER_MEGAMOE_COMBINE_DTYPE.get().strip().lower()
if envs.SGLANG_FLASHINFER_CUTEDSL_NVFP4_W4A16.get() and combine_dtype != "bf16":
raise ValueError(
"FlashInfer MegaMOE NVFP4 W4A16 requires "
"SGLANG_FLASHINFER_MEGAMOE_COMBINE_DTYPE=bf16."
)
if combine_dtype not in ("bf16", "mxfp8", "nvfp4"):
raise ValueError(
"SGLANG_FLASHINFER_MEGAMOE_COMBINE_DTYPE must be one of "
@@ -215,6 +220,8 @@ def validate_flashinfer_megamoe_model(server_args: Any) -> None:
"NemotronHPuzzleForCausalLM",
"Qwen2MoeForCausalLM",
"Qwen3MoeForCausalLM",
"KimiK3ForConditionalGeneration",
"KimiK3LinearForCausalLM",
)
if not any(
architecture in validated_architectures for architecture in architectures
@@ -228,13 +235,18 @@ def validate_flashinfer_megamoe_model(server_args: Any) -> None:
quantization = resolved_view(server_args).quantization
supports_megamoe_quantization = (
quantization in ("mxfp8", "modelopt_fp4")
or (
quantization == "nvfp4_online"
and envs.SGLANG_FLASHINFER_CUTEDSL_NVFP4_W4A16.get()
)
or model_config.is_fp4_experts
or model_config.nvfp4_moe_meta is not None
)
if not supports_megamoe_quantization:
raise ValueError(
"FlashInfer MegaMOE currently supports only MXFP8, ModelOpt "
"NVFP4, FP4-expert, or hybrid NVFP4 MoE checkpoints; got "
"NVFP4, FP4-expert, hybrid NVFP4 MoE checkpoints, or nvfp4_online "
"with SGLANG_FLASHINFER_CUTEDSL_NVFP4_W4A16=1; got "
f"quantization={quantization!r}. Standard FP8 MoE checkpoints "
"are not supported."
)
@@ -274,6 +286,13 @@ def handle_a2a_moe(server_args: Any):
if a2a_backend == "flashinfer_megamoe":
validate_flashinfer_megamoe_model(server_args)
validate_flashinfer_megamoe_envs()
if envs.SGLANG_FLASHINFER_CUTEDSL_NVFP4_W4A16.get():
import torch
if model_config_of(server_args).dtype != torch.bfloat16:
raise ValueError(
"FlashInfer MegaMOE NVFP4 W4A16 requires --dtype bfloat16."
)
assert attn_dp_enabled_of(cfg) and cfg.attn_dp_size == cfg.tp_size, (
"FlashInfer MegaMOE is only supported with --attn-dp-size equal to --tp-size"
)
@@ -289,6 +308,22 @@ def handle_a2a_moe(server_args: Any):
"FlashInfer MegaMOE currently requires an SM100-family "
"CUDA device for all supported quantization formats."
)
if cfg.nnodes == 1 and (
cfg.speculative_algorithm is None
or cfg.speculative_moe_a2a_backend in (None, "none")
):
# An unset draft A2A backend defaults to none for this target.
# MegaMoE kernels use directly mapped peer memory. Avoid NVSHMEM's
# uninitialized proxy global-exit request; see
# https://github.com/NVIDIA/nvshmem/pull/110.
# Disable remote transports too: the local-only flag cannot disable
# a full transport proxy.
# Device-side global_exit and proxy timeout polling are unavailable
# with these defaults. Preserve explicit NVSHMEM configuration and
# exclude drafts with their own A2A backend: these are process-wide.
os.environ.setdefault("NVSHMEM_REMOTE_TRANSPORT", "none")
os.environ.setdefault("NVSHMEM_IB_ENABLE_IBGDA", "0")
os.environ.setdefault("NVSHMEM_DISABLE_LOCAL_ONLY_PROXY", "1")
logger.info(
"FlashInfer MegaMOE is enabled. The expert parallel size is "
"adjusted to be the same as the tensor parallel size[%s].",
+11 -3
View File
@@ -1432,22 +1432,30 @@ def _moe_runner_backend_quant_constraints(view: Any) -> dict:
"""Resolve MoE runner backends from quantization."""
moe_runner_backend = view.moe_runner_backend
if view.quantization == "nvfp4_online":
use_megamoe_w4a16 = (
view.moe_a2a_backend == "flashinfer_megamoe"
and envs.SGLANG_FLASHINFER_CUTEDSL_NVFP4_W4A16.get()
)
if not get_platform().is_sm100:
raise ValueError(
"--quantization nvfp4_online is supported only on "
"NVIDIA Blackwell SM100/SM103 GPUs."
)
if moe_runner_backend == "auto":
moe_runner_backend = "flashinfer_trtllm"
moe_runner_backend = (
"flashinfer_megamoe" if use_megamoe_w4a16 else "flashinfer_trtllm"
)
elif moe_runner_backend not in [
"flashinfer_trtllm",
"flashinfer_trtllm_routed",
"flashinfer_cutedsl",
]:
] and not (moe_runner_backend == "flashinfer_megamoe" and use_megamoe_w4a16):
raise ValueError(
"--quantization nvfp4_online supports only "
"--moe-runner-backend flashinfer_trtllm or "
"flashinfer_trtllm_routed, or flashinfer_cutedsl."
"flashinfer_trtllm_routed, or flashinfer_cutedsl; "
"flashinfer_megamoe requires --moe-a2a-backend flashinfer_megamoe "
"and SGLANG_FLASHINFER_CUTEDSL_NVFP4_W4A16=1."
)
# Ascend runs MXFP8 MoE on the Ascend runner; every backend selected below is
# CUDA/ROCm-only. Forcing one here would not merely pick the wrong runner:
+4 -3
View File
@@ -1104,8 +1104,8 @@ class Envs:
# (derived from cuda_graph_max_bs / chunked_prefill_size when unset).
SGLANG_FLASHINFER_MEGAMOE_MAX_TOKENS_PER_RANK = EnvInt(0)
# Opt-in in-kernel FC2 top-k reduce (cross-rank REDG atomic-add) for the
# cutedsl mega kernels (NVFP4 / MXFP8). Permits in-kernel reduction;
# FlashInfer's tuning profile may still select external reduction.
# cutedsl mega kernels (NVFP4 W4A4 / W4A16 and MXFP8). Permits in-kernel
# reduction; FlashInfer's tuning profile may still select external reduction.
# BF16 atomic accumulation is nondeterministic -- keep off for
# bit-reproducibility.
# No effect on the DeepGEMM (block-FP8) mega path, which lacks the knob.
@@ -1118,7 +1118,8 @@ class Envs:
# Enable per-token FP32 activation scaling for serialized ModelOpt FP4 with
# FlashInfer TRT-LLM or CuTe DSL v2 MoE.
SGLANG_FLASHINFER_NVFP4_PER_TOKEN_ACTIVATION = EnvBool(False)
# Use BF16 activations with FlashInfer CuTe DSL NVFP4 dense and MoE weights.
# Use BF16 activations with FlashInfer CuTe DSL NVFP4 dense and MoE weights,
# including the flashinfer_megamoe runner (BF16 combine).
SGLANG_FLASHINFER_CUTEDSL_NVFP4_W4A16 = EnvBool(False)
# Launch the TRT-LLM MoE grouped GEMMs with PDL only at or below this
# token count.
@@ -347,41 +347,69 @@ def ensure_fp4_moe_layer_for_flashinfer_megamoe(layer: FusedMoE) -> Any:
)
def _nvfp4_megamoe_activation_kwargs(config: MoeRunnerConfig) -> dict[str, Any]:
if config.is_gated and config.activation == "silu":
return {"activation": "swiglu", "gate_up_clamp": config.swiglu_limit}
if config.is_gated and config.activation == "situ":
# Kimi K3 stores SiTU's tanh constants in the existing GEMM1 fields.
# The up-branch tanh scale is not a SwiGLU hard-clamp limit.
return {
"activation": "situ",
"situ_beta": config.gemm1_alpha,
"situ_linear_beta": config.gemm1_clamp_limit,
"gate_up_clamp": None,
}
raise ValueError("FlashInfer NVFP4 MegaMOE only supports gated SiLU or SiTU.")
def ensure_nvfp4_moe_layer_for_flashinfer_megamoe(layer: FusedMoE) -> Any:
mega = _get_or_init_flashinfer_megamoe_layer_state(layer)
if mega is not None:
return mega
from flashinfer.moe_ep import Nvfp4CutedslMegaMoeConfig
activation_kwargs = _nvfp4_megamoe_activation_kwargs(layer.moe_runner_config)
if envs.SGLANG_FLASHINFER_CUTEDSL_NVFP4_W4A16.get():
from flashinfer.moe_ep import Sm100_Bf16_Nvfp4_Bf16_Cutedsl_MegaMoeConfig
input_norm_const = layer._flashinfer_megamoe_input_norm_const
if input_norm_const is None:
if torch.cuda.is_available() and torch.cuda.is_current_stream_capturing():
raise RuntimeError(
"FlashInfer NVFP4 MegaMOE layer must be initialized before "
"CUDA graph capture."
megakernel_config = Sm100_Bf16_Nvfp4_Bf16_Cutedsl_MegaMoeConfig(
intermediate_size=layer.intermediate_size_per_partition,
top_k=layer.top_k,
enable_in_kernel_fc2_reduce=envs.SGLANG_FLASHINFER_MEGAMOE_IN_KERNEL_FC2_REDUCE.get(),
**activation_kwargs,
)
else:
from flashinfer.moe_ep import Nvfp4CutedslMegaMoeConfig
input_norm_const = layer._flashinfer_megamoe_input_norm_const
if input_norm_const is None:
if torch.cuda.is_available() and torch.cuda.is_current_stream_capturing():
raise RuntimeError(
"FlashInfer NVFP4 MegaMOE layer must be initialized before "
"CUDA graph capture."
)
logger.warning(
"FlashInfer NVFP4 MegaMOE layer[%s]: input_norm_const was not "
"precomputed at weight-load time; computing it lazily now via a "
"blocking device sync. This should only happen once per layer, "
"but if it happens during warmup it will look like a stall.",
layer.layer_id,
)
logger.warning(
"FlashInfer NVFP4 MegaMOE layer[%s]: input_norm_const was not "
"precomputed at weight-load time; computing it lazily now via a "
"blocking device sync. This should only happen once per layer, "
"but if it happens during warmup it will look like a stall.",
layer.layer_id,
)
input_norm_const = _scalar_float(layer.w13_input_scale_quant)
layer._flashinfer_megamoe_input_norm_const = input_norm_const
input_norm_const = _scalar_float(layer.w13_input_scale_quant)
layer._flashinfer_megamoe_input_norm_const = input_norm_const
return _ensure_flashinfer_megamoe_layer(
layer,
megakernel_config=Nvfp4CutedslMegaMoeConfig(
megakernel_config = Nvfp4CutedslMegaMoeConfig(
intermediate_size=layer.intermediate_size_per_partition,
top_k=layer.top_k,
gate_up_clamp=layer.moe_runner_config.swiglu_limit,
apply_topk_in_fc1=True,
enable_in_kernel_fc2_reduce=envs.SGLANG_FLASHINFER_MEGAMOE_IN_KERNEL_FC2_REDUCE.get(),
combine_dtype=resolve_flashinfer_megamoe_combine_dtype(),
input_norm_const=input_norm_const,
),
**activation_kwargs,
)
return _ensure_flashinfer_megamoe_layer(
layer,
megakernel_config=megakernel_config,
w13_scale=layer.w13_weight_scale,
w2_scale=layer.w2_weight_scale,
)
@@ -417,6 +445,11 @@ def prepare_fp4_moe_weights_for_flashinfer_megamoe(
current moe_ep API owns backend-specific weight preprocessing, including
DeepGEMM scale layout transforms.
"""
if (
not layer.moe_runner_config.is_gated
or layer.moe_runner_config.activation != "silu"
):
raise ValueError("FlashInfer DeepGEMM MegaMOE only supports gated SiLU.")
_init_flashinfer_megamoe_layer_state(layer)
from flashinfer.moe_ep import (
@@ -443,31 +476,56 @@ def prepare_fp4_moe_weights_for_flashinfer_megamoe(
)
def make_nvfp4_megamoe_weight_loader(
layer: FusedMoE, weight_loader: Callable
) -> Callable:
def load_weights(*args, **kwargs):
# Reload into canonical views of the existing prepared allocations.
# The online loader also writes block scales through its inner loader,
# so restore all four planes before handing it the first source tensor.
for name, (load_view, _) in getattr(
layer, "_flashinfer_megamoe_weight_views", {}
).items():
getattr(layer, name).data = load_view
return weight_loader(*args, **kwargs)
return load_weights
def prepare_nvfp4_moe_weights_for_flashinfer_megamoe(
layer: FusedMoE,
) -> None:
_init_flashinfer_megamoe_layer_state(layer)
use_w4a16 = envs.SGLANG_FLASHINFER_CUTEDSL_NVFP4_W4A16.get()
# Both NVFP4 modes reload into the same prepared allocations. Preserve the
# live layer, tuned profiles and pooled workspace owner for CUDA graphs.
mega = _get_or_init_flashinfer_megamoe_layer_state(layer)
from flashinfer.moe_ep import (
MoEWeightPack,
preprocess_nvfp4_cutedsl_mega_weights,
)
from flashinfer.moe_ep import MoEWeightPack
if layer.hidden_size % 128 != 0:
activation_kwargs = _nvfp4_megamoe_activation_kwargs(layer.moe_runner_config)
if layer.moe_runner_config.apply_router_weight_on_input:
raise ValueError(
"FlashInfer NVFP4 MegaMOE does not support routing weights on input."
)
hidden_alignment = 32 if use_w4a16 else 128
intermediate_alignment = 64 if use_w4a16 else 128
if layer.hidden_size % hidden_alignment != 0:
raise ValueError(
"FlashInfer NVFP4 MegaMOE requires hidden_size to be a multiple "
f"of 128, got {layer.hidden_size}."
f"of {hidden_alignment}, got {layer.hidden_size}."
)
if layer.quant_config.use_per_token_activation:
if layer.quant_config.use_per_token_activation and not use_w4a16:
raise ValueError(
"FlashInfer NVFP4 MegaMOE does not support per-token activation "
"scaling. Use flashinfer_trtllm/flashinfer_trtllm_routed for "
"ModelOpt NVFP4 per-token activation."
)
if layer.intermediate_size_per_partition % 128 != 0:
if layer.intermediate_size_per_partition % intermediate_alignment != 0:
raise ValueError(
"FlashInfer NVFP4 MegaMOE requires intermediate_size_per_partition "
f"to be a multiple of 128, got {layer.intermediate_size_per_partition}."
f"to be a multiple of {intermediate_alignment}, "
f"got {layer.intermediate_size_per_partition}."
)
if layer.num_experts % layer.moe_ep_size != 0:
raise ValueError(
@@ -476,36 +534,75 @@ def prepare_nvfp4_moe_weights_for_flashinfer_megamoe(
)
_validate_nvfp4_fc1_alpha(layer)
layer._flashinfer_megamoe_input_norm_const = _scalar_float(
layer.w13_input_scale_quant
)
gate_up_clamp = layer.moe_runner_config.swiglu_limit
weight_views = getattr(layer, "_flashinfer_megamoe_weight_views", None)
if weight_views is None:
load_layouts = {}
for name in ("w13_weight", "w13_weight_scale", "w2_weight", "w2_weight_scale"):
param = getattr(layer, name)
load_layouts[name] = (param.shape, param.stride(), param.dtype)
weights = MoEWeightPack(
w13=layer.w13_weight.data,
w2=layer.w2_weight.data,
w13_scale=layer.w13_weight_scale.data,
w2_scale=layer.w2_weight_scale.data,
)
transformed_weights = preprocess_nvfp4_cutedsl_mega_weights(
weights,
intermediate_size=layer.intermediate_size_per_partition,
hidden_size=layer.hidden_size,
gate_up_clamp=gate_up_clamp,
activation_clamp=None,
)
if use_w4a16:
from flashinfer.moe_ep import preprocess_bf16_nvfp4_cutedsl_mega_weights
transformed_weights = preprocess_bf16_nvfp4_cutedsl_mega_weights(
weights,
intermediate_size=layer.intermediate_size_per_partition,
hidden_size=layer.hidden_size,
)
else:
from flashinfer.moe_ep import preprocess_nvfp4_cutedsl_mega_weights
input_norm_const = _scalar_float(layer.w13_input_scale_quant)
if (
mega is not None
and input_norm_const != layer._flashinfer_megamoe_input_norm_const
):
# FlashInfer captures this scalar by value; updating Python config
# would leave existing CUDA graphs using the old normalization.
raise ValueError(
"FlashInfer NVFP4 W4A4 MegaMOE weight reload requires an unchanged "
"input activation scale. Restart the server to change it."
)
layer._flashinfer_megamoe_input_norm_const = input_norm_const
transformed_weights = preprocess_nvfp4_cutedsl_mega_weights(
weights,
intermediate_size=layer.intermediate_size_per_partition,
hidden_size=layer.hidden_size,
gate_up_clamp=activation_kwargs["gate_up_clamp"],
activation_clamp=None,
)
if weight_views is not None:
for name, (_, prepared_view) in weight_views.items():
getattr(layer, name).data = prepared_view
_bind_transformed_weights(
layer,
transformed_weights,
w13_scale_name="w13_weight_scale",
w2_scale_name="w2_weight_scale",
)
if weight_views is None:
# Padded scale storage can exceed the canonical load shape. as_strided
# exposes only the original extent without allocating another copy.
layer._flashinfer_megamoe_weight_views = {}
for name, (shape, stride, dtype) in load_layouts.items():
prepared_view = getattr(layer, name).data
load_view = torch.as_strided(prepared_view.view(dtype), shape, stride)
layer._flashinfer_megamoe_weight_views[name] = (load_view, prepared_view)
def prepare_mxfp8_moe_weights_for_flashinfer_megamoe(
layer: FusedMoE,
) -> None:
if (
not layer.moe_runner_config.is_gated
or layer.moe_runner_config.activation != "silu"
):
raise ValueError("FlashInfer MXFP8 MegaMOE only supports gated SiLU.")
_init_flashinfer_megamoe_layer_state(layer)
from flashinfer.moe_ep import (
@@ -578,7 +675,8 @@ def run_flashinfer_megamoe(
# directly into its final int64 workspace buffer. Keep this path copy-free.
topk_ids=topk_ids,
topk_weights=topk_weights.to(torch.float32),
# Stage per-layer scales on every call so FlashInfer can pool workspaces.
# Stage per-layer scales on every call so FlashInfer can share its
# reference-counted workspace across layers and observe weight reloads.
fc1_alpha=quant_info.fc1_alpha,
fc2_alpha=quant_info.fc2_alpha,
fc1_norm_const=quant_info.fc1_norm_const,
@@ -2439,6 +2439,12 @@ class ModelOptNvFp4FusedMoEMethod(FusedMoEMethodBase):
weight_loader = self.prepare_weight_loader(
layer, extra_weight_attrs.get("weight_loader")
)
if get_moe_runner_backend().is_flashinfer_megamoe():
from sglang.srt.layers.moe.flashinfer_megamoe import (
make_nvfp4_megamoe_weight_loader,
)
weight_loader = make_nvfp4_megamoe_weight_loader(layer, weight_loader)
# GEMM 1
num_shards = 2 if layer.moe_runner_config.is_gated else 1
@@ -2738,11 +2744,10 @@ class ModelOptNvFp4FusedMoEMethod(FusedMoEMethodBase):
w13_input_scale = layer.w13_input_scale.max(dim=-1).values.to(torch.float32)
w2_input_scale = layer.w2_input_scale
use_cutedsl_w4a16 = (
self._is_cutedsl_v2_standard
and envs.SGLANG_FLASHINFER_CUTEDSL_NVFP4_W4A16.get()
)
if self.quant_config.use_per_token_activation or use_cutedsl_w4a16:
use_w4a16 = (
self._is_cutedsl_v2_standard or moe_runner_backend.is_flashinfer_megamoe()
) and envs.SGLANG_FLASHINFER_CUTEDSL_NVFP4_W4A16.get()
if self.quant_config.use_per_token_activation or use_w4a16:
# FlashInfer computes activation scales dynamically per token, so
# the static checkpoint activation scale is intentionally neutral.
# CuTe DSL W4A16 keeps activations in BF16, so its GEMM alphas must
@@ -2825,7 +2830,7 @@ class ModelOptNvFp4FusedMoEMethod(FusedMoEMethodBase):
# TODO: for flashinfer always do MOE_NVFP4_DISPATCH
use_dispatch_fp4 = (
not self.quant_config.use_per_token_activation
and not use_cutedsl_w4a16
and not use_w4a16
and (
use_nvfp4_dispatch or should_use_flashinfer_cutlass_moe_fp4_allgather()
)
@@ -3117,7 +3122,11 @@ class ModelOptNvFp4FusedMoEMethod(FusedMoEMethodBase):
)
assert activation in _SUPPORTED_ACT_STRS or (
activation == "situ" and moe_runner_backend.is_flashinfer_trtllm()
activation == "situ"
and (
moe_runner_backend.is_flashinfer_trtllm()
or moe_runner_backend.is_flashinfer_megamoe()
)
), f"{activation=} is unsupported by {moe_runner_backend}"
if moe_runner_backend.is_flashinfer_megamoe():
@@ -3127,12 +3136,13 @@ class ModelOptNvFp4FusedMoEMethod(FusedMoEMethodBase):
)
mega = ensure_nvfp4_moe_layer_for_flashinfer_megamoe(layer)
use_w4a16 = envs.SGLANG_FLASHINFER_CUTEDSL_NVFP4_W4A16.get()
quant_info = FlashInferMegaMoeQuantInfo(
mega=mega,
mega_forward=layer._flashinfer_megamoe_forward,
fc1_alpha=layer.g1_alphas,
fc2_alpha=layer.g2_alphas,
fc1_norm_const=layer.w2_input_scale_quant,
fc1_norm_const=None if use_w4a16 else layer.w2_input_scale_quant,
apply_routed_scaling_factor=(
not layer.should_fuse_routed_scaling_factor_in_topk
),
@@ -12,6 +12,7 @@ from typing import Any, Callable, Dict, List, Optional
import torch
from sglang.srt.environ import envs
from sglang.srt.layers.moe import get_moe_runner_backend
from sglang.srt.layers.quantization.fp8_utils import (
block_quant_dequant,
inverse_transform_scale_ue8m0,
@@ -31,13 +32,14 @@ logger = logging.getLogger(__name__)
class NvFp4OnlineConfig(ModelOptQuantConfig):
"""Load-time NVFP4 with online per-token FP32 activation scales.
"""Load-time NVFP4 expert weights with online activation handling.
`--quantization nvfp4_online` exclusively means online per-token FP32
activation scaling. Use `modelopt_fp4` for per-tensor FP32 activation scales
or serialized NVFP4 checkpoints. This path converts BF16/FP16/FP8 MoE expert
weights as they load; dense layers retain their source precision or
quantization.
`--quantization nvfp4_online` uses per-token FP32 activation scales by
default. With `SGLANG_FLASHINFER_CUTEDSL_NVFP4_W4A16=1`, the flashinfer_cutedsl
and flashinfer_megamoe backends keep activations in BF16. Use `modelopt_fp4`
for per-tensor FP32 activation scales or serialized NVFP4 checkpoints.
This path converts BF16/FP16/FP8 MoE expert weights as they load; dense
layers retain their source precision or quantization.
"""
# Marker consumed by the ModelOpt FP4 layout and the model loader. Serialized
@@ -208,13 +210,19 @@ class ModelOptNvFp4OnlineFusedMoEMethod(ModelOptNvFp4FusedMoEMethod):
else layer_prefix
)
if quant_config.use_per_token_activation and not (
self.enable_flashinfer_trtllm_moe or self._is_cutedsl_v2_standard
self.enable_flashinfer_trtllm_moe
or self._is_cutedsl_v2_standard
or (
get_moe_runner_backend().is_flashinfer_megamoe()
and envs.SGLANG_FLASHINFER_CUTEDSL_NVFP4_W4A16.get()
)
):
raise ValueError(
"--quantization nvfp4_online requires online per-token FP32 "
"activation scales and supports flashinfer_trtllm, "
"--quantization nvfp4_online supports flashinfer_trtllm, "
"flashinfer_trtllm_routed, or flashinfer_cutedsl with no A2A "
"or FlashInfer A2A. Use --quantization modelopt_fp4 for "
"or FlashInfer A2A, or flashinfer_megamoe with "
"SGLANG_FLASHINFER_CUTEDSL_NVFP4_W4A16=1. "
"Use --quantization modelopt_fp4 for "
"per-tensor FP32 activation scales."
)
@@ -1770,7 +1770,8 @@ def initialize_dummy_weights(
"""
for name, param in model.state_dict().items():
if torch.is_floating_point(param):
if name.endswith("weight_scale_inv"):
# Neutral FP4 global scales keep fused gate/up projections compatible.
if name.endswith(("weight_scale_inv", "weight_scale_2")):
param.fill_(1.0)
continue
generator = torch.Generator(device=param.data.device)
+2
View File
@@ -438,6 +438,7 @@ def _uses_ep_a2a() -> bool:
_a2a_backend = get_moe_a2a_backend()
return (
_a2a_backend.is_megamoe()
or _a2a_backend.is_flashinfer_megamoe()
or _a2a_backend.is_deepep()
or _a2a_backend.is_mooncake()
or _a2a_backend.is_ascend_fuseep()
@@ -546,6 +547,7 @@ class KimiK3MoE(nn.Module):
)
# mega pre-dispatch consumes raw topk_ids/topk_weights
or get_moe_a2a_backend().is_megamoe()
or get_moe_a2a_backend().is_flashinfer_megamoe()
else None
),
)