[Feat] FlashInfer: Add NVFP4 W4A16 and SiTU support to MegaMoE (#44036)
This commit is contained in:
@@ -185,6 +185,11 @@ def handle_flashinfer_a2a_dispatch_type(server_args: Any):
|
||||
|
||||
def validate_flashinfer_megamoe_envs() -> None:
|
||||
combine_dtype = envs.SGLANG_FLASHINFER_MEGAMOE_COMBINE_DTYPE.get().strip().lower()
|
||||
if envs.SGLANG_FLASHINFER_CUTEDSL_NVFP4_W4A16.get() and combine_dtype != "bf16":
|
||||
raise ValueError(
|
||||
"FlashInfer MegaMOE NVFP4 W4A16 requires "
|
||||
"SGLANG_FLASHINFER_MEGAMOE_COMBINE_DTYPE=bf16."
|
||||
)
|
||||
if combine_dtype not in ("bf16", "mxfp8", "nvfp4"):
|
||||
raise ValueError(
|
||||
"SGLANG_FLASHINFER_MEGAMOE_COMBINE_DTYPE must be one of "
|
||||
@@ -215,6 +220,8 @@ def validate_flashinfer_megamoe_model(server_args: Any) -> None:
|
||||
"NemotronHPuzzleForCausalLM",
|
||||
"Qwen2MoeForCausalLM",
|
||||
"Qwen3MoeForCausalLM",
|
||||
"KimiK3ForConditionalGeneration",
|
||||
"KimiK3LinearForCausalLM",
|
||||
)
|
||||
if not any(
|
||||
architecture in validated_architectures for architecture in architectures
|
||||
@@ -228,13 +235,18 @@ def validate_flashinfer_megamoe_model(server_args: Any) -> None:
|
||||
quantization = resolved_view(server_args).quantization
|
||||
supports_megamoe_quantization = (
|
||||
quantization in ("mxfp8", "modelopt_fp4")
|
||||
or (
|
||||
quantization == "nvfp4_online"
|
||||
and envs.SGLANG_FLASHINFER_CUTEDSL_NVFP4_W4A16.get()
|
||||
)
|
||||
or model_config.is_fp4_experts
|
||||
or model_config.nvfp4_moe_meta is not None
|
||||
)
|
||||
if not supports_megamoe_quantization:
|
||||
raise ValueError(
|
||||
"FlashInfer MegaMOE currently supports only MXFP8, ModelOpt "
|
||||
"NVFP4, FP4-expert, or hybrid NVFP4 MoE checkpoints; got "
|
||||
"NVFP4, FP4-expert, hybrid NVFP4 MoE checkpoints, or nvfp4_online "
|
||||
"with SGLANG_FLASHINFER_CUTEDSL_NVFP4_W4A16=1; got "
|
||||
f"quantization={quantization!r}. Standard FP8 MoE checkpoints "
|
||||
"are not supported."
|
||||
)
|
||||
@@ -274,6 +286,13 @@ def handle_a2a_moe(server_args: Any):
|
||||
if a2a_backend == "flashinfer_megamoe":
|
||||
validate_flashinfer_megamoe_model(server_args)
|
||||
validate_flashinfer_megamoe_envs()
|
||||
if envs.SGLANG_FLASHINFER_CUTEDSL_NVFP4_W4A16.get():
|
||||
import torch
|
||||
|
||||
if model_config_of(server_args).dtype != torch.bfloat16:
|
||||
raise ValueError(
|
||||
"FlashInfer MegaMOE NVFP4 W4A16 requires --dtype bfloat16."
|
||||
)
|
||||
assert attn_dp_enabled_of(cfg) and cfg.attn_dp_size == cfg.tp_size, (
|
||||
"FlashInfer MegaMOE is only supported with --attn-dp-size equal to --tp-size"
|
||||
)
|
||||
@@ -289,6 +308,22 @@ def handle_a2a_moe(server_args: Any):
|
||||
"FlashInfer MegaMOE currently requires an SM100-family "
|
||||
"CUDA device for all supported quantization formats."
|
||||
)
|
||||
if cfg.nnodes == 1 and (
|
||||
cfg.speculative_algorithm is None
|
||||
or cfg.speculative_moe_a2a_backend in (None, "none")
|
||||
):
|
||||
# An unset draft A2A backend defaults to none for this target.
|
||||
# MegaMoE kernels use directly mapped peer memory. Avoid NVSHMEM's
|
||||
# uninitialized proxy global-exit request; see
|
||||
# https://github.com/NVIDIA/nvshmem/pull/110.
|
||||
# Disable remote transports too: the local-only flag cannot disable
|
||||
# a full transport proxy.
|
||||
# Device-side global_exit and proxy timeout polling are unavailable
|
||||
# with these defaults. Preserve explicit NVSHMEM configuration and
|
||||
# exclude drafts with their own A2A backend: these are process-wide.
|
||||
os.environ.setdefault("NVSHMEM_REMOTE_TRANSPORT", "none")
|
||||
os.environ.setdefault("NVSHMEM_IB_ENABLE_IBGDA", "0")
|
||||
os.environ.setdefault("NVSHMEM_DISABLE_LOCAL_ONLY_PROXY", "1")
|
||||
logger.info(
|
||||
"FlashInfer MegaMOE is enabled. The expert parallel size is "
|
||||
"adjusted to be the same as the tensor parallel size[%s].",
|
||||
|
||||
@@ -1432,22 +1432,30 @@ def _moe_runner_backend_quant_constraints(view: Any) -> dict:
|
||||
"""Resolve MoE runner backends from quantization."""
|
||||
moe_runner_backend = view.moe_runner_backend
|
||||
if view.quantization == "nvfp4_online":
|
||||
use_megamoe_w4a16 = (
|
||||
view.moe_a2a_backend == "flashinfer_megamoe"
|
||||
and envs.SGLANG_FLASHINFER_CUTEDSL_NVFP4_W4A16.get()
|
||||
)
|
||||
if not get_platform().is_sm100:
|
||||
raise ValueError(
|
||||
"--quantization nvfp4_online is supported only on "
|
||||
"NVIDIA Blackwell SM100/SM103 GPUs."
|
||||
)
|
||||
if moe_runner_backend == "auto":
|
||||
moe_runner_backend = "flashinfer_trtllm"
|
||||
moe_runner_backend = (
|
||||
"flashinfer_megamoe" if use_megamoe_w4a16 else "flashinfer_trtllm"
|
||||
)
|
||||
elif moe_runner_backend not in [
|
||||
"flashinfer_trtllm",
|
||||
"flashinfer_trtllm_routed",
|
||||
"flashinfer_cutedsl",
|
||||
]:
|
||||
] and not (moe_runner_backend == "flashinfer_megamoe" and use_megamoe_w4a16):
|
||||
raise ValueError(
|
||||
"--quantization nvfp4_online supports only "
|
||||
"--moe-runner-backend flashinfer_trtllm or "
|
||||
"flashinfer_trtllm_routed, or flashinfer_cutedsl."
|
||||
"flashinfer_trtllm_routed, or flashinfer_cutedsl; "
|
||||
"flashinfer_megamoe requires --moe-a2a-backend flashinfer_megamoe "
|
||||
"and SGLANG_FLASHINFER_CUTEDSL_NVFP4_W4A16=1."
|
||||
)
|
||||
# Ascend runs MXFP8 MoE on the Ascend runner; every backend selected below is
|
||||
# CUDA/ROCm-only. Forcing one here would not merely pick the wrong runner:
|
||||
|
||||
@@ -1104,8 +1104,8 @@ class Envs:
|
||||
# (derived from cuda_graph_max_bs / chunked_prefill_size when unset).
|
||||
SGLANG_FLASHINFER_MEGAMOE_MAX_TOKENS_PER_RANK = EnvInt(0)
|
||||
# Opt-in in-kernel FC2 top-k reduce (cross-rank REDG atomic-add) for the
|
||||
# cutedsl mega kernels (NVFP4 / MXFP8). Permits in-kernel reduction;
|
||||
# FlashInfer's tuning profile may still select external reduction.
|
||||
# cutedsl mega kernels (NVFP4 W4A4 / W4A16 and MXFP8). Permits in-kernel
|
||||
# reduction; FlashInfer's tuning profile may still select external reduction.
|
||||
# BF16 atomic accumulation is nondeterministic -- keep off for
|
||||
# bit-reproducibility.
|
||||
# No effect on the DeepGEMM (block-FP8) mega path, which lacks the knob.
|
||||
@@ -1118,7 +1118,8 @@ class Envs:
|
||||
# Enable per-token FP32 activation scaling for serialized ModelOpt FP4 with
|
||||
# FlashInfer TRT-LLM or CuTe DSL v2 MoE.
|
||||
SGLANG_FLASHINFER_NVFP4_PER_TOKEN_ACTIVATION = EnvBool(False)
|
||||
# Use BF16 activations with FlashInfer CuTe DSL NVFP4 dense and MoE weights.
|
||||
# Use BF16 activations with FlashInfer CuTe DSL NVFP4 dense and MoE weights,
|
||||
# including the flashinfer_megamoe runner (BF16 combine).
|
||||
SGLANG_FLASHINFER_CUTEDSL_NVFP4_W4A16 = EnvBool(False)
|
||||
# Launch the TRT-LLM MoE grouped GEMMs with PDL only at or below this
|
||||
# token count.
|
||||
|
||||
@@ -347,41 +347,69 @@ def ensure_fp4_moe_layer_for_flashinfer_megamoe(layer: FusedMoE) -> Any:
|
||||
)
|
||||
|
||||
|
||||
def _nvfp4_megamoe_activation_kwargs(config: MoeRunnerConfig) -> dict[str, Any]:
|
||||
if config.is_gated and config.activation == "silu":
|
||||
return {"activation": "swiglu", "gate_up_clamp": config.swiglu_limit}
|
||||
if config.is_gated and config.activation == "situ":
|
||||
# Kimi K3 stores SiTU's tanh constants in the existing GEMM1 fields.
|
||||
# The up-branch tanh scale is not a SwiGLU hard-clamp limit.
|
||||
return {
|
||||
"activation": "situ",
|
||||
"situ_beta": config.gemm1_alpha,
|
||||
"situ_linear_beta": config.gemm1_clamp_limit,
|
||||
"gate_up_clamp": None,
|
||||
}
|
||||
raise ValueError("FlashInfer NVFP4 MegaMOE only supports gated SiLU or SiTU.")
|
||||
|
||||
|
||||
def ensure_nvfp4_moe_layer_for_flashinfer_megamoe(layer: FusedMoE) -> Any:
|
||||
mega = _get_or_init_flashinfer_megamoe_layer_state(layer)
|
||||
if mega is not None:
|
||||
return mega
|
||||
|
||||
from flashinfer.moe_ep import Nvfp4CutedslMegaMoeConfig
|
||||
activation_kwargs = _nvfp4_megamoe_activation_kwargs(layer.moe_runner_config)
|
||||
if envs.SGLANG_FLASHINFER_CUTEDSL_NVFP4_W4A16.get():
|
||||
from flashinfer.moe_ep import Sm100_Bf16_Nvfp4_Bf16_Cutedsl_MegaMoeConfig
|
||||
|
||||
input_norm_const = layer._flashinfer_megamoe_input_norm_const
|
||||
if input_norm_const is None:
|
||||
if torch.cuda.is_available() and torch.cuda.is_current_stream_capturing():
|
||||
raise RuntimeError(
|
||||
"FlashInfer NVFP4 MegaMOE layer must be initialized before "
|
||||
"CUDA graph capture."
|
||||
megakernel_config = Sm100_Bf16_Nvfp4_Bf16_Cutedsl_MegaMoeConfig(
|
||||
intermediate_size=layer.intermediate_size_per_partition,
|
||||
top_k=layer.top_k,
|
||||
enable_in_kernel_fc2_reduce=envs.SGLANG_FLASHINFER_MEGAMOE_IN_KERNEL_FC2_REDUCE.get(),
|
||||
**activation_kwargs,
|
||||
)
|
||||
else:
|
||||
from flashinfer.moe_ep import Nvfp4CutedslMegaMoeConfig
|
||||
|
||||
input_norm_const = layer._flashinfer_megamoe_input_norm_const
|
||||
if input_norm_const is None:
|
||||
if torch.cuda.is_available() and torch.cuda.is_current_stream_capturing():
|
||||
raise RuntimeError(
|
||||
"FlashInfer NVFP4 MegaMOE layer must be initialized before "
|
||||
"CUDA graph capture."
|
||||
)
|
||||
logger.warning(
|
||||
"FlashInfer NVFP4 MegaMOE layer[%s]: input_norm_const was not "
|
||||
"precomputed at weight-load time; computing it lazily now via a "
|
||||
"blocking device sync. This should only happen once per layer, "
|
||||
"but if it happens during warmup it will look like a stall.",
|
||||
layer.layer_id,
|
||||
)
|
||||
logger.warning(
|
||||
"FlashInfer NVFP4 MegaMOE layer[%s]: input_norm_const was not "
|
||||
"precomputed at weight-load time; computing it lazily now via a "
|
||||
"blocking device sync. This should only happen once per layer, "
|
||||
"but if it happens during warmup it will look like a stall.",
|
||||
layer.layer_id,
|
||||
)
|
||||
input_norm_const = _scalar_float(layer.w13_input_scale_quant)
|
||||
layer._flashinfer_megamoe_input_norm_const = input_norm_const
|
||||
input_norm_const = _scalar_float(layer.w13_input_scale_quant)
|
||||
layer._flashinfer_megamoe_input_norm_const = input_norm_const
|
||||
|
||||
return _ensure_flashinfer_megamoe_layer(
|
||||
layer,
|
||||
megakernel_config=Nvfp4CutedslMegaMoeConfig(
|
||||
megakernel_config = Nvfp4CutedslMegaMoeConfig(
|
||||
intermediate_size=layer.intermediate_size_per_partition,
|
||||
top_k=layer.top_k,
|
||||
gate_up_clamp=layer.moe_runner_config.swiglu_limit,
|
||||
apply_topk_in_fc1=True,
|
||||
enable_in_kernel_fc2_reduce=envs.SGLANG_FLASHINFER_MEGAMOE_IN_KERNEL_FC2_REDUCE.get(),
|
||||
combine_dtype=resolve_flashinfer_megamoe_combine_dtype(),
|
||||
input_norm_const=input_norm_const,
|
||||
),
|
||||
**activation_kwargs,
|
||||
)
|
||||
|
||||
return _ensure_flashinfer_megamoe_layer(
|
||||
layer,
|
||||
megakernel_config=megakernel_config,
|
||||
w13_scale=layer.w13_weight_scale,
|
||||
w2_scale=layer.w2_weight_scale,
|
||||
)
|
||||
@@ -417,6 +445,11 @@ def prepare_fp4_moe_weights_for_flashinfer_megamoe(
|
||||
current moe_ep API owns backend-specific weight preprocessing, including
|
||||
DeepGEMM scale layout transforms.
|
||||
"""
|
||||
if (
|
||||
not layer.moe_runner_config.is_gated
|
||||
or layer.moe_runner_config.activation != "silu"
|
||||
):
|
||||
raise ValueError("FlashInfer DeepGEMM MegaMOE only supports gated SiLU.")
|
||||
_init_flashinfer_megamoe_layer_state(layer)
|
||||
|
||||
from flashinfer.moe_ep import (
|
||||
@@ -443,31 +476,56 @@ def prepare_fp4_moe_weights_for_flashinfer_megamoe(
|
||||
)
|
||||
|
||||
|
||||
def make_nvfp4_megamoe_weight_loader(
|
||||
layer: FusedMoE, weight_loader: Callable
|
||||
) -> Callable:
|
||||
def load_weights(*args, **kwargs):
|
||||
# Reload into canonical views of the existing prepared allocations.
|
||||
# The online loader also writes block scales through its inner loader,
|
||||
# so restore all four planes before handing it the first source tensor.
|
||||
for name, (load_view, _) in getattr(
|
||||
layer, "_flashinfer_megamoe_weight_views", {}
|
||||
).items():
|
||||
getattr(layer, name).data = load_view
|
||||
return weight_loader(*args, **kwargs)
|
||||
|
||||
return load_weights
|
||||
|
||||
|
||||
def prepare_nvfp4_moe_weights_for_flashinfer_megamoe(
|
||||
layer: FusedMoE,
|
||||
) -> None:
|
||||
_init_flashinfer_megamoe_layer_state(layer)
|
||||
use_w4a16 = envs.SGLANG_FLASHINFER_CUTEDSL_NVFP4_W4A16.get()
|
||||
# Both NVFP4 modes reload into the same prepared allocations. Preserve the
|
||||
# live layer, tuned profiles and pooled workspace owner for CUDA graphs.
|
||||
mega = _get_or_init_flashinfer_megamoe_layer_state(layer)
|
||||
|
||||
from flashinfer.moe_ep import (
|
||||
MoEWeightPack,
|
||||
preprocess_nvfp4_cutedsl_mega_weights,
|
||||
)
|
||||
from flashinfer.moe_ep import MoEWeightPack
|
||||
|
||||
if layer.hidden_size % 128 != 0:
|
||||
activation_kwargs = _nvfp4_megamoe_activation_kwargs(layer.moe_runner_config)
|
||||
if layer.moe_runner_config.apply_router_weight_on_input:
|
||||
raise ValueError(
|
||||
"FlashInfer NVFP4 MegaMOE does not support routing weights on input."
|
||||
)
|
||||
|
||||
hidden_alignment = 32 if use_w4a16 else 128
|
||||
intermediate_alignment = 64 if use_w4a16 else 128
|
||||
if layer.hidden_size % hidden_alignment != 0:
|
||||
raise ValueError(
|
||||
"FlashInfer NVFP4 MegaMOE requires hidden_size to be a multiple "
|
||||
f"of 128, got {layer.hidden_size}."
|
||||
f"of {hidden_alignment}, got {layer.hidden_size}."
|
||||
)
|
||||
if layer.quant_config.use_per_token_activation:
|
||||
if layer.quant_config.use_per_token_activation and not use_w4a16:
|
||||
raise ValueError(
|
||||
"FlashInfer NVFP4 MegaMOE does not support per-token activation "
|
||||
"scaling. Use flashinfer_trtllm/flashinfer_trtllm_routed for "
|
||||
"ModelOpt NVFP4 per-token activation."
|
||||
)
|
||||
if layer.intermediate_size_per_partition % 128 != 0:
|
||||
if layer.intermediate_size_per_partition % intermediate_alignment != 0:
|
||||
raise ValueError(
|
||||
"FlashInfer NVFP4 MegaMOE requires intermediate_size_per_partition "
|
||||
f"to be a multiple of 128, got {layer.intermediate_size_per_partition}."
|
||||
f"to be a multiple of {intermediate_alignment}, "
|
||||
f"got {layer.intermediate_size_per_partition}."
|
||||
)
|
||||
if layer.num_experts % layer.moe_ep_size != 0:
|
||||
raise ValueError(
|
||||
@@ -476,36 +534,75 @@ def prepare_nvfp4_moe_weights_for_flashinfer_megamoe(
|
||||
)
|
||||
|
||||
_validate_nvfp4_fc1_alpha(layer)
|
||||
layer._flashinfer_megamoe_input_norm_const = _scalar_float(
|
||||
layer.w13_input_scale_quant
|
||||
)
|
||||
|
||||
gate_up_clamp = layer.moe_runner_config.swiglu_limit
|
||||
|
||||
weight_views = getattr(layer, "_flashinfer_megamoe_weight_views", None)
|
||||
if weight_views is None:
|
||||
load_layouts = {}
|
||||
for name in ("w13_weight", "w13_weight_scale", "w2_weight", "w2_weight_scale"):
|
||||
param = getattr(layer, name)
|
||||
load_layouts[name] = (param.shape, param.stride(), param.dtype)
|
||||
weights = MoEWeightPack(
|
||||
w13=layer.w13_weight.data,
|
||||
w2=layer.w2_weight.data,
|
||||
w13_scale=layer.w13_weight_scale.data,
|
||||
w2_scale=layer.w2_weight_scale.data,
|
||||
)
|
||||
transformed_weights = preprocess_nvfp4_cutedsl_mega_weights(
|
||||
weights,
|
||||
intermediate_size=layer.intermediate_size_per_partition,
|
||||
hidden_size=layer.hidden_size,
|
||||
gate_up_clamp=gate_up_clamp,
|
||||
activation_clamp=None,
|
||||
)
|
||||
if use_w4a16:
|
||||
from flashinfer.moe_ep import preprocess_bf16_nvfp4_cutedsl_mega_weights
|
||||
|
||||
transformed_weights = preprocess_bf16_nvfp4_cutedsl_mega_weights(
|
||||
weights,
|
||||
intermediate_size=layer.intermediate_size_per_partition,
|
||||
hidden_size=layer.hidden_size,
|
||||
)
|
||||
else:
|
||||
from flashinfer.moe_ep import preprocess_nvfp4_cutedsl_mega_weights
|
||||
|
||||
input_norm_const = _scalar_float(layer.w13_input_scale_quant)
|
||||
if (
|
||||
mega is not None
|
||||
and input_norm_const != layer._flashinfer_megamoe_input_norm_const
|
||||
):
|
||||
# FlashInfer captures this scalar by value; updating Python config
|
||||
# would leave existing CUDA graphs using the old normalization.
|
||||
raise ValueError(
|
||||
"FlashInfer NVFP4 W4A4 MegaMOE weight reload requires an unchanged "
|
||||
"input activation scale. Restart the server to change it."
|
||||
)
|
||||
layer._flashinfer_megamoe_input_norm_const = input_norm_const
|
||||
transformed_weights = preprocess_nvfp4_cutedsl_mega_weights(
|
||||
weights,
|
||||
intermediate_size=layer.intermediate_size_per_partition,
|
||||
hidden_size=layer.hidden_size,
|
||||
gate_up_clamp=activation_kwargs["gate_up_clamp"],
|
||||
activation_clamp=None,
|
||||
)
|
||||
if weight_views is not None:
|
||||
for name, (_, prepared_view) in weight_views.items():
|
||||
getattr(layer, name).data = prepared_view
|
||||
_bind_transformed_weights(
|
||||
layer,
|
||||
transformed_weights,
|
||||
w13_scale_name="w13_weight_scale",
|
||||
w2_scale_name="w2_weight_scale",
|
||||
)
|
||||
if weight_views is None:
|
||||
# Padded scale storage can exceed the canonical load shape. as_strided
|
||||
# exposes only the original extent without allocating another copy.
|
||||
layer._flashinfer_megamoe_weight_views = {}
|
||||
for name, (shape, stride, dtype) in load_layouts.items():
|
||||
prepared_view = getattr(layer, name).data
|
||||
load_view = torch.as_strided(prepared_view.view(dtype), shape, stride)
|
||||
layer._flashinfer_megamoe_weight_views[name] = (load_view, prepared_view)
|
||||
|
||||
|
||||
def prepare_mxfp8_moe_weights_for_flashinfer_megamoe(
|
||||
layer: FusedMoE,
|
||||
) -> None:
|
||||
if (
|
||||
not layer.moe_runner_config.is_gated
|
||||
or layer.moe_runner_config.activation != "silu"
|
||||
):
|
||||
raise ValueError("FlashInfer MXFP8 MegaMOE only supports gated SiLU.")
|
||||
_init_flashinfer_megamoe_layer_state(layer)
|
||||
|
||||
from flashinfer.moe_ep import (
|
||||
@@ -578,7 +675,8 @@ def run_flashinfer_megamoe(
|
||||
# directly into its final int64 workspace buffer. Keep this path copy-free.
|
||||
topk_ids=topk_ids,
|
||||
topk_weights=topk_weights.to(torch.float32),
|
||||
# Stage per-layer scales on every call so FlashInfer can pool workspaces.
|
||||
# Stage per-layer scales on every call so FlashInfer can share its
|
||||
# reference-counted workspace across layers and observe weight reloads.
|
||||
fc1_alpha=quant_info.fc1_alpha,
|
||||
fc2_alpha=quant_info.fc2_alpha,
|
||||
fc1_norm_const=quant_info.fc1_norm_const,
|
||||
|
||||
@@ -2439,6 +2439,12 @@ class ModelOptNvFp4FusedMoEMethod(FusedMoEMethodBase):
|
||||
weight_loader = self.prepare_weight_loader(
|
||||
layer, extra_weight_attrs.get("weight_loader")
|
||||
)
|
||||
if get_moe_runner_backend().is_flashinfer_megamoe():
|
||||
from sglang.srt.layers.moe.flashinfer_megamoe import (
|
||||
make_nvfp4_megamoe_weight_loader,
|
||||
)
|
||||
|
||||
weight_loader = make_nvfp4_megamoe_weight_loader(layer, weight_loader)
|
||||
# GEMM 1
|
||||
num_shards = 2 if layer.moe_runner_config.is_gated else 1
|
||||
|
||||
@@ -2738,11 +2744,10 @@ class ModelOptNvFp4FusedMoEMethod(FusedMoEMethodBase):
|
||||
w13_input_scale = layer.w13_input_scale.max(dim=-1).values.to(torch.float32)
|
||||
w2_input_scale = layer.w2_input_scale
|
||||
|
||||
use_cutedsl_w4a16 = (
|
||||
self._is_cutedsl_v2_standard
|
||||
and envs.SGLANG_FLASHINFER_CUTEDSL_NVFP4_W4A16.get()
|
||||
)
|
||||
if self.quant_config.use_per_token_activation or use_cutedsl_w4a16:
|
||||
use_w4a16 = (
|
||||
self._is_cutedsl_v2_standard or moe_runner_backend.is_flashinfer_megamoe()
|
||||
) and envs.SGLANG_FLASHINFER_CUTEDSL_NVFP4_W4A16.get()
|
||||
if self.quant_config.use_per_token_activation or use_w4a16:
|
||||
# FlashInfer computes activation scales dynamically per token, so
|
||||
# the static checkpoint activation scale is intentionally neutral.
|
||||
# CuTe DSL W4A16 keeps activations in BF16, so its GEMM alphas must
|
||||
@@ -2825,7 +2830,7 @@ class ModelOptNvFp4FusedMoEMethod(FusedMoEMethodBase):
|
||||
# TODO: for flashinfer always do MOE_NVFP4_DISPATCH
|
||||
use_dispatch_fp4 = (
|
||||
not self.quant_config.use_per_token_activation
|
||||
and not use_cutedsl_w4a16
|
||||
and not use_w4a16
|
||||
and (
|
||||
use_nvfp4_dispatch or should_use_flashinfer_cutlass_moe_fp4_allgather()
|
||||
)
|
||||
@@ -3117,7 +3122,11 @@ class ModelOptNvFp4FusedMoEMethod(FusedMoEMethodBase):
|
||||
)
|
||||
|
||||
assert activation in _SUPPORTED_ACT_STRS or (
|
||||
activation == "situ" and moe_runner_backend.is_flashinfer_trtllm()
|
||||
activation == "situ"
|
||||
and (
|
||||
moe_runner_backend.is_flashinfer_trtllm()
|
||||
or moe_runner_backend.is_flashinfer_megamoe()
|
||||
)
|
||||
), f"{activation=} is unsupported by {moe_runner_backend}"
|
||||
|
||||
if moe_runner_backend.is_flashinfer_megamoe():
|
||||
@@ -3127,12 +3136,13 @@ class ModelOptNvFp4FusedMoEMethod(FusedMoEMethodBase):
|
||||
)
|
||||
|
||||
mega = ensure_nvfp4_moe_layer_for_flashinfer_megamoe(layer)
|
||||
use_w4a16 = envs.SGLANG_FLASHINFER_CUTEDSL_NVFP4_W4A16.get()
|
||||
quant_info = FlashInferMegaMoeQuantInfo(
|
||||
mega=mega,
|
||||
mega_forward=layer._flashinfer_megamoe_forward,
|
||||
fc1_alpha=layer.g1_alphas,
|
||||
fc2_alpha=layer.g2_alphas,
|
||||
fc1_norm_const=layer.w2_input_scale_quant,
|
||||
fc1_norm_const=None if use_w4a16 else layer.w2_input_scale_quant,
|
||||
apply_routed_scaling_factor=(
|
||||
not layer.should_fuse_routed_scaling_factor_in_topk
|
||||
),
|
||||
|
||||
@@ -12,6 +12,7 @@ from typing import Any, Callable, Dict, List, Optional
|
||||
import torch
|
||||
|
||||
from sglang.srt.environ import envs
|
||||
from sglang.srt.layers.moe import get_moe_runner_backend
|
||||
from sglang.srt.layers.quantization.fp8_utils import (
|
||||
block_quant_dequant,
|
||||
inverse_transform_scale_ue8m0,
|
||||
@@ -31,13 +32,14 @@ logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class NvFp4OnlineConfig(ModelOptQuantConfig):
|
||||
"""Load-time NVFP4 with online per-token FP32 activation scales.
|
||||
"""Load-time NVFP4 expert weights with online activation handling.
|
||||
|
||||
`--quantization nvfp4_online` exclusively means online per-token FP32
|
||||
activation scaling. Use `modelopt_fp4` for per-tensor FP32 activation scales
|
||||
or serialized NVFP4 checkpoints. This path converts BF16/FP16/FP8 MoE expert
|
||||
weights as they load; dense layers retain their source precision or
|
||||
quantization.
|
||||
`--quantization nvfp4_online` uses per-token FP32 activation scales by
|
||||
default. With `SGLANG_FLASHINFER_CUTEDSL_NVFP4_W4A16=1`, the flashinfer_cutedsl
|
||||
and flashinfer_megamoe backends keep activations in BF16. Use `modelopt_fp4`
|
||||
for per-tensor FP32 activation scales or serialized NVFP4 checkpoints.
|
||||
This path converts BF16/FP16/FP8 MoE expert weights as they load; dense
|
||||
layers retain their source precision or quantization.
|
||||
"""
|
||||
|
||||
# Marker consumed by the ModelOpt FP4 layout and the model loader. Serialized
|
||||
@@ -208,13 +210,19 @@ class ModelOptNvFp4OnlineFusedMoEMethod(ModelOptNvFp4FusedMoEMethod):
|
||||
else layer_prefix
|
||||
)
|
||||
if quant_config.use_per_token_activation and not (
|
||||
self.enable_flashinfer_trtllm_moe or self._is_cutedsl_v2_standard
|
||||
self.enable_flashinfer_trtllm_moe
|
||||
or self._is_cutedsl_v2_standard
|
||||
or (
|
||||
get_moe_runner_backend().is_flashinfer_megamoe()
|
||||
and envs.SGLANG_FLASHINFER_CUTEDSL_NVFP4_W4A16.get()
|
||||
)
|
||||
):
|
||||
raise ValueError(
|
||||
"--quantization nvfp4_online requires online per-token FP32 "
|
||||
"activation scales and supports flashinfer_trtllm, "
|
||||
"--quantization nvfp4_online supports flashinfer_trtllm, "
|
||||
"flashinfer_trtllm_routed, or flashinfer_cutedsl with no A2A "
|
||||
"or FlashInfer A2A. Use --quantization modelopt_fp4 for "
|
||||
"or FlashInfer A2A, or flashinfer_megamoe with "
|
||||
"SGLANG_FLASHINFER_CUTEDSL_NVFP4_W4A16=1. "
|
||||
"Use --quantization modelopt_fp4 for "
|
||||
"per-tensor FP32 activation scales."
|
||||
)
|
||||
|
||||
|
||||
@@ -1770,7 +1770,8 @@ def initialize_dummy_weights(
|
||||
"""
|
||||
for name, param in model.state_dict().items():
|
||||
if torch.is_floating_point(param):
|
||||
if name.endswith("weight_scale_inv"):
|
||||
# Neutral FP4 global scales keep fused gate/up projections compatible.
|
||||
if name.endswith(("weight_scale_inv", "weight_scale_2")):
|
||||
param.fill_(1.0)
|
||||
continue
|
||||
generator = torch.Generator(device=param.data.device)
|
||||
|
||||
@@ -438,6 +438,7 @@ def _uses_ep_a2a() -> bool:
|
||||
_a2a_backend = get_moe_a2a_backend()
|
||||
return (
|
||||
_a2a_backend.is_megamoe()
|
||||
or _a2a_backend.is_flashinfer_megamoe()
|
||||
or _a2a_backend.is_deepep()
|
||||
or _a2a_backend.is_mooncake()
|
||||
or _a2a_backend.is_ascend_fuseep()
|
||||
@@ -546,6 +547,7 @@ class KimiK3MoE(nn.Module):
|
||||
)
|
||||
# mega pre-dispatch consumes raw topk_ids/topk_weights
|
||||
or get_moe_a2a_backend().is_megamoe()
|
||||
or get_moe_a2a_backend().is_flashinfer_megamoe()
|
||||
else None
|
||||
),
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user