From c0bb04b67f269d87fa0a5b79799b268bc52069f2 Mon Sep 17 00:00:00 2001 From: Trevor Morris Date: Sun, 21 Jun 2026 19:35:14 -0700 Subject: [PATCH] [NVIDIA] Support NVFP4 MoE for DeepSeek-V4 (#25820) --- .../autoregressive/DeepSeek/DeepSeek-V4.mdx | 25 +++ .../deepseek-ai/deepseek-v4-benchmarks.jsx | 50 +++++ .../configs/deepseek-ai/deepseek-v4.jsx | 181 ++++++++++++++++++ .../sglang/srt/arg_groups/deepseek_v4_hook.py | 11 ++ python/sglang/srt/configs/model_config.py | 22 +++ python/sglang/srt/layers/moe/hash_topk.py | 12 +- .../moe/moe_runner/flashinfer_trtllm.py | 20 +- .../srt/layers/quantization/modelopt_quant.py | 54 ++++++ python/sglang/srt/model_loader/loader.py | 21 ++ python/sglang/srt/models/deepseek_v4.py | 6 +- 10 files changed, 385 insertions(+), 17 deletions(-) diff --git a/docs_new/cookbook/autoregressive/DeepSeek/DeepSeek-V4.mdx b/docs_new/cookbook/autoregressive/DeepSeek/DeepSeek-V4.mdx index be8d11c40..acde60f04 100644 --- a/docs_new/cookbook/autoregressive/DeepSeek/DeepSeek-V4.mdx +++ b/docs_new/cookbook/autoregressive/DeepSeek/DeepSeek-V4.mdx @@ -247,6 +247,31 @@ sglang serve \ --enable-deepseek-v4-fp4-indexer ``` +**NVFP4 Hybrid Checkpoints** + +The [`nvidia/DeepSeek-V4-Pro-NVFP4`](https://huggingface.co/nvidia/DeepSeek-V4-Pro-NVFP4) and +[`nvidia/DeepSeek-V4-Flash-NVFP4`](https://huggingface.co/nvidia/DeepSeek-V4-Flash-NVFP4) checkpoints +quantize MoE experts to **NVFP4** while keeping attention and dense layers in +**FP8**. It requires `--moe-runner-backend flashinfer_trtllm_routed` which will be automatically selected if not provided. + +```bash Command +sglang serve \ + --model-path nvidia/DeepSeek-V4-Pro-NVFP4 \ + --tp 8 +``` + +or + +```bash Command +sglang serve \ + --model-path nvidia/DeepSeek-V4-Flash-NVFP4 \ + --tp 8 +``` + +Requires Blackwell (SM100+). The MTP layer in this checkpoint stays +MXFP4-packed and is routed through the `Mxfp4FlashinferTrtllmMoEMethod` path +automatically. + **Hopper (H100 / H200) note** diff --git a/docs_new/src/snippets/configs/deepseek-ai/deepseek-v4-benchmarks.jsx b/docs_new/src/snippets/configs/deepseek-ai/deepseek-v4-benchmarks.jsx index 304edbabd..25d8af179 100644 --- a/docs_new/src/snippets/configs/deepseek-ai/deepseek-v4-benchmarks.jsx +++ b/docs_new/src/snippets/configs/deepseek-ai/deepseek-v4-benchmarks.jsx @@ -136,6 +136,31 @@ export const benchmarks = [ match: { hw: "gb200", variant: "pro", quant: "fp4", strategy: "high-throughput", nodes: "multi-2" }, }, // ==================================================================== + // GB200 + NVFP4 + // ==================================================================== + { + match: { hw: "gb200", variant: "flash", quant: "nvfp4", strategy: "low-latency", nodes: "single" }, + sglang_version: "PR #25820", + speed: [ + { workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 1 }, + ttft_ms: 323.85, tpot_ms: 3.62, tokens_per_sec_per_gpu: 496 }, + { workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 16 }, + ttft_ms: 397.31, tpot_ms: 8.11, tokens_per_sec_per_gpu: 3663 }, + ], + accuracy: { gsm8k_pct: 96.66 }, + }, + { + match: { hw: "gb200", variant: "pro", quant: "nvfp4", strategy: "low-latency", nodes: "multi-2" }, + sglang_version: "PR #25820", + speed: [ + { workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 1 }, + ttft_ms: 338.20, tpot_ms: 6.25, tokens_per_sec_per_gpu: 161 }, + { workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 16 }, + ttft_ms: 524.85, tpot_ms: 14.45, tokens_per_sec_per_gpu: 1015 }, + ], + accuracy: { gsm8k_pct: 95.98 }, + }, + // ==================================================================== // GB300 + FP4 // ==================================================================== { @@ -178,6 +203,31 @@ export const benchmarks = [ match: { hw: "gb300", variant: "pro", quant: "fp4", strategy: "high-throughput", nodes: "single" }, }, // ==================================================================== + // GB300 + NVFP4 + // ==================================================================== + { + match: { hw: "gb300", variant: "flash", quant: "nvfp4", strategy: "low-latency", nodes: "single" }, + sglang_version: "PR #25820", + speed: [ + { workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 1 }, + ttft_ms: 361.72 , tpot_ms: 3.62, tokens_per_sec_per_gpu: 480 }, + { workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 16 }, + ttft_ms: 422.96, tpot_ms: 8.19, tokens_per_sec_per_gpu: 3733 }, + ], + accuracy: { gsm8k_pct: 96.44 }, + }, + { + match: { hw: "gb300", variant: "pro", quant: "nvfp4", strategy: "low-latency", nodes: "single" }, + sglang_version: "PR #25820", + speed: [ + { workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 1 }, + ttft_ms: 364.33, tpot_ms: 5.33, tokens_per_sec_per_gpu: 381 }, + { workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 16 }, + ttft_ms: 694.58, tpot_ms: 16.03, tokens_per_sec_per_gpu: 1654 }, + ], + accuracy: { gsm8k_pct: 96.36 }, + }, + // ==================================================================== // H200 + FP8 // ==================================================================== { diff --git a/docs_new/src/snippets/configs/deepseek-ai/deepseek-v4.jsx b/docs_new/src/snippets/configs/deepseek-ai/deepseek-v4.jsx index 2450c213c..d16b6b689 100644 --- a/docs_new/src/snippets/configs/deepseek-ai/deepseek-v4.jsx +++ b/docs_new/src/snippets/configs/deepseek-ai/deepseek-v4.jsx @@ -25,6 +25,7 @@ export const config = { quantizations: [ { id: "fp8", label: "FP8" }, { id: "fp4", label: "FP4" }, + { id: "nvfp4", label: "NVFP4" }, ], strategies: [ { id: "low-latency", label: "Low-Latency" }, @@ -40,8 +41,10 @@ export const config = { modelNames: { "flash|fp4": "deepseek-ai/DeepSeek-V4-Flash", "flash|fp8": "deepseek-ai/DeepSeek-V4-Flash", + "flash|nvfp4": "nvidia/DeepSeek-V4-Flash-NVFP4", "pro|fp4": "deepseek-ai/DeepSeek-V4-Pro", "pro|fp8": "deepseek-ai/DeepSeek-V4-Pro", + "pro|nvfp4": "nvidia/DeepSeek-V4-Pro-NVFP4", // H200 FP8 needs the sgl-project repackaging (Hopper can't run FP4-mixed Instruct). "h200|flash|fp8": "sgl-project/DeepSeek-V4-Flash-FP8", "h200|pro|fp8": "sgl-project/DeepSeek-V4-Pro-FP8", @@ -592,6 +595,95 @@ sgl-eval run aime25 \\ ], }, + // ==================================================================== + // B200 + NVFP4 + // ==================================================================== + { + match: { hw: "b200", variant: "pro", quant: "nvfp4", strategy: "low-latency", nodes: "single" }, + verified: true, + env: [], + flags: [ + "--trust-remote-code", + "--model-path {{MODEL_NAME}}", + "--tp 8", + "--moe-runner-backend flashinfer_trtllm_routed", + "--speculative-algorithm EAGLE", + "--speculative-num-steps 3", + "--speculative-eagle-topk 1", + "--speculative-num-draft-tokens 4", + "--chunked-prefill-size 8192", + "--disable-flashinfer-autotune", + "--swa-full-tokens-ratio 0.1", + "--host {{HOST_IP}}", + "--port {{PORT}}", + ], + }, + + { + match: { hw: "b200", variant: "flash", quant: "nvfp4", strategy: "low-latency", nodes: "single" }, + verified: true, + env: [], + flags: [ + "--trust-remote-code", + "--model-path {{MODEL_NAME}}", + "--tp 4", + "--moe-runner-backend flashinfer_trtllm_routed", + "--speculative-algorithm EAGLE", + "--speculative-num-steps 3", + "--speculative-eagle-topk 1", + "--speculative-num-draft-tokens 4", + "--chunked-prefill-size 4096", + "--disable-flashinfer-autotune", + "--swa-full-tokens-ratio 0.1", + "--host {{HOST_IP}}", + "--port {{PORT}}", + ], + }, + // ==================================================================== + // B300 + NVFP4 + // ==================================================================== + { + match: { hw: "b300", variant: "flash", quant: "nvfp4", strategy: "low-latency", nodes: "single" }, + verified: true, + env: [], + flags: [ + "--trust-remote-code", + "--model-path {{MODEL_NAME}}", + "--tp 4", + "--moe-runner-backend flashinfer_trtllm_routed", + "--speculative-algorithm EAGLE", + "--speculative-num-steps 3", + "--speculative-eagle-topk 1", + "--speculative-num-draft-tokens 4", + "--chunked-prefill-size 4096", + "--disable-flashinfer-autotune", + "--swa-full-tokens-ratio 0.1", + "--host {{HOST_IP}}", + "--port {{PORT}}", + ], + }, + { + match: { hw: "b300", variant: "pro", quant: "nvfp4", strategy: "low-latency", nodes: "single" }, + verified: true, + env: [], + flags: [ + "--trust-remote-code", + "--model-path {{MODEL_NAME}}", + "--tp 8", + "--moe-runner-backend flashinfer_trtllm_routed", + "--speculative-algorithm EAGLE", + "--speculative-num-steps 3", + "--speculative-eagle-topk 1", + "--speculative-num-draft-tokens 4", + "--chunked-prefill-size 8192", + "--disable-flashinfer-autotune", + "--swa-full-tokens-ratio 0.1", + "--mem-fraction-static 0.90", + "--host {{HOST_IP}}", + "--port {{PORT}}", + ], + }, + // ==================================================================== // GB200 + FP4 // ==================================================================== @@ -726,6 +818,51 @@ sgl-eval run aime25 \\ ], }, + // ==================================================================== + // GB200 + NVFP4 + // ==================================================================== + { + match: { hw: "gb200", variant: "flash", quant: "nvfp4", strategy: "low-latency", nodes: "single" }, + verified: true, + env: [], + flags: [ + "--trust-remote-code", + "--model-path {{MODEL_NAME}}", + "--tp 4", + "--moe-runner-backend flashinfer_trtllm_routed", + "--speculative-algorithm EAGLE", + "--speculative-num-steps 3", + "--speculative-eagle-topk 1", + "--speculative-num-draft-tokens 4", + "--chunked-prefill-size 4096", + "--disable-flashinfer-autotune", + "--swa-full-tokens-ratio 0.1", + "--host {{HOST_IP}}", + "--port {{PORT}}", + ], + }, + { + match: { hw: "gb200", variant: "pro", quant: "nvfp4", strategy: "low-latency", nodes: "multi-2" }, + verified: true, + env: [], + flags: [ + "--trust-remote-code", + "--model-path {{MODEL_NAME}}", + "--tp 8", + "--moe-runner-backend flashinfer_trtllm_routed", + "--speculative-algorithm EAGLE", + "--speculative-num-steps 3", + "--speculative-eagle-topk 1", + "--speculative-num-draft-tokens 4", + "--chunked-prefill-size 8192", + "--disable-flashinfer-autotune", + "--swa-full-tokens-ratio 0.1", + "--mem-fraction-static 0.90", + "--host {{HOST_IP}}", + "--port {{PORT}}", + ], + }, + // ==================================================================== // GB300 + FP4 // ==================================================================== @@ -849,6 +986,50 @@ sgl-eval run aime25 \\ "--port {{PORT}}", ], }, + // ==================================================================== + // GB200 + NVFP4 + // ==================================================================== + { + match: { hw: "gb300", variant: "flash", quant: "nvfp4", strategy: "low-latency", nodes: "single" }, + verified: true, + env: [], + flags: [ + "--trust-remote-code", + "--model-path {{MODEL_NAME}}", + "--tp 4", + "--moe-runner-backend flashinfer_trtllm_routed", + "--speculative-algorithm EAGLE", + "--speculative-num-steps 3", + "--speculative-eagle-topk 1", + "--speculative-num-draft-tokens 4", + "--chunked-prefill-size 4096", + "--disable-flashinfer-autotune", + "--swa-full-tokens-ratio 0.1", + "--host {{HOST_IP}}", + "--port {{PORT}}", + ], + }, + { + match: { hw: "gb300", variant: "pro", quant: "nvfp4", strategy: "low-latency", nodes: "single" }, + verified: true, + env: [], + flags: [ + "--trust-remote-code", + "--model-path {{MODEL_NAME}}", + "--tp 4", + "--moe-runner-backend flashinfer_trtllm_routed", + "--speculative-algorithm EAGLE", + "--speculative-num-steps 3", + "--speculative-eagle-topk 1", + "--speculative-num-draft-tokens 4", + "--chunked-prefill-size 8192", + "--disable-flashinfer-autotune", + "--swa-full-tokens-ratio 0.1", + "--mem-fraction-static 0.90", + "--host {{HOST_IP}}", + "--port {{PORT}}", + ], + }, // ==================================================================== // H200 + FP8 (deepep, no Marlin) diff --git a/python/sglang/srt/arg_groups/deepseek_v4_hook.py b/python/sglang/srt/arg_groups/deepseek_v4_hook.py index 1af1a1e4f..19ea8e235 100644 --- a/python/sglang/srt/arg_groups/deepseek_v4_hook.py +++ b/python/sglang/srt/arg_groups/deepseek_v4_hook.py @@ -59,6 +59,17 @@ def apply_deepseek_v4_defaults(server_args: ServerArgs, model_arch: str) -> None f"Setting swa_full_tokens_ratio to {server_args.swa_full_tokens_ratio} for {model_arch}." ) + # nvidia/DeepSeek-V4-Pro-NVFP4 uses flashinfer_trtllm_routed MoE runner backend. + if ( + server_args.moe_runner_backend == "auto" + and server_args.get_model_config().nvfp4_moe_meta is not None + ): + server_args.moe_runner_backend = "flashinfer_trtllm_routed" + logger.info( + "Use flashinfer_trtllm_routed as MoE runner backend for " + f"{model_arch} hybrid FP8+NVFP4 checkpoint." + ) + def validate_deepseek_v4_cp(server_args: ServerArgs) -> None: """Validate DeepSeek V4 context-parallel configuration.""" diff --git a/python/sglang/srt/configs/model_config.py b/python/sglang/srt/configs/model_config.py index be029153a..964bdcd6f 100644 --- a/python/sglang/srt/configs/model_config.py +++ b/python/sglang/srt/configs/model_config.py @@ -302,6 +302,28 @@ class ModelConfig: if n_group is not None: self.hf_config.topk_group = n_group + # Handle hybrid NVFP4 moe (nvidia/DeepSeek-V4-Pro-NVFP4) + self.nvfp4_moe_meta: Optional[dict] = None + hybrid_quant_cfg = getattr(self.hf_config, "quantization_config", None) + if hybrid_quant_cfg is not None and not isinstance(hybrid_quant_cfg, dict): + hybrid_quant_cfg = hybrid_quant_cfg.to_dict() + if ( + hybrid_quant_cfg is not None + and str(hybrid_quant_cfg.get("quant_algo", "")).upper() == "MIXED_PRECISION" + and str(hybrid_quant_cfg.get("moe_quant_algo", "")).upper() == "NVFP4" + and hybrid_quant_cfg.get("group_size") is not None + ): + self.nvfp4_moe_meta = { + "group_size": int(hybrid_quant_cfg["group_size"]), + "exclude_modules": list(hybrid_quant_cfg.get("ignore") or []), + } + logger.info( + "Auto-detected hybrid FP8+NVFP4 checkpoint " + "(NVFP4 MoE group_size=%d, %d exclude_modules)", + self.nvfp4_moe_meta["group_size"], + len(self.nvfp4_moe_meta["exclude_modules"]), + ) + # Check model type self.attention_chunk_size = getattr( self.hf_text_config, "attention_chunk_size", None diff --git a/python/sglang/srt/layers/moe/hash_topk.py b/python/sglang/srt/layers/moe/hash_topk.py index bcc5f2c58..55ec9ef6f 100644 --- a/python/sglang/srt/layers/moe/hash_topk.py +++ b/python/sglang/srt/layers/moe/hash_topk.py @@ -65,7 +65,14 @@ class HashTopK(nn.Module): ) self._init_default_tid2eid() - assert not apply_routed_scaling_factor_on_output, "not implemented" + self.apply_routed_scaling_factor_on_output = ( + apply_routed_scaling_factor_on_output + ) + if apply_routed_scaling_factor_on_output and num_fused_shared_experts > 0: + raise NotImplementedError( + "HashTopK + apply_routed_scaling_factor_on_output is not supported " + "with fused shared experts; pass --disable-shared-experts-fusion." + ) def _init_default_tid2eid(self) -> None: topk = self.tid2eid.shape[1] @@ -188,6 +195,9 @@ class HashTopK(nn.Module): if _is_hip or _is_npu: topk_weights = topk_weights.to(torch.float32) + if self.apply_routed_scaling_factor_on_output: + topk_weights = topk_weights * self.routed_scaling_factor + log2phy_prob = None if ( expert_location_dispatch_info is not None diff --git a/python/sglang/srt/layers/moe/moe_runner/flashinfer_trtllm.py b/python/sglang/srt/layers/moe/moe_runner/flashinfer_trtllm.py index c8a88f8fb..09b77a5bb 100644 --- a/python/sglang/srt/layers/moe/moe_runner/flashinfer_trtllm.py +++ b/python/sglang/srt/layers/moe/moe_runner/flashinfer_trtllm.py @@ -3,7 +3,7 @@ from __future__ import annotations import contextvars from contextlib import contextmanager from dataclasses import dataclass -from typing import TYPE_CHECKING, Generator, cast +from typing import TYPE_CHECKING, Generator, Optional, cast import torch from torch.nn import Module @@ -857,6 +857,8 @@ class FlashInferTrtllmFp4MoeQuantInfo(MoeQuantInfo): routing_method_type: int use_per_token_activation: bool = False + gemm1_clamp_limit: Optional[torch.Tensor] = None + def quantize_hidden_states_fp4( hidden_states: torch.Tensor, @@ -953,18 +955,6 @@ def fused_experts_none_to_flashinfer_trtllm_fp4( runner_config.activation, is_gated=runner_config.is_gated ) - # Build per-expert clamp-limit tensor from the per-layer scalar. - _clamp_val = runner_config.gemm1_clamp_limit - if _clamp_val is not None: - gemm1_clamp_limit = torch.full( - (quant_info.local_num_experts,), - _clamp_val, - dtype=torch.float32, - device=hs_fp4.device, - ) - else: - gemm1_clamp_limit = None - # Fall back to routed path when topk was already materialized (e.g. sigmoid routing). if not use_routed_topk and TopKOutputChecker.format_is_standard(topk_output): use_routed_topk = True @@ -1020,7 +1010,7 @@ def fused_experts_none_to_flashinfer_trtllm_fp4( gemm1_bias=None, gemm1_alpha=None, gemm1_beta=None, - gemm1_clamp_limit=gemm1_clamp_limit, + gemm1_clamp_limit=quant_info.gemm1_clamp_limit, gemm2_weights=quant_info.w2_weight, gemm2_weights_scale=quant_info.w2_weight_scale.view(torch.float8_e4m3fn), gemm2_bias=None, @@ -1060,7 +1050,7 @@ def fused_experts_none_to_flashinfer_trtllm_fp4( gemm1_bias=None, gemm1_alpha=None, gemm1_beta=None, - gemm1_clamp_limit=gemm1_clamp_limit, + gemm1_clamp_limit=quant_info.gemm1_clamp_limit, gemm2_weights=quant_info.w2_weight, gemm2_weights_scale=quant_info.w2_weight_scale.view(torch.float8_e4m3fn), gemm2_bias=None, diff --git a/python/sglang/srt/layers/quantization/modelopt_quant.py b/python/sglang/srt/layers/quantization/modelopt_quant.py index 560e2ffc9..b31f996a7 100755 --- a/python/sglang/srt/layers/quantization/modelopt_quant.py +++ b/python/sglang/srt/layers/quantization/modelopt_quant.py @@ -41,6 +41,7 @@ from sglang.srt.layers.quantization.fp4_utils import ( fp4_quantize, get_fp4_gemm_runner_backend, ) +from sglang.srt.layers.quantization.fp8 import Fp8Config from sglang.srt.layers.quantization.fp8_kernel import scaled_fp8_quant from sglang.srt.layers.quantization.fp8_utils import ( apply_fp8_linear, @@ -1354,6 +1355,45 @@ class ModelOptFp4Config(ModelOptQuantConfig): ) +class HybridFp8NvFp4Config(Fp8Config): + """FP8 (linear/attention/MTP MoE) + NVFP4 (FusedMoE) hybrid quantization. + + For checkpoints like nvidia/DeepSeek-V4-Pro-NVFP4 where + config.json:quantization_config declares quant_method=fp8 and + moe_quant_algo=NVFP4. FusedMoE layers route through + ModelOptNvFp4FusedMoEMethod; linear / attention layers + delegate to the inherited Fp8Config dispatch. + """ + + def __init__(self, fp8_config: Fp8Config, nvfp4_config: ModelOptFp4Config): + # Inherit all of fp8_config's state without re-running its + # validation / logging (already happened at fp8_config build time). + self.__dict__.update(fp8_config.__dict__) + self.nvfp4_config = nvfp4_config + + def get_quant_method( + self, layer: torch.nn.Module, prefix: str + ) -> Optional[QuantizeMethodBase]: + from sglang.srt.layers.moe.fused_moe_triton import FusedMoE + + if isinstance(layer, FusedMoE): + if not self.nvfp4_config.is_layer_excluded(prefix): + return ModelOptNvFp4FusedMoEMethod(self.nvfp4_config) + # Fall back to MXFP4 for MTP MoE layers + if self.is_fp4_experts: + from sglang.srt.layers.quantization.fp8 import Fp8MoEMethod + from sglang.srt.layers.quantization.mxfp4_flashinfer_trtllm_moe import ( + Mxfp4FlashinferTrtllmMoEMethod, + ) + + return Mxfp4FlashinferTrtllmMoEMethod(Fp8MoEMethod(self), prefix=prefix) + return super().get_quant_method(layer, prefix) + + def apply_weight_name_mapper(self, hf_to_sglang_mapper: WeightsMapper): + super().apply_weight_name_mapper(hf_to_sglang_mapper) + self.nvfp4_config.apply_weight_name_mapper(hf_to_sglang_mapper) + + class ModelOptFp4LinearMethod(LinearMethodBase): """Linear method for NVFP4. Supports loading NVFP4 checkpoints with the following structure: @@ -2053,6 +2093,18 @@ class ModelOptNvFp4FusedMoEMethod(FusedMoEMethodBase): (1 / w2_input_scale).to(torch.float32), ) + swiglu_limit = layer.moe_runner_config.swiglu_limit + if ( + swiglu_limit is not None + and layer.moe_runner_config.is_gated + and self.enable_flashinfer_trtllm_moe + ): + copy_or_rebind_param( + layer, + "gemm1_clamp_limit", + (swiglu_limit / layer.g1_alphas).to(torch.float32), + ) + # TODO: for flashinfer always do MOE_NVFP4_DISPATCH layer.dispatcher.set_quant_config( { @@ -2338,6 +2390,7 @@ class ModelOptNvFp4FusedMoEMethod(FusedMoEMethodBase): layer, "routing_method_type", RoutingMethodType.Default ) + gemm1_clamp = getattr(layer, "gemm1_clamp_limit", None) quant_info = FlashInferTrtllmFp4MoeQuantInfo( w13_weight=layer.w13_weight.data, w2_weight=layer.w2_weight.data, @@ -2353,6 +2406,7 @@ class ModelOptNvFp4FusedMoEMethod(FusedMoEMethodBase): intermediate_size_per_partition=layer.intermediate_size_per_partition, routing_method_type=routing_method_type, use_per_token_activation=self.quant_config.use_per_token_activation, + gemm1_clamp_limit=gemm1_clamp.data if gemm1_clamp is not None else None, ) return self.runner.run(dispatch_output, quant_info) diff --git a/python/sglang/srt/model_loader/loader.py b/python/sglang/srt/model_loader/loader.py index 01756de35..edf06cb3e 100644 --- a/python/sglang/srt/model_loader/loader.py +++ b/python/sglang/srt/model_loader/loader.py @@ -245,6 +245,27 @@ def _get_quantization_config( if isinstance(quant_config, Fp8Config): quant_config.is_fp4_experts = model_config.is_fp4_experts + # Handle hybrid NVFP4 moe (nvidia/DeepSeek-V4-Pro-NVFP4) + nvfp4_meta = model_config.nvfp4_moe_meta + if nvfp4_meta is not None: + from sglang.srt.layers.quantization.modelopt_quant import ( + HybridFp8NvFp4Config, + ModelOptFp4Config, + ) + + # MTP MoE layers (model.decoder.*) are not NVFP4 quantized. + nvfp4_exclude_modules = list( + nvfp4_meta.get("exclude_modules") or [] + ) + ["model.decoder.*"] + nvfp4_config = ModelOptFp4Config( + is_checkpoint_nvfp4_serialized=True, + group_size=int(nvfp4_meta["group_size"]), + exclude_modules=nvfp4_exclude_modules, + packed_modules_mapping=quant_config.packed_modules_mapping, + ) + quant_config = HybridFp8NvFp4Config( + fp8_config=quant_config, nvfp4_config=nvfp4_config + ) if not _is_npu: major, minor = get_device_capability() diff --git a/python/sglang/srt/models/deepseek_v4.py b/python/sglang/srt/models/deepseek_v4.py index c2c60ce35..a5520d91c 100644 --- a/python/sglang/srt/models/deepseek_v4.py +++ b/python/sglang/srt/models/deepseek_v4.py @@ -2394,7 +2394,11 @@ class DeepseekV4ForCausalLM(nn.Module): assert len(cache_wqkv_a_weight) == 0, cache_wqkv_a_weight.keys() unloaded_params = params_dict.keys() - loaded_params - skipped_checking_patterns = ["attn_mqa.k_scale", "attn_mqa.v_scale"] + skipped_checking_patterns = [ + "attn_mqa.k_scale", + "attn_mqa.v_scale", + "blockscale_swizzled", + ] if not self.pp_group.is_first_rank: skipped_checking_patterns.append("embed_tokens") if not self.pp_group.is_last_rank: