[diffusion] Add Qwen-Image ModelOpt NVFP4 support (#28928)

Co-authored-by: jingyu-ml <jingyux@nvidia.com>
Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

python/sglang/multimodal_gen/registry.py

@@ -859,7 +859,7 @@ def _register_configs():
     register_configs(
         sampling_param_cls=QwenImageSamplingParams,
         pipeline_config_cls=QwenImagePipelineConfig,
-        hf_model_paths=["Qwen/Qwen-Image"],
+        hf_model_paths=["Qwen/Qwen-Image", "nvidia/Qwen-Image-NVFP4"],
         model_detectors=[
             lambda hf_id: (
                 "qwen-image" in hf_id.lower()

python/sglang/multimodal_gen/runtime/utils/quantization_utils.py

@@ -301,6 +301,22 @@ def get_metadata_from_safetensors_file(file_path: str):
         logger.warning(e)
 
 
+def _canonicalize_modulation_exclude(module_name: str) -> str:
+    """Map a serialized modulation weight's parent to the runtime linear prefix.
+
+    Qwen-Image wraps the modulation projection in ``nn.Sequential(SiLU, Linear)``,
+    so its weights serialize as ``...img_mod.1.weight`` while the runtime
+    ReplicatedLinear advertises ``...img_mod`` as its quant/exclusion prefix.
+    Strip the trailing Sequential index so a safetensors-inferred BF16 exclude
+    entry actually matches the linear (mirrors the ModelOpt FP8 converter, which
+    canonicalizes ``.img_mod.1``/``.txt_mod.1`` to ``.img_mod``/``.txt_mod``).
+    No-op for any other module name.
+    """
+    if module_name.endswith((".img_mod.1", ".txt_mod.1")):
+        return module_name.removesuffix(".1")
+    return module_name
+
+
 def _build_nvfp4_config_from_safetensors_files(
     file_paths: list[str],
     param_names_mapping_dict: Optional[dict] = None,
@@ -470,7 +486,9 @@ def _build_nvfp4_config_from_safetensors_files(
 
         exclude_modules.append(module_bfl)
 
-    exclude_modules = sorted(set(exclude_modules))
+    exclude_modules = sorted(
+        {_canonicalize_modulation_exclude(m) for m in exclude_modules}
+    )
 
     try:
         quant_cls = get_quantization_config("modelopt_fp4")

python/sglang/multimodal_gen/test/server/consistency_threshold.json

@@ -25,6 +25,12 @@
             "psnr_threshold": 18.0,
             "mean_abs_diff_threshold": 18.0
         },
+        "qwen_image_2512_modelopt_nvfp4_t2i": {
+            "clip_threshold": 0.95,
+            "ssim_threshold": 0.88,
+            "psnr_threshold": 23.0,
+            "mean_abs_diff_threshold": 9.0
+        },
         "flux_2_klein_image_t2i": {
             "clip_threshold": 0.94,
             "ssim_threshold": 0.78,

python/sglang/multimodal_gen/test/server/gpu_cases.py

@@ -9,6 +9,7 @@ from sglang.multimodal_gen.test.server.testcase_configs import (
     MODELOPT_FLUX2_NVFP4_WEIGHTS,
     MODELOPT_HUNYUANVIDEO_FP8_TRANSFORMER,
     MODELOPT_NVFP4_B200_ENV_VARS,
+    MODELOPT_QWEN_IMAGE_2512_NVFP4_MODEL,
     MODELOPT_QWEN_IMAGE_EDIT_FP8_TRANSFORMER,
     MODELOPT_QWEN_IMAGE_FP8_TRANSFORMER,
     MODELOPT_WAN22_FP8_MODEL,
@@ -20,6 +21,7 @@ from sglang.multimodal_gen.test.server.testcase_configs import (
     DiffusionTestCase,
     IDEOGRAM4_CI_sampling_params,
     LINGBOT_WORLD_REALTIME_sampling_params,
+    MODELOPT_QWEN_IMAGE_2512_NVFP4_CI_sampling_params,
     MODELOPT_T2I_CI_sampling_params,
     MODELOPT_T2V_CI_sampling_params,
     MODELOPT_TI2I_CI_sampling_params,
@@ -550,6 +552,15 @@ else:
             env_vars=MODELOPT_NVFP4_B200_ENV_VARS,
             run_consistency_check=True,
         ),
+        _make_modelopt_ci_case(
+            "qwen_image_2512_modelopt_nvfp4_t2i",
+            model_path=MODELOPT_QWEN_IMAGE_2512_NVFP4_MODEL,
+            modality="image",
+            sampling_params=MODELOPT_QWEN_IMAGE_2512_NVFP4_CI_sampling_params,
+            extras=[],
+            env_vars=MODELOPT_NVFP4_B200_ENV_VARS,
+            run_consistency_check=True,
+        ),
         _make_modelopt_ci_case(
             "wan22_modelopt_nvfp4_t2v",
             model_path=MODELOPT_WAN22_NVFP4_MODEL,

python/sglang/multimodal_gen/test/server/perf_baselines.json

@@ -2846,6 +2846,14 @@
             "expected_median_denoise_ms": 0.0,
             "estimated_full_test_time_s": 592.3
         },
+        "qwen_image_2512_modelopt_nvfp4_t2i": {
+            "stages_ms": {},
+            "denoise_step_ms": {},
+            "expected_e2e_ms": 0.0,
+            "expected_avg_denoise_ms": 0.0,
+            "expected_median_denoise_ms": 0.0,
+            "estimated_full_test_time_s": 120.0
+        },
         "wan22_modelopt_nvfp4_t2v": {
             "stages_ms": {},
             "denoise_step_ms": {},

python/sglang/multimodal_gen/test/server/testcase_configs.py

@@ -488,6 +488,11 @@ MODELOPT_T2I_CI_sampling_params = DiffusionSamplingParams(
     extras={"num_inference_steps": 12, "seed": 0},
 )
 
+MODELOPT_QWEN_IMAGE_2512_NVFP4_CI_sampling_params = replace(
+    MODELOPT_T2I_CI_sampling_params,
+    extras={"num_inference_steps": 50, "seed": 0},
+)
+
 MODELOPT_TI2I_CI_sampling_params = DiffusionSamplingParams(
     prompt="Convert 2D style to 3D style",
     image_path="https://github.com/lm-sys/lm-sys.github.io/releases/download/test/TI2I_Qwen_Image_Edit_Input.jpg",
@@ -636,6 +641,7 @@ MODELOPT_QWEN_IMAGE_EDIT_FP8_TRANSFORMER = (
 )
 MODELOPT_FLUX1_NVFP4_TRANSFORMER = "lmsys/flux1-dev-modelopt-nvfp4-sglang-transformer"
 MODELOPT_FLUX2_NVFP4_WEIGHTS = "black-forest-labs/FLUX.2-dev-NVFP4"
+MODELOPT_QWEN_IMAGE_2512_NVFP4_MODEL = "lmsys/qwen-image-2512-modelopt-nvfp4-sglang"
 MODELOPT_WAN22_NVFP4_MODEL = "nvidia/Wan2.2-T2V-A14B-Diffusers-NVFP4"
 MODELOPT_NVFP4_B200_ENV_VARS = {}
 MODELOPT_WAN22_NVFP4_B200_ENV_VARS = {}

python/sglang/multimodal_gen/test/test_utils.py

@@ -34,7 +34,7 @@ if TYPE_CHECKING:
 
 logger = init_logger(__name__)
 
-SGL_TEST_FILES_CI_DATA_REVISION = "14a963c21b96bcc4bb3811be02076977e8f8439b"
+SGL_TEST_FILES_CI_DATA_REVISION = "3c6e06ae99001d93f7901bc9b7fdf19ec6c2ce4e"
 
 if current_platform.is_npu():
     SGL_TEST_FILES_CI_DATA_REVISION = "670d66a8a290b62c0c3c077b3e9b0f4a4d9a44e7"