From 03b9278da06c7b93e5affaf9796f144241710490 Mon Sep 17 00:00:00 2001 From: Mick Date: Thu, 2 Jul 2026 01:32:51 +0800 Subject: [PATCH] [diffusion] CI: tighten multimodal-gen consistency thresholds (#29824) --- .../server/consistency_thresholds/h100.json | 282 +++++++++--------- 1 file changed, 141 insertions(+), 141 deletions(-) diff --git a/python/sglang/multimodal_gen/test/server/consistency_thresholds/h100.json b/python/sglang/multimodal_gen/test/server/consistency_thresholds/h100.json index 1d8eb9852..bf2209758 100644 --- a/python/sglang/multimodal_gen/test/server/consistency_thresholds/h100.json +++ b/python/sglang/multimodal_gen/test/server/consistency_thresholds/h100.json @@ -8,28 +8,28 @@ "mean_abs_diff_threshold": 13.3 }, "flux_image_t2i": { - "clip_threshold": 0.92, + "clip_threshold": 0.97, "ssim_threshold": 0.95, - "psnr_threshold": 24.0, - "mean_abs_diff_threshold": 8.0 + "psnr_threshold": 30.0, + "mean_abs_diff_threshold": 4.0 }, "flux1_modelopt_fp8_t2i": { - "clip_threshold": 0.92, + "clip_threshold": 0.96, "ssim_threshold": 0.94, "psnr_threshold": 28.0, - "mean_abs_diff_threshold": 8.0 + "mean_abs_diff_threshold": 6.3 }, "flux2_modelopt_nvfp4_t2i": { - "clip_threshold": 0.92, - "ssim_threshold": 0.86, + "clip_threshold": 0.98, + "ssim_threshold": 0.95, "psnr_threshold": 17.0, - "mean_abs_diff_threshold": 9.5 + "mean_abs_diff_threshold": 4.0 }, "ideogram4_nvfp4_t2i": { - "clip_threshold": 0.97, - "ssim_threshold": 0.78, + "clip_threshold": 0.98, + "ssim_threshold": 0.95, "psnr_threshold": 18.0, - "mean_abs_diff_threshold": 18.0 + "mean_abs_diff_threshold": 4.0 }, "qwen_image_2512_modelopt_nvfp4_t2i": { "clip_threshold": 0.95, @@ -38,142 +38,142 @@ "mean_abs_diff_threshold": 9.0 }, "flux_2_klein_image_t2i": { - "clip_threshold": 0.94, - "ssim_threshold": 0.78, + "clip_threshold": 0.98, + "ssim_threshold": 0.95, "psnr_threshold": 17.0, - "mean_abs_diff_threshold": 17.0 + "mean_abs_diff_threshold": 4.0 }, "zimage_image_t2i": { - "clip_threshold": 0.92, - "ssim_threshold": 0.86, - "psnr_threshold": 19.9, - "mean_abs_diff_threshold": 8.5 + "clip_threshold": 0.97, + "ssim_threshold": 0.95, + "psnr_threshold": 30.0, + "mean_abs_diff_threshold": 4.0 }, "zimage_image_t2i_multi_lora": { - "clip_threshold": 0.92, - "ssim_threshold": 0.89, + "clip_threshold": 0.98, + "ssim_threshold": 0.95, "psnr_threshold": 19.9, - "mean_abs_diff_threshold": 8.0 + "mean_abs_diff_threshold": 4.0 }, "qwen_image_t2i_cache_dit_enabled": { "clip_threshold": 0.98, - "ssim_threshold": 0.73, + "ssim_threshold": 0.78, "psnr_threshold": 13.5, - "mean_abs_diff_threshold": 26.0 + "mean_abs_diff_threshold": 17.2 }, "qwen_image_t2i_2_gpus": { "clip_threshold": 0.98, "ssim_threshold": 0.79, - "psnr_threshold": 14.3, - "mean_abs_diff_threshold": 20.0 + "psnr_threshold": 15.7, + "mean_abs_diff_threshold": 17.2 }, "flux_2_image_t2i": { "clip_threshold": 0.98, - "ssim_threshold": 0.86, + "ssim_threshold": 0.95, "psnr_threshold": 14.9, - "mean_abs_diff_threshold": 13.0 + "mean_abs_diff_threshold": 4.0 }, "flux_2_ti2i": { - "clip_threshold": 0.96, - "ssim_threshold": 0.88, + "clip_threshold": 0.98, + "ssim_threshold": 0.95, "psnr_threshold": 19.5, - "mean_abs_diff_threshold": 13.5 + "mean_abs_diff_threshold": 4.0 }, "zimage_image_t2i_fp8": { - "clip_threshold": 0.94, - "ssim_threshold": 0.88, - "psnr_threshold": 21.0, - "mean_abs_diff_threshold": 9.5 + "clip_threshold": 0.97, + "ssim_threshold": 0.95, + "psnr_threshold": 30.0, + "mean_abs_diff_threshold": 4.0 }, "qwen_image_edit_2509_ti2i": { - "clip_threshold": 0.75, - "ssim_threshold": 0.52, - "psnr_threshold": 10.5, - "mean_abs_diff_threshold": 46.5 + "clip_threshold": 0.91, + "ssim_threshold": 0.68, + "psnr_threshold": 11.3, + "mean_abs_diff_threshold": 26.4 }, "qwen_image_edit_ti2i": { - "clip_threshold": 0.96, - "ssim_threshold": 0.94, - "psnr_threshold": 25.4, - "mean_abs_diff_threshold": 10.0 + "clip_threshold": 0.97, + "ssim_threshold": 0.95, + "psnr_threshold": 30.0, + "mean_abs_diff_threshold": 4.0 }, "qwen_image_edit_2511_ti2i": { - "clip_threshold": 0.96, - "ssim_threshold": 0.83, - "psnr_threshold": 21.0, - "mean_abs_diff_threshold": 16.6 + "clip_threshold": 0.97, + "ssim_threshold": 0.95, + "psnr_threshold": 30.0, + "mean_abs_diff_threshold": 4.1 }, "qwen_image_layered_i2i": { - "clip_threshold": 0.92, - "ssim_threshold": 0.94, - "psnr_threshold": 28.0, - "mean_abs_diff_threshold": 8.0 + "clip_threshold": 0.97, + "ssim_threshold": 0.95, + "psnr_threshold": 30.0, + "mean_abs_diff_threshold": 4.2 }, "wan2_1_t2v_1_3b_lora_1gpu": { - "clip_threshold": 0.88, - "ssim_threshold": 0.75, - "psnr_threshold": 17.0, - "mean_abs_diff_threshold": 14.0 + "clip_threshold": 0.97, + "ssim_threshold": 0.94, + "psnr_threshold": 30.0, + "mean_abs_diff_threshold": 4.0 }, "wan2_1_t2v_1.3b": { - "clip_threshold": 0.94, - "ssim_threshold": 0.85, - "psnr_threshold": 25.0, - "mean_abs_diff_threshold": 8.0 + "clip_threshold": 0.97, + "ssim_threshold": 0.90, + "psnr_threshold": 30.0, + "mean_abs_diff_threshold": 5.6 }, "cosmos3_nano_t2v": { - "clip_threshold": 0.90, + "clip_threshold": 0.91, "ssim_threshold": 0.89, "psnr_threshold": 24.0, - "mean_abs_diff_threshold": 10.0 + "mean_abs_diff_threshold": 9.9 }, "ltx_2_two_stage_t2v": { - "clip_threshold": 0.90, + "clip_threshold": 0.97, "ssim_threshold": 0.89, - "psnr_threshold": 24.0, - "mean_abs_diff_threshold": 10.0 + "psnr_threshold": 30.0, + "mean_abs_diff_threshold": 5.9 }, "ltx_2.3_one_stage_ti2v": { - "clip_threshold": 0.84, + "clip_threshold": 0.87, "ssim_threshold": 0.78, "psnr_threshold": 20.5, "mean_abs_diff_threshold": 11.0 }, "ltx_2.3_two_stage_t2v_2gpus": { - "clip_threshold": 0.79, - "ssim_threshold": 0.12, + "clip_threshold": 0.94, + "ssim_threshold": 0.45, "psnr_threshold": 16.0, - "mean_abs_diff_threshold": 51.0 + "mean_abs_diff_threshold": 27.5 }, "wan2_1_t2v_1.3b_teacache_enabled": { - "clip_threshold": 0.93, + "clip_threshold": 0.97, "ssim_threshold": 0.92, - "psnr_threshold": 24.0, - "mean_abs_diff_threshold": 9.0 + "psnr_threshold": 30.0, + "mean_abs_diff_threshold": 5.5 }, "wan2_1_t2v_1.3b_upscaling_4x": { - "clip_threshold": 0.94, + "clip_threshold": 0.97, "ssim_threshold": 0.94, - "psnr_threshold": 26.0, - "mean_abs_diff_threshold": 8.0 + "psnr_threshold": 30.0, + "mean_abs_diff_threshold": 4.6 }, "wan2_2_ti2v_5b": { - "clip_threshold": 0.92, + "clip_threshold": 0.94, "ssim_threshold": 0.88, - "psnr_threshold": 22.0, - "mean_abs_diff_threshold": 9.0 + "psnr_threshold": 27.1, + "mean_abs_diff_threshold": 7.2 }, "wan22_modelopt_nvfp4_t2v": { - "clip_threshold": 0.90, - "ssim_threshold": 0.90, + "clip_threshold": 0.98, + "ssim_threshold": 0.95, "psnr_threshold": 24.0, - "mean_abs_diff_threshold": 10.0 + "mean_abs_diff_threshold": 4.0 }, "fastwan2_2_ti2v_5b": { - "clip_threshold": 0.90, - "ssim_threshold": 0.88, - "psnr_threshold": 24.0, - "mean_abs_diff_threshold": 10.0 + "clip_threshold": 0.97, + "ssim_threshold": 0.92, + "psnr_threshold": 30.0, + "mean_abs_diff_threshold": 4.7 }, "turbo_wan2_1_t2v_1.3b": { "clip_threshold": 0.90, @@ -182,34 +182,34 @@ "mean_abs_diff_threshold": 46.0 }, "fsdp-inference": { - "clip_threshold": 0.92, - "ssim_threshold": 0.90, - "psnr_threshold": 21.5, - "mean_abs_diff_threshold": 8.0 + "clip_threshold": 0.97, + "ssim_threshold": 0.95, + "psnr_threshold": 30.0, + "mean_abs_diff_threshold": 4.0 }, "zimage_image_t2i_2_gpus_non_square": { - "clip_threshold": 0.92, - "ssim_threshold": 0.76, - "psnr_threshold": 14.8, - "mean_abs_diff_threshold": 17.5 + "clip_threshold": 0.97, + "ssim_threshold": 0.95, + "psnr_threshold": 30.0, + "mean_abs_diff_threshold": 4.0 }, "flux_2_image_t2i_2_gpus": { - "clip_threshold": 0.54, - "ssim_threshold": 0.9, + "clip_threshold": 0.98, + "ssim_threshold": 0.95, "psnr_threshold": 19, - "mean_abs_diff_threshold": 8.0 + "mean_abs_diff_threshold": 4.0 }, "flux_2_image_t2i_upscaling_4x": { - "clip_threshold": 0.96, + "clip_threshold": 0.98, "ssim_threshold": 0.95, - "psnr_threshold": 28.0, - "mean_abs_diff_threshold": 8.0 + "psnr_threshold": 30.0, + "mean_abs_diff_threshold": 4.0 }, "flux_2_t2i_customized_vae_path": { - "clip_threshold": 0.96, + "clip_threshold": 0.98, "ssim_threshold": 0.95, "psnr_threshold": 28.0, - "mean_abs_diff_threshold": 8.0 + "mean_abs_diff_threshold": 4.0 }, "flux_2_ti2i_multi_image_cache_dit": { "clip_threshold": 0.94, @@ -218,70 +218,70 @@ "mean_abs_diff_threshold": 9.0 }, "zimage_image_t2i_2_gpus": { - "clip_threshold": 0.92, - "ssim_threshold": 0.90, - "psnr_threshold": 21.5, - "mean_abs_diff_threshold": 8.0 + "clip_threshold": 0.97, + "ssim_threshold": 0.95, + "psnr_threshold": 30.0, + "mean_abs_diff_threshold": 4.0 }, "flux_image_t2i_2_gpus": { - "clip_threshold": 0.92, - "ssim_threshold": 0.90, - "psnr_threshold": 18.7, - "mean_abs_diff_threshold": 8.0 + "clip_threshold": 0.97, + "ssim_threshold": 0.95, + "psnr_threshold": 25.8, + "mean_abs_diff_threshold": 4.0 }, "wan2_2_t2v_a14b_teacache_2gpu": { - "clip_threshold": 0.90, - "ssim_threshold": 0.72, - "psnr_threshold": 17.8, - "mean_abs_diff_threshold": 16.0 + "clip_threshold": 0.97, + "ssim_threshold": 0.90, + "psnr_threshold": 29.1, + "mean_abs_diff_threshold": 6.8 }, "wan2_1_t2v_14b_2gpu": { - "clip_threshold": 0.90, - "ssim_threshold": 0.84, - "psnr_threshold": 24.0, - "mean_abs_diff_threshold": 10.0 + "clip_threshold": 0.97, + "ssim_threshold": 0.91, + "psnr_threshold": 30.0, + "mean_abs_diff_threshold": 5.6 }, "mova_360p_ring1_uly2": { - "clip_threshold": 0.90, + "clip_threshold": 0.96, "ssim_threshold": 0.91, - "psnr_threshold": 24.0, - "mean_abs_diff_threshold": 10.0 + "psnr_threshold": 30.0, + "mean_abs_diff_threshold": 6.2 }, "wan2_1_i2v_14b_lora_2gpu": { - "clip_threshold": 0.90, - "ssim_threshold": 0.81, - "psnr_threshold": 24.0, - "mean_abs_diff_threshold": 10.0 + "clip_threshold": 0.97, + "ssim_threshold": 0.90, + "psnr_threshold": 30.0, + "mean_abs_diff_threshold": 5.3 }, "wan2_2_t2v_a14b_2gpu": { - "clip_threshold": 0.90, - "ssim_threshold": 0.72, - "psnr_threshold": 17.8, - "mean_abs_diff_threshold": 16.0 + "clip_threshold": 0.97, + "ssim_threshold": 0.90, + "psnr_threshold": 29.1, + "mean_abs_diff_threshold": 6.8 }, "wan2_2_t2v_a14b_lora_2gpu": { - "clip_threshold": 0.90, - "ssim_threshold": 0.81, + "clip_threshold": 0.98, + "ssim_threshold": 0.95, "psnr_threshold": 22.2, - "mean_abs_diff_threshold": 10.0 + "mean_abs_diff_threshold": 4.0 }, "wan2_1_i2v_14b_480P_2gpu": { - "clip_threshold": 0.76, - "ssim_threshold": 0.51, - "psnr_threshold": 14.8, - "mean_abs_diff_threshold": 23.0 + "clip_threshold": 0.92, + "ssim_threshold": 0.75, + "psnr_threshold": 21.4, + "mean_abs_diff_threshold": 9.7 }, "wan2_1_i2v_14b_720P_2gpu": { - "clip_threshold": 0.90, - "ssim_threshold": 0.89, - "psnr_threshold": 24.0, - "mean_abs_diff_threshold": 10.0 + "clip_threshold": 0.97, + "ssim_threshold": 0.90, + "psnr_threshold": 30.0, + "mean_abs_diff_threshold": 5.5 }, "ltx_2_3_hq_pipeline": { - "clip_threshold": 0.78, - "ssim_threshold": 0.48, - "psnr_threshold": 13.0, - "mean_abs_diff_threshold": 45.0 + "clip_threshold": 0.97, + "ssim_threshold": 0.88, + "psnr_threshold": 26.3, + "mean_abs_diff_threshold": 5.8 }, "ltx_2_3_two_stage_ti2v_2gpus": { "clip_threshold": 0.55,