diff --git a/python/sglang/multimodal_gen/test/server/gpu_cases.py b/python/sglang/multimodal_gen/test/server/gpu_cases.py index b8707da5be..717a3ee1f6 100644 --- a/python/sglang/multimodal_gen/test/server/gpu_cases.py +++ b/python/sglang/multimodal_gen/test/server/gpu_cases.py @@ -689,6 +689,7 @@ else: extras=["--transformer-path", MODELOPT_FLUX1_NVFP4_TRANSFORMER], env_vars=MODELOPT_NVFP4_B200_ENV_VARS, run_consistency_check=True, + run_perf_check=True, ), _make_modelopt_ci_case( "flux2_modelopt_nvfp4_t2i", @@ -698,6 +699,7 @@ else: extras=["--transformer-weights-path", MODELOPT_FLUX2_NVFP4_WEIGHTS], env_vars=MODELOPT_NVFP4_B200_ENV_VARS, run_consistency_check=True, + run_perf_check=True, ), _make_modelopt_ci_case( "ideogram4_nvfp4_t2i", @@ -707,6 +709,7 @@ else: extras=[], env_vars=MODELOPT_NVFP4_B200_ENV_VARS, run_consistency_check=True, + run_perf_check=True, ), _make_modelopt_ci_case( "qwen_image_2512_modelopt_nvfp4_t2i", @@ -716,6 +719,7 @@ else: extras=[], env_vars=MODELOPT_NVFP4_B200_ENV_VARS, run_consistency_check=True, + run_perf_check=True, ), _make_modelopt_ci_case( "wan22_modelopt_nvfp4_t2v", @@ -725,6 +729,7 @@ else: extras=[], env_vars=MODELOPT_WAN22_NVFP4_B200_ENV_VARS, run_consistency_check=True, + run_perf_check=True, ), ] diff --git a/python/sglang/multimodal_gen/test/server/perf_baselines/b200.json b/python/sglang/multimodal_gen/test/server/perf_baselines/b200.json index adbf1d0ad4..8b41fd60fb 100644 --- a/python/sglang/multimodal_gen/test/server/perf_baselines/b200.json +++ b/python/sglang/multimodal_gen/test/server/perf_baselines/b200.json @@ -2,8 +2,17 @@ "metadata": { "model": "Diffusion Server", "hardware": "CI B200 pool", - "description": "Reference estimates for B200-only diffusion cases, split out from the shared diffusion baseline file.", - "last_updated": "2026-07-01" + "description": "B200 diffusion references with stage, denoise and allocated-memory checks enabled; previously enforced E2E/load references and runner overrides retained.", + "last_updated": "2026-10-18", + "calibration_runs": [ + 38482436865, + 38554777367, + 38725436953, + 38979902269, + 39027561441, + 39077836460 + ], + "calibration_policy": "At least five comparable main CI runs per case. Newly enabled timing checks use the observed maximum plus 5%; memory references use the maximum plus 1%. Existing tolerances, E2E/load references and runner overrides remain unchanged." }, "runner_overrides": { "b200-cirrascale1-0123": { @@ -100,74 +109,82 @@ "flux1_modelopt_nvfp4_t2i": { "expected_load_ms": 38841.5, "stages_ms": { - "InputValidationStage": 0.04, - "TextEncodingStage": 16.55, - "LatentPreparationStage": 0.14, - "TimestepPreparationStage": 18.55, - "DenoisingStage": 623.33, - "DecodingStage": 18.67 + "InputValidationStage": 0.06, + "TextEncodingStage": 25.48, + "LatentPreparationStage": 0.23, + "TimestepPreparationStage": 2.09, + "DenoisingStage": 1097.5, + "DecodingStage": 15.42 }, "denoise_step_ms": { - "0": 61.5, + "0": 100.7, "1": 51.43, - "2": 51.19, + "2": 92.65, "3": 50.75, - "4": 51.09, + "4": 91.21, "5": 51.0, "6": 51.14, - "7": 50.65, + "7": 91.25, "8": 50.41, - "9": 50.45, + "9": 90.48, "10": 50.42, - "11": 50.9 + "11": 90.57 }, "expected_e2e_ms": 836.71, - "expected_avg_denoise_ms": 51.74, - "expected_median_denoise_ms": 50.95, - "estimated_full_test_time_s": 71.2 + "expected_avg_denoise_ms": 91.2, + "expected_median_denoise_ms": 90.58, + "estimated_full_test_time_s": 71.2, + "load_peak_vram_mb": 18297.16, + "runtime_peak_vram_mb": 23335.04, + "load_peak_allocated_mb": 18112.88, + "runtime_peak_allocated_mb": 19195.3 }, "flux2_modelopt_nvfp4_t2i": { "expected_load_ms": 71333.99, "stages_ms": { - "InputValidationStage": 0.04, - "TextEncodingStage": 330.43, + "InputValidationStage": 0.1, + "TextEncodingStage": 701.31, "ImageVAEEncodingStage": 0.01, - "LatentPreparationStage": 0.32, - "TimestepPreparationStage": 12.36, - "DenoisingStage": 1184.2, - "DecodingStage": 17.18 + "LatentPreparationStage": 0.82, + "TimestepPreparationStage": 4.35, + "DenoisingStage": 1501.8, + "DecodingStage": 15.72 }, "denoise_step_ms": { - "0": 96.2, + "0": 143.39, "1": 30.93, - "2": 66.88, + "2": 121.58, "3": 98.59, - "4": 97.53, + "4": 130.89, "5": 100.31, "6": 100.42, - "7": 98.55, + "7": 138.77, "8": 97.93, - "9": 98.11, + "9": 106.67, "10": 99.03, - "11": 99.53 + "11": 106.87 }, "expected_e2e_ms": 1990.84, - "expected_avg_denoise_ms": 90.33, - "expected_median_denoise_ms": 98.33, - "estimated_full_test_time_s": 592.3 + "expected_avg_denoise_ms": 115.55, + "expected_median_denoise_ms": 125.74, + "estimated_full_test_time_s": 592.3, + "load_peak_vram_mb": 26684.2, + "runtime_peak_vram_mb": 33020.94, + "load_peak_allocated_mb": 26206.26, + "runtime_peak_allocated_mb": 27679.14 }, "qwen_image_2512_modelopt_nvfp4_t2i": { "expected_load_ms": 46619.98, "stages_ms": { - "InputValidationStage": 0.04, - "TextEncodingStage": 221.4, - "LatentPreparationStage": 0.13, - "TimestepPreparationStage": 10.48, - "DenoisingStage": 9380.87, - "DecodingStage": 27.2 + "InputValidationStage": 0.07, + "TextEncodingStage": 247.17, + "LatentPreparationStage": 0.3, + "TimestepPreparationStage": 3.09, + "DenoisingStage": 18934.88, + "DecodingStage": 38.96 }, "denoise_step_ms": { - "0": 203.71, + "0": 398.42, "1": 190.75, "2": 186.34, "3": 186.88, @@ -177,7 +194,7 @@ "7": 186.61, "8": 185.09, "9": 190.7, - "10": 186.22, + "10": 370.1, "11": 185.68, "12": 190.85, "13": 186.54, @@ -187,7 +204,7 @@ "17": 186.04, "18": 185.14, "19": 185.39, - "20": 184.56, + "20": 342.95, "21": 188.41, "22": 184.77, "23": 185.5, @@ -196,7 +213,7 @@ "26": 185.62, "27": 191.78, "28": 186.4, - "29": 187.19, + "29": 356.39, "30": 185.61, "31": 188.86, "32": 185.2, @@ -206,7 +223,7 @@ "36": 186.95, "37": 189.05, "38": 187.4, - "39": 188.8, + "39": 333.99, "40": 193.13, "41": 187.5, "42": 184.56, @@ -216,53 +233,61 @@ "46": 188.3, "47": 191.32, "48": 186.01, - "49": 185.95 + "49": 595.74 }, "expected_e2e_ms": 9650.06, - "expected_avg_denoise_ms": 187.56, - "expected_median_denoise_ms": 186.57, - "estimated_full_test_time_s": 120.0 + "expected_avg_denoise_ms": 378.62, + "expected_median_denoise_ms": 345.81, + "estimated_full_test_time_s": 120.0, + "load_peak_vram_mb": 17806.3, + "runtime_peak_vram_mb": 21062.54, + "load_peak_allocated_mb": 17269.02, + "runtime_peak_allocated_mb": 18870.33 }, "wan22_modelopt_nvfp4_t2v": { "expected_load_ms": 79653.94, "stages_ms": { "InputValidationStage": 0.04, - "TextEncodingStage": 316.23, + "TextEncodingStage": 346.51, "LatentPreparationStage": 0.11, - "TimestepPreparationStage": 1.44, - "DenoisingStage": 5413.33, - "DecodingStage": 82.95 + "TimestepPreparationStage": 2.35, + "DenoisingStage": 5788.41, + "DecodingStage": 256.27 }, "denoise_step_ms": { - "0": 306.5, + "0": 375.77, "1": 410.26, - "2": 409.53, + "2": 446.02, "3": 407.43, - "4": 406.74, + "4": 443.38, "5": 406.78, "6": 406.75, - "7": 406.78, + "7": 443.45, "8": 899.72, - "9": 406.75, + "9": 562.42, "10": 406.77, - "11": 404.41 + "11": 591.1 }, "expected_e2e_ms": 5964.13, - "expected_avg_denoise_ms": 439.87, - "expected_median_denoise_ms": 406.77, - "estimated_full_test_time_s": 181.8 + "expected_avg_denoise_ms": 478.63, + "expected_median_denoise_ms": 443.42, + "estimated_full_test_time_s": 181.8, + "load_peak_vram_mb": 4920.72, + "runtime_peak_vram_mb": 9895.98, + "load_peak_allocated_mb": 4858.5, + "runtime_peak_allocated_mb": 6460.38 }, "ideogram4_nvfp4_t2i": { "expected_load_ms": 33439.28, "stages_ms": { "InputValidationStage": 0.04, - "Ideogram4TextEncodingStage": 129.35, - "LatentPreparationStage": 0.1, - "DenoisingStage": 5233.76, - "Ideogram4DecodingStage": 53.08 + "Ideogram4TextEncodingStage": 214.77, + "LatentPreparationStage": 0.13, + "DenoisingStage": 5818.9, + "Ideogram4DecodingStage": 56.33 }, "denoise_step_ms": { - "0": 88.71, + "0": 124.05, "1": 83.99, "2": 109.67, "3": 110.03, @@ -271,7 +296,7 @@ "6": 109.73, "7": 109.89, "8": 111.44, - "9": 108.37, + "9": 117.9, "10": 110.23, "11": 110.11, "12": 109.81, @@ -281,7 +306,7 @@ "16": 109.8, "17": 109.88, "18": 109.82, - "19": 110.23, + "19": 115.63, "20": 109.81, "21": 109.99, "22": 110.21, @@ -290,7 +315,7 @@ "25": 110.39, "26": 109.8, "27": 109.92, - "28": 109.83, + "28": 119.92, "29": 109.88, "30": 109.83, "31": 109.89, @@ -300,7 +325,7 @@ "35": 110.02, "36": 109.67, "37": 110.06, - "38": 109.8, + "38": 116.34, "39": 110.35, "40": 110.19, "41": 110.43, @@ -309,12 +334,16 @@ "44": 109.8, "45": 109.94, "46": 109.81, - "47": 110.02 + "47": 259.5 }, "expected_e2e_ms": 5421.76, - "expected_avg_denoise_ms": 108.99, - "expected_median_denoise_ms": 109.9, - "estimated_full_test_time_s": 300.0 + "expected_avg_denoise_ms": 121.13, + "expected_median_denoise_ms": 116.68, + "estimated_full_test_time_s": 300.0, + "load_peak_vram_mb": 13691.56, + "runtime_peak_vram_mb": 34903.58, + "load_peak_allocated_mb": 13661.57, + "runtime_peak_allocated_mb": 31120.37 } } } diff --git a/python/sglang/multimodal_gen/test/server/perf_baselines/h100.json b/python/sglang/multimodal_gen/test/server/perf_baselines/h100.json index aa88bf4e59..872b0b70c8 100644 --- a/python/sglang/multimodal_gen/test/server/perf_baselines/h100.json +++ b/python/sglang/multimodal_gen/test/server/perf_baselines/h100.json @@ -2,8 +2,17 @@ "metadata": { "model": "Diffusion Server", "hardware": "CI H100 80GB pool", - "description": "Reference numbers tightened from NVIDIA multimodal_gen CI at source revision 2ee0d38a8553d186faad9c50893480e3c1d79270.", - "last_updated": "2026-08-16" + "description": "Reference numbers from source revision 2ee0d38a8553d186faad9c50893480e3c1d79270, with selected stable metrics tightened using calibration_runs.", + "last_updated": "2026-10-18", + "calibration_runs": [ + 38554777367, + 38725436953, + 38821753958, + 38979902269, + 39027561441, + 39077836460 + ], + "calibration_policy": "At least five comparable main CI runs per metric; timing references are the observed maximum plus 5%, allocated-memory references plus 1%. Existing tolerances remain unchanged." }, "tolerances": { "long_term": { @@ -544,12 +553,12 @@ "denoise_step_ms": { "0": 25.99, "1": 12.61, - "2": 22.29, - "3": 57.75 + "2": 12.16, + "3": 39.07 }, "expected_e2e_ms": 399.62, - "expected_avg_denoise_ms": 30.0, - "expected_median_denoise_ms": 24.14, + "expected_avg_denoise_ms": 21.49, + "expected_median_denoise_ms": 19.63, "load_peak_vram_mb": 8504.0, "runtime_peak_vram_mb": 13732.0, "load_peak_allocated_mb": 8498.0, @@ -1317,7 +1326,7 @@ "DecodingStage": 361.6, "InputValidationStage": 0.03, "DenoisingStage": 7195.52, - "TextEncodingStage": 534.89, + "TextEncodingStage": 348.4, "LatentPreparationStage": 0.09, "TimestepPreparationStage": 2.07 }, @@ -2593,7 +2602,7 @@ "expected_load_ms": 30176.3, "stages_ms": { "InputValidationStage": 0.05, - "TextEncodingStage": 534.56, + "TextEncodingStage": 360.21, "LatentPreparationStage": 0.2, "TimestepPreparationStage": 2.72, "DenoisingStage": 7336.03, @@ -2738,7 +2747,7 @@ "DecodingStage": 362.62, "InputValidationStage": 0.04, "DenoisingStage": 7238.94, - "TextEncodingStage": 534.42, + "TextEncodingStage": 347.67, "LatentPreparationStage": 0.14, "TimestepPreparationStage": 2.35 }, @@ -2884,7 +2893,7 @@ "TimestepPreparationStage": 8.67, "LTX2AVLatentPreparationStage": 0.14, "LTX2ImageEncodingStage": 33.09, - "LTX2AVDenoisingStage": 16865.06, + "LTX2AVDenoisingStage": 9172.61, "LTX2AVDecodingStage": 332.72 }, "denoise_step_ms": { @@ -2894,34 +2903,34 @@ "3": 562.98, "4": 557.91, "5": 558.1, - "6": 560.08, + "6": 312.35, "7": 558.2, "8": 560.66, "9": 557.22, "10": 556.51, "11": 560.84, - "12": 558.08, + "12": 299.59, "13": 553.87, "14": 555.03, "15": 562.19, "16": 561.89, - "17": 560.13, + "17": 293.64, "18": 553.78, "19": 562.13, "20": 558.77, "21": 556.56, "22": 558.79, - "23": 556.63, + "23": 304.85, "24": 558.15, "25": 557.89, "26": 555.4, "27": 564.63, "28": 559.4, - "29": 554.83 + "29": 299.3 }, - "expected_e2e_ms": 18216.43, - "expected_avg_denoise_ms": 562.02, - "expected_median_denoise_ms": 556.39, + "expected_e2e_ms": 10160.35, + "expected_avg_denoise_ms": 305.63, + "expected_median_denoise_ms": 299.37, "load_peak_vram_mb": 46278.0, "runtime_peak_vram_mb": 49390.0, "load_peak_allocated_mb": 45460.0, @@ -3111,16 +3120,16 @@ "expected_load_ms": 32252.28, "stages_ms": { "InputValidationStage": 23.02, - "LongLive2TextEncodingStage": 327.98, + "LongLive2TextEncodingStage": 135.69, "LongLive2ImageVAEEncodingStage": 920.37, "LongLive2LatentPreparationStage": 0.12, - "LongLive2CausalDenoisingStage": 4326.16, + "LongLive2CausalDenoisingStage": 1457.5, "DecodingStage": 2548.02, "per_frame_generation": null }, "denoise_step_ms": {}, - "expected_e2e_ms": 9086.81, - "expected_avg_denoise_ms": 492.02, + "expected_e2e_ms": 5912.97, + "expected_avg_denoise_ms": 128.48, "expected_median_denoise_ms": 151.9, "load_peak_vram_mb": 34022.0, "runtime_peak_vram_mb": 62510.0, @@ -3294,7 +3303,7 @@ "33": 22.65, "34": 22.37 }, - "expected_e2e_ms": 1019.02, + "expected_e2e_ms": 812.05, "expected_avg_denoise_ms": 24.13, "expected_median_denoise_ms": 22.52, "estimated_full_test_time_s": 65.0 @@ -3316,7 +3325,7 @@ "2": 85.06, "3": 58.08 }, - "expected_e2e_ms": 499.7, + "expected_e2e_ms": 369.27, "expected_avg_denoise_ms": 73.47, "expected_median_denoise_ms": 73.29, "estimated_full_test_time_s": 65.0 @@ -3359,7 +3368,7 @@ "MiniMaxH3LatentPreparationStage": 24.37, "MiniMaxH3TimestepPreparationStage": 0.28, "MiniMaxH3DenoisingStage": 55168.59, - "MiniMaxH3DecodingStage": 3183.0 + "MiniMaxH3DecodingStage": 2758.73 }, "denoise_step_ms": { "0": 1064.67, @@ -3405,8 +3414,8 @@ "expected_median_denoise_ms": 2332.99, "load_peak_vram_mb": 45428.0, "runtime_peak_vram_mb": 63312.0, - "load_peak_allocated_mb": 15139.0, - "runtime_peak_allocated_mb": 22887.0, + "load_peak_allocated_mb": 3330.01, + "runtime_peak_allocated_mb": 18182.98, "estimated_full_test_time_s": 235.0 }, "mova_360p_tp2": { @@ -3528,23 +3537,23 @@ "wan2_2_animate_2_14b_ref2v": { "stages_ms": { "WanAnimate2BeforeDenoisingStage": 3783.78, - "WanAnimate2DenoisingStage": 94604.96, + "WanAnimate2DenoisingStage": 46049.96, "WanAnimate2OutputStage": 0.06 }, "denoise_step_ms": { "0": 25832.83, - "1": 7210.15, + "1": 4126.81, "2": 11187.28, - "3": 8460.7, - "4": 8407.93, + "3": 5101.22, + "4": 4998.42, "5": 8409.07, - "6": 8408.07, - "7": 8415.78 + "6": 5000.29, + "7": 4999.26 }, - "expected_e2e_ms": 98402.77, - "expected_load_ms": 62901.15, - "expected_avg_denoise_ms": 10791.48, - "expected_median_denoise_ms": 8412.42, + "expected_e2e_ms": 48412.15, + "expected_load_ms": 43351.75, + "expected_avg_denoise_ms": 5004.67, + "expected_median_denoise_ms": 4999.09, "load_peak_vram_mb": 1776.0, "runtime_peak_vram_mb": 36322.0, "load_peak_allocated_mb": 1737.0, diff --git a/python/sglang/multimodal_gen/test/server/testcase_configs.py b/python/sglang/multimodal_gen/test/server/testcase_configs.py index 3cd3c6829d..cc6ecbf0c7 100644 --- a/python/sglang/multimodal_gen/test/server/testcase_configs.py +++ b/python/sglang/multimodal_gen/test/server/testcase_configs.py @@ -1124,6 +1124,7 @@ def _make_modelopt_ci_case( extras: list[str], env_vars: dict[str, str] | None = None, run_consistency_check: bool = False, + run_perf_check: bool = False, ) -> DiffusionTestCase: return DiffusionTestCase( case_id, @@ -1134,7 +1135,7 @@ def _make_modelopt_ci_case( env_vars=env_vars or {}, ), sampling_params, - run_perf_check=False, + run_perf_check=run_perf_check, run_consistency_check=run_consistency_check, run_component_accuracy_check=False, )