diff --git a/python/sglang/multimodal_gen/test/server/gpu_cases.py b/python/sglang/multimodal_gen/test/server/gpu_cases.py index 7b5576371c..0d4c162d1e 100644 --- a/python/sglang/multimodal_gen/test/server/gpu_cases.py +++ b/python/sglang/multimodal_gen/test/server/gpu_cases.py @@ -1663,43 +1663,40 @@ if current_platform.is_hip(): # New standalone files may omit an estimate once to learn the real CI runtime. # CI will use a fallback estimate for sharding, run the test, then print a # measured value that must be copied into STANDALONE_FILE_EST_TIMES. +# scheduling estimates use full-process H100 CI medians, rounded up to 10s STANDALONE_FILE_EST_TIMES = { "bcg-diffusion": { "../single_test_file/test_diffusion_bcg_zimage_turbo.py": 420.0, }, "1-gpu": { "../single_test_file/test_generate_zimage_turbo_cli.py": 180.0, - "../single_test_file/test_update_weights_from_disk.py": 1200.0, + "../single_test_file/test_update_weights_from_disk.py": 200.0, }, "2-gpu": { - # Two disagg clusters × (~3 min startup + ~1 min generate) ≈ 8 min. - # Raise if CI reports a higher measured time. - "../single_test_file/test_disagg_server.py": 600.0, - "../single_test_file/test_ar_models.py": 600.0, + "../single_test_file/test_disagg_server.py": 780.0, + "../single_test_file/test_ar_models.py": 120.0, # no model load; the cost is the one-time JIT build of the sync kernels - "../single_test_file/test_ipc_a2a_2_gpu.py": 240.0, + "../single_test_file/test_ipc_a2a_2_gpu.py": 30.0, # 2 ranks here (4 ranks skip below four GPUs); ~50 s on H200 with warm JIT - "../single_test_file/test_ipc_a2a_multi_gpu.py": 180.0, + "../single_test_file/test_ipc_a2a_multi_gpu.py": 30.0, # 2 ranks here (4 ranks skip below four GPUs); ~45 s on B300 with warm JIT - "../single_test_file/test_usp_pipelined_attention_multi_gpu.py": 120.0, - "../single_test_file/test_encoder_fold_srt_linear_2_gpu.py": 120.0, - "../single_test_file/test_encoder_fold_srt_2_gpu.py": 240.0, - # ~60 s locally with a warm HF cache (load + one capture + 4 steps); - # padded for cold-cache CI. - "../single_test_file/test_diffusion_bcg_tp2_zimage_turbo.py": 180.0, + "../single_test_file/test_usp_pipelined_attention_multi_gpu.py": 40.0, + "../single_test_file/test_encoder_fold_srt_linear_2_gpu.py": 30.0, + "../single_test_file/test_encoder_fold_srt_2_gpu.py": 100.0, + "../single_test_file/test_diffusion_bcg_tp2_zimage_turbo.py": 70.0, # zimage server startup dominates; six short requests after warmup - "../single_test_file/test_dp_serving_2_gpu.py": 900.0, + "../single_test_file/test_dp_serving_2_gpu.py": 60.0, # one capture plus three replays on a 32K-element exchange - "../single_test_file/test_pynccl_a2a_capture_2_gpu.py": 180.0, + "../single_test_file/test_pynccl_a2a_capture_2_gpu.py": 30.0, # two SDPA parity checks on 128+6 rows - "../single_test_file/test_usp_replicated_parity_2_gpu.py": 180.0, + "../single_test_file/test_usp_replicated_parity_2_gpu.py": 30.0, # no model load; two small all-to-alls - "../single_test_file/test_vdn_ulysses_exchange_2_gpu.py": 60.0, + "../single_test_file/test_vdn_ulysses_exchange_2_gpu.py": 30.0, # the unit/ suite runs on one GPU; these files carry its 2-GPU cases - "../unit/test_async_output_save.py": 30.0, - "../unit/test_kandinsky6_sequence_parallel.py": 150.0, - "../unit/test_kandinsky6_sr_sequence_parallel.py": 150.0, - "../unit/test_qwen_image21_distributed.py": 90.0, + "../unit/test_async_output_save.py": 20.0, + "../unit/test_kandinsky6_sequence_parallel.py": 90.0, + "../unit/test_kandinsky6_sr_sequence_parallel.py": 90.0, + "../unit/test_qwen_image21_distributed.py": 40.0, }, } diff --git a/python/sglang/multimodal_gen/test/server/perf_baselines/h100.json b/python/sglang/multimodal_gen/test/server/perf_baselines/h100.json index 27b78a9b66..24290986e5 100644 --- a/python/sglang/multimodal_gen/test/server/perf_baselines/h100.json +++ b/python/sglang/multimodal_gen/test/server/perf_baselines/h100.json @@ -221,7 +221,7 @@ "runtime_peak_vram_mb": 46938.0, "load_peak_allocated_mb": 41798.0, "runtime_peak_allocated_mb": 44855.0, - "estimated_full_test_time_s": 65.6 + "estimated_full_test_time_s": 70.0 }, "qwen_image21_t2i_tp2": { "expected_load_ms": 45596.77, @@ -231,7 +231,7 @@ "expected_e2e_ms": 2914.82, "expected_avg_denoise_ms": 0.0, "expected_median_denoise_ms": 0.0, - "estimated_full_test_time_s": 300.0 + "estimated_full_test_time_s": 60.0 }, "qwen_image_t2i_2_gpus_extra_high": { "expected_load_ms": 46120.38, @@ -304,7 +304,7 @@ "warmup_peak_vram_mb": 43424.0, "load_peak_allocated_mb": 41798.14, "runtime_peak_allocated_mb": 44854.98, - "estimated_full_test_time_s": 54.4 + "estimated_full_test_time_s": 60.0 }, "flux2_modelopt_fp8_tp2_t2i": { "expected_load_ms": 50693.24, @@ -313,7 +313,7 @@ "expected_e2e_ms": 2089.62, "expected_avg_denoise_ms": 0.0, "expected_median_denoise_ms": 0.0, - "estimated_full_test_time_s": 75.4 + "estimated_full_test_time_s": 60.0 }, "joy_echo_t2v_2gpu": { "expected_load_ms": 52313.14, @@ -322,7 +322,7 @@ "expected_e2e_ms": 2469.07, "expected_avg_denoise_ms": 0.0, "expected_median_denoise_ms": 0.0, - "estimated_full_test_time_s": 62.8 + "estimated_full_test_time_s": 80.0 }, "ideogram4_fp8_tp2_t2i": { "expected_load_ms": 33515.44, @@ -331,7 +331,7 @@ "expected_e2e_ms": 8975.57, "expected_avg_denoise_ms": 0.0, "expected_median_denoise_ms": 0.0, - "estimated_full_test_time_s": 60.8 + "estimated_full_test_time_s": 60.0 }, "ideogram4_fp8_t2i": { "stages_ms": { @@ -850,7 +850,7 @@ "runtime_peak_vram_mb": 28148.0, "load_peak_allocated_mb": 23262.0, "runtime_peak_allocated_mb": 25541.0, - "estimated_full_test_time_s": 44.9 + "estimated_full_test_time_s": 50.0 }, "zimage_image_t2i": { "expected_load_ms": 28221.57, @@ -970,7 +970,7 @@ "runtime_peak_vram_mb": 17602.0, "load_peak_allocated_mb": 12852.0, "runtime_peak_allocated_mb": 15131.0, - "estimated_full_test_time_s": 39.3 + "estimated_full_test_time_s": 50.0 }, "qwen_image_edit_ti2i": { "expected_load_ms": 39090.52, @@ -1460,7 +1460,7 @@ "runtime_peak_vram_mb": 11972.0, "load_peak_allocated_mb": 5274.0, "runtime_peak_allocated_mb": 9251.0, - "estimated_full_test_time_s": 45.5 + "estimated_full_test_time_s": 60.0 }, "turbo_wan2_1_t2v_1.3b": { "expected_load_ms": 27199.67, @@ -1558,7 +1558,7 @@ "runtime_peak_vram_mb": 57528.0, "load_peak_allocated_mb": 43411.0, "runtime_peak_allocated_mb": 54736.0, - "estimated_full_test_time_s": 100.4 + "estimated_full_test_time_s": 100.0 }, "wan2_2_ti2v_5b": { "expected_load_ms": 37605.69, @@ -1875,7 +1875,7 @@ "runtime_peak_vram_mb": 26322.0, "load_peak_allocated_mb": 6177.0, "runtime_peak_allocated_mb": 19978.0, - "estimated_full_test_time_s": 168.7 + "estimated_full_test_time_s": 220.0 }, "wan2_1_i2v_14b_480P_2gpu": { "expected_load_ms": 57487.86, @@ -1949,7 +1949,7 @@ "warmup_peak_vram_mb": 45782.0, "load_peak_allocated_mb": 34033.0, "runtime_peak_allocated_mb": 38569.0, - "estimated_full_test_time_s": 128.8 + "estimated_full_test_time_s": 130.0 }, "wan2_1_i2v_14b_720P_2gpu": { "expected_load_ms": 47568.4, @@ -2022,7 +2022,7 @@ "runtime_peak_vram_mb": 51230.0, "load_peak_allocated_mb": 34033.0, "runtime_peak_allocated_mb": 44644.0, - "estimated_full_test_time_s": 182.2 + "estimated_full_test_time_s": 180.0 }, "wan2_2_t2v_a14b_2gpu": { "expected_load_ms": 65355.85, @@ -2084,7 +2084,7 @@ "warmup_peak_vram_mb": 26902.0, "load_peak_allocated_mb": 6175.0, "runtime_peak_allocated_mb": 15472.0, - "estimated_full_test_time_s": 188.0 + "estimated_full_test_time_s": 340.0 }, "wan2_1_t2v_14b_2gpu": { "expected_load_ms": 49038.47, @@ -2155,7 +2155,7 @@ "runtime_peak_vram_mb": 37788.0, "load_peak_allocated_mb": 29869.0, "runtime_peak_allocated_mb": 33843.0, - "estimated_full_test_time_s": 96.3 + "estimated_full_test_time_s": 90.0 }, "wan2_2_t2v_a14b_lora_2gpu": { "expected_load_ms": 61440.19, @@ -2216,7 +2216,7 @@ "runtime_peak_vram_mb": 21450.0, "load_peak_allocated_mb": 6499.0, "runtime_peak_allocated_mb": 14775.0, - "estimated_full_test_time_s": 576.3 + "estimated_full_test_time_s": 420.0 }, "wan2_1_t2v_1_3b_lora_1gpu": { "expected_load_ms": 30139.26, @@ -2432,7 +2432,7 @@ "runtime_peak_vram_mb": 38262.0, "load_peak_allocated_mb": 32851.0, "runtime_peak_allocated_mb": 34824.0, - "estimated_full_test_time_s": 75.0 + "estimated_full_test_time_s": 90.0 }, "qwen_image_edit_2511_ti2i": { "expected_load_ms": 38727.15, @@ -2524,7 +2524,7 @@ "runtime_peak_vram_mb": 12634.0, "load_peak_allocated_mb": 7033.0, "runtime_peak_allocated_mb": 9311.0, - "estimated_full_test_time_s": 61.3 + "estimated_full_test_time_s": 40.0 }, "hunyuan3d_shape_gen": { "expected_load_ms": 40355.77, @@ -2935,7 +2935,7 @@ "runtime_peak_vram_mb": 49390.0, "load_peak_allocated_mb": 45460.0, "runtime_peak_allocated_mb": 47725.0, - "estimated_full_test_time_s": 167.9 + "estimated_full_test_time_s": 170.0 }, "ltx_2.3_two_stage_t2v_2gpus": { "expected_load_ms": 45206.77, @@ -2997,7 +2997,7 @@ "runtime_peak_vram_mb": 59544.0, "load_peak_allocated_mb": 46411.0, "runtime_peak_allocated_mb": 56951.0, - "estimated_full_test_time_s": 186.0 + "estimated_full_test_time_s": 200.0 }, "ltx_2_5_diffusion_decoder_2gpus": { "expected_load_ms": 28133.24, @@ -3031,7 +3031,7 @@ "warmup_peak_vram_mb": 28396, "load_peak_allocated_mb": 1876.14, "runtime_peak_allocated_mb": 48399, - "estimated_full_test_time_s": 545.1 + "estimated_full_test_time_s": 170.0 }, "ltx_2_3_two_stage_ti2v_2gpus": { "expected_load_ms": 50003.37, @@ -3093,7 +3093,7 @@ "runtime_peak_vram_mb": 60286.0, "load_peak_allocated_mb": 46411.0, "runtime_peak_allocated_mb": 56951.0, - "estimated_full_test_time_s": 101.4 + "estimated_full_test_time_s": 100.0 }, "longlive2_t2v": { "expected_load_ms": 29328.4, @@ -3346,7 +3346,7 @@ "expected_e2e_ms": 82478.23, "expected_avg_denoise_ms": 0.0, "expected_median_denoise_ms": 0.0, - "estimated_full_test_time_s": 186.4 + "estimated_full_test_time_s": 350.0 }, "wan2_1_t2v_1_3b_cache_dit_sp_only_2gpu": { "expected_load_ms": 34366.19, @@ -3355,7 +3355,7 @@ "expected_e2e_ms": 1077.99, "expected_avg_denoise_ms": 0.0, "expected_median_denoise_ms": 0.0, - "estimated_full_test_time_s": 52.2 + "estimated_full_test_time_s": 50.0 }, "minimax_h3_ref2va_video_audio_2gpu_h100": { "expected_load_ms": 68100.42, @@ -3384,7 +3384,7 @@ "expected_median_denoise_ms": 7802.67, "load_peak_vram_mb": 15242.0, "runtime_peak_vram_mb": 34560.0, - "estimated_full_test_time_s": 340.0 + "estimated_full_test_time_s": 250.0 }, "minimax_h3_t2va_2gpu_h100": { "expected_load_ms": 79373.57, @@ -3416,7 +3416,7 @@ "runtime_peak_vram_mb": 63312.0, "load_peak_allocated_mb": 3330.01, "runtime_peak_allocated_mb": 18182.98, - "estimated_full_test_time_s": 235.0 + "estimated_full_test_time_s": 150.0 }, "mova_360p_tp2": { "expected_load_ms": 60480.85, @@ -3425,7 +3425,7 @@ "expected_e2e_ms": 58242.09, "expected_avg_denoise_ms": 0.0, "expected_median_denoise_ms": 0.0, - "estimated_full_test_time_s": 141.9 + "estimated_full_test_time_s": 160.0 }, "zimage_image_t2i_2_gpus_non_square": { "expected_load_ms": 30675.16, @@ -3455,7 +3455,7 @@ "runtime_peak_vram_mb": 17604.0, "load_peak_allocated_mb": 12852.0, "runtime_peak_allocated_mb": 15130.0, - "estimated_full_test_time_s": 40.5 + "estimated_full_test_time_s": 50.0 }, "flux1_modelopt_fp8_t2i": { "expected_load_ms": 33073.38, @@ -3614,7 +3614,7 @@ "runtime_peak_vram_mb": 36322.0, "load_peak_allocated_mb": 1737.0, "runtime_peak_allocated_mb": 25745.0, - "estimated_full_test_time_s": 404.0, + "estimated_full_test_time_s": 190.0, "warmup_peak_vram_mb": 14598.0 }, "llada_image_bf16_t2i": { diff --git a/scripts/ci/utils/diffusion/compute_diffusion_partitions.py b/scripts/ci/utils/diffusion/compute_diffusion_partitions.py index d2ba598533..de8d52c195 100755 --- a/scripts/ci/utils/diffusion/compute_diffusion_partitions.py +++ b/scripts/ci/utils/diffusion/compute_diffusion_partitions.py @@ -90,9 +90,10 @@ def compute_partition_count( fallback_count = math.ceil(total_time_seconds / target_time_seconds) return max(1, min(fallback_count, max_partitions)) - preferred_count = math.ceil(total_time_seconds / target_time_seconds) - preferred_count = max(1, min(preferred_count, max_partitions)) - return max(min_partition_count, min(preferred_count, max_partition_count)) + return min( + range(min_partition_count, max_partition_count + 1), + key=lambda count: abs(total_time_seconds / count - target_time_seconds), + ) def build_partition_items( diff --git a/scripts/ci/utils/diffusion/test_compute_diffusion_partitions.py b/scripts/ci/utils/diffusion/test_compute_diffusion_partitions.py new file mode 100644 index 0000000000..98cf63c120 --- /dev/null +++ b/scripts/ci/utils/diffusion/test_compute_diffusion_partitions.py @@ -0,0 +1,10 @@ +import runpy +from pathlib import Path + + +def test_observed_suite_prefers_thirty_minutes_over_extra_short_shard(monkeypatch): + """The 154-minute CI suite should use five shards, rather than six at 26m.""" + script_dir = Path(__file__).resolve().parent + monkeypatch.syspath_prepend(str(script_dir)) + namespace = runpy.run_path(str(script_dir / "compute_diffusion_partitions.py")) + assert namespace["compute_partition_count"](9255.4, 1200, 1800, 2400, 10) == 5