[diffusion] CI: let the 5090 consumer case runs two warm requests on the full recipe (#36032)

Co-authored-by: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
Mick
2026-08-23 15:39:32 +08:00
committed by GitHub
co-authored by Claude Fable 5
parent 155aa26c19
commit bd3cc97e7e
4 changed files with 22 additions and 11 deletions
@@ -1179,7 +1179,7 @@ def _make_5090_h3_consumer_budget_case() -> DiffusionTestCase:
"--layerwise-offload-components", "--layerwise-offload-components",
"dit,text_encoder,vae", "dit,text_encoder,vae",
"--layerwise-resident-layers", "--layerwise-resident-layers",
"video_vae=24", "video_vae=36",
], ],
env_vars={ env_vars={
"SGLANG_DIFFUSION_TEST_FORCE_HOST_AVAILABLE_GIB": "32", "SGLANG_DIFFUSION_TEST_FORCE_HOST_AVAILABLE_GIB": "32",
@@ -1212,6 +1212,7 @@ def _make_5090_h3_consumer_budget_case() -> DiffusionTestCase:
}, },
), ),
run_perf_check=True, run_perf_check=True,
perf_repeat_requests=2,
run_consistency_check=False, run_consistency_check=False,
run_component_accuracy_check=False, run_component_accuracy_check=False,
run_models_api_check=False, run_models_api_check=False,
@@ -138,7 +138,7 @@
"stages_ms": { "stages_ms": {
"MiniMaxH3TextEncodingStage": 11900.0, "MiniMaxH3TextEncodingStage": 11900.0,
"MiniMaxH3DenoisingStage": 76100.0, "MiniMaxH3DenoisingStage": 76100.0,
"MiniMaxH3DecodingStage": 26550.0 "MiniMaxH3DecodingStage": 13000.0
}, },
"denoise_step_ms": { "denoise_step_ms": {
"0": 14000.0, "0": 14000.0,
@@ -150,12 +150,12 @@
"6": 14000.0, "6": 14000.0,
"7": 14000.0 "7": 14000.0
}, },
"expected_e2e_ms": 125000.0, "expected_e2e_ms": 105000.0,
"expected_avg_denoise_ms": 12000.0, "expected_avg_denoise_ms": 12000.0,
"expected_median_denoise_ms": 11000.0, "expected_median_denoise_ms": 11000.0,
"load_peak_vram_mb": 6000.0, "load_peak_vram_mb": 6000.0,
"runtime_peak_vram_mb": 12288.0, "runtime_peak_vram_mb": 12288.0,
"estimated_full_test_time_s": 900.0 "estimated_full_test_time_s": 1050.0
} }
} }
} }
@@ -1546,14 +1546,18 @@ Pinned revision used by this check: {SGL_TEST_FILES_CI_DATA_REVISION}
sampling_params=case.sampling_params, sampling_params=case.sampling_params,
) )
# Single generation - output is reused for both validations # Generation - output of the last request is used for both validations.
# perf_repeat_requests > 1 asserts a warm second request meets the same
# baselines as the first: residency or courier state leaking between
# requests shows up here as degradation or an OOM.
is_realtime_case = case.sampling_params.realtime_num_chunks is not None is_realtime_case = case.sampling_params.realtime_num_chunks is not None
perf_record, content = self.run_and_collect( for _ in range(max(1, case.perf_repeat_requests)):
diffusion_server, perf_record, content = self.run_and_collect(
case.id, diffusion_server,
generate_fn, case.id,
collect_perf=not is_gt_gen_mode and not is_realtime_case, generate_fn,
) collect_perf=not is_gt_gen_mode and not is_realtime_case,
)
if is_gt_gen_mode: if is_gt_gen_mode:
# GT generation mode: save output and skip all validations/tests # GT generation mode: save output and skip all validations/tests
@@ -295,6 +295,12 @@ class DiffusionTestCase:
server_args: DiffusionServerArgs server_args: DiffusionServerArgs
sampling_params: DiffusionSamplingParams | None = None sampling_params: DiffusionSamplingParams | None = None
run_perf_check: bool = True run_perf_check: bool = True
# Send the request this many times in one server session; performance and
# consistency are validated on the last one. >1 asserts a warm second
# request meets the same baselines -- a leak in residency arming, courier
# in-flight tracking, or host copies shows up as the second request
# degrading or dying.
perf_repeat_requests: int = 1
run_consistency_check: bool = True run_consistency_check: bool = True
run_component_accuracy_check: bool = True run_component_accuracy_check: bool = True
run_models_api_check: bool = True run_models_api_check: bool = True