[diffusion] CI: let the 5090 consumer case runs two warm requests on the full recipe (#36032)
Co-authored-by: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
@@ -1179,7 +1179,7 @@ def _make_5090_h3_consumer_budget_case() -> DiffusionTestCase:
|
|||||||
"--layerwise-offload-components",
|
"--layerwise-offload-components",
|
||||||
"dit,text_encoder,vae",
|
"dit,text_encoder,vae",
|
||||||
"--layerwise-resident-layers",
|
"--layerwise-resident-layers",
|
||||||
"video_vae=24",
|
"video_vae=36",
|
||||||
],
|
],
|
||||||
env_vars={
|
env_vars={
|
||||||
"SGLANG_DIFFUSION_TEST_FORCE_HOST_AVAILABLE_GIB": "32",
|
"SGLANG_DIFFUSION_TEST_FORCE_HOST_AVAILABLE_GIB": "32",
|
||||||
@@ -1212,6 +1212,7 @@ def _make_5090_h3_consumer_budget_case() -> DiffusionTestCase:
|
|||||||
},
|
},
|
||||||
),
|
),
|
||||||
run_perf_check=True,
|
run_perf_check=True,
|
||||||
|
perf_repeat_requests=2,
|
||||||
run_consistency_check=False,
|
run_consistency_check=False,
|
||||||
run_component_accuracy_check=False,
|
run_component_accuracy_check=False,
|
||||||
run_models_api_check=False,
|
run_models_api_check=False,
|
||||||
|
|||||||
@@ -138,7 +138,7 @@
|
|||||||
"stages_ms": {
|
"stages_ms": {
|
||||||
"MiniMaxH3TextEncodingStage": 11900.0,
|
"MiniMaxH3TextEncodingStage": 11900.0,
|
||||||
"MiniMaxH3DenoisingStage": 76100.0,
|
"MiniMaxH3DenoisingStage": 76100.0,
|
||||||
"MiniMaxH3DecodingStage": 26550.0
|
"MiniMaxH3DecodingStage": 13000.0
|
||||||
},
|
},
|
||||||
"denoise_step_ms": {
|
"denoise_step_ms": {
|
||||||
"0": 14000.0,
|
"0": 14000.0,
|
||||||
@@ -150,12 +150,12 @@
|
|||||||
"6": 14000.0,
|
"6": 14000.0,
|
||||||
"7": 14000.0
|
"7": 14000.0
|
||||||
},
|
},
|
||||||
"expected_e2e_ms": 125000.0,
|
"expected_e2e_ms": 105000.0,
|
||||||
"expected_avg_denoise_ms": 12000.0,
|
"expected_avg_denoise_ms": 12000.0,
|
||||||
"expected_median_denoise_ms": 11000.0,
|
"expected_median_denoise_ms": 11000.0,
|
||||||
"load_peak_vram_mb": 6000.0,
|
"load_peak_vram_mb": 6000.0,
|
||||||
"runtime_peak_vram_mb": 12288.0,
|
"runtime_peak_vram_mb": 12288.0,
|
||||||
"estimated_full_test_time_s": 900.0
|
"estimated_full_test_time_s": 1050.0
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1546,14 +1546,18 @@ Pinned revision used by this check: {SGL_TEST_FILES_CI_DATA_REVISION}
|
|||||||
sampling_params=case.sampling_params,
|
sampling_params=case.sampling_params,
|
||||||
)
|
)
|
||||||
|
|
||||||
# Single generation - output is reused for both validations
|
# Generation - output of the last request is used for both validations.
|
||||||
|
# perf_repeat_requests > 1 asserts a warm second request meets the same
|
||||||
|
# baselines as the first: residency or courier state leaking between
|
||||||
|
# requests shows up here as degradation or an OOM.
|
||||||
is_realtime_case = case.sampling_params.realtime_num_chunks is not None
|
is_realtime_case = case.sampling_params.realtime_num_chunks is not None
|
||||||
perf_record, content = self.run_and_collect(
|
for _ in range(max(1, case.perf_repeat_requests)):
|
||||||
diffusion_server,
|
perf_record, content = self.run_and_collect(
|
||||||
case.id,
|
diffusion_server,
|
||||||
generate_fn,
|
case.id,
|
||||||
collect_perf=not is_gt_gen_mode and not is_realtime_case,
|
generate_fn,
|
||||||
)
|
collect_perf=not is_gt_gen_mode and not is_realtime_case,
|
||||||
|
)
|
||||||
|
|
||||||
if is_gt_gen_mode:
|
if is_gt_gen_mode:
|
||||||
# GT generation mode: save output and skip all validations/tests
|
# GT generation mode: save output and skip all validations/tests
|
||||||
|
|||||||
@@ -295,6 +295,12 @@ class DiffusionTestCase:
|
|||||||
server_args: DiffusionServerArgs
|
server_args: DiffusionServerArgs
|
||||||
sampling_params: DiffusionSamplingParams | None = None
|
sampling_params: DiffusionSamplingParams | None = None
|
||||||
run_perf_check: bool = True
|
run_perf_check: bool = True
|
||||||
|
# Send the request this many times in one server session; performance and
|
||||||
|
# consistency are validated on the last one. >1 asserts a warm second
|
||||||
|
# request meets the same baselines -- a leak in residency arming, courier
|
||||||
|
# in-flight tracking, or host copies shows up as the second request
|
||||||
|
# degrading or dying.
|
||||||
|
perf_repeat_requests: int = 1
|
||||||
run_consistency_check: bool = True
|
run_consistency_check: bool = True
|
||||||
run_component_accuracy_check: bool = True
|
run_component_accuracy_check: bool = True
|
||||||
run_models_api_check: bool = True
|
run_models_api_check: bool = True
|
||||||
|
|||||||
Reference in New Issue
Block a user