From 88a9bfd1ffdec1d13ebe86c7de53c93fd19d351d Mon Sep 17 00:00:00 2001 From: Mick Date: Tue, 8 Sep 2026 18:40:17 +0800 Subject: [PATCH] [diffusion] CI: add per-case timing tolerance for host-I/O-bound perf guards (#38457) Co-authored-by: Mick Qian Co-authored-by: Claude Opus 5 --- .../test/server/perf_baselines/5090.json | 3 +- .../test/server/test_server_common.py | 7 +++ .../test/server/test_server_utils.py | 30 ++++++--- .../test/server/testcase_configs.py | 6 ++ .../test/unit/test_performance_metrics.py | 62 +++++++++++++++++++ 5 files changed, 98 insertions(+), 10 deletions(-) diff --git a/python/sglang/multimodal_gen/test/server/perf_baselines/5090.json b/python/sglang/multimodal_gen/test/server/perf_baselines/5090.json index cd00d3ca8..275c8c558 100644 --- a/python/sglang/multimodal_gen/test/server/perf_baselines/5090.json +++ b/python/sglang/multimodal_gen/test/server/perf_baselines/5090.json @@ -223,7 +223,8 @@ "runtime_peak_allocated_mb": 11483.0, "load_peak_host_anon_mb": 32768.0, "runtime_peak_host_anon_mb": 32768.0, - "estimated_full_test_time_s": 283.0 + "estimated_full_test_time_s": 283.0, + "timing_tolerance": 0.9 } } } diff --git a/python/sglang/multimodal_gen/test/server/test_server_common.py b/python/sglang/multimodal_gen/test/server/test_server_common.py index 729400741..504873318 100644 --- a/python/sglang/multimodal_gen/test/server/test_server_common.py +++ b/python/sglang/multimodal_gen/test/server/test_server_common.py @@ -700,6 +700,13 @@ class DiffusionServerBase: if measured_full_time is not None: baseline["estimated_full_test_time_s"] = round(measured_full_time, 1) + # a per-case timing tolerance is a deliberate property of the case, not a + # measurement: carry it into the suggested entry so a baseline refresh + # cannot silently drop it + existing = BASELINE_CONFIG.scenarios.get(case.id) + if existing is not None and existing.timing_tolerance is not None: + baseline["timing_tolerance"] = existing.timing_tolerance + # Video-specific metrics if case.server_args.modality == "video": if "per_frame_generation" not in baseline["stages_ms"]: diff --git a/python/sglang/multimodal_gen/test/server/test_server_utils.py b/python/sglang/multimodal_gen/test/server/test_server_utils.py index 086929e30..ae9ffe31a 100644 --- a/python/sglang/multimodal_gen/test/server/test_server_utils.py +++ b/python/sglang/multimodal_gen/test/server/test_server_utils.py @@ -726,6 +726,18 @@ class PerformanceValidator: ) -> PerformanceSummary: return PerformanceSummary.from_req_perf_record(perf_record, self.step_fractions) + def _timing_tol(self, profile_tolerance: float) -> float: + """Tolerance for a wall-clock check, honoring a per-case override. + + A case whose runtime is dominated by shared-runner host I/O cannot be + guarded at the profile tolerance; ``timing_tolerance`` in its baseline + entry widens only the wall-clock checks, never the memory ones. + """ + override = self.scenario.timing_tolerance + if override is None: + return profile_tolerance + return max(profile_tolerance, override) + def _validate_e2e(self, summary: PerformanceSummary) -> None: """Validate end-to-end performance.""" assert summary.e2e_ms > 0, "E2E duration missing" @@ -733,7 +745,7 @@ class PerformanceValidator: "E2E Latency", summary.e2e_ms, self.scenario.expected_e2e_ms, - self.tolerances.e2e, + self._timing_tol(self.tolerances.e2e), ) def _validate_denoise_agg(self, summary: PerformanceSummary) -> None: @@ -744,13 +756,13 @@ class PerformanceValidator: "Average Denoise Step", summary.avg_denoise_ms, self.scenario.expected_avg_denoise_ms, - self.tolerances.denoise_agg, + self._timing_tol(self.tolerances.denoise_agg), ) self._assert_le( "Median Denoise Step", summary.median_denoise_ms, self.scenario.expected_median_denoise_ms, - self.tolerances.denoise_agg, + self._timing_tol(self.tolerances.denoise_agg), ) def _validate_denoise_steps(self, summary: PerformanceSummary) -> None: @@ -766,7 +778,7 @@ class PerformanceValidator: f"Denoise Step {idx}", actual, expected, - FIRST_DENOISE_STEP_TOLERANCE, + self._timing_tol(FIRST_DENOISE_STEP_TOLERANCE), min_abs_tolerance=FIRST_DENOISE_STEP_MIN_ABS_TOLERANCE_MS, ) continue @@ -775,7 +787,7 @@ class PerformanceValidator: f"Denoise Step {idx}", actual, expected, - self.tolerances.denoise_step, + self._timing_tol(self.tolerances.denoise_step), ) def _validate_stages(self, summary: PerformanceSummary) -> None: @@ -787,7 +799,7 @@ class PerformanceValidator: continue actual = summary.stage_metrics.get(stage) assert actual is not None, f"Stage {stage} timing missing" - tolerance = ( + tolerance = self._timing_tol( self.tolerances.denoise_stage if stage == "DenoisingStage" else self.tolerances.non_denoise_stage @@ -822,7 +834,7 @@ class VideoPerformanceValidator(PerformanceValidator): f"Denoise Step {idx}", actual, expected, - FIRST_DENOISE_STEP_TOLERANCE, + self._timing_tol(FIRST_DENOISE_STEP_TOLERANCE), min_abs_tolerance=FIRST_DENOISE_STEP_MIN_ABS_TOLERANCE_MS, ) continue @@ -833,7 +845,7 @@ class VideoPerformanceValidator(PerformanceValidator): f"Denoise Step {idx}", actual, expected, - self.tolerances.denoise_step, + self._timing_tol(self.tolerances.denoise_step), min_abs_tolerance=VIDEO_DENOISE_STEP_MIN_ABS_TOLERANCE_MS, ) @@ -863,7 +875,7 @@ class VideoPerformanceValidator(PerformanceValidator): "Average Frame Time", summary.avg_frame_time_ms, expected_frame_time, - self.tolerances.denoise_stage, + self._timing_tol(self.tolerances.denoise_stage), ) diff --git a/python/sglang/multimodal_gen/test/server/testcase_configs.py b/python/sglang/multimodal_gen/test/server/testcase_configs.py index 19067291e..d35dcb0d4 100644 --- a/python/sglang/multimodal_gen/test/server/testcase_configs.py +++ b/python/sglang/multimodal_gen/test/server/testcase_configs.py @@ -127,6 +127,11 @@ class ScenarioConfig: # Anonymous-host budget caps; None skips the check (older baselines). load_peak_host_anon_mb: float | None = None runtime_peak_host_anon_mb: float | None = None + # Per-case override for the wall-clock tolerances (e2e, denoise and stage + # timings) when a case's runtime is dominated by shared-runner host I/O + # rather than by the code under test. Memory guards keep the profile + # tolerance -- they are what such a case actually protects. + timing_tolerance: float | None = None @classmethod def from_dict(cls, cfg: dict[str, Any]) -> ScenarioConfig: @@ -148,6 +153,7 @@ class ScenarioConfig: runtime_peak_allocated_mb=optional_float("runtime_peak_allocated_mb"), load_peak_host_anon_mb=optional_float("load_peak_host_anon_mb"), runtime_peak_host_anon_mb=optional_float("runtime_peak_host_anon_mb"), + timing_tolerance=optional_float("timing_tolerance"), ) diff --git a/python/sglang/multimodal_gen/test/unit/test_performance_metrics.py b/python/sglang/multimodal_gen/test/unit/test_performance_metrics.py index ebfcd509d..76b2f6410 100644 --- a/python/sglang/multimodal_gen/test/unit/test_performance_metrics.py +++ b/python/sglang/multimodal_gen/test/unit/test_performance_metrics.py @@ -572,3 +572,65 @@ def test_worker_keeps_the_pool_when_no_probe_ran(monkeypatch): worker._release_warmup_pool(_warmup_req()) worker._release_warmup_pool(SimpleNamespace(is_warmup=False, extra={})) assert calls == [] + + +def _timing_validator(timing_tolerance: float | None) -> PerformanceValidator: + return PerformanceValidator( + scenario=ScenarioConfig( + stages_ms={"DenoisingStage": 1000.0}, + denoise_step_ms={1: 100.0}, + expected_e2e_ms=1000.0, + expected_avg_denoise_ms=100.0, + expected_median_denoise_ms=100.0, + timing_tolerance=timing_tolerance, + ), + tolerances=ToleranceConfig(0.25, 0.25, 0.25, 0.3, 0.2), + step_fractions=(), + ) + + +def _validate_scaled(validator: PerformanceValidator, scale: float) -> None: + summary = PerformanceSummary( + 1000.0 * scale, + 100.0 * scale, + 100.0 * scale, + {"DenoisingStage": 1000.0 * scale}, + [], + {1: 100.0 * scale}, + {1: 100.0 * scale}, + ) + validator.collect_metrics = lambda _record: summary + validator.validate(None) + + +def test_timing_tolerance_override_widens_wall_clock_checks_only(): + with patch.object(current_platform, "is_hip", return_value=False): + # 1.7x the baseline fails at the profile's 25% e2e tolerance + with pytest.raises(AssertionError, match="E2E Latency"): + _validate_scaled(_timing_validator(None), 1.7) + + # the same run passes for a case that declares a 90% timing tolerance + _validate_scaled(_timing_validator(0.9), 1.7) + + # the override is a ceiling, not a blank cheque + with pytest.raises(AssertionError, match="E2E Latency"): + _validate_scaled(_timing_validator(0.9), 2.5) + + +def test_timing_tolerance_override_leaves_memory_guards_alone(): + validator = _timing_validator(0.9) + regression = PerformanceSummary( + 0.0, + 0.0, + 0.0, + {}, + [], + {}, + {}, + load_peak_vram_mb=10_000.0, + runtime_peak_vram_mb=10_201.0, + ) + + with patch.object(current_platform, "is_hip", return_value=False): + with pytest.raises(AssertionError, match="Runtime Peak VRAM"): + validator.validate_peak_vram(regression, 10_000.0, 10_000.0)