[diffusion] CI: add per-case timing tolerance for host-I/O-bound perf guards (#38457)

Co-authored-by: Mick Qian <mickqian@radixark.ai>
Co-authored-by: Claude Opus 5 <noreply@anthropic.com>

python/sglang/multimodal_gen/test/server/perf_baselines/5090.json

@@ -223,7 +223,8 @@
             "runtime_peak_allocated_mb": 11483.0,
             "load_peak_host_anon_mb": 32768.0,
             "runtime_peak_host_anon_mb": 32768.0,
-            "estimated_full_test_time_s": 283.0
+            "estimated_full_test_time_s": 283.0,
+            "timing_tolerance": 0.9
         }
     }
 }

python/sglang/multimodal_gen/test/server/test_server_common.py

@@ -700,6 +700,13 @@ class DiffusionServerBase:
         if measured_full_time is not None:
             baseline["estimated_full_test_time_s"] = round(measured_full_time, 1)
 
+        # a per-case timing tolerance is a deliberate property of the case, not a
+        # measurement: carry it into the suggested entry so a baseline refresh
+        # cannot silently drop it
+        existing = BASELINE_CONFIG.scenarios.get(case.id)
+        if existing is not None and existing.timing_tolerance is not None:
+            baseline["timing_tolerance"] = existing.timing_tolerance
+
         # Video-specific metrics
         if case.server_args.modality == "video":
             if "per_frame_generation" not in baseline["stages_ms"]:

python/sglang/multimodal_gen/test/server/test_server_utils.py

@@ -726,6 +726,18 @@ class PerformanceValidator:
     ) -> PerformanceSummary:
         return PerformanceSummary.from_req_perf_record(perf_record, self.step_fractions)
 
+    def _timing_tol(self, profile_tolerance: float) -> float:
+        """Tolerance for a wall-clock check, honoring a per-case override.
+
+        A case whose runtime is dominated by shared-runner host I/O cannot be
+        guarded at the profile tolerance; ``timing_tolerance`` in its baseline
+        entry widens only the wall-clock checks, never the memory ones.
+        """
+        override = self.scenario.timing_tolerance
+        if override is None:
+            return profile_tolerance
+        return max(profile_tolerance, override)
+
     def _validate_e2e(self, summary: PerformanceSummary) -> None:
         """Validate end-to-end performance."""
         assert summary.e2e_ms > 0, "E2E duration missing"
@@ -733,7 +745,7 @@ class PerformanceValidator:
             "E2E Latency",
             summary.e2e_ms,
             self.scenario.expected_e2e_ms,
-            self.tolerances.e2e,
+            self._timing_tol(self.tolerances.e2e),
         )
 
     def _validate_denoise_agg(self, summary: PerformanceSummary) -> None:
@@ -744,13 +756,13 @@ class PerformanceValidator:
             "Average Denoise Step",
             summary.avg_denoise_ms,
             self.scenario.expected_avg_denoise_ms,
-            self.tolerances.denoise_agg,
+            self._timing_tol(self.tolerances.denoise_agg),
         )
         self._assert_le(
             "Median Denoise Step",
             summary.median_denoise_ms,
             self.scenario.expected_median_denoise_ms,
-            self.tolerances.denoise_agg,
+            self._timing_tol(self.tolerances.denoise_agg),
         )
 
     def _validate_denoise_steps(self, summary: PerformanceSummary) -> None:
@@ -766,7 +778,7 @@ class PerformanceValidator:
                     f"Denoise Step {idx}",
                     actual,
                     expected,
-                    FIRST_DENOISE_STEP_TOLERANCE,
+                    self._timing_tol(FIRST_DENOISE_STEP_TOLERANCE),
                     min_abs_tolerance=FIRST_DENOISE_STEP_MIN_ABS_TOLERANCE_MS,
                 )
                 continue
@@ -775,7 +787,7 @@ class PerformanceValidator:
                 f"Denoise Step {idx}",
                 actual,
                 expected,
-                self.tolerances.denoise_step,
+                self._timing_tol(self.tolerances.denoise_step),
             )
 
     def _validate_stages(self, summary: PerformanceSummary) -> None:
@@ -787,7 +799,7 @@ class PerformanceValidator:
                 continue
             actual = summary.stage_metrics.get(stage)
             assert actual is not None, f"Stage {stage} timing missing"
-            tolerance = (
+            tolerance = self._timing_tol(
                 self.tolerances.denoise_stage
                 if stage == "DenoisingStage"
                 else self.tolerances.non_denoise_stage
@@ -822,7 +834,7 @@ class VideoPerformanceValidator(PerformanceValidator):
                     f"Denoise Step {idx}",
                     actual,
                     expected,
-                    FIRST_DENOISE_STEP_TOLERANCE,
+                    self._timing_tol(FIRST_DENOISE_STEP_TOLERANCE),
                     min_abs_tolerance=FIRST_DENOISE_STEP_MIN_ABS_TOLERANCE_MS,
                 )
                 continue
@@ -833,7 +845,7 @@ class VideoPerformanceValidator(PerformanceValidator):
                 f"Denoise Step {idx}",
                 actual,
                 expected,
-                self.tolerances.denoise_step,
+                self._timing_tol(self.tolerances.denoise_step),
                 min_abs_tolerance=VIDEO_DENOISE_STEP_MIN_ABS_TOLERANCE_MS,
             )
 
@@ -863,7 +875,7 @@ class VideoPerformanceValidator(PerformanceValidator):
                 "Average Frame Time",
                 summary.avg_frame_time_ms,
                 expected_frame_time,
-                self.tolerances.denoise_stage,
+                self._timing_tol(self.tolerances.denoise_stage),
             )
 
 

python/sglang/multimodal_gen/test/server/testcase_configs.py

@@ -127,6 +127,11 @@ class ScenarioConfig:
     # Anonymous-host budget caps; None skips the check (older baselines).
     load_peak_host_anon_mb: float | None = None
     runtime_peak_host_anon_mb: float | None = None
+    # Per-case override for the wall-clock tolerances (e2e, denoise and stage
+    # timings) when a case's runtime is dominated by shared-runner host I/O
+    # rather than by the code under test. Memory guards keep the profile
+    # tolerance -- they are what such a case actually protects.
+    timing_tolerance: float | None = None
 
     @classmethod
     def from_dict(cls, cfg: dict[str, Any]) -> ScenarioConfig:
@@ -148,6 +153,7 @@ class ScenarioConfig:
             runtime_peak_allocated_mb=optional_float("runtime_peak_allocated_mb"),
             load_peak_host_anon_mb=optional_float("load_peak_host_anon_mb"),
             runtime_peak_host_anon_mb=optional_float("runtime_peak_host_anon_mb"),
+            timing_tolerance=optional_float("timing_tolerance"),
         )
 
 

python/sglang/multimodal_gen/test/unit/test_performance_metrics.py

@@ -572,3 +572,65 @@ def test_worker_keeps_the_pool_when_no_probe_ran(monkeypatch):
     worker._release_warmup_pool(_warmup_req())
     worker._release_warmup_pool(SimpleNamespace(is_warmup=False, extra={}))
     assert calls == []
+
+
+def _timing_validator(timing_tolerance: float | None) -> PerformanceValidator:
+    return PerformanceValidator(
+        scenario=ScenarioConfig(
+            stages_ms={"DenoisingStage": 1000.0},
+            denoise_step_ms={1: 100.0},
+            expected_e2e_ms=1000.0,
+            expected_avg_denoise_ms=100.0,
+            expected_median_denoise_ms=100.0,
+            timing_tolerance=timing_tolerance,
+        ),
+        tolerances=ToleranceConfig(0.25, 0.25, 0.25, 0.3, 0.2),
+        step_fractions=(),
+    )
+
+
+def _validate_scaled(validator: PerformanceValidator, scale: float) -> None:
+    summary = PerformanceSummary(
+        1000.0 * scale,
+        100.0 * scale,
+        100.0 * scale,
+        {"DenoisingStage": 1000.0 * scale},
+        [],
+        {1: 100.0 * scale},
+        {1: 100.0 * scale},
+    )
+    validator.collect_metrics = lambda _record: summary
+    validator.validate(None)
+
+
+def test_timing_tolerance_override_widens_wall_clock_checks_only():
+    with patch.object(current_platform, "is_hip", return_value=False):
+        # 1.7x the baseline fails at the profile's 25% e2e tolerance
+        with pytest.raises(AssertionError, match="E2E Latency"):
+            _validate_scaled(_timing_validator(None), 1.7)
+
+        # the same run passes for a case that declares a 90% timing tolerance
+        _validate_scaled(_timing_validator(0.9), 1.7)
+
+        # the override is a ceiling, not a blank cheque
+        with pytest.raises(AssertionError, match="E2E Latency"):
+            _validate_scaled(_timing_validator(0.9), 2.5)
+
+
+def test_timing_tolerance_override_leaves_memory_guards_alone():
+    validator = _timing_validator(0.9)
+    regression = PerformanceSummary(
+        0.0,
+        0.0,
+        0.0,
+        {},
+        [],
+        {},
+        {},
+        load_peak_vram_mb=10_000.0,
+        runtime_peak_vram_mb=10_201.0,
+    )
+
+    with patch.object(current_platform, "is_hip", return_value=False):
+        with pytest.raises(AssertionError, match="Runtime Peak VRAM"):
+            validator.validate_peak_vram(regression, 10_000.0, 10_000.0)