From dd2ffdb68058c96dec2b7d0fb707b50efc4f103e Mon Sep 17 00:00:00 2001 From: Mick Date: Mon, 5 Oct 2026 15:19:21 +0800 Subject: [PATCH] [diffusion] nightly: keep nightly regression baselines on a comparable methodology (#42226) Co-authored-by: Mick Qian --- .../unit/test_diffusion_nightly_comparison.py | 91 ++++++++++++++++++- .../diffusion/generate_diffusion_dashboard.py | 14 +++ 2 files changed, 104 insertions(+), 1 deletion(-) diff --git a/python/sglang/multimodal_gen/test/unit/test_diffusion_nightly_comparison.py b/python/sglang/multimodal_gen/test/unit/test_diffusion_nightly_comparison.py index 706f77c56a..5b351e536e 100644 --- a/python/sglang/multimodal_gen/test/unit/test_diffusion_nightly_comparison.py +++ b/python/sglang/multimodal_gen/test/unit/test_diffusion_nightly_comparison.py @@ -2,6 +2,7 @@ import importlib.util import io from pathlib import Path +import pytest from PIL import Image REPO_ROOT = Path(__file__).resolve().parents[5] @@ -224,13 +225,15 @@ def test_dashboard_uses_historical_median_and_shows_server_breakdown(): } history = [ { + "methodology": "client-e2e-v1", + "warmup_requests": 1, "results": [ { "case_id": "example", "framework": "sglang", "latency_s": value, } - ] + ], } for value in (10.0, 30.0, 9.8) ] @@ -249,3 +252,89 @@ def test_dashboard_uses_historical_median_and_shows_server_breakdown(): assert "Methodology `client-e2e-v1`" in markdown assert "## SGLang Server-Side Breakdown" in markdown assert "| model | 10.00 | 0.10 | 9.80 | 0.10 | 196.00 |" in markdown + + +@pytest.mark.parametrize( + "current_metadata,history_metadata,comparable", + [ + ({}, {}, True), + ({"methodology": "client-e2e-v1", "warmup_requests": 1}, {}, False), + ({}, {"methodology": "client-e2e-v1", "warmup_requests": 1}, False), + ( + {"methodology": "client-e2e-v1", "warmup_requests": 1}, + {"methodology": "client-e2e-v2", "warmup_requests": 1}, + False, + ), + ( + {"methodology": "client-e2e-v1", "warmup_requests": 1}, + {"methodology": "client-e2e-v1", "warmup_requests": 0}, + False, + ), + ( + {"methodology": "client-e2e-v1", "warmup_requests": 1}, + {"methodology": "client-e2e-v1", "warmup_requests": 1}, + True, + ), + ], +) +def test_dashboard_compares_only_matching_measurements( + current_metadata, history_metadata, comparable, tmp_path +): + current = { + **current_metadata, + "results": [ + { + "case_id": "example", + "framework": "sglang", + "model": "example/model", + "latency_s": 12.0, + } + ], + } + history = [ + { + **history_metadata, + "results": [ + {"case_id": "example", "framework": "sglang", "latency_s": 10.0} + ], + } + ] + charts_dir = tmp_path / "charts" + + markdown, alerts = dashboard.generate_dashboard( + current, history, str(charts_dir) if not comparable else None + ) + + assert bool(alerts) == comparable + assert ("Performance Regression Detected" in markdown) == comparable + assert ("Excluded 1 historical run(s)" in markdown) != comparable + assert not charts_dir.exists() + + +def test_dashboard_filters_before_limiting_baseline_window(): + metadata = {"methodology": "client-e2e-v1", "warmup_requests": 1} + current = { + **metadata, + "results": [ + { + "case_id": "example", + "framework": "sglang", + "model": "example/model", + "latency_s": 12.0, + } + ], + } + incompatible = { + "results": [{"case_id": "example", "framework": "sglang", "latency_s": 30.0}] + } + comparable = { + **metadata, + "results": [{"case_id": "example", "framework": "sglang", "latency_s": 10.0}], + } + history = [incompatible] * dashboard.HISTORICAL_BASELINE_RUNS + [comparable] + + markdown, alerts = dashboard.generate_dashboard(current, history) + + assert len(alerts) == 1 + assert "+20.0% vs 1-run median" in alerts[0] + assert "1-run median 10.00s" in markdown diff --git a/scripts/ci/utils/diffusion/generate_diffusion_dashboard.py b/scripts/ci/utils/diffusion/generate_diffusion_dashboard.py index 8fab7d5b20..c3cad4844e 100644 --- a/scripts/ci/utils/diffusion/generate_diffusion_dashboard.py +++ b/scripts/ci/utils/diffusion/generate_diffusion_dashboard.py @@ -296,6 +296,13 @@ def generate_dashboard( Returns the markdown string. """ + history_count = len(history) + history = [ + run + for run in history + if run.get("methodology") == current.get("methodology") + and run.get("warmup_requests") == current.get("warmup_requests") + ] lines: list[str] = [] lines.append("# SGLang-Diffusion Nightly Performance Dashboard\n") ts = current.get("timestamp", datetime.now(timezone.utc).isoformat()) @@ -311,6 +318,13 @@ def generate_dashboard( "First request = the first request after the server reports ready. " "Server perf dumps are telemetry only.*\n" ) + excluded_runs = history_count - len(history) + if excluded_runs: + lines.append( + f"*Excluded {excluded_runs} historical run(s) from baselines and trends " + "because their measurement methodology or warmup count differs. " + "Missing metadata is not treated as matching explicit metadata.*\n" + ) current_cases = _extract_case_results(current) current_records = _extract_case_records(current)