From e016de462c7288d7d7659117948e51a43780570c Mon Sep 17 00:00:00 2001 From: Mick Date: Fri, 11 Sep 2026 23:07:37 +0800 Subject: [PATCH] [diffusion] CI: expose nightly server telemetry coverage (#38782) Co-authored-by: Mick Qian --- docs/docs/sglang-diffusion/ci_perf.mdx | 9 ++++ .../unit/test_diffusion_nightly_comparison.py | 6 ++- .../diffusion/generate_diffusion_dashboard.py | 50 +++++++++++++++++-- 3 files changed, 61 insertions(+), 4 deletions(-) diff --git a/docs/docs/sglang-diffusion/ci_perf.mdx b/docs/docs/sglang-diffusion/ci_perf.mdx index c1f333aa1..80cdc2e76 100644 --- a/docs/docs/sglang-diffusion/ci_perf.mdx +++ b/docs/docs/sglang-diffusion/ci_perf.mdx @@ -62,3 +62,12 @@ Specify input/output paths and timeout: ```bash python python/sglang/multimodal_gen/test/scripts/gen_perf_baselines.py --baseline python/sglang/multimodal_gen/test/server/perf_baselines.json --out /tmp/perf_baselines.json --timeout 600 ``` + +## Nightly Comparison Telemetry + +The nightly comparison reports the median of three client-side measurements. +SGLang requests also write server perf dumps for stage-level timing. An +unreadable dump does not invalidate a completed client measurement, but the +dashboard reports server sample coverage (for example, `2/3`) and computes the +server total and stage medians only from readable dumps. Treat incomplete +coverage as a telemetry warning rather than a performance regression by itself. diff --git a/python/sglang/multimodal_gen/test/unit/test_diffusion_nightly_comparison.py b/python/sglang/multimodal_gen/test/unit/test_diffusion_nightly_comparison.py index 1f0af8407..f6f007150 100644 --- a/python/sglang/multimodal_gen/test/unit/test_diffusion_nightly_comparison.py +++ b/python/sglang/multimodal_gen/test/unit/test_diffusion_nightly_comparison.py @@ -117,7 +117,9 @@ def test_dashboard_uses_historical_median_and_shows_server_breakdown(): "latency_s": 10.4, "latency_samples_s": [10.3, 10.4, 10.5], "measurement_count": 3, + "server_latency_samples_s": [9.9, 10.0], "server_latency_s": 10.0, + "missing_perf_dumps": 1, "server_stage_medians_ms": { "TextEncodingStage": 100.0, "DenoisingStage": 9800.0, @@ -148,6 +150,8 @@ def test_dashboard_uses_historical_median_and_shows_server_breakdown(): assert baseline == 10.0 assert count == 3 assert alerts == [] - assert "| 3 | **10.40** |" in markdown + assert "Incomplete Server Telemetry" in markdown + assert "**model**: 2/3 server samples available" in markdown + assert "| 3 | 2/3 | **10.40** |" in markdown assert "## SGLang Server-Side Breakdown" in markdown assert "| model | 10.00 | 0.10 | 9.80 | 0.10 | 196.00 |" in markdown diff --git a/scripts/ci/utils/diffusion/generate_diffusion_dashboard.py b/scripts/ci/utils/diffusion/generate_diffusion_dashboard.py index 54a097e28..bfbfed42b 100644 --- a/scripts/ci/utils/diffusion/generate_diffusion_dashboard.py +++ b/scripts/ci/utils/diffusion/generate_diffusion_dashboard.py @@ -258,6 +258,23 @@ def _stage_group_seconds(result: dict, suffixes: str | tuple[str, ...]) -> float return sum(values) / 1000.0 +def _server_sample_coverage(result: dict) -> tuple[int, int] | None: + total = result.get("measurement_count") + if not isinstance(total, int) or total <= 0: + return None + + samples = result.get("server_latency_samples_s") + if isinstance(samples, list): + available = len(samples) + elif "missing_perf_dumps" in result: + available = total - int(result["missing_perf_dumps"]) + elif result.get("server_latency_s") is not None: + available = total + else: + available = 0 + return max(0, min(available, total)), total + + def _sanitize_filename(name: str) -> str: """Sanitize a case ID to be a safe filename.""" return name.replace("/", "_").replace(" ", "_").replace(":", "_") @@ -289,6 +306,16 @@ def generate_dashboard( current_records = _extract_case_records(current) case_ids = list(current_cases.keys()) + incomplete_server_telemetry = [] + for cid in case_ids: + record = current_records.get(cid, {}).get("sglang", {}) + coverage = _server_sample_coverage(record) + if coverage is not None and coverage[0] < coverage[1]: + model = record.get("model", cid).split("/")[-1] + incomplete_server_telemetry.append( + f"**{model}**: {coverage[0]}/{coverage[1]}" + ) + # ---- Regression detection ---- regressions: list[str] = [] if history: @@ -311,6 +338,16 @@ def generate_dashboard( lines.append(f"> - {reg}") lines.append("\n") + if incomplete_server_telemetry: + lines.append( + "> [!WARNING]\n> **Incomplete Server Telemetry**\n>\n" + "> Client-side latency includes every measured request, but server-side " + "stage medians use only the readable perf dumps." + ) + for coverage in incomplete_server_telemetry: + lines.append(f"> - {coverage} server samples available") + lines.append("\n") + # Discover all frameworks present in results all_frameworks = [] seen_fw = set() @@ -334,8 +371,8 @@ def generate_dashboard( risk_map[cid] = _assess_risk(cid, current_cases, history, other_frameworks) # Dynamic header - header = "| Model | Risk | Samples |" - sep = "|-------|------|---------|" + header = "| Model | Risk | Client samples | Server samples |" + sep = "|-------|------|----------------|----------------|" for fw in all_frameworks: header += f" {fw} median (s) |" sep += "---------|" @@ -359,10 +396,17 @@ def generate_dashboard( sample_count = sg_record.get("measurement_count") if not sample_count and sg_lat is not None: sample_count = 1 + server_coverage = _server_sample_coverage(sg_record) + server_samples = ( + f"{server_coverage[0]}/{server_coverage[1]}" + if server_coverage is not None + else "N/A" + ) risk_emoji, _ = risk_map.get(cid, ("✅", "")) row = ( - f"| {r['model'].split('/')[-1]} | {risk_emoji} | {sample_count or 'N/A'} |" + f"| {r['model'].split('/')[-1]} | {risk_emoji} | " + f"{sample_count or 'N/A'} | {server_samples} |" ) # Latency columns -- bold the fastest lats = {fw: case_fws.get(fw) for fw in all_frameworks}