[diffusion] CI: expose nightly server telemetry coverage (#38782)

Co-authored-by: Mick Qian <mickqian@users.noreply.github.com>

docs/docs/sglang-diffusion/ci_perf.mdx

@@ -62,3 +62,12 @@ Specify input/output paths and timeout:
 ```bash
 python python/sglang/multimodal_gen/test/scripts/gen_perf_baselines.py --baseline python/sglang/multimodal_gen/test/server/perf_baselines.json --out /tmp/perf_baselines.json --timeout 600
 ```
+
+## Nightly Comparison Telemetry
+
+The nightly comparison reports the median of three client-side measurements.
+SGLang requests also write server perf dumps for stage-level timing. An
+unreadable dump does not invalidate a completed client measurement, but the
+dashboard reports server sample coverage (for example, `2/3`) and computes the
+server total and stage medians only from readable dumps. Treat incomplete
+coverage as a telemetry warning rather than a performance regression by itself.

python/sglang/multimodal_gen/test/unit/test_diffusion_nightly_comparison.py

@@ -117,7 +117,9 @@ def test_dashboard_uses_historical_median_and_shows_server_breakdown():
                 "latency_s": 10.4,
                 "latency_samples_s": [10.3, 10.4, 10.5],
                 "measurement_count": 3,
+                "server_latency_samples_s": [9.9, 10.0],
                 "server_latency_s": 10.0,
+                "missing_perf_dumps": 1,
                 "server_stage_medians_ms": {
                     "TextEncodingStage": 100.0,
                     "DenoisingStage": 9800.0,
@@ -148,6 +150,8 @@ def test_dashboard_uses_historical_median_and_shows_server_breakdown():
     assert baseline == 10.0
     assert count == 3
     assert alerts == []
-    assert "| 3 | **10.40** |" in markdown
+    assert "Incomplete Server Telemetry" in markdown
+    assert "**model**: 2/3 server samples available" in markdown
+    assert "| 3 | 2/3 | **10.40** |" in markdown
     assert "## SGLang Server-Side Breakdown" in markdown
     assert "| model | 10.00 | 0.10 | 9.80 | 0.10 | 196.00 |" in markdown

scripts/ci/utils/diffusion/generate_diffusion_dashboard.py

@@ -258,6 +258,23 @@ def _stage_group_seconds(result: dict, suffixes: str | tuple[str, ...]) -> float
     return sum(values) / 1000.0
 
 
+def _server_sample_coverage(result: dict) -> tuple[int, int] | None:
+    total = result.get("measurement_count")
+    if not isinstance(total, int) or total <= 0:
+        return None
+
+    samples = result.get("server_latency_samples_s")
+    if isinstance(samples, list):
+        available = len(samples)
+    elif "missing_perf_dumps" in result:
+        available = total - int(result["missing_perf_dumps"])
+    elif result.get("server_latency_s") is not None:
+        available = total
+    else:
+        available = 0
+    return max(0, min(available, total)), total
+
+
 def _sanitize_filename(name: str) -> str:
     """Sanitize a case ID to be a safe filename."""
     return name.replace("/", "_").replace(" ", "_").replace(":", "_")
@@ -289,6 +306,16 @@ def generate_dashboard(
     current_records = _extract_case_records(current)
     case_ids = list(current_cases.keys())
 
+    incomplete_server_telemetry = []
+    for cid in case_ids:
+        record = current_records.get(cid, {}).get("sglang", {})
+        coverage = _server_sample_coverage(record)
+        if coverage is not None and coverage[0] < coverage[1]:
+            model = record.get("model", cid).split("/")[-1]
+            incomplete_server_telemetry.append(
+                f"**{model}**: {coverage[0]}/{coverage[1]}"
+            )
+
     # ---- Regression detection ----
     regressions: list[str] = []
     if history:
@@ -311,6 +338,16 @@ def generate_dashboard(
             lines.append(f"> - {reg}")
         lines.append("\n")
 
+    if incomplete_server_telemetry:
+        lines.append(
+            "> [!WARNING]\n> **Incomplete Server Telemetry**\n>\n"
+            "> Client-side latency includes every measured request, but server-side "
+            "stage medians use only the readable perf dumps."
+        )
+        for coverage in incomplete_server_telemetry:
+            lines.append(f"> - {coverage} server samples available")
+        lines.append("\n")
+
     # Discover all frameworks present in results
     all_frameworks = []
     seen_fw = set()
@@ -334,8 +371,8 @@ def generate_dashboard(
         risk_map[cid] = _assess_risk(cid, current_cases, history, other_frameworks)
 
     # Dynamic header
-    header = "| Model | Risk | Samples |"
-    sep = "|-------|------|---------|"
+    header = "| Model | Risk | Client samples | Server samples |"
+    sep = "|-------|------|----------------|----------------|"
     for fw in all_frameworks:
         header += f" {fw} median (s) |"
         sep += "---------|"
@@ -359,10 +396,17 @@ def generate_dashboard(
         sample_count = sg_record.get("measurement_count")
         if not sample_count and sg_lat is not None:
             sample_count = 1
+        server_coverage = _server_sample_coverage(sg_record)
+        server_samples = (
+            f"{server_coverage[0]}/{server_coverage[1]}"
+            if server_coverage is not None
+            else "N/A"
+        )
 
         risk_emoji, _ = risk_map.get(cid, ("✅", ""))
         row = (
-            f"| {r['model'].split('/')[-1]} | {risk_emoji} | {sample_count or 'N/A'} |"
+            f"| {r['model'].split('/')[-1]} | {risk_emoji} | "
+            f"{sample_count or 'N/A'} | {server_samples} |"
         )
         # Latency columns -- bold the fastest
         lats = {fw: case_fws.get(fw) for fw in all_frameworks}