[diffusion] CI: expose nightly server telemetry coverage (#38782)

Co-authored-by: Mick Qian <mickqian@users.noreply.github.com>

scripts/ci/utils/diffusion/generate_diffusion_dashboard.py

@@ -258,6 +258,23 @@ def _stage_group_seconds(result: dict, suffixes: str | tuple[str, ...]) -> float
     return sum(values) / 1000.0
 
 
+def _server_sample_coverage(result: dict) -> tuple[int, int] | None:
+    total = result.get("measurement_count")
+    if not isinstance(total, int) or total <= 0:
+        return None
+
+    samples = result.get("server_latency_samples_s")
+    if isinstance(samples, list):
+        available = len(samples)
+    elif "missing_perf_dumps" in result:
+        available = total - int(result["missing_perf_dumps"])
+    elif result.get("server_latency_s") is not None:
+        available = total
+    else:
+        available = 0
+    return max(0, min(available, total)), total
+
+
 def _sanitize_filename(name: str) -> str:
     """Sanitize a case ID to be a safe filename."""
     return name.replace("/", "_").replace(" ", "_").replace(":", "_")
@@ -289,6 +306,16 @@ def generate_dashboard(
     current_records = _extract_case_records(current)
     case_ids = list(current_cases.keys())
 
+    incomplete_server_telemetry = []
+    for cid in case_ids:
+        record = current_records.get(cid, {}).get("sglang", {})
+        coverage = _server_sample_coverage(record)
+        if coverage is not None and coverage[0] < coverage[1]:
+            model = record.get("model", cid).split("/")[-1]
+            incomplete_server_telemetry.append(
+                f"**{model}**: {coverage[0]}/{coverage[1]}"
+            )
+
     # ---- Regression detection ----
     regressions: list[str] = []
     if history:
@@ -311,6 +338,16 @@ def generate_dashboard(
             lines.append(f"> - {reg}")
         lines.append("\n")
 
+    if incomplete_server_telemetry:
+        lines.append(
+            "> [!WARNING]\n> **Incomplete Server Telemetry**\n>\n"
+            "> Client-side latency includes every measured request, but server-side "
+            "stage medians use only the readable perf dumps."
+        )
+        for coverage in incomplete_server_telemetry:
+            lines.append(f"> - {coverage} server samples available")
+        lines.append("\n")
+
     # Discover all frameworks present in results
     all_frameworks = []
     seen_fw = set()
@@ -334,8 +371,8 @@ def generate_dashboard(
         risk_map[cid] = _assess_risk(cid, current_cases, history, other_frameworks)
 
     # Dynamic header
-    header = "| Model | Risk | Samples |"
-    sep = "|-------|------|---------|"
+    header = "| Model | Risk | Client samples | Server samples |"
+    sep = "|-------|------|----------------|----------------|"
     for fw in all_frameworks:
         header += f" {fw} median (s) |"
         sep += "---------|"
@@ -359,10 +396,17 @@ def generate_dashboard(
         sample_count = sg_record.get("measurement_count")
         if not sample_count and sg_lat is not None:
             sample_count = 1
+        server_coverage = _server_sample_coverage(sg_record)
+        server_samples = (
+            f"{server_coverage[0]}/{server_coverage[1]}"
+            if server_coverage is not None
+            else "N/A"
+        )
 
         risk_emoji, _ = risk_map.get(cid, ("✅", ""))
         row = (
-            f"| {r['model'].split('/')[-1]} | {risk_emoji} | {sample_count or 'N/A'} |"
+            f"| {r['model'].split('/')[-1]} | {risk_emoji} | "
+            f"{sample_count or 'N/A'} | {server_samples} |"
         )
         # Latency columns -- bold the fastest
         lats = {fw: case_fws.get(fw) for fw in all_frameworks}