[diffusion] chore: make nightly performance measurements robust (#37915)

python/sglang/multimodal_gen/runtime/entrypoints/openai/image_api.py

@@ -424,6 +424,7 @@ async def edits(
     enable_upscaling: Optional[bool] = Form(False),
     upscaling_model_path: Optional[str] = Form(None),
     upscaling_scale: Optional[int] = Form(4),
+    perf_dump_path: Optional[str] = Form(None),
     num_frames: int = Form(1),
 ):
     request_id = generate_request_id()
@@ -484,6 +485,7 @@ async def edits(
             enable_upscaling=enable_upscaling,
             upscaling_model_path=upscaling_model_path,
             upscaling_scale=upscaling_scale,
+            perf_dump_path=perf_dump_path,
         )
         trace_headers = extract_trace_headers(raw_request.headers)
         batch = prepare_request(

python/sglang/multimodal_gen/runtime/entrypoints/openai/video_api.py

@@ -494,6 +494,7 @@ async def create_video(
     output_quality: Optional[str] = Form(None),
     output_compression: Optional[int] = Form(None),
     output_path: Optional[str] = Form(None),
+    perf_dump_path: Optional[str] = Form(None),
     extra_params: Optional[str] = Form(None),
     extra_body: Optional[str] = Form(None),
 ):
@@ -645,6 +646,7 @@ async def create_video(
             output_compression=form_value("output_compression", output_compression),
             output_quality=form_value("output_quality", output_quality),
             output_path=form_value("output_path", output_path),
+            perf_dump_path=form_value("perf_dump_path", perf_dump_path),
             diffusers_kwargs=form_value("diffusers_kwargs", None),
             **extra_request_fields,
         )

python/sglang/multimodal_gen/test/unit/test_diffusion_nightly_comparison.py

@@ -0,0 +1,153 @@
+import importlib.util
+from pathlib import Path
+
+REPO_ROOT = Path(__file__).resolve().parents[5]
+
+
+def _load_script(name: str, relative_path: str):
+    spec = importlib.util.spec_from_file_location(name, REPO_ROOT / relative_path)
+    assert spec is not None and spec.loader is not None
+    module = importlib.util.module_from_spec(spec)
+    spec.loader.exec_module(module)
+    return module
+
+
+runner = _load_script(
+    "diffusion_nightly_runner",
+    "scripts/ci/utils/diffusion/run_comparison.py",
+)
+dashboard = _load_script(
+    "diffusion_nightly_dashboard",
+    "scripts/ci/utils/diffusion/generate_diffusion_dashboard.py",
+)
+
+
+def test_sglang_server_warmup_matches_measured_shape():
+    case = {
+        "model": "example/model",
+        "num_gpus": 2,
+        "width": 768,
+        "height": 512,
+        "num_frames": 121,
+    }
+    command = runner._build_sglang_cmd(
+        case,
+        {"serve_args": "--warmup-mode server --tp-size 2"},
+        30000,
+    )
+
+    resolution_index = command.index("--warmup-resolutions")
+    frame_index = command.index("--warmup-num-frames")
+    assert command[resolution_index + 1] == "768x512"
+    assert command[frame_index + 1] == "121"
+
+
+def test_explicit_server_warmup_shape_is_preserved():
+    case = {
+        "model": "example/model",
+        "num_gpus": 1,
+        "width": 1024,
+        "height": 1024,
+        "num_frames": 81,
+    }
+    command = runner._build_sglang_cmd(
+        case,
+        {
+            "serve_args": (
+                "--warmup-mode server --warmup-resolutions 512x512 "
+                "--warmup-num-frames 25"
+            )
+        },
+        30000,
+    )
+
+    assert command.count("--warmup-resolutions") == 1
+    assert command.count("--warmup-num-frames") == 1
+    assert command[command.index("--warmup-resolutions") + 1] == "512x512"
+    assert command[command.index("--warmup-num-frames") + 1] == "25"
+
+
+def test_perf_dump_summary_uses_medians():
+    perf_dumps = [
+        {
+            "total_duration_ms": 1000.0,
+            "steps": [
+                {"name": "TextEncodingStage", "duration_ms": 100.0},
+                {"name": "DenoisingStage", "duration_ms": 800.0},
+            ],
+            "denoise_steps_ms": [{"duration_ms": 8.0}, {"duration_ms": 10.0}],
+        },
+        {
+            "total_duration_ms": 3000.0,
+            "steps": [
+                {"name": "TextEncodingStage", "duration_ms": 300.0},
+                {"name": "DenoisingStage", "duration_ms": 2400.0},
+            ],
+            "denoise_steps_ms": [{"duration_ms": 30.0}],
+        },
+        {
+            "total_duration_ms": 1100.0,
+            "steps": [
+                {"name": "TextEncodingStage", "duration_ms": 110.0},
+                {"name": "DenoisingStage", "duration_ms": 880.0},
+            ],
+            "denoise_steps_ms": [{"duration_ms": 11.0}],
+        },
+    ]
+
+    summary = runner._summarize_perf_dumps(perf_dumps)
+
+    assert summary["server_latency_s"] == 1.1
+    assert summary["server_stage_medians_ms"] == {
+        "DenoisingStage": 880.0,
+        "TextEncodingStage": 110.0,
+    }
+    assert summary["median_denoise_step_ms"] == 10.5
+
+
+def test_dashboard_uses_historical_median_and_shows_server_breakdown():
+    current = {
+        "timestamp": "2026-09-04T00:00:00+00:00",
+        "commit_sha": "abcdef123456",
+        "results": [
+            {
+                "case_id": "example",
+                "framework": "sglang",
+                "model": "example/model",
+                "latency_s": 10.4,
+                "latency_samples_s": [10.3, 10.4, 10.5],
+                "measurement_count": 3,
+                "server_latency_s": 10.0,
+                "server_stage_medians_ms": {
+                    "TextEncodingStage": 100.0,
+                    "DenoisingStage": 9800.0,
+                    "DecodingStage": 100.0,
+                },
+                "median_denoise_step_ms": 196.0,
+            }
+        ],
+    }
+    history = [
+        {
+            "results": [
+                {
+                    "case_id": "example",
+                    "framework": "sglang",
+                    "latency_s": value,
+                }
+            ]
+        }
+        for value in (10.0, 30.0, 9.8)
+    ]
+
+    baseline, count = dashboard._historical_latency_baseline(
+        "example", "sglang", history
+    )
+    markdown, alerts = dashboard.generate_dashboard(current, history)
+
+    assert baseline == 10.0
+    assert count == 3
+    assert alerts == []
+    assert "| 3 | **10.40** |" in markdown
+    assert "## SGLang Server-Side Breakdown" in markdown
+    assert "| model | 10.00 | 0.10 | 9.80 | 0.10 | 196.00 |" in markdown

python/sglang/multimodal_gen/test/unit/test_openai_image_api.py

@@ -1,3 +1,4 @@
+import inspect
 import os
 from dataclasses import fields
 
@@ -23,6 +24,7 @@ from sglang.multimodal_gen.runtime.entrypoints.openai.image_api import (
     _runtime_sampling_quality,
     _select_image_variant_cloud_url,
     _select_image_variant_path,
+    edits,
 )
 from sglang.multimodal_gen.runtime.entrypoints.openai.protocol import (
     ImageGenerationsRequest,
@@ -30,6 +32,10 @@ from sglang.multimodal_gen.runtime.entrypoints.openai.protocol import (
 from sglang.multimodal_gen.runtime.pipelines_core.schedule_batch import OutputBatch
 
 
+def test_image_edits_declares_perf_dump_path_form_field():
+    assert "perf_dump_path" in inspect.signature(edits).parameters
+
+
 def test_url_response_returns_one_item_per_output_path():
     paths = ["first.png", "second.png"]
 

python/sglang/multimodal_gen/test/unit/test_video_api_profiling.py

@@ -1,3 +1,4 @@
+import inspect
 from dataclasses import fields
 from types import SimpleNamespace
 from unittest.mock import patch
@@ -16,10 +17,15 @@ from sglang.multimodal_gen.runtime.entrypoints.openai.realtime.realtime_adapter
 from sglang.multimodal_gen.runtime.entrypoints.openai.video_api import (
     _build_video_sampling_params,
     _video_request_model_kwargs,
+    create_video,
 )
 from sglang.multimodal_gen.runtime.pipelines_core.schedule_batch import Req
 
 
+def test_multipart_video_declares_perf_dump_path_form_field():
+    assert "perf_dump_path" in inspect.signature(create_video).parameters
+
+
 def test_video_api_forwards_profiling_options():
     request = VideoGenerationsRequest(
         prompt="profile this request",

scripts/ci/utils/diffusion/comparison_configs.json

@@ -1,5 +1,6 @@
 {
     "_comment": "Per-model comparison config. Sampling params omitted where model defaults are correct — only override resolution, seed, and params that differ from defaults.",
+    "measurement_repeats": 3,
     "test_image_url": "https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/diffusers/cat.png",
     "cases": [
         {

scripts/ci/utils/diffusion/generate_diffusion_dashboard.py

@@ -15,6 +15,7 @@ Usage:
 import argparse
 import json
 import os
+import statistics
 from datetime import datetime, timezone
 
 # ---------------------------------------------------------------------------
@@ -26,6 +27,8 @@ CI_DATA_REPO_NAME = "ci-data-diffusion"
 CI_DATA_BRANCH = "main"
 HISTORY_PREFIX = "diffusion-comparisons"
 MAX_HISTORY_RUNS = 29
+HISTORICAL_BASELINE_RUNS = 5
+REGRESSION_THRESHOLD = 0.05
 
 # Base URL for chart images pushed to sgl-project/ci-data-diffusion
 CHARTS_RAW_BASE_URL = (
@@ -136,6 +139,19 @@ def _short_sha(sha: str) -> str:
     return sha[:7] if sha and sha != "unknown" else "?"
 
 
+def _historical_latency_baseline(
+    cid: str, framework: str, history: list[dict]
+) -> tuple[float | None, int]:
+    values = []
+    for run in history[:HISTORICAL_BASELINE_RUNS]:
+        latency = _extract_case_results(run).get(cid, {}).get(framework)
+        if latency is not None:
+            values.append(latency)
+    if not values:
+        return None, 0
+    return statistics.median(values), len(values)
+
+
 def _assess_risk(
     cid: str,
     current_cases: dict[str, dict[str, float | None]],
@@ -146,7 +162,7 @@ def _assess_risk(
 
     Rules (checked in order):
     - N/A latency → ❌ broken
-    - History exists: SGLang latency >5% vs avg of last 3 runs → ⚠️ regression
+    - History exists: SGLang latency >5% vs median of recent runs → ⚠️ regression
     - Competitor exists & SGLang slower → 🔴 competitive risk
     - SGLang faster than all competitors by >20% → 🟢 strong advantage
     - SGLang faster than all competitors by ≤20% → 🟡 moderate advantage
@@ -158,22 +174,17 @@ def _assess_risk(
     if sg_lat is None:
         return "❌", f"{cid}: SGLang latency is N/A (broken)"
 
-    # Check regression against 3-run historical average
+    baseline, baseline_runs = _historical_latency_baseline(cid, "sglang", history)
-    if history:
+    if (
-        hist_lats: list[float] = []
+        baseline is not None
-        for run in history[:3]:
+        and baseline > 0
-            run_cases = _extract_case_results(run)
+        and (sg_lat - baseline) / baseline > REGRESSION_THRESHOLD
-            h_lat = run_cases.get(cid, {}).get("sglang")
+    ):
-            if h_lat is not None:
+        pct = (sg_lat - baseline) / baseline * 100
-                hist_lats.append(h_lat)
-        if hist_lats:
-            avg_3 = sum(hist_lats) / len(hist_lats)
-            if avg_3 > 0 and (sg_lat - avg_3) / avg_3 > 0.05:
-                pct = (sg_lat - avg_3) / avg_3 * 100
         return (
             "⚠️",
-                    f"{cid}: SGLang regression +{pct:.1f}% vs 3-run avg "
+            f"{cid}: SGLang regression +{pct:.1f}% vs {baseline_runs}-run "
-                    f"({sg_lat:.2f}s vs {avg_3:.2f}s)",
+            f"median ({sg_lat:.2f}s vs {baseline:.2f}s)",
         )
 
     # Check competitive risk
@@ -229,6 +240,24 @@ def _extract_case_results(run_data: dict) -> dict[str, dict[str, float | None]]:
     return mapping
 
 
+def _extract_case_records(run_data: dict) -> dict[str, dict[str, dict]]:
+    """Extract {case_id: {framework: result record}} from a run."""
+    mapping: dict[str, dict[str, dict]] = {}
+    for result in run_data.get("results", []):
+        mapping.setdefault(result["case_id"], {})[result["framework"]] = result
+    return mapping
+
+
+def _stage_group_seconds(result: dict, suffixes: str | tuple[str, ...]) -> float | None:
+    if isinstance(suffixes, str):
+        suffixes = (suffixes,)
+    stages = result.get("server_stage_medians_ms") or {}
+    values = [value for name, value in stages.items() if name.endswith(suffixes)]
+    if not values:
+        return None
+    return sum(values) / 1000.0
+
+
 def _sanitize_filename(name: str) -> str:
     """Sanitize a case ID to be a safe filename."""
     return name.replace("/", "_").replace(" ", "_").replace(":", "_")
@@ -257,22 +286,22 @@ def generate_dashboard(
     lines.append(f"*Generated: {_short_date(ts)} | Commit: `{_short_sha(sha)}`*\n")
 
     current_cases = _extract_case_results(current)
+    current_records = _extract_case_records(current)
     case_ids = list(current_cases.keys())
 
     # ---- Regression detection ----
-    REGRESSION_THRESHOLD = 0.05  # 5%
     regressions: list[str] = []
     if history:
-        prev_cases = _extract_case_results(history[0])
         for cid in case_ids:
             for fw in ("sglang", "vllm-omni"):
                 cur = current_cases.get(cid, {}).get(fw)
-                prev = prev_cases.get(cid, {}).get(fw)
+                baseline, baseline_runs = _historical_latency_baseline(cid, fw, history)
-                if cur and prev and prev > 0:
+                if cur is not None and baseline is not None and baseline > 0:
-                    pct = (cur - prev) / prev
+                    pct = (cur - baseline) / baseline
                     if pct > REGRESSION_THRESHOLD:
                         regressions.append(
-                            f"**{cid}** ({fw}): {prev:.2f}s -> {cur:.2f}s "
+                            f"**{cid}** ({fw}): {cur:.2f}s vs "
+                            f"{baseline_runs}-run median {baseline:.2f}s "
                             f"(+{pct * 100:.1f}%)"
                         )
 
@@ -305,10 +334,10 @@ def generate_dashboard(
         risk_map[cid] = _assess_risk(cid, current_cases, history, other_frameworks)
 
     # Dynamic header
-    header = "| Model | Risk |"
+    header = "| Model | Risk | Samples |"
-    sep = "|-------|------|"
+    sep = "|-------|------|---------|"
     for fw in all_frameworks:
-        header += f" {fw} (s) |"
+        header += f" {fw} median (s) |"
         sep += "---------|"
     for ofw in other_frameworks:
         header += f" vs {ofw} |"
@@ -326,9 +355,15 @@ def generate_dashboard(
 
         case_fws = current_cases.get(cid, {})
         sg_lat = case_fws.get("sglang")
+        sg_record = current_records.get(cid, {}).get("sglang", {})
+        sample_count = sg_record.get("measurement_count")
+        if not sample_count and sg_lat is not None:
+            sample_count = 1
 
         risk_emoji, _ = risk_map.get(cid, ("✅", ""))
-        row = f"| {r['model'].split('/')[-1]} | {risk_emoji} |"
+        row = (
+            f"| {r['model'].split('/')[-1]} | {risk_emoji} | {sample_count or 'N/A'} |"
+        )
         # Latency columns -- bold the fastest
         lats = {fw: case_fws.get(fw) for fw in all_frameworks}
         valid_lats = [v for v in lats.values() if v is not None]
@@ -344,6 +379,38 @@ def generate_dashboard(
             row += f" {_fmt_speedup(sg_lat, case_fws.get(ofw))} |"
         lines.append(row)
 
+    server_records = [
+        current_records.get(cid, {}).get("sglang", {}) for cid in case_ids
+    ]
+    if any(record.get("server_latency_s") is not None for record in server_records):
+        lines.append("\n## SGLang Server-Side Breakdown\n")
+        lines.append(
+            "| Model | Server total (s) | Text encode (s) | Denoise (s) | "
+            "Decode (s) | Median denoise step (ms) |"
+        )
+        lines.append(
+            "|-------|------------------|-----------------|--------------|"
+            "------------|---------------------------|"
+        )
+        for record in server_records:
+            if record.get("server_latency_s") is None:
+                continue
+            model = record["model"].split("/")[-1]
+            text_encode = _stage_group_seconds(
+                record, ("TextEncodingStage", "TokenizationStage")
+            )
+            denoise = _stage_group_seconds(record, "DenoisingStage")
+            decode = _stage_group_seconds(record, "DecodingStage")
+            denoise_step = record.get("median_denoise_step_ms")
+            denoise_step_text = (
+                f"{denoise_step:.2f}" if denoise_step is not None else "N/A"
+            )
+            lines.append(
+                f"| {model} | {_fmt_latency(record['server_latency_s'])} | "
+                f"{_fmt_latency(text_encode)} | {_fmt_latency(denoise)} | "
+                f"{_fmt_latency(decode)} | {denoise_step_text} |"
+            )
+
     # ---- Section 2: Speedup-over-time vs. other frameworks (rendered only when present) ----
     if history and other_frameworks:
         lines.append("\n## SGLang vs vLLM-Omni Speedup Over Time\n")

scripts/ci/utils/diffusion/run_comparison.py

@@ -1,7 +1,7 @@
 """Diffusion serving benchmark for SGLang-Diffusion nightly CI.
 
-Launches an SGLang-Diffusion server for each test case, sends a single
+Launches an SGLang-Diffusion server for each test case, sends repeated
-request, measures end-to-end latency, and writes comparison-results.json.
+requests, measures median end-to-end latency, and writes comparison-results.json.
 The runner still supports extra frameworks via --frameworks, but the nightly
 config tracks SGLang-Diffusion only.
 
@@ -24,8 +24,10 @@ import base64
 import io
 import json
 import os
+import shlex
 import signal
 import socket
+import statistics
 import subprocess
 import sys
 import tempfile
@@ -84,12 +86,29 @@ def _build_sglang_cmd(case: dict, fw_cfg: dict, port: int) -> list[str]:
     ]
     if case["num_gpus"] > 1:
         cmd += ["--num-gpus", str(case["num_gpus"])]
-    if fw_cfg.get("serve_args", "").strip():
+    serve_args = shlex.split(fw_cfg.get("serve_args", ""))
-        cmd += fw_cfg["serve_args"].strip().split()
+    cmd += serve_args
-    # No explicit --warmup-resolutions: server-based warmup now defaults to the
+
-    # model's sampling-default resolution (see warmup_request_builder), which
+    def has_option(name: str) -> bool:
-    # already matches these single-resolution cases — the default warmup is
+        return any(arg == name or arg.startswith(f"{name}=") for arg in serve_args)
-    # sufficient, so we don't pin a resolution here.
+
+    server_warmup = any(
+        arg == "--warmup-mode=server"
+        or (
+            arg == "--warmup-mode"
+            and index + 1 < len(serve_args)
+            and serve_args[index + 1] == "server"
+        )
+        for index, arg in enumerate(serve_args)
+    )
+    if server_warmup and not has_option("--warmup-resolutions"):
+        cmd += ["--warmup-resolutions", f"{case['width']}x{case['height']}"]
+    if (
+        server_warmup
+        and case.get("num_frames") is not None
+        and not has_option("--warmup-num-frames")
+    ):
+        cmd += ["--warmup-num-frames", str(case["num_frames"])]
     return cmd
 
 
@@ -375,8 +394,8 @@ def _build_sglang_payload(case: dict) -> dict:
     return payload
 
 
-def _read_perf_dump(perf_dump_path: str, timeout: float = 10.0) -> float | None:
+def _read_perf_dump(perf_dump_path: str, timeout: float = 10.0) -> dict | None:
-    """Read total_duration_ms from a perf dump JSON written by the server.
+    """Read a perf dump JSON written by the server.
 
     The server writes the file asynchronously after the HTTP response,
     so we poll briefly.
@@ -386,9 +405,8 @@ def _read_perf_dump(perf_dump_path: str, timeout: float = 10.0) -> float | None:
         try:
             with open(perf_dump_path) as f:
                 data = json.load(f)
-            total_ms = data.get("total_duration_ms")
+            if data.get("total_duration_ms") is not None:
-            if total_ms is not None:
+                return data
-                return total_ms / 1000.0
         except (FileNotFoundError, json.JSONDecodeError):
             pass
         time.sleep(0.5)
@@ -415,16 +433,6 @@ def send_image_request_sglang(
     if "data" not in data or len(data["data"]) == 0:
         raise RuntimeError(f"Image request returned no data: {data}")
 
-    # Report client-side e2e latency to match vllm-omni / lightx2v (fair
-    # cross-framework comparison); server-side perf_dump is diagnostic only.
-    if perf_dump_path:
-        server_latency = _read_perf_dump(perf_dump_path)
-        if server_latency is not None:
-            print(
-                f"  Image generated in {client_latency:.2f}s (client e2e; "
-                f"server-side {server_latency:.2f}s, diagnostic)"
-            )
-            return client_latency
     print(f"  Image generated in {client_latency:.2f}s")
     return client_latency
 
@@ -468,16 +476,6 @@ def send_video_request_sglang(
 
     client_latency = time.time() - start
 
-    # Report client-side e2e latency to match vllm-omni / lightx2v (fair
-    # cross-framework comparison); server-side perf_dump is diagnostic only.
-    if perf_dump_path:
-        server_latency = _read_perf_dump(perf_dump_path)
-        if server_latency is not None:
-            print(
-                f"  Video generated in {client_latency:.2f}s (client e2e; "
-                f"server-side {server_latency:.2f}s, diagnostic)"
-            )
-            return client_latency
     print(f"  Video generated in {client_latency:.2f}s")
     return client_latency
 
@@ -556,16 +554,6 @@ def send_image_conditioned_request_sglang(
 
     client_latency = time.time() - start
 
-    # Report client-side e2e latency to match vllm-omni / lightx2v (fair
-    # cross-framework comparison); server-side perf_dump is diagnostic only.
-    if perf_dump_path:
-        server_latency = _read_perf_dump(perf_dump_path)
-        if server_latency is not None:
-            print(
-                f"  Generated in {client_latency:.2f}s (client e2e; "
-                f"server-side {server_latency:.2f}s, diagnostic)"
-            )
-            return client_latency
     print(f"  Generated in {client_latency:.2f}s (sglang, image-conditioned)")
     return client_latency
 
@@ -730,6 +718,39 @@ def send_request(
 # ---------------------------------------------------------------------------
 
 
+def _summarize_perf_dumps(perf_dumps: list[dict]) -> dict:
+    server_latency_samples_s = [
+        round(dump["total_duration_ms"] / 1000.0, 3) for dump in perf_dumps
+    ]
+    stage_samples: dict[str, list[float]] = {}
+    denoise_step_samples: list[float] = []
+    for dump in perf_dumps:
+        for stage in dump.get("steps", []):
+            name = stage.get("name")
+            duration_ms = stage.get("duration_ms")
+            if name and duration_ms is not None:
+                stage_samples.setdefault(name, []).append(float(duration_ms))
+        denoise_step_samples.extend(
+            float(step["duration_ms"])
+            for step in dump.get("denoise_steps_ms", [])
+            if step.get("duration_ms") is not None
+        )
+
+    summary = {
+        "server_latency_samples_s": server_latency_samples_s,
+        "server_latency_s": round(statistics.median(server_latency_samples_s), 3),
+        "server_stage_medians_ms": {
+            name: round(statistics.median(values), 3)
+            for name, values in sorted(stage_samples.items())
+        },
+    }
+    if denoise_step_samples:
+        summary["median_denoise_step_ms"] = round(
+            statistics.median(denoise_step_samples), 3
+        )
+    return summary
+
+
 def run_single(
     case: dict,
     framework: str,
@@ -737,6 +758,7 @@ def run_single(
     port: int,
     log_dir: Path,
     config: dict | None = None,
+    measurement_repeats: int = 1,
 ) -> dict:
     """Run a single (case, framework) combination. Returns result dict."""
     result = {
@@ -745,6 +767,8 @@ def run_single(
         "model": case["model"],
         "task": case["task"],
         "latency_s": None,
+        "latency_samples_s": [],
+        "measurement_count": 0,
         "error": None,
     }
 
@@ -754,11 +778,6 @@ def run_single(
     env = os.environ.copy()
     env.update(fw_cfg.get("extra_env", {}))
 
-    # perf_dump_path for SGLang server-side timing (passed in request, zero overhead when None)
-    perf_dump_path = None
-    if framework == "sglang":
-        perf_dump_path = os.path.join(str(log_dir), f"perf_{case['id']}_measured.json")
-
     log_file = log_dir / f"{case['id']}_{framework}.log"
     log_fh = open(log_file, "w", encoding="utf-8", buffering=1)
     log_thread = None
@@ -811,26 +830,54 @@ def run_single(
         base_url = f"http://{DEFAULT_HOST}:{port}"
         wait_for_health(base_url, framework)
 
-        # No client-side warmup: each framework relies on its own server-side
+        # SGLang server warmup uses the measured shape added by
-        # warmup before traffic. sglang's serve_args pass --warmup-mode server,
+        # _build_sglang_cmd. The repeated requests below absorb any remaining
-        # which primes kernels with a synthetic request at startup, before the
+        # request-path cold effects, including image-conditioned preprocessing.
-        # health check passes. This goes through the internal
-        # warmup path that bypasses sampling-param preset validation (e.g.
-        # Ideogram-4's preset-locked num_inference_steps), so no per-case warmup
-        # special-casing is needed here.
         # NOTE: vllm-omni / lightx2v configure no server-side warmup; if
         # cross-framework comparison is restored, they must add their own warmup
         # to stay on equal footing — otherwise their measured request pays the
         # full cold-start.
 
-        # Measured request — pass perf_dump_path for SGLang server-side timing
+        latency_samples: list[float] = []
-        if perf_dump_path and os.path.exists(perf_dump_path):
+        perf_dumps: list[dict] = []
+        for sample_index in range(measurement_repeats):
+            perf_dump_path = None
+            if framework == "sglang":
+                sample_suffix = "" if sample_index == 0 else f"_{sample_index + 1}"
+                perf_dump_path = str(
+                    (
+                        log_dir / f"perf_{case['id']}_measured{sample_suffix}.json"
+                    ).resolve()
+                )
+                if os.path.exists(perf_dump_path):
                     os.remove(perf_dump_path)
-        print("  Sending measured request...")
+
+            print(
+                f"  Sending measured request {sample_index + 1}/"
+                f"{measurement_repeats}..."
+            )
             latency = send_request(
                 base_url, case, framework, config, perf_dump_path=perf_dump_path
             )
-        result["latency_s"] = round(latency, 3)
+            latency_samples.append(round(latency, 3))
+
+            if perf_dump_path:
+                perf_dump = _read_perf_dump(perf_dump_path)
+                if perf_dump is None:
+                    raise RuntimeError(
+                        f"Server did not write performance data to {perf_dump_path}"
+                    )
+                perf_dumps.append(perf_dump)
+                print(
+                    "  Server-side latency: "
+                    f"{perf_dump['total_duration_ms'] / 1000.0:.2f}s"
+                )
+
+        result["latency_samples_s"] = latency_samples
+        result["measurement_count"] = len(latency_samples)
+        result["latency_s"] = round(statistics.median(latency_samples), 3)
+        if perf_dumps:
+            result.update(_summarize_perf_dumps(perf_dumps))
 
     except Exception as e:
         result["error"] = server_error.get("message", str(e))
@@ -889,6 +936,7 @@ def run_comparison(
     port: int = DEFAULT_PORT,
     output: str = "comparison-results.json",
     dry_run: bool = False,
+    measurement_repeats: int | None = None,
 ) -> dict:
     """Run all comparison cases, grouped by framework to minimize installs.
 
@@ -898,6 +946,13 @@ def run_comparison(
     timestamp = datetime.now(timezone.utc).isoformat()
     commit_sha = _get_checkout_commit_sha()
     run_id = os.environ.get("GITHUB_RUN_ID", "local")
+    repeats = (
+        measurement_repeats
+        if measurement_repeats is not None
+        else int(config.get("measurement_repeats", 1))
+    )
+    if repeats <= 0:
+        raise ValueError("measurement_repeats must be a positive integer")
 
     log_dir = Path("comparison-logs")
     log_dir.mkdir(exist_ok=True)
@@ -962,7 +1017,15 @@ def run_comparison(
                 )
                 continue
 
-            result = run_single(case, fw_name, fw_cfg, port, log_dir, config)
+            result = run_single(
+                case,
+                fw_name,
+                fw_cfg,
+                port,
+                log_dir,
+                config,
+                measurement_repeats=repeats,
+            )
             results.append(result)
 
             # Wait for GPU memory to clear
@@ -973,6 +1036,7 @@ def run_comparison(
         "timestamp": timestamp,
         "commit_sha": commit_sha,
         "run_id": run_id,
+        "measurement_repeats": repeats,
         "results": results,
     }
 
@@ -986,7 +1050,11 @@ def run_comparison(
     print("SUMMARY")
     print(f"{'=' * 60}")
     for r in results:
-        lat = f"{r['latency_s']:.2f}s" if r["latency_s"] else r.get("error", "N/A")
+        lat = (
+            f"{r['latency_s']:.2f}s median (n={r.get('measurement_count', 1)})"
+            if r["latency_s"]
+            else r.get("error", "N/A")
+        )
         print(f"  {r['case_id']:30s} | {r['framework']:12s} | {lat}")
 
     return output_data
@@ -1034,6 +1102,12 @@ def main():
         action="store_true",
         help="Parse config and print commands without launching servers",
     )
+    parser.add_argument(
+        "--measurement-repeats",
+        type=int,
+        default=None,
+        help="Measured requests per case (default: value from config)",
+    )
 
     args = parser.parse_args()
 
@@ -1049,6 +1123,7 @@ def main():
         port=args.port,
         output=args.output,
         dry_run=args.dry_run,
+        measurement_repeats=args.measurement_repeats,
     )
 
     # Exit with non-zero if any case had an error