From 88021b0734937c1efc19c2ceb2c1250a507d1fc8 Mon Sep 17 00:00:00 2001 From: Mick Date: Fri, 4 Sep 2026 22:03:25 +0800 Subject: [PATCH] [diffusion] chore: make nightly performance measurements robust (#37915) --- .../runtime/entrypoints/openai/image_api.py | 2 + .../runtime/entrypoints/openai/video_api.py | 2 + .../unit/test_diffusion_nightly_comparison.py | 153 +++++++++++++ .../test/unit/test_openai_image_api.py | 6 + .../test/unit/test_video_api_profiling.py | 6 + .../utils/diffusion/comparison_configs.json | 1 + .../diffusion/generate_diffusion_dashboard.py | 123 ++++++++--- scripts/ci/utils/diffusion/run_comparison.py | 205 ++++++++++++------ 8 files changed, 405 insertions(+), 93 deletions(-) create mode 100644 python/sglang/multimodal_gen/test/unit/test_diffusion_nightly_comparison.py diff --git a/python/sglang/multimodal_gen/runtime/entrypoints/openai/image_api.py b/python/sglang/multimodal_gen/runtime/entrypoints/openai/image_api.py index 88b40d0e7..2e5a13ba1 100644 --- a/python/sglang/multimodal_gen/runtime/entrypoints/openai/image_api.py +++ b/python/sglang/multimodal_gen/runtime/entrypoints/openai/image_api.py @@ -424,6 +424,7 @@ async def edits( enable_upscaling: Optional[bool] = Form(False), upscaling_model_path: Optional[str] = Form(None), upscaling_scale: Optional[int] = Form(4), + perf_dump_path: Optional[str] = Form(None), num_frames: int = Form(1), ): request_id = generate_request_id() @@ -484,6 +485,7 @@ async def edits( enable_upscaling=enable_upscaling, upscaling_model_path=upscaling_model_path, upscaling_scale=upscaling_scale, + perf_dump_path=perf_dump_path, ) trace_headers = extract_trace_headers(raw_request.headers) batch = prepare_request( diff --git a/python/sglang/multimodal_gen/runtime/entrypoints/openai/video_api.py b/python/sglang/multimodal_gen/runtime/entrypoints/openai/video_api.py index 47d884546..17245ec5e 100644 --- a/python/sglang/multimodal_gen/runtime/entrypoints/openai/video_api.py +++ b/python/sglang/multimodal_gen/runtime/entrypoints/openai/video_api.py @@ -494,6 +494,7 @@ async def create_video( output_quality: Optional[str] = Form(None), output_compression: Optional[int] = Form(None), output_path: Optional[str] = Form(None), + perf_dump_path: Optional[str] = Form(None), extra_params: Optional[str] = Form(None), extra_body: Optional[str] = Form(None), ): @@ -645,6 +646,7 @@ async def create_video( output_compression=form_value("output_compression", output_compression), output_quality=form_value("output_quality", output_quality), output_path=form_value("output_path", output_path), + perf_dump_path=form_value("perf_dump_path", perf_dump_path), diffusers_kwargs=form_value("diffusers_kwargs", None), **extra_request_fields, ) diff --git a/python/sglang/multimodal_gen/test/unit/test_diffusion_nightly_comparison.py b/python/sglang/multimodal_gen/test/unit/test_diffusion_nightly_comparison.py new file mode 100644 index 000000000..1f0af8407 --- /dev/null +++ b/python/sglang/multimodal_gen/test/unit/test_diffusion_nightly_comparison.py @@ -0,0 +1,153 @@ +import importlib.util +from pathlib import Path + +REPO_ROOT = Path(__file__).resolve().parents[5] + + +def _load_script(name: str, relative_path: str): + spec = importlib.util.spec_from_file_location(name, REPO_ROOT / relative_path) + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +runner = _load_script( + "diffusion_nightly_runner", + "scripts/ci/utils/diffusion/run_comparison.py", +) +dashboard = _load_script( + "diffusion_nightly_dashboard", + "scripts/ci/utils/diffusion/generate_diffusion_dashboard.py", +) + + +def test_sglang_server_warmup_matches_measured_shape(): + case = { + "model": "example/model", + "num_gpus": 2, + "width": 768, + "height": 512, + "num_frames": 121, + } + command = runner._build_sglang_cmd( + case, + {"serve_args": "--warmup-mode server --tp-size 2"}, + 30000, + ) + + resolution_index = command.index("--warmup-resolutions") + frame_index = command.index("--warmup-num-frames") + assert command[resolution_index + 1] == "768x512" + assert command[frame_index + 1] == "121" + + +def test_explicit_server_warmup_shape_is_preserved(): + case = { + "model": "example/model", + "num_gpus": 1, + "width": 1024, + "height": 1024, + "num_frames": 81, + } + command = runner._build_sglang_cmd( + case, + { + "serve_args": ( + "--warmup-mode server --warmup-resolutions 512x512 " + "--warmup-num-frames 25" + ) + }, + 30000, + ) + + assert command.count("--warmup-resolutions") == 1 + assert command.count("--warmup-num-frames") == 1 + assert command[command.index("--warmup-resolutions") + 1] == "512x512" + assert command[command.index("--warmup-num-frames") + 1] == "25" + + +def test_perf_dump_summary_uses_medians(): + perf_dumps = [ + { + "total_duration_ms": 1000.0, + "steps": [ + {"name": "TextEncodingStage", "duration_ms": 100.0}, + {"name": "DenoisingStage", "duration_ms": 800.0}, + ], + "denoise_steps_ms": [{"duration_ms": 8.0}, {"duration_ms": 10.0}], + }, + { + "total_duration_ms": 3000.0, + "steps": [ + {"name": "TextEncodingStage", "duration_ms": 300.0}, + {"name": "DenoisingStage", "duration_ms": 2400.0}, + ], + "denoise_steps_ms": [{"duration_ms": 30.0}], + }, + { + "total_duration_ms": 1100.0, + "steps": [ + {"name": "TextEncodingStage", "duration_ms": 110.0}, + {"name": "DenoisingStage", "duration_ms": 880.0}, + ], + "denoise_steps_ms": [{"duration_ms": 11.0}], + }, + ] + + summary = runner._summarize_perf_dumps(perf_dumps) + + assert summary["server_latency_s"] == 1.1 + assert summary["server_stage_medians_ms"] == { + "DenoisingStage": 880.0, + "TextEncodingStage": 110.0, + } + assert summary["median_denoise_step_ms"] == 10.5 + + +def test_dashboard_uses_historical_median_and_shows_server_breakdown(): + current = { + "timestamp": "2026-09-04T00:00:00+00:00", + "commit_sha": "abcdef123456", + "results": [ + { + "case_id": "example", + "framework": "sglang", + "model": "example/model", + "latency_s": 10.4, + "latency_samples_s": [10.3, 10.4, 10.5], + "measurement_count": 3, + "server_latency_s": 10.0, + "server_stage_medians_ms": { + "TextEncodingStage": 100.0, + "DenoisingStage": 9800.0, + "DecodingStage": 100.0, + }, + "median_denoise_step_ms": 196.0, + } + ], + } + history = [ + { + "results": [ + { + "case_id": "example", + "framework": "sglang", + "latency_s": value, + } + ] + } + for value in (10.0, 30.0, 9.8) + ] + + baseline, count = dashboard._historical_latency_baseline( + "example", "sglang", history + ) + markdown, alerts = dashboard.generate_dashboard(current, history) + + assert baseline == 10.0 + assert count == 3 + assert alerts == [] + assert "| 3 | **10.40** |" in markdown + assert "## SGLang Server-Side Breakdown" in markdown + assert "| model | 10.00 | 0.10 | 9.80 | 0.10 | 196.00 |" in markdown diff --git a/python/sglang/multimodal_gen/test/unit/test_openai_image_api.py b/python/sglang/multimodal_gen/test/unit/test_openai_image_api.py index 81ee23cd2..492d06577 100644 --- a/python/sglang/multimodal_gen/test/unit/test_openai_image_api.py +++ b/python/sglang/multimodal_gen/test/unit/test_openai_image_api.py @@ -1,3 +1,4 @@ +import inspect import os from dataclasses import fields @@ -23,6 +24,7 @@ from sglang.multimodal_gen.runtime.entrypoints.openai.image_api import ( _runtime_sampling_quality, _select_image_variant_cloud_url, _select_image_variant_path, + edits, ) from sglang.multimodal_gen.runtime.entrypoints.openai.protocol import ( ImageGenerationsRequest, @@ -30,6 +32,10 @@ from sglang.multimodal_gen.runtime.entrypoints.openai.protocol import ( from sglang.multimodal_gen.runtime.pipelines_core.schedule_batch import OutputBatch +def test_image_edits_declares_perf_dump_path_form_field(): + assert "perf_dump_path" in inspect.signature(edits).parameters + + def test_url_response_returns_one_item_per_output_path(): paths = ["first.png", "second.png"] diff --git a/python/sglang/multimodal_gen/test/unit/test_video_api_profiling.py b/python/sglang/multimodal_gen/test/unit/test_video_api_profiling.py index e2e14c3d6..a44542466 100644 --- a/python/sglang/multimodal_gen/test/unit/test_video_api_profiling.py +++ b/python/sglang/multimodal_gen/test/unit/test_video_api_profiling.py @@ -1,3 +1,4 @@ +import inspect from dataclasses import fields from types import SimpleNamespace from unittest.mock import patch @@ -16,10 +17,15 @@ from sglang.multimodal_gen.runtime.entrypoints.openai.realtime.realtime_adapter from sglang.multimodal_gen.runtime.entrypoints.openai.video_api import ( _build_video_sampling_params, _video_request_model_kwargs, + create_video, ) from sglang.multimodal_gen.runtime.pipelines_core.schedule_batch import Req +def test_multipart_video_declares_perf_dump_path_form_field(): + assert "perf_dump_path" in inspect.signature(create_video).parameters + + def test_video_api_forwards_profiling_options(): request = VideoGenerationsRequest( prompt="profile this request", diff --git a/scripts/ci/utils/diffusion/comparison_configs.json b/scripts/ci/utils/diffusion/comparison_configs.json index 755253c29..7d5df4d83 100644 --- a/scripts/ci/utils/diffusion/comparison_configs.json +++ b/scripts/ci/utils/diffusion/comparison_configs.json @@ -1,5 +1,6 @@ { "_comment": "Per-model comparison config. Sampling params omitted where model defaults are correct — only override resolution, seed, and params that differ from defaults.", + "measurement_repeats": 3, "test_image_url": "https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/diffusers/cat.png", "cases": [ { diff --git a/scripts/ci/utils/diffusion/generate_diffusion_dashboard.py b/scripts/ci/utils/diffusion/generate_diffusion_dashboard.py index 9ca9b41bc..54a097e28 100644 --- a/scripts/ci/utils/diffusion/generate_diffusion_dashboard.py +++ b/scripts/ci/utils/diffusion/generate_diffusion_dashboard.py @@ -15,6 +15,7 @@ Usage: import argparse import json import os +import statistics from datetime import datetime, timezone # --------------------------------------------------------------------------- @@ -26,6 +27,8 @@ CI_DATA_REPO_NAME = "ci-data-diffusion" CI_DATA_BRANCH = "main" HISTORY_PREFIX = "diffusion-comparisons" MAX_HISTORY_RUNS = 29 +HISTORICAL_BASELINE_RUNS = 5 +REGRESSION_THRESHOLD = 0.05 # Base URL for chart images pushed to sgl-project/ci-data-diffusion CHARTS_RAW_BASE_URL = ( @@ -136,6 +139,19 @@ def _short_sha(sha: str) -> str: return sha[:7] if sha and sha != "unknown" else "?" +def _historical_latency_baseline( + cid: str, framework: str, history: list[dict] +) -> tuple[float | None, int]: + values = [] + for run in history[:HISTORICAL_BASELINE_RUNS]: + latency = _extract_case_results(run).get(cid, {}).get(framework) + if latency is not None: + values.append(latency) + if not values: + return None, 0 + return statistics.median(values), len(values) + + def _assess_risk( cid: str, current_cases: dict[str, dict[str, float | None]], @@ -146,7 +162,7 @@ def _assess_risk( Rules (checked in order): - N/A latency → ❌ broken - - History exists: SGLang latency >5% vs avg of last 3 runs → ⚠️ regression + - History exists: SGLang latency >5% vs median of recent runs → ⚠️ regression - Competitor exists & SGLang slower → 🔴 competitive risk - SGLang faster than all competitors by >20% → 🟢 strong advantage - SGLang faster than all competitors by ≤20% → 🟡 moderate advantage @@ -158,23 +174,18 @@ def _assess_risk( if sg_lat is None: return "❌", f"{cid}: SGLang latency is N/A (broken)" - # Check regression against 3-run historical average - if history: - hist_lats: list[float] = [] - for run in history[:3]: - run_cases = _extract_case_results(run) - h_lat = run_cases.get(cid, {}).get("sglang") - if h_lat is not None: - hist_lats.append(h_lat) - if hist_lats: - avg_3 = sum(hist_lats) / len(hist_lats) - if avg_3 > 0 and (sg_lat - avg_3) / avg_3 > 0.05: - pct = (sg_lat - avg_3) / avg_3 * 100 - return ( - "⚠️", - f"{cid}: SGLang regression +{pct:.1f}% vs 3-run avg " - f"({sg_lat:.2f}s vs {avg_3:.2f}s)", - ) + baseline, baseline_runs = _historical_latency_baseline(cid, "sglang", history) + if ( + baseline is not None + and baseline > 0 + and (sg_lat - baseline) / baseline > REGRESSION_THRESHOLD + ): + pct = (sg_lat - baseline) / baseline * 100 + return ( + "⚠️", + f"{cid}: SGLang regression +{pct:.1f}% vs {baseline_runs}-run " + f"median ({sg_lat:.2f}s vs {baseline:.2f}s)", + ) # Check competitive risk if other_frameworks: @@ -229,6 +240,24 @@ def _extract_case_results(run_data: dict) -> dict[str, dict[str, float | None]]: return mapping +def _extract_case_records(run_data: dict) -> dict[str, dict[str, dict]]: + """Extract {case_id: {framework: result record}} from a run.""" + mapping: dict[str, dict[str, dict]] = {} + for result in run_data.get("results", []): + mapping.setdefault(result["case_id"], {})[result["framework"]] = result + return mapping + + +def _stage_group_seconds(result: dict, suffixes: str | tuple[str, ...]) -> float | None: + if isinstance(suffixes, str): + suffixes = (suffixes,) + stages = result.get("server_stage_medians_ms") or {} + values = [value for name, value in stages.items() if name.endswith(suffixes)] + if not values: + return None + return sum(values) / 1000.0 + + def _sanitize_filename(name: str) -> str: """Sanitize a case ID to be a safe filename.""" return name.replace("/", "_").replace(" ", "_").replace(":", "_") @@ -257,22 +286,22 @@ def generate_dashboard( lines.append(f"*Generated: {_short_date(ts)} | Commit: `{_short_sha(sha)}`*\n") current_cases = _extract_case_results(current) + current_records = _extract_case_records(current) case_ids = list(current_cases.keys()) # ---- Regression detection ---- - REGRESSION_THRESHOLD = 0.05 # 5% regressions: list[str] = [] if history: - prev_cases = _extract_case_results(history[0]) for cid in case_ids: for fw in ("sglang", "vllm-omni"): cur = current_cases.get(cid, {}).get(fw) - prev = prev_cases.get(cid, {}).get(fw) - if cur and prev and prev > 0: - pct = (cur - prev) / prev + baseline, baseline_runs = _historical_latency_baseline(cid, fw, history) + if cur is not None and baseline is not None and baseline > 0: + pct = (cur - baseline) / baseline if pct > REGRESSION_THRESHOLD: regressions.append( - f"**{cid}** ({fw}): {prev:.2f}s -> {cur:.2f}s " + f"**{cid}** ({fw}): {cur:.2f}s vs " + f"{baseline_runs}-run median {baseline:.2f}s " f"(+{pct * 100:.1f}%)" ) @@ -305,10 +334,10 @@ def generate_dashboard( risk_map[cid] = _assess_risk(cid, current_cases, history, other_frameworks) # Dynamic header - header = "| Model | Risk |" - sep = "|-------|------|" + header = "| Model | Risk | Samples |" + sep = "|-------|------|---------|" for fw in all_frameworks: - header += f" {fw} (s) |" + header += f" {fw} median (s) |" sep += "---------|" for ofw in other_frameworks: header += f" vs {ofw} |" @@ -326,9 +355,15 @@ def generate_dashboard( case_fws = current_cases.get(cid, {}) sg_lat = case_fws.get("sglang") + sg_record = current_records.get(cid, {}).get("sglang", {}) + sample_count = sg_record.get("measurement_count") + if not sample_count and sg_lat is not None: + sample_count = 1 risk_emoji, _ = risk_map.get(cid, ("✅", "")) - row = f"| {r['model'].split('/')[-1]} | {risk_emoji} |" + row = ( + f"| {r['model'].split('/')[-1]} | {risk_emoji} | {sample_count or 'N/A'} |" + ) # Latency columns -- bold the fastest lats = {fw: case_fws.get(fw) for fw in all_frameworks} valid_lats = [v for v in lats.values() if v is not None] @@ -344,6 +379,38 @@ def generate_dashboard( row += f" {_fmt_speedup(sg_lat, case_fws.get(ofw))} |" lines.append(row) + server_records = [ + current_records.get(cid, {}).get("sglang", {}) for cid in case_ids + ] + if any(record.get("server_latency_s") is not None for record in server_records): + lines.append("\n## SGLang Server-Side Breakdown\n") + lines.append( + "| Model | Server total (s) | Text encode (s) | Denoise (s) | " + "Decode (s) | Median denoise step (ms) |" + ) + lines.append( + "|-------|------------------|-----------------|--------------|" + "------------|---------------------------|" + ) + for record in server_records: + if record.get("server_latency_s") is None: + continue + model = record["model"].split("/")[-1] + text_encode = _stage_group_seconds( + record, ("TextEncodingStage", "TokenizationStage") + ) + denoise = _stage_group_seconds(record, "DenoisingStage") + decode = _stage_group_seconds(record, "DecodingStage") + denoise_step = record.get("median_denoise_step_ms") + denoise_step_text = ( + f"{denoise_step:.2f}" if denoise_step is not None else "N/A" + ) + lines.append( + f"| {model} | {_fmt_latency(record['server_latency_s'])} | " + f"{_fmt_latency(text_encode)} | {_fmt_latency(denoise)} | " + f"{_fmt_latency(decode)} | {denoise_step_text} |" + ) + # ---- Section 2: Speedup-over-time vs. other frameworks (rendered only when present) ---- if history and other_frameworks: lines.append("\n## SGLang vs vLLM-Omni Speedup Over Time\n") diff --git a/scripts/ci/utils/diffusion/run_comparison.py b/scripts/ci/utils/diffusion/run_comparison.py index 74cc651a5..31fe48a2b 100644 --- a/scripts/ci/utils/diffusion/run_comparison.py +++ b/scripts/ci/utils/diffusion/run_comparison.py @@ -1,7 +1,7 @@ """Diffusion serving benchmark for SGLang-Diffusion nightly CI. -Launches an SGLang-Diffusion server for each test case, sends a single -request, measures end-to-end latency, and writes comparison-results.json. +Launches an SGLang-Diffusion server for each test case, sends repeated +requests, measures median end-to-end latency, and writes comparison-results.json. The runner still supports extra frameworks via --frameworks, but the nightly config tracks SGLang-Diffusion only. @@ -24,8 +24,10 @@ import base64 import io import json import os +import shlex import signal import socket +import statistics import subprocess import sys import tempfile @@ -84,12 +86,29 @@ def _build_sglang_cmd(case: dict, fw_cfg: dict, port: int) -> list[str]: ] if case["num_gpus"] > 1: cmd += ["--num-gpus", str(case["num_gpus"])] - if fw_cfg.get("serve_args", "").strip(): - cmd += fw_cfg["serve_args"].strip().split() - # No explicit --warmup-resolutions: server-based warmup now defaults to the - # model's sampling-default resolution (see warmup_request_builder), which - # already matches these single-resolution cases — the default warmup is - # sufficient, so we don't pin a resolution here. + serve_args = shlex.split(fw_cfg.get("serve_args", "")) + cmd += serve_args + + def has_option(name: str) -> bool: + return any(arg == name or arg.startswith(f"{name}=") for arg in serve_args) + + server_warmup = any( + arg == "--warmup-mode=server" + or ( + arg == "--warmup-mode" + and index + 1 < len(serve_args) + and serve_args[index + 1] == "server" + ) + for index, arg in enumerate(serve_args) + ) + if server_warmup and not has_option("--warmup-resolutions"): + cmd += ["--warmup-resolutions", f"{case['width']}x{case['height']}"] + if ( + server_warmup + and case.get("num_frames") is not None + and not has_option("--warmup-num-frames") + ): + cmd += ["--warmup-num-frames", str(case["num_frames"])] return cmd @@ -375,8 +394,8 @@ def _build_sglang_payload(case: dict) -> dict: return payload -def _read_perf_dump(perf_dump_path: str, timeout: float = 10.0) -> float | None: - """Read total_duration_ms from a perf dump JSON written by the server. +def _read_perf_dump(perf_dump_path: str, timeout: float = 10.0) -> dict | None: + """Read a perf dump JSON written by the server. The server writes the file asynchronously after the HTTP response, so we poll briefly. @@ -386,9 +405,8 @@ def _read_perf_dump(perf_dump_path: str, timeout: float = 10.0) -> float | None: try: with open(perf_dump_path) as f: data = json.load(f) - total_ms = data.get("total_duration_ms") - if total_ms is not None: - return total_ms / 1000.0 + if data.get("total_duration_ms") is not None: + return data except (FileNotFoundError, json.JSONDecodeError): pass time.sleep(0.5) @@ -415,16 +433,6 @@ def send_image_request_sglang( if "data" not in data or len(data["data"]) == 0: raise RuntimeError(f"Image request returned no data: {data}") - # Report client-side e2e latency to match vllm-omni / lightx2v (fair - # cross-framework comparison); server-side perf_dump is diagnostic only. - if perf_dump_path: - server_latency = _read_perf_dump(perf_dump_path) - if server_latency is not None: - print( - f" Image generated in {client_latency:.2f}s (client e2e; " - f"server-side {server_latency:.2f}s, diagnostic)" - ) - return client_latency print(f" Image generated in {client_latency:.2f}s") return client_latency @@ -468,16 +476,6 @@ def send_video_request_sglang( client_latency = time.time() - start - # Report client-side e2e latency to match vllm-omni / lightx2v (fair - # cross-framework comparison); server-side perf_dump is diagnostic only. - if perf_dump_path: - server_latency = _read_perf_dump(perf_dump_path) - if server_latency is not None: - print( - f" Video generated in {client_latency:.2f}s (client e2e; " - f"server-side {server_latency:.2f}s, diagnostic)" - ) - return client_latency print(f" Video generated in {client_latency:.2f}s") return client_latency @@ -556,16 +554,6 @@ def send_image_conditioned_request_sglang( client_latency = time.time() - start - # Report client-side e2e latency to match vllm-omni / lightx2v (fair - # cross-framework comparison); server-side perf_dump is diagnostic only. - if perf_dump_path: - server_latency = _read_perf_dump(perf_dump_path) - if server_latency is not None: - print( - f" Generated in {client_latency:.2f}s (client e2e; " - f"server-side {server_latency:.2f}s, diagnostic)" - ) - return client_latency print(f" Generated in {client_latency:.2f}s (sglang, image-conditioned)") return client_latency @@ -730,6 +718,39 @@ def send_request( # --------------------------------------------------------------------------- +def _summarize_perf_dumps(perf_dumps: list[dict]) -> dict: + server_latency_samples_s = [ + round(dump["total_duration_ms"] / 1000.0, 3) for dump in perf_dumps + ] + stage_samples: dict[str, list[float]] = {} + denoise_step_samples: list[float] = [] + for dump in perf_dumps: + for stage in dump.get("steps", []): + name = stage.get("name") + duration_ms = stage.get("duration_ms") + if name and duration_ms is not None: + stage_samples.setdefault(name, []).append(float(duration_ms)) + denoise_step_samples.extend( + float(step["duration_ms"]) + for step in dump.get("denoise_steps_ms", []) + if step.get("duration_ms") is not None + ) + + summary = { + "server_latency_samples_s": server_latency_samples_s, + "server_latency_s": round(statistics.median(server_latency_samples_s), 3), + "server_stage_medians_ms": { + name: round(statistics.median(values), 3) + for name, values in sorted(stage_samples.items()) + }, + } + if denoise_step_samples: + summary["median_denoise_step_ms"] = round( + statistics.median(denoise_step_samples), 3 + ) + return summary + + def run_single( case: dict, framework: str, @@ -737,6 +758,7 @@ def run_single( port: int, log_dir: Path, config: dict | None = None, + measurement_repeats: int = 1, ) -> dict: """Run a single (case, framework) combination. Returns result dict.""" result = { @@ -745,6 +767,8 @@ def run_single( "model": case["model"], "task": case["task"], "latency_s": None, + "latency_samples_s": [], + "measurement_count": 0, "error": None, } @@ -754,11 +778,6 @@ def run_single( env = os.environ.copy() env.update(fw_cfg.get("extra_env", {})) - # perf_dump_path for SGLang server-side timing (passed in request, zero overhead when None) - perf_dump_path = None - if framework == "sglang": - perf_dump_path = os.path.join(str(log_dir), f"perf_{case['id']}_measured.json") - log_file = log_dir / f"{case['id']}_{framework}.log" log_fh = open(log_file, "w", encoding="utf-8", buffering=1) log_thread = None @@ -811,26 +830,54 @@ def run_single( base_url = f"http://{DEFAULT_HOST}:{port}" wait_for_health(base_url, framework) - # No client-side warmup: each framework relies on its own server-side - # warmup before traffic. sglang's serve_args pass --warmup-mode server, - # which primes kernels with a synthetic request at startup, before the - # health check passes. This goes through the internal - # warmup path that bypasses sampling-param preset validation (e.g. - # Ideogram-4's preset-locked num_inference_steps), so no per-case warmup - # special-casing is needed here. + # SGLang server warmup uses the measured shape added by + # _build_sglang_cmd. The repeated requests below absorb any remaining + # request-path cold effects, including image-conditioned preprocessing. # NOTE: vllm-omni / lightx2v configure no server-side warmup; if # cross-framework comparison is restored, they must add their own warmup # to stay on equal footing — otherwise their measured request pays the # full cold-start. - # Measured request — pass perf_dump_path for SGLang server-side timing - if perf_dump_path and os.path.exists(perf_dump_path): - os.remove(perf_dump_path) - print(" Sending measured request...") - latency = send_request( - base_url, case, framework, config, perf_dump_path=perf_dump_path - ) - result["latency_s"] = round(latency, 3) + latency_samples: list[float] = [] + perf_dumps: list[dict] = [] + for sample_index in range(measurement_repeats): + perf_dump_path = None + if framework == "sglang": + sample_suffix = "" if sample_index == 0 else f"_{sample_index + 1}" + perf_dump_path = str( + ( + log_dir / f"perf_{case['id']}_measured{sample_suffix}.json" + ).resolve() + ) + if os.path.exists(perf_dump_path): + os.remove(perf_dump_path) + + print( + f" Sending measured request {sample_index + 1}/" + f"{measurement_repeats}..." + ) + latency = send_request( + base_url, case, framework, config, perf_dump_path=perf_dump_path + ) + latency_samples.append(round(latency, 3)) + + if perf_dump_path: + perf_dump = _read_perf_dump(perf_dump_path) + if perf_dump is None: + raise RuntimeError( + f"Server did not write performance data to {perf_dump_path}" + ) + perf_dumps.append(perf_dump) + print( + " Server-side latency: " + f"{perf_dump['total_duration_ms'] / 1000.0:.2f}s" + ) + + result["latency_samples_s"] = latency_samples + result["measurement_count"] = len(latency_samples) + result["latency_s"] = round(statistics.median(latency_samples), 3) + if perf_dumps: + result.update(_summarize_perf_dumps(perf_dumps)) except Exception as e: result["error"] = server_error.get("message", str(e)) @@ -889,6 +936,7 @@ def run_comparison( port: int = DEFAULT_PORT, output: str = "comparison-results.json", dry_run: bool = False, + measurement_repeats: int | None = None, ) -> dict: """Run all comparison cases, grouped by framework to minimize installs. @@ -898,6 +946,13 @@ def run_comparison( timestamp = datetime.now(timezone.utc).isoformat() commit_sha = _get_checkout_commit_sha() run_id = os.environ.get("GITHUB_RUN_ID", "local") + repeats = ( + measurement_repeats + if measurement_repeats is not None + else int(config.get("measurement_repeats", 1)) + ) + if repeats <= 0: + raise ValueError("measurement_repeats must be a positive integer") log_dir = Path("comparison-logs") log_dir.mkdir(exist_ok=True) @@ -962,7 +1017,15 @@ def run_comparison( ) continue - result = run_single(case, fw_name, fw_cfg, port, log_dir, config) + result = run_single( + case, + fw_name, + fw_cfg, + port, + log_dir, + config, + measurement_repeats=repeats, + ) results.append(result) # Wait for GPU memory to clear @@ -973,6 +1036,7 @@ def run_comparison( "timestamp": timestamp, "commit_sha": commit_sha, "run_id": run_id, + "measurement_repeats": repeats, "results": results, } @@ -986,7 +1050,11 @@ def run_comparison( print("SUMMARY") print(f"{'=' * 60}") for r in results: - lat = f"{r['latency_s']:.2f}s" if r["latency_s"] else r.get("error", "N/A") + lat = ( + f"{r['latency_s']:.2f}s median (n={r.get('measurement_count', 1)})" + if r["latency_s"] + else r.get("error", "N/A") + ) print(f" {r['case_id']:30s} | {r['framework']:12s} | {lat}") return output_data @@ -1034,6 +1102,12 @@ def main(): action="store_true", help="Parse config and print commands without launching servers", ) + parser.add_argument( + "--measurement-repeats", + type=int, + default=None, + help="Measured requests per case (default: value from config)", + ) args = parser.parse_args() @@ -1049,6 +1123,7 @@ def main(): port=args.port, output=args.output, dry_run=args.dry_run, + measurement_repeats=args.measurement_repeats, ) # Exit with non-zero if any case had an error