[diffusion] chore: make nightly performance measurements robust (#37915)
This commit is contained in:
Mick
@@ -424,6 +424,7 @@ async def edits(
|
||||
enable_upscaling: Optional[bool] = Form(False),
|
||||
upscaling_model_path: Optional[str] = Form(None),
|
||||
upscaling_scale: Optional[int] = Form(4),
|
||||
perf_dump_path: Optional[str] = Form(None),
|
||||
num_frames: int = Form(1),
|
||||
):
|
||||
request_id = generate_request_id()
|
||||
@@ -484,6 +485,7 @@ async def edits(
|
||||
enable_upscaling=enable_upscaling,
|
||||
upscaling_model_path=upscaling_model_path,
|
||||
upscaling_scale=upscaling_scale,
|
||||
perf_dump_path=perf_dump_path,
|
||||
)
|
||||
trace_headers = extract_trace_headers(raw_request.headers)
|
||||
batch = prepare_request(
|
||||
|
||||
@@ -494,6 +494,7 @@ async def create_video(
|
||||
output_quality: Optional[str] = Form(None),
|
||||
output_compression: Optional[int] = Form(None),
|
||||
output_path: Optional[str] = Form(None),
|
||||
perf_dump_path: Optional[str] = Form(None),
|
||||
extra_params: Optional[str] = Form(None),
|
||||
extra_body: Optional[str] = Form(None),
|
||||
):
|
||||
@@ -645,6 +646,7 @@ async def create_video(
|
||||
output_compression=form_value("output_compression", output_compression),
|
||||
output_quality=form_value("output_quality", output_quality),
|
||||
output_path=form_value("output_path", output_path),
|
||||
perf_dump_path=form_value("perf_dump_path", perf_dump_path),
|
||||
diffusers_kwargs=form_value("diffusers_kwargs", None),
|
||||
**extra_request_fields,
|
||||
)
|
||||
|
||||
@@ -0,0 +1,153 @@
|
||||
import importlib.util
|
||||
from pathlib import Path
|
||||
|
||||
REPO_ROOT = Path(__file__).resolve().parents[5]
|
||||
|
||||
|
||||
def _load_script(name: str, relative_path: str):
|
||||
spec = importlib.util.spec_from_file_location(name, REPO_ROOT / relative_path)
|
||||
assert spec is not None and spec.loader is not None
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(module)
|
||||
return module
|
||||
|
||||
|
||||
runner = _load_script(
|
||||
"diffusion_nightly_runner",
|
||||
"scripts/ci/utils/diffusion/run_comparison.py",
|
||||
)
|
||||
dashboard = _load_script(
|
||||
"diffusion_nightly_dashboard",
|
||||
"scripts/ci/utils/diffusion/generate_diffusion_dashboard.py",
|
||||
)
|
||||
|
||||
|
||||
def test_sglang_server_warmup_matches_measured_shape():
|
||||
case = {
|
||||
"model": "example/model",
|
||||
"num_gpus": 2,
|
||||
"width": 768,
|
||||
"height": 512,
|
||||
"num_frames": 121,
|
||||
}
|
||||
command = runner._build_sglang_cmd(
|
||||
case,
|
||||
{"serve_args": "--warmup-mode server --tp-size 2"},
|
||||
30000,
|
||||
)
|
||||
|
||||
resolution_index = command.index("--warmup-resolutions")
|
||||
frame_index = command.index("--warmup-num-frames")
|
||||
assert command[resolution_index + 1] == "768x512"
|
||||
assert command[frame_index + 1] == "121"
|
||||
|
||||
|
||||
def test_explicit_server_warmup_shape_is_preserved():
|
||||
case = {
|
||||
"model": "example/model",
|
||||
"num_gpus": 1,
|
||||
"width": 1024,
|
||||
"height": 1024,
|
||||
"num_frames": 81,
|
||||
}
|
||||
command = runner._build_sglang_cmd(
|
||||
case,
|
||||
{
|
||||
"serve_args": (
|
||||
"--warmup-mode server --warmup-resolutions 512x512 "
|
||||
"--warmup-num-frames 25"
|
||||
)
|
||||
},
|
||||
30000,
|
||||
)
|
||||
|
||||
assert command.count("--warmup-resolutions") == 1
|
||||
assert command.count("--warmup-num-frames") == 1
|
||||
assert command[command.index("--warmup-resolutions") + 1] == "512x512"
|
||||
assert command[command.index("--warmup-num-frames") + 1] == "25"
|
||||
|
||||
|
||||
def test_perf_dump_summary_uses_medians():
|
||||
perf_dumps = [
|
||||
{
|
||||
"total_duration_ms": 1000.0,
|
||||
"steps": [
|
||||
{"name": "TextEncodingStage", "duration_ms": 100.0},
|
||||
{"name": "DenoisingStage", "duration_ms": 800.0},
|
||||
],
|
||||
"denoise_steps_ms": [{"duration_ms": 8.0}, {"duration_ms": 10.0}],
|
||||
},
|
||||
{
|
||||
"total_duration_ms": 3000.0,
|
||||
"steps": [
|
||||
{"name": "TextEncodingStage", "duration_ms": 300.0},
|
||||
{"name": "DenoisingStage", "duration_ms": 2400.0},
|
||||
],
|
||||
"denoise_steps_ms": [{"duration_ms": 30.0}],
|
||||
},
|
||||
{
|
||||
"total_duration_ms": 1100.0,
|
||||
"steps": [
|
||||
{"name": "TextEncodingStage", "duration_ms": 110.0},
|
||||
{"name": "DenoisingStage", "duration_ms": 880.0},
|
||||
],
|
||||
"denoise_steps_ms": [{"duration_ms": 11.0}],
|
||||
},
|
||||
]
|
||||
|
||||
summary = runner._summarize_perf_dumps(perf_dumps)
|
||||
|
||||
assert summary["server_latency_s"] == 1.1
|
||||
assert summary["server_stage_medians_ms"] == {
|
||||
"DenoisingStage": 880.0,
|
||||
"TextEncodingStage": 110.0,
|
||||
}
|
||||
assert summary["median_denoise_step_ms"] == 10.5
|
||||
|
||||
|
||||
def test_dashboard_uses_historical_median_and_shows_server_breakdown():
|
||||
current = {
|
||||
"timestamp": "2026-09-04T00:00:00+00:00",
|
||||
"commit_sha": "abcdef123456",
|
||||
"results": [
|
||||
{
|
||||
"case_id": "example",
|
||||
"framework": "sglang",
|
||||
"model": "example/model",
|
||||
"latency_s": 10.4,
|
||||
"latency_samples_s": [10.3, 10.4, 10.5],
|
||||
"measurement_count": 3,
|
||||
"server_latency_s": 10.0,
|
||||
"server_stage_medians_ms": {
|
||||
"TextEncodingStage": 100.0,
|
||||
"DenoisingStage": 9800.0,
|
||||
"DecodingStage": 100.0,
|
||||
},
|
||||
"median_denoise_step_ms": 196.0,
|
||||
}
|
||||
],
|
||||
}
|
||||
history = [
|
||||
{
|
||||
"results": [
|
||||
{
|
||||
"case_id": "example",
|
||||
"framework": "sglang",
|
||||
"latency_s": value,
|
||||
}
|
||||
]
|
||||
}
|
||||
for value in (10.0, 30.0, 9.8)
|
||||
]
|
||||
|
||||
baseline, count = dashboard._historical_latency_baseline(
|
||||
"example", "sglang", history
|
||||
)
|
||||
markdown, alerts = dashboard.generate_dashboard(current, history)
|
||||
|
||||
assert baseline == 10.0
|
||||
assert count == 3
|
||||
assert alerts == []
|
||||
assert "| 3 | **10.40** |" in markdown
|
||||
assert "## SGLang Server-Side Breakdown" in markdown
|
||||
assert "| model | 10.00 | 0.10 | 9.80 | 0.10 | 196.00 |" in markdown
|
||||
@@ -1,3 +1,4 @@
|
||||
import inspect
|
||||
import os
|
||||
from dataclasses import fields
|
||||
|
||||
@@ -23,6 +24,7 @@ from sglang.multimodal_gen.runtime.entrypoints.openai.image_api import (
|
||||
_runtime_sampling_quality,
|
||||
_select_image_variant_cloud_url,
|
||||
_select_image_variant_path,
|
||||
edits,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.entrypoints.openai.protocol import (
|
||||
ImageGenerationsRequest,
|
||||
@@ -30,6 +32,10 @@ from sglang.multimodal_gen.runtime.entrypoints.openai.protocol import (
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.schedule_batch import OutputBatch
|
||||
|
||||
|
||||
def test_image_edits_declares_perf_dump_path_form_field():
|
||||
assert "perf_dump_path" in inspect.signature(edits).parameters
|
||||
|
||||
|
||||
def test_url_response_returns_one_item_per_output_path():
|
||||
paths = ["first.png", "second.png"]
|
||||
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import inspect
|
||||
from dataclasses import fields
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import patch
|
||||
@@ -16,10 +17,15 @@ from sglang.multimodal_gen.runtime.entrypoints.openai.realtime.realtime_adapter
|
||||
from sglang.multimodal_gen.runtime.entrypoints.openai.video_api import (
|
||||
_build_video_sampling_params,
|
||||
_video_request_model_kwargs,
|
||||
create_video,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.schedule_batch import Req
|
||||
|
||||
|
||||
def test_multipart_video_declares_perf_dump_path_form_field():
|
||||
assert "perf_dump_path" in inspect.signature(create_video).parameters
|
||||
|
||||
|
||||
def test_video_api_forwards_profiling_options():
|
||||
request = VideoGenerationsRequest(
|
||||
prompt="profile this request",
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
{
|
||||
"_comment": "Per-model comparison config. Sampling params omitted where model defaults are correct — only override resolution, seed, and params that differ from defaults.",
|
||||
"measurement_repeats": 3,
|
||||
"test_image_url": "https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/diffusers/cat.png",
|
||||
"cases": [
|
||||
{
|
||||
|
||||
@@ -15,6 +15,7 @@ Usage:
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import statistics
|
||||
from datetime import datetime, timezone
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
@@ -26,6 +27,8 @@ CI_DATA_REPO_NAME = "ci-data-diffusion"
|
||||
CI_DATA_BRANCH = "main"
|
||||
HISTORY_PREFIX = "diffusion-comparisons"
|
||||
MAX_HISTORY_RUNS = 29
|
||||
HISTORICAL_BASELINE_RUNS = 5
|
||||
REGRESSION_THRESHOLD = 0.05
|
||||
|
||||
# Base URL for chart images pushed to sgl-project/ci-data-diffusion
|
||||
CHARTS_RAW_BASE_URL = (
|
||||
@@ -136,6 +139,19 @@ def _short_sha(sha: str) -> str:
|
||||
return sha[:7] if sha and sha != "unknown" else "?"
|
||||
|
||||
|
||||
def _historical_latency_baseline(
|
||||
cid: str, framework: str, history: list[dict]
|
||||
) -> tuple[float | None, int]:
|
||||
values = []
|
||||
for run in history[:HISTORICAL_BASELINE_RUNS]:
|
||||
latency = _extract_case_results(run).get(cid, {}).get(framework)
|
||||
if latency is not None:
|
||||
values.append(latency)
|
||||
if not values:
|
||||
return None, 0
|
||||
return statistics.median(values), len(values)
|
||||
|
||||
|
||||
def _assess_risk(
|
||||
cid: str,
|
||||
current_cases: dict[str, dict[str, float | None]],
|
||||
@@ -146,7 +162,7 @@ def _assess_risk(
|
||||
|
||||
Rules (checked in order):
|
||||
- N/A latency → ❌ broken
|
||||
- History exists: SGLang latency >5% vs avg of last 3 runs → ⚠️ regression
|
||||
- History exists: SGLang latency >5% vs median of recent runs → ⚠️ regression
|
||||
- Competitor exists & SGLang slower → 🔴 competitive risk
|
||||
- SGLang faster than all competitors by >20% → 🟢 strong advantage
|
||||
- SGLang faster than all competitors by ≤20% → 🟡 moderate advantage
|
||||
@@ -158,23 +174,18 @@ def _assess_risk(
|
||||
if sg_lat is None:
|
||||
return "❌", f"{cid}: SGLang latency is N/A (broken)"
|
||||
|
||||
# Check regression against 3-run historical average
|
||||
if history:
|
||||
hist_lats: list[float] = []
|
||||
for run in history[:3]:
|
||||
run_cases = _extract_case_results(run)
|
||||
h_lat = run_cases.get(cid, {}).get("sglang")
|
||||
if h_lat is not None:
|
||||
hist_lats.append(h_lat)
|
||||
if hist_lats:
|
||||
avg_3 = sum(hist_lats) / len(hist_lats)
|
||||
if avg_3 > 0 and (sg_lat - avg_3) / avg_3 > 0.05:
|
||||
pct = (sg_lat - avg_3) / avg_3 * 100
|
||||
return (
|
||||
"⚠️",
|
||||
f"{cid}: SGLang regression +{pct:.1f}% vs 3-run avg "
|
||||
f"({sg_lat:.2f}s vs {avg_3:.2f}s)",
|
||||
)
|
||||
baseline, baseline_runs = _historical_latency_baseline(cid, "sglang", history)
|
||||
if (
|
||||
baseline is not None
|
||||
and baseline > 0
|
||||
and (sg_lat - baseline) / baseline > REGRESSION_THRESHOLD
|
||||
):
|
||||
pct = (sg_lat - baseline) / baseline * 100
|
||||
return (
|
||||
"⚠️",
|
||||
f"{cid}: SGLang regression +{pct:.1f}% vs {baseline_runs}-run "
|
||||
f"median ({sg_lat:.2f}s vs {baseline:.2f}s)",
|
||||
)
|
||||
|
||||
# Check competitive risk
|
||||
if other_frameworks:
|
||||
@@ -229,6 +240,24 @@ def _extract_case_results(run_data: dict) -> dict[str, dict[str, float | None]]:
|
||||
return mapping
|
||||
|
||||
|
||||
def _extract_case_records(run_data: dict) -> dict[str, dict[str, dict]]:
|
||||
"""Extract {case_id: {framework: result record}} from a run."""
|
||||
mapping: dict[str, dict[str, dict]] = {}
|
||||
for result in run_data.get("results", []):
|
||||
mapping.setdefault(result["case_id"], {})[result["framework"]] = result
|
||||
return mapping
|
||||
|
||||
|
||||
def _stage_group_seconds(result: dict, suffixes: str | tuple[str, ...]) -> float | None:
|
||||
if isinstance(suffixes, str):
|
||||
suffixes = (suffixes,)
|
||||
stages = result.get("server_stage_medians_ms") or {}
|
||||
values = [value for name, value in stages.items() if name.endswith(suffixes)]
|
||||
if not values:
|
||||
return None
|
||||
return sum(values) / 1000.0
|
||||
|
||||
|
||||
def _sanitize_filename(name: str) -> str:
|
||||
"""Sanitize a case ID to be a safe filename."""
|
||||
return name.replace("/", "_").replace(" ", "_").replace(":", "_")
|
||||
@@ -257,22 +286,22 @@ def generate_dashboard(
|
||||
lines.append(f"*Generated: {_short_date(ts)} | Commit: `{_short_sha(sha)}`*\n")
|
||||
|
||||
current_cases = _extract_case_results(current)
|
||||
current_records = _extract_case_records(current)
|
||||
case_ids = list(current_cases.keys())
|
||||
|
||||
# ---- Regression detection ----
|
||||
REGRESSION_THRESHOLD = 0.05 # 5%
|
||||
regressions: list[str] = []
|
||||
if history:
|
||||
prev_cases = _extract_case_results(history[0])
|
||||
for cid in case_ids:
|
||||
for fw in ("sglang", "vllm-omni"):
|
||||
cur = current_cases.get(cid, {}).get(fw)
|
||||
prev = prev_cases.get(cid, {}).get(fw)
|
||||
if cur and prev and prev > 0:
|
||||
pct = (cur - prev) / prev
|
||||
baseline, baseline_runs = _historical_latency_baseline(cid, fw, history)
|
||||
if cur is not None and baseline is not None and baseline > 0:
|
||||
pct = (cur - baseline) / baseline
|
||||
if pct > REGRESSION_THRESHOLD:
|
||||
regressions.append(
|
||||
f"**{cid}** ({fw}): {prev:.2f}s -> {cur:.2f}s "
|
||||
f"**{cid}** ({fw}): {cur:.2f}s vs "
|
||||
f"{baseline_runs}-run median {baseline:.2f}s "
|
||||
f"(+{pct * 100:.1f}%)"
|
||||
)
|
||||
|
||||
@@ -305,10 +334,10 @@ def generate_dashboard(
|
||||
risk_map[cid] = _assess_risk(cid, current_cases, history, other_frameworks)
|
||||
|
||||
# Dynamic header
|
||||
header = "| Model | Risk |"
|
||||
sep = "|-------|------|"
|
||||
header = "| Model | Risk | Samples |"
|
||||
sep = "|-------|------|---------|"
|
||||
for fw in all_frameworks:
|
||||
header += f" {fw} (s) |"
|
||||
header += f" {fw} median (s) |"
|
||||
sep += "---------|"
|
||||
for ofw in other_frameworks:
|
||||
header += f" vs {ofw} |"
|
||||
@@ -326,9 +355,15 @@ def generate_dashboard(
|
||||
|
||||
case_fws = current_cases.get(cid, {})
|
||||
sg_lat = case_fws.get("sglang")
|
||||
sg_record = current_records.get(cid, {}).get("sglang", {})
|
||||
sample_count = sg_record.get("measurement_count")
|
||||
if not sample_count and sg_lat is not None:
|
||||
sample_count = 1
|
||||
|
||||
risk_emoji, _ = risk_map.get(cid, ("✅", ""))
|
||||
row = f"| {r['model'].split('/')[-1]} | {risk_emoji} |"
|
||||
row = (
|
||||
f"| {r['model'].split('/')[-1]} | {risk_emoji} | {sample_count or 'N/A'} |"
|
||||
)
|
||||
# Latency columns -- bold the fastest
|
||||
lats = {fw: case_fws.get(fw) for fw in all_frameworks}
|
||||
valid_lats = [v for v in lats.values() if v is not None]
|
||||
@@ -344,6 +379,38 @@ def generate_dashboard(
|
||||
row += f" {_fmt_speedup(sg_lat, case_fws.get(ofw))} |"
|
||||
lines.append(row)
|
||||
|
||||
server_records = [
|
||||
current_records.get(cid, {}).get("sglang", {}) for cid in case_ids
|
||||
]
|
||||
if any(record.get("server_latency_s") is not None for record in server_records):
|
||||
lines.append("\n## SGLang Server-Side Breakdown\n")
|
||||
lines.append(
|
||||
"| Model | Server total (s) | Text encode (s) | Denoise (s) | "
|
||||
"Decode (s) | Median denoise step (ms) |"
|
||||
)
|
||||
lines.append(
|
||||
"|-------|------------------|-----------------|--------------|"
|
||||
"------------|---------------------------|"
|
||||
)
|
||||
for record in server_records:
|
||||
if record.get("server_latency_s") is None:
|
||||
continue
|
||||
model = record["model"].split("/")[-1]
|
||||
text_encode = _stage_group_seconds(
|
||||
record, ("TextEncodingStage", "TokenizationStage")
|
||||
)
|
||||
denoise = _stage_group_seconds(record, "DenoisingStage")
|
||||
decode = _stage_group_seconds(record, "DecodingStage")
|
||||
denoise_step = record.get("median_denoise_step_ms")
|
||||
denoise_step_text = (
|
||||
f"{denoise_step:.2f}" if denoise_step is not None else "N/A"
|
||||
)
|
||||
lines.append(
|
||||
f"| {model} | {_fmt_latency(record['server_latency_s'])} | "
|
||||
f"{_fmt_latency(text_encode)} | {_fmt_latency(denoise)} | "
|
||||
f"{_fmt_latency(decode)} | {denoise_step_text} |"
|
||||
)
|
||||
|
||||
# ---- Section 2: Speedup-over-time vs. other frameworks (rendered only when present) ----
|
||||
if history and other_frameworks:
|
||||
lines.append("\n## SGLang vs vLLM-Omni Speedup Over Time\n")
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
"""Diffusion serving benchmark for SGLang-Diffusion nightly CI.
|
||||
|
||||
Launches an SGLang-Diffusion server for each test case, sends a single
|
||||
request, measures end-to-end latency, and writes comparison-results.json.
|
||||
Launches an SGLang-Diffusion server for each test case, sends repeated
|
||||
requests, measures median end-to-end latency, and writes comparison-results.json.
|
||||
The runner still supports extra frameworks via --frameworks, but the nightly
|
||||
config tracks SGLang-Diffusion only.
|
||||
|
||||
@@ -24,8 +24,10 @@ import base64
|
||||
import io
|
||||
import json
|
||||
import os
|
||||
import shlex
|
||||
import signal
|
||||
import socket
|
||||
import statistics
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
@@ -84,12 +86,29 @@ def _build_sglang_cmd(case: dict, fw_cfg: dict, port: int) -> list[str]:
|
||||
]
|
||||
if case["num_gpus"] > 1:
|
||||
cmd += ["--num-gpus", str(case["num_gpus"])]
|
||||
if fw_cfg.get("serve_args", "").strip():
|
||||
cmd += fw_cfg["serve_args"].strip().split()
|
||||
# No explicit --warmup-resolutions: server-based warmup now defaults to the
|
||||
# model's sampling-default resolution (see warmup_request_builder), which
|
||||
# already matches these single-resolution cases — the default warmup is
|
||||
# sufficient, so we don't pin a resolution here.
|
||||
serve_args = shlex.split(fw_cfg.get("serve_args", ""))
|
||||
cmd += serve_args
|
||||
|
||||
def has_option(name: str) -> bool:
|
||||
return any(arg == name or arg.startswith(f"{name}=") for arg in serve_args)
|
||||
|
||||
server_warmup = any(
|
||||
arg == "--warmup-mode=server"
|
||||
or (
|
||||
arg == "--warmup-mode"
|
||||
and index + 1 < len(serve_args)
|
||||
and serve_args[index + 1] == "server"
|
||||
)
|
||||
for index, arg in enumerate(serve_args)
|
||||
)
|
||||
if server_warmup and not has_option("--warmup-resolutions"):
|
||||
cmd += ["--warmup-resolutions", f"{case['width']}x{case['height']}"]
|
||||
if (
|
||||
server_warmup
|
||||
and case.get("num_frames") is not None
|
||||
and not has_option("--warmup-num-frames")
|
||||
):
|
||||
cmd += ["--warmup-num-frames", str(case["num_frames"])]
|
||||
return cmd
|
||||
|
||||
|
||||
@@ -375,8 +394,8 @@ def _build_sglang_payload(case: dict) -> dict:
|
||||
return payload
|
||||
|
||||
|
||||
def _read_perf_dump(perf_dump_path: str, timeout: float = 10.0) -> float | None:
|
||||
"""Read total_duration_ms from a perf dump JSON written by the server.
|
||||
def _read_perf_dump(perf_dump_path: str, timeout: float = 10.0) -> dict | None:
|
||||
"""Read a perf dump JSON written by the server.
|
||||
|
||||
The server writes the file asynchronously after the HTTP response,
|
||||
so we poll briefly.
|
||||
@@ -386,9 +405,8 @@ def _read_perf_dump(perf_dump_path: str, timeout: float = 10.0) -> float | None:
|
||||
try:
|
||||
with open(perf_dump_path) as f:
|
||||
data = json.load(f)
|
||||
total_ms = data.get("total_duration_ms")
|
||||
if total_ms is not None:
|
||||
return total_ms / 1000.0
|
||||
if data.get("total_duration_ms") is not None:
|
||||
return data
|
||||
except (FileNotFoundError, json.JSONDecodeError):
|
||||
pass
|
||||
time.sleep(0.5)
|
||||
@@ -415,16 +433,6 @@ def send_image_request_sglang(
|
||||
if "data" not in data or len(data["data"]) == 0:
|
||||
raise RuntimeError(f"Image request returned no data: {data}")
|
||||
|
||||
# Report client-side e2e latency to match vllm-omni / lightx2v (fair
|
||||
# cross-framework comparison); server-side perf_dump is diagnostic only.
|
||||
if perf_dump_path:
|
||||
server_latency = _read_perf_dump(perf_dump_path)
|
||||
if server_latency is not None:
|
||||
print(
|
||||
f" Image generated in {client_latency:.2f}s (client e2e; "
|
||||
f"server-side {server_latency:.2f}s, diagnostic)"
|
||||
)
|
||||
return client_latency
|
||||
print(f" Image generated in {client_latency:.2f}s")
|
||||
return client_latency
|
||||
|
||||
@@ -468,16 +476,6 @@ def send_video_request_sglang(
|
||||
|
||||
client_latency = time.time() - start
|
||||
|
||||
# Report client-side e2e latency to match vllm-omni / lightx2v (fair
|
||||
# cross-framework comparison); server-side perf_dump is diagnostic only.
|
||||
if perf_dump_path:
|
||||
server_latency = _read_perf_dump(perf_dump_path)
|
||||
if server_latency is not None:
|
||||
print(
|
||||
f" Video generated in {client_latency:.2f}s (client e2e; "
|
||||
f"server-side {server_latency:.2f}s, diagnostic)"
|
||||
)
|
||||
return client_latency
|
||||
print(f" Video generated in {client_latency:.2f}s")
|
||||
return client_latency
|
||||
|
||||
@@ -556,16 +554,6 @@ def send_image_conditioned_request_sglang(
|
||||
|
||||
client_latency = time.time() - start
|
||||
|
||||
# Report client-side e2e latency to match vllm-omni / lightx2v (fair
|
||||
# cross-framework comparison); server-side perf_dump is diagnostic only.
|
||||
if perf_dump_path:
|
||||
server_latency = _read_perf_dump(perf_dump_path)
|
||||
if server_latency is not None:
|
||||
print(
|
||||
f" Generated in {client_latency:.2f}s (client e2e; "
|
||||
f"server-side {server_latency:.2f}s, diagnostic)"
|
||||
)
|
||||
return client_latency
|
||||
print(f" Generated in {client_latency:.2f}s (sglang, image-conditioned)")
|
||||
return client_latency
|
||||
|
||||
@@ -730,6 +718,39 @@ def send_request(
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _summarize_perf_dumps(perf_dumps: list[dict]) -> dict:
|
||||
server_latency_samples_s = [
|
||||
round(dump["total_duration_ms"] / 1000.0, 3) for dump in perf_dumps
|
||||
]
|
||||
stage_samples: dict[str, list[float]] = {}
|
||||
denoise_step_samples: list[float] = []
|
||||
for dump in perf_dumps:
|
||||
for stage in dump.get("steps", []):
|
||||
name = stage.get("name")
|
||||
duration_ms = stage.get("duration_ms")
|
||||
if name and duration_ms is not None:
|
||||
stage_samples.setdefault(name, []).append(float(duration_ms))
|
||||
denoise_step_samples.extend(
|
||||
float(step["duration_ms"])
|
||||
for step in dump.get("denoise_steps_ms", [])
|
||||
if step.get("duration_ms") is not None
|
||||
)
|
||||
|
||||
summary = {
|
||||
"server_latency_samples_s": server_latency_samples_s,
|
||||
"server_latency_s": round(statistics.median(server_latency_samples_s), 3),
|
||||
"server_stage_medians_ms": {
|
||||
name: round(statistics.median(values), 3)
|
||||
for name, values in sorted(stage_samples.items())
|
||||
},
|
||||
}
|
||||
if denoise_step_samples:
|
||||
summary["median_denoise_step_ms"] = round(
|
||||
statistics.median(denoise_step_samples), 3
|
||||
)
|
||||
return summary
|
||||
|
||||
|
||||
def run_single(
|
||||
case: dict,
|
||||
framework: str,
|
||||
@@ -737,6 +758,7 @@ def run_single(
|
||||
port: int,
|
||||
log_dir: Path,
|
||||
config: dict | None = None,
|
||||
measurement_repeats: int = 1,
|
||||
) -> dict:
|
||||
"""Run a single (case, framework) combination. Returns result dict."""
|
||||
result = {
|
||||
@@ -745,6 +767,8 @@ def run_single(
|
||||
"model": case["model"],
|
||||
"task": case["task"],
|
||||
"latency_s": None,
|
||||
"latency_samples_s": [],
|
||||
"measurement_count": 0,
|
||||
"error": None,
|
||||
}
|
||||
|
||||
@@ -754,11 +778,6 @@ def run_single(
|
||||
env = os.environ.copy()
|
||||
env.update(fw_cfg.get("extra_env", {}))
|
||||
|
||||
# perf_dump_path for SGLang server-side timing (passed in request, zero overhead when None)
|
||||
perf_dump_path = None
|
||||
if framework == "sglang":
|
||||
perf_dump_path = os.path.join(str(log_dir), f"perf_{case['id']}_measured.json")
|
||||
|
||||
log_file = log_dir / f"{case['id']}_{framework}.log"
|
||||
log_fh = open(log_file, "w", encoding="utf-8", buffering=1)
|
||||
log_thread = None
|
||||
@@ -811,26 +830,54 @@ def run_single(
|
||||
base_url = f"http://{DEFAULT_HOST}:{port}"
|
||||
wait_for_health(base_url, framework)
|
||||
|
||||
# No client-side warmup: each framework relies on its own server-side
|
||||
# warmup before traffic. sglang's serve_args pass --warmup-mode server,
|
||||
# which primes kernels with a synthetic request at startup, before the
|
||||
# health check passes. This goes through the internal
|
||||
# warmup path that bypasses sampling-param preset validation (e.g.
|
||||
# Ideogram-4's preset-locked num_inference_steps), so no per-case warmup
|
||||
# special-casing is needed here.
|
||||
# SGLang server warmup uses the measured shape added by
|
||||
# _build_sglang_cmd. The repeated requests below absorb any remaining
|
||||
# request-path cold effects, including image-conditioned preprocessing.
|
||||
# NOTE: vllm-omni / lightx2v configure no server-side warmup; if
|
||||
# cross-framework comparison is restored, they must add their own warmup
|
||||
# to stay on equal footing — otherwise their measured request pays the
|
||||
# full cold-start.
|
||||
|
||||
# Measured request — pass perf_dump_path for SGLang server-side timing
|
||||
if perf_dump_path and os.path.exists(perf_dump_path):
|
||||
os.remove(perf_dump_path)
|
||||
print(" Sending measured request...")
|
||||
latency = send_request(
|
||||
base_url, case, framework, config, perf_dump_path=perf_dump_path
|
||||
)
|
||||
result["latency_s"] = round(latency, 3)
|
||||
latency_samples: list[float] = []
|
||||
perf_dumps: list[dict] = []
|
||||
for sample_index in range(measurement_repeats):
|
||||
perf_dump_path = None
|
||||
if framework == "sglang":
|
||||
sample_suffix = "" if sample_index == 0 else f"_{sample_index + 1}"
|
||||
perf_dump_path = str(
|
||||
(
|
||||
log_dir / f"perf_{case['id']}_measured{sample_suffix}.json"
|
||||
).resolve()
|
||||
)
|
||||
if os.path.exists(perf_dump_path):
|
||||
os.remove(perf_dump_path)
|
||||
|
||||
print(
|
||||
f" Sending measured request {sample_index + 1}/"
|
||||
f"{measurement_repeats}..."
|
||||
)
|
||||
latency = send_request(
|
||||
base_url, case, framework, config, perf_dump_path=perf_dump_path
|
||||
)
|
||||
latency_samples.append(round(latency, 3))
|
||||
|
||||
if perf_dump_path:
|
||||
perf_dump = _read_perf_dump(perf_dump_path)
|
||||
if perf_dump is None:
|
||||
raise RuntimeError(
|
||||
f"Server did not write performance data to {perf_dump_path}"
|
||||
)
|
||||
perf_dumps.append(perf_dump)
|
||||
print(
|
||||
" Server-side latency: "
|
||||
f"{perf_dump['total_duration_ms'] / 1000.0:.2f}s"
|
||||
)
|
||||
|
||||
result["latency_samples_s"] = latency_samples
|
||||
result["measurement_count"] = len(latency_samples)
|
||||
result["latency_s"] = round(statistics.median(latency_samples), 3)
|
||||
if perf_dumps:
|
||||
result.update(_summarize_perf_dumps(perf_dumps))
|
||||
|
||||
except Exception as e:
|
||||
result["error"] = server_error.get("message", str(e))
|
||||
@@ -889,6 +936,7 @@ def run_comparison(
|
||||
port: int = DEFAULT_PORT,
|
||||
output: str = "comparison-results.json",
|
||||
dry_run: bool = False,
|
||||
measurement_repeats: int | None = None,
|
||||
) -> dict:
|
||||
"""Run all comparison cases, grouped by framework to minimize installs.
|
||||
|
||||
@@ -898,6 +946,13 @@ def run_comparison(
|
||||
timestamp = datetime.now(timezone.utc).isoformat()
|
||||
commit_sha = _get_checkout_commit_sha()
|
||||
run_id = os.environ.get("GITHUB_RUN_ID", "local")
|
||||
repeats = (
|
||||
measurement_repeats
|
||||
if measurement_repeats is not None
|
||||
else int(config.get("measurement_repeats", 1))
|
||||
)
|
||||
if repeats <= 0:
|
||||
raise ValueError("measurement_repeats must be a positive integer")
|
||||
|
||||
log_dir = Path("comparison-logs")
|
||||
log_dir.mkdir(exist_ok=True)
|
||||
@@ -962,7 +1017,15 @@ def run_comparison(
|
||||
)
|
||||
continue
|
||||
|
||||
result = run_single(case, fw_name, fw_cfg, port, log_dir, config)
|
||||
result = run_single(
|
||||
case,
|
||||
fw_name,
|
||||
fw_cfg,
|
||||
port,
|
||||
log_dir,
|
||||
config,
|
||||
measurement_repeats=repeats,
|
||||
)
|
||||
results.append(result)
|
||||
|
||||
# Wait for GPU memory to clear
|
||||
@@ -973,6 +1036,7 @@ def run_comparison(
|
||||
"timestamp": timestamp,
|
||||
"commit_sha": commit_sha,
|
||||
"run_id": run_id,
|
||||
"measurement_repeats": repeats,
|
||||
"results": results,
|
||||
}
|
||||
|
||||
@@ -986,7 +1050,11 @@ def run_comparison(
|
||||
print("SUMMARY")
|
||||
print(f"{'=' * 60}")
|
||||
for r in results:
|
||||
lat = f"{r['latency_s']:.2f}s" if r["latency_s"] else r.get("error", "N/A")
|
||||
lat = (
|
||||
f"{r['latency_s']:.2f}s median (n={r.get('measurement_count', 1)})"
|
||||
if r["latency_s"]
|
||||
else r.get("error", "N/A")
|
||||
)
|
||||
print(f" {r['case_id']:30s} | {r['framework']:12s} | {lat}")
|
||||
|
||||
return output_data
|
||||
@@ -1034,6 +1102,12 @@ def main():
|
||||
action="store_true",
|
||||
help="Parse config and print commands without launching servers",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--measurement-repeats",
|
||||
type=int,
|
||||
default=None,
|
||||
help="Measured requests per case (default: value from config)",
|
||||
)
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
@@ -1049,6 +1123,7 @@ def main():
|
||||
port=args.port,
|
||||
output=args.output,
|
||||
dry_run=args.dry_run,
|
||||
measurement_repeats=args.measurement_repeats,
|
||||
)
|
||||
|
||||
# Exit with non-zero if any case had an error
|
||||
|
||||
Reference in New Issue
Block a user