minke.yu/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 12 Packages Projects Releases Wiki Activity

[diffusion] CI: improve diffusion comparison benchmark setting for realistic perf and auto-discover ut (#22086)

Browse Source 
Co-authored-by: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
Mick
2026-04-04 23:20:37 +08:00
committed by GitHub
co-authored by Claude Opus 4.6
parent 0f0f004f1f
commit efee62efa6
10 changed files with 278 additions and 110 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/ci/utils/diffusion/comparison_configs.json
+18 -17
View File
@@ -1,5 +1,5 @@
{
"_comment": "Per-model comparison config. Only frameworks listed under each case are tested. vLLM-Omni disabled until dep install issues resolved.",
"_comment": "Per-model comparison config. Sampling params omitted where model defaults are correct — only override resolution, seed, and params that differ from defaults.",
"test_image_url": "https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/diffusers/cat.png",
"cases": [
{
@@ -9,8 +9,6 @@
"prompt": "A futuristic cyberpunk city at night, neon lights reflecting on wet streets",
"width": 1024,
"height": 1024,
"num_inference_steps": 50,
"guidance_scale": 4.0,
"seed": 42,
"num_gpus": 1,
"frameworks": {
@@ -27,8 +25,6 @@
"prompt": "A futuristic cyberpunk city at night, neon lights reflecting on wet streets",
"width": 1024,
"height": 1024,
"num_inference_steps": 50,
"guidance_scale": 4.0,
"seed": 42,
"num_gpus": 1,
"frameworks": {
@@ -45,8 +41,6 @@
"prompt": "A futuristic cyberpunk city at night, neon lights reflecting on wet streets",
"width": 1024,
"height": 1024,
"num_inference_steps": 50,
"guidance_scale": 4.0,
"seed": 42,
"num_gpus": 1,
"frameworks": {
@@ -64,8 +58,6 @@
"reference_image": true,
"width": 1024,
"height": 1024,
"num_inference_steps": 50,
"guidance_scale": 4.0,
"seed": 42,
"num_gpus": 1,
"frameworks": {
@@ -82,8 +74,6 @@
"prompt": "A futuristic cyberpunk city at night, neon lights reflecting on wet streets",
"width": 1024,
"height": 1024,
"num_inference_steps": 9,
"guidance_scale": 4.0,
"seed": 42,
"num_gpus": 1,
"frameworks": {
@@ -101,8 +91,6 @@
"width": 1280,
"height": 720,
"num_frames": 81,
"num_inference_steps": 2,
"guidance_scale": 5.0,
"seed": 42,
"num_gpus": 4,
"frameworks": {
@@ -121,8 +109,6 @@
"width": 1280,
"height": 720,
"num_frames": 81,
"num_inference_steps": 50,
"guidance_scale": 5.0,
"seed": 42,
"num_gpus": 1,
"frameworks": {
@@ -132,6 +118,23 @@
}
}
},
{
"id": "ltx2_twostage_t2v",
"model": "Lightricks/LTX-2",
"task": "text-to-video",
"prompt": "A cat and a dog baking a cake together in a kitchen.",
"width": 768,
"height": 512,
"num_frames": 121,
"seed": 42,
"num_gpus": 2,
"frameworks": {
"sglang": {
"serve_args": "--enable-torch-compile --warmup --enable-cfg-parallel --pipeline-class-name LTX2TwoStagePipeline",
"extra_env": {}
}
}
},
{
"id": "wan22_i2v_a14b_720p",
"model": "Wan-AI/Wan2.2-I2V-A14B-Diffusers",
@@ -141,8 +144,6 @@
"width": 1280,
"height": 720,
"num_frames": 81,
"num_inference_steps": 2,
"guidance_scale": 5.0,
"seed": 42,
"num_gpus": 4,
"frameworks": {
scripts/ci/utils/diffusion/generate_diffusion_dashboard.py
+208 -44
View File
@@ -239,9 +239,12 @@ def generate_dashboard(
current: dict,
history: list[dict],
charts_dir: str | None = None,
) -> str:
) -> tuple[str, list[str]]:
"""Generate full markdown dashboard.
Returns (markdown_string, alert_reasons) where alert_reasons is a list of
human-readable strings for cases that need attention (empty if all is well).
If charts_dir is provided, saves chart PNGs as files to that directory
and references them via raw.githubusercontent URLs. Otherwise, charts
are omitted.
@@ -342,45 +345,7 @@ def generate_dashboard(
row += f" {_fmt_speedup(sg_lat, case_fws.get(ofw))} |"
lines.append(row)
# ---- Section 2: SGLang Performance Trend ----
if history:
lines.append(f"\n## SGLang Performance Trend (Last {len(history) + 1} Runs)\n")
# Build header
header = "| Date | Commit |"
sep = "|------|--------|"
for cid in case_ids:
header += f" {cid} (s) |"
sep += "---------|"
header += " Trend |"
sep += "-------|"
lines.append(header)
lines.append(sep)
# Current run first
all_runs = [current] + history
for i, run in enumerate(all_runs):
run_cases = _extract_case_results(run)
date = _short_date(run.get("timestamp", ""))
sha_s = _short_sha(run.get("commit_sha", ""))
row = f"| {date} | `{sha_s}` |"
for cid in case_ids:
lat = run_cases.get(cid, {}).get("sglang")
row += f" {_fmt_latency(lat)} |"
# Trend vs next (older) run
if i + 1 < len(all_runs):
prev_cases = _extract_case_results(all_runs[i + 1])
emojis = []
for cid in case_ids:
cur = run_cases.get(cid, {}).get("sglang")
prev = prev_cases.get(cid, {}).get("sglang")
emojis.append(_trend_emoji(cur, prev))
row += " ".join(emojis) + " |"
else:
row += " -- |"
lines.append(row)
# ---- Section 3: Cross-Framework Speedup Trend (only if multiple frameworks) ----
# ---- Section 2: Cross-Framework Speedup Trend (only if multiple frameworks) ----
if history and other_frameworks:
lines.append("\n## SGLang vs vLLM-Omni Speedup Over Time\n")
@@ -562,6 +527,41 @@ def generate_dashboard(
except ImportError:
lines.append("\n*Charts unavailable (matplotlib not installed)*\n")
# ---- SGLang Performance Trend (raw data table, at the end) ----
if history:
lines.append(f"\n## SGLang Performance Trend (Last {len(history) + 1} Runs)\n")
header = "| Date | Commit |"
sep = "|------|--------|"
for cid in case_ids:
header += f" {cid} (s) |"
sep += "---------|"
header += " Trend |"
sep += "-------|"
lines.append(header)
lines.append(sep)
all_runs = [current] + history
for i, run in enumerate(all_runs):
run_cases = _extract_case_results(run)
date = _short_date(run.get("timestamp", ""))
sha_s = _short_sha(run.get("commit_sha", ""))
row = f"| {date} | `{sha_s}` |"
for cid in case_ids:
lat = run_cases.get(cid, {}).get("sglang")
row += f" {_fmt_latency(lat)} |"
if i + 1 < len(all_runs):
prev_cases = _extract_case_results(all_runs[i + 1])
emojis = []
for cid in case_ids:
cur = run_cases.get(cid, {}).get("sglang")
prev = prev_cases.get(cid, {}).get("sglang")
emojis.append(_trend_emoji(cur, prev))
row += " ".join(emojis) + " |"
else:
row += " -- |"
lines.append(row)
# ---- Risk Notification ----
alert_cases = [
(cid, emoji, reason)
@@ -575,8 +575,7 @@ def generate_dashboard(
lines.append("> The following cases need attention:")
for _cid, _emoji, reason in alert_cases:
lines.append(f"> - {reason}")
lines.append(">")
lines.append("> cc @mickqian @bbuf @yhyang201\n")
lines.append("")
# Footer
lines.append("\n---")
@@ -584,7 +583,164 @@ def generate_dashboard(
"*Generated by `generate_diffusion_dashboard.py` in SGLang nightly CI.*"
)
return "\n".join(lines) + "\n"
alert_reasons = [reason for _, _, reason in alert_cases]
return "\n".join(lines) + "\n", alert_reasons
ALERT_ASSIGNEES = ["mickqian", "bbuf", "yhyang201"]
ALERT_LABEL = "perf-regression"
ALERT_ISSUE_TITLE = "[Diffusion CI] Performance regression tracker"
def _find_alert_issue(repo: str) -> tuple[str | None, bool]:
"""Find the perf-regression tracker issue (open OR closed).
Returns (issue_number, is_open). Prefers an open issue; if none,
returns the most recent closed one so it can be reopened.
"""
import subprocess
for state in ("open", "closed"):
result = subprocess.run(
[
"gh",
"issue",
"list",
"--repo",
repo,
"--label",
ALERT_LABEL,
"--state",
state,
"--json",
"number",
"--limit",
"1",
],
capture_output=True,
text=True,
timeout=30,
)
if result.returncode != 0 or not result.stdout.strip():
continue
issues = json.loads(result.stdout)
if issues:
return str(issues[0]["number"]), state == "open"
return None, False
def _create_alert_issue(alert_reasons: list[str]) -> None:
"""Create or update the single perf-regression tracker issue.
Logic:
- If an open issue exists → add a comment with the new alert.
- If a closed issue exists → reopen it, then add a comment.
- If no issue exists → create one.
This guarantees at most one tracker issue ever exists.
Uses `gh` (GitHub CLI) which is available in all GitHub Actions runners.
Falls back silently outside CI.
"""
import subprocess
run_url = ""
run_id = os.environ.get("GITHUB_RUN_ID", "")
repo = os.environ.get("GITHUB_REPOSITORY", "sgl-project/sglang")
server_url = os.environ.get("GITHUB_SERVER_URL", "https://github.com")
if run_id:
run_url = f"{server_url}/{repo}/actions/runs/{run_id}"
date = datetime.now(timezone.utc).strftime("%Y-%m-%d")
body_lines = [
f"## Performance Alert — {date}",
"",
"The nightly diffusion benchmark detected the following issue(s):",
"",
]
for reason in alert_reasons:
body_lines.append(f"- {reason}")
if run_url:
body_lines += ["", f"**CI Run:** {run_url}"]
body = "\n".join(body_lines)
try:
existing, is_open = _find_alert_issue(repo)
if existing:
# Reopen if closed
if not is_open:
subprocess.run(
[
"gh",
"issue",
"reopen",
existing,
"--repo",
repo,
],
capture_output=True,
text=True,
timeout=30,
)
print(f"Reopened alert issue #{existing}")
# Add comment
result = subprocess.run(
[
"gh",
"issue",
"comment",
existing,
"--repo",
repo,
"--body",
body,
],
capture_output=True,
text=True,
timeout=30,
)
if result.returncode == 0:
print(f"Commented on alert issue #{existing}")
else:
print(
f"Warning: failed to comment on issue #{existing} "
f"(rc={result.returncode}): {result.stderr.strip()}"
)
else:
# Create a new issue
cmd = [
"gh",
"issue",
"create",
"--repo",
repo,
"--title",
ALERT_ISSUE_TITLE,
"--body",
body,
"--label",
ALERT_LABEL,
]
for user in ALERT_ASSIGNEES:
cmd += ["--assignee", user]
result = subprocess.run(cmd, capture_output=True, text=True, timeout=30)
if result.returncode == 0:
print(f"Created alert issue: {result.stdout.strip()}")
else:
print(
f"Warning: failed to create alert issue "
f"(rc={result.returncode}): {result.stderr.strip()}"
)
except FileNotFoundError:
print("Warning: `gh` CLI not found — skipping alert issue creation")
except Exception as e:
print(f"Warning: failed to create/update alert issue: {e}")
# ---------------------------------------------------------------------------
@@ -649,7 +805,9 @@ def main():
print(f"Loaded {len(history)} historical run(s) from {args.history_dir}")
# Generate dashboard
markdown = generate_dashboard(current, history, charts_dir=args.charts_dir)
markdown, alert_reasons = generate_dashboard(
current, history, charts_dir=args.charts_dir
)
# Write output
os.makedirs(os.path.dirname(args.output) or ".", exist_ok=True)
@@ -667,6 +825,12 @@ def main():
else:
print("Warning: $GITHUB_STEP_SUMMARY not set, skipping")
# Create GitHub Issue for performance alerts (so assignees get notified)
if alert_reasons:
_create_alert_issue(alert_reasons)
else:
print("No performance alerts — skipping issue creation.")
if __name__ == "__main__":
main()