minke.yu/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 12 Packages Projects Releases Wiki Activity

[diffusion] CI: improve diffusion comparison benchmark setting for realistic perf and auto-discover ut (#22086)

Browse Source 
Co-authored-by: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
Mick
2026-04-04 23:20:37 +08:00
committed by GitHub
co-authored by Claude Opus 4.6
parent 0f0f004f1f
commit efee62efa6
10 changed files with 278 additions and 110 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/workflows/nightly-test-nvidia.yml
+1
View File
@@ -683,6 +683,7 @@ jobs:
if: always()
env:
GH_PAT_FOR_NIGHTLY_CI_DATA: ${{ secrets.GH_PAT_FOR_NIGHTLY_CI_DATA }}
GH_TOKEN: ${{ github.token }}
run: |
python3 scripts/ci/utils/diffusion/generate_diffusion_dashboard.py \
--results comparison-results.json \
python/sglang/cli/utils.py
+3
View File
@@ -78,6 +78,9 @@ def get_is_diffusion_model(model_path: str) -> bool:
if is_known_non_diffusers_diffusion_model(model_path):
return True
if _is_registered_diffusion_model(model_path):
return True
try:
if envs.SGLANG_USE_MODELSCOPE.get():
from modelscope import model_file_download
python/sglang/multimodal_gen/runtime/launch_server.py
+1 -1
View File
@@ -88,7 +88,7 @@ def launch_server(server_args: ServerArgs, launch_http_server: bool = True):
result_pipes_from_slaves_w.append(w)
# Launch all worker processes
master_port = server_args.master_port or (server_args.master_port + 100)
master_port = server_args.master_port
scheduler_pipe_readers = []
scheduler_pipe_writers = []
python/sglang/multimodal_gen/runtime/server_args.py
+14 -24
View File
@@ -188,8 +188,7 @@ class ServerArgs:
)
# Master port for distributed inference
# TODO: do not hard code
master_port: int | None = None
master_port: int = 30005
# http server endpoint config
host: str | None = "127.0.0.1"
@@ -386,36 +385,27 @@ class ServerArgs:
"Warmup enabled, the launch time is expected to be longer than usual"
)
@staticmethod
def _require_port(port: int, name: str) -> None:
"""Raise if *port* is occupied (used under ``--strict-ports``)."""
if not is_port_available(port):
raise RuntimeError(
f"{name} port {port} is unavailable and --strict-ports is enabled. "
f"Either use a different port or disable --strict-ports."
)
def _adjust_network_ports(self):
if self.strict_ports:
# Strict mode: fail if port is unavailable
if not is_port_available(self.port):
raise RuntimeError(
f"Port {self.port} is unavailable and --strict-ports is enabled. "
f"Either use a different port or remove --strict-ports to allow auto-selection."
)
if not is_port_available(self.scheduler_port):
raise RuntimeError(
f"Scheduler port {self.scheduler_port} is unavailable and --strict-ports is enabled. "
f"Either use a different port or remove --strict-ports to allow auto-selection."
)
if self.master_port is not None and not is_port_available(self.master_port):
raise RuntimeError(
f"Master port {self.master_port} is unavailable and --strict-ports is enabled. "
f"Either use a different port or remove --strict-ports to allow auto-selection."
)
self._require_port(self.port, "HTTP")
self._require_port(self.scheduler_port, "Scheduler")
self._require_port(self.master_port, "Master")
else:
self.port = self.settle_port(self.port)
initial_scheduler_port = self.scheduler_port + (
random.randint(0, 100) if self.scheduler_port == 5555 else 0
)
self.scheduler_port = self.settle_port(initial_scheduler_port)
initial_master_port = (
self.master_port
if self.master_port is not None
else (30005 + random.randint(0, 100))
)
self.master_port = self.settle_port(initial_master_port, 37)
self.master_port = self.settle_port(self.master_port, 37)
def _adjust_parallelism(self):
if self.tp_size is None:
python/sglang/multimodal_gen/test/run_suite.py
+13 -9
View File
@@ -28,17 +28,21 @@ _UPDATE_WEIGHTS_MODEL_PAIR_IDS = (
"Qwen-Image",
)
def _discover_unit_tests() -> list[str]:
"""Auto-discover all test_*.py files in the unit/ directory."""
unit_dir = Path(__file__).resolve().parent / "unit"
if not unit_dir.is_dir():
return []
return sorted(
f"../unit/{f.name}" for f in unit_dir.glob("test_*.py") if f.is_file()
)
SUITES = {
# no GPU required; safe to run on any CPU-only runner
"unit": [
"../unit/test_sampling_params.py",
"../unit/test_storage.py",
"../unit/test_lora_format_adapter.py",
"../unit/test_server_args.py",
"../unit/test_input_validation.py",
"../unit/test_resolve_prompts.py",
# add new unit tests here
],
# Auto-discovered from test/unit/test_*.py
"unit": _discover_unit_tests(),
"1-gpu": [
"test_server_a.py",
"test_server_b.py",
python/sglang/multimodal_gen/test/server/test_server_common.py
+4
View File
@@ -102,6 +102,10 @@ def diffusion_server(case: DiffusionTestCase) -> ServerContext:
if server_args.enable_warmup:
extra_args += " --warmup"
# Strict ports: fail immediately if port is occupied instead of silently
# picking another one (which causes the test client to connect to the wrong server).
extra_args += " --strict-ports"
for arg in server_args.extras:
extra_args += f" {arg}"
python/sglang/multimodal_gen/test/server/test_server_utils.py
+8 -7
View File
@@ -375,8 +375,10 @@ class ServerManager:
# Apply custom environment variables
env.update(self.env_vars)
# TODO: unify with run_command
logger.info(f"Running command: {shlex.join(command)}")
cmd_str = shlex.join(command)
# Use print (not logger) so the command always appears in CI output
# regardless of log-level configuration.
print(f"[server-test] Running command: {cmd_str}", flush=True)
process = subprocess.Popen(
command,
@@ -412,11 +414,10 @@ class ServerManager:
log_thread.daemon = True
log_thread.start()
logger.info(
"[server-test] Starting server pid=%s, model=%s, log=%s",
process.pid,
self.model,
stdout_path,
print(
f"[server-test] Starting server pid={process.pid}, "
f"model={self.model}, log={stdout_path}",
flush=True,
)
self._wait_for_ready(process, stdout_path)
python/sglang/srt/multimodal/processors/lfm2_vl.py
+8 -8
View File
@@ -12,9 +12,9 @@
# limitations under the License.
"""Multimodal processor for LFM2-VL models with SigLip2 NaFlex support."""
from typing import Any, Dict, List, Optional, Union
from typing import List, Union
from sglang.srt.managers.schedule_batch import Modality
from sglang.srt.managers.schedule_batch import Modality, MultimodalProcessorOutput
from sglang.srt.models.lfm2_vl import Lfm2VlForConditionalGeneration
from sglang.srt.multimodal.processors.base_processor import (
BaseMultimodalProcessor as SGLangBaseProcessor,
@@ -56,7 +56,7 @@ class Lfm2VlImageProcessor(SGLangBaseProcessor):
input_text: str,
request_obj,
**kwargs,
) -> Optional[Dict[str, Any]]:
):
if not image_data:
input_ids = self._tokenizer(
input_text, return_tensors="pt", add_special_tokens=False
@@ -77,8 +77,8 @@ class Lfm2VlImageProcessor(SGLangBaseProcessor):
base_output, self.mm_tokens
)
return {
"input_ids": input_ids.tolist(),
"mm_items": mm_items,
"im_token_id": self.IMAGE_TOKEN_ID,
}
return MultimodalProcessorOutput(
input_ids=input_ids.tolist(),
mm_items=mm_items,
im_token_id=self.IMAGE_TOKEN_ID,
)
scripts/ci/utils/diffusion/comparison_configs.json
+18 -17
View File
@@ -1,5 +1,5 @@
{
"_comment": "Per-model comparison config. Only frameworks listed under each case are tested. vLLM-Omni disabled until dep install issues resolved.",
"_comment": "Per-model comparison config. Sampling params omitted where model defaults are correct — only override resolution, seed, and params that differ from defaults.",
"test_image_url": "https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/diffusers/cat.png",
"cases": [
{
@@ -9,8 +9,6 @@
"prompt": "A futuristic cyberpunk city at night, neon lights reflecting on wet streets",
"width": 1024,
"height": 1024,
"num_inference_steps": 50,
"guidance_scale": 4.0,
"seed": 42,
"num_gpus": 1,
"frameworks": {
@@ -27,8 +25,6 @@
"prompt": "A futuristic cyberpunk city at night, neon lights reflecting on wet streets",
"width": 1024,
"height": 1024,
"num_inference_steps": 50,
"guidance_scale": 4.0,
"seed": 42,
"num_gpus": 1,
"frameworks": {
@@ -45,8 +41,6 @@
"prompt": "A futuristic cyberpunk city at night, neon lights reflecting on wet streets",
"width": 1024,
"height": 1024,
"num_inference_steps": 50,
"guidance_scale": 4.0,
"seed": 42,
"num_gpus": 1,
"frameworks": {
@@ -64,8 +58,6 @@
"reference_image": true,
"width": 1024,
"height": 1024,
"num_inference_steps": 50,
"guidance_scale": 4.0,
"seed": 42,
"num_gpus": 1,
"frameworks": {
@@ -82,8 +74,6 @@
"prompt": "A futuristic cyberpunk city at night, neon lights reflecting on wet streets",
"width": 1024,
"height": 1024,
"num_inference_steps": 9,
"guidance_scale": 4.0,
"seed": 42,
"num_gpus": 1,
"frameworks": {
@@ -101,8 +91,6 @@
"width": 1280,
"height": 720,
"num_frames": 81,
"num_inference_steps": 2,
"guidance_scale": 5.0,
"seed": 42,
"num_gpus": 4,
"frameworks": {
@@ -121,8 +109,6 @@
"width": 1280,
"height": 720,
"num_frames": 81,
"num_inference_steps": 50,
"guidance_scale": 5.0,
"seed": 42,
"num_gpus": 1,
"frameworks": {
@@ -132,6 +118,23 @@
}
}
},
{
"id": "ltx2_twostage_t2v",
"model": "Lightricks/LTX-2",
"task": "text-to-video",
"prompt": "A cat and a dog baking a cake together in a kitchen.",
"width": 768,
"height": 512,
"num_frames": 121,
"seed": 42,
"num_gpus": 2,
"frameworks": {
"sglang": {
"serve_args": "--enable-torch-compile --warmup --enable-cfg-parallel --pipeline-class-name LTX2TwoStagePipeline",
"extra_env": {}
}
}
},
{
"id": "wan22_i2v_a14b_720p",
"model": "Wan-AI/Wan2.2-I2V-A14B-Diffusers",
@@ -141,8 +144,6 @@
"width": 1280,
"height": 720,
"num_frames": 81,
"num_inference_steps": 2,
"guidance_scale": 5.0,
"seed": 42,
"num_gpus": 4,
"frameworks": {
scripts/ci/utils/diffusion/generate_diffusion_dashboard.py
+208 -44
View File
@@ -239,9 +239,12 @@ def generate_dashboard(
current: dict,
history: list[dict],
charts_dir: str | None = None,
) -> str:
) -> tuple[str, list[str]]:
"""Generate full markdown dashboard.
Returns (markdown_string, alert_reasons) where alert_reasons is a list of
human-readable strings for cases that need attention (empty if all is well).
If charts_dir is provided, saves chart PNGs as files to that directory
and references them via raw.githubusercontent URLs. Otherwise, charts
are omitted.
@@ -342,45 +345,7 @@ def generate_dashboard(
row += f" {_fmt_speedup(sg_lat, case_fws.get(ofw))} |"
lines.append(row)
# ---- Section 2: SGLang Performance Trend ----
if history:
lines.append(f"\n## SGLang Performance Trend (Last {len(history) + 1} Runs)\n")
# Build header
header = "| Date | Commit |"
sep = "|------|--------|"
for cid in case_ids:
header += f" {cid} (s) |"
sep += "---------|"
header += " Trend |"
sep += "-------|"
lines.append(header)
lines.append(sep)
# Current run first
all_runs = [current] + history
for i, run in enumerate(all_runs):
run_cases = _extract_case_results(run)
date = _short_date(run.get("timestamp", ""))
sha_s = _short_sha(run.get("commit_sha", ""))
row = f"| {date} | `{sha_s}` |"
for cid in case_ids:
lat = run_cases.get(cid, {}).get("sglang")
row += f" {_fmt_latency(lat)} |"
# Trend vs next (older) run
if i + 1 < len(all_runs):
prev_cases = _extract_case_results(all_runs[i + 1])
emojis = []
for cid in case_ids:
cur = run_cases.get(cid, {}).get("sglang")
prev = prev_cases.get(cid, {}).get("sglang")
emojis.append(_trend_emoji(cur, prev))
row += " ".join(emojis) + " |"
else:
row += " -- |"
lines.append(row)
# ---- Section 3: Cross-Framework Speedup Trend (only if multiple frameworks) ----
# ---- Section 2: Cross-Framework Speedup Trend (only if multiple frameworks) ----
if history and other_frameworks:
lines.append("\n## SGLang vs vLLM-Omni Speedup Over Time\n")
@@ -562,6 +527,41 @@ def generate_dashboard(
except ImportError:
lines.append("\n*Charts unavailable (matplotlib not installed)*\n")
# ---- SGLang Performance Trend (raw data table, at the end) ----
if history:
lines.append(f"\n## SGLang Performance Trend (Last {len(history) + 1} Runs)\n")
header = "| Date | Commit |"
sep = "|------|--------|"
for cid in case_ids:
header += f" {cid} (s) |"
sep += "---------|"
header += " Trend |"
sep += "-------|"
lines.append(header)
lines.append(sep)
all_runs = [current] + history
for i, run in enumerate(all_runs):
run_cases = _extract_case_results(run)
date = _short_date(run.get("timestamp", ""))
sha_s = _short_sha(run.get("commit_sha", ""))
row = f"| {date} | `{sha_s}` |"
for cid in case_ids:
lat = run_cases.get(cid, {}).get("sglang")
row += f" {_fmt_latency(lat)} |"
if i + 1 < len(all_runs):
prev_cases = _extract_case_results(all_runs[i + 1])
emojis = []
for cid in case_ids:
cur = run_cases.get(cid, {}).get("sglang")
prev = prev_cases.get(cid, {}).get("sglang")
emojis.append(_trend_emoji(cur, prev))
row += " ".join(emojis) + " |"
else:
row += " -- |"
lines.append(row)
# ---- Risk Notification ----
alert_cases = [
(cid, emoji, reason)
@@ -575,8 +575,7 @@ def generate_dashboard(
lines.append("> The following cases need attention:")
for _cid, _emoji, reason in alert_cases:
lines.append(f"> - {reason}")
lines.append(">")
lines.append("> cc @mickqian @bbuf @yhyang201\n")
lines.append("")
# Footer
lines.append("\n---")
@@ -584,7 +583,164 @@ def generate_dashboard(
"*Generated by `generate_diffusion_dashboard.py` in SGLang nightly CI.*"
)
return "\n".join(lines) + "\n"
alert_reasons = [reason for _, _, reason in alert_cases]
return "\n".join(lines) + "\n", alert_reasons
ALERT_ASSIGNEES = ["mickqian", "bbuf", "yhyang201"]
ALERT_LABEL = "perf-regression"
ALERT_ISSUE_TITLE = "[Diffusion CI] Performance regression tracker"
def _find_alert_issue(repo: str) -> tuple[str | None, bool]:
"""Find the perf-regression tracker issue (open OR closed).
Returns (issue_number, is_open). Prefers an open issue; if none,
returns the most recent closed one so it can be reopened.
"""
import subprocess
for state in ("open", "closed"):
result = subprocess.run(
[
"gh",
"issue",
"list",
"--repo",
repo,
"--label",
ALERT_LABEL,
"--state",
state,
"--json",
"number",
"--limit",
"1",
],
capture_output=True,
text=True,
timeout=30,
)
if result.returncode != 0 or not result.stdout.strip():
continue
issues = json.loads(result.stdout)
if issues:
return str(issues[0]["number"]), state == "open"
return None, False
def _create_alert_issue(alert_reasons: list[str]) -> None:
"""Create or update the single perf-regression tracker issue.
Logic:
- If an open issue exists → add a comment with the new alert.
- If a closed issue exists → reopen it, then add a comment.
- If no issue exists → create one.
This guarantees at most one tracker issue ever exists.
Uses `gh` (GitHub CLI) which is available in all GitHub Actions runners.
Falls back silently outside CI.
"""
import subprocess
run_url = ""
run_id = os.environ.get("GITHUB_RUN_ID", "")
repo = os.environ.get("GITHUB_REPOSITORY", "sgl-project/sglang")
server_url = os.environ.get("GITHUB_SERVER_URL", "https://github.com")
if run_id:
run_url = f"{server_url}/{repo}/actions/runs/{run_id}"
date = datetime.now(timezone.utc).strftime("%Y-%m-%d")
body_lines = [
f"## Performance Alert — {date}",
"",
"The nightly diffusion benchmark detected the following issue(s):",
"",
]
for reason in alert_reasons:
body_lines.append(f"- {reason}")
if run_url:
body_lines += ["", f"**CI Run:** {run_url}"]
body = "\n".join(body_lines)
try:
existing, is_open = _find_alert_issue(repo)
if existing:
# Reopen if closed
if not is_open:
subprocess.run(
[
"gh",
"issue",
"reopen",
existing,
"--repo",
repo,
],
capture_output=True,
text=True,
timeout=30,
)
print(f"Reopened alert issue #{existing}")
# Add comment
result = subprocess.run(
[
"gh",
"issue",
"comment",
existing,
"--repo",
repo,
"--body",
body,
],
capture_output=True,
text=True,
timeout=30,
)
if result.returncode == 0:
print(f"Commented on alert issue #{existing}")
else:
print(
f"Warning: failed to comment on issue #{existing} "
f"(rc={result.returncode}): {result.stderr.strip()}"
)
else:
# Create a new issue
cmd = [
"gh",
"issue",
"create",
"--repo",
repo,
"--title",
ALERT_ISSUE_TITLE,
"--body",
body,
"--label",
ALERT_LABEL,
]
for user in ALERT_ASSIGNEES:
cmd += ["--assignee", user]
result = subprocess.run(cmd, capture_output=True, text=True, timeout=30)
if result.returncode == 0:
print(f"Created alert issue: {result.stdout.strip()}")
else:
print(
f"Warning: failed to create alert issue "
f"(rc={result.returncode}): {result.stderr.strip()}"
)
except FileNotFoundError:
print("Warning: `gh` CLI not found — skipping alert issue creation")
except Exception as e:
print(f"Warning: failed to create/update alert issue: {e}")
# ---------------------------------------------------------------------------
@@ -649,7 +805,9 @@ def main():
print(f"Loaded {len(history)} historical run(s) from {args.history_dir}")
# Generate dashboard
markdown = generate_dashboard(current, history, charts_dir=args.charts_dir)
markdown, alert_reasons = generate_dashboard(
current, history, charts_dir=args.charts_dir
)
# Write output
os.makedirs(os.path.dirname(args.output) or ".", exist_ok=True)
@@ -667,6 +825,12 @@ def main():
else:
print("Warning: $GITHUB_STEP_SUMMARY not set, skipping")
# Create GitHub Issue for performance alerts (so assignees get notified)
if alert_reasons:
_create_alert_issue(alert_reasons)
else:
print("No performance alerts — skipping issue creation.")
if __name__ == "__main__":
main()