[diffusion] CI: improve diffusion comparison benchmark setting for realistic perf and auto-discover ut (#22086)

Co-authored-by: Claude Opus 4.6 <noreply@anthropic.com>

.github/workflows/nightly-test-nvidia.yml

@@ -683,6 +683,7 @@ jobs:
         if: always()
         env:
           GH_PAT_FOR_NIGHTLY_CI_DATA: ${{ secrets.GH_PAT_FOR_NIGHTLY_CI_DATA }}
+          GH_TOKEN: ${{ github.token }}
         run: |
           python3 scripts/ci/utils/diffusion/generate_diffusion_dashboard.py \
             --results comparison-results.json \

python/sglang/cli/utils.py

@@ -78,6 +78,9 @@ def get_is_diffusion_model(model_path: str) -> bool:
     if is_known_non_diffusers_diffusion_model(model_path):
         return True
 
+    if _is_registered_diffusion_model(model_path):
+        return True
+
     try:
         if envs.SGLANG_USE_MODELSCOPE.get():
             from modelscope import model_file_download

python/sglang/multimodal_gen/runtime/launch_server.py

@@ -88,7 +88,7 @@ def launch_server(server_args: ServerArgs, launch_http_server: bool = True):
         result_pipes_from_slaves_w.append(w)
 
     # Launch all worker processes
-    master_port = server_args.master_port or (server_args.master_port + 100)
+    master_port = server_args.master_port
     scheduler_pipe_readers = []
     scheduler_pipe_writers = []
 

python/sglang/multimodal_gen/runtime/server_args.py

@@ -188,8 +188,7 @@ class ServerArgs:
     )
 
     # Master port for distributed inference
-    # TODO: do not hard code
+    master_port: int = 30005
-    master_port: int | None = None
 
     # http server endpoint config
     host: str | None = "127.0.0.1"
@@ -386,36 +385,27 @@ class ServerArgs:
                 "Warmup enabled, the launch time is expected to be longer than usual"
             )
 
+    @staticmethod
+    def _require_port(port: int, name: str) -> None:
+        """Raise if *port* is occupied (used under ``--strict-ports``)."""
+        if not is_port_available(port):
+            raise RuntimeError(
+                f"{name} port {port} is unavailable and --strict-ports is enabled. "
+                f"Either use a different port or disable --strict-ports."
+            )
+
     def _adjust_network_ports(self):
         if self.strict_ports:
-            # Strict mode: fail if port is unavailable
+            self._require_port(self.port, "HTTP")
-            if not is_port_available(self.port):
+            self._require_port(self.scheduler_port, "Scheduler")
-                raise RuntimeError(
+            self._require_port(self.master_port, "Master")
-                    f"Port {self.port} is unavailable and --strict-ports is enabled. "
-                    f"Either use a different port or remove --strict-ports to allow auto-selection."
-                )
-            if not is_port_available(self.scheduler_port):
-                raise RuntimeError(
-                    f"Scheduler port {self.scheduler_port} is unavailable and --strict-ports is enabled. "
-                    f"Either use a different port or remove --strict-ports to allow auto-selection."
-                )
-            if self.master_port is not None and not is_port_available(self.master_port):
-                raise RuntimeError(
-                    f"Master port {self.master_port} is unavailable and --strict-ports is enabled. "
-                    f"Either use a different port or remove --strict-ports to allow auto-selection."
-                )
         else:
             self.port = self.settle_port(self.port)
             initial_scheduler_port = self.scheduler_port + (
                 random.randint(0, 100) if self.scheduler_port == 5555 else 0
             )
             self.scheduler_port = self.settle_port(initial_scheduler_port)
-            initial_master_port = (
+            self.master_port = self.settle_port(self.master_port, 37)
-                self.master_port
-                if self.master_port is not None
-                else (30005 + random.randint(0, 100))
-            )
-            self.master_port = self.settle_port(initial_master_port, 37)
 
     def _adjust_parallelism(self):
         if self.tp_size is None:

python/sglang/multimodal_gen/test/run_suite.py

@@ -28,17 +28,21 @@ _UPDATE_WEIGHTS_MODEL_PAIR_IDS = (
     "Qwen-Image",
 )
 
+
+def _discover_unit_tests() -> list[str]:
+    """Auto-discover all test_*.py files in the unit/ directory."""
+    unit_dir = Path(__file__).resolve().parent / "unit"
+    if not unit_dir.is_dir():
+        return []
+    return sorted(
+        f"../unit/{f.name}" for f in unit_dir.glob("test_*.py") if f.is_file()
+    )
+
+
 SUITES = {
     # no GPU required; safe to run on any CPU-only runner
-    "unit": [
+    # Auto-discovered from test/unit/test_*.py
-        "../unit/test_sampling_params.py",
+    "unit": _discover_unit_tests(),
-        "../unit/test_storage.py",
-        "../unit/test_lora_format_adapter.py",
-        "../unit/test_server_args.py",
-        "../unit/test_input_validation.py",
-        "../unit/test_resolve_prompts.py",
-        # add new unit tests here
-    ],
     "1-gpu": [
         "test_server_a.py",
         "test_server_b.py",

python/sglang/multimodal_gen/test/server/test_server_common.py

@@ -102,6 +102,10 @@ def diffusion_server(case: DiffusionTestCase) -> ServerContext:
     if server_args.enable_warmup:
         extra_args += " --warmup"
 
+    # Strict ports: fail immediately if port is occupied instead of silently
+    # picking another one (which causes the test client to connect to the wrong server).
+    extra_args += " --strict-ports"
+
     for arg in server_args.extras:
         extra_args += f" {arg}"
 

python/sglang/multimodal_gen/test/server/test_server_utils.py

@@ -375,8 +375,10 @@ class ServerManager:
         # Apply custom environment variables
         env.update(self.env_vars)
 
-        # TODO: unify with run_command
+        cmd_str = shlex.join(command)
-        logger.info(f"Running command: {shlex.join(command)}")
+        # Use print (not logger) so the command always appears in CI output
+        # regardless of log-level configuration.
+        print(f"[server-test] Running command: {cmd_str}", flush=True)
 
         process = subprocess.Popen(
             command,
@@ -412,11 +414,10 @@ class ServerManager:
             log_thread.daemon = True
             log_thread.start()
 
-        logger.info(
+        print(
-            "[server-test] Starting server pid=%s, model=%s, log=%s",
+            f"[server-test] Starting server pid={process.pid}, "
-            process.pid,
+            f"model={self.model}, log={stdout_path}",
-            self.model,
+            flush=True,
-            stdout_path,
         )
 
         self._wait_for_ready(process, stdout_path)

python/sglang/srt/multimodal/processors/lfm2_vl.py

@@ -12,9 +12,9 @@
 # limitations under the License.
 """Multimodal processor for LFM2-VL models with SigLip2 NaFlex support."""
 
-from typing import Any, Dict, List, Optional, Union
+from typing import List, Union
 
-from sglang.srt.managers.schedule_batch import Modality
+from sglang.srt.managers.schedule_batch import Modality, MultimodalProcessorOutput
 from sglang.srt.models.lfm2_vl import Lfm2VlForConditionalGeneration
 from sglang.srt.multimodal.processors.base_processor import (
     BaseMultimodalProcessor as SGLangBaseProcessor,
@@ -56,7 +56,7 @@ class Lfm2VlImageProcessor(SGLangBaseProcessor):
         input_text: str,
         request_obj,
         **kwargs,
-    ) -> Optional[Dict[str, Any]]:
+    ):
         if not image_data:
             input_ids = self._tokenizer(
                 input_text, return_tensors="pt", add_special_tokens=False
@@ -77,8 +77,8 @@ class Lfm2VlImageProcessor(SGLangBaseProcessor):
             base_output, self.mm_tokens
         )
 
-        return {
+        return MultimodalProcessorOutput(
-            "input_ids": input_ids.tolist(),
+            input_ids=input_ids.tolist(),
-            "mm_items": mm_items,
+            mm_items=mm_items,
-            "im_token_id": self.IMAGE_TOKEN_ID,
+            im_token_id=self.IMAGE_TOKEN_ID,
-        }
+        )

scripts/ci/utils/diffusion/comparison_configs.json

@@ -1,5 +1,5 @@
 {
-    "_comment": "Per-model comparison config. Only frameworks listed under each case are tested. vLLM-Omni disabled until dep install issues resolved.",
+    "_comment": "Per-model comparison config. Sampling params omitted where model defaults are correct — only override resolution, seed, and params that differ from defaults.",
     "test_image_url": "https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/diffusers/cat.png",
     "cases": [
         {
@@ -9,8 +9,6 @@
             "prompt": "A futuristic cyberpunk city at night, neon lights reflecting on wet streets",
             "width": 1024,
             "height": 1024,
-            "num_inference_steps": 50,
-            "guidance_scale": 4.0,
             "seed": 42,
             "num_gpus": 1,
             "frameworks": {
@@ -27,8 +25,6 @@
             "prompt": "A futuristic cyberpunk city at night, neon lights reflecting on wet streets",
             "width": 1024,
             "height": 1024,
-            "num_inference_steps": 50,
-            "guidance_scale": 4.0,
             "seed": 42,
             "num_gpus": 1,
             "frameworks": {
@@ -45,8 +41,6 @@
             "prompt": "A futuristic cyberpunk city at night, neon lights reflecting on wet streets",
             "width": 1024,
             "height": 1024,
-            "num_inference_steps": 50,
-            "guidance_scale": 4.0,
             "seed": 42,
             "num_gpus": 1,
             "frameworks": {
@@ -64,8 +58,6 @@
             "reference_image": true,
             "width": 1024,
             "height": 1024,
-            "num_inference_steps": 50,
-            "guidance_scale": 4.0,
             "seed": 42,
             "num_gpus": 1,
             "frameworks": {
@@ -82,8 +74,6 @@
             "prompt": "A futuristic cyberpunk city at night, neon lights reflecting on wet streets",
             "width": 1024,
             "height": 1024,
-            "num_inference_steps": 9,
-            "guidance_scale": 4.0,
             "seed": 42,
             "num_gpus": 1,
             "frameworks": {
@@ -101,8 +91,6 @@
             "width": 1280,
             "height": 720,
             "num_frames": 81,
-            "num_inference_steps": 2,
-            "guidance_scale": 5.0,
             "seed": 42,
             "num_gpus": 4,
             "frameworks": {
@@ -121,8 +109,6 @@
             "width": 1280,
             "height": 720,
             "num_frames": 81,
-            "num_inference_steps": 50,
-            "guidance_scale": 5.0,
             "seed": 42,
             "num_gpus": 1,
             "frameworks": {
@@ -132,6 +118,23 @@
                 }
             }
         },
+        {
+            "id": "ltx2_twostage_t2v",
+            "model": "Lightricks/LTX-2",
+            "task": "text-to-video",
+            "prompt": "A cat and a dog baking a cake together in a kitchen.",
+            "width": 768,
+            "height": 512,
+            "num_frames": 121,
+            "seed": 42,
+            "num_gpus": 2,
+            "frameworks": {
+                "sglang": {
+                    "serve_args": "--enable-torch-compile --warmup --enable-cfg-parallel --pipeline-class-name LTX2TwoStagePipeline",
+                    "extra_env": {}
+                }
+            }
+        },
         {
             "id": "wan22_i2v_a14b_720p",
             "model": "Wan-AI/Wan2.2-I2V-A14B-Diffusers",
@@ -141,8 +144,6 @@
             "width": 1280,
             "height": 720,
             "num_frames": 81,
-            "num_inference_steps": 2,
-            "guidance_scale": 5.0,
             "seed": 42,
             "num_gpus": 4,
             "frameworks": {

scripts/ci/utils/diffusion/generate_diffusion_dashboard.py

@@ -239,9 +239,12 @@ def generate_dashboard(
     current: dict,
     history: list[dict],
     charts_dir: str | None = None,
-) -> str:
+) -> tuple[str, list[str]]:
     """Generate full markdown dashboard.
 
+    Returns (markdown_string, alert_reasons) where alert_reasons is a list of
+    human-readable strings for cases that need attention (empty if all is well).
+
     If charts_dir is provided, saves chart PNGs as files to that directory
     and references them via raw.githubusercontent URLs. Otherwise, charts
     are omitted.
@@ -342,45 +345,7 @@ def generate_dashboard(
             row += f" {_fmt_speedup(sg_lat, case_fws.get(ofw))} |"
         lines.append(row)
 
-    # ---- Section 2: SGLang Performance Trend ----
+    # ---- Section 2: Cross-Framework Speedup Trend (only if multiple frameworks) ----
-    if history:
-        lines.append(f"\n## SGLang Performance Trend (Last {len(history) + 1} Runs)\n")
-
-        # Build header
-        header = "| Date | Commit |"
-        sep = "|------|--------|"
-        for cid in case_ids:
-            header += f" {cid} (s) |"
-            sep += "---------|"
-        header += " Trend |"
-        sep += "-------|"
-        lines.append(header)
-        lines.append(sep)
-
-        # Current run first
-        all_runs = [current] + history
-        for i, run in enumerate(all_runs):
-            run_cases = _extract_case_results(run)
-            date = _short_date(run.get("timestamp", ""))
-            sha_s = _short_sha(run.get("commit_sha", ""))
-            row = f"| {date} | `{sha_s}` |"
-            for cid in case_ids:
-                lat = run_cases.get(cid, {}).get("sglang")
-                row += f" {_fmt_latency(lat)} |"
-            # Trend vs next (older) run
-            if i + 1 < len(all_runs):
-                prev_cases = _extract_case_results(all_runs[i + 1])
-                emojis = []
-                for cid in case_ids:
-                    cur = run_cases.get(cid, {}).get("sglang")
-                    prev = prev_cases.get(cid, {}).get("sglang")
-                    emojis.append(_trend_emoji(cur, prev))
-                row += " ".join(emojis) + " |"
-            else:
-                row += " -- |"
-            lines.append(row)
-
-    # ---- Section 3: Cross-Framework Speedup Trend (only if multiple frameworks) ----
     if history and other_frameworks:
         lines.append("\n## SGLang vs vLLM-Omni Speedup Over Time\n")
 
@@ -562,6 +527,41 @@ def generate_dashboard(
         except ImportError:
             lines.append("\n*Charts unavailable (matplotlib not installed)*\n")
 
+    # ---- SGLang Performance Trend (raw data table, at the end) ----
+    if history:
+        lines.append(f"\n## SGLang Performance Trend (Last {len(history) + 1} Runs)\n")
+
+        header = "| Date | Commit |"
+        sep = "|------|--------|"
+        for cid in case_ids:
+            header += f" {cid} (s) |"
+            sep += "---------|"
+        header += " Trend |"
+        sep += "-------|"
+        lines.append(header)
+        lines.append(sep)
+
+        all_runs = [current] + history
+        for i, run in enumerate(all_runs):
+            run_cases = _extract_case_results(run)
+            date = _short_date(run.get("timestamp", ""))
+            sha_s = _short_sha(run.get("commit_sha", ""))
+            row = f"| {date} | `{sha_s}` |"
+            for cid in case_ids:
+                lat = run_cases.get(cid, {}).get("sglang")
+                row += f" {_fmt_latency(lat)} |"
+            if i + 1 < len(all_runs):
+                prev_cases = _extract_case_results(all_runs[i + 1])
+                emojis = []
+                for cid in case_ids:
+                    cur = run_cases.get(cid, {}).get("sglang")
+                    prev = prev_cases.get(cid, {}).get("sglang")
+                    emojis.append(_trend_emoji(cur, prev))
+                row += " ".join(emojis) + " |"
+            else:
+                row += " -- |"
+            lines.append(row)
+
     # ---- Risk Notification ----
     alert_cases = [
         (cid, emoji, reason)
@@ -575,8 +575,7 @@ def generate_dashboard(
         lines.append("> The following cases need attention:")
         for _cid, _emoji, reason in alert_cases:
             lines.append(f"> - {reason}")
-        lines.append(">")
+        lines.append("")
-        lines.append("> cc @mickqian @bbuf @yhyang201\n")
 
     # Footer
     lines.append("\n---")
@@ -584,7 +583,164 @@ def generate_dashboard(
         "*Generated by `generate_diffusion_dashboard.py` in SGLang nightly CI.*"
     )
 
-    return "\n".join(lines) + "\n"
+    alert_reasons = [reason for _, _, reason in alert_cases]
+    return "\n".join(lines) + "\n", alert_reasons
+
+
+ALERT_ASSIGNEES = ["mickqian", "bbuf", "yhyang201"]
+ALERT_LABEL = "perf-regression"
+
+
+ALERT_ISSUE_TITLE = "[Diffusion CI] Performance regression tracker"
+
+
+def _find_alert_issue(repo: str) -> tuple[str | None, bool]:
+    """Find the perf-regression tracker issue (open OR closed).
+
+    Returns (issue_number, is_open).  Prefers an open issue; if none,
+    returns the most recent closed one so it can be reopened.
+    """
+    import subprocess
+
+    for state in ("open", "closed"):
+        result = subprocess.run(
+            [
+                "gh",
+                "issue",
+                "list",
+                "--repo",
+                repo,
+                "--label",
+                ALERT_LABEL,
+                "--state",
+                state,
+                "--json",
+                "number",
+                "--limit",
+                "1",
+            ],
+            capture_output=True,
+            text=True,
+            timeout=30,
+        )
+        if result.returncode != 0 or not result.stdout.strip():
+            continue
+        issues = json.loads(result.stdout)
+        if issues:
+            return str(issues[0]["number"]), state == "open"
+    return None, False
+
+
+def _create_alert_issue(alert_reasons: list[str]) -> None:
+    """Create or update the single perf-regression tracker issue.
+
+    Logic:
+    - If an open issue exists  → add a comment with the new alert.
+    - If a closed issue exists → reopen it, then add a comment.
+    - If no issue exists       → create one.
+
+    This guarantees at most one tracker issue ever exists.
+
+    Uses `gh` (GitHub CLI) which is available in all GitHub Actions runners.
+    Falls back silently outside CI.
+    """
+    import subprocess
+
+    run_url = ""
+    run_id = os.environ.get("GITHUB_RUN_ID", "")
+    repo = os.environ.get("GITHUB_REPOSITORY", "sgl-project/sglang")
+    server_url = os.environ.get("GITHUB_SERVER_URL", "https://github.com")
+    if run_id:
+        run_url = f"{server_url}/{repo}/actions/runs/{run_id}"
+
+    date = datetime.now(timezone.utc).strftime("%Y-%m-%d")
+
+    body_lines = [
+        f"## Performance Alert — {date}",
+        "",
+        "The nightly diffusion benchmark detected the following issue(s):",
+        "",
+    ]
+    for reason in alert_reasons:
+        body_lines.append(f"- {reason}")
+    if run_url:
+        body_lines += ["", f"**CI Run:** {run_url}"]
+    body = "\n".join(body_lines)
+
+    try:
+        existing, is_open = _find_alert_issue(repo)
+
+        if existing:
+            # Reopen if closed
+            if not is_open:
+                subprocess.run(
+                    [
+                        "gh",
+                        "issue",
+                        "reopen",
+                        existing,
+                        "--repo",
+                        repo,
+                    ],
+                    capture_output=True,
+                    text=True,
+                    timeout=30,
+                )
+                print(f"Reopened alert issue #{existing}")
+
+            # Add comment
+            result = subprocess.run(
+                [
+                    "gh",
+                    "issue",
+                    "comment",
+                    existing,
+                    "--repo",
+                    repo,
+                    "--body",
+                    body,
+                ],
+                capture_output=True,
+                text=True,
+                timeout=30,
+            )
+            if result.returncode == 0:
+                print(f"Commented on alert issue #{existing}")
+            else:
+                print(
+                    f"Warning: failed to comment on issue #{existing} "
+                    f"(rc={result.returncode}): {result.stderr.strip()}"
+                )
+        else:
+            # Create a new issue
+            cmd = [
+                "gh",
+                "issue",
+                "create",
+                "--repo",
+                repo,
+                "--title",
+                ALERT_ISSUE_TITLE,
+                "--body",
+                body,
+                "--label",
+                ALERT_LABEL,
+            ]
+            for user in ALERT_ASSIGNEES:
+                cmd += ["--assignee", user]
+
+            result = subprocess.run(cmd, capture_output=True, text=True, timeout=30)
+            if result.returncode == 0:
+                print(f"Created alert issue: {result.stdout.strip()}")
+            else:
+                print(
+                    f"Warning: failed to create alert issue "
+                    f"(rc={result.returncode}): {result.stderr.strip()}"
+                )
+    except FileNotFoundError:
+        print("Warning: `gh` CLI not found — skipping alert issue creation")
+    except Exception as e:
+        print(f"Warning: failed to create/update alert issue: {e}")
 
 
 # ---------------------------------------------------------------------------
@@ -649,7 +805,9 @@ def main():
         print(f"Loaded {len(history)} historical run(s) from {args.history_dir}")
 
     # Generate dashboard
-    markdown = generate_dashboard(current, history, charts_dir=args.charts_dir)
+    markdown, alert_reasons = generate_dashboard(
+        current, history, charts_dir=args.charts_dir
+    )
 
     # Write output
     os.makedirs(os.path.dirname(args.output) or ".", exist_ok=True)
@@ -667,6 +825,12 @@ def main():
         else:
             print("Warning: $GITHUB_STEP_SUMMARY not set, skipping")
 
+    # Create GitHub Issue for performance alerts (so assignees get notified)
+    if alert_reasons:
+        _create_alert_issue(alert_reasons)
+    else:
+        print("No performance alerts — skipping issue creation.")
+
 
 if __name__ == "__main__":
     main()