[diffusion] nightly: track SGLang-Diffusion only (#29434)

.github/workflows/nightly-test-nvidia.yml

@@ -26,7 +26,7 @@ on:
           - 'nightly-test-specialized-8-gpu-b200'
           - 'nightly-test-perf-4-gpu-gb300'
           - 'nightly-test-kernel-1-gpu-h100'
-          - 'nightly-test-diffusion-comparison'
+          - 'nightly-test-diffusion'
           - 'nightly-test-kernel-8-gpu-h200'
           - 'nightly-test-precision-8-gpu-h200'
   workflow_call:
@@ -628,9 +628,9 @@ jobs:
       - uses: ./.github/actions/upload-cuda-coredumps
         if: failure()
 
-  # Diffusion cross-framework comparison
+  # SGLang-Diffusion nightly benchmark
-  nightly-test-diffusion-comparison:
+  nightly-test-diffusion:
-    if: github.repository == 'sgl-project/sglang' && (inputs.job_filter == '' || inputs.job_filter == 'all' || inputs.job_filter == 'nightly-test-diffusion-comparison')
+    if: github.repository == 'sgl-project/sglang' && (inputs.job_filter == '' || inputs.job_filter == 'all' || inputs.job_filter == 'nightly-test-diffusion')
     runs-on: 4-gpu-h100
     timeout-minutes: 300
     steps:
@@ -643,7 +643,7 @@ jobs:
         run: |
           bash scripts/ci/cuda/ci_install_dependency.sh diffusion
 
-      - name: Run cross-framework comparison
+      - name: Run diffusion benchmark
         env:
           GITHUB_SHA: ${{ github.sha }}
           GITHUB_RUN_ID: ${{ github.run_id }}
@@ -676,11 +676,11 @@ jobs:
             --dashboard dashboard.md \
             --charts-dir comparison-charts
 
-      - name: Upload comparison artifacts
+      - name: Upload benchmark artifacts
         if: always()
         uses: actions/upload-artifact@v4
         with:
-          name: diffusion-comparison-${{ github.run_id }}
+          name: diffusion-benchmark-${{ github.run_id }}
           path: |
             comparison-results.json
             dashboard.md
@@ -783,7 +783,7 @@ jobs:
       - nightly-test-perf-4-gpu-b200
       - nightly-test-specialized-8-gpu-b200
       - nightly-test-perf-4-gpu-gb300
-      - nightly-test-diffusion-comparison
+      - nightly-test-diffusion
       - nightly-test-precision-8-gpu-h200
       - consolidate-metrics
     runs-on: ubuntu-latest

scripts/ci/utils/diffusion/comparison_configs.json

@@ -118,23 +118,6 @@
                 }
             }
         },
-        {
-            "id": "ltx2_twostage_t2v",
-            "model": "Lightricks/LTX-2",
-            "task": "text-to-video",
-            "prompt": "A cat and a dog baking a cake together in a kitchen.",
-            "width": 768,
-            "height": 512,
-            "num_frames": 121,
-            "seed": 42,
-            "num_gpus": 2,
-            "frameworks": {
-                "sglang": {
-                    "serve_args": "--warmup --enable-cfg-parallel --pipeline-class-name LTX2TwoStagePipeline",
-                    "extra_env": {}
-                }
-            }
-        },
         {
             "id": "ltx2.3_twostage_ti2v_2gpus",
             "model": "Lightricks/LTX-2.3",
@@ -153,6 +136,39 @@
                 }
             }
         },
+        {
+            "id": "ideogram4_fp8_t2i_2gpu",
+            "model": "ideogram-ai/ideogram-4-fp8",
+            "task": "text-to-image",
+            "prompt": "A futuristic cyberpunk city at night, neon lights reflecting on wet streets",
+            "width": 1024,
+            "height": 1024,
+            "seed": 42,
+            "num_gpus": 2,
+            "frameworks": {
+                "sglang": {
+                    "serve_args": "--warmup --tp-size 2 --attention-backend fa",
+                    "extra_env": {}
+                }
+            }
+        },
+        {
+            "id": "cosmos3_super_t2v_2gpu",
+            "model": "nvidia/Cosmos3-Super",
+            "task": "text-to-video",
+            "prompt": "A cat and a dog baking a cake together in a kitchen.",
+            "width": 1280,
+            "height": 720,
+            "num_frames": 81,
+            "seed": 42,
+            "num_gpus": 2,
+            "frameworks": {
+                "sglang": {
+                    "serve_args": "--warmup --tp-size 2",
+                    "extra_env": {"SGLANG_DISABLE_COSMOS3_GUARDRAILS": "1"}
+                }
+            }
+        },
         {
             "id": "wan22_i2v_a14b_720p",
             "model": "Wan-AI/Wan2.2-I2V-A14B-Diffusers",

scripts/ci/utils/diffusion/generate_diffusion_dashboard.py

@@ -1,4 +1,4 @@
-"""Generate a Markdown dashboard for diffusion cross-framework comparisons.
+"""Generate a Markdown dashboard for SGLang-Diffusion nightly benchmarks.
 
 Reads current comparison results + historical data from sgl-project/ci-data repo
 and produces a Markdown report with tables and trend charts saved as PNG files.
@@ -252,7 +252,7 @@ def generate_dashboard(
     Returns the markdown string.
     """
     lines: list[str] = []
-    lines.append("# Diffusion Cross-Framework Performance Dashboard\n")
+    lines.append("# SGLang-Diffusion Nightly Performance Dashboard\n")
     ts = current.get("timestamp", datetime.now(timezone.utc).isoformat())
     sha = current.get("commit_sha", "unknown")
     lines.append(f"*Generated: {_short_date(ts)} | Commit: `{_short_sha(sha)}`*\n")
@@ -297,8 +297,8 @@ def generate_dashboard(
         all_frameworks.insert(0, "sglang")
     other_frameworks = [fw for fw in all_frameworks if fw != "sglang"]
 
-    # ---- Section 1: Cross-Framework Comparison (current run) ----
+    # ---- Section 1: SGLang-Diffusion performance (current run) ----
-    lines.append("## Cross-Framework Performance Comparison\n")
+    lines.append("## SGLang-Diffusion Performance\n")
 
     # Compute risk assessments for all cases
     risk_map: dict[str, tuple[str, str]] = {}
@@ -345,7 +345,7 @@ def generate_dashboard(
             row += f" {_fmt_speedup(sg_lat, case_fws.get(ofw))} |"
         lines.append(row)
 
-    # ---- Section 2: Cross-Framework Speedup Trend (only if multiple frameworks) ----
+    # ---- Section 2: Speedup-over-time vs. other frameworks (rendered only when present) ----
     if history and other_frameworks:
         lines.append("\n## SGLang vs vLLM-Omni Speedup Over Time\n")
 
@@ -750,7 +750,7 @@ def _create_alert_issue(alert_reasons: list[str]) -> None:
 
 def main():
     parser = argparse.ArgumentParser(
-        description="Generate diffusion cross-framework comparison dashboard"
+        description="Generate SGLang-Diffusion nightly benchmark dashboard"
     )
     parser.add_argument(
         "--results",

scripts/ci/utils/diffusion/publish_comparison_results.py

@@ -1,4 +1,4 @@
-"""Publish diffusion comparison results to sgl-project/ci-data repo.
+"""Publish SGLang-Diffusion nightly benchmark results to sgl-project/ci-data repo.
 
 Pushes comparison-results.json, dashboard.md, and chart PNG files to the
 ci-data repository for historical tracking. Chart PNGs are stored under
@@ -196,7 +196,7 @@ def publish_comparison(
 
 def main():
     parser = argparse.ArgumentParser(
-        description="Publish diffusion comparison results to ci-data"
+        description="Publish SGLang-Diffusion nightly benchmark results to ci-data"
     )
     parser.add_argument(
         "--results",

scripts/ci/utils/diffusion/run_comparison.py

@@ -1,7 +1,9 @@
-"""Cross-framework comparison benchmark for diffusion serving.
+"""Diffusion serving benchmark for SGLang-Diffusion nightly CI.
 
-Launches servers (SGLang, vLLM-Omni, LightX2V) for each test case, sends a
+Launches an SGLang-Diffusion server for each test case, sends a single
-single request, measures end-to-end latency, and writes comparison-results.json.
+request, measures end-to-end latency, and writes comparison-results.json.
+The runner still supports extra frameworks via --frameworks, but the nightly
+config tracks SGLang-Diffusion only.
 
 Usage:
     # Full run (requires GPU)
@@ -84,6 +86,10 @@ def _build_sglang_cmd(case: dict, fw_cfg: dict, port: int) -> list[str]:
         cmd += ["--num-gpus", str(case["num_gpus"])]
     if fw_cfg.get("serve_args", "").strip():
         cmd += fw_cfg["serve_args"].strip().split()
+    # No explicit --warmup-resolutions: server-based warmup now defaults to the
+    # model's sampling-default resolution (see warmup_request_builder), which
+    # already matches these single-resolution cases — the default warmup is
+    # sufficient, so we don't pin a resolution here.
     return cmd
 
 
@@ -399,14 +405,16 @@ def send_image_request_sglang(
     if "data" not in data or len(data["data"]) == 0:
         raise RuntimeError(f"Image request returned no data: {data}")
 
+    # Report client-side e2e latency to match vllm-omni / lightx2v (fair
+    # cross-framework comparison); server-side perf_dump is diagnostic only.
     if perf_dump_path:
         server_latency = _read_perf_dump(perf_dump_path)
         if server_latency is not None:
             print(
-                f"  Image generated in {server_latency:.2f}s (server-side), "
+                f"  Image generated in {client_latency:.2f}s (client e2e; "
-                f"client={client_latency:.2f}s"
+                f"server-side {server_latency:.2f}s, diagnostic)"
             )
-            return server_latency
+            return client_latency
     print(f"  Image generated in {client_latency:.2f}s")
     return client_latency
 
@@ -450,14 +458,16 @@ def send_video_request_sglang(
 
     client_latency = time.time() - start
 
+    # Report client-side e2e latency to match vllm-omni / lightx2v (fair
+    # cross-framework comparison); server-side perf_dump is diagnostic only.
     if perf_dump_path:
         server_latency = _read_perf_dump(perf_dump_path)
         if server_latency is not None:
             print(
-                f"  Video generated in {server_latency:.2f}s (server-side), "
+                f"  Video generated in {client_latency:.2f}s (client e2e; "
-                f"client={client_latency:.2f}s"
+                f"server-side {server_latency:.2f}s, diagnostic)"
             )
-            return server_latency
+            return client_latency
     print(f"  Video generated in {client_latency:.2f}s")
     return client_latency
 
@@ -536,14 +546,16 @@ def send_image_conditioned_request_sglang(
 
     client_latency = time.time() - start
 
+    # Report client-side e2e latency to match vllm-omni / lightx2v (fair
+    # cross-framework comparison); server-side perf_dump is diagnostic only.
     if perf_dump_path:
         server_latency = _read_perf_dump(perf_dump_path)
         if server_latency is not None:
             print(
-                f"  Generated in {server_latency:.2f}s (server-side), "
+                f"  Generated in {client_latency:.2f}s (client e2e; "
-                f"client={client_latency:.2f}s"
+                f"server-side {server_latency:.2f}s, diagnostic)"
             )
-            return server_latency
+            return client_latency
     print(f"  Generated in {client_latency:.2f}s (sglang, image-conditioned)")
     return client_latency
 
@@ -789,18 +801,17 @@ def run_single(
         base_url = f"http://{DEFAULT_HOST}:{port}"
         wait_for_health(base_url, framework)
 
-        # Warmup requests (not measured, no perf dump)
+        # No client-side warmup: each framework relies on its own server-side
-        # Use few steps to be fast — server's own warmup (warmup_steps=3) handles
+        # warmup before traffic. sglang's serve_args pass --warmup, which `serve`
-        # torch.compile compilation; these external warmups just stabilize triton
+        # resolves to server-based (synthetic) warmup that primes kernels at
-        # kernel specializations across requests.
+        # startup, before the health check passes. This goes through the internal
-        WARMUP_STEPS = 3
+        # warmup path that bypasses sampling-param preset validation (e.g.
-        warmup_case = {**case, "num_inference_steps": WARMUP_STEPS}
+        # Ideogram-4's preset-locked num_inference_steps), so no per-case warmup
-        for wi in range(1, 3):
+        # special-casing is needed here.
-            print(f"  Sending warmup request ({wi}/2, {WARMUP_STEPS} steps)...")
+        # NOTE: vllm-omni / lightx2v configure no server-side warmup; if
-            try:
+        # cross-framework comparison is restored, they must add their own warmup
-                send_request(base_url, warmup_case, framework, config)
+        # to stay on equal footing — otherwise their measured request pays the
-            except Exception as e:
+        # full cold-start.
-                raise RuntimeError(f"Warmup request {wi} failed: {e}") from e
 
         # Measured request — pass perf_dump_path for SGLang server-side timing
         if perf_dump_path and os.path.exists(perf_dump_path):
@@ -964,7 +975,7 @@ def run_comparison(
 
 def main():
     parser = argparse.ArgumentParser(
-        description="Cross-framework diffusion serving comparison benchmark"
+        description="SGLang-Diffusion serving benchmark (nightly CI)"
     )
     parser.add_argument(
         "--config",