diff --git a/.github/workflows/nightly-test-nvidia.yml b/.github/workflows/nightly-test-nvidia.yml index e9b3ed0dd..252b86e55 100644 --- a/.github/workflows/nightly-test-nvidia.yml +++ b/.github/workflows/nightly-test-nvidia.yml @@ -26,7 +26,7 @@ on: - 'nightly-test-specialized-8-gpu-b200' - 'nightly-test-perf-4-gpu-gb300' - 'nightly-test-kernel-1-gpu-h100' - - 'nightly-test-diffusion-comparison' + - 'nightly-test-diffusion' - 'nightly-test-kernel-8-gpu-h200' - 'nightly-test-precision-8-gpu-h200' workflow_call: @@ -628,9 +628,9 @@ jobs: - uses: ./.github/actions/upload-cuda-coredumps if: failure() - # Diffusion cross-framework comparison - nightly-test-diffusion-comparison: - if: github.repository == 'sgl-project/sglang' && (inputs.job_filter == '' || inputs.job_filter == 'all' || inputs.job_filter == 'nightly-test-diffusion-comparison') + # SGLang-Diffusion nightly benchmark + nightly-test-diffusion: + if: github.repository == 'sgl-project/sglang' && (inputs.job_filter == '' || inputs.job_filter == 'all' || inputs.job_filter == 'nightly-test-diffusion') runs-on: 4-gpu-h100 timeout-minutes: 300 steps: @@ -643,7 +643,7 @@ jobs: run: | bash scripts/ci/cuda/ci_install_dependency.sh diffusion - - name: Run cross-framework comparison + - name: Run diffusion benchmark env: GITHUB_SHA: ${{ github.sha }} GITHUB_RUN_ID: ${{ github.run_id }} @@ -676,11 +676,11 @@ jobs: --dashboard dashboard.md \ --charts-dir comparison-charts - - name: Upload comparison artifacts + - name: Upload benchmark artifacts if: always() uses: actions/upload-artifact@v4 with: - name: diffusion-comparison-${{ github.run_id }} + name: diffusion-benchmark-${{ github.run_id }} path: | comparison-results.json dashboard.md @@ -783,7 +783,7 @@ jobs: - nightly-test-perf-4-gpu-b200 - nightly-test-specialized-8-gpu-b200 - nightly-test-perf-4-gpu-gb300 - - nightly-test-diffusion-comparison + - nightly-test-diffusion - nightly-test-precision-8-gpu-h200 - consolidate-metrics runs-on: ubuntu-latest diff --git a/scripts/ci/utils/diffusion/comparison_configs.json b/scripts/ci/utils/diffusion/comparison_configs.json index b123dc80b..7694d3ac8 100644 --- a/scripts/ci/utils/diffusion/comparison_configs.json +++ b/scripts/ci/utils/diffusion/comparison_configs.json @@ -118,23 +118,6 @@ } } }, - { - "id": "ltx2_twostage_t2v", - "model": "Lightricks/LTX-2", - "task": "text-to-video", - "prompt": "A cat and a dog baking a cake together in a kitchen.", - "width": 768, - "height": 512, - "num_frames": 121, - "seed": 42, - "num_gpus": 2, - "frameworks": { - "sglang": { - "serve_args": "--warmup --enable-cfg-parallel --pipeline-class-name LTX2TwoStagePipeline", - "extra_env": {} - } - } - }, { "id": "ltx2.3_twostage_ti2v_2gpus", "model": "Lightricks/LTX-2.3", @@ -153,6 +136,39 @@ } } }, + { + "id": "ideogram4_fp8_t2i_2gpu", + "model": "ideogram-ai/ideogram-4-fp8", + "task": "text-to-image", + "prompt": "A futuristic cyberpunk city at night, neon lights reflecting on wet streets", + "width": 1024, + "height": 1024, + "seed": 42, + "num_gpus": 2, + "frameworks": { + "sglang": { + "serve_args": "--warmup --tp-size 2 --attention-backend fa", + "extra_env": {} + } + } + }, + { + "id": "cosmos3_super_t2v_2gpu", + "model": "nvidia/Cosmos3-Super", + "task": "text-to-video", + "prompt": "A cat and a dog baking a cake together in a kitchen.", + "width": 1280, + "height": 720, + "num_frames": 81, + "seed": 42, + "num_gpus": 2, + "frameworks": { + "sglang": { + "serve_args": "--warmup --tp-size 2", + "extra_env": {"SGLANG_DISABLE_COSMOS3_GUARDRAILS": "1"} + } + } + }, { "id": "wan22_i2v_a14b_720p", "model": "Wan-AI/Wan2.2-I2V-A14B-Diffusers", diff --git a/scripts/ci/utils/diffusion/generate_diffusion_dashboard.py b/scripts/ci/utils/diffusion/generate_diffusion_dashboard.py index 76e4d12e8..81e6b2728 100644 --- a/scripts/ci/utils/diffusion/generate_diffusion_dashboard.py +++ b/scripts/ci/utils/diffusion/generate_diffusion_dashboard.py @@ -1,4 +1,4 @@ -"""Generate a Markdown dashboard for diffusion cross-framework comparisons. +"""Generate a Markdown dashboard for SGLang-Diffusion nightly benchmarks. Reads current comparison results + historical data from sgl-project/ci-data repo and produces a Markdown report with tables and trend charts saved as PNG files. @@ -252,7 +252,7 @@ def generate_dashboard( Returns the markdown string. """ lines: list[str] = [] - lines.append("# Diffusion Cross-Framework Performance Dashboard\n") + lines.append("# SGLang-Diffusion Nightly Performance Dashboard\n") ts = current.get("timestamp", datetime.now(timezone.utc).isoformat()) sha = current.get("commit_sha", "unknown") lines.append(f"*Generated: {_short_date(ts)} | Commit: `{_short_sha(sha)}`*\n") @@ -297,8 +297,8 @@ def generate_dashboard( all_frameworks.insert(0, "sglang") other_frameworks = [fw for fw in all_frameworks if fw != "sglang"] - # ---- Section 1: Cross-Framework Comparison (current run) ---- - lines.append("## Cross-Framework Performance Comparison\n") + # ---- Section 1: SGLang-Diffusion performance (current run) ---- + lines.append("## SGLang-Diffusion Performance\n") # Compute risk assessments for all cases risk_map: dict[str, tuple[str, str]] = {} @@ -345,7 +345,7 @@ def generate_dashboard( row += f" {_fmt_speedup(sg_lat, case_fws.get(ofw))} |" lines.append(row) - # ---- Section 2: Cross-Framework Speedup Trend (only if multiple frameworks) ---- + # ---- Section 2: Speedup-over-time vs. other frameworks (rendered only when present) ---- if history and other_frameworks: lines.append("\n## SGLang vs vLLM-Omni Speedup Over Time\n") @@ -750,7 +750,7 @@ def _create_alert_issue(alert_reasons: list[str]) -> None: def main(): parser = argparse.ArgumentParser( - description="Generate diffusion cross-framework comparison dashboard" + description="Generate SGLang-Diffusion nightly benchmark dashboard" ) parser.add_argument( "--results", diff --git a/scripts/ci/utils/diffusion/publish_comparison_results.py b/scripts/ci/utils/diffusion/publish_comparison_results.py index 030f76ef3..a56a45876 100644 --- a/scripts/ci/utils/diffusion/publish_comparison_results.py +++ b/scripts/ci/utils/diffusion/publish_comparison_results.py @@ -1,4 +1,4 @@ -"""Publish diffusion comparison results to sgl-project/ci-data repo. +"""Publish SGLang-Diffusion nightly benchmark results to sgl-project/ci-data repo. Pushes comparison-results.json, dashboard.md, and chart PNG files to the ci-data repository for historical tracking. Chart PNGs are stored under @@ -196,7 +196,7 @@ def publish_comparison( def main(): parser = argparse.ArgumentParser( - description="Publish diffusion comparison results to ci-data" + description="Publish SGLang-Diffusion nightly benchmark results to ci-data" ) parser.add_argument( "--results", diff --git a/scripts/ci/utils/diffusion/run_comparison.py b/scripts/ci/utils/diffusion/run_comparison.py index eadd091c4..c625bfde9 100644 --- a/scripts/ci/utils/diffusion/run_comparison.py +++ b/scripts/ci/utils/diffusion/run_comparison.py @@ -1,7 +1,9 @@ -"""Cross-framework comparison benchmark for diffusion serving. +"""Diffusion serving benchmark for SGLang-Diffusion nightly CI. -Launches servers (SGLang, vLLM-Omni, LightX2V) for each test case, sends a -single request, measures end-to-end latency, and writes comparison-results.json. +Launches an SGLang-Diffusion server for each test case, sends a single +request, measures end-to-end latency, and writes comparison-results.json. +The runner still supports extra frameworks via --frameworks, but the nightly +config tracks SGLang-Diffusion only. Usage: # Full run (requires GPU) @@ -84,6 +86,10 @@ def _build_sglang_cmd(case: dict, fw_cfg: dict, port: int) -> list[str]: cmd += ["--num-gpus", str(case["num_gpus"])] if fw_cfg.get("serve_args", "").strip(): cmd += fw_cfg["serve_args"].strip().split() + # No explicit --warmup-resolutions: server-based warmup now defaults to the + # model's sampling-default resolution (see warmup_request_builder), which + # already matches these single-resolution cases — the default warmup is + # sufficient, so we don't pin a resolution here. return cmd @@ -399,14 +405,16 @@ def send_image_request_sglang( if "data" not in data or len(data["data"]) == 0: raise RuntimeError(f"Image request returned no data: {data}") + # Report client-side e2e latency to match vllm-omni / lightx2v (fair + # cross-framework comparison); server-side perf_dump is diagnostic only. if perf_dump_path: server_latency = _read_perf_dump(perf_dump_path) if server_latency is not None: print( - f" Image generated in {server_latency:.2f}s (server-side), " - f"client={client_latency:.2f}s" + f" Image generated in {client_latency:.2f}s (client e2e; " + f"server-side {server_latency:.2f}s, diagnostic)" ) - return server_latency + return client_latency print(f" Image generated in {client_latency:.2f}s") return client_latency @@ -450,14 +458,16 @@ def send_video_request_sglang( client_latency = time.time() - start + # Report client-side e2e latency to match vllm-omni / lightx2v (fair + # cross-framework comparison); server-side perf_dump is diagnostic only. if perf_dump_path: server_latency = _read_perf_dump(perf_dump_path) if server_latency is not None: print( - f" Video generated in {server_latency:.2f}s (server-side), " - f"client={client_latency:.2f}s" + f" Video generated in {client_latency:.2f}s (client e2e; " + f"server-side {server_latency:.2f}s, diagnostic)" ) - return server_latency + return client_latency print(f" Video generated in {client_latency:.2f}s") return client_latency @@ -536,14 +546,16 @@ def send_image_conditioned_request_sglang( client_latency = time.time() - start + # Report client-side e2e latency to match vllm-omni / lightx2v (fair + # cross-framework comparison); server-side perf_dump is diagnostic only. if perf_dump_path: server_latency = _read_perf_dump(perf_dump_path) if server_latency is not None: print( - f" Generated in {server_latency:.2f}s (server-side), " - f"client={client_latency:.2f}s" + f" Generated in {client_latency:.2f}s (client e2e; " + f"server-side {server_latency:.2f}s, diagnostic)" ) - return server_latency + return client_latency print(f" Generated in {client_latency:.2f}s (sglang, image-conditioned)") return client_latency @@ -789,18 +801,17 @@ def run_single( base_url = f"http://{DEFAULT_HOST}:{port}" wait_for_health(base_url, framework) - # Warmup requests (not measured, no perf dump) - # Use few steps to be fast — server's own warmup (warmup_steps=3) handles - # torch.compile compilation; these external warmups just stabilize triton - # kernel specializations across requests. - WARMUP_STEPS = 3 - warmup_case = {**case, "num_inference_steps": WARMUP_STEPS} - for wi in range(1, 3): - print(f" Sending warmup request ({wi}/2, {WARMUP_STEPS} steps)...") - try: - send_request(base_url, warmup_case, framework, config) - except Exception as e: - raise RuntimeError(f"Warmup request {wi} failed: {e}") from e + # No client-side warmup: each framework relies on its own server-side + # warmup before traffic. sglang's serve_args pass --warmup, which `serve` + # resolves to server-based (synthetic) warmup that primes kernels at + # startup, before the health check passes. This goes through the internal + # warmup path that bypasses sampling-param preset validation (e.g. + # Ideogram-4's preset-locked num_inference_steps), so no per-case warmup + # special-casing is needed here. + # NOTE: vllm-omni / lightx2v configure no server-side warmup; if + # cross-framework comparison is restored, they must add their own warmup + # to stay on equal footing — otherwise their measured request pays the + # full cold-start. # Measured request — pass perf_dump_path for SGLang server-side timing if perf_dump_path and os.path.exists(perf_dump_path): @@ -964,7 +975,7 @@ def run_comparison( def main(): parser = argparse.ArgumentParser( - description="Cross-framework diffusion serving comparison benchmark" + description="SGLang-Diffusion serving benchmark (nightly CI)" ) parser.add_argument( "--config",