diff --git a/.github/workflows/diffusion-ci-gt-gen.yml b/.github/workflows/diffusion-ci-gt-gen.yml index 036f6595a..5bbc8aa7b 100644 --- a/.github/workflows/diffusion-ci-gt-gen.yml +++ b/.github/workflows/diffusion-ci-gt-gen.yml @@ -19,7 +19,7 @@ on: default: '' type: string publish_target_dir: - description: 'Remote target directory in sgl-project/ci-data. Leave empty to use diffusion-ci/consistency_gt/sglang_generated.' + description: 'Remote target directory in sgl-project/ci-data. Leave empty to use sglang_generated, or official_generated when run_official_cases is true.' required: false default: '' type: string @@ -28,9 +28,24 @@ on: required: false default: '' type: string + run_official_cases: + description: 'Run official comparable GT cases instead of native SGLang GT cases.' + required: false + default: false + type: boolean + official_case_ids: + description: 'Specific official case IDs to run (space-separated). Leave empty to run all official comparable cases.' + required: false + default: '' + type: string + official_source_group: + description: 'Official GT source group filter: all, diffusers, wan21, or ltx23. Used only when run_official_cases is true.' + required: false + default: '' + type: string concurrency: - group: diffusion-ci-gt-gen-${{ github.ref }} + group: diffusion-ci-gt-gen-${{ github.ref }}-${{ inputs.output_name || inputs.case_ids || inputs.official_case_ids || inputs.official_source_group || inputs.run_official_cases || 'default' }} cancel-in-progress: true permissions: @@ -41,11 +56,290 @@ env: SGLANG_IS_IN_CI: true SGLANG_CUDA_COREDUMP: "1" OUTPUT_NAME: ${{ inputs.output_name || 'diffusion-ci-outputs' }} - PUBLISH_TARGET_DIR: ${{ inputs.publish_target_dir || 'diffusion-ci/consistency_gt/sglang_generated' }} + PUBLISH_TARGET_DIR: ${{ inputs.publish_target_dir || (inputs.run_official_cases && 'diffusion-ci/consistency_gt/official_generated' || 'diffusion-ci/consistency_gt/sglang_generated') }} jobs: + compute-official-gt-matrix: + if: github.repository == 'sgl-project/sglang' && inputs.run_official_cases + runs-on: ubuntu-latest + outputs: + matrix: ${{ steps.compute.outputs.matrix }} + case-count: ${{ steps.compute.outputs.case-count }} + steps: + - name: Compute official case matrix + id: compute + env: + OFFICIAL_CASE_IDS: ${{ inputs.official_case_ids }} + OFFICIAL_SOURCE_GROUP: ${{ inputs.official_source_group || 'all' }} + run: | + python3 - <<'PY' + import json + import os + + groups = { + "diffusers": [ + "flux_2_image_t2i", + "flux_2_klein_image_t2i", + "flux_2_ti2i", + "flux_image_t2i", + "qwen_image_edit_2509_ti2i", + "qwen_image_edit_2511_ti2i", + "qwen_image_edit_ti2i", + "qwen_image_layered_i2i", + "qwen_image_t2i", + "zimage_image_t2i", + ], + "wan21": ["wan2_1_t2v_1.3b"], + "ltx23": [ + "ltx_2.3_two_stage_t2v_2gpus", + "ltx_2.3_one_stage_ti2v", + ], + } + source_group = os.environ["OFFICIAL_SOURCE_GROUP"].strip() or "all" + if source_group == "all": + selected_groups = list(groups) + elif source_group in groups: + selected_groups = [source_group] + else: + raise SystemExit(f"unknown official_source_group: {source_group}") + + requested = os.environ["OFFICIAL_CASE_IDS"].split() + requested_set = set(requested) + h200_cases = { + "flux_2_image_t2i", + "flux_2_klein_image_t2i", + "flux_2_ti2i", + "ltx_2.3_two_stage_t2v_2gpus", + } + include = [] + for group in selected_groups: + for case_id in groups[group]: + if requested_set and case_id not in requested_set: + continue + include.append( + { + "source_group": group, + "case_id": case_id, + "runner": "8-gpu-h200" if case_id in h200_cases else "1-gpu-h100", + } + ) + + known_cases = {case for cases in groups.values() for case in cases} + unknown = sorted(requested_set - known_cases) + if unknown: + raise SystemExit(f"unknown official case id(s): {' '.join(unknown)}") + if not include: + raise SystemExit("official case matrix is empty") + + with open(os.environ["GITHUB_OUTPUT"], "a", encoding="utf-8") as f: + f.write(f"matrix={json.dumps({'include': include}, separators=(',', ':'))}\n") + f.write(f"case-count={len(include)}\n") + PY + + official-gt-hopper: + needs: compute-official-gt-matrix + if: needs.compute-official-gt-matrix.result == 'success' + runs-on: ${{ matrix.runner }} + timeout-minutes: 240 + strategy: + fail-fast: false + matrix: ${{ fromJson(needs.compute-official-gt-matrix.outputs.matrix) }} + steps: + - name: Checkout code + uses: actions/checkout@v4 + with: + ref: ${{ inputs.ref || github.ref }} + + - name: Checkout ci-data repro scripts + uses: actions/checkout@v4 + with: + repository: sgl-project/ci-data + ref: main + path: ci-data + token: ${{ secrets.GH_PAT_FOR_NIGHTLY_CI_DATA }} + + - name: Prepare sgl-kernel/dist for prebuilt wheel + if: inputs.kernel_artifact_run_id != '' + run: | + ls -alh sgl-kernel/dist || true + rm -rf sgl-kernel/dist/* || true + + - name: Download prebuilt sgl-kernel wheel + if: inputs.kernel_artifact_run_id != '' + uses: actions/download-artifact@v4 + with: + path: sgl-kernel/dist/ + merge-multiple: true + name: wheel-python3.10-cuda13.0 + run-id: ${{ inputs.kernel_artifact_run_id }} + github-token: ${{ secrets.GITHUB_TOKEN }} + + - name: Install dependencies + run: | + CUSTOM_BUILD_SGL_KERNEL="${{ inputs.kernel_artifact_run_id != '' && 'true' || 'false' }}" \ + bash scripts/ci/cuda/ci_install_dependency.sh diffusion + + - name: Install official LTX-2.3 repro dependencies + if: matrix.source_group == 'ltx23' + run: | + UV_SYSTEM_PYTHON=1 uv pip install \ + "transformers==4.52.4" \ + openimageio \ + --index-strategy unsafe-best-match \ + --prerelease allow + UV_SYSTEM_PYTHON=1 uv pip uninstall kernels + + - name: Checkout official Wan2.1 repo + if: matrix.source_group == 'wan21' + run: | + mkdir -p /tmp/mmgen-official-code + if [ ! -d /tmp/mmgen-official-code/Wan2.1/.git ]; then + git clone https://github.com/Wan-Video/Wan2.1.git /tmp/mmgen-official-code/Wan2.1 + fi + git -C /tmp/mmgen-official-code/Wan2.1 fetch --depth 1 origin 9737cba9c1c3c4d04b33fcad41c111989865d315 + git -C /tmp/mmgen-official-code/Wan2.1 checkout 9737cba9c1c3c4d04b33fcad41c111989865d315 + git -C /tmp/mmgen-official-code/Wan2.1 rev-parse HEAD + + - name: Checkout official LTX-2 repo + if: matrix.source_group == 'ltx23' + run: | + mkdir -p /tmp/mmgen-official-code + if [ ! -d /tmp/mmgen-official-code/LTX-2/.git ]; then + git clone https://github.com/Lightricks/LTX-2.git /tmp/mmgen-official-code/LTX-2 + fi + git -C /tmp/mmgen-official-code/LTX-2 fetch --depth 1 origin 41d924371612b692c0fd1e4d9d94c3dfb3c02cb3 + git -C /tmp/mmgen-official-code/LTX-2 checkout 41d924371612b692c0fd1e4d9d94c3dfb3c02cb3 + git -C /tmp/mmgen-official-code/LTX-2 rev-parse HEAD + + - name: Prepare official LTX-2.3 assets + if: matrix.source_group == 'ltx23' + env: + HF_TOKEN: ${{ secrets.SGLANG_DIFFUSION_CI_HF_TOKEN || secrets.HF_TOKEN }} + HUGGING_FACE_HUB_TOKEN: ${{ secrets.SGLANG_DIFFUSION_CI_HF_TOKEN || secrets.HF_TOKEN }} + run: | + python - <<'PY' + import os + from pathlib import Path + + from huggingface_hub import hf_hub_download, snapshot_download + + root = Path.home() / ".cache/sgl_diffusion/materialized_models/Lightricks__LTX-2.3-official-assets" + root.mkdir(parents=True, exist_ok=True) + text_encoder = root / "text_encoder" + tokenizer = root / "tokenizer" + text_encoder.mkdir(exist_ok=True) + tokenizer.mkdir(exist_ok=True) + + gemma_root = Path( + snapshot_download( + repo_id="google/gemma-3-12b-it-qat-q4_0-unquantized", + ignore_patterns=["*.onnx", "*.msgpack"], + max_workers=8, + ) + ) + tokenizer_names = { + "tokenizer.json", + "tokenizer.model", + "tokenizer_config.json", + "special_tokens_map.json", + } + + def link(src: Path, dst: Path) -> None: + if dst.exists() or dst.is_symlink(): + return + try: + dst.symlink_to(src) + except OSError: + os.link(src, dst) + + for src in gemma_root.iterdir(): + if not src.is_file(): + continue + link(src, text_encoder / src.name) + if src.name in tokenizer_names: + link(src, tokenizer / src.name) + + for filename in ( + "ltx-2.3-22b-distilled-lora-384.safetensors", + "ltx-2.3-spatial-upscaler-x2-1.1.safetensors", + ): + src = Path(hf_hub_download("Lightricks/LTX-2.3", filename)) + link(src, root / filename) + + print(root) + print(sorted(p.name for p in root.iterdir())) + PY + + - name: Generate official output + env: + HF_TOKEN: ${{ secrets.SGLANG_DIFFUSION_CI_HF_TOKEN || secrets.HF_TOKEN }} + HUGGING_FACE_HUB_TOKEN: ${{ secrets.SGLANG_DIFFUSION_CI_HF_TOKEN || secrets.HF_TOKEN }} + RUNAI_STREAMER_MEMORY_LIMIT: 0 + PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True + CUDA_VISIBLE_DEVICES: 0 + CASE_ID: ${{ matrix.case_id }} + SOURCE_GROUP: ${{ matrix.source_group }} + run: | + git -C ci-data rev-parse HEAD + mkdir -p "python/${{ env.OUTPUT_NAME }}" + if [ "$SOURCE_GROUP" = "diffusers" ]; then + cd python + python ../ci-data/diffusion-ci/repro_scripts/gen_official_diffusion_gt.py \ + --suite 1-gpu \ + --out-dir ./${{ env.OUTPUT_NAME }} \ + --case-ids "$CASE_ID" \ + --dtype bf16 \ + --device-map none \ + --generator-device cuda + elif [ "$SOURCE_GROUP" = "wan21" ]; then + cd python + python ../ci-data/diffusion-ci/repro_scripts/gen_official_diffusion_gt.py \ + --suite 1-gpu \ + --out-dir ./${{ env.OUTPUT_NAME }} \ + --case-ids "$CASE_ID" \ + --wan-official-repo-dir /tmp/mmgen-official-code/Wan2.1 \ + --dtype bf16 \ + --device-map none \ + --generator-device cuda + elif [ "$SOURCE_GROUP" = "ltx23" ]; then + cd python + set +e + PYTHONPATH=/tmp/mmgen-official-code/LTX-2/packages/ltx-core/src:/tmp/mmgen-official-code/LTX-2/packages/ltx-pipelines/src:$PWD:$PYTHONPATH \ + python ../ci-data/diffusion-ci/repro_scripts/gen_official_ltx23.py \ + --out-dir ./${{ env.OUTPUT_NAME }} \ + --case-ids "$CASE_ID" + status=$? + set -e + if [ "$status" -ne 0 ]; then + find ./${{ env.OUTPUT_NAME }} -name 'official_ltx23_manifest.json' -print -exec cat {} \; + exit "$status" + fi + else + echo "Unknown SOURCE_GROUP=$SOURCE_GROUP" + exit 1 + fi + + - name: Upload artifact + uses: actions/upload-artifact@v4 + with: + name: ${{ env.OUTPUT_NAME }}-${{ matrix.source_group }}-${{ matrix.case_id }} + path: | + python/${{ env.OUTPUT_NAME }}/*.jpg + python/${{ env.OUTPUT_NAME }}/*.png + python/${{ env.OUTPUT_NAME }}/official_gt_manifest_*.json + python/${{ env.OUTPUT_NAME }}/official_ltx23_manifest.json + retention-days: 7 + + - name: Publish official GT images to sgl-project/ci-data + env: + GITHUB_TOKEN: ${{ secrets.GH_PAT_FOR_NIGHTLY_CI_DATA }} + run: | + python scripts/ci/utils/diffusion/publish_diffusion_gt.py \ + --source-dir python/${{ env.OUTPUT_NAME }} \ + --target-dir "${{ env.PUBLISH_TARGET_DIR }}" + compute-diffusion-partitions: - if: github.repository == 'sgl-project/sglang' + if: github.repository == 'sgl-project/sglang' && !inputs.run_official_cases runs-on: ubuntu-latest outputs: matrix-1gpu: ${{ steps.compute.outputs.matrix-1gpu }}