[diffusion] CI: add diffusion GT generation (official implementation) to the workflow (#24270)

.github/workflows/diffusion-ci-gt-gen.yml

@@ -19,7 +19,7 @@ on:
         default: ''
         type: string
       publish_target_dir:
-        description: 'Remote target directory in sgl-project/ci-data. Leave empty to use diffusion-ci/consistency_gt/sglang_generated.'
+        description: 'Remote target directory in sgl-project/ci-data. Leave empty to use sglang_generated, or official_generated when run_official_cases is true.'
         required: false
         default: ''
         type: string
@@ -28,9 +28,24 @@ on:
         required: false
         default: ''
         type: string
+      run_official_cases:
+        description: 'Run official comparable GT cases instead of native SGLang GT cases.'
+        required: false
+        default: false
+        type: boolean
+      official_case_ids:
+        description: 'Specific official case IDs to run (space-separated). Leave empty to run all official comparable cases.'
+        required: false
+        default: ''
+        type: string
+      official_source_group:
+        description: 'Official GT source group filter: all, diffusers, wan21, or ltx23. Used only when run_official_cases is true.'
+        required: false
+        default: ''
+        type: string
 
 concurrency:
-  group: diffusion-ci-gt-gen-${{ github.ref }}
+  group: diffusion-ci-gt-gen-${{ github.ref }}-${{ inputs.output_name || inputs.case_ids || inputs.official_case_ids || inputs.official_source_group || inputs.run_official_cases || 'default' }}
   cancel-in-progress: true
 
 permissions:
@@ -41,11 +56,290 @@ env:
   SGLANG_IS_IN_CI: true
   SGLANG_CUDA_COREDUMP: "1"
   OUTPUT_NAME: ${{ inputs.output_name || 'diffusion-ci-outputs' }}
-  PUBLISH_TARGET_DIR: ${{ inputs.publish_target_dir || 'diffusion-ci/consistency_gt/sglang_generated' }}
+  PUBLISH_TARGET_DIR: ${{ inputs.publish_target_dir || (inputs.run_official_cases && 'diffusion-ci/consistency_gt/official_generated' || 'diffusion-ci/consistency_gt/sglang_generated') }}
 
 jobs:
+  compute-official-gt-matrix:
+    if: github.repository == 'sgl-project/sglang' && inputs.run_official_cases
+    runs-on: ubuntu-latest
+    outputs:
+      matrix: ${{ steps.compute.outputs.matrix }}
+      case-count: ${{ steps.compute.outputs.case-count }}
+    steps:
+      - name: Compute official case matrix
+        id: compute
+        env:
+          OFFICIAL_CASE_IDS: ${{ inputs.official_case_ids }}
+          OFFICIAL_SOURCE_GROUP: ${{ inputs.official_source_group || 'all' }}
+        run: |
+          python3 - <<'PY'
+          import json
+          import os
+
+          groups = {
+              "diffusers": [
+                  "flux_2_image_t2i",
+                  "flux_2_klein_image_t2i",
+                  "flux_2_ti2i",
+                  "flux_image_t2i",
+                  "qwen_image_edit_2509_ti2i",
+                  "qwen_image_edit_2511_ti2i",
+                  "qwen_image_edit_ti2i",
+                  "qwen_image_layered_i2i",
+                  "qwen_image_t2i",
+                  "zimage_image_t2i",
+              ],
+              "wan21": ["wan2_1_t2v_1.3b"],
+              "ltx23": [
+                  "ltx_2.3_two_stage_t2v_2gpus",
+                  "ltx_2.3_one_stage_ti2v",
+              ],
+          }
+          source_group = os.environ["OFFICIAL_SOURCE_GROUP"].strip() or "all"
+          if source_group == "all":
+              selected_groups = list(groups)
+          elif source_group in groups:
+              selected_groups = [source_group]
+          else:
+              raise SystemExit(f"unknown official_source_group: {source_group}")
+
+          requested = os.environ["OFFICIAL_CASE_IDS"].split()
+          requested_set = set(requested)
+          h200_cases = {
+              "flux_2_image_t2i",
+              "flux_2_klein_image_t2i",
+              "flux_2_ti2i",
+              "ltx_2.3_two_stage_t2v_2gpus",
+          }
+          include = []
+          for group in selected_groups:
+              for case_id in groups[group]:
+                  if requested_set and case_id not in requested_set:
+                      continue
+                  include.append(
+                      {
+                          "source_group": group,
+                          "case_id": case_id,
+                          "runner": "8-gpu-h200" if case_id in h200_cases else "1-gpu-h100",
+                      }
+                  )
+
+          known_cases = {case for cases in groups.values() for case in cases}
+          unknown = sorted(requested_set - known_cases)
+          if unknown:
+              raise SystemExit(f"unknown official case id(s): {' '.join(unknown)}")
+          if not include:
+              raise SystemExit("official case matrix is empty")
+
+          with open(os.environ["GITHUB_OUTPUT"], "a", encoding="utf-8") as f:
+              f.write(f"matrix={json.dumps({'include': include}, separators=(',', ':'))}\n")
+              f.write(f"case-count={len(include)}\n")
+          PY
+
+  official-gt-hopper:
+    needs: compute-official-gt-matrix
+    if: needs.compute-official-gt-matrix.result == 'success'
+    runs-on: ${{ matrix.runner }}
+    timeout-minutes: 240
+    strategy:
+      fail-fast: false
+      matrix: ${{ fromJson(needs.compute-official-gt-matrix.outputs.matrix) }}
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.ref || github.ref }}
+
+      - name: Checkout ci-data repro scripts
+        uses: actions/checkout@v4
+        with:
+          repository: sgl-project/ci-data
+          ref: main
+          path: ci-data
+          token: ${{ secrets.GH_PAT_FOR_NIGHTLY_CI_DATA }}
+
+      - name: Prepare sgl-kernel/dist for prebuilt wheel
+        if: inputs.kernel_artifact_run_id != ''
+        run: |
+          ls -alh sgl-kernel/dist || true
+          rm -rf sgl-kernel/dist/* || true
+
+      - name: Download prebuilt sgl-kernel wheel
+        if: inputs.kernel_artifact_run_id != ''
+        uses: actions/download-artifact@v4
+        with:
+          path: sgl-kernel/dist/
+          merge-multiple: true
+          name: wheel-python3.10-cuda13.0
+          run-id: ${{ inputs.kernel_artifact_run_id }}
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Install dependencies
+        run: |
+          CUSTOM_BUILD_SGL_KERNEL="${{ inputs.kernel_artifact_run_id != '' && 'true' || 'false' }}" \
+            bash scripts/ci/cuda/ci_install_dependency.sh diffusion
+
+      - name: Install official LTX-2.3 repro dependencies
+        if: matrix.source_group == 'ltx23'
+        run: |
+          UV_SYSTEM_PYTHON=1 uv pip install \
+            "transformers==4.52.4" \
+            openimageio \
+            --index-strategy unsafe-best-match \
+            --prerelease allow
+          UV_SYSTEM_PYTHON=1 uv pip uninstall kernels
+
+      - name: Checkout official Wan2.1 repo
+        if: matrix.source_group == 'wan21'
+        run: |
+          mkdir -p /tmp/mmgen-official-code
+          if [ ! -d /tmp/mmgen-official-code/Wan2.1/.git ]; then
+            git clone https://github.com/Wan-Video/Wan2.1.git /tmp/mmgen-official-code/Wan2.1
+          fi
+          git -C /tmp/mmgen-official-code/Wan2.1 fetch --depth 1 origin 9737cba9c1c3c4d04b33fcad41c111989865d315
+          git -C /tmp/mmgen-official-code/Wan2.1 checkout 9737cba9c1c3c4d04b33fcad41c111989865d315
+          git -C /tmp/mmgen-official-code/Wan2.1 rev-parse HEAD
+
+      - name: Checkout official LTX-2 repo
+        if: matrix.source_group == 'ltx23'
+        run: |
+          mkdir -p /tmp/mmgen-official-code
+          if [ ! -d /tmp/mmgen-official-code/LTX-2/.git ]; then
+            git clone https://github.com/Lightricks/LTX-2.git /tmp/mmgen-official-code/LTX-2
+          fi
+          git -C /tmp/mmgen-official-code/LTX-2 fetch --depth 1 origin 41d924371612b692c0fd1e4d9d94c3dfb3c02cb3
+          git -C /tmp/mmgen-official-code/LTX-2 checkout 41d924371612b692c0fd1e4d9d94c3dfb3c02cb3
+          git -C /tmp/mmgen-official-code/LTX-2 rev-parse HEAD
+
+      - name: Prepare official LTX-2.3 assets
+        if: matrix.source_group == 'ltx23'
+        env:
+          HF_TOKEN: ${{ secrets.SGLANG_DIFFUSION_CI_HF_TOKEN || secrets.HF_TOKEN }}
+          HUGGING_FACE_HUB_TOKEN: ${{ secrets.SGLANG_DIFFUSION_CI_HF_TOKEN || secrets.HF_TOKEN }}
+        run: |
+          python - <<'PY'
+          import os
+          from pathlib import Path
+
+          from huggingface_hub import hf_hub_download, snapshot_download
+
+          root = Path.home() / ".cache/sgl_diffusion/materialized_models/Lightricks__LTX-2.3-official-assets"
+          root.mkdir(parents=True, exist_ok=True)
+          text_encoder = root / "text_encoder"
+          tokenizer = root / "tokenizer"
+          text_encoder.mkdir(exist_ok=True)
+          tokenizer.mkdir(exist_ok=True)
+
+          gemma_root = Path(
+              snapshot_download(
+                  repo_id="google/gemma-3-12b-it-qat-q4_0-unquantized",
+                  ignore_patterns=["*.onnx", "*.msgpack"],
+                  max_workers=8,
+              )
+          )
+          tokenizer_names = {
+              "tokenizer.json",
+              "tokenizer.model",
+              "tokenizer_config.json",
+              "special_tokens_map.json",
+          }
+
+          def link(src: Path, dst: Path) -> None:
+              if dst.exists() or dst.is_symlink():
+                  return
+              try:
+                  dst.symlink_to(src)
+              except OSError:
+                  os.link(src, dst)
+
+          for src in gemma_root.iterdir():
+              if not src.is_file():
+                  continue
+              link(src, text_encoder / src.name)
+              if src.name in tokenizer_names:
+                  link(src, tokenizer / src.name)
+
+          for filename in (
+              "ltx-2.3-22b-distilled-lora-384.safetensors",
+              "ltx-2.3-spatial-upscaler-x2-1.1.safetensors",
+          ):
+              src = Path(hf_hub_download("Lightricks/LTX-2.3", filename))
+              link(src, root / filename)
+
+          print(root)
+          print(sorted(p.name for p in root.iterdir()))
+          PY
+
+      - name: Generate official output
+        env:
+          HF_TOKEN: ${{ secrets.SGLANG_DIFFUSION_CI_HF_TOKEN || secrets.HF_TOKEN }}
+          HUGGING_FACE_HUB_TOKEN: ${{ secrets.SGLANG_DIFFUSION_CI_HF_TOKEN || secrets.HF_TOKEN }}
+          RUNAI_STREAMER_MEMORY_LIMIT: 0
+          PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True
+          CUDA_VISIBLE_DEVICES: 0
+          CASE_ID: ${{ matrix.case_id }}
+          SOURCE_GROUP: ${{ matrix.source_group }}
+        run: |
+          git -C ci-data rev-parse HEAD
+          mkdir -p "python/${{ env.OUTPUT_NAME }}"
+          if [ "$SOURCE_GROUP" = "diffusers" ]; then
+            cd python
+            python ../ci-data/diffusion-ci/repro_scripts/gen_official_diffusion_gt.py \
+              --suite 1-gpu \
+              --out-dir ./${{ env.OUTPUT_NAME }} \
+              --case-ids "$CASE_ID" \
+              --dtype bf16 \
+              --device-map none \
+              --generator-device cuda
+          elif [ "$SOURCE_GROUP" = "wan21" ]; then
+            cd python
+            python ../ci-data/diffusion-ci/repro_scripts/gen_official_diffusion_gt.py \
+              --suite 1-gpu \
+              --out-dir ./${{ env.OUTPUT_NAME }} \
+              --case-ids "$CASE_ID" \
+              --wan-official-repo-dir /tmp/mmgen-official-code/Wan2.1 \
+              --dtype bf16 \
+              --device-map none \
+              --generator-device cuda
+          elif [ "$SOURCE_GROUP" = "ltx23" ]; then
+            cd python
+            set +e
+            PYTHONPATH=/tmp/mmgen-official-code/LTX-2/packages/ltx-core/src:/tmp/mmgen-official-code/LTX-2/packages/ltx-pipelines/src:$PWD:$PYTHONPATH \
+              python ../ci-data/diffusion-ci/repro_scripts/gen_official_ltx23.py \
+                --out-dir ./${{ env.OUTPUT_NAME }} \
+                --case-ids "$CASE_ID"
+            status=$?
+            set -e
+            if [ "$status" -ne 0 ]; then
+              find ./${{ env.OUTPUT_NAME }} -name 'official_ltx23_manifest.json' -print -exec cat {} \;
+              exit "$status"
+            fi
+          else
+            echo "Unknown SOURCE_GROUP=$SOURCE_GROUP"
+            exit 1
+          fi
+
+      - name: Upload artifact
+        uses: actions/upload-artifact@v4
+        with:
+          name: ${{ env.OUTPUT_NAME }}-${{ matrix.source_group }}-${{ matrix.case_id }}
+          path: |
+            python/${{ env.OUTPUT_NAME }}/*.jpg
+            python/${{ env.OUTPUT_NAME }}/*.png
+            python/${{ env.OUTPUT_NAME }}/official_gt_manifest_*.json
+            python/${{ env.OUTPUT_NAME }}/official_ltx23_manifest.json
+          retention-days: 7
+
+      - name: Publish official GT images to sgl-project/ci-data
+        env:
+          GITHUB_TOKEN: ${{ secrets.GH_PAT_FOR_NIGHTLY_CI_DATA }}
+        run: |
+          python scripts/ci/utils/diffusion/publish_diffusion_gt.py \
+            --source-dir python/${{ env.OUTPUT_NAME }} \
+            --target-dir "${{ env.PUBLISH_TARGET_DIR }}"
+
   compute-diffusion-partitions:
-    if: github.repository == 'sgl-project/sglang'
+    if: github.repository == 'sgl-project/sglang' && !inputs.run_official_cases
     runs-on: ubuntu-latest
     outputs:
       matrix-1gpu: ${{ steps.compute.outputs.matrix-1gpu }}