From d2f054d9168516e7e580ea060d36739eeb829b93 Mon Sep 17 00:00:00 2001 From: Mick Date: Sun, 13 Sep 2026 13:59:43 +0800 Subject: [PATCH] [diffusion] CI: make diffusion GT generation usable without the publish token, and cover the 5090 lane (#39255) Co-authored-by: Mick Qian Co-authored-by: Claude Opus 5 --- .github/workflows/diffusion-ci-gt-gen.yml | 89 ++++++++++++++++++++++- 1 file changed, 85 insertions(+), 4 deletions(-) diff --git a/.github/workflows/diffusion-ci-gt-gen.yml b/.github/workflows/diffusion-ci-gt-gen.yml index 6bdcf8dad..75c46d96c 100644 --- a/.github/workflows/diffusion-ci-gt-gen.yml +++ b/.github/workflows/diffusion-ci-gt-gen.yml @@ -48,6 +48,11 @@ on: required: false default: 'main' type: string + publish: + description: 'Publish the generated GT to sgl-project/ci-data-diffusion. false only uploads the run artifacts, which needs no GH_PAT_FOR_NIGHTLY_CI_DATA.' + required: false + default: true + type: boolean concurrency: group: diffusion-ci-gt-gen-${{ github.ref }}-${{ inputs.output_name || inputs.case_ids || inputs.official_case_ids || inputs.official_source_group || inputs.run_official_cases || 'default' }} @@ -73,6 +78,7 @@ jobs: case-count: ${{ steps.compute.outputs.case-count }} steps: - name: Verify write access to sgl-project/ci-data-diffusion + if: ${{ inputs.publish }} env: GH_TOKEN: ${{ secrets.GH_PAT_FOR_NIGHTLY_CI_DATA }} run: | @@ -307,7 +313,7 @@ jobs: - name: Validate generated GT images env: - GITHUB_TOKEN: ${{ secrets.GH_PAT_FOR_NIGHTLY_CI_DATA }} + GITHUB_TOKEN: ${{ inputs.publish && secrets.GH_PAT_FOR_NIGHTLY_CI_DATA || github.token }} run: | python scripts/ci/utils/diffusion/publish_diffusion_gt.py \ --source-dir python/${{ env.OUTPUT_NAME }} \ @@ -315,6 +321,7 @@ jobs: --check-only - name: Publish official GT images to sgl-project/ci-data-diffusion + if: ${{ inputs.publish }} env: GITHUB_TOKEN: ${{ secrets.GH_PAT_FOR_NIGHTLY_CI_DATA }} run: | @@ -347,6 +354,7 @@ jobs: python-version: '3.10' - name: Verify write access to sgl-project/ci-data-diffusion + if: ${{ inputs.publish }} env: GH_TOKEN: ${{ secrets.GH_PAT_FOR_NIGHTLY_CI_DATA }} run: | @@ -427,7 +435,7 @@ jobs: - name: Validate generated GT images env: - GITHUB_TOKEN: ${{ secrets.GH_PAT_FOR_NIGHTLY_CI_DATA }} + GITHUB_TOKEN: ${{ inputs.publish && secrets.GH_PAT_FOR_NIGHTLY_CI_DATA || github.token }} run: | python scripts/ci/utils/diffusion/publish_diffusion_gt.py \ --source-dir python/${{ env.OUTPUT_NAME }} \ @@ -435,6 +443,7 @@ jobs: --check-only - name: Publish GT images to sgl-project/ci-data-diffusion + if: ${{ inputs.publish }} env: GITHUB_TOKEN: ${{ secrets.GH_PAT_FOR_NIGHTLY_CI_DATA }} run: | @@ -504,7 +513,7 @@ jobs: - name: Validate generated GT images env: - GITHUB_TOKEN: ${{ secrets.GH_PAT_FOR_NIGHTLY_CI_DATA }} + GITHUB_TOKEN: ${{ inputs.publish && secrets.GH_PAT_FOR_NIGHTLY_CI_DATA || github.token }} run: | python scripts/ci/utils/diffusion/publish_diffusion_gt.py \ --source-dir python/${{ env.OUTPUT_NAME }} \ @@ -512,6 +521,7 @@ jobs: --check-only - name: Publish GT images to sgl-project/ci-data-diffusion + if: ${{ inputs.publish }} env: GITHUB_TOKEN: ${{ secrets.GH_PAT_FOR_NIGHTLY_CI_DATA }} run: | @@ -579,7 +589,7 @@ jobs: - name: Validate generated GT images env: - GITHUB_TOKEN: ${{ secrets.GH_PAT_FOR_NIGHTLY_CI_DATA }} + GITHUB_TOKEN: ${{ inputs.publish && secrets.GH_PAT_FOR_NIGHTLY_CI_DATA || github.token }} run: | python scripts/ci/utils/diffusion/publish_diffusion_gt.py \ --source-dir python/${{ env.OUTPUT_NAME }} \ @@ -587,6 +597,77 @@ jobs: --check-only - name: Publish GT images to sgl-project/ci-data-diffusion + if: ${{ inputs.publish }} + env: + GITHUB_TOKEN: ${{ secrets.GH_PAT_FOR_NIGHTLY_CI_DATA }} + run: | + python scripts/ci/utils/diffusion/publish_diffusion_gt.py \ + --source-dir python/${{ env.OUTPUT_NAME }} \ + --target-dir "${{ env.PUBLISH_TARGET_DIR }}" + + multimodal-diffusion-gen-5090: + # The 5090 canary suite is six cases; it is generated whole rather than + # partitioned, so it needs no entry in the partition plan. + needs: compute-diffusion-partitions + if: needs.compute-diffusion-partitions.result == 'success' + runs-on: 1-gpu-5090 + timeout-minutes: 150 + steps: + - name: Checkout code + uses: actions/checkout@v4 + with: + ref: ${{ inputs.ref || github.ref }} + + - name: Prepare AOT dist for prebuilt wheel + if: inputs.kernel_artifact_run_id != '' + run: | + ls -alh python/sglang/kernels/aot/dist || true + rm -rf python/sglang/kernels/aot/dist/* || true + + - name: Download prebuilt sgl-kernel wheel + if: inputs.kernel_artifact_run_id != '' + uses: actions/download-artifact@v4 + with: + path: python/sglang/kernels/aot/dist/ + merge-multiple: true + name: wheel-python3.10-cuda13.0 + run-id: ${{ inputs.kernel_artifact_run_id }} + github-token: ${{ secrets.GITHUB_TOKEN }} + + - name: Install dependencies + run: | + CUSTOM_BUILD_SGL_KERNEL="${{ inputs.kernel_artifact_run_id != '' && 'true' || 'false' }}" \ + bash scripts/ci/cuda/ci_install_dependency.sh diffusion + + - name: Generate outputs + env: + RUNAI_STREAMER_MEMORY_LIMIT: 0 + run: | + cd python + python -m sglang.multimodal_gen.test.scripts.gen_diffusion_ci_outputs \ + --suite 1-gpu-5090 \ + --out-dir ./${{ env.OUTPUT_NAME }} \ + ${{ inputs.case_ids != '' && format('--case-ids {0}', inputs.case_ids) || '' }} + + - name: Upload artifact + uses: actions/upload-artifact@v4 + with: + name: ${{ env.OUTPUT_NAME }}-5090 + path: python/${{ env.OUTPUT_NAME }} + retention-days: 7 + if-no-files-found: ignore + + - name: Validate generated GT images + env: + GITHUB_TOKEN: ${{ inputs.publish && secrets.GH_PAT_FOR_NIGHTLY_CI_DATA || github.token }} + run: | + python scripts/ci/utils/diffusion/publish_diffusion_gt.py \ + --source-dir python/${{ env.OUTPUT_NAME }} \ + --target-dir "${{ env.PUBLISH_TARGET_DIR }}" \ + --check-only + + - name: Publish GT images to sgl-project/ci-data-diffusion + if: ${{ inputs.publish }} env: GITHUB_TOKEN: ${{ secrets.GH_PAT_FOR_NIGHTLY_CI_DATA }} run: |