From b3632494235a22351e5b3185e3e1673b4937ab96 Mon Sep 17 00:00:00 2001 From: Michael <13900043+michaelzhang-ai@users.noreply.github.com> Date: Tue, 7 Jul 2026 16:02:58 -0700 Subject: [PATCH] [AMD] ci: run multimodal_gen unit suite on AMD (#30309) --- .github/workflows/pr-test-amd-rocm720.yml | 58 +++++++++++++++++++++ .github/workflows/pr-test-amd.yml | 61 +++++++++++++++++++++++ scripts/ci/utils/ci_coverage_report.py | 17 ++++++- 3 files changed, 135 insertions(+), 1 deletion(-) diff --git a/.github/workflows/pr-test-amd-rocm720.yml b/.github/workflows/pr-test-amd-rocm720.yml index 36b7de5fc..871aee1f1 100644 --- a/.github/workflows/pr-test-amd-rocm720.yml +++ b/.github/workflows/pr-test-amd-rocm720.yml @@ -46,6 +46,7 @@ on: - stage-b-test-2-gpu-large-amd-rocm720 - multimodal-gen-test-1-gpu-amd-rocm720 - multimodal-gen-test-2-gpu-amd-rocm720 + - multimodal-gen-unit-test-amd-rocm720 - stage-c-test-large-8-gpu-amd-rocm720 - stage-c-test-large-8-gpu-amd-mi35x-rocm720 - stage-b-test-large-8-gpu-mi35x-disaggregation-amd-rocm720 @@ -864,6 +865,62 @@ jobs: if-no-files-found: ignore retention-days: 7 + # ROCm 7.2.0 counterpart of `multimodal-gen-unit-test-amd`: run the portable + # mm_gen `unit` suite on ROCm 7.2.0 too. Skips the CUDA-only + # ltx2_vae_channels_last file (channels_last_3d assertions the ROCm conv path + # doesn't reproduce). + multimodal-gen-unit-test-amd-rocm720: + needs: [check-changes] + if: | + always() && + ( + (contains(format(',{0},', inputs.target_stage || inputs.target_stage_select), ',multimodal-gen-unit-test-amd-rocm720,')) || + ( + !(inputs.target_stage || inputs.target_stage_select) && + (!failure() && !cancelled()) && + ((needs.check-changes.outputs.main_package == 'true') || (needs.check-changes.outputs.sgl_kernel == 'true')) + ) + ) + runs-on: linux-mi325-1gpu-sglang + steps: + - name: Checkout code + uses: actions/checkout@v4 + with: + ref: ${{ inputs.pr_head_sha || inputs.ref || github.sha }} + + - name: Ensure VRAM is clear + run: bash scripts/ci/amd/ensure_vram_clear.sh rocm + + - name: Download artifacts + if: needs.check-changes.outputs.sgl_kernel == 'true' + uses: actions/download-artifact@v4 + with: + path: sgl-kernel/dist/ + merge-multiple: true + pattern: wheel-python3.10-cuda12.9 + + - name: Start CI container + run: bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version rocm720 + env: + GITHUB_WORKSPACE: ${{ github.workspace }} + + - name: Install dependencies + run: | + bash scripts/ci/amd/amd_ci_install_dependency.sh diffusion + docker exec ci_sglang pip install amdsmi + + - name: Run diffusion unit tests + timeout-minutes: 60 + run: | + # Skip ltx2_vae_channels_last: it asserts CUDA `channels_last_3d` + # memory-format behavior that the ROCm conv path doesn't reproduce. + docker exec \ + -e AITER_JIT_DIR=/sgl-data/aiter-kernels \ + -e MIOPEN_USER_DB_PATH=/sgl-data/miopen-cache \ + -w /sglang-checkout/python \ + ci_sglang python3 sglang/multimodal_gen/test/run_suite.py \ + --suite unit \ + -k "not ltx2_vae_channels_last" stage-c-test-4-gpu-amd-rocm720: needs: [check-changes, stage-b-test-1-gpu-small-amd-rocm720, stage-b-test-2-gpu-large-amd-rocm720] @@ -1261,6 +1318,7 @@ jobs: sgl-kernel-unit-test-2-gpu-amd-rocm720, multimodal-gen-test-1-gpu-amd-rocm720, multimodal-gen-test-2-gpu-amd-rocm720, + multimodal-gen-unit-test-amd-rocm720, stage-a-test-1-gpu-small-amd-rocm720, jit-kernel-unit-test-amd-rocm720, diff --git a/.github/workflows/pr-test-amd.yml b/.github/workflows/pr-test-amd.yml index c757a983c..50f692609 100644 --- a/.github/workflows/pr-test-amd.yml +++ b/.github/workflows/pr-test-amd.yml @@ -34,6 +34,7 @@ on: - stage-b-test-2-gpu-large-amd - multimodal-gen-test-1-gpu-amd - multimodal-gen-test-2-gpu-amd + - multimodal-gen-unit-test-amd - stage-c-test-4-gpu-amd - stage-c-test-large-8-gpu-amd - stage-c-test-large-8-gpu-amd-mi35x @@ -889,6 +890,65 @@ jobs: if-no-files-found: ignore retention-days: 7 + # AMD counterpart of the CUDA `multimodal-gen-unit-test` job + # (pr-test-multimodal-gen.yml): the mm_gen `unit` suite is portable, + # CPU-style unit tests (config / sampling params / storage / loaders / etc.) + # that don't require NVIDIA hardware, so they should run on AMD too. This + # closes the AMD coverage gap the dashboard surfaces for these tests. + multimodal-gen-unit-test-amd: + name: ${{ format('multimodal-gen-unit-test-amd (linux-{0}-1gpu-sglang)', inputs.runner_arch || 'mi325') }} + needs: [check-changes, call-gate] + if: | + always() && !cancelled() && + ( + (contains(format(',{0},', inputs.target_stage || inputs.target_stage_select), ',multimodal-gen-unit-test-amd,')) || + ( + !(inputs.target_stage || inputs.target_stage_select) && + (needs.call-gate.result == 'success' || needs.call-gate.result == 'skipped') && + needs.check-changes.outputs.multimodal_gen == 'true' + ) + ) + runs-on: ${{ format('linux-{0}-1gpu-sglang', inputs.runner_arch || 'mi325') }} + steps: + - name: Checkout code + uses: actions/checkout@v4 + with: + ref: ${{ inputs.pr_head_sha || inputs.ref || github.sha }} + + - name: Ensure VRAM is clear + run: bash scripts/ci/amd/ensure_vram_clear.sh rocm + + - name: Download artifacts + if: needs.check-changes.outputs.sgl_kernel == 'true' + uses: actions/download-artifact@v4 + with: + path: sgl-kernel/dist/ + merge-multiple: true + pattern: wheel-python3.10-cuda12.9 + + - name: Start CI container + run: bash scripts/ci/amd/amd_ci_start_container.sh + env: + GITHUB_WORKSPACE: ${{ github.workspace }} + + - name: Install dependencies + run: | + bash scripts/ci/amd/amd_ci_install_dependency.sh diffusion + + - name: Run diffusion unit tests + timeout-minutes: 60 + run: | + # Skip ltx2_vae_channels_last: it asserts CUDA `channels_last_3d` + # memory-format behavior that the ROCm conv path doesn't reproduce + # (CUDA-specific). The rest of the unit suite is portable. + docker exec \ + -e AITER_JIT_DIR=/sgl-data/aiter-kernels \ + -e MIOPEN_USER_DB_PATH=/sgl-data/miopen-cache \ + -w /sglang-checkout/python \ + ci_sglang python3 sglang/multimodal_gen/test/run_suite.py \ + --suite unit \ + -k "not ltx2_vae_channels_last" \ + ${{ needs.check-changes.outputs.continue_on_error == 'true' && '--continue-on-error' || '' }} wait-for-stage-b-amd: needs: [check-changes, call-gate, wait-for-stage-a-amd] @@ -1204,6 +1264,7 @@ jobs: sgl-kernel-unit-test-2-gpu-amd, multimodal-gen-test-1-gpu-amd, multimodal-gen-test-2-gpu-amd, + multimodal-gen-unit-test-amd, wait-for-stage-a-amd, stage-a-test-1-gpu-small-amd, diff --git a/scripts/ci/utils/ci_coverage_report.py b/scripts/ci/utils/ci_coverage_report.py index e04218f18..0cc7f0a06 100755 --- a/scripts/ci/utils/ci_coverage_report.py +++ b/scripts/ci/utils/ci_coverage_report.py @@ -64,9 +64,24 @@ _MM_GEN_SUBDIR_BACKENDS = { "server/musa": ("MUSA",), "server/ascend": ("NPU",), "layers": ("CUDA",), - "unit": ("CUDA",), + # unit/ are portable CPU-style unit tests. pr-test-amd now runs the `unit` + # suite on ROCm (multimodal-gen-unit-test-amd, both 7.0.0 and 7.2.0), so + # they are AMD-covered too, not CUDA-only. + "unit": ("CUDA", "AMD"), "cli": ("CUDA",), "manual": ("CUDA",), + # Standalone server/CLI single-file tests (restructured out of server/). + # Run on CUDA CI; AMD parity for these standalone files is TBD, so + # CUDA-only for now (previously matched no rule and were dropped entirely). + "single_test_file": ("CUDA",), + "single_test_file/component_accuracy": ("CUDA",), + # Nested unit suites run only on the CUDA lane today (they are not part of + # the AMD `unit` suite that multimodal-gen-unit-test-amd executes). + "unit/realtime": ("CUDA",), + "unit/sana_wm": ("CUDA",), + "unit/progressive_resolution": ("CUDA",), + # musa-named unit layer kernels. + "unit/musa/layers": ("MUSA",), } # Filenames that match `test_*.py` by convention but contain no real tests