diff --git a/.github/workflows/pr-test-amd-rocm720.yml b/.github/workflows/pr-test-amd-rocm720.yml index 41d27ce32..07a8a9ce1 100644 --- a/.github/workflows/pr-test-amd-rocm720.yml +++ b/.github/workflows/pr-test-amd-rocm720.yml @@ -43,7 +43,6 @@ on: - stage-b-test-2-gpu-large-amd-rocm720 - multimodal-gen-test-1-gpu-amd-rocm720 - multimodal-gen-test-2-gpu-amd-rocm720 - - multimodal-gen-unit-test-amd-rocm720 - stage-c-test-4-gpu-amd-rocm720 - stage-c-dsv4-flash-fp4-fp8-amd-mi35x-rocm720 - stage-c-dsv4-pro-fp4-amd-mi35x-rocm720 @@ -813,6 +812,24 @@ jobs: echo "=== Post-test System Memory Status ===" free -h + # The mm_gen `unit` suite is portable CPU-style tests that finish in a + # few minutes, which does not justify a job of its own: on mi300 the + # container pull plus dependency install costs ~2h before any test runs. + # Ride along on the shard that is already warm. Runs after the diffusion + # tests, and on failure too, so a diffusion regression cannot hide it. + # Skips the CUDA-only ltx2_vae_channels_last memory-format assertion. + - name: Run diffusion unit tests + if: matrix.part == 0 && (success() || failure()) + timeout-minutes: 30 + run: | + docker exec \ + -e MIOPEN_USER_DB_PATH=/sgl-data/miopen-cache \ + -w /sglang-checkout/python \ + ci_sglang python3 sglang/multimodal_gen/test/run_suite.py \ + --suite unit \ + -k "not ltx2_vae_channels_last" \ + ${{ needs.check-changes.outputs.continue_on_error == 'true' && '--continue-on-error' || '' }} + - name: Upload diffusion failure artifacts if: always() uses: actions/upload-artifact@v4 @@ -839,7 +856,12 @@ jobs: fail-fast: false max-parallel: 1 matrix: - part: [0, 1, 2] # run_suite.py load-balances the suite's parametrized cases and standalone files over these 3 partitions + # run_suite.py load-balances the suite's parametrized cases and + # standalone files over these partitions, so the count is free to + # choose. Two shards measured 45-65 min of case work each, inside the + # 180 min step timeout, and halve the ~2h per-job container pull plus + # dependency install. + part: [0, 1] runs-on: ${{ format('linux-{0}-2gpu-sglang', inputs.runner_arch || 'mi300') }} steps: - name: Checkout code @@ -936,7 +958,7 @@ jobs: ci_sglang python3 sglang/multimodal_gen/test/run_suite.py \ --suite 2-gpu \ --partition-id ${{ matrix.part }} \ - --total-partitions 3 \ + --total-partitions 2 \ ${{ needs.check-changes.outputs.continue_on_error == 'true' && '--continue-on-error' || '' }} # Post-test diagnostics @@ -952,65 +974,6 @@ jobs: if-no-files-found: ignore retention-days: 7 - # AMD counterpart of the CUDA `multimodal-gen-unit-test` job - # (pr-test-multimodal-gen.yml): the mm_gen `unit` suite is portable, - # CPU-style unit tests (config / sampling params / storage / loaders / etc.) - # that don't require NVIDIA hardware, so they should run on AMD too. This - # closes the AMD coverage gap the dashboard surfaces for these tests. - multimodal-gen-unit-test-amd-rocm720: - name: ${{ format('multimodal-gen-unit-test-amd-rocm720 (linux-{0}-1gpu-sglang)', inputs.runner_arch || 'mi300') }} - needs: [check-changes, call-gate] - if: | - always() && !cancelled() && - ( - (contains(format(',{0},', inputs.target_stage || inputs.target_stage_select), ',multimodal-gen-unit-test-amd-rocm720,')) || - ( - !(inputs.target_stage || inputs.target_stage_select) && - (needs.call-gate.result == 'success' || needs.call-gate.result == 'skipped') && - needs.check-changes.outputs.multimodal_gen == 'true' - ) - ) - runs-on: ${{ format('linux-{0}-1gpu-sglang', inputs.runner_arch || 'mi300') }} - steps: - - name: Checkout code - uses: actions/checkout@v4 - with: - ref: ${{ inputs.pr_head_sha || inputs.ref || github.sha }} - - - name: Ensure VRAM is clear - run: bash scripts/ci/amd/ensure_vram_clear.sh rocm - - - name: Download artifacts - if: needs.check-changes.outputs.sgl_kernel == 'true' - uses: actions/download-artifact@v4 - with: - path: python/sglang/kernels/aot/dist/ - merge-multiple: true - pattern: wheel-python3.10-cuda12.9 - - - name: Start CI container - run: bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version "$ROCM_VERSION" - env: - GITHUB_WORKSPACE: ${{ github.workspace }} - - - name: Install dependencies - run: | - bash scripts/ci/amd/amd_ci_install_dependency.sh diffusion - - - name: Run diffusion unit tests - timeout-minutes: 60 - run: | - # Skip ltx2_vae_channels_last: it asserts CUDA `channels_last_3d` - # memory-format behavior that the ROCm conv path doesn't reproduce - # (CUDA-specific). The rest of the unit suite is portable. - docker exec \ - -e MIOPEN_USER_DB_PATH=/sgl-data/miopen-cache \ - -w /sglang-checkout/python \ - ci_sglang python3 sglang/multimodal_gen/test/run_suite.py \ - --suite unit \ - -k "not ltx2_vae_channels_last" \ - ${{ needs.check-changes.outputs.continue_on_error == 'true' && '--continue-on-error' || '' }} - wait-for-stage-b-amd-rocm720: needs: [check-changes, call-gate, wait-for-stage-a-amd-rocm720] if: | @@ -1438,7 +1401,6 @@ jobs: sgl-kernel-unit-test-2-gpu-amd-rocm720, multimodal-gen-test-1-gpu-amd-rocm720, multimodal-gen-test-2-gpu-amd-rocm720, - multimodal-gen-unit-test-amd-rocm720, wait-for-stage-a-amd-rocm720, stage-a-test-1-gpu-small-amd-rocm720, diff --git a/.github/workflows/pr-test-amd.yml b/.github/workflows/pr-test-amd.yml index 0fec8a8eb..0e99d1f25 100644 --- a/.github/workflows/pr-test-amd.yml +++ b/.github/workflows/pr-test-amd.yml @@ -32,7 +32,6 @@ on: - stage-b-test-2-gpu-large-amd - multimodal-gen-test-1-gpu-amd - multimodal-gen-test-2-gpu-amd - - multimodal-gen-unit-test-amd - stage-c-test-4-gpu-amd - stage-c-test-large-8-gpu-amd - stage-c-test-large-8-gpu-amd-mi35x @@ -778,6 +777,24 @@ jobs: echo "=== Post-test System Memory Status ===" free -h + # The mm_gen `unit` suite is portable CPU-style tests that finish in a + # few minutes, which does not justify a job of its own: on mi300 the + # container pull plus dependency install costs ~2h before any test runs. + # Ride along on the shard that is already warm. Runs after the diffusion + # tests, and on failure too, so a diffusion regression cannot hide it. + # Skips the CUDA-only ltx2_vae_channels_last memory-format assertion. + - name: Run diffusion unit tests + if: matrix.part == 0 && (success() || failure()) + timeout-minutes: 30 + run: | + docker exec \ + -e MIOPEN_USER_DB_PATH=/sgl-data/miopen-cache \ + -w /sglang-checkout/python \ + ci_sglang python3 sglang/multimodal_gen/test/run_suite.py \ + --suite unit \ + -k "not ltx2_vae_channels_last" \ + ${{ needs.check-changes.outputs.continue_on_error == 'true' && '--continue-on-error' || '' }} + - name: Upload diffusion failure artifacts if: always() uses: actions/upload-artifact@v4 @@ -803,7 +820,12 @@ jobs: strategy: fail-fast: false matrix: - part: [0, 1, 2] # run_suite.py load-balances the suite's parametrized cases and standalone files over these 3 partitions + # run_suite.py load-balances the suite's parametrized cases and + # standalone files over these partitions, so the count is free to + # choose. Two shards measured 45-65 min of case work each, inside the + # 180 min step timeout, and halve the ~2h per-job container pull plus + # dependency install. + part: [0, 1] runs-on: ${{ format('linux-{0}-2gpu-sglang', inputs.runner_arch || 'mi300') }} steps: - name: Checkout code @@ -900,7 +922,7 @@ jobs: ci_sglang python3 sglang/multimodal_gen/test/run_suite.py \ --suite 2-gpu \ --partition-id ${{ matrix.part }} \ - --total-partitions 3 \ + --total-partitions 2 \ ${{ needs.check-changes.outputs.continue_on_error == 'true' && '--continue-on-error' || '' }} # Post-test diagnostics @@ -916,65 +938,6 @@ jobs: if-no-files-found: ignore retention-days: 7 - # AMD counterpart of the CUDA `multimodal-gen-unit-test` job - # (pr-test-multimodal-gen.yml): the mm_gen `unit` suite is portable, - # CPU-style unit tests (config / sampling params / storage / loaders / etc.) - # that don't require NVIDIA hardware, so they should run on AMD too. This - # closes the AMD coverage gap the dashboard surfaces for these tests. - multimodal-gen-unit-test-amd: - name: ${{ format('multimodal-gen-unit-test-amd (linux-{0}-1gpu-sglang)', inputs.runner_arch || 'mi300') }} - needs: [check-changes, call-gate] - if: | - always() && !cancelled() && - ( - (contains(format(',{0},', inputs.target_stage || inputs.target_stage_select), ',multimodal-gen-unit-test-amd,')) || - ( - !(inputs.target_stage || inputs.target_stage_select) && - (needs.call-gate.result == 'success' || needs.call-gate.result == 'skipped') && - needs.check-changes.outputs.multimodal_gen == 'true' - ) - ) - runs-on: ${{ format('linux-{0}-1gpu-sglang', inputs.runner_arch || 'mi300') }} - steps: - - name: Checkout code - uses: actions/checkout@v4 - with: - ref: ${{ inputs.pr_head_sha || inputs.ref || github.sha }} - - - name: Ensure VRAM is clear - run: bash scripts/ci/amd/ensure_vram_clear.sh rocm - - - name: Download artifacts - if: needs.check-changes.outputs.sgl_kernel == 'true' - uses: actions/download-artifact@v4 - with: - path: python/sglang/kernels/aot/dist/ - merge-multiple: true - pattern: wheel-python3.10-cuda12.9 - - - name: Start CI container - run: bash scripts/ci/amd/amd_ci_start_container.sh - env: - GITHUB_WORKSPACE: ${{ github.workspace }} - - - name: Install dependencies - run: | - bash scripts/ci/amd/amd_ci_install_dependency.sh diffusion - - - name: Run diffusion unit tests - timeout-minutes: 60 - run: | - # Skip ltx2_vae_channels_last: it asserts CUDA `channels_last_3d` - # memory-format behavior that the ROCm conv path doesn't reproduce - # (CUDA-specific). The rest of the unit suite is portable. - docker exec \ - -e MIOPEN_USER_DB_PATH=/sgl-data/miopen-cache \ - -w /sglang-checkout/python \ - ci_sglang python3 sglang/multimodal_gen/test/run_suite.py \ - --suite unit \ - -k "not ltx2_vae_channels_last" \ - ${{ needs.check-changes.outputs.continue_on_error == 'true' && '--continue-on-error' || '' }} - wait-for-stage-b-amd: needs: [check-changes, call-gate, wait-for-stage-a-amd] if: | @@ -1290,7 +1253,6 @@ jobs: sgl-kernel-unit-test-2-gpu-amd, multimodal-gen-test-1-gpu-amd, multimodal-gen-test-2-gpu-amd, - multimodal-gen-unit-test-amd, wait-for-stage-a-amd, stage-a-test-1-gpu-small-amd, diff --git a/scripts/ci/utils/ci_coverage_report.py b/scripts/ci/utils/ci_coverage_report.py index e8c802471..1a405cef9 100755 --- a/scripts/ci/utils/ci_coverage_report.py +++ b/scripts/ci/utils/ci_coverage_report.py @@ -64,9 +64,9 @@ _MM_GEN_SUBDIR_BACKENDS = { "server/musa": ("MUSA",), "server/ascend": ("NPU",), "layers": ("CUDA",), - # unit/ are portable CPU-style unit tests. pr-test-amd now runs the `unit` - # suite on ROCm (multimodal-gen-unit-test-amd, both 7.0.0 and 7.2.0), so - # they are AMD-covered too, not CUDA-only. + # unit/ are portable CPU-style unit tests. The `unit` suite also runs on + # ROCm (both 7.0.0 and 7.2.0), as a step of multimodal-gen-test-1-gpu-amd + # part 0, so they are AMD-covered too, not CUDA-only. "unit": ("CUDA", "AMD"), "cli": ("CUDA",), "manual": ("CUDA",),