[AMD] CI: cut two setup cycles from the AMD multimodal-gen lanes (#34483)
Co-authored-by: quitenode <quitenode@users.noreply.github.com>
This commit is contained in:
Michael
@@ -43,7 +43,6 @@ on:
|
||||
- stage-b-test-2-gpu-large-amd-rocm720
|
||||
- multimodal-gen-test-1-gpu-amd-rocm720
|
||||
- multimodal-gen-test-2-gpu-amd-rocm720
|
||||
- multimodal-gen-unit-test-amd-rocm720
|
||||
- stage-c-test-4-gpu-amd-rocm720
|
||||
- stage-c-dsv4-flash-fp4-fp8-amd-mi35x-rocm720
|
||||
- stage-c-dsv4-pro-fp4-amd-mi35x-rocm720
|
||||
@@ -813,6 +812,24 @@ jobs:
|
||||
echo "=== Post-test System Memory Status ==="
|
||||
free -h
|
||||
|
||||
# The mm_gen `unit` suite is portable CPU-style tests that finish in a
|
||||
# few minutes, which does not justify a job of its own: on mi300 the
|
||||
# container pull plus dependency install costs ~2h before any test runs.
|
||||
# Ride along on the shard that is already warm. Runs after the diffusion
|
||||
# tests, and on failure too, so a diffusion regression cannot hide it.
|
||||
# Skips the CUDA-only ltx2_vae_channels_last memory-format assertion.
|
||||
- name: Run diffusion unit tests
|
||||
if: matrix.part == 0 && (success() || failure())
|
||||
timeout-minutes: 30
|
||||
run: |
|
||||
docker exec \
|
||||
-e MIOPEN_USER_DB_PATH=/sgl-data/miopen-cache \
|
||||
-w /sglang-checkout/python \
|
||||
ci_sglang python3 sglang/multimodal_gen/test/run_suite.py \
|
||||
--suite unit \
|
||||
-k "not ltx2_vae_channels_last" \
|
||||
${{ needs.check-changes.outputs.continue_on_error == 'true' && '--continue-on-error' || '' }}
|
||||
|
||||
- name: Upload diffusion failure artifacts
|
||||
if: always()
|
||||
uses: actions/upload-artifact@v4
|
||||
@@ -839,7 +856,12 @@ jobs:
|
||||
fail-fast: false
|
||||
max-parallel: 1
|
||||
matrix:
|
||||
part: [0, 1, 2] # run_suite.py load-balances the suite's parametrized cases and standalone files over these 3 partitions
|
||||
# run_suite.py load-balances the suite's parametrized cases and
|
||||
# standalone files over these partitions, so the count is free to
|
||||
# choose. Two shards measured 45-65 min of case work each, inside the
|
||||
# 180 min step timeout, and halve the ~2h per-job container pull plus
|
||||
# dependency install.
|
||||
part: [0, 1]
|
||||
runs-on: ${{ format('linux-{0}-2gpu-sglang', inputs.runner_arch || 'mi300') }}
|
||||
steps:
|
||||
- name: Checkout code
|
||||
@@ -936,7 +958,7 @@ jobs:
|
||||
ci_sglang python3 sglang/multimodal_gen/test/run_suite.py \
|
||||
--suite 2-gpu \
|
||||
--partition-id ${{ matrix.part }} \
|
||||
--total-partitions 3 \
|
||||
--total-partitions 2 \
|
||||
${{ needs.check-changes.outputs.continue_on_error == 'true' && '--continue-on-error' || '' }}
|
||||
|
||||
# Post-test diagnostics
|
||||
@@ -952,65 +974,6 @@ jobs:
|
||||
if-no-files-found: ignore
|
||||
retention-days: 7
|
||||
|
||||
# AMD counterpart of the CUDA `multimodal-gen-unit-test` job
|
||||
# (pr-test-multimodal-gen.yml): the mm_gen `unit` suite is portable,
|
||||
# CPU-style unit tests (config / sampling params / storage / loaders / etc.)
|
||||
# that don't require NVIDIA hardware, so they should run on AMD too. This
|
||||
# closes the AMD coverage gap the dashboard surfaces for these tests.
|
||||
multimodal-gen-unit-test-amd-rocm720:
|
||||
name: ${{ format('multimodal-gen-unit-test-amd-rocm720 (linux-{0}-1gpu-sglang)', inputs.runner_arch || 'mi300') }}
|
||||
needs: [check-changes, call-gate]
|
||||
if: |
|
||||
always() && !cancelled() &&
|
||||
(
|
||||
(contains(format(',{0},', inputs.target_stage || inputs.target_stage_select), ',multimodal-gen-unit-test-amd-rocm720,')) ||
|
||||
(
|
||||
!(inputs.target_stage || inputs.target_stage_select) &&
|
||||
(needs.call-gate.result == 'success' || needs.call-gate.result == 'skipped') &&
|
||||
needs.check-changes.outputs.multimodal_gen == 'true'
|
||||
)
|
||||
)
|
||||
runs-on: ${{ format('linux-{0}-1gpu-sglang', inputs.runner_arch || 'mi300') }}
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
ref: ${{ inputs.pr_head_sha || inputs.ref || github.sha }}
|
||||
|
||||
- name: Ensure VRAM is clear
|
||||
run: bash scripts/ci/amd/ensure_vram_clear.sh rocm
|
||||
|
||||
- name: Download artifacts
|
||||
if: needs.check-changes.outputs.sgl_kernel == 'true'
|
||||
uses: actions/download-artifact@v4
|
||||
with:
|
||||
path: python/sglang/kernels/aot/dist/
|
||||
merge-multiple: true
|
||||
pattern: wheel-python3.10-cuda12.9
|
||||
|
||||
- name: Start CI container
|
||||
run: bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version "$ROCM_VERSION"
|
||||
env:
|
||||
GITHUB_WORKSPACE: ${{ github.workspace }}
|
||||
|
||||
- name: Install dependencies
|
||||
run: |
|
||||
bash scripts/ci/amd/amd_ci_install_dependency.sh diffusion
|
||||
|
||||
- name: Run diffusion unit tests
|
||||
timeout-minutes: 60
|
||||
run: |
|
||||
# Skip ltx2_vae_channels_last: it asserts CUDA `channels_last_3d`
|
||||
# memory-format behavior that the ROCm conv path doesn't reproduce
|
||||
# (CUDA-specific). The rest of the unit suite is portable.
|
||||
docker exec \
|
||||
-e MIOPEN_USER_DB_PATH=/sgl-data/miopen-cache \
|
||||
-w /sglang-checkout/python \
|
||||
ci_sglang python3 sglang/multimodal_gen/test/run_suite.py \
|
||||
--suite unit \
|
||||
-k "not ltx2_vae_channels_last" \
|
||||
${{ needs.check-changes.outputs.continue_on_error == 'true' && '--continue-on-error' || '' }}
|
||||
|
||||
wait-for-stage-b-amd-rocm720:
|
||||
needs: [check-changes, call-gate, wait-for-stage-a-amd-rocm720]
|
||||
if: |
|
||||
@@ -1438,7 +1401,6 @@ jobs:
|
||||
sgl-kernel-unit-test-2-gpu-amd-rocm720,
|
||||
multimodal-gen-test-1-gpu-amd-rocm720,
|
||||
multimodal-gen-test-2-gpu-amd-rocm720,
|
||||
multimodal-gen-unit-test-amd-rocm720,
|
||||
|
||||
wait-for-stage-a-amd-rocm720,
|
||||
stage-a-test-1-gpu-small-amd-rocm720,
|
||||
|
||||
@@ -32,7 +32,6 @@ on:
|
||||
- stage-b-test-2-gpu-large-amd
|
||||
- multimodal-gen-test-1-gpu-amd
|
||||
- multimodal-gen-test-2-gpu-amd
|
||||
- multimodal-gen-unit-test-amd
|
||||
- stage-c-test-4-gpu-amd
|
||||
- stage-c-test-large-8-gpu-amd
|
||||
- stage-c-test-large-8-gpu-amd-mi35x
|
||||
@@ -778,6 +777,24 @@ jobs:
|
||||
echo "=== Post-test System Memory Status ==="
|
||||
free -h
|
||||
|
||||
# The mm_gen `unit` suite is portable CPU-style tests that finish in a
|
||||
# few minutes, which does not justify a job of its own: on mi300 the
|
||||
# container pull plus dependency install costs ~2h before any test runs.
|
||||
# Ride along on the shard that is already warm. Runs after the diffusion
|
||||
# tests, and on failure too, so a diffusion regression cannot hide it.
|
||||
# Skips the CUDA-only ltx2_vae_channels_last memory-format assertion.
|
||||
- name: Run diffusion unit tests
|
||||
if: matrix.part == 0 && (success() || failure())
|
||||
timeout-minutes: 30
|
||||
run: |
|
||||
docker exec \
|
||||
-e MIOPEN_USER_DB_PATH=/sgl-data/miopen-cache \
|
||||
-w /sglang-checkout/python \
|
||||
ci_sglang python3 sglang/multimodal_gen/test/run_suite.py \
|
||||
--suite unit \
|
||||
-k "not ltx2_vae_channels_last" \
|
||||
${{ needs.check-changes.outputs.continue_on_error == 'true' && '--continue-on-error' || '' }}
|
||||
|
||||
- name: Upload diffusion failure artifacts
|
||||
if: always()
|
||||
uses: actions/upload-artifact@v4
|
||||
@@ -803,7 +820,12 @@ jobs:
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
part: [0, 1, 2] # run_suite.py load-balances the suite's parametrized cases and standalone files over these 3 partitions
|
||||
# run_suite.py load-balances the suite's parametrized cases and
|
||||
# standalone files over these partitions, so the count is free to
|
||||
# choose. Two shards measured 45-65 min of case work each, inside the
|
||||
# 180 min step timeout, and halve the ~2h per-job container pull plus
|
||||
# dependency install.
|
||||
part: [0, 1]
|
||||
runs-on: ${{ format('linux-{0}-2gpu-sglang', inputs.runner_arch || 'mi300') }}
|
||||
steps:
|
||||
- name: Checkout code
|
||||
@@ -900,7 +922,7 @@ jobs:
|
||||
ci_sglang python3 sglang/multimodal_gen/test/run_suite.py \
|
||||
--suite 2-gpu \
|
||||
--partition-id ${{ matrix.part }} \
|
||||
--total-partitions 3 \
|
||||
--total-partitions 2 \
|
||||
${{ needs.check-changes.outputs.continue_on_error == 'true' && '--continue-on-error' || '' }}
|
||||
|
||||
# Post-test diagnostics
|
||||
@@ -916,65 +938,6 @@ jobs:
|
||||
if-no-files-found: ignore
|
||||
retention-days: 7
|
||||
|
||||
# AMD counterpart of the CUDA `multimodal-gen-unit-test` job
|
||||
# (pr-test-multimodal-gen.yml): the mm_gen `unit` suite is portable,
|
||||
# CPU-style unit tests (config / sampling params / storage / loaders / etc.)
|
||||
# that don't require NVIDIA hardware, so they should run on AMD too. This
|
||||
# closes the AMD coverage gap the dashboard surfaces for these tests.
|
||||
multimodal-gen-unit-test-amd:
|
||||
name: ${{ format('multimodal-gen-unit-test-amd (linux-{0}-1gpu-sglang)', inputs.runner_arch || 'mi300') }}
|
||||
needs: [check-changes, call-gate]
|
||||
if: |
|
||||
always() && !cancelled() &&
|
||||
(
|
||||
(contains(format(',{0},', inputs.target_stage || inputs.target_stage_select), ',multimodal-gen-unit-test-amd,')) ||
|
||||
(
|
||||
!(inputs.target_stage || inputs.target_stage_select) &&
|
||||
(needs.call-gate.result == 'success' || needs.call-gate.result == 'skipped') &&
|
||||
needs.check-changes.outputs.multimodal_gen == 'true'
|
||||
)
|
||||
)
|
||||
runs-on: ${{ format('linux-{0}-1gpu-sglang', inputs.runner_arch || 'mi300') }}
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
ref: ${{ inputs.pr_head_sha || inputs.ref || github.sha }}
|
||||
|
||||
- name: Ensure VRAM is clear
|
||||
run: bash scripts/ci/amd/ensure_vram_clear.sh rocm
|
||||
|
||||
- name: Download artifacts
|
||||
if: needs.check-changes.outputs.sgl_kernel == 'true'
|
||||
uses: actions/download-artifact@v4
|
||||
with:
|
||||
path: python/sglang/kernels/aot/dist/
|
||||
merge-multiple: true
|
||||
pattern: wheel-python3.10-cuda12.9
|
||||
|
||||
- name: Start CI container
|
||||
run: bash scripts/ci/amd/amd_ci_start_container.sh
|
||||
env:
|
||||
GITHUB_WORKSPACE: ${{ github.workspace }}
|
||||
|
||||
- name: Install dependencies
|
||||
run: |
|
||||
bash scripts/ci/amd/amd_ci_install_dependency.sh diffusion
|
||||
|
||||
- name: Run diffusion unit tests
|
||||
timeout-minutes: 60
|
||||
run: |
|
||||
# Skip ltx2_vae_channels_last: it asserts CUDA `channels_last_3d`
|
||||
# memory-format behavior that the ROCm conv path doesn't reproduce
|
||||
# (CUDA-specific). The rest of the unit suite is portable.
|
||||
docker exec \
|
||||
-e MIOPEN_USER_DB_PATH=/sgl-data/miopen-cache \
|
||||
-w /sglang-checkout/python \
|
||||
ci_sglang python3 sglang/multimodal_gen/test/run_suite.py \
|
||||
--suite unit \
|
||||
-k "not ltx2_vae_channels_last" \
|
||||
${{ needs.check-changes.outputs.continue_on_error == 'true' && '--continue-on-error' || '' }}
|
||||
|
||||
wait-for-stage-b-amd:
|
||||
needs: [check-changes, call-gate, wait-for-stage-a-amd]
|
||||
if: |
|
||||
@@ -1290,7 +1253,6 @@ jobs:
|
||||
sgl-kernel-unit-test-2-gpu-amd,
|
||||
multimodal-gen-test-1-gpu-amd,
|
||||
multimodal-gen-test-2-gpu-amd,
|
||||
multimodal-gen-unit-test-amd,
|
||||
|
||||
wait-for-stage-a-amd,
|
||||
stage-a-test-1-gpu-small-amd,
|
||||
|
||||
Reference in New Issue
Block a user