[CI] Fix max_parallel for scheduled runs (#17046)
This commit is contained in:
@@ -110,12 +110,16 @@ jobs:
|
|||||||
- "python/*.toml"
|
- "python/*.toml"
|
||||||
- ".github/workflows/pr-test.yml"
|
- ".github/workflows/pr-test.yml"
|
||||||
|
|
||||||
- name: Set max-parallel based on high-priority label
|
- name: Set max-parallel based on run type
|
||||||
id: set-parallel
|
id: set-parallel
|
||||||
run: |
|
run: |
|
||||||
if [[ "${{ github.event_name }}" == "pull_request" && "${{ contains(github.event.pull_request.labels.*.name, 'high priority') }}" == "true" ]]; then
|
# Scheduled runs and high-priority PRs get full parallelism
|
||||||
echo "max_parallel=15" >> $GITHUB_OUTPUT
|
if [[ "${{ github.event_name }}" == "schedule" ]]; then
|
||||||
echo "High priority PR detected, setting max_parallel to 15"
|
echo "max_parallel=14" >> $GITHUB_OUTPUT
|
||||||
|
echo "Scheduled run detected, setting max_parallel to 14"
|
||||||
|
elif [[ "${{ github.event_name }}" == "pull_request" && "${{ contains(github.event.pull_request.labels.*.name, 'high priority') }}" == "true" ]]; then
|
||||||
|
echo "max_parallel=14" >> $GITHUB_OUTPUT
|
||||||
|
echo "High priority PR detected, setting max_parallel to 14"
|
||||||
else
|
else
|
||||||
echo "max_parallel=3" >> $GITHUB_OUTPUT
|
echo "max_parallel=3" >> $GITHUB_OUTPUT
|
||||||
echo "Using default max_parallel of 3"
|
echo "Using default max_parallel of 3"
|
||||||
@@ -772,6 +776,53 @@ jobs:
|
|||||||
fi
|
fi
|
||||||
python3 run_suite.py --hw cuda --suite stage-b-test-large-2-gpu --auto-partition-id ${{ matrix.partition }} --auto-partition-size 2 $CONTINUE_ON_ERROR_FLAG
|
python3 run_suite.py --hw cuda --suite stage-b-test-large-2-gpu --auto-partition-id ${{ matrix.partition }} --auto-partition-size 2 $CONTINUE_ON_ERROR_FLAG
|
||||||
|
|
||||||
|
stage-b-test-4-gpu-b200:
|
||||||
|
needs: [check-changes, call-gate, stage-a-test-1, sgl-kernel-build-wheels]
|
||||||
|
if: |
|
||||||
|
always() &&
|
||||||
|
(
|
||||||
|
(inputs.target_stage == 'stage-b-test-4-gpu-b200') ||
|
||||||
|
(
|
||||||
|
!inputs.target_stage &&
|
||||||
|
(github.event_name == 'schedule' || (!failure() && !cancelled())) &&
|
||||||
|
((needs.check-changes.outputs.main_package == 'true') || (needs.check-changes.outputs.sgl_kernel == 'true'))
|
||||||
|
)
|
||||||
|
)
|
||||||
|
runs-on: ${{ needs.check-changes.outputs.b200_runner }}
|
||||||
|
env:
|
||||||
|
RUNNER_LABELS: ${{ needs.check-changes.outputs.b200_runner }}
|
||||||
|
strategy:
|
||||||
|
fail-fast: false
|
||||||
|
|
||||||
|
steps:
|
||||||
|
- name: Checkout code
|
||||||
|
uses: actions/checkout@v4
|
||||||
|
with:
|
||||||
|
ref: ${{ inputs.pr_head_sha || inputs.ref || github.sha }}
|
||||||
|
|
||||||
|
- name: Download artifacts
|
||||||
|
if: needs.check-changes.outputs.sgl_kernel == 'true'
|
||||||
|
uses: actions/download-artifact@v6
|
||||||
|
with:
|
||||||
|
path: sgl-kernel/dist/
|
||||||
|
merge-multiple: true
|
||||||
|
pattern: wheel-python3.10-cuda12.9
|
||||||
|
|
||||||
|
- name: Install dependencies
|
||||||
|
timeout-minutes: 10
|
||||||
|
run: |
|
||||||
|
CUSTOM_BUILD_SGL_KERNEL=${{needs.check-changes.outputs.sgl_kernel}} IS_BLACKWELL=1 bash scripts/ci/ci_install_dependency.sh
|
||||||
|
|
||||||
|
- name: Run test
|
||||||
|
timeout-minutes: 30
|
||||||
|
run: |
|
||||||
|
cd test
|
||||||
|
CONTINUE_ON_ERROR_FLAG=""
|
||||||
|
if [[ "${{ needs.check-changes.outputs.continue_on_error }}" == "true" ]]; then
|
||||||
|
CONTINUE_ON_ERROR_FLAG="--continue-on-error"
|
||||||
|
fi
|
||||||
|
IS_BLACKWELL=1 python3 run_suite.py --hw cuda --suite stage-b-test-4-gpu-b200 $CONTINUE_ON_ERROR_FLAG
|
||||||
|
|
||||||
stage-c-test-large-4-gpu:
|
stage-c-test-large-4-gpu:
|
||||||
needs: [check-changes, call-gate, stage-b-test-small-1-gpu, stage-b-test-large-1-gpu, stage-b-test-large-2-gpu, sgl-kernel-build-wheels]
|
needs: [check-changes, call-gate, stage-b-test-small-1-gpu, stage-b-test-large-1-gpu, stage-b-test-large-2-gpu, sgl-kernel-build-wheels]
|
||||||
if: |
|
if: |
|
||||||
@@ -1002,53 +1053,6 @@ jobs:
|
|||||||
fi
|
fi
|
||||||
python3 run_suite.py --suite quantization_test $RETRY_FLAG $CONTINUE_ON_ERROR_FLAG
|
python3 run_suite.py --suite quantization_test $RETRY_FLAG $CONTINUE_ON_ERROR_FLAG
|
||||||
|
|
||||||
stage-b-test-4-gpu-b200:
|
|
||||||
needs: [check-changes, call-gate, stage-a-test-1, sgl-kernel-build-wheels]
|
|
||||||
if: |
|
|
||||||
always() &&
|
|
||||||
(
|
|
||||||
(inputs.target_stage == 'stage-b-test-4-gpu-b200') ||
|
|
||||||
(
|
|
||||||
!inputs.target_stage &&
|
|
||||||
(github.event_name == 'schedule' || (!failure() && !cancelled())) &&
|
|
||||||
((needs.check-changes.outputs.main_package == 'true') || (needs.check-changes.outputs.sgl_kernel == 'true'))
|
|
||||||
)
|
|
||||||
)
|
|
||||||
runs-on: ${{ needs.check-changes.outputs.b200_runner }}
|
|
||||||
env:
|
|
||||||
RUNNER_LABELS: ${{ needs.check-changes.outputs.b200_runner }}
|
|
||||||
strategy:
|
|
||||||
fail-fast: false
|
|
||||||
|
|
||||||
steps:
|
|
||||||
- name: Checkout code
|
|
||||||
uses: actions/checkout@v4
|
|
||||||
with:
|
|
||||||
ref: ${{ inputs.pr_head_sha || inputs.ref || github.sha }}
|
|
||||||
|
|
||||||
- name: Download artifacts
|
|
||||||
if: needs.check-changes.outputs.sgl_kernel == 'true'
|
|
||||||
uses: actions/download-artifact@v6
|
|
||||||
with:
|
|
||||||
path: sgl-kernel/dist/
|
|
||||||
merge-multiple: true
|
|
||||||
pattern: wheel-python3.10-cuda12.9
|
|
||||||
|
|
||||||
- name: Install dependencies
|
|
||||||
timeout-minutes: 10
|
|
||||||
run: |
|
|
||||||
CUSTOM_BUILD_SGL_KERNEL=${{needs.check-changes.outputs.sgl_kernel}} IS_BLACKWELL=1 bash scripts/ci/ci_install_dependency.sh
|
|
||||||
|
|
||||||
- name: Run test
|
|
||||||
timeout-minutes: 30
|
|
||||||
run: |
|
|
||||||
cd test
|
|
||||||
CONTINUE_ON_ERROR_FLAG=""
|
|
||||||
if [[ "${{ needs.check-changes.outputs.continue_on_error }}" == "true" ]]; then
|
|
||||||
CONTINUE_ON_ERROR_FLAG="--continue-on-error"
|
|
||||||
fi
|
|
||||||
IS_BLACKWELL=1 python3 run_suite.py --hw cuda --suite stage-b-test-4-gpu-b200 $CONTINUE_ON_ERROR_FLAG
|
|
||||||
|
|
||||||
unit-test-backend-4-gpu:
|
unit-test-backend-4-gpu:
|
||||||
needs: [check-changes, call-gate, stage-b-test-small-1-gpu, stage-b-test-large-1-gpu, stage-b-test-large-2-gpu, stage-b-test-4-gpu-b200]
|
needs: [check-changes, call-gate, stage-b-test-small-1-gpu, stage-b-test-large-1-gpu, stage-b-test-large-2-gpu, stage-b-test-4-gpu-b200]
|
||||||
if: |
|
if: |
|
||||||
|
|||||||
Reference in New Issue
Block a user