Re-enable unit-test-deepep-8-gpu and unit-test-backend-4-gpu-gb200 (#17438)
This commit is contained in:
Alison Shao
+100
-102
@@ -251,7 +251,6 @@ jobs:
|
|||||||
needs: [check-changes, call-gate, wait-for-stage-a]
|
needs: [check-changes, call-gate, wait-for-stage-a]
|
||||||
# Only run for PRs (not scheduled) and when not targeting a specific stage
|
# Only run for PRs (not scheduled) and when not targeting a specific stage
|
||||||
# Skip if call-gate failed (stage-b jobs will be skipped, nothing to wait for)
|
# Skip if call-gate failed (stage-b jobs will be skipped, nothing to wait for)
|
||||||
# !cancelled() ensures this job respects workflow cancellation from concurrency group
|
|
||||||
if: |
|
if: |
|
||||||
always() &&
|
always() &&
|
||||||
!cancelled() &&
|
!cancelled() &&
|
||||||
@@ -1645,54 +1644,54 @@ jobs:
|
|||||||
fi
|
fi
|
||||||
python3 run_suite.py --suite per-commit-4-gpu-deepep $RETRY_FLAG $CONTINUE_ON_ERROR_FLAG
|
python3 run_suite.py --suite per-commit-4-gpu-deepep $RETRY_FLAG $CONTINUE_ON_ERROR_FLAG
|
||||||
|
|
||||||
# Disabled: IBGDA/cudaHostRegister environment issues on 8-GPU runner, see #17175
|
unit-test-deepep-8-gpu:
|
||||||
# unit-test-deepep-8-gpu:
|
needs: [check-changes, call-gate, wait-for-stage-b]
|
||||||
# needs: [check-changes, call-gate, stage-b-test-small-1-gpu, stage-b-test-large-1-gpu, stage-b-test-large-2-gpu, stage-b-test-4-gpu-b200]
|
if: |
|
||||||
# if: |
|
always() &&
|
||||||
# always() &&
|
(
|
||||||
# (
|
(inputs.target_stage == 'unit-test-deepep-8-gpu') ||
|
||||||
# (inputs.target_stage == 'unit-test-deepep-8-gpu') ||
|
(
|
||||||
# (
|
!inputs.target_stage &&
|
||||||
# !inputs.target_stage &&
|
((github.event_name == 'schedule' || inputs.test_parallel_dispatch == true) || (!failure() && !cancelled())) &&
|
||||||
# (github.event_name == 'schedule' || (!failure() && !cancelled())) &&
|
((needs.check-changes.outputs.main_package == 'true') || (needs.check-changes.outputs.sgl_kernel == 'true'))
|
||||||
# ((needs.check-changes.outputs.main_package == 'true') || (needs.check-changes.outputs.sgl_kernel == 'true'))
|
)
|
||||||
# )
|
)
|
||||||
# )
|
runs-on: 8-gpu-h200
|
||||||
# runs-on: 8-gpu-h200
|
timeout-minutes: 60
|
||||||
# env:
|
env:
|
||||||
# RUNNER_LABELS: 8-gpu-h200
|
RUNNER_LABELS: 8-gpu-h200
|
||||||
# steps:
|
steps:
|
||||||
# - name: Checkout code
|
- name: Checkout code
|
||||||
# uses: actions/checkout@v4
|
uses: actions/checkout@v4
|
||||||
# with:
|
with:
|
||||||
# ref: ${{ inputs.pr_head_sha || inputs.ref || github.sha }}
|
ref: ${{ inputs.pr_head_sha || inputs.ref || github.sha }}
|
||||||
#
|
|
||||||
# - name: Download artifacts
|
- name: Download artifacts
|
||||||
# if: needs.check-changes.outputs.sgl_kernel == 'true'
|
if: needs.check-changes.outputs.sgl_kernel == 'true'
|
||||||
# uses: actions/download-artifact@v4
|
uses: actions/download-artifact@v4
|
||||||
# with:
|
with:
|
||||||
# path: sgl-kernel/dist/
|
path: sgl-kernel/dist/
|
||||||
# merge-multiple: true
|
merge-multiple: true
|
||||||
# pattern: wheel-python3.10-cuda12.9
|
pattern: wheel-python3.10-cuda12.9
|
||||||
#
|
|
||||||
# - name: Install dependencies
|
- name: Install dependencies
|
||||||
# timeout-minutes: 10
|
timeout-minutes: 10
|
||||||
# run: |
|
run: |
|
||||||
# CUSTOM_BUILD_SGL_KERNEL=${{needs.check-changes.outputs.sgl_kernel}} bash scripts/ci/ci_install_deepep.sh
|
CUSTOM_BUILD_SGL_KERNEL=${{needs.check-changes.outputs.sgl_kernel}} bash scripts/ci/ci_install_deepep.sh
|
||||||
#
|
|
||||||
# - name: Run test
|
- name: Run test
|
||||||
# timeout-minutes: 20
|
timeout-minutes: 45
|
||||||
# run: |
|
run: |
|
||||||
# cd test/srt
|
cd test/srt
|
||||||
# RETRY_FLAG=""
|
RETRY_FLAG=""
|
||||||
# if [[ "${{ needs.check-changes.outputs.enable_retry }}" == "true" ]]; then
|
if [[ "${{ needs.check-changes.outputs.enable_retry }}" == "true" ]]; then
|
||||||
# RETRY_FLAG="--enable-retry"
|
RETRY_FLAG="--enable-retry"
|
||||||
# fi
|
fi
|
||||||
# CONTINUE_ON_ERROR_FLAG=""
|
CONTINUE_ON_ERROR_FLAG=""
|
||||||
# if [[ "${{ needs.check-changes.outputs.continue_on_error }}" == "true" ]]; then
|
if [[ "${{ needs.check-changes.outputs.continue_on_error }}" == "true" ]]; then
|
||||||
# CONTINUE_ON_ERROR_FLAG="--continue-on-error"
|
CONTINUE_ON_ERROR_FLAG="--continue-on-error"
|
||||||
# fi
|
fi
|
||||||
# python3 run_suite.py --suite per-commit-8-gpu-h200-deepep $RETRY_FLAG $CONTINUE_ON_ERROR_FLAG
|
python3 run_suite.py --suite per-commit-8-gpu-h200-deepep $RETRY_FLAG $CONTINUE_ON_ERROR_FLAG
|
||||||
|
|
||||||
unit-test-backend-4-gpu-b200:
|
unit-test-backend-4-gpu-b200:
|
||||||
needs: [check-changes, call-gate, wait-for-stage-b]
|
needs: [check-changes, call-gate, wait-for-stage-b]
|
||||||
@@ -1748,57 +1747,56 @@ jobs:
|
|||||||
fi
|
fi
|
||||||
IS_BLACKWELL=1 python3 run_suite.py --suite per-commit-4-gpu-b200 --auto-partition-id ${{ matrix.part }} --auto-partition-size 3 --timeout-per-file 1800 $RETRY_FLAG $CONTINUE_ON_ERROR_FLAG
|
IS_BLACKWELL=1 python3 run_suite.py --suite per-commit-4-gpu-b200 --auto-partition-id ${{ matrix.part }} --auto-partition-size 3 --timeout-per-file 1800 $RETRY_FLAG $CONTINUE_ON_ERROR_FLAG
|
||||||
|
|
||||||
# Disabled: GB200 runner needs repair
|
unit-test-backend-4-gpu-gb200:
|
||||||
# unit-test-backend-4-gpu-gb200:
|
needs: [check-changes, call-gate, wait-for-stage-b, sgl-kernel-build-wheels-arm]
|
||||||
# needs: [check-changes, call-gate, wait-for-stage-b, sgl-kernel-build-wheels-arm]
|
if: |
|
||||||
# if: |
|
always() &&
|
||||||
# always() &&
|
(
|
||||||
# (
|
(inputs.target_stage == 'unit-test-backend-4-gpu-gb200') ||
|
||||||
# (inputs.target_stage == 'unit-test-backend-4-gpu-gb200') ||
|
(
|
||||||
# (
|
!inputs.target_stage &&
|
||||||
# !inputs.target_stage &&
|
((github.event_name == 'schedule' || inputs.test_parallel_dispatch == true) || (!failure() && !cancelled())) &&
|
||||||
# ((github.event_name == 'schedule' || inputs.test_parallel_dispatch == true) || (!failure() && !cancelled())) &&
|
((needs.check-changes.outputs.main_package == 'true') || (needs.check-changes.outputs.sgl_kernel == 'true'))
|
||||||
# ((needs.check-changes.outputs.main_package == 'true') || (needs.check-changes.outputs.sgl_kernel == 'true'))
|
)
|
||||||
# )
|
)
|
||||||
# )
|
runs-on: 4-gpu-gb200
|
||||||
# runs-on: 4-gpu-gb200
|
timeout-minutes: 60
|
||||||
# timeout-minutes: 60
|
env:
|
||||||
# env:
|
RUNNER_LABELS: 4-gpu-gb200
|
||||||
# RUNNER_LABELS: 4-gpu-gb200
|
strategy:
|
||||||
# strategy:
|
fail-fast: false
|
||||||
# fail-fast: false
|
steps:
|
||||||
# steps:
|
- name: Checkout code
|
||||||
# - name: Checkout code
|
uses: actions/checkout@v4
|
||||||
# uses: actions/checkout@v4
|
with:
|
||||||
# with:
|
ref: ${{ inputs.pr_head_sha || inputs.ref || github.sha }}
|
||||||
# ref: ${{ inputs.pr_head_sha || inputs.ref || github.sha }}
|
|
||||||
#
|
- name: Download artifacts
|
||||||
# - name: Download artifacts
|
if: needs.check-changes.outputs.sgl_kernel == 'true'
|
||||||
# if: needs.check-changes.outputs.sgl_kernel == 'true'
|
uses: actions/download-artifact@v4
|
||||||
# uses: actions/download-artifact@v4
|
with:
|
||||||
# with:
|
path: sgl-kernel/dist/
|
||||||
# path: sgl-kernel/dist/
|
merge-multiple: true
|
||||||
# merge-multiple: true
|
pattern: wheel-python3.10-cuda12.9-aarch64
|
||||||
# pattern: wheel-python3.10-cuda12.9-aarch64
|
|
||||||
#
|
- name: Install dependencies
|
||||||
# - name: Install dependencies
|
timeout-minutes: 10
|
||||||
# timeout-minutes: 10
|
run: |
|
||||||
# run: |
|
CUSTOM_BUILD_SGL_KERNEL=${{needs.check-changes.outputs.sgl_kernel}} IS_BLACKWELL=1 GRACE_BLACKWELL=1 bash scripts/ci/ci_install_deepep.sh
|
||||||
# CUSTOM_BUILD_SGL_KERNEL=${{needs.check-changes.outputs.sgl_kernel}} IS_BLACKWELL=1 GRACE_BLACKWELL=1 bash scripts/ci/ci_install_deepep.sh
|
|
||||||
#
|
- name: Run test
|
||||||
# - name: Run test
|
timeout-minutes: 45
|
||||||
# timeout-minutes: 45
|
run: |
|
||||||
# run: |
|
cd test/srt
|
||||||
# cd test/srt
|
RETRY_FLAG=""
|
||||||
# RETRY_FLAG=""
|
if [[ "${{ needs.check-changes.outputs.enable_retry }}" == "true" ]]; then
|
||||||
# if [[ "${{ needs.check-changes.outputs.enable_retry }}" == "true" ]]; then
|
RETRY_FLAG="--enable-retry"
|
||||||
# RETRY_FLAG="--enable-retry"
|
fi
|
||||||
# fi
|
CONTINUE_ON_ERROR_FLAG=""
|
||||||
# CONTINUE_ON_ERROR_FLAG=""
|
if [[ "${{ needs.check-changes.outputs.continue_on_error }}" == "true" ]]; then
|
||||||
# if [[ "${{ needs.check-changes.outputs.continue_on_error }}" == "true" ]]; then
|
CONTINUE_ON_ERROR_FLAG="--continue-on-error"
|
||||||
# CONTINUE_ON_ERROR_FLAG="--continue-on-error"
|
fi
|
||||||
# fi
|
python3 run_suite.py --suite per-commit-4-gpu-gb200 --auto-partition-id 0 --auto-partition-size 1 --timeout-per-file 3600 $RETRY_FLAG $CONTINUE_ON_ERROR_FLAG
|
||||||
# python3 run_suite.py --suite per-commit-4-gpu-gb200 --auto-partition-id 0 --auto-partition-size 1 --timeout-per-file 3600 $RETRY_FLAG $CONTINUE_ON_ERROR_FLAG
|
|
||||||
|
|
||||||
pr-test-finish:
|
pr-test-finish:
|
||||||
needs:
|
needs:
|
||||||
@@ -1836,9 +1834,9 @@ jobs:
|
|||||||
unit-test-backend-8-gpu-h20,
|
unit-test-backend-8-gpu-h20,
|
||||||
unit-test-backend-8-gpu-h200,
|
unit-test-backend-8-gpu-h200,
|
||||||
unit-test-deepep-4-gpu,
|
unit-test-deepep-4-gpu,
|
||||||
# unit-test-deepep-8-gpu, # Disabled, see #17175
|
unit-test-deepep-8-gpu,
|
||||||
unit-test-backend-4-gpu-b200,
|
unit-test-backend-4-gpu-b200,
|
||||||
# unit-test-backend-4-gpu-gb200, # Disabled: GB200 runner needs repair
|
unit-test-backend-4-gpu-gb200,
|
||||||
]
|
]
|
||||||
if: always()
|
if: always()
|
||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
|
|||||||
@@ -47,11 +47,9 @@ suites = {
|
|||||||
TestFile("ep/test_deepep_small.py", 531),
|
TestFile("ep/test_deepep_small.py", 531),
|
||||||
TestFile("ep/test_mooncake_ep_small.py", 660),
|
TestFile("ep/test_mooncake_ep_small.py", 660),
|
||||||
],
|
],
|
||||||
# Disabled: IBGDA/cudaHostRegister environment issues on 8-GPU runner, see #17175
|
"per-commit-8-gpu-h200-deepep": [
|
||||||
# 4-GPU DeepEP tests provide sufficient coverage
|
TestFile("ep/test_deepep_large.py", 563),
|
||||||
# "per-commit-8-gpu-h200-deepep": [
|
],
|
||||||
# TestFile("ep/test_deepep_large.py", 563),
|
|
||||||
# ],
|
|
||||||
# quantization_test suite migrated to test/registered/quant/
|
# quantization_test suite migrated to test/registered/quant/
|
||||||
"__not_in_ci__": [
|
"__not_in_ci__": [
|
||||||
TestFile("test_release_memory_occupation.py", 200), # Temporarily disabled
|
TestFile("test_release_memory_occupation.py", 200), # Temporarily disabled
|
||||||
@@ -64,7 +62,6 @@ suites = {
|
|||||||
TestFile(
|
TestFile(
|
||||||
"models/test_qwen3_next_models_pcg.py"
|
"models/test_qwen3_next_models_pcg.py"
|
||||||
), # Disabled: intermittent failures, see #17039
|
), # Disabled: intermittent failures, see #17039
|
||||||
TestFile("ep/test_deepep_large.py", 563), # Disabled: see #17175
|
|
||||||
],
|
],
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user