minke.yu/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 12 Packages Projects Releases Wiki Activity

ci: add continue-on-error for scheduled PR tests (#15701)

Browse Source
This commit is contained in:
Alison Shao
2025-12-25 13:26:14 -08:00
committed by GitHub
parent f2ccc44240
commit 4805118122
2 changed files with 125 additions and 27 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/workflows/pr-test.yml
+119 -27
View File
@@ -20,6 +20,11 @@ on:
required: false required: false
type: string type: string
default: "" default: ""
force_continue_on_error:
description: "Force continue-on-error (test scheduled CI behavior)"
required: false
type: boolean
default: false
concurrency: concurrency:
group: pr-test-${{ github.ref }} group: pr-test-${{ github.ref }}
@@ -39,6 +44,7 @@ jobs:
max_parallel: ${{ steps.set-parallel.outputs.max_parallel }} max_parallel: ${{ steps.set-parallel.outputs.max_parallel }}
b200_runner: ${{ steps.set-runner.outputs.b200_runner }} b200_runner: ${{ steps.set-runner.outputs.b200_runner }}
enable_retry: ${{ steps.set-retry.outputs.enable_retry }} enable_retry: ${{ steps.set-retry.outputs.enable_retry }}
continue_on_error: ${{ steps.set-continue-on-error.outputs.continue_on_error }}
steps: steps:
- name: Checkout code - name: Checkout code
uses: actions/checkout@v4 uses: actions/checkout@v4
@@ -98,19 +104,31 @@ jobs:
echo "enable_retry=true" >> $GITHUB_OUTPUT echo "enable_retry=true" >> $GITHUB_OUTPUT
echo "Retry logic enabled for CI" echo "Retry logic enabled for CI"
- name: Set continue-on-error for scheduled runs
id: set-continue-on-error
run: |
if [[ "${{ github.event_name }}" == "schedule" || "${{ inputs.force_continue_on_error }}" == "true" ]]; then
echo "continue_on_error=true" >> $GITHUB_OUTPUT
echo "Scheduled run or force flag detected, enabling continue-on-error to run all tests"
else
echo "continue_on_error=false" >> $GITHUB_OUTPUT
echo "Non-scheduled run, continue-on-error disabled"
fi
- name: Show filter results in summary (table) - name: Show filter results in summary (table)
run: | run: |
{ {
echo "## Change Detection" echo "## Change Detection"
echo "" echo ""
echo "| Component | Changed |" echo "| Component | Changed |"
echo "|----------------|---------|" echo "|-------------------|---------|"
echo "| main_package | ${{ steps.filter.outputs.main_package || steps.scheduled.outputs.main_package }} |" echo "| main_package | ${{ steps.filter.outputs.main_package || steps.scheduled.outputs.main_package }} |"
echo "| sgl_kernel | ${{ steps.filter.outputs.sgl_kernel || steps.scheduled.outputs.sgl_kernel }} |" echo "| sgl_kernel | ${{ steps.filter.outputs.sgl_kernel || steps.scheduled.outputs.sgl_kernel }} |"
echo "| multimodal_gen | ${{ steps.filter.outputs.multimodal_gen || steps.scheduled.outputs.multimodal_gen }} |" echo "| multimodal_gen | ${{ steps.filter.outputs.multimodal_gen || steps.scheduled.outputs.multimodal_gen }} |"
echo "| max_parallel | ${{ steps.set-parallel.outputs.max_parallel }} |" echo "| max_parallel | ${{ steps.set-parallel.outputs.max_parallel }} |"
echo "| b200_runner | ${{ steps.set-runner.outputs.b200_runner }} |" echo "| b200_runner | ${{ steps.set-runner.outputs.b200_runner }} |"
echo "| enable_retry | ${{ steps.set-retry.outputs.enable_retry }} |" echo "| enable_retry | ${{ steps.set-retry.outputs.enable_retry }} |"
echo "| continue_on_error | ${{ steps.set-continue-on-error.outputs.continue_on_error }} |"
} >> $GITHUB_STEP_SUMMARY } >> $GITHUB_STEP_SUMMARY
# =============================================== PR Gate ==================================================== # =============================================== PR Gate ====================================================
@@ -411,9 +429,13 @@ jobs:
timeout-minutes: 10 timeout-minutes: 10
run: | run: |
cd test/ cd test/
python3 run_suite.py --hw cuda --suite stage-a-test-1 CONTINUE_ON_ERROR_FLAG=""
if [[ "${{ needs.check-changes.outputs.continue_on_error }}" == "true" ]]; then
CONTINUE_ON_ERROR_FLAG="--continue-on-error"
fi
python3 run_suite.py --hw cuda --suite stage-a-test-1 $CONTINUE_ON_ERROR_FLAG
# temporarily put backend-independent cpu tests here # temporarily put backend-independent cpu tests here
python3 run_suite.py --hw cpu --suite default python3 run_suite.py --hw cpu --suite default $CONTINUE_ON_ERROR_FLAG
stage-b-test-small-1-gpu: stage-b-test-small-1-gpu:
needs: [check-changes, call-gate, stage-a-test-1, sgl-kernel-build-wheels] needs: [check-changes, call-gate, stage-a-test-1, sgl-kernel-build-wheels]
@@ -454,7 +476,11 @@ jobs:
timeout-minutes: 30 timeout-minutes: 30
run: | run: |
cd test/ cd test/
python3 run_suite.py --hw cuda --suite stage-b-test-small-1-gpu --auto-partition-id ${{ matrix.partition }} --auto-partition-size 4 CONTINUE_ON_ERROR_FLAG=""
if [[ "${{ needs.check-changes.outputs.continue_on_error }}" == "true" ]]; then
CONTINUE_ON_ERROR_FLAG="--continue-on-error"
fi
python3 run_suite.py --hw cuda --suite stage-b-test-small-1-gpu --auto-partition-id ${{ matrix.partition }} --auto-partition-size 4 $CONTINUE_ON_ERROR_FLAG
stage-b-test-large-1-gpu: stage-b-test-large-1-gpu:
needs: [check-changes, call-gate, stage-a-test-1, sgl-kernel-build-wheels] needs: [check-changes, call-gate, stage-a-test-1, sgl-kernel-build-wheels]
@@ -491,7 +517,11 @@ jobs:
timeout-minutes: 30 timeout-minutes: 30
run: | run: |
cd test/ cd test/
python3 run_suite.py --hw cuda --suite stage-b-test-large-1-gpu CONTINUE_ON_ERROR_FLAG=""
if [[ "${{ needs.check-changes.outputs.continue_on_error }}" == "true" ]]; then
CONTINUE_ON_ERROR_FLAG="--continue-on-error"
fi
python3 run_suite.py --hw cuda --suite stage-b-test-large-1-gpu $CONTINUE_ON_ERROR_FLAG
stage-b-test-large-2-gpu: stage-b-test-large-2-gpu:
needs: [check-changes, call-gate, stage-a-test-1, sgl-kernel-build-wheels] needs: [check-changes, call-gate, stage-a-test-1, sgl-kernel-build-wheels]
@@ -528,7 +558,11 @@ jobs:
timeout-minutes: 30 timeout-minutes: 30
run: | run: |
cd test/ cd test/
python3 run_suite.py --hw cuda --suite stage-b-test-large-2-gpu CONTINUE_ON_ERROR_FLAG=""
if [[ "${{ needs.check-changes.outputs.continue_on_error }}" == "true" ]]; then
CONTINUE_ON_ERROR_FLAG="--continue-on-error"
fi
python3 run_suite.py --hw cuda --suite stage-b-test-large-2-gpu $CONTINUE_ON_ERROR_FLAG
stage-c-test-large-4-gpu: stage-c-test-large-4-gpu:
needs: [check-changes, call-gate, stage-b-test-small-1-gpu, stage-b-test-large-1-gpu, stage-b-test-large-2-gpu, sgl-kernel-build-wheels] needs: [check-changes, call-gate, stage-b-test-small-1-gpu, stage-b-test-large-1-gpu, stage-b-test-large-2-gpu, sgl-kernel-build-wheels]
@@ -565,7 +599,11 @@ jobs:
timeout-minutes: 30 timeout-minutes: 30
run: | run: |
cd test/ cd test/
python3 run_suite.py --hw cuda --suite stage-c-test-large-4-gpu CONTINUE_ON_ERROR_FLAG=""
if [[ "${{ needs.check-changes.outputs.continue_on_error }}" == "true" ]]; then
CONTINUE_ON_ERROR_FLAG="--continue-on-error"
fi
python3 run_suite.py --hw cuda --suite stage-c-test-large-4-gpu $CONTINUE_ON_ERROR_FLAG
multimodal-gen-test-1-gpu: multimodal-gen-test-1-gpu:
needs: [check-changes, call-gate, sgl-kernel-build-wheels] needs: [check-changes, call-gate, sgl-kernel-build-wheels]
@@ -603,10 +641,15 @@ jobs:
timeout-minutes: 60 timeout-minutes: 60
run: | run: |
cd python cd python
CONTINUE_ON_ERROR_FLAG=""
if [[ "${{ needs.check-changes.outputs.continue_on_error }}" == "true" ]]; then
CONTINUE_ON_ERROR_FLAG="--continue-on-error"
fi
python3 sglang/multimodal_gen/test/run_suite.py \ python3 sglang/multimodal_gen/test/run_suite.py \
--suite 1-gpu \ --suite 1-gpu \
--partition-id ${{ matrix.part }} \ --partition-id ${{ matrix.part }} \
--total-partitions 2 --total-partitions 2 \
$CONTINUE_ON_ERROR_FLAG
multimodal-gen-test-2-gpu: multimodal-gen-test-2-gpu:
@@ -646,10 +689,15 @@ jobs:
timeout-minutes: 60 timeout-minutes: 60
run: | run: |
cd python cd python
CONTINUE_ON_ERROR_FLAG=""
if [[ "${{ needs.check-changes.outputs.continue_on_error }}" == "true" ]]; then
CONTINUE_ON_ERROR_FLAG="--continue-on-error"
fi
python3 sglang/multimodal_gen/test/run_suite.py \ python3 sglang/multimodal_gen/test/run_suite.py \
--suite 2-gpu \ --suite 2-gpu \
--partition-id ${{ matrix.part }} \ --partition-id ${{ matrix.part }} \
--total-partitions 2 --total-partitions 2 \
$CONTINUE_ON_ERROR_FLAG
quantization-test: quantization-test:
needs: [check-changes, call-gate, stage-a-test-1] needs: [check-changes, call-gate, stage-a-test-1]
@@ -688,7 +736,11 @@ jobs:
if [[ "${{ needs.check-changes.outputs.enable_retry }}" == "true" ]]; then if [[ "${{ needs.check-changes.outputs.enable_retry }}" == "true" ]]; then
RETRY_FLAG="--enable-retry" RETRY_FLAG="--enable-retry"
fi fi
python3 run_suite.py --suite quantization_test $RETRY_FLAG CONTINUE_ON_ERROR_FLAG=""
if [[ "${{ needs.check-changes.outputs.continue_on_error }}" == "true" ]]; then
CONTINUE_ON_ERROR_FLAG="--continue-on-error"
fi
python3 run_suite.py --suite quantization_test $RETRY_FLAG $CONTINUE_ON_ERROR_FLAG
unit-test-backend-1-gpu: unit-test-backend-1-gpu:
needs: [check-changes, call-gate, stage-a-test-1] needs: [check-changes, call-gate, stage-a-test-1]
@@ -734,7 +786,11 @@ jobs:
if [[ "${{ needs.check-changes.outputs.enable_retry }}" == "true" ]]; then if [[ "${{ needs.check-changes.outputs.enable_retry }}" == "true" ]]; then
RETRY_FLAG="--enable-retry" RETRY_FLAG="--enable-retry"
fi fi
python3 run_suite.py --suite per-commit-1-gpu --auto-partition-id ${{ matrix.part }} --auto-partition-size 13 $RETRY_FLAG CONTINUE_ON_ERROR_FLAG=""
if [[ "${{ needs.check-changes.outputs.continue_on_error }}" == "true" ]]; then
CONTINUE_ON_ERROR_FLAG="--continue-on-error"
fi
python3 run_suite.py --suite per-commit-1-gpu --auto-partition-id ${{ matrix.part }} --auto-partition-size 13 $RETRY_FLAG $CONTINUE_ON_ERROR_FLAG
stage-b-test-4-gpu-b200: stage-b-test-4-gpu-b200:
@@ -775,7 +831,11 @@ jobs:
timeout-minutes: 30 timeout-minutes: 30
run: | run: |
cd test cd test
IS_BLACKWELL=1 python3 run_suite.py --hw cuda --suite stage-b-test-4-gpu-b200 CONTINUE_ON_ERROR_FLAG=""
if [[ "${{ needs.check-changes.outputs.continue_on_error }}" == "true" ]]; then
CONTINUE_ON_ERROR_FLAG="--continue-on-error"
fi
IS_BLACKWELL=1 python3 run_suite.py --hw cuda --suite stage-b-test-4-gpu-b200 $CONTINUE_ON_ERROR_FLAG
unit-test-backend-2-gpu: unit-test-backend-2-gpu:
@@ -821,7 +881,11 @@ jobs:
if [[ "${{ needs.check-changes.outputs.enable_retry }}" == "true" ]]; then if [[ "${{ needs.check-changes.outputs.enable_retry }}" == "true" ]]; then
RETRY_FLAG="--enable-retry" RETRY_FLAG="--enable-retry"
fi fi
python3 run_suite.py --suite per-commit-2-gpu --auto-partition-id ${{ matrix.part }} --auto-partition-size 2 $RETRY_FLAG CONTINUE_ON_ERROR_FLAG=""
if [[ "${{ needs.check-changes.outputs.continue_on_error }}" == "true" ]]; then
CONTINUE_ON_ERROR_FLAG="--continue-on-error"
fi
python3 run_suite.py --suite per-commit-2-gpu --auto-partition-id ${{ matrix.part }} --auto-partition-size 2 $RETRY_FLAG $CONTINUE_ON_ERROR_FLAG
unit-test-backend-4-gpu: unit-test-backend-4-gpu:
needs: [check-changes, call-gate, unit-test-backend-2-gpu, stage-b-test-4-gpu-b200] needs: [check-changes, call-gate, unit-test-backend-2-gpu, stage-b-test-4-gpu-b200]
@@ -866,7 +930,11 @@ jobs:
if [[ "${{ needs.check-changes.outputs.enable_retry }}" == "true" ]]; then if [[ "${{ needs.check-changes.outputs.enable_retry }}" == "true" ]]; then
RETRY_FLAG="--enable-retry" RETRY_FLAG="--enable-retry"
fi fi
python3 run_suite.py --suite per-commit-4-gpu --auto-partition-id ${{ matrix.part }} --auto-partition-size 3 $RETRY_FLAG CONTINUE_ON_ERROR_FLAG=""
if [[ "${{ needs.check-changes.outputs.continue_on_error }}" == "true" ]]; then
CONTINUE_ON_ERROR_FLAG="--continue-on-error"
fi
python3 run_suite.py --suite per-commit-4-gpu --auto-partition-id ${{ matrix.part }} --auto-partition-size 3 $RETRY_FLAG $CONTINUE_ON_ERROR_FLAG
unit-test-backend-8-gpu-h200: unit-test-backend-8-gpu-h200:
needs: [check-changes, call-gate, unit-test-backend-2-gpu, stage-b-test-4-gpu-b200] needs: [check-changes, call-gate, unit-test-backend-2-gpu, stage-b-test-4-gpu-b200]
@@ -917,7 +985,11 @@ jobs:
if [[ "${{ needs.check-changes.outputs.enable_retry }}" == "true" ]]; then if [[ "${{ needs.check-changes.outputs.enable_retry }}" == "true" ]]; then
RETRY_FLAG="--enable-retry" RETRY_FLAG="--enable-retry"
fi fi
python3 run_suite.py --suite per-commit-8-gpu-h200 --auto-partition-id ${{ matrix.part }} --auto-partition-size 3 $RETRY_FLAG CONTINUE_ON_ERROR_FLAG=""
if [[ "${{ needs.check-changes.outputs.continue_on_error }}" == "true" ]]; then
CONTINUE_ON_ERROR_FLAG="--continue-on-error"
fi
python3 run_suite.py --suite per-commit-8-gpu-h200 --auto-partition-id ${{ matrix.part }} --auto-partition-size 3 $RETRY_FLAG $CONTINUE_ON_ERROR_FLAG
unit-test-backend-8-gpu-h20: unit-test-backend-8-gpu-h20:
needs: [check-changes, call-gate, unit-test-backend-2-gpu, stage-b-test-4-gpu-b200] needs: [check-changes, call-gate, unit-test-backend-2-gpu, stage-b-test-4-gpu-b200]
@@ -963,7 +1035,11 @@ jobs:
if [[ "${{ needs.check-changes.outputs.enable_retry }}" == "true" ]]; then if [[ "${{ needs.check-changes.outputs.enable_retry }}" == "true" ]]; then
RETRY_FLAG="--enable-retry" RETRY_FLAG="--enable-retry"
fi fi
python3 run_suite.py --suite per-commit-8-gpu-h20 --auto-partition-id ${{ matrix.part }} --auto-partition-size 2 $RETRY_FLAG CONTINUE_ON_ERROR_FLAG=""
if [[ "${{ needs.check-changes.outputs.continue_on_error }}" == "true" ]]; then
CONTINUE_ON_ERROR_FLAG="--continue-on-error"
fi
python3 run_suite.py --suite per-commit-8-gpu-h20 --auto-partition-id ${{ matrix.part }} --auto-partition-size 2 $RETRY_FLAG $CONTINUE_ON_ERROR_FLAG
performance-test-1-gpu-part-1: performance-test-1-gpu-part-1:
needs: [check-changes, call-gate, stage-a-test-1] needs: [check-changes, call-gate, stage-a-test-1]
@@ -1336,7 +1412,11 @@ jobs:
if [[ "${{ needs.check-changes.outputs.enable_retry }}" == "true" ]]; then if [[ "${{ needs.check-changes.outputs.enable_retry }}" == "true" ]]; then
RETRY_FLAG="--enable-retry" RETRY_FLAG="--enable-retry"
fi fi
python3 run_suite.py --suite per-commit-4-gpu-deepep $RETRY_FLAG CONTINUE_ON_ERROR_FLAG=""
if [[ "${{ needs.check-changes.outputs.continue_on_error }}" == "true" ]]; then
CONTINUE_ON_ERROR_FLAG="--continue-on-error"
fi
python3 run_suite.py --suite per-commit-4-gpu-deepep $RETRY_FLAG $CONTINUE_ON_ERROR_FLAG
unit-test-deepep-8-gpu: unit-test-deepep-8-gpu:
needs: [check-changes, call-gate, unit-test-backend-2-gpu, stage-b-test-4-gpu-b200] needs: [check-changes, call-gate, unit-test-backend-2-gpu, stage-b-test-4-gpu-b200]
@@ -1377,7 +1457,11 @@ jobs:
if [[ "${{ needs.check-changes.outputs.enable_retry }}" == "true" ]]; then if [[ "${{ needs.check-changes.outputs.enable_retry }}" == "true" ]]; then
RETRY_FLAG="--enable-retry" RETRY_FLAG="--enable-retry"
fi fi
python3 run_suite.py --suite per-commit-8-gpu-h200-deepep $RETRY_FLAG CONTINUE_ON_ERROR_FLAG=""
if [[ "${{ needs.check-changes.outputs.continue_on_error }}" == "true" ]]; then
CONTINUE_ON_ERROR_FLAG="--continue-on-error"
fi
python3 run_suite.py --suite per-commit-8-gpu-h200-deepep $RETRY_FLAG $CONTINUE_ON_ERROR_FLAG
unit-test-backend-4-gpu-b200: unit-test-backend-4-gpu-b200:
needs: [check-changes, call-gate, unit-test-backend-2-gpu, stage-b-test-4-gpu-b200] needs: [check-changes, call-gate, unit-test-backend-2-gpu, stage-b-test-4-gpu-b200]
@@ -1423,7 +1507,11 @@ jobs:
if [[ "${{ needs.check-changes.outputs.enable_retry }}" == "true" ]]; then if [[ "${{ needs.check-changes.outputs.enable_retry }}" == "true" ]]; then
RETRY_FLAG="--enable-retry" RETRY_FLAG="--enable-retry"
fi fi
IS_BLACKWELL=1 python3 run_suite.py --suite per-commit-4-gpu-b200 --auto-partition-id ${{ matrix.part }} --auto-partition-size 3 --timeout-per-file 1800 $RETRY_FLAG CONTINUE_ON_ERROR_FLAG=""
if [[ "${{ needs.check-changes.outputs.continue_on_error }}" == "true" ]]; then
CONTINUE_ON_ERROR_FLAG="--continue-on-error"
fi
IS_BLACKWELL=1 python3 run_suite.py --suite per-commit-4-gpu-b200 --auto-partition-id ${{ matrix.part }} --auto-partition-size 3 --timeout-per-file 1800 $RETRY_FLAG $CONTINUE_ON_ERROR_FLAG
unit-test-backend-4-gpu-gb200: unit-test-backend-4-gpu-gb200:
needs: [check-changes, call-gate, unit-test-backend-2-gpu, stage-b-test-4-gpu-b200, sgl-kernel-build-wheels-arm] needs: [check-changes, call-gate, unit-test-backend-2-gpu, stage-b-test-4-gpu-b200, sgl-kernel-build-wheels-arm]
@@ -1466,7 +1554,11 @@ jobs:
if [[ "${{ needs.check-changes.outputs.enable_retry }}" == "true" ]]; then if [[ "${{ needs.check-changes.outputs.enable_retry }}" == "true" ]]; then
RETRY_FLAG="--enable-retry" RETRY_FLAG="--enable-retry"
fi fi
python3 run_suite.py --suite per-commit-4-gpu-gb200 --auto-partition-id 0 --auto-partition-size 1 --timeout-per-file 3600 $RETRY_FLAG CONTINUE_ON_ERROR_FLAG=""
if [[ "${{ needs.check-changes.outputs.continue_on_error }}" == "true" ]]; then
CONTINUE_ON_ERROR_FLAG="--continue-on-error"
fi
python3 run_suite.py --suite per-commit-4-gpu-gb200 --auto-partition-id 0 --auto-partition-size 1 --timeout-per-file 3600 $RETRY_FLAG $CONTINUE_ON_ERROR_FLAG
pr-test-finish: pr-test-finish:
needs: needs:
python/sglang/multimodal_gen/test/run_suite.py
+6
View File
@@ -67,6 +67,12 @@ def parse_args():
default=None, default=None,
help="Pytest filter expression (passed to pytest -k)", help="Pytest filter expression (passed to pytest -k)",
) )
parser.add_argument(
"--continue-on-error",
action="store_true",
default=False,
help="Continue running remaining tests even if one fails (for CI consistency; pytest already continues by default)",
)
return parser.parse_args() return parser.parse_args()