minke.yu/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 12 Packages Projects Releases Wiki Activity

[AMD][CI] Clean up AMD nightly + pr-test workflows (#25266)

Browse Source 
Co-authored-by: Cursor Agent <cursoragent@cursor.com>
This commit is contained in:
Bingxu Chen
2026-05-20 23:30:26 -07:00
committed by GitHub
co-authored by Cursor Agent
parent b5b9c809e1
commit 45cadc215f
12 changed files with 1771 additions and 1664 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/workflows/nightly-test-amd-rocm720.yml
+988 -954
View File
File diff suppressed because it is too large Load Diff
.github/workflows/nightly-test-amd.yml
+727 -682
View File
File diff suppressed because it is too large Load Diff
.github/workflows/pr-test-amd-rocm720.yml
+38 -15
View File
@@ -46,7 +46,7 @@ on:
- multimodal-gen-test-2-gpu-amd-rocm720
- stage-c-test-large-8-gpu-amd-rocm720
- stage-c-test-large-8-gpu-amd-mi35x-rocm720
- stage-b-test-large-8-gpu-disaggregation-amd-rocm720
- stage-b-test-large-8-gpu-mi35x-disaggregation-amd-rocm720
- stage-c-test-4-gpu-amd-rocm720
target_stage:
description: "Or type comma-separated stage names (overrides dropdown if non-empty)"
@@ -68,6 +68,11 @@ on:
required: false
type: boolean
default: true
run_all_tests:
description: 'Run all tests (skip change detection). Ignored when target_stage / target_stage_select is set.'
required: false
type: boolean
default: false
workflow_call:
inputs:
ref:
@@ -116,6 +121,7 @@ jobs:
sgl_kernel: ${{ steps.filter.outputs.sgl_kernel || steps.run-mode.outputs.run_all_tests }}
jit_kernel: ${{ steps.filter.outputs.jit_kernel || steps.run-mode.outputs.run_all_tests }}
multimodal_gen: ${{ steps.filter.outputs.multimodal_gen || steps.run-mode.outputs.run_all_tests }}
continue_on_error: ${{ steps.set-continue-on-error.outputs.continue_on_error }}
steps:
- name: Checkout code
uses: actions/checkout@v4
@@ -135,6 +141,23 @@ jobs:
echo "Run mode: FILTERED (triggered by ${{ github.event_name }})"
fi
- name: Set continue-on-error for schedule/full runs
id: set-continue-on-error
run: |
# Mirror pr-test-amd.yml: continue-on-error when any of
# - run_all_tests was requested (workflow_dispatch checkbox or workflow_call input),
# - inputs.continue_on_error was explicitly set,
# - or this is a scheduled run.
# The schedule check is included explicitly here because (unlike pr-test-amd.yml's
# run-mode) the rocm720 run-mode does not roll schedule into run_all_tests.
if [[ "${{ steps.run-mode.outputs.run_all_tests }}" == "true" || "${{ inputs.continue_on_error }}" == "true" || "${{ github.event_name }}" == "schedule" ]]; then
echo "continue_on_error=true" >> $GITHUB_OUTPUT
echo "Continue-on-error: ENABLED (run_all_tests=${{ steps.run-mode.outputs.run_all_tests }}, input=${{ inputs.continue_on_error }}, event=${{ github.event_name }})"
else
echo "continue_on_error=false" >> $GITHUB_OUTPUT
echo "Continue-on-error: DISABLED"
fi
- name: Detect file changes
id: filter
uses: dorny/paths-filter@v3
@@ -289,7 +312,7 @@ jobs:
- name: Run test
timeout-minutes: 10
run: |
bash scripts/ci/amd/amd_ci_exec.sh -w "/sglang-checkout/test" python3 run_suite.py --hw amd --suite stage-a-test-1-gpu-small-amd ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }}
bash scripts/ci/amd/amd_ci_exec.sh -w "/sglang-checkout/test" python3 run_suite.py --hw amd --suite stage-a-test-1-gpu-small-amd ${{ needs.check-changes.outputs.continue_on_error == 'true' && '--continue-on-error' || '' }}
jit-kernel-unit-test-amd-rocm720:
needs: [check-changes]
@@ -366,7 +389,7 @@ jobs:
- name: Run test
timeout-minutes: 30
run: |
bash scripts/ci/amd/amd_ci_exec.sh -w "/sglang-checkout/test" python3 run_suite.py --hw amd --suite stage-b-test-1-gpu-small-amd --auto-partition-id ${{ matrix.part }} --auto-partition-size 14 --timeout-per-file 1800 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }}
bash scripts/ci/amd/amd_ci_exec.sh -w "/sglang-checkout/test" python3 run_suite.py --hw amd --suite stage-b-test-1-gpu-small-amd --auto-partition-id ${{ matrix.part }} --auto-partition-size 14 --timeout-per-file 1800 ${{ needs.check-changes.outputs.continue_on_error == 'true' && '--continue-on-error' || '' }}
stage-b-test-1-gpu-small-amd-nondeterministic-rocm720:
needs: [check-changes]
@@ -404,7 +427,7 @@ jobs:
- name: Run test
timeout-minutes: 30
run: |
bash scripts/ci/amd/amd_ci_exec.sh -w "/sglang-checkout/test" python3 run_suite.py --hw amd --suite stage-b-test-1-gpu-small-amd-nondeterministic --timeout-per-file 1800 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }}
bash scripts/ci/amd/amd_ci_exec.sh -w "/sglang-checkout/test" python3 run_suite.py --hw amd --suite stage-b-test-1-gpu-small-amd-nondeterministic --timeout-per-file 1800 ${{ needs.check-changes.outputs.continue_on_error == 'true' && '--continue-on-error' || '' }}
stage-b-test-1-gpu-small-amd-mi35x-rocm720:
needs: [check-changes]
@@ -442,7 +465,7 @@ jobs:
- name: Run test
timeout-minutes: 30
run: |
bash scripts/ci/amd/amd_ci_exec.sh -w "/sglang-checkout/test" python3 run_suite.py --hw amd --suite stage-b-test-1-gpu-small-amd-mi35x ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }}
bash scripts/ci/amd/amd_ci_exec.sh -w "/sglang-checkout/test" python3 run_suite.py --hw amd --suite stage-b-test-1-gpu-small-amd-mi35x ${{ needs.check-changes.outputs.continue_on_error == 'true' && '--continue-on-error' || '' }}
stage-b-test-1-gpu-large-amd-rocm720:
needs: [check-changes]
@@ -481,7 +504,7 @@ jobs:
- name: Run test
timeout-minutes: 30
run: |
bash scripts/ci/amd/amd_ci_exec.sh -w "/sglang-checkout/test" python3 run_suite.py --hw amd --suite stage-b-test-1-gpu-large-amd --auto-partition-id ${{ matrix.part }} --auto-partition-size 2 --timeout-per-file 1800 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }}
bash scripts/ci/amd/amd_ci_exec.sh -w "/sglang-checkout/test" python3 run_suite.py --hw amd --suite stage-b-test-1-gpu-large-amd --auto-partition-id ${{ matrix.part }} --auto-partition-size 2 --timeout-per-file 1800 ${{ needs.check-changes.outputs.continue_on_error == 'true' && '--continue-on-error' || '' }}
stage-b-test-2-gpu-large-amd-rocm720:
needs: [check-changes]
@@ -520,7 +543,7 @@ jobs:
- name: Run test
timeout-minutes: 30
run: |
bash scripts/ci/amd/amd_ci_exec.sh -w "/sglang-checkout/test" python3 run_suite.py --hw amd --suite stage-b-test-2-gpu-large-amd --auto-partition-id ${{ matrix.part }} --auto-partition-size 2 --timeout-per-file 1800 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }}
bash scripts/ci/amd/amd_ci_exec.sh -w "/sglang-checkout/test" python3 run_suite.py --hw amd --suite stage-b-test-2-gpu-large-amd --auto-partition-id ${{ matrix.part }} --auto-partition-size 2 --timeout-per-file 1800 ${{ needs.check-changes.outputs.continue_on_error == 'true' && '--continue-on-error' || '' }}
multimodal-gen-test-1-gpu-amd-rocm720:
needs: [check-changes]
@@ -828,7 +851,7 @@ jobs:
ref: ${{ inputs.pr_head_sha || inputs.ref || github.sha }}
- name: Ensure VRAM is clear
run: bash scripts/ensure_vram_clear.sh rocm
run: bash scripts/ci/amd/ensure_vram_clear.sh rocm
- name: Start CI container
run: bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version rocm720
@@ -857,7 +880,7 @@ jobs:
--max-attempts 2 \
--retry-wait-seconds 120 \
--retry-timeout-increase 0 \
${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }}
${{ needs.check-changes.outputs.continue_on_error == 'true' && '--continue-on-error' || '' }}
stage-c-test-large-8-gpu-amd-rocm720:
needs: [check-changes]
@@ -904,7 +927,7 @@ jobs:
- name: Run test
timeout-minutes: 120
run: |
bash scripts/ci/amd/amd_ci_exec.sh -w "/sglang-checkout/test" python3 run_suite.py --hw amd --suite stage-c-test-large-8-gpu-amd --auto-partition-id ${{ matrix.part }} --auto-partition-size 3 --timeout-per-file 5400 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }}
bash scripts/ci/amd/amd_ci_exec.sh -w "/sglang-checkout/test" python3 run_suite.py --hw amd --suite stage-c-test-large-8-gpu-amd --auto-partition-id ${{ matrix.part }} --auto-partition-size 3 --timeout-per-file 5400 ${{ needs.check-changes.outputs.continue_on_error == 'true' && '--continue-on-error' || '' }}
stage-c-test-large-8-gpu-amd-mi35x-rocm720:
needs: [check-changes]
@@ -943,15 +966,15 @@ jobs:
- name: Run test
timeout-minutes: 60
run: |
bash scripts/ci/amd/amd_ci_exec.sh -w "/sglang-checkout/test" python3 run_suite.py --hw amd --suite stage-c-test-large-8-gpu-amd-mi35x --auto-partition-id ${{ matrix.part }} --auto-partition-size 2 --timeout-per-file 3600 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }}
bash scripts/ci/amd/amd_ci_exec.sh -w "/sglang-checkout/test" python3 run_suite.py --hw amd --suite stage-c-test-large-8-gpu-amd-mi35x --auto-partition-id ${{ matrix.part }} --auto-partition-size 2 --timeout-per-file 3600 ${{ needs.check-changes.outputs.continue_on_error == 'true' && '--continue-on-error' || '' }}
# =============================================== Disaggregation ====================================================
stage-b-test-large-8-gpu-35x-disaggregation-amd-rocm720:
stage-b-test-large-8-gpu-mi35x-disaggregation-amd-rocm720:
needs: [check-changes]
if: |
always() &&
(
(contains(format(',{0},', inputs.target_stage || inputs.target_stage_select), ',stage-b-test-large-8-gpu-disaggregation-amd-rocm720,')) ||
(contains(format(',{0},', inputs.target_stage || inputs.target_stage_select), ',stage-b-test-large-8-gpu-mi35x-disaggregation-amd-rocm720,')) ||
(
!(inputs.target_stage || inputs.target_stage_select) &&
(!failure() && !cancelled()) &&
@@ -1055,7 +1078,7 @@ jobs:
run: |
bash scripts/ci/amd/amd_ci_exec.sh \
-e SGLANG_TEST_RDMA_DEVICE="${{ env.SGLANG_TEST_RDMA_DEVICE }}" \
-w "/sglang-checkout/test" python3 run_suite.py --hw amd --suite stage-b-test-large-8-gpu-35x-disaggregation-amd --timeout-per-file 1800 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }}
-w "/sglang-checkout/test" python3 run_suite.py --hw amd --suite stage-b-test-large-8-gpu-mi35x-disaggregation-amd --timeout-per-file 1800 ${{ needs.check-changes.outputs.continue_on_error == 'true' && '--continue-on-error' || '' }}
pr-test-amd-rocm720-finish:
needs:
@@ -1075,7 +1098,7 @@ jobs:
stage-b-test-1-gpu-small-amd-mi35x-rocm720,
stage-b-test-1-gpu-large-amd-rocm720,
stage-b-test-2-gpu-large-amd-rocm720,
stage-b-test-large-8-gpu-35x-disaggregation-amd-rocm720,
stage-b-test-large-8-gpu-mi35x-disaggregation-amd-rocm720,
stage-c-test-4-gpu-amd-rocm720,
stage-c-test-large-8-gpu-amd-rocm720,
stage-c-test-large-8-gpu-amd-mi35x-rocm720,
.github/workflows/pr-test-amd.yml
+10 -5
View File
@@ -38,7 +38,7 @@ on:
- stage-c-test-4-gpu-amd
- stage-c-test-large-8-gpu-amd
- stage-c-test-large-8-gpu-amd-mi35x
- stage-b-test-large-8-gpu-35x-disaggregation-amd
- stage-b-test-large-8-gpu-mi35x-disaggregation-amd
target_stage:
description: "Or type comma-separated stage names (overrides dropdown if non-empty)"
required: false
@@ -67,6 +67,11 @@ on:
options:
- mi300
- mi325
run_all_tests:
description: 'Run all tests (skip change detection). Ignored when target_stage / target_stage_select is set.'
required: false
type: boolean
default: false
workflow_call:
inputs:
ref:
@@ -1028,12 +1033,12 @@ jobs:
bash scripts/ci/amd/amd_ci_exec.sh -w "/sglang-checkout/test" python3 run_suite.py --hw amd --suite stage-c-test-large-8-gpu-amd-mi35x --auto-partition-id ${{ matrix.part }} --auto-partition-size 2 --timeout-per-file 3600 ${{ needs.check-changes.outputs.continue_on_error == 'true' && '--continue-on-error' || '' }}
# =============================================== Disaggregation ====================================================
stage-b-test-large-8-gpu-35x-disaggregation-amd:
stage-b-test-large-8-gpu-mi35x-disaggregation-amd:
needs: [check-changes, wait-for-stage-a-amd]
if: |
always() &&
(
(contains(format(',{0},', inputs.target_stage || inputs.target_stage_select), ',stage-b-test-large-8-gpu-35x-disaggregation-amd,')) ||
(contains(format(',{0},', inputs.target_stage || inputs.target_stage_select), ',stage-b-test-large-8-gpu-mi35x-disaggregation-amd,')) ||
(
!(inputs.target_stage || inputs.target_stage_select) &&
((github.event_name == 'schedule') || (!failure() && !cancelled())) &&
@@ -1137,7 +1142,7 @@ jobs:
run: |
bash scripts/ci/amd/amd_ci_exec.sh \
-e SGLANG_TEST_RDMA_DEVICE="${{ env.SGLANG_TEST_RDMA_DEVICE }}" \
-w "/sglang-checkout/test" python3 run_suite.py --hw amd --suite stage-b-test-large-8-gpu-35x-disaggregation-amd --timeout-per-file 1800 ${{ needs.check-changes.outputs.continue_on_error == 'true' && '--continue-on-error' || '' }}
-w "/sglang-checkout/test" python3 run_suite.py --hw amd --suite stage-b-test-large-8-gpu-mi35x-disaggregation-amd --timeout-per-file 1800 ${{ needs.check-changes.outputs.continue_on_error == 'true' && '--continue-on-error' || '' }}
pr-test-amd-finish:
needs:
@@ -1159,7 +1164,7 @@ jobs:
stage-b-test-1-gpu-small-amd-mi35x,
stage-b-test-1-gpu-large-amd,
stage-b-test-2-gpu-large-amd,
stage-b-test-large-8-gpu-35x-disaggregation-amd,
stage-b-test-large-8-gpu-mi35x-disaggregation-amd,
stage-c-test-4-gpu-amd,
stage-c-test-large-8-gpu-amd,
stage-c-test-large-8-gpu-amd-mi35x,