minke.yu/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 12 Packages Projects Releases Wiki Activity

[AMD][CI] Clean up AMD nightly + pr-test workflows (#25266)

Browse Source 
Co-authored-by: Cursor Agent <cursoragent@cursor.com>
This commit is contained in:
Bingxu Chen
2026-05-20 23:30:26 -07:00
committed by GitHub
co-authored by Cursor Agent
parent b5b9c809e1
commit 45cadc215f
12 changed files with 1771 additions and 1664 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.claude/skills/write-sglang-test/SKILL.md
+1 -1
View File
@@ -88,7 +88,7 @@ Defined in `python/sglang/test/test_utils.py`:
| `stage-b-test-1-gpu-small-amd-mi35x` | `linux-mi35x-gpu-1` | 1-GPU tests on MI35x hardware |
| `stage-b-test-1-gpu-large-amd` | `linux-mi325-1gpu-sglang` | Large 1-GPU AMD tests (2 partitions) |
| `stage-b-test-2-gpu-large-amd` | `linux-mi325-2gpu-sglang` | 2-GPU ROCm correctness and parallel setups |
| `stage-b-test-large-8-gpu-35x-disaggregation-amd` | `linux-mi35x-gpu-8.fabric` | PD disaggregation and RDMA on 8×MI35x fabric |
| `stage-b-test-large-8-gpu-mi35x-disaggregation-amd` | `linux-mi35x-gpu-8.fabric` | PD disaggregation and RDMA on 8×MI35x fabric |
| `stage-c-test-4-gpu-amd` | `linux-mi325-4gpu-sglang` | 4-GPU AMD integration (2 partitions) |
| `stage-c-test-large-8-gpu-amd` | `linux-mi325-8gpu-sglang` | 8-GPU MI325 scaling and integration |
| `stage-c-test-large-8-gpu-amd-mi35x` | `linux-mi35x-gpu-8` | 8-GPU MI35x scaling (2 partitions) |
.github/workflows/nightly-test-amd-rocm720.yml
+988 -954
View File
File diff suppressed because it is too large Load Diff
.github/workflows/nightly-test-amd.yml
+727 -682
View File
File diff suppressed because it is too large Load Diff
.github/workflows/pr-test-amd-rocm720.yml
+38 -15
View File
@@ -46,7 +46,7 @@ on:
- multimodal-gen-test-2-gpu-amd-rocm720
- stage-c-test-large-8-gpu-amd-rocm720
- stage-c-test-large-8-gpu-amd-mi35x-rocm720
- stage-b-test-large-8-gpu-disaggregation-amd-rocm720
- stage-b-test-large-8-gpu-mi35x-disaggregation-amd-rocm720
- stage-c-test-4-gpu-amd-rocm720
target_stage:
description: "Or type comma-separated stage names (overrides dropdown if non-empty)"
@@ -68,6 +68,11 @@ on:
required: false
type: boolean
default: true
run_all_tests:
description: 'Run all tests (skip change detection). Ignored when target_stage / target_stage_select is set.'
required: false
type: boolean
default: false
workflow_call:
inputs:
ref:
@@ -116,6 +121,7 @@ jobs:
sgl_kernel: ${{ steps.filter.outputs.sgl_kernel || steps.run-mode.outputs.run_all_tests }}
jit_kernel: ${{ steps.filter.outputs.jit_kernel || steps.run-mode.outputs.run_all_tests }}
multimodal_gen: ${{ steps.filter.outputs.multimodal_gen || steps.run-mode.outputs.run_all_tests }}
continue_on_error: ${{ steps.set-continue-on-error.outputs.continue_on_error }}
steps:
- name: Checkout code
uses: actions/checkout@v4
@@ -135,6 +141,23 @@ jobs:
echo "Run mode: FILTERED (triggered by ${{ github.event_name }})"
fi
- name: Set continue-on-error for schedule/full runs
id: set-continue-on-error
run: |
# Mirror pr-test-amd.yml: continue-on-error when any of
# - run_all_tests was requested (workflow_dispatch checkbox or workflow_call input),
# - inputs.continue_on_error was explicitly set,
# - or this is a scheduled run.
# The schedule check is included explicitly here because (unlike pr-test-amd.yml's
# run-mode) the rocm720 run-mode does not roll schedule into run_all_tests.
if [[ "${{ steps.run-mode.outputs.run_all_tests }}" == "true" || "${{ inputs.continue_on_error }}" == "true" || "${{ github.event_name }}" == "schedule" ]]; then
echo "continue_on_error=true" >> $GITHUB_OUTPUT
echo "Continue-on-error: ENABLED (run_all_tests=${{ steps.run-mode.outputs.run_all_tests }}, input=${{ inputs.continue_on_error }}, event=${{ github.event_name }})"
else
echo "continue_on_error=false" >> $GITHUB_OUTPUT
echo "Continue-on-error: DISABLED"
fi
- name: Detect file changes
id: filter
uses: dorny/paths-filter@v3
@@ -289,7 +312,7 @@ jobs:
- name: Run test
timeout-minutes: 10
run: |
bash scripts/ci/amd/amd_ci_exec.sh -w "/sglang-checkout/test" python3 run_suite.py --hw amd --suite stage-a-test-1-gpu-small-amd ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }}
bash scripts/ci/amd/amd_ci_exec.sh -w "/sglang-checkout/test" python3 run_suite.py --hw amd --suite stage-a-test-1-gpu-small-amd ${{ needs.check-changes.outputs.continue_on_error == 'true' && '--continue-on-error' || '' }}
jit-kernel-unit-test-amd-rocm720:
needs: [check-changes]
@@ -366,7 +389,7 @@ jobs:
- name: Run test
timeout-minutes: 30
run: |
bash scripts/ci/amd/amd_ci_exec.sh -w "/sglang-checkout/test" python3 run_suite.py --hw amd --suite stage-b-test-1-gpu-small-amd --auto-partition-id ${{ matrix.part }} --auto-partition-size 14 --timeout-per-file 1800 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }}
bash scripts/ci/amd/amd_ci_exec.sh -w "/sglang-checkout/test" python3 run_suite.py --hw amd --suite stage-b-test-1-gpu-small-amd --auto-partition-id ${{ matrix.part }} --auto-partition-size 14 --timeout-per-file 1800 ${{ needs.check-changes.outputs.continue_on_error == 'true' && '--continue-on-error' || '' }}
stage-b-test-1-gpu-small-amd-nondeterministic-rocm720:
needs: [check-changes]
@@ -404,7 +427,7 @@ jobs:
- name: Run test
timeout-minutes: 30
run: |
bash scripts/ci/amd/amd_ci_exec.sh -w "/sglang-checkout/test" python3 run_suite.py --hw amd --suite stage-b-test-1-gpu-small-amd-nondeterministic --timeout-per-file 1800 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }}
bash scripts/ci/amd/amd_ci_exec.sh -w "/sglang-checkout/test" python3 run_suite.py --hw amd --suite stage-b-test-1-gpu-small-amd-nondeterministic --timeout-per-file 1800 ${{ needs.check-changes.outputs.continue_on_error == 'true' && '--continue-on-error' || '' }}
stage-b-test-1-gpu-small-amd-mi35x-rocm720:
needs: [check-changes]
@@ -442,7 +465,7 @@ jobs:
- name: Run test
timeout-minutes: 30
run: |
bash scripts/ci/amd/amd_ci_exec.sh -w "/sglang-checkout/test" python3 run_suite.py --hw amd --suite stage-b-test-1-gpu-small-amd-mi35x ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }}
bash scripts/ci/amd/amd_ci_exec.sh -w "/sglang-checkout/test" python3 run_suite.py --hw amd --suite stage-b-test-1-gpu-small-amd-mi35x ${{ needs.check-changes.outputs.continue_on_error == 'true' && '--continue-on-error' || '' }}
stage-b-test-1-gpu-large-amd-rocm720:
needs: [check-changes]
@@ -481,7 +504,7 @@ jobs:
- name: Run test
timeout-minutes: 30
run: |
bash scripts/ci/amd/amd_ci_exec.sh -w "/sglang-checkout/test" python3 run_suite.py --hw amd --suite stage-b-test-1-gpu-large-amd --auto-partition-id ${{ matrix.part }} --auto-partition-size 2 --timeout-per-file 1800 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }}
bash scripts/ci/amd/amd_ci_exec.sh -w "/sglang-checkout/test" python3 run_suite.py --hw amd --suite stage-b-test-1-gpu-large-amd --auto-partition-id ${{ matrix.part }} --auto-partition-size 2 --timeout-per-file 1800 ${{ needs.check-changes.outputs.continue_on_error == 'true' && '--continue-on-error' || '' }}
stage-b-test-2-gpu-large-amd-rocm720:
needs: [check-changes]
@@ -520,7 +543,7 @@ jobs:
- name: Run test
timeout-minutes: 30
run: |
bash scripts/ci/amd/amd_ci_exec.sh -w "/sglang-checkout/test" python3 run_suite.py --hw amd --suite stage-b-test-2-gpu-large-amd --auto-partition-id ${{ matrix.part }} --auto-partition-size 2 --timeout-per-file 1800 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }}
bash scripts/ci/amd/amd_ci_exec.sh -w "/sglang-checkout/test" python3 run_suite.py --hw amd --suite stage-b-test-2-gpu-large-amd --auto-partition-id ${{ matrix.part }} --auto-partition-size 2 --timeout-per-file 1800 ${{ needs.check-changes.outputs.continue_on_error == 'true' && '--continue-on-error' || '' }}
multimodal-gen-test-1-gpu-amd-rocm720:
needs: [check-changes]
@@ -828,7 +851,7 @@ jobs:
ref: ${{ inputs.pr_head_sha || inputs.ref || github.sha }}
- name: Ensure VRAM is clear
run: bash scripts/ensure_vram_clear.sh rocm
run: bash scripts/ci/amd/ensure_vram_clear.sh rocm
- name: Start CI container
run: bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version rocm720
@@ -857,7 +880,7 @@ jobs:
--max-attempts 2 \
--retry-wait-seconds 120 \
--retry-timeout-increase 0 \
${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }}
${{ needs.check-changes.outputs.continue_on_error == 'true' && '--continue-on-error' || '' }}
stage-c-test-large-8-gpu-amd-rocm720:
needs: [check-changes]
@@ -904,7 +927,7 @@ jobs:
- name: Run test
timeout-minutes: 120
run: |
bash scripts/ci/amd/amd_ci_exec.sh -w "/sglang-checkout/test" python3 run_suite.py --hw amd --suite stage-c-test-large-8-gpu-amd --auto-partition-id ${{ matrix.part }} --auto-partition-size 3 --timeout-per-file 5400 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }}
bash scripts/ci/amd/amd_ci_exec.sh -w "/sglang-checkout/test" python3 run_suite.py --hw amd --suite stage-c-test-large-8-gpu-amd --auto-partition-id ${{ matrix.part }} --auto-partition-size 3 --timeout-per-file 5400 ${{ needs.check-changes.outputs.continue_on_error == 'true' && '--continue-on-error' || '' }}
stage-c-test-large-8-gpu-amd-mi35x-rocm720:
needs: [check-changes]
@@ -943,15 +966,15 @@ jobs:
- name: Run test
timeout-minutes: 60
run: |
bash scripts/ci/amd/amd_ci_exec.sh -w "/sglang-checkout/test" python3 run_suite.py --hw amd --suite stage-c-test-large-8-gpu-amd-mi35x --auto-partition-id ${{ matrix.part }} --auto-partition-size 2 --timeout-per-file 3600 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }}
bash scripts/ci/amd/amd_ci_exec.sh -w "/sglang-checkout/test" python3 run_suite.py --hw amd --suite stage-c-test-large-8-gpu-amd-mi35x --auto-partition-id ${{ matrix.part }} --auto-partition-size 2 --timeout-per-file 3600 ${{ needs.check-changes.outputs.continue_on_error == 'true' && '--continue-on-error' || '' }}
# =============================================== Disaggregation ====================================================
stage-b-test-large-8-gpu-35x-disaggregation-amd-rocm720:
stage-b-test-large-8-gpu-mi35x-disaggregation-amd-rocm720:
needs: [check-changes]
if: |
always() &&
(
(contains(format(',{0},', inputs.target_stage || inputs.target_stage_select), ',stage-b-test-large-8-gpu-disaggregation-amd-rocm720,')) ||
(contains(format(',{0},', inputs.target_stage || inputs.target_stage_select), ',stage-b-test-large-8-gpu-mi35x-disaggregation-amd-rocm720,')) ||
(
!(inputs.target_stage || inputs.target_stage_select) &&
(!failure() && !cancelled()) &&
@@ -1055,7 +1078,7 @@ jobs:
run: |
bash scripts/ci/amd/amd_ci_exec.sh \
-e SGLANG_TEST_RDMA_DEVICE="${{ env.SGLANG_TEST_RDMA_DEVICE }}" \
-w "/sglang-checkout/test" python3 run_suite.py --hw amd --suite stage-b-test-large-8-gpu-35x-disaggregation-amd --timeout-per-file 1800 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }}
-w "/sglang-checkout/test" python3 run_suite.py --hw amd --suite stage-b-test-large-8-gpu-mi35x-disaggregation-amd --timeout-per-file 1800 ${{ needs.check-changes.outputs.continue_on_error == 'true' && '--continue-on-error' || '' }}
pr-test-amd-rocm720-finish:
needs:
@@ -1075,7 +1098,7 @@ jobs:
stage-b-test-1-gpu-small-amd-mi35x-rocm720,
stage-b-test-1-gpu-large-amd-rocm720,
stage-b-test-2-gpu-large-amd-rocm720,
stage-b-test-large-8-gpu-35x-disaggregation-amd-rocm720,
stage-b-test-large-8-gpu-mi35x-disaggregation-amd-rocm720,
stage-c-test-4-gpu-amd-rocm720,
stage-c-test-large-8-gpu-amd-rocm720,
stage-c-test-large-8-gpu-amd-mi35x-rocm720,
.github/workflows/pr-test-amd.yml
+10 -5
View File
@@ -38,7 +38,7 @@ on:
- stage-c-test-4-gpu-amd
- stage-c-test-large-8-gpu-amd
- stage-c-test-large-8-gpu-amd-mi35x
- stage-b-test-large-8-gpu-35x-disaggregation-amd
- stage-b-test-large-8-gpu-mi35x-disaggregation-amd
target_stage:
description: "Or type comma-separated stage names (overrides dropdown if non-empty)"
required: false
@@ -67,6 +67,11 @@ on:
options:
- mi300
- mi325
run_all_tests:
description: 'Run all tests (skip change detection). Ignored when target_stage / target_stage_select is set.'
required: false
type: boolean
default: false
workflow_call:
inputs:
ref:
@@ -1028,12 +1033,12 @@ jobs:
bash scripts/ci/amd/amd_ci_exec.sh -w "/sglang-checkout/test" python3 run_suite.py --hw amd --suite stage-c-test-large-8-gpu-amd-mi35x --auto-partition-id ${{ matrix.part }} --auto-partition-size 2 --timeout-per-file 3600 ${{ needs.check-changes.outputs.continue_on_error == 'true' && '--continue-on-error' || '' }}
# =============================================== Disaggregation ====================================================
stage-b-test-large-8-gpu-35x-disaggregation-amd:
stage-b-test-large-8-gpu-mi35x-disaggregation-amd:
needs: [check-changes, wait-for-stage-a-amd]
if: |
always() &&
(
(contains(format(',{0},', inputs.target_stage || inputs.target_stage_select), ',stage-b-test-large-8-gpu-35x-disaggregation-amd,')) ||
(contains(format(',{0},', inputs.target_stage || inputs.target_stage_select), ',stage-b-test-large-8-gpu-mi35x-disaggregation-amd,')) ||
(
!(inputs.target_stage || inputs.target_stage_select) &&
((github.event_name == 'schedule') || (!failure() && !cancelled())) &&
@@ -1137,7 +1142,7 @@ jobs:
run: |
bash scripts/ci/amd/amd_ci_exec.sh \
-e SGLANG_TEST_RDMA_DEVICE="${{ env.SGLANG_TEST_RDMA_DEVICE }}" \
-w "/sglang-checkout/test" python3 run_suite.py --hw amd --suite stage-b-test-large-8-gpu-35x-disaggregation-amd --timeout-per-file 1800 ${{ needs.check-changes.outputs.continue_on_error == 'true' && '--continue-on-error' || '' }}
-w "/sglang-checkout/test" python3 run_suite.py --hw amd --suite stage-b-test-large-8-gpu-mi35x-disaggregation-amd --timeout-per-file 1800 ${{ needs.check-changes.outputs.continue_on_error == 'true' && '--continue-on-error' || '' }}
pr-test-amd-finish:
needs:
@@ -1159,7 +1164,7 @@ jobs:
stage-b-test-1-gpu-small-amd-mi35x,
stage-b-test-1-gpu-large-amd,
stage-b-test-2-gpu-large-amd,
stage-b-test-large-8-gpu-35x-disaggregation-amd,
stage-b-test-large-8-gpu-mi35x-disaggregation-amd,
stage-c-test-4-gpu-amd,
stage-c-test-large-8-gpu-amd,
stage-c-test-large-8-gpu-amd-mi35x,
test/registered/amd/accuracy/mi30x/test_glm51_eval_amd.py
+1 -1
View File
@@ -161,7 +161,7 @@ class TestGLM51EvalAMD(unittest.TestCase):
def setUpClass(cls):
cls.models = GLM51_MODELS
cls.base_url = DEFAULT_URL_FOR_TEST
cls.num_questions = int(os.environ.get("GSM8K_NUM_QUESTIONS", "200"))
cls.num_questions = int(os.environ.get("GSM8K_NUM_QUESTIONS", "1319"))
def test_glm51_accuracy(self):
all_results = []
test/registered/amd/accuracy/mi35x/test_glm51_eval_mi35x.py
+1 -1
View File
@@ -165,7 +165,7 @@ class TestGLM51EvalMI35x(unittest.TestCase):
def setUpClass(cls):
cls.models = MI35X_GLM51_MODELS
cls.base_url = DEFAULT_URL_FOR_TEST
cls.num_questions = int(os.environ.get("GSM8K_NUM_QUESTIONS", "200"))
cls.num_questions = int(os.environ.get("GSM8K_NUM_QUESTIONS", "1319"))
def test_glm51_accuracy(self):
all_results = []
test/registered/amd/accuracy/mi35x/test_glm5_mxfp4_eval_mi35x.py
+1 -1
View File
@@ -191,7 +191,7 @@ class TestGLM5MXFP4EvalMI35x(unittest.TestCase):
def setUpClass(cls):
cls.models = get_glm5_mxfp4_models()
cls.base_url = DEFAULT_URL_FOR_TEST
cls.num_questions = int(os.environ.get("GSM8K_NUM_QUESTIONS", "200"))
cls.num_questions = int(os.environ.get("GSM8K_NUM_QUESTIONS", "1319"))
def test_glm5_mxfp4_accuracy(self):
"""Test GLM-5-MXFP4 with GSM8K completion benchmark."""
test/registered/amd/disaggregation/test_disaggregation_basic.py
+1 -1
View File
@@ -18,7 +18,7 @@ from sglang.test.test_utils import (
popen_launch_pd_server,
)
register_amd_ci(est_time=600, suite="stage-b-test-large-8-gpu-35x-disaggregation-amd")
register_amd_ci(est_time=600, suite="stage-b-test-large-8-gpu-mi35x-disaggregation-amd")
class TestDisaggregationAccuracy(PDDisaggregationServerBase):
test/registered/amd/disaggregation/test_disaggregation_pp.py
+1 -1
View File
@@ -15,7 +15,7 @@ from sglang.test.test_utils import (
try_cached_model,
)
register_amd_ci(est_time=600, suite="stage-b-test-large-8-gpu-35x-disaggregation-amd")
register_amd_ci(est_time=600, suite="stage-b-test-large-8-gpu-mi35x-disaggregation-amd")
class TestDisaggregationPrefillPPAccuracy(PDDisaggregationServerBase):
test/registered/amd/disaggregation/test_mori_transfer_engine_e2e.py
+1 -1
View File
@@ -14,7 +14,7 @@ from sglang.test.test_utils import (
try_cached_model,
)
register_amd_ci(est_time=300, suite="stage-b-test-large-8-gpu-35x-disaggregation-amd")
register_amd_ci(est_time=300, suite="stage-b-test-large-8-gpu-mi35x-disaggregation-amd")
class MoriTransferEngineBase(PDDisaggregationServerBase):
test/run_suite.py
+1 -1
View File
@@ -33,7 +33,7 @@ PER_COMMIT_SUITES = {
"stage-b-test-1-gpu-small-amd",
"stage-b-test-1-gpu-small-amd-nondeterministic",
"stage-b-test-1-gpu-small-amd-mi35x",
"stage-b-test-large-8-gpu-35x-disaggregation-amd",
"stage-b-test-large-8-gpu-mi35x-disaggregation-amd",
"stage-b-test-1-gpu-large-amd",
"stage-b-test-2-gpu-large-amd",
"jit-kernel-unit-test-amd",