[CI] [AMD] miles: Fix the ROCm 10 nightly suites that never finish (#45469)

This commit is contained in:
Michael
2026-10-21 17:24:51 -07:00
committed by GitHub
parent eacd0a07d7
commit 5fff58a169
2 changed files with 24 additions and 7 deletions
@@ -34,15 +34,18 @@ jobs:
- suite: nightly-stage-c-2-gpu-mi350
runner: linux-mi300-2gpu-sglang
timeout: 180
# Single-request decode of Qwen3.8-27B-FP8 runs at ~20 tok/s on MI300X, so long
# generations outlast the test client's 180 s request timeout.
skip_tests: tests/e2e/sglang/test_session_server_multi_role/test_qwen38small.py
# test_qwen38small: single-request decode of Qwen3.8-27B-FP8 runs at ~20 tok/s on
# MI300X, so long generations outlast the test client's 180 s request timeout.
# test_glm5_2_744b_a40b_5layer_ci: its tilelang indexer kernel needs 98 KiB of LDS
# per block; gfx942 has 64 KiB.
skip_tests: >-
tests/e2e/sglang/test_session_server_multi_role/test_qwen38small.py
tests/e2e/megatron/model_scripts/test_glm5_2_744b_a40b_5layer_ci.py
# The 4- and 8-GPU suites run past 180 min on MI300X; they take ~155 and ~112 min on MI355X.
- suite: nightly-stage-c-4-gpu-mi350
runner: linux-mi300-4gpu-sglang
timeout: 300
# Its tilelang indexer kernel needs 98 KiB of LDS per block; gfx942 has 64 KiB.
skip_tests: tests/e2e/megatron/model_scripts/test_glm5_2_744b_a40b_5layer_ci.py
skip_tests: ""
- suite: nightly-stage-c-8-gpu-mi350
runner: linux-mi300-8gpu-sglang
timeout: 300
@@ -24,15 +24,28 @@ concurrency:
jobs:
test-miles-mi35x:
if: github.repository == 'sgl-project/sglang'
name: test-miles-mi35x (${{ matrix.suite }}, ${{ matrix.runner }}${{ matrix.partition_size && format(', part {0}', matrix.part) || '' }})
strategy:
fail-fast: false
matrix:
# All suites use the 8-GPU MI35X pool. The 2-GPU suite includes the
# GLM-5.2 5-layer test, which takes the linux-mi35x-gpu-2 runner down
# ("runner lost communication") but passes on an 8-GPU runner.
include:
- suite: nightly-stage-c-2-gpu-mi350
runner: linux-mi35x-gpu-2
# Use 8-GPU MI35X pool for 4-GPU suite.
runner: linux-mi35x-gpu-8
# The 4-GPU suite is ~6 h of estimated test time; split it so each
# half fits in the step timeout.
- suite: nightly-stage-c-4-gpu-mi350
runner: linux-mi35x-gpu-8
partition_id: 0
partition_size: 2
part: 1/2
- suite: nightly-stage-c-4-gpu-mi350
runner: linux-mi35x-gpu-8
partition_id: 1
partition_size: 2
part: 2/2
- suite: nightly-stage-c-8-gpu-mi350
runner: linux-mi35x-gpu-8
runs-on: ${{ matrix.runner }}
@@ -88,6 +101,7 @@ jobs:
-e PYTHONPATH=/root/miles:/opt/tilelang \
-e GITHUB_STEP_SUMMARY=/sglang-checkout/github_summary.md \
python3 tests/ci/run_suite.py --hw rocm --suite ${{ matrix.suite }} --cadence nightly \
${{ matrix.partition_size && format('--auto-partition-id {0} --auto-partition-size {1}', matrix.partition_id, matrix.partition_size) || '' }} \
${{ (github.event_name != 'workflow_dispatch' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$?
cat "${GITHUB_WORKSPACE}/github_summary.md" >> "$GITHUB_STEP_SUMMARY" || true
exit ${TEST_EXIT_CODE}