From 5fff58a16980bcd7da0bf0526a799dccf4f49d43 Mon Sep 17 00:00:00 2001 From: Michael <13900043+michaelzhang-ai@users.noreply.github.com> Date: Wed, 21 Oct 2026 17:24:51 -0700 Subject: [PATCH] [CI] [AMD] miles: Fix the ROCm 10 nightly suites that never finish (#45469) --- .../nightly-test-amd-miles-rocm10-mi30x.yml | 13 ++++++++----- .../nightly-test-amd-miles-rocm10.yml | 18 ++++++++++++++++-- 2 files changed, 24 insertions(+), 7 deletions(-) diff --git a/.github/workflows/nightly-test-amd-miles-rocm10-mi30x.yml b/.github/workflows/nightly-test-amd-miles-rocm10-mi30x.yml index 6730c7adf6..05747a6e9b 100644 --- a/.github/workflows/nightly-test-amd-miles-rocm10-mi30x.yml +++ b/.github/workflows/nightly-test-amd-miles-rocm10-mi30x.yml @@ -34,15 +34,18 @@ jobs: - suite: nightly-stage-c-2-gpu-mi350 runner: linux-mi300-2gpu-sglang timeout: 180 - # Single-request decode of Qwen3.8-27B-FP8 runs at ~20 tok/s on MI300X, so long - # generations outlast the test client's 180 s request timeout. - skip_tests: tests/e2e/sglang/test_session_server_multi_role/test_qwen38small.py + # test_qwen38small: single-request decode of Qwen3.8-27B-FP8 runs at ~20 tok/s on + # MI300X, so long generations outlast the test client's 180 s request timeout. + # test_glm5_2_744b_a40b_5layer_ci: its tilelang indexer kernel needs 98 KiB of LDS + # per block; gfx942 has 64 KiB. + skip_tests: >- + tests/e2e/sglang/test_session_server_multi_role/test_qwen38small.py + tests/e2e/megatron/model_scripts/test_glm5_2_744b_a40b_5layer_ci.py # The 4- and 8-GPU suites run past 180 min on MI300X; they take ~155 and ~112 min on MI355X. - suite: nightly-stage-c-4-gpu-mi350 runner: linux-mi300-4gpu-sglang timeout: 300 - # Its tilelang indexer kernel needs 98 KiB of LDS per block; gfx942 has 64 KiB. - skip_tests: tests/e2e/megatron/model_scripts/test_glm5_2_744b_a40b_5layer_ci.py + skip_tests: "" - suite: nightly-stage-c-8-gpu-mi350 runner: linux-mi300-8gpu-sglang timeout: 300 diff --git a/.github/workflows/nightly-test-amd-miles-rocm10.yml b/.github/workflows/nightly-test-amd-miles-rocm10.yml index e13b810d43..2c4964c831 100644 --- a/.github/workflows/nightly-test-amd-miles-rocm10.yml +++ b/.github/workflows/nightly-test-amd-miles-rocm10.yml @@ -24,15 +24,28 @@ concurrency: jobs: test-miles-mi35x: if: github.repository == 'sgl-project/sglang' + name: test-miles-mi35x (${{ matrix.suite }}, ${{ matrix.runner }}${{ matrix.partition_size && format(', part {0}', matrix.part) || '' }}) strategy: fail-fast: false matrix: + # All suites use the 8-GPU MI35X pool. The 2-GPU suite includes the + # GLM-5.2 5-layer test, which takes the linux-mi35x-gpu-2 runner down + # ("runner lost communication") but passes on an 8-GPU runner. include: - suite: nightly-stage-c-2-gpu-mi350 - runner: linux-mi35x-gpu-2 - # Use 8-GPU MI35X pool for 4-GPU suite. + runner: linux-mi35x-gpu-8 + # The 4-GPU suite is ~6 h of estimated test time; split it so each + # half fits in the step timeout. - suite: nightly-stage-c-4-gpu-mi350 runner: linux-mi35x-gpu-8 + partition_id: 0 + partition_size: 2 + part: 1/2 + - suite: nightly-stage-c-4-gpu-mi350 + runner: linux-mi35x-gpu-8 + partition_id: 1 + partition_size: 2 + part: 2/2 - suite: nightly-stage-c-8-gpu-mi350 runner: linux-mi35x-gpu-8 runs-on: ${{ matrix.runner }} @@ -88,6 +101,7 @@ jobs: -e PYTHONPATH=/root/miles:/opt/tilelang \ -e GITHUB_STEP_SUMMARY=/sglang-checkout/github_summary.md \ python3 tests/ci/run_suite.py --hw rocm --suite ${{ matrix.suite }} --cadence nightly \ + ${{ matrix.partition_size && format('--auto-partition-id {0} --auto-partition-size {1}', matrix.partition_id, matrix.partition_size) || '' }} \ ${{ (github.event_name != 'workflow_dispatch' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$? cat "${GITHUB_WORKSPACE}/github_summary.md" >> "$GITHUB_STEP_SUMMARY" || true exit ${TEST_EXIT_CODE}