diff --git a/.github/workflows/amd-ci-job-monitor.yml b/.github/workflows/amd-ci-job-monitor.yml index 19b657c83..e1dd2a198 100644 --- a/.github/workflows/amd-ci-job-monitor.yml +++ b/.github/workflows/amd-ci-job-monitor.yml @@ -129,26 +129,45 @@ jobs: - name: Parse workflow files id: parse run: | + set -euo pipefail + # Parse the default ROCm 7.2 PR gate (exclude utility jobs) - # Excluded: call-gate, check-changes, finish, cancel, check-all-jobs + # Excluded: call-gate, check-changes, finish, cancel, check-all-jobs, + # plus the stage waits -- they run on ubuntu-latest, so the + # self-hosted-only snapshot has nothing for them. Keep the extra call: + # its delegated GPU jobs use the explicit caller name as their prefix. + # Job ids keep the -rocm720 lane suffix but display names no longer do, + # so strip it to get the prefix query_job_status.py matches against. pr_jobs=$(yq -r '.jobs | keys | .[]' .github/workflows/pr-test-amd-rocm720.yml | \ - grep -v -E '^(call-gate|check-changes|pr-test-amd-rocm720-finish|cancel|check-all-jobs)$' | \ + grep -v -E '^(call-gate|check-changes|pr-test-amd-rocm720-finish|cancel|check-all-jobs|wait-for-stage-[ab]-amd-rocm720)$' | \ + sed 's/-rocm720$//' | \ jq -R -s -c 'split("\n") | map(select(length > 0))') echo "pr_jobs=$pr_jobs" >> $GITHUB_OUTPUT echo "PR jobs: $pr_jobs" # Parse the default ROCm 7.2 nightly (exclude utility jobs) # Excluded: check-all-jobs - nightly_jobs=$(yq -r '.jobs | keys | .[]' .github/workflows/nightly-test-amd-rocm720.yml | \ - grep -v -E '^(check-all-jobs)$' | \ + # One nightly run covers every flavor, so ask for each by its full name: + # a stem-only prefix matches every flavor and merges them into a single + # row, hiding a regression that only reproduces on one image. Nightly + # names carry the runner too, so pair each job id with its runs-on. + # Keep in sync with the rocm_version matrix in the nightly workflow. + rocm_flavors="rocm724 rocm720" + nightly_jobs=$(yq -r '.jobs | to_entries[] | .key + " " + .value."runs-on"' .github/workflows/nightly-test-amd-rocm720.yml | \ + grep -v -E '^check-all-jobs ' | \ + while read -r id runner; do + stem=${id%-rocm720} + for f in $rocm_flavors; do echo "$stem ($f, $runner)"; done + done | \ jq -R -s -c 'split("\n") | map(select(length > 0))') echo "nightly_jobs=$nightly_jobs" >> $GITHUB_OUTPUT echo "Nightly jobs: $nightly_jobs" # Parse the ROCm 7.0 rollback shadow (exclude utility jobs) - # Excluded: call-gate, check-changes, finish, cancel, check-all-jobs + # Excluded: call-gate, check-changes, finish, and the stage waits. + # Keep the reusable extra caller so its nested GPU jobs remain covered. pr_rocm700_jobs=$(yq -r '.jobs | keys | .[]' .github/workflows/pr-test-amd.yml | \ - grep -v -E '^(call-gate|check-changes|pr-test-amd-finish|cancel|check-all-jobs)$' | \ + grep -v -E '^(call-gate|check-changes|pr-test-amd-finish|wait-for-stage-[ab]-amd)$' | \ jq -R -s -c 'split("\n") | map(select(length > 0))') echo "pr_rocm700_jobs=$pr_rocm700_jobs" >> $GITHUB_OUTPUT echo "PR ROCm 7.0 shadow jobs: $pr_rocm700_jobs" diff --git a/.github/workflows/nightly-test-amd-rocm720.yml b/.github/workflows/nightly-test-amd-rocm720.yml index 202f809c8..253d50b8f 100644 --- a/.github/workflows/nightly-test-amd-rocm720.yml +++ b/.github/workflows/nightly-test-amd-rocm720.yml @@ -161,6 +161,7 @@ jobs: # ============================================================================== nightly-test-1-gpu-unit-rocm720: + name: ${{ format('nightly-test-1-gpu-unit ({0}, linux-mi300-1gpu-sglang)', matrix.rocm_version) }} strategy: fail-fast: false matrix: @@ -176,7 +177,7 @@ jobs: - name: Ensure VRAM is clear run: bash scripts/ci/amd/ensure_vram_clear.sh rocm - - name: Setup docker (ROCm 7.2) + - name: Setup docker (${{ matrix.rocm_version }}) run: | touch github_summary.md bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version ${{ matrix.rocm_version }} @@ -195,6 +196,7 @@ jobs: exit ${TEST_EXIT_CODE:-0} nightly-test-1-gpu-kernel-rocm720: + name: ${{ format('nightly-test-1-gpu-kernel ({0}, linux-mi300-1gpu-sglang)', matrix.rocm_version) }} strategy: fail-fast: false matrix: @@ -210,7 +212,7 @@ jobs: - name: Ensure VRAM is clear run: bash scripts/ci/amd/ensure_vram_clear.sh rocm - - name: Setup docker (ROCm 7.2) + - name: Setup docker (${{ matrix.rocm_version }}) run: | touch github_summary.md bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version ${{ matrix.rocm_version }} @@ -229,6 +231,7 @@ jobs: exit ${TEST_EXIT_CODE:-0} nightly-test-1-gpu-mi35x-rocm720: + name: ${{ format('nightly-test-1-gpu-mi35x ({0}, linux-mi35x-gpu-1)', matrix.rocm_version) }} strategy: fail-fast: false matrix: @@ -244,7 +247,7 @@ jobs: - name: Ensure VRAM is clear run: bash scripts/ci/amd/ensure_vram_clear.sh rocm - - name: Setup docker (ROCm 7.2) + - name: Setup docker (${{ matrix.rocm_version }}) run: | touch github_summary.md bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version ${{ matrix.rocm_version }} @@ -268,6 +271,7 @@ jobs: # ============================================================================== nightly-accuracy-2-gpu-rocm720: + name: ${{ format('nightly-accuracy-2-gpu ({0}, linux-mi300-2gpu-sglang)', matrix.rocm_version) }} strategy: fail-fast: false matrix: @@ -283,7 +287,7 @@ jobs: - name: Ensure VRAM is clear run: bash scripts/ci/amd/ensure_vram_clear.sh rocm - - name: Setup docker (ROCm 7.2) + - name: Setup docker (${{ matrix.rocm_version }}) run: | touch github_summary.md bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version ${{ matrix.rocm_version }} @@ -302,6 +306,7 @@ jobs: exit ${TEST_EXIT_CODE:-0} nightly-2-gpu-mi35x-glm51-mxfp4-rocm720: + name: ${{ format('nightly-2-gpu-mi35x-glm51-mxfp4 ({0}, linux-mi35x-gpu-2)', matrix.rocm_version) }} strategy: fail-fast: false matrix: @@ -317,7 +322,7 @@ jobs: - name: Ensure VRAM is clear run: bash scripts/ci/amd/ensure_vram_clear.sh rocm - - name: Setup docker (ROCm 7.2) + - name: Setup docker (${{ matrix.rocm_version }}) run: | touch github_summary.md bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version ${{ matrix.rocm_version }} @@ -340,6 +345,7 @@ jobs: exit ${TEST_EXIT_CODE:-0} nightly-2-gpu-mi35x-deepseek-r1-mxfp4-tp2-rocm720: + name: ${{ format('nightly-2-gpu-mi35x-deepseek-r1-mxfp4-tp2 ({0}, linux-mi35x-gpu-2)', matrix.rocm_version) }} strategy: fail-fast: false matrix: @@ -355,7 +361,7 @@ jobs: - name: Ensure VRAM is clear run: bash scripts/ci/amd/ensure_vram_clear.sh rocm - - name: Setup docker (ROCm 7.2) + - name: Setup docker (${{ matrix.rocm_version }}) run: | touch github_summary.md bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version ${{ matrix.rocm_version }} @@ -379,6 +385,7 @@ jobs: exit ${TEST_EXIT_CODE:-0} nightly-accuracy-2-gpu-vlm-rocm720: + name: ${{ format('nightly-accuracy-2-gpu-vlm ({0}, linux-mi300-2gpu-sglang)', matrix.rocm_version) }} strategy: fail-fast: false matrix: @@ -394,7 +401,7 @@ jobs: - name: Ensure VRAM is clear run: bash scripts/ci/amd/ensure_vram_clear.sh rocm - - name: Setup docker (ROCm 7.2) + - name: Setup docker (${{ matrix.rocm_version }}) run: | touch github_summary.md bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version ${{ matrix.rocm_version }} @@ -414,6 +421,7 @@ jobs: exit ${TEST_EXIT_CODE:-0} nightly-perf-2-gpu-text-rocm720: + name: ${{ format('nightly-perf-2-gpu-text ({0}, linux-mi300-2gpu-sglang)', matrix.rocm_version) }} strategy: fail-fast: false matrix: @@ -429,7 +437,7 @@ jobs: - name: Ensure VRAM is clear run: bash scripts/ci/amd/ensure_vram_clear.sh rocm - - name: Setup docker (ROCm 7.2) + - name: Setup docker (${{ matrix.rocm_version }}) run: | touch github_summary.md bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version ${{ matrix.rocm_version }} @@ -450,6 +458,7 @@ jobs: exit ${TEST_EXIT_CODE:-0} nightly-perf-2-gpu-vlm-rocm720: + name: ${{ format('nightly-perf-2-gpu-vlm ({0}, linux-mi300-2gpu-sglang)', matrix.rocm_version) }} strategy: fail-fast: false matrix: @@ -465,7 +474,7 @@ jobs: - name: Ensure VRAM is clear run: bash scripts/ci/amd/ensure_vram_clear.sh rocm - - name: Setup docker (ROCm 7.2) + - name: Setup docker (${{ matrix.rocm_version }}) run: | touch github_summary.md bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version ${{ matrix.rocm_version }} @@ -486,6 +495,7 @@ jobs: exit ${TEST_EXIT_CODE:-0} nightly-4-gpu-rocm720: + name: ${{ format('nightly-4-gpu ({0}, linux-mi300-4gpu-sglang)', matrix.rocm_version) }} strategy: fail-fast: false matrix: @@ -501,7 +511,7 @@ jobs: - name: Ensure VRAM is clear run: bash scripts/ci/amd/ensure_vram_clear.sh rocm - - name: Setup docker (ROCm 7.2) + - name: Setup docker (${{ matrix.rocm_version }}) run: | touch github_summary.md bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version ${{ matrix.rocm_version }} @@ -526,6 +536,7 @@ jobs: # ============================================================================== nightly-accuracy-8-gpu-rocm720: + name: ${{ format('nightly-accuracy-8-gpu ({0}, linux-mi300-8gpu-sglang)', matrix.rocm_version) }} strategy: fail-fast: false matrix: @@ -541,7 +552,7 @@ jobs: - name: Ensure VRAM is clear run: bash scripts/ci/amd/ensure_vram_clear.sh rocm - - name: Setup docker (ROCm 7.2) + - name: Setup docker (${{ matrix.rocm_version }}) run: | touch github_summary.md bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version ${{ matrix.rocm_version }} @@ -578,6 +589,7 @@ jobs: exit ${TEST_EXIT_CODE:-0} nightly-accuracy-8-gpu-mi35x-rocm720: + name: ${{ format('nightly-accuracy-8-gpu-mi35x ({0}, linux-mi35x-gpu-8)', matrix.rocm_version) }} strategy: fail-fast: false matrix: @@ -593,7 +605,7 @@ jobs: - name: Ensure VRAM is clear run: bash scripts/ci/amd/ensure_vram_clear.sh rocm - - name: Setup docker (ROCm 7.2) + - name: Setup docker (${{ matrix.rocm_version }}) run: | touch github_summary.md bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version ${{ matrix.rocm_version }} @@ -634,6 +646,7 @@ jobs: # ============================================================================== nightly-8-gpu-grok1-int4-rocm720: + name: ${{ format('nightly-8-gpu-grok1-int4 ({0}, linux-mi300-8gpu-sglang)', matrix.rocm_version) }} strategy: fail-fast: false matrix: @@ -649,7 +662,7 @@ jobs: - name: Ensure VRAM is clear run: bash scripts/ci/amd/ensure_vram_clear.sh rocm - - name: Setup docker (ROCm 7.2) + - name: Setup docker (${{ matrix.rocm_version }}) run: | touch github_summary.md bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version ${{ matrix.rocm_version }} @@ -683,6 +696,7 @@ jobs: exit ${TEST_EXIT_CODE:-0} nightly-8-gpu-mi35x-grok1-int4-rocm720: + name: ${{ format('nightly-8-gpu-mi35x-grok1-int4 ({0}, linux-mi35x-gpu-8)', matrix.rocm_version) }} strategy: fail-fast: false matrix: @@ -698,7 +712,7 @@ jobs: - name: Ensure VRAM is clear run: bash scripts/ci/amd/ensure_vram_clear.sh rocm - - name: Setup docker (ROCm 7.2) + - name: Setup docker (${{ matrix.rocm_version }}) run: | touch github_summary.md bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version ${{ matrix.rocm_version }} @@ -739,6 +753,7 @@ jobs: # ============================================================================== nightly-8-gpu-grok2-rocm720: + name: ${{ format('nightly-8-gpu-grok2 ({0}, linux-mi300-8gpu-sglang)', matrix.rocm_version) }} strategy: fail-fast: false matrix: @@ -754,7 +769,7 @@ jobs: - name: Ensure VRAM is clear run: bash scripts/ci/amd/ensure_vram_clear.sh rocm - - name: Setup docker (ROCm 7.2) + - name: Setup docker (${{ matrix.rocm_version }}) run: | touch github_summary.md bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version ${{ matrix.rocm_version }} @@ -788,6 +803,7 @@ jobs: exit ${TEST_EXIT_CODE:-0} nightly-8-gpu-mi35x-grok2-rocm720: + name: ${{ format('nightly-8-gpu-mi35x-grok2 ({0}, linux-mi35x-gpu-8)', matrix.rocm_version) }} strategy: fail-fast: false matrix: @@ -803,7 +819,7 @@ jobs: - name: Ensure VRAM is clear run: bash scripts/ci/amd/ensure_vram_clear.sh rocm - - name: Setup docker (ROCm 7.2) + - name: Setup docker (${{ matrix.rocm_version }}) run: | touch github_summary.md bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version ${{ matrix.rocm_version }} @@ -844,6 +860,7 @@ jobs: # ============================================================================== nightly-8-gpu-deepseek-v31-rocm720: + name: ${{ format('nightly-8-gpu-deepseek-v31 ({0}, linux-mi300-8gpu-sglang)', matrix.rocm_version) }} strategy: fail-fast: false matrix: @@ -859,7 +876,7 @@ jobs: - name: Ensure VRAM is clear run: bash scripts/ci/amd/ensure_vram_clear.sh rocm - - name: Setup docker (ROCm 7.2) + - name: Setup docker (${{ matrix.rocm_version }}) run: | touch github_summary.md bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version ${{ matrix.rocm_version }} @@ -894,6 +911,7 @@ jobs: exit ${TEST_EXIT_CODE:-0} nightly-8-gpu-deepseek-v32-rocm720: + name: ${{ format('nightly-8-gpu-deepseek-v32 ({0}, linux-mi300-8gpu-sglang)', matrix.rocm_version) }} strategy: fail-fast: false matrix: @@ -909,7 +927,7 @@ jobs: - name: Ensure VRAM is clear run: bash scripts/ci/amd/ensure_vram_clear.sh rocm - - name: Setup docker (ROCm 7.2) + - name: Setup docker (${{ matrix.rocm_version }}) run: | touch github_summary.md bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version ${{ matrix.rocm_version }} @@ -942,6 +960,7 @@ jobs: exit ${TEST_EXIT_CODE:-0} nightly-8-gpu-deepseek-v32-mtp-rocm720: + name: ${{ format('nightly-8-gpu-deepseek-v32-mtp ({0}, linux-mi300-8gpu-sglang)', matrix.rocm_version) }} strategy: fail-fast: false matrix: @@ -957,7 +976,7 @@ jobs: - name: Ensure VRAM is clear run: bash scripts/ci/amd/ensure_vram_clear.sh rocm - - name: Setup docker (ROCm 7.2) + - name: Setup docker (${{ matrix.rocm_version }}) run: | touch github_summary.md bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version ${{ matrix.rocm_version }} @@ -990,6 +1009,7 @@ jobs: exit ${TEST_EXIT_CODE:-0} nightly-8-gpu-deepseek-v3-kv-fp8-rocm720: + name: ${{ format('nightly-8-gpu-deepseek-v3-kv-fp8 ({0}, linux-mi300-8gpu-sglang)', matrix.rocm_version) }} strategy: fail-fast: false matrix: @@ -1005,7 +1025,7 @@ jobs: - name: Ensure VRAM is clear run: bash scripts/ci/amd/ensure_vram_clear.sh rocm - - name: Setup docker (ROCm 7.2) + - name: Setup docker (${{ matrix.rocm_version }}) run: | touch github_summary.md bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version ${{ matrix.rocm_version }} @@ -1031,6 +1051,7 @@ jobs: # ============================================================================== nightly-accuracy-8-gpu-mi35x-deepseek-v32-rocm720: + name: ${{ format('nightly-accuracy-8-gpu-mi35x-deepseek-v32 ({0}, linux-mi35x-gpu-8)', matrix.rocm_version) }} strategy: fail-fast: false matrix: @@ -1046,7 +1067,7 @@ jobs: - name: Ensure VRAM is clear run: bash scripts/ci/amd/ensure_vram_clear.sh rocm - - name: Setup docker (ROCm 7.2) + - name: Setup docker (${{ matrix.rocm_version }}) run: | touch github_summary.md bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version ${{ matrix.rocm_version }} @@ -1071,6 +1092,7 @@ jobs: exit ${TEST_EXIT_CODE:-0} nightly-accuracy-8-gpu-mi35x-deepseek-v32-mtp-rocm720: + name: ${{ format('nightly-accuracy-8-gpu-mi35x-deepseek-v32-mtp ({0}, linux-mi35x-gpu-8)', matrix.rocm_version) }} strategy: fail-fast: false matrix: @@ -1086,7 +1108,7 @@ jobs: - name: Ensure VRAM is clear run: bash scripts/ci/amd/ensure_vram_clear.sh rocm - - name: Setup docker (ROCm 7.2) + - name: Setup docker (${{ matrix.rocm_version }}) run: | touch github_summary.md bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version ${{ matrix.rocm_version }} @@ -1111,6 +1133,7 @@ jobs: exit ${TEST_EXIT_CODE:-0} nightly-perf-8-gpu-mi35x-deepseek-v32-basic-rocm720: + name: ${{ format('nightly-perf-8-gpu-mi35x-deepseek-v32-basic ({0}, linux-mi35x-gpu-8)', matrix.rocm_version) }} strategy: fail-fast: false matrix: @@ -1126,7 +1149,7 @@ jobs: - name: Ensure VRAM is clear run: bash scripts/ci/amd/ensure_vram_clear.sh rocm - - name: Setup docker (ROCm 7.2) + - name: Setup docker (${{ matrix.rocm_version }}) run: | touch github_summary.md bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version ${{ matrix.rocm_version }} @@ -1151,6 +1174,7 @@ jobs: exit ${TEST_EXIT_CODE:-0} nightly-perf-8-gpu-mi35x-deepseek-v32-mtp-rocm720: + name: ${{ format('nightly-perf-8-gpu-mi35x-deepseek-v32-mtp ({0}, linux-mi35x-gpu-8)', matrix.rocm_version) }} strategy: fail-fast: false matrix: @@ -1166,7 +1190,7 @@ jobs: - name: Ensure VRAM is clear run: bash scripts/ci/amd/ensure_vram_clear.sh rocm - - name: Setup docker (ROCm 7.2) + - name: Setup docker (${{ matrix.rocm_version }}) run: | touch github_summary.md bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version ${{ matrix.rocm_version }} @@ -1195,6 +1219,7 @@ jobs: # ============================================================================== nightly-8-gpu-mi35x-deepseek-r1-mxfp4-rocm720: + name: ${{ format('nightly-8-gpu-mi35x-deepseek-r1-mxfp4 ({0}, linux-mi35x-gpu-8)', matrix.rocm_version) }} strategy: fail-fast: false matrix: @@ -1210,7 +1235,7 @@ jobs: - name: Ensure VRAM is clear run: bash scripts/ci/amd/ensure_vram_clear.sh rocm - - name: Setup docker (ROCm 7.2) + - name: Setup docker (${{ matrix.rocm_version }}) run: | touch github_summary.md bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version ${{ matrix.rocm_version }} @@ -1246,6 +1271,7 @@ jobs: exit ${TEST_EXIT_CODE:-0} nightly-8-gpu-mi35x-deepseek-r1-mxfp4-tp4-rocm720: + name: ${{ format('nightly-8-gpu-mi35x-deepseek-r1-mxfp4-tp4 ({0}, linux-mi35x-gpu-8)', matrix.rocm_version) }} strategy: fail-fast: false matrix: @@ -1261,7 +1287,7 @@ jobs: - name: Ensure VRAM is clear run: bash scripts/ci/amd/ensure_vram_clear.sh rocm - - name: Setup docker (ROCm 7.2) + - name: Setup docker (${{ matrix.rocm_version }}) run: | touch github_summary.md bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version ${{ matrix.rocm_version }} @@ -1296,6 +1322,7 @@ jobs: exit ${TEST_EXIT_CODE:-0} nightly-8-gpu-mi35x-deepseek-r1-mxfp4-kv-fp8-rocm720: + name: ${{ format('nightly-8-gpu-mi35x-deepseek-r1-mxfp4-kv-fp8 ({0}, linux-mi35x-gpu-8)', matrix.rocm_version) }} strategy: fail-fast: false matrix: @@ -1311,7 +1338,7 @@ jobs: - name: Ensure VRAM is clear run: bash scripts/ci/amd/ensure_vram_clear.sh rocm - - name: Setup docker (ROCm 7.2) + - name: Setup docker (${{ matrix.rocm_version }}) run: | touch github_summary.md bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version ${{ matrix.rocm_version }} @@ -1347,6 +1374,7 @@ jobs: exit ${TEST_EXIT_CODE:-0} nightly-8-gpu-mi35x-deepseek-r1-mxfp4-ar-fusion-rocm720: + name: ${{ format('nightly-8-gpu-mi35x-deepseek-r1-mxfp4-ar-fusion ({0}, linux-mi35x-gpu-8)', matrix.rocm_version) }} strategy: fail-fast: false matrix: @@ -1362,7 +1390,7 @@ jobs: - name: Ensure VRAM is clear run: bash scripts/ci/amd/ensure_vram_clear.sh rocm - - name: Setup docker (ROCm 7.2) + - name: Setup docker (${{ matrix.rocm_version }}) run: | touch github_summary.md bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version ${{ matrix.rocm_version }} @@ -1398,6 +1426,7 @@ jobs: exit ${TEST_EXIT_CODE:-0} nightly-8-gpu-mi35x-deepseek-r1-hicache-rocm720: + name: ${{ format('nightly-8-gpu-mi35x-deepseek-r1-hicache ({0}, linux-mi35x-gpu-8)', matrix.rocm_version) }} strategy: fail-fast: false matrix: @@ -1413,7 +1442,7 @@ jobs: - name: Ensure VRAM is clear run: bash scripts/ci/amd/ensure_vram_clear.sh rocm - - name: Setup docker (ROCm 7.2) + - name: Setup docker (${{ matrix.rocm_version }}) run: | touch github_summary.md bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version ${{ matrix.rocm_version }} @@ -1440,6 +1469,7 @@ jobs: # ============================================================================== nightly-8-gpu-mi35x-deepseek-v4-flash-rocm720: + name: ${{ format('nightly-8-gpu-mi35x-deepseek-v4-flash ({0}, linux-mi35x-gpu-8)', matrix.rocm_version) }} strategy: fail-fast: false matrix: @@ -1455,7 +1485,7 @@ jobs: - name: Ensure VRAM is clear run: bash scripts/ci/amd/ensure_vram_clear.sh rocm - - name: Setup docker (ROCm 7.2) + - name: Setup docker (${{ matrix.rocm_version }}) run: | touch github_summary.md bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version ${{ matrix.rocm_version }} @@ -1495,6 +1525,7 @@ jobs: echo "$(> $GITHUB_STEP_SUMMARY || true nightly-8-gpu-mi35x-deepseek-v4-pro-rocm720: + name: ${{ format('nightly-8-gpu-mi35x-deepseek-v4-pro ({0}, linux-mi35x-gpu-8)', matrix.rocm_version) }} strategy: fail-fast: false matrix: @@ -1510,7 +1541,7 @@ jobs: - name: Ensure VRAM is clear run: bash scripts/ci/amd/ensure_vram_clear.sh rocm - - name: Setup docker (ROCm 7.2) + - name: Setup docker (${{ matrix.rocm_version }}) run: | touch github_summary.md bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version ${{ matrix.rocm_version }} @@ -1550,6 +1581,7 @@ jobs: echo "$(> $GITHUB_STEP_SUMMARY || true nightly-8-gpu-mi35x-deepseek-v4-pro-mtp-rocm720: + name: ${{ format('nightly-8-gpu-mi35x-deepseek-v4-pro-mtp ({0}, linux-mi35x-gpu-8)', matrix.rocm_version) }} strategy: fail-fast: false matrix: @@ -1565,7 +1597,7 @@ jobs: - name: Ensure VRAM is clear run: bash scripts/ci/amd/ensure_vram_clear.sh rocm - - name: Setup docker (ROCm 7.2) + - name: Setup docker (${{ matrix.rocm_version }}) run: | touch github_summary.md bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version ${{ matrix.rocm_version }} @@ -1605,6 +1637,7 @@ jobs: echo "$(> $GITHUB_STEP_SUMMARY || true nightly-8-gpu-mi35x-deepseek-v4-pro-dspark-rocm720: + name: ${{ format('nightly-8-gpu-mi35x-deepseek-v4-pro-dspark ({0}, linux-mi35x-gpu-8)', matrix.rocm_version) }} strategy: fail-fast: false matrix: @@ -1620,7 +1653,7 @@ jobs: - name: Ensure VRAM is clear run: bash scripts/ci/amd/ensure_vram_clear.sh rocm - - name: Setup docker (ROCm 7.2) + - name: Setup docker (${{ matrix.rocm_version }}) run: | touch github_summary.md bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version ${{ matrix.rocm_version }} @@ -1654,6 +1687,7 @@ jobs: # ============================================================================== nightly-8-gpu-kimi-k26-rocm720: + name: ${{ format('nightly-8-gpu-kimi-k26 ({0}, linux-mi300-8gpu-sglang)', matrix.rocm_version) }} strategy: fail-fast: false matrix: @@ -1669,7 +1703,7 @@ jobs: - name: Ensure VRAM is clear run: bash scripts/ci/amd/ensure_vram_clear.sh rocm - - name: Setup docker (ROCm 7.2) + - name: Setup docker (${{ matrix.rocm_version }}) run: | touch github_summary.md bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version ${{ matrix.rocm_version }} @@ -1695,6 +1729,7 @@ jobs: # ============================================================================== nightly-8-gpu-mi35x-kimi-k3-rocm720: + name: ${{ format('nightly-8-gpu-mi35x-kimi-k3 ({0}, linux-mi35x-gpu-8)', matrix.rocm_version) }} strategy: fail-fast: false matrix: @@ -1710,7 +1745,7 @@ jobs: - name: Ensure VRAM is clear run: bash scripts/ci/amd/ensure_vram_clear.sh rocm - - name: Setup docker (ROCm 7.2) + - name: Setup docker (${{ matrix.rocm_version }}) run: | touch github_summary.md bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version ${{ matrix.rocm_version }} @@ -1758,6 +1793,7 @@ jobs: # ============================================================================== nightly-8-gpu-qwen35-rocm720: + name: ${{ format('nightly-8-gpu-qwen35 ({0}, linux-mi300-8gpu-sglang)', matrix.rocm_version) }} strategy: fail-fast: false matrix: @@ -1773,7 +1809,7 @@ jobs: - name: Ensure VRAM is clear run: bash scripts/ci/amd/ensure_vram_clear.sh rocm - - name: Setup docker (ROCm 7.2) + - name: Setup docker (${{ matrix.rocm_version }}) run: | touch github_summary.md bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version ${{ matrix.rocm_version }} @@ -1810,6 +1846,7 @@ jobs: exit ${TEST_EXIT_CODE:-0} nightly-8-gpu-mi35x-qwen35-rocm720: + name: ${{ format('nightly-8-gpu-mi35x-qwen35 ({0}, linux-mi35x-gpu-8)', matrix.rocm_version) }} strategy: fail-fast: false matrix: @@ -1825,7 +1862,7 @@ jobs: - name: Ensure VRAM is clear run: bash scripts/ci/amd/ensure_vram_clear.sh rocm - - name: Setup docker (ROCm 7.2) + - name: Setup docker (${{ matrix.rocm_version }}) run: | touch github_summary.md bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version ${{ matrix.rocm_version }} @@ -1861,6 +1898,7 @@ jobs: exit ${TEST_EXIT_CODE:-0} nightly-8-gpu-mi35x-qwen35-triton-dcp-rocm720: + name: ${{ format('nightly-8-gpu-mi35x-qwen35-triton-dcp ({0}, linux-mi35x-gpu-8)', matrix.rocm_version) }} strategy: fail-fast: false matrix: @@ -1876,7 +1914,7 @@ jobs: - name: Ensure VRAM is clear run: bash scripts/ci/amd/ensure_vram_clear.sh rocm - - name: Setup docker (ROCm 7.2) + - name: Setup docker (${{ matrix.rocm_version }}) run: | touch github_summary.md bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version ${{ matrix.rocm_version }} @@ -1903,6 +1941,7 @@ jobs: # ============================================================================== nightly-8-gpu-mi35x-qwen38-mxfp4-rocm720: + name: ${{ format('nightly-8-gpu-mi35x-qwen38-mxfp4 ({0}, linux-mi35x-gpu-8)', matrix.rocm_version) }} strategy: fail-fast: false matrix: @@ -1918,7 +1957,7 @@ jobs: - name: Ensure VRAM is clear run: bash scripts/ci/amd/ensure_vram_clear.sh rocm - - name: Setup docker (ROCm 7.2) + - name: Setup docker (${{ matrix.rocm_version }}) run: | touch github_summary.md bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version ${{ matrix.rocm_version }} @@ -1960,6 +1999,7 @@ jobs: # ============================================================================== nightly-8-gpu-glm51-rocm720: + name: ${{ format('nightly-8-gpu-glm51 ({0}, linux-mi300-8gpu-sglang)', matrix.rocm_version) }} strategy: fail-fast: false matrix: @@ -1975,7 +2015,7 @@ jobs: - name: Ensure VRAM is clear run: bash scripts/ci/amd/ensure_vram_clear.sh rocm - - name: Setup docker (ROCm 7.2) + - name: Setup docker (${{ matrix.rocm_version }}) run: | touch github_summary.md bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version ${{ matrix.rocm_version }} @@ -2013,6 +2053,7 @@ jobs: # ============================================================================== nightly-8-gpu-mi35x-glm52-fp8-rocm720: + name: ${{ format('nightly-8-gpu-mi35x-glm52-fp8 ({0}, linux-mi35x-gpu-8)', matrix.rocm_version) }} strategy: fail-fast: false matrix: @@ -2028,7 +2069,7 @@ jobs: - name: Ensure VRAM is clear run: bash scripts/ci/amd/ensure_vram_clear.sh rocm - - name: Setup docker (ROCm 7.2) + - name: Setup docker (${{ matrix.rocm_version }}) run: | touch github_summary.md bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version ${{ matrix.rocm_version }} @@ -2071,6 +2112,7 @@ jobs: # ============================================================================== nightly-8-gpu-mi35x-glm5-mxfp4-rocm720: + name: ${{ format('nightly-8-gpu-mi35x-glm5-mxfp4 ({0}, linux-mi35x-gpu-8)', matrix.rocm_version) }} strategy: fail-fast: false matrix: @@ -2086,7 +2128,7 @@ jobs: - name: Ensure VRAM is clear run: bash scripts/ci/amd/ensure_vram_clear.sh rocm - - name: Setup docker (ROCm 7.2) + - name: Setup docker (${{ matrix.rocm_version }}) run: | touch github_summary.md bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version ${{ matrix.rocm_version }} @@ -2126,6 +2168,7 @@ jobs: # ============================================================================== nightly-4-gpu-mi35x-minimax-m25-rocm720: + name: ${{ format('nightly-4-gpu-mi35x-minimax-m25 ({0}, linux-mi35x-gpu-8)', matrix.rocm_version) }} strategy: fail-fast: false matrix: @@ -2141,7 +2184,7 @@ jobs: - name: Ensure VRAM is clear run: bash scripts/ci/amd/ensure_vram_clear.sh rocm - - name: Setup docker (ROCm 7.2) + - name: Setup docker (${{ matrix.rocm_version }}) run: | touch github_summary.md bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version ${{ matrix.rocm_version }} @@ -2169,6 +2212,7 @@ jobs: # ============================================================================== nightly-4-gpu-mi35x-minimax-m3-rocm720: + name: ${{ format('nightly-4-gpu-mi35x-minimax-m3 ({0}, linux-mi35x-gpu-8)', matrix.rocm_version) }} strategy: fail-fast: false matrix: @@ -2184,7 +2228,7 @@ jobs: - name: Ensure VRAM is clear run: bash scripts/ci/amd/ensure_vram_clear.sh rocm - - name: Setup docker (ROCm 7.2) + - name: Setup docker (${{ matrix.rocm_version }}) run: | touch github_summary.md bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version ${{ matrix.rocm_version }} @@ -2211,6 +2255,7 @@ jobs: # ============================================================================== nightly-8-gpu-minimax-m27-rocm720: + name: ${{ format('nightly-8-gpu-minimax-m27 ({0}, linux-mi300-8gpu-sglang)', matrix.rocm_version) }} strategy: fail-fast: false matrix: @@ -2226,7 +2271,7 @@ jobs: - name: Ensure VRAM is clear run: bash scripts/ci/amd/ensure_vram_clear.sh rocm - - name: Setup docker (ROCm 7.2) + - name: Setup docker (${{ matrix.rocm_version }}) run: | touch github_summary.md bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version ${{ matrix.rocm_version }} @@ -2265,6 +2310,7 @@ jobs: # ============================================================================== nightly-1-gpu-zimage-turbo-rocm720: + name: ${{ format('nightly-1-gpu-zimage-turbo ({0}, linux-mi300-1gpu-sglang)', matrix.rocm_version) }} strategy: fail-fast: false matrix: @@ -2280,7 +2326,7 @@ jobs: - name: Ensure VRAM is clear run: bash scripts/ci/amd/ensure_vram_clear.sh rocm - - name: Setup docker (ROCm 7.2) + - name: Setup docker (${{ matrix.rocm_version }}) run: | touch github_summary.md bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version ${{ matrix.rocm_version }} diff --git a/.github/workflows/nightly-test-amd.yml b/.github/workflows/nightly-test-amd.yml index 7d04f0095..863ebc726 100644 --- a/.github/workflows/nightly-test-amd.yml +++ b/.github/workflows/nightly-test-amd.yml @@ -139,6 +139,7 @@ jobs: # ============================================================================== nightly-test-1-gpu-unit: + name: nightly-test-1-gpu-unit (rocm700, linux-mi300-1gpu-sglang) if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-test-1-gpu-unit,')) runs-on: linux-mi300-1gpu-sglang steps: @@ -170,6 +171,7 @@ jobs: exit ${TEST_EXIT_CODE:-0} nightly-test-1-gpu-kernel: + name: nightly-test-1-gpu-kernel (rocm700, linux-mi300-1gpu-sglang) if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-test-1-gpu-kernel,')) runs-on: linux-mi300-1gpu-sglang steps: @@ -201,6 +203,7 @@ jobs: exit ${TEST_EXIT_CODE:-0} nightly-test-1-gpu-mi35x: + name: nightly-test-1-gpu-mi35x (rocm700, linux-mi35x-gpu-1) if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-test-1-gpu-mi35x,')) runs-on: linux-mi35x-gpu-1 steps: @@ -239,6 +242,7 @@ jobs: # ============================================================================== nightly-accuracy-2-gpu: + name: nightly-accuracy-2-gpu (rocm700, linux-mi300-2gpu-sglang) if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-accuracy-2-gpu,')) runs-on: linux-mi300-2gpu-sglang steps: @@ -270,6 +274,7 @@ jobs: exit ${TEST_EXIT_CODE:-0} nightly-2-gpu-mi35x-glm51-mxfp4: + name: nightly-2-gpu-mi35x-glm51-mxfp4 (rocm700, linux-mi35x-gpu-2) if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-2-gpu-mi35x-glm51-mxfp4,')) runs-on: linux-mi35x-gpu-2 steps: @@ -304,6 +309,7 @@ jobs: exit ${TEST_EXIT_CODE:-0} nightly-2-gpu-mi35x-deepseek-r1-mxfp4-tp2: + name: nightly-2-gpu-mi35x-deepseek-r1-mxfp4-tp2 (rocm700, linux-mi35x-gpu-2) if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-2-gpu-mi35x-deepseek-r1-mxfp4-tp2,')) runs-on: linux-mi35x-gpu-2 steps: @@ -339,6 +345,7 @@ jobs: exit ${TEST_EXIT_CODE:-0} nightly-accuracy-2-gpu-vlm: + name: nightly-accuracy-2-gpu-vlm (rocm700, linux-mi300-2gpu-sglang) if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-accuracy-2-gpu-vlm,')) runs-on: linux-mi300-2gpu-sglang steps: @@ -371,6 +378,7 @@ jobs: exit ${TEST_EXIT_CODE:-0} nightly-perf-2-gpu-text: + name: nightly-perf-2-gpu-text (rocm700, linux-mi300-2gpu-sglang) if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-perf-2-gpu-text,')) runs-on: linux-mi300-2gpu-sglang steps: @@ -404,6 +412,7 @@ jobs: exit ${TEST_EXIT_CODE:-0} nightly-perf-2-gpu-vlm: + name: nightly-perf-2-gpu-vlm (rocm700, linux-mi300-2gpu-sglang) if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-perf-2-gpu-vlm,')) runs-on: linux-mi300-2gpu-sglang steps: @@ -437,6 +446,7 @@ jobs: exit ${TEST_EXIT_CODE:-0} nightly-4-gpu: + name: nightly-4-gpu (rocm700, linux-mi300-4gpu-sglang) if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-4-gpu,')) runs-on: linux-mi300-4gpu-sglang steps: @@ -473,6 +483,7 @@ jobs: # ============================================================================== nightly-accuracy-8-gpu: + name: nightly-accuracy-8-gpu (rocm700, linux-mi300-8gpu-sglang) if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-accuracy-8-gpu,')) runs-on: linux-mi300-8gpu-sglang steps: @@ -504,6 +515,7 @@ jobs: exit ${TEST_EXIT_CODE:-0} nightly-accuracy-8-gpu-mi35x: + name: nightly-accuracy-8-gpu-mi35x (rocm700, linux-mi35x-gpu-8) if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-accuracy-8-gpu-mi35x,')) runs-on: linux-mi35x-gpu-8 steps: @@ -542,6 +554,7 @@ jobs: # ============================================================================== nightly-8-gpu-grok1-int4: + name: nightly-8-gpu-grok1-int4 (rocm700, linux-mi300-8gpu-sglang) if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-grok1-int4,')) runs-on: linux-mi300-8gpu-sglang steps: @@ -587,6 +600,7 @@ jobs: exit ${TEST_EXIT_CODE:-0} nightly-8-gpu-mi35x-grok1-int4: + name: nightly-8-gpu-mi35x-grok1-int4 (rocm700, linux-mi35x-gpu-8) if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-mi35x-grok1-int4,')) runs-on: linux-mi35x-gpu-8 steps: @@ -639,6 +653,7 @@ jobs: # ============================================================================== nightly-8-gpu-grok2: + name: nightly-8-gpu-grok2 (rocm700, linux-mi300-8gpu-sglang) if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-grok2,')) runs-on: linux-mi300-8gpu-sglang steps: @@ -684,6 +699,7 @@ jobs: exit ${TEST_EXIT_CODE:-0} nightly-8-gpu-mi35x-grok2: + name: nightly-8-gpu-mi35x-grok2 (rocm700, linux-mi35x-gpu-8) if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-mi35x-grok2,')) runs-on: linux-mi35x-gpu-8 steps: @@ -736,6 +752,7 @@ jobs: # ============================================================================== nightly-8-gpu-deepseek-v31: + name: nightly-8-gpu-deepseek-v31 (rocm700, linux-mi300-8gpu-sglang) if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-deepseek-v31,')) runs-on: linux-mi300-8gpu-sglang steps: @@ -782,6 +799,7 @@ jobs: exit ${TEST_EXIT_CODE:-0} nightly-8-gpu-deepseek-v32: + name: nightly-8-gpu-deepseek-v32 (rocm700, linux-mi300-8gpu-sglang) if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-deepseek-v32,')) runs-on: linux-mi300-8gpu-sglang steps: @@ -826,6 +844,7 @@ jobs: exit ${TEST_EXIT_CODE:-0} nightly-8-gpu-deepseek-v32-mtp: + name: nightly-8-gpu-deepseek-v32-mtp (rocm700, linux-mi300-8gpu-sglang) if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-deepseek-v32-mtp,')) runs-on: linux-mi300-8gpu-sglang steps: @@ -870,6 +889,7 @@ jobs: exit ${TEST_EXIT_CODE:-0} nightly-8-gpu-deepseek-v3-kv-fp8: + name: nightly-8-gpu-deepseek-v3-kv-fp8 (rocm700, linux-mi300-8gpu-sglang) if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-deepseek-v3-kv-fp8,')) runs-on: linux-mi300-8gpu-sglang steps: @@ -907,6 +927,7 @@ jobs: # ============================================================================== nightly-accuracy-8-gpu-mi35x-deepseek-v32: + name: nightly-accuracy-8-gpu-mi35x-deepseek-v32 (rocm700, linux-mi35x-gpu-8) if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-accuracy-8-gpu-mi35x-deepseek-v32,')) runs-on: linux-mi35x-gpu-8 steps: @@ -943,6 +964,7 @@ jobs: exit ${TEST_EXIT_CODE:-0} nightly-accuracy-8-gpu-mi35x-deepseek-v32-mtp: + name: nightly-accuracy-8-gpu-mi35x-deepseek-v32-mtp (rocm700, linux-mi35x-gpu-8) if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-accuracy-8-gpu-mi35x-deepseek-v32-mtp,')) runs-on: linux-mi35x-gpu-8 steps: @@ -979,6 +1001,7 @@ jobs: exit ${TEST_EXIT_CODE:-0} nightly-perf-8-gpu-mi35x-deepseek-v32-basic: + name: nightly-perf-8-gpu-mi35x-deepseek-v32-basic (rocm700, linux-mi35x-gpu-8) if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-perf-8-gpu-mi35x-deepseek-v32-basic,')) runs-on: linux-mi35x-gpu-8 steps: @@ -1015,6 +1038,7 @@ jobs: exit ${TEST_EXIT_CODE:-0} nightly-perf-8-gpu-mi35x-deepseek-v32-mtp: + name: nightly-perf-8-gpu-mi35x-deepseek-v32-mtp (rocm700, linux-mi35x-gpu-8) if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-perf-8-gpu-mi35x-deepseek-v32-mtp,')) runs-on: linux-mi35x-gpu-8 steps: @@ -1055,6 +1079,7 @@ jobs: # ============================================================================== nightly-8-gpu-mi35x-deepseek-r1-mxfp4: + name: nightly-8-gpu-mi35x-deepseek-r1-mxfp4 (rocm700, linux-mi35x-gpu-8) if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-mi35x-deepseek-r1-mxfp4,')) runs-on: linux-mi35x-gpu-8 steps: @@ -1102,6 +1127,7 @@ jobs: exit ${TEST_EXIT_CODE:-0} nightly-8-gpu-mi35x-deepseek-r1-mxfp4-tp4: + name: nightly-8-gpu-mi35x-deepseek-r1-mxfp4-tp4 (rocm700, linux-mi35x-gpu-8) if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-mi35x-deepseek-r1-mxfp4-tp4,')) runs-on: linux-mi35x-gpu-8 steps: @@ -1148,6 +1174,7 @@ jobs: exit ${TEST_EXIT_CODE:-0} nightly-8-gpu-mi35x-deepseek-r1-mxfp4-kv-fp8: + name: nightly-8-gpu-mi35x-deepseek-r1-mxfp4-kv-fp8 (rocm700, linux-mi35x-gpu-8) if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-mi35x-deepseek-r1-mxfp4-kv-fp8,')) runs-on: linux-mi35x-gpu-8 steps: @@ -1195,6 +1222,7 @@ jobs: exit ${TEST_EXIT_CODE:-0} nightly-8-gpu-mi35x-deepseek-r1-mxfp4-ar-fusion: + name: nightly-8-gpu-mi35x-deepseek-r1-mxfp4-ar-fusion (rocm700, linux-mi35x-gpu-8) if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-mi35x-deepseek-r1-mxfp4-ar-fusion,')) runs-on: linux-mi35x-gpu-8 steps: @@ -1242,6 +1270,7 @@ jobs: exit ${TEST_EXIT_CODE:-0} nightly-8-gpu-mi35x-deepseek-r1-hicache: + name: nightly-8-gpu-mi35x-deepseek-r1-hicache (rocm700, linux-mi35x-gpu-8) if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-mi35x-deepseek-r1-hicache,')) runs-on: linux-mi35x-gpu-8 steps: @@ -1280,6 +1309,7 @@ jobs: # ============================================================================== nightly-8-gpu-kimi-k26: + name: nightly-8-gpu-kimi-k26 (rocm700, linux-mi300-8gpu-sglang) if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-kimi-k26,')) runs-on: linux-mi300-8gpu-sglang steps: @@ -1313,6 +1343,7 @@ jobs: exit ${TEST_EXIT_CODE:-0} nightly-8-gpu-mi35x-kimi-k26: + name: nightly-8-gpu-mi35x-kimi-k26 (rocm700, linux-mi35x-gpu-8) if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-mi35x-kimi-k26,')) runs-on: linux-mi35x-gpu-8 steps: @@ -1353,6 +1384,7 @@ jobs: # ============================================================================== nightly-8-gpu-qwen3-235b: + name: nightly-8-gpu-qwen3-235b (rocm700, linux-mi300-8gpu-sglang) if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-qwen3-235b,')) runs-on: linux-mi300-8gpu-sglang steps: @@ -1386,6 +1418,7 @@ jobs: exit ${TEST_EXIT_CODE:-0} nightly-8-gpu-mi35x-qwen3-235b-mxfp4: + name: nightly-8-gpu-mi35x-qwen3-235b-mxfp4 (rocm700, linux-mi35x-gpu-8) if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-mi35x-qwen3-235b-mxfp4,')) runs-on: linux-mi35x-gpu-8 steps: @@ -1425,6 +1458,7 @@ jobs: # ============================================================================== nightly-8-gpu-qwen35: + name: nightly-8-gpu-qwen35 (rocm700, linux-mi300-8gpu-sglang) if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-qwen35,')) runs-on: linux-mi300-8gpu-sglang steps: @@ -1473,6 +1507,7 @@ jobs: exit ${TEST_EXIT_CODE:-0} nightly-8-gpu-mi35x-qwen35: + name: nightly-8-gpu-mi35x-qwen35 (rocm700, linux-mi35x-gpu-8) if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-mi35x-qwen35,')) runs-on: linux-mi35x-gpu-8 steps: @@ -1520,6 +1555,7 @@ jobs: exit ${TEST_EXIT_CODE:-0} nightly-8-gpu-mi35x-qwen35-triton-dcp: + name: nightly-8-gpu-mi35x-qwen35-triton-dcp (rocm700, linux-mi35x-gpu-8) if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-mi35x-qwen35-triton-dcp,')) runs-on: linux-mi35x-gpu-8 steps: @@ -1558,6 +1594,7 @@ jobs: # ============================================================================== nightly-8-gpu-glm51: + name: nightly-8-gpu-glm51 (rocm700, linux-mi300-8gpu-sglang) if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-glm51,')) runs-on: linux-mi300-8gpu-sglang steps: @@ -1603,6 +1640,7 @@ jobs: exit ${TEST_EXIT_CODE:-0} nightly-8-gpu-mi35x-glm51: + name: nightly-8-gpu-mi35x-glm51 (rocm700, linux-mi35x-gpu-8) if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-mi35x-glm51,')) runs-on: linux-mi35x-gpu-8 steps: @@ -1652,6 +1690,7 @@ jobs: # ============================================================================== nightly-8-gpu-mi35x-glm5-mxfp4: + name: nightly-8-gpu-mi35x-glm5-mxfp4 (rocm700, linux-mi35x-gpu-8) if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-mi35x-glm5-mxfp4,')) runs-on: linux-mi35x-gpu-8 steps: @@ -1703,6 +1742,7 @@ jobs: # ============================================================================== nightly-4-gpu-mi35x-minimax-m25: + name: nightly-4-gpu-mi35x-minimax-m25 (rocm700, linux-mi35x-gpu-8) if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-4-gpu-mi35x-minimax-m25,')) runs-on: linux-mi35x-gpu-8 steps: @@ -1742,6 +1782,7 @@ jobs: # ============================================================================== nightly-4-gpu-mi35x-minimax-m3: + name: nightly-4-gpu-mi35x-minimax-m3 (rocm700, linux-mi35x-gpu-8) if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-4-gpu-mi35x-minimax-m3,')) runs-on: linux-mi35x-gpu-8 steps: @@ -1780,6 +1821,7 @@ jobs: # ============================================================================== nightly-8-gpu-minimax-m27: + name: nightly-8-gpu-minimax-m27 (rocm700, linux-mi300-8gpu-sglang) if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-minimax-m27,')) runs-on: linux-mi300-8gpu-sglang steps: @@ -1830,6 +1872,7 @@ jobs: # ============================================================================== nightly-1-gpu-zimage-turbo: + name: nightly-1-gpu-zimage-turbo (rocm700, linux-mi300-1gpu-sglang) if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-1-gpu-zimage-turbo,')) runs-on: linux-mi300-1gpu-sglang steps: diff --git a/.github/workflows/pr-test-amd-extra.yml b/.github/workflows/pr-test-amd-extra.yml index f3a9db140..a99d27d08 100644 --- a/.github/workflows/pr-test-amd-extra.yml +++ b/.github/workflows/pr-test-amd-extra.yml @@ -142,7 +142,7 @@ jobs: # setup across scarce AMD GPUs to shave only a couple minutes of test time, # so one GPU running the whole suite sequentially is the better trade. extra-a-test-1-gpu-small-amd: - name: ${{ format('extra-a-test-1-gpu-small-amd{0} (linux-{1}-1gpu-sglang)', inputs.rocm_version && inputs.rocm_version != 'rocm724' && format('-{0}', inputs.rocm_version) || '', inputs.runner_arch || 'mi300') }} + name: ${{ format('extra-a-test-1-gpu-small-amd ({0}, linux-{1}-1gpu-sglang)', inputs.rocm_version || 'rocm724', inputs.runner_arch || 'mi300') }} needs: [call-gate] if: ${{ !cancelled() && needs.call-gate.result == 'success' }} runs-on: ${{ format('linux-{0}-1gpu-sglang', inputs.runner_arch || 'mi300') }} @@ -175,7 +175,7 @@ jobs: # pool as small (AMD GPUs carry enough VRAM that "large" here is a CUDA # memory-tier label, not a separate AMD runner pool). extra-a-test-1-gpu-large-amd: - name: ${{ format('extra-a-test-1-gpu-large-amd{0} (linux-{1}-1gpu-sglang)', inputs.rocm_version && inputs.rocm_version != 'rocm724' && format('-{0}', inputs.rocm_version) || '', inputs.runner_arch || 'mi300') }} + name: ${{ format('extra-a-test-1-gpu-large-amd ({0}, linux-{1}-1gpu-sglang)', inputs.rocm_version || 'rocm724', inputs.runner_arch || 'mi300') }} needs: [call-gate] if: ${{ !cancelled() && needs.call-gate.result == 'success' }} runs-on: ${{ format('linux-{0}-1gpu-sglang', inputs.runner_arch || 'mi300') }} @@ -205,7 +205,7 @@ jobs: # Multi-GPU TP / PP / PD mock-model + kv_canary e2e tests. Mirrors CUDA's # extra-a 2-gpu-large stage; runs on the 2-GPU AMD pool. extra-a-test-2-gpu-large-amd: - name: ${{ format('extra-a-test-2-gpu-large-amd{0} (linux-{1}-2gpu-sglang)', inputs.rocm_version && inputs.rocm_version != 'rocm724' && format('-{0}', inputs.rocm_version) || '', inputs.runner_arch || 'mi300') }} + name: ${{ format('extra-a-test-2-gpu-large-amd ({0}, linux-{1}-2gpu-sglang)', inputs.rocm_version || 'rocm724', inputs.runner_arch || 'mi300') }} needs: [call-gate] if: ${{ !cancelled() && needs.call-gate.result == 'success' }} runs-on: ${{ format('linux-{0}-2gpu-sglang', inputs.runner_arch || 'mi300') }} diff --git a/.github/workflows/pr-test-amd-rocm720.yml b/.github/workflows/pr-test-amd-rocm720.yml index 07a8a9ce1..59225ee64 100644 --- a/.github/workflows/pr-test-amd-rocm720.yml +++ b/.github/workflows/pr-test-amd-rocm720.yml @@ -242,6 +242,7 @@ jobs: # Not added to `pr-test-amd-rocm720-finish` so the base AMD gate never depends on # the opt-in extra suite. call-pr-test-amd-extra-rocm720: + name: call-pr-test-amd-extra if: | (github.event_name == 'schedule' || inputs.run_all_tests == true) && !(inputs.target_stage || inputs.target_stage_select) @@ -257,7 +258,7 @@ jobs: # =============================================== sgl-kernel ==================================================== sgl-kernel-unit-test-amd-rocm720: - name: ${{ format('sgl-kernel-unit-test-amd-rocm720 (linux-{0}-1gpu-sglang)', inputs.runner_arch || 'mi300') }} + name: ${{ format('sgl-kernel-unit-test-amd ({0}, linux-{1}-1gpu-sglang)', inputs.rocm_version || 'rocm724', inputs.runner_arch || 'mi300') }} needs: [check-changes, call-gate] if: | always() && !cancelled() && @@ -315,7 +316,7 @@ jobs: exit $failures sgl-kernel-unit-test-2-gpu-amd-rocm720: - name: ${{ format('sgl-kernel-unit-test-2-gpu-amd-rocm720 (linux-{0}-2gpu-sglang)', inputs.runner_arch || 'mi300') }} + name: ${{ format('sgl-kernel-unit-test-2-gpu-amd ({0}, linux-{1}-2gpu-sglang)', inputs.rocm_version || 'rocm724', inputs.runner_arch || 'mi300') }} needs: [check-changes, call-gate] if: | always() && !cancelled() && @@ -356,7 +357,7 @@ jobs: # =============================================== primary ==================================================== stage-a-test-1-gpu-small-amd-rocm720: - name: ${{ format('stage-a-test-1-gpu-small-amd-rocm720 (linux-{0}-1gpu-sglang)', inputs.runner_arch || 'mi300') }} + name: ${{ format('stage-a-test-1-gpu-small-amd ({0}, linux-{1}-1gpu-sglang)', inputs.rocm_version || 'rocm724', inputs.runner_arch || 'mi300') }} needs: [check-changes, call-gate] if: | always() && !cancelled() && @@ -393,7 +394,7 @@ jobs: bash scripts/ci/amd/amd_ci_exec.sh -w "/sglang-checkout/test" python3 run_suite.py --hw amd --suite stage-a-test-1-gpu-small-amd ${{ needs.check-changes.outputs.continue_on_error == 'true' && '--continue-on-error' || '' }} jit-kernel-unit-test-amd-rocm720: - name: ${{ format('jit-kernel-unit-test-amd-rocm720 (linux-{0}-1gpu-sglang)', inputs.runner_arch || 'mi300') }} + name: ${{ format('jit-kernel-unit-test-amd ({0}, linux-{1}-1gpu-sglang)', inputs.rocm_version || 'rocm724', inputs.runner_arch || 'mi300') }} needs: [check-changes, call-gate] if: | always() && !cancelled() && @@ -430,7 +431,7 @@ jobs: bash scripts/ci/amd/amd_ci_exec.sh -w "/sglang-checkout/test" python3 run_suite.py --hw amd --suite jit-kernel-unit-test-amd ${{ needs.check-changes.outputs.continue_on_error == 'true' && '--continue-on-error' || '' }} jit-kernel-benchmark-test-amd-rocm720: - name: ${{ format('jit-kernel-benchmark-test-amd-rocm720 (linux-{0}-1gpu-sglang)', inputs.runner_arch || 'mi300') }} + name: ${{ format('jit-kernel-benchmark-test-amd ({0}, linux-{1}-1gpu-sglang)', inputs.rocm_version || 'rocm724', inputs.runner_arch || 'mi300') }} needs: [check-changes, call-gate] if: | always() && !cancelled() && @@ -472,6 +473,7 @@ jobs: # For scheduled runs: wait jobs are skipped, enabling parallel execution of all stages. wait-for-stage-a-amd-rocm720: + name: wait-for-stage-a-amd needs: [check-changes, call-gate] if: | always() && @@ -488,12 +490,12 @@ jobs: - uses: ./.github/actions/wait-for-jobs id: wait with: - stage-name: stage-a-amd-rocm720 - jobs: '[{"prefix": "stage-a-test-1-gpu-small-amd-rocm720", "expected_count": 1}]' + stage-name: stage-a-amd + jobs: '[{"prefix": "stage-a-test-1-gpu-small-amd", "expected_count": 1}]' max-wait-minutes: '240' stage-b-test-1-gpu-small-amd-rocm720: - name: ${{ format('stage-b-test-1-gpu-small-amd-rocm720 (linux-{0}-1gpu-sglang, {1})', inputs.runner_arch || 'mi300', matrix.part) }} + name: ${{ format('stage-b-test-1-gpu-small-amd ({0}, linux-{1}-1gpu-sglang, {2})', inputs.rocm_version || 'rocm724', inputs.runner_arch || 'mi300', matrix.part) }} needs: [check-changes, wait-for-stage-a-amd-rocm720] if: | always() && @@ -534,7 +536,7 @@ jobs: bash scripts/ci/amd/amd_ci_exec.sh -w "/sglang-checkout/test" python3 run_suite.py --hw amd --suite stage-b-test-1-gpu-small-amd --auto-partition-id ${{ matrix.part }} --auto-partition-size 14 --timeout-per-file 2400 ${{ needs.check-changes.outputs.continue_on_error == 'true' && '--continue-on-error' || '' }} stage-b-test-1-gpu-small-amd-nondeterministic-rocm720: - name: ${{ format('stage-b-test-1-gpu-small-amd-nondeterministic-rocm720 (linux-{0}-1gpu-sglang)', inputs.runner_arch || 'mi300') }} + name: ${{ format('stage-b-test-1-gpu-small-amd-nondeterministic ({0}, linux-{1}-1gpu-sglang)', inputs.rocm_version || 'rocm724', inputs.runner_arch || 'mi300') }} needs: [check-changes, wait-for-stage-a-amd-rocm720] if: | always() && @@ -570,6 +572,7 @@ jobs: bash scripts/ci/amd/amd_ci_exec.sh -w "/sglang-checkout/test" python3 run_suite.py --hw amd --suite stage-b-test-1-gpu-small-amd-nondeterministic --timeout-per-file 3600 ${{ needs.check-changes.outputs.continue_on_error == 'true' && '--continue-on-error' || '' }} stage-b-test-1-gpu-small-amd-mi35x-rocm720: + name: ${{ format('stage-b-test-1-gpu-small-amd-mi35x ({0}, {1})', inputs.rocm_version || 'rocm724', matrix.runner) }} needs: [check-changes, wait-for-stage-a-amd-rocm720] if: | always() && @@ -609,7 +612,7 @@ jobs: bash scripts/ci/amd/amd_ci_exec.sh -w "/sglang-checkout/test" python3 run_suite.py --hw amd --suite stage-b-test-1-gpu-small-amd-mi35x ${{ needs.check-changes.outputs.continue_on_error == 'true' && '--continue-on-error' || '' }} stage-b-test-1-gpu-large-amd-rocm720: - name: ${{ format('stage-b-test-1-gpu-large-amd-rocm720 (linux-{0}-1gpu-sglang, {1})', inputs.runner_arch || 'mi300', matrix.part) }} + name: ${{ format('stage-b-test-1-gpu-large-amd ({0}, linux-{1}-1gpu-sglang, {2})', inputs.rocm_version || 'rocm724', inputs.runner_arch || 'mi300', matrix.part) }} needs: [check-changes, wait-for-stage-a-amd-rocm720] if: | always() && @@ -650,7 +653,7 @@ jobs: bash scripts/ci/amd/amd_ci_exec.sh -w "/sglang-checkout/test" python3 run_suite.py --hw amd --suite stage-b-test-1-gpu-large-amd --auto-partition-id ${{ matrix.part }} --auto-partition-size 3 --timeout-per-file 2700 ${{ needs.check-changes.outputs.continue_on_error == 'true' && '--continue-on-error' || '' }} stage-b-test-2-gpu-large-amd-rocm720: - name: ${{ format('stage-b-test-2-gpu-large-amd-rocm720 (linux-{0}-2gpu-sglang, {1})', inputs.runner_arch || 'mi300', matrix.part) }} + name: ${{ format('stage-b-test-2-gpu-large-amd ({0}, linux-{1}-2gpu-sglang, {2})', inputs.rocm_version || 'rocm724', inputs.runner_arch || 'mi300', matrix.part) }} needs: [check-changes, wait-for-stage-a-amd-rocm720] if: | always() && @@ -691,7 +694,7 @@ jobs: bash scripts/ci/amd/amd_ci_exec.sh -w "/sglang-checkout/test" python3 run_suite.py --hw amd --suite stage-b-test-2-gpu-large-amd --auto-partition-id ${{ matrix.part }} --auto-partition-size 2 --timeout-per-file ${{ (inputs.runner_arch || 'mi300') == 'mi300' && 5400 || 2700 }} ${{ needs.check-changes.outputs.continue_on_error == 'true' && '--continue-on-error' || '' }} multimodal-gen-test-1-gpu-amd-rocm720: - name: ${{ format('multimodal-gen-test-1-gpu-amd-rocm720 (linux-{0}-1gpu-sglang, {1})', inputs.runner_arch || 'mi300', matrix.part) }} + name: ${{ format('multimodal-gen-test-1-gpu-amd ({0}, linux-{1}-1gpu-sglang, {2})', inputs.rocm_version || 'rocm724', inputs.runner_arch || 'mi300', matrix.part) }} needs: [check-changes, call-gate] if: | always() && !cancelled() && @@ -840,7 +843,7 @@ jobs: retention-days: 7 multimodal-gen-test-2-gpu-amd-rocm720: - name: ${{ format('multimodal-gen-test-2-gpu-amd-rocm720 (linux-{0}-2gpu-sglang, {1})', inputs.runner_arch || 'mi300', matrix.part) }} + name: ${{ format('multimodal-gen-test-2-gpu-amd ({0}, linux-{1}-2gpu-sglang, {2})', inputs.rocm_version || 'rocm724', inputs.runner_arch || 'mi300', matrix.part) }} needs: [check-changes, call-gate] if: | always() && !cancelled() && @@ -975,6 +978,7 @@ jobs: retention-days: 7 wait-for-stage-b-amd-rocm720: + name: wait-for-stage-b-amd needs: [check-changes, call-gate, wait-for-stage-a-amd-rocm720] if: | always() && @@ -992,17 +996,17 @@ jobs: - uses: ./.github/actions/wait-for-jobs id: wait with: - stage-name: stage-b-amd-rocm720 + stage-name: stage-b-amd jobs: | [ - {"prefix": "stage-b-test-1-gpu-small-amd-rocm720", "expected_count": 14}, - {"prefix": "stage-b-test-1-gpu-large-amd-rocm720", "expected_count": 3}, - {"prefix": "stage-b-test-2-gpu-large-amd-rocm720", "expected_count": 2} + {"prefix": "stage-b-test-1-gpu-small-amd", "expected_count": 14}, + {"prefix": "stage-b-test-1-gpu-large-amd", "expected_count": 3}, + {"prefix": "stage-b-test-2-gpu-large-amd", "expected_count": 2} ] max-wait-minutes: '480' stage-c-test-4-gpu-amd-rocm720: - name: ${{ format('stage-c-test-4-gpu-amd-rocm720 (linux-{0}-4gpu-sglang, {1})', inputs.runner_arch || 'mi300', matrix.part) }} + name: ${{ format('stage-c-test-4-gpu-amd ({0}, linux-{1}-4gpu-sglang, {2})', inputs.rocm_version || 'rocm724', inputs.runner_arch || 'mi300', matrix.part) }} needs: [check-changes, call-gate, wait-for-stage-b-amd-rocm720] if: | always() && @@ -1058,7 +1062,7 @@ jobs: ${{ needs.check-changes.outputs.continue_on_error == 'true' && '--continue-on-error' || '' }} stage-c-test-large-8-gpu-amd-rocm720: - name: ${{ format('stage-c-test-large-8-gpu-amd-rocm720 (linux-{0}-8gpu-sglang, {1})', inputs.runner_arch || 'mi300', matrix.part) }} + name: ${{ format('stage-c-test-large-8-gpu-amd ({0}, linux-{1}-8gpu-sglang, {2})', inputs.rocm_version || 'rocm724', inputs.runner_arch || 'mi300', matrix.part) }} needs: [check-changes, call-gate, wait-for-stage-b-amd-rocm720] if: | always() && @@ -1107,6 +1111,7 @@ jobs: bash scripts/ci/amd/amd_ci_exec.sh -w "/sglang-checkout/test" python3 run_suite.py --hw amd --suite stage-c-test-large-8-gpu-amd --auto-partition-id ${{ matrix.part }} --auto-partition-size 4 --timeout-per-file 5400 ${{ needs.check-changes.outputs.continue_on_error == 'true' && '--continue-on-error' || '' }} stage-c-test-large-8-gpu-amd-mi35x-rocm720: + name: ${{ format('stage-c-test-large-8-gpu-amd-mi35x ({0}, {1}, {2})', inputs.rocm_version || 'rocm724', matrix.runner, matrix.part) }} needs: [check-changes, call-gate, wait-for-stage-b-amd-rocm720] if: | always() && @@ -1150,6 +1155,7 @@ jobs: # GSM8K accuracy on the nightly dsv4 suites, ~20min each. Gated as stage-C so # every PR that touches the main package covers DeepSeek-V4. stage-c-dsv4-flash-fp4-fp8-amd-mi35x-rocm720: + name: ${{ format('stage-c-dsv4-flash-fp4-fp8-amd-mi35x ({0}, {1})', inputs.rocm_version || 'rocm724', matrix.runner) }} needs: [check-changes, call-gate, wait-for-stage-b-amd-rocm720] if: | always() && @@ -1204,6 +1210,7 @@ jobs: exit ${TEST_EXIT_CODE:-0} stage-c-dsv4-pro-fp4-amd-mi35x-rocm720: + name: ${{ format('stage-c-dsv4-pro-fp4-amd-mi35x ({0}, {1})', inputs.rocm_version || 'rocm724', matrix.runner) }} needs: [check-changes, call-gate, wait-for-stage-b-amd-rocm720] if: | always() && @@ -1259,6 +1266,7 @@ jobs: # =============================================== Disaggregation ==================================================== stage-b-test-large-8-gpu-mi35x-disaggregation-amd-rocm720: + name: ${{ format('stage-b-test-large-8-gpu-mi35x-disaggregation-amd ({0}, {1})', inputs.rocm_version || 'rocm724', matrix.runner) }} needs: [check-changes, wait-for-stage-a-amd-rocm720] if: | always() && @@ -1392,6 +1400,7 @@ jobs: -w "/sglang-checkout/test" python3 run_suite.py --hw amd --suite stage-b-test-large-8-gpu-mi35x-disaggregation-amd --timeout-per-file 1800 ${{ needs.check-changes.outputs.continue_on_error == 'true' && '--continue-on-error' || '' }} pr-test-amd-rocm720-finish: + name: pr-test-amd-finish needs: [ call-gate, diff --git a/.github/workflows/pr-test-amd.yml b/.github/workflows/pr-test-amd.yml index 0e99d1f25..edc6fcb60 100644 --- a/.github/workflows/pr-test-amd.yml +++ b/.github/workflows/pr-test-amd.yml @@ -226,7 +226,7 @@ jobs: # =============================================== sgl-kernel ==================================================== sgl-kernel-unit-test-amd: - name: ${{ format('sgl-kernel-unit-test-amd (linux-{0}-1gpu-sglang)', inputs.runner_arch || 'mi300') }} + name: ${{ format('sgl-kernel-unit-test-amd (rocm700, linux-{0}-1gpu-sglang)', inputs.runner_arch || 'mi300') }} needs: [check-changes, call-gate] if: | always() && !cancelled() && @@ -284,7 +284,7 @@ jobs: exit $failures sgl-kernel-unit-test-2-gpu-amd: - name: ${{ format('sgl-kernel-unit-test-2-gpu-amd (linux-{0}-2gpu-sglang)', inputs.runner_arch || 'mi300') }} + name: ${{ format('sgl-kernel-unit-test-2-gpu-amd (rocm700, linux-{0}-2gpu-sglang)', inputs.runner_arch || 'mi300') }} needs: [check-changes, call-gate] if: | always() && !cancelled() && @@ -325,7 +325,7 @@ jobs: # =============================================== primary ==================================================== stage-a-test-1-gpu-small-amd: - name: ${{ format('stage-a-test-1-gpu-small-amd (linux-{0}-1gpu-sglang)', inputs.runner_arch || 'mi300') }} + name: ${{ format('stage-a-test-1-gpu-small-amd (rocm700, linux-{0}-1gpu-sglang)', inputs.runner_arch || 'mi300') }} needs: [check-changes, call-gate] if: | always() && !cancelled() && @@ -362,7 +362,7 @@ jobs: bash scripts/ci/amd/amd_ci_exec.sh -w "/sglang-checkout/test" python3 run_suite.py --hw amd --suite stage-a-test-1-gpu-small-amd ${{ needs.check-changes.outputs.continue_on_error == 'true' && '--continue-on-error' || '' }} jit-kernel-unit-test-amd: - name: ${{ format('jit-kernel-unit-test-amd (linux-{0}-1gpu-sglang)', inputs.runner_arch || 'mi300') }} + name: ${{ format('jit-kernel-unit-test-amd (rocm700, linux-{0}-1gpu-sglang)', inputs.runner_arch || 'mi300') }} needs: [check-changes, call-gate] if: | always() && !cancelled() && @@ -399,7 +399,7 @@ jobs: bash scripts/ci/amd/amd_ci_exec.sh -w "/sglang-checkout/test" python3 run_suite.py --hw amd --suite jit-kernel-unit-test-amd ${{ needs.check-changes.outputs.continue_on_error == 'true' && '--continue-on-error' || '' }} jit-kernel-benchmark-test-amd: - name: ${{ format('jit-kernel-benchmark-test-amd (linux-{0}-1gpu-sglang)', inputs.runner_arch || 'mi300') }} + name: ${{ format('jit-kernel-benchmark-test-amd (rocm700, linux-{0}-1gpu-sglang)', inputs.runner_arch || 'mi300') }} needs: [check-changes, call-gate] if: | always() && !cancelled() && @@ -462,7 +462,7 @@ jobs: max-wait-minutes: '240' stage-b-test-1-gpu-small-amd: - name: ${{ format('stage-b-test-1-gpu-small-amd (linux-{0}-1gpu-sglang, {1})', inputs.runner_arch || 'mi300', matrix.part) }} + name: ${{ format('stage-b-test-1-gpu-small-amd (rocm700, linux-{0}-1gpu-sglang, {1})', inputs.runner_arch || 'mi300', matrix.part) }} needs: [check-changes, wait-for-stage-a-amd] if: | always() && @@ -502,7 +502,7 @@ jobs: bash scripts/ci/amd/amd_ci_exec.sh -w "/sglang-checkout/test" python3 run_suite.py --hw amd --suite stage-b-test-1-gpu-small-amd --auto-partition-id ${{ matrix.part }} --auto-partition-size 14 --timeout-per-file 2400 ${{ needs.check-changes.outputs.continue_on_error == 'true' && '--continue-on-error' || '' }} stage-b-test-1-gpu-small-amd-nondeterministic: - name: ${{ format('stage-b-test-1-gpu-small-amd-nondeterministic (linux-{0}-1gpu-sglang)', inputs.runner_arch || 'mi300') }} + name: ${{ format('stage-b-test-1-gpu-small-amd-nondeterministic (rocm700, linux-{0}-1gpu-sglang)', inputs.runner_arch || 'mi300') }} needs: [check-changes, wait-for-stage-a-amd] if: | always() && @@ -538,6 +538,7 @@ jobs: bash scripts/ci/amd/amd_ci_exec.sh -w "/sglang-checkout/test" python3 run_suite.py --hw amd --suite stage-b-test-1-gpu-small-amd-nondeterministic --timeout-per-file 3600 ${{ needs.check-changes.outputs.continue_on_error == 'true' && '--continue-on-error' || '' }} stage-b-test-1-gpu-small-amd-mi35x: + name: ${{ format('stage-b-test-1-gpu-small-amd-mi35x (rocm700, {0})', matrix.runner) }} needs: [check-changes, wait-for-stage-a-amd] if: | always() && @@ -577,7 +578,7 @@ jobs: bash scripts/ci/amd/amd_ci_exec.sh -w "/sglang-checkout/test" python3 run_suite.py --hw amd --suite stage-b-test-1-gpu-small-amd-mi35x ${{ needs.check-changes.outputs.continue_on_error == 'true' && '--continue-on-error' || '' }} stage-b-test-1-gpu-large-amd: - name: ${{ format('stage-b-test-1-gpu-large-amd (linux-{0}-1gpu-sglang, {1})', inputs.runner_arch || 'mi300', matrix.part) }} + name: ${{ format('stage-b-test-1-gpu-large-amd (rocm700, linux-{0}-1gpu-sglang, {1})', inputs.runner_arch || 'mi300', matrix.part) }} needs: [check-changes, wait-for-stage-a-amd] if: | always() && @@ -617,7 +618,7 @@ jobs: bash scripts/ci/amd/amd_ci_exec.sh -w "/sglang-checkout/test" python3 run_suite.py --hw amd --suite stage-b-test-1-gpu-large-amd --auto-partition-id ${{ matrix.part }} --auto-partition-size 3 --timeout-per-file 2700 ${{ needs.check-changes.outputs.continue_on_error == 'true' && '--continue-on-error' || '' }} stage-b-test-2-gpu-large-amd: - name: ${{ format('stage-b-test-2-gpu-large-amd (linux-{0}-2gpu-sglang, {1})', inputs.runner_arch || 'mi300', matrix.part) }} + name: ${{ format('stage-b-test-2-gpu-large-amd (rocm700, linux-{0}-2gpu-sglang, {1})', inputs.runner_arch || 'mi300', matrix.part) }} needs: [check-changes, wait-for-stage-a-amd] if: | always() && @@ -657,7 +658,7 @@ jobs: bash scripts/ci/amd/amd_ci_exec.sh -w "/sglang-checkout/test" python3 run_suite.py --hw amd --suite stage-b-test-2-gpu-large-amd --auto-partition-id ${{ matrix.part }} --auto-partition-size 2 --timeout-per-file ${{ (inputs.runner_arch || 'mi300') == 'mi300' && 5400 || 2700 }} ${{ needs.check-changes.outputs.continue_on_error == 'true' && '--continue-on-error' || '' }} multimodal-gen-test-1-gpu-amd: - name: ${{ format('multimodal-gen-test-1-gpu-amd (linux-{0}-1gpu-sglang, {1})', inputs.runner_arch || 'mi300', matrix.part) }} + name: ${{ format('multimodal-gen-test-1-gpu-amd (rocm700, linux-{0}-1gpu-sglang, {1})', inputs.runner_arch || 'mi300', matrix.part) }} needs: [check-changes, call-gate] if: | always() && !cancelled() && @@ -805,7 +806,7 @@ jobs: retention-days: 7 multimodal-gen-test-2-gpu-amd: - name: ${{ format('multimodal-gen-test-2-gpu-amd (linux-{0}-2gpu-sglang, {1})', inputs.runner_arch || 'mi300', matrix.part) }} + name: ${{ format('multimodal-gen-test-2-gpu-amd (rocm700, linux-{0}-2gpu-sglang, {1})', inputs.runner_arch || 'mi300', matrix.part) }} needs: [check-changes, call-gate] if: | always() && !cancelled() && @@ -966,7 +967,7 @@ jobs: max-wait-minutes: '480' stage-c-test-4-gpu-amd: - name: ${{ format('stage-c-test-4-gpu-amd (linux-{0}-4gpu-sglang, {1})', inputs.runner_arch || 'mi300', matrix.part) }} + name: ${{ format('stage-c-test-4-gpu-amd (rocm700, linux-{0}-4gpu-sglang, {1})', inputs.runner_arch || 'mi300', matrix.part) }} needs: [check-changes, call-gate, wait-for-stage-b-amd] if: | always() && @@ -1022,7 +1023,7 @@ jobs: ${{ needs.check-changes.outputs.continue_on_error == 'true' && '--continue-on-error' || '' }} stage-c-test-large-8-gpu-amd: - name: ${{ format('stage-c-test-large-8-gpu-amd (linux-{0}-8gpu-sglang, {1})', inputs.runner_arch || 'mi300', matrix.part) }} + name: ${{ format('stage-c-test-large-8-gpu-amd (rocm700, linux-{0}-8gpu-sglang, {1})', inputs.runner_arch || 'mi300', matrix.part) }} needs: [check-changes, call-gate, wait-for-stage-b-amd] if: | always() && @@ -1070,6 +1071,7 @@ jobs: bash scripts/ci/amd/amd_ci_exec.sh -w "/sglang-checkout/test" python3 run_suite.py --hw amd --suite stage-c-test-large-8-gpu-amd --auto-partition-id ${{ matrix.part }} --auto-partition-size 4 --timeout-per-file 5400 ${{ needs.check-changes.outputs.continue_on_error == 'true' && '--continue-on-error' || '' }} stage-c-test-large-8-gpu-amd-mi35x: + name: ${{ format('stage-c-test-large-8-gpu-amd-mi35x (rocm700, {0}, {1})', matrix.runner, matrix.part) }} needs: [check-changes, call-gate, wait-for-stage-b-amd] if: | always() && @@ -1111,6 +1113,7 @@ jobs: # =============================================== Disaggregation ==================================================== stage-b-test-large-8-gpu-mi35x-disaggregation-amd: + name: ${{ format('stage-b-test-large-8-gpu-mi35x-disaggregation-amd (rocm700, {0})', matrix.runner) }} needs: [check-changes, wait-for-stage-a-amd] if: | always() && diff --git a/scripts/ci/utils/query_job_status.py b/scripts/ci/utils/query_job_status.py index 02e4cf244..e4ddafa13 100755 --- a/scripts/ci/utils/query_job_status.py +++ b/scripts/ci/utils/query_job_status.py @@ -6,7 +6,9 @@ Usage: # Per-job reports (original mode) python scripts/ci/utils/query_job_status.py --job "stage-c-test-large-8-gpu-amd-mi35x" python scripts/ci/utils/query_job_status.py --job "stage-c-test-large-8-gpu-amd-mi35x" --hours 48 - python scripts/ci/utils/query_job_status.py --job "stage-c-test-large-8-gpu-amd-mi35x-rocm720" --workflow "pr-test-amd-rocm720.yml" --input-data-file actions-job-snapshot.json --summary + # A stem matches every ROCm flavor; use the full display name to pin one + python scripts/ci/utils/query_job_status.py --job "stage-c-test-large-8-gpu-amd-mi35x" --workflow "pr-test-amd-rocm720.yml" --input-data-file actions-job-snapshot.json --summary + python scripts/ci/utils/query_job_status.py --job "nightly-test-1-gpu-unit (rocm724, linux-mi300-1gpu-sglang)" --workflow "nightly-test-amd-rocm720.yml" --input-data-file actions-job-snapshot.json --summary # Runner fleet report (cross-workflow runner analytics) python scripts/ci/utils/query_job_status.py --runner-report --workflow "pr-test-amd-rocm720.yml,nightly-test-amd-rocm720.yml" --hours 24 diff --git a/scripts/ci_monitor/ci_failures_analysis.py b/scripts/ci_monitor/ci_failures_analysis.py index 7328a6e21..d06532801 100644 --- a/scripts/ci_monitor/ci_failures_analysis.py +++ b/scripts/ci_monitor/ci_failures_analysis.py @@ -27,6 +27,51 @@ from typing import Dict, List, Optional, Tuple import requests +def _filter_legacy_amd_job_rows(job_data: Dict[str, Dict]) -> Dict[str, Dict]: + """Drop pre-cutover AMD names without changing the shared analyzer.""" + + filtered = {} + for full_name, data in job_data.items(): + # This caller was renamed by the AMD job-name cutover. Other outer + # callers, including AITER's *-rocm720 callers, are still current. + name_parts = full_name.split(" / ") + if "call-pr-test-amd-extra-rocm720" in name_parts[:-1]: + continue + + leaf_name = name_parts[-1] + if leaf_name.startswith(("wait-for-stage-a-amd", "wait-for-stage-b-amd")): + continue + if leaf_name in { + "call-gate", + "call-pr-test-amd-extra", + "check-all-jobs", + "check-changes", + "pr-gate", + "pr-test-amd-extra-finish", + "pr-test-amd-finish", + "pr-test-amd-rocm720-finish", + }: + continue + + # Old display-name stems ended in -rocm. Their version was not + # reliable enough to map forward: a -rocm720 PR job could run rocm724. + stem = leaf_name.split(" (", 1)[0] + _, separator, version = stem.rpartition("-rocm") + if separator and version.isdigit(): + continue + + # An intermediate nightly schema showed only the ROCm flavor. Current + # names always include both the flavor and runner inside parentheses. + if leaf_name.endswith(")") and " (" in leaf_name: + details = leaf_name.rsplit(" (", 1)[1][:-1] + if details.startswith("rocm") and details[4:].isdigit(): + continue + + filtered[full_name] = data + + return filtered + + class SGLangFailuresAnalyzer: """Analyzes consecutive failures in GitHub Actions workflows.""" @@ -2648,6 +2693,21 @@ def main(): else ({}, {}) ) + # AMD renamed its display names to carry the actual ROCm flavor. Reset + # only AMD history at that boundary so legacy failures cannot remain + # "current" under names that no longer exist. Other platforms continue + # to use the unmodified shared analyzer results. General AMD reports + # intentionally omit old-schema rows from branches that have not moved + # to the new names; their ROCm flavor cannot be mapped reliably. + pr_test_amd_scheduled_data = _filter_legacy_amd_job_rows( + pr_test_amd_scheduled_data + ) + nightly_amd_scheduled_data = _filter_legacy_amd_job_rows( + nightly_amd_scheduled_data + ) + pr_test_amd_general_data = _filter_legacy_amd_job_rows(pr_test_amd_general_data) + nightly_amd_general_data = _filter_legacy_amd_job_rows(nightly_amd_general_data) + # Analyze runner health and consecutive failures on all runs ( runner_stats, diff --git a/scripts/ci_monitor/test_ci_failures_analysis.py b/scripts/ci_monitor/test_ci_failures_analysis.py new file mode 100644 index 000000000..3434f5ad0 --- /dev/null +++ b/scripts/ci_monitor/test_ci_failures_analysis.py @@ -0,0 +1,57 @@ +"""AMD job-name cutover tests. + +Run with: + python -m unittest discover -s scripts/ci_monitor -p 'test_ci_failures_analysis.py' +""" + +import os +import sys +import unittest + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +from ci_failures_analysis import _filter_legacy_amd_job_rows # noqa: E402 + + +class TestFilterLegacyAmdJobRows(unittest.TestCase): + def test_drops_legacy_names_and_nested_utilities(self): + rows = { + "stage-b-test-1-gpu-small-amd-rocm720 (linux-mi300-1gpu-sglang, 0)": {}, + "nightly-accuracy-2-gpu-rocm720 (rocm724)": {}, + "nightly-accuracy-2-gpu-rocm724": {}, + "nightly-test-1-gpu-unit (rocm724)": {}, + "call-pr-test-amd-rocm720 / call-pr-test-amd-extra-rocm720 / extra-a-test-1-gpu-small-amd (linux-mi300-1gpu-sglang)": {}, + "wait-for-stage-a-amd": {}, + "call-pr-test-amd-extra / pr-test-amd-extra-finish": {}, + "call-pr-test-amd-extra / call-gate / pr-gate": {}, + } + + self.assertEqual(_filter_legacy_amd_job_rows(rows), {}) + + def test_keeps_current_flavors_and_nested_callers_separate(self): + new_success = {"current_streak": 0} + rows = { + "stage-b-test-1-gpu-small-amd (rocm724, linux-mi300-1gpu-sglang, 0)": new_success, + "nightly-accuracy-2-gpu (rocm720, linux-mi300-2gpu-sglang)": { + "current_streak": 1 + }, + "call-pr-test-amd-rocm720 / stage-c-test-4-gpu-amd (rocm724, linux-mi300-4gpu-sglang, 0)": { + "current_streak": 0 + }, + "call-pr-test-amd-rocm720 / call-pr-test-amd-extra / extra-a-test-1-gpu-small-amd (rocm724, linux-mi300-1gpu-sglang)": { + "current_streak": 0 + }, + } + + filtered = _filter_legacy_amd_job_rows(rows) + + self.assertEqual(set(filtered), set(rows)) + self.assertIs( + filtered[ + "stage-b-test-1-gpu-small-amd (rocm724, linux-mi300-1gpu-sglang, 0)" + ], + new_success, + ) + + +if __name__ == "__main__": + unittest.main()