From 07821e9d568d2732575ab769b079ec59d99265ba Mon Sep 17 00:00:00 2001 From: Michael <13900043+michaelzhang-ai@users.noreply.github.com> Date: Thu, 13 Aug 2026 07:22:42 -0700 Subject: [PATCH] [AMD][CI] Stop scheduling Grok-1 and Grok-2 on MI30x (#34643) Co-authored-by: Cursor Agent Co-authored-by: Michael --- .../workflows/nightly-test-amd-rocm720.yml | 122 ++---------------- .github/workflows/nightly-test-amd.yml | 122 ++---------------- 2 files changed, 18 insertions(+), 226 deletions(-) diff --git a/.github/workflows/nightly-test-amd-rocm720.yml b/.github/workflows/nightly-test-amd-rocm720.yml index 16f0d3088..ec46b74a6 100644 --- a/.github/workflows/nightly-test-amd-rocm720.yml +++ b/.github/workflows/nightly-test-amd-rocm720.yml @@ -41,14 +41,12 @@ on: - nightly-2-gpu-mi35x-glm51-mxfp4-rocm720 # 2-GPU DeepSeek-R1-MXFP4 TP2 (MI35x only) - nightly-2-gpu-mi35x-deepseek-r1-mxfp4-tp2-rocm720 - # 8-GPU GPT-OSS (MI30x mixes Grok1-FP8; MI35x mixes Qwen3-Coder-Next) + # 8-GPU GPT-OSS (MI30x + MI35x) - nightly-accuracy-8-gpu-rocm720 - nightly-accuracy-8-gpu-mi35x-rocm720 - # 8-GPU Grok1-INT4 (MI30x + MI35x) - - nightly-8-gpu-grok1-int4-rocm720 + # 8-GPU Grok1-INT4 (MI35x) - nightly-8-gpu-mi35x-grok1-int4-rocm720 - # 8-GPU Grok2 (MI30x + MI35x) - - nightly-8-gpu-grok2-rocm720 + # 8-GPU Grok2 (MI35x) - nightly-8-gpu-mi35x-grok2-rocm720 # 8-GPU DeepSeek-V3.x (MI30x) - nightly-8-gpu-deepseek-v31-rocm720 @@ -465,7 +463,7 @@ jobs: exit ${TEST_EXIT_CODE:-0} # ============================================================================== - # 8-GPU GPT-OSS (MI30x mixes Grok1-FP8; MI35x mixes Qwen3-Coder-Next) + # 8-GPU GPT-OSS (MI30x + MI35x) # ============================================================================== nightly-accuracy-8-gpu-rocm720: @@ -500,16 +498,6 @@ jobs: echo "$(> $GITHUB_STEP_SUMMARY || true exit ${TEST_EXIT_CODE:-0} - - name: Accuracy Test ROCm 7.2 (8-GPU Grok1-FP8) - timeout-minutes: 60 - run: | - bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ - -e RCCL_MSCCL_ENABLE=0 \ - -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ - python3 run_suite.py --hw amd --suite nightly-amd-accuracy-8-gpu-grok1-fp8 --nightly --timeout-per-file 3600 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$? - echo "$(> $GITHUB_STEP_SUMMARY || true - exit ${TEST_EXIT_CODE:-0} - nightly-accuracy-8-gpu-mi35x-rocm720: if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-accuracy-8-gpu-mi35x-rocm720,')) runs-on: linux-mi35x-gpu-8 @@ -545,54 +533,9 @@ jobs: exit ${TEST_EXIT_CODE:-0} # ============================================================================== - # 8-GPU Grok1-INT4 (MI30x + MI35x) + # 8-GPU Grok1-INT4 (MI35x) # ============================================================================== - nightly-8-gpu-grok1-int4-rocm720: - if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-grok1-int4-rocm720,')) - runs-on: linux-mi300-8gpu-sglang - steps: - - name: Checkout code - uses: actions/checkout@v4 - with: - ref: ${{ inputs.ref || github.sha }} - - - name: Ensure VRAM is clear - run: bash scripts/ci/amd/ensure_vram_clear.sh rocm - - - name: Setup docker (ROCm 7.2) - run: | - touch github_summary.md - bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version rocm720 - env: - GITHUB_WORKSPACE: ${{ github.workspace }} - - - name: Install dependencies - run: bash scripts/ci/amd/amd_ci_install_dependency.sh --skip-test-time-deps - - - name: Accuracy Test ROCm 7.2 (8-GPU Grok1-INT4) - timeout-minutes: 60 - run: | - > github_summary.md # Clear summary file - bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ - -e RCCL_MSCCL_ENABLE=0 \ - -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ - python3 run_suite.py --hw amd --suite nightly-amd-accuracy-8-gpu-grok1-int4 --nightly --timeout-per-file 3600 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$? - echo "$(> $GITHUB_STEP_SUMMARY || true - exit ${TEST_EXIT_CODE:-0} - - - name: Performance Test ROCm 7.2 (8-GPU Grok1-INT4) - timeout-minutes: 60 - continue-on-error: true - run: | - > github_summary.md # Clear summary file - bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ - -e RCCL_MSCCL_ENABLE=0 \ - -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ - python3 run_suite.py --hw amd --suite nightly-perf-8-gpu-grok1-int4 --nightly --timeout-per-file 3600 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$? - echo "$(> $GITHUB_STEP_SUMMARY || true - exit ${TEST_EXIT_CODE:-0} - nightly-8-gpu-mi35x-grok1-int4-rocm720: if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-mi35x-grok1-int4-rocm720,')) runs-on: linux-mi35x-gpu-8 @@ -642,54 +585,9 @@ jobs: exit ${TEST_EXIT_CODE:-0} # ============================================================================== - # 8-GPU Grok2 (MI30x + MI35x) + # 8-GPU Grok2 (MI35x) # ============================================================================== - nightly-8-gpu-grok2-rocm720: - if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-grok2-rocm720,')) - runs-on: linux-mi300-8gpu-sglang - steps: - - name: Checkout code - uses: actions/checkout@v4 - with: - ref: ${{ inputs.ref || github.sha }} - - - name: Ensure VRAM is clear - run: bash scripts/ci/amd/ensure_vram_clear.sh rocm - - - name: Setup docker (ROCm 7.2) - run: | - touch github_summary.md - bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version rocm720 - env: - GITHUB_WORKSPACE: ${{ github.workspace }} - - - name: Install dependencies - run: bash scripts/ci/amd/amd_ci_install_dependency.sh --skip-test-time-deps - - - name: Accuracy Test ROCm 7.2 (8-GPU Grok2) - timeout-minutes: 60 - run: | - > github_summary.md # Clear summary file - bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ - -e RCCL_MSCCL_ENABLE=0 \ - -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ - python3 run_suite.py --hw amd --suite nightly-amd-accuracy-8-gpu-grok2 --nightly --timeout-per-file 3600 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$? - echo "$(> $GITHUB_STEP_SUMMARY || true - exit ${TEST_EXIT_CODE:-0} - - - name: Performance Test ROCm 7.2 (8-GPU Grok2) - timeout-minutes: 60 - continue-on-error: true - run: | - > github_summary.md # Clear summary file - bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ - -e RCCL_MSCCL_ENABLE=0 \ - -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ - python3 run_suite.py --hw amd --suite nightly-perf-8-gpu-grok2 --nightly --timeout-per-file 3600 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$? - echo "$(> $GITHUB_STEP_SUMMARY || true - exit ${TEST_EXIT_CODE:-0} - nightly-8-gpu-mi35x-grok2-rocm720: if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-mi35x-grok2-rocm720,')) runs-on: linux-mi35x-gpu-8 @@ -2087,14 +1985,12 @@ jobs: - nightly-4-gpu-rocm720 - nightly-2-gpu-mi35x-glm51-mxfp4-rocm720 - nightly-2-gpu-mi35x-deepseek-r1-mxfp4-tp2-rocm720 - # 8-GPU GPT-OSS (MI30x mixes Grok1-FP8; MI35x mixes Qwen3-Coder-Next) + # 8-GPU GPT-OSS (MI30x + MI35x) - nightly-accuracy-8-gpu-rocm720 - nightly-accuracy-8-gpu-mi35x-rocm720 - # 8-GPU Grok1-INT4 (MI30x + MI35x) - - nightly-8-gpu-grok1-int4-rocm720 + # 8-GPU Grok1-INT4 (MI35x) - nightly-8-gpu-mi35x-grok1-int4-rocm720 - # 8-GPU Grok2 (MI30x + MI35x) - - nightly-8-gpu-grok2-rocm720 + # 8-GPU Grok2 (MI35x) - nightly-8-gpu-mi35x-grok2-rocm720 # 8-GPU DeepSeek-V3.x (MI30x) - nightly-8-gpu-deepseek-v31-rocm720 diff --git a/.github/workflows/nightly-test-amd.yml b/.github/workflows/nightly-test-amd.yml index 783ff96b3..0b79259b8 100644 --- a/.github/workflows/nightly-test-amd.yml +++ b/.github/workflows/nightly-test-amd.yml @@ -41,14 +41,12 @@ on: - nightly-2-gpu-mi35x-glm51-mxfp4 # 2-GPU DeepSeek-R1-MXFP4 TP2 (MI35x only) - nightly-2-gpu-mi35x-deepseek-r1-mxfp4-tp2 - # 8-GPU GPT-OSS (MI30x mixes Grok1-FP8; MI35x mixes Qwen3-Coder-Next) + # 8-GPU GPT-OSS (MI30x + MI35x) - nightly-accuracy-8-gpu - nightly-accuracy-8-gpu-mi35x - # 8-GPU Grok1-INT4 (MI30x + MI35x) - - nightly-8-gpu-grok1-int4 + # 8-GPU Grok1-INT4 (MI35x) - nightly-8-gpu-mi35x-grok1-int4 - # 8-GPU Grok2 (MI30x + MI35x) - - nightly-8-gpu-grok2 + # 8-GPU Grok2 (MI35x) - nightly-8-gpu-mi35x-grok2 # 8-GPU DeepSeek-V3.x (MI30x) - nightly-8-gpu-deepseek-v31 @@ -469,7 +467,7 @@ jobs: exit ${TEST_EXIT_CODE:-0} # ============================================================================== - # 8-GPU GPT-OSS (MI30x mixes Grok1-FP8; MI35x mixes Qwen3-Coder-Next) + # 8-GPU GPT-OSS (MI30x + MI35x) # ============================================================================== nightly-accuracy-8-gpu: @@ -503,16 +501,6 @@ jobs: echo "$(> $GITHUB_STEP_SUMMARY || true exit ${TEST_EXIT_CODE:-0} - - name: Accuracy Test (8-GPU Grok1-FP8) - timeout-minutes: 60 - run: | - bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ - -e RCCL_MSCCL_ENABLE=0 \ - -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ - python3 run_suite.py --hw amd --suite nightly-amd-accuracy-8-gpu-grok1-fp8 --nightly --timeout-per-file 3600 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$? - echo "$(> $GITHUB_STEP_SUMMARY || true - exit ${TEST_EXIT_CODE:-0} - nightly-accuracy-8-gpu-mi35x: if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-accuracy-8-gpu-mi35x,')) runs-on: linux-mi35x-gpu-8 @@ -548,54 +536,9 @@ jobs: exit ${TEST_EXIT_CODE:-0} # ============================================================================== - # 8-GPU Grok1-INT4 (MI30x + MI35x) + # 8-GPU Grok1-INT4 (MI35x) # ============================================================================== - nightly-8-gpu-grok1-int4: - if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-grok1-int4,')) - runs-on: linux-mi300-8gpu-sglang - steps: - - name: Checkout code - uses: actions/checkout@v4 - with: - ref: ${{ inputs.ref || github.sha }} - - - name: Ensure VRAM is clear - run: bash scripts/ci/amd/ensure_vram_clear.sh rocm - - - name: Setup docker - run: | - touch github_summary.md - bash scripts/ci/amd/amd_ci_start_container.sh - env: - GITHUB_WORKSPACE: ${{ github.workspace }} - - - name: Install dependencies - run: bash scripts/ci/amd/amd_ci_install_dependency.sh - - - name: Accuracy Test (8-GPU Grok1-INT4) - timeout-minutes: 60 - run: | - > github_summary.md # Clear summary file - bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ - -e RCCL_MSCCL_ENABLE=0 \ - -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ - python3 run_suite.py --hw amd --suite nightly-amd-accuracy-8-gpu-grok1-int4 --nightly --timeout-per-file 3600 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$? - echo "$(> $GITHUB_STEP_SUMMARY || true - exit ${TEST_EXIT_CODE:-0} - - - name: Performance Test (8-GPU Grok1-INT4) - timeout-minutes: 60 - continue-on-error: true # Perf test failure doesn't fail the job if accuracy passed - run: | - > github_summary.md # Clear summary file - bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ - -e RCCL_MSCCL_ENABLE=0 \ - -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ - python3 run_suite.py --hw amd --suite nightly-perf-8-gpu-grok1-int4 --nightly --timeout-per-file 3600 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$? - echo "$(> $GITHUB_STEP_SUMMARY || true - exit ${TEST_EXIT_CODE:-0} - nightly-8-gpu-mi35x-grok1-int4: if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-mi35x-grok1-int4,')) runs-on: linux-mi35x-gpu-8 @@ -645,54 +588,9 @@ jobs: exit ${TEST_EXIT_CODE:-0} # ============================================================================== - # 8-GPU Grok2 (MI30x + MI35x) + # 8-GPU Grok2 (MI35x) # ============================================================================== - nightly-8-gpu-grok2: - if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-grok2,')) - runs-on: linux-mi300-8gpu-sglang - steps: - - name: Checkout code - uses: actions/checkout@v4 - with: - ref: ${{ inputs.ref || github.sha }} - - - name: Ensure VRAM is clear - run: bash scripts/ci/amd/ensure_vram_clear.sh rocm - - - name: Setup docker - run: | - touch github_summary.md - bash scripts/ci/amd/amd_ci_start_container.sh - env: - GITHUB_WORKSPACE: ${{ github.workspace }} - - - name: Install dependencies - run: bash scripts/ci/amd/amd_ci_install_dependency.sh - - - name: Accuracy Test (8-GPU Grok2) - timeout-minutes: 60 - run: | - > github_summary.md # Clear summary file - bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ - -e RCCL_MSCCL_ENABLE=0 \ - -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ - python3 run_suite.py --hw amd --suite nightly-amd-accuracy-8-gpu-grok2 --nightly --timeout-per-file 3600 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$? - echo "$(> $GITHUB_STEP_SUMMARY || true - exit ${TEST_EXIT_CODE:-0} - - - name: Performance Test (8-GPU Grok2) - timeout-minutes: 60 - continue-on-error: true # Perf test failure doesn't fail the job if accuracy passed - run: | - > github_summary.md # Clear summary file - bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ - -e RCCL_MSCCL_ENABLE=0 \ - -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ - python3 run_suite.py --hw amd --suite nightly-perf-8-gpu-grok2 --nightly --timeout-per-file 3600 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$? - echo "$(> $GITHUB_STEP_SUMMARY || true - exit ${TEST_EXIT_CODE:-0} - nightly-8-gpu-mi35x-grok2: if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-mi35x-grok2,')) runs-on: linux-mi35x-gpu-8 @@ -1894,14 +1792,12 @@ jobs: - nightly-4-gpu - nightly-2-gpu-mi35x-glm51-mxfp4 - nightly-2-gpu-mi35x-deepseek-r1-mxfp4-tp2 - # 8-GPU GPT-OSS (MI30x mixes Grok1-FP8; MI35x mixes Qwen3-Coder-Next) + # 8-GPU GPT-OSS (MI30x + MI35x) - nightly-accuracy-8-gpu - nightly-accuracy-8-gpu-mi35x - # 8-GPU Grok1-INT4 (MI30x + MI35x) - - nightly-8-gpu-grok1-int4 + # 8-GPU Grok1-INT4 (MI35x) - nightly-8-gpu-mi35x-grok1-int4 - # 8-GPU Grok2 (MI30x + MI35x) - - nightly-8-gpu-grok2 + # 8-GPU Grok2 (MI35x) - nightly-8-gpu-mi35x-grok2 # 8-GPU DeepSeek-V3.x (MI30x) - nightly-8-gpu-deepseek-v31