diff --git a/.github/workflows/nightly-test-amd-rocm720.yml b/.github/workflows/nightly-test-amd-rocm720.yml index ec46b74a6..2245c81ea 100644 --- a/.github/workflows/nightly-test-amd-rocm720.yml +++ b/.github/workflows/nightly-test-amd-rocm720.yml @@ -41,12 +41,14 @@ on: - nightly-2-gpu-mi35x-glm51-mxfp4-rocm720 # 2-GPU DeepSeek-R1-MXFP4 TP2 (MI35x only) - nightly-2-gpu-mi35x-deepseek-r1-mxfp4-tp2-rocm720 - # 8-GPU GPT-OSS (MI30x + MI35x) + # 8-GPU GPT-OSS (MI35x mixes Qwen3-Coder-Next) - nightly-accuracy-8-gpu-rocm720 - nightly-accuracy-8-gpu-mi35x-rocm720 - # 8-GPU Grok1-INT4 (MI35x) + # 8-GPU Grok1-INT4 (MI30x + MI35x) + - nightly-8-gpu-grok1-int4-rocm720 - nightly-8-gpu-mi35x-grok1-int4-rocm720 - # 8-GPU Grok2 (MI35x) + # 8-GPU Grok2 (MI30x + MI35x) + - nightly-8-gpu-grok2-rocm720 - nightly-8-gpu-mi35x-grok2-rocm720 # 8-GPU DeepSeek-V3.x (MI30x) - nightly-8-gpu-deepseek-v31-rocm720 @@ -463,7 +465,7 @@ jobs: exit ${TEST_EXIT_CODE:-0} # ============================================================================== - # 8-GPU GPT-OSS (MI30x + MI35x) + # 8-GPU GPT-OSS (MI35x mixes Qwen3-Coder-Next) # ============================================================================== nightly-accuracy-8-gpu-rocm720: @@ -533,9 +535,54 @@ jobs: exit ${TEST_EXIT_CODE:-0} # ============================================================================== - # 8-GPU Grok1-INT4 (MI35x) + # 8-GPU Grok1-INT4 (MI30x + MI35x) # ============================================================================== + nightly-8-gpu-grok1-int4-rocm720: + if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-grok1-int4-rocm720,')) + runs-on: linux-mi300-8gpu-sglang + steps: + - name: Checkout code + uses: actions/checkout@v4 + with: + ref: ${{ inputs.ref || github.sha }} + + - name: Ensure VRAM is clear + run: bash scripts/ci/amd/ensure_vram_clear.sh rocm + + - name: Setup docker (ROCm 7.2) + run: | + touch github_summary.md + bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version rocm720 + env: + GITHUB_WORKSPACE: ${{ github.workspace }} + + - name: Install dependencies + run: bash scripts/ci/amd/amd_ci_install_dependency.sh --skip-test-time-deps + + - name: Accuracy Test ROCm 7.2 (8-GPU Grok1-INT4) + timeout-minutes: 60 + run: | + > github_summary.md # Clear summary file + bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ + -e RCCL_MSCCL_ENABLE=0 \ + -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ + python3 run_suite.py --hw amd --suite nightly-amd-accuracy-8-gpu-grok1-int4 --nightly --timeout-per-file 3600 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$? + echo "$(> $GITHUB_STEP_SUMMARY || true + exit ${TEST_EXIT_CODE:-0} + + - name: Performance Test ROCm 7.2 (8-GPU Grok1-INT4) + timeout-minutes: 60 + continue-on-error: true + run: | + > github_summary.md # Clear summary file + bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ + -e RCCL_MSCCL_ENABLE=0 \ + -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ + python3 run_suite.py --hw amd --suite nightly-perf-8-gpu-grok1-int4 --nightly --timeout-per-file 3600 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$? + echo "$(> $GITHUB_STEP_SUMMARY || true + exit ${TEST_EXIT_CODE:-0} + nightly-8-gpu-mi35x-grok1-int4-rocm720: if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-mi35x-grok1-int4-rocm720,')) runs-on: linux-mi35x-gpu-8 @@ -585,9 +632,54 @@ jobs: exit ${TEST_EXIT_CODE:-0} # ============================================================================== - # 8-GPU Grok2 (MI35x) + # 8-GPU Grok2 (MI30x + MI35x) # ============================================================================== + nightly-8-gpu-grok2-rocm720: + if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-grok2-rocm720,')) + runs-on: linux-mi300-8gpu-sglang + steps: + - name: Checkout code + uses: actions/checkout@v4 + with: + ref: ${{ inputs.ref || github.sha }} + + - name: Ensure VRAM is clear + run: bash scripts/ci/amd/ensure_vram_clear.sh rocm + + - name: Setup docker (ROCm 7.2) + run: | + touch github_summary.md + bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version rocm720 + env: + GITHUB_WORKSPACE: ${{ github.workspace }} + + - name: Install dependencies + run: bash scripts/ci/amd/amd_ci_install_dependency.sh --skip-test-time-deps + + - name: Accuracy Test ROCm 7.2 (8-GPU Grok2) + timeout-minutes: 60 + run: | + > github_summary.md # Clear summary file + bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ + -e RCCL_MSCCL_ENABLE=0 \ + -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ + python3 run_suite.py --hw amd --suite nightly-amd-accuracy-8-gpu-grok2 --nightly --timeout-per-file 3600 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$? + echo "$(> $GITHUB_STEP_SUMMARY || true + exit ${TEST_EXIT_CODE:-0} + + - name: Performance Test ROCm 7.2 (8-GPU Grok2) + timeout-minutes: 60 + continue-on-error: true + run: | + > github_summary.md # Clear summary file + bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ + -e RCCL_MSCCL_ENABLE=0 \ + -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ + python3 run_suite.py --hw amd --suite nightly-perf-8-gpu-grok2 --nightly --timeout-per-file 3600 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$? + echo "$(> $GITHUB_STEP_SUMMARY || true + exit ${TEST_EXIT_CODE:-0} + nightly-8-gpu-mi35x-grok2-rocm720: if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-mi35x-grok2-rocm720,')) runs-on: linux-mi35x-gpu-8 @@ -1985,12 +2077,14 @@ jobs: - nightly-4-gpu-rocm720 - nightly-2-gpu-mi35x-glm51-mxfp4-rocm720 - nightly-2-gpu-mi35x-deepseek-r1-mxfp4-tp2-rocm720 - # 8-GPU GPT-OSS (MI30x + MI35x) + # 8-GPU GPT-OSS (MI35x mixes Qwen3-Coder-Next) - nightly-accuracy-8-gpu-rocm720 - nightly-accuracy-8-gpu-mi35x-rocm720 - # 8-GPU Grok1-INT4 (MI35x) + # 8-GPU Grok1-INT4 (MI30x + MI35x) + - nightly-8-gpu-grok1-int4-rocm720 - nightly-8-gpu-mi35x-grok1-int4-rocm720 - # 8-GPU Grok2 (MI35x) + # 8-GPU Grok2 (MI30x + MI35x) + - nightly-8-gpu-grok2-rocm720 - nightly-8-gpu-mi35x-grok2-rocm720 # 8-GPU DeepSeek-V3.x (MI30x) - nightly-8-gpu-deepseek-v31-rocm720 diff --git a/.github/workflows/nightly-test-amd.yml b/.github/workflows/nightly-test-amd.yml index 0b79259b8..7d04f0095 100644 --- a/.github/workflows/nightly-test-amd.yml +++ b/.github/workflows/nightly-test-amd.yml @@ -41,12 +41,14 @@ on: - nightly-2-gpu-mi35x-glm51-mxfp4 # 2-GPU DeepSeek-R1-MXFP4 TP2 (MI35x only) - nightly-2-gpu-mi35x-deepseek-r1-mxfp4-tp2 - # 8-GPU GPT-OSS (MI30x + MI35x) + # 8-GPU GPT-OSS (MI35x mixes Qwen3-Coder-Next) - nightly-accuracy-8-gpu - nightly-accuracy-8-gpu-mi35x - # 8-GPU Grok1-INT4 (MI35x) + # 8-GPU Grok1-INT4 (MI30x + MI35x) + - nightly-8-gpu-grok1-int4 - nightly-8-gpu-mi35x-grok1-int4 - # 8-GPU Grok2 (MI35x) + # 8-GPU Grok2 (MI30x + MI35x) + - nightly-8-gpu-grok2 - nightly-8-gpu-mi35x-grok2 # 8-GPU DeepSeek-V3.x (MI30x) - nightly-8-gpu-deepseek-v31 @@ -467,7 +469,7 @@ jobs: exit ${TEST_EXIT_CODE:-0} # ============================================================================== - # 8-GPU GPT-OSS (MI30x + MI35x) + # 8-GPU GPT-OSS (MI35x mixes Qwen3-Coder-Next) # ============================================================================== nightly-accuracy-8-gpu: @@ -536,9 +538,54 @@ jobs: exit ${TEST_EXIT_CODE:-0} # ============================================================================== - # 8-GPU Grok1-INT4 (MI35x) + # 8-GPU Grok1-INT4 (MI30x + MI35x) # ============================================================================== + nightly-8-gpu-grok1-int4: + if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-grok1-int4,')) + runs-on: linux-mi300-8gpu-sglang + steps: + - name: Checkout code + uses: actions/checkout@v4 + with: + ref: ${{ inputs.ref || github.sha }} + + - name: Ensure VRAM is clear + run: bash scripts/ci/amd/ensure_vram_clear.sh rocm + + - name: Setup docker + run: | + touch github_summary.md + bash scripts/ci/amd/amd_ci_start_container.sh + env: + GITHUB_WORKSPACE: ${{ github.workspace }} + + - name: Install dependencies + run: bash scripts/ci/amd/amd_ci_install_dependency.sh + + - name: Accuracy Test (8-GPU Grok1-INT4) + timeout-minutes: 60 + run: | + > github_summary.md # Clear summary file + bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ + -e RCCL_MSCCL_ENABLE=0 \ + -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ + python3 run_suite.py --hw amd --suite nightly-amd-accuracy-8-gpu-grok1-int4 --nightly --timeout-per-file 3600 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$? + echo "$(> $GITHUB_STEP_SUMMARY || true + exit ${TEST_EXIT_CODE:-0} + + - name: Performance Test (8-GPU Grok1-INT4) + timeout-minutes: 60 + continue-on-error: true # Perf test failure doesn't fail the job if accuracy passed + run: | + > github_summary.md # Clear summary file + bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ + -e RCCL_MSCCL_ENABLE=0 \ + -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ + python3 run_suite.py --hw amd --suite nightly-perf-8-gpu-grok1-int4 --nightly --timeout-per-file 3600 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$? + echo "$(> $GITHUB_STEP_SUMMARY || true + exit ${TEST_EXIT_CODE:-0} + nightly-8-gpu-mi35x-grok1-int4: if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-mi35x-grok1-int4,')) runs-on: linux-mi35x-gpu-8 @@ -588,9 +635,54 @@ jobs: exit ${TEST_EXIT_CODE:-0} # ============================================================================== - # 8-GPU Grok2 (MI35x) + # 8-GPU Grok2 (MI30x + MI35x) # ============================================================================== + nightly-8-gpu-grok2: + if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-grok2,')) + runs-on: linux-mi300-8gpu-sglang + steps: + - name: Checkout code + uses: actions/checkout@v4 + with: + ref: ${{ inputs.ref || github.sha }} + + - name: Ensure VRAM is clear + run: bash scripts/ci/amd/ensure_vram_clear.sh rocm + + - name: Setup docker + run: | + touch github_summary.md + bash scripts/ci/amd/amd_ci_start_container.sh + env: + GITHUB_WORKSPACE: ${{ github.workspace }} + + - name: Install dependencies + run: bash scripts/ci/amd/amd_ci_install_dependency.sh + + - name: Accuracy Test (8-GPU Grok2) + timeout-minutes: 60 + run: | + > github_summary.md # Clear summary file + bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ + -e RCCL_MSCCL_ENABLE=0 \ + -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ + python3 run_suite.py --hw amd --suite nightly-amd-accuracy-8-gpu-grok2 --nightly --timeout-per-file 3600 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$? + echo "$(> $GITHUB_STEP_SUMMARY || true + exit ${TEST_EXIT_CODE:-0} + + - name: Performance Test (8-GPU Grok2) + timeout-minutes: 60 + continue-on-error: true # Perf test failure doesn't fail the job if accuracy passed + run: | + > github_summary.md # Clear summary file + bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ + -e RCCL_MSCCL_ENABLE=0 \ + -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ + python3 run_suite.py --hw amd --suite nightly-perf-8-gpu-grok2 --nightly --timeout-per-file 3600 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$? + echo "$(> $GITHUB_STEP_SUMMARY || true + exit ${TEST_EXIT_CODE:-0} + nightly-8-gpu-mi35x-grok2: if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-mi35x-grok2,')) runs-on: linux-mi35x-gpu-8 @@ -1792,12 +1884,14 @@ jobs: - nightly-4-gpu - nightly-2-gpu-mi35x-glm51-mxfp4 - nightly-2-gpu-mi35x-deepseek-r1-mxfp4-tp2 - # 8-GPU GPT-OSS (MI30x + MI35x) + # 8-GPU GPT-OSS (MI35x mixes Qwen3-Coder-Next) - nightly-accuracy-8-gpu - nightly-accuracy-8-gpu-mi35x - # 8-GPU Grok1-INT4 (MI35x) + # 8-GPU Grok1-INT4 (MI30x + MI35x) + - nightly-8-gpu-grok1-int4 - nightly-8-gpu-mi35x-grok1-int4 - # 8-GPU Grok2 (MI35x) + # 8-GPU Grok2 (MI30x + MI35x) + - nightly-8-gpu-grok2 - nightly-8-gpu-mi35x-grok2 # 8-GPU DeepSeek-V3.x (MI30x) - nightly-8-gpu-deepseek-v31