diff --git a/.claude/skills/write-sglang-test/SKILL.md b/.claude/skills/write-sglang-test/SKILL.md index 3ed22137d..7f04c4bde 100644 --- a/.claude/skills/write-sglang-test/SKILL.md +++ b/.claude/skills/write-sglang-test/SKILL.md @@ -88,7 +88,7 @@ Defined in `python/sglang/test/test_utils.py`: | `stage-b-test-1-gpu-small-amd-mi35x` | `linux-mi35x-gpu-1` | 1-GPU tests on MI35x hardware | | `stage-b-test-1-gpu-large-amd` | `linux-mi325-1gpu-sglang` | Large 1-GPU AMD tests (2 partitions) | | `stage-b-test-2-gpu-large-amd` | `linux-mi325-2gpu-sglang` | 2-GPU ROCm correctness and parallel setups | -| `stage-b-test-large-8-gpu-35x-disaggregation-amd` | `linux-mi35x-gpu-8.fabric` | PD disaggregation and RDMA on 8×MI35x fabric | +| `stage-b-test-large-8-gpu-mi35x-disaggregation-amd` | `linux-mi35x-gpu-8.fabric` | PD disaggregation and RDMA on 8×MI35x fabric | | `stage-c-test-4-gpu-amd` | `linux-mi325-4gpu-sglang` | 4-GPU AMD integration (2 partitions) | | `stage-c-test-large-8-gpu-amd` | `linux-mi325-8gpu-sglang` | 8-GPU MI325 scaling and integration | | `stage-c-test-large-8-gpu-amd-mi35x` | `linux-mi35x-gpu-8` | 8-GPU MI35x scaling (2 partitions) | diff --git a/.github/workflows/nightly-test-amd-rocm720.yml b/.github/workflows/nightly-test-amd-rocm720.yml index cab07292b..696c16d72 100644 --- a/.github/workflows/nightly-test-amd-rocm720.yml +++ b/.github/workflows/nightly-test-amd-rocm720.yml @@ -27,43 +27,59 @@ on: default: 'all' options: - 'all' + # 1-GPU Unit Tests (MI30x + MI35x) - nightly-test-1-gpu-unit-rocm720 + - nightly-test-1-gpu-mi35x-rocm720 + # 2-GPU and 4-GPU Tests (MI30x) - nightly-accuracy-2-gpu-rocm720 - nightly-accuracy-2-gpu-vlm-rocm720 - nightly-perf-2-gpu-text-rocm720 - nightly-perf-2-gpu-vlm-rocm720 - nightly-4-gpu-rocm720 + # 8-GPU GPT-OSS (MI30x mixes Grok1-FP8; MI35x mixes Qwen3-Coder-Next) - nightly-accuracy-8-gpu-rocm720 + - nightly-accuracy-8-gpu-mi35x-rocm720 + # 8-GPU Grok1-INT4 (MI30x + MI35x) - nightly-8-gpu-grok1-int4-rocm720 + - nightly-8-gpu-mi35x-grok1-int4-rocm720 + # 8-GPU Grok2 (MI30x + MI35x) - nightly-8-gpu-grok2-rocm720 + - nightly-8-gpu-mi35x-grok2-rocm720 + # 8-GPU DeepSeek-V3.x (MI30x) - nightly-8-gpu-deepseek-v31-rocm720 - nightly-8-gpu-deepseek-v32-rocm720 - nightly-8-gpu-deepseek-v32-mtp-rocm720 - nightly-8-gpu-deepseek-v3-kv-fp8-rocm720 - - nightly-8-gpu-kimi-k26-rocm720 - - nightly-8-gpu-qwen3-235b-rocm720 - - nightly-8-gpu-qwen35-rocm720 - - nightly-8-gpu-glm51-rocm720 - - nightly-8-gpu-minimax-m27-rocm720 - - nightly-1-gpu-zimage-turbo-rocm720 - - nightly-test-1-gpu-mi35x-rocm720 - - nightly-accuracy-8-gpu-mi35x-rocm720 - - nightly-8-gpu-mi35x-grok1-int4-rocm720 - - nightly-8-gpu-mi35x-grok2-rocm720 - - nightly-8-gpu-mi35x-deepseek-r1-mxfp4-rocm720 - - nightly-8-gpu-mi35x-deepseek-r1-mxfp4-kv-fp8-rocm720 - - nightly-8-gpu-mi35x-deepseek-r1-mxfp4-ar-fusion-rocm720 + # 8-GPU DeepSeek-V3.2 (MI35x) - nightly-accuracy-8-gpu-mi35x-deepseek-v32-rocm720 - nightly-accuracy-8-gpu-mi35x-deepseek-v32-mtp-rocm720 - nightly-perf-8-gpu-mi35x-deepseek-v32-basic-rocm720 - nightly-perf-8-gpu-mi35x-deepseek-v32-mtp-rocm720 + # 8-GPU DeepSeek-R1-MXFP4 (MI35x only) + - nightly-8-gpu-mi35x-deepseek-r1-mxfp4-rocm720 + - nightly-8-gpu-mi35x-deepseek-r1-mxfp4-kv-fp8-rocm720 + - nightly-8-gpu-mi35x-deepseek-r1-mxfp4-ar-fusion-rocm720 + # 8-GPU DeepSeek-V4 (MI35x only) - nightly-8-gpu-mi35x-deepseek-v4-flash-rocm720 - nightly-8-gpu-mi35x-deepseek-v4-pro-rocm720 + # 8-GPU Kimi-K2.6 (MI30x + MI35x) + - nightly-8-gpu-kimi-k26-rocm720 - nightly-8-gpu-mi35x-kimi-k26-rocm720 + # 8-GPU Qwen3-235B (MI30x + MI35x MXFP4) + - nightly-8-gpu-qwen3-235b-rocm720 - nightly-8-gpu-mi35x-qwen3-235b-mxfp4-rocm720 + # 8-GPU Qwen 3.5 (MI30x + MI35x) + - nightly-8-gpu-qwen35-rocm720 - nightly-8-gpu-mi35x-qwen35-rocm720 + # 8-GPU GLM-5.1 (MI30x + MI35x) + - nightly-8-gpu-glm51-rocm720 - nightly-8-gpu-mi35x-glm51-rocm720 + # 8-GPU GLM-5-MXFP4 (MI35x only) - nightly-8-gpu-mi35x-glm5-mxfp4-rocm720 + # 8-GPU MiniMax-M2.7 (MI30x only) + - nightly-8-gpu-minimax-m27-rocm720 + # Diffusion (MI30x) + - nightly-1-gpu-zimage-turbo-rocm720 job_filter: description: 'Or type comma-separated job names (overrides dropdown if non-empty)' required: false @@ -108,6 +124,11 @@ concurrency: jobs: # ============================================== MI30x ROCm 7.2 Unit Tests ============================================== # 1-GPU Unit Tests - LoRA, debug utils, scheduler, etc. (MI30x ROCm 7.2) + + # ============================================================================== + # 1-GPU Unit Tests (MI30x + MI35x) + # ============================================================================== + nightly-test-1-gpu-unit-rocm720: if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-test-1-gpu-unit-rocm720,')) runs-on: linux-mi325-1gpu-sglang @@ -138,8 +159,41 @@ jobs: echo "$(> $GITHUB_STEP_SUMMARY || true exit ${TEST_EXIT_CODE:-0} - # ============================================== MI30x ROCm 7.2 Accuracy Tests ============================================== - # 2-GPU Accuracy Tests - GSM8K eval (MI30x ROCm 7.2) + nightly-test-1-gpu-mi35x-rocm720: + if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-test-1-gpu-mi35x-rocm720,')) + runs-on: linux-mi35x-gpu-1 + steps: + - name: Checkout code + uses: actions/checkout@v4 + with: + ref: ${{ inputs.ref || github.sha }} + + - name: Ensure VRAM is clear + run: bash scripts/ci/amd/ensure_vram_clear.sh rocm + + - name: Setup docker (ROCm 7.2) + run: | + touch github_summary.md + bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version rocm720 + env: + GITHUB_WORKSPACE: ${{ github.workspace }} + + - name: Install dependencies + run: | + bash scripts/ci/amd/amd_ci_install_dependency.sh + - name: Nightly Test MI35x ROCm 7.2 (1-GPU) + timeout-minutes: 90 + run: | + bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ + -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ + python3 run_suite.py --hw amd --suite nightly-amd-1-gpu-mi35x --nightly --timeout-per-file 900 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$? + echo "$(> $GITHUB_STEP_SUMMARY || true + exit ${TEST_EXIT_CODE:-0} + + # ============================================================================== + # 2-GPU and 4-GPU Tests (MI30x) + # ============================================================================== + nightly-accuracy-2-gpu-rocm720: if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-accuracy-2-gpu-rocm720,')) runs-on: linux-mi325-2gpu-sglang @@ -170,7 +224,6 @@ jobs: echo "$(> $GITHUB_STEP_SUMMARY || true exit ${TEST_EXIT_CODE:-0} - # 2-GPU VLM Accuracy Tests - Vision-Language Models MMMU evaluation (ROCm 7.2) nightly-accuracy-2-gpu-vlm-rocm720: if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-accuracy-2-gpu-vlm-rocm720,')) runs-on: linux-mi325-2gpu-sglang @@ -202,7 +255,6 @@ jobs: echo "$(> $GITHUB_STEP_SUMMARY || true exit ${TEST_EXIT_CODE:-0} - # 2-GPU Text Models Performance Tests (ROCm 7.2) nightly-perf-2-gpu-text-rocm720: if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-perf-2-gpu-text-rocm720,')) runs-on: linux-mi325-2gpu-sglang @@ -235,7 +287,6 @@ jobs: echo "$(> $GITHUB_STEP_SUMMARY || true exit ${TEST_EXIT_CODE:-0} - # 2-GPU VLM Performance Tests (ROCm 7.2) nightly-perf-2-gpu-vlm-rocm720: if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-perf-2-gpu-vlm-rocm720,')) runs-on: linux-mi325-2gpu-sglang @@ -268,8 +319,6 @@ jobs: echo "$(> $GITHUB_STEP_SUMMARY || true exit ${TEST_EXIT_CODE:-0} - # ============================================== MI30x ROCm 7.2 4-GPU Tests ============================================== - # 4-GPU Nightly Tests - Dumper/Comparator E2E, VLM Encoder DP (ROCm 7.2) nightly-4-gpu-rocm720: if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-4-gpu-rocm720,')) runs-on: linux-mi325-4gpu-sglang @@ -302,7 +351,10 @@ jobs: echo "$(> $GITHUB_STEP_SUMMARY || true exit ${TEST_EXIT_CODE:-0} - # 8-GPU Accuracy Tests - GPT-OSS, Grok1-FP8 (ROCm 7.2) + # ============================================================================== + # 8-GPU GPT-OSS (MI30x mixes Grok1-FP8; MI35x mixes Qwen3-Coder-Next) + # ============================================================================== + nightly-accuracy-8-gpu-rocm720: if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-accuracy-8-gpu-rocm720,')) runs-on: linux-mi325-8gpu-sglang @@ -344,8 +396,44 @@ jobs: echo "$(> $GITHUB_STEP_SUMMARY || true exit ${TEST_EXIT_CODE:-0} - # ============================================== MI30x ROCm 7.2 Combined Accuracy + Performance Tests ============================================== - # 8-GPU Grok1-INT4 (Accuracy + Performance) ROCm 7.2 + nightly-accuracy-8-gpu-mi35x-rocm720: + if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-accuracy-8-gpu-mi35x-rocm720,')) + runs-on: linux-mi35x-gpu-8 + steps: + - name: Checkout code + uses: actions/checkout@v4 + with: + ref: ${{ inputs.ref || github.sha }} + + - name: Ensure VRAM is clear + run: bash scripts/ci/amd/ensure_vram_clear.sh rocm + + - name: Setup docker (ROCm 7.2) + run: | + touch github_summary.md + bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version rocm720 + env: + GITHUB_WORKSPACE: ${{ github.workspace }} + + - name: Install dependencies + run: | + bash scripts/ci/amd/amd_ci_install_dependency.sh --skip-test-time-deps + # Install tabulate for run_suite.py (missing in MI35x container) + bash scripts/ci/amd/amd_ci_exec.sh pip install tabulate + + - name: Accuracy Test MI35x ROCm 7.2 (8-GPU GPT-OSS) + timeout-minutes: 180 + run: | + bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ + -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ + python3 run_suite.py --hw amd --suite nightly-amd-8-gpu-mi35x --nightly --timeout-per-file 7200 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$? + echo "$(> $GITHUB_STEP_SUMMARY || true + exit ${TEST_EXIT_CODE:-0} + + # ============================================================================== + # 8-GPU Grok1-INT4 (MI30x + MI35x) + # ============================================================================== + nightly-8-gpu-grok1-int4-rocm720: if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-grok1-int4-rocm720,')) runs-on: linux-mi325-8gpu-sglang @@ -391,537 +479,6 @@ jobs: echo "$(> $GITHUB_STEP_SUMMARY || true exit ${TEST_EXIT_CODE:-0} - # 8-GPU Grok2 (Accuracy + Performance) ROCm 7.2 - nightly-8-gpu-grok2-rocm720: - if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-grok2-rocm720,')) - runs-on: linux-mi325-8gpu-sglang - steps: - - name: Checkout code - uses: actions/checkout@v4 - with: - ref: ${{ inputs.ref || github.sha }} - - - name: Ensure VRAM is clear - run: bash scripts/ci/amd/ensure_vram_clear.sh rocm - - - name: Setup docker (ROCm 7.2) - run: | - touch github_summary.md - bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version rocm720 - env: - GITHUB_WORKSPACE: ${{ github.workspace }} - - - name: Install dependencies - run: bash scripts/ci/amd/amd_ci_install_dependency.sh --skip-test-time-deps - - - name: Accuracy Test ROCm 7.2 (8-GPU Grok2) - timeout-minutes: 60 - run: | - > github_summary.md # Clear summary file - bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ - -e RCCL_MSCCL_ENABLE=0 \ - -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ - python3 run_suite.py --hw amd --suite nightly-amd-accuracy-8-gpu-grok2 --nightly --timeout-per-file 3600 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$? - echo "$(> $GITHUB_STEP_SUMMARY || true - exit ${TEST_EXIT_CODE:-0} - - - name: Performance Test ROCm 7.2 (8-GPU Grok2) - timeout-minutes: 60 - continue-on-error: true - run: | - > github_summary.md # Clear summary file - bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ - -e RCCL_MSCCL_ENABLE=0 \ - -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ - python3 run_suite.py --hw amd --suite nightly-perf-8-gpu-grok2 --nightly --timeout-per-file 3600 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$? - echo "$(> $GITHUB_STEP_SUMMARY || true - exit ${TEST_EXIT_CODE:-0} - - # 8-GPU DeepSeek-V3.1 (Accuracy + Performance) ROCm 7.2 - nightly-8-gpu-deepseek-v31-rocm720: - if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-deepseek-v31-rocm720,')) - runs-on: linux-mi325-8gpu-sglang - steps: - - name: Checkout code - uses: actions/checkout@v4 - with: - ref: ${{ inputs.ref || github.sha }} - - - name: Ensure VRAM is clear - run: bash scripts/ci/amd/ensure_vram_clear.sh rocm - - - name: Setup docker (ROCm 7.2) - run: | - touch github_summary.md - bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version rocm720 - env: - GITHUB_WORKSPACE: ${{ github.workspace }} - - - name: Install dependencies - run: bash scripts/ci/amd/amd_ci_install_dependency.sh --skip-test-time-deps - - - name: Accuracy Test ROCm 7.2 (8-GPU DeepSeek-V3.1) - timeout-minutes: 120 - run: | - > github_summary.md # Clear summary file - bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ - -e SGLANG_USE_AITER=1 \ - -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ - python3 run_suite.py --hw amd --suite nightly-amd-accuracy-8-gpu-deepseek-v31 --nightly --timeout-per-file 3600 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$? - echo "$(> $GITHUB_STEP_SUMMARY || true - exit ${TEST_EXIT_CODE:-0} - - - name: Performance Test ROCm 7.2 (8-GPU DeepSeek-V3.1) - timeout-minutes: 300 - continue-on-error: true - run: | - > github_summary.md # Clear summary file - bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ - -e SGLANG_USE_ROCM700A=1 \ - -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ - python3 run_suite.py --hw amd --suite nightly-perf-8-gpu-deepseek-v31 --nightly --timeout-per-file 18000 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$? - echo "$(> $GITHUB_STEP_SUMMARY || true - exit ${TEST_EXIT_CODE:-0} - - # 8-GPU DeepSeek-V3.2 (Basic Accuracy + Perf) ROCm 7.2 - nightly-8-gpu-deepseek-v32-rocm720: - if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-deepseek-v32-rocm720,')) - runs-on: linux-mi325-8gpu-sglang - steps: - - name: Checkout code - uses: actions/checkout@v4 - with: - ref: ${{ inputs.ref || github.sha }} - - - name: Ensure VRAM is clear - run: bash scripts/ci/amd/ensure_vram_clear.sh rocm - - - name: Setup docker (ROCm 7.2) - run: | - touch github_summary.md - bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version rocm720 - env: - GITHUB_WORKSPACE: ${{ github.workspace }} - - - name: Install dependencies - run: bash scripts/ci/amd/amd_ci_install_dependency.sh --skip-test-time-deps - - - name: Accuracy Test ROCm 7.2 (8-GPU DeepSeek-V3.2 Basic) - timeout-minutes: 120 - run: | - > github_summary.md # Clear summary file - bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ - -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ - python3 run_suite.py --hw amd --suite nightly-amd-accuracy-8-gpu-deepseek-v32 --nightly --timeout-per-file 3600 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$? - echo "$(> $GITHUB_STEP_SUMMARY || true - exit ${TEST_EXIT_CODE:-0} - - - name: Performance Test ROCm 7.2 (8-GPU DeepSeek-V3.2 Basic) - timeout-minutes: 150 - continue-on-error: true - run: | - > github_summary.md # Clear summary file - bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ - -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ - python3 run_suite.py --hw amd --suite nightly-perf-8-gpu-deepseek-v32-basic --nightly --timeout-per-file 5400 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$? - echo "$(> $GITHUB_STEP_SUMMARY || true - exit ${TEST_EXIT_CODE:-0} - - # 8-GPU DeepSeek-V3.2 MTP (MTP Accuracy + Perf) ROCm 7.2 - nightly-8-gpu-deepseek-v32-mtp-rocm720: - if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-deepseek-v32-mtp-rocm720,')) - runs-on: linux-mi325-8gpu-sglang - steps: - - name: Checkout code - uses: actions/checkout@v4 - with: - ref: ${{ inputs.ref || github.sha }} - - - name: Ensure VRAM is clear - run: bash scripts/ci/amd/ensure_vram_clear.sh rocm - - - name: Setup docker (ROCm 7.2) - run: | - touch github_summary.md - bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version rocm720 - env: - GITHUB_WORKSPACE: ${{ github.workspace }} - - - name: Install dependencies - run: bash scripts/ci/amd/amd_ci_install_dependency.sh --skip-test-time-deps - - - name: Accuracy Test ROCm 7.2 (8-GPU DeepSeek-V3.2 MTP) - timeout-minutes: 120 - run: | - > github_summary.md # Clear summary file - bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ - -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ - python3 run_suite.py --hw amd --suite nightly-amd-accuracy-8-gpu-deepseek-v32-mtp --nightly --timeout-per-file 3600 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$? - echo "$(> $GITHUB_STEP_SUMMARY || true - exit ${TEST_EXIT_CODE:-0} - - - name: Performance Test ROCm 7.2 (8-GPU DeepSeek-V3.2 MTP) - timeout-minutes: 180 - continue-on-error: true - run: | - > github_summary.md # Clear summary file - bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ - -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ - python3 run_suite.py --hw amd --suite nightly-perf-8-gpu-deepseek-v32-mtp --nightly --timeout-per-file 7200 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$? - echo "$(> $GITHUB_STEP_SUMMARY || true - exit ${TEST_EXIT_CODE:-0} - - # 8-GPU DeepSeek-V3 KV FP8 (Basic + MTP with --kv-cache-dtype fp8_e4m3) ROCm 7.2 - nightly-8-gpu-deepseek-v3-kv-fp8-rocm720: - if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-deepseek-v3-kv-fp8-rocm720,')) - runs-on: linux-mi325-8gpu-sglang - steps: - - name: Checkout code - uses: actions/checkout@v4 - with: - ref: ${{ inputs.ref || github.sha }} - - - name: Ensure VRAM is clear - run: bash scripts/ci/amd/ensure_vram_clear.sh rocm - - - name: Setup docker (ROCm 7.2) - run: | - touch github_summary.md - bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version rocm720 - env: - GITHUB_WORKSPACE: ${{ github.workspace }} - - - name: Install dependencies - run: bash scripts/ci/amd/amd_ci_install_dependency.sh --skip-test-time-deps - - - name: DeepSeek-V3 KV FP8 Test ROCm 7.2 (8-GPU Basic + MTP) - timeout-minutes: 120 - run: | - > github_summary.md # Clear summary file - bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ - -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ - python3 run_suite.py --hw amd --suite nightly-amd-8-gpu-deepseek-v3-kv-fp8 --nightly --timeout-per-file 3600 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$? - echo "$(> $GITHUB_STEP_SUMMARY || true - exit ${TEST_EXIT_CODE:-0} - - # 8-GPU Kimi-K2.6 (Accuracy) ROCm 7.2 - nightly-8-gpu-kimi-k26-rocm720: - if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-kimi-k26-rocm720,')) - runs-on: linux-mi325-8gpu-sglang - steps: - - name: Checkout code - uses: actions/checkout@v4 - with: - ref: ${{ inputs.ref || github.sha }} - - - name: Ensure VRAM is clear - run: bash scripts/ci/amd/ensure_vram_clear.sh rocm - - - name: Setup docker (ROCm 7.2) - run: | - touch github_summary.md - bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version rocm720 - env: - GITHUB_WORKSPACE: ${{ github.workspace }} - - - name: Install dependencies - run: bash scripts/ci/amd/amd_ci_install_dependency.sh --skip-test-time-deps - - - name: Accuracy Test ROCm 7.2 (8-GPU Kimi-K2.6) - timeout-minutes: 120 - run: | - > github_summary.md # Clear summary file - bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ - -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ - python3 run_suite.py --hw amd --suite nightly-amd-accuracy-8-gpu-kimi-k26 --nightly --timeout-per-file 3600 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$? - echo "$(> $GITHUB_STEP_SUMMARY || true - exit ${TEST_EXIT_CODE:-0} - - # 8-GPU Qwen3-235B (Accuracy + Performance) ROCm 7.2 - nightly-8-gpu-qwen3-235b-rocm720: - if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-qwen3-235b-rocm720,')) - runs-on: linux-mi325-8gpu-sglang - steps: - - name: Checkout code - uses: actions/checkout@v4 - with: - ref: ${{ inputs.ref || github.sha }} - - - name: Ensure VRAM is clear - run: bash scripts/ci/amd/ensure_vram_clear.sh rocm - - - name: Setup docker (ROCm 7.2) - run: | - touch github_summary.md - bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version rocm720 - env: - GITHUB_WORKSPACE: ${{ github.workspace }} - - - name: Install dependencies - run: bash scripts/ci/amd/amd_ci_install_dependency.sh --skip-test-time-deps - - - name: Accuracy Test + Performance Test ROCm 7.2 (8-GPU Qwen3) - timeout-minutes: 120 - run: | - > github_summary.md # Clear summary file - bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ - -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ - python3 run_suite.py --hw amd --suite nightly-8-gpu-qwen3-235b --nightly --timeout-per-file 3600 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$? - echo "$(> $GITHUB_STEP_SUMMARY || true - exit ${TEST_EXIT_CODE:-0} - - # 8-GPU Qwen 3.5 (Accuracy + Performance combined) ROCm 7.2 - nightly-8-gpu-qwen35-rocm720: - if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-qwen35-rocm720,')) - runs-on: linux-mi325-8gpu-sglang - steps: - - name: Checkout code - uses: actions/checkout@v4 - with: - ref: ${{ inputs.ref || github.sha }} - - - name: Ensure VRAM is clear - run: bash scripts/ci/amd/ensure_vram_clear.sh rocm - - - name: Setup docker (ROCm 7.2) - run: | - touch github_summary.md - bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version rocm720 - env: - GITHUB_WORKSPACE: ${{ github.workspace }} - - - name: Install dependencies - run: | - bash scripts/ci/amd/amd_ci_install_dependency.sh --skip-aiter-build --skip-test-time-deps - bash scripts/ci/amd/amd_ci_exec.sh pip install mistral-common "lm-eval[api]" - - - name: Accuracy Test ROCm 7.2 (8-GPU Qwen 3.5) - timeout-minutes: 120 - run: | - > github_summary.md # Clear summary file - bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ - -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ - python3 run_suite.py --hw amd --suite nightly-amd-accuracy-8-gpu-qwen35 --nightly --timeout-per-file 3600 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$? - echo "$(> $GITHUB_STEP_SUMMARY || true - exit ${TEST_EXIT_CODE:-0} - - - name: Performance Test ROCm 7.2 (8-GPU Qwen 3.5 FP8) - timeout-minutes: 120 - continue-on-error: true - run: | - > github_summary.md # Clear summary file - bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ - -e SGLANG_USE_AITER=1 \ - -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ - python3 run_suite.py --hw amd --suite nightly-perf-8-gpu-qwen35-fp8 --nightly --timeout-per-file 5400 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$? - echo "$(> $GITHUB_STEP_SUMMARY || true - exit ${TEST_EXIT_CODE:-0} - - # 8-GPU GLM-5.1 (Accuracy + Performance combined) ROCm 7.2 - nightly-8-gpu-glm51-rocm720: - if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-glm51-rocm720,')) - runs-on: linux-mi325-8gpu-sglang - steps: - - name: Checkout code - uses: actions/checkout@v4 - with: - ref: ${{ inputs.ref || github.sha }} - - - name: Ensure VRAM is clear - run: bash scripts/ci/amd/ensure_vram_clear.sh rocm - - - name: Setup docker (ROCm 7.2) - run: | - touch github_summary.md - bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version rocm720 - env: - GITHUB_WORKSPACE: ${{ github.workspace }} - - - name: Install dependencies - run: | - bash scripts/ci/amd/amd_ci_install_dependency.sh --skip-test-time-deps - bash scripts/ci/amd/amd_ci_exec.sh pip install git+https://github.com/huggingface/transformers.git@96f807a33b75 - - - name: Accuracy Test ROCm 7.2 (8-GPU GLM-5.1 DSA) - timeout-minutes: 120 - run: | - > github_summary.md # Clear summary file - bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ - -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ - python3 run_suite.py --hw amd --suite nightly-amd-accuracy-8-gpu-glm51 --nightly --timeout-per-file 3600 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$? - echo "$(> $GITHUB_STEP_SUMMARY || true - exit ${TEST_EXIT_CODE:-0} - - - name: Performance Test ROCm 7.2 (8-GPU GLM-5.1) - timeout-minutes: 120 - continue-on-error: true - run: | - > github_summary.md # Clear summary file - bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ - -e SGLANG_USE_AITER=1 \ - -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ - python3 run_suite.py --hw amd --suite nightly-perf-8-gpu-glm51 --nightly --timeout-per-file 5400 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$? - echo "$(> $GITHUB_STEP_SUMMARY || true - exit ${TEST_EXIT_CODE:-0} - - # 8-GPU MiniMax-M2.7 (Accuracy + Performance combined, replaces M2.5) ROCm 7.2 - nightly-8-gpu-minimax-m27-rocm720: - if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-minimax-m27-rocm720,')) - runs-on: linux-mi325-8gpu-sglang - steps: - - name: Checkout code - uses: actions/checkout@v4 - with: - ref: ${{ inputs.ref || github.sha }} - - - name: Ensure VRAM is clear - run: bash scripts/ci/amd/ensure_vram_clear.sh rocm - - - name: Setup docker (ROCm 7.2) - run: | - touch github_summary.md - bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version rocm720 - env: - GITHUB_WORKSPACE: ${{ github.workspace }} - - - name: Install dependencies - run: bash scripts/ci/amd/amd_ci_install_dependency.sh --skip-test-time-deps - - - name: Accuracy Test ROCm 7.2 (8-GPU MiniMax-M2.7) - timeout-minutes: 120 - run: | - > github_summary.md # Clear summary file - bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ - -e SGLANG_USE_AITER=1 \ - -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ - python3 run_suite.py --hw amd --suite nightly-amd-accuracy-8-gpu-minimax-m27 --nightly --timeout-per-file 3600 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$? - echo "$(> $GITHUB_STEP_SUMMARY || true - exit ${TEST_EXIT_CODE:-0} - - - name: Performance Test ROCm 7.2 (8-GPU MiniMax-M2.7) - timeout-minutes: 120 - continue-on-error: true # Perf test failure doesn't fail the job if accuracy passed - run: | - > github_summary.md # Clear summary file - bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ - -e SGLANG_USE_AITER=1 \ - -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ - python3 run_suite.py --hw amd --suite nightly-perf-8-gpu-minimax-m27 --nightly --timeout-per-file 5400 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$? - echo "$(> $GITHUB_STEP_SUMMARY || true - exit ${TEST_EXIT_CODE:-0} - - # ============================================== MI30x ROCm 7.2 Diffusion Tests ============================================== - # 1-GPU Z-Image-Turbo (Diffusion T2I) ROCm 7.2 - nightly-1-gpu-zimage-turbo-rocm720: - if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-1-gpu-zimage-turbo-rocm720,')) - runs-on: linux-mi325-1gpu-sglang - steps: - - name: Checkout code - uses: actions/checkout@v4 - with: - ref: ${{ inputs.ref || github.sha }} - - - name: Ensure VRAM is clear - run: bash scripts/ci/amd/ensure_vram_clear.sh rocm - - - name: Setup docker (ROCm 7.2) - run: | - touch github_summary.md - bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version rocm720 - env: - GITHUB_WORKSPACE: ${{ github.workspace }} - - - name: Install dependencies - run: bash scripts/ci/amd/amd_ci_install_dependency.sh - - - name: Z-Image-Turbo Diffusion Test ROCm 7.2 (1-GPU) - timeout-minutes: 45 - run: | - bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout \ - -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ - -e SGLANG_DIFFUSION_ARTIFACT_DIR="/sglang-checkout/diffusion-artifacts" \ - pytest test/registered/amd/test_zimage_turbo.py -v -s ${{ inputs.continue_on_error && '|| true' || '' }} || TEST_EXIT_CODE=$? - echo "$(> $GITHUB_STEP_SUMMARY || true - exit ${TEST_EXIT_CODE:-0} - - - name: Upload generated images - if: always() - uses: actions/upload-artifact@v4 - with: - name: zimage-turbo-outputs-rocm720 - path: diffusion-artifacts/ - if-no-files-found: ignore - retention-days: 30 - - # ============================================== MI35x ROCm 7.2 Tests ============================================== - # MI35x 1-GPU ROCm 7.2 tests - nightly-test-1-gpu-mi35x-rocm720: - if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-test-1-gpu-mi35x-rocm720,')) - runs-on: linux-mi35x-gpu-1 - steps: - - name: Checkout code - uses: actions/checkout@v4 - with: - ref: ${{ inputs.ref || github.sha }} - - - name: Ensure VRAM is clear - run: bash scripts/ci/amd/ensure_vram_clear.sh rocm - - - name: Setup docker (ROCm 7.2) - run: | - touch github_summary.md - bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version rocm720 - env: - GITHUB_WORKSPACE: ${{ github.workspace }} - - - name: Install dependencies - run: | - bash scripts/ci/amd/amd_ci_install_dependency.sh - - name: Nightly Test MI35x ROCm 7.2 (1-GPU) - timeout-minutes: 90 - run: | - bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ - -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ - python3 run_suite.py --hw amd --suite nightly-amd-1-gpu-mi35x --nightly --timeout-per-file 900 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$? - echo "$(> $GITHUB_STEP_SUMMARY || true - exit ${TEST_EXIT_CODE:-0} - - # MI35x 8-GPU Accuracy Tests - GPT-OSS (ROCm 7.2) - nightly-accuracy-8-gpu-mi35x-rocm720: - if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-accuracy-8-gpu-mi35x-rocm720,')) - runs-on: linux-mi35x-gpu-8 - steps: - - name: Checkout code - uses: actions/checkout@v4 - with: - ref: ${{ inputs.ref || github.sha }} - - - name: Ensure VRAM is clear - run: bash scripts/ci/amd/ensure_vram_clear.sh rocm - - - name: Setup docker (ROCm 7.2) - run: | - touch github_summary.md - bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version rocm720 - env: - GITHUB_WORKSPACE: ${{ github.workspace }} - - - name: Install dependencies - run: | - bash scripts/ci/amd/amd_ci_install_dependency.sh --skip-test-time-deps - # Install tabulate for run_suite.py (missing in MI35x container) - bash scripts/ci/amd/amd_ci_exec.sh pip install tabulate - - - name: Accuracy Test MI35x ROCm 7.2 (8-GPU GPT-OSS) - timeout-minutes: 180 - run: | - bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ - -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ - python3 run_suite.py --hw amd --suite nightly-amd-8-gpu-mi35x --nightly --timeout-per-file 7200 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$? - echo "$(> $GITHUB_STEP_SUMMARY || true - exit ${TEST_EXIT_CODE:-0} - - # MI35x 8-GPU Grok1-INT4 (Accuracy + Performance) ROCm 7.2 nightly-8-gpu-mi35x-grok1-int4-rocm720: if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-mi35x-grok1-int4-rocm720,')) runs-on: linux-mi35x-gpu-8 @@ -970,7 +527,55 @@ jobs: echo "$(> $GITHUB_STEP_SUMMARY || true exit ${TEST_EXIT_CODE:-0} - # MI35x 8-GPU Grok2 (Accuracy + Performance) ROCm 7.2 + # ============================================================================== + # 8-GPU Grok2 (MI30x + MI35x) + # ============================================================================== + + nightly-8-gpu-grok2-rocm720: + if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-grok2-rocm720,')) + runs-on: linux-mi325-8gpu-sglang + steps: + - name: Checkout code + uses: actions/checkout@v4 + with: + ref: ${{ inputs.ref || github.sha }} + + - name: Ensure VRAM is clear + run: bash scripts/ci/amd/ensure_vram_clear.sh rocm + + - name: Setup docker (ROCm 7.2) + run: | + touch github_summary.md + bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version rocm720 + env: + GITHUB_WORKSPACE: ${{ github.workspace }} + + - name: Install dependencies + run: bash scripts/ci/amd/amd_ci_install_dependency.sh --skip-test-time-deps + + - name: Accuracy Test ROCm 7.2 (8-GPU Grok2) + timeout-minutes: 60 + run: | + > github_summary.md # Clear summary file + bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ + -e RCCL_MSCCL_ENABLE=0 \ + -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ + python3 run_suite.py --hw amd --suite nightly-amd-accuracy-8-gpu-grok2 --nightly --timeout-per-file 3600 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$? + echo "$(> $GITHUB_STEP_SUMMARY || true + exit ${TEST_EXIT_CODE:-0} + + - name: Performance Test ROCm 7.2 (8-GPU Grok2) + timeout-minutes: 60 + continue-on-error: true + run: | + > github_summary.md # Clear summary file + bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ + -e RCCL_MSCCL_ENABLE=0 \ + -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ + python3 run_suite.py --hw amd --suite nightly-perf-8-gpu-grok2 --nightly --timeout-per-file 3600 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$? + echo "$(> $GITHUB_STEP_SUMMARY || true + exit ${TEST_EXIT_CODE:-0} + nightly-8-gpu-mi35x-grok2-rocm720: if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-mi35x-grok2-rocm720,')) runs-on: linux-mi35x-gpu-8 @@ -1019,7 +624,321 @@ jobs: echo "$(> $GITHUB_STEP_SUMMARY || true exit ${TEST_EXIT_CODE:-0} - # MI35x 8-GPU DeepSeek-R1-MXFP4 (Accuracy + Performance) ROCm 7.2 + # ============================================================================== + # 8-GPU DeepSeek-V3.x (MI30x) + # ============================================================================== + + nightly-8-gpu-deepseek-v31-rocm720: + if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-deepseek-v31-rocm720,')) + runs-on: linux-mi325-8gpu-sglang + steps: + - name: Checkout code + uses: actions/checkout@v4 + with: + ref: ${{ inputs.ref || github.sha }} + + - name: Ensure VRAM is clear + run: bash scripts/ci/amd/ensure_vram_clear.sh rocm + + - name: Setup docker (ROCm 7.2) + run: | + touch github_summary.md + bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version rocm720 + env: + GITHUB_WORKSPACE: ${{ github.workspace }} + + - name: Install dependencies + run: bash scripts/ci/amd/amd_ci_install_dependency.sh --skip-test-time-deps + + - name: Accuracy Test ROCm 7.2 (8-GPU DeepSeek-V3.1) + timeout-minutes: 120 + run: | + > github_summary.md # Clear summary file + bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ + -e SGLANG_USE_AITER=1 \ + -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ + python3 run_suite.py --hw amd --suite nightly-amd-accuracy-8-gpu-deepseek-v31 --nightly --timeout-per-file 3600 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$? + echo "$(> $GITHUB_STEP_SUMMARY || true + exit ${TEST_EXIT_CODE:-0} + + - name: Performance Test ROCm 7.2 (8-GPU DeepSeek-V3.1) + timeout-minutes: 300 + continue-on-error: true + run: | + > github_summary.md # Clear summary file + bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ + -e SGLANG_USE_ROCM700A=1 \ + -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ + python3 run_suite.py --hw amd --suite nightly-perf-8-gpu-deepseek-v31 --nightly --timeout-per-file 18000 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$? + echo "$(> $GITHUB_STEP_SUMMARY || true + exit ${TEST_EXIT_CODE:-0} + + nightly-8-gpu-deepseek-v32-rocm720: + if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-deepseek-v32-rocm720,')) + runs-on: linux-mi325-8gpu-sglang + steps: + - name: Checkout code + uses: actions/checkout@v4 + with: + ref: ${{ inputs.ref || github.sha }} + + - name: Ensure VRAM is clear + run: bash scripts/ci/amd/ensure_vram_clear.sh rocm + + - name: Setup docker (ROCm 7.2) + run: | + touch github_summary.md + bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version rocm720 + env: + GITHUB_WORKSPACE: ${{ github.workspace }} + + - name: Install dependencies + run: bash scripts/ci/amd/amd_ci_install_dependency.sh --skip-test-time-deps + + - name: Accuracy Test ROCm 7.2 (8-GPU DeepSeek-V3.2 Basic) + timeout-minutes: 120 + run: | + > github_summary.md # Clear summary file + bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ + -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ + python3 run_suite.py --hw amd --suite nightly-amd-accuracy-8-gpu-deepseek-v32 --nightly --timeout-per-file 3600 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$? + echo "$(> $GITHUB_STEP_SUMMARY || true + exit ${TEST_EXIT_CODE:-0} + + - name: Performance Test ROCm 7.2 (8-GPU DeepSeek-V3.2 Basic) + timeout-minutes: 150 + continue-on-error: true + run: | + > github_summary.md # Clear summary file + bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ + -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ + python3 run_suite.py --hw amd --suite nightly-perf-8-gpu-deepseek-v32-basic --nightly --timeout-per-file 5400 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$? + echo "$(> $GITHUB_STEP_SUMMARY || true + exit ${TEST_EXIT_CODE:-0} + + nightly-8-gpu-deepseek-v32-mtp-rocm720: + if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-deepseek-v32-mtp-rocm720,')) + runs-on: linux-mi325-8gpu-sglang + steps: + - name: Checkout code + uses: actions/checkout@v4 + with: + ref: ${{ inputs.ref || github.sha }} + + - name: Ensure VRAM is clear + run: bash scripts/ci/amd/ensure_vram_clear.sh rocm + + - name: Setup docker (ROCm 7.2) + run: | + touch github_summary.md + bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version rocm720 + env: + GITHUB_WORKSPACE: ${{ github.workspace }} + + - name: Install dependencies + run: bash scripts/ci/amd/amd_ci_install_dependency.sh --skip-test-time-deps + + - name: Accuracy Test ROCm 7.2 (8-GPU DeepSeek-V3.2 MTP) + timeout-minutes: 120 + run: | + > github_summary.md # Clear summary file + bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ + -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ + python3 run_suite.py --hw amd --suite nightly-amd-accuracy-8-gpu-deepseek-v32-mtp --nightly --timeout-per-file 3600 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$? + echo "$(> $GITHUB_STEP_SUMMARY || true + exit ${TEST_EXIT_CODE:-0} + + - name: Performance Test ROCm 7.2 (8-GPU DeepSeek-V3.2 MTP) + timeout-minutes: 180 + continue-on-error: true + run: | + > github_summary.md # Clear summary file + bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ + -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ + python3 run_suite.py --hw amd --suite nightly-perf-8-gpu-deepseek-v32-mtp --nightly --timeout-per-file 7200 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$? + echo "$(> $GITHUB_STEP_SUMMARY || true + exit ${TEST_EXIT_CODE:-0} + + nightly-8-gpu-deepseek-v3-kv-fp8-rocm720: + if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-deepseek-v3-kv-fp8-rocm720,')) + runs-on: linux-mi325-8gpu-sglang + steps: + - name: Checkout code + uses: actions/checkout@v4 + with: + ref: ${{ inputs.ref || github.sha }} + + - name: Ensure VRAM is clear + run: bash scripts/ci/amd/ensure_vram_clear.sh rocm + + - name: Setup docker (ROCm 7.2) + run: | + touch github_summary.md + bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version rocm720 + env: + GITHUB_WORKSPACE: ${{ github.workspace }} + + - name: Install dependencies + run: bash scripts/ci/amd/amd_ci_install_dependency.sh --skip-test-time-deps + + - name: DeepSeek-V3 KV FP8 Test ROCm 7.2 (8-GPU Basic + MTP) + timeout-minutes: 120 + run: | + > github_summary.md # Clear summary file + bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ + -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ + python3 run_suite.py --hw amd --suite nightly-amd-8-gpu-deepseek-v3-kv-fp8 --nightly --timeout-per-file 3600 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$? + echo "$(> $GITHUB_STEP_SUMMARY || true + exit ${TEST_EXIT_CODE:-0} + + # ============================================================================== + # 8-GPU DeepSeek-V3.2 (MI35x) + # ============================================================================== + + nightly-accuracy-8-gpu-mi35x-deepseek-v32-rocm720: + if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-accuracy-8-gpu-mi35x-deepseek-v32-rocm720,')) + runs-on: linux-mi35x-gpu-8 + steps: + - name: Checkout code + uses: actions/checkout@v4 + with: + ref: ${{ inputs.ref || github.sha }} + + - name: Ensure VRAM is clear + run: bash scripts/ci/amd/ensure_vram_clear.sh rocm + + - name: Setup docker (ROCm 7.2) + run: | + touch github_summary.md + bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version rocm720 + env: + GITHUB_WORKSPACE: ${{ github.workspace }} + + - name: Install dependencies + run: | + bash scripts/ci/amd/amd_ci_install_dependency.sh --skip-test-time-deps + # Install tabulate for run_suite.py (missing in MI35x container) + bash scripts/ci/amd/amd_ci_exec.sh pip install tabulate + + - name: Accuracy Test MI35x ROCm 7.2 (8-GPU DeepSeek-V3.2) + timeout-minutes: 120 + run: | + > github_summary.md # Clear summary file + bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ + -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ + python3 run_suite.py --hw amd --suite nightly-amd-8-gpu-mi35x-deepseek-v32 --nightly --timeout-per-file 3600 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$? + echo "$(> $GITHUB_STEP_SUMMARY || true + exit ${TEST_EXIT_CODE:-0} + + nightly-accuracy-8-gpu-mi35x-deepseek-v32-mtp-rocm720: + if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-accuracy-8-gpu-mi35x-deepseek-v32-mtp-rocm720,')) + runs-on: linux-mi35x-gpu-8 + steps: + - name: Checkout code + uses: actions/checkout@v4 + with: + ref: ${{ inputs.ref || github.sha }} + + - name: Ensure VRAM is clear + run: bash scripts/ci/amd/ensure_vram_clear.sh rocm + + - name: Setup docker (ROCm 7.2) + run: | + touch github_summary.md + bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version rocm720 + env: + GITHUB_WORKSPACE: ${{ github.workspace }} + + - name: Install dependencies + run: | + bash scripts/ci/amd/amd_ci_install_dependency.sh --skip-test-time-deps + # Install tabulate for run_suite.py (missing in MI35x container) + bash scripts/ci/amd/amd_ci_exec.sh pip install tabulate + + - name: Accuracy Test MI35x ROCm 7.2 (8-GPU DeepSeek-V3.2 TP+MTP) + timeout-minutes: 120 + run: | + > github_summary.md # Clear summary file + bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ + -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ + python3 run_suite.py --hw amd --suite nightly-amd-accuracy-8-gpu-mi35x-deepseek-v32-mtp --nightly --timeout-per-file 3600 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$? + echo "$(> $GITHUB_STEP_SUMMARY || true + exit ${TEST_EXIT_CODE:-0} + + nightly-perf-8-gpu-mi35x-deepseek-v32-basic-rocm720: + if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-perf-8-gpu-mi35x-deepseek-v32-basic-rocm720,')) + runs-on: linux-mi35x-gpu-8 + steps: + - name: Checkout code + uses: actions/checkout@v4 + with: + ref: ${{ inputs.ref || github.sha }} + + - name: Ensure VRAM is clear + run: bash scripts/ci/amd/ensure_vram_clear.sh rocm + + - name: Setup docker (ROCm 7.2) + run: | + touch github_summary.md + bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version rocm720 + env: + GITHUB_WORKSPACE: ${{ github.workspace }} + + - name: Install dependencies + run: | + bash scripts/ci/amd/amd_ci_install_dependency.sh --skip-test-time-deps + # Install tabulate for run_suite.py (missing in MI35x container) + bash scripts/ci/amd/amd_ci_exec.sh pip install tabulate + + - name: Performance Test MI35x ROCm 7.2 (8-GPU DeepSeek-V3.2 Basic) + timeout-minutes: 150 + run: | + > github_summary.md # Clear summary file + bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ + -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ + python3 run_suite.py --hw amd --suite nightly-perf-8-gpu-mi35x-deepseek-v32-basic --nightly --timeout-per-file 5400 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$? + echo "$(> $GITHUB_STEP_SUMMARY || true + exit ${TEST_EXIT_CODE:-0} + + nightly-perf-8-gpu-mi35x-deepseek-v32-mtp-rocm720: + if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-perf-8-gpu-mi35x-deepseek-v32-mtp-rocm720,')) + runs-on: linux-mi35x-gpu-8 + steps: + - name: Checkout code + uses: actions/checkout@v4 + with: + ref: ${{ inputs.ref || github.sha }} + + - name: Ensure VRAM is clear + run: bash scripts/ci/amd/ensure_vram_clear.sh rocm + + - name: Setup docker (ROCm 7.2) + run: | + touch github_summary.md + bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version rocm720 + env: + GITHUB_WORKSPACE: ${{ github.workspace }} + + - name: Install dependencies + run: | + bash scripts/ci/amd/amd_ci_install_dependency.sh --skip-test-time-deps + # Install tabulate for run_suite.py (missing in MI35x container) + bash scripts/ci/amd/amd_ci_exec.sh pip install tabulate + + - name: Performance Test MI35x ROCm 7.2 (8-GPU DeepSeek-V3.2 MTP) + timeout-minutes: 180 + run: | + > github_summary.md # Clear summary file + bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ + -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ + python3 run_suite.py --hw amd --suite nightly-perf-8-gpu-mi35x-deepseek-v32-mtp --nightly --timeout-per-file 7200 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$? + echo "$(> $GITHUB_STEP_SUMMARY || true + exit ${TEST_EXIT_CODE:-0} + + # ============================================================================== + # 8-GPU DeepSeek-R1-MXFP4 (MI35x only) + # ============================================================================== + nightly-8-gpu-mi35x-deepseek-r1-mxfp4-rocm720: if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-mi35x-deepseek-r1-mxfp4-rocm720,')) runs-on: linux-mi35x-gpu-8 @@ -1066,7 +985,6 @@ jobs: echo "$(> $GITHUB_STEP_SUMMARY || true exit ${TEST_EXIT_CODE:-0} - # MI35x 8-GPU DeepSeek-R1-MXFP4 KV FP8 (Accuracy + Performance) ROCm 7.2 nightly-8-gpu-mi35x-deepseek-r1-mxfp4-kv-fp8-rocm720: if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-mi35x-deepseek-r1-mxfp4-kv-fp8-rocm720,')) runs-on: linux-mi35x-gpu-8 @@ -1113,7 +1031,6 @@ jobs: echo "$(> $GITHUB_STEP_SUMMARY || true exit ${TEST_EXIT_CODE:-0} - # MI35x 8-GPU DeepSeek-R1-MXFP4 AllReduce Fusion (Accuracy + Performance) ROCm 7.2 nightly-8-gpu-mi35x-deepseek-r1-mxfp4-ar-fusion-rocm720: if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-mi35x-deepseek-r1-mxfp4-ar-fusion-rocm720,')) runs-on: linux-mi35x-gpu-8 @@ -1160,378 +1077,10 @@ jobs: echo "$(> $GITHUB_STEP_SUMMARY || true exit ${TEST_EXIT_CODE:-0} - # MI35x 8-GPU DeepSeek-V3.2 Accuracy Test (ROCm 7.2) - nightly-accuracy-8-gpu-mi35x-deepseek-v32-rocm720: - if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-accuracy-8-gpu-mi35x-deepseek-v32-rocm720,')) - runs-on: linux-mi35x-gpu-8 - steps: - - name: Checkout code - uses: actions/checkout@v4 - with: - ref: ${{ inputs.ref || github.sha }} + # ============================================================================== + # 8-GPU DeepSeek-V4 (MI35x only) + # ============================================================================== - - name: Ensure VRAM is clear - run: bash scripts/ci/amd/ensure_vram_clear.sh rocm - - - name: Setup docker (ROCm 7.2) - run: | - touch github_summary.md - bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version rocm720 - env: - GITHUB_WORKSPACE: ${{ github.workspace }} - - - name: Install dependencies - run: | - bash scripts/ci/amd/amd_ci_install_dependency.sh --skip-test-time-deps - # Install tabulate for run_suite.py (missing in MI35x container) - bash scripts/ci/amd/amd_ci_exec.sh pip install tabulate - - - name: Accuracy Test MI35x ROCm 7.2 (8-GPU DeepSeek-V3.2) - timeout-minutes: 120 - run: | - > github_summary.md # Clear summary file - bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ - -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ - python3 run_suite.py --hw amd --suite nightly-amd-8-gpu-mi35x-deepseek-v32 --nightly --timeout-per-file 3600 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$? - echo "$(> $GITHUB_STEP_SUMMARY || true - exit ${TEST_EXIT_CODE:-0} - - # MI35x 8-GPU DeepSeek-V3.2 TP+MTP Accuracy Test (ROCm 7.2) - nightly-accuracy-8-gpu-mi35x-deepseek-v32-mtp-rocm720: - if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-accuracy-8-gpu-mi35x-deepseek-v32-mtp-rocm720,')) - runs-on: linux-mi35x-gpu-8 - steps: - - name: Checkout code - uses: actions/checkout@v4 - with: - ref: ${{ inputs.ref || github.sha }} - - - name: Ensure VRAM is clear - run: bash scripts/ci/amd/ensure_vram_clear.sh rocm - - - name: Setup docker (ROCm 7.2) - run: | - touch github_summary.md - bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version rocm720 - env: - GITHUB_WORKSPACE: ${{ github.workspace }} - - - name: Install dependencies - run: | - bash scripts/ci/amd/amd_ci_install_dependency.sh --skip-test-time-deps - # Install tabulate for run_suite.py (missing in MI35x container) - bash scripts/ci/amd/amd_ci_exec.sh pip install tabulate - - - name: Accuracy Test MI35x ROCm 7.2 (8-GPU DeepSeek-V3.2 TP+MTP) - timeout-minutes: 120 - run: | - > github_summary.md # Clear summary file - bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ - -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ - python3 run_suite.py --hw amd --suite nightly-amd-accuracy-8-gpu-mi35x-deepseek-v32-mtp --nightly --timeout-per-file 3600 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$? - echo "$(> $GITHUB_STEP_SUMMARY || true - exit ${TEST_EXIT_CODE:-0} - - # MI35x 8-GPU DeepSeek-V3.2 Performance Test (Basic) ROCm 7.2 - nightly-perf-8-gpu-mi35x-deepseek-v32-basic-rocm720: - if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-perf-8-gpu-mi35x-deepseek-v32-basic-rocm720,')) - runs-on: linux-mi35x-gpu-8 - steps: - - name: Checkout code - uses: actions/checkout@v4 - with: - ref: ${{ inputs.ref || github.sha }} - - - name: Ensure VRAM is clear - run: bash scripts/ci/amd/ensure_vram_clear.sh rocm - - - name: Setup docker (ROCm 7.2) - run: | - touch github_summary.md - bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version rocm720 - env: - GITHUB_WORKSPACE: ${{ github.workspace }} - - - name: Install dependencies - run: | - bash scripts/ci/amd/amd_ci_install_dependency.sh --skip-test-time-deps - # Install tabulate for run_suite.py (missing in MI35x container) - bash scripts/ci/amd/amd_ci_exec.sh pip install tabulate - - - name: Performance Test MI35x ROCm 7.2 (8-GPU DeepSeek-V3.2 Basic) - timeout-minutes: 150 - run: | - > github_summary.md # Clear summary file - bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ - -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ - python3 run_suite.py --hw amd --suite nightly-perf-8-gpu-mi35x-deepseek-v32-basic --nightly --timeout-per-file 5400 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$? - echo "$(> $GITHUB_STEP_SUMMARY || true - exit ${TEST_EXIT_CODE:-0} - - # MI35x 8-GPU Kimi-K2.6 (Accuracy) ROCm 7.2 - nightly-8-gpu-mi35x-kimi-k26-rocm720: - if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-mi35x-kimi-k26-rocm720,')) - runs-on: linux-mi35x-gpu-8 - steps: - - name: Checkout code - uses: actions/checkout@v4 - with: - ref: ${{ inputs.ref || github.sha }} - - - name: Ensure VRAM is clear - run: bash scripts/ci/amd/ensure_vram_clear.sh rocm - - - name: Setup docker (ROCm 7.2) - run: | - touch github_summary.md - bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version rocm720 - env: - GITHUB_WORKSPACE: ${{ github.workspace }} - - - name: Install dependencies - run: | - bash scripts/ci/amd/amd_ci_install_dependency.sh --skip-test-time-deps - # Install tabulate for run_suite.py (missing in MI35x container) - bash scripts/ci/amd/amd_ci_exec.sh pip install tabulate - - - name: Accuracy Test MI35x ROCm 7.2 (8-GPU Kimi-K2.6) - timeout-minutes: 180 - run: | - > github_summary.md # Clear summary file - bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ - -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ - python3 run_suite.py --hw amd --suite nightly-amd-accuracy-8-gpu-mi35x-kimi-k26 --nightly --timeout-per-file 7200 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$? - echo "$(> $GITHUB_STEP_SUMMARY || true - exit ${TEST_EXIT_CODE:-0} - - # MI35x 8-GPU Qwen3-235B-MXFP4 (Accuracy + Performance) ROCm 7.2 - nightly-8-gpu-mi35x-qwen3-235b-mxfp4-rocm720: - if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-mi35x-qwen3-235b-mxfp4-rocm720,')) - runs-on: linux-mi35x-gpu-8 - steps: - - name: Checkout code - uses: actions/checkout@v4 - with: - ref: ${{ inputs.ref || github.sha }} - - - name: Ensure VRAM is clear - run: bash scripts/ci/amd/ensure_vram_clear.sh rocm - - - name: Setup docker (ROCm 7.2) - run: | - touch github_summary.md - bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version rocm720 - env: - GITHUB_WORKSPACE: ${{ github.workspace }} - - - name: Install dependencies - run: | - bash scripts/ci/amd/amd_ci_install_dependency.sh --skip-test-time-deps - # Install tabulate for run_suite.py (missing in MI35x container) - bash scripts/ci/amd/amd_ci_exec.sh pip install tabulate - - - name: Accuracy Test + Performance Test MI35x ROCm 7.2 (8-GPU Qwen3-235B-MXFP4) - timeout-minutes: 120 - run: | - > github_summary.md # Clear summary file - bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ - -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ - python3 run_suite.py --hw amd --suite nightly-8-gpu-mi35x-qwen3-235b-mxfp4 --nightly --timeout-per-file 3600 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$? - echo "$(> $GITHUB_STEP_SUMMARY || true - exit ${TEST_EXIT_CODE:-0} - - # MI35x 8-GPU Qwen 3.5 (Accuracy + Performance combined) ROCm 7.2 - nightly-8-gpu-mi35x-qwen35-rocm720: - if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-mi35x-qwen35-rocm720,')) - runs-on: linux-mi35x-gpu-8 - steps: - - name: Checkout code - uses: actions/checkout@v4 - with: - ref: ${{ inputs.ref || github.sha }} - - - name: Ensure VRAM is clear - run: bash scripts/ci/amd/ensure_vram_clear.sh rocm - - - name: Setup docker (ROCm 7.2) - run: | - touch github_summary.md - bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version rocm720 - env: - GITHUB_WORKSPACE: ${{ github.workspace }} - - - name: Install dependencies - run: | - bash scripts/ci/amd/amd_ci_install_dependency.sh --skip-aiter-build --skip-test-time-deps - bash scripts/ci/amd/amd_ci_exec.sh pip install tabulate - bash scripts/ci/amd/amd_ci_exec.sh pip install mistral-common "lm-eval[api]" - - - name: Accuracy Test MI35x ROCm 7.2 (8-GPU Qwen 3.5) - timeout-minutes: 120 - run: | - > github_summary.md # Clear summary file - bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ - -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ - python3 run_suite.py --hw amd --suite nightly-amd-accuracy-8-gpu-mi35x-qwen35 --nightly --timeout-per-file 3600 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$? - echo "$(> $GITHUB_STEP_SUMMARY || true - exit ${TEST_EXIT_CODE:-0} - - - name: Performance Test MI35x ROCm 7.2 (8-GPU Qwen 3.5 FP8) - timeout-minutes: 120 - continue-on-error: true - run: | - > github_summary.md # Clear summary file - bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ - -e SGLANG_USE_AITER=1 \ - -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ - python3 run_suite.py --hw amd --suite nightly-perf-8-gpu-mi35x-qwen35-fp8 --nightly --timeout-per-file 5400 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$? - echo "$(> $GITHUB_STEP_SUMMARY || true - exit ${TEST_EXIT_CODE:-0} - - # MI35x 8-GPU GLM-5.1 (Accuracy + Performance combined) ROCm 7.2 - nightly-8-gpu-mi35x-glm51-rocm720: - if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-mi35x-glm51-rocm720,')) - runs-on: linux-mi35x-gpu-8 - steps: - - name: Checkout code - uses: actions/checkout@v4 - with: - ref: ${{ inputs.ref || github.sha }} - - - name: Ensure VRAM is clear - run: bash scripts/ci/amd/ensure_vram_clear.sh rocm - - - name: Setup docker (ROCm 7.2) - run: | - touch github_summary.md - bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version rocm720 - env: - GITHUB_WORKSPACE: ${{ github.workspace }} - - - name: Install dependencies - run: | - bash scripts/ci/amd/amd_ci_install_dependency.sh --skip-test-time-deps - bash scripts/ci/amd/amd_ci_exec.sh pip install tabulate - bash scripts/ci/amd/amd_ci_exec.sh pip install git+https://github.com/huggingface/transformers.git@96f807a33b75 - - - name: Accuracy Test MI35x ROCm 7.2 (8-GPU GLM-5.1 DSA) - timeout-minutes: 180 - run: | - > github_summary.md # Clear summary file - bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ - -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ - python3 run_suite.py --hw amd --suite nightly-amd-8-gpu-mi35x-glm51 --nightly --timeout-per-file 7200 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$? - echo "$(> $GITHUB_STEP_SUMMARY || true - exit ${TEST_EXIT_CODE:-0} - - - name: Performance Test MI35x ROCm 7.2 (8-GPU GLM-5.1) - timeout-minutes: 120 - continue-on-error: true - run: | - > github_summary.md # Clear summary file - bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ - -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ - python3 run_suite.py --hw amd --suite nightly-perf-8-gpu-mi35x-glm51 --nightly --timeout-per-file 5400 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$? - echo "$(> $GITHUB_STEP_SUMMARY || true - exit ${TEST_EXIT_CODE:-0} - - # MI35x 8-GPU GLM-5-MXFP4 (Accuracy + Performance combined) ROCm 7.2 - nightly-8-gpu-mi35x-glm5-mxfp4-rocm720: - if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-mi35x-glm5-mxfp4-rocm720,')) - runs-on: linux-mi35x-gpu-8 - steps: - - name: Checkout code - uses: actions/checkout@v4 - with: - ref: ${{ inputs.ref || github.sha }} - - - name: Ensure VRAM is clear - run: bash scripts/ci/amd/ensure_vram_clear.sh rocm - - - name: Setup docker (ROCm 7.2) - run: | - touch github_summary.md - bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version rocm720 - env: - GITHUB_WORKSPACE: ${{ github.workspace }} - - - name: Install dependencies - run: | - bash scripts/ci/amd/amd_ci_install_dependency.sh --skip-test-time-deps - bash scripts/ci/amd/amd_ci_exec.sh pip install tabulate - bash scripts/ci/amd/amd_ci_exec.sh pip install git+https://github.com/huggingface/transformers.git@96f807a33b75 - - - name: Accuracy Test MI35x ROCm 7.2 (8-GPU GLM-5-MXFP4) - timeout-minutes: 180 - run: | - > github_summary.md - bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ - -e SGLANG_USE_AITER=1 \ - -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ - python3 run_suite.py --hw amd --suite nightly-amd-8-gpu-mi35x-glm5-mxfp4 --nightly --timeout-per-file 7200 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$? - echo "$(> $GITHUB_STEP_SUMMARY || true - exit ${TEST_EXIT_CODE:-0} - - - name: Performance Test MI35x ROCm 7.2 (8-GPU GLM-5-MXFP4) - timeout-minutes: 300 - continue-on-error: true - run: | - > github_summary.md - bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ - -e SGLANG_USE_AITER=1 \ - -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ - python3 registered/amd/perf/mi35x/test_glm5_mxfp4_perf_mi35x.py || TEST_EXIT_CODE=$? - echo "$(> $GITHUB_STEP_SUMMARY || true - exit ${TEST_EXIT_CODE:-0} - - # MI35x 8-GPU DeepSeek-V3.2 Performance Test (MTP) ROCm 7.2 - nightly-perf-8-gpu-mi35x-deepseek-v32-mtp-rocm720: - if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-perf-8-gpu-mi35x-deepseek-v32-mtp-rocm720,')) - runs-on: linux-mi35x-gpu-8 - steps: - - name: Checkout code - uses: actions/checkout@v4 - with: - ref: ${{ inputs.ref || github.sha }} - - - name: Ensure VRAM is clear - run: bash scripts/ci/amd/ensure_vram_clear.sh rocm - - - name: Setup docker (ROCm 7.2) - run: | - touch github_summary.md - bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version rocm720 - env: - GITHUB_WORKSPACE: ${{ github.workspace }} - - - name: Install dependencies - run: | - bash scripts/ci/amd/amd_ci_install_dependency.sh --skip-test-time-deps - # Install tabulate for run_suite.py (missing in MI35x container) - bash scripts/ci/amd/amd_ci_exec.sh pip install tabulate - - - name: Performance Test MI35x ROCm 7.2 (8-GPU DeepSeek-V3.2 MTP) - timeout-minutes: 180 - run: | - > github_summary.md # Clear summary file - bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ - -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ - python3 run_suite.py --hw amd --suite nightly-perf-8-gpu-mi35x-deepseek-v32-mtp --nightly --timeout-per-file 7200 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$? - echo "$(> $GITHUB_STEP_SUMMARY || true - exit ${TEST_EXIT_CODE:-0} - - # MI35x 8-GPU DeepSeek-V4-Flash FP8 + FP4 (Accuracy + Performance combined) ROCm 7.2 - # NOTE on runtime sourcing: the DSv4 docker image (tag suffix `-DSv4`) bakes - # in sglang built from a specific commit of the amd/deepseek_v4 branch (the - # 7-char sha in the image tag is that commit). To keep the runtime as exactly - # that image-frozen sglang/aiter, we pass `--skip-sglang-build` and - # `--skip-aiter-build` so install_dependency.sh does NOT `pip install -e - # /sglang-checkout/python` (which would override the image's sglang with - # whatever this checkout happens to be) and does NOT rebuild aiter from this - # checkout's docker/rocm.Dockerfile. The /sglang-checkout mount is still used - # for shell scripts and for run_suite.py discovering test files; it does not - # poison Python imports because the image's site-packages .pth points at - # /sgl-workspace/sglang/python (a different path). nightly-8-gpu-mi35x-deepseek-v4-flash-rocm720: if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-mi35x-deepseek-v4-flash-rocm720,')) runs-on: linux-mi35x-gpu-8 @@ -1599,11 +1148,6 @@ jobs: echo "$(> $GITHUB_STEP_SUMMARY || true exit ${TEST_EXIT_CODE:-0} - # MI35x 8-GPU DeepSeek-V4-Pro FP8 + FP4 (Accuracy + Performance combined) ROCm 7.2 - # Pro is 1.6T (vs Flash 285B); load + warmup is much longer, so timeout-per-file - # and the job timeout are both larger than the Flash job. - # Same image / branch / install strategy as the Flash job above — see the comment - # block on `nightly-8-gpu-mi35x-deepseek-v4-flash-rocm720` for the rationale. nightly-8-gpu-mi35x-deepseek-v4-pro-rocm720: if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-mi35x-deepseek-v4-pro-rocm720,')) runs-on: linux-mi35x-gpu-8 @@ -1662,53 +1206,543 @@ jobs: echo "$(> $GITHUB_STEP_SUMMARY || true exit ${TEST_EXIT_CODE:-0} + # ============================================================================== + # 8-GPU Kimi-K2.6 (MI30x + MI35x) + # ============================================================================== + + nightly-8-gpu-kimi-k26-rocm720: + if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-kimi-k26-rocm720,')) + runs-on: linux-mi325-8gpu-sglang + steps: + - name: Checkout code + uses: actions/checkout@v4 + with: + ref: ${{ inputs.ref || github.sha }} + + - name: Ensure VRAM is clear + run: bash scripts/ci/amd/ensure_vram_clear.sh rocm + + - name: Setup docker (ROCm 7.2) + run: | + touch github_summary.md + bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version rocm720 + env: + GITHUB_WORKSPACE: ${{ github.workspace }} + + - name: Install dependencies + run: bash scripts/ci/amd/amd_ci_install_dependency.sh --skip-test-time-deps + + - name: Accuracy Test ROCm 7.2 (8-GPU Kimi-K2.6) + timeout-minutes: 120 + run: | + > github_summary.md # Clear summary file + bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ + -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ + python3 run_suite.py --hw amd --suite nightly-amd-accuracy-8-gpu-kimi-k26 --nightly --timeout-per-file 3600 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$? + echo "$(> $GITHUB_STEP_SUMMARY || true + exit ${TEST_EXIT_CODE:-0} + + nightly-8-gpu-mi35x-kimi-k26-rocm720: + if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-mi35x-kimi-k26-rocm720,')) + runs-on: linux-mi35x-gpu-8 + steps: + - name: Checkout code + uses: actions/checkout@v4 + with: + ref: ${{ inputs.ref || github.sha }} + + - name: Ensure VRAM is clear + run: bash scripts/ci/amd/ensure_vram_clear.sh rocm + + - name: Setup docker (ROCm 7.2) + run: | + touch github_summary.md + bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version rocm720 + env: + GITHUB_WORKSPACE: ${{ github.workspace }} + + - name: Install dependencies + run: | + bash scripts/ci/amd/amd_ci_install_dependency.sh --skip-test-time-deps + # Install tabulate for run_suite.py (missing in MI35x container) + bash scripts/ci/amd/amd_ci_exec.sh pip install tabulate + + - name: Accuracy Test MI35x ROCm 7.2 (8-GPU Kimi-K2.6) + timeout-minutes: 180 + run: | + > github_summary.md # Clear summary file + bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ + -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ + python3 run_suite.py --hw amd --suite nightly-amd-accuracy-8-gpu-mi35x-kimi-k26 --nightly --timeout-per-file 7200 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$? + echo "$(> $GITHUB_STEP_SUMMARY || true + exit ${TEST_EXIT_CODE:-0} + + # ============================================================================== + # 8-GPU Qwen3-235B (MI30x + MI35x MXFP4) + # ============================================================================== + + nightly-8-gpu-qwen3-235b-rocm720: + if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-qwen3-235b-rocm720,')) + runs-on: linux-mi325-8gpu-sglang + steps: + - name: Checkout code + uses: actions/checkout@v4 + with: + ref: ${{ inputs.ref || github.sha }} + + - name: Ensure VRAM is clear + run: bash scripts/ci/amd/ensure_vram_clear.sh rocm + + - name: Setup docker (ROCm 7.2) + run: | + touch github_summary.md + bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version rocm720 + env: + GITHUB_WORKSPACE: ${{ github.workspace }} + + - name: Install dependencies + run: bash scripts/ci/amd/amd_ci_install_dependency.sh --skip-test-time-deps + + - name: Accuracy Test + Performance Test ROCm 7.2 (8-GPU Qwen3) + timeout-minutes: 120 + run: | + > github_summary.md # Clear summary file + bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ + -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ + python3 run_suite.py --hw amd --suite nightly-8-gpu-qwen3-235b --nightly --timeout-per-file 3600 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$? + echo "$(> $GITHUB_STEP_SUMMARY || true + exit ${TEST_EXIT_CODE:-0} + + nightly-8-gpu-mi35x-qwen3-235b-mxfp4-rocm720: + if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-mi35x-qwen3-235b-mxfp4-rocm720,')) + runs-on: linux-mi35x-gpu-8 + steps: + - name: Checkout code + uses: actions/checkout@v4 + with: + ref: ${{ inputs.ref || github.sha }} + + - name: Ensure VRAM is clear + run: bash scripts/ci/amd/ensure_vram_clear.sh rocm + + - name: Setup docker (ROCm 7.2) + run: | + touch github_summary.md + bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version rocm720 + env: + GITHUB_WORKSPACE: ${{ github.workspace }} + + - name: Install dependencies + run: | + bash scripts/ci/amd/amd_ci_install_dependency.sh --skip-test-time-deps + # Install tabulate for run_suite.py (missing in MI35x container) + bash scripts/ci/amd/amd_ci_exec.sh pip install tabulate + + - name: Accuracy Test + Performance Test MI35x ROCm 7.2 (8-GPU Qwen3-235B-MXFP4) + timeout-minutes: 120 + run: | + > github_summary.md # Clear summary file + bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ + -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ + python3 run_suite.py --hw amd --suite nightly-8-gpu-mi35x-qwen3-235b-mxfp4 --nightly --timeout-per-file 3600 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$? + echo "$(> $GITHUB_STEP_SUMMARY || true + exit ${TEST_EXIT_CODE:-0} + + # ============================================================================== + # 8-GPU Qwen 3.5 (MI30x + MI35x) + # ============================================================================== + + nightly-8-gpu-qwen35-rocm720: + if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-qwen35-rocm720,')) + runs-on: linux-mi325-8gpu-sglang + steps: + - name: Checkout code + uses: actions/checkout@v4 + with: + ref: ${{ inputs.ref || github.sha }} + + - name: Ensure VRAM is clear + run: bash scripts/ci/amd/ensure_vram_clear.sh rocm + + - name: Setup docker (ROCm 7.2) + run: | + touch github_summary.md + bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version rocm720 + env: + GITHUB_WORKSPACE: ${{ github.workspace }} + + - name: Install dependencies + run: | + bash scripts/ci/amd/amd_ci_install_dependency.sh --skip-aiter-build --skip-test-time-deps + bash scripts/ci/amd/amd_ci_exec.sh pip install mistral-common "lm-eval[api]" + + - name: Accuracy Test ROCm 7.2 (8-GPU Qwen 3.5) + timeout-minutes: 120 + run: | + > github_summary.md # Clear summary file + bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ + -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ + python3 run_suite.py --hw amd --suite nightly-amd-accuracy-8-gpu-qwen35 --nightly --timeout-per-file 3600 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$? + echo "$(> $GITHUB_STEP_SUMMARY || true + exit ${TEST_EXIT_CODE:-0} + + - name: Performance Test ROCm 7.2 (8-GPU Qwen 3.5 FP8) + timeout-minutes: 120 + continue-on-error: true + run: | + > github_summary.md # Clear summary file + bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ + -e SGLANG_USE_AITER=1 \ + -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ + python3 run_suite.py --hw amd --suite nightly-perf-8-gpu-qwen35-fp8 --nightly --timeout-per-file 5400 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$? + echo "$(> $GITHUB_STEP_SUMMARY || true + exit ${TEST_EXIT_CODE:-0} + + nightly-8-gpu-mi35x-qwen35-rocm720: + if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-mi35x-qwen35-rocm720,')) + runs-on: linux-mi35x-gpu-8 + steps: + - name: Checkout code + uses: actions/checkout@v4 + with: + ref: ${{ inputs.ref || github.sha }} + + - name: Ensure VRAM is clear + run: bash scripts/ci/amd/ensure_vram_clear.sh rocm + + - name: Setup docker (ROCm 7.2) + run: | + touch github_summary.md + bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version rocm720 + env: + GITHUB_WORKSPACE: ${{ github.workspace }} + + - name: Install dependencies + run: | + bash scripts/ci/amd/amd_ci_install_dependency.sh --skip-aiter-build --skip-test-time-deps + bash scripts/ci/amd/amd_ci_exec.sh pip install tabulate + bash scripts/ci/amd/amd_ci_exec.sh pip install mistral-common "lm-eval[api]" + + - name: Accuracy Test MI35x ROCm 7.2 (8-GPU Qwen 3.5) + timeout-minutes: 120 + run: | + > github_summary.md # Clear summary file + bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ + -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ + python3 run_suite.py --hw amd --suite nightly-amd-accuracy-8-gpu-mi35x-qwen35 --nightly --timeout-per-file 3600 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$? + echo "$(> $GITHUB_STEP_SUMMARY || true + exit ${TEST_EXIT_CODE:-0} + + - name: Performance Test MI35x ROCm 7.2 (8-GPU Qwen 3.5 FP8) + timeout-minutes: 120 + continue-on-error: true + run: | + > github_summary.md # Clear summary file + bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ + -e SGLANG_USE_AITER=1 \ + -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ + python3 run_suite.py --hw amd --suite nightly-perf-8-gpu-mi35x-qwen35-fp8 --nightly --timeout-per-file 5400 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$? + echo "$(> $GITHUB_STEP_SUMMARY || true + exit ${TEST_EXIT_CODE:-0} + + # ============================================================================== + # 8-GPU GLM-5.1 (MI30x + MI35x) + # ============================================================================== + + nightly-8-gpu-glm51-rocm720: + if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-glm51-rocm720,')) + runs-on: linux-mi325-8gpu-sglang + steps: + - name: Checkout code + uses: actions/checkout@v4 + with: + ref: ${{ inputs.ref || github.sha }} + + - name: Ensure VRAM is clear + run: bash scripts/ci/amd/ensure_vram_clear.sh rocm + + - name: Setup docker (ROCm 7.2) + run: | + touch github_summary.md + bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version rocm720 + env: + GITHUB_WORKSPACE: ${{ github.workspace }} + + - name: Install dependencies + run: | + bash scripts/ci/amd/amd_ci_install_dependency.sh --skip-test-time-deps + bash scripts/ci/amd/amd_ci_exec.sh pip install git+https://github.com/huggingface/transformers.git@96f807a33b75 + + - name: Accuracy Test ROCm 7.2 (8-GPU GLM-5.1 DSA) + timeout-minutes: 120 + run: | + > github_summary.md # Clear summary file + bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ + -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ + python3 run_suite.py --hw amd --suite nightly-amd-accuracy-8-gpu-glm51 --nightly --timeout-per-file 3600 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$? + echo "$(> $GITHUB_STEP_SUMMARY || true + exit ${TEST_EXIT_CODE:-0} + + - name: Performance Test ROCm 7.2 (8-GPU GLM-5.1) + timeout-minutes: 120 + continue-on-error: true + run: | + > github_summary.md # Clear summary file + bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ + -e SGLANG_USE_AITER=1 \ + -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ + python3 run_suite.py --hw amd --suite nightly-perf-8-gpu-glm51 --nightly --timeout-per-file 5400 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$? + echo "$(> $GITHUB_STEP_SUMMARY || true + exit ${TEST_EXIT_CODE:-0} + + nightly-8-gpu-mi35x-glm51-rocm720: + if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-mi35x-glm51-rocm720,')) + runs-on: linux-mi35x-gpu-8 + steps: + - name: Checkout code + uses: actions/checkout@v4 + with: + ref: ${{ inputs.ref || github.sha }} + + - name: Ensure VRAM is clear + run: bash scripts/ci/amd/ensure_vram_clear.sh rocm + + - name: Setup docker (ROCm 7.2) + run: | + touch github_summary.md + bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version rocm720 + env: + GITHUB_WORKSPACE: ${{ github.workspace }} + + - name: Install dependencies + run: | + bash scripts/ci/amd/amd_ci_install_dependency.sh --skip-test-time-deps + bash scripts/ci/amd/amd_ci_exec.sh pip install tabulate + bash scripts/ci/amd/amd_ci_exec.sh pip install git+https://github.com/huggingface/transformers.git@96f807a33b75 + + - name: Accuracy Test MI35x ROCm 7.2 (8-GPU GLM-5.1 DSA) + timeout-minutes: 180 + run: | + > github_summary.md # Clear summary file + bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ + -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ + python3 run_suite.py --hw amd --suite nightly-amd-8-gpu-mi35x-glm51 --nightly --timeout-per-file 7200 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$? + echo "$(> $GITHUB_STEP_SUMMARY || true + exit ${TEST_EXIT_CODE:-0} + + - name: Performance Test MI35x ROCm 7.2 (8-GPU GLM-5.1) + timeout-minutes: 120 + continue-on-error: true + run: | + > github_summary.md # Clear summary file + bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ + -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ + python3 run_suite.py --hw amd --suite nightly-perf-8-gpu-mi35x-glm51 --nightly --timeout-per-file 5400 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$? + echo "$(> $GITHUB_STEP_SUMMARY || true + exit ${TEST_EXIT_CODE:-0} + + # ============================================================================== + # 8-GPU GLM-5-MXFP4 (MI35x only) + # ============================================================================== + + nightly-8-gpu-mi35x-glm5-mxfp4-rocm720: + if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-mi35x-glm5-mxfp4-rocm720,')) + runs-on: linux-mi35x-gpu-8 + steps: + - name: Checkout code + uses: actions/checkout@v4 + with: + ref: ${{ inputs.ref || github.sha }} + + - name: Ensure VRAM is clear + run: bash scripts/ci/amd/ensure_vram_clear.sh rocm + + - name: Setup docker (ROCm 7.2) + run: | + touch github_summary.md + bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version rocm720 + env: + GITHUB_WORKSPACE: ${{ github.workspace }} + + - name: Install dependencies + run: | + bash scripts/ci/amd/amd_ci_install_dependency.sh --skip-test-time-deps + bash scripts/ci/amd/amd_ci_exec.sh pip install tabulate + bash scripts/ci/amd/amd_ci_exec.sh pip install git+https://github.com/huggingface/transformers.git@96f807a33b75 + + - name: Accuracy Test MI35x ROCm 7.2 (8-GPU GLM-5-MXFP4) + timeout-minutes: 180 + run: | + > github_summary.md + bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ + -e SGLANG_USE_AITER=1 \ + -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ + python3 run_suite.py --hw amd --suite nightly-amd-8-gpu-mi35x-glm5-mxfp4 --nightly --timeout-per-file 7200 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$? + echo "$(> $GITHUB_STEP_SUMMARY || true + exit ${TEST_EXIT_CODE:-0} + + - name: Performance Test MI35x ROCm 7.2 (8-GPU GLM-5-MXFP4) + timeout-minutes: 300 + continue-on-error: true + run: | + > github_summary.md + bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ + -e SGLANG_USE_AITER=1 \ + -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ + python3 registered/amd/perf/mi35x/test_glm5_mxfp4_perf_mi35x.py || TEST_EXIT_CODE=$? + echo "$(> $GITHUB_STEP_SUMMARY || true + exit ${TEST_EXIT_CODE:-0} + + # ============================================================================== + # 8-GPU MiniMax-M2.7 (MI30x only) + # ============================================================================== + + nightly-8-gpu-minimax-m27-rocm720: + if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-minimax-m27-rocm720,')) + runs-on: linux-mi325-8gpu-sglang + steps: + - name: Checkout code + uses: actions/checkout@v4 + with: + ref: ${{ inputs.ref || github.sha }} + + - name: Ensure VRAM is clear + run: bash scripts/ci/amd/ensure_vram_clear.sh rocm + + - name: Setup docker (ROCm 7.2) + run: | + touch github_summary.md + bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version rocm720 + env: + GITHUB_WORKSPACE: ${{ github.workspace }} + + - name: Install dependencies + run: bash scripts/ci/amd/amd_ci_install_dependency.sh --skip-test-time-deps + + - name: Accuracy Test ROCm 7.2 (8-GPU MiniMax-M2.7) + timeout-minutes: 120 + run: | + > github_summary.md # Clear summary file + bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ + -e SGLANG_USE_AITER=1 \ + -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ + python3 run_suite.py --hw amd --suite nightly-amd-accuracy-8-gpu-minimax-m27 --nightly --timeout-per-file 3600 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$? + echo "$(> $GITHUB_STEP_SUMMARY || true + exit ${TEST_EXIT_CODE:-0} + + - name: Performance Test ROCm 7.2 (8-GPU MiniMax-M2.7) + timeout-minutes: 120 + continue-on-error: true # Perf test failure doesn't fail the job if accuracy passed + run: | + > github_summary.md # Clear summary file + bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ + -e SGLANG_USE_AITER=1 \ + -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ + python3 run_suite.py --hw amd --suite nightly-perf-8-gpu-minimax-m27 --nightly --timeout-per-file 5400 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$? + echo "$(> $GITHUB_STEP_SUMMARY || true + exit ${TEST_EXIT_CODE:-0} + + # ============================================================================== + # Diffusion (MI30x) + # ============================================================================== + + nightly-1-gpu-zimage-turbo-rocm720: + if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-1-gpu-zimage-turbo-rocm720,')) + runs-on: linux-mi325-1gpu-sglang + steps: + - name: Checkout code + uses: actions/checkout@v4 + with: + ref: ${{ inputs.ref || github.sha }} + + - name: Ensure VRAM is clear + run: bash scripts/ci/amd/ensure_vram_clear.sh rocm + + - name: Setup docker (ROCm 7.2) + run: | + touch github_summary.md + bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version rocm720 + env: + GITHUB_WORKSPACE: ${{ github.workspace }} + + - name: Install dependencies + run: bash scripts/ci/amd/amd_ci_install_dependency.sh + + - name: Z-Image-Turbo Diffusion Test ROCm 7.2 (1-GPU) + timeout-minutes: 45 + run: | + bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout \ + -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ + -e SGLANG_DIFFUSION_ARTIFACT_DIR="/sglang-checkout/diffusion-artifacts" \ + pytest test/registered/amd/test_zimage_turbo.py -v -s ${{ inputs.continue_on_error && '|| true' || '' }} || TEST_EXIT_CODE=$? + echo "$(> $GITHUB_STEP_SUMMARY || true + exit ${TEST_EXIT_CODE:-0} + + - name: Upload generated images + if: always() + uses: actions/upload-artifact@v4 + with: + name: zimage-turbo-outputs-rocm720 + path: diffusion-artifacts/ + if-no-files-found: ignore + retention-days: 30 + check-all-jobs: if: always() && (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request' || github.event_name == 'workflow_dispatch') needs: - # MI30x ROCm 7.2 Unit Tests + # 1-GPU Unit Tests (MI30x + MI35x) - nightly-test-1-gpu-unit-rocm720 - # MI30x ROCm 7.2 Accuracy Tests + - nightly-test-1-gpu-mi35x-rocm720 + # 2-GPU and 4-GPU Tests (MI30x) - nightly-accuracy-2-gpu-rocm720 - nightly-accuracy-2-gpu-vlm-rocm720 - # MI30x ROCm 7.2 Performance Tests - nightly-perf-2-gpu-text-rocm720 - nightly-perf-2-gpu-vlm-rocm720 - # MI30x ROCm 7.2 4-GPU Tests - nightly-4-gpu-rocm720 + # 8-GPU GPT-OSS (MI30x mixes Grok1-FP8; MI35x mixes Qwen3-Coder-Next) - nightly-accuracy-8-gpu-rocm720 - # MI30x ROCm 7.2 Combined Accuracy + Performance Tests + - nightly-accuracy-8-gpu-mi35x-rocm720 + # 8-GPU Grok1-INT4 (MI30x + MI35x) - nightly-8-gpu-grok1-int4-rocm720 + - nightly-8-gpu-mi35x-grok1-int4-rocm720 + # 8-GPU Grok2 (MI30x + MI35x) - nightly-8-gpu-grok2-rocm720 + - nightly-8-gpu-mi35x-grok2-rocm720 + # 8-GPU DeepSeek-V3.x (MI30x) - nightly-8-gpu-deepseek-v31-rocm720 - nightly-8-gpu-deepseek-v32-rocm720 - nightly-8-gpu-deepseek-v32-mtp-rocm720 - nightly-8-gpu-deepseek-v3-kv-fp8-rocm720 - - nightly-8-gpu-kimi-k26-rocm720 - - nightly-8-gpu-qwen3-235b-rocm720 - - nightly-8-gpu-qwen35-rocm720 - - nightly-8-gpu-glm51-rocm720 - - nightly-8-gpu-minimax-m27-rocm720 - # MI30x ROCm 7.2 Diffusion Tests - - nightly-1-gpu-zimage-turbo-rocm720 - # MI35x ROCm 7.2 jobs - - nightly-test-1-gpu-mi35x-rocm720 - - nightly-accuracy-8-gpu-mi35x-rocm720 - - nightly-8-gpu-mi35x-grok1-int4-rocm720 - - nightly-8-gpu-mi35x-grok2-rocm720 - - nightly-8-gpu-mi35x-deepseek-r1-mxfp4-rocm720 - - nightly-8-gpu-mi35x-deepseek-r1-mxfp4-kv-fp8-rocm720 - - nightly-8-gpu-mi35x-deepseek-r1-mxfp4-ar-fusion-rocm720 + # 8-GPU DeepSeek-V3.2 (MI35x) - nightly-accuracy-8-gpu-mi35x-deepseek-v32-rocm720 - nightly-accuracy-8-gpu-mi35x-deepseek-v32-mtp-rocm720 - nightly-perf-8-gpu-mi35x-deepseek-v32-basic-rocm720 - nightly-perf-8-gpu-mi35x-deepseek-v32-mtp-rocm720 + # 8-GPU DeepSeek-R1-MXFP4 (MI35x only) + - nightly-8-gpu-mi35x-deepseek-r1-mxfp4-rocm720 + - nightly-8-gpu-mi35x-deepseek-r1-mxfp4-kv-fp8-rocm720 + - nightly-8-gpu-mi35x-deepseek-r1-mxfp4-ar-fusion-rocm720 + # 8-GPU DeepSeek-V4 (MI35x only) - nightly-8-gpu-mi35x-deepseek-v4-flash-rocm720 - nightly-8-gpu-mi35x-deepseek-v4-pro-rocm720 + # 8-GPU Kimi-K2.6 (MI30x + MI35x) + - nightly-8-gpu-kimi-k26-rocm720 - nightly-8-gpu-mi35x-kimi-k26-rocm720 + # 8-GPU Qwen3-235B (MI30x + MI35x MXFP4) + - nightly-8-gpu-qwen3-235b-rocm720 - nightly-8-gpu-mi35x-qwen3-235b-mxfp4-rocm720 + # 8-GPU Qwen 3.5 (MI30x + MI35x) + - nightly-8-gpu-qwen35-rocm720 - nightly-8-gpu-mi35x-qwen35-rocm720 + # 8-GPU GLM-5.1 (MI30x + MI35x) + - nightly-8-gpu-glm51-rocm720 - nightly-8-gpu-mi35x-glm51-rocm720 + # 8-GPU GLM-5-MXFP4 (MI35x only) - nightly-8-gpu-mi35x-glm5-mxfp4-rocm720 + # 8-GPU MiniMax-M2.7 (MI30x only) + - nightly-8-gpu-minimax-m27-rocm720 + # Diffusion (MI30x) + - nightly-1-gpu-zimage-turbo-rocm720 runs-on: ubuntu-latest steps: - name: Check if any job failed diff --git a/.github/workflows/nightly-test-amd.yml b/.github/workflows/nightly-test-amd.yml index 01543b668..c50e10b53 100644 --- a/.github/workflows/nightly-test-amd.yml +++ b/.github/workflows/nightly-test-amd.yml @@ -27,41 +27,56 @@ on: default: 'all' options: - 'all' + # 1-GPU Unit Tests (MI30x + MI35x) - nightly-test-1-gpu-unit + - nightly-test-1-gpu-mi35x + # 2-GPU and 4-GPU Tests (MI30x) - nightly-accuracy-2-gpu - nightly-accuracy-2-gpu-vlm - nightly-perf-2-gpu-text - nightly-perf-2-gpu-vlm - nightly-4-gpu + # 8-GPU GPT-OSS (MI30x mixes Grok1-FP8; MI35x mixes Qwen3-Coder-Next) - nightly-accuracy-8-gpu + - nightly-accuracy-8-gpu-mi35x + # 8-GPU Grok1-INT4 (MI30x + MI35x) - nightly-8-gpu-grok1-int4 + - nightly-8-gpu-mi35x-grok1-int4 + # 8-GPU Grok2 (MI30x + MI35x) - nightly-8-gpu-grok2 + - nightly-8-gpu-mi35x-grok2 + # 8-GPU DeepSeek-V3.x (MI30x) - nightly-8-gpu-deepseek-v31 - nightly-8-gpu-deepseek-v32 - nightly-8-gpu-deepseek-v32-mtp - nightly-8-gpu-deepseek-v3-kv-fp8 - - nightly-8-gpu-kimi-k26 - - nightly-8-gpu-qwen3-235b - - nightly-8-gpu-qwen35 - - nightly-8-gpu-glm51 - - nightly-8-gpu-minimax-m27 - - nightly-1-gpu-zimage-turbo - - nightly-test-1-gpu-mi35x - - nightly-accuracy-8-gpu-mi35x - - nightly-8-gpu-mi35x-grok1-int4 - - nightly-8-gpu-mi35x-grok2 - - nightly-8-gpu-mi35x-deepseek-r1-mxfp4 - - nightly-8-gpu-mi35x-deepseek-r1-mxfp4-kv-fp8 - - nightly-8-gpu-mi35x-deepseek-r1-mxfp4-ar-fusion + # 8-GPU DeepSeek-V3.2 (MI35x) - nightly-accuracy-8-gpu-mi35x-deepseek-v32 - nightly-accuracy-8-gpu-mi35x-deepseek-v32-mtp - nightly-perf-8-gpu-mi35x-deepseek-v32-basic - nightly-perf-8-gpu-mi35x-deepseek-v32-mtp + # 8-GPU DeepSeek-R1-MXFP4 (MI35x only) + - nightly-8-gpu-mi35x-deepseek-r1-mxfp4 + - nightly-8-gpu-mi35x-deepseek-r1-mxfp4-kv-fp8 + - nightly-8-gpu-mi35x-deepseek-r1-mxfp4-ar-fusion + # 8-GPU Kimi-K2.6 (MI30x + MI35x) + - nightly-8-gpu-kimi-k26 - nightly-8-gpu-mi35x-kimi-k26 + # 8-GPU Qwen3-235B (MI30x + MI35x MXFP4) + - nightly-8-gpu-qwen3-235b - nightly-8-gpu-mi35x-qwen3-235b-mxfp4 + # 8-GPU Qwen 3.5 (MI30x + MI35x) + - nightly-8-gpu-qwen35 - nightly-8-gpu-mi35x-qwen35 + # 8-GPU GLM-5.1 (MI30x + MI35x) + - nightly-8-gpu-glm51 - nightly-8-gpu-mi35x-glm51 + # 8-GPU GLM-5-MXFP4 (MI35x only) - nightly-8-gpu-mi35x-glm5-mxfp4 + # 8-GPU MiniMax-M2.7 (MI30x only) + - nightly-8-gpu-minimax-m27 + # Diffusion (MI30x) + - nightly-1-gpu-zimage-turbo job_filter: description: 'Or type comma-separated job names (overrides dropdown if non-empty)' required: false @@ -106,6 +121,11 @@ concurrency: jobs: # ============================================== MI30x Unit Tests ============================================== # 1-GPU Unit Tests - LoRA, debug utils, scheduler, etc. (MI30x only) + + # ============================================================================== + # 1-GPU Unit Tests (MI30x + MI35x) + # ============================================================================== + nightly-test-1-gpu-unit: if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-test-1-gpu-unit,')) runs-on: linux-mi325-1gpu-sglang @@ -137,8 +157,44 @@ jobs: echo "$(> $GITHUB_STEP_SUMMARY || true exit ${TEST_EXIT_CODE:-0} - # ============================================== MI30x Accuracy Tests ============================================== - # 2-GPU Accuracy Tests - GSM8K eval (MI30x only) + nightly-test-1-gpu-mi35x: + if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-test-1-gpu-mi35x,')) + runs-on: linux-mi35x-gpu-1 + steps: + - name: Checkout code + uses: actions/checkout@v4 + with: + ref: ${{ inputs.ref || github.sha }} + + - name: Ensure VRAM is clear + run: bash scripts/ci/amd/ensure_vram_clear.sh rocm + + - name: Setup docker + run: | + touch github_summary.md + bash scripts/ci/amd/amd_ci_start_container.sh + env: + GITHUB_WORKSPACE: ${{ github.workspace }} + + - name: Install dependencies + run: | + bash scripts/ci/amd/amd_ci_install_dependency.sh + # Install tabulate for run_suite.py (missing in MI35x container) + bash scripts/ci/amd/amd_ci_exec.sh pip install tabulate + + - name: Nightly Test MI35x (1-GPU) + timeout-minutes: 90 + run: | + bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ + -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ + python3 run_suite.py --hw amd --suite nightly-amd-1-gpu-mi35x --nightly --timeout-per-file 900 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$? + echo "$(> $GITHUB_STEP_SUMMARY || true + exit ${TEST_EXIT_CODE:-0} + + # ============================================================================== + # 2-GPU and 4-GPU Tests (MI30x) + # ============================================================================== + nightly-accuracy-2-gpu: if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-accuracy-2-gpu,')) runs-on: linux-mi325-2gpu-sglang @@ -170,7 +226,6 @@ jobs: echo "$(> $GITHUB_STEP_SUMMARY || true exit ${TEST_EXIT_CODE:-0} - # 2-GPU VLM Accuracy Tests - Vision-Language Models MMMU evaluation nightly-accuracy-2-gpu-vlm: if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-accuracy-2-gpu-vlm,')) runs-on: linux-mi325-2gpu-sglang @@ -203,7 +258,6 @@ jobs: echo "$(> $GITHUB_STEP_SUMMARY || true exit ${TEST_EXIT_CODE:-0} - # 2-GPU Text Models Performance Tests nightly-perf-2-gpu-text: if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-perf-2-gpu-text,')) runs-on: linux-mi325-2gpu-sglang @@ -237,7 +291,6 @@ jobs: echo "$(> $GITHUB_STEP_SUMMARY || true exit ${TEST_EXIT_CODE:-0} - # 2-GPU VLM Performance Tests nightly-perf-2-gpu-vlm: if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-perf-2-gpu-vlm,')) runs-on: linux-mi325-2gpu-sglang @@ -271,8 +324,6 @@ jobs: echo "$(> $GITHUB_STEP_SUMMARY || true exit ${TEST_EXIT_CODE:-0} - # ============================================== MI30x 4-GPU Tests ============================================== - # 4-GPU Nightly Tests - Dumper/Comparator E2E, VLM Encoder DP nightly-4-gpu: if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-4-gpu,')) runs-on: linux-mi325-4gpu-sglang @@ -305,7 +356,10 @@ jobs: echo "$(> $GITHUB_STEP_SUMMARY || true exit ${TEST_EXIT_CODE:-0} - # 8-GPU Accuracy Tests - GPT-OSS, Grok1-FP8 (accuracy only) + # ============================================================================== + # 8-GPU GPT-OSS (MI30x mixes Grok1-FP8; MI35x mixes Qwen3-Coder-Next) + # ============================================================================== + nightly-accuracy-8-gpu: if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-accuracy-8-gpu,')) runs-on: linux-mi325-8gpu-sglang @@ -347,8 +401,44 @@ jobs: echo "$(> $GITHUB_STEP_SUMMARY || true exit ${TEST_EXIT_CODE:-0} - # ============================================== MI30x Combined Accuracy + Performance Tests ============================================== - # 8-GPU Grok1-INT4 (Accuracy + Performance combined) + nightly-accuracy-8-gpu-mi35x: + if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-accuracy-8-gpu-mi35x,')) + runs-on: linux-mi35x-gpu-8 + steps: + - name: Checkout code + uses: actions/checkout@v4 + with: + ref: ${{ inputs.ref || github.sha }} + + - name: Ensure VRAM is clear + run: bash scripts/ci/amd/ensure_vram_clear.sh rocm + + - name: Setup docker + run: | + touch github_summary.md + bash scripts/ci/amd/amd_ci_start_container.sh + env: + GITHUB_WORKSPACE: ${{ github.workspace }} + + - name: Install dependencies + run: | + bash scripts/ci/amd/amd_ci_install_dependency.sh + # Install tabulate for run_suite.py (missing in MI35x container) + bash scripts/ci/amd/amd_ci_exec.sh pip install tabulate + + - name: Accuracy Test MI35x (8-GPU GPT-OSS) + timeout-minutes: 180 + run: | + bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ + -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ + python3 run_suite.py --hw amd --suite nightly-amd-8-gpu-mi35x --nightly --timeout-per-file 7200 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$? + echo "$(> $GITHUB_STEP_SUMMARY || true + exit ${TEST_EXIT_CODE:-0} + + # ============================================================================== + # 8-GPU Grok1-INT4 (MI30x + MI35x) + # ============================================================================== + nightly-8-gpu-grok1-int4: if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-grok1-int4,')) runs-on: linux-mi325-8gpu-sglang @@ -394,539 +484,6 @@ jobs: echo "$(> $GITHUB_STEP_SUMMARY || true exit ${TEST_EXIT_CODE:-0} - # 8-GPU Grok2 (Accuracy + Performance combined) - nightly-8-gpu-grok2: - if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-grok2,')) - runs-on: linux-mi325-8gpu-sglang - steps: - - name: Checkout code - uses: actions/checkout@v4 - with: - ref: ${{ inputs.ref || github.sha }} - - - name: Ensure VRAM is clear - run: bash scripts/ci/amd/ensure_vram_clear.sh rocm - - - name: Setup docker - run: | - touch github_summary.md - bash scripts/ci/amd/amd_ci_start_container.sh - env: - GITHUB_WORKSPACE: ${{ github.workspace }} - - - name: Install dependencies - run: bash scripts/ci/amd/amd_ci_install_dependency.sh - - - name: Accuracy Test (8-GPU Grok2) - timeout-minutes: 60 - run: | - > github_summary.md # Clear summary file - bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ - -e RCCL_MSCCL_ENABLE=0 \ - -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ - python3 run_suite.py --hw amd --suite nightly-amd-accuracy-8-gpu-grok2 --nightly --timeout-per-file 3600 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$? - echo "$(> $GITHUB_STEP_SUMMARY || true - exit ${TEST_EXIT_CODE:-0} - - - name: Performance Test (8-GPU Grok2) - timeout-minutes: 60 - continue-on-error: true # Perf test failure doesn't fail the job if accuracy passed - run: | - > github_summary.md # Clear summary file - bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ - -e RCCL_MSCCL_ENABLE=0 \ - -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ - python3 run_suite.py --hw amd --suite nightly-perf-8-gpu-grok2 --nightly --timeout-per-file 3600 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$? - echo "$(> $GITHUB_STEP_SUMMARY || true - exit ${TEST_EXIT_CODE:-0} - - # 8-GPU DeepSeek-V3.1 (Accuracy + Performance combined) - nightly-8-gpu-deepseek-v31: - if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-deepseek-v31,')) - runs-on: linux-mi325-8gpu-sglang - steps: - - name: Checkout code - uses: actions/checkout@v4 - with: - ref: ${{ inputs.ref || github.sha }} - - - name: Ensure VRAM is clear - run: bash scripts/ci/amd/ensure_vram_clear.sh rocm - - - name: Setup docker - run: | - touch github_summary.md - bash scripts/ci/amd/amd_ci_start_container.sh - env: - GITHUB_WORKSPACE: ${{ github.workspace }} - - - name: Install dependencies - run: bash scripts/ci/amd/amd_ci_install_dependency.sh - - - name: Accuracy Test (8-GPU DeepSeek-V3.1) - timeout-minutes: 120 - run: | - > github_summary.md # Clear summary file - bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ - -e SGLANG_USE_AITER=1 \ - -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ - python3 run_suite.py --hw amd --suite nightly-amd-accuracy-8-gpu-deepseek-v31 --nightly --timeout-per-file 3600 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$? - echo "$(> $GITHUB_STEP_SUMMARY || true - exit ${TEST_EXIT_CODE:-0} - - - name: Performance Test (8-GPU DeepSeek-V3.1) - timeout-minutes: 300 - continue-on-error: true # Perf test failure doesn't fail the job if accuracy passed - run: | - > github_summary.md # Clear summary file - bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ - -e SGLANG_USE_ROCM700A=1 \ - -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ - python3 run_suite.py --hw amd --suite nightly-perf-8-gpu-deepseek-v31 --nightly --timeout-per-file 18000 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$? - echo "$(> $GITHUB_STEP_SUMMARY || true - exit ${TEST_EXIT_CODE:-0} - - # 8-GPU DeepSeek-V3.2 (Basic Accuracy + Perf) - nightly-8-gpu-deepseek-v32: - if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-deepseek-v32,')) - runs-on: linux-mi325-8gpu-sglang - steps: - - name: Checkout code - uses: actions/checkout@v4 - with: - ref: ${{ inputs.ref || github.sha }} - - - name: Ensure VRAM is clear - run: bash scripts/ci/amd/ensure_vram_clear.sh rocm - - - name: Setup docker - run: | - touch github_summary.md - bash scripts/ci/amd/amd_ci_start_container.sh - env: - GITHUB_WORKSPACE: ${{ github.workspace }} - - - name: Install dependencies - run: bash scripts/ci/amd/amd_ci_install_dependency.sh - - - name: Accuracy Test (8-GPU DeepSeek-V3.2 Basic) - timeout-minutes: 120 - run: | - > github_summary.md # Clear summary file - bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ - -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ - python3 run_suite.py --hw amd --suite nightly-amd-accuracy-8-gpu-deepseek-v32 --nightly --timeout-per-file 3600 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$? - echo "$(> $GITHUB_STEP_SUMMARY || true - exit ${TEST_EXIT_CODE:-0} - - - name: Performance Test (8-GPU DeepSeek-V3.2 Basic) - timeout-minutes: 150 - continue-on-error: true # Perf test failure doesn't fail the job if accuracy passed - run: | - > github_summary.md # Clear summary file - bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ - -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ - python3 run_suite.py --hw amd --suite nightly-perf-8-gpu-deepseek-v32-basic --nightly --timeout-per-file 5400 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$? - echo "$(> $GITHUB_STEP_SUMMARY || true - exit ${TEST_EXIT_CODE:-0} - - # 8-GPU DeepSeek-V3.2 MTP (MTP Accuracy + Perf) - nightly-8-gpu-deepseek-v32-mtp: - if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-deepseek-v32-mtp,')) - runs-on: linux-mi325-8gpu-sglang - steps: - - name: Checkout code - uses: actions/checkout@v4 - with: - ref: ${{ inputs.ref || github.sha }} - - - name: Ensure VRAM is clear - run: bash scripts/ci/amd/ensure_vram_clear.sh rocm - - - name: Setup docker - run: | - touch github_summary.md - bash scripts/ci/amd/amd_ci_start_container.sh - env: - GITHUB_WORKSPACE: ${{ github.workspace }} - - - name: Install dependencies - run: bash scripts/ci/amd/amd_ci_install_dependency.sh - - - name: Accuracy Test (8-GPU DeepSeek-V3.2 MTP) - timeout-minutes: 120 - run: | - > github_summary.md # Clear summary file - bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ - -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ - python3 run_suite.py --hw amd --suite nightly-amd-accuracy-8-gpu-deepseek-v32-mtp --nightly --timeout-per-file 3600 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$? - echo "$(> $GITHUB_STEP_SUMMARY || true - exit ${TEST_EXIT_CODE:-0} - - - name: Performance Test (8-GPU DeepSeek-V3.2 MTP) - timeout-minutes: 180 - continue-on-error: true # Perf test failure doesn't fail the job if accuracy passed - run: | - > github_summary.md # Clear summary file - bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ - -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ - python3 run_suite.py --hw amd --suite nightly-perf-8-gpu-deepseek-v32-mtp --nightly --timeout-per-file 7200 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$? - echo "$(> $GITHUB_STEP_SUMMARY || true - exit ${TEST_EXIT_CODE:-0} - - # 8-GPU DeepSeek-V3 KV FP8 (Basic + MTP with --kv-cache-dtype fp8_e4m3) - nightly-8-gpu-deepseek-v3-kv-fp8: - if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-deepseek-v3-kv-fp8,')) - runs-on: linux-mi325-8gpu-sglang - steps: - - name: Checkout code - uses: actions/checkout@v4 - with: - ref: ${{ inputs.ref || github.sha }} - - - name: Ensure VRAM is clear - run: bash scripts/ci/amd/ensure_vram_clear.sh rocm - - - name: Setup docker - run: | - touch github_summary.md - bash scripts/ci/amd/amd_ci_start_container.sh - env: - GITHUB_WORKSPACE: ${{ github.workspace }} - - - name: Install dependencies - run: bash scripts/ci/amd/amd_ci_install_dependency.sh - - - name: DeepSeek-V3 KV FP8 Test (8-GPU Basic + MTP) - timeout-minutes: 120 - run: | - > github_summary.md # Clear summary file - bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ - -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ - python3 run_suite.py --hw amd --suite nightly-amd-8-gpu-deepseek-v3-kv-fp8 --nightly --timeout-per-file 3600 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$? - echo "$(> $GITHUB_STEP_SUMMARY || true - exit ${TEST_EXIT_CODE:-0} - - # 8-GPU Kimi-K2.6 (Accuracy) - nightly-8-gpu-kimi-k26: - if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-kimi-k26,')) - runs-on: linux-mi325-8gpu-sglang - steps: - - name: Checkout code - uses: actions/checkout@v4 - with: - ref: ${{ inputs.ref || github.sha }} - - - name: Ensure VRAM is clear - run: bash scripts/ci/amd/ensure_vram_clear.sh rocm - - - name: Setup docker - run: | - touch github_summary.md - bash scripts/ci/amd/amd_ci_start_container.sh - env: - GITHUB_WORKSPACE: ${{ github.workspace }} - - - name: Install dependencies - run: bash scripts/ci/amd/amd_ci_install_dependency.sh - - - name: Accuracy Test (8-GPU Kimi-K2.6) - timeout-minutes: 120 - run: | - > github_summary.md # Clear summary file - bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ - -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ - python3 run_suite.py --hw amd --suite nightly-amd-accuracy-8-gpu-kimi-k26 --nightly --timeout-per-file 3600 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$? - echo "$(> $GITHUB_STEP_SUMMARY || true - exit ${TEST_EXIT_CODE:-0} - - nightly-8-gpu-qwen3-235b: - if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-qwen3-235b,')) - runs-on: linux-mi325-8gpu-sglang - steps: - - name: Checkout code - uses: actions/checkout@v4 - with: - ref: ${{ inputs.ref || github.sha }} - - - name: Ensure VRAM is clear - run: bash scripts/ci/amd/ensure_vram_clear.sh rocm - - - name: Setup docker - run: | - touch github_summary.md - bash scripts/ci/amd/amd_ci_start_container.sh - env: - GITHUB_WORKSPACE: ${{ github.workspace }} - - - name: Install dependencies - run: bash scripts/ci/amd/amd_ci_install_dependency.sh - - - name: Accuracy Test + Performance Test (8-GPU Qwen3) - timeout-minutes: 120 - run: | - > github_summary.md # Clear summary file - bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ - -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ - python3 run_suite.py --hw amd --suite nightly-8-gpu-qwen3-235b --nightly --timeout-per-file 3600 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$? - echo "$(> $GITHUB_STEP_SUMMARY || true - exit ${TEST_EXIT_CODE:-0} - - # 8-GPU Qwen 3.5 (Accuracy + Performance combined) - nightly-8-gpu-qwen35: - if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-qwen35,')) - runs-on: linux-mi325-8gpu-sglang - steps: - - name: Checkout code - uses: actions/checkout@v4 - with: - ref: ${{ inputs.ref || github.sha }} - - - name: Ensure VRAM is clear - run: bash scripts/ci/amd/ensure_vram_clear.sh rocm - - - name: Setup docker - run: | - touch github_summary.md - bash scripts/ci/amd/amd_ci_start_container.sh - env: - GITHUB_WORKSPACE: ${{ github.workspace }} - - - name: Install dependencies - run: | - bash scripts/ci/amd/amd_ci_install_dependency.sh - bash scripts/ci/amd/amd_ci_exec.sh pip install mistral-common "lm-eval[api]" - - - name: Accuracy Test (8-GPU Qwen 3.5) - timeout-minutes: 120 - run: | - > github_summary.md # Clear summary file - bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ - -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ - python3 run_suite.py --hw amd --suite nightly-amd-accuracy-8-gpu-qwen35 --nightly --timeout-per-file 3600 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$? - echo "$(> $GITHUB_STEP_SUMMARY || true - exit ${TEST_EXIT_CODE:-0} - - - name: Performance Test (8-GPU Qwen 3.5 FP8) - timeout-minutes: 120 - continue-on-error: true - run: | - > github_summary.md # Clear summary file - bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ - -e SGLANG_USE_AITER=1 \ - -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ - python3 run_suite.py --hw amd --suite nightly-perf-8-gpu-qwen35-fp8 --nightly --timeout-per-file 5400 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$? - echo "$(> $GITHUB_STEP_SUMMARY || true - exit ${TEST_EXIT_CODE:-0} - - # 8-GPU GLM-5.1 (Accuracy + Performance combined) - nightly-8-gpu-glm51: - if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-glm51,')) - runs-on: linux-mi325-8gpu-sglang - steps: - - name: Checkout code - uses: actions/checkout@v4 - with: - ref: ${{ inputs.ref || github.sha }} - - - name: Ensure VRAM is clear - run: bash scripts/ci/amd/ensure_vram_clear.sh rocm - - - name: Setup docker - run: | - touch github_summary.md - bash scripts/ci/amd/amd_ci_start_container.sh - env: - GITHUB_WORKSPACE: ${{ github.workspace }} - - - name: Install dependencies - run: | - bash scripts/ci/amd/amd_ci_install_dependency.sh - bash scripts/ci/amd/amd_ci_exec.sh pip install git+https://github.com/huggingface/transformers.git@96f807a33b75 - - - name: Accuracy Test (8-GPU GLM-5.1 DSA) - timeout-minutes: 120 - run: | - > github_summary.md # Clear summary file - bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ - -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ - python3 run_suite.py --hw amd --suite nightly-amd-accuracy-8-gpu-glm51 --nightly --timeout-per-file 3600 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$? - echo "$(> $GITHUB_STEP_SUMMARY || true - exit ${TEST_EXIT_CODE:-0} - - - name: Performance Test (8-GPU GLM-5.1) - timeout-minutes: 120 - continue-on-error: true - run: | - > github_summary.md # Clear summary file - bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ - -e SGLANG_USE_AITER=1 \ - -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ - python3 run_suite.py --hw amd --suite nightly-perf-8-gpu-glm51 --nightly --timeout-per-file 5400 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$? - echo "$(> $GITHUB_STEP_SUMMARY || true - exit ${TEST_EXIT_CODE:-0} - - # 8-GPU MiniMax-M2.7 (Accuracy + Performance combined, replaces M2.5) - nightly-8-gpu-minimax-m27: - if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-minimax-m27,')) - runs-on: linux-mi325-8gpu-sglang - steps: - - name: Checkout code - uses: actions/checkout@v4 - with: - ref: ${{ inputs.ref || github.sha }} - - - name: Ensure VRAM is clear - run: bash scripts/ci/amd/ensure_vram_clear.sh rocm - - - name: Setup docker - run: | - touch github_summary.md - bash scripts/ci/amd/amd_ci_start_container.sh - env: - GITHUB_WORKSPACE: ${{ github.workspace }} - - - name: Install dependencies - run: bash scripts/ci/amd/amd_ci_install_dependency.sh - - - name: Accuracy Test (8-GPU MiniMax-M2.7) - timeout-minutes: 120 - run: | - > github_summary.md # Clear summary file - bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ - -e SGLANG_USE_AITER=1 \ - -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ - python3 run_suite.py --hw amd --suite nightly-amd-accuracy-8-gpu-minimax-m27 --nightly --timeout-per-file 3600 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$? - echo "$(> $GITHUB_STEP_SUMMARY || true - exit ${TEST_EXIT_CODE:-0} - - - name: Performance Test (8-GPU MiniMax-M2.7) - timeout-minutes: 120 - continue-on-error: true # Perf test failure doesn't fail the job if accuracy passed - run: | - > github_summary.md # Clear summary file - bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ - -e SGLANG_USE_AITER=1 \ - -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ - python3 run_suite.py --hw amd --suite nightly-perf-8-gpu-minimax-m27 --nightly --timeout-per-file 5400 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$? - echo "$(> $GITHUB_STEP_SUMMARY || true - exit ${TEST_EXIT_CODE:-0} - - # ============================================== MI30x Diffusion Tests ============================================== - # 1-GPU Z-Image-Turbo (Diffusion T2I) - nightly-1-gpu-zimage-turbo: - if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-1-gpu-zimage-turbo,')) - runs-on: linux-mi325-1gpu-sglang - steps: - - name: Checkout code - uses: actions/checkout@v4 - with: - ref: ${{ inputs.ref || github.sha }} - - - name: Ensure VRAM is clear - run: bash scripts/ci/amd/ensure_vram_clear.sh rocm - - - name: Setup docker - run: | - touch github_summary.md - bash scripts/ci/amd/amd_ci_start_container.sh - env: - GITHUB_WORKSPACE: ${{ github.workspace }} - - - name: Install dependencies - run: bash scripts/ci/amd/amd_ci_install_dependency.sh - - - name: Z-Image-Turbo Diffusion Test (1-GPU) - timeout-minutes: 45 - run: | - bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout \ - -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ - -e SGLANG_DIFFUSION_ARTIFACT_DIR="/sglang-checkout/diffusion-artifacts" \ - pytest test/registered/amd/test_zimage_turbo.py -v -s ${{ inputs.continue_on_error && '|| true' || '' }} || TEST_EXIT_CODE=$? - echo "$(> $GITHUB_STEP_SUMMARY || true - exit ${TEST_EXIT_CODE:-0} - - - name: Upload generated images - if: always() - uses: actions/upload-artifact@v4 - with: - name: zimage-turbo-outputs - path: diffusion-artifacts/ - if-no-files-found: ignore - retention-days: 30 - - # ============================================== MI35x Tests ============================================== - # MI35x 1-GPU tests - platform-agnostic tests that may work on CDNA4 (gfx950) - nightly-test-1-gpu-mi35x: - if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-test-1-gpu-mi35x,')) - runs-on: linux-mi35x-gpu-1 - steps: - - name: Checkout code - uses: actions/checkout@v4 - with: - ref: ${{ inputs.ref || github.sha }} - - - name: Ensure VRAM is clear - run: bash scripts/ci/amd/ensure_vram_clear.sh rocm - - - name: Setup docker - run: | - touch github_summary.md - bash scripts/ci/amd/amd_ci_start_container.sh - env: - GITHUB_WORKSPACE: ${{ github.workspace }} - - - name: Install dependencies - run: | - bash scripts/ci/amd/amd_ci_install_dependency.sh - # Install tabulate for run_suite.py (missing in MI35x container) - bash scripts/ci/amd/amd_ci_exec.sh pip install tabulate - - - name: Nightly Test MI35x (1-GPU) - timeout-minutes: 90 - run: | - bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ - -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ - python3 run_suite.py --hw amd --suite nightly-amd-1-gpu-mi35x --nightly --timeout-per-file 900 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$? - echo "$(> $GITHUB_STEP_SUMMARY || true - exit ${TEST_EXIT_CODE:-0} - - # MI35x 8-GPU Accuracy Tests - GPT-OSS (accuracy only) - nightly-accuracy-8-gpu-mi35x: - if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-accuracy-8-gpu-mi35x,')) - runs-on: linux-mi35x-gpu-8 - steps: - - name: Checkout code - uses: actions/checkout@v4 - with: - ref: ${{ inputs.ref || github.sha }} - - - name: Ensure VRAM is clear - run: bash scripts/ci/amd/ensure_vram_clear.sh rocm - - - name: Setup docker - run: | - touch github_summary.md - bash scripts/ci/amd/amd_ci_start_container.sh - env: - GITHUB_WORKSPACE: ${{ github.workspace }} - - - name: Install dependencies - run: | - bash scripts/ci/amd/amd_ci_install_dependency.sh - # Install tabulate for run_suite.py (missing in MI35x container) - bash scripts/ci/amd/amd_ci_exec.sh pip install tabulate - - - name: Accuracy Test MI35x (8-GPU GPT-OSS) - timeout-minutes: 180 - run: | - bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ - -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ - python3 run_suite.py --hw amd --suite nightly-amd-8-gpu-mi35x --nightly --timeout-per-file 7200 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$? - echo "$(> $GITHUB_STEP_SUMMARY || true - exit ${TEST_EXIT_CODE:-0} - - # MI35x 8-GPU Grok1-INT4 (Accuracy + Performance combined) nightly-8-gpu-mi35x-grok1-int4: if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-mi35x-grok1-int4,')) runs-on: linux-mi35x-gpu-8 @@ -975,7 +532,55 @@ jobs: echo "$(> $GITHUB_STEP_SUMMARY || true exit ${TEST_EXIT_CODE:-0} - # MI35x 8-GPU Grok2 (Accuracy + Performance combined) + # ============================================================================== + # 8-GPU Grok2 (MI30x + MI35x) + # ============================================================================== + + nightly-8-gpu-grok2: + if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-grok2,')) + runs-on: linux-mi325-8gpu-sglang + steps: + - name: Checkout code + uses: actions/checkout@v4 + with: + ref: ${{ inputs.ref || github.sha }} + + - name: Ensure VRAM is clear + run: bash scripts/ci/amd/ensure_vram_clear.sh rocm + + - name: Setup docker + run: | + touch github_summary.md + bash scripts/ci/amd/amd_ci_start_container.sh + env: + GITHUB_WORKSPACE: ${{ github.workspace }} + + - name: Install dependencies + run: bash scripts/ci/amd/amd_ci_install_dependency.sh + + - name: Accuracy Test (8-GPU Grok2) + timeout-minutes: 60 + run: | + > github_summary.md # Clear summary file + bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ + -e RCCL_MSCCL_ENABLE=0 \ + -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ + python3 run_suite.py --hw amd --suite nightly-amd-accuracy-8-gpu-grok2 --nightly --timeout-per-file 3600 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$? + echo "$(> $GITHUB_STEP_SUMMARY || true + exit ${TEST_EXIT_CODE:-0} + + - name: Performance Test (8-GPU Grok2) + timeout-minutes: 60 + continue-on-error: true # Perf test failure doesn't fail the job if accuracy passed + run: | + > github_summary.md # Clear summary file + bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ + -e RCCL_MSCCL_ENABLE=0 \ + -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ + python3 run_suite.py --hw amd --suite nightly-perf-8-gpu-grok2 --nightly --timeout-per-file 3600 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$? + echo "$(> $GITHUB_STEP_SUMMARY || true + exit ${TEST_EXIT_CODE:-0} + nightly-8-gpu-mi35x-grok2: if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-mi35x-grok2,')) runs-on: linux-mi35x-gpu-8 @@ -1024,7 +629,321 @@ jobs: echo "$(> $GITHUB_STEP_SUMMARY || true exit ${TEST_EXIT_CODE:-0} - # MI35x 8-GPU DeepSeek-R1-MXFP4 (Accuracy + Performance combined) + # ============================================================================== + # 8-GPU DeepSeek-V3.x (MI30x) + # ============================================================================== + + nightly-8-gpu-deepseek-v31: + if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-deepseek-v31,')) + runs-on: linux-mi325-8gpu-sglang + steps: + - name: Checkout code + uses: actions/checkout@v4 + with: + ref: ${{ inputs.ref || github.sha }} + + - name: Ensure VRAM is clear + run: bash scripts/ci/amd/ensure_vram_clear.sh rocm + + - name: Setup docker + run: | + touch github_summary.md + bash scripts/ci/amd/amd_ci_start_container.sh + env: + GITHUB_WORKSPACE: ${{ github.workspace }} + + - name: Install dependencies + run: bash scripts/ci/amd/amd_ci_install_dependency.sh + + - name: Accuracy Test (8-GPU DeepSeek-V3.1) + timeout-minutes: 120 + run: | + > github_summary.md # Clear summary file + bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ + -e SGLANG_USE_AITER=1 \ + -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ + python3 run_suite.py --hw amd --suite nightly-amd-accuracy-8-gpu-deepseek-v31 --nightly --timeout-per-file 3600 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$? + echo "$(> $GITHUB_STEP_SUMMARY || true + exit ${TEST_EXIT_CODE:-0} + + - name: Performance Test (8-GPU DeepSeek-V3.1) + timeout-minutes: 300 + continue-on-error: true # Perf test failure doesn't fail the job if accuracy passed + run: | + > github_summary.md # Clear summary file + bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ + -e SGLANG_USE_ROCM700A=1 \ + -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ + python3 run_suite.py --hw amd --suite nightly-perf-8-gpu-deepseek-v31 --nightly --timeout-per-file 18000 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$? + echo "$(> $GITHUB_STEP_SUMMARY || true + exit ${TEST_EXIT_CODE:-0} + + nightly-8-gpu-deepseek-v32: + if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-deepseek-v32,')) + runs-on: linux-mi325-8gpu-sglang + steps: + - name: Checkout code + uses: actions/checkout@v4 + with: + ref: ${{ inputs.ref || github.sha }} + + - name: Ensure VRAM is clear + run: bash scripts/ci/amd/ensure_vram_clear.sh rocm + + - name: Setup docker + run: | + touch github_summary.md + bash scripts/ci/amd/amd_ci_start_container.sh + env: + GITHUB_WORKSPACE: ${{ github.workspace }} + + - name: Install dependencies + run: bash scripts/ci/amd/amd_ci_install_dependency.sh + + - name: Accuracy Test (8-GPU DeepSeek-V3.2 Basic) + timeout-minutes: 120 + run: | + > github_summary.md # Clear summary file + bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ + -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ + python3 run_suite.py --hw amd --suite nightly-amd-accuracy-8-gpu-deepseek-v32 --nightly --timeout-per-file 3600 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$? + echo "$(> $GITHUB_STEP_SUMMARY || true + exit ${TEST_EXIT_CODE:-0} + + - name: Performance Test (8-GPU DeepSeek-V3.2 Basic) + timeout-minutes: 150 + continue-on-error: true # Perf test failure doesn't fail the job if accuracy passed + run: | + > github_summary.md # Clear summary file + bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ + -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ + python3 run_suite.py --hw amd --suite nightly-perf-8-gpu-deepseek-v32-basic --nightly --timeout-per-file 5400 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$? + echo "$(> $GITHUB_STEP_SUMMARY || true + exit ${TEST_EXIT_CODE:-0} + + nightly-8-gpu-deepseek-v32-mtp: + if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-deepseek-v32-mtp,')) + runs-on: linux-mi325-8gpu-sglang + steps: + - name: Checkout code + uses: actions/checkout@v4 + with: + ref: ${{ inputs.ref || github.sha }} + + - name: Ensure VRAM is clear + run: bash scripts/ci/amd/ensure_vram_clear.sh rocm + + - name: Setup docker + run: | + touch github_summary.md + bash scripts/ci/amd/amd_ci_start_container.sh + env: + GITHUB_WORKSPACE: ${{ github.workspace }} + + - name: Install dependencies + run: bash scripts/ci/amd/amd_ci_install_dependency.sh + + - name: Accuracy Test (8-GPU DeepSeek-V3.2 MTP) + timeout-minutes: 120 + run: | + > github_summary.md # Clear summary file + bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ + -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ + python3 run_suite.py --hw amd --suite nightly-amd-accuracy-8-gpu-deepseek-v32-mtp --nightly --timeout-per-file 3600 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$? + echo "$(> $GITHUB_STEP_SUMMARY || true + exit ${TEST_EXIT_CODE:-0} + + - name: Performance Test (8-GPU DeepSeek-V3.2 MTP) + timeout-minutes: 180 + continue-on-error: true # Perf test failure doesn't fail the job if accuracy passed + run: | + > github_summary.md # Clear summary file + bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ + -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ + python3 run_suite.py --hw amd --suite nightly-perf-8-gpu-deepseek-v32-mtp --nightly --timeout-per-file 7200 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$? + echo "$(> $GITHUB_STEP_SUMMARY || true + exit ${TEST_EXIT_CODE:-0} + + nightly-8-gpu-deepseek-v3-kv-fp8: + if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-deepseek-v3-kv-fp8,')) + runs-on: linux-mi325-8gpu-sglang + steps: + - name: Checkout code + uses: actions/checkout@v4 + with: + ref: ${{ inputs.ref || github.sha }} + + - name: Ensure VRAM is clear + run: bash scripts/ci/amd/ensure_vram_clear.sh rocm + + - name: Setup docker + run: | + touch github_summary.md + bash scripts/ci/amd/amd_ci_start_container.sh + env: + GITHUB_WORKSPACE: ${{ github.workspace }} + + - name: Install dependencies + run: bash scripts/ci/amd/amd_ci_install_dependency.sh + + - name: DeepSeek-V3 KV FP8 Test (8-GPU Basic + MTP) + timeout-minutes: 120 + run: | + > github_summary.md # Clear summary file + bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ + -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ + python3 run_suite.py --hw amd --suite nightly-amd-8-gpu-deepseek-v3-kv-fp8 --nightly --timeout-per-file 3600 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$? + echo "$(> $GITHUB_STEP_SUMMARY || true + exit ${TEST_EXIT_CODE:-0} + + # ============================================================================== + # 8-GPU DeepSeek-V3.2 (MI35x) + # ============================================================================== + + nightly-accuracy-8-gpu-mi35x-deepseek-v32: + if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-accuracy-8-gpu-mi35x-deepseek-v32,')) + runs-on: linux-mi35x-gpu-8 + steps: + - name: Checkout code + uses: actions/checkout@v4 + with: + ref: ${{ inputs.ref || github.sha }} + + - name: Ensure VRAM is clear + run: bash scripts/ci/amd/ensure_vram_clear.sh rocm + + - name: Setup docker + run: | + touch github_summary.md + bash scripts/ci/amd/amd_ci_start_container.sh + env: + GITHUB_WORKSPACE: ${{ github.workspace }} + + - name: Install dependencies + run: | + bash scripts/ci/amd/amd_ci_install_dependency.sh + # Install tabulate for run_suite.py (missing in MI35x container) + bash scripts/ci/amd/amd_ci_exec.sh pip install tabulate + + - name: Accuracy Test MI35x (8-GPU DeepSeek-V3.2) + timeout-minutes: 120 + run: | + > github_summary.md # Clear summary file + bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ + -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ + python3 run_suite.py --hw amd --suite nightly-amd-8-gpu-mi35x-deepseek-v32 --nightly --timeout-per-file 3600 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$? + echo "$(> $GITHUB_STEP_SUMMARY || true + exit ${TEST_EXIT_CODE:-0} + + nightly-accuracy-8-gpu-mi35x-deepseek-v32-mtp: + if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-accuracy-8-gpu-mi35x-deepseek-v32-mtp,')) + runs-on: linux-mi35x-gpu-8 + steps: + - name: Checkout code + uses: actions/checkout@v4 + with: + ref: ${{ inputs.ref || github.sha }} + + - name: Ensure VRAM is clear + run: bash scripts/ci/amd/ensure_vram_clear.sh rocm + + - name: Setup docker + run: | + touch github_summary.md + bash scripts/ci/amd/amd_ci_start_container.sh + env: + GITHUB_WORKSPACE: ${{ github.workspace }} + + - name: Install dependencies + run: | + bash scripts/ci/amd/amd_ci_install_dependency.sh + # Install tabulate for run_suite.py (missing in MI35x container) + bash scripts/ci/amd/amd_ci_exec.sh pip install tabulate + + - name: Accuracy Test MI35x (8-GPU DeepSeek-V3.2 TP+MTP) + timeout-minutes: 120 + run: | + > github_summary.md # Clear summary file + bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ + -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ + python3 run_suite.py --hw amd --suite nightly-amd-accuracy-8-gpu-mi35x-deepseek-v32-mtp --nightly --timeout-per-file 3600 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$? + echo "$(> $GITHUB_STEP_SUMMARY || true + exit ${TEST_EXIT_CODE:-0} + + nightly-perf-8-gpu-mi35x-deepseek-v32-basic: + if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-perf-8-gpu-mi35x-deepseek-v32-basic,')) + runs-on: linux-mi35x-gpu-8 + steps: + - name: Checkout code + uses: actions/checkout@v4 + with: + ref: ${{ inputs.ref || github.sha }} + + - name: Ensure VRAM is clear + run: bash scripts/ci/amd/ensure_vram_clear.sh rocm + + - name: Setup docker + run: | + touch github_summary.md + bash scripts/ci/amd/amd_ci_start_container.sh + env: + GITHUB_WORKSPACE: ${{ github.workspace }} + + - name: Install dependencies + run: | + bash scripts/ci/amd/amd_ci_install_dependency.sh + # Install tabulate for run_suite.py (missing in MI35x container) + bash scripts/ci/amd/amd_ci_exec.sh pip install tabulate + + - name: Performance Test MI35x (8-GPU DeepSeek-V3.2 Basic) + timeout-minutes: 150 + run: | + > github_summary.md # Clear summary file + bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ + -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ + python3 run_suite.py --hw amd --suite nightly-perf-8-gpu-mi35x-deepseek-v32-basic --nightly --timeout-per-file 5400 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$? + echo "$(> $GITHUB_STEP_SUMMARY || true + exit ${TEST_EXIT_CODE:-0} + + nightly-perf-8-gpu-mi35x-deepseek-v32-mtp: + if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-perf-8-gpu-mi35x-deepseek-v32-mtp,')) + runs-on: linux-mi35x-gpu-8 + steps: + - name: Checkout code + uses: actions/checkout@v4 + with: + ref: ${{ inputs.ref || github.sha }} + + - name: Ensure VRAM is clear + run: bash scripts/ci/amd/ensure_vram_clear.sh rocm + + - name: Setup docker + run: | + touch github_summary.md + bash scripts/ci/amd/amd_ci_start_container.sh + env: + GITHUB_WORKSPACE: ${{ github.workspace }} + + - name: Install dependencies + run: | + bash scripts/ci/amd/amd_ci_install_dependency.sh + # Install tabulate for run_suite.py (missing in MI35x container) + bash scripts/ci/amd/amd_ci_exec.sh pip install tabulate + + - name: Performance Test MI35x (8-GPU DeepSeek-V3.2 MTP) + timeout-minutes: 180 + run: | + > github_summary.md # Clear summary file + bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ + -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ + python3 run_suite.py --hw amd --suite nightly-perf-8-gpu-mi35x-deepseek-v32-mtp --nightly --timeout-per-file 7200 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$? + echo "$(> $GITHUB_STEP_SUMMARY || true + exit ${TEST_EXIT_CODE:-0} + + # ============================================================================== + # 8-GPU DeepSeek-R1-MXFP4 (MI35x only) + # ============================================================================== + nightly-8-gpu-mi35x-deepseek-r1-mxfp4: if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-mi35x-deepseek-r1-mxfp4,')) runs-on: linux-mi35x-gpu-8 @@ -1071,7 +990,6 @@ jobs: echo "$(> $GITHUB_STEP_SUMMARY || true exit ${TEST_EXIT_CODE:-0} - # MI35x 8-GPU DeepSeek-R1-MXFP4 KV FP8 (Accuracy + Performance combined) nightly-8-gpu-mi35x-deepseek-r1-mxfp4-kv-fp8: if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-mi35x-deepseek-r1-mxfp4-kv-fp8,')) runs-on: linux-mi35x-gpu-8 @@ -1118,7 +1036,6 @@ jobs: echo "$(> $GITHUB_STEP_SUMMARY || true exit ${TEST_EXIT_CODE:-0} - # MI35x 8-GPU DeepSeek-R1-MXFP4 AllReduce Fusion (Accuracy + Performance combined) nightly-8-gpu-mi35x-deepseek-r1-mxfp4-ar-fusion: if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-mi35x-deepseek-r1-mxfp4-ar-fusion,')) runs-on: linux-mi35x-gpu-8 @@ -1165,10 +1082,13 @@ jobs: echo "$(> $GITHUB_STEP_SUMMARY || true exit ${TEST_EXIT_CODE:-0} - # MI35x 8-GPU DeepSeek-V3.2 Accuracy Test - nightly-accuracy-8-gpu-mi35x-deepseek-v32: - if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-accuracy-8-gpu-mi35x-deepseek-v32,')) - runs-on: linux-mi35x-gpu-8 + # ============================================================================== + # 8-GPU Kimi-K2.6 (MI30x + MI35x) + # ============================================================================== + + nightly-8-gpu-kimi-k26: + if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-kimi-k26,')) + runs-on: linux-mi325-8gpu-sglang steps: - name: Checkout code uses: actions/checkout@v4 @@ -1186,94 +1106,18 @@ jobs: GITHUB_WORKSPACE: ${{ github.workspace }} - name: Install dependencies - run: | - bash scripts/ci/amd/amd_ci_install_dependency.sh - # Install tabulate for run_suite.py (missing in MI35x container) - bash scripts/ci/amd/amd_ci_exec.sh pip install tabulate + run: bash scripts/ci/amd/amd_ci_install_dependency.sh - - name: Accuracy Test MI35x (8-GPU DeepSeek-V3.2) + - name: Accuracy Test (8-GPU Kimi-K2.6) timeout-minutes: 120 run: | > github_summary.md # Clear summary file bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ - python3 run_suite.py --hw amd --suite nightly-amd-8-gpu-mi35x-deepseek-v32 --nightly --timeout-per-file 3600 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$? + python3 run_suite.py --hw amd --suite nightly-amd-accuracy-8-gpu-kimi-k26 --nightly --timeout-per-file 3600 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$? echo "$(> $GITHUB_STEP_SUMMARY || true exit ${TEST_EXIT_CODE:-0} - # MI35x 8-GPU DeepSeek-V3.2 TP+MTP Accuracy Test - nightly-accuracy-8-gpu-mi35x-deepseek-v32-mtp: - if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-accuracy-8-gpu-mi35x-deepseek-v32-mtp,')) - runs-on: linux-mi35x-gpu-8 - steps: - - name: Checkout code - uses: actions/checkout@v4 - with: - ref: ${{ inputs.ref || github.sha }} - - - name: Ensure VRAM is clear - run: bash scripts/ci/amd/ensure_vram_clear.sh rocm - - - name: Setup docker - run: | - touch github_summary.md - bash scripts/ci/amd/amd_ci_start_container.sh - env: - GITHUB_WORKSPACE: ${{ github.workspace }} - - - name: Install dependencies - run: | - bash scripts/ci/amd/amd_ci_install_dependency.sh - # Install tabulate for run_suite.py (missing in MI35x container) - bash scripts/ci/amd/amd_ci_exec.sh pip install tabulate - - - name: Accuracy Test MI35x (8-GPU DeepSeek-V3.2 TP+MTP) - timeout-minutes: 120 - run: | - > github_summary.md # Clear summary file - bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ - -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ - python3 run_suite.py --hw amd --suite nightly-amd-accuracy-8-gpu-mi35x-deepseek-v32-mtp --nightly --timeout-per-file 3600 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$? - echo "$(> $GITHUB_STEP_SUMMARY || true - exit ${TEST_EXIT_CODE:-0} - - # MI35x 8-GPU DeepSeek-V3.2 Performance Test (Basic) - nightly-perf-8-gpu-mi35x-deepseek-v32-basic: - if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-perf-8-gpu-mi35x-deepseek-v32-basic,')) - runs-on: linux-mi35x-gpu-8 - steps: - - name: Checkout code - uses: actions/checkout@v4 - with: - ref: ${{ inputs.ref || github.sha }} - - - name: Ensure VRAM is clear - run: bash scripts/ci/amd/ensure_vram_clear.sh rocm - - - name: Setup docker - run: | - touch github_summary.md - bash scripts/ci/amd/amd_ci_start_container.sh - env: - GITHUB_WORKSPACE: ${{ github.workspace }} - - - name: Install dependencies - run: | - bash scripts/ci/amd/amd_ci_install_dependency.sh - # Install tabulate for run_suite.py (missing in MI35x container) - bash scripts/ci/amd/amd_ci_exec.sh pip install tabulate - - - name: Performance Test MI35x (8-GPU DeepSeek-V3.2 Basic) - timeout-minutes: 150 - run: | - > github_summary.md # Clear summary file - bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ - -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ - python3 run_suite.py --hw amd --suite nightly-perf-8-gpu-mi35x-deepseek-v32-basic --nightly --timeout-per-file 5400 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$? - echo "$(> $GITHUB_STEP_SUMMARY || true - exit ${TEST_EXIT_CODE:-0} - - # MI35x 8-GPU Kimi-K2.6 (Accuracy) nightly-8-gpu-mi35x-kimi-k26: if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-mi35x-kimi-k26,')) runs-on: linux-mi35x-gpu-8 @@ -1309,7 +1153,42 @@ jobs: echo "$(> $GITHUB_STEP_SUMMARY || true exit ${TEST_EXIT_CODE:-0} - # MI35x 8-GPU Qwen3-235B-MXFP4 (Accuracy + Performance) + # ============================================================================== + # 8-GPU Qwen3-235B (MI30x + MI35x MXFP4) + # ============================================================================== + + nightly-8-gpu-qwen3-235b: + if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-qwen3-235b,')) + runs-on: linux-mi325-8gpu-sglang + steps: + - name: Checkout code + uses: actions/checkout@v4 + with: + ref: ${{ inputs.ref || github.sha }} + + - name: Ensure VRAM is clear + run: bash scripts/ci/amd/ensure_vram_clear.sh rocm + + - name: Setup docker + run: | + touch github_summary.md + bash scripts/ci/amd/amd_ci_start_container.sh + env: + GITHUB_WORKSPACE: ${{ github.workspace }} + + - name: Install dependencies + run: bash scripts/ci/amd/amd_ci_install_dependency.sh + + - name: Accuracy Test + Performance Test (8-GPU Qwen3) + timeout-minutes: 120 + run: | + > github_summary.md # Clear summary file + bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ + -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ + python3 run_suite.py --hw amd --suite nightly-8-gpu-qwen3-235b --nightly --timeout-per-file 3600 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$? + echo "$(> $GITHUB_STEP_SUMMARY || true + exit ${TEST_EXIT_CODE:-0} + nightly-8-gpu-mi35x-qwen3-235b-mxfp4: if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-mi35x-qwen3-235b-mxfp4,')) runs-on: linux-mi35x-gpu-8 @@ -1345,7 +1224,56 @@ jobs: echo "$(> $GITHUB_STEP_SUMMARY || true exit ${TEST_EXIT_CODE:-0} - # MI35x 8-GPU Qwen 3.5 (Accuracy + Performance combined) + # ============================================================================== + # 8-GPU Qwen 3.5 (MI30x + MI35x) + # ============================================================================== + + nightly-8-gpu-qwen35: + if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-qwen35,')) + runs-on: linux-mi325-8gpu-sglang + steps: + - name: Checkout code + uses: actions/checkout@v4 + with: + ref: ${{ inputs.ref || github.sha }} + + - name: Ensure VRAM is clear + run: bash scripts/ci/amd/ensure_vram_clear.sh rocm + + - name: Setup docker + run: | + touch github_summary.md + bash scripts/ci/amd/amd_ci_start_container.sh + env: + GITHUB_WORKSPACE: ${{ github.workspace }} + + - name: Install dependencies + run: | + bash scripts/ci/amd/amd_ci_install_dependency.sh + bash scripts/ci/amd/amd_ci_exec.sh pip install mistral-common "lm-eval[api]" + + - name: Accuracy Test (8-GPU Qwen 3.5) + timeout-minutes: 120 + run: | + > github_summary.md # Clear summary file + bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ + -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ + python3 run_suite.py --hw amd --suite nightly-amd-accuracy-8-gpu-qwen35 --nightly --timeout-per-file 3600 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$? + echo "$(> $GITHUB_STEP_SUMMARY || true + exit ${TEST_EXIT_CODE:-0} + + - name: Performance Test (8-GPU Qwen 3.5 FP8) + timeout-minutes: 120 + continue-on-error: true + run: | + > github_summary.md # Clear summary file + bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ + -e SGLANG_USE_AITER=1 \ + -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ + python3 run_suite.py --hw amd --suite nightly-perf-8-gpu-qwen35-fp8 --nightly --timeout-per-file 5400 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$? + echo "$(> $GITHUB_STEP_SUMMARY || true + exit ${TEST_EXIT_CODE:-0} + nightly-8-gpu-mi35x-qwen35: if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-mi35x-qwen35,')) runs-on: linux-mi35x-gpu-8 @@ -1393,7 +1321,56 @@ jobs: echo "$(> $GITHUB_STEP_SUMMARY || true exit ${TEST_EXIT_CODE:-0} - # MI35x 8-GPU GLM-5.1 (Accuracy + Performance combined) + # ============================================================================== + # 8-GPU GLM-5.1 (MI30x + MI35x) + # ============================================================================== + + nightly-8-gpu-glm51: + if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-glm51,')) + runs-on: linux-mi325-8gpu-sglang + steps: + - name: Checkout code + uses: actions/checkout@v4 + with: + ref: ${{ inputs.ref || github.sha }} + + - name: Ensure VRAM is clear + run: bash scripts/ci/amd/ensure_vram_clear.sh rocm + + - name: Setup docker + run: | + touch github_summary.md + bash scripts/ci/amd/amd_ci_start_container.sh + env: + GITHUB_WORKSPACE: ${{ github.workspace }} + + - name: Install dependencies + run: | + bash scripts/ci/amd/amd_ci_install_dependency.sh + bash scripts/ci/amd/amd_ci_exec.sh pip install git+https://github.com/huggingface/transformers.git@96f807a33b75 + + - name: Accuracy Test (8-GPU GLM-5.1 DSA) + timeout-minutes: 120 + run: | + > github_summary.md # Clear summary file + bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ + -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ + python3 run_suite.py --hw amd --suite nightly-amd-accuracy-8-gpu-glm51 --nightly --timeout-per-file 3600 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$? + echo "$(> $GITHUB_STEP_SUMMARY || true + exit ${TEST_EXIT_CODE:-0} + + - name: Performance Test (8-GPU GLM-5.1) + timeout-minutes: 120 + continue-on-error: true + run: | + > github_summary.md # Clear summary file + bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ + -e SGLANG_USE_AITER=1 \ + -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ + python3 run_suite.py --hw amd --suite nightly-perf-8-gpu-glm51 --nightly --timeout-per-file 5400 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$? + echo "$(> $GITHUB_STEP_SUMMARY || true + exit ${TEST_EXIT_CODE:-0} + nightly-8-gpu-mi35x-glm51: if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-mi35x-glm51,')) runs-on: linux-mi35x-gpu-8 @@ -1440,7 +1417,10 @@ jobs: echo "$(> $GITHUB_STEP_SUMMARY || true exit ${TEST_EXIT_CODE:-0} - # MI35x 8-GPU GLM-5-MXFP4 (Accuracy + Performance combined) + # ============================================================================== + # 8-GPU GLM-5-MXFP4 (MI35x only) + # ============================================================================== + nightly-8-gpu-mi35x-glm5-mxfp4: if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-mi35x-glm5-mxfp4,')) runs-on: linux-mi35x-gpu-8 @@ -1489,10 +1469,13 @@ jobs: echo "$(> $GITHUB_STEP_SUMMARY || true exit ${TEST_EXIT_CODE:-0} - # MI35x 8-GPU DeepSeek-V3.2 Performance Test (MTP) - nightly-perf-8-gpu-mi35x-deepseek-v32-mtp: - if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-perf-8-gpu-mi35x-deepseek-v32-mtp,')) - runs-on: linux-mi35x-gpu-8 + # ============================================================================== + # 8-GPU MiniMax-M2.7 (MI30x only) + # ============================================================================== + + nightly-8-gpu-minimax-m27: + if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-minimax-m27,')) + runs-on: linux-mi325-8gpu-sglang steps: - name: Checkout code uses: actions/checkout@v4 @@ -1510,67 +1493,129 @@ jobs: GITHUB_WORKSPACE: ${{ github.workspace }} - name: Install dependencies - run: | - bash scripts/ci/amd/amd_ci_install_dependency.sh - # Install tabulate for run_suite.py (missing in MI35x container) - bash scripts/ci/amd/amd_ci_exec.sh pip install tabulate + run: bash scripts/ci/amd/amd_ci_install_dependency.sh - - name: Performance Test MI35x (8-GPU DeepSeek-V3.2 MTP) - timeout-minutes: 180 + - name: Accuracy Test (8-GPU MiniMax-M2.7) + timeout-minutes: 120 run: | > github_summary.md # Clear summary file bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ + -e SGLANG_USE_AITER=1 \ -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ - python3 run_suite.py --hw amd --suite nightly-perf-8-gpu-mi35x-deepseek-v32-mtp --nightly --timeout-per-file 7200 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$? + python3 run_suite.py --hw amd --suite nightly-amd-accuracy-8-gpu-minimax-m27 --nightly --timeout-per-file 3600 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$? echo "$(> $GITHUB_STEP_SUMMARY || true exit ${TEST_EXIT_CODE:-0} + - name: Performance Test (8-GPU MiniMax-M2.7) + timeout-minutes: 120 + continue-on-error: true # Perf test failure doesn't fail the job if accuracy passed + run: | + > github_summary.md # Clear summary file + bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ + -e SGLANG_USE_AITER=1 \ + -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ + python3 run_suite.py --hw amd --suite nightly-perf-8-gpu-minimax-m27 --nightly --timeout-per-file 5400 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$? + echo "$(> $GITHUB_STEP_SUMMARY || true + exit ${TEST_EXIT_CODE:-0} + + # ============================================================================== + # Diffusion (MI30x) + # ============================================================================== + + nightly-1-gpu-zimage-turbo: + if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-1-gpu-zimage-turbo,')) + runs-on: linux-mi325-1gpu-sglang + steps: + - name: Checkout code + uses: actions/checkout@v4 + with: + ref: ${{ inputs.ref || github.sha }} + + - name: Ensure VRAM is clear + run: bash scripts/ci/amd/ensure_vram_clear.sh rocm + + - name: Setup docker + run: | + touch github_summary.md + bash scripts/ci/amd/amd_ci_start_container.sh + env: + GITHUB_WORKSPACE: ${{ github.workspace }} + + - name: Install dependencies + run: bash scripts/ci/amd/amd_ci_install_dependency.sh + + - name: Z-Image-Turbo Diffusion Test (1-GPU) + timeout-minutes: 45 + run: | + bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout \ + -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ + -e SGLANG_DIFFUSION_ARTIFACT_DIR="/sglang-checkout/diffusion-artifacts" \ + pytest test/registered/amd/test_zimage_turbo.py -v -s ${{ inputs.continue_on_error && '|| true' || '' }} || TEST_EXIT_CODE=$? + echo "$(> $GITHUB_STEP_SUMMARY || true + exit ${TEST_EXIT_CODE:-0} + + - name: Upload generated images + if: always() + uses: actions/upload-artifact@v4 + with: + name: zimage-turbo-outputs + path: diffusion-artifacts/ + if-no-files-found: ignore + retention-days: 30 + check-all-jobs: if: always() && (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request' || github.event_name == 'workflow_dispatch') needs: - # MI30x Unit Tests + # 1-GPU Unit Tests (MI30x + MI35x) - nightly-test-1-gpu-unit - # MI30x Accuracy Tests + - nightly-test-1-gpu-mi35x + # 2-GPU and 4-GPU Tests (MI30x) - nightly-accuracy-2-gpu - nightly-accuracy-2-gpu-vlm - # MI30x 4-GPU Tests + # - nightly-perf-2-gpu-text # excluded: perf failures don't block CI + # - nightly-perf-2-gpu-vlm # excluded: perf failures don't block CI - nightly-4-gpu + # 8-GPU GPT-OSS (MI30x mixes Grok1-FP8; MI35x mixes Qwen3-Coder-Next) - nightly-accuracy-8-gpu - # MI30x Performance Tests - excluded from check (perf failures don't block CI) - # - nightly-perf-2-gpu-text - # - nightly-perf-2-gpu-vlm - # MI30x Combined Accuracy + Performance Tests + - nightly-accuracy-8-gpu-mi35x + # 8-GPU Grok1-INT4 (MI30x + MI35x) - nightly-8-gpu-grok1-int4 + - nightly-8-gpu-mi35x-grok1-int4 + # 8-GPU Grok2 (MI30x + MI35x) - nightly-8-gpu-grok2 + - nightly-8-gpu-mi35x-grok2 + # 8-GPU DeepSeek-V3.x (MI30x) - nightly-8-gpu-deepseek-v31 - nightly-8-gpu-deepseek-v32 - nightly-8-gpu-deepseek-v32-mtp - nightly-8-gpu-deepseek-v3-kv-fp8 - - nightly-8-gpu-kimi-k26 - - nightly-8-gpu-qwen3-235b - - nightly-8-gpu-qwen35 - - nightly-8-gpu-glm51 - - nightly-8-gpu-minimax-m27 - # MI30x Diffusion Tests - - nightly-1-gpu-zimage-turbo - # MI35x jobs - - nightly-test-1-gpu-mi35x - - nightly-accuracy-8-gpu-mi35x - - nightly-8-gpu-mi35x-grok1-int4 - - nightly-8-gpu-mi35x-grok2 + # 8-GPU DeepSeek-V3.2 (MI35x) + - nightly-accuracy-8-gpu-mi35x-deepseek-v32 + - nightly-accuracy-8-gpu-mi35x-deepseek-v32-mtp + # - nightly-perf-8-gpu-mi35x-deepseek-v32-basic # excluded: perf failures don't block CI + # - nightly-perf-8-gpu-mi35x-deepseek-v32-mtp # excluded: perf failures don't block CI + # 8-GPU DeepSeek-R1-MXFP4 (MI35x only) - nightly-8-gpu-mi35x-deepseek-r1-mxfp4 - nightly-8-gpu-mi35x-deepseek-r1-mxfp4-kv-fp8 - nightly-8-gpu-mi35x-deepseek-r1-mxfp4-ar-fusion - - nightly-accuracy-8-gpu-mi35x-deepseek-v32 - - nightly-accuracy-8-gpu-mi35x-deepseek-v32-mtp + # 8-GPU Kimi-K2.6 (MI30x + MI35x) + - nightly-8-gpu-kimi-k26 - nightly-8-gpu-mi35x-kimi-k26 + # 8-GPU Qwen3-235B (MI30x + MI35x MXFP4) + - nightly-8-gpu-qwen3-235b - nightly-8-gpu-mi35x-qwen3-235b-mxfp4 + # 8-GPU Qwen 3.5 (MI30x + MI35x) + - nightly-8-gpu-qwen35 - nightly-8-gpu-mi35x-qwen35 + # 8-GPU GLM-5.1 (MI30x + MI35x) + - nightly-8-gpu-glm51 - nightly-8-gpu-mi35x-glm51 + # 8-GPU GLM-5-MXFP4 (MI35x only) - nightly-8-gpu-mi35x-glm5-mxfp4 - # MI35x perf jobs excluded from check - perf failures don't block CI - # - nightly-perf-8-gpu-mi35x-deepseek-v32-basic - # - nightly-perf-8-gpu-mi35x-deepseek-v32-mtp + # 8-GPU MiniMax-M2.7 (MI30x only) + - nightly-8-gpu-minimax-m27 + # Diffusion (MI30x) + - nightly-1-gpu-zimage-turbo runs-on: ubuntu-latest steps: - name: Check if any job failed diff --git a/.github/workflows/pr-test-amd-rocm720.yml b/.github/workflows/pr-test-amd-rocm720.yml index 319e41e04..88c8320a9 100644 --- a/.github/workflows/pr-test-amd-rocm720.yml +++ b/.github/workflows/pr-test-amd-rocm720.yml @@ -46,7 +46,7 @@ on: - multimodal-gen-test-2-gpu-amd-rocm720 - stage-c-test-large-8-gpu-amd-rocm720 - stage-c-test-large-8-gpu-amd-mi35x-rocm720 - - stage-b-test-large-8-gpu-disaggregation-amd-rocm720 + - stage-b-test-large-8-gpu-mi35x-disaggregation-amd-rocm720 - stage-c-test-4-gpu-amd-rocm720 target_stage: description: "Or type comma-separated stage names (overrides dropdown if non-empty)" @@ -68,6 +68,11 @@ on: required: false type: boolean default: true + run_all_tests: + description: 'Run all tests (skip change detection). Ignored when target_stage / target_stage_select is set.' + required: false + type: boolean + default: false workflow_call: inputs: ref: @@ -116,6 +121,7 @@ jobs: sgl_kernel: ${{ steps.filter.outputs.sgl_kernel || steps.run-mode.outputs.run_all_tests }} jit_kernel: ${{ steps.filter.outputs.jit_kernel || steps.run-mode.outputs.run_all_tests }} multimodal_gen: ${{ steps.filter.outputs.multimodal_gen || steps.run-mode.outputs.run_all_tests }} + continue_on_error: ${{ steps.set-continue-on-error.outputs.continue_on_error }} steps: - name: Checkout code uses: actions/checkout@v4 @@ -135,6 +141,23 @@ jobs: echo "Run mode: FILTERED (triggered by ${{ github.event_name }})" fi + - name: Set continue-on-error for schedule/full runs + id: set-continue-on-error + run: | + # Mirror pr-test-amd.yml: continue-on-error when any of + # - run_all_tests was requested (workflow_dispatch checkbox or workflow_call input), + # - inputs.continue_on_error was explicitly set, + # - or this is a scheduled run. + # The schedule check is included explicitly here because (unlike pr-test-amd.yml's + # run-mode) the rocm720 run-mode does not roll schedule into run_all_tests. + if [[ "${{ steps.run-mode.outputs.run_all_tests }}" == "true" || "${{ inputs.continue_on_error }}" == "true" || "${{ github.event_name }}" == "schedule" ]]; then + echo "continue_on_error=true" >> $GITHUB_OUTPUT + echo "Continue-on-error: ENABLED (run_all_tests=${{ steps.run-mode.outputs.run_all_tests }}, input=${{ inputs.continue_on_error }}, event=${{ github.event_name }})" + else + echo "continue_on_error=false" >> $GITHUB_OUTPUT + echo "Continue-on-error: DISABLED" + fi + - name: Detect file changes id: filter uses: dorny/paths-filter@v3 @@ -289,7 +312,7 @@ jobs: - name: Run test timeout-minutes: 10 run: | - bash scripts/ci/amd/amd_ci_exec.sh -w "/sglang-checkout/test" python3 run_suite.py --hw amd --suite stage-a-test-1-gpu-small-amd ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} + bash scripts/ci/amd/amd_ci_exec.sh -w "/sglang-checkout/test" python3 run_suite.py --hw amd --suite stage-a-test-1-gpu-small-amd ${{ needs.check-changes.outputs.continue_on_error == 'true' && '--continue-on-error' || '' }} jit-kernel-unit-test-amd-rocm720: needs: [check-changes] @@ -366,7 +389,7 @@ jobs: - name: Run test timeout-minutes: 30 run: | - bash scripts/ci/amd/amd_ci_exec.sh -w "/sglang-checkout/test" python3 run_suite.py --hw amd --suite stage-b-test-1-gpu-small-amd --auto-partition-id ${{ matrix.part }} --auto-partition-size 14 --timeout-per-file 1800 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} + bash scripts/ci/amd/amd_ci_exec.sh -w "/sglang-checkout/test" python3 run_suite.py --hw amd --suite stage-b-test-1-gpu-small-amd --auto-partition-id ${{ matrix.part }} --auto-partition-size 14 --timeout-per-file 1800 ${{ needs.check-changes.outputs.continue_on_error == 'true' && '--continue-on-error' || '' }} stage-b-test-1-gpu-small-amd-nondeterministic-rocm720: needs: [check-changes] @@ -404,7 +427,7 @@ jobs: - name: Run test timeout-minutes: 30 run: | - bash scripts/ci/amd/amd_ci_exec.sh -w "/sglang-checkout/test" python3 run_suite.py --hw amd --suite stage-b-test-1-gpu-small-amd-nondeterministic --timeout-per-file 1800 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} + bash scripts/ci/amd/amd_ci_exec.sh -w "/sglang-checkout/test" python3 run_suite.py --hw amd --suite stage-b-test-1-gpu-small-amd-nondeterministic --timeout-per-file 1800 ${{ needs.check-changes.outputs.continue_on_error == 'true' && '--continue-on-error' || '' }} stage-b-test-1-gpu-small-amd-mi35x-rocm720: needs: [check-changes] @@ -442,7 +465,7 @@ jobs: - name: Run test timeout-minutes: 30 run: | - bash scripts/ci/amd/amd_ci_exec.sh -w "/sglang-checkout/test" python3 run_suite.py --hw amd --suite stage-b-test-1-gpu-small-amd-mi35x ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} + bash scripts/ci/amd/amd_ci_exec.sh -w "/sglang-checkout/test" python3 run_suite.py --hw amd --suite stage-b-test-1-gpu-small-amd-mi35x ${{ needs.check-changes.outputs.continue_on_error == 'true' && '--continue-on-error' || '' }} stage-b-test-1-gpu-large-amd-rocm720: needs: [check-changes] @@ -481,7 +504,7 @@ jobs: - name: Run test timeout-minutes: 30 run: | - bash scripts/ci/amd/amd_ci_exec.sh -w "/sglang-checkout/test" python3 run_suite.py --hw amd --suite stage-b-test-1-gpu-large-amd --auto-partition-id ${{ matrix.part }} --auto-partition-size 2 --timeout-per-file 1800 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} + bash scripts/ci/amd/amd_ci_exec.sh -w "/sglang-checkout/test" python3 run_suite.py --hw amd --suite stage-b-test-1-gpu-large-amd --auto-partition-id ${{ matrix.part }} --auto-partition-size 2 --timeout-per-file 1800 ${{ needs.check-changes.outputs.continue_on_error == 'true' && '--continue-on-error' || '' }} stage-b-test-2-gpu-large-amd-rocm720: needs: [check-changes] @@ -520,7 +543,7 @@ jobs: - name: Run test timeout-minutes: 30 run: | - bash scripts/ci/amd/amd_ci_exec.sh -w "/sglang-checkout/test" python3 run_suite.py --hw amd --suite stage-b-test-2-gpu-large-amd --auto-partition-id ${{ matrix.part }} --auto-partition-size 2 --timeout-per-file 1800 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} + bash scripts/ci/amd/amd_ci_exec.sh -w "/sglang-checkout/test" python3 run_suite.py --hw amd --suite stage-b-test-2-gpu-large-amd --auto-partition-id ${{ matrix.part }} --auto-partition-size 2 --timeout-per-file 1800 ${{ needs.check-changes.outputs.continue_on_error == 'true' && '--continue-on-error' || '' }} multimodal-gen-test-1-gpu-amd-rocm720: needs: [check-changes] @@ -828,7 +851,7 @@ jobs: ref: ${{ inputs.pr_head_sha || inputs.ref || github.sha }} - name: Ensure VRAM is clear - run: bash scripts/ensure_vram_clear.sh rocm + run: bash scripts/ci/amd/ensure_vram_clear.sh rocm - name: Start CI container run: bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version rocm720 @@ -857,7 +880,7 @@ jobs: --max-attempts 2 \ --retry-wait-seconds 120 \ --retry-timeout-increase 0 \ - ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} + ${{ needs.check-changes.outputs.continue_on_error == 'true' && '--continue-on-error' || '' }} stage-c-test-large-8-gpu-amd-rocm720: needs: [check-changes] @@ -904,7 +927,7 @@ jobs: - name: Run test timeout-minutes: 120 run: | - bash scripts/ci/amd/amd_ci_exec.sh -w "/sglang-checkout/test" python3 run_suite.py --hw amd --suite stage-c-test-large-8-gpu-amd --auto-partition-id ${{ matrix.part }} --auto-partition-size 3 --timeout-per-file 5400 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} + bash scripts/ci/amd/amd_ci_exec.sh -w "/sglang-checkout/test" python3 run_suite.py --hw amd --suite stage-c-test-large-8-gpu-amd --auto-partition-id ${{ matrix.part }} --auto-partition-size 3 --timeout-per-file 5400 ${{ needs.check-changes.outputs.continue_on_error == 'true' && '--continue-on-error' || '' }} stage-c-test-large-8-gpu-amd-mi35x-rocm720: needs: [check-changes] @@ -943,15 +966,15 @@ jobs: - name: Run test timeout-minutes: 60 run: | - bash scripts/ci/amd/amd_ci_exec.sh -w "/sglang-checkout/test" python3 run_suite.py --hw amd --suite stage-c-test-large-8-gpu-amd-mi35x --auto-partition-id ${{ matrix.part }} --auto-partition-size 2 --timeout-per-file 3600 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} + bash scripts/ci/amd/amd_ci_exec.sh -w "/sglang-checkout/test" python3 run_suite.py --hw amd --suite stage-c-test-large-8-gpu-amd-mi35x --auto-partition-id ${{ matrix.part }} --auto-partition-size 2 --timeout-per-file 3600 ${{ needs.check-changes.outputs.continue_on_error == 'true' && '--continue-on-error' || '' }} # =============================================== Disaggregation ==================================================== - stage-b-test-large-8-gpu-35x-disaggregation-amd-rocm720: + stage-b-test-large-8-gpu-mi35x-disaggregation-amd-rocm720: needs: [check-changes] if: | always() && ( - (contains(format(',{0},', inputs.target_stage || inputs.target_stage_select), ',stage-b-test-large-8-gpu-disaggregation-amd-rocm720,')) || + (contains(format(',{0},', inputs.target_stage || inputs.target_stage_select), ',stage-b-test-large-8-gpu-mi35x-disaggregation-amd-rocm720,')) || ( !(inputs.target_stage || inputs.target_stage_select) && (!failure() && !cancelled()) && @@ -1055,7 +1078,7 @@ jobs: run: | bash scripts/ci/amd/amd_ci_exec.sh \ -e SGLANG_TEST_RDMA_DEVICE="${{ env.SGLANG_TEST_RDMA_DEVICE }}" \ - -w "/sglang-checkout/test" python3 run_suite.py --hw amd --suite stage-b-test-large-8-gpu-35x-disaggregation-amd --timeout-per-file 1800 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} + -w "/sglang-checkout/test" python3 run_suite.py --hw amd --suite stage-b-test-large-8-gpu-mi35x-disaggregation-amd --timeout-per-file 1800 ${{ needs.check-changes.outputs.continue_on_error == 'true' && '--continue-on-error' || '' }} pr-test-amd-rocm720-finish: needs: @@ -1075,7 +1098,7 @@ jobs: stage-b-test-1-gpu-small-amd-mi35x-rocm720, stage-b-test-1-gpu-large-amd-rocm720, stage-b-test-2-gpu-large-amd-rocm720, - stage-b-test-large-8-gpu-35x-disaggregation-amd-rocm720, + stage-b-test-large-8-gpu-mi35x-disaggregation-amd-rocm720, stage-c-test-4-gpu-amd-rocm720, stage-c-test-large-8-gpu-amd-rocm720, stage-c-test-large-8-gpu-amd-mi35x-rocm720, diff --git a/.github/workflows/pr-test-amd.yml b/.github/workflows/pr-test-amd.yml index 9e46a5b41..e596adc8f 100644 --- a/.github/workflows/pr-test-amd.yml +++ b/.github/workflows/pr-test-amd.yml @@ -38,7 +38,7 @@ on: - stage-c-test-4-gpu-amd - stage-c-test-large-8-gpu-amd - stage-c-test-large-8-gpu-amd-mi35x - - stage-b-test-large-8-gpu-35x-disaggregation-amd + - stage-b-test-large-8-gpu-mi35x-disaggregation-amd target_stage: description: "Or type comma-separated stage names (overrides dropdown if non-empty)" required: false @@ -67,6 +67,11 @@ on: options: - mi300 - mi325 + run_all_tests: + description: 'Run all tests (skip change detection). Ignored when target_stage / target_stage_select is set.' + required: false + type: boolean + default: false workflow_call: inputs: ref: @@ -1028,12 +1033,12 @@ jobs: bash scripts/ci/amd/amd_ci_exec.sh -w "/sglang-checkout/test" python3 run_suite.py --hw amd --suite stage-c-test-large-8-gpu-amd-mi35x --auto-partition-id ${{ matrix.part }} --auto-partition-size 2 --timeout-per-file 3600 ${{ needs.check-changes.outputs.continue_on_error == 'true' && '--continue-on-error' || '' }} # =============================================== Disaggregation ==================================================== - stage-b-test-large-8-gpu-35x-disaggregation-amd: + stage-b-test-large-8-gpu-mi35x-disaggregation-amd: needs: [check-changes, wait-for-stage-a-amd] if: | always() && ( - (contains(format(',{0},', inputs.target_stage || inputs.target_stage_select), ',stage-b-test-large-8-gpu-35x-disaggregation-amd,')) || + (contains(format(',{0},', inputs.target_stage || inputs.target_stage_select), ',stage-b-test-large-8-gpu-mi35x-disaggregation-amd,')) || ( !(inputs.target_stage || inputs.target_stage_select) && ((github.event_name == 'schedule') || (!failure() && !cancelled())) && @@ -1137,7 +1142,7 @@ jobs: run: | bash scripts/ci/amd/amd_ci_exec.sh \ -e SGLANG_TEST_RDMA_DEVICE="${{ env.SGLANG_TEST_RDMA_DEVICE }}" \ - -w "/sglang-checkout/test" python3 run_suite.py --hw amd --suite stage-b-test-large-8-gpu-35x-disaggregation-amd --timeout-per-file 1800 ${{ needs.check-changes.outputs.continue_on_error == 'true' && '--continue-on-error' || '' }} + -w "/sglang-checkout/test" python3 run_suite.py --hw amd --suite stage-b-test-large-8-gpu-mi35x-disaggregation-amd --timeout-per-file 1800 ${{ needs.check-changes.outputs.continue_on_error == 'true' && '--continue-on-error' || '' }} pr-test-amd-finish: needs: @@ -1159,7 +1164,7 @@ jobs: stage-b-test-1-gpu-small-amd-mi35x, stage-b-test-1-gpu-large-amd, stage-b-test-2-gpu-large-amd, - stage-b-test-large-8-gpu-35x-disaggregation-amd, + stage-b-test-large-8-gpu-mi35x-disaggregation-amd, stage-c-test-4-gpu-amd, stage-c-test-large-8-gpu-amd, stage-c-test-large-8-gpu-amd-mi35x, diff --git a/test/registered/amd/accuracy/mi30x/test_glm51_eval_amd.py b/test/registered/amd/accuracy/mi30x/test_glm51_eval_amd.py index 531d97c38..ee5948a10 100644 --- a/test/registered/amd/accuracy/mi30x/test_glm51_eval_amd.py +++ b/test/registered/amd/accuracy/mi30x/test_glm51_eval_amd.py @@ -161,7 +161,7 @@ class TestGLM51EvalAMD(unittest.TestCase): def setUpClass(cls): cls.models = GLM51_MODELS cls.base_url = DEFAULT_URL_FOR_TEST - cls.num_questions = int(os.environ.get("GSM8K_NUM_QUESTIONS", "200")) + cls.num_questions = int(os.environ.get("GSM8K_NUM_QUESTIONS", "1319")) def test_glm51_accuracy(self): all_results = [] diff --git a/test/registered/amd/accuracy/mi35x/test_glm51_eval_mi35x.py b/test/registered/amd/accuracy/mi35x/test_glm51_eval_mi35x.py index 907df4f49..09ac35eb0 100644 --- a/test/registered/amd/accuracy/mi35x/test_glm51_eval_mi35x.py +++ b/test/registered/amd/accuracy/mi35x/test_glm51_eval_mi35x.py @@ -165,7 +165,7 @@ class TestGLM51EvalMI35x(unittest.TestCase): def setUpClass(cls): cls.models = MI35X_GLM51_MODELS cls.base_url = DEFAULT_URL_FOR_TEST - cls.num_questions = int(os.environ.get("GSM8K_NUM_QUESTIONS", "200")) + cls.num_questions = int(os.environ.get("GSM8K_NUM_QUESTIONS", "1319")) def test_glm51_accuracy(self): all_results = [] diff --git a/test/registered/amd/accuracy/mi35x/test_glm5_mxfp4_eval_mi35x.py b/test/registered/amd/accuracy/mi35x/test_glm5_mxfp4_eval_mi35x.py index 856d8c4a4..e6cb0f659 100644 --- a/test/registered/amd/accuracy/mi35x/test_glm5_mxfp4_eval_mi35x.py +++ b/test/registered/amd/accuracy/mi35x/test_glm5_mxfp4_eval_mi35x.py @@ -191,7 +191,7 @@ class TestGLM5MXFP4EvalMI35x(unittest.TestCase): def setUpClass(cls): cls.models = get_glm5_mxfp4_models() cls.base_url = DEFAULT_URL_FOR_TEST - cls.num_questions = int(os.environ.get("GSM8K_NUM_QUESTIONS", "200")) + cls.num_questions = int(os.environ.get("GSM8K_NUM_QUESTIONS", "1319")) def test_glm5_mxfp4_accuracy(self): """Test GLM-5-MXFP4 with GSM8K completion benchmark.""" diff --git a/test/registered/amd/disaggregation/test_disaggregation_basic.py b/test/registered/amd/disaggregation/test_disaggregation_basic.py index 123fee945..8e00d2ba1 100644 --- a/test/registered/amd/disaggregation/test_disaggregation_basic.py +++ b/test/registered/amd/disaggregation/test_disaggregation_basic.py @@ -18,7 +18,7 @@ from sglang.test.test_utils import ( popen_launch_pd_server, ) -register_amd_ci(est_time=600, suite="stage-b-test-large-8-gpu-35x-disaggregation-amd") +register_amd_ci(est_time=600, suite="stage-b-test-large-8-gpu-mi35x-disaggregation-amd") class TestDisaggregationAccuracy(PDDisaggregationServerBase): diff --git a/test/registered/amd/disaggregation/test_disaggregation_pp.py b/test/registered/amd/disaggregation/test_disaggregation_pp.py index 829d4dd71..7f0d9ca87 100644 --- a/test/registered/amd/disaggregation/test_disaggregation_pp.py +++ b/test/registered/amd/disaggregation/test_disaggregation_pp.py @@ -15,7 +15,7 @@ from sglang.test.test_utils import ( try_cached_model, ) -register_amd_ci(est_time=600, suite="stage-b-test-large-8-gpu-35x-disaggregation-amd") +register_amd_ci(est_time=600, suite="stage-b-test-large-8-gpu-mi35x-disaggregation-amd") class TestDisaggregationPrefillPPAccuracy(PDDisaggregationServerBase): diff --git a/test/registered/amd/disaggregation/test_mori_transfer_engine_e2e.py b/test/registered/amd/disaggregation/test_mori_transfer_engine_e2e.py index 3a9e1d459..3f9626872 100644 --- a/test/registered/amd/disaggregation/test_mori_transfer_engine_e2e.py +++ b/test/registered/amd/disaggregation/test_mori_transfer_engine_e2e.py @@ -14,7 +14,7 @@ from sglang.test.test_utils import ( try_cached_model, ) -register_amd_ci(est_time=300, suite="stage-b-test-large-8-gpu-35x-disaggregation-amd") +register_amd_ci(est_time=300, suite="stage-b-test-large-8-gpu-mi35x-disaggregation-amd") class MoriTransferEngineBase(PDDisaggregationServerBase): diff --git a/test/run_suite.py b/test/run_suite.py index ceff2a2b7..18da6c9fb 100644 --- a/test/run_suite.py +++ b/test/run_suite.py @@ -33,7 +33,7 @@ PER_COMMIT_SUITES = { "stage-b-test-1-gpu-small-amd", "stage-b-test-1-gpu-small-amd-nondeterministic", "stage-b-test-1-gpu-small-amd-mi35x", - "stage-b-test-large-8-gpu-35x-disaggregation-amd", + "stage-b-test-large-8-gpu-mi35x-disaggregation-amd", "stage-b-test-1-gpu-large-amd", "stage-b-test-2-gpu-large-amd", "jit-kernel-unit-test-amd",