[AMD] Add 4-GPU test suite for MI325 runners (#20294)

2026-03-24 14:04:49 -07:00
parent c4db64c16b
commit 6cb1c2d53d
12 changed files with 252 additions and 13 deletions
@@ -32,6 +32,7 @@ on:
          - nightly-accuracy-2-gpu-vlm-rocm720
          - nightly-perf-2-gpu-text-rocm720
          - nightly-perf-2-gpu-vlm-rocm720
+          - nightly-4-gpu-rocm720
          - nightly-accuracy-8-gpu-rocm720
          - nightly-8-gpu-grok1-int4-rocm720
          - nightly-8-gpu-grok2-rocm720
@@ -247,6 +248,37 @@ jobs:
          echo "$(<github_summary.md )" >> $GITHUB_STEP_SUMMARY || true
          exit ${TEST_EXIT_CODE:-0}

+  # ============================================== MI30x ROCm 7.2 4-GPU Tests ==============================================
+  # 4-GPU Nightly Tests - Dumper/Comparator E2E, VLM Encoder DP (ROCm 7.2)
+  nightly-4-gpu-rocm720:
+    if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-4-gpu-rocm720,'))
+    runs-on: linux-mi325-4gpu-sglang
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.ref || github.ref }}
+
+      - name: Setup docker (ROCm 7.2)
+        run: |
+          touch github_summary.md
+          bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version rocm720
+        env:
+          GITHUB_WORKSPACE: ${{ github.workspace }}
+
+      - name: Install dependencies
+        run: bash scripts/ci/amd/amd_ci_install_dependency.sh
+
+      - name: Nightly Test ROCm 7.2 (4-GPU)
+        timeout-minutes: 120
+        run: |
+          > github_summary.md
+          bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \
+            -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \
+            python3 run_suite.py --hw amd --suite nightly-amd-4-gpu --nightly --continue-on-error --timeout-per-file 3600 || TEST_EXIT_CODE=$?
+          echo "$(<github_summary.md )" >> $GITHUB_STEP_SUMMARY || true
+          exit ${TEST_EXIT_CODE:-0}
+
  # 8-GPU Accuracy Tests - GPT-OSS, Grok1-FP8 (ROCm 7.2)
  nightly-accuracy-8-gpu-rocm720:
    if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-accuracy-8-gpu-rocm720,'))
@@ -1317,6 +1349,8 @@ jobs:
      # MI30x ROCm 7.2 Performance Tests
      - nightly-perf-2-gpu-text-rocm720
      - nightly-perf-2-gpu-vlm-rocm720
+      # MI30x ROCm 7.2 4-GPU Tests
+      - nightly-4-gpu-rocm720
      - nightly-accuracy-8-gpu-rocm720
      # MI30x ROCm 7.2 Combined Accuracy + Performance Tests
      - nightly-8-gpu-grok1-int4-rocm720
@@ -32,6 +32,7 @@ on:
          - nightly-accuracy-2-gpu-vlm
          - nightly-perf-2-gpu-text
          - nightly-perf-2-gpu-vlm
+          - nightly-4-gpu
          - nightly-accuracy-8-gpu
          - nightly-8-gpu-grok1-int4
          - nightly-8-gpu-grok2
@@ -252,6 +253,37 @@ jobs:
          echo "$(<github_summary.md )" >> $GITHUB_STEP_SUMMARY || true
          exit ${TEST_EXIT_CODE:-0}

+  # ============================================== MI30x 4-GPU Tests ==============================================
+  # 4-GPU Nightly Tests - Dumper/Comparator E2E, VLM Encoder DP
+  nightly-4-gpu:
+    if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-4-gpu,'))
+    runs-on: linux-mi325-4gpu-sglang
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.ref || github.ref }}
+
+      - name: Setup docker
+        run: |
+          touch github_summary.md
+          bash scripts/ci/amd/amd_ci_start_container.sh
+        env:
+          GITHUB_WORKSPACE: ${{ github.workspace }}
+
+      - name: Install dependencies
+        run: bash scripts/ci/amd/amd_ci_install_dependency.sh
+
+      - name: Nightly Test (4-GPU)
+        timeout-minutes: 120
+        run: |
+          > github_summary.md
+          bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \
+            -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \
+            python3 run_suite.py --hw amd --suite nightly-amd-4-gpu --nightly --continue-on-error --timeout-per-file 3600 || TEST_EXIT_CODE=$?
+          echo "$(<github_summary.md )" >> $GITHUB_STEP_SUMMARY || true
+          exit ${TEST_EXIT_CODE:-0}
+
  # 8-GPU Accuracy Tests - GPT-OSS, Grok1-FP8 (accuracy only)
  nightly-accuracy-8-gpu:
    if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-accuracy-8-gpu,'))
@@ -1320,6 +1352,8 @@ jobs:
      # MI30x Accuracy Tests
      - nightly-accuracy-2-gpu
      - nightly-accuracy-2-gpu-vlm
+      # MI30x 4-GPU Tests
+      - nightly-4-gpu
      - nightly-accuracy-8-gpu
      # MI30x Performance Tests - excluded from check (perf failures don't block CI)
      # - nightly-perf-2-gpu-text
@@ -47,6 +47,7 @@ on:
          - stage-c-test-large-8-gpu-amd
          - stage-c-test-large-8-gpu-amd-mi35x
          - stage-b-test-large-8-gpu-disaggregation-amd
+          - stage-c-test-4-gpu-amd
      target_stage:
        description: "Or type comma-separated stage names (overrides dropdown if non-empty)"
        required: false
@@ -775,6 +776,62 @@ jobs:
          free -h


+  stage-c-test-4-gpu-amd:
+    needs: [check-changes, stage-b-test-small-1-gpu-amd, stage-b-test-large-2-gpu-amd]
+    if: |
+      always() &&
+      (
+        (contains(format(',{0},', inputs.target_stage || inputs.target_stage_select), ',stage-c-test-4-gpu-amd,')) ||
+        (
+          !(inputs.target_stage || inputs.target_stage_select) &&
+          (!failure() && !cancelled()) &&
+          ((needs.check-changes.outputs.main_package == 'true') || (needs.check-changes.outputs.sgl_kernel == 'true'))
+        )
+      )
+    strategy:
+      fail-fast: false
+      matrix:
+        runner: [linux-mi325-4gpu-sglang]
+        part: [0, 1]
+    runs-on: ${{matrix.runner}}
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.pr_head_sha || inputs.ref || github.sha }}
+
+      - name: Ensure VRAM is clear
+        run: bash scripts/ensure_vram_clear.sh rocm
+
+      - name: Start CI container
+        run: bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version rocm720
+        env:
+          GITHUB_WORKSPACE: ${{ github.workspace }}
+
+      - name: Install dependencies
+        run: bash scripts/ci/amd/amd_ci_install_dependency.sh
+
+      - name: Run test
+        timeout-minutes: 60
+        run: |
+          bash scripts/ci/amd/amd_ci_exec.sh \
+            -e NCCL_CUMEM_ENABLE=0 \
+            -e NCCL_NVLS_ENABLE=0 \
+            -e RCCL_MSCCL_ENABLE=0 \
+            -e SGLANG_USE_ROCM700A=1 \
+            -w "/sglang-checkout/test" \
+            python3 run_suite.py \
+              --hw amd \
+              --suite stage-c-test-4-gpu-amd \
+              --auto-partition-id ${{ matrix.part }} \
+              --auto-partition-size 2 \
+              --timeout-per-file 1800 \
+              --enable-retry \
+              --max-attempts 2 \
+              --retry-wait-seconds 120 \
+              --retry-timeout-increase 0 \
+              ${{ inputs.continue_on_error && '--continue-on-error' || '' }}
+
  stage-c-test-large-8-gpu-amd:
    needs: [check-changes]
    if: |
@@ -992,6 +1049,7 @@ jobs:
        stage-b-test-1-gpu-large-amd,
        stage-b-test-2-gpu-large-amd,
        stage-b-test-large-8-gpu-35x-disaggregation-amd,
+        stage-c-test-4-gpu-amd,
        stage-c-test-large-8-gpu-amd,
        stage-c-test-large-8-gpu-amd-mi35x,
      ]
@@ -42,6 +42,7 @@ on:
          - stage-b-test-2-gpu-large-amd
          - multimodal-gen-test-1-gpu-amd
          - multimodal-gen-test-2-gpu-amd
+          - stage-c-test-4-gpu-amd
          - stage-c-test-large-8-gpu-amd
          - stage-c-test-large-8-gpu-amd-mi35x
          - stage-b-test-large-8-gpu-disaggregation-amd
@@ -778,6 +779,62 @@ jobs:
          free -h


+  stage-c-test-4-gpu-amd:
+    needs: [check-changes, call-gate, stage-b-test-small-1-gpu-amd, stage-b-test-large-2-gpu-amd]
+    if: |
+      always() &&
+      (
+        (contains(format(',{0},', inputs.target_stage || inputs.target_stage_select), ',stage-c-test-4-gpu-amd,')) ||
+        (
+          !(inputs.target_stage || inputs.target_stage_select) &&
+          (!failure() && !cancelled()) &&
+          ((needs.check-changes.outputs.main_package == 'true') || (needs.check-changes.outputs.sgl_kernel == 'true'))
+        )
+      )
+    strategy:
+      fail-fast: false
+      matrix:
+        runner: [linux-mi325-4gpu-sglang]
+        part: [0, 1]
+    runs-on: ${{matrix.runner}}
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.pr_head_sha || inputs.ref || github.sha }}
+
+      - name: Ensure VRAM is clear
+        run: bash scripts/ensure_vram_clear.sh rocm
+
+      - name: Start CI container
+        run: bash scripts/ci/amd/amd_ci_start_container.sh
+        env:
+          GITHUB_WORKSPACE: ${{ github.workspace }}
+
+      - name: Install dependencies
+        run: bash scripts/ci/amd/amd_ci_install_dependency.sh
+
+      - name: Run test
+        timeout-minutes: 60
+        run: |
+          bash scripts/ci/amd/amd_ci_exec.sh \
+            -e NCCL_CUMEM_ENABLE=0 \
+            -e NCCL_NVLS_ENABLE=0 \
+            -e RCCL_MSCCL_ENABLE=0 \
+            -e SGLANG_USE_ROCM700A=1 \
+            -w "/sglang-checkout/test" \
+            python3 run_suite.py \
+              --hw amd \
+              --suite stage-c-test-4-gpu-amd \
+              --auto-partition-id ${{ matrix.part }} \
+              --auto-partition-size 2 \
+              --timeout-per-file 1800 \
+              --enable-retry \
+              --max-attempts 2 \
+              --retry-wait-seconds 120 \
+              --retry-timeout-increase 0 \
+              ${{ inputs.continue_on_error && '--continue-on-error' || '' }}
+
  stage-c-test-large-8-gpu-amd:
    needs: [check-changes, call-gate, stage-b-test-1-gpu-small-amd, stage-b-test-2-gpu-large-amd]
    if: |
@@ -997,6 +1054,7 @@ jobs:
        stage-b-test-1-gpu-large-amd,
        stage-b-test-2-gpu-large-amd,
        stage-b-test-large-8-gpu-35x-disaggregation-amd,
+        stage-c-test-4-gpu-amd,
        stage-c-test-large-8-gpu-amd,
        stage-c-test-large-8-gpu-amd-mi35x,
      ]