[AMD] Replace MI325 with MI300 CI Runners (#31409)

This commit is contained in:
Bingxu Chen
2026-07-29 08:18:07 -07:00
committed by GitHub
parent e1f2f9d1fa
commit bfc450248e
9 changed files with 200 additions and 174 deletions
+20 -20
View File
@@ -144,7 +144,7 @@ jobs:
nightly-test-1-gpu-unit-rocm720:
if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-test-1-gpu-unit-rocm720,'))
runs-on: linux-mi325-1gpu-sglang
runs-on: linux-mi300-1gpu-sglang
steps:
- name: Checkout code
uses: actions/checkout@v4
@@ -174,7 +174,7 @@ jobs:
nightly-test-1-gpu-kernel-rocm720:
if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-test-1-gpu-kernel-rocm720,'))
runs-on: linux-mi325-1gpu-sglang
runs-on: linux-mi300-1gpu-sglang
steps:
- name: Checkout code
uses: actions/checkout@v4
@@ -239,7 +239,7 @@ jobs:
nightly-accuracy-2-gpu-rocm720:
if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-accuracy-2-gpu-rocm720,'))
runs-on: linux-mi325-2gpu-sglang
runs-on: linux-mi300-2gpu-sglang
steps:
- name: Checkout code
uses: actions/checkout@v4
@@ -338,7 +338,7 @@ jobs:
nightly-accuracy-2-gpu-vlm-rocm720:
if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-accuracy-2-gpu-vlm-rocm720,'))
runs-on: linux-mi325-2gpu-sglang
runs-on: linux-mi300-2gpu-sglang
steps:
- name: Checkout code
uses: actions/checkout@v4
@@ -369,7 +369,7 @@ jobs:
nightly-perf-2-gpu-text-rocm720:
if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-perf-2-gpu-text-rocm720,'))
runs-on: linux-mi325-2gpu-sglang
runs-on: linux-mi300-2gpu-sglang
steps:
- name: Checkout code
uses: actions/checkout@v4
@@ -401,7 +401,7 @@ jobs:
nightly-perf-2-gpu-vlm-rocm720:
if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-perf-2-gpu-vlm-rocm720,'))
runs-on: linux-mi325-2gpu-sglang
runs-on: linux-mi300-2gpu-sglang
steps:
- name: Checkout code
uses: actions/checkout@v4
@@ -433,7 +433,7 @@ jobs:
nightly-4-gpu-rocm720:
if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-4-gpu-rocm720,'))
runs-on: linux-mi325-4gpu-sglang
runs-on: linux-mi300-4gpu-sglang
steps:
- name: Checkout code
uses: actions/checkout@v4
@@ -469,7 +469,7 @@ jobs:
nightly-accuracy-8-gpu-rocm720:
if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-accuracy-8-gpu-rocm720,'))
runs-on: linux-mi325-8gpu-sglang
runs-on: linux-mi300-8gpu-sglang
steps:
- name: Checkout code
uses: actions/checkout@v4
@@ -548,7 +548,7 @@ jobs:
nightly-8-gpu-grok1-int4-rocm720:
if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-grok1-int4-rocm720,'))
runs-on: linux-mi325-8gpu-sglang
runs-on: linux-mi300-8gpu-sglang
steps:
- name: Checkout code
uses: actions/checkout@v4
@@ -645,7 +645,7 @@ jobs:
nightly-8-gpu-grok2-rocm720:
if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-grok2-rocm720,'))
runs-on: linux-mi325-8gpu-sglang
runs-on: linux-mi300-8gpu-sglang
steps:
- name: Checkout code
uses: actions/checkout@v4
@@ -742,7 +742,7 @@ jobs:
nightly-8-gpu-deepseek-v31-rocm720:
if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-deepseek-v31-rocm720,'))
runs-on: linux-mi325-8gpu-sglang
runs-on: linux-mi300-8gpu-sglang
steps:
- name: Checkout code
uses: actions/checkout@v4
@@ -788,7 +788,7 @@ jobs:
nightly-8-gpu-deepseek-v32-rocm720:
if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-deepseek-v32-rocm720,'))
runs-on: linux-mi325-8gpu-sglang
runs-on: linux-mi300-8gpu-sglang
steps:
- name: Checkout code
uses: actions/checkout@v4
@@ -832,7 +832,7 @@ jobs:
nightly-8-gpu-deepseek-v32-mtp-rocm720:
if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-deepseek-v32-mtp-rocm720,'))
runs-on: linux-mi325-8gpu-sglang
runs-on: linux-mi300-8gpu-sglang
steps:
- name: Checkout code
uses: actions/checkout@v4
@@ -876,7 +876,7 @@ jobs:
nightly-8-gpu-deepseek-v3-kv-fp8-rocm720:
if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-deepseek-v3-kv-fp8-rocm720,'))
runs-on: linux-mi325-8gpu-sglang
runs-on: linux-mi300-8gpu-sglang
steps:
- name: Checkout code
uses: actions/checkout@v4
@@ -1437,7 +1437,7 @@ jobs:
nightly-8-gpu-kimi-k26-rocm720:
if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-kimi-k26-rocm720,'))
runs-on: linux-mi325-8gpu-sglang
runs-on: linux-mi300-8gpu-sglang
steps:
- name: Checkout code
uses: actions/checkout@v4
@@ -1510,7 +1510,7 @@ jobs:
nightly-8-gpu-qwen3-235b-rocm720:
if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-qwen3-235b-rocm720,'))
runs-on: linux-mi325-8gpu-sglang
runs-on: linux-mi300-8gpu-sglang
steps:
- name: Checkout code
uses: actions/checkout@v4
@@ -1582,7 +1582,7 @@ jobs:
nightly-8-gpu-qwen35-rocm720:
if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-qwen35-rocm720,'))
runs-on: linux-mi325-8gpu-sglang
runs-on: linux-mi300-8gpu-sglang
steps:
- name: Checkout code
uses: actions/checkout@v4
@@ -1714,7 +1714,7 @@ jobs:
nightly-8-gpu-glm51-rocm720:
if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-glm51-rocm720,'))
runs-on: linux-mi325-8gpu-sglang
runs-on: linux-mi300-8gpu-sglang
steps:
- name: Checkout code
uses: actions/checkout@v4
@@ -1936,7 +1936,7 @@ jobs:
nightly-8-gpu-minimax-m27-rocm720:
if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-minimax-m27-rocm720,'))
runs-on: linux-mi325-8gpu-sglang
runs-on: linux-mi300-8gpu-sglang
steps:
- name: Checkout code
uses: actions/checkout@v4
@@ -1986,7 +1986,7 @@ jobs:
nightly-1-gpu-zimage-turbo-rocm720:
if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-1-gpu-zimage-turbo-rocm720,'))
runs-on: linux-mi325-1gpu-sglang
runs-on: linux-mi300-1gpu-sglang
steps:
- name: Checkout code
uses: actions/checkout@v4
+20 -20
View File
@@ -140,7 +140,7 @@ jobs:
nightly-test-1-gpu-unit:
if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-test-1-gpu-unit,'))
runs-on: linux-mi325-1gpu-sglang
runs-on: linux-mi300-1gpu-sglang
steps:
- name: Checkout code
uses: actions/checkout@v4
@@ -171,7 +171,7 @@ jobs:
nightly-test-1-gpu-kernel:
if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-test-1-gpu-kernel,'))
runs-on: linux-mi325-1gpu-sglang
runs-on: linux-mi300-1gpu-sglang
steps:
- name: Checkout code
uses: actions/checkout@v4
@@ -240,7 +240,7 @@ jobs:
nightly-accuracy-2-gpu:
if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-accuracy-2-gpu,'))
runs-on: linux-mi325-2gpu-sglang
runs-on: linux-mi300-2gpu-sglang
steps:
- name: Checkout code
uses: actions/checkout@v4
@@ -340,7 +340,7 @@ jobs:
nightly-accuracy-2-gpu-vlm:
if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-accuracy-2-gpu-vlm,'))
runs-on: linux-mi325-2gpu-sglang
runs-on: linux-mi300-2gpu-sglang
steps:
- name: Checkout code
uses: actions/checkout@v4
@@ -372,7 +372,7 @@ jobs:
nightly-perf-2-gpu-text:
if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-perf-2-gpu-text,'))
runs-on: linux-mi325-2gpu-sglang
runs-on: linux-mi300-2gpu-sglang
steps:
- name: Checkout code
uses: actions/checkout@v4
@@ -405,7 +405,7 @@ jobs:
nightly-perf-2-gpu-vlm:
if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-perf-2-gpu-vlm,'))
runs-on: linux-mi325-2gpu-sglang
runs-on: linux-mi300-2gpu-sglang
steps:
- name: Checkout code
uses: actions/checkout@v4
@@ -438,7 +438,7 @@ jobs:
nightly-4-gpu:
if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-4-gpu,'))
runs-on: linux-mi325-4gpu-sglang
runs-on: linux-mi300-4gpu-sglang
steps:
- name: Checkout code
uses: actions/checkout@v4
@@ -474,7 +474,7 @@ jobs:
nightly-accuracy-8-gpu:
if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-accuracy-8-gpu,'))
runs-on: linux-mi325-8gpu-sglang
runs-on: linux-mi300-8gpu-sglang
steps:
- name: Checkout code
uses: actions/checkout@v4
@@ -553,7 +553,7 @@ jobs:
nightly-8-gpu-grok1-int4:
if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-grok1-int4,'))
runs-on: linux-mi325-8gpu-sglang
runs-on: linux-mi300-8gpu-sglang
steps:
- name: Checkout code
uses: actions/checkout@v4
@@ -650,7 +650,7 @@ jobs:
nightly-8-gpu-grok2:
if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-grok2,'))
runs-on: linux-mi325-8gpu-sglang
runs-on: linux-mi300-8gpu-sglang
steps:
- name: Checkout code
uses: actions/checkout@v4
@@ -747,7 +747,7 @@ jobs:
nightly-8-gpu-deepseek-v31:
if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-deepseek-v31,'))
runs-on: linux-mi325-8gpu-sglang
runs-on: linux-mi300-8gpu-sglang
steps:
- name: Checkout code
uses: actions/checkout@v4
@@ -793,7 +793,7 @@ jobs:
nightly-8-gpu-deepseek-v32:
if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-deepseek-v32,'))
runs-on: linux-mi325-8gpu-sglang
runs-on: linux-mi300-8gpu-sglang
steps:
- name: Checkout code
uses: actions/checkout@v4
@@ -837,7 +837,7 @@ jobs:
nightly-8-gpu-deepseek-v32-mtp:
if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-deepseek-v32-mtp,'))
runs-on: linux-mi325-8gpu-sglang
runs-on: linux-mi300-8gpu-sglang
steps:
- name: Checkout code
uses: actions/checkout@v4
@@ -881,7 +881,7 @@ jobs:
nightly-8-gpu-deepseek-v3-kv-fp8:
if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-deepseek-v3-kv-fp8,'))
runs-on: linux-mi325-8gpu-sglang
runs-on: linux-mi300-8gpu-sglang
steps:
- name: Checkout code
uses: actions/checkout@v4
@@ -1291,7 +1291,7 @@ jobs:
nightly-8-gpu-kimi-k26:
if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-kimi-k26,'))
runs-on: linux-mi325-8gpu-sglang
runs-on: linux-mi300-8gpu-sglang
steps:
- name: Checkout code
uses: actions/checkout@v4
@@ -1364,7 +1364,7 @@ jobs:
nightly-8-gpu-qwen3-235b:
if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-qwen3-235b,'))
runs-on: linux-mi325-8gpu-sglang
runs-on: linux-mi300-8gpu-sglang
steps:
- name: Checkout code
uses: actions/checkout@v4
@@ -1436,7 +1436,7 @@ jobs:
nightly-8-gpu-qwen35:
if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-qwen35,'))
runs-on: linux-mi325-8gpu-sglang
runs-on: linux-mi300-8gpu-sglang
steps:
- name: Checkout code
uses: actions/checkout@v4
@@ -1568,7 +1568,7 @@ jobs:
nightly-8-gpu-glm51:
if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-glm51,'))
runs-on: linux-mi325-8gpu-sglang
runs-on: linux-mi300-8gpu-sglang
steps:
- name: Checkout code
uses: actions/checkout@v4
@@ -1790,7 +1790,7 @@ jobs:
nightly-8-gpu-minimax-m27:
if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-minimax-m27,'))
runs-on: linux-mi325-8gpu-sglang
runs-on: linux-mi300-8gpu-sglang
steps:
- name: Checkout code
uses: actions/checkout@v4
@@ -1840,7 +1840,7 @@ jobs:
nightly-1-gpu-zimage-turbo:
if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-1-gpu-zimage-turbo,'))
runs-on: linux-mi325-1gpu-sglang
runs-on: linux-mi300-1gpu-sglang
steps:
- name: Checkout code
uses: actions/checkout@v4
+12 -7
View File
@@ -32,7 +32,7 @@ on:
description: 'AMD runner pool to dispatch GPU jobs to'
required: false
type: choice
default: mi325
default: mi300
options:
- mi300
- mi325
@@ -61,6 +61,11 @@ on:
required: false
type: string
default: ''
runner_arch:
description: 'AMD runner pool to dispatch GPU jobs to'
required: false
type: string
default: mi300
rocm_version:
description: 'ROCm container variant (empty = Dockerfile default; rocm720 = ROCm 7.2.0)'
required: false
@@ -125,10 +130,10 @@ jobs:
# setup across scarce AMD GPUs to shave only a couple minutes of test time,
# so one GPU running the whole suite sequentially is the better trade.
extra-a-test-1-gpu-small-amd:
name: ${{ format('extra-a-test-1-gpu-small-amd{0} (linux-{1}-1gpu-sglang)', inputs.rocm_version && format('-{0}', inputs.rocm_version) || '', inputs.runner_arch || 'mi325') }}
name: ${{ format('extra-a-test-1-gpu-small-amd{0} (linux-{1}-1gpu-sglang)', inputs.rocm_version && format('-{0}', inputs.rocm_version) || '', inputs.runner_arch || 'mi300') }}
needs: [call-gate]
if: ${{ !cancelled() && needs.call-gate.result == 'success' }}
runs-on: ${{ format('linux-{0}-1gpu-sglang', inputs.runner_arch || 'mi325') }}
runs-on: ${{ format('linux-{0}-1gpu-sglang', inputs.runner_arch || 'mi300') }}
steps:
- name: Checkout code
uses: actions/checkout@v4
@@ -157,10 +162,10 @@ jobs:
# pool as small (AMD GPUs carry enough VRAM that "large" here is a CUDA
# memory-tier label, not a separate AMD runner pool).
extra-a-test-1-gpu-large-amd:
name: ${{ format('extra-a-test-1-gpu-large-amd{0} (linux-{1}-1gpu-sglang)', inputs.rocm_version && format('-{0}', inputs.rocm_version) || '', inputs.runner_arch || 'mi325') }}
name: ${{ format('extra-a-test-1-gpu-large-amd{0} (linux-{1}-1gpu-sglang)', inputs.rocm_version && format('-{0}', inputs.rocm_version) || '', inputs.runner_arch || 'mi300') }}
needs: [call-gate]
if: ${{ !cancelled() && needs.call-gate.result == 'success' }}
runs-on: ${{ format('linux-{0}-1gpu-sglang', inputs.runner_arch || 'mi325') }}
runs-on: ${{ format('linux-{0}-1gpu-sglang', inputs.runner_arch || 'mi300') }}
steps:
- name: Checkout code
uses: actions/checkout@v4
@@ -187,10 +192,10 @@ jobs:
# Multi-GPU TP / PP / PD mock-model + kv_canary e2e tests. Mirrors CUDA's
# extra-a 2-gpu-large stage; runs on the 2-GPU AMD pool.
extra-a-test-2-gpu-large-amd:
name: ${{ format('extra-a-test-2-gpu-large-amd{0} (linux-{1}-2gpu-sglang)', inputs.rocm_version && format('-{0}', inputs.rocm_version) || '', inputs.runner_arch || 'mi325') }}
name: ${{ format('extra-a-test-2-gpu-large-amd{0} (linux-{1}-2gpu-sglang)', inputs.rocm_version && format('-{0}', inputs.rocm_version) || '', inputs.runner_arch || 'mi300') }}
needs: [call-gate]
if: ${{ !cancelled() && needs.call-gate.result == 'success' }}
runs-on: ${{ format('linux-{0}-2gpu-sglang', inputs.runner_arch || 'mi325') }}
runs-on: ${{ format('linux-{0}-2gpu-sglang', inputs.runner_arch || 'mi300') }}
steps:
- name: Checkout code
uses: actions/checkout@v4
+36 -49
View File
@@ -229,6 +229,7 @@ jobs:
uses: ./.github/workflows/pr-test-amd-extra.yml
with:
ref: ${{ inputs.pr_head_sha || inputs.ref || '' }}
runner_arch: mi300
rocm_version: rocm720
aiter_ref: ${{ inputs.aiter_ref }}
continue_on_error: true
@@ -249,7 +250,7 @@ jobs:
strategy:
fail-fast: false
matrix:
runner: [linux-mi325-1gpu-sglang]
runner: [linux-mi300-1gpu-sglang]
runs-on: ${{matrix.runner}}
steps:
- name: Checkout code
@@ -295,7 +296,7 @@ jobs:
strategy:
fail-fast: false
matrix:
runner: [linux-mi325-2gpu-sglang]
runner: [linux-mi300-2gpu-sglang]
runs-on: ${{matrix.runner}}
steps:
- name: Checkout code
@@ -336,7 +337,7 @@ jobs:
strategy:
fail-fast: false
matrix:
runner: [linux-mi325-1gpu-sglang]
runner: [linux-mi300-1gpu-sglang]
runs-on: ${{matrix.runner}}
steps:
- name: Checkout code
@@ -374,7 +375,7 @@ jobs:
strategy:
fail-fast: false
matrix:
runner: [linux-mi325-1gpu-sglang]
runner: [linux-mi300-1gpu-sglang]
runs-on: ${{matrix.runner}}
steps:
- name: Checkout code
@@ -412,7 +413,7 @@ jobs:
strategy:
fail-fast: false
matrix:
runner: [linux-mi325-1gpu-sglang]
runner: [linux-mi300-1gpu-sglang]
runs-on: ${{matrix.runner}}
steps:
- name: Checkout code
@@ -450,8 +451,9 @@ jobs:
)
strategy:
fail-fast: false
max-parallel: 4
matrix:
runner: [linux-mi325-1gpu-sglang]
runner: [linux-mi300-1gpu-sglang]
part: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13]
runs-on: ${{matrix.runner}}
steps:
@@ -471,9 +473,9 @@ jobs:
- name: Install dependencies
run: bash scripts/ci/amd/amd_ci_install_dependency.sh
- name: Run test
timeout-minutes: 60
timeout-minutes: 75
run: |
bash scripts/ci/amd/amd_ci_exec.sh -w "/sglang-checkout/test" python3 run_suite.py --hw amd --suite stage-b-test-1-gpu-small-amd --auto-partition-id ${{ matrix.part }} --auto-partition-size 14 --timeout-per-file 1800 ${{ needs.check-changes.outputs.continue_on_error == 'true' && '--continue-on-error' || '' }}
bash scripts/ci/amd/amd_ci_exec.sh -w "/sglang-checkout/test" python3 run_suite.py --hw amd --suite stage-b-test-1-gpu-small-amd --auto-partition-id ${{ matrix.part }} --auto-partition-size 14 --timeout-per-file 2400 ${{ needs.check-changes.outputs.continue_on_error == 'true' && '--continue-on-error' || '' }}
stage-b-test-1-gpu-small-amd-nondeterministic-rocm720:
needs: [check-changes]
@@ -490,7 +492,7 @@ jobs:
strategy:
fail-fast: false
matrix:
runner: [linux-mi325-1gpu-sglang]
runner: [linux-mi300-1gpu-sglang]
runs-on: ${{matrix.runner}}
steps:
- name: Checkout code
@@ -565,8 +567,9 @@ jobs:
)
strategy:
fail-fast: false
max-parallel: 1
matrix:
runner: [linux-mi325-1gpu-sglang]
runner: [linux-mi300-1gpu-sglang]
part: [0, 1, 2]
runs-on: ${{matrix.runner}}
steps:
@@ -586,9 +589,9 @@ jobs:
- name: Install dependencies
run: bash scripts/ci/amd/amd_ci_install_dependency.sh
- name: Run test
timeout-minutes: 45
timeout-minutes: 60
run: |
bash scripts/ci/amd/amd_ci_exec.sh -w "/sglang-checkout/test" python3 run_suite.py --hw amd --suite stage-b-test-1-gpu-large-amd --auto-partition-id ${{ matrix.part }} --auto-partition-size 3 --timeout-per-file 1800 ${{ needs.check-changes.outputs.continue_on_error == 'true' && '--continue-on-error' || '' }}
bash scripts/ci/amd/amd_ci_exec.sh -w "/sglang-checkout/test" python3 run_suite.py --hw amd --suite stage-b-test-1-gpu-large-amd --auto-partition-id ${{ matrix.part }} --auto-partition-size 3 --timeout-per-file 2700 ${{ needs.check-changes.outputs.continue_on_error == 'true' && '--continue-on-error' || '' }}
stage-b-test-2-gpu-large-amd-rocm720:
needs: [check-changes]
@@ -604,8 +607,9 @@ jobs:
)
strategy:
fail-fast: false
max-parallel: 1
matrix:
runner: [linux-mi325-2gpu-sglang]
runner: [linux-mi300-2gpu-sglang]
part: [0, 1]
runs-on: ${{matrix.runner}}
steps:
@@ -625,9 +629,9 @@ jobs:
- name: Install dependencies
run: bash scripts/ci/amd/amd_ci_install_dependency.sh
- name: Run test
timeout-minutes: 45
timeout-minutes: 120
run: |
bash scripts/ci/amd/amd_ci_exec.sh -w "/sglang-checkout/test" python3 run_suite.py --hw amd --suite stage-b-test-2-gpu-large-amd --auto-partition-id ${{ matrix.part }} --auto-partition-size 2 --timeout-per-file 1800 ${{ needs.check-changes.outputs.continue_on_error == 'true' && '--continue-on-error' || '' }}
bash scripts/ci/amd/amd_ci_exec.sh -w "/sglang-checkout/test" python3 run_suite.py --hw amd --suite stage-b-test-2-gpu-large-amd --auto-partition-id ${{ matrix.part }} --auto-partition-size 2 --timeout-per-file 5400 ${{ needs.check-changes.outputs.continue_on_error == 'true' && '--continue-on-error' || '' }}
multimodal-gen-test-1-gpu-amd-rocm720:
needs: [check-changes]
@@ -643,8 +647,9 @@ jobs:
)
strategy:
fail-fast: false
max-parallel: 1
matrix:
runner: [linux-mi325-1gpu-sglang]
runner: [linux-mi300-1gpu-sglang]
part: [0, 1, 2, 3]
runs-on: ${{matrix.runner}}
steps:
@@ -676,16 +681,7 @@ jobs:
- name: Setup kernel caches
run: |
# Use the persistent /sgl-data directory (mounted from /home/runner/sgl-data)
# This directory persists across container restarts on the self-hosted runner
docker exec ci_sglang mkdir -p /sgl-data/aiter-kernels /sgl-data/miopen-cache /sgl-data/hf-cache/hub
# Clear pre-built AITER kernels from Docker image to avoid segfaults
# The image may have stale/incompatible kernels at /sgl-workspace/aiter/aiter/jit/
echo "Clearing pre-built AITER kernels from Docker image..."
docker exec ci_sglang rm -rf /sgl-workspace/aiter/aiter/jit/*.so 2>/dev/null || true
docker exec ci_sglang rm -rf /sgl-data/aiter-kernels/*.so 2>/dev/null || true
echo "AITER kernels cleared - will be rebuilt on first use"
docker exec ci_sglang mkdir -p /sgl-data/miopen-cache /sgl-data/hf-cache/hub
# Create persistent cache marker if /sgl-data is a real mount (not ephemeral)
# This tells the test cleanup code to NOT delete downloaded models
@@ -706,7 +702,7 @@ jobs:
free -h
echo ""
echo "=== Disk Space ==="
df -h /home/runner/sgl-data 2>/dev/null || df -h
df -h /home/runner/sglang-data 2>/dev/null || df -h
echo ""
echo "=== HF Cache Directory Structure ==="
docker exec ci_sglang ls -la /sgl-data/hf-cache/ 2>/dev/null || echo "HF cache dir not found"
@@ -728,7 +724,7 @@ jobs:
docker exec ci_sglang rocm-smi --showmeminfo vram 2>/dev/null || echo "rocm-smi not available"
- name: Run diffusion server tests (1-GPU)
timeout-minutes: 60
timeout-minutes: 90
run: |
# AMD CI: All 1-GPU tests except FLUX.2 (FLUX.1 covers same code path)
# Tests: T2V, T2I, I2V, LoRA
@@ -745,7 +741,6 @@ jobs:
-e SGLANG_SKIP_CONSISTENCY=1 \
-e SGLANG_TEST_NUM_INFERENCE_STEPS=5 \
-e SGLANG_DIFFUSION_ARTIFACT_DIR=/sglang-checkout/diffusion-failures \
-e AITER_JIT_DIR=/sgl-data/aiter-kernels \
-e MIOPEN_USER_DB_PATH=/sgl-data/miopen-cache \
-e HF_HUB_ENABLE_HF_TRANSFER=1 \
-e HF_HUB_DISABLE_SYMLINKS_WARNING=1 \
@@ -783,8 +778,9 @@ jobs:
)
strategy:
fail-fast: false
max-parallel: 1
matrix:
runner: [linux-mi325-2gpu-sglang]
runner: [linux-mi300-2gpu-sglang]
part: [0, 1, 2] # 3 partitions: 2 parametrized + 1 standalone (single_test_file/test_disagg_server.py)
runs-on: ${{matrix.runner}}
steps:
@@ -816,15 +812,7 @@ jobs:
- name: Setup kernel caches
run: |
# Use the persistent /sgl-data directory (mounted from /home/runner/sgl-data)
docker exec ci_sglang mkdir -p /sgl-data/aiter-kernels /sgl-data/miopen-cache /sgl-data/hf-cache/hub
# Clear pre-built AITER kernels from Docker image to avoid segfaults
# The image may have stale/incompatible kernels at /sgl-workspace/aiter/aiter/jit/
echo "Clearing pre-built AITER kernels from Docker image..."
docker exec ci_sglang rm -rf /sgl-workspace/aiter/aiter/jit/*.so 2>/dev/null || true
docker exec ci_sglang rm -rf /sgl-data/aiter-kernels/*.so 2>/dev/null || true
echo "AITER kernels cleared - will be rebuilt on first use"
docker exec ci_sglang mkdir -p /sgl-data/miopen-cache /sgl-data/hf-cache/hub
# Create persistent cache marker if /sgl-data is a real mount (not ephemeral)
# This tells the test cleanup code to NOT delete downloaded models
@@ -845,7 +833,7 @@ jobs:
free -h
echo ""
echo "=== Disk Space ==="
df -h /home/runner/sgl-data 2>/dev/null || df -h
df -h /home/runner/sglang-data 2>/dev/null || df -h
echo ""
echo "=== HF Cache Directory Structure ==="
docker exec ci_sglang ls -la /sgl-data/hf-cache/ 2>/dev/null || echo "HF cache dir not found"
@@ -867,7 +855,7 @@ jobs:
docker exec ci_sglang rocm-smi --showmeminfo vram 2>/dev/null || echo "rocm-smi not available"
- name: Run diffusion server tests (2-GPU)
timeout-minutes: 150
timeout-minutes: 180
run: |
# AMD CI: All 2-GPU tests including LoRA
# Tests: T2V, T2I, I2V, LoRA
@@ -884,7 +872,6 @@ jobs:
-e SGLANG_SKIP_CONSISTENCY=1 \
-e SGLANG_TEST_NUM_INFERENCE_STEPS=5 \
-e SGLANG_DIFFUSION_ARTIFACT_DIR=/sglang-checkout/diffusion-failures \
-e AITER_JIT_DIR=/sgl-data/aiter-kernels \
-e MIOPEN_USER_DB_PATH=/sgl-data/miopen-cache \
-e HF_HUB_ENABLE_HF_TRANSFER=1 \
-e HF_HUB_DISABLE_SYMLINKS_WARNING=1 \
@@ -923,7 +910,7 @@ jobs:
((needs.check-changes.outputs.main_package == 'true') || (needs.check-changes.outputs.sgl_kernel == 'true'))
)
)
runs-on: linux-mi325-1gpu-sglang
runs-on: linux-mi300-1gpu-sglang
steps:
- name: Checkout code
uses: actions/checkout@v4
@@ -957,7 +944,6 @@ jobs:
# Skip ltx2_vae_channels_last: it asserts CUDA `channels_last_3d`
# memory-format behavior that the ROCm conv path doesn't reproduce.
docker exec \
-e AITER_JIT_DIR=/sgl-data/aiter-kernels \
-e MIOPEN_USER_DB_PATH=/sgl-data/miopen-cache \
-w /sglang-checkout/python \
ci_sglang python3 sglang/multimodal_gen/test/run_suite.py \
@@ -979,7 +965,7 @@ jobs:
strategy:
fail-fast: false
matrix:
runner: [linux-mi325-4gpu-sglang]
runner: [linux-mi300-4gpu-sglang]
part: [0]
runs-on: ${{matrix.runner}}
steps:
@@ -1000,7 +986,7 @@ jobs:
run: bash scripts/ci/amd/amd_ci_install_dependency.sh
- name: Run test
timeout-minutes: 60
timeout-minutes: 90
run: |
bash scripts/ci/amd/amd_ci_exec.sh \
-e NCCL_CUMEM_ENABLE=0 \
@@ -1013,7 +999,7 @@ jobs:
--suite stage-c-test-4-gpu-amd \
--auto-partition-id ${{ matrix.part }} \
--auto-partition-size 1 \
--timeout-per-file 1800 \
--timeout-per-file 5400 \
--enable-retry \
--max-attempts 2 \
--retry-wait-seconds 120 \
@@ -1033,11 +1019,12 @@ jobs:
)
)
env:
RUNNER_LABELS: linux-mi325-8gpu-sglang
RUNNER_LABELS: linux-mi300-8gpu-sglang
strategy:
fail-fast: false
max-parallel: 1
matrix:
runner: [linux-mi325-8gpu-sglang]
runner: [linux-mi300-8gpu-sglang]
part: [0, 1, 2, 3]
runs-on: ${{matrix.runner}}
steps:
+51 -59
View File
@@ -64,7 +64,7 @@ on:
description: 'AMD runner pool to dispatch GPU jobs to'
required: false
type: choice
default: mi325
default: mi300
options:
- mi300
- mi325
@@ -80,6 +80,11 @@ on:
required: false
type: string
default: ''
runner_arch:
description: 'AMD runner pool to dispatch GPU jobs to'
required: false
type: string
default: mi300
run_all_tests:
description: "Run all tests (for releasing or testing purpose)"
required: false
@@ -217,13 +222,14 @@ jobs:
uses: ./.github/workflows/pr-test-amd-extra.yml
with:
ref: ${{ inputs.pr_head_sha || inputs.ref || '' }}
runner_arch: ${{ inputs.runner_arch || 'mi300' }}
aiter_ref: ${{ inputs.aiter_ref }}
continue_on_error: true
secrets: inherit
# =============================================== sgl-kernel ====================================================
sgl-kernel-unit-test-amd:
name: ${{ format('sgl-kernel-unit-test-amd (linux-{0}-1gpu-sglang)', inputs.runner_arch || 'mi325') }}
name: ${{ format('sgl-kernel-unit-test-amd (linux-{0}-1gpu-sglang)', inputs.runner_arch || 'mi300') }}
needs: [check-changes, call-gate]
if: |
always() && !cancelled() &&
@@ -235,7 +241,7 @@ jobs:
needs.check-changes.outputs.sgl_kernel == 'true'
)
)
runs-on: ${{ format('linux-{0}-1gpu-sglang', inputs.runner_arch || 'mi325') }}
runs-on: ${{ format('linux-{0}-1gpu-sglang', inputs.runner_arch || 'mi300') }}
steps:
- name: Checkout code
uses: actions/checkout@v4
@@ -281,7 +287,7 @@ jobs:
exit $failures
sgl-kernel-unit-test-2-gpu-amd:
name: ${{ format('sgl-kernel-unit-test-2-gpu-amd (linux-{0}-2gpu-sglang)', inputs.runner_arch || 'mi325') }}
name: ${{ format('sgl-kernel-unit-test-2-gpu-amd (linux-{0}-2gpu-sglang)', inputs.runner_arch || 'mi300') }}
needs: [check-changes, call-gate]
if: |
always() && !cancelled() &&
@@ -293,7 +299,7 @@ jobs:
needs.check-changes.outputs.sgl_kernel == 'true'
)
)
runs-on: ${{ format('linux-{0}-2gpu-sglang', inputs.runner_arch || 'mi325') }}
runs-on: ${{ format('linux-{0}-2gpu-sglang', inputs.runner_arch || 'mi300') }}
steps:
- name: Checkout code
uses: actions/checkout@v4
@@ -322,7 +328,7 @@ jobs:
# =============================================== primary ====================================================
stage-a-test-1-gpu-small-amd:
name: ${{ format('stage-a-test-1-gpu-small-amd (linux-{0}-1gpu-sglang)', inputs.runner_arch || 'mi325') }}
name: ${{ format('stage-a-test-1-gpu-small-amd (linux-{0}-1gpu-sglang)', inputs.runner_arch || 'mi300') }}
needs: [check-changes, call-gate]
if: |
always() && !cancelled() &&
@@ -334,7 +340,7 @@ jobs:
((needs.check-changes.outputs.main_package == 'true') || (needs.check-changes.outputs.sgl_kernel == 'true'))
)
)
runs-on: ${{ format('linux-{0}-1gpu-sglang', inputs.runner_arch || 'mi325') }}
runs-on: ${{ format('linux-{0}-1gpu-sglang', inputs.runner_arch || 'mi300') }}
steps:
- name: Checkout code
uses: actions/checkout@v4
@@ -359,7 +365,7 @@ jobs:
bash scripts/ci/amd/amd_ci_exec.sh -w "/sglang-checkout/test" python3 run_suite.py --hw amd --suite stage-a-test-1-gpu-small-amd ${{ needs.check-changes.outputs.continue_on_error == 'true' && '--continue-on-error' || '' }}
jit-kernel-unit-test-amd:
name: ${{ format('jit-kernel-unit-test-amd (linux-{0}-1gpu-sglang)', inputs.runner_arch || 'mi325') }}
name: ${{ format('jit-kernel-unit-test-amd (linux-{0}-1gpu-sglang)', inputs.runner_arch || 'mi300') }}
needs: [check-changes, call-gate]
if: |
always() && !cancelled() &&
@@ -371,7 +377,7 @@ jobs:
needs.check-changes.outputs.jit_kernel == 'true'
)
)
runs-on: ${{ format('linux-{0}-1gpu-sglang', inputs.runner_arch || 'mi325') }}
runs-on: ${{ format('linux-{0}-1gpu-sglang', inputs.runner_arch || 'mi300') }}
steps:
- name: Checkout code
uses: actions/checkout@v4
@@ -396,7 +402,7 @@ jobs:
bash scripts/ci/amd/amd_ci_exec.sh -w "/sglang-checkout/test" python3 run_suite.py --hw amd --suite jit-kernel-unit-test-amd ${{ needs.check-changes.outputs.continue_on_error == 'true' && '--continue-on-error' || '' }}
jit-kernel-benchmark-test-amd:
name: ${{ format('jit-kernel-benchmark-test-amd (linux-{0}-1gpu-sglang)', inputs.runner_arch || 'mi325') }}
name: ${{ format('jit-kernel-benchmark-test-amd (linux-{0}-1gpu-sglang)', inputs.runner_arch || 'mi300') }}
needs: [check-changes, call-gate]
if: |
always() && !cancelled() &&
@@ -408,7 +414,7 @@ jobs:
needs.check-changes.outputs.jit_kernel == 'true'
)
)
runs-on: ${{ format('linux-{0}-1gpu-sglang', inputs.runner_arch || 'mi325') }}
runs-on: ${{ format('linux-{0}-1gpu-sglang', inputs.runner_arch || 'mi300') }}
steps:
- name: Checkout code
uses: actions/checkout@v4
@@ -459,7 +465,7 @@ jobs:
max-wait-minutes: '240'
stage-b-test-1-gpu-small-amd:
name: ${{ format('stage-b-test-1-gpu-small-amd (linux-{0}-1gpu-sglang, {1})', inputs.runner_arch || 'mi325', matrix.part) }}
name: ${{ format('stage-b-test-1-gpu-small-amd (linux-{0}-1gpu-sglang, {1})', inputs.runner_arch || 'mi300', matrix.part) }}
needs: [check-changes, wait-for-stage-a-amd]
if: |
always() &&
@@ -473,9 +479,10 @@ jobs:
)
strategy:
fail-fast: false
max-parallel: 4
matrix:
part: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13]
runs-on: ${{ format('linux-{0}-1gpu-sglang', inputs.runner_arch || 'mi325') }}
runs-on: ${{ format('linux-{0}-1gpu-sglang', inputs.runner_arch || 'mi300') }}
steps:
- name: Checkout code
uses: actions/checkout@v4
@@ -494,12 +501,12 @@ jobs:
run: bash scripts/ci/amd/amd_ci_install_dependency.sh
- name: Run test
timeout-minutes: 60
timeout-minutes: ${{ (inputs.runner_arch || 'mi300') == 'mi300' && 75 || 60 }}
run: |
bash scripts/ci/amd/amd_ci_exec.sh -w "/sglang-checkout/test" python3 run_suite.py --hw amd --suite stage-b-test-1-gpu-small-amd --auto-partition-id ${{ matrix.part }} --auto-partition-size 14 --timeout-per-file 2400 ${{ needs.check-changes.outputs.continue_on_error == 'true' && '--continue-on-error' || '' }}
stage-b-test-1-gpu-small-amd-nondeterministic:
name: ${{ format('stage-b-test-1-gpu-small-amd-nondeterministic (linux-{0}-1gpu-sglang)', inputs.runner_arch || 'mi325') }}
name: ${{ format('stage-b-test-1-gpu-small-amd-nondeterministic (linux-{0}-1gpu-sglang)', inputs.runner_arch || 'mi300') }}
needs: [check-changes, wait-for-stage-a-amd]
if: |
always() &&
@@ -511,7 +518,7 @@ jobs:
((needs.check-changes.outputs.main_package == 'true') || (needs.check-changes.outputs.sgl_kernel == 'true'))
)
)
runs-on: ${{ format('linux-{0}-1gpu-sglang', inputs.runner_arch || 'mi325') }}
runs-on: ${{ format('linux-{0}-1gpu-sglang', inputs.runner_arch || 'mi300') }}
steps:
- name: Checkout code
uses: actions/checkout@v4
@@ -574,7 +581,7 @@ jobs:
bash scripts/ci/amd/amd_ci_exec.sh -w "/sglang-checkout/test" python3 run_suite.py --hw amd --suite stage-b-test-1-gpu-small-amd-mi35x ${{ needs.check-changes.outputs.continue_on_error == 'true' && '--continue-on-error' || '' }}
stage-b-test-1-gpu-large-amd:
name: ${{ format('stage-b-test-1-gpu-large-amd (linux-{0}-1gpu-sglang, {1})', inputs.runner_arch || 'mi325', matrix.part) }}
name: ${{ format('stage-b-test-1-gpu-large-amd (linux-{0}-1gpu-sglang, {1})', inputs.runner_arch || 'mi300', matrix.part) }}
needs: [check-changes, wait-for-stage-a-amd]
if: |
always() &&
@@ -588,9 +595,10 @@ jobs:
)
strategy:
fail-fast: false
max-parallel: 1
matrix:
part: [0, 1, 2]
runs-on: ${{ format('linux-{0}-1gpu-sglang', inputs.runner_arch || 'mi325') }}
runs-on: ${{ format('linux-{0}-1gpu-sglang', inputs.runner_arch || 'mi300') }}
steps:
- name: Checkout code
uses: actions/checkout@v4
@@ -609,12 +617,12 @@ jobs:
run: bash scripts/ci/amd/amd_ci_install_dependency.sh
- name: Run test
timeout-minutes: 45
timeout-minutes: ${{ (inputs.runner_arch || 'mi300') == 'mi300' && 60 || 45 }}
run: |
bash scripts/ci/amd/amd_ci_exec.sh -w "/sglang-checkout/test" python3 run_suite.py --hw amd --suite stage-b-test-1-gpu-large-amd --auto-partition-id ${{ matrix.part }} --auto-partition-size 3 --timeout-per-file 2700 ${{ needs.check-changes.outputs.continue_on_error == 'true' && '--continue-on-error' || '' }}
stage-b-test-2-gpu-large-amd:
name: ${{ format('stage-b-test-2-gpu-large-amd (linux-{0}-2gpu-sglang, {1})', inputs.runner_arch || 'mi325', matrix.part) }}
name: ${{ format('stage-b-test-2-gpu-large-amd (linux-{0}-2gpu-sglang, {1})', inputs.runner_arch || 'mi300', matrix.part) }}
needs: [check-changes, wait-for-stage-a-amd]
if: |
always() &&
@@ -628,9 +636,10 @@ jobs:
)
strategy:
fail-fast: false
max-parallel: 1
matrix:
part: [0, 1]
runs-on: ${{ format('linux-{0}-2gpu-sglang', inputs.runner_arch || 'mi325') }}
runs-on: ${{ format('linux-{0}-2gpu-sglang', inputs.runner_arch || 'mi300') }}
steps:
- name: Checkout code
uses: actions/checkout@v4
@@ -649,12 +658,12 @@ jobs:
run: bash scripts/ci/amd/amd_ci_install_dependency.sh
- name: Run test
timeout-minutes: 45
timeout-minutes: ${{ (inputs.runner_arch || 'mi300') == 'mi300' && 120 || 45 }}
run: |
bash scripts/ci/amd/amd_ci_exec.sh -w "/sglang-checkout/test" python3 run_suite.py --hw amd --suite stage-b-test-2-gpu-large-amd --auto-partition-id ${{ matrix.part }} --auto-partition-size 2 --timeout-per-file 2700 ${{ needs.check-changes.outputs.continue_on_error == 'true' && '--continue-on-error' || '' }}
bash scripts/ci/amd/amd_ci_exec.sh -w "/sglang-checkout/test" python3 run_suite.py --hw amd --suite stage-b-test-2-gpu-large-amd --auto-partition-id ${{ matrix.part }} --auto-partition-size 2 --timeout-per-file ${{ (inputs.runner_arch || 'mi300') == 'mi300' && 5400 || 2700 }} ${{ needs.check-changes.outputs.continue_on_error == 'true' && '--continue-on-error' || '' }}
multimodal-gen-test-1-gpu-amd:
name: ${{ format('multimodal-gen-test-1-gpu-amd (linux-{0}-1gpu-sglang, {1})', inputs.runner_arch || 'mi325', matrix.part) }}
name: ${{ format('multimodal-gen-test-1-gpu-amd (linux-{0}-1gpu-sglang, {1})', inputs.runner_arch || 'mi300', matrix.part) }}
needs: [check-changes, call-gate]
if: |
always() && !cancelled() &&
@@ -668,9 +677,10 @@ jobs:
)
strategy:
fail-fast: false
max-parallel: 1
matrix:
part: [0, 1, 2, 3]
runs-on: ${{ format('linux-{0}-1gpu-sglang', inputs.runner_arch || 'mi325') }}
runs-on: ${{ format('linux-{0}-1gpu-sglang', inputs.runner_arch || 'mi300') }}
steps:
- name: Checkout code
uses: actions/checkout@v4
@@ -699,16 +709,7 @@ jobs:
- name: Setup kernel caches
run: |
# Use the persistent /sgl-data directory (mounted from /home/runner/sgl-data)
# This directory persists across container restarts on the self-hosted runner
docker exec ci_sglang mkdir -p /sgl-data/aiter-kernels /sgl-data/miopen-cache /sgl-data/hf-cache/hub
# Clear pre-built AITER kernels from Docker image to avoid segfaults
# The image may have stale/incompatible kernels at /sgl-workspace/aiter/aiter/jit/
echo "Clearing pre-built AITER kernels from Docker image..."
docker exec ci_sglang rm -rf /sgl-workspace/aiter/aiter/jit/*.so 2>/dev/null || true
docker exec ci_sglang rm -rf /sgl-data/aiter-kernels/*.so 2>/dev/null || true
echo "AITER kernels cleared - will be rebuilt on first use"
docker exec ci_sglang mkdir -p /sgl-data/miopen-cache /sgl-data/hf-cache/hub
# Create persistent cache marker if /sgl-data is a real mount (not ephemeral)
# This tells the test cleanup code to NOT delete downloaded models
@@ -729,7 +730,7 @@ jobs:
free -h
echo ""
echo "=== Disk Space ==="
df -h /home/runner/sgl-data 2>/dev/null || df -h
df -h /home/runner/sglang-data 2>/dev/null || df -h
echo ""
echo "=== HF Cache Directory Structure ==="
docker exec ci_sglang ls -la /sgl-data/hf-cache/ 2>/dev/null || echo "HF cache dir not found"
@@ -768,7 +769,6 @@ jobs:
-e SGLANG_SKIP_CONSISTENCY=1 \
-e SGLANG_TEST_NUM_INFERENCE_STEPS=5 \
-e SGLANG_DIFFUSION_ARTIFACT_DIR=/sglang-checkout/diffusion-failures \
-e AITER_JIT_DIR=/sgl-data/aiter-kernels \
-e MIOPEN_USER_DB_PATH=/sgl-data/miopen-cache \
-e HF_HUB_ENABLE_HF_TRANSFER=1 \
-e HF_HUB_DISABLE_SYMLINKS_WARNING=1 \
@@ -794,7 +794,7 @@ jobs:
retention-days: 7
multimodal-gen-test-2-gpu-amd:
name: ${{ format('multimodal-gen-test-2-gpu-amd (linux-{0}-2gpu-sglang, {1})', inputs.runner_arch || 'mi325', matrix.part) }}
name: ${{ format('multimodal-gen-test-2-gpu-amd (linux-{0}-2gpu-sglang, {1})', inputs.runner_arch || 'mi300', matrix.part) }}
needs: [check-changes, call-gate]
if: |
always() && !cancelled() &&
@@ -808,9 +808,10 @@ jobs:
)
strategy:
fail-fast: false
max-parallel: 1
matrix:
part: [0, 1, 2] # 3 partitions: 2 parametrized + 1 standalone (single_test_file/test_disagg_server.py)
runs-on: ${{ format('linux-{0}-2gpu-sglang', inputs.runner_arch || 'mi325') }}
runs-on: ${{ format('linux-{0}-2gpu-sglang', inputs.runner_arch || 'mi300') }}
steps:
- name: Checkout code
uses: actions/checkout@v4
@@ -839,15 +840,7 @@ jobs:
- name: Setup kernel caches
run: |
# Use the persistent /sgl-data directory (mounted from /home/runner/sgl-data)
docker exec ci_sglang mkdir -p /sgl-data/aiter-kernels /sgl-data/miopen-cache /sgl-data/hf-cache/hub
# Clear pre-built AITER kernels from Docker image to avoid segfaults
# The image may have stale/incompatible kernels at /sgl-workspace/aiter/aiter/jit/
echo "Clearing pre-built AITER kernels from Docker image..."
docker exec ci_sglang rm -rf /sgl-workspace/aiter/aiter/jit/*.so 2>/dev/null || true
docker exec ci_sglang rm -rf /sgl-data/aiter-kernels/*.so 2>/dev/null || true
echo "AITER kernels cleared - will be rebuilt on first use"
docker exec ci_sglang mkdir -p /sgl-data/miopen-cache /sgl-data/hf-cache/hub
# Create persistent cache marker if /sgl-data is a real mount (not ephemeral)
# This tells the test cleanup code to NOT delete downloaded models
@@ -868,7 +861,7 @@ jobs:
free -h
echo ""
echo "=== Disk Space ==="
df -h /home/runner/sgl-data 2>/dev/null || df -h
df -h /home/runner/sglang-data 2>/dev/null || df -h
echo ""
echo "=== HF Cache Directory Structure ==="
docker exec ci_sglang ls -la /sgl-data/hf-cache/ 2>/dev/null || echo "HF cache dir not found"
@@ -890,7 +883,7 @@ jobs:
docker exec ci_sglang rocm-smi --showmeminfo vram 2>/dev/null || echo "rocm-smi not available"
- name: Run diffusion server tests (2-GPU)
timeout-minutes: 150
timeout-minutes: ${{ (inputs.runner_arch || 'mi300') == 'mi300' && 180 || 150 }}
run: |
# AMD CI: All 2-GPU tests including LoRA
# Tests: T2V, T2I, I2V, LoRA
@@ -907,7 +900,6 @@ jobs:
-e SGLANG_SKIP_CONSISTENCY=1 \
-e SGLANG_TEST_NUM_INFERENCE_STEPS=5 \
-e SGLANG_DIFFUSION_ARTIFACT_DIR=/sglang-checkout/diffusion-failures \
-e AITER_JIT_DIR=/sgl-data/aiter-kernels \
-e MIOPEN_USER_DB_PATH=/sgl-data/miopen-cache \
-e HF_HUB_ENABLE_HF_TRANSFER=1 \
-e HF_HUB_DISABLE_SYMLINKS_WARNING=1 \
@@ -937,7 +929,7 @@ jobs:
# that don't require NVIDIA hardware, so they should run on AMD too. This
# closes the AMD coverage gap the dashboard surfaces for these tests.
multimodal-gen-unit-test-amd:
name: ${{ format('multimodal-gen-unit-test-amd (linux-{0}-1gpu-sglang)', inputs.runner_arch || 'mi325') }}
name: ${{ format('multimodal-gen-unit-test-amd (linux-{0}-1gpu-sglang)', inputs.runner_arch || 'mi300') }}
needs: [check-changes, call-gate]
if: |
always() && !cancelled() &&
@@ -949,7 +941,7 @@ jobs:
needs.check-changes.outputs.multimodal_gen == 'true'
)
)
runs-on: ${{ format('linux-{0}-1gpu-sglang', inputs.runner_arch || 'mi325') }}
runs-on: ${{ format('linux-{0}-1gpu-sglang', inputs.runner_arch || 'mi300') }}
steps:
- name: Checkout code
uses: actions/checkout@v4
@@ -983,7 +975,6 @@ jobs:
# memory-format behavior that the ROCm conv path doesn't reproduce
# (CUDA-specific). The rest of the unit suite is portable.
docker exec \
-e AITER_JIT_DIR=/sgl-data/aiter-kernels \
-e MIOPEN_USER_DB_PATH=/sgl-data/miopen-cache \
-w /sglang-checkout/python \
ci_sglang python3 sglang/multimodal_gen/test/run_suite.py \
@@ -1019,7 +1010,7 @@ jobs:
max-wait-minutes: '480'
stage-c-test-4-gpu-amd:
name: ${{ format('stage-c-test-4-gpu-amd (linux-{0}-4gpu-sglang, {1})', inputs.runner_arch || 'mi325', matrix.part) }}
name: ${{ format('stage-c-test-4-gpu-amd (linux-{0}-4gpu-sglang, {1})', inputs.runner_arch || 'mi300', matrix.part) }}
needs: [check-changes, call-gate, wait-for-stage-b-amd]
if: |
always() &&
@@ -1035,7 +1026,7 @@ jobs:
fail-fast: false
matrix:
part: [0]
runs-on: ${{ format('linux-{0}-4gpu-sglang', inputs.runner_arch || 'mi325') }}
runs-on: ${{ format('linux-{0}-4gpu-sglang', inputs.runner_arch || 'mi300') }}
steps:
- name: Checkout code
uses: actions/checkout@v4
@@ -1075,7 +1066,7 @@ jobs:
${{ needs.check-changes.outputs.continue_on_error == 'true' && '--continue-on-error' || '' }}
stage-c-test-large-8-gpu-amd:
name: ${{ format('stage-c-test-large-8-gpu-amd (linux-{0}-8gpu-sglang, {1})', inputs.runner_arch || 'mi325', matrix.part) }}
name: ${{ format('stage-c-test-large-8-gpu-amd (linux-{0}-8gpu-sglang, {1})', inputs.runner_arch || 'mi300', matrix.part) }}
needs: [check-changes, call-gate, wait-for-stage-b-amd]
if: |
always() &&
@@ -1088,12 +1079,13 @@ jobs:
)
)
env:
RUNNER_LABELS: ${{ format('linux-{0}-8gpu-sglang', inputs.runner_arch || 'mi325') }}
RUNNER_LABELS: ${{ format('linux-{0}-8gpu-sglang', inputs.runner_arch || 'mi300') }}
strategy:
fail-fast: false
max-parallel: 1
matrix:
part: [0, 1, 2, 3]
runs-on: ${{ format('linux-{0}-8gpu-sglang', inputs.runner_arch || 'mi325') }}
runs-on: ${{ format('linux-{0}-8gpu-sglang', inputs.runner_arch || 'mi300') }}
steps:
- name: Checkout code
uses: actions/checkout@v4