diff --git a/.github/workflows/nightly-test-amd-rocm720.yml b/.github/workflows/nightly-test-amd-rocm720.yml index 6808f2f20..8af8e9838 100644 --- a/.github/workflows/nightly-test-amd-rocm720.yml +++ b/.github/workflows/nightly-test-amd-rocm720.yml @@ -144,7 +144,7 @@ jobs: nightly-test-1-gpu-unit-rocm720: if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-test-1-gpu-unit-rocm720,')) - runs-on: linux-mi325-1gpu-sglang + runs-on: linux-mi300-1gpu-sglang steps: - name: Checkout code uses: actions/checkout@v4 @@ -174,7 +174,7 @@ jobs: nightly-test-1-gpu-kernel-rocm720: if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-test-1-gpu-kernel-rocm720,')) - runs-on: linux-mi325-1gpu-sglang + runs-on: linux-mi300-1gpu-sglang steps: - name: Checkout code uses: actions/checkout@v4 @@ -239,7 +239,7 @@ jobs: nightly-accuracy-2-gpu-rocm720: if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-accuracy-2-gpu-rocm720,')) - runs-on: linux-mi325-2gpu-sglang + runs-on: linux-mi300-2gpu-sglang steps: - name: Checkout code uses: actions/checkout@v4 @@ -338,7 +338,7 @@ jobs: nightly-accuracy-2-gpu-vlm-rocm720: if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-accuracy-2-gpu-vlm-rocm720,')) - runs-on: linux-mi325-2gpu-sglang + runs-on: linux-mi300-2gpu-sglang steps: - name: Checkout code uses: actions/checkout@v4 @@ -369,7 +369,7 @@ jobs: nightly-perf-2-gpu-text-rocm720: if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-perf-2-gpu-text-rocm720,')) - runs-on: linux-mi325-2gpu-sglang + runs-on: linux-mi300-2gpu-sglang steps: - name: Checkout code uses: actions/checkout@v4 @@ -401,7 +401,7 @@ jobs: nightly-perf-2-gpu-vlm-rocm720: if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-perf-2-gpu-vlm-rocm720,')) - runs-on: linux-mi325-2gpu-sglang + runs-on: linux-mi300-2gpu-sglang steps: - name: Checkout code uses: actions/checkout@v4 @@ -433,7 +433,7 @@ jobs: nightly-4-gpu-rocm720: if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-4-gpu-rocm720,')) - runs-on: linux-mi325-4gpu-sglang + runs-on: linux-mi300-4gpu-sglang steps: - name: Checkout code uses: actions/checkout@v4 @@ -469,7 +469,7 @@ jobs: nightly-accuracy-8-gpu-rocm720: if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-accuracy-8-gpu-rocm720,')) - runs-on: linux-mi325-8gpu-sglang + runs-on: linux-mi300-8gpu-sglang steps: - name: Checkout code uses: actions/checkout@v4 @@ -548,7 +548,7 @@ jobs: nightly-8-gpu-grok1-int4-rocm720: if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-grok1-int4-rocm720,')) - runs-on: linux-mi325-8gpu-sglang + runs-on: linux-mi300-8gpu-sglang steps: - name: Checkout code uses: actions/checkout@v4 @@ -645,7 +645,7 @@ jobs: nightly-8-gpu-grok2-rocm720: if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-grok2-rocm720,')) - runs-on: linux-mi325-8gpu-sglang + runs-on: linux-mi300-8gpu-sglang steps: - name: Checkout code uses: actions/checkout@v4 @@ -742,7 +742,7 @@ jobs: nightly-8-gpu-deepseek-v31-rocm720: if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-deepseek-v31-rocm720,')) - runs-on: linux-mi325-8gpu-sglang + runs-on: linux-mi300-8gpu-sglang steps: - name: Checkout code uses: actions/checkout@v4 @@ -788,7 +788,7 @@ jobs: nightly-8-gpu-deepseek-v32-rocm720: if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-deepseek-v32-rocm720,')) - runs-on: linux-mi325-8gpu-sglang + runs-on: linux-mi300-8gpu-sglang steps: - name: Checkout code uses: actions/checkout@v4 @@ -832,7 +832,7 @@ jobs: nightly-8-gpu-deepseek-v32-mtp-rocm720: if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-deepseek-v32-mtp-rocm720,')) - runs-on: linux-mi325-8gpu-sglang + runs-on: linux-mi300-8gpu-sglang steps: - name: Checkout code uses: actions/checkout@v4 @@ -876,7 +876,7 @@ jobs: nightly-8-gpu-deepseek-v3-kv-fp8-rocm720: if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-deepseek-v3-kv-fp8-rocm720,')) - runs-on: linux-mi325-8gpu-sglang + runs-on: linux-mi300-8gpu-sglang steps: - name: Checkout code uses: actions/checkout@v4 @@ -1437,7 +1437,7 @@ jobs: nightly-8-gpu-kimi-k26-rocm720: if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-kimi-k26-rocm720,')) - runs-on: linux-mi325-8gpu-sglang + runs-on: linux-mi300-8gpu-sglang steps: - name: Checkout code uses: actions/checkout@v4 @@ -1510,7 +1510,7 @@ jobs: nightly-8-gpu-qwen3-235b-rocm720: if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-qwen3-235b-rocm720,')) - runs-on: linux-mi325-8gpu-sglang + runs-on: linux-mi300-8gpu-sglang steps: - name: Checkout code uses: actions/checkout@v4 @@ -1582,7 +1582,7 @@ jobs: nightly-8-gpu-qwen35-rocm720: if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-qwen35-rocm720,')) - runs-on: linux-mi325-8gpu-sglang + runs-on: linux-mi300-8gpu-sglang steps: - name: Checkout code uses: actions/checkout@v4 @@ -1714,7 +1714,7 @@ jobs: nightly-8-gpu-glm51-rocm720: if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-glm51-rocm720,')) - runs-on: linux-mi325-8gpu-sglang + runs-on: linux-mi300-8gpu-sglang steps: - name: Checkout code uses: actions/checkout@v4 @@ -1936,7 +1936,7 @@ jobs: nightly-8-gpu-minimax-m27-rocm720: if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-minimax-m27-rocm720,')) - runs-on: linux-mi325-8gpu-sglang + runs-on: linux-mi300-8gpu-sglang steps: - name: Checkout code uses: actions/checkout@v4 @@ -1986,7 +1986,7 @@ jobs: nightly-1-gpu-zimage-turbo-rocm720: if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-1-gpu-zimage-turbo-rocm720,')) - runs-on: linux-mi325-1gpu-sglang + runs-on: linux-mi300-1gpu-sglang steps: - name: Checkout code uses: actions/checkout@v4 diff --git a/.github/workflows/nightly-test-amd.yml b/.github/workflows/nightly-test-amd.yml index 73b4dcd8b..da995a2c3 100644 --- a/.github/workflows/nightly-test-amd.yml +++ b/.github/workflows/nightly-test-amd.yml @@ -140,7 +140,7 @@ jobs: nightly-test-1-gpu-unit: if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-test-1-gpu-unit,')) - runs-on: linux-mi325-1gpu-sglang + runs-on: linux-mi300-1gpu-sglang steps: - name: Checkout code uses: actions/checkout@v4 @@ -171,7 +171,7 @@ jobs: nightly-test-1-gpu-kernel: if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-test-1-gpu-kernel,')) - runs-on: linux-mi325-1gpu-sglang + runs-on: linux-mi300-1gpu-sglang steps: - name: Checkout code uses: actions/checkout@v4 @@ -240,7 +240,7 @@ jobs: nightly-accuracy-2-gpu: if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-accuracy-2-gpu,')) - runs-on: linux-mi325-2gpu-sglang + runs-on: linux-mi300-2gpu-sglang steps: - name: Checkout code uses: actions/checkout@v4 @@ -340,7 +340,7 @@ jobs: nightly-accuracy-2-gpu-vlm: if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-accuracy-2-gpu-vlm,')) - runs-on: linux-mi325-2gpu-sglang + runs-on: linux-mi300-2gpu-sglang steps: - name: Checkout code uses: actions/checkout@v4 @@ -372,7 +372,7 @@ jobs: nightly-perf-2-gpu-text: if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-perf-2-gpu-text,')) - runs-on: linux-mi325-2gpu-sglang + runs-on: linux-mi300-2gpu-sglang steps: - name: Checkout code uses: actions/checkout@v4 @@ -405,7 +405,7 @@ jobs: nightly-perf-2-gpu-vlm: if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-perf-2-gpu-vlm,')) - runs-on: linux-mi325-2gpu-sglang + runs-on: linux-mi300-2gpu-sglang steps: - name: Checkout code uses: actions/checkout@v4 @@ -438,7 +438,7 @@ jobs: nightly-4-gpu: if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-4-gpu,')) - runs-on: linux-mi325-4gpu-sglang + runs-on: linux-mi300-4gpu-sglang steps: - name: Checkout code uses: actions/checkout@v4 @@ -474,7 +474,7 @@ jobs: nightly-accuracy-8-gpu: if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-accuracy-8-gpu,')) - runs-on: linux-mi325-8gpu-sglang + runs-on: linux-mi300-8gpu-sglang steps: - name: Checkout code uses: actions/checkout@v4 @@ -553,7 +553,7 @@ jobs: nightly-8-gpu-grok1-int4: if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-grok1-int4,')) - runs-on: linux-mi325-8gpu-sglang + runs-on: linux-mi300-8gpu-sglang steps: - name: Checkout code uses: actions/checkout@v4 @@ -650,7 +650,7 @@ jobs: nightly-8-gpu-grok2: if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-grok2,')) - runs-on: linux-mi325-8gpu-sglang + runs-on: linux-mi300-8gpu-sglang steps: - name: Checkout code uses: actions/checkout@v4 @@ -747,7 +747,7 @@ jobs: nightly-8-gpu-deepseek-v31: if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-deepseek-v31,')) - runs-on: linux-mi325-8gpu-sglang + runs-on: linux-mi300-8gpu-sglang steps: - name: Checkout code uses: actions/checkout@v4 @@ -793,7 +793,7 @@ jobs: nightly-8-gpu-deepseek-v32: if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-deepseek-v32,')) - runs-on: linux-mi325-8gpu-sglang + runs-on: linux-mi300-8gpu-sglang steps: - name: Checkout code uses: actions/checkout@v4 @@ -837,7 +837,7 @@ jobs: nightly-8-gpu-deepseek-v32-mtp: if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-deepseek-v32-mtp,')) - runs-on: linux-mi325-8gpu-sglang + runs-on: linux-mi300-8gpu-sglang steps: - name: Checkout code uses: actions/checkout@v4 @@ -881,7 +881,7 @@ jobs: nightly-8-gpu-deepseek-v3-kv-fp8: if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-deepseek-v3-kv-fp8,')) - runs-on: linux-mi325-8gpu-sglang + runs-on: linux-mi300-8gpu-sglang steps: - name: Checkout code uses: actions/checkout@v4 @@ -1291,7 +1291,7 @@ jobs: nightly-8-gpu-kimi-k26: if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-kimi-k26,')) - runs-on: linux-mi325-8gpu-sglang + runs-on: linux-mi300-8gpu-sglang steps: - name: Checkout code uses: actions/checkout@v4 @@ -1364,7 +1364,7 @@ jobs: nightly-8-gpu-qwen3-235b: if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-qwen3-235b,')) - runs-on: linux-mi325-8gpu-sglang + runs-on: linux-mi300-8gpu-sglang steps: - name: Checkout code uses: actions/checkout@v4 @@ -1436,7 +1436,7 @@ jobs: nightly-8-gpu-qwen35: if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-qwen35,')) - runs-on: linux-mi325-8gpu-sglang + runs-on: linux-mi300-8gpu-sglang steps: - name: Checkout code uses: actions/checkout@v4 @@ -1568,7 +1568,7 @@ jobs: nightly-8-gpu-glm51: if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-glm51,')) - runs-on: linux-mi325-8gpu-sglang + runs-on: linux-mi300-8gpu-sglang steps: - name: Checkout code uses: actions/checkout@v4 @@ -1790,7 +1790,7 @@ jobs: nightly-8-gpu-minimax-m27: if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-minimax-m27,')) - runs-on: linux-mi325-8gpu-sglang + runs-on: linux-mi300-8gpu-sglang steps: - name: Checkout code uses: actions/checkout@v4 @@ -1840,7 +1840,7 @@ jobs: nightly-1-gpu-zimage-turbo: if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-1-gpu-zimage-turbo,')) - runs-on: linux-mi325-1gpu-sglang + runs-on: linux-mi300-1gpu-sglang steps: - name: Checkout code uses: actions/checkout@v4 diff --git a/.github/workflows/pr-test-amd-extra.yml b/.github/workflows/pr-test-amd-extra.yml index 35f5c99fe..1f0d440eb 100644 --- a/.github/workflows/pr-test-amd-extra.yml +++ b/.github/workflows/pr-test-amd-extra.yml @@ -32,7 +32,7 @@ on: description: 'AMD runner pool to dispatch GPU jobs to' required: false type: choice - default: mi325 + default: mi300 options: - mi300 - mi325 @@ -61,6 +61,11 @@ on: required: false type: string default: '' + runner_arch: + description: 'AMD runner pool to dispatch GPU jobs to' + required: false + type: string + default: mi300 rocm_version: description: 'ROCm container variant (empty = Dockerfile default; rocm720 = ROCm 7.2.0)' required: false @@ -125,10 +130,10 @@ jobs: # setup across scarce AMD GPUs to shave only a couple minutes of test time, # so one GPU running the whole suite sequentially is the better trade. extra-a-test-1-gpu-small-amd: - name: ${{ format('extra-a-test-1-gpu-small-amd{0} (linux-{1}-1gpu-sglang)', inputs.rocm_version && format('-{0}', inputs.rocm_version) || '', inputs.runner_arch || 'mi325') }} + name: ${{ format('extra-a-test-1-gpu-small-amd{0} (linux-{1}-1gpu-sglang)', inputs.rocm_version && format('-{0}', inputs.rocm_version) || '', inputs.runner_arch || 'mi300') }} needs: [call-gate] if: ${{ !cancelled() && needs.call-gate.result == 'success' }} - runs-on: ${{ format('linux-{0}-1gpu-sglang', inputs.runner_arch || 'mi325') }} + runs-on: ${{ format('linux-{0}-1gpu-sglang', inputs.runner_arch || 'mi300') }} steps: - name: Checkout code uses: actions/checkout@v4 @@ -157,10 +162,10 @@ jobs: # pool as small (AMD GPUs carry enough VRAM that "large" here is a CUDA # memory-tier label, not a separate AMD runner pool). extra-a-test-1-gpu-large-amd: - name: ${{ format('extra-a-test-1-gpu-large-amd{0} (linux-{1}-1gpu-sglang)', inputs.rocm_version && format('-{0}', inputs.rocm_version) || '', inputs.runner_arch || 'mi325') }} + name: ${{ format('extra-a-test-1-gpu-large-amd{0} (linux-{1}-1gpu-sglang)', inputs.rocm_version && format('-{0}', inputs.rocm_version) || '', inputs.runner_arch || 'mi300') }} needs: [call-gate] if: ${{ !cancelled() && needs.call-gate.result == 'success' }} - runs-on: ${{ format('linux-{0}-1gpu-sglang', inputs.runner_arch || 'mi325') }} + runs-on: ${{ format('linux-{0}-1gpu-sglang', inputs.runner_arch || 'mi300') }} steps: - name: Checkout code uses: actions/checkout@v4 @@ -187,10 +192,10 @@ jobs: # Multi-GPU TP / PP / PD mock-model + kv_canary e2e tests. Mirrors CUDA's # extra-a 2-gpu-large stage; runs on the 2-GPU AMD pool. extra-a-test-2-gpu-large-amd: - name: ${{ format('extra-a-test-2-gpu-large-amd{0} (linux-{1}-2gpu-sglang)', inputs.rocm_version && format('-{0}', inputs.rocm_version) || '', inputs.runner_arch || 'mi325') }} + name: ${{ format('extra-a-test-2-gpu-large-amd{0} (linux-{1}-2gpu-sglang)', inputs.rocm_version && format('-{0}', inputs.rocm_version) || '', inputs.runner_arch || 'mi300') }} needs: [call-gate] if: ${{ !cancelled() && needs.call-gate.result == 'success' }} - runs-on: ${{ format('linux-{0}-2gpu-sglang', inputs.runner_arch || 'mi325') }} + runs-on: ${{ format('linux-{0}-2gpu-sglang', inputs.runner_arch || 'mi300') }} steps: - name: Checkout code uses: actions/checkout@v4 diff --git a/.github/workflows/pr-test-amd-rocm720.yml b/.github/workflows/pr-test-amd-rocm720.yml index d437e13e8..eb92573cf 100644 --- a/.github/workflows/pr-test-amd-rocm720.yml +++ b/.github/workflows/pr-test-amd-rocm720.yml @@ -229,6 +229,7 @@ jobs: uses: ./.github/workflows/pr-test-amd-extra.yml with: ref: ${{ inputs.pr_head_sha || inputs.ref || '' }} + runner_arch: mi300 rocm_version: rocm720 aiter_ref: ${{ inputs.aiter_ref }} continue_on_error: true @@ -249,7 +250,7 @@ jobs: strategy: fail-fast: false matrix: - runner: [linux-mi325-1gpu-sglang] + runner: [linux-mi300-1gpu-sglang] runs-on: ${{matrix.runner}} steps: - name: Checkout code @@ -295,7 +296,7 @@ jobs: strategy: fail-fast: false matrix: - runner: [linux-mi325-2gpu-sglang] + runner: [linux-mi300-2gpu-sglang] runs-on: ${{matrix.runner}} steps: - name: Checkout code @@ -336,7 +337,7 @@ jobs: strategy: fail-fast: false matrix: - runner: [linux-mi325-1gpu-sglang] + runner: [linux-mi300-1gpu-sglang] runs-on: ${{matrix.runner}} steps: - name: Checkout code @@ -374,7 +375,7 @@ jobs: strategy: fail-fast: false matrix: - runner: [linux-mi325-1gpu-sglang] + runner: [linux-mi300-1gpu-sglang] runs-on: ${{matrix.runner}} steps: - name: Checkout code @@ -412,7 +413,7 @@ jobs: strategy: fail-fast: false matrix: - runner: [linux-mi325-1gpu-sglang] + runner: [linux-mi300-1gpu-sglang] runs-on: ${{matrix.runner}} steps: - name: Checkout code @@ -450,8 +451,9 @@ jobs: ) strategy: fail-fast: false + max-parallel: 4 matrix: - runner: [linux-mi325-1gpu-sglang] + runner: [linux-mi300-1gpu-sglang] part: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13] runs-on: ${{matrix.runner}} steps: @@ -471,9 +473,9 @@ jobs: - name: Install dependencies run: bash scripts/ci/amd/amd_ci_install_dependency.sh - name: Run test - timeout-minutes: 60 + timeout-minutes: 75 run: | - bash scripts/ci/amd/amd_ci_exec.sh -w "/sglang-checkout/test" python3 run_suite.py --hw amd --suite stage-b-test-1-gpu-small-amd --auto-partition-id ${{ matrix.part }} --auto-partition-size 14 --timeout-per-file 1800 ${{ needs.check-changes.outputs.continue_on_error == 'true' && '--continue-on-error' || '' }} + bash scripts/ci/amd/amd_ci_exec.sh -w "/sglang-checkout/test" python3 run_suite.py --hw amd --suite stage-b-test-1-gpu-small-amd --auto-partition-id ${{ matrix.part }} --auto-partition-size 14 --timeout-per-file 2400 ${{ needs.check-changes.outputs.continue_on_error == 'true' && '--continue-on-error' || '' }} stage-b-test-1-gpu-small-amd-nondeterministic-rocm720: needs: [check-changes] @@ -490,7 +492,7 @@ jobs: strategy: fail-fast: false matrix: - runner: [linux-mi325-1gpu-sglang] + runner: [linux-mi300-1gpu-sglang] runs-on: ${{matrix.runner}} steps: - name: Checkout code @@ -565,8 +567,9 @@ jobs: ) strategy: fail-fast: false + max-parallel: 1 matrix: - runner: [linux-mi325-1gpu-sglang] + runner: [linux-mi300-1gpu-sglang] part: [0, 1, 2] runs-on: ${{matrix.runner}} steps: @@ -586,9 +589,9 @@ jobs: - name: Install dependencies run: bash scripts/ci/amd/amd_ci_install_dependency.sh - name: Run test - timeout-minutes: 45 + timeout-minutes: 60 run: | - bash scripts/ci/amd/amd_ci_exec.sh -w "/sglang-checkout/test" python3 run_suite.py --hw amd --suite stage-b-test-1-gpu-large-amd --auto-partition-id ${{ matrix.part }} --auto-partition-size 3 --timeout-per-file 1800 ${{ needs.check-changes.outputs.continue_on_error == 'true' && '--continue-on-error' || '' }} + bash scripts/ci/amd/amd_ci_exec.sh -w "/sglang-checkout/test" python3 run_suite.py --hw amd --suite stage-b-test-1-gpu-large-amd --auto-partition-id ${{ matrix.part }} --auto-partition-size 3 --timeout-per-file 2700 ${{ needs.check-changes.outputs.continue_on_error == 'true' && '--continue-on-error' || '' }} stage-b-test-2-gpu-large-amd-rocm720: needs: [check-changes] @@ -604,8 +607,9 @@ jobs: ) strategy: fail-fast: false + max-parallel: 1 matrix: - runner: [linux-mi325-2gpu-sglang] + runner: [linux-mi300-2gpu-sglang] part: [0, 1] runs-on: ${{matrix.runner}} steps: @@ -625,9 +629,9 @@ jobs: - name: Install dependencies run: bash scripts/ci/amd/amd_ci_install_dependency.sh - name: Run test - timeout-minutes: 45 + timeout-minutes: 120 run: | - bash scripts/ci/amd/amd_ci_exec.sh -w "/sglang-checkout/test" python3 run_suite.py --hw amd --suite stage-b-test-2-gpu-large-amd --auto-partition-id ${{ matrix.part }} --auto-partition-size 2 --timeout-per-file 1800 ${{ needs.check-changes.outputs.continue_on_error == 'true' && '--continue-on-error' || '' }} + bash scripts/ci/amd/amd_ci_exec.sh -w "/sglang-checkout/test" python3 run_suite.py --hw amd --suite stage-b-test-2-gpu-large-amd --auto-partition-id ${{ matrix.part }} --auto-partition-size 2 --timeout-per-file 5400 ${{ needs.check-changes.outputs.continue_on_error == 'true' && '--continue-on-error' || '' }} multimodal-gen-test-1-gpu-amd-rocm720: needs: [check-changes] @@ -643,8 +647,9 @@ jobs: ) strategy: fail-fast: false + max-parallel: 1 matrix: - runner: [linux-mi325-1gpu-sglang] + runner: [linux-mi300-1gpu-sglang] part: [0, 1, 2, 3] runs-on: ${{matrix.runner}} steps: @@ -676,16 +681,7 @@ jobs: - name: Setup kernel caches run: | - # Use the persistent /sgl-data directory (mounted from /home/runner/sgl-data) - # This directory persists across container restarts on the self-hosted runner - docker exec ci_sglang mkdir -p /sgl-data/aiter-kernels /sgl-data/miopen-cache /sgl-data/hf-cache/hub - - # Clear pre-built AITER kernels from Docker image to avoid segfaults - # The image may have stale/incompatible kernels at /sgl-workspace/aiter/aiter/jit/ - echo "Clearing pre-built AITER kernels from Docker image..." - docker exec ci_sglang rm -rf /sgl-workspace/aiter/aiter/jit/*.so 2>/dev/null || true - docker exec ci_sglang rm -rf /sgl-data/aiter-kernels/*.so 2>/dev/null || true - echo "AITER kernels cleared - will be rebuilt on first use" + docker exec ci_sglang mkdir -p /sgl-data/miopen-cache /sgl-data/hf-cache/hub # Create persistent cache marker if /sgl-data is a real mount (not ephemeral) # This tells the test cleanup code to NOT delete downloaded models @@ -706,7 +702,7 @@ jobs: free -h echo "" echo "=== Disk Space ===" - df -h /home/runner/sgl-data 2>/dev/null || df -h + df -h /home/runner/sglang-data 2>/dev/null || df -h echo "" echo "=== HF Cache Directory Structure ===" docker exec ci_sglang ls -la /sgl-data/hf-cache/ 2>/dev/null || echo "HF cache dir not found" @@ -728,7 +724,7 @@ jobs: docker exec ci_sglang rocm-smi --showmeminfo vram 2>/dev/null || echo "rocm-smi not available" - name: Run diffusion server tests (1-GPU) - timeout-minutes: 60 + timeout-minutes: 90 run: | # AMD CI: All 1-GPU tests except FLUX.2 (FLUX.1 covers same code path) # Tests: T2V, T2I, I2V, LoRA @@ -745,7 +741,6 @@ jobs: -e SGLANG_SKIP_CONSISTENCY=1 \ -e SGLANG_TEST_NUM_INFERENCE_STEPS=5 \ -e SGLANG_DIFFUSION_ARTIFACT_DIR=/sglang-checkout/diffusion-failures \ - -e AITER_JIT_DIR=/sgl-data/aiter-kernels \ -e MIOPEN_USER_DB_PATH=/sgl-data/miopen-cache \ -e HF_HUB_ENABLE_HF_TRANSFER=1 \ -e HF_HUB_DISABLE_SYMLINKS_WARNING=1 \ @@ -783,8 +778,9 @@ jobs: ) strategy: fail-fast: false + max-parallel: 1 matrix: - runner: [linux-mi325-2gpu-sglang] + runner: [linux-mi300-2gpu-sglang] part: [0, 1, 2] # 3 partitions: 2 parametrized + 1 standalone (single_test_file/test_disagg_server.py) runs-on: ${{matrix.runner}} steps: @@ -816,15 +812,7 @@ jobs: - name: Setup kernel caches run: | - # Use the persistent /sgl-data directory (mounted from /home/runner/sgl-data) - docker exec ci_sglang mkdir -p /sgl-data/aiter-kernels /sgl-data/miopen-cache /sgl-data/hf-cache/hub - - # Clear pre-built AITER kernels from Docker image to avoid segfaults - # The image may have stale/incompatible kernels at /sgl-workspace/aiter/aiter/jit/ - echo "Clearing pre-built AITER kernels from Docker image..." - docker exec ci_sglang rm -rf /sgl-workspace/aiter/aiter/jit/*.so 2>/dev/null || true - docker exec ci_sglang rm -rf /sgl-data/aiter-kernels/*.so 2>/dev/null || true - echo "AITER kernels cleared - will be rebuilt on first use" + docker exec ci_sglang mkdir -p /sgl-data/miopen-cache /sgl-data/hf-cache/hub # Create persistent cache marker if /sgl-data is a real mount (not ephemeral) # This tells the test cleanup code to NOT delete downloaded models @@ -845,7 +833,7 @@ jobs: free -h echo "" echo "=== Disk Space ===" - df -h /home/runner/sgl-data 2>/dev/null || df -h + df -h /home/runner/sglang-data 2>/dev/null || df -h echo "" echo "=== HF Cache Directory Structure ===" docker exec ci_sglang ls -la /sgl-data/hf-cache/ 2>/dev/null || echo "HF cache dir not found" @@ -867,7 +855,7 @@ jobs: docker exec ci_sglang rocm-smi --showmeminfo vram 2>/dev/null || echo "rocm-smi not available" - name: Run diffusion server tests (2-GPU) - timeout-minutes: 150 + timeout-minutes: 180 run: | # AMD CI: All 2-GPU tests including LoRA # Tests: T2V, T2I, I2V, LoRA @@ -884,7 +872,6 @@ jobs: -e SGLANG_SKIP_CONSISTENCY=1 \ -e SGLANG_TEST_NUM_INFERENCE_STEPS=5 \ -e SGLANG_DIFFUSION_ARTIFACT_DIR=/sglang-checkout/diffusion-failures \ - -e AITER_JIT_DIR=/sgl-data/aiter-kernels \ -e MIOPEN_USER_DB_PATH=/sgl-data/miopen-cache \ -e HF_HUB_ENABLE_HF_TRANSFER=1 \ -e HF_HUB_DISABLE_SYMLINKS_WARNING=1 \ @@ -923,7 +910,7 @@ jobs: ((needs.check-changes.outputs.main_package == 'true') || (needs.check-changes.outputs.sgl_kernel == 'true')) ) ) - runs-on: linux-mi325-1gpu-sglang + runs-on: linux-mi300-1gpu-sglang steps: - name: Checkout code uses: actions/checkout@v4 @@ -957,7 +944,6 @@ jobs: # Skip ltx2_vae_channels_last: it asserts CUDA `channels_last_3d` # memory-format behavior that the ROCm conv path doesn't reproduce. docker exec \ - -e AITER_JIT_DIR=/sgl-data/aiter-kernels \ -e MIOPEN_USER_DB_PATH=/sgl-data/miopen-cache \ -w /sglang-checkout/python \ ci_sglang python3 sglang/multimodal_gen/test/run_suite.py \ @@ -979,7 +965,7 @@ jobs: strategy: fail-fast: false matrix: - runner: [linux-mi325-4gpu-sglang] + runner: [linux-mi300-4gpu-sglang] part: [0] runs-on: ${{matrix.runner}} steps: @@ -1000,7 +986,7 @@ jobs: run: bash scripts/ci/amd/amd_ci_install_dependency.sh - name: Run test - timeout-minutes: 60 + timeout-minutes: 90 run: | bash scripts/ci/amd/amd_ci_exec.sh \ -e NCCL_CUMEM_ENABLE=0 \ @@ -1013,7 +999,7 @@ jobs: --suite stage-c-test-4-gpu-amd \ --auto-partition-id ${{ matrix.part }} \ --auto-partition-size 1 \ - --timeout-per-file 1800 \ + --timeout-per-file 5400 \ --enable-retry \ --max-attempts 2 \ --retry-wait-seconds 120 \ @@ -1033,11 +1019,12 @@ jobs: ) ) env: - RUNNER_LABELS: linux-mi325-8gpu-sglang + RUNNER_LABELS: linux-mi300-8gpu-sglang strategy: fail-fast: false + max-parallel: 1 matrix: - runner: [linux-mi325-8gpu-sglang] + runner: [linux-mi300-8gpu-sglang] part: [0, 1, 2, 3] runs-on: ${{matrix.runner}} steps: diff --git a/.github/workflows/pr-test-amd.yml b/.github/workflows/pr-test-amd.yml index 4e5320805..7dba7448d 100644 --- a/.github/workflows/pr-test-amd.yml +++ b/.github/workflows/pr-test-amd.yml @@ -64,7 +64,7 @@ on: description: 'AMD runner pool to dispatch GPU jobs to' required: false type: choice - default: mi325 + default: mi300 options: - mi300 - mi325 @@ -80,6 +80,11 @@ on: required: false type: string default: '' + runner_arch: + description: 'AMD runner pool to dispatch GPU jobs to' + required: false + type: string + default: mi300 run_all_tests: description: "Run all tests (for releasing or testing purpose)" required: false @@ -217,13 +222,14 @@ jobs: uses: ./.github/workflows/pr-test-amd-extra.yml with: ref: ${{ inputs.pr_head_sha || inputs.ref || '' }} + runner_arch: ${{ inputs.runner_arch || 'mi300' }} aiter_ref: ${{ inputs.aiter_ref }} continue_on_error: true secrets: inherit # =============================================== sgl-kernel ==================================================== sgl-kernel-unit-test-amd: - name: ${{ format('sgl-kernel-unit-test-amd (linux-{0}-1gpu-sglang)', inputs.runner_arch || 'mi325') }} + name: ${{ format('sgl-kernel-unit-test-amd (linux-{0}-1gpu-sglang)', inputs.runner_arch || 'mi300') }} needs: [check-changes, call-gate] if: | always() && !cancelled() && @@ -235,7 +241,7 @@ jobs: needs.check-changes.outputs.sgl_kernel == 'true' ) ) - runs-on: ${{ format('linux-{0}-1gpu-sglang', inputs.runner_arch || 'mi325') }} + runs-on: ${{ format('linux-{0}-1gpu-sglang', inputs.runner_arch || 'mi300') }} steps: - name: Checkout code uses: actions/checkout@v4 @@ -281,7 +287,7 @@ jobs: exit $failures sgl-kernel-unit-test-2-gpu-amd: - name: ${{ format('sgl-kernel-unit-test-2-gpu-amd (linux-{0}-2gpu-sglang)', inputs.runner_arch || 'mi325') }} + name: ${{ format('sgl-kernel-unit-test-2-gpu-amd (linux-{0}-2gpu-sglang)', inputs.runner_arch || 'mi300') }} needs: [check-changes, call-gate] if: | always() && !cancelled() && @@ -293,7 +299,7 @@ jobs: needs.check-changes.outputs.sgl_kernel == 'true' ) ) - runs-on: ${{ format('linux-{0}-2gpu-sglang', inputs.runner_arch || 'mi325') }} + runs-on: ${{ format('linux-{0}-2gpu-sglang', inputs.runner_arch || 'mi300') }} steps: - name: Checkout code uses: actions/checkout@v4 @@ -322,7 +328,7 @@ jobs: # =============================================== primary ==================================================== stage-a-test-1-gpu-small-amd: - name: ${{ format('stage-a-test-1-gpu-small-amd (linux-{0}-1gpu-sglang)', inputs.runner_arch || 'mi325') }} + name: ${{ format('stage-a-test-1-gpu-small-amd (linux-{0}-1gpu-sglang)', inputs.runner_arch || 'mi300') }} needs: [check-changes, call-gate] if: | always() && !cancelled() && @@ -334,7 +340,7 @@ jobs: ((needs.check-changes.outputs.main_package == 'true') || (needs.check-changes.outputs.sgl_kernel == 'true')) ) ) - runs-on: ${{ format('linux-{0}-1gpu-sglang', inputs.runner_arch || 'mi325') }} + runs-on: ${{ format('linux-{0}-1gpu-sglang', inputs.runner_arch || 'mi300') }} steps: - name: Checkout code uses: actions/checkout@v4 @@ -359,7 +365,7 @@ jobs: bash scripts/ci/amd/amd_ci_exec.sh -w "/sglang-checkout/test" python3 run_suite.py --hw amd --suite stage-a-test-1-gpu-small-amd ${{ needs.check-changes.outputs.continue_on_error == 'true' && '--continue-on-error' || '' }} jit-kernel-unit-test-amd: - name: ${{ format('jit-kernel-unit-test-amd (linux-{0}-1gpu-sglang)', inputs.runner_arch || 'mi325') }} + name: ${{ format('jit-kernel-unit-test-amd (linux-{0}-1gpu-sglang)', inputs.runner_arch || 'mi300') }} needs: [check-changes, call-gate] if: | always() && !cancelled() && @@ -371,7 +377,7 @@ jobs: needs.check-changes.outputs.jit_kernel == 'true' ) ) - runs-on: ${{ format('linux-{0}-1gpu-sglang', inputs.runner_arch || 'mi325') }} + runs-on: ${{ format('linux-{0}-1gpu-sglang', inputs.runner_arch || 'mi300') }} steps: - name: Checkout code uses: actions/checkout@v4 @@ -396,7 +402,7 @@ jobs: bash scripts/ci/amd/amd_ci_exec.sh -w "/sglang-checkout/test" python3 run_suite.py --hw amd --suite jit-kernel-unit-test-amd ${{ needs.check-changes.outputs.continue_on_error == 'true' && '--continue-on-error' || '' }} jit-kernel-benchmark-test-amd: - name: ${{ format('jit-kernel-benchmark-test-amd (linux-{0}-1gpu-sglang)', inputs.runner_arch || 'mi325') }} + name: ${{ format('jit-kernel-benchmark-test-amd (linux-{0}-1gpu-sglang)', inputs.runner_arch || 'mi300') }} needs: [check-changes, call-gate] if: | always() && !cancelled() && @@ -408,7 +414,7 @@ jobs: needs.check-changes.outputs.jit_kernel == 'true' ) ) - runs-on: ${{ format('linux-{0}-1gpu-sglang', inputs.runner_arch || 'mi325') }} + runs-on: ${{ format('linux-{0}-1gpu-sglang', inputs.runner_arch || 'mi300') }} steps: - name: Checkout code uses: actions/checkout@v4 @@ -459,7 +465,7 @@ jobs: max-wait-minutes: '240' stage-b-test-1-gpu-small-amd: - name: ${{ format('stage-b-test-1-gpu-small-amd (linux-{0}-1gpu-sglang, {1})', inputs.runner_arch || 'mi325', matrix.part) }} + name: ${{ format('stage-b-test-1-gpu-small-amd (linux-{0}-1gpu-sglang, {1})', inputs.runner_arch || 'mi300', matrix.part) }} needs: [check-changes, wait-for-stage-a-amd] if: | always() && @@ -473,9 +479,10 @@ jobs: ) strategy: fail-fast: false + max-parallel: 4 matrix: part: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13] - runs-on: ${{ format('linux-{0}-1gpu-sglang', inputs.runner_arch || 'mi325') }} + runs-on: ${{ format('linux-{0}-1gpu-sglang', inputs.runner_arch || 'mi300') }} steps: - name: Checkout code uses: actions/checkout@v4 @@ -494,12 +501,12 @@ jobs: run: bash scripts/ci/amd/amd_ci_install_dependency.sh - name: Run test - timeout-minutes: 60 + timeout-minutes: ${{ (inputs.runner_arch || 'mi300') == 'mi300' && 75 || 60 }} run: | bash scripts/ci/amd/amd_ci_exec.sh -w "/sglang-checkout/test" python3 run_suite.py --hw amd --suite stage-b-test-1-gpu-small-amd --auto-partition-id ${{ matrix.part }} --auto-partition-size 14 --timeout-per-file 2400 ${{ needs.check-changes.outputs.continue_on_error == 'true' && '--continue-on-error' || '' }} stage-b-test-1-gpu-small-amd-nondeterministic: - name: ${{ format('stage-b-test-1-gpu-small-amd-nondeterministic (linux-{0}-1gpu-sglang)', inputs.runner_arch || 'mi325') }} + name: ${{ format('stage-b-test-1-gpu-small-amd-nondeterministic (linux-{0}-1gpu-sglang)', inputs.runner_arch || 'mi300') }} needs: [check-changes, wait-for-stage-a-amd] if: | always() && @@ -511,7 +518,7 @@ jobs: ((needs.check-changes.outputs.main_package == 'true') || (needs.check-changes.outputs.sgl_kernel == 'true')) ) ) - runs-on: ${{ format('linux-{0}-1gpu-sglang', inputs.runner_arch || 'mi325') }} + runs-on: ${{ format('linux-{0}-1gpu-sglang', inputs.runner_arch || 'mi300') }} steps: - name: Checkout code uses: actions/checkout@v4 @@ -574,7 +581,7 @@ jobs: bash scripts/ci/amd/amd_ci_exec.sh -w "/sglang-checkout/test" python3 run_suite.py --hw amd --suite stage-b-test-1-gpu-small-amd-mi35x ${{ needs.check-changes.outputs.continue_on_error == 'true' && '--continue-on-error' || '' }} stage-b-test-1-gpu-large-amd: - name: ${{ format('stage-b-test-1-gpu-large-amd (linux-{0}-1gpu-sglang, {1})', inputs.runner_arch || 'mi325', matrix.part) }} + name: ${{ format('stage-b-test-1-gpu-large-amd (linux-{0}-1gpu-sglang, {1})', inputs.runner_arch || 'mi300', matrix.part) }} needs: [check-changes, wait-for-stage-a-amd] if: | always() && @@ -588,9 +595,10 @@ jobs: ) strategy: fail-fast: false + max-parallel: 1 matrix: part: [0, 1, 2] - runs-on: ${{ format('linux-{0}-1gpu-sglang', inputs.runner_arch || 'mi325') }} + runs-on: ${{ format('linux-{0}-1gpu-sglang', inputs.runner_arch || 'mi300') }} steps: - name: Checkout code uses: actions/checkout@v4 @@ -609,12 +617,12 @@ jobs: run: bash scripts/ci/amd/amd_ci_install_dependency.sh - name: Run test - timeout-minutes: 45 + timeout-minutes: ${{ (inputs.runner_arch || 'mi300') == 'mi300' && 60 || 45 }} run: | bash scripts/ci/amd/amd_ci_exec.sh -w "/sglang-checkout/test" python3 run_suite.py --hw amd --suite stage-b-test-1-gpu-large-amd --auto-partition-id ${{ matrix.part }} --auto-partition-size 3 --timeout-per-file 2700 ${{ needs.check-changes.outputs.continue_on_error == 'true' && '--continue-on-error' || '' }} stage-b-test-2-gpu-large-amd: - name: ${{ format('stage-b-test-2-gpu-large-amd (linux-{0}-2gpu-sglang, {1})', inputs.runner_arch || 'mi325', matrix.part) }} + name: ${{ format('stage-b-test-2-gpu-large-amd (linux-{0}-2gpu-sglang, {1})', inputs.runner_arch || 'mi300', matrix.part) }} needs: [check-changes, wait-for-stage-a-amd] if: | always() && @@ -628,9 +636,10 @@ jobs: ) strategy: fail-fast: false + max-parallel: 1 matrix: part: [0, 1] - runs-on: ${{ format('linux-{0}-2gpu-sglang', inputs.runner_arch || 'mi325') }} + runs-on: ${{ format('linux-{0}-2gpu-sglang', inputs.runner_arch || 'mi300') }} steps: - name: Checkout code uses: actions/checkout@v4 @@ -649,12 +658,12 @@ jobs: run: bash scripts/ci/amd/amd_ci_install_dependency.sh - name: Run test - timeout-minutes: 45 + timeout-minutes: ${{ (inputs.runner_arch || 'mi300') == 'mi300' && 120 || 45 }} run: | - bash scripts/ci/amd/amd_ci_exec.sh -w "/sglang-checkout/test" python3 run_suite.py --hw amd --suite stage-b-test-2-gpu-large-amd --auto-partition-id ${{ matrix.part }} --auto-partition-size 2 --timeout-per-file 2700 ${{ needs.check-changes.outputs.continue_on_error == 'true' && '--continue-on-error' || '' }} + bash scripts/ci/amd/amd_ci_exec.sh -w "/sglang-checkout/test" python3 run_suite.py --hw amd --suite stage-b-test-2-gpu-large-amd --auto-partition-id ${{ matrix.part }} --auto-partition-size 2 --timeout-per-file ${{ (inputs.runner_arch || 'mi300') == 'mi300' && 5400 || 2700 }} ${{ needs.check-changes.outputs.continue_on_error == 'true' && '--continue-on-error' || '' }} multimodal-gen-test-1-gpu-amd: - name: ${{ format('multimodal-gen-test-1-gpu-amd (linux-{0}-1gpu-sglang, {1})', inputs.runner_arch || 'mi325', matrix.part) }} + name: ${{ format('multimodal-gen-test-1-gpu-amd (linux-{0}-1gpu-sglang, {1})', inputs.runner_arch || 'mi300', matrix.part) }} needs: [check-changes, call-gate] if: | always() && !cancelled() && @@ -668,9 +677,10 @@ jobs: ) strategy: fail-fast: false + max-parallel: 1 matrix: part: [0, 1, 2, 3] - runs-on: ${{ format('linux-{0}-1gpu-sglang', inputs.runner_arch || 'mi325') }} + runs-on: ${{ format('linux-{0}-1gpu-sglang', inputs.runner_arch || 'mi300') }} steps: - name: Checkout code uses: actions/checkout@v4 @@ -699,16 +709,7 @@ jobs: - name: Setup kernel caches run: | - # Use the persistent /sgl-data directory (mounted from /home/runner/sgl-data) - # This directory persists across container restarts on the self-hosted runner - docker exec ci_sglang mkdir -p /sgl-data/aiter-kernels /sgl-data/miopen-cache /sgl-data/hf-cache/hub - - # Clear pre-built AITER kernels from Docker image to avoid segfaults - # The image may have stale/incompatible kernels at /sgl-workspace/aiter/aiter/jit/ - echo "Clearing pre-built AITER kernels from Docker image..." - docker exec ci_sglang rm -rf /sgl-workspace/aiter/aiter/jit/*.so 2>/dev/null || true - docker exec ci_sglang rm -rf /sgl-data/aiter-kernels/*.so 2>/dev/null || true - echo "AITER kernels cleared - will be rebuilt on first use" + docker exec ci_sglang mkdir -p /sgl-data/miopen-cache /sgl-data/hf-cache/hub # Create persistent cache marker if /sgl-data is a real mount (not ephemeral) # This tells the test cleanup code to NOT delete downloaded models @@ -729,7 +730,7 @@ jobs: free -h echo "" echo "=== Disk Space ===" - df -h /home/runner/sgl-data 2>/dev/null || df -h + df -h /home/runner/sglang-data 2>/dev/null || df -h echo "" echo "=== HF Cache Directory Structure ===" docker exec ci_sglang ls -la /sgl-data/hf-cache/ 2>/dev/null || echo "HF cache dir not found" @@ -768,7 +769,6 @@ jobs: -e SGLANG_SKIP_CONSISTENCY=1 \ -e SGLANG_TEST_NUM_INFERENCE_STEPS=5 \ -e SGLANG_DIFFUSION_ARTIFACT_DIR=/sglang-checkout/diffusion-failures \ - -e AITER_JIT_DIR=/sgl-data/aiter-kernels \ -e MIOPEN_USER_DB_PATH=/sgl-data/miopen-cache \ -e HF_HUB_ENABLE_HF_TRANSFER=1 \ -e HF_HUB_DISABLE_SYMLINKS_WARNING=1 \ @@ -794,7 +794,7 @@ jobs: retention-days: 7 multimodal-gen-test-2-gpu-amd: - name: ${{ format('multimodal-gen-test-2-gpu-amd (linux-{0}-2gpu-sglang, {1})', inputs.runner_arch || 'mi325', matrix.part) }} + name: ${{ format('multimodal-gen-test-2-gpu-amd (linux-{0}-2gpu-sglang, {1})', inputs.runner_arch || 'mi300', matrix.part) }} needs: [check-changes, call-gate] if: | always() && !cancelled() && @@ -808,9 +808,10 @@ jobs: ) strategy: fail-fast: false + max-parallel: 1 matrix: part: [0, 1, 2] # 3 partitions: 2 parametrized + 1 standalone (single_test_file/test_disagg_server.py) - runs-on: ${{ format('linux-{0}-2gpu-sglang', inputs.runner_arch || 'mi325') }} + runs-on: ${{ format('linux-{0}-2gpu-sglang', inputs.runner_arch || 'mi300') }} steps: - name: Checkout code uses: actions/checkout@v4 @@ -839,15 +840,7 @@ jobs: - name: Setup kernel caches run: | - # Use the persistent /sgl-data directory (mounted from /home/runner/sgl-data) - docker exec ci_sglang mkdir -p /sgl-data/aiter-kernels /sgl-data/miopen-cache /sgl-data/hf-cache/hub - - # Clear pre-built AITER kernels from Docker image to avoid segfaults - # The image may have stale/incompatible kernels at /sgl-workspace/aiter/aiter/jit/ - echo "Clearing pre-built AITER kernels from Docker image..." - docker exec ci_sglang rm -rf /sgl-workspace/aiter/aiter/jit/*.so 2>/dev/null || true - docker exec ci_sglang rm -rf /sgl-data/aiter-kernels/*.so 2>/dev/null || true - echo "AITER kernels cleared - will be rebuilt on first use" + docker exec ci_sglang mkdir -p /sgl-data/miopen-cache /sgl-data/hf-cache/hub # Create persistent cache marker if /sgl-data is a real mount (not ephemeral) # This tells the test cleanup code to NOT delete downloaded models @@ -868,7 +861,7 @@ jobs: free -h echo "" echo "=== Disk Space ===" - df -h /home/runner/sgl-data 2>/dev/null || df -h + df -h /home/runner/sglang-data 2>/dev/null || df -h echo "" echo "=== HF Cache Directory Structure ===" docker exec ci_sglang ls -la /sgl-data/hf-cache/ 2>/dev/null || echo "HF cache dir not found" @@ -890,7 +883,7 @@ jobs: docker exec ci_sglang rocm-smi --showmeminfo vram 2>/dev/null || echo "rocm-smi not available" - name: Run diffusion server tests (2-GPU) - timeout-minutes: 150 + timeout-minutes: ${{ (inputs.runner_arch || 'mi300') == 'mi300' && 180 || 150 }} run: | # AMD CI: All 2-GPU tests including LoRA # Tests: T2V, T2I, I2V, LoRA @@ -907,7 +900,6 @@ jobs: -e SGLANG_SKIP_CONSISTENCY=1 \ -e SGLANG_TEST_NUM_INFERENCE_STEPS=5 \ -e SGLANG_DIFFUSION_ARTIFACT_DIR=/sglang-checkout/diffusion-failures \ - -e AITER_JIT_DIR=/sgl-data/aiter-kernels \ -e MIOPEN_USER_DB_PATH=/sgl-data/miopen-cache \ -e HF_HUB_ENABLE_HF_TRANSFER=1 \ -e HF_HUB_DISABLE_SYMLINKS_WARNING=1 \ @@ -937,7 +929,7 @@ jobs: # that don't require NVIDIA hardware, so they should run on AMD too. This # closes the AMD coverage gap the dashboard surfaces for these tests. multimodal-gen-unit-test-amd: - name: ${{ format('multimodal-gen-unit-test-amd (linux-{0}-1gpu-sglang)', inputs.runner_arch || 'mi325') }} + name: ${{ format('multimodal-gen-unit-test-amd (linux-{0}-1gpu-sglang)', inputs.runner_arch || 'mi300') }} needs: [check-changes, call-gate] if: | always() && !cancelled() && @@ -949,7 +941,7 @@ jobs: needs.check-changes.outputs.multimodal_gen == 'true' ) ) - runs-on: ${{ format('linux-{0}-1gpu-sglang', inputs.runner_arch || 'mi325') }} + runs-on: ${{ format('linux-{0}-1gpu-sglang', inputs.runner_arch || 'mi300') }} steps: - name: Checkout code uses: actions/checkout@v4 @@ -983,7 +975,6 @@ jobs: # memory-format behavior that the ROCm conv path doesn't reproduce # (CUDA-specific). The rest of the unit suite is portable. docker exec \ - -e AITER_JIT_DIR=/sgl-data/aiter-kernels \ -e MIOPEN_USER_DB_PATH=/sgl-data/miopen-cache \ -w /sglang-checkout/python \ ci_sglang python3 sglang/multimodal_gen/test/run_suite.py \ @@ -1019,7 +1010,7 @@ jobs: max-wait-minutes: '480' stage-c-test-4-gpu-amd: - name: ${{ format('stage-c-test-4-gpu-amd (linux-{0}-4gpu-sglang, {1})', inputs.runner_arch || 'mi325', matrix.part) }} + name: ${{ format('stage-c-test-4-gpu-amd (linux-{0}-4gpu-sglang, {1})', inputs.runner_arch || 'mi300', matrix.part) }} needs: [check-changes, call-gate, wait-for-stage-b-amd] if: | always() && @@ -1035,7 +1026,7 @@ jobs: fail-fast: false matrix: part: [0] - runs-on: ${{ format('linux-{0}-4gpu-sglang', inputs.runner_arch || 'mi325') }} + runs-on: ${{ format('linux-{0}-4gpu-sglang', inputs.runner_arch || 'mi300') }} steps: - name: Checkout code uses: actions/checkout@v4 @@ -1075,7 +1066,7 @@ jobs: ${{ needs.check-changes.outputs.continue_on_error == 'true' && '--continue-on-error' || '' }} stage-c-test-large-8-gpu-amd: - name: ${{ format('stage-c-test-large-8-gpu-amd (linux-{0}-8gpu-sglang, {1})', inputs.runner_arch || 'mi325', matrix.part) }} + name: ${{ format('stage-c-test-large-8-gpu-amd (linux-{0}-8gpu-sglang, {1})', inputs.runner_arch || 'mi300', matrix.part) }} needs: [check-changes, call-gate, wait-for-stage-b-amd] if: | always() && @@ -1088,12 +1079,13 @@ jobs: ) ) env: - RUNNER_LABELS: ${{ format('linux-{0}-8gpu-sglang', inputs.runner_arch || 'mi325') }} + RUNNER_LABELS: ${{ format('linux-{0}-8gpu-sglang', inputs.runner_arch || 'mi300') }} strategy: fail-fast: false + max-parallel: 1 matrix: part: [0, 1, 2, 3] - runs-on: ${{ format('linux-{0}-8gpu-sglang', inputs.runner_arch || 'mi325') }} + runs-on: ${{ format('linux-{0}-8gpu-sglang', inputs.runner_arch || 'mi300') }} steps: - name: Checkout code uses: actions/checkout@v4 diff --git a/scripts/ci/amd/amd_ci_start_container.sh b/scripts/ci/amd/amd_ci_start_container.sh index 21bdd07a5..68ee717d2 100755 --- a/scripts/ci/amd/amd_ci_start_container.sh +++ b/scripts/ci/amd/amd_ci_start_container.sh @@ -57,7 +57,7 @@ while [[ $# -gt 0 ]]; do echo "" echo "Environment:" echo " ENABLE_CACHE_HOST=1|0" - echo " Mount /home/runner/sglang-data to /sgl-data. Defaults to 0." + echo " Mount /home/runner/sglang-data to /sgl-data. Defaults to 1 when RUNNER_NAME contains 300 or 35x, otherwise 0. Missing host cache falls back to container-local /sgl-data." exit 0 ;; *) echo "Unknown option $1"; exit 1;; @@ -288,15 +288,24 @@ else fi CACHE_HOST=/home/runner/sglang-data -ENABLE_CACHE_HOST="${ENABLE_CACHE_HOST:-0}" +if [[ -z "${ENABLE_CACHE_HOST:-}" ]]; then + RUNNER_NAME_LOWER="${RUNNER_NAME:-}" + RUNNER_NAME_LOWER="${RUNNER_NAME_LOWER,,}" + if [[ "${RUNNER_NAME_LOWER}" == *300* || "${RUNNER_NAME_LOWER}" == *35x* ]]; then + ENABLE_CACHE_HOST="1" + else + ENABLE_CACHE_HOST="0" + fi +fi case "${ENABLE_CACHE_HOST,,}" in 1|true|yes|on|pvc|persistent) - if [[ ! -d "$CACHE_HOST" ]]; then - echo "Error: ENABLE_CACHE_HOST=1 but ${CACHE_HOST} does not exist." >&2 - exit 1 + if [[ -d "$CACHE_HOST" ]]; then + CACHE_VOLUME="-v $CACHE_HOST:/sgl-data" + echo "Mounting persistent CI data: ${CACHE_HOST} -> /sgl-data" + else + CACHE_VOLUME="" + echo "Warning: ${CACHE_HOST} does not exist; using container-local /sgl-data." >&2 fi - CACHE_VOLUME="-v $CACHE_HOST:/sgl-data" - echo "Mounting persistent CI data: ${CACHE_HOST} -> /sgl-data" ;; 0|false|no|off|"") CACHE_VOLUME="" @@ -332,8 +341,7 @@ docker run -dt --user root --device=/dev/kfd ${DEVICE_FLAG} \ docker exec ci_sglang mkdir -p \ /sgl-data/hf-cache/hub \ /sgl-data/pip-cache \ - /sgl-data/miopen-cache \ - /sgl-data/aiter-kernels + /sgl-data/miopen-cache # The checkout is owned by the runner (non-root) but the container runs as # root. Git >= 2.35.2 rejects cross-user repos; mark the mount as safe so diff --git a/scripts/ci/amd/amd_ci_start_container_disagg.sh b/scripts/ci/amd/amd_ci_start_container_disagg.sh index 7d404619b..e46c2a43d 100755 --- a/scripts/ci/amd/amd_ci_start_container_disagg.sh +++ b/scripts/ci/amd/amd_ci_start_container_disagg.sh @@ -228,13 +228,36 @@ else retry_with_backoff 6 docker pull "${IMAGE}" fi -# CACHE_HOST=/home/runner/sgl-data -CACHE_HOST=/home/runner/temp-sglang-data -if [[ -d "$CACHE_HOST" ]]; then - CACHE_VOLUME="-v $CACHE_HOST:/sgl-data" -else - CACHE_VOLUME="" +CACHE_HOST=/home/runner/sglang-data +if [[ -z "${ENABLE_CACHE_HOST:-}" ]]; then + RUNNER_NAME_LOWER="${RUNNER_NAME:-}" + RUNNER_NAME_LOWER="${RUNNER_NAME_LOWER,,}" + if [[ "${RUNNER_NAME_LOWER}" == *300* || "${RUNNER_NAME_LOWER}" == *35x* ]]; then + ENABLE_CACHE_HOST="1" + else + ENABLE_CACHE_HOST="0" + fi fi +case "${ENABLE_CACHE_HOST,,}" in + 1|true|yes|on|pvc|persistent) + if [[ -d "$CACHE_HOST" ]]; then + CACHE_VOLUME="-v $CACHE_HOST:/sgl-data" + echo "Mounting persistent CI data: ${CACHE_HOST} -> /sgl-data" + else + CACHE_VOLUME="" + echo "Warning: ${CACHE_HOST} does not exist; using container-local /sgl-data." >&2 + fi + ;; + 0|false|no|off|"") + CACHE_VOLUME="" + echo "Not mounting ${CACHE_HOST}; /sgl-data will be container-local." + ;; + *) + echo "Error: unsupported ENABLE_CACHE_HOST='${ENABLE_CACHE_HOST}'" >&2 + echo "Use 1/true/pvc/persistent or 0/false/off." >&2 + exit 1 + ;; +esac # Detect libionic library for RDMA support LIBIONIC_MOUNT="" @@ -325,6 +348,11 @@ docker run -dt --user root \ --name ci_sglang \ "${IMAGE}" +docker exec ci_sglang mkdir -p \ + /sgl-data/hf-cache/hub \ + /sgl-data/pip-cache \ + /sgl-data/miopen-cache + # The checkout is owned by the runner (non-root) but the container runs as # root. Git >= 2.35.2 rejects cross-user repos; mark the mount as safe so # setuptools-scm / vcs_versioning can resolve the package version. diff --git a/test/registered/amd/accuracy/mi30x/test_minimax_m27_eval_amd.py b/test/registered/amd/accuracy/mi30x/test_minimax_m27_eval_amd.py index b272986ac..c33e93eb8 100644 --- a/test/registered/amd/accuracy/mi30x/test_minimax_m27_eval_amd.py +++ b/test/registered/amd/accuracy/mi30x/test_minimax_m27_eval_amd.py @@ -167,7 +167,7 @@ class TestMiniMaxM27EvalAMD(unittest.TestCase): def setUpClass(cls): cls.models = MINIMAX_M27_MODELS cls.base_url = DEFAULT_URL_FOR_TEST - cls.num_questions = int(os.environ.get("GSM8K_NUM_QUESTIONS", "200")) + cls.num_questions = int(os.environ.get("GSM8K_NUM_QUESTIONS", "1319")) def test_minimax_m27_accuracy(self): """Test MiniMax-M2.7 with GSM8K completion benchmark.""" diff --git a/test/registered/perf/test_bench_serving_1gpu_part1.py b/test/registered/perf/test_bench_serving_1gpu_part1.py index ef257e4b7..09bc9905f 100644 --- a/test/registered/perf/test_bench_serving_1gpu_part1.py +++ b/test/registered/perf/test_bench_serving_1gpu_part1.py @@ -117,7 +117,7 @@ class TestBenchServing1GPUPart1(CustomTestCase): f"Output throughput: {res['output_throughput']:.2f} token/s\n" ) if is_in_amd_ci(): - self.assertGreater(res["output_throughput"], 3500) + self.assertGreater(res["output_throughput"], 2700) else: self.assertGreater(res["output_throughput"], 3700) @@ -150,7 +150,10 @@ class TestBenchServing1GPUPart1(CustomTestCase): f"median_e2e_latency_ms: {res['median_e2e_latency_ms']:.2f} ms\n" f"median_ttft_ms: {res['median_ttft_ms']:.2f} ms\n" ) - self.assertLess(res["median_e2e_latency_ms"], 2400) + if is_in_amd_ci(): + self.assertLess(res["median_e2e_latency_ms"], 3320) + else: + self.assertLess(res["median_e2e_latency_ms"], 2400) # relax for mi300x (LoRA TTFT ~2x slower than mi325) if is_in_amd_ci(): self.assertLess(res["median_ttft_ms"], 100) @@ -166,7 +169,10 @@ class TestBenchServing1GPUPart1(CustomTestCase): f"median_e2e_latency_ms: {res['median_e2e_latency_ms']:.2f} ms\n" f"median_ttft_ms: {res['median_ttft_ms']:.2f} ms\n" ) - self.assertLess(res["median_e2e_latency_ms"], 4000) + if is_in_amd_ci(): + self.assertLess(res["median_e2e_latency_ms"], 6000) + else: + self.assertLess(res["median_e2e_latency_ms"], 4000) # relax for mi300x (LoRA TTFT ~2x slower than mi325) if is_in_amd_ci(): self.assertLess(res["median_ttft_ms"], 130)