[CI] Drive per-commit stage jobs from a runner table instead of copied job blocks (#40495)
This commit is contained in:
Liangsheng Yin
@@ -13,9 +13,6 @@ on:
|
||||
description: 'Artifact of prebuilt Rust extension modules, from rust-ext-build. Empty, or a download that fails, falls back to the cache; a miss there compiles during install.'
|
||||
type: string
|
||||
default: ''
|
||||
runner_config:
|
||||
required: true
|
||||
type: string
|
||||
runs_on_map:
|
||||
required: true
|
||||
type: string
|
||||
@@ -43,15 +40,36 @@ env:
|
||||
SKIP_PR_TEST_HEALTH_CHECK: ${{ inputs.skip_pr_test_health_check == true && 'true' || 'false' }}
|
||||
|
||||
jobs:
|
||||
jit-kernel-unit-test:
|
||||
# `name:` reproduces the job names this workflow published before the table.
|
||||
jit-kernel-test:
|
||||
name: ${{ matrix.job_name }}
|
||||
# Runs whenever call-jit-kernel-tests dispatches this workflow. That caller is the
|
||||
# single gate (PR jit_kernel changes, or scheduled/parallel-dispatch full runs), so
|
||||
# the sub-jobs no longer re-exclude schedule/parallel-dispatch here.
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
partition: [0, 1]
|
||||
runs-on: 1-gpu-h100
|
||||
include:
|
||||
- job_name: jit-kernel-unit-test (0)
|
||||
runner_config: 1-gpu-large
|
||||
suite: base-b-kernel-unit-test-1-gpu-large
|
||||
suite_args: --auto-partition-id 0 --auto-partition-size 2 --fork-worker-batch-size 20
|
||||
- job_name: jit-kernel-unit-test (1)
|
||||
runner_config: 1-gpu-large
|
||||
suite: base-b-kernel-unit-test-1-gpu-large
|
||||
suite_args: --auto-partition-id 1 --auto-partition-size 2 --fork-worker-batch-size 20
|
||||
- job_name: jit-kernel-multigpu-unit-test
|
||||
runner_config: 8-gpu-h200
|
||||
suite: base-b-kernel-unit-test-8-gpu-h200
|
||||
# Alone among these, it has never run the health check.
|
||||
skip_health_check: true
|
||||
- job_name: jit-kernel-benchmark-test
|
||||
runner_config: 1-gpu-large
|
||||
suite: base-b-kernel-benchmark-test-1-gpu-large
|
||||
- job_name: jit-kernel-b200-test
|
||||
runner_config: 4-gpu-b200
|
||||
suite: base-b-kernel-unit-test-4-gpu-b200
|
||||
runs-on: ${{ fromJson(inputs.runs_on_map)[matrix.runner_config] }}
|
||||
timeout-minutes: 240
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
@@ -59,6 +77,7 @@ jobs:
|
||||
ref: ${{ inputs.git_ref || github.sha }}
|
||||
|
||||
- uses: ./.github/actions/check-pr-test-health
|
||||
if: ${{ !matrix.skip_health_check }}
|
||||
|
||||
- uses: ./.github/actions/check-maintenance
|
||||
|
||||
@@ -89,138 +108,4 @@ jobs:
|
||||
timeout-minutes: 60
|
||||
run: |
|
||||
cd test/
|
||||
python3 run_suite.py --hw cuda \
|
||||
--suite base-b-kernel-unit-test-1-gpu-large \
|
||||
--auto-partition-id ${{ matrix.partition }} \
|
||||
--auto-partition-size 2 \
|
||||
--fork-worker-batch-size 20
|
||||
|
||||
jit-kernel-multigpu-unit-test:
|
||||
# Runs whenever call-jit-kernel-tests dispatches this workflow. That caller is the
|
||||
# single gate (PR jit_kernel changes, or scheduled/parallel-dispatch full runs), so
|
||||
# the sub-jobs no longer re-exclude schedule/parallel-dispatch here.
|
||||
runs-on: 8-gpu-h200
|
||||
timeout-minutes: 240
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
ref: ${{ inputs.git_ref || github.sha }}
|
||||
|
||||
- uses: ./.github/actions/check-maintenance
|
||||
|
||||
- name: Cleanup
|
||||
if: inputs.sgl_kernel == 'true'
|
||||
run: |
|
||||
ls -alh python/sglang/kernels/aot/dist || true
|
||||
rm -rf python/sglang/kernels/aot/dist/* || true
|
||||
|
||||
- name: Download artifacts
|
||||
if: inputs.sgl_kernel == 'true'
|
||||
uses: actions/download-artifact@v4
|
||||
with:
|
||||
path: python/sglang/kernels/aot/dist/
|
||||
merge-multiple: true
|
||||
pattern: wheel-python3.10-cuda13.0
|
||||
|
||||
- uses: ./.github/actions/download-rust-ext
|
||||
with:
|
||||
artifact_name: ${{ inputs.rust_ext_artifact }}
|
||||
|
||||
- name: Install dependencies
|
||||
timeout-minutes: 20
|
||||
run: |
|
||||
CUSTOM_BUILD_SGL_KERNEL=${{ inputs.sgl_kernel }} bash scripts/ci/cuda/ci_install_dependency.sh diffusion
|
||||
|
||||
- name: Run multi-GPU test
|
||||
timeout-minutes: 45
|
||||
run: |
|
||||
cd test/
|
||||
python3 run_suite.py --hw cuda --suite base-b-kernel-unit-test-8-gpu-h200
|
||||
|
||||
jit-kernel-benchmark-test:
|
||||
# Runs whenever call-jit-kernel-tests dispatches this workflow. That caller is the
|
||||
# single gate (PR jit_kernel changes, or scheduled/parallel-dispatch full runs), so
|
||||
# the sub-jobs no longer re-exclude schedule/parallel-dispatch here.
|
||||
runs-on: 1-gpu-h100
|
||||
timeout-minutes: 240
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
ref: ${{ inputs.git_ref || github.sha }}
|
||||
|
||||
- uses: ./.github/actions/check-pr-test-health
|
||||
|
||||
- uses: ./.github/actions/check-maintenance
|
||||
|
||||
- name: Cleanup
|
||||
if: inputs.sgl_kernel == 'true'
|
||||
run: |
|
||||
ls -alh python/sglang/kernels/aot/dist || true
|
||||
rm -rf python/sglang/kernels/aot/dist/* || true
|
||||
|
||||
- name: Download artifacts
|
||||
if: inputs.sgl_kernel == 'true'
|
||||
uses: actions/download-artifact@v4
|
||||
with:
|
||||
path: python/sglang/kernels/aot/dist/
|
||||
merge-multiple: true
|
||||
pattern: wheel-python3.10-cuda13.0
|
||||
|
||||
- uses: ./.github/actions/download-rust-ext
|
||||
with:
|
||||
artifact_name: ${{ inputs.rust_ext_artifact }}
|
||||
|
||||
- name: Install dependencies
|
||||
timeout-minutes: 20
|
||||
run: |
|
||||
CUSTOM_BUILD_SGL_KERNEL=${{ inputs.sgl_kernel }} bash scripts/ci/cuda/ci_install_dependency.sh diffusion
|
||||
|
||||
- name: Run benchmark tests
|
||||
timeout-minutes: 45
|
||||
run: |
|
||||
cd test/
|
||||
python3 run_suite.py --hw cuda --suite base-b-kernel-benchmark-test-1-gpu-large
|
||||
|
||||
jit-kernel-b200-test:
|
||||
# Runs whenever call-jit-kernel-tests dispatches this workflow. That caller is the
|
||||
# single gate (PR jit_kernel changes, or scheduled/parallel-dispatch full runs), so
|
||||
# the sub-jobs no longer re-exclude schedule/parallel-dispatch here.
|
||||
runs-on: ${{ fromJson(inputs.runs_on_map)[inputs.runner_config] }}
|
||||
timeout-minutes: 240
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
ref: ${{ inputs.git_ref || github.sha }}
|
||||
|
||||
- uses: ./.github/actions/check-pr-test-health
|
||||
|
||||
- uses: ./.github/actions/check-maintenance
|
||||
|
||||
- name: Cleanup
|
||||
if: inputs.sgl_kernel == 'true'
|
||||
run: |
|
||||
ls -alh python/sglang/kernels/aot/dist || true
|
||||
rm -rf python/sglang/kernels/aot/dist/* || true
|
||||
|
||||
- name: Download artifacts
|
||||
if: inputs.sgl_kernel == 'true'
|
||||
uses: actions/download-artifact@v4
|
||||
with:
|
||||
path: python/sglang/kernels/aot/dist/
|
||||
merge-multiple: true
|
||||
pattern: wheel-python3.10-cuda13.0
|
||||
|
||||
- uses: ./.github/actions/download-rust-ext
|
||||
with:
|
||||
artifact_name: ${{ inputs.rust_ext_artifact }}
|
||||
|
||||
- name: Install dependencies
|
||||
timeout-minutes: 20
|
||||
run: |
|
||||
CUSTOM_BUILD_SGL_KERNEL=${{ inputs.sgl_kernel }} bash scripts/ci/cuda/ci_install_dependency.sh diffusion
|
||||
|
||||
- name: Run B200 diffusion test
|
||||
timeout-minutes: 30
|
||||
run: |
|
||||
cd test/
|
||||
python3 run_suite.py --hw cuda --suite base-b-kernel-unit-test-4-gpu-b200
|
||||
python3 run_suite.py --hw cuda --suite ${{ matrix.suite }} ${{ matrix.suite_args }}
|
||||
|
||||
Reference in New Issue
Block a user