[CI] Drive per-commit stage jobs from a runner table instead of copied job blocks (#40495)
This commit is contained in:
@@ -159,107 +159,32 @@ jobs:
|
|||||||
rust_ext_artifact: ${{ needs.rust-ext-build.outputs.artifact_name }}
|
rust_ext_artifact: ${{ needs.rust-ext-build.outputs.artifact_name }}
|
||||||
# No `secrets: inherit`: this hosted CPU job has no secret consumer.
|
# No `secrets: inherit`: this hosted CPU job has no secret consumer.
|
||||||
|
|
||||||
# =============================================== extra-a (1-/2-gpu) ===============================================
|
# =============================================== extra-a / extra-b ===============================================
|
||||||
extra-a-test-1-gpu-small:
|
# `name:` keeps the job names wait-for-jobs and check-pr-test-health match on.
|
||||||
|
extra-test:
|
||||||
|
name: ${{ matrix.stage }}-test-${{ matrix.runner_config }}
|
||||||
needs: [check-changes, call-gate, sgl-kernel-build-wheels, rust-ext-build]
|
needs: [check-changes, call-gate, sgl-kernel-build-wheels, rust-ext-build]
|
||||||
if: ${{ !failure() && !cancelled() && needs.check-changes.result == 'success' && (needs.call-gate.result == 'success' || needs.call-gate.result == 'skipped') }}
|
if: ${{ !failure() && !cancelled() && needs.check-changes.result == 'success' && (needs.call-gate.result == 'success' || needs.call-gate.result == 'skipped') }}
|
||||||
|
strategy:
|
||||||
|
fail-fast: false
|
||||||
|
matrix:
|
||||||
|
include:
|
||||||
|
- { stage: extra-a, runner_config: 1-gpu-small, timeout_per_file: '' }
|
||||||
|
- { stage: extra-a, runner_config: 1-gpu-large, timeout_per_file: '1800' }
|
||||||
|
- { stage: extra-a, runner_config: 2-gpu-large, timeout_per_file: '' }
|
||||||
|
- { stage: extra-b, runner_config: 4-gpu-h100, timeout_per_file: '3600' }
|
||||||
|
- { stage: extra-b, runner_config: 4-gpu-b200, timeout_per_file: '1800' }
|
||||||
|
- { stage: extra-b, runner_config: 8-gpu-h200, timeout_per_file: '' }
|
||||||
|
- { stage: extra-b, runner_config: 8-gpu-b300, timeout_per_file: '3600' }
|
||||||
uses: ./.github/workflows/_pr-test-stage.yml
|
uses: ./.github/workflows/_pr-test-stage.yml
|
||||||
with:
|
with:
|
||||||
self_name: extra-a-test-1-gpu-small
|
self_name: ${{ matrix.stage }}-test-${{ matrix.runner_config }}
|
||||||
runner_config: 1-gpu-small
|
runner_config: ${{ matrix.runner_config }}
|
||||||
check_changes: ${{ toJson(needs.check-changes.outputs) }}
|
check_changes: ${{ toJson(needs.check-changes.outputs) }}
|
||||||
caller_inputs: ${{ toJson(inputs) }}
|
caller_inputs: ${{ toJson(inputs) }}
|
||||||
partitions: ${{ needs.check-changes.outputs.partitions }}
|
partitions: ${{ needs.check-changes.outputs.partitions }}
|
||||||
run_timeout_minutes: '60'
|
run_timeout_minutes: '60'
|
||||||
rust_ext_artifact: ${{ needs.rust-ext-build.outputs.artifact_name }}
|
timeout_per_file: ${{ matrix.timeout_per_file }}
|
||||||
secrets: inherit
|
|
||||||
|
|
||||||
extra-a-test-1-gpu-large:
|
|
||||||
needs: [check-changes, call-gate, sgl-kernel-build-wheels, rust-ext-build]
|
|
||||||
if: ${{ !failure() && !cancelled() && needs.check-changes.result == 'success' && (needs.call-gate.result == 'success' || needs.call-gate.result == 'skipped') }}
|
|
||||||
uses: ./.github/workflows/_pr-test-stage.yml
|
|
||||||
with:
|
|
||||||
self_name: extra-a-test-1-gpu-large
|
|
||||||
runner_config: 1-gpu-large
|
|
||||||
check_changes: ${{ toJson(needs.check-changes.outputs) }}
|
|
||||||
caller_inputs: ${{ toJson(inputs) }}
|
|
||||||
partitions: ${{ needs.check-changes.outputs.partitions }}
|
|
||||||
run_timeout_minutes: '60'
|
|
||||||
timeout_per_file: '1800'
|
|
||||||
rust_ext_artifact: ${{ needs.rust-ext-build.outputs.artifact_name }}
|
|
||||||
secrets: inherit
|
|
||||||
|
|
||||||
extra-a-test-2-gpu-large:
|
|
||||||
needs: [check-changes, call-gate, sgl-kernel-build-wheels, rust-ext-build]
|
|
||||||
if: ${{ !failure() && !cancelled() && needs.check-changes.result == 'success' && (needs.call-gate.result == 'success' || needs.call-gate.result == 'skipped') }}
|
|
||||||
uses: ./.github/workflows/_pr-test-stage.yml
|
|
||||||
with:
|
|
||||||
self_name: extra-a-test-2-gpu-large
|
|
||||||
runner_config: 2-gpu-large
|
|
||||||
check_changes: ${{ toJson(needs.check-changes.outputs) }}
|
|
||||||
caller_inputs: ${{ toJson(inputs) }}
|
|
||||||
partitions: ${{ needs.check-changes.outputs.partitions }}
|
|
||||||
run_timeout_minutes: '60'
|
|
||||||
rust_ext_artifact: ${{ needs.rust-ext-build.outputs.artifact_name }}
|
|
||||||
secrets: inherit
|
|
||||||
|
|
||||||
# =============================================== extra-b (4-/8-gpu) ===============================================
|
|
||||||
extra-b-test-4-gpu-h100:
|
|
||||||
needs: [check-changes, call-gate, sgl-kernel-build-wheels, rust-ext-build]
|
|
||||||
if: ${{ !failure() && !cancelled() && needs.check-changes.result == 'success' && (needs.call-gate.result == 'success' || needs.call-gate.result == 'skipped') }}
|
|
||||||
uses: ./.github/workflows/_pr-test-stage.yml
|
|
||||||
with:
|
|
||||||
self_name: extra-b-test-4-gpu-h100
|
|
||||||
runner_config: 4-gpu-h100
|
|
||||||
check_changes: ${{ toJson(needs.check-changes.outputs) }}
|
|
||||||
caller_inputs: ${{ toJson(inputs) }}
|
|
||||||
partitions: ${{ needs.check-changes.outputs.partitions }}
|
|
||||||
run_timeout_minutes: '60'
|
|
||||||
timeout_per_file: '3600'
|
|
||||||
rust_ext_artifact: ${{ needs.rust-ext-build.outputs.artifact_name }}
|
|
||||||
secrets: inherit
|
|
||||||
|
|
||||||
extra-b-test-4-gpu-b200:
|
|
||||||
needs: [check-changes, call-gate, sgl-kernel-build-wheels, rust-ext-build]
|
|
||||||
if: ${{ !failure() && !cancelled() && needs.check-changes.result == 'success' && (needs.call-gate.result == 'success' || needs.call-gate.result == 'skipped') }}
|
|
||||||
uses: ./.github/workflows/_pr-test-stage.yml
|
|
||||||
with:
|
|
||||||
self_name: extra-b-test-4-gpu-b200
|
|
||||||
runner_config: 4-gpu-b200
|
|
||||||
check_changes: ${{ toJson(needs.check-changes.outputs) }}
|
|
||||||
caller_inputs: ${{ toJson(inputs) }}
|
|
||||||
partitions: ${{ needs.check-changes.outputs.partitions }}
|
|
||||||
run_timeout_minutes: '60'
|
|
||||||
timeout_per_file: '1800'
|
|
||||||
rust_ext_artifact: ${{ needs.rust-ext-build.outputs.artifact_name }}
|
|
||||||
secrets: inherit
|
|
||||||
|
|
||||||
extra-b-test-8-gpu-h200:
|
|
||||||
needs: [check-changes, call-gate, sgl-kernel-build-wheels, rust-ext-build]
|
|
||||||
if: ${{ !failure() && !cancelled() && needs.check-changes.result == 'success' && (needs.call-gate.result == 'success' || needs.call-gate.result == 'skipped') }}
|
|
||||||
uses: ./.github/workflows/_pr-test-stage.yml
|
|
||||||
with:
|
|
||||||
self_name: extra-b-test-8-gpu-h200
|
|
||||||
runner_config: 8-gpu-h200
|
|
||||||
check_changes: ${{ toJson(needs.check-changes.outputs) }}
|
|
||||||
caller_inputs: ${{ toJson(inputs) }}
|
|
||||||
partitions: ${{ needs.check-changes.outputs.partitions }}
|
|
||||||
run_timeout_minutes: '60'
|
|
||||||
rust_ext_artifact: ${{ needs.rust-ext-build.outputs.artifact_name }}
|
|
||||||
secrets: inherit
|
|
||||||
|
|
||||||
extra-b-test-8-gpu-b300:
|
|
||||||
needs: [check-changes, call-gate, sgl-kernel-build-wheels, rust-ext-build]
|
|
||||||
if: ${{ !failure() && !cancelled() && needs.check-changes.result == 'success' && (needs.call-gate.result == 'success' || needs.call-gate.result == 'skipped') }}
|
|
||||||
uses: ./.github/workflows/_pr-test-stage.yml
|
|
||||||
with:
|
|
||||||
self_name: extra-b-test-8-gpu-b300
|
|
||||||
runner_config: 8-gpu-b300
|
|
||||||
check_changes: ${{ toJson(needs.check-changes.outputs) }}
|
|
||||||
caller_inputs: ${{ toJson(inputs) }}
|
|
||||||
partitions: ${{ needs.check-changes.outputs.partitions }}
|
|
||||||
run_timeout_minutes: '60'
|
|
||||||
timeout_per_file: '3600'
|
|
||||||
rust_ext_artifact: ${{ needs.rust-ext-build.outputs.artifact_name }}
|
rust_ext_artifact: ${{ needs.rust-ext-build.outputs.artifact_name }}
|
||||||
secrets: inherit
|
secrets: inherit
|
||||||
|
|
||||||
@@ -275,13 +200,7 @@ jobs:
|
|||||||
sgl-kernel-build-wheels,
|
sgl-kernel-build-wheels,
|
||||||
rust-ext-build,
|
rust-ext-build,
|
||||||
simulator-test-cpu,
|
simulator-test-cpu,
|
||||||
extra-a-test-1-gpu-small,
|
extra-test,
|
||||||
extra-a-test-1-gpu-large,
|
|
||||||
extra-a-test-2-gpu-large,
|
|
||||||
extra-b-test-4-gpu-h100,
|
|
||||||
extra-b-test-4-gpu-b200,
|
|
||||||
extra-b-test-8-gpu-h200,
|
|
||||||
extra-b-test-8-gpu-b300,
|
|
||||||
]
|
]
|
||||||
if: always()
|
if: always()
|
||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
|
|||||||
@@ -13,9 +13,6 @@ on:
|
|||||||
description: 'Artifact of prebuilt Rust extension modules, from rust-ext-build. Empty, or a download that fails, falls back to the cache; a miss there compiles during install.'
|
description: 'Artifact of prebuilt Rust extension modules, from rust-ext-build. Empty, or a download that fails, falls back to the cache; a miss there compiles during install.'
|
||||||
type: string
|
type: string
|
||||||
default: ''
|
default: ''
|
||||||
runner_config:
|
|
||||||
required: true
|
|
||||||
type: string
|
|
||||||
runs_on_map:
|
runs_on_map:
|
||||||
required: true
|
required: true
|
||||||
type: string
|
type: string
|
||||||
@@ -43,15 +40,36 @@ env:
|
|||||||
SKIP_PR_TEST_HEALTH_CHECK: ${{ inputs.skip_pr_test_health_check == true && 'true' || 'false' }}
|
SKIP_PR_TEST_HEALTH_CHECK: ${{ inputs.skip_pr_test_health_check == true && 'true' || 'false' }}
|
||||||
|
|
||||||
jobs:
|
jobs:
|
||||||
jit-kernel-unit-test:
|
# `name:` reproduces the job names this workflow published before the table.
|
||||||
|
jit-kernel-test:
|
||||||
|
name: ${{ matrix.job_name }}
|
||||||
# Runs whenever call-jit-kernel-tests dispatches this workflow. That caller is the
|
# Runs whenever call-jit-kernel-tests dispatches this workflow. That caller is the
|
||||||
# single gate (PR jit_kernel changes, or scheduled/parallel-dispatch full runs), so
|
# single gate (PR jit_kernel changes, or scheduled/parallel-dispatch full runs), so
|
||||||
# the sub-jobs no longer re-exclude schedule/parallel-dispatch here.
|
# the sub-jobs no longer re-exclude schedule/parallel-dispatch here.
|
||||||
strategy:
|
strategy:
|
||||||
fail-fast: false
|
fail-fast: false
|
||||||
matrix:
|
matrix:
|
||||||
partition: [0, 1]
|
include:
|
||||||
runs-on: 1-gpu-h100
|
- job_name: jit-kernel-unit-test (0)
|
||||||
|
runner_config: 1-gpu-large
|
||||||
|
suite: base-b-kernel-unit-test-1-gpu-large
|
||||||
|
suite_args: --auto-partition-id 0 --auto-partition-size 2 --fork-worker-batch-size 20
|
||||||
|
- job_name: jit-kernel-unit-test (1)
|
||||||
|
runner_config: 1-gpu-large
|
||||||
|
suite: base-b-kernel-unit-test-1-gpu-large
|
||||||
|
suite_args: --auto-partition-id 1 --auto-partition-size 2 --fork-worker-batch-size 20
|
||||||
|
- job_name: jit-kernel-multigpu-unit-test
|
||||||
|
runner_config: 8-gpu-h200
|
||||||
|
suite: base-b-kernel-unit-test-8-gpu-h200
|
||||||
|
# Alone among these, it has never run the health check.
|
||||||
|
skip_health_check: true
|
||||||
|
- job_name: jit-kernel-benchmark-test
|
||||||
|
runner_config: 1-gpu-large
|
||||||
|
suite: base-b-kernel-benchmark-test-1-gpu-large
|
||||||
|
- job_name: jit-kernel-b200-test
|
||||||
|
runner_config: 4-gpu-b200
|
||||||
|
suite: base-b-kernel-unit-test-4-gpu-b200
|
||||||
|
runs-on: ${{ fromJson(inputs.runs_on_map)[matrix.runner_config] }}
|
||||||
timeout-minutes: 240
|
timeout-minutes: 240
|
||||||
steps:
|
steps:
|
||||||
- uses: actions/checkout@v4
|
- uses: actions/checkout@v4
|
||||||
@@ -59,6 +77,7 @@ jobs:
|
|||||||
ref: ${{ inputs.git_ref || github.sha }}
|
ref: ${{ inputs.git_ref || github.sha }}
|
||||||
|
|
||||||
- uses: ./.github/actions/check-pr-test-health
|
- uses: ./.github/actions/check-pr-test-health
|
||||||
|
if: ${{ !matrix.skip_health_check }}
|
||||||
|
|
||||||
- uses: ./.github/actions/check-maintenance
|
- uses: ./.github/actions/check-maintenance
|
||||||
|
|
||||||
@@ -89,138 +108,4 @@ jobs:
|
|||||||
timeout-minutes: 60
|
timeout-minutes: 60
|
||||||
run: |
|
run: |
|
||||||
cd test/
|
cd test/
|
||||||
python3 run_suite.py --hw cuda \
|
python3 run_suite.py --hw cuda --suite ${{ matrix.suite }} ${{ matrix.suite_args }}
|
||||||
--suite base-b-kernel-unit-test-1-gpu-large \
|
|
||||||
--auto-partition-id ${{ matrix.partition }} \
|
|
||||||
--auto-partition-size 2 \
|
|
||||||
--fork-worker-batch-size 20
|
|
||||||
|
|
||||||
jit-kernel-multigpu-unit-test:
|
|
||||||
# Runs whenever call-jit-kernel-tests dispatches this workflow. That caller is the
|
|
||||||
# single gate (PR jit_kernel changes, or scheduled/parallel-dispatch full runs), so
|
|
||||||
# the sub-jobs no longer re-exclude schedule/parallel-dispatch here.
|
|
||||||
runs-on: 8-gpu-h200
|
|
||||||
timeout-minutes: 240
|
|
||||||
steps:
|
|
||||||
- uses: actions/checkout@v4
|
|
||||||
with:
|
|
||||||
ref: ${{ inputs.git_ref || github.sha }}
|
|
||||||
|
|
||||||
- uses: ./.github/actions/check-maintenance
|
|
||||||
|
|
||||||
- name: Cleanup
|
|
||||||
if: inputs.sgl_kernel == 'true'
|
|
||||||
run: |
|
|
||||||
ls -alh python/sglang/kernels/aot/dist || true
|
|
||||||
rm -rf python/sglang/kernels/aot/dist/* || true
|
|
||||||
|
|
||||||
- name: Download artifacts
|
|
||||||
if: inputs.sgl_kernel == 'true'
|
|
||||||
uses: actions/download-artifact@v4
|
|
||||||
with:
|
|
||||||
path: python/sglang/kernels/aot/dist/
|
|
||||||
merge-multiple: true
|
|
||||||
pattern: wheel-python3.10-cuda13.0
|
|
||||||
|
|
||||||
- uses: ./.github/actions/download-rust-ext
|
|
||||||
with:
|
|
||||||
artifact_name: ${{ inputs.rust_ext_artifact }}
|
|
||||||
|
|
||||||
- name: Install dependencies
|
|
||||||
timeout-minutes: 20
|
|
||||||
run: |
|
|
||||||
CUSTOM_BUILD_SGL_KERNEL=${{ inputs.sgl_kernel }} bash scripts/ci/cuda/ci_install_dependency.sh diffusion
|
|
||||||
|
|
||||||
- name: Run multi-GPU test
|
|
||||||
timeout-minutes: 45
|
|
||||||
run: |
|
|
||||||
cd test/
|
|
||||||
python3 run_suite.py --hw cuda --suite base-b-kernel-unit-test-8-gpu-h200
|
|
||||||
|
|
||||||
jit-kernel-benchmark-test:
|
|
||||||
# Runs whenever call-jit-kernel-tests dispatches this workflow. That caller is the
|
|
||||||
# single gate (PR jit_kernel changes, or scheduled/parallel-dispatch full runs), so
|
|
||||||
# the sub-jobs no longer re-exclude schedule/parallel-dispatch here.
|
|
||||||
runs-on: 1-gpu-h100
|
|
||||||
timeout-minutes: 240
|
|
||||||
steps:
|
|
||||||
- uses: actions/checkout@v4
|
|
||||||
with:
|
|
||||||
ref: ${{ inputs.git_ref || github.sha }}
|
|
||||||
|
|
||||||
- uses: ./.github/actions/check-pr-test-health
|
|
||||||
|
|
||||||
- uses: ./.github/actions/check-maintenance
|
|
||||||
|
|
||||||
- name: Cleanup
|
|
||||||
if: inputs.sgl_kernel == 'true'
|
|
||||||
run: |
|
|
||||||
ls -alh python/sglang/kernels/aot/dist || true
|
|
||||||
rm -rf python/sglang/kernels/aot/dist/* || true
|
|
||||||
|
|
||||||
- name: Download artifacts
|
|
||||||
if: inputs.sgl_kernel == 'true'
|
|
||||||
uses: actions/download-artifact@v4
|
|
||||||
with:
|
|
||||||
path: python/sglang/kernels/aot/dist/
|
|
||||||
merge-multiple: true
|
|
||||||
pattern: wheel-python3.10-cuda13.0
|
|
||||||
|
|
||||||
- uses: ./.github/actions/download-rust-ext
|
|
||||||
with:
|
|
||||||
artifact_name: ${{ inputs.rust_ext_artifact }}
|
|
||||||
|
|
||||||
- name: Install dependencies
|
|
||||||
timeout-minutes: 20
|
|
||||||
run: |
|
|
||||||
CUSTOM_BUILD_SGL_KERNEL=${{ inputs.sgl_kernel }} bash scripts/ci/cuda/ci_install_dependency.sh diffusion
|
|
||||||
|
|
||||||
- name: Run benchmark tests
|
|
||||||
timeout-minutes: 45
|
|
||||||
run: |
|
|
||||||
cd test/
|
|
||||||
python3 run_suite.py --hw cuda --suite base-b-kernel-benchmark-test-1-gpu-large
|
|
||||||
|
|
||||||
jit-kernel-b200-test:
|
|
||||||
# Runs whenever call-jit-kernel-tests dispatches this workflow. That caller is the
|
|
||||||
# single gate (PR jit_kernel changes, or scheduled/parallel-dispatch full runs), so
|
|
||||||
# the sub-jobs no longer re-exclude schedule/parallel-dispatch here.
|
|
||||||
runs-on: ${{ fromJson(inputs.runs_on_map)[inputs.runner_config] }}
|
|
||||||
timeout-minutes: 240
|
|
||||||
steps:
|
|
||||||
- uses: actions/checkout@v4
|
|
||||||
with:
|
|
||||||
ref: ${{ inputs.git_ref || github.sha }}
|
|
||||||
|
|
||||||
- uses: ./.github/actions/check-pr-test-health
|
|
||||||
|
|
||||||
- uses: ./.github/actions/check-maintenance
|
|
||||||
|
|
||||||
- name: Cleanup
|
|
||||||
if: inputs.sgl_kernel == 'true'
|
|
||||||
run: |
|
|
||||||
ls -alh python/sglang/kernels/aot/dist || true
|
|
||||||
rm -rf python/sglang/kernels/aot/dist/* || true
|
|
||||||
|
|
||||||
- name: Download artifacts
|
|
||||||
if: inputs.sgl_kernel == 'true'
|
|
||||||
uses: actions/download-artifact@v4
|
|
||||||
with:
|
|
||||||
path: python/sglang/kernels/aot/dist/
|
|
||||||
merge-multiple: true
|
|
||||||
pattern: wheel-python3.10-cuda13.0
|
|
||||||
|
|
||||||
- uses: ./.github/actions/download-rust-ext
|
|
||||||
with:
|
|
||||||
artifact_name: ${{ inputs.rust_ext_artifact }}
|
|
||||||
|
|
||||||
- name: Install dependencies
|
|
||||||
timeout-minutes: 20
|
|
||||||
run: |
|
|
||||||
CUSTOM_BUILD_SGL_KERNEL=${{ inputs.sgl_kernel }} bash scripts/ci/cuda/ci_install_dependency.sh diffusion
|
|
||||||
|
|
||||||
- name: Run B200 diffusion test
|
|
||||||
timeout-minutes: 30
|
|
||||||
run: |
|
|
||||||
cd test/
|
|
||||||
python3 run_suite.py --hw cuda --suite base-b-kernel-unit-test-4-gpu-b200
|
|
||||||
|
|||||||
+77
-144
@@ -290,7 +290,6 @@ jobs:
|
|||||||
needs.check-changes.outputs.jit_kernel == 'true'
|
needs.check-changes.outputs.jit_kernel == 'true'
|
||||||
uses: ./.github/workflows/pr-test-jit-kernel.yml
|
uses: ./.github/workflows/pr-test-jit-kernel.yml
|
||||||
with:
|
with:
|
||||||
runner_config: 4-gpu-b200
|
|
||||||
runs_on_map: ${{ needs.check-changes.outputs.runs_on_map }}
|
runs_on_map: ${{ needs.check-changes.outputs.runs_on_map }}
|
||||||
jit_kernel: ${{ needs.check-changes.outputs.jit_kernel }}
|
jit_kernel: ${{ needs.check-changes.outputs.jit_kernel }}
|
||||||
# On scheduled/parallel-dispatch runs sgl-kernel-build-wheels is skipped, so the wheel
|
# On scheduled/parallel-dispatch runs sgl-kernel-build-wheels is skipped, so the wheel
|
||||||
@@ -305,18 +304,27 @@ jobs:
|
|||||||
|
|
||||||
# =============================================== primary ====================================================
|
# =============================================== primary ====================================================
|
||||||
|
|
||||||
# Runs on 5090 (32GB, SM120)
|
# `name:` is load-bearing: wait-for-jobs gates a stage by job-name prefix,
|
||||||
base-a-test-1-gpu-small:
|
# and check-pr-test-health carves h20 out by the same name.
|
||||||
|
base-a-test:
|
||||||
|
name: base-a-test-${{ matrix.runner_config }}
|
||||||
needs: [check-changes, call-gate, sgl-kernel-build-wheels, rust-ext-build]
|
needs: [check-changes, call-gate, sgl-kernel-build-wheels, rust-ext-build]
|
||||||
if: ${{ !failure() && !cancelled() }}
|
if: ${{ !failure() && !cancelled() }}
|
||||||
|
strategy:
|
||||||
|
fail-fast: false
|
||||||
|
matrix:
|
||||||
|
include:
|
||||||
|
# 5090 (32GB, SM120)
|
||||||
|
- { runner_config: 1-gpu-small, run_timeout: '10', timeout_per_file: '' }
|
||||||
uses: ./.github/workflows/_pr-test-stage.yml
|
uses: ./.github/workflows/_pr-test-stage.yml
|
||||||
with:
|
with:
|
||||||
self_name: base-a-test-1-gpu-small
|
self_name: base-a-test-${{ matrix.runner_config }}
|
||||||
runner_config: 1-gpu-small
|
runner_config: ${{ matrix.runner_config }}
|
||||||
check_changes: ${{ toJson(needs.check-changes.outputs) }}
|
check_changes: ${{ toJson(needs.check-changes.outputs) }}
|
||||||
caller_inputs: ${{ toJson(inputs) }}
|
caller_inputs: ${{ toJson(inputs) }}
|
||||||
partitions: ${{ needs.check-changes.outputs.partitions }}
|
partitions: ${{ needs.check-changes.outputs.partitions }}
|
||||||
run_timeout_minutes: '10'
|
run_timeout_minutes: ${{ matrix.run_timeout }}
|
||||||
|
timeout_per_file: ${{ matrix.timeout_per_file }}
|
||||||
rust_ext_artifact: ${{ needs.rust-ext-build.outputs.artifact_name }}
|
rust_ext_artifact: ${{ needs.rust-ext-build.outputs.artifact_name }}
|
||||||
secrets: inherit
|
secrets: inherit
|
||||||
|
|
||||||
@@ -337,62 +345,28 @@ jobs:
|
|||||||
# No `secrets: inherit`: this stage has no secret consumer, unlike the GPU
|
# No `secrets: inherit`: this stage has no secret consumer, unlike the GPU
|
||||||
# stages' coredump upload. GITHUB_TOKEN and permissions inherit regardless.
|
# stages' coredump upload. GITHUB_TOKEN and permissions inherit regardless.
|
||||||
|
|
||||||
# Runs on 5090 (32GB, SM120)
|
base-b-test:
|
||||||
base-b-test-1-gpu-small:
|
name: base-b-test-${{ matrix.runner_config }}
|
||||||
needs: [check-changes, call-gate, wait-for-base-a, sgl-kernel-build-wheels, rust-ext-build]
|
needs: [check-changes, call-gate, wait-for-base-a, sgl-kernel-build-wheels, rust-ext-build]
|
||||||
if: ${{ !failure() && !cancelled() }}
|
if: ${{ !failure() && !cancelled() }}
|
||||||
|
strategy:
|
||||||
|
fail-fast: false
|
||||||
|
matrix:
|
||||||
|
include:
|
||||||
|
# 5090 (32GB, SM120)
|
||||||
|
- { runner_config: 1-gpu-small, run_timeout: '30', timeout_per_file: '' }
|
||||||
|
- { runner_config: 1-gpu-large, run_timeout: '60', timeout_per_file: '1800' }
|
||||||
|
- { runner_config: 2-gpu-large, run_timeout: '30', timeout_per_file: '' }
|
||||||
|
- { runner_config: 4-gpu-b200, run_timeout: '40', timeout_per_file: '' }
|
||||||
uses: ./.github/workflows/_pr-test-stage.yml
|
uses: ./.github/workflows/_pr-test-stage.yml
|
||||||
with:
|
with:
|
||||||
self_name: base-b-test-1-gpu-small
|
self_name: base-b-test-${{ matrix.runner_config }}
|
||||||
runner_config: 1-gpu-small
|
runner_config: ${{ matrix.runner_config }}
|
||||||
check_changes: ${{ toJson(needs.check-changes.outputs) }}
|
check_changes: ${{ toJson(needs.check-changes.outputs) }}
|
||||||
caller_inputs: ${{ toJson(inputs) }}
|
caller_inputs: ${{ toJson(inputs) }}
|
||||||
partitions: ${{ needs.check-changes.outputs.partitions }}
|
partitions: ${{ needs.check-changes.outputs.partitions }}
|
||||||
run_timeout_minutes: '30'
|
run_timeout_minutes: ${{ matrix.run_timeout }}
|
||||||
rust_ext_artifact: ${{ needs.rust-ext-build.outputs.artifact_name }}
|
timeout_per_file: ${{ matrix.timeout_per_file }}
|
||||||
secrets: inherit
|
|
||||||
|
|
||||||
# Runs on H100 (80GB, SM90) - tests that don't pass on 5090 (FA3, FP8, high VRAM, etc.)
|
|
||||||
base-b-test-1-gpu-large:
|
|
||||||
needs: [check-changes, call-gate, wait-for-base-a, sgl-kernel-build-wheels, rust-ext-build]
|
|
||||||
if: ${{ !failure() && !cancelled() }}
|
|
||||||
uses: ./.github/workflows/_pr-test-stage.yml
|
|
||||||
with:
|
|
||||||
self_name: base-b-test-1-gpu-large
|
|
||||||
runner_config: 1-gpu-large
|
|
||||||
check_changes: ${{ toJson(needs.check-changes.outputs) }}
|
|
||||||
caller_inputs: ${{ toJson(inputs) }}
|
|
||||||
partitions: ${{ needs.check-changes.outputs.partitions }}
|
|
||||||
run_timeout_minutes: '60'
|
|
||||||
timeout_per_file: '1800'
|
|
||||||
rust_ext_artifact: ${{ needs.rust-ext-build.outputs.artifact_name }}
|
|
||||||
secrets: inherit
|
|
||||||
|
|
||||||
base-b-test-2-gpu-large:
|
|
||||||
needs: [check-changes, call-gate, wait-for-base-a, sgl-kernel-build-wheels, rust-ext-build]
|
|
||||||
if: ${{ !failure() && !cancelled() }}
|
|
||||||
uses: ./.github/workflows/_pr-test-stage.yml
|
|
||||||
with:
|
|
||||||
self_name: base-b-test-2-gpu-large
|
|
||||||
runner_config: 2-gpu-large
|
|
||||||
check_changes: ${{ toJson(needs.check-changes.outputs) }}
|
|
||||||
caller_inputs: ${{ toJson(inputs) }}
|
|
||||||
partitions: ${{ needs.check-changes.outputs.partitions }}
|
|
||||||
run_timeout_minutes: '30'
|
|
||||||
rust_ext_artifact: ${{ needs.rust-ext-build.outputs.artifact_name }}
|
|
||||||
secrets: inherit
|
|
||||||
|
|
||||||
base-b-test-4-gpu-b200:
|
|
||||||
needs: [check-changes, call-gate, wait-for-base-a, sgl-kernel-build-wheels, rust-ext-build]
|
|
||||||
if: ${{ !failure() && !cancelled() }}
|
|
||||||
uses: ./.github/workflows/_pr-test-stage.yml
|
|
||||||
with:
|
|
||||||
self_name: base-b-test-4-gpu-b200
|
|
||||||
runner_config: 4-gpu-b200
|
|
||||||
check_changes: ${{ toJson(needs.check-changes.outputs) }}
|
|
||||||
caller_inputs: ${{ toJson(inputs) }}
|
|
||||||
partitions: ${{ needs.check-changes.outputs.partitions }}
|
|
||||||
run_timeout_minutes: '40'
|
|
||||||
rust_ext_artifact: ${{ needs.rust-ext-build.outputs.artifact_name }}
|
rust_ext_artifact: ${{ needs.rust-ext-build.outputs.artifact_name }}
|
||||||
secrets: inherit
|
secrets: inherit
|
||||||
|
|
||||||
@@ -416,104 +390,70 @@ jobs:
|
|||||||
skip_pr_test_health_check: ${{ (inputs.skip_pr_test_health_check == true || inputs.test_parallel_dispatch == true || inputs.run_all_tests == true) && 'true' || 'false' }}
|
skip_pr_test_health_check: ${{ (inputs.skip_pr_test_health_check == true || inputs.test_parallel_dispatch == true || inputs.run_all_tests == true) && 'true' || 'false' }}
|
||||||
secrets: inherit
|
secrets: inherit
|
||||||
|
|
||||||
base-c-test-4-gpu-h100:
|
base-c-test:
|
||||||
|
name: base-c-test-${{ matrix.runner_config }}
|
||||||
needs: [check-changes, call-gate, wait-for-base-b, sgl-kernel-build-wheels, rust-ext-build]
|
needs: [check-changes, call-gate, wait-for-base-b, sgl-kernel-build-wheels, rust-ext-build]
|
||||||
if: ${{ !failure() && !cancelled() }}
|
if: ${{ !failure() && !cancelled() }}
|
||||||
|
strategy:
|
||||||
|
fail-fast: false
|
||||||
|
matrix:
|
||||||
|
include:
|
||||||
|
- {
|
||||||
|
runner_config: 4-gpu-h100,
|
||||||
|
run_timeout: '30',
|
||||||
|
timeout_per_file: '',
|
||||||
|
warmup_deep_gemm_models: 'lmsys/sglang-ci-dsv3-test:4',
|
||||||
|
warmup_server_models: 'lmsys/sglang-ci-dsv3-test:4',
|
||||||
|
}
|
||||||
|
- {
|
||||||
|
runner_config: 8-gpu-h200,
|
||||||
|
run_timeout: '30',
|
||||||
|
timeout_per_file: '1800',
|
||||||
|
warmup_deep_gemm_models: 'deepseek-ai/DeepSeek-V3-0324:8 deepseek-ai/DeepSeek-V3.2:8 zai-org/GLM-5-FP8:8 XiaomiMiMo/MiMo-V2-Flash:4 XiaomiMiMo/MiMo-V2.5:8',
|
||||||
|
warmup_timeout: '60',
|
||||||
|
}
|
||||||
|
- { runner_config: 8-gpu-h20, run_timeout: '30', timeout_per_file: '' }
|
||||||
|
- { runner_config: 4-gpu-b200, run_timeout: '30', timeout_per_file: '1800' }
|
||||||
|
- { runner_config: 8-gpu-b300, run_timeout: '60', timeout_per_file: '3600' }
|
||||||
uses: ./.github/workflows/_pr-test-stage.yml
|
uses: ./.github/workflows/_pr-test-stage.yml
|
||||||
with:
|
with:
|
||||||
self_name: base-c-test-4-gpu-h100
|
self_name: base-c-test-${{ matrix.runner_config }}
|
||||||
runner_config: 4-gpu-h100
|
runner_config: ${{ matrix.runner_config }}
|
||||||
check_changes: ${{ toJson(needs.check-changes.outputs) }}
|
check_changes: ${{ toJson(needs.check-changes.outputs) }}
|
||||||
caller_inputs: ${{ toJson(inputs) }}
|
caller_inputs: ${{ toJson(inputs) }}
|
||||||
partitions: ${{ needs.check-changes.outputs.partitions }}
|
partitions: ${{ needs.check-changes.outputs.partitions }}
|
||||||
run_timeout_minutes: '30'
|
run_timeout_minutes: ${{ matrix.run_timeout }}
|
||||||
warmup_deep_gemm_models: 'lmsys/sglang-ci-dsv3-test:4'
|
timeout_per_file: ${{ matrix.timeout_per_file }}
|
||||||
warmup_server_models: 'lmsys/sglang-ci-dsv3-test:4'
|
warmup_deep_gemm_models: ${{ matrix.warmup_deep_gemm_models }}
|
||||||
|
warmup_server_models: ${{ matrix.warmup_server_models }}
|
||||||
|
# Rows that do not set it keep _pr-test-stage.yml's own default.
|
||||||
|
warmup_timeout_minutes: ${{ matrix.warmup_timeout || '25' }}
|
||||||
rust_ext_artifact: ${{ needs.rust-ext-build.outputs.artifact_name }}
|
rust_ext_artifact: ${{ needs.rust-ext-build.outputs.artifact_name }}
|
||||||
secrets: inherit
|
secrets: inherit
|
||||||
|
|
||||||
base-c-test-8-gpu-h200:
|
# Separate only because `needs` cannot vary per matrix row: this stage takes
|
||||||
needs: [check-changes, call-gate, wait-for-base-b, sgl-kernel-build-wheels, rust-ext-build]
|
# the aarch64 build, and neither table should wait on the other's.
|
||||||
if: ${{ !failure() && !cancelled() }}
|
base-c-test-aarch64:
|
||||||
uses: ./.github/workflows/_pr-test-stage.yml
|
name: base-c-test-${{ matrix.runner_config }}
|
||||||
with:
|
|
||||||
self_name: base-c-test-8-gpu-h200
|
|
||||||
runner_config: 8-gpu-h200
|
|
||||||
check_changes: ${{ toJson(needs.check-changes.outputs) }}
|
|
||||||
caller_inputs: ${{ toJson(inputs) }}
|
|
||||||
partitions: ${{ needs.check-changes.outputs.partitions }}
|
|
||||||
run_timeout_minutes: '30'
|
|
||||||
timeout_per_file: '1800'
|
|
||||||
# Per-model TP must match the test's launch in test/registered/ -- see
|
|
||||||
# FALLBACK_ARGS in scripts/ci/cuda/warmup_deep_gemm.py for extra dp/ep
|
|
||||||
# flags. Only models that actually invoke DeepGEMM kernels at runtime
|
|
||||||
# are listed. Cold-cache ~13 min; warm-cache <=30 s via marker file.
|
|
||||||
# Server CUDA Graph warmup is combined into this step (warmup_server_models unset).
|
|
||||||
warmup_deep_gemm_models: 'deepseek-ai/DeepSeek-V3-0324:8 deepseek-ai/DeepSeek-V3.2:8 zai-org/GLM-5-FP8:8 XiaomiMiMo/MiMo-V2-Flash:4 XiaomiMiMo/MiMo-V2.5:8'
|
|
||||||
warmup_timeout_minutes: '60'
|
|
||||||
rust_ext_artifact: ${{ needs.rust-ext-build.outputs.artifact_name }}
|
|
||||||
secrets: inherit
|
|
||||||
|
|
||||||
base-c-test-8-gpu-h20:
|
|
||||||
needs: [check-changes, call-gate, wait-for-base-b, sgl-kernel-build-wheels, rust-ext-build]
|
|
||||||
if: ${{ !failure() && !cancelled() }}
|
|
||||||
uses: ./.github/workflows/_pr-test-stage.yml
|
|
||||||
with:
|
|
||||||
self_name: base-c-test-8-gpu-h20
|
|
||||||
runner_config: 8-gpu-h20
|
|
||||||
check_changes: ${{ toJson(needs.check-changes.outputs) }}
|
|
||||||
caller_inputs: ${{ toJson(inputs) }}
|
|
||||||
partitions: ${{ needs.check-changes.outputs.partitions }}
|
|
||||||
run_timeout_minutes: '30'
|
|
||||||
rust_ext_artifact: ${{ needs.rust-ext-build.outputs.artifact_name }}
|
|
||||||
secrets: inherit
|
|
||||||
|
|
||||||
base-c-test-4-gpu-b200:
|
|
||||||
needs: [check-changes, call-gate, wait-for-base-b, sgl-kernel-build-wheels, rust-ext-build]
|
|
||||||
if: ${{ !failure() && !cancelled() }}
|
|
||||||
uses: ./.github/workflows/_pr-test-stage.yml
|
|
||||||
with:
|
|
||||||
self_name: base-c-test-4-gpu-b200
|
|
||||||
runner_config: 4-gpu-b200
|
|
||||||
check_changes: ${{ toJson(needs.check-changes.outputs) }}
|
|
||||||
caller_inputs: ${{ toJson(inputs) }}
|
|
||||||
partitions: ${{ needs.check-changes.outputs.partitions }}
|
|
||||||
run_timeout_minutes: '30'
|
|
||||||
timeout_per_file: '1800'
|
|
||||||
rust_ext_artifact: ${{ needs.rust-ext-build.outputs.artifact_name }}
|
|
||||||
secrets: inherit
|
|
||||||
|
|
||||||
base-c-test-4-gpu-gb300:
|
|
||||||
needs: [check-changes, call-gate, wait-for-base-b, sgl-kernel-build-wheels, rust-ext-build-aarch64]
|
needs: [check-changes, call-gate, wait-for-base-b, sgl-kernel-build-wheels, rust-ext-build-aarch64]
|
||||||
if: ${{ !failure() && !cancelled() }}
|
if: ${{ !failure() && !cancelled() }}
|
||||||
|
strategy:
|
||||||
|
fail-fast: false
|
||||||
|
matrix:
|
||||||
|
include:
|
||||||
|
- { runner_config: 4-gpu-gb300, run_timeout: '30', timeout_per_file: '1800' }
|
||||||
uses: ./.github/workflows/_pr-test-stage.yml
|
uses: ./.github/workflows/_pr-test-stage.yml
|
||||||
with:
|
with:
|
||||||
self_name: base-c-test-4-gpu-gb300
|
self_name: base-c-test-${{ matrix.runner_config }}
|
||||||
runner_config: 4-gpu-gb300
|
runner_config: ${{ matrix.runner_config }}
|
||||||
check_changes: ${{ toJson(needs.check-changes.outputs) }}
|
check_changes: ${{ toJson(needs.check-changes.outputs) }}
|
||||||
caller_inputs: ${{ toJson(inputs) }}
|
caller_inputs: ${{ toJson(inputs) }}
|
||||||
partitions: ${{ needs.check-changes.outputs.partitions }}
|
partitions: ${{ needs.check-changes.outputs.partitions }}
|
||||||
run_timeout_minutes: '30'
|
run_timeout_minutes: ${{ matrix.run_timeout }}
|
||||||
timeout_per_file: '1800'
|
timeout_per_file: ${{ matrix.timeout_per_file }}
|
||||||
# The one aarch64 stage, so it takes the aarch64 build, not rust-ext-build's.
|
|
||||||
rust_ext_artifact: ${{ needs.rust-ext-build-aarch64.outputs.artifact_name }}
|
rust_ext_artifact: ${{ needs.rust-ext-build-aarch64.outputs.artifact_name }}
|
||||||
secrets: inherit
|
secrets: inherit
|
||||||
|
|
||||||
base-c-test-8-gpu-b300:
|
|
||||||
needs: [check-changes, call-gate, wait-for-base-b, sgl-kernel-build-wheels, rust-ext-build]
|
|
||||||
if: ${{ !failure() && !cancelled() }}
|
|
||||||
uses: ./.github/workflows/_pr-test-stage.yml
|
|
||||||
with:
|
|
||||||
self_name: base-c-test-8-gpu-b300
|
|
||||||
runner_config: 8-gpu-b300
|
|
||||||
check_changes: ${{ toJson(needs.check-changes.outputs) }}
|
|
||||||
caller_inputs: ${{ toJson(inputs) }}
|
|
||||||
partitions: ${{ needs.check-changes.outputs.partitions }}
|
|
||||||
run_timeout_minutes: '60'
|
|
||||||
timeout_per_file: '3600'
|
|
||||||
rust_ext_artifact: ${{ needs.rust-ext-build.outputs.artifact_name }}
|
|
||||||
secrets: inherit
|
|
||||||
|
|
||||||
# List every build and test job: `skipped` passes here, so an omission turns that
|
# List every build and test job: `skipped` passes here, so an omission turns that
|
||||||
# job's failure into a green run with no tests.
|
# job's failure into a green run with no tests.
|
||||||
pr-test-finish:
|
pr-test-finish:
|
||||||
@@ -536,18 +476,11 @@ jobs:
|
|||||||
|
|
||||||
call-multimodal-gen-tests,
|
call-multimodal-gen-tests,
|
||||||
|
|
||||||
base-a-test-1-gpu-small,
|
base-a-test,
|
||||||
base-a-test-cpu,
|
base-a-test-cpu,
|
||||||
base-b-test-1-gpu-small,
|
base-b-test,
|
||||||
base-b-test-1-gpu-large,
|
base-c-test,
|
||||||
base-b-test-2-gpu-large,
|
base-c-test-aarch64,
|
||||||
base-b-test-4-gpu-b200,
|
|
||||||
base-c-test-4-gpu-h100,
|
|
||||||
base-c-test-8-gpu-h20,
|
|
||||||
base-c-test-8-gpu-h200,
|
|
||||||
base-c-test-4-gpu-b200,
|
|
||||||
base-c-test-4-gpu-gb300,
|
|
||||||
base-c-test-8-gpu-b300,
|
|
||||||
]
|
]
|
||||||
if: always()
|
if: always()
|
||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
|
|||||||
Reference in New Issue
Block a user