ba214ef3d3
Co-authored-by: hnyls2002 <lsyincs@gmail.com> Co-authored-by: Liangsheng Yin <hnyls2002@gmail.com>
625 lines
25 KiB
YAML
625 lines
25 KiB
YAML
name: PR Test
|
|
# Dynamic run-name for /rerun-stage commands to enable URL lookup
|
|
# Format: "[stage-name] sha" for fork PRs, "[stage-name]" for non-fork, default for normal runs
|
|
run-name: ${{ inputs.target_stage && (inputs.pr_head_sha && format('[{0}] {1}', inputs.target_stage, inputs.pr_head_sha) || format('[{0}]', inputs.target_stage)) || '' }}
|
|
|
|
on:
|
|
schedule:
|
|
- cron: '0 1,9,17 * * *' # Run 3x daily: 2am / 10am / 6pm Pacific (PDT)
|
|
pull_request:
|
|
branches: [main]
|
|
workflow_dispatch:
|
|
inputs:
|
|
target_stage:
|
|
description: "Specific stage to run (optional, for quick testing)"
|
|
required: false
|
|
type: string
|
|
default: ""
|
|
force_continue_on_error:
|
|
description: "Force continue-on-error (test scheduled CI behavior)"
|
|
required: false
|
|
type: boolean
|
|
default: false
|
|
pr_head_sha:
|
|
description: "PR head SHA to checkout (for /rerun-stage on fork PRs)"
|
|
required: false
|
|
type: string
|
|
default: ""
|
|
include_wheel_build:
|
|
description: "When set with target_stage, also run sgl-kernel-build-wheels so the target stage uses the freshly-built kernel (for /rerun-stage on PRs that modify sgl-kernel/)"
|
|
required: false
|
|
type: boolean
|
|
default: false
|
|
test_parallel_dispatch:
|
|
description: "Test parallel dispatch behavior (simulates scheduled run)"
|
|
required: false
|
|
type: boolean
|
|
default: false
|
|
workflow_call:
|
|
inputs:
|
|
git_ref:
|
|
description: 'Git ref (branch, tag, or SHA) to test. If not provided, uses the default branch.'
|
|
required: false
|
|
type: string
|
|
default: ''
|
|
run_all_tests:
|
|
description: "Run all tests (for releasing or testing purpose)"
|
|
required: false
|
|
type: boolean
|
|
default: false
|
|
skip_stage_health_check:
|
|
description: "Skip stage health check fast-fail (e.g. for release branch cuts)"
|
|
required: false
|
|
type: boolean
|
|
default: false
|
|
|
|
concurrency:
|
|
# Concurrency group structure: pr-test-{event}-{branch}-{pr_sha}-{stage}
|
|
# - event_name prevents scheduled runs from colliding with fork PRs whose branch is named 'main'
|
|
# (without it, both resolve the branch segment to 'main' and block each other)
|
|
# - github.head_ref (pull_request) or github.ref_name (workflow_dispatch) normalizes to branch name
|
|
# - pr_head_sha isolates /rerun-stage from main branch runs
|
|
# - target_stage allows parallel stage dispatches to run independently
|
|
group: pr-test-${{ github.event_name }}-${{ github.head_ref || github.ref_name || 'default' }}-${{ inputs.pr_head_sha || 'current' }}-${{ inputs.target_stage || inputs.git_ref || 'all' }}
|
|
cancel-in-progress: ${{ github.event_name != 'workflow_call' }}
|
|
|
|
env:
|
|
SGLANG_IS_IN_CI: true
|
|
SGLANG_CUDA_COREDUMP: "1"
|
|
SGLANG_JIT_DEEPGEMM_FAST_WARMUP: true
|
|
SKIP_STAGE_HEALTH_CHECK: ${{ inputs.skip_stage_health_check == true && 'true' || 'false' }}
|
|
# TEMP: rebuild deepep against the new torch for torch-211-merge PR only — revert before merging to main.
|
|
FORCE_REBUILD_DEEPEP: '1'
|
|
# Schedule / main-branch dispatch / workflow_call from main use refs/heads/main; PR events use refs/pull/*/merge
|
|
PR_TEST_BYPASS_MAINTENANCE_ON_MAIN: ${{ github.ref == 'refs/heads/main' && 'true' || 'false' }}
|
|
USE_VENV: false
|
|
|
|
permissions:
|
|
actions: write
|
|
contents: read
|
|
issues: read
|
|
pull-requests: read
|
|
|
|
jobs:
|
|
# =============================================== check changes ====================================================
|
|
check-changes:
|
|
uses: ./.github/workflows/_pr-test-check-changes.yml
|
|
with:
|
|
pr_head_sha: ${{ inputs.pr_head_sha || '' }}
|
|
git_ref: ${{ inputs.git_ref || '' }}
|
|
target_stage: ${{ inputs.target_stage || '' }}
|
|
include_wheel_build: ${{ inputs.include_wheel_build == true }}
|
|
run_all_tests: ${{ inputs.run_all_tests == true }}
|
|
force_continue_on_error: ${{ inputs.force_continue_on_error == true }}
|
|
secrets: inherit
|
|
|
|
# =============================================== Wait Jobs for Sequential PR Execution ====================================================
|
|
# These jobs poll GitHub API to wait for previous stages to complete.
|
|
# For PR runs: wait jobs run and enforce sequential execution via polling.
|
|
# For scheduled runs: wait jobs are skipped, enabling parallel execution for easier retry.
|
|
# For PRs with the `bypass-fastfail` label: wait jobs run but return success immediately
|
|
# (handled inside the wait-for-jobs action), so downstream stages dispatch in parallel.
|
|
|
|
wait-for-stage-a:
|
|
needs: [check-changes, call-gate]
|
|
if: |
|
|
always() &&
|
|
!cancelled() &&
|
|
github.event_name == 'pull_request' &&
|
|
!inputs.target_stage &&
|
|
inputs.test_parallel_dispatch != true &&
|
|
(needs.check-changes.outputs.main_package == 'true' || needs.check-changes.outputs.sgl_kernel == 'true') &&
|
|
(needs.call-gate.result == 'success' || needs.call-gate.result == 'skipped')
|
|
runs-on: ubuntu-latest
|
|
outputs:
|
|
stage_a_result: ${{ steps.wait.outputs.result }}
|
|
steps:
|
|
- uses: actions/checkout@v4
|
|
|
|
- uses: ./.github/actions/check-maintenance
|
|
|
|
- uses: ./.github/actions/wait-for-jobs
|
|
id: wait
|
|
with:
|
|
stage-name: stage-a
|
|
jobs: |
|
|
[
|
|
{"prefix": "stage-a-test-1-gpu-small", "expected_count": ${{ fromJson(needs.check-changes.outputs.partitions)['stage-a-test-1-gpu-small'].size }}},
|
|
{"prefix": "stage-a-test-cpu", "expected_count": ${{ fromJson(needs.check-changes.outputs.partitions)['stage-a-test-cpu'].size }}}
|
|
]
|
|
max-wait-minutes: '240'
|
|
|
|
wait-for-stage-b:
|
|
needs: [check-changes, call-gate, wait-for-stage-a]
|
|
if: |
|
|
always() &&
|
|
!cancelled() &&
|
|
github.event_name == 'pull_request' &&
|
|
!inputs.target_stage &&
|
|
inputs.test_parallel_dispatch != true &&
|
|
(needs.check-changes.outputs.main_package == 'true' || needs.check-changes.outputs.sgl_kernel == 'true') &&
|
|
(needs.wait-for-stage-a.result == 'success' || needs.wait-for-stage-a.result == 'skipped') &&
|
|
(needs.call-gate.result == 'success' || needs.call-gate.result == 'skipped')
|
|
runs-on: ubuntu-latest
|
|
outputs:
|
|
stage_b_result: ${{ steps.wait.outputs.result }}
|
|
steps:
|
|
- uses: actions/checkout@v4
|
|
|
|
- uses: ./.github/actions/check-maintenance
|
|
|
|
- uses: ./.github/actions/wait-for-jobs
|
|
id: wait
|
|
with:
|
|
stage-name: stage-b
|
|
jobs: |
|
|
[
|
|
{"prefix": "stage-b-test-1-gpu-small", "expected_count": ${{ fromJson(needs.check-changes.outputs.partitions)['stage-b-test-1-gpu-small'].size }}},
|
|
{"prefix": "stage-b-test-1-gpu-large", "expected_count": ${{ fromJson(needs.check-changes.outputs.partitions)['stage-b-test-1-gpu-large'].size }}},
|
|
{"prefix": "stage-b-test-2-gpu-large", "expected_count": ${{ fromJson(needs.check-changes.outputs.partitions)['stage-b-test-2-gpu-large'].size }}},
|
|
{"prefix": "stage-b-test-4-gpu-b200", "expected_count": ${{ fromJson(needs.check-changes.outputs.partitions)['stage-b-test-4-gpu-b200'].size }}}
|
|
]
|
|
max-wait-minutes: '480'
|
|
|
|
# =============================================== PR Gate ====================================================
|
|
call-gate:
|
|
needs: check-changes
|
|
# Skip for scheduled runs (they run all tests) and when target_stage is specified
|
|
if: |
|
|
github.event_name != 'schedule' &&
|
|
inputs.test_parallel_dispatch != true &&
|
|
!inputs.target_stage &&
|
|
(
|
|
needs.check-changes.outputs.main_package == 'true' ||
|
|
needs.check-changes.outputs.sgl_kernel == 'true' ||
|
|
needs.check-changes.outputs.jit_kernel == 'true' ||
|
|
needs.check-changes.outputs.multimodal_gen == 'true'
|
|
)
|
|
uses: ./.github/workflows/pr-gate.yml
|
|
secrets: inherit
|
|
|
|
# =============================================== sgl-kernel ====================================================
|
|
|
|
sgl-kernel-build-wheels:
|
|
needs: [check-changes, call-gate]
|
|
# Skip for scheduled runs (they run stages independently). Runs in target_stage mode only when
|
|
# include_wheel_build is true (i.e. /rerun-stage on a PR with sgl-kernel changes), so the
|
|
# target stage can download the freshly-built wheel.
|
|
#
|
|
# `always()` lets us run when call-gate is skipped (which it always is in target_stage mode by
|
|
# design). The explicit needs.<x>.result checks preserve old gating for the normal PR path.
|
|
if: |
|
|
always() &&
|
|
github.event_name != 'schedule' &&
|
|
inputs.test_parallel_dispatch != true &&
|
|
needs.check-changes.result == 'success' &&
|
|
needs.check-changes.outputs.sgl_kernel == 'true' &&
|
|
(
|
|
(!inputs.target_stage && needs.call-gate.result == 'success') ||
|
|
(inputs.target_stage && inputs.include_wheel_build)
|
|
)
|
|
uses: ./.github/workflows/_pr-test-sgl-kernel-build.yml
|
|
with:
|
|
runs_on: x64-kernel-build-node
|
|
job_display_name: Build Wheel
|
|
pr_head_sha: ${{ inputs.pr_head_sha || '' }}
|
|
git_ref: ${{ inputs.git_ref || '' }}
|
|
skip_stage_health_check: ${{ inputs.skip_stage_health_check == true }}
|
|
secrets: inherit
|
|
|
|
sgl-kernel-build-wheels-arm:
|
|
needs: [check-changes, call-gate]
|
|
# Skip for scheduled runs (they run stages independently). Runs in target_stage mode only when
|
|
# include_wheel_build is true (i.e. /rerun-stage on a PR with sgl-kernel changes).
|
|
#
|
|
# See sgl-kernel-build-wheels above for the always() + result-check rationale.
|
|
if: |
|
|
always() &&
|
|
github.event_name != 'schedule' &&
|
|
inputs.test_parallel_dispatch != true &&
|
|
needs.check-changes.result == 'success' &&
|
|
needs.check-changes.outputs.sgl_kernel == 'true' &&
|
|
(
|
|
(!inputs.target_stage && needs.call-gate.result == 'success') ||
|
|
(inputs.target_stage && inputs.include_wheel_build)
|
|
)
|
|
uses: ./.github/workflows/_pr-test-sgl-kernel-build.yml
|
|
with:
|
|
runs_on: arm-kernel-build-node
|
|
job_display_name: Build Wheel Arm
|
|
arch_suffix: '-aarch64'
|
|
pr_head_sha: ${{ inputs.pr_head_sha || '' }}
|
|
git_ref: ${{ inputs.git_ref || '' }}
|
|
skip_stage_health_check: ${{ inputs.skip_stage_health_check == true }}
|
|
secrets: inherit
|
|
|
|
call-sgl-kernel-tests:
|
|
needs: [check-changes, call-gate, sgl-kernel-build-wheels]
|
|
if: |
|
|
github.event_name != 'schedule' &&
|
|
inputs.test_parallel_dispatch != true &&
|
|
!inputs.target_stage &&
|
|
needs.check-changes.outputs.sgl_kernel == 'true'
|
|
uses: ./.github/workflows/pr-test-sgl-kernel.yml
|
|
with:
|
|
sgl_kernel: ${{ needs.check-changes.outputs.sgl_kernel }}
|
|
b200_runner: ${{ needs.check-changes.outputs.b200_runner }}
|
|
pr_head_sha: ${{ inputs.pr_head_sha || '' }}
|
|
git_ref: ${{ inputs.git_ref || '' }}
|
|
skip_stage_health_check: ${{ inputs.skip_stage_health_check == true }}
|
|
secrets: inherit
|
|
|
|
# =============================================== jit-kernel ====================================================
|
|
|
|
call-jit-kernel-tests:
|
|
needs: [check-changes, call-gate, sgl-kernel-build-wheels]
|
|
if: |
|
|
always() &&
|
|
!failure() && !cancelled() &&
|
|
github.event_name != 'schedule' &&
|
|
inputs.test_parallel_dispatch != true &&
|
|
!inputs.target_stage &&
|
|
needs.check-changes.outputs.jit_kernel == 'true'
|
|
uses: ./.github/workflows/pr-test-jit-kernel.yml
|
|
with:
|
|
jit_kernel: ${{ needs.check-changes.outputs.jit_kernel }}
|
|
sgl_kernel: ${{ needs.check-changes.outputs.sgl_kernel }}
|
|
b200_runner: ${{ needs.check-changes.outputs.b200_runner }}
|
|
pr_head_sha: ${{ inputs.pr_head_sha || '' }}
|
|
git_ref: ${{ inputs.git_ref || '' }}
|
|
target_stage: ${{ inputs.target_stage || '' }}
|
|
test_parallel_dispatch: ${{ inputs.test_parallel_dispatch == true && 'true' || 'false' }}
|
|
skip_stage_health_check: ${{ inputs.skip_stage_health_check == true }}
|
|
secrets: inherit
|
|
|
|
# =============================================== primary ====================================================
|
|
|
|
# Runs on 5090 (32GB, SM120)
|
|
stage-a-test-1-gpu-small:
|
|
needs: [check-changes, call-gate, sgl-kernel-build-wheels]
|
|
if: ${{ !failure() && !cancelled() }}
|
|
uses: ./.github/workflows/_pr-test-stage.yml
|
|
with:
|
|
self_name: stage-a-test-1-gpu-small
|
|
runner_config: 1-gpu-small
|
|
runs_on: 1-gpu-5090
|
|
check_changes: ${{ toJson(needs.check-changes.outputs) }}
|
|
caller_inputs: ${{ toJson(inputs) }}
|
|
partitions: ${{ needs.check-changes.outputs.partitions }}
|
|
run_timeout_minutes: '10'
|
|
secrets: inherit
|
|
|
|
stage-a-test-cpu:
|
|
needs: [check-changes, call-gate]
|
|
if: |
|
|
always() &&
|
|
(
|
|
(inputs.target_stage == 'stage-a-test-cpu') ||
|
|
(
|
|
!inputs.target_stage &&
|
|
((github.event_name == 'schedule' || inputs.test_parallel_dispatch == true) || (!failure() && !cancelled())) &&
|
|
(needs.check-changes.outputs.main_package == 'true')
|
|
)
|
|
)
|
|
runs-on: ubuntu-latest
|
|
timeout-minutes: 240
|
|
strategy:
|
|
fail-fast: false
|
|
max-parallel: ${{ fromJson(needs.check-changes.outputs.partitions)['stage-a-test-cpu'].max_parallel }}
|
|
matrix:
|
|
partition: ${{ fromJson(needs.check-changes.outputs.partitions)['stage-a-test-cpu'].arr }}
|
|
steps:
|
|
- name: Free disk space
|
|
run: |
|
|
sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc
|
|
df -h
|
|
|
|
- name: Checkout code
|
|
uses: actions/checkout@v4
|
|
with:
|
|
ref: ${{ inputs.pr_head_sha || inputs.git_ref || github.sha }}
|
|
|
|
- uses: ./.github/actions/check-stage-health
|
|
|
|
- uses: ./.github/actions/check-maintenance
|
|
|
|
- name: Set up Python
|
|
uses: actions/setup-python@v5
|
|
with:
|
|
python-version: '3.10'
|
|
|
|
- name: Install uv
|
|
uses: astral-sh/setup-uv@v5
|
|
|
|
# Needed by setuptools-rust to build the bundled native gRPC extension
|
|
# (rust/sglang-grpc) when installing the main `sglang` wheel from source.
|
|
- name: Install protoc + Rust toolchain
|
|
timeout-minutes: 10
|
|
run: bash scripts/ci/utils/install_rust_protoc.sh
|
|
|
|
- name: Rust cache (sglang-grpc)
|
|
uses: Swatinem/rust-cache@v2
|
|
with:
|
|
workspaces: rust/sglang-grpc
|
|
shared-key: "sglang-grpc-cpu"
|
|
save-if: ${{ matrix.partition == 0 }}
|
|
|
|
# uv pip targets a venv by default; setup-python has no venv — install into that interpreter (see UV_SYSTEM_PYTHON in https://docs.astral.sh/uv/guides/integration/github/)
|
|
- name: Install dependencies
|
|
timeout-minutes: 20
|
|
env:
|
|
UV_SYSTEM_PYTHON: "1"
|
|
run: |
|
|
uv pip install -e "python[dev]" --index-strategy unsafe-best-match --prerelease allow
|
|
|
|
- name: Run test
|
|
timeout-minutes: 10
|
|
env:
|
|
CONTINUE_ON_ERROR_FLAG: ${{ needs.check-changes.outputs.continue_on_error == 'true' && '--continue-on-error' || '' }}
|
|
run: |
|
|
cd test/
|
|
python3 run_suite.py --hw cpu --suite stage-a-test-cpu --auto-partition-id ${{ matrix.partition }} --auto-partition-size ${{ fromJson(needs.check-changes.outputs.partitions)['stage-a-test-cpu'].size }} $CONTINUE_ON_ERROR_FLAG
|
|
|
|
# Runs on 5090 (32GB, SM120)
|
|
stage-b-test-1-gpu-small:
|
|
needs: [check-changes, call-gate, wait-for-stage-a, sgl-kernel-build-wheels]
|
|
if: ${{ !failure() && !cancelled() }}
|
|
uses: ./.github/workflows/_pr-test-stage.yml
|
|
with:
|
|
self_name: stage-b-test-1-gpu-small
|
|
runner_config: 1-gpu-small
|
|
runs_on: 1-gpu-5090
|
|
check_changes: ${{ toJson(needs.check-changes.outputs) }}
|
|
caller_inputs: ${{ toJson(inputs) }}
|
|
partitions: ${{ needs.check-changes.outputs.partitions }}
|
|
run_timeout_minutes: '30'
|
|
secrets: inherit
|
|
|
|
# Runs on H100 (80GB, SM90) - tests that don't pass on 5090 (FA3, FP8, high VRAM, etc.)
|
|
stage-b-test-1-gpu-large:
|
|
needs: [check-changes, call-gate, wait-for-stage-a, sgl-kernel-build-wheels]
|
|
if: ${{ !failure() && !cancelled() }}
|
|
uses: ./.github/workflows/_pr-test-stage.yml
|
|
with:
|
|
self_name: stage-b-test-1-gpu-large
|
|
runner_config: 1-gpu-large
|
|
runs_on: 1-gpu-h100
|
|
check_changes: ${{ toJson(needs.check-changes.outputs) }}
|
|
caller_inputs: ${{ toJson(inputs) }}
|
|
partitions: ${{ needs.check-changes.outputs.partitions }}
|
|
run_timeout_minutes: '30'
|
|
timeout_per_file: '1800'
|
|
secrets: inherit
|
|
|
|
stage-b-test-2-gpu-large:
|
|
needs: [check-changes, call-gate, wait-for-stage-a, sgl-kernel-build-wheels]
|
|
if: ${{ !failure() && !cancelled() }}
|
|
uses: ./.github/workflows/_pr-test-stage.yml
|
|
with:
|
|
self_name: stage-b-test-2-gpu-large
|
|
runner_config: 2-gpu-large
|
|
runs_on: 2-gpu-h100
|
|
check_changes: ${{ toJson(needs.check-changes.outputs) }}
|
|
caller_inputs: ${{ toJson(inputs) }}
|
|
partitions: ${{ needs.check-changes.outputs.partitions }}
|
|
run_timeout_minutes: '30'
|
|
secrets: inherit
|
|
|
|
stage-b-test-4-gpu-b200:
|
|
needs: [check-changes, call-gate, wait-for-stage-a, sgl-kernel-build-wheels]
|
|
if: ${{ !failure() && !cancelled() }}
|
|
uses: ./.github/workflows/_pr-test-stage.yml
|
|
with:
|
|
self_name: stage-b-test-4-gpu-b200
|
|
runner_config: 4-gpu-b200
|
|
runs_on: ${{ needs.check-changes.outputs.b200_runner }}
|
|
check_changes: ${{ toJson(needs.check-changes.outputs) }}
|
|
caller_inputs: ${{ toJson(inputs) }}
|
|
partitions: ${{ needs.check-changes.outputs.partitions }}
|
|
run_timeout_minutes: '40'
|
|
extra_pytest_path: 'python/sglang/jit_kernel/tests/test_flash_attention_4.py'
|
|
secrets: inherit
|
|
|
|
call-multimodal-gen-tests:
|
|
needs: [check-changes, call-gate, sgl-kernel-build-wheels]
|
|
if: |
|
|
always() &&
|
|
!cancelled() &&
|
|
(
|
|
inputs.target_stage == 'multimodal-gen-test-1-gpu' ||
|
|
inputs.target_stage == 'multimodal-gen-test-2-gpu' ||
|
|
inputs.target_stage == 'multimodal-gen-component-accuracy' ||
|
|
inputs.target_stage == 'multimodal-gen-component-accuracy-1-gpu' ||
|
|
inputs.target_stage == 'multimodal-gen-component-accuracy-2-gpu' ||
|
|
inputs.target_stage == 'multimodal-gen-test-1-b200' ||
|
|
inputs.target_stage == 'multimodal-gen-unit-test' ||
|
|
(
|
|
!inputs.target_stage &&
|
|
((github.event_name == 'schedule' || inputs.test_parallel_dispatch == true) || (!failure() && !cancelled())) &&
|
|
needs.check-changes.outputs.multimodal_gen == 'true'
|
|
)
|
|
)
|
|
uses: ./.github/workflows/pr-test-multimodal-gen.yml
|
|
with:
|
|
multimodal_gen: ${{ needs.check-changes.outputs.multimodal_gen }}
|
|
sgl_kernel: ${{ needs.check-changes.outputs.sgl_kernel }}
|
|
b200_runner: ${{ needs.check-changes.outputs.b200_runner }}
|
|
continue_on_error: ${{ needs.check-changes.outputs.continue_on_error }}
|
|
pr_head_sha: ${{ inputs.pr_head_sha || '' }}
|
|
git_ref: ${{ inputs.git_ref || '' }}
|
|
target_stage: ${{ inputs.target_stage || '' }}
|
|
test_parallel_dispatch: ${{ inputs.test_parallel_dispatch == true && 'true' || 'false' }}
|
|
caller_needs_failure: ${{ (needs.call-gate.result == 'failure' || needs.sgl-kernel-build-wheels.result == 'failure' || needs.check-changes.result == 'failure') && 'true' || 'false' }}
|
|
skip_stage_health_check: ${{ inputs.skip_stage_health_check == true && 'true' || 'false' }}
|
|
secrets: inherit
|
|
|
|
stage-c-test-4-gpu-h100:
|
|
needs: [check-changes, call-gate, wait-for-stage-b, sgl-kernel-build-wheels]
|
|
if: ${{ !failure() && !cancelled() }}
|
|
uses: ./.github/workflows/_pr-test-stage.yml
|
|
with:
|
|
self_name: stage-c-test-4-gpu-h100
|
|
runner_config: 4-gpu-h100
|
|
runs_on: 4-gpu-h100
|
|
check_changes: ${{ toJson(needs.check-changes.outputs) }}
|
|
caller_inputs: ${{ toJson(inputs) }}
|
|
partitions: ${{ needs.check-changes.outputs.partitions }}
|
|
run_timeout_minutes: '30'
|
|
secrets: inherit
|
|
|
|
stage-c-test-8-gpu-h200:
|
|
needs: [check-changes, call-gate, wait-for-stage-b, sgl-kernel-build-wheels]
|
|
if: ${{ !failure() && !cancelled() }}
|
|
uses: ./.github/workflows/_pr-test-stage.yml
|
|
with:
|
|
self_name: stage-c-test-8-gpu-h200
|
|
runner_config: 8-gpu-h200
|
|
runs_on: 8-gpu-h200
|
|
check_changes: ${{ toJson(needs.check-changes.outputs) }}
|
|
caller_inputs: ${{ toJson(inputs) }}
|
|
partitions: ${{ needs.check-changes.outputs.partitions }}
|
|
run_timeout_minutes: '30'
|
|
# Per-model TP must match the test's launch in test/registered/ -- see
|
|
# FALLBACK_ARGS in scripts/ci/cuda/warmup_deep_gemm.py for extra dp/ep
|
|
# flags. Only models that actually invoke DeepGEMM kernels at runtime
|
|
# are listed. Cold-cache ~13 min; warm-cache <=30 s via marker file.
|
|
# Server CUDA Graph warmup is combined into this step (warmup_server_models unset).
|
|
warmup_deep_gemm_models: 'deepseek-ai/DeepSeek-V3-0324:8 deepseek-ai/DeepSeek-V3.2:8 zai-org/GLM-5-FP8:8 XiaomiMiMo/MiMo-V2-Flash:4 XiaomiMiMo/MiMo-V2.5:8'
|
|
warmup_timeout_minutes: '60'
|
|
secrets: inherit
|
|
|
|
stage-c-test-8-gpu-h20:
|
|
needs: [check-changes, call-gate, wait-for-stage-b, sgl-kernel-build-wheels]
|
|
if: ${{ !failure() && !cancelled() }}
|
|
uses: ./.github/workflows/_pr-test-stage.yml
|
|
with:
|
|
self_name: stage-c-test-8-gpu-h20
|
|
runner_config: 8-gpu-h20
|
|
runs_on: 8-gpu-h20
|
|
check_changes: ${{ toJson(needs.check-changes.outputs) }}
|
|
caller_inputs: ${{ toJson(inputs) }}
|
|
partitions: ${{ needs.check-changes.outputs.partitions }}
|
|
run_timeout_minutes: '30'
|
|
secrets: inherit
|
|
|
|
stage-c-test-deepep-4-gpu-h100:
|
|
needs: [check-changes, call-gate, wait-for-stage-b, sgl-kernel-build-wheels]
|
|
if: ${{ !failure() && !cancelled() }}
|
|
uses: ./.github/workflows/_pr-test-stage.yml
|
|
with:
|
|
self_name: stage-c-test-deepep-4-gpu-h100
|
|
runner_config: deepep-4-gpu-h100
|
|
runs_on: 4-gpu-h100
|
|
check_changes: ${{ toJson(needs.check-changes.outputs) }}
|
|
caller_inputs: ${{ toJson(inputs) }}
|
|
partitions: ${{ needs.check-changes.outputs.partitions }}
|
|
run_timeout_minutes: '30'
|
|
warmup_deep_gemm_models: 'lmsys/sglang-ci-dsv3-test:4'
|
|
warmup_server_models: 'lmsys/sglang-ci-dsv3-test:4'
|
|
secrets: inherit
|
|
|
|
stage-c-test-4-gpu-b200:
|
|
needs: [check-changes, call-gate, wait-for-stage-b, sgl-kernel-build-wheels]
|
|
if: ${{ !failure() && !cancelled() }}
|
|
uses: ./.github/workflows/_pr-test-stage.yml
|
|
with:
|
|
self_name: stage-c-test-4-gpu-b200
|
|
runner_config: 4-gpu-b200
|
|
runs_on: ${{ needs.check-changes.outputs.b200_runner }}
|
|
check_changes: ${{ toJson(needs.check-changes.outputs) }}
|
|
caller_inputs: ${{ toJson(inputs) }}
|
|
partitions: ${{ needs.check-changes.outputs.partitions }}
|
|
run_timeout_minutes: '30'
|
|
timeout_per_file: '1800'
|
|
secrets: inherit
|
|
|
|
stage-c-test-dsv4-4-gpu-b200:
|
|
needs: [check-changes, call-gate, wait-for-stage-b, sgl-kernel-build-wheels]
|
|
if: ${{ !failure() && !cancelled() }}
|
|
uses: ./.github/workflows/_pr-test-stage.yml
|
|
with:
|
|
self_name: stage-c-test-dsv4-4-gpu-b200
|
|
runner_config: dsv4-4-gpu-b200
|
|
runs_on: ${{ needs.check-changes.outputs.b200_runner }}
|
|
check_changes: ${{ toJson(needs.check-changes.outputs) }}
|
|
caller_inputs: ${{ toJson(inputs) }}
|
|
partitions: ${{ needs.check-changes.outputs.partitions }}
|
|
run_timeout_minutes: '45'
|
|
timeout_per_file: '1800'
|
|
secrets: inherit
|
|
|
|
stage-c-test-dsv4-8-gpu-h200:
|
|
needs: [check-changes, call-gate, wait-for-stage-b, sgl-kernel-build-wheels]
|
|
if: ${{ !failure() && !cancelled() }}
|
|
uses: ./.github/workflows/_pr-test-stage.yml
|
|
with:
|
|
self_name: stage-c-test-dsv4-8-gpu-h200
|
|
runner_config: dsv4-8-gpu-h200
|
|
runs_on: 8-gpu-h200
|
|
check_changes: ${{ toJson(needs.check-changes.outputs) }}
|
|
caller_inputs: ${{ toJson(inputs) }}
|
|
partitions: ${{ needs.check-changes.outputs.partitions }}
|
|
run_timeout_minutes: '30'
|
|
timeout_per_file: '1800'
|
|
secrets: inherit
|
|
|
|
pr-test-finish:
|
|
needs:
|
|
[
|
|
call-gate,
|
|
check-changes,
|
|
|
|
sgl-kernel-build-wheels,
|
|
sgl-kernel-build-wheels-arm,
|
|
call-sgl-kernel-tests,
|
|
|
|
wait-for-stage-a,
|
|
wait-for-stage-b,
|
|
|
|
call-jit-kernel-tests,
|
|
|
|
call-multimodal-gen-tests,
|
|
|
|
stage-a-test-1-gpu-small,
|
|
stage-a-test-cpu,
|
|
stage-b-test-1-gpu-small,
|
|
stage-b-test-1-gpu-large,
|
|
stage-b-test-2-gpu-large,
|
|
stage-b-test-4-gpu-b200,
|
|
stage-c-test-4-gpu-h100,
|
|
stage-c-test-8-gpu-h20,
|
|
stage-c-test-8-gpu-h200,
|
|
stage-c-test-deepep-4-gpu-h100,
|
|
stage-c-test-4-gpu-b200,
|
|
stage-c-test-dsv4-4-gpu-b200,
|
|
stage-c-test-dsv4-8-gpu-h200,
|
|
# stage-c-test-4-gpu-gb200, # Temporarily disabled — no GB200 runner
|
|
]
|
|
if: always()
|
|
runs-on: ubuntu-latest
|
|
steps:
|
|
- name: Check all dependent job statuses
|
|
run: |
|
|
# Convert the 'needs' context to a JSON string
|
|
json_needs='${{ toJson(needs) }}'
|
|
|
|
# Get a list of all job names from the JSON keys
|
|
job_names=$(echo "$json_needs" | jq -r 'keys_unsorted[]')
|
|
|
|
for job in $job_names; do
|
|
# For each job, extract its result
|
|
result=$(echo "$json_needs" | jq -r --arg j "$job" '.[$j].result')
|
|
|
|
# Print the job name and its result
|
|
echo "$job: $result"
|
|
|
|
# Check for failure or cancellation and exit if found
|
|
if [[ "$result" == "failure" || "$result" == "cancelled" ]]; then
|
|
echo "The above jobs failed."
|
|
exit 1
|
|
fi
|
|
done
|
|
# If the loop completes, all jobs were successful
|
|
echo "All jobs completed successfully"
|
|
exit 0
|