1379 lines
50 KiB
YAML
1379 lines
50 KiB
YAML
name: PR Test
|
|
# Dynamic run-name for /rerun-stage commands to enable URL lookup
|
|
# Format: "[stage-name] sha" for fork PRs, "[stage-name]" for non-fork, default for normal runs
|
|
run-name: ${{ inputs.target_stage && (inputs.pr_head_sha && format('[{0}] {1}', inputs.target_stage, inputs.pr_head_sha) || format('[{0}]', inputs.target_stage)) || '' }}
|
|
|
|
on:
|
|
schedule:
|
|
- cron: '0 1,9,17 * * *' # Run 3x daily: 2am / 10am / 6pm Pacific (PDT)
|
|
pull_request:
|
|
branches: [main]
|
|
workflow_dispatch:
|
|
inputs:
|
|
target_stage:
|
|
description: "Specific stage to run (optional, for quick testing)"
|
|
required: false
|
|
type: string
|
|
default: ""
|
|
force_continue_on_error:
|
|
description: "Force continue-on-error (test scheduled CI behavior)"
|
|
required: false
|
|
type: boolean
|
|
default: false
|
|
pr_head_sha:
|
|
description: "PR head SHA to checkout (for /rerun-stage on fork PRs)"
|
|
required: false
|
|
type: string
|
|
default: ""
|
|
include_wheel_build:
|
|
description: "When set with target_stage, also run sgl-kernel-build-wheels so the target stage uses the freshly-built kernel (for /rerun-stage on PRs that modify sgl-kernel/)"
|
|
required: false
|
|
type: boolean
|
|
default: false
|
|
test_parallel_dispatch:
|
|
description: "Test parallel dispatch behavior (simulates scheduled run)"
|
|
required: false
|
|
type: boolean
|
|
default: false
|
|
workflow_call:
|
|
inputs:
|
|
git_ref:
|
|
description: 'Git ref (branch, tag, or SHA) to test. If not provided, uses the default branch.'
|
|
required: false
|
|
type: string
|
|
default: ''
|
|
run_all_tests:
|
|
description: "Run all tests (for releasing or testing purpose)"
|
|
required: false
|
|
type: boolean
|
|
default: false
|
|
skip_stage_health_check:
|
|
description: "Skip stage health check fast-fail (e.g. for release branch cuts)"
|
|
required: false
|
|
type: boolean
|
|
default: false
|
|
|
|
concurrency:
|
|
# Concurrency group structure: pr-test-{event}-{branch}-{pr_sha}-{stage}
|
|
# - event_name prevents scheduled runs from colliding with fork PRs whose branch is named 'main'
|
|
# (without it, both resolve the branch segment to 'main' and block each other)
|
|
# - github.head_ref (pull_request) or github.ref_name (workflow_dispatch) normalizes to branch name
|
|
# - pr_head_sha isolates /rerun-stage from main branch runs
|
|
# - target_stage allows parallel stage dispatches to run independently
|
|
group: pr-test-${{ github.event_name }}-${{ github.head_ref || github.ref_name || 'default' }}-${{ inputs.pr_head_sha || 'current' }}-${{ inputs.target_stage || inputs.git_ref || 'all' }}
|
|
cancel-in-progress: ${{ github.event_name != 'workflow_call' }}
|
|
|
|
env:
|
|
SGLANG_IS_IN_CI: true
|
|
SGLANG_CUDA_COREDUMP: "1"
|
|
SGLANG_JIT_DEEPGEMM_FAST_WARMUP: true
|
|
SKIP_STAGE_HEALTH_CHECK: ${{ inputs.skip_stage_health_check == true && 'true' || 'false' }}
|
|
# TEMP: rebuild deepep against the new torch for torch-211-merge PR only — revert before merging to main.
|
|
FORCE_REBUILD_DEEPEP: '1'
|
|
# Schedule / main-branch dispatch / workflow_call from main use refs/heads/main; PR events use refs/pull/*/merge
|
|
PR_TEST_BYPASS_MAINTENANCE_ON_MAIN: ${{ github.ref == 'refs/heads/main' && 'true' || 'false' }}
|
|
USE_VENV: false
|
|
|
|
permissions:
|
|
actions: write
|
|
contents: read
|
|
issues: read
|
|
pull-requests: read
|
|
|
|
jobs:
|
|
# =============================================== check changes ====================================================
|
|
check-changes:
|
|
uses: ./.github/workflows/_pr-test-check-changes.yml
|
|
with:
|
|
pr_head_sha: ${{ inputs.pr_head_sha || '' }}
|
|
git_ref: ${{ inputs.git_ref || '' }}
|
|
target_stage: ${{ inputs.target_stage || '' }}
|
|
include_wheel_build: ${{ inputs.include_wheel_build == true }}
|
|
run_all_tests: ${{ inputs.run_all_tests == true }}
|
|
force_continue_on_error: ${{ inputs.force_continue_on_error == true }}
|
|
secrets: inherit
|
|
|
|
# =============================================== Wait Jobs for Sequential PR Execution ====================================================
|
|
# These jobs poll GitHub API to wait for previous stages to complete.
|
|
# For PR runs: wait jobs run and enforce sequential execution via polling.
|
|
# For scheduled runs: wait jobs are skipped, enabling parallel execution for easier retry.
|
|
# For PRs with the `bypass-fastfail` label: wait jobs run but return success immediately
|
|
# (handled inside the wait-for-jobs action), so downstream stages dispatch in parallel.
|
|
|
|
wait-for-stage-a:
|
|
needs: [check-changes, call-gate]
|
|
if: |
|
|
always() &&
|
|
!cancelled() &&
|
|
github.event_name == 'pull_request' &&
|
|
!inputs.target_stage &&
|
|
inputs.test_parallel_dispatch != true &&
|
|
(needs.check-changes.outputs.main_package == 'true' || needs.check-changes.outputs.sgl_kernel == 'true') &&
|
|
(needs.call-gate.result == 'success' || needs.call-gate.result == 'skipped')
|
|
runs-on: ubuntu-latest
|
|
outputs:
|
|
stage_a_result: ${{ steps.wait.outputs.result }}
|
|
steps:
|
|
- uses: actions/checkout@v4
|
|
|
|
- uses: ./.github/actions/check-maintenance
|
|
|
|
- uses: ./.github/actions/wait-for-jobs
|
|
id: wait
|
|
with:
|
|
stage-name: stage-a
|
|
jobs: '["stage-a-test-1-gpu-small", {"prefix": "stage-a-test-cpu", "expected_count": 4}]'
|
|
max-wait-minutes: '240'
|
|
|
|
wait-for-stage-b:
|
|
needs: [check-changes, call-gate, wait-for-stage-a]
|
|
if: |
|
|
always() &&
|
|
!cancelled() &&
|
|
github.event_name == 'pull_request' &&
|
|
!inputs.target_stage &&
|
|
inputs.test_parallel_dispatch != true &&
|
|
(needs.check-changes.outputs.main_package == 'true' || needs.check-changes.outputs.sgl_kernel == 'true') &&
|
|
(needs.wait-for-stage-a.result == 'success' || needs.wait-for-stage-a.result == 'skipped') &&
|
|
(needs.call-gate.result == 'success' || needs.call-gate.result == 'skipped')
|
|
runs-on: ubuntu-latest
|
|
outputs:
|
|
stage_b_result: ${{ steps.wait.outputs.result }}
|
|
steps:
|
|
- uses: actions/checkout@v4
|
|
|
|
- uses: ./.github/actions/check-maintenance
|
|
|
|
- uses: ./.github/actions/wait-for-jobs
|
|
id: wait
|
|
with:
|
|
stage-name: stage-b
|
|
jobs: |
|
|
[
|
|
{"prefix": "stage-b-test-1-gpu-small", "expected_count": 8},
|
|
{"prefix": "stage-b-test-1-gpu-large", "expected_count": 14},
|
|
{"prefix": "stage-b-test-2-gpu-large", "expected_count": 4},
|
|
{"prefix": "stage-b-test-4-gpu-b200", "expected_count": 1}
|
|
]
|
|
max-wait-minutes: '480'
|
|
|
|
# =============================================== PR Gate ====================================================
|
|
call-gate:
|
|
needs: check-changes
|
|
# Skip for scheduled runs (they run all tests) and when target_stage is specified
|
|
if: |
|
|
github.event_name != 'schedule' &&
|
|
inputs.test_parallel_dispatch != true &&
|
|
!inputs.target_stage &&
|
|
(
|
|
needs.check-changes.outputs.main_package == 'true' ||
|
|
needs.check-changes.outputs.sgl_kernel == 'true' ||
|
|
needs.check-changes.outputs.jit_kernel == 'true' ||
|
|
needs.check-changes.outputs.multimodal_gen == 'true'
|
|
)
|
|
uses: ./.github/workflows/pr-gate.yml
|
|
secrets: inherit
|
|
|
|
# =============================================== sgl-kernel ====================================================
|
|
|
|
sgl-kernel-build-wheels:
|
|
needs: [check-changes, call-gate]
|
|
# Skip for scheduled runs (they run stages independently). Runs in target_stage mode only when
|
|
# include_wheel_build is true (i.e. /rerun-stage on a PR with sgl-kernel changes), so the
|
|
# target stage can download the freshly-built wheel.
|
|
#
|
|
# `always()` lets us run when call-gate is skipped (which it always is in target_stage mode by
|
|
# design). The explicit needs.<x>.result checks preserve old gating for the normal PR path.
|
|
if: |
|
|
always() &&
|
|
github.event_name != 'schedule' &&
|
|
inputs.test_parallel_dispatch != true &&
|
|
needs.check-changes.result == 'success' &&
|
|
needs.check-changes.outputs.sgl_kernel == 'true' &&
|
|
(
|
|
(!inputs.target_stage && needs.call-gate.result == 'success') ||
|
|
(inputs.target_stage && inputs.include_wheel_build)
|
|
)
|
|
runs-on: x64-kernel-build-node
|
|
timeout-minutes: 240
|
|
strategy:
|
|
matrix:
|
|
include:
|
|
- python-version: "3.10"
|
|
cuda-version: "13.0"
|
|
- python-version: "3.10"
|
|
cuda-version: "12.9"
|
|
name: Build Wheel
|
|
steps:
|
|
- name: Cleanup
|
|
run: |
|
|
sudo rm -rf $GITHUB_WORKSPACE/* || true
|
|
|
|
- uses: actions/checkout@v4
|
|
with:
|
|
submodules: "recursive"
|
|
ref: ${{ inputs.pr_head_sha || inputs.git_ref || github.sha }}
|
|
|
|
- uses: ./.github/actions/check-maintenance
|
|
|
|
- name: Set up Python ${{ matrix.python-version }}
|
|
uses: actions/setup-python@v5
|
|
with:
|
|
python-version: ${{ matrix.python-version }}
|
|
|
|
- name: Free Docker disk space
|
|
run: |
|
|
set -x
|
|
# build.sh retags sgl-kernel-deps:cuda${CUDA_VERSION}-${PY_TAG}-${ARCH}
|
|
# on every run, leaving the previous image as a dangling <none>:<none>
|
|
# entry (~16-23 GB each). Prune them before building so the runner
|
|
# doesn't fill up. The local buildx cache at ~/.cache/sgl-kernel/buildx
|
|
# and the tagged sgl-kernel-deps image are not affected.
|
|
# `until=12h` avoids racing with a sibling matrix cell (cuda 12.9 vs
|
|
# 13.0) that may have just orphaned an image seconds ago.
|
|
docker image prune -f --filter "until=12h"
|
|
# Drop orphaned buildx builder volumes from past `docker buildx create`
|
|
# invocations. The active `sgl-kernel-builder` volume is held open and
|
|
# would fail to remove anyway, but skip it explicitly for clarity.
|
|
for v in $(docker volume ls -q | grep '^buildx_buildkit_' | grep -v '^buildx_buildkit_sgl-kernel-builder' || true); do
|
|
echo "Removing orphan buildx volume: $v"
|
|
docker volume rm "$v" || true
|
|
done
|
|
df -h /
|
|
|
|
- name: Build wheel for Python ${{ matrix.python-version }} and CUDA ${{ matrix.cuda-version }}
|
|
run: |
|
|
cd sgl-kernel
|
|
./build.sh "${{ matrix.python-version }}" "${{ matrix.cuda-version }}"
|
|
env:
|
|
USE_CCACHE: 1
|
|
|
|
- name: Verify wheel artifacts
|
|
run: |
|
|
ls -alh sgl-kernel/dist
|
|
ls -alh sgl-kernel/dist/*.whl
|
|
|
|
- name: Upload artifacts
|
|
uses: actions/upload-artifact@v4
|
|
with:
|
|
name: wheel-python${{ matrix.python-version }}-cuda${{ matrix.cuda-version }}
|
|
path: sgl-kernel/dist/*
|
|
if-no-files-found: error
|
|
|
|
sgl-kernel-build-wheels-arm:
|
|
needs: [check-changes, call-gate]
|
|
# Skip for scheduled runs (they run stages independently). Runs in target_stage mode only when
|
|
# include_wheel_build is true (i.e. /rerun-stage on a PR with sgl-kernel changes).
|
|
#
|
|
# See sgl-kernel-build-wheels above for the always() + result-check rationale.
|
|
if: |
|
|
always() &&
|
|
github.event_name != 'schedule' &&
|
|
inputs.test_parallel_dispatch != true &&
|
|
needs.check-changes.result == 'success' &&
|
|
needs.check-changes.outputs.sgl_kernel == 'true' &&
|
|
(
|
|
(!inputs.target_stage && needs.call-gate.result == 'success') ||
|
|
(inputs.target_stage && inputs.include_wheel_build)
|
|
)
|
|
runs-on: arm-kernel-build-node
|
|
timeout-minutes: 240
|
|
strategy:
|
|
matrix:
|
|
include:
|
|
- python-version: "3.10"
|
|
cuda-version: "13.0"
|
|
- python-version: "3.10"
|
|
cuda-version: "12.9"
|
|
name: Build Wheel Arm
|
|
steps:
|
|
- name: Cleanup
|
|
run: |
|
|
if [ -d "$GITHUB_WORKSPACE" ]; then
|
|
sudo rm -rf "$GITHUB_WORKSPACE"/* || true
|
|
else
|
|
echo "$GITHUB_WORKSPACE does not exist, nothing to clean"
|
|
fi
|
|
|
|
- uses: actions/checkout@v4
|
|
with:
|
|
submodules: "recursive"
|
|
ref: ${{ inputs.pr_head_sha || inputs.git_ref || github.sha }}
|
|
|
|
- uses: ./.github/actions/check-maintenance
|
|
|
|
- name: Set up Python ${{ matrix.python-version }}
|
|
uses: actions/setup-python@v5
|
|
with:
|
|
python-version: ${{ matrix.python-version }}
|
|
|
|
- name: Free Docker disk space
|
|
run: |
|
|
set -x
|
|
# See sgl-kernel-build-wheels above for the rationale.
|
|
docker image prune -f --filter "until=12h"
|
|
for v in $(docker volume ls -q | grep '^buildx_buildkit_' | grep -v '^buildx_buildkit_sgl-kernel-builder' || true); do
|
|
echo "Removing orphan buildx volume: $v"
|
|
docker volume rm "$v" || true
|
|
done
|
|
df -h /
|
|
|
|
- name: Build wheel for Python ${{ matrix.python-version }} and CUDA ${{ matrix.cuda-version }}
|
|
run: |
|
|
cd sgl-kernel
|
|
./build.sh "${{ matrix.python-version }}" "${{ matrix.cuda-version }}"
|
|
env:
|
|
USE_CCACHE: 1
|
|
|
|
- name: Verify wheel artifacts
|
|
run: |
|
|
ls -alh sgl-kernel/dist
|
|
ls -alh sgl-kernel/dist/*.whl
|
|
|
|
- name: Upload artifacts
|
|
uses: actions/upload-artifact@v4
|
|
with:
|
|
name: wheel-python${{ matrix.python-version }}-cuda${{ matrix.cuda-version }}-aarch64
|
|
path: sgl-kernel/dist/*
|
|
if-no-files-found: error
|
|
|
|
call-sgl-kernel-tests:
|
|
needs: [check-changes, call-gate, sgl-kernel-build-wheels]
|
|
if: |
|
|
github.event_name != 'schedule' &&
|
|
inputs.test_parallel_dispatch != true &&
|
|
!inputs.target_stage &&
|
|
needs.check-changes.outputs.sgl_kernel == 'true'
|
|
uses: ./.github/workflows/pr-test-sgl-kernel.yml
|
|
with:
|
|
sgl_kernel: ${{ needs.check-changes.outputs.sgl_kernel }}
|
|
b200_runner: ${{ needs.check-changes.outputs.b200_runner }}
|
|
pr_head_sha: ${{ inputs.pr_head_sha || '' }}
|
|
git_ref: ${{ inputs.git_ref || '' }}
|
|
skip_stage_health_check: ${{ inputs.skip_stage_health_check == true }}
|
|
secrets: inherit
|
|
|
|
# =============================================== jit-kernel ====================================================
|
|
|
|
call-jit-kernel-tests:
|
|
needs: [check-changes, call-gate, sgl-kernel-build-wheels]
|
|
if: |
|
|
always() &&
|
|
!failure() && !cancelled() &&
|
|
github.event_name != 'schedule' &&
|
|
inputs.test_parallel_dispatch != true &&
|
|
!inputs.target_stage &&
|
|
needs.check-changes.outputs.jit_kernel == 'true'
|
|
uses: ./.github/workflows/pr-test-jit-kernel.yml
|
|
with:
|
|
jit_kernel: ${{ needs.check-changes.outputs.jit_kernel }}
|
|
sgl_kernel: ${{ needs.check-changes.outputs.sgl_kernel }}
|
|
b200_runner: ${{ needs.check-changes.outputs.b200_runner }}
|
|
pr_head_sha: ${{ inputs.pr_head_sha || '' }}
|
|
git_ref: ${{ inputs.git_ref || '' }}
|
|
target_stage: ${{ inputs.target_stage || '' }}
|
|
test_parallel_dispatch: ${{ inputs.test_parallel_dispatch == true && 'true' || 'false' }}
|
|
skip_stage_health_check: ${{ inputs.skip_stage_health_check == true }}
|
|
secrets: inherit
|
|
|
|
# =============================================== primary ====================================================
|
|
|
|
# Runs on 5090 (32GB, SM120)
|
|
stage-a-test-1-gpu-small:
|
|
needs: [check-changes, call-gate, sgl-kernel-build-wheels]
|
|
if: |
|
|
always() &&
|
|
(
|
|
(inputs.target_stage == 'stage-a-test-1-gpu-small') ||
|
|
(
|
|
!inputs.target_stage &&
|
|
((github.event_name == 'schedule' || inputs.test_parallel_dispatch == true) || (!failure() && !cancelled())) &&
|
|
((needs.check-changes.outputs.main_package == 'true') || (needs.check-changes.outputs.sgl_kernel == 'true'))
|
|
)
|
|
)
|
|
runs-on: 1-gpu-5090
|
|
timeout-minutes: 240
|
|
steps:
|
|
- name: Checkout code
|
|
uses: actions/checkout@v4
|
|
with:
|
|
ref: ${{ inputs.pr_head_sha || inputs.git_ref || github.sha }}
|
|
|
|
- uses: ./.github/actions/check-stage-health
|
|
|
|
- uses: ./.github/actions/check-maintenance
|
|
|
|
- name: Download artifacts
|
|
if: needs.check-changes.outputs.sgl_kernel == 'true'
|
|
uses: actions/download-artifact@v4
|
|
with:
|
|
path: sgl-kernel/dist/
|
|
merge-multiple: true
|
|
pattern: wheel-python3.10-cuda*
|
|
|
|
- name: Install dependencies
|
|
timeout-minutes: 20
|
|
run: |
|
|
CUSTOM_BUILD_SGL_KERNEL=${{needs.check-changes.outputs.sgl_kernel}} bash scripts/ci/cuda/ci_install_dependency.sh
|
|
|
|
- name: Run test
|
|
timeout-minutes: 10
|
|
env:
|
|
CONTINUE_ON_ERROR_FLAG: ${{ needs.check-changes.outputs.continue_on_error == 'true' && '--continue-on-error' || '' }}
|
|
run: |
|
|
cd test/
|
|
python3 run_suite.py --hw cuda --suite stage-a-test-1-gpu-small $CONTINUE_ON_ERROR_FLAG
|
|
|
|
- uses: ./.github/actions/upload-cuda-coredumps
|
|
if: failure()
|
|
|
|
- name: Cleanup venv
|
|
if: always()
|
|
run: bash scripts/ci/cuda/ci_cleanup_venv.sh
|
|
|
|
stage-a-test-cpu:
|
|
needs: [check-changes, call-gate]
|
|
if: |
|
|
always() &&
|
|
(
|
|
(inputs.target_stage == 'stage-a-test-cpu') ||
|
|
(
|
|
!inputs.target_stage &&
|
|
((github.event_name == 'schedule' || inputs.test_parallel_dispatch == true) || (!failure() && !cancelled())) &&
|
|
(needs.check-changes.outputs.main_package == 'true')
|
|
)
|
|
)
|
|
runs-on: ubuntu-latest
|
|
timeout-minutes: 240
|
|
strategy:
|
|
fail-fast: false
|
|
matrix:
|
|
partition: [0, 1, 2, 3]
|
|
steps:
|
|
- name: Free disk space
|
|
run: |
|
|
sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc
|
|
df -h
|
|
|
|
- name: Checkout code
|
|
uses: actions/checkout@v4
|
|
with:
|
|
ref: ${{ inputs.pr_head_sha || inputs.git_ref || github.sha }}
|
|
|
|
- uses: ./.github/actions/check-stage-health
|
|
|
|
- uses: ./.github/actions/check-maintenance
|
|
|
|
- name: Set up Python
|
|
uses: actions/setup-python@v5
|
|
with:
|
|
python-version: '3.10'
|
|
|
|
- name: Install uv
|
|
uses: astral-sh/setup-uv@v5
|
|
|
|
# Needed by setuptools-rust to build the bundled native gRPC extension
|
|
# (rust/sglang-grpc) when installing the main `sglang` wheel from source.
|
|
- name: Install protoc + Rust toolchain
|
|
timeout-minutes: 10
|
|
run: bash scripts/ci/utils/install_rust_protoc.sh
|
|
|
|
- name: Rust cache (sglang-grpc)
|
|
uses: Swatinem/rust-cache@v2
|
|
with:
|
|
workspaces: rust/sglang-grpc
|
|
shared-key: "sglang-grpc-cpu"
|
|
save-if: ${{ matrix.partition == 0 }}
|
|
|
|
# uv pip targets a venv by default; setup-python has no venv — install into that interpreter (see UV_SYSTEM_PYTHON in https://docs.astral.sh/uv/guides/integration/github/)
|
|
- name: Install dependencies
|
|
timeout-minutes: 20
|
|
env:
|
|
UV_SYSTEM_PYTHON: "1"
|
|
run: |
|
|
uv pip install -e "python[dev]" --index-strategy unsafe-best-match --prerelease allow
|
|
|
|
- name: Run test
|
|
timeout-minutes: 10
|
|
env:
|
|
CONTINUE_ON_ERROR_FLAG: ${{ needs.check-changes.outputs.continue_on_error == 'true' && '--continue-on-error' || '' }}
|
|
run: |
|
|
cd test/
|
|
python3 run_suite.py --hw cpu --suite stage-a-test-cpu --auto-partition-id ${{ matrix.partition }} --auto-partition-size 4 $CONTINUE_ON_ERROR_FLAG
|
|
|
|
# Runs on 5090 (32GB, SM120)
|
|
stage-b-test-1-gpu-small:
|
|
needs: [check-changes, call-gate, wait-for-stage-a, sgl-kernel-build-wheels]
|
|
if: |
|
|
always() &&
|
|
(
|
|
(inputs.target_stage == 'stage-b-test-1-gpu-small') ||
|
|
(
|
|
!inputs.target_stage &&
|
|
((github.event_name == 'schedule' || inputs.test_parallel_dispatch == true) || (!failure() && !cancelled())) &&
|
|
((needs.check-changes.outputs.main_package == 'true') || (needs.check-changes.outputs.sgl_kernel == 'true'))
|
|
)
|
|
)
|
|
runs-on: 1-gpu-5090
|
|
timeout-minutes: 240
|
|
strategy:
|
|
fail-fast: false
|
|
max-parallel: ${{ fromJson(needs.check-changes.outputs.max_parallel_small) }}
|
|
matrix:
|
|
partition: [0, 1, 2, 3, 4, 5, 6, 7]
|
|
steps:
|
|
- name: Checkout code
|
|
uses: actions/checkout@v4
|
|
with:
|
|
ref: ${{ inputs.pr_head_sha || inputs.git_ref || github.sha }}
|
|
|
|
- uses: ./.github/actions/check-stage-health
|
|
|
|
- uses: ./.github/actions/check-maintenance
|
|
|
|
- name: Download artifacts
|
|
if: needs.check-changes.outputs.sgl_kernel == 'true'
|
|
uses: actions/download-artifact@v4
|
|
with:
|
|
path: sgl-kernel/dist/
|
|
merge-multiple: true
|
|
pattern: wheel-python3.10-cuda*
|
|
|
|
- name: Install dependencies
|
|
timeout-minutes: 20
|
|
run: |
|
|
CUSTOM_BUILD_SGL_KERNEL=${{needs.check-changes.outputs.sgl_kernel}} bash scripts/ci/cuda/ci_install_dependency.sh
|
|
|
|
- name: Run test
|
|
timeout-minutes: 30
|
|
env:
|
|
CONTINUE_ON_ERROR_FLAG: ${{ needs.check-changes.outputs.continue_on_error == 'true' && '--continue-on-error' || '' }}
|
|
run: |
|
|
cd test/
|
|
python3 run_suite.py --hw cuda --suite stage-b-test-1-gpu-small --auto-partition-id ${{ matrix.partition }} --auto-partition-size 8 $CONTINUE_ON_ERROR_FLAG
|
|
|
|
- uses: ./.github/actions/upload-cuda-coredumps
|
|
if: failure()
|
|
with:
|
|
artifact-suffix: ${{ matrix.partition }}
|
|
|
|
- name: Cleanup venv
|
|
if: always()
|
|
run: bash scripts/ci/cuda/ci_cleanup_venv.sh
|
|
|
|
# Runs on H100 (80GB, SM90) - tests that don't pass on 5090 (FA3, FP8, high VRAM, etc.)
|
|
stage-b-test-1-gpu-large:
|
|
needs: [check-changes, call-gate, wait-for-stage-a, sgl-kernel-build-wheels]
|
|
if: |
|
|
always() &&
|
|
(
|
|
(inputs.target_stage == 'stage-b-test-1-gpu-large') ||
|
|
(
|
|
!inputs.target_stage &&
|
|
((github.event_name == 'schedule' || inputs.test_parallel_dispatch == true) || (!failure() && !cancelled())) &&
|
|
((needs.check-changes.outputs.main_package == 'true') || (needs.check-changes.outputs.sgl_kernel == 'true'))
|
|
)
|
|
)
|
|
runs-on: 1-gpu-h100
|
|
timeout-minutes: 240
|
|
strategy:
|
|
fail-fast: false
|
|
max-parallel: ${{ fromJson(needs.check-changes.outputs.max_parallel) }}
|
|
matrix:
|
|
partition: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13]
|
|
steps:
|
|
- name: Checkout code
|
|
uses: actions/checkout@v4
|
|
with:
|
|
ref: ${{ inputs.pr_head_sha || inputs.git_ref || github.sha }}
|
|
|
|
- uses: ./.github/actions/check-stage-health
|
|
|
|
- uses: ./.github/actions/check-maintenance
|
|
|
|
- name: Download artifacts
|
|
if: needs.check-changes.outputs.sgl_kernel == 'true'
|
|
uses: actions/download-artifact@v4
|
|
with:
|
|
path: sgl-kernel/dist/
|
|
merge-multiple: true
|
|
pattern: wheel-python3.10-cuda*
|
|
|
|
- name: Install dependencies
|
|
timeout-minutes: 20
|
|
run: |
|
|
CUSTOM_BUILD_SGL_KERNEL=${{needs.check-changes.outputs.sgl_kernel}} bash scripts/ci/cuda/ci_install_dependency.sh
|
|
|
|
- name: Run test
|
|
timeout-minutes: 30
|
|
env:
|
|
CONTINUE_ON_ERROR_FLAG: ${{ needs.check-changes.outputs.continue_on_error == 'true' && '--continue-on-error' || '' }}
|
|
run: |
|
|
cd test/
|
|
python3 run_suite.py --hw cuda --suite stage-b-test-1-gpu-large --auto-partition-id ${{ matrix.partition }} --auto-partition-size 14 --timeout-per-file 1800 $CONTINUE_ON_ERROR_FLAG
|
|
|
|
- uses: ./.github/actions/upload-cuda-coredumps
|
|
if: failure()
|
|
with:
|
|
artifact-suffix: ${{ matrix.partition }}
|
|
|
|
- name: Cleanup venv
|
|
if: always()
|
|
run: bash scripts/ci/cuda/ci_cleanup_venv.sh
|
|
|
|
stage-b-test-2-gpu-large:
|
|
needs: [check-changes, call-gate, wait-for-stage-a, sgl-kernel-build-wheels]
|
|
if: |
|
|
always() &&
|
|
(
|
|
(inputs.target_stage == 'stage-b-test-2-gpu-large') ||
|
|
(
|
|
!inputs.target_stage &&
|
|
((github.event_name == 'schedule' || inputs.test_parallel_dispatch == true) || (!failure() && !cancelled())) &&
|
|
((needs.check-changes.outputs.main_package == 'true') || (needs.check-changes.outputs.sgl_kernel == 'true'))
|
|
)
|
|
)
|
|
runs-on: 2-gpu-h100
|
|
timeout-minutes: 240
|
|
strategy:
|
|
fail-fast: false
|
|
max-parallel: ${{ fromJson(needs.check-changes.outputs.max_parallel_2gpu) }}
|
|
matrix:
|
|
partition: [0, 1, 2, 3]
|
|
steps:
|
|
- name: Checkout code
|
|
uses: actions/checkout@v4
|
|
with:
|
|
ref: ${{ inputs.pr_head_sha || inputs.git_ref || github.sha }}
|
|
|
|
- uses: ./.github/actions/check-stage-health
|
|
|
|
- uses: ./.github/actions/check-maintenance
|
|
|
|
- name: Download artifacts
|
|
if: needs.check-changes.outputs.sgl_kernel == 'true'
|
|
uses: actions/download-artifact@v4
|
|
with:
|
|
path: sgl-kernel/dist/
|
|
merge-multiple: true
|
|
pattern: wheel-python3.10-cuda*
|
|
|
|
- name: Install dependencies
|
|
timeout-minutes: 20
|
|
run: |
|
|
CUSTOM_BUILD_SGL_KERNEL=${{needs.check-changes.outputs.sgl_kernel}} bash scripts/ci/cuda/ci_install_dependency.sh
|
|
|
|
- name: Run test
|
|
timeout-minutes: 30
|
|
env:
|
|
CONTINUE_ON_ERROR_FLAG: ${{ needs.check-changes.outputs.continue_on_error == 'true' && '--continue-on-error' || '' }}
|
|
run: |
|
|
cd test/
|
|
python3 run_suite.py --hw cuda --suite stage-b-test-2-gpu-large --auto-partition-id ${{ matrix.partition }} --auto-partition-size 4 $CONTINUE_ON_ERROR_FLAG
|
|
|
|
- uses: ./.github/actions/upload-cuda-coredumps
|
|
if: failure()
|
|
with:
|
|
artifact-suffix: ${{ matrix.partition }}
|
|
|
|
- name: Cleanup venv
|
|
if: always()
|
|
run: bash scripts/ci/cuda/ci_cleanup_venv.sh
|
|
|
|
stage-b-test-4-gpu-b200:
|
|
needs: [check-changes, call-gate, wait-for-stage-a, sgl-kernel-build-wheels]
|
|
if: |
|
|
always() &&
|
|
(
|
|
(inputs.target_stage == 'stage-b-test-4-gpu-b200') ||
|
|
(
|
|
!inputs.target_stage &&
|
|
((github.event_name == 'schedule' || inputs.test_parallel_dispatch == true) || (!failure() && !cancelled())) &&
|
|
((needs.check-changes.outputs.main_package == 'true') || (needs.check-changes.outputs.sgl_kernel == 'true'))
|
|
)
|
|
)
|
|
runs-on: ${{ needs.check-changes.outputs.b200_runner }}
|
|
timeout-minutes: 240
|
|
strategy:
|
|
fail-fast: false
|
|
|
|
steps:
|
|
- name: Checkout code
|
|
uses: actions/checkout@v4
|
|
with:
|
|
ref: ${{ inputs.pr_head_sha || inputs.git_ref || github.sha }}
|
|
|
|
- uses: ./.github/actions/check-stage-health
|
|
|
|
- uses: ./.github/actions/check-maintenance
|
|
|
|
- name: Download artifacts
|
|
if: needs.check-changes.outputs.sgl_kernel == 'true'
|
|
uses: actions/download-artifact@v6
|
|
with:
|
|
path: sgl-kernel/dist/
|
|
merge-multiple: true
|
|
pattern: wheel-python3.10-cuda*
|
|
|
|
- name: Install dependencies
|
|
timeout-minutes: 20
|
|
run: |
|
|
CUSTOM_BUILD_SGL_KERNEL=${{needs.check-changes.outputs.sgl_kernel}} bash scripts/ci/cuda/ci_install_dependency.sh
|
|
|
|
- name: Run test
|
|
timeout-minutes: 40
|
|
env:
|
|
CONTINUE_ON_ERROR_FLAG: ${{ needs.check-changes.outputs.continue_on_error == 'true' && '--continue-on-error' || '' }}
|
|
run: |
|
|
cd test
|
|
python3 run_suite.py --hw cuda --suite stage-b-test-4-gpu-b200 $CONTINUE_ON_ERROR_FLAG
|
|
|
|
- name: Run FA4 jit_kernel tests (SM100+)
|
|
timeout-minutes: 10
|
|
run: |
|
|
python3 -m pytest -q python/sglang/jit_kernel/tests/test_flash_attention_4.py
|
|
|
|
- uses: ./.github/actions/upload-cuda-coredumps
|
|
if: failure()
|
|
|
|
- name: Cleanup venv
|
|
if: always()
|
|
run: bash scripts/ci/cuda/ci_cleanup_venv.sh
|
|
|
|
call-multimodal-gen-tests:
|
|
needs: [check-changes, call-gate, sgl-kernel-build-wheels]
|
|
if: |
|
|
always() &&
|
|
!cancelled() &&
|
|
(
|
|
inputs.target_stage == 'multimodal-gen-test-1-gpu' ||
|
|
inputs.target_stage == 'multimodal-gen-test-2-gpu' ||
|
|
inputs.target_stage == 'multimodal-gen-component-accuracy' ||
|
|
inputs.target_stage == 'multimodal-gen-component-accuracy-1-gpu' ||
|
|
inputs.target_stage == 'multimodal-gen-component-accuracy-2-gpu' ||
|
|
inputs.target_stage == 'multimodal-gen-test-1-b200' ||
|
|
inputs.target_stage == 'multimodal-gen-unit-test' ||
|
|
(
|
|
!inputs.target_stage &&
|
|
((github.event_name == 'schedule' || inputs.test_parallel_dispatch == true) || (!failure() && !cancelled())) &&
|
|
needs.check-changes.outputs.multimodal_gen == 'true'
|
|
)
|
|
)
|
|
uses: ./.github/workflows/pr-test-multimodal-gen.yml
|
|
with:
|
|
multimodal_gen: ${{ needs.check-changes.outputs.multimodal_gen }}
|
|
sgl_kernel: ${{ needs.check-changes.outputs.sgl_kernel }}
|
|
b200_runner: ${{ needs.check-changes.outputs.b200_runner }}
|
|
continue_on_error: ${{ needs.check-changes.outputs.continue_on_error }}
|
|
pr_head_sha: ${{ inputs.pr_head_sha || '' }}
|
|
git_ref: ${{ inputs.git_ref || '' }}
|
|
target_stage: ${{ inputs.target_stage || '' }}
|
|
test_parallel_dispatch: ${{ inputs.test_parallel_dispatch == true && 'true' || 'false' }}
|
|
caller_needs_failure: ${{ (needs.call-gate.result == 'failure' || needs.sgl-kernel-build-wheels.result == 'failure' || needs.check-changes.result == 'failure') && 'true' || 'false' }}
|
|
skip_stage_health_check: ${{ inputs.skip_stage_health_check == true && 'true' || 'false' }}
|
|
secrets: inherit
|
|
|
|
stage-c-test-4-gpu-h100:
|
|
needs: [check-changes, call-gate, wait-for-stage-b, sgl-kernel-build-wheels]
|
|
if: |
|
|
always() &&
|
|
(
|
|
(inputs.target_stage == 'stage-c-test-4-gpu-h100') ||
|
|
(
|
|
!inputs.target_stage &&
|
|
((github.event_name == 'schedule' || inputs.test_parallel_dispatch == true) || (!failure() && !cancelled())) &&
|
|
((needs.check-changes.outputs.main_package == 'true') || (needs.check-changes.outputs.sgl_kernel == 'true'))
|
|
)
|
|
)
|
|
runs-on: 4-gpu-h100
|
|
timeout-minutes: 240
|
|
strategy:
|
|
fail-fast: false
|
|
matrix:
|
|
part: [0, 1, 2]
|
|
steps:
|
|
- name: Checkout code
|
|
uses: actions/checkout@v4
|
|
with:
|
|
ref: ${{ inputs.pr_head_sha || inputs.git_ref || github.sha }}
|
|
|
|
- uses: ./.github/actions/check-stage-health
|
|
|
|
- uses: ./.github/actions/check-maintenance
|
|
|
|
- name: Download artifacts
|
|
if: needs.check-changes.outputs.sgl_kernel == 'true'
|
|
uses: actions/download-artifact@v4
|
|
with:
|
|
path: sgl-kernel/dist/
|
|
merge-multiple: true
|
|
pattern: wheel-python3.10-cuda*
|
|
|
|
- name: Install dependencies
|
|
timeout-minutes: 20
|
|
run: |
|
|
CUSTOM_BUILD_SGL_KERNEL=${{needs.check-changes.outputs.sgl_kernel}} bash scripts/ci/cuda/ci_install_dependency.sh
|
|
|
|
- name: Run test
|
|
timeout-minutes: 30
|
|
env:
|
|
CONTINUE_ON_ERROR_FLAG: ${{ needs.check-changes.outputs.continue_on_error == 'true' && '--continue-on-error' || '' }}
|
|
run: |
|
|
cd test
|
|
python3 run_suite.py --hw cuda --suite stage-c-test-4-gpu-h100 --auto-partition-id ${{ matrix.part }} --auto-partition-size 3 $CONTINUE_ON_ERROR_FLAG
|
|
|
|
- uses: ./.github/actions/upload-cuda-coredumps
|
|
if: failure()
|
|
with:
|
|
artifact-suffix: ${{ matrix.part }}
|
|
|
|
- name: Cleanup venv
|
|
if: always()
|
|
run: bash scripts/ci/cuda/ci_cleanup_venv.sh
|
|
|
|
stage-c-test-8-gpu-h200:
|
|
needs: [check-changes, call-gate, wait-for-stage-b, sgl-kernel-build-wheels]
|
|
if: |
|
|
always() &&
|
|
(
|
|
(inputs.target_stage == 'stage-c-test-8-gpu-h200') ||
|
|
(
|
|
!inputs.target_stage &&
|
|
((github.event_name == 'schedule' || inputs.test_parallel_dispatch == true) || (!failure() && !cancelled())) &&
|
|
((needs.check-changes.outputs.main_package == 'true') || (needs.check-changes.outputs.sgl_kernel == 'true'))
|
|
)
|
|
)
|
|
runs-on: 8-gpu-h200
|
|
timeout-minutes: 240
|
|
strategy:
|
|
fail-fast: false
|
|
matrix:
|
|
part: [0, 1, 2, 3]
|
|
steps:
|
|
- name: Checkout code
|
|
uses: actions/checkout@v4
|
|
with:
|
|
ref: ${{ inputs.pr_head_sha || inputs.git_ref || github.sha }}
|
|
|
|
- uses: ./.github/actions/check-stage-health
|
|
|
|
- uses: ./.github/actions/check-maintenance
|
|
|
|
- name: Download artifacts
|
|
if: needs.check-changes.outputs.sgl_kernel == 'true'
|
|
uses: actions/download-artifact@v4
|
|
with:
|
|
path: sgl-kernel/dist/
|
|
merge-multiple: true
|
|
pattern: wheel-python3.10-cuda*
|
|
|
|
- name: Install dependencies
|
|
timeout-minutes: 20
|
|
run: |
|
|
CUSTOM_BUILD_SGL_KERNEL=${{needs.check-changes.outputs.sgl_kernel}} bash scripts/ci/cuda/ci_install_dependency.sh
|
|
|
|
- name: Warmup DeepGEMM JIT Compilation
|
|
timeout-minutes: 25
|
|
run: |
|
|
# Activate venv if available (GITHUB_ENV may have failed to propagate)
|
|
[ -f "${SGLANG_CI_VENV_PATH:-/dev/null}/bin/activate" ] && source "${SGLANG_CI_VENV_PATH}/bin/activate"
|
|
[ -f "${SGLANG_CI_VENV_PATH:-/dev/null}/env.sh" ] && source "${SGLANG_CI_VENV_PATH}/env.sh"
|
|
python3 scripts/ci/cuda/warmup_deep_gemm.py \
|
|
deepseek-ai/DeepSeek-V3-0324:8 \
|
|
deepseek-ai/DeepSeek-V3.2-Exp:8
|
|
|
|
- name: Warmup Server CUDA Graphs
|
|
timeout-minutes: 25
|
|
run: |
|
|
[ -f "${SGLANG_CI_VENV_PATH:-/dev/null}/bin/activate" ] && source "${SGLANG_CI_VENV_PATH}/bin/activate"
|
|
[ -f "${SGLANG_CI_VENV_PATH:-/dev/null}/env.sh" ] && source "${SGLANG_CI_VENV_PATH}/env.sh"
|
|
python3 scripts/ci/cuda/warmup_server.py \
|
|
deepseek-ai/DeepSeek-V3-0324:8 \
|
|
inclusionAI/Ring-2.5-1T:8
|
|
|
|
- name: Run test
|
|
timeout-minutes: 30
|
|
env:
|
|
CONTINUE_ON_ERROR_FLAG: ${{ needs.check-changes.outputs.continue_on_error == 'true' && '--continue-on-error' || '' }}
|
|
run: |
|
|
cd test
|
|
python3 run_suite.py --hw cuda --suite stage-c-test-8-gpu-h200 --auto-partition-id ${{ matrix.part }} --auto-partition-size 4 $CONTINUE_ON_ERROR_FLAG
|
|
|
|
- uses: ./.github/actions/upload-cuda-coredumps
|
|
if: failure()
|
|
with:
|
|
artifact-suffix: ${{ matrix.part }}
|
|
|
|
- name: Cleanup venv
|
|
if: always()
|
|
run: bash scripts/ci/cuda/ci_cleanup_venv.sh
|
|
|
|
stage-c-test-8-gpu-h20:
|
|
needs: [check-changes, call-gate, wait-for-stage-b, sgl-kernel-build-wheels]
|
|
if: |
|
|
always() &&
|
|
(
|
|
(inputs.target_stage == 'stage-c-test-8-gpu-h20') ||
|
|
(
|
|
!inputs.target_stage &&
|
|
((github.event_name == 'schedule' || inputs.test_parallel_dispatch == true) || (!failure() && !cancelled())) &&
|
|
((needs.check-changes.outputs.main_package == 'true') || (needs.check-changes.outputs.sgl_kernel == 'true'))
|
|
)
|
|
)
|
|
runs-on: 8-gpu-h20
|
|
timeout-minutes: 240
|
|
env:
|
|
SGLANG_CI_RDMA_ALL_DEVICES: "mlx5_1,mlx5_2,mlx5_3,mlx5_4"
|
|
steps:
|
|
- name: Checkout code
|
|
uses: actions/checkout@v4
|
|
with:
|
|
ref: ${{ inputs.pr_head_sha || inputs.git_ref || github.sha }}
|
|
|
|
- uses: ./.github/actions/check-stage-health
|
|
|
|
- uses: ./.github/actions/check-maintenance
|
|
|
|
- name: Download artifacts
|
|
if: needs.check-changes.outputs.sgl_kernel == 'true'
|
|
uses: actions/download-artifact@v4
|
|
with:
|
|
path: sgl-kernel/dist/
|
|
merge-multiple: true
|
|
pattern: wheel-python3.10-cuda*
|
|
|
|
- name: Install dependencies
|
|
timeout-minutes: 20
|
|
run: |
|
|
CUSTOM_BUILD_SGL_KERNEL=${{needs.check-changes.outputs.sgl_kernel}} bash scripts/ci/cuda/ci_install_deepep.sh
|
|
|
|
- name: Run test
|
|
timeout-minutes: 30
|
|
env:
|
|
CONTINUE_ON_ERROR_FLAG: ${{ needs.check-changes.outputs.continue_on_error == 'true' && '--continue-on-error' || '' }}
|
|
run: |
|
|
cd test
|
|
python3 run_suite.py --hw cuda --suite stage-c-test-8-gpu-h20 $CONTINUE_ON_ERROR_FLAG
|
|
|
|
- uses: ./.github/actions/upload-cuda-coredumps
|
|
if: failure()
|
|
|
|
- name: Cleanup venv
|
|
if: always()
|
|
run: bash scripts/ci/cuda/ci_cleanup_venv.sh
|
|
|
|
stage-c-test-deepep-4-gpu-h100:
|
|
needs: [check-changes, call-gate, wait-for-stage-b, sgl-kernel-build-wheels]
|
|
if: |
|
|
always() &&
|
|
(
|
|
(inputs.target_stage == 'stage-c-test-deepep-4-gpu-h100') ||
|
|
(
|
|
!inputs.target_stage &&
|
|
((github.event_name == 'schedule' || inputs.test_parallel_dispatch == true) || (!failure() && !cancelled())) &&
|
|
((needs.check-changes.outputs.main_package == 'true') || (needs.check-changes.outputs.sgl_kernel == 'true'))
|
|
)
|
|
)
|
|
runs-on: 4-gpu-h100
|
|
timeout-minutes: 240
|
|
steps:
|
|
- name: Checkout code
|
|
uses: actions/checkout@v4
|
|
with:
|
|
ref: ${{ inputs.pr_head_sha || inputs.git_ref || github.sha }}
|
|
|
|
- uses: ./.github/actions/check-stage-health
|
|
|
|
- uses: ./.github/actions/check-maintenance
|
|
|
|
- name: Download artifacts
|
|
if: needs.check-changes.outputs.sgl_kernel == 'true'
|
|
uses: actions/download-artifact@v4
|
|
with:
|
|
path: sgl-kernel/dist/
|
|
merge-multiple: true
|
|
pattern: wheel-python3.10-cuda*
|
|
|
|
- name: Install dependencies
|
|
timeout-minutes: 20
|
|
run: |
|
|
CUSTOM_BUILD_SGL_KERNEL=${{needs.check-changes.outputs.sgl_kernel}} bash scripts/ci/cuda/ci_install_deepep.sh
|
|
|
|
- name: Warmup DeepGEMM JIT Compilation
|
|
timeout-minutes: 25
|
|
run: |
|
|
# Activate venv if available (GITHUB_ENV may have failed to propagate)
|
|
[ -f "${SGLANG_CI_VENV_PATH:-/dev/null}/bin/activate" ] && source "${SGLANG_CI_VENV_PATH}/bin/activate"
|
|
[ -f "${SGLANG_CI_VENV_PATH:-/dev/null}/env.sh" ] && source "${SGLANG_CI_VENV_PATH}/env.sh"
|
|
python3 scripts/ci/cuda/warmup_deep_gemm.py \
|
|
lmsys/sglang-ci-dsv3-test:4
|
|
|
|
- name: Warmup Server CUDA Graphs
|
|
timeout-minutes: 25
|
|
run: |
|
|
[ -f "${SGLANG_CI_VENV_PATH:-/dev/null}/bin/activate" ] && source "${SGLANG_CI_VENV_PATH}/bin/activate"
|
|
[ -f "${SGLANG_CI_VENV_PATH:-/dev/null}/env.sh" ] && source "${SGLANG_CI_VENV_PATH}/env.sh"
|
|
python3 scripts/ci/cuda/warmup_server.py \
|
|
lmsys/sglang-ci-dsv3-test:4
|
|
|
|
- name: Run test
|
|
timeout-minutes: 30
|
|
env:
|
|
CONTINUE_ON_ERROR_FLAG: ${{ needs.check-changes.outputs.continue_on_error == 'true' && '--continue-on-error' || '' }}
|
|
run: |
|
|
cd test
|
|
python3 run_suite.py --hw cuda --suite stage-c-test-deepep-4-gpu-h100 $CONTINUE_ON_ERROR_FLAG
|
|
|
|
- uses: ./.github/actions/upload-cuda-coredumps
|
|
if: failure()
|
|
|
|
- name: Cleanup venv
|
|
if: always()
|
|
run: bash scripts/ci/cuda/ci_cleanup_venv.sh
|
|
|
|
stage-c-test-deepep-8-gpu-h200:
|
|
needs: [check-changes, call-gate, wait-for-stage-b, sgl-kernel-build-wheels]
|
|
if: |
|
|
always() &&
|
|
(
|
|
(inputs.target_stage == 'stage-c-test-deepep-8-gpu-h200') ||
|
|
(
|
|
!inputs.target_stage &&
|
|
((github.event_name == 'schedule' || inputs.test_parallel_dispatch == true) || (!failure() && !cancelled())) &&
|
|
((needs.check-changes.outputs.main_package == 'true') || (needs.check-changes.outputs.sgl_kernel == 'true'))
|
|
)
|
|
)
|
|
runs-on: 8-gpu-h200-deepep
|
|
timeout-minutes: 240
|
|
steps:
|
|
- name: Checkout code
|
|
uses: actions/checkout@v4
|
|
with:
|
|
ref: ${{ inputs.pr_head_sha || inputs.git_ref || github.sha }}
|
|
|
|
- uses: ./.github/actions/check-stage-health
|
|
|
|
- uses: ./.github/actions/check-maintenance
|
|
|
|
- name: Download artifacts
|
|
if: needs.check-changes.outputs.sgl_kernel == 'true'
|
|
uses: actions/download-artifact@v4
|
|
with:
|
|
path: sgl-kernel/dist/
|
|
merge-multiple: true
|
|
pattern: wheel-python3.10-cuda*
|
|
|
|
- name: Install dependencies
|
|
timeout-minutes: 20
|
|
run: |
|
|
CUSTOM_BUILD_SGL_KERNEL=${{needs.check-changes.outputs.sgl_kernel}} bash scripts/ci/cuda/ci_install_deepep.sh
|
|
|
|
- name: Warmup DeepGEMM JIT Compilation
|
|
timeout-minutes: 25
|
|
run: |
|
|
# Activate venv if available (GITHUB_ENV may have failed to propagate)
|
|
[ -f "${SGLANG_CI_VENV_PATH:-/dev/null}/bin/activate" ] && source "${SGLANG_CI_VENV_PATH}/bin/activate"
|
|
[ -f "${SGLANG_CI_VENV_PATH:-/dev/null}/env.sh" ] && source "${SGLANG_CI_VENV_PATH}/env.sh"
|
|
python3 scripts/ci/cuda/warmup_deep_gemm.py \
|
|
deepseek-ai/DeepSeek-V3-0324:8 \
|
|
deepseek-ai/DeepSeek-V3.2-Exp:8
|
|
|
|
- name: Warmup Server CUDA Graphs
|
|
timeout-minutes: 25
|
|
run: |
|
|
[ -f "${SGLANG_CI_VENV_PATH:-/dev/null}/bin/activate" ] && source "${SGLANG_CI_VENV_PATH}/bin/activate"
|
|
[ -f "${SGLANG_CI_VENV_PATH:-/dev/null}/env.sh" ] && source "${SGLANG_CI_VENV_PATH}/env.sh"
|
|
python3 scripts/ci/cuda/warmup_server.py \
|
|
deepseek-ai/DeepSeek-V3-0324:8
|
|
|
|
- name: Run test
|
|
timeout-minutes: 45
|
|
env:
|
|
CONTINUE_ON_ERROR_FLAG: ${{ needs.check-changes.outputs.continue_on_error == 'true' && '--continue-on-error' || '' }}
|
|
run: |
|
|
cd test
|
|
python3 run_suite.py --hw cuda --suite stage-c-test-deepep-8-gpu-h200 $CONTINUE_ON_ERROR_FLAG
|
|
|
|
- uses: ./.github/actions/upload-cuda-coredumps
|
|
if: failure()
|
|
|
|
- name: Cleanup venv
|
|
if: always()
|
|
run: bash scripts/ci/cuda/ci_cleanup_venv.sh
|
|
|
|
stage-c-test-4-gpu-b200:
|
|
needs: [check-changes, call-gate, wait-for-stage-b, sgl-kernel-build-wheels]
|
|
if: |
|
|
always() &&
|
|
(
|
|
(inputs.target_stage == 'stage-c-test-4-gpu-b200') ||
|
|
(
|
|
!inputs.target_stage &&
|
|
((github.event_name == 'schedule' || inputs.test_parallel_dispatch == true) || (!failure() && !cancelled())) &&
|
|
((needs.check-changes.outputs.main_package == 'true') || (needs.check-changes.outputs.sgl_kernel == 'true'))
|
|
)
|
|
)
|
|
runs-on: ${{ needs.check-changes.outputs.b200_runner }}
|
|
timeout-minutes: 240
|
|
strategy:
|
|
fail-fast: false
|
|
matrix:
|
|
part: [0, 1, 2, 3, 4, 5]
|
|
|
|
steps:
|
|
- name: Checkout code
|
|
uses: actions/checkout@v4
|
|
with:
|
|
ref: ${{ inputs.pr_head_sha || inputs.git_ref || github.sha }}
|
|
|
|
- uses: ./.github/actions/check-stage-health
|
|
|
|
- uses: ./.github/actions/check-maintenance
|
|
|
|
- name: Download artifacts
|
|
if: needs.check-changes.outputs.sgl_kernel == 'true'
|
|
uses: actions/download-artifact@v6
|
|
with:
|
|
path: sgl-kernel/dist/
|
|
merge-multiple: true
|
|
pattern: wheel-python3.10-cuda*
|
|
|
|
- name: Install dependencies
|
|
timeout-minutes: 20
|
|
run: |
|
|
CUSTOM_BUILD_SGL_KERNEL=${{needs.check-changes.outputs.sgl_kernel}} bash scripts/ci/cuda/ci_install_dependency.sh
|
|
|
|
- name: Run test
|
|
timeout-minutes: 30
|
|
env:
|
|
CONTINUE_ON_ERROR_FLAG: ${{ needs.check-changes.outputs.continue_on_error == 'true' && '--continue-on-error' || '' }}
|
|
run: |
|
|
cd test
|
|
python3 run_suite.py --hw cuda --suite stage-c-test-4-gpu-b200 --auto-partition-id ${{ matrix.part }} --auto-partition-size 6 --timeout-per-file 1800 $CONTINUE_ON_ERROR_FLAG
|
|
|
|
- uses: ./.github/actions/upload-cuda-coredumps
|
|
if: failure()
|
|
with:
|
|
artifact-suffix: ${{ matrix.part }}
|
|
|
|
- name: Cleanup venv
|
|
if: always()
|
|
run: bash scripts/ci/cuda/ci_cleanup_venv.sh
|
|
|
|
stage-c-test-dsv4-4-gpu-b200:
|
|
needs: [check-changes, call-gate, wait-for-stage-b, sgl-kernel-build-wheels]
|
|
if: |
|
|
always() &&
|
|
(
|
|
(inputs.target_stage == 'stage-c-test-dsv4-4-gpu-b200') ||
|
|
(
|
|
!inputs.target_stage &&
|
|
((github.event_name == 'schedule' || inputs.test_parallel_dispatch == true) || (!failure() && !cancelled())) &&
|
|
((needs.check-changes.outputs.main_package == 'true') || (needs.check-changes.outputs.sgl_kernel == 'true'))
|
|
)
|
|
)
|
|
runs-on: ${{ needs.check-changes.outputs.b200_runner }}
|
|
timeout-minutes: 240
|
|
steps:
|
|
- name: Checkout code
|
|
uses: actions/checkout@v4
|
|
with:
|
|
ref: ${{ inputs.pr_head_sha || inputs.git_ref || github.sha }}
|
|
|
|
- uses: ./.github/actions/check-stage-health
|
|
|
|
- uses: ./.github/actions/check-maintenance
|
|
|
|
- name: Download artifacts
|
|
if: needs.check-changes.outputs.sgl_kernel == 'true'
|
|
uses: actions/download-artifact@v6
|
|
with:
|
|
path: sgl-kernel/dist/
|
|
merge-multiple: true
|
|
pattern: wheel-python3.10-cuda*
|
|
|
|
- name: Install dependencies
|
|
timeout-minutes: 30
|
|
run: |
|
|
CUSTOM_BUILD_SGL_KERNEL=${{needs.check-changes.outputs.sgl_kernel}} bash scripts/ci/cuda/ci_install_dsv4_dep.sh
|
|
|
|
- name: Run test
|
|
timeout-minutes: 30
|
|
env:
|
|
CONTINUE_ON_ERROR_FLAG: ${{ needs.check-changes.outputs.continue_on_error == 'true' && '--continue-on-error' || '' }}
|
|
run: |
|
|
cd test
|
|
python3 run_suite.py --hw cuda --suite stage-c-test-dsv4-4-gpu-b200 --timeout-per-file 1800 $CONTINUE_ON_ERROR_FLAG
|
|
|
|
- uses: ./.github/actions/upload-cuda-coredumps
|
|
if: failure()
|
|
|
|
- name: Cleanup venv
|
|
if: always()
|
|
run: bash scripts/ci/cuda/ci_cleanup_venv.sh
|
|
|
|
stage-c-test-dsv4-8-gpu-h200:
|
|
needs: [check-changes, call-gate, wait-for-stage-b, sgl-kernel-build-wheels]
|
|
if: |
|
|
always() &&
|
|
(
|
|
(inputs.target_stage == 'stage-c-test-dsv4-8-gpu-h200') ||
|
|
(
|
|
!inputs.target_stage &&
|
|
((github.event_name == 'schedule' || inputs.test_parallel_dispatch == true) || (!failure() && !cancelled())) &&
|
|
((needs.check-changes.outputs.main_package == 'true') || (needs.check-changes.outputs.sgl_kernel == 'true'))
|
|
)
|
|
)
|
|
runs-on: 8-gpu-h200
|
|
timeout-minutes: 240
|
|
steps:
|
|
- name: Checkout code
|
|
uses: actions/checkout@v4
|
|
with:
|
|
ref: ${{ inputs.pr_head_sha || inputs.git_ref || github.sha }}
|
|
|
|
- uses: ./.github/actions/check-stage-health
|
|
|
|
- uses: ./.github/actions/check-maintenance
|
|
|
|
- name: Download artifacts
|
|
if: needs.check-changes.outputs.sgl_kernel == 'true'
|
|
uses: actions/download-artifact@v4
|
|
with:
|
|
path: sgl-kernel/dist/
|
|
merge-multiple: true
|
|
pattern: wheel-python3.10-cuda*
|
|
|
|
- name: Install dependencies
|
|
timeout-minutes: 30
|
|
run: |
|
|
CUSTOM_BUILD_SGL_KERNEL=${{needs.check-changes.outputs.sgl_kernel}} bash scripts/ci/cuda/ci_install_dsv4_dep.sh
|
|
|
|
- name: Run test
|
|
timeout-minutes: 30
|
|
env:
|
|
CONTINUE_ON_ERROR_FLAG: ${{ needs.check-changes.outputs.continue_on_error == 'true' && '--continue-on-error' || '' }}
|
|
run: |
|
|
cd test
|
|
python3 run_suite.py --hw cuda --suite stage-c-test-dsv4-8-gpu-h200 --timeout-per-file 1800 $CONTINUE_ON_ERROR_FLAG
|
|
|
|
- uses: ./.github/actions/upload-cuda-coredumps
|
|
if: failure()
|
|
|
|
- name: Cleanup venv
|
|
if: always()
|
|
run: bash scripts/ci/cuda/ci_cleanup_venv.sh
|
|
|
|
# NOTE: GB200 stage temporarily disabled — no company-owned GB200 runner available yet.
|
|
# Re-enable when a 4-gpu-gb200 runner is provisioned.
|
|
# stage-c-test-4-gpu-gb200:
|
|
# needs: [check-changes, call-gate, wait-for-stage-b, sgl-kernel-build-wheels-arm]
|
|
# if: |
|
|
# always() &&
|
|
# (
|
|
# (inputs.target_stage == 'stage-c-test-4-gpu-gb200') ||
|
|
# (
|
|
# !inputs.target_stage &&
|
|
# ((github.event_name == 'schedule' || inputs.test_parallel_dispatch == true) || (!failure() && !cancelled())) &&
|
|
# ((needs.check-changes.outputs.main_package == 'true') || (needs.check-changes.outputs.sgl_kernel == 'true'))
|
|
# )
|
|
# )
|
|
# runs-on: 4-gpu-gb200
|
|
# timeout-minutes: 240
|
|
# strategy:
|
|
# fail-fast: false
|
|
# steps:
|
|
# - uses: ./.github/actions/check-maintenance
|
|
# with:
|
|
# github-token: ${{ github.token }}
|
|
#
|
|
# - name: Checkout code
|
|
# uses: actions/checkout@v4
|
|
# with:
|
|
# ref: ${{ inputs.pr_head_sha || inputs.git_ref || github.sha }}
|
|
#
|
|
# - name: Download artifacts
|
|
# if: needs.check-changes.outputs.sgl_kernel == 'true'
|
|
# uses: actions/download-artifact@v4
|
|
# with:
|
|
# path: sgl-kernel/dist/
|
|
# merge-multiple: true
|
|
# pattern: wheel-python3.10-cuda13.0-aarch64
|
|
#
|
|
# - name: Install dependencies
|
|
# timeout-minutes: 20
|
|
# run: |
|
|
# CUSTOM_BUILD_SGL_KERNEL=${{needs.check-changes.outputs.sgl_kernel}} GRACE_BLACKWELL=1 bash scripts/ci/cuda/ci_install_deepep.sh
|
|
#
|
|
# - name: Run test
|
|
# timeout-minutes: 45
|
|
# env:
|
|
# CONTINUE_ON_ERROR_FLAG: ${{ needs.check-changes.outputs.continue_on_error == 'true' && '--continue-on-error' || '' }}
|
|
# run: |
|
|
# cd test
|
|
# python3 run_suite.py --hw cuda --suite stage-c-test-4-gpu-gb200 --timeout-per-file 3600 $CONTINUE_ON_ERROR_FLAG
|
|
#
|
|
# - uses: ./.github/actions/upload-cuda-coredumps
|
|
# if: failure()
|
|
|
|
pr-test-finish:
|
|
needs:
|
|
[
|
|
call-gate,
|
|
check-changes,
|
|
|
|
sgl-kernel-build-wheels,
|
|
sgl-kernel-build-wheels-arm,
|
|
call-sgl-kernel-tests,
|
|
|
|
wait-for-stage-a,
|
|
wait-for-stage-b,
|
|
|
|
call-jit-kernel-tests,
|
|
|
|
call-multimodal-gen-tests,
|
|
|
|
stage-a-test-1-gpu-small,
|
|
stage-a-test-cpu,
|
|
stage-b-test-1-gpu-small,
|
|
stage-b-test-1-gpu-large,
|
|
stage-b-test-2-gpu-large,
|
|
stage-b-test-4-gpu-b200,
|
|
stage-c-test-4-gpu-h100,
|
|
stage-c-test-8-gpu-h20,
|
|
stage-c-test-8-gpu-h200,
|
|
stage-c-test-deepep-4-gpu-h100,
|
|
stage-c-test-deepep-8-gpu-h200,
|
|
stage-c-test-4-gpu-b200,
|
|
stage-c-test-dsv4-4-gpu-b200,
|
|
stage-c-test-dsv4-8-gpu-h200,
|
|
# stage-c-test-4-gpu-gb200, # Temporarily disabled — no GB200 runner
|
|
]
|
|
if: always()
|
|
runs-on: ubuntu-latest
|
|
steps:
|
|
- name: Check all dependent job statuses
|
|
run: |
|
|
# Convert the 'needs' context to a JSON string
|
|
json_needs='${{ toJson(needs) }}'
|
|
|
|
# Get a list of all job names from the JSON keys
|
|
job_names=$(echo "$json_needs" | jq -r 'keys_unsorted[]')
|
|
|
|
for job in $job_names; do
|
|
# For each job, extract its result
|
|
result=$(echo "$json_needs" | jq -r --arg j "$job" '.[$j].result')
|
|
|
|
# Print the job name and its result
|
|
echo "$job: $result"
|
|
|
|
# Check for failure or cancellation and exit if found
|
|
if [[ "$result" == "failure" || "$result" == "cancelled" ]]; then
|
|
echo "The above jobs failed."
|
|
exit 1
|
|
fi
|
|
done
|
|
# If the loop completes, all jobs were successful
|
|
echo "All jobs completed successfully"
|
|
exit 0
|