minke.yu/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 13 Packages Projects Releases Wiki Activity

[CI] Add per-job uv venv isolation and upgrade CI version to Cuda 13 (#23119)

Browse Source 
Co-authored-by: Kangyan Zhou <zky314343421@gmail.com>
Co-authored-by: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
Co-authored-by: Alison Shao <a.shao@wustl.edu>
Co-authored-by: Mick <mickjagger19@icloud.com>
This commit is contained in:
Baizhou Zhang
2026-04-19 05:32:36 -07:00
committed by GitHub
co-authored by Kangyan Zhou Claude Opus 4.7 Alison Shao Mick
parent 03828f4205
commit 6ecd6f84db
39 changed files with 892 additions and 239 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/workflows/pr-test.yml
+75 -22
View File
@@ -65,6 +65,7 @@ env:
SKIP_STAGE_HEALTH_CHECK: ${{ inputs.skip_stage_health_check == true && 'true' || 'false' }}
# Schedule / main-branch dispatch / workflow_call from main use refs/heads/main; PR events use refs/pull/*/merge
SGLANG_PR_TEST_BYPASS_MAINTENANCE_ON_MAIN: ${{ github.ref == 'refs/heads/main' && 'true' || 'false' }}
USE_VENV: false
permissions:
actions: write
@@ -340,9 +341,6 @@ jobs:
wait-for-stage-a:
needs: [check-changes, call-gate]
# Only run for PRs (not scheduled) and when not targeting a specific stage
# Skip if call-gate failed (stage-a jobs will be skipped, nothing to wait for)
# !cancelled() ensures this job respects workflow cancellation from concurrency group
if: |
always() &&
!cancelled() &&
@@ -368,8 +366,6 @@ jobs:
wait-for-stage-b:
needs: [check-changes, call-gate, wait-for-stage-a]
# Only run for PRs (not scheduled) and when not targeting a specific stage
# Skip if call-gate failed (stage-b jobs will be skipped, nothing to wait for)
if: |
always() &&
!cancelled() &&
@@ -429,10 +425,7 @@ jobs:
matrix:
include:
- python-version: "3.10"
cuda-version: "12.9"
# Add back when CUDA 13.0 is supported on CI
# - python-version: "3.10"
# cuda-version: "13.0"
cuda-version: "13.0"
name: Build Wheel
steps:
- name: Cleanup
@@ -480,7 +473,7 @@ jobs:
matrix:
include:
- python-version: "3.10"
cuda-version: "12.9"
cuda-version: "13.0"
name: Build Wheel Arm
steps:
- name: Cleanup
@@ -587,7 +580,7 @@ jobs:
with:
path: sgl-kernel/dist/
merge-multiple: true
pattern: wheel-python3.10-cuda12.9
pattern: wheel-python3.10-cuda13.0
- name: Install dependencies
timeout-minutes: 20
@@ -605,6 +598,10 @@ jobs:
- uses: ./.github/actions/upload-cuda-coredumps
if: failure()
- name: Cleanup venv
if: always()
run: bash scripts/ci/cuda/ci_cleanup_venv.sh
stage-a-test-cpu:
needs: [check-changes, call-gate]
if: |
@@ -694,7 +691,7 @@ jobs:
with:
path: sgl-kernel/dist/
merge-multiple: true
pattern: wheel-python3.10-cuda12.9
pattern: wheel-python3.10-cuda13.0
- name: Install dependencies
timeout-minutes: 20
@@ -716,6 +713,10 @@ jobs:
with:
artifact-suffix: ${{ matrix.partition }}
- name: Cleanup venv
if: always()
run: bash scripts/ci/cuda/ci_cleanup_venv.sh
# Runs on H100 (80GB, SM90) - tests that don't pass on 5090 (FA3, FP8, high VRAM, etc.)
stage-b-test-1-gpu-large:
needs: [check-changes, call-gate, wait-for-stage-a, sgl-kernel-build-wheels]
@@ -752,7 +753,7 @@ jobs:
with:
path: sgl-kernel/dist/
merge-multiple: true
pattern: wheel-python3.10-cuda12.9
pattern: wheel-python3.10-cuda13.0
- name: Install dependencies
timeout-minutes: 20
@@ -772,6 +773,10 @@ jobs:
with:
artifact-suffix: ${{ matrix.partition }}
- name: Cleanup venv
if: always()
run: bash scripts/ci/cuda/ci_cleanup_venv.sh
stage-b-test-2-gpu-large:
needs: [check-changes, call-gate, wait-for-stage-a, sgl-kernel-build-wheels]
if: |
@@ -807,7 +812,7 @@ jobs:
with:
path: sgl-kernel/dist/
merge-multiple: true
pattern: wheel-python3.10-cuda12.9
pattern: wheel-python3.10-cuda13.0
- name: Install dependencies
timeout-minutes: 20
@@ -827,6 +832,10 @@ jobs:
with:
artifact-suffix: ${{ matrix.partition }}
- name: Cleanup venv
if: always()
run: bash scripts/ci/cuda/ci_cleanup_venv.sh
stage-b-test-4-gpu-b200:
needs: [check-changes, call-gate, wait-for-stage-a, sgl-kernel-build-wheels]
if: |
@@ -860,7 +869,7 @@ jobs:
with:
path: sgl-kernel/dist/
merge-multiple: true
pattern: wheel-python3.10-cuda12.9
pattern: wheel-python3.10-cuda13.0
- name: Install dependencies
timeout-minutes: 20
@@ -883,6 +892,10 @@ jobs:
- uses: ./.github/actions/upload-cuda-coredumps
if: failure()
- name: Cleanup venv
if: always()
run: bash scripts/ci/cuda/ci_cleanup_venv.sh
call-multimodal-gen-tests:
needs: [check-changes, call-gate, sgl-kernel-build-wheels]
if: |
@@ -950,7 +963,7 @@ jobs:
with:
path: sgl-kernel/dist/
merge-multiple: true
pattern: wheel-python3.10-cuda12.9
pattern: wheel-python3.10-cuda13.0
- name: Install dependencies
timeout-minutes: 20
@@ -970,6 +983,10 @@ jobs:
with:
artifact-suffix: ${{ matrix.part }}
- name: Cleanup venv
if: always()
run: bash scripts/ci/cuda/ci_cleanup_venv.sh
stage-c-test-8-gpu-h200:
needs: [check-changes, call-gate, wait-for-stage-b]
if: |
@@ -1004,7 +1021,7 @@ jobs:
with:
path: sgl-kernel/dist/
merge-multiple: true
pattern: wheel-python3.10-cuda12.9
pattern: wheel-python3.10-cuda13.0
- name: Install dependencies
timeout-minutes: 20
@@ -1014,6 +1031,9 @@ jobs:
- name: Warmup DeepGEMM JIT Compilation
timeout-minutes: 25
run: |
# Activate venv if available (GITHUB_ENV may have failed to propagate)
[ -f "${SGLANG_CI_VENV_PATH:-/dev/null}/bin/activate" ] && source "${SGLANG_CI_VENV_PATH}/bin/activate"
[ -f "${SGLANG_CI_VENV_PATH:-/dev/null}/env.sh" ] && source "${SGLANG_CI_VENV_PATH}/env.sh"
python3 scripts/ci/cuda/warmup_deep_gemm.py \
deepseek-ai/DeepSeek-V3-0324:8 \
deepseek-ai/DeepSeek-V3.2-Exp:8
@@ -1021,6 +1041,8 @@ jobs:
- name: Warmup Server CUDA Graphs
timeout-minutes: 25
run: |
[ -f "${SGLANG_CI_VENV_PATH:-/dev/null}/bin/activate" ] && source "${SGLANG_CI_VENV_PATH}/bin/activate"
[ -f "${SGLANG_CI_VENV_PATH:-/dev/null}/env.sh" ] && source "${SGLANG_CI_VENV_PATH}/env.sh"
python3 scripts/ci/cuda/warmup_server.py \
deepseek-ai/DeepSeek-V3-0324:8 \
inclusionAI/Ring-2.5-1T:8
@@ -1038,6 +1060,10 @@ jobs:
with:
artifact-suffix: ${{ matrix.part }}
- name: Cleanup venv
if: always()
run: bash scripts/ci/cuda/ci_cleanup_venv.sh
stage-c-test-8-gpu-h20:
needs: [check-changes, call-gate, wait-for-stage-b]
if: |
@@ -1054,6 +1080,7 @@ jobs:
timeout-minutes: 240
env:
SGLANG_CI_RDMA_ALL_DEVICES: "mlx5_1,mlx5_2,mlx5_3,mlx5_4"
CU_VERSION: cu129
strategy:
fail-fast: false
matrix:
@@ -1074,7 +1101,7 @@ jobs:
with:
path: sgl-kernel/dist/
merge-multiple: true
pattern: wheel-python3.10-cuda12.9
pattern: wheel-python3.10-cuda13.0
- name: Install dependencies
timeout-minutes: 20
@@ -1094,6 +1121,10 @@ jobs:
with:
artifact-suffix: ${{ matrix.part }}
- name: Cleanup venv
if: always()
run: bash scripts/ci/cuda/ci_cleanup_venv.sh
stage-c-test-deepep-4-gpu-h100:
needs: [check-changes, call-gate, wait-for-stage-b]
if: |
@@ -1124,7 +1155,7 @@ jobs:
with:
path: sgl-kernel/dist/
merge-multiple: true
pattern: wheel-python3.10-cuda12.9
pattern: wheel-python3.10-cuda13.0
- name: Install dependencies
timeout-minutes: 20
@@ -1134,12 +1165,17 @@ jobs:
- name: Warmup DeepGEMM JIT Compilation
timeout-minutes: 25
run: |
# Activate venv if available (GITHUB_ENV may have failed to propagate)
[ -f "${SGLANG_CI_VENV_PATH:-/dev/null}/bin/activate" ] && source "${SGLANG_CI_VENV_PATH}/bin/activate"
[ -f "${SGLANG_CI_VENV_PATH:-/dev/null}/env.sh" ] && source "${SGLANG_CI_VENV_PATH}/env.sh"
python3 scripts/ci/cuda/warmup_deep_gemm.py \
lmsys/sglang-ci-dsv3-test:4
- name: Warmup Server CUDA Graphs
timeout-minutes: 25
run: |
[ -f "${SGLANG_CI_VENV_PATH:-/dev/null}/bin/activate" ] && source "${SGLANG_CI_VENV_PATH}/bin/activate"
[ -f "${SGLANG_CI_VENV_PATH:-/dev/null}/env.sh" ] && source "${SGLANG_CI_VENV_PATH}/env.sh"
python3 scripts/ci/cuda/warmup_server.py \
lmsys/sglang-ci-dsv3-test:4
@@ -1154,6 +1190,10 @@ jobs:
- uses: ./.github/actions/upload-cuda-coredumps
if: failure()
- name: Cleanup venv
if: always()
run: bash scripts/ci/cuda/ci_cleanup_venv.sh
stage-c-test-deepep-8-gpu-h200:
needs: [check-changes, call-gate, wait-for-stage-b]
if: |
@@ -1184,7 +1224,7 @@ jobs:
with:
path: sgl-kernel/dist/
merge-multiple: true
pattern: wheel-python3.10-cuda12.9
pattern: wheel-python3.10-cuda13.0
- name: Install dependencies
timeout-minutes: 20
@@ -1194,6 +1234,9 @@ jobs:
- name: Warmup DeepGEMM JIT Compilation
timeout-minutes: 25
run: |
# Activate venv if available (GITHUB_ENV may have failed to propagate)
[ -f "${SGLANG_CI_VENV_PATH:-/dev/null}/bin/activate" ] && source "${SGLANG_CI_VENV_PATH}/bin/activate"
[ -f "${SGLANG_CI_VENV_PATH:-/dev/null}/env.sh" ] && source "${SGLANG_CI_VENV_PATH}/env.sh"
python3 scripts/ci/cuda/warmup_deep_gemm.py \
deepseek-ai/DeepSeek-V3-0324:8 \
deepseek-ai/DeepSeek-V3.2-Exp:8
@@ -1201,6 +1244,8 @@ jobs:
- name: Warmup Server CUDA Graphs
timeout-minutes: 25
run: |
[ -f "${SGLANG_CI_VENV_PATH:-/dev/null}/bin/activate" ] && source "${SGLANG_CI_VENV_PATH}/bin/activate"
[ -f "${SGLANG_CI_VENV_PATH:-/dev/null}/env.sh" ] && source "${SGLANG_CI_VENV_PATH}/env.sh"
python3 scripts/ci/cuda/warmup_server.py \
deepseek-ai/DeepSeek-V3-0324:8
@@ -1215,6 +1260,10 @@ jobs:
- uses: ./.github/actions/upload-cuda-coredumps
if: failure()
- name: Cleanup venv
if: always()
run: bash scripts/ci/cuda/ci_cleanup_venv.sh
stage-c-test-4-gpu-b200:
needs: [check-changes, call-gate, wait-for-stage-b]
if: |
@@ -1250,7 +1299,7 @@ jobs:
with:
path: sgl-kernel/dist/
merge-multiple: true
pattern: wheel-python3.10-cuda12.9
pattern: wheel-python3.10-cuda13.0
- name: Install dependencies
timeout-minutes: 20
@@ -1270,6 +1319,10 @@ jobs:
with:
artifact-suffix: ${{ matrix.part }}
- name: Cleanup venv
if: always()
run: bash scripts/ci/cuda/ci_cleanup_venv.sh
# NOTE: GB200 stage temporarily disabled — no company-owned GB200 runner available yet.
# Re-enable when a 4-gpu-gb200 runner is provisioned.
# stage-c-test-4-gpu-gb200:
@@ -1304,7 +1357,7 @@ jobs:
# with:
# path: sgl-kernel/dist/
# merge-multiple: true
# pattern: wheel-python3.10-cuda12.9-aarch64
# pattern: wheel-python3.10-cuda13.0-aarch64
#
# - name: Install dependencies
# timeout-minutes: 20