minke.yu/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 10 Packages Projects Releases Wiki Activity

[Deps] Retire the CUDA 12 lane (#38404)

Browse Source 
Co-authored-by: Mohammad Angkad <mohammad.angkad@radixark.ai>
This commit is contained in:
Mohammad Miadh Angkad
2026-09-10 16:58:09 -07:00
committed by GitHub
co-authored by Mohammad Angkad
parent fae8cd84cb
commit 52c191da52
38 changed files with 170 additions and 737 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/workflows/_docker-build-and-publish.yml
+21 -92
View File
@@ -1,7 +1,7 @@
name: Build and Publish Multi-Arch Docker Images
# Reusable workflow: builds CUDA 12 + CUDA 13 images for amd64 and arm64,
# then creates multi-arch manifests with caller-specified tags.
# Reusable workflow: builds CUDA 13 images for amd64 and arm64, then creates
# multi-arch manifests with caller-specified tags.
on:
workflow_call:
@@ -26,7 +26,7 @@ on:
type: string
default: ""
tag_config:
description: 'JSON array of {"cuda":"cu129|cu130","tags":["tag1","tag2"]}. Tags support {version} substitution.'
description: 'JSON array of {"cuda":"cu130","tags":["tag1","tag2"]}. Tags support {version} substitution.'
required: true
type: string
use_environment:
@@ -50,7 +50,6 @@ jobs:
SGL_VERSION: ${{ inputs.sgl_version }}
IMAGE_REPO: ${{ inputs.image_repo }}
outputs:
digest-cu129: ${{ steps.build-cu129.outputs.digest }}
digest-cu130: ${{ steps.build-cu130.outputs.digest }}
steps:
- name: Delete huge unnecessary tools folder
@@ -69,16 +68,14 @@ jobs:
set -euo pipefail
BUILD_COMMIT="$(git rev-parse HEAD)"
BUILD_URL="${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}"
for CUDA_VARIANT in cu129 cu130; do
python3 scripts/ci/utils/docker_build_metadata_args.py \
--cuda "${CUDA_VARIANT}" \
--tag-config "${TAG_CONFIG}" \
--image-repo "${IMAGE_REPO}" \
--sgl-version "${SGL_VERSION}" \
--build-commit "${BUILD_COMMIT}" \
--build-url "${BUILD_URL}" \
> "/tmp/docker-metadata-${CUDA_VARIANT}.args"
done
python3 scripts/ci/utils/docker_build_metadata_args.py \
--cuda cu130 \
--tag-config "${TAG_CONFIG}" \
--image-repo "${IMAGE_REPO}" \
--sgl-version "${SGL_VERSION}" \
--build-commit "${BUILD_COMMIT}" \
--build-url "${BUILD_URL}" \
> /tmp/docker-metadata-cu130.args
- name: Free disk space
uses: jlumbroso/free-disk-space@main
@@ -106,34 +103,6 @@ jobs:
username: ${{ secrets.DOCKERHUB_USERNAME }}
password: ${{ secrets.DOCKERHUB_TOKEN }}
- name: Build and push AMD64 image (CUDA 12)
id: build-cu129
run: |
VERSION_ARG=""
if [ -n "${SGL_VERSION}" ]; then
VERSION_ARG="--build-arg SGL_VERSION=${SGL_VERSION}"
fi
mapfile -t METADATA_ARGS < /tmp/docker-metadata-cu129.args
docker buildx build \
--target ${{ inputs.docker_target }} \
--platform linux/amd64 \
--output type=image,name=${{ inputs.image_repo }},push-by-digest=true,name-canonical=true,push=true \
-f docker/Dockerfile \
--build-arg CUDA_VERSION=12.9.2 \
--build-arg BUILD_TYPE=all \
--build-arg INSTALL_FLASHINFER_JIT_CACHE=1 \
"${METADATA_ARGS[@]}" \
${VERSION_ARG} \
${{ inputs.extra_build_args }} \
--metadata-file /tmp/metadata-cu129.json \
--no-cache \
.
DIGEST=$(python3 -c "import json; print(json.load(open('/tmp/metadata-cu129.json'))['containerimage.digest'])")
echo "Pushed digest: ${DIGEST}"
echo "digest=${DIGEST}" >> $GITHUB_OUTPUT
- name: Build and push AMD64 image (CUDA 13)
id: build-cu130
run: |
@@ -171,7 +140,6 @@ jobs:
SGL_VERSION: ${{ inputs.sgl_version }}
IMAGE_REPO: ${{ inputs.image_repo }}
outputs:
digest-cu129: ${{ steps.build-cu129.outputs.digest }}
digest-cu130: ${{ steps.build-cu130.outputs.digest }}
steps:
- name: Delete huge unnecessary tools folder
@@ -190,16 +158,14 @@ jobs:
set -euo pipefail
BUILD_COMMIT="$(git rev-parse HEAD)"
BUILD_URL="${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}"
for CUDA_VARIANT in cu129 cu130; do
python3 scripts/ci/utils/docker_build_metadata_args.py \
--cuda "${CUDA_VARIANT}" \
--tag-config "${TAG_CONFIG}" \
--image-repo "${IMAGE_REPO}" \
--sgl-version "${SGL_VERSION}" \
--build-commit "${BUILD_COMMIT}" \
--build-url "${BUILD_URL}" \
> "/tmp/docker-metadata-${CUDA_VARIANT}.args"
done
python3 scripts/ci/utils/docker_build_metadata_args.py \
--cuda cu130 \
--tag-config "${TAG_CONFIG}" \
--image-repo "${IMAGE_REPO}" \
--sgl-version "${SGL_VERSION}" \
--build-commit "${BUILD_COMMIT}" \
--build-url "${BUILD_URL}" \
> /tmp/docker-metadata-cu130.args
- name: Prune Docker to reclaim disk space
run: |
@@ -216,34 +182,6 @@ jobs:
username: ${{ secrets.DOCKERHUB_USERNAME }}
password: ${{ secrets.DOCKERHUB_TOKEN }}
- name: Build and push ARM64 image (CUDA 12)
id: build-cu129
run: |
VERSION_ARG=""
if [ -n "${SGL_VERSION}" ]; then
VERSION_ARG="--build-arg SGL_VERSION=${SGL_VERSION}"
fi
mapfile -t METADATA_ARGS < /tmp/docker-metadata-cu129.args
docker buildx build \
--target ${{ inputs.docker_target }} \
--platform linux/arm64 \
--output type=image,name=${{ inputs.image_repo }},push-by-digest=true,name-canonical=true,push=true \
-f docker/Dockerfile \
--build-arg CUDA_VERSION=12.9.2 \
--build-arg BUILD_TYPE=all \
--build-arg INSTALL_FLASHINFER_JIT_CACHE=1 \
"${METADATA_ARGS[@]}" \
${VERSION_ARG} \
${{ inputs.extra_build_args }} \
--metadata-file /tmp/metadata-cu129.json \
--no-cache \
.
DIGEST=$(python3 -c "import json; print(json.load(open('/tmp/metadata-cu129.json'))['containerimage.digest'])")
echo "Pushed digest: ${DIGEST}"
echo "digest=${DIGEST}" >> $GITHUB_OUTPUT
- name: Build and push ARM64 image (CUDA 13)
id: build-cu130
run: |
@@ -292,22 +230,13 @@ jobs:
TAG_CONFIG: ${{ inputs.tag_config }}
SGL_VERSION: ${{ inputs.sgl_version }}
IMAGE_REPO: ${{ inputs.image_repo }}
X86_CU129: ${{ needs.build-x86.outputs.digest-cu129 }}
X86_CU130: ${{ needs.build-x86.outputs.digest-cu130 }}
ARM64_CU129: ${{ needs.build-arm64.outputs.digest-cu129 }}
ARM64_CU130: ${{ needs.build-arm64.outputs.digest-cu130 }}
SHORT_SHA: ${{ github.sha }}
run: |
echo "${TAG_CONFIG}" | jq -c '.[]' | while read -r entry; do
CUDA=$(echo "${entry}" | jq -r '.cuda')
if [ "${CUDA}" = "cu129" ]; then
X86_DIGEST="${X86_CU129}"
ARM64_DIGEST="${ARM64_CU129}"
else
X86_DIGEST="${X86_CU130}"
ARM64_DIGEST="${ARM64_CU130}"
fi
X86_DIGEST="${X86_CU130}"
ARM64_DIGEST="${ARM64_CU130}"
TAG_ARGS=""
for tag in $(echo "${entry}" | jq -r '.tags[]'); do
.github/workflows/patch-docker-dev.yml
+1 -1
View File
@@ -8,7 +8,7 @@ on:
required: false
default: ""
image_tag:
description: "Base image tag to patch (e.g. dev, dev-cu13, dev-cu12)"
description: "Base image tag to patch (e.g. dev, dev-cu13)"
required: true
output_tag:
description: "Tag to publish as. Overwrites it if it already exists."
.github/workflows/release-docker-dev.yml
+6 -8
View File
@@ -25,7 +25,7 @@ on:
required: false
default: ""
overlay_cudas:
description: "Comma-separated cuda variants to overlay onto (cu12, cu13). Default 'cu13'."
description: "Comma-separated cuda variants to overlay onto (cu13). Default 'cu13'."
required: false
default: "cu13"
overlay_tag_suffix:
@@ -86,9 +86,9 @@ jobs:
# consumers pinned to the pre-flip names.
if [ -z "${SUFFIX}" ]; then
# Nightly: include dated tags
TAG_CONFIG='[{"cuda":"cu129","tags":["dev-cu12","nightly-dev-cu12-{date}-{short_sha}"]},{"cuda":"cu130","tags":["dev","dev-cu13","nightly-dev-{date}-{short_sha}","nightly-dev-cu13-{date}-{short_sha}"]}]'
TAG_CONFIG='[{"cuda":"cu130","tags":["dev","dev-cu13","nightly-dev-{date}-{short_sha}","nightly-dev-cu13-{date}-{short_sha}"]}]'
else
TAG_CONFIG="[{\"cuda\":\"cu129\",\"tags\":[\"dev-cu12${SUFFIX}\"]},{\"cuda\":\"cu130\",\"tags\":[\"dev${SUFFIX}\",\"dev-cu13${SUFFIX}\"]}]"
TAG_CONFIG="[{\"cuda\":\"cu130\",\"tags\":[\"dev${SUFFIX}\",\"dev-cu13${SUFFIX}\"]}]"
fi
echo "tag_config=${TAG_CONFIG}" >> $GITHUB_OUTPUT
@@ -97,7 +97,7 @@ jobs:
# SUFFIX was computed above from inputs.tag / inputs.pr_number.
if [ -n "${OVERLAY_DOCKERFILE_INPUT}" ]; then
# Refuse to overlay without a tag/pr_number: SUFFIX would be empty,
# so base would point at the moving 'dev'/'dev-cu12'/'dev-cu13'
# so base would point at the moving 'dev'/'dev-cu13'
# latest tags and overwrite mode would clobber them.
if [ -z "${SUFFIX}" ]; then
echo "overlay_dockerfile is set but neither 'tag' nor 'pr_number' was given." >&2
@@ -112,10 +112,8 @@ jobs:
C="$(echo "${C}" | xargs)"
if [ "${C}" = "cu13" ]; then
BASES='["dev'"${SUFFIX}"'","dev-cu13'"${SUFFIX}"'"]'
elif [ "${C}" = "cu12" ]; then
BASES='["dev-cu12'"${SUFFIX}"'"]'
else
echo "Unknown overlay cuda variant: ${C} (expected cu12 or cu13)" >&2
echo "Unknown overlay cuda variant: ${C} (expected cu13)" >&2
exit 1
fi
if [ -n "${OVERLAY_SUFFIX_INPUT}" ]; then
@@ -213,7 +211,7 @@ jobs:
if: ${{ !inputs.build_only && !inputs.tag && !inputs.pr_number }}
uses: ./.github/workflows/_docker-cleanup-nightly.yml
with:
tag_prefixes: '["nightly-dev", "nightly-dev-cu12", "nightly-dev-cu13"]'
tag_prefixes: '["nightly-dev", "nightly-dev-cu13"]'
image_repo: ${{ inputs.image_repo || 'lmsysorg/sglang' }}
secrets: inherit
.github/workflows/release-docker-runtime.yml
+2 -3
View File
@@ -2,7 +2,7 @@ name: Release Docker Runtime Images
#
# Builds and publishes runtime Docker images (production-optimized, ~50% smaller):
# - lmsysorg/sglang:v{version}-runtime, lmsysorg/sglang:latest-runtime
# - lmsysorg/sglang:v{version}-cu129-runtime, lmsysorg/sglang:latest-cu129-runtime
# - lmsysorg/sglang:v{version}-cu130-runtime, lmsysorg/sglang:latest-cu130-runtime
#
on:
push:
@@ -49,7 +49,6 @@ jobs:
image_repo: ${{ inputs.image_repo || 'lmsysorg/sglang' }}
tag_config: |
[
{"cuda": "cu130", "tags": ["v${{ needs.resolve-version.outputs.version }}-runtime", "latest-runtime", "v${{ needs.resolve-version.outputs.version }}-cu130-runtime", "latest-cu130-runtime"]},
{"cuda": "cu129", "tags": ["v${{ needs.resolve-version.outputs.version }}-cu129-runtime", "latest-cu129-runtime"]}
{"cuda": "cu130", "tags": ["v${{ needs.resolve-version.outputs.version }}-runtime", "latest-runtime", "v${{ needs.resolve-version.outputs.version }}-cu130-runtime", "latest-cu130-runtime"]}
]
secrets: inherit
.github/workflows/release-docker.yml
+2 -3
View File
@@ -2,7 +2,7 @@ name: Release Docker Images
#
# Builds and publishes framework Docker images (full development environment):
# - lmsysorg/sglang:v{version}, lmsysorg/sglang:latest (cuda 13)
# - lmsysorg/sglang:v{version}-cu129, lmsysorg/sglang:latest-cu129
# - lmsysorg/sglang:v{version}-cu130, lmsysorg/sglang:latest-cu130
#
on:
push:
@@ -49,7 +49,6 @@ jobs:
image_repo: ${{ inputs.image_repo || 'lmsysorg/sglang' }}
tag_config: |
[
{"cuda": "cu130", "tags": ["v${{ needs.resolve-version.outputs.version }}", "latest", "v${{ needs.resolve-version.outputs.version }}-cu130", "latest-cu130"]},
{"cuda": "cu129", "tags": ["v${{ needs.resolve-version.outputs.version }}-cu129", "latest-cu129"]}
{"cuda": "cu130", "tags": ["v${{ needs.resolve-version.outputs.version }}", "latest", "v${{ needs.resolve-version.outputs.version }}-cu130", "latest-cu130"]}
]
secrets: inherit
.github/workflows/release-pypi-nightly.yml
+5 -5
View File
@@ -113,13 +113,13 @@ jobs:
environment: 'prod'
strategy:
fail-fast: false
# The wheel is CUDA-agnostic and built once — we just register the same
# artifact under cu129/sglang/ and cu130/sglang/ wheel indexes so users
# can install via either --extra-index-url. Serialize because both matrix
# runs clone and push to the same sgl-whl branch.
# The wheel is CUDA-agnostic and built once — we just register it under
# the cu130/sglang/ wheel index so users can install via
# --extra-index-url. Serialize because matrix runs clone and push to the
# same sgl-whl branch.
max-parallel: 1
matrix:
cuda_version: ['129', '130']
cuda_version: ['130']
steps:
- uses: actions/checkout@v4
.github/workflows/release-whl-deepep.yml
+7 -103
View File
@@ -7,15 +7,6 @@ on:
description: "Public wheel version (for example, 0.1.0 or 0.1.1rc0)"
type: string
required: true
target:
description: "CUDA target"
type: choice
required: false
default: "all"
options:
- "all"
- "cu129"
- "cu130"
packaging-ref:
description: "DeepEP ref containing the shared sgl_deep_ep overlay"
type: string
@@ -30,75 +21,8 @@ env:
TORCH_VERSION: "2.13.0"
jobs:
build-cu129-matrix:
if: |
github.repository == 'sgl-project/sglang' &&
(inputs.target == 'all' || inputs.target == 'cu129')
strategy:
fail-fast: false
matrix:
python-version: ["3.10", "3.12"]
arch: [x86_64, aarch64]
include:
- arch: x86_64
runner: x64-kernel-build-node
source-branch: sgl-deepep-cu12-x86
- arch: aarch64
runner: arm-kernel-build-node
source-branch: sgl-deepep-cu12-arm
runs-on: ${{ matrix.runner }}
steps:
- name: Clean workspace
run: |
docker run --rm -v "${{ github.workspace }}:/workspace" alpine:3 \
sh -c 'rm -rf /workspace/..?* /workspace/.[!.]* /workspace/*' || true
- name: Checkout SGLang
uses: actions/checkout@v4
- name: Checkout DeepEP implementation
uses: actions/checkout@v4
with:
repository: sgl-project/DeepEP
ref: ${{ matrix.source-branch }}
path: DeepEP-source
submodules: recursive
- name: Checkout DeepEP packaging overlay
uses: actions/checkout@v4
with:
repository: sgl-project/DeepEP
ref: ${{ inputs['packaging-ref'] || 'sgl-deepep-packaging' }}
path: DeepEP-packaging
sparse-checkout: sgl_deep_ep
- name: Set wheel version
env:
RELEASE_VERSION: ${{ inputs.version }}
run: |
printf '%s' "${RELEASE_VERSION}" > DeepEP-packaging/sgl_deep_ep/VERSION
- name: Build wheel
run: |
chmod +x scripts/build_sgl_deepep.sh DeepEP-packaging/sgl_deep_ep/build_sgl_deep_ep.sh
scripts/build_sgl_deepep.sh \
"${{ matrix.python-version }}" \
"12.9" \
"${{ github.workspace }}/DeepEP-source" \
"${{ github.workspace }}/DeepEP-packaging/sgl_deep_ep" \
"${{ matrix.arch }}"
- name: Upload wheel
uses: actions/upload-artifact@v4
with:
name: deepep-wheel-cu129-py${{ matrix.python-version }}-${{ matrix.arch }}
path: DeepEP-source/dist/*.whl
if-no-files-found: error
build-cu130-matrix:
if: |
github.repository == 'sgl-project/sglang' &&
(inputs.target == 'all' || inputs.target == 'cu130')
if: github.repository == 'sgl-project/sglang'
strategy:
fail-fast: false
matrix:
@@ -168,19 +92,10 @@ jobs:
if-no-files-found: error
release-whl:
needs: [build-cu129-matrix, build-cu130-matrix]
needs: build-cu130-matrix
if: |
always() &&
github.repository == 'sgl-project/sglang' &&
(
(inputs.target == 'cu129' && needs.build-cu129-matrix.result == 'success') ||
(inputs.target == 'cu130' && needs.build-cu130-matrix.result == 'success') ||
(
inputs.target == 'all' &&
needs.build-cu129-matrix.result == 'success' &&
needs.build-cu130-matrix.result == 'success'
)
)
needs.build-cu130-matrix.result == 'success'
runs-on: ubuntu-latest
steps:
- name: Checkout SGLang
@@ -209,14 +124,8 @@ jobs:
git clone --branch gh-pages \
"https://oauth2:${WHL_TOKEN}@github.com/sgl-project/whl.git" sgl-whl
- name: Update wheel indexes
run: |
if [[ "${{ inputs.target }}" == "all" || "${{ inputs.target }}" == "cu129" ]]; then
python3 scripts/update_deepep_whl_index.py --cuda 129
fi
if [[ "${{ inputs.target }}" == "all" || "${{ inputs.target }}" == "cu130" ]]; then
python3 scripts/update_deepep_whl_index.py --cuda 130
fi
- name: Update wheel index
run: python3 scripts/update_deepep_whl_index.py --cuda 130
- name: Push wheel indexes
env:
@@ -234,15 +143,10 @@ jobs:
fi
release-pypi:
needs: [build-cu129-matrix, build-cu130-matrix]
needs: build-cu130-matrix
if: |
always() &&
github.repository == 'sgl-project/sglang' &&
needs.build-cu130-matrix.result == 'success' &&
(
inputs.target == 'cu130' ||
(inputs.target == 'all' && needs.build-cu129-matrix.result == 'success')
)
needs.build-cu130-matrix.result == 'success'
runs-on: ubuntu-latest
steps:
- name: Download PyPI wheels
.github/workflows/release-whl-deepgemm.yml
-87
View File
@@ -14,7 +14,6 @@ on:
default: 'all'
options:
- 'all'
- 'cu129'
- 'cu130'
branch:
description: "DeepGEMM branch to build from (default: dev)"
@@ -33,92 +32,6 @@ env:
TORCH_VER: "2.13.0"
jobs:
build-cu129-matrix:
if: |
github.repository == 'sgl-project/sglang' &&
(github.event.inputs.target == 'all' || github.event.inputs.target == 'cu129')
strategy:
matrix:
python-version: ["3.12"]
cuda-version: ["12.9"]
arch: [x86_64, aarch64]
include:
- arch: x86_64
runner: x64-kernel-build-node
- arch: aarch64
runner: arm-kernel-build-node
runs-on: ${{ matrix.runner }}
steps:
- name: Clean workspace (remove root-owned files from prior runs)
run: |
docker run --rm -v "${{ github.workspace }}:/workspace" alpine:3 \
sh -c 'rm -rf /workspace/..?* /workspace/.[!.]* /workspace/*' || true
- uses: actions/checkout@v4
- name: Checkout DeepGEMM
uses: actions/checkout@v4
with:
repository: sgl-project/DeepGEMM
ref: ${{ inputs.branch || 'dev' }}
path: DeepGEMM
submodules: recursive
- name: Set wheel version
run: |
echo -n "${{ inputs.version }}" > DeepGEMM/sgl_deep_gemm/VERSION
cat DeepGEMM/sgl_deep_gemm/VERSION
- name: Build wheel
run: |
chmod +x ./scripts/build_sgl_deep_gemm.sh ./scripts/rename_sgl_deep_gemm_whl.sh
./scripts/build_sgl_deep_gemm.sh "${{ matrix.python-version }}" "${{ matrix.cuda-version }}" "${{ github.workspace }}/DeepGEMM" "${{ matrix.arch }}"
- name: Upload artifacts
uses: actions/upload-artifact@v4
with:
name: deepgemm-wheel-cuda${{ matrix.cuda-version }}-${{ matrix.arch }}
path: DeepGEMM/dist/*.whl
release-cu129:
needs: build-cu129-matrix
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- name: Download artifacts
uses: actions/download-artifact@v4
with:
path: dist/
merge-multiple: true
pattern: deepgemm-wheel-cuda12.9-*
- name: Release
uses: softprops/action-gh-release@v2
with:
tag_name: v${{ inputs.version }}
repository: sgl-project/whl
token: ${{ secrets.GH_PAT_FOR_WHL_RELEASE }}
files: |
dist/*
- name: Clone wheel index
run: git clone https://oauth2:${WHL_TOKEN}@github.com/sgl-project/whl.git sgl-whl
env:
WHL_TOKEN: ${{ secrets.GH_PAT_FOR_WHL_RELEASE }}
- name: Update wheel index
run: python3 scripts/update_deepgemm_whl_index.py --cuda 129
- name: Push wheel index
run: |
cd sgl-whl
git config --local user.name "sglang-bot"
git config --local user.email "sglangbot@gmail.com"
git add -A
git commit -m "update sgl-deep-gemm whl index for v${{ inputs.version }}"
git push
build-cu130-matrix:
if: |
github.repository == 'sgl-project/sglang' &&
.github/workflows/release-whl-kernel.yml
-105
View File
@@ -15,7 +15,6 @@ on:
default: 'all'
options:
- 'all'
- 'cu129'
- 'cu130'
- 'rocm700'
- 'rocm720'
@@ -35,110 +34,6 @@ concurrency:
cancel-in-progress: true
jobs:
# cu130 is the PyPI-released variant; cu129 wheels are published only to the
# sgl-project/whl index (consumed via `pip install ...+cu129` for the legacy
# cuda 12.9 path), not to PyPI.
build-cu129-matrix:
if: |
github.repository == 'sgl-project/sglang' &&
(github.event_name == 'push' || github.event.inputs.target == 'all' || github.event.inputs.target == 'cu129')
strategy:
matrix:
python-version: ["3.10"]
cuda-version: ["12.9"]
arch: [x86_64, aarch64]
include:
- arch: x86_64
runner: x64-kernel-build-node
- arch: aarch64
runner: arm-kernel-build-node
runs-on: ${{ matrix.runner }}
steps:
# Self-hosted build nodes retain the workspace across jobs. Prior builds
# leave root-owned artifacts under python/sglang/kernels/aot/build/ that actions/checkout
# cannot remove, causing EACCES on rmdir. Wipe them via a throwaway root
# container before checkout recreates the workspace.
- name: Clean workspace (remove root-owned files from prior runs)
run: |
docker run --rm -v "${{ github.workspace }}:/workspace" alpine:3 \
sh -c 'rm -rf /workspace/..?* /workspace/.[!.]* /workspace/*' || true
- uses: actions/checkout@v4
with:
submodules: "recursive"
ref: ${{ inputs.pr_number && format('refs/pull/{0}/head', inputs.pr_number) || '' }}
- name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@v5
with:
python-version: ${{ matrix.python-version }}
- name: Build wheels
run: |
cd python/sglang/kernels/aot
chmod +x ./build.sh
./build.sh "${{ matrix.python-version }}" "${{ matrix.cuda-version }}" ${{ matrix.arch == 'aarch64' && 'aarch64' || '' }}
env:
BUILD_JOBS: 64
NVCC_THREADS: 8
- name: Upload artifacts
uses: actions/upload-artifact@v4
with:
name: wheel-python${{ matrix.python-version }}-cuda${{ matrix.cuda-version }}${{ matrix.arch == 'aarch64' && '-aarch64' || '' }}
path: python/sglang/kernels/aot/dist/*
release-cu129:
needs: build-cu129-matrix
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
with:
ref: ${{ inputs.pr_number && format('refs/pull/{0}/head', inputs.pr_number) || '' }}
- name: Download artifacts
uses: actions/download-artifact@v4
with:
path: python/sglang/kernels/aot/dist/
merge-multiple: true
pattern: wheel-*
- name: Set tag name
id: set_tag_name
run: |
if [ -z "${{ inputs.tag_name }}" ]; then
TAG_NAME="v$(cat python/sglang/kernels/aot/python/sgl_kernel/version.py | cut -d'"' -f2)"
echo "tag_name=$TAG_NAME" >> $GITHUB_OUTPUT
else
echo "tag_name=${{ inputs.tag_name }}" >> $GITHUB_OUTPUT
fi
- name: Release
uses: softprops/action-gh-release@v2
with:
tag_name: ${{ steps.set_tag_name.outputs.tag_name }}
repository: sgl-project/whl
token: ${{ secrets.GH_PAT_FOR_WHL_RELEASE }}
files: |
python/sglang/kernels/aot/dist/*
- name: Clone wheel index
run: git clone https://oauth2:${WHL_TOKEN}@github.com/sgl-project/whl.git sgl-whl
env:
WHL_TOKEN: ${{ secrets.GH_PAT_FOR_WHL_RELEASE }}
- name: Update wheel index
run: python3 scripts/update_kernel_whl_index.py --cuda 129
- name: Push wheel index
run: |
cd sgl-whl
git config --local user.name "sglang-bot"
git config --local user.email "sglangbot@gmail.com"
git add -A
git commit -m "update whl index"
git push
build-cu130-matrix:
if: |
github.repository == 'sgl-project/sglang' &&
.github/workflows/retag-docker.yml
+1 -1
View File
@@ -4,7 +4,7 @@ on:
workflow_dispatch:
inputs:
source_tag:
description: "Existing image tag (e.g., v0.4.7-cu129-amd64)"
description: "Existing image tag (e.g., v0.4.7-cu130-amd64)"
required: true
target_tag:
description: "New tag to apply (e.g., latest)"
.github/workflows/trivy-scan-dev.yml
+1 -1
View File
@@ -22,7 +22,7 @@ jobs:
strategy:
fail-fast: false
matrix:
tag: ${{ inputs.tag && fromJSON(format('["{0}"]', inputs.tag)) || fromJSON('["dev", "dev-cu12"]') }}
tag: ${{ inputs.tag && fromJSON(format('["{0}"]', inputs.tag)) || fromJSON('["dev", "dev-cu13"]') }}
steps:
- name: Cleanup workspace (remove root-owned files from prior runs)
run: sudo rm -rf "$GITHUB_WORKSPACE"/* || true
docker/Dockerfile
+15 -77
View File
@@ -6,7 +6,6 @@ ARG BUILD_TYPE=all
ARG BRANCH_TYPE=remote
ARG SGL_KERNEL_VERSION=0.4.6.post1
ARG SGL_VERSION
ARG SGL_DEEP_GEMM_VERSION=0.1.7
ARG USE_LATEST_SGLANG=0
ARG GDRCOPY_VERSION=2.5.1
ARG SGL_NCCL_VERSION=2.30.7
@@ -185,25 +184,11 @@ RUN curl --proto '=https' --tlsv1.2 --retry 3 --retry-delay 2 -sSf https://sh.ru
RUN --mount=type=cache,target=/root/.cache/pip \
python3 -m pip install --upgrade pip setuptools wheel html5lib six \
&& case "$CUDA_VERSION" in \
12.6.3) CUINDEX=126 ;; \
12.9.2) CUINDEX=129 ;; \
13.0.3) CUINDEX=130 ;; \
13.0.3) ;; \
*) echo "Unsupported CUDA version: $CUDA_VERSION" && exit 1 ;; \
esac \
&& if [ "$CUDA_VERSION" = "12.6.3" ]; then \
python3 -m pip install https://${GITHUB_ARTIFACTORY}/sgl-project/whl/releases/download/v${SGL_KERNEL_VERSION}/sglang_kernel-${SGL_KERNEL_VERSION}+cu124-cp310-abi3-manylinux2014_$(uname -m).whl --force-reinstall --no-deps \
; \
elif [ "$CUDA_VERSION" = "12.9.2" ]; then \
python3 -m pip install https://${GITHUB_ARTIFACTORY}/sgl-project/whl/releases/download/v${SGL_KERNEL_VERSION}/sglang_kernel-${SGL_KERNEL_VERSION}+cu129-cp310-abi3-manylinux2014_$(uname -m).whl --force-reinstall --no-deps \
; \
elif [ "$CUDA_VERSION" = "13.0.3" ]; then \
# --no-deps prevents pip from pulling torch from default PyPI
python3 -m pip install sglang-kernel==${SGL_KERNEL_VERSION} --force-reinstall --no-deps \
; \
else \
echo "Unsupported CUDA version: $CUDA_VERSION" && exit 1 \
; \
fi
# --no-deps prevents pip from pulling torch from default PyPI
&& python3 -m pip install sglang-kernel==${SGL_KERNEL_VERSION} --force-reinstall --no-deps
# Copy dep spec + Rust crate source + proto files. setuptools-rust compiles the
# Rust extension during the stub wheel build; the crate's build.rs references
@@ -214,15 +199,11 @@ COPY rust/sglang-grpc /tmp/sglang_deps/rust/sglang-grpc
COPY rust/sglang-mm /tmp/sglang_deps/rust/sglang-mm
COPY proto /tmp/sglang_deps/proto
# Install sglang dependencies (torch, transformers, etc.). CUDA 12 DeepEP
# wheels live only on the SGLang index, so preinstall the local-version wheel;
# it satisfies the public-version pyproject pin during the full dependency solve.
# Install sglang dependencies (torch, transformers, etc.).
# Generate constraints.txt to prevent reinstalling these deps in later stages.
RUN --mount=type=cache,target=/root/.cache/pip \
--mount=type=cache,target=/root/.cargo/registry \
case "$CUDA_VERSION" in \
12.6.3) CUINDEX=126 ;; \
12.9.2) CUINDEX=129 ;; \
13.0.3) CUINDEX=130 ;; \
*) echo "Unsupported CUDA version: $CUDA_VERSION" && exit 1 ;; \
esac \
@@ -232,27 +213,7 @@ RUN --mount=type=cache,target=/root/.cache/pip \
&& echo '__version__ = "0.0.0"' > sglang/version.py \
&& touch README.md \
&& touch LICENSE \
&& SGL_DEEP_EP_VERSION="$(sed -n 's/^[[:space:]]*"sgl-deep-ep==\([^"]*\)",/\1/p' pyproject.toml)" \
&& test -n "${SGL_DEEP_EP_VERSION}" \
&& if [ "${CUDA_VERSION%%.*}" = "12" ]; then \
python3 -m pip install \
"sgl-deep-ep==${SGL_DEEP_EP_VERSION}+cu129" \
--index-url "https://docs.sglang.ai/whl/cu129/" \
--no-deps; \
fi \
&& if [ "${CUDA_VERSION%%.*}" = "12" ]; then \
sed -i 's/cuda-python>=13\.0/cuda-python>=12,<13/' pyproject.toml && \
sed -i 's/flashinfer_python\[cu13\]/flashinfer_python[cu12]/' pyproject.toml && \
sed -i 's/nvidia-cutlass-dsl\[cu13\]/nvidia-cutlass-dsl/' pyproject.toml; \
fi \
&& python3 -m pip install --extra-index-url https://download.pytorch.org/whl/cu${CUINDEX} ".[${BUILD_TYPE}]" \
&& if [ "${CUDA_VERSION%%.*}" = "12" ]; then \
pip list --format=freeze | awk -F'==' '/-cu13(==|$)/ {print $1}' \
| xargs -r python3 -m pip uninstall -y && \
python3 -m pip install --index-url https://download.pytorch.org/whl/cu${CUINDEX} \
torch==2.13.0 torchvision==0.28.0 torchaudio==2.11.0 --force-reinstall; \
python3 -m pip install https://github.com/sgl-project/whl/releases/download/v${SGL_DEEP_GEMM_VERSION}/sgl_deep_gemm-${SGL_DEEP_GEMM_VERSION}+cu129-py3-none-manylinux2014_$(uname -m).whl --force-reinstall; \
fi \
&& cd /sgl-workspace \
&& rm -rf /tmp/sglang_deps \
&& pip freeze | grep -v "^sglang==" > /sgl-workspace/constraints.txt
@@ -315,8 +276,6 @@ ARG FLASHINFER_VERSION
# Stage jit-cache/cubin artifacts into /flashinfer_jit_output for clean COPY later
RUN --mount=type=cache,target=/root/.cache/pip \
case "$CUDA_VERSION" in \
12.6.3) CUINDEX=126 ;; \
12.9.2) CUINDEX=129 ;; \
13.0.3) CUINDEX=130 ;; \
*) echo "Unsupported CUDA version: $CUDA_VERSION" && exit 1 ;; \
esac \
@@ -503,21 +462,12 @@ RUN --mount=type=cache,target=/var/cache/apt,id=framework-apt \
# Install Mooncake
RUN --mount=type=cache,target=/root/.cache/pip \
CUDA_MAJOR="${CUDA_VERSION%%.*}" && \
if [ "$CUDA_MAJOR" -ge 13 ]; then \
python3 -m pip install mooncake-transfer-engine-cuda13==${MOONCAKE_VERSION}; \
else \
python3 -m pip install mooncake-transfer-engine==${MOONCAKE_VERSION}; \
fi
python3 -m pip install mooncake-transfer-engine-cuda13==${MOONCAKE_VERSION}
# Install MSCCL++ Python dependencies and package (builds extension via CMake through pip)
RUN --mount=type=cache,target=/root/.cache/pip \
git clone --depth=1 --branch ${MSCCLPP_VERSION} https://${GITHUB_ARTIFACTORY}/microsoft/mscclpp.git /tmp/mscclpp \
&& case "${CUDA_VERSION}" in \
12.*) \
CMAKE_ARGS="-DMSCCLPP_BYPASS_GPU_CHECK=ON -DMSCCLPP_USE_CUDA=ON -DMSCCLPP_GPU_ARCHS=80,90,100,100a,103,103a" \
python3 -m pip install "/tmp/mscclpp[cuda12]"; \
;; \
13.*) \
CMAKE_ARGS="-DMSCCLPP_BYPASS_GPU_CHECK=ON -DMSCCLPP_USE_CUDA=ON -DMSCCLPP_GPU_ARCHS=80,90,100,100a,103,103a" \
python3 -m pip install "/tmp/mscclpp[cuda13]"; \
@@ -553,22 +503,15 @@ RUN --mount=type=cache,target=/root/.cache/pip \
# resolving dependencies and generating constraints.txt. DeepEP needs 2.30.7
# at runtime; apply the override only after the constrained dependency solve.
RUN --mount=type=cache,target=/root/.cache/pip \
if [ "${CUDA_VERSION%%.*}" = "13" ]; then \
python3 -m pip install --force-reinstall --no-deps \
"nvidia-nccl-cu13==${SGL_NCCL_VERSION}"; \
fi
python3 -m pip install --force-reinstall --no-deps \
"nvidia-nccl-cu13==${SGL_NCCL_VERSION}"
# Per-CUDA-major package installs. The `nixl` stub package is needed (it owns
# the `nixl` import path) but unconditionally requires nixl-cu12, so we install
# it with --no-deps and pair it with the matching nixl-cu12 / nixl-cu13 binary
# to avoid shipping wrong-CUDA libs on cu13 images.
RUN --mount=type=cache,target=/root/.cache/pip if [ "${CUDA_VERSION%%.*}" = "12" ]; then \
python3 -m pip install nixl nixl-cu12 --no-deps ; \
python3 -m pip install "cuda-python>=12,<13" ; \
elif [ "${CUDA_VERSION%%.*}" = "13" ]; then \
python3 -m pip install nixl nixl-cu13 --no-deps ; \
python3 -m pip install "cuda-python>=13,<14" ; \
fi
# The `nixl` stub package is needed (it owns the `nixl` import path) but
# unconditionally requires nixl-cu12, so install it with --no-deps and pair it
# with the nixl-cu13 binary to avoid shipping wrong-CUDA libs.
RUN --mount=type=cache,target=/root/.cache/pip \
python3 -m pip install nixl nixl-cu13 --no-deps \
&& python3 -m pip install "cuda-python>=13,<14"
# Install the latest available AI Dynamo prerelease from NVIDIA's package index.
RUN --mount=type=cache,target=/root/.cache/pip \
@@ -638,11 +581,6 @@ RUN if [ "$BRANCH_TYPE" = "local" ]; then \
# Clean up __pycache__/tests/pyc in same RUN to avoid writing ~28k files to layer
RUN --mount=type=cache,target=/root/.cache/pip \
cd /sgl-workspace/sglang \
&& if [ "${CUDA_VERSION%%.*}" = "12" ]; then \
sed -i 's/cuda-python>=13\.0/cuda-python>=12,<13/' python/pyproject.toml && \
sed -i 's/flashinfer_python\[cu13\]/flashinfer_python[cu12]/' python/pyproject.toml && \
sed -i 's/nvidia-cutlass-dsl\[cu13\]/nvidia-cutlass-dsl/' python/pyproject.toml; \
fi \
&& python3 -m pip install --no-deps -e "python[${BUILD_TYPE}]" \
&& kernels lock python \
&& ( success=0; \
@@ -659,8 +597,8 @@ RUN --mount=type=cache,target=/root/.cache/pip \
echo "sgl-kernel cubin download failed, retrying in 30s..." && sleep 30; \
done; \
# x86: if no prebuilt sgl-flash-attn3 variant matches this torch+CUDA \
# combo (e.g. cu129 publishes no torch>=2.10 cubin), fall back to \
# runtime JIT instead of failing the build, mirroring the aarch64 branch. \
# combo, fall back to runtime JIT instead of failing the build, \
# mirroring the aarch64 branch. \
if [ "$success" != "1" ]; then \
echo "WARNING: no matching sgl-flash-attn3 cubin variant for this torch+CUDA; kernels will be JIT-compiled at runtime"; \
success=1; \
docker/Dockerfile.cu134
+13 -68
View File
@@ -493,19 +493,11 @@ RUN curl --proto '=https' --tlsv1.2 --retry 3 --retry-delay 2 -sSf https://sh.ru
RUN --mount=type=cache,target=/root/.cache/pip \
python3 -m pip install --upgrade pip setuptools wheel html5lib six \
&& case "$CUDA_VERSION" in \
12.6.3) CUINDEX=126 ;; \
12.9.2) CUINDEX=129 ;; \
13.0.3) CUINDEX=130 ;; \
13.4.0) CUINDEX=134 ;; \
*) echo "Unsupported CUDA version: $CUDA_VERSION" && exit 1 ;; \
esac \
&& if [ "$CUDA_VERSION" = "12.6.3" ]; then \
python3 -m pip install https://${GITHUB_ARTIFACTORY}/sgl-project/whl/releases/download/v${SGL_KERNEL_VERSION}/sglang_kernel-${SGL_KERNEL_VERSION}+cu124-cp310-abi3-manylinux2014_$(uname -m).whl --force-reinstall --no-deps \
; \
elif [ "$CUDA_VERSION" = "12.9.2" ]; then \
python3 -m pip install https://${GITHUB_ARTIFACTORY}/sgl-project/whl/releases/download/v${SGL_KERNEL_VERSION}/sglang_kernel-${SGL_KERNEL_VERSION}+cu129-cp310-abi3-manylinux2014_$(uname -m).whl --force-reinstall --no-deps \
; \
elif [ "$CUDA_VERSION" = "13.0.3" ]; then \
&& if [ "$CUDA_VERSION" = "13.0.3" ]; then \
# --no-deps prevents pip from pulling torch from default PyPI
python3 -m pip install sglang-kernel==${SGL_KERNEL_VERSION} --force-reinstall --no-deps \
; \
@@ -527,15 +519,11 @@ COPY rust/sglang-grpc /tmp/sglang_deps/rust/sglang-grpc
COPY rust/sglang-mm /tmp/sglang_deps/rust/sglang-mm
COPY proto /tmp/sglang_deps/proto
# Install sglang dependencies (torch, transformers, etc.). CUDA 12 DeepEP
# wheels live only on the SGLang index, so preinstall the local-version wheel;
# it satisfies the public-version pyproject pin during the full dependency solve.
# Install sglang dependencies (torch, transformers, etc.).
# Generate constraints.txt to prevent reinstalling these deps in later stages.
RUN --mount=type=cache,target=/root/.cache/pip \
--mount=type=cache,target=/root/.cargo/registry \
case "$CUDA_VERSION" in \
12.6.3) CUINDEX=126 ;; \
12.9.2) CUINDEX=129 ;; \
13.0.3) CUINDEX=130 ;; \
13.4.0) CUINDEX=134 ;; \
*) echo "Unsupported CUDA version: $CUDA_VERSION" && exit 1 ;; \
@@ -546,19 +534,6 @@ RUN --mount=type=cache,target=/root/.cache/pip \
&& echo '__version__ = "0.0.0"' > sglang/version.py \
&& touch README.md \
&& touch LICENSE \
&& SGL_DEEP_EP_VERSION="$(sed -n 's/^[[:space:]]*"sgl-deep-ep==\([^"]*\)",/\1/p' pyproject.toml)" \
&& test -n "${SGL_DEEP_EP_VERSION}" \
&& if [ "${CUDA_VERSION%%.*}" = "12" ]; then \
python3 -m pip install \
"sgl-deep-ep==${SGL_DEEP_EP_VERSION}+cu129" \
--index-url "https://docs.sglang.ai/whl/cu129/" \
--no-deps; \
fi \
&& if [ "${CUDA_VERSION%%.*}" = "12" ]; then \
sed -i 's/cuda-python>=13\.0/cuda-python>=12,<13/' pyproject.toml && \
sed -i 's/flashinfer_python\[cu13\]/flashinfer_python[cu12]/' pyproject.toml && \
sed -i 's/nvidia-cutlass-dsl\[cu13\]/nvidia-cutlass-dsl/' pyproject.toml; \
fi \
&& if [ "$CUDA_VERSION" = "13.4.0" ]; then \
sed -i "s|^ \"flashinfer_python\[cu13\]==.*| \"flashinfer_python[cu13]==${FLASHINFER_VERSION}\",|" pyproject.toml; \
fi \
@@ -579,17 +554,8 @@ RUN --mount=type=cache,target=/root/.cache/pip \
TORCH_INDEX="https://download.pytorch.org/whl/cu${CUINDEX}"; \
fi \
&& python3 -m pip install --extra-index-url "${TORCH_INDEX}" ".[${BUILD_TYPE}]" \
&& if [ "${CUDA_VERSION%%.*}" = "13" ]; then \
python3 -m pip install --force-reinstall --no-deps \
"nvidia-nccl-cu13==${NCCL_VERSION}"; \
fi \
&& if [ "${CUDA_VERSION%%.*}" = "12" ]; then \
pip list --format=freeze | awk -F'==' '/-cu13(==|$)/ {print $1}' \
| xargs -r python3 -m pip uninstall -y && \
python3 -m pip install --index-url https://download.pytorch.org/whl/cu${CUINDEX} \
torch==2.13.0 torchvision==0.28.0 torchaudio==2.11.0 --force-reinstall; \
python3 -m pip install https://github.com/sgl-project/whl/releases/download/v${SGL_DEEP_GEMM_VERSION}/sgl_deep_gemm-${SGL_DEEP_GEMM_VERSION}+cu129-py3-none-manylinux2014_$(uname -m).whl --force-reinstall; \
fi \
&& python3 -m pip install --force-reinstall --no-deps \
"nvidia-nccl-cu13==${NCCL_VERSION}" \
&& if [ "$CUDA_VERSION" = "13.4.0" ]; then \
python3 -m pip install --force-reinstall --no-deps /tmp/cu134_wheels/*.whl; \
fi \
@@ -643,8 +609,6 @@ ARG FLASHINFER_VERSION
# Stage jit-cache/cubin artifacts into /flashinfer_jit_output for clean COPY later
RUN --mount=type=cache,target=/root/.cache/pip \
case "$CUDA_VERSION" in \
12.6.3) CUINDEX=126 ;; \
12.9.2) CUINDEX=129 ;; \
13.0.3) CUINDEX=130 ;; \
13.4.0) CUINDEX=134 ;; \
*) echo "Unsupported CUDA version: $CUDA_VERSION" && exit 1 ;; \
@@ -830,21 +794,12 @@ RUN --mount=type=cache,target=/var/cache/apt,id=framework-apt \
# Install Mooncake
RUN --mount=type=cache,target=/root/.cache/pip \
CUDA_MAJOR="${CUDA_VERSION%%.*}" && \
if [ "$CUDA_MAJOR" -ge 13 ]; then \
python3 -m pip install mooncake-transfer-engine-cuda13==${MOONCAKE_VERSION}; \
else \
python3 -m pip install mooncake-transfer-engine==${MOONCAKE_VERSION}; \
fi
python3 -m pip install mooncake-transfer-engine-cuda13==${MOONCAKE_VERSION}
# Install MSCCL++ Python dependencies and package (builds extension via CMake through pip)
RUN --mount=type=cache,target=/root/.cache/pip \
git clone --depth=1 --branch ${MSCCLPP_VERSION} https://${GITHUB_ARTIFACTORY}/microsoft/mscclpp.git /tmp/mscclpp \
&& case "${CUDA_VERSION}" in \
12.*) \
CMAKE_ARGS="-DMSCCLPP_BYPASS_GPU_CHECK=ON -DMSCCLPP_USE_CUDA=ON -DMSCCLPP_GPU_ARCHS=80,90,100,100a,103,103a" \
python3 -m pip install "/tmp/mscclpp[cuda12]"; \
;; \
13.*) \
CMAKE_ARGS="-DMSCCLPP_BYPASS_GPU_CHECK=ON -DMSCCLPP_USE_CUDA=ON -DMSCCLPP_GPU_ARCHS=80,90,100,100a,103,103a" \
python3 -m pip install "/tmp/mscclpp[cuda13]"; \
@@ -882,17 +837,12 @@ RUN --mount=type=cache,target=/root/.cache/pip \
termplotlib \
"runai-model-streamer[s3,gcs,azure]>=0.15.7"
# Per-CUDA-major package installs. The `nixl` stub package is needed (it owns
# the `nixl` import path) but unconditionally requires nixl-cu12, so we install
# it with --no-deps and pair it with the matching nixl-cu12 / nixl-cu13 binary
# to avoid shipping wrong-CUDA libs on cu13 images.
RUN --mount=type=cache,target=/root/.cache/pip if [ "${CUDA_VERSION%%.*}" = "12" ]; then \
python3 -m pip install nixl nixl-cu12 --no-deps ; \
python3 -m pip install "cuda-python>=12,<13" ; \
elif [ "${CUDA_VERSION%%.*}" = "13" ]; then \
python3 -m pip install nixl nixl-cu13 --no-deps ; \
python3 -m pip install "cuda-python>=13,<14" ; \
fi
# The `nixl` stub package is needed (it owns the `nixl` import path) but
# unconditionally requires nixl-cu12, so install it with --no-deps and pair it
# with the nixl-cu13 binary to avoid shipping wrong-CUDA libs.
RUN --mount=type=cache,target=/root/.cache/pip \
python3 -m pip install nixl nixl-cu13 --no-deps \
&& python3 -m pip install "cuda-python>=13,<14"
# Install the latest available AI Dynamo prerelease from NVIDIA's package index.
RUN --mount=type=cache,target=/root/.cache/pip \
@@ -965,11 +915,6 @@ RUN if [ "$BRANCH_TYPE" = "local" ]; then \
# Clean up __pycache__/tests/pyc in same RUN to avoid writing ~28k files to layer
RUN --mount=type=cache,target=/root/.cache/pip \
cd /sgl-workspace/sglang \
&& if [ "${CUDA_VERSION%%.*}" = "12" ]; then \
sed -i 's/cuda-python>=13\.0/cuda-python>=12,<13/' python/pyproject.toml && \
sed -i 's/flashinfer_python\[cu13\]/flashinfer_python[cu12]/' python/pyproject.toml && \
sed -i 's/nvidia-cutlass-dsl\[cu13\]/nvidia-cutlass-dsl/' python/pyproject.toml; \
fi \
&& if [ "$CUDA_VERSION" = "13.4.0" ]; then \
sed -i "s|^ \"flashinfer_python\[cu13\]==.*| \"flashinfer_python[cu13]==${FLASHINFER_VERSION}\",|" python/pyproject.toml; \
fi \
@@ -989,8 +934,8 @@ RUN --mount=type=cache,target=/root/.cache/pip \
echo "sgl-kernel cubin download failed, retrying in 30s..." && sleep 30; \
done; \
# x86: if no prebuilt sgl-flash-attn3 variant matches this torch+CUDA \
# combo (e.g. cu129 publishes no torch>=2.10 cubin), fall back to \
# runtime JIT instead of failing the build, mirroring the aarch64 branch. \
# combo, fall back to runtime JIT instead of failing the build, \
# mirroring the aarch64 branch. \
if [ "$success" != "1" ]; then \
echo "WARNING: no matching sgl-flash-attn3 cubin variant for this torch+CUDA; kernels will be JIT-compiled at runtime"; \
success=1; \
docker/sgl-deep-ep.Dockerfile
+2 -18
View File
@@ -6,7 +6,6 @@ FROM ${BASE_IMAGE}:cuda${CUDA_VERSION}
ARG ARCHITECTURE=x86_64
ARG CUDA_TAG=cu130
ARG CUDA_VERSION=13.0
ARG GDRCOPY_VERSION=2.5.1
ARG NCCL_VERSION=2.30.7
ARG PYTHON_TAG=cp312-cp312
ARG TORCH_VERSION=2.13.0
@@ -49,19 +48,6 @@ RUN set -eux; \
ln -sf "${cuda_stub}" /usr/lib64/libcuda.so; \
ln -sf "${cuda_stub}" "/usr/lib/${ARCHITECTURE}-linux-gnu/libcuda.so"
# DeepEP v2 uses NCCL Gin on CUDA 13. Keep GDRCopy only for the CUDA 12
# legacy NVSHMEM/IBGDA build.
RUN set -eux; \
if [ "${CUDA_TAG}" = cu129 ]; then \
git clone --depth 1 --branch "v${GDRCOPY_VERSION}" \
https://github.com/NVIDIA/gdrcopy.git /opt/gdrcopy; \
make -C /opt/gdrcopy CUDA="${CUDA_HOME}" prefix=/usr/local lib_install; \
printf '%s\n' /usr/local/lib > /etc/ld.so.conf.d/gdrcopy.conf; \
ldconfig; \
test -f /usr/local/include/gdrapi.h; \
ldconfig -p | grep -q libgdrapi; \
fi
RUN --mount=type=cache,id=sgl-deep-ep-pip-${CUDA_TAG}-${PYTHON_TAG}-${ARCHITECTURE},target=/root/.cache/pip \
set -eux; \
"${PYTHON_BIN}" -m pip uninstall -y deep-ep sgl-deep-ep || true; \
@@ -69,10 +55,8 @@ RUN --mount=type=cache,id=sgl-deep-ep-pip-${CUDA_TAG}-${PYTHON_TAG}-${ARCHITECTU
"${PYTHON_BIN}" -m pip install --force-reinstall \
"torch==${TORCH_VERSION}" \
--index-url "https://download.pytorch.org/whl/${CUDA_TAG}"; \
if [ "${CUDA_TAG}" = cu130 ]; then \
"${PYTHON_BIN}" -m pip install --force-reinstall --no-deps \
"nvidia-nccl-cu13==${NCCL_VERSION}"; \
fi; \
"${PYTHON_BIN}" -m pip install --force-reinstall --no-deps \
"nvidia-nccl-cu13==${NCCL_VERSION}"; \
"${PYTHON_BIN}" -m pip install \
"auditwheel>=6.0" \
build \
docker/sgl-deep-gemm.Dockerfile
+1 -2
View File
@@ -27,8 +27,7 @@ RUN --mount=type=cache,id=sgl-deep-gemm-pip,target=/root/.cache/pip \
set -eux; \
case "${CUDA_VERSION}" in \
13.0) CU_TAG=cu130 ;; \
12.9) CU_TAG=cu129 ;; \
*) CU_TAG=cu130 ;; \
*) echo "Unsupported CUDA version: ${CUDA_VERSION}" && exit 1 ;; \
esac; \
${PYTHON_ROOT_PATH}/bin/pip install torch==${TORCH_VER} --index-url https://${PYTORCH_MIRROR}/whl/${CU_TAG}; \
${PYTHON_ROOT_PATH}/bin/pip install --index-url ${PIP_DEFAULT_INDEX} \
docs/cookbook/autoregressive/MiniMax/MiniMax-M3.mdx
+1 -1
View File
@@ -39,7 +39,7 @@ Then run the **Python** output of the command panel below in that environment. T
docker pull lmsysorg/sglang:dev-cu13-minimax-m3
```
The command panel below fills in the right tag per platform: `dev-cu13-minimax-m3` (CUDA 13 — B300, GB200, GB300), `dev-cu12-minimax-m3` (CUDA 12 — Hopper H200), or `dev-minimax-m3` (default). On AMD Instinct it uses the matching ROCm image (MI300X/MI325X → `aigmkt/minimax-m3-sglang-rocm700-mi30x`, MI350X/MI355X → `aigmkt/minimax-m3-sglang-rocm720-mi35x`). For how to launch the image, see [Install → Method 3: Using Docker](../../../docs/get-started/install#method-3-using-docker), substituting the inner `sglang serve ...` with what the command generator produces.
The command panel below fills in the right tag per platform: `dev-cu13-minimax-m3` (CUDA 13 — B300, GB200, GB300), `dev-cu12-minimax-m3` (CUDA 12 — Hopper H200; final build, the CUDA 12 lane is retired), or `dev-minimax-m3` (default). On AMD Instinct it uses the matching ROCm image (MI300X/MI325X → `aigmkt/minimax-m3-sglang-rocm700-mi30x`, MI350X/MI355X → `aigmkt/minimax-m3-sglang-rocm720-mi35x`). For how to launch the image, see [Install → Method 3: Using Docker](../../../docs/get-started/install#method-3-using-docker), substituting the inner `sglang serve ...` with what the command generator produces.
<Note>
These M3 dev images now **bundle MiniMax's MSA sparse-attention kernel** (`fmha_sm100`), so Blackwell users get the recommended fast path automatically — no manual install needed (see **§2.1**). On a custom image without it, the same recipe still serves on the built-in Triton sparse path.
docs/cookbook/autoregressive/OpenBMB/MiniCPM-V-4_6.mdx
+2 -1
View File
@@ -32,7 +32,8 @@ Pull the nightly Docker image (rolling tag, tracks `main`):
# CUDA 13 (Hopper / Blackwell, default)
docker pull lmsysorg/sglang:dev
# CUDA 12 (Ampere or older drivers)
# CUDA 12 (Ampere or older drivers). Final build: the CUDA 12 lane is retired,
# so this tag no longer tracks main.
docker pull lmsysorg/sglang:dev-cu12
```
docs/cookbook/autoregressive/ThinkingMachines/Inkling-Small.mdx
+2 -2
View File
@@ -35,7 +35,7 @@ There are two multi-arch (amd64 / arm64) CUDA builds plus a ROCm build; pick the
```bash Command
docker pull lmsysorg/sglang:dev-inkling-dspark # CUDA 13
docker pull lmsysorg/sglang:dev-cu12-inkling-dspark # CUDA 12
docker pull lmsysorg/sglang:dev-cu12-inkling-dspark # CUDA 12 (final build; lane retired)
docker pull lmsysorg/sglang:dev-inkling-small-dgx-spark # DGX Spark (GB10 / SM121)
docker pull lmsysorg/sglang-rocm:dev-rocm720-mi35x-inkling-dspark # AMD MI350X / MI355X
```
@@ -309,4 +309,4 @@ To try it, select the **Long Context** strategy in the Deploy panel above for an
The **DSpark** deploy strategy is the second speculative-decoding path for Inkling-Small. Unlike **MTP**, which drives Inkling-Small's own multi-layer draft head, DSpark runs a **separate draft checkpoint** — `RadixArk/Inkling-Small-DSpark` — served unquantized alongside the NVFP4 target.
DSpark support ships in the images listed in §1 (`dev-inkling-dspark` for CUDA 13, `dev-cu12-inkling-dspark` for CUDA 12), so no separate build is needed. Verified end-to-end on B200 (TP=8, NVFP4).
DSpark support ships in the images listed in §1 (`dev-inkling-dspark` for CUDA 13, `dev-cu12-inkling-dspark` for CUDA 12 — a final build, since that lane is retired), so no separate build is needed. Verified end-to-end on B200 (TP=8, NVFP4).
docs/cookbook/autoregressive/ThinkingMachines/Inkling.mdx
+2 -2
View File
@@ -35,7 +35,7 @@ There are two multi-arch (amd64 / arm64) CUDA builds plus a ROCm build; pick the
```bash Command
docker pull lmsysorg/sglang:dev-inkling-dspark # CUDA 13
docker pull lmsysorg/sglang:dev-cu12-inkling-dspark # CUDA 12
docker pull lmsysorg/sglang:dev-cu12-inkling-dspark # CUDA 12 (final build; lane retired)
docker pull lmsysorg/sglang-rocm:dev-rocm720-mi35x-inkling-dspark # AMD MI350X / MI355X
```
@@ -306,4 +306,4 @@ To try it, select the **Long Context** strategy in the Deploy panel above for an
The **DSpark** deploy strategy is the second speculative-decoding path for Inkling. Unlike **MTP**, which drives Inkling's own multi-layer draft head, DSpark runs a **separate draft checkpoint** — `RadixArk/Inkling-DSpark-Preview` — served unquantized alongside the NVFP4 target.
DSpark support ships in the images listed in §1 (`dev-inkling-dspark` for CUDA 13, `dev-cu12-inkling-dspark` for CUDA 12), so no separate build is needed. Verified end-to-end on B200 (TP=8, NVFP4).
DSpark support ships in the images listed in §1 (`dev-inkling-dspark` for CUDA 13, `dev-cu12-inkling-dspark` for CUDA 12 — a final build, since that lane is retired), so no separate build is needed. Verified end-to-end on B200 (TP=8, NVFP4).
docs/docs/developer_guide/contribution_guide.mdx
+4 -8
View File
@@ -206,9 +206,9 @@ Develop SGLang's customized DeepGEMM package on the [`dev` branch of `sgl-projec
### Update sgl-deep-ep
Develop `sgl-deep-ep` in [`sgl-project/DeepEP`](https://github.com/sgl-project/DeepEP). Use the implementation branch for the target platform: `sgl-deepep` for CUDA 13 on x86_64 or aarch64, `sgl-deepep-cu12-x86` for CUDA 12.9 on x86_64, or `sgl-deepep-cu12-arm` for CUDA 12.9 on aarch64. Merge packaging changes into `sgl-deepep-packaging`. The [`sgl-deep-ep` README](https://github.com/sgl-project/DeepEP/blob/sgl-deepep-packaging/sgl_deep_ep/README.md) describes the platform prerequisites and release matrix.
Develop `sgl-deep-ep` in [`sgl-project/DeepEP`](https://github.com/sgl-project/DeepEP). Use `sgl-deepep`, the implementation branch for CUDA 13 on x86_64 or aarch64. Merge packaging changes into `sgl-deepep-packaging`. The [`sgl-deep-ep` README](https://github.com/sgl-project/DeepEP/blob/sgl-deepep-packaging/sgl_deep_ep/README.md) describes the platform prerequisites and release matrix.
To validate locally, check out the selected implementation branch as `DeepEP-source` and the packaging branch as `DeepEP-packaging`. Install the required build dependencies first; CUDA 12.9 builds also require GDRCopy. The following CUDA 13 example builds a wheel for the host architecture, installs that exact wheel, and verifies that its guarded package import succeeds:
To validate locally, check out the selected implementation branch as `DeepEP-source` and the packaging branch as `DeepEP-packaging`. Install the required build dependencies first. The following example builds a wheel for the host architecture, installs that exact wheel, and verifies that its guarded package import succeeds:
```bash
DEEPEP_OUTPUT_DIR="$(mktemp -d)"
@@ -223,17 +223,13 @@ python3 -m pip install --force-reinstall --no-deps \
python3 -c "import deep_ep; print(deep_ep.__file__)"
```
Use `12.9` instead of `13.0` for a CUDA 12.9 build. The import check validates packaging and binary loading, but not communication correctness. On a configured multi-GPU host, also run the test appropriate for the implementation branch:
The import check validates packaging and binary loading, but not communication correctness. On a configured multi-GPU host, also run:
```bash
# CUDA 13 implementation branch
python3 DeepEP-source/tests/elastic/test_ep.py --num-processes 8
# CUDA 12.9 implementation branches
python3 DeepEP-source/tests/test_intranode.py --num-processes 8
```
Adjust `--num-processes` to the available GPUs and run the internode or low-latency tests when those transports changed. After local validation, ask the SGLang team to run the [sgl-deep-ep release workflow](https://github.com/sgl-project/sglang/actions/workflows/release-whl-deepep.yml) with the new version, CUDA target, and packaging ref. After verifying the published wheels for the supported Python versions and architectures, update the `sgl-deep-ep` pin in `python/pyproject.toml` before landing dependent SGLang changes.
Adjust `--num-processes` to the available GPUs and run the internode or low-latency tests when those transports changed. After local validation, ask the SGLang team to run the [sgl-deep-ep release workflow](https://github.com/sgl-project/sglang/actions/workflows/release-whl-deepep.yml) with the new version and packaging ref. After verifying the published wheels for the supported Python versions and architectures, update the `sgl-deep-ep` pin in `python/pyproject.toml` before landing dependent SGLang changes.
## Tips for newcomers
docs/docs/get-started/install.mdx
+6 -17
View File
@@ -29,15 +29,11 @@ uv pip install --prerelease=allow sglang
Some of SGLang's dependencies only publish pre-releases on PyPI, so without `--prerelease=allow` uv older than 0.12.0 silently installs SGLang 0.5.9. On [uv 0.12.0](https://github.com/astral-sh/uv/releases/tag/0.12.0) and newer the flag is a harmless no-op.
</Note>
The major version of Cuda is 13 by default. To install sglang under Cuda 12 with pip or uv, please try the following commands:
```bash Command
pip install --upgrade pip
pip install uv
uv pip install --prerelease=allow sglang
uv pip install --force-reinstall torch==2.13.0 torchaudio==2.11.0 torchvision --index-url https://download.pytorch.org/whl/cu129
uv pip install --force-reinstall sglang-kernel --index-url https://docs.sglang.ai/whl/cu129/
uv pip install --force-reinstall sgl-deep-gemm --index-url https://docs.sglang.ai/whl/cu129/ --no-deps
```
<Note>
SGLang requires CUDA 13. The CUDA 12 (`cu129`) wheels and images are retired,
because PyTorch 2.14 publishes no CUDA 12.9 builds. SGLang 0.5.19 is the last
release with a CUDA 12 lane.
</Note>
### Nightly builds
@@ -49,13 +45,6 @@ pip install uv
uv pip install --prerelease=allow --index-strategy unsafe-best-match --extra-index-url https://docs.sglang.ai/whl/cu130/ sglang
```
To install a nightly build under Cuda 12, swap the index to `cu129`:
```bash Command
pip install --upgrade pip
pip install uv
uv pip install --prerelease=allow --index-strategy unsafe-best-match --extra-index-url https://docs.sglang.ai/whl/cu129/ sglang
```
### Quick fixes to common problems
- If you encounter `OSError: CUDA_HOME environment variable is not set`. Please set it to your CUDA install root with either of the following solutions:
1. Use `export CUDA_HOME=/usr/local/cuda-<your-cuda-version>` to set the `CUDA_HOME` environment variable.
@@ -113,7 +102,7 @@ docker run --gpus all \
You can also find the nightly docker images [here](https://hub.docker.com/r/lmsysorg/sglang/tags?name=nightly).
Notes:
- SGLang is shipped with CUDA 13 environment by default. To run SGLang on CUDA 12 environment, please use images with `-cu12` or `-cu129` suffix, such as `lmsysorg/sglang:latest-cu129` or `lmsysorg/sglang:dev-cu12`.
- SGLang images ship a CUDA 13 environment. The `-cu12` / `-cu129` images are no longer published; `lmsysorg/sglang:v0.5.19-cu129` is the last CUDA 12 tag.
## Method 4: Using Kubernetes
docs/src/snippets/configs/MiniMaxAI/minimax-m3.jsx
+4 -3
View File
@@ -93,9 +93,10 @@ sgl-eval run mmmu_pro \\
dockerImages: {
// M3-specific dev images (multi-arch amd64+arm64). cu13 carries the sm_103
// (B300/GB300) + Grace arm64 builds; cu12 is the Hopper/CUDA-12 build;
// dev-minimax-m3 is the rolling default. M3 model support is not yet in a
// tagged release, so :latest cannot serve it.
// (B300/GB300) + Grace arm64 builds; cu12 is the final Hopper/CUDA-12 build
// (that lane is retired, so it no longer rebuilds); dev-minimax-m3 is the
// rolling default. M3 model support is not yet in a tagged release, so
// :latest cannot serve it.
b200: "lmsysorg/sglang:dev-minimax-m3",
b300: "lmsysorg/sglang:dev-cu13-minimax-m3",
gb200: "lmsysorg/sglang:dev-cu13-minimax-m3",
docs/src/snippets/configs/thinkingmachines/inkling-small.jsx
+2 -1
View File
@@ -70,7 +70,8 @@ export const config = {
-d '{ "model": "{{MODEL_NAME}}", "messages": [{"role":"user","content":"Hello"}] }'`,
// NVIDIA: two multi-arch CUDA builds (dev-inkling-dspark for CUDA 13,
// dev-cu12-inkling-dspark for CUDA 12) — pick by your CUDA version, not by GPU.
// dev-cu12-inkling-dspark for CUDA 12, a final build since that lane is
// retired) — pick by your CUDA version, not by GPU.
// Panel defaults to cu13. AMD: dev-rocm720-mi35x-inkling-dspark (sglang-rocm repo).
// DGX Spark uses a dedicated arm64 CUDA 13 image with NCCL 2.30.7.
// All tiers ship from the same images, DSpark included (except DGX Spark).
docs/src/snippets/configs/thinkingmachines/inkling.jsx
+2 -1
View File
@@ -73,7 +73,8 @@ export const config = {
-d '{ "model": "{{MODEL_NAME}}", "messages": [{"role":"user","content":"Hello"}] }'`,
// NVIDIA: two multi-arch CUDA builds (dev-inkling-dspark for CUDA 13,
// dev-cu12-inkling-dspark for CUDA 12) — pick by your CUDA version, not by GPU.
// dev-cu12-inkling-dspark for CUDA 12, a final build since that lane is
// retired) — pick by your CUDA version, not by GPU.
// Panel defaults to cu13. AMD: dev-rocm720-mi35x-inkling-dspark (sglang-rocm repo).
// All tiers ship from the same images, DSpark included.
dockerImages: {
python/sglang/kernels/aot/Dockerfile
+1 -2
View File
@@ -98,8 +98,7 @@ RUN --mount=type=cache,id=sgl-kernel-pip,target=/root/.cache/pip \
set -eux; \
case "${CUDA_VERSION}" in \
13.0) TORCH_VER=2.13.0; CU_TAG=cu130 ;; \
12.9) TORCH_VER=2.13.0; CU_TAG=cu129 ;; \
*) TORCH_VER=2.13.0; CU_TAG=cu126 ;; \
*) echo "Unsupported CUDA version: ${CUDA_VERSION}" && exit 1 ;; \
esac; \
${PYTHON_ROOT_PATH}/bin/pip install torch==${TORCH_VER} --index-url ${PYTORCH_INDEX_BASE}/${CU_TAG}; \
${PYTHON_ROOT_PATH}/bin/pip install ninja setuptools==75.0.0 wheel==0.41.0 numpy uv scikit-build-core --index-url ${PIP_DEFAULT_INDEX}
python/sglang/kernels/aot/python/sgl_kernel/load_utils.py
+1 -6
View File
@@ -167,12 +167,7 @@ def _load_architecture_specific_ops():
# All attempts failed
cuda_version = torch.version.cuda
if cuda_version and cuda_version.startswith("12"):
install_hint = (
"pip install sglang-kernel --index-url https://docs.sglang.ai/whl/cu129/"
)
else:
install_hint = "pip install --upgrade sglang-kernel"
install_hint = "pip install --upgrade sglang-kernel"
error_msg = f"""
[sgl_kernel] CRITICAL: Could not load any common_ops library!
python/sglang/kernels/aot/rename_wheels.sh
+2 -6
View File
@@ -1,5 +1,5 @@
#!/usr/bin/env bash
# Align CUDA wheel filenames (+cu124/+cu129/+cu130) with internal METADATA Version and
# Align CUDA wheel filenames (+cu130) with internal METADATA Version and
# WHEEL tags after build (fixes pip "inconsistent version" when only the .whl name changed).
# Unpack → patch WHEEL/METADATA → wheel pack (RECORD regenerated; no hand-editing).
set -ex
@@ -7,11 +7,7 @@ set -ex
WHEEL_DIR="dist"
detect_cuda_suffix() {
if ls /usr/local/ 2>/dev/null | grep -q "12.4"; then
echo "+cu124"
elif ls /usr/local/ 2>/dev/null | grep -q "12.9"; then
echo "+cu129"
elif ls /usr/local/ 2>/dev/null | grep -q "13.0"; then
if ls /usr/local/ 2>/dev/null | grep -q "13.0"; then
echo "+cu130"
else
echo ""
python/sglang/kernels/jit/utils/arch.py
+1 -1
View File
@@ -42,7 +42,7 @@ def _jit_cuda_version() -> tuple[int, ...]:
"""CUDA version of the nvcc that JIT builds actually run.
The target has to match the compiler, not the toolkit PyTorch was built
against: a cu129 wheel on a CUDA 12.8 toolkit would otherwise select
against: a cu130 wheel on a CUDA 12.8 toolkit would otherwise select
`sm_120f`, which nvcc 12.8 rejects. Resolve nvcc the way tvm-ffi does
(`CUDA_HOME` / `CUDA_PATH`, then `$PATH`, then `/usr/local/cuda`) and fall
back to `torch.version.cuda` when it cannot be probed.
scripts/build_sgl_deep_gemm.sh
+2 -3
View File
@@ -3,12 +3,12 @@
#
# Usage: build_sgl_deep_gemm.sh <PYTHON_VERSION> <CUDA_VERSION> <DEEPGEMM_SRC> [ARCH]
# PYTHON_VERSION: e.g. 3.10
# CUDA_VERSION: e.g. 12.9 or 13.0
# CUDA_VERSION: e.g. 13.0
# DEEPGEMM_SRC: path to a checkout of sgl-project/DeepGEMM
# ARCH: x86_64 (default) or aarch64
#
# Writes:
# <DEEPGEMM_SRC>/dist/ — wheel(s) tagged +cu129 / +cu130 and manylinux
# <DEEPGEMM_SRC>/dist/ — wheel(s) tagged +cu130 and manylinux
# <DEEPGEMM_SRC>/dist-pypi/ — cu130 only: same wheel(s) with +cu130 stripped
# (PyPI rejects local-version segments)
set -ex
@@ -25,7 +25,6 @@ ARCH="${4:-$(uname -i)}"
case "${CUDA_VERSION}" in
13.0) CU_TAG=cu130 ;;
12.9) CU_TAG=cu129 ;;
*)
echo "Unsupported CUDA_VERSION: ${CUDA_VERSION}" >&2
exit 1
scripts/build_sgl_deepep.sh
+3 -6
View File
@@ -4,8 +4,8 @@
# Usage:
# build_sgl_deepep.sh <python-version> <cuda-version> <deepep-source> <packaging-overlay> [architecture]
#
# Writes CUDA-tagged wheels to <deepep-source>/dist. CUDA 13 builds also write
# PyPI-ready wheels without the local CUDA version to <deepep-source>/dist-pypi.
# Writes CUDA-tagged wheels to <deepep-source>/dist, plus PyPI-ready wheels
# without the local CUDA version in <deepep-source>/dist-pypi.
set -euo pipefail
@@ -14,7 +14,7 @@ usage() {
Usage: build_sgl_deepep.sh <python-version> <cuda-version> <deepep-source> <packaging-overlay> [architecture]
python-version: 3.10, 3.11, 3.12, or 3.13
cuda-version: 12.9 or 13.0
cuda-version: 13.0
deepep-source: checkout of the selected DeepEP implementation branch
packaging-overlay: path to the shared DeepEP sgl_deep_ep directory
architecture: x86_64 or aarch64 (defaults to the current machine)
@@ -45,9 +45,6 @@ case "${PYTHON_VERSION}" in
esac
case "${CUDA_VERSION}" in
12.9)
CUDA_TAG=cu129
;;
13.0)
CUDA_TAG=cu130
;;
scripts/ci/cuda/ci_install_dependency.sh
+32 -78
View File
@@ -2,7 +2,7 @@
# Install dependencies for CUDA CI jobs.
#
# CU_VERSION (default: cu130) controls PyTorch index URL, FlashInfer JIT cache
# index, and nvrtc variant selection.
# index, and the sglang wheel index. CUDA 13 only.
set -euxo pipefail
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
@@ -32,10 +32,14 @@ mark_step_done() {
configure_environment() {
# CU_VERSION controls PyTorch index URL, FlashInfer JIT cache index, and
# nvrtc variant selection (cu12 vs cu13).
# the sglang wheel index. Only CUDA 13 lanes exist: PyTorch 2.14 publishes
# no CUDA 12 wheels for the cu129 index the retired cu12 lane used.
CU_VERSION="${CU_VERSION:-cu130}"
CU_STRIP="${CU_VERSION#cu}"
CU_MAJOR="${CU_STRIP:0:2}"
case "${CU_STRIP}" in
13*) ;;
*) echo "FATAL: unsupported CU_VERSION=${CU_VERSION}; only CUDA 13 is supported"; exit 1 ;;
esac
OPTIONAL_DEPS="${1:-}"
@@ -366,11 +370,6 @@ remove_stale_cuda12_nvidia_wheels() {
local -a NVIDIA_WHEELS_TO_RESTORE=()
local -a STALE_CUDA12_NVIDIA_WHEELS=()
if [ "$CU_MAJOR" != "13" ]; then
mark_step_done "${FUNCNAME[0]}"
return
fi
mapfile -t INSTALLED_NVIDIA_WHEELS < <(
python3 -m pip list --format=freeze | sed -n '/^nvidia-.*==/p'
)
@@ -462,30 +461,6 @@ install_pytorch_stack() {
mark_step_done "${FUNCNAME[0]}"
}
install_cuda12_deepep_wheel() {
if [ "$CU_MAJOR" = "13" ]; then
echo "CUDA 13 uses the public sgl-deep-ep wheel declared in python/pyproject.toml"
mark_step_done "${FUNCNAME[0]}"
return
fi
local version
version=$(grep -Po -m1 '"sgl-deep-ep==\K[^"]+' python/pyproject.toml || true)
if [ -z "$version" ]; then
echo "ERROR: python/pyproject.toml must pin sgl-deep-ep"
exit 1
fi
# CUDA 12 wheels intentionally live only on the SGLang wheel index. Their
# local version satisfies the public-version pyproject pin, so the later
# editable SGLang install keeps this CUDA-matched wheel.
$PIP_CMD install "sgl-deep-ep==${version}+${CU_VERSION}" \
--index-url "https://docs.sglang.ai/whl/${CU_VERSION}/" \
--force-reinstall --no-deps $PIP_INSTALL_SUFFIX
mark_step_done "${FUNCNAME[0]}"
}
require_prebuilt_rust_exts() {
# Stages whose download succeeded set this to none. Runs before
# setup_pip_toolchain uninstalls sglang, so clearing it here still reaches
@@ -551,14 +526,10 @@ install_sglang() {
}
install_nccl() {
if [ "$CU_MAJOR" = "13" ]; then
# PyTorch pins 2.29.7, so this override must run after every command
# that resolves Python dependencies (including lmms-eval).
$PIP_CMD install "nvidia-nccl-cu13==2.30.7" \
--force-reinstall --no-deps $PIP_INSTALL_SUFFIX
else
echo "CUDA ${CU_MAJOR} does not require the NCCL Gin wheel"
fi
# PyTorch pins 2.29.7, so this override must run after every command
# that resolves Python dependencies (including lmms-eval).
$PIP_CMD install "nvidia-nccl-cu13==2.30.7" \
--force-reinstall --no-deps $PIP_INSTALL_SUFFIX
mark_step_done "${FUNCNAME[0]}"
}
@@ -628,8 +599,8 @@ install_sglang_kernel() {
if [ "${CUSTOM_BUILD_SGL_KERNEL:-}" != "true" ]; then
# The PyPI default wheel tracks one CUDA version (currently cu130); other
# runners (e.g. h20 / cu129) need the +${CU_VERSION}-tagged wheel from the
# sglang index, linked against the right libnvrtc.
# runners need the +${CU_VERSION}-tagged wheel from the sglang index,
# linked against the right libnvrtc.
SGL_KERNEL_WANTED="${SGL_KERNEL_VERSION_FROM_SRT}+${CU_VERSION}"
if installed_wheel_ok sglang-kernel "${SGL_KERNEL_WANTED}" reject-local; then
echo "sglang-kernel==${SGL_KERNEL_WANTED} already installed, keeping it"
@@ -640,18 +611,11 @@ install_sglang_kernel() {
echo "CUSTOM_BUILD_SGL_KERNEL=true: keeping freshly built sgl-kernel wheel."
fi
SGL_DEEP_GEMM_VERSION=$(grep -Po -m1 '(?<=sgl-deep-gemm==)[0-9A-Za-z\.\-]+' python/pyproject.toml)
if [ "$CU_MAJOR" = "13" ]; then
SGL_DEEP_GEMM_WANTED="${SGL_DEEP_GEMM_VERSION}"
else
SGL_DEEP_GEMM_WANTED="${SGL_DEEP_GEMM_VERSION}+cu129"
fi
# No reject-local: nothing builds sgl-deep-gemm locally.
if installed_wheel_ok sgl-deep-gemm "${SGL_DEEP_GEMM_WANTED}"; then
echo "sgl-deep-gemm==${SGL_DEEP_GEMM_WANTED} already installed, keeping it"
elif [ "$CU_MAJOR" = "13" ]; then
$PIP_CMD install "sgl-deep-gemm==${SGL_DEEP_GEMM_VERSION}" --force-reinstall $PIP_INSTALL_SUFFIX
if installed_wheel_ok sgl-deep-gemm "${SGL_DEEP_GEMM_VERSION}"; then
echo "sgl-deep-gemm==${SGL_DEEP_GEMM_VERSION} already installed, keeping it"
else
$PIP_CMD install "https://github.com/sgl-project/whl/releases/download/v${SGL_DEEP_GEMM_VERSION}/sgl_deep_gemm-${SGL_DEEP_GEMM_VERSION}+cu129-py3-none-manylinux2014_$(uname -m).whl" --force-reinstall $PIP_INSTALL_SUFFIX
$PIP_CMD install "sgl-deep-gemm==${SGL_DEEP_GEMM_VERSION}" --force-reinstall $PIP_INSTALL_SUFFIX
fi
mark_step_done "${FUNCNAME[0]}"
@@ -739,17 +703,10 @@ stabilize_flashinfer_jit_paths() {
install_extra_deps() {
MOONCAKE_VERSION="0.3.13"
NIXL_VERSION="1.3.0"
if [ "$CU_MAJOR" = "13" ]; then
MOONCAKE_PKG="mooncake-transfer-engine-cuda13==${MOONCAKE_VERSION}"
MOONCAKE_STALE_PKG="mooncake-transfer-engine"
NIXL_BIN_NAME="nixl-cu13"
EXTRA_NVIDIA_SPECS="nvidia-cuda-nvrtc"
else
MOONCAKE_PKG="mooncake-transfer-engine==${MOONCAKE_VERSION}"
MOONCAKE_STALE_PKG="mooncake-transfer-engine-cuda13"
NIXL_BIN_NAME="nixl-cu12"
EXTRA_NVIDIA_SPECS="nvidia-cuda-nvrtc-cu12"
fi
MOONCAKE_PKG="mooncake-transfer-engine-cuda13==${MOONCAKE_VERSION}"
MOONCAKE_STALE_PKG="mooncake-transfer-engine"
NIXL_BIN_NAME="nixl-cu13"
EXTRA_NVIDIA_SPECS="nvidia-cuda-nvrtc"
# Both variants own the same mooncake/ package files and bin/ scripts
# (mooncake_master, etc.). Uninstalling the stale variant deletes shared
# files that the live variant's RECORD still references, so we force a
@@ -834,20 +791,18 @@ verify_imports() {
import ctypes
import importlib.metadata
import os
import sys
if sys.argv[1] == "13":
if importlib.metadata.version("nvidia-nccl-cu13") != "2.30.7":
raise SystemExit("nvidia-nccl-cu13 was changed after the final CI override")
nccl = ctypes.CDLL("libnccl.so.2")
nccl_version = ctypes.c_int()
status = nccl.ncclGetVersion(ctypes.byref(nccl_version))
if status != 0 or nccl_version.value != 23007:
raise SystemExit(
f"expected NCCL runtime 2.30.7, got status={status}, "
f"raw_version={nccl_version.value}"
)
print("NCCL package and runtime versions are 2.30.7")
if importlib.metadata.version("nvidia-nccl-cu13") != "2.30.7":
raise SystemExit("nvidia-nccl-cu13 was changed after the final CI override")
nccl = ctypes.CDLL("libnccl.so.2")
nccl_version = ctypes.c_int()
status = nccl.ncclGetVersion(ctypes.byref(nccl_version))
if status != 0 or nccl_version.value != 23007:
raise SystemExit(
f"expected NCCL runtime 2.30.7, got status={status}, "
f"raw_version={nccl_version.value}"
)
print("NCCL package and runtime versions are 2.30.7")
import torch
print(torch.version.cuda)
@@ -882,7 +837,7 @@ for mod in ("server", "grpc", "multimodal"):
except Exception as exc:
raise SystemExit(f"{name} is present but does not load: {exc!r}")
print(f"{name} loads")
' "$CU_MAJOR"
'
mark_step_done "${FUNCNAME[0]}"
}
@@ -904,7 +859,6 @@ main() {
remove_stale_cuda12_nvidia_wheels
uninstall_stale_flashinfer
install_pytorch_stack
install_cuda12_deepep_wheel
setup_cargo_cache
install_sglang
release_cargo_cache_lock
scripts/ci/utils/docker_build_metadata_args.py
+1 -1
View File
@@ -3,7 +3,7 @@ import datetime
import json
import sys
MOVING_TAGS = {"dev", "dev-cu12", "dev-cu13", "latest"}
MOVING_TAGS = {"dev", "dev-cu13", "latest"}
def render_tag_template(tag: str, version: str, date: str, short_sha: str) -> str:
scripts/rename_sgl_deep_gemm_whl.sh
+1 -1
View File
@@ -8,7 +8,7 @@
#
# Usage: rename_wheels.sh <WHEEL_DIR> <CU_TAG> <ARCH>
# WHEEL_DIR: directory containing the *.whl file (e.g. DeepGEMM/dist)
# CU_TAG: cu129 | cu130
# CU_TAG: cu130
# ARCH: x86_64 | aarch64
set -ex
scripts/update_deepep_whl_index.py
+1 -1
View File
@@ -5,7 +5,7 @@ import hashlib
import pathlib
import re
SUPPORTED_CUDA_VERSIONS = ("129", "130")
SUPPORTED_CUDA_VERSIONS = ("130",)
WHEEL_PATTERN = re.compile(
r"^sgl_deep_ep-(?P<version>[0-9][^-]*)-[^-]+-[^-]+-[^-]+\.whl$"
)
scripts/update_deepgemm_whl_index.py
+2 -2
View File
@@ -1,14 +1,14 @@
# Generates a PEP 503 simple index for sgl-deep-gemm wheels under
# sgl-whl/cu<version>/sgl-deep-gemm/index.html. Mirrors the layout used by
# update_kernel_whl_index.py so consumers can `pip install
# sgl-deep-gemm --extra-index-url https://...whl/cu129`.
# sgl-deep-gemm --extra-index-url https://...whl/cu130`.
import argparse
import hashlib
import pathlib
import re
SUPPORTED_CUDA_VERSIONS = ["129", "130"]
SUPPORTED_CUDA_VERSIONS = ["130"]
def update_wheel_index(cuda_version, wheel_dir):
scripts/update_kernel_whl_index.py
+12 -11
View File
@@ -5,9 +5,9 @@ import hashlib
import pathlib
import re
# All the CUDA versions that the wheels will cover
SUPPORTED_CUDA_VERSIONS = ["129", "130"]
DEFAULT_CUDA_VERSION = "130"
# Local version a CUDA wheel carries, e.g. sglang_kernel-0.4.6.post1+cu130-...whl
CUDA_LOCAL_VERSION_PATTERN = re.compile(r"\+cu(\d+)")
def check_wheel_cuda_version(path_name, target_cuda_version):
@@ -19,16 +19,17 @@ def check_wheel_cuda_version(path_name, target_cuda_version):
):
return False
# For other CUDA versions, the wheel path name will contain the cuda version suffix, e.g. sglang_kernel-0.4.0+cu130-cp310-abi3-manylinux2014_x86_64.whl
if target_cuda_version != DEFAULT_CUDA_VERSION:
return target_cuda_version in path_name
# Match on the wheel's own +cuNNN tag rather than a list of known versions,
# so a wheel built for a CUDA version this script has never heard of is
# rejected instead of landing in the target index.
match = CUDA_LOCAL_VERSION_PATTERN.search(path_name)
if match is not None:
return match.group(1) == target_cuda_version
# For the default CUDA version, the wheel path name will not contain any cuda version suffix, e.g. sglang_kernel-0.4.0-cp310-abi3-manylinux2014_x86_64.whl
# So we need to check if the wheel path name contains any other cuda version suffix
for cuda_version in SUPPORTED_CUDA_VERSIONS:
if cuda_version != DEFAULT_CUDA_VERSION and cuda_version in path_name:
return False
return True
# An untagged wheel is the default-CUDA build, e.g.
# sglang_kernel-0.4.0-cp310-abi3-manylinux2014_x86_64.whl (PyPI rejects
# local versions, so that upload strips the tag).
return target_cuda_version == DEFAULT_CUDA_VERSION
def update_wheel_index(cuda_version=DEFAULT_CUDA_VERSION, rocm_version=None):
scripts/update_nightly_whl_index.py
+2 -2
View File
@@ -35,7 +35,7 @@ def update_wheel_index(
Args:
commit_hash: Short git commit hash (e.g., 'c5f1e86')
nightly_version: Full nightly version string (e.g., '0.5.6.post1.dev7716+gc5f1e86')
cuda_version: CUDA version string (e.g., '129' or '130')
cuda_version: CUDA version string (e.g., '130')
build_date: Build date in YYYY-MM-DD format (e.g., '2025-12-13')
"""
dist_dir = pathlib.Path("dist")
@@ -176,7 +176,7 @@ def main():
"--cuda-version",
type=str,
default="130",
help="CUDA version (e.g., '129' or '130'). Defaults to '130'.",
help="CUDA version (e.g., '130'). Defaults to '130'.",
)
parser.add_argument(
"--build-date",
test/registered/unit/tools/test_docker_build_metadata_args.py
+7 -7
View File
@@ -76,9 +76,9 @@ class TestDockerBuildMetadataArgs(unittest.TestCase):
def test_release_metadata_prefers_versioned_tag(self):
args = self.run_helper(
cuda="cu129",
cuda="cu134",
tag_config=[
{"cuda": "cu129", "tags": ["v{version}", "latest"]},
{"cuda": "cu134", "tags": ["v{version}", "latest"]},
{"cuda": "cu130", "tags": ["v{version}-cu130", "latest-cu130"]},
],
)
@@ -99,7 +99,7 @@ class TestDockerBuildMetadataArgs(unittest.TestCase):
cuda="cu130",
image_repo="lmsysorg/sglang-staging",
tag_config=[
{"cuda": "cu129", "tags": ["v{version}-runtime", "latest-runtime"]},
{"cuda": "cu134", "tags": ["v{version}-runtime", "latest-runtime"]},
{
"cuda": "cu130",
"tags": ["v{version}-cu130-runtime", "latest-cu130-runtime"],
@@ -114,11 +114,11 @@ class TestDockerBuildMetadataArgs(unittest.TestCase):
def test_dev_nightly_metadata_prefers_unique_tag_from_checked_out_commit(self):
args = self.run_helper(
cuda="cu129",
cuda="cu134",
version="",
build_commit="1234567890abcdef",
tag_config=[
{"cuda": "cu129", "tags": ["dev", "nightly-dev-{date}-{short_sha}"]},
{"cuda": "cu134", "tags": ["dev", "nightly-dev-{date}-{short_sha}"]},
{
"cuda": "cu130",
"tags": ["dev-cu13", "nightly-dev-cu13-{date}-{short_sha}"],
@@ -140,7 +140,7 @@ class TestDockerBuildMetadataArgs(unittest.TestCase):
cuda="cu130",
version="",
tag_config=[
{"cuda": "cu129", "tags": ["dev-my-test"]},
{"cuda": "cu134", "tags": ["dev-my-test"]},
{"cuda": "cu130", "tags": ["dev-cu13-my-test"]},
],
)
@@ -153,7 +153,7 @@ class TestDockerBuildMetadataArgs(unittest.TestCase):
def test_missing_cuda_entry_fails(self):
with self.assertRaisesRegex(ValueError, "cu130"):
self.helper.select_tag(
json.dumps([{"cuda": "cu129", "tags": ["v{version}"]}]),
json.dumps([{"cuda": "cu134", "tags": ["v{version}"]}]),
"cu130",
"0.6.0",
"20260429",