[NVIDIA] Update deepgemm, deep-ep, sgl-kernel in CUDA 13.4 image, use cuda base image (#40987)

This commit is contained in:
Trevor Morris
2026-09-28 14:12:46 -07:00
committed by GitHub
parent 6fa3fe69e2
commit b07cb9e9e7
+27 -100
View File
@@ -1,100 +1,28 @@
ARG CUDA_VERSION=13.4.0
ARG UBUNTU_BASE_IMAGE=ubuntu:24.04
ARG CUDA_PKG_VERSION=13-4
ARG CUDA_PREVIEW_REPO=https://packages.nvidia.com/noble
ARG CUDA_PREVIEW_SUITE=prerelease/cuda/13.4.0
# CUDA 13.4 only args
# Torch nighly args
ARG TORCH_NIGHTLY_INDEX=https://download.pytorch.org/whl/nightly/cu134
ARG TORCH_NIGHTLY_VERSION=2.15.0.dev20260818+cu134
ARG TORCHVISION_NIGHTLY_VERSION=0.30.0.dev20260819+cu134
ARG TORCHAUDIO_NIGHTLY_VERSION=2.11.0.dev20260818+cu134
# Wheel build args
ARG MANYLINUX_IMAGE=pytorch/manylinuxaarch64-builder:cuda13.4
ARG DEEPGEMM_REF=dev
ARG DEEPGEMM_REF=v0.2.0
ARG DEEPEP_SOURCE_REF=sgl-deepep
ARG DEEPEP_PACKAGING_REF=sgl-deepep-packaging
ARG SGL_DEEP_EP_VERSION=0.1.0
ARG SGL_DEEP_EP_VERSION=0.1.2
########################################################
# CUDA base: Ubuntu + CUDA 13.4 developer preview toolkit
########################################################
FROM ${UBUNTU_BASE_IMAGE} AS cuda_base
ARG CUDA_PKG_VERSION
ARG CUDA_PREVIEW_REPO
ARG CUDA_PREVIEW_SUITE
RUN printf 'Acquire::Retries "5";\nAcquire::http::Timeout "30";\nAcquire::https::Timeout "30";\n' \
> /etc/apt/apt.conf.d/80-net-hardening
RUN export DEBIAN_FRONTEND=noninteractive \
&& apt-get update \
&& apt-get install -y --no-install-recommends ca-certificates wget gnupg \
&& wget -q -O /tmp/nvidia-preview-keyring.deb "${CUDA_PREVIEW_REPO}/nvidia-preview-keyring.deb" \
&& dpkg -i /tmp/nvidia-preview-keyring.deb \
&& rm -f /tmp/nvidia-preview-keyring.deb \
&& printf '%s\n' \
'X-Repolib-Name: NVIDIA Packages (frozen)' \
'Types: deb' \
"URIs: ${CUDA_PREVIEW_REPO}" \
"Suites: ${CUDA_PREVIEW_SUITE}" \
'Components: main' \
'Signed-By: /usr/share/keyrings/nvidia-packages-preview.gpg' \
'Enabled: yes' \
> /etc/apt/sources.list.d/nvidia-packages-preview.sources \
&& apt-get update \
&& apt-get install -y --no-install-recommends "cuda-toolkit-${CUDA_PKG_VERSION}" \
&& cuda_dir="$(ls -d /usr/local/cuda-1* 2>/dev/null | head -1)" \
&& test -n "${cuda_dir}" \
&& ln -sfn "${cuda_dir}" /usr/local/cuda \
&& /usr/local/cuda/bin/nvcc --version \
&& rm -rf /var/lib/apt/lists/*
# The 13.4 preview repo ships no NCCL, but later stages need libnccl2 /
# libnccl-dev: pynccl_allocator JIT-compiles with -lnccl under symmetric memory.
# The repo must be removed again in this same layer, not left behind an apt pin:
# the devtools repo that nsight-systems-cli adds later publishes byte-identical
# release metadata (o=NVIDIA, l=NVIDIA CUDA, c=) on this same host, so any pin
# that stops the CUDA repo's older cuda-* packages from shadowing the 13.4
# toolkit also blocks nsight-systems-cli. Removal is safe: the later
# `apt-get install libnccl2 libnccl-dev` resolves from dpkg status.
RUN export DEBIAN_FRONTEND=noninteractive \
&& case "$(dpkg --print-architecture)" in \
arm64) repo_arch=sbsa ;; \
amd64) repo_arch=x86_64 ;; \
*) echo "unsupported arch: $(dpkg --print-architecture)" >&2; exit 1 ;; \
esac \
&& wget -q -O /tmp/cuda-keyring.deb \
"https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2404/${repo_arch}/cuda-keyring_1.1-1_all.deb" \
&& dpkg -i /tmp/cuda-keyring.deb \
&& rm -f /tmp/cuda-keyring.deb \
&& apt-get update \
&& apt-get install -y --no-install-recommends libnccl2 libnccl-dev \
&& test -e "/usr/lib/$(uname -m)-linux-gnu/libnccl.so" \
&& rm -f /etc/apt/sources.list.d/cuda*.list /etc/apt/sources.list.d/cuda*.sources \
&& dpkg -r cuda-keyring \
&& rm -rf /var/lib/apt/lists/*
ENV PATH=/usr/local/cuda/bin:${PATH} \
LD_LIBRARY_PATH=/usr/local/cuda/lib64 \
NVIDIA_VISIBLE_DEVICES=all \
NVIDIA_DRIVER_CAPABILITIES=compute,utility
########################################################
# Base stage
########################################################
FROM cuda_base AS base
ARG CUDA_VERSION=13.4.1
FROM nvidia/cuda:${CUDA_VERSION}-cudnn-devel-ubuntu24.04 AS base
ARG TARGETARCH
ARG BUILD_TYPE=all
ARG BRANCH_TYPE=remote
ARG SGL_KERNEL_VERSION=0.4.6.post1
ARG SGL_KERNEL_VERSION=0.4.7
ARG SGL_VERSION
ARG SGL_DEEP_GEMM_VERSION=0.1.5.post2
ARG SGL_DEEP_GEMM_VERSION=0.2.0
ARG USE_LATEST_SGLANG=0
ARG GDRCOPY_VERSION=2.5.1
ARG NCCL_VERSION=2.30.7
ARG SGL_NCCL_VERSION=2.30.7
ARG PIP_DEFAULT_INDEX
ARG UBUNTU_MIRROR
ARG GITHUB_ARTIFACTORY=github.com
@@ -351,7 +279,7 @@ RUN --mount=type=cache,id=cu134-sgl-kernel-ccache,target=/ccache \
FROM cu134_wheel_base AS cu134_wheel_deepgemm
ARG DEEPGEMM_REF
ARG SGL_DEEP_GEMM_VERSION=0.1.5.post2
ARG SGL_DEEP_GEMM_VERSION=0.2.0
ARG TVM_FFI_VERSION=0.1.11
# DeepJIT's exception handling requires elfutils/libdwfl.h and libelf headers.
@@ -370,8 +298,6 @@ COPY scripts/rename_sgl_deep_gemm_whl.sh /rename_sgl_deep_gemm_whl.sh
RUN --mount=type=cache,id=cu134-wheels-pip,target=/root/.cache/pip \
set -eux; \
cd /deepgemm; \
sed -i "s/'-std=c++17'/'-std=c++20'/g" \
build_sgl_deep_gemm.sh setup.py sgl_deep_gemm/__init__.py; \
printf '%s' "${SGL_DEEP_GEMM_VERSION}" > sgl_deep_gemm/VERSION; \
bash build_sgl_deep_gemm.sh; \
bash /rename_sgl_deep_gemm_whl.sh dist cu134 "$(uname -m)"; \
@@ -414,7 +340,6 @@ RUN git clone --recursive --depth 1 --branch "${DEEPEP_SOURCE_REF}" \
RUN set -eux; \
printf '%s' "${SGL_DEEP_EP_VERSION}" > /deepep-packaging/sgl_deep_ep/VERSION; \
sed -i 's/"-std=c++17"/"-std=c++20"/g' /deepep-source/setup.py; \
${PYTHON_ROOT_PATH}/bin/python - <<'PY'
import pathlib
p = pathlib.Path("/deepep-packaging/sgl_deep_ep/build_sgl_deep_ep.sh")
@@ -473,7 +398,9 @@ ARG CUDA_VERSION
ARG BUILD_TYPE
ARG SGL_KERNEL_VERSION
ARG GITHUB_ARTIFACTORY
ARG NCCL_VERSION
# Renamed from NCCL_VERSION: the nvidia/cuda base image sets ENV NCCL_VERSION
# (Debian apt format, e.g. 2.31.2-1), which shadows a same-named ARG inside RUN.
ARG SGL_NCCL_VERSION
ARG TORCH_NIGHTLY_INDEX
ARG TORCH_NIGHTLY_VERSION
ARG TORCHVISION_NIGHTLY_VERSION
@@ -496,14 +423,14 @@ RUN --mount=type=cache,target=/root/.cache/pip \
python3 -m pip install --upgrade pip setuptools wheel html5lib six \
&& case "$CUDA_VERSION" in \
13.0.3) CUINDEX=130 ;; \
13.4.0) CUINDEX=134 ;; \
13.4.1) CUINDEX=134 ;; \
*) echo "Unsupported CUDA version: $CUDA_VERSION" && exit 1 ;; \
esac \
&& if [ "$CUDA_VERSION" = "13.0.3" ]; then \
# --no-deps prevents pip from pulling torch from default PyPI
python3 -m pip install sglang-kernel==${SGL_KERNEL_VERSION} --force-reinstall --no-deps \
; \
elif [ "$CUDA_VERSION" = "13.4.0" ]; then \
elif [ "$CUDA_VERSION" = "13.4.1" ]; then \
# Use locally built sgl-* wheels.
python3 -m pip install --force-reinstall --no-deps /tmp/cu134_wheels/*.whl \
; \
@@ -527,7 +454,7 @@ RUN --mount=type=cache,target=/root/.cache/pip \
--mount=type=cache,target=/root/.cargo/registry \
case "$CUDA_VERSION" in \
13.0.3) CUINDEX=130 ;; \
13.4.0) CUINDEX=134 ;; \
13.4.1) CUINDEX=134 ;; \
*) echo "Unsupported CUDA version: $CUDA_VERSION" && exit 1 ;; \
esac \
&& cd /tmp/sglang_deps/python \
@@ -536,10 +463,10 @@ RUN --mount=type=cache,target=/root/.cache/pip \
&& echo '__version__ = "0.0.0"' > sglang/version.py \
&& touch README.md \
&& touch LICENSE \
&& if [ "$CUDA_VERSION" = "13.4.0" ]; then \
&& if [ "$CUDA_VERSION" = "13.4.1" ]; then \
sed -i "s|^ \"flashinfer_python\[cu13\]==.*| \"flashinfer_python[cu13]==${FLASHINFER_VERSION}\",|" pyproject.toml; \
fi \
&& if [ "$CUDA_VERSION" = "13.4.0" ]; then \
&& if [ "$CUDA_VERSION" = "13.4.1" ]; then \
TORCH_INDEX="${TORCH_NIGHTLY_INDEX}"; \
python3 -m pip install --index-url "${TORCH_NIGHTLY_INDEX}" \
"torch==${TORCH_NIGHTLY_VERSION}" \
@@ -557,8 +484,8 @@ RUN --mount=type=cache,target=/root/.cache/pip \
fi \
&& python3 -m pip install --extra-index-url "${TORCH_INDEX}" ".[${BUILD_TYPE}]" \
&& python3 -m pip install --force-reinstall --no-deps \
"nvidia-nccl-cu13==${NCCL_VERSION}" \
&& if [ "$CUDA_VERSION" = "13.4.0" ]; then \
"nvidia-nccl-cu13==${SGL_NCCL_VERSION}" \
&& if [ "$CUDA_VERSION" = "13.4.1" ]; then \
python3 -m pip install --force-reinstall --no-deps /tmp/cu134_wheels/*.whl; \
fi \
&& cd /sgl-workspace \
@@ -571,7 +498,7 @@ RUN python3 -m pip install --ignore-installed --no-deps distro
# constraints.txt pins the cu134 nightly torch, which does not exist on PyPI;
# later stages cannot satisfy it without the nightly index in scope.
RUN if [ "$CUDA_VERSION" = "13.4.0" ]; then \
RUN if [ "$CUDA_VERSION" = "13.4.1" ]; then \
python3 -m pip config set global.extra-index-url "${TORCH_NIGHTLY_INDEX}"; \
fi
@@ -612,7 +539,7 @@ ARG FLASHINFER_VERSION
RUN --mount=type=cache,target=/root/.cache/pip \
case "$CUDA_VERSION" in \
13.0.3) CUINDEX=130 ;; \
13.4.0) CUINDEX=134 ;; \
13.4.1) CUINDEX=134 ;; \
*) echo "Unsupported CUDA version: $CUDA_VERSION" && exit 1 ;; \
esac \
&& mkdir -p /flashinfer_jit_output \
@@ -818,7 +745,7 @@ RUN --mount=type=cache,target=/root/.cache/pip \
# one, so the constraints-pinned cu134 nightly must be restated here; otherwise
# runai-model-streamer's "torch<3.0.0,>=2.0.0" backtracks the whole resolve.
RUN --mount=type=cache,target=/root/.cache/pip \
if [ "$CUDA_VERSION" = "13.4.0" ]; then TORCH_SPEC="torch==${TORCH_NIGHTLY_VERSION}"; else TORCH_SPEC=""; fi \
if [ "$CUDA_VERSION" = "13.4.1" ]; then TORCH_SPEC="torch==${TORCH_NIGHTLY_VERSION}"; else TORCH_SPEC=""; fi \
&& python3 -m pip install -c /sgl-workspace/constraints.txt \
${TORCH_SPEC} \
datamodel_code_generator \
@@ -917,7 +844,7 @@ RUN if [ "$BRANCH_TYPE" = "local" ]; then \
# Clean up __pycache__/tests/pyc in same RUN to avoid writing ~28k files to layer
RUN --mount=type=cache,target=/root/.cache/pip \
cd /sgl-workspace/sglang \
&& if [ "$CUDA_VERSION" = "13.4.0" ]; then \
&& if [ "$CUDA_VERSION" = "13.4.1" ]; then \
sed -i "s|^ \"flashinfer_python\[cu13\]==.*| \"flashinfer_python[cu13]==${FLASHINFER_VERSION}\",|" python/pyproject.toml; \
fi \
&& python3 -m pip install --no-deps -e "python[${BUILD_TYPE}]" \
@@ -958,7 +885,7 @@ RUN --mount=type=cache,target=/root/.cache/pip \
# quack-kernels hard-pins nvidia-cutlass-dsl==4.6.2, override.
RUN --mount=type=cache,target=/root/.cache/pip \
if [ "$CUDA_VERSION" = "13.4.0" ]; then \
if [ "$CUDA_VERSION" = "13.4.1" ]; then \
python3 -m pip uninstall -y \
nvidia-cutlass-dsl \
nvidia-cutlass-dsl-libs-base \
@@ -1005,7 +932,7 @@ LABEL org.opencontainers.image.source="https://github.com/sgl-project/sglang" \
# - Ready-to-run inference server environment
#
# Note: Uses devel base for complete NVCC toolchain required by DeepGEMM JIT
FROM cuda_base AS runtime
FROM nvidia/cuda:${CUDA_VERSION}-cudnn-devel-ubuntu24.04 AS runtime
ARG CUDA_VERSION
ARG TARGETARCH