[NVIDIA] Update deepgemm, deep-ep, sgl-kernel in CUDA 13.4 image, use cuda base image (#40987)
This commit is contained in:
+27
-100
@@ -1,100 +1,28 @@
|
||||
ARG CUDA_VERSION=13.4.0
|
||||
ARG UBUNTU_BASE_IMAGE=ubuntu:24.04
|
||||
ARG CUDA_PKG_VERSION=13-4
|
||||
ARG CUDA_PREVIEW_REPO=https://packages.nvidia.com/noble
|
||||
ARG CUDA_PREVIEW_SUITE=prerelease/cuda/13.4.0
|
||||
|
||||
# CUDA 13.4 only args
|
||||
# Torch nighly args
|
||||
ARG TORCH_NIGHTLY_INDEX=https://download.pytorch.org/whl/nightly/cu134
|
||||
ARG TORCH_NIGHTLY_VERSION=2.15.0.dev20260818+cu134
|
||||
ARG TORCHVISION_NIGHTLY_VERSION=0.30.0.dev20260819+cu134
|
||||
ARG TORCHAUDIO_NIGHTLY_VERSION=2.11.0.dev20260818+cu134
|
||||
|
||||
# Wheel build args
|
||||
ARG MANYLINUX_IMAGE=pytorch/manylinuxaarch64-builder:cuda13.4
|
||||
ARG DEEPGEMM_REF=dev
|
||||
ARG DEEPGEMM_REF=v0.2.0
|
||||
ARG DEEPEP_SOURCE_REF=sgl-deepep
|
||||
ARG DEEPEP_PACKAGING_REF=sgl-deepep-packaging
|
||||
ARG SGL_DEEP_EP_VERSION=0.1.0
|
||||
ARG SGL_DEEP_EP_VERSION=0.1.2
|
||||
|
||||
########################################################
|
||||
# CUDA base: Ubuntu + CUDA 13.4 developer preview toolkit
|
||||
########################################################
|
||||
FROM ${UBUNTU_BASE_IMAGE} AS cuda_base
|
||||
|
||||
ARG CUDA_PKG_VERSION
|
||||
ARG CUDA_PREVIEW_REPO
|
||||
ARG CUDA_PREVIEW_SUITE
|
||||
|
||||
RUN printf 'Acquire::Retries "5";\nAcquire::http::Timeout "30";\nAcquire::https::Timeout "30";\n' \
|
||||
> /etc/apt/apt.conf.d/80-net-hardening
|
||||
|
||||
RUN export DEBIAN_FRONTEND=noninteractive \
|
||||
&& apt-get update \
|
||||
&& apt-get install -y --no-install-recommends ca-certificates wget gnupg \
|
||||
&& wget -q -O /tmp/nvidia-preview-keyring.deb "${CUDA_PREVIEW_REPO}/nvidia-preview-keyring.deb" \
|
||||
&& dpkg -i /tmp/nvidia-preview-keyring.deb \
|
||||
&& rm -f /tmp/nvidia-preview-keyring.deb \
|
||||
&& printf '%s\n' \
|
||||
'X-Repolib-Name: NVIDIA Packages (frozen)' \
|
||||
'Types: deb' \
|
||||
"URIs: ${CUDA_PREVIEW_REPO}" \
|
||||
"Suites: ${CUDA_PREVIEW_SUITE}" \
|
||||
'Components: main' \
|
||||
'Signed-By: /usr/share/keyrings/nvidia-packages-preview.gpg' \
|
||||
'Enabled: yes' \
|
||||
> /etc/apt/sources.list.d/nvidia-packages-preview.sources \
|
||||
&& apt-get update \
|
||||
&& apt-get install -y --no-install-recommends "cuda-toolkit-${CUDA_PKG_VERSION}" \
|
||||
&& cuda_dir="$(ls -d /usr/local/cuda-1* 2>/dev/null | head -1)" \
|
||||
&& test -n "${cuda_dir}" \
|
||||
&& ln -sfn "${cuda_dir}" /usr/local/cuda \
|
||||
&& /usr/local/cuda/bin/nvcc --version \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
# The 13.4 preview repo ships no NCCL, but later stages need libnccl2 /
|
||||
# libnccl-dev: pynccl_allocator JIT-compiles with -lnccl under symmetric memory.
|
||||
# The repo must be removed again in this same layer, not left behind an apt pin:
|
||||
# the devtools repo that nsight-systems-cli adds later publishes byte-identical
|
||||
# release metadata (o=NVIDIA, l=NVIDIA CUDA, c=) on this same host, so any pin
|
||||
# that stops the CUDA repo's older cuda-* packages from shadowing the 13.4
|
||||
# toolkit also blocks nsight-systems-cli. Removal is safe: the later
|
||||
# `apt-get install libnccl2 libnccl-dev` resolves from dpkg status.
|
||||
RUN export DEBIAN_FRONTEND=noninteractive \
|
||||
&& case "$(dpkg --print-architecture)" in \
|
||||
arm64) repo_arch=sbsa ;; \
|
||||
amd64) repo_arch=x86_64 ;; \
|
||||
*) echo "unsupported arch: $(dpkg --print-architecture)" >&2; exit 1 ;; \
|
||||
esac \
|
||||
&& wget -q -O /tmp/cuda-keyring.deb \
|
||||
"https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2404/${repo_arch}/cuda-keyring_1.1-1_all.deb" \
|
||||
&& dpkg -i /tmp/cuda-keyring.deb \
|
||||
&& rm -f /tmp/cuda-keyring.deb \
|
||||
&& apt-get update \
|
||||
&& apt-get install -y --no-install-recommends libnccl2 libnccl-dev \
|
||||
&& test -e "/usr/lib/$(uname -m)-linux-gnu/libnccl.so" \
|
||||
&& rm -f /etc/apt/sources.list.d/cuda*.list /etc/apt/sources.list.d/cuda*.sources \
|
||||
&& dpkg -r cuda-keyring \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
ENV PATH=/usr/local/cuda/bin:${PATH} \
|
||||
LD_LIBRARY_PATH=/usr/local/cuda/lib64 \
|
||||
NVIDIA_VISIBLE_DEVICES=all \
|
||||
NVIDIA_DRIVER_CAPABILITIES=compute,utility
|
||||
|
||||
|
||||
########################################################
|
||||
# Base stage
|
||||
########################################################
|
||||
FROM cuda_base AS base
|
||||
ARG CUDA_VERSION=13.4.1
|
||||
FROM nvidia/cuda:${CUDA_VERSION}-cudnn-devel-ubuntu24.04 AS base
|
||||
|
||||
ARG TARGETARCH
|
||||
ARG BUILD_TYPE=all
|
||||
ARG BRANCH_TYPE=remote
|
||||
ARG SGL_KERNEL_VERSION=0.4.6.post1
|
||||
ARG SGL_KERNEL_VERSION=0.4.7
|
||||
ARG SGL_VERSION
|
||||
ARG SGL_DEEP_GEMM_VERSION=0.1.5.post2
|
||||
ARG SGL_DEEP_GEMM_VERSION=0.2.0
|
||||
ARG USE_LATEST_SGLANG=0
|
||||
ARG GDRCOPY_VERSION=2.5.1
|
||||
ARG NCCL_VERSION=2.30.7
|
||||
ARG SGL_NCCL_VERSION=2.30.7
|
||||
ARG PIP_DEFAULT_INDEX
|
||||
ARG UBUNTU_MIRROR
|
||||
ARG GITHUB_ARTIFACTORY=github.com
|
||||
@@ -351,7 +279,7 @@ RUN --mount=type=cache,id=cu134-sgl-kernel-ccache,target=/ccache \
|
||||
FROM cu134_wheel_base AS cu134_wheel_deepgemm
|
||||
|
||||
ARG DEEPGEMM_REF
|
||||
ARG SGL_DEEP_GEMM_VERSION=0.1.5.post2
|
||||
ARG SGL_DEEP_GEMM_VERSION=0.2.0
|
||||
ARG TVM_FFI_VERSION=0.1.11
|
||||
|
||||
# DeepJIT's exception handling requires elfutils/libdwfl.h and libelf headers.
|
||||
@@ -370,8 +298,6 @@ COPY scripts/rename_sgl_deep_gemm_whl.sh /rename_sgl_deep_gemm_whl.sh
|
||||
RUN --mount=type=cache,id=cu134-wheels-pip,target=/root/.cache/pip \
|
||||
set -eux; \
|
||||
cd /deepgemm; \
|
||||
sed -i "s/'-std=c++17'/'-std=c++20'/g" \
|
||||
build_sgl_deep_gemm.sh setup.py sgl_deep_gemm/__init__.py; \
|
||||
printf '%s' "${SGL_DEEP_GEMM_VERSION}" > sgl_deep_gemm/VERSION; \
|
||||
bash build_sgl_deep_gemm.sh; \
|
||||
bash /rename_sgl_deep_gemm_whl.sh dist cu134 "$(uname -m)"; \
|
||||
@@ -414,7 +340,6 @@ RUN git clone --recursive --depth 1 --branch "${DEEPEP_SOURCE_REF}" \
|
||||
|
||||
RUN set -eux; \
|
||||
printf '%s' "${SGL_DEEP_EP_VERSION}" > /deepep-packaging/sgl_deep_ep/VERSION; \
|
||||
sed -i 's/"-std=c++17"/"-std=c++20"/g' /deepep-source/setup.py; \
|
||||
${PYTHON_ROOT_PATH}/bin/python - <<'PY'
|
||||
import pathlib
|
||||
p = pathlib.Path("/deepep-packaging/sgl_deep_ep/build_sgl_deep_ep.sh")
|
||||
@@ -473,7 +398,9 @@ ARG CUDA_VERSION
|
||||
ARG BUILD_TYPE
|
||||
ARG SGL_KERNEL_VERSION
|
||||
ARG GITHUB_ARTIFACTORY
|
||||
ARG NCCL_VERSION
|
||||
# Renamed from NCCL_VERSION: the nvidia/cuda base image sets ENV NCCL_VERSION
|
||||
# (Debian apt format, e.g. 2.31.2-1), which shadows a same-named ARG inside RUN.
|
||||
ARG SGL_NCCL_VERSION
|
||||
ARG TORCH_NIGHTLY_INDEX
|
||||
ARG TORCH_NIGHTLY_VERSION
|
||||
ARG TORCHVISION_NIGHTLY_VERSION
|
||||
@@ -496,14 +423,14 @@ RUN --mount=type=cache,target=/root/.cache/pip \
|
||||
python3 -m pip install --upgrade pip setuptools wheel html5lib six \
|
||||
&& case "$CUDA_VERSION" in \
|
||||
13.0.3) CUINDEX=130 ;; \
|
||||
13.4.0) CUINDEX=134 ;; \
|
||||
13.4.1) CUINDEX=134 ;; \
|
||||
*) echo "Unsupported CUDA version: $CUDA_VERSION" && exit 1 ;; \
|
||||
esac \
|
||||
&& if [ "$CUDA_VERSION" = "13.0.3" ]; then \
|
||||
# --no-deps prevents pip from pulling torch from default PyPI
|
||||
python3 -m pip install sglang-kernel==${SGL_KERNEL_VERSION} --force-reinstall --no-deps \
|
||||
; \
|
||||
elif [ "$CUDA_VERSION" = "13.4.0" ]; then \
|
||||
elif [ "$CUDA_VERSION" = "13.4.1" ]; then \
|
||||
# Use locally built sgl-* wheels.
|
||||
python3 -m pip install --force-reinstall --no-deps /tmp/cu134_wheels/*.whl \
|
||||
; \
|
||||
@@ -527,7 +454,7 @@ RUN --mount=type=cache,target=/root/.cache/pip \
|
||||
--mount=type=cache,target=/root/.cargo/registry \
|
||||
case "$CUDA_VERSION" in \
|
||||
13.0.3) CUINDEX=130 ;; \
|
||||
13.4.0) CUINDEX=134 ;; \
|
||||
13.4.1) CUINDEX=134 ;; \
|
||||
*) echo "Unsupported CUDA version: $CUDA_VERSION" && exit 1 ;; \
|
||||
esac \
|
||||
&& cd /tmp/sglang_deps/python \
|
||||
@@ -536,10 +463,10 @@ RUN --mount=type=cache,target=/root/.cache/pip \
|
||||
&& echo '__version__ = "0.0.0"' > sglang/version.py \
|
||||
&& touch README.md \
|
||||
&& touch LICENSE \
|
||||
&& if [ "$CUDA_VERSION" = "13.4.0" ]; then \
|
||||
&& if [ "$CUDA_VERSION" = "13.4.1" ]; then \
|
||||
sed -i "s|^ \"flashinfer_python\[cu13\]==.*| \"flashinfer_python[cu13]==${FLASHINFER_VERSION}\",|" pyproject.toml; \
|
||||
fi \
|
||||
&& if [ "$CUDA_VERSION" = "13.4.0" ]; then \
|
||||
&& if [ "$CUDA_VERSION" = "13.4.1" ]; then \
|
||||
TORCH_INDEX="${TORCH_NIGHTLY_INDEX}"; \
|
||||
python3 -m pip install --index-url "${TORCH_NIGHTLY_INDEX}" \
|
||||
"torch==${TORCH_NIGHTLY_VERSION}" \
|
||||
@@ -557,8 +484,8 @@ RUN --mount=type=cache,target=/root/.cache/pip \
|
||||
fi \
|
||||
&& python3 -m pip install --extra-index-url "${TORCH_INDEX}" ".[${BUILD_TYPE}]" \
|
||||
&& python3 -m pip install --force-reinstall --no-deps \
|
||||
"nvidia-nccl-cu13==${NCCL_VERSION}" \
|
||||
&& if [ "$CUDA_VERSION" = "13.4.0" ]; then \
|
||||
"nvidia-nccl-cu13==${SGL_NCCL_VERSION}" \
|
||||
&& if [ "$CUDA_VERSION" = "13.4.1" ]; then \
|
||||
python3 -m pip install --force-reinstall --no-deps /tmp/cu134_wheels/*.whl; \
|
||||
fi \
|
||||
&& cd /sgl-workspace \
|
||||
@@ -571,7 +498,7 @@ RUN python3 -m pip install --ignore-installed --no-deps distro
|
||||
|
||||
# constraints.txt pins the cu134 nightly torch, which does not exist on PyPI;
|
||||
# later stages cannot satisfy it without the nightly index in scope.
|
||||
RUN if [ "$CUDA_VERSION" = "13.4.0" ]; then \
|
||||
RUN if [ "$CUDA_VERSION" = "13.4.1" ]; then \
|
||||
python3 -m pip config set global.extra-index-url "${TORCH_NIGHTLY_INDEX}"; \
|
||||
fi
|
||||
|
||||
@@ -612,7 +539,7 @@ ARG FLASHINFER_VERSION
|
||||
RUN --mount=type=cache,target=/root/.cache/pip \
|
||||
case "$CUDA_VERSION" in \
|
||||
13.0.3) CUINDEX=130 ;; \
|
||||
13.4.0) CUINDEX=134 ;; \
|
||||
13.4.1) CUINDEX=134 ;; \
|
||||
*) echo "Unsupported CUDA version: $CUDA_VERSION" && exit 1 ;; \
|
||||
esac \
|
||||
&& mkdir -p /flashinfer_jit_output \
|
||||
@@ -818,7 +745,7 @@ RUN --mount=type=cache,target=/root/.cache/pip \
|
||||
# one, so the constraints-pinned cu134 nightly must be restated here; otherwise
|
||||
# runai-model-streamer's "torch<3.0.0,>=2.0.0" backtracks the whole resolve.
|
||||
RUN --mount=type=cache,target=/root/.cache/pip \
|
||||
if [ "$CUDA_VERSION" = "13.4.0" ]; then TORCH_SPEC="torch==${TORCH_NIGHTLY_VERSION}"; else TORCH_SPEC=""; fi \
|
||||
if [ "$CUDA_VERSION" = "13.4.1" ]; then TORCH_SPEC="torch==${TORCH_NIGHTLY_VERSION}"; else TORCH_SPEC=""; fi \
|
||||
&& python3 -m pip install -c /sgl-workspace/constraints.txt \
|
||||
${TORCH_SPEC} \
|
||||
datamodel_code_generator \
|
||||
@@ -917,7 +844,7 @@ RUN if [ "$BRANCH_TYPE" = "local" ]; then \
|
||||
# Clean up __pycache__/tests/pyc in same RUN to avoid writing ~28k files to layer
|
||||
RUN --mount=type=cache,target=/root/.cache/pip \
|
||||
cd /sgl-workspace/sglang \
|
||||
&& if [ "$CUDA_VERSION" = "13.4.0" ]; then \
|
||||
&& if [ "$CUDA_VERSION" = "13.4.1" ]; then \
|
||||
sed -i "s|^ \"flashinfer_python\[cu13\]==.*| \"flashinfer_python[cu13]==${FLASHINFER_VERSION}\",|" python/pyproject.toml; \
|
||||
fi \
|
||||
&& python3 -m pip install --no-deps -e "python[${BUILD_TYPE}]" \
|
||||
@@ -958,7 +885,7 @@ RUN --mount=type=cache,target=/root/.cache/pip \
|
||||
|
||||
# quack-kernels hard-pins nvidia-cutlass-dsl==4.6.2, override.
|
||||
RUN --mount=type=cache,target=/root/.cache/pip \
|
||||
if [ "$CUDA_VERSION" = "13.4.0" ]; then \
|
||||
if [ "$CUDA_VERSION" = "13.4.1" ]; then \
|
||||
python3 -m pip uninstall -y \
|
||||
nvidia-cutlass-dsl \
|
||||
nvidia-cutlass-dsl-libs-base \
|
||||
@@ -1005,7 +932,7 @@ LABEL org.opencontainers.image.source="https://github.com/sgl-project/sglang" \
|
||||
# - Ready-to-run inference server environment
|
||||
#
|
||||
# Note: Uses devel base for complete NVCC toolchain required by DeepGEMM JIT
|
||||
FROM cuda_base AS runtime
|
||||
FROM nvidia/cuda:${CUDA_VERSION}-cudnn-devel-ubuntu24.04 AS runtime
|
||||
|
||||
ARG CUDA_VERSION
|
||||
ARG TARGETARCH
|
||||
|
||||
Reference in New Issue
Block a user