From b07cb9e9e73acc60c04b5fcf6554380b4ec78770 Mon Sep 17 00:00:00 2001 From: Trevor Morris Date: Mon, 28 Sep 2026 14:12:46 -0700 Subject: [PATCH] [NVIDIA] Update deepgemm, deep-ep, sgl-kernel in CUDA 13.4 image, use cuda base image (#40987) --- docker/Dockerfile.cu134 | 127 +++++++++------------------------------- 1 file changed, 27 insertions(+), 100 deletions(-) diff --git a/docker/Dockerfile.cu134 b/docker/Dockerfile.cu134 index 86e58bfe58..57c01d2adf 100644 --- a/docker/Dockerfile.cu134 +++ b/docker/Dockerfile.cu134 @@ -1,100 +1,28 @@ -ARG CUDA_VERSION=13.4.0 -ARG UBUNTU_BASE_IMAGE=ubuntu:24.04 -ARG CUDA_PKG_VERSION=13-4 -ARG CUDA_PREVIEW_REPO=https://packages.nvidia.com/noble -ARG CUDA_PREVIEW_SUITE=prerelease/cuda/13.4.0 - +# CUDA 13.4 only args +# Torch nighly args ARG TORCH_NIGHTLY_INDEX=https://download.pytorch.org/whl/nightly/cu134 ARG TORCH_NIGHTLY_VERSION=2.15.0.dev20260818+cu134 ARG TORCHVISION_NIGHTLY_VERSION=0.30.0.dev20260819+cu134 ARG TORCHAUDIO_NIGHTLY_VERSION=2.11.0.dev20260818+cu134 - +# Wheel build args ARG MANYLINUX_IMAGE=pytorch/manylinuxaarch64-builder:cuda13.4 -ARG DEEPGEMM_REF=dev +ARG DEEPGEMM_REF=v0.2.0 ARG DEEPEP_SOURCE_REF=sgl-deepep ARG DEEPEP_PACKAGING_REF=sgl-deepep-packaging -ARG SGL_DEEP_EP_VERSION=0.1.0 +ARG SGL_DEEP_EP_VERSION=0.1.2 -######################################################## -# CUDA base: Ubuntu + CUDA 13.4 developer preview toolkit -######################################################## -FROM ${UBUNTU_BASE_IMAGE} AS cuda_base - -ARG CUDA_PKG_VERSION -ARG CUDA_PREVIEW_REPO -ARG CUDA_PREVIEW_SUITE - -RUN printf 'Acquire::Retries "5";\nAcquire::http::Timeout "30";\nAcquire::https::Timeout "30";\n' \ - > /etc/apt/apt.conf.d/80-net-hardening - -RUN export DEBIAN_FRONTEND=noninteractive \ - && apt-get update \ - && apt-get install -y --no-install-recommends ca-certificates wget gnupg \ - && wget -q -O /tmp/nvidia-preview-keyring.deb "${CUDA_PREVIEW_REPO}/nvidia-preview-keyring.deb" \ - && dpkg -i /tmp/nvidia-preview-keyring.deb \ - && rm -f /tmp/nvidia-preview-keyring.deb \ - && printf '%s\n' \ - 'X-Repolib-Name: NVIDIA Packages (frozen)' \ - 'Types: deb' \ - "URIs: ${CUDA_PREVIEW_REPO}" \ - "Suites: ${CUDA_PREVIEW_SUITE}" \ - 'Components: main' \ - 'Signed-By: /usr/share/keyrings/nvidia-packages-preview.gpg' \ - 'Enabled: yes' \ - > /etc/apt/sources.list.d/nvidia-packages-preview.sources \ - && apt-get update \ - && apt-get install -y --no-install-recommends "cuda-toolkit-${CUDA_PKG_VERSION}" \ - && cuda_dir="$(ls -d /usr/local/cuda-1* 2>/dev/null | head -1)" \ - && test -n "${cuda_dir}" \ - && ln -sfn "${cuda_dir}" /usr/local/cuda \ - && /usr/local/cuda/bin/nvcc --version \ - && rm -rf /var/lib/apt/lists/* - -# The 13.4 preview repo ships no NCCL, but later stages need libnccl2 / -# libnccl-dev: pynccl_allocator JIT-compiles with -lnccl under symmetric memory. -# The repo must be removed again in this same layer, not left behind an apt pin: -# the devtools repo that nsight-systems-cli adds later publishes byte-identical -# release metadata (o=NVIDIA, l=NVIDIA CUDA, c=) on this same host, so any pin -# that stops the CUDA repo's older cuda-* packages from shadowing the 13.4 -# toolkit also blocks nsight-systems-cli. Removal is safe: the later -# `apt-get install libnccl2 libnccl-dev` resolves from dpkg status. -RUN export DEBIAN_FRONTEND=noninteractive \ - && case "$(dpkg --print-architecture)" in \ - arm64) repo_arch=sbsa ;; \ - amd64) repo_arch=x86_64 ;; \ - *) echo "unsupported arch: $(dpkg --print-architecture)" >&2; exit 1 ;; \ - esac \ - && wget -q -O /tmp/cuda-keyring.deb \ - "https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2404/${repo_arch}/cuda-keyring_1.1-1_all.deb" \ - && dpkg -i /tmp/cuda-keyring.deb \ - && rm -f /tmp/cuda-keyring.deb \ - && apt-get update \ - && apt-get install -y --no-install-recommends libnccl2 libnccl-dev \ - && test -e "/usr/lib/$(uname -m)-linux-gnu/libnccl.so" \ - && rm -f /etc/apt/sources.list.d/cuda*.list /etc/apt/sources.list.d/cuda*.sources \ - && dpkg -r cuda-keyring \ - && rm -rf /var/lib/apt/lists/* - -ENV PATH=/usr/local/cuda/bin:${PATH} \ - LD_LIBRARY_PATH=/usr/local/cuda/lib64 \ - NVIDIA_VISIBLE_DEVICES=all \ - NVIDIA_DRIVER_CAPABILITIES=compute,utility - - -######################################################## -# Base stage -######################################################## -FROM cuda_base AS base +ARG CUDA_VERSION=13.4.1 +FROM nvidia/cuda:${CUDA_VERSION}-cudnn-devel-ubuntu24.04 AS base ARG TARGETARCH ARG BUILD_TYPE=all ARG BRANCH_TYPE=remote -ARG SGL_KERNEL_VERSION=0.4.6.post1 +ARG SGL_KERNEL_VERSION=0.4.7 ARG SGL_VERSION -ARG SGL_DEEP_GEMM_VERSION=0.1.5.post2 +ARG SGL_DEEP_GEMM_VERSION=0.2.0 ARG USE_LATEST_SGLANG=0 ARG GDRCOPY_VERSION=2.5.1 -ARG NCCL_VERSION=2.30.7 +ARG SGL_NCCL_VERSION=2.30.7 ARG PIP_DEFAULT_INDEX ARG UBUNTU_MIRROR ARG GITHUB_ARTIFACTORY=github.com @@ -351,7 +279,7 @@ RUN --mount=type=cache,id=cu134-sgl-kernel-ccache,target=/ccache \ FROM cu134_wheel_base AS cu134_wheel_deepgemm ARG DEEPGEMM_REF -ARG SGL_DEEP_GEMM_VERSION=0.1.5.post2 +ARG SGL_DEEP_GEMM_VERSION=0.2.0 ARG TVM_FFI_VERSION=0.1.11 # DeepJIT's exception handling requires elfutils/libdwfl.h and libelf headers. @@ -370,8 +298,6 @@ COPY scripts/rename_sgl_deep_gemm_whl.sh /rename_sgl_deep_gemm_whl.sh RUN --mount=type=cache,id=cu134-wheels-pip,target=/root/.cache/pip \ set -eux; \ cd /deepgemm; \ - sed -i "s/'-std=c++17'/'-std=c++20'/g" \ - build_sgl_deep_gemm.sh setup.py sgl_deep_gemm/__init__.py; \ printf '%s' "${SGL_DEEP_GEMM_VERSION}" > sgl_deep_gemm/VERSION; \ bash build_sgl_deep_gemm.sh; \ bash /rename_sgl_deep_gemm_whl.sh dist cu134 "$(uname -m)"; \ @@ -414,7 +340,6 @@ RUN git clone --recursive --depth 1 --branch "${DEEPEP_SOURCE_REF}" \ RUN set -eux; \ printf '%s' "${SGL_DEEP_EP_VERSION}" > /deepep-packaging/sgl_deep_ep/VERSION; \ - sed -i 's/"-std=c++17"/"-std=c++20"/g' /deepep-source/setup.py; \ ${PYTHON_ROOT_PATH}/bin/python - <<'PY' import pathlib p = pathlib.Path("/deepep-packaging/sgl_deep_ep/build_sgl_deep_ep.sh") @@ -473,7 +398,9 @@ ARG CUDA_VERSION ARG BUILD_TYPE ARG SGL_KERNEL_VERSION ARG GITHUB_ARTIFACTORY -ARG NCCL_VERSION +# Renamed from NCCL_VERSION: the nvidia/cuda base image sets ENV NCCL_VERSION +# (Debian apt format, e.g. 2.31.2-1), which shadows a same-named ARG inside RUN. +ARG SGL_NCCL_VERSION ARG TORCH_NIGHTLY_INDEX ARG TORCH_NIGHTLY_VERSION ARG TORCHVISION_NIGHTLY_VERSION @@ -496,14 +423,14 @@ RUN --mount=type=cache,target=/root/.cache/pip \ python3 -m pip install --upgrade pip setuptools wheel html5lib six \ && case "$CUDA_VERSION" in \ 13.0.3) CUINDEX=130 ;; \ - 13.4.0) CUINDEX=134 ;; \ + 13.4.1) CUINDEX=134 ;; \ *) echo "Unsupported CUDA version: $CUDA_VERSION" && exit 1 ;; \ esac \ && if [ "$CUDA_VERSION" = "13.0.3" ]; then \ # --no-deps prevents pip from pulling torch from default PyPI python3 -m pip install sglang-kernel==${SGL_KERNEL_VERSION} --force-reinstall --no-deps \ ; \ - elif [ "$CUDA_VERSION" = "13.4.0" ]; then \ + elif [ "$CUDA_VERSION" = "13.4.1" ]; then \ # Use locally built sgl-* wheels. python3 -m pip install --force-reinstall --no-deps /tmp/cu134_wheels/*.whl \ ; \ @@ -527,7 +454,7 @@ RUN --mount=type=cache,target=/root/.cache/pip \ --mount=type=cache,target=/root/.cargo/registry \ case "$CUDA_VERSION" in \ 13.0.3) CUINDEX=130 ;; \ - 13.4.0) CUINDEX=134 ;; \ + 13.4.1) CUINDEX=134 ;; \ *) echo "Unsupported CUDA version: $CUDA_VERSION" && exit 1 ;; \ esac \ && cd /tmp/sglang_deps/python \ @@ -536,10 +463,10 @@ RUN --mount=type=cache,target=/root/.cache/pip \ && echo '__version__ = "0.0.0"' > sglang/version.py \ && touch README.md \ && touch LICENSE \ - && if [ "$CUDA_VERSION" = "13.4.0" ]; then \ + && if [ "$CUDA_VERSION" = "13.4.1" ]; then \ sed -i "s|^ \"flashinfer_python\[cu13\]==.*| \"flashinfer_python[cu13]==${FLASHINFER_VERSION}\",|" pyproject.toml; \ fi \ - && if [ "$CUDA_VERSION" = "13.4.0" ]; then \ + && if [ "$CUDA_VERSION" = "13.4.1" ]; then \ TORCH_INDEX="${TORCH_NIGHTLY_INDEX}"; \ python3 -m pip install --index-url "${TORCH_NIGHTLY_INDEX}" \ "torch==${TORCH_NIGHTLY_VERSION}" \ @@ -557,8 +484,8 @@ RUN --mount=type=cache,target=/root/.cache/pip \ fi \ && python3 -m pip install --extra-index-url "${TORCH_INDEX}" ".[${BUILD_TYPE}]" \ && python3 -m pip install --force-reinstall --no-deps \ - "nvidia-nccl-cu13==${NCCL_VERSION}" \ - && if [ "$CUDA_VERSION" = "13.4.0" ]; then \ + "nvidia-nccl-cu13==${SGL_NCCL_VERSION}" \ + && if [ "$CUDA_VERSION" = "13.4.1" ]; then \ python3 -m pip install --force-reinstall --no-deps /tmp/cu134_wheels/*.whl; \ fi \ && cd /sgl-workspace \ @@ -571,7 +498,7 @@ RUN python3 -m pip install --ignore-installed --no-deps distro # constraints.txt pins the cu134 nightly torch, which does not exist on PyPI; # later stages cannot satisfy it without the nightly index in scope. -RUN if [ "$CUDA_VERSION" = "13.4.0" ]; then \ +RUN if [ "$CUDA_VERSION" = "13.4.1" ]; then \ python3 -m pip config set global.extra-index-url "${TORCH_NIGHTLY_INDEX}"; \ fi @@ -612,7 +539,7 @@ ARG FLASHINFER_VERSION RUN --mount=type=cache,target=/root/.cache/pip \ case "$CUDA_VERSION" in \ 13.0.3) CUINDEX=130 ;; \ - 13.4.0) CUINDEX=134 ;; \ + 13.4.1) CUINDEX=134 ;; \ *) echo "Unsupported CUDA version: $CUDA_VERSION" && exit 1 ;; \ esac \ && mkdir -p /flashinfer_jit_output \ @@ -818,7 +745,7 @@ RUN --mount=type=cache,target=/root/.cache/pip \ # one, so the constraints-pinned cu134 nightly must be restated here; otherwise # runai-model-streamer's "torch<3.0.0,>=2.0.0" backtracks the whole resolve. RUN --mount=type=cache,target=/root/.cache/pip \ - if [ "$CUDA_VERSION" = "13.4.0" ]; then TORCH_SPEC="torch==${TORCH_NIGHTLY_VERSION}"; else TORCH_SPEC=""; fi \ + if [ "$CUDA_VERSION" = "13.4.1" ]; then TORCH_SPEC="torch==${TORCH_NIGHTLY_VERSION}"; else TORCH_SPEC=""; fi \ && python3 -m pip install -c /sgl-workspace/constraints.txt \ ${TORCH_SPEC} \ datamodel_code_generator \ @@ -917,7 +844,7 @@ RUN if [ "$BRANCH_TYPE" = "local" ]; then \ # Clean up __pycache__/tests/pyc in same RUN to avoid writing ~28k files to layer RUN --mount=type=cache,target=/root/.cache/pip \ cd /sgl-workspace/sglang \ - && if [ "$CUDA_VERSION" = "13.4.0" ]; then \ + && if [ "$CUDA_VERSION" = "13.4.1" ]; then \ sed -i "s|^ \"flashinfer_python\[cu13\]==.*| \"flashinfer_python[cu13]==${FLASHINFER_VERSION}\",|" python/pyproject.toml; \ fi \ && python3 -m pip install --no-deps -e "python[${BUILD_TYPE}]" \ @@ -958,7 +885,7 @@ RUN --mount=type=cache,target=/root/.cache/pip \ # quack-kernels hard-pins nvidia-cutlass-dsl==4.6.2, override. RUN --mount=type=cache,target=/root/.cache/pip \ - if [ "$CUDA_VERSION" = "13.4.0" ]; then \ + if [ "$CUDA_VERSION" = "13.4.1" ]; then \ python3 -m pip uninstall -y \ nvidia-cutlass-dsl \ nvidia-cutlass-dsl-libs-base \ @@ -1005,7 +932,7 @@ LABEL org.opencontainers.image.source="https://github.com/sgl-project/sglang" \ # - Ready-to-run inference server environment # # Note: Uses devel base for complete NVCC toolchain required by DeepGEMM JIT -FROM cuda_base AS runtime +FROM nvidia/cuda:${CUDA_VERSION}-cudnn-devel-ubuntu24.04 AS runtime ARG CUDA_VERSION ARG TARGETARCH