diff --git a/.github/workflows/_pr-test-rust-ext-build.yml b/.github/workflows/_pr-test-rust-ext-build.yml
index 9c2034f6a4..73d19a6024 100644
--- a/.github/workflows/_pr-test-rust-ext-build.yml
+++ b/.github/workflows/_pr-test-rust-ext-build.yml
@@ -214,7 +214,7 @@ jobs:
uv venv "${venv}" --python "${{ steps.py.outputs.python-path }}" --seed
# shellcheck disable=SC1091
source "${venv}/bin/activate"
- uv pip install "setuptools>=61.0" "setuptools-rust>=1.11" "setuptools-scm>=8.0" "torch==2.13.0" wheel
+ uv pip install "setuptools>=61.0" "setuptools-rust>=1.11" "setuptools-scm>=8.0" "torch==2.14.1" wheel
# torch-sys resolves libtorch from the active interpreter and bakes that
# path into the cargo cache. venv is per-run and deleted on exit, so a
# persistent target dir would reuse a "Fresh torch-sys" whose -L points at
@@ -250,7 +250,7 @@ jobs:
root / "python/sglang/srt/rust_extensions/torch_build.py"
)
build = helper["torch_build_configuration"](
- compat_header=root / "rust/sglang-radix-tree/torch_2_13_compat.h",
+ compat_header=root / "rust/sglang-radix-tree/torch_compat.h",
python_module="sglang.srt.mem_cache.rust_tree_core.mem_cache",
torch_module=torch,
include_absolute_rpath=False,
diff --git a/.github/workflows/release-pypi-nightly.yml b/.github/workflows/release-pypi-nightly.yml
index 8a069e5de8..68da0726b8 100644
--- a/.github/workflows/release-pypi-nightly.yml
+++ b/.github/workflows/release-pypi-nightly.yml
@@ -44,7 +44,7 @@ jobs:
run: |
pip install \
auditwheel build patchelf wheel "setuptools>=61.0" \
- "setuptools-rust>=1.11" "setuptools-scm>=8.0" "torch==2.13.0"
+ "setuptools-rust>=1.11" "setuptools-scm>=8.0" "torch==2.14.1"
# Needed by setuptools-rust to build the bundled native gRPC extension
# (rust/sglang-grpc) when `python -m build` builds the sglang wheel.
diff --git a/.github/workflows/release-pypi-pr.yml b/.github/workflows/release-pypi-pr.yml
index 627f3750cb..4ac208a91c 100644
--- a/.github/workflows/release-pypi-pr.yml
+++ b/.github/workflows/release-pypi-pr.yml
@@ -84,7 +84,7 @@ jobs:
run: |
pip install \
auditwheel build patchelf wheel "setuptools>=61.0" \
- "setuptools-rust>=1.11" "setuptools-scm>=8.0" "torch==2.13.0"
+ "setuptools-rust>=1.11" "setuptools-scm>=8.0" "torch==2.14.1"
- name: Build wheel
run: |
diff --git a/.github/workflows/release-pypi.yml b/.github/workflows/release-pypi.yml
index 591bcde68a..06f255d75e 100644
--- a/.github/workflows/release-pypi.yml
+++ b/.github/workflows/release-pypi.yml
@@ -63,7 +63,7 @@ jobs:
cp ../README.md ../LICENSE .
pip install \
build wheel "setuptools>=61.0" "setuptools-rust>=1.11" \
- "setuptools-scm>=8.0" "torch==2.13.0"
+ "setuptools-scm>=8.0" "torch==2.14.1"
if [ -n "$RELEASE_VERSION" ]; then
export SETUPTOOLS_SCM_PRETEND_VERSION="${RELEASE_VERSION#v}"
echo "Pinning wheel version to $SETUPTOOLS_SCM_PRETEND_VERSION"
diff --git a/.github/workflows/release-whl-deepep.yml b/.github/workflows/release-whl-deepep.yml
index 93b71f8aca..c988201c45 100644
--- a/.github/workflows/release-whl-deepep.yml
+++ b/.github/workflows/release-whl-deepep.yml
@@ -18,7 +18,7 @@ concurrency:
cancel-in-progress: false
env:
- TORCH_VERSION: "2.13.0"
+ TORCH_VERSION: "2.14.1"
jobs:
build-cu130-matrix:
diff --git a/.github/workflows/release-whl-deepgemm.yml b/.github/workflows/release-whl-deepgemm.yml
index 7037bce67b..147df91ab9 100644
--- a/.github/workflows/release-whl-deepgemm.yml
+++ b/.github/workflows/release-whl-deepgemm.yml
@@ -25,6 +25,11 @@ on:
type: boolean
required: false
default: false
+ skip_tests:
+ description: "Skip the GPU test suite (publishes untested wheels)"
+ type: boolean
+ required: false
+ default: false
concurrency:
group: release-sgl-deepgemm-${{ github.ref }}
@@ -34,7 +39,7 @@ env:
NCCL_NVLS_ENABLE: "0"
# Must match TORCH_VER in docker/sgl-deep-gemm.Dockerfile — the wheel's
# pre-compiled _C.so links against this torch ABI.
- TORCH_VER: "2.13.0"
+ TORCH_VER: "2.14.1"
jobs:
build-cu130-matrix:
@@ -98,6 +103,7 @@ jobs:
test-cu130:
needs: build-cu130-matrix
+ if: ${{ !inputs.skip_tests }}
strategy:
fail-fast: false
matrix:
@@ -146,20 +152,26 @@ jobs:
# deep_gemm imports torch before loading _C.so, so torch must preload the
# libcudart the cu130 wheel links. Pin the CUDA-matched torch (see sgl-deep-gemm.Dockerfile).
python3 -m pip install "torch==${TORCH_VER}" --index-url "https://download.pytorch.org/whl/${CU_TAG}" --force-reinstall
- python3 -m pip install numpy "tilelang==0.1.9" "tile-kernels==1.0.0" "sgl-deep-ep==0.1.2"
- python3 -m pip install dist/*.whl
- # Torch pins NCCL 2.29.7, but this DeepEP wheel requires 2.30.7.
- # Override it after every dependency-resolving pip command.
+ # sgl-deep-ep's own torch pin overrides the line above; it must pin TORCH_VER.
+ python3 -m pip install numpy "tilelang==0.1.9" "tile-kernels==1.0.0" "sgl-deep-ep==0.1.2.post1"
+ # --force-reinstall: a base image may already carry this exact version,
+ # in which case pip skips the wheel and the job tests a stale _C.so.
+ python3 -m pip install dist/*.whl --force-reinstall --no-deps
+ # This DeepEP wheel requires NCCL 2.30.7, which Torch 2.14 also pins;
+ # reinstall it after every dependency-resolving pip command.
python3 -m pip install "nvidia-nccl-cu13==2.30.7" --force-reinstall --no-deps
nccl_lib=$(python3 -c "from importlib.metadata import distribution; print(distribution('nvidia-nccl-cu13').locate_file('nvidia/nccl/lib'))")
export LD_LIBRARY_PATH="${nccl_lib}${LD_LIBRARY_PATH:+:${LD_LIBRARY_PATH}}"
echo "LD_LIBRARY_PATH=${LD_LIBRARY_PATH}" >> "${GITHUB_ENV}"
python3 -c "import deep_gemm; print('deep_gemm:', deep_gemm.__file__)"
- python3 - <<'PY'
+ # tvm-ffi loads _C.so lazily and a later RTLD_NOW dlopen resolves nothing,
+ # so bind eagerly to fail here on any missing symbol.
+ LD_BIND_NOW=1 python3 - <<'PY'
import ctypes
import sys
import torch
import deep_ep
+ import deep_gemm
from tile_kernels.moe import top2_sum_gate
# torch.cuda.nccl.version() reports Torch's compiled version, not the loaded library.
@@ -212,7 +224,10 @@ jobs:
"${{ github.workspace }}/DeepGEMM/sgl_deep_gemm/run_tests.sh" --release
release-cu130:
- if: ${{ !inputs.validation_only }}
+ if: |
+ !cancelled() && !inputs.validation_only &&
+ needs.build-cu130-matrix.result == 'success' &&
+ (needs.test-cu130.result == 'success' || needs.test-cu130.result == 'skipped')
needs: [build-cu130-matrix, test-cu130]
runs-on: ubuntu-latest
steps:
@@ -232,6 +247,12 @@ jobs:
merge-multiple: true
pattern: deepgemm-pypi-cuda13.0-*
+ - name: Record that the GPU test suite was skipped
+ if: ${{ inputs.skip_tests }}
+ run: |
+ echo "Releasing v${{ inputs.version }} with the GPU test suite skipped." \
+ >> "$GITHUB_STEP_SUMMARY"
+
- name: Upload to PyPI
run: |
pip install --upgrade twine "packaging>=24.2"
@@ -260,5 +281,5 @@ jobs:
git config --local user.name "sglang-bot"
git config --local user.email "sglangbot@gmail.com"
git add -A
- git commit -m "update sgl-deep-gemm whl index for v${{ inputs.version }}"
+ git commit -m "update sgl-deep-gemm whl index for v${{ inputs.version }}${{ inputs.skip_tests && ' (GPU tests skipped)' || '' }}"
git push
diff --git a/docker/Dockerfile b/docker/Dockerfile
index 6176ead24e..8a5e2d4877 100644
--- a/docker/Dockerfile
+++ b/docker/Dockerfile
@@ -4,7 +4,7 @@ FROM nvidia/cuda:${CUDA_VERSION}-cudnn-devel-ubuntu24.04 AS base
ARG TARGETARCH
ARG BUILD_TYPE=all
ARG BRANCH_TYPE=remote
-ARG SGL_KERNEL_VERSION=0.4.8
+ARG SGL_KERNEL_VERSION=0.4.9
ARG SGL_VERSION
ARG USE_LATEST_SGLANG=0
ARG GDRCOPY_VERSION=2.5.1
@@ -498,9 +498,8 @@ RUN --mount=type=cache,target=/root/.cache/pip \
termplotlib \
"runai-model-streamer[s3,gcs,azure]>=0.15.7"
-# PyTorch 2.13 requires nvidia-nccl-cu13==2.29.7, so keep that version while
-# resolving dependencies and generating constraints.txt. DeepEP needs 2.30.7
-# at runtime; apply the override only after the constrained dependency solve.
+# DeepEP needs SGL_NCCL_VERSION at runtime even if torch pins another version;
+# install it after the constrained dependency solve so the solve cannot revert it.
RUN --mount=type=cache,target=/root/.cache/pip \
python3 -m pip install --force-reinstall --no-deps \
"nvidia-nccl-cu13==${SGL_NCCL_VERSION}"
diff --git a/docker/sgl-deep-ep.Dockerfile b/docker/sgl-deep-ep.Dockerfile
index 4ccf930f32..28cfb3806a 100644
--- a/docker/sgl-deep-ep.Dockerfile
+++ b/docker/sgl-deep-ep.Dockerfile
@@ -8,7 +8,7 @@ ARG CUDA_TAG=cu130
ARG CUDA_VERSION=13.0
ARG NCCL_VERSION=2.30.7
ARG PYTHON_TAG=cp312-cp312
-ARG TORCH_VERSION=2.13.0
+ARG TORCH_VERSION=2.14.1
ENV CUDA_HOME=/usr/local/cuda
ENV LD_LIBRARY_PATH=/usr/local/lib:/usr/local/lib64:/usr/local/cuda/lib64:${LD_LIBRARY_PATH}
diff --git a/docker/sgl-deep-gemm.Dockerfile b/docker/sgl-deep-gemm.Dockerfile
index 6d89bf884e..81466f3a44 100644
--- a/docker/sgl-deep-gemm.Dockerfile
+++ b/docker/sgl-deep-gemm.Dockerfile
@@ -7,7 +7,7 @@ ARG ARCH=x86_64
ARG CUDA_VERSION=13.0
ARG PYTHON_VERSION=3.12
ARG PYTHON_TAG=cp312-cp312
-ARG TORCH_VER=2.13.0
+ARG TORCH_VER=2.14.1
ARG TVM_FFI_VER=0.1.11
ARG PIP_DEFAULT_INDEX=https://pypi.python.org/simple
ARG PYTORCH_MIRROR=download.pytorch.org
diff --git a/docs/docs/get-started/install.mdx b/docs/docs/get-started/install.mdx
index b32da3dc88..e24082785f 100644
--- a/docs/docs/get-started/install.mdx
+++ b/docs/docs/get-started/install.mdx
@@ -65,7 +65,7 @@ Use Python 3.10 or higher and a compatible CUDA 13 environment.
-SGLang requires CUDA 13. CUDA 12 (`cu129`) wheels and images are retired because PyTorch 2.14 publishes no CUDA 12.9 builds. SGLang 0.5.19 is the last release with a CUDA 12 lane.
+SGLang requires CUDA 13. CUDA 12 (`cu129`) wheels and images were retired with the upgrade to PyTorch 2.14, which publishes no CUDA 12.9 builds. SGLang 0.5.19 is the last release with a CUDA 12 lane (PyTorch 2.13).
diff --git a/python/pyproject.toml b/python/pyproject.toml
index 1e53fcfd62..2f9a520060 100755
--- a/python/pyproject.toml
+++ b/python/pyproject.toml
@@ -3,7 +3,7 @@ requires = [
"setuptools>=61.0",
"setuptools-rust>=1.11",
"setuptools-scm>=8.0",
- "torch==2.13.0",
+ "torch==2.14.1",
"wheel",
]
build-backend = "setuptools.build_meta"
@@ -76,9 +76,9 @@ dependencies = [
"scipy",
"sentencepiece==0.2.1", # 0.2.2 rejects null pieces in InternVL tokenizers.
"setproctitle",
- "sgl-deep-ep==0.1.2",
- "sgl-deep-gemm==0.2.0",
- "sglang-kernel==0.4.8",
+ "sgl-deep-ep==0.1.2.post1",
+ "sgl-deep-gemm==0.2.1.post1",
+ "sglang-kernel==0.4.9",
"smg-grpc-servicer>=0.9.0",
"soundfile==0.13.1",
"tiktoken",
@@ -87,10 +87,10 @@ dependencies = [
"tokenizers==0.23.2",
"tokenspeed_mla==0.1.8",
"tomli ; python_version < '3.11'",
- "torch==2.13.0",
+ "torch==2.14.1",
"torch_memory_saver>=0.0.9.post1",
"torchaudio==2.11.0",
- "torchcodec==0.15.0 ; sys_platform != 'linux' or (sys_platform == 'linux' and platform_machine != 'aarch64' and platform_machine != 'arm64' and platform_machine != 'armv7l')", # Not available on Linux ARM.
+ "torchcodec==0.17.0 ; sys_platform != 'linux' or (sys_platform == 'linux' and platform_machine != 'aarch64' and platform_machine != 'arm64' and platform_machine != 'armv7l')", # Not available on Linux ARM.
"torchvision",
"tqdm",
"transformers==5.17.0",
diff --git a/python/sglang/kernels/ops/attention/decode_attention.py b/python/sglang/kernels/ops/attention/decode_attention.py
index f711172ea3..24ffe864e4 100644
--- a/python/sglang/kernels/ops/attention/decode_attention.py
+++ b/python/sglang/kernels/ops/attention/decode_attention.py
@@ -332,7 +332,9 @@ def _fwd_kernel_stage1(
mask=(offs_n[:, None] < split_kv_end) & (mask_d[None, :]),
other=0.0,
)
- qk = tl.sum(q[None, :] * k, 1)
+ # Reduce in fp32; a bf16 tl.sum rounds each partial sum in compiler order.
+ # Cast the product, not q and k, to avoid a fp32 [BLOCK_N, D] tile.
+ qk = tl.sum((q[None, :] * k).to(tl.float32), 1)
qk *= sm_scale_withk
if logit_cap > 0:
diff --git a/python/sglang/kernels/ops/attention/dsv4/decode_attention_sm100_gluon.py b/python/sglang/kernels/ops/attention/dsv4/decode_attention_sm100_gluon.py
index afe4617a13..ec34f68e12 100644
--- a/python/sglang/kernels/ops/attention/dsv4/decode_attention_sm100_gluon.py
+++ b/python/sglang/kernels/ops/attention/dsv4/decode_attention_sm100_gluon.py
@@ -16,8 +16,8 @@ try:
from triton.experimental.gluon.language.nvidia.blackwell import get_tmem_reg_layout
except ImportError:
# Triton 3.8 moved this onto the descriptor type (triton-lang/triton#9594)
- # TODO(tmorris): After we update to triton 3.8 (torch 2.15), use new API directly
- # instead of this compatibility wrapper.
+ # TODO: once ROCm leaves triton 3.7 (swapab_gluon_hip imports _load_v4
+ # from here), drop the try import above and call get_reg_layout directly.
from triton.experimental.gluon.language.nvidia.blackwell import (
tensor_memory_descriptor_type,
)
diff --git a/python/sglang/multimodal_gen/runtime/layers/attention/backends/sdpa.py b/python/sglang/multimodal_gen/runtime/layers/attention/backends/sdpa.py
index d185f9acec..7481a5b41f 100644
--- a/python/sglang/multimodal_gen/runtime/layers/attention/backends/sdpa.py
+++ b/python/sglang/multimodal_gen/runtime/layers/attention/backends/sdpa.py
@@ -152,7 +152,9 @@ class SDPAImpl(AttentionImpl):
):
# Keep the existing Flash SDPA arithmetic while consuming all
# packed windows in one call, including ragged and empty windows.
- return torch_varlen_attn(
+ # Call the Flash op directly: from torch 2.14, varlen_attn silently
+ # routes sm90/sm100 to cuDNN, which changes outputs versus the Flash path.
+ output, *_ = torch.ops.aten._flash_attention_forward(
query,
key,
value,
@@ -160,9 +162,14 @@ class SDPAImpl(AttentionImpl):
cu_seqlens,
max_seqlen,
max_seqlen,
+ 0.0,
+ self.causal,
+ return_debug_mask=False,
scale=self.softmax_scale,
- window_size=(-1, 0) if self.causal else (-1, -1),
+ window_size_left=-1,
+ window_size_right=0 if self.causal else -1,
)
+ return output
bounds = (
cu_seqlens_host
if cu_seqlens_host is not None
diff --git a/python/sglang/multimodal_gen/test/server/consistency_thresholds/h100.json b/python/sglang/multimodal_gen/test/server/consistency_thresholds/h100.json
index df68fcdff8..270ef2b809 100644
--- a/python/sglang/multimodal_gen/test/server/consistency_thresholds/h100.json
+++ b/python/sglang/multimodal_gen/test/server/consistency_thresholds/h100.json
@@ -40,8 +40,8 @@
"zimage_image_t2i": {
"clip_threshold": 0.97,
"ssim_threshold": 0.86,
- "psnr_threshold": 19.9,
- "mean_abs_diff_threshold": 8.5
+ "psnr_threshold": 19.3,
+ "mean_abs_diff_threshold": 9.8
},
"zimage_image_t2i_multi_lora": {
"clip_threshold": 0.98,
diff --git a/python/sglang/multimodal_gen/test/test_utils.py b/python/sglang/multimodal_gen/test/test_utils.py
index f7196de4d5..8abac1a7f6 100644
--- a/python/sglang/multimodal_gen/test/test_utils.py
+++ b/python/sglang/multimodal_gen/test/test_utils.py
@@ -40,7 +40,7 @@ logger = init_logger(__name__)
# NPU/ascend) is read from sgl-project/ci-data-diffusion, where the GT-gen workflows
# publish.
SGL_TEST_FILES_CI_DATA_REPO = "sgl-project/ci-data-diffusion"
-SGL_TEST_FILES_CI_DATA_REVISION = "dbb70135da54b1f2b172886b28e813d2bcdd16c1"
+SGL_TEST_FILES_CI_DATA_REVISION = "972007201d82d7dcce9e35b7e4b151c2b1181c30"
# The NPU pin is kept as a separate branch so ascend GT can be bumped independently
# when it's regenerated on its own cadence.
diff --git a/python/sglang/multimodal_gen/test/unit/test_ltx2_vae_channels_last.py b/python/sglang/multimodal_gen/test/unit/test_ltx2_vae_channels_last.py
index 1eea62767d..d61382f4ca 100644
--- a/python/sglang/multimodal_gen/test/unit/test_ltx2_vae_channels_last.py
+++ b/python/sglang/multimodal_gen/test/unit/test_ltx2_vae_channels_last.py
@@ -13,6 +13,13 @@ class TestLTX2CausalConvChannelsLast(unittest.TestCase):
the original repeat()+concatenate() temporal padding (only the conv kernel's
floating-point accumulation order may differ, which fp32 keeps negligible)."""
+ def setUp(self):
+ # From torch 2.14 (cuDNN 9.24), cuDNN picks a TF32 kernel for the
+ # channels_last_3d conv, which this fp32 comparison would read as drift.
+ flags = torch.backends.cudnn.flags(enabled=True, allow_tf32=False)
+ flags.__enter__()
+ self.addCleanup(flags.__exit__, None, None, None)
+
def _check(self, causal: bool, kernel_size):
device = "cuda" if torch.cuda.is_available() else "cpu"
torch.manual_seed(0)
diff --git a/python/sglang/srt/entrypoints/engine.py b/python/sglang/srt/entrypoints/engine.py
index 85a23dc6e1..47bad3e6ee 100644
--- a/python/sglang/srt/entrypoints/engine.py
+++ b/python/sglang/srt/entrypoints/engine.py
@@ -1856,7 +1856,7 @@ def _set_envs_and_config(server_args: ServerArgs):
if _is_cuda:
assert_pkg_version(
"sglang-kernel",
- "0.4.8",
+ "0.4.9",
"Please reinstall the latest version with `pip install sglang-kernel --force-reinstall`",
)
diff --git a/python/sglang/srt/layers/moe/topk.py b/python/sglang/srt/layers/moe/topk.py
index 801bbb6b8b..4cde43a1ec 100644
--- a/python/sglang/srt/layers/moe/topk.py
+++ b/python/sglang/srt/layers/moe/topk.py
@@ -59,7 +59,7 @@ try:
):
if simulated_ep != 1:
raise NotImplementedError(
- "simulated_ep routing is not supported with triton_kernels 3.7.1"
+ "simulated_ep routing is not supported with triton_kernels"
)
if sm_first:
diff --git a/python/sglang/srt/layers/quantization/mxfp4.py b/python/sglang/srt/layers/quantization/mxfp4.py
index bd79467318..81883eb084 100644
--- a/python/sglang/srt/layers/quantization/mxfp4.py
+++ b/python/sglang/srt/layers/quantization/mxfp4.py
@@ -18,7 +18,7 @@
from __future__ import annotations
import os
-from dataclasses import replace
+from dataclasses import fields, replace
from functools import lru_cache
from typing import TYPE_CHECKING, List, Optional
@@ -227,6 +227,20 @@ def _pad_hopper_mxfp4_scale(scale, k_size):
return torch.nn.functional.pad(scale, (0, want - scale.shape[-1]), value=_UE8M0_ONE)
+@lru_cache(maxsize=1)
+def _mx_block_kwargs():
+ # triton_kernels 3.8 rejects an mx scale without its block size; the field is
+ # absent in 3.7, which ROCm still ships, so probe before importing the value.
+ from triton_kernels.matmul import PrecisionConfig
+
+ if "b_microblock_size" not in {f.name for f in fields(PrecisionConfig)}:
+ return {}
+
+ from triton_kernels.numerics_details.mxfp import MXFP_BLOCK_SIZE
+
+ return {"b_microblock_size": int(MXFP_BLOCK_SIZE)}
+
+
def _swizzle_mxfp4(quant_tensor, scale, num_warps):
"""weight swizzle for mxfp4 moe, used for OAI mxfp4 kernel"""
import triton_kernels.matmul_details.opt_flags as opt_flags
@@ -1042,11 +1056,12 @@ class Mxfp4MoEMethod(FusedMoEMethodBase):
layer.w2_weight, layer.w2_weight_scale, num_warps
)
+ mx_block = _mx_block_kwargs()
self.w13_precision_config = PrecisionConfig(
- b_mx_scale=w13_scale, flex_ctx=FlexCtx(rhs_data=w13_flex)
+ b_mx_scale=w13_scale, flex_ctx=FlexCtx(rhs_data=w13_flex), **mx_block
)
self.w2_precision_config = PrecisionConfig(
- b_mx_scale=w2_scale, flex_ctx=FlexCtx(rhs_data=w2_flex)
+ b_mx_scale=w2_scale, flex_ctx=FlexCtx(rhs_data=w2_flex), **mx_block
)
self.w13_weight_triton_tensor = w13_weight
diff --git a/python/sglang/srt/mem_cache/rust_tree_core/extension.py b/python/sglang/srt/mem_cache/rust_tree_core/extension.py
index d2b05dd464..c42d7b57b5 100644
--- a/python/sglang/srt/mem_cache/rust_tree_core/extension.py
+++ b/python/sglang/srt/mem_cache/rust_tree_core/extension.py
@@ -11,7 +11,7 @@ from sglang.srt.rust_extensions.torch_build import torch_build_configuration
_PYTHON_MODULE = "sglang.srt.mem_cache.rust_tree_core.mem_cache"
_INSPECTION_MODULE = "sglang.srt.mem_cache.rust_tree_core.mem_cache_inspection"
_CRATE_DIR = Path(__file__).resolve().parents[5] / "rust" / "sglang-radix-tree"
-_TORCH_COMPAT_HEADER = _CRATE_DIR / "torch_2_13_compat.h"
+_TORCH_COMPAT_HEADER = _CRATE_DIR / "torch_compat.h"
def load_tree_core_extension(*, inspection: bool = False):
diff --git a/python/sglang/srt/rust_extensions/torch_build.py b/python/sglang/srt/rust_extensions/torch_build.py
index 65629b90c7..e54c593576 100644
--- a/python/sglang/srt/rust_extensions/torch_build.py
+++ b/python/sglang/srt/rust_extensions/torch_build.py
@@ -13,7 +13,7 @@ from types import ModuleType
from typing import Mapping
_MIN_SUPPORTED_TORCH = (2, 11)
-_MAX_SUPPORTED_TORCH = (2, 13)
+_MAX_SUPPORTED_TORCH = (2, 14)
@dataclass(frozen=True)
@@ -80,7 +80,7 @@ def torch_build_configuration(
environment["LIBTORCH_LIB"] = os.fspath(torch_root)
environment["LIBTORCH_CXX11_ABI"] = "1" if cxx11_abi else "0"
# tch 0.24 targets Torch 2.11. The compatibility header below covers the
- # API removals in the supported 2.12/2.13 builds, after this explicit gate.
+ # API removals in the supported 2.12-2.14 builds, after this explicit gate.
environment["LIBTORCH_BYPASS_VERSION_CHECK"] = "1"
environment["PYO3_PYTHON"] = sys.executable
environment["PATH"] = os.pathsep.join(
@@ -90,10 +90,16 @@ def torch_build_configuration(
filter(None, (os.fspath(torch_lib), environment.get("LD_LIBRARY_PATH")))
)
- cxxflags = environment.get("CXXFLAGS", "")
- environment["CXXFLAGS"] = (
- f"{cxxflags} -include {shlex.quote(os.fspath(compat_header.resolve()))}"
- ).strip()
+ # cc-rs splits *FLAGS on whitespace unless told they are shell words,
+ # which the quoted header path below needs to survive a space.
+ environment["CC_SHELL_ESCAPED_FLAGS"] = "1"
+ cxxflags = [environment.get("CXXFLAGS", "")]
+ # torch-sys 0.24 hardcodes -std=c++17, but libtorch declares C++20 from 2.12 on;
+ # cc-rs appends CXXFLAGS after a build script's own flags, so this one wins.
+ if major_minor >= (2, 12):
+ cxxflags.append("-std=c++20")
+ cxxflags.append(f"-include {shlex.quote(os.fspath(compat_header.resolve()))}")
+ environment["CXXFLAGS"] = " ".join(filter(None, cxxflags)).strip()
package_depth = len(python_module.split(".")) - 1
bundled_torch_lib = "$ORIGIN/" + "../" * package_depth + "torch/lib"
diff --git a/python/sglang/srt/utils/common.py b/python/sglang/srt/utils/common.py
index 2cc2421c52..3f17f45226 100644
--- a/python/sglang/srt/utils/common.py
+++ b/python/sglang/srt/utils/common.py
@@ -595,7 +595,12 @@ def get_device_module():
return torch.xpu
if is_musa():
return torch.musa
- return torch.get_device_module()
+ # From torch 2.14, a bare torch.get_device_module() is torch.cuda on a CUDA wheel
+ # even with no usable device; require an available accelerator instead.
+ accelerator = torch.accelerator.current_accelerator(check_available=True)
+ if accelerator is None:
+ return torch.cpu
+ return torch.get_device_module(accelerator)
def create_device_stream(device):
diff --git a/rust/sglang-radix-tree/Cargo.toml b/rust/sglang-radix-tree/Cargo.toml
index 426fe97d40..ed92327132 100644
--- a/rust/sglang-radix-tree/Cargo.toml
+++ b/rust/sglang-radix-tree/Cargo.toml
@@ -10,7 +10,7 @@ repository = "https://github.com/sgl-project/sglang"
python-module = "sglang.srt.mem_cache.rust_tree_core.mem_cache"
debug = false
features = ["python-extension"]
-torch-compat-header = "torch_2_13_compat.h"
+torch-compat-header = "torch_compat.h"
[lib]
name = "mem_cache"
diff --git a/rust/sglang-radix-tree/README.md b/rust/sglang-radix-tree/README.md
index 82f209fa2d..9c8c77956a 100644
--- a/rust/sglang-radix-tree/README.md
+++ b/rust/sglang-radix-tree/README.md
@@ -11,7 +11,7 @@ Python in these cases:
- C128 or other unsupported components.
- Custom component overrides.
- Non-Linux platforms.
-- PyTorch versions outside 2.11 through 2.13.
+- PyTorch versions outside 2.11 through 2.14.
- Devices other than CPU or CUDA.
- Installations containing neither the Rust extension nor its sources.
- Source builds with a missing or unusable Rust toolchain.
@@ -39,8 +39,10 @@ Standard SGLang wheels bundle the production extension; some platform
distributions omit it. A source checkout falls back to
the shared fingerprinted Rust-extension cache; it never writes a shared object
into the Python package. LibTorch and the Python headers come from the running
-interpreter's PyTorch install. PyTorch 2.11 through 2.13 are accepted explicitly,
-and `torch_2_13_compat.h` covers two alignment APIs removed in PyTorch 2.13.
+interpreter's PyTorch install. PyTorch 2.11 through 2.14 are accepted explicitly,
+and `torch_compat.h` covers two alignment APIs removed in PyTorch 2.13 plus
+the `cholesky` and `qr` aliases removed in 2.14. Torch 2.12 and later declare
+C++20, so the builds below pass `-std=c++20` over torch-sys's own `-std=c++17`.
Source checkouts need working `cargo` and `rustc` to use Rust, including for
fingerprinted cache lookup. If either tool is unavailable, the registry selects
Python. Trusted bundled extensions do not require a Rust compiler.
@@ -60,13 +62,13 @@ tree's mutex; both tree cores implement the same contract.
cd rust/sglang-radix-tree
LIBTORCH_USE_PYTORCH=1 \
LIBTORCH_BYPASS_VERSION_CHECK=1 \
- CXXFLAGS="-include $PWD/torch_2_13_compat.h" \
+ CXXFLAGS="-std=c++20 -include $PWD/torch_compat.h" \
cargo build --release --locked --features python-extension
# Native tests do not enable pyo3's extension-module feature:
TORCH_ROOT=$(python3 -c 'import pathlib, torch; print(pathlib.Path(torch.__file__).parent)')
LIBTORCH_USE_PYTORCH=1 LIBTORCH_BYPASS_VERSION_CHECK=1 \
- CXXFLAGS="-include $PWD/torch_2_13_compat.h" \
+ CXXFLAGS="-std=c++20 -include $PWD/torch_compat.h" \
LD_LIBRARY_PATH="$TORCH_ROOT/lib${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}" \
cargo test --locked
```
diff --git a/rust/sglang-radix-tree/torch_2_13_compat.h b/rust/sglang-radix-tree/torch_2_13_compat.h
deleted file mode 100644
index f70196ec80..0000000000
--- a/rust/sglang-radix-tree/torch_2_13_compat.h
+++ /dev/null
@@ -1,15 +0,0 @@
-#pragma once
-
-#include
-#include
-
-#if TORCH_VERSION_MAJOR > 2 || \
- (TORCH_VERSION_MAJOR == 2 && TORCH_VERSION_MINOR >= 13)
-// Keep tch 0.24's removed alignment wrappers as explicit runtime errors.
-#define align_as(...) \
- alias(); \
- throw std::runtime_error("align_as is unavailable in PyTorch 2.13+")
-#define align_tensors(...) \
- autograd::variable_list{}; \
- throw std::runtime_error("align_tensors is unavailable in PyTorch 2.13+")
-#endif
diff --git a/rust/sglang-radix-tree/torch_compat.h b/rust/sglang-radix-tree/torch_compat.h
new file mode 100644
index 0000000000..564160ab66
--- /dev/null
+++ b/rust/sglang-radix-tree/torch_compat.h
@@ -0,0 +1,57 @@
+#pragma once
+
+#include
+#include
+
+#if TORCH_VERSION_MAJOR > 2 || \
+ (TORCH_VERSION_MAJOR == 2 && TORCH_VERSION_MINOR >= 13)
+// Keep tch 0.24's removed alignment wrappers as explicit runtime errors.
+#define align_as(...) \
+ alias(); \
+ throw std::runtime_error("align_as is unavailable in PyTorch 2.13+")
+#define align_tensors(...) \
+ autograd::variable_list{}; \
+ throw std::runtime_error("align_tensors is unavailable in PyTorch 2.13+")
+#endif
+
+#if TORCH_VERSION_MAJOR > 2 || \
+ (TORCH_VERSION_MAJOR == 2 && TORCH_VERSION_MINOR >= 14)
+// tch 0.24 still binds the `cholesky` and `qr` aliases PyTorch 2.14 removed;
+// linalg differs in errors and qr's signature, so stub rather than remap.
+#include
+
+#include
+
+namespace torch {
+
+inline at::Tensor cholesky_unavailable(const at::Tensor &, bool) {
+ throw std::runtime_error(
+ "cholesky is unavailable in PyTorch 2.14+; use linalg_cholesky");
+}
+
+inline at::Tensor &cholesky_out_unavailable(at::Tensor &, const at::Tensor &,
+ bool) {
+ throw std::runtime_error(
+ "cholesky_out is unavailable in PyTorch 2.14+; use linalg_cholesky_out");
+}
+
+inline std::tuple qr_unavailable(const at::Tensor &,
+ bool) {
+ throw std::runtime_error("qr is unavailable in PyTorch 2.14+; use linalg_qr");
+}
+
+inline std::tuple
+qr_out_unavailable(at::Tensor &, at::Tensor &, const at::Tensor &, bool) {
+ throw std::runtime_error(
+ "qr_out is unavailable in PyTorch 2.14+; use linalg_qr_out");
+}
+
+} // namespace torch
+
+// Safe as global macros only because no PyTorch 2.14 header uses these names;
+// the include above declares at::Tensor for the signatures.
+#define cholesky(...) cholesky_unavailable(__VA_ARGS__)
+#define cholesky_out(...) cholesky_out_unavailable(__VA_ARGS__)
+#define qr(...) qr_unavailable(__VA_ARGS__)
+#define qr_out(...) qr_out_unavailable(__VA_ARGS__)
+#endif
diff --git a/scripts/build_sgl_deepep.sh b/scripts/build_sgl_deepep.sh
index eb76f14a95..f9ad699973 100755
--- a/scripts/build_sgl_deepep.sh
+++ b/scripts/build_sgl_deepep.sh
@@ -111,7 +111,7 @@ docker build \
--build-arg CUDA_TAG="${CUDA_TAG}" \
--build-arg PYTHON_TAG="${PYTHON_TAG}" \
--build-arg ARCHITECTURE="${ARCHITECTURE}" \
- --build-arg TORCH_VERSION="${TORCH_VERSION:-2.13.0}" \
+ --build-arg TORCH_VERSION="${TORCH_VERSION:-2.14.1}" \
--tag "${IMAGE_TAG}" \
--network=host \
"${REPOSITORY_ROOT}/docker"
diff --git a/scripts/ci/cuda/ci_install_dependency.sh b/scripts/ci/cuda/ci_install_dependency.sh
index ee6d1a6f5e..c57c84b14a 100755
--- a/scripts/ci/cuda/ci_install_dependency.sh
+++ b/scripts/ci/cuda/ci_install_dependency.sh
@@ -589,8 +589,8 @@ install_sglang() {
}
install_nccl() {
- # PyTorch pins 2.29.7, so this override must run after every command
- # that resolves Python dependencies (including lmms-eval).
+ # DeepEP needs NCCL 2.30.7 whatever torch pins, so this must run
+ # after every command that resolves Python dependencies (including lmms-eval).
$PIP_CMD install "nvidia-nccl-cu13==2.30.7" \
--force-reinstall --no-deps $PIP_INSTALL_SUFFIX
diff --git a/test/registered/rust/test_run_sglang_radix_tree_rust_tests.py b/test/registered/rust/test_run_sglang_radix_tree_rust_tests.py
index 43f08bd89f..5272a963b5 100644
--- a/test/registered/rust/test_run_sglang_radix_tree_rust_tests.py
+++ b/test/registered/rust/test_run_sglang_radix_tree_rust_tests.py
@@ -34,7 +34,7 @@ class TestSGLangRadixTreeCargo(CustomTestCase):
f"sglang-radix-tree manifest not found at {SGLANG_RADIX_TREE_MANIFEST}",
)
build = torch_build_configuration(
- compat_header=SGLANG_RADIX_TREE_MANIFEST.parent / "torch_2_13_compat.h",
+ compat_header=SGLANG_RADIX_TREE_MANIFEST.parent / "torch_compat.h",
python_module="sglang.srt.mem_cache.rust_tree_core.mem_cache",
)
proc = subprocess.run(
diff --git a/test/registered/rust/test_rust_extension.py b/test/registered/rust/test_rust_extension.py
index 10b5712d62..2873e6c8ea 100644
--- a/test/registered/rust/test_rust_extension.py
+++ b/test/registered/rust/test_rust_extension.py
@@ -201,7 +201,7 @@ crate-type = ["cdylib"]
crate,
features=(*crate.features, "inspection"),
extension_module="demo._inspection",
- build_fingerprint={"torch": "2.13"},
+ build_fingerprint={"torch": "2.14"},
)
self.assertNotEqual(changed_source.fingerprint, inspection.fingerprint)
self.assertNotEqual(
@@ -458,7 +458,7 @@ crate-type = ["cdylib"]
compat_header = root / "compat.h"
compat_header.write_text("// compatibility\n", encoding="utf-8")
fake_torch = SimpleNamespace(
- __version__="2.13.0+cu130",
+ __version__="2.14.1+cu130",
__file__=str(torch_init),
compiled_with_cxx11_abi=lambda: True,
version=SimpleNamespace(cuda="13.0", hip=None),
@@ -483,11 +483,14 @@ crate-type = ["cdylib"]
self.assertEqual(build.environment["LIBTORCH_CXX11_ABI"], "1")
self.assertEqual(build.environment["LIBTORCH_BYPASS_VERSION_CHECK"], "1")
self.assertIn(str(compat_header), build.environment["CXXFLAGS"])
+ # -std=c++20 must trail the inherited CXXFLAGS to beat a caller's -std=;
+ # torch-sys's own -std=c++17 loses anyway, as cc-rs appends CXXFLAGS.
+ self.assertRegex(build.environment["CXXFLAGS"], r"^-O2\b.*\s-std=c\+\+20\b")
self.assertIn(
"$ORIGIN/../../../../torch/lib", build.environment["RUSTFLAGS"]
)
self.assertIn(str(torch_root / "lib"), build.environment["RUSTFLAGS"])
- self.assertEqual(build.fingerprint["torch_version"], "2.13.0+cu130")
+ self.assertEqual(build.fingerprint["torch_version"], "2.14.1+cu130")
self.assertTrue(build.fingerprint["torch_cxx11_abi"])
wheel_build = torch_build_configuration(
@@ -502,8 +505,37 @@ crate-type = ["cdylib"]
)
self.assertFalse(wheel_build.fingerprint["include_absolute_rpath"])
- fake_torch.__version__ = "2.14.0"
- with self.assertRaisesRegex(RuntimeError, "PyTorch 2.11 through 2.13"):
+ # libtorch declares C++20 from 2.12 on; 2.11 is the only C++17 release.
+ for version, expected in (("2.12.0+cu128", True), ("2.11.0+cu128", False)):
+ fake_torch.__version__ = version
+ older_build = torch_build_configuration(
+ compat_header=compat_header,
+ python_module="sglang.srt.mem_cache.rust_tree_core.mem_cache",
+ torch_module=fake_torch,
+ base_environment={},
+ )
+ self.assertEqual(
+ "-std=c++20" in older_build.environment["CXXFLAGS"],
+ expected,
+ msg=version,
+ )
+
+ # cc-rs keeps a quoted, spaced header path whole only in shell-words mode.
+ fake_torch.__version__ = "2.14.1+cu130"
+ spaced_header = root / "compat dir" / "compat.h"
+ spaced_header.parent.mkdir()
+ spaced_header.write_text("// compatibility\n", encoding="utf-8")
+ spaced_build = torch_build_configuration(
+ compat_header=spaced_header,
+ python_module="sglang.srt.mem_cache.rust_tree_core.mem_cache",
+ torch_module=fake_torch,
+ base_environment={},
+ )
+ self.assertEqual(spaced_build.environment["CC_SHELL_ESCAPED_FLAGS"], "1")
+ self.assertIn(f"'{spaced_header}'", spaced_build.environment["CXXFLAGS"])
+
+ fake_torch.__version__ = "2.15.0"
+ with self.assertRaisesRegex(RuntimeError, "PyTorch 2.11 through 2.14"):
torch_build_configuration(
compat_header=compat_header,
python_module="sglang.srt.mem_cache.rust_tree_core.mem_cache",
diff --git a/test/registered/unit/mem_cache/test_tree_core_registry.py b/test/registered/unit/mem_cache/test_tree_core_registry.py
index ad7cb03055..ead7ddbacb 100644
--- a/test/registered/unit/mem_cache/test_tree_core_registry.py
+++ b/test/registered/unit/mem_cache/test_tree_core_registry.py
@@ -280,7 +280,7 @@ class UnifiedRadixCacheTreeCoreSelectionTest(CustomTestCase):
mock.patch.dict(os.environ),
mock.patch.dict(tree_core_registry.sys.modules),
mock.patch.object(tree_core_registry.sys, "platform", "linux"),
- mock.patch.object(tree_core_registry.torch, "__version__", "2.13.0"),
+ mock.patch.object(tree_core_registry.torch, "__version__", "2.14.1"),
mock.patch.object(
tree_core_registry.importlib.util,
"find_spec",
@@ -327,7 +327,7 @@ class UnifiedRadixCacheTreeCoreSelectionTest(CustomTestCase):
mock.patch.dict(tree_core_registry.sys.modules),
mock.patch.object(tree_core_registry.sys, "platform", "linux"),
mock.patch.object(
- tree_core_registry.torch, "__version__", "2.13.0"
+ tree_core_registry.torch, "__version__", "2.14.1"
),
mock.patch.object(
tree_core_registry.importlib.util,
@@ -357,7 +357,7 @@ class TreeCoreDefaultCompatibilityTest(CustomTestCase):
mock.patch.dict(os.environ, {"SGLANG_RUST_BUILD_MODE": "auto"}),
mock.patch.dict(tree_core_registry.sys.modules),
mock.patch.object(tree_core_registry.sys, "platform", "linux"),
- mock.patch.object(tree_core_registry.torch, "__version__", "2.13.0"),
+ mock.patch.object(tree_core_registry.torch, "__version__", "2.14.1"),
mock.patch.object(
tree_core_registry.importlib.util, "find_spec", return_value=None
),
@@ -497,7 +497,7 @@ class TreeCoreDefaultCompatibilityTest(CustomTestCase):
self.assertEqual(
select_tree_core_backend(_cache_init_params()), "python"
)
- for version in ("2.10.0", "2.14.0", "unknown"):
+ for version in ("2.10.0", "2.15.0", "unknown"):
with (
self.subTest(version=version),
mock.patch.object(tree_core_registry.torch, "__version__", version),