diff --git a/.github/workflows/_pr-test-rust-ext-build.yml b/.github/workflows/_pr-test-rust-ext-build.yml index 9c2034f6a4..73d19a6024 100644 --- a/.github/workflows/_pr-test-rust-ext-build.yml +++ b/.github/workflows/_pr-test-rust-ext-build.yml @@ -214,7 +214,7 @@ jobs: uv venv "${venv}" --python "${{ steps.py.outputs.python-path }}" --seed # shellcheck disable=SC1091 source "${venv}/bin/activate" - uv pip install "setuptools>=61.0" "setuptools-rust>=1.11" "setuptools-scm>=8.0" "torch==2.13.0" wheel + uv pip install "setuptools>=61.0" "setuptools-rust>=1.11" "setuptools-scm>=8.0" "torch==2.14.1" wheel # torch-sys resolves libtorch from the active interpreter and bakes that # path into the cargo cache. venv is per-run and deleted on exit, so a # persistent target dir would reuse a "Fresh torch-sys" whose -L points at @@ -250,7 +250,7 @@ jobs: root / "python/sglang/srt/rust_extensions/torch_build.py" ) build = helper["torch_build_configuration"]( - compat_header=root / "rust/sglang-radix-tree/torch_2_13_compat.h", + compat_header=root / "rust/sglang-radix-tree/torch_compat.h", python_module="sglang.srt.mem_cache.rust_tree_core.mem_cache", torch_module=torch, include_absolute_rpath=False, diff --git a/.github/workflows/release-pypi-nightly.yml b/.github/workflows/release-pypi-nightly.yml index 8a069e5de8..68da0726b8 100644 --- a/.github/workflows/release-pypi-nightly.yml +++ b/.github/workflows/release-pypi-nightly.yml @@ -44,7 +44,7 @@ jobs: run: | pip install \ auditwheel build patchelf wheel "setuptools>=61.0" \ - "setuptools-rust>=1.11" "setuptools-scm>=8.0" "torch==2.13.0" + "setuptools-rust>=1.11" "setuptools-scm>=8.0" "torch==2.14.1" # Needed by setuptools-rust to build the bundled native gRPC extension # (rust/sglang-grpc) when `python -m build` builds the sglang wheel. diff --git a/.github/workflows/release-pypi-pr.yml b/.github/workflows/release-pypi-pr.yml index 627f3750cb..4ac208a91c 100644 --- a/.github/workflows/release-pypi-pr.yml +++ b/.github/workflows/release-pypi-pr.yml @@ -84,7 +84,7 @@ jobs: run: | pip install \ auditwheel build patchelf wheel "setuptools>=61.0" \ - "setuptools-rust>=1.11" "setuptools-scm>=8.0" "torch==2.13.0" + "setuptools-rust>=1.11" "setuptools-scm>=8.0" "torch==2.14.1" - name: Build wheel run: | diff --git a/.github/workflows/release-pypi.yml b/.github/workflows/release-pypi.yml index 591bcde68a..06f255d75e 100644 --- a/.github/workflows/release-pypi.yml +++ b/.github/workflows/release-pypi.yml @@ -63,7 +63,7 @@ jobs: cp ../README.md ../LICENSE . pip install \ build wheel "setuptools>=61.0" "setuptools-rust>=1.11" \ - "setuptools-scm>=8.0" "torch==2.13.0" + "setuptools-scm>=8.0" "torch==2.14.1" if [ -n "$RELEASE_VERSION" ]; then export SETUPTOOLS_SCM_PRETEND_VERSION="${RELEASE_VERSION#v}" echo "Pinning wheel version to $SETUPTOOLS_SCM_PRETEND_VERSION" diff --git a/.github/workflows/release-whl-deepep.yml b/.github/workflows/release-whl-deepep.yml index 93b71f8aca..c988201c45 100644 --- a/.github/workflows/release-whl-deepep.yml +++ b/.github/workflows/release-whl-deepep.yml @@ -18,7 +18,7 @@ concurrency: cancel-in-progress: false env: - TORCH_VERSION: "2.13.0" + TORCH_VERSION: "2.14.1" jobs: build-cu130-matrix: diff --git a/.github/workflows/release-whl-deepgemm.yml b/.github/workflows/release-whl-deepgemm.yml index 7037bce67b..147df91ab9 100644 --- a/.github/workflows/release-whl-deepgemm.yml +++ b/.github/workflows/release-whl-deepgemm.yml @@ -25,6 +25,11 @@ on: type: boolean required: false default: false + skip_tests: + description: "Skip the GPU test suite (publishes untested wheels)" + type: boolean + required: false + default: false concurrency: group: release-sgl-deepgemm-${{ github.ref }} @@ -34,7 +39,7 @@ env: NCCL_NVLS_ENABLE: "0" # Must match TORCH_VER in docker/sgl-deep-gemm.Dockerfile — the wheel's # pre-compiled _C.so links against this torch ABI. - TORCH_VER: "2.13.0" + TORCH_VER: "2.14.1" jobs: build-cu130-matrix: @@ -98,6 +103,7 @@ jobs: test-cu130: needs: build-cu130-matrix + if: ${{ !inputs.skip_tests }} strategy: fail-fast: false matrix: @@ -146,20 +152,26 @@ jobs: # deep_gemm imports torch before loading _C.so, so torch must preload the # libcudart the cu130 wheel links. Pin the CUDA-matched torch (see sgl-deep-gemm.Dockerfile). python3 -m pip install "torch==${TORCH_VER}" --index-url "https://download.pytorch.org/whl/${CU_TAG}" --force-reinstall - python3 -m pip install numpy "tilelang==0.1.9" "tile-kernels==1.0.0" "sgl-deep-ep==0.1.2" - python3 -m pip install dist/*.whl - # Torch pins NCCL 2.29.7, but this DeepEP wheel requires 2.30.7. - # Override it after every dependency-resolving pip command. + # sgl-deep-ep's own torch pin overrides the line above; it must pin TORCH_VER. + python3 -m pip install numpy "tilelang==0.1.9" "tile-kernels==1.0.0" "sgl-deep-ep==0.1.2.post1" + # --force-reinstall: a base image may already carry this exact version, + # in which case pip skips the wheel and the job tests a stale _C.so. + python3 -m pip install dist/*.whl --force-reinstall --no-deps + # This DeepEP wheel requires NCCL 2.30.7, which Torch 2.14 also pins; + # reinstall it after every dependency-resolving pip command. python3 -m pip install "nvidia-nccl-cu13==2.30.7" --force-reinstall --no-deps nccl_lib=$(python3 -c "from importlib.metadata import distribution; print(distribution('nvidia-nccl-cu13').locate_file('nvidia/nccl/lib'))") export LD_LIBRARY_PATH="${nccl_lib}${LD_LIBRARY_PATH:+:${LD_LIBRARY_PATH}}" echo "LD_LIBRARY_PATH=${LD_LIBRARY_PATH}" >> "${GITHUB_ENV}" python3 -c "import deep_gemm; print('deep_gemm:', deep_gemm.__file__)" - python3 - <<'PY' + # tvm-ffi loads _C.so lazily and a later RTLD_NOW dlopen resolves nothing, + # so bind eagerly to fail here on any missing symbol. + LD_BIND_NOW=1 python3 - <<'PY' import ctypes import sys import torch import deep_ep + import deep_gemm from tile_kernels.moe import top2_sum_gate # torch.cuda.nccl.version() reports Torch's compiled version, not the loaded library. @@ -212,7 +224,10 @@ jobs: "${{ github.workspace }}/DeepGEMM/sgl_deep_gemm/run_tests.sh" --release release-cu130: - if: ${{ !inputs.validation_only }} + if: | + !cancelled() && !inputs.validation_only && + needs.build-cu130-matrix.result == 'success' && + (needs.test-cu130.result == 'success' || needs.test-cu130.result == 'skipped') needs: [build-cu130-matrix, test-cu130] runs-on: ubuntu-latest steps: @@ -232,6 +247,12 @@ jobs: merge-multiple: true pattern: deepgemm-pypi-cuda13.0-* + - name: Record that the GPU test suite was skipped + if: ${{ inputs.skip_tests }} + run: | + echo "Releasing v${{ inputs.version }} with the GPU test suite skipped." \ + >> "$GITHUB_STEP_SUMMARY" + - name: Upload to PyPI run: | pip install --upgrade twine "packaging>=24.2" @@ -260,5 +281,5 @@ jobs: git config --local user.name "sglang-bot" git config --local user.email "sglangbot@gmail.com" git add -A - git commit -m "update sgl-deep-gemm whl index for v${{ inputs.version }}" + git commit -m "update sgl-deep-gemm whl index for v${{ inputs.version }}${{ inputs.skip_tests && ' (GPU tests skipped)' || '' }}" git push diff --git a/docker/Dockerfile b/docker/Dockerfile index 6176ead24e..8a5e2d4877 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -4,7 +4,7 @@ FROM nvidia/cuda:${CUDA_VERSION}-cudnn-devel-ubuntu24.04 AS base ARG TARGETARCH ARG BUILD_TYPE=all ARG BRANCH_TYPE=remote -ARG SGL_KERNEL_VERSION=0.4.8 +ARG SGL_KERNEL_VERSION=0.4.9 ARG SGL_VERSION ARG USE_LATEST_SGLANG=0 ARG GDRCOPY_VERSION=2.5.1 @@ -498,9 +498,8 @@ RUN --mount=type=cache,target=/root/.cache/pip \ termplotlib \ "runai-model-streamer[s3,gcs,azure]>=0.15.7" -# PyTorch 2.13 requires nvidia-nccl-cu13==2.29.7, so keep that version while -# resolving dependencies and generating constraints.txt. DeepEP needs 2.30.7 -# at runtime; apply the override only after the constrained dependency solve. +# DeepEP needs SGL_NCCL_VERSION at runtime even if torch pins another version; +# install it after the constrained dependency solve so the solve cannot revert it. RUN --mount=type=cache,target=/root/.cache/pip \ python3 -m pip install --force-reinstall --no-deps \ "nvidia-nccl-cu13==${SGL_NCCL_VERSION}" diff --git a/docker/sgl-deep-ep.Dockerfile b/docker/sgl-deep-ep.Dockerfile index 4ccf930f32..28cfb3806a 100644 --- a/docker/sgl-deep-ep.Dockerfile +++ b/docker/sgl-deep-ep.Dockerfile @@ -8,7 +8,7 @@ ARG CUDA_TAG=cu130 ARG CUDA_VERSION=13.0 ARG NCCL_VERSION=2.30.7 ARG PYTHON_TAG=cp312-cp312 -ARG TORCH_VERSION=2.13.0 +ARG TORCH_VERSION=2.14.1 ENV CUDA_HOME=/usr/local/cuda ENV LD_LIBRARY_PATH=/usr/local/lib:/usr/local/lib64:/usr/local/cuda/lib64:${LD_LIBRARY_PATH} diff --git a/docker/sgl-deep-gemm.Dockerfile b/docker/sgl-deep-gemm.Dockerfile index 6d89bf884e..81466f3a44 100644 --- a/docker/sgl-deep-gemm.Dockerfile +++ b/docker/sgl-deep-gemm.Dockerfile @@ -7,7 +7,7 @@ ARG ARCH=x86_64 ARG CUDA_VERSION=13.0 ARG PYTHON_VERSION=3.12 ARG PYTHON_TAG=cp312-cp312 -ARG TORCH_VER=2.13.0 +ARG TORCH_VER=2.14.1 ARG TVM_FFI_VER=0.1.11 ARG PIP_DEFAULT_INDEX=https://pypi.python.org/simple ARG PYTORCH_MIRROR=download.pytorch.org diff --git a/docs/docs/get-started/install.mdx b/docs/docs/get-started/install.mdx index b32da3dc88..e24082785f 100644 --- a/docs/docs/get-started/install.mdx +++ b/docs/docs/get-started/install.mdx @@ -65,7 +65,7 @@ Use Python 3.10 or higher and a compatible CUDA 13 environment. -SGLang requires CUDA 13. CUDA 12 (`cu129`) wheels and images are retired because PyTorch 2.14 publishes no CUDA 12.9 builds. SGLang 0.5.19 is the last release with a CUDA 12 lane. +SGLang requires CUDA 13. CUDA 12 (`cu129`) wheels and images were retired with the upgrade to PyTorch 2.14, which publishes no CUDA 12.9 builds. SGLang 0.5.19 is the last release with a CUDA 12 lane (PyTorch 2.13). diff --git a/python/pyproject.toml b/python/pyproject.toml index 1e53fcfd62..2f9a520060 100755 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -3,7 +3,7 @@ requires = [ "setuptools>=61.0", "setuptools-rust>=1.11", "setuptools-scm>=8.0", - "torch==2.13.0", + "torch==2.14.1", "wheel", ] build-backend = "setuptools.build_meta" @@ -76,9 +76,9 @@ dependencies = [ "scipy", "sentencepiece==0.2.1", # 0.2.2 rejects null pieces in InternVL tokenizers. "setproctitle", - "sgl-deep-ep==0.1.2", - "sgl-deep-gemm==0.2.0", - "sglang-kernel==0.4.8", + "sgl-deep-ep==0.1.2.post1", + "sgl-deep-gemm==0.2.1.post1", + "sglang-kernel==0.4.9", "smg-grpc-servicer>=0.9.0", "soundfile==0.13.1", "tiktoken", @@ -87,10 +87,10 @@ dependencies = [ "tokenizers==0.23.2", "tokenspeed_mla==0.1.8", "tomli ; python_version < '3.11'", - "torch==2.13.0", + "torch==2.14.1", "torch_memory_saver>=0.0.9.post1", "torchaudio==2.11.0", - "torchcodec==0.15.0 ; sys_platform != 'linux' or (sys_platform == 'linux' and platform_machine != 'aarch64' and platform_machine != 'arm64' and platform_machine != 'armv7l')", # Not available on Linux ARM. + "torchcodec==0.17.0 ; sys_platform != 'linux' or (sys_platform == 'linux' and platform_machine != 'aarch64' and platform_machine != 'arm64' and platform_machine != 'armv7l')", # Not available on Linux ARM. "torchvision", "tqdm", "transformers==5.17.0", diff --git a/python/sglang/kernels/ops/attention/decode_attention.py b/python/sglang/kernels/ops/attention/decode_attention.py index f711172ea3..24ffe864e4 100644 --- a/python/sglang/kernels/ops/attention/decode_attention.py +++ b/python/sglang/kernels/ops/attention/decode_attention.py @@ -332,7 +332,9 @@ def _fwd_kernel_stage1( mask=(offs_n[:, None] < split_kv_end) & (mask_d[None, :]), other=0.0, ) - qk = tl.sum(q[None, :] * k, 1) + # Reduce in fp32; a bf16 tl.sum rounds each partial sum in compiler order. + # Cast the product, not q and k, to avoid a fp32 [BLOCK_N, D] tile. + qk = tl.sum((q[None, :] * k).to(tl.float32), 1) qk *= sm_scale_withk if logit_cap > 0: diff --git a/python/sglang/kernels/ops/attention/dsv4/decode_attention_sm100_gluon.py b/python/sglang/kernels/ops/attention/dsv4/decode_attention_sm100_gluon.py index afe4617a13..ec34f68e12 100644 --- a/python/sglang/kernels/ops/attention/dsv4/decode_attention_sm100_gluon.py +++ b/python/sglang/kernels/ops/attention/dsv4/decode_attention_sm100_gluon.py @@ -16,8 +16,8 @@ try: from triton.experimental.gluon.language.nvidia.blackwell import get_tmem_reg_layout except ImportError: # Triton 3.8 moved this onto the descriptor type (triton-lang/triton#9594) - # TODO(tmorris): After we update to triton 3.8 (torch 2.15), use new API directly - # instead of this compatibility wrapper. + # TODO: once ROCm leaves triton 3.7 (swapab_gluon_hip imports _load_v4 + # from here), drop the try import above and call get_reg_layout directly. from triton.experimental.gluon.language.nvidia.blackwell import ( tensor_memory_descriptor_type, ) diff --git a/python/sglang/multimodal_gen/runtime/layers/attention/backends/sdpa.py b/python/sglang/multimodal_gen/runtime/layers/attention/backends/sdpa.py index d185f9acec..7481a5b41f 100644 --- a/python/sglang/multimodal_gen/runtime/layers/attention/backends/sdpa.py +++ b/python/sglang/multimodal_gen/runtime/layers/attention/backends/sdpa.py @@ -152,7 +152,9 @@ class SDPAImpl(AttentionImpl): ): # Keep the existing Flash SDPA arithmetic while consuming all # packed windows in one call, including ragged and empty windows. - return torch_varlen_attn( + # Call the Flash op directly: from torch 2.14, varlen_attn silently + # routes sm90/sm100 to cuDNN, which changes outputs versus the Flash path. + output, *_ = torch.ops.aten._flash_attention_forward( query, key, value, @@ -160,9 +162,14 @@ class SDPAImpl(AttentionImpl): cu_seqlens, max_seqlen, max_seqlen, + 0.0, + self.causal, + return_debug_mask=False, scale=self.softmax_scale, - window_size=(-1, 0) if self.causal else (-1, -1), + window_size_left=-1, + window_size_right=0 if self.causal else -1, ) + return output bounds = ( cu_seqlens_host if cu_seqlens_host is not None diff --git a/python/sglang/multimodal_gen/test/server/consistency_thresholds/h100.json b/python/sglang/multimodal_gen/test/server/consistency_thresholds/h100.json index df68fcdff8..270ef2b809 100644 --- a/python/sglang/multimodal_gen/test/server/consistency_thresholds/h100.json +++ b/python/sglang/multimodal_gen/test/server/consistency_thresholds/h100.json @@ -40,8 +40,8 @@ "zimage_image_t2i": { "clip_threshold": 0.97, "ssim_threshold": 0.86, - "psnr_threshold": 19.9, - "mean_abs_diff_threshold": 8.5 + "psnr_threshold": 19.3, + "mean_abs_diff_threshold": 9.8 }, "zimage_image_t2i_multi_lora": { "clip_threshold": 0.98, diff --git a/python/sglang/multimodal_gen/test/test_utils.py b/python/sglang/multimodal_gen/test/test_utils.py index f7196de4d5..8abac1a7f6 100644 --- a/python/sglang/multimodal_gen/test/test_utils.py +++ b/python/sglang/multimodal_gen/test/test_utils.py @@ -40,7 +40,7 @@ logger = init_logger(__name__) # NPU/ascend) is read from sgl-project/ci-data-diffusion, where the GT-gen workflows # publish. SGL_TEST_FILES_CI_DATA_REPO = "sgl-project/ci-data-diffusion" -SGL_TEST_FILES_CI_DATA_REVISION = "dbb70135da54b1f2b172886b28e813d2bcdd16c1" +SGL_TEST_FILES_CI_DATA_REVISION = "972007201d82d7dcce9e35b7e4b151c2b1181c30" # The NPU pin is kept as a separate branch so ascend GT can be bumped independently # when it's regenerated on its own cadence. diff --git a/python/sglang/multimodal_gen/test/unit/test_ltx2_vae_channels_last.py b/python/sglang/multimodal_gen/test/unit/test_ltx2_vae_channels_last.py index 1eea62767d..d61382f4ca 100644 --- a/python/sglang/multimodal_gen/test/unit/test_ltx2_vae_channels_last.py +++ b/python/sglang/multimodal_gen/test/unit/test_ltx2_vae_channels_last.py @@ -13,6 +13,13 @@ class TestLTX2CausalConvChannelsLast(unittest.TestCase): the original repeat()+concatenate() temporal padding (only the conv kernel's floating-point accumulation order may differ, which fp32 keeps negligible).""" + def setUp(self): + # From torch 2.14 (cuDNN 9.24), cuDNN picks a TF32 kernel for the + # channels_last_3d conv, which this fp32 comparison would read as drift. + flags = torch.backends.cudnn.flags(enabled=True, allow_tf32=False) + flags.__enter__() + self.addCleanup(flags.__exit__, None, None, None) + def _check(self, causal: bool, kernel_size): device = "cuda" if torch.cuda.is_available() else "cpu" torch.manual_seed(0) diff --git a/python/sglang/srt/entrypoints/engine.py b/python/sglang/srt/entrypoints/engine.py index 85a23dc6e1..47bad3e6ee 100644 --- a/python/sglang/srt/entrypoints/engine.py +++ b/python/sglang/srt/entrypoints/engine.py @@ -1856,7 +1856,7 @@ def _set_envs_and_config(server_args: ServerArgs): if _is_cuda: assert_pkg_version( "sglang-kernel", - "0.4.8", + "0.4.9", "Please reinstall the latest version with `pip install sglang-kernel --force-reinstall`", ) diff --git a/python/sglang/srt/layers/moe/topk.py b/python/sglang/srt/layers/moe/topk.py index 801bbb6b8b..4cde43a1ec 100644 --- a/python/sglang/srt/layers/moe/topk.py +++ b/python/sglang/srt/layers/moe/topk.py @@ -59,7 +59,7 @@ try: ): if simulated_ep != 1: raise NotImplementedError( - "simulated_ep routing is not supported with triton_kernels 3.7.1" + "simulated_ep routing is not supported with triton_kernels" ) if sm_first: diff --git a/python/sglang/srt/layers/quantization/mxfp4.py b/python/sglang/srt/layers/quantization/mxfp4.py index bd79467318..81883eb084 100644 --- a/python/sglang/srt/layers/quantization/mxfp4.py +++ b/python/sglang/srt/layers/quantization/mxfp4.py @@ -18,7 +18,7 @@ from __future__ import annotations import os -from dataclasses import replace +from dataclasses import fields, replace from functools import lru_cache from typing import TYPE_CHECKING, List, Optional @@ -227,6 +227,20 @@ def _pad_hopper_mxfp4_scale(scale, k_size): return torch.nn.functional.pad(scale, (0, want - scale.shape[-1]), value=_UE8M0_ONE) +@lru_cache(maxsize=1) +def _mx_block_kwargs(): + # triton_kernels 3.8 rejects an mx scale without its block size; the field is + # absent in 3.7, which ROCm still ships, so probe before importing the value. + from triton_kernels.matmul import PrecisionConfig + + if "b_microblock_size" not in {f.name for f in fields(PrecisionConfig)}: + return {} + + from triton_kernels.numerics_details.mxfp import MXFP_BLOCK_SIZE + + return {"b_microblock_size": int(MXFP_BLOCK_SIZE)} + + def _swizzle_mxfp4(quant_tensor, scale, num_warps): """weight swizzle for mxfp4 moe, used for OAI mxfp4 kernel""" import triton_kernels.matmul_details.opt_flags as opt_flags @@ -1042,11 +1056,12 @@ class Mxfp4MoEMethod(FusedMoEMethodBase): layer.w2_weight, layer.w2_weight_scale, num_warps ) + mx_block = _mx_block_kwargs() self.w13_precision_config = PrecisionConfig( - b_mx_scale=w13_scale, flex_ctx=FlexCtx(rhs_data=w13_flex) + b_mx_scale=w13_scale, flex_ctx=FlexCtx(rhs_data=w13_flex), **mx_block ) self.w2_precision_config = PrecisionConfig( - b_mx_scale=w2_scale, flex_ctx=FlexCtx(rhs_data=w2_flex) + b_mx_scale=w2_scale, flex_ctx=FlexCtx(rhs_data=w2_flex), **mx_block ) self.w13_weight_triton_tensor = w13_weight diff --git a/python/sglang/srt/mem_cache/rust_tree_core/extension.py b/python/sglang/srt/mem_cache/rust_tree_core/extension.py index d2b05dd464..c42d7b57b5 100644 --- a/python/sglang/srt/mem_cache/rust_tree_core/extension.py +++ b/python/sglang/srt/mem_cache/rust_tree_core/extension.py @@ -11,7 +11,7 @@ from sglang.srt.rust_extensions.torch_build import torch_build_configuration _PYTHON_MODULE = "sglang.srt.mem_cache.rust_tree_core.mem_cache" _INSPECTION_MODULE = "sglang.srt.mem_cache.rust_tree_core.mem_cache_inspection" _CRATE_DIR = Path(__file__).resolve().parents[5] / "rust" / "sglang-radix-tree" -_TORCH_COMPAT_HEADER = _CRATE_DIR / "torch_2_13_compat.h" +_TORCH_COMPAT_HEADER = _CRATE_DIR / "torch_compat.h" def load_tree_core_extension(*, inspection: bool = False): diff --git a/python/sglang/srt/rust_extensions/torch_build.py b/python/sglang/srt/rust_extensions/torch_build.py index 65629b90c7..e54c593576 100644 --- a/python/sglang/srt/rust_extensions/torch_build.py +++ b/python/sglang/srt/rust_extensions/torch_build.py @@ -13,7 +13,7 @@ from types import ModuleType from typing import Mapping _MIN_SUPPORTED_TORCH = (2, 11) -_MAX_SUPPORTED_TORCH = (2, 13) +_MAX_SUPPORTED_TORCH = (2, 14) @dataclass(frozen=True) @@ -80,7 +80,7 @@ def torch_build_configuration( environment["LIBTORCH_LIB"] = os.fspath(torch_root) environment["LIBTORCH_CXX11_ABI"] = "1" if cxx11_abi else "0" # tch 0.24 targets Torch 2.11. The compatibility header below covers the - # API removals in the supported 2.12/2.13 builds, after this explicit gate. + # API removals in the supported 2.12-2.14 builds, after this explicit gate. environment["LIBTORCH_BYPASS_VERSION_CHECK"] = "1" environment["PYO3_PYTHON"] = sys.executable environment["PATH"] = os.pathsep.join( @@ -90,10 +90,16 @@ def torch_build_configuration( filter(None, (os.fspath(torch_lib), environment.get("LD_LIBRARY_PATH"))) ) - cxxflags = environment.get("CXXFLAGS", "") - environment["CXXFLAGS"] = ( - f"{cxxflags} -include {shlex.quote(os.fspath(compat_header.resolve()))}" - ).strip() + # cc-rs splits *FLAGS on whitespace unless told they are shell words, + # which the quoted header path below needs to survive a space. + environment["CC_SHELL_ESCAPED_FLAGS"] = "1" + cxxflags = [environment.get("CXXFLAGS", "")] + # torch-sys 0.24 hardcodes -std=c++17, but libtorch declares C++20 from 2.12 on; + # cc-rs appends CXXFLAGS after a build script's own flags, so this one wins. + if major_minor >= (2, 12): + cxxflags.append("-std=c++20") + cxxflags.append(f"-include {shlex.quote(os.fspath(compat_header.resolve()))}") + environment["CXXFLAGS"] = " ".join(filter(None, cxxflags)).strip() package_depth = len(python_module.split(".")) - 1 bundled_torch_lib = "$ORIGIN/" + "../" * package_depth + "torch/lib" diff --git a/python/sglang/srt/utils/common.py b/python/sglang/srt/utils/common.py index 2cc2421c52..3f17f45226 100644 --- a/python/sglang/srt/utils/common.py +++ b/python/sglang/srt/utils/common.py @@ -595,7 +595,12 @@ def get_device_module(): return torch.xpu if is_musa(): return torch.musa - return torch.get_device_module() + # From torch 2.14, a bare torch.get_device_module() is torch.cuda on a CUDA wheel + # even with no usable device; require an available accelerator instead. + accelerator = torch.accelerator.current_accelerator(check_available=True) + if accelerator is None: + return torch.cpu + return torch.get_device_module(accelerator) def create_device_stream(device): diff --git a/rust/sglang-radix-tree/Cargo.toml b/rust/sglang-radix-tree/Cargo.toml index 426fe97d40..ed92327132 100644 --- a/rust/sglang-radix-tree/Cargo.toml +++ b/rust/sglang-radix-tree/Cargo.toml @@ -10,7 +10,7 @@ repository = "https://github.com/sgl-project/sglang" python-module = "sglang.srt.mem_cache.rust_tree_core.mem_cache" debug = false features = ["python-extension"] -torch-compat-header = "torch_2_13_compat.h" +torch-compat-header = "torch_compat.h" [lib] name = "mem_cache" diff --git a/rust/sglang-radix-tree/README.md b/rust/sglang-radix-tree/README.md index 82f209fa2d..9c8c77956a 100644 --- a/rust/sglang-radix-tree/README.md +++ b/rust/sglang-radix-tree/README.md @@ -11,7 +11,7 @@ Python in these cases: - C128 or other unsupported components. - Custom component overrides. - Non-Linux platforms. -- PyTorch versions outside 2.11 through 2.13. +- PyTorch versions outside 2.11 through 2.14. - Devices other than CPU or CUDA. - Installations containing neither the Rust extension nor its sources. - Source builds with a missing or unusable Rust toolchain. @@ -39,8 +39,10 @@ Standard SGLang wheels bundle the production extension; some platform distributions omit it. A source checkout falls back to the shared fingerprinted Rust-extension cache; it never writes a shared object into the Python package. LibTorch and the Python headers come from the running -interpreter's PyTorch install. PyTorch 2.11 through 2.13 are accepted explicitly, -and `torch_2_13_compat.h` covers two alignment APIs removed in PyTorch 2.13. +interpreter's PyTorch install. PyTorch 2.11 through 2.14 are accepted explicitly, +and `torch_compat.h` covers two alignment APIs removed in PyTorch 2.13 plus +the `cholesky` and `qr` aliases removed in 2.14. Torch 2.12 and later declare +C++20, so the builds below pass `-std=c++20` over torch-sys's own `-std=c++17`. Source checkouts need working `cargo` and `rustc` to use Rust, including for fingerprinted cache lookup. If either tool is unavailable, the registry selects Python. Trusted bundled extensions do not require a Rust compiler. @@ -60,13 +62,13 @@ tree's mutex; both tree cores implement the same contract. cd rust/sglang-radix-tree LIBTORCH_USE_PYTORCH=1 \ LIBTORCH_BYPASS_VERSION_CHECK=1 \ - CXXFLAGS="-include $PWD/torch_2_13_compat.h" \ + CXXFLAGS="-std=c++20 -include $PWD/torch_compat.h" \ cargo build --release --locked --features python-extension # Native tests do not enable pyo3's extension-module feature: TORCH_ROOT=$(python3 -c 'import pathlib, torch; print(pathlib.Path(torch.__file__).parent)') LIBTORCH_USE_PYTORCH=1 LIBTORCH_BYPASS_VERSION_CHECK=1 \ - CXXFLAGS="-include $PWD/torch_2_13_compat.h" \ + CXXFLAGS="-std=c++20 -include $PWD/torch_compat.h" \ LD_LIBRARY_PATH="$TORCH_ROOT/lib${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}" \ cargo test --locked ``` diff --git a/rust/sglang-radix-tree/torch_2_13_compat.h b/rust/sglang-radix-tree/torch_2_13_compat.h deleted file mode 100644 index f70196ec80..0000000000 --- a/rust/sglang-radix-tree/torch_2_13_compat.h +++ /dev/null @@ -1,15 +0,0 @@ -#pragma once - -#include -#include - -#if TORCH_VERSION_MAJOR > 2 || \ - (TORCH_VERSION_MAJOR == 2 && TORCH_VERSION_MINOR >= 13) -// Keep tch 0.24's removed alignment wrappers as explicit runtime errors. -#define align_as(...) \ - alias(); \ - throw std::runtime_error("align_as is unavailable in PyTorch 2.13+") -#define align_tensors(...) \ - autograd::variable_list{}; \ - throw std::runtime_error("align_tensors is unavailable in PyTorch 2.13+") -#endif diff --git a/rust/sglang-radix-tree/torch_compat.h b/rust/sglang-radix-tree/torch_compat.h new file mode 100644 index 0000000000..564160ab66 --- /dev/null +++ b/rust/sglang-radix-tree/torch_compat.h @@ -0,0 +1,57 @@ +#pragma once + +#include +#include + +#if TORCH_VERSION_MAJOR > 2 || \ + (TORCH_VERSION_MAJOR == 2 && TORCH_VERSION_MINOR >= 13) +// Keep tch 0.24's removed alignment wrappers as explicit runtime errors. +#define align_as(...) \ + alias(); \ + throw std::runtime_error("align_as is unavailable in PyTorch 2.13+") +#define align_tensors(...) \ + autograd::variable_list{}; \ + throw std::runtime_error("align_tensors is unavailable in PyTorch 2.13+") +#endif + +#if TORCH_VERSION_MAJOR > 2 || \ + (TORCH_VERSION_MAJOR == 2 && TORCH_VERSION_MINOR >= 14) +// tch 0.24 still binds the `cholesky` and `qr` aliases PyTorch 2.14 removed; +// linalg differs in errors and qr's signature, so stub rather than remap. +#include + +#include + +namespace torch { + +inline at::Tensor cholesky_unavailable(const at::Tensor &, bool) { + throw std::runtime_error( + "cholesky is unavailable in PyTorch 2.14+; use linalg_cholesky"); +} + +inline at::Tensor &cholesky_out_unavailable(at::Tensor &, const at::Tensor &, + bool) { + throw std::runtime_error( + "cholesky_out is unavailable in PyTorch 2.14+; use linalg_cholesky_out"); +} + +inline std::tuple qr_unavailable(const at::Tensor &, + bool) { + throw std::runtime_error("qr is unavailable in PyTorch 2.14+; use linalg_qr"); +} + +inline std::tuple +qr_out_unavailable(at::Tensor &, at::Tensor &, const at::Tensor &, bool) { + throw std::runtime_error( + "qr_out is unavailable in PyTorch 2.14+; use linalg_qr_out"); +} + +} // namespace torch + +// Safe as global macros only because no PyTorch 2.14 header uses these names; +// the include above declares at::Tensor for the signatures. +#define cholesky(...) cholesky_unavailable(__VA_ARGS__) +#define cholesky_out(...) cholesky_out_unavailable(__VA_ARGS__) +#define qr(...) qr_unavailable(__VA_ARGS__) +#define qr_out(...) qr_out_unavailable(__VA_ARGS__) +#endif diff --git a/scripts/build_sgl_deepep.sh b/scripts/build_sgl_deepep.sh index eb76f14a95..f9ad699973 100755 --- a/scripts/build_sgl_deepep.sh +++ b/scripts/build_sgl_deepep.sh @@ -111,7 +111,7 @@ docker build \ --build-arg CUDA_TAG="${CUDA_TAG}" \ --build-arg PYTHON_TAG="${PYTHON_TAG}" \ --build-arg ARCHITECTURE="${ARCHITECTURE}" \ - --build-arg TORCH_VERSION="${TORCH_VERSION:-2.13.0}" \ + --build-arg TORCH_VERSION="${TORCH_VERSION:-2.14.1}" \ --tag "${IMAGE_TAG}" \ --network=host \ "${REPOSITORY_ROOT}/docker" diff --git a/scripts/ci/cuda/ci_install_dependency.sh b/scripts/ci/cuda/ci_install_dependency.sh index ee6d1a6f5e..c57c84b14a 100755 --- a/scripts/ci/cuda/ci_install_dependency.sh +++ b/scripts/ci/cuda/ci_install_dependency.sh @@ -589,8 +589,8 @@ install_sglang() { } install_nccl() { - # PyTorch pins 2.29.7, so this override must run after every command - # that resolves Python dependencies (including lmms-eval). + # DeepEP needs NCCL 2.30.7 whatever torch pins, so this must run + # after every command that resolves Python dependencies (including lmms-eval). $PIP_CMD install "nvidia-nccl-cu13==2.30.7" \ --force-reinstall --no-deps $PIP_INSTALL_SUFFIX diff --git a/test/registered/rust/test_run_sglang_radix_tree_rust_tests.py b/test/registered/rust/test_run_sglang_radix_tree_rust_tests.py index 43f08bd89f..5272a963b5 100644 --- a/test/registered/rust/test_run_sglang_radix_tree_rust_tests.py +++ b/test/registered/rust/test_run_sglang_radix_tree_rust_tests.py @@ -34,7 +34,7 @@ class TestSGLangRadixTreeCargo(CustomTestCase): f"sglang-radix-tree manifest not found at {SGLANG_RADIX_TREE_MANIFEST}", ) build = torch_build_configuration( - compat_header=SGLANG_RADIX_TREE_MANIFEST.parent / "torch_2_13_compat.h", + compat_header=SGLANG_RADIX_TREE_MANIFEST.parent / "torch_compat.h", python_module="sglang.srt.mem_cache.rust_tree_core.mem_cache", ) proc = subprocess.run( diff --git a/test/registered/rust/test_rust_extension.py b/test/registered/rust/test_rust_extension.py index 10b5712d62..2873e6c8ea 100644 --- a/test/registered/rust/test_rust_extension.py +++ b/test/registered/rust/test_rust_extension.py @@ -201,7 +201,7 @@ crate-type = ["cdylib"] crate, features=(*crate.features, "inspection"), extension_module="demo._inspection", - build_fingerprint={"torch": "2.13"}, + build_fingerprint={"torch": "2.14"}, ) self.assertNotEqual(changed_source.fingerprint, inspection.fingerprint) self.assertNotEqual( @@ -458,7 +458,7 @@ crate-type = ["cdylib"] compat_header = root / "compat.h" compat_header.write_text("// compatibility\n", encoding="utf-8") fake_torch = SimpleNamespace( - __version__="2.13.0+cu130", + __version__="2.14.1+cu130", __file__=str(torch_init), compiled_with_cxx11_abi=lambda: True, version=SimpleNamespace(cuda="13.0", hip=None), @@ -483,11 +483,14 @@ crate-type = ["cdylib"] self.assertEqual(build.environment["LIBTORCH_CXX11_ABI"], "1") self.assertEqual(build.environment["LIBTORCH_BYPASS_VERSION_CHECK"], "1") self.assertIn(str(compat_header), build.environment["CXXFLAGS"]) + # -std=c++20 must trail the inherited CXXFLAGS to beat a caller's -std=; + # torch-sys's own -std=c++17 loses anyway, as cc-rs appends CXXFLAGS. + self.assertRegex(build.environment["CXXFLAGS"], r"^-O2\b.*\s-std=c\+\+20\b") self.assertIn( "$ORIGIN/../../../../torch/lib", build.environment["RUSTFLAGS"] ) self.assertIn(str(torch_root / "lib"), build.environment["RUSTFLAGS"]) - self.assertEqual(build.fingerprint["torch_version"], "2.13.0+cu130") + self.assertEqual(build.fingerprint["torch_version"], "2.14.1+cu130") self.assertTrue(build.fingerprint["torch_cxx11_abi"]) wheel_build = torch_build_configuration( @@ -502,8 +505,37 @@ crate-type = ["cdylib"] ) self.assertFalse(wheel_build.fingerprint["include_absolute_rpath"]) - fake_torch.__version__ = "2.14.0" - with self.assertRaisesRegex(RuntimeError, "PyTorch 2.11 through 2.13"): + # libtorch declares C++20 from 2.12 on; 2.11 is the only C++17 release. + for version, expected in (("2.12.0+cu128", True), ("2.11.0+cu128", False)): + fake_torch.__version__ = version + older_build = torch_build_configuration( + compat_header=compat_header, + python_module="sglang.srt.mem_cache.rust_tree_core.mem_cache", + torch_module=fake_torch, + base_environment={}, + ) + self.assertEqual( + "-std=c++20" in older_build.environment["CXXFLAGS"], + expected, + msg=version, + ) + + # cc-rs keeps a quoted, spaced header path whole only in shell-words mode. + fake_torch.__version__ = "2.14.1+cu130" + spaced_header = root / "compat dir" / "compat.h" + spaced_header.parent.mkdir() + spaced_header.write_text("// compatibility\n", encoding="utf-8") + spaced_build = torch_build_configuration( + compat_header=spaced_header, + python_module="sglang.srt.mem_cache.rust_tree_core.mem_cache", + torch_module=fake_torch, + base_environment={}, + ) + self.assertEqual(spaced_build.environment["CC_SHELL_ESCAPED_FLAGS"], "1") + self.assertIn(f"'{spaced_header}'", spaced_build.environment["CXXFLAGS"]) + + fake_torch.__version__ = "2.15.0" + with self.assertRaisesRegex(RuntimeError, "PyTorch 2.11 through 2.14"): torch_build_configuration( compat_header=compat_header, python_module="sglang.srt.mem_cache.rust_tree_core.mem_cache", diff --git a/test/registered/unit/mem_cache/test_tree_core_registry.py b/test/registered/unit/mem_cache/test_tree_core_registry.py index ad7cb03055..ead7ddbacb 100644 --- a/test/registered/unit/mem_cache/test_tree_core_registry.py +++ b/test/registered/unit/mem_cache/test_tree_core_registry.py @@ -280,7 +280,7 @@ class UnifiedRadixCacheTreeCoreSelectionTest(CustomTestCase): mock.patch.dict(os.environ), mock.patch.dict(tree_core_registry.sys.modules), mock.patch.object(tree_core_registry.sys, "platform", "linux"), - mock.patch.object(tree_core_registry.torch, "__version__", "2.13.0"), + mock.patch.object(tree_core_registry.torch, "__version__", "2.14.1"), mock.patch.object( tree_core_registry.importlib.util, "find_spec", @@ -327,7 +327,7 @@ class UnifiedRadixCacheTreeCoreSelectionTest(CustomTestCase): mock.patch.dict(tree_core_registry.sys.modules), mock.patch.object(tree_core_registry.sys, "platform", "linux"), mock.patch.object( - tree_core_registry.torch, "__version__", "2.13.0" + tree_core_registry.torch, "__version__", "2.14.1" ), mock.patch.object( tree_core_registry.importlib.util, @@ -357,7 +357,7 @@ class TreeCoreDefaultCompatibilityTest(CustomTestCase): mock.patch.dict(os.environ, {"SGLANG_RUST_BUILD_MODE": "auto"}), mock.patch.dict(tree_core_registry.sys.modules), mock.patch.object(tree_core_registry.sys, "platform", "linux"), - mock.patch.object(tree_core_registry.torch, "__version__", "2.13.0"), + mock.patch.object(tree_core_registry.torch, "__version__", "2.14.1"), mock.patch.object( tree_core_registry.importlib.util, "find_spec", return_value=None ), @@ -497,7 +497,7 @@ class TreeCoreDefaultCompatibilityTest(CustomTestCase): self.assertEqual( select_tree_core_backend(_cache_init_params()), "python" ) - for version in ("2.10.0", "2.14.0", "unknown"): + for version in ("2.10.0", "2.15.0", "unknown"): with ( self.subTest(version=version), mock.patch.object(tree_core_registry.torch, "__version__", version),