[Deps] Upgrade the CUDA PyTorch stack to 2.14 (#38641)
Co-authored-by: Mohammad Angkad <mohammad.angkad@radixark.ai>
This commit is contained in:
co-authored by
Mohammad Angkad
parent
9631f6f2d0
commit
0bdd0cc8dc
@@ -214,7 +214,7 @@ jobs:
|
||||
uv venv "${venv}" --python "${{ steps.py.outputs.python-path }}" --seed
|
||||
# shellcheck disable=SC1091
|
||||
source "${venv}/bin/activate"
|
||||
uv pip install "setuptools>=61.0" "setuptools-rust>=1.11" "setuptools-scm>=8.0" "torch==2.13.0" wheel
|
||||
uv pip install "setuptools>=61.0" "setuptools-rust>=1.11" "setuptools-scm>=8.0" "torch==2.14.1" wheel
|
||||
# torch-sys resolves libtorch from the active interpreter and bakes that
|
||||
# path into the cargo cache. venv is per-run and deleted on exit, so a
|
||||
# persistent target dir would reuse a "Fresh torch-sys" whose -L points at
|
||||
@@ -250,7 +250,7 @@ jobs:
|
||||
root / "python/sglang/srt/rust_extensions/torch_build.py"
|
||||
)
|
||||
build = helper["torch_build_configuration"](
|
||||
compat_header=root / "rust/sglang-radix-tree/torch_2_13_compat.h",
|
||||
compat_header=root / "rust/sglang-radix-tree/torch_compat.h",
|
||||
python_module="sglang.srt.mem_cache.rust_tree_core.mem_cache",
|
||||
torch_module=torch,
|
||||
include_absolute_rpath=False,
|
||||
|
||||
@@ -44,7 +44,7 @@ jobs:
|
||||
run: |
|
||||
pip install \
|
||||
auditwheel build patchelf wheel "setuptools>=61.0" \
|
||||
"setuptools-rust>=1.11" "setuptools-scm>=8.0" "torch==2.13.0"
|
||||
"setuptools-rust>=1.11" "setuptools-scm>=8.0" "torch==2.14.1"
|
||||
|
||||
# Needed by setuptools-rust to build the bundled native gRPC extension
|
||||
# (rust/sglang-grpc) when `python -m build` builds the sglang wheel.
|
||||
|
||||
@@ -84,7 +84,7 @@ jobs:
|
||||
run: |
|
||||
pip install \
|
||||
auditwheel build patchelf wheel "setuptools>=61.0" \
|
||||
"setuptools-rust>=1.11" "setuptools-scm>=8.0" "torch==2.13.0"
|
||||
"setuptools-rust>=1.11" "setuptools-scm>=8.0" "torch==2.14.1"
|
||||
|
||||
- name: Build wheel
|
||||
run: |
|
||||
|
||||
@@ -63,7 +63,7 @@ jobs:
|
||||
cp ../README.md ../LICENSE .
|
||||
pip install \
|
||||
build wheel "setuptools>=61.0" "setuptools-rust>=1.11" \
|
||||
"setuptools-scm>=8.0" "torch==2.13.0"
|
||||
"setuptools-scm>=8.0" "torch==2.14.1"
|
||||
if [ -n "$RELEASE_VERSION" ]; then
|
||||
export SETUPTOOLS_SCM_PRETEND_VERSION="${RELEASE_VERSION#v}"
|
||||
echo "Pinning wheel version to $SETUPTOOLS_SCM_PRETEND_VERSION"
|
||||
|
||||
@@ -18,7 +18,7 @@ concurrency:
|
||||
cancel-in-progress: false
|
||||
|
||||
env:
|
||||
TORCH_VERSION: "2.13.0"
|
||||
TORCH_VERSION: "2.14.1"
|
||||
|
||||
jobs:
|
||||
build-cu130-matrix:
|
||||
|
||||
@@ -25,6 +25,11 @@ on:
|
||||
type: boolean
|
||||
required: false
|
||||
default: false
|
||||
skip_tests:
|
||||
description: "Skip the GPU test suite (publishes untested wheels)"
|
||||
type: boolean
|
||||
required: false
|
||||
default: false
|
||||
|
||||
concurrency:
|
||||
group: release-sgl-deepgemm-${{ github.ref }}
|
||||
@@ -34,7 +39,7 @@ env:
|
||||
NCCL_NVLS_ENABLE: "0"
|
||||
# Must match TORCH_VER in docker/sgl-deep-gemm.Dockerfile — the wheel's
|
||||
# pre-compiled _C.so links against this torch ABI.
|
||||
TORCH_VER: "2.13.0"
|
||||
TORCH_VER: "2.14.1"
|
||||
|
||||
jobs:
|
||||
build-cu130-matrix:
|
||||
@@ -98,6 +103,7 @@ jobs:
|
||||
|
||||
test-cu130:
|
||||
needs: build-cu130-matrix
|
||||
if: ${{ !inputs.skip_tests }}
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
@@ -146,20 +152,26 @@ jobs:
|
||||
# deep_gemm imports torch before loading _C.so, so torch must preload the
|
||||
# libcudart the cu130 wheel links. Pin the CUDA-matched torch (see sgl-deep-gemm.Dockerfile).
|
||||
python3 -m pip install "torch==${TORCH_VER}" --index-url "https://download.pytorch.org/whl/${CU_TAG}" --force-reinstall
|
||||
python3 -m pip install numpy "tilelang==0.1.9" "tile-kernels==1.0.0" "sgl-deep-ep==0.1.2"
|
||||
python3 -m pip install dist/*.whl
|
||||
# Torch pins NCCL 2.29.7, but this DeepEP wheel requires 2.30.7.
|
||||
# Override it after every dependency-resolving pip command.
|
||||
# sgl-deep-ep's own torch pin overrides the line above; it must pin TORCH_VER.
|
||||
python3 -m pip install numpy "tilelang==0.1.9" "tile-kernels==1.0.0" "sgl-deep-ep==0.1.2.post1"
|
||||
# --force-reinstall: a base image may already carry this exact version,
|
||||
# in which case pip skips the wheel and the job tests a stale _C.so.
|
||||
python3 -m pip install dist/*.whl --force-reinstall --no-deps
|
||||
# This DeepEP wheel requires NCCL 2.30.7, which Torch 2.14 also pins;
|
||||
# reinstall it after every dependency-resolving pip command.
|
||||
python3 -m pip install "nvidia-nccl-cu13==2.30.7" --force-reinstall --no-deps
|
||||
nccl_lib=$(python3 -c "from importlib.metadata import distribution; print(distribution('nvidia-nccl-cu13').locate_file('nvidia/nccl/lib'))")
|
||||
export LD_LIBRARY_PATH="${nccl_lib}${LD_LIBRARY_PATH:+:${LD_LIBRARY_PATH}}"
|
||||
echo "LD_LIBRARY_PATH=${LD_LIBRARY_PATH}" >> "${GITHUB_ENV}"
|
||||
python3 -c "import deep_gemm; print('deep_gemm:', deep_gemm.__file__)"
|
||||
python3 - <<'PY'
|
||||
# tvm-ffi loads _C.so lazily and a later RTLD_NOW dlopen resolves nothing,
|
||||
# so bind eagerly to fail here on any missing symbol.
|
||||
LD_BIND_NOW=1 python3 - <<'PY'
|
||||
import ctypes
|
||||
import sys
|
||||
import torch
|
||||
import deep_ep
|
||||
import deep_gemm
|
||||
from tile_kernels.moe import top2_sum_gate
|
||||
|
||||
# torch.cuda.nccl.version() reports Torch's compiled version, not the loaded library.
|
||||
@@ -212,7 +224,10 @@ jobs:
|
||||
"${{ github.workspace }}/DeepGEMM/sgl_deep_gemm/run_tests.sh" --release
|
||||
|
||||
release-cu130:
|
||||
if: ${{ !inputs.validation_only }}
|
||||
if: |
|
||||
!cancelled() && !inputs.validation_only &&
|
||||
needs.build-cu130-matrix.result == 'success' &&
|
||||
(needs.test-cu130.result == 'success' || needs.test-cu130.result == 'skipped')
|
||||
needs: [build-cu130-matrix, test-cu130]
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
@@ -232,6 +247,12 @@ jobs:
|
||||
merge-multiple: true
|
||||
pattern: deepgemm-pypi-cuda13.0-*
|
||||
|
||||
- name: Record that the GPU test suite was skipped
|
||||
if: ${{ inputs.skip_tests }}
|
||||
run: |
|
||||
echo "Releasing v${{ inputs.version }} with the GPU test suite skipped." \
|
||||
>> "$GITHUB_STEP_SUMMARY"
|
||||
|
||||
- name: Upload to PyPI
|
||||
run: |
|
||||
pip install --upgrade twine "packaging>=24.2"
|
||||
@@ -260,5 +281,5 @@ jobs:
|
||||
git config --local user.name "sglang-bot"
|
||||
git config --local user.email "sglangbot@gmail.com"
|
||||
git add -A
|
||||
git commit -m "update sgl-deep-gemm whl index for v${{ inputs.version }}"
|
||||
git commit -m "update sgl-deep-gemm whl index for v${{ inputs.version }}${{ inputs.skip_tests && ' (GPU tests skipped)' || '' }}"
|
||||
git push
|
||||
|
||||
+3
-4
@@ -4,7 +4,7 @@ FROM nvidia/cuda:${CUDA_VERSION}-cudnn-devel-ubuntu24.04 AS base
|
||||
ARG TARGETARCH
|
||||
ARG BUILD_TYPE=all
|
||||
ARG BRANCH_TYPE=remote
|
||||
ARG SGL_KERNEL_VERSION=0.4.8
|
||||
ARG SGL_KERNEL_VERSION=0.4.9
|
||||
ARG SGL_VERSION
|
||||
ARG USE_LATEST_SGLANG=0
|
||||
ARG GDRCOPY_VERSION=2.5.1
|
||||
@@ -498,9 +498,8 @@ RUN --mount=type=cache,target=/root/.cache/pip \
|
||||
termplotlib \
|
||||
"runai-model-streamer[s3,gcs,azure]>=0.15.7"
|
||||
|
||||
# PyTorch 2.13 requires nvidia-nccl-cu13==2.29.7, so keep that version while
|
||||
# resolving dependencies and generating constraints.txt. DeepEP needs 2.30.7
|
||||
# at runtime; apply the override only after the constrained dependency solve.
|
||||
# DeepEP needs SGL_NCCL_VERSION at runtime even if torch pins another version;
|
||||
# install it after the constrained dependency solve so the solve cannot revert it.
|
||||
RUN --mount=type=cache,target=/root/.cache/pip \
|
||||
python3 -m pip install --force-reinstall --no-deps \
|
||||
"nvidia-nccl-cu13==${SGL_NCCL_VERSION}"
|
||||
|
||||
@@ -8,7 +8,7 @@ ARG CUDA_TAG=cu130
|
||||
ARG CUDA_VERSION=13.0
|
||||
ARG NCCL_VERSION=2.30.7
|
||||
ARG PYTHON_TAG=cp312-cp312
|
||||
ARG TORCH_VERSION=2.13.0
|
||||
ARG TORCH_VERSION=2.14.1
|
||||
|
||||
ENV CUDA_HOME=/usr/local/cuda
|
||||
ENV LD_LIBRARY_PATH=/usr/local/lib:/usr/local/lib64:/usr/local/cuda/lib64:${LD_LIBRARY_PATH}
|
||||
|
||||
@@ -7,7 +7,7 @@ ARG ARCH=x86_64
|
||||
ARG CUDA_VERSION=13.0
|
||||
ARG PYTHON_VERSION=3.12
|
||||
ARG PYTHON_TAG=cp312-cp312
|
||||
ARG TORCH_VER=2.13.0
|
||||
ARG TORCH_VER=2.14.1
|
||||
ARG TVM_FFI_VER=0.1.11
|
||||
ARG PIP_DEFAULT_INDEX=https://pypi.python.org/simple
|
||||
ARG PYTORCH_MIRROR=download.pytorch.org
|
||||
|
||||
@@ -65,7 +65,7 @@ Use Python 3.10 or higher and a compatible CUDA 13 environment.
|
||||
|
||||
<Accordion title="CUDA version compatibility">
|
||||
|
||||
SGLang requires CUDA 13. CUDA 12 (`cu129`) wheels and images are retired because PyTorch 2.14 publishes no CUDA 12.9 builds. SGLang 0.5.19 is the last release with a CUDA 12 lane.
|
||||
SGLang requires CUDA 13. CUDA 12 (`cu129`) wheels and images were retired with the upgrade to PyTorch 2.14, which publishes no CUDA 12.9 builds. SGLang 0.5.19 is the last release with a CUDA 12 lane (PyTorch 2.13).
|
||||
|
||||
</Accordion>
|
||||
|
||||
|
||||
@@ -3,7 +3,7 @@ requires = [
|
||||
"setuptools>=61.0",
|
||||
"setuptools-rust>=1.11",
|
||||
"setuptools-scm>=8.0",
|
||||
"torch==2.13.0",
|
||||
"torch==2.14.1",
|
||||
"wheel",
|
||||
]
|
||||
build-backend = "setuptools.build_meta"
|
||||
@@ -76,9 +76,9 @@ dependencies = [
|
||||
"scipy",
|
||||
"sentencepiece==0.2.1", # 0.2.2 rejects null pieces in InternVL tokenizers.
|
||||
"setproctitle",
|
||||
"sgl-deep-ep==0.1.2",
|
||||
"sgl-deep-gemm==0.2.0",
|
||||
"sglang-kernel==0.4.8",
|
||||
"sgl-deep-ep==0.1.2.post1",
|
||||
"sgl-deep-gemm==0.2.1.post1",
|
||||
"sglang-kernel==0.4.9",
|
||||
"smg-grpc-servicer>=0.9.0",
|
||||
"soundfile==0.13.1",
|
||||
"tiktoken",
|
||||
@@ -87,10 +87,10 @@ dependencies = [
|
||||
"tokenizers==0.23.2",
|
||||
"tokenspeed_mla==0.1.8",
|
||||
"tomli ; python_version < '3.11'",
|
||||
"torch==2.13.0",
|
||||
"torch==2.14.1",
|
||||
"torch_memory_saver>=0.0.9.post1",
|
||||
"torchaudio==2.11.0",
|
||||
"torchcodec==0.15.0 ; sys_platform != 'linux' or (sys_platform == 'linux' and platform_machine != 'aarch64' and platform_machine != 'arm64' and platform_machine != 'armv7l')", # Not available on Linux ARM.
|
||||
"torchcodec==0.17.0 ; sys_platform != 'linux' or (sys_platform == 'linux' and platform_machine != 'aarch64' and platform_machine != 'arm64' and platform_machine != 'armv7l')", # Not available on Linux ARM.
|
||||
"torchvision",
|
||||
"tqdm",
|
||||
"transformers==5.17.0",
|
||||
|
||||
@@ -332,7 +332,9 @@ def _fwd_kernel_stage1(
|
||||
mask=(offs_n[:, None] < split_kv_end) & (mask_d[None, :]),
|
||||
other=0.0,
|
||||
)
|
||||
qk = tl.sum(q[None, :] * k, 1)
|
||||
# Reduce in fp32; a bf16 tl.sum rounds each partial sum in compiler order.
|
||||
# Cast the product, not q and k, to avoid a fp32 [BLOCK_N, D] tile.
|
||||
qk = tl.sum((q[None, :] * k).to(tl.float32), 1)
|
||||
qk *= sm_scale_withk
|
||||
|
||||
if logit_cap > 0:
|
||||
|
||||
@@ -16,8 +16,8 @@ try:
|
||||
from triton.experimental.gluon.language.nvidia.blackwell import get_tmem_reg_layout
|
||||
except ImportError:
|
||||
# Triton 3.8 moved this onto the descriptor type (triton-lang/triton#9594)
|
||||
# TODO(tmorris): After we update to triton 3.8 (torch 2.15), use new API directly
|
||||
# instead of this compatibility wrapper.
|
||||
# TODO: once ROCm leaves triton 3.7 (swapab_gluon_hip imports _load_v4
|
||||
# from here), drop the try import above and call get_reg_layout directly.
|
||||
from triton.experimental.gluon.language.nvidia.blackwell import (
|
||||
tensor_memory_descriptor_type,
|
||||
)
|
||||
|
||||
@@ -152,7 +152,9 @@ class SDPAImpl(AttentionImpl):
|
||||
):
|
||||
# Keep the existing Flash SDPA arithmetic while consuming all
|
||||
# packed windows in one call, including ragged and empty windows.
|
||||
return torch_varlen_attn(
|
||||
# Call the Flash op directly: from torch 2.14, varlen_attn silently
|
||||
# routes sm90/sm100 to cuDNN, which changes outputs versus the Flash path.
|
||||
output, *_ = torch.ops.aten._flash_attention_forward(
|
||||
query,
|
||||
key,
|
||||
value,
|
||||
@@ -160,9 +162,14 @@ class SDPAImpl(AttentionImpl):
|
||||
cu_seqlens,
|
||||
max_seqlen,
|
||||
max_seqlen,
|
||||
0.0,
|
||||
self.causal,
|
||||
return_debug_mask=False,
|
||||
scale=self.softmax_scale,
|
||||
window_size=(-1, 0) if self.causal else (-1, -1),
|
||||
window_size_left=-1,
|
||||
window_size_right=0 if self.causal else -1,
|
||||
)
|
||||
return output
|
||||
bounds = (
|
||||
cu_seqlens_host
|
||||
if cu_seqlens_host is not None
|
||||
|
||||
@@ -40,8 +40,8 @@
|
||||
"zimage_image_t2i": {
|
||||
"clip_threshold": 0.97,
|
||||
"ssim_threshold": 0.86,
|
||||
"psnr_threshold": 19.9,
|
||||
"mean_abs_diff_threshold": 8.5
|
||||
"psnr_threshold": 19.3,
|
||||
"mean_abs_diff_threshold": 9.8
|
||||
},
|
||||
"zimage_image_t2i_multi_lora": {
|
||||
"clip_threshold": 0.98,
|
||||
|
||||
@@ -40,7 +40,7 @@ logger = init_logger(__name__)
|
||||
# NPU/ascend) is read from sgl-project/ci-data-diffusion, where the GT-gen workflows
|
||||
# publish.
|
||||
SGL_TEST_FILES_CI_DATA_REPO = "sgl-project/ci-data-diffusion"
|
||||
SGL_TEST_FILES_CI_DATA_REVISION = "dbb70135da54b1f2b172886b28e813d2bcdd16c1"
|
||||
SGL_TEST_FILES_CI_DATA_REVISION = "972007201d82d7dcce9e35b7e4b151c2b1181c30"
|
||||
|
||||
# The NPU pin is kept as a separate branch so ascend GT can be bumped independently
|
||||
# when it's regenerated on its own cadence.
|
||||
|
||||
@@ -13,6 +13,13 @@ class TestLTX2CausalConvChannelsLast(unittest.TestCase):
|
||||
the original repeat()+concatenate() temporal padding (only the conv kernel's
|
||||
floating-point accumulation order may differ, which fp32 keeps negligible)."""
|
||||
|
||||
def setUp(self):
|
||||
# From torch 2.14 (cuDNN 9.24), cuDNN picks a TF32 kernel for the
|
||||
# channels_last_3d conv, which this fp32 comparison would read as drift.
|
||||
flags = torch.backends.cudnn.flags(enabled=True, allow_tf32=False)
|
||||
flags.__enter__()
|
||||
self.addCleanup(flags.__exit__, None, None, None)
|
||||
|
||||
def _check(self, causal: bool, kernel_size):
|
||||
device = "cuda" if torch.cuda.is_available() else "cpu"
|
||||
torch.manual_seed(0)
|
||||
|
||||
@@ -1856,7 +1856,7 @@ def _set_envs_and_config(server_args: ServerArgs):
|
||||
if _is_cuda:
|
||||
assert_pkg_version(
|
||||
"sglang-kernel",
|
||||
"0.4.8",
|
||||
"0.4.9",
|
||||
"Please reinstall the latest version with `pip install sglang-kernel --force-reinstall`",
|
||||
)
|
||||
|
||||
|
||||
@@ -59,7 +59,7 @@ try:
|
||||
):
|
||||
if simulated_ep != 1:
|
||||
raise NotImplementedError(
|
||||
"simulated_ep routing is not supported with triton_kernels 3.7.1"
|
||||
"simulated_ep routing is not supported with triton_kernels"
|
||||
)
|
||||
|
||||
if sm_first:
|
||||
|
||||
@@ -18,7 +18,7 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
from dataclasses import replace
|
||||
from dataclasses import fields, replace
|
||||
from functools import lru_cache
|
||||
from typing import TYPE_CHECKING, List, Optional
|
||||
|
||||
@@ -227,6 +227,20 @@ def _pad_hopper_mxfp4_scale(scale, k_size):
|
||||
return torch.nn.functional.pad(scale, (0, want - scale.shape[-1]), value=_UE8M0_ONE)
|
||||
|
||||
|
||||
@lru_cache(maxsize=1)
|
||||
def _mx_block_kwargs():
|
||||
# triton_kernels 3.8 rejects an mx scale without its block size; the field is
|
||||
# absent in 3.7, which ROCm still ships, so probe before importing the value.
|
||||
from triton_kernels.matmul import PrecisionConfig
|
||||
|
||||
if "b_microblock_size" not in {f.name for f in fields(PrecisionConfig)}:
|
||||
return {}
|
||||
|
||||
from triton_kernels.numerics_details.mxfp import MXFP_BLOCK_SIZE
|
||||
|
||||
return {"b_microblock_size": int(MXFP_BLOCK_SIZE)}
|
||||
|
||||
|
||||
def _swizzle_mxfp4(quant_tensor, scale, num_warps):
|
||||
"""weight swizzle for mxfp4 moe, used for OAI mxfp4 kernel"""
|
||||
import triton_kernels.matmul_details.opt_flags as opt_flags
|
||||
@@ -1042,11 +1056,12 @@ class Mxfp4MoEMethod(FusedMoEMethodBase):
|
||||
layer.w2_weight, layer.w2_weight_scale, num_warps
|
||||
)
|
||||
|
||||
mx_block = _mx_block_kwargs()
|
||||
self.w13_precision_config = PrecisionConfig(
|
||||
b_mx_scale=w13_scale, flex_ctx=FlexCtx(rhs_data=w13_flex)
|
||||
b_mx_scale=w13_scale, flex_ctx=FlexCtx(rhs_data=w13_flex), **mx_block
|
||||
)
|
||||
self.w2_precision_config = PrecisionConfig(
|
||||
b_mx_scale=w2_scale, flex_ctx=FlexCtx(rhs_data=w2_flex)
|
||||
b_mx_scale=w2_scale, flex_ctx=FlexCtx(rhs_data=w2_flex), **mx_block
|
||||
)
|
||||
|
||||
self.w13_weight_triton_tensor = w13_weight
|
||||
|
||||
@@ -11,7 +11,7 @@ from sglang.srt.rust_extensions.torch_build import torch_build_configuration
|
||||
_PYTHON_MODULE = "sglang.srt.mem_cache.rust_tree_core.mem_cache"
|
||||
_INSPECTION_MODULE = "sglang.srt.mem_cache.rust_tree_core.mem_cache_inspection"
|
||||
_CRATE_DIR = Path(__file__).resolve().parents[5] / "rust" / "sglang-radix-tree"
|
||||
_TORCH_COMPAT_HEADER = _CRATE_DIR / "torch_2_13_compat.h"
|
||||
_TORCH_COMPAT_HEADER = _CRATE_DIR / "torch_compat.h"
|
||||
|
||||
|
||||
def load_tree_core_extension(*, inspection: bool = False):
|
||||
|
||||
@@ -13,7 +13,7 @@ from types import ModuleType
|
||||
from typing import Mapping
|
||||
|
||||
_MIN_SUPPORTED_TORCH = (2, 11)
|
||||
_MAX_SUPPORTED_TORCH = (2, 13)
|
||||
_MAX_SUPPORTED_TORCH = (2, 14)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
@@ -80,7 +80,7 @@ def torch_build_configuration(
|
||||
environment["LIBTORCH_LIB"] = os.fspath(torch_root)
|
||||
environment["LIBTORCH_CXX11_ABI"] = "1" if cxx11_abi else "0"
|
||||
# tch 0.24 targets Torch 2.11. The compatibility header below covers the
|
||||
# API removals in the supported 2.12/2.13 builds, after this explicit gate.
|
||||
# API removals in the supported 2.12-2.14 builds, after this explicit gate.
|
||||
environment["LIBTORCH_BYPASS_VERSION_CHECK"] = "1"
|
||||
environment["PYO3_PYTHON"] = sys.executable
|
||||
environment["PATH"] = os.pathsep.join(
|
||||
@@ -90,10 +90,16 @@ def torch_build_configuration(
|
||||
filter(None, (os.fspath(torch_lib), environment.get("LD_LIBRARY_PATH")))
|
||||
)
|
||||
|
||||
cxxflags = environment.get("CXXFLAGS", "")
|
||||
environment["CXXFLAGS"] = (
|
||||
f"{cxxflags} -include {shlex.quote(os.fspath(compat_header.resolve()))}"
|
||||
).strip()
|
||||
# cc-rs splits *FLAGS on whitespace unless told they are shell words,
|
||||
# which the quoted header path below needs to survive a space.
|
||||
environment["CC_SHELL_ESCAPED_FLAGS"] = "1"
|
||||
cxxflags = [environment.get("CXXFLAGS", "")]
|
||||
# torch-sys 0.24 hardcodes -std=c++17, but libtorch declares C++20 from 2.12 on;
|
||||
# cc-rs appends CXXFLAGS after a build script's own flags, so this one wins.
|
||||
if major_minor >= (2, 12):
|
||||
cxxflags.append("-std=c++20")
|
||||
cxxflags.append(f"-include {shlex.quote(os.fspath(compat_header.resolve()))}")
|
||||
environment["CXXFLAGS"] = " ".join(filter(None, cxxflags)).strip()
|
||||
|
||||
package_depth = len(python_module.split(".")) - 1
|
||||
bundled_torch_lib = "$ORIGIN/" + "../" * package_depth + "torch/lib"
|
||||
|
||||
@@ -595,7 +595,12 @@ def get_device_module():
|
||||
return torch.xpu
|
||||
if is_musa():
|
||||
return torch.musa
|
||||
return torch.get_device_module()
|
||||
# From torch 2.14, a bare torch.get_device_module() is torch.cuda on a CUDA wheel
|
||||
# even with no usable device; require an available accelerator instead.
|
||||
accelerator = torch.accelerator.current_accelerator(check_available=True)
|
||||
if accelerator is None:
|
||||
return torch.cpu
|
||||
return torch.get_device_module(accelerator)
|
||||
|
||||
|
||||
def create_device_stream(device):
|
||||
|
||||
@@ -10,7 +10,7 @@ repository = "https://github.com/sgl-project/sglang"
|
||||
python-module = "sglang.srt.mem_cache.rust_tree_core.mem_cache"
|
||||
debug = false
|
||||
features = ["python-extension"]
|
||||
torch-compat-header = "torch_2_13_compat.h"
|
||||
torch-compat-header = "torch_compat.h"
|
||||
|
||||
[lib]
|
||||
name = "mem_cache"
|
||||
|
||||
@@ -11,7 +11,7 @@ Python in these cases:
|
||||
- C128 or other unsupported components.
|
||||
- Custom component overrides.
|
||||
- Non-Linux platforms.
|
||||
- PyTorch versions outside 2.11 through 2.13.
|
||||
- PyTorch versions outside 2.11 through 2.14.
|
||||
- Devices other than CPU or CUDA.
|
||||
- Installations containing neither the Rust extension nor its sources.
|
||||
- Source builds with a missing or unusable Rust toolchain.
|
||||
@@ -39,8 +39,10 @@ Standard SGLang wheels bundle the production extension; some platform
|
||||
distributions omit it. A source checkout falls back to
|
||||
the shared fingerprinted Rust-extension cache; it never writes a shared object
|
||||
into the Python package. LibTorch and the Python headers come from the running
|
||||
interpreter's PyTorch install. PyTorch 2.11 through 2.13 are accepted explicitly,
|
||||
and `torch_2_13_compat.h` covers two alignment APIs removed in PyTorch 2.13.
|
||||
interpreter's PyTorch install. PyTorch 2.11 through 2.14 are accepted explicitly,
|
||||
and `torch_compat.h` covers two alignment APIs removed in PyTorch 2.13 plus
|
||||
the `cholesky` and `qr` aliases removed in 2.14. Torch 2.12 and later declare
|
||||
C++20, so the builds below pass `-std=c++20` over torch-sys's own `-std=c++17`.
|
||||
Source checkouts need working `cargo` and `rustc` to use Rust, including for
|
||||
fingerprinted cache lookup. If either tool is unavailable, the registry selects
|
||||
Python. Trusted bundled extensions do not require a Rust compiler.
|
||||
@@ -60,13 +62,13 @@ tree's mutex; both tree cores implement the same contract.
|
||||
cd rust/sglang-radix-tree
|
||||
LIBTORCH_USE_PYTORCH=1 \
|
||||
LIBTORCH_BYPASS_VERSION_CHECK=1 \
|
||||
CXXFLAGS="-include $PWD/torch_2_13_compat.h" \
|
||||
CXXFLAGS="-std=c++20 -include $PWD/torch_compat.h" \
|
||||
cargo build --release --locked --features python-extension
|
||||
|
||||
# Native tests do not enable pyo3's extension-module feature:
|
||||
TORCH_ROOT=$(python3 -c 'import pathlib, torch; print(pathlib.Path(torch.__file__).parent)')
|
||||
LIBTORCH_USE_PYTORCH=1 LIBTORCH_BYPASS_VERSION_CHECK=1 \
|
||||
CXXFLAGS="-include $PWD/torch_2_13_compat.h" \
|
||||
CXXFLAGS="-std=c++20 -include $PWD/torch_compat.h" \
|
||||
LD_LIBRARY_PATH="$TORCH_ROOT/lib${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}" \
|
||||
cargo test --locked
|
||||
```
|
||||
|
||||
@@ -1,15 +0,0 @@
|
||||
#pragma once
|
||||
|
||||
#include <stdexcept>
|
||||
#include <torch/version.h>
|
||||
|
||||
#if TORCH_VERSION_MAJOR > 2 || \
|
||||
(TORCH_VERSION_MAJOR == 2 && TORCH_VERSION_MINOR >= 13)
|
||||
// Keep tch 0.24's removed alignment wrappers as explicit runtime errors.
|
||||
#define align_as(...) \
|
||||
alias(); \
|
||||
throw std::runtime_error("align_as is unavailable in PyTorch 2.13+")
|
||||
#define align_tensors(...) \
|
||||
autograd::variable_list{}; \
|
||||
throw std::runtime_error("align_tensors is unavailable in PyTorch 2.13+")
|
||||
#endif
|
||||
@@ -0,0 +1,57 @@
|
||||
#pragma once
|
||||
|
||||
#include <stdexcept>
|
||||
#include <torch/version.h>
|
||||
|
||||
#if TORCH_VERSION_MAJOR > 2 || \
|
||||
(TORCH_VERSION_MAJOR == 2 && TORCH_VERSION_MINOR >= 13)
|
||||
// Keep tch 0.24's removed alignment wrappers as explicit runtime errors.
|
||||
#define align_as(...) \
|
||||
alias(); \
|
||||
throw std::runtime_error("align_as is unavailable in PyTorch 2.13+")
|
||||
#define align_tensors(...) \
|
||||
autograd::variable_list{}; \
|
||||
throw std::runtime_error("align_tensors is unavailable in PyTorch 2.13+")
|
||||
#endif
|
||||
|
||||
#if TORCH_VERSION_MAJOR > 2 || \
|
||||
(TORCH_VERSION_MAJOR == 2 && TORCH_VERSION_MINOR >= 14)
|
||||
// tch 0.24 still binds the `cholesky` and `qr` aliases PyTorch 2.14 removed;
|
||||
// linalg differs in errors and qr's signature, so stub rather than remap.
|
||||
#include <ATen/core/Tensor.h>
|
||||
|
||||
#include <tuple>
|
||||
|
||||
namespace torch {
|
||||
|
||||
inline at::Tensor cholesky_unavailable(const at::Tensor &, bool) {
|
||||
throw std::runtime_error(
|
||||
"cholesky is unavailable in PyTorch 2.14+; use linalg_cholesky");
|
||||
}
|
||||
|
||||
inline at::Tensor &cholesky_out_unavailable(at::Tensor &, const at::Tensor &,
|
||||
bool) {
|
||||
throw std::runtime_error(
|
||||
"cholesky_out is unavailable in PyTorch 2.14+; use linalg_cholesky_out");
|
||||
}
|
||||
|
||||
inline std::tuple<at::Tensor, at::Tensor> qr_unavailable(const at::Tensor &,
|
||||
bool) {
|
||||
throw std::runtime_error("qr is unavailable in PyTorch 2.14+; use linalg_qr");
|
||||
}
|
||||
|
||||
inline std::tuple<at::Tensor &, at::Tensor &>
|
||||
qr_out_unavailable(at::Tensor &, at::Tensor &, const at::Tensor &, bool) {
|
||||
throw std::runtime_error(
|
||||
"qr_out is unavailable in PyTorch 2.14+; use linalg_qr_out");
|
||||
}
|
||||
|
||||
} // namespace torch
|
||||
|
||||
// Safe as global macros only because no PyTorch 2.14 header uses these names;
|
||||
// the include above declares at::Tensor for the signatures.
|
||||
#define cholesky(...) cholesky_unavailable(__VA_ARGS__)
|
||||
#define cholesky_out(...) cholesky_out_unavailable(__VA_ARGS__)
|
||||
#define qr(...) qr_unavailable(__VA_ARGS__)
|
||||
#define qr_out(...) qr_out_unavailable(__VA_ARGS__)
|
||||
#endif
|
||||
@@ -111,7 +111,7 @@ docker build \
|
||||
--build-arg CUDA_TAG="${CUDA_TAG}" \
|
||||
--build-arg PYTHON_TAG="${PYTHON_TAG}" \
|
||||
--build-arg ARCHITECTURE="${ARCHITECTURE}" \
|
||||
--build-arg TORCH_VERSION="${TORCH_VERSION:-2.13.0}" \
|
||||
--build-arg TORCH_VERSION="${TORCH_VERSION:-2.14.1}" \
|
||||
--tag "${IMAGE_TAG}" \
|
||||
--network=host \
|
||||
"${REPOSITORY_ROOT}/docker"
|
||||
|
||||
@@ -589,8 +589,8 @@ install_sglang() {
|
||||
}
|
||||
|
||||
install_nccl() {
|
||||
# PyTorch pins 2.29.7, so this override must run after every command
|
||||
# that resolves Python dependencies (including lmms-eval).
|
||||
# DeepEP needs NCCL 2.30.7 whatever torch pins, so this must run
|
||||
# after every command that resolves Python dependencies (including lmms-eval).
|
||||
$PIP_CMD install "nvidia-nccl-cu13==2.30.7" \
|
||||
--force-reinstall --no-deps $PIP_INSTALL_SUFFIX
|
||||
|
||||
|
||||
@@ -34,7 +34,7 @@ class TestSGLangRadixTreeCargo(CustomTestCase):
|
||||
f"sglang-radix-tree manifest not found at {SGLANG_RADIX_TREE_MANIFEST}",
|
||||
)
|
||||
build = torch_build_configuration(
|
||||
compat_header=SGLANG_RADIX_TREE_MANIFEST.parent / "torch_2_13_compat.h",
|
||||
compat_header=SGLANG_RADIX_TREE_MANIFEST.parent / "torch_compat.h",
|
||||
python_module="sglang.srt.mem_cache.rust_tree_core.mem_cache",
|
||||
)
|
||||
proc = subprocess.run(
|
||||
|
||||
@@ -201,7 +201,7 @@ crate-type = ["cdylib"]
|
||||
crate,
|
||||
features=(*crate.features, "inspection"),
|
||||
extension_module="demo._inspection",
|
||||
build_fingerprint={"torch": "2.13"},
|
||||
build_fingerprint={"torch": "2.14"},
|
||||
)
|
||||
self.assertNotEqual(changed_source.fingerprint, inspection.fingerprint)
|
||||
self.assertNotEqual(
|
||||
@@ -458,7 +458,7 @@ crate-type = ["cdylib"]
|
||||
compat_header = root / "compat.h"
|
||||
compat_header.write_text("// compatibility\n", encoding="utf-8")
|
||||
fake_torch = SimpleNamespace(
|
||||
__version__="2.13.0+cu130",
|
||||
__version__="2.14.1+cu130",
|
||||
__file__=str(torch_init),
|
||||
compiled_with_cxx11_abi=lambda: True,
|
||||
version=SimpleNamespace(cuda="13.0", hip=None),
|
||||
@@ -483,11 +483,14 @@ crate-type = ["cdylib"]
|
||||
self.assertEqual(build.environment["LIBTORCH_CXX11_ABI"], "1")
|
||||
self.assertEqual(build.environment["LIBTORCH_BYPASS_VERSION_CHECK"], "1")
|
||||
self.assertIn(str(compat_header), build.environment["CXXFLAGS"])
|
||||
# -std=c++20 must trail the inherited CXXFLAGS to beat a caller's -std=;
|
||||
# torch-sys's own -std=c++17 loses anyway, as cc-rs appends CXXFLAGS.
|
||||
self.assertRegex(build.environment["CXXFLAGS"], r"^-O2\b.*\s-std=c\+\+20\b")
|
||||
self.assertIn(
|
||||
"$ORIGIN/../../../../torch/lib", build.environment["RUSTFLAGS"]
|
||||
)
|
||||
self.assertIn(str(torch_root / "lib"), build.environment["RUSTFLAGS"])
|
||||
self.assertEqual(build.fingerprint["torch_version"], "2.13.0+cu130")
|
||||
self.assertEqual(build.fingerprint["torch_version"], "2.14.1+cu130")
|
||||
self.assertTrue(build.fingerprint["torch_cxx11_abi"])
|
||||
|
||||
wheel_build = torch_build_configuration(
|
||||
@@ -502,8 +505,37 @@ crate-type = ["cdylib"]
|
||||
)
|
||||
self.assertFalse(wheel_build.fingerprint["include_absolute_rpath"])
|
||||
|
||||
fake_torch.__version__ = "2.14.0"
|
||||
with self.assertRaisesRegex(RuntimeError, "PyTorch 2.11 through 2.13"):
|
||||
# libtorch declares C++20 from 2.12 on; 2.11 is the only C++17 release.
|
||||
for version, expected in (("2.12.0+cu128", True), ("2.11.0+cu128", False)):
|
||||
fake_torch.__version__ = version
|
||||
older_build = torch_build_configuration(
|
||||
compat_header=compat_header,
|
||||
python_module="sglang.srt.mem_cache.rust_tree_core.mem_cache",
|
||||
torch_module=fake_torch,
|
||||
base_environment={},
|
||||
)
|
||||
self.assertEqual(
|
||||
"-std=c++20" in older_build.environment["CXXFLAGS"],
|
||||
expected,
|
||||
msg=version,
|
||||
)
|
||||
|
||||
# cc-rs keeps a quoted, spaced header path whole only in shell-words mode.
|
||||
fake_torch.__version__ = "2.14.1+cu130"
|
||||
spaced_header = root / "compat dir" / "compat.h"
|
||||
spaced_header.parent.mkdir()
|
||||
spaced_header.write_text("// compatibility\n", encoding="utf-8")
|
||||
spaced_build = torch_build_configuration(
|
||||
compat_header=spaced_header,
|
||||
python_module="sglang.srt.mem_cache.rust_tree_core.mem_cache",
|
||||
torch_module=fake_torch,
|
||||
base_environment={},
|
||||
)
|
||||
self.assertEqual(spaced_build.environment["CC_SHELL_ESCAPED_FLAGS"], "1")
|
||||
self.assertIn(f"'{spaced_header}'", spaced_build.environment["CXXFLAGS"])
|
||||
|
||||
fake_torch.__version__ = "2.15.0"
|
||||
with self.assertRaisesRegex(RuntimeError, "PyTorch 2.11 through 2.14"):
|
||||
torch_build_configuration(
|
||||
compat_header=compat_header,
|
||||
python_module="sglang.srt.mem_cache.rust_tree_core.mem_cache",
|
||||
|
||||
@@ -280,7 +280,7 @@ class UnifiedRadixCacheTreeCoreSelectionTest(CustomTestCase):
|
||||
mock.patch.dict(os.environ),
|
||||
mock.patch.dict(tree_core_registry.sys.modules),
|
||||
mock.patch.object(tree_core_registry.sys, "platform", "linux"),
|
||||
mock.patch.object(tree_core_registry.torch, "__version__", "2.13.0"),
|
||||
mock.patch.object(tree_core_registry.torch, "__version__", "2.14.1"),
|
||||
mock.patch.object(
|
||||
tree_core_registry.importlib.util,
|
||||
"find_spec",
|
||||
@@ -327,7 +327,7 @@ class UnifiedRadixCacheTreeCoreSelectionTest(CustomTestCase):
|
||||
mock.patch.dict(tree_core_registry.sys.modules),
|
||||
mock.patch.object(tree_core_registry.sys, "platform", "linux"),
|
||||
mock.patch.object(
|
||||
tree_core_registry.torch, "__version__", "2.13.0"
|
||||
tree_core_registry.torch, "__version__", "2.14.1"
|
||||
),
|
||||
mock.patch.object(
|
||||
tree_core_registry.importlib.util,
|
||||
@@ -357,7 +357,7 @@ class TreeCoreDefaultCompatibilityTest(CustomTestCase):
|
||||
mock.patch.dict(os.environ, {"SGLANG_RUST_BUILD_MODE": "auto"}),
|
||||
mock.patch.dict(tree_core_registry.sys.modules),
|
||||
mock.patch.object(tree_core_registry.sys, "platform", "linux"),
|
||||
mock.patch.object(tree_core_registry.torch, "__version__", "2.13.0"),
|
||||
mock.patch.object(tree_core_registry.torch, "__version__", "2.14.1"),
|
||||
mock.patch.object(
|
||||
tree_core_registry.importlib.util, "find_spec", return_value=None
|
||||
),
|
||||
@@ -497,7 +497,7 @@ class TreeCoreDefaultCompatibilityTest(CustomTestCase):
|
||||
self.assertEqual(
|
||||
select_tree_core_backend(_cache_init_params()), "python"
|
||||
)
|
||||
for version in ("2.10.0", "2.14.0", "unknown"):
|
||||
for version in ("2.10.0", "2.15.0", "unknown"):
|
||||
with (
|
||||
self.subTest(version=version),
|
||||
mock.patch.object(tree_core_registry.torch, "__version__", version),
|
||||
|
||||
Reference in New Issue
Block a user