[Deps] Upgrade the CUDA PyTorch stack to 2.14 (#38641)
Co-authored-by: Mohammad Angkad <mohammad.angkad@radixark.ai>
This commit is contained in:
co-authored by
Mohammad Angkad
parent
9631f6f2d0
commit
0bdd0cc8dc
@@ -3,7 +3,7 @@ requires = [
|
||||
"setuptools>=61.0",
|
||||
"setuptools-rust>=1.11",
|
||||
"setuptools-scm>=8.0",
|
||||
"torch==2.13.0",
|
||||
"torch==2.14.1",
|
||||
"wheel",
|
||||
]
|
||||
build-backend = "setuptools.build_meta"
|
||||
@@ -76,9 +76,9 @@ dependencies = [
|
||||
"scipy",
|
||||
"sentencepiece==0.2.1", # 0.2.2 rejects null pieces in InternVL tokenizers.
|
||||
"setproctitle",
|
||||
"sgl-deep-ep==0.1.2",
|
||||
"sgl-deep-gemm==0.2.0",
|
||||
"sglang-kernel==0.4.8",
|
||||
"sgl-deep-ep==0.1.2.post1",
|
||||
"sgl-deep-gemm==0.2.1.post1",
|
||||
"sglang-kernel==0.4.9",
|
||||
"smg-grpc-servicer>=0.9.0",
|
||||
"soundfile==0.13.1",
|
||||
"tiktoken",
|
||||
@@ -87,10 +87,10 @@ dependencies = [
|
||||
"tokenizers==0.23.2",
|
||||
"tokenspeed_mla==0.1.8",
|
||||
"tomli ; python_version < '3.11'",
|
||||
"torch==2.13.0",
|
||||
"torch==2.14.1",
|
||||
"torch_memory_saver>=0.0.9.post1",
|
||||
"torchaudio==2.11.0",
|
||||
"torchcodec==0.15.0 ; sys_platform != 'linux' or (sys_platform == 'linux' and platform_machine != 'aarch64' and platform_machine != 'arm64' and platform_machine != 'armv7l')", # Not available on Linux ARM.
|
||||
"torchcodec==0.17.0 ; sys_platform != 'linux' or (sys_platform == 'linux' and platform_machine != 'aarch64' and platform_machine != 'arm64' and platform_machine != 'armv7l')", # Not available on Linux ARM.
|
||||
"torchvision",
|
||||
"tqdm",
|
||||
"transformers==5.17.0",
|
||||
|
||||
@@ -332,7 +332,9 @@ def _fwd_kernel_stage1(
|
||||
mask=(offs_n[:, None] < split_kv_end) & (mask_d[None, :]),
|
||||
other=0.0,
|
||||
)
|
||||
qk = tl.sum(q[None, :] * k, 1)
|
||||
# Reduce in fp32; a bf16 tl.sum rounds each partial sum in compiler order.
|
||||
# Cast the product, not q and k, to avoid a fp32 [BLOCK_N, D] tile.
|
||||
qk = tl.sum((q[None, :] * k).to(tl.float32), 1)
|
||||
qk *= sm_scale_withk
|
||||
|
||||
if logit_cap > 0:
|
||||
|
||||
@@ -16,8 +16,8 @@ try:
|
||||
from triton.experimental.gluon.language.nvidia.blackwell import get_tmem_reg_layout
|
||||
except ImportError:
|
||||
# Triton 3.8 moved this onto the descriptor type (triton-lang/triton#9594)
|
||||
# TODO(tmorris): After we update to triton 3.8 (torch 2.15), use new API directly
|
||||
# instead of this compatibility wrapper.
|
||||
# TODO: once ROCm leaves triton 3.7 (swapab_gluon_hip imports _load_v4
|
||||
# from here), drop the try import above and call get_reg_layout directly.
|
||||
from triton.experimental.gluon.language.nvidia.blackwell import (
|
||||
tensor_memory_descriptor_type,
|
||||
)
|
||||
|
||||
@@ -152,7 +152,9 @@ class SDPAImpl(AttentionImpl):
|
||||
):
|
||||
# Keep the existing Flash SDPA arithmetic while consuming all
|
||||
# packed windows in one call, including ragged and empty windows.
|
||||
return torch_varlen_attn(
|
||||
# Call the Flash op directly: from torch 2.14, varlen_attn silently
|
||||
# routes sm90/sm100 to cuDNN, which changes outputs versus the Flash path.
|
||||
output, *_ = torch.ops.aten._flash_attention_forward(
|
||||
query,
|
||||
key,
|
||||
value,
|
||||
@@ -160,9 +162,14 @@ class SDPAImpl(AttentionImpl):
|
||||
cu_seqlens,
|
||||
max_seqlen,
|
||||
max_seqlen,
|
||||
0.0,
|
||||
self.causal,
|
||||
return_debug_mask=False,
|
||||
scale=self.softmax_scale,
|
||||
window_size=(-1, 0) if self.causal else (-1, -1),
|
||||
window_size_left=-1,
|
||||
window_size_right=0 if self.causal else -1,
|
||||
)
|
||||
return output
|
||||
bounds = (
|
||||
cu_seqlens_host
|
||||
if cu_seqlens_host is not None
|
||||
|
||||
@@ -40,8 +40,8 @@
|
||||
"zimage_image_t2i": {
|
||||
"clip_threshold": 0.97,
|
||||
"ssim_threshold": 0.86,
|
||||
"psnr_threshold": 19.9,
|
||||
"mean_abs_diff_threshold": 8.5
|
||||
"psnr_threshold": 19.3,
|
||||
"mean_abs_diff_threshold": 9.8
|
||||
},
|
||||
"zimage_image_t2i_multi_lora": {
|
||||
"clip_threshold": 0.98,
|
||||
|
||||
@@ -40,7 +40,7 @@ logger = init_logger(__name__)
|
||||
# NPU/ascend) is read from sgl-project/ci-data-diffusion, where the GT-gen workflows
|
||||
# publish.
|
||||
SGL_TEST_FILES_CI_DATA_REPO = "sgl-project/ci-data-diffusion"
|
||||
SGL_TEST_FILES_CI_DATA_REVISION = "dbb70135da54b1f2b172886b28e813d2bcdd16c1"
|
||||
SGL_TEST_FILES_CI_DATA_REVISION = "972007201d82d7dcce9e35b7e4b151c2b1181c30"
|
||||
|
||||
# The NPU pin is kept as a separate branch so ascend GT can be bumped independently
|
||||
# when it's regenerated on its own cadence.
|
||||
|
||||
@@ -13,6 +13,13 @@ class TestLTX2CausalConvChannelsLast(unittest.TestCase):
|
||||
the original repeat()+concatenate() temporal padding (only the conv kernel's
|
||||
floating-point accumulation order may differ, which fp32 keeps negligible)."""
|
||||
|
||||
def setUp(self):
|
||||
# From torch 2.14 (cuDNN 9.24), cuDNN picks a TF32 kernel for the
|
||||
# channels_last_3d conv, which this fp32 comparison would read as drift.
|
||||
flags = torch.backends.cudnn.flags(enabled=True, allow_tf32=False)
|
||||
flags.__enter__()
|
||||
self.addCleanup(flags.__exit__, None, None, None)
|
||||
|
||||
def _check(self, causal: bool, kernel_size):
|
||||
device = "cuda" if torch.cuda.is_available() else "cpu"
|
||||
torch.manual_seed(0)
|
||||
|
||||
@@ -1856,7 +1856,7 @@ def _set_envs_and_config(server_args: ServerArgs):
|
||||
if _is_cuda:
|
||||
assert_pkg_version(
|
||||
"sglang-kernel",
|
||||
"0.4.8",
|
||||
"0.4.9",
|
||||
"Please reinstall the latest version with `pip install sglang-kernel --force-reinstall`",
|
||||
)
|
||||
|
||||
|
||||
@@ -59,7 +59,7 @@ try:
|
||||
):
|
||||
if simulated_ep != 1:
|
||||
raise NotImplementedError(
|
||||
"simulated_ep routing is not supported with triton_kernels 3.7.1"
|
||||
"simulated_ep routing is not supported with triton_kernels"
|
||||
)
|
||||
|
||||
if sm_first:
|
||||
|
||||
@@ -18,7 +18,7 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
from dataclasses import replace
|
||||
from dataclasses import fields, replace
|
||||
from functools import lru_cache
|
||||
from typing import TYPE_CHECKING, List, Optional
|
||||
|
||||
@@ -227,6 +227,20 @@ def _pad_hopper_mxfp4_scale(scale, k_size):
|
||||
return torch.nn.functional.pad(scale, (0, want - scale.shape[-1]), value=_UE8M0_ONE)
|
||||
|
||||
|
||||
@lru_cache(maxsize=1)
|
||||
def _mx_block_kwargs():
|
||||
# triton_kernels 3.8 rejects an mx scale without its block size; the field is
|
||||
# absent in 3.7, which ROCm still ships, so probe before importing the value.
|
||||
from triton_kernels.matmul import PrecisionConfig
|
||||
|
||||
if "b_microblock_size" not in {f.name for f in fields(PrecisionConfig)}:
|
||||
return {}
|
||||
|
||||
from triton_kernels.numerics_details.mxfp import MXFP_BLOCK_SIZE
|
||||
|
||||
return {"b_microblock_size": int(MXFP_BLOCK_SIZE)}
|
||||
|
||||
|
||||
def _swizzle_mxfp4(quant_tensor, scale, num_warps):
|
||||
"""weight swizzle for mxfp4 moe, used for OAI mxfp4 kernel"""
|
||||
import triton_kernels.matmul_details.opt_flags as opt_flags
|
||||
@@ -1042,11 +1056,12 @@ class Mxfp4MoEMethod(FusedMoEMethodBase):
|
||||
layer.w2_weight, layer.w2_weight_scale, num_warps
|
||||
)
|
||||
|
||||
mx_block = _mx_block_kwargs()
|
||||
self.w13_precision_config = PrecisionConfig(
|
||||
b_mx_scale=w13_scale, flex_ctx=FlexCtx(rhs_data=w13_flex)
|
||||
b_mx_scale=w13_scale, flex_ctx=FlexCtx(rhs_data=w13_flex), **mx_block
|
||||
)
|
||||
self.w2_precision_config = PrecisionConfig(
|
||||
b_mx_scale=w2_scale, flex_ctx=FlexCtx(rhs_data=w2_flex)
|
||||
b_mx_scale=w2_scale, flex_ctx=FlexCtx(rhs_data=w2_flex), **mx_block
|
||||
)
|
||||
|
||||
self.w13_weight_triton_tensor = w13_weight
|
||||
|
||||
@@ -11,7 +11,7 @@ from sglang.srt.rust_extensions.torch_build import torch_build_configuration
|
||||
_PYTHON_MODULE = "sglang.srt.mem_cache.rust_tree_core.mem_cache"
|
||||
_INSPECTION_MODULE = "sglang.srt.mem_cache.rust_tree_core.mem_cache_inspection"
|
||||
_CRATE_DIR = Path(__file__).resolve().parents[5] / "rust" / "sglang-radix-tree"
|
||||
_TORCH_COMPAT_HEADER = _CRATE_DIR / "torch_2_13_compat.h"
|
||||
_TORCH_COMPAT_HEADER = _CRATE_DIR / "torch_compat.h"
|
||||
|
||||
|
||||
def load_tree_core_extension(*, inspection: bool = False):
|
||||
|
||||
@@ -13,7 +13,7 @@ from types import ModuleType
|
||||
from typing import Mapping
|
||||
|
||||
_MIN_SUPPORTED_TORCH = (2, 11)
|
||||
_MAX_SUPPORTED_TORCH = (2, 13)
|
||||
_MAX_SUPPORTED_TORCH = (2, 14)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
@@ -80,7 +80,7 @@ def torch_build_configuration(
|
||||
environment["LIBTORCH_LIB"] = os.fspath(torch_root)
|
||||
environment["LIBTORCH_CXX11_ABI"] = "1" if cxx11_abi else "0"
|
||||
# tch 0.24 targets Torch 2.11. The compatibility header below covers the
|
||||
# API removals in the supported 2.12/2.13 builds, after this explicit gate.
|
||||
# API removals in the supported 2.12-2.14 builds, after this explicit gate.
|
||||
environment["LIBTORCH_BYPASS_VERSION_CHECK"] = "1"
|
||||
environment["PYO3_PYTHON"] = sys.executable
|
||||
environment["PATH"] = os.pathsep.join(
|
||||
@@ -90,10 +90,16 @@ def torch_build_configuration(
|
||||
filter(None, (os.fspath(torch_lib), environment.get("LD_LIBRARY_PATH")))
|
||||
)
|
||||
|
||||
cxxflags = environment.get("CXXFLAGS", "")
|
||||
environment["CXXFLAGS"] = (
|
||||
f"{cxxflags} -include {shlex.quote(os.fspath(compat_header.resolve()))}"
|
||||
).strip()
|
||||
# cc-rs splits *FLAGS on whitespace unless told they are shell words,
|
||||
# which the quoted header path below needs to survive a space.
|
||||
environment["CC_SHELL_ESCAPED_FLAGS"] = "1"
|
||||
cxxflags = [environment.get("CXXFLAGS", "")]
|
||||
# torch-sys 0.24 hardcodes -std=c++17, but libtorch declares C++20 from 2.12 on;
|
||||
# cc-rs appends CXXFLAGS after a build script's own flags, so this one wins.
|
||||
if major_minor >= (2, 12):
|
||||
cxxflags.append("-std=c++20")
|
||||
cxxflags.append(f"-include {shlex.quote(os.fspath(compat_header.resolve()))}")
|
||||
environment["CXXFLAGS"] = " ".join(filter(None, cxxflags)).strip()
|
||||
|
||||
package_depth = len(python_module.split(".")) - 1
|
||||
bundled_torch_lib = "$ORIGIN/" + "../" * package_depth + "torch/lib"
|
||||
|
||||
@@ -595,7 +595,12 @@ def get_device_module():
|
||||
return torch.xpu
|
||||
if is_musa():
|
||||
return torch.musa
|
||||
return torch.get_device_module()
|
||||
# From torch 2.14, a bare torch.get_device_module() is torch.cuda on a CUDA wheel
|
||||
# even with no usable device; require an available accelerator instead.
|
||||
accelerator = torch.accelerator.current_accelerator(check_available=True)
|
||||
if accelerator is None:
|
||||
return torch.cpu
|
||||
return torch.get_device_module(accelerator)
|
||||
|
||||
|
||||
def create_device_stream(device):
|
||||
|
||||
Reference in New Issue
Block a user