From eae808903ff571d9e00e65cea8ef03ba9a1b309f Mon Sep 17 00:00:00 2001 From: Mohammad Miadh Angkad <176301910+mmangkad@users.noreply.github.com> Date: Wed, 30 Sep 2026 19:42:19 +0000 Subject: [PATCH] [Deps] Bump FlashInfer to 0.7.0.post1 (#40709) Co-authored-by: Mohammad Angkad --- docker/Dockerfile | 4 +-- docker/Dockerfile.cu134 | 8 ++--- python/pyproject.toml | 6 ++-- .../attention/flash_attn/cute/pyproject.toml | 6 ++-- .../ops/attention/flash_attn/cute/utils.py | 5 ++- .../backends/subblock_sparse/README.md | 2 +- python/sglang/srt/entrypoints/engine.py | 2 +- .../attention/flashinfer_mla_backend.py | 5 +++ python/sglang/srt/utils/common.py | 11 +------ .../ci/cuda/ci_check_flashinfer_jit_cache.sh | 33 +++++++++++++++++++ .../cuda/ci_download_flashinfer_jit_cache.sh | 29 ++++++++++------ scripts/ci/cuda/ci_install_dependency.sh | 26 +++++++++++---- 12 files changed, 93 insertions(+), 44 deletions(-) create mode 100755 scripts/ci/cuda/ci_check_flashinfer_jit_cache.sh diff --git a/docker/Dockerfile b/docker/Dockerfile index f235c15ff8..1a35b6c76e 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -13,7 +13,7 @@ ARG PIP_DEFAULT_INDEX ARG UBUNTU_MIRROR ARG GITHUB_ARTIFACTORY=github.com ARG INSTALL_FLASHINFER_JIT_CACHE=0 -ARG FLASHINFER_VERSION=0.6.18 +ARG FLASHINFER_VERSION=0.7.0.post1 ARG MOONCAKE_VERSION=0.3.13.post1 ARG MSCCLPP_VERSION=sglang-v0.10.1 @@ -286,7 +286,7 @@ RUN --mount=type=cache,target=/root/.cache/pip \ && if [ "$INSTALL_FLASHINFER_JIT_CACHE" = "1" ]; then \ python3 -m pip install flashinfer-jit-cache==${FLASHINFER_VERSION} --index-url https://flashinfer.ai/whl/cu${CUINDEX} \ && cp -r /opt/sglang/lib/python3.12/site-packages/flashinfer_jit_cache /flashinfer_jit_output/ \ - && cp -r /opt/sglang/lib/python3.12/site-packages/flashinfer_jit_cache-*.dist-info /flashinfer_jit_output/ ; \ + && cp -r /opt/sglang/lib/python3.12/site-packages/flashinfer_jit_cache*.dist-info /flashinfer_jit_output/ ; \ fi ######################################################## diff --git a/docker/Dockerfile.cu134 b/docker/Dockerfile.cu134 index a6e321109f..09ce19d455 100644 --- a/docker/Dockerfile.cu134 +++ b/docker/Dockerfile.cu134 @@ -27,16 +27,14 @@ ARG PIP_DEFAULT_INDEX ARG UBUNTU_MIRROR ARG GITHUB_ARTIFACTORY=github.com ARG INSTALL_FLASHINFER_JIT_CACHE=1 -ARG FLASHINFER_VERSION=0.6.18 -ARG CUTLASS_DSL_VERSION=4.8.0.dev0 +ARG FLASHINFER_VERSION=0.7.0.post1 ARG MOONCAKE_VERSION=0.3.12.post1 ARG MSCCLPP_VERSION=sglang-v0.9.1 ENV DEBIAN_FRONTEND=noninteractive \ CUDA_HOME=/usr/local/cuda \ GDRCOPY_HOME=/usr/src/gdrdrv-${GDRCOPY_VERSION}/ \ - FLASHINFER_VERSION=${FLASHINFER_VERSION} \ - CUTLASS_DSL_VERSION=${CUTLASS_DSL_VERSION} + FLASHINFER_VERSION=${FLASHINFER_VERSION} # Add GKE default lib and bin locations ENV PATH="${PATH}:/usr/local/nvidia/bin" \ @@ -549,7 +547,7 @@ RUN --mount=type=cache,target=/root/.cache/pip \ && if [ "$INSTALL_FLASHINFER_JIT_CACHE" = "1" ]; then \ python3 -m pip install flashinfer-jit-cache==${FLASHINFER_VERSION} --index-url https://flashinfer.ai/whl/cu${CUINDEX} \ && cp -r /opt/sglang/lib/python3.12/site-packages/flashinfer_jit_cache /flashinfer_jit_output/ \ - && cp -r /opt/sglang/lib/python3.12/site-packages/flashinfer_jit_cache-*.dist-info /flashinfer_jit_output/ ; \ + && cp -r /opt/sglang/lib/python3.12/site-packages/flashinfer_jit_cache*.dist-info /flashinfer_jit_output/ ; \ fi ######################################################## diff --git a/python/pyproject.toml b/python/pyproject.toml index c600c399e1..feda3c4626 100755 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -39,7 +39,7 @@ dependencies = [ "einops", "fastapi", "flash-attn-4>=4.0.0b18", - "flashinfer_python[cu13]==0.6.18", # keep it aligned with jit-cache version in Dockerfile + "flashinfer_python[cu13]==0.7.0.post1", # keep it aligned with jit-cache version in Dockerfile "gguf", "humming-kernels[cu13]==0.1.12", "interegular", @@ -52,7 +52,7 @@ dependencies = [ "ninja", "numba==0.65.1", "numpy", - "nvidia-cutlass-dsl[cu13]==4.6.2", + "nvidia-cutlass-dsl[cu13]==4.8.0", "nvidia-mathdx==25.6.0", "nvidia-ml-py", "nvshmem4py-cu13", @@ -70,7 +70,7 @@ dependencies = [ "pydantic", "python-multipart", "pyzmq>=25.1.2", - "quack-kernels==0.6.4", + "quack-kernels==0.6.5", "regex", "requests", "scipy", diff --git a/python/sglang/kernels/ops/attention/flash_attn/cute/pyproject.toml b/python/sglang/kernels/ops/attention/flash_attn/cute/pyproject.toml index 929595b698..031817c750 100644 --- a/python/sglang/kernels/ops/attention/flash_attn/cute/pyproject.toml +++ b/python/sglang/kernels/ops/attention/flash_attn/cute/pyproject.toml @@ -22,17 +22,17 @@ classifiers = [ ] dependencies = [ - "nvidia-cutlass-dsl==4.6.2", + "nvidia-cutlass-dsl[cu13]==4.8.0", "torch", "einops", "typing_extensions", "apache-tvm-ffi>=0.1.5,<0.2", "torch-c-dlpack-ext", - "quack-kernels==0.6.4", + "quack-kernels==0.6.5", ] [project.optional-dependencies] -cu13 = ["nvidia-cutlass-dsl[cu13]==4.6.2"] +cu13 = ["nvidia-cutlass-dsl[cu13]==4.8.0"] dev = [ "pytest", "pytest-xdist", diff --git a/python/sglang/kernels/ops/attention/flash_attn/cute/utils.py b/python/sglang/kernels/ops/attention/flash_attn/cute/utils.py index 293d461a14..1afa681441 100644 --- a/python/sglang/kernels/ops/attention/flash_attn/cute/utils.py +++ b/python/sglang/kernels/ops/attention/flash_attn/cute/utils.py @@ -9,7 +9,6 @@ from typing import Callable, NamedTuple, Optional, Tuple, Type, overload import cutlass import cutlass.cute as cute -import quack.activation from cutlass import Float32, Int32, const_expr from cutlass._mlir.dialects import llvm, nvvm from cutlass.cute import FastDivmodDivisor @@ -925,10 +924,10 @@ def ex2_emulation_2( ) # The integer floor of x & y are now in the last 8 bits of xy_rounded # We want the next 2 ops to round to nearest even. The rounding mode is important. - xy_rounded_back = quack.activation.sub_packed_f32x2( + xy_rounded_back = cute.arch.sub_packed_f32x2( xy_rounded, (fp32_round_int, fp32_round_int) ) - xy_frac = quack.activation.sub_packed_f32x2(xy_clamped, xy_rounded_back) + xy_frac = cute.arch.sub_packed_f32x2(xy_clamped, xy_rounded_back) xy_frac_ex2 = evaluate_polynomial_2(*xy_frac, POLY_EX2[poly_degree], loc=loc, ip=ip) x_out = combine_int_frac_ex2(xy_rounded[0], xy_frac_ex2[0], loc=loc, ip=ip) y_out = combine_int_frac_ex2(xy_rounded[1], xy_frac_ex2[1], loc=loc, ip=ip) diff --git a/python/sglang/multimodal_gen/runtime/layers/attention/backends/subblock_sparse/README.md b/python/sglang/multimodal_gen/runtime/layers/attention/backends/subblock_sparse/README.md index ac773b816f..61639571c2 100644 --- a/python/sglang/multimodal_gen/runtime/layers/attention/backends/subblock_sparse/README.md +++ b/python/sglang/multimodal_gen/runtime/layers/attention/backends/subblock_sparse/README.md @@ -109,7 +109,7 @@ pip install git+https://github.com/thu-ml/SpargeAttn.git --no-build-isolation SM120 requires FlashInfer's CuTe-DSL SM120 Sage backend and `quantize_sage_qkv_sm120`, added in [FlashInfer #4691](https://github.com/flashinfer-ai/flashinfer/pull/4691). -The default FlashInfer `0.6.18` pin does not include these APIs. After installing +The default FlashInfer `0.7.0.post1` pin does not include these APIs. After installing SGLang, install the FlashInfer source revision used for SM120 validation: ```bash diff --git a/python/sglang/srt/entrypoints/engine.py b/python/sglang/srt/entrypoints/engine.py index 4df8182334..af49cbe0cb 100644 --- a/python/sglang/srt/entrypoints/engine.py +++ b/python/sglang/srt/entrypoints/engine.py @@ -1848,7 +1848,7 @@ def _set_envs_and_config(server_args: ServerArgs): ): assert_pkg_version( "flashinfer_python", - "0.6.18", + "0.7.0.post1", "Please uninstall the old version and " "reinstall the latest version by following the instructions " "at https://docs.flashinfer.ai/installation.html.", diff --git a/python/sglang/srt/layers/attention/flashinfer_mla_backend.py b/python/sglang/srt/layers/attention/flashinfer_mla_backend.py index 329223f877..0ee38fbacb 100644 --- a/python/sglang/srt/layers/attention/flashinfer_mla_backend.py +++ b/python/sglang/srt/layers/attention/flashinfer_mla_backend.py @@ -934,6 +934,11 @@ class FlashInferMLAIndicesUpdaterDecode: valid.copy_(translator.translate_dcp_read_ids(valid)) else: kv_indptr, kv_indices = spec_info.kv_indptr, spec_info.kv_indices + if not init_metadata_replay: + # FlashInfer >= 0.7.0 rejects a plan whose kv_len_arr disagrees with + # kv_indptr, and a draft branch spans its prefix plus one per step. + # Replay plans through fast_mla_decode_plan, which skips that check. + kv_lens = (kv_indptr[1:] - kv_indptr[:-1]).to(torch.int32) if not init_metadata_replay: wrapper.plan( diff --git a/python/sglang/srt/utils/common.py b/python/sglang/srt/utils/common.py index 67bf51b698..f4e38cf9e8 100644 --- a/python/sglang/srt/utils/common.py +++ b/python/sglang/srt/utils/common.py @@ -2264,16 +2264,7 @@ def assert_pkg_version(pkg: str, min_version: str, message: str): def check_pkg_version_at_least(pkg: str, min_version: str) -> bool: - """ - Check if a package is installed and meets the minimum version requirement. - - Args: - pkg: Package name (distribution name, e.g., "flashinfer-python") - min_version: Minimum version required (e.g., "0.6.18") - - Returns: - True if package is installed and version >= min_version, False otherwise - """ + """Check if a package is installed and meets the minimum version requirement.""" if _should_skip_kernel_pkg_version_check(pkg): return True diff --git a/scripts/ci/cuda/ci_check_flashinfer_jit_cache.sh b/scripts/ci/cuda/ci_check_flashinfer_jit_cache.sh new file mode 100755 index 0000000000..8a39f63e21 --- /dev/null +++ b/scripts/ci/cuda/ci_check_flashinfer_jit_cache.sh @@ -0,0 +1,33 @@ +#!/bin/bash +# Exit 0 when the installed flashinfer-jit-cache can actually serve cubins. +# Since 0.7.0 they live in per-arch dists found through entry points, so a shim +# with none registered still imports and still reports the expected version. +python3 - <<'PY' +import sys + +try: + import flashinfer_jit_cache +except Exception as error: + print(f"flashinfer-jit-cache is not importable: {error}") + sys.exit(1) + +try: + from flashinfer_jit_cache import get_jit_cache_providers +except ImportError: + # Pre-0.7.0 layout: the cubins ship inside the package itself. + print(f"flashinfer-jit-cache {flashinfer_jit_cache.__version__} (pre-provider layout)") + sys.exit(0) + +providers = get_jit_cache_providers() +if not providers: + print( + f"flashinfer-jit-cache {flashinfer_jit_cache.__version__} has no usable " + "provider; the per-arch packages are missing or their entry points were lost" + ) + sys.exit(1) + +print( + f"flashinfer-jit-cache {flashinfer_jit_cache.__version__} with " + f"{len(providers)} provider(s): {', '.join(p.provider_id for p in providers)}" +) +PY diff --git a/scripts/ci/cuda/ci_download_flashinfer_jit_cache.sh b/scripts/ci/cuda/ci_download_flashinfer_jit_cache.sh index 9dbe9c47a5..47e7dd306f 100755 --- a/scripts/ci/cuda/ci_download_flashinfer_jit_cache.sh +++ b/scripts/ci/cuda/ci_download_flashinfer_jit_cache.sh @@ -1,6 +1,6 @@ #!/bin/bash # Install flashinfer-jit-cache with caching and retry logic (flashinfer.ai can have transient DNS issues). -# The jit-cache wheel is 1.2+ GB, so we skip the download entirely if already installed. +# The jit-cache is 1.2+ GB across its wheels, so we skip the download entirely if already installed. # # Required environment (caller must export or set): # UNINSTALL_JIT_CACHE — literal true/false (skip download when false) @@ -25,17 +25,19 @@ if [ "$FLASHINFER_JIT_CACHE_INSTALLED" = false ]; then FLASHINFER_CACHE_DIR="${HOME}/.cache/flashinfer-wheels" mkdir -p "${FLASHINFER_CACHE_DIR}" - FLASHINFER_WHEEL_PATTERN="flashinfer_jit_cache-${FLASHINFER_PYTHON_REQUIRED}+${CU_VERSION}*.whl" - CACHED_WHEEL=$(find "${FLASHINFER_CACHE_DIR}" -name "${FLASHINFER_WHEEL_PATTERN}" -type f 2>/dev/null | head -n 1) + # The per-arch wheel names are unclaimed on PyPI, so resolve the whole set + # from disk. The glob also matches the single pre-0.7.0 wheel. + FLASHINFER_WHEEL_GLOB="${FLASHINFER_CACHE_DIR}/flashinfer_jit_cache*-${FLASHINFER_PYTHON_REQUIRED}+${CU_VERSION}-*.whl" - if [ -n "$CACHED_WHEEL" ] && [ -f "$CACHED_WHEEL" ]; then - echo "Found cached flashinfer wheel: $CACHED_WHEEL" - if $PIP_CMD install "$CACHED_WHEEL" $PIP_INSTALL_SUFFIX; then + set -- $FLASHINFER_WHEEL_GLOB + if [ -f "$1" ]; then + echo "Found $# cached flashinfer wheel(s)" + if $PIP_CMD install --no-index "$@" $PIP_INSTALL_SUFFIX; then FLASHINFER_JIT_CACHE_INSTALLED=true echo "Successfully installed flashinfer-jit-cache from cache" else echo "Failed to install from cache, will try downloading..." - rm -f "$CACHED_WHEEL" + rm -f "$@" fi fi @@ -46,9 +48,9 @@ if [ "$FLASHINFER_JIT_CACHE_INSTALLED" = false ]; then --index-url "https://flashinfer.ai/whl/${CU_VERSION}" \ -d "${FLASHINFER_CACHE_DIR}"; then - CACHED_WHEEL=$(find "${FLASHINFER_CACHE_DIR}" -name "${FLASHINFER_WHEEL_PATTERN}" -type f 2>/dev/null | head -n 1) - if [ -n "$CACHED_WHEEL" ] && [ -f "$CACHED_WHEEL" ]; then - if $PIP_CMD install "$CACHED_WHEEL" $PIP_INSTALL_SUFFIX; then + set -- $FLASHINFER_WHEEL_GLOB + if [ -f "$1" ]; then + if $PIP_CMD install --no-index "$@" $PIP_INSTALL_SUFFIX; then FLASHINFER_JIT_CACHE_INSTALLED=true echo "Successfully downloaded and installed flashinfer-jit-cache" break @@ -67,3 +69,10 @@ if [ "$FLASHINFER_JIT_CACHE_INSTALLED" = false ]; then echo "ERROR: Failed to install flashinfer-jit-cache after 5 attempts" exit 1 fi + +# A resolve can succeed and still register no provider, which degrades silently. +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +if ! bash "${SCRIPT_DIR}/ci_check_flashinfer_jit_cache.sh"; then + echo "ERROR: flashinfer-jit-cache installed but serves no cubins" + exit 1 +fi diff --git a/scripts/ci/cuda/ci_install_dependency.sh b/scripts/ci/cuda/ci_install_dependency.sh index 3df540aeb3..084f5cd817 100755 --- a/scripts/ci/cuda/ci_install_dependency.sh +++ b/scripts/ci/cuda/ci_install_dependency.sh @@ -424,9 +424,9 @@ uninstall_stale_flashinfer() { FLASHINFER_PYTHON_REQUIRED=$(grep -Po -m1 'flashinfer_python(\[[^]]+\])?==\K[0-9A-Za-z\.\-]+' python/pyproject.toml || echo "") # flashinfer-cubin is no longer a pyproject dependency (installed explicitly below), tracks the same version as flashinfer_python FLASHINFER_CUBIN_REQUIRED="$FLASHINFER_PYTHON_REQUIRED" - FLASHINFER_CUBIN_INSTALLED=$(pip show flashinfer-cubin 2>/dev/null | grep "^Version:" | awk '{print $2}' || echo "") - FLASHINFER_JIT_INSTALLED=$(pip show flashinfer-jit-cache 2>/dev/null | grep "^Version:" | awk '{print $2}' | sed 's/+.*//' || echo "") - FLASHINFER_JIT_CU_VERSION=$(pip show flashinfer-jit-cache 2>/dev/null | grep "^Version:" | awk '{print $2}' | sed -n 's/.*+//p' || echo "") + FLASHINFER_CUBIN_INSTALLED=$(python3 -m pip show flashinfer-cubin 2>/dev/null | grep "^Version:" | awk '{print $2}' || echo "") + FLASHINFER_JIT_INSTALLED=$(python3 -m pip show flashinfer-jit-cache 2>/dev/null | grep "^Version:" | awk '{print $2}' | sed 's/+.*//' || echo "") + FLASHINFER_JIT_CU_VERSION=$(python3 -m pip show flashinfer-jit-cache 2>/dev/null | grep "^Version:" | awk '{print $2}' | sed -n 's/.*+//p' || echo "") UNINSTALL_CUBIN=true UNINSTALL_JIT_CACHE=true @@ -439,8 +439,13 @@ uninstall_stale_flashinfer() { fi if [ "$FLASHINFER_JIT_INSTALLED" = "$FLASHINFER_PYTHON_REQUIRED" ] && [ -n "$FLASHINFER_PYTHON_REQUIRED" ]; then - echo "flashinfer-jit-cache==${FLASHINFER_PYTHON_REQUIRED} already installed, keeping it" - UNINSTALL_JIT_CACHE=false + # Since 0.7.0 that version is the shim's; it does not imply a provider. + if bash "${SCRIPT_DIR}/ci_check_flashinfer_jit_cache.sh"; then + echo "flashinfer-jit-cache==${FLASHINFER_PYTHON_REQUIRED} already installed, keeping it" + UNINSTALL_JIT_CACHE=false + else + echo "flashinfer-jit-cache==${FLASHINFER_PYTHON_REQUIRED} installed but serves no cubins, will reinstall" + fi else echo "flashinfer-jit-cache version mismatch (installed: ${FLASHINFER_JIT_INSTALLED:-none}, required: ${FLASHINFER_PYTHON_REQUIRED}), will reinstall" fi @@ -452,7 +457,16 @@ uninstall_stale_flashinfer() { FLASHINFER_UNINSTALL="flashinfer-python" [ "$UNINSTALL_CUBIN" = true ] && FLASHINFER_UNINSTALL="$FLASHINFER_UNINSTALL flashinfer-cubin" - [ "$UNINSTALL_JIT_CACHE" = true ] && FLASHINFER_UNINSTALL="$FLASHINFER_UNINSTALL flashinfer-jit-cache" + if [ "$UNINSTALL_JIT_CACHE" = true ]; then + # Which per-arch packages exist varies by release, so drop whatever is + # installed rather than a fixed list. + JIT_CACHE_PKGS=$(python3 -m pip list --format=freeze) || { + echo "ERROR: pip list failed; cannot enumerate per-arch jit-cache packages" + return 1 + } + JIT_CACHE_ARCH=$(printf '%s\n' "$JIT_CACHE_PKGS" | sed -n 's/^\(flashinfer-jit-cache-[0-9a-z]*\)==.*/\1/p' | tr '\n' ' ') + FLASHINFER_UNINSTALL="$FLASHINFER_UNINSTALL flashinfer-jit-cache $JIT_CACHE_ARCH" + fi $PIP_UNINSTALL_CMD $FLASHINFER_UNINSTALL $PIP_UNINSTALL_SUFFIX || true $PIP_UNINSTALL_CMD opencv-python opencv-python-headless $PIP_UNINSTALL_SUFFIX || true