minke.yu/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 13 Packages Projects Releases Wiki Activity

enable flashinfer-jit-cache in image build and ci install to speed up model launch (#14959)

Browse Source
This commit is contained in:
gongwei-130
2025-12-11 23:20:44 -08:00
committed by GitHub
parent 4dabfbc827
commit 94e1251131
2 changed files with 8 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/workflows/release-docker.yml
+5
View File
@@ -69,6 +69,7 @@ jobs:
--build-arg CUDA_VERSION=${{ matrix.variant.cuda_version }} \ --build-arg CUDA_VERSION=${{ matrix.variant.cuda_version }} \
--build-arg BUILD_TYPE=${{ matrix.variant.build_type }} \ --build-arg BUILD_TYPE=${{ matrix.variant.build_type }} \
--build-arg GRACE_BLACKWELL=${{ matrix.variant.grace_blackwell }} \ --build-arg GRACE_BLACKWELL=${{ matrix.variant.grace_blackwell }} \
--build-arg INSTALL_FLASHINFER_JIT_CACHE=1 \
-t lmsysorg/sglang:${tag} \ -t lmsysorg/sglang:${tag} \
--no-cache \ --no-cache \
. .
@@ -86,6 +87,7 @@ jobs:
--build-arg CUDA_VERSION=${{ matrix.variant.cuda_version }} \ --build-arg CUDA_VERSION=${{ matrix.variant.cuda_version }} \
--build-arg BUILD_TYPE=${{ matrix.variant.build_type }} \ --build-arg BUILD_TYPE=${{ matrix.variant.build_type }} \
--build-arg GRACE_BLACKWELL=${{ matrix.variant.grace_blackwell }} \ --build-arg GRACE_BLACKWELL=${{ matrix.variant.grace_blackwell }} \
--build-arg INSTALL_FLASHINFER_JIT_CACHE=1 \
-t lmsysorg/sglang:${tag} \ -t lmsysorg/sglang:${tag} \
--no-cache \ --no-cache \
. .
@@ -102,6 +104,7 @@ jobs:
-f docker/Dockerfile \ -f docker/Dockerfile \
--build-arg CUDA_VERSION=13.0.1 \ --build-arg CUDA_VERSION=13.0.1 \
--build-arg BUILD_TYPE=${{ matrix.variant.build_type }} \ --build-arg BUILD_TYPE=${{ matrix.variant.build_type }} \
--build-arg INSTALL_FLASHINFER_JIT_CACHE=1 \
--build-arg GRACE_BLACKWELL=0 \ --build-arg GRACE_BLACKWELL=0 \
-t lmsysorg/sglang:${tag} \ -t lmsysorg/sglang:${tag} \
--no-cache \ --no-cache \
@@ -146,6 +149,7 @@ jobs:
--build-arg CUDA_VERSION=${{ matrix.variant.cuda_version }} \ --build-arg CUDA_VERSION=${{ matrix.variant.cuda_version }} \
--build-arg BUILD_TYPE=${{ matrix.variant.build_type }} \ --build-arg BUILD_TYPE=${{ matrix.variant.build_type }} \
--build-arg GRACE_BLACKWELL=${{ matrix.variant.grace_blackwell }} \ --build-arg GRACE_BLACKWELL=${{ matrix.variant.grace_blackwell }} \
--build-arg INSTALL_FLASHINFER_JIT_CACHE=1 \
-t lmsysorg/sglang:${tag} \ -t lmsysorg/sglang:${tag} \
--no-cache \ --no-cache \
. .
@@ -163,6 +167,7 @@ jobs:
--build-arg CUDA_VERSION=${{ matrix.variant.cuda_version }} \ --build-arg CUDA_VERSION=${{ matrix.variant.cuda_version }} \
--build-arg BUILD_TYPE=${{ matrix.variant.build_type }} \ --build-arg BUILD_TYPE=${{ matrix.variant.build_type }} \
--build-arg GRACE_BLACKWELL=${{ matrix.variant.grace_blackwell }} \ --build-arg GRACE_BLACKWELL=${{ matrix.variant.grace_blackwell }} \
--build-arg INSTALL_FLASHINFER_JIT_CACHE=1 \
-t lmsysorg/sglang:${tag} \ -t lmsysorg/sglang:${tag} \
--no-cache \ --no-cache \
. .
scripts/ci/ci_install_dependency.sh
+3
View File
@@ -5,6 +5,7 @@ set -euxo pipefail
# Set up environment variables # Set up environment variables
IS_BLACKWELL=${IS_BLACKWELL:-0} IS_BLACKWELL=${IS_BLACKWELL:-0}
CU_VERSION="cu129" CU_VERSION="cu129"
FLASHINFER_VERSION=0.5.3
OPTIONAL_DEPS="${1:-}" OPTIONAL_DEPS="${1:-}"
# Detect system architecture # Detect system architecture
@@ -154,6 +155,8 @@ $PIP_CMD install nvidia-nvshmem-cu12==3.4.5 --force-reinstall $PIP_INSTALL_SUFFI
$PIP_CMD install nvidia-cudnn-cu12==9.16.0.29 --force-reinstall $PIP_INSTALL_SUFFIX $PIP_CMD install nvidia-cudnn-cu12==9.16.0.29 --force-reinstall $PIP_INSTALL_SUFFIX
$PIP_CMD uninstall xformers || true $PIP_CMD uninstall xformers || true
$PIP_CMD install flashinfer-jit-cache==${FLASHINFER_VERSION} --index-url https://flashinfer.ai/whl/${CU_VERSION}
# Show current packages # Show current packages
$PIP_CMD list $PIP_CMD list
python3 -c "import torch; print(torch.version.cuda)" python3 -c "import torch; print(torch.version.cuda)"