[CI] Skip flashinfer-cubin reinstall when version matches (#19470)
Co-authored-by: Alison Shao <alisonshao@MacBook-Pro-D2W773R9CD.local> Co-authored-by: Alison Shao <alisonshao@Mac.attlocal.net> Co-authored-by: Alison Shao <alisonshao@mac.lan>
This commit is contained in:
Alison Shao
co-authored by
Alison Shao
Alison Shao
Alison Shao
Alison Shao
Alison Shao
Alison Shao
parent
b3cfad0a80
commit
1c1712d8e5
@@ -118,7 +118,35 @@ fi
|
||||
|
||||
# Clean up existing installations
|
||||
$PIP_UNINSTALL_CMD sgl-kernel sglang $PIP_UNINSTALL_SUFFIX || true
|
||||
$PIP_UNINSTALL_CMD flashinfer-python flashinfer-cubin flashinfer-jit-cache $PIP_UNINSTALL_SUFFIX || true
|
||||
# Keep flashinfer packages installed if version matches to avoid re-downloading:
|
||||
# - flashinfer-cubin: 150+ MB, plus extra cubins from ci_download_flashinfer_cubin.sh
|
||||
# - flashinfer-jit-cache: 1.2+ GB, by far the largest download in CI
|
||||
FLASHINFER_CUBIN_REQUIRED=$(grep -Po -m1 '(?<=flashinfer_cubin==)[0-9A-Za-z\.\-]+' python/pyproject.toml || echo "")
|
||||
FLASHINFER_CUBIN_INSTALLED=$(pip show flashinfer-cubin 2>/dev/null | grep "^Version:" | awk '{print $2}' || echo "")
|
||||
FLASHINFER_JIT_INSTALLED=$(pip show flashinfer-jit-cache 2>/dev/null | grep "^Version:" | awk '{print $2}' | sed 's/+.*//' || echo "")
|
||||
|
||||
UNINSTALL_CUBIN=true
|
||||
UNINSTALL_JIT_CACHE=true
|
||||
|
||||
if [ "$FLASHINFER_CUBIN_INSTALLED" = "$FLASHINFER_CUBIN_REQUIRED" ] && [ -n "$FLASHINFER_CUBIN_REQUIRED" ]; then
|
||||
echo "flashinfer-cubin==${FLASHINFER_CUBIN_REQUIRED} already installed, keeping it"
|
||||
UNINSTALL_CUBIN=false
|
||||
else
|
||||
echo "flashinfer-cubin version mismatch (installed: ${FLASHINFER_CUBIN_INSTALLED:-none}, required: ${FLASHINFER_CUBIN_REQUIRED}), reinstalling"
|
||||
fi
|
||||
|
||||
if [ "$FLASHINFER_JIT_INSTALLED" = "$FLASHINFER_VERSION" ] && [ -n "$FLASHINFER_VERSION" ]; then
|
||||
echo "flashinfer-jit-cache==${FLASHINFER_VERSION} already installed, keeping it"
|
||||
UNINSTALL_JIT_CACHE=false
|
||||
else
|
||||
echo "flashinfer-jit-cache version mismatch (installed: ${FLASHINFER_JIT_INSTALLED:-none}, required: ${FLASHINFER_VERSION}), will reinstall"
|
||||
fi
|
||||
|
||||
# Build uninstall list based on what needs updating
|
||||
FLASHINFER_UNINSTALL="flashinfer-python"
|
||||
[ "$UNINSTALL_CUBIN" = true ] && FLASHINFER_UNINSTALL="$FLASHINFER_UNINSTALL flashinfer-cubin"
|
||||
[ "$UNINSTALL_JIT_CACHE" = true ] && FLASHINFER_UNINSTALL="$FLASHINFER_UNINSTALL flashinfer-jit-cache"
|
||||
$PIP_UNINSTALL_CMD $FLASHINFER_UNINSTALL $PIP_UNINSTALL_SUFFIX || true
|
||||
$PIP_UNINSTALL_CMD opencv-python opencv-python-headless $PIP_UNINSTALL_SUFFIX || true
|
||||
|
||||
# Install the main package
|
||||
@@ -236,52 +264,60 @@ fi
|
||||
$PIP_CMD uninstall xformers || true
|
||||
|
||||
# Install flashinfer-jit-cache with caching and retry logic (flashinfer.ai can have transient DNS issues)
|
||||
# Cache directory for flashinfer wheels (persists across CI runs on self-hosted runners)
|
||||
FLASHINFER_CACHE_DIR="${HOME}/.cache/flashinfer-wheels"
|
||||
mkdir -p "${FLASHINFER_CACHE_DIR}"
|
||||
|
||||
# Clean up old versions to avoid cache bloat
|
||||
find "${FLASHINFER_CACHE_DIR}" -name "flashinfer_jit_cache-*.whl" ! -name "flashinfer_jit_cache-${FLASHINFER_VERSION}*" -type f -delete 2>/dev/null || true
|
||||
|
||||
FLASHINFER_WHEEL_PATTERN="flashinfer_jit_cache-${FLASHINFER_VERSION}*.whl"
|
||||
CACHED_WHEEL=$(find "${FLASHINFER_CACHE_DIR}" -name "${FLASHINFER_WHEEL_PATTERN}" -type f 2>/dev/null | head -n 1)
|
||||
|
||||
# The jit-cache wheel is 1.2+ GB, so we skip the download entirely if already installed.
|
||||
FLASHINFER_INSTALLED=false
|
||||
|
||||
# Try to install from cache first
|
||||
if [ -n "$CACHED_WHEEL" ] && [ -f "$CACHED_WHEEL" ]; then
|
||||
echo "Found cached flashinfer wheel: $CACHED_WHEEL"
|
||||
if $PIP_CMD install "$CACHED_WHEEL" $PIP_INSTALL_SUFFIX; then
|
||||
FLASHINFER_INSTALLED=true
|
||||
echo "Successfully installed flashinfer-jit-cache from cache"
|
||||
else
|
||||
echo "Failed to install from cache, will try downloading..."
|
||||
rm -f "$CACHED_WHEEL"
|
||||
fi
|
||||
if [ "$UNINSTALL_JIT_CACHE" = false ]; then
|
||||
FLASHINFER_INSTALLED=true
|
||||
echo "flashinfer-jit-cache already at correct version, skipping download"
|
||||
fi
|
||||
|
||||
# If not installed from cache, download with retry logic
|
||||
if [ "$FLASHINFER_INSTALLED" = false ]; then
|
||||
for i in {1..5}; do
|
||||
# Download wheel to cache directory (use pip directly as uv pip doesn't support download)
|
||||
if pip download flashinfer-jit-cache==${FLASHINFER_VERSION} \
|
||||
--index-url https://flashinfer.ai/whl/${CU_VERSION} \
|
||||
-d "${FLASHINFER_CACHE_DIR}"; then
|
||||
# Cache directory for flashinfer wheels (persists across CI runs on self-hosted runners)
|
||||
FLASHINFER_CACHE_DIR="${HOME}/.cache/flashinfer-wheels"
|
||||
mkdir -p "${FLASHINFER_CACHE_DIR}"
|
||||
|
||||
CACHED_WHEEL=$(find "${FLASHINFER_CACHE_DIR}" -name "${FLASHINFER_WHEEL_PATTERN}" -type f 2>/dev/null | head -n 1)
|
||||
if [ -n "$CACHED_WHEEL" ] && [ -f "$CACHED_WHEEL" ]; then
|
||||
if $PIP_CMD install "$CACHED_WHEEL" $PIP_INSTALL_SUFFIX; then
|
||||
FLASHINFER_INSTALLED=true
|
||||
echo "Successfully downloaded and installed flashinfer-jit-cache"
|
||||
break
|
||||
fi
|
||||
else
|
||||
echo "Warning: Download succeeded but wheel file not found"
|
||||
fi
|
||||
# Clean up old versions to avoid cache bloat
|
||||
find "${FLASHINFER_CACHE_DIR}" -name "flashinfer_jit_cache-*.whl" ! -name "flashinfer_jit_cache-${FLASHINFER_VERSION}*" -type f -delete 2>/dev/null || true
|
||||
|
||||
FLASHINFER_WHEEL_PATTERN="flashinfer_jit_cache-${FLASHINFER_VERSION}*.whl"
|
||||
CACHED_WHEEL=$(find "${FLASHINFER_CACHE_DIR}" -name "${FLASHINFER_WHEEL_PATTERN}" -type f 2>/dev/null | head -n 1)
|
||||
|
||||
# Try to install from cache first
|
||||
if [ -n "$CACHED_WHEEL" ] && [ -f "$CACHED_WHEEL" ]; then
|
||||
echo "Found cached flashinfer wheel: $CACHED_WHEEL"
|
||||
if $PIP_CMD install "$CACHED_WHEEL" $PIP_INSTALL_SUFFIX; then
|
||||
FLASHINFER_INSTALLED=true
|
||||
echo "Successfully installed flashinfer-jit-cache from cache"
|
||||
else
|
||||
echo "Failed to install from cache, will try downloading..."
|
||||
rm -f "$CACHED_WHEEL"
|
||||
fi
|
||||
echo "Attempt $i to download flashinfer-jit-cache failed, retrying in 10 seconds..."
|
||||
sleep 10
|
||||
done
|
||||
fi
|
||||
|
||||
# If not installed from cache, download with retry logic
|
||||
if [ "$FLASHINFER_INSTALLED" = false ]; then
|
||||
for i in {1..5}; do
|
||||
# Download wheel to cache directory (use pip directly as uv pip doesn't support download)
|
||||
# Timeout after 10 minutes — the wheel is ~1.2 GB
|
||||
if timeout 600 pip download flashinfer-jit-cache==${FLASHINFER_VERSION} \
|
||||
--index-url https://flashinfer.ai/whl/${CU_VERSION} \
|
||||
-d "${FLASHINFER_CACHE_DIR}"; then
|
||||
|
||||
CACHED_WHEEL=$(find "${FLASHINFER_CACHE_DIR}" -name "${FLASHINFER_WHEEL_PATTERN}" -type f 2>/dev/null | head -n 1)
|
||||
if [ -n "$CACHED_WHEEL" ] && [ -f "$CACHED_WHEEL" ]; then
|
||||
if $PIP_CMD install "$CACHED_WHEEL" $PIP_INSTALL_SUFFIX; then
|
||||
FLASHINFER_INSTALLED=true
|
||||
echo "Successfully downloaded and installed flashinfer-jit-cache"
|
||||
break
|
||||
fi
|
||||
else
|
||||
echo "Warning: Download succeeded but wheel file not found"
|
||||
fi
|
||||
fi
|
||||
echo "Attempt $i to download flashinfer-jit-cache failed, retrying in 10 seconds..."
|
||||
sleep 10
|
||||
done
|
||||
fi
|
||||
fi
|
||||
|
||||
if [ "$FLASHINFER_INSTALLED" = false ]; then
|
||||
|
||||
Reference in New Issue
Block a user