[CI] Speed up dependency install: dual-ABI Rust ext cache and prevalidation pruning (#33619)

scripts/ci/cuda/ci_install_dependency.sh

@@ -223,7 +223,10 @@ setup_cargo_cache() {
 }
 
 setup_pip_toolchain() {
-    python3 -m pip install --upgrade pip
+    if [ "$USE_VENV" = "1" ]; then
+        # The bootstrap upgrade hit system pip; this upgrades the venv's own.
+        python3 -m pip install --upgrade pip
+    fi
 
     if [ "$USE_VENV" != "1" ]; then
         export UV_SYSTEM_PYTHON=1
@@ -468,7 +471,6 @@ install_sglang_kernel() {
 
 install_sglang_router() {
     $PIP_CMD install sglang-router $PIP_INSTALL_SUFFIX
-    $PIP_CMD list
 
     mark_step_done "${FUNCNAME[0]}"
 }
@@ -632,9 +634,9 @@ prepare_runner() {
 
 setup_ld_library_path() {
     # NVIDIA pip packages and torch ship .so files under site-packages that are
-    # not on the default LD_LIBRARY_PATH.
+    # not on the default LD_LIBRARY_PATH; lib/ always nests under nvidia/.
     SITE_PACKAGES=$(python3 -c "import site, sys; print(site.getsitepackages()[0])")
-    NVIDIA_LIBS=$(find "$SITE_PACKAGES" -path "*/nvidia/*/lib" -type d 2>/dev/null | tr '\n' ':')
+    NVIDIA_LIBS=$( (find "$SITE_PACKAGES/nvidia" -type d -name lib 2>/dev/null || true) | tr '\n' ':')
     TORCH_LIB="$SITE_PACKAGES/torch/lib"
     VENV_LD="${NVIDIA_LIBS}${TORCH_LIB}"
     export LD_LIBRARY_PATH="${VENV_LD}${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}"
@@ -652,13 +654,18 @@ setup_ld_library_path() {
 
 verify_imports() {
     $PIP_CMD list
-    python3 -c "import torch; print(torch.version.cuda)"
-    python3 -c "import cutlass; import cutlass.cute;"
 
-    # A shadowed sglang still imports, so without this the failure only surfaces
-    # as a missing submodule during the test step. find_spec, not import: the
-    # finders alone answer this and importing would pull in torch for nothing.
+    # One process; torch/cutlass do not import sglang, so the find_spec check
+    # still runs ahead of any sglang import.
     SGLANG_EXPECTED_INIT="${REPO_ROOT}/python/sglang/__init__.py" python3 -c '
+import torch
+print(torch.version.cuda)
+import cutlass
+import cutlass.cute
+
+# A shadowed sglang still imports, so without this the failure only surfaces
+# as a missing submodule during the test step. find_spec, not import: the
+# finders alone answer this without importing sglang.
 import importlib.util, os
 want = os.environ["SGLANG_EXPECTED_INIT"]
 spec = importlib.util.find_spec("sglang")
@@ -671,11 +678,9 @@ if spec.origin != want:
         "something in site-packages is shadowing the checkout"
     )
 print(f"sglang resolves to {spec.origin}")
-'
 
-    # Import, not find_spec: the finders locate an extension without dlopening it,
-    # so a .so that cannot load passes find_spec and only fails inside some suite.
-    python3 -c '
+# Import, not find_spec: the finders locate an extension without dlopening it,
+# so a .so that cannot load passes find_spec and only fails inside some suite.
 import importlib
 for mod in ("server", "grpc", "multimodal"):
     name = f"sglang.srt.{mod}._core"

scripts/ci/cuda/prepare_runner.sh

@@ -1,5 +1,5 @@
 #!/bin/bash
-# Prepare the CI runner by cleaning up stale HuggingFace cache artifacts and validating models
+# Prepare the CI runner by cleaning up stale HuggingFace cache artifacts
 set -euo pipefail
 
 SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
@@ -7,13 +7,9 @@ SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
 echo "Preparing CI runner..."
 echo ""
 
-# Clean up stale HuggingFace cache artifacts from previous failed downloads
+# Clean up stale HuggingFace cache artifacts from previous failed downloads.
+# No prevalidation: launch/load-time validation covers and repairs each cache.
 python3 "${SCRIPT_DIR}/../utils/cleanup_hf_cache.py"
 echo ""
 
-# Pre-validate cached models and write markers for offline mode
-# This allows tests to run with HF_HUB_OFFLINE=1 for models that are fully cached
-python3 "${SCRIPT_DIR}/../utils/prevalidate_cached_models.py"
-echo ""
-
 echo "CI runner preparation complete!"