[Refactore] [CI] Remove redundant CI test runs step 2 (#17584)

2026-01-24 23:39:48 -08:00
parent 9bd92ba0f6
commit d1042e0d62
36 changed files with 199 additions and 196 deletions
@@ -0,0 +1,89 @@
+#!/bin/bash
+# Install the dependency in CI.
+set -euxo pipefail
+
+bash scripts/ci/cuda/ci_install_dependency.sh
+
+export GDRCOPY_HOME=/usr/src/gdrdrv-2.5.1/
+export CUDA_HOME=/usr/local/cuda
+
+GRACE_BLACKWELL=${GRACE_BLACKWELL:-0}
+# Detect architecture
+ARCH=$(uname -m)
+if [ "$ARCH" != "x86_64" ] && [ "$ARCH" != "aarch64" ]; then
+    echo "Unsupported architecture: $ARCH"
+    exit 1
+fi
+
+if python3 -c "import deep_ep" >/dev/null 2>&1; then
+    echo "deep_ep is already installed or importable. Skipping installation."
+    exit 0
+fi
+
+# Install system dependencies
+apt install -y curl wget git sudo rdma-core infiniband-diags openssh-server perftest libibumad3 libibverbs-dev libibverbs1 ibverbs-providers ibverbs-utils libnl-3-200 libnl-route-3-200 librdmacm1 build-essential cmake
+
+# Install GDRCopy
+rm -rf /opt/gdrcopy && mkdir -p /opt/gdrcopy
+cd /opt/gdrcopy
+git clone https://github.com/NVIDIA/gdrcopy.git .
+git checkout v2.5.1
+apt update
+apt install -y nvidia-dkms-580
+apt install -y build-essential devscripts debhelper fakeroot pkg-config dkms
+apt install -y check libsubunit0 libsubunit-dev python3-venv
+cd packages
+CUDA=/usr/local/cuda ./build-deb-packages.sh
+dpkg -i gdrdrv-dkms_*.deb
+dpkg -i libgdrapi_*.deb
+dpkg -i gdrcopy-tests_*.deb
+dpkg -i gdrcopy_*.deb
+
+# Set up library paths based on architecture
+LIB_PATH="/usr/lib/$ARCH-linux-gnu"
+if [ ! -e "$LIB_PATH/libmlx5.so" ]; then
+    ln -s $LIB_PATH/libmlx5.so.1 $LIB_PATH/libmlx5.so
+fi
+apt-get update && apt-get install -y libfabric-dev
+
+# Install DeepEP
+DEEPEP_DIR=/root/.cache/deepep
+rm -rf ${DEEPEP_DIR}
+if [ "$GRACE_BLACKWELL" = "1" ]; then
+    # We use Tom's DeepEP fork for GB200 for now, which supports fp4 dispatch.
+    GRACE_BLACKWELL_DEEPEP_BRANCH=gb200_blog_part_2
+    git clone https://github.com/fzyzcjy/DeepEP.git ${DEEPEP_DIR} && \
+    pushd ${DEEPEP_DIR} && \
+    git checkout ${GRACE_BLACKWELL_DEEPEP_BRANCH} && \
+    sed -i 's/#define NUM_CPU_TIMEOUT_SECS 100/#define NUM_CPU_TIMEOUT_SECS 1000/' csrc/kernels/configs.cuh && \
+    popd
+else
+    git clone https://github.com/deepseek-ai/DeepEP.git ${DEEPEP_DIR} && \
+    pushd ${DEEPEP_DIR} && \
+    git checkout 9af0e0d0e74f3577af1979c9b9e1ac2cad0104ee && \
+    popd
+fi
+
+cd ${DEEPEP_DIR}
+if [ "$GRACE_BLACKWELL" = "1" ]; then
+    CUDA_VERSION=$(nvidia-smi | grep "CUDA Version" | head -n1 | awk '{print $9}')
+    if [ "$CUDA_VERSION" = "12.8" ]; then
+        CHOSEN_TORCH_CUDA_ARCH_LIST='10.0'
+    elif awk -v ver="$CUDA_VERSION" 'BEGIN {exit !(ver > 12.8)}'; then
+        # With cuda > 12.8, the compiler supports 10.3, so we should use
+        # CHOSEN_TORCH_CUDA_ARCH_LIST='10.0;10.3'
+        #
+        # However, our CI machine has a weird setup and nvidia-smi reports wrong CUDA version in the container.
+        # The container is actually cuda 12.8, but nvidia-smi reports 13.0, leading to compilation errors. so we
+        # drop 10.3.
+        CHOSEN_TORCH_CUDA_ARCH_LIST='10.0'
+    else
+        echo "Unsupported CUDA version for Grace Blackwell: $CUDA_VERSION" && exit 1
+    fi && \
+    if [ "${CUDA_VERSION%%.*}" = "13" ]; then \
+        sed -i "/^    include_dirs = \['csrc\/'\]/a\    include_dirs.append('${CUDA_HOME}/include/cccl')" setup.py; \
+    fi
+    TORCH_CUDA_ARCH_LIST="${CHOSEN_TORCH_CUDA_ARCH_LIST}" pip install --no-build-isolation .
+else
+    python3 setup.py install
+fi
@@ -0,0 +1,256 @@
+#!/bin/bash
+# Install the dependency in CI.
+set -euxo pipefail
+
+# Set up environment variables
+IS_BLACKWELL=${IS_BLACKWELL:-0}
+CU_VERSION="cu129"
+FLASHINFER_VERSION=0.6.1
+OPTIONAL_DEPS="${1:-}"
+
+# Detect system architecture
+ARCH=$(uname -m)
+echo "Detected architecture: ${ARCH}"
+
+if [ "$CU_VERSION" = "cu130" ]; then
+    NVRTC_SPEC="nvidia-cuda-nvrtc"
+else
+    NVRTC_SPEC="nvidia-cuda-nvrtc-cu12"
+fi
+
+# Kill existing processes
+SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+bash "${SCRIPT_DIR}/../../killall_sglang.sh"
+echo "CUDA_VISIBLE_DEVICES=${CUDA_VISIBLE_DEVICES:-}"
+
+# Clear torch compilation cache
+python3 -c 'import os, shutil, tempfile, getpass; cache_dir = os.environ.get("TORCHINDUCTOR_CACHE_DIR") or os.path.join(tempfile.gettempdir(), "torchinductor_" + getpass.getuser()); shutil.rmtree(cache_dir, ignore_errors=True)'
+
+# Install apt packages
+apt install -y git libnuma-dev libssl-dev pkg-config libibverbs-dev libibverbs1 ibverbs-providers ibverbs-utils
+
+# Check if protoc of correct architecture is already installed
+if command -v protoc >/dev/null 2>&1; then
+    if protoc --version >/dev/null 2>&1; then
+        echo "protoc already installed: $(protoc --version)"
+    else
+        echo "protoc found but not runnable, reinstalling..."
+        INSTALL_PROTOC=1
+    fi
+else
+    INSTALL_PROTOC=1
+fi
+
+# Install protoc for router build (gRPC protobuf compilation)
+if [ "${INSTALL_PROTOC:-0}" = "1" ]; then
+    # TODO: move this to a separate script
+    echo "Installing protoc..."
+    if command -v apt-get &> /dev/null; then
+        # Ubuntu/Debian
+        apt-get update
+        apt-get install -y wget unzip gcc g++ perl make
+    elif command -v yum &> /dev/null; then
+        # RHEL/CentOS
+        yum update -y
+        yum install -y wget unzip gcc gcc-c++ perl-core make
+    fi
+
+    cd /tmp
+    # Determine protoc architecture
+    if [ "$ARCH" = "aarch64" ] || [ "$ARCH" = "arm64" ]; then
+        PROTOC_ARCH="aarch_64"
+    else
+        PROTOC_ARCH="x86_64"
+    fi
+    PROTOC_ZIP="protoc-32.0-linux-${PROTOC_ARCH}.zip"
+    wget https://github.com/protocolbuffers/protobuf/releases/download/v32.0/${PROTOC_ZIP}
+    unzip -o ${PROTOC_ZIP} -d /usr/local
+    rm ${PROTOC_ZIP}
+    protoc --version
+    cd -
+else
+    echo "protoc already installed: $(protoc --version)"
+fi
+
+# Install uv
+pip install --upgrade pip
+
+if [ "$IS_BLACKWELL" = "1" ]; then
+    # The blackwell CI runner has some issues with pip and uv,
+    # so we can only use pip with `--break-system-packages`
+    PIP_CMD="pip"
+    PIP_INSTALL_SUFFIX="--break-system-packages"
+    PIP_UNINSTALL_CMD="pip uninstall -y"
+    PIP_UNINSTALL_SUFFIX="--break-system-packages"
+else
+    # In normal cases, we use uv, which is much faster than pip.
+    pip install uv
+    export UV_SYSTEM_PYTHON=true
+
+    PIP_CMD="uv pip"
+    PIP_INSTALL_SUFFIX="--index-strategy unsafe-best-match --prerelease allow"
+    PIP_UNINSTALL_CMD="uv pip uninstall"
+    PIP_UNINSTALL_SUFFIX=""
+fi
+
+# Clean up existing installations
+$PIP_UNINSTALL_CMD sgl-kernel sglang $PIP_UNINSTALL_SUFFIX || true
+$PIP_UNINSTALL_CMD flashinfer-python flashinfer-cubin flashinfer-jit-cache $PIP_UNINSTALL_SUFFIX || true
+$PIP_UNINSTALL_CMD opencv-python opencv-python-headless $PIP_UNINSTALL_SUFFIX || true
+
+# Install the main package
+EXTRAS="dev"
+if [ -n "$OPTIONAL_DEPS" ]; then
+    EXTRAS="dev,${OPTIONAL_DEPS}"
+fi
+echo "Installing python extras: [${EXTRAS}]"
+
+$PIP_CMD install -e "python[${EXTRAS}]" --extra-index-url https://download.pytorch.org/whl/${CU_VERSION} $PIP_INSTALL_SUFFIX
+
+# Install router for pd-disagg test
+$PIP_CMD install sglang-router $PIP_INSTALL_SUFFIX
+
+# Remove flash_attn folder to avoid conflicts
+PYTHON_LIB_PATH=$(python3 -c "import site; print(site.getsitepackages()[0])")
+FLASH_ATTN_PATH="${PYTHON_LIB_PATH}/flash_attn"
+
+if [ -d "$FLASH_ATTN_PATH" ]; then
+    echo "Directory $FLASH_ATTN_PATH exists. Removing..."
+    rm -rf "$FLASH_ATTN_PATH"
+else
+    echo "Directory $FLASH_ATTN_PATH does not exist."
+fi
+
+# Install sgl-kernel
+SGL_KERNEL_VERSION_FROM_KERNEL=$(grep -Po '(?<=^version = ")[^"]*' sgl-kernel/pyproject.toml)
+SGL_KERNEL_VERSION_FROM_SRT=$(grep -Po -m1 '(?<=sgl-kernel==)[0-9A-Za-z\.\-]+' python/pyproject.toml)
+echo "SGL_KERNEL_VERSION_FROM_KERNEL=${SGL_KERNEL_VERSION_FROM_KERNEL} SGL_KERNEL_VERSION_FROM_SRT=${SGL_KERNEL_VERSION_FROM_SRT}"
+
+if [ "${CUSTOM_BUILD_SGL_KERNEL:-}" = "true" ] && [ -d "sgl-kernel/dist" ]; then
+    ls -alh sgl-kernel/dist
+    # Determine wheel architecture
+    if [ "$ARCH" = "aarch64" ] || [ "$ARCH" = "arm64" ]; then
+        WHEEL_ARCH="aarch64"
+    else
+        WHEEL_ARCH="x86_64"
+    fi
+    $PIP_CMD install sgl-kernel/dist/sgl_kernel-${SGL_KERNEL_VERSION_FROM_KERNEL}-cp310-abi3-manylinux2014_${WHEEL_ARCH}.whl --force-reinstall $PIP_INSTALL_SUFFIX
+elif [ "${CUSTOM_BUILD_SGL_KERNEL:-}" = "true" ] && [ ! -d "sgl-kernel/dist" ]; then
+    # CUSTOM_BUILD_SGL_KERNEL was set but artifacts not available (e.g., stage rerun without wheel build)
+    # Fall back to installing from PyPI
+    echo "WARNING: CUSTOM_BUILD_SGL_KERNEL=true but sgl-kernel/dist not found, falling back to PyPI"
+    $PIP_CMD install sgl-kernel==${SGL_KERNEL_VERSION_FROM_SRT} --force-reinstall $PIP_INSTALL_SUFFIX
+else
+    # On Blackwell machines, skip reinstall if correct version already installed to avoid race conditions
+    if [ "$IS_BLACKWELL" = "1" ]; then
+        INSTALLED_SGL_KERNEL=$(pip show sgl-kernel 2>/dev/null | grep "^Version:" | awk '{print $2}' || echo "")
+        if [ "$INSTALLED_SGL_KERNEL" = "$SGL_KERNEL_VERSION_FROM_SRT" ]; then
+            echo "sgl-kernel==${SGL_KERNEL_VERSION_FROM_SRT} already installed, skipping reinstall"
+        else
+            echo "Installing sgl-kernel==${SGL_KERNEL_VERSION_FROM_SRT} (current: ${INSTALLED_SGL_KERNEL:-none})"
+            $PIP_CMD install sgl-kernel==${SGL_KERNEL_VERSION_FROM_SRT} $PIP_INSTALL_SUFFIX
+        fi
+    else
+        $PIP_CMD install sgl-kernel==${SGL_KERNEL_VERSION_FROM_SRT} --force-reinstall $PIP_INSTALL_SUFFIX
+    fi
+fi
+
+# Show current packages
+$PIP_CMD list
+
+# Install other python dependencies
+$PIP_CMD install mooncake-transfer-engine==0.3.8.post1 "${NVRTC_SPEC}" py-spy scipy huggingface_hub[hf_xet] pytest $PIP_INSTALL_SUFFIX
+
+if [ "$IS_BLACKWELL" != "1" ]; then
+    # For lmms_evals evaluating MMMU
+    git clone --branch v0.5 --depth 1 https://github.com/EvolvingLMMs-Lab/lmms-eval.git
+    $PIP_CMD install -e lmms-eval/ $PIP_INSTALL_SUFFIX
+fi
+
+# DeepEP depends on nvshmem 3.4.5
+# On Blackwell machines, skip reinstall if correct version already installed to avoid race conditions
+if [ "$IS_BLACKWELL" = "1" ]; then
+    INSTALLED_NVSHMEM=$(pip show nvidia-nvshmem-cu12 2>/dev/null | grep "^Version:" | awk '{print $2}' || echo "")
+    if [ "$INSTALLED_NVSHMEM" = "3.4.5" ]; then
+        echo "nvidia-nvshmem-cu12==3.4.5 already installed, skipping reinstall"
+    else
+        $PIP_CMD install nvidia-nvshmem-cu12==3.4.5 $PIP_INSTALL_SUFFIX
+    fi
+else
+    $PIP_CMD install nvidia-nvshmem-cu12==3.4.5 --force-reinstall $PIP_INSTALL_SUFFIX
+fi
+
+# Cudnn with version less than 9.16.0.29 will cause performance regression on Conv3D kernel
+# On Blackwell machines, skip reinstall if correct version already installed to avoid race conditions
+if [ "$IS_BLACKWELL" = "1" ]; then
+    INSTALLED_CUDNN=$(pip show nvidia-cudnn-cu12 2>/dev/null | grep "^Version:" | awk '{print $2}' || echo "")
+    if [ "$INSTALLED_CUDNN" = "9.16.0.29" ]; then
+        echo "nvidia-cudnn-cu12==9.16.0.29 already installed, skipping reinstall"
+    else
+        $PIP_CMD install nvidia-cudnn-cu12==9.16.0.29 $PIP_INSTALL_SUFFIX
+    fi
+else
+    $PIP_CMD install nvidia-cudnn-cu12==9.16.0.29 --force-reinstall $PIP_INSTALL_SUFFIX
+fi
+$PIP_CMD uninstall xformers || true
+
+# Install flashinfer-jit-cache with caching and retry logic (flashinfer.ai can have transient DNS issues)
+# Cache directory for flashinfer wheels (persists across CI runs on self-hosted runners)
+FLASHINFER_CACHE_DIR="${HOME}/.cache/flashinfer-wheels"
+mkdir -p "${FLASHINFER_CACHE_DIR}"
+
+# Clean up old versions to avoid cache bloat
+find "${FLASHINFER_CACHE_DIR}" -name "flashinfer_jit_cache-*.whl" ! -name "flashinfer_jit_cache-${FLASHINFER_VERSION}*" -type f -delete 2>/dev/null || true
+
+FLASHINFER_WHEEL_PATTERN="flashinfer_jit_cache-${FLASHINFER_VERSION}*.whl"
+CACHED_WHEEL=$(find "${FLASHINFER_CACHE_DIR}" -name "${FLASHINFER_WHEEL_PATTERN}" -type f 2>/dev/null | head -n 1)
+
+FLASHINFER_INSTALLED=false
+
+# Try to install from cache first
+if [ -n "$CACHED_WHEEL" ] && [ -f "$CACHED_WHEEL" ]; then
+    echo "Found cached flashinfer wheel: $CACHED_WHEEL"
+    if $PIP_CMD install "$CACHED_WHEEL" $PIP_INSTALL_SUFFIX; then
+        FLASHINFER_INSTALLED=true
+        echo "Successfully installed flashinfer-jit-cache from cache"
+    else
+        echo "Failed to install from cache, will try downloading..."
+        rm -f "$CACHED_WHEEL"
+    fi
+fi
+
+# If not installed from cache, download with retry logic
+if [ "$FLASHINFER_INSTALLED" = false ]; then
+    for i in {1..5}; do
+        # Download wheel to cache directory (use pip directly as uv pip doesn't support download)
+        if pip download flashinfer-jit-cache==${FLASHINFER_VERSION} \
+            --index-url https://flashinfer.ai/whl/${CU_VERSION} \
+            -d "${FLASHINFER_CACHE_DIR}"; then
+
+            CACHED_WHEEL=$(find "${FLASHINFER_CACHE_DIR}" -name "${FLASHINFER_WHEEL_PATTERN}" -type f 2>/dev/null | head -n 1)
+            if [ -n "$CACHED_WHEEL" ] && [ -f "$CACHED_WHEEL" ]; then
+                if $PIP_CMD install "$CACHED_WHEEL" $PIP_INSTALL_SUFFIX; then
+                    FLASHINFER_INSTALLED=true
+                    echo "Successfully downloaded and installed flashinfer-jit-cache"
+                    break
+                fi
+            else
+                echo "Warning: Download succeeded but wheel file not found"
+            fi
+        fi
+        echo "Attempt $i to download flashinfer-jit-cache failed, retrying in 10 seconds..."
+        sleep 10
+    done
+fi
+
+if [ "$FLASHINFER_INSTALLED" = false ]; then
+    echo "ERROR: Failed to install flashinfer-jit-cache after 5 attempts"
+    exit 1
+fi
+
+# Show current packages
+$PIP_CMD list
+python3 -c "import torch; print(torch.version.cuda)"
+
+# Prepare the CI runner (cleanup HuggingFace cache, etc.)
+bash "${SCRIPT_DIR}/prepare_runner.sh"
@@ -0,0 +1,24 @@
+#!/bin/bash
+set -euxo pipefail
+
+# Check if sudo is available
+if command -v sudo >/dev/null 2>&1; then
+    sudo apt-get update
+    sudo apt-get install -y libssl-dev pkg-config protobuf-compiler redis-server
+else
+    apt-get update
+    apt-get install -y libssl-dev pkg-config protobuf-compiler redis-server
+fi
+
+# Install rustup (Rust installer and version manager)
+curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain 1.90
+
+
+# Follow the installation prompts, then reload your shell
+. "$HOME/.cargo/env"
+source $HOME/.cargo/env
+
+# Verify installation
+rustc --version
+cargo --version
+protoc --version
@@ -0,0 +1,106 @@
+#!/bin/bash
+set -euo pipefail
+
+# Optional: set DISAGG_READY_FILE to a filepath; when all servers are healthy, the script will
+# create this file as a readiness signal (useful for CI to proceed to next steps).
+DISAGG_READY_FILE="${DISAGG_READY_FILE:-}"
+
+MODEL_PATH="/raid/models/meta-llama/Llama-3.1-8B-Instruct"
+
+# Function to find the first available active IB device
+find_active_ib_device() {
+    for device in mlx5_{0..11}; do
+        if ibv_devinfo $device >/dev/null 2>&1; then
+            state=$(ibv_devinfo $device | grep "state:" | head -1 | awk '{print $2}')
+            if [[ "$state" == "PORT_ACTIVE" ]]; then
+                echo "$device"
+                return 0
+            fi
+        fi
+    done
+    echo "No active IB device found" >&2
+    return 1
+}
+
+# Get the first available active IB device
+DEVICE=$(find_active_ib_device)
+echo "Using IB device: $DEVICE"
+
+# Launch prefill servers on GPU 0–3
+for i in {0..3}; do
+  PORT=$((30001 + i))
+  BOOTSTRAP_PORT=$((9001 + i))
+  HOST="127.0.0.$((i + 1))"
+  echo "Launching PREFILL server on GPU $i at $HOST:$PORT (bootstrap: $BOOTSTRAP_PORT)"
+  CUDA_VISIBLE_DEVICES=$i \
+  python3 -m sglang.launch_server \
+    --model-path "$MODEL_PATH" \
+    --disaggregation-mode prefill \
+    --host "$HOST" \
+    --port "$PORT" \
+    --disaggregation-ib-device "$DEVICE" \
+    --disaggregation-bootstrap-port "$BOOTSTRAP_PORT" &
+done
+
+# Launch decode servers on GPU 4–7
+for i in {4..7}; do
+  PORT=$((30001 + i))
+  HOST="127.0.0.$((i + 1))"
+  echo "Launching DECODE server on GPU $i at $HOST:$PORT"
+  CUDA_VISIBLE_DEVICES=$i \
+  python3 -m sglang.launch_server \
+    --model-path "$MODEL_PATH" \
+    --disaggregation-mode decode \
+    --host "$HOST" \
+    --port "$PORT" \
+    --disaggregation-ib-device "$DEVICE" \
+    --base-gpu-id 0 &
+done
+
+# Wait for disaggregation servers to initialize
+echo "Waiting for disaggregation servers to initialize..."
+
+# Health check with 5-minute timeout
+TIMEOUT=300
+START_TIME=$(date +%s)
+
+echo "Checking health of all 8 servers..."
+while true; do
+    CURRENT_TIME=$(date +%s)
+    ELAPSED=$((CURRENT_TIME - START_TIME))
+
+    if [ $ELAPSED -ge $TIMEOUT ]; then
+        echo "❌ Timeout: Servers did not become healthy within 5 minutes"
+        exit 1
+    fi
+
+    HEALTHY_COUNT=0
+    # Check all 8 servers (127.0.0.1-8:30001-30008)
+    for i in {1..8}; do
+        if curl -s -f "http://127.0.0.$i:$((30000 + i))/health" >/dev/null 2>&1; then
+            HEALTHY_COUNT=$((HEALTHY_COUNT + 1))
+        fi
+    done
+
+    echo "Healthy servers: $HEALTHY_COUNT/8 (elapsed: ${ELAPSED}s)"
+
+    if [ $HEALTHY_COUNT -eq 8 ]; then
+        echo "✅ All 8 servers are healthy!"
+        # Emit readiness signal file if requested
+        if [ -n "$DISAGG_READY_FILE" ]; then
+            echo "Creating readiness flag: $DISAGG_READY_FILE"
+            # Ensure parent dir exists; ignore errors
+            mkdir -p "$(dirname "$DISAGG_READY_FILE")" 2>/dev/null || true
+            touch "$DISAGG_READY_FILE"
+        fi
+        break
+    else
+        sleep 10  # Wait 10 seconds before next check
+    fi
+done
+
+# Don't launch router here - just keep servers running
+echo "✅ All disaggregation servers are ready and waiting for router connections"
+
+# Keep the script running
+wait
@@ -0,0 +1,19 @@
+#!/bin/bash
+# Prepare the CI runner by cleaning up stale HuggingFace cache artifacts and validating models
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+
+echo "Preparing CI runner..."
+echo ""
+
+# Clean up stale HuggingFace cache artifacts from previous failed downloads
+python3 "${SCRIPT_DIR}/../utils/cleanup_hf_cache.py"
+echo ""
+
+# Pre-validate cached models and write markers for offline mode
+# This allows tests to run with HF_HUB_OFFLINE=1 for models that are fully cached
+python3 "${SCRIPT_DIR}/../utils/prevalidate_cached_models.py"
+echo ""
+
+echo "CI runner preparation complete!"