Resolve HF download issue and download models before CI run starts for 8-gpu-h200 runners (#12952)

scripts/ci/prepare_runner.sh

@@ -1,47 +1,29 @@
 #!/bin/bash
-# Prepare the CI runner by cleaning up incomplete HuggingFace download files
+# Prepare the CI runner by cleaning up stale HuggingFace cache artifacts and validating models
 set -euo pipefail
 
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+
 echo "Preparing CI runner..."
+echo ""
 
-# Clean up incomplete HuggingFace download files
-echo "Cleaning up incomplete HuggingFace download files..."
-python3 << 'EOF'
-import os
-import glob
+# Clean up stale HuggingFace cache artifacts from previous failed downloads
+python3 "${SCRIPT_DIR}/cleanup_hf_cache.py"
+echo ""
 
-try:
-    from huggingface_hub import constants
-    hf_cache_dir = constants.HF_HUB_CACHE
-except Exception as e:
-    print(f"Warning: Could not import huggingface_hub constants: {e}")
-    # Fallback to checking HF_HOME env var or default location
-    hf_home = os.environ.get('HF_HOME', os.path.expanduser("~/.cache/huggingface"))
-    hf_cache_dir = os.path.join(hf_home, "hub")
+# Validate model integrity for configured runners
+echo "Validating model integrity..."
 
-if os.path.exists(hf_cache_dir):
-    print(f"Checking HuggingFace cache directory: {hf_cache_dir}")
+# Enable accelerated HuggingFace downloads (10x faster on high-bandwidth networks)
+export HF_HUB_ENABLE_HF_TRANSFER=1
 
-    # Clean up incomplete marker files, temporary files, and lock files
-    patterns = ['**/*.incomplete', '**/*.tmp', '**/*.lock']
-    cleaned_count = 0
+python3 "${SCRIPT_DIR}/validate_and_download_models.py"
+VALIDATION_EXIT_CODE=$?
 
-    for pattern in patterns:
-        files = glob.glob(os.path.join(hf_cache_dir, pattern), recursive=True)
-        for file_path in files:
-            try:
-                os.remove(file_path)
-                print(f"Removed: {file_path}")
-                cleaned_count += 1
-            except Exception as e:
-                print(f"Warning: Could not remove {file_path}: {e}")
-
-    if cleaned_count > 0:
-        print(f"Cleaned up {cleaned_count} incomplete HuggingFace download file(s)")
-    else:
-        print("No incomplete HuggingFace download files found")
-else:
-    print(f"HuggingFace cache directory does not exist: {hf_cache_dir}")
-EOF
+if [ $VALIDATION_EXIT_CODE -ne 0 ]; then
+    echo "Model validation failed with exit code: $VALIDATION_EXIT_CODE"
+    exit $VALIDATION_EXIT_CODE
+fi
 
+echo ""
 echo "CI runner preparation complete!"