[CI] Build the Rust extension modules once per run instead of in every CUDA job (#33384)

scripts/ci/cuda/ci_install_dependency.sh

@@ -326,6 +326,40 @@ uninstall_stale_flashinfer() {
     mark_step_done "${FUNCNAME[0]}"
 }
 
+require_prebuilt_rust_exts() {
+    # Stages whose download succeeded set this to none. Runs before
+    # setup_pip_toolchain uninstalls sglang, so clearing it here still reaches
+    # install_sglang below - setup.py reads it from the environment at build time.
+    if [ "${SGLANG_BUILD_RUST_EXTS:-}" != "none" ]; then
+        mark_step_done "${FUNCNAME[0]}"
+        return
+    fi
+
+    # Exact EXT_SUFFIX rather than a _core*.so glob: no crate sets abi3, so a module
+    # built for another minor version satisfies the glob while the import system
+    # ignores it, leaving is_rust_server_built() false and the Rust-server tests
+    # silently skipped. Stages have no setup-python, so the interpreter is whatever
+    # the image ships, and the pools are not on one version (h20 is 3.12 while
+    # h100 is 3.10) - a mismatch is drift to route around, not a failure.
+    local suffix
+    suffix=$(python3 -c 'import sysconfig; print(sysconfig.get_config_var("EXT_SUFFIX"))')
+    local missing=()
+    local pkg
+    for pkg in server grpc multimodal; do
+        [ -f "python/sglang/srt/${pkg}/_core${suffix}" ] || missing+=("${pkg}")
+    done
+    if [ ${#missing[@]} -gt 0 ]; then
+        echo "::warning::no prebuilt _core${suffix} for: ${missing[*]}; building from source"
+        ls -l python/sglang/srt/*/_core*.so 2>/dev/null || echo "(no extension modules at all)"
+        export SGLANG_BUILD_RUST_EXTS=
+        mark_step_done "${FUNCNAME[0]}"
+        return
+    fi
+    echo "Using prebuilt Rust extension modules; skipping the cargo build."
+
+    mark_step_done "${FUNCNAME[0]}"
+}
+
 install_sglang() {
     EXTRAS="dev,runai,tracing"
     if [ -n "$OPTIONAL_DEPS" ]; then
@@ -654,6 +688,7 @@ main() {
     install_apt_packages
     clean_site_packages
     setup_cargo_cache
+    require_prebuilt_rust_exts
     setup_pip_toolchain
     remove_stale_cuda12_nvidia_wheels
     uninstall_stale_flashinfer

scripts/ci/utils/stage_rust_ext_modules.sh

@@ -0,0 +1,43 @@
+#!/bin/bash
+# Copy the built PyO3 extension modules into rust-ext-staging/<pkg>/ for
+# upload-artifact. Shared by both jobs of _pr-test-rust-ext-build.yml, so the
+# archive layout and the module-count check cannot drift between them.
+#
+# MAX_GLIBC (optional): also reject a module requiring a newer GLIBC symbol
+# version than the test runners have. Only set where the modules were just
+# compiled - on a cache hit these are the same bytes that already passed.
+set -euo pipefail
+shopt -s nullglob
+# upload-artifact strips the longest common prefix it matched, so a missing
+# module would silently shift the archive layout.
+rm -rf rust-ext-staging
+built=()
+for pkg in server grpc multimodal; do
+    found=(python/sglang/srt/"${pkg}"/_core*.so)
+    if [ ${#found[@]} -ne 1 ]; then
+        echo "::error::expected exactly one extension module for ${pkg}, found ${#found[@]}"
+        exit 1
+    fi
+    mkdir -p "rust-ext-staging/${pkg}"
+    cp "${found[0]}" "rust-ext-staging/${pkg}/"
+    built+=("${found[0]}")
+done
+max_allowed="${MAX_GLIBC:-}"
+[ -n "${max_allowed}" ] || exit 0
+
+# Newer glibc than the test runners fails at import: "GLIBC_2.xx not found".
+status=0
+for so in "${built[@]}"; do
+    # grep exits 1 with no match; without `|| true` pipefail kills the script.
+    needed=$(objdump -T "$so" \
+        | grep -oE 'GLIBC_2\.[0-9]+' \
+        | sed 's/GLIBC_//' \
+        | sort -V | tail -1 || true)
+    echo "${so}: requires glibc <= ${needed:-none}"
+    if [ -n "${needed}" ] \
+       && [ "$(printf '%s\n%s\n' "${max_allowed}" "${needed}" | sort -V | tail -1)" != "${max_allowed}" ]; then
+        echo "::error::${so} requires glibc ${needed} > ${max_allowed} supported by the test runners; build on an older image"
+        status=1
+    fi
+done
+exit $status