[CI][XPU] Stabilize XPU CI: pin UMD/IGC, retry infra flakes, right-size EAGLE3 (#32438)

Co-authored-by: Ma Mingfei <mingfei.ma@intel.com>
2026-08-04 16:28:52 +08:00
parent 17d19081d9
commit 53804d609c
6 changed files with 137 additions and 7 deletions
@@ -113,7 +113,7 @@ jobs:
      - name: Run stage-a tests
        timeout-minutes: 30
        run: |
-          docker exec ci_sglang_xpu bash -c "source /opt/venv/bin/activate && cd /sglang-checkout/test && python3 run_suite.py --hw xpu --suite stage-a-test-1-gpu-xpu"
+          docker exec ci_sglang_xpu bash -c "source /opt/venv/bin/activate && cd /sglang-checkout/test && python3 run_suite.py --hw xpu --suite stage-a-test-1-gpu-xpu --enable-retry"

      - name: Cleanup container
        if: always()
@@ -136,6 +136,19 @@ jobs:
            .pytest_cache || true
          find . -type d -name "__pycache__" -prune -exec rm -rf {} + || true
          find . -type f -name "*.pyc" -delete || true
+          # SIGTERM sglang and drain GPU context before `docker rm -f`;
+          # SIGKILL leaves the xe/GuC exec queue registered and triggers a
+          # GT reset (+ devcoredump) on B580.
+          if docker ps --format '{{.Names}}' | grep -qx ci_sglang_xpu; then
+            docker exec ci_sglang_xpu bash -c '
+              pkill -TERM -f "sglang|run_suite|python3.*test_" 2>/dev/null || true
+              for _ in $(seq 1 30); do
+                pgrep -f "sglang::|sglang.launch_server" >/dev/null || break
+                sleep 1
+              done
+              pkill -KILL -f "sglang|run_suite" 2>/dev/null || true
+            ' || true
+          fi
          docker rm -f ci_sglang_xpu || true
          if [[ -n "${CI_SGLANG_XPU_IMAGE:-}" ]]; then
            docker rmi -f "${CI_SGLANG_XPU_IMAGE}" || true
@@ -189,7 +202,7 @@ jobs:
      - name: Run stage-b tests
        timeout-minutes: 60
        run: |
-          docker exec ci_sglang_xpu bash -c "source /opt/venv/bin/activate && cd /sglang-checkout/test && python3 run_suite.py --hw xpu --suite stage-b-test-1-gpu-xpu"
+          docker exec ci_sglang_xpu bash -c "source /opt/venv/bin/activate && cd /sglang-checkout/test && python3 run_suite.py --hw xpu --suite stage-b-test-1-gpu-xpu --enable-retry"

      - name: Cleanup container
        if: always()
@@ -212,6 +225,19 @@ jobs:
            .pytest_cache || true
          find . -type d -name "__pycache__" -prune -exec rm -rf {} + || true
          find . -type f -name "*.pyc" -delete || true
+          # SIGTERM sglang and drain GPU context before `docker rm -f`;
+          # SIGKILL leaves the xe/GuC exec queue registered and triggers a
+          # GT reset (+ devcoredump) on B580.
+          if docker ps --format '{{.Names}}' | grep -qx ci_sglang_xpu; then
+            docker exec ci_sglang_xpu bash -c '
+              pkill -TERM -f "sglang|run_suite|python3.*test_" 2>/dev/null || true
+              for _ in $(seq 1 30); do
+                pgrep -f "sglang::|sglang.launch_server" >/dev/null || break
+                sleep 1
+              done
+              pkill -KILL -f "sglang|run_suite" 2>/dev/null || true
+            ' || true
+          fi
          docker rm -f ci_sglang_xpu || true
          if [[ -n "${CI_SGLANG_XPU_IMAGE:-}" ]]; then
            docker rmi -f "${CI_SGLANG_XPU_IMAGE}" || true
@@ -17,14 +17,40 @@ ARG SG_LANG_KERNEL_BRANCH=main

 USER root

-# Install the latest UMD driver for SYCL-TLA
-RUN apt-get update && apt-get install -y software-properties-common && \
+# Pin Level-Zero UMD + IGC (rolling PPA once faulted libze on B580; see sgl-kernel-xpu#296).
+# Keep in lockstep with the host xe KMD; override via --build-arg.
+ARG COMPUTE_RUNTIME_VERSION=26.05.37020.3
+ARG IGC_VERSION=2.28.4+20760
+ARG GMM_VERSION=22.9.0
+RUN apt-get update && apt-get install -y software-properties-common curl && \
    add-apt-repository -y ppa:kobuk-team/intel-graphics && \
    apt-get update && \
+    # Loader + media/metrics from the PPA; the GPU driver is pinned below.
    apt-get install -y \
-        libze-intel-gpu1 libze1 intel-metrics-discovery intel-opencl-icd clinfo intel-gsc \
+        libze1 intel-metrics-discovery clinfo intel-gsc \
        intel-media-va-driver-non-free libmfx-gen1 libvpl2 libvpl-tools libva-glx2 va-driver-all vainfo \
-        libze-dev intel-ocloc && \
+        libze-dev && \
+    cd /tmp && \
+    igc_url="https://github.com/intel/intel-graphics-compiler/releases/download/v${IGC_VERSION%%+*}" && \
+    cr_url="https://github.com/intel/compute-runtime/releases/download/${COMPUTE_RUNTIME_VERSION}" && \
+    # IGC first: libze-intel-gpu1 / intel-opencl-icd depend on its exact version.
+    curl -fsSL -O "${igc_url}/intel-igc-core-2_${IGC_VERSION}_amd64.deb" && \
+    curl -fsSL -O "${igc_url}/intel-igc-opencl-2_${IGC_VERSION}_amd64.deb" && \
+    curl -fsSL -O "${cr_url}/libze-intel-gpu1_${COMPUTE_RUNTIME_VERSION}-0_amd64.deb" && \
+    curl -fsSL -O "${cr_url}/intel-opencl-icd_${COMPUTE_RUNTIME_VERSION}-0_amd64.deb" && \
+    curl -fsSL -O "${cr_url}/intel-ocloc_${COMPUTE_RUNTIME_VERSION}-0_amd64.deb" && \
+    curl -fsSL -O "${cr_url}/libigdgmm12_${GMM_VERSION}_amd64.deb" && \
+    apt-get install -y --allow-downgrades \
+        ./intel-igc-core-2_${IGC_VERSION}_amd64.deb \
+        ./intel-igc-opencl-2_${IGC_VERSION}_amd64.deb \
+        ./libigdgmm12_${GMM_VERSION}_amd64.deb \
+        ./libze-intel-gpu1_${COMPUTE_RUNTIME_VERSION}-0_amd64.deb \
+        ./intel-opencl-icd_${COMPUTE_RUNTIME_VERSION}-0_amd64.deb \
+        ./intel-ocloc_${COMPUTE_RUNTIME_VERSION}-0_amd64.deb && \
+    rm -f /tmp/*.deb && \
+    # Hold so later apt upgrades can't pull the rolling PPA version back.
+    apt-mark hold libze-intel-gpu1 intel-opencl-icd intel-ocloc libigdgmm12 \
+        intel-igc-core-2 intel-igc-opencl-2 && \
    rm -rf /var/lib/apt/lists/*


@@ -36,6 +36,14 @@ RETRIABLE_PATTERNS = [
    r"timeout",
 ]

+# XPU/B580 device-resource flakes. Matched BEFORE the non-retriable list so
+# transient GPU OOMs / slow cold-cache server starts get one clean re-run.
+INFRA_RETRIABLE_PATTERNS = [
+    r"UR_RESULT_ERROR_OUT_OF_RESOURCES",
+    r"XPU out of memory",
+    r"Server failed to start within the timeout",
+]
+
 # Patterns that indicate non-retriable failures (real code errors)
 NON_RETRIABLE_PATTERNS = [
    r"SyntaxError",
@@ -61,6 +69,11 @@ def is_retriable_failure(output: str) -> tuple[bool, str]:
    Returns:
        tuple: (is_retriable, reason)
    """
+    # XPU infra flakes take precedence over the non-retriable list.
+    for pattern in INFRA_RETRIABLE_PATTERNS:
+        if re.search(pattern, output, re.IGNORECASE):
+            return True, f"retriable XPU infra flake: {pattern}"
+
    # Check for non-retriable patterns first
    for pattern in NON_RETRIABLE_PATTERNS:
        if re.search(pattern, output, re.IGNORECASE):
@@ -269,6 +282,17 @@ def run_unittest_files(
                # record the timeout cap as an upper bound so the file still
                # appears in the TIMINGS block below.
                file_elapsed[filename] = float(timeout_per_file)
+                # Retry once on timeout: usually a stuck server / hung device.
+                # A real hang times out again and is reported.
+                if enable_retry and attempt < max_attempts:
+                    logger.info(
+                        f"\n[CI Retry] {filename} timed out after "
+                        f"{timeout_per_file}s; waiting {retry_wait_seconds}s "
+                        f"before retry (attempt {attempt + 1}/{max_attempts})\n"
+                    )
+                    time.sleep(retry_wait_seconds)
+                    attempt += 1
+                    continue
                logger.info(
                    f"\n✗ TIMEOUT: {filename} after {timeout_per_file} seconds\n"
                )
@@ -248,6 +248,9 @@ if is_blackwell_system():
 if is_h200_system():
    DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH = 3600

+if is_in_ci() and is_xpu():
+    DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH = 1800
+

 def call_generate_lightllm(prompt, temperature, max_tokens, stop=None, url=None):
    assert url is not None
@@ -89,6 +89,18 @@ fi
 VIDEO_GID=$(getent group video | cut -d: -f3)
 RENDER_GID=$(getent group render | cut -d: -f3)

+# Forward ZE_AFFINITY_MASK so each runner pins to its own GPU (else all pile onto L0 dev 0).
+# ONEAPI_DEVICE_SELECTOR keeps SYCL consistent with the L0-filtered device.
+GPU_AFFINITY_ARGS=()
+if [[ -n "${ZE_AFFINITY_MASK:-}" ]]; then
+  echo "Pinning container to GPU via ZE_AFFINITY_MASK=${ZE_AFFINITY_MASK}"
+  GPU_AFFINITY_ARGS+=(-e "ZE_AFFINITY_MASK=${ZE_AFFINITY_MASK}")
+  GPU_AFFINITY_ARGS+=(-e "ONEAPI_DEVICE_SELECTOR=level_zero:0")
+else
+  echo "Warning: ZE_AFFINITY_MASK is not set; container will default to GPU 0." >&2
+  echo "         Set ZE_AFFINITY_MASK per runner to spread jobs across GPUs." >&2
+fi
+
 HF_TOKEN_FILE="${HOME}/huggingface_token.txt"
 HF_TOKEN_VALUE=""
 if [[ -n "${HF_TOKEN:-}" ]]; then
@@ -97,6 +109,21 @@ elif [[ -r "${HF_TOKEN_FILE}" ]]; then
  HF_TOKEN_VALUE=$(cat "${HF_TOKEN_FILE}")
 fi

+# Persistent JIT kernel cache (Triton/Inductor/NEO/SYCL) keyed by GPU mask.
+# Cold JIT compile can push test_xpu_basic past its 1200s timeout on B580.
+XPU_KERNEL_CACHE_HOST="${XPU_KERNEL_CACHE_DIR:-${HOME}/.cache/sglang-xpu-ci/kernel-cache-gpu${ZE_AFFINITY_MASK:-shared}}"
+mkdir -p "${XPU_KERNEL_CACHE_HOST}"/{triton,inductor,neo,sycl}
+echo "Using persistent XPU kernel cache: ${XPU_KERNEL_CACHE_HOST}"
+
+# Cap the cache (default 5 GiB); over-cap resets it (misses just recompile).
+XPU_KERNEL_CACHE_MAX_MB="${XPU_KERNEL_CACHE_MAX_MB:-5120}"
+cache_mb=$(du -sm "${XPU_KERNEL_CACHE_HOST}" 2>/dev/null | cut -f1)
+if [[ -n "${cache_mb}" && "${cache_mb}" -gt "${XPU_KERNEL_CACHE_MAX_MB}" ]]; then
+  echo "XPU kernel cache is ${cache_mb} MiB (> ${XPU_KERNEL_CACHE_MAX_MB} MiB cap); resetting it."
+  rm -rf "${XPU_KERNEL_CACHE_HOST:?}"/{triton,inductor,neo,sycl}
+  mkdir -p "${XPU_KERNEL_CACHE_HOST}"/{triton,inductor,neo,sycl}
+fi
+
 echo "Launching container: ${CONTAINER_NAME} from ${IMAGE}"
 # SGLANG_SERVER_LAUNCH_TIMEOUT=36000 matches /data/pgirijal/scripts/setup_upstream_env.sh:
 # 4-GPU MoE loads (Qwen3.5-35B-A3B, gemma-4-26B-A4B, ...) on Arc Pro B60 can
@@ -110,11 +137,35 @@ docker run -dt \
  -v /dev/dri/by-path:/dev/dri/by-path \
  -v "${HOME}/.cache/huggingface:/root/.cache/huggingface" \
  -v "${GITHUB_WORKSPACE:-$PWD}:/sglang-checkout" \
+  -v "${XPU_KERNEL_CACHE_HOST}:/root/.cache/sglang-xpu" \
  -e HF_TOKEN="${HF_TOKEN_VALUE}" \
  -e SGLANG_SERVER_LAUNCH_TIMEOUT=36000 \
+  -e TRITON_CACHE_DIR=/root/.cache/sglang-xpu/triton \
+  -e TORCHINDUCTOR_CACHE_DIR=/root/.cache/sglang-xpu/inductor \
+  -e NEO_CACHE_DIR=/root/.cache/sglang-xpu/neo \
+  -e NEO_CACHE_PERSISTENT=1 \
+  -e SYCL_CACHE_DIR=/root/.cache/sglang-xpu/sycl \
+  -e SYCL_CACHE_PERSISTENT=1 \
+  "${GPU_AFFINITY_ARGS[@]}" \
  --name "${CONTAINER_NAME}" \
  "${IMAGE}"

 # Mark the workspace mount as a safe directory so git operations as root
 # inside the container don't trip the cross-user repo guard.
 docker exec "${CONTAINER_NAME}" git config --global --add safe.directory /sglang-checkout || true
+
+# Pre-warm the HF cache for models used by tests that time out on cold download.
+# popen_launch_server's inner timeout counts network time, so a slow HF Hub can
+# eat the whole window before shard loading begins. Best-effort: on failure the
+# test still tries a live download.
+if [[ -n "${HF_TOKEN_VALUE}" ]]; then
+  docker exec "${CONTAINER_NAME}" /bin/bash -c \
+    "/opt/venv/bin/hf auth login --token '${HF_TOKEN_VALUE}' >/dev/null 2>&1 || true"
+fi
+for model in \
+    "meta-llama/Llama-3.2-1B-Instruct" \
+    "rescommons/SpecForge-EAGLE3-Llama-3.2-1B-Instruct"; do
+  echo "Pre-downloading HF model: ${model}"
+  docker exec "${CONTAINER_NAME}" /opt/venv/bin/hf download "${model}" \
+    >/dev/null 2>&1 || echo "Warning: pre-download of ${model} failed; test will retry online" >&2
+done
@@ -13,7 +13,7 @@ from sglang.test.kits.spec_server_kits import SpecParityKit
 from sglang.test.server_fixtures.spec_eagle_fixture import Eagle3Base

 register_cuda_ci(est_time=360, stage="base-b", runner_config="1-gpu-large")
-register_xpu_ci(est_time=360, stage="stage-b", runner_config="1-gpu-xpu")
+register_xpu_ci(est_time=360, suite="nightly-xpu-1-gpu", nightly=True)

 _is_xpu = is_xpu()