From 53804d609cb3d7e1a3dfaed0c3e9557b7749767d Mon Sep 17 00:00:00 2001 From: ashwini rathi Date: Tue, 4 Aug 2026 13:58:52 +0530 Subject: [PATCH] [CI][XPU] Stabilize XPU CI: pin UMD/IGC, retry infra flakes, right-size EAGLE3 (#32438) Co-authored-by: Ma Mingfei --- .github/workflows/pr-test-xpu.yml | 30 ++++++++++- docker/xpu.Dockerfile | 34 +++++++++++-- python/sglang/test/ci/ci_utils.py | 24 +++++++++ python/sglang/test/test_utils.py | 3 ++ scripts/ci/xpu/xpu_ci_start_container.sh | 51 +++++++++++++++++++ .../spec/eagle/test_spec_eagle_parity.py | 2 +- 6 files changed, 137 insertions(+), 7 deletions(-) diff --git a/.github/workflows/pr-test-xpu.yml b/.github/workflows/pr-test-xpu.yml index ce94e6dae..5c4adecca 100644 --- a/.github/workflows/pr-test-xpu.yml +++ b/.github/workflows/pr-test-xpu.yml @@ -113,7 +113,7 @@ jobs: - name: Run stage-a tests timeout-minutes: 30 run: | - docker exec ci_sglang_xpu bash -c "source /opt/venv/bin/activate && cd /sglang-checkout/test && python3 run_suite.py --hw xpu --suite stage-a-test-1-gpu-xpu" + docker exec ci_sglang_xpu bash -c "source /opt/venv/bin/activate && cd /sglang-checkout/test && python3 run_suite.py --hw xpu --suite stage-a-test-1-gpu-xpu --enable-retry" - name: Cleanup container if: always() @@ -136,6 +136,19 @@ jobs: .pytest_cache || true find . -type d -name "__pycache__" -prune -exec rm -rf {} + || true find . -type f -name "*.pyc" -delete || true + # SIGTERM sglang and drain GPU context before `docker rm -f`; + # SIGKILL leaves the xe/GuC exec queue registered and triggers a + # GT reset (+ devcoredump) on B580. + if docker ps --format '{{.Names}}' | grep -qx ci_sglang_xpu; then + docker exec ci_sglang_xpu bash -c ' + pkill -TERM -f "sglang|run_suite|python3.*test_" 2>/dev/null || true + for _ in $(seq 1 30); do + pgrep -f "sglang::|sglang.launch_server" >/dev/null || break + sleep 1 + done + pkill -KILL -f "sglang|run_suite" 2>/dev/null || true + ' || true + fi docker rm -f ci_sglang_xpu || true if [[ -n "${CI_SGLANG_XPU_IMAGE:-}" ]]; then docker rmi -f "${CI_SGLANG_XPU_IMAGE}" || true @@ -189,7 +202,7 @@ jobs: - name: Run stage-b tests timeout-minutes: 60 run: | - docker exec ci_sglang_xpu bash -c "source /opt/venv/bin/activate && cd /sglang-checkout/test && python3 run_suite.py --hw xpu --suite stage-b-test-1-gpu-xpu" + docker exec ci_sglang_xpu bash -c "source /opt/venv/bin/activate && cd /sglang-checkout/test && python3 run_suite.py --hw xpu --suite stage-b-test-1-gpu-xpu --enable-retry" - name: Cleanup container if: always() @@ -212,6 +225,19 @@ jobs: .pytest_cache || true find . -type d -name "__pycache__" -prune -exec rm -rf {} + || true find . -type f -name "*.pyc" -delete || true + # SIGTERM sglang and drain GPU context before `docker rm -f`; + # SIGKILL leaves the xe/GuC exec queue registered and triggers a + # GT reset (+ devcoredump) on B580. + if docker ps --format '{{.Names}}' | grep -qx ci_sglang_xpu; then + docker exec ci_sglang_xpu bash -c ' + pkill -TERM -f "sglang|run_suite|python3.*test_" 2>/dev/null || true + for _ in $(seq 1 30); do + pgrep -f "sglang::|sglang.launch_server" >/dev/null || break + sleep 1 + done + pkill -KILL -f "sglang|run_suite" 2>/dev/null || true + ' || true + fi docker rm -f ci_sglang_xpu || true if [[ -n "${CI_SGLANG_XPU_IMAGE:-}" ]]; then docker rmi -f "${CI_SGLANG_XPU_IMAGE}" || true diff --git a/docker/xpu.Dockerfile b/docker/xpu.Dockerfile index 6d169f958..b2dbae89b 100644 --- a/docker/xpu.Dockerfile +++ b/docker/xpu.Dockerfile @@ -17,14 +17,40 @@ ARG SG_LANG_KERNEL_BRANCH=main USER root -# Install the latest UMD driver for SYCL-TLA -RUN apt-get update && apt-get install -y software-properties-common && \ +# Pin Level-Zero UMD + IGC (rolling PPA once faulted libze on B580; see sgl-kernel-xpu#296). +# Keep in lockstep with the host xe KMD; override via --build-arg. +ARG COMPUTE_RUNTIME_VERSION=26.05.37020.3 +ARG IGC_VERSION=2.28.4+20760 +ARG GMM_VERSION=22.9.0 +RUN apt-get update && apt-get install -y software-properties-common curl && \ add-apt-repository -y ppa:kobuk-team/intel-graphics && \ apt-get update && \ + # Loader + media/metrics from the PPA; the GPU driver is pinned below. apt-get install -y \ - libze-intel-gpu1 libze1 intel-metrics-discovery intel-opencl-icd clinfo intel-gsc \ + libze1 intel-metrics-discovery clinfo intel-gsc \ intel-media-va-driver-non-free libmfx-gen1 libvpl2 libvpl-tools libva-glx2 va-driver-all vainfo \ - libze-dev intel-ocloc && \ + libze-dev && \ + cd /tmp && \ + igc_url="https://github.com/intel/intel-graphics-compiler/releases/download/v${IGC_VERSION%%+*}" && \ + cr_url="https://github.com/intel/compute-runtime/releases/download/${COMPUTE_RUNTIME_VERSION}" && \ + # IGC first: libze-intel-gpu1 / intel-opencl-icd depend on its exact version. + curl -fsSL -O "${igc_url}/intel-igc-core-2_${IGC_VERSION}_amd64.deb" && \ + curl -fsSL -O "${igc_url}/intel-igc-opencl-2_${IGC_VERSION}_amd64.deb" && \ + curl -fsSL -O "${cr_url}/libze-intel-gpu1_${COMPUTE_RUNTIME_VERSION}-0_amd64.deb" && \ + curl -fsSL -O "${cr_url}/intel-opencl-icd_${COMPUTE_RUNTIME_VERSION}-0_amd64.deb" && \ + curl -fsSL -O "${cr_url}/intel-ocloc_${COMPUTE_RUNTIME_VERSION}-0_amd64.deb" && \ + curl -fsSL -O "${cr_url}/libigdgmm12_${GMM_VERSION}_amd64.deb" && \ + apt-get install -y --allow-downgrades \ + ./intel-igc-core-2_${IGC_VERSION}_amd64.deb \ + ./intel-igc-opencl-2_${IGC_VERSION}_amd64.deb \ + ./libigdgmm12_${GMM_VERSION}_amd64.deb \ + ./libze-intel-gpu1_${COMPUTE_RUNTIME_VERSION}-0_amd64.deb \ + ./intel-opencl-icd_${COMPUTE_RUNTIME_VERSION}-0_amd64.deb \ + ./intel-ocloc_${COMPUTE_RUNTIME_VERSION}-0_amd64.deb && \ + rm -f /tmp/*.deb && \ + # Hold so later apt upgrades can't pull the rolling PPA version back. + apt-mark hold libze-intel-gpu1 intel-opencl-icd intel-ocloc libigdgmm12 \ + intel-igc-core-2 intel-igc-opencl-2 && \ rm -rf /var/lib/apt/lists/* diff --git a/python/sglang/test/ci/ci_utils.py b/python/sglang/test/ci/ci_utils.py index 2eaf654a0..dd8907fd9 100644 --- a/python/sglang/test/ci/ci_utils.py +++ b/python/sglang/test/ci/ci_utils.py @@ -36,6 +36,14 @@ RETRIABLE_PATTERNS = [ r"timeout", ] +# XPU/B580 device-resource flakes. Matched BEFORE the non-retriable list so +# transient GPU OOMs / slow cold-cache server starts get one clean re-run. +INFRA_RETRIABLE_PATTERNS = [ + r"UR_RESULT_ERROR_OUT_OF_RESOURCES", + r"XPU out of memory", + r"Server failed to start within the timeout", +] + # Patterns that indicate non-retriable failures (real code errors) NON_RETRIABLE_PATTERNS = [ r"SyntaxError", @@ -61,6 +69,11 @@ def is_retriable_failure(output: str) -> tuple[bool, str]: Returns: tuple: (is_retriable, reason) """ + # XPU infra flakes take precedence over the non-retriable list. + for pattern in INFRA_RETRIABLE_PATTERNS: + if re.search(pattern, output, re.IGNORECASE): + return True, f"retriable XPU infra flake: {pattern}" + # Check for non-retriable patterns first for pattern in NON_RETRIABLE_PATTERNS: if re.search(pattern, output, re.IGNORECASE): @@ -269,6 +282,17 @@ def run_unittest_files( # record the timeout cap as an upper bound so the file still # appears in the TIMINGS block below. file_elapsed[filename] = float(timeout_per_file) + # Retry once on timeout: usually a stuck server / hung device. + # A real hang times out again and is reported. + if enable_retry and attempt < max_attempts: + logger.info( + f"\n[CI Retry] {filename} timed out after " + f"{timeout_per_file}s; waiting {retry_wait_seconds}s " + f"before retry (attempt {attempt + 1}/{max_attempts})\n" + ) + time.sleep(retry_wait_seconds) + attempt += 1 + continue logger.info( f"\n✗ TIMEOUT: {filename} after {timeout_per_file} seconds\n" ) diff --git a/python/sglang/test/test_utils.py b/python/sglang/test/test_utils.py index 2f12b7d1e..aa7535ca6 100644 --- a/python/sglang/test/test_utils.py +++ b/python/sglang/test/test_utils.py @@ -248,6 +248,9 @@ if is_blackwell_system(): if is_h200_system(): DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH = 3600 +if is_in_ci() and is_xpu(): + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH = 1800 + def call_generate_lightllm(prompt, temperature, max_tokens, stop=None, url=None): assert url is not None diff --git a/scripts/ci/xpu/xpu_ci_start_container.sh b/scripts/ci/xpu/xpu_ci_start_container.sh index 61e6566b7..11314a792 100755 --- a/scripts/ci/xpu/xpu_ci_start_container.sh +++ b/scripts/ci/xpu/xpu_ci_start_container.sh @@ -89,6 +89,18 @@ fi VIDEO_GID=$(getent group video | cut -d: -f3) RENDER_GID=$(getent group render | cut -d: -f3) +# Forward ZE_AFFINITY_MASK so each runner pins to its own GPU (else all pile onto L0 dev 0). +# ONEAPI_DEVICE_SELECTOR keeps SYCL consistent with the L0-filtered device. +GPU_AFFINITY_ARGS=() +if [[ -n "${ZE_AFFINITY_MASK:-}" ]]; then + echo "Pinning container to GPU via ZE_AFFINITY_MASK=${ZE_AFFINITY_MASK}" + GPU_AFFINITY_ARGS+=(-e "ZE_AFFINITY_MASK=${ZE_AFFINITY_MASK}") + GPU_AFFINITY_ARGS+=(-e "ONEAPI_DEVICE_SELECTOR=level_zero:0") +else + echo "Warning: ZE_AFFINITY_MASK is not set; container will default to GPU 0." >&2 + echo " Set ZE_AFFINITY_MASK per runner to spread jobs across GPUs." >&2 +fi + HF_TOKEN_FILE="${HOME}/huggingface_token.txt" HF_TOKEN_VALUE="" if [[ -n "${HF_TOKEN:-}" ]]; then @@ -97,6 +109,21 @@ elif [[ -r "${HF_TOKEN_FILE}" ]]; then HF_TOKEN_VALUE=$(cat "${HF_TOKEN_FILE}") fi +# Persistent JIT kernel cache (Triton/Inductor/NEO/SYCL) keyed by GPU mask. +# Cold JIT compile can push test_xpu_basic past its 1200s timeout on B580. +XPU_KERNEL_CACHE_HOST="${XPU_KERNEL_CACHE_DIR:-${HOME}/.cache/sglang-xpu-ci/kernel-cache-gpu${ZE_AFFINITY_MASK:-shared}}" +mkdir -p "${XPU_KERNEL_CACHE_HOST}"/{triton,inductor,neo,sycl} +echo "Using persistent XPU kernel cache: ${XPU_KERNEL_CACHE_HOST}" + +# Cap the cache (default 5 GiB); over-cap resets it (misses just recompile). +XPU_KERNEL_CACHE_MAX_MB="${XPU_KERNEL_CACHE_MAX_MB:-5120}" +cache_mb=$(du -sm "${XPU_KERNEL_CACHE_HOST}" 2>/dev/null | cut -f1) +if [[ -n "${cache_mb}" && "${cache_mb}" -gt "${XPU_KERNEL_CACHE_MAX_MB}" ]]; then + echo "XPU kernel cache is ${cache_mb} MiB (> ${XPU_KERNEL_CACHE_MAX_MB} MiB cap); resetting it." + rm -rf "${XPU_KERNEL_CACHE_HOST:?}"/{triton,inductor,neo,sycl} + mkdir -p "${XPU_KERNEL_CACHE_HOST}"/{triton,inductor,neo,sycl} +fi + echo "Launching container: ${CONTAINER_NAME} from ${IMAGE}" # SGLANG_SERVER_LAUNCH_TIMEOUT=36000 matches /data/pgirijal/scripts/setup_upstream_env.sh: # 4-GPU MoE loads (Qwen3.5-35B-A3B, gemma-4-26B-A4B, ...) on Arc Pro B60 can @@ -110,11 +137,35 @@ docker run -dt \ -v /dev/dri/by-path:/dev/dri/by-path \ -v "${HOME}/.cache/huggingface:/root/.cache/huggingface" \ -v "${GITHUB_WORKSPACE:-$PWD}:/sglang-checkout" \ + -v "${XPU_KERNEL_CACHE_HOST}:/root/.cache/sglang-xpu" \ -e HF_TOKEN="${HF_TOKEN_VALUE}" \ -e SGLANG_SERVER_LAUNCH_TIMEOUT=36000 \ + -e TRITON_CACHE_DIR=/root/.cache/sglang-xpu/triton \ + -e TORCHINDUCTOR_CACHE_DIR=/root/.cache/sglang-xpu/inductor \ + -e NEO_CACHE_DIR=/root/.cache/sglang-xpu/neo \ + -e NEO_CACHE_PERSISTENT=1 \ + -e SYCL_CACHE_DIR=/root/.cache/sglang-xpu/sycl \ + -e SYCL_CACHE_PERSISTENT=1 \ + "${GPU_AFFINITY_ARGS[@]}" \ --name "${CONTAINER_NAME}" \ "${IMAGE}" # Mark the workspace mount as a safe directory so git operations as root # inside the container don't trip the cross-user repo guard. docker exec "${CONTAINER_NAME}" git config --global --add safe.directory /sglang-checkout || true + +# Pre-warm the HF cache for models used by tests that time out on cold download. +# popen_launch_server's inner timeout counts network time, so a slow HF Hub can +# eat the whole window before shard loading begins. Best-effort: on failure the +# test still tries a live download. +if [[ -n "${HF_TOKEN_VALUE}" ]]; then + docker exec "${CONTAINER_NAME}" /bin/bash -c \ + "/opt/venv/bin/hf auth login --token '${HF_TOKEN_VALUE}' >/dev/null 2>&1 || true" +fi +for model in \ + "meta-llama/Llama-3.2-1B-Instruct" \ + "rescommons/SpecForge-EAGLE3-Llama-3.2-1B-Instruct"; do + echo "Pre-downloading HF model: ${model}" + docker exec "${CONTAINER_NAME}" /opt/venv/bin/hf download "${model}" \ + >/dev/null 2>&1 || echo "Warning: pre-download of ${model} failed; test will retry online" >&2 +done diff --git a/test/registered/spec/eagle/test_spec_eagle_parity.py b/test/registered/spec/eagle/test_spec_eagle_parity.py index 9059dd4ee..8371a7e3d 100644 --- a/test/registered/spec/eagle/test_spec_eagle_parity.py +++ b/test/registered/spec/eagle/test_spec_eagle_parity.py @@ -13,7 +13,7 @@ from sglang.test.kits.spec_server_kits import SpecParityKit from sglang.test.server_fixtures.spec_eagle_fixture import Eagle3Base register_cuda_ci(est_time=360, stage="base-b", runner_config="1-gpu-large") -register_xpu_ci(est_time=360, stage="stage-b", runner_config="1-gpu-xpu") +register_xpu_ci(est_time=360, suite="nightly-xpu-1-gpu", nightly=True) _is_xpu = is_xpu()