From fafa302e414750884f696dac5efbfb95825d9eb3 Mon Sep 17 00:00:00 2001 From: ashwini rathi Date: Mon, 20 Jul 2026 13:45:50 +0530 Subject: [PATCH] [XPU][NIGHTLY] Add 8 XPU nightly tests, enable 1-gpu suite (#30246) Co-authored-by: arathi-hlab Co-authored-by: Claude Opus 4.8 (1M context) --- .github/workflows/nightly-test-intel.yml | 88 +++++++- .../test/xpu/simple_eval_gsm8k_xpu_mixin.py | 23 +-- scripts/ci/xpu/xpu_ci_start_container.sh | 4 + .../llm_models/test_xpu_gemma_4_26b_a4b.py | 42 ++++ .../xpu/llm_models/test_xpu_llama_3_1_8b.py | 3 +- .../test_xpu_nemotron_3_nano_30b_a3b.py | 59 ++++++ ...qwen3_32b.py => test_xpu_qwen3_30b_a3b.py} | 21 +- .../llm_models/test_xpu_qwen3_5_35b_a3b.py | 43 ++++ .../xpu/llm_models/test_xpu_qwen3_5_9b.py | 41 ++++ .../xpu/test_deepseek_ocr_2_olmbench.py | 171 ++++++++++++++++ .../xpu/test_triton_attention_backend.py | 2 +- test/registered/xpu/test_xpu_flux2_dev.py | 188 ++++++++++++++++++ test/registered/xpu/test_xpu_zimage_turbo.py | 179 +++++++++++++++++ 13 files changed, 828 insertions(+), 36 deletions(-) create mode 100644 test/registered/xpu/llm_models/test_xpu_gemma_4_26b_a4b.py create mode 100644 test/registered/xpu/llm_models/test_xpu_nemotron_3_nano_30b_a3b.py rename test/registered/xpu/llm_models/{test_xpu_qwen3_32b.py => test_xpu_qwen3_30b_a3b.py} (55%) create mode 100644 test/registered/xpu/llm_models/test_xpu_qwen3_5_35b_a3b.py create mode 100644 test/registered/xpu/llm_models/test_xpu_qwen3_5_9b.py create mode 100644 test/registered/xpu/test_deepseek_ocr_2_olmbench.py create mode 100644 test/registered/xpu/test_xpu_flux2_dev.py create mode 100644 test/registered/xpu/test_xpu_zimage_turbo.py diff --git a/.github/workflows/nightly-test-intel.yml b/.github/workflows/nightly-test-intel.yml index 37817a80e..2bb9e3a9f 100644 --- a/.github/workflows/nightly-test-intel.yml +++ b/.github/workflows/nightly-test-intel.yml @@ -35,13 +35,78 @@ concurrency: jobs: nightly-xpu-1-gpu: - # Placeholder: no models currently registered to nightly-xpu-1-gpu run on - # intel-bmg-nightly. Add a model test to this suite to re-enable. - if: false + if: github.repository == 'sgl-project/sglang' runs-on: intel-bmg-nightly + env: + DOCKERHUB_INTEL_USERNAME: ${{ secrets.DOCKERHUB_INTEL_USERNAME }} + DOCKERHUB_INTEL_TOKEN: ${{ secrets.DOCKERHUB_INTEL_TOKEN }} steps: - - name: Placeholder - run: echo "nightly-xpu-1-gpu has no validated models; skipping." + - name: Reset workspace ownership + run: | + docker run --rm -v "${{ github.workspace }}:/w" busybox:latest \ + chown -R "$(id -u):$(id -g)" /w || true + + - name: Checkout code + uses: actions/checkout@v4 + with: + fetch-depth: 0 + ref: ${{ inputs.ref || github.sha }} + + - name: Start CI container (pull intel/sglang-dev:latest) + run: | + export HF_TOKEN="$(cat ~/huggingface_token.txt)" + bash scripts/ci/xpu/xpu_ci_start_container.sh + env: + GITHUB_WORKSPACE: ${{ github.workspace }} + + - name: HF login + install run_suite extras + timeout-minutes: 10 + run: | + docker exec ci_sglang_xpu /opt/venv/bin/python3 -m pip install --no-cache-dir tabulate + docker exec ci_sglang_xpu /opt/venv/bin/python3 -m pip install --no-cache-dir --no-deps xgrammar==0.1.33 + docker exec ci_sglang_xpu /opt/venv/bin/python3 -m pip install --no-cache-dir "lm-eval==0.4.9" + docker exec ci_sglang_xpu /opt/venv/bin/python3 -m pip install --no-cache-dir pytest + # Diffusion extras (needed by test_xpu_flux2_dev, test_xpu_zimage_turbo). + # Pins mirror python/pyproject_xpu.toml [project.optional-dependencies.diffusion]. + docker exec ci_sglang_xpu /opt/venv/bin/python3 -m pip install --no-cache-dir "sglang[diffusion]" + docker exec ci_sglang_xpu /bin/bash -c '/opt/venv/bin/hf auth login --token ${HF_TOKEN}' + + - name: Download olmOCR-bench dataset (for test_deepseek_ocr_2_olmbench) + timeout-minutes: 30 + run: | + # PyMuPDF renders the bench PDFs to images; bench_sglang.py errors + # every sample without it. + docker exec ci_sglang_xpu /opt/venv/bin/python3 -m pip install --no-cache-dir pymupdf + docker exec ci_sglang_xpu /bin/bash -c ' + /opt/venv/bin/hf download --repo-type dataset allenai/olmOCR-bench \ + --local-dir /sglang-checkout/olmOCR-bench' + + - name: Nightly Test (1-GPU XPU) + timeout-minutes: 240 + run: | + touch github_summary.md + docker exec ci_sglang_xpu bash -c " + source /opt/venv/bin/activate && + cd /sglang-checkout/test && + OLMOCR_BENCH_DIR=/sglang-checkout/olmOCR-bench/bench_data \ + GITHUB_STEP_SUMMARY=/sglang-checkout/github_summary.md \ + python3 run_suite.py --hw xpu --suite nightly-xpu-1-gpu --nightly --timeout-per-file 7200 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} + " || TEST_EXIT_CODE=$? + echo "$(> $GITHUB_STEP_SUMMARY || true + exit ${TEST_EXIT_CODE:-0} + + - name: Cleanup container + if: always() + run: | + docker run --rm -v "${{ github.workspace }}:/w" busybox:latest \ + chown -R "$(id -u):$(id -g)" /w || true + rm -rf test/result.jsonl test/results test/.pytest_cache .pytest_cache || true + find . -type d -name "__pycache__" -prune -exec rm -rf {} + || true + find . -type f -name "*.pyc" -delete || true + docker rm -f ci_sglang_xpu || true + if [[ -n "${CI_SGLANG_XPU_IMAGE:-}" ]]; then + docker rmi -f "${CI_SGLANG_XPU_IMAGE}" || true + fi nightly-xpu-2-gpu: if: github.repository == 'sgl-project/sglang' @@ -74,10 +139,14 @@ jobs: docker exec ci_sglang_xpu /opt/venv/bin/python3 -m pip install --no-cache-dir tabulate docker exec ci_sglang_xpu /opt/venv/bin/python3 -m pip install --no-cache-dir --no-deps xgrammar==0.1.33 docker exec ci_sglang_xpu /opt/venv/bin/python3 -m pip install --no-cache-dir "lm-eval==0.4.9" + docker exec ci_sglang_xpu /opt/venv/bin/python3 -m pip install --no-cache-dir pytest + # Diffusion extras (needed by test_xpu_flux2_dev, test_xpu_zimage_turbo). + # Pins mirror python/pyproject_xpu.toml [project.optional-dependencies.diffusion]. + docker exec ci_sglang_xpu /opt/venv/bin/python3 -m pip install --no-cache-dir "sglang[diffusion]" docker exec ci_sglang_xpu /bin/bash -c '/opt/venv/bin/hf auth login --token ${HF_TOKEN}' - name: Nightly Test (2-GPU XPU) - timeout-minutes: 60 + timeout-minutes: 240 run: | touch github_summary.md docker exec ci_sglang_xpu bash -c " @@ -133,10 +202,14 @@ jobs: docker exec ci_sglang_xpu /opt/venv/bin/python3 -m pip install --no-cache-dir tabulate docker exec ci_sglang_xpu /opt/venv/bin/python3 -m pip install --no-cache-dir --no-deps xgrammar==0.1.33 docker exec ci_sglang_xpu /opt/venv/bin/python3 -m pip install --no-cache-dir "lm-eval==0.4.9" + docker exec ci_sglang_xpu /opt/venv/bin/python3 -m pip install --no-cache-dir pytest + # Diffusion extras (needed by test_xpu_flux2_dev, test_xpu_zimage_turbo). + # Pins mirror python/pyproject_xpu.toml [project.optional-dependencies.diffusion]. + docker exec ci_sglang_xpu /opt/venv/bin/python3 -m pip install --no-cache-dir "sglang[diffusion]" docker exec ci_sglang_xpu /bin/bash -c '/opt/venv/bin/hf auth login --token ${HF_TOKEN}' - name: Nightly Test (4-GPU XPU) - timeout-minutes: 120 + timeout-minutes: 480 run: | touch github_summary.md docker exec ci_sglang_xpu bash -c " @@ -164,6 +237,7 @@ jobs: check-all-jobs: if: always() && (github.repository == 'sgl-project/sglang' || github.event_name == 'workflow_dispatch') needs: + - nightly-xpu-1-gpu - nightly-xpu-2-gpu - nightly-xpu-4-gpu runs-on: ubuntu-latest diff --git a/python/sglang/test/xpu/simple_eval_gsm8k_xpu_mixin.py b/python/sglang/test/xpu/simple_eval_gsm8k_xpu_mixin.py index cb909037b..9b6e1a936 100644 --- a/python/sglang/test/xpu/simple_eval_gsm8k_xpu_mixin.py +++ b/python/sglang/test/xpu/simple_eval_gsm8k_xpu_mixin.py @@ -1,9 +1,8 @@ """simple-evals GSM8K accuracy mixin for Intel XPU nightly tests. -Mirrors the AMD/NVIDIA nightly flow (``test_gsm8k_eval_amd.py`` / -``test_text_models_gsm8k_eval.py``): launch an SGLang server with XPU -flags, then call ``sglang.test.run_eval`` with ``eval_name="gsm8k"`` so -the same ``simple_eval_gsm8k.GSM8KEval`` evaluator scores every backend. +Launches an SGLang server with XPU flags, then calls ``sglang.test.run_eval`` +with ``eval_name="gsm8k"`` so the ``simple_eval_gsm8k.GSM8KEval`` evaluator +scores the run. Subclasses set ``model``, ``tp_size``, ``accuracy``, and may override ``other_args`` / ``env`` / ``num_examples`` / ``num_threads``. @@ -46,20 +45,12 @@ class SimpleEvalGSM8KXPUMixin(ABC): env: dict | None = None server_cmd: str = "" - # 200 questions matches the limit used by the XPU 70B lm-eval YAML and - # fits inside run_suite's per-file timeout when num_threads=1 keeps - # throughput low. Subclasses on cheaper-per-token hardware (TP=1, no - # Level Zero wedge) can raise this or set None for the full 1319-question - # GSM8K test set, matching the AMD/NVIDIA nightly defaults. + # Subset that fits run_suite's per-file timeout; set None for the full set. num_examples: int | None = 200 - # Single-stream eval: intel_xpu attention at TP>=2 wedges the Level Zero - # driver in ur_command_list_manager::appendUSMMemcpy on concurrent prefill. - # Subclasses on hardware that handles parallel prefill cleanly may bump. + # Single-stream: intel_xpu attention at TP>=2 wedges the Level Zero driver + # on concurrent prefill. Subclasses may bump on hardware that handles it. num_threads: int = 1 - # Short generations reduce the rate of prefill->decode->prefill handoffs, - # which is what trips the same Level Zero wedge on TP>=2 (observed at the - # default 2048; 512 matches the original few_shot_gsm8k limit and is still - # enough for GSM8K CoT answers). + # Short generations reduce prefill->decode handoffs that trip the same wedge. max_tokens: int = 512 @classmethod diff --git a/scripts/ci/xpu/xpu_ci_start_container.sh b/scripts/ci/xpu/xpu_ci_start_container.sh index 4f75dca42..61e6566b7 100755 --- a/scripts/ci/xpu/xpu_ci_start_container.sh +++ b/scripts/ci/xpu/xpu_ci_start_container.sh @@ -98,6 +98,9 @@ elif [[ -r "${HF_TOKEN_FILE}" ]]; then fi echo "Launching container: ${CONTAINER_NAME} from ${IMAGE}" +# SGLANG_SERVER_LAUNCH_TIMEOUT=36000 matches /data/pgirijal/scripts/setup_upstream_env.sh: +# 4-GPU MoE loads (Qwen3.5-35B-A3B, gemma-4-26B-A4B, ...) on Arc Pro B60 can +# take >1h from a cold HF cache, so give sglang server startup a 10h ceiling. docker run -dt \ --shm-size 8g \ --group-add 992 \ @@ -108,6 +111,7 @@ docker run -dt \ -v "${HOME}/.cache/huggingface:/root/.cache/huggingface" \ -v "${GITHUB_WORKSPACE:-$PWD}:/sglang-checkout" \ -e HF_TOKEN="${HF_TOKEN_VALUE}" \ + -e SGLANG_SERVER_LAUNCH_TIMEOUT=36000 \ --name "${CONTAINER_NAME}" \ "${IMAGE}" diff --git a/test/registered/xpu/llm_models/test_xpu_gemma_4_26b_a4b.py b/test/registered/xpu/llm_models/test_xpu_gemma_4_26b_a4b.py new file mode 100644 index 000000000..156888af8 --- /dev/null +++ b/test/registered/xpu/llm_models/test_xpu_gemma_4_26b_a4b.py @@ -0,0 +1,42 @@ +"""gemma-4-26B-A4B GSM8K accuracy on Intel XPU (TP=4). + +Scored by ``simple_eval_gsm8k.GSM8KEval``. +""" + +import unittest + +import torch + +from sglang.test.ci.ci_register import register_xpu_ci +from sglang.test.test_utils import CustomTestCase +from sglang.test.xpu.simple_eval_gsm8k_xpu_mixin import SimpleEvalGSM8KXPUMixin + +register_xpu_ci(est_time=2400, suite="nightly-xpu-4-gpu", nightly=True) + + +@unittest.skipUnless( + torch.xpu.is_available(), + "Intel XPU not available (torch.xpu.is_available() returned False)", +) +class TestGemma4_26BA4BXPU(SimpleEvalGSM8KXPUMixin, CustomTestCase): + model = "google/gemma-4-26B-A4B-it" + tp_size = 4 + accuracy = 0.90 + timeout_for_server_launch = 3600 + env = {"SGLANG_USE_SGL_XPU": "1"} + + # Gemma-4 hybrid-attention kernels crash under chunked prefill on XPU. + other_args = SimpleEvalGSM8KXPUMixin.other_args + [ + "--page-size", + "64", + "--max-total-tokens", + "65536", + "--mem-fraction-static", + "0.9", + "--chunked-prefill-size", + "-1", + ] + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/xpu/llm_models/test_xpu_llama_3_1_8b.py b/test/registered/xpu/llm_models/test_xpu_llama_3_1_8b.py index 1e933ce79..6194e1324 100644 --- a/test/registered/xpu/llm_models/test_xpu_llama_3_1_8b.py +++ b/test/registered/xpu/llm_models/test_xpu_llama_3_1_8b.py @@ -3,8 +3,7 @@ TP=4 wedges the Level Zero driver during the first prefill batch on Arc/BMG; TP=2 runs cleanly with the same model and serves at ~18 tok/s. -Scored by ``simple_eval_gsm8k.GSM8KEval`` (the same evaluator AMD and -NVIDIA nightlies use); threshold mirrors theirs. +Scored by ``simple_eval_gsm8k.GSM8KEval``. """ import unittest diff --git a/test/registered/xpu/llm_models/test_xpu_nemotron_3_nano_30b_a3b.py b/test/registered/xpu/llm_models/test_xpu_nemotron_3_nano_30b_a3b.py new file mode 100644 index 000000000..174099c15 --- /dev/null +++ b/test/registered/xpu/llm_models/test_xpu_nemotron_3_nano_30b_a3b.py @@ -0,0 +1,59 @@ +"""NVIDIA-Nemotron-3-Nano-30B-A3B GSM8K accuracy on Intel XPU (TP=4). + +Scored by ``simple_eval_gsm8k.GSM8KEval``. +""" + +import unittest + +import torch + +from sglang.test.ci.ci_register import register_xpu_ci +from sglang.test.test_utils import CustomTestCase +from sglang.test.xpu.simple_eval_gsm8k_xpu_mixin import SimpleEvalGSM8KXPUMixin + +register_xpu_ci(est_time=2400, suite="nightly-xpu-4-gpu", nightly=True) + + +@unittest.skipUnless( + torch.xpu.is_available(), + "Intel XPU not available (torch.xpu.is_available() returned False)", +) +class TestNemotron3Nano30BA3BXPU(SimpleEvalGSM8KXPUMixin, CustomTestCase): + model = "nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-BF16" + tp_size = 4 + accuracy = 0.72 + timeout_for_server_launch = 3600 + # Generation cap for the GSM8K eval (mixin default is 512). + max_tokens = 8192 + # Client-side eval concurrency (mixin default is 1). + num_threads = 4 + env = {"SGLANG_USE_SGL_XPU": "1"} + + # Hybrid-mamba layout needs --model-impl sglang, a fixed page size, and + # the nemotron_3 reasoning / qwen3_coder tool-call parsers. + other_args = SimpleEvalGSM8KXPUMixin.other_args + [ + "--max-total-tokens", + "65536", + "--mem-fraction-static", + "0.85", + "--context-length", + "16384", + "--page-size", + "64", + "--chunked-prefill-size", + "1024", + "--max-running-requests", + "8", + "--watchdog-timeout", + "1200", + "--model-impl", + "sglang", + "--tool-call-parser", + "qwen3_coder", + "--reasoning-parser", + "nemotron_3", + ] + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/xpu/llm_models/test_xpu_qwen3_32b.py b/test/registered/xpu/llm_models/test_xpu_qwen3_30b_a3b.py similarity index 55% rename from test/registered/xpu/llm_models/test_xpu_qwen3_32b.py rename to test/registered/xpu/llm_models/test_xpu_qwen3_30b_a3b.py index 14dda840f..1548b950b 100644 --- a/test/registered/xpu/llm_models/test_xpu_qwen3_32b.py +++ b/test/registered/xpu/llm_models/test_xpu_qwen3_30b_a3b.py @@ -1,7 +1,6 @@ -"""Qwen3-32B GSM8K accuracy on Intel XPU (TP=4). +"""Qwen3-30B-A3B GSM8K accuracy on Intel XPU (TP=4). -Scored by ``simple_eval_gsm8k.GSM8KEval`` (the same evaluator AMD and -NVIDIA nightlies use). +Scored by ``simple_eval_gsm8k.GSM8KEval``. """ import unittest @@ -12,21 +11,23 @@ from sglang.test.ci.ci_register import register_xpu_ci from sglang.test.test_utils import CustomTestCase from sglang.test.xpu.simple_eval_gsm8k_xpu_mixin import SimpleEvalGSM8KXPUMixin -register_xpu_ci(est_time=1800, suite="nightly-xpu-4-gpu", nightly=True) +register_xpu_ci(est_time=2400, suite="nightly-xpu-4-gpu", nightly=True) @unittest.skipUnless( torch.xpu.is_available(), "Intel XPU not available (torch.xpu.is_available() returned False)", ) -class TestQwen3_32BXPU(SimpleEvalGSM8KXPUMixin, CustomTestCase): - model = "Qwen/Qwen3-32B" +class TestQwen3_30BA3BXPU(SimpleEvalGSM8KXPUMixin, CustomTestCase): + model = "Qwen/Qwen3-30B-A3B" tp_size = 4 - accuracy = 0.85 - # 64GB BF16 weights split across 4 ranks take ~9 min to load on Intel - # Arc Pro B60; the default 600s timeout fires mid-startup. Mirror the - # XPU 70B test's 1-hour budget. + accuracy = 0.90 timeout_for_server_launch = 3600 + # SGL XPU MoE kernels gate on this env var. + env = {"SGLANG_USE_SGL_XPU": "1"} + num_examples = 50 + num_threads = 4 + max_tokens = 8192 other_args = SimpleEvalGSM8KXPUMixin.other_args + [ "--max-total-tokens", diff --git a/test/registered/xpu/llm_models/test_xpu_qwen3_5_35b_a3b.py b/test/registered/xpu/llm_models/test_xpu_qwen3_5_35b_a3b.py new file mode 100644 index 000000000..0d5befc28 --- /dev/null +++ b/test/registered/xpu/llm_models/test_xpu_qwen3_5_35b_a3b.py @@ -0,0 +1,43 @@ +"""Qwen3.5-35B-A3B GSM8K accuracy on Intel XPU (TP=4). + +Scored by ``simple_eval_gsm8k.GSM8KEval``. +""" + +import unittest + +import torch + +from sglang.test.ci.ci_register import register_xpu_ci +from sglang.test.test_utils import CustomTestCase +from sglang.test.xpu.simple_eval_gsm8k_xpu_mixin import SimpleEvalGSM8KXPUMixin + +register_xpu_ci(est_time=2400, suite="nightly-xpu-4-gpu", nightly=True) + + +@unittest.skipUnless( + torch.xpu.is_available(), + "Intel XPU not available (torch.xpu.is_available() returned False)", +) +class TestQwen3_5_35BA3BXPU(SimpleEvalGSM8KXPUMixin, CustomTestCase): + model = "Qwen/Qwen3.5-35B-A3B" + tp_size = 4 + accuracy = 0.90 + timeout_for_server_launch = 3600 + # SGL XPU MoE kernels gate on this env var. + env = {"SGLANG_USE_SGL_XPU": "1"} + num_examples = 50 + num_threads = 4 + max_tokens = 8192 + + other_args = SimpleEvalGSM8KXPUMixin.other_args + [ + "--page-size", + "128", + "--max-total-tokens", + "65536", + "--mem-fraction-static", + "0.85", + ] + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/xpu/llm_models/test_xpu_qwen3_5_9b.py b/test/registered/xpu/llm_models/test_xpu_qwen3_5_9b.py new file mode 100644 index 000000000..aacbe320c --- /dev/null +++ b/test/registered/xpu/llm_models/test_xpu_qwen3_5_9b.py @@ -0,0 +1,41 @@ +"""Qwen3.5-9B GSM8K accuracy on Intel XPU (TP=4). + +Scored by ``simple_eval_gsm8k.GSM8KEval``. +""" + +import unittest + +import torch + +from sglang.test.ci.ci_register import register_xpu_ci +from sglang.test.test_utils import CustomTestCase +from sglang.test.xpu.simple_eval_gsm8k_xpu_mixin import SimpleEvalGSM8KXPUMixin + +register_xpu_ci(est_time=2400, suite="nightly-xpu-4-gpu", nightly=True) + + +@unittest.skipUnless( + torch.xpu.is_available(), + "Intel XPU not available (torch.xpu.is_available() returned False)", +) +class TestQwen3_5_9BXPU(SimpleEvalGSM8KXPUMixin, CustomTestCase): + model = "Qwen/Qwen3.5-9B" + tp_size = 4 + accuracy = 0.90 + # max_tokens=8192 lets the GSM8K CoT complete under num_threads=4. + num_examples = 50 + num_threads = 4 + max_tokens = 8192 + + other_args = SimpleEvalGSM8KXPUMixin.other_args + [ + "--page-size", + "128", + "--max-total-tokens", + "65536", + "--mem-fraction-static", + "0.85", + ] + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/xpu/test_deepseek_ocr_2_olmbench.py b/test/registered/xpu/test_deepseek_ocr_2_olmbench.py new file mode 100644 index 000000000..7e3eb5dc5 --- /dev/null +++ b/test/registered/xpu/test_deepseek_ocr_2_olmbench.py @@ -0,0 +1,171 @@ +"""DeepSeek-OCR-2 olmOCR-bench accuracy on Intel XPU (1-GPU nightly). + +Launches the server with the OCR serving config, runs the full olmOCR-bench +via ``benchmark/ocr/bench_sglang.py``, asserts the aggregate score >= 0.80, +and writes the per-split breakdown to the GitHub step summary. + +The olmOCR-bench dataset is downloaded by the nightly workflow step and its +location passed via the OLMOCR_BENCH_DIR env var. +""" + +import json +import os +import subprocess +import sys +import unittest +from pathlib import Path +from urllib.parse import urlparse + +import torch + +from sglang.srt.utils import kill_process_tree +from sglang.test.ci.ci_register import register_xpu_ci +from sglang.test.test_utils import ( + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + CustomTestCase, + popen_launch_server, + write_github_step_summary, +) + +register_xpu_ci(est_time=7200, suite="nightly-xpu-1-gpu", nightly=True) + +# Repo root: test/registered/xpu/ -> parents[3]. +_REPO_ROOT = Path(__file__).resolve().parents[3] +# Default matches the workflow's --local-dir; override with OLMOCR_BENCH_DIR. +_DEFAULT_BENCH_DIR = _REPO_ROOT / "olmOCR-bench" / "bench_data" + + +@unittest.skipUnless( + torch.xpu.is_available(), + "Intel XPU not available (torch.xpu.is_available() returned False)", +) +class TestDeepSeekOCR2OlmBenchXPU(CustomTestCase): + model = "deepseek-ai/DeepSeek-OCR-2" + # Aggregate score (total_passed / total_tests) must clear this. + accuracy = 0.80 + timeout_for_server_launch = DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH + + # Full bench by default; overridable via env for quick sanity runs. + concurrency = int(os.environ.get("OLMOCR_BENCH_CONCURRENCY", "26")) + split = os.environ.get("OLMOCR_BENCH_SPLIT", "all") + max_samples = int(os.environ.get("OLMOCR_BENCH_MAX_SAMPLES", "-1")) + + other_args = [ + "--dtype", + "bfloat16", + "--trust-remote-code", + "--disable-radix-cache", + "--attention-backend", + "intel_xpu", + "--mem-fraction-static", + "0.65", + "--max-running-requests", + "26", + "--enable-mixed-chunk", + "--chunked-prefill-size", + "8192", + "--disable-cuda-graph", + ] + env = {"SGLANG_USE_SGL_XPU": "1"} + + @classmethod + def setUpClass(cls): + cls.base_url = DEFAULT_URL_FOR_TEST + cls.bench_dir = Path(os.environ.get("OLMOCR_BENCH_DIR", _DEFAULT_BENCH_DIR)) + cls.output_dir = _REPO_ROOT / "ocr_bench_results" + env = {**os.environ, **cls.env} + try: + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=cls.timeout_for_server_launch, + other_args=list(cls.other_args), + env=env, + ) + except Exception as e: + write_github_step_summary(f"Failed to launch server for {cls.model}: {e}") + raise AssertionError(f"Test failed for {cls.model}: {e}") + + @classmethod + def tearDownClass(cls): + if getattr(cls, "process", None): + kill_process_tree(cls.process.pid) + + def test_olmocr_bench(self): + if not self.bench_dir.exists(): + self.fail( + f"olmOCR-bench data not found at {self.bench_dir}. Download it first:\n" + " hf download --repo-type dataset allenai/olmOCR-bench " + "--local-dir ./olmOCR-bench" + ) + + port = urlparse(self.base_url).port + cmd = [ + sys.executable, + str(_REPO_ROOT / "benchmark" / "ocr" / "bench_sglang.py"), + "--port", + str(port), + "--split", + self.split, + "--concurrency", + str(self.concurrency), + "--model", + self.model, + *(["--max-samples", str(self.max_samples)] if self.max_samples > 0 else []), + "--bench-dir", + str(self.bench_dir), + "--output-dir", + str(self.output_dir), + ] + + try: + subprocess.run(cmd, check=True, cwd=str(_REPO_ROOT)) + except subprocess.CalledProcessError as e: + self.fail(f"olmOCR-bench run failed for {self.model}: {e}") + + summary_path = self.output_dir / "summary.json" + if not summary_path.exists(): + self.fail(f"Benchmark produced no summary at {summary_path}") + + with open(summary_path, encoding="utf-8") as f: + results = json.load(f) + + total_tests = sum(r.get("total_tests", 0) for r in results.values()) + total_passed = sum(r.get("total_passed", 0) for r in results.values()) + total_errored = sum(r.get("error_samples", 0) for r in results.values()) + score = total_passed / total_tests if total_tests else 0.0 + + lines = [ + f"## DeepSeek-OCR-2 olmOCR-bench (XPU, concurrency {self.concurrency})", + "", + "| Split | Tests | Passed | Score | Errored |", + "| --- | ---: | ---: | ---: | ---: |", + ] + for split, r in results.items(): + lines.append( + f"| {split} | {r.get('total_tests', 0)} | " + f"{r.get('total_passed', 0)} | {r.get('overall_score', 0.0):.1f}% | " + f"{r.get('error_samples', 0)} |" + ) + lines.append( + f"| **TOTAL** | {total_tests} | {total_passed} | " + f"**{100.0 * score:.1f}%** | {total_errored} |" + ) + write_github_step_summary("\n".join(lines) + "\n") + + # Guard against a silent empty run before comparing the score. + self.assertGreater( + total_tests, 0, f"olmOCR-bench scored 0 tests for {self.model}" + ) + self.assertGreaterEqual( + score, + self.accuracy, + f"olmOCR-bench aggregate for {self.model} is {100.0 * score:.1f}%, " + f"below the {100.0 * self.accuracy:.0f}% threshold " + f"({total_errored} samples errored)", + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/xpu/test_triton_attention_backend.py b/test/registered/xpu/test_triton_attention_backend.py index 72e0a99e7..ba0ea1688 100644 --- a/test/registered/xpu/test_triton_attention_backend.py +++ b/test/registered/xpu/test_triton_attention_backend.py @@ -13,7 +13,7 @@ from sglang.test.test_utils import ( run_bench_serving, ) -register_xpu_ci(est_time=600, suite="stage-b-test-1-gpu-xpu") +register_xpu_ci(est_time=600, suite="nightly-xpu-1-gpu", nightly=True) def triton_attention_benchmark(extra_args=None, mem_fraction_static="0.84"): diff --git a/test/registered/xpu/test_xpu_flux2_dev.py b/test/registered/xpu/test_xpu_flux2_dev.py new file mode 100644 index 000000000..198f88e1d --- /dev/null +++ b/test/registered/xpu/test_xpu_flux2_dev.py @@ -0,0 +1,188 @@ +"""FLUX.2-dev text-to-image on Intel XPU (4-GPU nightly). + +Mirrors ``test/registered/amd/test_zimage_turbo.py`` but targets FLUX.2-dev +with ``num_gpus=4`` and registers to the XPU 4-GPU nightly suite. The +diffusion server harness is device-agnostic; XPU dispatch is picked up by +``current_platform`` inside multimodal_gen at server launch. +""" + +from __future__ import annotations + +import io +import logging +import os + +import pytest +import torch + +from sglang.multimodal_gen.test.server.test_server_common import ( # noqa: F401 + DiffusionServerBase, + diffusion_server, +) +from sglang.multimodal_gen.test.server.test_server_utils import ( + ServerContext, + get_generate_fn, +) +from sglang.multimodal_gen.test.server.testcase_configs import ( + DiffusionSamplingParams, + DiffusionServerArgs, + DiffusionTestCase, +) +from sglang.test.ci.ci_register import register_xpu_ci + +logger = logging.getLogger(__name__) + +register_xpu_ci(est_time=3600, suite="nightly-xpu-4-gpu", nightly=True) + +XPU_FLUX2_CASES = [ + DiffusionTestCase( + "flux2_dev_image_t2i", + DiffusionServerArgs( + model_path="black-forest-labs/FLUX.2-dev", + modality="image", + num_gpus=4, + tp_size=4, + dit_layerwise_offload=True, + extras=[ + "--dit-precision", + "bf16", + "--vae-precision", + "bf16", + "--text-encoder-precisions", + "bf16", + ], + ), + DiffusionSamplingParams( + prompt="A curious raccoon in a top hat, oil painting", + output_size="1024x1024", + ), + # XPU has no perf/consistency baseline in + # multimodal_gen/test/server/perf_baselines/. CLIP-score guard below + # is the accuracy check; skip the CUDA-only latency/consistency ones. + run_perf_check=False, + run_consistency_check=False, + run_component_accuracy_check=False, + ), +] + +CLIP_SCORE_THRESHOLD = 0.20 + +ARTIFACT_DIR = os.environ.get( + "SGLANG_DIFFUSION_ARTIFACT_DIR", "/tmp/diffusion-artifacts" +) + + +def _save_image_and_write_summary( + case_id: str, prompt: str, image_bytes: bytes, clip_score: float | None = None +): + ext = "jpg" if image_bytes[:2] == b"\xff\xd8" else "png" + os.makedirs(ARTIFACT_DIR, exist_ok=True) + img_path = os.path.join(ARTIFACT_DIR, f"{case_id}.{ext}") + with open(img_path, "wb") as f: + f.write(image_bytes) + logger.info("Saved image artifact: %s (%d bytes)", img_path, len(image_bytes)) + + summary_file = os.environ.get("GITHUB_STEP_SUMMARY") + if not summary_file: + return + + clip_line = "" + if clip_score is not None: + status = "PASS" if clip_score >= CLIP_SCORE_THRESHOLD else "FAIL" + clip_line = ( + f"| CLIP Score | {clip_score:.4f} " + f"({status}, threshold: {CLIP_SCORE_THRESHOLD}) |\n" + ) + + md = ( + f"### FLUX.2-dev — `{case_id}`\n\n" + f"| | |\n|---|---|\n" + f"| Prompt | {prompt} |\n" + f"| Size | {len(image_bytes):,} bytes |\n" + f"{clip_line}" + f"| Artifact | `{case_id}.{ext}` (download from Artifacts section above) |\n\n" + ) + + with open(summary_file, "a") as f: + f.write(md) + + +def _compute_clip_score(image_bytes: bytes, prompt: str) -> float | None: + try: + from PIL import Image + from transformers import CLIPModel, CLIPProcessor + + model_name = "openai/clip-vit-base-patch32" + processor = CLIPProcessor.from_pretrained(model_name) + model = CLIPModel.from_pretrained(model_name) + model.eval() + + image = Image.open(io.BytesIO(image_bytes)).convert("RGB") + inputs = processor(text=[prompt], images=image, return_tensors="pt") + + with torch.no_grad(): + outputs = model(**inputs) + score = outputs.logits_per_image.item() / 100.0 + + logger.info("CLIP score for '%s': %.4f", prompt, score) + return score + except Exception as e: + logger.warning("CLIP score computation failed: %s", e) + return None + + +@pytest.mark.skipif( + not (hasattr(torch, "xpu") and torch.xpu.is_available()), + reason="Intel XPU not available (torch.xpu.is_available() returned False)", +) +class TestFlux2DevXPU(DiffusionServerBase): + """Intel XPU nightly test for FLUX.2-dev text-to-image generation.""" + + @classmethod + def teardown_class(cls): + try: + super().teardown_class() + except AttributeError: + pass + + @pytest.fixture(params=XPU_FLUX2_CASES, ids=lambda c: c.id) + def case(self, request) -> DiffusionTestCase: + return request.param + + def test_diffusion_generation( + self, + case: DiffusionTestCase, + diffusion_server: ServerContext, + ): + generate_fn = get_generate_fn( + model_path=case.server_args.model_path, + modality=case.server_args.modality, + sampling_params=case.sampling_params, + ) + + perf_record, content = self.run_and_collect( + diffusion_server, case.id, generate_fn + ) + + self._validate_and_record(case, perf_record) + self._test_v1_models_endpoint(diffusion_server, case) + + prompt = case.sampling_params.prompt or "" + clip_score = _compute_clip_score(content, prompt) + + if clip_score is not None: + logger.info( + "CLIP score: %.4f (threshold: %.2f)", clip_score, CLIP_SCORE_THRESHOLD + ) + assert clip_score >= CLIP_SCORE_THRESHOLD, ( + f"CLIP score {clip_score:.4f} below threshold {CLIP_SCORE_THRESHOLD} " + f"for prompt '{prompt}'" + ) + + _save_image_and_write_summary(case.id, prompt, content, clip_score) + + +if __name__ == "__main__": + import sys + + sys.exit(pytest.main([__file__, "-v"])) diff --git a/test/registered/xpu/test_xpu_zimage_turbo.py b/test/registered/xpu/test_xpu_zimage_turbo.py new file mode 100644 index 000000000..ab20ed819 --- /dev/null +++ b/test/registered/xpu/test_xpu_zimage_turbo.py @@ -0,0 +1,179 @@ +"""Z-Image-Turbo text-to-image on Intel XPU (1-GPU nightly). + +Mirrors ``test/registered/amd/test_zimage_turbo.py`` but registers to the +XPU 1-GPU nightly suite. The diffusion server harness is device-agnostic; +XPU dispatch is picked up by ``current_platform`` inside multimodal_gen at +server launch. +""" + +from __future__ import annotations + +import io +import logging +import os + +import pytest +import torch + +from sglang.multimodal_gen.test.server.test_server_common import ( # noqa: F401 + DiffusionServerBase, + diffusion_server, +) +from sglang.multimodal_gen.test.server.test_server_utils import ( + ServerContext, + get_generate_fn, +) +from sglang.multimodal_gen.test.server.testcase_configs import ( + DiffusionSamplingParams, + DiffusionServerArgs, + DiffusionTestCase, +) +from sglang.test.ci.ci_register import register_xpu_ci + +logger = logging.getLogger(__name__) + +register_xpu_ci(est_time=1800, suite="nightly-xpu-1-gpu", nightly=True) + +XPU_ZIMAGE_CASES = [ + DiffusionTestCase( + "zimage_image_t2i", + DiffusionServerArgs( + model_path="Tongyi-MAI/Z-Image-Turbo", + modality="image", + num_gpus=1, + tp_size=1, + ), + DiffusionSamplingParams( + prompt="Doraemon is eating dorayaki", + output_size="1024x1024", + ), + # XPU has no perf/consistency baseline in + # multimodal_gen/test/server/perf_baselines/. CLIP-score guard below + # is the accuracy check; skip the CUDA-only latency/consistency ones. + run_perf_check=False, + run_consistency_check=False, + run_component_accuracy_check=False, + ), +] + +CLIP_SCORE_THRESHOLD = 0.20 + +ARTIFACT_DIR = os.environ.get( + "SGLANG_DIFFUSION_ARTIFACT_DIR", "/tmp/diffusion-artifacts" +) + + +def _save_image_and_write_summary( + case_id: str, prompt: str, image_bytes: bytes, clip_score: float | None = None +): + ext = "jpg" if image_bytes[:2] == b"\xff\xd8" else "png" + os.makedirs(ARTIFACT_DIR, exist_ok=True) + img_path = os.path.join(ARTIFACT_DIR, f"{case_id}.{ext}") + with open(img_path, "wb") as f: + f.write(image_bytes) + logger.info("Saved image artifact: %s (%d bytes)", img_path, len(image_bytes)) + + summary_file = os.environ.get("GITHUB_STEP_SUMMARY") + if not summary_file: + return + + clip_line = "" + if clip_score is not None: + status = "PASS" if clip_score >= CLIP_SCORE_THRESHOLD else "FAIL" + clip_line = ( + f"| CLIP Score | {clip_score:.4f} " + f"({status}, threshold: {CLIP_SCORE_THRESHOLD}) |\n" + ) + + md = ( + f"### Z-Image-Turbo — `{case_id}`\n\n" + f"| | |\n|---|---|\n" + f"| Prompt | {prompt} |\n" + f"| Size | {len(image_bytes):,} bytes |\n" + f"{clip_line}" + f"| Artifact | `{case_id}.{ext}` (download from Artifacts section above) |\n\n" + ) + + with open(summary_file, "a") as f: + f.write(md) + + +def _compute_clip_score(image_bytes: bytes, prompt: str) -> float | None: + try: + from PIL import Image + from transformers import CLIPModel, CLIPProcessor + + model_name = "openai/clip-vit-base-patch32" + processor = CLIPProcessor.from_pretrained(model_name) + model = CLIPModel.from_pretrained(model_name) + model.eval() + + image = Image.open(io.BytesIO(image_bytes)).convert("RGB") + inputs = processor(text=[prompt], images=image, return_tensors="pt") + + with torch.no_grad(): + outputs = model(**inputs) + score = outputs.logits_per_image.item() / 100.0 + + logger.info("CLIP score for '%s': %.4f", prompt, score) + return score + except Exception as e: + logger.warning("CLIP score computation failed: %s", e) + return None + + +@pytest.mark.skipif( + not (hasattr(torch, "xpu") and torch.xpu.is_available()), + reason="Intel XPU not available (torch.xpu.is_available() returned False)", +) +class TestZImageTurboXPU(DiffusionServerBase): + """Intel XPU nightly test for Z-Image-Turbo text-to-image generation.""" + + @classmethod + def teardown_class(cls): + try: + super().teardown_class() + except AttributeError: + pass + + @pytest.fixture(params=XPU_ZIMAGE_CASES, ids=lambda c: c.id) + def case(self, request) -> DiffusionTestCase: + return request.param + + def test_diffusion_generation( + self, + case: DiffusionTestCase, + diffusion_server: ServerContext, + ): + generate_fn = get_generate_fn( + model_path=case.server_args.model_path, + modality=case.server_args.modality, + sampling_params=case.sampling_params, + ) + + perf_record, content = self.run_and_collect( + diffusion_server, case.id, generate_fn + ) + + self._validate_and_record(case, perf_record) + self._test_v1_models_endpoint(diffusion_server, case) + + prompt = case.sampling_params.prompt or "" + clip_score = _compute_clip_score(content, prompt) + + if clip_score is not None: + logger.info( + "CLIP score: %.4f (threshold: %.2f)", clip_score, CLIP_SCORE_THRESHOLD + ) + assert clip_score >= CLIP_SCORE_THRESHOLD, ( + f"CLIP score {clip_score:.4f} below threshold {CLIP_SCORE_THRESHOLD} " + f"for prompt '{prompt}'" + ) + + _save_image_and_write_summary(case.id, prompt, content, clip_score) + + +if __name__ == "__main__": + import sys + + sys.exit(pytest.main([__file__, "-v"]))