minke.yu/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 12 Packages Projects Releases Wiki Activity

[XPU][NIGHTLY] Add 8 XPU nightly tests, enable 1-gpu suite (#30246)

Browse Source 
Co-authored-by: arathi-hlab <arathi-hlab@users.noreply.github.com>
Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
ashwini rathi
2026-07-20 16:15:50 +08:00
committed by GitHub
co-authored by arathi-hlab Claude Opus 4.8
parent 50c118704a
commit fafa302e41
13 changed files with 828 additions and 36 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/workflows/nightly-test-intel.yml
+81 -7
View File
@@ -35,13 +35,78 @@ concurrency:
jobs:
nightly-xpu-1-gpu:
# Placeholder: no models currently registered to nightly-xpu-1-gpu run on
# intel-bmg-nightly. Add a model test to this suite to re-enable.
if: false
if: github.repository == 'sgl-project/sglang'
runs-on: intel-bmg-nightly
env:
DOCKERHUB_INTEL_USERNAME: ${{ secrets.DOCKERHUB_INTEL_USERNAME }}
DOCKERHUB_INTEL_TOKEN: ${{ secrets.DOCKERHUB_INTEL_TOKEN }}
steps:
- name: Placeholder
run: echo "nightly-xpu-1-gpu has no validated models; skipping."
- name: Reset workspace ownership
run: |
docker run --rm -v "${{ github.workspace }}:/w" busybox:latest \
chown -R "$(id -u):$(id -g)" /w || true
- name: Checkout code
uses: actions/checkout@v4
with:
fetch-depth: 0
ref: ${{ inputs.ref || github.sha }}
- name: Start CI container (pull intel/sglang-dev:latest)
run: |
export HF_TOKEN="$(cat ~/huggingface_token.txt)"
bash scripts/ci/xpu/xpu_ci_start_container.sh
env:
GITHUB_WORKSPACE: ${{ github.workspace }}
- name: HF login + install run_suite extras
timeout-minutes: 10
run: |
docker exec ci_sglang_xpu /opt/venv/bin/python3 -m pip install --no-cache-dir tabulate
docker exec ci_sglang_xpu /opt/venv/bin/python3 -m pip install --no-cache-dir --no-deps xgrammar==0.1.33
docker exec ci_sglang_xpu /opt/venv/bin/python3 -m pip install --no-cache-dir "lm-eval==0.4.9"
docker exec ci_sglang_xpu /opt/venv/bin/python3 -m pip install --no-cache-dir pytest
# Diffusion extras (needed by test_xpu_flux2_dev, test_xpu_zimage_turbo).
# Pins mirror python/pyproject_xpu.toml [project.optional-dependencies.diffusion].
docker exec ci_sglang_xpu /opt/venv/bin/python3 -m pip install --no-cache-dir "sglang[diffusion]"
docker exec ci_sglang_xpu /bin/bash -c '/opt/venv/bin/hf auth login --token ${HF_TOKEN}'
- name: Download olmOCR-bench dataset (for test_deepseek_ocr_2_olmbench)
timeout-minutes: 30
run: |
# PyMuPDF renders the bench PDFs to images; bench_sglang.py errors
# every sample without it.
docker exec ci_sglang_xpu /opt/venv/bin/python3 -m pip install --no-cache-dir pymupdf
docker exec ci_sglang_xpu /bin/bash -c '
/opt/venv/bin/hf download --repo-type dataset allenai/olmOCR-bench \
--local-dir /sglang-checkout/olmOCR-bench'
- name: Nightly Test (1-GPU XPU)
timeout-minutes: 240
run: |
touch github_summary.md
docker exec ci_sglang_xpu bash -c "
source /opt/venv/bin/activate &&
cd /sglang-checkout/test &&
OLMOCR_BENCH_DIR=/sglang-checkout/olmOCR-bench/bench_data \
GITHUB_STEP_SUMMARY=/sglang-checkout/github_summary.md \
python3 run_suite.py --hw xpu --suite nightly-xpu-1-gpu --nightly --timeout-per-file 7200 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }}
" || TEST_EXIT_CODE=$?
echo "$(<github_summary.md)" >> $GITHUB_STEP_SUMMARY || true
exit ${TEST_EXIT_CODE:-0}
- name: Cleanup container
if: always()
run: |
docker run --rm -v "${{ github.workspace }}:/w" busybox:latest \
chown -R "$(id -u):$(id -g)" /w || true
rm -rf test/result.jsonl test/results test/.pytest_cache .pytest_cache || true
find . -type d -name "__pycache__" -prune -exec rm -rf {} + || true
find . -type f -name "*.pyc" -delete || true
docker rm -f ci_sglang_xpu || true
if [[ -n "${CI_SGLANG_XPU_IMAGE:-}" ]]; then
docker rmi -f "${CI_SGLANG_XPU_IMAGE}" || true
fi
nightly-xpu-2-gpu:
if: github.repository == 'sgl-project/sglang'
@@ -74,10 +139,14 @@ jobs:
docker exec ci_sglang_xpu /opt/venv/bin/python3 -m pip install --no-cache-dir tabulate
docker exec ci_sglang_xpu /opt/venv/bin/python3 -m pip install --no-cache-dir --no-deps xgrammar==0.1.33
docker exec ci_sglang_xpu /opt/venv/bin/python3 -m pip install --no-cache-dir "lm-eval==0.4.9"
docker exec ci_sglang_xpu /opt/venv/bin/python3 -m pip install --no-cache-dir pytest
# Diffusion extras (needed by test_xpu_flux2_dev, test_xpu_zimage_turbo).
# Pins mirror python/pyproject_xpu.toml [project.optional-dependencies.diffusion].
docker exec ci_sglang_xpu /opt/venv/bin/python3 -m pip install --no-cache-dir "sglang[diffusion]"
docker exec ci_sglang_xpu /bin/bash -c '/opt/venv/bin/hf auth login --token ${HF_TOKEN}'
- name: Nightly Test (2-GPU XPU)
timeout-minutes: 60
timeout-minutes: 240
run: |
touch github_summary.md
docker exec ci_sglang_xpu bash -c "
@@ -133,10 +202,14 @@ jobs:
docker exec ci_sglang_xpu /opt/venv/bin/python3 -m pip install --no-cache-dir tabulate
docker exec ci_sglang_xpu /opt/venv/bin/python3 -m pip install --no-cache-dir --no-deps xgrammar==0.1.33
docker exec ci_sglang_xpu /opt/venv/bin/python3 -m pip install --no-cache-dir "lm-eval==0.4.9"
docker exec ci_sglang_xpu /opt/venv/bin/python3 -m pip install --no-cache-dir pytest
# Diffusion extras (needed by test_xpu_flux2_dev, test_xpu_zimage_turbo).
# Pins mirror python/pyproject_xpu.toml [project.optional-dependencies.diffusion].
docker exec ci_sglang_xpu /opt/venv/bin/python3 -m pip install --no-cache-dir "sglang[diffusion]"
docker exec ci_sglang_xpu /bin/bash -c '/opt/venv/bin/hf auth login --token ${HF_TOKEN}'
- name: Nightly Test (4-GPU XPU)
timeout-minutes: 120
timeout-minutes: 480
run: |
touch github_summary.md
docker exec ci_sglang_xpu bash -c "
@@ -164,6 +237,7 @@ jobs:
check-all-jobs:
if: always() && (github.repository == 'sgl-project/sglang' || github.event_name == 'workflow_dispatch')
needs:
- nightly-xpu-1-gpu
- nightly-xpu-2-gpu
- nightly-xpu-4-gpu
runs-on: ubuntu-latest
python/sglang/test/xpu/simple_eval_gsm8k_xpu_mixin.py
+7 -16
View File
@@ -1,9 +1,8 @@
"""simple-evals GSM8K accuracy mixin for Intel XPU nightly tests.
Mirrors the AMD/NVIDIA nightly flow (``test_gsm8k_eval_amd.py`` /
``test_text_models_gsm8k_eval.py``): launch an SGLang server with XPU
flags, then call ``sglang.test.run_eval`` with ``eval_name="gsm8k"`` so
the same ``simple_eval_gsm8k.GSM8KEval`` evaluator scores every backend.
Launches an SGLang server with XPU flags, then calls ``sglang.test.run_eval``
with ``eval_name="gsm8k"`` so the ``simple_eval_gsm8k.GSM8KEval`` evaluator
scores the run.
Subclasses set ``model``, ``tp_size``, ``accuracy``, and may override
``other_args`` / ``env`` / ``num_examples`` / ``num_threads``.
@@ -46,20 +45,12 @@ class SimpleEvalGSM8KXPUMixin(ABC):
env: dict | None = None
server_cmd: str = ""
# 200 questions matches the limit used by the XPU 70B lm-eval YAML and
# fits inside run_suite's per-file timeout when num_threads=1 keeps
# throughput low. Subclasses on cheaper-per-token hardware (TP=1, no
# Level Zero wedge) can raise this or set None for the full 1319-question
# GSM8K test set, matching the AMD/NVIDIA nightly defaults.
# Subset that fits run_suite's per-file timeout; set None for the full set.
num_examples: int | None = 200
# Single-stream eval: intel_xpu attention at TP>=2 wedges the Level Zero
# driver in ur_command_list_manager::appendUSMMemcpy on concurrent prefill.
# Subclasses on hardware that handles parallel prefill cleanly may bump.
# Single-stream: intel_xpu attention at TP>=2 wedges the Level Zero driver
# on concurrent prefill. Subclasses may bump on hardware that handles it.
num_threads: int = 1
# Short generations reduce the rate of prefill->decode->prefill handoffs,
# which is what trips the same Level Zero wedge on TP>=2 (observed at the
# default 2048; 512 matches the original few_shot_gsm8k limit and is still
# enough for GSM8K CoT answers).
# Short generations reduce prefill->decode handoffs that trip the same wedge.
max_tokens: int = 512
@classmethod
scripts/ci/xpu/xpu_ci_start_container.sh
+4
View File
@@ -98,6 +98,9 @@ elif [[ -r "${HF_TOKEN_FILE}" ]]; then
fi
echo "Launching container: ${CONTAINER_NAME} from ${IMAGE}"
# SGLANG_SERVER_LAUNCH_TIMEOUT=36000 matches /data/pgirijal/scripts/setup_upstream_env.sh:
# 4-GPU MoE loads (Qwen3.5-35B-A3B, gemma-4-26B-A4B, ...) on Arc Pro B60 can
# take >1h from a cold HF cache, so give sglang server startup a 10h ceiling.
docker run -dt \
--shm-size 8g \
--group-add 992 \
@@ -108,6 +111,7 @@ docker run -dt \
-v "${HOME}/.cache/huggingface:/root/.cache/huggingface" \
-v "${GITHUB_WORKSPACE:-$PWD}:/sglang-checkout" \
-e HF_TOKEN="${HF_TOKEN_VALUE}" \
-e SGLANG_SERVER_LAUNCH_TIMEOUT=36000 \
--name "${CONTAINER_NAME}" \
"${IMAGE}"
test/registered/xpu/llm_models/test_xpu_gemma_4_26b_a4b.py
+42
View File
@@ -0,0 +1,42 @@
"""gemma-4-26B-A4B GSM8K accuracy on Intel XPU (TP=4).
Scored by ``simple_eval_gsm8k.GSM8KEval``.
"""
import unittest
import torch
from sglang.test.ci.ci_register import register_xpu_ci
from sglang.test.test_utils import CustomTestCase
from sglang.test.xpu.simple_eval_gsm8k_xpu_mixin import SimpleEvalGSM8KXPUMixin
register_xpu_ci(est_time=2400, suite="nightly-xpu-4-gpu", nightly=True)
@unittest.skipUnless(
torch.xpu.is_available(),
"Intel XPU not available (torch.xpu.is_available() returned False)",
)
class TestGemma4_26BA4BXPU(SimpleEvalGSM8KXPUMixin, CustomTestCase):
model = "google/gemma-4-26B-A4B-it"
tp_size = 4
accuracy = 0.90
timeout_for_server_launch = 3600
env = {"SGLANG_USE_SGL_XPU": "1"}
# Gemma-4 hybrid-attention kernels crash under chunked prefill on XPU.
other_args = SimpleEvalGSM8KXPUMixin.other_args + [
"--page-size",
"64",
"--max-total-tokens",
"65536",
"--mem-fraction-static",
"0.9",
"--chunked-prefill-size",
"-1",
]
if __name__ == "__main__":
unittest.main()
test/registered/xpu/llm_models/test_xpu_llama_3_1_8b.py
+1 -2
View File
@@ -3,8 +3,7 @@
TP=4 wedges the Level Zero driver during the first prefill batch on Arc/BMG;
TP=2 runs cleanly with the same model and serves at ~18 tok/s.
Scored by ``simple_eval_gsm8k.GSM8KEval`` (the same evaluator AMD and
NVIDIA nightlies use); threshold mirrors theirs.
Scored by ``simple_eval_gsm8k.GSM8KEval``.
"""
import unittest
test/registered/xpu/llm_models/test_xpu_nemotron_3_nano_30b_a3b.py
+59
View File
@@ -0,0 +1,59 @@
"""NVIDIA-Nemotron-3-Nano-30B-A3B GSM8K accuracy on Intel XPU (TP=4).
Scored by ``simple_eval_gsm8k.GSM8KEval``.
"""
import unittest
import torch
from sglang.test.ci.ci_register import register_xpu_ci
from sglang.test.test_utils import CustomTestCase
from sglang.test.xpu.simple_eval_gsm8k_xpu_mixin import SimpleEvalGSM8KXPUMixin
register_xpu_ci(est_time=2400, suite="nightly-xpu-4-gpu", nightly=True)
@unittest.skipUnless(
torch.xpu.is_available(),
"Intel XPU not available (torch.xpu.is_available() returned False)",
)
class TestNemotron3Nano30BA3BXPU(SimpleEvalGSM8KXPUMixin, CustomTestCase):
model = "nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-BF16"
tp_size = 4
accuracy = 0.72
timeout_for_server_launch = 3600
# Generation cap for the GSM8K eval (mixin default is 512).
max_tokens = 8192
# Client-side eval concurrency (mixin default is 1).
num_threads = 4
env = {"SGLANG_USE_SGL_XPU": "1"}
# Hybrid-mamba layout needs --model-impl sglang, a fixed page size, and
# the nemotron_3 reasoning / qwen3_coder tool-call parsers.
other_args = SimpleEvalGSM8KXPUMixin.other_args + [
"--max-total-tokens",
"65536",
"--mem-fraction-static",
"0.85",
"--context-length",
"16384",
"--page-size",
"64",
"--chunked-prefill-size",
"1024",
"--max-running-requests",
"8",
"--watchdog-timeout",
"1200",
"--model-impl",
"sglang",
"--tool-call-parser",
"qwen3_coder",
"--reasoning-parser",
"nemotron_3",
]
if __name__ == "__main__":
unittest.main()
test/registered/xpu/llm_models/test_xpu_qwen3_32b.py → test/registered/xpu/llm_models/test_xpu_qwen3_30b_a3b.py
+11 -10
View File
@@ -1,7 +1,6 @@
"""Qwen3-32B GSM8K accuracy on Intel XPU (TP=4).
"""Qwen3-30B-A3B GSM8K accuracy on Intel XPU (TP=4).
Scored by ``simple_eval_gsm8k.GSM8KEval`` (the same evaluator AMD and
NVIDIA nightlies use).
Scored by ``simple_eval_gsm8k.GSM8KEval``.
"""
import unittest
@@ -12,21 +11,23 @@ from sglang.test.ci.ci_register import register_xpu_ci
from sglang.test.test_utils import CustomTestCase
from sglang.test.xpu.simple_eval_gsm8k_xpu_mixin import SimpleEvalGSM8KXPUMixin
register_xpu_ci(est_time=1800, suite="nightly-xpu-4-gpu", nightly=True)
register_xpu_ci(est_time=2400, suite="nightly-xpu-4-gpu", nightly=True)
@unittest.skipUnless(
torch.xpu.is_available(),
"Intel XPU not available (torch.xpu.is_available() returned False)",
)
class TestQwen3_32BXPU(SimpleEvalGSM8KXPUMixin, CustomTestCase):
model = "Qwen/Qwen3-32B"
class TestQwen3_30BA3BXPU(SimpleEvalGSM8KXPUMixin, CustomTestCase):
model = "Qwen/Qwen3-30B-A3B"
tp_size = 4
accuracy = 0.85
# 64GB BF16 weights split across 4 ranks take ~9 min to load on Intel
# Arc Pro B60; the default 600s timeout fires mid-startup. Mirror the
# XPU 70B test's 1-hour budget.
accuracy = 0.90
timeout_for_server_launch = 3600
# SGL XPU MoE kernels gate on this env var.
env = {"SGLANG_USE_SGL_XPU": "1"}
num_examples = 50
num_threads = 4
max_tokens = 8192
other_args = SimpleEvalGSM8KXPUMixin.other_args + [
"--max-total-tokens",
test/registered/xpu/llm_models/test_xpu_qwen3_5_35b_a3b.py
+43
View File
@@ -0,0 +1,43 @@
"""Qwen3.5-35B-A3B GSM8K accuracy on Intel XPU (TP=4).
Scored by ``simple_eval_gsm8k.GSM8KEval``.
"""
import unittest
import torch
from sglang.test.ci.ci_register import register_xpu_ci
from sglang.test.test_utils import CustomTestCase
from sglang.test.xpu.simple_eval_gsm8k_xpu_mixin import SimpleEvalGSM8KXPUMixin
register_xpu_ci(est_time=2400, suite="nightly-xpu-4-gpu", nightly=True)
@unittest.skipUnless(
torch.xpu.is_available(),
"Intel XPU not available (torch.xpu.is_available() returned False)",
)
class TestQwen3_5_35BA3BXPU(SimpleEvalGSM8KXPUMixin, CustomTestCase):
model = "Qwen/Qwen3.5-35B-A3B"
tp_size = 4
accuracy = 0.90
timeout_for_server_launch = 3600
# SGL XPU MoE kernels gate on this env var.
env = {"SGLANG_USE_SGL_XPU": "1"}
num_examples = 50
num_threads = 4
max_tokens = 8192
other_args = SimpleEvalGSM8KXPUMixin.other_args + [
"--page-size",
"128",
"--max-total-tokens",
"65536",
"--mem-fraction-static",
"0.85",
]
if __name__ == "__main__":
unittest.main()
test/registered/xpu/llm_models/test_xpu_qwen3_5_9b.py
+41
View File
@@ -0,0 +1,41 @@
"""Qwen3.5-9B GSM8K accuracy on Intel XPU (TP=4).
Scored by ``simple_eval_gsm8k.GSM8KEval``.
"""
import unittest
import torch
from sglang.test.ci.ci_register import register_xpu_ci
from sglang.test.test_utils import CustomTestCase
from sglang.test.xpu.simple_eval_gsm8k_xpu_mixin import SimpleEvalGSM8KXPUMixin
register_xpu_ci(est_time=2400, suite="nightly-xpu-4-gpu", nightly=True)
@unittest.skipUnless(
torch.xpu.is_available(),
"Intel XPU not available (torch.xpu.is_available() returned False)",
)
class TestQwen3_5_9BXPU(SimpleEvalGSM8KXPUMixin, CustomTestCase):
model = "Qwen/Qwen3.5-9B"
tp_size = 4
accuracy = 0.90
# max_tokens=8192 lets the GSM8K CoT complete under num_threads=4.
num_examples = 50
num_threads = 4
max_tokens = 8192
other_args = SimpleEvalGSM8KXPUMixin.other_args + [
"--page-size",
"128",
"--max-total-tokens",
"65536",
"--mem-fraction-static",
"0.85",
]
if __name__ == "__main__":
unittest.main()
test/registered/xpu/test_deepseek_ocr_2_olmbench.py
+171
View File
@@ -0,0 +1,171 @@
"""DeepSeek-OCR-2 olmOCR-bench accuracy on Intel XPU (1-GPU nightly).
Launches the server with the OCR serving config, runs the full olmOCR-bench
via ``benchmark/ocr/bench_sglang.py``, asserts the aggregate score >= 0.80,
and writes the per-split breakdown to the GitHub step summary.
The olmOCR-bench dataset is downloaded by the nightly workflow step and its
location passed via the OLMOCR_BENCH_DIR env var.
"""
import json
import os
import subprocess
import sys
import unittest
from pathlib import Path
from urllib.parse import urlparse
import torch
from sglang.srt.utils import kill_process_tree
from sglang.test.ci.ci_register import register_xpu_ci
from sglang.test.test_utils import (
DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
DEFAULT_URL_FOR_TEST,
CustomTestCase,
popen_launch_server,
write_github_step_summary,
)
register_xpu_ci(est_time=7200, suite="nightly-xpu-1-gpu", nightly=True)
# Repo root: test/registered/xpu/<this file> -> parents[3].
_REPO_ROOT = Path(__file__).resolve().parents[3]
# Default matches the workflow's --local-dir; override with OLMOCR_BENCH_DIR.
_DEFAULT_BENCH_DIR = _REPO_ROOT / "olmOCR-bench" / "bench_data"
@unittest.skipUnless(
torch.xpu.is_available(),
"Intel XPU not available (torch.xpu.is_available() returned False)",
)
class TestDeepSeekOCR2OlmBenchXPU(CustomTestCase):
model = "deepseek-ai/DeepSeek-OCR-2"
# Aggregate score (total_passed / total_tests) must clear this.
accuracy = 0.80
timeout_for_server_launch = DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH
# Full bench by default; overridable via env for quick sanity runs.
concurrency = int(os.environ.get("OLMOCR_BENCH_CONCURRENCY", "26"))
split = os.environ.get("OLMOCR_BENCH_SPLIT", "all")
max_samples = int(os.environ.get("OLMOCR_BENCH_MAX_SAMPLES", "-1"))
other_args = [
"--dtype",
"bfloat16",
"--trust-remote-code",
"--disable-radix-cache",
"--attention-backend",
"intel_xpu",
"--mem-fraction-static",
"0.65",
"--max-running-requests",
"26",
"--enable-mixed-chunk",
"--chunked-prefill-size",
"8192",
"--disable-cuda-graph",
]
env = {"SGLANG_USE_SGL_XPU": "1"}
@classmethod
def setUpClass(cls):
cls.base_url = DEFAULT_URL_FOR_TEST
cls.bench_dir = Path(os.environ.get("OLMOCR_BENCH_DIR", _DEFAULT_BENCH_DIR))
cls.output_dir = _REPO_ROOT / "ocr_bench_results"
env = {**os.environ, **cls.env}
try:
cls.process = popen_launch_server(
cls.model,
cls.base_url,
timeout=cls.timeout_for_server_launch,
other_args=list(cls.other_args),
env=env,
)
except Exception as e:
write_github_step_summary(f"Failed to launch server for {cls.model}: {e}")
raise AssertionError(f"Test failed for {cls.model}: {e}")
@classmethod
def tearDownClass(cls):
if getattr(cls, "process", None):
kill_process_tree(cls.process.pid)
def test_olmocr_bench(self):
if not self.bench_dir.exists():
self.fail(
f"olmOCR-bench data not found at {self.bench_dir}. Download it first:\n"
" hf download --repo-type dataset allenai/olmOCR-bench "
"--local-dir ./olmOCR-bench"
)
port = urlparse(self.base_url).port
cmd = [
sys.executable,
str(_REPO_ROOT / "benchmark" / "ocr" / "bench_sglang.py"),
"--port",
str(port),
"--split",
self.split,
"--concurrency",
str(self.concurrency),
"--model",
self.model,
*(["--max-samples", str(self.max_samples)] if self.max_samples > 0 else []),
"--bench-dir",
str(self.bench_dir),
"--output-dir",
str(self.output_dir),
]
try:
subprocess.run(cmd, check=True, cwd=str(_REPO_ROOT))
except subprocess.CalledProcessError as e:
self.fail(f"olmOCR-bench run failed for {self.model}: {e}")
summary_path = self.output_dir / "summary.json"
if not summary_path.exists():
self.fail(f"Benchmark produced no summary at {summary_path}")
with open(summary_path, encoding="utf-8") as f:
results = json.load(f)
total_tests = sum(r.get("total_tests", 0) for r in results.values())
total_passed = sum(r.get("total_passed", 0) for r in results.values())
total_errored = sum(r.get("error_samples", 0) for r in results.values())
score = total_passed / total_tests if total_tests else 0.0
lines = [
f"## DeepSeek-OCR-2 olmOCR-bench (XPU, concurrency {self.concurrency})",
"",
"| Split | Tests | Passed | Score | Errored |",
"| --- | ---: | ---: | ---: | ---: |",
]
for split, r in results.items():
lines.append(
f"| {split} | {r.get('total_tests', 0)} | "
f"{r.get('total_passed', 0)} | {r.get('overall_score', 0.0):.1f}% | "
f"{r.get('error_samples', 0)} |"
)
lines.append(
f"| **TOTAL** | {total_tests} | {total_passed} | "
f"**{100.0 * score:.1f}%** | {total_errored} |"
)
write_github_step_summary("\n".join(lines) + "\n")
# Guard against a silent empty run before comparing the score.
self.assertGreater(
total_tests, 0, f"olmOCR-bench scored 0 tests for {self.model}"
)
self.assertGreaterEqual(
score,
self.accuracy,
f"olmOCR-bench aggregate for {self.model} is {100.0 * score:.1f}%, "
f"below the {100.0 * self.accuracy:.0f}% threshold "
f"({total_errored} samples errored)",
)
if __name__ == "__main__":
unittest.main()
test/registered/xpu/test_triton_attention_backend.py
+1 -1
View File
@@ -13,7 +13,7 @@ from sglang.test.test_utils import (
run_bench_serving,
)
register_xpu_ci(est_time=600, suite="stage-b-test-1-gpu-xpu")
register_xpu_ci(est_time=600, suite="nightly-xpu-1-gpu", nightly=True)
def triton_attention_benchmark(extra_args=None, mem_fraction_static="0.84"):
test/registered/xpu/test_xpu_flux2_dev.py
+188
View File
@@ -0,0 +1,188 @@
"""FLUX.2-dev text-to-image on Intel XPU (4-GPU nightly).
Mirrors ``test/registered/amd/test_zimage_turbo.py`` but targets FLUX.2-dev
with ``num_gpus=4`` and registers to the XPU 4-GPU nightly suite. The
diffusion server harness is device-agnostic; XPU dispatch is picked up by
``current_platform`` inside multimodal_gen at server launch.
"""
from __future__ import annotations
import io
import logging
import os
import pytest
import torch
from sglang.multimodal_gen.test.server.test_server_common import ( # noqa: F401
DiffusionServerBase,
diffusion_server,
)
from sglang.multimodal_gen.test.server.test_server_utils import (
ServerContext,
get_generate_fn,
)
from sglang.multimodal_gen.test.server.testcase_configs import (
DiffusionSamplingParams,
DiffusionServerArgs,
DiffusionTestCase,
)
from sglang.test.ci.ci_register import register_xpu_ci
logger = logging.getLogger(__name__)
register_xpu_ci(est_time=3600, suite="nightly-xpu-4-gpu", nightly=True)
XPU_FLUX2_CASES = [
DiffusionTestCase(
"flux2_dev_image_t2i",
DiffusionServerArgs(
model_path="black-forest-labs/FLUX.2-dev",
modality="image",
num_gpus=4,
tp_size=4,
dit_layerwise_offload=True,
extras=[
"--dit-precision",
"bf16",
"--vae-precision",
"bf16",
"--text-encoder-precisions",
"bf16",
],
),
DiffusionSamplingParams(
prompt="A curious raccoon in a top hat, oil painting",
output_size="1024x1024",
),
# XPU has no perf/consistency baseline in
# multimodal_gen/test/server/perf_baselines/. CLIP-score guard below
# is the accuracy check; skip the CUDA-only latency/consistency ones.
run_perf_check=False,
run_consistency_check=False,
run_component_accuracy_check=False,
),
]
CLIP_SCORE_THRESHOLD = 0.20
ARTIFACT_DIR = os.environ.get(
"SGLANG_DIFFUSION_ARTIFACT_DIR", "/tmp/diffusion-artifacts"
)
def _save_image_and_write_summary(
case_id: str, prompt: str, image_bytes: bytes, clip_score: float | None = None
):
ext = "jpg" if image_bytes[:2] == b"\xff\xd8" else "png"
os.makedirs(ARTIFACT_DIR, exist_ok=True)
img_path = os.path.join(ARTIFACT_DIR, f"{case_id}.{ext}")
with open(img_path, "wb") as f:
f.write(image_bytes)
logger.info("Saved image artifact: %s (%d bytes)", img_path, len(image_bytes))
summary_file = os.environ.get("GITHUB_STEP_SUMMARY")
if not summary_file:
return
clip_line = ""
if clip_score is not None:
status = "PASS" if clip_score >= CLIP_SCORE_THRESHOLD else "FAIL"
clip_line = (
f"| CLIP Score | {clip_score:.4f} "
f"({status}, threshold: {CLIP_SCORE_THRESHOLD}) |\n"
)
md = (
f"### FLUX.2-dev — `{case_id}`\n\n"
f"| | |\n|---|---|\n"
f"| Prompt | {prompt} |\n"
f"| Size | {len(image_bytes):,} bytes |\n"
f"{clip_line}"
f"| Artifact | `{case_id}.{ext}` (download from Artifacts section above) |\n\n"
)
with open(summary_file, "a") as f:
f.write(md)
def _compute_clip_score(image_bytes: bytes, prompt: str) -> float | None:
try:
from PIL import Image
from transformers import CLIPModel, CLIPProcessor
model_name = "openai/clip-vit-base-patch32"
processor = CLIPProcessor.from_pretrained(model_name)
model = CLIPModel.from_pretrained(model_name)
model.eval()
image = Image.open(io.BytesIO(image_bytes)).convert("RGB")
inputs = processor(text=[prompt], images=image, return_tensors="pt")
with torch.no_grad():
outputs = model(**inputs)
score = outputs.logits_per_image.item() / 100.0
logger.info("CLIP score for '%s': %.4f", prompt, score)
return score
except Exception as e:
logger.warning("CLIP score computation failed: %s", e)
return None
@pytest.mark.skipif(
not (hasattr(torch, "xpu") and torch.xpu.is_available()),
reason="Intel XPU not available (torch.xpu.is_available() returned False)",
)
class TestFlux2DevXPU(DiffusionServerBase):
"""Intel XPU nightly test for FLUX.2-dev text-to-image generation."""
@classmethod
def teardown_class(cls):
try:
super().teardown_class()
except AttributeError:
pass
@pytest.fixture(params=XPU_FLUX2_CASES, ids=lambda c: c.id)
def case(self, request) -> DiffusionTestCase:
return request.param
def test_diffusion_generation(
self,
case: DiffusionTestCase,
diffusion_server: ServerContext,
):
generate_fn = get_generate_fn(
model_path=case.server_args.model_path,
modality=case.server_args.modality,
sampling_params=case.sampling_params,
)
perf_record, content = self.run_and_collect(
diffusion_server, case.id, generate_fn
)
self._validate_and_record(case, perf_record)
self._test_v1_models_endpoint(diffusion_server, case)
prompt = case.sampling_params.prompt or ""
clip_score = _compute_clip_score(content, prompt)
if clip_score is not None:
logger.info(
"CLIP score: %.4f (threshold: %.2f)", clip_score, CLIP_SCORE_THRESHOLD
)
assert clip_score >= CLIP_SCORE_THRESHOLD, (
f"CLIP score {clip_score:.4f} below threshold {CLIP_SCORE_THRESHOLD} "
f"for prompt '{prompt}'"
)
_save_image_and_write_summary(case.id, prompt, content, clip_score)
if __name__ == "__main__":
import sys
sys.exit(pytest.main([__file__, "-v"]))
test/registered/xpu/test_xpu_zimage_turbo.py
+179
View File
@@ -0,0 +1,179 @@
"""Z-Image-Turbo text-to-image on Intel XPU (1-GPU nightly).
Mirrors ``test/registered/amd/test_zimage_turbo.py`` but registers to the
XPU 1-GPU nightly suite. The diffusion server harness is device-agnostic;
XPU dispatch is picked up by ``current_platform`` inside multimodal_gen at
server launch.
"""
from __future__ import annotations
import io
import logging
import os
import pytest
import torch
from sglang.multimodal_gen.test.server.test_server_common import ( # noqa: F401
DiffusionServerBase,
diffusion_server,
)
from sglang.multimodal_gen.test.server.test_server_utils import (
ServerContext,
get_generate_fn,
)
from sglang.multimodal_gen.test.server.testcase_configs import (
DiffusionSamplingParams,
DiffusionServerArgs,
DiffusionTestCase,
)
from sglang.test.ci.ci_register import register_xpu_ci
logger = logging.getLogger(__name__)
register_xpu_ci(est_time=1800, suite="nightly-xpu-1-gpu", nightly=True)
XPU_ZIMAGE_CASES = [
DiffusionTestCase(
"zimage_image_t2i",
DiffusionServerArgs(
model_path="Tongyi-MAI/Z-Image-Turbo",
modality="image",
num_gpus=1,
tp_size=1,
),
DiffusionSamplingParams(
prompt="Doraemon is eating dorayaki",
output_size="1024x1024",
),
# XPU has no perf/consistency baseline in
# multimodal_gen/test/server/perf_baselines/. CLIP-score guard below
# is the accuracy check; skip the CUDA-only latency/consistency ones.
run_perf_check=False,
run_consistency_check=False,
run_component_accuracy_check=False,
),
]
CLIP_SCORE_THRESHOLD = 0.20
ARTIFACT_DIR = os.environ.get(
"SGLANG_DIFFUSION_ARTIFACT_DIR", "/tmp/diffusion-artifacts"
)
def _save_image_and_write_summary(
case_id: str, prompt: str, image_bytes: bytes, clip_score: float | None = None
):
ext = "jpg" if image_bytes[:2] == b"\xff\xd8" else "png"
os.makedirs(ARTIFACT_DIR, exist_ok=True)
img_path = os.path.join(ARTIFACT_DIR, f"{case_id}.{ext}")
with open(img_path, "wb") as f:
f.write(image_bytes)
logger.info("Saved image artifact: %s (%d bytes)", img_path, len(image_bytes))
summary_file = os.environ.get("GITHUB_STEP_SUMMARY")
if not summary_file:
return
clip_line = ""
if clip_score is not None:
status = "PASS" if clip_score >= CLIP_SCORE_THRESHOLD else "FAIL"
clip_line = (
f"| CLIP Score | {clip_score:.4f} "
f"({status}, threshold: {CLIP_SCORE_THRESHOLD}) |\n"
)
md = (
f"### Z-Image-Turbo — `{case_id}`\n\n"
f"| | |\n|---|---|\n"
f"| Prompt | {prompt} |\n"
f"| Size | {len(image_bytes):,} bytes |\n"
f"{clip_line}"
f"| Artifact | `{case_id}.{ext}` (download from Artifacts section above) |\n\n"
)
with open(summary_file, "a") as f:
f.write(md)
def _compute_clip_score(image_bytes: bytes, prompt: str) -> float | None:
try:
from PIL import Image
from transformers import CLIPModel, CLIPProcessor
model_name = "openai/clip-vit-base-patch32"
processor = CLIPProcessor.from_pretrained(model_name)
model = CLIPModel.from_pretrained(model_name)
model.eval()
image = Image.open(io.BytesIO(image_bytes)).convert("RGB")
inputs = processor(text=[prompt], images=image, return_tensors="pt")
with torch.no_grad():
outputs = model(**inputs)
score = outputs.logits_per_image.item() / 100.0
logger.info("CLIP score for '%s': %.4f", prompt, score)
return score
except Exception as e:
logger.warning("CLIP score computation failed: %s", e)
return None
@pytest.mark.skipif(
not (hasattr(torch, "xpu") and torch.xpu.is_available()),
reason="Intel XPU not available (torch.xpu.is_available() returned False)",
)
class TestZImageTurboXPU(DiffusionServerBase):
"""Intel XPU nightly test for Z-Image-Turbo text-to-image generation."""
@classmethod
def teardown_class(cls):
try:
super().teardown_class()
except AttributeError:
pass
@pytest.fixture(params=XPU_ZIMAGE_CASES, ids=lambda c: c.id)
def case(self, request) -> DiffusionTestCase:
return request.param
def test_diffusion_generation(
self,
case: DiffusionTestCase,
diffusion_server: ServerContext,
):
generate_fn = get_generate_fn(
model_path=case.server_args.model_path,
modality=case.server_args.modality,
sampling_params=case.sampling_params,
)
perf_record, content = self.run_and_collect(
diffusion_server, case.id, generate_fn
)
self._validate_and_record(case, perf_record)
self._test_v1_models_endpoint(diffusion_server, case)
prompt = case.sampling_params.prompt or ""
clip_score = _compute_clip_score(content, prompt)
if clip_score is not None:
logger.info(
"CLIP score: %.4f (threshold: %.2f)", clip_score, CLIP_SCORE_THRESHOLD
)
assert clip_score >= CLIP_SCORE_THRESHOLD, (
f"CLIP score {clip_score:.4f} below threshold {CLIP_SCORE_THRESHOLD} "
f"for prompt '{prompt}'"
)
_save_image_and_write_summary(case.id, prompt, content, clip_score)
if __name__ == "__main__":
import sys
sys.exit(pytest.main([__file__, "-v"]))