[XPU] Add registry mechanism for XPU CI tests (#25405)

Co-authored-by: Ma Mingfei <mingfei.ma@intel.com>

.github/workflows/pr-test-xpu.yml

@@ -68,7 +68,8 @@ jobs:
     uses: ./.github/workflows/pr-gate.yml
     secrets: inherit
 
-  build-and-test:
+  # ==================== Stage A ==================== #
+  stage-a-test-1-gpu-xpu:
     needs: [check-changes, pr-gate]
     if: needs.check-changes.outputs.main_package == 'true'
     runs-on: intel-bmg
@@ -108,16 +109,80 @@ jobs:
         run: |
           cid="${{ steps.start_container.outputs.container_id }}"
           docker exec "$cid" /home/sdp/miniforge3/envs/py3.12/bin/python3 -m pip install --upgrade pip
-          docker exec "$cid" /home/sdp/miniforge3/envs/py3.12/bin/python3 -m pip install pytest expecttest ray huggingface_hub
+          docker exec "$cid" /home/sdp/miniforge3/envs/py3.12/bin/python3 -m pip install pytest expecttest ray huggingface_hub tabulate
           docker exec "$cid" /home/sdp/miniforge3/envs/py3.12/bin/python3 -m pip uninstall -y flashinfer-python
           docker exec "$cid" /bin/bash -c '/home/sdp/miniforge3/envs/py3.12/bin/hf auth login --token ${HF_TOKEN} '
 
+      - name: Run stage-a tests
+        timeout-minutes: 30
+        run: |
+          cid="${{ steps.start_container.outputs.container_id }}"
+          docker exec "$cid" bash -c "source /home/sdp/miniforge3/bin/activate && conda activate py3.12 && cd /home/sdp/sglang/test && python3 run_suite.py --hw xpu --suite stage-a-test-1-gpu-xpu"
 
-      - name: Run E2E Bfloat16 tests
+      - name: Cleanup container
+        if: always()
+        run: |
+          cid="${{ steps.start_container.outputs.container_id }}"
+          docker rm -f "$cid" || true
+
+  # ==================== Wait for Stage A ==================== #
+  wait-for-stage-a:
+    needs: [stage-a-test-1-gpu-xpu]
+    runs-on: ubuntu-latest
+    steps:
+      - run: echo "stage-a passed"
+
+  # ==================== Stage B ==================== #
+  stage-b-test-1-gpu-xpu:
+    needs: [check-changes, pr-gate, wait-for-stage-a]
+    if: needs.check-changes.outputs.main_package == 'true'
+    runs-on: intel-bmg
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+          ref: ${{ inputs.ref || github.ref }}
+
+      - name: Build Docker image
+        run: |
+          PR_REPO=${{ github.event.pull_request.head.repo.clone_url }}
+          PR_HEAD_REF=${{ github.head_ref }}
+          docker build \
+            ${PR_REPO:+--build-arg SG_LANG_REPO=$PR_REPO} \
+            ${PR_HEAD_REF:+--build-arg SG_LANG_BRANCH=$PR_HEAD_REF} \
+            --no-cache --progress=plain -f docker/xpu.Dockerfile -t xpu_sglang_main:bmg .
+
+      - name: Run container
+        id: start_container
+        run: |
+          container_id=$(docker run -dt \
+            --group-add 992 \
+            --group-add $(getent group video | cut -d: -f3) \
+            --group-add $(getent group render | cut -d: -f3) \
+            -v $HOME/.cache/huggingface:/root/.cache/huggingface \
+            --device /dev/dri \
+            -v /dev/dri/by-path:/dev/dri/by-path \
+            -e HF_TOKEN="$(cat ~/huggingface_token.txt)" \
+            xpu_sglang_main:bmg)
+          echo "Started container: $container_id"
+          echo "container_id=$container_id" >> "$GITHUB_OUTPUT"
+
+      - name: Install Dependency
         timeout-minutes: 20
         run: |
           cid="${{ steps.start_container.outputs.container_id }}"
-          docker exec "$cid" bash -c "source /home/sdp/miniforge3/bin/activate && conda activate py3.12 && cd /home/sdp/sglang/test/srt && python3 run_suite.py --suite per-commit-xpu"
+          docker exec "$cid" /home/sdp/miniforge3/envs/py3.12/bin/python3 -m pip install --upgrade pip
+          docker exec "$cid" /home/sdp/miniforge3/envs/py3.12/bin/python3 -m pip install pytest expecttest ray huggingface_hub tabulate
+          docker exec "$cid" /home/sdp/miniforge3/envs/py3.12/bin/python3 -m pip uninstall -y flashinfer-python
+          docker exec "$cid" /bin/bash -c '/home/sdp/miniforge3/envs/py3.12/bin/hf auth login --token ${HF_TOKEN} '
+
+      - name: Run stage-b tests
+        timeout-minutes: 60
+        run: |
+          cid="${{ steps.start_container.outputs.container_id }}"
+          docker exec "$cid" bash -c "source /home/sdp/miniforge3/bin/activate && conda activate py3.12 && cd /home/sdp/sglang/test && python3 run_suite.py --hw xpu --suite stage-b-test-1-gpu-xpu"
+
       - name: Cleanup container
         if: always()
         run: |
@@ -126,15 +191,20 @@ jobs:
 
   finish:
     if: always()
-    needs: [build-and-test, pr-gate]
+    needs: [stage-a-test-1-gpu-xpu, stage-b-test-1-gpu-xpu, pr-gate]
     runs-on: ubuntu-latest
     steps:
       - name: Check job status
         run: |
-          result="${{ needs.build-and-test.result }}"
+          stage_a="${{ needs.stage-a-test-1-gpu-xpu.result }}"
-          if [ "$result" != "success" ] && [ "$result" != "skipped" ]; then
+          stage_b="${{ needs.stage-b-test-1-gpu-xpu.result }}"
-            echo "Job failed with result: $result"
+          if [ "$stage_a" != "success" ] && [ "$stage_a" != "skipped" ]; then
+            echo "stage-a failed with result: $stage_a"
             exit 1
           fi
-          echo "All jobs completed successfully (result: $result)"
+          if [ "$stage_b" != "success" ] && [ "$stage_b" != "skipped" ]; then
+            echo "stage-b failed with result: $stage_b"
+            exit 1
+          fi
+          echo "All jobs completed successfully"
           exit 0

python/sglang/test/ci/ci_register.py

@@ -13,6 +13,7 @@ __all__ = [
     "register_cuda_ci",
     "register_amd_ci",
     "register_npu_ci",
+    "register_xpu_ci",
     "ut_parse_one_file",
 ]
 
@@ -107,9 +108,12 @@ def register_npu_ci(
 
 def register_xpu_ci(
     est_time: float,
-    suite: str,
+    suite: Optional[str] = None,
     nightly: bool = False,
     disabled: Optional[str] = None,
+    *,
+    stage: Optional[str] = None,
+    runner_config: Optional[str] = None,
 ):
     """Marker for XPU CI registration (parsed via AST; runtime no-op)."""
     return None

test/registered/attention/test_chunk_gated_delta_rule.py

@@ -10,7 +10,7 @@ from sglang.srt.utils import get_device
 from sglang.test.ci.ci_register import register_cuda_ci, register_xpu_ci
 
 register_cuda_ci(est_time=11, stage="base-b", runner_config="1-gpu-large")
-register_xpu_ci(est_time=30, suite="xpu")
+register_xpu_ci(est_time=900, suite="stage-b-test-1-gpu-xpu")
 
 
 @unittest.skipIf(

test/registered/xpu/test_deepseek_ocr.py

@@ -11,6 +11,7 @@ import requests
 
 from sglang.srt.utils import kill_process_tree
 from sglang.srt.utils.hf_transformers import get_tokenizer
+from sglang.test.ci.ci_register import register_xpu_ci
 from sglang.test.test_utils import (
     DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
     DEFAULT_URL_FOR_TEST,
@@ -18,6 +19,8 @@ from sglang.test.test_utils import (
     popen_launch_server,
 )
 
+register_xpu_ci(est_time=360, suite="stage-b-test-1-gpu-xpu")
+
 
 class TestDeepSeekOCR(CustomTestCase):
     @classmethod

test/registered/xpu/test_deepseek_ocr_triton.py

@@ -9,12 +9,19 @@ from pathlib import Path
 from test_deepseek_ocr import TestDeepSeekOCR
 
 from sglang.srt.utils.hf_transformers import get_tokenizer
+from sglang.test.ci.ci_register import register_xpu_ci
 from sglang.test.test_utils import (
     DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
     DEFAULT_URL_FOR_TEST,
     popen_launch_server,
 )
 
+register_xpu_ci(
+    est_time=360,
+    suite="stage-b-test-1-gpu-xpu",
+    disabled="Temporarily disabled until Triton-XPU upgrade",
+)
+
 
 # TODO: Temporarily disable this test and re-enable it after Triton-XPU is upgraded.
 @unittest.skip("Temporarily disabled until Triton-XPU upgrade")

test/registered/xpu/test_intel_xpu_backend.py

@@ -6,6 +6,7 @@ python3 -m unittest test_intel_xpu_backend.TestIntelXPUBackend.test_latency_qwen
 import unittest
 from functools import wraps
 
+from sglang.test.ci.ci_register import register_xpu_ci
 from sglang.test.test_utils import (
     DEFAULT_MODEL_NAME_FOR_TEST_FP8_WITH_MOE,
     DEFAULT_SMALL_MODEL_NAME_FOR_TEST_BASE,
@@ -15,6 +16,8 @@ from sglang.test.test_utils import (
     run_bench_one_batch,
 )
 
+register_xpu_ci(est_time=600, suite="stage-b-test-1-gpu-xpu")
+
 
 def intel_xpu_benchmark(
     extra_args=None, min_throughput=None, mem_fraction_static="0.4"

test/registered/xpu/test_xpu_basic.py

@@ -0,0 +1,47 @@
+"""
+Basic XPU test: verifies the server starts and produces a non-empty
+response on Intel XPU with the default attention backend.
+
+Assigned to stage-a so it gates stage-b before the heavier tests run.
+
+Usage:
+python3 -m unittest test_xpu_basic.TestXPUBasic.test_basic_generation
+"""
+
+import unittest
+
+from sglang.test.ci.ci_register import register_xpu_ci
+from sglang.test.test_utils import (
+    DEFAULT_SMALL_MODEL_NAME_FOR_TEST_QWEN,
+    CustomTestCase,
+    is_in_ci,
+    run_bench_one_batch,
+)
+
+register_xpu_ci(est_time=300, suite="stage-a-test-1-gpu-xpu")
+
+
+class TestXPUBasic(CustomTestCase):
+
+    def test_basic_generation(self):
+        """Server starts on XPU and completes at least one decode step."""
+        args = [
+            "--device",
+            "xpu",
+            "--disable-radix-cache",
+            "--mem-fraction-static",
+            "0.6",
+            "--batch-size",
+            "1",
+        ]
+        if is_in_ci():
+            args += ["--input", "64", "--output", "4"]
+
+        _, decode_throughput, _ = run_bench_one_batch(
+            DEFAULT_SMALL_MODEL_NAME_FOR_TEST_QWEN, args
+        )
+        self.assertGreater(decode_throughput, 0, "XPU decode throughput must be > 0")
+
+
+if __name__ == "__main__":
+    unittest.main()

test/run_suite.py

@@ -20,6 +20,7 @@ HW_MAPPING = {
     "cuda": HWBackend.CUDA,
     "amd": HWBackend.AMD,
     "npu": HWBackend.NPU,
+    "xpu": HWBackend.XPU,
 }
 
 # Per-commit test suites (run on every PR).
@@ -77,6 +78,10 @@ PER_COMMIT_SUITES = {
         "stage-b-test-4-npu-a3",
         "stage-b-test-16-npu-a3",
     ],
+    HWBackend.XPU: [
+        "stage-a-test-1-gpu-xpu",
+        "stage-b-test-1-gpu-xpu",
+    ],
 }
 
 # Nightly test suites (run nightly, organized by GPU configuration)
@@ -127,6 +132,7 @@ NIGHTLY_SUITES = {
         "full-8-npu-a3",
         "full-16-npu-a3",
     ],
+    HWBackend.XPU: [],
 }
 
 
@@ -141,7 +147,7 @@ OTHER_SUITES = {
 }
 
 
-_SUITE_CHECKED_BACKENDS = {HWBackend.CUDA, HWBackend.CPU}
+_SUITE_CHECKED_BACKENDS = {HWBackend.CUDA, HWBackend.CPU, HWBackend.XPU}
 
 
 def _valid_suites_by_backend() -> dict:

test/srt/run_suite.py

@@ -87,16 +87,10 @@ suite_xeon = {
     ],
 }
 
-# Add Intel XPU tests
+# XPU tests migrated to test/registered/xpu/ using register_xpu_ci().
-# NOTE: please sort the test cases alphabetically by the test file name
+# The legacy per-commit-xpu suite is replaced by stage-a-test-1-gpu-xpu
-suite_xpu = {
+# and stage-b-test-1-gpu-xpu in test/run_suite.py (registry-based).
-    "per-commit-xpu": [
+suite_xpu = {}
-        TestFile("xpu/test_deepseek_ocr.py", 360),
-        TestFile("xpu/test_deepseek_ocr_triton.py", 360),
-        # TestFile("xpu/test_internvl.py"),
-        TestFile("xpu/test_intel_xpu_backend.py"),
-    ],
-}
 
 suites.update(suite_amd)
 suites.update(suite_arm64)