diff --git a/.github/workflows/pr-test-xpu.yml b/.github/workflows/pr-test-xpu.yml index aa0554c72..22dac03fc 100644 --- a/.github/workflows/pr-test-xpu.yml +++ b/.github/workflows/pr-test-xpu.yml @@ -68,7 +68,8 @@ jobs: uses: ./.github/workflows/pr-gate.yml secrets: inherit - build-and-test: + # ==================== Stage A ==================== # + stage-a-test-1-gpu-xpu: needs: [check-changes, pr-gate] if: needs.check-changes.outputs.main_package == 'true' runs-on: intel-bmg @@ -108,16 +109,80 @@ jobs: run: | cid="${{ steps.start_container.outputs.container_id }}" docker exec "$cid" /home/sdp/miniforge3/envs/py3.12/bin/python3 -m pip install --upgrade pip - docker exec "$cid" /home/sdp/miniforge3/envs/py3.12/bin/python3 -m pip install pytest expecttest ray huggingface_hub + docker exec "$cid" /home/sdp/miniforge3/envs/py3.12/bin/python3 -m pip install pytest expecttest ray huggingface_hub tabulate docker exec "$cid" /home/sdp/miniforge3/envs/py3.12/bin/python3 -m pip uninstall -y flashinfer-python docker exec "$cid" /bin/bash -c '/home/sdp/miniforge3/envs/py3.12/bin/hf auth login --token ${HF_TOKEN} ' + - name: Run stage-a tests + timeout-minutes: 30 + run: | + cid="${{ steps.start_container.outputs.container_id }}" + docker exec "$cid" bash -c "source /home/sdp/miniforge3/bin/activate && conda activate py3.12 && cd /home/sdp/sglang/test && python3 run_suite.py --hw xpu --suite stage-a-test-1-gpu-xpu" - - name: Run E2E Bfloat16 tests + - name: Cleanup container + if: always() + run: | + cid="${{ steps.start_container.outputs.container_id }}" + docker rm -f "$cid" || true + + # ==================== Wait for Stage A ==================== # + wait-for-stage-a: + needs: [stage-a-test-1-gpu-xpu] + runs-on: ubuntu-latest + steps: + - run: echo "stage-a passed" + + # ==================== Stage B ==================== # + stage-b-test-1-gpu-xpu: + needs: [check-changes, pr-gate, wait-for-stage-a] + if: needs.check-changes.outputs.main_package == 'true' + runs-on: intel-bmg + steps: + - name: Checkout code + uses: actions/checkout@v4 + with: + fetch-depth: 0 + ref: ${{ inputs.ref || github.ref }} + + - name: Build Docker image + run: | + PR_REPO=${{ github.event.pull_request.head.repo.clone_url }} + PR_HEAD_REF=${{ github.head_ref }} + docker build \ + ${PR_REPO:+--build-arg SG_LANG_REPO=$PR_REPO} \ + ${PR_HEAD_REF:+--build-arg SG_LANG_BRANCH=$PR_HEAD_REF} \ + --no-cache --progress=plain -f docker/xpu.Dockerfile -t xpu_sglang_main:bmg . + + - name: Run container + id: start_container + run: | + container_id=$(docker run -dt \ + --group-add 992 \ + --group-add $(getent group video | cut -d: -f3) \ + --group-add $(getent group render | cut -d: -f3) \ + -v $HOME/.cache/huggingface:/root/.cache/huggingface \ + --device /dev/dri \ + -v /dev/dri/by-path:/dev/dri/by-path \ + -e HF_TOKEN="$(cat ~/huggingface_token.txt)" \ + xpu_sglang_main:bmg) + echo "Started container: $container_id" + echo "container_id=$container_id" >> "$GITHUB_OUTPUT" + + - name: Install Dependency timeout-minutes: 20 run: | cid="${{ steps.start_container.outputs.container_id }}" - docker exec "$cid" bash -c "source /home/sdp/miniforge3/bin/activate && conda activate py3.12 && cd /home/sdp/sglang/test/srt && python3 run_suite.py --suite per-commit-xpu" + docker exec "$cid" /home/sdp/miniforge3/envs/py3.12/bin/python3 -m pip install --upgrade pip + docker exec "$cid" /home/sdp/miniforge3/envs/py3.12/bin/python3 -m pip install pytest expecttest ray huggingface_hub tabulate + docker exec "$cid" /home/sdp/miniforge3/envs/py3.12/bin/python3 -m pip uninstall -y flashinfer-python + docker exec "$cid" /bin/bash -c '/home/sdp/miniforge3/envs/py3.12/bin/hf auth login --token ${HF_TOKEN} ' + + - name: Run stage-b tests + timeout-minutes: 60 + run: | + cid="${{ steps.start_container.outputs.container_id }}" + docker exec "$cid" bash -c "source /home/sdp/miniforge3/bin/activate && conda activate py3.12 && cd /home/sdp/sglang/test && python3 run_suite.py --hw xpu --suite stage-b-test-1-gpu-xpu" + - name: Cleanup container if: always() run: | @@ -126,15 +191,20 @@ jobs: finish: if: always() - needs: [build-and-test, pr-gate] + needs: [stage-a-test-1-gpu-xpu, stage-b-test-1-gpu-xpu, pr-gate] runs-on: ubuntu-latest steps: - name: Check job status run: | - result="${{ needs.build-and-test.result }}" - if [ "$result" != "success" ] && [ "$result" != "skipped" ]; then - echo "Job failed with result: $result" + stage_a="${{ needs.stage-a-test-1-gpu-xpu.result }}" + stage_b="${{ needs.stage-b-test-1-gpu-xpu.result }}" + if [ "$stage_a" != "success" ] && [ "$stage_a" != "skipped" ]; then + echo "stage-a failed with result: $stage_a" exit 1 fi - echo "All jobs completed successfully (result: $result)" + if [ "$stage_b" != "success" ] && [ "$stage_b" != "skipped" ]; then + echo "stage-b failed with result: $stage_b" + exit 1 + fi + echo "All jobs completed successfully" exit 0 diff --git a/python/sglang/test/ci/ci_register.py b/python/sglang/test/ci/ci_register.py index 1935775d9..d832527c9 100644 --- a/python/sglang/test/ci/ci_register.py +++ b/python/sglang/test/ci/ci_register.py @@ -13,6 +13,7 @@ __all__ = [ "register_cuda_ci", "register_amd_ci", "register_npu_ci", + "register_xpu_ci", "ut_parse_one_file", ] @@ -107,9 +108,12 @@ def register_npu_ci( def register_xpu_ci( est_time: float, - suite: str, + suite: Optional[str] = None, nightly: bool = False, disabled: Optional[str] = None, + *, + stage: Optional[str] = None, + runner_config: Optional[str] = None, ): """Marker for XPU CI registration (parsed via AST; runtime no-op).""" return None diff --git a/test/registered/attention/test_chunk_gated_delta_rule.py b/test/registered/attention/test_chunk_gated_delta_rule.py index c487f2577..680c2b50e 100644 --- a/test/registered/attention/test_chunk_gated_delta_rule.py +++ b/test/registered/attention/test_chunk_gated_delta_rule.py @@ -10,7 +10,7 @@ from sglang.srt.utils import get_device from sglang.test.ci.ci_register import register_cuda_ci, register_xpu_ci register_cuda_ci(est_time=11, stage="base-b", runner_config="1-gpu-large") -register_xpu_ci(est_time=30, suite="xpu") +register_xpu_ci(est_time=900, suite="stage-b-test-1-gpu-xpu") @unittest.skipIf( diff --git a/test/srt/xpu/test_deepseek_ocr.py b/test/registered/xpu/test_deepseek_ocr.py similarity index 96% rename from test/srt/xpu/test_deepseek_ocr.py rename to test/registered/xpu/test_deepseek_ocr.py index 1d8606e23..33e38c789 100644 --- a/test/srt/xpu/test_deepseek_ocr.py +++ b/test/registered/xpu/test_deepseek_ocr.py @@ -11,6 +11,7 @@ import requests from sglang.srt.utils import kill_process_tree from sglang.srt.utils.hf_transformers import get_tokenizer +from sglang.test.ci.ci_register import register_xpu_ci from sglang.test.test_utils import ( DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, DEFAULT_URL_FOR_TEST, @@ -18,6 +19,8 @@ from sglang.test.test_utils import ( popen_launch_server, ) +register_xpu_ci(est_time=360, suite="stage-b-test-1-gpu-xpu") + class TestDeepSeekOCR(CustomTestCase): @classmethod diff --git a/test/srt/xpu/test_deepseek_ocr_triton.py b/test/registered/xpu/test_deepseek_ocr_triton.py similarity index 88% rename from test/srt/xpu/test_deepseek_ocr_triton.py rename to test/registered/xpu/test_deepseek_ocr_triton.py index 67b8c54da..b17c29710 100644 --- a/test/srt/xpu/test_deepseek_ocr_triton.py +++ b/test/registered/xpu/test_deepseek_ocr_triton.py @@ -9,12 +9,19 @@ from pathlib import Path from test_deepseek_ocr import TestDeepSeekOCR from sglang.srt.utils.hf_transformers import get_tokenizer +from sglang.test.ci.ci_register import register_xpu_ci from sglang.test.test_utils import ( DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, DEFAULT_URL_FOR_TEST, popen_launch_server, ) +register_xpu_ci( + est_time=360, + suite="stage-b-test-1-gpu-xpu", + disabled="Temporarily disabled until Triton-XPU upgrade", +) + # TODO: Temporarily disable this test and re-enable it after Triton-XPU is upgraded. @unittest.skip("Temporarily disabled until Triton-XPU upgrade") diff --git a/test/srt/xpu/test_intel_xpu_backend.py b/test/registered/xpu/test_intel_xpu_backend.py similarity index 95% rename from test/srt/xpu/test_intel_xpu_backend.py rename to test/registered/xpu/test_intel_xpu_backend.py index 568ecc47d..b38c57984 100644 --- a/test/srt/xpu/test_intel_xpu_backend.py +++ b/test/registered/xpu/test_intel_xpu_backend.py @@ -6,6 +6,7 @@ python3 -m unittest test_intel_xpu_backend.TestIntelXPUBackend.test_latency_qwen import unittest from functools import wraps +from sglang.test.ci.ci_register import register_xpu_ci from sglang.test.test_utils import ( DEFAULT_MODEL_NAME_FOR_TEST_FP8_WITH_MOE, DEFAULT_SMALL_MODEL_NAME_FOR_TEST_BASE, @@ -15,6 +16,8 @@ from sglang.test.test_utils import ( run_bench_one_batch, ) +register_xpu_ci(est_time=600, suite="stage-b-test-1-gpu-xpu") + def intel_xpu_benchmark( extra_args=None, min_throughput=None, mem_fraction_static="0.4" diff --git a/test/registered/xpu/test_xpu_basic.py b/test/registered/xpu/test_xpu_basic.py new file mode 100644 index 000000000..c1e4c30ea --- /dev/null +++ b/test/registered/xpu/test_xpu_basic.py @@ -0,0 +1,47 @@ +""" +Basic XPU test: verifies the server starts and produces a non-empty +response on Intel XPU with the default attention backend. + +Assigned to stage-a so it gates stage-b before the heavier tests run. + +Usage: +python3 -m unittest test_xpu_basic.TestXPUBasic.test_basic_generation +""" + +import unittest + +from sglang.test.ci.ci_register import register_xpu_ci +from sglang.test.test_utils import ( + DEFAULT_SMALL_MODEL_NAME_FOR_TEST_QWEN, + CustomTestCase, + is_in_ci, + run_bench_one_batch, +) + +register_xpu_ci(est_time=300, suite="stage-a-test-1-gpu-xpu") + + +class TestXPUBasic(CustomTestCase): + + def test_basic_generation(self): + """Server starts on XPU and completes at least one decode step.""" + args = [ + "--device", + "xpu", + "--disable-radix-cache", + "--mem-fraction-static", + "0.6", + "--batch-size", + "1", + ] + if is_in_ci(): + args += ["--input", "64", "--output", "4"] + + _, decode_throughput, _ = run_bench_one_batch( + DEFAULT_SMALL_MODEL_NAME_FOR_TEST_QWEN, args + ) + self.assertGreater(decode_throughput, 0, "XPU decode throughput must be > 0") + + +if __name__ == "__main__": + unittest.main() diff --git a/test/run_suite.py b/test/run_suite.py index 18da6c9fb..36e1dbfb9 100644 --- a/test/run_suite.py +++ b/test/run_suite.py @@ -20,6 +20,7 @@ HW_MAPPING = { "cuda": HWBackend.CUDA, "amd": HWBackend.AMD, "npu": HWBackend.NPU, + "xpu": HWBackend.XPU, } # Per-commit test suites (run on every PR). @@ -77,6 +78,10 @@ PER_COMMIT_SUITES = { "stage-b-test-4-npu-a3", "stage-b-test-16-npu-a3", ], + HWBackend.XPU: [ + "stage-a-test-1-gpu-xpu", + "stage-b-test-1-gpu-xpu", + ], } # Nightly test suites (run nightly, organized by GPU configuration) @@ -127,6 +132,7 @@ NIGHTLY_SUITES = { "full-8-npu-a3", "full-16-npu-a3", ], + HWBackend.XPU: [], } @@ -141,7 +147,7 @@ OTHER_SUITES = { } -_SUITE_CHECKED_BACKENDS = {HWBackend.CUDA, HWBackend.CPU} +_SUITE_CHECKED_BACKENDS = {HWBackend.CUDA, HWBackend.CPU, HWBackend.XPU} def _valid_suites_by_backend() -> dict: diff --git a/test/srt/run_suite.py b/test/srt/run_suite.py index e45d361c3..029853c16 100644 --- a/test/srt/run_suite.py +++ b/test/srt/run_suite.py @@ -87,16 +87,10 @@ suite_xeon = { ], } -# Add Intel XPU tests -# NOTE: please sort the test cases alphabetically by the test file name -suite_xpu = { - "per-commit-xpu": [ - TestFile("xpu/test_deepseek_ocr.py", 360), - TestFile("xpu/test_deepseek_ocr_triton.py", 360), - # TestFile("xpu/test_internvl.py"), - TestFile("xpu/test_intel_xpu_backend.py"), - ], -} +# XPU tests migrated to test/registered/xpu/ using register_xpu_ci(). +# The legacy per-commit-xpu suite is replaced by stage-a-test-1-gpu-xpu +# and stage-b-test-1-gpu-xpu in test/run_suite.py (registry-based). +suite_xpu = {} suites.update(suite_amd) suites.update(suite_arm64)