diff --git a/.github/workflows/pr-test-xpu.yml b/.github/workflows/pr-test-xpu.yml index 867d03f3a..2f5ca1e5e 100644 --- a/.github/workflows/pr-test-xpu.yml +++ b/.github/workflows/pr-test-xpu.yml @@ -127,74 +127,70 @@ jobs: steps: - run: echo "stage-a passed" - # ==================== Stage B (disabled — tests moved to Stage A) ==================== # - # stage-b-test-1-gpu-xpu: - # needs: [check-changes, pr-gate, wait-for-stage-a] - # if: needs.check-changes.outputs.main_package == 'true' - # runs-on: intel-bmg - # steps: - # - name: Checkout code - # uses: actions/checkout@v4 - # with: - # fetch-depth: 0 - # ref: ${{ inputs.ref || github.ref }} - # - # - name: Build Docker image - # run: | - # PR_REPO=${{ github.event.pull_request.head.repo.clone_url }} - # PR_HEAD_REF=${{ github.head_ref }} - # docker build \ - # ${PR_REPO:+--build-arg SG_LANG_REPO=$PR_REPO} \ - # ${PR_HEAD_REF:+--build-arg SG_LANG_BRANCH=$PR_HEAD_REF} \ - # --no-cache --progress=plain -f docker/xpu.Dockerfile -t xpu_sglang_main:bmg . - # - # - name: Run container - # id: start_container - # run: | - # container_id=$(docker run -dt \ - # --group-add 992 \ - # --group-add $(getent group video | cut -d: -f3) \ - # --group-add $(getent group render | cut -d: -f3) \ - # -v $HOME/.cache/huggingface:/root/.cache/huggingface \ - # --device /dev/dri \ - # -v /dev/dri/by-path:/dev/dri/by-path \ - # -e HF_TOKEN="$(cat ~/huggingface_token.txt)" \ - # xpu_sglang_main:bmg) - # echo "Started container: $container_id" - # echo "container_id=$container_id" >> "$GITHUB_OUTPUT" - # - # - name: Install Dependency - # timeout-minutes: 20 - # run: | - # cid="${{ steps.start_container.outputs.container_id }}" - # docker exec "$cid" /opt/venv/bin/python3 -m pip install --upgrade pip - # docker exec "$cid" /opt/venv/bin/python3 -m pip install pytest expecttest ray huggingface_hub tabulate - # docker exec "$cid" /opt/venv/bin/python3 -m pip uninstall -y flashinfer-python - # docker exec "$cid" /bin/bash -c '/opt/venv/bin/hf auth login --token ${HF_TOKEN} ' - # - # - name: Run stage-b tests - # timeout-minutes: 60 - # run: | - # cid="${{ steps.start_container.outputs.container_id }}" - # docker exec "$cid" bash -c "source /opt/venv/bin/activate && cd /sgl-workspace/sglang/test && python3 run_suite.py --hw xpu --suite stage-b-test-1-gpu-xpu" - # - # - name: Cleanup container - # if: always() - # run: | - # cid="${{ steps.start_container.outputs.container_id }}" - # docker rm -f "$cid" || true + # ==================== Stage B ==================== # + stage-b-test-1-gpu-xpu: + needs: [check-changes, pr-gate, wait-for-stage-a] + if: needs.check-changes.outputs.main_package == 'true' + runs-on: intel-bmg + env: + DOCKERHUB_INTEL_USERNAME: ${{ secrets.DOCKERHUB_INTEL_USERNAME }} + DOCKERHUB_INTEL_TOKEN: ${{ secrets.DOCKERHUB_INTEL_TOKEN }} + steps: + - name: Reset workspace ownership + run: | + docker run --rm -v "${{ github.workspace }}:/w" busybox:latest \ + chown -R "$(id -u):$(id -g)" /w || true + + - name: Checkout code + uses: actions/checkout@v4 + with: + fetch-depth: 0 + ref: ${{ inputs.ref || github.ref }} + + - name: Start CI container + run: | + export HF_TOKEN="$(cat ~/huggingface_token.txt)" + bash scripts/ci/xpu/xpu_ci_start_container.sh + env: + GITHUB_WORKSPACE: ${{ github.workspace }} + + - name: Install Dependency + timeout-minutes: 60 + run: | + docker exec ci_sglang_xpu /opt/venv/bin/python3 -m pip install --upgrade pip + docker exec ci_sglang_xpu /opt/venv/bin/python3 -m pip install pytest expecttest ray huggingface_hub tabulate + docker exec ci_sglang_xpu /opt/venv/bin/python3 -m pip uninstall -y flashinfer-python sgl-kernel sglang + docker exec ci_sglang_xpu cp /sglang-checkout/python/pyproject_xpu.toml /sglang-checkout/python/pyproject.toml + docker exec -w /sglang-checkout/python ci_sglang_xpu /opt/venv/bin/python3 -m pip install --no-cache-dir . --extra-index-url https://download.pytorch.org/whl/xpu + docker exec ci_sglang_xpu /opt/venv/bin/python3 -m pip install --no-cache-dir --no-deps xgrammar==0.1.33 + docker exec ci_sglang_xpu /opt/venv/bin/python3 -m pip install triton-xpu==3.7.1 --index-url https://download.pytorch.org/whl/test/xpu --force-reinstall + docker exec ci_sglang_xpu /bin/bash -c '/opt/venv/bin/hf auth login --token ${HF_TOKEN}' + + - name: Run stage-b tests + timeout-minutes: 60 + run: | + docker exec ci_sglang_xpu bash -c "source /opt/venv/bin/activate && cd /sglang-checkout/test && python3 run_suite.py --hw xpu --suite stage-b-test-1-gpu-xpu" + + - name: Cleanup container + if: always() + run: docker rm -f ci_sglang_xpu || true finish: if: always() - needs: [stage-a-test-1-gpu-xpu, pr-gate] + needs: [stage-a-test-1-gpu-xpu, stage-b-test-1-gpu-xpu, pr-gate] runs-on: ubuntu-latest steps: - name: Check job status run: | stage_a="${{ needs.stage-a-test-1-gpu-xpu.result }}" + stage_b="${{ needs.stage-b-test-1-gpu-xpu.result }}" if [ "$stage_a" != "success" ] && [ "$stage_a" != "skipped" ]; then echo "stage-a failed with result: $stage_a" exit 1 fi + if [ "$stage_b" != "success" ] && [ "$stage_b" != "skipped" ]; then + echo "stage-b failed with result: $stage_b" + exit 1 + fi echo "All jobs completed successfully" exit 0 diff --git a/test/registered/attention/test_chunk_gated_delta_rule.py b/test/registered/attention/test_chunk_gated_delta_rule.py index bbff6078e..680c2b50e 100644 --- a/test/registered/attention/test_chunk_gated_delta_rule.py +++ b/test/registered/attention/test_chunk_gated_delta_rule.py @@ -10,7 +10,7 @@ from sglang.srt.utils import get_device from sglang.test.ci.ci_register import register_cuda_ci, register_xpu_ci register_cuda_ci(est_time=11, stage="base-b", runner_config="1-gpu-large") -register_xpu_ci(est_time=900, suite="stage-a-test-1-gpu-xpu") +register_xpu_ci(est_time=900, suite="stage-b-test-1-gpu-xpu") @unittest.skipIf( diff --git a/test/registered/xpu/test_deepseek_ocr.py b/test/registered/xpu/test_deepseek_ocr.py index 060e04df5..33e38c789 100644 --- a/test/registered/xpu/test_deepseek_ocr.py +++ b/test/registered/xpu/test_deepseek_ocr.py @@ -19,7 +19,7 @@ from sglang.test.test_utils import ( popen_launch_server, ) -register_xpu_ci(est_time=360, suite="stage-a-test-1-gpu-xpu") +register_xpu_ci(est_time=360, suite="stage-b-test-1-gpu-xpu") class TestDeepSeekOCR(CustomTestCase): diff --git a/test/registered/xpu/test_deepseek_ocr_triton.py b/test/registered/xpu/test_deepseek_ocr_triton.py index f605cf41e..b17c29710 100644 --- a/test/registered/xpu/test_deepseek_ocr_triton.py +++ b/test/registered/xpu/test_deepseek_ocr_triton.py @@ -18,7 +18,7 @@ from sglang.test.test_utils import ( register_xpu_ci( est_time=360, - suite="stage-a-test-1-gpu-xpu", + suite="stage-b-test-1-gpu-xpu", disabled="Temporarily disabled until Triton-XPU upgrade", ) diff --git a/test/registered/xpu/test_gemma_4_e2b.py b/test/registered/xpu/test_gemma_4_e2b.py index cd37335bf..ebe1e2dd2 100644 --- a/test/registered/xpu/test_gemma_4_e2b.py +++ b/test/registered/xpu/test_gemma_4_e2b.py @@ -130,7 +130,11 @@ class TestGemma4E2BXPU(CustomTestCase): from sglang.test.ci.ci_register import register_xpu_ci # Single e2e test: boot + a short Q&A. -register_xpu_ci(est_time=240, suite="stage-b-test-1-gpu-xpu") +register_xpu_ci( + est_time=240, + suite="stage-b-test-1-gpu-xpu", + disabled="OOM on stage-b XPU runner (server launch fails with --mem-fraction-static)", +) if __name__ == "__main__": unittest.main() diff --git a/test/registered/xpu/test_intel_xpu_backend.py b/test/registered/xpu/test_intel_xpu_backend.py index c73f30102..b38c57984 100644 --- a/test/registered/xpu/test_intel_xpu_backend.py +++ b/test/registered/xpu/test_intel_xpu_backend.py @@ -16,7 +16,7 @@ from sglang.test.test_utils import ( run_bench_one_batch, ) -register_xpu_ci(est_time=600, suite="stage-a-test-1-gpu-xpu") +register_xpu_ci(est_time=600, suite="stage-b-test-1-gpu-xpu") def intel_xpu_benchmark( diff --git a/test/registered/xpu/test_topk.py b/test/registered/xpu/test_topk.py index 2522950b8..2ddc0be59 100644 --- a/test/registered/xpu/test_topk.py +++ b/test/registered/xpu/test_topk.py @@ -11,7 +11,7 @@ from sglang.srt.layers.moe.topk import ( from sglang.test.ci.ci_register import register_xpu_ci from sglang.test.test_utils import CustomTestCase -register_xpu_ci(est_time=5, suite="stage-a-test-1-gpu-xpu") +register_xpu_ci(est_time=5, suite="stage-b-test-1-gpu-xpu") # Nemotron-3 uses biased_grouped_topk