name: PR Test (Xeon) on: push: branches: [ main ] pull_request: branches: [ main ] workflow_dispatch: workflow_call: inputs: ref: description: 'Git ref (branch, tag, or SHA) to test. If not provided, uses the default branch.' required: false type: string default: '' run_all_tests: description: "Run all tests (for releasing or testing purpose)" required: false type: boolean default: false concurrency: group: pr-test-xeon-${{ inputs.ref || github.ref }} cancel-in-progress: ${{ github.event_name != 'workflow_call' }} jobs: # ==================== Check Changes ==================== # check-changes: runs-on: ubuntu-latest outputs: main_package: ${{ steps.filter.outputs.main_package || steps.run-mode.outputs.run_all_tests}} env: PYTHONDONTWRITEBYTECODE: 1 steps: - name: clean up run: | sudo rm -rf ${{ github.workspace }} || true sudo mkdir -p ${{ github.workspace }} sudo chown -R $(id -u):$(id -g) ${{ github.workspace }} - name: Checkout code uses: actions/checkout@v4 with: clean: false ref: ${{ inputs.ref || github.ref }} - name: Determine run mode id: run-mode run: | # Run all tests for workflow_call (when ref input is provided) # Note: github.event_name is inherited from caller, so we detect workflow_call by checking inputs.ref if [[ "${{ inputs.run_all_tests }}" == "true" ]]; then echo "run_all_tests=true" >> $GITHUB_OUTPUT echo "Run mode: ALL TESTS (run_all_tests=${{ inputs.run_all_tests }})" else echo "run_all_tests=false" >> $GITHUB_OUTPUT echo "Run mode: FILTERED (triggered by ${{ github.event_name }})" fi - name: Detect file changes id: filter uses: dorny/paths-filter@v3 if: steps.run-mode.outputs.run_all_tests != 'true' with: filters: | main_package: - "python/sglang/!(multimodal_gen)/**/!(*.md)" - "python/pyproject_cpu.toml" - "test/**/!(*.md)" - "python/sglang/kernels/aot/**/!(*.md|THIRDPARTYNOTICES.txt|LICENSE)" - ".github/workflows/pr-test-xeon.yml" - "docker/xeon.Dockerfile" # ==================== PR Gate ==================== # pr-gate: needs: check-changes if: needs.check-changes.outputs.main_package == 'true' uses: ./.github/workflows/pr-gate.yml secrets: inherit build-test: needs: [check-changes, pr-gate] if: needs.check-changes.outputs.main_package == 'true' runs-on: ${{ matrix.runner }} strategy: fail-fast: true matrix: include: # GNR runs only the TP suite (needs >=6 NUMA nodes for --tp 6), # full machine, no partitioning. 4 files: 3 TP + autoround. - runner: xeon-gnr role: gnr suite: "stage-a-tp-test-cpu-intel" name: gnr # Each SPR box (2 sockets / 2 NUMA, 32 physical cores each) runs TWO # socket-pinned containers in parallel, each taking one partition of a # 4-way split over the merged non-TP pool. Each container is capped # with --memory so sglang (which reads cgroup memory.max, PR #38260 + # the empty-NUMA-node fix) sizes its KV pool against one socket's # budget, not the whole host -- so two parallel servers no longer # exhaust host RAM. 2 boxes x 2 sockets = 4 partitions. - runner: xeon-spr role: spr suite: "stage-a-test-cpu-intel,stage-b-test-cpu-intel" name: spr1 part_lo: 0 part_hi: 1 part_size: 4 - runner: xeon-spr role: spr suite: "stage-a-test-cpu-intel,stage-b-test-cpu-intel" name: spr2 part_lo: 2 part_hi: 3 part_size: 4 steps: - name: Cleanup workspace run: | sudo rm -rf ${{ github.workspace }} || true sudo mkdir -p ${{ github.workspace }} sudo chown $(id -u):$(id -g) ${{ github.workspace }} - name: Checkout repository uses: actions/checkout@v4 with: clean: false ref: ${{ inputs.ref || github.ref }} - name: Build image run: | # Fixed tag (no SHA): a new build overwrites the previous image so a # self-hosted box does not accumulate multi-GB images across PRs. # --no-cache keeps the CPU image clean (past sgl-eval/accelerate gaps # came from stale layers). One build per job, one container per box. PR_REPO=${{ github.event.pull_request.head.repo.clone_url }} PR_HEAD_REF=${{ github.head_ref }} docker build \ ${PR_REPO:+--build-arg SGLANG_REPO=$PR_REPO} \ ${PR_HEAD_REF:+--build-arg VER_SGLANG=$PR_HEAD_REF} \ . -f docker/xeon.Dockerfile -t sglang_xeon --no-cache # ==================== GNR: full machine, TP suite ==================== # - name: Run container (GNR) if: matrix.role == 'gnr' run: | docker rm -f ci_sglang_${{ matrix.name }} || true docker run -dt \ -v ${{ github.workspace }}:/sglang-checkout/ --shm-size=64g \ -v $HOME/.cache/huggingface:/root/.cache/huggingface \ -v $HOME/.cache/sgl_eval:/root/.cache/sgl_eval \ -e HF_TOKEN="$(cat ~/huggingface_token.txt)" \ --name ci_sglang_${{ matrix.name }} \ sglang_xeon - name: Check AMX support (GNR) if: matrix.role == 'gnr' timeout-minutes: 5 run: | docker exec -w /sglang-checkout/ ci_sglang_${{ matrix.name }} \ bash -c "source /opt/.venv/bin/activate && python3 -c 'import torch; import sgl_kernel; assert torch.cpu._is_amx_tile_supported(); assert hasattr(torch.ops.sgl_kernel, \"convert_weight_packed\"); '" - name: Run unit tests (GNR) if: matrix.role == 'gnr' timeout-minutes: 120 run: | docker exec -w /sglang-checkout/ ci_sglang_${{ matrix.name }} \ bash -c "source /opt/.venv/bin/activate && hf auth login --token \$HF_TOKEN && cd ./test && python3 run_suite.py --hw cpu --suite ${{ matrix.suite }} --timeout-from-est-time" # ========= SPR: two socket-pinned containers, run in parallel ======== # - name: Run unit tests (SPR, two sockets in parallel) if: matrix.role == 'spr' timeout-minutes: 120 run: | set -uo pipefail HF_TOKEN_VAL="$(cat ~/huggingface_token.txt)" # One socket-pinned container. Physical cores only (0-31 = socket0, # 32-63 = socket1); logical/HT siblings (64-127) are never bound. # --memory caps the cgroup so sglang sizes KV against one socket, and # --cpuset-mems keeps allocations on the local memory bank. # $1 socket id, $2 cpuset-cpus, $3 cpuset-mems, $4 partition id run_socket () { local sid=$1 cpus=$2 mems=$3 pid=$4 local cname=ci_sglang_${{ matrix.name }}_s${sid} docker rm -f "$cname" >/dev/null 2>&1 || true docker run -dt \ --cpuset-cpus="$cpus" --cpuset-mems="$mems" --memory=240g \ -v ${{ github.workspace }}:/sglang-checkout/ --shm-size=32g \ -v $HOME/.cache/huggingface:/root/.cache/huggingface \ -v $HOME/.cache/sgl_eval:/root/.cache/sgl_eval \ -e HF_TOKEN="$HF_TOKEN_VAL" \ -e SGLANG_CPU_OMP_THREADS_BIND="$cpus" \ --name "$cname" \ sglang_xeon docker exec -w /sglang-checkout/ "$cname" bash -c \ "source /opt/.venv/bin/activate && python3 -c 'import torch; import sgl_kernel; assert torch.cpu._is_amx_tile_supported(); assert hasattr(torch.ops.sgl_kernel, \"convert_weight_packed\")' && \ hf auth login --token \$HF_TOKEN && \ cd ./test && python3 run_suite.py --hw cpu --suite ${{ matrix.suite }} --auto-partition-id $pid --auto-partition-size ${{ matrix.part_size }} --timeout-from-est-time" } # socket0 -> part_lo, socket1 -> part_hi; run concurrently, wait both. run_socket 0 "0-31" 0 ${{ matrix.part_lo }} > s0.log 2>&1 & P0=$! run_socket 1 "32-63" 1 ${{ matrix.part_hi }} > s1.log 2>&1 & P1=$! rc=0 wait $P0 || rc=1 wait $P1 || rc=1 echo "==================== socket0 (partition ${{ matrix.part_lo }}) ====================" cat s0.log || true echo "==================== socket1 (partition ${{ matrix.part_hi }}) ====================" cat s1.log || true exit $rc # ============================ Common ================================= # - name: Change permission if: always() timeout-minutes: 2 run: | # GNR has one container (ci_sglang_); SPR has two socket # containers (ci_sglang__s0/_s1). Chown from whichever exist. for c in ci_sglang_${{ matrix.name }} ci_sglang_${{ matrix.name }}_s0 ci_sglang_${{ matrix.name }}_s1; do docker exec -u root "$c" bash -c "rm -rf /tmp/ci-home && chown -R $(id -u):$(id -g) /sglang-checkout/ 2>/dev/null || true" 2>/dev/null || true done - name: Cleanup container if: always() run: | for c in ci_sglang_${{ matrix.name }} ci_sglang_${{ matrix.name }}_s0 ci_sglang_${{ matrix.name }}_s1; do docker rm -f "$c" || true done