Files
sglang/.github/workflows/pr-test-npu.yml
T

1242 lines
56 KiB
YAML

name: PR Test (NPU)
on:
schedule:
- cron: '0 12 * * *' # Run daily at 12:00 UTC
pull_request:
branches:
- main
- 'npu/**'
- 'npu-*'
workflow_dispatch:
inputs:
coverage_mode:
description: 'Run in coverage collection mode (uses coverage runners, enables coverage instrumentation, runs setup-covstub)'
required: false
type: boolean
default: false
workflow_call:
inputs:
ref:
description: 'Git ref (branch, tag, or SHA) to test. If not provided, uses the event commit SHA.'
required: false
type: string
default: ''
run_all_tests:
description: "Run all tests (for releasing or testing purpose)"
required: false
type: boolean
default: false
coverage_mode:
description: 'Run in coverage collection mode (uses coverage runners, enables coverage instrumentation, runs setup-covstub)'
required: false
type: boolean
default: false
concurrency:
group: pr-test-npu-${{ inputs.coverage_mode == true && 'coverage-' || '' }}${{ inputs.ref || github.ref }}
cancel-in-progress: ${{ github.event_name != 'workflow_call' }}
jobs:
# ==================== Check Changes ==================== #
check-changes:
runs-on: ubuntu-latest
outputs:
changes_exist: ${{ steps.filter.outputs.main_package == 'true' || steps.filter.outputs.multimodal_gen == 'true' || steps.run-mode.outputs.run_all_tests == 'true'}}
main_package: ${{ steps.filter.outputs.main_package == 'true' || steps.run-mode.outputs.run_all_tests == 'true' }}
multimodal_gen: ${{ steps.filter.outputs.multimodal_gen == 'true' || steps.run-mode.outputs.run_all_tests == 'true' }}
steps:
- name: Checkout code
uses: actions/checkout@v4
with:
# Pin to the event SHA (not the branch name) so every job in this run
# checks out the same commit even if the branch advances mid-run.
ref: ${{ inputs.ref || github.sha }}
- name: Determine run mode
id: run-mode
run: |
# Run all tests for scheduled runs and workflow_call (when ref input is provided)
# Note: github.event_name is inherited from caller, so we detect workflow_call by checking inputs.ref
if [[ "${{ github.event_name }}" == "schedule" || "${{ inputs.run_all_tests }}" == "true" ]]; then
echo "run_all_tests=true" >> $GITHUB_OUTPUT
echo "Run mode: ALL TESTS (schedule=${{ github.event_name == 'schedule' }}, run_all_tests=${{ inputs.run_all_tests }})"
else
echo "run_all_tests=false" >> $GITHUB_OUTPUT
echo "Run mode: FILTERED (triggered by ${{ github.event_name }})"
fi
- name: Detect file changes
id: filter
uses: dorny/paths-filter@v3
if: steps.run-mode.outputs.run_all_tests != 'true'
with:
filters: |
# Everything under python/sglang except multimodal_gen/ and kernels/ops/diffusion/.
# Exclusions have to be extglobs inside a single pattern: paths-filter ORs the
# patterns of a filter (predicate-quantifier defaults to "some"), so a standalone
# "!some/path/**" entry matches every file outside that path and makes the whole
# filter unconditionally true.
main_package:
- "python/sglang/!(*.md)"
- "python/sglang/!(multimodal_gen|kernels)/**/!(*.md)"
- "python/sglang/kernels/!(*.md)"
- "python/sglang/kernels/!(ops)/**/!(*.md)"
- "python/sglang/kernels/ops/!(*.md)"
- "python/sglang/kernels/ops/!(diffusion)/**/!(*.md)"
- "python/pyproject_npu.toml"
- "scripts/ci/npu/npu_ci_install_dependency.sh"
- "test/registered/npu/**"
- "test/registered/unit/npu/**"
- ".github/workflows/pr-test-npu.yml"
multimodal_gen:
- "python/sglang/multimodal_gen/**/!(*.md|*.ipynb)"
- "python/sglang/kernels/ops/diffusion/**"
- "python/sglang/srt/**"
- "python/pyproject_npu.toml"
- "scripts/ci/npu/npu_ci_install_dependency.sh"
- ".github/workflows/pr-test-npu.yml"
# ==================== PR Gate ==================== #
pr-gate:
needs: check-changes
if: ${{ inputs.coverage_mode != true && needs.check-changes.outputs.changes_exist == 'true' }}
uses: ./.github/workflows/pr-gate.yml
secrets: inherit
set-image-config:
runs-on: ubuntu-latest
outputs:
CANN_image_a3: ${{ steps.set-vars.outputs.CANN_image_a3 }}
steps:
# When triggered by PR, no inputs parameters are used. The latest community code is tested by default.
- name: Set image config
id: set-vars
run: |
echo "CANN_image_a3=swr.cn-southwest-2.myhuaweicloud.com/base_image/ascend-ci/cann:9.1.0-a3-ubuntu22.04-py3.12" >> $GITHUB_OUTPUT
# Stage-A gate: CPU-only unit tests. The NPU attention-backend UTs stub out
# torch_npu / sgl_kernel_npu before importing, so a hosted runner suffices.
base-a-test-npu:
needs: [check-changes, pr-gate]
if: ${{ !failure() && !cancelled() && inputs.coverage_mode != true && needs.check-changes.outputs.main_package == 'true' }}
runs-on: ubuntu-latest
timeout-minutes: 45
env:
SGLANG_IS_IN_CI: true
steps:
- name: Checkout code
uses: actions/checkout@v4
with:
# Pin to the event SHA so this job tests the same commit as the rest of the run.
ref: ${{ inputs.ref || github.sha }}
# Same gate the NPU stages run first: fail before the NPU jobs are scheduled.
- uses: ./.github/actions/check-pr-test-health
- name: Set up Python
uses: actions/setup-python@v5
with:
python-version: '3.12'
- name: Install uv
uses: astral-sh/setup-uv@v5
with:
enable-cache: true
cache-dependency-glob: 'python/pyproject_npu.toml'
- name: Install dependencies
timeout-minutes: 30
env:
# uv pip targets a venv by default; setup-python has no venv.
UV_SYSTEM_PYTHON: "1"
run: |
# Resolve the same dependency set as the NPU stages, minus Ascend-only wheels.
rm -f python/pyproject.toml
mv python/pyproject_npu.toml python/pyproject.toml
# Versions mirror npu_ci_install_dependency.sh's defaults; keep them in sync.
# triton: sglang.srt.utils.common imports it unconditionally and the CPU
# wheel does not pull it in (the NPU image uses triton-ascend instead).
# unsafe-best-match: --index-url is the lowest-priority index, so the
# default strategy would resolve PyPI's CUDA torch, not the +cpu build.
uv pip install torch==2.10.0 torchvision==0.25.0 torchaudio==2.10.0 triton \
--index-url https://download.pytorch.org/whl/cpu \
--extra-index-url https://pypi.org/simple/ \
--index-strategy unsafe-best-match
uv pip install -e "python[dev_npu]" --index-strategy unsafe-best-match
- name: Run NPU unit tests
timeout-minutes: 10
run: |
cd test
python3 run_suite.py --hw npu --suite base-a-test-npu
# Coverage-mode-only counterpart of the base-a PR gate above. Stays on
# ubuntu-latest like the gate (its UTs mock the NPU stack and need a CPU
# triton; the NPU image's triton-ascend breaks those mocks), so the shared
# template (unconditional CANN container:) cannot be used and coverage
# data reaches setup-covstub via artifact instead of the shared disk.
base-a-test-npu-cov:
needs: [set-image-config]
if: ${{ inputs.coverage_mode == true }}
runs-on: ubuntu-latest
timeout-minutes: 45
env:
SGLANG_IS_IN_CI: true
steps:
- name: Checkout code
uses: actions/checkout@v4
with:
# Pin to the event SHA so this job tests the same commit as the rest of the run.
ref: ${{ inputs.ref || github.sha }}
- name: Set up Python
uses: actions/setup-python@v5
with:
python-version: '3.12'
- name: Install uv
uses: astral-sh/setup-uv@v5
with:
enable-cache: true
cache-dependency-glob: 'python/pyproject_npu.toml'
- name: Install dependencies
timeout-minutes: 30
env:
# uv pip targets a venv by default; setup-python has no venv.
UV_SYSTEM_PYTHON: "1"
run: |
# Same dependency set as the base-a PR gate above (CPU torch + CPU
# triton, no Ascend wheels); keep the two blocks in sync.
rm -f python/pyproject.toml
mv python/pyproject_npu.toml python/pyproject.toml
uv pip install torch==2.10.0 torchvision==0.25.0 torchaudio==2.10.0 triton \
--index-url https://download.pytorch.org/whl/cpu \
--extra-index-url https://pypi.org/simple/ \
--index-strategy unsafe-best-match
uv pip install -e "python[dev_npu]" --index-strategy unsafe-best-match
pip install coverage==7.8.*
- name: Run test with coverage
timeout-minutes: 15
run: |
# Hosted runners cannot write the shared disk, so stage locally;
# run_tests_with_coverage.sh derives RUN_DIR/date tag from run-level
# env vars, so the artifact merges 1:1 into the shared-disk layout.
export PRECISE_TEST_CACHE_ROOT=/tmp/precise-test
cd test
python3 ../scripts/ci/npu/precise-test/list_tests.py --hw npu --suite base-a-test-npu \
-o /tmp/selected_tests.txt
echo "Selected test files:"
cat /tmp/selected_tests.txt
if [ -s /tmp/selected_tests.txt ]; then
bash ../scripts/ci/npu/precise-test/run_tests_with_coverage.sh $(cat /tmp/selected_tests.txt)
fi
- name: Upload coverage outputs
if: always()
uses: actions/upload-artifact@v4
with:
name: coverage-outputs-base-a-test-npu
path: /tmp/precise-test/
retention-days: 1
if-no-files-found: warn
base-b-test-1-npu-a3:
needs: [check-changes, pr-gate, set-image-config, base-a-test-npu, base-a-test-npu-cov, recommend-tests-from-coverage]
if: >-
${{ !failure() && !cancelled() &&
(inputs.coverage_mode == true || needs.check-changes.outputs.main_package == 'true') &&
needs.recommend-tests-from-coverage.outputs.should_run_base_b_test_1_npu_a3 != 'false' }}
uses: ./.github/workflows/_npu-pr-test-stage.yml
with:
self_name: base-b-test-1-npu-a3
runner_config: ${{ inputs.coverage_mode == true && 'linux-aarch64-a3-800t-2' || 'linux-aarch64-a3-2-' }}
image: ${{ needs.set-image-config.outputs.CANN_image_a3 }}
run_timeout_minutes: '60'
timeout_per_file: '3600'
only_tests: ${{ needs.recommend-tests-from-coverage.outputs.only_tests_base_b_test_1_npu_a3 }}
use_coverage_runner: ${{ inputs.coverage_mode == true }}
secrets: inherit
base-b-test-2-npu-a3:
needs: [check-changes, pr-gate, set-image-config, base-a-test-npu, base-a-test-npu-cov, recommend-tests-from-coverage]
if: >-
${{ !failure() && !cancelled() &&
(inputs.coverage_mode == true || needs.check-changes.outputs.main_package == 'true') &&
needs.recommend-tests-from-coverage.outputs.should_run_base_b_test_2_npu_a3 != 'false' }}
uses: ./.github/workflows/_npu-pr-test-stage.yml
with:
self_name: base-b-test-2-npu-a3
runner_config: ${{ inputs.coverage_mode == true && 'linux-aarch64-a3-800t-2' || 'linux-aarch64-a3-2-' }}
image: ${{ needs.set-image-config.outputs.CANN_image_a3 }}
run_timeout_minutes: '60'
timeout_per_file: '3600'
only_tests: ${{ needs.recommend-tests-from-coverage.outputs.only_tests_base_b_test_2_npu_a3 }}
use_coverage_runner: ${{ inputs.coverage_mode == true }}
secrets: inherit
base-b-test-4-npu-a3:
needs: [check-changes, pr-gate, set-image-config, base-a-test-npu, base-a-test-npu-cov, recommend-tests-from-coverage]
if: >-
${{ !failure() && !cancelled() &&
(inputs.coverage_mode == true || needs.check-changes.outputs.main_package == 'true') &&
needs.recommend-tests-from-coverage.outputs.should_run_base_b_test_4_npu_a3 != 'false' }}
uses: ./.github/workflows/_npu-pr-test-stage.yml
with:
self_name: base-b-test-4-npu-a3
runner_config: ${{ inputs.coverage_mode == true && 'linux-aarch64-a3-800t-4' || 'linux-aarch64-a3-4-' }}
image: ${{ needs.set-image-config.outputs.CANN_image_a3 }}
run_timeout_minutes: '120'
timeout_per_file: '3600'
partitions: '{"size":2,"arr":[0, 1]}'
only_tests: ${{ needs.recommend-tests-from-coverage.outputs.only_tests_base_b_test_4_npu_a3 }}
use_coverage_runner: ${{ inputs.coverage_mode == true }}
secrets: inherit
base-b-test-8-npu-a3:
needs: [check-changes, pr-gate, set-image-config, base-a-test-npu, base-a-test-npu-cov, recommend-tests-from-coverage]
if: >-
${{ !failure() && !cancelled() &&
(inputs.coverage_mode == true || needs.check-changes.outputs.main_package == 'true') &&
needs.recommend-tests-from-coverage.outputs.should_run_base_b_test_8_npu_a3 != 'false' }}
uses: ./.github/workflows/_npu-pr-test-stage.yml
with:
self_name: base-b-test-8-npu-a3
runner_config: ${{ inputs.coverage_mode == true && 'linux-aarch64-a3-800t-8' || 'linux-aarch64-a3-8-' }}
image: ${{ needs.set-image-config.outputs.CANN_image_a3 }}
run_timeout_minutes: '60'
timeout_per_file: '3600'
only_tests: ${{ needs.recommend-tests-from-coverage.outputs.only_tests_base_b_test_8_npu_a3 }}
use_coverage_runner: ${{ inputs.coverage_mode == true }}
secrets: inherit
base-b-test-16-npu-a3:
needs: [check-changes, pr-gate, set-image-config, base-a-test-npu, base-a-test-npu-cov, recommend-tests-from-coverage]
if: >-
${{ !failure() && !cancelled() &&
(inputs.coverage_mode == true || needs.check-changes.outputs.main_package == 'true') &&
needs.recommend-tests-from-coverage.outputs.should_run_base_b_test_16_npu_a3 != 'false' }}
uses: ./.github/workflows/_npu-pr-test-stage.yml
with:
self_name: base-b-test-16-npu-a3
runner_config: ${{ inputs.coverage_mode == true && 'linux-aarch64-a3-800t-16' || 'linux-aarch64-a3-16-' }}
image: ${{ needs.set-image-config.outputs.CANN_image_a3 }}
run_timeout_minutes: '120'
timeout_per_file: '3600'
only_tests: ${{ needs.recommend-tests-from-coverage.outputs.only_tests_base_b_test_16_npu_a3 }}
use_coverage_runner: ${{ inputs.coverage_mode == true }}
secrets: inherit
multimodal-gen-test-1-npu-a3:
needs:
[
check-changes,
pr-gate,
set-image-config,
base-a-test-npu,
base-a-test-npu-cov,
recommend-tests-from-coverage,
]
if: >-
${{ !failure() && !cancelled() &&
(inputs.coverage_mode == true || needs.check-changes.outputs.multimodal_gen == 'true') &&
needs.recommend-tests-from-coverage.outputs.should_run_multimodal_gen_test_1_npu_a3 != 'false' }}
runs-on: linux-aarch64-a3-800t-2
strategy:
fail-fast: false
matrix:
part: [0, 1]
container:
image: ${{ needs.set-image-config.outputs.CANN_image_a3 }}
steps:
- name: Checkout code
uses: actions/checkout@v4
- name: Mark repository safe
run: |
git config --system --add safe.directory ${GITHUB_WORKSPACE}
- name: Install dependencies
env:
TORCH_CACHE_URL: "http://cache-service.nginx-pypi-cache.svc.cluster.local/whl/cpu"
PYPI_CACHE_URL: "http://cache-service.nginx-pypi-cache.svc.cluster.local/pypi/simple"
UV_INDEX_URL: "http://cache-service.nginx-pypi-cache.svc.cluster.local/pypi/simple"
GITHUB_PROXY_URL: "https://gh-proxy.test.osinfra.cn/"
RUSTUP_CACHE_URL: "http://cache-service.nginx-pypi-cache.svc.cluster.local:8082"
run: |
# speed up by using infra cache services
CACHING_URL="cache-service.nginx-pypi-cache.svc.cluster.local"
sed -Ei "s@(ports|archive).ubuntu.com@${CACHING_URL}:8081@g" /etc/apt/sources.list
pip config set global.index-url http://${CACHING_URL}/pypi/simple
pip config set global.trusted-host "${CACHING_URL}"
bash scripts/ci/npu/npu_ci_install_dependency.sh a3
# copy required file from our daily cache
cp ~/.cache/modelscope/hub/datasets/otavia/ShareGPT_Vicuna_unfiltered/ShareGPT_V3_unfiltered_cleaned_split.json /tmp
# copy gsm8k dataset from cache
cp ~/.cache/modelscope/hub/datasets/tmp/test.jsonl /tmp
- name: Prefetch consistency GT
env:
GITHUB_PROXY_URL: "https://gh-proxy.test.osinfra.cn/"
run: |
python3 scripts/ci/utils/diffusion/prefetch_consistency_gt.py
- name: Prefetch image-edit input
env:
GITHUB_PROXY_URL: "https://gh-proxy.test.osinfra.cn/"
run: python3 scripts/ci/utils/diffusion/prefetch_input_image.py
- name: Run test
timeout-minutes: ${{ inputs.coverage_mode == true && 150 || 60 }}
shell: bash
env:
SGLANG_USE_MODELSCOPE: true
SGLANG_IS_IN_CI: true
HF_ENDPOINT: https://hf-mirror.com
TORCH_EXTENSIONS_DIR: /tmp/torch_extensions
PYTORCH_NPU_ALLOC_CONF: "expandable_segments:True"
STREAMS_PER_DEVICE: 32
SGLANG_DIFFUSION_ARTIFACT_DIR: ${{ github.workspace }}/diffusion-failures
ONLY_TESTS: ${{ needs.recommend-tests-from-coverage.outputs.only_tests_multimodal_gen_test_1_npu_a3 }}
run: |
# Coverage mode: enumerate this shard's targets with an NPU-only CI
# helper (it reuses the shared diffusion runner read-only) and run
# each target in its own `coverage run -m pytest` invocation. The
# runner normally batches a whole shard into one pytest call, which
# cannot be attributed per test case by the precise-test selector.
if [[ "${{ inputs.coverage_mode }}" == "true" ]]; then
set -euo pipefail
pip install coverage==7.8.*
# Coverage tracing slows down the Python glue, which shows up in
# wall-clock assertions. Relax only the timing tolerances for the
# collection run; PR runs (no coverage) keep the strict profile and
# the memory/VRAM checks are untouched.
export SGLANG_E2E_TOLERANCE=1.0
export SGLANG_STAGE_TIME_TOLERANCE=1.0
export SGLANG_NON_DENOISE_STAGE_TIME_TOLERANCE=2.0
export SGLANG_DENOISE_STEP_TOLERANCE=1.0
export SGLANG_DENOISE_AGG_TOLERANCE=1.0
python3 scripts/ci/npu/precise-test/list_multimodal_tests.py \
--suite 1-npu \
--partition-id ${{ matrix.part }} \
--total-partitions 2 \
-o /tmp/selected_tests.txt
echo "Selected test targets:"
cat /tmp/selected_tests.txt
if [ -s /tmp/selected_tests.txt ]; then
bash scripts/ci/npu/precise-test/run_tests_with_coverage.sh $(cat /tmp/selected_tests.txt)
fi
exit 0
fi
# Precision mode: ONLY_TESTS holds one pytest -k token per line
# ('[case_id]' for parametrized cases, a file stem for standalone
# files). It is empty when nothing was recommended, precision mode
# is off, or a safety gate forced the full suite; then no filter is
# applied and the shard runs its whole assignment.
FILTER=""
while IFS= read -r line; do
if [ -z "${line}" ]; then
continue
fi
# Every line must be a -k token ('[case_id]' or a standalone file
# stem). A path means the bucket contract drifted: with a path the
# -k expression matches nothing, pytest exits 5, and the shard
# would report success after running zero tests.
if [[ "${line}" == */* || "${line}" == *.py ]]; then
echo "::error::Unexpected only_tests entry '${line}' (expected a pytest -k token)"
exit 1
fi
if [ -n "${FILTER}" ]; then
FILTER="${FILTER} or "
fi
FILTER="${FILTER}${line}"
done <<< "${ONLY_TESTS}"
FILTER_ARGS=()
if [ -n "${FILTER}" ]; then
FILTER_ARGS+=(-k "${FILTER}")
echo "Precision mode: filtered to ${FILTER}"
else
echo "Precision mode: no filter (full suite)"
fi
cd python
python3 sglang/multimodal_gen/test/run_suite.py \
--suite 1-npu \
--partition-id ${{ matrix.part }} \
--total-partitions 2 \
"${FILTER_ARGS[@]}"
- name: Upload diffusion failure artifacts
if: always()
uses: actions/upload-artifact@v4
with:
name: diffusion-failures-npu-1-part${{ matrix.part }}-${{ github.run_attempt }}
path: diffusion-failures/
if-no-files-found: ignore
retention-days: 7
multimodal-gen-test-4-npu-a3:
needs:
[
check-changes,
pr-gate,
set-image-config,
base-a-test-npu,
base-a-test-npu-cov,
recommend-tests-from-coverage,
]
if: >-
${{ !failure() && !cancelled() &&
(inputs.coverage_mode == true || needs.check-changes.outputs.multimodal_gen == 'true') &&
needs.recommend-tests-from-coverage.outputs.should_run_multimodal_gen_test_4_npu_a3 != 'false' }}
runs-on: linux-aarch64-a3-800t-4
strategy:
fail-fast: false
matrix:
part: [0, 1]
container:
image: ${{ needs.set-image-config.outputs.CANN_image_a3 }}
steps:
- name: Checkout code
uses: actions/checkout@v4
- name: Mark repository safe
run: |
git config --system --add safe.directory ${GITHUB_WORKSPACE}
- name: Install dependencies
env:
TORCH_CACHE_URL: "http://cache-service.nginx-pypi-cache.svc.cluster.local/whl/cpu"
PYPI_CACHE_URL: "http://cache-service.nginx-pypi-cache.svc.cluster.local/pypi/simple"
UV_INDEX_URL: "http://cache-service.nginx-pypi-cache.svc.cluster.local/pypi/simple"
GITHUB_PROXY_URL: "https://gh-proxy.test.osinfra.cn/"
RUSTUP_CACHE_URL: "http://cache-service.nginx-pypi-cache.svc.cluster.local:8082"
run: |
# speed up by using infra cache services
CACHING_URL="cache-service.nginx-pypi-cache.svc.cluster.local"
sed -Ei "s@(ports|archive).ubuntu.com@${CACHING_URL}:8081@g" /etc/apt/sources.list
pip config set global.index-url http://${CACHING_URL}/pypi/simple
pip config set global.trusted-host "${CACHING_URL}"
bash scripts/ci/npu/npu_ci_install_dependency.sh a3
# copy required file from our daily cache
cp ~/.cache/modelscope/hub/datasets/otavia/ShareGPT_Vicuna_unfiltered/ShareGPT_V3_unfiltered_cleaned_split.json /tmp
# copy gsm8k dataset
cp ~/.cache/modelscope/hub/datasets/tmp/test.jsonl /tmp
- name: Prefetch consistency GT
env:
GITHUB_PROXY_URL: "https://gh-proxy.test.osinfra.cn/"
run: |
python3 scripts/ci/utils/diffusion/prefetch_consistency_gt.py
- name: Run test
timeout-minutes: ${{ inputs.coverage_mode == true && 150 || 60 }}
shell: bash
env:
SGLANG_USE_MODELSCOPE: true
SGLANG_IS_IN_CI: true
HF_ENDPOINT: https://hf-mirror.com
TORCH_EXTENSIONS_DIR: /tmp/torch_extensions
PYTORCH_NPU_ALLOC_CONF: "expandable_segments:True"
STREAMS_PER_DEVICE: 32
SGLANG_DIFFUSION_ARTIFACT_DIR: ${{ github.workspace }}/diffusion-failures
ONLY_TESTS: ${{ needs.recommend-tests-from-coverage.outputs.only_tests_multimodal_gen_test_4_npu_a3 }}
run: |
# Coverage mode: enumerate this shard's targets with an NPU-only CI
# helper (it reuses the shared diffusion runner read-only) and run
# each target in its own `coverage run -m pytest` invocation. The
# runner normally batches a whole shard into one pytest call, which
# cannot be attributed per test case by the precise-test selector.
if [[ "${{ inputs.coverage_mode }}" == "true" ]]; then
set -euo pipefail
pip install coverage==7.8.*
# Coverage tracing slows down the Python glue, which shows up in
# wall-clock assertions. Relax only the timing tolerances for the
# collection run; PR runs (no coverage) keep the strict profile and
# the memory/VRAM checks are untouched.
export SGLANG_E2E_TOLERANCE=1.0
export SGLANG_STAGE_TIME_TOLERANCE=1.0
export SGLANG_NON_DENOISE_STAGE_TIME_TOLERANCE=2.0
export SGLANG_DENOISE_STEP_TOLERANCE=1.0
export SGLANG_DENOISE_AGG_TOLERANCE=1.0
python3 scripts/ci/npu/precise-test/list_multimodal_tests.py \
--suite 2-npu \
--partition-id ${{ matrix.part }} \
--total-partitions 2 \
-o /tmp/selected_tests.txt
echo "Selected test targets:"
cat /tmp/selected_tests.txt
if [ -s /tmp/selected_tests.txt ]; then
bash scripts/ci/npu/precise-test/run_tests_with_coverage.sh $(cat /tmp/selected_tests.txt)
fi
exit 0
fi
# Precision mode: ONLY_TESTS holds one pytest -k token per line
# ('[case_id]' for parametrized cases, a file stem for standalone
# files). It is empty when nothing was recommended, precision mode
# is off, or a safety gate forced the full suite; then no filter is
# applied and the shard runs its whole assignment.
FILTER=""
while IFS= read -r line; do
if [ -z "${line}" ]; then
continue
fi
# Every line must be a -k token ('[case_id]' or a standalone file
# stem). A path means the bucket contract drifted: with a path the
# -k expression matches nothing, pytest exits 5, and the shard
# would report success after running zero tests.
if [[ "${line}" == */* || "${line}" == *.py ]]; then
echo "::error::Unexpected only_tests entry '${line}' (expected a pytest -k token)"
exit 1
fi
if [ -n "${FILTER}" ]; then
FILTER="${FILTER} or "
fi
FILTER="${FILTER}${line}"
done <<< "${ONLY_TESTS}"
FILTER_ARGS=()
if [ -n "${FILTER}" ]; then
FILTER_ARGS+=(-k "${FILTER}")
echo "Precision mode: filtered to ${FILTER}"
else
echo "Precision mode: no filter (full suite)"
fi
cd python
python3 sglang/multimodal_gen/test/run_suite.py \
--suite 2-npu \
--partition-id ${{ matrix.part }} \
--total-partitions 2 \
"${FILTER_ARGS[@]}"
- name: Upload diffusion failure artifacts
if: always()
uses: actions/upload-artifact@v4
with:
name: diffusion-failures-npu-2-part${{ matrix.part }}-${{ github.run_attempt }}
path: diffusion-failures/
if-no-files-found: ignore
retention-days: 7
base-c-test-acc-2-npu-a3:
name: base-c-test-acc-2-npu-a3
needs: [ check-changes, pr-gate, set-image-config, base-b-test-1-npu-a3, base-b-test-2-npu-a3, base-b-test-4-npu-a3, base-b-test-8-npu-a3, base-b-test-16-npu-a3, recommend-tests-from-coverage ]
if: >-
${{ !failure() && !cancelled() &&
(inputs.coverage_mode == true || needs.check-changes.outputs.main_package == 'true') &&
needs.recommend-tests-from-coverage.outputs.should_run_base_c_test_acc_2_npu_a3 != 'false' }}
uses: ./.github/workflows/_npu-single-node-test-stage.yml
with:
runner: ${{ inputs.coverage_mode == true && 'linux-aarch64-a3-800t-2' || 'linux-aarch64-a3-2-' }}
test_type: 'accuracy'
test_suite: base-c-test-acc-2-npu-a3
image: ${{ needs.set-image-config.outputs.CANN_image_a3 }}
install_sglang_deps: true
device_type_for_deps: 'a3'
partitions: '{"size":2,"arr":[0,1]}'
only_tests: ${{ needs.recommend-tests-from-coverage.outputs.only_tests_base_c_test_acc_2_npu_a3 }}
use_coverage_runner: ${{ inputs.coverage_mode == true }}
base-c-test-acc-16-npu-a3:
name: base-c-test-acc-16-npu-a3
needs: [ check-changes, pr-gate, set-image-config, base-b-test-1-npu-a3, base-b-test-2-npu-a3, base-b-test-4-npu-a3, base-b-test-8-npu-a3, base-b-test-16-npu-a3, recommend-tests-from-coverage ]
if: >-
${{ !failure() && !cancelled() &&
(inputs.coverage_mode == true || needs.check-changes.outputs.main_package == 'true') &&
needs.recommend-tests-from-coverage.outputs.should_run_base_c_test_acc_16_npu_a3 != 'false' }}
uses: ./.github/workflows/_npu-single-node-test-stage.yml
with:
runner: ${{ inputs.coverage_mode == true && 'linux-aarch64-a3-800t-16' || 'linux-aarch64-a3-16-' }}
test_type: 'accuracy'
test_suite: base-c-test-acc-16-npu-a3
image: ${{ needs.set-image-config.outputs.CANN_image_a3 }}
install_sglang_deps: true
device_type_for_deps: 'a3'
only_tests: ${{ needs.recommend-tests-from-coverage.outputs.only_tests_base_c_test_acc_16_npu_a3 }}
use_coverage_runner: ${{ inputs.coverage_mode == true }}
base-c-test-perf-2-npu-a3:
name: base-c-test-perf-2-npu-a3
needs: [ check-changes, pr-gate, set-image-config, base-c-test-acc-2-npu-a3, base-c-test-acc-16-npu-a3, recommend-tests-from-coverage ]
if: >-
${{ !failure() && !cancelled() &&
(inputs.coverage_mode == true || needs.check-changes.outputs.main_package == 'true') &&
needs.recommend-tests-from-coverage.outputs.should_run_base_c_test_perf_2_npu_a3 != 'false' }}
uses: ./.github/workflows/_npu-single-node-test-stage.yml
with:
runner: linux-aarch64-a3-800t-2
test_type: 'perf'
test_suite: base-c-test-perf-2-npu-a3
image: ${{ needs.set-image-config.outputs.CANN_image_a3 }}
install_sglang_deps: true
device_type_for_deps: 'a3'
only_tests: ${{ needs.recommend-tests-from-coverage.outputs.only_tests_base_c_test_perf_2_npu_a3 }}
use_coverage_runner: ${{ inputs.coverage_mode == true }}
base-c-test-perf-16-npu-a3:
name: base-c-test-perf-16-npu-a3
needs: [ check-changes, pr-gate, set-image-config, base-c-test-acc-2-npu-a3, base-c-test-acc-16-npu-a3, recommend-tests-from-coverage ]
if: >-
${{ !failure() && !cancelled() &&
(inputs.coverage_mode == true || needs.check-changes.outputs.main_package == 'true') &&
needs.recommend-tests-from-coverage.outputs.should_run_base_c_test_perf_16_npu_a3 != 'false' }}
uses: ./.github/workflows/_npu-single-node-test-stage.yml
with:
runner: linux-aarch64-a3-800t-16
test_type: 'perf'
test_suite: base-c-test-perf-16-npu-a3
image: ${{ needs.set-image-config.outputs.CANN_image_a3 }}
install_sglang_deps: true
device_type_for_deps: 'a3'
test_timeout_minutes: '180'
only_tests: ${{ needs.recommend-tests-from-coverage.outputs.only_tests_base_c_test_perf_16_npu_a3 }}
use_coverage_runner: ${{ inputs.coverage_mode == true }}
# ==================== Recommend Tests from Coverage ==================== #
# Recommends PR-affected test cases from the pre-built coverage baseline.
# continue-on-error so it never blocks the PR gate.
recommend-tests-from-coverage:
name: Recommend tests from coverage
needs: [check-changes, pr-gate]
runs-on: ubuntu-latest
continue-on-error: true
if: >-
${{ !failure() && !cancelled() &&
github.event_name == 'pull_request' &&
inputs.coverage_mode != true &&
needs.check-changes.outputs.changes_exist == 'true' }}
outputs:
should_run_base_b_test_1_npu_a3: ${{ steps.bucket-tests.outputs.should_run_base_b_test_1_npu_a3 }}
only_tests_base_b_test_1_npu_a3: ${{ steps.bucket-tests.outputs.only_tests_base_b_test_1_npu_a3 }}
should_run_base_b_test_2_npu_a3: ${{ steps.bucket-tests.outputs.should_run_base_b_test_2_npu_a3 }}
only_tests_base_b_test_2_npu_a3: ${{ steps.bucket-tests.outputs.only_tests_base_b_test_2_npu_a3 }}
should_run_base_b_test_4_npu_a3: ${{ steps.bucket-tests.outputs.should_run_base_b_test_4_npu_a3 }}
only_tests_base_b_test_4_npu_a3: ${{ steps.bucket-tests.outputs.only_tests_base_b_test_4_npu_a3 }}
should_run_base_b_test_8_npu_a3: ${{ steps.bucket-tests.outputs.should_run_base_b_test_8_npu_a3 }}
only_tests_base_b_test_8_npu_a3: ${{ steps.bucket-tests.outputs.only_tests_base_b_test_8_npu_a3 }}
should_run_base_b_test_16_npu_a3: ${{ steps.bucket-tests.outputs.should_run_base_b_test_16_npu_a3 }}
only_tests_base_b_test_16_npu_a3: ${{ steps.bucket-tests.outputs.only_tests_base_b_test_16_npu_a3 }}
should_run_base_c_test_acc_2_npu_a3: ${{ steps.bucket-tests.outputs.should_run_base_c_test_acc_2_npu_a3 }}
only_tests_base_c_test_acc_2_npu_a3: ${{ steps.bucket-tests.outputs.only_tests_base_c_test_acc_2_npu_a3 }}
should_run_base_c_test_acc_16_npu_a3: ${{ steps.bucket-tests.outputs.should_run_base_c_test_acc_16_npu_a3 }}
only_tests_base_c_test_acc_16_npu_a3: ${{ steps.bucket-tests.outputs.only_tests_base_c_test_acc_16_npu_a3 }}
should_run_base_c_test_perf_2_npu_a3: ${{ steps.bucket-tests.outputs.should_run_base_c_test_perf_2_npu_a3 }}
only_tests_base_c_test_perf_2_npu_a3: ${{ steps.bucket-tests.outputs.only_tests_base_c_test_perf_2_npu_a3 }}
should_run_base_c_test_perf_16_npu_a3: ${{ steps.bucket-tests.outputs.should_run_base_c_test_perf_16_npu_a3 }}
only_tests_base_c_test_perf_16_npu_a3: ${{ steps.bucket-tests.outputs.only_tests_base_c_test_perf_16_npu_a3 }}
should_run_multimodal_gen_test_1_npu_a3: ${{ steps.bucket-tests.outputs.should_run_multimodal_gen_test_1_npu_a3 }}
only_tests_multimodal_gen_test_1_npu_a3: ${{ steps.bucket-tests.outputs.only_tests_multimodal_gen_test_1_npu_a3 }}
should_run_multimodal_gen_test_4_npu_a3: ${{ steps.bucket-tests.outputs.should_run_multimodal_gen_test_4_npu_a3 }}
only_tests_multimodal_gen_test_4_npu_a3: ${{ steps.bucket-tests.outputs.only_tests_multimodal_gen_test_4_npu_a3 }}
env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
steps:
# Every step carries step-level continue-on-error so the job conclusion
# is ALWAYS 'success' — the fast-fail health check in other jobs filters
# by job conclusion, and job-level continue-on-error still reports
# 'failure' via the API. Failed steps stay visible (red ✗ + annotation)
# but never cascade to test jobs.
- name: Checkout sglang scripts
continue-on-error: true
uses: actions/checkout@v4
with:
# test/registered + python/sglang/test/ci are needed by
# list_tests.py to enumerate each suite's test files for bucketing.
# The diffusion case config is read (AST only) by
# bucket_recommended_tests.py to enumerate multimodal cases.
sparse-checkout: |
scripts
test/registered
python/sglang/test/ci
python/sglang/multimodal_gen/test/server/ascend/testcase_configs_npu.py
sparse-checkout-cone-mode: false
- name: Download test case map
continue-on-error: true
run: |
set -euo pipefail
curl -fsSL \
--retry 3 \
--retry-delay 5 \
--retry-all-errors \
"https://sglang-npu.obs.cn-southwest-2.myhuaweicloud.com/coverage/test_case_map.json" \
-o test_case_map.json
test -s test_case_map.json
echo "Downloaded test_case_map.json"
du -h test_case_map.json
- name: Download coverage package and extract covstub
continue-on-error: true
run: |
set -euo pipefail
curl -fsSL \
--retry 3 \
--retry-delay 5 \
--retry-all-errors \
"https://sglang-npu.obs.cn-southwest-2.myhuaweicloud.com/coverage/outputs.tar.gz" \
-o outputs.tar.gz
test -s outputs.tar.gz
echo "Downloaded outputs.tar.gz"
du -h outputs.tar.gz
tar xzf outputs.tar.gz outputs/covstub
test -d outputs/covstub
echo "Extracted outputs/covstub"
find outputs/covstub -maxdepth 2 -type d | head -20 || true
- name: Recommend tests for current PR
continue-on-error: true
run: |
set -euo pipefail
if ! python3 -c "import regex" >/dev/null 2>&1; then
echo "regex not found for $(python3 -V); bootstrapping pip and installing regex"
curl -fsSL https://bootstrap.pypa.io/get-pip.py -o get-pip.py
python3 get-pip.py --break-system-packages
python3 -m pip install --break-system-packages regex
fi
python3 -c "import regex; print('regex ok:', regex.__file__)"
PR_SPEC="${{ github.repository }}#${{ github.event.pull_request.number }}"
echo "Selecting tests for PR: ${PR_SPEC}"
# test_selector.py resolves relative paths against the script's own dir,
# so pass absolute paths for files downloaded to the workspace root.
python3 scripts/ci/npu/precise-test/test_selector.py \
--github-pr "${PR_SPEC}" \
--source-dir "$PWD/outputs/covstub" \
--map-file "$PWD/test_case_map.json"
# Bucket the recommended tests per suite. Downstream jobs consume two
# outputs per suite: should_run_<suite> ('false' -> skip the job, e.g.
# empty bucket in precision mode) and only_tests_<suite> (non-empty only
# in precision mode, gated inside the script). On failure outputs stay
# empty -> callers run full suites.
- name: Bucket recommended tests per suite
id: bucket-tests
continue-on-error: true
if: always()
env:
# Precision-test switch: 'true' = the a3 suites run only the
# recommended subset, 'false' = full suites.
PRECISION_MODE: 'true'
run: |
set -euo pipefail
python3 scripts/ci/npu/precise-test/bucket_recommended_tests.py \
--recommended scripts/ci/npu/precise-test/recommended_pytest_paths.txt \
--changed-files scripts/ci/npu/precise-test/pr_changed_files.txt \
--suites "base-b-test-1-npu-a3,base-b-test-2-npu-a3,base-b-test-4-npu-a3,base-b-test-8-npu-a3,base-b-test-16-npu-a3,base-c-test-acc-2-npu-a3,base-c-test-acc-16-npu-a3,base-c-test-perf-2-npu-a3,base-c-test-perf-16-npu-a3" \
--diffusion-suites "multimodal-gen-test-1-npu-a3:1-npu,multimodal-gen-test-4-npu-a3:2-npu" \
--case-map "$PWD/test_case_map.json" \
--precision-mode "${PRECISION_MODE}" \
--github-output "${GITHUB_OUTPUT}" \
--github-summary "${GITHUB_STEP_SUMMARY:-}" \
--report-file recommended-tests-report.md
# Same report as the step summary, downloadable from the run page.
- name: Upload recommended tests report
continue-on-error: true
if: always()
uses: actions/upload-artifact@v4
with:
name: recommended-tests-report
path: recommended-tests-report.md
if-no-files-found: warn
retention-days: 7
# ==================== Coverage Assembly & Publish ==================== #
# Assembles coverage data and publishes the baseline to the shared disk.
# !failure() && !cancelled() (not success()): skipped upstream poisons success().
setup-covstub:
needs:
[
base-a-test-npu-cov,
base-b-test-1-npu-a3,
base-b-test-2-npu-a3,
base-b-test-4-npu-a3,
base-b-test-8-npu-a3,
base-b-test-16-npu-a3,
base-c-test-acc-2-npu-a3,
base-c-test-acc-16-npu-a3,
base-c-test-perf-2-npu-a3,
base-c-test-perf-16-npu-a3,
multimodal-gen-test-1-npu-a3,
multimodal-gen-test-4-npu-a3,
]
if: ${{ !failure() && !cancelled() && inputs.coverage_mode == true }}
runs-on: linux-amd64-cpu-4
container:
image: swr.ap-southeast-1.myhuaweicloud.com/base_image/ascend-ci/sglang:main-x86
steps:
- name: Checkout code
uses: actions/checkout@v4
with:
# Pin to the event SHA (not the branch name) so every job in this run
# checks out the same commit even if the branch advances mid-run.
ref: ${{ inputs.ref || github.sha }}
- name: Collect base-a coverage artifact
# Runs on ubuntu-latest (no shared disk) — hence the artifact relay.
uses: actions/download-artifact@v4
with:
name: coverage-outputs-base-a-test-npu
path: /tmp/base-a-coverage
- name: Merge base-a coverage into shared run dir
run: |
# Artifact stages {RUN_DIR}/outputs/... — merges 1:1 into the shared layout.
CACHE_ROOT="/root/.cache/tests/precise-test"
mkdir -p "${CACHE_ROOT}"
cp -a /tmp/base-a-coverage/. "${CACHE_ROOT}/"
echo "Merged base-a coverage into ${CACHE_ROOT}:"
find "${CACHE_ROOT}" -maxdepth 3 -type d | head -20
- name: Copy sglang source to covstub
run: |
RUN_DIR="/root/.cache/tests/precise-test/${{ github.run_id }}-attempt-${{ github.run_attempt }}"
COVSTUB="${RUN_DIR}/outputs/covstub"
mkdir -p "${COVSTUB}"
cp -r python/sglang "${COVSTUB}/"
# _version.py is build-generated (git-ignored); the placeholder keeps
# coverage html from failing on a missing recorded source file.
echo '__version__ = "0.0.0"' > "${COVSTUB}/sglang/_version.py"
- name: Build test case map
shell: bash
run: |
set -euo pipefail
RUN_DIR="/root/.cache/tests/precise-test/${{ github.run_id }}-attempt-${{ github.run_attempt }}"
# Must match run_tests_with_coverage.sh: GITHUB_RUN_STARTED_AT may be
# unset on the k8s runner (only explicitly-set envs are injected), so
# fall back to local date the same way.
if [ -n "${GITHUB_RUN_STARTED_AT:-}" ]; then
COV_DATE_TAG="${GITHUB_RUN_STARTED_AT:0:10}"
COV_DATE_TAG="${COV_DATE_TAG//-/}"
else
COV_DATE_TAG="$(date +%Y%m%d)"
fi
COV_ROOT="${RUN_DIR}/outputs/sglang@${COV_DATE_TAG}"
COVSTUB="${RUN_DIR}/outputs/covstub"
# Standalone top-level deliverable, never packed inside outputs/.
MAP_FILE="${RUN_DIR}/test_case_map.json"
echo "Coverage dir: ${COV_ROOT}"
echo "Source dir: ${COVSTUB}"
# Coverage data may be absent when no test targets were enumerated
# (run_tests_with_coverage.sh never ran). Skip map building then.
if [ ! -d "${COV_ROOT}" ]; then
echo "::warning::Coverage dir not found: ${COV_ROOT}, skip building test case map"
exit 0
fi
COVERAGE_FILE_COUNT=$(find "${COV_ROOT}" -name 'coverage.*' -type f | wc -l)
echo "Found ${COVERAGE_FILE_COUNT} coverage files"
if [ "${COVERAGE_FILE_COUNT}" -eq 0 ]; then
echo "::warning::No coverage files found under ${COV_ROOT}, skip building test case map"
exit 0
fi
if ! python3 -c "import regex" 2>/dev/null; then
python3 -m pip install regex \
-i http://cache-service.nginx-pypi-cache.svc.cluster.local/pypi/simple \
--trusted-host cache-service.nginx-pypi-cache.svc.cluster.local \
--retries 3 --timeout 60 \
|| python3 -m pip install regex -i https://mirrors.tuna.tsinghua.edu.cn/pypi/web/simple
fi
python3 scripts/ci/npu/precise-test/test_selector.py \
--build-map \
--coverage-dir "${COV_ROOT}" \
--source-dir "${COVSTUB}" \
--map-file "${MAP_FILE}"
test -s "${MAP_FILE}"
du -h "${MAP_FILE}"
- name: Generate HTML coverage report
# Best-effort add-on: continue-on-error keeps the job green and later
# steps running, while a failed generation still shows a red ✗ in the UI.
continue-on-error: true
shell: bash
run: |
set -euo pipefail
RUN_DIR="/root/.cache/tests/precise-test/${{ github.run_id }}-attempt-${{ github.run_attempt }}"
# Date tag derivation must match run_tests_with_coverage.sh.
if [ -n "${GITHUB_RUN_STARTED_AT:-}" ]; then
COV_DATE_TAG="${GITHUB_RUN_STARTED_AT:0:10}"
COV_DATE_TAG="${COV_DATE_TAG//-/}"
else
COV_DATE_TAG="$(date +%Y%m%d)"
fi
COV_ROOT="${RUN_DIR}/outputs/sglang@${COV_DATE_TAG}"
COVSTUB="${RUN_DIR}/outputs/covstub"
# Standalone top-level dir, packaged into htmlcov.tar.gz at publish.
REPORT_DIR="${RUN_DIR}/htmlcov"
# Same skip conditions as map building: no data -> nothing to report.
if [ ! -d "${COV_ROOT}" ] \
|| [ "$(find "${COV_ROOT}" -name 'coverage.*' -type f | wc -l)" -eq 0 ]; then
echo "::warning::No coverage data found, skip HTML report"
exit 0
fi
# Failures below exit non-zero (errexit) — red ✗, job stays green.
# Do NOT convert failures to exit 0 behind an `if ! ( set -e; ... )`
# guard: bash ignores -e inside a subshell in a condition context,
# which silently swallowed three failures in run 38713909169.
# Pin matches the test jobs' coverage==7.8.* so combine can read
# their SQLite data.
if ! python3 -c "import coverage" 2>/dev/null; then
python3 -m pip install coverage==7.8.* \
-i http://cache-service.nginx-pypi-cache.svc.cluster.local/pypi/simple \
--trusted-host cache-service.nginx-pypi-cache.svc.cluster.local \
--retries 3 --timeout 60 \
|| python3 -m pip install coverage==7.8.* -i https://mirrors.tuna.tsinghua.edu.cn/pypi/web/simple
fi
# Alias runner-specific source paths onto covstub; first entry must
# be a real local path (no wildcards).
{
echo "[run]"
echo "branch = True"
echo "[paths]"
echo "sglang ="
echo " ${COVSTUB}/sglang/"
echo " */python/sglang/"
} > /tmp/report-rc
# --keep preserves the per-test raw coverage files: coverage-data.tar.gz
# archives them after this step for manual map rebuilds.
COVERAGE_FILE=/tmp/.coverage-all \
python3 -m coverage combine --rcfile=/tmp/report-rc --keep \
$(find "${COV_ROOT}" -name 'coverage.*' -type f)
rm -rf "${REPORT_DIR}"
COVERAGE_FILE=/tmp/.coverage-all \
python3 -m coverage html --rcfile=/tmp/report-rc \
--directory="${REPORT_DIR}"
# NPU-only view: [run] source does not filter at report time (a
# source-based rcfile yields the full report), so instead touch the
# unexecuted npu .py files into a data copy (os.walk, so the PEP 420
# namespace tree counts) and filter with --include, which works.
cp /tmp/.coverage-all /tmp/.coverage-npu
python3 - "${COVSTUB}" <<'PYEOF'
import os
import sys
from coverage import CoverageData
npu_root = os.path.join(sys.argv[1], "sglang", "srt", "hardware_backend", "npu")
data = CoverageData(basename="/tmp/.coverage-npu")
data.read()
measured = set(data.measured_files())
added = 0
for root, _, files in os.walk(npu_root):
for f in files:
if f.endswith(".py"):
p = os.path.join(root, f)
if p not in measured:
data.touch_file(p)
added += 1
data.write()
print(f"NPU view: {len(measured)} measured, touched {added} unexecuted files")
PYEOF
rm -rf "${REPORT_DIR}-npu"
COVERAGE_FILE=/tmp/.coverage-npu \
python3 -m coverage html --rcfile=/tmp/report-rc \
--include='*/sglang/srt/hardware_backend/npu/*' \
--directory="${REPORT_DIR}-npu"
# Surface the totals directly on the run page.
# GITHUB_STEP_SUMMARY may be unset on the k8s runner pool.
if [ -n "${GITHUB_STEP_SUMMARY:-}" ]; then
{
echo "### Coverage"
echo '```'
echo "python/sglang (measured files only):"
COVERAGE_FILE=/tmp/.coverage-all \
python3 -m coverage report --rcfile=/tmp/report-rc | tail -n 1
echo "python/sglang/srt/hardware_backend/npu (full-tree denominator):"
COVERAGE_FILE=/tmp/.coverage-npu \
python3 -m coverage report --rcfile=/tmp/report-rc \
--include='*/sglang/srt/hardware_backend/npu/*' | tail -n 1
echo '```'
echo "Full line-by-line report: download the \`coverage-html-report\` artifact and open \`htmlcov/index.html\` (NPU-only view: \`htmlcov-npu/index.html\`)."
} >> "${GITHUB_STEP_SUMMARY}"
fi
echo "HTML coverage report: ${REPORT_DIR}"
echo "NPU-only HTML coverage report: ${REPORT_DIR}-npu"
- name: Upload HTML coverage report
# Report upload must never block the baseline publish pipeline.
continue-on-error: true
uses: actions/upload-artifact@v4
with:
name: coverage-html-report
# Two paths -> artifact root becomes the run dir, so the archive
# holds htmlcov/ and htmlcov-npu/ side by side.
path: |
/root/.cache/tests/precise-test/${{ github.run_id }}-attempt-${{ github.run_attempt }}/htmlcov
/root/.cache/tests/precise-test/${{ github.run_id }}-attempt-${{ github.run_attempt }}/htmlcov-npu
if-no-files-found: ignore
retention-days: 14
- name: Archive outputs
run: |
RUN_DIR="/root/.cache/tests/precise-test/${{ github.run_id }}-attempt-${{ github.run_attempt }}"
# Source-only baseline: recommend mode needs covstub only, and it
# shrinks the package from ~300MB to a few MB.
tar -czf "${RUN_DIR}/outputs.tar.gz" -C "${RUN_DIR}" outputs/covstub
# Raw coverage data for manual map rebuilds: extract both archives
# under one root to reconstruct outputs/. Absent when no data.
# Date tag derivation must match run_tests_with_coverage.sh.
if [ -n "${GITHUB_RUN_STARTED_AT:-}" ]; then
COV_DATE_TAG="${GITHUB_RUN_STARTED_AT:0:10}"
COV_DATE_TAG="${COV_DATE_TAG//-/}"
else
COV_DATE_TAG="$(date +%Y%m%d)"
fi
if [ -d "${RUN_DIR}/outputs/sglang@${COV_DATE_TAG}" ]; then
tar -czf "${RUN_DIR}/coverage-data.tar.gz" -C "${RUN_DIR}" "outputs/sglang@${COV_DATE_TAG}"
fi
- name: Upload outputs and test case map
# Never blocks the publish pipeline; must run before publish mv's
# the files to the fixed PUBLISH_DIR (races with concurrent runs).
continue-on-error: true
uses: actions/upload-artifact@v4
with:
name: precise-test-outputs
# Two paths -> artifact root becomes RUN_DIR, so the archive
# holds outputs.tar.gz and test_case_map.json side by side.
path: |
/root/.cache/tests/precise-test/${{ github.run_id }}-attempt-${{ github.run_attempt }}/outputs.tar.gz
/root/.cache/tests/precise-test/${{ github.run_id }}-attempt-${{ github.run_attempt }}/test_case_map.json
if-no-files-found: ignore
retention-days: 14
- name: Upload raw coverage data
# Separate artifact keeps the ~300MB bulk out of the baseline package;
# must run before publish mv's the file to the fixed PUBLISH_DIR.
continue-on-error: true
uses: actions/upload-artifact@v4
with:
name: precise-test-coverage-data
path: /root/.cache/tests/precise-test/${{ github.run_id }}-attempt-${{ github.run_attempt }}/coverage-data.tar.gz
# Absent when all test jobs were skipped (no coverage data).
if-no-files-found: ignore
retention-days: 14
- name: Publish outputs and map to shared disk
shell: bash
run: |
set -euo pipefail
RUN_DIR="/root/.cache/tests/precise-test/${{ github.run_id }}-attempt-${{ github.run_attempt }}"
# Fixed-name copy for manual use; the recommend job's baseline lives
# in OBS (upload outputs.tar.gz + test_case_map.json manually after
# each run). coverage-data.tar.gz has no automatic consumer — no OBS
# copy, only this dir and the precise-test-coverage-data artifact.
PUBLISH_DIR="/root/.cache/tests/precise-test/coverage"
# Skip publish when no coverage data this run — never overwrite
# the historical baseline with an incomplete one.
if [ ! -s "${RUN_DIR}/test_case_map.json" ]; then
echo "::warning::test_case_map.json missing (no coverage data this run), skip publish"
exit 0
fi
mkdir -p "${PUBLISH_DIR}"
# Same-filesystem mv is atomic per file, so readers never see a
# partial file (files from two concurrent publishes can interleave).
mv "${RUN_DIR}/outputs.tar.gz" "${PUBLISH_DIR}/outputs.tar.gz"
mv "${RUN_DIR}/test_case_map.json" "${PUBLISH_DIR}/test_case_map.json"
# Guarded: absent in the unlikely case this run collected no
# coverage data while still producing a map.
if [ -f "${RUN_DIR}/coverage-data.tar.gz" ]; then
mv "${RUN_DIR}/coverage-data.tar.gz" "${PUBLISH_DIR}/coverage-data.tar.gz"
fi
# HTML reports too, fixed names — latest wins, like the baseline.
# [ -d ] guards the continue-on-error generation step. A tar failure
# aborts red after the baseline mv completed; Cleanup stays skipped,
# keeping RUN_DIR for recovery.
if [ -d "${RUN_DIR}/htmlcov" ]; then
tar -czf "${PUBLISH_DIR}/htmlcov.tar.gz" -C "${RUN_DIR}" htmlcov
fi
if [ -d "${RUN_DIR}/htmlcov-npu" ]; then
tar -czf "${PUBLISH_DIR}/htmlcov-npu.tar.gz" -C "${RUN_DIR}" htmlcov-npu
fi
echo "Published baseline to ${PUBLISH_DIR}"
ls -lh "${PUBLISH_DIR}" | tail -n +2
- name: Clean up per-run directory
# Skipped on publish failure, keeping RUN_DIR for manual recovery.
shell: bash
run: |
set -euo pipefail
RUN_DIR="/root/.cache/tests/precise-test/${{ github.run_id }}-attempt-${{ github.run_attempt }}"
rm -rf "${RUN_DIR}"
echo "Removed ${RUN_DIR}"
pr-test-npu-finish:
needs:
[
check-changes,
base-a-test-npu,
base-a-test-npu-cov,
base-b-test-1-npu-a3,
base-b-test-2-npu-a3,
base-b-test-4-npu-a3,
base-b-test-8-npu-a3,
base-b-test-16-npu-a3,
multimodal-gen-test-1-npu-a3,
multimodal-gen-test-4-npu-a3,
base-c-test-acc-2-npu-a3,
base-c-test-acc-16-npu-a3,
base-c-test-perf-2-npu-a3,
base-c-test-perf-16-npu-a3,
setup-covstub,
]
if: always()
runs-on: ubuntu-latest
steps:
- name: Check all dependent job statuses
run: |
# Convert the 'needs' context to a JSON string
json_needs='${{ toJson(needs) }}'
# Get a list of all job names from the JSON keys
job_names=$(echo "$json_needs" | jq -r 'keys_unsorted[]')
for job in $job_names; do
# For each job, extract its result
result=$(echo "$json_needs" | jq -r --arg j "$job" '.[$j].result')
# Print the job name and its result
echo "$job: $result"
# Check for failure or cancellation and exit if found
if [[ "$result" == "failure" || "$result" == "cancelled" ]]; then
echo "The above jobs failed."
exit 1
fi
done
# If the loop completes, all jobs were successful
echo "All jobs completed successfully"
exit 0