1238 lines
56 KiB
YAML
1238 lines
56 KiB
YAML
name: PR Test (NPU)
|
|
|
|
on:
|
|
schedule:
|
|
- cron: '0 12 * * *' # Run daily at 12:00 UTC
|
|
pull_request:
|
|
branches:
|
|
- main
|
|
- 'npu/**'
|
|
- 'npu-*'
|
|
workflow_dispatch:
|
|
inputs:
|
|
coverage_mode:
|
|
description: 'Run in coverage collection mode (uses coverage runners, enables coverage instrumentation, runs setup-covstub)'
|
|
required: false
|
|
type: boolean
|
|
default: false
|
|
workflow_call:
|
|
inputs:
|
|
ref:
|
|
description: 'Git ref (branch, tag, or SHA) to test. If not provided, uses the event commit SHA.'
|
|
required: false
|
|
type: string
|
|
default: ''
|
|
run_all_tests:
|
|
description: "Run all tests (for releasing or testing purpose)"
|
|
required: false
|
|
type: boolean
|
|
default: false
|
|
coverage_mode:
|
|
description: 'Run in coverage collection mode (uses coverage runners, enables coverage instrumentation, runs setup-covstub)'
|
|
required: false
|
|
type: boolean
|
|
default: false
|
|
|
|
concurrency:
|
|
group: pr-test-npu-${{ inputs.coverage_mode == true && 'coverage-' || '' }}${{ inputs.ref || github.ref }}
|
|
cancel-in-progress: ${{ github.event_name != 'workflow_call' }}
|
|
|
|
jobs:
|
|
# ==================== Check Changes ==================== #
|
|
check-changes:
|
|
runs-on: ubuntu-latest
|
|
outputs:
|
|
changes_exist: ${{ steps.filter.outputs.main_package == 'true' || steps.filter.outputs.multimodal_gen == 'true' || steps.run-mode.outputs.run_all_tests == 'true'}}
|
|
main_package: ${{ steps.filter.outputs.main_package == 'true' || steps.run-mode.outputs.run_all_tests == 'true' }}
|
|
multimodal_gen: ${{ steps.filter.outputs.multimodal_gen == 'true' || steps.run-mode.outputs.run_all_tests == 'true' }}
|
|
steps:
|
|
- name: Checkout code
|
|
uses: actions/checkout@v4
|
|
with:
|
|
# Pin to the event SHA (not the branch name) so every job in this run
|
|
# checks out the same commit even if the branch advances mid-run.
|
|
ref: ${{ inputs.ref || github.sha }}
|
|
|
|
- name: Determine run mode
|
|
id: run-mode
|
|
run: |
|
|
# Run all tests for scheduled runs and workflow_call (when ref input is provided)
|
|
# Note: github.event_name is inherited from caller, so we detect workflow_call by checking inputs.ref
|
|
if [[ "${{ github.event_name }}" == "schedule" || "${{ inputs.run_all_tests }}" == "true" ]]; then
|
|
echo "run_all_tests=true" >> $GITHUB_OUTPUT
|
|
echo "Run mode: ALL TESTS (schedule=${{ github.event_name == 'schedule' }}, run_all_tests=${{ inputs.run_all_tests }})"
|
|
else
|
|
echo "run_all_tests=false" >> $GITHUB_OUTPUT
|
|
echo "Run mode: FILTERED (triggered by ${{ github.event_name }})"
|
|
fi
|
|
|
|
- name: Detect file changes
|
|
id: filter
|
|
uses: dorny/paths-filter@v3
|
|
if: steps.run-mode.outputs.run_all_tests != 'true'
|
|
with:
|
|
filters: |
|
|
# Everything under python/sglang except multimodal_gen/ and kernels/ops/diffusion/.
|
|
# Exclusions have to be extglobs inside a single pattern: paths-filter ORs the
|
|
# patterns of a filter (predicate-quantifier defaults to "some"), so a standalone
|
|
# "!some/path/**" entry matches every file outside that path and makes the whole
|
|
# filter unconditionally true.
|
|
main_package:
|
|
- "python/sglang/!(*.md)"
|
|
- "python/sglang/!(multimodal_gen|kernels)/**/!(*.md)"
|
|
- "python/sglang/kernels/!(*.md)"
|
|
- "python/sglang/kernels/!(ops)/**/!(*.md)"
|
|
- "python/sglang/kernels/ops/!(*.md)"
|
|
- "python/sglang/kernels/ops/!(diffusion)/**/!(*.md)"
|
|
- "python/pyproject_npu.toml"
|
|
- "scripts/ci/npu/npu_ci_install_dependency.sh"
|
|
- "test/registered/npu/**"
|
|
- "test/registered/unit/npu/**"
|
|
- ".github/workflows/pr-test-npu.yml"
|
|
multimodal_gen:
|
|
- "python/sglang/multimodal_gen/**/!(*.md|*.ipynb)"
|
|
- "python/sglang/kernels/ops/diffusion/**"
|
|
- "python/sglang/srt/**"
|
|
- "python/pyproject_npu.toml"
|
|
- "scripts/ci/npu/npu_ci_install_dependency.sh"
|
|
- ".github/workflows/pr-test-npu.yml"
|
|
|
|
# ==================== PR Gate ==================== #
|
|
pr-gate:
|
|
needs: check-changes
|
|
if: ${{ inputs.coverage_mode != true && needs.check-changes.outputs.changes_exist == 'true' }}
|
|
uses: ./.github/workflows/pr-gate.yml
|
|
secrets: inherit
|
|
|
|
set-image-config:
|
|
runs-on: ubuntu-latest
|
|
outputs:
|
|
CANN_image_a3: ${{ steps.set-vars.outputs.CANN_image_a3 }}
|
|
steps:
|
|
# When triggered by PR, no inputs parameters are used. The latest community code is tested by default.
|
|
- name: Set image config
|
|
id: set-vars
|
|
run: |
|
|
echo "CANN_image_a3=swr.cn-southwest-2.myhuaweicloud.com/base_image/ascend-ci/cann:9.1.0-a3-ubuntu22.04-py3.12" >> $GITHUB_OUTPUT
|
|
|
|
# Stage-A gate: CPU-only unit tests. The NPU attention-backend UTs stub out
|
|
# torch_npu / sgl_kernel_npu before importing, so a hosted runner suffices.
|
|
base-a-test-npu:
|
|
needs: [check-changes, pr-gate]
|
|
if: ${{ !failure() && !cancelled() && inputs.coverage_mode != true && needs.check-changes.outputs.main_package == 'true' }}
|
|
runs-on: ubuntu-latest
|
|
timeout-minutes: 45
|
|
env:
|
|
SGLANG_IS_IN_CI: true
|
|
steps:
|
|
- name: Checkout code
|
|
uses: actions/checkout@v4
|
|
with:
|
|
# Pin to the event SHA so this job tests the same commit as the rest of the run.
|
|
ref: ${{ inputs.ref || github.sha }}
|
|
|
|
# Same gate the NPU stages run first: fail before the NPU jobs are scheduled.
|
|
- uses: ./.github/actions/check-pr-test-health
|
|
|
|
- name: Set up Python
|
|
uses: actions/setup-python@v5
|
|
with:
|
|
python-version: '3.12'
|
|
|
|
- name: Install uv
|
|
uses: astral-sh/setup-uv@v5
|
|
with:
|
|
enable-cache: true
|
|
cache-dependency-glob: 'python/pyproject_npu.toml'
|
|
|
|
- name: Install dependencies
|
|
timeout-minutes: 30
|
|
env:
|
|
# uv pip targets a venv by default; setup-python has no venv.
|
|
UV_SYSTEM_PYTHON: "1"
|
|
run: |
|
|
# Resolve the same dependency set as the NPU stages, minus Ascend-only wheels.
|
|
rm -f python/pyproject.toml
|
|
mv python/pyproject_npu.toml python/pyproject.toml
|
|
# Versions mirror npu_ci_install_dependency.sh's defaults; keep them in sync.
|
|
# triton: sglang.srt.utils.common imports it unconditionally and the CPU
|
|
# wheel does not pull it in (the NPU image uses triton-ascend instead).
|
|
# unsafe-best-match: --index-url is the lowest-priority index, so the
|
|
# default strategy would resolve PyPI's CUDA torch, not the +cpu build.
|
|
uv pip install torch==2.10.0 torchvision==0.25.0 torchaudio==2.10.0 triton \
|
|
--index-url https://download.pytorch.org/whl/cpu \
|
|
--extra-index-url https://pypi.org/simple/ \
|
|
--index-strategy unsafe-best-match
|
|
uv pip install -e "python[dev_npu]" --index-strategy unsafe-best-match
|
|
|
|
- name: Run NPU unit tests
|
|
timeout-minutes: 10
|
|
run: |
|
|
cd test
|
|
python3 run_suite.py --hw npu --suite base-a-test-npu
|
|
|
|
# Coverage-mode-only counterpart of the base-a PR gate above. Stays on
|
|
# ubuntu-latest like the gate (its UTs mock the NPU stack and need a CPU
|
|
# triton; the NPU image's triton-ascend breaks those mocks), so the shared
|
|
# template (unconditional CANN container:) cannot be used and coverage
|
|
# data reaches setup-covstub via artifact instead of the shared disk.
|
|
base-a-test-npu-cov:
|
|
needs: [set-image-config]
|
|
if: ${{ inputs.coverage_mode == true }}
|
|
runs-on: ubuntu-latest
|
|
timeout-minutes: 45
|
|
env:
|
|
SGLANG_IS_IN_CI: true
|
|
steps:
|
|
- name: Checkout code
|
|
uses: actions/checkout@v4
|
|
with:
|
|
# Pin to the event SHA so this job tests the same commit as the rest of the run.
|
|
ref: ${{ inputs.ref || github.sha }}
|
|
|
|
- name: Set up Python
|
|
uses: actions/setup-python@v5
|
|
with:
|
|
python-version: '3.12'
|
|
|
|
- name: Install uv
|
|
uses: astral-sh/setup-uv@v5
|
|
with:
|
|
enable-cache: true
|
|
cache-dependency-glob: 'python/pyproject_npu.toml'
|
|
|
|
- name: Install dependencies
|
|
timeout-minutes: 30
|
|
env:
|
|
# uv pip targets a venv by default; setup-python has no venv.
|
|
UV_SYSTEM_PYTHON: "1"
|
|
run: |
|
|
# Same dependency set as the base-a PR gate above (CPU torch + CPU
|
|
# triton, no Ascend wheels); keep the two blocks in sync.
|
|
rm -f python/pyproject.toml
|
|
mv python/pyproject_npu.toml python/pyproject.toml
|
|
uv pip install torch==2.10.0 torchvision==0.25.0 torchaudio==2.10.0 triton \
|
|
--index-url https://download.pytorch.org/whl/cpu \
|
|
--extra-index-url https://pypi.org/simple/ \
|
|
--index-strategy unsafe-best-match
|
|
uv pip install -e "python[dev_npu]" --index-strategy unsafe-best-match
|
|
pip install coverage==7.8.*
|
|
|
|
- name: Run test with coverage
|
|
timeout-minutes: 15
|
|
run: |
|
|
# Hosted runners cannot write the shared disk, so stage locally;
|
|
# run_tests_with_coverage.sh derives RUN_DIR/date tag from run-level
|
|
# env vars, so the artifact merges 1:1 into the shared-disk layout.
|
|
export PRECISE_TEST_CACHE_ROOT=/tmp/precise-test
|
|
cd test
|
|
python3 ../scripts/ci/npu/precise-test/list_tests.py --hw npu --suite base-a-test-npu \
|
|
-o /tmp/selected_tests.txt
|
|
echo "Selected test files:"
|
|
cat /tmp/selected_tests.txt
|
|
if [ -s /tmp/selected_tests.txt ]; then
|
|
bash ../scripts/ci/npu/precise-test/run_tests_with_coverage.sh $(cat /tmp/selected_tests.txt)
|
|
fi
|
|
|
|
- name: Upload coverage outputs
|
|
if: always()
|
|
uses: actions/upload-artifact@v4
|
|
with:
|
|
name: coverage-outputs-base-a-test-npu
|
|
path: /tmp/precise-test/
|
|
retention-days: 1
|
|
if-no-files-found: warn
|
|
|
|
base-b-test-1-npu-a3:
|
|
needs: [check-changes, pr-gate, set-image-config, base-a-test-npu, base-a-test-npu-cov, recommend-tests-from-coverage]
|
|
if: >-
|
|
${{ !failure() && !cancelled() &&
|
|
(inputs.coverage_mode == true || needs.check-changes.outputs.main_package == 'true') &&
|
|
needs.recommend-tests-from-coverage.outputs.should_run_base_b_test_1_npu_a3 != 'false' }}
|
|
uses: ./.github/workflows/_npu-pr-test-stage.yml
|
|
with:
|
|
self_name: base-b-test-1-npu-a3
|
|
runner_config: ${{ inputs.coverage_mode == true && 'linux-aarch64-a3-800t-2' || 'linux-aarch64-a3-2-' }}
|
|
image: ${{ needs.set-image-config.outputs.CANN_image_a3 }}
|
|
run_timeout_minutes: '60'
|
|
timeout_per_file: '3600'
|
|
only_tests: ${{ needs.recommend-tests-from-coverage.outputs.only_tests_base_b_test_1_npu_a3 }}
|
|
use_coverage_runner: ${{ inputs.coverage_mode == true }}
|
|
secrets: inherit
|
|
|
|
base-b-test-2-npu-a3:
|
|
needs: [check-changes, pr-gate, set-image-config, base-a-test-npu, base-a-test-npu-cov, recommend-tests-from-coverage]
|
|
if: >-
|
|
${{ !failure() && !cancelled() &&
|
|
(inputs.coverage_mode == true || needs.check-changes.outputs.main_package == 'true') &&
|
|
needs.recommend-tests-from-coverage.outputs.should_run_base_b_test_2_npu_a3 != 'false' }}
|
|
uses: ./.github/workflows/_npu-pr-test-stage.yml
|
|
with:
|
|
self_name: base-b-test-2-npu-a3
|
|
runner_config: ${{ inputs.coverage_mode == true && 'linux-aarch64-a3-800t-2' || 'linux-aarch64-a3-2-' }}
|
|
image: ${{ needs.set-image-config.outputs.CANN_image_a3 }}
|
|
run_timeout_minutes: '60'
|
|
timeout_per_file: '3600'
|
|
only_tests: ${{ needs.recommend-tests-from-coverage.outputs.only_tests_base_b_test_2_npu_a3 }}
|
|
use_coverage_runner: ${{ inputs.coverage_mode == true }}
|
|
secrets: inherit
|
|
|
|
base-b-test-4-npu-a3:
|
|
needs: [check-changes, pr-gate, set-image-config, base-a-test-npu, base-a-test-npu-cov, recommend-tests-from-coverage]
|
|
if: >-
|
|
${{ !failure() && !cancelled() &&
|
|
(inputs.coverage_mode == true || needs.check-changes.outputs.main_package == 'true') &&
|
|
needs.recommend-tests-from-coverage.outputs.should_run_base_b_test_4_npu_a3 != 'false' }}
|
|
uses: ./.github/workflows/_npu-pr-test-stage.yml
|
|
with:
|
|
self_name: base-b-test-4-npu-a3
|
|
runner_config: ${{ inputs.coverage_mode == true && 'linux-aarch64-a3-800t-4' || 'linux-aarch64-a3-4-' }}
|
|
image: ${{ needs.set-image-config.outputs.CANN_image_a3 }}
|
|
run_timeout_minutes: '120'
|
|
timeout_per_file: '3600'
|
|
partitions: '{"size":2,"arr":[0, 1]}'
|
|
only_tests: ${{ needs.recommend-tests-from-coverage.outputs.only_tests_base_b_test_4_npu_a3 }}
|
|
use_coverage_runner: ${{ inputs.coverage_mode == true }}
|
|
secrets: inherit
|
|
|
|
base-b-test-8-npu-a3:
|
|
needs: [check-changes, pr-gate, set-image-config, base-a-test-npu, base-a-test-npu-cov, recommend-tests-from-coverage]
|
|
if: >-
|
|
${{ !failure() && !cancelled() &&
|
|
(inputs.coverage_mode == true || needs.check-changes.outputs.main_package == 'true') &&
|
|
needs.recommend-tests-from-coverage.outputs.should_run_base_b_test_8_npu_a3 != 'false' }}
|
|
uses: ./.github/workflows/_npu-pr-test-stage.yml
|
|
with:
|
|
self_name: base-b-test-8-npu-a3
|
|
runner_config: ${{ inputs.coverage_mode == true && 'linux-aarch64-a3-800t-8' || 'linux-aarch64-a3-8-' }}
|
|
image: ${{ needs.set-image-config.outputs.CANN_image_a3 }}
|
|
run_timeout_minutes: '60'
|
|
timeout_per_file: '3600'
|
|
only_tests: ${{ needs.recommend-tests-from-coverage.outputs.only_tests_base_b_test_8_npu_a3 }}
|
|
use_coverage_runner: ${{ inputs.coverage_mode == true }}
|
|
secrets: inherit
|
|
|
|
base-b-test-16-npu-a3:
|
|
needs: [check-changes, pr-gate, set-image-config, base-a-test-npu, base-a-test-npu-cov, recommend-tests-from-coverage]
|
|
if: >-
|
|
${{ !failure() && !cancelled() &&
|
|
(inputs.coverage_mode == true || needs.check-changes.outputs.main_package == 'true') &&
|
|
needs.recommend-tests-from-coverage.outputs.should_run_base_b_test_16_npu_a3 != 'false' }}
|
|
uses: ./.github/workflows/_npu-pr-test-stage.yml
|
|
with:
|
|
self_name: base-b-test-16-npu-a3
|
|
runner_config: ${{ inputs.coverage_mode == true && 'linux-aarch64-a3-800t-16' || 'linux-aarch64-a3-16-' }}
|
|
image: ${{ needs.set-image-config.outputs.CANN_image_a3 }}
|
|
run_timeout_minutes: '120'
|
|
timeout_per_file: '3600'
|
|
only_tests: ${{ needs.recommend-tests-from-coverage.outputs.only_tests_base_b_test_16_npu_a3 }}
|
|
use_coverage_runner: ${{ inputs.coverage_mode == true }}
|
|
secrets: inherit
|
|
|
|
multimodal-gen-test-1-npu-a3:
|
|
needs:
|
|
[
|
|
check-changes,
|
|
pr-gate,
|
|
set-image-config,
|
|
base-a-test-npu,
|
|
base-a-test-npu-cov,
|
|
recommend-tests-from-coverage,
|
|
]
|
|
if: >-
|
|
${{ !failure() && !cancelled() &&
|
|
(inputs.coverage_mode == true || needs.check-changes.outputs.multimodal_gen == 'true') &&
|
|
needs.recommend-tests-from-coverage.outputs.should_run_multimodal_gen_test_1_npu_a3 != 'false' }}
|
|
runs-on: linux-aarch64-a3-800t-2
|
|
strategy:
|
|
fail-fast: false
|
|
matrix:
|
|
part: [0, 1]
|
|
container:
|
|
image: ${{ needs.set-image-config.outputs.CANN_image_a3 }}
|
|
steps:
|
|
- name: Checkout code
|
|
uses: actions/checkout@v4
|
|
|
|
- name: Mark repository safe
|
|
run: |
|
|
git config --system --add safe.directory ${GITHUB_WORKSPACE}
|
|
|
|
- name: Install dependencies
|
|
env:
|
|
TORCH_CACHE_URL: "http://cache-service.nginx-pypi-cache.svc.cluster.local/whl/cpu"
|
|
PYPI_CACHE_URL: "http://cache-service.nginx-pypi-cache.svc.cluster.local/pypi/simple"
|
|
UV_INDEX_URL: "http://cache-service.nginx-pypi-cache.svc.cluster.local/pypi/simple"
|
|
GITHUB_PROXY_URL: "https://gh-proxy.test.osinfra.cn/"
|
|
RUSTUP_CACHE_URL: "http://cache-service.nginx-pypi-cache.svc.cluster.local:8082"
|
|
run: |
|
|
# speed up by using infra cache services
|
|
CACHING_URL="cache-service.nginx-pypi-cache.svc.cluster.local"
|
|
sed -Ei "s@(ports|archive).ubuntu.com@${CACHING_URL}:8081@g" /etc/apt/sources.list
|
|
pip config set global.index-url http://${CACHING_URL}/pypi/simple
|
|
pip config set global.trusted-host "${CACHING_URL}"
|
|
|
|
bash scripts/ci/npu/npu_ci_install_dependency.sh a3
|
|
# copy required file from our daily cache
|
|
cp ~/.cache/modelscope/hub/datasets/otavia/ShareGPT_Vicuna_unfiltered/ShareGPT_V3_unfiltered_cleaned_split.json /tmp
|
|
# copy gsm8k dataset from cache
|
|
cp ~/.cache/modelscope/hub/datasets/tmp/test.jsonl /tmp
|
|
|
|
- name: Prefetch consistency GT
|
|
env:
|
|
GITHUB_PROXY_URL: "https://gh-proxy.test.osinfra.cn/"
|
|
run: |
|
|
python3 scripts/ci/utils/diffusion/prefetch_consistency_gt.py
|
|
|
|
- name: Prefetch image-edit input
|
|
env:
|
|
GITHUB_PROXY_URL: "https://gh-proxy.test.osinfra.cn/"
|
|
run: python3 scripts/ci/utils/diffusion/prefetch_input_image.py
|
|
|
|
- name: Run test
|
|
timeout-minutes: ${{ inputs.coverage_mode == true && 150 || 60 }}
|
|
shell: bash
|
|
env:
|
|
SGLANG_USE_MODELSCOPE: true
|
|
SGLANG_IS_IN_CI: true
|
|
HF_ENDPOINT: https://hf-mirror.com
|
|
TORCH_EXTENSIONS_DIR: /tmp/torch_extensions
|
|
PYTORCH_NPU_ALLOC_CONF: "expandable_segments:True"
|
|
STREAMS_PER_DEVICE: 32
|
|
SGLANG_DIFFUSION_ARTIFACT_DIR: ${{ github.workspace }}/diffusion-failures
|
|
ONLY_TESTS: ${{ needs.recommend-tests-from-coverage.outputs.only_tests_multimodal_gen_test_1_npu_a3 }}
|
|
run: |
|
|
# Coverage mode: enumerate this shard's targets with an NPU-only CI
|
|
# helper (it reuses the shared diffusion runner read-only) and run
|
|
# each target in its own `coverage run -m pytest` invocation. The
|
|
# runner normally batches a whole shard into one pytest call, which
|
|
# cannot be attributed per test case by the precise-test selector.
|
|
if [[ "${{ inputs.coverage_mode }}" == "true" ]]; then
|
|
set -euo pipefail
|
|
pip install coverage==7.8.*
|
|
# Coverage tracing slows down the Python glue, which shows up in
|
|
# wall-clock assertions. Relax only the timing tolerances for the
|
|
# collection run; PR runs (no coverage) keep the strict profile and
|
|
# the memory/VRAM checks are untouched.
|
|
export SGLANG_E2E_TOLERANCE=1.0
|
|
export SGLANG_STAGE_TIME_TOLERANCE=1.0
|
|
export SGLANG_NON_DENOISE_STAGE_TIME_TOLERANCE=2.0
|
|
export SGLANG_DENOISE_STEP_TOLERANCE=1.0
|
|
export SGLANG_DENOISE_AGG_TOLERANCE=1.0
|
|
python3 scripts/ci/npu/precise-test/list_multimodal_tests.py \
|
|
--suite 1-npu \
|
|
--partition-id ${{ matrix.part }} \
|
|
--total-partitions 2 \
|
|
-o /tmp/selected_tests.txt
|
|
echo "Selected test targets:"
|
|
cat /tmp/selected_tests.txt
|
|
if [ -s /tmp/selected_tests.txt ]; then
|
|
bash scripts/ci/npu/precise-test/run_tests_with_coverage.sh $(cat /tmp/selected_tests.txt)
|
|
fi
|
|
exit 0
|
|
fi
|
|
|
|
# Precision mode: ONLY_TESTS holds one pytest -k token per line
|
|
# ('[case_id]' for parametrized cases, a file stem for standalone
|
|
# files). It is empty when nothing was recommended, precision mode
|
|
# is off, or a safety gate forced the full suite; then no filter is
|
|
# applied and the shard runs its whole assignment.
|
|
FILTER=""
|
|
while IFS= read -r line; do
|
|
if [ -z "${line}" ]; then
|
|
continue
|
|
fi
|
|
# Every line must be a -k token ('[case_id]' or a standalone file
|
|
# stem). A path means the bucket contract drifted: with a path the
|
|
# -k expression matches nothing, pytest exits 5, and the shard
|
|
# would report success after running zero tests.
|
|
if [[ "${line}" == */* || "${line}" == *.py ]]; then
|
|
echo "::error::Unexpected only_tests entry '${line}' (expected a pytest -k token)"
|
|
exit 1
|
|
fi
|
|
if [ -n "${FILTER}" ]; then
|
|
FILTER="${FILTER} or "
|
|
fi
|
|
FILTER="${FILTER}${line}"
|
|
done <<< "${ONLY_TESTS}"
|
|
FILTER_ARGS=()
|
|
if [ -n "${FILTER}" ]; then
|
|
FILTER_ARGS+=(-k "${FILTER}")
|
|
echo "Precision mode: filtered to ${FILTER}"
|
|
else
|
|
echo "Precision mode: no filter (full suite)"
|
|
fi
|
|
|
|
cd python
|
|
python3 sglang/multimodal_gen/test/run_suite.py \
|
|
--suite 1-npu \
|
|
--partition-id ${{ matrix.part }} \
|
|
--total-partitions 2 \
|
|
"${FILTER_ARGS[@]}"
|
|
|
|
- name: Upload diffusion failure artifacts
|
|
if: always()
|
|
uses: actions/upload-artifact@v4
|
|
with:
|
|
name: diffusion-failures-npu-1-part${{ matrix.part }}-${{ github.run_attempt }}
|
|
path: diffusion-failures/
|
|
if-no-files-found: ignore
|
|
retention-days: 7
|
|
|
|
multimodal-gen-test-4-npu-a3:
|
|
needs:
|
|
[
|
|
check-changes,
|
|
pr-gate,
|
|
set-image-config,
|
|
base-a-test-npu,
|
|
base-a-test-npu-cov,
|
|
recommend-tests-from-coverage,
|
|
]
|
|
if: >-
|
|
${{ !failure() && !cancelled() &&
|
|
(inputs.coverage_mode == true || needs.check-changes.outputs.multimodal_gen == 'true') &&
|
|
needs.recommend-tests-from-coverage.outputs.should_run_multimodal_gen_test_4_npu_a3 != 'false' }}
|
|
runs-on: linux-aarch64-a3-800t-4
|
|
strategy:
|
|
fail-fast: false
|
|
matrix:
|
|
part: [0, 1]
|
|
container:
|
|
image: ${{ needs.set-image-config.outputs.CANN_image_a3 }}
|
|
steps:
|
|
- name: Checkout code
|
|
uses: actions/checkout@v4
|
|
|
|
- name: Mark repository safe
|
|
run: |
|
|
git config --system --add safe.directory ${GITHUB_WORKSPACE}
|
|
|
|
- name: Install dependencies
|
|
env:
|
|
TORCH_CACHE_URL: "http://cache-service.nginx-pypi-cache.svc.cluster.local/whl/cpu"
|
|
PYPI_CACHE_URL: "http://cache-service.nginx-pypi-cache.svc.cluster.local/pypi/simple"
|
|
UV_INDEX_URL: "http://cache-service.nginx-pypi-cache.svc.cluster.local/pypi/simple"
|
|
GITHUB_PROXY_URL: "https://gh-proxy.test.osinfra.cn/"
|
|
RUSTUP_CACHE_URL: "http://cache-service.nginx-pypi-cache.svc.cluster.local:8082"
|
|
run: |
|
|
# speed up by using infra cache services
|
|
CACHING_URL="cache-service.nginx-pypi-cache.svc.cluster.local"
|
|
sed -Ei "s@(ports|archive).ubuntu.com@${CACHING_URL}:8081@g" /etc/apt/sources.list
|
|
pip config set global.index-url http://${CACHING_URL}/pypi/simple
|
|
pip config set global.trusted-host "${CACHING_URL}"
|
|
|
|
bash scripts/ci/npu/npu_ci_install_dependency.sh a3
|
|
# copy required file from our daily cache
|
|
cp ~/.cache/modelscope/hub/datasets/otavia/ShareGPT_Vicuna_unfiltered/ShareGPT_V3_unfiltered_cleaned_split.json /tmp
|
|
# copy gsm8k dataset
|
|
cp ~/.cache/modelscope/hub/datasets/tmp/test.jsonl /tmp
|
|
|
|
- name: Prefetch consistency GT
|
|
env:
|
|
GITHUB_PROXY_URL: "https://gh-proxy.test.osinfra.cn/"
|
|
run: |
|
|
python3 scripts/ci/utils/diffusion/prefetch_consistency_gt.py
|
|
|
|
- name: Run test
|
|
timeout-minutes: ${{ inputs.coverage_mode == true && 150 || 60 }}
|
|
shell: bash
|
|
env:
|
|
SGLANG_USE_MODELSCOPE: true
|
|
SGLANG_IS_IN_CI: true
|
|
HF_ENDPOINT: https://hf-mirror.com
|
|
TORCH_EXTENSIONS_DIR: /tmp/torch_extensions
|
|
PYTORCH_NPU_ALLOC_CONF: "expandable_segments:True"
|
|
STREAMS_PER_DEVICE: 32
|
|
SGLANG_DIFFUSION_ARTIFACT_DIR: ${{ github.workspace }}/diffusion-failures
|
|
ONLY_TESTS: ${{ needs.recommend-tests-from-coverage.outputs.only_tests_multimodal_gen_test_4_npu_a3 }}
|
|
run: |
|
|
# Coverage mode: enumerate this shard's targets with an NPU-only CI
|
|
# helper (it reuses the shared diffusion runner read-only) and run
|
|
# each target in its own `coverage run -m pytest` invocation. The
|
|
# runner normally batches a whole shard into one pytest call, which
|
|
# cannot be attributed per test case by the precise-test selector.
|
|
if [[ "${{ inputs.coverage_mode }}" == "true" ]]; then
|
|
set -euo pipefail
|
|
pip install coverage==7.8.*
|
|
# Coverage tracing slows down the Python glue, which shows up in
|
|
# wall-clock assertions. Relax only the timing tolerances for the
|
|
# collection run; PR runs (no coverage) keep the strict profile and
|
|
# the memory/VRAM checks are untouched.
|
|
export SGLANG_E2E_TOLERANCE=1.0
|
|
export SGLANG_STAGE_TIME_TOLERANCE=1.0
|
|
export SGLANG_NON_DENOISE_STAGE_TIME_TOLERANCE=2.0
|
|
export SGLANG_DENOISE_STEP_TOLERANCE=1.0
|
|
export SGLANG_DENOISE_AGG_TOLERANCE=1.0
|
|
python3 scripts/ci/npu/precise-test/list_multimodal_tests.py \
|
|
--suite 2-npu \
|
|
--partition-id ${{ matrix.part }} \
|
|
--total-partitions 2 \
|
|
-o /tmp/selected_tests.txt
|
|
echo "Selected test targets:"
|
|
cat /tmp/selected_tests.txt
|
|
if [ -s /tmp/selected_tests.txt ]; then
|
|
bash scripts/ci/npu/precise-test/run_tests_with_coverage.sh $(cat /tmp/selected_tests.txt)
|
|
fi
|
|
exit 0
|
|
fi
|
|
|
|
# Precision mode: ONLY_TESTS holds one pytest -k token per line
|
|
# ('[case_id]' for parametrized cases, a file stem for standalone
|
|
# files). It is empty when nothing was recommended, precision mode
|
|
# is off, or a safety gate forced the full suite; then no filter is
|
|
# applied and the shard runs its whole assignment.
|
|
FILTER=""
|
|
while IFS= read -r line; do
|
|
if [ -z "${line}" ]; then
|
|
continue
|
|
fi
|
|
# Every line must be a -k token ('[case_id]' or a standalone file
|
|
# stem). A path means the bucket contract drifted: with a path the
|
|
# -k expression matches nothing, pytest exits 5, and the shard
|
|
# would report success after running zero tests.
|
|
if [[ "${line}" == */* || "${line}" == *.py ]]; then
|
|
echo "::error::Unexpected only_tests entry '${line}' (expected a pytest -k token)"
|
|
exit 1
|
|
fi
|
|
if [ -n "${FILTER}" ]; then
|
|
FILTER="${FILTER} or "
|
|
fi
|
|
FILTER="${FILTER}${line}"
|
|
done <<< "${ONLY_TESTS}"
|
|
FILTER_ARGS=()
|
|
if [ -n "${FILTER}" ]; then
|
|
FILTER_ARGS+=(-k "${FILTER}")
|
|
echo "Precision mode: filtered to ${FILTER}"
|
|
else
|
|
echo "Precision mode: no filter (full suite)"
|
|
fi
|
|
|
|
cd python
|
|
python3 sglang/multimodal_gen/test/run_suite.py \
|
|
--suite 2-npu \
|
|
--partition-id ${{ matrix.part }} \
|
|
--total-partitions 2 \
|
|
"${FILTER_ARGS[@]}"
|
|
|
|
- name: Upload diffusion failure artifacts
|
|
if: always()
|
|
uses: actions/upload-artifact@v4
|
|
with:
|
|
name: diffusion-failures-npu-2-part${{ matrix.part }}-${{ github.run_attempt }}
|
|
path: diffusion-failures/
|
|
if-no-files-found: ignore
|
|
retention-days: 7
|
|
|
|
base-c-test-acc-2-npu-a3:
|
|
name: base-c-test-acc-2-npu-a3
|
|
needs: [ check-changes, pr-gate, set-image-config, base-b-test-1-npu-a3, base-b-test-2-npu-a3, base-b-test-4-npu-a3, base-b-test-8-npu-a3, base-b-test-16-npu-a3, recommend-tests-from-coverage ]
|
|
if: >-
|
|
${{ !failure() && !cancelled() &&
|
|
(inputs.coverage_mode == true || needs.check-changes.outputs.main_package == 'true') &&
|
|
needs.recommend-tests-from-coverage.outputs.should_run_base_c_test_acc_2_npu_a3 != 'false' }}
|
|
uses: ./.github/workflows/_npu-single-node-test-stage.yml
|
|
with:
|
|
runner: ${{ inputs.coverage_mode == true && 'linux-aarch64-a3-800t-2' || 'linux-aarch64-a3-2-' }}
|
|
test_type: 'accuracy'
|
|
test_suite: base-c-test-acc-2-npu-a3
|
|
image: ${{ needs.set-image-config.outputs.CANN_image_a3 }}
|
|
install_sglang_deps: true
|
|
device_type_for_deps: 'a3'
|
|
partitions: '{"size":2,"arr":[0,1]}'
|
|
only_tests: ${{ needs.recommend-tests-from-coverage.outputs.only_tests_base_c_test_acc_2_npu_a3 }}
|
|
use_coverage_runner: ${{ inputs.coverage_mode == true }}
|
|
|
|
base-c-test-acc-16-npu-a3:
|
|
name: base-c-test-acc-16-npu-a3
|
|
needs: [ check-changes, pr-gate, set-image-config, base-b-test-1-npu-a3, base-b-test-2-npu-a3, base-b-test-4-npu-a3, base-b-test-8-npu-a3, base-b-test-16-npu-a3, recommend-tests-from-coverage ]
|
|
if: >-
|
|
${{ !failure() && !cancelled() &&
|
|
(inputs.coverage_mode == true || needs.check-changes.outputs.main_package == 'true') &&
|
|
needs.recommend-tests-from-coverage.outputs.should_run_base_c_test_acc_16_npu_a3 != 'false' }}
|
|
uses: ./.github/workflows/_npu-single-node-test-stage.yml
|
|
with:
|
|
runner: ${{ inputs.coverage_mode == true && 'linux-aarch64-a3-800t-16' || 'linux-aarch64-a3-16-' }}
|
|
test_type: 'accuracy'
|
|
test_suite: base-c-test-acc-16-npu-a3
|
|
image: ${{ needs.set-image-config.outputs.CANN_image_a3 }}
|
|
install_sglang_deps: true
|
|
device_type_for_deps: 'a3'
|
|
only_tests: ${{ needs.recommend-tests-from-coverage.outputs.only_tests_base_c_test_acc_16_npu_a3 }}
|
|
use_coverage_runner: ${{ inputs.coverage_mode == true }}
|
|
|
|
base-c-test-perf-2-npu-a3:
|
|
name: base-c-test-perf-2-npu-a3
|
|
needs: [ check-changes, pr-gate, set-image-config, base-c-test-acc-2-npu-a3, base-c-test-acc-16-npu-a3, recommend-tests-from-coverage ]
|
|
if: >-
|
|
${{ !failure() && !cancelled() &&
|
|
(inputs.coverage_mode == true || needs.check-changes.outputs.main_package == 'true') &&
|
|
needs.recommend-tests-from-coverage.outputs.should_run_base_c_test_perf_2_npu_a3 != 'false' }}
|
|
uses: ./.github/workflows/_npu-single-node-test-stage.yml
|
|
with:
|
|
runner: linux-aarch64-a3-800t-2
|
|
test_type: 'perf'
|
|
test_suite: base-c-test-perf-2-npu-a3
|
|
image: ${{ needs.set-image-config.outputs.CANN_image_a3 }}
|
|
install_sglang_deps: true
|
|
device_type_for_deps: 'a3'
|
|
only_tests: ${{ needs.recommend-tests-from-coverage.outputs.only_tests_base_c_test_perf_2_npu_a3 }}
|
|
use_coverage_runner: ${{ inputs.coverage_mode == true }}
|
|
|
|
base-c-test-perf-16-npu-a3:
|
|
name: base-c-test-perf-16-npu-a3
|
|
needs: [ check-changes, pr-gate, set-image-config, base-c-test-acc-2-npu-a3, base-c-test-acc-16-npu-a3, recommend-tests-from-coverage ]
|
|
if: >-
|
|
${{ !failure() && !cancelled() &&
|
|
(inputs.coverage_mode == true || needs.check-changes.outputs.main_package == 'true') &&
|
|
needs.recommend-tests-from-coverage.outputs.should_run_base_c_test_perf_16_npu_a3 != 'false' }}
|
|
uses: ./.github/workflows/_npu-single-node-test-stage.yml
|
|
with:
|
|
runner: linux-aarch64-a3-800t-16
|
|
test_type: 'perf'
|
|
test_suite: base-c-test-perf-16-npu-a3
|
|
image: ${{ needs.set-image-config.outputs.CANN_image_a3 }}
|
|
install_sglang_deps: true
|
|
device_type_for_deps: 'a3'
|
|
test_timeout_minutes: '180'
|
|
only_tests: ${{ needs.recommend-tests-from-coverage.outputs.only_tests_base_c_test_perf_16_npu_a3 }}
|
|
use_coverage_runner: ${{ inputs.coverage_mode == true }}
|
|
|
|
|
|
# ==================== Recommend Tests from Coverage ==================== #
|
|
# Recommends PR-affected test cases from the pre-built coverage baseline.
|
|
# continue-on-error so it never blocks the PR gate.
|
|
recommend-tests-from-coverage:
|
|
name: Recommend tests from coverage
|
|
needs: pr-gate
|
|
runs-on: ubuntu-latest
|
|
continue-on-error: true
|
|
if: ${{ !failure() && !cancelled() && github.event_name == 'pull_request' && inputs.coverage_mode != true }}
|
|
outputs:
|
|
should_run_base_b_test_1_npu_a3: ${{ steps.bucket-tests.outputs.should_run_base_b_test_1_npu_a3 }}
|
|
only_tests_base_b_test_1_npu_a3: ${{ steps.bucket-tests.outputs.only_tests_base_b_test_1_npu_a3 }}
|
|
should_run_base_b_test_2_npu_a3: ${{ steps.bucket-tests.outputs.should_run_base_b_test_2_npu_a3 }}
|
|
only_tests_base_b_test_2_npu_a3: ${{ steps.bucket-tests.outputs.only_tests_base_b_test_2_npu_a3 }}
|
|
should_run_base_b_test_4_npu_a3: ${{ steps.bucket-tests.outputs.should_run_base_b_test_4_npu_a3 }}
|
|
only_tests_base_b_test_4_npu_a3: ${{ steps.bucket-tests.outputs.only_tests_base_b_test_4_npu_a3 }}
|
|
should_run_base_b_test_8_npu_a3: ${{ steps.bucket-tests.outputs.should_run_base_b_test_8_npu_a3 }}
|
|
only_tests_base_b_test_8_npu_a3: ${{ steps.bucket-tests.outputs.only_tests_base_b_test_8_npu_a3 }}
|
|
should_run_base_b_test_16_npu_a3: ${{ steps.bucket-tests.outputs.should_run_base_b_test_16_npu_a3 }}
|
|
only_tests_base_b_test_16_npu_a3: ${{ steps.bucket-tests.outputs.only_tests_base_b_test_16_npu_a3 }}
|
|
should_run_base_c_test_acc_2_npu_a3: ${{ steps.bucket-tests.outputs.should_run_base_c_test_acc_2_npu_a3 }}
|
|
only_tests_base_c_test_acc_2_npu_a3: ${{ steps.bucket-tests.outputs.only_tests_base_c_test_acc_2_npu_a3 }}
|
|
should_run_base_c_test_acc_16_npu_a3: ${{ steps.bucket-tests.outputs.should_run_base_c_test_acc_16_npu_a3 }}
|
|
only_tests_base_c_test_acc_16_npu_a3: ${{ steps.bucket-tests.outputs.only_tests_base_c_test_acc_16_npu_a3 }}
|
|
should_run_base_c_test_perf_2_npu_a3: ${{ steps.bucket-tests.outputs.should_run_base_c_test_perf_2_npu_a3 }}
|
|
only_tests_base_c_test_perf_2_npu_a3: ${{ steps.bucket-tests.outputs.only_tests_base_c_test_perf_2_npu_a3 }}
|
|
should_run_base_c_test_perf_16_npu_a3: ${{ steps.bucket-tests.outputs.should_run_base_c_test_perf_16_npu_a3 }}
|
|
only_tests_base_c_test_perf_16_npu_a3: ${{ steps.bucket-tests.outputs.only_tests_base_c_test_perf_16_npu_a3 }}
|
|
should_run_multimodal_gen_test_1_npu_a3: ${{ steps.bucket-tests.outputs.should_run_multimodal_gen_test_1_npu_a3 }}
|
|
only_tests_multimodal_gen_test_1_npu_a3: ${{ steps.bucket-tests.outputs.only_tests_multimodal_gen_test_1_npu_a3 }}
|
|
should_run_multimodal_gen_test_4_npu_a3: ${{ steps.bucket-tests.outputs.should_run_multimodal_gen_test_4_npu_a3 }}
|
|
only_tests_multimodal_gen_test_4_npu_a3: ${{ steps.bucket-tests.outputs.only_tests_multimodal_gen_test_4_npu_a3 }}
|
|
env:
|
|
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
|
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
|
steps:
|
|
# Every step carries step-level continue-on-error so the job conclusion
|
|
# is ALWAYS 'success' — the fast-fail health check in other jobs filters
|
|
# by job conclusion, and job-level continue-on-error still reports
|
|
# 'failure' via the API. Failed steps stay visible (red ✗ + annotation)
|
|
# but never cascade to test jobs.
|
|
- name: Checkout sglang scripts
|
|
continue-on-error: true
|
|
uses: actions/checkout@v4
|
|
with:
|
|
# test/registered + python/sglang/test/ci are needed by
|
|
# list_tests.py to enumerate each suite's test files for bucketing.
|
|
# The diffusion case config is read (AST only) by
|
|
# bucket_recommended_tests.py to enumerate multimodal cases.
|
|
sparse-checkout: |
|
|
scripts
|
|
test/registered
|
|
python/sglang/test/ci
|
|
python/sglang/multimodal_gen/test/server/ascend/testcase_configs_npu.py
|
|
sparse-checkout-cone-mode: false
|
|
|
|
- name: Download test case map
|
|
continue-on-error: true
|
|
run: |
|
|
set -euo pipefail
|
|
curl -fsSL \
|
|
--retry 3 \
|
|
--retry-delay 5 \
|
|
--retry-all-errors \
|
|
"https://sglang-npu.obs.cn-southwest-2.myhuaweicloud.com/coverage/test_case_map.json" \
|
|
-o test_case_map.json
|
|
test -s test_case_map.json
|
|
echo "Downloaded test_case_map.json"
|
|
du -h test_case_map.json
|
|
|
|
- name: Download coverage package and extract covstub
|
|
continue-on-error: true
|
|
run: |
|
|
set -euo pipefail
|
|
curl -fsSL \
|
|
--retry 3 \
|
|
--retry-delay 5 \
|
|
--retry-all-errors \
|
|
"https://sglang-npu.obs.cn-southwest-2.myhuaweicloud.com/coverage/outputs.tar.gz" \
|
|
-o outputs.tar.gz
|
|
test -s outputs.tar.gz
|
|
echo "Downloaded outputs.tar.gz"
|
|
du -h outputs.tar.gz
|
|
tar xzf outputs.tar.gz outputs/covstub
|
|
test -d outputs/covstub
|
|
echo "Extracted outputs/covstub"
|
|
find outputs/covstub -maxdepth 2 -type d | head -20 || true
|
|
|
|
- name: Recommend tests for current PR
|
|
continue-on-error: true
|
|
run: |
|
|
set -euo pipefail
|
|
if ! python3 -c "import regex" >/dev/null 2>&1; then
|
|
echo "regex not found for $(python3 -V); bootstrapping pip and installing regex"
|
|
curl -fsSL https://bootstrap.pypa.io/get-pip.py -o get-pip.py
|
|
python3 get-pip.py --break-system-packages
|
|
python3 -m pip install --break-system-packages regex
|
|
fi
|
|
python3 -c "import regex; print('regex ok:', regex.__file__)"
|
|
PR_SPEC="${{ github.repository }}#${{ github.event.pull_request.number }}"
|
|
echo "Selecting tests for PR: ${PR_SPEC}"
|
|
# test_selector.py resolves relative paths against the script's own dir,
|
|
# so pass absolute paths for files downloaded to the workspace root.
|
|
python3 scripts/ci/npu/precise-test/test_selector.py \
|
|
--github-pr "${PR_SPEC}" \
|
|
--source-dir "$PWD/outputs/covstub" \
|
|
--map-file "$PWD/test_case_map.json"
|
|
|
|
# Bucket the recommended tests per suite. Downstream jobs consume two
|
|
# outputs per suite: should_run_<suite> ('false' -> skip the job, e.g.
|
|
# empty bucket in precision mode) and only_tests_<suite> (non-empty only
|
|
# in precision mode, gated inside the script). On failure outputs stay
|
|
# empty -> callers run full suites.
|
|
- name: Bucket recommended tests per suite
|
|
id: bucket-tests
|
|
continue-on-error: true
|
|
if: always()
|
|
env:
|
|
# Precision-test switch: 'true' = the a3 suites run only the
|
|
# recommended subset, 'false' = full suites.
|
|
PRECISION_MODE: 'true'
|
|
run: |
|
|
set -euo pipefail
|
|
python3 scripts/ci/npu/precise-test/bucket_recommended_tests.py \
|
|
--recommended scripts/ci/npu/precise-test/recommended_pytest_paths.txt \
|
|
--changed-files scripts/ci/npu/precise-test/pr_changed_files.txt \
|
|
--suites "base-b-test-1-npu-a3,base-b-test-2-npu-a3,base-b-test-4-npu-a3,base-b-test-8-npu-a3,base-b-test-16-npu-a3,base-c-test-acc-2-npu-a3,base-c-test-acc-16-npu-a3,base-c-test-perf-2-npu-a3,base-c-test-perf-16-npu-a3" \
|
|
--diffusion-suites "multimodal-gen-test-1-npu-a3:1-npu,multimodal-gen-test-4-npu-a3:2-npu" \
|
|
--case-map "$PWD/test_case_map.json" \
|
|
--precision-mode "${PRECISION_MODE}" \
|
|
--github-output "${GITHUB_OUTPUT}" \
|
|
--github-summary "${GITHUB_STEP_SUMMARY:-}" \
|
|
--report-file recommended-tests-report.md
|
|
|
|
# Same report as the step summary, downloadable from the run page.
|
|
- name: Upload recommended tests report
|
|
continue-on-error: true
|
|
if: always()
|
|
uses: actions/upload-artifact@v4
|
|
with:
|
|
name: recommended-tests-report
|
|
path: recommended-tests-report.md
|
|
if-no-files-found: warn
|
|
retention-days: 7
|
|
|
|
# ==================== Coverage Assembly & Publish ==================== #
|
|
# Assembles coverage data and publishes the baseline to the shared disk.
|
|
# !failure() && !cancelled() (not success()): skipped upstream poisons success().
|
|
setup-covstub:
|
|
needs:
|
|
[
|
|
base-a-test-npu-cov,
|
|
base-b-test-1-npu-a3,
|
|
base-b-test-2-npu-a3,
|
|
base-b-test-4-npu-a3,
|
|
base-b-test-8-npu-a3,
|
|
base-b-test-16-npu-a3,
|
|
base-c-test-acc-2-npu-a3,
|
|
base-c-test-acc-16-npu-a3,
|
|
base-c-test-perf-2-npu-a3,
|
|
base-c-test-perf-16-npu-a3,
|
|
multimodal-gen-test-1-npu-a3,
|
|
multimodal-gen-test-4-npu-a3,
|
|
]
|
|
if: ${{ !failure() && !cancelled() && inputs.coverage_mode == true }}
|
|
runs-on: linux-amd64-cpu-4
|
|
container:
|
|
image: swr.ap-southeast-1.myhuaweicloud.com/base_image/ascend-ci/sglang:main-x86
|
|
steps:
|
|
- name: Checkout code
|
|
uses: actions/checkout@v4
|
|
with:
|
|
# Pin to the event SHA (not the branch name) so every job in this run
|
|
# checks out the same commit even if the branch advances mid-run.
|
|
ref: ${{ inputs.ref || github.sha }}
|
|
|
|
- name: Collect base-a coverage artifact
|
|
# Runs on ubuntu-latest (no shared disk) — hence the artifact relay.
|
|
uses: actions/download-artifact@v4
|
|
with:
|
|
name: coverage-outputs-base-a-test-npu
|
|
path: /tmp/base-a-coverage
|
|
|
|
- name: Merge base-a coverage into shared run dir
|
|
run: |
|
|
# Artifact stages {RUN_DIR}/outputs/... — merges 1:1 into the shared layout.
|
|
CACHE_ROOT="/root/.cache/tests/precise-test"
|
|
mkdir -p "${CACHE_ROOT}"
|
|
cp -a /tmp/base-a-coverage/. "${CACHE_ROOT}/"
|
|
echo "Merged base-a coverage into ${CACHE_ROOT}:"
|
|
find "${CACHE_ROOT}" -maxdepth 3 -type d | head -20
|
|
|
|
- name: Copy sglang source to covstub
|
|
run: |
|
|
RUN_DIR="/root/.cache/tests/precise-test/${{ github.run_id }}-attempt-${{ github.run_attempt }}"
|
|
COVSTUB="${RUN_DIR}/outputs/covstub"
|
|
mkdir -p "${COVSTUB}"
|
|
cp -r python/sglang "${COVSTUB}/"
|
|
# _version.py is build-generated (git-ignored); the placeholder keeps
|
|
# coverage html from failing on a missing recorded source file.
|
|
echo '__version__ = "0.0.0"' > "${COVSTUB}/sglang/_version.py"
|
|
|
|
- name: Build test case map
|
|
shell: bash
|
|
run: |
|
|
set -euo pipefail
|
|
|
|
RUN_DIR="/root/.cache/tests/precise-test/${{ github.run_id }}-attempt-${{ github.run_attempt }}"
|
|
# Must match run_tests_with_coverage.sh: GITHUB_RUN_STARTED_AT may be
|
|
# unset on the k8s runner (only explicitly-set envs are injected), so
|
|
# fall back to local date the same way.
|
|
if [ -n "${GITHUB_RUN_STARTED_AT:-}" ]; then
|
|
COV_DATE_TAG="${GITHUB_RUN_STARTED_AT:0:10}"
|
|
COV_DATE_TAG="${COV_DATE_TAG//-/}"
|
|
else
|
|
COV_DATE_TAG="$(date +%Y%m%d)"
|
|
fi
|
|
COV_ROOT="${RUN_DIR}/outputs/sglang@${COV_DATE_TAG}"
|
|
COVSTUB="${RUN_DIR}/outputs/covstub"
|
|
# Standalone top-level deliverable, never packed inside outputs/.
|
|
MAP_FILE="${RUN_DIR}/test_case_map.json"
|
|
|
|
echo "Coverage dir: ${COV_ROOT}"
|
|
echo "Source dir: ${COVSTUB}"
|
|
|
|
# Coverage data may be absent when no test targets were enumerated
|
|
# (run_tests_with_coverage.sh never ran). Skip map building then.
|
|
if [ ! -d "${COV_ROOT}" ]; then
|
|
echo "::warning::Coverage dir not found: ${COV_ROOT}, skip building test case map"
|
|
exit 0
|
|
fi
|
|
|
|
COVERAGE_FILE_COUNT=$(find "${COV_ROOT}" -name 'coverage.*' -type f | wc -l)
|
|
echo "Found ${COVERAGE_FILE_COUNT} coverage files"
|
|
if [ "${COVERAGE_FILE_COUNT}" -eq 0 ]; then
|
|
echo "::warning::No coverage files found under ${COV_ROOT}, skip building test case map"
|
|
exit 0
|
|
fi
|
|
|
|
if ! python3 -c "import regex" 2>/dev/null; then
|
|
python3 -m pip install regex \
|
|
-i http://cache-service.nginx-pypi-cache.svc.cluster.local/pypi/simple \
|
|
--trusted-host cache-service.nginx-pypi-cache.svc.cluster.local \
|
|
--retries 3 --timeout 60 \
|
|
|| python3 -m pip install regex -i https://mirrors.tuna.tsinghua.edu.cn/pypi/web/simple
|
|
fi
|
|
|
|
python3 scripts/ci/npu/precise-test/test_selector.py \
|
|
--build-map \
|
|
--coverage-dir "${COV_ROOT}" \
|
|
--source-dir "${COVSTUB}" \
|
|
--map-file "${MAP_FILE}"
|
|
test -s "${MAP_FILE}"
|
|
du -h "${MAP_FILE}"
|
|
|
|
- name: Generate HTML coverage report
|
|
# Best-effort add-on: continue-on-error keeps the job green and later
|
|
# steps running, while a failed generation still shows a red ✗ in the UI.
|
|
continue-on-error: true
|
|
shell: bash
|
|
run: |
|
|
set -euo pipefail
|
|
|
|
RUN_DIR="/root/.cache/tests/precise-test/${{ github.run_id }}-attempt-${{ github.run_attempt }}"
|
|
# Date tag derivation must match run_tests_with_coverage.sh.
|
|
if [ -n "${GITHUB_RUN_STARTED_AT:-}" ]; then
|
|
COV_DATE_TAG="${GITHUB_RUN_STARTED_AT:0:10}"
|
|
COV_DATE_TAG="${COV_DATE_TAG//-/}"
|
|
else
|
|
COV_DATE_TAG="$(date +%Y%m%d)"
|
|
fi
|
|
COV_ROOT="${RUN_DIR}/outputs/sglang@${COV_DATE_TAG}"
|
|
COVSTUB="${RUN_DIR}/outputs/covstub"
|
|
# Standalone top-level dir, packaged into htmlcov.tar.gz at publish.
|
|
REPORT_DIR="${RUN_DIR}/htmlcov"
|
|
|
|
# Same skip conditions as map building: no data -> nothing to report.
|
|
if [ ! -d "${COV_ROOT}" ] \
|
|
|| [ "$(find "${COV_ROOT}" -name 'coverage.*' -type f | wc -l)" -eq 0 ]; then
|
|
echo "::warning::No coverage data found, skip HTML report"
|
|
exit 0
|
|
fi
|
|
|
|
# Failures below exit non-zero (errexit) — red ✗, job stays green.
|
|
# Do NOT convert failures to exit 0 behind an `if ! ( set -e; ... )`
|
|
# guard: bash ignores -e inside a subshell in a condition context,
|
|
# which silently swallowed three failures in run 38713909169.
|
|
|
|
# Pin matches the test jobs' coverage==7.8.* so combine can read
|
|
# their SQLite data.
|
|
if ! python3 -c "import coverage" 2>/dev/null; then
|
|
python3 -m pip install coverage==7.8.* \
|
|
-i http://cache-service.nginx-pypi-cache.svc.cluster.local/pypi/simple \
|
|
--trusted-host cache-service.nginx-pypi-cache.svc.cluster.local \
|
|
--retries 3 --timeout 60 \
|
|
|| python3 -m pip install coverage==7.8.* -i https://mirrors.tuna.tsinghua.edu.cn/pypi/web/simple
|
|
fi
|
|
|
|
# Alias runner-specific source paths onto covstub; first entry must
|
|
# be a real local path (no wildcards).
|
|
{
|
|
echo "[run]"
|
|
echo "branch = True"
|
|
echo "[paths]"
|
|
echo "sglang ="
|
|
echo " ${COVSTUB}/sglang/"
|
|
echo " */python/sglang/"
|
|
} > /tmp/report-rc
|
|
|
|
# --keep preserves the per-test raw coverage files: coverage-data.tar.gz
|
|
# archives them after this step for manual map rebuilds.
|
|
COVERAGE_FILE=/tmp/.coverage-all \
|
|
python3 -m coverage combine --rcfile=/tmp/report-rc --keep \
|
|
$(find "${COV_ROOT}" -name 'coverage.*' -type f)
|
|
|
|
rm -rf "${REPORT_DIR}"
|
|
COVERAGE_FILE=/tmp/.coverage-all \
|
|
python3 -m coverage html --rcfile=/tmp/report-rc \
|
|
--directory="${REPORT_DIR}"
|
|
|
|
# NPU-only view: [run] source does not filter at report time (a
|
|
# source-based rcfile yields the full report), so instead touch the
|
|
# unexecuted npu .py files into a data copy (os.walk, so the PEP 420
|
|
# namespace tree counts) and filter with --include, which works.
|
|
cp /tmp/.coverage-all /tmp/.coverage-npu
|
|
python3 - "${COVSTUB}" <<'PYEOF'
|
|
import os
|
|
import sys
|
|
from coverage import CoverageData
|
|
|
|
npu_root = os.path.join(sys.argv[1], "sglang", "srt", "hardware_backend", "npu")
|
|
data = CoverageData(basename="/tmp/.coverage-npu")
|
|
data.read()
|
|
measured = set(data.measured_files())
|
|
added = 0
|
|
for root, _, files in os.walk(npu_root):
|
|
for f in files:
|
|
if f.endswith(".py"):
|
|
p = os.path.join(root, f)
|
|
if p not in measured:
|
|
data.touch_file(p)
|
|
added += 1
|
|
data.write()
|
|
print(f"NPU view: {len(measured)} measured, touched {added} unexecuted files")
|
|
PYEOF
|
|
rm -rf "${REPORT_DIR}-npu"
|
|
COVERAGE_FILE=/tmp/.coverage-npu \
|
|
python3 -m coverage html --rcfile=/tmp/report-rc \
|
|
--include='*/sglang/srt/hardware_backend/npu/*' \
|
|
--directory="${REPORT_DIR}-npu"
|
|
|
|
# Surface the totals directly on the run page.
|
|
# GITHUB_STEP_SUMMARY may be unset on the k8s runner pool.
|
|
if [ -n "${GITHUB_STEP_SUMMARY:-}" ]; then
|
|
{
|
|
echo "### Coverage"
|
|
echo '```'
|
|
echo "python/sglang (measured files only):"
|
|
COVERAGE_FILE=/tmp/.coverage-all \
|
|
python3 -m coverage report --rcfile=/tmp/report-rc | tail -n 1
|
|
echo "python/sglang/srt/hardware_backend/npu (full-tree denominator):"
|
|
COVERAGE_FILE=/tmp/.coverage-npu \
|
|
python3 -m coverage report --rcfile=/tmp/report-rc \
|
|
--include='*/sglang/srt/hardware_backend/npu/*' | tail -n 1
|
|
echo '```'
|
|
echo "Full line-by-line report: download the \`coverage-html-report\` artifact and open \`htmlcov/index.html\` (NPU-only view: \`htmlcov-npu/index.html\`)."
|
|
} >> "${GITHUB_STEP_SUMMARY}"
|
|
fi
|
|
|
|
echo "HTML coverage report: ${REPORT_DIR}"
|
|
echo "NPU-only HTML coverage report: ${REPORT_DIR}-npu"
|
|
|
|
- name: Upload HTML coverage report
|
|
# Report upload must never block the baseline publish pipeline.
|
|
continue-on-error: true
|
|
uses: actions/upload-artifact@v4
|
|
with:
|
|
name: coverage-html-report
|
|
# Two paths -> artifact root becomes the run dir, so the archive
|
|
# holds htmlcov/ and htmlcov-npu/ side by side.
|
|
path: |
|
|
/root/.cache/tests/precise-test/${{ github.run_id }}-attempt-${{ github.run_attempt }}/htmlcov
|
|
/root/.cache/tests/precise-test/${{ github.run_id }}-attempt-${{ github.run_attempt }}/htmlcov-npu
|
|
if-no-files-found: ignore
|
|
retention-days: 14
|
|
|
|
- name: Archive outputs
|
|
run: |
|
|
RUN_DIR="/root/.cache/tests/precise-test/${{ github.run_id }}-attempt-${{ github.run_attempt }}"
|
|
# Source-only baseline: recommend mode needs covstub only, and it
|
|
# shrinks the package from ~300MB to a few MB.
|
|
tar -czf "${RUN_DIR}/outputs.tar.gz" -C "${RUN_DIR}" outputs/covstub
|
|
# Raw coverage data for manual map rebuilds: extract both archives
|
|
# under one root to reconstruct outputs/. Absent when no data.
|
|
# Date tag derivation must match run_tests_with_coverage.sh.
|
|
if [ -n "${GITHUB_RUN_STARTED_AT:-}" ]; then
|
|
COV_DATE_TAG="${GITHUB_RUN_STARTED_AT:0:10}"
|
|
COV_DATE_TAG="${COV_DATE_TAG//-/}"
|
|
else
|
|
COV_DATE_TAG="$(date +%Y%m%d)"
|
|
fi
|
|
if [ -d "${RUN_DIR}/outputs/sglang@${COV_DATE_TAG}" ]; then
|
|
tar -czf "${RUN_DIR}/coverage-data.tar.gz" -C "${RUN_DIR}" "outputs/sglang@${COV_DATE_TAG}"
|
|
fi
|
|
|
|
- name: Upload outputs and test case map
|
|
# Never blocks the publish pipeline; must run before publish mv's
|
|
# the files to the fixed PUBLISH_DIR (races with concurrent runs).
|
|
continue-on-error: true
|
|
uses: actions/upload-artifact@v4
|
|
with:
|
|
name: precise-test-outputs
|
|
# Two paths -> artifact root becomes RUN_DIR, so the archive
|
|
# holds outputs.tar.gz and test_case_map.json side by side.
|
|
path: |
|
|
/root/.cache/tests/precise-test/${{ github.run_id }}-attempt-${{ github.run_attempt }}/outputs.tar.gz
|
|
/root/.cache/tests/precise-test/${{ github.run_id }}-attempt-${{ github.run_attempt }}/test_case_map.json
|
|
if-no-files-found: ignore
|
|
retention-days: 14
|
|
|
|
- name: Upload raw coverage data
|
|
# Separate artifact keeps the ~300MB bulk out of the baseline package;
|
|
# must run before publish mv's the file to the fixed PUBLISH_DIR.
|
|
continue-on-error: true
|
|
uses: actions/upload-artifact@v4
|
|
with:
|
|
name: precise-test-coverage-data
|
|
path: /root/.cache/tests/precise-test/${{ github.run_id }}-attempt-${{ github.run_attempt }}/coverage-data.tar.gz
|
|
# Absent when all test jobs were skipped (no coverage data).
|
|
if-no-files-found: ignore
|
|
retention-days: 14
|
|
|
|
- name: Publish outputs and map to shared disk
|
|
shell: bash
|
|
run: |
|
|
set -euo pipefail
|
|
|
|
RUN_DIR="/root/.cache/tests/precise-test/${{ github.run_id }}-attempt-${{ github.run_attempt }}"
|
|
# Fixed-name copy for manual use; the recommend job's baseline lives
|
|
# in OBS (upload outputs.tar.gz + test_case_map.json manually after
|
|
# each run). coverage-data.tar.gz has no automatic consumer — no OBS
|
|
# copy, only this dir and the precise-test-coverage-data artifact.
|
|
PUBLISH_DIR="/root/.cache/tests/precise-test/coverage"
|
|
|
|
# Skip publish when no coverage data this run — never overwrite
|
|
# the historical baseline with an incomplete one.
|
|
if [ ! -s "${RUN_DIR}/test_case_map.json" ]; then
|
|
echo "::warning::test_case_map.json missing (no coverage data this run), skip publish"
|
|
exit 0
|
|
fi
|
|
|
|
mkdir -p "${PUBLISH_DIR}"
|
|
|
|
# Same-filesystem mv is atomic per file, so readers never see a
|
|
# partial file (files from two concurrent publishes can interleave).
|
|
mv "${RUN_DIR}/outputs.tar.gz" "${PUBLISH_DIR}/outputs.tar.gz"
|
|
mv "${RUN_DIR}/test_case_map.json" "${PUBLISH_DIR}/test_case_map.json"
|
|
# Guarded: absent in the unlikely case this run collected no
|
|
# coverage data while still producing a map.
|
|
if [ -f "${RUN_DIR}/coverage-data.tar.gz" ]; then
|
|
mv "${RUN_DIR}/coverage-data.tar.gz" "${PUBLISH_DIR}/coverage-data.tar.gz"
|
|
fi
|
|
|
|
# HTML reports too, fixed names — latest wins, like the baseline.
|
|
# [ -d ] guards the continue-on-error generation step. A tar failure
|
|
# aborts red after the baseline mv completed; Cleanup stays skipped,
|
|
# keeping RUN_DIR for recovery.
|
|
if [ -d "${RUN_DIR}/htmlcov" ]; then
|
|
tar -czf "${PUBLISH_DIR}/htmlcov.tar.gz" -C "${RUN_DIR}" htmlcov
|
|
fi
|
|
if [ -d "${RUN_DIR}/htmlcov-npu" ]; then
|
|
tar -czf "${PUBLISH_DIR}/htmlcov-npu.tar.gz" -C "${RUN_DIR}" htmlcov-npu
|
|
fi
|
|
|
|
echo "Published baseline to ${PUBLISH_DIR}"
|
|
ls -lh "${PUBLISH_DIR}" | tail -n +2
|
|
|
|
- name: Clean up per-run directory
|
|
# Skipped on publish failure, keeping RUN_DIR for manual recovery.
|
|
shell: bash
|
|
run: |
|
|
set -euo pipefail
|
|
RUN_DIR="/root/.cache/tests/precise-test/${{ github.run_id }}-attempt-${{ github.run_attempt }}"
|
|
rm -rf "${RUN_DIR}"
|
|
echo "Removed ${RUN_DIR}"
|
|
|
|
pr-test-npu-finish:
|
|
needs:
|
|
[
|
|
check-changes,
|
|
|
|
base-a-test-npu,
|
|
base-a-test-npu-cov,
|
|
base-b-test-1-npu-a3,
|
|
base-b-test-2-npu-a3,
|
|
base-b-test-4-npu-a3,
|
|
base-b-test-8-npu-a3,
|
|
base-b-test-16-npu-a3,
|
|
|
|
multimodal-gen-test-1-npu-a3,
|
|
multimodal-gen-test-4-npu-a3,
|
|
|
|
base-c-test-acc-2-npu-a3,
|
|
base-c-test-acc-16-npu-a3,
|
|
base-c-test-perf-2-npu-a3,
|
|
base-c-test-perf-16-npu-a3,
|
|
|
|
setup-covstub,
|
|
]
|
|
if: always()
|
|
runs-on: ubuntu-latest
|
|
steps:
|
|
- name: Check all dependent job statuses
|
|
run: |
|
|
# Convert the 'needs' context to a JSON string
|
|
json_needs='${{ toJson(needs) }}'
|
|
|
|
# Get a list of all job names from the JSON keys
|
|
job_names=$(echo "$json_needs" | jq -r 'keys_unsorted[]')
|
|
|
|
for job in $job_names; do
|
|
# For each job, extract its result
|
|
result=$(echo "$json_needs" | jq -r --arg j "$job" '.[$j].result')
|
|
|
|
# Print the job name and its result
|
|
echo "$job: $result"
|
|
|
|
# Check for failure or cancellation and exit if found
|
|
if [[ "$result" == "failure" || "$result" == "cancelled" ]]; then
|
|
echo "The above jobs failed."
|
|
exit 1
|
|
fi
|
|
done
|
|
# If the loop completes, all jobs were successful
|
|
echo "All jobs completed successfully"
|
|
exit 0
|