693 lines
29 KiB
YAML
693 lines
29 KiB
YAML
name: PR Test (NPU)
|
|
|
|
on:
|
|
push:
|
|
branches: [ main ]
|
|
schedule:
|
|
- cron: '0 12 * * *' # Run daily at 12:00 UTC
|
|
pull_request:
|
|
workflow_dispatch:
|
|
inputs:
|
|
coverage_mode:
|
|
description: 'Run in coverage collection mode (uses coverage runners, enables coverage instrumentation, runs setup-covstub)'
|
|
required: false
|
|
type: boolean
|
|
default: false
|
|
workflow_call:
|
|
inputs:
|
|
ref:
|
|
description: 'Git ref (branch, tag, or SHA) to test. If not provided, uses the event commit SHA.'
|
|
required: false
|
|
type: string
|
|
default: ''
|
|
run_all_tests:
|
|
description: "Run all tests (for releasing or testing purpose)"
|
|
required: false
|
|
type: boolean
|
|
default: false
|
|
coverage_mode:
|
|
description: 'Run in coverage collection mode (uses coverage runners, enables coverage instrumentation, runs setup-covstub)'
|
|
required: false
|
|
type: boolean
|
|
default: false
|
|
|
|
concurrency:
|
|
group: pr-test-npu-${{ inputs.coverage_mode == true && 'coverage-' || '' }}${{ inputs.ref || github.ref }}
|
|
cancel-in-progress: ${{ github.event_name != 'workflow_call' }}
|
|
|
|
jobs:
|
|
# ==================== Check Changes ==================== #
|
|
check-changes:
|
|
runs-on: ubuntu-latest
|
|
outputs:
|
|
changes_exist: ${{ steps.filter.outputs.main_package == 'true' || steps.filter.outputs.multimodal_gen == 'true' || steps.run-mode.outputs.run_all_tests == 'true'}}
|
|
main_package: ${{ steps.filter.outputs.main_package == 'true' || steps.run-mode.outputs.run_all_tests == 'true' }}
|
|
multimodal_gen: ${{ steps.filter.outputs.multimodal_gen == 'true' || steps.run-mode.outputs.run_all_tests == 'true' }}
|
|
steps:
|
|
- name: Checkout code
|
|
uses: actions/checkout@v4
|
|
with:
|
|
# Pin to the event SHA (not the branch name) so every job in this run
|
|
# checks out the same commit even if the branch advances mid-run.
|
|
ref: ${{ inputs.ref || github.sha }}
|
|
|
|
- name: Determine run mode
|
|
id: run-mode
|
|
run: |
|
|
# Run all tests for scheduled runs and workflow_call (when ref input is provided)
|
|
# Note: github.event_name is inherited from caller, so we detect workflow_call by checking inputs.ref
|
|
if [[ "${{ github.event_name }}" == "schedule" || "${{ inputs.run_all_tests }}" == "true" ]]; then
|
|
echo "run_all_tests=true" >> $GITHUB_OUTPUT
|
|
echo "Run mode: ALL TESTS (schedule=${{ github.event_name == 'schedule' }}, run_all_tests=${{ inputs.run_all_tests }})"
|
|
else
|
|
echo "run_all_tests=false" >> $GITHUB_OUTPUT
|
|
echo "Run mode: FILTERED (triggered by ${{ github.event_name }})"
|
|
fi
|
|
|
|
- name: Detect file changes
|
|
id: filter
|
|
uses: dorny/paths-filter@v3
|
|
if: steps.run-mode.outputs.run_all_tests != 'true'
|
|
with:
|
|
filters: |
|
|
# Everything under python/sglang except multimodal_gen/ and kernels/ops/diffusion/.
|
|
# Exclusions have to be extglobs inside a single pattern: paths-filter ORs the
|
|
# patterns of a filter (predicate-quantifier defaults to "some"), so a standalone
|
|
# "!some/path/**" entry matches every file outside that path and makes the whole
|
|
# filter unconditionally true.
|
|
main_package:
|
|
- "python/sglang/!(*.md)"
|
|
- "python/sglang/!(multimodal_gen|kernels)/**/!(*.md)"
|
|
- "python/sglang/kernels/!(*.md)"
|
|
- "python/sglang/kernels/!(ops)/**/!(*.md)"
|
|
- "python/sglang/kernels/ops/!(*.md)"
|
|
- "python/sglang/kernels/ops/!(diffusion)/**/!(*.md)"
|
|
- "python/pyproject_npu.toml"
|
|
- "scripts/ci/npu/npu_ci_install_dependency.sh"
|
|
- "test/registered/npu/**"
|
|
- "test/registered/unit/npu/**"
|
|
- ".github/workflows/pr-test-npu.yml"
|
|
multimodal_gen:
|
|
- "python/sglang/multimodal_gen/**/!(*.md|*.ipynb)"
|
|
- "python/sglang/kernels/ops/diffusion/**"
|
|
- "python/sglang/srt/**"
|
|
- "python/pyproject_npu.toml"
|
|
- "scripts/ci/npu/npu_ci_install_dependency.sh"
|
|
- ".github/workflows/pr-test-npu.yml"
|
|
|
|
# ==================== PR Gate ==================== #
|
|
pr-gate:
|
|
needs: check-changes
|
|
if: ${{ inputs.coverage_mode != true && needs.check-changes.outputs.changes_exist == 'true' }}
|
|
uses: ./.github/workflows/pr-gate.yml
|
|
secrets: inherit
|
|
|
|
set-image-config:
|
|
runs-on: ubuntu-latest
|
|
outputs:
|
|
CANN_image_a3: ${{ steps.set-vars.outputs.CANN_image_a3 }}
|
|
CANN_image_910b: ${{ steps.set-vars.outputs.CANN_image_910b }}
|
|
steps:
|
|
# When triggered by PR, no inputs parameters are used. The latest community code is tested by default.
|
|
- name: Set image config
|
|
id: set-vars
|
|
run: |
|
|
echo "CANN_image_a3=swr.cn-southwest-2.myhuaweicloud.com/base_image/ascend-ci/cann:9.0.0-a3-ubuntu22.04-py3.11" >> $GITHUB_OUTPUT
|
|
echo "CANN_image_910b=swr.cn-southwest-2.myhuaweicloud.com/base_image/ascend-ci/cann:9.0.0-910b-ubuntu22.04-py3.11" >> $GITHUB_OUTPUT
|
|
|
|
base-a-test-1-npu-a2:
|
|
needs: [check-changes, pr-gate, set-image-config]
|
|
if: ${{ !failure() && !cancelled() && inputs.coverage_mode != true && needs.check-changes.outputs.main_package == 'true' }}
|
|
uses: ./.github/workflows/_npu-pr-test-stage.yml
|
|
with:
|
|
self_name: base-a-test-1-npu-a2
|
|
runner_config: linux-aarch64-a2-1
|
|
image: ${{ needs.set-image-config.outputs.CANN_image_910b }}
|
|
npu_device_type: 910b
|
|
run_timeout_minutes: '15'
|
|
secrets: inherit
|
|
|
|
base-b-test-1-npu-a3:
|
|
needs: [check-changes, pr-gate, set-image-config, base-a-test-1-npu-a2]
|
|
if: ${{ !failure() && !cancelled() && (inputs.coverage_mode == true || needs.check-changes.outputs.main_package == 'true') }}
|
|
uses: ./.github/workflows/_npu-pr-test-stage.yml
|
|
with:
|
|
self_name: base-b-test-1-npu-a3
|
|
runner_config: ${{ inputs.coverage_mode == true && 'linux-aarch64-a3-800t-2' || 'linux-aarch64-a3-2-' }}
|
|
image: ${{ needs.set-image-config.outputs.CANN_image_a3 }}
|
|
run_timeout_minutes: '60'
|
|
timeout_per_file: '3600'
|
|
use_coverage_runner: ${{ inputs.coverage_mode == true }}
|
|
upload_test_logs: ${{ inputs.coverage_mode != true }}
|
|
secrets: inherit
|
|
|
|
base-b-test-2-npu-a3:
|
|
needs: [check-changes, pr-gate, set-image-config, base-a-test-1-npu-a2]
|
|
if: ${{ !failure() && !cancelled() && (inputs.coverage_mode == true || needs.check-changes.outputs.main_package == 'true') }}
|
|
uses: ./.github/workflows/_npu-pr-test-stage.yml
|
|
with:
|
|
self_name: base-b-test-2-npu-a3
|
|
runner_config: ${{ inputs.coverage_mode == true && 'linux-aarch64-a3-800t-2' || 'linux-aarch64-a3-2-' }}
|
|
image: ${{ needs.set-image-config.outputs.CANN_image_a3 }}
|
|
run_timeout_minutes: '60'
|
|
timeout_per_file: '3600'
|
|
use_coverage_runner: ${{ inputs.coverage_mode == true }}
|
|
upload_test_logs: ${{ inputs.coverage_mode != true }}
|
|
secrets: inherit
|
|
|
|
base-b-test-4-npu-a3:
|
|
needs: [check-changes, pr-gate, set-image-config, base-a-test-1-npu-a2]
|
|
if: ${{ !failure() && !cancelled() && (inputs.coverage_mode == true || needs.check-changes.outputs.main_package == 'true') }}
|
|
uses: ./.github/workflows/_npu-pr-test-stage.yml
|
|
with:
|
|
self_name: base-b-test-4-npu-a3
|
|
runner_config: ${{ inputs.coverage_mode == true && 'linux-aarch64-a3-800t-4' || 'linux-aarch64-a3-4-' }}
|
|
image: ${{ needs.set-image-config.outputs.CANN_image_a3 }}
|
|
run_timeout_minutes: '120'
|
|
timeout_per_file: '3600'
|
|
partitions: '{"size":2,"arr":[0, 1]}'
|
|
use_coverage_runner: ${{ inputs.coverage_mode == true }}
|
|
upload_test_logs: ${{ inputs.coverage_mode != true }}
|
|
secrets: inherit
|
|
|
|
base-b-test-8-npu-a3:
|
|
needs: [check-changes, pr-gate, set-image-config, base-a-test-1-npu-a2]
|
|
if: ${{ !failure() && !cancelled() && (inputs.coverage_mode == true || needs.check-changes.outputs.main_package == 'true') }}
|
|
uses: ./.github/workflows/_npu-pr-test-stage.yml
|
|
with:
|
|
self_name: base-b-test-8-npu-a3
|
|
runner_config: ${{ inputs.coverage_mode == true && 'linux-aarch64-a3-800t-8' || 'linux-aarch64-a3-8-' }}
|
|
image: ${{ needs.set-image-config.outputs.CANN_image_a3 }}
|
|
run_timeout_minutes: '60'
|
|
timeout_per_file: '3600'
|
|
use_coverage_runner: ${{ inputs.coverage_mode == true }}
|
|
upload_test_logs: ${{ inputs.coverage_mode != true }}
|
|
secrets: inherit
|
|
|
|
base-b-test-16-npu-a3:
|
|
needs: [check-changes, pr-gate, set-image-config, base-a-test-1-npu-a2]
|
|
if: ${{ !failure() && !cancelled() && (inputs.coverage_mode == true || needs.check-changes.outputs.main_package == 'true') }}
|
|
uses: ./.github/workflows/_npu-pr-test-stage.yml
|
|
with:
|
|
self_name: base-b-test-16-npu-a3
|
|
runner_config: ${{ inputs.coverage_mode == true && 'linux-aarch64-a3-800t-16' || 'linux-aarch64-a3-16-' }}
|
|
image: ${{ needs.set-image-config.outputs.CANN_image_a3 }}
|
|
run_timeout_minutes: '120'
|
|
timeout_per_file: '3600'
|
|
use_coverage_runner: ${{ inputs.coverage_mode == true }}
|
|
upload_test_logs: ${{ inputs.coverage_mode != true }}
|
|
secrets: inherit
|
|
|
|
multimodal-gen-test-1-npu-a3:
|
|
needs: [check-changes, pr-gate, set-image-config, base-a-test-1-npu-a2]
|
|
if: ${{ !failure() && !cancelled() && inputs.coverage_mode != true && needs.check-changes.outputs.multimodal_gen == 'true' }}
|
|
runs-on: linux-aarch64-a3-800t-2
|
|
strategy:
|
|
fail-fast: false
|
|
matrix:
|
|
part: [0, 1]
|
|
container:
|
|
image: ${{ needs.set-image-config.outputs.CANN_image_a3 }}
|
|
steps:
|
|
- name: Checkout code
|
|
uses: actions/checkout@v4
|
|
|
|
- name: Mark repository safe
|
|
run: |
|
|
git config --system --add safe.directory ${GITHUB_WORKSPACE}
|
|
|
|
- name: Install dependencies
|
|
env:
|
|
TORCH_CACHE_URL: "http://cache-service.nginx-pypi-cache.svc.cluster.local/whl/cpu"
|
|
PYPI_CACHE_URL: "http://cache-service.nginx-pypi-cache.svc.cluster.local/pypi/simple"
|
|
UV_INDEX_URL: "http://cache-service.nginx-pypi-cache.svc.cluster.local/pypi/simple"
|
|
GITHUB_PROXY_URL: "https://gh-proxy.test.osinfra.cn/"
|
|
RUSTUP_CACHE_URL: "http://cache-service.nginx-pypi-cache.svc.cluster.local:8082"
|
|
run: |
|
|
# speed up by using infra cache services
|
|
CACHING_URL="cache-service.nginx-pypi-cache.svc.cluster.local"
|
|
sed -Ei "s@(ports|archive).ubuntu.com@${CACHING_URL}:8081@g" /etc/apt/sources.list
|
|
pip config set global.index-url http://${CACHING_URL}/pypi/simple
|
|
pip config set global.trusted-host "${CACHING_URL}"
|
|
|
|
bash scripts/ci/npu/npu_ci_install_dependency.sh a3
|
|
# copy required file from our daily cache
|
|
cp ~/.cache/modelscope/hub/datasets/otavia/ShareGPT_Vicuna_unfiltered/ShareGPT_V3_unfiltered_cleaned_split.json /tmp
|
|
# copy gsm8k dataset from cache
|
|
cp ~/.cache/modelscope/hub/datasets/tmp/test.jsonl /tmp
|
|
|
|
- name: Run test
|
|
timeout-minutes: 60
|
|
env:
|
|
SGLANG_USE_MODELSCOPE: true
|
|
SGLANG_IS_IN_CI: true
|
|
HF_ENDPOINT: https://hf-mirror.com
|
|
TORCH_EXTENSIONS_DIR: /tmp/torch_extensions
|
|
PYTORCH_NPU_ALLOC_CONF: "expandable_segments:True"
|
|
STREAMS_PER_DEVICE: 32
|
|
SGLANG_DIFFUSION_ARTIFACT_DIR: ${{ github.workspace }}/diffusion-failures
|
|
run: |
|
|
cd python
|
|
python3 sglang/multimodal_gen/test/run_suite.py \
|
|
--suite 1-npu \
|
|
--partition-id ${{ matrix.part }} \
|
|
--total-partitions 2
|
|
|
|
- name: Upload diffusion failure artifacts
|
|
if: always()
|
|
uses: actions/upload-artifact@v4
|
|
with:
|
|
name: diffusion-failures-npu-1-part${{ matrix.part }}-${{ github.run_attempt }}
|
|
path: diffusion-failures/
|
|
if-no-files-found: ignore
|
|
retention-days: 7
|
|
|
|
multimodal-gen-test-4-npu-a3:
|
|
needs: [check-changes, pr-gate, set-image-config, base-a-test-1-npu-a2]
|
|
if: ${{ !failure() && !cancelled() && inputs.coverage_mode != true && needs.check-changes.outputs.multimodal_gen == 'true' }}
|
|
runs-on: linux-aarch64-a3-800t-4
|
|
strategy:
|
|
fail-fast: false
|
|
matrix:
|
|
part: [0, 1]
|
|
container:
|
|
image: ${{ needs.set-image-config.outputs.CANN_image_a3 }}
|
|
steps:
|
|
- name: Checkout code
|
|
uses: actions/checkout@v4
|
|
|
|
- name: Mark repository safe
|
|
run: |
|
|
git config --system --add safe.directory ${GITHUB_WORKSPACE}
|
|
|
|
- name: Install dependencies
|
|
env:
|
|
TORCH_CACHE_URL: "http://cache-service.nginx-pypi-cache.svc.cluster.local/whl/cpu"
|
|
PYPI_CACHE_URL: "http://cache-service.nginx-pypi-cache.svc.cluster.local/pypi/simple"
|
|
UV_INDEX_URL: "http://cache-service.nginx-pypi-cache.svc.cluster.local/pypi/simple"
|
|
GITHUB_PROXY_URL: "https://gh-proxy.test.osinfra.cn/"
|
|
RUSTUP_CACHE_URL: "http://cache-service.nginx-pypi-cache.svc.cluster.local:8082"
|
|
run: |
|
|
# speed up by using infra cache services
|
|
CACHING_URL="cache-service.nginx-pypi-cache.svc.cluster.local"
|
|
sed -Ei "s@(ports|archive).ubuntu.com@${CACHING_URL}:8081@g" /etc/apt/sources.list
|
|
pip config set global.index-url http://${CACHING_URL}/pypi/simple
|
|
pip config set global.trusted-host "${CACHING_URL}"
|
|
|
|
bash scripts/ci/npu/npu_ci_install_dependency.sh a3
|
|
# copy required file from our daily cache
|
|
cp ~/.cache/modelscope/hub/datasets/otavia/ShareGPT_Vicuna_unfiltered/ShareGPT_V3_unfiltered_cleaned_split.json /tmp
|
|
# copy gsm8k dataset
|
|
cp ~/.cache/modelscope/hub/datasets/tmp/test.jsonl /tmp
|
|
|
|
- name: Run test
|
|
timeout-minutes: 60
|
|
env:
|
|
SGLANG_USE_MODELSCOPE: true
|
|
SGLANG_IS_IN_CI: true
|
|
HF_ENDPOINT: https://hf-mirror.com
|
|
TORCH_EXTENSIONS_DIR: /tmp/torch_extensions
|
|
PYTORCH_NPU_ALLOC_CONF: "expandable_segments:True"
|
|
STREAMS_PER_DEVICE: 32
|
|
SGLANG_DIFFUSION_ARTIFACT_DIR: ${{ github.workspace }}/diffusion-failures
|
|
run: |
|
|
cd python
|
|
python3 sglang/multimodal_gen/test/run_suite.py \
|
|
--suite 2-npu \
|
|
--partition-id ${{ matrix.part }} \
|
|
--total-partitions 2
|
|
|
|
- name: Upload diffusion failure artifacts
|
|
if: always()
|
|
uses: actions/upload-artifact@v4
|
|
with:
|
|
name: diffusion-failures-npu-2-part${{ matrix.part }}-${{ github.run_attempt }}
|
|
path: diffusion-failures/
|
|
if-no-files-found: ignore
|
|
retention-days: 7
|
|
|
|
base-c-test-acc-2-npu-a3:
|
|
name: base-c-test-acc-2-npu-a3
|
|
needs: [ check-changes, pr-gate, set-image-config, base-b-test-1-npu-a3, base-b-test-2-npu-a3, base-b-test-4-npu-a3, base-b-test-8-npu-a3, base-b-test-16-npu-a3 ]
|
|
if: ${{ !failure() && !cancelled() && (inputs.coverage_mode == true || needs.check-changes.outputs.main_package == 'true') }}
|
|
uses: ./.github/workflows/_npu-single-node-test-stage.yml
|
|
with:
|
|
runner: ${{ inputs.coverage_mode == true && 'linux-aarch64-a3-800t-2' || 'linux-aarch64-a3-2-' }}
|
|
test_type: 'accuracy'
|
|
test_suite: base-c-test-acc-2-npu-a3
|
|
image: ${{ needs.set-image-config.outputs.CANN_image_a3 }}
|
|
install_sglang_deps: true
|
|
device_type_for_deps: 'a3'
|
|
partitions: '{"size":2,"arr":[0,1]}'
|
|
use_coverage_runner: ${{ inputs.coverage_mode == true }}
|
|
upload_test_logs: ${{ inputs.coverage_mode != true }}
|
|
|
|
base-c-test-acc-16-npu-a3:
|
|
name: base-c-test-acc-16-npu-a3
|
|
needs: [ check-changes, pr-gate, set-image-config, base-b-test-1-npu-a3, base-b-test-2-npu-a3, base-b-test-4-npu-a3, base-b-test-8-npu-a3, base-b-test-16-npu-a3 ]
|
|
if: ${{ !failure() && !cancelled() && (inputs.coverage_mode == true || needs.check-changes.outputs.main_package == 'true') }}
|
|
uses: ./.github/workflows/_npu-single-node-test-stage.yml
|
|
with:
|
|
runner: ${{ inputs.coverage_mode == true && 'linux-aarch64-a3-800t-16' || 'linux-aarch64-a3-16-' }}
|
|
test_type: 'accuracy'
|
|
test_suite: base-c-test-acc-16-npu-a3
|
|
image: ${{ needs.set-image-config.outputs.CANN_image_a3 }}
|
|
install_sglang_deps: true
|
|
device_type_for_deps: 'a3'
|
|
use_coverage_runner: ${{ inputs.coverage_mode == true }}
|
|
upload_test_logs: ${{ inputs.coverage_mode != true }}
|
|
|
|
base-c-test-perf-2-npu-a3:
|
|
name: base-c-test-perf-2-npu-a3
|
|
needs: [ check-changes, pr-gate, set-image-config, base-c-test-acc-2-npu-a3, base-c-test-acc-16-npu-a3 ]
|
|
if: ${{ !failure() && !cancelled() && (inputs.coverage_mode == true || needs.check-changes.outputs.main_package == 'true') }}
|
|
uses: ./.github/workflows/_npu-single-node-test-stage.yml
|
|
with:
|
|
runner: linux-aarch64-a3-800t-2
|
|
test_type: 'perf'
|
|
test_suite: base-c-test-perf-2-npu-a3
|
|
image: ${{ needs.set-image-config.outputs.CANN_image_a3 }}
|
|
install_sglang_deps: true
|
|
device_type_for_deps: 'a3'
|
|
use_coverage_runner: ${{ inputs.coverage_mode == true }}
|
|
upload_test_logs: ${{ inputs.coverage_mode != true }}
|
|
|
|
base-c-test-perf-16-npu-a3:
|
|
name: base-c-test-perf-16-npu-a3
|
|
needs: [ check-changes, pr-gate, set-image-config, base-c-test-acc-2-npu-a3, base-c-test-acc-16-npu-a3 ]
|
|
if: ${{ !failure() && !cancelled() && (inputs.coverage_mode == true || needs.check-changes.outputs.main_package == 'true') }}
|
|
uses: ./.github/workflows/_npu-single-node-test-stage.yml
|
|
with:
|
|
runner: linux-aarch64-a3-800t-16
|
|
test_type: 'perf'
|
|
test_suite: base-c-test-perf-16-npu-a3
|
|
image: ${{ needs.set-image-config.outputs.CANN_image_a3 }}
|
|
install_sglang_deps: true
|
|
device_type_for_deps: 'a3'
|
|
test_timeout_minutes: '180'
|
|
use_coverage_runner: ${{ inputs.coverage_mode == true }}
|
|
upload_test_logs: ${{ inputs.coverage_mode != true }}
|
|
|
|
|
|
# ==================== Recommend Tests from Coverage ==================== #
|
|
# Recommends PR-affected test cases from the pre-built coverage baseline.
|
|
# continue-on-error so it never blocks the PR gate.
|
|
recommend-tests-from-coverage:
|
|
name: Recommend tests from coverage
|
|
runs-on: ubuntu-latest
|
|
continue-on-error: true
|
|
if: ${{ !cancelled() && github.event_name == 'pull_request' && inputs.coverage_mode != true }}
|
|
outputs:
|
|
coverage_paths: ${{ steps.export-paths.outputs.coverage_paths }}
|
|
env:
|
|
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
|
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
|
steps:
|
|
- name: Checkout sglang scripts
|
|
uses: actions/checkout@v4
|
|
with:
|
|
sparse-checkout: |
|
|
scripts
|
|
sparse-checkout-cone-mode: false
|
|
|
|
- name: Download test case map
|
|
run: |
|
|
set -euo pipefail
|
|
curl -fsSL \
|
|
--retry 3 \
|
|
--retry-delay 5 \
|
|
--retry-all-errors \
|
|
"https://sglang-npu.obs.cn-southwest-2.myhuaweicloud.com/coverage/test_case_map.json" \
|
|
-o test_case_map.json
|
|
test -s test_case_map.json
|
|
echo "Downloaded test_case_map.json"
|
|
du -h test_case_map.json
|
|
|
|
- name: Download coverage package and extract covstub
|
|
run: |
|
|
set -euo pipefail
|
|
curl -fsSL \
|
|
--retry 3 \
|
|
--retry-delay 5 \
|
|
--retry-all-errors \
|
|
"https://sglang-npu.obs.cn-southwest-2.myhuaweicloud.com/coverage/outputs.tar.gz" \
|
|
-o outputs.tar.gz
|
|
test -s outputs.tar.gz
|
|
echo "Downloaded outputs.tar.gz"
|
|
du -h outputs.tar.gz
|
|
tar xzf outputs.tar.gz outputs/covstub
|
|
test -d outputs/covstub
|
|
echo "Extracted outputs/covstub"
|
|
find outputs/covstub -maxdepth 2 -type d | head -20 || true
|
|
|
|
- name: Recommend tests for current PR
|
|
run: |
|
|
set -euo pipefail
|
|
if ! python3 -c "import regex" >/dev/null 2>&1; then
|
|
echo "regex not found for $(python3 -V); bootstrapping pip and installing regex"
|
|
curl -fsSL https://bootstrap.pypa.io/get-pip.py -o get-pip.py
|
|
python3 get-pip.py --break-system-packages
|
|
python3 -m pip install --break-system-packages regex
|
|
fi
|
|
python3 -c "import regex; print('regex ok:', regex.__file__)"
|
|
PR_SPEC="${{ github.repository }}#${{ github.event.pull_request.number }}"
|
|
echo "Selecting tests for PR: ${PR_SPEC}"
|
|
# test_selector.py resolves relative paths against the script's own dir,
|
|
# so pass absolute paths for files downloaded to the workspace root.
|
|
python3 scripts/ci/npu/precise-test/test_selector.py \
|
|
--github-pr "${PR_SPEC}" \
|
|
--source-dir "$PWD/outputs/covstub" \
|
|
--map-file "$PWD/test_case_map.json"
|
|
|
|
- name: Export coverage paths output
|
|
id: export-paths
|
|
if: always()
|
|
run: |
|
|
FILE="scripts/ci/npu/precise-test/recommended_pytest_paths.txt"
|
|
if [ -f "$FILE" ]; then
|
|
{
|
|
echo "coverage_paths<<EOF"
|
|
cat "$FILE"
|
|
echo "EOF"
|
|
} >> "$GITHUB_OUTPUT"
|
|
echo "Exported $(wc -l < "$FILE") recommended paths"
|
|
else
|
|
echo "coverage_paths=" >> "$GITHUB_OUTPUT"
|
|
echo "::notice::recommended_pytest_paths.txt not found"
|
|
fi
|
|
|
|
# ==================== Analyze Failure Report ==================== #
|
|
# Cross-references test failures with coverage recommendations.
|
|
# !cancelled() only: skipped jobs in needs poison success().
|
|
analyze-failure-report:
|
|
name: Analyze failure report
|
|
needs:
|
|
[
|
|
base-a-test-1-npu-a2,
|
|
base-b-test-1-npu-a3,
|
|
base-b-test-2-npu-a3,
|
|
base-b-test-4-npu-a3,
|
|
base-b-test-8-npu-a3,
|
|
base-b-test-16-npu-a3,
|
|
|
|
multimodal-gen-test-1-npu-a3,
|
|
multimodal-gen-test-4-npu-a3,
|
|
|
|
base-c-test-acc-2-npu-a3,
|
|
base-c-test-acc-16-npu-a3,
|
|
base-c-test-perf-2-npu-a3,
|
|
base-c-test-perf-16-npu-a3,
|
|
|
|
recommend-tests-from-coverage,
|
|
]
|
|
if: ${{ !cancelled() && inputs.coverage_mode != true }}
|
|
uses: ./.github/workflows/_npu-analyze-failure.yml
|
|
with:
|
|
log_artifact_pattern: selected-test-logs-*
|
|
recommendations_content: ${{ needs.recommend-tests-from-coverage.outputs.coverage_paths || '' }}
|
|
|
|
# ==================== Coverage Assembly & Publish ==================== #
|
|
# Assembles coverage data and publishes the baseline to the shared disk.
|
|
# !failure() && !cancelled() (not success()): skipped upstream poisons success().
|
|
setup-covstub:
|
|
needs:
|
|
[
|
|
set-image-config,
|
|
base-b-test-1-npu-a3,
|
|
base-b-test-2-npu-a3,
|
|
base-b-test-4-npu-a3,
|
|
base-b-test-8-npu-a3,
|
|
base-b-test-16-npu-a3,
|
|
base-c-test-acc-2-npu-a3,
|
|
base-c-test-acc-16-npu-a3,
|
|
base-c-test-perf-2-npu-a3,
|
|
base-c-test-perf-16-npu-a3,
|
|
]
|
|
if: ${{ !failure() && !cancelled() && inputs.coverage_mode == true }}
|
|
runs-on: linux-aarch64-a3-800t-2
|
|
container:
|
|
image: ${{ needs.set-image-config.outputs.CANN_image_a3 }}
|
|
steps:
|
|
- name: Checkout code
|
|
uses: actions/checkout@v4
|
|
with:
|
|
# Pin to the event SHA (not the branch name) so every job in this run
|
|
# checks out the same commit even if the branch advances mid-run.
|
|
ref: ${{ inputs.ref || github.sha }}
|
|
|
|
- name: Copy sglang source to covstub
|
|
run: |
|
|
RUN_DIR="/root/.cache/tests/precise-test/${{ github.run_id }}-attempt-${{ github.run_attempt }}"
|
|
COVSTUB="${RUN_DIR}/outputs/covstub"
|
|
mkdir -p "${COVSTUB}"
|
|
cp -r python/sglang "${COVSTUB}/"
|
|
|
|
- name: Build test case map
|
|
shell: bash
|
|
run: |
|
|
set -euo pipefail
|
|
|
|
RUN_DIR="/root/.cache/tests/precise-test/${{ github.run_id }}-attempt-${{ github.run_attempt }}"
|
|
# Date tag derivation must match scripts/ci/npu/precise-test/run_tests_with_coverage.sh.
|
|
# The custom k8s runner only injects explicitly-set env vars, so
|
|
# GITHUB_RUN_STARTED_AT may be absent (set -u would abort); fall back
|
|
# to local date exactly like run_tests_with_coverage.sh does, so both
|
|
# sides derive the same tag.
|
|
if [ -n "${GITHUB_RUN_STARTED_AT:-}" ]; then
|
|
COV_DATE_TAG="${GITHUB_RUN_STARTED_AT:0:10}"
|
|
COV_DATE_TAG="${COV_DATE_TAG//-/}"
|
|
else
|
|
COV_DATE_TAG="$(date +%Y%m%d)"
|
|
fi
|
|
COV_ROOT="${RUN_DIR}/outputs/sglang@${COV_DATE_TAG}"
|
|
COVSTUB="${RUN_DIR}/outputs/covstub"
|
|
# Placed outside outputs/ so it is NOT included in outputs.tar.gz.
|
|
MAP_FILE="${RUN_DIR}/test_case_map.json"
|
|
|
|
echo "Coverage dir: ${COV_ROOT}"
|
|
echo "Source dir: ${COVSTUB}"
|
|
|
|
# Coverage data may be absent when all test jobs were skipped
|
|
# (e.g. by the changes filter). Skip map building in that case.
|
|
if [ ! -d "${COV_ROOT}" ]; then
|
|
echo "::warning::Coverage dir not found: ${COV_ROOT}, skip building test case map"
|
|
exit 0
|
|
fi
|
|
|
|
COVERAGE_FILE_COUNT=$(find "${COV_ROOT}" -name 'coverage.*' -type f | wc -l)
|
|
echo "Found ${COVERAGE_FILE_COUNT} coverage files"
|
|
if [ "${COVERAGE_FILE_COUNT}" -eq 0 ]; then
|
|
echo "::warning::No coverage files found under ${COV_ROOT}, skip building test case map"
|
|
exit 0
|
|
fi
|
|
|
|
if ! python3 -c "import regex" 2>/dev/null; then
|
|
pip install regex \
|
|
-i http://cache-service.nginx-pypi-cache.svc.cluster.local/pypi/simple \
|
|
--trusted-host cache-service.nginx-pypi-cache.svc.cluster.local \
|
|
--retries 3 --timeout 60 \
|
|
|| pip install regex -i https://mirrors.tuna.tsinghua.edu.cn/pypi/web/simple
|
|
fi
|
|
|
|
python3 scripts/ci/npu/precise-test/test_selector.py \
|
|
--build-map \
|
|
--coverage-dir "${COV_ROOT}" \
|
|
--source-dir "${COVSTUB}" \
|
|
--map-file "${MAP_FILE}"
|
|
test -s "${MAP_FILE}"
|
|
du -h "${MAP_FILE}"
|
|
|
|
- name: Archive outputs
|
|
run: |
|
|
RUN_DIR="/root/.cache/tests/precise-test/${{ github.run_id }}-attempt-${{ github.run_attempt }}"
|
|
tar -czf "${RUN_DIR}/outputs.tar.gz" -C "${RUN_DIR}" outputs
|
|
|
|
- name: Publish outputs and map to shared disk
|
|
shell: bash
|
|
run: |
|
|
set -euo pipefail
|
|
|
|
RUN_DIR="/root/.cache/tests/precise-test/${{ github.run_id }}-attempt-${{ github.run_attempt }}"
|
|
# Fixed publish dir with no RUN_DIR layer: other pipelines have a
|
|
# different GITHUB_RUN_ID and must be able to read the baseline.
|
|
# Must match PUBLISH_DIR in the recommend-tests-from-coverage job.
|
|
PUBLISH_DIR="/root/.cache/tests/precise-test/coverage"
|
|
|
|
# Skip publish when no coverage data was collected this run (map
|
|
# build was skipped), so we never overwrite the historical
|
|
# baseline on the shared disk with an incomplete one.
|
|
if [ ! -s "${RUN_DIR}/test_case_map.json" ]; then
|
|
echo "::warning::test_case_map.json missing (no coverage data this run), skip publish"
|
|
exit 0
|
|
fi
|
|
|
|
mkdir -p "${PUBLISH_DIR}"
|
|
|
|
# mv (rename) within the same shared filesystem is atomic, so a
|
|
# concurrent reader sees either the old or the new baseline,
|
|
# never a partial file.
|
|
mv "${RUN_DIR}/outputs.tar.gz" "${PUBLISH_DIR}/outputs.tar.gz"
|
|
mv "${RUN_DIR}/test_case_map.json" "${PUBLISH_DIR}/test_case_map.json"
|
|
|
|
echo "Published baseline to ${PUBLISH_DIR}"
|
|
du -h "${PUBLISH_DIR}/outputs.tar.gz" "${PUBLISH_DIR}/test_case_map.json"
|
|
|
|
- name: Clean up per-run directory
|
|
# Skipped on publish failure, keeping RUN_DIR for manual recovery.
|
|
shell: bash
|
|
run: |
|
|
set -euo pipefail
|
|
RUN_DIR="/root/.cache/tests/precise-test/${{ github.run_id }}-attempt-${{ github.run_attempt }}"
|
|
rm -rf "${RUN_DIR}"
|
|
echo "Removed ${RUN_DIR}"
|
|
|
|
pr-test-npu-finish:
|
|
needs:
|
|
[
|
|
check-changes,
|
|
|
|
base-a-test-1-npu-a2,
|
|
base-b-test-1-npu-a3,
|
|
base-b-test-2-npu-a3,
|
|
base-b-test-4-npu-a3,
|
|
base-b-test-8-npu-a3,
|
|
base-b-test-16-npu-a3,
|
|
|
|
multimodal-gen-test-1-npu-a3,
|
|
multimodal-gen-test-4-npu-a3,
|
|
|
|
base-c-test-acc-2-npu-a3,
|
|
base-c-test-acc-16-npu-a3,
|
|
base-c-test-perf-2-npu-a3,
|
|
base-c-test-perf-16-npu-a3,
|
|
|
|
setup-covstub,
|
|
]
|
|
if: always()
|
|
runs-on: ubuntu-latest
|
|
steps:
|
|
- name: Check all dependent job statuses
|
|
run: |
|
|
# Convert the 'needs' context to a JSON string
|
|
json_needs='${{ toJson(needs) }}'
|
|
|
|
# Get a list of all job names from the JSON keys
|
|
job_names=$(echo "$json_needs" | jq -r 'keys_unsorted[]')
|
|
|
|
for job in $job_names; do
|
|
# For each job, extract its result
|
|
result=$(echo "$json_needs" | jq -r --arg j "$job" '.[$j].result')
|
|
|
|
# Print the job name and its result
|
|
echo "$job: $result"
|
|
|
|
# Check for failure or cancellation and exit if found
|
|
if [[ "$result" == "failure" || "$result" == "cancelled" ]]; then
|
|
echo "The above jobs failed."
|
|
exit 1
|
|
fi
|
|
done
|
|
# If the loop completes, all jobs were successful
|
|
echo "All jobs completed successfully"
|
|
exit 0
|