606 lines
25 KiB
YAML
606 lines
25 KiB
YAML
name: Nightly Test (NPU)
|
||
on:
|
||
schedule:
|
||
- cron: '0 16 * * *' # Execute at 0:00 a.m. Beijing Time every day
|
||
pull_request:
|
||
branches:
|
||
- main
|
||
paths:
|
||
- ".github/workflows/nightly-test-npu.yml"
|
||
workflow_dispatch:
|
||
inputs:
|
||
ref:
|
||
description: 'Git ref (branch, tag, or SHA) to test. If not provided, uses the default branch.'
|
||
required: false
|
||
type: string
|
||
default: ''
|
||
job_filter:
|
||
description: 'Select which job to run (leave empty or "all" to run all jobs)'
|
||
required: false
|
||
type: string
|
||
default: 'all'
|
||
image_a2:
|
||
description: 'The a2 running docker image of the test task.'
|
||
required: false
|
||
type: string
|
||
default: 'swr.cn-southwest-2.myhuaweicloud.com/base_image/dockerhub/lmsysorg/sglang:main-cann9.0.0-910b'
|
||
image_a3:
|
||
description: 'The a3 running docker image of the test task.'
|
||
required: false
|
||
type: string
|
||
default: 'swr.cn-southwest-2.myhuaweicloud.com/base_image/dockerhub/lmsysorg/sglang:main-cann9.0.0-a3'
|
||
image_a3_cann910:
|
||
description: 'The a3 running docker image of the test task with cann 9.1.0.'
|
||
required: false
|
||
type: string
|
||
default: 'swr.cn-southwest-2.myhuaweicloud.com/base_image/dockerhub/lmsysorg/sglang:cann9.1.0-a3-nightly'
|
||
image_a5:
|
||
description: 'The a5 running docker image of the test task.'
|
||
required: false
|
||
type: string
|
||
default: 'swr.cn-southwest-2.myhuaweicloud.com/base_image/dockerhub/lmsysorg/sglang:cann9.1.0-950-nightly'
|
||
skip_install_flag:
|
||
description: 'Indicates whether to skip the installation of sglang, defaulting to false.'
|
||
required: false
|
||
type: string
|
||
default: 'true'
|
||
workflow_call:
|
||
inputs:
|
||
ref:
|
||
description: 'Git ref (branch, tag, or SHA) to test. If not provided, uses the default branch.'
|
||
required: false
|
||
type: string
|
||
default: ''
|
||
job_filter:
|
||
description: 'Select which job to run (leave empty or "all" to run all jobs)'
|
||
required: false
|
||
type: string
|
||
default: 'all'
|
||
image_a2:
|
||
description: 'The a2 running docker image of the test task.'
|
||
required: false
|
||
type: string
|
||
default: ''
|
||
image_a3:
|
||
description: 'The a3 running docker image of the test task.'
|
||
required: false
|
||
type: string
|
||
default: ''
|
||
image_a3_cann910:
|
||
description: 'The a3 running docker image of the test task with cann 9.1.0.'
|
||
required: false
|
||
type: string
|
||
default: ''
|
||
image_a5:
|
||
description: 'The a5 running docker image of the test task.'
|
||
required: false
|
||
type: string
|
||
default: ''
|
||
skip_install_flag:
|
||
description: 'Indicates whether to skip the installation of sglang, defaulting to false.'
|
||
required: false
|
||
type: string
|
||
default: ''
|
||
concurrency:
|
||
group: nightly-test-npu-${{ inputs.ref || github.ref }}
|
||
cancel-in-progress: ${{ github.event_name != 'workflow_call' }}
|
||
jobs:
|
||
set-image-config:
|
||
runs-on: ubuntu-latest
|
||
outputs:
|
||
ref: ${{ steps.set-vars.outputs.ref }}
|
||
job_filter: ${{ steps.set-vars.outputs.job_filter }}
|
||
image_a2: ${{ steps.set-vars.outputs.image_a2 }}
|
||
image_a3: ${{ steps.set-vars.outputs.image_a3 }}
|
||
image_a3_cann910: ${{ steps.set-vars.outputs.image_a3_cann910 }}
|
||
image_a5: ${{ steps.set-vars.outputs.image_a5 }}
|
||
skip_install_flag: ${{ steps.set-vars.outputs.skip_install_flag }}
|
||
run_start_metadata: ${{ steps.set-vars.outputs.run_start_metadata }}
|
||
steps:
|
||
# When triggered by PR, no inputs parameters are used. The latest community code is tested by default.
|
||
- name: Set image config
|
||
id: set-vars
|
||
env:
|
||
# Pass untrusted PR metadata through environment variables to avoid shell‑script interpolation,
|
||
# mitigating script injection risks from fork branch names.
|
||
HEAD_LABEL: ${{ github.event.pull_request.head.label }}
|
||
run: |
|
||
if [ -z "${{ inputs.ref }}" ]; then
|
||
echo "ref=" >> $GITHUB_OUTPUT
|
||
else
|
||
echo "ref=${{ inputs.ref }}" >> $GITHUB_OUTPUT
|
||
fi
|
||
if [ -z "${{ inputs.job_filter }}" ]; then
|
||
echo "job_filter=all" >> $GITHUB_OUTPUT
|
||
else
|
||
echo "job_filter=${{ inputs.job_filter }}" >> $GITHUB_OUTPUT
|
||
fi
|
||
if [ -z "${{ inputs.image_a2 }}" ]; then
|
||
echo "image_a2=swr.cn-southwest-2.myhuaweicloud.com/base_image/dockerhub/lmsysorg/sglang:main-cann9.0.0-910b" >> $GITHUB_OUTPUT
|
||
else
|
||
echo "image_a2=${{ inputs.image_a2 }}" >> $GITHUB_OUTPUT
|
||
fi
|
||
if [ -z "${{ inputs.image_a3 }}" ]; then
|
||
echo "image_a3=swr.cn-southwest-2.myhuaweicloud.com/base_image/dockerhub/lmsysorg/sglang:main-cann9.0.0-a3" >> $GITHUB_OUTPUT
|
||
else
|
||
echo "image_a3=${{ inputs.image_a3 }}" >> $GITHUB_OUTPUT
|
||
fi
|
||
if [ -z "${{ inputs.image_a3_cann910 }}" ]; then
|
||
echo "image_a3_cann910=swr.cn-southwest-2.myhuaweicloud.com/base_image/dockerhub/lmsysorg/sglang:cann9.1.0-a3-nightly" >> $GITHUB_OUTPUT
|
||
else
|
||
echo "image_a3_cann910=${{ inputs.image_a3_cann910 }}" >> $GITHUB_OUTPUT
|
||
fi
|
||
if [ -z "${{ inputs.image_a5 }}" ]; then
|
||
echo "image_a5=swr.cn-southwest-2.myhuaweicloud.com/base_image/dockerhub/lmsysorg/sglang:cann9.1.0-950-nightly" >> $GITHUB_OUTPUT
|
||
else
|
||
echo "image_a5=${{ inputs.image_a5 }}" >> $GITHUB_OUTPUT
|
||
fi
|
||
if [ -z "${{ inputs.skip_install_flag }}" ]; then
|
||
echo "skip_install_flag=false" >> $GITHUB_OUTPUT
|
||
else
|
||
echo "skip_install_flag=${{ inputs.skip_install_flag }}" >> $GITHUB_OUTPUT
|
||
fi
|
||
# If the nightly pipeline is triggered by a PR, this variable takes the source branch label of the PR;
|
||
# otherwise, it uses the branch or tag name of the current ref.
|
||
BRANCH_LABEL="$HEAD_LABEL"
|
||
if [ -z "${BRANCH_LABEL}" ]; then
|
||
BRANCH_LABEL="${{ github.ref_name }}"
|
||
fi
|
||
BRANCH_LABEL="$(echo "${BRANCH_LABEL}" | tr -cd 'a-zA-Z0-9._/:' | tr '/:' '--')"
|
||
# Package run metadata (branch/workflow/create_time) as a single JSON for reuse by all
|
||
# test jobs; create_time is UTC+8 date + time (minute precision) recorded once at run
|
||
# start, shared even across midnight.
|
||
RUN_START_METADATA="$(python3 -c "import json,sys; print(json.dumps({'branch_label':sys.argv[1],'workflow_name':sys.argv[2],'create_time':sys.argv[3]}))" "$BRANCH_LABEL" "${{ github.workflow }}" "$(date -u -d '+8 hours' +%Y%m%d-%H%M)")"
|
||
# Write to GITHUB_OUTPUT and print to the log in one command.
|
||
echo "run_start_metadata=${RUN_START_METADATA}" | tee -a $GITHUB_OUTPUT
|
||
|
||
nightly-1-npu-a3:
|
||
name: nightly-1-npu-a3
|
||
if: ${{ !cancelled() }}
|
||
needs: [set-image-config]
|
||
uses: ./.github/workflows/_npu-pr-test-stage.yml
|
||
with:
|
||
self_name: nightly-1-npu-a3
|
||
runner_config: linux-aarch64-a3-2-
|
||
image: ${{ needs.set-image-config.outputs.image_a3 }}
|
||
run_timeout_minutes: '90'
|
||
timeout_per_file: '3600'
|
||
is_nightly_pipeline_job: true
|
||
skip_pr_test_health_check: 'true'
|
||
secrets: inherit
|
||
|
||
nightly-2-npu-a3:
|
||
name: nightly-2-npu-a3
|
||
if: ${{ !cancelled() }}
|
||
needs: [set-image-config]
|
||
uses: ./.github/workflows/_npu-pr-test-stage.yml
|
||
with:
|
||
self_name: nightly-2-npu-a3
|
||
runner_config: linux-aarch64-a3-2-
|
||
image: ${{ needs.set-image-config.outputs.image_a3 }}
|
||
run_timeout_minutes: '90'
|
||
timeout_per_file: '3600'
|
||
is_nightly_pipeline_job: true
|
||
skip_pr_test_health_check: 'true'
|
||
secrets: inherit
|
||
|
||
# The nightly-4-npu-a3 suite is split across 2 parallel partition jobs, mirroring base-b-test-4-npu-a3 in the PR pipeline.
|
||
nightly-4-npu-a3:
|
||
name: nightly-4-npu-a3
|
||
if: ${{ !cancelled() }}
|
||
needs: [set-image-config]
|
||
uses: ./.github/workflows/_npu-pr-test-stage.yml
|
||
with:
|
||
self_name: nightly-4-npu-a3
|
||
runner_config: linux-aarch64-a3-4-
|
||
image: ${{ needs.set-image-config.outputs.image_a3 }}
|
||
run_timeout_minutes: '120'
|
||
timeout_per_file: '3600'
|
||
partitions: '{"size":2,"arr":[0,1]}'
|
||
is_nightly_pipeline_job: true
|
||
skip_pr_test_health_check: 'true'
|
||
secrets: inherit
|
||
|
||
nightly-8-npu-a3:
|
||
name: nightly-8-npu-a3
|
||
if: ${{ !cancelled() }}
|
||
needs: [set-image-config]
|
||
uses: ./.github/workflows/_npu-pr-test-stage.yml
|
||
with:
|
||
self_name: nightly-8-npu-a3
|
||
runner_config: linux-aarch64-a3-8-
|
||
image: ${{ needs.set-image-config.outputs.image_a3 }}
|
||
run_timeout_minutes: '60'
|
||
timeout_per_file: '3600'
|
||
is_nightly_pipeline_job: true
|
||
skip_pr_test_health_check: 'true'
|
||
secrets: inherit
|
||
|
||
nightly-16-npu-a3:
|
||
name: nightly-16-npu-a3
|
||
if: ${{ !cancelled() }}
|
||
needs: [set-image-config]
|
||
uses: ./.github/workflows/_npu-pr-test-stage.yml
|
||
with:
|
||
self_name: nightly-16-npu-a3
|
||
runner_config: linux-aarch64-a3-16-
|
||
image: ${{ needs.set-image-config.outputs.image_a3 }}
|
||
run_timeout_minutes: '120'
|
||
timeout_per_file: '3600'
|
||
is_nightly_pipeline_job: true
|
||
skip_pr_test_health_check: 'true'
|
||
secrets: inherit
|
||
|
||
nightly-perf-2-npu-a3:
|
||
name: nightly-perf-2-npu-a3
|
||
if: ${{ !cancelled() }}
|
||
needs: [set-image-config]
|
||
uses: ./.github/workflows/_npu-single-node-test-stage.yml
|
||
with:
|
||
runner: linux-aarch64-a3-800t-2
|
||
test_type: 'perf'
|
||
test_suite: nightly-perf-2-npu-a3
|
||
is_nightly_pipeline_job: true
|
||
skip_pr_test_health_check: 'true'
|
||
image: ${{ needs.set-image-config.outputs.image_a3 }}
|
||
install_sglang_deps: false
|
||
device_type_for_deps: 'a3'
|
||
run_start_metadata: ${{ needs.set-image-config.outputs.run_start_metadata }}
|
||
|
||
nightly-perf-4-npu-a3:
|
||
name: nightly-perf-4-npu-a3
|
||
if: ${{ !cancelled() }}
|
||
needs: [set-image-config]
|
||
uses: ./.github/workflows/_npu-single-node-test-stage.yml
|
||
with:
|
||
runner: linux-aarch64-a3-800t-4
|
||
test_type: 'perf'
|
||
test_suite: nightly-perf-4-npu-a3
|
||
is_nightly_pipeline_job: true
|
||
skip_pr_test_health_check: 'true'
|
||
image: ${{ needs.set-image-config.outputs.image_a3 }}
|
||
install_sglang_deps: false
|
||
device_type_for_deps: 'a3'
|
||
run_start_metadata: ${{ needs.set-image-config.outputs.run_start_metadata }}
|
||
|
||
nightly-perf-16-npu-a3:
|
||
name: nightly-perf-16-npu-a3
|
||
if: ${{ !cancelled() }}
|
||
needs: [set-image-config]
|
||
uses: ./.github/workflows/_npu-single-node-test-stage.yml
|
||
with:
|
||
runner: linux-aarch64-a3-800t-16
|
||
test_type: 'perf'
|
||
test_suite: nightly-perf-16-npu-a3
|
||
is_nightly_pipeline_job: true
|
||
skip_pr_test_health_check: 'true'
|
||
image: ${{ needs.set-image-config.outputs.image_a3 }}
|
||
install_sglang_deps: false
|
||
device_type_for_deps: 'a3'
|
||
run_start_metadata: ${{ needs.set-image-config.outputs.run_start_metadata }}
|
||
test_timeout_minutes: '180'
|
||
|
||
# The acc-2 suite is split across 4 parallel partition jobs via the single `partitions` input.
|
||
nightly-acc-2-npu-a3:
|
||
name: nightly-acc-2-npu-a3
|
||
if: ${{ !cancelled() }}
|
||
needs: [set-image-config]
|
||
uses: ./.github/workflows/_npu-single-node-test-stage.yml
|
||
with:
|
||
runner: linux-aarch64-a3-2-
|
||
test_type: 'accuracy'
|
||
test_suite: nightly-acc-2-npu-a3
|
||
is_nightly_pipeline_job: true
|
||
partitions: '{"size":4,"arr":[0,1,2,3]}'
|
||
skip_pr_test_health_check: 'true'
|
||
image: ${{ needs.set-image-config.outputs.image_a3 }}
|
||
install_sglang_deps: false
|
||
device_type_for_deps: 'a3'
|
||
run_start_metadata: ${{ needs.set-image-config.outputs.run_start_metadata }}
|
||
|
||
nightly-acc-16-npu-a3:
|
||
name: nightly-acc-16-npu-a3
|
||
if: ${{ !cancelled() }}
|
||
needs: [set-image-config]
|
||
uses: ./.github/workflows/_npu-single-node-test-stage.yml
|
||
with:
|
||
runner: linux-aarch64-a3-16-
|
||
test_type: 'accuracy'
|
||
test_suite: nightly-acc-16-npu-a3
|
||
is_nightly_pipeline_job: true
|
||
skip_pr_test_health_check: 'true'
|
||
image: ${{ needs.set-image-config.outputs.image_a3 }}
|
||
install_sglang_deps: false
|
||
device_type_for_deps: 'a3'
|
||
run_start_metadata: ${{ needs.set-image-config.outputs.run_start_metadata }}
|
||
|
||
nightly-acc-4-npu-a5:
|
||
name: nightly-acc-4-npu-a5
|
||
if: ${{ !cancelled() }}
|
||
needs: [set-image-config]
|
||
uses: ./.github/workflows/_npu-single-node-test-stage.yml
|
||
with:
|
||
runner: linux-amd64-a5-4
|
||
test_type: 'accuracy'
|
||
test_suite: nightly-acc-4-npu-a5
|
||
is_nightly_pipeline_job: true
|
||
skip_pr_test_health_check: 'true'
|
||
image: ${{ needs.set-image-config.outputs.image_a5 }}
|
||
install_sglang_deps: false
|
||
device_type_for_deps: 'a5'
|
||
run_start_metadata: ${{ needs.set-image-config.outputs.run_start_metadata }}
|
||
|
||
# Suites suffixed with `-cann910` run against the CANN 9.1.0 image.
|
||
nightly-perf-16-npu-a3-cann910:
|
||
name: nightly-perf-16-npu-a3-cann910
|
||
if: ${{ !cancelled() }}
|
||
needs: [set-image-config]
|
||
uses: ./.github/workflows/_npu-single-node-test-stage.yml
|
||
with:
|
||
runner: linux-aarch64-a3-800t-16
|
||
test_type: 'perf'
|
||
test_suite: nightly-perf-16-npu-a3-cann910
|
||
is_nightly_pipeline_job: true
|
||
skip_pr_test_health_check: 'true'
|
||
image: ${{ needs.set-image-config.outputs.image_a3_cann910 }}
|
||
install_sglang_deps: false
|
||
device_type_for_deps: 'a3'
|
||
run_start_metadata: ${{ needs.set-image-config.outputs.run_start_metadata }}
|
||
test_timeout_minutes: '180'
|
||
|
||
nightly-acc-16-npu-a3-cann910:
|
||
name: nightly-acc-16-npu-a3-cann910
|
||
if: ${{ !cancelled() }}
|
||
needs: [set-image-config]
|
||
uses: ./.github/workflows/_npu-single-node-test-stage.yml
|
||
with:
|
||
runner: linux-aarch64-a3-16-
|
||
test_type: 'accuracy'
|
||
test_suite: nightly-acc-16-npu-a3-cann910
|
||
is_nightly_pipeline_job: true
|
||
skip_pr_test_health_check: 'true'
|
||
image: ${{ needs.set-image-config.outputs.image_a3_cann910 }}
|
||
install_sglang_deps: false
|
||
device_type_for_deps: 'a3'
|
||
run_start_metadata: ${{ needs.set-image-config.outputs.run_start_metadata }}
|
||
|
||
nightly-poc-multi-node-tests:
|
||
name: multi-node-poc
|
||
if: ${{ !cancelled() }}
|
||
needs: [set-image-config, nightly-perf-2-npu-a3, nightly-perf-4-npu-a3, nightly-perf-16-npu-a3, nightly-acc-2-npu-a3, nightly-acc-16-npu-a3]
|
||
strategy:
|
||
fail-fast: false
|
||
max-parallel: 1
|
||
matrix:
|
||
test_config:
|
||
# glm5_1 performance tests
|
||
# Disable test case temporarily due to feature refactor in PR #38293.
|
||
# - name: glm5_1_w4a8_1p1d_32p_in64k_out1k_50ms
|
||
# prefill_size: 2
|
||
# decode_size: 2
|
||
# router_size: 1
|
||
# test_case: test/registered/npu/performance/glm5_1/test_npu_glm5_1_w4a8_1p1d_32p_in64k_out1k_50ms.py
|
||
# test_type: 'perf'
|
||
# prefill_decode_deployment: 'separation'
|
||
# - name: glm5_1_w4a8_1p1d_32p_in64k_out1k_50ms_aime26
|
||
# prefill_size: 2
|
||
# decode_size: 2
|
||
# router_size: 1
|
||
# test_case: test/registered/npu/accuracy/glm5_1/test_npu_glm5_1_w4a8_1p1d_32p_in64k_out1k_50ms_aime26.py
|
||
# test_type: 'accuracy'
|
||
# prefill_decode_deployment: 'separation'
|
||
# deepseek_v4_flash performance tests
|
||
- name: deepseek_v4_flash_w8a8_1p1d_16p_in8k_out1k_50ms
|
||
prefill_size: 1
|
||
decode_size: 1
|
||
router_size: 1
|
||
test_case: test/registered/npu/performance/deepseek_v4_flash/test_npu_deepseek_v4_flash_w8a8_1p1d_16p_in8k_out1k_50ms.py
|
||
test_type: 'perf'
|
||
prefill_decode_deployment: 'separation'
|
||
uses: ./.github/workflows/nightly-test-npu-e2e-multi-node.yml
|
||
with:
|
||
runner: linux-amd64-cpu-4
|
||
test_type: ${{ matrix.test_config.test_type || 'perf' }}
|
||
test_config_name: ${{ matrix.test_config.name }}
|
||
prefill_size: ${{ matrix.test_config.prefill_size }}
|
||
decode_size: ${{ matrix.test_config.decode_size }}
|
||
router_size: ${{ matrix.test_config.router_size }}
|
||
test_case: ${{ matrix.test_config.test_case }}
|
||
image: ${{ needs.set-image-config.outputs.image_a3 }}
|
||
install_sglang_from_source: false
|
||
prefill_decode_deployment: ${{ matrix.test_config.prefill_decode_deployment }}
|
||
transformers_version: ''
|
||
run_start_metadata: ${{ needs.set-image-config.outputs.run_start_metadata }}
|
||
|
||
nightly-poc-multi-node-mix-tests:
|
||
name: multi-node-mix-poc
|
||
if: ${{ !cancelled() }}
|
||
needs: [set-image-config, nightly-perf-2-npu-a3, nightly-perf-4-npu-a3, nightly-perf-16-npu-a3, nightly-acc-2-npu-a3, nightly-acc-16-npu-a3, nightly-poc-multi-node-tests]
|
||
strategy:
|
||
fail-fast: false
|
||
max-parallel: 1
|
||
matrix:
|
||
test_config:
|
||
# glm_5_2 accuracy tests
|
||
- name: glm_5_2_w8a8_16p_gpqa
|
||
node_size: 2
|
||
test_case: test/registered/npu/accuracy/glm5_2/test_npu_glm_5_2_w8a8_16p_gpqa.py
|
||
test_type: 'accuracy'
|
||
# kimi_k3 accuracy tests
|
||
- name: kimi_k3_w4a8_32p_gpqa
|
||
node_size: 4
|
||
test_case: test/registered/npu/accuracy/kimi_k3/test_npu_kimi_k3_w4a8_32p_gpqa.py
|
||
test_type: 'accuracy'
|
||
uses: ./.github/workflows/nightly-test-npu-e2e-multi-node.yml
|
||
with:
|
||
runner: linux-amd64-cpu-4
|
||
test_type: ${{ matrix.test_config.test_type || 'perf' }}
|
||
test_config_name: ${{ matrix.test_config.name }}
|
||
node_size: ${{ matrix.test_config.node_size }}
|
||
test_case: ${{ matrix.test_config.test_case }}
|
||
image: ${{ needs.set-image-config.outputs.image_a3 }}
|
||
install_sglang_from_source: false
|
||
prefill_decode_deployment: 'mix'
|
||
transformers_version: ''
|
||
run_start_metadata: ${{ needs.set-image-config.outputs.run_start_metadata }}
|
||
|
||
nightly-poc-multi-node-tests-cann910:
|
||
name: multi-node-poc-cann910
|
||
if: ${{ !cancelled() }}
|
||
# Multi-node jobs run strictly one at a time to avoid contending for the
|
||
# same A3 multi-node resources: 9.0.0 multi -> 9.0.0 mix -> cann910 multi -> cann910 mix.
|
||
needs: [set-image-config, nightly-perf-16-npu-a3-cann910, nightly-acc-16-npu-a3-cann910, nightly-poc-multi-node-tests, nightly-poc-multi-node-mix-tests]
|
||
strategy:
|
||
fail-fast: false
|
||
max-parallel: 1
|
||
matrix:
|
||
test_config:
|
||
# deepseek_v4_flash performance tests
|
||
- name: deepseek_v4_flash_w8a8_1p1d_16p_in8k_out1k_50ms
|
||
prefill_size: 1
|
||
decode_size: 1
|
||
router_size: 1
|
||
test_case: test/registered/npu/performance/deepseek_v4_flash/test_npu_deepseek_v4_flash_w8a8_1p1d_16p_in8k_out1k_50ms.py
|
||
test_type: 'perf'
|
||
prefill_decode_deployment: 'separation'
|
||
uses: ./.github/workflows/nightly-test-npu-e2e-multi-node.yml
|
||
with:
|
||
runner: linux-amd64-cpu-4
|
||
test_type: ${{ matrix.test_config.test_type || 'perf' }}
|
||
test_config_name: ${{ matrix.test_config.name }}
|
||
prefill_size: ${{ matrix.test_config.prefill_size }}
|
||
decode_size: ${{ matrix.test_config.decode_size }}
|
||
router_size: ${{ matrix.test_config.router_size }}
|
||
test_case: ${{ matrix.test_config.test_case }}
|
||
image: ${{ needs.set-image-config.outputs.image_a3_cann910 }}
|
||
install_sglang_from_source: false
|
||
prefill_decode_deployment: ${{ matrix.test_config.prefill_decode_deployment }}
|
||
transformers_version: ''
|
||
run_start_metadata: ${{ needs.set-image-config.outputs.run_start_metadata }}
|
||
|
||
nightly-poc-multi-node-mix-tests-cann910:
|
||
name: multi-node-mix-poc-cann910
|
||
if: false
|
||
needs: [set-image-config, nightly-poc-multi-node-tests-cann910]
|
||
strategy:
|
||
fail-fast: false
|
||
max-parallel: 1
|
||
matrix:
|
||
test_config:
|
||
# glm_5_2 accuracy tests
|
||
- name: glm_5_2_w8a8_16p_gpqa
|
||
node_size: 2
|
||
test_case: test/registered/npu/accuracy/glm5_2/test_npu_glm_5_2_w8a8_16p_gpqa.py
|
||
test_type: 'accuracy'
|
||
# kimi_k3 accuracy tests
|
||
- name: kimi_k3_w4a8_32p_gpqa
|
||
node_size: 4
|
||
test_case: test/registered/npu/accuracy/kimi_k3/test_npu_kimi_k3_w4a8_32p_gpqa.py
|
||
test_type: 'accuracy'
|
||
uses: ./.github/workflows/nightly-test-npu-e2e-multi-node.yml
|
||
with:
|
||
runner: linux-amd64-cpu-4
|
||
test_type: ${{ matrix.test_config.test_type || 'perf' }}
|
||
test_config_name: ${{ matrix.test_config.name }}
|
||
node_size: ${{ matrix.test_config.node_size }}
|
||
test_case: ${{ matrix.test_config.test_case }}
|
||
image: ${{ needs.set-image-config.outputs.image_a3_cann910 }}
|
||
install_sglang_from_source: false
|
||
prefill_decode_deployment: 'mix'
|
||
transformers_version: ''
|
||
run_start_metadata: ${{ needs.set-image-config.outputs.run_start_metadata }}
|
||
|
||
check-all-jobs:
|
||
if: ${{ !cancelled() }}
|
||
needs:
|
||
- nightly-1-npu-a3
|
||
- nightly-2-npu-a3
|
||
- nightly-4-npu-a3
|
||
- nightly-8-npu-a3
|
||
- nightly-16-npu-a3
|
||
- nightly-perf-2-npu-a3
|
||
- nightly-perf-4-npu-a3
|
||
- nightly-perf-16-npu-a3
|
||
- nightly-acc-2-npu-a3
|
||
- nightly-acc-16-npu-a3
|
||
- nightly-perf-16-npu-a3-cann910
|
||
- nightly-acc-16-npu-a3-cann910
|
||
- nightly-acc-4-npu-a5
|
||
- nightly-poc-multi-node-tests
|
||
- nightly-poc-multi-node-mix-tests
|
||
- nightly-poc-multi-node-tests-cann910
|
||
- nightly-poc-multi-node-mix-tests-cann910
|
||
runs-on: ubuntu-latest
|
||
steps:
|
||
- name: Generate results table
|
||
run: |
|
||
multi_result="${{ needs.nightly-poc-multi-node-tests.result }}"
|
||
mix_result="${{ needs.nightly-poc-multi-node-mix-tests.result }}"
|
||
multi_cann910_result="${{ needs.nightly-poc-multi-node-tests-cann910.result }}"
|
||
mix_cann910_result="${{ needs.nightly-poc-multi-node-mix-tests-cann910.result }}"
|
||
|
||
# single-node suites are reported per suite; overall status is success if none failed
|
||
single_result="success"
|
||
for r in \
|
||
"${{ needs.nightly-1-npu-a3.result }}" \
|
||
"${{ needs.nightly-2-npu-a3.result }}" \
|
||
"${{ needs.nightly-4-npu-a3.result }}" \
|
||
"${{ needs.nightly-8-npu-a3.result }}" \
|
||
"${{ needs.nightly-16-npu-a3.result }}" \
|
||
"${{ needs.nightly-perf-2-npu-a3.result }}" \
|
||
"${{ needs.nightly-perf-4-npu-a3.result }}" \
|
||
"${{ needs.nightly-perf-16-npu-a3.result }}" \
|
||
"${{ needs.nightly-acc-2-npu-a3.result }}" \
|
||
"${{ needs.nightly-acc-16-npu-a3.result }}" \
|
||
"${{ needs.nightly-perf-16-npu-a3-cann910.result }}" \
|
||
"${{ needs.nightly-acc-16-npu-a3-cann910.result }}" \
|
||
"${{ needs.nightly-acc-4-npu-a5.result }}"; do
|
||
if [ "${r}" != "success" ] && [ "${r}" != "skipped" ]; then
|
||
single_result="failure"
|
||
fi
|
||
done
|
||
|
||
group_icon() {
|
||
case "$1" in
|
||
success) echo "✅" ;;
|
||
failure) echo "❌" ;;
|
||
cancelled) echo "⏭️" ;;
|
||
skipped) echo "⏭️" ;;
|
||
*) echo "❓" ;;
|
||
esac
|
||
}
|
||
|
||
echo "## Nightly Test Results" >> $GITHUB_STEP_SUMMARY
|
||
echo "" >> $GITHUB_STEP_SUMMARY
|
||
echo "| Group | Status |" >> $GITHUB_STEP_SUMMARY
|
||
echo "|-------|--------|" >> $GITHUB_STEP_SUMMARY
|
||
for entry in \
|
||
"nightly-1-npu-a3:${{ needs.nightly-1-npu-a3.result }}" \
|
||
"nightly-2-npu-a3:${{ needs.nightly-2-npu-a3.result }}" \
|
||
"nightly-4-npu-a3:${{ needs.nightly-4-npu-a3.result }}" \
|
||
"nightly-8-npu-a3:${{ needs.nightly-8-npu-a3.result }}" \
|
||
"nightly-16-npu-a3:${{ needs.nightly-16-npu-a3.result }}" \
|
||
"nightly-perf-2-npu-a3:${{ needs.nightly-perf-2-npu-a3.result }}" \
|
||
"nightly-perf-4-npu-a3:${{ needs.nightly-perf-4-npu-a3.result }}" \
|
||
"nightly-perf-16-npu-a3:${{ needs.nightly-perf-16-npu-a3.result }}" \
|
||
"nightly-acc-2-npu-a3:${{ needs.nightly-acc-2-npu-a3.result }}" \
|
||
"nightly-acc-16-npu-a3:${{ needs.nightly-acc-16-npu-a3.result }}" \
|
||
"nightly-perf-16-npu-a3-cann910:${{ needs.nightly-perf-16-npu-a3-cann910.result }}" \
|
||
"nightly-acc-16-npu-a3-cann910:${{ needs.nightly-acc-16-npu-a3-cann910.result }}" \
|
||
"nightly-acc-4-npu-a5:${{ needs.nightly-acc-4-npu-a5.result }}"; do
|
||
suite="${entry%%:*}"
|
||
r="${entry##*:}"
|
||
echo "| ${suite} | $(group_icon ${r}) ${r} |" >> $GITHUB_STEP_SUMMARY
|
||
done
|
||
echo "| multi-node-poc | $(group_icon ${multi_result}) ${multi_result} |" >> $GITHUB_STEP_SUMMARY
|
||
echo "| multi-node-mix-poc | $(group_icon ${mix_result}) ${mix_result} |" >> $GITHUB_STEP_SUMMARY
|
||
echo "| multi-node-poc-cann910 | $(group_icon ${multi_cann910_result}) ${multi_cann910_result} |" >> $GITHUB_STEP_SUMMARY
|
||
echo "| multi-node-mix-poc-cann910 | $(group_icon ${mix_cann910_result}) ${mix_cann910_result} |" >> $GITHUB_STEP_SUMMARY
|
||
echo "" >> $GITHUB_STEP_SUMMARY
|
||
|
||
FAIL=0
|
||
for r in "${single_result}" "${multi_result}" "${mix_result}" "${multi_cann910_result}" "${mix_cann910_result}"; do
|
||
if [ "${r}" != "success" ] && [ "${r}" != "skipped" ]; then FAIL=1; fi
|
||
done
|
||
exit $FAIL
|