minke.yu/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 13 Packages Projects Releases Wiki Activity

[NPU CI] Reorganize test output/log directory structure with workflow context (#33685)

Browse Source
This commit is contained in:
pllimax
2026-08-18 23:46:00 +08:00
committed by GitHub
parent 63d783bbe0
commit 499e90a125
139 changed files with 779 additions and 688 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/workflows/_npu-pr-test-stage.yml
+40 -1
View File
@@ -40,6 +40,10 @@ on:
description: 'Git ref (branch, tag, or SHA) to test. If not provided, uses the default branch.'
type: string
default: 'false'
is_nightly_pipeline_job:
description: 'Run the test suite with --nightly (collects nightly-registered tests) and --continue-on-error'
type: boolean
default: false
github-token:
description: 'GitHub token for API calls'
type: string
@@ -177,6 +181,8 @@ jobs:
}
- name: Install dependencies
# Only PR jobs install dependencies
if: ${{ inputs.is_nightly_pipeline_job != true }}
env:
TORCH_CACHE_URL: "http://cache-service.nginx-pypi-cache.svc.cluster.local/whl/cpu"
PYPI_CACHE_URL: "http://cache-service.nginx-pypi-cache.svc.cluster.local/pypi/simple"
@@ -204,10 +210,43 @@ jobs:
timeout-minutes: ${{ fromJson(inputs.run_timeout_minutes) }}
env:
CONTINUE_ON_ERROR_FLAG: ${{ inputs.continue_on_error == 'true' && '--continue-on-error' || '' }}
shell: bash
run: |
# Fail fast on any command error, undefined variable, or pipe failure.
set -euo pipefail
# Install missing python deps (skip if already importable).
PYTHON_FOR_SGLANG="python"
PIP_INDEXES=("https://pypi.org/simple/" "https://mirrors.tuna.tsinghua.edu.cn/pypi/web/simple" "http://cache-service.nginx-pypi-cache.svc.cluster.local/pypi/simple")
PIP_TRUSTED_HOST=cache-service.nginx-pypi-cache.svc.cluster.local
install_pkg() {
local pkg="$1"
ok=0
for index in "${PIP_INDEXES[@]}"; do
echo "${pkg}: try via ${index}"
if ${PYTHON_FOR_SGLANG} -m pip install "${pkg}" --retries 3 --timeout 60 \
-i "${index}" --trusted-host "${PIP_TRUSTED_HOST}"; then
ok=1; break
fi
done
[ "${ok}" = "1" ] || { echo "ERROR: failed to install ${pkg}"; exit 1; }
}
REQUIRED_PIP_PKGS=(tabulate)
for pkg in "${REQUIRED_PIP_PKGS[@]}"; do
${PYTHON_FOR_SGLANG} -c "import ${pkg}" 2>/dev/null && { echo "${pkg}: available, skip"; continue; }
install_pkg "${pkg}"
done
cd test
# Append `--nightly` and `--continue-on-error` when the job belongs to the nightly pipeline.
# --nightly: Collect test cases registered for nightly runs.
# --continue-on-error: Keep running remaining cases even if one fails.
NIGHTLY_FLAG=""
if [ "${{ inputs.is_nightly_pipeline_job }}" = "true" ]; then
NIGHTLY_FLAG="--nightly --continue-on-error"
fi
python3 run_suite.py --hw npu --suite ${{ inputs.self_name }} \
--auto-partition-id ${{ matrix.partition }} \
--auto-partition-size ${{ fromJson(inputs.partitions).size }} \
${{ inputs.timeout_per_file && format('--timeout-per-file {0}', inputs.timeout_per_file) || '' }} \
$CONTINUE_ON_ERROR_FLAG
$NIGHTLY_FLAG $CONTINUE_ON_ERROR_FLAG
.github/workflows/_npu-single-node-test-stage.yml
+135 -12
View File
@@ -37,10 +37,23 @@ on:
type: string
default: ""
description: "The transformers version number for running sglang. Use default version in image if keep empty."
run_start_metadata:
required: false
type: string
default: '{}'
description: 'JSON run metadata {branch_label, workflow_name, create_time}, recorded once at workflow start'
skip_pr_test_health_check:
description: 'Git ref (branch, tag, or SHA) to test. If not provided, uses the default branch.'
type: string
default: 'false'
is_nightly_pipeline_job:
description: 'Run the test suite with --nightly (collects nightly-registered tests) and --continue-on-error'
type: boolean
default: false
partitions:
description: 'Partition config to parallelize a suite across jobs, e.g. {"size":3,"arr":[0,1,2]}. Each element of arr is a partition id; the template expands one matrix job per element. size is passed to run_suite.py --auto-partition-size.'
type: string
default: '{"size":1,"arr":[0]}'
github-token:
description: 'GitHub token for API calls'
type: string
@@ -56,6 +69,10 @@ concurrency:
jobs:
e2e:
name: ${{ inputs.test_suite }}
strategy:
fail-fast: false
matrix:
partition: ${{ fromJson(inputs.partitions).arr }}
runs-on: ${{ inputs.runner }}
container:
image: ${{ inputs.image }}
@@ -195,9 +212,11 @@ jobs:
SGLANG_IS_IN_CI: true
TRANSFORMERS_VERBOSITY: "error"
GDN_ATTN_BACKEND_TRITON: 1
SGLANG_TEST_METRICS_OUTPUT: /root/.cache/tests/output/metrics/metrics
shell: bash
run: |
# Fail fast on any command error, undefined variable, or pipe failure.
set -euo pipefail
sglang_source_path=$(pwd)
echo "Source code path: ${sglang_source_path}"
ln -sf ${sglang_source_path} /root/sglang
@@ -207,27 +226,90 @@ jobs:
echo "Test mode: suite (${test_suite})"
tc_name="${test_suite}"
# The ID is used to locate the current job instance,
# size determines whether parallel execution is enabled.
PARTITION_ID=${{ matrix.partition }}
PARTITION_SIZE=${{ fromJson(inputs.partitions).size }}
# Append directory suffixes when there are parallel partition jobs
# within the test suite to prevent collision of result artifact directories.
if [ "${PARTITION_SIZE}" -gt 1 ]; then
tc_name="${test_suite}-p${PARTITION_ID}"
fi
# Provided automatically by the GitHub server.
# run_id is the unique identifier for the workflow run,
# attempt represents the re‑run attempt count of the job.
run_id=${{ github.run_id }}
run_attempt=${{ github.run_attempt }}
# Parse run metadata (branch_label / workflow_name / create_time)
# from run_start_metadata JSON input using GitHub fromJson expression, same as partitions input.
branch_label=$(echo "${{ fromJson(inputs.run_start_metadata).branch_label }}" | tr '/:' '--')
workflow_name=$(echo "${{ fromJson(inputs.run_start_metadata).workflow_name }}" | tr ' ' '_' | tr -d '()')
create_time="${{ fromJson(inputs.run_start_metadata).create_time }}"
if [ -z "${create_time}" ]; then create_time=$(date -u -d '+8 hours' +%Y%m%d-%H%M); fi
# Print persistence‑directory‑prefix variables
echo "Persistence prefix: run_id=${run_id} run_attempt=${run_attempt} branch_label=${branch_label} workflow_name=${workflow_name} create_time=${create_time}"
if [ "${{ inputs.is_nightly_pipeline_job }}" = "true" ]; then
test_data_output_path=/root/.cache/tests/output/${branch_label}-${create_time}-${run_id}-${run_attempt}/${workflow_name}/${{ inputs.test_type }}/${tc_name}
mkdir -p ${test_data_output_path}
# Set and print the output path for test‑case results and metrics persistence.
export METRICS_DATA_FILE=${test_data_output_path}
export SGLANG_TEST_METRICS_OUTPUT=${test_data_output_path}/metrics
echo "METRICS_DATA_FILE=${METRICS_DATA_FILE}"
echo "SGLANG_TEST_METRICS_OUTPUT=${SGLANG_TEST_METRICS_OUTPUT}"
fi
export TRANSFORMERS_VERSION_FOR_SGLANG="${{ inputs.transformers_version }}"
PYTHON_FOR_SGLANG="python"
PIP_FOR_SGLANG="pip"
# Shared pip index list (public PyPI, tsinghua mirror, then the in-cluster cache).
PIP_INDEXES=("https://pypi.org/simple/" "https://mirrors.tuna.tsinghua.edu.cn/pypi/web/simple" "http://cache-service.nginx-pypi-cache.svc.cluster.local/pypi/simple")
PIP_TRUSTED_HOST=cache-service.nginx-pypi-cache.svc.cluster.local
# Shared pip installer: try each index until one succeeds, fail the job if all fail.
install_pkg() {
local pkg="$1"
ok=0
for index in "${PIP_INDEXES[@]}"; do
echo "${pkg}: try via ${index}"
if ${PYTHON_FOR_SGLANG} -m pip install "${pkg}" --retries 3 --timeout 60 \
-i "${index}" --trusted-host "${PIP_TRUSTED_HOST}"; then
ok=1; break
fi
done
[ "${ok}" = "1" ] || { echo "ERROR: failed to install ${pkg}"; exit 1; }
}
# Transformers (pinned version): prefer the offline wheel cache, fall back to online install.
if [ -n "${TRANSFORMERS_VERSION_FOR_SGLANG}" ];then
echo "===== Install transformers for sglang - Begin ====="
TRANSFORMERS_PKG_PATH_SOURCE=/root/.cache/.cache/transformers/${TRANSFORMERS_VERSION_FOR_SGLANG}
if [ ! -d "${TRANSFORMERS_PKG_PATH_SOURCE}" ]; then
echo "The dependent transformers package does not exist: ${TRANSFORMERS_PKG_PATH_SOURCE}."
echo "Install transformers ${TRANSFORMERS_VERSION_FOR_SGLANG} online."
pip install transformers=="${TRANSFORMERS_VERSION_FOR_SGLANG}" -i https://mirrors.tuna.tsinghua.edu.cn/pypi/web/simple
install_pkg "transformers==${TRANSFORMERS_VERSION_FOR_SGLANG}"
else
echo "Install transformers ${TRANSFORMERS_VERSION_FOR_SGLANG} locally."
TRANSFORMERS_PKG_PATH_TARGET=/tmp/transformers/${TRANSFORMERS_VERSION_FOR_SGLANG}
mkdir -p "${TRANSFORMERS_PKG_PATH_TARGET}"
cp "${TRANSFORMERS_PKG_PATH_SOURCE}/*" "${TRANSFORMERS_PKG_PATH_TARGET}/"
pip install --no-index --find-links="${TRANSFORMERS_PKG_PATH_TARGET}" transformers=="${TRANSFORMERS_VERSION_FOR_SGLANG}"
cp "${TRANSFORMERS_PKG_PATH_SOURCE}/"* "${TRANSFORMERS_PKG_PATH_TARGET}/"
${PYTHON_FOR_SGLANG} -m pip install --no-index --find-links="${TRANSFORMERS_PKG_PATH_TARGET}" transformers=="${TRANSFORMERS_VERSION_FOR_SGLANG}"
fi
echo "===== Install transformers for sglang in virtual env - End ====="
fi
echo "Transformers version for sglang: $(${PIP_FOR_SGLANG} show transformers | grep Version | cut -d: -f2)"
# Install missing python deps (skip if already importable)
REQUIRED_PIP_PKGS=(tabulate)
for pkg in "${REQUIRED_PIP_PKGS[@]}"; do
${PYTHON_FOR_SGLANG} -c "import ${pkg}" 2>/dev/null && { echo "${pkg}: available, skip"; continue; }
install_pkg "${pkg}"
done
echo "scaling_governor performance num: \
$(cat /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor | grep performance | wc -l)"
echo "swappiness: $(cat /proc/sys/vm/swappiness)"
@@ -246,21 +328,64 @@ jobs:
mv ${ascend_test_util_path} ${ascend_test_util_path}_bak
cp -r ${sglang_source_path}/python/sglang/test/ascend ${ascend_test_util_path}
# External env scripts may reference unset shell vars (e.g. ZSH_VERSION),
# which abort the whole script under `set -u`. Temporarily disable -u.
set +u
source /usr/local/Ascend/cann/set_env.sh || true
source /usr/local/Ascend/nnal/atb/set_env.sh || true
source /usr/local/Ascend/ascend-toolkit/latest/opp/vendors/customize/bin/set_env.bash || true
source /usr/local/Ascend/ascend-toolkit/latest/opp/vendors/custom_transformer/bin/set_env.bash || true
set -u
# Set environment of cann
log_path="/root/.cache/tests/logs/log/${current_date}/${tc_name}/${HOSTNAME}"
rm -rf ${log_path}
mkdir -p ${log_path}
# Nightly log path mirrors the output layout; PR keeps the original {date}/{testcase}.
if [ "${{ inputs.is_nightly_pipeline_job }}" = "true" ]; then
log_path="/root/.cache/tests/logs/log/${branch_label}-${create_time}-${run_id}-${run_attempt}/${workflow_name}/${{ inputs.test_type }}/${tc_name}/${HOSTNAME}"
else
current_date=$(date -u -d '+8 hours' +%Y%m%d)
log_path="/root/.cache/tests/logs/log/${current_date}/${tc_name}/${HOSTNAME}"
fi
rm -rf "${log_path}"
mkdir -p "${log_path}"
echo "Log path: ${log_path}"
echo "Running test: ${tc_name}"
test_exit_code=0
cd test
${PYTHON_FOR_SGLANG} -u run_suite.py --hw npu --suite ${test_suite} --timeout-per-file 7200 2>&1 | tee /tmp/test_output.log || test_exit_code=$?
# Append `--nightly` and `--continue-on-error` arguments when the job belongs to the nightly pipeline.
# --nightly: Collect test cases registered for nightly runs.
# --continue-on-error: Keep running remaining cases even if one fails.
NIGHTLY_FLAG=""
if [ "${{ inputs.is_nightly_pipeline_job }}" = "true" ]; then
NIGHTLY_FLAG="--nightly --continue-on-error"
fi
# Append configuration arguments when the test suite is split into multiple parallel jobs.
PARTITION_ARGS=""
if [ "${PARTITION_SIZE}" -gt 1 ] 2>/dev/null; then
PARTITION_ARGS="--auto-partition-id ${PARTITION_ID} --auto-partition-size ${PARTITION_SIZE}"
fi
# Append `--timeout-from-est‑time` when the job belongs to the nightly pipeline.
# per‑case timeout is derived from each case's est_time.
TIMEOUT_FLAG=""
if [ "${{ inputs.is_nightly_pipeline_job }}" = "true" ]; then
TIMEOUT_FLAG="--timeout-from-est-time"
fi
# Nightly additionally persists the full suite log to ${log_path}/${tc_name}.log
# under the structured path; PR runs only tee to /tmp/test_output.log.
# Use an array so the extra target is omitted when empty: passing an empty
# string arg makes GNU tee fail (exit code 1) and flips the pipeline exit code.
LOG_TEE_TARGETS=("/tmp/test_output.log")
if [ "${{ inputs.is_nightly_pipeline_job }}" = "true" ]; then
LOG_TEE_TARGETS+=("${log_path}/${tc_name}.log")
fi
# Run NPU test suite with run_suite.py.
# Capture mode (--enable-retry --max-attempts 1) keeps test subprocesses off the
# tee pipe, so leftover server processes cannot block the pipeline on exit.
${PYTHON_FOR_SGLANG} -u run_suite.py --hw npu --suite ${test_suite} ${NIGHTLY_FLAG} ${PARTITION_ARGS} ${TIMEOUT_FLAG} --enable-retry --max-attempts 1 --timeout-per-file 7200 2>&1 | tee "${LOG_TEE_TARGETS[@]}" || test_exit_code=$?
echo "Finished test: ${tc_name}"
@@ -272,10 +397,8 @@ jobs:
status_icon="❌"
fi
echo "test_status=${test_status}" >> $GITHUB_ENV
echo "tc_name=${tc_name}" >> $GITHUB_ENV
echo "tc_name=${tc_name}"
export test_status tc_name
echo "## ${tc_name} ${status_icon} ${test_status}" >> $GITHUB_STEP_SUMMARY
echo "" >> $GITHUB_STEP_SUMMARY
metric_count=$(grep -c '\[METRIC\]' /tmp/test_output.log 2>/dev/null || echo 0)
.github/workflows/nightly-test-npu-e2e-multi-node.yml
+27 -8
View File
@@ -59,6 +59,11 @@ on:
type: string
default: ""
description: "The transformers version number for running sglang. Use default version in image if keep empty."
run_start_metadata:
required: false
type: string
default: '{}'
description: 'JSON run metadata {branch_label, workflow_name, create_time}, recorded once at workflow start'
concurrency:
group: ascend-nightly-multi-node-${{ github.workflow_ref }}-${{ github.ref }}-${{ inputs.test_config_name }}
@@ -135,14 +140,28 @@ jobs:
fi
# prepare for output data
current_date=$(date +%Y%m%d)
test_data_output_path=/root/.cache/tests/output/${{ inputs.test_type }}/${current_date}
mkdir -p ${test_data_output_path}
tc_name=${test_case##*/}
tc_name=${tc_name%.*}
metrics_data_path=${test_data_output_path}/${tc_name}
mkdir -p ${metrics_data_path}
echo "Metrics file path: ${metrics_data_path}"
# Provided automatically by the GitHub server.
# run_id is the unique identifier for the workflow run,
# attempt represents the re‑run attempt count of the job.
run_id=${{ github.run_id }}
run_attempt=${{ github.run_attempt }}
# Parse run metadata (branch_label / workflow_name / create_time)
# from run_start_metadata JSON input using GitHub fromJson expression, same as partitions input.
branch_label=$(echo "${{ fromJson(inputs.run_start_metadata).branch_label }}" | tr '/:' '--')
workflow_name=$(echo "${{ fromJson(inputs.run_start_metadata).workflow_name }}" | tr ' ' '_' | tr -d '()')
create_time="${{ fromJson(inputs.run_start_metadata).create_time }}"
if [ -z "${create_time}" ]; then create_time=$(date -u -d '+8 hours' +%Y%m%d-%H%M); fi
# Print persistence‑directory‑prefix variables
echo "Persistence prefix: run_id=${run_id} run_attempt=${run_attempt} branch_label=${branch_label} workflow_name=${workflow_name} create_time=${create_time}"
sglang_test_results_output_path=/root/.cache/tests/output/${branch_label}-${create_time}-${run_id}-${run_attempt}/${workflow_name}/${{ inputs.test_type }}/${tc_name}
mkdir -p ${sglang_test_results_output_path}
echo "Metrics file path: ${sglang_test_results_output_path}"
prefill_decode_deployment="${{ inputs.prefill_decode_deployment }}"
kube_job_type=""
@@ -161,7 +180,7 @@ jobs:
--env ci \
--image ${{ inputs.image }} \
--sglang-source-relative-path ${sglang_source_relative_path} \
--metrics-data-file ${metrics_data_path} \
--metrics-data-file ${sglang_test_results_output_path} \
--test-case ${test_case} \
--kube-name-space ${NAMESPACE} \
--kube-job-type ${kube_job_type} \
@@ -189,7 +208,7 @@ jobs:
echo "Install sglang from source."
CMD="${CMD} --install-sglang-from-source"
commit_id=${{ github.sha }}
echo "commit id: ${commit_id}" > ${test_data_output_path}/commit_id
echo "commit id: ${commit_id}" > "$(dirname "${sglang_test_results_output_path}")/commit_id"
else
echo "Use sglang from image: ${{ inputs.image }}"
fi
.github/workflows/nightly-test-npu-e2e-single-node.yml
-257
View File
@@ -1,257 +0,0 @@
name: 'Nightly Test (NPU) Single Node Template'
on:
workflow_call:
inputs:
runner:
required: true
type: string
default: linux-aarch64-a3-16
test_type:
required: false
type: string
default: perf
description: perf or accuracy
test_config_name:
required: true
type: string
description: test config name
test_case:
required: true
type: string
description: path of test case file
image:
required: false
type: string
description: image for pods
default: "swr.cn-southwest-2.myhuaweicloud.com/base_image/dockerhub/lmsysorg/sglang:main-cann9.0.0-a3"
install_sglang_from_source:
required: false
type: boolean
default: false
description: use sglang from source code or from docker image
install_sglang_deps:
required: false
type: boolean
default: false
description: install sglang dependencies (e.g. PyTorch, CANN packages) when using source installation
device_type_for_deps:
required: false
type: string
default: 'a3'
description: device type for dependency installation (a3 or 910b)
transformers_version:
required: false
type: string
default: ""
description: "The transformers version number for running sglang. Use default version in image if keep empty."
concurrency:
group: ascend-nightly-e2e-singlenode-${{ github.workflow_ref }}-${{ github.ref }}-${{ inputs.test_config_name }}
cancel-in-progress: true
jobs:
e2e:
name: ${{ inputs.test_config_name }}
runs-on: ${{ inputs.runner }}
container:
image: ${{ inputs.image }}
steps:
- name: Checkout code
uses: actions/checkout@v4
- name: Check npu info
run: |
npu-smi info
- name: Install sglang dependencies
if: ${{ inputs.install_sglang_deps == true }}
shell: bash
env:
TORCH_CACHE_URL: "http://cache-service.nginx-pypi-cache.svc.cluster.local/whl/cpu"
PYPI_CACHE_URL: "http://cache-service.nginx-pypi-cache.svc.cluster.local/pypi/simple"
UV_INDEX_URL: "http://cache-service.nginx-pypi-cache.svc.cluster.local/pypi/simple"
GITHUB_PROXY_URL: "https://gh-proxy.test.osinfra.cn/"
RUSTUP_CACHE_URL: "http://cache-service.nginx-pypi-cache.svc.cluster.local:8082"
run: |
CACHING_URL="cache-service.nginx-pypi-cache.svc.cluster.local"
sed -Ei "s@(ports|archive).ubuntu.com@${CACHING_URL}:8081@g" /etc/apt/sources.list
pip config set global.index-url http://${CACHING_URL}/pypi/simple
pip config set global.trusted-host "${CACHING_URL}"
bash scripts/ci/npu/npu_ci_install_dependency.sh ${{ inputs.device_type_for_deps }}
cp ~/.cache/modelscope/hub/datasets/otavia/ShareGPT_Vicuna_unfiltered/ShareGPT_V3_unfiltered_cleaned_split.json /tmp
- name: Run test
timeout-minutes: 300
env:
SGLANG_USE_MODELSCOPE: true
HF_ENDPOINT: https://hf-mirror.com
SGLANG_IS_IN_CI: true
TRANSFORMERS_VERBOSITY: "error"
GDN_ATTN_BACKEND_TRITON: 1
SGLANG_TEST_METRICS_OUTPUT: /root/.cache/tests/output/metrics/metrics
shell: bash
run: |
sglang_source_path=$(pwd)
echo "Source code path: ${sglang_source_path}"
ln -sf ${sglang_source_path} /root/sglang
test_case=${{ inputs.test_case }}
if [ ! -f "${sglang_source_path}/${test_case}" ]; then
echo "The testcase does not exit: ${sglang_source_path}/${test_case}"
exit 1
fi
# prepare for output data
current_date=$(date +%Y%m%d)
test_data_output_path=/root/.cache/tests/output/${{ inputs.test_type }}/${current_date}
mkdir -p ${test_data_output_path}
tc_name=${test_case##*/}
tc_name=${tc_name%.*}
export METRICS_DATA_FILE=${test_data_output_path}/${tc_name}
mkdir -p ${METRICS_DATA_FILE}
echo "Metrics file path: ${METRICS_DATA_FILE}"
# copy required file from our daily cache
cp ~/.cache/modelscope/hub/datasets/otavia/ShareGPT_Vicuna_unfiltered/ShareGPT_V3_unfiltered_cleaned_split.json /tmp
curl -o /tmp/test.jsonl -L https://gh-proxy.test.osinfra.cn/https://raw.githubusercontent.com/openai/grade-school-math/master/grade_school_math/data/test.jsonl
export TRANSFORMERS_VERSION_FOR_SGLANG="${{ inputs.transformers_version }}"
PYTHON_FOR_SGLANG="python"
PIP_FOR_SGLANG="pip"
if [ -n "${TRANSFORMERS_VERSION_FOR_SGLANG}" ];then
echo "===== Install transformers for sglang - Begin ====="
TRANSFORMERS_PKG_PATH_SOURCE=/root/.cache/.cache/transformers/${TRANSFORMERS_VERSION_FOR_SGLANG}
if [ ! -d "${TRANSFORMERS_PKG_PATH_SOURCE}" ]; then
echo "The dependent transformers package does not exist: ${TRANSFORMERS_PKG_PATH_SOURCE}."
echo "Install transformers ${TRANSFORMERS_VERSION_FOR_SGLANG} online."
pip install transformers=="${TRANSFORMERS_VERSION_FOR_SGLANG}" -i https://mirrors.tuna.tsinghua.edu.cn/pypi/web/simple
else
echo "Install transformers ${TRANSFORMERS_VERSION_FOR_SGLANG} locally."
TRANSFORMERS_PKG_PATH_TARGET=/tmp/transformers/${TRANSFORMERS_VERSION_FOR_SGLANG}
mkdir -p "${TRANSFORMERS_PKG_PATH_TARGET}"
cp "${TRANSFORMERS_PKG_PATH_SOURCE}/*" "${TRANSFORMERS_PKG_PATH_TARGET}/"
pip install --no-index --find-links="${TRANSFORMERS_PKG_PATH_TARGET}" transformers=="${TRANSFORMERS_VERSION_FOR_SGLANG}"
fi
echo "===== Install transformers for sglang in virtual env - End ====="
fi
echo "Transformers version for sglang: $(${PIP_FOR_SGLANG} show transformers | grep Version | cut -d: -f2)"
echo "scaling_governor performance num: \
$(cat /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor | grep performance | wc -l)"
echo "swappiness: $(cat /proc/sys/vm/swappiness)"
echo "numa_balancing: $(cat /proc/sys/kernel/numa_balancing)"
echo "sched_migration_cost_ns: $(cat /proc/sys/kernel/sched_migration_cost_ns)"
export SGLANG_TEST_MAX_RETRY=0
export SGLANG_SET_CPU_AFFINITY=1
echo "SGLANG_SET_CPU_AFFINITY: $SGLANG_SET_CPU_AFFINITY"
install_sglang_from_source=${{ inputs.install_sglang_from_source }}
if [ "$install_sglang_from_source" = "true" ] || [ "$install_sglang_from_source" = "True" ];then
echo "Install sglang from source"
commit_id=${{ github.sha }}
echo "commit id: ${commit_id}" > ${test_data_output_path}/commit_id
export PYTHONPATH=${sglang_source_path}/python:$PYTHONPATH
else
echo "Use sglang from image: ${{ inputs.image }}"
sglang_pkg_path=/sgl-workspace/sglang/python
ascend_test_util_path=${sglang_pkg_path}/sglang/test/ascend
mkdir -p ${ascend_test_util_path}
mv ${ascend_test_util_path} ${ascend_test_util_path}_bak
cp -r ${sglang_source_path}/python/sglang/test/ascend ${ascend_test_util_path}
fi
source /usr/local/Ascend/cann/set_env.sh || true
source /usr/local/Ascend/nnal/atb/set_env.sh || true
source /usr/local/Ascend/ascend-toolkit/latest/opp/vendors/customize/bin/set_env.bash || true
source /usr/local/Ascend/ascend-toolkit/latest/opp/vendors/custom_transformer/bin/set_env.bash || true
# Set environment of cann
log_path="/root/.cache/tests/logs/log/${current_date}/${tc_name}/${HOSTNAME}"
rm -rf ${log_path}
mkdir -p ${log_path}
echo "Log path: ${log_path}"
echo "Running test case ${test_case}"
test_exit_code=0
${PYTHON_FOR_SGLANG} -u ${test_case} 2>&1 | tee /tmp/test_output.log || test_exit_code=$?
echo "Finished test case ${test_case}"
if [ "${test_exit_code}" = "0" ]; then
test_status="pass"
status_icon="✅"
else
test_status="fail"
status_icon="❌"
fi
echo "test_status=${test_status}" >> $GITHUB_ENV
echo "tc_name=${tc_name}" >> $GITHUB_ENV
export test_status tc_name
echo "## ${tc_name} ${status_icon} ${test_status}" >> $GITHUB_STEP_SUMMARY
echo "" >> $GITHUB_STEP_SUMMARY
metric_count=$(grep -c '\[METRIC\]' /tmp/test_output.log 2>/dev/null || echo 0)
if [ "${metric_count}" -gt 0 ]; then
echo "| Metric | Value | Pass |" >> $GITHUB_STEP_SUMMARY
echo "|--------|-------|------|" >> $GITHUB_STEP_SUMMARY
grep '\[METRIC\]' /tmp/test_output.log | while IFS= read -r line; do
metric_name=$(echo "$line" | sed -E 's/.*\[METRIC\] ([^=]+)=.*/\1/')
metric_value=$(echo "$line" | sed -E 's/.*\[METRIC\] [^=]+=([^ ]+).*/\1/')
echo "| ${metric_name} | ${metric_value} | ${status_icon} |" >> $GITHUB_STEP_SUMMARY
done
else
echo "No metrics collected (test may have failed before producing results)." >> $GITHUB_STEP_SUMMARY
fi
echo "" >> $GITHUB_STEP_SUMMARY
echo "import json, re, os" > /tmp/dump_metrics.py
echo "tc = os.environ.get('tc_name', 'unknown')" >> /tmp/dump_metrics.py
echo "st = os.environ.get('test_status', 'unknown')" >> /tmp/dump_metrics.py
echo "tp = '${{ inputs.test_type }}'" >> /tmp/dump_metrics.py
echo "metrics = {}" >> /tmp/dump_metrics.py
echo "with open('/tmp/test_output.log') as f:" >> /tmp/dump_metrics.py
echo " for line in f:" >> /tmp/dump_metrics.py
echo " m = re.match(r'\[METRIC\] (\S+)=(\S+)', line)" >> /tmp/dump_metrics.py
echo " if m:" >> /tmp/dump_metrics.py
echo " v = m.group(2)" >> /tmp/dump_metrics.py
echo " try:" >> /tmp/dump_metrics.py
echo " v = float(v)" >> /tmp/dump_metrics.py
echo " except ValueError:" >> /tmp/dump_metrics.py
echo " pass" >> /tmp/dump_metrics.py
echo " metrics[m.group(1)] = v" >> /tmp/dump_metrics.py
echo "baselines = {}" >> /tmp/dump_metrics.py
echo "for k, v in list(metrics.items()):" >> /tmp/dump_metrics.py
echo " if k.endswith('_baseline'):" >> /tmp/dump_metrics.py
echo " baselines[k[:-9]] = v" >> /tmp/dump_metrics.py
echo " del metrics[k]" >> /tmp/dump_metrics.py
echo "with open('/tmp/metrics.json', 'w') as f:" >> /tmp/dump_metrics.py
echo " json.dump({'test_case': tc, 'test_type': tp, 'status': st, 'metrics': metrics, 'baselines': baselines}, f)" >> /tmp/dump_metrics.py
python3 /tmp/dump_metrics.py
exit ${test_exit_code}
- name: Upload metrics
if: always()
uses: actions/upload-artifact@v4
with:
name: metrics-${{ inputs.test_config_name }}
path: /tmp/metrics.json
retention-days: 7
- name: Backup plog
if: always()
shell: bash
run: |
plog_path="/root/ascend/log/debug/plog"
if [ -d "$plog_path" ];then
echo "Plog files found. Begin to backup them."
tc_name=${{ inputs.test_case }}
tc_name=${tc_name##*/}
tc_name=${tc_name%.*}
target_plog_path="/root/.cache/tests/logs/plog/${tc_name}/${HOSTNAME}"
echo "Save path: ${target_plog_path}"
rm -rf ${target_plog_path}
mkdir -p ${target_plog_path}
cp ${plog_path}/* ${target_plog_path}
fi
.github/workflows/nightly-test-npu.yml
+292 -219
View File
@@ -1,4 +1,4 @@
name: Nightly Test (NPU)
name: Nightly Test (NPU)
on:
schedule:
- cron: '0 16 * * *' # Execute at 0:00 a.m. Beijing Time every day
@@ -73,10 +73,15 @@ jobs:
image_a2: ${{ steps.set-vars.outputs.image_a2 }}
image_a3: ${{ steps.set-vars.outputs.image_a3 }}
skip_install_flag: ${{ steps.set-vars.outputs.skip_install_flag }}
run_start_metadata: ${{ steps.set-vars.outputs.run_start_metadata }}
steps:
# When triggered by PR, no inputs parameters are used. The latest community code is tested by default.
- name: Set image config
id: set-vars
env:
# Pass untrusted PR metadata through environment variables to avoid shell‑script interpolation,
# mitigating script injection risks from fork branch names.
HEAD_LABEL: ${{ github.event.pull_request.head.label }}
run: |
if [ -z "${{ inputs.ref }}" ]; then
echo "ref=" >> $GITHUB_OUTPUT
@@ -103,211 +108,231 @@ jobs:
else
echo "skip_install_flag=${{ inputs.skip_install_flag }}" >> $GITHUB_OUTPUT
fi
nightly-poc-single-node-a2-tests:
name: single-node-poc-a2
if: ${{ !cancelled() }}
needs: [ set-image-config ]
strategy:
fail-fast: false
max-parallel: 6
matrix:
test_config:
- name: qwen3_32b_w8a8_2p_in3k5_out1k5_50ms_a2
runner: linux-aarch64-a2-4
test_case: test/registered/npu/performance/qwen3_32b/test_npu_qwen3_32b_w8a8_2p_in3k5_out1k5_50ms_a2.py
test_type: 'perf'
- name: qwen3_32b_w8a8_2p_in3k5_out1k5_50ms_gpqa_a2
runner: linux-aarch64-a2-4
test_case: test/registered/npu/accuracy/qwen3_32b/test_npu_qwen3_32b_w8a8_2p_in3k5_out1k5_50ms_gpqa_a2.py
uses: ./.github/workflows/nightly-test-npu-e2e-single-node.yml
with:
runner: ${{ matrix.test_config.runner }}
test_type: ${{ matrix.test_config.test_type }}
test_config_name: ${{ matrix.test_config.name }}
test_case: ${{ matrix.test_config.test_case }}
image: ${{ needs.set-image-config.outputs.image_a2 }}
install_sglang_from_source: false
transformers_version: ''
# If the nightly pipeline is triggered by a PR, this variable takes the source branch label of the PR;
# otherwise, it uses the branch or tag name of the current ref.
BRANCH_LABEL="$HEAD_LABEL"
if [ -z "${BRANCH_LABEL}" ]; then
BRANCH_LABEL="${{ github.ref_name }}"
fi
BRANCH_LABEL="$(echo "${BRANCH_LABEL}" | tr -cd 'a-zA-Z0-9._/:' | tr '/:' '--')"
# Package run metadata (branch/workflow/create_time) as a single JSON for reuse by all
# test jobs; create_time is UTC+8 date + time (minute precision) recorded once at run
# start, shared even across midnight.
RUN_START_METADATA="$(python3 -c "import json,sys; print(json.dumps({'branch_label':sys.argv[1],'workflow_name':sys.argv[2],'create_time':sys.argv[3]}))" "$BRANCH_LABEL" "${{ github.workflow }}" "$(date -u -d '+8 hours' +%Y%m%d-%H%M)")"
# Write to GITHUB_OUTPUT and print to the log in one command.
echo "run_start_metadata=${RUN_START_METADATA}" | tee -a $GITHUB_OUTPUT
nightly-poc-single-node-tests:
name: single-node-poc
nightly-1-npu-a2:
name: nightly-1-npu-a2
if: ${{ !cancelled() }}
needs: [set-image-config]
strategy:
fail-fast: false
max-parallel: 6
matrix:
test_config:
# qwen3_6_35b_a3b performance tests
- name: qwen3_6_35b_a3b_1p_in3k5_out1k5_50ms
runner: linux-aarch64-a3-2
test_case: test/registered/npu/performance/qwen3_6_35b_a3b/test_npu_qwen3_6_35b_a3b_1p_in3k5_out1k5_50ms.py
test_type: 'perf'
- name: qwen3_6_35b_a3b_1p_aime26
runner: linux-aarch64-a3-2
test_case: test/registered/npu/accuracy/qwen3_6_35b_a3b/test_npu_qwen3_6_35b_a3b_1p_aime26.py
- name: qwen3_6_35b_a3b_1p_in64k_out1k_50ms
runner: linux-aarch64-a3-2
test_case: test/registered/npu/performance/qwen3_6_35b_a3b/test_npu_qwen3_6_35b_a3b_1p_in64k_out1k_50ms.py
test_type: 'perf'
- name: qwen3_6_35b_a3b_1p_in128k_out1k_50ms
runner: linux-aarch64-a3-2
test_case: test/registered/npu/performance/qwen3_6_35b_a3b/test_npu_qwen3_6_35b_a3b_1p_in128k_out1k_50ms.py
test_type: 'perf'
- name: qwen3_6_35b_a3b_1p_in64k_out1k_prefix90_50ms
runner: linux-aarch64-a3-2
test_case: test/registered/npu/performance/qwen3_6_35b_a3b/test_npu_qwen3_6_35b_a3b_1p_in64k_out1k_prefix90_50ms.py
test_type: 'perf'
- name: qwen3_6_35b_a3b_1p_in64k_out1k_prefix90_50ms_aime26
runner: linux-aarch64-a3-2
test_case: test/registered/npu/accuracy/qwen3_6_35b_a3b/test_npu_qwen3_6_35b_a3b_1p_in64k_out1k_prefix90_50ms_aime26.py
- name: qwen3_6_35b_a3b_1p_in128k_out1k_prefix90_50ms
runner: linux-aarch64-a3-2
test_case: test/registered/npu/performance/qwen3_6_35b_a3b/test_npu_qwen3_6_35b_a3b_1p_in128k_out1k_prefix90_50ms.py
test_type: 'perf'
# qwen3_6_27b performance tests
- name: qwen3_6_27b_w8a8_1p_in3k5_out1k5_50ms
runner: linux-aarch64-a3-2
test_case: test/registered/npu/performance/qwen3_6_27b/test_npu_qwen3_6_27b_w8a8_1p_in3k5_out1k5_50ms.py
test_type: 'perf'
- name: qwen3_6_27b_w8a8_1p_in3k5_out1k5_50ms_gpqa
runner: linux-aarch64-a3-2
test_case: test/registered/npu/accuracy/qwen3_6_27b/test_npu_qwen3_6_27b_w8a8_1p_in3k5_out1k5_50ms_gpqa.py
- name: qwen3_6_27b_w8a8_2p_in16k_out1k_50ms
runner: linux-aarch64-a3-4
test_case: test/registered/npu/performance/qwen3_6_27b/test_npu_qwen3_6_27b_w8a8_2p_in16k_out1k_50ms.py
test_type: 'perf'
- name: qwen3_6_27b_w8a8_1p_in64k_out1k_50ms
runner: linux-aarch64-a3-4
test_case: test/registered/npu/performance/qwen3_6_27b/test_npu_qwen3_6_27b_w8a8_1p_in64k_out1k_50ms.py
test_type: 'perf'
- name: qwen3_6_27b_w8a8_2p_in128k_out1k_50ms
runner: linux-aarch64-a3-4
test_case: test/registered/npu/performance/qwen3_6_27b/test_npu_qwen3_6_27b_w8a8_2p_in128k_out1k_50ms.py
test_type: 'perf'
- name: qwen3_6_27b_2p_in64k_out1k_prefix90_50ms
runner: linux-aarch64-a3-4
test_case: test/registered/npu/performance/qwen3_6_27b/test_npu_qwen3_6_27b_2p_in64k_out1k_prefix90_50ms.py
test_type: 'perf'
- name: qwen3_6_27b_1p_in1024x1024_30_out1024_50ms
runner: linux-aarch64-a3-2
test_case: test/registered/npu/performance/qwen3_6_27b/test_npu_qwen3_6_27b_1p_in1024x1024_30_out1024_50ms.py
test_type: 'perf'
- name: qwen3_6_27b_1p_in1080p_30_out256_50ms
runner: linux-aarch64-a3-2
test_case: test/registered/npu/performance/qwen3_6_27b/test_npu_qwen3_6_27b_1p_in1080p_30_out256_50ms.py
test_type: 'perf'
- name: qwen3_6_27b_1p_gpqa
runner: linux-aarch64-a3-2
test_case: test/registered/npu/accuracy/qwen3_6_27b/test_npu_qwen3_6_27b_1p_gpqa.py
# qwen3_32b performance tests
- name: qwen3_32b_w8a8_2p_in3k5_out1k5_50ms
runner: linux-aarch64-a3-4
test_case: test/registered/npu/performance/qwen3_32b/test_npu_qwen3_32b_w8a8_2p_in3k5_out1k5_50ms.py
test_type: 'perf'
- name: qwen3_32b_w8a8_2p_in3k5_out1k5_50ms_gpqa
runner: linux-aarch64-a3-4
test_case: test/registered/npu/accuracy/qwen3_32b/test_npu_qwen3_32b_w8a8_2p_in3k5_out1k5_50ms_gpqa.py
- name: qwen3_32b_bf16_8p_in18k_out4k_6ms
runner: linux-aarch64-a3-16
test_case: test/registered/npu/performance/qwen3_32b/test_npu_qwen3_32b_bf16_8p_in18k_out4k_6ms.py
test_type: 'perf'
- name: qwen3_32b_bf16_8p_gpqa
runner: linux-aarch64-a3-16
test_case: test/registered/npu/accuracy/qwen3_32b/test_npu_qwen3_32b_bf16_8p_gpqa.py
# qwen3_30b_a3b performance tests
- name: qwen3_30b_w8a8_1p_in3k5_out1k5_50ms
runner: linux-aarch64-a3-2
test_case: test/registered/npu/performance/qwen3_30b_a3b/test_npu_qwen3_30b_w8a8_1p_in3k5_out1k5_50ms.py
test_type: 'perf'
- name: qwen3_30b_w8a8_1p_in3k5_out1k5_50ms_aime25
runner: linux-aarch64-a3-2
test_case: test/registered/npu/accuracy/qwen3_30b_a3b/test_npu_qwen3_30b_w8a8_1p_in3k5_out1k5_50ms_aime25.py
# qwen3-8b performance tests
- name: qwen3_8b_w8a8_1p_in3k5_out1k5_50ms
runner: linux-aarch64-a3-2
test_case: test/registered/npu/performance/qwen3-8b/test_npu_qwen3_8b_w8a8_1p_in3k5_out1k5_50ms.py
test_type: 'perf'
- name: qwen3_8b_w8a8_1p_in3k5_out1k5_50ms_gpqa
runner: linux-aarch64-a3-2
test_case: test/registered/npu/accuracy/qwen3-8b/test_npu_qwen3_8b_w8a8_1p_in3k5_out1k5_50ms_gpqa.py
- name: qwen3_8b_w8a8_1p_in6k_out1k5_bs16
runner: linux-aarch64-a3-2
test_case: test/registered/npu/performance/qwen3-8b/test_npu_qwen3_8b_w8a8_1p_in6k_out1k5_bs16.py
test_type: 'perf'
- name: qwen3_8b_w8a8_1p_in6k_out1k5_bs16_gpqa
runner: linux-aarch64-a3-2
test_case: test/registered/npu/accuracy/qwen3-8b/test_npu_qwen3_8b_w8a8_1p_in6k_out1k5_bs16_gpqa.py
# qwen3_next_80b_a3b_instruct performance tests
- name: qwen3_next_80b_w8a8_2p_in6k_out1k5_bs16
runner: linux-aarch64-a3-4
test_case: test/registered/npu/performance/qwen3_next_80b_a3b_instruct/test_npu_qwen3_next_80b_w8a8_2p_in6k_out1k5_bs16.py
test_type: 'perf'
- name: qwen3_next_80b_w8a8_2p_in6k_out1k5_bs16_aime25
runner: linux-aarch64-a3-4
test_case: test/registered/npu/accuracy/qwen3_next_80b_a3b_instruct/test_npu_qwen3_next_80b_w8a8_2p_in6k_out1k5_bs16_aime25.py
# minimax_m2_5 performance tests
- name: minimax_m2_5_w8a8_8p_in3k5_out1k5_50ms
runner: linux-aarch64-a3-16
test_case: test/registered/npu/performance/minimax_m2_5/test_npu_minimax_m2_5_w8a8_8p_in3k5_out1k5_50ms.py
test_type: 'perf'
- name: minimax_m2_5_w8a8_8p_in3k5_out1k5_50ms_gpqa
runner: linux-aarch64-a3-16
test_case: test/registered/npu/accuracy/minimax_m2_5/test_npu_minimax_m2_5_w8a8_8p_in3k5_out1k5_50ms_gpqa.py
- name: minimax_m2_5_w8a8_4p_in64k_out1k_prefix90_50ms
runner: linux-aarch64-a3-16
test_case: test/registered/npu/performance/minimax_m2_5/test_npu_minimax_m2_5_w8a8_4p_in64k_out1k_prefix90_50ms.py
test_type: 'perf'
- name: minimax_m2_5_w8a8_4p_in64k_out1k_prefix90_50ms_gpqa
runner: linux-aarch64-a3-16
test_case: test/registered/npu/accuracy/minimax_m2_5/test_npu_minimax_m2_5_w8a8_4p_in64k_out1k_prefix90_50ms_gpqa.py
# deepseek_v3_2 accuracy tests
- name: deepseek_v3_2_8p_aime25
runner: linux-aarch64-a3-16
test_case: test/registered/npu/accuracy/deepseek_v3_2/test_npu_deepseek_v3_2_8p_aime25.py
# glm4_7_flash accuracy tests
- name: glm4_7_flash_1p_aime25
runner: linux-aarch64-a3-2
test_case: test/registered/npu/accuracy/glm4_7_flash/test_npu_glm4_7_flash_1p_aime25.py
# qwen3_vl_8b_thinking accuracy tests
- name: qwen3_vl_8b_thinking_1p_mmmu
runner: linux-aarch64-a3-2
test_case: test/registered/npu/accuracy/qwen3_vl_8b_thinking/test_npu_qwen3_vl_8b_thinking_1p_mmmu.py
# qwen3_vl_30b_a3b_thinking accuracy tests
- name: qwen3_vl_30b_a3b_thinking_1p_mmmu
runner: linux-aarch64-a3-2
test_case: test/registered/npu/accuracy/qwen3_vl_30b_a3b_thinking/test_npu_qwen3_vl_30b_a3b_thinking_1p_mmmu.py
- name: qwen3_235b_w8a8_8p_in3k5_out1k5_50ms
runner: linux-aarch64-a3-16
test_case: test/registered/npu/performance/qwen3_235b_a22b/test_npu_qwen3_235b_w8a8_8p_in3k5_out1k5_50ms.py
test_type: 'perf'
- name: qwen3_5_397b_w4a8_8p_in3k5_out1k5_50ms
runner: linux-aarch64-a3-16
test_case: test/registered/npu/performance/qwen3_5_397b/test_npu_qwen3_5_397b_w4a8_8p_in3k5_out1k5_50ms.py
test_type: 'perf'
- name: kimi_k2_6_w4a8_8p_in3k5_out1k5_20ms
runner: linux-aarch64-a3-16
test_case: test/registered/npu/performance/kimi_k2_6/test_npu_kimi_k2_6_w4a8_8p_in3k5_out1k5_20ms.py
test_type: 'perf'
# deepseek_v4_flash performance tests
- name: deepseek_v4_flash_w8a8_8p_in8k_out1k_50ms
runner: linux-aarch64-a3-16
test_case: test/registered/npu/performance/deepseek_v4_flash/test_npu_deepseek_v4_flash_w8a8_8p_in8k_out1k_50ms.py
test_type: 'perf'
uses: ./.github/workflows/nightly-test-npu-e2e-single-node.yml
uses: ./.github/workflows/_npu-single-node-test-stage.yml
with:
runner: ${{ matrix.test_config.runner }}
test_type: ${{ matrix.test_config.test_type }}
test_config_name: ${{ matrix.test_config.name }}
test_case: ${{ matrix.test_config.test_case }}
runner: linux-aarch64-a2-4
test_type: 'perf'
test_suite: nightly-1-npu-a2
is_nightly_pipeline_job: true
skip_pr_test_health_check: 'true'
image: ${{ needs.set-image-config.outputs.image_a2 }}
install_sglang_deps: false
device_type_for_deps: '910b'
run_start_metadata: ${{ needs.set-image-config.outputs.run_start_metadata }}
nightly-1-npu-a3:
name: nightly-1-npu-a3
if: ${{ !cancelled() }}
needs: [set-image-config]
uses: ./.github/workflows/_npu-pr-test-stage.yml
with:
self_name: nightly-1-npu-a3
runner_config: linux-aarch64-a3-2-
image: ${{ needs.set-image-config.outputs.image_a3 }}
install_sglang_from_source: false
transformers_version: ''
run_timeout_minutes: '60'
timeout_per_file: '3600'
is_nightly_pipeline_job: true
skip_pr_test_health_check: 'true'
secrets: inherit
nightly-2-npu-a3:
name: nightly-2-npu-a3
if: ${{ !cancelled() }}
needs: [set-image-config]
uses: ./.github/workflows/_npu-pr-test-stage.yml
with:
self_name: nightly-2-npu-a3
runner_config: linux-aarch64-a3-2-
image: ${{ needs.set-image-config.outputs.image_a3 }}
run_timeout_minutes: '60'
timeout_per_file: '3600'
is_nightly_pipeline_job: true
skip_pr_test_health_check: 'true'
secrets: inherit
# The nightly-4-npu-a3 suite is split across 2 parallel partition jobs, mirroring base-b-test-4-npu-a3 in the PR pipeline.
nightly-4-npu-a3:
name: nightly-4-npu-a3
if: ${{ !cancelled() }}
needs: [set-image-config]
uses: ./.github/workflows/_npu-pr-test-stage.yml
with:
self_name: nightly-4-npu-a3
runner_config: linux-aarch64-a3-4-
image: ${{ needs.set-image-config.outputs.image_a3 }}
run_timeout_minutes: '120'
timeout_per_file: '3600'
partitions: '{"size":2,"arr":[0,1]}'
is_nightly_pipeline_job: true
skip_pr_test_health_check: 'true'
secrets: inherit
nightly-8-npu-a3:
name: nightly-8-npu-a3
if: ${{ !cancelled() }}
needs: [set-image-config]
uses: ./.github/workflows/_npu-pr-test-stage.yml
with:
self_name: nightly-8-npu-a3
runner_config: linux-aarch64-a3-8-
image: ${{ needs.set-image-config.outputs.image_a3 }}
run_timeout_minutes: '60'
timeout_per_file: '3600'
is_nightly_pipeline_job: true
skip_pr_test_health_check: 'true'
secrets: inherit
nightly-16-npu-a3:
name: nightly-16-npu-a3
if: ${{ !cancelled() }}
needs: [set-image-config]
uses: ./.github/workflows/_npu-pr-test-stage.yml
with:
self_name: nightly-16-npu-a3
runner_config: linux-aarch64-a3-16-
image: ${{ needs.set-image-config.outputs.image_a3 }}
run_timeout_minutes: '120'
timeout_per_file: '3600'
is_nightly_pipeline_job: true
skip_pr_test_health_check: 'true'
secrets: inherit
nightly-perf-2-npu-a3:
name: nightly-perf-2-npu-a3
if: ${{ !cancelled() }}
needs: [set-image-config]
uses: ./.github/workflows/_npu-single-node-test-stage.yml
with:
runner: linux-aarch64-a3-800t-2
test_type: 'perf'
test_suite: nightly-perf-2-npu-a3
is_nightly_pipeline_job: true
skip_pr_test_health_check: 'true'
image: ${{ needs.set-image-config.outputs.image_a3 }}
install_sglang_deps: false
device_type_for_deps: 'a3'
run_start_metadata: ${{ needs.set-image-config.outputs.run_start_metadata }}
nightly-perf-4-npu-a3:
name: nightly-perf-4-npu-a3
if: ${{ !cancelled() }}
needs: [set-image-config]
uses: ./.github/workflows/_npu-single-node-test-stage.yml
with:
runner: linux-aarch64-a3-800t-4
test_type: 'perf'
test_suite: nightly-perf-4-npu-a3
is_nightly_pipeline_job: true
skip_pr_test_health_check: 'true'
image: ${{ needs.set-image-config.outputs.image_a3 }}
install_sglang_deps: false
device_type_for_deps: 'a3'
run_start_metadata: ${{ needs.set-image-config.outputs.run_start_metadata }}
nightly-perf-8-npu-a3:
name: nightly-perf-8-npu-a3
if: ${{ !cancelled() }}
needs: [set-image-config]
uses: ./.github/workflows/_npu-single-node-test-stage.yml
with:
runner: linux-aarch64-a3-800t-8
test_type: 'perf'
test_suite: nightly-perf-8-npu-a3
is_nightly_pipeline_job: true
skip_pr_test_health_check: 'true'
image: ${{ needs.set-image-config.outputs.image_a3 }}
install_sglang_deps: false
device_type_for_deps: 'a3'
run_start_metadata: ${{ needs.set-image-config.outputs.run_start_metadata }}
nightly-perf-16-npu-a3:
name: nightly-perf-16-npu-a3
if: ${{ !cancelled() }}
needs: [set-image-config]
uses: ./.github/workflows/_npu-single-node-test-stage.yml
with:
runner: linux-aarch64-a3-800t-16
test_type: 'perf'
test_suite: nightly-perf-16-npu-a3
is_nightly_pipeline_job: true
skip_pr_test_health_check: 'true'
image: ${{ needs.set-image-config.outputs.image_a3 }}
install_sglang_deps: false
device_type_for_deps: 'a3'
run_start_metadata: ${{ needs.set-image-config.outputs.run_start_metadata }}
# The acc-2 suite is split across 4 parallel partition jobs via the single `partitions` input.
nightly-acc-2-npu-a3:
name: nightly-acc-2-npu-a3
if: ${{ !cancelled() }}
needs: [set-image-config]
uses: ./.github/workflows/_npu-single-node-test-stage.yml
with:
runner: linux-aarch64-a3-2-
test_type: 'accuracy'
test_suite: nightly-acc-2-npu-a3
is_nightly_pipeline_job: true
partitions: '{"size":4,"arr":[0,1,2,3]}'
skip_pr_test_health_check: 'true'
image: ${{ needs.set-image-config.outputs.image_a3 }}
install_sglang_deps: false
device_type_for_deps: 'a3'
run_start_metadata: ${{ needs.set-image-config.outputs.run_start_metadata }}
nightly-acc-4-npu-a3:
name: nightly-acc-4-npu-a3
if: ${{ !cancelled() }}
needs: [set-image-config]
uses: ./.github/workflows/_npu-single-node-test-stage.yml
with:
runner: linux-aarch64-a3-4-
test_type: 'accuracy'
test_suite: nightly-acc-4-npu-a3
is_nightly_pipeline_job: true
skip_pr_test_health_check: 'true'
image: ${{ needs.set-image-config.outputs.image_a3 }}
install_sglang_deps: false
device_type_for_deps: 'a3'
run_start_metadata: ${{ needs.set-image-config.outputs.run_start_metadata }}
nightly-acc-16-npu-a3:
name: nightly-acc-16-npu-a3
if: ${{ !cancelled() }}
needs: [set-image-config]
uses: ./.github/workflows/_npu-single-node-test-stage.yml
with:
runner: linux-aarch64-a3-16-
test_type: 'accuracy'
test_suite: nightly-acc-16-npu-a3
is_nightly_pipeline_job: true
skip_pr_test_health_check: 'true'
image: ${{ needs.set-image-config.outputs.image_a3 }}
install_sglang_deps: false
device_type_for_deps: 'a3'
run_start_metadata: ${{ needs.set-image-config.outputs.run_start_metadata }}
nightly-poc-multi-node-tests:
name: multi-node-poc
if: ${{ !cancelled() }}
needs: [set-image-config, nightly-poc-single-node-tests]
needs: [set-image-config, nightly-perf-2-npu-a3, nightly-perf-4-npu-a3, nightly-perf-16-npu-a3, nightly-acc-2-npu-a3, nightly-acc-4-npu-a3, nightly-acc-16-npu-a3]
strategy:
fail-fast: false
max-parallel: 1
@@ -326,6 +351,7 @@ jobs:
decode_size: 2
router_size: 1
test_case: test/registered/npu/accuracy/glm5_1/test_npu_glm5_1_w4a8_1p1d_32p_in64k_out1k_50ms_aime26.py
test_type: 'accuracy'
prefill_decode_deployment: 'separation'
# mimo_v2_flash performance tests
- name: test_npu_mimo_v2_flash_1p1d_12p_in16k_out1_ttft_5s
@@ -344,8 +370,8 @@ jobs:
prefill_decode_deployment: 'separation'
uses: ./.github/workflows/nightly-test-npu-e2e-multi-node.yml
with:
runner: linux-amd64-cpu-8
test_type: ${{ matrix.test_config.test_type }}
runner: linux-amd64-cpu-4
test_type: ${{ matrix.test_config.test_type || 'perf' }}
test_config_name: ${{ matrix.test_config.name }}
prefill_size: ${{ matrix.test_config.prefill_size }}
decode_size: ${{ matrix.test_config.decode_size }}
@@ -355,11 +381,12 @@ jobs:
install_sglang_from_source: false
prefill_decode_deployment: ${{ matrix.test_config.prefill_decode_deployment }}
transformers_version: ''
run_start_metadata: ${{ needs.set-image-config.outputs.run_start_metadata }}
nightly-poc-multi-node-mix-tests:
name: multi-node-mix-poc
if: ${{ !cancelled() }}
needs: [set-image-config, nightly-poc-single-node-tests, nightly-poc-multi-node-tests]
needs: [set-image-config, nightly-perf-2-npu-a3, nightly-perf-4-npu-a3, nightly-perf-16-npu-a3, nightly-acc-2-npu-a3, nightly-acc-4-npu-a3, nightly-acc-16-npu-a3, nightly-poc-multi-node-tests]
strategy:
fail-fast: false
max-parallel: 1
@@ -373,10 +400,11 @@ jobs:
- name: kimi_k2_6_w4a8_16p_in64k_out1k_100ms_aime25
node_size: 2
test_case: test/registered/npu/accuracy/kimi_k2_6/test_npu_kimi_k2_6_w4a8_16p_in64k_out1k_100ms_aime25.py
test_type: 'accuracy'
uses: ./.github/workflows/nightly-test-npu-e2e-multi-node.yml
with:
runner: linux-amd64-cpu-8
test_type: ${{ matrix.test_config.test_type }}
runner: linux-amd64-cpu-4
test_type: ${{ matrix.test_config.test_type || 'perf' }}
test_config_name: ${{ matrix.test_config.name }}
node_size: ${{ matrix.test_config.node_size }}
test_case: ${{ matrix.test_config.test_case }}
@@ -384,11 +412,24 @@ jobs:
install_sglang_from_source: false
prefill_decode_deployment: 'mix'
transformers_version: ''
run_start_metadata: ${{ needs.set-image-config.outputs.run_start_metadata }}
check-all-jobs:
if: ${{ !cancelled() }}
needs:
- nightly-poc-single-node-a2-tests
- nightly-poc-single-node-tests
- nightly-1-npu-a2
- nightly-1-npu-a3
- nightly-2-npu-a3
- nightly-4-npu-a3
- nightly-8-npu-a3
- nightly-16-npu-a3
- nightly-perf-2-npu-a3
- nightly-perf-4-npu-a3
- nightly-perf-8-npu-a3
- nightly-perf-16-npu-a3
- nightly-acc-2-npu-a3
- nightly-acc-4-npu-a3
- nightly-acc-16-npu-a3
- nightly-poc-multi-node-tests
- nightly-poc-multi-node-mix-tests
runs-on: ubuntu-latest
@@ -402,19 +443,30 @@ jobs:
- name: Generate results table
run: |
status_emoji() {
case "$1" in
pass) echo "✅" ;;
fail) echo "❌" ;;
*) echo "❓" ;;
esac
}
single_result_a2="${{ needs.nightly-poc-single-node-a2-tests.result }}"
single_result="${{ needs.nightly-poc-single-node-tests.result }}"
single_result_a2="${{ needs.nightly-1-npu-a2.result }}"
multi_result="${{ needs.nightly-poc-multi-node-tests.result }}"
mix_result="${{ needs.nightly-poc-multi-node-mix-tests.result }}"
# single-node suites are reported per suite; overall status is success if none failed
single_result="success"
for r in \
"${{ needs.nightly-1-npu-a3.result }}" \
"${{ needs.nightly-2-npu-a3.result }}" \
"${{ needs.nightly-4-npu-a3.result }}" \
"${{ needs.nightly-8-npu-a3.result }}" \
"${{ needs.nightly-16-npu-a3.result }}" \
"${{ needs.nightly-perf-2-npu-a3.result }}" \
"${{ needs.nightly-perf-4-npu-a3.result }}" \
"${{ needs.nightly-perf-8-npu-a3.result }}" \
"${{ needs.nightly-perf-16-npu-a3.result }}" \
"${{ needs.nightly-acc-2-npu-a3.result }}" \
"${{ needs.nightly-acc-4-npu-a3.result }}" \
"${{ needs.nightly-acc-16-npu-a3.result }}"; do
if [ "${r}" != "success" ] && [ "${r}" != "skipped" ]; then
single_result="failure"
fi
done
group_icon() {
case "$1" in
success) echo "✅" ;;
@@ -429,8 +481,24 @@ jobs:
echo "" >> $GITHUB_STEP_SUMMARY
echo "| Group | Status |" >> $GITHUB_STEP_SUMMARY
echo "|-------|--------|" >> $GITHUB_STEP_SUMMARY
echo "| single-node-poc-a2 | $(group_icon ${single_result_a2}) ${single_result_a2} |" >> $GITHUB_STEP_SUMMARY
echo "| single-node-poc | $(group_icon ${single_result}) ${single_result} |" >> $GITHUB_STEP_SUMMARY
echo "| nightly-1-npu-a2 | $(group_icon ${single_result_a2}) ${single_result_a2} |" >> $GITHUB_STEP_SUMMARY
for entry in \
"nightly-1-npu-a3:${{ needs.nightly-1-npu-a3.result }}" \
"nightly-2-npu-a3:${{ needs.nightly-2-npu-a3.result }}" \
"nightly-4-npu-a3:${{ needs.nightly-4-npu-a3.result }}" \
"nightly-8-npu-a3:${{ needs.nightly-8-npu-a3.result }}" \
"nightly-16-npu-a3:${{ needs.nightly-16-npu-a3.result }}" \
"nightly-perf-2-npu-a3:${{ needs.nightly-perf-2-npu-a3.result }}" \
"nightly-perf-4-npu-a3:${{ needs.nightly-perf-4-npu-a3.result }}" \
"nightly-perf-8-npu-a3:${{ needs.nightly-perf-8-npu-a3.result }}" \
"nightly-perf-16-npu-a3:${{ needs.nightly-perf-16-npu-a3.result }}" \
"nightly-acc-2-npu-a3:${{ needs.nightly-acc-2-npu-a3.result }}" \
"nightly-acc-4-npu-a3:${{ needs.nightly-acc-4-npu-a3.result }}" \
"nightly-acc-16-npu-a3:${{ needs.nightly-acc-16-npu-a3.result }}"; do
suite="${entry%%:*}"
r="${entry##*:}"
echo "| ${suite} | $(group_icon ${r}) ${r} |" >> $GITHUB_STEP_SUMMARY
done
echo "| multi-node-poc | $(group_icon ${multi_result}) ${multi_result} |" >> $GITHUB_STEP_SUMMARY
echo "| multi-node-mix-poc | $(group_icon ${mix_result}) ${mix_result} |" >> $GITHUB_STEP_SUMMARY
echo "" >> $GITHUB_STEP_SUMMARY
@@ -439,8 +507,11 @@ jobs:
echo "" >> $GITHUB_STEP_SUMMARY
has_metrics=false
for dir in /tmp/metrics/metrics-*/; do
file="${dir}metrics.json"
# Each suite job uploads its METRICS_DATA_FILE directory, which
# contains per-case subdirectories with their own metrics.json.
# Recursively find every metrics.json and render one table row per
# test case.
for file in $(find /tmp/metrics -name metrics.json 2>/dev/null); do
if [ -f "$file" ]; then
has_metrics=true
echo "import json" > /tmp/parse_metrics.py
@@ -469,5 +540,7 @@ jobs:
echo "" >> $GITHUB_STEP_SUMMARY
FAIL=0
if [ "${single_result}" != "success" ] && [ "${single_result}" != "skipped" ]; then FAIL=1; fi
for r in "${single_result_a2}" "${single_result}" "${multi_result}" "${mix_result}"; do
if [ "${r}" != "success" ] && [ "${r}" != "skipped" ]; then FAIL=1; fi
done
exit $FAIL