[NPU] Improve the execution efficiency and maintainability of pr‑test‑npu (#33724)

Co-authored-by: Even Zhou <even.y.zhou@outlook.com>
Co-authored-by: sglang-npu-bot <sglangnpu@163.com>
This commit is contained in:
Cherry_ming
2026-08-08 16:42:44 +08:00
committed by GitHub
co-authored by Even Zhou sglang-npu-bot
parent e732c0a9dc
commit dd5d82bead
60 changed files with 978 additions and 565 deletions
+213
View File
@@ -0,0 +1,213 @@
name: PR Test Stage for NPU
# Reusable workflow for one CUDA test stage. Caller pr-test-npu.yml forwards
on:
workflow_call:
inputs:
self_name:
description: 'Caller job key; used for partitions[suite] lookup.'
type: string
required: true
runner_config:
description: 'Key in scripts/ci/runner_configs.yml. Resolves install script, artifact version, install timeout, runs-on label, and rdma_devices.'
type: string
required: true
image:
description: 'The container image for NPU test environment.'
type: string
required: true
npu_device_type:
description: 'NPU device type, e.g. 910b, a3.'
type: string
default: 'a3'
run_timeout_minutes:
description: 'timeout-minutes for the Run test step. Required so compute_partitions.py can read it from pr-test.yml without a duplicated default constant.'
type: string
required: true
timeout_per_file:
description: 'run_suite.py --timeout-per-file value (empty = unset).'
type: string
default: ''
partitions:
description: 'partitions config, e.g. {"size":1,"arr":[0]}'
type: string
default: '{"size":1,"arr":[0]}'
ref:
description: 'Git ref (branch, tag, or SHA) to test. If not provided, uses the default branch.'
type: string
default: ''
skip_pr_test_health_check:
description: 'Git ref (branch, tag, or SHA) to test. If not provided, uses the default branch.'
type: string
default: 'false'
github-token:
description: 'GitHub token for API calls'
type: string
default: ${{ github.token }}
env:
SKIP_PR_TEST_HEALTH_CHECK: ${{ inputs.skip_pr_test_health_check }}
SGLANG_USE_MODELSCOPE: true
SGLANG_IS_IN_CI: true
HF_ENDPOINT: https://hf-mirror.com
TORCH_EXTENSIONS_DIR: /tmp/torch_extensions
PYTORCH_NPU_ALLOC_CONF: "expandable_segments:True"
STREAMS_PER_DEVICE: 32
jobs:
run:
runs-on: ${{ inputs.runner_config }}
container:
image: ${{ inputs.image }}
timeout-minutes: 240
strategy:
fail-fast: false
matrix:
partition: ${{ fromJson(inputs.partitions).arr }}
steps:
- name: Checkout code
uses: actions/checkout@v4
with:
ref: ${{ inputs.ref || github.ref }}
- name: Mark repository safe
run: |
git config --system --add safe.directory ${GITHUB_WORKSPACE}
- name: Check PR test health
uses: actions/github-script@v8
env:
SKIP_PR_TEST_HEALTH_CHECK: ${{ env.SKIP_PR_TEST_HEALTH_CHECK }}
with:
github-token: ${{ inputs.github-token || github.token }}
script: |
core.notice(`[health-check] START — event=${context.eventName}, runId=${context.runId}`);
// Skip when explicitly requested via env var (e.g. release branch cut)
if (process.env.SKIP_PR_TEST_HEALTH_CHECK === 'true') {
core.notice('[health-check] SKIP: SKIP_PR_TEST_HEALTH_CHECK=true');
return;
}
// Skip for scheduled runs — they should collect all failures, not fast-fail
if (context.eventName === 'schedule') {
core.notice('[health-check] SKIP: scheduled run');
return;
}
// Check lint status from the separate Lint workflow (lint.yml).
// listJobsForWorkflowRun only sees jobs within the SAME run, so we use
// checks.listForRef which queries by commit SHA across ALL workflows.
const ref = context.payload.pull_request?.head?.sha || context.sha;
core.info(`[health-check] Checking lint for ref=${ref}`);
const { data } = await github.rest.checks.listForRef({
owner: context.repo.owner,
repo: context.repo.repo,
ref: ref,
check_name: 'lint',
});
const lintRun = data.check_runs.find(
cr => cr.app?.slug === 'github-actions'
);
core.info(`[health-check] Lint check: status=${lintRun?.status}, conclusion=${lintRun?.conclusion}`);
if (lintRun?.status === 'completed' && lintRun?.conclusion === 'failure') {
core.setFailed('Fast-fail: lint check failed');
return;
}
// Skip the jobs-failed check when the PR carries the bypass-fastfail label.
// Lint check above still runs.
let labels = [];
if (context.payload.pull_request?.labels) {
labels = context.payload.pull_request.labels.map(l => l.name);
} else {
const { data: prs } = await github.rest.repos.listPullRequestsAssociatedWithCommit({
owner: context.repo.owner,
repo: context.repo.repo,
commit_sha: ref,
});
if (prs.length > 0) {
labels = prs[0].labels.map(l => l.name);
}
}
core.info(`[health-check] PR labels: [${labels.join(', ')}]`);
if (labels.includes('bypass-fastfail')) {
core.notice('[health-check] SKIP jobs-failed check: bypass-fastfail label present');
return;
}
const jobs = await github.paginate(github.rest.actions.listJobsForWorkflowRun, {
owner: context.repo.owner,
repo: context.repo.repo,
run_id: context.runId,
per_page: 100,
});
core.info(`[health-check] Total jobs in run: ${jobs.length}`);
const failedJobs = jobs.filter(j => j.status === 'completed' && j.conclusion === 'failure');
core.info(`[health-check] Failed jobs (before filtering): ${failedJobs.map(j => `${j.name}(${j.conclusion})`).join(', ') || 'none'}`);
// Find jobs that failed from a real error, not from fast-fail cascade
const rootCauseFailures = jobs.filter(j => {
if (j.status !== 'completed' || j.conclusion !== 'failure') return false;
// h20 runners are flaky (dirty GPU state from prior runs); their failures
// should not cascade fast-fail to other stages. j.name shape from
// listJobsForWorkflowRun: "<job-key>" + optional " / <reusable-job>"
// + optional " (<matrix>)". Split off the base job key before exact
// match so we cover both inline + reusable forms without confusing
// 'h20' with the 'h200' prefix.
const baseName = j.name.split(/[ /]/)[0];
if (baseName === 'base-c-test-8-gpu-h20') {
core.info(`[health-check] Filtered out h20 job: ${j.name}`);
return false;
}
// If the failing step is the health check, it's a cascade — skip it
const failedStep = (j.steps || []).find(s => s.conclusion === 'failure');
if (failedStep && (failedStep.name.includes('check-pr-test-health') || failedStep.name.includes('Check PR test health'))) {
core.info(`[health-check] Filtered out cascade failure: ${j.name} (failed step: ${failedStep.name})`);
return false;
}
return true;
});
core.info(`[health-check] Root cause failures (after filtering): ${rootCauseFailures.map(j => j.name).join(', ') || 'none'}`);
if (rootCauseFailures.length > 0) {
core.setFailed(`Fast-fail: skipping — root cause job(s): ${rootCauseFailures.map(j => j.name).join(', ')}`);
} else {
core.notice('[health-check] PASS: no root cause failures detected');
}
- name: Install dependencies
env:
TORCH_CACHE_URL: "http://cache-service.nginx-pypi-cache.svc.cluster.local/whl/cpu"
PYPI_CACHE_URL: "http://cache-service.nginx-pypi-cache.svc.cluster.local/pypi/simple"
UV_INDEX_URL: "http://cache-service.nginx-pypi-cache.svc.cluster.local/pypi/simple"
GITHUB_PROXY_URL: "https://gh-proxy.test.osinfra.cn/"
RUSTUP_CACHE_URL: "http://cache-service.nginx-pypi-cache.svc.cluster.local:8082"
run: |
# speed up by using infra cache services
CACHING_URL="cache-service.nginx-pypi-cache.svc.cluster.local"
sed -Ei "s@(ports|archive).ubuntu.com@${CACHING_URL}:8081@g" /etc/apt/sources.list
pip config set global.index-url http://${CACHING_URL}/pypi/simple
pip config set global.trusted-host "${CACHING_URL}"
bash scripts/ci/npu/npu_ci_install_dependency.sh ${{ inputs.npu_device_type }}
# copy required file from our daily cache
cp ~/.cache/modelscope/hub/datasets/otavia/ShareGPT_Vicuna_unfiltered/ShareGPT_V3_unfiltered_cleaned_split.json /tmp
# copy gsm8k dataset
cp ~/.cache/modelscope/hub/datasets/tmp/test.jsonl /tmp
# install sglang_router
apt-get install -y libssl-dev
pip install sglang_router
- name: Run test
timeout-minutes: ${{ fromJson(inputs.run_timeout_minutes) }}
env:
CONTINUE_ON_ERROR_FLAG: ${{ inputs.continue_on_error == 'true' && '--continue-on-error' || '' }}
run: |
cd test
python3 run_suite.py --hw npu --suite ${{ inputs.self_name }} \
--auto-partition-id ${{ matrix.partition }} \
--auto-partition-size ${{ fromJson(inputs.partitions).size }} \
${{ inputs.timeout_per_file && format('--timeout-per-file {0}', inputs.timeout_per_file) || '' }} \
$CONTINUE_ON_ERROR_FLAG
@@ -0,0 +1,295 @@
name: 'Single Node Template for E2E performance and accuracy tests'
on:
workflow_call:
inputs:
runner:
required: true
type: string
default: linux-aarch64-a3-16
test_type:
required: true
type: string
default: perf
description: perf or accuracy
test_suite:
required: true
type: string
default: ''
description: name of test suite to run via run_suite.py (mutually exclusive with test_case)
image:
required: true
type: string
description: image for pods
default: "swr.cn-southwest-2.myhuaweicloud.com/base_image/dockerhub/lmsysorg/sglang:main-cann9.0.0-a3"
install_sglang_deps:
required: false
type: boolean
default: false
description: install sglang dependencies (e.g. PyTorch, CANN packages) when using source installation
device_type_for_deps:
required: false
type: string
default: 'a3'
description: device type for dependency installation (a3 or 910b)
transformers_version:
required: false
type: string
default: ""
description: "The transformers version number for running sglang. Use default version in image if keep empty."
skip_pr_test_health_check:
description: 'Git ref (branch, tag, or SHA) to test. If not provided, uses the default branch.'
type: string
default: 'false'
github-token:
description: 'GitHub token for API calls'
type: string
default: ${{ github.token }}
env:
SKIP_PR_TEST_HEALTH_CHECK: ${{ inputs.skip_pr_test_health_check }}
concurrency:
group: ascend-nightly-e2e-singlenode-${{ github.workflow_ref }}-${{ github.ref }}-${{ inputs.test_suite }}
cancel-in-progress: true
jobs:
e2e:
name: ${{ inputs.test_suite }}
runs-on: ${{ inputs.runner }}
container:
image: ${{ inputs.image }}
steps:
- name: Checkout code
uses: actions/checkout@v4
- name: Check PR test health
uses: actions/github-script@v8
env:
SKIP_PR_TEST_HEALTH_CHECK: ${{ env.SKIP_PR_TEST_HEALTH_CHECK }}
with:
github-token: ${{ inputs.github-token || github.token }}
script: |
core.notice(`[health-check] START — event=${context.eventName}, runId=${context.runId}`);
// Skip when explicitly requested via env var (e.g. release branch cut)
if (process.env.SKIP_PR_TEST_HEALTH_CHECK === 'true') {
core.notice('[health-check] SKIP: SKIP_PR_TEST_HEALTH_CHECK=true');
return;
}
// Skip for scheduled runs — they should collect all failures, not fast-fail
if (context.eventName === 'schedule') {
core.notice('[health-check] SKIP: scheduled run');
return;
}
// Check lint status from the separate Lint workflow (lint.yml).
// listJobsForWorkflowRun only sees jobs within the SAME run, so we use
// checks.listForRef which queries by commit SHA across ALL workflows.
const ref = context.payload.pull_request?.head?.sha || context.sha;
core.info(`[health-check] Checking lint for ref=${ref}`);
const { data } = await github.rest.checks.listForRef({
owner: context.repo.owner,
repo: context.repo.repo,
ref: ref,
check_name: 'lint',
});
const lintRun = data.check_runs.find(
cr => cr.app?.slug === 'github-actions'
);
core.info(`[health-check] Lint check: status=${lintRun?.status}, conclusion=${lintRun?.conclusion}`);
if (lintRun?.status === 'completed' && lintRun?.conclusion === 'failure') {
core.setFailed('Fast-fail: lint check failed');
return;
}
// Skip the jobs-failed check when the PR carries the bypass-fastfail label.
// Lint check above still runs.
let labels = [];
if (context.payload.pull_request?.labels) {
labels = context.payload.pull_request.labels.map(l => l.name);
} else {
const { data: prs } = await github.rest.repos.listPullRequestsAssociatedWithCommit({
owner: context.repo.owner,
repo: context.repo.repo,
commit_sha: ref,
});
if (prs.length > 0) {
labels = prs[0].labels.map(l => l.name);
}
}
core.info(`[health-check] PR labels: [${labels.join(', ')}]`);
if (labels.includes('bypass-fastfail')) {
core.notice('[health-check] SKIP jobs-failed check: bypass-fastfail label present');
return;
}
const jobs = await github.paginate(github.rest.actions.listJobsForWorkflowRun, {
owner: context.repo.owner,
repo: context.repo.repo,
run_id: context.runId,
per_page: 100,
});
core.info(`[health-check] Total jobs in run: ${jobs.length}`);
const failedJobs = jobs.filter(j => j.status === 'completed' && j.conclusion === 'failure');
core.info(`[health-check] Failed jobs (before filtering): ${failedJobs.map(j => `${j.name}(${j.conclusion})`).join(', ') || 'none'}`);
// Find jobs that failed from a real error, not from fast-fail cascade
const rootCauseFailures = jobs.filter(j => {
if (j.status !== 'completed' || j.conclusion !== 'failure') return false;
// h20 runners are flaky (dirty GPU state from prior runs); their failures
// should not cascade fast-fail to other stages. j.name shape from
// listJobsForWorkflowRun: "<job-key>" + optional " / <reusable-job>"
// + optional " (<matrix>)". Split off the base job key before exact
// match so we cover both inline + reusable forms without confusing
// 'h20' with the 'h200' prefix.
const baseName = j.name.split(/[ /]/)[0];
if (baseName === 'base-c-test-8-gpu-h20') {
core.info(`[health-check] Filtered out h20 job: ${j.name}`);
return false;
}
// If the failing step is the health check, it's a cascade — skip it
const failedStep = (j.steps || []).find(s => s.conclusion === 'failure');
if (failedStep && (failedStep.name.includes('check-pr-test-health') || failedStep.name.includes('Check PR test health'))) {
core.info(`[health-check] Filtered out cascade failure: ${j.name} (failed step: ${failedStep.name})`);
return false;
}
return true;
});
core.info(`[health-check] Root cause failures (after filtering): ${rootCauseFailures.map(j => j.name).join(', ') || 'none'}`);
if (rootCauseFailures.length > 0) {
core.setFailed(`Fast-fail: skipping — root cause job(s): ${rootCauseFailures.map(j => j.name).join(', ')}`);
} else {
core.notice('[health-check] PASS: no root cause failures detected');
}
- name: Check npu info
run: |
npu-smi info
- name: Install sglang dependencies
if: ${{ inputs.install_sglang_deps == true }}
shell: bash
env:
TORCH_CACHE_URL: "http://cache-service.nginx-pypi-cache.svc.cluster.local/whl/cpu"
PYPI_CACHE_URL: "http://cache-service.nginx-pypi-cache.svc.cluster.local/pypi/simple"
UV_INDEX_URL: "http://cache-service.nginx-pypi-cache.svc.cluster.local/pypi/simple"
GITHUB_PROXY_URL: "https://gh-proxy.test.osinfra.cn/"
RUSTUP_CACHE_URL: "http://cache-service.nginx-pypi-cache.svc.cluster.local:8082"
run: |
CACHING_URL="cache-service.nginx-pypi-cache.svc.cluster.local"
sed -Ei "s@(ports|archive).ubuntu.com@${CACHING_URL}:8081@g" /etc/apt/sources.list
pip config set global.index-url http://${CACHING_URL}/pypi/simple
pip config set global.trusted-host "${CACHING_URL}"
bash scripts/ci/npu/npu_ci_install_dependency.sh ${{ inputs.device_type_for_deps }}
cp ~/.cache/modelscope/hub/datasets/otavia/ShareGPT_Vicuna_unfiltered/ShareGPT_V3_unfiltered_cleaned_split.json /tmp
curl -o /tmp/test.jsonl -L https://gh-proxy.test.osinfra.cn/https://raw.githubusercontent.com/openai/grade-school-math/master/grade_school_math/data/test.jsonl
- name: Run test
timeout-minutes: 300
env:
SGLANG_USE_MODELSCOPE: true
HF_ENDPOINT: https://hf-mirror.com
SGLANG_IS_IN_CI: true
TRANSFORMERS_VERBOSITY: "error"
GDN_ATTN_BACKEND_TRITON: 1
SGLANG_TEST_METRICS_OUTPUT: /root/.cache/tests/output/metrics/metrics
shell: bash
run: |
sglang_source_path=$(pwd)
echo "Source code path: ${sglang_source_path}"
ln -sf ${sglang_source_path} /root/sglang
# Determine test mode: suite (run_suite.py --suite) or single case file.
test_suite="${{ inputs.test_suite }}"
echo "Test mode: suite (${test_suite})"
tc_name="${test_suite}"
export TRANSFORMERS_VERSION_FOR_SGLANG="${{ inputs.transformers_version }}"
PYTHON_FOR_SGLANG="python"
PIP_FOR_SGLANG="pip"
if [ -n "${TRANSFORMERS_VERSION_FOR_SGLANG}" ];then
echo "===== Install transformers for sglang - Begin ====="
TRANSFORMERS_PKG_PATH_SOURCE=/root/.cache/.cache/transformers/${TRANSFORMERS_VERSION_FOR_SGLANG}
if [ ! -d "${TRANSFORMERS_PKG_PATH_SOURCE}" ]; then
echo "The dependent transformers package does not exist: ${TRANSFORMERS_PKG_PATH_SOURCE}."
echo "Install transformers ${TRANSFORMERS_VERSION_FOR_SGLANG} online."
pip install transformers=="${TRANSFORMERS_VERSION_FOR_SGLANG}" -i https://mirrors.tuna.tsinghua.edu.cn/pypi/web/simple
else
echo "Install transformers ${TRANSFORMERS_VERSION_FOR_SGLANG} locally."
TRANSFORMERS_PKG_PATH_TARGET=/tmp/transformers/${TRANSFORMERS_VERSION_FOR_SGLANG}
mkdir -p "${TRANSFORMERS_PKG_PATH_TARGET}"
cp "${TRANSFORMERS_PKG_PATH_SOURCE}/*" "${TRANSFORMERS_PKG_PATH_TARGET}/"
pip install --no-index --find-links="${TRANSFORMERS_PKG_PATH_TARGET}" transformers=="${TRANSFORMERS_VERSION_FOR_SGLANG}"
fi
echo "===== Install transformers for sglang in virtual env - End ====="
fi
echo "Transformers version for sglang: $(${PIP_FOR_SGLANG} show transformers | grep Version | cut -d: -f2)"
echo "scaling_governor performance num: \
$(cat /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor | grep performance | wc -l)"
echo "swappiness: $(cat /proc/sys/vm/swappiness)"
echo "numa_balancing: $(cat /proc/sys/kernel/numa_balancing)"
echo "sched_migration_cost_ns: $(cat /proc/sys/kernel/sched_migration_cost_ns)"
export SGLANG_TEST_MAX_RETRY=0
export SGLANG_SET_CPU_AFFINITY=1
echo "SGLANG_SET_CPU_AFFINITY: $SGLANG_SET_CPU_AFFINITY"
# Copy the checked‑out test common utility code to the sglang installation directory.
sglang_pkg_path=/sgl-workspace/sglang/python
ascend_test_util_path=${sglang_pkg_path}/sglang/test/ascend
mkdir -p ${ascend_test_util_path}
mv ${ascend_test_util_path} ${ascend_test_util_path}_bak
cp -r ${sglang_source_path}/python/sglang/test/ascend ${ascend_test_util_path}
source /usr/local/Ascend/cann/set_env.sh || true
source /usr/local/Ascend/nnal/atb/set_env.sh || true
source /usr/local/Ascend/ascend-toolkit/latest/opp/vendors/customize/bin/set_env.bash || true
source /usr/local/Ascend/ascend-toolkit/latest/opp/vendors/custom_transformer/bin/set_env.bash || true
# Set environment of cann
log_path="/root/.cache/tests/logs/log/${current_date}/${tc_name}/${HOSTNAME}"
rm -rf ${log_path}
mkdir -p ${log_path}
echo "Log path: ${log_path}"
echo "Running test: ${tc_name}"
test_exit_code=0
cd test
${PYTHON_FOR_SGLANG} -u run_suite.py --hw npu --suite ${test_suite} --timeout-per-file 3600 2>&1 | tee /tmp/test_output.log || test_exit_code=$?
echo "Finished test: ${tc_name}"
if [ "${test_exit_code}" = "0" ]; then
test_status="pass"
status_icon="✅"
else
test_status="fail"
status_icon="❌"
fi
echo "test_status=${test_status}" >> $GITHUB_ENV
echo "tc_name=${tc_name}" >> $GITHUB_ENV
export test_status tc_name
echo "## ${tc_name} ${status_icon} ${test_status}" >> $GITHUB_STEP_SUMMARY
echo "" >> $GITHUB_STEP_SUMMARY
metric_count=$(grep -c '\[METRIC\]' /tmp/test_output.log 2>/dev/null || echo 0)
if [ "${metric_count}" -gt 0 ]; then
echo "| Metric | Value | Pass |" >> $GITHUB_STEP_SUMMARY
echo "|--------|-------|------|" >> $GITHUB_STEP_SUMMARY
grep '\[METRIC\]' /tmp/test_output.log | while IFS= read -r line; do
metric_name=$(echo "$line" | sed -E 's/.*\[METRIC\] ([^=]+)=.*/\1/')
metric_value=$(echo "$line" | sed -E 's/.*\[METRIC\] [^=]+=([^ ]+).*/\1/')
echo "| ${metric_name} | ${metric_value} | ${status_icon} |" >> $GITHUB_STEP_SUMMARY
done
else
echo "No metrics collected (test may have failed before producing results)." >> $GITHUB_STEP_SUMMARY
fi
echo "" >> $GITHUB_STEP_SUMMARY
exit ${test_exit_code}
+171 -404
View File
@@ -90,303 +90,82 @@ jobs:
echo "CANN_image_a3=swr.cn-southwest-2.myhuaweicloud.com/base_image/ascend-ci/cann:9.0.0-a3-ubuntu22.04-py3.11" >> $GITHUB_OUTPUT
echo "CANN_image_910b=swr.cn-southwest-2.myhuaweicloud.com/base_image/ascend-ci/cann:9.0.0-910b-ubuntu22.04-py3.11" >> $GITHUB_OUTPUT
stage-a-unit-test-npu:
base-a-test-1-npu-a2:
needs: [check-changes, pr-gate, set-image-config]
if: needs.check-changes.outputs.main_package == 'true'
runs-on: linux-aarch64-a2-1
container:
if: ${{ !failure() && !cancelled() && needs.check-changes.outputs.main_package == 'true' }}
uses: ./.github/workflows/_npu-pr-test-stage.yml
with:
self_name: base-a-test-1-npu-a2
runner_config: linux-aarch64-a2-1
image: ${{ needs.set-image-config.outputs.CANN_image_910b }}
steps:
- name: Checkout code
uses: actions/checkout@v4
with:
ref: ${{ inputs.ref || github.ref }}
npu_device_type: 910b
run_timeout_minutes: '15'
secrets: inherit
- name: Mark repository safe
run: |
git config --system --add safe.directory ${GITHUB_WORKSPACE}
- name: Install dependencies
env:
TORCH_CACHE_URL: "http://cache-service.nginx-pypi-cache.svc.cluster.local/whl/cpu"
PYPI_CACHE_URL: "http://cache-service.nginx-pypi-cache.svc.cluster.local/pypi/simple"
UV_INDEX_URL: "http://cache-service.nginx-pypi-cache.svc.cluster.local/pypi/simple"
GITHUB_PROXY_URL: "https://gh-proxy.test.osinfra.cn/"
RUSTUP_CACHE_URL: "http://cache-service.nginx-pypi-cache.svc.cluster.local:8082"
run: |
# speed up by using infra cache services
CACHING_URL="cache-service.nginx-pypi-cache.svc.cluster.local"
sed -Ei "s@(ports|archive).ubuntu.com@${CACHING_URL}:8081@g" /etc/apt/sources.list
pip config set global.index-url http://${CACHING_URL}/pypi/simple
pip config set global.trusted-host "${CACHING_URL}"
bash scripts/ci/npu/npu_ci_install_dependency.sh 910b
- name: Run test
timeout-minutes: 15
env:
SGLANG_IS_IN_CI: true
run: |
cd test
python3 run_suite.py --hw npu --suite stage-a-unit-test-npu
stage-b-test-1-npu-a3:
base-b-test-1-npu-a3:
needs: [check-changes, pr-gate, set-image-config]
if: needs.check-changes.outputs.main_package == 'true'
runs-on: linux-aarch64-a3-2-
container:
if: ${{ !failure() && !cancelled() && needs.check-changes.outputs.main_package == 'true' }}
uses: ./.github/workflows/_npu-pr-test-stage.yml
with:
self_name: base-b-test-1-npu-a3
runner_config: linux-aarch64-a3-2-
image: ${{ needs.set-image-config.outputs.CANN_image_a3 }}
steps:
- name: Checkout code
uses: actions/checkout@v4
with:
ref: ${{ inputs.ref || github.ref }}
run_timeout_minutes: '60'
timeout_per_file: '3600'
secrets: inherit
- name: Mark repository safe
run: |
git config --system --add safe.directory ${GITHUB_WORKSPACE}
- name: Install dependencies
env:
TORCH_CACHE_URL: "http://cache-service.nginx-pypi-cache.svc.cluster.local/whl/cpu"
PYPI_CACHE_URL: "http://cache-service.nginx-pypi-cache.svc.cluster.local/pypi/simple"
UV_INDEX_URL: "http://cache-service.nginx-pypi-cache.svc.cluster.local/pypi/simple"
GITHUB_PROXY_URL: "https://gh-proxy.test.osinfra.cn/"
RUSTUP_CACHE_URL: "http://cache-service.nginx-pypi-cache.svc.cluster.local:8082"
run: |
# speed up by using infra cache services
CACHING_URL="cache-service.nginx-pypi-cache.svc.cluster.local"
sed -Ei "s@(ports|archive).ubuntu.com@${CACHING_URL}:8081@g" /etc/apt/sources.list
pip config set global.index-url http://${CACHING_URL}/pypi/simple
pip config set global.trusted-host "${CACHING_URL}"
bash scripts/ci/npu/npu_ci_install_dependency.sh a3
# copy required file from our daily cache
cp ~/.cache/modelscope/hub/datasets/otavia/ShareGPT_Vicuna_unfiltered/ShareGPT_V3_unfiltered_cleaned_split.json /tmp
# copy gsm8k dataset
cp ~/.cache/modelscope/hub/datasets/tmp/test.jsonl /tmp
- name: Run test
timeout-minutes: 60
env:
SGLANG_USE_MODELSCOPE: true
SGLANG_IS_IN_CI: true
HF_ENDPOINT: https://hf-mirror.com
TORCH_EXTENSIONS_DIR: /tmp/torch_extensions
PYTORCH_NPU_ALLOC_CONF: "expandable_segments:True"
STREAMS_PER_DEVICE: 32
run: |
cd test
python3 run_suite.py --hw npu --suite stage-b-test-1-npu-a3 --timeout-per-file 3600
stage-b-test-2-npu-a3:
base-b-test-2-npu-a3:
needs: [check-changes, pr-gate, set-image-config]
if: needs.check-changes.outputs.main_package == 'true'
runs-on: linux-aarch64-a3-2-
container:
if: ${{ !failure() && !cancelled() && needs.check-changes.outputs.main_package == 'true' }}
uses: ./.github/workflows/_npu-pr-test-stage.yml
with:
self_name: base-b-test-2-npu-a3
runner_config: linux-aarch64-a3-2-
image: ${{ needs.set-image-config.outputs.CANN_image_a3 }}
steps:
- name: Checkout code
uses: actions/checkout@v4
with:
ref: ${{ inputs.ref || github.ref }}
run_timeout_minutes: '60'
timeout_per_file: '3600'
secrets: inherit
- name: Mark repository safe
run: |
git config --system --add safe.directory ${GITHUB_WORKSPACE}
- name: Install dependencies
env:
TORCH_CACHE_URL: "http://cache-service.nginx-pypi-cache.svc.cluster.local/whl/cpu"
PYPI_CACHE_URL: "http://cache-service.nginx-pypi-cache.svc.cluster.local/pypi/simple"
UV_INDEX_URL: "http://cache-service.nginx-pypi-cache.svc.cluster.local/pypi/simple"
GITHUB_PROXY_URL: "https://gh-proxy.test.osinfra.cn/"
RUSTUP_CACHE_URL: "http://cache-service.nginx-pypi-cache.svc.cluster.local:8082"
run: |
# speed up by using infra cache services
CACHING_URL="cache-service.nginx-pypi-cache.svc.cluster.local"
sed -Ei "s@(ports|archive).ubuntu.com@${CACHING_URL}:8081@g" /etc/apt/sources.list
pip config set global.index-url http://${CACHING_URL}/pypi/simple
pip config set global.trusted-host "${CACHING_URL}"
bash scripts/ci/npu/npu_ci_install_dependency.sh a3
# copy required file from our daily cache
cp ~/.cache/modelscope/hub/datasets/otavia/ShareGPT_Vicuna_unfiltered/ShareGPT_V3_unfiltered_cleaned_split.json /tmp
# copy gsm8k dataset
cp ~/.cache/modelscope/hub/datasets/tmp/test.jsonl /tmp
- name: Run test
timeout-minutes: 60
env:
SGLANG_USE_MODELSCOPE: true
SGLANG_IS_IN_CI: true
HF_ENDPOINT: https://hf-mirror.com
TORCH_EXTENSIONS_DIR: /tmp/torch_extensions
PYTORCH_NPU_ALLOC_CONF: "expandable_segments:True"
STREAMS_PER_DEVICE: 32
run: |
cd test
python3 run_suite.py --hw npu --suite stage-b-test-2-npu-a3 --timeout-per-file 3600
stage-b-test-4-npu-a3:
base-b-test-4-npu-a3:
needs: [check-changes, pr-gate, set-image-config]
if: needs.check-changes.outputs.main_package == 'true'
runs-on: linux-aarch64-a3-4-
strategy:
fail-fast: false
matrix:
part: [ 0, 1 ]
container:
if: ${{ !failure() && !cancelled() && needs.check-changes.outputs.main_package == 'true' }}
uses: ./.github/workflows/_npu-pr-test-stage.yml
with:
self_name: base-b-test-4-npu-a3
runner_config: linux-aarch64-a3-4-
image: ${{ needs.set-image-config.outputs.CANN_image_a3 }}
steps:
- name: Checkout code
uses: actions/checkout@v4
with:
ref: ${{ inputs.ref || github.ref }}
run_timeout_minutes: '120'
timeout_per_file: '3600'
partitions: '{"size":2,"arr":[0, 1]}'
secrets: inherit
- name: Mark repository safe
run: |
git config --system --add safe.directory ${GITHUB_WORKSPACE}
- name: Install dependencies
env:
TORCH_CACHE_URL: "http://cache-service.nginx-pypi-cache.svc.cluster.local/whl/cpu"
PYPI_CACHE_URL: "http://cache-service.nginx-pypi-cache.svc.cluster.local/pypi/simple"
UV_INDEX_URL: "http://cache-service.nginx-pypi-cache.svc.cluster.local/pypi/simple"
GITHUB_PROXY_URL: "https://gh-proxy.test.osinfra.cn/"
RUSTUP_CACHE_URL: "http://cache-service.nginx-pypi-cache.svc.cluster.local:8082"
run: |
# speed up by using infra cache services
CACHING_URL="cache-service.nginx-pypi-cache.svc.cluster.local"
sed -Ei "s@(ports|archive).ubuntu.com@${CACHING_URL}:8081@g" /etc/apt/sources.list
pip config set global.index-url http://${CACHING_URL}/pypi/simple
pip config set global.trusted-host "${CACHING_URL}"
bash scripts/ci/npu/npu_ci_install_dependency.sh a3
# copy required file from our daily cache
cp ~/.cache/modelscope/hub/datasets/otavia/ShareGPT_Vicuna_unfiltered/ShareGPT_V3_unfiltered_cleaned_split.json /tmp
# copy gsm8k dataset
cp ~/.cache/modelscope/hub/datasets/tmp/test.jsonl /tmp
- name: Run test
timeout-minutes: 120
env:
SGLANG_USE_MODELSCOPE: true
SGLANG_IS_IN_CI: true
HF_ENDPOINT: https://hf-mirror.com
TORCH_EXTENSIONS_DIR: /tmp/torch_extensions
PYTORCH_NPU_ALLOC_CONF: "expandable_segments:True"
STREAMS_PER_DEVICE: 32
run: |
pip install sglang_router
cd test
python3 run_suite.py --hw npu --suite stage-b-test-4-npu-a3 --auto-partition-id ${{ matrix.part }} --auto-partition-size 2 --timeout-per-file 3600
stage-b-test-8-npu-a3:
base-b-test-8-npu-a3:
needs: [check-changes, pr-gate, set-image-config]
if: needs.check-changes.outputs.main_package == 'true'
runs-on: linux-aarch64-a3-8-
container:
if: ${{ !failure() && !cancelled() && needs.check-changes.outputs.main_package == 'true' }}
uses: ./.github/workflows/_npu-pr-test-stage.yml
with:
self_name: base-b-test-8-npu-a3
runner_config: linux-aarch64-a3-8-
image: ${{ needs.set-image-config.outputs.CANN_image_a3 }}
steps:
- name: Checkout code
uses: actions/checkout@v4
with:
ref: ${{ inputs.ref || github.ref }}
run_timeout_minutes: '60'
timeout_per_file: '3600'
secrets: inherit
- name: Mark repository safe
run: |
git config --system --add safe.directory ${GITHUB_WORKSPACE}
- name: Install dependencies
env:
TORCH_CACHE_URL: "http://cache-service.nginx-pypi-cache.svc.cluster.local/whl/cpu"
PYPI_CACHE_URL: "http://cache-service.nginx-pypi-cache.svc.cluster.local/pypi/simple"
UV_INDEX_URL: "http://cache-service.nginx-pypi-cache.svc.cluster.local/pypi/simple"
GITHUB_PROXY_URL: "https://gh-proxy.test.osinfra.cn/"
RUSTUP_CACHE_URL: "http://cache-service.nginx-pypi-cache.svc.cluster.local:8082"
run: |
# speed up by using infra cache services
CACHING_URL="cache-service.nginx-pypi-cache.svc.cluster.local"
sed -Ei "s@(ports|archive).ubuntu.com@${CACHING_URL}:8081@g" /etc/apt/sources.list
pip config set global.index-url http://${CACHING_URL}/pypi/simple
pip config set global.trusted-host "${CACHING_URL}"
bash scripts/ci/npu/npu_ci_install_dependency.sh a3
# copy required file from our daily cache
cp ~/.cache/modelscope/hub/datasets/otavia/ShareGPT_Vicuna_unfiltered/ShareGPT_V3_unfiltered_cleaned_split.json /tmp
# copy gsm8k dataset
cp ~/.cache/modelscope/hub/datasets/tmp/test.jsonl /tmp
- name: Run test
timeout-minutes: 60
env:
SGLANG_USE_MODELSCOPE: true
SGLANG_IS_IN_CI: true
HF_ENDPOINT: https://hf-mirror.com
TORCH_EXTENSIONS_DIR: /tmp/torch_extensions
PYTORCH_NPU_ALLOC_CONF: "expandable_segments:True"
STREAMS_PER_DEVICE: 32
run: |
cd test
python3 run_suite.py --hw npu --suite stage-b-test-8-npu-a3 --timeout-per-file 3600
stage-b-test-16-npu-a3:
base-b-test-16-npu-a3:
needs: [check-changes, pr-gate, set-image-config]
if: needs.check-changes.outputs.main_package == 'true'
runs-on: linux-aarch64-a3-16-
container:
if: ${{ !failure() && !cancelled() && needs.check-changes.outputs.main_package == 'true' }}
uses: ./.github/workflows/_npu-pr-test-stage.yml
with:
self_name: base-b-test-16-npu-a3
runner_config: linux-aarch64-a3-16-
image: ${{ needs.set-image-config.outputs.CANN_image_a3 }}
steps:
- name: Checkout code
uses: actions/checkout@v4
with:
ref: ${{ inputs.ref || github.ref }}
- name: Mark repository safe
run: |
git config --system --add safe.directory ${GITHUB_WORKSPACE}
- name: Install dependencies
env:
TORCH_CACHE_URL: "http://cache-service.nginx-pypi-cache.svc.cluster.local/whl/cpu"
PYPI_CACHE_URL: "http://cache-service.nginx-pypi-cache.svc.cluster.local/pypi/simple"
UV_INDEX_URL: "http://cache-service.nginx-pypi-cache.svc.cluster.local/pypi/simple"
GITHUB_PROXY_URL: "https://gh-proxy.test.osinfra.cn/"
RUSTUP_CACHE_URL: "http://cache-service.nginx-pypi-cache.svc.cluster.local:8082"
run: |
# speed up by using infra cache services
CACHING_URL="cache-service.nginx-pypi-cache.svc.cluster.local"
sed -Ei "s@(ports|archive).ubuntu.com@${CACHING_URL}:8081@g" /etc/apt/sources.list
pip config set global.index-url http://${CACHING_URL}/pypi/simple
pip config set global.trusted-host "${CACHING_URL}"
bash scripts/ci/npu/npu_ci_install_dependency.sh a3
# copy required file from our daily cache
cp ~/.cache/modelscope/hub/datasets/otavia/ShareGPT_Vicuna_unfiltered/ShareGPT_V3_unfiltered_cleaned_split.json /tmp
# copy gsm8k dataset
cp ~/.cache/modelscope/hub/datasets/tmp/test.jsonl /tmp
- name: Run test
timeout-minutes: 120
env:
SGLANG_USE_MODELSCOPE: true
SGLANG_IS_IN_CI: true
HF_ENDPOINT: https://hf-mirror.com
TORCH_EXTENSIONS_DIR: /tmp/torch_extensions
PYTORCH_NPU_ALLOC_CONF: "expandable_segments:True"
STREAMS_PER_DEVICE: 32
run: |
apt-get install -y libssl-dev
pip install sglang_router
cd test
python3 run_suite.py --hw npu --suite stage-b-test-16-npu-a3 --timeout-per-file 3600
run_timeout_minutes: '120'
timeout_per_file: '3600'
secrets: inherit
multimodal-gen-test-1-npu-a3:
needs: [check-changes, pr-gate, set-image-config]
if: needs.check-changes.outputs.multimodal_gen == 'true'
if: ${{ !failure() && !cancelled() && needs.check-changes.outputs.multimodal_gen == 'true' }}
runs-on: linux-aarch64-a3-800t-2
container:
image: ${{ needs.set-image-config.outputs.CANN_image_a3 }}
@@ -443,7 +222,7 @@ jobs:
multimodal-gen-test-2-npu-a3:
needs: [check-changes, pr-gate, set-image-config]
if: needs.check-changes.outputs.multimodal_gen == 'true'
if: ${{ !failure() && !cancelled() && needs.check-changes.outputs.multimodal_gen == 'true' }}
runs-on: linux-aarch64-a3-800t-2
container:
image: ${{ needs.set-image-config.outputs.CANN_image_a3 }}
@@ -498,146 +277,134 @@ jobs:
if-no-files-found: ignore
retention-days: 7
pr-single-node-tests:
name: single-node-poc
needs: [check-changes, pr-gate, set-image-config]
if: needs.check-changes.outputs.main_package == 'true'
strategy:
fail-fast: false
max-parallel: 6
matrix:
test_config:
# qwen3_6_27b performance tests
- name: qwen3_6_27b_w8a8_1p_in64k_out1k_50ms
runner: linux-aarch64-a3-800t-2
test_case: test/registered/npu/performance/qwen3_6_27b/test_npu_qwen3_6_27b_w8a8_1p_in64k_out1k_50ms.py
test_type: 'perf'
# - name: qwen3_6_27b_1p_in1024x1024_30_out1024_50ms
# runner: linux-aarch64-a3-800t-2
# test_case: test/registered/npu/performance/qwen3_6_27b/test_npu_qwen3_6_27b_1p_in1024x1024_30_out1024_50ms.py
# test_type: 'perf'
# # qwen3_8b performance tests
# - name: qwen3_8b_w8a8_1p_in3k5_out1k5_50ms
# runner: linux-aarch64-a3-800t-2
# test_case: test/registered/npu/performance/qwen3-8b/test_npu_qwen3_8b_w8a8_1p_in3k5_out1k5_50ms.py
# test_type: 'perf'
# # qwen3_30b_a3b performance tests
# - name: qwen3_30b_w8a8_1p_in3k5_out1k5_50ms
# runner: linux-aarch64-a3-800t-2
# test_case: test/registered/npu/performance/qwen3_30b_a3b/test_npu_qwen3_30b_w8a8_1p_in3k5_out1k5_50ms.py
# test_type: 'perf'
# # qwen3_6_35b_a3b performance tests
# - name: qwen3_6_35b_a3b_1p_in64k_out1k_prefix90_50ms
# runner: linux-aarch64-a3-800t-2
# test_case: test/registered/npu/performance/qwen3_6_35b_a3b/test_npu_qwen3_6_35b_a3b_1p_in64k_out1k_prefix90_50ms.py
# test_type: 'perf'
# # qwen3_vl_8b_thinking accuracy tests
# - name: qwen3_vl_8b_thinking_1p_mmmu
# runner: linux-aarch64-a3-2-
# test_case: test/registered/npu/accuracy/qwen3_vl_8b_thinking/test_npu_qwen3_vl_8b_thinking_1p_mmmu.py
# test_type: 'accuracy'
# # qwen3_32b performance tests
# - name: qwen3_32b_w8a8_2p_in3k5_out1k5_50ms
# runner: linux-aarch64-a3-800t-4
# test_case: test/registered/npu/performance/qwen3_32b/test_npu_qwen3_32b_w8a8_2p_in3k5_out1k5_50ms.py
# test_type: 'perf'
# # qwen3_next_80b_a3b performance tests
# - name: qwen3_next_80b_w8a8_2p_in6k_out1k5_bs16
# runner: linux-aarch64-a3-800t-4
# test_case: test/registered/npu/performance/qwen3_next_80b_a3b_instruct/test_npu_qwen3_next_80b_w8a8_2p_in6k_out1k5_bs16.py
# test_type: 'perf'
# # minimax_m2_5 performance tests
# - name: minimax_m2_5_w8a8_4p_in64k_out1k_prefix90_50ms
# runner: linux-aarch64-a3-800t-8
# test_case: test/registered/npu/performance/minimax_m2_5/test_npu_minimax_m2_5_w8a8_4p_in64k_out1k_prefix90_50ms.py
# test_type: 'perf'
# # deepseek_v4_flash performance tests
# - name: deepseek_v4_flash_w8a8_8p_in8k_out1k_50ms
# runner: linux-aarch64-a3-800t-16
# test_case: test/registered/npu/performance/deepseek_v4_flash/test_npu_deepseek_v4_flash_w8a8_8p_in8k_out1k_50ms.py
# test_type: 'perf'
# # kimi_k2_6 performance tests
# - name: kimi_k2_6_w4a8_8p_in3k5_out1k5_20ms
# runner: linux-aarch64-a3-800t-16
# test_case: test/registered/npu/performance/kimi_k2_6/test_npu_kimi_k2_6_w4a8_8p_in3k5_out1k5_20ms.py
# test_type: 'perf'
# # qwen3_235b performance tests
# - name: qwen3_235b_w8a8_8p_in3k5_out1k5_50ms
# runner: linux-aarch64-a3-800t-16
# test_case: test/registered/npu/performance/qwen3_235b_a22b/test_npu_qwen3_235b_w8a8_8p_in3k5_out1k5_50ms.py
# test_type: 'perf'
# # qwen3_5_397b performance tests
# - name: qwen3_5_397b_w4a8_8p_in3k5_out1k5_50ms
# runner: linux-aarch64-a3-800t-16
# test_case: test/registered/npu/performance/qwen3_5_397b/test_npu_qwen3_5_397b_w4a8_8p_in3k5_out1k5_50ms.py
# test_type: 'perf'
# NPU accuracy tests
# - name: glm4_7_flash_1p_gsm8k
# runner: linux-aarch64-a3-2-
# test_case: test/registered/npu/accuracy/glm4_7_flash/test_npu_glm4_7_flash_1p_gsm8k.py
# test_type: 'accuracy'
# - name: qwen3_vl_30b_a3b_bf16_2p_gsm8k
# runner: linux-aarch64-a3-4-
# test_case: test/registered/npu/accuracy/qwen3_vl_30b_a3b/test_npu_qwen3_vl_30b_a3b_bf16_2p_gsm8k.py
# test_type: 'accuracy'
- name: glm5_top64_pruned_bf16_8p_gsm8k
runner: linux-aarch64-a3-16-
test_case: test/registered/npu/accuracy/glm5_top64_pruned/test_npu_glm5_top64_pruned_bf16_8p_gsm8k.py
test_type: 'accuracy'
# - name: moonshotai_moonlight_16b_a3b_bf16_1p_gsm8k
# runner: linux-aarch64-a3-2-
# test_case: test/registered/npu/accuracy/moonshotai_moonlight_16b_a3b/test_npu_moonlight_16b_a3b_bf16_1p_gsm8k.py
# test_type: 'accuracy'
# - name: qwen3_5_9b_bf16_1p_gsm8k
# runner: linux-aarch64-a3-2-
# test_case: test/registered/npu/accuracy/qwen3_5_9b/test_npu_qwen3_5_9b_bf16_1p_gsm8k.py
# test_type: 'accuracy'
# - name: qwen3_vl_8b_bf16_2p_gsm8k
# runner: linux-aarch64-a3-4-
# test_case: test/registered/npu/accuracy/qwen3_vl_8b/test_npu_qwen3_vl_8b_bf16_2p_gsm8k.py
# test_type: 'accuracy'
uses: ./.github/workflows/nightly-test-npu-e2e-single-node.yml
base-c-test-acc-2-npu-a3:
name: base-c-test-acc-2-npu-a3
needs: [ check-changes, pr-gate, set-image-config ]
if: ${{ !failure() && !cancelled() && needs.check-changes.outputs.main_package == 'true' }}
uses: ./.github/workflows/_npu-single-node-test-stage.yml
with:
runner: ${{ matrix.test_config.runner }}
test_type: ${{ matrix.test_config.test_type }}
test_config_name: ${{ matrix.test_config.name }}
test_case: ${{ matrix.test_config.test_case }}
runner: linux-aarch64-a3-2-
test_type: 'accuracy'
test_suite: base-c-test-acc-2-npu-a3
image: ${{ needs.set-image-config.outputs.CANN_image_a3 }}
install_sglang_from_source: false
install_sglang_deps: true
device_type_for_deps: 'a3'
transformers_version: ''
base-c-test-acc-4-npu-a3:
name: base-c-test-acc-4-npu-a3
needs: [ check-changes, pr-gate, set-image-config ]
if: ${{ !failure() && !cancelled() && needs.check-changes.outputs.main_package == 'true' }}
uses: ./.github/workflows/_npu-single-node-test-stage.yml
with:
runner: linux-aarch64-a3-4-
test_type: 'accuracy'
test_suite: base-c-test-acc-4-npu-a3
image: ${{ needs.set-image-config.outputs.CANN_image_a3 }}
install_sglang_deps: true
device_type_for_deps: 'a3'
base-c-test-acc-8-npu-a3:
name: base-c-test-acc-8-npu-a3
needs: [ check-changes, pr-gate, set-image-config ]
if: ${{ !failure() && !cancelled() && needs.check-changes.outputs.main_package == 'true' }}
uses: ./.github/workflows/_npu-single-node-test-stage.yml
with:
runner: linux-aarch64-a3-8-
test_type: 'accuracy'
test_suite: base-c-test-acc-8-npu-a3
image: ${{ needs.set-image-config.outputs.CANN_image_a3 }}
install_sglang_deps: true
device_type_for_deps: 'a3'
base-c-test-acc-16-npu-a3:
name: base-c-test-acc-16-npu-a3
needs: [ check-changes, pr-gate, set-image-config ]
if: ${{ !failure() && !cancelled() && needs.check-changes.outputs.main_package == 'true' }}
uses: ./.github/workflows/_npu-single-node-test-stage.yml
with:
runner: linux-aarch64-a3-16-
test_type: 'accuracy'
test_suite: base-c-test-acc-16-npu-a3
image: ${{ needs.set-image-config.outputs.CANN_image_a3 }}
install_sglang_deps: true
device_type_for_deps: 'a3'
base-c-test-perf-2-npu-a3:
name: base-c-test-perf-2-npu-a3
needs: [ check-changes, pr-gate, set-image-config, base-c-test-acc-2-npu-a3 ]
if: ${{ !failure() && !cancelled() && needs.check-changes.outputs.main_package == 'true' }}
uses: ./.github/workflows/_npu-single-node-test-stage.yml
with:
runner: linux-aarch64-a3-800t-2
test_type: 'perf'
test_suite: base-c-test-perf-2-npu-a3
image: ${{ needs.set-image-config.outputs.CANN_image_a3 }}
install_sglang_deps: true
device_type_for_deps: 'a3'
base-c-test-perf-4-npu-a3:
name: base-c-test-perf-4-npu-a3
needs: [ check-changes, pr-gate, set-image-config, base-c-test-acc-4-npu-a3 ]
if: ${{ !failure() && !cancelled() && needs.check-changes.outputs.main_package == 'true' }}
uses: ./.github/workflows/_npu-single-node-test-stage.yml
with:
runner: linux-aarch64-a3-800t-4
test_type: 'perf'
test_suite: base-c-test-perf-4-npu-a3
image: ${{ needs.set-image-config.outputs.CANN_image_a3 }}
install_sglang_deps: true
device_type_for_deps: 'a3'
base-c-test-perf-8-npu-a3:
name: base-c-test-perf-8-npu-a3
needs: [ check-changes, pr-gate, set-image-config, base-c-test-acc-8-npu-a3 ]
if: ${{ !failure() && !cancelled() && needs.check-changes.outputs.main_package == 'true' }}
uses: ./.github/workflows/_npu-single-node-test-stage.yml
with:
runner: linux-aarch64-a3-800t-8
test_type: 'perf'
test_suite: base-c-test-perf-8-npu-a3
image: ${{ needs.set-image-config.outputs.CANN_image_a3 }}
install_sglang_deps: true
device_type_for_deps: 'a3'
base-c-test-perf-16-npu-a3:
name: base-c-test-perf-16-npu-a3
needs: [ check-changes, pr-gate, set-image-config, base-c-test-acc-16-npu-a3 ]
if: ${{ !failure() && !cancelled() && needs.check-changes.outputs.main_package == 'true' }}
uses: ./.github/workflows/_npu-single-node-test-stage.yml
with:
runner: linux-aarch64-a3-800t-16
test_type: 'perf'
test_suite: base-c-test-perf-16-npu-a3
image: ${{ needs.set-image-config.outputs.CANN_image_a3 }}
install_sglang_deps: true
device_type_for_deps: 'a3'
pr-test-npu-finish:
needs:
[
check-changes,
stage-a-unit-test-npu,
stage-b-test-1-npu-a3,
stage-b-test-2-npu-a3,
stage-b-test-4-npu-a3,
stage-b-test-8-npu-a3,
stage-b-test-16-npu-a3,
base-a-test-1-npu-a2,
base-b-test-1-npu-a3,
base-b-test-2-npu-a3,
base-b-test-4-npu-a3,
base-b-test-8-npu-a3,
base-b-test-16-npu-a3,
multimodal-gen-test-1-npu-a3,
multimodal-gen-test-2-npu-a3,
pr-single-node-tests,
base-c-test-acc-2-npu-a3,
base-c-test-acc-4-npu-a3,
base-c-test-acc-8-npu-a3,
base-c-test-acc-16-npu-a3,
base-c-test-perf-2-npu-a3,
base-c-test-perf-4-npu-a3,
base-c-test-perf-8-npu-a3,
base-c-test-perf-16-npu-a3,
]
if: always()
runs-on: ubuntu-latest
@@ -1,10 +1,15 @@
import glob
import inspect
import json
import logging
import os
import re
import shutil
import subprocess
import sys
import threading
import time
from datetime import datetime
from urllib.parse import urlparse
from sglang.srt.utils import kill_process_tree
@@ -289,9 +294,117 @@ class TestNpuAccuracyTestCaseBase(CustomTestCase):
max_attempts = 2
n_runs = 3
accuracy = 0.1
test_type = "accuracy"
@classmethod
def _get_tc_name(cls):
"""Derive the test case name from the test file (filename without
extension). Mirrors the workflow's ``tc_name=${test_case##*/}`` logic
so each case in a suite writes to its own output path."""
try:
tc_file = inspect.getfile(cls)
except (TypeError, OSError):
tc_file = getattr(sys.modules.get(cls.__module__), "__file__", "")
return os.path.splitext(os.path.basename(tc_file))[0]
@classmethod
def _setup_per_case_output(cls):
"""Set up per-case output directories and env vars.
Extracted from ``nightly-test-npu-e2e-single-node.yml`` so that when a
suite is executed, each case writes its metrics/plog to a path derived
from the case file rather than the suite name.
"""
cls.tc_name = cls._get_tc_name()
current_date = datetime.now().strftime("%Y%m%d")
test_type = getattr(cls, "test_type", "accuracy")
base_output = f"/root/.cache/tests/output/{test_type}/{current_date}"
os.makedirs(base_output, exist_ok=True)
cls.metrics_data_file = os.path.join(base_output, cls.tc_name)
os.makedirs(cls.metrics_data_file, exist_ok=True)
# Override env vars so evalscope/dump_metric write to per-case paths.
os.environ["METRICS_DATA_FILE"] = cls.metrics_data_file
os.environ["SGLANG_TEST_METRICS_OUTPUT"] = os.path.join(
cls.metrics_data_file, "metrics"
)
logger.info(
"Per-case output: tc_name=%s metrics_data_file=%s",
cls.tc_name,
cls.metrics_data_file,
)
@classmethod
def _save_metrics_json(cls):
"""Write per-case ``metrics.json`` from ``dump_metric`` JSONL files.
Replaces the workflow's stdout-parsing + ``dump_metrics.py`` logic so
each case in a suite persists its own metrics snapshot.
"""
if not getattr(cls, "metrics_data_file", None):
return
metrics = {}
baselines = {}
pattern = os.path.join(cls.metrics_data_file, "metrics.*.jsonl")
for jsonl_path in glob.glob(pattern):
try:
with open(jsonl_path, "r", encoding="utf-8") as f:
for line in f:
line = line.strip()
if not line:
continue
record = json.loads(line)
name = record.get("metric_name")
value = record.get("value")
if name is None:
continue
if name.endswith("_baseline"):
baselines[name[: -len("_baseline")]] = value
else:
metrics[name] = value
except Exception as e:
logger.warning("Failed to read %s: %s", jsonl_path, e)
out_path = os.path.join(cls.metrics_data_file, "metrics.json")
payload = {
"test_case": cls.tc_name,
"test_type": getattr(cls, "test_type", "accuracy"),
"metrics": metrics,
"baselines": baselines,
}
try:
with open(out_path, "w", encoding="utf-8") as f:
json.dump(payload, f, ensure_ascii=False)
logger.info("Saved per-case metrics to %s", out_path)
except Exception as e:
logger.warning("Failed to write metrics.json: %s", e)
@classmethod
def _backup_plog(cls):
"""Backup Ascend plog files to a per-case path.
Replaces the workflow's ``Backup plog`` step so each case in a suite
gets its own plog snapshot instead of all cases sharing the suite name.
"""
plog_path = "/root/ascend/log/debug/plog"
if not os.path.isdir(plog_path):
return
tc_name = getattr(cls, "tc_name", None)
if not tc_name:
return
hostname = os.getenv("HOSTNAME", "unknown")
target = os.path.join("/root/.cache/tests/logs/plog", tc_name, hostname)
os.makedirs(target, exist_ok=True)
for name in os.listdir(plog_path):
src = os.path.join(plog_path, name)
if os.path.isfile(src):
try:
shutil.copy2(src, os.path.join(target, name))
except Exception as e:
logger.warning("Failed to copy plog %s: %s", name, e)
logger.info("Backed up plog to %s", target)
@classmethod
def setUpClass(cls):
cls._setup_per_case_output()
cls.base_url = DEFAULT_URL_FOR_TEST
env = os.environ.copy()
for key, value in env.items():
@@ -318,6 +431,8 @@ class TestNpuAccuracyTestCaseBase(CustomTestCase):
kill_process_tree(cls.process.pid)
except Exception as e:
logger.error(f"Error during tearDown: {e}")
cls._save_metrics_json()
cls._backup_plog()
def run_accuracy(self):
parsed_url = urlparse(self.base_url)
@@ -1,9 +1,15 @@
import glob
import inspect
import json
import logging
import os
import re
import shutil
import subprocess
import sys
import threading
import time
from datetime import datetime
from functools import wraps
from urllib.parse import urlparse
@@ -909,8 +915,115 @@ class TestNpuPerformanceTestCaseBase(CustomTestCase):
generation_kwargs = None
pop_sglang_is_in_ci_for_gsp = False
@classmethod
def _get_tc_name(cls):
"""Derive the test case name from the test file (filename without
extension). Mirrors the workflow's ``tc_name=${test_case##*/}`` logic
so each case in a suite writes to its own output path."""
try:
tc_file = inspect.getfile(cls)
except (TypeError, OSError):
tc_file = getattr(sys.modules.get(cls.__module__), "__file__", "")
return os.path.splitext(os.path.basename(tc_file))[0]
@classmethod
def _setup_per_case_output(cls):
"""Set up per-case output directories and env vars.
Extracted from ``nightly-test-npu-e2e-single-node.yml`` so that when a
suite is executed, each case writes its metrics/plog to a path derived
from the case file rather than the suite name.
"""
cls.tc_name = cls._get_tc_name()
current_date = datetime.now().strftime("%Y%m%d")
test_type = getattr(cls, "test_type", "perf")
base_output = f"/root/.cache/tests/output/{test_type}/{current_date}"
os.makedirs(base_output, exist_ok=True)
cls.metrics_data_file = os.path.join(base_output, cls.tc_name)
os.makedirs(cls.metrics_data_file, exist_ok=True)
# Override env vars so evalscope/dump_metric write to per-case paths.
os.environ["METRICS_DATA_FILE"] = cls.metrics_data_file
os.environ["SGLANG_TEST_METRICS_OUTPUT"] = os.path.join(
cls.metrics_data_file, "metrics"
)
logger.info(
"Per-case output: tc_name=%s metrics_data_file=%s",
cls.tc_name,
cls.metrics_data_file,
)
@classmethod
def _save_metrics_json(cls):
"""Write per-case ``metrics.json`` from ``dump_metric`` JSONL files.
Replaces the workflow's stdout-parsing + ``dump_metrics.py`` logic so
each case in a suite persists its own metrics snapshot.
"""
if not getattr(cls, "metrics_data_file", None):
return
metrics = {}
baselines = {}
pattern = os.path.join(cls.metrics_data_file, "metrics.*.jsonl")
for jsonl_path in glob.glob(pattern):
try:
with open(jsonl_path, "r", encoding="utf-8") as f:
for line in f:
line = line.strip()
if not line:
continue
record = json.loads(line)
name = record.get("metric_name")
value = record.get("value")
if name is None:
continue
if name.endswith("_baseline"):
baselines[name[: -len("_baseline")]] = value
else:
metrics[name] = value
except Exception as e:
logger.warning("Failed to read %s: %s", jsonl_path, e)
out_path = os.path.join(cls.metrics_data_file, "metrics.json")
payload = {
"test_case": cls.tc_name,
"test_type": getattr(cls, "test_type", "accuracy"),
"metrics": metrics,
"baselines": baselines,
}
try:
with open(out_path, "w", encoding="utf-8") as f:
json.dump(payload, f, ensure_ascii=False)
logger.info("Saved per-case metrics to %s", out_path)
except Exception as e:
logger.warning("Failed to write metrics.json: %s", e)
@classmethod
def _backup_plog(cls):
"""Backup Ascend plog files to a per-case path.
Replaces the workflow's ``Backup plog`` step so each case in a suite
gets its own plog snapshot instead of all cases sharing the suite name.
"""
plog_path = "/root/ascend/log/debug/plog"
if not os.path.isdir(plog_path):
return
tc_name = getattr(cls, "tc_name", None)
if not tc_name:
return
hostname = os.getenv("HOSTNAME", "unknown")
target = os.path.join("/root/.cache/tests/logs/plog", tc_name, hostname)
os.makedirs(target, exist_ok=True)
for name in os.listdir(plog_path):
src = os.path.join(plog_path, name)
if os.path.isfile(src):
try:
shutil.copy2(src, os.path.join(target, name))
except Exception as e:
logger.warning("Failed to copy plog %s: %s", name, e)
logger.info("Backed up plog to %s", target)
@classmethod
def setUpClass(cls):
cls._setup_per_case_output()
cls.base_url = DEFAULT_URL_FOR_TEST
env = os.environ.copy()
for key, value in env.items():
@@ -937,6 +1050,8 @@ class TestNpuPerformanceTestCaseBase(CustomTestCase):
kill_process_tree(cls.process.pid)
except Exception as e:
logger.error(f"Error during tearDown: {e}")
cls._save_metrics_json()
cls._backup_plog()
@retry()
def run_throughput(self):
@@ -6,12 +6,7 @@ from sglang.test.ascend.e2e.test_npu_accuracy_utils import (
from sglang.test.ascend.e2e.test_npu_performance_utils import GLM_4_7_FLASH_MODEL_PATH
from sglang.test.ci.ci_register import register_npu_ci
register_npu_ci(
est_time=3600,
suite="",
nightly=True,
disabled="accuracy testcase",
)
register_npu_ci(est_time=3600, suite="base-c-test-acc-2-npu-a3")
ENVS = {
"PYTORCH_NPU_ALLOC_CONF": "expandable_segments:True",
@@ -8,12 +8,7 @@ from sglang.test.ascend.e2e.test_npu_performance_utils import (
)
from sglang.test.ci.ci_register import register_npu_ci
register_npu_ci(
est_time=3600,
suite="",
nightly=True,
disabled="accuracy testcase",
)
register_npu_ci(est_time=3600, suite="base-c-test-acc-16-npu-a3")
ENVS = {
"SGLANG_SET_CPU_AFFINITY": "1",
@@ -54,7 +49,7 @@ class TestNPUGLM5_Top64_Pruned_GSM8K(TestNpuAccuracyTestCaseBase):
model = GLM5_TOP64_PRUNED_GSM8K_MODEL_PATH
envs = ENVS
other_args = OTHER_ARGS
accuracy = 0.50
accuracy = 0.48
datasets = ["gsm8k"]
generation_config = {
"max_tokens": 2048,
@@ -8,12 +8,7 @@ from sglang.test.ascend.e2e.test_npu_performance_utils import (
)
from sglang.test.ci.ci_register import register_npu_ci
register_npu_ci(
est_time=3600,
suite="",
nightly=True,
disabled="accuracy testcase",
)
register_npu_ci(est_time=3600, suite="base-c-test-acc-2-npu-a3")
MODEL_ENVS = {
"SGLANG_SET_CPU_AFFINITY": "1",
@@ -8,12 +8,7 @@ from sglang.test.ascend.e2e.test_npu_performance_utils import (
)
from sglang.test.ci.ci_register import register_npu_ci
register_npu_ci(
est_time=3600,
suite="stage-b-test-2-npu-a3",
nightly=True,
disabled="accuracy testcase",
)
register_npu_ci(est_time=3600, suite="base-c-test-acc-2-npu-a3")
QWEN3_5_9B_ENVS = {
"SGLANG_SET_CPU_AFFINITY": "1",
@@ -8,12 +8,7 @@ from sglang.test.ascend.e2e.test_npu_performance_utils import (
)
from sglang.test.ci.ci_register import register_npu_ci
register_npu_ci(
est_time=3600,
suite="stage-b-test-4-npu-a3",
nightly=True,
disabled="accuracy testcase",
)
register_npu_ci(est_time=3600, suite="base-c-test-acc-4-npu-a3")
QWEN3_VL_30B_A3B_ENVS = {
"SGLANG_SET_CPU_AFFINITY": "1",
@@ -8,12 +8,7 @@ from sglang.test.ascend.e2e.test_npu_performance_utils import (
)
from sglang.test.ci.ci_register import register_npu_ci
register_npu_ci(
est_time=3600,
suite="stage-b-test-4-npu-a3",
nightly=True,
disabled="accuracy testcase",
)
register_npu_ci(est_time=3600, suite="base-c-test-acc-4-npu-a3")
QWEN3_VL_8B_ENVS = {
"SGLANG_SET_CPU_AFFINITY": "1",
@@ -9,12 +9,7 @@ from sglang.test.ascend.e2e.test_npu_performance_utils import (
)
from sglang.test.ci.ci_register import register_npu_ci
register_npu_ci(
est_time=3600,
suite="",
nightly=True,
disabled="performance testcase",
)
register_npu_ci(est_time=3600, suite="base-c-test-perf-2-npu-a3")
_is_pr_pipeline = os.environ.get("GITHUB_EVENT_NAME") == "pull_request"
@@ -13,7 +13,7 @@ from sglang.test.test_utils import (
popen_launch_server,
)
register_npu_ci(est_time=400, suite="stage-b-test-1-npu-a3", nightly=False)
register_npu_ci(est_time=400, suite="base-b-test-1-npu-a3")
register_npu_ci(est_time=400, suite="nightly-1-npu-a3", nightly=True)
TEST_MODEL_MATRIX = {
@@ -13,7 +13,7 @@ from sglang.test.test_utils import (
popen_launch_server,
)
register_npu_ci(est_time=400, suite="stage-b-test-4-npu-a3", nightly=False)
register_npu_ci(est_time=400, suite="base-b-test-4-npu-a3")
register_npu_ci(est_time=400, suite="nightly-4-npu-a3", nightly=True)
TEST_MODEL_MATRIX = {
@@ -14,7 +14,7 @@ from sglang.test.test_utils import (
popen_launch_server,
)
register_npu_ci(est_time=400, suite="stage-b-test-1-npu-a3", nightly=False)
register_npu_ci(est_time=400, suite="base-b-test-1-npu-a3")
register_npu_ci(est_time=400, suite="nightly-1-npu-a3", nightly=True)
@@ -10,7 +10,7 @@ from sglang.test.test_utils import (
write_github_step_summary,
)
register_npu_ci(est_time=400, suite="stage-b-test-4-npu-a3", nightly=False)
register_npu_ci(est_time=400, suite="base-b-test-4-npu-a3")
register_npu_ci(est_time=400, suite="nightly-1-npu-a3", nightly=True)
@@ -23,7 +23,7 @@ from sglang.test.test_utils import (
popen_launch_server,
)
register_npu_ci(est_time=400, suite="stage-b-test-4-npu-a3", nightly=False)
register_npu_ci(est_time=400, suite="base-b-test-4-npu-a3")
register_npu_ci(est_time=400, suite="nightly-4-npu-a3", nightly=True)
@@ -14,7 +14,7 @@ from sglang.test.test_utils import (
popen_launch_server,
)
register_npu_ci(est_time=400, suite="stage-b-test-1-npu-a3", nightly=False)
register_npu_ci(est_time=400, suite="base-b-test-1-npu-a3")
register_npu_ci(est_time=400, suite="nightly-1-npu-a3", nightly=True)
TEST_MODEL_MATRIX = {
@@ -12,7 +12,7 @@ from sglang.test.test_utils import (
run_bench_one_batch,
)
register_npu_ci(est_time=400, suite="stage-b-test-1-npu-a3", nightly=False)
register_npu_ci(est_time=400, suite="base-b-test-1-npu-a3")
register_npu_ci(est_time=400, suite="nightly-1-npu-a3", nightly=True)
@@ -18,7 +18,7 @@ from sglang.test.test_utils import (
popen_launch_server,
)
register_npu_ci(est_time=700, suite="stage-b-test-16-npu-a3", nightly=False)
register_npu_ci(est_time=700, suite="base-b-test-16-npu-a3")
register_npu_ci(est_time=700, suite="nightly-16-npu-a3", nightly=True)
@@ -23,7 +23,7 @@ from sglang.test.test_utils import (
popen_launch_pd_server,
)
register_npu_ci(est_time=3600, suite="stage-b-test-16-npu-a3", nightly=False)
register_npu_ci(est_time=3600, suite="base-b-test-16-npu-a3")
register_npu_ci(est_time=3600, suite="nightly-16-npu-a3", nightly=True)
load_balance_method_options = [
@@ -13,7 +13,7 @@ from sglang.test.test_utils import (
popen_launch_server,
)
register_npu_ci(est_time=400, suite="stage-b-test-16-npu-a3", nightly=False)
register_npu_ci(est_time=400, suite="base-b-test-16-npu-a3")
register_npu_ci(est_time=400, suite="nightly-16-npu-a3", nightly=True)
TEST_MODEL_MATRIX = {
@@ -14,7 +14,7 @@ from sglang.test.test_utils import (
popen_launch_server,
)
register_npu_ci(est_time=400, suite="stage-b-test-8-npu-a3", nightly=False)
register_npu_ci(est_time=400, suite="base-b-test-8-npu-a3")
register_npu_ci(est_time=400, suite="nightly-8-npu-a3", nightly=True)
@@ -17,7 +17,7 @@ from sglang.test.test_utils import (
popen_launch_server,
)
register_npu_ci(est_time=400, suite="stage-b-test-2-npu-a3", nightly=False)
register_npu_ci(est_time=400, suite="base-b-test-2-npu-a3")
register_npu_ci(est_time=400, suite="nightly-2-npu-a3", nightly=True)
@@ -13,7 +13,7 @@ from sglang.test.test_utils import (
popen_launch_server,
)
register_npu_ci(est_time=400, suite="stage-b-test-2-npu-a3", nightly=False)
register_npu_ci(est_time=400, suite="base-b-test-2-npu-a3")
register_npu_ci(est_time=400, suite="nightly-2-npu-a3", nightly=True)
@@ -12,7 +12,7 @@ from sglang.test.test_utils import (
popen_launch_server,
)
register_npu_ci(est_time=400, suite="stage-b-test-2-npu-a3", nightly=False)
register_npu_ci(est_time=400, suite="base-b-test-2-npu-a3")
register_npu_ci(est_time=400, suite="nightly-2-npu-a3", nightly=True)
@@ -21,7 +21,7 @@ from sglang.test.test_utils import (
popen_with_error_check,
)
register_npu_ci(est_time=400, suite="stage-b-test-16-npu-a3", nightly=False)
register_npu_ci(est_time=400, suite="base-b-test-16-npu-a3")
register_npu_ci(est_time=400, suite="nightly-16-npu-a3", nightly=True)
@@ -15,7 +15,7 @@ from sglang.test.test_utils import (
popen_launch_server,
)
register_npu_ci(est_time=400, suite="stage-b-test-1-npu-a3", nightly=False)
register_npu_ci(est_time=400, suite="base-b-test-1-npu-a3")
register_npu_ci(est_time=400, suite="nightly-1-npu-a3", nightly=True)
logger = logging.getLogger(__name__)
@@ -16,7 +16,7 @@ from sglang.test.test_utils import (
popen_launch_server,
)
register_npu_ci(est_time=400, suite="stage-b-test-1-npu-a3", nightly=False)
register_npu_ci(est_time=400, suite="base-b-test-1-npu-a3")
register_npu_ci(est_time=400, suite="nightly-1-npu-a3", nightly=True)
logger = logging.getLogger(__name__)
@@ -17,7 +17,7 @@ from sglang.test.test_utils import (
popen_launch_server,
)
register_npu_ci(est_time=400, suite="stage-b-test-1-npu-a3", nightly=False)
register_npu_ci(est_time=400, suite="base-b-test-1-npu-a3")
register_npu_ci(est_time=400, suite="nightly-1-npu-a3", nightly=True)
logger = logging.getLogger(__name__)
@@ -13,7 +13,7 @@ from sglang.test.ascend.test_ascend_utils import (
from sglang.test.ci.ci_register import register_npu_ci
from sglang.test.test_utils import CustomTestCase
register_npu_ci(est_time=400, suite="stage-b-test-4-npu-a3", nightly=False)
register_npu_ci(est_time=400, suite="base-b-test-4-npu-a3")
register_npu_ci(est_time=400, suite="nightly-4-npu-a3", nightly=True)
@@ -14,7 +14,7 @@ from sglang.test.test_utils import (
popen_launch_server,
)
register_npu_ci(est_time=400, suite="stage-b-test-2-npu-a3", nightly=False)
register_npu_ci(est_time=400, suite="base-b-test-2-npu-a3")
register_npu_ci(est_time=400, suite="nightly-2-npu-a3", nightly=True)
TEST_MODEL_MATRIX = {
@@ -13,7 +13,7 @@ from sglang.test.test_utils import (
popen_launch_server,
)
register_npu_ci(est_time=400, suite="stage-b-test-4-npu-a3", nightly=False)
register_npu_ci(est_time=400, suite="base-b-test-4-npu-a3")
register_npu_ci(est_time=400, suite="nightly-4-npu-a3", nightly=True)
TEST_MODEL_MATRIX = {
@@ -13,7 +13,7 @@ from sglang.test.test_utils import (
popen_launch_server,
)
register_npu_ci(est_time=400, suite="stage-b-test-1-npu-a3", nightly=False)
register_npu_ci(est_time=400, suite="base-b-test-1-npu-a3")
register_npu_ci(est_time=400, suite="nightly-1-npu-a3", nightly=True)
TEST_MODEL_MATRIX = {
@@ -13,7 +13,7 @@ from sglang.test.test_utils import (
popen_launch_server,
)
register_npu_ci(est_time=400, suite="stage-b-test-2-npu-a3", nightly=False)
register_npu_ci(est_time=400, suite="base-b-test-2-npu-a3")
register_npu_ci(est_time=400, suite="nightly-2-npu-a3", nightly=True)
TEST_MODEL_MATRIX = {
@@ -14,7 +14,7 @@ from sglang.test.test_utils import (
popen_launch_server,
)
register_npu_ci(est_time=400, suite="stage-b-test-2-npu-a3", nightly=False)
register_npu_ci(est_time=400, suite="base-b-test-2-npu-a3")
register_npu_ci(est_time=400, suite="nightly-2-npu-a3", nightly=True)
TEST_MODEL_MATRIX = {
@@ -12,7 +12,7 @@ from sglang.test.test_utils import (
popen_launch_server,
)
register_npu_ci(est_time=400, suite="stage-b-test-4-npu-a3", nightly=False)
register_npu_ci(est_time=400, suite="base-b-test-4-npu-a3")
register_npu_ci(est_time=400, suite="nightly-4-npu-a3", nightly=True)
TEST_MODEL_MATRIX = {
@@ -17,7 +17,7 @@ from sglang.test.test_utils import (
popen_launch_server,
)
register_npu_ci(est_time=200, suite="stage-b-test-1-npu-a3", nightly=False)
register_npu_ci(est_time=200, suite="base-b-test-1-npu-a3")
register_npu_ci(est_time=200, suite="nightly-1-npu-a3", nightly=True)
@@ -9,7 +9,7 @@ from sglang.test.ascend.test_ascend_utils import (
from sglang.test.ci.ci_register import register_npu_ci
from sglang.test.test_utils import CustomTestCase
register_npu_ci(est_time=400, suite="stage-b-test-1-npu-a3", nightly=False)
register_npu_ci(est_time=400, suite="base-b-test-1-npu-a3")
register_npu_ci(est_time=400, suite="nightly-1-npu-a3", nightly=True)
@@ -17,11 +17,7 @@ from sglang.test.test_utils import (
popen_launch_server,
)
register_npu_ci(
est_time=400,
suite="stage-b-test-4-npu-a3",
nightly=False,
)
register_npu_ci(est_time=400, suite="base-b-test-4-npu-a3")
register_npu_ci(
est_time=400,
suite="nightly-4-npu-a3",
@@ -16,7 +16,7 @@ from sglang.test.test_utils import (
popen_launch_server,
)
register_npu_ci(est_time=400, suite="stage-b-test-16-npu-a3", nightly=False)
register_npu_ci(est_time=400, suite="base-b-test-16-npu-a3")
register_npu_ci(est_time=400, suite="nightly-16-npu-a3", nightly=True)
MODEL_PATH = DEEPSEEK_R1_0528_W4A8_PER_CHANNEL_WEIGHTS_PATH
@@ -16,7 +16,7 @@ from sglang.test.test_utils import (
popen_launch_server,
)
register_npu_ci(est_time=400, suite="stage-b-test-16-npu-a3", nightly=False)
register_npu_ci(est_time=400, suite="base-b-test-16-npu-a3")
register_npu_ci(est_time=400, suite="nightly-16-npu-a3", nightly=True)
@@ -16,7 +16,7 @@ from sglang.test.test_utils import (
popen_launch_server,
)
register_npu_ci(est_time=400, suite="stage-b-test-4-npu-a3", nightly=False)
register_npu_ci(est_time=400, suite="base-b-test-4-npu-a3")
register_npu_ci(est_time=400, suite="nightly-4-npu-a3", nightly=True)
_ASCEND_BACKEND = "ascend"
@@ -20,7 +20,7 @@ from sglang.test.test_utils import (
popen_launch_server,
)
register_npu_ci(est_time=300, suite="stage-b-test-4-npu-a3", nightly=False)
register_npu_ci(est_time=300, suite="base-b-test-4-npu-a3")
register_npu_ci(est_time=300, suite="nightly-4-npu-a3", nightly=True)
@@ -8,12 +8,7 @@ from sglang.test.ascend.e2e.test_npu_performance_utils import (
)
from sglang.test.ci.ci_register import register_npu_ci
register_npu_ci(
est_time=3600,
suite="",
nightly=True,
disabled="performance testcase",
)
register_npu_ci(est_time=3600, suite="base-c-test-perf-16-npu-a3")
# Environment variables for DSV4-Flash single-node PD-mix deployment.
DEEPSEEK_V4_FLASH_W8A8_8P_ENVS = {
@@ -10,12 +10,7 @@ from sglang.test.ascend.e2e.test_npu_performance_utils import (
)
from sglang.test.ci.ci_register import register_npu_ci
register_npu_ci(
est_time=1800,
suite="full-16-npu-a3",
nightly=True,
disabled="Currently it is executed by the npu performance workflow.",
)
register_npu_ci(est_time=1800, suite="base-c-test-perf-16-npu-a3")
KIMI_K2_6_ENVS = {
"PYTORCH_NPU_ALLOC_CONF": "expandable_segments:True",
@@ -9,12 +9,7 @@ from sglang.test.ascend.e2e.test_npu_performance_utils import (
)
from sglang.test.ci.ci_register import register_npu_ci
register_npu_ci(
est_time=3600,
suite="npu-performance",
nightly=True,
disabled="performance testcase",
)
register_npu_ci(est_time=3600, suite="base-c-test-perf-8-npu-a3")
MINIMAX_M2_5_W8A8_4P_IN64K_OUT1K_PREFIX90_ENVS = {
"PYTORCH_NPU_ALLOC_CONF": "expandable_segments:True",
@@ -9,12 +9,7 @@ from sglang.test.ascend.e2e.test_npu_performance_utils import (
)
from sglang.test.ci.ci_register import register_npu_ci
register_npu_ci(
est_time=3600,
suite="",
nightly=True,
disabled="performance testcase",
)
register_npu_ci(est_time=3600, suite="base-c-test-perf-2-npu-a3")
QWEN3_8B_ENVS = {
"SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT": "600",
@@ -9,12 +9,7 @@ from sglang.test.ascend.e2e.test_npu_performance_utils import (
)
from sglang.test.ci.ci_register import register_npu_ci
register_npu_ci(
est_time=3600,
suite="",
nightly=True,
disabled="performance testcase",
)
register_npu_ci(est_time=3600, suite="base-c-test-perf-16-npu-a3")
QWEN3_235B_ENVS = {
"PYTORCH_NPU_ALLOC_CONF": "expandable_segments:True",
@@ -9,12 +9,7 @@ from sglang.test.ascend.e2e.test_npu_performance_utils import (
)
from sglang.test.ci.ci_register import register_npu_ci
register_npu_ci(
est_time=3600,
suite="",
nightly=True,
disabled="performance testcase",
)
register_npu_ci(est_time=3600, suite="base-c-test-perf-2-npu-a3")
QWEN3_30B_A3B_ENVS = {
"ASCEND_LAUNCH_BLOCKING": "0",
@@ -9,12 +9,7 @@ from sglang.test.ascend.e2e.test_npu_performance_utils import (
)
from sglang.test.ci.ci_register import register_npu_ci
register_npu_ci(
est_time=3600,
suite="",
nightly=True,
disabled="performance testcase",
)
register_npu_ci(est_time=3600, suite="base-c-test-perf-4-npu-a3")
QWEN3_32B_ENVS = {
"SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT": "600",
@@ -8,12 +8,7 @@ from sglang.test.ascend.e2e.test_npu_performance_utils import (
)
from sglang.test.ci.ci_register import register_npu_ci
register_npu_ci(
est_time=3600,
suite="nightly-16-npu-a3",
nightly=True,
disabled="performance testcase",
)
register_npu_ci(est_time=3600, suite="base-c-test-perf-16-npu-a3")
QWEN3_5_397B_A17B_ENVS = {
"PYTORCH_NPU_ALLOC_CONF": "expandable_segments:True",
@@ -8,12 +8,7 @@ from sglang.test.ascend.e2e.test_npu_performance_utils import (
)
from sglang.test.ci.ci_register import register_npu_ci
register_npu_ci(
est_time=3600,
suite="full-2-npu-a3",
nightly=True,
disabled="performance case",
)
register_npu_ci(est_time=3600, suite="base-c-test-perf-2-npu-a3")
QWEN3_6_27B_1024_ENVS = {
"PYTORCH_NPU_ALLOC_CONF": "expandable_segments:True",
@@ -8,12 +8,7 @@ from sglang.test.ascend.e2e.test_npu_performance_utils import (
)
from sglang.test.ci.ci_register import register_npu_ci
register_npu_ci(
est_time=3600,
suite="",
nightly=True,
disabled="performance testcase",
)
register_npu_ci(est_time=3600, suite="base-c-test-perf-2-npu-a3")
QWEN3_6_27B_64K_1K_ENVS = {
"STREAMS_PER_DEVICE": "32",
@@ -8,12 +8,7 @@ from sglang.test.ascend.e2e.test_npu_performance_utils import (
)
from sglang.test.ci.ci_register import register_npu_ci
register_npu_ci(
est_time=3600,
suite="",
nightly=True,
disabled="performance testcase",
)
register_npu_ci(est_time=3600, suite="base-c-test-perf-2-npu-a3")
QWEN3_6_35B_A3B_64K_PREFIX_ENVS = {
"PYTORCH_NPU_ALLOC_CONF": "expandable_segments:True",
@@ -7,12 +7,7 @@ from sglang.test.ascend.e2e.test_npu_performance_utils import (
)
from sglang.test.ci.ci_register import register_npu_ci
register_npu_ci(
est_time=3600,
suite="",
nightly=True,
disabled="performance testcase",
)
register_npu_ci(est_time=3600, suite="base-c-test-perf-4-npu-a3")
QWEN3_NEXT_80B_A3B_ENVS = {
"PYTORCH_NPU_ALLOC_CONF": "expandable_segments:True",
@@ -11,7 +11,7 @@ from sglang.srt.layers.attention import vision
from sglang.test.ci.ci_register import register_cpu_ci, register_npu_ci
register_cpu_ci(est_time=2, suite="base-a-test-cpu")
register_npu_ci(est_time=2, suite="stage-b-test-1-npu-a3")
register_npu_ci(est_time=2, suite="base-b-test-1-npu-a3")
register_npu_ci(est_time=2, suite="nightly-1-npu-a3", nightly=True)
@@ -12,7 +12,7 @@ import torch
from sglang.test.ci.ci_register import register_npu_ci
register_npu_ci(est_time=5, suite="stage-a-unit-test-npu")
register_npu_ci(est_time=5, suite="base-a-test-1-npu-a2")
# Mock NPU-only modules before importing the source module.
for _ in (
@@ -12,7 +12,7 @@ import torch
from sglang.test.ci.ci_register import register_npu_ci
register_npu_ci(est_time=4, suite="stage-a-unit-test-npu")
register_npu_ci(est_time=4, suite="base-a-test-1-npu-a2")
for mod in (
"torch_npu",
+14 -7
View File
@@ -93,13 +93,20 @@ PER_COMMIT_SUITES = {
"extra-b-test-8-gpu-h200",
],
HWBackend.NPU: [
"base-a-test-1-gpu-small",
"stage-a-unit-test-npu",
"stage-b-test-1-npu-a3",
"stage-b-test-2-npu-a3",
"stage-b-test-4-npu-a3",
"stage-b-test-8-npu-a3",
"stage-b-test-16-npu-a3",
"base-a-test-1-npu-a2",
"base-b-test-1-npu-a3",
"base-b-test-2-npu-a3",
"base-b-test-4-npu-a3",
"base-b-test-8-npu-a3",
"base-b-test-16-npu-a3",
"base-c-test-acc-2-npu-a3",
"base-c-test-acc-4-npu-a3",
"base-c-test-acc-8-npu-a3",
"base-c-test-acc-16-npu-a3",
"base-c-test-perf-2-npu-a3",
"base-c-test-perf-4-npu-a3",
"base-c-test-perf-8-npu-a3",
"base-c-test-perf-16-npu-a3",
],
HWBackend.XPU: [
"stage-a-test-1-gpu-xpu",