[NPU] Improve the execution efficiency and maintainability of pr‑test‑npu (#33724)
Co-authored-by: Even Zhou <even.y.zhou@outlook.com> Co-authored-by: sglang-npu-bot <sglangnpu@163.com>
This commit is contained in:
co-authored by
Even Zhou
sglang-npu-bot
parent
e732c0a9dc
commit
dd5d82bead
@@ -0,0 +1,213 @@
|
||||
name: PR Test Stage for NPU
|
||||
# Reusable workflow for one CUDA test stage. Caller pr-test-npu.yml forwards
|
||||
|
||||
on:
|
||||
workflow_call:
|
||||
inputs:
|
||||
self_name:
|
||||
description: 'Caller job key; used for partitions[suite] lookup.'
|
||||
type: string
|
||||
required: true
|
||||
runner_config:
|
||||
description: 'Key in scripts/ci/runner_configs.yml. Resolves install script, artifact version, install timeout, runs-on label, and rdma_devices.'
|
||||
type: string
|
||||
required: true
|
||||
image:
|
||||
description: 'The container image for NPU test environment.'
|
||||
type: string
|
||||
required: true
|
||||
npu_device_type:
|
||||
description: 'NPU device type, e.g. 910b, a3.'
|
||||
type: string
|
||||
default: 'a3'
|
||||
run_timeout_minutes:
|
||||
description: 'timeout-minutes for the Run test step. Required so compute_partitions.py can read it from pr-test.yml without a duplicated default constant.'
|
||||
type: string
|
||||
required: true
|
||||
timeout_per_file:
|
||||
description: 'run_suite.py --timeout-per-file value (empty = unset).'
|
||||
type: string
|
||||
default: ''
|
||||
partitions:
|
||||
description: 'partitions config, e.g. {"size":1,"arr":[0]}'
|
||||
type: string
|
||||
default: '{"size":1,"arr":[0]}'
|
||||
ref:
|
||||
description: 'Git ref (branch, tag, or SHA) to test. If not provided, uses the default branch.'
|
||||
type: string
|
||||
default: ''
|
||||
skip_pr_test_health_check:
|
||||
description: 'Git ref (branch, tag, or SHA) to test. If not provided, uses the default branch.'
|
||||
type: string
|
||||
default: 'false'
|
||||
github-token:
|
||||
description: 'GitHub token for API calls'
|
||||
type: string
|
||||
default: ${{ github.token }}
|
||||
|
||||
env:
|
||||
SKIP_PR_TEST_HEALTH_CHECK: ${{ inputs.skip_pr_test_health_check }}
|
||||
SGLANG_USE_MODELSCOPE: true
|
||||
SGLANG_IS_IN_CI: true
|
||||
HF_ENDPOINT: https://hf-mirror.com
|
||||
TORCH_EXTENSIONS_DIR: /tmp/torch_extensions
|
||||
PYTORCH_NPU_ALLOC_CONF: "expandable_segments:True"
|
||||
STREAMS_PER_DEVICE: 32
|
||||
|
||||
jobs:
|
||||
run:
|
||||
runs-on: ${{ inputs.runner_config }}
|
||||
container:
|
||||
image: ${{ inputs.image }}
|
||||
timeout-minutes: 240
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
partition: ${{ fromJson(inputs.partitions).arr }}
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
ref: ${{ inputs.ref || github.ref }}
|
||||
|
||||
- name: Mark repository safe
|
||||
run: |
|
||||
git config --system --add safe.directory ${GITHUB_WORKSPACE}
|
||||
|
||||
- name: Check PR test health
|
||||
uses: actions/github-script@v8
|
||||
env:
|
||||
SKIP_PR_TEST_HEALTH_CHECK: ${{ env.SKIP_PR_TEST_HEALTH_CHECK }}
|
||||
with:
|
||||
github-token: ${{ inputs.github-token || github.token }}
|
||||
script: |
|
||||
core.notice(`[health-check] START — event=${context.eventName}, runId=${context.runId}`);
|
||||
|
||||
// Skip when explicitly requested via env var (e.g. release branch cut)
|
||||
if (process.env.SKIP_PR_TEST_HEALTH_CHECK === 'true') {
|
||||
core.notice('[health-check] SKIP: SKIP_PR_TEST_HEALTH_CHECK=true');
|
||||
return;
|
||||
}
|
||||
|
||||
// Skip for scheduled runs — they should collect all failures, not fast-fail
|
||||
if (context.eventName === 'schedule') {
|
||||
core.notice('[health-check] SKIP: scheduled run');
|
||||
return;
|
||||
}
|
||||
|
||||
// Check lint status from the separate Lint workflow (lint.yml).
|
||||
// listJobsForWorkflowRun only sees jobs within the SAME run, so we use
|
||||
// checks.listForRef which queries by commit SHA across ALL workflows.
|
||||
const ref = context.payload.pull_request?.head?.sha || context.sha;
|
||||
core.info(`[health-check] Checking lint for ref=${ref}`);
|
||||
const { data } = await github.rest.checks.listForRef({
|
||||
owner: context.repo.owner,
|
||||
repo: context.repo.repo,
|
||||
ref: ref,
|
||||
check_name: 'lint',
|
||||
});
|
||||
const lintRun = data.check_runs.find(
|
||||
cr => cr.app?.slug === 'github-actions'
|
||||
);
|
||||
core.info(`[health-check] Lint check: status=${lintRun?.status}, conclusion=${lintRun?.conclusion}`);
|
||||
if (lintRun?.status === 'completed' && lintRun?.conclusion === 'failure') {
|
||||
core.setFailed('Fast-fail: lint check failed');
|
||||
return;
|
||||
}
|
||||
|
||||
// Skip the jobs-failed check when the PR carries the bypass-fastfail label.
|
||||
// Lint check above still runs.
|
||||
let labels = [];
|
||||
if (context.payload.pull_request?.labels) {
|
||||
labels = context.payload.pull_request.labels.map(l => l.name);
|
||||
} else {
|
||||
const { data: prs } = await github.rest.repos.listPullRequestsAssociatedWithCommit({
|
||||
owner: context.repo.owner,
|
||||
repo: context.repo.repo,
|
||||
commit_sha: ref,
|
||||
});
|
||||
if (prs.length > 0) {
|
||||
labels = prs[0].labels.map(l => l.name);
|
||||
}
|
||||
}
|
||||
core.info(`[health-check] PR labels: [${labels.join(', ')}]`);
|
||||
if (labels.includes('bypass-fastfail')) {
|
||||
core.notice('[health-check] SKIP jobs-failed check: bypass-fastfail label present');
|
||||
return;
|
||||
}
|
||||
|
||||
const jobs = await github.paginate(github.rest.actions.listJobsForWorkflowRun, {
|
||||
owner: context.repo.owner,
|
||||
repo: context.repo.repo,
|
||||
run_id: context.runId,
|
||||
per_page: 100,
|
||||
});
|
||||
core.info(`[health-check] Total jobs in run: ${jobs.length}`);
|
||||
const failedJobs = jobs.filter(j => j.status === 'completed' && j.conclusion === 'failure');
|
||||
core.info(`[health-check] Failed jobs (before filtering): ${failedJobs.map(j => `${j.name}(${j.conclusion})`).join(', ') || 'none'}`);
|
||||
|
||||
// Find jobs that failed from a real error, not from fast-fail cascade
|
||||
const rootCauseFailures = jobs.filter(j => {
|
||||
if (j.status !== 'completed' || j.conclusion !== 'failure') return false;
|
||||
// h20 runners are flaky (dirty GPU state from prior runs); their failures
|
||||
// should not cascade fast-fail to other stages. j.name shape from
|
||||
// listJobsForWorkflowRun: "<job-key>" + optional " / <reusable-job>"
|
||||
// + optional " (<matrix>)". Split off the base job key before exact
|
||||
// match so we cover both inline + reusable forms without confusing
|
||||
// 'h20' with the 'h200' prefix.
|
||||
const baseName = j.name.split(/[ /]/)[0];
|
||||
if (baseName === 'base-c-test-8-gpu-h20') {
|
||||
core.info(`[health-check] Filtered out h20 job: ${j.name}`);
|
||||
return false;
|
||||
}
|
||||
// If the failing step is the health check, it's a cascade — skip it
|
||||
const failedStep = (j.steps || []).find(s => s.conclusion === 'failure');
|
||||
if (failedStep && (failedStep.name.includes('check-pr-test-health') || failedStep.name.includes('Check PR test health'))) {
|
||||
core.info(`[health-check] Filtered out cascade failure: ${j.name} (failed step: ${failedStep.name})`);
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
});
|
||||
core.info(`[health-check] Root cause failures (after filtering): ${rootCauseFailures.map(j => j.name).join(', ') || 'none'}`);
|
||||
|
||||
if (rootCauseFailures.length > 0) {
|
||||
core.setFailed(`Fast-fail: skipping — root cause job(s): ${rootCauseFailures.map(j => j.name).join(', ')}`);
|
||||
} else {
|
||||
core.notice('[health-check] PASS: no root cause failures detected');
|
||||
}
|
||||
|
||||
- name: Install dependencies
|
||||
env:
|
||||
TORCH_CACHE_URL: "http://cache-service.nginx-pypi-cache.svc.cluster.local/whl/cpu"
|
||||
PYPI_CACHE_URL: "http://cache-service.nginx-pypi-cache.svc.cluster.local/pypi/simple"
|
||||
UV_INDEX_URL: "http://cache-service.nginx-pypi-cache.svc.cluster.local/pypi/simple"
|
||||
GITHUB_PROXY_URL: "https://gh-proxy.test.osinfra.cn/"
|
||||
RUSTUP_CACHE_URL: "http://cache-service.nginx-pypi-cache.svc.cluster.local:8082"
|
||||
run: |
|
||||
# speed up by using infra cache services
|
||||
CACHING_URL="cache-service.nginx-pypi-cache.svc.cluster.local"
|
||||
sed -Ei "s@(ports|archive).ubuntu.com@${CACHING_URL}:8081@g" /etc/apt/sources.list
|
||||
pip config set global.index-url http://${CACHING_URL}/pypi/simple
|
||||
pip config set global.trusted-host "${CACHING_URL}"
|
||||
|
||||
bash scripts/ci/npu/npu_ci_install_dependency.sh ${{ inputs.npu_device_type }}
|
||||
# copy required file from our daily cache
|
||||
cp ~/.cache/modelscope/hub/datasets/otavia/ShareGPT_Vicuna_unfiltered/ShareGPT_V3_unfiltered_cleaned_split.json /tmp
|
||||
# copy gsm8k dataset
|
||||
cp ~/.cache/modelscope/hub/datasets/tmp/test.jsonl /tmp
|
||||
|
||||
# install sglang_router
|
||||
apt-get install -y libssl-dev
|
||||
pip install sglang_router
|
||||
|
||||
- name: Run test
|
||||
timeout-minutes: ${{ fromJson(inputs.run_timeout_minutes) }}
|
||||
env:
|
||||
CONTINUE_ON_ERROR_FLAG: ${{ inputs.continue_on_error == 'true' && '--continue-on-error' || '' }}
|
||||
run: |
|
||||
cd test
|
||||
python3 run_suite.py --hw npu --suite ${{ inputs.self_name }} \
|
||||
--auto-partition-id ${{ matrix.partition }} \
|
||||
--auto-partition-size ${{ fromJson(inputs.partitions).size }} \
|
||||
${{ inputs.timeout_per_file && format('--timeout-per-file {0}', inputs.timeout_per_file) || '' }} \
|
||||
$CONTINUE_ON_ERROR_FLAG
|
||||
@@ -0,0 +1,295 @@
|
||||
name: 'Single Node Template for E2E performance and accuracy tests'
|
||||
|
||||
on:
|
||||
workflow_call:
|
||||
inputs:
|
||||
runner:
|
||||
required: true
|
||||
type: string
|
||||
default: linux-aarch64-a3-16
|
||||
test_type:
|
||||
required: true
|
||||
type: string
|
||||
default: perf
|
||||
description: perf or accuracy
|
||||
test_suite:
|
||||
required: true
|
||||
type: string
|
||||
default: ''
|
||||
description: name of test suite to run via run_suite.py (mutually exclusive with test_case)
|
||||
image:
|
||||
required: true
|
||||
type: string
|
||||
description: image for pods
|
||||
default: "swr.cn-southwest-2.myhuaweicloud.com/base_image/dockerhub/lmsysorg/sglang:main-cann9.0.0-a3"
|
||||
install_sglang_deps:
|
||||
required: false
|
||||
type: boolean
|
||||
default: false
|
||||
description: install sglang dependencies (e.g. PyTorch, CANN packages) when using source installation
|
||||
device_type_for_deps:
|
||||
required: false
|
||||
type: string
|
||||
default: 'a3'
|
||||
description: device type for dependency installation (a3 or 910b)
|
||||
transformers_version:
|
||||
required: false
|
||||
type: string
|
||||
default: ""
|
||||
description: "The transformers version number for running sglang. Use default version in image if keep empty."
|
||||
skip_pr_test_health_check:
|
||||
description: 'Git ref (branch, tag, or SHA) to test. If not provided, uses the default branch.'
|
||||
type: string
|
||||
default: 'false'
|
||||
github-token:
|
||||
description: 'GitHub token for API calls'
|
||||
type: string
|
||||
default: ${{ github.token }}
|
||||
|
||||
env:
|
||||
SKIP_PR_TEST_HEALTH_CHECK: ${{ inputs.skip_pr_test_health_check }}
|
||||
|
||||
concurrency:
|
||||
group: ascend-nightly-e2e-singlenode-${{ github.workflow_ref }}-${{ github.ref }}-${{ inputs.test_suite }}
|
||||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
e2e:
|
||||
name: ${{ inputs.test_suite }}
|
||||
runs-on: ${{ inputs.runner }}
|
||||
container:
|
||||
image: ${{ inputs.image }}
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Check PR test health
|
||||
uses: actions/github-script@v8
|
||||
env:
|
||||
SKIP_PR_TEST_HEALTH_CHECK: ${{ env.SKIP_PR_TEST_HEALTH_CHECK }}
|
||||
with:
|
||||
github-token: ${{ inputs.github-token || github.token }}
|
||||
script: |
|
||||
core.notice(`[health-check] START — event=${context.eventName}, runId=${context.runId}`);
|
||||
|
||||
// Skip when explicitly requested via env var (e.g. release branch cut)
|
||||
if (process.env.SKIP_PR_TEST_HEALTH_CHECK === 'true') {
|
||||
core.notice('[health-check] SKIP: SKIP_PR_TEST_HEALTH_CHECK=true');
|
||||
return;
|
||||
}
|
||||
|
||||
// Skip for scheduled runs — they should collect all failures, not fast-fail
|
||||
if (context.eventName === 'schedule') {
|
||||
core.notice('[health-check] SKIP: scheduled run');
|
||||
return;
|
||||
}
|
||||
|
||||
// Check lint status from the separate Lint workflow (lint.yml).
|
||||
// listJobsForWorkflowRun only sees jobs within the SAME run, so we use
|
||||
// checks.listForRef which queries by commit SHA across ALL workflows.
|
||||
const ref = context.payload.pull_request?.head?.sha || context.sha;
|
||||
core.info(`[health-check] Checking lint for ref=${ref}`);
|
||||
const { data } = await github.rest.checks.listForRef({
|
||||
owner: context.repo.owner,
|
||||
repo: context.repo.repo,
|
||||
ref: ref,
|
||||
check_name: 'lint',
|
||||
});
|
||||
const lintRun = data.check_runs.find(
|
||||
cr => cr.app?.slug === 'github-actions'
|
||||
);
|
||||
core.info(`[health-check] Lint check: status=${lintRun?.status}, conclusion=${lintRun?.conclusion}`);
|
||||
if (lintRun?.status === 'completed' && lintRun?.conclusion === 'failure') {
|
||||
core.setFailed('Fast-fail: lint check failed');
|
||||
return;
|
||||
}
|
||||
|
||||
// Skip the jobs-failed check when the PR carries the bypass-fastfail label.
|
||||
// Lint check above still runs.
|
||||
let labels = [];
|
||||
if (context.payload.pull_request?.labels) {
|
||||
labels = context.payload.pull_request.labels.map(l => l.name);
|
||||
} else {
|
||||
const { data: prs } = await github.rest.repos.listPullRequestsAssociatedWithCommit({
|
||||
owner: context.repo.owner,
|
||||
repo: context.repo.repo,
|
||||
commit_sha: ref,
|
||||
});
|
||||
if (prs.length > 0) {
|
||||
labels = prs[0].labels.map(l => l.name);
|
||||
}
|
||||
}
|
||||
core.info(`[health-check] PR labels: [${labels.join(', ')}]`);
|
||||
if (labels.includes('bypass-fastfail')) {
|
||||
core.notice('[health-check] SKIP jobs-failed check: bypass-fastfail label present');
|
||||
return;
|
||||
}
|
||||
|
||||
const jobs = await github.paginate(github.rest.actions.listJobsForWorkflowRun, {
|
||||
owner: context.repo.owner,
|
||||
repo: context.repo.repo,
|
||||
run_id: context.runId,
|
||||
per_page: 100,
|
||||
});
|
||||
core.info(`[health-check] Total jobs in run: ${jobs.length}`);
|
||||
const failedJobs = jobs.filter(j => j.status === 'completed' && j.conclusion === 'failure');
|
||||
core.info(`[health-check] Failed jobs (before filtering): ${failedJobs.map(j => `${j.name}(${j.conclusion})`).join(', ') || 'none'}`);
|
||||
|
||||
// Find jobs that failed from a real error, not from fast-fail cascade
|
||||
const rootCauseFailures = jobs.filter(j => {
|
||||
if (j.status !== 'completed' || j.conclusion !== 'failure') return false;
|
||||
// h20 runners are flaky (dirty GPU state from prior runs); their failures
|
||||
// should not cascade fast-fail to other stages. j.name shape from
|
||||
// listJobsForWorkflowRun: "<job-key>" + optional " / <reusable-job>"
|
||||
// + optional " (<matrix>)". Split off the base job key before exact
|
||||
// match so we cover both inline + reusable forms without confusing
|
||||
// 'h20' with the 'h200' prefix.
|
||||
const baseName = j.name.split(/[ /]/)[0];
|
||||
if (baseName === 'base-c-test-8-gpu-h20') {
|
||||
core.info(`[health-check] Filtered out h20 job: ${j.name}`);
|
||||
return false;
|
||||
}
|
||||
// If the failing step is the health check, it's a cascade — skip it
|
||||
const failedStep = (j.steps || []).find(s => s.conclusion === 'failure');
|
||||
if (failedStep && (failedStep.name.includes('check-pr-test-health') || failedStep.name.includes('Check PR test health'))) {
|
||||
core.info(`[health-check] Filtered out cascade failure: ${j.name} (failed step: ${failedStep.name})`);
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
});
|
||||
core.info(`[health-check] Root cause failures (after filtering): ${rootCauseFailures.map(j => j.name).join(', ') || 'none'}`);
|
||||
|
||||
if (rootCauseFailures.length > 0) {
|
||||
core.setFailed(`Fast-fail: skipping — root cause job(s): ${rootCauseFailures.map(j => j.name).join(', ')}`);
|
||||
} else {
|
||||
core.notice('[health-check] PASS: no root cause failures detected');
|
||||
}
|
||||
|
||||
- name: Check npu info
|
||||
run: |
|
||||
npu-smi info
|
||||
|
||||
- name: Install sglang dependencies
|
||||
if: ${{ inputs.install_sglang_deps == true }}
|
||||
shell: bash
|
||||
env:
|
||||
TORCH_CACHE_URL: "http://cache-service.nginx-pypi-cache.svc.cluster.local/whl/cpu"
|
||||
PYPI_CACHE_URL: "http://cache-service.nginx-pypi-cache.svc.cluster.local/pypi/simple"
|
||||
UV_INDEX_URL: "http://cache-service.nginx-pypi-cache.svc.cluster.local/pypi/simple"
|
||||
GITHUB_PROXY_URL: "https://gh-proxy.test.osinfra.cn/"
|
||||
RUSTUP_CACHE_URL: "http://cache-service.nginx-pypi-cache.svc.cluster.local:8082"
|
||||
run: |
|
||||
CACHING_URL="cache-service.nginx-pypi-cache.svc.cluster.local"
|
||||
sed -Ei "s@(ports|archive).ubuntu.com@${CACHING_URL}:8081@g" /etc/apt/sources.list
|
||||
pip config set global.index-url http://${CACHING_URL}/pypi/simple
|
||||
pip config set global.trusted-host "${CACHING_URL}"
|
||||
bash scripts/ci/npu/npu_ci_install_dependency.sh ${{ inputs.device_type_for_deps }}
|
||||
cp ~/.cache/modelscope/hub/datasets/otavia/ShareGPT_Vicuna_unfiltered/ShareGPT_V3_unfiltered_cleaned_split.json /tmp
|
||||
curl -o /tmp/test.jsonl -L https://gh-proxy.test.osinfra.cn/https://raw.githubusercontent.com/openai/grade-school-math/master/grade_school_math/data/test.jsonl
|
||||
|
||||
- name: Run test
|
||||
timeout-minutes: 300
|
||||
env:
|
||||
SGLANG_USE_MODELSCOPE: true
|
||||
HF_ENDPOINT: https://hf-mirror.com
|
||||
SGLANG_IS_IN_CI: true
|
||||
TRANSFORMERS_VERBOSITY: "error"
|
||||
GDN_ATTN_BACKEND_TRITON: 1
|
||||
SGLANG_TEST_METRICS_OUTPUT: /root/.cache/tests/output/metrics/metrics
|
||||
shell: bash
|
||||
run: |
|
||||
sglang_source_path=$(pwd)
|
||||
echo "Source code path: ${sglang_source_path}"
|
||||
ln -sf ${sglang_source_path} /root/sglang
|
||||
|
||||
# Determine test mode: suite (run_suite.py --suite) or single case file.
|
||||
test_suite="${{ inputs.test_suite }}"
|
||||
echo "Test mode: suite (${test_suite})"
|
||||
tc_name="${test_suite}"
|
||||
|
||||
export TRANSFORMERS_VERSION_FOR_SGLANG="${{ inputs.transformers_version }}"
|
||||
PYTHON_FOR_SGLANG="python"
|
||||
PIP_FOR_SGLANG="pip"
|
||||
if [ -n "${TRANSFORMERS_VERSION_FOR_SGLANG}" ];then
|
||||
echo "===== Install transformers for sglang - Begin ====="
|
||||
TRANSFORMERS_PKG_PATH_SOURCE=/root/.cache/.cache/transformers/${TRANSFORMERS_VERSION_FOR_SGLANG}
|
||||
if [ ! -d "${TRANSFORMERS_PKG_PATH_SOURCE}" ]; then
|
||||
echo "The dependent transformers package does not exist: ${TRANSFORMERS_PKG_PATH_SOURCE}."
|
||||
echo "Install transformers ${TRANSFORMERS_VERSION_FOR_SGLANG} online."
|
||||
pip install transformers=="${TRANSFORMERS_VERSION_FOR_SGLANG}" -i https://mirrors.tuna.tsinghua.edu.cn/pypi/web/simple
|
||||
else
|
||||
echo "Install transformers ${TRANSFORMERS_VERSION_FOR_SGLANG} locally."
|
||||
TRANSFORMERS_PKG_PATH_TARGET=/tmp/transformers/${TRANSFORMERS_VERSION_FOR_SGLANG}
|
||||
mkdir -p "${TRANSFORMERS_PKG_PATH_TARGET}"
|
||||
cp "${TRANSFORMERS_PKG_PATH_SOURCE}/*" "${TRANSFORMERS_PKG_PATH_TARGET}/"
|
||||
pip install --no-index --find-links="${TRANSFORMERS_PKG_PATH_TARGET}" transformers=="${TRANSFORMERS_VERSION_FOR_SGLANG}"
|
||||
fi
|
||||
echo "===== Install transformers for sglang in virtual env - End ====="
|
||||
fi
|
||||
echo "Transformers version for sglang: $(${PIP_FOR_SGLANG} show transformers | grep Version | cut -d: -f2)"
|
||||
|
||||
echo "scaling_governor performance num: \
|
||||
$(cat /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor | grep performance | wc -l)"
|
||||
echo "swappiness: $(cat /proc/sys/vm/swappiness)"
|
||||
echo "numa_balancing: $(cat /proc/sys/kernel/numa_balancing)"
|
||||
echo "sched_migration_cost_ns: $(cat /proc/sys/kernel/sched_migration_cost_ns)"
|
||||
|
||||
export SGLANG_TEST_MAX_RETRY=0
|
||||
export SGLANG_SET_CPU_AFFINITY=1
|
||||
echo "SGLANG_SET_CPU_AFFINITY: $SGLANG_SET_CPU_AFFINITY"
|
||||
|
||||
|
||||
# Copy the checked‑out test common utility code to the sglang installation directory.
|
||||
sglang_pkg_path=/sgl-workspace/sglang/python
|
||||
ascend_test_util_path=${sglang_pkg_path}/sglang/test/ascend
|
||||
mkdir -p ${ascend_test_util_path}
|
||||
mv ${ascend_test_util_path} ${ascend_test_util_path}_bak
|
||||
cp -r ${sglang_source_path}/python/sglang/test/ascend ${ascend_test_util_path}
|
||||
|
||||
source /usr/local/Ascend/cann/set_env.sh || true
|
||||
source /usr/local/Ascend/nnal/atb/set_env.sh || true
|
||||
source /usr/local/Ascend/ascend-toolkit/latest/opp/vendors/customize/bin/set_env.bash || true
|
||||
source /usr/local/Ascend/ascend-toolkit/latest/opp/vendors/custom_transformer/bin/set_env.bash || true
|
||||
|
||||
# Set environment of cann
|
||||
log_path="/root/.cache/tests/logs/log/${current_date}/${tc_name}/${HOSTNAME}"
|
||||
rm -rf ${log_path}
|
||||
mkdir -p ${log_path}
|
||||
echo "Log path: ${log_path}"
|
||||
|
||||
echo "Running test: ${tc_name}"
|
||||
test_exit_code=0
|
||||
cd test
|
||||
${PYTHON_FOR_SGLANG} -u run_suite.py --hw npu --suite ${test_suite} --timeout-per-file 3600 2>&1 | tee /tmp/test_output.log || test_exit_code=$?
|
||||
|
||||
echo "Finished test: ${tc_name}"
|
||||
|
||||
if [ "${test_exit_code}" = "0" ]; then
|
||||
test_status="pass"
|
||||
status_icon="✅"
|
||||
else
|
||||
test_status="fail"
|
||||
status_icon="❌"
|
||||
fi
|
||||
|
||||
echo "test_status=${test_status}" >> $GITHUB_ENV
|
||||
echo "tc_name=${tc_name}" >> $GITHUB_ENV
|
||||
export test_status tc_name
|
||||
|
||||
echo "## ${tc_name} ${status_icon} ${test_status}" >> $GITHUB_STEP_SUMMARY
|
||||
echo "" >> $GITHUB_STEP_SUMMARY
|
||||
metric_count=$(grep -c '\[METRIC\]' /tmp/test_output.log 2>/dev/null || echo 0)
|
||||
if [ "${metric_count}" -gt 0 ]; then
|
||||
echo "| Metric | Value | Pass |" >> $GITHUB_STEP_SUMMARY
|
||||
echo "|--------|-------|------|" >> $GITHUB_STEP_SUMMARY
|
||||
grep '\[METRIC\]' /tmp/test_output.log | while IFS= read -r line; do
|
||||
metric_name=$(echo "$line" | sed -E 's/.*\[METRIC\] ([^=]+)=.*/\1/')
|
||||
metric_value=$(echo "$line" | sed -E 's/.*\[METRIC\] [^=]+=([^ ]+).*/\1/')
|
||||
echo "| ${metric_name} | ${metric_value} | ${status_icon} |" >> $GITHUB_STEP_SUMMARY
|
||||
done
|
||||
else
|
||||
echo "No metrics collected (test may have failed before producing results)." >> $GITHUB_STEP_SUMMARY
|
||||
fi
|
||||
echo "" >> $GITHUB_STEP_SUMMARY
|
||||
|
||||
exit ${test_exit_code}
|
||||
+171
-404
@@ -90,303 +90,82 @@ jobs:
|
||||
echo "CANN_image_a3=swr.cn-southwest-2.myhuaweicloud.com/base_image/ascend-ci/cann:9.0.0-a3-ubuntu22.04-py3.11" >> $GITHUB_OUTPUT
|
||||
echo "CANN_image_910b=swr.cn-southwest-2.myhuaweicloud.com/base_image/ascend-ci/cann:9.0.0-910b-ubuntu22.04-py3.11" >> $GITHUB_OUTPUT
|
||||
|
||||
stage-a-unit-test-npu:
|
||||
base-a-test-1-npu-a2:
|
||||
needs: [check-changes, pr-gate, set-image-config]
|
||||
if: needs.check-changes.outputs.main_package == 'true'
|
||||
runs-on: linux-aarch64-a2-1
|
||||
container:
|
||||
if: ${{ !failure() && !cancelled() && needs.check-changes.outputs.main_package == 'true' }}
|
||||
uses: ./.github/workflows/_npu-pr-test-stage.yml
|
||||
with:
|
||||
self_name: base-a-test-1-npu-a2
|
||||
runner_config: linux-aarch64-a2-1
|
||||
image: ${{ needs.set-image-config.outputs.CANN_image_910b }}
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
ref: ${{ inputs.ref || github.ref }}
|
||||
npu_device_type: 910b
|
||||
run_timeout_minutes: '15'
|
||||
secrets: inherit
|
||||
|
||||
- name: Mark repository safe
|
||||
run: |
|
||||
git config --system --add safe.directory ${GITHUB_WORKSPACE}
|
||||
|
||||
- name: Install dependencies
|
||||
env:
|
||||
TORCH_CACHE_URL: "http://cache-service.nginx-pypi-cache.svc.cluster.local/whl/cpu"
|
||||
PYPI_CACHE_URL: "http://cache-service.nginx-pypi-cache.svc.cluster.local/pypi/simple"
|
||||
UV_INDEX_URL: "http://cache-service.nginx-pypi-cache.svc.cluster.local/pypi/simple"
|
||||
GITHUB_PROXY_URL: "https://gh-proxy.test.osinfra.cn/"
|
||||
RUSTUP_CACHE_URL: "http://cache-service.nginx-pypi-cache.svc.cluster.local:8082"
|
||||
run: |
|
||||
# speed up by using infra cache services
|
||||
CACHING_URL="cache-service.nginx-pypi-cache.svc.cluster.local"
|
||||
sed -Ei "s@(ports|archive).ubuntu.com@${CACHING_URL}:8081@g" /etc/apt/sources.list
|
||||
pip config set global.index-url http://${CACHING_URL}/pypi/simple
|
||||
pip config set global.trusted-host "${CACHING_URL}"
|
||||
|
||||
bash scripts/ci/npu/npu_ci_install_dependency.sh 910b
|
||||
|
||||
- name: Run test
|
||||
timeout-minutes: 15
|
||||
env:
|
||||
SGLANG_IS_IN_CI: true
|
||||
run: |
|
||||
cd test
|
||||
python3 run_suite.py --hw npu --suite stage-a-unit-test-npu
|
||||
|
||||
stage-b-test-1-npu-a3:
|
||||
base-b-test-1-npu-a3:
|
||||
needs: [check-changes, pr-gate, set-image-config]
|
||||
if: needs.check-changes.outputs.main_package == 'true'
|
||||
runs-on: linux-aarch64-a3-2-
|
||||
container:
|
||||
image: ${{ needs.set-image-config.outputs.CANN_image_a3 }}
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v4
|
||||
if: ${{ !failure() && !cancelled() && needs.check-changes.outputs.main_package == 'true' }}
|
||||
uses: ./.github/workflows/_npu-pr-test-stage.yml
|
||||
with:
|
||||
ref: ${{ inputs.ref || github.ref }}
|
||||
self_name: base-b-test-1-npu-a3
|
||||
runner_config: linux-aarch64-a3-2-
|
||||
image: ${{ needs.set-image-config.outputs.CANN_image_a3 }}
|
||||
run_timeout_minutes: '60'
|
||||
timeout_per_file: '3600'
|
||||
secrets: inherit
|
||||
|
||||
- name: Mark repository safe
|
||||
run: |
|
||||
git config --system --add safe.directory ${GITHUB_WORKSPACE}
|
||||
|
||||
- name: Install dependencies
|
||||
env:
|
||||
TORCH_CACHE_URL: "http://cache-service.nginx-pypi-cache.svc.cluster.local/whl/cpu"
|
||||
PYPI_CACHE_URL: "http://cache-service.nginx-pypi-cache.svc.cluster.local/pypi/simple"
|
||||
UV_INDEX_URL: "http://cache-service.nginx-pypi-cache.svc.cluster.local/pypi/simple"
|
||||
GITHUB_PROXY_URL: "https://gh-proxy.test.osinfra.cn/"
|
||||
RUSTUP_CACHE_URL: "http://cache-service.nginx-pypi-cache.svc.cluster.local:8082"
|
||||
run: |
|
||||
# speed up by using infra cache services
|
||||
CACHING_URL="cache-service.nginx-pypi-cache.svc.cluster.local"
|
||||
sed -Ei "s@(ports|archive).ubuntu.com@${CACHING_URL}:8081@g" /etc/apt/sources.list
|
||||
pip config set global.index-url http://${CACHING_URL}/pypi/simple
|
||||
pip config set global.trusted-host "${CACHING_URL}"
|
||||
|
||||
bash scripts/ci/npu/npu_ci_install_dependency.sh a3
|
||||
# copy required file from our daily cache
|
||||
cp ~/.cache/modelscope/hub/datasets/otavia/ShareGPT_Vicuna_unfiltered/ShareGPT_V3_unfiltered_cleaned_split.json /tmp
|
||||
# copy gsm8k dataset
|
||||
cp ~/.cache/modelscope/hub/datasets/tmp/test.jsonl /tmp
|
||||
|
||||
- name: Run test
|
||||
timeout-minutes: 60
|
||||
env:
|
||||
SGLANG_USE_MODELSCOPE: true
|
||||
SGLANG_IS_IN_CI: true
|
||||
HF_ENDPOINT: https://hf-mirror.com
|
||||
TORCH_EXTENSIONS_DIR: /tmp/torch_extensions
|
||||
PYTORCH_NPU_ALLOC_CONF: "expandable_segments:True"
|
||||
STREAMS_PER_DEVICE: 32
|
||||
run: |
|
||||
cd test
|
||||
python3 run_suite.py --hw npu --suite stage-b-test-1-npu-a3 --timeout-per-file 3600
|
||||
|
||||
|
||||
stage-b-test-2-npu-a3:
|
||||
base-b-test-2-npu-a3:
|
||||
needs: [check-changes, pr-gate, set-image-config]
|
||||
if: needs.check-changes.outputs.main_package == 'true'
|
||||
runs-on: linux-aarch64-a3-2-
|
||||
container:
|
||||
image: ${{ needs.set-image-config.outputs.CANN_image_a3 }}
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v4
|
||||
if: ${{ !failure() && !cancelled() && needs.check-changes.outputs.main_package == 'true' }}
|
||||
uses: ./.github/workflows/_npu-pr-test-stage.yml
|
||||
with:
|
||||
ref: ${{ inputs.ref || github.ref }}
|
||||
self_name: base-b-test-2-npu-a3
|
||||
runner_config: linux-aarch64-a3-2-
|
||||
image: ${{ needs.set-image-config.outputs.CANN_image_a3 }}
|
||||
run_timeout_minutes: '60'
|
||||
timeout_per_file: '3600'
|
||||
secrets: inherit
|
||||
|
||||
- name: Mark repository safe
|
||||
run: |
|
||||
git config --system --add safe.directory ${GITHUB_WORKSPACE}
|
||||
|
||||
- name: Install dependencies
|
||||
env:
|
||||
TORCH_CACHE_URL: "http://cache-service.nginx-pypi-cache.svc.cluster.local/whl/cpu"
|
||||
PYPI_CACHE_URL: "http://cache-service.nginx-pypi-cache.svc.cluster.local/pypi/simple"
|
||||
UV_INDEX_URL: "http://cache-service.nginx-pypi-cache.svc.cluster.local/pypi/simple"
|
||||
GITHUB_PROXY_URL: "https://gh-proxy.test.osinfra.cn/"
|
||||
RUSTUP_CACHE_URL: "http://cache-service.nginx-pypi-cache.svc.cluster.local:8082"
|
||||
run: |
|
||||
# speed up by using infra cache services
|
||||
CACHING_URL="cache-service.nginx-pypi-cache.svc.cluster.local"
|
||||
sed -Ei "s@(ports|archive).ubuntu.com@${CACHING_URL}:8081@g" /etc/apt/sources.list
|
||||
pip config set global.index-url http://${CACHING_URL}/pypi/simple
|
||||
pip config set global.trusted-host "${CACHING_URL}"
|
||||
|
||||
bash scripts/ci/npu/npu_ci_install_dependency.sh a3
|
||||
# copy required file from our daily cache
|
||||
cp ~/.cache/modelscope/hub/datasets/otavia/ShareGPT_Vicuna_unfiltered/ShareGPT_V3_unfiltered_cleaned_split.json /tmp
|
||||
# copy gsm8k dataset
|
||||
cp ~/.cache/modelscope/hub/datasets/tmp/test.jsonl /tmp
|
||||
|
||||
- name: Run test
|
||||
timeout-minutes: 60
|
||||
env:
|
||||
SGLANG_USE_MODELSCOPE: true
|
||||
SGLANG_IS_IN_CI: true
|
||||
HF_ENDPOINT: https://hf-mirror.com
|
||||
TORCH_EXTENSIONS_DIR: /tmp/torch_extensions
|
||||
PYTORCH_NPU_ALLOC_CONF: "expandable_segments:True"
|
||||
STREAMS_PER_DEVICE: 32
|
||||
run: |
|
||||
cd test
|
||||
python3 run_suite.py --hw npu --suite stage-b-test-2-npu-a3 --timeout-per-file 3600
|
||||
|
||||
|
||||
stage-b-test-4-npu-a3:
|
||||
base-b-test-4-npu-a3:
|
||||
needs: [check-changes, pr-gate, set-image-config]
|
||||
if: needs.check-changes.outputs.main_package == 'true'
|
||||
runs-on: linux-aarch64-a3-4-
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
part: [ 0, 1 ]
|
||||
container:
|
||||
image: ${{ needs.set-image-config.outputs.CANN_image_a3 }}
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v4
|
||||
if: ${{ !failure() && !cancelled() && needs.check-changes.outputs.main_package == 'true' }}
|
||||
uses: ./.github/workflows/_npu-pr-test-stage.yml
|
||||
with:
|
||||
ref: ${{ inputs.ref || github.ref }}
|
||||
self_name: base-b-test-4-npu-a3
|
||||
runner_config: linux-aarch64-a3-4-
|
||||
image: ${{ needs.set-image-config.outputs.CANN_image_a3 }}
|
||||
run_timeout_minutes: '120'
|
||||
timeout_per_file: '3600'
|
||||
partitions: '{"size":2,"arr":[0, 1]}'
|
||||
secrets: inherit
|
||||
|
||||
- name: Mark repository safe
|
||||
run: |
|
||||
git config --system --add safe.directory ${GITHUB_WORKSPACE}
|
||||
|
||||
- name: Install dependencies
|
||||
env:
|
||||
TORCH_CACHE_URL: "http://cache-service.nginx-pypi-cache.svc.cluster.local/whl/cpu"
|
||||
PYPI_CACHE_URL: "http://cache-service.nginx-pypi-cache.svc.cluster.local/pypi/simple"
|
||||
UV_INDEX_URL: "http://cache-service.nginx-pypi-cache.svc.cluster.local/pypi/simple"
|
||||
GITHUB_PROXY_URL: "https://gh-proxy.test.osinfra.cn/"
|
||||
RUSTUP_CACHE_URL: "http://cache-service.nginx-pypi-cache.svc.cluster.local:8082"
|
||||
run: |
|
||||
# speed up by using infra cache services
|
||||
CACHING_URL="cache-service.nginx-pypi-cache.svc.cluster.local"
|
||||
sed -Ei "s@(ports|archive).ubuntu.com@${CACHING_URL}:8081@g" /etc/apt/sources.list
|
||||
pip config set global.index-url http://${CACHING_URL}/pypi/simple
|
||||
pip config set global.trusted-host "${CACHING_URL}"
|
||||
|
||||
bash scripts/ci/npu/npu_ci_install_dependency.sh a3
|
||||
# copy required file from our daily cache
|
||||
cp ~/.cache/modelscope/hub/datasets/otavia/ShareGPT_Vicuna_unfiltered/ShareGPT_V3_unfiltered_cleaned_split.json /tmp
|
||||
# copy gsm8k dataset
|
||||
cp ~/.cache/modelscope/hub/datasets/tmp/test.jsonl /tmp
|
||||
|
||||
- name: Run test
|
||||
timeout-minutes: 120
|
||||
env:
|
||||
SGLANG_USE_MODELSCOPE: true
|
||||
SGLANG_IS_IN_CI: true
|
||||
HF_ENDPOINT: https://hf-mirror.com
|
||||
TORCH_EXTENSIONS_DIR: /tmp/torch_extensions
|
||||
PYTORCH_NPU_ALLOC_CONF: "expandable_segments:True"
|
||||
STREAMS_PER_DEVICE: 32
|
||||
run: |
|
||||
pip install sglang_router
|
||||
cd test
|
||||
python3 run_suite.py --hw npu --suite stage-b-test-4-npu-a3 --auto-partition-id ${{ matrix.part }} --auto-partition-size 2 --timeout-per-file 3600
|
||||
|
||||
stage-b-test-8-npu-a3:
|
||||
base-b-test-8-npu-a3:
|
||||
needs: [check-changes, pr-gate, set-image-config]
|
||||
if: needs.check-changes.outputs.main_package == 'true'
|
||||
runs-on: linux-aarch64-a3-8-
|
||||
container:
|
||||
image: ${{ needs.set-image-config.outputs.CANN_image_a3 }}
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v4
|
||||
if: ${{ !failure() && !cancelled() && needs.check-changes.outputs.main_package == 'true' }}
|
||||
uses: ./.github/workflows/_npu-pr-test-stage.yml
|
||||
with:
|
||||
ref: ${{ inputs.ref || github.ref }}
|
||||
self_name: base-b-test-8-npu-a3
|
||||
runner_config: linux-aarch64-a3-8-
|
||||
image: ${{ needs.set-image-config.outputs.CANN_image_a3 }}
|
||||
run_timeout_minutes: '60'
|
||||
timeout_per_file: '3600'
|
||||
secrets: inherit
|
||||
|
||||
- name: Mark repository safe
|
||||
run: |
|
||||
git config --system --add safe.directory ${GITHUB_WORKSPACE}
|
||||
|
||||
- name: Install dependencies
|
||||
env:
|
||||
TORCH_CACHE_URL: "http://cache-service.nginx-pypi-cache.svc.cluster.local/whl/cpu"
|
||||
PYPI_CACHE_URL: "http://cache-service.nginx-pypi-cache.svc.cluster.local/pypi/simple"
|
||||
UV_INDEX_URL: "http://cache-service.nginx-pypi-cache.svc.cluster.local/pypi/simple"
|
||||
GITHUB_PROXY_URL: "https://gh-proxy.test.osinfra.cn/"
|
||||
RUSTUP_CACHE_URL: "http://cache-service.nginx-pypi-cache.svc.cluster.local:8082"
|
||||
run: |
|
||||
# speed up by using infra cache services
|
||||
CACHING_URL="cache-service.nginx-pypi-cache.svc.cluster.local"
|
||||
sed -Ei "s@(ports|archive).ubuntu.com@${CACHING_URL}:8081@g" /etc/apt/sources.list
|
||||
pip config set global.index-url http://${CACHING_URL}/pypi/simple
|
||||
pip config set global.trusted-host "${CACHING_URL}"
|
||||
|
||||
bash scripts/ci/npu/npu_ci_install_dependency.sh a3
|
||||
# copy required file from our daily cache
|
||||
cp ~/.cache/modelscope/hub/datasets/otavia/ShareGPT_Vicuna_unfiltered/ShareGPT_V3_unfiltered_cleaned_split.json /tmp
|
||||
# copy gsm8k dataset
|
||||
cp ~/.cache/modelscope/hub/datasets/tmp/test.jsonl /tmp
|
||||
|
||||
- name: Run test
|
||||
timeout-minutes: 60
|
||||
env:
|
||||
SGLANG_USE_MODELSCOPE: true
|
||||
SGLANG_IS_IN_CI: true
|
||||
HF_ENDPOINT: https://hf-mirror.com
|
||||
TORCH_EXTENSIONS_DIR: /tmp/torch_extensions
|
||||
PYTORCH_NPU_ALLOC_CONF: "expandable_segments:True"
|
||||
STREAMS_PER_DEVICE: 32
|
||||
run: |
|
||||
cd test
|
||||
python3 run_suite.py --hw npu --suite stage-b-test-8-npu-a3 --timeout-per-file 3600
|
||||
|
||||
stage-b-test-16-npu-a3:
|
||||
base-b-test-16-npu-a3:
|
||||
needs: [check-changes, pr-gate, set-image-config]
|
||||
if: needs.check-changes.outputs.main_package == 'true'
|
||||
runs-on: linux-aarch64-a3-16-
|
||||
container:
|
||||
image: ${{ needs.set-image-config.outputs.CANN_image_a3 }}
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v4
|
||||
if: ${{ !failure() && !cancelled() && needs.check-changes.outputs.main_package == 'true' }}
|
||||
uses: ./.github/workflows/_npu-pr-test-stage.yml
|
||||
with:
|
||||
ref: ${{ inputs.ref || github.ref }}
|
||||
|
||||
- name: Mark repository safe
|
||||
run: |
|
||||
git config --system --add safe.directory ${GITHUB_WORKSPACE}
|
||||
|
||||
- name: Install dependencies
|
||||
env:
|
||||
TORCH_CACHE_URL: "http://cache-service.nginx-pypi-cache.svc.cluster.local/whl/cpu"
|
||||
PYPI_CACHE_URL: "http://cache-service.nginx-pypi-cache.svc.cluster.local/pypi/simple"
|
||||
UV_INDEX_URL: "http://cache-service.nginx-pypi-cache.svc.cluster.local/pypi/simple"
|
||||
GITHUB_PROXY_URL: "https://gh-proxy.test.osinfra.cn/"
|
||||
RUSTUP_CACHE_URL: "http://cache-service.nginx-pypi-cache.svc.cluster.local:8082"
|
||||
run: |
|
||||
# speed up by using infra cache services
|
||||
CACHING_URL="cache-service.nginx-pypi-cache.svc.cluster.local"
|
||||
sed -Ei "s@(ports|archive).ubuntu.com@${CACHING_URL}:8081@g" /etc/apt/sources.list
|
||||
pip config set global.index-url http://${CACHING_URL}/pypi/simple
|
||||
pip config set global.trusted-host "${CACHING_URL}"
|
||||
|
||||
bash scripts/ci/npu/npu_ci_install_dependency.sh a3
|
||||
# copy required file from our daily cache
|
||||
cp ~/.cache/modelscope/hub/datasets/otavia/ShareGPT_Vicuna_unfiltered/ShareGPT_V3_unfiltered_cleaned_split.json /tmp
|
||||
# copy gsm8k dataset
|
||||
cp ~/.cache/modelscope/hub/datasets/tmp/test.jsonl /tmp
|
||||
|
||||
- name: Run test
|
||||
timeout-minutes: 120
|
||||
env:
|
||||
SGLANG_USE_MODELSCOPE: true
|
||||
SGLANG_IS_IN_CI: true
|
||||
HF_ENDPOINT: https://hf-mirror.com
|
||||
TORCH_EXTENSIONS_DIR: /tmp/torch_extensions
|
||||
PYTORCH_NPU_ALLOC_CONF: "expandable_segments:True"
|
||||
STREAMS_PER_DEVICE: 32
|
||||
run: |
|
||||
apt-get install -y libssl-dev
|
||||
pip install sglang_router
|
||||
cd test
|
||||
python3 run_suite.py --hw npu --suite stage-b-test-16-npu-a3 --timeout-per-file 3600
|
||||
self_name: base-b-test-16-npu-a3
|
||||
runner_config: linux-aarch64-a3-16-
|
||||
image: ${{ needs.set-image-config.outputs.CANN_image_a3 }}
|
||||
run_timeout_minutes: '120'
|
||||
timeout_per_file: '3600'
|
||||
secrets: inherit
|
||||
|
||||
multimodal-gen-test-1-npu-a3:
|
||||
needs: [check-changes, pr-gate, set-image-config]
|
||||
if: needs.check-changes.outputs.multimodal_gen == 'true'
|
||||
if: ${{ !failure() && !cancelled() && needs.check-changes.outputs.multimodal_gen == 'true' }}
|
||||
runs-on: linux-aarch64-a3-800t-2
|
||||
container:
|
||||
image: ${{ needs.set-image-config.outputs.CANN_image_a3 }}
|
||||
@@ -443,7 +222,7 @@ jobs:
|
||||
|
||||
multimodal-gen-test-2-npu-a3:
|
||||
needs: [check-changes, pr-gate, set-image-config]
|
||||
if: needs.check-changes.outputs.multimodal_gen == 'true'
|
||||
if: ${{ !failure() && !cancelled() && needs.check-changes.outputs.multimodal_gen == 'true' }}
|
||||
runs-on: linux-aarch64-a3-800t-2
|
||||
container:
|
||||
image: ${{ needs.set-image-config.outputs.CANN_image_a3 }}
|
||||
@@ -498,146 +277,134 @@ jobs:
|
||||
if-no-files-found: ignore
|
||||
retention-days: 7
|
||||
|
||||
pr-single-node-tests:
|
||||
name: single-node-poc
|
||||
needs: [check-changes, pr-gate, set-image-config]
|
||||
if: needs.check-changes.outputs.main_package == 'true'
|
||||
strategy:
|
||||
fail-fast: false
|
||||
max-parallel: 6
|
||||
matrix:
|
||||
test_config:
|
||||
# qwen3_6_27b performance tests
|
||||
- name: qwen3_6_27b_w8a8_1p_in64k_out1k_50ms
|
||||
runner: linux-aarch64-a3-800t-2
|
||||
test_case: test/registered/npu/performance/qwen3_6_27b/test_npu_qwen3_6_27b_w8a8_1p_in64k_out1k_50ms.py
|
||||
test_type: 'perf'
|
||||
|
||||
# - name: qwen3_6_27b_1p_in1024x1024_30_out1024_50ms
|
||||
# runner: linux-aarch64-a3-800t-2
|
||||
# test_case: test/registered/npu/performance/qwen3_6_27b/test_npu_qwen3_6_27b_1p_in1024x1024_30_out1024_50ms.py
|
||||
# test_type: 'perf'
|
||||
|
||||
# # qwen3_8b performance tests
|
||||
# - name: qwen3_8b_w8a8_1p_in3k5_out1k5_50ms
|
||||
# runner: linux-aarch64-a3-800t-2
|
||||
# test_case: test/registered/npu/performance/qwen3-8b/test_npu_qwen3_8b_w8a8_1p_in3k5_out1k5_50ms.py
|
||||
# test_type: 'perf'
|
||||
|
||||
# # qwen3_30b_a3b performance tests
|
||||
# - name: qwen3_30b_w8a8_1p_in3k5_out1k5_50ms
|
||||
# runner: linux-aarch64-a3-800t-2
|
||||
# test_case: test/registered/npu/performance/qwen3_30b_a3b/test_npu_qwen3_30b_w8a8_1p_in3k5_out1k5_50ms.py
|
||||
# test_type: 'perf'
|
||||
|
||||
# # qwen3_6_35b_a3b performance tests
|
||||
# - name: qwen3_6_35b_a3b_1p_in64k_out1k_prefix90_50ms
|
||||
# runner: linux-aarch64-a3-800t-2
|
||||
# test_case: test/registered/npu/performance/qwen3_6_35b_a3b/test_npu_qwen3_6_35b_a3b_1p_in64k_out1k_prefix90_50ms.py
|
||||
# test_type: 'perf'
|
||||
|
||||
# # qwen3_vl_8b_thinking accuracy tests
|
||||
# - name: qwen3_vl_8b_thinking_1p_mmmu
|
||||
# runner: linux-aarch64-a3-2-
|
||||
# test_case: test/registered/npu/accuracy/qwen3_vl_8b_thinking/test_npu_qwen3_vl_8b_thinking_1p_mmmu.py
|
||||
# test_type: 'accuracy'
|
||||
|
||||
# # qwen3_32b performance tests
|
||||
# - name: qwen3_32b_w8a8_2p_in3k5_out1k5_50ms
|
||||
# runner: linux-aarch64-a3-800t-4
|
||||
# test_case: test/registered/npu/performance/qwen3_32b/test_npu_qwen3_32b_w8a8_2p_in3k5_out1k5_50ms.py
|
||||
# test_type: 'perf'
|
||||
|
||||
# # qwen3_next_80b_a3b performance tests
|
||||
# - name: qwen3_next_80b_w8a8_2p_in6k_out1k5_bs16
|
||||
# runner: linux-aarch64-a3-800t-4
|
||||
# test_case: test/registered/npu/performance/qwen3_next_80b_a3b_instruct/test_npu_qwen3_next_80b_w8a8_2p_in6k_out1k5_bs16.py
|
||||
# test_type: 'perf'
|
||||
|
||||
# # minimax_m2_5 performance tests
|
||||
# - name: minimax_m2_5_w8a8_4p_in64k_out1k_prefix90_50ms
|
||||
# runner: linux-aarch64-a3-800t-8
|
||||
# test_case: test/registered/npu/performance/minimax_m2_5/test_npu_minimax_m2_5_w8a8_4p_in64k_out1k_prefix90_50ms.py
|
||||
# test_type: 'perf'
|
||||
|
||||
# # deepseek_v4_flash performance tests
|
||||
# - name: deepseek_v4_flash_w8a8_8p_in8k_out1k_50ms
|
||||
# runner: linux-aarch64-a3-800t-16
|
||||
# test_case: test/registered/npu/performance/deepseek_v4_flash/test_npu_deepseek_v4_flash_w8a8_8p_in8k_out1k_50ms.py
|
||||
# test_type: 'perf'
|
||||
|
||||
# # kimi_k2_6 performance tests
|
||||
# - name: kimi_k2_6_w4a8_8p_in3k5_out1k5_20ms
|
||||
# runner: linux-aarch64-a3-800t-16
|
||||
# test_case: test/registered/npu/performance/kimi_k2_6/test_npu_kimi_k2_6_w4a8_8p_in3k5_out1k5_20ms.py
|
||||
# test_type: 'perf'
|
||||
|
||||
# # qwen3_235b performance tests
|
||||
# - name: qwen3_235b_w8a8_8p_in3k5_out1k5_50ms
|
||||
# runner: linux-aarch64-a3-800t-16
|
||||
# test_case: test/registered/npu/performance/qwen3_235b_a22b/test_npu_qwen3_235b_w8a8_8p_in3k5_out1k5_50ms.py
|
||||
# test_type: 'perf'
|
||||
|
||||
# # qwen3_5_397b performance tests
|
||||
# - name: qwen3_5_397b_w4a8_8p_in3k5_out1k5_50ms
|
||||
# runner: linux-aarch64-a3-800t-16
|
||||
# test_case: test/registered/npu/performance/qwen3_5_397b/test_npu_qwen3_5_397b_w4a8_8p_in3k5_out1k5_50ms.py
|
||||
# test_type: 'perf'
|
||||
|
||||
# NPU accuracy tests
|
||||
# - name: glm4_7_flash_1p_gsm8k
|
||||
# runner: linux-aarch64-a3-2-
|
||||
# test_case: test/registered/npu/accuracy/glm4_7_flash/test_npu_glm4_7_flash_1p_gsm8k.py
|
||||
# test_type: 'accuracy'
|
||||
# - name: qwen3_vl_30b_a3b_bf16_2p_gsm8k
|
||||
# runner: linux-aarch64-a3-4-
|
||||
# test_case: test/registered/npu/accuracy/qwen3_vl_30b_a3b/test_npu_qwen3_vl_30b_a3b_bf16_2p_gsm8k.py
|
||||
# test_type: 'accuracy'
|
||||
- name: glm5_top64_pruned_bf16_8p_gsm8k
|
||||
runner: linux-aarch64-a3-16-
|
||||
test_case: test/registered/npu/accuracy/glm5_top64_pruned/test_npu_glm5_top64_pruned_bf16_8p_gsm8k.py
|
||||
test_type: 'accuracy'
|
||||
# - name: moonshotai_moonlight_16b_a3b_bf16_1p_gsm8k
|
||||
# runner: linux-aarch64-a3-2-
|
||||
# test_case: test/registered/npu/accuracy/moonshotai_moonlight_16b_a3b/test_npu_moonlight_16b_a3b_bf16_1p_gsm8k.py
|
||||
# test_type: 'accuracy'
|
||||
# - name: qwen3_5_9b_bf16_1p_gsm8k
|
||||
# runner: linux-aarch64-a3-2-
|
||||
# test_case: test/registered/npu/accuracy/qwen3_5_9b/test_npu_qwen3_5_9b_bf16_1p_gsm8k.py
|
||||
# test_type: 'accuracy'
|
||||
# - name: qwen3_vl_8b_bf16_2p_gsm8k
|
||||
# runner: linux-aarch64-a3-4-
|
||||
# test_case: test/registered/npu/accuracy/qwen3_vl_8b/test_npu_qwen3_vl_8b_bf16_2p_gsm8k.py
|
||||
# test_type: 'accuracy'
|
||||
|
||||
uses: ./.github/workflows/nightly-test-npu-e2e-single-node.yml
|
||||
base-c-test-acc-2-npu-a3:
|
||||
name: base-c-test-acc-2-npu-a3
|
||||
needs: [ check-changes, pr-gate, set-image-config ]
|
||||
if: ${{ !failure() && !cancelled() && needs.check-changes.outputs.main_package == 'true' }}
|
||||
uses: ./.github/workflows/_npu-single-node-test-stage.yml
|
||||
with:
|
||||
runner: ${{ matrix.test_config.runner }}
|
||||
test_type: ${{ matrix.test_config.test_type }}
|
||||
test_config_name: ${{ matrix.test_config.name }}
|
||||
test_case: ${{ matrix.test_config.test_case }}
|
||||
runner: linux-aarch64-a3-2-
|
||||
test_type: 'accuracy'
|
||||
test_suite: base-c-test-acc-2-npu-a3
|
||||
image: ${{ needs.set-image-config.outputs.CANN_image_a3 }}
|
||||
install_sglang_from_source: false
|
||||
install_sglang_deps: true
|
||||
device_type_for_deps: 'a3'
|
||||
transformers_version: ''
|
||||
|
||||
base-c-test-acc-4-npu-a3:
|
||||
name: base-c-test-acc-4-npu-a3
|
||||
needs: [ check-changes, pr-gate, set-image-config ]
|
||||
if: ${{ !failure() && !cancelled() && needs.check-changes.outputs.main_package == 'true' }}
|
||||
uses: ./.github/workflows/_npu-single-node-test-stage.yml
|
||||
with:
|
||||
runner: linux-aarch64-a3-4-
|
||||
test_type: 'accuracy'
|
||||
test_suite: base-c-test-acc-4-npu-a3
|
||||
image: ${{ needs.set-image-config.outputs.CANN_image_a3 }}
|
||||
install_sglang_deps: true
|
||||
device_type_for_deps: 'a3'
|
||||
|
||||
base-c-test-acc-8-npu-a3:
|
||||
name: base-c-test-acc-8-npu-a3
|
||||
needs: [ check-changes, pr-gate, set-image-config ]
|
||||
if: ${{ !failure() && !cancelled() && needs.check-changes.outputs.main_package == 'true' }}
|
||||
uses: ./.github/workflows/_npu-single-node-test-stage.yml
|
||||
with:
|
||||
runner: linux-aarch64-a3-8-
|
||||
test_type: 'accuracy'
|
||||
test_suite: base-c-test-acc-8-npu-a3
|
||||
image: ${{ needs.set-image-config.outputs.CANN_image_a3 }}
|
||||
install_sglang_deps: true
|
||||
device_type_for_deps: 'a3'
|
||||
|
||||
base-c-test-acc-16-npu-a3:
|
||||
name: base-c-test-acc-16-npu-a3
|
||||
needs: [ check-changes, pr-gate, set-image-config ]
|
||||
if: ${{ !failure() && !cancelled() && needs.check-changes.outputs.main_package == 'true' }}
|
||||
uses: ./.github/workflows/_npu-single-node-test-stage.yml
|
||||
with:
|
||||
runner: linux-aarch64-a3-16-
|
||||
test_type: 'accuracy'
|
||||
test_suite: base-c-test-acc-16-npu-a3
|
||||
image: ${{ needs.set-image-config.outputs.CANN_image_a3 }}
|
||||
install_sglang_deps: true
|
||||
device_type_for_deps: 'a3'
|
||||
|
||||
base-c-test-perf-2-npu-a3:
|
||||
name: base-c-test-perf-2-npu-a3
|
||||
needs: [ check-changes, pr-gate, set-image-config, base-c-test-acc-2-npu-a3 ]
|
||||
if: ${{ !failure() && !cancelled() && needs.check-changes.outputs.main_package == 'true' }}
|
||||
uses: ./.github/workflows/_npu-single-node-test-stage.yml
|
||||
with:
|
||||
runner: linux-aarch64-a3-800t-2
|
||||
test_type: 'perf'
|
||||
test_suite: base-c-test-perf-2-npu-a3
|
||||
image: ${{ needs.set-image-config.outputs.CANN_image_a3 }}
|
||||
install_sglang_deps: true
|
||||
device_type_for_deps: 'a3'
|
||||
|
||||
base-c-test-perf-4-npu-a3:
|
||||
name: base-c-test-perf-4-npu-a3
|
||||
needs: [ check-changes, pr-gate, set-image-config, base-c-test-acc-4-npu-a3 ]
|
||||
if: ${{ !failure() && !cancelled() && needs.check-changes.outputs.main_package == 'true' }}
|
||||
uses: ./.github/workflows/_npu-single-node-test-stage.yml
|
||||
with:
|
||||
runner: linux-aarch64-a3-800t-4
|
||||
test_type: 'perf'
|
||||
test_suite: base-c-test-perf-4-npu-a3
|
||||
image: ${{ needs.set-image-config.outputs.CANN_image_a3 }}
|
||||
install_sglang_deps: true
|
||||
device_type_for_deps: 'a3'
|
||||
|
||||
base-c-test-perf-8-npu-a3:
|
||||
name: base-c-test-perf-8-npu-a3
|
||||
needs: [ check-changes, pr-gate, set-image-config, base-c-test-acc-8-npu-a3 ]
|
||||
if: ${{ !failure() && !cancelled() && needs.check-changes.outputs.main_package == 'true' }}
|
||||
uses: ./.github/workflows/_npu-single-node-test-stage.yml
|
||||
with:
|
||||
runner: linux-aarch64-a3-800t-8
|
||||
test_type: 'perf'
|
||||
test_suite: base-c-test-perf-8-npu-a3
|
||||
image: ${{ needs.set-image-config.outputs.CANN_image_a3 }}
|
||||
install_sglang_deps: true
|
||||
device_type_for_deps: 'a3'
|
||||
|
||||
base-c-test-perf-16-npu-a3:
|
||||
name: base-c-test-perf-16-npu-a3
|
||||
needs: [ check-changes, pr-gate, set-image-config, base-c-test-acc-16-npu-a3 ]
|
||||
if: ${{ !failure() && !cancelled() && needs.check-changes.outputs.main_package == 'true' }}
|
||||
uses: ./.github/workflows/_npu-single-node-test-stage.yml
|
||||
with:
|
||||
runner: linux-aarch64-a3-800t-16
|
||||
test_type: 'perf'
|
||||
test_suite: base-c-test-perf-16-npu-a3
|
||||
image: ${{ needs.set-image-config.outputs.CANN_image_a3 }}
|
||||
install_sglang_deps: true
|
||||
device_type_for_deps: 'a3'
|
||||
|
||||
|
||||
pr-test-npu-finish:
|
||||
needs:
|
||||
[
|
||||
check-changes,
|
||||
|
||||
stage-a-unit-test-npu,
|
||||
stage-b-test-1-npu-a3,
|
||||
stage-b-test-2-npu-a3,
|
||||
stage-b-test-4-npu-a3,
|
||||
stage-b-test-8-npu-a3,
|
||||
stage-b-test-16-npu-a3,
|
||||
base-a-test-1-npu-a2,
|
||||
base-b-test-1-npu-a3,
|
||||
base-b-test-2-npu-a3,
|
||||
base-b-test-4-npu-a3,
|
||||
base-b-test-8-npu-a3,
|
||||
base-b-test-16-npu-a3,
|
||||
|
||||
multimodal-gen-test-1-npu-a3,
|
||||
multimodal-gen-test-2-npu-a3,
|
||||
|
||||
pr-single-node-tests,
|
||||
base-c-test-acc-2-npu-a3,
|
||||
base-c-test-acc-4-npu-a3,
|
||||
base-c-test-acc-8-npu-a3,
|
||||
base-c-test-acc-16-npu-a3,
|
||||
base-c-test-perf-2-npu-a3,
|
||||
base-c-test-perf-4-npu-a3,
|
||||
base-c-test-perf-8-npu-a3,
|
||||
base-c-test-perf-16-npu-a3,
|
||||
]
|
||||
if: always()
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
@@ -1,10 +1,15 @@
|
||||
import glob
|
||||
import inspect
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
from datetime import datetime
|
||||
from urllib.parse import urlparse
|
||||
|
||||
from sglang.srt.utils import kill_process_tree
|
||||
@@ -289,9 +294,117 @@ class TestNpuAccuracyTestCaseBase(CustomTestCase):
|
||||
max_attempts = 2
|
||||
n_runs = 3
|
||||
accuracy = 0.1
|
||||
test_type = "accuracy"
|
||||
|
||||
@classmethod
|
||||
def _get_tc_name(cls):
|
||||
"""Derive the test case name from the test file (filename without
|
||||
extension). Mirrors the workflow's ``tc_name=${test_case##*/}`` logic
|
||||
so each case in a suite writes to its own output path."""
|
||||
try:
|
||||
tc_file = inspect.getfile(cls)
|
||||
except (TypeError, OSError):
|
||||
tc_file = getattr(sys.modules.get(cls.__module__), "__file__", "")
|
||||
return os.path.splitext(os.path.basename(tc_file))[0]
|
||||
|
||||
@classmethod
|
||||
def _setup_per_case_output(cls):
|
||||
"""Set up per-case output directories and env vars.
|
||||
|
||||
Extracted from ``nightly-test-npu-e2e-single-node.yml`` so that when a
|
||||
suite is executed, each case writes its metrics/plog to a path derived
|
||||
from the case file rather than the suite name.
|
||||
"""
|
||||
cls.tc_name = cls._get_tc_name()
|
||||
current_date = datetime.now().strftime("%Y%m%d")
|
||||
test_type = getattr(cls, "test_type", "accuracy")
|
||||
base_output = f"/root/.cache/tests/output/{test_type}/{current_date}"
|
||||
os.makedirs(base_output, exist_ok=True)
|
||||
cls.metrics_data_file = os.path.join(base_output, cls.tc_name)
|
||||
os.makedirs(cls.metrics_data_file, exist_ok=True)
|
||||
# Override env vars so evalscope/dump_metric write to per-case paths.
|
||||
os.environ["METRICS_DATA_FILE"] = cls.metrics_data_file
|
||||
os.environ["SGLANG_TEST_METRICS_OUTPUT"] = os.path.join(
|
||||
cls.metrics_data_file, "metrics"
|
||||
)
|
||||
logger.info(
|
||||
"Per-case output: tc_name=%s metrics_data_file=%s",
|
||||
cls.tc_name,
|
||||
cls.metrics_data_file,
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def _save_metrics_json(cls):
|
||||
"""Write per-case ``metrics.json`` from ``dump_metric`` JSONL files.
|
||||
|
||||
Replaces the workflow's stdout-parsing + ``dump_metrics.py`` logic so
|
||||
each case in a suite persists its own metrics snapshot.
|
||||
"""
|
||||
if not getattr(cls, "metrics_data_file", None):
|
||||
return
|
||||
metrics = {}
|
||||
baselines = {}
|
||||
pattern = os.path.join(cls.metrics_data_file, "metrics.*.jsonl")
|
||||
for jsonl_path in glob.glob(pattern):
|
||||
try:
|
||||
with open(jsonl_path, "r", encoding="utf-8") as f:
|
||||
for line in f:
|
||||
line = line.strip()
|
||||
if not line:
|
||||
continue
|
||||
record = json.loads(line)
|
||||
name = record.get("metric_name")
|
||||
value = record.get("value")
|
||||
if name is None:
|
||||
continue
|
||||
if name.endswith("_baseline"):
|
||||
baselines[name[: -len("_baseline")]] = value
|
||||
else:
|
||||
metrics[name] = value
|
||||
except Exception as e:
|
||||
logger.warning("Failed to read %s: %s", jsonl_path, e)
|
||||
out_path = os.path.join(cls.metrics_data_file, "metrics.json")
|
||||
payload = {
|
||||
"test_case": cls.tc_name,
|
||||
"test_type": getattr(cls, "test_type", "accuracy"),
|
||||
"metrics": metrics,
|
||||
"baselines": baselines,
|
||||
}
|
||||
try:
|
||||
with open(out_path, "w", encoding="utf-8") as f:
|
||||
json.dump(payload, f, ensure_ascii=False)
|
||||
logger.info("Saved per-case metrics to %s", out_path)
|
||||
except Exception as e:
|
||||
logger.warning("Failed to write metrics.json: %s", e)
|
||||
|
||||
@classmethod
|
||||
def _backup_plog(cls):
|
||||
"""Backup Ascend plog files to a per-case path.
|
||||
|
||||
Replaces the workflow's ``Backup plog`` step so each case in a suite
|
||||
gets its own plog snapshot instead of all cases sharing the suite name.
|
||||
"""
|
||||
plog_path = "/root/ascend/log/debug/plog"
|
||||
if not os.path.isdir(plog_path):
|
||||
return
|
||||
tc_name = getattr(cls, "tc_name", None)
|
||||
if not tc_name:
|
||||
return
|
||||
hostname = os.getenv("HOSTNAME", "unknown")
|
||||
target = os.path.join("/root/.cache/tests/logs/plog", tc_name, hostname)
|
||||
os.makedirs(target, exist_ok=True)
|
||||
for name in os.listdir(plog_path):
|
||||
src = os.path.join(plog_path, name)
|
||||
if os.path.isfile(src):
|
||||
try:
|
||||
shutil.copy2(src, os.path.join(target, name))
|
||||
except Exception as e:
|
||||
logger.warning("Failed to copy plog %s: %s", name, e)
|
||||
logger.info("Backed up plog to %s", target)
|
||||
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
cls._setup_per_case_output()
|
||||
cls.base_url = DEFAULT_URL_FOR_TEST
|
||||
env = os.environ.copy()
|
||||
for key, value in env.items():
|
||||
@@ -318,6 +431,8 @@ class TestNpuAccuracyTestCaseBase(CustomTestCase):
|
||||
kill_process_tree(cls.process.pid)
|
||||
except Exception as e:
|
||||
logger.error(f"Error during tearDown: {e}")
|
||||
cls._save_metrics_json()
|
||||
cls._backup_plog()
|
||||
|
||||
def run_accuracy(self):
|
||||
parsed_url = urlparse(self.base_url)
|
||||
|
||||
@@ -1,9 +1,15 @@
|
||||
import glob
|
||||
import inspect
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
from datetime import datetime
|
||||
from functools import wraps
|
||||
from urllib.parse import urlparse
|
||||
|
||||
@@ -909,8 +915,115 @@ class TestNpuPerformanceTestCaseBase(CustomTestCase):
|
||||
generation_kwargs = None
|
||||
pop_sglang_is_in_ci_for_gsp = False
|
||||
|
||||
@classmethod
|
||||
def _get_tc_name(cls):
|
||||
"""Derive the test case name from the test file (filename without
|
||||
extension). Mirrors the workflow's ``tc_name=${test_case##*/}`` logic
|
||||
so each case in a suite writes to its own output path."""
|
||||
try:
|
||||
tc_file = inspect.getfile(cls)
|
||||
except (TypeError, OSError):
|
||||
tc_file = getattr(sys.modules.get(cls.__module__), "__file__", "")
|
||||
return os.path.splitext(os.path.basename(tc_file))[0]
|
||||
|
||||
@classmethod
|
||||
def _setup_per_case_output(cls):
|
||||
"""Set up per-case output directories and env vars.
|
||||
|
||||
Extracted from ``nightly-test-npu-e2e-single-node.yml`` so that when a
|
||||
suite is executed, each case writes its metrics/plog to a path derived
|
||||
from the case file rather than the suite name.
|
||||
"""
|
||||
cls.tc_name = cls._get_tc_name()
|
||||
current_date = datetime.now().strftime("%Y%m%d")
|
||||
test_type = getattr(cls, "test_type", "perf")
|
||||
base_output = f"/root/.cache/tests/output/{test_type}/{current_date}"
|
||||
os.makedirs(base_output, exist_ok=True)
|
||||
cls.metrics_data_file = os.path.join(base_output, cls.tc_name)
|
||||
os.makedirs(cls.metrics_data_file, exist_ok=True)
|
||||
# Override env vars so evalscope/dump_metric write to per-case paths.
|
||||
os.environ["METRICS_DATA_FILE"] = cls.metrics_data_file
|
||||
os.environ["SGLANG_TEST_METRICS_OUTPUT"] = os.path.join(
|
||||
cls.metrics_data_file, "metrics"
|
||||
)
|
||||
logger.info(
|
||||
"Per-case output: tc_name=%s metrics_data_file=%s",
|
||||
cls.tc_name,
|
||||
cls.metrics_data_file,
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def _save_metrics_json(cls):
|
||||
"""Write per-case ``metrics.json`` from ``dump_metric`` JSONL files.
|
||||
|
||||
Replaces the workflow's stdout-parsing + ``dump_metrics.py`` logic so
|
||||
each case in a suite persists its own metrics snapshot.
|
||||
"""
|
||||
if not getattr(cls, "metrics_data_file", None):
|
||||
return
|
||||
metrics = {}
|
||||
baselines = {}
|
||||
pattern = os.path.join(cls.metrics_data_file, "metrics.*.jsonl")
|
||||
for jsonl_path in glob.glob(pattern):
|
||||
try:
|
||||
with open(jsonl_path, "r", encoding="utf-8") as f:
|
||||
for line in f:
|
||||
line = line.strip()
|
||||
if not line:
|
||||
continue
|
||||
record = json.loads(line)
|
||||
name = record.get("metric_name")
|
||||
value = record.get("value")
|
||||
if name is None:
|
||||
continue
|
||||
if name.endswith("_baseline"):
|
||||
baselines[name[: -len("_baseline")]] = value
|
||||
else:
|
||||
metrics[name] = value
|
||||
except Exception as e:
|
||||
logger.warning("Failed to read %s: %s", jsonl_path, e)
|
||||
out_path = os.path.join(cls.metrics_data_file, "metrics.json")
|
||||
payload = {
|
||||
"test_case": cls.tc_name,
|
||||
"test_type": getattr(cls, "test_type", "accuracy"),
|
||||
"metrics": metrics,
|
||||
"baselines": baselines,
|
||||
}
|
||||
try:
|
||||
with open(out_path, "w", encoding="utf-8") as f:
|
||||
json.dump(payload, f, ensure_ascii=False)
|
||||
logger.info("Saved per-case metrics to %s", out_path)
|
||||
except Exception as e:
|
||||
logger.warning("Failed to write metrics.json: %s", e)
|
||||
|
||||
@classmethod
|
||||
def _backup_plog(cls):
|
||||
"""Backup Ascend plog files to a per-case path.
|
||||
|
||||
Replaces the workflow's ``Backup plog`` step so each case in a suite
|
||||
gets its own plog snapshot instead of all cases sharing the suite name.
|
||||
"""
|
||||
plog_path = "/root/ascend/log/debug/plog"
|
||||
if not os.path.isdir(plog_path):
|
||||
return
|
||||
tc_name = getattr(cls, "tc_name", None)
|
||||
if not tc_name:
|
||||
return
|
||||
hostname = os.getenv("HOSTNAME", "unknown")
|
||||
target = os.path.join("/root/.cache/tests/logs/plog", tc_name, hostname)
|
||||
os.makedirs(target, exist_ok=True)
|
||||
for name in os.listdir(plog_path):
|
||||
src = os.path.join(plog_path, name)
|
||||
if os.path.isfile(src):
|
||||
try:
|
||||
shutil.copy2(src, os.path.join(target, name))
|
||||
except Exception as e:
|
||||
logger.warning("Failed to copy plog %s: %s", name, e)
|
||||
logger.info("Backed up plog to %s", target)
|
||||
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
cls._setup_per_case_output()
|
||||
cls.base_url = DEFAULT_URL_FOR_TEST
|
||||
env = os.environ.copy()
|
||||
for key, value in env.items():
|
||||
@@ -937,6 +1050,8 @@ class TestNpuPerformanceTestCaseBase(CustomTestCase):
|
||||
kill_process_tree(cls.process.pid)
|
||||
except Exception as e:
|
||||
logger.error(f"Error during tearDown: {e}")
|
||||
cls._save_metrics_json()
|
||||
cls._backup_plog()
|
||||
|
||||
@retry()
|
||||
def run_throughput(self):
|
||||
|
||||
@@ -6,12 +6,7 @@ from sglang.test.ascend.e2e.test_npu_accuracy_utils import (
|
||||
from sglang.test.ascend.e2e.test_npu_performance_utils import GLM_4_7_FLASH_MODEL_PATH
|
||||
from sglang.test.ci.ci_register import register_npu_ci
|
||||
|
||||
register_npu_ci(
|
||||
est_time=3600,
|
||||
suite="",
|
||||
nightly=True,
|
||||
disabled="accuracy testcase",
|
||||
)
|
||||
register_npu_ci(est_time=3600, suite="base-c-test-acc-2-npu-a3")
|
||||
|
||||
ENVS = {
|
||||
"PYTORCH_NPU_ALLOC_CONF": "expandable_segments:True",
|
||||
|
||||
+2
-7
@@ -8,12 +8,7 @@ from sglang.test.ascend.e2e.test_npu_performance_utils import (
|
||||
)
|
||||
from sglang.test.ci.ci_register import register_npu_ci
|
||||
|
||||
register_npu_ci(
|
||||
est_time=3600,
|
||||
suite="",
|
||||
nightly=True,
|
||||
disabled="accuracy testcase",
|
||||
)
|
||||
register_npu_ci(est_time=3600, suite="base-c-test-acc-16-npu-a3")
|
||||
|
||||
ENVS = {
|
||||
"SGLANG_SET_CPU_AFFINITY": "1",
|
||||
@@ -54,7 +49,7 @@ class TestNPUGLM5_Top64_Pruned_GSM8K(TestNpuAccuracyTestCaseBase):
|
||||
model = GLM5_TOP64_PRUNED_GSM8K_MODEL_PATH
|
||||
envs = ENVS
|
||||
other_args = OTHER_ARGS
|
||||
accuracy = 0.50
|
||||
accuracy = 0.48
|
||||
datasets = ["gsm8k"]
|
||||
generation_config = {
|
||||
"max_tokens": 2048,
|
||||
|
||||
+1
-6
@@ -8,12 +8,7 @@ from sglang.test.ascend.e2e.test_npu_performance_utils import (
|
||||
)
|
||||
from sglang.test.ci.ci_register import register_npu_ci
|
||||
|
||||
register_npu_ci(
|
||||
est_time=3600,
|
||||
suite="",
|
||||
nightly=True,
|
||||
disabled="accuracy testcase",
|
||||
)
|
||||
register_npu_ci(est_time=3600, suite="base-c-test-acc-2-npu-a3")
|
||||
|
||||
MODEL_ENVS = {
|
||||
"SGLANG_SET_CPU_AFFINITY": "1",
|
||||
|
||||
@@ -8,12 +8,7 @@ from sglang.test.ascend.e2e.test_npu_performance_utils import (
|
||||
)
|
||||
from sglang.test.ci.ci_register import register_npu_ci
|
||||
|
||||
register_npu_ci(
|
||||
est_time=3600,
|
||||
suite="stage-b-test-2-npu-a3",
|
||||
nightly=True,
|
||||
disabled="accuracy testcase",
|
||||
)
|
||||
register_npu_ci(est_time=3600, suite="base-c-test-acc-2-npu-a3")
|
||||
|
||||
QWEN3_5_9B_ENVS = {
|
||||
"SGLANG_SET_CPU_AFFINITY": "1",
|
||||
|
||||
+1
-6
@@ -8,12 +8,7 @@ from sglang.test.ascend.e2e.test_npu_performance_utils import (
|
||||
)
|
||||
from sglang.test.ci.ci_register import register_npu_ci
|
||||
|
||||
register_npu_ci(
|
||||
est_time=3600,
|
||||
suite="stage-b-test-4-npu-a3",
|
||||
nightly=True,
|
||||
disabled="accuracy testcase",
|
||||
)
|
||||
register_npu_ci(est_time=3600, suite="base-c-test-acc-4-npu-a3")
|
||||
|
||||
QWEN3_VL_30B_A3B_ENVS = {
|
||||
"SGLANG_SET_CPU_AFFINITY": "1",
|
||||
|
||||
@@ -8,12 +8,7 @@ from sglang.test.ascend.e2e.test_npu_performance_utils import (
|
||||
)
|
||||
from sglang.test.ci.ci_register import register_npu_ci
|
||||
|
||||
register_npu_ci(
|
||||
est_time=3600,
|
||||
suite="stage-b-test-4-npu-a3",
|
||||
nightly=True,
|
||||
disabled="accuracy testcase",
|
||||
)
|
||||
register_npu_ci(est_time=3600, suite="base-c-test-acc-4-npu-a3")
|
||||
|
||||
QWEN3_VL_8B_ENVS = {
|
||||
"SGLANG_SET_CPU_AFFINITY": "1",
|
||||
|
||||
+1
-6
@@ -9,12 +9,7 @@ from sglang.test.ascend.e2e.test_npu_performance_utils import (
|
||||
)
|
||||
from sglang.test.ci.ci_register import register_npu_ci
|
||||
|
||||
register_npu_ci(
|
||||
est_time=3600,
|
||||
suite="",
|
||||
nightly=True,
|
||||
disabled="performance testcase",
|
||||
)
|
||||
register_npu_ci(est_time=3600, suite="base-c-test-perf-2-npu-a3")
|
||||
|
||||
_is_pr_pipeline = os.environ.get("GITHUB_EVENT_NAME") == "pull_request"
|
||||
|
||||
|
||||
@@ -13,7 +13,7 @@ from sglang.test.test_utils import (
|
||||
popen_launch_server,
|
||||
)
|
||||
|
||||
register_npu_ci(est_time=400, suite="stage-b-test-1-npu-a3", nightly=False)
|
||||
register_npu_ci(est_time=400, suite="base-b-test-1-npu-a3")
|
||||
register_npu_ci(est_time=400, suite="nightly-1-npu-a3", nightly=True)
|
||||
|
||||
TEST_MODEL_MATRIX = {
|
||||
|
||||
@@ -13,7 +13,7 @@ from sglang.test.test_utils import (
|
||||
popen_launch_server,
|
||||
)
|
||||
|
||||
register_npu_ci(est_time=400, suite="stage-b-test-4-npu-a3", nightly=False)
|
||||
register_npu_ci(est_time=400, suite="base-b-test-4-npu-a3")
|
||||
register_npu_ci(est_time=400, suite="nightly-4-npu-a3", nightly=True)
|
||||
|
||||
TEST_MODEL_MATRIX = {
|
||||
|
||||
@@ -14,7 +14,7 @@ from sglang.test.test_utils import (
|
||||
popen_launch_server,
|
||||
)
|
||||
|
||||
register_npu_ci(est_time=400, suite="stage-b-test-1-npu-a3", nightly=False)
|
||||
register_npu_ci(est_time=400, suite="base-b-test-1-npu-a3")
|
||||
register_npu_ci(est_time=400, suite="nightly-1-npu-a3", nightly=True)
|
||||
|
||||
|
||||
|
||||
@@ -10,7 +10,7 @@ from sglang.test.test_utils import (
|
||||
write_github_step_summary,
|
||||
)
|
||||
|
||||
register_npu_ci(est_time=400, suite="stage-b-test-4-npu-a3", nightly=False)
|
||||
register_npu_ci(est_time=400, suite="base-b-test-4-npu-a3")
|
||||
register_npu_ci(est_time=400, suite="nightly-1-npu-a3", nightly=True)
|
||||
|
||||
|
||||
|
||||
@@ -23,7 +23,7 @@ from sglang.test.test_utils import (
|
||||
popen_launch_server,
|
||||
)
|
||||
|
||||
register_npu_ci(est_time=400, suite="stage-b-test-4-npu-a3", nightly=False)
|
||||
register_npu_ci(est_time=400, suite="base-b-test-4-npu-a3")
|
||||
register_npu_ci(est_time=400, suite="nightly-4-npu-a3", nightly=True)
|
||||
|
||||
|
||||
|
||||
+1
-1
@@ -14,7 +14,7 @@ from sglang.test.test_utils import (
|
||||
popen_launch_server,
|
||||
)
|
||||
|
||||
register_npu_ci(est_time=400, suite="stage-b-test-1-npu-a3", nightly=False)
|
||||
register_npu_ci(est_time=400, suite="base-b-test-1-npu-a3")
|
||||
register_npu_ci(est_time=400, suite="nightly-1-npu-a3", nightly=True)
|
||||
|
||||
TEST_MODEL_MATRIX = {
|
||||
|
||||
+1
-1
@@ -12,7 +12,7 @@ from sglang.test.test_utils import (
|
||||
run_bench_one_batch,
|
||||
)
|
||||
|
||||
register_npu_ci(est_time=400, suite="stage-b-test-1-npu-a3", nightly=False)
|
||||
register_npu_ci(est_time=400, suite="base-b-test-1-npu-a3")
|
||||
register_npu_ci(est_time=400, suite="nightly-1-npu-a3", nightly=True)
|
||||
|
||||
|
||||
|
||||
+1
-1
@@ -18,7 +18,7 @@ from sglang.test.test_utils import (
|
||||
popen_launch_server,
|
||||
)
|
||||
|
||||
register_npu_ci(est_time=700, suite="stage-b-test-16-npu-a3", nightly=False)
|
||||
register_npu_ci(est_time=700, suite="base-b-test-16-npu-a3")
|
||||
register_npu_ci(est_time=700, suite="nightly-16-npu-a3", nightly=True)
|
||||
|
||||
|
||||
|
||||
+1
-1
@@ -23,7 +23,7 @@ from sglang.test.test_utils import (
|
||||
popen_launch_pd_server,
|
||||
)
|
||||
|
||||
register_npu_ci(est_time=3600, suite="stage-b-test-16-npu-a3", nightly=False)
|
||||
register_npu_ci(est_time=3600, suite="base-b-test-16-npu-a3")
|
||||
register_npu_ci(est_time=3600, suite="nightly-16-npu-a3", nightly=True)
|
||||
|
||||
load_balance_method_options = [
|
||||
|
||||
+1
-1
@@ -13,7 +13,7 @@ from sglang.test.test_utils import (
|
||||
popen_launch_server,
|
||||
)
|
||||
|
||||
register_npu_ci(est_time=400, suite="stage-b-test-16-npu-a3", nightly=False)
|
||||
register_npu_ci(est_time=400, suite="base-b-test-16-npu-a3")
|
||||
register_npu_ci(est_time=400, suite="nightly-16-npu-a3", nightly=True)
|
||||
|
||||
TEST_MODEL_MATRIX = {
|
||||
|
||||
+1
-1
@@ -14,7 +14,7 @@ from sglang.test.test_utils import (
|
||||
popen_launch_server,
|
||||
)
|
||||
|
||||
register_npu_ci(est_time=400, suite="stage-b-test-8-npu-a3", nightly=False)
|
||||
register_npu_ci(est_time=400, suite="base-b-test-8-npu-a3")
|
||||
register_npu_ci(est_time=400, suite="nightly-8-npu-a3", nightly=True)
|
||||
|
||||
|
||||
|
||||
+1
-1
@@ -17,7 +17,7 @@ from sglang.test.test_utils import (
|
||||
popen_launch_server,
|
||||
)
|
||||
|
||||
register_npu_ci(est_time=400, suite="stage-b-test-2-npu-a3", nightly=False)
|
||||
register_npu_ci(est_time=400, suite="base-b-test-2-npu-a3")
|
||||
register_npu_ci(est_time=400, suite="nightly-2-npu-a3", nightly=True)
|
||||
|
||||
|
||||
|
||||
+1
-1
@@ -13,7 +13,7 @@ from sglang.test.test_utils import (
|
||||
popen_launch_server,
|
||||
)
|
||||
|
||||
register_npu_ci(est_time=400, suite="stage-b-test-2-npu-a3", nightly=False)
|
||||
register_npu_ci(est_time=400, suite="base-b-test-2-npu-a3")
|
||||
register_npu_ci(est_time=400, suite="nightly-2-npu-a3", nightly=True)
|
||||
|
||||
|
||||
|
||||
+1
-1
@@ -12,7 +12,7 @@ from sglang.test.test_utils import (
|
||||
popen_launch_server,
|
||||
)
|
||||
|
||||
register_npu_ci(est_time=400, suite="stage-b-test-2-npu-a3", nightly=False)
|
||||
register_npu_ci(est_time=400, suite="base-b-test-2-npu-a3")
|
||||
register_npu_ci(est_time=400, suite="nightly-2-npu-a3", nightly=True)
|
||||
|
||||
|
||||
|
||||
@@ -21,7 +21,7 @@ from sglang.test.test_utils import (
|
||||
popen_with_error_check,
|
||||
)
|
||||
|
||||
register_npu_ci(est_time=400, suite="stage-b-test-16-npu-a3", nightly=False)
|
||||
register_npu_ci(est_time=400, suite="base-b-test-16-npu-a3")
|
||||
register_npu_ci(est_time=400, suite="nightly-16-npu-a3", nightly=True)
|
||||
|
||||
|
||||
|
||||
@@ -15,7 +15,7 @@ from sglang.test.test_utils import (
|
||||
popen_launch_server,
|
||||
)
|
||||
|
||||
register_npu_ci(est_time=400, suite="stage-b-test-1-npu-a3", nightly=False)
|
||||
register_npu_ci(est_time=400, suite="base-b-test-1-npu-a3")
|
||||
register_npu_ci(est_time=400, suite="nightly-1-npu-a3", nightly=True)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -16,7 +16,7 @@ from sglang.test.test_utils import (
|
||||
popen_launch_server,
|
||||
)
|
||||
|
||||
register_npu_ci(est_time=400, suite="stage-b-test-1-npu-a3", nightly=False)
|
||||
register_npu_ci(est_time=400, suite="base-b-test-1-npu-a3")
|
||||
register_npu_ci(est_time=400, suite="nightly-1-npu-a3", nightly=True)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -17,7 +17,7 @@ from sglang.test.test_utils import (
|
||||
popen_launch_server,
|
||||
)
|
||||
|
||||
register_npu_ci(est_time=400, suite="stage-b-test-1-npu-a3", nightly=False)
|
||||
register_npu_ci(est_time=400, suite="base-b-test-1-npu-a3")
|
||||
register_npu_ci(est_time=400, suite="nightly-1-npu-a3", nightly=True)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -13,7 +13,7 @@ from sglang.test.ascend.test_ascend_utils import (
|
||||
from sglang.test.ci.ci_register import register_npu_ci
|
||||
from sglang.test.test_utils import CustomTestCase
|
||||
|
||||
register_npu_ci(est_time=400, suite="stage-b-test-4-npu-a3", nightly=False)
|
||||
register_npu_ci(est_time=400, suite="base-b-test-4-npu-a3")
|
||||
register_npu_ci(est_time=400, suite="nightly-4-npu-a3", nightly=True)
|
||||
|
||||
|
||||
|
||||
@@ -14,7 +14,7 @@ from sglang.test.test_utils import (
|
||||
popen_launch_server,
|
||||
)
|
||||
|
||||
register_npu_ci(est_time=400, suite="stage-b-test-2-npu-a3", nightly=False)
|
||||
register_npu_ci(est_time=400, suite="base-b-test-2-npu-a3")
|
||||
register_npu_ci(est_time=400, suite="nightly-2-npu-a3", nightly=True)
|
||||
|
||||
TEST_MODEL_MATRIX = {
|
||||
|
||||
@@ -13,7 +13,7 @@ from sglang.test.test_utils import (
|
||||
popen_launch_server,
|
||||
)
|
||||
|
||||
register_npu_ci(est_time=400, suite="stage-b-test-4-npu-a3", nightly=False)
|
||||
register_npu_ci(est_time=400, suite="base-b-test-4-npu-a3")
|
||||
register_npu_ci(est_time=400, suite="nightly-4-npu-a3", nightly=True)
|
||||
|
||||
TEST_MODEL_MATRIX = {
|
||||
|
||||
@@ -13,7 +13,7 @@ from sglang.test.test_utils import (
|
||||
popen_launch_server,
|
||||
)
|
||||
|
||||
register_npu_ci(est_time=400, suite="stage-b-test-1-npu-a3", nightly=False)
|
||||
register_npu_ci(est_time=400, suite="base-b-test-1-npu-a3")
|
||||
register_npu_ci(est_time=400, suite="nightly-1-npu-a3", nightly=True)
|
||||
|
||||
TEST_MODEL_MATRIX = {
|
||||
|
||||
@@ -13,7 +13,7 @@ from sglang.test.test_utils import (
|
||||
popen_launch_server,
|
||||
)
|
||||
|
||||
register_npu_ci(est_time=400, suite="stage-b-test-2-npu-a3", nightly=False)
|
||||
register_npu_ci(est_time=400, suite="base-b-test-2-npu-a3")
|
||||
register_npu_ci(est_time=400, suite="nightly-2-npu-a3", nightly=True)
|
||||
|
||||
TEST_MODEL_MATRIX = {
|
||||
|
||||
@@ -14,7 +14,7 @@ from sglang.test.test_utils import (
|
||||
popen_launch_server,
|
||||
)
|
||||
|
||||
register_npu_ci(est_time=400, suite="stage-b-test-2-npu-a3", nightly=False)
|
||||
register_npu_ci(est_time=400, suite="base-b-test-2-npu-a3")
|
||||
register_npu_ci(est_time=400, suite="nightly-2-npu-a3", nightly=True)
|
||||
|
||||
TEST_MODEL_MATRIX = {
|
||||
|
||||
@@ -12,7 +12,7 @@ from sglang.test.test_utils import (
|
||||
popen_launch_server,
|
||||
)
|
||||
|
||||
register_npu_ci(est_time=400, suite="stage-b-test-4-npu-a3", nightly=False)
|
||||
register_npu_ci(est_time=400, suite="base-b-test-4-npu-a3")
|
||||
register_npu_ci(est_time=400, suite="nightly-4-npu-a3", nightly=True)
|
||||
|
||||
TEST_MODEL_MATRIX = {
|
||||
|
||||
+1
-1
@@ -17,7 +17,7 @@ from sglang.test.test_utils import (
|
||||
popen_launch_server,
|
||||
)
|
||||
|
||||
register_npu_ci(est_time=200, suite="stage-b-test-1-npu-a3", nightly=False)
|
||||
register_npu_ci(est_time=200, suite="base-b-test-1-npu-a3")
|
||||
register_npu_ci(est_time=200, suite="nightly-1-npu-a3", nightly=True)
|
||||
|
||||
|
||||
|
||||
@@ -9,7 +9,7 @@ from sglang.test.ascend.test_ascend_utils import (
|
||||
from sglang.test.ci.ci_register import register_npu_ci
|
||||
from sglang.test.test_utils import CustomTestCase
|
||||
|
||||
register_npu_ci(est_time=400, suite="stage-b-test-1-npu-a3", nightly=False)
|
||||
register_npu_ci(est_time=400, suite="base-b-test-1-npu-a3")
|
||||
register_npu_ci(est_time=400, suite="nightly-1-npu-a3", nightly=True)
|
||||
|
||||
|
||||
|
||||
+1
-5
@@ -17,11 +17,7 @@ from sglang.test.test_utils import (
|
||||
popen_launch_server,
|
||||
)
|
||||
|
||||
register_npu_ci(
|
||||
est_time=400,
|
||||
suite="stage-b-test-4-npu-a3",
|
||||
nightly=False,
|
||||
)
|
||||
register_npu_ci(est_time=400, suite="base-b-test-4-npu-a3")
|
||||
register_npu_ci(
|
||||
est_time=400,
|
||||
suite="nightly-4-npu-a3",
|
||||
|
||||
+1
-1
@@ -16,7 +16,7 @@ from sglang.test.test_utils import (
|
||||
popen_launch_server,
|
||||
)
|
||||
|
||||
register_npu_ci(est_time=400, suite="stage-b-test-16-npu-a3", nightly=False)
|
||||
register_npu_ci(est_time=400, suite="base-b-test-16-npu-a3")
|
||||
register_npu_ci(est_time=400, suite="nightly-16-npu-a3", nightly=True)
|
||||
|
||||
MODEL_PATH = DEEPSEEK_R1_0528_W4A8_PER_CHANNEL_WEIGHTS_PATH
|
||||
|
||||
+1
-1
@@ -16,7 +16,7 @@ from sglang.test.test_utils import (
|
||||
popen_launch_server,
|
||||
)
|
||||
|
||||
register_npu_ci(est_time=400, suite="stage-b-test-16-npu-a3", nightly=False)
|
||||
register_npu_ci(est_time=400, suite="base-b-test-16-npu-a3")
|
||||
register_npu_ci(est_time=400, suite="nightly-16-npu-a3", nightly=True)
|
||||
|
||||
|
||||
|
||||
+1
-1
@@ -16,7 +16,7 @@ from sglang.test.test_utils import (
|
||||
popen_launch_server,
|
||||
)
|
||||
|
||||
register_npu_ci(est_time=400, suite="stage-b-test-4-npu-a3", nightly=False)
|
||||
register_npu_ci(est_time=400, suite="base-b-test-4-npu-a3")
|
||||
register_npu_ci(est_time=400, suite="nightly-4-npu-a3", nightly=True)
|
||||
|
||||
_ASCEND_BACKEND = "ascend"
|
||||
|
||||
+1
-1
@@ -20,7 +20,7 @@ from sglang.test.test_utils import (
|
||||
popen_launch_server,
|
||||
)
|
||||
|
||||
register_npu_ci(est_time=300, suite="stage-b-test-4-npu-a3", nightly=False)
|
||||
register_npu_ci(est_time=300, suite="base-b-test-4-npu-a3")
|
||||
register_npu_ci(est_time=300, suite="nightly-4-npu-a3", nightly=True)
|
||||
|
||||
|
||||
|
||||
+1
-6
@@ -8,12 +8,7 @@ from sglang.test.ascend.e2e.test_npu_performance_utils import (
|
||||
)
|
||||
from sglang.test.ci.ci_register import register_npu_ci
|
||||
|
||||
register_npu_ci(
|
||||
est_time=3600,
|
||||
suite="",
|
||||
nightly=True,
|
||||
disabled="performance testcase",
|
||||
)
|
||||
register_npu_ci(est_time=3600, suite="base-c-test-perf-16-npu-a3")
|
||||
|
||||
# Environment variables for DSV4-Flash single-node PD-mix deployment.
|
||||
DEEPSEEK_V4_FLASH_W8A8_8P_ENVS = {
|
||||
|
||||
+1
-6
@@ -10,12 +10,7 @@ from sglang.test.ascend.e2e.test_npu_performance_utils import (
|
||||
)
|
||||
from sglang.test.ci.ci_register import register_npu_ci
|
||||
|
||||
register_npu_ci(
|
||||
est_time=1800,
|
||||
suite="full-16-npu-a3",
|
||||
nightly=True,
|
||||
disabled="Currently it is executed by the npu performance workflow.",
|
||||
)
|
||||
register_npu_ci(est_time=1800, suite="base-c-test-perf-16-npu-a3")
|
||||
|
||||
KIMI_K2_6_ENVS = {
|
||||
"PYTORCH_NPU_ALLOC_CONF": "expandable_segments:True",
|
||||
|
||||
+1
-6
@@ -9,12 +9,7 @@ from sglang.test.ascend.e2e.test_npu_performance_utils import (
|
||||
)
|
||||
from sglang.test.ci.ci_register import register_npu_ci
|
||||
|
||||
register_npu_ci(
|
||||
est_time=3600,
|
||||
suite="npu-performance",
|
||||
nightly=True,
|
||||
disabled="performance testcase",
|
||||
)
|
||||
register_npu_ci(est_time=3600, suite="base-c-test-perf-8-npu-a3")
|
||||
|
||||
MINIMAX_M2_5_W8A8_4P_IN64K_OUT1K_PREFIX90_ENVS = {
|
||||
"PYTORCH_NPU_ALLOC_CONF": "expandable_segments:True",
|
||||
|
||||
+1
-6
@@ -9,12 +9,7 @@ from sglang.test.ascend.e2e.test_npu_performance_utils import (
|
||||
)
|
||||
from sglang.test.ci.ci_register import register_npu_ci
|
||||
|
||||
register_npu_ci(
|
||||
est_time=3600,
|
||||
suite="",
|
||||
nightly=True,
|
||||
disabled="performance testcase",
|
||||
)
|
||||
register_npu_ci(est_time=3600, suite="base-c-test-perf-2-npu-a3")
|
||||
|
||||
QWEN3_8B_ENVS = {
|
||||
"SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT": "600",
|
||||
|
||||
+1
-6
@@ -9,12 +9,7 @@ from sglang.test.ascend.e2e.test_npu_performance_utils import (
|
||||
)
|
||||
from sglang.test.ci.ci_register import register_npu_ci
|
||||
|
||||
register_npu_ci(
|
||||
est_time=3600,
|
||||
suite="",
|
||||
nightly=True,
|
||||
disabled="performance testcase",
|
||||
)
|
||||
register_npu_ci(est_time=3600, suite="base-c-test-perf-16-npu-a3")
|
||||
|
||||
QWEN3_235B_ENVS = {
|
||||
"PYTORCH_NPU_ALLOC_CONF": "expandable_segments:True",
|
||||
|
||||
+1
-6
@@ -9,12 +9,7 @@ from sglang.test.ascend.e2e.test_npu_performance_utils import (
|
||||
)
|
||||
from sglang.test.ci.ci_register import register_npu_ci
|
||||
|
||||
register_npu_ci(
|
||||
est_time=3600,
|
||||
suite="",
|
||||
nightly=True,
|
||||
disabled="performance testcase",
|
||||
)
|
||||
register_npu_ci(est_time=3600, suite="base-c-test-perf-2-npu-a3")
|
||||
|
||||
QWEN3_30B_A3B_ENVS = {
|
||||
"ASCEND_LAUNCH_BLOCKING": "0",
|
||||
|
||||
+1
-6
@@ -9,12 +9,7 @@ from sglang.test.ascend.e2e.test_npu_performance_utils import (
|
||||
)
|
||||
from sglang.test.ci.ci_register import register_npu_ci
|
||||
|
||||
register_npu_ci(
|
||||
est_time=3600,
|
||||
suite="",
|
||||
nightly=True,
|
||||
disabled="performance testcase",
|
||||
)
|
||||
register_npu_ci(est_time=3600, suite="base-c-test-perf-4-npu-a3")
|
||||
|
||||
QWEN3_32B_ENVS = {
|
||||
"SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT": "600",
|
||||
|
||||
+1
-6
@@ -8,12 +8,7 @@ from sglang.test.ascend.e2e.test_npu_performance_utils import (
|
||||
)
|
||||
from sglang.test.ci.ci_register import register_npu_ci
|
||||
|
||||
register_npu_ci(
|
||||
est_time=3600,
|
||||
suite="nightly-16-npu-a3",
|
||||
nightly=True,
|
||||
disabled="performance testcase",
|
||||
)
|
||||
register_npu_ci(est_time=3600, suite="base-c-test-perf-16-npu-a3")
|
||||
|
||||
QWEN3_5_397B_A17B_ENVS = {
|
||||
"PYTORCH_NPU_ALLOC_CONF": "expandable_segments:True",
|
||||
|
||||
+1
-6
@@ -8,12 +8,7 @@ from sglang.test.ascend.e2e.test_npu_performance_utils import (
|
||||
)
|
||||
from sglang.test.ci.ci_register import register_npu_ci
|
||||
|
||||
register_npu_ci(
|
||||
est_time=3600,
|
||||
suite="full-2-npu-a3",
|
||||
nightly=True,
|
||||
disabled="performance case",
|
||||
)
|
||||
register_npu_ci(est_time=3600, suite="base-c-test-perf-2-npu-a3")
|
||||
|
||||
QWEN3_6_27B_1024_ENVS = {
|
||||
"PYTORCH_NPU_ALLOC_CONF": "expandable_segments:True",
|
||||
|
||||
+1
-6
@@ -8,12 +8,7 @@ from sglang.test.ascend.e2e.test_npu_performance_utils import (
|
||||
)
|
||||
from sglang.test.ci.ci_register import register_npu_ci
|
||||
|
||||
register_npu_ci(
|
||||
est_time=3600,
|
||||
suite="",
|
||||
nightly=True,
|
||||
disabled="performance testcase",
|
||||
)
|
||||
register_npu_ci(est_time=3600, suite="base-c-test-perf-2-npu-a3")
|
||||
|
||||
QWEN3_6_27B_64K_1K_ENVS = {
|
||||
"STREAMS_PER_DEVICE": "32",
|
||||
|
||||
+1
-6
@@ -8,12 +8,7 @@ from sglang.test.ascend.e2e.test_npu_performance_utils import (
|
||||
)
|
||||
from sglang.test.ci.ci_register import register_npu_ci
|
||||
|
||||
register_npu_ci(
|
||||
est_time=3600,
|
||||
suite="",
|
||||
nightly=True,
|
||||
disabled="performance testcase",
|
||||
)
|
||||
register_npu_ci(est_time=3600, suite="base-c-test-perf-2-npu-a3")
|
||||
|
||||
QWEN3_6_35B_A3B_64K_PREFIX_ENVS = {
|
||||
"PYTORCH_NPU_ALLOC_CONF": "expandable_segments:True",
|
||||
|
||||
+1
-6
@@ -7,12 +7,7 @@ from sglang.test.ascend.e2e.test_npu_performance_utils import (
|
||||
)
|
||||
from sglang.test.ci.ci_register import register_npu_ci
|
||||
|
||||
register_npu_ci(
|
||||
est_time=3600,
|
||||
suite="",
|
||||
nightly=True,
|
||||
disabled="performance testcase",
|
||||
)
|
||||
register_npu_ci(est_time=3600, suite="base-c-test-perf-4-npu-a3")
|
||||
|
||||
QWEN3_NEXT_80B_A3B_ENVS = {
|
||||
"PYTORCH_NPU_ALLOC_CONF": "expandable_segments:True",
|
||||
|
||||
@@ -11,7 +11,7 @@ from sglang.srt.layers.attention import vision
|
||||
from sglang.test.ci.ci_register import register_cpu_ci, register_npu_ci
|
||||
|
||||
register_cpu_ci(est_time=2, suite="base-a-test-cpu")
|
||||
register_npu_ci(est_time=2, suite="stage-b-test-1-npu-a3")
|
||||
register_npu_ci(est_time=2, suite="base-b-test-1-npu-a3")
|
||||
register_npu_ci(est_time=2, suite="nightly-1-npu-a3", nightly=True)
|
||||
|
||||
|
||||
|
||||
@@ -12,7 +12,7 @@ import torch
|
||||
|
||||
from sglang.test.ci.ci_register import register_npu_ci
|
||||
|
||||
register_npu_ci(est_time=5, suite="stage-a-unit-test-npu")
|
||||
register_npu_ci(est_time=5, suite="base-a-test-1-npu-a2")
|
||||
|
||||
# Mock NPU-only modules before importing the source module.
|
||||
for _ in (
|
||||
|
||||
@@ -12,7 +12,7 @@ import torch
|
||||
|
||||
from sglang.test.ci.ci_register import register_npu_ci
|
||||
|
||||
register_npu_ci(est_time=4, suite="stage-a-unit-test-npu")
|
||||
register_npu_ci(est_time=4, suite="base-a-test-1-npu-a2")
|
||||
|
||||
for mod in (
|
||||
"torch_npu",
|
||||
|
||||
+14
-7
@@ -93,13 +93,20 @@ PER_COMMIT_SUITES = {
|
||||
"extra-b-test-8-gpu-h200",
|
||||
],
|
||||
HWBackend.NPU: [
|
||||
"base-a-test-1-gpu-small",
|
||||
"stage-a-unit-test-npu",
|
||||
"stage-b-test-1-npu-a3",
|
||||
"stage-b-test-2-npu-a3",
|
||||
"stage-b-test-4-npu-a3",
|
||||
"stage-b-test-8-npu-a3",
|
||||
"stage-b-test-16-npu-a3",
|
||||
"base-a-test-1-npu-a2",
|
||||
"base-b-test-1-npu-a3",
|
||||
"base-b-test-2-npu-a3",
|
||||
"base-b-test-4-npu-a3",
|
||||
"base-b-test-8-npu-a3",
|
||||
"base-b-test-16-npu-a3",
|
||||
"base-c-test-acc-2-npu-a3",
|
||||
"base-c-test-acc-4-npu-a3",
|
||||
"base-c-test-acc-8-npu-a3",
|
||||
"base-c-test-acc-16-npu-a3",
|
||||
"base-c-test-perf-2-npu-a3",
|
||||
"base-c-test-perf-4-npu-a3",
|
||||
"base-c-test-perf-8-npu-a3",
|
||||
"base-c-test-perf-16-npu-a3",
|
||||
],
|
||||
HWBackend.XPU: [
|
||||
"stage-a-test-1-gpu-xpu",
|
||||
|
||||
Reference in New Issue
Block a user