[NPU] Improve the execution efficiency and maintainability of pr‑test‑npu (#33724)
Co-authored-by: Even Zhou <even.y.zhou@outlook.com> Co-authored-by: sglang-npu-bot <sglangnpu@163.com>
This commit is contained in:
co-authored by
Even Zhou
sglang-npu-bot
parent
e732c0a9dc
commit
dd5d82bead
@@ -0,0 +1,213 @@
|
|||||||
|
name: PR Test Stage for NPU
|
||||||
|
# Reusable workflow for one CUDA test stage. Caller pr-test-npu.yml forwards
|
||||||
|
|
||||||
|
on:
|
||||||
|
workflow_call:
|
||||||
|
inputs:
|
||||||
|
self_name:
|
||||||
|
description: 'Caller job key; used for partitions[suite] lookup.'
|
||||||
|
type: string
|
||||||
|
required: true
|
||||||
|
runner_config:
|
||||||
|
description: 'Key in scripts/ci/runner_configs.yml. Resolves install script, artifact version, install timeout, runs-on label, and rdma_devices.'
|
||||||
|
type: string
|
||||||
|
required: true
|
||||||
|
image:
|
||||||
|
description: 'The container image for NPU test environment.'
|
||||||
|
type: string
|
||||||
|
required: true
|
||||||
|
npu_device_type:
|
||||||
|
description: 'NPU device type, e.g. 910b, a3.'
|
||||||
|
type: string
|
||||||
|
default: 'a3'
|
||||||
|
run_timeout_minutes:
|
||||||
|
description: 'timeout-minutes for the Run test step. Required so compute_partitions.py can read it from pr-test.yml without a duplicated default constant.'
|
||||||
|
type: string
|
||||||
|
required: true
|
||||||
|
timeout_per_file:
|
||||||
|
description: 'run_suite.py --timeout-per-file value (empty = unset).'
|
||||||
|
type: string
|
||||||
|
default: ''
|
||||||
|
partitions:
|
||||||
|
description: 'partitions config, e.g. {"size":1,"arr":[0]}'
|
||||||
|
type: string
|
||||||
|
default: '{"size":1,"arr":[0]}'
|
||||||
|
ref:
|
||||||
|
description: 'Git ref (branch, tag, or SHA) to test. If not provided, uses the default branch.'
|
||||||
|
type: string
|
||||||
|
default: ''
|
||||||
|
skip_pr_test_health_check:
|
||||||
|
description: 'Git ref (branch, tag, or SHA) to test. If not provided, uses the default branch.'
|
||||||
|
type: string
|
||||||
|
default: 'false'
|
||||||
|
github-token:
|
||||||
|
description: 'GitHub token for API calls'
|
||||||
|
type: string
|
||||||
|
default: ${{ github.token }}
|
||||||
|
|
||||||
|
env:
|
||||||
|
SKIP_PR_TEST_HEALTH_CHECK: ${{ inputs.skip_pr_test_health_check }}
|
||||||
|
SGLANG_USE_MODELSCOPE: true
|
||||||
|
SGLANG_IS_IN_CI: true
|
||||||
|
HF_ENDPOINT: https://hf-mirror.com
|
||||||
|
TORCH_EXTENSIONS_DIR: /tmp/torch_extensions
|
||||||
|
PYTORCH_NPU_ALLOC_CONF: "expandable_segments:True"
|
||||||
|
STREAMS_PER_DEVICE: 32
|
||||||
|
|
||||||
|
jobs:
|
||||||
|
run:
|
||||||
|
runs-on: ${{ inputs.runner_config }}
|
||||||
|
container:
|
||||||
|
image: ${{ inputs.image }}
|
||||||
|
timeout-minutes: 240
|
||||||
|
strategy:
|
||||||
|
fail-fast: false
|
||||||
|
matrix:
|
||||||
|
partition: ${{ fromJson(inputs.partitions).arr }}
|
||||||
|
steps:
|
||||||
|
- name: Checkout code
|
||||||
|
uses: actions/checkout@v4
|
||||||
|
with:
|
||||||
|
ref: ${{ inputs.ref || github.ref }}
|
||||||
|
|
||||||
|
- name: Mark repository safe
|
||||||
|
run: |
|
||||||
|
git config --system --add safe.directory ${GITHUB_WORKSPACE}
|
||||||
|
|
||||||
|
- name: Check PR test health
|
||||||
|
uses: actions/github-script@v8
|
||||||
|
env:
|
||||||
|
SKIP_PR_TEST_HEALTH_CHECK: ${{ env.SKIP_PR_TEST_HEALTH_CHECK }}
|
||||||
|
with:
|
||||||
|
github-token: ${{ inputs.github-token || github.token }}
|
||||||
|
script: |
|
||||||
|
core.notice(`[health-check] START — event=${context.eventName}, runId=${context.runId}`);
|
||||||
|
|
||||||
|
// Skip when explicitly requested via env var (e.g. release branch cut)
|
||||||
|
if (process.env.SKIP_PR_TEST_HEALTH_CHECK === 'true') {
|
||||||
|
core.notice('[health-check] SKIP: SKIP_PR_TEST_HEALTH_CHECK=true');
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Skip for scheduled runs — they should collect all failures, not fast-fail
|
||||||
|
if (context.eventName === 'schedule') {
|
||||||
|
core.notice('[health-check] SKIP: scheduled run');
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Check lint status from the separate Lint workflow (lint.yml).
|
||||||
|
// listJobsForWorkflowRun only sees jobs within the SAME run, so we use
|
||||||
|
// checks.listForRef which queries by commit SHA across ALL workflows.
|
||||||
|
const ref = context.payload.pull_request?.head?.sha || context.sha;
|
||||||
|
core.info(`[health-check] Checking lint for ref=${ref}`);
|
||||||
|
const { data } = await github.rest.checks.listForRef({
|
||||||
|
owner: context.repo.owner,
|
||||||
|
repo: context.repo.repo,
|
||||||
|
ref: ref,
|
||||||
|
check_name: 'lint',
|
||||||
|
});
|
||||||
|
const lintRun = data.check_runs.find(
|
||||||
|
cr => cr.app?.slug === 'github-actions'
|
||||||
|
);
|
||||||
|
core.info(`[health-check] Lint check: status=${lintRun?.status}, conclusion=${lintRun?.conclusion}`);
|
||||||
|
if (lintRun?.status === 'completed' && lintRun?.conclusion === 'failure') {
|
||||||
|
core.setFailed('Fast-fail: lint check failed');
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Skip the jobs-failed check when the PR carries the bypass-fastfail label.
|
||||||
|
// Lint check above still runs.
|
||||||
|
let labels = [];
|
||||||
|
if (context.payload.pull_request?.labels) {
|
||||||
|
labels = context.payload.pull_request.labels.map(l => l.name);
|
||||||
|
} else {
|
||||||
|
const { data: prs } = await github.rest.repos.listPullRequestsAssociatedWithCommit({
|
||||||
|
owner: context.repo.owner,
|
||||||
|
repo: context.repo.repo,
|
||||||
|
commit_sha: ref,
|
||||||
|
});
|
||||||
|
if (prs.length > 0) {
|
||||||
|
labels = prs[0].labels.map(l => l.name);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
core.info(`[health-check] PR labels: [${labels.join(', ')}]`);
|
||||||
|
if (labels.includes('bypass-fastfail')) {
|
||||||
|
core.notice('[health-check] SKIP jobs-failed check: bypass-fastfail label present');
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
const jobs = await github.paginate(github.rest.actions.listJobsForWorkflowRun, {
|
||||||
|
owner: context.repo.owner,
|
||||||
|
repo: context.repo.repo,
|
||||||
|
run_id: context.runId,
|
||||||
|
per_page: 100,
|
||||||
|
});
|
||||||
|
core.info(`[health-check] Total jobs in run: ${jobs.length}`);
|
||||||
|
const failedJobs = jobs.filter(j => j.status === 'completed' && j.conclusion === 'failure');
|
||||||
|
core.info(`[health-check] Failed jobs (before filtering): ${failedJobs.map(j => `${j.name}(${j.conclusion})`).join(', ') || 'none'}`);
|
||||||
|
|
||||||
|
// Find jobs that failed from a real error, not from fast-fail cascade
|
||||||
|
const rootCauseFailures = jobs.filter(j => {
|
||||||
|
if (j.status !== 'completed' || j.conclusion !== 'failure') return false;
|
||||||
|
// h20 runners are flaky (dirty GPU state from prior runs); their failures
|
||||||
|
// should not cascade fast-fail to other stages. j.name shape from
|
||||||
|
// listJobsForWorkflowRun: "<job-key>" + optional " / <reusable-job>"
|
||||||
|
// + optional " (<matrix>)". Split off the base job key before exact
|
||||||
|
// match so we cover both inline + reusable forms without confusing
|
||||||
|
// 'h20' with the 'h200' prefix.
|
||||||
|
const baseName = j.name.split(/[ /]/)[0];
|
||||||
|
if (baseName === 'base-c-test-8-gpu-h20') {
|
||||||
|
core.info(`[health-check] Filtered out h20 job: ${j.name}`);
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
// If the failing step is the health check, it's a cascade — skip it
|
||||||
|
const failedStep = (j.steps || []).find(s => s.conclusion === 'failure');
|
||||||
|
if (failedStep && (failedStep.name.includes('check-pr-test-health') || failedStep.name.includes('Check PR test health'))) {
|
||||||
|
core.info(`[health-check] Filtered out cascade failure: ${j.name} (failed step: ${failedStep.name})`);
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
return true;
|
||||||
|
});
|
||||||
|
core.info(`[health-check] Root cause failures (after filtering): ${rootCauseFailures.map(j => j.name).join(', ') || 'none'}`);
|
||||||
|
|
||||||
|
if (rootCauseFailures.length > 0) {
|
||||||
|
core.setFailed(`Fast-fail: skipping — root cause job(s): ${rootCauseFailures.map(j => j.name).join(', ')}`);
|
||||||
|
} else {
|
||||||
|
core.notice('[health-check] PASS: no root cause failures detected');
|
||||||
|
}
|
||||||
|
|
||||||
|
- name: Install dependencies
|
||||||
|
env:
|
||||||
|
TORCH_CACHE_URL: "http://cache-service.nginx-pypi-cache.svc.cluster.local/whl/cpu"
|
||||||
|
PYPI_CACHE_URL: "http://cache-service.nginx-pypi-cache.svc.cluster.local/pypi/simple"
|
||||||
|
UV_INDEX_URL: "http://cache-service.nginx-pypi-cache.svc.cluster.local/pypi/simple"
|
||||||
|
GITHUB_PROXY_URL: "https://gh-proxy.test.osinfra.cn/"
|
||||||
|
RUSTUP_CACHE_URL: "http://cache-service.nginx-pypi-cache.svc.cluster.local:8082"
|
||||||
|
run: |
|
||||||
|
# speed up by using infra cache services
|
||||||
|
CACHING_URL="cache-service.nginx-pypi-cache.svc.cluster.local"
|
||||||
|
sed -Ei "s@(ports|archive).ubuntu.com@${CACHING_URL}:8081@g" /etc/apt/sources.list
|
||||||
|
pip config set global.index-url http://${CACHING_URL}/pypi/simple
|
||||||
|
pip config set global.trusted-host "${CACHING_URL}"
|
||||||
|
|
||||||
|
bash scripts/ci/npu/npu_ci_install_dependency.sh ${{ inputs.npu_device_type }}
|
||||||
|
# copy required file from our daily cache
|
||||||
|
cp ~/.cache/modelscope/hub/datasets/otavia/ShareGPT_Vicuna_unfiltered/ShareGPT_V3_unfiltered_cleaned_split.json /tmp
|
||||||
|
# copy gsm8k dataset
|
||||||
|
cp ~/.cache/modelscope/hub/datasets/tmp/test.jsonl /tmp
|
||||||
|
|
||||||
|
# install sglang_router
|
||||||
|
apt-get install -y libssl-dev
|
||||||
|
pip install sglang_router
|
||||||
|
|
||||||
|
- name: Run test
|
||||||
|
timeout-minutes: ${{ fromJson(inputs.run_timeout_minutes) }}
|
||||||
|
env:
|
||||||
|
CONTINUE_ON_ERROR_FLAG: ${{ inputs.continue_on_error == 'true' && '--continue-on-error' || '' }}
|
||||||
|
run: |
|
||||||
|
cd test
|
||||||
|
python3 run_suite.py --hw npu --suite ${{ inputs.self_name }} \
|
||||||
|
--auto-partition-id ${{ matrix.partition }} \
|
||||||
|
--auto-partition-size ${{ fromJson(inputs.partitions).size }} \
|
||||||
|
${{ inputs.timeout_per_file && format('--timeout-per-file {0}', inputs.timeout_per_file) || '' }} \
|
||||||
|
$CONTINUE_ON_ERROR_FLAG
|
||||||
@@ -0,0 +1,295 @@
|
|||||||
|
name: 'Single Node Template for E2E performance and accuracy tests'
|
||||||
|
|
||||||
|
on:
|
||||||
|
workflow_call:
|
||||||
|
inputs:
|
||||||
|
runner:
|
||||||
|
required: true
|
||||||
|
type: string
|
||||||
|
default: linux-aarch64-a3-16
|
||||||
|
test_type:
|
||||||
|
required: true
|
||||||
|
type: string
|
||||||
|
default: perf
|
||||||
|
description: perf or accuracy
|
||||||
|
test_suite:
|
||||||
|
required: true
|
||||||
|
type: string
|
||||||
|
default: ''
|
||||||
|
description: name of test suite to run via run_suite.py (mutually exclusive with test_case)
|
||||||
|
image:
|
||||||
|
required: true
|
||||||
|
type: string
|
||||||
|
description: image for pods
|
||||||
|
default: "swr.cn-southwest-2.myhuaweicloud.com/base_image/dockerhub/lmsysorg/sglang:main-cann9.0.0-a3"
|
||||||
|
install_sglang_deps:
|
||||||
|
required: false
|
||||||
|
type: boolean
|
||||||
|
default: false
|
||||||
|
description: install sglang dependencies (e.g. PyTorch, CANN packages) when using source installation
|
||||||
|
device_type_for_deps:
|
||||||
|
required: false
|
||||||
|
type: string
|
||||||
|
default: 'a3'
|
||||||
|
description: device type for dependency installation (a3 or 910b)
|
||||||
|
transformers_version:
|
||||||
|
required: false
|
||||||
|
type: string
|
||||||
|
default: ""
|
||||||
|
description: "The transformers version number for running sglang. Use default version in image if keep empty."
|
||||||
|
skip_pr_test_health_check:
|
||||||
|
description: 'Git ref (branch, tag, or SHA) to test. If not provided, uses the default branch.'
|
||||||
|
type: string
|
||||||
|
default: 'false'
|
||||||
|
github-token:
|
||||||
|
description: 'GitHub token for API calls'
|
||||||
|
type: string
|
||||||
|
default: ${{ github.token }}
|
||||||
|
|
||||||
|
env:
|
||||||
|
SKIP_PR_TEST_HEALTH_CHECK: ${{ inputs.skip_pr_test_health_check }}
|
||||||
|
|
||||||
|
concurrency:
|
||||||
|
group: ascend-nightly-e2e-singlenode-${{ github.workflow_ref }}-${{ github.ref }}-${{ inputs.test_suite }}
|
||||||
|
cancel-in-progress: true
|
||||||
|
|
||||||
|
jobs:
|
||||||
|
e2e:
|
||||||
|
name: ${{ inputs.test_suite }}
|
||||||
|
runs-on: ${{ inputs.runner }}
|
||||||
|
container:
|
||||||
|
image: ${{ inputs.image }}
|
||||||
|
steps:
|
||||||
|
- name: Checkout code
|
||||||
|
uses: actions/checkout@v4
|
||||||
|
|
||||||
|
- name: Check PR test health
|
||||||
|
uses: actions/github-script@v8
|
||||||
|
env:
|
||||||
|
SKIP_PR_TEST_HEALTH_CHECK: ${{ env.SKIP_PR_TEST_HEALTH_CHECK }}
|
||||||
|
with:
|
||||||
|
github-token: ${{ inputs.github-token || github.token }}
|
||||||
|
script: |
|
||||||
|
core.notice(`[health-check] START — event=${context.eventName}, runId=${context.runId}`);
|
||||||
|
|
||||||
|
// Skip when explicitly requested via env var (e.g. release branch cut)
|
||||||
|
if (process.env.SKIP_PR_TEST_HEALTH_CHECK === 'true') {
|
||||||
|
core.notice('[health-check] SKIP: SKIP_PR_TEST_HEALTH_CHECK=true');
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Skip for scheduled runs — they should collect all failures, not fast-fail
|
||||||
|
if (context.eventName === 'schedule') {
|
||||||
|
core.notice('[health-check] SKIP: scheduled run');
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Check lint status from the separate Lint workflow (lint.yml).
|
||||||
|
// listJobsForWorkflowRun only sees jobs within the SAME run, so we use
|
||||||
|
// checks.listForRef which queries by commit SHA across ALL workflows.
|
||||||
|
const ref = context.payload.pull_request?.head?.sha || context.sha;
|
||||||
|
core.info(`[health-check] Checking lint for ref=${ref}`);
|
||||||
|
const { data } = await github.rest.checks.listForRef({
|
||||||
|
owner: context.repo.owner,
|
||||||
|
repo: context.repo.repo,
|
||||||
|
ref: ref,
|
||||||
|
check_name: 'lint',
|
||||||
|
});
|
||||||
|
const lintRun = data.check_runs.find(
|
||||||
|
cr => cr.app?.slug === 'github-actions'
|
||||||
|
);
|
||||||
|
core.info(`[health-check] Lint check: status=${lintRun?.status}, conclusion=${lintRun?.conclusion}`);
|
||||||
|
if (lintRun?.status === 'completed' && lintRun?.conclusion === 'failure') {
|
||||||
|
core.setFailed('Fast-fail: lint check failed');
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Skip the jobs-failed check when the PR carries the bypass-fastfail label.
|
||||||
|
// Lint check above still runs.
|
||||||
|
let labels = [];
|
||||||
|
if (context.payload.pull_request?.labels) {
|
||||||
|
labels = context.payload.pull_request.labels.map(l => l.name);
|
||||||
|
} else {
|
||||||
|
const { data: prs } = await github.rest.repos.listPullRequestsAssociatedWithCommit({
|
||||||
|
owner: context.repo.owner,
|
||||||
|
repo: context.repo.repo,
|
||||||
|
commit_sha: ref,
|
||||||
|
});
|
||||||
|
if (prs.length > 0) {
|
||||||
|
labels = prs[0].labels.map(l => l.name);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
core.info(`[health-check] PR labels: [${labels.join(', ')}]`);
|
||||||
|
if (labels.includes('bypass-fastfail')) {
|
||||||
|
core.notice('[health-check] SKIP jobs-failed check: bypass-fastfail label present');
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
const jobs = await github.paginate(github.rest.actions.listJobsForWorkflowRun, {
|
||||||
|
owner: context.repo.owner,
|
||||||
|
repo: context.repo.repo,
|
||||||
|
run_id: context.runId,
|
||||||
|
per_page: 100,
|
||||||
|
});
|
||||||
|
core.info(`[health-check] Total jobs in run: ${jobs.length}`);
|
||||||
|
const failedJobs = jobs.filter(j => j.status === 'completed' && j.conclusion === 'failure');
|
||||||
|
core.info(`[health-check] Failed jobs (before filtering): ${failedJobs.map(j => `${j.name}(${j.conclusion})`).join(', ') || 'none'}`);
|
||||||
|
|
||||||
|
// Find jobs that failed from a real error, not from fast-fail cascade
|
||||||
|
const rootCauseFailures = jobs.filter(j => {
|
||||||
|
if (j.status !== 'completed' || j.conclusion !== 'failure') return false;
|
||||||
|
// h20 runners are flaky (dirty GPU state from prior runs); their failures
|
||||||
|
// should not cascade fast-fail to other stages. j.name shape from
|
||||||
|
// listJobsForWorkflowRun: "<job-key>" + optional " / <reusable-job>"
|
||||||
|
// + optional " (<matrix>)". Split off the base job key before exact
|
||||||
|
// match so we cover both inline + reusable forms without confusing
|
||||||
|
// 'h20' with the 'h200' prefix.
|
||||||
|
const baseName = j.name.split(/[ /]/)[0];
|
||||||
|
if (baseName === 'base-c-test-8-gpu-h20') {
|
||||||
|
core.info(`[health-check] Filtered out h20 job: ${j.name}`);
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
// If the failing step is the health check, it's a cascade — skip it
|
||||||
|
const failedStep = (j.steps || []).find(s => s.conclusion === 'failure');
|
||||||
|
if (failedStep && (failedStep.name.includes('check-pr-test-health') || failedStep.name.includes('Check PR test health'))) {
|
||||||
|
core.info(`[health-check] Filtered out cascade failure: ${j.name} (failed step: ${failedStep.name})`);
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
return true;
|
||||||
|
});
|
||||||
|
core.info(`[health-check] Root cause failures (after filtering): ${rootCauseFailures.map(j => j.name).join(', ') || 'none'}`);
|
||||||
|
|
||||||
|
if (rootCauseFailures.length > 0) {
|
||||||
|
core.setFailed(`Fast-fail: skipping — root cause job(s): ${rootCauseFailures.map(j => j.name).join(', ')}`);
|
||||||
|
} else {
|
||||||
|
core.notice('[health-check] PASS: no root cause failures detected');
|
||||||
|
}
|
||||||
|
|
||||||
|
- name: Check npu info
|
||||||
|
run: |
|
||||||
|
npu-smi info
|
||||||
|
|
||||||
|
- name: Install sglang dependencies
|
||||||
|
if: ${{ inputs.install_sglang_deps == true }}
|
||||||
|
shell: bash
|
||||||
|
env:
|
||||||
|
TORCH_CACHE_URL: "http://cache-service.nginx-pypi-cache.svc.cluster.local/whl/cpu"
|
||||||
|
PYPI_CACHE_URL: "http://cache-service.nginx-pypi-cache.svc.cluster.local/pypi/simple"
|
||||||
|
UV_INDEX_URL: "http://cache-service.nginx-pypi-cache.svc.cluster.local/pypi/simple"
|
||||||
|
GITHUB_PROXY_URL: "https://gh-proxy.test.osinfra.cn/"
|
||||||
|
RUSTUP_CACHE_URL: "http://cache-service.nginx-pypi-cache.svc.cluster.local:8082"
|
||||||
|
run: |
|
||||||
|
CACHING_URL="cache-service.nginx-pypi-cache.svc.cluster.local"
|
||||||
|
sed -Ei "s@(ports|archive).ubuntu.com@${CACHING_URL}:8081@g" /etc/apt/sources.list
|
||||||
|
pip config set global.index-url http://${CACHING_URL}/pypi/simple
|
||||||
|
pip config set global.trusted-host "${CACHING_URL}"
|
||||||
|
bash scripts/ci/npu/npu_ci_install_dependency.sh ${{ inputs.device_type_for_deps }}
|
||||||
|
cp ~/.cache/modelscope/hub/datasets/otavia/ShareGPT_Vicuna_unfiltered/ShareGPT_V3_unfiltered_cleaned_split.json /tmp
|
||||||
|
curl -o /tmp/test.jsonl -L https://gh-proxy.test.osinfra.cn/https://raw.githubusercontent.com/openai/grade-school-math/master/grade_school_math/data/test.jsonl
|
||||||
|
|
||||||
|
- name: Run test
|
||||||
|
timeout-minutes: 300
|
||||||
|
env:
|
||||||
|
SGLANG_USE_MODELSCOPE: true
|
||||||
|
HF_ENDPOINT: https://hf-mirror.com
|
||||||
|
SGLANG_IS_IN_CI: true
|
||||||
|
TRANSFORMERS_VERBOSITY: "error"
|
||||||
|
GDN_ATTN_BACKEND_TRITON: 1
|
||||||
|
SGLANG_TEST_METRICS_OUTPUT: /root/.cache/tests/output/metrics/metrics
|
||||||
|
shell: bash
|
||||||
|
run: |
|
||||||
|
sglang_source_path=$(pwd)
|
||||||
|
echo "Source code path: ${sglang_source_path}"
|
||||||
|
ln -sf ${sglang_source_path} /root/sglang
|
||||||
|
|
||||||
|
# Determine test mode: suite (run_suite.py --suite) or single case file.
|
||||||
|
test_suite="${{ inputs.test_suite }}"
|
||||||
|
echo "Test mode: suite (${test_suite})"
|
||||||
|
tc_name="${test_suite}"
|
||||||
|
|
||||||
|
export TRANSFORMERS_VERSION_FOR_SGLANG="${{ inputs.transformers_version }}"
|
||||||
|
PYTHON_FOR_SGLANG="python"
|
||||||
|
PIP_FOR_SGLANG="pip"
|
||||||
|
if [ -n "${TRANSFORMERS_VERSION_FOR_SGLANG}" ];then
|
||||||
|
echo "===== Install transformers for sglang - Begin ====="
|
||||||
|
TRANSFORMERS_PKG_PATH_SOURCE=/root/.cache/.cache/transformers/${TRANSFORMERS_VERSION_FOR_SGLANG}
|
||||||
|
if [ ! -d "${TRANSFORMERS_PKG_PATH_SOURCE}" ]; then
|
||||||
|
echo "The dependent transformers package does not exist: ${TRANSFORMERS_PKG_PATH_SOURCE}."
|
||||||
|
echo "Install transformers ${TRANSFORMERS_VERSION_FOR_SGLANG} online."
|
||||||
|
pip install transformers=="${TRANSFORMERS_VERSION_FOR_SGLANG}" -i https://mirrors.tuna.tsinghua.edu.cn/pypi/web/simple
|
||||||
|
else
|
||||||
|
echo "Install transformers ${TRANSFORMERS_VERSION_FOR_SGLANG} locally."
|
||||||
|
TRANSFORMERS_PKG_PATH_TARGET=/tmp/transformers/${TRANSFORMERS_VERSION_FOR_SGLANG}
|
||||||
|
mkdir -p "${TRANSFORMERS_PKG_PATH_TARGET}"
|
||||||
|
cp "${TRANSFORMERS_PKG_PATH_SOURCE}/*" "${TRANSFORMERS_PKG_PATH_TARGET}/"
|
||||||
|
pip install --no-index --find-links="${TRANSFORMERS_PKG_PATH_TARGET}" transformers=="${TRANSFORMERS_VERSION_FOR_SGLANG}"
|
||||||
|
fi
|
||||||
|
echo "===== Install transformers for sglang in virtual env - End ====="
|
||||||
|
fi
|
||||||
|
echo "Transformers version for sglang: $(${PIP_FOR_SGLANG} show transformers | grep Version | cut -d: -f2)"
|
||||||
|
|
||||||
|
echo "scaling_governor performance num: \
|
||||||
|
$(cat /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor | grep performance | wc -l)"
|
||||||
|
echo "swappiness: $(cat /proc/sys/vm/swappiness)"
|
||||||
|
echo "numa_balancing: $(cat /proc/sys/kernel/numa_balancing)"
|
||||||
|
echo "sched_migration_cost_ns: $(cat /proc/sys/kernel/sched_migration_cost_ns)"
|
||||||
|
|
||||||
|
export SGLANG_TEST_MAX_RETRY=0
|
||||||
|
export SGLANG_SET_CPU_AFFINITY=1
|
||||||
|
echo "SGLANG_SET_CPU_AFFINITY: $SGLANG_SET_CPU_AFFINITY"
|
||||||
|
|
||||||
|
|
||||||
|
# Copy the checked‑out test common utility code to the sglang installation directory.
|
||||||
|
sglang_pkg_path=/sgl-workspace/sglang/python
|
||||||
|
ascend_test_util_path=${sglang_pkg_path}/sglang/test/ascend
|
||||||
|
mkdir -p ${ascend_test_util_path}
|
||||||
|
mv ${ascend_test_util_path} ${ascend_test_util_path}_bak
|
||||||
|
cp -r ${sglang_source_path}/python/sglang/test/ascend ${ascend_test_util_path}
|
||||||
|
|
||||||
|
source /usr/local/Ascend/cann/set_env.sh || true
|
||||||
|
source /usr/local/Ascend/nnal/atb/set_env.sh || true
|
||||||
|
source /usr/local/Ascend/ascend-toolkit/latest/opp/vendors/customize/bin/set_env.bash || true
|
||||||
|
source /usr/local/Ascend/ascend-toolkit/latest/opp/vendors/custom_transformer/bin/set_env.bash || true
|
||||||
|
|
||||||
|
# Set environment of cann
|
||||||
|
log_path="/root/.cache/tests/logs/log/${current_date}/${tc_name}/${HOSTNAME}"
|
||||||
|
rm -rf ${log_path}
|
||||||
|
mkdir -p ${log_path}
|
||||||
|
echo "Log path: ${log_path}"
|
||||||
|
|
||||||
|
echo "Running test: ${tc_name}"
|
||||||
|
test_exit_code=0
|
||||||
|
cd test
|
||||||
|
${PYTHON_FOR_SGLANG} -u run_suite.py --hw npu --suite ${test_suite} --timeout-per-file 3600 2>&1 | tee /tmp/test_output.log || test_exit_code=$?
|
||||||
|
|
||||||
|
echo "Finished test: ${tc_name}"
|
||||||
|
|
||||||
|
if [ "${test_exit_code}" = "0" ]; then
|
||||||
|
test_status="pass"
|
||||||
|
status_icon="✅"
|
||||||
|
else
|
||||||
|
test_status="fail"
|
||||||
|
status_icon="❌"
|
||||||
|
fi
|
||||||
|
|
||||||
|
echo "test_status=${test_status}" >> $GITHUB_ENV
|
||||||
|
echo "tc_name=${tc_name}" >> $GITHUB_ENV
|
||||||
|
export test_status tc_name
|
||||||
|
|
||||||
|
echo "## ${tc_name} ${status_icon} ${test_status}" >> $GITHUB_STEP_SUMMARY
|
||||||
|
echo "" >> $GITHUB_STEP_SUMMARY
|
||||||
|
metric_count=$(grep -c '\[METRIC\]' /tmp/test_output.log 2>/dev/null || echo 0)
|
||||||
|
if [ "${metric_count}" -gt 0 ]; then
|
||||||
|
echo "| Metric | Value | Pass |" >> $GITHUB_STEP_SUMMARY
|
||||||
|
echo "|--------|-------|------|" >> $GITHUB_STEP_SUMMARY
|
||||||
|
grep '\[METRIC\]' /tmp/test_output.log | while IFS= read -r line; do
|
||||||
|
metric_name=$(echo "$line" | sed -E 's/.*\[METRIC\] ([^=]+)=.*/\1/')
|
||||||
|
metric_value=$(echo "$line" | sed -E 's/.*\[METRIC\] [^=]+=([^ ]+).*/\1/')
|
||||||
|
echo "| ${metric_name} | ${metric_value} | ${status_icon} |" >> $GITHUB_STEP_SUMMARY
|
||||||
|
done
|
||||||
|
else
|
||||||
|
echo "No metrics collected (test may have failed before producing results)." >> $GITHUB_STEP_SUMMARY
|
||||||
|
fi
|
||||||
|
echo "" >> $GITHUB_STEP_SUMMARY
|
||||||
|
|
||||||
|
exit ${test_exit_code}
|
||||||
+170
-403
@@ -90,303 +90,82 @@ jobs:
|
|||||||
echo "CANN_image_a3=swr.cn-southwest-2.myhuaweicloud.com/base_image/ascend-ci/cann:9.0.0-a3-ubuntu22.04-py3.11" >> $GITHUB_OUTPUT
|
echo "CANN_image_a3=swr.cn-southwest-2.myhuaweicloud.com/base_image/ascend-ci/cann:9.0.0-a3-ubuntu22.04-py3.11" >> $GITHUB_OUTPUT
|
||||||
echo "CANN_image_910b=swr.cn-southwest-2.myhuaweicloud.com/base_image/ascend-ci/cann:9.0.0-910b-ubuntu22.04-py3.11" >> $GITHUB_OUTPUT
|
echo "CANN_image_910b=swr.cn-southwest-2.myhuaweicloud.com/base_image/ascend-ci/cann:9.0.0-910b-ubuntu22.04-py3.11" >> $GITHUB_OUTPUT
|
||||||
|
|
||||||
stage-a-unit-test-npu:
|
base-a-test-1-npu-a2:
|
||||||
needs: [check-changes, pr-gate, set-image-config]
|
needs: [check-changes, pr-gate, set-image-config]
|
||||||
if: needs.check-changes.outputs.main_package == 'true'
|
if: ${{ !failure() && !cancelled() && needs.check-changes.outputs.main_package == 'true' }}
|
||||||
runs-on: linux-aarch64-a2-1
|
uses: ./.github/workflows/_npu-pr-test-stage.yml
|
||||||
container:
|
with:
|
||||||
|
self_name: base-a-test-1-npu-a2
|
||||||
|
runner_config: linux-aarch64-a2-1
|
||||||
image: ${{ needs.set-image-config.outputs.CANN_image_910b }}
|
image: ${{ needs.set-image-config.outputs.CANN_image_910b }}
|
||||||
steps:
|
npu_device_type: 910b
|
||||||
- name: Checkout code
|
run_timeout_minutes: '15'
|
||||||
uses: actions/checkout@v4
|
secrets: inherit
|
||||||
with:
|
|
||||||
ref: ${{ inputs.ref || github.ref }}
|
|
||||||
|
|
||||||
- name: Mark repository safe
|
base-b-test-1-npu-a3:
|
||||||
run: |
|
|
||||||
git config --system --add safe.directory ${GITHUB_WORKSPACE}
|
|
||||||
|
|
||||||
- name: Install dependencies
|
|
||||||
env:
|
|
||||||
TORCH_CACHE_URL: "http://cache-service.nginx-pypi-cache.svc.cluster.local/whl/cpu"
|
|
||||||
PYPI_CACHE_URL: "http://cache-service.nginx-pypi-cache.svc.cluster.local/pypi/simple"
|
|
||||||
UV_INDEX_URL: "http://cache-service.nginx-pypi-cache.svc.cluster.local/pypi/simple"
|
|
||||||
GITHUB_PROXY_URL: "https://gh-proxy.test.osinfra.cn/"
|
|
||||||
RUSTUP_CACHE_URL: "http://cache-service.nginx-pypi-cache.svc.cluster.local:8082"
|
|
||||||
run: |
|
|
||||||
# speed up by using infra cache services
|
|
||||||
CACHING_URL="cache-service.nginx-pypi-cache.svc.cluster.local"
|
|
||||||
sed -Ei "s@(ports|archive).ubuntu.com@${CACHING_URL}:8081@g" /etc/apt/sources.list
|
|
||||||
pip config set global.index-url http://${CACHING_URL}/pypi/simple
|
|
||||||
pip config set global.trusted-host "${CACHING_URL}"
|
|
||||||
|
|
||||||
bash scripts/ci/npu/npu_ci_install_dependency.sh 910b
|
|
||||||
|
|
||||||
- name: Run test
|
|
||||||
timeout-minutes: 15
|
|
||||||
env:
|
|
||||||
SGLANG_IS_IN_CI: true
|
|
||||||
run: |
|
|
||||||
cd test
|
|
||||||
python3 run_suite.py --hw npu --suite stage-a-unit-test-npu
|
|
||||||
|
|
||||||
stage-b-test-1-npu-a3:
|
|
||||||
needs: [check-changes, pr-gate, set-image-config]
|
needs: [check-changes, pr-gate, set-image-config]
|
||||||
if: needs.check-changes.outputs.main_package == 'true'
|
if: ${{ !failure() && !cancelled() && needs.check-changes.outputs.main_package == 'true' }}
|
||||||
runs-on: linux-aarch64-a3-2-
|
uses: ./.github/workflows/_npu-pr-test-stage.yml
|
||||||
container:
|
|
||||||
image: ${{ needs.set-image-config.outputs.CANN_image_a3 }}
|
|
||||||
steps:
|
|
||||||
- name: Checkout code
|
|
||||||
uses: actions/checkout@v4
|
|
||||||
with:
|
with:
|
||||||
ref: ${{ inputs.ref || github.ref }}
|
self_name: base-b-test-1-npu-a3
|
||||||
|
runner_config: linux-aarch64-a3-2-
|
||||||
|
image: ${{ needs.set-image-config.outputs.CANN_image_a3 }}
|
||||||
|
run_timeout_minutes: '60'
|
||||||
|
timeout_per_file: '3600'
|
||||||
|
secrets: inherit
|
||||||
|
|
||||||
- name: Mark repository safe
|
base-b-test-2-npu-a3:
|
||||||
run: |
|
|
||||||
git config --system --add safe.directory ${GITHUB_WORKSPACE}
|
|
||||||
|
|
||||||
- name: Install dependencies
|
|
||||||
env:
|
|
||||||
TORCH_CACHE_URL: "http://cache-service.nginx-pypi-cache.svc.cluster.local/whl/cpu"
|
|
||||||
PYPI_CACHE_URL: "http://cache-service.nginx-pypi-cache.svc.cluster.local/pypi/simple"
|
|
||||||
UV_INDEX_URL: "http://cache-service.nginx-pypi-cache.svc.cluster.local/pypi/simple"
|
|
||||||
GITHUB_PROXY_URL: "https://gh-proxy.test.osinfra.cn/"
|
|
||||||
RUSTUP_CACHE_URL: "http://cache-service.nginx-pypi-cache.svc.cluster.local:8082"
|
|
||||||
run: |
|
|
||||||
# speed up by using infra cache services
|
|
||||||
CACHING_URL="cache-service.nginx-pypi-cache.svc.cluster.local"
|
|
||||||
sed -Ei "s@(ports|archive).ubuntu.com@${CACHING_URL}:8081@g" /etc/apt/sources.list
|
|
||||||
pip config set global.index-url http://${CACHING_URL}/pypi/simple
|
|
||||||
pip config set global.trusted-host "${CACHING_URL}"
|
|
||||||
|
|
||||||
bash scripts/ci/npu/npu_ci_install_dependency.sh a3
|
|
||||||
# copy required file from our daily cache
|
|
||||||
cp ~/.cache/modelscope/hub/datasets/otavia/ShareGPT_Vicuna_unfiltered/ShareGPT_V3_unfiltered_cleaned_split.json /tmp
|
|
||||||
# copy gsm8k dataset
|
|
||||||
cp ~/.cache/modelscope/hub/datasets/tmp/test.jsonl /tmp
|
|
||||||
|
|
||||||
- name: Run test
|
|
||||||
timeout-minutes: 60
|
|
||||||
env:
|
|
||||||
SGLANG_USE_MODELSCOPE: true
|
|
||||||
SGLANG_IS_IN_CI: true
|
|
||||||
HF_ENDPOINT: https://hf-mirror.com
|
|
||||||
TORCH_EXTENSIONS_DIR: /tmp/torch_extensions
|
|
||||||
PYTORCH_NPU_ALLOC_CONF: "expandable_segments:True"
|
|
||||||
STREAMS_PER_DEVICE: 32
|
|
||||||
run: |
|
|
||||||
cd test
|
|
||||||
python3 run_suite.py --hw npu --suite stage-b-test-1-npu-a3 --timeout-per-file 3600
|
|
||||||
|
|
||||||
|
|
||||||
stage-b-test-2-npu-a3:
|
|
||||||
needs: [check-changes, pr-gate, set-image-config]
|
needs: [check-changes, pr-gate, set-image-config]
|
||||||
if: needs.check-changes.outputs.main_package == 'true'
|
if: ${{ !failure() && !cancelled() && needs.check-changes.outputs.main_package == 'true' }}
|
||||||
runs-on: linux-aarch64-a3-2-
|
uses: ./.github/workflows/_npu-pr-test-stage.yml
|
||||||
container:
|
|
||||||
image: ${{ needs.set-image-config.outputs.CANN_image_a3 }}
|
|
||||||
steps:
|
|
||||||
- name: Checkout code
|
|
||||||
uses: actions/checkout@v4
|
|
||||||
with:
|
with:
|
||||||
ref: ${{ inputs.ref || github.ref }}
|
self_name: base-b-test-2-npu-a3
|
||||||
|
runner_config: linux-aarch64-a3-2-
|
||||||
|
image: ${{ needs.set-image-config.outputs.CANN_image_a3 }}
|
||||||
|
run_timeout_minutes: '60'
|
||||||
|
timeout_per_file: '3600'
|
||||||
|
secrets: inherit
|
||||||
|
|
||||||
- name: Mark repository safe
|
base-b-test-4-npu-a3:
|
||||||
run: |
|
|
||||||
git config --system --add safe.directory ${GITHUB_WORKSPACE}
|
|
||||||
|
|
||||||
- name: Install dependencies
|
|
||||||
env:
|
|
||||||
TORCH_CACHE_URL: "http://cache-service.nginx-pypi-cache.svc.cluster.local/whl/cpu"
|
|
||||||
PYPI_CACHE_URL: "http://cache-service.nginx-pypi-cache.svc.cluster.local/pypi/simple"
|
|
||||||
UV_INDEX_URL: "http://cache-service.nginx-pypi-cache.svc.cluster.local/pypi/simple"
|
|
||||||
GITHUB_PROXY_URL: "https://gh-proxy.test.osinfra.cn/"
|
|
||||||
RUSTUP_CACHE_URL: "http://cache-service.nginx-pypi-cache.svc.cluster.local:8082"
|
|
||||||
run: |
|
|
||||||
# speed up by using infra cache services
|
|
||||||
CACHING_URL="cache-service.nginx-pypi-cache.svc.cluster.local"
|
|
||||||
sed -Ei "s@(ports|archive).ubuntu.com@${CACHING_URL}:8081@g" /etc/apt/sources.list
|
|
||||||
pip config set global.index-url http://${CACHING_URL}/pypi/simple
|
|
||||||
pip config set global.trusted-host "${CACHING_URL}"
|
|
||||||
|
|
||||||
bash scripts/ci/npu/npu_ci_install_dependency.sh a3
|
|
||||||
# copy required file from our daily cache
|
|
||||||
cp ~/.cache/modelscope/hub/datasets/otavia/ShareGPT_Vicuna_unfiltered/ShareGPT_V3_unfiltered_cleaned_split.json /tmp
|
|
||||||
# copy gsm8k dataset
|
|
||||||
cp ~/.cache/modelscope/hub/datasets/tmp/test.jsonl /tmp
|
|
||||||
|
|
||||||
- name: Run test
|
|
||||||
timeout-minutes: 60
|
|
||||||
env:
|
|
||||||
SGLANG_USE_MODELSCOPE: true
|
|
||||||
SGLANG_IS_IN_CI: true
|
|
||||||
HF_ENDPOINT: https://hf-mirror.com
|
|
||||||
TORCH_EXTENSIONS_DIR: /tmp/torch_extensions
|
|
||||||
PYTORCH_NPU_ALLOC_CONF: "expandable_segments:True"
|
|
||||||
STREAMS_PER_DEVICE: 32
|
|
||||||
run: |
|
|
||||||
cd test
|
|
||||||
python3 run_suite.py --hw npu --suite stage-b-test-2-npu-a3 --timeout-per-file 3600
|
|
||||||
|
|
||||||
|
|
||||||
stage-b-test-4-npu-a3:
|
|
||||||
needs: [check-changes, pr-gate, set-image-config]
|
needs: [check-changes, pr-gate, set-image-config]
|
||||||
if: needs.check-changes.outputs.main_package == 'true'
|
if: ${{ !failure() && !cancelled() && needs.check-changes.outputs.main_package == 'true' }}
|
||||||
runs-on: linux-aarch64-a3-4-
|
uses: ./.github/workflows/_npu-pr-test-stage.yml
|
||||||
strategy:
|
|
||||||
fail-fast: false
|
|
||||||
matrix:
|
|
||||||
part: [ 0, 1 ]
|
|
||||||
container:
|
|
||||||
image: ${{ needs.set-image-config.outputs.CANN_image_a3 }}
|
|
||||||
steps:
|
|
||||||
- name: Checkout code
|
|
||||||
uses: actions/checkout@v4
|
|
||||||
with:
|
with:
|
||||||
ref: ${{ inputs.ref || github.ref }}
|
self_name: base-b-test-4-npu-a3
|
||||||
|
runner_config: linux-aarch64-a3-4-
|
||||||
|
image: ${{ needs.set-image-config.outputs.CANN_image_a3 }}
|
||||||
|
run_timeout_minutes: '120'
|
||||||
|
timeout_per_file: '3600'
|
||||||
|
partitions: '{"size":2,"arr":[0, 1]}'
|
||||||
|
secrets: inherit
|
||||||
|
|
||||||
- name: Mark repository safe
|
base-b-test-8-npu-a3:
|
||||||
run: |
|
|
||||||
git config --system --add safe.directory ${GITHUB_WORKSPACE}
|
|
||||||
|
|
||||||
- name: Install dependencies
|
|
||||||
env:
|
|
||||||
TORCH_CACHE_URL: "http://cache-service.nginx-pypi-cache.svc.cluster.local/whl/cpu"
|
|
||||||
PYPI_CACHE_URL: "http://cache-service.nginx-pypi-cache.svc.cluster.local/pypi/simple"
|
|
||||||
UV_INDEX_URL: "http://cache-service.nginx-pypi-cache.svc.cluster.local/pypi/simple"
|
|
||||||
GITHUB_PROXY_URL: "https://gh-proxy.test.osinfra.cn/"
|
|
||||||
RUSTUP_CACHE_URL: "http://cache-service.nginx-pypi-cache.svc.cluster.local:8082"
|
|
||||||
run: |
|
|
||||||
# speed up by using infra cache services
|
|
||||||
CACHING_URL="cache-service.nginx-pypi-cache.svc.cluster.local"
|
|
||||||
sed -Ei "s@(ports|archive).ubuntu.com@${CACHING_URL}:8081@g" /etc/apt/sources.list
|
|
||||||
pip config set global.index-url http://${CACHING_URL}/pypi/simple
|
|
||||||
pip config set global.trusted-host "${CACHING_URL}"
|
|
||||||
|
|
||||||
bash scripts/ci/npu/npu_ci_install_dependency.sh a3
|
|
||||||
# copy required file from our daily cache
|
|
||||||
cp ~/.cache/modelscope/hub/datasets/otavia/ShareGPT_Vicuna_unfiltered/ShareGPT_V3_unfiltered_cleaned_split.json /tmp
|
|
||||||
# copy gsm8k dataset
|
|
||||||
cp ~/.cache/modelscope/hub/datasets/tmp/test.jsonl /tmp
|
|
||||||
|
|
||||||
- name: Run test
|
|
||||||
timeout-minutes: 120
|
|
||||||
env:
|
|
||||||
SGLANG_USE_MODELSCOPE: true
|
|
||||||
SGLANG_IS_IN_CI: true
|
|
||||||
HF_ENDPOINT: https://hf-mirror.com
|
|
||||||
TORCH_EXTENSIONS_DIR: /tmp/torch_extensions
|
|
||||||
PYTORCH_NPU_ALLOC_CONF: "expandable_segments:True"
|
|
||||||
STREAMS_PER_DEVICE: 32
|
|
||||||
run: |
|
|
||||||
pip install sglang_router
|
|
||||||
cd test
|
|
||||||
python3 run_suite.py --hw npu --suite stage-b-test-4-npu-a3 --auto-partition-id ${{ matrix.part }} --auto-partition-size 2 --timeout-per-file 3600
|
|
||||||
|
|
||||||
stage-b-test-8-npu-a3:
|
|
||||||
needs: [check-changes, pr-gate, set-image-config]
|
needs: [check-changes, pr-gate, set-image-config]
|
||||||
if: needs.check-changes.outputs.main_package == 'true'
|
if: ${{ !failure() && !cancelled() && needs.check-changes.outputs.main_package == 'true' }}
|
||||||
runs-on: linux-aarch64-a3-8-
|
uses: ./.github/workflows/_npu-pr-test-stage.yml
|
||||||
container:
|
|
||||||
image: ${{ needs.set-image-config.outputs.CANN_image_a3 }}
|
|
||||||
steps:
|
|
||||||
- name: Checkout code
|
|
||||||
uses: actions/checkout@v4
|
|
||||||
with:
|
with:
|
||||||
ref: ${{ inputs.ref || github.ref }}
|
self_name: base-b-test-8-npu-a3
|
||||||
|
runner_config: linux-aarch64-a3-8-
|
||||||
|
image: ${{ needs.set-image-config.outputs.CANN_image_a3 }}
|
||||||
|
run_timeout_minutes: '60'
|
||||||
|
timeout_per_file: '3600'
|
||||||
|
secrets: inherit
|
||||||
|
|
||||||
- name: Mark repository safe
|
base-b-test-16-npu-a3:
|
||||||
run: |
|
|
||||||
git config --system --add safe.directory ${GITHUB_WORKSPACE}
|
|
||||||
|
|
||||||
- name: Install dependencies
|
|
||||||
env:
|
|
||||||
TORCH_CACHE_URL: "http://cache-service.nginx-pypi-cache.svc.cluster.local/whl/cpu"
|
|
||||||
PYPI_CACHE_URL: "http://cache-service.nginx-pypi-cache.svc.cluster.local/pypi/simple"
|
|
||||||
UV_INDEX_URL: "http://cache-service.nginx-pypi-cache.svc.cluster.local/pypi/simple"
|
|
||||||
GITHUB_PROXY_URL: "https://gh-proxy.test.osinfra.cn/"
|
|
||||||
RUSTUP_CACHE_URL: "http://cache-service.nginx-pypi-cache.svc.cluster.local:8082"
|
|
||||||
run: |
|
|
||||||
# speed up by using infra cache services
|
|
||||||
CACHING_URL="cache-service.nginx-pypi-cache.svc.cluster.local"
|
|
||||||
sed -Ei "s@(ports|archive).ubuntu.com@${CACHING_URL}:8081@g" /etc/apt/sources.list
|
|
||||||
pip config set global.index-url http://${CACHING_URL}/pypi/simple
|
|
||||||
pip config set global.trusted-host "${CACHING_URL}"
|
|
||||||
|
|
||||||
bash scripts/ci/npu/npu_ci_install_dependency.sh a3
|
|
||||||
# copy required file from our daily cache
|
|
||||||
cp ~/.cache/modelscope/hub/datasets/otavia/ShareGPT_Vicuna_unfiltered/ShareGPT_V3_unfiltered_cleaned_split.json /tmp
|
|
||||||
# copy gsm8k dataset
|
|
||||||
cp ~/.cache/modelscope/hub/datasets/tmp/test.jsonl /tmp
|
|
||||||
|
|
||||||
- name: Run test
|
|
||||||
timeout-minutes: 60
|
|
||||||
env:
|
|
||||||
SGLANG_USE_MODELSCOPE: true
|
|
||||||
SGLANG_IS_IN_CI: true
|
|
||||||
HF_ENDPOINT: https://hf-mirror.com
|
|
||||||
TORCH_EXTENSIONS_DIR: /tmp/torch_extensions
|
|
||||||
PYTORCH_NPU_ALLOC_CONF: "expandable_segments:True"
|
|
||||||
STREAMS_PER_DEVICE: 32
|
|
||||||
run: |
|
|
||||||
cd test
|
|
||||||
python3 run_suite.py --hw npu --suite stage-b-test-8-npu-a3 --timeout-per-file 3600
|
|
||||||
|
|
||||||
stage-b-test-16-npu-a3:
|
|
||||||
needs: [check-changes, pr-gate, set-image-config]
|
needs: [check-changes, pr-gate, set-image-config]
|
||||||
if: needs.check-changes.outputs.main_package == 'true'
|
if: ${{ !failure() && !cancelled() && needs.check-changes.outputs.main_package == 'true' }}
|
||||||
runs-on: linux-aarch64-a3-16-
|
uses: ./.github/workflows/_npu-pr-test-stage.yml
|
||||||
container:
|
|
||||||
image: ${{ needs.set-image-config.outputs.CANN_image_a3 }}
|
|
||||||
steps:
|
|
||||||
- name: Checkout code
|
|
||||||
uses: actions/checkout@v4
|
|
||||||
with:
|
with:
|
||||||
ref: ${{ inputs.ref || github.ref }}
|
self_name: base-b-test-16-npu-a3
|
||||||
|
runner_config: linux-aarch64-a3-16-
|
||||||
- name: Mark repository safe
|
image: ${{ needs.set-image-config.outputs.CANN_image_a3 }}
|
||||||
run: |
|
run_timeout_minutes: '120'
|
||||||
git config --system --add safe.directory ${GITHUB_WORKSPACE}
|
timeout_per_file: '3600'
|
||||||
|
secrets: inherit
|
||||||
- name: Install dependencies
|
|
||||||
env:
|
|
||||||
TORCH_CACHE_URL: "http://cache-service.nginx-pypi-cache.svc.cluster.local/whl/cpu"
|
|
||||||
PYPI_CACHE_URL: "http://cache-service.nginx-pypi-cache.svc.cluster.local/pypi/simple"
|
|
||||||
UV_INDEX_URL: "http://cache-service.nginx-pypi-cache.svc.cluster.local/pypi/simple"
|
|
||||||
GITHUB_PROXY_URL: "https://gh-proxy.test.osinfra.cn/"
|
|
||||||
RUSTUP_CACHE_URL: "http://cache-service.nginx-pypi-cache.svc.cluster.local:8082"
|
|
||||||
run: |
|
|
||||||
# speed up by using infra cache services
|
|
||||||
CACHING_URL="cache-service.nginx-pypi-cache.svc.cluster.local"
|
|
||||||
sed -Ei "s@(ports|archive).ubuntu.com@${CACHING_URL}:8081@g" /etc/apt/sources.list
|
|
||||||
pip config set global.index-url http://${CACHING_URL}/pypi/simple
|
|
||||||
pip config set global.trusted-host "${CACHING_URL}"
|
|
||||||
|
|
||||||
bash scripts/ci/npu/npu_ci_install_dependency.sh a3
|
|
||||||
# copy required file from our daily cache
|
|
||||||
cp ~/.cache/modelscope/hub/datasets/otavia/ShareGPT_Vicuna_unfiltered/ShareGPT_V3_unfiltered_cleaned_split.json /tmp
|
|
||||||
# copy gsm8k dataset
|
|
||||||
cp ~/.cache/modelscope/hub/datasets/tmp/test.jsonl /tmp
|
|
||||||
|
|
||||||
- name: Run test
|
|
||||||
timeout-minutes: 120
|
|
||||||
env:
|
|
||||||
SGLANG_USE_MODELSCOPE: true
|
|
||||||
SGLANG_IS_IN_CI: true
|
|
||||||
HF_ENDPOINT: https://hf-mirror.com
|
|
||||||
TORCH_EXTENSIONS_DIR: /tmp/torch_extensions
|
|
||||||
PYTORCH_NPU_ALLOC_CONF: "expandable_segments:True"
|
|
||||||
STREAMS_PER_DEVICE: 32
|
|
||||||
run: |
|
|
||||||
apt-get install -y libssl-dev
|
|
||||||
pip install sglang_router
|
|
||||||
cd test
|
|
||||||
python3 run_suite.py --hw npu --suite stage-b-test-16-npu-a3 --timeout-per-file 3600
|
|
||||||
|
|
||||||
multimodal-gen-test-1-npu-a3:
|
multimodal-gen-test-1-npu-a3:
|
||||||
needs: [check-changes, pr-gate, set-image-config]
|
needs: [check-changes, pr-gate, set-image-config]
|
||||||
if: needs.check-changes.outputs.multimodal_gen == 'true'
|
if: ${{ !failure() && !cancelled() && needs.check-changes.outputs.multimodal_gen == 'true' }}
|
||||||
runs-on: linux-aarch64-a3-800t-2
|
runs-on: linux-aarch64-a3-800t-2
|
||||||
container:
|
container:
|
||||||
image: ${{ needs.set-image-config.outputs.CANN_image_a3 }}
|
image: ${{ needs.set-image-config.outputs.CANN_image_a3 }}
|
||||||
@@ -443,7 +222,7 @@ jobs:
|
|||||||
|
|
||||||
multimodal-gen-test-2-npu-a3:
|
multimodal-gen-test-2-npu-a3:
|
||||||
needs: [check-changes, pr-gate, set-image-config]
|
needs: [check-changes, pr-gate, set-image-config]
|
||||||
if: needs.check-changes.outputs.multimodal_gen == 'true'
|
if: ${{ !failure() && !cancelled() && needs.check-changes.outputs.multimodal_gen == 'true' }}
|
||||||
runs-on: linux-aarch64-a3-800t-2
|
runs-on: linux-aarch64-a3-800t-2
|
||||||
container:
|
container:
|
||||||
image: ${{ needs.set-image-config.outputs.CANN_image_a3 }}
|
image: ${{ needs.set-image-config.outputs.CANN_image_a3 }}
|
||||||
@@ -498,146 +277,134 @@ jobs:
|
|||||||
if-no-files-found: ignore
|
if-no-files-found: ignore
|
||||||
retention-days: 7
|
retention-days: 7
|
||||||
|
|
||||||
pr-single-node-tests:
|
base-c-test-acc-2-npu-a3:
|
||||||
name: single-node-poc
|
name: base-c-test-acc-2-npu-a3
|
||||||
needs: [ check-changes, pr-gate, set-image-config ]
|
needs: [ check-changes, pr-gate, set-image-config ]
|
||||||
if: needs.check-changes.outputs.main_package == 'true'
|
if: ${{ !failure() && !cancelled() && needs.check-changes.outputs.main_package == 'true' }}
|
||||||
strategy:
|
uses: ./.github/workflows/_npu-single-node-test-stage.yml
|
||||||
fail-fast: false
|
|
||||||
max-parallel: 6
|
|
||||||
matrix:
|
|
||||||
test_config:
|
|
||||||
# qwen3_6_27b performance tests
|
|
||||||
- name: qwen3_6_27b_w8a8_1p_in64k_out1k_50ms
|
|
||||||
runner: linux-aarch64-a3-800t-2
|
|
||||||
test_case: test/registered/npu/performance/qwen3_6_27b/test_npu_qwen3_6_27b_w8a8_1p_in64k_out1k_50ms.py
|
|
||||||
test_type: 'perf'
|
|
||||||
|
|
||||||
# - name: qwen3_6_27b_1p_in1024x1024_30_out1024_50ms
|
|
||||||
# runner: linux-aarch64-a3-800t-2
|
|
||||||
# test_case: test/registered/npu/performance/qwen3_6_27b/test_npu_qwen3_6_27b_1p_in1024x1024_30_out1024_50ms.py
|
|
||||||
# test_type: 'perf'
|
|
||||||
|
|
||||||
# # qwen3_8b performance tests
|
|
||||||
# - name: qwen3_8b_w8a8_1p_in3k5_out1k5_50ms
|
|
||||||
# runner: linux-aarch64-a3-800t-2
|
|
||||||
# test_case: test/registered/npu/performance/qwen3-8b/test_npu_qwen3_8b_w8a8_1p_in3k5_out1k5_50ms.py
|
|
||||||
# test_type: 'perf'
|
|
||||||
|
|
||||||
# # qwen3_30b_a3b performance tests
|
|
||||||
# - name: qwen3_30b_w8a8_1p_in3k5_out1k5_50ms
|
|
||||||
# runner: linux-aarch64-a3-800t-2
|
|
||||||
# test_case: test/registered/npu/performance/qwen3_30b_a3b/test_npu_qwen3_30b_w8a8_1p_in3k5_out1k5_50ms.py
|
|
||||||
# test_type: 'perf'
|
|
||||||
|
|
||||||
# # qwen3_6_35b_a3b performance tests
|
|
||||||
# - name: qwen3_6_35b_a3b_1p_in64k_out1k_prefix90_50ms
|
|
||||||
# runner: linux-aarch64-a3-800t-2
|
|
||||||
# test_case: test/registered/npu/performance/qwen3_6_35b_a3b/test_npu_qwen3_6_35b_a3b_1p_in64k_out1k_prefix90_50ms.py
|
|
||||||
# test_type: 'perf'
|
|
||||||
|
|
||||||
# # qwen3_vl_8b_thinking accuracy tests
|
|
||||||
# - name: qwen3_vl_8b_thinking_1p_mmmu
|
|
||||||
# runner: linux-aarch64-a3-2-
|
|
||||||
# test_case: test/registered/npu/accuracy/qwen3_vl_8b_thinking/test_npu_qwen3_vl_8b_thinking_1p_mmmu.py
|
|
||||||
# test_type: 'accuracy'
|
|
||||||
|
|
||||||
# # qwen3_32b performance tests
|
|
||||||
# - name: qwen3_32b_w8a8_2p_in3k5_out1k5_50ms
|
|
||||||
# runner: linux-aarch64-a3-800t-4
|
|
||||||
# test_case: test/registered/npu/performance/qwen3_32b/test_npu_qwen3_32b_w8a8_2p_in3k5_out1k5_50ms.py
|
|
||||||
# test_type: 'perf'
|
|
||||||
|
|
||||||
# # qwen3_next_80b_a3b performance tests
|
|
||||||
# - name: qwen3_next_80b_w8a8_2p_in6k_out1k5_bs16
|
|
||||||
# runner: linux-aarch64-a3-800t-4
|
|
||||||
# test_case: test/registered/npu/performance/qwen3_next_80b_a3b_instruct/test_npu_qwen3_next_80b_w8a8_2p_in6k_out1k5_bs16.py
|
|
||||||
# test_type: 'perf'
|
|
||||||
|
|
||||||
# # minimax_m2_5 performance tests
|
|
||||||
# - name: minimax_m2_5_w8a8_4p_in64k_out1k_prefix90_50ms
|
|
||||||
# runner: linux-aarch64-a3-800t-8
|
|
||||||
# test_case: test/registered/npu/performance/minimax_m2_5/test_npu_minimax_m2_5_w8a8_4p_in64k_out1k_prefix90_50ms.py
|
|
||||||
# test_type: 'perf'
|
|
||||||
|
|
||||||
# # deepseek_v4_flash performance tests
|
|
||||||
# - name: deepseek_v4_flash_w8a8_8p_in8k_out1k_50ms
|
|
||||||
# runner: linux-aarch64-a3-800t-16
|
|
||||||
# test_case: test/registered/npu/performance/deepseek_v4_flash/test_npu_deepseek_v4_flash_w8a8_8p_in8k_out1k_50ms.py
|
|
||||||
# test_type: 'perf'
|
|
||||||
|
|
||||||
# # kimi_k2_6 performance tests
|
|
||||||
# - name: kimi_k2_6_w4a8_8p_in3k5_out1k5_20ms
|
|
||||||
# runner: linux-aarch64-a3-800t-16
|
|
||||||
# test_case: test/registered/npu/performance/kimi_k2_6/test_npu_kimi_k2_6_w4a8_8p_in3k5_out1k5_20ms.py
|
|
||||||
# test_type: 'perf'
|
|
||||||
|
|
||||||
# # qwen3_235b performance tests
|
|
||||||
# - name: qwen3_235b_w8a8_8p_in3k5_out1k5_50ms
|
|
||||||
# runner: linux-aarch64-a3-800t-16
|
|
||||||
# test_case: test/registered/npu/performance/qwen3_235b_a22b/test_npu_qwen3_235b_w8a8_8p_in3k5_out1k5_50ms.py
|
|
||||||
# test_type: 'perf'
|
|
||||||
|
|
||||||
# # qwen3_5_397b performance tests
|
|
||||||
# - name: qwen3_5_397b_w4a8_8p_in3k5_out1k5_50ms
|
|
||||||
# runner: linux-aarch64-a3-800t-16
|
|
||||||
# test_case: test/registered/npu/performance/qwen3_5_397b/test_npu_qwen3_5_397b_w4a8_8p_in3k5_out1k5_50ms.py
|
|
||||||
# test_type: 'perf'
|
|
||||||
|
|
||||||
# NPU accuracy tests
|
|
||||||
# - name: glm4_7_flash_1p_gsm8k
|
|
||||||
# runner: linux-aarch64-a3-2-
|
|
||||||
# test_case: test/registered/npu/accuracy/glm4_7_flash/test_npu_glm4_7_flash_1p_gsm8k.py
|
|
||||||
# test_type: 'accuracy'
|
|
||||||
# - name: qwen3_vl_30b_a3b_bf16_2p_gsm8k
|
|
||||||
# runner: linux-aarch64-a3-4-
|
|
||||||
# test_case: test/registered/npu/accuracy/qwen3_vl_30b_a3b/test_npu_qwen3_vl_30b_a3b_bf16_2p_gsm8k.py
|
|
||||||
# test_type: 'accuracy'
|
|
||||||
- name: glm5_top64_pruned_bf16_8p_gsm8k
|
|
||||||
runner: linux-aarch64-a3-16-
|
|
||||||
test_case: test/registered/npu/accuracy/glm5_top64_pruned/test_npu_glm5_top64_pruned_bf16_8p_gsm8k.py
|
|
||||||
test_type: 'accuracy'
|
|
||||||
# - name: moonshotai_moonlight_16b_a3b_bf16_1p_gsm8k
|
|
||||||
# runner: linux-aarch64-a3-2-
|
|
||||||
# test_case: test/registered/npu/accuracy/moonshotai_moonlight_16b_a3b/test_npu_moonlight_16b_a3b_bf16_1p_gsm8k.py
|
|
||||||
# test_type: 'accuracy'
|
|
||||||
# - name: qwen3_5_9b_bf16_1p_gsm8k
|
|
||||||
# runner: linux-aarch64-a3-2-
|
|
||||||
# test_case: test/registered/npu/accuracy/qwen3_5_9b/test_npu_qwen3_5_9b_bf16_1p_gsm8k.py
|
|
||||||
# test_type: 'accuracy'
|
|
||||||
# - name: qwen3_vl_8b_bf16_2p_gsm8k
|
|
||||||
# runner: linux-aarch64-a3-4-
|
|
||||||
# test_case: test/registered/npu/accuracy/qwen3_vl_8b/test_npu_qwen3_vl_8b_bf16_2p_gsm8k.py
|
|
||||||
# test_type: 'accuracy'
|
|
||||||
|
|
||||||
uses: ./.github/workflows/nightly-test-npu-e2e-single-node.yml
|
|
||||||
with:
|
with:
|
||||||
runner: ${{ matrix.test_config.runner }}
|
runner: linux-aarch64-a3-2-
|
||||||
test_type: ${{ matrix.test_config.test_type }}
|
test_type: 'accuracy'
|
||||||
test_config_name: ${{ matrix.test_config.name }}
|
test_suite: base-c-test-acc-2-npu-a3
|
||||||
test_case: ${{ matrix.test_config.test_case }}
|
|
||||||
image: ${{ needs.set-image-config.outputs.CANN_image_a3 }}
|
image: ${{ needs.set-image-config.outputs.CANN_image_a3 }}
|
||||||
install_sglang_from_source: false
|
|
||||||
install_sglang_deps: true
|
install_sglang_deps: true
|
||||||
device_type_for_deps: 'a3'
|
device_type_for_deps: 'a3'
|
||||||
transformers_version: ''
|
|
||||||
|
base-c-test-acc-4-npu-a3:
|
||||||
|
name: base-c-test-acc-4-npu-a3
|
||||||
|
needs: [ check-changes, pr-gate, set-image-config ]
|
||||||
|
if: ${{ !failure() && !cancelled() && needs.check-changes.outputs.main_package == 'true' }}
|
||||||
|
uses: ./.github/workflows/_npu-single-node-test-stage.yml
|
||||||
|
with:
|
||||||
|
runner: linux-aarch64-a3-4-
|
||||||
|
test_type: 'accuracy'
|
||||||
|
test_suite: base-c-test-acc-4-npu-a3
|
||||||
|
image: ${{ needs.set-image-config.outputs.CANN_image_a3 }}
|
||||||
|
install_sglang_deps: true
|
||||||
|
device_type_for_deps: 'a3'
|
||||||
|
|
||||||
|
base-c-test-acc-8-npu-a3:
|
||||||
|
name: base-c-test-acc-8-npu-a3
|
||||||
|
needs: [ check-changes, pr-gate, set-image-config ]
|
||||||
|
if: ${{ !failure() && !cancelled() && needs.check-changes.outputs.main_package == 'true' }}
|
||||||
|
uses: ./.github/workflows/_npu-single-node-test-stage.yml
|
||||||
|
with:
|
||||||
|
runner: linux-aarch64-a3-8-
|
||||||
|
test_type: 'accuracy'
|
||||||
|
test_suite: base-c-test-acc-8-npu-a3
|
||||||
|
image: ${{ needs.set-image-config.outputs.CANN_image_a3 }}
|
||||||
|
install_sglang_deps: true
|
||||||
|
device_type_for_deps: 'a3'
|
||||||
|
|
||||||
|
base-c-test-acc-16-npu-a3:
|
||||||
|
name: base-c-test-acc-16-npu-a3
|
||||||
|
needs: [ check-changes, pr-gate, set-image-config ]
|
||||||
|
if: ${{ !failure() && !cancelled() && needs.check-changes.outputs.main_package == 'true' }}
|
||||||
|
uses: ./.github/workflows/_npu-single-node-test-stage.yml
|
||||||
|
with:
|
||||||
|
runner: linux-aarch64-a3-16-
|
||||||
|
test_type: 'accuracy'
|
||||||
|
test_suite: base-c-test-acc-16-npu-a3
|
||||||
|
image: ${{ needs.set-image-config.outputs.CANN_image_a3 }}
|
||||||
|
install_sglang_deps: true
|
||||||
|
device_type_for_deps: 'a3'
|
||||||
|
|
||||||
|
base-c-test-perf-2-npu-a3:
|
||||||
|
name: base-c-test-perf-2-npu-a3
|
||||||
|
needs: [ check-changes, pr-gate, set-image-config, base-c-test-acc-2-npu-a3 ]
|
||||||
|
if: ${{ !failure() && !cancelled() && needs.check-changes.outputs.main_package == 'true' }}
|
||||||
|
uses: ./.github/workflows/_npu-single-node-test-stage.yml
|
||||||
|
with:
|
||||||
|
runner: linux-aarch64-a3-800t-2
|
||||||
|
test_type: 'perf'
|
||||||
|
test_suite: base-c-test-perf-2-npu-a3
|
||||||
|
image: ${{ needs.set-image-config.outputs.CANN_image_a3 }}
|
||||||
|
install_sglang_deps: true
|
||||||
|
device_type_for_deps: 'a3'
|
||||||
|
|
||||||
|
base-c-test-perf-4-npu-a3:
|
||||||
|
name: base-c-test-perf-4-npu-a3
|
||||||
|
needs: [ check-changes, pr-gate, set-image-config, base-c-test-acc-4-npu-a3 ]
|
||||||
|
if: ${{ !failure() && !cancelled() && needs.check-changes.outputs.main_package == 'true' }}
|
||||||
|
uses: ./.github/workflows/_npu-single-node-test-stage.yml
|
||||||
|
with:
|
||||||
|
runner: linux-aarch64-a3-800t-4
|
||||||
|
test_type: 'perf'
|
||||||
|
test_suite: base-c-test-perf-4-npu-a3
|
||||||
|
image: ${{ needs.set-image-config.outputs.CANN_image_a3 }}
|
||||||
|
install_sglang_deps: true
|
||||||
|
device_type_for_deps: 'a3'
|
||||||
|
|
||||||
|
base-c-test-perf-8-npu-a3:
|
||||||
|
name: base-c-test-perf-8-npu-a3
|
||||||
|
needs: [ check-changes, pr-gate, set-image-config, base-c-test-acc-8-npu-a3 ]
|
||||||
|
if: ${{ !failure() && !cancelled() && needs.check-changes.outputs.main_package == 'true' }}
|
||||||
|
uses: ./.github/workflows/_npu-single-node-test-stage.yml
|
||||||
|
with:
|
||||||
|
runner: linux-aarch64-a3-800t-8
|
||||||
|
test_type: 'perf'
|
||||||
|
test_suite: base-c-test-perf-8-npu-a3
|
||||||
|
image: ${{ needs.set-image-config.outputs.CANN_image_a3 }}
|
||||||
|
install_sglang_deps: true
|
||||||
|
device_type_for_deps: 'a3'
|
||||||
|
|
||||||
|
base-c-test-perf-16-npu-a3:
|
||||||
|
name: base-c-test-perf-16-npu-a3
|
||||||
|
needs: [ check-changes, pr-gate, set-image-config, base-c-test-acc-16-npu-a3 ]
|
||||||
|
if: ${{ !failure() && !cancelled() && needs.check-changes.outputs.main_package == 'true' }}
|
||||||
|
uses: ./.github/workflows/_npu-single-node-test-stage.yml
|
||||||
|
with:
|
||||||
|
runner: linux-aarch64-a3-800t-16
|
||||||
|
test_type: 'perf'
|
||||||
|
test_suite: base-c-test-perf-16-npu-a3
|
||||||
|
image: ${{ needs.set-image-config.outputs.CANN_image_a3 }}
|
||||||
|
install_sglang_deps: true
|
||||||
|
device_type_for_deps: 'a3'
|
||||||
|
|
||||||
|
|
||||||
pr-test-npu-finish:
|
pr-test-npu-finish:
|
||||||
needs:
|
needs:
|
||||||
[
|
[
|
||||||
check-changes,
|
check-changes,
|
||||||
|
|
||||||
stage-a-unit-test-npu,
|
base-a-test-1-npu-a2,
|
||||||
stage-b-test-1-npu-a3,
|
base-b-test-1-npu-a3,
|
||||||
stage-b-test-2-npu-a3,
|
base-b-test-2-npu-a3,
|
||||||
stage-b-test-4-npu-a3,
|
base-b-test-4-npu-a3,
|
||||||
stage-b-test-8-npu-a3,
|
base-b-test-8-npu-a3,
|
||||||
stage-b-test-16-npu-a3,
|
base-b-test-16-npu-a3,
|
||||||
|
|
||||||
multimodal-gen-test-1-npu-a3,
|
multimodal-gen-test-1-npu-a3,
|
||||||
multimodal-gen-test-2-npu-a3,
|
multimodal-gen-test-2-npu-a3,
|
||||||
|
|
||||||
pr-single-node-tests,
|
base-c-test-acc-2-npu-a3,
|
||||||
|
base-c-test-acc-4-npu-a3,
|
||||||
|
base-c-test-acc-8-npu-a3,
|
||||||
|
base-c-test-acc-16-npu-a3,
|
||||||
|
base-c-test-perf-2-npu-a3,
|
||||||
|
base-c-test-perf-4-npu-a3,
|
||||||
|
base-c-test-perf-8-npu-a3,
|
||||||
|
base-c-test-perf-16-npu-a3,
|
||||||
]
|
]
|
||||||
if: always()
|
if: always()
|
||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
|
|||||||
@@ -1,10 +1,15 @@
|
|||||||
|
import glob
|
||||||
|
import inspect
|
||||||
import json
|
import json
|
||||||
import logging
|
import logging
|
||||||
import os
|
import os
|
||||||
import re
|
import re
|
||||||
|
import shutil
|
||||||
import subprocess
|
import subprocess
|
||||||
|
import sys
|
||||||
import threading
|
import threading
|
||||||
import time
|
import time
|
||||||
|
from datetime import datetime
|
||||||
from urllib.parse import urlparse
|
from urllib.parse import urlparse
|
||||||
|
|
||||||
from sglang.srt.utils import kill_process_tree
|
from sglang.srt.utils import kill_process_tree
|
||||||
@@ -289,9 +294,117 @@ class TestNpuAccuracyTestCaseBase(CustomTestCase):
|
|||||||
max_attempts = 2
|
max_attempts = 2
|
||||||
n_runs = 3
|
n_runs = 3
|
||||||
accuracy = 0.1
|
accuracy = 0.1
|
||||||
|
test_type = "accuracy"
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def _get_tc_name(cls):
|
||||||
|
"""Derive the test case name from the test file (filename without
|
||||||
|
extension). Mirrors the workflow's ``tc_name=${test_case##*/}`` logic
|
||||||
|
so each case in a suite writes to its own output path."""
|
||||||
|
try:
|
||||||
|
tc_file = inspect.getfile(cls)
|
||||||
|
except (TypeError, OSError):
|
||||||
|
tc_file = getattr(sys.modules.get(cls.__module__), "__file__", "")
|
||||||
|
return os.path.splitext(os.path.basename(tc_file))[0]
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def _setup_per_case_output(cls):
|
||||||
|
"""Set up per-case output directories and env vars.
|
||||||
|
|
||||||
|
Extracted from ``nightly-test-npu-e2e-single-node.yml`` so that when a
|
||||||
|
suite is executed, each case writes its metrics/plog to a path derived
|
||||||
|
from the case file rather than the suite name.
|
||||||
|
"""
|
||||||
|
cls.tc_name = cls._get_tc_name()
|
||||||
|
current_date = datetime.now().strftime("%Y%m%d")
|
||||||
|
test_type = getattr(cls, "test_type", "accuracy")
|
||||||
|
base_output = f"/root/.cache/tests/output/{test_type}/{current_date}"
|
||||||
|
os.makedirs(base_output, exist_ok=True)
|
||||||
|
cls.metrics_data_file = os.path.join(base_output, cls.tc_name)
|
||||||
|
os.makedirs(cls.metrics_data_file, exist_ok=True)
|
||||||
|
# Override env vars so evalscope/dump_metric write to per-case paths.
|
||||||
|
os.environ["METRICS_DATA_FILE"] = cls.metrics_data_file
|
||||||
|
os.environ["SGLANG_TEST_METRICS_OUTPUT"] = os.path.join(
|
||||||
|
cls.metrics_data_file, "metrics"
|
||||||
|
)
|
||||||
|
logger.info(
|
||||||
|
"Per-case output: tc_name=%s metrics_data_file=%s",
|
||||||
|
cls.tc_name,
|
||||||
|
cls.metrics_data_file,
|
||||||
|
)
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def _save_metrics_json(cls):
|
||||||
|
"""Write per-case ``metrics.json`` from ``dump_metric`` JSONL files.
|
||||||
|
|
||||||
|
Replaces the workflow's stdout-parsing + ``dump_metrics.py`` logic so
|
||||||
|
each case in a suite persists its own metrics snapshot.
|
||||||
|
"""
|
||||||
|
if not getattr(cls, "metrics_data_file", None):
|
||||||
|
return
|
||||||
|
metrics = {}
|
||||||
|
baselines = {}
|
||||||
|
pattern = os.path.join(cls.metrics_data_file, "metrics.*.jsonl")
|
||||||
|
for jsonl_path in glob.glob(pattern):
|
||||||
|
try:
|
||||||
|
with open(jsonl_path, "r", encoding="utf-8") as f:
|
||||||
|
for line in f:
|
||||||
|
line = line.strip()
|
||||||
|
if not line:
|
||||||
|
continue
|
||||||
|
record = json.loads(line)
|
||||||
|
name = record.get("metric_name")
|
||||||
|
value = record.get("value")
|
||||||
|
if name is None:
|
||||||
|
continue
|
||||||
|
if name.endswith("_baseline"):
|
||||||
|
baselines[name[: -len("_baseline")]] = value
|
||||||
|
else:
|
||||||
|
metrics[name] = value
|
||||||
|
except Exception as e:
|
||||||
|
logger.warning("Failed to read %s: %s", jsonl_path, e)
|
||||||
|
out_path = os.path.join(cls.metrics_data_file, "metrics.json")
|
||||||
|
payload = {
|
||||||
|
"test_case": cls.tc_name,
|
||||||
|
"test_type": getattr(cls, "test_type", "accuracy"),
|
||||||
|
"metrics": metrics,
|
||||||
|
"baselines": baselines,
|
||||||
|
}
|
||||||
|
try:
|
||||||
|
with open(out_path, "w", encoding="utf-8") as f:
|
||||||
|
json.dump(payload, f, ensure_ascii=False)
|
||||||
|
logger.info("Saved per-case metrics to %s", out_path)
|
||||||
|
except Exception as e:
|
||||||
|
logger.warning("Failed to write metrics.json: %s", e)
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def _backup_plog(cls):
|
||||||
|
"""Backup Ascend plog files to a per-case path.
|
||||||
|
|
||||||
|
Replaces the workflow's ``Backup plog`` step so each case in a suite
|
||||||
|
gets its own plog snapshot instead of all cases sharing the suite name.
|
||||||
|
"""
|
||||||
|
plog_path = "/root/ascend/log/debug/plog"
|
||||||
|
if not os.path.isdir(plog_path):
|
||||||
|
return
|
||||||
|
tc_name = getattr(cls, "tc_name", None)
|
||||||
|
if not tc_name:
|
||||||
|
return
|
||||||
|
hostname = os.getenv("HOSTNAME", "unknown")
|
||||||
|
target = os.path.join("/root/.cache/tests/logs/plog", tc_name, hostname)
|
||||||
|
os.makedirs(target, exist_ok=True)
|
||||||
|
for name in os.listdir(plog_path):
|
||||||
|
src = os.path.join(plog_path, name)
|
||||||
|
if os.path.isfile(src):
|
||||||
|
try:
|
||||||
|
shutil.copy2(src, os.path.join(target, name))
|
||||||
|
except Exception as e:
|
||||||
|
logger.warning("Failed to copy plog %s: %s", name, e)
|
||||||
|
logger.info("Backed up plog to %s", target)
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
def setUpClass(cls):
|
def setUpClass(cls):
|
||||||
|
cls._setup_per_case_output()
|
||||||
cls.base_url = DEFAULT_URL_FOR_TEST
|
cls.base_url = DEFAULT_URL_FOR_TEST
|
||||||
env = os.environ.copy()
|
env = os.environ.copy()
|
||||||
for key, value in env.items():
|
for key, value in env.items():
|
||||||
@@ -318,6 +431,8 @@ class TestNpuAccuracyTestCaseBase(CustomTestCase):
|
|||||||
kill_process_tree(cls.process.pid)
|
kill_process_tree(cls.process.pid)
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
logger.error(f"Error during tearDown: {e}")
|
logger.error(f"Error during tearDown: {e}")
|
||||||
|
cls._save_metrics_json()
|
||||||
|
cls._backup_plog()
|
||||||
|
|
||||||
def run_accuracy(self):
|
def run_accuracy(self):
|
||||||
parsed_url = urlparse(self.base_url)
|
parsed_url = urlparse(self.base_url)
|
||||||
|
|||||||
@@ -1,9 +1,15 @@
|
|||||||
|
import glob
|
||||||
|
import inspect
|
||||||
|
import json
|
||||||
import logging
|
import logging
|
||||||
import os
|
import os
|
||||||
import re
|
import re
|
||||||
|
import shutil
|
||||||
import subprocess
|
import subprocess
|
||||||
|
import sys
|
||||||
import threading
|
import threading
|
||||||
import time
|
import time
|
||||||
|
from datetime import datetime
|
||||||
from functools import wraps
|
from functools import wraps
|
||||||
from urllib.parse import urlparse
|
from urllib.parse import urlparse
|
||||||
|
|
||||||
@@ -909,8 +915,115 @@ class TestNpuPerformanceTestCaseBase(CustomTestCase):
|
|||||||
generation_kwargs = None
|
generation_kwargs = None
|
||||||
pop_sglang_is_in_ci_for_gsp = False
|
pop_sglang_is_in_ci_for_gsp = False
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def _get_tc_name(cls):
|
||||||
|
"""Derive the test case name from the test file (filename without
|
||||||
|
extension). Mirrors the workflow's ``tc_name=${test_case##*/}`` logic
|
||||||
|
so each case in a suite writes to its own output path."""
|
||||||
|
try:
|
||||||
|
tc_file = inspect.getfile(cls)
|
||||||
|
except (TypeError, OSError):
|
||||||
|
tc_file = getattr(sys.modules.get(cls.__module__), "__file__", "")
|
||||||
|
return os.path.splitext(os.path.basename(tc_file))[0]
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def _setup_per_case_output(cls):
|
||||||
|
"""Set up per-case output directories and env vars.
|
||||||
|
|
||||||
|
Extracted from ``nightly-test-npu-e2e-single-node.yml`` so that when a
|
||||||
|
suite is executed, each case writes its metrics/plog to a path derived
|
||||||
|
from the case file rather than the suite name.
|
||||||
|
"""
|
||||||
|
cls.tc_name = cls._get_tc_name()
|
||||||
|
current_date = datetime.now().strftime("%Y%m%d")
|
||||||
|
test_type = getattr(cls, "test_type", "perf")
|
||||||
|
base_output = f"/root/.cache/tests/output/{test_type}/{current_date}"
|
||||||
|
os.makedirs(base_output, exist_ok=True)
|
||||||
|
cls.metrics_data_file = os.path.join(base_output, cls.tc_name)
|
||||||
|
os.makedirs(cls.metrics_data_file, exist_ok=True)
|
||||||
|
# Override env vars so evalscope/dump_metric write to per-case paths.
|
||||||
|
os.environ["METRICS_DATA_FILE"] = cls.metrics_data_file
|
||||||
|
os.environ["SGLANG_TEST_METRICS_OUTPUT"] = os.path.join(
|
||||||
|
cls.metrics_data_file, "metrics"
|
||||||
|
)
|
||||||
|
logger.info(
|
||||||
|
"Per-case output: tc_name=%s metrics_data_file=%s",
|
||||||
|
cls.tc_name,
|
||||||
|
cls.metrics_data_file,
|
||||||
|
)
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def _save_metrics_json(cls):
|
||||||
|
"""Write per-case ``metrics.json`` from ``dump_metric`` JSONL files.
|
||||||
|
|
||||||
|
Replaces the workflow's stdout-parsing + ``dump_metrics.py`` logic so
|
||||||
|
each case in a suite persists its own metrics snapshot.
|
||||||
|
"""
|
||||||
|
if not getattr(cls, "metrics_data_file", None):
|
||||||
|
return
|
||||||
|
metrics = {}
|
||||||
|
baselines = {}
|
||||||
|
pattern = os.path.join(cls.metrics_data_file, "metrics.*.jsonl")
|
||||||
|
for jsonl_path in glob.glob(pattern):
|
||||||
|
try:
|
||||||
|
with open(jsonl_path, "r", encoding="utf-8") as f:
|
||||||
|
for line in f:
|
||||||
|
line = line.strip()
|
||||||
|
if not line:
|
||||||
|
continue
|
||||||
|
record = json.loads(line)
|
||||||
|
name = record.get("metric_name")
|
||||||
|
value = record.get("value")
|
||||||
|
if name is None:
|
||||||
|
continue
|
||||||
|
if name.endswith("_baseline"):
|
||||||
|
baselines[name[: -len("_baseline")]] = value
|
||||||
|
else:
|
||||||
|
metrics[name] = value
|
||||||
|
except Exception as e:
|
||||||
|
logger.warning("Failed to read %s: %s", jsonl_path, e)
|
||||||
|
out_path = os.path.join(cls.metrics_data_file, "metrics.json")
|
||||||
|
payload = {
|
||||||
|
"test_case": cls.tc_name,
|
||||||
|
"test_type": getattr(cls, "test_type", "accuracy"),
|
||||||
|
"metrics": metrics,
|
||||||
|
"baselines": baselines,
|
||||||
|
}
|
||||||
|
try:
|
||||||
|
with open(out_path, "w", encoding="utf-8") as f:
|
||||||
|
json.dump(payload, f, ensure_ascii=False)
|
||||||
|
logger.info("Saved per-case metrics to %s", out_path)
|
||||||
|
except Exception as e:
|
||||||
|
logger.warning("Failed to write metrics.json: %s", e)
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def _backup_plog(cls):
|
||||||
|
"""Backup Ascend plog files to a per-case path.
|
||||||
|
|
||||||
|
Replaces the workflow's ``Backup plog`` step so each case in a suite
|
||||||
|
gets its own plog snapshot instead of all cases sharing the suite name.
|
||||||
|
"""
|
||||||
|
plog_path = "/root/ascend/log/debug/plog"
|
||||||
|
if not os.path.isdir(plog_path):
|
||||||
|
return
|
||||||
|
tc_name = getattr(cls, "tc_name", None)
|
||||||
|
if not tc_name:
|
||||||
|
return
|
||||||
|
hostname = os.getenv("HOSTNAME", "unknown")
|
||||||
|
target = os.path.join("/root/.cache/tests/logs/plog", tc_name, hostname)
|
||||||
|
os.makedirs(target, exist_ok=True)
|
||||||
|
for name in os.listdir(plog_path):
|
||||||
|
src = os.path.join(plog_path, name)
|
||||||
|
if os.path.isfile(src):
|
||||||
|
try:
|
||||||
|
shutil.copy2(src, os.path.join(target, name))
|
||||||
|
except Exception as e:
|
||||||
|
logger.warning("Failed to copy plog %s: %s", name, e)
|
||||||
|
logger.info("Backed up plog to %s", target)
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
def setUpClass(cls):
|
def setUpClass(cls):
|
||||||
|
cls._setup_per_case_output()
|
||||||
cls.base_url = DEFAULT_URL_FOR_TEST
|
cls.base_url = DEFAULT_URL_FOR_TEST
|
||||||
env = os.environ.copy()
|
env = os.environ.copy()
|
||||||
for key, value in env.items():
|
for key, value in env.items():
|
||||||
@@ -937,6 +1050,8 @@ class TestNpuPerformanceTestCaseBase(CustomTestCase):
|
|||||||
kill_process_tree(cls.process.pid)
|
kill_process_tree(cls.process.pid)
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
logger.error(f"Error during tearDown: {e}")
|
logger.error(f"Error during tearDown: {e}")
|
||||||
|
cls._save_metrics_json()
|
||||||
|
cls._backup_plog()
|
||||||
|
|
||||||
@retry()
|
@retry()
|
||||||
def run_throughput(self):
|
def run_throughput(self):
|
||||||
|
|||||||
@@ -6,12 +6,7 @@ from sglang.test.ascend.e2e.test_npu_accuracy_utils import (
|
|||||||
from sglang.test.ascend.e2e.test_npu_performance_utils import GLM_4_7_FLASH_MODEL_PATH
|
from sglang.test.ascend.e2e.test_npu_performance_utils import GLM_4_7_FLASH_MODEL_PATH
|
||||||
from sglang.test.ci.ci_register import register_npu_ci
|
from sglang.test.ci.ci_register import register_npu_ci
|
||||||
|
|
||||||
register_npu_ci(
|
register_npu_ci(est_time=3600, suite="base-c-test-acc-2-npu-a3")
|
||||||
est_time=3600,
|
|
||||||
suite="",
|
|
||||||
nightly=True,
|
|
||||||
disabled="accuracy testcase",
|
|
||||||
)
|
|
||||||
|
|
||||||
ENVS = {
|
ENVS = {
|
||||||
"PYTORCH_NPU_ALLOC_CONF": "expandable_segments:True",
|
"PYTORCH_NPU_ALLOC_CONF": "expandable_segments:True",
|
||||||
|
|||||||
+2
-7
@@ -8,12 +8,7 @@ from sglang.test.ascend.e2e.test_npu_performance_utils import (
|
|||||||
)
|
)
|
||||||
from sglang.test.ci.ci_register import register_npu_ci
|
from sglang.test.ci.ci_register import register_npu_ci
|
||||||
|
|
||||||
register_npu_ci(
|
register_npu_ci(est_time=3600, suite="base-c-test-acc-16-npu-a3")
|
||||||
est_time=3600,
|
|
||||||
suite="",
|
|
||||||
nightly=True,
|
|
||||||
disabled="accuracy testcase",
|
|
||||||
)
|
|
||||||
|
|
||||||
ENVS = {
|
ENVS = {
|
||||||
"SGLANG_SET_CPU_AFFINITY": "1",
|
"SGLANG_SET_CPU_AFFINITY": "1",
|
||||||
@@ -54,7 +49,7 @@ class TestNPUGLM5_Top64_Pruned_GSM8K(TestNpuAccuracyTestCaseBase):
|
|||||||
model = GLM5_TOP64_PRUNED_GSM8K_MODEL_PATH
|
model = GLM5_TOP64_PRUNED_GSM8K_MODEL_PATH
|
||||||
envs = ENVS
|
envs = ENVS
|
||||||
other_args = OTHER_ARGS
|
other_args = OTHER_ARGS
|
||||||
accuracy = 0.50
|
accuracy = 0.48
|
||||||
datasets = ["gsm8k"]
|
datasets = ["gsm8k"]
|
||||||
generation_config = {
|
generation_config = {
|
||||||
"max_tokens": 2048,
|
"max_tokens": 2048,
|
||||||
|
|||||||
+1
-6
@@ -8,12 +8,7 @@ from sglang.test.ascend.e2e.test_npu_performance_utils import (
|
|||||||
)
|
)
|
||||||
from sglang.test.ci.ci_register import register_npu_ci
|
from sglang.test.ci.ci_register import register_npu_ci
|
||||||
|
|
||||||
register_npu_ci(
|
register_npu_ci(est_time=3600, suite="base-c-test-acc-2-npu-a3")
|
||||||
est_time=3600,
|
|
||||||
suite="",
|
|
||||||
nightly=True,
|
|
||||||
disabled="accuracy testcase",
|
|
||||||
)
|
|
||||||
|
|
||||||
MODEL_ENVS = {
|
MODEL_ENVS = {
|
||||||
"SGLANG_SET_CPU_AFFINITY": "1",
|
"SGLANG_SET_CPU_AFFINITY": "1",
|
||||||
|
|||||||
@@ -8,12 +8,7 @@ from sglang.test.ascend.e2e.test_npu_performance_utils import (
|
|||||||
)
|
)
|
||||||
from sglang.test.ci.ci_register import register_npu_ci
|
from sglang.test.ci.ci_register import register_npu_ci
|
||||||
|
|
||||||
register_npu_ci(
|
register_npu_ci(est_time=3600, suite="base-c-test-acc-2-npu-a3")
|
||||||
est_time=3600,
|
|
||||||
suite="stage-b-test-2-npu-a3",
|
|
||||||
nightly=True,
|
|
||||||
disabled="accuracy testcase",
|
|
||||||
)
|
|
||||||
|
|
||||||
QWEN3_5_9B_ENVS = {
|
QWEN3_5_9B_ENVS = {
|
||||||
"SGLANG_SET_CPU_AFFINITY": "1",
|
"SGLANG_SET_CPU_AFFINITY": "1",
|
||||||
|
|||||||
+1
-6
@@ -8,12 +8,7 @@ from sglang.test.ascend.e2e.test_npu_performance_utils import (
|
|||||||
)
|
)
|
||||||
from sglang.test.ci.ci_register import register_npu_ci
|
from sglang.test.ci.ci_register import register_npu_ci
|
||||||
|
|
||||||
register_npu_ci(
|
register_npu_ci(est_time=3600, suite="base-c-test-acc-4-npu-a3")
|
||||||
est_time=3600,
|
|
||||||
suite="stage-b-test-4-npu-a3",
|
|
||||||
nightly=True,
|
|
||||||
disabled="accuracy testcase",
|
|
||||||
)
|
|
||||||
|
|
||||||
QWEN3_VL_30B_A3B_ENVS = {
|
QWEN3_VL_30B_A3B_ENVS = {
|
||||||
"SGLANG_SET_CPU_AFFINITY": "1",
|
"SGLANG_SET_CPU_AFFINITY": "1",
|
||||||
|
|||||||
@@ -8,12 +8,7 @@ from sglang.test.ascend.e2e.test_npu_performance_utils import (
|
|||||||
)
|
)
|
||||||
from sglang.test.ci.ci_register import register_npu_ci
|
from sglang.test.ci.ci_register import register_npu_ci
|
||||||
|
|
||||||
register_npu_ci(
|
register_npu_ci(est_time=3600, suite="base-c-test-acc-4-npu-a3")
|
||||||
est_time=3600,
|
|
||||||
suite="stage-b-test-4-npu-a3",
|
|
||||||
nightly=True,
|
|
||||||
disabled="accuracy testcase",
|
|
||||||
)
|
|
||||||
|
|
||||||
QWEN3_VL_8B_ENVS = {
|
QWEN3_VL_8B_ENVS = {
|
||||||
"SGLANG_SET_CPU_AFFINITY": "1",
|
"SGLANG_SET_CPU_AFFINITY": "1",
|
||||||
|
|||||||
+1
-6
@@ -9,12 +9,7 @@ from sglang.test.ascend.e2e.test_npu_performance_utils import (
|
|||||||
)
|
)
|
||||||
from sglang.test.ci.ci_register import register_npu_ci
|
from sglang.test.ci.ci_register import register_npu_ci
|
||||||
|
|
||||||
register_npu_ci(
|
register_npu_ci(est_time=3600, suite="base-c-test-perf-2-npu-a3")
|
||||||
est_time=3600,
|
|
||||||
suite="",
|
|
||||||
nightly=True,
|
|
||||||
disabled="performance testcase",
|
|
||||||
)
|
|
||||||
|
|
||||||
_is_pr_pipeline = os.environ.get("GITHUB_EVENT_NAME") == "pull_request"
|
_is_pr_pipeline = os.environ.get("GITHUB_EVENT_NAME") == "pull_request"
|
||||||
|
|
||||||
|
|||||||
@@ -13,7 +13,7 @@ from sglang.test.test_utils import (
|
|||||||
popen_launch_server,
|
popen_launch_server,
|
||||||
)
|
)
|
||||||
|
|
||||||
register_npu_ci(est_time=400, suite="stage-b-test-1-npu-a3", nightly=False)
|
register_npu_ci(est_time=400, suite="base-b-test-1-npu-a3")
|
||||||
register_npu_ci(est_time=400, suite="nightly-1-npu-a3", nightly=True)
|
register_npu_ci(est_time=400, suite="nightly-1-npu-a3", nightly=True)
|
||||||
|
|
||||||
TEST_MODEL_MATRIX = {
|
TEST_MODEL_MATRIX = {
|
||||||
|
|||||||
@@ -13,7 +13,7 @@ from sglang.test.test_utils import (
|
|||||||
popen_launch_server,
|
popen_launch_server,
|
||||||
)
|
)
|
||||||
|
|
||||||
register_npu_ci(est_time=400, suite="stage-b-test-4-npu-a3", nightly=False)
|
register_npu_ci(est_time=400, suite="base-b-test-4-npu-a3")
|
||||||
register_npu_ci(est_time=400, suite="nightly-4-npu-a3", nightly=True)
|
register_npu_ci(est_time=400, suite="nightly-4-npu-a3", nightly=True)
|
||||||
|
|
||||||
TEST_MODEL_MATRIX = {
|
TEST_MODEL_MATRIX = {
|
||||||
|
|||||||
@@ -14,7 +14,7 @@ from sglang.test.test_utils import (
|
|||||||
popen_launch_server,
|
popen_launch_server,
|
||||||
)
|
)
|
||||||
|
|
||||||
register_npu_ci(est_time=400, suite="stage-b-test-1-npu-a3", nightly=False)
|
register_npu_ci(est_time=400, suite="base-b-test-1-npu-a3")
|
||||||
register_npu_ci(est_time=400, suite="nightly-1-npu-a3", nightly=True)
|
register_npu_ci(est_time=400, suite="nightly-1-npu-a3", nightly=True)
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -10,7 +10,7 @@ from sglang.test.test_utils import (
|
|||||||
write_github_step_summary,
|
write_github_step_summary,
|
||||||
)
|
)
|
||||||
|
|
||||||
register_npu_ci(est_time=400, suite="stage-b-test-4-npu-a3", nightly=False)
|
register_npu_ci(est_time=400, suite="base-b-test-4-npu-a3")
|
||||||
register_npu_ci(est_time=400, suite="nightly-1-npu-a3", nightly=True)
|
register_npu_ci(est_time=400, suite="nightly-1-npu-a3", nightly=True)
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -23,7 +23,7 @@ from sglang.test.test_utils import (
|
|||||||
popen_launch_server,
|
popen_launch_server,
|
||||||
)
|
)
|
||||||
|
|
||||||
register_npu_ci(est_time=400, suite="stage-b-test-4-npu-a3", nightly=False)
|
register_npu_ci(est_time=400, suite="base-b-test-4-npu-a3")
|
||||||
register_npu_ci(est_time=400, suite="nightly-4-npu-a3", nightly=True)
|
register_npu_ci(est_time=400, suite="nightly-4-npu-a3", nightly=True)
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
+1
-1
@@ -14,7 +14,7 @@ from sglang.test.test_utils import (
|
|||||||
popen_launch_server,
|
popen_launch_server,
|
||||||
)
|
)
|
||||||
|
|
||||||
register_npu_ci(est_time=400, suite="stage-b-test-1-npu-a3", nightly=False)
|
register_npu_ci(est_time=400, suite="base-b-test-1-npu-a3")
|
||||||
register_npu_ci(est_time=400, suite="nightly-1-npu-a3", nightly=True)
|
register_npu_ci(est_time=400, suite="nightly-1-npu-a3", nightly=True)
|
||||||
|
|
||||||
TEST_MODEL_MATRIX = {
|
TEST_MODEL_MATRIX = {
|
||||||
|
|||||||
+1
-1
@@ -12,7 +12,7 @@ from sglang.test.test_utils import (
|
|||||||
run_bench_one_batch,
|
run_bench_one_batch,
|
||||||
)
|
)
|
||||||
|
|
||||||
register_npu_ci(est_time=400, suite="stage-b-test-1-npu-a3", nightly=False)
|
register_npu_ci(est_time=400, suite="base-b-test-1-npu-a3")
|
||||||
register_npu_ci(est_time=400, suite="nightly-1-npu-a3", nightly=True)
|
register_npu_ci(est_time=400, suite="nightly-1-npu-a3", nightly=True)
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
+1
-1
@@ -18,7 +18,7 @@ from sglang.test.test_utils import (
|
|||||||
popen_launch_server,
|
popen_launch_server,
|
||||||
)
|
)
|
||||||
|
|
||||||
register_npu_ci(est_time=700, suite="stage-b-test-16-npu-a3", nightly=False)
|
register_npu_ci(est_time=700, suite="base-b-test-16-npu-a3")
|
||||||
register_npu_ci(est_time=700, suite="nightly-16-npu-a3", nightly=True)
|
register_npu_ci(est_time=700, suite="nightly-16-npu-a3", nightly=True)
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
+1
-1
@@ -23,7 +23,7 @@ from sglang.test.test_utils import (
|
|||||||
popen_launch_pd_server,
|
popen_launch_pd_server,
|
||||||
)
|
)
|
||||||
|
|
||||||
register_npu_ci(est_time=3600, suite="stage-b-test-16-npu-a3", nightly=False)
|
register_npu_ci(est_time=3600, suite="base-b-test-16-npu-a3")
|
||||||
register_npu_ci(est_time=3600, suite="nightly-16-npu-a3", nightly=True)
|
register_npu_ci(est_time=3600, suite="nightly-16-npu-a3", nightly=True)
|
||||||
|
|
||||||
load_balance_method_options = [
|
load_balance_method_options = [
|
||||||
|
|||||||
+1
-1
@@ -13,7 +13,7 @@ from sglang.test.test_utils import (
|
|||||||
popen_launch_server,
|
popen_launch_server,
|
||||||
)
|
)
|
||||||
|
|
||||||
register_npu_ci(est_time=400, suite="stage-b-test-16-npu-a3", nightly=False)
|
register_npu_ci(est_time=400, suite="base-b-test-16-npu-a3")
|
||||||
register_npu_ci(est_time=400, suite="nightly-16-npu-a3", nightly=True)
|
register_npu_ci(est_time=400, suite="nightly-16-npu-a3", nightly=True)
|
||||||
|
|
||||||
TEST_MODEL_MATRIX = {
|
TEST_MODEL_MATRIX = {
|
||||||
|
|||||||
+1
-1
@@ -14,7 +14,7 @@ from sglang.test.test_utils import (
|
|||||||
popen_launch_server,
|
popen_launch_server,
|
||||||
)
|
)
|
||||||
|
|
||||||
register_npu_ci(est_time=400, suite="stage-b-test-8-npu-a3", nightly=False)
|
register_npu_ci(est_time=400, suite="base-b-test-8-npu-a3")
|
||||||
register_npu_ci(est_time=400, suite="nightly-8-npu-a3", nightly=True)
|
register_npu_ci(est_time=400, suite="nightly-8-npu-a3", nightly=True)
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
+1
-1
@@ -17,7 +17,7 @@ from sglang.test.test_utils import (
|
|||||||
popen_launch_server,
|
popen_launch_server,
|
||||||
)
|
)
|
||||||
|
|
||||||
register_npu_ci(est_time=400, suite="stage-b-test-2-npu-a3", nightly=False)
|
register_npu_ci(est_time=400, suite="base-b-test-2-npu-a3")
|
||||||
register_npu_ci(est_time=400, suite="nightly-2-npu-a3", nightly=True)
|
register_npu_ci(est_time=400, suite="nightly-2-npu-a3", nightly=True)
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
+1
-1
@@ -13,7 +13,7 @@ from sglang.test.test_utils import (
|
|||||||
popen_launch_server,
|
popen_launch_server,
|
||||||
)
|
)
|
||||||
|
|
||||||
register_npu_ci(est_time=400, suite="stage-b-test-2-npu-a3", nightly=False)
|
register_npu_ci(est_time=400, suite="base-b-test-2-npu-a3")
|
||||||
register_npu_ci(est_time=400, suite="nightly-2-npu-a3", nightly=True)
|
register_npu_ci(est_time=400, suite="nightly-2-npu-a3", nightly=True)
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
+1
-1
@@ -12,7 +12,7 @@ from sglang.test.test_utils import (
|
|||||||
popen_launch_server,
|
popen_launch_server,
|
||||||
)
|
)
|
||||||
|
|
||||||
register_npu_ci(est_time=400, suite="stage-b-test-2-npu-a3", nightly=False)
|
register_npu_ci(est_time=400, suite="base-b-test-2-npu-a3")
|
||||||
register_npu_ci(est_time=400, suite="nightly-2-npu-a3", nightly=True)
|
register_npu_ci(est_time=400, suite="nightly-2-npu-a3", nightly=True)
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -21,7 +21,7 @@ from sglang.test.test_utils import (
|
|||||||
popen_with_error_check,
|
popen_with_error_check,
|
||||||
)
|
)
|
||||||
|
|
||||||
register_npu_ci(est_time=400, suite="stage-b-test-16-npu-a3", nightly=False)
|
register_npu_ci(est_time=400, suite="base-b-test-16-npu-a3")
|
||||||
register_npu_ci(est_time=400, suite="nightly-16-npu-a3", nightly=True)
|
register_npu_ci(est_time=400, suite="nightly-16-npu-a3", nightly=True)
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -15,7 +15,7 @@ from sglang.test.test_utils import (
|
|||||||
popen_launch_server,
|
popen_launch_server,
|
||||||
)
|
)
|
||||||
|
|
||||||
register_npu_ci(est_time=400, suite="stage-b-test-1-npu-a3", nightly=False)
|
register_npu_ci(est_time=400, suite="base-b-test-1-npu-a3")
|
||||||
register_npu_ci(est_time=400, suite="nightly-1-npu-a3", nightly=True)
|
register_npu_ci(est_time=400, suite="nightly-1-npu-a3", nightly=True)
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
|
|||||||
@@ -16,7 +16,7 @@ from sglang.test.test_utils import (
|
|||||||
popen_launch_server,
|
popen_launch_server,
|
||||||
)
|
)
|
||||||
|
|
||||||
register_npu_ci(est_time=400, suite="stage-b-test-1-npu-a3", nightly=False)
|
register_npu_ci(est_time=400, suite="base-b-test-1-npu-a3")
|
||||||
register_npu_ci(est_time=400, suite="nightly-1-npu-a3", nightly=True)
|
register_npu_ci(est_time=400, suite="nightly-1-npu-a3", nightly=True)
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
|
|||||||
@@ -17,7 +17,7 @@ from sglang.test.test_utils import (
|
|||||||
popen_launch_server,
|
popen_launch_server,
|
||||||
)
|
)
|
||||||
|
|
||||||
register_npu_ci(est_time=400, suite="stage-b-test-1-npu-a3", nightly=False)
|
register_npu_ci(est_time=400, suite="base-b-test-1-npu-a3")
|
||||||
register_npu_ci(est_time=400, suite="nightly-1-npu-a3", nightly=True)
|
register_npu_ci(est_time=400, suite="nightly-1-npu-a3", nightly=True)
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
|
|||||||
@@ -13,7 +13,7 @@ from sglang.test.ascend.test_ascend_utils import (
|
|||||||
from sglang.test.ci.ci_register import register_npu_ci
|
from sglang.test.ci.ci_register import register_npu_ci
|
||||||
from sglang.test.test_utils import CustomTestCase
|
from sglang.test.test_utils import CustomTestCase
|
||||||
|
|
||||||
register_npu_ci(est_time=400, suite="stage-b-test-4-npu-a3", nightly=False)
|
register_npu_ci(est_time=400, suite="base-b-test-4-npu-a3")
|
||||||
register_npu_ci(est_time=400, suite="nightly-4-npu-a3", nightly=True)
|
register_npu_ci(est_time=400, suite="nightly-4-npu-a3", nightly=True)
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -14,7 +14,7 @@ from sglang.test.test_utils import (
|
|||||||
popen_launch_server,
|
popen_launch_server,
|
||||||
)
|
)
|
||||||
|
|
||||||
register_npu_ci(est_time=400, suite="stage-b-test-2-npu-a3", nightly=False)
|
register_npu_ci(est_time=400, suite="base-b-test-2-npu-a3")
|
||||||
register_npu_ci(est_time=400, suite="nightly-2-npu-a3", nightly=True)
|
register_npu_ci(est_time=400, suite="nightly-2-npu-a3", nightly=True)
|
||||||
|
|
||||||
TEST_MODEL_MATRIX = {
|
TEST_MODEL_MATRIX = {
|
||||||
|
|||||||
@@ -13,7 +13,7 @@ from sglang.test.test_utils import (
|
|||||||
popen_launch_server,
|
popen_launch_server,
|
||||||
)
|
)
|
||||||
|
|
||||||
register_npu_ci(est_time=400, suite="stage-b-test-4-npu-a3", nightly=False)
|
register_npu_ci(est_time=400, suite="base-b-test-4-npu-a3")
|
||||||
register_npu_ci(est_time=400, suite="nightly-4-npu-a3", nightly=True)
|
register_npu_ci(est_time=400, suite="nightly-4-npu-a3", nightly=True)
|
||||||
|
|
||||||
TEST_MODEL_MATRIX = {
|
TEST_MODEL_MATRIX = {
|
||||||
|
|||||||
@@ -13,7 +13,7 @@ from sglang.test.test_utils import (
|
|||||||
popen_launch_server,
|
popen_launch_server,
|
||||||
)
|
)
|
||||||
|
|
||||||
register_npu_ci(est_time=400, suite="stage-b-test-1-npu-a3", nightly=False)
|
register_npu_ci(est_time=400, suite="base-b-test-1-npu-a3")
|
||||||
register_npu_ci(est_time=400, suite="nightly-1-npu-a3", nightly=True)
|
register_npu_ci(est_time=400, suite="nightly-1-npu-a3", nightly=True)
|
||||||
|
|
||||||
TEST_MODEL_MATRIX = {
|
TEST_MODEL_MATRIX = {
|
||||||
|
|||||||
@@ -13,7 +13,7 @@ from sglang.test.test_utils import (
|
|||||||
popen_launch_server,
|
popen_launch_server,
|
||||||
)
|
)
|
||||||
|
|
||||||
register_npu_ci(est_time=400, suite="stage-b-test-2-npu-a3", nightly=False)
|
register_npu_ci(est_time=400, suite="base-b-test-2-npu-a3")
|
||||||
register_npu_ci(est_time=400, suite="nightly-2-npu-a3", nightly=True)
|
register_npu_ci(est_time=400, suite="nightly-2-npu-a3", nightly=True)
|
||||||
|
|
||||||
TEST_MODEL_MATRIX = {
|
TEST_MODEL_MATRIX = {
|
||||||
|
|||||||
@@ -14,7 +14,7 @@ from sglang.test.test_utils import (
|
|||||||
popen_launch_server,
|
popen_launch_server,
|
||||||
)
|
)
|
||||||
|
|
||||||
register_npu_ci(est_time=400, suite="stage-b-test-2-npu-a3", nightly=False)
|
register_npu_ci(est_time=400, suite="base-b-test-2-npu-a3")
|
||||||
register_npu_ci(est_time=400, suite="nightly-2-npu-a3", nightly=True)
|
register_npu_ci(est_time=400, suite="nightly-2-npu-a3", nightly=True)
|
||||||
|
|
||||||
TEST_MODEL_MATRIX = {
|
TEST_MODEL_MATRIX = {
|
||||||
|
|||||||
@@ -12,7 +12,7 @@ from sglang.test.test_utils import (
|
|||||||
popen_launch_server,
|
popen_launch_server,
|
||||||
)
|
)
|
||||||
|
|
||||||
register_npu_ci(est_time=400, suite="stage-b-test-4-npu-a3", nightly=False)
|
register_npu_ci(est_time=400, suite="base-b-test-4-npu-a3")
|
||||||
register_npu_ci(est_time=400, suite="nightly-4-npu-a3", nightly=True)
|
register_npu_ci(est_time=400, suite="nightly-4-npu-a3", nightly=True)
|
||||||
|
|
||||||
TEST_MODEL_MATRIX = {
|
TEST_MODEL_MATRIX = {
|
||||||
|
|||||||
+1
-1
@@ -17,7 +17,7 @@ from sglang.test.test_utils import (
|
|||||||
popen_launch_server,
|
popen_launch_server,
|
||||||
)
|
)
|
||||||
|
|
||||||
register_npu_ci(est_time=200, suite="stage-b-test-1-npu-a3", nightly=False)
|
register_npu_ci(est_time=200, suite="base-b-test-1-npu-a3")
|
||||||
register_npu_ci(est_time=200, suite="nightly-1-npu-a3", nightly=True)
|
register_npu_ci(est_time=200, suite="nightly-1-npu-a3", nightly=True)
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -9,7 +9,7 @@ from sglang.test.ascend.test_ascend_utils import (
|
|||||||
from sglang.test.ci.ci_register import register_npu_ci
|
from sglang.test.ci.ci_register import register_npu_ci
|
||||||
from sglang.test.test_utils import CustomTestCase
|
from sglang.test.test_utils import CustomTestCase
|
||||||
|
|
||||||
register_npu_ci(est_time=400, suite="stage-b-test-1-npu-a3", nightly=False)
|
register_npu_ci(est_time=400, suite="base-b-test-1-npu-a3")
|
||||||
register_npu_ci(est_time=400, suite="nightly-1-npu-a3", nightly=True)
|
register_npu_ci(est_time=400, suite="nightly-1-npu-a3", nightly=True)
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
+1
-5
@@ -17,11 +17,7 @@ from sglang.test.test_utils import (
|
|||||||
popen_launch_server,
|
popen_launch_server,
|
||||||
)
|
)
|
||||||
|
|
||||||
register_npu_ci(
|
register_npu_ci(est_time=400, suite="base-b-test-4-npu-a3")
|
||||||
est_time=400,
|
|
||||||
suite="stage-b-test-4-npu-a3",
|
|
||||||
nightly=False,
|
|
||||||
)
|
|
||||||
register_npu_ci(
|
register_npu_ci(
|
||||||
est_time=400,
|
est_time=400,
|
||||||
suite="nightly-4-npu-a3",
|
suite="nightly-4-npu-a3",
|
||||||
|
|||||||
+1
-1
@@ -16,7 +16,7 @@ from sglang.test.test_utils import (
|
|||||||
popen_launch_server,
|
popen_launch_server,
|
||||||
)
|
)
|
||||||
|
|
||||||
register_npu_ci(est_time=400, suite="stage-b-test-16-npu-a3", nightly=False)
|
register_npu_ci(est_time=400, suite="base-b-test-16-npu-a3")
|
||||||
register_npu_ci(est_time=400, suite="nightly-16-npu-a3", nightly=True)
|
register_npu_ci(est_time=400, suite="nightly-16-npu-a3", nightly=True)
|
||||||
|
|
||||||
MODEL_PATH = DEEPSEEK_R1_0528_W4A8_PER_CHANNEL_WEIGHTS_PATH
|
MODEL_PATH = DEEPSEEK_R1_0528_W4A8_PER_CHANNEL_WEIGHTS_PATH
|
||||||
|
|||||||
+1
-1
@@ -16,7 +16,7 @@ from sglang.test.test_utils import (
|
|||||||
popen_launch_server,
|
popen_launch_server,
|
||||||
)
|
)
|
||||||
|
|
||||||
register_npu_ci(est_time=400, suite="stage-b-test-16-npu-a3", nightly=False)
|
register_npu_ci(est_time=400, suite="base-b-test-16-npu-a3")
|
||||||
register_npu_ci(est_time=400, suite="nightly-16-npu-a3", nightly=True)
|
register_npu_ci(est_time=400, suite="nightly-16-npu-a3", nightly=True)
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
+1
-1
@@ -16,7 +16,7 @@ from sglang.test.test_utils import (
|
|||||||
popen_launch_server,
|
popen_launch_server,
|
||||||
)
|
)
|
||||||
|
|
||||||
register_npu_ci(est_time=400, suite="stage-b-test-4-npu-a3", nightly=False)
|
register_npu_ci(est_time=400, suite="base-b-test-4-npu-a3")
|
||||||
register_npu_ci(est_time=400, suite="nightly-4-npu-a3", nightly=True)
|
register_npu_ci(est_time=400, suite="nightly-4-npu-a3", nightly=True)
|
||||||
|
|
||||||
_ASCEND_BACKEND = "ascend"
|
_ASCEND_BACKEND = "ascend"
|
||||||
|
|||||||
+1
-1
@@ -20,7 +20,7 @@ from sglang.test.test_utils import (
|
|||||||
popen_launch_server,
|
popen_launch_server,
|
||||||
)
|
)
|
||||||
|
|
||||||
register_npu_ci(est_time=300, suite="stage-b-test-4-npu-a3", nightly=False)
|
register_npu_ci(est_time=300, suite="base-b-test-4-npu-a3")
|
||||||
register_npu_ci(est_time=300, suite="nightly-4-npu-a3", nightly=True)
|
register_npu_ci(est_time=300, suite="nightly-4-npu-a3", nightly=True)
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
+1
-6
@@ -8,12 +8,7 @@ from sglang.test.ascend.e2e.test_npu_performance_utils import (
|
|||||||
)
|
)
|
||||||
from sglang.test.ci.ci_register import register_npu_ci
|
from sglang.test.ci.ci_register import register_npu_ci
|
||||||
|
|
||||||
register_npu_ci(
|
register_npu_ci(est_time=3600, suite="base-c-test-perf-16-npu-a3")
|
||||||
est_time=3600,
|
|
||||||
suite="",
|
|
||||||
nightly=True,
|
|
||||||
disabled="performance testcase",
|
|
||||||
)
|
|
||||||
|
|
||||||
# Environment variables for DSV4-Flash single-node PD-mix deployment.
|
# Environment variables for DSV4-Flash single-node PD-mix deployment.
|
||||||
DEEPSEEK_V4_FLASH_W8A8_8P_ENVS = {
|
DEEPSEEK_V4_FLASH_W8A8_8P_ENVS = {
|
||||||
|
|||||||
+1
-6
@@ -10,12 +10,7 @@ from sglang.test.ascend.e2e.test_npu_performance_utils import (
|
|||||||
)
|
)
|
||||||
from sglang.test.ci.ci_register import register_npu_ci
|
from sglang.test.ci.ci_register import register_npu_ci
|
||||||
|
|
||||||
register_npu_ci(
|
register_npu_ci(est_time=1800, suite="base-c-test-perf-16-npu-a3")
|
||||||
est_time=1800,
|
|
||||||
suite="full-16-npu-a3",
|
|
||||||
nightly=True,
|
|
||||||
disabled="Currently it is executed by the npu performance workflow.",
|
|
||||||
)
|
|
||||||
|
|
||||||
KIMI_K2_6_ENVS = {
|
KIMI_K2_6_ENVS = {
|
||||||
"PYTORCH_NPU_ALLOC_CONF": "expandable_segments:True",
|
"PYTORCH_NPU_ALLOC_CONF": "expandable_segments:True",
|
||||||
|
|||||||
+1
-6
@@ -9,12 +9,7 @@ from sglang.test.ascend.e2e.test_npu_performance_utils import (
|
|||||||
)
|
)
|
||||||
from sglang.test.ci.ci_register import register_npu_ci
|
from sglang.test.ci.ci_register import register_npu_ci
|
||||||
|
|
||||||
register_npu_ci(
|
register_npu_ci(est_time=3600, suite="base-c-test-perf-8-npu-a3")
|
||||||
est_time=3600,
|
|
||||||
suite="npu-performance",
|
|
||||||
nightly=True,
|
|
||||||
disabled="performance testcase",
|
|
||||||
)
|
|
||||||
|
|
||||||
MINIMAX_M2_5_W8A8_4P_IN64K_OUT1K_PREFIX90_ENVS = {
|
MINIMAX_M2_5_W8A8_4P_IN64K_OUT1K_PREFIX90_ENVS = {
|
||||||
"PYTORCH_NPU_ALLOC_CONF": "expandable_segments:True",
|
"PYTORCH_NPU_ALLOC_CONF": "expandable_segments:True",
|
||||||
|
|||||||
+1
-6
@@ -9,12 +9,7 @@ from sglang.test.ascend.e2e.test_npu_performance_utils import (
|
|||||||
)
|
)
|
||||||
from sglang.test.ci.ci_register import register_npu_ci
|
from sglang.test.ci.ci_register import register_npu_ci
|
||||||
|
|
||||||
register_npu_ci(
|
register_npu_ci(est_time=3600, suite="base-c-test-perf-2-npu-a3")
|
||||||
est_time=3600,
|
|
||||||
suite="",
|
|
||||||
nightly=True,
|
|
||||||
disabled="performance testcase",
|
|
||||||
)
|
|
||||||
|
|
||||||
QWEN3_8B_ENVS = {
|
QWEN3_8B_ENVS = {
|
||||||
"SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT": "600",
|
"SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT": "600",
|
||||||
|
|||||||
+1
-6
@@ -9,12 +9,7 @@ from sglang.test.ascend.e2e.test_npu_performance_utils import (
|
|||||||
)
|
)
|
||||||
from sglang.test.ci.ci_register import register_npu_ci
|
from sglang.test.ci.ci_register import register_npu_ci
|
||||||
|
|
||||||
register_npu_ci(
|
register_npu_ci(est_time=3600, suite="base-c-test-perf-16-npu-a3")
|
||||||
est_time=3600,
|
|
||||||
suite="",
|
|
||||||
nightly=True,
|
|
||||||
disabled="performance testcase",
|
|
||||||
)
|
|
||||||
|
|
||||||
QWEN3_235B_ENVS = {
|
QWEN3_235B_ENVS = {
|
||||||
"PYTORCH_NPU_ALLOC_CONF": "expandable_segments:True",
|
"PYTORCH_NPU_ALLOC_CONF": "expandable_segments:True",
|
||||||
|
|||||||
+1
-6
@@ -9,12 +9,7 @@ from sglang.test.ascend.e2e.test_npu_performance_utils import (
|
|||||||
)
|
)
|
||||||
from sglang.test.ci.ci_register import register_npu_ci
|
from sglang.test.ci.ci_register import register_npu_ci
|
||||||
|
|
||||||
register_npu_ci(
|
register_npu_ci(est_time=3600, suite="base-c-test-perf-2-npu-a3")
|
||||||
est_time=3600,
|
|
||||||
suite="",
|
|
||||||
nightly=True,
|
|
||||||
disabled="performance testcase",
|
|
||||||
)
|
|
||||||
|
|
||||||
QWEN3_30B_A3B_ENVS = {
|
QWEN3_30B_A3B_ENVS = {
|
||||||
"ASCEND_LAUNCH_BLOCKING": "0",
|
"ASCEND_LAUNCH_BLOCKING": "0",
|
||||||
|
|||||||
+1
-6
@@ -9,12 +9,7 @@ from sglang.test.ascend.e2e.test_npu_performance_utils import (
|
|||||||
)
|
)
|
||||||
from sglang.test.ci.ci_register import register_npu_ci
|
from sglang.test.ci.ci_register import register_npu_ci
|
||||||
|
|
||||||
register_npu_ci(
|
register_npu_ci(est_time=3600, suite="base-c-test-perf-4-npu-a3")
|
||||||
est_time=3600,
|
|
||||||
suite="",
|
|
||||||
nightly=True,
|
|
||||||
disabled="performance testcase",
|
|
||||||
)
|
|
||||||
|
|
||||||
QWEN3_32B_ENVS = {
|
QWEN3_32B_ENVS = {
|
||||||
"SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT": "600",
|
"SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT": "600",
|
||||||
|
|||||||
+1
-6
@@ -8,12 +8,7 @@ from sglang.test.ascend.e2e.test_npu_performance_utils import (
|
|||||||
)
|
)
|
||||||
from sglang.test.ci.ci_register import register_npu_ci
|
from sglang.test.ci.ci_register import register_npu_ci
|
||||||
|
|
||||||
register_npu_ci(
|
register_npu_ci(est_time=3600, suite="base-c-test-perf-16-npu-a3")
|
||||||
est_time=3600,
|
|
||||||
suite="nightly-16-npu-a3",
|
|
||||||
nightly=True,
|
|
||||||
disabled="performance testcase",
|
|
||||||
)
|
|
||||||
|
|
||||||
QWEN3_5_397B_A17B_ENVS = {
|
QWEN3_5_397B_A17B_ENVS = {
|
||||||
"PYTORCH_NPU_ALLOC_CONF": "expandable_segments:True",
|
"PYTORCH_NPU_ALLOC_CONF": "expandable_segments:True",
|
||||||
|
|||||||
+1
-6
@@ -8,12 +8,7 @@ from sglang.test.ascend.e2e.test_npu_performance_utils import (
|
|||||||
)
|
)
|
||||||
from sglang.test.ci.ci_register import register_npu_ci
|
from sglang.test.ci.ci_register import register_npu_ci
|
||||||
|
|
||||||
register_npu_ci(
|
register_npu_ci(est_time=3600, suite="base-c-test-perf-2-npu-a3")
|
||||||
est_time=3600,
|
|
||||||
suite="full-2-npu-a3",
|
|
||||||
nightly=True,
|
|
||||||
disabled="performance case",
|
|
||||||
)
|
|
||||||
|
|
||||||
QWEN3_6_27B_1024_ENVS = {
|
QWEN3_6_27B_1024_ENVS = {
|
||||||
"PYTORCH_NPU_ALLOC_CONF": "expandable_segments:True",
|
"PYTORCH_NPU_ALLOC_CONF": "expandable_segments:True",
|
||||||
|
|||||||
+1
-6
@@ -8,12 +8,7 @@ from sglang.test.ascend.e2e.test_npu_performance_utils import (
|
|||||||
)
|
)
|
||||||
from sglang.test.ci.ci_register import register_npu_ci
|
from sglang.test.ci.ci_register import register_npu_ci
|
||||||
|
|
||||||
register_npu_ci(
|
register_npu_ci(est_time=3600, suite="base-c-test-perf-2-npu-a3")
|
||||||
est_time=3600,
|
|
||||||
suite="",
|
|
||||||
nightly=True,
|
|
||||||
disabled="performance testcase",
|
|
||||||
)
|
|
||||||
|
|
||||||
QWEN3_6_27B_64K_1K_ENVS = {
|
QWEN3_6_27B_64K_1K_ENVS = {
|
||||||
"STREAMS_PER_DEVICE": "32",
|
"STREAMS_PER_DEVICE": "32",
|
||||||
|
|||||||
+1
-6
@@ -8,12 +8,7 @@ from sglang.test.ascend.e2e.test_npu_performance_utils import (
|
|||||||
)
|
)
|
||||||
from sglang.test.ci.ci_register import register_npu_ci
|
from sglang.test.ci.ci_register import register_npu_ci
|
||||||
|
|
||||||
register_npu_ci(
|
register_npu_ci(est_time=3600, suite="base-c-test-perf-2-npu-a3")
|
||||||
est_time=3600,
|
|
||||||
suite="",
|
|
||||||
nightly=True,
|
|
||||||
disabled="performance testcase",
|
|
||||||
)
|
|
||||||
|
|
||||||
QWEN3_6_35B_A3B_64K_PREFIX_ENVS = {
|
QWEN3_6_35B_A3B_64K_PREFIX_ENVS = {
|
||||||
"PYTORCH_NPU_ALLOC_CONF": "expandable_segments:True",
|
"PYTORCH_NPU_ALLOC_CONF": "expandable_segments:True",
|
||||||
|
|||||||
+1
-6
@@ -7,12 +7,7 @@ from sglang.test.ascend.e2e.test_npu_performance_utils import (
|
|||||||
)
|
)
|
||||||
from sglang.test.ci.ci_register import register_npu_ci
|
from sglang.test.ci.ci_register import register_npu_ci
|
||||||
|
|
||||||
register_npu_ci(
|
register_npu_ci(est_time=3600, suite="base-c-test-perf-4-npu-a3")
|
||||||
est_time=3600,
|
|
||||||
suite="",
|
|
||||||
nightly=True,
|
|
||||||
disabled="performance testcase",
|
|
||||||
)
|
|
||||||
|
|
||||||
QWEN3_NEXT_80B_A3B_ENVS = {
|
QWEN3_NEXT_80B_A3B_ENVS = {
|
||||||
"PYTORCH_NPU_ALLOC_CONF": "expandable_segments:True",
|
"PYTORCH_NPU_ALLOC_CONF": "expandable_segments:True",
|
||||||
|
|||||||
@@ -11,7 +11,7 @@ from sglang.srt.layers.attention import vision
|
|||||||
from sglang.test.ci.ci_register import register_cpu_ci, register_npu_ci
|
from sglang.test.ci.ci_register import register_cpu_ci, register_npu_ci
|
||||||
|
|
||||||
register_cpu_ci(est_time=2, suite="base-a-test-cpu")
|
register_cpu_ci(est_time=2, suite="base-a-test-cpu")
|
||||||
register_npu_ci(est_time=2, suite="stage-b-test-1-npu-a3")
|
register_npu_ci(est_time=2, suite="base-b-test-1-npu-a3")
|
||||||
register_npu_ci(est_time=2, suite="nightly-1-npu-a3", nightly=True)
|
register_npu_ci(est_time=2, suite="nightly-1-npu-a3", nightly=True)
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -12,7 +12,7 @@ import torch
|
|||||||
|
|
||||||
from sglang.test.ci.ci_register import register_npu_ci
|
from sglang.test.ci.ci_register import register_npu_ci
|
||||||
|
|
||||||
register_npu_ci(est_time=5, suite="stage-a-unit-test-npu")
|
register_npu_ci(est_time=5, suite="base-a-test-1-npu-a2")
|
||||||
|
|
||||||
# Mock NPU-only modules before importing the source module.
|
# Mock NPU-only modules before importing the source module.
|
||||||
for _ in (
|
for _ in (
|
||||||
|
|||||||
@@ -12,7 +12,7 @@ import torch
|
|||||||
|
|
||||||
from sglang.test.ci.ci_register import register_npu_ci
|
from sglang.test.ci.ci_register import register_npu_ci
|
||||||
|
|
||||||
register_npu_ci(est_time=4, suite="stage-a-unit-test-npu")
|
register_npu_ci(est_time=4, suite="base-a-test-1-npu-a2")
|
||||||
|
|
||||||
for mod in (
|
for mod in (
|
||||||
"torch_npu",
|
"torch_npu",
|
||||||
|
|||||||
+14
-7
@@ -93,13 +93,20 @@ PER_COMMIT_SUITES = {
|
|||||||
"extra-b-test-8-gpu-h200",
|
"extra-b-test-8-gpu-h200",
|
||||||
],
|
],
|
||||||
HWBackend.NPU: [
|
HWBackend.NPU: [
|
||||||
"base-a-test-1-gpu-small",
|
"base-a-test-1-npu-a2",
|
||||||
"stage-a-unit-test-npu",
|
"base-b-test-1-npu-a3",
|
||||||
"stage-b-test-1-npu-a3",
|
"base-b-test-2-npu-a3",
|
||||||
"stage-b-test-2-npu-a3",
|
"base-b-test-4-npu-a3",
|
||||||
"stage-b-test-4-npu-a3",
|
"base-b-test-8-npu-a3",
|
||||||
"stage-b-test-8-npu-a3",
|
"base-b-test-16-npu-a3",
|
||||||
"stage-b-test-16-npu-a3",
|
"base-c-test-acc-2-npu-a3",
|
||||||
|
"base-c-test-acc-4-npu-a3",
|
||||||
|
"base-c-test-acc-8-npu-a3",
|
||||||
|
"base-c-test-acc-16-npu-a3",
|
||||||
|
"base-c-test-perf-2-npu-a3",
|
||||||
|
"base-c-test-perf-4-npu-a3",
|
||||||
|
"base-c-test-perf-8-npu-a3",
|
||||||
|
"base-c-test-perf-16-npu-a3",
|
||||||
],
|
],
|
||||||
HWBackend.XPU: [
|
HWBackend.XPU: [
|
||||||
"stage-a-test-1-gpu-xpu",
|
"stage-a-test-1-gpu-xpu",
|
||||||
|
|||||||
Reference in New Issue
Block a user