[NPU][CI] Constrain evalscope dependency versions (#40133)

This commit is contained in:
pllimax
2026-09-22 20:15:11 +08:00
committed by GitHub
parent bc3a63ebdb
commit 244db08d60
3 changed files with 251 additions and 9 deletions
+107 -2
View File
@@ -39,7 +39,7 @@ on:
type: string
default: ''
skip_pr_test_health_check:
description: 'Set to true to skip the PR test health check (fail-fast gate).'
description: 'Set to true to skip the PR test health check (fast-fail gate).'
type: string
default: 'false'
is_nightly_pipeline_job:
@@ -90,7 +90,112 @@ jobs:
run: |
git config --system --add safe.directory ${GITHUB_WORKSPACE}
- uses: ./.github/actions/check-pr-test-health
- name: Check PR test health
uses: actions/github-script@v8
env:
SKIP_PR_TEST_HEALTH_CHECK: ${{ env.SKIP_PR_TEST_HEALTH_CHECK }}
with:
github-token: ${{ inputs.github-token || github.token }}
script: |
core.notice(`[health-check] START — event=${context.eventName}, runId=${context.runId}`);
// Skip when explicitly requested via env var (e.g. release branch cut)
if (process.env.SKIP_PR_TEST_HEALTH_CHECK === 'true') {
core.notice('[health-check] SKIP: SKIP_PR_TEST_HEALTH_CHECK=true');
return;
}
// Skip for scheduled runs — they should collect all failures, not fast-fail
if (context.eventName === 'schedule') {
core.notice('[health-check] SKIP: scheduled run');
return;
}
// Check lint status from the separate Lint workflow (lint.yml).
// listJobsForWorkflowRun only sees jobs within the SAME run, so we use
// checks.listForRef which queries by commit SHA across ALL workflows.
const ref = context.payload.pull_request?.head?.sha || context.sha;
core.info(`[health-check] Checking lint for ref=${ref}`);
const { data } = await github.rest.checks.listForRef({
owner: context.repo.owner,
repo: context.repo.repo,
ref: ref,
check_name: 'lint',
});
const lintRun = data.check_runs.find(
cr => cr.app?.slug === 'github-actions'
);
core.info(`[health-check] Lint check: status=${lintRun?.status}, conclusion=${lintRun?.conclusion}`);
if (lintRun?.status === 'completed' && lintRun?.conclusion === 'failure') {
core.setFailed('Fast-fail: lint check failed');
return;
}
// Skip the jobs-failed check when the PR carries the bypass-fastfail label.
// Lint check above still runs.
let labels = [];
if (context.payload.pull_request?.labels) {
labels = context.payload.pull_request.labels.map(l => l.name);
} else {
const { data: prs } = await github.rest.repos.listPullRequestsAssociatedWithCommit({
owner: context.repo.owner,
repo: context.repo.repo,
commit_sha: ref,
});
if (prs.length > 0) {
labels = prs[0].labels.map(l => l.name);
}
}
core.info(`[health-check] PR labels: [${labels.join(', ')}]`);
if (labels.includes('bypass-fastfail')) {
core.notice('[health-check] SKIP jobs-failed check: bypass-fastfail label present');
return;
}
const jobs = await github.paginate(github.rest.actions.listJobsForWorkflowRun, {
owner: context.repo.owner,
repo: context.repo.repo,
run_id: context.runId,
per_page: 100,
});
core.info(`[health-check] Total jobs in run: ${jobs.length}`);
const failedJobs = jobs.filter(j => j.status === 'completed' && j.conclusion === 'failure');
core.info(`[health-check] Failed jobs (before filtering): ${failedJobs.map(j => `${j.name}(${j.conclusion})`).join(', ') || 'none'}`);
// Find jobs that failed from a real error, not from fast-fail cascade
const rootCauseFailures = jobs.filter(j => {
if (j.status !== 'completed' || j.conclusion !== 'failure') return false;
// h20 runners are flaky (dirty GPU state from prior runs); their failures
// should not cascade fast-fail to other stages. j.name shape from
// listJobsForWorkflowRun: "<job-key>" + optional " / <reusable-job>"
// + optional " (<matrix>)". Split off the base job key before exact
// match so we cover both inline + reusable forms without confusing
// 'h20' with the 'h200' prefix.
const baseName = j.name.split(/[ /]/)[0];
if (baseName === 'base-c-test-8-gpu-h20') {
core.info(`[health-check] Filtered out h20 job: ${j.name}`);
return false;
}
// multimodal-gen NPU tests should not cascade fast-fail to perf/accuracy stages.
if (baseName === 'multimodal-gen-test-1-npu-a3' || baseName === 'multimodal-gen-test-2-npu-a3') {
core.info(`[health-check] Filtered out multimodal-gen NPU job: ${j.name}`);
return false;
}
// If the failing step is the health check, it's a cascade — skip it
const failedStep = (j.steps || []).find(s => s.conclusion === 'failure');
if (failedStep && (failedStep.name.includes('check-pr-test-health') || failedStep.name.includes('Check PR test health'))) {
core.info(`[health-check] Filtered out cascade failure: ${j.name} (failed step: ${failedStep.name})`);
return false;
}
return true;
});
core.info(`[health-check] Root cause failures (after filtering): ${rootCauseFailures.map(j => j.name).join(', ') || 'none'}`);
if (rootCauseFailures.length > 0) {
core.setFailed(`Fast-fail: skipping — root cause job(s): ${rootCauseFailures.map(j => j.name).join(', ')}`);
} else {
core.notice('[health-check] PASS: no root cause failures detected');
}
- name: Install dependencies
env:
@@ -43,7 +43,7 @@ on:
default: '{}'
description: 'JSON run metadata {branch_label, workflow_name, create_time}, recorded once at workflow start'
skip_pr_test_health_check:
description: 'Set to true to skip the PR test health check (fail-fast gate).'
description: 'Set to true to skip the PR test health check (fast-fail gate).'
type: string
default: 'false'
is_nightly_pipeline_job:
@@ -101,7 +101,112 @@ jobs:
# checks out the same commit even if the branch advances mid-run.
ref: ${{ inputs.ref || github.sha }}
- uses: ./.github/actions/check-pr-test-health
- name: Check PR test health
uses: actions/github-script@v8
env:
SKIP_PR_TEST_HEALTH_CHECK: ${{ env.SKIP_PR_TEST_HEALTH_CHECK }}
with:
github-token: ${{ inputs.github-token || github.token }}
script: |
core.notice(`[health-check] START — event=${context.eventName}, runId=${context.runId}`);
// Skip when explicitly requested via env var (e.g. release branch cut)
if (process.env.SKIP_PR_TEST_HEALTH_CHECK === 'true') {
core.notice('[health-check] SKIP: SKIP_PR_TEST_HEALTH_CHECK=true');
return;
}
// Skip for scheduled runs — they should collect all failures, not fast-fail
if (context.eventName === 'schedule') {
core.notice('[health-check] SKIP: scheduled run');
return;
}
// Check lint status from the separate Lint workflow (lint.yml).
// listJobsForWorkflowRun only sees jobs within the SAME run, so we use
// checks.listForRef which queries by commit SHA across ALL workflows.
const ref = context.payload.pull_request?.head?.sha || context.sha;
core.info(`[health-check] Checking lint for ref=${ref}`);
const { data } = await github.rest.checks.listForRef({
owner: context.repo.owner,
repo: context.repo.repo,
ref: ref,
check_name: 'lint',
});
const lintRun = data.check_runs.find(
cr => cr.app?.slug === 'github-actions'
);
core.info(`[health-check] Lint check: status=${lintRun?.status}, conclusion=${lintRun?.conclusion}`);
if (lintRun?.status === 'completed' && lintRun?.conclusion === 'failure') {
core.setFailed('Fast-fail: lint check failed');
return;
}
// Skip the jobs-failed check when the PR carries the bypass-fastfail label.
// Lint check above still runs.
let labels = [];
if (context.payload.pull_request?.labels) {
labels = context.payload.pull_request.labels.map(l => l.name);
} else {
const { data: prs } = await github.rest.repos.listPullRequestsAssociatedWithCommit({
owner: context.repo.owner,
repo: context.repo.repo,
commit_sha: ref,
});
if (prs.length > 0) {
labels = prs[0].labels.map(l => l.name);
}
}
core.info(`[health-check] PR labels: [${labels.join(', ')}]`);
if (labels.includes('bypass-fastfail')) {
core.notice('[health-check] SKIP jobs-failed check: bypass-fastfail label present');
return;
}
const jobs = await github.paginate(github.rest.actions.listJobsForWorkflowRun, {
owner: context.repo.owner,
repo: context.repo.repo,
run_id: context.runId,
per_page: 100,
});
core.info(`[health-check] Total jobs in run: ${jobs.length}`);
const failedJobs = jobs.filter(j => j.status === 'completed' && j.conclusion === 'failure');
core.info(`[health-check] Failed jobs (before filtering): ${failedJobs.map(j => `${j.name}(${j.conclusion})`).join(', ') || 'none'}`);
// Find jobs that failed from a real error, not from fast-fail cascade
const rootCauseFailures = jobs.filter(j => {
if (j.status !== 'completed' || j.conclusion !== 'failure') return false;
// h20 runners are flaky (dirty GPU state from prior runs); their failures
// should not cascade fast-fail to other stages. j.name shape from
// listJobsForWorkflowRun: "<job-key>" + optional " / <reusable-job>"
// + optional " (<matrix>)". Split off the base job key before exact
// match so we cover both inline + reusable forms without confusing
// 'h20' with the 'h200' prefix.
const baseName = j.name.split(/[ /]/)[0];
if (baseName === 'base-c-test-8-gpu-h20') {
core.info(`[health-check] Filtered out h20 job: ${j.name}`);
return false;
}
// multimodal-gen NPU tests should not cascade fast-fail to perf/accuracy stages.
if (baseName === 'multimodal-gen-test-1-npu-a3' || baseName === 'multimodal-gen-test-2-npu-a3') {
core.info(`[health-check] Filtered out multimodal-gen NPU job: ${j.name}`);
return false;
}
// If the failing step is the health check, it's a cascade — skip it
const failedStep = (j.steps || []).find(s => s.conclusion === 'failure');
if (failedStep && (failedStep.name.includes('check-pr-test-health') || failedStep.name.includes('Check PR test health'))) {
core.info(`[health-check] Filtered out cascade failure: ${j.name} (failed step: ${failedStep.name})`);
return false;
}
return true;
});
core.info(`[health-check] Root cause failures (after filtering): ${rootCauseFailures.map(j => j.name).join(', ') || 'none'}`);
if (rootCauseFailures.length > 0) {
core.setFailed(`Fast-fail: skipping — root cause job(s): ${rootCauseFailures.map(j => j.name).join(', ')}`);
} else {
core.notice('[health-check] PASS: no root cause failures detected');
}
- name: Check npu info
run: |
+37 -5
View File
@@ -3,26 +3,58 @@
set -e
PYTHON_ENV_FOR_EVALSCOPE=test_env_evalscope
PYTHON_FOR_EVALSCOPE=${PYTHON_ENV_FOR_EVALSCOPE}/bin/python
PIP_FOR_EVALSCOPE=${PYTHON_ENV_FOR_EVALSCOPE}/bin/pip
EVALSCOPE_SOURCE_PATH=/root/.cache/.cache/evalscope
pip_mirror_source="https://mirrors.tuna.tsinghua.edu.cn/pypi/web/simple"
if [ -d "${PYTHON_ENV_FOR_EVALSCOPE}" ]; then
echo "Virtual env ${PYTHON_ENV_FOR_EVALSCOPE} already exists, skip installation."
# Bound key deps so the resolver cannot fall back to ancient versions.
EVALSCOPE_CONSTRAINTS=(
"aiohttp>=3.11,<4"
"httpx>=0.28,<1"
)
# Fail fast when the pip install hangs instead of timing out the whole job.
EVALSCOPE_INSTALL_TIMEOUT=1200
# Highest priority: reuse a system-wide evalscope (e.g. pre-installed in the
# image); create ${PYTHON_FOR_EVALSCOPE} with --system-site-packages when needed.
if python -c "import evalscope" >/dev/null 2>&1; then
echo "evalscope found in the system python, reuse it instead of installing."
if ! ${PYTHON_FOR_EVALSCOPE} -c "import evalscope" >/dev/null 2>&1; then
python -m venv --system-site-packages ${PYTHON_ENV_FOR_EVALSCOPE}
fi
# Only skip the install when the env really imports evalscope.
if ${PYTHON_FOR_EVALSCOPE} -c "import evalscope" >/dev/null 2>&1; then
exit 0
fi
echo "The env cannot provide evalscope, install it instead."
fi
# Otherwise reuse the virtual env when it already imports evalscope.
if [ -x "${PYTHON_FOR_EVALSCOPE}" ] && ${PYTHON_FOR_EVALSCOPE} -c "import evalscope" >/dev/null 2>&1; then
echo "evalscope already exists in ${PYTHON_ENV_FOR_EVALSCOPE}, skip installation."
exit 0
fi
echo "===== Install evalscope in virtual env - Begin ====="
python -m venv ${PYTHON_ENV_FOR_EVALSCOPE}
echo "===== Install evalscope in virtual env - Begin ====="
if [ ! -d "${EVALSCOPE_SOURCE_PATH}" ]; then
echo "The evalscope source does not exist: ${EVALSCOPE_SOURCE_PATH}."
echo "Install evalscope online."
${PIP_FOR_EVALSCOPE} install -U pip -i ${pip_mirror_source}
${PIP_FOR_EVALSCOPE} install evalscope -i ${pip_mirror_source}
timeout ${EVALSCOPE_INSTALL_TIMEOUT} ${PIP_FOR_EVALSCOPE} install evalscope "${EVALSCOPE_CONSTRAINTS[@]}" -i ${pip_mirror_source} || {
echo "ERROR: evalscope install timed out after ${EVALSCOPE_INSTALL_TIMEOUT}s."
exit 1
}
else
echo "Install evalscope from local source: ${EVALSCOPE_SOURCE_PATH}"
${PIP_FOR_EVALSCOPE} install -U pip -i ${pip_mirror_source}
${PIP_FOR_EVALSCOPE} install -e ${EVALSCOPE_SOURCE_PATH} -i ${pip_mirror_source}
timeout ${EVALSCOPE_INSTALL_TIMEOUT} ${PIP_FOR_EVALSCOPE} install -e ${EVALSCOPE_SOURCE_PATH} "${EVALSCOPE_CONSTRAINTS[@]}" -i ${pip_mirror_source} || {
echo "ERROR: evalscope install timed out after ${EVALSCOPE_INSTALL_TIMEOUT}s."
exit 1
}
fi
echo "===== Install evalscope in virtual env - End ====="