From 244db08d60cc05275f71079fba0dabc2bbde1eb9 Mon Sep 17 00:00:00 2001 From: pllimax Date: Tue, 22 Sep 2026 20:15:11 +0800 Subject: [PATCH] [NPU][CI] Constrain evalscope dependency versions (#40133) --- .github/workflows/_npu-pr-test-stage.yml | 109 +++++++++++++++++- .../workflows/_npu-single-node-test-stage.yml | 109 +++++++++++++++++- .../sglang/test/ascend/e2e/run_evalscope.sh | 42 ++++++- 3 files changed, 251 insertions(+), 9 deletions(-) diff --git a/.github/workflows/_npu-pr-test-stage.yml b/.github/workflows/_npu-pr-test-stage.yml index fed2b1317b..0bf8a37c9c 100644 --- a/.github/workflows/_npu-pr-test-stage.yml +++ b/.github/workflows/_npu-pr-test-stage.yml @@ -39,7 +39,7 @@ on: type: string default: '' skip_pr_test_health_check: - description: 'Set to true to skip the PR test health check (fail-fast gate).' + description: 'Set to true to skip the PR test health check (fast-fail gate).' type: string default: 'false' is_nightly_pipeline_job: @@ -90,7 +90,112 @@ jobs: run: | git config --system --add safe.directory ${GITHUB_WORKSPACE} - - uses: ./.github/actions/check-pr-test-health + - name: Check PR test health + uses: actions/github-script@v8 + env: + SKIP_PR_TEST_HEALTH_CHECK: ${{ env.SKIP_PR_TEST_HEALTH_CHECK }} + with: + github-token: ${{ inputs.github-token || github.token }} + script: | + core.notice(`[health-check] START — event=${context.eventName}, runId=${context.runId}`); + + // Skip when explicitly requested via env var (e.g. release branch cut) + if (process.env.SKIP_PR_TEST_HEALTH_CHECK === 'true') { + core.notice('[health-check] SKIP: SKIP_PR_TEST_HEALTH_CHECK=true'); + return; + } + + // Skip for scheduled runs — they should collect all failures, not fast-fail + if (context.eventName === 'schedule') { + core.notice('[health-check] SKIP: scheduled run'); + return; + } + + // Check lint status from the separate Lint workflow (lint.yml). + // listJobsForWorkflowRun only sees jobs within the SAME run, so we use + // checks.listForRef which queries by commit SHA across ALL workflows. + const ref = context.payload.pull_request?.head?.sha || context.sha; + core.info(`[health-check] Checking lint for ref=${ref}`); + const { data } = await github.rest.checks.listForRef({ + owner: context.repo.owner, + repo: context.repo.repo, + ref: ref, + check_name: 'lint', + }); + const lintRun = data.check_runs.find( + cr => cr.app?.slug === 'github-actions' + ); + core.info(`[health-check] Lint check: status=${lintRun?.status}, conclusion=${lintRun?.conclusion}`); + if (lintRun?.status === 'completed' && lintRun?.conclusion === 'failure') { + core.setFailed('Fast-fail: lint check failed'); + return; + } + + // Skip the jobs-failed check when the PR carries the bypass-fastfail label. + // Lint check above still runs. + let labels = []; + if (context.payload.pull_request?.labels) { + labels = context.payload.pull_request.labels.map(l => l.name); + } else { + const { data: prs } = await github.rest.repos.listPullRequestsAssociatedWithCommit({ + owner: context.repo.owner, + repo: context.repo.repo, + commit_sha: ref, + }); + if (prs.length > 0) { + labels = prs[0].labels.map(l => l.name); + } + } + core.info(`[health-check] PR labels: [${labels.join(', ')}]`); + if (labels.includes('bypass-fastfail')) { + core.notice('[health-check] SKIP jobs-failed check: bypass-fastfail label present'); + return; + } + + const jobs = await github.paginate(github.rest.actions.listJobsForWorkflowRun, { + owner: context.repo.owner, + repo: context.repo.repo, + run_id: context.runId, + per_page: 100, + }); + core.info(`[health-check] Total jobs in run: ${jobs.length}`); + const failedJobs = jobs.filter(j => j.status === 'completed' && j.conclusion === 'failure'); + core.info(`[health-check] Failed jobs (before filtering): ${failedJobs.map(j => `${j.name}(${j.conclusion})`).join(', ') || 'none'}`); + + // Find jobs that failed from a real error, not from fast-fail cascade + const rootCauseFailures = jobs.filter(j => { + if (j.status !== 'completed' || j.conclusion !== 'failure') return false; + // h20 runners are flaky (dirty GPU state from prior runs); their failures + // should not cascade fast-fail to other stages. j.name shape from + // listJobsForWorkflowRun: "" + optional " / " + // + optional " ()". Split off the base job key before exact + // match so we cover both inline + reusable forms without confusing + // 'h20' with the 'h200' prefix. + const baseName = j.name.split(/[ /]/)[0]; + if (baseName === 'base-c-test-8-gpu-h20') { + core.info(`[health-check] Filtered out h20 job: ${j.name}`); + return false; + } + // multimodal-gen NPU tests should not cascade fast-fail to perf/accuracy stages. + if (baseName === 'multimodal-gen-test-1-npu-a3' || baseName === 'multimodal-gen-test-2-npu-a3') { + core.info(`[health-check] Filtered out multimodal-gen NPU job: ${j.name}`); + return false; + } + // If the failing step is the health check, it's a cascade — skip it + const failedStep = (j.steps || []).find(s => s.conclusion === 'failure'); + if (failedStep && (failedStep.name.includes('check-pr-test-health') || failedStep.name.includes('Check PR test health'))) { + core.info(`[health-check] Filtered out cascade failure: ${j.name} (failed step: ${failedStep.name})`); + return false; + } + return true; + }); + core.info(`[health-check] Root cause failures (after filtering): ${rootCauseFailures.map(j => j.name).join(', ') || 'none'}`); + + if (rootCauseFailures.length > 0) { + core.setFailed(`Fast-fail: skipping — root cause job(s): ${rootCauseFailures.map(j => j.name).join(', ')}`); + } else { + core.notice('[health-check] PASS: no root cause failures detected'); + } - name: Install dependencies env: diff --git a/.github/workflows/_npu-single-node-test-stage.yml b/.github/workflows/_npu-single-node-test-stage.yml index 9ad0b0f303..2a6b434e38 100644 --- a/.github/workflows/_npu-single-node-test-stage.yml +++ b/.github/workflows/_npu-single-node-test-stage.yml @@ -43,7 +43,7 @@ on: default: '{}' description: 'JSON run metadata {branch_label, workflow_name, create_time}, recorded once at workflow start' skip_pr_test_health_check: - description: 'Set to true to skip the PR test health check (fail-fast gate).' + description: 'Set to true to skip the PR test health check (fast-fail gate).' type: string default: 'false' is_nightly_pipeline_job: @@ -101,7 +101,112 @@ jobs: # checks out the same commit even if the branch advances mid-run. ref: ${{ inputs.ref || github.sha }} - - uses: ./.github/actions/check-pr-test-health + - name: Check PR test health + uses: actions/github-script@v8 + env: + SKIP_PR_TEST_HEALTH_CHECK: ${{ env.SKIP_PR_TEST_HEALTH_CHECK }} + with: + github-token: ${{ inputs.github-token || github.token }} + script: | + core.notice(`[health-check] START — event=${context.eventName}, runId=${context.runId}`); + + // Skip when explicitly requested via env var (e.g. release branch cut) + if (process.env.SKIP_PR_TEST_HEALTH_CHECK === 'true') { + core.notice('[health-check] SKIP: SKIP_PR_TEST_HEALTH_CHECK=true'); + return; + } + + // Skip for scheduled runs — they should collect all failures, not fast-fail + if (context.eventName === 'schedule') { + core.notice('[health-check] SKIP: scheduled run'); + return; + } + + // Check lint status from the separate Lint workflow (lint.yml). + // listJobsForWorkflowRun only sees jobs within the SAME run, so we use + // checks.listForRef which queries by commit SHA across ALL workflows. + const ref = context.payload.pull_request?.head?.sha || context.sha; + core.info(`[health-check] Checking lint for ref=${ref}`); + const { data } = await github.rest.checks.listForRef({ + owner: context.repo.owner, + repo: context.repo.repo, + ref: ref, + check_name: 'lint', + }); + const lintRun = data.check_runs.find( + cr => cr.app?.slug === 'github-actions' + ); + core.info(`[health-check] Lint check: status=${lintRun?.status}, conclusion=${lintRun?.conclusion}`); + if (lintRun?.status === 'completed' && lintRun?.conclusion === 'failure') { + core.setFailed('Fast-fail: lint check failed'); + return; + } + + // Skip the jobs-failed check when the PR carries the bypass-fastfail label. + // Lint check above still runs. + let labels = []; + if (context.payload.pull_request?.labels) { + labels = context.payload.pull_request.labels.map(l => l.name); + } else { + const { data: prs } = await github.rest.repos.listPullRequestsAssociatedWithCommit({ + owner: context.repo.owner, + repo: context.repo.repo, + commit_sha: ref, + }); + if (prs.length > 0) { + labels = prs[0].labels.map(l => l.name); + } + } + core.info(`[health-check] PR labels: [${labels.join(', ')}]`); + if (labels.includes('bypass-fastfail')) { + core.notice('[health-check] SKIP jobs-failed check: bypass-fastfail label present'); + return; + } + + const jobs = await github.paginate(github.rest.actions.listJobsForWorkflowRun, { + owner: context.repo.owner, + repo: context.repo.repo, + run_id: context.runId, + per_page: 100, + }); + core.info(`[health-check] Total jobs in run: ${jobs.length}`); + const failedJobs = jobs.filter(j => j.status === 'completed' && j.conclusion === 'failure'); + core.info(`[health-check] Failed jobs (before filtering): ${failedJobs.map(j => `${j.name}(${j.conclusion})`).join(', ') || 'none'}`); + + // Find jobs that failed from a real error, not from fast-fail cascade + const rootCauseFailures = jobs.filter(j => { + if (j.status !== 'completed' || j.conclusion !== 'failure') return false; + // h20 runners are flaky (dirty GPU state from prior runs); their failures + // should not cascade fast-fail to other stages. j.name shape from + // listJobsForWorkflowRun: "" + optional " / " + // + optional " ()". Split off the base job key before exact + // match so we cover both inline + reusable forms without confusing + // 'h20' with the 'h200' prefix. + const baseName = j.name.split(/[ /]/)[0]; + if (baseName === 'base-c-test-8-gpu-h20') { + core.info(`[health-check] Filtered out h20 job: ${j.name}`); + return false; + } + // multimodal-gen NPU tests should not cascade fast-fail to perf/accuracy stages. + if (baseName === 'multimodal-gen-test-1-npu-a3' || baseName === 'multimodal-gen-test-2-npu-a3') { + core.info(`[health-check] Filtered out multimodal-gen NPU job: ${j.name}`); + return false; + } + // If the failing step is the health check, it's a cascade — skip it + const failedStep = (j.steps || []).find(s => s.conclusion === 'failure'); + if (failedStep && (failedStep.name.includes('check-pr-test-health') || failedStep.name.includes('Check PR test health'))) { + core.info(`[health-check] Filtered out cascade failure: ${j.name} (failed step: ${failedStep.name})`); + return false; + } + return true; + }); + core.info(`[health-check] Root cause failures (after filtering): ${rootCauseFailures.map(j => j.name).join(', ') || 'none'}`); + + if (rootCauseFailures.length > 0) { + core.setFailed(`Fast-fail: skipping — root cause job(s): ${rootCauseFailures.map(j => j.name).join(', ')}`); + } else { + core.notice('[health-check] PASS: no root cause failures detected'); + } - name: Check npu info run: | diff --git a/python/sglang/test/ascend/e2e/run_evalscope.sh b/python/sglang/test/ascend/e2e/run_evalscope.sh index 4b882b7fc3..57723d9070 100755 --- a/python/sglang/test/ascend/e2e/run_evalscope.sh +++ b/python/sglang/test/ascend/e2e/run_evalscope.sh @@ -3,26 +3,58 @@ set -e PYTHON_ENV_FOR_EVALSCOPE=test_env_evalscope +PYTHON_FOR_EVALSCOPE=${PYTHON_ENV_FOR_EVALSCOPE}/bin/python PIP_FOR_EVALSCOPE=${PYTHON_ENV_FOR_EVALSCOPE}/bin/pip EVALSCOPE_SOURCE_PATH=/root/.cache/.cache/evalscope pip_mirror_source="https://mirrors.tuna.tsinghua.edu.cn/pypi/web/simple" -if [ -d "${PYTHON_ENV_FOR_EVALSCOPE}" ]; then - echo "Virtual env ${PYTHON_ENV_FOR_EVALSCOPE} already exists, skip installation." +# Bound key deps so the resolver cannot fall back to ancient versions. +EVALSCOPE_CONSTRAINTS=( + "aiohttp>=3.11,<4" + "httpx>=0.28,<1" +) + +# Fail fast when the pip install hangs instead of timing out the whole job. +EVALSCOPE_INSTALL_TIMEOUT=1200 + +# Highest priority: reuse a system-wide evalscope (e.g. pre-installed in the +# image); create ${PYTHON_FOR_EVALSCOPE} with --system-site-packages when needed. +if python -c "import evalscope" >/dev/null 2>&1; then + echo "evalscope found in the system python, reuse it instead of installing." + if ! ${PYTHON_FOR_EVALSCOPE} -c "import evalscope" >/dev/null 2>&1; then + python -m venv --system-site-packages ${PYTHON_ENV_FOR_EVALSCOPE} + fi + # Only skip the install when the env really imports evalscope. + if ${PYTHON_FOR_EVALSCOPE} -c "import evalscope" >/dev/null 2>&1; then + exit 0 + fi + echo "The env cannot provide evalscope, install it instead." +fi + +# Otherwise reuse the virtual env when it already imports evalscope. +if [ -x "${PYTHON_FOR_EVALSCOPE}" ] && ${PYTHON_FOR_EVALSCOPE} -c "import evalscope" >/dev/null 2>&1; then + echo "evalscope already exists in ${PYTHON_ENV_FOR_EVALSCOPE}, skip installation." exit 0 fi -echo "===== Install evalscope in virtual env - Begin =====" python -m venv ${PYTHON_ENV_FOR_EVALSCOPE} +echo "===== Install evalscope in virtual env - Begin =====" + if [ ! -d "${EVALSCOPE_SOURCE_PATH}" ]; then echo "The evalscope source does not exist: ${EVALSCOPE_SOURCE_PATH}." echo "Install evalscope online." ${PIP_FOR_EVALSCOPE} install -U pip -i ${pip_mirror_source} - ${PIP_FOR_EVALSCOPE} install evalscope -i ${pip_mirror_source} + timeout ${EVALSCOPE_INSTALL_TIMEOUT} ${PIP_FOR_EVALSCOPE} install evalscope "${EVALSCOPE_CONSTRAINTS[@]}" -i ${pip_mirror_source} || { + echo "ERROR: evalscope install timed out after ${EVALSCOPE_INSTALL_TIMEOUT}s." + exit 1 + } else echo "Install evalscope from local source: ${EVALSCOPE_SOURCE_PATH}" ${PIP_FOR_EVALSCOPE} install -U pip -i ${pip_mirror_source} - ${PIP_FOR_EVALSCOPE} install -e ${EVALSCOPE_SOURCE_PATH} -i ${pip_mirror_source} + timeout ${EVALSCOPE_INSTALL_TIMEOUT} ${PIP_FOR_EVALSCOPE} install -e ${EVALSCOPE_SOURCE_PATH} "${EVALSCOPE_CONSTRAINTS[@]}" -i ${pip_mirror_source} || { + echo "ERROR: evalscope install timed out after ${EVALSCOPE_INSTALL_TIMEOUT}s." + exit 1 + } fi echo "===== Install evalscope in virtual env - End ====="