diff --git a/.github/workflows/_npu-pr-test-stage.yml b/.github/workflows/_npu-pr-test-stage.yml new file mode 100644 index 000000000..db1171fa9 --- /dev/null +++ b/.github/workflows/_npu-pr-test-stage.yml @@ -0,0 +1,213 @@ +name: PR Test Stage for NPU +# Reusable workflow for one CUDA test stage. Caller pr-test-npu.yml forwards + +on: + workflow_call: + inputs: + self_name: + description: 'Caller job key; used for partitions[suite] lookup.' + type: string + required: true + runner_config: + description: 'Key in scripts/ci/runner_configs.yml. Resolves install script, artifact version, install timeout, runs-on label, and rdma_devices.' + type: string + required: true + image: + description: 'The container image for NPU test environment.' + type: string + required: true + npu_device_type: + description: 'NPU device type, e.g. 910b, a3.' + type: string + default: 'a3' + run_timeout_minutes: + description: 'timeout-minutes for the Run test step. Required so compute_partitions.py can read it from pr-test.yml without a duplicated default constant.' + type: string + required: true + timeout_per_file: + description: 'run_suite.py --timeout-per-file value (empty = unset).' + type: string + default: '' + partitions: + description: 'partitions config, e.g. {"size":1,"arr":[0]}' + type: string + default: '{"size":1,"arr":[0]}' + ref: + description: 'Git ref (branch, tag, or SHA) to test. If not provided, uses the default branch.' + type: string + default: '' + skip_pr_test_health_check: + description: 'Git ref (branch, tag, or SHA) to test. If not provided, uses the default branch.' + type: string + default: 'false' + github-token: + description: 'GitHub token for API calls' + type: string + default: ${{ github.token }} + +env: + SKIP_PR_TEST_HEALTH_CHECK: ${{ inputs.skip_pr_test_health_check }} + SGLANG_USE_MODELSCOPE: true + SGLANG_IS_IN_CI: true + HF_ENDPOINT: https://hf-mirror.com + TORCH_EXTENSIONS_DIR: /tmp/torch_extensions + PYTORCH_NPU_ALLOC_CONF: "expandable_segments:True" + STREAMS_PER_DEVICE: 32 + +jobs: + run: + runs-on: ${{ inputs.runner_config }} + container: + image: ${{ inputs.image }} + timeout-minutes: 240 + strategy: + fail-fast: false + matrix: + partition: ${{ fromJson(inputs.partitions).arr }} + steps: + - name: Checkout code + uses: actions/checkout@v4 + with: + ref: ${{ inputs.ref || github.ref }} + + - name: Mark repository safe + run: | + git config --system --add safe.directory ${GITHUB_WORKSPACE} + + - name: Check PR test health + uses: actions/github-script@v8 + env: + SKIP_PR_TEST_HEALTH_CHECK: ${{ env.SKIP_PR_TEST_HEALTH_CHECK }} + with: + github-token: ${{ inputs.github-token || github.token }} + script: | + core.notice(`[health-check] START — event=${context.eventName}, runId=${context.runId}`); + + // Skip when explicitly requested via env var (e.g. release branch cut) + if (process.env.SKIP_PR_TEST_HEALTH_CHECK === 'true') { + core.notice('[health-check] SKIP: SKIP_PR_TEST_HEALTH_CHECK=true'); + return; + } + + // Skip for scheduled runs — they should collect all failures, not fast-fail + if (context.eventName === 'schedule') { + core.notice('[health-check] SKIP: scheduled run'); + return; + } + + // Check lint status from the separate Lint workflow (lint.yml). + // listJobsForWorkflowRun only sees jobs within the SAME run, so we use + // checks.listForRef which queries by commit SHA across ALL workflows. + const ref = context.payload.pull_request?.head?.sha || context.sha; + core.info(`[health-check] Checking lint for ref=${ref}`); + const { data } = await github.rest.checks.listForRef({ + owner: context.repo.owner, + repo: context.repo.repo, + ref: ref, + check_name: 'lint', + }); + const lintRun = data.check_runs.find( + cr => cr.app?.slug === 'github-actions' + ); + core.info(`[health-check] Lint check: status=${lintRun?.status}, conclusion=${lintRun?.conclusion}`); + if (lintRun?.status === 'completed' && lintRun?.conclusion === 'failure') { + core.setFailed('Fast-fail: lint check failed'); + return; + } + + // Skip the jobs-failed check when the PR carries the bypass-fastfail label. + // Lint check above still runs. + let labels = []; + if (context.payload.pull_request?.labels) { + labels = context.payload.pull_request.labels.map(l => l.name); + } else { + const { data: prs } = await github.rest.repos.listPullRequestsAssociatedWithCommit({ + owner: context.repo.owner, + repo: context.repo.repo, + commit_sha: ref, + }); + if (prs.length > 0) { + labels = prs[0].labels.map(l => l.name); + } + } + core.info(`[health-check] PR labels: [${labels.join(', ')}]`); + if (labels.includes('bypass-fastfail')) { + core.notice('[health-check] SKIP jobs-failed check: bypass-fastfail label present'); + return; + } + + const jobs = await github.paginate(github.rest.actions.listJobsForWorkflowRun, { + owner: context.repo.owner, + repo: context.repo.repo, + run_id: context.runId, + per_page: 100, + }); + core.info(`[health-check] Total jobs in run: ${jobs.length}`); + const failedJobs = jobs.filter(j => j.status === 'completed' && j.conclusion === 'failure'); + core.info(`[health-check] Failed jobs (before filtering): ${failedJobs.map(j => `${j.name}(${j.conclusion})`).join(', ') || 'none'}`); + + // Find jobs that failed from a real error, not from fast-fail cascade + const rootCauseFailures = jobs.filter(j => { + if (j.status !== 'completed' || j.conclusion !== 'failure') return false; + // h20 runners are flaky (dirty GPU state from prior runs); their failures + // should not cascade fast-fail to other stages. j.name shape from + // listJobsForWorkflowRun: "" + optional " / " + // + optional " ()". Split off the base job key before exact + // match so we cover both inline + reusable forms without confusing + // 'h20' with the 'h200' prefix. + const baseName = j.name.split(/[ /]/)[0]; + if (baseName === 'base-c-test-8-gpu-h20') { + core.info(`[health-check] Filtered out h20 job: ${j.name}`); + return false; + } + // If the failing step is the health check, it's a cascade — skip it + const failedStep = (j.steps || []).find(s => s.conclusion === 'failure'); + if (failedStep && (failedStep.name.includes('check-pr-test-health') || failedStep.name.includes('Check PR test health'))) { + core.info(`[health-check] Filtered out cascade failure: ${j.name} (failed step: ${failedStep.name})`); + return false; + } + return true; + }); + core.info(`[health-check] Root cause failures (after filtering): ${rootCauseFailures.map(j => j.name).join(', ') || 'none'}`); + + if (rootCauseFailures.length > 0) { + core.setFailed(`Fast-fail: skipping — root cause job(s): ${rootCauseFailures.map(j => j.name).join(', ')}`); + } else { + core.notice('[health-check] PASS: no root cause failures detected'); + } + + - name: Install dependencies + env: + TORCH_CACHE_URL: "http://cache-service.nginx-pypi-cache.svc.cluster.local/whl/cpu" + PYPI_CACHE_URL: "http://cache-service.nginx-pypi-cache.svc.cluster.local/pypi/simple" + UV_INDEX_URL: "http://cache-service.nginx-pypi-cache.svc.cluster.local/pypi/simple" + GITHUB_PROXY_URL: "https://gh-proxy.test.osinfra.cn/" + RUSTUP_CACHE_URL: "http://cache-service.nginx-pypi-cache.svc.cluster.local:8082" + run: | + # speed up by using infra cache services + CACHING_URL="cache-service.nginx-pypi-cache.svc.cluster.local" + sed -Ei "s@(ports|archive).ubuntu.com@${CACHING_URL}:8081@g" /etc/apt/sources.list + pip config set global.index-url http://${CACHING_URL}/pypi/simple + pip config set global.trusted-host "${CACHING_URL}" + + bash scripts/ci/npu/npu_ci_install_dependency.sh ${{ inputs.npu_device_type }} + # copy required file from our daily cache + cp ~/.cache/modelscope/hub/datasets/otavia/ShareGPT_Vicuna_unfiltered/ShareGPT_V3_unfiltered_cleaned_split.json /tmp + # copy gsm8k dataset + cp ~/.cache/modelscope/hub/datasets/tmp/test.jsonl /tmp + + # install sglang_router + apt-get install -y libssl-dev + pip install sglang_router + + - name: Run test + timeout-minutes: ${{ fromJson(inputs.run_timeout_minutes) }} + env: + CONTINUE_ON_ERROR_FLAG: ${{ inputs.continue_on_error == 'true' && '--continue-on-error' || '' }} + run: | + cd test + python3 run_suite.py --hw npu --suite ${{ inputs.self_name }} \ + --auto-partition-id ${{ matrix.partition }} \ + --auto-partition-size ${{ fromJson(inputs.partitions).size }} \ + ${{ inputs.timeout_per_file && format('--timeout-per-file {0}', inputs.timeout_per_file) || '' }} \ + $CONTINUE_ON_ERROR_FLAG diff --git a/.github/workflows/_npu-single-node-test-stage.yml b/.github/workflows/_npu-single-node-test-stage.yml new file mode 100644 index 000000000..f04757db7 --- /dev/null +++ b/.github/workflows/_npu-single-node-test-stage.yml @@ -0,0 +1,295 @@ +name: 'Single Node Template for E2E performance and accuracy tests' + +on: + workflow_call: + inputs: + runner: + required: true + type: string + default: linux-aarch64-a3-16 + test_type: + required: true + type: string + default: perf + description: perf or accuracy + test_suite: + required: true + type: string + default: '' + description: name of test suite to run via run_suite.py (mutually exclusive with test_case) + image: + required: true + type: string + description: image for pods + default: "swr.cn-southwest-2.myhuaweicloud.com/base_image/dockerhub/lmsysorg/sglang:main-cann9.0.0-a3" + install_sglang_deps: + required: false + type: boolean + default: false + description: install sglang dependencies (e.g. PyTorch, CANN packages) when using source installation + device_type_for_deps: + required: false + type: string + default: 'a3' + description: device type for dependency installation (a3 or 910b) + transformers_version: + required: false + type: string + default: "" + description: "The transformers version number for running sglang. Use default version in image if keep empty." + skip_pr_test_health_check: + description: 'Git ref (branch, tag, or SHA) to test. If not provided, uses the default branch.' + type: string + default: 'false' + github-token: + description: 'GitHub token for API calls' + type: string + default: ${{ github.token }} + +env: + SKIP_PR_TEST_HEALTH_CHECK: ${{ inputs.skip_pr_test_health_check }} + +concurrency: + group: ascend-nightly-e2e-singlenode-${{ github.workflow_ref }}-${{ github.ref }}-${{ inputs.test_suite }} + cancel-in-progress: true + +jobs: + e2e: + name: ${{ inputs.test_suite }} + runs-on: ${{ inputs.runner }} + container: + image: ${{ inputs.image }} + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Check PR test health + uses: actions/github-script@v8 + env: + SKIP_PR_TEST_HEALTH_CHECK: ${{ env.SKIP_PR_TEST_HEALTH_CHECK }} + with: + github-token: ${{ inputs.github-token || github.token }} + script: | + core.notice(`[health-check] START — event=${context.eventName}, runId=${context.runId}`); + + // Skip when explicitly requested via env var (e.g. release branch cut) + if (process.env.SKIP_PR_TEST_HEALTH_CHECK === 'true') { + core.notice('[health-check] SKIP: SKIP_PR_TEST_HEALTH_CHECK=true'); + return; + } + + // Skip for scheduled runs — they should collect all failures, not fast-fail + if (context.eventName === 'schedule') { + core.notice('[health-check] SKIP: scheduled run'); + return; + } + + // Check lint status from the separate Lint workflow (lint.yml). + // listJobsForWorkflowRun only sees jobs within the SAME run, so we use + // checks.listForRef which queries by commit SHA across ALL workflows. + const ref = context.payload.pull_request?.head?.sha || context.sha; + core.info(`[health-check] Checking lint for ref=${ref}`); + const { data } = await github.rest.checks.listForRef({ + owner: context.repo.owner, + repo: context.repo.repo, + ref: ref, + check_name: 'lint', + }); + const lintRun = data.check_runs.find( + cr => cr.app?.slug === 'github-actions' + ); + core.info(`[health-check] Lint check: status=${lintRun?.status}, conclusion=${lintRun?.conclusion}`); + if (lintRun?.status === 'completed' && lintRun?.conclusion === 'failure') { + core.setFailed('Fast-fail: lint check failed'); + return; + } + + // Skip the jobs-failed check when the PR carries the bypass-fastfail label. + // Lint check above still runs. + let labels = []; + if (context.payload.pull_request?.labels) { + labels = context.payload.pull_request.labels.map(l => l.name); + } else { + const { data: prs } = await github.rest.repos.listPullRequestsAssociatedWithCommit({ + owner: context.repo.owner, + repo: context.repo.repo, + commit_sha: ref, + }); + if (prs.length > 0) { + labels = prs[0].labels.map(l => l.name); + } + } + core.info(`[health-check] PR labels: [${labels.join(', ')}]`); + if (labels.includes('bypass-fastfail')) { + core.notice('[health-check] SKIP jobs-failed check: bypass-fastfail label present'); + return; + } + + const jobs = await github.paginate(github.rest.actions.listJobsForWorkflowRun, { + owner: context.repo.owner, + repo: context.repo.repo, + run_id: context.runId, + per_page: 100, + }); + core.info(`[health-check] Total jobs in run: ${jobs.length}`); + const failedJobs = jobs.filter(j => j.status === 'completed' && j.conclusion === 'failure'); + core.info(`[health-check] Failed jobs (before filtering): ${failedJobs.map(j => `${j.name}(${j.conclusion})`).join(', ') || 'none'}`); + + // Find jobs that failed from a real error, not from fast-fail cascade + const rootCauseFailures = jobs.filter(j => { + if (j.status !== 'completed' || j.conclusion !== 'failure') return false; + // h20 runners are flaky (dirty GPU state from prior runs); their failures + // should not cascade fast-fail to other stages. j.name shape from + // listJobsForWorkflowRun: "" + optional " / " + // + optional " ()". Split off the base job key before exact + // match so we cover both inline + reusable forms without confusing + // 'h20' with the 'h200' prefix. + const baseName = j.name.split(/[ /]/)[0]; + if (baseName === 'base-c-test-8-gpu-h20') { + core.info(`[health-check] Filtered out h20 job: ${j.name}`); + return false; + } + // If the failing step is the health check, it's a cascade — skip it + const failedStep = (j.steps || []).find(s => s.conclusion === 'failure'); + if (failedStep && (failedStep.name.includes('check-pr-test-health') || failedStep.name.includes('Check PR test health'))) { + core.info(`[health-check] Filtered out cascade failure: ${j.name} (failed step: ${failedStep.name})`); + return false; + } + return true; + }); + core.info(`[health-check] Root cause failures (after filtering): ${rootCauseFailures.map(j => j.name).join(', ') || 'none'}`); + + if (rootCauseFailures.length > 0) { + core.setFailed(`Fast-fail: skipping — root cause job(s): ${rootCauseFailures.map(j => j.name).join(', ')}`); + } else { + core.notice('[health-check] PASS: no root cause failures detected'); + } + + - name: Check npu info + run: | + npu-smi info + + - name: Install sglang dependencies + if: ${{ inputs.install_sglang_deps == true }} + shell: bash + env: + TORCH_CACHE_URL: "http://cache-service.nginx-pypi-cache.svc.cluster.local/whl/cpu" + PYPI_CACHE_URL: "http://cache-service.nginx-pypi-cache.svc.cluster.local/pypi/simple" + UV_INDEX_URL: "http://cache-service.nginx-pypi-cache.svc.cluster.local/pypi/simple" + GITHUB_PROXY_URL: "https://gh-proxy.test.osinfra.cn/" + RUSTUP_CACHE_URL: "http://cache-service.nginx-pypi-cache.svc.cluster.local:8082" + run: | + CACHING_URL="cache-service.nginx-pypi-cache.svc.cluster.local" + sed -Ei "s@(ports|archive).ubuntu.com@${CACHING_URL}:8081@g" /etc/apt/sources.list + pip config set global.index-url http://${CACHING_URL}/pypi/simple + pip config set global.trusted-host "${CACHING_URL}" + bash scripts/ci/npu/npu_ci_install_dependency.sh ${{ inputs.device_type_for_deps }} + cp ~/.cache/modelscope/hub/datasets/otavia/ShareGPT_Vicuna_unfiltered/ShareGPT_V3_unfiltered_cleaned_split.json /tmp + curl -o /tmp/test.jsonl -L https://gh-proxy.test.osinfra.cn/https://raw.githubusercontent.com/openai/grade-school-math/master/grade_school_math/data/test.jsonl + + - name: Run test + timeout-minutes: 300 + env: + SGLANG_USE_MODELSCOPE: true + HF_ENDPOINT: https://hf-mirror.com + SGLANG_IS_IN_CI: true + TRANSFORMERS_VERBOSITY: "error" + GDN_ATTN_BACKEND_TRITON: 1 + SGLANG_TEST_METRICS_OUTPUT: /root/.cache/tests/output/metrics/metrics + shell: bash + run: | + sglang_source_path=$(pwd) + echo "Source code path: ${sglang_source_path}" + ln -sf ${sglang_source_path} /root/sglang + + # Determine test mode: suite (run_suite.py --suite) or single case file. + test_suite="${{ inputs.test_suite }}" + echo "Test mode: suite (${test_suite})" + tc_name="${test_suite}" + + export TRANSFORMERS_VERSION_FOR_SGLANG="${{ inputs.transformers_version }}" + PYTHON_FOR_SGLANG="python" + PIP_FOR_SGLANG="pip" + if [ -n "${TRANSFORMERS_VERSION_FOR_SGLANG}" ];then + echo "===== Install transformers for sglang - Begin =====" + TRANSFORMERS_PKG_PATH_SOURCE=/root/.cache/.cache/transformers/${TRANSFORMERS_VERSION_FOR_SGLANG} + if [ ! -d "${TRANSFORMERS_PKG_PATH_SOURCE}" ]; then + echo "The dependent transformers package does not exist: ${TRANSFORMERS_PKG_PATH_SOURCE}." + echo "Install transformers ${TRANSFORMERS_VERSION_FOR_SGLANG} online." + pip install transformers=="${TRANSFORMERS_VERSION_FOR_SGLANG}" -i https://mirrors.tuna.tsinghua.edu.cn/pypi/web/simple + else + echo "Install transformers ${TRANSFORMERS_VERSION_FOR_SGLANG} locally." + TRANSFORMERS_PKG_PATH_TARGET=/tmp/transformers/${TRANSFORMERS_VERSION_FOR_SGLANG} + mkdir -p "${TRANSFORMERS_PKG_PATH_TARGET}" + cp "${TRANSFORMERS_PKG_PATH_SOURCE}/*" "${TRANSFORMERS_PKG_PATH_TARGET}/" + pip install --no-index --find-links="${TRANSFORMERS_PKG_PATH_TARGET}" transformers=="${TRANSFORMERS_VERSION_FOR_SGLANG}" + fi + echo "===== Install transformers for sglang in virtual env - End =====" + fi + echo "Transformers version for sglang: $(${PIP_FOR_SGLANG} show transformers | grep Version | cut -d: -f2)" + + echo "scaling_governor performance num: \ + $(cat /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor | grep performance | wc -l)" + echo "swappiness: $(cat /proc/sys/vm/swappiness)" + echo "numa_balancing: $(cat /proc/sys/kernel/numa_balancing)" + echo "sched_migration_cost_ns: $(cat /proc/sys/kernel/sched_migration_cost_ns)" + + export SGLANG_TEST_MAX_RETRY=0 + export SGLANG_SET_CPU_AFFINITY=1 + echo "SGLANG_SET_CPU_AFFINITY: $SGLANG_SET_CPU_AFFINITY" + + + # Copy the checked‑out test common utility code to the sglang installation directory. + sglang_pkg_path=/sgl-workspace/sglang/python + ascend_test_util_path=${sglang_pkg_path}/sglang/test/ascend + mkdir -p ${ascend_test_util_path} + mv ${ascend_test_util_path} ${ascend_test_util_path}_bak + cp -r ${sglang_source_path}/python/sglang/test/ascend ${ascend_test_util_path} + + source /usr/local/Ascend/cann/set_env.sh || true + source /usr/local/Ascend/nnal/atb/set_env.sh || true + source /usr/local/Ascend/ascend-toolkit/latest/opp/vendors/customize/bin/set_env.bash || true + source /usr/local/Ascend/ascend-toolkit/latest/opp/vendors/custom_transformer/bin/set_env.bash || true + + # Set environment of cann + log_path="/root/.cache/tests/logs/log/${current_date}/${tc_name}/${HOSTNAME}" + rm -rf ${log_path} + mkdir -p ${log_path} + echo "Log path: ${log_path}" + + echo "Running test: ${tc_name}" + test_exit_code=0 + cd test + ${PYTHON_FOR_SGLANG} -u run_suite.py --hw npu --suite ${test_suite} --timeout-per-file 3600 2>&1 | tee /tmp/test_output.log || test_exit_code=$? + + echo "Finished test: ${tc_name}" + + if [ "${test_exit_code}" = "0" ]; then + test_status="pass" + status_icon="✅" + else + test_status="fail" + status_icon="❌" + fi + + echo "test_status=${test_status}" >> $GITHUB_ENV + echo "tc_name=${tc_name}" >> $GITHUB_ENV + export test_status tc_name + + echo "## ${tc_name} ${status_icon} ${test_status}" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + metric_count=$(grep -c '\[METRIC\]' /tmp/test_output.log 2>/dev/null || echo 0) + if [ "${metric_count}" -gt 0 ]; then + echo "| Metric | Value | Pass |" >> $GITHUB_STEP_SUMMARY + echo "|--------|-------|------|" >> $GITHUB_STEP_SUMMARY + grep '\[METRIC\]' /tmp/test_output.log | while IFS= read -r line; do + metric_name=$(echo "$line" | sed -E 's/.*\[METRIC\] ([^=]+)=.*/\1/') + metric_value=$(echo "$line" | sed -E 's/.*\[METRIC\] [^=]+=([^ ]+).*/\1/') + echo "| ${metric_name} | ${metric_value} | ${status_icon} |" >> $GITHUB_STEP_SUMMARY + done + else + echo "No metrics collected (test may have failed before producing results)." >> $GITHUB_STEP_SUMMARY + fi + echo "" >> $GITHUB_STEP_SUMMARY + + exit ${test_exit_code} diff --git a/.github/workflows/pr-test-npu.yml b/.github/workflows/pr-test-npu.yml index 3387c6b78..f829631f4 100644 --- a/.github/workflows/pr-test-npu.yml +++ b/.github/workflows/pr-test-npu.yml @@ -90,303 +90,82 @@ jobs: echo "CANN_image_a3=swr.cn-southwest-2.myhuaweicloud.com/base_image/ascend-ci/cann:9.0.0-a3-ubuntu22.04-py3.11" >> $GITHUB_OUTPUT echo "CANN_image_910b=swr.cn-southwest-2.myhuaweicloud.com/base_image/ascend-ci/cann:9.0.0-910b-ubuntu22.04-py3.11" >> $GITHUB_OUTPUT - stage-a-unit-test-npu: + base-a-test-1-npu-a2: needs: [check-changes, pr-gate, set-image-config] - if: needs.check-changes.outputs.main_package == 'true' - runs-on: linux-aarch64-a2-1 - container: + if: ${{ !failure() && !cancelled() && needs.check-changes.outputs.main_package == 'true' }} + uses: ./.github/workflows/_npu-pr-test-stage.yml + with: + self_name: base-a-test-1-npu-a2 + runner_config: linux-aarch64-a2-1 image: ${{ needs.set-image-config.outputs.CANN_image_910b }} - steps: - - name: Checkout code - uses: actions/checkout@v4 - with: - ref: ${{ inputs.ref || github.ref }} + npu_device_type: 910b + run_timeout_minutes: '15' + secrets: inherit - - name: Mark repository safe - run: | - git config --system --add safe.directory ${GITHUB_WORKSPACE} - - - name: Install dependencies - env: - TORCH_CACHE_URL: "http://cache-service.nginx-pypi-cache.svc.cluster.local/whl/cpu" - PYPI_CACHE_URL: "http://cache-service.nginx-pypi-cache.svc.cluster.local/pypi/simple" - UV_INDEX_URL: "http://cache-service.nginx-pypi-cache.svc.cluster.local/pypi/simple" - GITHUB_PROXY_URL: "https://gh-proxy.test.osinfra.cn/" - RUSTUP_CACHE_URL: "http://cache-service.nginx-pypi-cache.svc.cluster.local:8082" - run: | - # speed up by using infra cache services - CACHING_URL="cache-service.nginx-pypi-cache.svc.cluster.local" - sed -Ei "s@(ports|archive).ubuntu.com@${CACHING_URL}:8081@g" /etc/apt/sources.list - pip config set global.index-url http://${CACHING_URL}/pypi/simple - pip config set global.trusted-host "${CACHING_URL}" - - bash scripts/ci/npu/npu_ci_install_dependency.sh 910b - - - name: Run test - timeout-minutes: 15 - env: - SGLANG_IS_IN_CI: true - run: | - cd test - python3 run_suite.py --hw npu --suite stage-a-unit-test-npu - - stage-b-test-1-npu-a3: + base-b-test-1-npu-a3: needs: [check-changes, pr-gate, set-image-config] - if: needs.check-changes.outputs.main_package == 'true' - runs-on: linux-aarch64-a3-2- - container: + if: ${{ !failure() && !cancelled() && needs.check-changes.outputs.main_package == 'true' }} + uses: ./.github/workflows/_npu-pr-test-stage.yml + with: + self_name: base-b-test-1-npu-a3 + runner_config: linux-aarch64-a3-2- image: ${{ needs.set-image-config.outputs.CANN_image_a3 }} - steps: - - name: Checkout code - uses: actions/checkout@v4 - with: - ref: ${{ inputs.ref || github.ref }} + run_timeout_minutes: '60' + timeout_per_file: '3600' + secrets: inherit - - name: Mark repository safe - run: | - git config --system --add safe.directory ${GITHUB_WORKSPACE} - - - name: Install dependencies - env: - TORCH_CACHE_URL: "http://cache-service.nginx-pypi-cache.svc.cluster.local/whl/cpu" - PYPI_CACHE_URL: "http://cache-service.nginx-pypi-cache.svc.cluster.local/pypi/simple" - UV_INDEX_URL: "http://cache-service.nginx-pypi-cache.svc.cluster.local/pypi/simple" - GITHUB_PROXY_URL: "https://gh-proxy.test.osinfra.cn/" - RUSTUP_CACHE_URL: "http://cache-service.nginx-pypi-cache.svc.cluster.local:8082" - run: | - # speed up by using infra cache services - CACHING_URL="cache-service.nginx-pypi-cache.svc.cluster.local" - sed -Ei "s@(ports|archive).ubuntu.com@${CACHING_URL}:8081@g" /etc/apt/sources.list - pip config set global.index-url http://${CACHING_URL}/pypi/simple - pip config set global.trusted-host "${CACHING_URL}" - - bash scripts/ci/npu/npu_ci_install_dependency.sh a3 - # copy required file from our daily cache - cp ~/.cache/modelscope/hub/datasets/otavia/ShareGPT_Vicuna_unfiltered/ShareGPT_V3_unfiltered_cleaned_split.json /tmp - # copy gsm8k dataset - cp ~/.cache/modelscope/hub/datasets/tmp/test.jsonl /tmp - - - name: Run test - timeout-minutes: 60 - env: - SGLANG_USE_MODELSCOPE: true - SGLANG_IS_IN_CI: true - HF_ENDPOINT: https://hf-mirror.com - TORCH_EXTENSIONS_DIR: /tmp/torch_extensions - PYTORCH_NPU_ALLOC_CONF: "expandable_segments:True" - STREAMS_PER_DEVICE: 32 - run: | - cd test - python3 run_suite.py --hw npu --suite stage-b-test-1-npu-a3 --timeout-per-file 3600 - - - stage-b-test-2-npu-a3: + base-b-test-2-npu-a3: needs: [check-changes, pr-gate, set-image-config] - if: needs.check-changes.outputs.main_package == 'true' - runs-on: linux-aarch64-a3-2- - container: + if: ${{ !failure() && !cancelled() && needs.check-changes.outputs.main_package == 'true' }} + uses: ./.github/workflows/_npu-pr-test-stage.yml + with: + self_name: base-b-test-2-npu-a3 + runner_config: linux-aarch64-a3-2- image: ${{ needs.set-image-config.outputs.CANN_image_a3 }} - steps: - - name: Checkout code - uses: actions/checkout@v4 - with: - ref: ${{ inputs.ref || github.ref }} + run_timeout_minutes: '60' + timeout_per_file: '3600' + secrets: inherit - - name: Mark repository safe - run: | - git config --system --add safe.directory ${GITHUB_WORKSPACE} - - - name: Install dependencies - env: - TORCH_CACHE_URL: "http://cache-service.nginx-pypi-cache.svc.cluster.local/whl/cpu" - PYPI_CACHE_URL: "http://cache-service.nginx-pypi-cache.svc.cluster.local/pypi/simple" - UV_INDEX_URL: "http://cache-service.nginx-pypi-cache.svc.cluster.local/pypi/simple" - GITHUB_PROXY_URL: "https://gh-proxy.test.osinfra.cn/" - RUSTUP_CACHE_URL: "http://cache-service.nginx-pypi-cache.svc.cluster.local:8082" - run: | - # speed up by using infra cache services - CACHING_URL="cache-service.nginx-pypi-cache.svc.cluster.local" - sed -Ei "s@(ports|archive).ubuntu.com@${CACHING_URL}:8081@g" /etc/apt/sources.list - pip config set global.index-url http://${CACHING_URL}/pypi/simple - pip config set global.trusted-host "${CACHING_URL}" - - bash scripts/ci/npu/npu_ci_install_dependency.sh a3 - # copy required file from our daily cache - cp ~/.cache/modelscope/hub/datasets/otavia/ShareGPT_Vicuna_unfiltered/ShareGPT_V3_unfiltered_cleaned_split.json /tmp - # copy gsm8k dataset - cp ~/.cache/modelscope/hub/datasets/tmp/test.jsonl /tmp - - - name: Run test - timeout-minutes: 60 - env: - SGLANG_USE_MODELSCOPE: true - SGLANG_IS_IN_CI: true - HF_ENDPOINT: https://hf-mirror.com - TORCH_EXTENSIONS_DIR: /tmp/torch_extensions - PYTORCH_NPU_ALLOC_CONF: "expandable_segments:True" - STREAMS_PER_DEVICE: 32 - run: | - cd test - python3 run_suite.py --hw npu --suite stage-b-test-2-npu-a3 --timeout-per-file 3600 - - - stage-b-test-4-npu-a3: + base-b-test-4-npu-a3: needs: [check-changes, pr-gate, set-image-config] - if: needs.check-changes.outputs.main_package == 'true' - runs-on: linux-aarch64-a3-4- - strategy: - fail-fast: false - matrix: - part: [ 0, 1 ] - container: + if: ${{ !failure() && !cancelled() && needs.check-changes.outputs.main_package == 'true' }} + uses: ./.github/workflows/_npu-pr-test-stage.yml + with: + self_name: base-b-test-4-npu-a3 + runner_config: linux-aarch64-a3-4- image: ${{ needs.set-image-config.outputs.CANN_image_a3 }} - steps: - - name: Checkout code - uses: actions/checkout@v4 - with: - ref: ${{ inputs.ref || github.ref }} + run_timeout_minutes: '120' + timeout_per_file: '3600' + partitions: '{"size":2,"arr":[0, 1]}' + secrets: inherit - - name: Mark repository safe - run: | - git config --system --add safe.directory ${GITHUB_WORKSPACE} - - - name: Install dependencies - env: - TORCH_CACHE_URL: "http://cache-service.nginx-pypi-cache.svc.cluster.local/whl/cpu" - PYPI_CACHE_URL: "http://cache-service.nginx-pypi-cache.svc.cluster.local/pypi/simple" - UV_INDEX_URL: "http://cache-service.nginx-pypi-cache.svc.cluster.local/pypi/simple" - GITHUB_PROXY_URL: "https://gh-proxy.test.osinfra.cn/" - RUSTUP_CACHE_URL: "http://cache-service.nginx-pypi-cache.svc.cluster.local:8082" - run: | - # speed up by using infra cache services - CACHING_URL="cache-service.nginx-pypi-cache.svc.cluster.local" - sed -Ei "s@(ports|archive).ubuntu.com@${CACHING_URL}:8081@g" /etc/apt/sources.list - pip config set global.index-url http://${CACHING_URL}/pypi/simple - pip config set global.trusted-host "${CACHING_URL}" - - bash scripts/ci/npu/npu_ci_install_dependency.sh a3 - # copy required file from our daily cache - cp ~/.cache/modelscope/hub/datasets/otavia/ShareGPT_Vicuna_unfiltered/ShareGPT_V3_unfiltered_cleaned_split.json /tmp - # copy gsm8k dataset - cp ~/.cache/modelscope/hub/datasets/tmp/test.jsonl /tmp - - - name: Run test - timeout-minutes: 120 - env: - SGLANG_USE_MODELSCOPE: true - SGLANG_IS_IN_CI: true - HF_ENDPOINT: https://hf-mirror.com - TORCH_EXTENSIONS_DIR: /tmp/torch_extensions - PYTORCH_NPU_ALLOC_CONF: "expandable_segments:True" - STREAMS_PER_DEVICE: 32 - run: | - pip install sglang_router - cd test - python3 run_suite.py --hw npu --suite stage-b-test-4-npu-a3 --auto-partition-id ${{ matrix.part }} --auto-partition-size 2 --timeout-per-file 3600 - - stage-b-test-8-npu-a3: + base-b-test-8-npu-a3: needs: [check-changes, pr-gate, set-image-config] - if: needs.check-changes.outputs.main_package == 'true' - runs-on: linux-aarch64-a3-8- - container: + if: ${{ !failure() && !cancelled() && needs.check-changes.outputs.main_package == 'true' }} + uses: ./.github/workflows/_npu-pr-test-stage.yml + with: + self_name: base-b-test-8-npu-a3 + runner_config: linux-aarch64-a3-8- image: ${{ needs.set-image-config.outputs.CANN_image_a3 }} - steps: - - name: Checkout code - uses: actions/checkout@v4 - with: - ref: ${{ inputs.ref || github.ref }} + run_timeout_minutes: '60' + timeout_per_file: '3600' + secrets: inherit - - name: Mark repository safe - run: | - git config --system --add safe.directory ${GITHUB_WORKSPACE} - - - name: Install dependencies - env: - TORCH_CACHE_URL: "http://cache-service.nginx-pypi-cache.svc.cluster.local/whl/cpu" - PYPI_CACHE_URL: "http://cache-service.nginx-pypi-cache.svc.cluster.local/pypi/simple" - UV_INDEX_URL: "http://cache-service.nginx-pypi-cache.svc.cluster.local/pypi/simple" - GITHUB_PROXY_URL: "https://gh-proxy.test.osinfra.cn/" - RUSTUP_CACHE_URL: "http://cache-service.nginx-pypi-cache.svc.cluster.local:8082" - run: | - # speed up by using infra cache services - CACHING_URL="cache-service.nginx-pypi-cache.svc.cluster.local" - sed -Ei "s@(ports|archive).ubuntu.com@${CACHING_URL}:8081@g" /etc/apt/sources.list - pip config set global.index-url http://${CACHING_URL}/pypi/simple - pip config set global.trusted-host "${CACHING_URL}" - - bash scripts/ci/npu/npu_ci_install_dependency.sh a3 - # copy required file from our daily cache - cp ~/.cache/modelscope/hub/datasets/otavia/ShareGPT_Vicuna_unfiltered/ShareGPT_V3_unfiltered_cleaned_split.json /tmp - # copy gsm8k dataset - cp ~/.cache/modelscope/hub/datasets/tmp/test.jsonl /tmp - - - name: Run test - timeout-minutes: 60 - env: - SGLANG_USE_MODELSCOPE: true - SGLANG_IS_IN_CI: true - HF_ENDPOINT: https://hf-mirror.com - TORCH_EXTENSIONS_DIR: /tmp/torch_extensions - PYTORCH_NPU_ALLOC_CONF: "expandable_segments:True" - STREAMS_PER_DEVICE: 32 - run: | - cd test - python3 run_suite.py --hw npu --suite stage-b-test-8-npu-a3 --timeout-per-file 3600 - - stage-b-test-16-npu-a3: + base-b-test-16-npu-a3: needs: [check-changes, pr-gate, set-image-config] - if: needs.check-changes.outputs.main_package == 'true' - runs-on: linux-aarch64-a3-16- - container: + if: ${{ !failure() && !cancelled() && needs.check-changes.outputs.main_package == 'true' }} + uses: ./.github/workflows/_npu-pr-test-stage.yml + with: + self_name: base-b-test-16-npu-a3 + runner_config: linux-aarch64-a3-16- image: ${{ needs.set-image-config.outputs.CANN_image_a3 }} - steps: - - name: Checkout code - uses: actions/checkout@v4 - with: - ref: ${{ inputs.ref || github.ref }} - - - name: Mark repository safe - run: | - git config --system --add safe.directory ${GITHUB_WORKSPACE} - - - name: Install dependencies - env: - TORCH_CACHE_URL: "http://cache-service.nginx-pypi-cache.svc.cluster.local/whl/cpu" - PYPI_CACHE_URL: "http://cache-service.nginx-pypi-cache.svc.cluster.local/pypi/simple" - UV_INDEX_URL: "http://cache-service.nginx-pypi-cache.svc.cluster.local/pypi/simple" - GITHUB_PROXY_URL: "https://gh-proxy.test.osinfra.cn/" - RUSTUP_CACHE_URL: "http://cache-service.nginx-pypi-cache.svc.cluster.local:8082" - run: | - # speed up by using infra cache services - CACHING_URL="cache-service.nginx-pypi-cache.svc.cluster.local" - sed -Ei "s@(ports|archive).ubuntu.com@${CACHING_URL}:8081@g" /etc/apt/sources.list - pip config set global.index-url http://${CACHING_URL}/pypi/simple - pip config set global.trusted-host "${CACHING_URL}" - - bash scripts/ci/npu/npu_ci_install_dependency.sh a3 - # copy required file from our daily cache - cp ~/.cache/modelscope/hub/datasets/otavia/ShareGPT_Vicuna_unfiltered/ShareGPT_V3_unfiltered_cleaned_split.json /tmp - # copy gsm8k dataset - cp ~/.cache/modelscope/hub/datasets/tmp/test.jsonl /tmp - - - name: Run test - timeout-minutes: 120 - env: - SGLANG_USE_MODELSCOPE: true - SGLANG_IS_IN_CI: true - HF_ENDPOINT: https://hf-mirror.com - TORCH_EXTENSIONS_DIR: /tmp/torch_extensions - PYTORCH_NPU_ALLOC_CONF: "expandable_segments:True" - STREAMS_PER_DEVICE: 32 - run: | - apt-get install -y libssl-dev - pip install sglang_router - cd test - python3 run_suite.py --hw npu --suite stage-b-test-16-npu-a3 --timeout-per-file 3600 + run_timeout_minutes: '120' + timeout_per_file: '3600' + secrets: inherit multimodal-gen-test-1-npu-a3: needs: [check-changes, pr-gate, set-image-config] - if: needs.check-changes.outputs.multimodal_gen == 'true' + if: ${{ !failure() && !cancelled() && needs.check-changes.outputs.multimodal_gen == 'true' }} runs-on: linux-aarch64-a3-800t-2 container: image: ${{ needs.set-image-config.outputs.CANN_image_a3 }} @@ -443,7 +222,7 @@ jobs: multimodal-gen-test-2-npu-a3: needs: [check-changes, pr-gate, set-image-config] - if: needs.check-changes.outputs.multimodal_gen == 'true' + if: ${{ !failure() && !cancelled() && needs.check-changes.outputs.multimodal_gen == 'true' }} runs-on: linux-aarch64-a3-800t-2 container: image: ${{ needs.set-image-config.outputs.CANN_image_a3 }} @@ -498,146 +277,134 @@ jobs: if-no-files-found: ignore retention-days: 7 - pr-single-node-tests: - name: single-node-poc - needs: [check-changes, pr-gate, set-image-config] - if: needs.check-changes.outputs.main_package == 'true' - strategy: - fail-fast: false - max-parallel: 6 - matrix: - test_config: - # qwen3_6_27b performance tests - - name: qwen3_6_27b_w8a8_1p_in64k_out1k_50ms - runner: linux-aarch64-a3-800t-2 - test_case: test/registered/npu/performance/qwen3_6_27b/test_npu_qwen3_6_27b_w8a8_1p_in64k_out1k_50ms.py - test_type: 'perf' - - # - name: qwen3_6_27b_1p_in1024x1024_30_out1024_50ms - # runner: linux-aarch64-a3-800t-2 - # test_case: test/registered/npu/performance/qwen3_6_27b/test_npu_qwen3_6_27b_1p_in1024x1024_30_out1024_50ms.py - # test_type: 'perf' - - # # qwen3_8b performance tests - # - name: qwen3_8b_w8a8_1p_in3k5_out1k5_50ms - # runner: linux-aarch64-a3-800t-2 - # test_case: test/registered/npu/performance/qwen3-8b/test_npu_qwen3_8b_w8a8_1p_in3k5_out1k5_50ms.py - # test_type: 'perf' - - # # qwen3_30b_a3b performance tests - # - name: qwen3_30b_w8a8_1p_in3k5_out1k5_50ms - # runner: linux-aarch64-a3-800t-2 - # test_case: test/registered/npu/performance/qwen3_30b_a3b/test_npu_qwen3_30b_w8a8_1p_in3k5_out1k5_50ms.py - # test_type: 'perf' - - # # qwen3_6_35b_a3b performance tests - # - name: qwen3_6_35b_a3b_1p_in64k_out1k_prefix90_50ms - # runner: linux-aarch64-a3-800t-2 - # test_case: test/registered/npu/performance/qwen3_6_35b_a3b/test_npu_qwen3_6_35b_a3b_1p_in64k_out1k_prefix90_50ms.py - # test_type: 'perf' - - # # qwen3_vl_8b_thinking accuracy tests - # - name: qwen3_vl_8b_thinking_1p_mmmu - # runner: linux-aarch64-a3-2- - # test_case: test/registered/npu/accuracy/qwen3_vl_8b_thinking/test_npu_qwen3_vl_8b_thinking_1p_mmmu.py - # test_type: 'accuracy' - - # # qwen3_32b performance tests - # - name: qwen3_32b_w8a8_2p_in3k5_out1k5_50ms - # runner: linux-aarch64-a3-800t-4 - # test_case: test/registered/npu/performance/qwen3_32b/test_npu_qwen3_32b_w8a8_2p_in3k5_out1k5_50ms.py - # test_type: 'perf' - - # # qwen3_next_80b_a3b performance tests - # - name: qwen3_next_80b_w8a8_2p_in6k_out1k5_bs16 - # runner: linux-aarch64-a3-800t-4 - # test_case: test/registered/npu/performance/qwen3_next_80b_a3b_instruct/test_npu_qwen3_next_80b_w8a8_2p_in6k_out1k5_bs16.py - # test_type: 'perf' - - # # minimax_m2_5 performance tests - # - name: minimax_m2_5_w8a8_4p_in64k_out1k_prefix90_50ms - # runner: linux-aarch64-a3-800t-8 - # test_case: test/registered/npu/performance/minimax_m2_5/test_npu_minimax_m2_5_w8a8_4p_in64k_out1k_prefix90_50ms.py - # test_type: 'perf' - - # # deepseek_v4_flash performance tests - # - name: deepseek_v4_flash_w8a8_8p_in8k_out1k_50ms - # runner: linux-aarch64-a3-800t-16 - # test_case: test/registered/npu/performance/deepseek_v4_flash/test_npu_deepseek_v4_flash_w8a8_8p_in8k_out1k_50ms.py - # test_type: 'perf' - - # # kimi_k2_6 performance tests - # - name: kimi_k2_6_w4a8_8p_in3k5_out1k5_20ms - # runner: linux-aarch64-a3-800t-16 - # test_case: test/registered/npu/performance/kimi_k2_6/test_npu_kimi_k2_6_w4a8_8p_in3k5_out1k5_20ms.py - # test_type: 'perf' - - # # qwen3_235b performance tests - # - name: qwen3_235b_w8a8_8p_in3k5_out1k5_50ms - # runner: linux-aarch64-a3-800t-16 - # test_case: test/registered/npu/performance/qwen3_235b_a22b/test_npu_qwen3_235b_w8a8_8p_in3k5_out1k5_50ms.py - # test_type: 'perf' - - # # qwen3_5_397b performance tests - # - name: qwen3_5_397b_w4a8_8p_in3k5_out1k5_50ms - # runner: linux-aarch64-a3-800t-16 - # test_case: test/registered/npu/performance/qwen3_5_397b/test_npu_qwen3_5_397b_w4a8_8p_in3k5_out1k5_50ms.py - # test_type: 'perf' - - # NPU accuracy tests - # - name: glm4_7_flash_1p_gsm8k - # runner: linux-aarch64-a3-2- - # test_case: test/registered/npu/accuracy/glm4_7_flash/test_npu_glm4_7_flash_1p_gsm8k.py - # test_type: 'accuracy' - # - name: qwen3_vl_30b_a3b_bf16_2p_gsm8k - # runner: linux-aarch64-a3-4- - # test_case: test/registered/npu/accuracy/qwen3_vl_30b_a3b/test_npu_qwen3_vl_30b_a3b_bf16_2p_gsm8k.py - # test_type: 'accuracy' - - name: glm5_top64_pruned_bf16_8p_gsm8k - runner: linux-aarch64-a3-16- - test_case: test/registered/npu/accuracy/glm5_top64_pruned/test_npu_glm5_top64_pruned_bf16_8p_gsm8k.py - test_type: 'accuracy' - # - name: moonshotai_moonlight_16b_a3b_bf16_1p_gsm8k - # runner: linux-aarch64-a3-2- - # test_case: test/registered/npu/accuracy/moonshotai_moonlight_16b_a3b/test_npu_moonlight_16b_a3b_bf16_1p_gsm8k.py - # test_type: 'accuracy' - # - name: qwen3_5_9b_bf16_1p_gsm8k - # runner: linux-aarch64-a3-2- - # test_case: test/registered/npu/accuracy/qwen3_5_9b/test_npu_qwen3_5_9b_bf16_1p_gsm8k.py - # test_type: 'accuracy' - # - name: qwen3_vl_8b_bf16_2p_gsm8k - # runner: linux-aarch64-a3-4- - # test_case: test/registered/npu/accuracy/qwen3_vl_8b/test_npu_qwen3_vl_8b_bf16_2p_gsm8k.py - # test_type: 'accuracy' - - uses: ./.github/workflows/nightly-test-npu-e2e-single-node.yml + base-c-test-acc-2-npu-a3: + name: base-c-test-acc-2-npu-a3 + needs: [ check-changes, pr-gate, set-image-config ] + if: ${{ !failure() && !cancelled() && needs.check-changes.outputs.main_package == 'true' }} + uses: ./.github/workflows/_npu-single-node-test-stage.yml with: - runner: ${{ matrix.test_config.runner }} - test_type: ${{ matrix.test_config.test_type }} - test_config_name: ${{ matrix.test_config.name }} - test_case: ${{ matrix.test_config.test_case }} + runner: linux-aarch64-a3-2- + test_type: 'accuracy' + test_suite: base-c-test-acc-2-npu-a3 image: ${{ needs.set-image-config.outputs.CANN_image_a3 }} - install_sglang_from_source: false install_sglang_deps: true device_type_for_deps: 'a3' - transformers_version: '' + + base-c-test-acc-4-npu-a3: + name: base-c-test-acc-4-npu-a3 + needs: [ check-changes, pr-gate, set-image-config ] + if: ${{ !failure() && !cancelled() && needs.check-changes.outputs.main_package == 'true' }} + uses: ./.github/workflows/_npu-single-node-test-stage.yml + with: + runner: linux-aarch64-a3-4- + test_type: 'accuracy' + test_suite: base-c-test-acc-4-npu-a3 + image: ${{ needs.set-image-config.outputs.CANN_image_a3 }} + install_sglang_deps: true + device_type_for_deps: 'a3' + + base-c-test-acc-8-npu-a3: + name: base-c-test-acc-8-npu-a3 + needs: [ check-changes, pr-gate, set-image-config ] + if: ${{ !failure() && !cancelled() && needs.check-changes.outputs.main_package == 'true' }} + uses: ./.github/workflows/_npu-single-node-test-stage.yml + with: + runner: linux-aarch64-a3-8- + test_type: 'accuracy' + test_suite: base-c-test-acc-8-npu-a3 + image: ${{ needs.set-image-config.outputs.CANN_image_a3 }} + install_sglang_deps: true + device_type_for_deps: 'a3' + + base-c-test-acc-16-npu-a3: + name: base-c-test-acc-16-npu-a3 + needs: [ check-changes, pr-gate, set-image-config ] + if: ${{ !failure() && !cancelled() && needs.check-changes.outputs.main_package == 'true' }} + uses: ./.github/workflows/_npu-single-node-test-stage.yml + with: + runner: linux-aarch64-a3-16- + test_type: 'accuracy' + test_suite: base-c-test-acc-16-npu-a3 + image: ${{ needs.set-image-config.outputs.CANN_image_a3 }} + install_sglang_deps: true + device_type_for_deps: 'a3' + + base-c-test-perf-2-npu-a3: + name: base-c-test-perf-2-npu-a3 + needs: [ check-changes, pr-gate, set-image-config, base-c-test-acc-2-npu-a3 ] + if: ${{ !failure() && !cancelled() && needs.check-changes.outputs.main_package == 'true' }} + uses: ./.github/workflows/_npu-single-node-test-stage.yml + with: + runner: linux-aarch64-a3-800t-2 + test_type: 'perf' + test_suite: base-c-test-perf-2-npu-a3 + image: ${{ needs.set-image-config.outputs.CANN_image_a3 }} + install_sglang_deps: true + device_type_for_deps: 'a3' + + base-c-test-perf-4-npu-a3: + name: base-c-test-perf-4-npu-a3 + needs: [ check-changes, pr-gate, set-image-config, base-c-test-acc-4-npu-a3 ] + if: ${{ !failure() && !cancelled() && needs.check-changes.outputs.main_package == 'true' }} + uses: ./.github/workflows/_npu-single-node-test-stage.yml + with: + runner: linux-aarch64-a3-800t-4 + test_type: 'perf' + test_suite: base-c-test-perf-4-npu-a3 + image: ${{ needs.set-image-config.outputs.CANN_image_a3 }} + install_sglang_deps: true + device_type_for_deps: 'a3' + + base-c-test-perf-8-npu-a3: + name: base-c-test-perf-8-npu-a3 + needs: [ check-changes, pr-gate, set-image-config, base-c-test-acc-8-npu-a3 ] + if: ${{ !failure() && !cancelled() && needs.check-changes.outputs.main_package == 'true' }} + uses: ./.github/workflows/_npu-single-node-test-stage.yml + with: + runner: linux-aarch64-a3-800t-8 + test_type: 'perf' + test_suite: base-c-test-perf-8-npu-a3 + image: ${{ needs.set-image-config.outputs.CANN_image_a3 }} + install_sglang_deps: true + device_type_for_deps: 'a3' + + base-c-test-perf-16-npu-a3: + name: base-c-test-perf-16-npu-a3 + needs: [ check-changes, pr-gate, set-image-config, base-c-test-acc-16-npu-a3 ] + if: ${{ !failure() && !cancelled() && needs.check-changes.outputs.main_package == 'true' }} + uses: ./.github/workflows/_npu-single-node-test-stage.yml + with: + runner: linux-aarch64-a3-800t-16 + test_type: 'perf' + test_suite: base-c-test-perf-16-npu-a3 + image: ${{ needs.set-image-config.outputs.CANN_image_a3 }} + install_sglang_deps: true + device_type_for_deps: 'a3' + pr-test-npu-finish: needs: [ check-changes, - stage-a-unit-test-npu, - stage-b-test-1-npu-a3, - stage-b-test-2-npu-a3, - stage-b-test-4-npu-a3, - stage-b-test-8-npu-a3, - stage-b-test-16-npu-a3, + base-a-test-1-npu-a2, + base-b-test-1-npu-a3, + base-b-test-2-npu-a3, + base-b-test-4-npu-a3, + base-b-test-8-npu-a3, + base-b-test-16-npu-a3, multimodal-gen-test-1-npu-a3, multimodal-gen-test-2-npu-a3, - pr-single-node-tests, + base-c-test-acc-2-npu-a3, + base-c-test-acc-4-npu-a3, + base-c-test-acc-8-npu-a3, + base-c-test-acc-16-npu-a3, + base-c-test-perf-2-npu-a3, + base-c-test-perf-4-npu-a3, + base-c-test-perf-8-npu-a3, + base-c-test-perf-16-npu-a3, ] if: always() runs-on: ubuntu-latest diff --git a/python/sglang/test/ascend/e2e/test_npu_accuracy_utils.py b/python/sglang/test/ascend/e2e/test_npu_accuracy_utils.py index 6dc75173a..5fcdf3d5c 100644 --- a/python/sglang/test/ascend/e2e/test_npu_accuracy_utils.py +++ b/python/sglang/test/ascend/e2e/test_npu_accuracy_utils.py @@ -1,10 +1,15 @@ +import glob +import inspect import json import logging import os import re +import shutil import subprocess +import sys import threading import time +from datetime import datetime from urllib.parse import urlparse from sglang.srt.utils import kill_process_tree @@ -289,9 +294,117 @@ class TestNpuAccuracyTestCaseBase(CustomTestCase): max_attempts = 2 n_runs = 3 accuracy = 0.1 + test_type = "accuracy" + + @classmethod + def _get_tc_name(cls): + """Derive the test case name from the test file (filename without + extension). Mirrors the workflow's ``tc_name=${test_case##*/}`` logic + so each case in a suite writes to its own output path.""" + try: + tc_file = inspect.getfile(cls) + except (TypeError, OSError): + tc_file = getattr(sys.modules.get(cls.__module__), "__file__", "") + return os.path.splitext(os.path.basename(tc_file))[0] + + @classmethod + def _setup_per_case_output(cls): + """Set up per-case output directories and env vars. + + Extracted from ``nightly-test-npu-e2e-single-node.yml`` so that when a + suite is executed, each case writes its metrics/plog to a path derived + from the case file rather than the suite name. + """ + cls.tc_name = cls._get_tc_name() + current_date = datetime.now().strftime("%Y%m%d") + test_type = getattr(cls, "test_type", "accuracy") + base_output = f"/root/.cache/tests/output/{test_type}/{current_date}" + os.makedirs(base_output, exist_ok=True) + cls.metrics_data_file = os.path.join(base_output, cls.tc_name) + os.makedirs(cls.metrics_data_file, exist_ok=True) + # Override env vars so evalscope/dump_metric write to per-case paths. + os.environ["METRICS_DATA_FILE"] = cls.metrics_data_file + os.environ["SGLANG_TEST_METRICS_OUTPUT"] = os.path.join( + cls.metrics_data_file, "metrics" + ) + logger.info( + "Per-case output: tc_name=%s metrics_data_file=%s", + cls.tc_name, + cls.metrics_data_file, + ) + + @classmethod + def _save_metrics_json(cls): + """Write per-case ``metrics.json`` from ``dump_metric`` JSONL files. + + Replaces the workflow's stdout-parsing + ``dump_metrics.py`` logic so + each case in a suite persists its own metrics snapshot. + """ + if not getattr(cls, "metrics_data_file", None): + return + metrics = {} + baselines = {} + pattern = os.path.join(cls.metrics_data_file, "metrics.*.jsonl") + for jsonl_path in glob.glob(pattern): + try: + with open(jsonl_path, "r", encoding="utf-8") as f: + for line in f: + line = line.strip() + if not line: + continue + record = json.loads(line) + name = record.get("metric_name") + value = record.get("value") + if name is None: + continue + if name.endswith("_baseline"): + baselines[name[: -len("_baseline")]] = value + else: + metrics[name] = value + except Exception as e: + logger.warning("Failed to read %s: %s", jsonl_path, e) + out_path = os.path.join(cls.metrics_data_file, "metrics.json") + payload = { + "test_case": cls.tc_name, + "test_type": getattr(cls, "test_type", "accuracy"), + "metrics": metrics, + "baselines": baselines, + } + try: + with open(out_path, "w", encoding="utf-8") as f: + json.dump(payload, f, ensure_ascii=False) + logger.info("Saved per-case metrics to %s", out_path) + except Exception as e: + logger.warning("Failed to write metrics.json: %s", e) + + @classmethod + def _backup_plog(cls): + """Backup Ascend plog files to a per-case path. + + Replaces the workflow's ``Backup plog`` step so each case in a suite + gets its own plog snapshot instead of all cases sharing the suite name. + """ + plog_path = "/root/ascend/log/debug/plog" + if not os.path.isdir(plog_path): + return + tc_name = getattr(cls, "tc_name", None) + if not tc_name: + return + hostname = os.getenv("HOSTNAME", "unknown") + target = os.path.join("/root/.cache/tests/logs/plog", tc_name, hostname) + os.makedirs(target, exist_ok=True) + for name in os.listdir(plog_path): + src = os.path.join(plog_path, name) + if os.path.isfile(src): + try: + shutil.copy2(src, os.path.join(target, name)) + except Exception as e: + logger.warning("Failed to copy plog %s: %s", name, e) + logger.info("Backed up plog to %s", target) @classmethod def setUpClass(cls): + cls._setup_per_case_output() cls.base_url = DEFAULT_URL_FOR_TEST env = os.environ.copy() for key, value in env.items(): @@ -318,6 +431,8 @@ class TestNpuAccuracyTestCaseBase(CustomTestCase): kill_process_tree(cls.process.pid) except Exception as e: logger.error(f"Error during tearDown: {e}") + cls._save_metrics_json() + cls._backup_plog() def run_accuracy(self): parsed_url = urlparse(self.base_url) diff --git a/python/sglang/test/ascend/e2e/test_npu_performance_utils.py b/python/sglang/test/ascend/e2e/test_npu_performance_utils.py index 21cd57171..dfa30e6b8 100644 --- a/python/sglang/test/ascend/e2e/test_npu_performance_utils.py +++ b/python/sglang/test/ascend/e2e/test_npu_performance_utils.py @@ -1,9 +1,15 @@ +import glob +import inspect +import json import logging import os import re +import shutil import subprocess +import sys import threading import time +from datetime import datetime from functools import wraps from urllib.parse import urlparse @@ -909,8 +915,115 @@ class TestNpuPerformanceTestCaseBase(CustomTestCase): generation_kwargs = None pop_sglang_is_in_ci_for_gsp = False + @classmethod + def _get_tc_name(cls): + """Derive the test case name from the test file (filename without + extension). Mirrors the workflow's ``tc_name=${test_case##*/}`` logic + so each case in a suite writes to its own output path.""" + try: + tc_file = inspect.getfile(cls) + except (TypeError, OSError): + tc_file = getattr(sys.modules.get(cls.__module__), "__file__", "") + return os.path.splitext(os.path.basename(tc_file))[0] + + @classmethod + def _setup_per_case_output(cls): + """Set up per-case output directories and env vars. + + Extracted from ``nightly-test-npu-e2e-single-node.yml`` so that when a + suite is executed, each case writes its metrics/plog to a path derived + from the case file rather than the suite name. + """ + cls.tc_name = cls._get_tc_name() + current_date = datetime.now().strftime("%Y%m%d") + test_type = getattr(cls, "test_type", "perf") + base_output = f"/root/.cache/tests/output/{test_type}/{current_date}" + os.makedirs(base_output, exist_ok=True) + cls.metrics_data_file = os.path.join(base_output, cls.tc_name) + os.makedirs(cls.metrics_data_file, exist_ok=True) + # Override env vars so evalscope/dump_metric write to per-case paths. + os.environ["METRICS_DATA_FILE"] = cls.metrics_data_file + os.environ["SGLANG_TEST_METRICS_OUTPUT"] = os.path.join( + cls.metrics_data_file, "metrics" + ) + logger.info( + "Per-case output: tc_name=%s metrics_data_file=%s", + cls.tc_name, + cls.metrics_data_file, + ) + + @classmethod + def _save_metrics_json(cls): + """Write per-case ``metrics.json`` from ``dump_metric`` JSONL files. + + Replaces the workflow's stdout-parsing + ``dump_metrics.py`` logic so + each case in a suite persists its own metrics snapshot. + """ + if not getattr(cls, "metrics_data_file", None): + return + metrics = {} + baselines = {} + pattern = os.path.join(cls.metrics_data_file, "metrics.*.jsonl") + for jsonl_path in glob.glob(pattern): + try: + with open(jsonl_path, "r", encoding="utf-8") as f: + for line in f: + line = line.strip() + if not line: + continue + record = json.loads(line) + name = record.get("metric_name") + value = record.get("value") + if name is None: + continue + if name.endswith("_baseline"): + baselines[name[: -len("_baseline")]] = value + else: + metrics[name] = value + except Exception as e: + logger.warning("Failed to read %s: %s", jsonl_path, e) + out_path = os.path.join(cls.metrics_data_file, "metrics.json") + payload = { + "test_case": cls.tc_name, + "test_type": getattr(cls, "test_type", "accuracy"), + "metrics": metrics, + "baselines": baselines, + } + try: + with open(out_path, "w", encoding="utf-8") as f: + json.dump(payload, f, ensure_ascii=False) + logger.info("Saved per-case metrics to %s", out_path) + except Exception as e: + logger.warning("Failed to write metrics.json: %s", e) + + @classmethod + def _backup_plog(cls): + """Backup Ascend plog files to a per-case path. + + Replaces the workflow's ``Backup plog`` step so each case in a suite + gets its own plog snapshot instead of all cases sharing the suite name. + """ + plog_path = "/root/ascend/log/debug/plog" + if not os.path.isdir(plog_path): + return + tc_name = getattr(cls, "tc_name", None) + if not tc_name: + return + hostname = os.getenv("HOSTNAME", "unknown") + target = os.path.join("/root/.cache/tests/logs/plog", tc_name, hostname) + os.makedirs(target, exist_ok=True) + for name in os.listdir(plog_path): + src = os.path.join(plog_path, name) + if os.path.isfile(src): + try: + shutil.copy2(src, os.path.join(target, name)) + except Exception as e: + logger.warning("Failed to copy plog %s: %s", name, e) + logger.info("Backed up plog to %s", target) + @classmethod def setUpClass(cls): + cls._setup_per_case_output() cls.base_url = DEFAULT_URL_FOR_TEST env = os.environ.copy() for key, value in env.items(): @@ -937,6 +1050,8 @@ class TestNpuPerformanceTestCaseBase(CustomTestCase): kill_process_tree(cls.process.pid) except Exception as e: logger.error(f"Error during tearDown: {e}") + cls._save_metrics_json() + cls._backup_plog() @retry() def run_throughput(self): diff --git a/test/registered/npu/accuracy/glm4_7_flash/test_npu_glm4_7_flash_1p_gsm8k.py b/test/registered/npu/accuracy/glm4_7_flash/test_npu_glm4_7_flash_1p_gsm8k.py index f2de0b3d2..7a24cae35 100644 --- a/test/registered/npu/accuracy/glm4_7_flash/test_npu_glm4_7_flash_1p_gsm8k.py +++ b/test/registered/npu/accuracy/glm4_7_flash/test_npu_glm4_7_flash_1p_gsm8k.py @@ -6,12 +6,7 @@ from sglang.test.ascend.e2e.test_npu_accuracy_utils import ( from sglang.test.ascend.e2e.test_npu_performance_utils import GLM_4_7_FLASH_MODEL_PATH from sglang.test.ci.ci_register import register_npu_ci -register_npu_ci( - est_time=3600, - suite="", - nightly=True, - disabled="accuracy testcase", -) +register_npu_ci(est_time=3600, suite="base-c-test-acc-2-npu-a3") ENVS = { "PYTORCH_NPU_ALLOC_CONF": "expandable_segments:True", diff --git a/test/registered/npu/accuracy/glm5_top64_pruned/test_npu_glm5_top64_pruned_bf16_8p_gsm8k.py b/test/registered/npu/accuracy/glm5_top64_pruned/test_npu_glm5_top64_pruned_bf16_8p_gsm8k.py index d56b2d5ba..7ae8616ff 100644 --- a/test/registered/npu/accuracy/glm5_top64_pruned/test_npu_glm5_top64_pruned_bf16_8p_gsm8k.py +++ b/test/registered/npu/accuracy/glm5_top64_pruned/test_npu_glm5_top64_pruned_bf16_8p_gsm8k.py @@ -8,12 +8,7 @@ from sglang.test.ascend.e2e.test_npu_performance_utils import ( ) from sglang.test.ci.ci_register import register_npu_ci -register_npu_ci( - est_time=3600, - suite="", - nightly=True, - disabled="accuracy testcase", -) +register_npu_ci(est_time=3600, suite="base-c-test-acc-16-npu-a3") ENVS = { "SGLANG_SET_CPU_AFFINITY": "1", @@ -54,7 +49,7 @@ class TestNPUGLM5_Top64_Pruned_GSM8K(TestNpuAccuracyTestCaseBase): model = GLM5_TOP64_PRUNED_GSM8K_MODEL_PATH envs = ENVS other_args = OTHER_ARGS - accuracy = 0.50 + accuracy = 0.48 datasets = ["gsm8k"] generation_config = { "max_tokens": 2048, diff --git a/test/registered/npu/accuracy/moonshotai_moonlight_16b_a3b/test_npu_moonlight_16b_a3b_bf16_1p_gsm8k.py b/test/registered/npu/accuracy/moonshotai_moonlight_16b_a3b/test_npu_moonlight_16b_a3b_bf16_1p_gsm8k.py index ff490bb46..2a4675e16 100644 --- a/test/registered/npu/accuracy/moonshotai_moonlight_16b_a3b/test_npu_moonlight_16b_a3b_bf16_1p_gsm8k.py +++ b/test/registered/npu/accuracy/moonshotai_moonlight_16b_a3b/test_npu_moonlight_16b_a3b_bf16_1p_gsm8k.py @@ -8,12 +8,7 @@ from sglang.test.ascend.e2e.test_npu_performance_utils import ( ) from sglang.test.ci.ci_register import register_npu_ci -register_npu_ci( - est_time=3600, - suite="", - nightly=True, - disabled="accuracy testcase", -) +register_npu_ci(est_time=3600, suite="base-c-test-acc-2-npu-a3") MODEL_ENVS = { "SGLANG_SET_CPU_AFFINITY": "1", diff --git a/test/registered/npu/accuracy/qwen3_5_9b/test_npu_qwen3_5_9b_bf16_1p_gsm8k.py b/test/registered/npu/accuracy/qwen3_5_9b/test_npu_qwen3_5_9b_bf16_1p_gsm8k.py index 7b49dd52e..c997de8bf 100644 --- a/test/registered/npu/accuracy/qwen3_5_9b/test_npu_qwen3_5_9b_bf16_1p_gsm8k.py +++ b/test/registered/npu/accuracy/qwen3_5_9b/test_npu_qwen3_5_9b_bf16_1p_gsm8k.py @@ -8,12 +8,7 @@ from sglang.test.ascend.e2e.test_npu_performance_utils import ( ) from sglang.test.ci.ci_register import register_npu_ci -register_npu_ci( - est_time=3600, - suite="stage-b-test-2-npu-a3", - nightly=True, - disabled="accuracy testcase", -) +register_npu_ci(est_time=3600, suite="base-c-test-acc-2-npu-a3") QWEN3_5_9B_ENVS = { "SGLANG_SET_CPU_AFFINITY": "1", diff --git a/test/registered/npu/accuracy/qwen3_vl_30b_a3b/test_npu_qwen3_vl_30b_a3b_bf16_2p_gsm8k.py b/test/registered/npu/accuracy/qwen3_vl_30b_a3b/test_npu_qwen3_vl_30b_a3b_bf16_2p_gsm8k.py index ffaf7e9de..0d9e06817 100644 --- a/test/registered/npu/accuracy/qwen3_vl_30b_a3b/test_npu_qwen3_vl_30b_a3b_bf16_2p_gsm8k.py +++ b/test/registered/npu/accuracy/qwen3_vl_30b_a3b/test_npu_qwen3_vl_30b_a3b_bf16_2p_gsm8k.py @@ -8,12 +8,7 @@ from sglang.test.ascend.e2e.test_npu_performance_utils import ( ) from sglang.test.ci.ci_register import register_npu_ci -register_npu_ci( - est_time=3600, - suite="stage-b-test-4-npu-a3", - nightly=True, - disabled="accuracy testcase", -) +register_npu_ci(est_time=3600, suite="base-c-test-acc-4-npu-a3") QWEN3_VL_30B_A3B_ENVS = { "SGLANG_SET_CPU_AFFINITY": "1", diff --git a/test/registered/npu/accuracy/qwen3_vl_8b/test_npu_qwen3_vl_8b_bf16_2p_gsm8k.py b/test/registered/npu/accuracy/qwen3_vl_8b/test_npu_qwen3_vl_8b_bf16_2p_gsm8k.py index 6d44fc382..ffca0f1ed 100644 --- a/test/registered/npu/accuracy/qwen3_vl_8b/test_npu_qwen3_vl_8b_bf16_2p_gsm8k.py +++ b/test/registered/npu/accuracy/qwen3_vl_8b/test_npu_qwen3_vl_8b_bf16_2p_gsm8k.py @@ -8,12 +8,7 @@ from sglang.test.ascend.e2e.test_npu_performance_utils import ( ) from sglang.test.ci.ci_register import register_npu_ci -register_npu_ci( - est_time=3600, - suite="stage-b-test-4-npu-a3", - nightly=True, - disabled="accuracy testcase", -) +register_npu_ci(est_time=3600, suite="base-c-test-acc-4-npu-a3") QWEN3_VL_8B_ENVS = { "SGLANG_SET_CPU_AFFINITY": "1", diff --git a/test/registered/npu/accuracy/qwen3_vl_8b_thinking/test_npu_qwen3_vl_8b_thinking_1p_mmmu.py b/test/registered/npu/accuracy/qwen3_vl_8b_thinking/test_npu_qwen3_vl_8b_thinking_1p_mmmu.py index 2f9ce2f8d..1cfefbc1f 100644 --- a/test/registered/npu/accuracy/qwen3_vl_8b_thinking/test_npu_qwen3_vl_8b_thinking_1p_mmmu.py +++ b/test/registered/npu/accuracy/qwen3_vl_8b_thinking/test_npu_qwen3_vl_8b_thinking_1p_mmmu.py @@ -9,12 +9,7 @@ from sglang.test.ascend.e2e.test_npu_performance_utils import ( ) from sglang.test.ci.ci_register import register_npu_ci -register_npu_ci( - est_time=3600, - suite="", - nightly=True, - disabled="performance testcase", -) +register_npu_ci(est_time=3600, suite="base-c-test-perf-2-npu-a3") _is_pr_pipeline = os.environ.get("GITHUB_EVENT_NAME") == "pull_request" diff --git a/test/registered/npu/basic_function/HiCache/test_npu_hicache_mha.py b/test/registered/npu/basic_function/HiCache/test_npu_hicache_mha.py index b83fdaf70..99b843ed2 100644 --- a/test/registered/npu/basic_function/HiCache/test_npu_hicache_mha.py +++ b/test/registered/npu/basic_function/HiCache/test_npu_hicache_mha.py @@ -13,7 +13,7 @@ from sglang.test.test_utils import ( popen_launch_server, ) -register_npu_ci(est_time=400, suite="stage-b-test-1-npu-a3", nightly=False) +register_npu_ci(est_time=400, suite="base-b-test-1-npu-a3") register_npu_ci(est_time=400, suite="nightly-1-npu-a3", nightly=True) TEST_MODEL_MATRIX = { diff --git a/test/registered/npu/basic_function/HiCache/test_npu_hicache_mla.py b/test/registered/npu/basic_function/HiCache/test_npu_hicache_mla.py index 9448071ed..496a38373 100644 --- a/test/registered/npu/basic_function/HiCache/test_npu_hicache_mla.py +++ b/test/registered/npu/basic_function/HiCache/test_npu_hicache_mla.py @@ -13,7 +13,7 @@ from sglang.test.test_utils import ( popen_launch_server, ) -register_npu_ci(est_time=400, suite="stage-b-test-4-npu-a3", nightly=False) +register_npu_ci(est_time=400, suite="base-b-test-4-npu-a3") register_npu_ci(est_time=400, suite="nightly-4-npu-a3", nightly=True) TEST_MODEL_MATRIX = { diff --git a/test/registered/npu/basic_function/backends/test_npu_sampling_backend.py b/test/registered/npu/basic_function/backends/test_npu_sampling_backend.py index 8b278d3f5..df93ae543 100644 --- a/test/registered/npu/basic_function/backends/test_npu_sampling_backend.py +++ b/test/registered/npu/basic_function/backends/test_npu_sampling_backend.py @@ -14,7 +14,7 @@ from sglang.test.test_utils import ( popen_launch_server, ) -register_npu_ci(est_time=400, suite="stage-b-test-1-npu-a3", nightly=False) +register_npu_ci(est_time=400, suite="base-b-test-1-npu-a3") register_npu_ci(est_time=400, suite="nightly-1-npu-a3", nightly=True) diff --git a/test/registered/npu/basic_function/dllm/test_npu_llada2_mini.py b/test/registered/npu/basic_function/dllm/test_npu_llada2_mini.py index 4d55a335a..dc55d5565 100644 --- a/test/registered/npu/basic_function/dllm/test_npu_llada2_mini.py +++ b/test/registered/npu/basic_function/dllm/test_npu_llada2_mini.py @@ -10,7 +10,7 @@ from sglang.test.test_utils import ( write_github_step_summary, ) -register_npu_ci(est_time=400, suite="stage-b-test-4-npu-a3", nightly=False) +register_npu_ci(est_time=400, suite="base-b-test-4-npu-a3") register_npu_ci(est_time=400, suite="nightly-1-npu-a3", nightly=True) diff --git a/test/registered/npu/basic_function/dp_attn/test_npu_dp_attention.py b/test/registered/npu/basic_function/dp_attn/test_npu_dp_attention.py index a93a94312..a830ea2b6 100644 --- a/test/registered/npu/basic_function/dp_attn/test_npu_dp_attention.py +++ b/test/registered/npu/basic_function/dp_attn/test_npu_dp_attention.py @@ -23,7 +23,7 @@ from sglang.test.test_utils import ( popen_launch_server, ) -register_npu_ci(est_time=400, suite="stage-b-test-4-npu-a3", nightly=False) +register_npu_ci(est_time=400, suite="base-b-test-4-npu-a3") register_npu_ci(est_time=400, suite="nightly-4-npu-a3", nightly=True) diff --git a/test/registered/npu/basic_function/optimization_debug/test_npu_compile_graph_tp1_bf16.py b/test/registered/npu/basic_function/optimization_debug/test_npu_compile_graph_tp1_bf16.py index 9255b5337..05cad7f0e 100644 --- a/test/registered/npu/basic_function/optimization_debug/test_npu_compile_graph_tp1_bf16.py +++ b/test/registered/npu/basic_function/optimization_debug/test_npu_compile_graph_tp1_bf16.py @@ -14,7 +14,7 @@ from sglang.test.test_utils import ( popen_launch_server, ) -register_npu_ci(est_time=400, suite="stage-b-test-1-npu-a3", nightly=False) +register_npu_ci(est_time=400, suite="base-b-test-1-npu-a3") register_npu_ci(est_time=400, suite="nightly-1-npu-a3", nightly=True) TEST_MODEL_MATRIX = { diff --git a/test/registered/npu/basic_function/optimization_debug/test_npu_piecewise_graph_prefill.py b/test/registered/npu/basic_function/optimization_debug/test_npu_piecewise_graph_prefill.py index b4b620c9f..c3230817b 100644 --- a/test/registered/npu/basic_function/optimization_debug/test_npu_piecewise_graph_prefill.py +++ b/test/registered/npu/basic_function/optimization_debug/test_npu_piecewise_graph_prefill.py @@ -12,7 +12,7 @@ from sglang.test.test_utils import ( run_bench_one_batch, ) -register_npu_ci(est_time=400, suite="stage-b-test-1-npu-a3", nightly=False) +register_npu_ci(est_time=400, suite="base-b-test-1-npu-a3") register_npu_ci(est_time=400, suite="nightly-1-npu-a3", nightly=True) diff --git a/test/registered/npu/basic_function/parallel_strategy/data_parallelism/test_npu_load_balance_method.py b/test/registered/npu/basic_function/parallel_strategy/data_parallelism/test_npu_load_balance_method.py index 7f2a60492..cf347e0be 100644 --- a/test/registered/npu/basic_function/parallel_strategy/data_parallelism/test_npu_load_balance_method.py +++ b/test/registered/npu/basic_function/parallel_strategy/data_parallelism/test_npu_load_balance_method.py @@ -18,7 +18,7 @@ from sglang.test.test_utils import ( popen_launch_server, ) -register_npu_ci(est_time=700, suite="stage-b-test-16-npu-a3", nightly=False) +register_npu_ci(est_time=700, suite="base-b-test-16-npu-a3") register_npu_ci(est_time=700, suite="nightly-16-npu-a3", nightly=True) diff --git a/test/registered/npu/basic_function/parallel_strategy/data_parallelism/test_npu_load_balance_method_pd_disaggregation.py b/test/registered/npu/basic_function/parallel_strategy/data_parallelism/test_npu_load_balance_method_pd_disaggregation.py index 6a4075cd4..730ae28ec 100644 --- a/test/registered/npu/basic_function/parallel_strategy/data_parallelism/test_npu_load_balance_method_pd_disaggregation.py +++ b/test/registered/npu/basic_function/parallel_strategy/data_parallelism/test_npu_load_balance_method_pd_disaggregation.py @@ -23,7 +23,7 @@ from sglang.test.test_utils import ( popen_launch_pd_server, ) -register_npu_ci(est_time=3600, suite="stage-b-test-16-npu-a3", nightly=False) +register_npu_ci(est_time=3600, suite="base-b-test-16-npu-a3") register_npu_ci(est_time=3600, suite="nightly-16-npu-a3", nightly=True) load_balance_method_options = [ diff --git a/test/registered/npu/basic_function/parallel_strategy/expert_parallelism/test_npu_deepep.py b/test/registered/npu/basic_function/parallel_strategy/expert_parallelism/test_npu_deepep.py index 949701643..e7b158622 100644 --- a/test/registered/npu/basic_function/parallel_strategy/expert_parallelism/test_npu_deepep.py +++ b/test/registered/npu/basic_function/parallel_strategy/expert_parallelism/test_npu_deepep.py @@ -13,7 +13,7 @@ from sglang.test.test_utils import ( popen_launch_server, ) -register_npu_ci(est_time=400, suite="stage-b-test-16-npu-a3", nightly=False) +register_npu_ci(est_time=400, suite="base-b-test-16-npu-a3") register_npu_ci(est_time=400, suite="nightly-16-npu-a3", nightly=True) TEST_MODEL_MATRIX = { diff --git a/test/registered/npu/basic_function/parallel_strategy/expert_parallelism/test_npu_eplb_min_rebalancing_utilization_threshold.py b/test/registered/npu/basic_function/parallel_strategy/expert_parallelism/test_npu_eplb_min_rebalancing_utilization_threshold.py index 057a09b51..1c23e7df1 100644 --- a/test/registered/npu/basic_function/parallel_strategy/expert_parallelism/test_npu_eplb_min_rebalancing_utilization_threshold.py +++ b/test/registered/npu/basic_function/parallel_strategy/expert_parallelism/test_npu_eplb_min_rebalancing_utilization_threshold.py @@ -14,7 +14,7 @@ from sglang.test.test_utils import ( popen_launch_server, ) -register_npu_ci(est_time=400, suite="stage-b-test-8-npu-a3", nightly=False) +register_npu_ci(est_time=400, suite="base-b-test-8-npu-a3") register_npu_ci(est_time=400, suite="nightly-8-npu-a3", nightly=True) diff --git a/test/registered/npu/basic_function/parallel_strategy/expert_parallelism/test_npu_expert_distribution_recorder_mode.py b/test/registered/npu/basic_function/parallel_strategy/expert_parallelism/test_npu_expert_distribution_recorder_mode.py index 27497235b..8d96cc9b4 100644 --- a/test/registered/npu/basic_function/parallel_strategy/expert_parallelism/test_npu_expert_distribution_recorder_mode.py +++ b/test/registered/npu/basic_function/parallel_strategy/expert_parallelism/test_npu_expert_distribution_recorder_mode.py @@ -17,7 +17,7 @@ from sglang.test.test_utils import ( popen_launch_server, ) -register_npu_ci(est_time=400, suite="stage-b-test-2-npu-a3", nightly=False) +register_npu_ci(est_time=400, suite="base-b-test-2-npu-a3") register_npu_ci(est_time=400, suite="nightly-2-npu-a3", nightly=True) diff --git a/test/registered/npu/basic_function/parallel_strategy/expert_parallelism/test_npu_moe_dense_tp_size.py b/test/registered/npu/basic_function/parallel_strategy/expert_parallelism/test_npu_moe_dense_tp_size.py index 1e1364a7c..cdc39ff86 100644 --- a/test/registered/npu/basic_function/parallel_strategy/expert_parallelism/test_npu_moe_dense_tp_size.py +++ b/test/registered/npu/basic_function/parallel_strategy/expert_parallelism/test_npu_moe_dense_tp_size.py @@ -13,7 +13,7 @@ from sglang.test.test_utils import ( popen_launch_server, ) -register_npu_ci(est_time=400, suite="stage-b-test-2-npu-a3", nightly=False) +register_npu_ci(est_time=400, suite="base-b-test-2-npu-a3") register_npu_ci(est_time=400, suite="nightly-2-npu-a3", nightly=True) diff --git a/test/registered/npu/basic_function/parallel_strategy/expert_parallelism/test_npu_moe_runner_backend.py b/test/registered/npu/basic_function/parallel_strategy/expert_parallelism/test_npu_moe_runner_backend.py index 24f6c6ca9..a2db0c6f7 100644 --- a/test/registered/npu/basic_function/parallel_strategy/expert_parallelism/test_npu_moe_runner_backend.py +++ b/test/registered/npu/basic_function/parallel_strategy/expert_parallelism/test_npu_moe_runner_backend.py @@ -12,7 +12,7 @@ from sglang.test.test_utils import ( popen_launch_server, ) -register_npu_ci(est_time=400, suite="stage-b-test-2-npu-a3", nightly=False) +register_npu_ci(est_time=400, suite="base-b-test-2-npu-a3") register_npu_ci(est_time=400, suite="nightly-2-npu-a3", nightly=True) diff --git a/test/registered/npu/basic_function/pd_disaggregation/test_npu_pd_disaggregation.py b/test/registered/npu/basic_function/pd_disaggregation/test_npu_pd_disaggregation.py index 7585c27c8..e3e339375 100644 --- a/test/registered/npu/basic_function/pd_disaggregation/test_npu_pd_disaggregation.py +++ b/test/registered/npu/basic_function/pd_disaggregation/test_npu_pd_disaggregation.py @@ -21,7 +21,7 @@ from sglang.test.test_utils import ( popen_with_error_check, ) -register_npu_ci(est_time=400, suite="stage-b-test-16-npu-a3", nightly=False) +register_npu_ci(est_time=400, suite="base-b-test-16-npu-a3") register_npu_ci(est_time=400, suite="nightly-16-npu-a3", nightly=True) diff --git a/test/registered/npu/basic_function/quant/test_npu_autoround_dense.py b/test/registered/npu/basic_function/quant/test_npu_autoround_dense.py index f684e5990..97a8249ec 100644 --- a/test/registered/npu/basic_function/quant/test_npu_autoround_dense.py +++ b/test/registered/npu/basic_function/quant/test_npu_autoround_dense.py @@ -15,7 +15,7 @@ from sglang.test.test_utils import ( popen_launch_server, ) -register_npu_ci(est_time=400, suite="stage-b-test-1-npu-a3", nightly=False) +register_npu_ci(est_time=400, suite="base-b-test-1-npu-a3") register_npu_ci(est_time=400, suite="nightly-1-npu-a3", nightly=True) logger = logging.getLogger(__name__) diff --git a/test/registered/npu/basic_function/quant/test_npu_autoround_moe.py b/test/registered/npu/basic_function/quant/test_npu_autoround_moe.py index a9641ec69..1c831a938 100644 --- a/test/registered/npu/basic_function/quant/test_npu_autoround_moe.py +++ b/test/registered/npu/basic_function/quant/test_npu_autoround_moe.py @@ -16,7 +16,7 @@ from sglang.test.test_utils import ( popen_launch_server, ) -register_npu_ci(est_time=400, suite="stage-b-test-1-npu-a3", nightly=False) +register_npu_ci(est_time=400, suite="base-b-test-1-npu-a3") register_npu_ci(est_time=400, suite="nightly-1-npu-a3", nightly=True) logger = logging.getLogger(__name__) diff --git a/test/registered/npu/basic_function/quant/test_npu_gptq_moe.py b/test/registered/npu/basic_function/quant/test_npu_gptq_moe.py index b305f2872..bf9d4a296 100644 --- a/test/registered/npu/basic_function/quant/test_npu_gptq_moe.py +++ b/test/registered/npu/basic_function/quant/test_npu_gptq_moe.py @@ -17,7 +17,7 @@ from sglang.test.test_utils import ( popen_launch_server, ) -register_npu_ci(est_time=400, suite="stage-b-test-1-npu-a3", nightly=False) +register_npu_ci(est_time=400, suite="base-b-test-1-npu-a3") register_npu_ci(est_time=400, suite="nightly-1-npu-a3", nightly=True) logger = logging.getLogger(__name__) diff --git a/test/registered/npu/basic_function/quant/test_npu_w4a4_quantization.py b/test/registered/npu/basic_function/quant/test_npu_w4a4_quantization.py index 3db391e6c..f3743bcb0 100644 --- a/test/registered/npu/basic_function/quant/test_npu_w4a4_quantization.py +++ b/test/registered/npu/basic_function/quant/test_npu_w4a4_quantization.py @@ -13,7 +13,7 @@ from sglang.test.ascend.test_ascend_utils import ( from sglang.test.ci.ci_register import register_npu_ci from sglang.test.test_utils import CustomTestCase -register_npu_ci(est_time=400, suite="stage-b-test-4-npu-a3", nightly=False) +register_npu_ci(est_time=400, suite="base-b-test-4-npu-a3") register_npu_ci(est_time=400, suite="nightly-4-npu-a3", nightly=True) diff --git a/test/registered/npu/basic_function/runtime_opts/test_npu_mla_fia_w8a8int8.py b/test/registered/npu/basic_function/runtime_opts/test_npu_mla_fia_w8a8int8.py index 75d4bcc1a..0fb4236be 100644 --- a/test/registered/npu/basic_function/runtime_opts/test_npu_mla_fia_w8a8int8.py +++ b/test/registered/npu/basic_function/runtime_opts/test_npu_mla_fia_w8a8int8.py @@ -14,7 +14,7 @@ from sglang.test.test_utils import ( popen_launch_server, ) -register_npu_ci(est_time=400, suite="stage-b-test-2-npu-a3", nightly=False) +register_npu_ci(est_time=400, suite="base-b-test-2-npu-a3") register_npu_ci(est_time=400, suite="nightly-2-npu-a3", nightly=True) TEST_MODEL_MATRIX = { diff --git a/test/registered/npu/basic_function/runtime_opts/test_npu_mla_w8a8int8.py b/test/registered/npu/basic_function/runtime_opts/test_npu_mla_w8a8int8.py index 94c76b258..50dcd88fa 100644 --- a/test/registered/npu/basic_function/runtime_opts/test_npu_mla_w8a8int8.py +++ b/test/registered/npu/basic_function/runtime_opts/test_npu_mla_w8a8int8.py @@ -13,7 +13,7 @@ from sglang.test.test_utils import ( popen_launch_server, ) -register_npu_ci(est_time=400, suite="stage-b-test-4-npu-a3", nightly=False) +register_npu_ci(est_time=400, suite="base-b-test-4-npu-a3") register_npu_ci(est_time=400, suite="nightly-4-npu-a3", nightly=True) TEST_MODEL_MATRIX = { diff --git a/test/registered/npu/basic_function/runtime_opts/test_npu_tp1_bf16.py b/test/registered/npu/basic_function/runtime_opts/test_npu_tp1_bf16.py index e9740342e..1556c62d7 100644 --- a/test/registered/npu/basic_function/runtime_opts/test_npu_tp1_bf16.py +++ b/test/registered/npu/basic_function/runtime_opts/test_npu_tp1_bf16.py @@ -13,7 +13,7 @@ from sglang.test.test_utils import ( popen_launch_server, ) -register_npu_ci(est_time=400, suite="stage-b-test-1-npu-a3", nightly=False) +register_npu_ci(est_time=400, suite="base-b-test-1-npu-a3") register_npu_ci(est_time=400, suite="nightly-1-npu-a3", nightly=True) TEST_MODEL_MATRIX = { diff --git a/test/registered/npu/basic_function/runtime_opts/test_npu_tp2_bf16.py b/test/registered/npu/basic_function/runtime_opts/test_npu_tp2_bf16.py index 2ec3d16dd..5a88ad3b0 100644 --- a/test/registered/npu/basic_function/runtime_opts/test_npu_tp2_bf16.py +++ b/test/registered/npu/basic_function/runtime_opts/test_npu_tp2_bf16.py @@ -13,7 +13,7 @@ from sglang.test.test_utils import ( popen_launch_server, ) -register_npu_ci(est_time=400, suite="stage-b-test-2-npu-a3", nightly=False) +register_npu_ci(est_time=400, suite="base-b-test-2-npu-a3") register_npu_ci(est_time=400, suite="nightly-2-npu-a3", nightly=True) TEST_MODEL_MATRIX = { diff --git a/test/registered/npu/basic_function/runtime_opts/test_npu_tp2_fia_bf16.py b/test/registered/npu/basic_function/runtime_opts/test_npu_tp2_fia_bf16.py index 821f09abe..bf2ff3361 100644 --- a/test/registered/npu/basic_function/runtime_opts/test_npu_tp2_fia_bf16.py +++ b/test/registered/npu/basic_function/runtime_opts/test_npu_tp2_fia_bf16.py @@ -14,7 +14,7 @@ from sglang.test.test_utils import ( popen_launch_server, ) -register_npu_ci(est_time=400, suite="stage-b-test-2-npu-a3", nightly=False) +register_npu_ci(est_time=400, suite="base-b-test-2-npu-a3") register_npu_ci(est_time=400, suite="nightly-2-npu-a3", nightly=True) TEST_MODEL_MATRIX = { diff --git a/test/registered/npu/basic_function/runtime_opts/test_npu_tp4_bf16.py b/test/registered/npu/basic_function/runtime_opts/test_npu_tp4_bf16.py index b94887783..f9842d10b 100644 --- a/test/registered/npu/basic_function/runtime_opts/test_npu_tp4_bf16.py +++ b/test/registered/npu/basic_function/runtime_opts/test_npu_tp4_bf16.py @@ -12,7 +12,7 @@ from sglang.test.test_utils import ( popen_launch_server, ) -register_npu_ci(est_time=400, suite="stage-b-test-4-npu-a3", nightly=False) +register_npu_ci(est_time=400, suite="base-b-test-4-npu-a3") register_npu_ci(est_time=400, suite="nightly-4-npu-a3", nightly=True) TEST_MODEL_MATRIX = { diff --git a/test/registered/npu/basic_function/speculative_inference/test_npu_basic_sanity_eagle3.py b/test/registered/npu/basic_function/speculative_inference/test_npu_basic_sanity_eagle3.py index 7254361fb..979a01bee 100644 --- a/test/registered/npu/basic_function/speculative_inference/test_npu_basic_sanity_eagle3.py +++ b/test/registered/npu/basic_function/speculative_inference/test_npu_basic_sanity_eagle3.py @@ -17,7 +17,7 @@ from sglang.test.test_utils import ( popen_launch_server, ) -register_npu_ci(est_time=200, suite="stage-b-test-1-npu-a3", nightly=False) +register_npu_ci(est_time=200, suite="base-b-test-1-npu-a3") register_npu_ci(est_time=200, suite="nightly-1-npu-a3", nightly=True) diff --git a/test/registered/npu/basic_function/speculative_inference/test_npu_eagle3.py b/test/registered/npu/basic_function/speculative_inference/test_npu_eagle3.py index 5c56818c3..21c914aeb 100644 --- a/test/registered/npu/basic_function/speculative_inference/test_npu_eagle3.py +++ b/test/registered/npu/basic_function/speculative_inference/test_npu_eagle3.py @@ -9,7 +9,7 @@ from sglang.test.ascend.test_ascend_utils import ( from sglang.test.ci.ci_register import register_npu_ci from sglang.test.test_utils import CustomTestCase -register_npu_ci(est_time=400, suite="stage-b-test-1-npu-a3", nightly=False) +register_npu_ci(est_time=400, suite="base-b-test-1-npu-a3") register_npu_ci(est_time=400, suite="nightly-1-npu-a3", nightly=True) diff --git a/test/registered/npu/basic_function/speculative_inference/test_npu_speculative_attention_mode.py b/test/registered/npu/basic_function/speculative_inference/test_npu_speculative_attention_mode.py index c6cec152a..cfc9dddec 100644 --- a/test/registered/npu/basic_function/speculative_inference/test_npu_speculative_attention_mode.py +++ b/test/registered/npu/basic_function/speculative_inference/test_npu_speculative_attention_mode.py @@ -17,11 +17,7 @@ from sglang.test.test_utils import ( popen_launch_server, ) -register_npu_ci( - est_time=400, - suite="stage-b-test-4-npu-a3", - nightly=False, -) +register_npu_ci(est_time=400, suite="base-b-test-4-npu-a3") register_npu_ci( est_time=400, suite="nightly-4-npu-a3", diff --git a/test/registered/npu/basic_function/speculative_inference/test_npu_speculative_draft_attention_backend.py b/test/registered/npu/basic_function/speculative_inference/test_npu_speculative_draft_attention_backend.py index c61021bb9..485e54b4f 100644 --- a/test/registered/npu/basic_function/speculative_inference/test_npu_speculative_draft_attention_backend.py +++ b/test/registered/npu/basic_function/speculative_inference/test_npu_speculative_draft_attention_backend.py @@ -16,7 +16,7 @@ from sglang.test.test_utils import ( popen_launch_server, ) -register_npu_ci(est_time=400, suite="stage-b-test-16-npu-a3", nightly=False) +register_npu_ci(est_time=400, suite="base-b-test-16-npu-a3") register_npu_ci(est_time=400, suite="nightly-16-npu-a3", nightly=True) MODEL_PATH = DEEPSEEK_R1_0528_W4A8_PER_CHANNEL_WEIGHTS_PATH diff --git a/test/registered/npu/basic_function/speculative_inference/test_npu_speculative_moe_a2a_backend.py b/test/registered/npu/basic_function/speculative_inference/test_npu_speculative_moe_a2a_backend.py index e02518785..56573a783 100644 --- a/test/registered/npu/basic_function/speculative_inference/test_npu_speculative_moe_a2a_backend.py +++ b/test/registered/npu/basic_function/speculative_inference/test_npu_speculative_moe_a2a_backend.py @@ -16,7 +16,7 @@ from sglang.test.test_utils import ( popen_launch_server, ) -register_npu_ci(est_time=400, suite="stage-b-test-16-npu-a3", nightly=False) +register_npu_ci(est_time=400, suite="base-b-test-16-npu-a3") register_npu_ci(est_time=400, suite="nightly-16-npu-a3", nightly=True) diff --git a/test/registered/npu/basic_function/speculative_inference/test_npu_speculative_multi_npu.py b/test/registered/npu/basic_function/speculative_inference/test_npu_speculative_multi_npu.py index e9185a06a..8a9564b2d 100644 --- a/test/registered/npu/basic_function/speculative_inference/test_npu_speculative_multi_npu.py +++ b/test/registered/npu/basic_function/speculative_inference/test_npu_speculative_multi_npu.py @@ -16,7 +16,7 @@ from sglang.test.test_utils import ( popen_launch_server, ) -register_npu_ci(est_time=400, suite="stage-b-test-4-npu-a3", nightly=False) +register_npu_ci(est_time=400, suite="base-b-test-4-npu-a3") register_npu_ci(est_time=400, suite="nightly-4-npu-a3", nightly=True) _ASCEND_BACKEND = "ascend" diff --git a/test/registered/npu/basic_function/speculative_inference/test_npu_speculative_token_map.py b/test/registered/npu/basic_function/speculative_inference/test_npu_speculative_token_map.py index 8d0400b28..3cde909da 100644 --- a/test/registered/npu/basic_function/speculative_inference/test_npu_speculative_token_map.py +++ b/test/registered/npu/basic_function/speculative_inference/test_npu_speculative_token_map.py @@ -20,7 +20,7 @@ from sglang.test.test_utils import ( popen_launch_server, ) -register_npu_ci(est_time=300, suite="stage-b-test-4-npu-a3", nightly=False) +register_npu_ci(est_time=300, suite="base-b-test-4-npu-a3") register_npu_ci(est_time=300, suite="nightly-4-npu-a3", nightly=True) diff --git a/test/registered/npu/performance/deepseek_v4_flash/test_npu_deepseek_v4_flash_w8a8_8p_in8k_out1k_50ms.py b/test/registered/npu/performance/deepseek_v4_flash/test_npu_deepseek_v4_flash_w8a8_8p_in8k_out1k_50ms.py index 62fbef9a7..e8ae7403d 100644 --- a/test/registered/npu/performance/deepseek_v4_flash/test_npu_deepseek_v4_flash_w8a8_8p_in8k_out1k_50ms.py +++ b/test/registered/npu/performance/deepseek_v4_flash/test_npu_deepseek_v4_flash_w8a8_8p_in8k_out1k_50ms.py @@ -8,12 +8,7 @@ from sglang.test.ascend.e2e.test_npu_performance_utils import ( ) from sglang.test.ci.ci_register import register_npu_ci -register_npu_ci( - est_time=3600, - suite="", - nightly=True, - disabled="performance testcase", -) +register_npu_ci(est_time=3600, suite="base-c-test-perf-16-npu-a3") # Environment variables for DSV4-Flash single-node PD-mix deployment. DEEPSEEK_V4_FLASH_W8A8_8P_ENVS = { diff --git a/test/registered/npu/performance/kimi_k2_6/test_npu_kimi_k2_6_w4a8_8p_in3k5_out1k5_20ms.py b/test/registered/npu/performance/kimi_k2_6/test_npu_kimi_k2_6_w4a8_8p_in3k5_out1k5_20ms.py index 43c7cc761..8ad487567 100644 --- a/test/registered/npu/performance/kimi_k2_6/test_npu_kimi_k2_6_w4a8_8p_in3k5_out1k5_20ms.py +++ b/test/registered/npu/performance/kimi_k2_6/test_npu_kimi_k2_6_w4a8_8p_in3k5_out1k5_20ms.py @@ -10,12 +10,7 @@ from sglang.test.ascend.e2e.test_npu_performance_utils import ( ) from sglang.test.ci.ci_register import register_npu_ci -register_npu_ci( - est_time=1800, - suite="full-16-npu-a3", - nightly=True, - disabled="Currently it is executed by the npu performance workflow.", -) +register_npu_ci(est_time=1800, suite="base-c-test-perf-16-npu-a3") KIMI_K2_6_ENVS = { "PYTORCH_NPU_ALLOC_CONF": "expandable_segments:True", diff --git a/test/registered/npu/performance/minimax_m2_5/test_npu_minimax_m2_5_w8a8_4p_in64k_out1k_prefix90_50ms.py b/test/registered/npu/performance/minimax_m2_5/test_npu_minimax_m2_5_w8a8_4p_in64k_out1k_prefix90_50ms.py index a1d35c737..d37cf0f55 100644 --- a/test/registered/npu/performance/minimax_m2_5/test_npu_minimax_m2_5_w8a8_4p_in64k_out1k_prefix90_50ms.py +++ b/test/registered/npu/performance/minimax_m2_5/test_npu_minimax_m2_5_w8a8_4p_in64k_out1k_prefix90_50ms.py @@ -9,12 +9,7 @@ from sglang.test.ascend.e2e.test_npu_performance_utils import ( ) from sglang.test.ci.ci_register import register_npu_ci -register_npu_ci( - est_time=3600, - suite="npu-performance", - nightly=True, - disabled="performance testcase", -) +register_npu_ci(est_time=3600, suite="base-c-test-perf-8-npu-a3") MINIMAX_M2_5_W8A8_4P_IN64K_OUT1K_PREFIX90_ENVS = { "PYTORCH_NPU_ALLOC_CONF": "expandable_segments:True", diff --git a/test/registered/npu/performance/qwen3-8b/test_npu_qwen3_8b_w8a8_1p_in3k5_out1k5_50ms.py b/test/registered/npu/performance/qwen3-8b/test_npu_qwen3_8b_w8a8_1p_in3k5_out1k5_50ms.py index 4509463c4..125c55ac5 100644 --- a/test/registered/npu/performance/qwen3-8b/test_npu_qwen3_8b_w8a8_1p_in3k5_out1k5_50ms.py +++ b/test/registered/npu/performance/qwen3-8b/test_npu_qwen3_8b_w8a8_1p_in3k5_out1k5_50ms.py @@ -9,12 +9,7 @@ from sglang.test.ascend.e2e.test_npu_performance_utils import ( ) from sglang.test.ci.ci_register import register_npu_ci -register_npu_ci( - est_time=3600, - suite="", - nightly=True, - disabled="performance testcase", -) +register_npu_ci(est_time=3600, suite="base-c-test-perf-2-npu-a3") QWEN3_8B_ENVS = { "SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT": "600", diff --git a/test/registered/npu/performance/qwen3_235b_a22b/test_npu_qwen3_235b_w8a8_8p_in3k5_out1k5_50ms.py b/test/registered/npu/performance/qwen3_235b_a22b/test_npu_qwen3_235b_w8a8_8p_in3k5_out1k5_50ms.py index 74335bafc..d115b37ff 100644 --- a/test/registered/npu/performance/qwen3_235b_a22b/test_npu_qwen3_235b_w8a8_8p_in3k5_out1k5_50ms.py +++ b/test/registered/npu/performance/qwen3_235b_a22b/test_npu_qwen3_235b_w8a8_8p_in3k5_out1k5_50ms.py @@ -9,12 +9,7 @@ from sglang.test.ascend.e2e.test_npu_performance_utils import ( ) from sglang.test.ci.ci_register import register_npu_ci -register_npu_ci( - est_time=3600, - suite="", - nightly=True, - disabled="performance testcase", -) +register_npu_ci(est_time=3600, suite="base-c-test-perf-16-npu-a3") QWEN3_235B_ENVS = { "PYTORCH_NPU_ALLOC_CONF": "expandable_segments:True", diff --git a/test/registered/npu/performance/qwen3_30b_a3b/test_npu_qwen3_30b_w8a8_1p_in3k5_out1k5_50ms.py b/test/registered/npu/performance/qwen3_30b_a3b/test_npu_qwen3_30b_w8a8_1p_in3k5_out1k5_50ms.py index 3bf536ac5..50ebda549 100644 --- a/test/registered/npu/performance/qwen3_30b_a3b/test_npu_qwen3_30b_w8a8_1p_in3k5_out1k5_50ms.py +++ b/test/registered/npu/performance/qwen3_30b_a3b/test_npu_qwen3_30b_w8a8_1p_in3k5_out1k5_50ms.py @@ -9,12 +9,7 @@ from sglang.test.ascend.e2e.test_npu_performance_utils import ( ) from sglang.test.ci.ci_register import register_npu_ci -register_npu_ci( - est_time=3600, - suite="", - nightly=True, - disabled="performance testcase", -) +register_npu_ci(est_time=3600, suite="base-c-test-perf-2-npu-a3") QWEN3_30B_A3B_ENVS = { "ASCEND_LAUNCH_BLOCKING": "0", diff --git a/test/registered/npu/performance/qwen3_32b/test_npu_qwen3_32b_w8a8_2p_in3k5_out1k5_50ms.py b/test/registered/npu/performance/qwen3_32b/test_npu_qwen3_32b_w8a8_2p_in3k5_out1k5_50ms.py index 89cf2a304..7752ee5b5 100644 --- a/test/registered/npu/performance/qwen3_32b/test_npu_qwen3_32b_w8a8_2p_in3k5_out1k5_50ms.py +++ b/test/registered/npu/performance/qwen3_32b/test_npu_qwen3_32b_w8a8_2p_in3k5_out1k5_50ms.py @@ -9,12 +9,7 @@ from sglang.test.ascend.e2e.test_npu_performance_utils import ( ) from sglang.test.ci.ci_register import register_npu_ci -register_npu_ci( - est_time=3600, - suite="", - nightly=True, - disabled="performance testcase", -) +register_npu_ci(est_time=3600, suite="base-c-test-perf-4-npu-a3") QWEN3_32B_ENVS = { "SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT": "600", diff --git a/test/registered/npu/performance/qwen3_5_397b/test_npu_qwen3_5_397b_w4a8_8p_in3k5_out1k5_50ms.py b/test/registered/npu/performance/qwen3_5_397b/test_npu_qwen3_5_397b_w4a8_8p_in3k5_out1k5_50ms.py index 68d5d75ef..c77ef4810 100644 --- a/test/registered/npu/performance/qwen3_5_397b/test_npu_qwen3_5_397b_w4a8_8p_in3k5_out1k5_50ms.py +++ b/test/registered/npu/performance/qwen3_5_397b/test_npu_qwen3_5_397b_w4a8_8p_in3k5_out1k5_50ms.py @@ -8,12 +8,7 @@ from sglang.test.ascend.e2e.test_npu_performance_utils import ( ) from sglang.test.ci.ci_register import register_npu_ci -register_npu_ci( - est_time=3600, - suite="nightly-16-npu-a3", - nightly=True, - disabled="performance testcase", -) +register_npu_ci(est_time=3600, suite="base-c-test-perf-16-npu-a3") QWEN3_5_397B_A17B_ENVS = { "PYTORCH_NPU_ALLOC_CONF": "expandable_segments:True", diff --git a/test/registered/npu/performance/qwen3_6_27b/test_npu_qwen3_6_27b_1p_in1024x1024_30_out1024_50ms.py b/test/registered/npu/performance/qwen3_6_27b/test_npu_qwen3_6_27b_1p_in1024x1024_30_out1024_50ms.py index fd501d5ff..0bd8aab8d 100644 --- a/test/registered/npu/performance/qwen3_6_27b/test_npu_qwen3_6_27b_1p_in1024x1024_30_out1024_50ms.py +++ b/test/registered/npu/performance/qwen3_6_27b/test_npu_qwen3_6_27b_1p_in1024x1024_30_out1024_50ms.py @@ -8,12 +8,7 @@ from sglang.test.ascend.e2e.test_npu_performance_utils import ( ) from sglang.test.ci.ci_register import register_npu_ci -register_npu_ci( - est_time=3600, - suite="full-2-npu-a3", - nightly=True, - disabled="performance case", -) +register_npu_ci(est_time=3600, suite="base-c-test-perf-2-npu-a3") QWEN3_6_27B_1024_ENVS = { "PYTORCH_NPU_ALLOC_CONF": "expandable_segments:True", diff --git a/test/registered/npu/performance/qwen3_6_27b/test_npu_qwen3_6_27b_w8a8_1p_in64k_out1k_50ms.py b/test/registered/npu/performance/qwen3_6_27b/test_npu_qwen3_6_27b_w8a8_1p_in64k_out1k_50ms.py index 68f8f341c..14b869e4c 100644 --- a/test/registered/npu/performance/qwen3_6_27b/test_npu_qwen3_6_27b_w8a8_1p_in64k_out1k_50ms.py +++ b/test/registered/npu/performance/qwen3_6_27b/test_npu_qwen3_6_27b_w8a8_1p_in64k_out1k_50ms.py @@ -8,12 +8,7 @@ from sglang.test.ascend.e2e.test_npu_performance_utils import ( ) from sglang.test.ci.ci_register import register_npu_ci -register_npu_ci( - est_time=3600, - suite="", - nightly=True, - disabled="performance testcase", -) +register_npu_ci(est_time=3600, suite="base-c-test-perf-2-npu-a3") QWEN3_6_27B_64K_1K_ENVS = { "STREAMS_PER_DEVICE": "32", diff --git a/test/registered/npu/performance/qwen3_6_35b_a3b/test_npu_qwen3_6_35b_a3b_1p_in64k_out1k_prefix90_50ms.py b/test/registered/npu/performance/qwen3_6_35b_a3b/test_npu_qwen3_6_35b_a3b_1p_in64k_out1k_prefix90_50ms.py index 371a9cf43..cd191e7a4 100644 --- a/test/registered/npu/performance/qwen3_6_35b_a3b/test_npu_qwen3_6_35b_a3b_1p_in64k_out1k_prefix90_50ms.py +++ b/test/registered/npu/performance/qwen3_6_35b_a3b/test_npu_qwen3_6_35b_a3b_1p_in64k_out1k_prefix90_50ms.py @@ -8,12 +8,7 @@ from sglang.test.ascend.e2e.test_npu_performance_utils import ( ) from sglang.test.ci.ci_register import register_npu_ci -register_npu_ci( - est_time=3600, - suite="", - nightly=True, - disabled="performance testcase", -) +register_npu_ci(est_time=3600, suite="base-c-test-perf-2-npu-a3") QWEN3_6_35B_A3B_64K_PREFIX_ENVS = { "PYTORCH_NPU_ALLOC_CONF": "expandable_segments:True", diff --git a/test/registered/npu/performance/qwen3_next_80b_a3b_instruct/test_npu_qwen3_next_80b_w8a8_2p_in6k_out1k5_bs16.py b/test/registered/npu/performance/qwen3_next_80b_a3b_instruct/test_npu_qwen3_next_80b_w8a8_2p_in6k_out1k5_bs16.py index 1952c1ef8..155ed324a 100644 --- a/test/registered/npu/performance/qwen3_next_80b_a3b_instruct/test_npu_qwen3_next_80b_w8a8_2p_in6k_out1k5_bs16.py +++ b/test/registered/npu/performance/qwen3_next_80b_a3b_instruct/test_npu_qwen3_next_80b_w8a8_2p_in6k_out1k5_bs16.py @@ -7,12 +7,7 @@ from sglang.test.ascend.e2e.test_npu_performance_utils import ( ) from sglang.test.ci.ci_register import register_npu_ci -register_npu_ci( - est_time=3600, - suite="", - nightly=True, - disabled="performance testcase", -) +register_npu_ci(est_time=3600, suite="base-c-test-perf-4-npu-a3") QWEN3_NEXT_80B_A3B_ENVS = { "PYTORCH_NPU_ALLOC_CONF": "expandable_segments:True", diff --git a/test/registered/unit/layers/attention/test_vision_backend_selection.py b/test/registered/unit/layers/attention/test_vision_backend_selection.py index b7e6296fa..3ed5a4d64 100644 --- a/test/registered/unit/layers/attention/test_vision_backend_selection.py +++ b/test/registered/unit/layers/attention/test_vision_backend_selection.py @@ -11,7 +11,7 @@ from sglang.srt.layers.attention import vision from sglang.test.ci.ci_register import register_cpu_ci, register_npu_ci register_cpu_ci(est_time=2, suite="base-a-test-cpu") -register_npu_ci(est_time=2, suite="stage-b-test-1-npu-a3") +register_npu_ci(est_time=2, suite="base-b-test-1-npu-a3") register_npu_ci(est_time=2, suite="nightly-1-npu-a3", nightly=True) diff --git a/test/registered/unit/npu/attention/test_npu_ascend_backend.py b/test/registered/unit/npu/attention/test_npu_ascend_backend.py index 867a4a38a..ddc6e9093 100644 --- a/test/registered/unit/npu/attention/test_npu_ascend_backend.py +++ b/test/registered/unit/npu/attention/test_npu_ascend_backend.py @@ -12,7 +12,7 @@ import torch from sglang.test.ci.ci_register import register_npu_ci -register_npu_ci(est_time=5, suite="stage-a-unit-test-npu") +register_npu_ci(est_time=5, suite="base-a-test-1-npu-a2") # Mock NPU-only modules before importing the source module. for _ in ( diff --git a/test/registered/unit/npu/attention/test_npu_ascend_dsv4_backend.py b/test/registered/unit/npu/attention/test_npu_ascend_dsv4_backend.py index 5cf5f1e16..c244c1d24 100644 --- a/test/registered/unit/npu/attention/test_npu_ascend_dsv4_backend.py +++ b/test/registered/unit/npu/attention/test_npu_ascend_dsv4_backend.py @@ -12,7 +12,7 @@ import torch from sglang.test.ci.ci_register import register_npu_ci -register_npu_ci(est_time=4, suite="stage-a-unit-test-npu") +register_npu_ci(est_time=4, suite="base-a-test-1-npu-a2") for mod in ( "torch_npu", diff --git a/test/run_suite.py b/test/run_suite.py index 1af0d5b20..b4ac59623 100644 --- a/test/run_suite.py +++ b/test/run_suite.py @@ -93,13 +93,20 @@ PER_COMMIT_SUITES = { "extra-b-test-8-gpu-h200", ], HWBackend.NPU: [ - "base-a-test-1-gpu-small", - "stage-a-unit-test-npu", - "stage-b-test-1-npu-a3", - "stage-b-test-2-npu-a3", - "stage-b-test-4-npu-a3", - "stage-b-test-8-npu-a3", - "stage-b-test-16-npu-a3", + "base-a-test-1-npu-a2", + "base-b-test-1-npu-a3", + "base-b-test-2-npu-a3", + "base-b-test-4-npu-a3", + "base-b-test-8-npu-a3", + "base-b-test-16-npu-a3", + "base-c-test-acc-2-npu-a3", + "base-c-test-acc-4-npu-a3", + "base-c-test-acc-8-npu-a3", + "base-c-test-acc-16-npu-a3", + "base-c-test-perf-2-npu-a3", + "base-c-test-perf-4-npu-a3", + "base-c-test-perf-8-npu-a3", + "base-c-test-perf-16-npu-a3", ], HWBackend.XPU: [ "stage-a-test-1-gpu-xpu",