diff --git a/.github/workflows/_npu-pr-test-stage.yml b/.github/workflows/_npu-pr-test-stage.yml index db1171fa9..79bb16954 100644 --- a/.github/workflows/_npu-pr-test-stage.yml +++ b/.github/workflows/_npu-pr-test-stage.yml @@ -40,6 +40,10 @@ on: description: 'Git ref (branch, tag, or SHA) to test. If not provided, uses the default branch.' type: string default: 'false' + is_nightly_pipeline_job: + description: 'Run the test suite with --nightly (collects nightly-registered tests) and --continue-on-error' + type: boolean + default: false github-token: description: 'GitHub token for API calls' type: string @@ -177,6 +181,8 @@ jobs: } - name: Install dependencies + # Only PR jobs install dependencies + if: ${{ inputs.is_nightly_pipeline_job != true }} env: TORCH_CACHE_URL: "http://cache-service.nginx-pypi-cache.svc.cluster.local/whl/cpu" PYPI_CACHE_URL: "http://cache-service.nginx-pypi-cache.svc.cluster.local/pypi/simple" @@ -204,10 +210,43 @@ jobs: timeout-minutes: ${{ fromJson(inputs.run_timeout_minutes) }} env: CONTINUE_ON_ERROR_FLAG: ${{ inputs.continue_on_error == 'true' && '--continue-on-error' || '' }} + shell: bash run: | + # Fail fast on any command error, undefined variable, or pipe failure. + set -euo pipefail + + # Install missing python deps (skip if already importable). + PYTHON_FOR_SGLANG="python" + PIP_INDEXES=("https://pypi.org/simple/" "https://mirrors.tuna.tsinghua.edu.cn/pypi/web/simple" "http://cache-service.nginx-pypi-cache.svc.cluster.local/pypi/simple") + PIP_TRUSTED_HOST=cache-service.nginx-pypi-cache.svc.cluster.local + install_pkg() { + local pkg="$1" + ok=0 + for index in "${PIP_INDEXES[@]}"; do + echo "${pkg}: try via ${index}" + if ${PYTHON_FOR_SGLANG} -m pip install "${pkg}" --retries 3 --timeout 60 \ + -i "${index}" --trusted-host "${PIP_TRUSTED_HOST}"; then + ok=1; break + fi + done + [ "${ok}" = "1" ] || { echo "ERROR: failed to install ${pkg}"; exit 1; } + } + REQUIRED_PIP_PKGS=(tabulate) + for pkg in "${REQUIRED_PIP_PKGS[@]}"; do + ${PYTHON_FOR_SGLANG} -c "import ${pkg}" 2>/dev/null && { echo "${pkg}: available, skip"; continue; } + install_pkg "${pkg}" + done + cd test + # Append `--nightly` and `--continue-on-error` when the job belongs to the nightly pipeline. + # --nightly: Collect test cases registered for nightly runs. + # --continue-on-error: Keep running remaining cases even if one fails. + NIGHTLY_FLAG="" + if [ "${{ inputs.is_nightly_pipeline_job }}" = "true" ]; then + NIGHTLY_FLAG="--nightly --continue-on-error" + fi python3 run_suite.py --hw npu --suite ${{ inputs.self_name }} \ --auto-partition-id ${{ matrix.partition }} \ --auto-partition-size ${{ fromJson(inputs.partitions).size }} \ ${{ inputs.timeout_per_file && format('--timeout-per-file {0}', inputs.timeout_per_file) || '' }} \ - $CONTINUE_ON_ERROR_FLAG + $NIGHTLY_FLAG $CONTINUE_ON_ERROR_FLAG diff --git a/.github/workflows/_npu-single-node-test-stage.yml b/.github/workflows/_npu-single-node-test-stage.yml index be5bda0e4..1674e444d 100644 --- a/.github/workflows/_npu-single-node-test-stage.yml +++ b/.github/workflows/_npu-single-node-test-stage.yml @@ -37,10 +37,23 @@ on: type: string default: "" description: "The transformers version number for running sglang. Use default version in image if keep empty." + run_start_metadata: + required: false + type: string + default: '{}' + description: 'JSON run metadata {branch_label, workflow_name, create_time}, recorded once at workflow start' skip_pr_test_health_check: description: 'Git ref (branch, tag, or SHA) to test. If not provided, uses the default branch.' type: string default: 'false' + is_nightly_pipeline_job: + description: 'Run the test suite with --nightly (collects nightly-registered tests) and --continue-on-error' + type: boolean + default: false + partitions: + description: 'Partition config to parallelize a suite across jobs, e.g. {"size":3,"arr":[0,1,2]}. Each element of arr is a partition id; the template expands one matrix job per element. size is passed to run_suite.py --auto-partition-size.' + type: string + default: '{"size":1,"arr":[0]}' github-token: description: 'GitHub token for API calls' type: string @@ -56,6 +69,10 @@ concurrency: jobs: e2e: name: ${{ inputs.test_suite }} + strategy: + fail-fast: false + matrix: + partition: ${{ fromJson(inputs.partitions).arr }} runs-on: ${{ inputs.runner }} container: image: ${{ inputs.image }} @@ -195,9 +212,11 @@ jobs: SGLANG_IS_IN_CI: true TRANSFORMERS_VERBOSITY: "error" GDN_ATTN_BACKEND_TRITON: 1 - SGLANG_TEST_METRICS_OUTPUT: /root/.cache/tests/output/metrics/metrics shell: bash run: | + # Fail fast on any command error, undefined variable, or pipe failure. + set -euo pipefail + sglang_source_path=$(pwd) echo "Source code path: ${sglang_source_path}" ln -sf ${sglang_source_path} /root/sglang @@ -207,27 +226,90 @@ jobs: echo "Test mode: suite (${test_suite})" tc_name="${test_suite}" + # The ID is used to locate the current job instance, + # size determines whether parallel execution is enabled. + PARTITION_ID=${{ matrix.partition }} + PARTITION_SIZE=${{ fromJson(inputs.partitions).size }} + # Append directory suffixes when there are parallel partition jobs + # within the test suite to prevent collision of result artifact directories. + if [ "${PARTITION_SIZE}" -gt 1 ]; then + tc_name="${test_suite}-p${PARTITION_ID}" + fi + + # Provided automatically by the GitHub server. + # run_id is the unique identifier for the workflow run, + # attempt represents the re‑run attempt count of the job. + run_id=${{ github.run_id }} + run_attempt=${{ github.run_attempt }} + + # Parse run metadata (branch_label / workflow_name / create_time) + # from run_start_metadata JSON input using GitHub fromJson expression, same as partitions input. + branch_label=$(echo "${{ fromJson(inputs.run_start_metadata).branch_label }}" | tr '/:' '--') + workflow_name=$(echo "${{ fromJson(inputs.run_start_metadata).workflow_name }}" | tr ' ' '_' | tr -d '()') + create_time="${{ fromJson(inputs.run_start_metadata).create_time }}" + if [ -z "${create_time}" ]; then create_time=$(date -u -d '+8 hours' +%Y%m%d-%H%M); fi + + # Print persistence‑directory‑prefix variables + echo "Persistence prefix: run_id=${run_id} run_attempt=${run_attempt} branch_label=${branch_label} workflow_name=${workflow_name} create_time=${create_time}" + + if [ "${{ inputs.is_nightly_pipeline_job }}" = "true" ]; then + test_data_output_path=/root/.cache/tests/output/${branch_label}-${create_time}-${run_id}-${run_attempt}/${workflow_name}/${{ inputs.test_type }}/${tc_name} + mkdir -p ${test_data_output_path} + + # Set and print the output path for test‑case results and metrics persistence. + export METRICS_DATA_FILE=${test_data_output_path} + export SGLANG_TEST_METRICS_OUTPUT=${test_data_output_path}/metrics + echo "METRICS_DATA_FILE=${METRICS_DATA_FILE}" + echo "SGLANG_TEST_METRICS_OUTPUT=${SGLANG_TEST_METRICS_OUTPUT}" + fi + export TRANSFORMERS_VERSION_FOR_SGLANG="${{ inputs.transformers_version }}" PYTHON_FOR_SGLANG="python" PIP_FOR_SGLANG="pip" + # Shared pip index list (public PyPI, tsinghua mirror, then the in-cluster cache). + PIP_INDEXES=("https://pypi.org/simple/" "https://mirrors.tuna.tsinghua.edu.cn/pypi/web/simple" "http://cache-service.nginx-pypi-cache.svc.cluster.local/pypi/simple") + PIP_TRUSTED_HOST=cache-service.nginx-pypi-cache.svc.cluster.local + + # Shared pip installer: try each index until one succeeds, fail the job if all fail. + install_pkg() { + local pkg="$1" + ok=0 + for index in "${PIP_INDEXES[@]}"; do + echo "${pkg}: try via ${index}" + if ${PYTHON_FOR_SGLANG} -m pip install "${pkg}" --retries 3 --timeout 60 \ + -i "${index}" --trusted-host "${PIP_TRUSTED_HOST}"; then + ok=1; break + fi + done + [ "${ok}" = "1" ] || { echo "ERROR: failed to install ${pkg}"; exit 1; } + } + + # Transformers (pinned version): prefer the offline wheel cache, fall back to online install. if [ -n "${TRANSFORMERS_VERSION_FOR_SGLANG}" ];then echo "===== Install transformers for sglang - Begin =====" TRANSFORMERS_PKG_PATH_SOURCE=/root/.cache/.cache/transformers/${TRANSFORMERS_VERSION_FOR_SGLANG} if [ ! -d "${TRANSFORMERS_PKG_PATH_SOURCE}" ]; then echo "The dependent transformers package does not exist: ${TRANSFORMERS_PKG_PATH_SOURCE}." echo "Install transformers ${TRANSFORMERS_VERSION_FOR_SGLANG} online." - pip install transformers=="${TRANSFORMERS_VERSION_FOR_SGLANG}" -i https://mirrors.tuna.tsinghua.edu.cn/pypi/web/simple + install_pkg "transformers==${TRANSFORMERS_VERSION_FOR_SGLANG}" else echo "Install transformers ${TRANSFORMERS_VERSION_FOR_SGLANG} locally." TRANSFORMERS_PKG_PATH_TARGET=/tmp/transformers/${TRANSFORMERS_VERSION_FOR_SGLANG} mkdir -p "${TRANSFORMERS_PKG_PATH_TARGET}" - cp "${TRANSFORMERS_PKG_PATH_SOURCE}/*" "${TRANSFORMERS_PKG_PATH_TARGET}/" - pip install --no-index --find-links="${TRANSFORMERS_PKG_PATH_TARGET}" transformers=="${TRANSFORMERS_VERSION_FOR_SGLANG}" + cp "${TRANSFORMERS_PKG_PATH_SOURCE}/"* "${TRANSFORMERS_PKG_PATH_TARGET}/" + ${PYTHON_FOR_SGLANG} -m pip install --no-index --find-links="${TRANSFORMERS_PKG_PATH_TARGET}" transformers=="${TRANSFORMERS_VERSION_FOR_SGLANG}" fi echo "===== Install transformers for sglang in virtual env - End =====" fi echo "Transformers version for sglang: $(${PIP_FOR_SGLANG} show transformers | grep Version | cut -d: -f2)" + # Install missing python deps (skip if already importable) + REQUIRED_PIP_PKGS=(tabulate) + for pkg in "${REQUIRED_PIP_PKGS[@]}"; do + ${PYTHON_FOR_SGLANG} -c "import ${pkg}" 2>/dev/null && { echo "${pkg}: available, skip"; continue; } + install_pkg "${pkg}" + done + echo "scaling_governor performance num: \ $(cat /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor | grep performance | wc -l)" echo "swappiness: $(cat /proc/sys/vm/swappiness)" @@ -246,21 +328,64 @@ jobs: mv ${ascend_test_util_path} ${ascend_test_util_path}_bak cp -r ${sglang_source_path}/python/sglang/test/ascend ${ascend_test_util_path} + # External env scripts may reference unset shell vars (e.g. ZSH_VERSION), + # which abort the whole script under `set -u`. Temporarily disable -u. + set +u source /usr/local/Ascend/cann/set_env.sh || true source /usr/local/Ascend/nnal/atb/set_env.sh || true source /usr/local/Ascend/ascend-toolkit/latest/opp/vendors/customize/bin/set_env.bash || true source /usr/local/Ascend/ascend-toolkit/latest/opp/vendors/custom_transformer/bin/set_env.bash || true + set -u - # Set environment of cann - log_path="/root/.cache/tests/logs/log/${current_date}/${tc_name}/${HOSTNAME}" - rm -rf ${log_path} - mkdir -p ${log_path} + # Nightly log path mirrors the output layout; PR keeps the original {date}/{testcase}. + if [ "${{ inputs.is_nightly_pipeline_job }}" = "true" ]; then + log_path="/root/.cache/tests/logs/log/${branch_label}-${create_time}-${run_id}-${run_attempt}/${workflow_name}/${{ inputs.test_type }}/${tc_name}/${HOSTNAME}" + else + current_date=$(date -u -d '+8 hours' +%Y%m%d) + log_path="/root/.cache/tests/logs/log/${current_date}/${tc_name}/${HOSTNAME}" + fi + rm -rf "${log_path}" + mkdir -p "${log_path}" echo "Log path: ${log_path}" echo "Running test: ${tc_name}" test_exit_code=0 cd test - ${PYTHON_FOR_SGLANG} -u run_suite.py --hw npu --suite ${test_suite} --timeout-per-file 7200 2>&1 | tee /tmp/test_output.log || test_exit_code=$? + + # Append `--nightly` and `--continue-on-error` arguments when the job belongs to the nightly pipeline. + # --nightly: Collect test cases registered for nightly runs. + # --continue-on-error: Keep running remaining cases even if one fails. + NIGHTLY_FLAG="" + if [ "${{ inputs.is_nightly_pipeline_job }}" = "true" ]; then + NIGHTLY_FLAG="--nightly --continue-on-error" + fi + + # Append configuration arguments when the test suite is split into multiple parallel jobs. + PARTITION_ARGS="" + if [ "${PARTITION_SIZE}" -gt 1 ] 2>/dev/null; then + PARTITION_ARGS="--auto-partition-id ${PARTITION_ID} --auto-partition-size ${PARTITION_SIZE}" + fi + + # Append `--timeout-from-est‑time` when the job belongs to the nightly pipeline. + # per‑case timeout is derived from each case's est_time. + TIMEOUT_FLAG="" + if [ "${{ inputs.is_nightly_pipeline_job }}" = "true" ]; then + TIMEOUT_FLAG="--timeout-from-est-time" + fi + + # Nightly additionally persists the full suite log to ${log_path}/${tc_name}.log + # under the structured path; PR runs only tee to /tmp/test_output.log. + # Use an array so the extra target is omitted when empty: passing an empty + # string arg makes GNU tee fail (exit code 1) and flips the pipeline exit code. + LOG_TEE_TARGETS=("/tmp/test_output.log") + if [ "${{ inputs.is_nightly_pipeline_job }}" = "true" ]; then + LOG_TEE_TARGETS+=("${log_path}/${tc_name}.log") + fi + + # Run NPU test suite with run_suite.py. + # Capture mode (--enable-retry --max-attempts 1) keeps test subprocesses off the + # tee pipe, so leftover server processes cannot block the pipeline on exit. + ${PYTHON_FOR_SGLANG} -u run_suite.py --hw npu --suite ${test_suite} ${NIGHTLY_FLAG} ${PARTITION_ARGS} ${TIMEOUT_FLAG} --enable-retry --max-attempts 1 --timeout-per-file 7200 2>&1 | tee "${LOG_TEE_TARGETS[@]}" || test_exit_code=$? echo "Finished test: ${tc_name}" @@ -272,10 +397,8 @@ jobs: status_icon="❌" fi - echo "test_status=${test_status}" >> $GITHUB_ENV - echo "tc_name=${tc_name}" >> $GITHUB_ENV + echo "tc_name=${tc_name}" export test_status tc_name - echo "## ${tc_name} ${status_icon} ${test_status}" >> $GITHUB_STEP_SUMMARY echo "" >> $GITHUB_STEP_SUMMARY metric_count=$(grep -c '\[METRIC\]' /tmp/test_output.log 2>/dev/null || echo 0) diff --git a/.github/workflows/nightly-test-npu-e2e-multi-node.yml b/.github/workflows/nightly-test-npu-e2e-multi-node.yml index 89b02ed6b..d591b1c27 100644 --- a/.github/workflows/nightly-test-npu-e2e-multi-node.yml +++ b/.github/workflows/nightly-test-npu-e2e-multi-node.yml @@ -59,6 +59,11 @@ on: type: string default: "" description: "The transformers version number for running sglang. Use default version in image if keep empty." + run_start_metadata: + required: false + type: string + default: '{}' + description: 'JSON run metadata {branch_label, workflow_name, create_time}, recorded once at workflow start' concurrency: group: ascend-nightly-multi-node-${{ github.workflow_ref }}-${{ github.ref }}-${{ inputs.test_config_name }} @@ -135,14 +140,28 @@ jobs: fi # prepare for output data - current_date=$(date +%Y%m%d) - test_data_output_path=/root/.cache/tests/output/${{ inputs.test_type }}/${current_date} - mkdir -p ${test_data_output_path} tc_name=${test_case##*/} tc_name=${tc_name%.*} - metrics_data_path=${test_data_output_path}/${tc_name} - mkdir -p ${metrics_data_path} - echo "Metrics file path: ${metrics_data_path}" + + # Provided automatically by the GitHub server. + # run_id is the unique identifier for the workflow run, + # attempt represents the re‑run attempt count of the job. + run_id=${{ github.run_id }} + run_attempt=${{ github.run_attempt }} + + # Parse run metadata (branch_label / workflow_name / create_time) + # from run_start_metadata JSON input using GitHub fromJson expression, same as partitions input. + branch_label=$(echo "${{ fromJson(inputs.run_start_metadata).branch_label }}" | tr '/:' '--') + workflow_name=$(echo "${{ fromJson(inputs.run_start_metadata).workflow_name }}" | tr ' ' '_' | tr -d '()') + create_time="${{ fromJson(inputs.run_start_metadata).create_time }}" + if [ -z "${create_time}" ]; then create_time=$(date -u -d '+8 hours' +%Y%m%d-%H%M); fi + + # Print persistence‑directory‑prefix variables + echo "Persistence prefix: run_id=${run_id} run_attempt=${run_attempt} branch_label=${branch_label} workflow_name=${workflow_name} create_time=${create_time}" + + sglang_test_results_output_path=/root/.cache/tests/output/${branch_label}-${create_time}-${run_id}-${run_attempt}/${workflow_name}/${{ inputs.test_type }}/${tc_name} + mkdir -p ${sglang_test_results_output_path} + echo "Metrics file path: ${sglang_test_results_output_path}" prefill_decode_deployment="${{ inputs.prefill_decode_deployment }}" kube_job_type="" @@ -161,7 +180,7 @@ jobs: --env ci \ --image ${{ inputs.image }} \ --sglang-source-relative-path ${sglang_source_relative_path} \ - --metrics-data-file ${metrics_data_path} \ + --metrics-data-file ${sglang_test_results_output_path} \ --test-case ${test_case} \ --kube-name-space ${NAMESPACE} \ --kube-job-type ${kube_job_type} \ @@ -189,7 +208,7 @@ jobs: echo "Install sglang from source." CMD="${CMD} --install-sglang-from-source" commit_id=${{ github.sha }} - echo "commit id: ${commit_id}" > ${test_data_output_path}/commit_id + echo "commit id: ${commit_id}" > "$(dirname "${sglang_test_results_output_path}")/commit_id" else echo "Use sglang from image: ${{ inputs.image }}" fi diff --git a/.github/workflows/nightly-test-npu-e2e-single-node.yml b/.github/workflows/nightly-test-npu-e2e-single-node.yml deleted file mode 100644 index 940127cdc..000000000 --- a/.github/workflows/nightly-test-npu-e2e-single-node.yml +++ /dev/null @@ -1,257 +0,0 @@ -name: 'Nightly Test (NPU) Single Node Template' - -on: - workflow_call: - inputs: - runner: - required: true - type: string - default: linux-aarch64-a3-16 - test_type: - required: false - type: string - default: perf - description: perf or accuracy - test_config_name: - required: true - type: string - description: test config name - test_case: - required: true - type: string - description: path of test case file - image: - required: false - type: string - description: image for pods - default: "swr.cn-southwest-2.myhuaweicloud.com/base_image/dockerhub/lmsysorg/sglang:main-cann9.0.0-a3" - install_sglang_from_source: - required: false - type: boolean - default: false - description: use sglang from source code or from docker image - install_sglang_deps: - required: false - type: boolean - default: false - description: install sglang dependencies (e.g. PyTorch, CANN packages) when using source installation - device_type_for_deps: - required: false - type: string - default: 'a3' - description: device type for dependency installation (a3 or 910b) - transformers_version: - required: false - type: string - default: "" - description: "The transformers version number for running sglang. Use default version in image if keep empty." - -concurrency: - group: ascend-nightly-e2e-singlenode-${{ github.workflow_ref }}-${{ github.ref }}-${{ inputs.test_config_name }} - cancel-in-progress: true - -jobs: - e2e: - name: ${{ inputs.test_config_name }} - runs-on: ${{ inputs.runner }} - container: - image: ${{ inputs.image }} - steps: - - name: Checkout code - uses: actions/checkout@v4 - - - name: Check npu info - run: | - npu-smi info - - - name: Install sglang dependencies - if: ${{ inputs.install_sglang_deps == true }} - shell: bash - env: - TORCH_CACHE_URL: "http://cache-service.nginx-pypi-cache.svc.cluster.local/whl/cpu" - PYPI_CACHE_URL: "http://cache-service.nginx-pypi-cache.svc.cluster.local/pypi/simple" - UV_INDEX_URL: "http://cache-service.nginx-pypi-cache.svc.cluster.local/pypi/simple" - GITHUB_PROXY_URL: "https://gh-proxy.test.osinfra.cn/" - RUSTUP_CACHE_URL: "http://cache-service.nginx-pypi-cache.svc.cluster.local:8082" - run: | - CACHING_URL="cache-service.nginx-pypi-cache.svc.cluster.local" - sed -Ei "s@(ports|archive).ubuntu.com@${CACHING_URL}:8081@g" /etc/apt/sources.list - pip config set global.index-url http://${CACHING_URL}/pypi/simple - pip config set global.trusted-host "${CACHING_URL}" - bash scripts/ci/npu/npu_ci_install_dependency.sh ${{ inputs.device_type_for_deps }} - cp ~/.cache/modelscope/hub/datasets/otavia/ShareGPT_Vicuna_unfiltered/ShareGPT_V3_unfiltered_cleaned_split.json /tmp - - - name: Run test - timeout-minutes: 300 - env: - SGLANG_USE_MODELSCOPE: true - HF_ENDPOINT: https://hf-mirror.com - SGLANG_IS_IN_CI: true - TRANSFORMERS_VERBOSITY: "error" - GDN_ATTN_BACKEND_TRITON: 1 - SGLANG_TEST_METRICS_OUTPUT: /root/.cache/tests/output/metrics/metrics - shell: bash - run: | - sglang_source_path=$(pwd) - echo "Source code path: ${sglang_source_path}" - ln -sf ${sglang_source_path} /root/sglang - - test_case=${{ inputs.test_case }} - if [ ! -f "${sglang_source_path}/${test_case}" ]; then - echo "The testcase does not exit: ${sglang_source_path}/${test_case}" - exit 1 - fi - - # prepare for output data - current_date=$(date +%Y%m%d) - test_data_output_path=/root/.cache/tests/output/${{ inputs.test_type }}/${current_date} - mkdir -p ${test_data_output_path} - tc_name=${test_case##*/} - tc_name=${tc_name%.*} - export METRICS_DATA_FILE=${test_data_output_path}/${tc_name} - mkdir -p ${METRICS_DATA_FILE} - echo "Metrics file path: ${METRICS_DATA_FILE}" - - # copy required file from our daily cache - cp ~/.cache/modelscope/hub/datasets/otavia/ShareGPT_Vicuna_unfiltered/ShareGPT_V3_unfiltered_cleaned_split.json /tmp - curl -o /tmp/test.jsonl -L https://gh-proxy.test.osinfra.cn/https://raw.githubusercontent.com/openai/grade-school-math/master/grade_school_math/data/test.jsonl - - export TRANSFORMERS_VERSION_FOR_SGLANG="${{ inputs.transformers_version }}" - PYTHON_FOR_SGLANG="python" - PIP_FOR_SGLANG="pip" - if [ -n "${TRANSFORMERS_VERSION_FOR_SGLANG}" ];then - echo "===== Install transformers for sglang - Begin =====" - TRANSFORMERS_PKG_PATH_SOURCE=/root/.cache/.cache/transformers/${TRANSFORMERS_VERSION_FOR_SGLANG} - if [ ! -d "${TRANSFORMERS_PKG_PATH_SOURCE}" ]; then - echo "The dependent transformers package does not exist: ${TRANSFORMERS_PKG_PATH_SOURCE}." - echo "Install transformers ${TRANSFORMERS_VERSION_FOR_SGLANG} online." - pip install transformers=="${TRANSFORMERS_VERSION_FOR_SGLANG}" -i https://mirrors.tuna.tsinghua.edu.cn/pypi/web/simple - else - echo "Install transformers ${TRANSFORMERS_VERSION_FOR_SGLANG} locally." - TRANSFORMERS_PKG_PATH_TARGET=/tmp/transformers/${TRANSFORMERS_VERSION_FOR_SGLANG} - mkdir -p "${TRANSFORMERS_PKG_PATH_TARGET}" - cp "${TRANSFORMERS_PKG_PATH_SOURCE}/*" "${TRANSFORMERS_PKG_PATH_TARGET}/" - pip install --no-index --find-links="${TRANSFORMERS_PKG_PATH_TARGET}" transformers=="${TRANSFORMERS_VERSION_FOR_SGLANG}" - fi - echo "===== Install transformers for sglang in virtual env - End =====" - fi - echo "Transformers version for sglang: $(${PIP_FOR_SGLANG} show transformers | grep Version | cut -d: -f2)" - - echo "scaling_governor performance num: \ - $(cat /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor | grep performance | wc -l)" - echo "swappiness: $(cat /proc/sys/vm/swappiness)" - echo "numa_balancing: $(cat /proc/sys/kernel/numa_balancing)" - echo "sched_migration_cost_ns: $(cat /proc/sys/kernel/sched_migration_cost_ns)" - - export SGLANG_TEST_MAX_RETRY=0 - export SGLANG_SET_CPU_AFFINITY=1 - echo "SGLANG_SET_CPU_AFFINITY: $SGLANG_SET_CPU_AFFINITY" - - install_sglang_from_source=${{ inputs.install_sglang_from_source }} - if [ "$install_sglang_from_source" = "true" ] || [ "$install_sglang_from_source" = "True" ];then - echo "Install sglang from source" - commit_id=${{ github.sha }} - echo "commit id: ${commit_id}" > ${test_data_output_path}/commit_id - export PYTHONPATH=${sglang_source_path}/python:$PYTHONPATH - else - echo "Use sglang from image: ${{ inputs.image }}" - sglang_pkg_path=/sgl-workspace/sglang/python - ascend_test_util_path=${sglang_pkg_path}/sglang/test/ascend - mkdir -p ${ascend_test_util_path} - mv ${ascend_test_util_path} ${ascend_test_util_path}_bak - cp -r ${sglang_source_path}/python/sglang/test/ascend ${ascend_test_util_path} - fi - - source /usr/local/Ascend/cann/set_env.sh || true - source /usr/local/Ascend/nnal/atb/set_env.sh || true - source /usr/local/Ascend/ascend-toolkit/latest/opp/vendors/customize/bin/set_env.bash || true - source /usr/local/Ascend/ascend-toolkit/latest/opp/vendors/custom_transformer/bin/set_env.bash || true - - # Set environment of cann - log_path="/root/.cache/tests/logs/log/${current_date}/${tc_name}/${HOSTNAME}" - rm -rf ${log_path} - mkdir -p ${log_path} - echo "Log path: ${log_path}" - - echo "Running test case ${test_case}" - test_exit_code=0 - ${PYTHON_FOR_SGLANG} -u ${test_case} 2>&1 | tee /tmp/test_output.log || test_exit_code=$? - echo "Finished test case ${test_case}" - - if [ "${test_exit_code}" = "0" ]; then - test_status="pass" - status_icon="✅" - else - test_status="fail" - status_icon="❌" - fi - - echo "test_status=${test_status}" >> $GITHUB_ENV - echo "tc_name=${tc_name}" >> $GITHUB_ENV - export test_status tc_name - - echo "## ${tc_name} ${status_icon} ${test_status}" >> $GITHUB_STEP_SUMMARY - echo "" >> $GITHUB_STEP_SUMMARY - metric_count=$(grep -c '\[METRIC\]' /tmp/test_output.log 2>/dev/null || echo 0) - if [ "${metric_count}" -gt 0 ]; then - echo "| Metric | Value | Pass |" >> $GITHUB_STEP_SUMMARY - echo "|--------|-------|------|" >> $GITHUB_STEP_SUMMARY - grep '\[METRIC\]' /tmp/test_output.log | while IFS= read -r line; do - metric_name=$(echo "$line" | sed -E 's/.*\[METRIC\] ([^=]+)=.*/\1/') - metric_value=$(echo "$line" | sed -E 's/.*\[METRIC\] [^=]+=([^ ]+).*/\1/') - echo "| ${metric_name} | ${metric_value} | ${status_icon} |" >> $GITHUB_STEP_SUMMARY - done - else - echo "No metrics collected (test may have failed before producing results)." >> $GITHUB_STEP_SUMMARY - fi - echo "" >> $GITHUB_STEP_SUMMARY - - echo "import json, re, os" > /tmp/dump_metrics.py - echo "tc = os.environ.get('tc_name', 'unknown')" >> /tmp/dump_metrics.py - echo "st = os.environ.get('test_status', 'unknown')" >> /tmp/dump_metrics.py - echo "tp = '${{ inputs.test_type }}'" >> /tmp/dump_metrics.py - echo "metrics = {}" >> /tmp/dump_metrics.py - echo "with open('/tmp/test_output.log') as f:" >> /tmp/dump_metrics.py - echo " for line in f:" >> /tmp/dump_metrics.py - echo " m = re.match(r'\[METRIC\] (\S+)=(\S+)', line)" >> /tmp/dump_metrics.py - echo " if m:" >> /tmp/dump_metrics.py - echo " v = m.group(2)" >> /tmp/dump_metrics.py - echo " try:" >> /tmp/dump_metrics.py - echo " v = float(v)" >> /tmp/dump_metrics.py - echo " except ValueError:" >> /tmp/dump_metrics.py - echo " pass" >> /tmp/dump_metrics.py - echo " metrics[m.group(1)] = v" >> /tmp/dump_metrics.py - echo "baselines = {}" >> /tmp/dump_metrics.py - echo "for k, v in list(metrics.items()):" >> /tmp/dump_metrics.py - echo " if k.endswith('_baseline'):" >> /tmp/dump_metrics.py - echo " baselines[k[:-9]] = v" >> /tmp/dump_metrics.py - echo " del metrics[k]" >> /tmp/dump_metrics.py - echo "with open('/tmp/metrics.json', 'w') as f:" >> /tmp/dump_metrics.py - echo " json.dump({'test_case': tc, 'test_type': tp, 'status': st, 'metrics': metrics, 'baselines': baselines}, f)" >> /tmp/dump_metrics.py - python3 /tmp/dump_metrics.py - exit ${test_exit_code} - - - name: Upload metrics - if: always() - uses: actions/upload-artifact@v4 - with: - name: metrics-${{ inputs.test_config_name }} - path: /tmp/metrics.json - retention-days: 7 - - - name: Backup plog - if: always() - shell: bash - run: | - plog_path="/root/ascend/log/debug/plog" - if [ -d "$plog_path" ];then - echo "Plog files found. Begin to backup them." - tc_name=${{ inputs.test_case }} - tc_name=${tc_name##*/} - tc_name=${tc_name%.*} - target_plog_path="/root/.cache/tests/logs/plog/${tc_name}/${HOSTNAME}" - echo "Save path: ${target_plog_path}" - rm -rf ${target_plog_path} - mkdir -p ${target_plog_path} - cp ${plog_path}/* ${target_plog_path} - fi diff --git a/.github/workflows/nightly-test-npu.yml b/.github/workflows/nightly-test-npu.yml index 7b6216230..c6abe178d 100644 --- a/.github/workflows/nightly-test-npu.yml +++ b/.github/workflows/nightly-test-npu.yml @@ -1,4 +1,4 @@ -name: Nightly Test (NPU) +name: Nightly Test (NPU) on: schedule: - cron: '0 16 * * *' # Execute at 0:00 a.m. Beijing Time every day @@ -73,10 +73,15 @@ jobs: image_a2: ${{ steps.set-vars.outputs.image_a2 }} image_a3: ${{ steps.set-vars.outputs.image_a3 }} skip_install_flag: ${{ steps.set-vars.outputs.skip_install_flag }} + run_start_metadata: ${{ steps.set-vars.outputs.run_start_metadata }} steps: # When triggered by PR, no inputs parameters are used. The latest community code is tested by default. - name: Set image config id: set-vars + env: + # Pass untrusted PR metadata through environment variables to avoid shell‑script interpolation, + # mitigating script injection risks from fork branch names. + HEAD_LABEL: ${{ github.event.pull_request.head.label }} run: | if [ -z "${{ inputs.ref }}" ]; then echo "ref=" >> $GITHUB_OUTPUT @@ -103,211 +108,231 @@ jobs: else echo "skip_install_flag=${{ inputs.skip_install_flag }}" >> $GITHUB_OUTPUT fi - nightly-poc-single-node-a2-tests: - name: single-node-poc-a2 - if: ${{ !cancelled() }} - needs: [ set-image-config ] - strategy: - fail-fast: false - max-parallel: 6 - matrix: - test_config: - - name: qwen3_32b_w8a8_2p_in3k5_out1k5_50ms_a2 - runner: linux-aarch64-a2-4 - test_case: test/registered/npu/performance/qwen3_32b/test_npu_qwen3_32b_w8a8_2p_in3k5_out1k5_50ms_a2.py - test_type: 'perf' - - name: qwen3_32b_w8a8_2p_in3k5_out1k5_50ms_gpqa_a2 - runner: linux-aarch64-a2-4 - test_case: test/registered/npu/accuracy/qwen3_32b/test_npu_qwen3_32b_w8a8_2p_in3k5_out1k5_50ms_gpqa_a2.py - uses: ./.github/workflows/nightly-test-npu-e2e-single-node.yml - with: - runner: ${{ matrix.test_config.runner }} - test_type: ${{ matrix.test_config.test_type }} - test_config_name: ${{ matrix.test_config.name }} - test_case: ${{ matrix.test_config.test_case }} - image: ${{ needs.set-image-config.outputs.image_a2 }} - install_sglang_from_source: false - transformers_version: '' + # If the nightly pipeline is triggered by a PR, this variable takes the source branch label of the PR; + # otherwise, it uses the branch or tag name of the current ref. + BRANCH_LABEL="$HEAD_LABEL" + if [ -z "${BRANCH_LABEL}" ]; then + BRANCH_LABEL="${{ github.ref_name }}" + fi + BRANCH_LABEL="$(echo "${BRANCH_LABEL}" | tr -cd 'a-zA-Z0-9._/:' | tr '/:' '--')" + # Package run metadata (branch/workflow/create_time) as a single JSON for reuse by all + # test jobs; create_time is UTC+8 date + time (minute precision) recorded once at run + # start, shared even across midnight. + RUN_START_METADATA="$(python3 -c "import json,sys; print(json.dumps({'branch_label':sys.argv[1],'workflow_name':sys.argv[2],'create_time':sys.argv[3]}))" "$BRANCH_LABEL" "${{ github.workflow }}" "$(date -u -d '+8 hours' +%Y%m%d-%H%M)")" + # Write to GITHUB_OUTPUT and print to the log in one command. + echo "run_start_metadata=${RUN_START_METADATA}" | tee -a $GITHUB_OUTPUT - nightly-poc-single-node-tests: - name: single-node-poc + nightly-1-npu-a2: + name: nightly-1-npu-a2 if: ${{ !cancelled() }} needs: [set-image-config] - strategy: - fail-fast: false - max-parallel: 6 - matrix: - test_config: - # qwen3_6_35b_a3b performance tests - - name: qwen3_6_35b_a3b_1p_in3k5_out1k5_50ms - runner: linux-aarch64-a3-2 - test_case: test/registered/npu/performance/qwen3_6_35b_a3b/test_npu_qwen3_6_35b_a3b_1p_in3k5_out1k5_50ms.py - test_type: 'perf' - - name: qwen3_6_35b_a3b_1p_aime26 - runner: linux-aarch64-a3-2 - test_case: test/registered/npu/accuracy/qwen3_6_35b_a3b/test_npu_qwen3_6_35b_a3b_1p_aime26.py - - name: qwen3_6_35b_a3b_1p_in64k_out1k_50ms - runner: linux-aarch64-a3-2 - test_case: test/registered/npu/performance/qwen3_6_35b_a3b/test_npu_qwen3_6_35b_a3b_1p_in64k_out1k_50ms.py - test_type: 'perf' - - name: qwen3_6_35b_a3b_1p_in128k_out1k_50ms - runner: linux-aarch64-a3-2 - test_case: test/registered/npu/performance/qwen3_6_35b_a3b/test_npu_qwen3_6_35b_a3b_1p_in128k_out1k_50ms.py - test_type: 'perf' - - name: qwen3_6_35b_a3b_1p_in64k_out1k_prefix90_50ms - runner: linux-aarch64-a3-2 - test_case: test/registered/npu/performance/qwen3_6_35b_a3b/test_npu_qwen3_6_35b_a3b_1p_in64k_out1k_prefix90_50ms.py - test_type: 'perf' - - name: qwen3_6_35b_a3b_1p_in64k_out1k_prefix90_50ms_aime26 - runner: linux-aarch64-a3-2 - test_case: test/registered/npu/accuracy/qwen3_6_35b_a3b/test_npu_qwen3_6_35b_a3b_1p_in64k_out1k_prefix90_50ms_aime26.py - - name: qwen3_6_35b_a3b_1p_in128k_out1k_prefix90_50ms - runner: linux-aarch64-a3-2 - test_case: test/registered/npu/performance/qwen3_6_35b_a3b/test_npu_qwen3_6_35b_a3b_1p_in128k_out1k_prefix90_50ms.py - test_type: 'perf' - # qwen3_6_27b performance tests - - name: qwen3_6_27b_w8a8_1p_in3k5_out1k5_50ms - runner: linux-aarch64-a3-2 - test_case: test/registered/npu/performance/qwen3_6_27b/test_npu_qwen3_6_27b_w8a8_1p_in3k5_out1k5_50ms.py - test_type: 'perf' - - name: qwen3_6_27b_w8a8_1p_in3k5_out1k5_50ms_gpqa - runner: linux-aarch64-a3-2 - test_case: test/registered/npu/accuracy/qwen3_6_27b/test_npu_qwen3_6_27b_w8a8_1p_in3k5_out1k5_50ms_gpqa.py - - name: qwen3_6_27b_w8a8_2p_in16k_out1k_50ms - runner: linux-aarch64-a3-4 - test_case: test/registered/npu/performance/qwen3_6_27b/test_npu_qwen3_6_27b_w8a8_2p_in16k_out1k_50ms.py - test_type: 'perf' - - name: qwen3_6_27b_w8a8_1p_in64k_out1k_50ms - runner: linux-aarch64-a3-4 - test_case: test/registered/npu/performance/qwen3_6_27b/test_npu_qwen3_6_27b_w8a8_1p_in64k_out1k_50ms.py - test_type: 'perf' - - name: qwen3_6_27b_w8a8_2p_in128k_out1k_50ms - runner: linux-aarch64-a3-4 - test_case: test/registered/npu/performance/qwen3_6_27b/test_npu_qwen3_6_27b_w8a8_2p_in128k_out1k_50ms.py - test_type: 'perf' - - name: qwen3_6_27b_2p_in64k_out1k_prefix90_50ms - runner: linux-aarch64-a3-4 - test_case: test/registered/npu/performance/qwen3_6_27b/test_npu_qwen3_6_27b_2p_in64k_out1k_prefix90_50ms.py - test_type: 'perf' - - name: qwen3_6_27b_1p_in1024x1024_30_out1024_50ms - runner: linux-aarch64-a3-2 - test_case: test/registered/npu/performance/qwen3_6_27b/test_npu_qwen3_6_27b_1p_in1024x1024_30_out1024_50ms.py - test_type: 'perf' - - name: qwen3_6_27b_1p_in1080p_30_out256_50ms - runner: linux-aarch64-a3-2 - test_case: test/registered/npu/performance/qwen3_6_27b/test_npu_qwen3_6_27b_1p_in1080p_30_out256_50ms.py - test_type: 'perf' - - name: qwen3_6_27b_1p_gpqa - runner: linux-aarch64-a3-2 - test_case: test/registered/npu/accuracy/qwen3_6_27b/test_npu_qwen3_6_27b_1p_gpqa.py - # qwen3_32b performance tests - - name: qwen3_32b_w8a8_2p_in3k5_out1k5_50ms - runner: linux-aarch64-a3-4 - test_case: test/registered/npu/performance/qwen3_32b/test_npu_qwen3_32b_w8a8_2p_in3k5_out1k5_50ms.py - test_type: 'perf' - - name: qwen3_32b_w8a8_2p_in3k5_out1k5_50ms_gpqa - runner: linux-aarch64-a3-4 - test_case: test/registered/npu/accuracy/qwen3_32b/test_npu_qwen3_32b_w8a8_2p_in3k5_out1k5_50ms_gpqa.py - - name: qwen3_32b_bf16_8p_in18k_out4k_6ms - runner: linux-aarch64-a3-16 - test_case: test/registered/npu/performance/qwen3_32b/test_npu_qwen3_32b_bf16_8p_in18k_out4k_6ms.py - test_type: 'perf' - - name: qwen3_32b_bf16_8p_gpqa - runner: linux-aarch64-a3-16 - test_case: test/registered/npu/accuracy/qwen3_32b/test_npu_qwen3_32b_bf16_8p_gpqa.py - # qwen3_30b_a3b performance tests - - name: qwen3_30b_w8a8_1p_in3k5_out1k5_50ms - runner: linux-aarch64-a3-2 - test_case: test/registered/npu/performance/qwen3_30b_a3b/test_npu_qwen3_30b_w8a8_1p_in3k5_out1k5_50ms.py - test_type: 'perf' - - name: qwen3_30b_w8a8_1p_in3k5_out1k5_50ms_aime25 - runner: linux-aarch64-a3-2 - test_case: test/registered/npu/accuracy/qwen3_30b_a3b/test_npu_qwen3_30b_w8a8_1p_in3k5_out1k5_50ms_aime25.py - # qwen3-8b performance tests - - name: qwen3_8b_w8a8_1p_in3k5_out1k5_50ms - runner: linux-aarch64-a3-2 - test_case: test/registered/npu/performance/qwen3-8b/test_npu_qwen3_8b_w8a8_1p_in3k5_out1k5_50ms.py - test_type: 'perf' - - name: qwen3_8b_w8a8_1p_in3k5_out1k5_50ms_gpqa - runner: linux-aarch64-a3-2 - test_case: test/registered/npu/accuracy/qwen3-8b/test_npu_qwen3_8b_w8a8_1p_in3k5_out1k5_50ms_gpqa.py - - name: qwen3_8b_w8a8_1p_in6k_out1k5_bs16 - runner: linux-aarch64-a3-2 - test_case: test/registered/npu/performance/qwen3-8b/test_npu_qwen3_8b_w8a8_1p_in6k_out1k5_bs16.py - test_type: 'perf' - - name: qwen3_8b_w8a8_1p_in6k_out1k5_bs16_gpqa - runner: linux-aarch64-a3-2 - test_case: test/registered/npu/accuracy/qwen3-8b/test_npu_qwen3_8b_w8a8_1p_in6k_out1k5_bs16_gpqa.py - # qwen3_next_80b_a3b_instruct performance tests - - name: qwen3_next_80b_w8a8_2p_in6k_out1k5_bs16 - runner: linux-aarch64-a3-4 - test_case: test/registered/npu/performance/qwen3_next_80b_a3b_instruct/test_npu_qwen3_next_80b_w8a8_2p_in6k_out1k5_bs16.py - test_type: 'perf' - - name: qwen3_next_80b_w8a8_2p_in6k_out1k5_bs16_aime25 - runner: linux-aarch64-a3-4 - test_case: test/registered/npu/accuracy/qwen3_next_80b_a3b_instruct/test_npu_qwen3_next_80b_w8a8_2p_in6k_out1k5_bs16_aime25.py - # minimax_m2_5 performance tests - - name: minimax_m2_5_w8a8_8p_in3k5_out1k5_50ms - runner: linux-aarch64-a3-16 - test_case: test/registered/npu/performance/minimax_m2_5/test_npu_minimax_m2_5_w8a8_8p_in3k5_out1k5_50ms.py - test_type: 'perf' - - name: minimax_m2_5_w8a8_8p_in3k5_out1k5_50ms_gpqa - runner: linux-aarch64-a3-16 - test_case: test/registered/npu/accuracy/minimax_m2_5/test_npu_minimax_m2_5_w8a8_8p_in3k5_out1k5_50ms_gpqa.py - - name: minimax_m2_5_w8a8_4p_in64k_out1k_prefix90_50ms - runner: linux-aarch64-a3-16 - test_case: test/registered/npu/performance/minimax_m2_5/test_npu_minimax_m2_5_w8a8_4p_in64k_out1k_prefix90_50ms.py - test_type: 'perf' - - name: minimax_m2_5_w8a8_4p_in64k_out1k_prefix90_50ms_gpqa - runner: linux-aarch64-a3-16 - test_case: test/registered/npu/accuracy/minimax_m2_5/test_npu_minimax_m2_5_w8a8_4p_in64k_out1k_prefix90_50ms_gpqa.py - # deepseek_v3_2 accuracy tests - - name: deepseek_v3_2_8p_aime25 - runner: linux-aarch64-a3-16 - test_case: test/registered/npu/accuracy/deepseek_v3_2/test_npu_deepseek_v3_2_8p_aime25.py - # glm4_7_flash accuracy tests - - name: glm4_7_flash_1p_aime25 - runner: linux-aarch64-a3-2 - test_case: test/registered/npu/accuracy/glm4_7_flash/test_npu_glm4_7_flash_1p_aime25.py - # qwen3_vl_8b_thinking accuracy tests - - name: qwen3_vl_8b_thinking_1p_mmmu - runner: linux-aarch64-a3-2 - test_case: test/registered/npu/accuracy/qwen3_vl_8b_thinking/test_npu_qwen3_vl_8b_thinking_1p_mmmu.py - # qwen3_vl_30b_a3b_thinking accuracy tests - - name: qwen3_vl_30b_a3b_thinking_1p_mmmu - runner: linux-aarch64-a3-2 - test_case: test/registered/npu/accuracy/qwen3_vl_30b_a3b_thinking/test_npu_qwen3_vl_30b_a3b_thinking_1p_mmmu.py - - name: qwen3_235b_w8a8_8p_in3k5_out1k5_50ms - runner: linux-aarch64-a3-16 - test_case: test/registered/npu/performance/qwen3_235b_a22b/test_npu_qwen3_235b_w8a8_8p_in3k5_out1k5_50ms.py - test_type: 'perf' - - name: qwen3_5_397b_w4a8_8p_in3k5_out1k5_50ms - runner: linux-aarch64-a3-16 - test_case: test/registered/npu/performance/qwen3_5_397b/test_npu_qwen3_5_397b_w4a8_8p_in3k5_out1k5_50ms.py - test_type: 'perf' - - name: kimi_k2_6_w4a8_8p_in3k5_out1k5_20ms - runner: linux-aarch64-a3-16 - test_case: test/registered/npu/performance/kimi_k2_6/test_npu_kimi_k2_6_w4a8_8p_in3k5_out1k5_20ms.py - test_type: 'perf' - # deepseek_v4_flash performance tests - - name: deepseek_v4_flash_w8a8_8p_in8k_out1k_50ms - runner: linux-aarch64-a3-16 - test_case: test/registered/npu/performance/deepseek_v4_flash/test_npu_deepseek_v4_flash_w8a8_8p_in8k_out1k_50ms.py - test_type: 'perf' - uses: ./.github/workflows/nightly-test-npu-e2e-single-node.yml + uses: ./.github/workflows/_npu-single-node-test-stage.yml with: - runner: ${{ matrix.test_config.runner }} - test_type: ${{ matrix.test_config.test_type }} - test_config_name: ${{ matrix.test_config.name }} - test_case: ${{ matrix.test_config.test_case }} + runner: linux-aarch64-a2-4 + test_type: 'perf' + test_suite: nightly-1-npu-a2 + is_nightly_pipeline_job: true + skip_pr_test_health_check: 'true' + image: ${{ needs.set-image-config.outputs.image_a2 }} + install_sglang_deps: false + device_type_for_deps: '910b' + run_start_metadata: ${{ needs.set-image-config.outputs.run_start_metadata }} + + nightly-1-npu-a3: + name: nightly-1-npu-a3 + if: ${{ !cancelled() }} + needs: [set-image-config] + uses: ./.github/workflows/_npu-pr-test-stage.yml + with: + self_name: nightly-1-npu-a3 + runner_config: linux-aarch64-a3-2- image: ${{ needs.set-image-config.outputs.image_a3 }} - install_sglang_from_source: false - transformers_version: '' + run_timeout_minutes: '60' + timeout_per_file: '3600' + is_nightly_pipeline_job: true + skip_pr_test_health_check: 'true' + secrets: inherit + + nightly-2-npu-a3: + name: nightly-2-npu-a3 + if: ${{ !cancelled() }} + needs: [set-image-config] + uses: ./.github/workflows/_npu-pr-test-stage.yml + with: + self_name: nightly-2-npu-a3 + runner_config: linux-aarch64-a3-2- + image: ${{ needs.set-image-config.outputs.image_a3 }} + run_timeout_minutes: '60' + timeout_per_file: '3600' + is_nightly_pipeline_job: true + skip_pr_test_health_check: 'true' + secrets: inherit + + # The nightly-4-npu-a3 suite is split across 2 parallel partition jobs, mirroring base-b-test-4-npu-a3 in the PR pipeline. + nightly-4-npu-a3: + name: nightly-4-npu-a3 + if: ${{ !cancelled() }} + needs: [set-image-config] + uses: ./.github/workflows/_npu-pr-test-stage.yml + with: + self_name: nightly-4-npu-a3 + runner_config: linux-aarch64-a3-4- + image: ${{ needs.set-image-config.outputs.image_a3 }} + run_timeout_minutes: '120' + timeout_per_file: '3600' + partitions: '{"size":2,"arr":[0,1]}' + is_nightly_pipeline_job: true + skip_pr_test_health_check: 'true' + secrets: inherit + + nightly-8-npu-a3: + name: nightly-8-npu-a3 + if: ${{ !cancelled() }} + needs: [set-image-config] + uses: ./.github/workflows/_npu-pr-test-stage.yml + with: + self_name: nightly-8-npu-a3 + runner_config: linux-aarch64-a3-8- + image: ${{ needs.set-image-config.outputs.image_a3 }} + run_timeout_minutes: '60' + timeout_per_file: '3600' + is_nightly_pipeline_job: true + skip_pr_test_health_check: 'true' + secrets: inherit + + nightly-16-npu-a3: + name: nightly-16-npu-a3 + if: ${{ !cancelled() }} + needs: [set-image-config] + uses: ./.github/workflows/_npu-pr-test-stage.yml + with: + self_name: nightly-16-npu-a3 + runner_config: linux-aarch64-a3-16- + image: ${{ needs.set-image-config.outputs.image_a3 }} + run_timeout_minutes: '120' + timeout_per_file: '3600' + is_nightly_pipeline_job: true + skip_pr_test_health_check: 'true' + secrets: inherit + + nightly-perf-2-npu-a3: + name: nightly-perf-2-npu-a3 + if: ${{ !cancelled() }} + needs: [set-image-config] + uses: ./.github/workflows/_npu-single-node-test-stage.yml + with: + runner: linux-aarch64-a3-800t-2 + test_type: 'perf' + test_suite: nightly-perf-2-npu-a3 + is_nightly_pipeline_job: true + skip_pr_test_health_check: 'true' + image: ${{ needs.set-image-config.outputs.image_a3 }} + install_sglang_deps: false + device_type_for_deps: 'a3' + run_start_metadata: ${{ needs.set-image-config.outputs.run_start_metadata }} + + nightly-perf-4-npu-a3: + name: nightly-perf-4-npu-a3 + if: ${{ !cancelled() }} + needs: [set-image-config] + uses: ./.github/workflows/_npu-single-node-test-stage.yml + with: + runner: linux-aarch64-a3-800t-4 + test_type: 'perf' + test_suite: nightly-perf-4-npu-a3 + is_nightly_pipeline_job: true + skip_pr_test_health_check: 'true' + image: ${{ needs.set-image-config.outputs.image_a3 }} + install_sglang_deps: false + device_type_for_deps: 'a3' + run_start_metadata: ${{ needs.set-image-config.outputs.run_start_metadata }} + + nightly-perf-8-npu-a3: + name: nightly-perf-8-npu-a3 + if: ${{ !cancelled() }} + needs: [set-image-config] + uses: ./.github/workflows/_npu-single-node-test-stage.yml + with: + runner: linux-aarch64-a3-800t-8 + test_type: 'perf' + test_suite: nightly-perf-8-npu-a3 + is_nightly_pipeline_job: true + skip_pr_test_health_check: 'true' + image: ${{ needs.set-image-config.outputs.image_a3 }} + install_sglang_deps: false + device_type_for_deps: 'a3' + run_start_metadata: ${{ needs.set-image-config.outputs.run_start_metadata }} + + nightly-perf-16-npu-a3: + name: nightly-perf-16-npu-a3 + if: ${{ !cancelled() }} + needs: [set-image-config] + uses: ./.github/workflows/_npu-single-node-test-stage.yml + with: + runner: linux-aarch64-a3-800t-16 + test_type: 'perf' + test_suite: nightly-perf-16-npu-a3 + is_nightly_pipeline_job: true + skip_pr_test_health_check: 'true' + image: ${{ needs.set-image-config.outputs.image_a3 }} + install_sglang_deps: false + device_type_for_deps: 'a3' + run_start_metadata: ${{ needs.set-image-config.outputs.run_start_metadata }} + + # The acc-2 suite is split across 4 parallel partition jobs via the single `partitions` input. + nightly-acc-2-npu-a3: + name: nightly-acc-2-npu-a3 + if: ${{ !cancelled() }} + needs: [set-image-config] + uses: ./.github/workflows/_npu-single-node-test-stage.yml + with: + runner: linux-aarch64-a3-2- + test_type: 'accuracy' + test_suite: nightly-acc-2-npu-a3 + is_nightly_pipeline_job: true + partitions: '{"size":4,"arr":[0,1,2,3]}' + skip_pr_test_health_check: 'true' + image: ${{ needs.set-image-config.outputs.image_a3 }} + install_sglang_deps: false + device_type_for_deps: 'a3' + run_start_metadata: ${{ needs.set-image-config.outputs.run_start_metadata }} + + nightly-acc-4-npu-a3: + name: nightly-acc-4-npu-a3 + if: ${{ !cancelled() }} + needs: [set-image-config] + uses: ./.github/workflows/_npu-single-node-test-stage.yml + with: + runner: linux-aarch64-a3-4- + test_type: 'accuracy' + test_suite: nightly-acc-4-npu-a3 + is_nightly_pipeline_job: true + skip_pr_test_health_check: 'true' + image: ${{ needs.set-image-config.outputs.image_a3 }} + install_sglang_deps: false + device_type_for_deps: 'a3' + run_start_metadata: ${{ needs.set-image-config.outputs.run_start_metadata }} + + nightly-acc-16-npu-a3: + name: nightly-acc-16-npu-a3 + if: ${{ !cancelled() }} + needs: [set-image-config] + uses: ./.github/workflows/_npu-single-node-test-stage.yml + with: + runner: linux-aarch64-a3-16- + test_type: 'accuracy' + test_suite: nightly-acc-16-npu-a3 + is_nightly_pipeline_job: true + skip_pr_test_health_check: 'true' + image: ${{ needs.set-image-config.outputs.image_a3 }} + install_sglang_deps: false + device_type_for_deps: 'a3' + run_start_metadata: ${{ needs.set-image-config.outputs.run_start_metadata }} nightly-poc-multi-node-tests: name: multi-node-poc if: ${{ !cancelled() }} - needs: [set-image-config, nightly-poc-single-node-tests] + needs: [set-image-config, nightly-perf-2-npu-a3, nightly-perf-4-npu-a3, nightly-perf-16-npu-a3, nightly-acc-2-npu-a3, nightly-acc-4-npu-a3, nightly-acc-16-npu-a3] strategy: fail-fast: false max-parallel: 1 @@ -326,6 +351,7 @@ jobs: decode_size: 2 router_size: 1 test_case: test/registered/npu/accuracy/glm5_1/test_npu_glm5_1_w4a8_1p1d_32p_in64k_out1k_50ms_aime26.py + test_type: 'accuracy' prefill_decode_deployment: 'separation' # mimo_v2_flash performance tests - name: test_npu_mimo_v2_flash_1p1d_12p_in16k_out1_ttft_5s @@ -344,8 +370,8 @@ jobs: prefill_decode_deployment: 'separation' uses: ./.github/workflows/nightly-test-npu-e2e-multi-node.yml with: - runner: linux-amd64-cpu-8 - test_type: ${{ matrix.test_config.test_type }} + runner: linux-amd64-cpu-4 + test_type: ${{ matrix.test_config.test_type || 'perf' }} test_config_name: ${{ matrix.test_config.name }} prefill_size: ${{ matrix.test_config.prefill_size }} decode_size: ${{ matrix.test_config.decode_size }} @@ -355,11 +381,12 @@ jobs: install_sglang_from_source: false prefill_decode_deployment: ${{ matrix.test_config.prefill_decode_deployment }} transformers_version: '' + run_start_metadata: ${{ needs.set-image-config.outputs.run_start_metadata }} nightly-poc-multi-node-mix-tests: name: multi-node-mix-poc if: ${{ !cancelled() }} - needs: [set-image-config, nightly-poc-single-node-tests, nightly-poc-multi-node-tests] + needs: [set-image-config, nightly-perf-2-npu-a3, nightly-perf-4-npu-a3, nightly-perf-16-npu-a3, nightly-acc-2-npu-a3, nightly-acc-4-npu-a3, nightly-acc-16-npu-a3, nightly-poc-multi-node-tests] strategy: fail-fast: false max-parallel: 1 @@ -373,10 +400,11 @@ jobs: - name: kimi_k2_6_w4a8_16p_in64k_out1k_100ms_aime25 node_size: 2 test_case: test/registered/npu/accuracy/kimi_k2_6/test_npu_kimi_k2_6_w4a8_16p_in64k_out1k_100ms_aime25.py + test_type: 'accuracy' uses: ./.github/workflows/nightly-test-npu-e2e-multi-node.yml with: - runner: linux-amd64-cpu-8 - test_type: ${{ matrix.test_config.test_type }} + runner: linux-amd64-cpu-4 + test_type: ${{ matrix.test_config.test_type || 'perf' }} test_config_name: ${{ matrix.test_config.name }} node_size: ${{ matrix.test_config.node_size }} test_case: ${{ matrix.test_config.test_case }} @@ -384,11 +412,24 @@ jobs: install_sglang_from_source: false prefill_decode_deployment: 'mix' transformers_version: '' + run_start_metadata: ${{ needs.set-image-config.outputs.run_start_metadata }} + check-all-jobs: if: ${{ !cancelled() }} needs: - - nightly-poc-single-node-a2-tests - - nightly-poc-single-node-tests + - nightly-1-npu-a2 + - nightly-1-npu-a3 + - nightly-2-npu-a3 + - nightly-4-npu-a3 + - nightly-8-npu-a3 + - nightly-16-npu-a3 + - nightly-perf-2-npu-a3 + - nightly-perf-4-npu-a3 + - nightly-perf-8-npu-a3 + - nightly-perf-16-npu-a3 + - nightly-acc-2-npu-a3 + - nightly-acc-4-npu-a3 + - nightly-acc-16-npu-a3 - nightly-poc-multi-node-tests - nightly-poc-multi-node-mix-tests runs-on: ubuntu-latest @@ -402,19 +443,30 @@ jobs: - name: Generate results table run: | - status_emoji() { - case "$1" in - pass) echo "✅" ;; - fail) echo "❌" ;; - *) echo "❓" ;; - esac - } - - single_result_a2="${{ needs.nightly-poc-single-node-a2-tests.result }}" - single_result="${{ needs.nightly-poc-single-node-tests.result }}" + single_result_a2="${{ needs.nightly-1-npu-a2.result }}" multi_result="${{ needs.nightly-poc-multi-node-tests.result }}" mix_result="${{ needs.nightly-poc-multi-node-mix-tests.result }}" + # single-node suites are reported per suite; overall status is success if none failed + single_result="success" + for r in \ + "${{ needs.nightly-1-npu-a3.result }}" \ + "${{ needs.nightly-2-npu-a3.result }}" \ + "${{ needs.nightly-4-npu-a3.result }}" \ + "${{ needs.nightly-8-npu-a3.result }}" \ + "${{ needs.nightly-16-npu-a3.result }}" \ + "${{ needs.nightly-perf-2-npu-a3.result }}" \ + "${{ needs.nightly-perf-4-npu-a3.result }}" \ + "${{ needs.nightly-perf-8-npu-a3.result }}" \ + "${{ needs.nightly-perf-16-npu-a3.result }}" \ + "${{ needs.nightly-acc-2-npu-a3.result }}" \ + "${{ needs.nightly-acc-4-npu-a3.result }}" \ + "${{ needs.nightly-acc-16-npu-a3.result }}"; do + if [ "${r}" != "success" ] && [ "${r}" != "skipped" ]; then + single_result="failure" + fi + done + group_icon() { case "$1" in success) echo "✅" ;; @@ -429,8 +481,24 @@ jobs: echo "" >> $GITHUB_STEP_SUMMARY echo "| Group | Status |" >> $GITHUB_STEP_SUMMARY echo "|-------|--------|" >> $GITHUB_STEP_SUMMARY - echo "| single-node-poc-a2 | $(group_icon ${single_result_a2}) ${single_result_a2} |" >> $GITHUB_STEP_SUMMARY - echo "| single-node-poc | $(group_icon ${single_result}) ${single_result} |" >> $GITHUB_STEP_SUMMARY + echo "| nightly-1-npu-a2 | $(group_icon ${single_result_a2}) ${single_result_a2} |" >> $GITHUB_STEP_SUMMARY + for entry in \ + "nightly-1-npu-a3:${{ needs.nightly-1-npu-a3.result }}" \ + "nightly-2-npu-a3:${{ needs.nightly-2-npu-a3.result }}" \ + "nightly-4-npu-a3:${{ needs.nightly-4-npu-a3.result }}" \ + "nightly-8-npu-a3:${{ needs.nightly-8-npu-a3.result }}" \ + "nightly-16-npu-a3:${{ needs.nightly-16-npu-a3.result }}" \ + "nightly-perf-2-npu-a3:${{ needs.nightly-perf-2-npu-a3.result }}" \ + "nightly-perf-4-npu-a3:${{ needs.nightly-perf-4-npu-a3.result }}" \ + "nightly-perf-8-npu-a3:${{ needs.nightly-perf-8-npu-a3.result }}" \ + "nightly-perf-16-npu-a3:${{ needs.nightly-perf-16-npu-a3.result }}" \ + "nightly-acc-2-npu-a3:${{ needs.nightly-acc-2-npu-a3.result }}" \ + "nightly-acc-4-npu-a3:${{ needs.nightly-acc-4-npu-a3.result }}" \ + "nightly-acc-16-npu-a3:${{ needs.nightly-acc-16-npu-a3.result }}"; do + suite="${entry%%:*}" + r="${entry##*:}" + echo "| ${suite} | $(group_icon ${r}) ${r} |" >> $GITHUB_STEP_SUMMARY + done echo "| multi-node-poc | $(group_icon ${multi_result}) ${multi_result} |" >> $GITHUB_STEP_SUMMARY echo "| multi-node-mix-poc | $(group_icon ${mix_result}) ${mix_result} |" >> $GITHUB_STEP_SUMMARY echo "" >> $GITHUB_STEP_SUMMARY @@ -439,8 +507,11 @@ jobs: echo "" >> $GITHUB_STEP_SUMMARY has_metrics=false - for dir in /tmp/metrics/metrics-*/; do - file="${dir}metrics.json" + # Each suite job uploads its METRICS_DATA_FILE directory, which + # contains per-case subdirectories with their own metrics.json. + # Recursively find every metrics.json and render one table row per + # test case. + for file in $(find /tmp/metrics -name metrics.json 2>/dev/null); do if [ -f "$file" ]; then has_metrics=true echo "import json" > /tmp/parse_metrics.py @@ -469,5 +540,7 @@ jobs: echo "" >> $GITHUB_STEP_SUMMARY FAIL=0 - if [ "${single_result}" != "success" ] && [ "${single_result}" != "skipped" ]; then FAIL=1; fi + for r in "${single_result_a2}" "${single_result}" "${multi_result}" "${mix_result}"; do + if [ "${r}" != "success" ] && [ "${r}" != "skipped" ]; then FAIL=1; fi + done exit $FAIL diff --git a/python/sglang/test/ascend/e2e/k8s_multi_pd_mix.yaml.jinja2 b/python/sglang/test/ascend/e2e/k8s_multi_pd_mix.yaml.jinja2 index 1c58d7457..3ad20a9c6 100644 --- a/python/sglang/test/ascend/e2e/k8s_multi_pd_mix.yaml.jinja2 +++ b/python/sglang/test/ascend/e2e/k8s_multi_pd_mix.yaml.jinja2 @@ -60,6 +60,12 @@ spec: value: "https://hf-mirror.com" - name: SGLANG_IS_IN_CI value: "{{ sglang_is_in_ci }}" + - name: RUN_LABEL + value: "{{ run_label }}" + - name: TROUBLE_SHOTTING + value: "{{ trouble_shotting }}" + - name: TRANSFORMERS_VERSION_FOR_SGLANG + value: "{{ transformers_version }}" command: ["/bin/bash", "-c"] args: - | diff --git a/python/sglang/test/ascend/e2e/k8s_multi_pd_mix_green.yaml.jinja2 b/python/sglang/test/ascend/e2e/k8s_multi_pd_mix_green.yaml.jinja2 index f1c971f98..eddcc878f 100644 --- a/python/sglang/test/ascend/e2e/k8s_multi_pd_mix_green.yaml.jinja2 +++ b/python/sglang/test/ascend/e2e/k8s_multi_pd_mix_green.yaml.jinja2 @@ -52,6 +52,8 @@ spec: value: "https://hf-mirror.com" - name: METRICS_DATA_FILE value: "{{ metrics_data_file }}" + - name: RUN_LABEL + value: "{{ run_label }}" - name: SGLANG_IS_IN_CI value: "{{ sglang_is_in_ci }}" - name: TRANSFORMERS_VERSION_FOR_SGLANG diff --git a/python/sglang/test/ascend/e2e/k8s_multi_pd_separation.yaml.jinja2 b/python/sglang/test/ascend/e2e/k8s_multi_pd_separation.yaml.jinja2 index bbffb69a6..9c25516c7 100644 --- a/python/sglang/test/ascend/e2e/k8s_multi_pd_separation.yaml.jinja2 +++ b/python/sglang/test/ascend/e2e/k8s_multi_pd_separation.yaml.jinja2 @@ -60,6 +60,10 @@ spec: value: "{{ metrics_data_file }}" - name: SGLANG_IS_IN_CI value: "{{ sglang_is_in_ci }}" + - name: RUN_LABEL + value: "{{ run_label }}" + - name: TROUBLE_SHOTTING + value: "{{ trouble_shotting }}" - name: TRANSFORMERS_VERSION_FOR_SGLANG value: "{{ transformers_version }}" command: ["/bin/bash", "-c"] @@ -172,6 +176,10 @@ spec: value: "{{ metrics_data_file }}" - name: SGLANG_IS_IN_CI value: "{{ sglang_is_in_ci }}" + - name: RUN_LABEL + value: "{{ run_label }}" + - name: TROUBLE_SHOTTING + value: "{{ trouble_shotting }}" - name: TRANSFORMERS_VERSION_FOR_SGLANG value: "{{ transformers_version }}" command: ["/bin/bash", "-c"] @@ -274,6 +282,8 @@ spec: value: "{{ install_sglang_from_source }}" - name: METRICS_DATA_FILE value: "{{ metrics_data_file }}" + - name: RUN_LABEL + value: "{{ run_label }}" - name: KUBECONFIG value: "{{ kube_config }}" - name: NAMESPACE @@ -284,6 +294,8 @@ spec: value: "https://hf-mirror.com" - name: SGLANG_IS_IN_CI value: "{{ sglang_is_in_ci }}" + - name: TROUBLE_SHOTTING + value: "{{ trouble_shotting }}" - name: TRANSFORMERS_VERSION_FOR_SGLANG value: "{{ transformers_version }}" command: ["/bin/bash", "-c"] diff --git a/python/sglang/test/ascend/e2e/k8s_multi_pd_separation_green.yaml.jinja2 b/python/sglang/test/ascend/e2e/k8s_multi_pd_separation_green.yaml.jinja2 index fa8f5b04e..d51dce478 100644 --- a/python/sglang/test/ascend/e2e/k8s_multi_pd_separation_green.yaml.jinja2 +++ b/python/sglang/test/ascend/e2e/k8s_multi_pd_separation_green.yaml.jinja2 @@ -53,6 +53,8 @@ spec: value: "https://hf-mirror.com" - name: METRICS_DATA_FILE value: "{{ metrics_data_file }}" + - name: RUN_LABEL + value: "{{ run_label }}" - name: SGLANG_IS_IN_CI value: "{{ sglang_is_in_ci }}" command: ["/bin/bash", "-c"] @@ -154,6 +156,8 @@ spec: value: "https://hf-mirror.com" - name: METRICS_DATA_FILE value: "{{ metrics_data_file }}" + - name: RUN_LABEL + value: "{{ run_label }}" - name: SGLANG_IS_IN_CI value: "{{ sglang_is_in_ci }}" command: ["/bin/bash", "-c"] @@ -254,6 +258,8 @@ spec: value: "https://hf-mirror.com" - name: METRICS_DATA_FILE value: "{{ metrics_data_file }}" + - name: RUN_LABEL + value: "{{ run_label }}" - name: SGLANG_IS_IN_CI value: "{{ sglang_is_in_ci }}" command: ["/bin/bash", "-c"] diff --git a/python/sglang/test/ascend/e2e/k8s_single.yaml.jinja2 b/python/sglang/test/ascend/e2e/k8s_single.yaml.jinja2 index dff5a5591..af4246bab 100644 --- a/python/sglang/test/ascend/e2e/k8s_single.yaml.jinja2 +++ b/python/sglang/test/ascend/e2e/k8s_single.yaml.jinja2 @@ -51,8 +51,13 @@ spec: value: "{{ kube_config }}" - name: HF_ENDPOINT value: "https://hf-mirror.com" + # Env vars consumed by run_npu_testcase.sh inside the pod - name: TROUBLE_SHOTTING value: "{{ trouble_shotting }}" + - name: RUN_LABEL + value: "{{ run_label }}" + - name: SGLANG_IS_IN_CI + value: "{{ sglang_is_in_ci }}" - name: TRANSFORMERS_VERSION_FOR_SGLANG value: "{{ transformers_version }}" command: ["/bin/bash", "-c"] diff --git a/python/sglang/test/ascend/e2e/run_npu_e2e_test.py b/python/sglang/test/ascend/e2e/run_npu_e2e_test.py index 28295fdda..81d53e8bd 100644 --- a/python/sglang/test/ascend/e2e/run_npu_e2e_test.py +++ b/python/sglang/test/ascend/e2e/run_npu_e2e_test.py @@ -518,10 +518,17 @@ def generate_metrics_json(metrics_data_file, test_case, status): tc_name = test_case.rsplit("/", 1)[-1].rsplit(".", 1)[0] test_type = "unknown" + # nightly: .../output/{branch}-{date}-{run_id}-{run_attempt}/{workflow}/{test_type}/... + # PR: .../output/{test_type}/{date}/{tc_name} + # After `output`, rest >= 4 segments -> test_type at parts[i+3]; otherwise at parts[i+1]. parts = metrics_data_file.split("/") for i, part in enumerate(parts): - if part == "output" and i + 1 < len(parts): - test_type = parts[i + 1] + if part == "output": + rest = len(parts) - (i + 1) + if rest >= 4: + test_type = parts[i + 3] + elif rest >= 1: + test_type = parts[i + 1] break output = { @@ -576,6 +583,22 @@ def run_npu_e2e_test_case( kube_config_map = f"sglang-configmap-{random_str}" final_kube_job_name = f"{kube_job_name_prefix}-{random_str}" + # run_label is injected into the pod as RUN_LABEL to build the pod log directory prefix. + # nightly (>=4 segments after `output`): first two segments {branch}-{date}-{run_id}-{run_attempt}/{workflow} + # PR legacy layout: fall back to the date segment to keep the original {date}/{tc_name}/{host} path. + parts = ( + metrics_data_file.split("/output/")[-1] + if "/output/" in metrics_data_file + else "" + ) + if parts: + segments = parts.split("/") + if len(segments) >= 4: + run_label = "/".join(segments[:2]) + else: + run_label = segments[1] if len(segments) > 1 else "unknown" + else: + run_label = "unknown" kube_yaml_file_dict = { KUBE_JOB_SINGLE: f"k8s_single_{random_str}.yaml", @@ -605,6 +628,7 @@ def run_npu_e2e_test_case( "env": env, "trouble_shotting": trouble_shotting, "transformers_version": transformers_version, + "run_label": run_label, } create_kube_yaml( kube_yaml_template=KUBE_YAML_TEMPLATE.get(kube_job_type), @@ -627,6 +651,7 @@ def run_npu_e2e_test_case( "env": env, "trouble_shotting": trouble_shotting, "transformers_version": transformers_version, + "run_label": run_label, } template_key = ( KUBE_JOB_MULTI_PD_MIX_GREEN if env == "green" else kube_job_type @@ -654,6 +679,7 @@ def run_npu_e2e_test_case( "env": env, "trouble_shotting": trouble_shotting, "transformers_version": transformers_version, + "run_label": run_label, } template_key = ( KUBE_JOB_MULTI_PD_SEPARATION_GREEN if env == "green" else kube_job_type diff --git a/python/sglang/test/ascend/e2e/run_npu_testcase.sh b/python/sglang/test/ascend/e2e/run_npu_testcase.sh index 439fdb6a9..4ce070093 100644 --- a/python/sglang/test/ascend/e2e/run_npu_testcase.sh +++ b/python/sglang/test/ascend/e2e/run_npu_testcase.sh @@ -34,7 +34,7 @@ if [ -n "${TRANSFORMERS_VERSION_FOR_SGLANG}" ];then echo "Install transformers ${TRANSFORMERS_VERSION_FOR_SGLANG} locally." TRANSFORMERS_PKG_PATH_TARGET=/tmp/transformers/${TRANSFORMERS_VERSION_FOR_SGLANG} mkdir -p "${TRANSFORMERS_PKG_PATH_TARGET}" - cp "${TRANSFORMERS_PKG_PATH_SOURCE}/*" "${TRANSFORMERS_PKG_PATH_TARGET}/" + cp "${TRANSFORMERS_PKG_PATH_SOURCE}/"* "${TRANSFORMERS_PKG_PATH_TARGET}/" pip install --no-index --find-links="${TRANSFORMERS_PKG_PATH_TARGET}" transformers=="${TRANSFORMERS_VERSION_FOR_SGLANG}" fi echo "===== Install transformers for sglang in virtual env - End =====" @@ -114,10 +114,11 @@ fi echo "Running test case ${test_case}" tc_name=${test_case##*/} tc_name=${tc_name%.*} -current_date=$(date +%Y%m%d) -log_path="/root/sglang/debug/logs/log/${current_date}/${tc_name}/${HOSTNAME}" +run_label="${RUN_LABEL:-unknown}" +log_path="/root/sglang/debug/logs/log/${run_label}/${tc_name}/${HOSTNAME}" if [ "${SGLANG_IS_IN_CI}" = "true" ] || [ "${SGLANG_IS_IN_CI}" = "True" ];then - log_path="/root/.cache/tests/logs/log/${current_date}/${tc_name}/${HOSTNAME}" + # In CI, persist logs under /root/.cache/tests/logs so they can be collected + log_path="/root/.cache/tests/logs/log/${run_label}/${tc_name}/${HOSTNAME}" fi rm -rf "${log_path}" mkdir -p "${log_path}" @@ -134,6 +135,7 @@ echo "Finished test case ${test_case}" if [ -n "${METRICS_DATA_FILE}" ]; then mkdir -p "${METRICS_DATA_FILE}" + # Archive the test log into the output directory for result collection cp "${log_path}/${tc_name}.log" "${METRICS_DATA_FILE}/test_output.log" echo "Metrics log saved to ${METRICS_DATA_FILE}/test_output.log" fi diff --git a/python/sglang/test/ascend/e2e/test_npu_accuracy_utils.py b/python/sglang/test/ascend/e2e/test_npu_accuracy_utils.py index 94d746c9d..9a7785cd3 100644 --- a/python/sglang/test/ascend/e2e/test_npu_accuracy_utils.py +++ b/python/sglang/test/ascend/e2e/test_npu_accuracy_utils.py @@ -311,16 +311,27 @@ class TestNpuAccuracyTestCaseBase(CustomTestCase): def _setup_per_case_output(cls): """Set up per-case output directories and env vars. - Extracted from ``nightly-test-npu-e2e-single-node.yml`` so that when a - suite is executed, each case writes its metrics/plog to a path derived - from the case file rather than the suite name. + When the workflow sets METRICS_DATA_FILE to a suite-level directory + (e.g. .../output/{branch_label}-{create_date}-{run_id}-{run_attempt}/ + {workflow_name}/{test_type}/{suite}), each case in the suite + writes to its own subdirectory under it, so results stay in the + structured layout and are keyed by the case id. Falls back to the + legacy per-case layout when the env var is not set. """ cls.tc_name = cls._get_tc_name() - current_date = datetime.now().strftime("%Y%m%d") - test_type = getattr(cls, "test_type", "accuracy") - base_output = f"/root/.cache/tests/output/{test_type}/{current_date}" - os.makedirs(base_output, exist_ok=True) - cls.metrics_data_file = os.path.join(base_output, cls.tc_name) + suite_output = os.environ.get("METRICS_DATA_FILE") + if suite_output: + # Append the case id under the suite output prefix. + cls.metrics_data_file = os.path.join(suite_output, cls.tc_name) + # Mirror the output prefix to the plog location (drop the test_type/suite tail). + suite_plog = suite_output.replace("/output/", "/logs/plog/", 1) + cls.plog_base = os.path.dirname(os.path.dirname(suite_plog)) + else: + current_date = datetime.now().strftime("%Y%m%d") + test_type = getattr(cls, "test_type", "accuracy") + base_output = f"/root/.cache/tests/output/{test_type}/{current_date}" + cls.metrics_data_file = os.path.join(base_output, cls.tc_name) + cls.plog_base = f"/root/.cache/tests/logs/plog" os.makedirs(cls.metrics_data_file, exist_ok=True) # Override env vars so evalscope/dump_metric write to per-case paths. os.environ["METRICS_DATA_FILE"] = cls.metrics_data_file @@ -376,6 +387,12 @@ class TestNpuAccuracyTestCaseBase(CustomTestCase): logger.info("Saved per-case metrics to %s", out_path) except Exception as e: logger.warning("Failed to write metrics.json: %s", e) + # Remove the intermediate JSONL records, keeping only the final metrics.json. + for jsonl_path in glob.glob(pattern): + try: + os.remove(jsonl_path) + except Exception as e: + logger.warning("Failed to remove %s: %s", jsonl_path, e) @classmethod def _backup_plog(cls): @@ -391,7 +408,8 @@ class TestNpuAccuracyTestCaseBase(CustomTestCase): if not tc_name: return hostname = os.getenv("HOSTNAME", "unknown") - target = os.path.join("/root/.cache/tests/logs/plog", tc_name, hostname) + plog_base = getattr(cls, "plog_base", "/root/.cache/tests/logs/plog") + target = os.path.join(plog_base, tc_name, hostname) os.makedirs(target, exist_ok=True) for name in os.listdir(plog_path): src = os.path.join(plog_path, name) diff --git a/python/sglang/test/ascend/e2e/test_npu_performance_utils.py b/python/sglang/test/ascend/e2e/test_npu_performance_utils.py index dfa30e6b8..61c299621 100644 --- a/python/sglang/test/ascend/e2e/test_npu_performance_utils.py +++ b/python/sglang/test/ascend/e2e/test_npu_performance_utils.py @@ -930,16 +930,27 @@ class TestNpuPerformanceTestCaseBase(CustomTestCase): def _setup_per_case_output(cls): """Set up per-case output directories and env vars. - Extracted from ``nightly-test-npu-e2e-single-node.yml`` so that when a - suite is executed, each case writes its metrics/plog to a path derived - from the case file rather than the suite name. + When the workflow sets METRICS_DATA_FILE to a suite-level directory + (e.g. .../output/{branch_label}-{create_date}-{run_id}-{run_attempt}/ + {workflow_name}/{test_type}/{suite}), each case in the suite + writes to its own subdirectory under it, so results stay in the + structured layout and are keyed by the case id. Falls back to the + legacy per-case layout when the env var is not set. """ cls.tc_name = cls._get_tc_name() - current_date = datetime.now().strftime("%Y%m%d") - test_type = getattr(cls, "test_type", "perf") - base_output = f"/root/.cache/tests/output/{test_type}/{current_date}" - os.makedirs(base_output, exist_ok=True) - cls.metrics_data_file = os.path.join(base_output, cls.tc_name) + suite_output = os.environ.get("METRICS_DATA_FILE") + if suite_output: + # Append the case id under the suite output prefix. + cls.metrics_data_file = os.path.join(suite_output, cls.tc_name) + # Mirror the output prefix to the plog location (drop the test_type/suite tail). + suite_plog = suite_output.replace("/output/", "/logs/plog/", 1) + cls.plog_base = os.path.dirname(os.path.dirname(suite_plog)) + else: + current_date = datetime.now().strftime("%Y%m%d") + test_type = getattr(cls, "test_type", "perf") + base_output = f"/root/.cache/tests/output/{test_type}/{current_date}" + cls.metrics_data_file = os.path.join(base_output, cls.tc_name) + cls.plog_base = f"/root/.cache/tests/logs/plog" os.makedirs(cls.metrics_data_file, exist_ok=True) # Override env vars so evalscope/dump_metric write to per-case paths. os.environ["METRICS_DATA_FILE"] = cls.metrics_data_file @@ -985,7 +996,7 @@ class TestNpuPerformanceTestCaseBase(CustomTestCase): out_path = os.path.join(cls.metrics_data_file, "metrics.json") payload = { "test_case": cls.tc_name, - "test_type": getattr(cls, "test_type", "accuracy"), + "test_type": getattr(cls, "test_type", "perf"), "metrics": metrics, "baselines": baselines, } @@ -995,6 +1006,12 @@ class TestNpuPerformanceTestCaseBase(CustomTestCase): logger.info("Saved per-case metrics to %s", out_path) except Exception as e: logger.warning("Failed to write metrics.json: %s", e) + # Remove the intermediate JSONL records, keeping only the final metrics.json. + for jsonl_path in glob.glob(pattern): + try: + os.remove(jsonl_path) + except Exception as e: + logger.warning("Failed to remove %s: %s", jsonl_path, e) @classmethod def _backup_plog(cls): @@ -1010,7 +1027,8 @@ class TestNpuPerformanceTestCaseBase(CustomTestCase): if not tc_name: return hostname = os.getenv("HOSTNAME", "unknown") - target = os.path.join("/root/.cache/tests/logs/plog", tc_name, hostname) + plog_base = getattr(cls, "plog_base", "/root/.cache/tests/logs/plog") + target = os.path.join(plog_base, tc_name, hostname) os.makedirs(target, exist_ok=True) for name in os.listdir(plog_path): src = os.path.join(plog_path, name) diff --git a/test/registered/npu/accuracy/deepseek_v3_2/test_npu_deepseek_v3_2_8p_aime25.py b/test/registered/npu/accuracy/deepseek_v3_2/test_npu_deepseek_v3_2_8p_aime25.py index 32a1a39b2..00e5a080b 100644 --- a/test/registered/npu/accuracy/deepseek_v3_2/test_npu_deepseek_v3_2_8p_aime25.py +++ b/test/registered/npu/accuracy/deepseek_v3_2/test_npu_deepseek_v3_2_8p_aime25.py @@ -7,10 +7,9 @@ from sglang.test.ascend.test_ascend_utils import DEEPSEEK_V3_2_EXP_W8A8_WEIGHTS_ from sglang.test.ci.ci_register import register_npu_ci register_npu_ci( - est_time=3600, - suite="", + est_time=4800, + suite="nightly-acc-16-npu-a3", nightly=True, - disabled="accuracy testcase", ) OTHER_ARGS = [ diff --git a/test/registered/npu/accuracy/glm4_6v_flash/test_npu_glm4_6v_flash_1p_mmmu.py b/test/registered/npu/accuracy/glm4_6v_flash/test_npu_glm4_6v_flash_1p_mmmu.py index 2cfe5d563..8b122225f 100644 --- a/test/registered/npu/accuracy/glm4_6v_flash/test_npu_glm4_6v_flash_1p_mmmu.py +++ b/test/registered/npu/accuracy/glm4_6v_flash/test_npu_glm4_6v_flash_1p_mmmu.py @@ -7,7 +7,7 @@ from sglang.test.ascend.e2e.test_npu_performance_utils import GLM_4_6V_FLASH_MOD from sglang.test.ci.ci_register import register_npu_ci register_npu_ci( - est_time=3600, + est_time=4800, suite="", nightly=True, disabled="performance testcase", diff --git a/test/registered/npu/accuracy/glm4_7_flash/test_npu_glm4_7_flash_1p_aime25.py b/test/registered/npu/accuracy/glm4_7_flash/test_npu_glm4_7_flash_1p_aime25.py index 679c78a4a..9394ca6b1 100644 --- a/test/registered/npu/accuracy/glm4_7_flash/test_npu_glm4_7_flash_1p_aime25.py +++ b/test/registered/npu/accuracy/glm4_7_flash/test_npu_glm4_7_flash_1p_aime25.py @@ -7,10 +7,9 @@ from sglang.test.ascend.e2e.test_npu_performance_utils import GLM_4_7_FLASH_MODE from sglang.test.ci.ci_register import register_npu_ci register_npu_ci( - est_time=3600, - suite="", + est_time=6500, + suite="nightly-acc-2-npu-a3", nightly=True, - disabled="accuracy testcase", ) ENVS = { diff --git a/test/registered/npu/accuracy/glm4_7_flash/test_npu_glm4_7_flash_1p_gsm8k.py b/test/registered/npu/accuracy/glm4_7_flash/test_npu_glm4_7_flash_1p_gsm8k.py index 7a24cae35..1551d0558 100644 --- a/test/registered/npu/accuracy/glm4_7_flash/test_npu_glm4_7_flash_1p_gsm8k.py +++ b/test/registered/npu/accuracy/glm4_7_flash/test_npu_glm4_7_flash_1p_gsm8k.py @@ -7,6 +7,7 @@ from sglang.test.ascend.e2e.test_npu_performance_utils import GLM_4_7_FLASH_MODE from sglang.test.ci.ci_register import register_npu_ci register_npu_ci(est_time=3600, suite="base-c-test-acc-2-npu-a3") +register_npu_ci(est_time=6500, suite="nightly-acc-2-npu-a3", nightly=True) ENVS = { "PYTORCH_NPU_ALLOC_CONF": "expandable_segments:True", diff --git a/test/registered/npu/accuracy/glm5_1/test_npu_glm5_1_w4a8_1p1d_32p_in64k_out1k_50ms_aime26.py b/test/registered/npu/accuracy/glm5_1/test_npu_glm5_1_w4a8_1p1d_32p_in64k_out1k_50ms_aime26.py index bd977dfb6..ac127ba43 100644 --- a/test/registered/npu/accuracy/glm5_1/test_npu_glm5_1_w4a8_1p1d_32p_in64k_out1k_50ms_aime26.py +++ b/test/registered/npu/accuracy/glm5_1/test_npu_glm5_1_w4a8_1p1d_32p_in64k_out1k_50ms_aime26.py @@ -10,7 +10,7 @@ from sglang.test.ascend.e2e.test_npu_performance_utils import ( from sglang.test.ci.ci_register import register_npu_ci register_npu_ci( - est_time=3600, + est_time=4800, suite="", nightly=True, disabled="accuracy testcase", diff --git a/test/registered/npu/accuracy/glm5_top64_pruned/test_npu_glm5_top64_pruned_bf16_8p_gsm8k.py b/test/registered/npu/accuracy/glm5_top64_pruned/test_npu_glm5_top64_pruned_bf16_8p_gsm8k.py index 7ae8616ff..ccfe108af 100644 --- a/test/registered/npu/accuracy/glm5_top64_pruned/test_npu_glm5_top64_pruned_bf16_8p_gsm8k.py +++ b/test/registered/npu/accuracy/glm5_top64_pruned/test_npu_glm5_top64_pruned_bf16_8p_gsm8k.py @@ -9,6 +9,7 @@ from sglang.test.ascend.e2e.test_npu_performance_utils import ( from sglang.test.ci.ci_register import register_npu_ci register_npu_ci(est_time=3600, suite="base-c-test-acc-16-npu-a3") +register_npu_ci(est_time=4800, suite="nightly-acc-16-npu-a3", nightly=True) ENVS = { "SGLANG_SET_CPU_AFFINITY": "1", diff --git a/test/registered/npu/accuracy/kimi_k2_6/test_npu_kimi_k2_6_w4a8_16p_in64k_out1k_100ms_aime25.py b/test/registered/npu/accuracy/kimi_k2_6/test_npu_kimi_k2_6_w4a8_16p_in64k_out1k_100ms_aime25.py index 24cd79560..780c8b2f9 100644 --- a/test/registered/npu/accuracy/kimi_k2_6/test_npu_kimi_k2_6_w4a8_16p_in64k_out1k_100ms_aime25.py +++ b/test/registered/npu/accuracy/kimi_k2_6/test_npu_kimi_k2_6_w4a8_16p_in64k_out1k_100ms_aime25.py @@ -12,7 +12,7 @@ from sglang.test.ci.ci_register import register_npu_ci register_npu_ci( est_time=1800, - suite="nightly-8-npu-a3", + suite="full-8-npu-a3", nightly=True, disabled="accuracy testcase", ) diff --git a/test/registered/npu/accuracy/minimax_m2_5/test_npu_minimax_m2_5_w8a8_4p_in64k_out1k_prefix90_50ms_gpqa.py b/test/registered/npu/accuracy/minimax_m2_5/test_npu_minimax_m2_5_w8a8_4p_in64k_out1k_prefix90_50ms_gpqa.py index bd7d53bfd..955f656f2 100644 --- a/test/registered/npu/accuracy/minimax_m2_5/test_npu_minimax_m2_5_w8a8_4p_in64k_out1k_prefix90_50ms_gpqa.py +++ b/test/registered/npu/accuracy/minimax_m2_5/test_npu_minimax_m2_5_w8a8_4p_in64k_out1k_prefix90_50ms_gpqa.py @@ -11,10 +11,9 @@ from sglang.test.ascend.e2e.test_npu_performance_utils import ( from sglang.test.ci.ci_register import register_npu_ci register_npu_ci( - est_time=3600, - suite="", + est_time=7200, + suite="nightly-acc-16-npu-a3", nightly=True, - disabled="accuracy testcase", ) MINIMAX_M2_5_W8A8_4P_IN64K_OUT1K_PREFIX90_ENVS = { diff --git a/test/registered/npu/accuracy/minimax_m2_5/test_npu_minimax_m2_5_w8a8_8p_in3k5_out1k5_50ms_gpqa.py b/test/registered/npu/accuracy/minimax_m2_5/test_npu_minimax_m2_5_w8a8_8p_in3k5_out1k5_50ms_gpqa.py index a92462eea..21eba280f 100644 --- a/test/registered/npu/accuracy/minimax_m2_5/test_npu_minimax_m2_5_w8a8_8p_in3k5_out1k5_50ms_gpqa.py +++ b/test/registered/npu/accuracy/minimax_m2_5/test_npu_minimax_m2_5_w8a8_8p_in3k5_out1k5_50ms_gpqa.py @@ -11,10 +11,9 @@ from sglang.test.ascend.e2e.test_npu_performance_utils import ( from sglang.test.ci.ci_register import register_npu_ci register_npu_ci( - est_time=3600, - suite="full-16-npu-a3", + est_time=4800, + suite="nightly-acc-16-npu-a3", nightly=True, - disabled="accuracy testcase", ) MINIMAX_M2_5_HIGH_THROUGHPUT_ENVS = { diff --git a/test/registered/npu/accuracy/moonshotai_moonlight_16b_a3b/test_npu_moonlight_16b_a3b_bf16_1p_gsm8k.py b/test/registered/npu/accuracy/moonshotai_moonlight_16b_a3b/test_npu_moonlight_16b_a3b_bf16_1p_gsm8k.py index 2a4675e16..08a971e0b 100644 --- a/test/registered/npu/accuracy/moonshotai_moonlight_16b_a3b/test_npu_moonlight_16b_a3b_bf16_1p_gsm8k.py +++ b/test/registered/npu/accuracy/moonshotai_moonlight_16b_a3b/test_npu_moonlight_16b_a3b_bf16_1p_gsm8k.py @@ -9,6 +9,7 @@ from sglang.test.ascend.e2e.test_npu_performance_utils import ( from sglang.test.ci.ci_register import register_npu_ci register_npu_ci(est_time=3600, suite="base-c-test-acc-2-npu-a3") +register_npu_ci(est_time=1300, suite="nightly-acc-2-npu-a3", nightly=True) MODEL_ENVS = { "SGLANG_SET_CPU_AFFINITY": "1", diff --git a/test/registered/npu/accuracy/qwen3-8b/test_npu_qwen3_8b_w8a8_1p_in3k5_out1k5_50ms_gpqa.py b/test/registered/npu/accuracy/qwen3-8b/test_npu_qwen3_8b_w8a8_1p_in3k5_out1k5_50ms_gpqa.py index 6eb55f49c..abae77090 100644 --- a/test/registered/npu/accuracy/qwen3-8b/test_npu_qwen3_8b_w8a8_1p_in3k5_out1k5_50ms_gpqa.py +++ b/test/registered/npu/accuracy/qwen3-8b/test_npu_qwen3_8b_w8a8_1p_in3k5_out1k5_50ms_gpqa.py @@ -10,10 +10,9 @@ from sglang.test.ascend.e2e.test_npu_performance_utils import ( from sglang.test.ci.ci_register import register_npu_ci register_npu_ci( - est_time=3600, - suite="", + est_time=3700, + suite="nightly-acc-2-npu-a3", nightly=True, - disabled="accuracy testcase", ) QWEN3_8B_ENVS = { diff --git a/test/registered/npu/accuracy/qwen3-8b/test_npu_qwen3_8b_w8a8_1p_in6k_out1k5_bs16_gpqa.py b/test/registered/npu/accuracy/qwen3-8b/test_npu_qwen3_8b_w8a8_1p_in6k_out1k5_bs16_gpqa.py index b336a5087..e31bfe394 100644 --- a/test/registered/npu/accuracy/qwen3-8b/test_npu_qwen3_8b_w8a8_1p_in6k_out1k5_bs16_gpqa.py +++ b/test/registered/npu/accuracy/qwen3-8b/test_npu_qwen3_8b_w8a8_1p_in6k_out1k5_bs16_gpqa.py @@ -10,10 +10,9 @@ from sglang.test.ascend.e2e.test_npu_performance_utils import ( from sglang.test.ci.ci_register import register_npu_ci register_npu_ci( - est_time=3600, - suite="", + est_time=3700, + suite="nightly-acc-2-npu-a3", nightly=True, - disabled="accuracy testcase", ) QWEN3_8B_ENVS = { diff --git a/test/registered/npu/accuracy/qwen3_30b_a3b/test_npu_qwen3_30b_w8a8_1p_in3k5_out1k5_50ms_aime25.py b/test/registered/npu/accuracy/qwen3_30b_a3b/test_npu_qwen3_30b_w8a8_1p_in3k5_out1k5_50ms_aime25.py index 802441071..7e4e4d0de 100644 --- a/test/registered/npu/accuracy/qwen3_30b_a3b/test_npu_qwen3_30b_w8a8_1p_in3k5_out1k5_50ms_aime25.py +++ b/test/registered/npu/accuracy/qwen3_30b_a3b/test_npu_qwen3_30b_w8a8_1p_in3k5_out1k5_50ms_aime25.py @@ -10,10 +10,9 @@ from sglang.test.ascend.e2e.test_npu_performance_utils import ( from sglang.test.ci.ci_register import register_npu_ci register_npu_ci( - est_time=3600, - suite="", + est_time=2800, + suite="nightly-acc-2-npu-a3", nightly=True, - disabled="accuracy testcase", ) QWEN3_30B_A3B_ENVS = { diff --git a/test/registered/npu/accuracy/qwen3_32b/test_npu_qwen3_32b_bf16_8p_gpqa.py b/test/registered/npu/accuracy/qwen3_32b/test_npu_qwen3_32b_bf16_8p_gpqa.py index d3f2ba2d4..e8776f041 100644 --- a/test/registered/npu/accuracy/qwen3_32b/test_npu_qwen3_32b_bf16_8p_gpqa.py +++ b/test/registered/npu/accuracy/qwen3_32b/test_npu_qwen3_32b_bf16_8p_gpqa.py @@ -10,10 +10,9 @@ from sglang.test.ascend.e2e.test_npu_performance_utils import ( from sglang.test.ci.ci_register import register_npu_ci register_npu_ci( - est_time=3600, - suite="", + est_time=4800, + suite="nightly-acc-16-npu-a3", nightly=True, - disabled="performance testcase", ) QWEN3_32B_ENVS = { diff --git a/test/registered/npu/accuracy/qwen3_32b/test_npu_qwen3_32b_w8a8_2p_in3k5_out1k5_50ms_gpqa.py b/test/registered/npu/accuracy/qwen3_32b/test_npu_qwen3_32b_w8a8_2p_in3k5_out1k5_50ms_gpqa.py index 6fb1dad81..b1ee89074 100644 --- a/test/registered/npu/accuracy/qwen3_32b/test_npu_qwen3_32b_w8a8_2p_in3k5_out1k5_50ms_gpqa.py +++ b/test/registered/npu/accuracy/qwen3_32b/test_npu_qwen3_32b_w8a8_2p_in3k5_out1k5_50ms_gpqa.py @@ -10,10 +10,9 @@ from sglang.test.ascend.e2e.test_npu_performance_utils import ( from sglang.test.ci.ci_register import register_npu_ci register_npu_ci( - est_time=3600, - suite="", + est_time=4800, + suite="nightly-acc-4-npu-a3", nightly=True, - disabled="accuracy testcase", ) QWEN3_32B_ENVS = { diff --git a/test/registered/npu/accuracy/qwen3_32b/test_npu_qwen3_32b_w8a8_2p_in3k5_out1k5_50ms_gpqa_a2.py b/test/registered/npu/accuracy/qwen3_32b/test_npu_qwen3_32b_w8a8_2p_in3k5_out1k5_50ms_gpqa_a2.py index eacac5ec0..b46999437 100644 --- a/test/registered/npu/accuracy/qwen3_32b/test_npu_qwen3_32b_w8a8_2p_in3k5_out1k5_50ms_gpqa_a2.py +++ b/test/registered/npu/accuracy/qwen3_32b/test_npu_qwen3_32b_w8a8_2p_in3k5_out1k5_50ms_gpqa_a2.py @@ -10,10 +10,9 @@ from sglang.test.ascend.e2e.test_npu_performance_utils import ( from sglang.test.ci.ci_register import register_npu_ci register_npu_ci( - est_time=3600, - suite="", + est_time=4800, + suite="nightly-1-npu-a2", nightly=True, - disabled="accuracy testcase", ) QWEN3_32B_ENVS = { diff --git a/test/registered/npu/accuracy/qwen3_5_9b/test_npu_qwen3_5_9b_bf16_1p_gsm8k.py b/test/registered/npu/accuracy/qwen3_5_9b/test_npu_qwen3_5_9b_bf16_1p_gsm8k.py index c997de8bf..00fe80710 100644 --- a/test/registered/npu/accuracy/qwen3_5_9b/test_npu_qwen3_5_9b_bf16_1p_gsm8k.py +++ b/test/registered/npu/accuracy/qwen3_5_9b/test_npu_qwen3_5_9b_bf16_1p_gsm8k.py @@ -9,6 +9,7 @@ from sglang.test.ascend.e2e.test_npu_performance_utils import ( from sglang.test.ci.ci_register import register_npu_ci register_npu_ci(est_time=3600, suite="base-c-test-acc-2-npu-a3") +register_npu_ci(est_time=2800, suite="nightly-acc-2-npu-a3", nightly=True) QWEN3_5_9B_ENVS = { "SGLANG_SET_CPU_AFFINITY": "1", diff --git a/test/registered/npu/accuracy/qwen3_6_27b/test_npu_qwen3_6_27b_1p_gpqa.py b/test/registered/npu/accuracy/qwen3_6_27b/test_npu_qwen3_6_27b_1p_gpqa.py index ca8edf142..a799c97b9 100644 --- a/test/registered/npu/accuracy/qwen3_6_27b/test_npu_qwen3_6_27b_1p_gpqa.py +++ b/test/registered/npu/accuracy/qwen3_6_27b/test_npu_qwen3_6_27b_1p_gpqa.py @@ -9,10 +9,13 @@ from sglang.test.ascend.e2e.test_npu_performance_utils import ( from sglang.test.ci.ci_register import register_npu_ci register_npu_ci( - est_time=3600, - suite="", + # Measured ~45min/round in CI (2026-08-17 nightly-acc-2-npu-a3). + # est_time = 2x single-round so derive_timeout_per_file (1.5x est) + # reserves ~3 rounds of budget for accuracy retries. + # Normalized to the shared 6500 ladder value for ~42-54min cases. + est_time=6500, + suite="nightly-acc-2-npu-a3", nightly=True, - disabled="performance testcase", ) QWEN3_6_27B_64K_PREFIX_ENVS = { diff --git a/test/registered/npu/accuracy/qwen3_6_27b/test_npu_qwen3_6_27b_w8a8_1p_in3k5_out1k5_50ms_gpqa.py b/test/registered/npu/accuracy/qwen3_6_27b/test_npu_qwen3_6_27b_w8a8_1p_in3k5_out1k5_50ms_gpqa.py index 30bbc0d75..8ecdc892d 100644 --- a/test/registered/npu/accuracy/qwen3_6_27b/test_npu_qwen3_6_27b_w8a8_1p_in3k5_out1k5_50ms_gpqa.py +++ b/test/registered/npu/accuracy/qwen3_6_27b/test_npu_qwen3_6_27b_w8a8_1p_in3k5_out1k5_50ms_gpqa.py @@ -9,10 +9,9 @@ from sglang.test.ascend.e2e.test_npu_performance_utils import ( from sglang.test.ci.ci_register import register_npu_ci register_npu_ci( - est_time=3600, - suite="full-2-npu-a3", + est_time=8400, + suite="nightly-acc-2-npu-a3", nightly=True, - disabled="accuracy testcase", ) QWEN3_6_27B_3K5_1K5_ENVS = { diff --git a/test/registered/npu/accuracy/qwen3_6_35b_a3b/test_npu_qwen3_6_35b_a3b_1p_aime26.py b/test/registered/npu/accuracy/qwen3_6_35b_a3b/test_npu_qwen3_6_35b_a3b_1p_aime26.py index f2f42d355..a6cf52913 100644 --- a/test/registered/npu/accuracy/qwen3_6_35b_a3b/test_npu_qwen3_6_35b_a3b_1p_aime26.py +++ b/test/registered/npu/accuracy/qwen3_6_35b_a3b/test_npu_qwen3_6_35b_a3b_1p_aime26.py @@ -8,12 +8,7 @@ from sglang.test.ascend.e2e.test_npu_performance_utils import ( ) from sglang.test.ci.ci_register import register_npu_ci -register_npu_ci( - est_time=3600, - suite="full-2-npu-a3", - nightly=True, - disabled="performance testcase", -) +register_npu_ci(est_time=6500, suite="nightly-acc-2-npu-a3", nightly=True) QWEN3_6_35B_A3B_3K5_1K5_ENVS = { "PYTORCH_NPU_ALLOC_CONF": "expandable_segments:True", diff --git a/test/registered/npu/accuracy/qwen3_6_35b_a3b/test_npu_qwen3_6_35b_a3b_1p_in64k_out1k_prefix90_50ms_aime26.py b/test/registered/npu/accuracy/qwen3_6_35b_a3b/test_npu_qwen3_6_35b_a3b_1p_in64k_out1k_prefix90_50ms_aime26.py index 66befc0f3..ce81a32b0 100644 --- a/test/registered/npu/accuracy/qwen3_6_35b_a3b/test_npu_qwen3_6_35b_a3b_1p_in64k_out1k_prefix90_50ms_aime26.py +++ b/test/registered/npu/accuracy/qwen3_6_35b_a3b/test_npu_qwen3_6_35b_a3b_1p_in64k_out1k_prefix90_50ms_aime26.py @@ -9,10 +9,9 @@ from sglang.test.ascend.e2e.test_npu_performance_utils import ( from sglang.test.ci.ci_register import register_npu_ci register_npu_ci( - est_time=3600, - suite="", + est_time=2800, + suite="nightly-acc-2-npu-a3", nightly=True, - disabled="accuracy testcase", ) QWEN3_6_35B_A3B_64K_PREFIX_ENVS = { diff --git a/test/registered/npu/accuracy/qwen3_next_80b_a3b_instruct/test_npu_qwen3_next_80b_w8a8_2p_in6k_out1k5_bs16_aime25.py b/test/registered/npu/accuracy/qwen3_next_80b_a3b_instruct/test_npu_qwen3_next_80b_w8a8_2p_in6k_out1k5_bs16_aime25.py index c6e3aaea3..b69a2a499 100644 --- a/test/registered/npu/accuracy/qwen3_next_80b_a3b_instruct/test_npu_qwen3_next_80b_w8a8_2p_in6k_out1k5_bs16_aime25.py +++ b/test/registered/npu/accuracy/qwen3_next_80b_a3b_instruct/test_npu_qwen3_next_80b_w8a8_2p_in6k_out1k5_bs16_aime25.py @@ -10,10 +10,9 @@ from sglang.test.ascend.e2e.test_npu_performance_utils import ( from sglang.test.ci.ci_register import register_npu_ci register_npu_ci( - est_time=3600, - suite="", + est_time=4800, + suite="nightly-acc-4-npu-a3", nightly=True, - disabled="accuracy testcase", ) QWEN3_NEXT_80B_A3B_ENVS = { diff --git a/test/registered/npu/accuracy/qwen3_omni_30b_a3b_thinking/test_npu_qwen3_omni_30b_a3b_thinking_1p_mmmu.py b/test/registered/npu/accuracy/qwen3_omni_30b_a3b_thinking/test_npu_qwen3_omni_30b_a3b_thinking_1p_mmmu.py index 7bcf21f3b..7a473229d 100644 --- a/test/registered/npu/accuracy/qwen3_omni_30b_a3b_thinking/test_npu_qwen3_omni_30b_a3b_thinking_1p_mmmu.py +++ b/test/registered/npu/accuracy/qwen3_omni_30b_a3b_thinking/test_npu_qwen3_omni_30b_a3b_thinking_1p_mmmu.py @@ -9,7 +9,7 @@ from sglang.test.ascend.e2e.test_npu_performance_utils import ( from sglang.test.ci.ci_register import register_npu_ci register_npu_ci( - est_time=3600, + est_time=4800, suite="", nightly=True, disabled="performance testcase", diff --git a/test/registered/npu/accuracy/qwen3_vl_30b_a3b/test_npu_qwen3_vl_30b_a3b_bf16_2p_gsm8k.py b/test/registered/npu/accuracy/qwen3_vl_30b_a3b/test_npu_qwen3_vl_30b_a3b_bf16_2p_gsm8k.py index 0d9e06817..85d43f872 100644 --- a/test/registered/npu/accuracy/qwen3_vl_30b_a3b/test_npu_qwen3_vl_30b_a3b_bf16_2p_gsm8k.py +++ b/test/registered/npu/accuracy/qwen3_vl_30b_a3b/test_npu_qwen3_vl_30b_a3b_bf16_2p_gsm8k.py @@ -9,6 +9,7 @@ from sglang.test.ascend.e2e.test_npu_performance_utils import ( from sglang.test.ci.ci_register import register_npu_ci register_npu_ci(est_time=3600, suite="base-c-test-acc-4-npu-a3") +register_npu_ci(est_time=4800, suite="nightly-acc-4-npu-a3", nightly=True) QWEN3_VL_30B_A3B_ENVS = { "SGLANG_SET_CPU_AFFINITY": "1", diff --git a/test/registered/npu/accuracy/qwen3_vl_30b_a3b_thinking/test_npu_qwen3_vl_30b_a3b_thinking_1p_mmmu.py b/test/registered/npu/accuracy/qwen3_vl_30b_a3b_thinking/test_npu_qwen3_vl_30b_a3b_thinking_1p_mmmu.py index 7481faede..d6ee9b7f8 100644 --- a/test/registered/npu/accuracy/qwen3_vl_30b_a3b_thinking/test_npu_qwen3_vl_30b_a3b_thinking_1p_mmmu.py +++ b/test/registered/npu/accuracy/qwen3_vl_30b_a3b_thinking/test_npu_qwen3_vl_30b_a3b_thinking_1p_mmmu.py @@ -9,10 +9,9 @@ from sglang.test.ascend.e2e.test_npu_performance_utils import ( from sglang.test.ci.ci_register import register_npu_ci register_npu_ci( - est_time=3600, - suite="", + est_time=8400, + suite="nightly-acc-2-npu-a3", nightly=True, - disabled="performance testcase", ) ENVS = { diff --git a/test/registered/npu/accuracy/qwen3_vl_8b/test_npu_qwen3_vl_8b_bf16_2p_gsm8k.py b/test/registered/npu/accuracy/qwen3_vl_8b/test_npu_qwen3_vl_8b_bf16_2p_gsm8k.py index ffca0f1ed..dd1a4769f 100644 --- a/test/registered/npu/accuracy/qwen3_vl_8b/test_npu_qwen3_vl_8b_bf16_2p_gsm8k.py +++ b/test/registered/npu/accuracy/qwen3_vl_8b/test_npu_qwen3_vl_8b_bf16_2p_gsm8k.py @@ -9,6 +9,7 @@ from sglang.test.ascend.e2e.test_npu_performance_utils import ( from sglang.test.ci.ci_register import register_npu_ci register_npu_ci(est_time=3600, suite="base-c-test-acc-4-npu-a3") +register_npu_ci(est_time=4800, suite="nightly-acc-4-npu-a3", nightly=True) QWEN3_VL_8B_ENVS = { "SGLANG_SET_CPU_AFFINITY": "1", diff --git a/test/registered/npu/accuracy/qwen3_vl_8b_thinking/test_npu_qwen3_vl_8b_thinking_1p_mmmu.py b/test/registered/npu/accuracy/qwen3_vl_8b_thinking/test_npu_qwen3_vl_8b_thinking_1p_mmmu.py index 1cfefbc1f..8c5aa85d3 100644 --- a/test/registered/npu/accuracy/qwen3_vl_8b_thinking/test_npu_qwen3_vl_8b_thinking_1p_mmmu.py +++ b/test/registered/npu/accuracy/qwen3_vl_8b_thinking/test_npu_qwen3_vl_8b_thinking_1p_mmmu.py @@ -10,6 +10,7 @@ from sglang.test.ascend.e2e.test_npu_performance_utils import ( from sglang.test.ci.ci_register import register_npu_ci register_npu_ci(est_time=3600, suite="base-c-test-perf-2-npu-a3") +register_npu_ci(est_time=6500, suite="nightly-acc-2-npu-a3", nightly=True) _is_pr_pipeline = os.environ.get("GITHUB_EVENT_NAME") == "pull_request" diff --git a/test/registered/npu/basic_function/HiCache/test_npu_hierarchical_cache.py b/test/registered/npu/basic_function/HiCache/test_npu_hierarchical_cache.py index 93a0a25e8..011240353 100644 --- a/test/registered/npu/basic_function/HiCache/test_npu_hierarchical_cache.py +++ b/test/registered/npu/basic_function/HiCache/test_npu_hierarchical_cache.py @@ -12,7 +12,7 @@ from sglang.test.test_utils import ( popen_launch_server, ) -register_npu_ci(est_time=400, suite="nightly-1-npu-a3", nightly=True) +register_npu_ci(est_time=400, suite="full-1-npu-a3", nightly=True) class TestNPUHierarchicalCache(CustomTestCase): diff --git a/test/registered/npu/basic_function/HiCache/test_npu_hierarchical_cache_mla.py b/test/registered/npu/basic_function/HiCache/test_npu_hierarchical_cache_mla.py index 8e38fafb6..1553cb34b 100644 --- a/test/registered/npu/basic_function/HiCache/test_npu_hierarchical_cache_mla.py +++ b/test/registered/npu/basic_function/HiCache/test_npu_hierarchical_cache_mla.py @@ -10,7 +10,7 @@ from sglang.test.test_utils import CustomTestCase register_npu_ci( est_time=400, - suite="nightly-16-npu-a3", + suite="full-16-npu-a3", nightly=True, ) diff --git a/test/registered/npu/basic_function/HiCache/test_npu_hierarchical_cache_mutually_exclusive.py b/test/registered/npu/basic_function/HiCache/test_npu_hierarchical_cache_mutually_exclusive.py index 1f29db1ac..fc3cf62c0 100644 --- a/test/registered/npu/basic_function/HiCache/test_npu_hierarchical_cache_mutually_exclusive.py +++ b/test/registered/npu/basic_function/HiCache/test_npu_hierarchical_cache_mutually_exclusive.py @@ -12,7 +12,7 @@ from sglang.test.test_utils import ( register_npu_ci( est_time=400, - suite="nightly-1-npu-a3", + suite="full-1-npu-a3", nightly=True, ) diff --git a/test/registered/npu/basic_function/HiCache/test_npu_hierarchical_cache_ttft_mha.py b/test/registered/npu/basic_function/HiCache/test_npu_hierarchical_cache_ttft_mha.py index 0189eae1a..57493b5c1 100644 --- a/test/registered/npu/basic_function/HiCache/test_npu_hierarchical_cache_ttft_mha.py +++ b/test/registered/npu/basic_function/HiCache/test_npu_hierarchical_cache_ttft_mha.py @@ -9,7 +9,7 @@ from sglang.test.test_utils import CustomTestCase register_npu_ci( est_time=400, - suite="nightly-2-npu-a3", + suite="full-2-npu-a3", nightly=True, ) diff --git a/test/registered/npu/basic_function/HiCache/test_npu_radix_cache.py b/test/registered/npu/basic_function/HiCache/test_npu_radix_cache.py index ff5fc4e5e..aa99bc4f5 100644 --- a/test/registered/npu/basic_function/HiCache/test_npu_radix_cache.py +++ b/test/registered/npu/basic_function/HiCache/test_npu_radix_cache.py @@ -12,7 +12,7 @@ from sglang.test.test_utils import ( popen_launch_server, ) -register_npu_ci(est_time=400, suite="nightly-1-npu-a3", nightly=True) +register_npu_ci(est_time=400, suite="full-1-npu-a3", nightly=True) class TestNPURadixCache(CustomTestCase): diff --git a/test/registered/npu/basic_function/dllm/test_npu_llada2_mini.py b/test/registered/npu/basic_function/dllm/test_npu_llada2_mini.py index dc55d5565..e34ee4b93 100644 --- a/test/registered/npu/basic_function/dllm/test_npu_llada2_mini.py +++ b/test/registered/npu/basic_function/dllm/test_npu_llada2_mini.py @@ -11,7 +11,7 @@ from sglang.test.test_utils import ( ) register_npu_ci(est_time=400, suite="base-b-test-4-npu-a3") -register_npu_ci(est_time=400, suite="nightly-1-npu-a3", nightly=True) +register_npu_ci(est_time=400, suite="nightly-4-npu-a3", nightly=True) class TestLLaDA2Mini(GSM8KAscendMixin, CustomTestCase): diff --git a/test/registered/npu/basic_function/memory_and_scheduling/test_npu_no_chunked_prefill.py b/test/registered/npu/basic_function/memory_and_scheduling/test_npu_no_chunked_prefill.py index 3127674da..f5d28eae5 100644 --- a/test/registered/npu/basic_function/memory_and_scheduling/test_npu_no_chunked_prefill.py +++ b/test/registered/npu/basic_function/memory_and_scheduling/test_npu_no_chunked_prefill.py @@ -13,7 +13,7 @@ from sglang.test.test_utils import ( run_bench_serving, ) -register_npu_ci(est_time=400, suite="nightly-1-npu-a3", nightly=True) +register_npu_ci(est_time=400, suite="full-1-npu-a3", nightly=True) class TestNoChunkedPrefill(CustomTestCase): diff --git a/test/registered/npu/basic_function/offloading/test_npu_offload_modes.py b/test/registered/npu/basic_function/offloading/test_npu_offload_modes.py index f69e801af..fd29cdb57 100644 --- a/test/registered/npu/basic_function/offloading/test_npu_offload_modes.py +++ b/test/registered/npu/basic_function/offloading/test_npu_offload_modes.py @@ -13,7 +13,7 @@ from sglang.test.test_utils import ( popen_launch_server, ) -register_npu_ci(est_time=800, suite="nightly-2-npu-a3", nightly=True) +register_npu_ci(est_time=800, suite="full-2-npu-a3", nightly=True) TEST_MODEL_MATRIX = { DEEPSEEK_CODER_V2_LITE_WEIGHTS_PATH, diff --git a/test/registered/npu/basic_function/parallel_strategy/expert_parallelism/test_npu_deepep_auto_deepseek_v3_2_w8a8.py b/test/registered/npu/basic_function/parallel_strategy/expert_parallelism/test_npu_deepep_auto_deepseek_v3_2_w8a8.py index a09963d70..2952a766c 100644 --- a/test/registered/npu/basic_function/parallel_strategy/expert_parallelism/test_npu_deepep_auto_deepseek_v3_2_w8a8.py +++ b/test/registered/npu/basic_function/parallel_strategy/expert_parallelism/test_npu_deepep_auto_deepseek_v3_2_w8a8.py @@ -7,7 +7,7 @@ from sglang.test.ascend.test_mmlu import TestMMLU from sglang.test.ci.ci_register import register_npu_ci from sglang.test.test_utils import CustomTestCase -register_npu_ci(est_time=400, suite="nightly-16-npu-a3", nightly=True) +register_npu_ci(est_time=400, suite="full-16-npu-a3", nightly=True) class TestDeepEpDeepseekV32(GSM8KAscendMixin, TestMMLU, CustomTestCase): diff --git a/test/registered/npu/basic_function/parallel_strategy/expert_parallelism/test_npu_deepep_auto_qwen3_480b.py b/test/registered/npu/basic_function/parallel_strategy/expert_parallelism/test_npu_deepep_auto_qwen3_480b.py index 166641984..d7adbac6e 100644 --- a/test/registered/npu/basic_function/parallel_strategy/expert_parallelism/test_npu_deepep_auto_qwen3_480b.py +++ b/test/registered/npu/basic_function/parallel_strategy/expert_parallelism/test_npu_deepep_auto_qwen3_480b.py @@ -9,7 +9,7 @@ from sglang.test.ascend.test_mmlu import TestMMLU from sglang.test.ci.ci_register import register_npu_ci from sglang.test.test_utils import CustomTestCase -register_npu_ci(est_time=200, suite="nightly-16-npu-a3", nightly=True) +register_npu_ci(est_time=200, suite="full-16-npu-a3", nightly=True) class TestDeepEpQwen(GSM8KAscendMixin, TestMMLU, CustomTestCase): diff --git a/test/registered/npu/basic_function/parallel_strategy/expert_parallelism/test_npu_deepep_auto_qwen3_next.py b/test/registered/npu/basic_function/parallel_strategy/expert_parallelism/test_npu_deepep_auto_qwen3_next.py index 8c3db8099..76f3870e5 100644 --- a/test/registered/npu/basic_function/parallel_strategy/expert_parallelism/test_npu_deepep_auto_qwen3_next.py +++ b/test/registered/npu/basic_function/parallel_strategy/expert_parallelism/test_npu_deepep_auto_qwen3_next.py @@ -11,7 +11,7 @@ from sglang.test.test_utils import CustomTestCase register_npu_ci( est_time=200, - suite="nightly-8-npu-a3", + suite="full-8-npu-a3", nightly=True, ) diff --git a/test/registered/npu/basic_function/parallel_strategy/expert_parallelism/test_npu_deepep_low_latency_deepseek_v3_2_w8a8.py b/test/registered/npu/basic_function/parallel_strategy/expert_parallelism/test_npu_deepep_low_latency_deepseek_v3_2_w8a8.py index c7e2563e3..6d5607ab2 100644 --- a/test/registered/npu/basic_function/parallel_strategy/expert_parallelism/test_npu_deepep_low_latency_deepseek_v3_2_w8a8.py +++ b/test/registered/npu/basic_function/parallel_strategy/expert_parallelism/test_npu_deepep_low_latency_deepseek_v3_2_w8a8.py @@ -7,7 +7,7 @@ from sglang.test.ascend.test_mmlu import TestMMLU from sglang.test.ci.ci_register import register_npu_ci from sglang.test.test_utils import CustomTestCase -register_npu_ci(est_time=200, suite="nightly-16-npu-a3", nightly=True) +register_npu_ci(est_time=200, suite="full-16-npu-a3", nightly=True) class TestDeepEpDeepseekV32(GSM8KAscendMixin, TestMMLU, CustomTestCase): diff --git a/test/registered/npu/basic_function/parallel_strategy/expert_parallelism/test_npu_deepep_low_latency_qwen3_480b.py b/test/registered/npu/basic_function/parallel_strategy/expert_parallelism/test_npu_deepep_low_latency_qwen3_480b.py index 54ababf38..bdfcd4f03 100644 --- a/test/registered/npu/basic_function/parallel_strategy/expert_parallelism/test_npu_deepep_low_latency_qwen3_480b.py +++ b/test/registered/npu/basic_function/parallel_strategy/expert_parallelism/test_npu_deepep_low_latency_qwen3_480b.py @@ -9,7 +9,7 @@ from sglang.test.ascend.test_mmlu import TestMMLU from sglang.test.ci.ci_register import register_npu_ci from sglang.test.test_utils import CustomTestCase -register_npu_ci(est_time=200, suite="nightly-16-npu-a3", nightly=True) +register_npu_ci(est_time=200, suite="full-16-npu-a3", nightly=True) class TestDeepEpQwen(GSM8KAscendMixin, TestMMLU, CustomTestCase): diff --git a/test/registered/npu/basic_function/parallel_strategy/expert_parallelism/test_npu_deepep_low_latency_qwen3_next.py b/test/registered/npu/basic_function/parallel_strategy/expert_parallelism/test_npu_deepep_low_latency_qwen3_next.py index e1b804c1e..51f905a04 100644 --- a/test/registered/npu/basic_function/parallel_strategy/expert_parallelism/test_npu_deepep_low_latency_qwen3_next.py +++ b/test/registered/npu/basic_function/parallel_strategy/expert_parallelism/test_npu_deepep_low_latency_qwen3_next.py @@ -11,7 +11,7 @@ from sglang.test.test_utils import CustomTestCase register_npu_ci( est_time=200, - suite="nightly-8-npu-a3", + suite="full-8-npu-a3", nightly=True, ) diff --git a/test/registered/npu/basic_function/parameter/test_npu_fim_completion.py b/test/registered/npu/basic_function/parameter/test_npu_fim_completion.py index 1bd45a68d..c335320a2 100644 --- a/test/registered/npu/basic_function/parameter/test_npu_fim_completion.py +++ b/test/registered/npu/basic_function/parameter/test_npu_fim_completion.py @@ -18,7 +18,7 @@ from sglang.test.test_utils import ( register_npu_ci( est_time=400, - suite="nightly-1-npu-a3", + suite="full-1-npu-a3", nightly=True, ) diff --git a/test/registered/npu/basic_function/parameter/test_npu_log_level.py b/test/registered/npu/basic_function/parameter/test_npu_log_level.py index 63e49bcf8..d99d8da0c 100644 --- a/test/registered/npu/basic_function/parameter/test_npu_log_level.py +++ b/test/registered/npu/basic_function/parameter/test_npu_log_level.py @@ -13,7 +13,7 @@ from sglang.test.test_utils import ( popen_launch_server, ) -register_npu_ci(est_time=400, suite="nightly-1-npu-a3", nightly=True) +register_npu_ci(est_time=400, suite="full-1-npu-a3", nightly=True) class TestLogLevel(CustomTestCase): diff --git a/test/registered/npu/basic_function/parameter/test_npu_no_overlap_scheduler.py b/test/registered/npu/basic_function/parameter/test_npu_no_overlap_scheduler.py index 27c46a5ef..84a8ea744 100644 --- a/test/registered/npu/basic_function/parameter/test_npu_no_overlap_scheduler.py +++ b/test/registered/npu/basic_function/parameter/test_npu_no_overlap_scheduler.py @@ -5,7 +5,7 @@ from sglang.test.test_utils import CustomTestCase, run_mmlu_test register_npu_ci( est_time=400, - suite="nightly-1-npu-a3", + suite="full-1-npu-a3", nightly=True, ) diff --git a/test/registered/npu/basic_function/parameter/test_npu_original_logprobs.py b/test/registered/npu/basic_function/parameter/test_npu_original_logprobs.py index e60b88909..375751bde 100644 --- a/test/registered/npu/basic_function/parameter/test_npu_original_logprobs.py +++ b/test/registered/npu/basic_function/parameter/test_npu_original_logprobs.py @@ -46,7 +46,7 @@ if torch.cuda.is_available(): torch.backends.cuda.matmul.allow_tf32 = False torch.backends.cudnn.allow_tf32 = False -register_npu_ci(est_time=400, suite="nightly-1-npu-a3", nightly=True) +register_npu_ci(est_time=400, suite="full-1-npu-a3", nightly=True) class TestOriginalLogprob(unittest.TestCase): diff --git a/test/registered/npu/basic_function/parameter/test_npu_warmups.py b/test/registered/npu/basic_function/parameter/test_npu_warmups.py index d6d7fa3fe..0852ec24d 100644 --- a/test/registered/npu/basic_function/parameter/test_npu_warmups.py +++ b/test/registered/npu/basic_function/parameter/test_npu_warmups.py @@ -14,7 +14,7 @@ from sglang.test.test_utils import ( register_npu_ci( est_time=400, - suite="nightly-4-npu-a3", + suite="full-4-npu-a3", nightly=True, ) diff --git a/test/registered/npu/basic_function/quant/test_npu_gguf.py b/test/registered/npu/basic_function/quant/test_npu_gguf.py index c02412e01..a81d367f6 100644 --- a/test/registered/npu/basic_function/quant/test_npu_gguf.py +++ b/test/registered/npu/basic_function/quant/test_npu_gguf.py @@ -14,7 +14,7 @@ from sglang.test.test_utils import ( popen_launch_server, ) -register_npu_ci(est_time=300, suite="nightly-1-npu-a3", nightly=True) +register_npu_ci(est_time=300, suite="full-1-npu-a3", nightly=True) logger = logging.getLogger(__name__) diff --git a/test/registered/npu/basic_function/quant/test_npu_gguf_moe.py b/test/registered/npu/basic_function/quant/test_npu_gguf_moe.py index 25f6a0d97..9b7ffa31c 100644 --- a/test/registered/npu/basic_function/quant/test_npu_gguf_moe.py +++ b/test/registered/npu/basic_function/quant/test_npu_gguf_moe.py @@ -16,7 +16,7 @@ from sglang.test.test_utils import ( popen_launch_server, ) -register_npu_ci(est_time=500, suite="nightly-2-npu-a3", nightly=True) +register_npu_ci(est_time=500, suite="full-2-npu-a3", nightly=True) logger = logging.getLogger(__name__) diff --git a/test/registered/npu/interface/test_npu_api_abort_request.py b/test/registered/npu/interface/test_npu_api_abort_request.py index 4a5baaef7..7e62c4d6d 100644 --- a/test/registered/npu/interface/test_npu_api_abort_request.py +++ b/test/registered/npu/interface/test_npu_api_abort_request.py @@ -22,7 +22,7 @@ def send_requests(url, **kwargs): responses.append(response) -register_npu_ci(est_time=400, suite="nightly-1-npu-a3", nightly=True) +register_npu_ci(est_time=400, suite="full-1-npu-a3", nightly=True) class TestNpuApi(CustomTestCase): diff --git a/test/registered/npu/interface/test_npu_api_encode.py b/test/registered/npu/interface/test_npu_api_encode.py index 800d774c8..0723d0174 100644 --- a/test/registered/npu/interface/test_npu_api_encode.py +++ b/test/registered/npu/interface/test_npu_api_encode.py @@ -20,7 +20,7 @@ logging.basicConfig( ) logger = logging.getLogger(__name__) -register_npu_ci(est_time=400, suite="nightly-1-npu-a3", nightly=True) +register_npu_ci(est_time=400, suite="full-1-npu-a3", nightly=True) class TestNpuApi(CustomTestCase): diff --git a/test/registered/npu/interface/test_npu_enable_thinking.py b/test/registered/npu/interface/test_npu_enable_thinking.py index f2db5b567..edb61820e 100644 --- a/test/registered/npu/interface/test_npu_enable_thinking.py +++ b/test/registered/npu/interface/test_npu_enable_thinking.py @@ -15,7 +15,7 @@ from sglang.test.test_utils import ( register_npu_ci( est_time=400, - suite="nightly-2-npu-a3", + suite="full-2-npu-a3", nightly=True, ) diff --git a/test/registered/npu/interface/test_npu_matched_stop.py b/test/registered/npu/interface/test_npu_matched_stop.py index 5dcf40ff5..d679290cf 100644 --- a/test/registered/npu/interface/test_npu_matched_stop.py +++ b/test/registered/npu/interface/test_npu_matched_stop.py @@ -19,7 +19,7 @@ Each section should be as comprehensive as possible to create a rich and immersi The story should span multiple events, challenges, and character developments over time. Aim to make the story at least 3,000 words long. """ -register_npu_ci(est_time=400, suite="nightly-1-npu-a3", nightly=True) +register_npu_ci(est_time=400, suite="full-1-npu-a3", nightly=True) class TestMatchedStop(CustomTestCase): diff --git a/test/registered/npu/interface/test_npu_openai_function_calling.py b/test/registered/npu/interface/test_npu_openai_function_calling.py index fbecc981f..daea9486b 100644 --- a/test/registered/npu/interface/test_npu_openai_function_calling.py +++ b/test/registered/npu/interface/test_npu_openai_function_calling.py @@ -16,7 +16,7 @@ from sglang.test.test_utils import ( register_npu_ci( est_time=400, - suite="nightly-1-npu-a3", + suite="full-1-npu-a3", nightly=True, ) diff --git a/test/registered/npu/interface/test_npu_openai_server_ignore_eos.py b/test/registered/npu/interface/test_npu_openai_server_ignore_eos.py index 0fee4ecac..9ae4aa388 100644 --- a/test/registered/npu/interface/test_npu_openai_server_ignore_eos.py +++ b/test/registered/npu/interface/test_npu_openai_server_ignore_eos.py @@ -13,7 +13,7 @@ from sglang.test.test_utils import ( popen_launch_server, ) -register_npu_ci(est_time=400, suite="nightly-2-npu-a3", nightly=True) +register_npu_ci(est_time=400, suite="full-2-npu-a3", nightly=True) class TestOpenAIServerIgnoreEOS(CustomTestCase): diff --git a/test/registered/npu/interface/test_npu_penalty.py b/test/registered/npu/interface/test_npu_penalty.py index f8c3f9dec..0b0720cef 100644 --- a/test/registered/npu/interface/test_npu_penalty.py +++ b/test/registered/npu/interface/test_npu_penalty.py @@ -15,7 +15,7 @@ from sglang.test.test_utils import ( popen_launch_server, ) -register_npu_ci(est_time=400, suite="nightly-1-npu-a3", nightly=True) +register_npu_ci(est_time=400, suite="full-1-npu-a3", nightly=True) class TestPenalty(CustomTestCase): diff --git a/test/registered/npu/llm_models/test_ascend_dbrx_instruct.py b/test/registered/npu/llm_models/test_ascend_dbrx_instruct.py index 189e9fe90..3ae7bb19f 100644 --- a/test/registered/npu/llm_models/test_ascend_dbrx_instruct.py +++ b/test/registered/npu/llm_models/test_ascend_dbrx_instruct.py @@ -4,7 +4,7 @@ from sglang.test.ascend.gsm8k_ascend_mixin import GSM8KAscendMixin from sglang.test.ci.ci_register import register_npu_ci from sglang.test.test_utils import CustomTestCase -register_npu_ci(est_time=400, suite="nightly-8-npu-a3", nightly=True) +register_npu_ci(est_time=400, suite="full-8-npu-a3", nightly=True) class TestDbrx(GSM8KAscendMixin, CustomTestCase): diff --git a/test/registered/npu/llm_models/test_ascend_minimax_m2.py b/test/registered/npu/llm_models/test_ascend_minimax_m2.py index 6f9db9508..4160c140c 100644 --- a/test/registered/npu/llm_models/test_ascend_minimax_m2.py +++ b/test/registered/npu/llm_models/test_ascend_minimax_m2.py @@ -7,7 +7,7 @@ from sglang.test.test_utils import CustomTestCase register_npu_ci( est_time=400, - suite="nightly-8-npu-a3", + suite="full-8-npu-a3", nightly=True, ) diff --git a/test/registered/npu/llm_models/test_ascend_trinity_mini.py b/test/registered/npu/llm_models/test_ascend_trinity_mini.py index a8fb1d316..00cdb9b6c 100644 --- a/test/registered/npu/llm_models/test_ascend_trinity_mini.py +++ b/test/registered/npu/llm_models/test_ascend_trinity_mini.py @@ -7,7 +7,7 @@ from sglang.test.test_utils import CustomTestCase register_npu_ci( est_time=400, - suite="nightly-2-npu-a3", + suite="full-2-npu-a3", nightly=True, ) diff --git a/test/registered/npu/llm_models/test_npu_baichuan2_13b_chat.py b/test/registered/npu/llm_models/test_npu_baichuan2_13b_chat.py index 34c7732f2..19cc857d0 100644 --- a/test/registered/npu/llm_models/test_npu_baichuan2_13b_chat.py +++ b/test/registered/npu/llm_models/test_npu_baichuan2_13b_chat.py @@ -5,7 +5,7 @@ from sglang.test.ascend.test_ascend_utils import BAICHUAN2_13B_CHAT_WEIGHTS_PATH from sglang.test.ci.ci_register import register_npu_ci from sglang.test.test_utils import CustomTestCase -register_npu_ci(est_time=400, suite="nightly-1-npu-a3", nightly=True) +register_npu_ci(est_time=400, suite="full-1-npu-a3", nightly=True) class TestBaichuan(GSM8KAscendMixin, CustomTestCase): diff --git a/test/registered/npu/llm_models/test_npu_chatglm2_6b.py b/test/registered/npu/llm_models/test_npu_chatglm2_6b.py index 9681219bf..92a430db3 100644 --- a/test/registered/npu/llm_models/test_npu_chatglm2_6b.py +++ b/test/registered/npu/llm_models/test_npu_chatglm2_6b.py @@ -5,7 +5,7 @@ from sglang.test.ascend.test_ascend_utils import CHATGLM2_6B_WEIGHTS_PATH from sglang.test.ci.ci_register import register_npu_ci from sglang.test.test_utils import CustomTestCase -register_npu_ci(est_time=400, suite="nightly-1-npu-a3", nightly=True) +register_npu_ci(est_time=400, suite="full-1-npu-a3", nightly=True) class TestChatGlm2(GSM8KAscendMixin, CustomTestCase): diff --git a/test/registered/npu/llm_models/test_npu_deepseek_v3_2_exp_w8a8.py b/test/registered/npu/llm_models/test_npu_deepseek_v3_2_exp_w8a8.py index 7cf589196..aa2d54675 100644 --- a/test/registered/npu/llm_models/test_npu_deepseek_v3_2_exp_w8a8.py +++ b/test/registered/npu/llm_models/test_npu_deepseek_v3_2_exp_w8a8.py @@ -5,7 +5,7 @@ from sglang.test.ascend.test_ascend_utils import DEEPSEEK_V3_2_EXP_W8A8_WEIGHTS_ from sglang.test.ci.ci_register import register_npu_ci from sglang.test.test_utils import CustomTestCase -register_npu_ci(est_time=400, suite="nightly-16-npu-a3", nightly=True) +register_npu_ci(est_time=400, suite="full-16-npu-a3", nightly=True) class TestDeepSeekV32(GSM8KAscendMixin, CustomTestCase): diff --git a/test/registered/npu/llm_models/test_npu_gemma_3_4b_it_llm.py b/test/registered/npu/llm_models/test_npu_gemma_3_4b_it_llm.py index 293deb22f..04debcfb2 100644 --- a/test/registered/npu/llm_models/test_npu_gemma_3_4b_it_llm.py +++ b/test/registered/npu/llm_models/test_npu_gemma_3_4b_it_llm.py @@ -7,7 +7,7 @@ from sglang.test.test_utils import CustomTestCase register_npu_ci( est_time=400, - suite="nightly-1-npu-a3", + suite="full-1-npu-a3", nightly=True, ) diff --git a/test/registered/npu/llm_models/test_npu_glm4_9b_chat.py b/test/registered/npu/llm_models/test_npu_glm4_9b_chat.py index e19eb3105..4864ca0ff 100644 --- a/test/registered/npu/llm_models/test_npu_glm4_9b_chat.py +++ b/test/registered/npu/llm_models/test_npu_glm4_9b_chat.py @@ -7,7 +7,7 @@ from sglang.test.test_utils import CustomTestCase register_npu_ci( est_time=400, - suite="nightly-1-npu-a3", + suite="full-1-npu-a3", nightly=True, ) diff --git a/test/registered/npu/llm_models/test_npu_internlm2_7b.py b/test/registered/npu/llm_models/test_npu_internlm2_7b.py index 0f319cfad..3812e5234 100644 --- a/test/registered/npu/llm_models/test_npu_internlm2_7b.py +++ b/test/registered/npu/llm_models/test_npu_internlm2_7b.py @@ -8,7 +8,7 @@ from sglang.test.test_utils import CustomTestCase register_npu_ci( est_time=400, - suite="nightly-1-npu-a3", + suite="full-1-npu-a3", nightly=True, ) diff --git a/test/registered/npu/llm_models/test_npu_llama4_scount_17b_16e.py b/test/registered/npu/llm_models/test_npu_llama4_scount_17b_16e.py index d7a40d6a1..c7dca88b3 100644 --- a/test/registered/npu/llm_models/test_npu_llama4_scount_17b_16e.py +++ b/test/registered/npu/llm_models/test_npu_llama4_scount_17b_16e.py @@ -7,7 +7,7 @@ from sglang.test.ascend.test_ascend_utils import ( from sglang.test.ci.ci_register import register_npu_ci from sglang.test.test_utils import CustomTestCase -register_npu_ci(est_time=400, suite="nightly-4-npu-a3", nightly=True) +register_npu_ci(est_time=400, suite="full-4-npu-a3", nightly=True) class TestLlama4(GSM8KAscendMixin, CustomTestCase): diff --git a/test/registered/npu/llm_models/test_npu_llama_2_7b.py b/test/registered/npu/llm_models/test_npu_llama_2_7b.py index 1391d5482..579aea704 100644 --- a/test/registered/npu/llm_models/test_npu_llama_2_7b.py +++ b/test/registered/npu/llm_models/test_npu_llama_2_7b.py @@ -5,7 +5,7 @@ from sglang.test.ascend.test_ascend_utils import LLAMA_2_7B_WEIGHTS_PATH from sglang.test.ci.ci_register import register_npu_ci from sglang.test.test_utils import CustomTestCase -register_npu_ci(est_time=400, suite="nightly-1-npu-a3", nightly=True) +register_npu_ci(est_time=400, suite="full-1-npu-a3", nightly=True) class TestLlama(GSM8KAscendMixin, CustomTestCase): diff --git a/test/registered/npu/llm_models/test_npu_llama_2_7b_communications_compression.py b/test/registered/npu/llm_models/test_npu_llama_2_7b_communications_compression.py index e7b7fe361..0c8374d54 100644 --- a/test/registered/npu/llm_models/test_npu_llama_2_7b_communications_compression.py +++ b/test/registered/npu/llm_models/test_npu_llama_2_7b_communications_compression.py @@ -5,7 +5,7 @@ from sglang.test.ascend.test_ascend_utils import LLAMA_2_7B_WEIGHTS_PATH from sglang.test.ci.ci_register import register_npu_ci from sglang.test.test_utils import CustomTestCase -register_npu_ci(est_time=400, suite="nightly-2-npu-a3", nightly=True) +register_npu_ci(est_time=400, suite="full-2-npu-a3", nightly=True) class TestLlama(GSM8KAscendMixin, CustomTestCase): diff --git a/test/registered/npu/llm_models/test_npu_minicpm3_4b.py b/test/registered/npu/llm_models/test_npu_minicpm3_4b.py index 21b00a651..362bbb9b3 100644 --- a/test/registered/npu/llm_models/test_npu_minicpm3_4b.py +++ b/test/registered/npu/llm_models/test_npu_minicpm3_4b.py @@ -5,7 +5,7 @@ from sglang.test.ascend.test_ascend_utils import MINICPM3_4B_WEIGHTS_PATH from sglang.test.ci.ci_register import register_npu_ci from sglang.test.test_utils import CustomTestCase -register_npu_ci(est_time=400, suite="nightly-1-npu-a3", nightly=True) +register_npu_ci(est_time=400, suite="full-1-npu-a3", nightly=True) class TestMiniCPM3(GSM8KAscendMixin, CustomTestCase): diff --git a/test/registered/npu/llm_models/test_npu_mistral_7b.py b/test/registered/npu/llm_models/test_npu_mistral_7b.py index f13790134..5c5e41b0a 100644 --- a/test/registered/npu/llm_models/test_npu_mistral_7b.py +++ b/test/registered/npu/llm_models/test_npu_mistral_7b.py @@ -5,7 +5,7 @@ from sglang.test.ascend.test_ascend_utils import MISTRAL_7B_INSTRUCT_V0_2_WEIGHT from sglang.test.ci.ci_register import register_npu_ci from sglang.test.test_utils import CustomTestCase -register_npu_ci(est_time=400, suite="nightly-1-npu-a3", nightly=True) +register_npu_ci(est_time=400, suite="full-1-npu-a3", nightly=True) class TestMistral7B(GSM8KAscendMixin, CustomTestCase): diff --git a/test/registered/npu/llm_models/test_npu_phi_4_multimodal_llm.py b/test/registered/npu/llm_models/test_npu_phi_4_multimodal_llm.py index 19b1e327d..7fcafaee9 100644 --- a/test/registered/npu/llm_models/test_npu_phi_4_multimodal_llm.py +++ b/test/registered/npu/llm_models/test_npu_phi_4_multimodal_llm.py @@ -5,7 +5,7 @@ from sglang.test.ascend.test_ascend_utils import PHI_4_MULTIMODAL_INSTRUCT_WEIGH from sglang.test.ci.ci_register import register_npu_ci from sglang.test.test_utils import CustomTestCase -register_npu_ci(est_time=400, suite="nightly-1-npu-a3", nightly=True) +register_npu_ci(est_time=400, suite="full-1-npu-a3", nightly=True) class TestPhi4(GSM8KAscendMixin, CustomTestCase): diff --git a/test/registered/npu/llm_models/test_npu_qwen3_0_6b.py b/test/registered/npu/llm_models/test_npu_qwen3_0_6b.py index a1ba1cb5e..49805c0c5 100644 --- a/test/registered/npu/llm_models/test_npu_qwen3_0_6b.py +++ b/test/registered/npu/llm_models/test_npu_qwen3_0_6b.py @@ -5,7 +5,7 @@ from sglang.test.ascend.test_ascend_utils import QWEN3_0_6B_WEIGHTS_PATH from sglang.test.ci.ci_register import register_npu_ci from sglang.test.test_utils import CustomTestCase -register_npu_ci(est_time=400, suite="nightly-1-npu-a3", nightly=True) +register_npu_ci(est_time=400, suite="full-1-npu-a3", nightly=True) class TestQwen306B(GSM8KAscendMixin, CustomTestCase): diff --git a/test/registered/npu/llm_models/test_npu_qwen3_235b_a22b_w8a8.py b/test/registered/npu/llm_models/test_npu_qwen3_235b_a22b_w8a8.py index 944a0bc7c..43b1d1088 100644 --- a/test/registered/npu/llm_models/test_npu_qwen3_235b_a22b_w8a8.py +++ b/test/registered/npu/llm_models/test_npu_qwen3_235b_a22b_w8a8.py @@ -5,7 +5,7 @@ from sglang.test.ascend.test_ascend_utils import QWEN3_235B_A22B_W8A8_WEIGHTS_PA from sglang.test.ci.ci_register import register_npu_ci from sglang.test.test_utils import CustomTestCase -register_npu_ci(est_time=400, suite="nightly-8-npu-a3", nightly=True) +register_npu_ci(est_time=400, suite="full-8-npu-a3", nightly=True) class TestQwen3235BA22BW8A8(GSM8KAscendMixin, CustomTestCase): diff --git a/test/registered/npu/llm_models/test_npu_qwen3_30b.py b/test/registered/npu/llm_models/test_npu_qwen3_30b.py index 3204aab07..cdda7659a 100644 --- a/test/registered/npu/llm_models/test_npu_qwen3_30b.py +++ b/test/registered/npu/llm_models/test_npu_qwen3_30b.py @@ -7,7 +7,7 @@ from sglang.test.ascend.test_ascend_utils import ( from sglang.test.ci.ci_register import register_npu_ci from sglang.test.test_utils import CustomTestCase -register_npu_ci(est_time=400, suite="nightly-2-npu-a3", nightly=True) +register_npu_ci(est_time=400, suite="full-2-npu-a3", nightly=True) class TestQwen330B(GSM8KAscendMixin, CustomTestCase): diff --git a/test/registered/npu/llm_models/test_npu_qwen3_30b_attn_cp.py b/test/registered/npu/llm_models/test_npu_qwen3_30b_attn_cp.py index 201b85b50..b16b0d174 100644 --- a/test/registered/npu/llm_models/test_npu_qwen3_30b_attn_cp.py +++ b/test/registered/npu/llm_models/test_npu_qwen3_30b_attn_cp.py @@ -6,7 +6,7 @@ from sglang.test.ascend.test_ascend_utils import QWEN3_30B_A3B_WEIGHTS_PATH from sglang.test.ci.ci_register import register_npu_ci from sglang.test.test_utils import CustomTestCase -register_npu_ci(est_time=500, suite="nightly-4-npu-a3", nightly=True) +register_npu_ci(est_time=500, suite="full-4-npu-a3", nightly=True) class TestQwen330BAttnCP(GSM8KAscendMixin, CustomTestCase): diff --git a/test/registered/npu/llm_models/test_npu_qwen3_32b.py b/test/registered/npu/llm_models/test_npu_qwen3_32b.py index b6d638d92..08e638f44 100644 --- a/test/registered/npu/llm_models/test_npu_qwen3_32b.py +++ b/test/registered/npu/llm_models/test_npu_qwen3_32b.py @@ -7,7 +7,7 @@ from sglang.test.test_utils import CustomTestCase register_npu_ci( est_time=400, - suite="nightly-4-npu-a3", + suite="full-4-npu-a3", nightly=True, ) diff --git a/test/registered/npu/llm_models/test_npu_qwen3_8b_communications_quantization.py b/test/registered/npu/llm_models/test_npu_qwen3_8b_communications_quantization.py index 6f8d0f5b3..b103d3724 100644 --- a/test/registered/npu/llm_models/test_npu_qwen3_8b_communications_quantization.py +++ b/test/registered/npu/llm_models/test_npu_qwen3_8b_communications_quantization.py @@ -5,7 +5,7 @@ from sglang.test.ascend.test_ascend_utils import QWEN3_8B_WEIGHTS_PATH from sglang.test.ci.ci_register import register_npu_ci from sglang.test.test_utils import CustomTestCase -register_npu_ci(est_time=400, suite="nightly-2-npu-a3", nightly=True) +register_npu_ci(est_time=400, suite="full-2-npu-a3", nightly=True) class TestQwen38BCommQuantization(GSM8KAscendMixin, CustomTestCase): diff --git a/test/registered/npu/llm_models/test_npu_qwen3_coder_480b_a35b.py b/test/registered/npu/llm_models/test_npu_qwen3_coder_480b_a35b.py index ada30cf75..bdc33d567 100644 --- a/test/registered/npu/llm_models/test_npu_qwen3_coder_480b_a35b.py +++ b/test/registered/npu/llm_models/test_npu_qwen3_coder_480b_a35b.py @@ -9,7 +9,7 @@ from sglang.test.test_utils import CustomTestCase register_npu_ci( est_time=400, - suite="nightly-16-npu-a3", + suite="full-16-npu-a3", nightly=True, ) diff --git a/test/registered/npu/llm_models/test_npu_qwq_32b_w8a8.py b/test/registered/npu/llm_models/test_npu_qwq_32b_w8a8.py index 4a394b808..2f3d2b968 100644 --- a/test/registered/npu/llm_models/test_npu_qwq_32b_w8a8.py +++ b/test/registered/npu/llm_models/test_npu_qwq_32b_w8a8.py @@ -6,7 +6,7 @@ from sglang.test.ascend.test_ascend_utils import QWQ_32B_W8A8_WEIGHTS_PATH from sglang.test.ci.ci_register import register_npu_ci from sglang.test.test_utils import CustomTestCase -register_npu_ci(est_time=400, suite="nightly-2-npu-a3", nightly=True) +register_npu_ci(est_time=400, suite="full-2-npu-a3", nightly=True) class TestQWQ32BW8A8(GSM8KAscendMixin, CustomTestCase): diff --git a/test/registered/npu/performance/deepseek_v4_flash/test_npu_deepseek_v4_flash_w8a8_8p_in8k_out1k_50ms.py b/test/registered/npu/performance/deepseek_v4_flash/test_npu_deepseek_v4_flash_w8a8_8p_in8k_out1k_50ms.py index e8ae7403d..6ca06051c 100644 --- a/test/registered/npu/performance/deepseek_v4_flash/test_npu_deepseek_v4_flash_w8a8_8p_in8k_out1k_50ms.py +++ b/test/registered/npu/performance/deepseek_v4_flash/test_npu_deepseek_v4_flash_w8a8_8p_in8k_out1k_50ms.py @@ -8,7 +8,8 @@ from sglang.test.ascend.e2e.test_npu_performance_utils import ( ) from sglang.test.ci.ci_register import register_npu_ci -register_npu_ci(est_time=3600, suite="base-c-test-perf-16-npu-a3") +register_npu_ci(est_time=1200, suite="base-c-test-perf-16-npu-a3") +register_npu_ci(est_time=1200, suite="nightly-perf-16-npu-a3", nightly=True) # Environment variables for DSV4-Flash single-node PD-mix deployment. DEEPSEEK_V4_FLASH_W8A8_8P_ENVS = { @@ -55,7 +56,7 @@ DEEPSEEK_V4_FLASH_W8A8_8P_OTHER_ARGS = [ "--watchdog-timeout", 9000, "--mem-fraction-static", - 0.7, + 0.6, "--prefill-max-requests", 2, "--disable-radix-cache", diff --git a/test/registered/npu/performance/glm5_1/test_npu_glm5_1_w4a8_1p1d_32p_in64k_out1k_50ms.py b/test/registered/npu/performance/glm5_1/test_npu_glm5_1_w4a8_1p1d_32p_in64k_out1k_50ms.py index 59bf7ec5d..634e49e36 100644 --- a/test/registered/npu/performance/glm5_1/test_npu_glm5_1_w4a8_1p1d_32p_in64k_out1k_50ms.py +++ b/test/registered/npu/performance/glm5_1/test_npu_glm5_1_w4a8_1p1d_32p_in64k_out1k_50ms.py @@ -70,7 +70,7 @@ GLM_5_1_PD_SEP_PREFILL_ARGS = [ "--served-model-name", "glm-5", "--chunked-prefill-size", - 16384, + 32768, "--max-prefill-tokens", 180000, "--moe-a2a-backend", @@ -151,6 +151,14 @@ GLM_5_1_PD_SEP_DECODE_ARGS = [ "glm45", "--tool-call-parser", "glm47", + "--speculative-algorithm", + "NEXTN", + "--speculative-num-steps", + 3, + "--speculative-eagle-topk", + 1, + "--speculative-num-draft-tokens", + 4, ] GLM_5_1_PD_SEP_MODEL_CONFIG = { @@ -171,8 +179,8 @@ class TestNPUGLM5_1_W4A8_PD_SEP_In3k5_Out1k5(TestNpuPerfMultiNodePdSepTestCaseBa benchmark_tool = BENCHMARK_TOOL_DEFAULT dataset_type = AISBENCHMARK_DATASET_DEFAULT dataset_name = "random" - max_concurrency = 1 - num_prompts = 1 + max_concurrency = 32 + num_prompts = 64 input_len = 65536 output_len = 1024 random_range_ratio = 1 diff --git a/test/registered/npu/performance/kimi_k2_6/test_npu_kimi_k2_6_w4a8_16p_in64k_out1k_100ms.py b/test/registered/npu/performance/kimi_k2_6/test_npu_kimi_k2_6_w4a8_16p_in64k_out1k_100ms.py index 18e2d0504..b3fad7885 100644 --- a/test/registered/npu/performance/kimi_k2_6/test_npu_kimi_k2_6_w4a8_16p_in64k_out1k_100ms.py +++ b/test/registered/npu/performance/kimi_k2_6/test_npu_kimi_k2_6_w4a8_16p_in64k_out1k_100ms.py @@ -12,7 +12,7 @@ from sglang.test.ci.ci_register import register_npu_ci register_npu_ci( est_time=1800, - suite="nightly-8-npu-a3", + suite="full-8-npu-a3", nightly=True, disabled="Currently it is executed by the npu performance workflow.", ) diff --git a/test/registered/npu/performance/kimi_k2_6/test_npu_kimi_k2_6_w4a8_8p_in3k5_out1k5_20ms.py b/test/registered/npu/performance/kimi_k2_6/test_npu_kimi_k2_6_w4a8_8p_in3k5_out1k5_20ms.py index d8b798cb6..dea5d7736 100644 --- a/test/registered/npu/performance/kimi_k2_6/test_npu_kimi_k2_6_w4a8_8p_in3k5_out1k5_20ms.py +++ b/test/registered/npu/performance/kimi_k2_6/test_npu_kimi_k2_6_w4a8_8p_in3k5_out1k5_20ms.py @@ -10,7 +10,8 @@ from sglang.test.ascend.e2e.test_npu_performance_utils import ( ) from sglang.test.ci.ci_register import register_npu_ci -register_npu_ci(est_time=1800, suite="base-c-test-perf-16-npu-a3") +register_npu_ci(est_time=1200, suite="base-c-test-perf-16-npu-a3") +register_npu_ci(est_time=1200, suite="nightly-perf-16-npu-a3", nightly=True) KIMI_K2_6_ENVS = { "PYTORCH_NPU_ALLOC_CONF": "expandable_segments:True", diff --git a/test/registered/npu/performance/minimax_m2_5/test_npu_minimax_m2_5_w8a8_4p_in64k_out1k_prefix90_50ms.py b/test/registered/npu/performance/minimax_m2_5/test_npu_minimax_m2_5_w8a8_4p_in64k_out1k_prefix90_50ms.py index b5b0f4495..ab454ee57 100644 --- a/test/registered/npu/performance/minimax_m2_5/test_npu_minimax_m2_5_w8a8_4p_in64k_out1k_prefix90_50ms.py +++ b/test/registered/npu/performance/minimax_m2_5/test_npu_minimax_m2_5_w8a8_4p_in64k_out1k_prefix90_50ms.py @@ -10,6 +10,7 @@ from sglang.test.ascend.e2e.test_npu_performance_utils import ( from sglang.test.ci.ci_register import register_npu_ci register_npu_ci(est_time=3600, suite="base-c-test-perf-8-npu-a3") +register_npu_ci(est_time=3600, suite="nightly-perf-8-npu-a3", nightly=True) MINIMAX_M2_5_W8A8_4P_IN64K_OUT1K_PREFIX90_ENVS = { "PYTORCH_NPU_ALLOC_CONF": "expandable_segments:True", diff --git a/test/registered/npu/performance/minimax_m2_5/test_npu_minimax_m2_5_w8a8_8p_in3k5_out1k5_50ms.py b/test/registered/npu/performance/minimax_m2_5/test_npu_minimax_m2_5_w8a8_8p_in3k5_out1k5_50ms.py index fa395166b..d60fb67c7 100644 --- a/test/registered/npu/performance/minimax_m2_5/test_npu_minimax_m2_5_w8a8_8p_in3k5_out1k5_50ms.py +++ b/test/registered/npu/performance/minimax_m2_5/test_npu_minimax_m2_5_w8a8_8p_in3k5_out1k5_50ms.py @@ -12,9 +12,8 @@ from sglang.test.ci.ci_register import register_npu_ci register_npu_ci( est_time=3600, - suite="full-16-npu-a3", + suite="nightly-perf-16-npu-a3", nightly=True, - disabled="performance testcase", ) MINIMAX_M2_5_HIGH_THROUGHPUT_ENVS = { diff --git a/test/registered/npu/performance/qwen3-8b/test_npu_qwen3_8b_w8a8_1p_in3k5_out1k5_50ms.py b/test/registered/npu/performance/qwen3-8b/test_npu_qwen3_8b_w8a8_1p_in3k5_out1k5_50ms.py index f14459056..9eb0bc744 100644 --- a/test/registered/npu/performance/qwen3-8b/test_npu_qwen3_8b_w8a8_1p_in3k5_out1k5_50ms.py +++ b/test/registered/npu/performance/qwen3-8b/test_npu_qwen3_8b_w8a8_1p_in3k5_out1k5_50ms.py @@ -10,6 +10,7 @@ from sglang.test.ascend.e2e.test_npu_performance_utils import ( from sglang.test.ci.ci_register import register_npu_ci register_npu_ci(est_time=3600, suite="base-c-test-perf-2-npu-a3") +register_npu_ci(est_time=3600, suite="nightly-perf-2-npu-a3", nightly=True) QWEN3_8B_ENVS = { "SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT": "600", diff --git a/test/registered/npu/performance/qwen3-8b/test_npu_qwen3_8b_w8a8_1p_in6k_out1k5_bs16.py b/test/registered/npu/performance/qwen3-8b/test_npu_qwen3_8b_w8a8_1p_in6k_out1k5_bs16.py index 592b806f3..7e92f9d19 100644 --- a/test/registered/npu/performance/qwen3-8b/test_npu_qwen3_8b_w8a8_1p_in6k_out1k5_bs16.py +++ b/test/registered/npu/performance/qwen3-8b/test_npu_qwen3_8b_w8a8_1p_in6k_out1k5_bs16.py @@ -9,9 +9,8 @@ from sglang.test.ci.ci_register import register_npu_ci register_npu_ci( est_time=3600, - suite="", + suite="nightly-perf-2-npu-a3", nightly=True, - disabled="performance testcase", ) QWEN3_8B_ENVS = { diff --git a/test/registered/npu/performance/qwen3_235b_a22b/test_npu_qwen3_235b_w8a8_8p_in3k5_out1k5_50ms.py b/test/registered/npu/performance/qwen3_235b_a22b/test_npu_qwen3_235b_w8a8_8p_in3k5_out1k5_50ms.py index 1362bf032..780bdfb3f 100644 --- a/test/registered/npu/performance/qwen3_235b_a22b/test_npu_qwen3_235b_w8a8_8p_in3k5_out1k5_50ms.py +++ b/test/registered/npu/performance/qwen3_235b_a22b/test_npu_qwen3_235b_w8a8_8p_in3k5_out1k5_50ms.py @@ -10,6 +10,7 @@ from sglang.test.ascend.e2e.test_npu_performance_utils import ( from sglang.test.ci.ci_register import register_npu_ci register_npu_ci(est_time=3600, suite="base-c-test-perf-16-npu-a3") +register_npu_ci(est_time=3600, suite="nightly-perf-16-npu-a3", nightly=True) QWEN3_235B_ENVS = { "PYTORCH_NPU_ALLOC_CONF": "expandable_segments:True", diff --git a/test/registered/npu/performance/qwen3_30b_a3b/test_npu_qwen3_30b_w8a8_1p_in3k5_out1k5_50ms.py b/test/registered/npu/performance/qwen3_30b_a3b/test_npu_qwen3_30b_w8a8_1p_in3k5_out1k5_50ms.py index 4c0f4a029..f5e67f18b 100644 --- a/test/registered/npu/performance/qwen3_30b_a3b/test_npu_qwen3_30b_w8a8_1p_in3k5_out1k5_50ms.py +++ b/test/registered/npu/performance/qwen3_30b_a3b/test_npu_qwen3_30b_w8a8_1p_in3k5_out1k5_50ms.py @@ -10,6 +10,7 @@ from sglang.test.ascend.e2e.test_npu_performance_utils import ( from sglang.test.ci.ci_register import register_npu_ci register_npu_ci(est_time=3600, suite="base-c-test-perf-2-npu-a3") +register_npu_ci(est_time=3600, suite="nightly-perf-2-npu-a3", nightly=True) QWEN3_30B_A3B_ENVS = { "ASCEND_LAUNCH_BLOCKING": "0", diff --git a/test/registered/npu/performance/qwen3_32b/test_npu_qwen3_32b_bf16_8p_in18k_out4k_6ms.py b/test/registered/npu/performance/qwen3_32b/test_npu_qwen3_32b_bf16_8p_in18k_out4k_6ms.py index 835c93fe8..5b96b211a 100644 --- a/test/registered/npu/performance/qwen3_32b/test_npu_qwen3_32b_bf16_8p_in18k_out4k_6ms.py +++ b/test/registered/npu/performance/qwen3_32b/test_npu_qwen3_32b_bf16_8p_in18k_out4k_6ms.py @@ -11,9 +11,8 @@ from sglang.test.ci.ci_register import register_npu_ci register_npu_ci( est_time=3600, - suite="", + suite="nightly-perf-16-npu-a3", nightly=True, - disabled="performance testcase", ) QWEN3_32B_ENVS = { diff --git a/test/registered/npu/performance/qwen3_32b/test_npu_qwen3_32b_w8a8_2p_in3k5_out1k5_50ms.py b/test/registered/npu/performance/qwen3_32b/test_npu_qwen3_32b_w8a8_2p_in3k5_out1k5_50ms.py index a3bb52bed..5ca08a877 100644 --- a/test/registered/npu/performance/qwen3_32b/test_npu_qwen3_32b_w8a8_2p_in3k5_out1k5_50ms.py +++ b/test/registered/npu/performance/qwen3_32b/test_npu_qwen3_32b_w8a8_2p_in3k5_out1k5_50ms.py @@ -10,6 +10,7 @@ from sglang.test.ascend.e2e.test_npu_performance_utils import ( from sglang.test.ci.ci_register import register_npu_ci register_npu_ci(est_time=3600, suite="base-c-test-perf-4-npu-a3") +register_npu_ci(est_time=3600, suite="nightly-perf-4-npu-a3", nightly=True) QWEN3_32B_ENVS = { "SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT": "600", diff --git a/test/registered/npu/performance/qwen3_32b/test_npu_qwen3_32b_w8a8_2p_in3k5_out1k5_50ms_a2.py b/test/registered/npu/performance/qwen3_32b/test_npu_qwen3_32b_w8a8_2p_in3k5_out1k5_50ms_a2.py index dd012b889..7742c900f 100644 --- a/test/registered/npu/performance/qwen3_32b/test_npu_qwen3_32b_w8a8_2p_in3k5_out1k5_50ms_a2.py +++ b/test/registered/npu/performance/qwen3_32b/test_npu_qwen3_32b_w8a8_2p_in3k5_out1k5_50ms_a2.py @@ -11,9 +11,8 @@ from sglang.test.ci.ci_register import register_npu_ci register_npu_ci( est_time=3600, - suite="", + suite="nightly-1-npu-a2", nightly=True, - disabled="performance testcase", ) QWEN3_32B_ENVS = { diff --git a/test/registered/npu/performance/qwen3_5_397b/test_npu_qwen3_5_397b_w4a8_8p_in3k5_out1k5_50ms.py b/test/registered/npu/performance/qwen3_5_397b/test_npu_qwen3_5_397b_w4a8_8p_in3k5_out1k5_50ms.py index c77ef4810..69f487b4e 100644 --- a/test/registered/npu/performance/qwen3_5_397b/test_npu_qwen3_5_397b_w4a8_8p_in3k5_out1k5_50ms.py +++ b/test/registered/npu/performance/qwen3_5_397b/test_npu_qwen3_5_397b_w4a8_8p_in3k5_out1k5_50ms.py @@ -9,6 +9,7 @@ from sglang.test.ascend.e2e.test_npu_performance_utils import ( from sglang.test.ci.ci_register import register_npu_ci register_npu_ci(est_time=3600, suite="base-c-test-perf-16-npu-a3") +register_npu_ci(est_time=3600, suite="nightly-perf-16-npu-a3", nightly=True) QWEN3_5_397B_A17B_ENVS = { "PYTORCH_NPU_ALLOC_CONF": "expandable_segments:True", diff --git a/test/registered/npu/performance/qwen3_6_27b/test_npu_qwen3_6_27b_1p_in1024x1024_30_out1024_50ms.py b/test/registered/npu/performance/qwen3_6_27b/test_npu_qwen3_6_27b_1p_in1024x1024_30_out1024_50ms.py index a04e3c422..63daf043b 100644 --- a/test/registered/npu/performance/qwen3_6_27b/test_npu_qwen3_6_27b_1p_in1024x1024_30_out1024_50ms.py +++ b/test/registered/npu/performance/qwen3_6_27b/test_npu_qwen3_6_27b_1p_in1024x1024_30_out1024_50ms.py @@ -9,6 +9,7 @@ from sglang.test.ascend.e2e.test_npu_performance_utils import ( from sglang.test.ci.ci_register import register_npu_ci register_npu_ci(est_time=3600, suite="base-c-test-perf-2-npu-a3") +register_npu_ci(est_time=3600, suite="nightly-perf-2-npu-a3", nightly=True) QWEN3_6_27B_1024_ENVS = { "PYTORCH_NPU_ALLOC_CONF": "expandable_segments:True", diff --git a/test/registered/npu/performance/qwen3_6_27b/test_npu_qwen3_6_27b_1p_in1080p_30_out256_50ms.py b/test/registered/npu/performance/qwen3_6_27b/test_npu_qwen3_6_27b_1p_in1080p_30_out256_50ms.py index 08e9bc143..bd90e0ffe 100644 --- a/test/registered/npu/performance/qwen3_6_27b/test_npu_qwen3_6_27b_1p_in1080p_30_out256_50ms.py +++ b/test/registered/npu/performance/qwen3_6_27b/test_npu_qwen3_6_27b_1p_in1080p_30_out256_50ms.py @@ -10,9 +10,8 @@ from sglang.test.ci.ci_register import register_npu_ci register_npu_ci( est_time=3600, - suite="full-2-npu-a3", + suite="nightly-perf-2-npu-a3", nightly=True, - disabled="performance testcase", ) QWEN3_6_27B_1080P_ENVS = { diff --git a/test/registered/npu/performance/qwen3_6_27b/test_npu_qwen3_6_27b_2p_in64k_out1k_prefix90_50ms.py b/test/registered/npu/performance/qwen3_6_27b/test_npu_qwen3_6_27b_2p_in64k_out1k_prefix90_50ms.py index 7e06a9608..1ab4e74f4 100644 --- a/test/registered/npu/performance/qwen3_6_27b/test_npu_qwen3_6_27b_2p_in64k_out1k_prefix90_50ms.py +++ b/test/registered/npu/performance/qwen3_6_27b/test_npu_qwen3_6_27b_2p_in64k_out1k_prefix90_50ms.py @@ -9,9 +9,8 @@ from sglang.test.ci.ci_register import register_npu_ci register_npu_ci( est_time=3600, - suite="", + suite="nightly-perf-4-npu-a3", nightly=True, - disabled="performance testcase", ) QWEN3_6_27B_64K_PREFIX_ENVS = { diff --git a/test/registered/npu/performance/qwen3_6_27b/test_npu_qwen3_6_27b_w8a8_1p_in3k5_out1k5_50ms.py b/test/registered/npu/performance/qwen3_6_27b/test_npu_qwen3_6_27b_w8a8_1p_in3k5_out1k5_50ms.py index 92764ae9f..dec90357e 100644 --- a/test/registered/npu/performance/qwen3_6_27b/test_npu_qwen3_6_27b_w8a8_1p_in3k5_out1k5_50ms.py +++ b/test/registered/npu/performance/qwen3_6_27b/test_npu_qwen3_6_27b_w8a8_1p_in3k5_out1k5_50ms.py @@ -10,9 +10,8 @@ from sglang.test.ci.ci_register import register_npu_ci register_npu_ci( est_time=3600, - suite="full-2-npu-a3", + suite="nightly-perf-2-npu-a3", nightly=True, - disabled="performance testcase", ) QWEN3_6_27B_3K5_1K5_ENVS = { diff --git a/test/registered/npu/performance/qwen3_6_27b/test_npu_qwen3_6_27b_w8a8_1p_in64k_out1k_50ms.py b/test/registered/npu/performance/qwen3_6_27b/test_npu_qwen3_6_27b_w8a8_1p_in64k_out1k_50ms.py index 14b869e4c..aff01ed9f 100644 --- a/test/registered/npu/performance/qwen3_6_27b/test_npu_qwen3_6_27b_w8a8_1p_in64k_out1k_50ms.py +++ b/test/registered/npu/performance/qwen3_6_27b/test_npu_qwen3_6_27b_w8a8_1p_in64k_out1k_50ms.py @@ -9,6 +9,7 @@ from sglang.test.ascend.e2e.test_npu_performance_utils import ( from sglang.test.ci.ci_register import register_npu_ci register_npu_ci(est_time=3600, suite="base-c-test-perf-2-npu-a3") +register_npu_ci(est_time=3600, suite="nightly-perf-4-npu-a3", nightly=True) QWEN3_6_27B_64K_1K_ENVS = { "STREAMS_PER_DEVICE": "32", diff --git a/test/registered/npu/performance/qwen3_6_27b/test_npu_qwen3_6_27b_w8a8_2p_in128k_out1k_50ms.py b/test/registered/npu/performance/qwen3_6_27b/test_npu_qwen3_6_27b_w8a8_2p_in128k_out1k_50ms.py index 170da0d6f..062273708 100644 --- a/test/registered/npu/performance/qwen3_6_27b/test_npu_qwen3_6_27b_w8a8_2p_in128k_out1k_50ms.py +++ b/test/registered/npu/performance/qwen3_6_27b/test_npu_qwen3_6_27b_w8a8_2p_in128k_out1k_50ms.py @@ -10,9 +10,8 @@ from sglang.test.ci.ci_register import register_npu_ci register_npu_ci( est_time=3600, - suite="", + suite="nightly-perf-4-npu-a3", nightly=True, - disabled="performance testcase", ) QWEN3_6_27B_128K_ENVS = { diff --git a/test/registered/npu/performance/qwen3_6_27b/test_npu_qwen3_6_27b_w8a8_2p_in16k_out1k_50ms.py b/test/registered/npu/performance/qwen3_6_27b/test_npu_qwen3_6_27b_w8a8_2p_in16k_out1k_50ms.py index d1e823bcd..5d8eb24ac 100644 --- a/test/registered/npu/performance/qwen3_6_27b/test_npu_qwen3_6_27b_w8a8_2p_in16k_out1k_50ms.py +++ b/test/registered/npu/performance/qwen3_6_27b/test_npu_qwen3_6_27b_w8a8_2p_in16k_out1k_50ms.py @@ -10,9 +10,8 @@ from sglang.test.ci.ci_register import register_npu_ci register_npu_ci( est_time=3600, - suite="", + suite="nightly-perf-4-npu-a3", nightly=True, - disabled="performance testcase", ) QWEN3_6_27B_16K_1k_ENVS = { diff --git a/test/registered/npu/performance/qwen3_6_35b_a3b/test_npu_qwen3_6_35b_a3b_1p_in128k_out1k_50ms.py b/test/registered/npu/performance/qwen3_6_35b_a3b/test_npu_qwen3_6_35b_a3b_1p_in128k_out1k_50ms.py index 80edd0221..6a9600b4a 100644 --- a/test/registered/npu/performance/qwen3_6_35b_a3b/test_npu_qwen3_6_35b_a3b_1p_in128k_out1k_50ms.py +++ b/test/registered/npu/performance/qwen3_6_35b_a3b/test_npu_qwen3_6_35b_a3b_1p_in128k_out1k_50ms.py @@ -10,9 +10,8 @@ from sglang.test.ci.ci_register import register_npu_ci register_npu_ci( est_time=3600, - suite="", + suite="nightly-perf-2-npu-a3", nightly=True, - disabled="performance testcase", ) QWEN3_6_35B_A3B_128K_1K_ENVS = { diff --git a/test/registered/npu/performance/qwen3_6_35b_a3b/test_npu_qwen3_6_35b_a3b_1p_in128k_out1k_prefix90_50ms.py b/test/registered/npu/performance/qwen3_6_35b_a3b/test_npu_qwen3_6_35b_a3b_1p_in128k_out1k_prefix90_50ms.py index 4827fa521..70f1f4174 100644 --- a/test/registered/npu/performance/qwen3_6_35b_a3b/test_npu_qwen3_6_35b_a3b_1p_in128k_out1k_prefix90_50ms.py +++ b/test/registered/npu/performance/qwen3_6_35b_a3b/test_npu_qwen3_6_35b_a3b_1p_in128k_out1k_prefix90_50ms.py @@ -9,9 +9,8 @@ from sglang.test.ci.ci_register import register_npu_ci register_npu_ci( est_time=3600, - suite="", + suite="nightly-perf-2-npu-a3", nightly=True, - disabled="performance testcase", ) QWEN3_6_35B_A3B_128K_PREFIX_ENVS = { diff --git a/test/registered/npu/performance/qwen3_6_35b_a3b/test_npu_qwen3_6_35b_a3b_1p_in3k5_out1k5_50ms.py b/test/registered/npu/performance/qwen3_6_35b_a3b/test_npu_qwen3_6_35b_a3b_1p_in3k5_out1k5_50ms.py index 3280f63ba..e23edefeb 100644 --- a/test/registered/npu/performance/qwen3_6_35b_a3b/test_npu_qwen3_6_35b_a3b_1p_in3k5_out1k5_50ms.py +++ b/test/registered/npu/performance/qwen3_6_35b_a3b/test_npu_qwen3_6_35b_a3b_1p_in3k5_out1k5_50ms.py @@ -8,12 +8,7 @@ from sglang.test.ascend.e2e.test_npu_performance_utils import ( ) from sglang.test.ci.ci_register import register_npu_ci -register_npu_ci( - est_time=3600, - suite="full-2-npu-a3", - nightly=True, - disabled="performance testcase", -) +register_npu_ci(est_time=3600, suite="nightly-perf-2-npu-a3", nightly=True) QWEN3_6_35B_A3B_3K5_1K5_ENVS = { "PYTORCH_NPU_ALLOC_CONF": "expandable_segments:True", diff --git a/test/registered/npu/performance/qwen3_6_35b_a3b/test_npu_qwen3_6_35b_a3b_1p_in64k_out1k_50ms.py b/test/registered/npu/performance/qwen3_6_35b_a3b/test_npu_qwen3_6_35b_a3b_1p_in64k_out1k_50ms.py index eb88d7f1c..034014d00 100644 --- a/test/registered/npu/performance/qwen3_6_35b_a3b/test_npu_qwen3_6_35b_a3b_1p_in64k_out1k_50ms.py +++ b/test/registered/npu/performance/qwen3_6_35b_a3b/test_npu_qwen3_6_35b_a3b_1p_in64k_out1k_50ms.py @@ -8,12 +8,7 @@ from sglang.test.ascend.e2e.test_npu_performance_utils import ( ) from sglang.test.ci.ci_register import register_npu_ci -register_npu_ci( - est_time=3600, - suite="full-2-npu-a3", - nightly=True, - disabled="performance testcase", -) +register_npu_ci(est_time=3600, suite="nightly-perf-2-npu-a3", nightly=True) QWEN3_6_35B_A3B_64K_1K_ENVS = { "PYTORCH_NPU_ALLOC_CONF": "expandable_segments:True", diff --git a/test/registered/npu/performance/qwen3_6_35b_a3b/test_npu_qwen3_6_35b_a3b_1p_in64k_out1k_prefix90_50ms.py b/test/registered/npu/performance/qwen3_6_35b_a3b/test_npu_qwen3_6_35b_a3b_1p_in64k_out1k_prefix90_50ms.py index cd191e7a4..c803d6fbc 100644 --- a/test/registered/npu/performance/qwen3_6_35b_a3b/test_npu_qwen3_6_35b_a3b_1p_in64k_out1k_prefix90_50ms.py +++ b/test/registered/npu/performance/qwen3_6_35b_a3b/test_npu_qwen3_6_35b_a3b_1p_in64k_out1k_prefix90_50ms.py @@ -9,6 +9,7 @@ from sglang.test.ascend.e2e.test_npu_performance_utils import ( from sglang.test.ci.ci_register import register_npu_ci register_npu_ci(est_time=3600, suite="base-c-test-perf-2-npu-a3") +register_npu_ci(est_time=3600, suite="nightly-perf-2-npu-a3", nightly=True) QWEN3_6_35B_A3B_64K_PREFIX_ENVS = { "PYTORCH_NPU_ALLOC_CONF": "expandable_segments:True", diff --git a/test/registered/npu/performance/qwen3_next_80b_a3b_instruct/test_npu_qwen3_next_80b_w8a8_2p_in6k_out1k5_bs16.py b/test/registered/npu/performance/qwen3_next_80b_a3b_instruct/test_npu_qwen3_next_80b_w8a8_2p_in6k_out1k5_bs16.py index 155ed324a..f1f69862b 100644 --- a/test/registered/npu/performance/qwen3_next_80b_a3b_instruct/test_npu_qwen3_next_80b_w8a8_2p_in6k_out1k5_bs16.py +++ b/test/registered/npu/performance/qwen3_next_80b_a3b_instruct/test_npu_qwen3_next_80b_w8a8_2p_in6k_out1k5_bs16.py @@ -8,6 +8,7 @@ from sglang.test.ascend.e2e.test_npu_performance_utils import ( from sglang.test.ci.ci_register import register_npu_ci register_npu_ci(est_time=3600, suite="base-c-test-perf-4-npu-a3") +register_npu_ci(est_time=3600, suite="nightly-perf-4-npu-a3", nightly=True) QWEN3_NEXT_80B_A3B_ENVS = { "PYTORCH_NPU_ALLOC_CONF": "expandable_segments:True", diff --git a/test/registered/npu/rerank_models/test_npu_bge_reranker_v2_m3.py b/test/registered/npu/rerank_models/test_npu_bge_reranker_v2_m3.py index 800a2f824..8b552741a 100644 --- a/test/registered/npu/rerank_models/test_npu_bge_reranker_v2_m3.py +++ b/test/registered/npu/rerank_models/test_npu_bge_reranker_v2_m3.py @@ -10,7 +10,7 @@ from sglang.test.test_utils import CustomTestCase register_npu_ci( est_time=400, - suite="nightly-1-npu-a3", + suite="full-1-npu-a3", nightly=True, ) diff --git a/test/registered/npu/test_npu_memory_consumption.py b/test/registered/npu/test_npu_memory_consumption.py index 99d30d22a..b9dd8b0f6 100644 --- a/test/registered/npu/test_npu_memory_consumption.py +++ b/test/registered/npu/test_npu_memory_consumption.py @@ -19,7 +19,7 @@ from sglang.test.test_utils import ( register_npu_ci( est_time=400, - suite="nightly-2-npu-a3", + suite="full-2-npu-a3", nightly=True, ) diff --git a/test/registered/npu/vlm_models/test_ascend_glm_4_5v.py b/test/registered/npu/vlm_models/test_ascend_glm_4_5v.py index ec28a57e0..c8447a7c2 100644 --- a/test/registered/npu/vlm_models/test_ascend_glm_4_5v.py +++ b/test/registered/npu/vlm_models/test_ascend_glm_4_5v.py @@ -3,7 +3,7 @@ import unittest from sglang.test.ascend.vlm_utils import TestVLMModels from sglang.test.ci.ci_register import register_npu_ci -register_npu_ci(est_time=400, suite="nightly-8-npu-a3", nightly=True) +register_npu_ci(est_time=400, suite="full-8-npu-a3", nightly=True) class TestGLM4Models(TestVLMModels): diff --git a/test/registered/npu/vlm_models/test_npu_deepseek_vl2.py b/test/registered/npu/vlm_models/test_npu_deepseek_vl2.py index 32f08ad17..27ae7dedc 100644 --- a/test/registered/npu/vlm_models/test_npu_deepseek_vl2.py +++ b/test/registered/npu/vlm_models/test_npu_deepseek_vl2.py @@ -6,7 +6,7 @@ from sglang.test.ci.ci_register import register_npu_ci register_npu_ci( est_time=400, - suite="nightly-4-npu-a3", + suite="full-4-npu-a3", nightly=True, ) diff --git a/test/registered/npu/vlm_models/test_npu_gemma_3_4b_it.py b/test/registered/npu/vlm_models/test_npu_gemma_3_4b_it.py index 289a8e98a..0e4ded15c 100644 --- a/test/registered/npu/vlm_models/test_npu_gemma_3_4b_it.py +++ b/test/registered/npu/vlm_models/test_npu_gemma_3_4b_it.py @@ -4,7 +4,7 @@ from sglang.test.ascend.test_ascend_utils import GEMMA_3_4B_IT_WEIGHTS_PATH from sglang.test.ascend.vlm_utils import TestVLMModels from sglang.test.ci.ci_register import register_npu_ci -register_npu_ci(est_time=400, suite="nightly-4-npu-a3", nightly=True) +register_npu_ci(est_time=400, suite="full-4-npu-a3", nightly=True) class TestGemma34bModels(TestVLMModels): diff --git a/test/registered/npu/vlm_models/test_npu_janus_pro_1b.py b/test/registered/npu/vlm_models/test_npu_janus_pro_1b.py index 6409158fc..dc0252aea 100644 --- a/test/registered/npu/vlm_models/test_npu_janus_pro_1b.py +++ b/test/registered/npu/vlm_models/test_npu_janus_pro_1b.py @@ -4,7 +4,7 @@ from sglang.test.ascend.test_ascend_utils import JANUS_PRO_1B_WEIGHTS_PATH from sglang.test.ascend.vlm_utils import TestVLMModels from sglang.test.ci.ci_register import register_npu_ci -register_npu_ci(est_time=400, suite="nightly-4-npu-a3", nightly=True) +register_npu_ci(est_time=400, suite="full-4-npu-a3", nightly=True) class TestJanusPro1B(TestVLMModels): diff --git a/test/registered/npu/vlm_models/test_npu_janus_pro_7b.py b/test/registered/npu/vlm_models/test_npu_janus_pro_7b.py index 9dee3f8d8..f9399f20c 100644 --- a/test/registered/npu/vlm_models/test_npu_janus_pro_7b.py +++ b/test/registered/npu/vlm_models/test_npu_janus_pro_7b.py @@ -4,7 +4,7 @@ from sglang.test.ascend.test_ascend_utils import JANUS_PRO_7B_WEIGHTS_PATH from sglang.test.ascend.vlm_utils import TestVLMModels from sglang.test.ci.ci_register import register_npu_ci -register_npu_ci(est_time=400, suite="nightly-4-npu-a3", nightly=True) +register_npu_ci(est_time=400, suite="full-4-npu-a3", nightly=True) class TestJanusPro7B(TestVLMModels): diff --git a/test/registered/npu/vlm_models/test_npu_kimi_vl_a3b_instruct.py b/test/registered/npu/vlm_models/test_npu_kimi_vl_a3b_instruct.py index 3f6754c61..c7c3d0923 100644 --- a/test/registered/npu/vlm_models/test_npu_kimi_vl_a3b_instruct.py +++ b/test/registered/npu/vlm_models/test_npu_kimi_vl_a3b_instruct.py @@ -6,7 +6,7 @@ from sglang.test.ci.ci_register import register_npu_ci register_npu_ci( est_time=400, - suite="nightly-4-npu-a3", + suite="full-4-npu-a3", nightly=True, ) diff --git a/test/registered/npu/vlm_models/test_npu_llama_3_2_11b_vision_instruct.py b/test/registered/npu/vlm_models/test_npu_llama_3_2_11b_vision_instruct.py index a7c527dd9..12c8b3200 100644 --- a/test/registered/npu/vlm_models/test_npu_llama_3_2_11b_vision_instruct.py +++ b/test/registered/npu/vlm_models/test_npu_llama_3_2_11b_vision_instruct.py @@ -6,7 +6,7 @@ from sglang.test.ascend.test_ascend_utils import ( from sglang.test.ascend.vlm_utils import TestVLMModels from sglang.test.ci.ci_register import register_npu_ci -register_npu_ci(est_time=400, suite="nightly-1-npu-a3", nightly=True) +register_npu_ci(est_time=400, suite="full-1-npu-a3", nightly=True) class TestLlama3211BVisionInstruct(TestVLMModels): diff --git a/test/registered/npu/vlm_models/test_npu_mimo_vl_7b_rl.py b/test/registered/npu/vlm_models/test_npu_mimo_vl_7b_rl.py index 12f11ccef..366adbc59 100644 --- a/test/registered/npu/vlm_models/test_npu_mimo_vl_7b_rl.py +++ b/test/registered/npu/vlm_models/test_npu_mimo_vl_7b_rl.py @@ -4,7 +4,7 @@ from sglang.test.ascend.test_ascend_utils import MIMO_VL_7B_RL_WEIGHTS_PATH from sglang.test.ascend.vlm_utils import TestVLMModels from sglang.test.ci.ci_register import register_npu_ci -register_npu_ci(est_time=400, suite="nightly-4-npu-a3", nightly=True) +register_npu_ci(est_time=400, suite="full-4-npu-a3", nightly=True) class TestMiMoModels(TestVLMModels): diff --git a/test/registered/npu/vlm_models/test_npu_minicpm_o_2_6.py b/test/registered/npu/vlm_models/test_npu_minicpm_o_2_6.py index a45bb38e7..fb184189e 100644 --- a/test/registered/npu/vlm_models/test_npu_minicpm_o_2_6.py +++ b/test/registered/npu/vlm_models/test_npu_minicpm_o_2_6.py @@ -6,7 +6,7 @@ from sglang.test.ci.ci_register import register_npu_ci register_npu_ci( est_time=400, - suite="nightly-4-npu-a3", + suite="full-4-npu-a3", nightly=True, ) diff --git a/test/registered/npu/vlm_models/test_npu_minicpm_v_2_6.py b/test/registered/npu/vlm_models/test_npu_minicpm_v_2_6.py index 74a5efd1b..846c38fc9 100644 --- a/test/registered/npu/vlm_models/test_npu_minicpm_v_2_6.py +++ b/test/registered/npu/vlm_models/test_npu_minicpm_v_2_6.py @@ -4,7 +4,7 @@ from sglang.test.ascend.test_ascend_utils import MINICPM_V_2_6_WEIGHTS_PATH from sglang.test.ascend.vlm_utils import TestVLMModels from sglang.test.ci.ci_register import register_npu_ci -register_npu_ci(est_time=400, suite="nightly-4-npu-a3", nightly=True) +register_npu_ci(est_time=400, suite="full-4-npu-a3", nightly=True) class TestMiniCPMModelsV(TestVLMModels): diff --git a/test/registered/npu/vlm_models/test_npu_mistral_small_3_1_24b_instruct_2503.py b/test/registered/npu/vlm_models/test_npu_mistral_small_3_1_24b_instruct_2503.py index 14bd9bb27..5263678af 100644 --- a/test/registered/npu/vlm_models/test_npu_mistral_small_3_1_24b_instruct_2503.py +++ b/test/registered/npu/vlm_models/test_npu_mistral_small_3_1_24b_instruct_2503.py @@ -6,7 +6,7 @@ from sglang.test.ascend.test_ascend_utils import ( from sglang.test.ascend.vlm_utils import TestVLMModels from sglang.test.ci.ci_register import register_npu_ci -register_npu_ci(est_time=400, suite="nightly-4-npu-a3", nightly=True) +register_npu_ci(est_time=400, suite="full-4-npu-a3", nightly=True) class TestMistralModels(TestVLMModels): diff --git a/test/registered/npu/vlm_models/test_npu_phi4_multimodal_instruct.py b/test/registered/npu/vlm_models/test_npu_phi4_multimodal_instruct.py index c6d2c00bc..ca66c32cd 100644 --- a/test/registered/npu/vlm_models/test_npu_phi4_multimodal_instruct.py +++ b/test/registered/npu/vlm_models/test_npu_phi4_multimodal_instruct.py @@ -4,7 +4,7 @@ from sglang.test.ascend.test_ascend_utils import PHI_4_MULTIMODAL_INSTRUCT_WEIGH from sglang.test.ascend.vlm_utils import TestVLMModels from sglang.test.ci.ci_register import register_npu_ci -register_npu_ci(est_time=400, suite="nightly-4-npu-a3", nightly=True) +register_npu_ci(est_time=400, suite="full-4-npu-a3", nightly=True) class TestPhi4Multimodal(TestVLMModels): diff --git a/test/registered/npu/vlm_models/test_npu_qwen2_5_vl_3b_instruct.py b/test/registered/npu/vlm_models/test_npu_qwen2_5_vl_3b_instruct.py index 245e74dff..e34691bb3 100644 --- a/test/registered/npu/vlm_models/test_npu_qwen2_5_vl_3b_instruct.py +++ b/test/registered/npu/vlm_models/test_npu_qwen2_5_vl_3b_instruct.py @@ -4,7 +4,7 @@ from sglang.test.ascend.test_ascend_utils import QWEN2_5_VL_3B_INSTRUCT_WEIGHTS_ from sglang.test.ascend.vlm_utils import TestVLMModels from sglang.test.ci.ci_register import register_npu_ci -register_npu_ci(est_time=400, suite="nightly-4-npu-a3", nightly=True) +register_npu_ci(est_time=400, suite="full-4-npu-a3", nightly=True) class TestQwen25VL3B(TestVLMModels): diff --git a/test/registered/npu/vlm_models/test_npu_qwen2_5_vl_72b_instruct.py b/test/registered/npu/vlm_models/test_npu_qwen2_5_vl_72b_instruct.py index 8755c6a08..d8fb45b16 100644 --- a/test/registered/npu/vlm_models/test_npu_qwen2_5_vl_72b_instruct.py +++ b/test/registered/npu/vlm_models/test_npu_qwen2_5_vl_72b_instruct.py @@ -4,7 +4,7 @@ from sglang.test.ascend.test_ascend_utils import QWEN2_5_VL_72B_INSTRUCT_WEIGHTS from sglang.test.ascend.vlm_utils import TestVLMModels from sglang.test.ci.ci_register import register_npu_ci -register_npu_ci(est_time=400, suite="nightly-8-npu-a3", nightly=True) +register_npu_ci(est_time=400, suite="full-8-npu-a3", nightly=True) class TestQwen25VL72B(TestVLMModels): diff --git a/test/registered/npu/vlm_models/test_npu_qwen3_vl_235b_a22b_instruct.py b/test/registered/npu/vlm_models/test_npu_qwen3_vl_235b_a22b_instruct.py index bcd91e236..659c581b5 100644 --- a/test/registered/npu/vlm_models/test_npu_qwen3_vl_235b_a22b_instruct.py +++ b/test/registered/npu/vlm_models/test_npu_qwen3_vl_235b_a22b_instruct.py @@ -6,7 +6,7 @@ from sglang.test.ascend.test_ascend_utils import ( from sglang.test.ascend.vlm_utils import TestVLMModels from sglang.test.ci.ci_register import register_npu_ci -register_npu_ci(est_time=400, suite="nightly-16-npu-a3", nightly=True) +register_npu_ci(est_time=400, suite="full-16-npu-a3", nightly=True) class TestQwen3VL235BA22B(TestVLMModels): diff --git a/test/registered/npu/vlm_models/test_npu_qwen3_vl_30b_a3b_instruct.py b/test/registered/npu/vlm_models/test_npu_qwen3_vl_30b_a3b_instruct.py index 2e08ff4f6..f8b765655 100644 --- a/test/registered/npu/vlm_models/test_npu_qwen3_vl_30b_a3b_instruct.py +++ b/test/registered/npu/vlm_models/test_npu_qwen3_vl_30b_a3b_instruct.py @@ -4,7 +4,7 @@ from sglang.test.ascend.test_ascend_utils import QWEN3_VL_30B_A3B_INSTRUCT_WEIGH from sglang.test.ascend.vlm_utils import TestVLMModels from sglang.test.ci.ci_register import register_npu_ci -register_npu_ci(est_time=400, suite="nightly-4-npu-a3", nightly=True) +register_npu_ci(est_time=400, suite="full-4-npu-a3", nightly=True) class TestQwen3VL30BA3B(TestVLMModels): diff --git a/test/registered/npu/vlm_models/test_npu_qwen3_vl_8b_instruct.py b/test/registered/npu/vlm_models/test_npu_qwen3_vl_8b_instruct.py index 1a59569e9..5f74a031f 100644 --- a/test/registered/npu/vlm_models/test_npu_qwen3_vl_8b_instruct.py +++ b/test/registered/npu/vlm_models/test_npu_qwen3_vl_8b_instruct.py @@ -4,7 +4,7 @@ from sglang.test.ascend.test_ascend_utils import QWEN3_VL_8B_INSTRUCT_WEIGHTS_PA from sglang.test.ascend.vlm_utils import TestVLMModels from sglang.test.ci.ci_register import register_npu_ci -register_npu_ci(est_time=400, suite="nightly-4-npu-a3", nightly=True) +register_npu_ci(est_time=400, suite="full-4-npu-a3", nightly=True) class TestQwen3VL8B(TestVLMModels): diff --git a/test/run_suite.py b/test/run_suite.py index dece7e03f..ef753276e 100644 --- a/test/run_suite.py +++ b/test/run_suite.py @@ -153,11 +153,20 @@ NIGHTLY_SUITES = { ], HWBackend.CPU: [], HWBackend.NPU: [ + "nightly-1-npu-a2", "nightly-1-npu-a3", "nightly-2-npu-a3", "nightly-4-npu-a3", "nightly-8-npu-a3", "nightly-16-npu-a3", + "nightly-acc-2-npu-a3", + "nightly-acc-4-npu-a3", + "nightly-acc-8-npu-a3", + "nightly-acc-16-npu-a3", + "nightly-perf-2-npu-a3", + "nightly-perf-4-npu-a3", + "nightly-perf-8-npu-a3", + "nightly-perf-16-npu-a3", "full-1-npu-a3", "full-2-npu-a3", "full-4-npu-a3",