[NPU CI] Reorganize test output/log directory structure with workflow context (#33685)

2026-08-18 23:46:00 +08:00
parent 63d783bbe0
commit 499e90a125
139 changed files with 779 additions and 688 deletions
@@ -40,6 +40,10 @@ on:
        description: 'Git ref (branch, tag, or SHA) to test. If not provided, uses the default branch.'
        type: string
        default: 'false'
+      is_nightly_pipeline_job:
+        description: 'Run the test suite with --nightly (collects nightly-registered tests) and --continue-on-error'
+        type: boolean
+        default: false
      github-token:
        description: 'GitHub token for API calls'
        type: string
@@ -177,6 +181,8 @@ jobs:
            }

      - name: Install dependencies
+        # Only PR jobs install dependencies
+        if: ${{ inputs.is_nightly_pipeline_job != true }}
        env:
          TORCH_CACHE_URL: "http://cache-service.nginx-pypi-cache.svc.cluster.local/whl/cpu"
          PYPI_CACHE_URL: "http://cache-service.nginx-pypi-cache.svc.cluster.local/pypi/simple"
@@ -204,10 +210,43 @@ jobs:
        timeout-minutes: ${{ fromJson(inputs.run_timeout_minutes) }}
        env:
          CONTINUE_ON_ERROR_FLAG: ${{ inputs.continue_on_error == 'true' && '--continue-on-error' || '' }}
+        shell: bash
        run: |
+          # Fail fast on any command error, undefined variable, or pipe failure.
+          set -euo pipefail
+
+          # Install missing python deps (skip if already importable).
+          PYTHON_FOR_SGLANG="python"
+          PIP_INDEXES=("https://pypi.org/simple/" "https://mirrors.tuna.tsinghua.edu.cn/pypi/web/simple" "http://cache-service.nginx-pypi-cache.svc.cluster.local/pypi/simple")
+          PIP_TRUSTED_HOST=cache-service.nginx-pypi-cache.svc.cluster.local
+          install_pkg() {
+            local pkg="$1"
+            ok=0
+            for index in "${PIP_INDEXES[@]}"; do
+              echo "${pkg}: try via ${index}"
+              if ${PYTHON_FOR_SGLANG} -m pip install "${pkg}" --retries 3 --timeout 60 \
+                  -i "${index}" --trusted-host "${PIP_TRUSTED_HOST}"; then
+                ok=1; break
+              fi
+            done
+            [ "${ok}" = "1" ] || { echo "ERROR: failed to install ${pkg}"; exit 1; }
+          }
+          REQUIRED_PIP_PKGS=(tabulate)
+          for pkg in "${REQUIRED_PIP_PKGS[@]}"; do
+            ${PYTHON_FOR_SGLANG} -c "import ${pkg}" 2>/dev/null && { echo "${pkg}: available, skip"; continue; }
+            install_pkg "${pkg}"
+          done
+
          cd test
+          # Append `--nightly` and `--continue-on-error` when the job belongs to the nightly pipeline.
+          # --nightly: Collect test cases registered for nightly runs.
+          # --continue-on-error: Keep running remaining cases even if one fails.
+          NIGHTLY_FLAG=""
+          if [ "${{ inputs.is_nightly_pipeline_job }}" = "true" ]; then
+            NIGHTLY_FLAG="--nightly --continue-on-error"
+          fi
          python3 run_suite.py --hw npu --suite ${{ inputs.self_name }} \
            --auto-partition-id ${{ matrix.partition }} \
            --auto-partition-size ${{ fromJson(inputs.partitions).size }} \
            ${{ inputs.timeout_per_file && format('--timeout-per-file {0}', inputs.timeout_per_file) || '' }} \
-            $CONTINUE_ON_ERROR_FLAG
+            $NIGHTLY_FLAG $CONTINUE_ON_ERROR_FLAG
@@ -37,10 +37,23 @@ on:
        type: string
        default: ""
        description: "The transformers version number for running sglang. Use default version in image if keep empty."
+      run_start_metadata:
+        required: false
+        type: string
+        default: '{}'
+        description: 'JSON run metadata {branch_label, workflow_name, create_time}, recorded once at workflow start'
      skip_pr_test_health_check:
        description: 'Git ref (branch, tag, or SHA) to test. If not provided, uses the default branch.'
        type: string
        default: 'false'
+      is_nightly_pipeline_job:
+        description: 'Run the test suite with --nightly (collects nightly-registered tests) and --continue-on-error'
+        type: boolean
+        default: false
+      partitions:
+        description: 'Partition config to parallelize a suite across jobs, e.g. {"size":3,"arr":[0,1,2]}. Each element of arr is a partition id; the template expands one matrix job per element. size is passed to run_suite.py --auto-partition-size.'
+        type: string
+        default: '{"size":1,"arr":[0]}'
      github-token:
        description: 'GitHub token for API calls'
        type: string
@@ -56,6 +69,10 @@ concurrency:
 jobs:
  e2e:
    name: ${{ inputs.test_suite }}
+    strategy:
+      fail-fast: false
+      matrix:
+        partition: ${{ fromJson(inputs.partitions).arr }}
    runs-on: ${{ inputs.runner }}
    container:
      image: ${{ inputs.image }}
@@ -195,9 +212,11 @@ jobs:
          SGLANG_IS_IN_CI: true
          TRANSFORMERS_VERBOSITY: "error"
          GDN_ATTN_BACKEND_TRITON: 1
-          SGLANG_TEST_METRICS_OUTPUT: /root/.cache/tests/output/metrics/metrics
        shell: bash
        run: |
+          # Fail fast on any command error, undefined variable, or pipe failure.
+          set -euo pipefail
+
          sglang_source_path=$(pwd)
          echo "Source code path: ${sglang_source_path}"
          ln -sf ${sglang_source_path} /root/sglang
@@ -207,27 +226,90 @@ jobs:
          echo "Test mode: suite (${test_suite})"
          tc_name="${test_suite}"

+          # The ID is used to locate the current job instance,
+          # size determines whether parallel execution is enabled.
+          PARTITION_ID=${{ matrix.partition }}
+          PARTITION_SIZE=${{ fromJson(inputs.partitions).size }}
+          # Append directory suffixes when there are parallel partition jobs
+          # within the test suite to prevent collision of result artifact directories.
+          if [ "${PARTITION_SIZE}" -gt 1 ]; then
+            tc_name="${test_suite}-p${PARTITION_ID}"
+          fi
+
+          # Provided automatically by the GitHub server.
+          # run_id is the unique identifier for the workflow run,
+          # attempt represents the re‑run attempt count of the job.
+          run_id=${{ github.run_id }}
+          run_attempt=${{ github.run_attempt }}
+
+          # Parse run metadata (branch_label / workflow_name / create_time)
+          # from run_start_metadata JSON input using GitHub fromJson expression, same as partitions input.
+          branch_label=$(echo "${{ fromJson(inputs.run_start_metadata).branch_label }}" | tr '/:' '--')
+          workflow_name=$(echo "${{ fromJson(inputs.run_start_metadata).workflow_name }}" | tr ' ' '_' | tr -d '()')
+          create_time="${{ fromJson(inputs.run_start_metadata).create_time }}"
+          if [ -z "${create_time}" ]; then create_time=$(date -u -d '+8 hours' +%Y%m%d-%H%M); fi
+
+          # Print persistence‑directory‑prefix variables
+          echo "Persistence prefix: run_id=${run_id} run_attempt=${run_attempt} branch_label=${branch_label} workflow_name=${workflow_name} create_time=${create_time}"
+
+          if [ "${{ inputs.is_nightly_pipeline_job }}" = "true" ]; then
+            test_data_output_path=/root/.cache/tests/output/${branch_label}-${create_time}-${run_id}-${run_attempt}/${workflow_name}/${{ inputs.test_type }}/${tc_name}
+            mkdir -p ${test_data_output_path}
+
+            # Set and print the output path for test‑case results and metrics persistence.
+            export METRICS_DATA_FILE=${test_data_output_path}
+            export SGLANG_TEST_METRICS_OUTPUT=${test_data_output_path}/metrics
+            echo "METRICS_DATA_FILE=${METRICS_DATA_FILE}"
+            echo "SGLANG_TEST_METRICS_OUTPUT=${SGLANG_TEST_METRICS_OUTPUT}"
+          fi
+
          export TRANSFORMERS_VERSION_FOR_SGLANG="${{ inputs.transformers_version }}"
          PYTHON_FOR_SGLANG="python"
          PIP_FOR_SGLANG="pip"
+          # Shared pip index list (public PyPI, tsinghua mirror, then the in-cluster cache).
+          PIP_INDEXES=("https://pypi.org/simple/" "https://mirrors.tuna.tsinghua.edu.cn/pypi/web/simple" "http://cache-service.nginx-pypi-cache.svc.cluster.local/pypi/simple")
+          PIP_TRUSTED_HOST=cache-service.nginx-pypi-cache.svc.cluster.local
+
+          # Shared pip installer: try each index until one succeeds, fail the job if all fail.
+          install_pkg() {
+            local pkg="$1"
+            ok=0
+            for index in "${PIP_INDEXES[@]}"; do
+              echo "${pkg}: try via ${index}"
+              if ${PYTHON_FOR_SGLANG} -m pip install "${pkg}" --retries 3 --timeout 60 \
+                  -i "${index}" --trusted-host "${PIP_TRUSTED_HOST}"; then
+                ok=1; break
+              fi
+            done
+            [ "${ok}" = "1" ] || { echo "ERROR: failed to install ${pkg}"; exit 1; }
+          }
+
+          # Transformers (pinned version): prefer the offline wheel cache, fall back to online install.
          if [ -n "${TRANSFORMERS_VERSION_FOR_SGLANG}" ];then
            echo "===== Install transformers for sglang - Begin ====="
            TRANSFORMERS_PKG_PATH_SOURCE=/root/.cache/.cache/transformers/${TRANSFORMERS_VERSION_FOR_SGLANG}
            if [ ! -d "${TRANSFORMERS_PKG_PATH_SOURCE}" ]; then
              echo "The dependent transformers package does not exist: ${TRANSFORMERS_PKG_PATH_SOURCE}."
              echo "Install transformers ${TRANSFORMERS_VERSION_FOR_SGLANG} online."
-              pip install transformers=="${TRANSFORMERS_VERSION_FOR_SGLANG}" -i https://mirrors.tuna.tsinghua.edu.cn/pypi/web/simple
+              install_pkg "transformers==${TRANSFORMERS_VERSION_FOR_SGLANG}"
            else
              echo "Install transformers ${TRANSFORMERS_VERSION_FOR_SGLANG} locally."
              TRANSFORMERS_PKG_PATH_TARGET=/tmp/transformers/${TRANSFORMERS_VERSION_FOR_SGLANG}
              mkdir -p "${TRANSFORMERS_PKG_PATH_TARGET}"
-              cp "${TRANSFORMERS_PKG_PATH_SOURCE}/*" "${TRANSFORMERS_PKG_PATH_TARGET}/"
-              pip install --no-index --find-links="${TRANSFORMERS_PKG_PATH_TARGET}" transformers=="${TRANSFORMERS_VERSION_FOR_SGLANG}"
+              cp "${TRANSFORMERS_PKG_PATH_SOURCE}/"* "${TRANSFORMERS_PKG_PATH_TARGET}/"
+              ${PYTHON_FOR_SGLANG} -m pip install --no-index --find-links="${TRANSFORMERS_PKG_PATH_TARGET}" transformers=="${TRANSFORMERS_VERSION_FOR_SGLANG}"
            fi
            echo "===== Install transformers for sglang in virtual env - End ====="
          fi
          echo "Transformers version for sglang: $(${PIP_FOR_SGLANG} show transformers | grep Version | cut -d: -f2)"

+          # Install missing python deps (skip if already importable)
+          REQUIRED_PIP_PKGS=(tabulate)
+          for pkg in "${REQUIRED_PIP_PKGS[@]}"; do
+            ${PYTHON_FOR_SGLANG} -c "import ${pkg}" 2>/dev/null && { echo "${pkg}: available, skip"; continue; }
+            install_pkg "${pkg}"
+          done
+
          echo "scaling_governor performance num: \
            $(cat /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor | grep performance | wc -l)"
          echo "swappiness: $(cat /proc/sys/vm/swappiness)"
@@ -246,21 +328,64 @@ jobs:
          mv ${ascend_test_util_path} ${ascend_test_util_path}_bak
          cp -r ${sglang_source_path}/python/sglang/test/ascend ${ascend_test_util_path}

+          # External env scripts may reference unset shell vars (e.g. ZSH_VERSION),
+          # which abort the whole script under `set -u`. Temporarily disable -u.
+          set +u
          source /usr/local/Ascend/cann/set_env.sh || true
          source /usr/local/Ascend/nnal/atb/set_env.sh || true
          source /usr/local/Ascend/ascend-toolkit/latest/opp/vendors/customize/bin/set_env.bash || true
          source /usr/local/Ascend/ascend-toolkit/latest/opp/vendors/custom_transformer/bin/set_env.bash || true
+          set -u

-          # Set environment of cann
-          log_path="/root/.cache/tests/logs/log/${current_date}/${tc_name}/${HOSTNAME}"
-          rm -rf ${log_path}
-          mkdir -p ${log_path}
+          # Nightly log path mirrors the output layout; PR keeps the original {date}/{testcase}.
+          if [ "${{ inputs.is_nightly_pipeline_job }}" = "true" ]; then
+            log_path="/root/.cache/tests/logs/log/${branch_label}-${create_time}-${run_id}-${run_attempt}/${workflow_name}/${{ inputs.test_type }}/${tc_name}/${HOSTNAME}"
+          else
+            current_date=$(date -u -d '+8 hours' +%Y%m%d)
+            log_path="/root/.cache/tests/logs/log/${current_date}/${tc_name}/${HOSTNAME}"
+          fi
+          rm -rf "${log_path}"
+          mkdir -p "${log_path}"
          echo "Log path: ${log_path}"

          echo "Running test: ${tc_name}"
          test_exit_code=0
          cd test
-          ${PYTHON_FOR_SGLANG} -u run_suite.py --hw npu --suite ${test_suite} --timeout-per-file 7200 2>&1 | tee /tmp/test_output.log || test_exit_code=$?
+
+          # Append `--nightly` and `--continue-on-error` arguments when the job belongs to the nightly pipeline.
+          # --nightly: Collect test cases registered for nightly runs.
+          # --continue-on-error: Keep running remaining cases even if one fails.
+          NIGHTLY_FLAG=""
+          if [ "${{ inputs.is_nightly_pipeline_job }}" = "true" ]; then
+            NIGHTLY_FLAG="--nightly --continue-on-error"
+          fi
+
+          # Append configuration arguments when the test suite is split into multiple parallel jobs.
+          PARTITION_ARGS=""
+          if [ "${PARTITION_SIZE}" -gt 1 ] 2>/dev/null; then
+            PARTITION_ARGS="--auto-partition-id ${PARTITION_ID} --auto-partition-size ${PARTITION_SIZE}"
+          fi
+
+          # Append `--timeout-from-est‑time` when the job belongs to the nightly pipeline.
+          # per‑case timeout is derived from each case's est_time.
+          TIMEOUT_FLAG=""
+          if [ "${{ inputs.is_nightly_pipeline_job }}" = "true" ]; then
+            TIMEOUT_FLAG="--timeout-from-est-time"
+          fi
+
+          # Nightly additionally persists the full suite log to ${log_path}/${tc_name}.log
+          # under the structured path; PR runs only tee to /tmp/test_output.log.
+          # Use an array so the extra target is omitted when empty: passing an empty
+          # string arg makes GNU tee fail (exit code 1) and flips the pipeline exit code.
+          LOG_TEE_TARGETS=("/tmp/test_output.log")
+          if [ "${{ inputs.is_nightly_pipeline_job }}" = "true" ]; then
+            LOG_TEE_TARGETS+=("${log_path}/${tc_name}.log")
+          fi
+
+          # Run NPU test suite with run_suite.py.
+          # Capture mode (--enable-retry --max-attempts 1) keeps test subprocesses off the
+          # tee pipe, so leftover server processes cannot block the pipeline on exit.
+          ${PYTHON_FOR_SGLANG} -u run_suite.py --hw npu --suite ${test_suite} ${NIGHTLY_FLAG} ${PARTITION_ARGS} ${TIMEOUT_FLAG} --enable-retry --max-attempts 1 --timeout-per-file 7200 2>&1 | tee "${LOG_TEE_TARGETS[@]}" || test_exit_code=$?

          echo "Finished test: ${tc_name}"

@@ -272,10 +397,8 @@ jobs:
            status_icon="❌"
          fi

-          echo "test_status=${test_status}" >> $GITHUB_ENV
-          echo "tc_name=${tc_name}" >> $GITHUB_ENV
+          echo "tc_name=${tc_name}"
          export test_status tc_name
-
          echo "## ${tc_name}  ${status_icon} ${test_status}" >> $GITHUB_STEP_SUMMARY
          echo "" >> $GITHUB_STEP_SUMMARY
          metric_count=$(grep -c '\[METRIC\]' /tmp/test_output.log 2>/dev/null || echo 0)
@@ -59,6 +59,11 @@ on:
        type: string
        default: ""
        description: "The transformers version number for running sglang. Use default version in image if keep empty."
+      run_start_metadata:
+        required: false
+        type: string
+        default: '{}'
+        description: 'JSON run metadata {branch_label, workflow_name, create_time}, recorded once at workflow start'

 concurrency:
  group: ascend-nightly-multi-node-${{ github.workflow_ref }}-${{ github.ref }}-${{ inputs.test_config_name }}
@@ -135,14 +140,28 @@ jobs:
          fi

          # prepare for output data
-          current_date=$(date +%Y%m%d)
-          test_data_output_path=/root/.cache/tests/output/${{ inputs.test_type }}/${current_date}
-          mkdir -p ${test_data_output_path}
          tc_name=${test_case##*/}
          tc_name=${tc_name%.*}
-          metrics_data_path=${test_data_output_path}/${tc_name}
-          mkdir -p ${metrics_data_path}
-          echo "Metrics file path: ${metrics_data_path}"
+
+          # Provided automatically by the GitHub server.
+          # run_id is the unique identifier for the workflow run,
+          # attempt represents the re‑run attempt count of the job.
+          run_id=${{ github.run_id }}
+          run_attempt=${{ github.run_attempt }}
+
+          # Parse run metadata (branch_label / workflow_name / create_time)
+          # from run_start_metadata JSON input using GitHub fromJson expression, same as partitions input.
+          branch_label=$(echo "${{ fromJson(inputs.run_start_metadata).branch_label }}" | tr '/:' '--')
+          workflow_name=$(echo "${{ fromJson(inputs.run_start_metadata).workflow_name }}" | tr ' ' '_' | tr -d '()')
+          create_time="${{ fromJson(inputs.run_start_metadata).create_time }}"
+          if [ -z "${create_time}" ]; then create_time=$(date -u -d '+8 hours' +%Y%m%d-%H%M); fi
+
+          # Print persistence‑directory‑prefix variables
+          echo "Persistence prefix: run_id=${run_id} run_attempt=${run_attempt} branch_label=${branch_label} workflow_name=${workflow_name} create_time=${create_time}"
+
+          sglang_test_results_output_path=/root/.cache/tests/output/${branch_label}-${create_time}-${run_id}-${run_attempt}/${workflow_name}/${{ inputs.test_type }}/${tc_name}
+          mkdir -p ${sglang_test_results_output_path}
+          echo "Metrics file path: ${sglang_test_results_output_path}"

          prefill_decode_deployment="${{ inputs.prefill_decode_deployment }}"
          kube_job_type=""
@@ -161,7 +180,7 @@ jobs:
            --env ci \
            --image ${{ inputs.image }} \
            --sglang-source-relative-path ${sglang_source_relative_path} \
-            --metrics-data-file ${metrics_data_path} \
+            --metrics-data-file ${sglang_test_results_output_path} \
            --test-case ${test_case} \
            --kube-name-space ${NAMESPACE} \
            --kube-job-type ${kube_job_type} \
@@ -189,7 +208,7 @@ jobs:
            echo "Install sglang from source."
            CMD="${CMD} --install-sglang-from-source"
            commit_id=${{ github.sha }}
-            echo "commit id: ${commit_id}" > ${test_data_output_path}/commit_id
+            echo "commit id: ${commit_id}" > "$(dirname "${sglang_test_results_output_path}")/commit_id"
          else
            echo "Use sglang from image: ${{ inputs.image }}"
          fi
@@ -1,257 +0,0 @@
-name: 'Nightly Test (NPU) Single Node Template'
-
-on:
-  workflow_call:
-    inputs:
-      runner:
-        required: true
-        type: string
-        default: linux-aarch64-a3-16
-      test_type:
-        required: false
-        type: string
-        default: perf
-        description: perf or accuracy
-      test_config_name:
-        required: true
-        type: string
-        description: test config name
-      test_case:
-        required: true
-        type: string
-        description: path of test case file
-      image:
-        required: false
-        type: string
-        description: image for pods
-        default: "swr.cn-southwest-2.myhuaweicloud.com/base_image/dockerhub/lmsysorg/sglang:main-cann9.0.0-a3"
-      install_sglang_from_source:
-        required: false
-        type: boolean
-        default: false
-        description: use sglang from source code or from docker image
-      install_sglang_deps:
-        required: false
-        type: boolean
-        default: false
-        description: install sglang dependencies (e.g. PyTorch, CANN packages) when using source installation
-      device_type_for_deps:
-        required: false
-        type: string
-        default: 'a3'
-        description: device type for dependency installation (a3 or 910b)
-      transformers_version:
-        required: false
-        type: string
-        default: ""
-        description: "The transformers version number for running sglang. Use default version in image if keep empty."
-
-concurrency:
-  group: ascend-nightly-e2e-singlenode-${{ github.workflow_ref }}-${{ github.ref }}-${{ inputs.test_config_name }}
-  cancel-in-progress: true
-
-jobs:
-  e2e:
-    name: ${{ inputs.test_config_name }}
-    runs-on: ${{ inputs.runner }}
-    container:
-      image: ${{ inputs.image }}
-    steps:
-      - name: Checkout code
-        uses: actions/checkout@v4
-
-      - name: Check npu info
-        run: |
-          npu-smi info
-
-      - name: Install sglang dependencies
-        if: ${{ inputs.install_sglang_deps == true }}
-        shell: bash
-        env:
-          TORCH_CACHE_URL: "http://cache-service.nginx-pypi-cache.svc.cluster.local/whl/cpu"
-          PYPI_CACHE_URL: "http://cache-service.nginx-pypi-cache.svc.cluster.local/pypi/simple"
-          UV_INDEX_URL: "http://cache-service.nginx-pypi-cache.svc.cluster.local/pypi/simple"
-          GITHUB_PROXY_URL: "https://gh-proxy.test.osinfra.cn/"
-          RUSTUP_CACHE_URL: "http://cache-service.nginx-pypi-cache.svc.cluster.local:8082"
-        run: |
-          CACHING_URL="cache-service.nginx-pypi-cache.svc.cluster.local"
-          sed -Ei "s@(ports|archive).ubuntu.com@${CACHING_URL}:8081@g" /etc/apt/sources.list
-          pip config set global.index-url http://${CACHING_URL}/pypi/simple
-          pip config set global.trusted-host "${CACHING_URL}"
-          bash scripts/ci/npu/npu_ci_install_dependency.sh ${{ inputs.device_type_for_deps }}
-          cp ~/.cache/modelscope/hub/datasets/otavia/ShareGPT_Vicuna_unfiltered/ShareGPT_V3_unfiltered_cleaned_split.json /tmp
-
-      - name: Run test
-        timeout-minutes: 300
-        env:
-          SGLANG_USE_MODELSCOPE: true
-          HF_ENDPOINT: https://hf-mirror.com
-          SGLANG_IS_IN_CI: true
-          TRANSFORMERS_VERBOSITY: "error"
-          GDN_ATTN_BACKEND_TRITON: 1
-          SGLANG_TEST_METRICS_OUTPUT: /root/.cache/tests/output/metrics/metrics
-        shell: bash
-        run: |
-          sglang_source_path=$(pwd)
-          echo "Source code path: ${sglang_source_path}"
-          ln -sf ${sglang_source_path} /root/sglang
-
-          test_case=${{ inputs.test_case }}
-          if [ ! -f "${sglang_source_path}/${test_case}" ]; then
-            echo "The testcase does not exit: ${sglang_source_path}/${test_case}"
-            exit 1
-          fi
-
-          # prepare for output data
-          current_date=$(date +%Y%m%d)
-          test_data_output_path=/root/.cache/tests/output/${{ inputs.test_type }}/${current_date}
-          mkdir -p ${test_data_output_path}
-          tc_name=${test_case##*/}
-          tc_name=${tc_name%.*}
-          export METRICS_DATA_FILE=${test_data_output_path}/${tc_name}
-          mkdir -p ${METRICS_DATA_FILE}
-          echo "Metrics file path: ${METRICS_DATA_FILE}"
-
-          # copy required file from our daily cache
-          cp ~/.cache/modelscope/hub/datasets/otavia/ShareGPT_Vicuna_unfiltered/ShareGPT_V3_unfiltered_cleaned_split.json /tmp
-          curl -o /tmp/test.jsonl -L https://gh-proxy.test.osinfra.cn/https://raw.githubusercontent.com/openai/grade-school-math/master/grade_school_math/data/test.jsonl
-
-          export TRANSFORMERS_VERSION_FOR_SGLANG="${{ inputs.transformers_version }}"
-          PYTHON_FOR_SGLANG="python"
-          PIP_FOR_SGLANG="pip"
-          if [ -n "${TRANSFORMERS_VERSION_FOR_SGLANG}" ];then
-            echo "===== Install transformers for sglang - Begin ====="
-            TRANSFORMERS_PKG_PATH_SOURCE=/root/.cache/.cache/transformers/${TRANSFORMERS_VERSION_FOR_SGLANG}
-            if [ ! -d "${TRANSFORMERS_PKG_PATH_SOURCE}" ]; then
-              echo "The dependent transformers package does not exist: ${TRANSFORMERS_PKG_PATH_SOURCE}."
-              echo "Install transformers ${TRANSFORMERS_VERSION_FOR_SGLANG} online."
-              pip install transformers=="${TRANSFORMERS_VERSION_FOR_SGLANG}" -i https://mirrors.tuna.tsinghua.edu.cn/pypi/web/simple
-            else
-              echo "Install transformers ${TRANSFORMERS_VERSION_FOR_SGLANG} locally."
-              TRANSFORMERS_PKG_PATH_TARGET=/tmp/transformers/${TRANSFORMERS_VERSION_FOR_SGLANG}
-              mkdir -p "${TRANSFORMERS_PKG_PATH_TARGET}"
-              cp "${TRANSFORMERS_PKG_PATH_SOURCE}/*" "${TRANSFORMERS_PKG_PATH_TARGET}/"
-              pip install --no-index --find-links="${TRANSFORMERS_PKG_PATH_TARGET}" transformers=="${TRANSFORMERS_VERSION_FOR_SGLANG}"
-            fi
-            echo "===== Install transformers for sglang in virtual env - End ====="
-          fi
-          echo "Transformers version for sglang: $(${PIP_FOR_SGLANG} show transformers | grep Version | cut -d: -f2)"
-
-          echo "scaling_governor performance num: \
-            $(cat /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor | grep performance | wc -l)"
-          echo "swappiness: $(cat /proc/sys/vm/swappiness)"
-          echo "numa_balancing: $(cat /proc/sys/kernel/numa_balancing)"
-          echo "sched_migration_cost_ns: $(cat /proc/sys/kernel/sched_migration_cost_ns)"
-
-          export SGLANG_TEST_MAX_RETRY=0
-          export SGLANG_SET_CPU_AFFINITY=1
-          echo "SGLANG_SET_CPU_AFFINITY: $SGLANG_SET_CPU_AFFINITY"
-
-          install_sglang_from_source=${{ inputs.install_sglang_from_source }}
-          if [ "$install_sglang_from_source" = "true" ] || [ "$install_sglang_from_source" = "True" ];then
-            echo "Install sglang from source"
-            commit_id=${{ github.sha }}
-            echo "commit id: ${commit_id}" > ${test_data_output_path}/commit_id
-            export PYTHONPATH=${sglang_source_path}/python:$PYTHONPATH
-          else
-            echo "Use sglang from image: ${{ inputs.image }}"
-            sglang_pkg_path=/sgl-workspace/sglang/python
-            ascend_test_util_path=${sglang_pkg_path}/sglang/test/ascend
-            mkdir -p ${ascend_test_util_path}
-            mv ${ascend_test_util_path} ${ascend_test_util_path}_bak
-            cp -r ${sglang_source_path}/python/sglang/test/ascend ${ascend_test_util_path}
-          fi
-
-          source /usr/local/Ascend/cann/set_env.sh || true
-          source /usr/local/Ascend/nnal/atb/set_env.sh || true
-          source /usr/local/Ascend/ascend-toolkit/latest/opp/vendors/customize/bin/set_env.bash || true
-          source /usr/local/Ascend/ascend-toolkit/latest/opp/vendors/custom_transformer/bin/set_env.bash || true
-
-          # Set environment of cann
-          log_path="/root/.cache/tests/logs/log/${current_date}/${tc_name}/${HOSTNAME}"
-          rm -rf ${log_path}
-          mkdir -p ${log_path}
-          echo "Log path: ${log_path}"
-
-          echo "Running test case ${test_case}"
-          test_exit_code=0
-          ${PYTHON_FOR_SGLANG} -u ${test_case} 2>&1 | tee /tmp/test_output.log || test_exit_code=$?
-          echo "Finished test case ${test_case}"
-
-          if [ "${test_exit_code}" = "0" ]; then
-            test_status="pass"
-            status_icon="✅"
-          else
-            test_status="fail"
-            status_icon="❌"
-          fi
-
-          echo "test_status=${test_status}" >> $GITHUB_ENV
-          echo "tc_name=${tc_name}" >> $GITHUB_ENV
-          export test_status tc_name
-
-          echo "## ${tc_name}  ${status_icon} ${test_status}" >> $GITHUB_STEP_SUMMARY
-          echo "" >> $GITHUB_STEP_SUMMARY
-          metric_count=$(grep -c '\[METRIC\]' /tmp/test_output.log 2>/dev/null || echo 0)
-          if [ "${metric_count}" -gt 0 ]; then
-            echo "| Metric | Value | Pass |" >> $GITHUB_STEP_SUMMARY
-            echo "|--------|-------|------|" >> $GITHUB_STEP_SUMMARY
-            grep '\[METRIC\]' /tmp/test_output.log | while IFS= read -r line; do
-              metric_name=$(echo "$line" | sed -E 's/.*\[METRIC\] ([^=]+)=.*/\1/')
-              metric_value=$(echo "$line" | sed -E 's/.*\[METRIC\] [^=]+=([^ ]+).*/\1/')
-              echo "| ${metric_name} | ${metric_value} | ${status_icon} |" >> $GITHUB_STEP_SUMMARY
-            done
-          else
-            echo "No metrics collected (test may have failed before producing results)." >> $GITHUB_STEP_SUMMARY
-          fi
-          echo "" >> $GITHUB_STEP_SUMMARY
-
-          echo "import json, re, os" > /tmp/dump_metrics.py
-          echo "tc = os.environ.get('tc_name', 'unknown')" >> /tmp/dump_metrics.py
-          echo "st = os.environ.get('test_status', 'unknown')" >> /tmp/dump_metrics.py
-          echo "tp = '${{ inputs.test_type }}'" >> /tmp/dump_metrics.py
-          echo "metrics = {}" >> /tmp/dump_metrics.py
-          echo "with open('/tmp/test_output.log') as f:" >> /tmp/dump_metrics.py
-          echo "    for line in f:" >> /tmp/dump_metrics.py
-          echo "        m = re.match(r'\[METRIC\] (\S+)=(\S+)', line)" >> /tmp/dump_metrics.py
-          echo "        if m:" >> /tmp/dump_metrics.py
-          echo "            v = m.group(2)" >> /tmp/dump_metrics.py
-          echo "            try:" >> /tmp/dump_metrics.py
-          echo "                v = float(v)" >> /tmp/dump_metrics.py
-          echo "            except ValueError:" >> /tmp/dump_metrics.py
-          echo "                pass" >> /tmp/dump_metrics.py
-          echo "            metrics[m.group(1)] = v" >> /tmp/dump_metrics.py
-          echo "baselines = {}" >> /tmp/dump_metrics.py
-          echo "for k, v in list(metrics.items()):" >> /tmp/dump_metrics.py
-          echo "    if k.endswith('_baseline'):" >> /tmp/dump_metrics.py
-          echo "        baselines[k[:-9]] = v" >> /tmp/dump_metrics.py
-          echo "        del metrics[k]" >> /tmp/dump_metrics.py
-          echo "with open('/tmp/metrics.json', 'w') as f:" >> /tmp/dump_metrics.py
-          echo "    json.dump({'test_case': tc, 'test_type': tp, 'status': st, 'metrics': metrics, 'baselines': baselines}, f)" >> /tmp/dump_metrics.py
-          python3 /tmp/dump_metrics.py
-          exit ${test_exit_code}
-
-      - name: Upload metrics
-        if: always()
-        uses: actions/upload-artifact@v4
-        with:
-          name: metrics-${{ inputs.test_config_name }}
-          path: /tmp/metrics.json
-          retention-days: 7
-
-      - name: Backup plog
-        if: always()
-        shell: bash
-        run: |
-          plog_path="/root/ascend/log/debug/plog"
-          if [ -d "$plog_path" ];then
-            echo "Plog files found. Begin to backup them."
-            tc_name=${{ inputs.test_case }}
-            tc_name=${tc_name##*/}
-            tc_name=${tc_name%.*}
-            target_plog_path="/root/.cache/tests/logs/plog/${tc_name}/${HOSTNAME}"
-            echo "Save path: ${target_plog_path}"
-            rm -rf ${target_plog_path}
-            mkdir -p ${target_plog_path}
-            cp ${plog_path}/* ${target_plog_path}
-          fi
@@ -1,4 +1,4 @@
-name: Nightly Test (NPU)
+name: Nightly Test (NPU)
 on:
  schedule:
    - cron: '0 16 * * *'  # Execute at 0:00 a.m. Beijing Time every day
@@ -73,10 +73,15 @@ jobs:
      image_a2: ${{ steps.set-vars.outputs.image_a2 }}
      image_a3: ${{ steps.set-vars.outputs.image_a3 }}
      skip_install_flag: ${{ steps.set-vars.outputs.skip_install_flag }}
+      run_start_metadata: ${{ steps.set-vars.outputs.run_start_metadata }}
    steps:
      # When triggered by PR, no inputs parameters are used. The latest community code is tested by default.
      - name: Set image config
        id: set-vars
+        env:
+          # Pass untrusted PR metadata through environment variables to avoid shell‑script interpolation,
+          # mitigating script injection risks from fork branch names.
+          HEAD_LABEL: ${{ github.event.pull_request.head.label }}
        run: |
          if [ -z "${{ inputs.ref }}" ]; then
            echo "ref=" >> $GITHUB_OUTPUT
@@ -103,211 +108,231 @@ jobs:
          else
            echo "skip_install_flag=${{ inputs.skip_install_flag }}" >> $GITHUB_OUTPUT
          fi
-  nightly-poc-single-node-a2-tests:
-    name: single-node-poc-a2
-    if: ${{ !cancelled() }}
-    needs: [ set-image-config ]
-    strategy:
-      fail-fast: false
-      max-parallel: 6
-      matrix:
-        test_config:
-          - name: qwen3_32b_w8a8_2p_in3k5_out1k5_50ms_a2
-            runner: linux-aarch64-a2-4
-            test_case: test/registered/npu/performance/qwen3_32b/test_npu_qwen3_32b_w8a8_2p_in3k5_out1k5_50ms_a2.py
-            test_type: 'perf'
-          - name: qwen3_32b_w8a8_2p_in3k5_out1k5_50ms_gpqa_a2
-            runner: linux-aarch64-a2-4
-            test_case: test/registered/npu/accuracy/qwen3_32b/test_npu_qwen3_32b_w8a8_2p_in3k5_out1k5_50ms_gpqa_a2.py
-    uses: ./.github/workflows/nightly-test-npu-e2e-single-node.yml
-    with:
-      runner: ${{ matrix.test_config.runner }}
-      test_type: ${{ matrix.test_config.test_type }}
-      test_config_name: ${{ matrix.test_config.name }}
-      test_case: ${{ matrix.test_config.test_case }}
-      image: ${{ needs.set-image-config.outputs.image_a2 }}
-      install_sglang_from_source: false
-      transformers_version: ''
+          # If the nightly pipeline is triggered by a PR, this variable takes the source branch label of the PR;
+          # otherwise, it uses the branch or tag name of the current ref.
+          BRANCH_LABEL="$HEAD_LABEL"
+          if [ -z "${BRANCH_LABEL}" ]; then
+            BRANCH_LABEL="${{ github.ref_name }}"
+          fi
+          BRANCH_LABEL="$(echo "${BRANCH_LABEL}" | tr -cd 'a-zA-Z0-9._/:' | tr '/:' '--')"
+          # Package run metadata (branch/workflow/create_time) as a single JSON for reuse by all
+          # test jobs; create_time is UTC+8 date + time (minute precision) recorded once at run
+          # start, shared even across midnight.
+          RUN_START_METADATA="$(python3 -c "import json,sys; print(json.dumps({'branch_label':sys.argv[1],'workflow_name':sys.argv[2],'create_time':sys.argv[3]}))" "$BRANCH_LABEL" "${{ github.workflow }}" "$(date -u -d '+8 hours' +%Y%m%d-%H%M)")"
+          # Write to GITHUB_OUTPUT and print to the log in one command.
+          echo "run_start_metadata=${RUN_START_METADATA}" | tee -a $GITHUB_OUTPUT

-  nightly-poc-single-node-tests:
-    name: single-node-poc
+  nightly-1-npu-a2:
+    name: nightly-1-npu-a2
    if: ${{ !cancelled() }}
    needs: [set-image-config]
-    strategy:
-      fail-fast: false
-      max-parallel: 6
-      matrix:
-        test_config:
-          # qwen3_6_35b_a3b performance tests
-          - name: qwen3_6_35b_a3b_1p_in3k5_out1k5_50ms
-            runner: linux-aarch64-a3-2
-            test_case: test/registered/npu/performance/qwen3_6_35b_a3b/test_npu_qwen3_6_35b_a3b_1p_in3k5_out1k5_50ms.py
-            test_type: 'perf'
-          - name: qwen3_6_35b_a3b_1p_aime26
-            runner: linux-aarch64-a3-2
-            test_case: test/registered/npu/accuracy/qwen3_6_35b_a3b/test_npu_qwen3_6_35b_a3b_1p_aime26.py
-          - name: qwen3_6_35b_a3b_1p_in64k_out1k_50ms
-            runner: linux-aarch64-a3-2
-            test_case: test/registered/npu/performance/qwen3_6_35b_a3b/test_npu_qwen3_6_35b_a3b_1p_in64k_out1k_50ms.py
-            test_type: 'perf'
-          - name: qwen3_6_35b_a3b_1p_in128k_out1k_50ms
-            runner: linux-aarch64-a3-2
-            test_case: test/registered/npu/performance/qwen3_6_35b_a3b/test_npu_qwen3_6_35b_a3b_1p_in128k_out1k_50ms.py
-            test_type: 'perf'
-          - name: qwen3_6_35b_a3b_1p_in64k_out1k_prefix90_50ms
-            runner: linux-aarch64-a3-2
-            test_case: test/registered/npu/performance/qwen3_6_35b_a3b/test_npu_qwen3_6_35b_a3b_1p_in64k_out1k_prefix90_50ms.py
-            test_type: 'perf'
-          - name: qwen3_6_35b_a3b_1p_in64k_out1k_prefix90_50ms_aime26
-            runner: linux-aarch64-a3-2
-            test_case: test/registered/npu/accuracy/qwen3_6_35b_a3b/test_npu_qwen3_6_35b_a3b_1p_in64k_out1k_prefix90_50ms_aime26.py
-          - name: qwen3_6_35b_a3b_1p_in128k_out1k_prefix90_50ms
-            runner: linux-aarch64-a3-2
-            test_case: test/registered/npu/performance/qwen3_6_35b_a3b/test_npu_qwen3_6_35b_a3b_1p_in128k_out1k_prefix90_50ms.py
-            test_type: 'perf'
-          # qwen3_6_27b performance tests
-          - name: qwen3_6_27b_w8a8_1p_in3k5_out1k5_50ms
-            runner: linux-aarch64-a3-2
-            test_case: test/registered/npu/performance/qwen3_6_27b/test_npu_qwen3_6_27b_w8a8_1p_in3k5_out1k5_50ms.py
-            test_type: 'perf'
-          - name: qwen3_6_27b_w8a8_1p_in3k5_out1k5_50ms_gpqa
-            runner: linux-aarch64-a3-2
-            test_case: test/registered/npu/accuracy/qwen3_6_27b/test_npu_qwen3_6_27b_w8a8_1p_in3k5_out1k5_50ms_gpqa.py
-          - name: qwen3_6_27b_w8a8_2p_in16k_out1k_50ms
-            runner: linux-aarch64-a3-4
-            test_case: test/registered/npu/performance/qwen3_6_27b/test_npu_qwen3_6_27b_w8a8_2p_in16k_out1k_50ms.py
-            test_type: 'perf'
-          - name: qwen3_6_27b_w8a8_1p_in64k_out1k_50ms
-            runner: linux-aarch64-a3-4
-            test_case: test/registered/npu/performance/qwen3_6_27b/test_npu_qwen3_6_27b_w8a8_1p_in64k_out1k_50ms.py
-            test_type: 'perf'
-          - name: qwen3_6_27b_w8a8_2p_in128k_out1k_50ms
-            runner: linux-aarch64-a3-4
-            test_case: test/registered/npu/performance/qwen3_6_27b/test_npu_qwen3_6_27b_w8a8_2p_in128k_out1k_50ms.py
-            test_type: 'perf'
-          - name: qwen3_6_27b_2p_in64k_out1k_prefix90_50ms
-            runner: linux-aarch64-a3-4
-            test_case: test/registered/npu/performance/qwen3_6_27b/test_npu_qwen3_6_27b_2p_in64k_out1k_prefix90_50ms.py
-            test_type: 'perf'
-          - name: qwen3_6_27b_1p_in1024x1024_30_out1024_50ms
-            runner: linux-aarch64-a3-2
-            test_case: test/registered/npu/performance/qwen3_6_27b/test_npu_qwen3_6_27b_1p_in1024x1024_30_out1024_50ms.py
-            test_type: 'perf'
-          - name: qwen3_6_27b_1p_in1080p_30_out256_50ms
-            runner: linux-aarch64-a3-2
-            test_case: test/registered/npu/performance/qwen3_6_27b/test_npu_qwen3_6_27b_1p_in1080p_30_out256_50ms.py
-            test_type: 'perf'
-          - name: qwen3_6_27b_1p_gpqa
-            runner: linux-aarch64-a3-2
-            test_case: test/registered/npu/accuracy/qwen3_6_27b/test_npu_qwen3_6_27b_1p_gpqa.py
-          # qwen3_32b performance tests
-          - name: qwen3_32b_w8a8_2p_in3k5_out1k5_50ms
-            runner: linux-aarch64-a3-4
-            test_case: test/registered/npu/performance/qwen3_32b/test_npu_qwen3_32b_w8a8_2p_in3k5_out1k5_50ms.py
-            test_type: 'perf'
-          - name: qwen3_32b_w8a8_2p_in3k5_out1k5_50ms_gpqa
-            runner: linux-aarch64-a3-4
-            test_case: test/registered/npu/accuracy/qwen3_32b/test_npu_qwen3_32b_w8a8_2p_in3k5_out1k5_50ms_gpqa.py
-          - name: qwen3_32b_bf16_8p_in18k_out4k_6ms
-            runner: linux-aarch64-a3-16
-            test_case: test/registered/npu/performance/qwen3_32b/test_npu_qwen3_32b_bf16_8p_in18k_out4k_6ms.py
-            test_type: 'perf'
-          - name: qwen3_32b_bf16_8p_gpqa
-            runner: linux-aarch64-a3-16
-            test_case: test/registered/npu/accuracy/qwen3_32b/test_npu_qwen3_32b_bf16_8p_gpqa.py
-          # qwen3_30b_a3b performance tests
-          - name: qwen3_30b_w8a8_1p_in3k5_out1k5_50ms
-            runner: linux-aarch64-a3-2
-            test_case: test/registered/npu/performance/qwen3_30b_a3b/test_npu_qwen3_30b_w8a8_1p_in3k5_out1k5_50ms.py
-            test_type: 'perf'
-          - name: qwen3_30b_w8a8_1p_in3k5_out1k5_50ms_aime25
-            runner: linux-aarch64-a3-2
-            test_case: test/registered/npu/accuracy/qwen3_30b_a3b/test_npu_qwen3_30b_w8a8_1p_in3k5_out1k5_50ms_aime25.py
-          # qwen3-8b performance tests
-          - name: qwen3_8b_w8a8_1p_in3k5_out1k5_50ms
-            runner: linux-aarch64-a3-2
-            test_case: test/registered/npu/performance/qwen3-8b/test_npu_qwen3_8b_w8a8_1p_in3k5_out1k5_50ms.py
-            test_type: 'perf'
-          - name: qwen3_8b_w8a8_1p_in3k5_out1k5_50ms_gpqa
-            runner: linux-aarch64-a3-2
-            test_case: test/registered/npu/accuracy/qwen3-8b/test_npu_qwen3_8b_w8a8_1p_in3k5_out1k5_50ms_gpqa.py
-          - name: qwen3_8b_w8a8_1p_in6k_out1k5_bs16
-            runner: linux-aarch64-a3-2
-            test_case: test/registered/npu/performance/qwen3-8b/test_npu_qwen3_8b_w8a8_1p_in6k_out1k5_bs16.py
-            test_type: 'perf'
-          - name: qwen3_8b_w8a8_1p_in6k_out1k5_bs16_gpqa
-            runner: linux-aarch64-a3-2
-            test_case: test/registered/npu/accuracy/qwen3-8b/test_npu_qwen3_8b_w8a8_1p_in6k_out1k5_bs16_gpqa.py
-          # qwen3_next_80b_a3b_instruct performance tests
-          - name: qwen3_next_80b_w8a8_2p_in6k_out1k5_bs16
-            runner: linux-aarch64-a3-4
-            test_case: test/registered/npu/performance/qwen3_next_80b_a3b_instruct/test_npu_qwen3_next_80b_w8a8_2p_in6k_out1k5_bs16.py
-            test_type: 'perf'
-          - name: qwen3_next_80b_w8a8_2p_in6k_out1k5_bs16_aime25
-            runner: linux-aarch64-a3-4
-            test_case: test/registered/npu/accuracy/qwen3_next_80b_a3b_instruct/test_npu_qwen3_next_80b_w8a8_2p_in6k_out1k5_bs16_aime25.py
-          # minimax_m2_5 performance tests
-          - name: minimax_m2_5_w8a8_8p_in3k5_out1k5_50ms
-            runner: linux-aarch64-a3-16
-            test_case: test/registered/npu/performance/minimax_m2_5/test_npu_minimax_m2_5_w8a8_8p_in3k5_out1k5_50ms.py
-            test_type: 'perf'
-          - name: minimax_m2_5_w8a8_8p_in3k5_out1k5_50ms_gpqa
-            runner: linux-aarch64-a3-16
-            test_case: test/registered/npu/accuracy/minimax_m2_5/test_npu_minimax_m2_5_w8a8_8p_in3k5_out1k5_50ms_gpqa.py
-          - name: minimax_m2_5_w8a8_4p_in64k_out1k_prefix90_50ms
-            runner: linux-aarch64-a3-16
-            test_case: test/registered/npu/performance/minimax_m2_5/test_npu_minimax_m2_5_w8a8_4p_in64k_out1k_prefix90_50ms.py
-            test_type: 'perf'
-          - name: minimax_m2_5_w8a8_4p_in64k_out1k_prefix90_50ms_gpqa
-            runner: linux-aarch64-a3-16
-            test_case: test/registered/npu/accuracy/minimax_m2_5/test_npu_minimax_m2_5_w8a8_4p_in64k_out1k_prefix90_50ms_gpqa.py
-          # deepseek_v3_2 accuracy tests
-          - name: deepseek_v3_2_8p_aime25
-            runner: linux-aarch64-a3-16
-            test_case: test/registered/npu/accuracy/deepseek_v3_2/test_npu_deepseek_v3_2_8p_aime25.py
-          # glm4_7_flash accuracy tests
-          - name: glm4_7_flash_1p_aime25
-            runner: linux-aarch64-a3-2
-            test_case: test/registered/npu/accuracy/glm4_7_flash/test_npu_glm4_7_flash_1p_aime25.py
-          # qwen3_vl_8b_thinking accuracy tests
-          - name: qwen3_vl_8b_thinking_1p_mmmu
-            runner: linux-aarch64-a3-2
-            test_case: test/registered/npu/accuracy/qwen3_vl_8b_thinking/test_npu_qwen3_vl_8b_thinking_1p_mmmu.py
-          # qwen3_vl_30b_a3b_thinking accuracy tests
-          - name: qwen3_vl_30b_a3b_thinking_1p_mmmu
-            runner: linux-aarch64-a3-2
-            test_case: test/registered/npu/accuracy/qwen3_vl_30b_a3b_thinking/test_npu_qwen3_vl_30b_a3b_thinking_1p_mmmu.py
-          - name: qwen3_235b_w8a8_8p_in3k5_out1k5_50ms
-            runner: linux-aarch64-a3-16
-            test_case: test/registered/npu/performance/qwen3_235b_a22b/test_npu_qwen3_235b_w8a8_8p_in3k5_out1k5_50ms.py
-            test_type: 'perf'
-          - name: qwen3_5_397b_w4a8_8p_in3k5_out1k5_50ms
-            runner: linux-aarch64-a3-16
-            test_case: test/registered/npu/performance/qwen3_5_397b/test_npu_qwen3_5_397b_w4a8_8p_in3k5_out1k5_50ms.py
-            test_type: 'perf'
-          - name: kimi_k2_6_w4a8_8p_in3k5_out1k5_20ms
-            runner: linux-aarch64-a3-16
-            test_case: test/registered/npu/performance/kimi_k2_6/test_npu_kimi_k2_6_w4a8_8p_in3k5_out1k5_20ms.py
-            test_type: 'perf'
-          # deepseek_v4_flash performance tests
-          - name: deepseek_v4_flash_w8a8_8p_in8k_out1k_50ms
-            runner: linux-aarch64-a3-16
-            test_case: test/registered/npu/performance/deepseek_v4_flash/test_npu_deepseek_v4_flash_w8a8_8p_in8k_out1k_50ms.py
-            test_type: 'perf'
-    uses: ./.github/workflows/nightly-test-npu-e2e-single-node.yml
+    uses: ./.github/workflows/_npu-single-node-test-stage.yml
    with:
-      runner: ${{ matrix.test_config.runner }}
-      test_type: ${{ matrix.test_config.test_type }}
-      test_config_name: ${{ matrix.test_config.name }}
-      test_case: ${{ matrix.test_config.test_case }}
+      runner: linux-aarch64-a2-4
+      test_type: 'perf'
+      test_suite: nightly-1-npu-a2
+      is_nightly_pipeline_job: true
+      skip_pr_test_health_check: 'true'
+      image: ${{ needs.set-image-config.outputs.image_a2 }}
+      install_sglang_deps: false
+      device_type_for_deps: '910b'
+      run_start_metadata: ${{ needs.set-image-config.outputs.run_start_metadata }}
+
+  nightly-1-npu-a3:
+    name: nightly-1-npu-a3
+    if: ${{ !cancelled() }}
+    needs: [set-image-config]
+    uses: ./.github/workflows/_npu-pr-test-stage.yml
+    with:
+      self_name: nightly-1-npu-a3
+      runner_config: linux-aarch64-a3-2-
      image: ${{ needs.set-image-config.outputs.image_a3 }}
-      install_sglang_from_source: false
-      transformers_version: ''
+      run_timeout_minutes: '60'
+      timeout_per_file: '3600'
+      is_nightly_pipeline_job: true
+      skip_pr_test_health_check: 'true'
+    secrets: inherit
+
+  nightly-2-npu-a3:
+    name: nightly-2-npu-a3
+    if: ${{ !cancelled() }}
+    needs: [set-image-config]
+    uses: ./.github/workflows/_npu-pr-test-stage.yml
+    with:
+      self_name: nightly-2-npu-a3
+      runner_config: linux-aarch64-a3-2-
+      image: ${{ needs.set-image-config.outputs.image_a3 }}
+      run_timeout_minutes: '60'
+      timeout_per_file: '3600'
+      is_nightly_pipeline_job: true
+      skip_pr_test_health_check: 'true'
+    secrets: inherit
+
+  # The nightly-4-npu-a3 suite is split across 2 parallel partition jobs, mirroring base-b-test-4-npu-a3 in the PR pipeline.
+  nightly-4-npu-a3:
+    name: nightly-4-npu-a3
+    if: ${{ !cancelled() }}
+    needs: [set-image-config]
+    uses: ./.github/workflows/_npu-pr-test-stage.yml
+    with:
+      self_name: nightly-4-npu-a3
+      runner_config: linux-aarch64-a3-4-
+      image: ${{ needs.set-image-config.outputs.image_a3 }}
+      run_timeout_minutes: '120'
+      timeout_per_file: '3600'
+      partitions: '{"size":2,"arr":[0,1]}'
+      is_nightly_pipeline_job: true
+      skip_pr_test_health_check: 'true'
+    secrets: inherit
+
+  nightly-8-npu-a3:
+    name: nightly-8-npu-a3
+    if: ${{ !cancelled() }}
+    needs: [set-image-config]
+    uses: ./.github/workflows/_npu-pr-test-stage.yml
+    with:
+      self_name: nightly-8-npu-a3
+      runner_config: linux-aarch64-a3-8-
+      image: ${{ needs.set-image-config.outputs.image_a3 }}
+      run_timeout_minutes: '60'
+      timeout_per_file: '3600'
+      is_nightly_pipeline_job: true
+      skip_pr_test_health_check: 'true'
+    secrets: inherit
+
+  nightly-16-npu-a3:
+    name: nightly-16-npu-a3
+    if: ${{ !cancelled() }}
+    needs: [set-image-config]
+    uses: ./.github/workflows/_npu-pr-test-stage.yml
+    with:
+      self_name: nightly-16-npu-a3
+      runner_config: linux-aarch64-a3-16-
+      image: ${{ needs.set-image-config.outputs.image_a3 }}
+      run_timeout_minutes: '120'
+      timeout_per_file: '3600'
+      is_nightly_pipeline_job: true
+      skip_pr_test_health_check: 'true'
+    secrets: inherit
+
+  nightly-perf-2-npu-a3:
+    name: nightly-perf-2-npu-a3
+    if: ${{ !cancelled() }}
+    needs: [set-image-config]
+    uses: ./.github/workflows/_npu-single-node-test-stage.yml
+    with:
+      runner: linux-aarch64-a3-800t-2
+      test_type: 'perf'
+      test_suite: nightly-perf-2-npu-a3
+      is_nightly_pipeline_job: true
+      skip_pr_test_health_check: 'true'
+      image: ${{ needs.set-image-config.outputs.image_a3 }}
+      install_sglang_deps: false
+      device_type_for_deps: 'a3'
+      run_start_metadata: ${{ needs.set-image-config.outputs.run_start_metadata }}
+
+  nightly-perf-4-npu-a3:
+    name: nightly-perf-4-npu-a3
+    if: ${{ !cancelled() }}
+    needs: [set-image-config]
+    uses: ./.github/workflows/_npu-single-node-test-stage.yml
+    with:
+      runner: linux-aarch64-a3-800t-4
+      test_type: 'perf'
+      test_suite: nightly-perf-4-npu-a3
+      is_nightly_pipeline_job: true
+      skip_pr_test_health_check: 'true'
+      image: ${{ needs.set-image-config.outputs.image_a3 }}
+      install_sglang_deps: false
+      device_type_for_deps: 'a3'
+      run_start_metadata: ${{ needs.set-image-config.outputs.run_start_metadata }}
+
+  nightly-perf-8-npu-a3:
+    name: nightly-perf-8-npu-a3
+    if: ${{ !cancelled() }}
+    needs: [set-image-config]
+    uses: ./.github/workflows/_npu-single-node-test-stage.yml
+    with:
+      runner: linux-aarch64-a3-800t-8
+      test_type: 'perf'
+      test_suite: nightly-perf-8-npu-a3
+      is_nightly_pipeline_job: true
+      skip_pr_test_health_check: 'true'
+      image: ${{ needs.set-image-config.outputs.image_a3 }}
+      install_sglang_deps: false
+      device_type_for_deps: 'a3'
+      run_start_metadata: ${{ needs.set-image-config.outputs.run_start_metadata }}
+
+  nightly-perf-16-npu-a3:
+    name: nightly-perf-16-npu-a3
+    if: ${{ !cancelled() }}
+    needs: [set-image-config]
+    uses: ./.github/workflows/_npu-single-node-test-stage.yml
+    with:
+      runner: linux-aarch64-a3-800t-16
+      test_type: 'perf'
+      test_suite: nightly-perf-16-npu-a3
+      is_nightly_pipeline_job: true
+      skip_pr_test_health_check: 'true'
+      image: ${{ needs.set-image-config.outputs.image_a3 }}
+      install_sglang_deps: false
+      device_type_for_deps: 'a3'
+      run_start_metadata: ${{ needs.set-image-config.outputs.run_start_metadata }}
+
+  # The acc-2 suite is split across 4 parallel partition jobs via the single `partitions` input.
+  nightly-acc-2-npu-a3:
+    name: nightly-acc-2-npu-a3
+    if: ${{ !cancelled() }}
+    needs: [set-image-config]
+    uses: ./.github/workflows/_npu-single-node-test-stage.yml
+    with:
+      runner: linux-aarch64-a3-2-
+      test_type: 'accuracy'
+      test_suite: nightly-acc-2-npu-a3
+      is_nightly_pipeline_job: true
+      partitions: '{"size":4,"arr":[0,1,2,3]}'
+      skip_pr_test_health_check: 'true'
+      image: ${{ needs.set-image-config.outputs.image_a3 }}
+      install_sglang_deps: false
+      device_type_for_deps: 'a3'
+      run_start_metadata: ${{ needs.set-image-config.outputs.run_start_metadata }}
+
+  nightly-acc-4-npu-a3:
+    name: nightly-acc-4-npu-a3
+    if: ${{ !cancelled() }}
+    needs: [set-image-config]
+    uses: ./.github/workflows/_npu-single-node-test-stage.yml
+    with:
+      runner: linux-aarch64-a3-4-
+      test_type: 'accuracy'
+      test_suite: nightly-acc-4-npu-a3
+      is_nightly_pipeline_job: true
+      skip_pr_test_health_check: 'true'
+      image: ${{ needs.set-image-config.outputs.image_a3 }}
+      install_sglang_deps: false
+      device_type_for_deps: 'a3'
+      run_start_metadata: ${{ needs.set-image-config.outputs.run_start_metadata }}
+
+  nightly-acc-16-npu-a3:
+    name: nightly-acc-16-npu-a3
+    if: ${{ !cancelled() }}
+    needs: [set-image-config]
+    uses: ./.github/workflows/_npu-single-node-test-stage.yml
+    with:
+      runner: linux-aarch64-a3-16-
+      test_type: 'accuracy'
+      test_suite: nightly-acc-16-npu-a3
+      is_nightly_pipeline_job: true
+      skip_pr_test_health_check: 'true'
+      image: ${{ needs.set-image-config.outputs.image_a3 }}
+      install_sglang_deps: false
+      device_type_for_deps: 'a3'
+      run_start_metadata: ${{ needs.set-image-config.outputs.run_start_metadata }}

  nightly-poc-multi-node-tests:
    name: multi-node-poc
    if: ${{ !cancelled() }}
-    needs: [set-image-config, nightly-poc-single-node-tests]
+    needs: [set-image-config, nightly-perf-2-npu-a3, nightly-perf-4-npu-a3, nightly-perf-16-npu-a3, nightly-acc-2-npu-a3, nightly-acc-4-npu-a3, nightly-acc-16-npu-a3]
    strategy:
      fail-fast: false
      max-parallel: 1
@@ -326,6 +351,7 @@ jobs:
            decode_size: 2
            router_size: 1
            test_case: test/registered/npu/accuracy/glm5_1/test_npu_glm5_1_w4a8_1p1d_32p_in64k_out1k_50ms_aime26.py
+            test_type: 'accuracy'
            prefill_decode_deployment: 'separation'
          # mimo_v2_flash performance tests
          - name: test_npu_mimo_v2_flash_1p1d_12p_in16k_out1_ttft_5s
@@ -344,8 +370,8 @@ jobs:
            prefill_decode_deployment: 'separation'
    uses: ./.github/workflows/nightly-test-npu-e2e-multi-node.yml
    with:
-      runner: linux-amd64-cpu-8
-      test_type: ${{ matrix.test_config.test_type }}
+      runner: linux-amd64-cpu-4
+      test_type: ${{ matrix.test_config.test_type || 'perf' }}
      test_config_name: ${{ matrix.test_config.name }}
      prefill_size: ${{ matrix.test_config.prefill_size }}
      decode_size: ${{ matrix.test_config.decode_size }}
@@ -355,11 +381,12 @@ jobs:
      install_sglang_from_source: false
      prefill_decode_deployment: ${{ matrix.test_config.prefill_decode_deployment }}
      transformers_version: ''
+      run_start_metadata: ${{ needs.set-image-config.outputs.run_start_metadata }}

  nightly-poc-multi-node-mix-tests:
    name: multi-node-mix-poc
    if: ${{ !cancelled() }}
-    needs: [set-image-config, nightly-poc-single-node-tests, nightly-poc-multi-node-tests]
+    needs: [set-image-config, nightly-perf-2-npu-a3, nightly-perf-4-npu-a3, nightly-perf-16-npu-a3, nightly-acc-2-npu-a3, nightly-acc-4-npu-a3, nightly-acc-16-npu-a3, nightly-poc-multi-node-tests]
    strategy:
      fail-fast: false
      max-parallel: 1
@@ -373,10 +400,11 @@ jobs:
          - name: kimi_k2_6_w4a8_16p_in64k_out1k_100ms_aime25
            node_size: 2
            test_case: test/registered/npu/accuracy/kimi_k2_6/test_npu_kimi_k2_6_w4a8_16p_in64k_out1k_100ms_aime25.py
+            test_type: 'accuracy'
    uses: ./.github/workflows/nightly-test-npu-e2e-multi-node.yml
    with:
-      runner: linux-amd64-cpu-8
-      test_type: ${{ matrix.test_config.test_type }}
+      runner: linux-amd64-cpu-4
+      test_type: ${{ matrix.test_config.test_type || 'perf' }}
      test_config_name: ${{ matrix.test_config.name }}
      node_size: ${{ matrix.test_config.node_size }}
      test_case: ${{ matrix.test_config.test_case }}
@@ -384,11 +412,24 @@ jobs:
      install_sglang_from_source: false
      prefill_decode_deployment: 'mix'
      transformers_version: ''
+      run_start_metadata: ${{ needs.set-image-config.outputs.run_start_metadata }}
+
  check-all-jobs:
    if: ${{ !cancelled() }}
    needs:
-      - nightly-poc-single-node-a2-tests
-      - nightly-poc-single-node-tests
+      - nightly-1-npu-a2
+      - nightly-1-npu-a3
+      - nightly-2-npu-a3
+      - nightly-4-npu-a3
+      - nightly-8-npu-a3
+      - nightly-16-npu-a3
+      - nightly-perf-2-npu-a3
+      - nightly-perf-4-npu-a3
+      - nightly-perf-8-npu-a3
+      - nightly-perf-16-npu-a3
+      - nightly-acc-2-npu-a3
+      - nightly-acc-4-npu-a3
+      - nightly-acc-16-npu-a3
      - nightly-poc-multi-node-tests
      - nightly-poc-multi-node-mix-tests
    runs-on: ubuntu-latest
@@ -402,19 +443,30 @@ jobs:

      - name: Generate results table
        run: |
-          status_emoji() {
-            case "$1" in
-              pass)   echo "✅" ;;
-              fail)   echo "❌" ;;
-              *)      echo "❓" ;;
-            esac
-          }
-
-          single_result_a2="${{ needs.nightly-poc-single-node-a2-tests.result }}"
-          single_result="${{ needs.nightly-poc-single-node-tests.result }}"
+          single_result_a2="${{ needs.nightly-1-npu-a2.result }}"
          multi_result="${{ needs.nightly-poc-multi-node-tests.result }}"
          mix_result="${{ needs.nightly-poc-multi-node-mix-tests.result }}"

+          # single-node suites are reported per suite; overall status is success if none failed
+          single_result="success"
+          for r in \
+            "${{ needs.nightly-1-npu-a3.result }}" \
+            "${{ needs.nightly-2-npu-a3.result }}" \
+            "${{ needs.nightly-4-npu-a3.result }}" \
+            "${{ needs.nightly-8-npu-a3.result }}" \
+            "${{ needs.nightly-16-npu-a3.result }}" \
+            "${{ needs.nightly-perf-2-npu-a3.result }}" \
+            "${{ needs.nightly-perf-4-npu-a3.result }}" \
+            "${{ needs.nightly-perf-8-npu-a3.result }}" \
+            "${{ needs.nightly-perf-16-npu-a3.result }}" \
+            "${{ needs.nightly-acc-2-npu-a3.result }}" \
+            "${{ needs.nightly-acc-4-npu-a3.result }}" \
+            "${{ needs.nightly-acc-16-npu-a3.result }}"; do
+            if [ "${r}" != "success" ] && [ "${r}" != "skipped" ]; then
+              single_result="failure"
+            fi
+          done
+
          group_icon() {
            case "$1" in
              success)   echo "✅" ;;
@@ -429,8 +481,24 @@ jobs:
          echo "" >> $GITHUB_STEP_SUMMARY
          echo "| Group | Status |" >> $GITHUB_STEP_SUMMARY
          echo "|-------|--------|" >> $GITHUB_STEP_SUMMARY
-          echo "| single-node-poc-a2 | $(group_icon ${single_result_a2}) ${single_result_a2} |" >> $GITHUB_STEP_SUMMARY
-          echo "| single-node-poc | $(group_icon ${single_result}) ${single_result} |" >> $GITHUB_STEP_SUMMARY
+          echo "| nightly-1-npu-a2 | $(group_icon ${single_result_a2}) ${single_result_a2} |" >> $GITHUB_STEP_SUMMARY
+          for entry in \
+            "nightly-1-npu-a3:${{ needs.nightly-1-npu-a3.result }}" \
+            "nightly-2-npu-a3:${{ needs.nightly-2-npu-a3.result }}" \
+            "nightly-4-npu-a3:${{ needs.nightly-4-npu-a3.result }}" \
+            "nightly-8-npu-a3:${{ needs.nightly-8-npu-a3.result }}" \
+            "nightly-16-npu-a3:${{ needs.nightly-16-npu-a3.result }}" \
+            "nightly-perf-2-npu-a3:${{ needs.nightly-perf-2-npu-a3.result }}" \
+            "nightly-perf-4-npu-a3:${{ needs.nightly-perf-4-npu-a3.result }}" \
+            "nightly-perf-8-npu-a3:${{ needs.nightly-perf-8-npu-a3.result }}" \
+            "nightly-perf-16-npu-a3:${{ needs.nightly-perf-16-npu-a3.result }}" \
+            "nightly-acc-2-npu-a3:${{ needs.nightly-acc-2-npu-a3.result }}" \
+            "nightly-acc-4-npu-a3:${{ needs.nightly-acc-4-npu-a3.result }}" \
+            "nightly-acc-16-npu-a3:${{ needs.nightly-acc-16-npu-a3.result }}"; do
+            suite="${entry%%:*}"
+            r="${entry##*:}"
+            echo "| ${suite} | $(group_icon ${r}) ${r} |" >> $GITHUB_STEP_SUMMARY
+          done
          echo "| multi-node-poc | $(group_icon ${multi_result}) ${multi_result} |" >> $GITHUB_STEP_SUMMARY
          echo "| multi-node-mix-poc | $(group_icon ${mix_result}) ${mix_result} |" >> $GITHUB_STEP_SUMMARY
          echo "" >> $GITHUB_STEP_SUMMARY
@@ -439,8 +507,11 @@ jobs:
          echo "" >> $GITHUB_STEP_SUMMARY

          has_metrics=false
-          for dir in /tmp/metrics/metrics-*/; do
-            file="${dir}metrics.json"
+          # Each suite job uploads its METRICS_DATA_FILE directory, which
+          # contains per-case subdirectories with their own metrics.json.
+          # Recursively find every metrics.json and render one table row per
+          # test case.
+          for file in $(find /tmp/metrics -name metrics.json 2>/dev/null); do
            if [ -f "$file" ]; then
              has_metrics=true
              echo "import json" > /tmp/parse_metrics.py
@@ -469,5 +540,7 @@ jobs:
          echo "" >> $GITHUB_STEP_SUMMARY

          FAIL=0
-          if [ "${single_result}" != "success" ] && [ "${single_result}" != "skipped" ]; then FAIL=1; fi
+          for r in "${single_result_a2}" "${single_result}" "${multi_result}" "${mix_result}"; do
+            if [ "${r}" != "success" ] && [ "${r}" != "skipped" ]; then FAIL=1; fi
+          done
          exit $FAIL
@@ -60,6 +60,12 @@ spec:
            value: "https://hf-mirror.com"
          - name: SGLANG_IS_IN_CI
            value: "{{ sglang_is_in_ci }}"
+          - name: RUN_LABEL
+            value: "{{ run_label }}"
+          - name: TROUBLE_SHOTTING
+            value: "{{ trouble_shotting }}"
+          - name: TRANSFORMERS_VERSION_FOR_SGLANG
+            value: "{{ transformers_version }}"
          command: ["/bin/bash", "-c"]
          args:
            - |
@@ -52,6 +52,8 @@ spec:
          value: "https://hf-mirror.com"
        - name: METRICS_DATA_FILE
          value: "{{ metrics_data_file }}"
+        - name: RUN_LABEL
+          value: "{{ run_label }}"
        - name: SGLANG_IS_IN_CI
          value: "{{ sglang_is_in_ci }}"
        - name: TRANSFORMERS_VERSION_FOR_SGLANG
@@ -60,6 +60,10 @@ spec:
            value: "{{ metrics_data_file }}"
          - name: SGLANG_IS_IN_CI
            value: "{{ sglang_is_in_ci }}"
+          - name: RUN_LABEL
+            value: "{{ run_label }}"
+          - name: TROUBLE_SHOTTING
+            value: "{{ trouble_shotting }}"
          - name: TRANSFORMERS_VERSION_FOR_SGLANG
            value: "{{ transformers_version }}"
          command: ["/bin/bash", "-c"]
@@ -172,6 +176,10 @@ spec:
            value: "{{ metrics_data_file }}"
          - name: SGLANG_IS_IN_CI
            value: "{{ sglang_is_in_ci }}"
+          - name: RUN_LABEL
+            value: "{{ run_label }}"
+          - name: TROUBLE_SHOTTING
+            value: "{{ trouble_shotting }}"
          - name: TRANSFORMERS_VERSION_FOR_SGLANG
            value: "{{ transformers_version }}"
          command: ["/bin/bash", "-c"]
@@ -274,6 +282,8 @@ spec:
            value: "{{ install_sglang_from_source }}"
          - name: METRICS_DATA_FILE
            value: "{{ metrics_data_file }}"
+          - name: RUN_LABEL
+            value: "{{ run_label }}"
          - name: KUBECONFIG
            value: "{{ kube_config }}"
          - name: NAMESPACE
@@ -284,6 +294,8 @@ spec:
            value: "https://hf-mirror.com"
          - name: SGLANG_IS_IN_CI
            value: "{{ sglang_is_in_ci }}"
+          - name: TROUBLE_SHOTTING
+            value: "{{ trouble_shotting }}"
          - name: TRANSFORMERS_VERSION_FOR_SGLANG
            value: "{{ transformers_version }}"
          command: ["/bin/bash", "-c"]
@@ -53,6 +53,8 @@ spec:
          value: "https://hf-mirror.com"
        - name: METRICS_DATA_FILE
          value: "{{ metrics_data_file }}"
+        - name: RUN_LABEL
+          value: "{{ run_label }}"
        - name: SGLANG_IS_IN_CI
          value: "{{ sglang_is_in_ci }}"
        command: ["/bin/bash", "-c"]
@@ -154,6 +156,8 @@ spec:
          value: "https://hf-mirror.com"
        - name: METRICS_DATA_FILE
          value: "{{ metrics_data_file }}"
+        - name: RUN_LABEL
+          value: "{{ run_label }}"
        - name: SGLANG_IS_IN_CI
          value: "{{ sglang_is_in_ci }}"
        command: ["/bin/bash", "-c"]
@@ -254,6 +258,8 @@ spec:
          value: "https://hf-mirror.com"
        - name: METRICS_DATA_FILE
          value: "{{ metrics_data_file }}"
+        - name: RUN_LABEL
+          value: "{{ run_label }}"
        - name: SGLANG_IS_IN_CI
          value: "{{ sglang_is_in_ci }}"
        command: ["/bin/bash", "-c"]
@@ -51,8 +51,13 @@ spec:
            value: "{{ kube_config }}"
          - name: HF_ENDPOINT
            value: "https://hf-mirror.com"
+          # Env vars consumed by run_npu_testcase.sh inside the pod
          - name: TROUBLE_SHOTTING
            value: "{{ trouble_shotting }}"
+          - name: RUN_LABEL
+            value: "{{ run_label }}"
+          - name: SGLANG_IS_IN_CI
+            value: "{{ sglang_is_in_ci }}"
          - name: TRANSFORMERS_VERSION_FOR_SGLANG
            value: "{{ transformers_version }}"
          command: ["/bin/bash", "-c"]
@@ -518,10 +518,17 @@ def generate_metrics_json(metrics_data_file, test_case, status):
    tc_name = test_case.rsplit("/", 1)[-1].rsplit(".", 1)[0]

    test_type = "unknown"
+    # nightly: .../output/{branch}-{date}-{run_id}-{run_attempt}/{workflow}/{test_type}/...
+    # PR:      .../output/{test_type}/{date}/{tc_name}
+    # After `output`, rest >= 4 segments -> test_type at parts[i+3]; otherwise at parts[i+1].
    parts = metrics_data_file.split("/")
    for i, part in enumerate(parts):
-        if part == "output" and i + 1 < len(parts):
-            test_type = parts[i + 1]
+        if part == "output":
+            rest = len(parts) - (i + 1)
+            if rest >= 4:
+                test_type = parts[i + 3]
+            elif rest >= 1:
+                test_type = parts[i + 1]
            break

    output = {
@@ -576,6 +583,22 @@ def run_npu_e2e_test_case(

    kube_config_map = f"sglang-configmap-{random_str}"
    final_kube_job_name = f"{kube_job_name_prefix}-{random_str}"
+    # run_label is injected into the pod as RUN_LABEL to build the pod log directory prefix.
+    # nightly (>=4 segments after `output`): first two segments {branch}-{date}-{run_id}-{run_attempt}/{workflow}
+    # PR legacy layout: fall back to the date segment to keep the original {date}/{tc_name}/{host} path.
+    parts = (
+        metrics_data_file.split("/output/")[-1]
+        if "/output/" in metrics_data_file
+        else ""
+    )
+    if parts:
+        segments = parts.split("/")
+        if len(segments) >= 4:
+            run_label = "/".join(segments[:2])
+        else:
+            run_label = segments[1] if len(segments) > 1 else "unknown"
+    else:
+        run_label = "unknown"

    kube_yaml_file_dict = {
        KUBE_JOB_SINGLE: f"k8s_single_{random_str}.yaml",
@@ -605,6 +628,7 @@ def run_npu_e2e_test_case(
                "env": env,
                "trouble_shotting": trouble_shotting,
                "transformers_version": transformers_version,
+                "run_label": run_label,
            }
            create_kube_yaml(
                kube_yaml_template=KUBE_YAML_TEMPLATE.get(kube_job_type),
@@ -627,6 +651,7 @@ def run_npu_e2e_test_case(
                "env": env,
                "trouble_shotting": trouble_shotting,
                "transformers_version": transformers_version,
+                "run_label": run_label,
            }
            template_key = (
                KUBE_JOB_MULTI_PD_MIX_GREEN if env == "green" else kube_job_type
@@ -654,6 +679,7 @@ def run_npu_e2e_test_case(
                "env": env,
                "trouble_shotting": trouble_shotting,
                "transformers_version": transformers_version,
+                "run_label": run_label,
            }
            template_key = (
                KUBE_JOB_MULTI_PD_SEPARATION_GREEN if env == "green" else kube_job_type
@@ -34,7 +34,7 @@ if [ -n "${TRANSFORMERS_VERSION_FOR_SGLANG}" ];then
    echo "Install transformers ${TRANSFORMERS_VERSION_FOR_SGLANG} locally."
    TRANSFORMERS_PKG_PATH_TARGET=/tmp/transformers/${TRANSFORMERS_VERSION_FOR_SGLANG}
    mkdir -p "${TRANSFORMERS_PKG_PATH_TARGET}"
-    cp "${TRANSFORMERS_PKG_PATH_SOURCE}/*" "${TRANSFORMERS_PKG_PATH_TARGET}/"
+    cp "${TRANSFORMERS_PKG_PATH_SOURCE}/"* "${TRANSFORMERS_PKG_PATH_TARGET}/"
    pip install --no-index --find-links="${TRANSFORMERS_PKG_PATH_TARGET}" transformers=="${TRANSFORMERS_VERSION_FOR_SGLANG}"
  fi
  echo "===== Install transformers for sglang in virtual env - End ====="
@@ -114,10 +114,11 @@ fi
 echo "Running test case ${test_case}"
 tc_name=${test_case##*/}
 tc_name=${tc_name%.*}
-current_date=$(date +%Y%m%d)
-log_path="/root/sglang/debug/logs/log/${current_date}/${tc_name}/${HOSTNAME}"
+run_label="${RUN_LABEL:-unknown}"
+log_path="/root/sglang/debug/logs/log/${run_label}/${tc_name}/${HOSTNAME}"
 if [ "${SGLANG_IS_IN_CI}" = "true" ] || [ "${SGLANG_IS_IN_CI}" = "True" ];then
-    log_path="/root/.cache/tests/logs/log/${current_date}/${tc_name}/${HOSTNAME}"
+    # In CI, persist logs under /root/.cache/tests/logs so they can be collected
+    log_path="/root/.cache/tests/logs/log/${run_label}/${tc_name}/${HOSTNAME}"
 fi
 rm -rf "${log_path}"
 mkdir -p "${log_path}"
@@ -134,6 +135,7 @@ echo "Finished test case ${test_case}"

 if [ -n "${METRICS_DATA_FILE}" ]; then
    mkdir -p "${METRICS_DATA_FILE}"
+    # Archive the test log into the output directory for result collection
    cp "${log_path}/${tc_name}.log" "${METRICS_DATA_FILE}/test_output.log"
    echo "Metrics log saved to ${METRICS_DATA_FILE}/test_output.log"
 fi
@@ -311,16 +311,27 @@ class TestNpuAccuracyTestCaseBase(CustomTestCase):
    def _setup_per_case_output(cls):
        """Set up per-case output directories and env vars.

-        Extracted from ``nightly-test-npu-e2e-single-node.yml`` so that when a
-        suite is executed, each case writes its metrics/plog to a path derived
-        from the case file rather than the suite name.
+        When the workflow sets METRICS_DATA_FILE to a suite-level directory
+        (e.g. .../output/{branch_label}-{create_date}-{run_id}-{run_attempt}/
+        {workflow_name}/{test_type}/{suite}), each case in the suite
+        writes to its own subdirectory under it, so results stay in the
+        structured layout and are keyed by the case id. Falls back to the
+        legacy per-case layout when the env var is not set.
        """
        cls.tc_name = cls._get_tc_name()
-        current_date = datetime.now().strftime("%Y%m%d")
-        test_type = getattr(cls, "test_type", "accuracy")
-        base_output = f"/root/.cache/tests/output/{test_type}/{current_date}"
-        os.makedirs(base_output, exist_ok=True)
-        cls.metrics_data_file = os.path.join(base_output, cls.tc_name)
+        suite_output = os.environ.get("METRICS_DATA_FILE")
+        if suite_output:
+            # Append the case id under the suite output prefix.
+            cls.metrics_data_file = os.path.join(suite_output, cls.tc_name)
+            # Mirror the output prefix to the plog location (drop the test_type/suite tail).
+            suite_plog = suite_output.replace("/output/", "/logs/plog/", 1)
+            cls.plog_base = os.path.dirname(os.path.dirname(suite_plog))
+        else:
+            current_date = datetime.now().strftime("%Y%m%d")
+            test_type = getattr(cls, "test_type", "accuracy")
+            base_output = f"/root/.cache/tests/output/{test_type}/{current_date}"
+            cls.metrics_data_file = os.path.join(base_output, cls.tc_name)
+            cls.plog_base = f"/root/.cache/tests/logs/plog"
        os.makedirs(cls.metrics_data_file, exist_ok=True)
        # Override env vars so evalscope/dump_metric write to per-case paths.
        os.environ["METRICS_DATA_FILE"] = cls.metrics_data_file
@@ -376,6 +387,12 @@ class TestNpuAccuracyTestCaseBase(CustomTestCase):
            logger.info("Saved per-case metrics to %s", out_path)
        except Exception as e:
            logger.warning("Failed to write metrics.json: %s", e)
+        # Remove the intermediate JSONL records, keeping only the final metrics.json.
+        for jsonl_path in glob.glob(pattern):
+            try:
+                os.remove(jsonl_path)
+            except Exception as e:
+                logger.warning("Failed to remove %s: %s", jsonl_path, e)

    @classmethod
    def _backup_plog(cls):
@@ -391,7 +408,8 @@ class TestNpuAccuracyTestCaseBase(CustomTestCase):
        if not tc_name:
            return
        hostname = os.getenv("HOSTNAME", "unknown")
-        target = os.path.join("/root/.cache/tests/logs/plog", tc_name, hostname)
+        plog_base = getattr(cls, "plog_base", "/root/.cache/tests/logs/plog")
+        target = os.path.join(plog_base, tc_name, hostname)
        os.makedirs(target, exist_ok=True)
        for name in os.listdir(plog_path):
            src = os.path.join(plog_path, name)
@@ -930,16 +930,27 @@ class TestNpuPerformanceTestCaseBase(CustomTestCase):
    def _setup_per_case_output(cls):
        """Set up per-case output directories and env vars.

-        Extracted from ``nightly-test-npu-e2e-single-node.yml`` so that when a
-        suite is executed, each case writes its metrics/plog to a path derived
-        from the case file rather than the suite name.
+        When the workflow sets METRICS_DATA_FILE to a suite-level directory
+        (e.g. .../output/{branch_label}-{create_date}-{run_id}-{run_attempt}/
+        {workflow_name}/{test_type}/{suite}), each case in the suite
+        writes to its own subdirectory under it, so results stay in the
+        structured layout and are keyed by the case id. Falls back to the
+        legacy per-case layout when the env var is not set.
        """
        cls.tc_name = cls._get_tc_name()
-        current_date = datetime.now().strftime("%Y%m%d")
-        test_type = getattr(cls, "test_type", "perf")
-        base_output = f"/root/.cache/tests/output/{test_type}/{current_date}"
-        os.makedirs(base_output, exist_ok=True)
-        cls.metrics_data_file = os.path.join(base_output, cls.tc_name)
+        suite_output = os.environ.get("METRICS_DATA_FILE")
+        if suite_output:
+            # Append the case id under the suite output prefix.
+            cls.metrics_data_file = os.path.join(suite_output, cls.tc_name)
+            # Mirror the output prefix to the plog location (drop the test_type/suite tail).
+            suite_plog = suite_output.replace("/output/", "/logs/plog/", 1)
+            cls.plog_base = os.path.dirname(os.path.dirname(suite_plog))
+        else:
+            current_date = datetime.now().strftime("%Y%m%d")
+            test_type = getattr(cls, "test_type", "perf")
+            base_output = f"/root/.cache/tests/output/{test_type}/{current_date}"
+            cls.metrics_data_file = os.path.join(base_output, cls.tc_name)
+            cls.plog_base = f"/root/.cache/tests/logs/plog"
        os.makedirs(cls.metrics_data_file, exist_ok=True)
        # Override env vars so evalscope/dump_metric write to per-case paths.
        os.environ["METRICS_DATA_FILE"] = cls.metrics_data_file
@@ -985,7 +996,7 @@ class TestNpuPerformanceTestCaseBase(CustomTestCase):
        out_path = os.path.join(cls.metrics_data_file, "metrics.json")
        payload = {
            "test_case": cls.tc_name,
-            "test_type": getattr(cls, "test_type", "accuracy"),
+            "test_type": getattr(cls, "test_type", "perf"),
            "metrics": metrics,
            "baselines": baselines,
        }
@@ -995,6 +1006,12 @@ class TestNpuPerformanceTestCaseBase(CustomTestCase):
            logger.info("Saved per-case metrics to %s", out_path)
        except Exception as e:
            logger.warning("Failed to write metrics.json: %s", e)
+        # Remove the intermediate JSONL records, keeping only the final metrics.json.
+        for jsonl_path in glob.glob(pattern):
+            try:
+                os.remove(jsonl_path)
+            except Exception as e:
+                logger.warning("Failed to remove %s: %s", jsonl_path, e)

    @classmethod
    def _backup_plog(cls):
@@ -1010,7 +1027,8 @@ class TestNpuPerformanceTestCaseBase(CustomTestCase):
        if not tc_name:
            return
        hostname = os.getenv("HOSTNAME", "unknown")
-        target = os.path.join("/root/.cache/tests/logs/plog", tc_name, hostname)
+        plog_base = getattr(cls, "plog_base", "/root/.cache/tests/logs/plog")
+        target = os.path.join(plog_base, tc_name, hostname)
        os.makedirs(target, exist_ok=True)
        for name in os.listdir(plog_path):
            src = os.path.join(plog_path, name)
@@ -7,10 +7,9 @@ from sglang.test.ascend.test_ascend_utils import DEEPSEEK_V3_2_EXP_W8A8_WEIGHTS_
 from sglang.test.ci.ci_register import register_npu_ci

 register_npu_ci(
-    est_time=3600,
-    suite="",
+    est_time=4800,
+    suite="nightly-acc-16-npu-a3",
    nightly=True,
-    disabled="accuracy testcase",
 )

 OTHER_ARGS = [
@@ -7,7 +7,7 @@ from sglang.test.ascend.e2e.test_npu_performance_utils import GLM_4_6V_FLASH_MOD
 from sglang.test.ci.ci_register import register_npu_ci

 register_npu_ci(
-    est_time=3600,
+    est_time=4800,
    suite="",
    nightly=True,
    disabled="performance testcase",
@@ -7,10 +7,9 @@ from sglang.test.ascend.e2e.test_npu_performance_utils import GLM_4_7_FLASH_MODE
 from sglang.test.ci.ci_register import register_npu_ci

 register_npu_ci(
-    est_time=3600,
-    suite="",
+    est_time=6500,
+    suite="nightly-acc-2-npu-a3",
    nightly=True,
-    disabled="accuracy testcase",
 )

 ENVS = {
@@ -7,6 +7,7 @@ from sglang.test.ascend.e2e.test_npu_performance_utils import GLM_4_7_FLASH_MODE
 from sglang.test.ci.ci_register import register_npu_ci

 register_npu_ci(est_time=3600, suite="base-c-test-acc-2-npu-a3")
+register_npu_ci(est_time=6500, suite="nightly-acc-2-npu-a3", nightly=True)

 ENVS = {
    "PYTORCH_NPU_ALLOC_CONF": "expandable_segments:True",
@@ -10,7 +10,7 @@ from sglang.test.ascend.e2e.test_npu_performance_utils import (
 from sglang.test.ci.ci_register import register_npu_ci

 register_npu_ci(
-    est_time=3600,
+    est_time=4800,
    suite="",
    nightly=True,
    disabled="accuracy testcase",
@@ -9,6 +9,7 @@ from sglang.test.ascend.e2e.test_npu_performance_utils import (
 from sglang.test.ci.ci_register import register_npu_ci

 register_npu_ci(est_time=3600, suite="base-c-test-acc-16-npu-a3")
+register_npu_ci(est_time=4800, suite="nightly-acc-16-npu-a3", nightly=True)

 ENVS = {
    "SGLANG_SET_CPU_AFFINITY": "1",
@@ -12,7 +12,7 @@ from sglang.test.ci.ci_register import register_npu_ci

 register_npu_ci(
    est_time=1800,
-    suite="nightly-8-npu-a3",
+    suite="full-8-npu-a3",
    nightly=True,
    disabled="accuracy testcase",
 )
@@ -11,10 +11,9 @@ from sglang.test.ascend.e2e.test_npu_performance_utils import (
 from sglang.test.ci.ci_register import register_npu_ci

 register_npu_ci(
-    est_time=3600,
-    suite="",
+    est_time=7200,
+    suite="nightly-acc-16-npu-a3",
    nightly=True,
-    disabled="accuracy testcase",
 )

 MINIMAX_M2_5_W8A8_4P_IN64K_OUT1K_PREFIX90_ENVS = {
@@ -11,10 +11,9 @@ from sglang.test.ascend.e2e.test_npu_performance_utils import (
 from sglang.test.ci.ci_register import register_npu_ci

 register_npu_ci(
-    est_time=3600,
-    suite="full-16-npu-a3",
+    est_time=4800,
+    suite="nightly-acc-16-npu-a3",
    nightly=True,
-    disabled="accuracy testcase",
 )

 MINIMAX_M2_5_HIGH_THROUGHPUT_ENVS = {
@@ -9,6 +9,7 @@ from sglang.test.ascend.e2e.test_npu_performance_utils import (
 from sglang.test.ci.ci_register import register_npu_ci

 register_npu_ci(est_time=3600, suite="base-c-test-acc-2-npu-a3")
+register_npu_ci(est_time=1300, suite="nightly-acc-2-npu-a3", nightly=True)

 MODEL_ENVS = {
    "SGLANG_SET_CPU_AFFINITY": "1",
@@ -10,10 +10,9 @@ from sglang.test.ascend.e2e.test_npu_performance_utils import (
 from sglang.test.ci.ci_register import register_npu_ci

 register_npu_ci(
-    est_time=3600,
-    suite="",
+    est_time=3700,
+    suite="nightly-acc-2-npu-a3",
    nightly=True,
-    disabled="accuracy testcase",
 )

 QWEN3_8B_ENVS = {
@@ -10,10 +10,9 @@ from sglang.test.ascend.e2e.test_npu_performance_utils import (
 from sglang.test.ci.ci_register import register_npu_ci

 register_npu_ci(
-    est_time=3600,
-    suite="",
+    est_time=3700,
+    suite="nightly-acc-2-npu-a3",
    nightly=True,
-    disabled="accuracy testcase",
 )

 QWEN3_8B_ENVS = {
@@ -10,10 +10,9 @@ from sglang.test.ascend.e2e.test_npu_performance_utils import (
 from sglang.test.ci.ci_register import register_npu_ci

 register_npu_ci(
-    est_time=3600,
-    suite="",
+    est_time=2800,
+    suite="nightly-acc-2-npu-a3",
    nightly=True,
-    disabled="accuracy testcase",
 )

 QWEN3_30B_A3B_ENVS = {
@@ -10,10 +10,9 @@ from sglang.test.ascend.e2e.test_npu_performance_utils import (
 from sglang.test.ci.ci_register import register_npu_ci

 register_npu_ci(
-    est_time=3600,
-    suite="",
+    est_time=4800,
+    suite="nightly-acc-16-npu-a3",
    nightly=True,
-    disabled="performance testcase",
 )

 QWEN3_32B_ENVS = {
@@ -10,10 +10,9 @@ from sglang.test.ascend.e2e.test_npu_performance_utils import (
 from sglang.test.ci.ci_register import register_npu_ci

 register_npu_ci(
-    est_time=3600,
-    suite="",
+    est_time=4800,
+    suite="nightly-acc-4-npu-a3",
    nightly=True,
-    disabled="accuracy testcase",
 )

 QWEN3_32B_ENVS = {
@@ -10,10 +10,9 @@ from sglang.test.ascend.e2e.test_npu_performance_utils import (
 from sglang.test.ci.ci_register import register_npu_ci

 register_npu_ci(
-    est_time=3600,
-    suite="",
+    est_time=4800,
+    suite="nightly-1-npu-a2",
    nightly=True,
-    disabled="accuracy testcase",
 )

 QWEN3_32B_ENVS = {
@@ -9,6 +9,7 @@ from sglang.test.ascend.e2e.test_npu_performance_utils import (
 from sglang.test.ci.ci_register import register_npu_ci

 register_npu_ci(est_time=3600, suite="base-c-test-acc-2-npu-a3")
+register_npu_ci(est_time=2800, suite="nightly-acc-2-npu-a3", nightly=True)

 QWEN3_5_9B_ENVS = {
    "SGLANG_SET_CPU_AFFINITY": "1",
@@ -9,10 +9,13 @@ from sglang.test.ascend.e2e.test_npu_performance_utils import (
 from sglang.test.ci.ci_register import register_npu_ci

 register_npu_ci(
-    est_time=3600,
-    suite="",
+    # Measured ~45min/round in CI (2026-08-17 nightly-acc-2-npu-a3).
+    # est_time = 2x single-round so derive_timeout_per_file (1.5x est)
+    # reserves ~3 rounds of budget for accuracy retries.
+    # Normalized to the shared 6500 ladder value for ~42-54min cases.
+    est_time=6500,
+    suite="nightly-acc-2-npu-a3",
    nightly=True,
-    disabled="performance testcase",
 )

 QWEN3_6_27B_64K_PREFIX_ENVS = {
@@ -9,10 +9,9 @@ from sglang.test.ascend.e2e.test_npu_performance_utils import (
 from sglang.test.ci.ci_register import register_npu_ci

 register_npu_ci(
-    est_time=3600,
-    suite="full-2-npu-a3",
+    est_time=8400,
+    suite="nightly-acc-2-npu-a3",
    nightly=True,
-    disabled="accuracy testcase",
 )

 QWEN3_6_27B_3K5_1K5_ENVS = {
@@ -8,12 +8,7 @@ from sglang.test.ascend.e2e.test_npu_performance_utils import (
 )
 from sglang.test.ci.ci_register import register_npu_ci

-register_npu_ci(
-    est_time=3600,
-    suite="full-2-npu-a3",
-    nightly=True,
-    disabled="performance testcase",
-)
+register_npu_ci(est_time=6500, suite="nightly-acc-2-npu-a3", nightly=True)

 QWEN3_6_35B_A3B_3K5_1K5_ENVS = {
    "PYTORCH_NPU_ALLOC_CONF": "expandable_segments:True",
@@ -9,10 +9,9 @@ from sglang.test.ascend.e2e.test_npu_performance_utils import (
 from sglang.test.ci.ci_register import register_npu_ci

 register_npu_ci(
-    est_time=3600,
-    suite="",
+    est_time=2800,
+    suite="nightly-acc-2-npu-a3",
    nightly=True,
-    disabled="accuracy testcase",
 )

 QWEN3_6_35B_A3B_64K_PREFIX_ENVS = {
@@ -10,10 +10,9 @@ from sglang.test.ascend.e2e.test_npu_performance_utils import (
 from sglang.test.ci.ci_register import register_npu_ci

 register_npu_ci(
-    est_time=3600,
-    suite="",
+    est_time=4800,
+    suite="nightly-acc-4-npu-a3",
    nightly=True,
-    disabled="accuracy testcase",
 )

 QWEN3_NEXT_80B_A3B_ENVS = {
@@ -9,7 +9,7 @@ from sglang.test.ascend.e2e.test_npu_performance_utils import (
 from sglang.test.ci.ci_register import register_npu_ci

 register_npu_ci(
-    est_time=3600,
+    est_time=4800,
    suite="",
    nightly=True,
    disabled="performance testcase",
@@ -9,6 +9,7 @@ from sglang.test.ascend.e2e.test_npu_performance_utils import (
 from sglang.test.ci.ci_register import register_npu_ci

 register_npu_ci(est_time=3600, suite="base-c-test-acc-4-npu-a3")
+register_npu_ci(est_time=4800, suite="nightly-acc-4-npu-a3", nightly=True)

 QWEN3_VL_30B_A3B_ENVS = {
    "SGLANG_SET_CPU_AFFINITY": "1",
@@ -9,10 +9,9 @@ from sglang.test.ascend.e2e.test_npu_performance_utils import (
 from sglang.test.ci.ci_register import register_npu_ci

 register_npu_ci(
-    est_time=3600,
-    suite="",
+    est_time=8400,
+    suite="nightly-acc-2-npu-a3",
    nightly=True,
-    disabled="performance testcase",
 )

 ENVS = {
@@ -9,6 +9,7 @@ from sglang.test.ascend.e2e.test_npu_performance_utils import (
 from sglang.test.ci.ci_register import register_npu_ci

 register_npu_ci(est_time=3600, suite="base-c-test-acc-4-npu-a3")
+register_npu_ci(est_time=4800, suite="nightly-acc-4-npu-a3", nightly=True)

 QWEN3_VL_8B_ENVS = {
    "SGLANG_SET_CPU_AFFINITY": "1",
@@ -10,6 +10,7 @@ from sglang.test.ascend.e2e.test_npu_performance_utils import (
 from sglang.test.ci.ci_register import register_npu_ci

 register_npu_ci(est_time=3600, suite="base-c-test-perf-2-npu-a3")
+register_npu_ci(est_time=6500, suite="nightly-acc-2-npu-a3", nightly=True)

 _is_pr_pipeline = os.environ.get("GITHUB_EVENT_NAME") == "pull_request"

@@ -12,7 +12,7 @@ from sglang.test.test_utils import (
    popen_launch_server,
 )

-register_npu_ci(est_time=400, suite="nightly-1-npu-a3", nightly=True)
+register_npu_ci(est_time=400, suite="full-1-npu-a3", nightly=True)


 class TestNPUHierarchicalCache(CustomTestCase):
@@ -10,7 +10,7 @@ from sglang.test.test_utils import CustomTestCase

 register_npu_ci(
    est_time=400,
-    suite="nightly-16-npu-a3",
+    suite="full-16-npu-a3",
    nightly=True,
 )

@@ -12,7 +12,7 @@ from sglang.test.test_utils import (

 register_npu_ci(
    est_time=400,
-    suite="nightly-1-npu-a3",
+    suite="full-1-npu-a3",
    nightly=True,
 )

@@ -9,7 +9,7 @@ from sglang.test.test_utils import CustomTestCase

 register_npu_ci(
    est_time=400,
-    suite="nightly-2-npu-a3",
+    suite="full-2-npu-a3",
    nightly=True,
 )

@@ -12,7 +12,7 @@ from sglang.test.test_utils import (
    popen_launch_server,
 )

-register_npu_ci(est_time=400, suite="nightly-1-npu-a3", nightly=True)
+register_npu_ci(est_time=400, suite="full-1-npu-a3", nightly=True)


 class TestNPURadixCache(CustomTestCase):
@@ -11,7 +11,7 @@ from sglang.test.test_utils import (
 )

 register_npu_ci(est_time=400, suite="base-b-test-4-npu-a3")
-register_npu_ci(est_time=400, suite="nightly-1-npu-a3", nightly=True)
+register_npu_ci(est_time=400, suite="nightly-4-npu-a3", nightly=True)


 class TestLLaDA2Mini(GSM8KAscendMixin, CustomTestCase):
@@ -13,7 +13,7 @@ from sglang.test.test_utils import (
    run_bench_serving,
 )

-register_npu_ci(est_time=400, suite="nightly-1-npu-a3", nightly=True)
+register_npu_ci(est_time=400, suite="full-1-npu-a3", nightly=True)


 class TestNoChunkedPrefill(CustomTestCase):
@@ -13,7 +13,7 @@ from sglang.test.test_utils import (
    popen_launch_server,
 )

-register_npu_ci(est_time=800, suite="nightly-2-npu-a3", nightly=True)
+register_npu_ci(est_time=800, suite="full-2-npu-a3", nightly=True)

 TEST_MODEL_MATRIX = {
    DEEPSEEK_CODER_V2_LITE_WEIGHTS_PATH,
@@ -7,7 +7,7 @@ from sglang.test.ascend.test_mmlu import TestMMLU
 from sglang.test.ci.ci_register import register_npu_ci
 from sglang.test.test_utils import CustomTestCase

-register_npu_ci(est_time=400, suite="nightly-16-npu-a3", nightly=True)
+register_npu_ci(est_time=400, suite="full-16-npu-a3", nightly=True)


 class TestDeepEpDeepseekV32(GSM8KAscendMixin, TestMMLU, CustomTestCase):
@@ -9,7 +9,7 @@ from sglang.test.ascend.test_mmlu import TestMMLU
 from sglang.test.ci.ci_register import register_npu_ci
 from sglang.test.test_utils import CustomTestCase

-register_npu_ci(est_time=200, suite="nightly-16-npu-a3", nightly=True)
+register_npu_ci(est_time=200, suite="full-16-npu-a3", nightly=True)


 class TestDeepEpQwen(GSM8KAscendMixin, TestMMLU, CustomTestCase):
@@ -11,7 +11,7 @@ from sglang.test.test_utils import CustomTestCase

 register_npu_ci(
    est_time=200,
-    suite="nightly-8-npu-a3",
+    suite="full-8-npu-a3",
    nightly=True,
 )

@@ -7,7 +7,7 @@ from sglang.test.ascend.test_mmlu import TestMMLU
 from sglang.test.ci.ci_register import register_npu_ci
 from sglang.test.test_utils import CustomTestCase

-register_npu_ci(est_time=200, suite="nightly-16-npu-a3", nightly=True)
+register_npu_ci(est_time=200, suite="full-16-npu-a3", nightly=True)


 class TestDeepEpDeepseekV32(GSM8KAscendMixin, TestMMLU, CustomTestCase):
@@ -9,7 +9,7 @@ from sglang.test.ascend.test_mmlu import TestMMLU
 from sglang.test.ci.ci_register import register_npu_ci
 from sglang.test.test_utils import CustomTestCase

-register_npu_ci(est_time=200, suite="nightly-16-npu-a3", nightly=True)
+register_npu_ci(est_time=200, suite="full-16-npu-a3", nightly=True)


 class TestDeepEpQwen(GSM8KAscendMixin, TestMMLU, CustomTestCase):
@@ -11,7 +11,7 @@ from sglang.test.test_utils import CustomTestCase

 register_npu_ci(
    est_time=200,
-    suite="nightly-8-npu-a3",
+    suite="full-8-npu-a3",
    nightly=True,
 )

@@ -18,7 +18,7 @@ from sglang.test.test_utils import (

 register_npu_ci(
    est_time=400,
-    suite="nightly-1-npu-a3",
+    suite="full-1-npu-a3",
    nightly=True,
 )

@@ -13,7 +13,7 @@ from sglang.test.test_utils import (
    popen_launch_server,
 )

-register_npu_ci(est_time=400, suite="nightly-1-npu-a3", nightly=True)
+register_npu_ci(est_time=400, suite="full-1-npu-a3", nightly=True)


 class TestLogLevel(CustomTestCase):
@@ -5,7 +5,7 @@ from sglang.test.test_utils import CustomTestCase, run_mmlu_test

 register_npu_ci(
    est_time=400,
-    suite="nightly-1-npu-a3",
+    suite="full-1-npu-a3",
    nightly=True,
 )

@@ -46,7 +46,7 @@ if torch.cuda.is_available():
    torch.backends.cuda.matmul.allow_tf32 = False
    torch.backends.cudnn.allow_tf32 = False

-register_npu_ci(est_time=400, suite="nightly-1-npu-a3", nightly=True)
+register_npu_ci(est_time=400, suite="full-1-npu-a3", nightly=True)


 class TestOriginalLogprob(unittest.TestCase):
@@ -14,7 +14,7 @@ from sglang.test.test_utils import (

 register_npu_ci(
    est_time=400,
-    suite="nightly-4-npu-a3",
+    suite="full-4-npu-a3",
    nightly=True,
 )

@@ -14,7 +14,7 @@ from sglang.test.test_utils import (
    popen_launch_server,
 )

-register_npu_ci(est_time=300, suite="nightly-1-npu-a3", nightly=True)
+register_npu_ci(est_time=300, suite="full-1-npu-a3", nightly=True)

 logger = logging.getLogger(__name__)

@@ -16,7 +16,7 @@ from sglang.test.test_utils import (
    popen_launch_server,
 )

-register_npu_ci(est_time=500, suite="nightly-2-npu-a3", nightly=True)
+register_npu_ci(est_time=500, suite="full-2-npu-a3", nightly=True)

 logger = logging.getLogger(__name__)

@@ -22,7 +22,7 @@ def send_requests(url, **kwargs):
    responses.append(response)


-register_npu_ci(est_time=400, suite="nightly-1-npu-a3", nightly=True)
+register_npu_ci(est_time=400, suite="full-1-npu-a3", nightly=True)


 class TestNpuApi(CustomTestCase):
@@ -20,7 +20,7 @@ logging.basicConfig(
 )
 logger = logging.getLogger(__name__)

-register_npu_ci(est_time=400, suite="nightly-1-npu-a3", nightly=True)
+register_npu_ci(est_time=400, suite="full-1-npu-a3", nightly=True)


 class TestNpuApi(CustomTestCase):
@@ -15,7 +15,7 @@ from sglang.test.test_utils import (

 register_npu_ci(
    est_time=400,
-    suite="nightly-2-npu-a3",
+    suite="full-2-npu-a3",
    nightly=True,
 )

@@ -19,7 +19,7 @@ Each section should be as comprehensive as possible to create a rich and immersi
 The story should span multiple events, challenges, and character developments over time. Aim to make the story at least 3,000 words long.
 """

-register_npu_ci(est_time=400, suite="nightly-1-npu-a3", nightly=True)
+register_npu_ci(est_time=400, suite="full-1-npu-a3", nightly=True)


 class TestMatchedStop(CustomTestCase):
@@ -16,7 +16,7 @@ from sglang.test.test_utils import (

 register_npu_ci(
    est_time=400,
-    suite="nightly-1-npu-a3",
+    suite="full-1-npu-a3",
    nightly=True,
 )

@@ -13,7 +13,7 @@ from sglang.test.test_utils import (
    popen_launch_server,
 )

-register_npu_ci(est_time=400, suite="nightly-2-npu-a3", nightly=True)
+register_npu_ci(est_time=400, suite="full-2-npu-a3", nightly=True)


 class TestOpenAIServerIgnoreEOS(CustomTestCase):
@@ -15,7 +15,7 @@ from sglang.test.test_utils import (
    popen_launch_server,
 )

-register_npu_ci(est_time=400, suite="nightly-1-npu-a3", nightly=True)
+register_npu_ci(est_time=400, suite="full-1-npu-a3", nightly=True)


 class TestPenalty(CustomTestCase):
@@ -4,7 +4,7 @@ from sglang.test.ascend.gsm8k_ascend_mixin import GSM8KAscendMixin
 from sglang.test.ci.ci_register import register_npu_ci
 from sglang.test.test_utils import CustomTestCase

-register_npu_ci(est_time=400, suite="nightly-8-npu-a3", nightly=True)
+register_npu_ci(est_time=400, suite="full-8-npu-a3", nightly=True)


 class TestDbrx(GSM8KAscendMixin, CustomTestCase):
@@ -7,7 +7,7 @@ from sglang.test.test_utils import CustomTestCase

 register_npu_ci(
    est_time=400,
-    suite="nightly-8-npu-a3",
+    suite="full-8-npu-a3",
    nightly=True,
 )

@@ -7,7 +7,7 @@ from sglang.test.test_utils import CustomTestCase

 register_npu_ci(
    est_time=400,
-    suite="nightly-2-npu-a3",
+    suite="full-2-npu-a3",
    nightly=True,
 )

@@ -5,7 +5,7 @@ from sglang.test.ascend.test_ascend_utils import BAICHUAN2_13B_CHAT_WEIGHTS_PATH
 from sglang.test.ci.ci_register import register_npu_ci
 from sglang.test.test_utils import CustomTestCase

-register_npu_ci(est_time=400, suite="nightly-1-npu-a3", nightly=True)
+register_npu_ci(est_time=400, suite="full-1-npu-a3", nightly=True)


 class TestBaichuan(GSM8KAscendMixin, CustomTestCase):
@@ -5,7 +5,7 @@ from sglang.test.ascend.test_ascend_utils import CHATGLM2_6B_WEIGHTS_PATH
 from sglang.test.ci.ci_register import register_npu_ci
 from sglang.test.test_utils import CustomTestCase

-register_npu_ci(est_time=400, suite="nightly-1-npu-a3", nightly=True)
+register_npu_ci(est_time=400, suite="full-1-npu-a3", nightly=True)


 class TestChatGlm2(GSM8KAscendMixin, CustomTestCase):
@@ -5,7 +5,7 @@ from sglang.test.ascend.test_ascend_utils import DEEPSEEK_V3_2_EXP_W8A8_WEIGHTS_
 from sglang.test.ci.ci_register import register_npu_ci
 from sglang.test.test_utils import CustomTestCase

-register_npu_ci(est_time=400, suite="nightly-16-npu-a3", nightly=True)
+register_npu_ci(est_time=400, suite="full-16-npu-a3", nightly=True)


 class TestDeepSeekV32(GSM8KAscendMixin, CustomTestCase):
@@ -7,7 +7,7 @@ from sglang.test.test_utils import CustomTestCase

 register_npu_ci(
    est_time=400,
-    suite="nightly-1-npu-a3",
+    suite="full-1-npu-a3",
    nightly=True,
 )

@@ -7,7 +7,7 @@ from sglang.test.test_utils import CustomTestCase

 register_npu_ci(
    est_time=400,
-    suite="nightly-1-npu-a3",
+    suite="full-1-npu-a3",
    nightly=True,
 )

@@ -8,7 +8,7 @@ from sglang.test.test_utils import CustomTestCase

 register_npu_ci(
    est_time=400,
-    suite="nightly-1-npu-a3",
+    suite="full-1-npu-a3",
    nightly=True,
 )

@@ -7,7 +7,7 @@ from sglang.test.ascend.test_ascend_utils import (
 from sglang.test.ci.ci_register import register_npu_ci
 from sglang.test.test_utils import CustomTestCase

-register_npu_ci(est_time=400, suite="nightly-4-npu-a3", nightly=True)
+register_npu_ci(est_time=400, suite="full-4-npu-a3", nightly=True)


 class TestLlama4(GSM8KAscendMixin, CustomTestCase):
@@ -5,7 +5,7 @@ from sglang.test.ascend.test_ascend_utils import LLAMA_2_7B_WEIGHTS_PATH
 from sglang.test.ci.ci_register import register_npu_ci
 from sglang.test.test_utils import CustomTestCase

-register_npu_ci(est_time=400, suite="nightly-1-npu-a3", nightly=True)
+register_npu_ci(est_time=400, suite="full-1-npu-a3", nightly=True)


 class TestLlama(GSM8KAscendMixin, CustomTestCase):
@@ -5,7 +5,7 @@ from sglang.test.ascend.test_ascend_utils import LLAMA_2_7B_WEIGHTS_PATH
 from sglang.test.ci.ci_register import register_npu_ci
 from sglang.test.test_utils import CustomTestCase

-register_npu_ci(est_time=400, suite="nightly-2-npu-a3", nightly=True)
+register_npu_ci(est_time=400, suite="full-2-npu-a3", nightly=True)


 class TestLlama(GSM8KAscendMixin, CustomTestCase):
@@ -5,7 +5,7 @@ from sglang.test.ascend.test_ascend_utils import MINICPM3_4B_WEIGHTS_PATH
 from sglang.test.ci.ci_register import register_npu_ci
 from sglang.test.test_utils import CustomTestCase

-register_npu_ci(est_time=400, suite="nightly-1-npu-a3", nightly=True)
+register_npu_ci(est_time=400, suite="full-1-npu-a3", nightly=True)


 class TestMiniCPM3(GSM8KAscendMixin, CustomTestCase):
@@ -5,7 +5,7 @@ from sglang.test.ascend.test_ascend_utils import MISTRAL_7B_INSTRUCT_V0_2_WEIGHT
 from sglang.test.ci.ci_register import register_npu_ci
 from sglang.test.test_utils import CustomTestCase

-register_npu_ci(est_time=400, suite="nightly-1-npu-a3", nightly=True)
+register_npu_ci(est_time=400, suite="full-1-npu-a3", nightly=True)


 class TestMistral7B(GSM8KAscendMixin, CustomTestCase):
@@ -5,7 +5,7 @@ from sglang.test.ascend.test_ascend_utils import PHI_4_MULTIMODAL_INSTRUCT_WEIGH
 from sglang.test.ci.ci_register import register_npu_ci
 from sglang.test.test_utils import CustomTestCase

-register_npu_ci(est_time=400, suite="nightly-1-npu-a3", nightly=True)
+register_npu_ci(est_time=400, suite="full-1-npu-a3", nightly=True)


 class TestPhi4(GSM8KAscendMixin, CustomTestCase):
@@ -5,7 +5,7 @@ from sglang.test.ascend.test_ascend_utils import QWEN3_0_6B_WEIGHTS_PATH
 from sglang.test.ci.ci_register import register_npu_ci
 from sglang.test.test_utils import CustomTestCase

-register_npu_ci(est_time=400, suite="nightly-1-npu-a3", nightly=True)
+register_npu_ci(est_time=400, suite="full-1-npu-a3", nightly=True)


 class TestQwen306B(GSM8KAscendMixin, CustomTestCase):
@@ -5,7 +5,7 @@ from sglang.test.ascend.test_ascend_utils import QWEN3_235B_A22B_W8A8_WEIGHTS_PA
 from sglang.test.ci.ci_register import register_npu_ci
 from sglang.test.test_utils import CustomTestCase

-register_npu_ci(est_time=400, suite="nightly-8-npu-a3", nightly=True)
+register_npu_ci(est_time=400, suite="full-8-npu-a3", nightly=True)


 class TestQwen3235BA22BW8A8(GSM8KAscendMixin, CustomTestCase):
@@ -7,7 +7,7 @@ from sglang.test.ascend.test_ascend_utils import (
 from sglang.test.ci.ci_register import register_npu_ci
 from sglang.test.test_utils import CustomTestCase

-register_npu_ci(est_time=400, suite="nightly-2-npu-a3", nightly=True)
+register_npu_ci(est_time=400, suite="full-2-npu-a3", nightly=True)


 class TestQwen330B(GSM8KAscendMixin, CustomTestCase):
@@ -6,7 +6,7 @@ from sglang.test.ascend.test_ascend_utils import QWEN3_30B_A3B_WEIGHTS_PATH
 from sglang.test.ci.ci_register import register_npu_ci
 from sglang.test.test_utils import CustomTestCase

-register_npu_ci(est_time=500, suite="nightly-4-npu-a3", nightly=True)
+register_npu_ci(est_time=500, suite="full-4-npu-a3", nightly=True)


 class TestQwen330BAttnCP(GSM8KAscendMixin, CustomTestCase):
@@ -7,7 +7,7 @@ from sglang.test.test_utils import CustomTestCase

 register_npu_ci(
    est_time=400,
-    suite="nightly-4-npu-a3",
+    suite="full-4-npu-a3",
    nightly=True,
 )

@@ -5,7 +5,7 @@ from sglang.test.ascend.test_ascend_utils import QWEN3_8B_WEIGHTS_PATH
 from sglang.test.ci.ci_register import register_npu_ci
 from sglang.test.test_utils import CustomTestCase

-register_npu_ci(est_time=400, suite="nightly-2-npu-a3", nightly=True)
+register_npu_ci(est_time=400, suite="full-2-npu-a3", nightly=True)


 class TestQwen38BCommQuantization(GSM8KAscendMixin, CustomTestCase):
@@ -9,7 +9,7 @@ from sglang.test.test_utils import CustomTestCase

 register_npu_ci(
    est_time=400,
-    suite="nightly-16-npu-a3",
+    suite="full-16-npu-a3",
    nightly=True,
 )

@@ -6,7 +6,7 @@ from sglang.test.ascend.test_ascend_utils import QWQ_32B_W8A8_WEIGHTS_PATH
 from sglang.test.ci.ci_register import register_npu_ci
 from sglang.test.test_utils import CustomTestCase

-register_npu_ci(est_time=400, suite="nightly-2-npu-a3", nightly=True)
+register_npu_ci(est_time=400, suite="full-2-npu-a3", nightly=True)


 class TestQWQ32BW8A8(GSM8KAscendMixin, CustomTestCase):
@@ -8,7 +8,8 @@ from sglang.test.ascend.e2e.test_npu_performance_utils import (
 )
 from sglang.test.ci.ci_register import register_npu_ci

-register_npu_ci(est_time=3600, suite="base-c-test-perf-16-npu-a3")
+register_npu_ci(est_time=1200, suite="base-c-test-perf-16-npu-a3")
+register_npu_ci(est_time=1200, suite="nightly-perf-16-npu-a3", nightly=True)

 # Environment variables for DSV4-Flash single-node PD-mix deployment.
 DEEPSEEK_V4_FLASH_W8A8_8P_ENVS = {
@@ -55,7 +56,7 @@ DEEPSEEK_V4_FLASH_W8A8_8P_OTHER_ARGS = [
    "--watchdog-timeout",
    9000,
    "--mem-fraction-static",
-    0.7,
+    0.6,
    "--prefill-max-requests",
    2,
    "--disable-radix-cache",
@@ -70,7 +70,7 @@ GLM_5_1_PD_SEP_PREFILL_ARGS = [
    "--served-model-name",
    "glm-5",
    "--chunked-prefill-size",
-    16384,
+    32768,
    "--max-prefill-tokens",
    180000,
    "--moe-a2a-backend",
@@ -151,6 +151,14 @@ GLM_5_1_PD_SEP_DECODE_ARGS = [
    "glm45",
    "--tool-call-parser",
    "glm47",
+    "--speculative-algorithm",
+    "NEXTN",
+    "--speculative-num-steps",
+    3,
+    "--speculative-eagle-topk",
+    1,
+    "--speculative-num-draft-tokens",
+    4,
 ]

 GLM_5_1_PD_SEP_MODEL_CONFIG = {
@@ -171,8 +179,8 @@ class TestNPUGLM5_1_W4A8_PD_SEP_In3k5_Out1k5(TestNpuPerfMultiNodePdSepTestCaseBa
    benchmark_tool = BENCHMARK_TOOL_DEFAULT
    dataset_type = AISBENCHMARK_DATASET_DEFAULT
    dataset_name = "random"
-    max_concurrency = 1
-    num_prompts = 1
+    max_concurrency = 32
+    num_prompts = 64
    input_len = 65536
    output_len = 1024
    random_range_ratio = 1
@@ -12,7 +12,7 @@ from sglang.test.ci.ci_register import register_npu_ci

 register_npu_ci(
    est_time=1800,
-    suite="nightly-8-npu-a3",
+    suite="full-8-npu-a3",
    nightly=True,
    disabled="Currently it is executed by the npu performance workflow.",
 )
@@ -10,7 +10,8 @@ from sglang.test.ascend.e2e.test_npu_performance_utils import (
 )
 from sglang.test.ci.ci_register import register_npu_ci

-register_npu_ci(est_time=1800, suite="base-c-test-perf-16-npu-a3")
+register_npu_ci(est_time=1200, suite="base-c-test-perf-16-npu-a3")
+register_npu_ci(est_time=1200, suite="nightly-perf-16-npu-a3", nightly=True)

 KIMI_K2_6_ENVS = {
    "PYTORCH_NPU_ALLOC_CONF": "expandable_segments:True",
@@ -10,6 +10,7 @@ from sglang.test.ascend.e2e.test_npu_performance_utils import (
 from sglang.test.ci.ci_register import register_npu_ci

 register_npu_ci(est_time=3600, suite="base-c-test-perf-8-npu-a3")
+register_npu_ci(est_time=3600, suite="nightly-perf-8-npu-a3", nightly=True)

 MINIMAX_M2_5_W8A8_4P_IN64K_OUT1K_PREFIX90_ENVS = {
    "PYTORCH_NPU_ALLOC_CONF": "expandable_segments:True",
@@ -12,9 +12,8 @@ from sglang.test.ci.ci_register import register_npu_ci

 register_npu_ci(
    est_time=3600,
-    suite="full-16-npu-a3",
+    suite="nightly-perf-16-npu-a3",
    nightly=True,
-    disabled="performance testcase",
 )

 MINIMAX_M2_5_HIGH_THROUGHPUT_ENVS = {
@@ -10,6 +10,7 @@ from sglang.test.ascend.e2e.test_npu_performance_utils import (
 from sglang.test.ci.ci_register import register_npu_ci

 register_npu_ci(est_time=3600, suite="base-c-test-perf-2-npu-a3")
+register_npu_ci(est_time=3600, suite="nightly-perf-2-npu-a3", nightly=True)

 QWEN3_8B_ENVS = {
    "SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT": "600",
@@ -9,9 +9,8 @@ from sglang.test.ci.ci_register import register_npu_ci

 register_npu_ci(
    est_time=3600,
-    suite="",
+    suite="nightly-perf-2-npu-a3",
    nightly=True,
-    disabled="performance testcase",
 )

 QWEN3_8B_ENVS = {
--- a/Show More
+++ b/Show More