[NPU] add coverage-based precision test selection pipeline (#38339)

.github/workflows/_npu-analyze-failure.yml

@@ -0,0 +1,96 @@
+# Reusable workflow: cross-reference CI test failures with coverage-based
+# test recommendations. Called by pr-test-npu.yml's analyze-failure-report job.
+name: Analyze Failure Report
+
+on:
+  workflow_call:
+    inputs:
+      log_artifact_pattern:
+        type: string
+        required: true
+        description: 'Glob pattern for test log artifacts (e.g. selected-test-logs-*)'
+      recommendations_content:
+        type: string
+        required: false
+        default: ''
+        description: 'Content of recommendations from upstream job output'
+
+defaults:
+  run:
+    shell: bash -el {0}
+
+jobs:
+  analyze:
+    name: cross-reference failures with recommendations
+    continue-on-error: true  # Non-blocking on parse/read failure.
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout sglang repo
+        uses: actions/checkout@v4
+
+      # ----- Step 1: Locate recommendations -----
+      - name: Locate recommendations file
+        id: locate-recs
+        run: |
+          mkdir -p ./test-logs
+
+          if [ -n "${{ inputs.recommendations_content }}" ]; then
+            echo "has_recommendations=true" >> $GITHUB_OUTPUT
+            echo "Using workflow output recommendations"
+            echo "${{ inputs.recommendations_content }}" > ./test-logs/_recommended.txt
+            echo "RECS_FILE=./test-logs/_recommended.txt" >> $GITHUB_ENV
+            echo "RECS_SOURCE=output" >> $GITHUB_ENV
+            COUNT=$(wc -l < ./test-logs/_recommended.txt)
+            echo "Recommendations count: ${COUNT}"
+          else
+            echo "has_recommendations=false" >> $GITHUB_OUTPUT
+            echo "::notice::No recommended test cases detected, skipping subsequent analysis tasks."
+          fi
+
+      # ----- Step 2: Download test logs -----
+      - name: Download test logs
+        uses: actions/download-artifact@v7
+        with:
+          pattern: ${{ inputs.log_artifact_pattern }}
+          path: ./test-logs
+          merge-multiple: true
+          continue-on-error: true
+
+      - name: Show downloaded logs structure
+        run: |
+          echo "Downloaded log files:"
+          find ./test-logs -type f | head -50 || echo "  (no logs found)"
+
+      # ----- Step 3: Run analysis -----
+      - name: Ensure regex is available
+        if: steps.locate-recs.outputs.has_recommendations != 'false'
+        run: |
+          set -euo pipefail
+          if ! python3 -c "import regex" >/dev/null 2>&1; then
+            echo "regex not found for $(python3 -V); bootstrapping pip and installing regex"
+            curl -fsSL https://bootstrap.pypa.io/get-pip.py -o get-pip.py
+            python3 get-pip.py --break-system-packages
+            python3 -m pip install --break-system-packages regex
+          fi
+          python3 -c "import regex; print('regex ok:', regex.__file__)"
+
+      - name: Run failure analysis
+        if: steps.locate-recs.outputs.has_recommendations != 'false'
+        id: analysis
+        continue-on-error: true
+        run: |
+          python3 scripts/ci/npu/precise-test/analyze_failure_report.py \
+            --log-dir ./test-logs \
+            --recommendations-file "${RECS_FILE}" \
+            --recommendations-source "${RECS_SOURCE:-none}" \
+            --output ./test-logs/failure_report.md
+
+      # ----- Step 4: Upload report -----
+      - name: Upload failure analysis report
+        if: always()
+        uses: actions/upload-artifact@v7
+        with:
+          name: failure-analysis-report
+          path: ./test-logs/failure_report.md
+          if-no-files-found: ignore
+          retention-days: 14

.github/workflows/_npu-pr-test-stage.yml

@@ -1,5 +1,7 @@
 name: PR Test Stage for NPU
-# Reusable workflow for one CUDA test stage. Caller pr-test-npu.yml forwards
+# Reusable workflow for one NPU test stage. Caller pr-test-npu.yml forwards
+# job parameters (runner, image, partitions) and optionally delegates test
+# execution to the coverage runner (use_coverage_runner=true).
 
 on:
   workflow_call:
@@ -33,17 +35,25 @@ on:
         type: string
         default: '{"size":1,"arr":[0]}'
       ref:
-        description: 'Git ref (branch, tag, or SHA) to test. If not provided, uses the default branch.'
+        description: 'Git ref (branch, tag, or SHA) to test. If not provided, uses the event commit SHA.'
         type: string
         default: ''
       skip_pr_test_health_check:
-        description: 'Git ref (branch, tag, or SHA) to test. If not provided, uses the default branch.'
+        description: 'Set to true to skip the PR test health check (fast-fail gate).'
         type: string
         default: 'false'
       is_nightly_pipeline_job:
         description: 'Run the test suite with --nightly (collects nightly-registered tests) and --continue-on-error'
         type: boolean
         default: false
+      use_coverage_runner:
+        description: 'If true, delegate test execution to scripts/ci/npu/precise-test/run_tests_with_coverage.sh instead of running via run_suite.py directly'
+        type: boolean
+        default: false
+      upload_test_logs:
+        description: 'If true, upload the test logs as a GitHub Actions artifact (only enabled by pr-test-npu)'
+        type: boolean
+        default: false
       github-token:
         description: 'GitHub token for API calls'
         type: string
@@ -72,7 +82,9 @@ jobs:
       - name: Checkout code
         uses: actions/checkout@v4
         with:
-          ref: ${{ inputs.ref || github.ref }}
+          # Pin to the event SHA (not the branch name) so every job in this run
+          # checks out the same commit even if the branch advances mid-run.
+          ref: ${{ inputs.ref || github.sha }}
 
       - name: Mark repository safe
         run: |
@@ -228,10 +240,12 @@ jobs:
         timeout-minutes: ${{ fromJson(inputs.run_timeout_minutes) }}
         env:
           CONTINUE_ON_ERROR_FLAG: ${{ inputs.continue_on_error == 'true' && '--continue-on-error' || '' }}
+          TEST_LOG_DIR: /tmp/test-logs
         shell: bash
         run: |
           # Fail fast on any command error, undefined variable, or pipe failure.
           set -euo pipefail
+          mkdir -p "${TEST_LOG_DIR}"
 
           # Install missing python deps (skip if already importable).
           PYTHON_FOR_SGLANG="python"
@@ -266,8 +280,38 @@ jobs:
           if [ "${{ inputs.is_nightly_pipeline_job }}" = "true" ]; then
             NIGHTLY_FLAG="--nightly --continue-on-error"
           fi
+
+          # Optional: delegate execution to scripts/ci/npu/precise-test/run_tests_with_coverage.sh when the
+          # caller sets use_coverage_runner=true. list_tests.py (same directory) enumerates the
+          # selected test files; the original run_suite.py execution below is preserved
+          # unchanged for the default (false) path.
+          LOG_FILE="${TEST_LOG_DIR}/${{ inputs.self_name }}-part${{ matrix.partition }}.log"
+
+          if [[ "${{ inputs.use_coverage_runner }}" == "true" ]]; then
+            pip install coverage==7.8.*
+            python3 ../scripts/ci/npu/precise-test/list_tests.py --hw npu --suite ${{ inputs.self_name }} \
+              --auto-partition-id ${{ matrix.partition }} \
+              --auto-partition-size ${{ fromJson(inputs.partitions).size }} \
+              -o /tmp/selected_tests.txt
+            echo "Selected test files:"
+            cat /tmp/selected_tests.txt
+            if [ -s /tmp/selected_tests.txt ]; then
+              bash ../scripts/ci/npu/precise-test/run_tests_with_coverage.sh $(cat /tmp/selected_tests.txt)
+            fi
+            exit 0
+          fi
+
           python3 run_suite.py --hw npu --suite ${{ inputs.self_name }} \
             --auto-partition-id ${{ matrix.partition }} \
             --auto-partition-size ${{ fromJson(inputs.partitions).size }} \
             ${{ inputs.timeout_per_file && format('--timeout-per-file {0}', inputs.timeout_per_file) || '' }} \
-            $NIGHTLY_FLAG $CONTINUE_ON_ERROR_FLAG
+            $NIGHTLY_FLAG $CONTINUE_ON_ERROR_FLAG 2>&1 | tee "${LOG_FILE}"
+
+      - name: Upload test logs
+        if: always() && inputs.use_coverage_runner != true && inputs.upload_test_logs == true
+        uses: actions/upload-artifact@v7
+        with:
+          name: selected-test-logs-${{ inputs.self_name }}-part${{ matrix.partition }}
+          path: /tmp/test-logs/
+          if-no-files-found: ignore
+          retention-days: 14

.github/workflows/_npu-single-node-test-stage.yml

@@ -43,7 +43,7 @@ on:
         default: '{}'
         description: 'JSON run metadata {branch_label, workflow_name, create_time}, recorded once at workflow start'
       skip_pr_test_health_check:
-        description: 'Git ref (branch, tag, or SHA) to test. If not provided, uses the default branch.'
+        description: 'Set to true to skip the PR test health check (fast-fail gate).'
         type: string
         default: 'false'
       is_nightly_pipeline_job:
@@ -54,6 +54,10 @@ on:
         description: 'Partition config to parallelize a suite across jobs, e.g. {"size":3,"arr":[0,1,2]}. Each element of arr is a partition id; the template expands one matrix job per element. size is passed to run_suite.py --auto-partition-size.'
         type: string
         default: '{"size":1,"arr":[0]}'
+      ref:
+        description: 'Git ref (branch, tag, or SHA) to test. If not provided, uses the event SHA.'
+        type: string
+        default: ''
       github-token:
         description: 'GitHub token for API calls'
         type: string
@@ -63,6 +67,14 @@ on:
         type: string
         default: '300'
         description: 'timeout-minutes for the Run test step'
+      use_coverage_runner:
+        description: 'If true, delegate test execution to scripts/ci/npu/precise-test/run_tests_with_coverage.sh instead of running via run_suite.py directly'
+        type: boolean
+        default: false
+      upload_test_logs:
+        description: 'If true, upload the test logs as a GitHub Actions artifact (only enabled by pr-test-npu)'
+        type: boolean
+        default: false
 
 env:
   SKIP_PR_TEST_HEALTH_CHECK: ${{ inputs.skip_pr_test_health_check }}
@@ -84,6 +96,10 @@ jobs:
     steps:
       - name: Checkout code
         uses: actions/checkout@v4
+        with:
+          # Pin to the event SHA (not the branch name) so every job in this run
+          # checks out the same commit even if the branch advances mid-run.
+          ref: ${{ inputs.ref || github.sha }}
 
       - name: Check PR test health
         uses: actions/github-script@v8
@@ -227,6 +243,9 @@ jobs:
           # Fail fast on any command error, undefined variable, or pipe failure.
           set -euo pipefail
 
+          TEST_LOG_DIR="/tmp/test-logs"
+          mkdir -p "${TEST_LOG_DIR}"
+
           sglang_source_path=$(pwd)
           echo "Source code path: ${sglang_source_path}"
           ln -sf ${sglang_source_path} /root/sglang
@@ -390,14 +409,29 @@ jobs:
           fi
 
           # Nightly additionally persists the full suite log to ${log_path}/${tc_name}.log
-          # under the structured path; PR runs only tee to /tmp/test_output.log.
+          # under the structured path; PR runs tee to ${TEST_LOG_DIR}/${tc_name}.log.
           # Use an array so the extra target is omitted when empty: passing an empty
           # string arg makes GNU tee fail (exit code 1) and flips the pipeline exit code.
-          LOG_TEE_TARGETS=("/tmp/test_output.log")
+          LOG_TEE_TARGETS=("${TEST_LOG_DIR}/${tc_name}.log")
           if [ "${{ inputs.is_nightly_pipeline_job }}" = "true" ]; then
             LOG_TEE_TARGETS+=("${log_path}/${tc_name}.log")
           fi
 
+          # Optional: delegate execution to scripts/ci/npu/precise-test/run_tests_with_coverage.sh when the
+          # caller sets use_coverage_runner=true. list_tests.py (same directory) enumerates the
+          # selected test files; the original run_suite.py execution below is preserved
+          # unchanged for the default (false) path.
+          if [[ "${{ inputs.use_coverage_runner }}" == "true" ]]; then
+            pip install coverage==7.8.*
+            ${PYTHON_FOR_SGLANG} -u ../scripts/ci/npu/precise-test/list_tests.py --hw npu --suite ${test_suite} ${PARTITION_ARGS} -o /tmp/selected_tests.txt
+            echo "Selected test files:"
+            cat /tmp/selected_tests.txt
+            if [ -s /tmp/selected_tests.txt ]; then
+              bash ../scripts/ci/npu/precise-test/run_tests_with_coverage.sh $(cat /tmp/selected_tests.txt)
+            fi
+            exit 0
+          fi
+
           # Run NPU test suite with run_suite.py.
           # Capture mode (--enable-retry --max-attempts 1) keeps test subprocesses off the
           # tee pipe, so leftover server processes cannot block the pipeline on exit.
@@ -418,11 +452,11 @@ jobs:
           echo "## ${tc_name}  ${status_icon} ${test_status}" >> $GITHUB_STEP_SUMMARY
           echo "" >> $GITHUB_STEP_SUMMARY
           if [ "${{ inputs.is_nightly_pipeline_job }}" != "true" ]; then
-            metric_count=$(grep -c '\[METRIC\]' /tmp/test_output.log 2>/dev/null || echo 0)
+            metric_count=$(grep -c '\[METRIC\]' "${TEST_LOG_DIR}/${tc_name}.log" 2>/dev/null || echo 0)
             if [ "${metric_count}" -gt 0 ]; then
               echo "| Metric | Value | Pass |" >> $GITHUB_STEP_SUMMARY
               echo "|--------|-------|------|" >> $GITHUB_STEP_SUMMARY
-              grep '\[METRIC\]' /tmp/test_output.log | while IFS= read -r line; do
+              grep '\[METRIC\]' "${TEST_LOG_DIR}/${tc_name}.log" | while IFS= read -r line; do
                 metric_name=$(echo "$line" | sed -E 's/.*\[METRIC\] ([^=]+)=.*/\1/')
                 metric_value=$(echo "$line" | sed -E 's/.*\[METRIC\] [^=]+=([^ ]+).*/\1/')
                 echo "| ${metric_name} | ${metric_value} | ${status_icon} |" >> $GITHUB_STEP_SUMMARY
@@ -433,3 +467,12 @@ jobs:
             echo "" >> $GITHUB_STEP_SUMMARY
           fi
           exit ${test_exit_code}
+
+      - name: Upload test logs
+        if: always() && inputs.use_coverage_runner != true && inputs.upload_test_logs == true
+        uses: actions/upload-artifact@v7
+        with:
+          name: selected-test-logs-${{ inputs.test_suite }}-part${{ matrix.partition }}
+          path: /tmp/test-logs/
+          if-no-files-found: ignore
+          retention-days: 14

.github/workflows/coverage-collection-npu.yml

@@ -0,0 +1,27 @@
+# Coverage data collection pipeline for the NPU precision test selection.
+# Delegates to pr-test-npu.yml with coverage_mode=true, which switches:
+#   - Runners to the coverage pool (shared /root/.cache with setup-covstub)
+#   - use_coverage_runner=true on all test jobs
+#   - Skips pr-gate, base-a (910b), multimodal-gen, recommend, analyze
+#   - Runs setup-covstub to assemble coverage data and publish it to the
+#     shared disk at a fixed path (/root/.cache/tests/precise-test/coverage)
+# pr-test-npu-finish is the final gate in the called workflow.
+#
+# Manually triggered only (workflow_dispatch): PRs run the standard
+# pr-test-npu.yml instead; this pipeline builds the full coverage baseline
+# on demand.
+name: Coverage Collection (NPU)
+
+on:
+  workflow_dispatch:
+
+concurrency:
+  group: npu-coverage-collection-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  coverage:
+    uses: ./.github/workflows/pr-test-npu.yml
+    with:
+      coverage_mode: true
+    secrets: inherit

.github/workflows/pr-test-npu.yml

@@ -7,10 +7,16 @@ on:
     - cron: '0 12 * * *'  # Run daily at 12:00 UTC
   pull_request:
   workflow_dispatch:
+    inputs:
+      coverage_mode:
+        description: 'Run in coverage collection mode (uses coverage runners, enables coverage instrumentation, runs setup-covstub)'
+        required: false
+        type: boolean
+        default: false
   workflow_call:
     inputs:
       ref:
-        description: 'Git ref (branch, tag, or SHA) to test. If not provided, uses the default branch.'
+        description: 'Git ref (branch, tag, or SHA) to test. If not provided, uses the event commit SHA.'
         required: false
         type: string
         default: ''
@@ -19,9 +25,14 @@ on:
         required: false
         type: boolean
         default: false
+      coverage_mode:
+        description: 'Run in coverage collection mode (uses coverage runners, enables coverage instrumentation, runs setup-covstub)'
+        required: false
+        type: boolean
+        default: false
 
 concurrency:
-  group: pr-test-npu-${{ inputs.ref || github.ref }}
+  group: pr-test-npu-${{ inputs.coverage_mode == true && 'coverage-' || '' }}${{ inputs.ref || github.ref }}
   cancel-in-progress: ${{ github.event_name != 'workflow_call' }}
 
 jobs:
@@ -36,7 +47,9 @@ jobs:
       - name: Checkout code
         uses: actions/checkout@v4
         with:
-          ref: ${{ inputs.ref || github.ref }}
+          # Pin to the event SHA (not the branch name) so every job in this run
+          # checks out the same commit even if the branch advances mid-run.
+          ref: ${{ inputs.ref || github.sha }}
 
       - name: Determine run mode
         id: run-mode
@@ -77,7 +90,7 @@ jobs:
   # ==================== PR Gate ==================== #
   pr-gate:
     needs: check-changes
-    if: needs.check-changes.outputs.changes_exist == 'true'
+    if: ${{ inputs.coverage_mode != true && needs.check-changes.outputs.changes_exist == 'true' }}
     uses: ./.github/workflows/pr-gate.yml
     secrets: inherit
 
@@ -96,7 +109,7 @@ jobs:
 
   base-a-test-1-npu-a2:
     needs: [check-changes, pr-gate, set-image-config]
-    if: ${{ !failure() && !cancelled() && needs.check-changes.outputs.main_package == 'true' }}
+    if: ${{ !failure() && !cancelled() && inputs.coverage_mode != true && needs.check-changes.outputs.main_package == 'true' }}
     uses: ./.github/workflows/_npu-pr-test-stage.yml
     with:
       self_name:  base-a-test-1-npu-a2
@@ -108,68 +121,78 @@ jobs:
 
   base-b-test-1-npu-a3:
     needs: [check-changes, pr-gate, set-image-config, base-a-test-1-npu-a2]
-    if: ${{ !failure() && !cancelled() && needs.check-changes.outputs.main_package == 'true' }}
+    if: ${{ !failure() && !cancelled() && (inputs.coverage_mode == true || needs.check-changes.outputs.main_package == 'true') }}
     uses: ./.github/workflows/_npu-pr-test-stage.yml
     with:
       self_name: base-b-test-1-npu-a3
-      runner_config: linux-aarch64-a3-2-
+      runner_config: ${{ inputs.coverage_mode == true && 'linux-aarch64-a3-800t-2' || 'linux-aarch64-a3-2-' }}
       image: ${{ needs.set-image-config.outputs.CANN_image_a3 }}
       run_timeout_minutes: '60'
       timeout_per_file: '3600'
+      use_coverage_runner: ${{ inputs.coverage_mode == true }}
+      upload_test_logs: ${{ inputs.coverage_mode != true }}
     secrets: inherit
 
   base-b-test-2-npu-a3:
     needs: [check-changes, pr-gate, set-image-config, base-a-test-1-npu-a2]
-    if: ${{ !failure() && !cancelled() && needs.check-changes.outputs.main_package == 'true' }}
+    if: ${{ !failure() && !cancelled() && (inputs.coverage_mode == true || needs.check-changes.outputs.main_package == 'true') }}
     uses: ./.github/workflows/_npu-pr-test-stage.yml
     with:
       self_name: base-b-test-2-npu-a3
-      runner_config: linux-aarch64-a3-2-
+      runner_config: ${{ inputs.coverage_mode == true && 'linux-aarch64-a3-800t-2' || 'linux-aarch64-a3-2-' }}
       image: ${{ needs.set-image-config.outputs.CANN_image_a3 }}
       run_timeout_minutes: '60'
       timeout_per_file: '3600'
+      use_coverage_runner: ${{ inputs.coverage_mode == true }}
+      upload_test_logs: ${{ inputs.coverage_mode != true }}
     secrets: inherit
 
   base-b-test-4-npu-a3:
     needs: [check-changes, pr-gate, set-image-config, base-a-test-1-npu-a2]
-    if: ${{ !failure() && !cancelled() && needs.check-changes.outputs.main_package == 'true' }}
+    if: ${{ !failure() && !cancelled() && (inputs.coverage_mode == true || needs.check-changes.outputs.main_package == 'true') }}
     uses: ./.github/workflows/_npu-pr-test-stage.yml
     with:
       self_name: base-b-test-4-npu-a3
-      runner_config: linux-aarch64-a3-4-
+      runner_config: ${{ inputs.coverage_mode == true && 'linux-aarch64-a3-800t-4' || 'linux-aarch64-a3-4-' }}
       image: ${{ needs.set-image-config.outputs.CANN_image_a3 }}
       run_timeout_minutes: '120'
       timeout_per_file: '3600'
       partitions: '{"size":2,"arr":[0, 1]}'
+      use_coverage_runner: ${{ inputs.coverage_mode == true }}
+      upload_test_logs: ${{ inputs.coverage_mode != true }}
     secrets: inherit
 
   base-b-test-8-npu-a3:
     needs: [check-changes, pr-gate, set-image-config, base-a-test-1-npu-a2]
-    if: ${{ !failure() && !cancelled() && needs.check-changes.outputs.main_package == 'true' }}
+    if: ${{ !failure() && !cancelled() && (inputs.coverage_mode == true || needs.check-changes.outputs.main_package == 'true') }}
     uses: ./.github/workflows/_npu-pr-test-stage.yml
     with:
       self_name: base-b-test-8-npu-a3
-      runner_config: linux-aarch64-a3-8-
+      runner_config: ${{ inputs.coverage_mode == true && 'linux-aarch64-a3-800t-8' || 'linux-aarch64-a3-8-' }}
       image: ${{ needs.set-image-config.outputs.CANN_image_a3 }}
       run_timeout_minutes: '60'
       timeout_per_file: '3600'
+      use_coverage_runner: ${{ inputs.coverage_mode == true }}
+      upload_test_logs: ${{ inputs.coverage_mode != true }}
     secrets: inherit
 
   base-b-test-16-npu-a3:
     needs: [check-changes, pr-gate, set-image-config, base-a-test-1-npu-a2]
-    if: ${{ !failure() && !cancelled() && needs.check-changes.outputs.main_package == 'true' }}
+    if: ${{ !failure() && !cancelled() && (inputs.coverage_mode == true || needs.check-changes.outputs.main_package == 'true') }}
     uses: ./.github/workflows/_npu-pr-test-stage.yml
     with:
       self_name: base-b-test-16-npu-a3
-      runner_config: linux-aarch64-a3-16-
+      runner_config: ${{ inputs.coverage_mode == true && 'linux-aarch64-a3-800t-16' || 'linux-aarch64-a3-16-' }}
       image: ${{ needs.set-image-config.outputs.CANN_image_a3 }}
       run_timeout_minutes: '120'
       timeout_per_file: '3600'
+      use_coverage_runner: ${{ inputs.coverage_mode == true }}
+      upload_test_logs: ${{ inputs.coverage_mode != true }}
     secrets: inherit
 
   multimodal-gen-test-1-npu-a3:
     needs: [check-changes, pr-gate, set-image-config, base-a-test-1-npu-a2]
-    if: ${{ !failure() && !cancelled() && needs.check-changes.outputs.multimodal_gen == 'true' }}
+    if: ${{ !failure() && !cancelled() && inputs.coverage_mode != true && needs.check-changes.outputs.multimodal_gen == 'true' }}
     runs-on: linux-aarch64-a3-800t-2
     strategy:
       fail-fast: false
@@ -233,7 +256,7 @@ jobs:
 
   multimodal-gen-test-4-npu-a3:
     needs: [check-changes, pr-gate, set-image-config, base-a-test-1-npu-a2]
-    if: ${{ !failure() && !cancelled() && needs.check-changes.outputs.multimodal_gen == 'true' }}
+    if: ${{ !failure() && !cancelled() && inputs.coverage_mode != true && needs.check-changes.outputs.multimodal_gen == 'true' }}
     runs-on: linux-aarch64-a3-800t-4
     strategy:
       fail-fast: false
@@ -298,34 +321,38 @@ jobs:
   base-c-test-acc-2-npu-a3:
     name: base-c-test-acc-2-npu-a3
     needs: [ check-changes, pr-gate, set-image-config, base-b-test-1-npu-a3, base-b-test-2-npu-a3, base-b-test-4-npu-a3, base-b-test-8-npu-a3, base-b-test-16-npu-a3 ]
-    if: ${{ !failure() && !cancelled() && needs.check-changes.outputs.main_package == 'true' }}
+    if: ${{ !failure() && !cancelled() && (inputs.coverage_mode == true || needs.check-changes.outputs.main_package == 'true') }}
     uses: ./.github/workflows/_npu-single-node-test-stage.yml
     with:
-      runner: linux-aarch64-a3-2-
+      runner: ${{ inputs.coverage_mode == true && 'linux-aarch64-a3-800t-2' || 'linux-aarch64-a3-2-' }}
       test_type: 'accuracy'
       test_suite: base-c-test-acc-2-npu-a3
       image: ${{ needs.set-image-config.outputs.CANN_image_a3 }}
       install_sglang_deps: true
       device_type_for_deps: 'a3'
       partitions: '{"size":2,"arr":[0,1]}'
+      use_coverage_runner: ${{ inputs.coverage_mode == true }}
+      upload_test_logs: ${{ inputs.coverage_mode != true }}
 
   base-c-test-acc-16-npu-a3:
     name: base-c-test-acc-16-npu-a3
     needs: [ check-changes, pr-gate, set-image-config, base-b-test-1-npu-a3, base-b-test-2-npu-a3, base-b-test-4-npu-a3, base-b-test-8-npu-a3, base-b-test-16-npu-a3 ]
-    if: ${{ !failure() && !cancelled() && needs.check-changes.outputs.main_package == 'true' }}
+    if: ${{ !failure() && !cancelled() && (inputs.coverage_mode == true || needs.check-changes.outputs.main_package == 'true') }}
     uses: ./.github/workflows/_npu-single-node-test-stage.yml
     with:
-      runner: linux-aarch64-a3-16-
+      runner: ${{ inputs.coverage_mode == true && 'linux-aarch64-a3-800t-16' || 'linux-aarch64-a3-16-' }}
       test_type: 'accuracy'
       test_suite: base-c-test-acc-16-npu-a3
       image: ${{ needs.set-image-config.outputs.CANN_image_a3 }}
       install_sglang_deps: true
       device_type_for_deps: 'a3'
+      use_coverage_runner: ${{ inputs.coverage_mode == true }}
+      upload_test_logs: ${{ inputs.coverage_mode != true }}
 
   base-c-test-perf-2-npu-a3:
     name: base-c-test-perf-2-npu-a3
     needs: [ check-changes, pr-gate, set-image-config, base-c-test-acc-2-npu-a3, base-c-test-acc-16-npu-a3 ]
-    if: ${{ !failure() && !cancelled() && needs.check-changes.outputs.main_package == 'true' }}
+    if: ${{ !failure() && !cancelled() && (inputs.coverage_mode == true || needs.check-changes.outputs.main_package == 'true') }}
     uses: ./.github/workflows/_npu-single-node-test-stage.yml
     with:
       runner: linux-aarch64-a3-800t-2
@@ -334,11 +361,13 @@ jobs:
       image: ${{ needs.set-image-config.outputs.CANN_image_a3 }}
       install_sglang_deps: true
       device_type_for_deps: 'a3'
+      use_coverage_runner: ${{ inputs.coverage_mode == true }}
+      upload_test_logs: ${{ inputs.coverage_mode != true }}
 
   base-c-test-perf-16-npu-a3:
     name: base-c-test-perf-16-npu-a3
     needs: [ check-changes, pr-gate, set-image-config, base-c-test-acc-2-npu-a3, base-c-test-acc-16-npu-a3 ]
-    if: ${{ !failure() && !cancelled() && needs.check-changes.outputs.main_package == 'true' }}
+    if: ${{ !failure() && !cancelled() && (inputs.coverage_mode == true || needs.check-changes.outputs.main_package == 'true') }}
     uses: ./.github/workflows/_npu-single-node-test-stage.yml
     with:
       runner: linux-aarch64-a3-800t-16
@@ -348,8 +377,262 @@ jobs:
       install_sglang_deps: true
       device_type_for_deps: 'a3'
       test_timeout_minutes: '180'
+      use_coverage_runner: ${{ inputs.coverage_mode == true }}
+      upload_test_logs: ${{ inputs.coverage_mode != true }}
 
 
+  # ==================== Recommend Tests from Coverage ==================== #
+  # Recommends PR-affected test cases from the pre-built coverage baseline.
+  # continue-on-error so it never blocks the PR gate.
+  recommend-tests-from-coverage:
+    name: Recommend tests from coverage
+    runs-on: ubuntu-latest
+    continue-on-error: true
+    if: ${{ !cancelled() && github.event_name == 'pull_request' && inputs.coverage_mode != true }}
+    outputs:
+      coverage_paths: ${{ steps.export-paths.outputs.coverage_paths }}
+    env:
+      GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+    steps:
+      - name: Checkout sglang scripts
+        uses: actions/checkout@v4
+        with:
+          sparse-checkout: |
+            scripts
+          sparse-checkout-cone-mode: false
+
+      - name: Download test case map
+        run: |
+          set -euo pipefail
+          curl -fsSL \
+            --retry 3 \
+            --retry-delay 5 \
+            --retry-all-errors \
+            "https://sglang-npu.obs.cn-southwest-2.myhuaweicloud.com/coverage/test_case_map.json" \
+            -o test_case_map.json
+          test -s test_case_map.json
+          echo "Downloaded test_case_map.json"
+          du -h test_case_map.json
+
+      - name: Download coverage package and extract covstub
+        run: |
+          set -euo pipefail
+          curl -fsSL \
+            --retry 3 \
+            --retry-delay 5 \
+            --retry-all-errors \
+            "https://sglang-npu.obs.cn-southwest-2.myhuaweicloud.com/coverage/outputs.tar.gz" \
+            -o outputs.tar.gz
+          test -s outputs.tar.gz
+          echo "Downloaded outputs.tar.gz"
+          du -h outputs.tar.gz
+          tar xzf outputs.tar.gz outputs/covstub
+          test -d outputs/covstub
+          echo "Extracted outputs/covstub"
+          find outputs/covstub -maxdepth 2 -type d | head -20 || true
+
+      - name: Recommend tests for current PR
+        run: |
+          set -euo pipefail
+          if ! python3 -c "import regex" >/dev/null 2>&1; then
+            echo "regex not found for $(python3 -V); bootstrapping pip and installing regex"
+            curl -fsSL https://bootstrap.pypa.io/get-pip.py -o get-pip.py
+            python3 get-pip.py --break-system-packages
+            python3 -m pip install --break-system-packages regex
+          fi
+          python3 -c "import regex; print('regex ok:', regex.__file__)"
+          PR_SPEC="${{ github.repository }}#${{ github.event.pull_request.number }}"
+          echo "Selecting tests for PR: ${PR_SPEC}"
+          # test_selector.py resolves relative paths against the script's own dir,
+          # so pass absolute paths for files downloaded to the workspace root.
+          python3 scripts/ci/npu/precise-test/test_selector.py \
+            --github-pr "${PR_SPEC}" \
+            --source-dir "$PWD/outputs/covstub" \
+            --map-file "$PWD/test_case_map.json"
+
+      - name: Export coverage paths output
+        id: export-paths
+        if: always()
+        run: |
+          FILE="scripts/ci/npu/precise-test/recommended_pytest_paths.txt"
+          if [ -f "$FILE" ]; then
+            {
+              echo "coverage_paths<<EOF"
+              cat "$FILE"
+              echo "EOF"
+            } >> "$GITHUB_OUTPUT"
+            echo "Exported $(wc -l < "$FILE") recommended paths"
+          else
+            echo "coverage_paths=" >> "$GITHUB_OUTPUT"
+            echo "::notice::recommended_pytest_paths.txt not found"
+          fi
+
+  # ==================== Analyze Failure Report ==================== #
+  # Cross-references test failures with coverage recommendations.
+  # !cancelled() only: skipped jobs in needs poison success().
+  analyze-failure-report:
+    name: Analyze failure report
+    needs:
+      [
+        base-a-test-1-npu-a2,
+        base-b-test-1-npu-a3,
+        base-b-test-2-npu-a3,
+        base-b-test-4-npu-a3,
+        base-b-test-8-npu-a3,
+        base-b-test-16-npu-a3,
+
+        multimodal-gen-test-1-npu-a3,
+        multimodal-gen-test-4-npu-a3,
+
+        base-c-test-acc-2-npu-a3,
+        base-c-test-acc-16-npu-a3,
+        base-c-test-perf-2-npu-a3,
+        base-c-test-perf-16-npu-a3,
+
+        recommend-tests-from-coverage,
+      ]
+    if: ${{ !cancelled() && inputs.coverage_mode != true }}
+    uses: ./.github/workflows/_npu-analyze-failure.yml
+    with:
+      log_artifact_pattern: selected-test-logs-*
+      recommendations_content: ${{ needs.recommend-tests-from-coverage.outputs.coverage_paths || '' }}
+
+  # ==================== Coverage Assembly & Publish ==================== #
+  # Assembles coverage data and publishes the baseline to the shared disk.
+  # !failure() && !cancelled() (not success()): skipped upstream poisons success().
+  setup-covstub:
+    needs:
+      [
+        set-image-config,
+        base-b-test-1-npu-a3,
+        base-b-test-2-npu-a3,
+        base-b-test-4-npu-a3,
+        base-b-test-8-npu-a3,
+        base-b-test-16-npu-a3,
+        base-c-test-acc-2-npu-a3,
+        base-c-test-acc-16-npu-a3,
+        base-c-test-perf-2-npu-a3,
+        base-c-test-perf-16-npu-a3,
+      ]
+    if: ${{ !failure() && !cancelled() && inputs.coverage_mode == true }}
+    runs-on: linux-aarch64-a3-800t-2
+    container:
+      image: ${{ needs.set-image-config.outputs.CANN_image_a3 }}
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+        with:
+          # Pin to the event SHA (not the branch name) so every job in this run
+          # checks out the same commit even if the branch advances mid-run.
+          ref: ${{ inputs.ref || github.sha }}
+
+      - name: Copy sglang source to covstub
+        run: |
+          RUN_DIR="/root/.cache/tests/precise-test/${{ github.run_id }}-attempt-${{ github.run_attempt }}"
+          COVSTUB="${RUN_DIR}/outputs/covstub"
+          mkdir -p "${COVSTUB}"
+          cp -r python/sglang "${COVSTUB}/"
+
+      - name: Build test case map
+        shell: bash
+        run: |
+          set -euo pipefail
+
+          RUN_DIR="/root/.cache/tests/precise-test/${{ github.run_id }}-attempt-${{ github.run_attempt }}"
+          # Date tag derivation must match scripts/ci/npu/precise-test/run_tests_with_coverage.sh.
+          # The custom k8s runner only injects explicitly-set env vars, so
+          # GITHUB_RUN_STARTED_AT may be absent (set -u would abort); fall back
+          # to local date exactly like run_tests_with_coverage.sh does, so both
+          # sides derive the same tag.
+          if [ -n "${GITHUB_RUN_STARTED_AT:-}" ]; then
+            COV_DATE_TAG="${GITHUB_RUN_STARTED_AT:0:10}"
+            COV_DATE_TAG="${COV_DATE_TAG//-/}"
+          else
+            COV_DATE_TAG="$(date +%Y%m%d)"
+          fi
+          COV_ROOT="${RUN_DIR}/outputs/sglang@${COV_DATE_TAG}"
+          COVSTUB="${RUN_DIR}/outputs/covstub"
+          # Placed outside outputs/ so it is NOT included in outputs.tar.gz.
+          MAP_FILE="${RUN_DIR}/test_case_map.json"
+
+          echo "Coverage dir: ${COV_ROOT}"
+          echo "Source dir: ${COVSTUB}"
+
+          # Coverage data may be absent when all test jobs were skipped
+          # (e.g. by the changes filter). Skip map building in that case.
+          if [ ! -d "${COV_ROOT}" ]; then
+            echo "::warning::Coverage dir not found: ${COV_ROOT}, skip building test case map"
+            exit 0
+          fi
+
+          COVERAGE_FILE_COUNT=$(find "${COV_ROOT}" -name 'coverage.*' -type f | wc -l)
+          echo "Found ${COVERAGE_FILE_COUNT} coverage files"
+          if [ "${COVERAGE_FILE_COUNT}" -eq 0 ]; then
+            echo "::warning::No coverage files found under ${COV_ROOT}, skip building test case map"
+            exit 0
+          fi
+
+          if ! python3 -c "import regex" 2>/dev/null; then
+            pip install regex \
+              -i http://cache-service.nginx-pypi-cache.svc.cluster.local/pypi/simple \
+              --trusted-host cache-service.nginx-pypi-cache.svc.cluster.local \
+              --retries 3 --timeout 60 \
+              || pip install regex -i https://mirrors.tuna.tsinghua.edu.cn/pypi/web/simple
+          fi
+
+          python3 scripts/ci/npu/precise-test/test_selector.py \
+            --build-map \
+            --coverage-dir "${COV_ROOT}" \
+            --source-dir "${COVSTUB}" \
+            --map-file "${MAP_FILE}"
+          test -s "${MAP_FILE}"
+          du -h "${MAP_FILE}"
+
+      - name: Archive outputs
+        run: |
+          RUN_DIR="/root/.cache/tests/precise-test/${{ github.run_id }}-attempt-${{ github.run_attempt }}"
+          tar -czf "${RUN_DIR}/outputs.tar.gz" -C "${RUN_DIR}" outputs
+
+      - name: Publish outputs and map to shared disk
+        shell: bash
+        run: |
+          set -euo pipefail
+
+          RUN_DIR="/root/.cache/tests/precise-test/${{ github.run_id }}-attempt-${{ github.run_attempt }}"
+          # Fixed publish dir with no RUN_DIR layer: other pipelines have a
+          # different GITHUB_RUN_ID and must be able to read the baseline.
+          # Must match PUBLISH_DIR in the recommend-tests-from-coverage job.
+          PUBLISH_DIR="/root/.cache/tests/precise-test/coverage"
+
+          # Skip publish when no coverage data was collected this run (map
+          # build was skipped), so we never overwrite the historical
+          # baseline on the shared disk with an incomplete one.
+          if [ ! -s "${RUN_DIR}/test_case_map.json" ]; then
+            echo "::warning::test_case_map.json missing (no coverage data this run), skip publish"
+            exit 0
+          fi
+
+          mkdir -p "${PUBLISH_DIR}"
+
+          # mv (rename) within the same shared filesystem is atomic, so a
+          # concurrent reader sees either the old or the new baseline,
+          # never a partial file.
+          mv "${RUN_DIR}/outputs.tar.gz" "${PUBLISH_DIR}/outputs.tar.gz"
+          mv "${RUN_DIR}/test_case_map.json" "${PUBLISH_DIR}/test_case_map.json"
+
+          echo "Published baseline to ${PUBLISH_DIR}"
+          du -h "${PUBLISH_DIR}/outputs.tar.gz" "${PUBLISH_DIR}/test_case_map.json"
+
+      - name: Clean up per-run directory
+        # Skipped on publish failure, keeping RUN_DIR for manual recovery.
+        shell: bash
+        run: |
+          set -euo pipefail
+          RUN_DIR="/root/.cache/tests/precise-test/${{ github.run_id }}-attempt-${{ github.run_attempt }}"
+          rm -rf "${RUN_DIR}"
+          echo "Removed ${RUN_DIR}"
+
   pr-test-npu-finish:
     needs:
       [
@@ -369,6 +652,8 @@ jobs:
         base-c-test-acc-16-npu-a3,
         base-c-test-perf-2-npu-a3,
         base-c-test-perf-16-npu-a3,
+
+        setup-covstub,
       ]
     if: always()
     runs-on: ubuntu-latest

scripts/ci/npu/precise-test/analyze_failure_report.py

@@ -0,0 +1,525 @@
+#!/usr/bin/env python3
+"""
+analyze_failure_report.py
+
+Cross-reference CI test failures with test recommendations.
+
+Pipeline:
+  1. Scan each log file for failed tests using three methods:
+     a. TIMINGS JSON block (machine-readable, from ci_utils.py)
+     b. ci_utils.py "✗ FAILED:" summary section (structured text)
+     c. pytest "short test summary info" block (for pytest-style logs)
+  2. Read recommended_pytest_paths.txt
+  3. Match: exact match + file-level match
+  4. Generate a Markdown report
+
+Usage:
+  python analyze_failure_report.py --log-dir LOG_DIR --recommendations-file RECOMMENDED.txt [--output report.md]
+"""
+
+import argparse
+import contextlib
+import json
+import sys
+from pathlib import Path
+
+import regex as re
+
+# ============================================================
+#  Utility: strip CI log noise
+# ============================================================
+
+
+def strip_ansi(text):
+    """Remove ANSI color codes like \x1b[31m, \x1b[0m, etc."""
+    return re.sub(r"\x1b\[[0-9;]*m", "", text)
+
+
+def strip_timestamp(line):
+    """Remove GitHub Actions timestamp prefix: YYYY-MM-DDTHH:MM:SS.fffffffZ"""
+    return re.sub(r"^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d+Z\s+", "", line)
+
+
+def clean_line(line):
+    """Strip BOM, ANSI codes, and timestamp from one log line."""
+    line = line.lstrip("\ufeff")  # UTF-8 BOM marker
+    return strip_ansi(strip_timestamp(line)).strip()
+
+
+# ============================================================
+#  Step 1: Extract FAILED and ERROR tests from log files
+# ============================================================
+
+
+# Match pytest-style FAILED/ERROR lines with the test/ prefix (sglang convention).
+FAILED_PATTERN = re.compile(r"^(?:FAILED|ERROR)\s+(test/\S+?\.py(?:::\S+?)?)\s")
+SUMMARY_SEPARATOR_PATTERN = re.compile(r"^=+\s")
+CPU_LOG_PATH_PATTERN = re.compile(r"(?:^|-)cpu-\d+card(?:-|$)", re.IGNORECASE)
+CPU_FAILURE_LABEL = "cpu-ut"
+
+# ci_utils.py summary: "✗ FAILED:" section lines like "  /path/to/test/registered/test_xxx.py (exit code 1)".
+# Paths are absolute (from os.path.abspath in run_suite.py's glob).
+CI_UTILS_FAILED_PATTERN = re.compile(r"^[✗X]\s*FAILED:\s*$")
+CI_UTILS_FAILED_LINE_PATTERN = re.compile(r"^\s{2,}(\S+\.py)\s*\(")
+
+# TIMINGS block: machine-readable JSON lines with "passed": false.
+TIMINGS_BEGIN_PATTERN = re.compile(r"^=+\s*TIMINGS\s+BEGIN\s*=+")
+TIMINGS_END_PATTERN = re.compile(r"^=+\s*TIMINGS\s+END\s*=+")
+
+
+def _extract_from_timings(lines):
+    """Extract failed test file paths from the TIMINGS JSON block (ci_utils.py)."""
+    failed = []
+    in_timings = False
+    for line in lines:
+        text = clean_line(line)
+        if TIMINGS_BEGIN_PATTERN.search(text):
+            in_timings = True
+            continue
+        if not in_timings:
+            continue
+        if TIMINGS_END_PATTERN.search(text):
+            break
+        try:
+            entry = json.loads(text)
+            if not entry.get("passed", True) and entry.get("file"):
+                failed.append(entry["file"])
+        except (json.JSONDecodeError, ValueError):
+            continue
+    return failed
+
+
+def _extract_from_ci_utils_summary(lines):
+    """Extract failed test file paths from ci_utils.py's '✗ FAILED:' summary section."""
+    failed = []
+    in_failed_section = False
+    for line in lines:
+        text = clean_line(line)
+        if CI_UTILS_FAILED_PATTERN.match(text):
+            in_failed_section = True
+            continue
+        if not in_failed_section:
+            continue
+        if SUMMARY_SEPARATOR_PATTERN.match(text):
+            break
+        match = CI_UTILS_FAILED_LINE_PATTERN.match(line)
+        if match:
+            # Paths in this section are absolute (e.g. "/__w/sglang/sglang/test/registered/test_xxx.py").
+            # Strip everything up to and including the "/sglang/" marker to get
+            # the repo-relative form (e.g. "test/registered/test_xxx.py").
+            path = match.group(1)
+            marker = "/sglang/"
+            idx = path.rfind(marker)
+            if idx >= 0:
+                path = path[idx + len(marker) :]
+            failed.append(path)
+    return failed
+
+
+def extract_failed_from_log(log_path):
+    """Extract failed test paths from one log file.
+
+    Tries three methods in order of reliability:
+      1. TIMINGS JSON block (machine-readable, ci_utils.py)
+      2. ci_utils.py '✗ FAILED:' summary section (human-readable but structured)
+      3. pytest 'short test summary info' block (for pytest-style logs)
+    """
+    try:
+        lines = log_path.read_text(encoding="utf-8", errors="replace").splitlines()
+    except Exception as exc:
+        print(f"::warning:: Cannot read {log_path}: {exc}")
+        return []
+
+    # Method 1: TIMINGS block (most reliable, machine-readable).
+    failed = _extract_from_timings(lines)
+    if failed:
+        return failed
+
+    # Method 2: ci_utils.py summary section.
+    failed = _extract_from_ci_utils_summary(lines)
+    if failed:
+        return failed
+
+    # Method 3: pytest-style "short test summary info" block.
+    failed = []
+    in_summary = False
+    for line in lines:
+        text = clean_line(line)
+
+        if "short test summary info" in text:
+            in_summary = True
+            continue
+
+        if not in_summary:
+            continue
+
+        if SUMMARY_SEPARATOR_PATTERN.match(text):
+            in_summary = False
+            continue
+
+        match = FAILED_PATTERN.match(text)
+        if match:
+            failed.append(match.group(1))
+
+    return failed
+
+
+def is_cpu_log(log_path):
+    """Return whether a log belongs to a CPU selected-test artifact."""
+    if log_path.stem.lower().endswith("-cpu-ut"):
+        return True
+
+    return any(CPU_LOG_PATH_PATTERN.search(part) for part in log_path.parent.parts)
+
+
+def extract_failed_from_logs(log_dir):
+    """
+    Scan CPU logs first and represent all CPU failures as one ``cpu-ut`` item.
+    Scan all remaining logs with the existing pytest node-ID behavior.
+    """
+    base = Path(log_dir)
+    if not base.is_dir():
+        print(f"::warning:: Log directory not found: {log_dir}")
+        return []
+
+    # Scan .log files (from NPU test stages) and .txt files (legacy/mock).
+    candidates = []
+    candidates.extend(base.rglob("*.log"))
+    candidates.extend(base.rglob("*.txt"))
+    candidates = [
+        candidate
+        for candidate in sorted(candidates)
+        if candidate.suffix != ".txt" or "run-selected-tests" in candidate.name
+    ]
+
+    cpu_candidates = []
+    regular_candidates = []
+    for candidate in candidates:
+        target = cpu_candidates if is_cpu_log(candidate) else regular_candidates
+        target.append(candidate)
+
+    all_failed = []
+    seen = set()
+
+    cpu_failed = False
+    for candidate in cpu_candidates:
+        if extract_failed_from_log(candidate):
+            cpu_failed = True
+
+    if cpu_failed:
+        seen.add(CPU_FAILURE_LABEL)
+        all_failed.append(CPU_FAILURE_LABEL)
+
+    for candidate in regular_candidates:
+        for test_path in extract_failed_from_log(candidate):
+            if test_path not in seen:
+                seen.add(test_path)
+                all_failed.append(test_path)
+
+    return all_failed
+
+
+# ============================================================
+#  Step 2: Read recommendations
+# ============================================================
+
+
+def read_recommended(recommendations_file):
+    """
+    recommended_pytest_paths.txt contains one pytest path per line, e.g.:
+        test/ops/test_matmul.py::test_bf16
+        test/layers/test_attention.py
+    """
+    path = Path(recommendations_file)
+    if not path.exists():
+        print(f"::warning:: Recommendations file not found: {recommendations_file}")
+        return []
+    raw = path.read_text(encoding="utf-8").lstrip("\ufeff")
+    return [
+        line.strip()
+        for line in raw.splitlines()
+        if line.strip() and not line.startswith("ERROR")
+    ]
+
+
+# ============================================================
+#  Step 3: Match
+# ============================================================
+
+
+def normalize_test_path(test_path):
+    """Return a stable comparison key for a pytest path or node ID."""
+    normalized = test_path.strip().replace("\\", "/").removeprefix("./")
+    file_path, separator, test_name = normalized.partition("::")
+    file_path = file_path.removesuffix(".py")
+    if separator:
+        test_name = test_name.partition("[")[0]
+    return f"{file_path}{separator}{test_name}" if separator else file_path
+
+
+def match_failed_vs_recommended(failed, recommended):
+    """
+    Two-level matching:
+      Level 1 - File-level: recommended "test/foo.py" (no function)
+                  matches failed "test/foo.py::anything"
+      Level 2 - Exact: "test/foo.py::test_bar" in both lists
+
+    Returns {"hit": [...], "miss": [...], "untested": [...]}
+      hit:       failed AND recommended
+      miss:      failed but NOT recommended
+      untested:  recommended but NOT in failed list
+    """
+    recommended_files = {
+        normalize_test_path(item) for item in recommended if "::" not in item
+    }
+    recommended_functions = {
+        normalize_test_path(item) for item in recommended if "::" in item
+    }
+
+    hit = []
+    miss = []
+
+    normalized_failed = {item: normalize_test_path(item) for item in failed}
+    for original, normalized in normalized_failed.items():
+        failed_file = normalized.split("::", 1)[0]
+        if failed_file in recommended_files or normalized in recommended_functions:
+            hit.append(original)
+        else:
+            miss.append(original)
+
+    # Recommended but not failed
+    failed_functions = set(normalized_failed.values())
+    failed_files = {item.split("::", 1)[0] for item in failed_functions}
+    untested = []
+    for item in recommended:
+        normalized = normalize_test_path(item)
+        has_failure = (
+            normalized in failed_functions
+            if "::" in item
+            else normalized in failed_files
+        )
+        if not has_failure:
+            untested.append(item)
+
+    return {"hit": hit, "miss": miss, "untested": untested}
+
+
+# ============================================================
+#  Step 4: Generate Markdown report
+# ============================================================
+
+
+def generate_report(
+    failed, recommended, matched, log_dir, recommendations_source="none"
+):
+    """Produce a Markdown summary table."""
+    hit = matched["hit"]
+    miss = matched["miss"]
+    untested = matched["untested"]
+
+    out = []
+    out.append("# Test Failure vs Recommendation Report")
+    out.append("")
+    out.append(f"**Log source**: `{log_dir}`")
+    out.append("")
+
+    # Recommendation source indicator
+    if recommendations_source == "output":
+        out.append(
+            "> **[Source: Workflow Output]** Recommended cases are passed from coverage recommendations outputs"
+        )
+    elif recommendations_source == "committed":
+        out.append(
+            "> **[Source: Local File]** Recommended test cases come from a txt file in the repository"
+        )
+    else:
+        out.append("> **[Source: None]** No recommended test cases found")
+    out.append("")
+
+    # ================================================================
+    #  Section 1: Full Failed Test List
+    # ================================================================
+    out.append("---")
+    out.append("")
+    out.append(f"## Failed Test Cases（ {len(failed)} total）")
+    out.append("")
+    if failed:
+        for i, t in enumerate(failed, 1):
+            tag = (
+                " **[Matched Recommendation]**"
+                if t in hit
+                else " **[Not Matched Recommendation]**"
+            )
+            out.append(f"{i}. `{t}`{tag}")
+        out.append("")
+    else:
+        out.append("> No failed test cases")
+        out.append("")
+
+    # ================================================================
+    #  Section 2: Full Recommended Test List
+    # ================================================================
+    out.append("---")
+    out.append("")
+    out.append(f"## Recommended Test Cases（ {len(recommended)} total）")
+    out.append("")
+    if recommended:
+        normalized_failed = {normalize_test_path(item) for item in failed}
+        failed_file_set = {item.split("::", 1)[0] for item in normalized_failed}
+        for i, item in enumerate(recommended, 1):
+            normalized = normalize_test_path(item)
+            has_failure = (
+                normalized in normalized_failed
+                if "::" in item
+                else normalized in failed_file_set
+            )
+            tag = " **[Already Failed]**" if has_failure else ""
+            out.append(f"{i}. `{item}`{tag}")
+        out.append("")
+    else:
+        out.append("> No recommended test cases")
+        out.append("")
+
+    # ================================================================
+    #  Section 3: Core Conclusion
+    # ================================================================
+    out.append("---")
+    out.append("")
+    out.append("## Core Conclusion")
+    out.append("")
+    if not failed:
+        out.append(
+            "> No failed cases in this CI run; no need to compare against the recommendation list."
+        )
+    elif len(miss) == 0:
+        out.append("> **All failed test cases are within the recommended scope.**")
+    else:
+        total_failed = len(failed)
+        out.append(
+            f"> ** {len(miss)}/{total_failed} failed cases are outside the recommended scope.**"
+        )
+    out.append("")
+
+    # ================================================================
+    #  Section 4: Detail table
+    # ================================================================
+    out.append("| Category | Count |")
+    out.append("|---|---|")
+    out.append(f"| Failed & Matched Recommendation | {len(hit)} |")
+    out.append(f"| Failed but Not Matched Recommendation | {len(miss)} |")
+    out.append(f"| Recommended but Not Failed | {len(untested)} |")
+    out.append("")
+
+    if hit:
+        out.append("## Failed & Matched Recommendation")
+        out.append("")
+        out.append("| # | Failed test |")
+        out.append("|---|---|")
+        for i, t in enumerate(hit, 1):
+            out.append(f"| {i} | `{t}` |")
+        out.append("")
+
+    if miss:
+        out.append("## Failed but Not Matched Recommendation")
+        out.append("")
+        out.append(
+            "> Possible causes: uncovered modules, environment issues, flaky tests."
+        )
+        out.append("")
+        for t in miss:
+            out.append(f"- `{t}`")
+        out.append("")
+
+    if untested:
+        out.append("## Recommended but Not Failed")
+        out.append("")
+        out.append(
+            "> These test cases were recommended but did not fail this run (passed or not executed)."
+        )
+        out.append("")
+        for t in untested:
+            out.append(f"- `{t}`")
+        out.append("")
+
+    if not hit and not miss:
+        out.append("## No failed cases")
+        out.append("")
+
+    out.append("---")
+    out.append("*Generated by analyze_failure_report.py*")
+    return "\n".join(out)
+
+
+def main():
+    parser = argparse.ArgumentParser(
+        description="Cross-reference CI test failures with test recommendations"
+    )
+    parser.add_argument(
+        "--log-dir", required=True, help="Directory containing CI .log files"
+    )
+    parser.add_argument(
+        "--recommendations-file",
+        help="Path to recommended_pytest_paths.txt",
+    )
+    parser.add_argument(
+        "--output",
+        default="failure_report.md",
+        help="Output Markdown report path (default: failure_report.md)",
+    )
+    parser.add_argument(
+        "--recommendations-source",
+        default="none",
+        choices=["committed", "output", "none"],
+        help="Where recommendations came from",
+    )
+    args = parser.parse_args()
+
+    if not args.recommendations_file:
+        parser.error("--recommendations-file is required")
+
+    # For Windows console: force UTF-8 if possible
+    if sys.platform == "win32":
+        with contextlib.suppress(Exception):
+            sys.stdout.reconfigure(encoding="utf-8", errors="replace")
+
+    print("=" * 50)
+    print("Step 1: Extract failed tests from CI logs")
+    print("=" * 50)
+    failed = extract_failed_from_logs(args.log_dir)
+    print(f"Failed: {len(failed)}")
+
+    print()
+    print("=" * 50)
+    print("Step 2: Read recommendations")
+    print("=" * 50)
+    recommended = read_recommended(args.recommendations_file)
+    print(f"Recommended: {len(recommended)}")
+
+    print()
+    print("=" * 50)
+    print("Step 3: Match")
+    print("=" * 50)
+    matched = match_failed_vs_recommended(failed, recommended)
+    print(f"Hit (failed + recommended): {len(matched['hit'])}")
+    print(f"Miss (failed, not recommended): {len(matched['miss'])}")
+    print(f"Untested (recommended, no failure): {len(matched['untested'])}")
+
+    report = generate_report(
+        failed, recommended, matched, args.log_dir, args.recommendations_source
+    )
+    output_path = Path(args.output)
+    output_path.parent.mkdir(parents=True, exist_ok=True)
+    output_path.write_text(report, encoding="utf-8")
+    print()
+    print(f"Report => {output_path}")
+    print()
+
+    # Print report to stdout (safe fallback for Windows encoding)
+    try:
+        print(report)
+    except UnicodeEncodeError:
+        print(report.encode("ascii", errors="replace").decode("ascii"))
+
+
+if __name__ == "__main__":
+    main()

scripts/ci/npu/precise-test/coveragerc

@@ -0,0 +1,16 @@
+[run]
+branch = True
+relative_files = False
+parallel = True
+concurrency = thread,multiprocessing
+sigterm = False
+disable_warnings = no-data-collected,module-not-python
+
+include =
+    */python/sglang/*
+
+omit =
+    */.local/*
+    /usr/*
+    */sglang/test/*
+    */sglang/benchmark/*

scripts/ci/npu/precise-test/list_tests.py

@@ -0,0 +1,139 @@
+#!/usr/bin/env python3
+"""
+List the test files selected for a CI suite without running them.
+
+Standalone extraction of `test/run_suite.py --list-tests-output`: discover
+the registered tests under test/registered/, filter them by hardware
+backend / suite (PR per-commit tests only; nightly-registered tests are
+excluded), then write the selected test file paths (one per line) to the
+output file.
+
+Usage:
+    python3 list_tests.py --hw npu --suite base-b-test-1-npu-a3 \
+        [--auto-partition-id N --auto-partition-size M] \
+        -o /tmp/selected_tests.txt
+"""
+
+import argparse
+import glob
+import os
+import sys
+from pathlib import Path
+
+# Repo layout: this script lives at <repo>/scripts/ci/npu/precise-test/.
+SCRIPT_DIR = Path(__file__).resolve().parent
+REPO_ROOT = SCRIPT_DIR.parents[3]
+
+# ci_register.py is stdlib-only; import it directly (bypassing the sglang
+# package __init__, which pulls in torch) so this script runs anywhere.
+sys.path.insert(0, str(REPO_ROOT / "python" / "sglang" / "test" / "ci"))
+
+from ci_register import HWBackend, auto_partition, collect_tests  # noqa: E402
+
+HW_MAPPING = {
+    "cpu": HWBackend.CPU,
+    "cuda": HWBackend.CUDA,
+    "amd": HWBackend.AMD,
+    "musa": HWBackend.MUSA,
+    "npu": HWBackend.NPU,
+    "xpu": HWBackend.XPU,
+    "mlx": HWBackend.MLX,
+}
+
+
+def main():
+    parser = argparse.ArgumentParser(
+        description=(
+            "Write the test files selected for a CI suite (one per line) "
+            "without running them."
+        )
+    )
+    parser.add_argument(
+        "--hw",
+        type=str,
+        choices=HW_MAPPING.keys(),
+        required=True,
+        help="Hardware backend to select tests for.",
+    )
+    parser.add_argument(
+        "--suite",
+        type=str,
+        required=True,
+        help=(
+            "Test suite to select. Accepts a comma-separated list of suites; "
+            "their tests are unioned."
+        ),
+    )
+    parser.add_argument(
+        "--auto-partition-id",
+        type=int,
+        help="Use auto load balancing. The part id.",
+    )
+    parser.add_argument(
+        "--auto-partition-size",
+        type=int,
+        help="Use auto load balancing. The number of parts.",
+    )
+    parser.add_argument(
+        "--output",
+        "-o",
+        type=str,
+        required=True,
+        help="Write selected test file paths (one per line) to this file.",
+    )
+    args = parser.parse_args()
+
+    # Validate auto-partition arguments (same rules as run_suite.py).
+    if (args.auto_partition_id is not None) != (args.auto_partition_size is not None):
+        parser.error(
+            "--auto-partition-id and --auto-partition-size must be specified together."
+        )
+    if args.auto_partition_size is not None:
+        if args.auto_partition_size <= 0:
+            parser.error("--auto-partition-size must be positive.")
+        if not 0 <= args.auto_partition_id < args.auto_partition_size:
+            parser.error(
+                f"--auto-partition-id must be in range [0, {args.auto_partition_size}), "
+                f"but got {args.auto_partition_id}"
+            )
+
+    hw = HW_MAPPING[args.hw]
+    suites = {s.strip() for s in args.suite.split(",") if s.strip()}
+
+    # Registered tests under <repo>/test/registered/
+    files = [
+        f
+        for f in glob.glob(
+            str(REPO_ROOT / "test" / "registered" / "**" / "*.py"), recursive=True
+        )
+        # conftest.py / __init__.py are pytest+package structure, never
+        # registered tests, and must not be listed as one.
+        if os.path.basename(f) not in ("conftest.py", "__init__.py")
+    ]
+    all_tests = collect_tests(files)
+
+    # Same filter as run_suite.py PR mode: backend + suite, per-commit
+    # (non-nightly) tests only, enabled only.
+    ci_tests = [
+        t
+        for t in all_tests
+        if t.backend == hw
+        and t.effective_suite in suites
+        and not t.nightly
+        and t.disabled is None
+    ]
+
+    # Shard the selected tests across runners (LPT, same as run_suite.py).
+    # NPU workflows rely on this to split one suite across matrix jobs.
+    if args.auto_partition_size:
+        ci_tests = auto_partition(
+            ci_tests, args.auto_partition_id, args.auto_partition_size
+        )
+
+    with open(args.output, "w") as f:
+        for t in ci_tests:
+            f.write(t.filename + "\n")
+
+
+if __name__ == "__main__":
+    main()

scripts/ci/npu/precise-test/run_tests_with_coverage.sh

@@ -0,0 +1,126 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+
+CACHE_ROOT="/root/.cache/tests/precise-test"
+
+# Per-CI-run, per-attempt unique directory (no cross-run residue, no overwrite on re-run)
+RUN_ID="${GITHUB_RUN_ID:-local}"
+RUN_ATTEMPT="${GITHUB_RUN_ATTEMPT:-1}"
+RUN_DIR="${RUN_ID}-attempt-${RUN_ATTEMPT}"
+
+# Date tag for grouping coverage data. In CI, prefer GITHUB_RUN_STARTED_AT
+# (same value across all jobs in one run, immune to midnight rollover).
+# Fall back to local date for non-CI execution.
+if [ -n "${GITHUB_RUN_STARTED_AT:-}" ]; then
+  COV_DATE_TAG="${GITHUB_RUN_STARTED_AT:0:10}"
+  COV_DATE_TAG="${COV_DATE_TAG//-/}"
+else
+  COV_DATE_TAG="$(date +%Y%m%d)"
+fi
+COV_ROOT="${CACHE_ROOT}/${RUN_DIR}/outputs/sglang@${COV_DATE_TAG}"
+
+mkdir -p "${COV_ROOT}"
+
+targets=("$@")
+if [ "${#targets[@]}" -eq 0 ]; then
+  echo "Usage: $0 <test> [test ...]"
+  exit 1
+fi
+
+overall_status=0
+
+results=()
+
+# Derive a filesystem-safe directory name from a test target:
+#   1. strip the trailing ".py"
+#   2. flatten path separators:  /  ->  __
+#   3. flatten pytest separators: ::  ->  --
+#   4. replace any remaining unsafe character with "_"
+# Each test gets its own COVERAGE_FILE so results never collide.
+setup_coverage() {
+  local target="$1"
+  local name="${target%.py}"
+  name="${name//\//__}"
+  name="${name//::/--}"
+  name="${name//[^a-zA-Z0-9_.-]/_}"
+  local covdir="${COV_ROOT}/${name}"
+  mkdir -p "${covdir}"
+  export COVERAGE_FILE="${covdir}/coverage"
+}
+
+run_one() {
+  local target="$1"
+
+  echo "=== Running: ${target} ==="
+  setup_coverage "${target}"
+
+  set +e
+  python -m coverage run --rcfile="${SCRIPT_DIR}/coveragerc" -m pytest -sv --color=yes "${target}" 2>&1
+  local status=$?
+  set -e
+
+  if [ "${status}" -ne 0 ]; then
+    echo "1" > "$(dirname "${COVERAGE_FILE}")/FAILED"
+    echo "=== FAILED: ${target} ==="
+    overall_status=1
+    results+=("${target}|FAILED")
+  else
+    echo "=== PASSED: ${target} ==="
+    results+=("${target}|PASSED")
+  fi
+}
+
+for target in "${targets[@]}"; do
+  run_one "${target}"
+done
+
+# ====================
+# Test result summary
+# ====================
+passed_list=()
+failed_list=()
+
+for entry in "${results[@]}"; do
+  test_name="${entry%%|*}"
+  test_status="${entry##*|}"
+  if [ "${test_status}" = "PASSED" ]; then
+    passed_list+=("${test_name}")
+  else
+    failed_list+=("${test_name}")
+  fi
+done
+
+passed_count="${#passed_list[@]}"
+failed_count="${#failed_list[@]}"
+total_count=$((passed_count + failed_count))
+
+echo
+echo "============================================================"
+echo "Test Summary: total ${total_count}, passed ${passed_count}, failed ${failed_count}"
+echo "============================================================"
+
+if [ "${passed_count}" -gt 0 ]; then
+  echo "✓ PASSED:"
+  for t in "${passed_list[@]}"; do
+    echo "  ${t}"
+  done
+fi
+
+if [ "${failed_count}" -gt 0 ]; then
+  echo
+  echo "✗ FAILED:"
+  for t in "${failed_list[@]}"; do
+    echo "  ${t}"
+  done
+fi
+
+echo "============================================================"
+
+if [ "${failed_count}" -gt 0 ]; then
+  echo "ERROR: Some tests failed."
+fi
+echo "Coverage: ${COV_ROOT}/"
+
+exit "${overall_status}"