[NPU] add coverage-based precision test selection pipeline (#38339)

.github/workflows/_npu-analyze-failure.yml

@@ -0,0 +1,96 @@
+# Reusable workflow: cross-reference CI test failures with coverage-based
+# test recommendations. Called by pr-test-npu.yml's analyze-failure-report job.
+name: Analyze Failure Report
+
+on:
+  workflow_call:
+    inputs:
+      log_artifact_pattern:
+        type: string
+        required: true
+        description: 'Glob pattern for test log artifacts (e.g. selected-test-logs-*)'
+      recommendations_content:
+        type: string
+        required: false
+        default: ''
+        description: 'Content of recommendations from upstream job output'
+
+defaults:
+  run:
+    shell: bash -el {0}
+
+jobs:
+  analyze:
+    name: cross-reference failures with recommendations
+    continue-on-error: true  # Non-blocking on parse/read failure.
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout sglang repo
+        uses: actions/checkout@v4
+
+      # ----- Step 1: Locate recommendations -----
+      - name: Locate recommendations file
+        id: locate-recs
+        run: |
+          mkdir -p ./test-logs
+
+          if [ -n "${{ inputs.recommendations_content }}" ]; then
+            echo "has_recommendations=true" >> $GITHUB_OUTPUT
+            echo "Using workflow output recommendations"
+            echo "${{ inputs.recommendations_content }}" > ./test-logs/_recommended.txt
+            echo "RECS_FILE=./test-logs/_recommended.txt" >> $GITHUB_ENV
+            echo "RECS_SOURCE=output" >> $GITHUB_ENV
+            COUNT=$(wc -l < ./test-logs/_recommended.txt)
+            echo "Recommendations count: ${COUNT}"
+          else
+            echo "has_recommendations=false" >> $GITHUB_OUTPUT
+            echo "::notice::No recommended test cases detected, skipping subsequent analysis tasks."
+          fi
+
+      # ----- Step 2: Download test logs -----
+      - name: Download test logs
+        uses: actions/download-artifact@v7
+        with:
+          pattern: ${{ inputs.log_artifact_pattern }}
+          path: ./test-logs
+          merge-multiple: true
+          continue-on-error: true
+
+      - name: Show downloaded logs structure
+        run: |
+          echo "Downloaded log files:"
+          find ./test-logs -type f | head -50 || echo "  (no logs found)"
+
+      # ----- Step 3: Run analysis -----
+      - name: Ensure regex is available
+        if: steps.locate-recs.outputs.has_recommendations != 'false'
+        run: |
+          set -euo pipefail
+          if ! python3 -c "import regex" >/dev/null 2>&1; then
+            echo "regex not found for $(python3 -V); bootstrapping pip and installing regex"
+            curl -fsSL https://bootstrap.pypa.io/get-pip.py -o get-pip.py
+            python3 get-pip.py --break-system-packages
+            python3 -m pip install --break-system-packages regex
+          fi
+          python3 -c "import regex; print('regex ok:', regex.__file__)"
+
+      - name: Run failure analysis
+        if: steps.locate-recs.outputs.has_recommendations != 'false'
+        id: analysis
+        continue-on-error: true
+        run: |
+          python3 scripts/ci/npu/precise-test/analyze_failure_report.py \
+            --log-dir ./test-logs \
+            --recommendations-file "${RECS_FILE}" \
+            --recommendations-source "${RECS_SOURCE:-none}" \
+            --output ./test-logs/failure_report.md
+
+      # ----- Step 4: Upload report -----
+      - name: Upload failure analysis report
+        if: always()
+        uses: actions/upload-artifact@v7
+        with:
+          name: failure-analysis-report
+          path: ./test-logs/failure_report.md
+          if-no-files-found: ignore
+          retention-days: 14

.github/workflows/_npu-pr-test-stage.yml

@@ -1,5 +1,7 @@
 name: PR Test Stage for NPU
-# Reusable workflow for one CUDA test stage. Caller pr-test-npu.yml forwards
+# Reusable workflow for one NPU test stage. Caller pr-test-npu.yml forwards
+# job parameters (runner, image, partitions) and optionally delegates test
+# execution to the coverage runner (use_coverage_runner=true).
 
 on:
   workflow_call:
@@ -33,17 +35,25 @@ on:
         type: string
         default: '{"size":1,"arr":[0]}'
       ref:
-        description: 'Git ref (branch, tag, or SHA) to test. If not provided, uses the default branch.'
+        description: 'Git ref (branch, tag, or SHA) to test. If not provided, uses the event commit SHA.'
         type: string
         default: ''
       skip_pr_test_health_check:
-        description: 'Git ref (branch, tag, or SHA) to test. If not provided, uses the default branch.'
+        description: 'Set to true to skip the PR test health check (fast-fail gate).'
         type: string
         default: 'false'
       is_nightly_pipeline_job:
         description: 'Run the test suite with --nightly (collects nightly-registered tests) and --continue-on-error'
         type: boolean
         default: false
+      use_coverage_runner:
+        description: 'If true, delegate test execution to scripts/ci/npu/precise-test/run_tests_with_coverage.sh instead of running via run_suite.py directly'
+        type: boolean
+        default: false
+      upload_test_logs:
+        description: 'If true, upload the test logs as a GitHub Actions artifact (only enabled by pr-test-npu)'
+        type: boolean
+        default: false
       github-token:
         description: 'GitHub token for API calls'
         type: string
@@ -72,7 +82,9 @@ jobs:
       - name: Checkout code
         uses: actions/checkout@v4
         with:
-          ref: ${{ inputs.ref || github.ref }}
+          # Pin to the event SHA (not the branch name) so every job in this run
+          # checks out the same commit even if the branch advances mid-run.
+          ref: ${{ inputs.ref || github.sha }}
 
       - name: Mark repository safe
         run: |
@@ -228,10 +240,12 @@ jobs:
         timeout-minutes: ${{ fromJson(inputs.run_timeout_minutes) }}
         env:
           CONTINUE_ON_ERROR_FLAG: ${{ inputs.continue_on_error == 'true' && '--continue-on-error' || '' }}
+          TEST_LOG_DIR: /tmp/test-logs
         shell: bash
         run: |
           # Fail fast on any command error, undefined variable, or pipe failure.
           set -euo pipefail
+          mkdir -p "${TEST_LOG_DIR}"
 
           # Install missing python deps (skip if already importable).
           PYTHON_FOR_SGLANG="python"
@@ -266,8 +280,38 @@ jobs:
           if [ "${{ inputs.is_nightly_pipeline_job }}" = "true" ]; then
             NIGHTLY_FLAG="--nightly --continue-on-error"
           fi
+
+          # Optional: delegate execution to scripts/ci/npu/precise-test/run_tests_with_coverage.sh when the
+          # caller sets use_coverage_runner=true. list_tests.py (same directory) enumerates the
+          # selected test files; the original run_suite.py execution below is preserved
+          # unchanged for the default (false) path.
+          LOG_FILE="${TEST_LOG_DIR}/${{ inputs.self_name }}-part${{ matrix.partition }}.log"
+
+          if [[ "${{ inputs.use_coverage_runner }}" == "true" ]]; then
+            pip install coverage==7.8.*
+            python3 ../scripts/ci/npu/precise-test/list_tests.py --hw npu --suite ${{ inputs.self_name }} \
+              --auto-partition-id ${{ matrix.partition }} \
+              --auto-partition-size ${{ fromJson(inputs.partitions).size }} \
+              -o /tmp/selected_tests.txt
+            echo "Selected test files:"
+            cat /tmp/selected_tests.txt
+            if [ -s /tmp/selected_tests.txt ]; then
+              bash ../scripts/ci/npu/precise-test/run_tests_with_coverage.sh $(cat /tmp/selected_tests.txt)
+            fi
+            exit 0
+          fi
+
           python3 run_suite.py --hw npu --suite ${{ inputs.self_name }} \
             --auto-partition-id ${{ matrix.partition }} \
             --auto-partition-size ${{ fromJson(inputs.partitions).size }} \
             ${{ inputs.timeout_per_file && format('--timeout-per-file {0}', inputs.timeout_per_file) || '' }} \
-            $NIGHTLY_FLAG $CONTINUE_ON_ERROR_FLAG
+            $NIGHTLY_FLAG $CONTINUE_ON_ERROR_FLAG 2>&1 | tee "${LOG_FILE}"
+
+      - name: Upload test logs
+        if: always() && inputs.use_coverage_runner != true && inputs.upload_test_logs == true
+        uses: actions/upload-artifact@v7
+        with:
+          name: selected-test-logs-${{ inputs.self_name }}-part${{ matrix.partition }}
+          path: /tmp/test-logs/
+          if-no-files-found: ignore
+          retention-days: 14

.github/workflows/_npu-single-node-test-stage.yml

@@ -43,7 +43,7 @@ on:
         default: '{}'
         description: 'JSON run metadata {branch_label, workflow_name, create_time}, recorded once at workflow start'
       skip_pr_test_health_check:
-        description: 'Git ref (branch, tag, or SHA) to test. If not provided, uses the default branch.'
+        description: 'Set to true to skip the PR test health check (fast-fail gate).'
         type: string
         default: 'false'
       is_nightly_pipeline_job:
@@ -54,6 +54,10 @@ on:
         description: 'Partition config to parallelize a suite across jobs, e.g. {"size":3,"arr":[0,1,2]}. Each element of arr is a partition id; the template expands one matrix job per element. size is passed to run_suite.py --auto-partition-size.'
         type: string
         default: '{"size":1,"arr":[0]}'
+      ref:
+        description: 'Git ref (branch, tag, or SHA) to test. If not provided, uses the event SHA.'
+        type: string
+        default: ''
       github-token:
         description: 'GitHub token for API calls'
         type: string
@@ -63,6 +67,14 @@ on:
         type: string
         default: '300'
         description: 'timeout-minutes for the Run test step'
+      use_coverage_runner:
+        description: 'If true, delegate test execution to scripts/ci/npu/precise-test/run_tests_with_coverage.sh instead of running via run_suite.py directly'
+        type: boolean
+        default: false
+      upload_test_logs:
+        description: 'If true, upload the test logs as a GitHub Actions artifact (only enabled by pr-test-npu)'
+        type: boolean
+        default: false
 
 env:
   SKIP_PR_TEST_HEALTH_CHECK: ${{ inputs.skip_pr_test_health_check }}
@@ -84,6 +96,10 @@ jobs:
     steps:
       - name: Checkout code
         uses: actions/checkout@v4
+        with:
+          # Pin to the event SHA (not the branch name) so every job in this run
+          # checks out the same commit even if the branch advances mid-run.
+          ref: ${{ inputs.ref || github.sha }}
 
       - name: Check PR test health
         uses: actions/github-script@v8
@@ -227,6 +243,9 @@ jobs:
           # Fail fast on any command error, undefined variable, or pipe failure.
           set -euo pipefail
 
+          TEST_LOG_DIR="/tmp/test-logs"
+          mkdir -p "${TEST_LOG_DIR}"
+
           sglang_source_path=$(pwd)
           echo "Source code path: ${sglang_source_path}"
           ln -sf ${sglang_source_path} /root/sglang
@@ -390,14 +409,29 @@ jobs:
           fi
 
           # Nightly additionally persists the full suite log to ${log_path}/${tc_name}.log
-          # under the structured path; PR runs only tee to /tmp/test_output.log.
+          # under the structured path; PR runs tee to ${TEST_LOG_DIR}/${tc_name}.log.
           # Use an array so the extra target is omitted when empty: passing an empty
           # string arg makes GNU tee fail (exit code 1) and flips the pipeline exit code.
-          LOG_TEE_TARGETS=("/tmp/test_output.log")
+          LOG_TEE_TARGETS=("${TEST_LOG_DIR}/${tc_name}.log")
           if [ "${{ inputs.is_nightly_pipeline_job }}" = "true" ]; then
             LOG_TEE_TARGETS+=("${log_path}/${tc_name}.log")
           fi
 
+          # Optional: delegate execution to scripts/ci/npu/precise-test/run_tests_with_coverage.sh when the
+          # caller sets use_coverage_runner=true. list_tests.py (same directory) enumerates the
+          # selected test files; the original run_suite.py execution below is preserved
+          # unchanged for the default (false) path.
+          if [[ "${{ inputs.use_coverage_runner }}" == "true" ]]; then
+            pip install coverage==7.8.*
+            ${PYTHON_FOR_SGLANG} -u ../scripts/ci/npu/precise-test/list_tests.py --hw npu --suite ${test_suite} ${PARTITION_ARGS} -o /tmp/selected_tests.txt
+            echo "Selected test files:"
+            cat /tmp/selected_tests.txt
+            if [ -s /tmp/selected_tests.txt ]; then
+              bash ../scripts/ci/npu/precise-test/run_tests_with_coverage.sh $(cat /tmp/selected_tests.txt)
+            fi
+            exit 0
+          fi
+
           # Run NPU test suite with run_suite.py.
           # Capture mode (--enable-retry --max-attempts 1) keeps test subprocesses off the
           # tee pipe, so leftover server processes cannot block the pipeline on exit.
@@ -418,11 +452,11 @@ jobs:
           echo "## ${tc_name}  ${status_icon} ${test_status}" >> $GITHUB_STEP_SUMMARY
           echo "" >> $GITHUB_STEP_SUMMARY
           if [ "${{ inputs.is_nightly_pipeline_job }}" != "true" ]; then
-            metric_count=$(grep -c '\[METRIC\]' /tmp/test_output.log 2>/dev/null || echo 0)
+            metric_count=$(grep -c '\[METRIC\]' "${TEST_LOG_DIR}/${tc_name}.log" 2>/dev/null || echo 0)
             if [ "${metric_count}" -gt 0 ]; then
               echo "| Metric | Value | Pass |" >> $GITHUB_STEP_SUMMARY
               echo "|--------|-------|------|" >> $GITHUB_STEP_SUMMARY
-              grep '\[METRIC\]' /tmp/test_output.log | while IFS= read -r line; do
+              grep '\[METRIC\]' "${TEST_LOG_DIR}/${tc_name}.log" | while IFS= read -r line; do
                 metric_name=$(echo "$line" | sed -E 's/.*\[METRIC\] ([^=]+)=.*/\1/')
                 metric_value=$(echo "$line" | sed -E 's/.*\[METRIC\] [^=]+=([^ ]+).*/\1/')
                 echo "| ${metric_name} | ${metric_value} | ${status_icon} |" >> $GITHUB_STEP_SUMMARY
@@ -433,3 +467,12 @@ jobs:
             echo "" >> $GITHUB_STEP_SUMMARY
           fi
           exit ${test_exit_code}
+
+      - name: Upload test logs
+        if: always() && inputs.use_coverage_runner != true && inputs.upload_test_logs == true
+        uses: actions/upload-artifact@v7
+        with:
+          name: selected-test-logs-${{ inputs.test_suite }}-part${{ matrix.partition }}
+          path: /tmp/test-logs/
+          if-no-files-found: ignore
+          retention-days: 14

.github/workflows/coverage-collection-npu.yml

@@ -0,0 +1,27 @@
+# Coverage data collection pipeline for the NPU precision test selection.
+# Delegates to pr-test-npu.yml with coverage_mode=true, which switches:
+#   - Runners to the coverage pool (shared /root/.cache with setup-covstub)
+#   - use_coverage_runner=true on all test jobs
+#   - Skips pr-gate, base-a (910b), multimodal-gen, recommend, analyze
+#   - Runs setup-covstub to assemble coverage data and publish it to the
+#     shared disk at a fixed path (/root/.cache/tests/precise-test/coverage)
+# pr-test-npu-finish is the final gate in the called workflow.
+#
+# Manually triggered only (workflow_dispatch): PRs run the standard
+# pr-test-npu.yml instead; this pipeline builds the full coverage baseline
+# on demand.
+name: Coverage Collection (NPU)
+
+on:
+  workflow_dispatch:
+
+concurrency:
+  group: npu-coverage-collection-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  coverage:
+    uses: ./.github/workflows/pr-test-npu.yml
+    with:
+      coverage_mode: true
+    secrets: inherit

.github/workflows/pr-test-npu.yml

@@ -7,10 +7,16 @@ on:
     - cron: '0 12 * * *'  # Run daily at 12:00 UTC
   pull_request:
   workflow_dispatch:
+    inputs:
+      coverage_mode:
+        description: 'Run in coverage collection mode (uses coverage runners, enables coverage instrumentation, runs setup-covstub)'
+        required: false
+        type: boolean
+        default: false
   workflow_call:
     inputs:
       ref:
-        description: 'Git ref (branch, tag, or SHA) to test. If not provided, uses the default branch.'
+        description: 'Git ref (branch, tag, or SHA) to test. If not provided, uses the event commit SHA.'
         required: false
         type: string
         default: ''
@@ -19,9 +25,14 @@ on:
         required: false
         type: boolean
         default: false
+      coverage_mode:
+        description: 'Run in coverage collection mode (uses coverage runners, enables coverage instrumentation, runs setup-covstub)'
+        required: false
+        type: boolean
+        default: false
 
 concurrency:
-  group: pr-test-npu-${{ inputs.ref || github.ref }}
+  group: pr-test-npu-${{ inputs.coverage_mode == true && 'coverage-' || '' }}${{ inputs.ref || github.ref }}
   cancel-in-progress: ${{ github.event_name != 'workflow_call' }}
 
 jobs:
@@ -36,7 +47,9 @@ jobs:
       - name: Checkout code
         uses: actions/checkout@v4
         with:
-          ref: ${{ inputs.ref || github.ref }}
+          # Pin to the event SHA (not the branch name) so every job in this run
+          # checks out the same commit even if the branch advances mid-run.
+          ref: ${{ inputs.ref || github.sha }}
 
       - name: Determine run mode
         id: run-mode
@@ -77,7 +90,7 @@ jobs:
   # ==================== PR Gate ==================== #
   pr-gate:
     needs: check-changes
-    if: needs.check-changes.outputs.changes_exist == 'true'
+    if: ${{ inputs.coverage_mode != true && needs.check-changes.outputs.changes_exist == 'true' }}
     uses: ./.github/workflows/pr-gate.yml
     secrets: inherit
 
@@ -96,7 +109,7 @@ jobs:
 
   base-a-test-1-npu-a2:
     needs: [check-changes, pr-gate, set-image-config]
-    if: ${{ !failure() && !cancelled() && needs.check-changes.outputs.main_package == 'true' }}
+    if: ${{ !failure() && !cancelled() && inputs.coverage_mode != true && needs.check-changes.outputs.main_package == 'true' }}
     uses: ./.github/workflows/_npu-pr-test-stage.yml
     with:
       self_name:  base-a-test-1-npu-a2
@@ -108,68 +121,78 @@ jobs:
 
   base-b-test-1-npu-a3:
     needs: [check-changes, pr-gate, set-image-config, base-a-test-1-npu-a2]
-    if: ${{ !failure() && !cancelled() && needs.check-changes.outputs.main_package == 'true' }}
+    if: ${{ !failure() && !cancelled() && (inputs.coverage_mode == true || needs.check-changes.outputs.main_package == 'true') }}
     uses: ./.github/workflows/_npu-pr-test-stage.yml
     with:
       self_name: base-b-test-1-npu-a3
-      runner_config: linux-aarch64-a3-2-
+      runner_config: ${{ inputs.coverage_mode == true && 'linux-aarch64-a3-800t-2' || 'linux-aarch64-a3-2-' }}
       image: ${{ needs.set-image-config.outputs.CANN_image_a3 }}
       run_timeout_minutes: '60'
       timeout_per_file: '3600'
+      use_coverage_runner: ${{ inputs.coverage_mode == true }}
+      upload_test_logs: ${{ inputs.coverage_mode != true }}
     secrets: inherit
 
   base-b-test-2-npu-a3:
     needs: [check-changes, pr-gate, set-image-config, base-a-test-1-npu-a2]
-    if: ${{ !failure() && !cancelled() && needs.check-changes.outputs.main_package == 'true' }}
+    if: ${{ !failure() && !cancelled() && (inputs.coverage_mode == true || needs.check-changes.outputs.main_package == 'true') }}
     uses: ./.github/workflows/_npu-pr-test-stage.yml
     with:
       self_name: base-b-test-2-npu-a3
-      runner_config: linux-aarch64-a3-2-
+      runner_config: ${{ inputs.coverage_mode == true && 'linux-aarch64-a3-800t-2' || 'linux-aarch64-a3-2-' }}
       image: ${{ needs.set-image-config.outputs.CANN_image_a3 }}
       run_timeout_minutes: '60'
       timeout_per_file: '3600'
+      use_coverage_runner: ${{ inputs.coverage_mode == true }}
+      upload_test_logs: ${{ inputs.coverage_mode != true }}
     secrets: inherit
 
   base-b-test-4-npu-a3:
     needs: [check-changes, pr-gate, set-image-config, base-a-test-1-npu-a2]
-    if: ${{ !failure() && !cancelled() && needs.check-changes.outputs.main_package == 'true' }}
+    if: ${{ !failure() && !cancelled() && (inputs.coverage_mode == true || needs.check-changes.outputs.main_package == 'true') }}
     uses: ./.github/workflows/_npu-pr-test-stage.yml
     with:
       self_name: base-b-test-4-npu-a3
-      runner_config: linux-aarch64-a3-4-
+      runner_config: ${{ inputs.coverage_mode == true && 'linux-aarch64-a3-800t-4' || 'linux-aarch64-a3-4-' }}
       image: ${{ needs.set-image-config.outputs.CANN_image_a3 }}
       run_timeout_minutes: '120'
       timeout_per_file: '3600'
       partitions: '{"size":2,"arr":[0, 1]}'
+      use_coverage_runner: ${{ inputs.coverage_mode == true }}
+      upload_test_logs: ${{ inputs.coverage_mode != true }}
     secrets: inherit
 
   base-b-test-8-npu-a3:
     needs: [check-changes, pr-gate, set-image-config, base-a-test-1-npu-a2]
-    if: ${{ !failure() && !cancelled() && needs.check-changes.outputs.main_package == 'true' }}
+    if: ${{ !failure() && !cancelled() && (inputs.coverage_mode == true || needs.check-changes.outputs.main_package == 'true') }}
     uses: ./.github/workflows/_npu-pr-test-stage.yml
     with:
       self_name: base-b-test-8-npu-a3
-      runner_config: linux-aarch64-a3-8-
+      runner_config: ${{ inputs.coverage_mode == true && 'linux-aarch64-a3-800t-8' || 'linux-aarch64-a3-8-' }}
       image: ${{ needs.set-image-config.outputs.CANN_image_a3 }}
       run_timeout_minutes: '60'
       timeout_per_file: '3600'
+      use_coverage_runner: ${{ inputs.coverage_mode == true }}
+      upload_test_logs: ${{ inputs.coverage_mode != true }}
     secrets: inherit
 
   base-b-test-16-npu-a3:
     needs: [check-changes, pr-gate, set-image-config, base-a-test-1-npu-a2]
-    if: ${{ !failure() && !cancelled() && needs.check-changes.outputs.main_package == 'true' }}
+    if: ${{ !failure() && !cancelled() && (inputs.coverage_mode == true || needs.check-changes.outputs.main_package == 'true') }}
     uses: ./.github/workflows/_npu-pr-test-stage.yml
     with:
       self_name: base-b-test-16-npu-a3
-      runner_config: linux-aarch64-a3-16-
+      runner_config: ${{ inputs.coverage_mode == true && 'linux-aarch64-a3-800t-16' || 'linux-aarch64-a3-16-' }}
       image: ${{ needs.set-image-config.outputs.CANN_image_a3 }}
       run_timeout_minutes: '120'
       timeout_per_file: '3600'
+      use_coverage_runner: ${{ inputs.coverage_mode == true }}
+      upload_test_logs: ${{ inputs.coverage_mode != true }}
     secrets: inherit
 
   multimodal-gen-test-1-npu-a3:
     needs: [check-changes, pr-gate, set-image-config, base-a-test-1-npu-a2]
-    if: ${{ !failure() && !cancelled() && needs.check-changes.outputs.multimodal_gen == 'true' }}
+    if: ${{ !failure() && !cancelled() && inputs.coverage_mode != true && needs.check-changes.outputs.multimodal_gen == 'true' }}
     runs-on: linux-aarch64-a3-800t-2
     strategy:
       fail-fast: false
@@ -233,7 +256,7 @@ jobs:
 
   multimodal-gen-test-4-npu-a3:
     needs: [check-changes, pr-gate, set-image-config, base-a-test-1-npu-a2]
-    if: ${{ !failure() && !cancelled() && needs.check-changes.outputs.multimodal_gen == 'true' }}
+    if: ${{ !failure() && !cancelled() && inputs.coverage_mode != true && needs.check-changes.outputs.multimodal_gen == 'true' }}
     runs-on: linux-aarch64-a3-800t-4
     strategy:
       fail-fast: false
@@ -298,34 +321,38 @@ jobs:
   base-c-test-acc-2-npu-a3:
     name: base-c-test-acc-2-npu-a3
     needs: [ check-changes, pr-gate, set-image-config, base-b-test-1-npu-a3, base-b-test-2-npu-a3, base-b-test-4-npu-a3, base-b-test-8-npu-a3, base-b-test-16-npu-a3 ]
-    if: ${{ !failure() && !cancelled() && needs.check-changes.outputs.main_package == 'true' }}
+    if: ${{ !failure() && !cancelled() && (inputs.coverage_mode == true || needs.check-changes.outputs.main_package == 'true') }}
     uses: ./.github/workflows/_npu-single-node-test-stage.yml
     with:
-      runner: linux-aarch64-a3-2-
+      runner: ${{ inputs.coverage_mode == true && 'linux-aarch64-a3-800t-2' || 'linux-aarch64-a3-2-' }}
       test_type: 'accuracy'
       test_suite: base-c-test-acc-2-npu-a3
       image: ${{ needs.set-image-config.outputs.CANN_image_a3 }}
       install_sglang_deps: true
       device_type_for_deps: 'a3'
       partitions: '{"size":2,"arr":[0,1]}'
+      use_coverage_runner: ${{ inputs.coverage_mode == true }}
+      upload_test_logs: ${{ inputs.coverage_mode != true }}
 
   base-c-test-acc-16-npu-a3:
     name: base-c-test-acc-16-npu-a3
     needs: [ check-changes, pr-gate, set-image-config, base-b-test-1-npu-a3, base-b-test-2-npu-a3, base-b-test-4-npu-a3, base-b-test-8-npu-a3, base-b-test-16-npu-a3 ]
-    if: ${{ !failure() && !cancelled() && needs.check-changes.outputs.main_package == 'true' }}
+    if: ${{ !failure() && !cancelled() && (inputs.coverage_mode == true || needs.check-changes.outputs.main_package == 'true') }}
     uses: ./.github/workflows/_npu-single-node-test-stage.yml
     with:
-      runner: linux-aarch64-a3-16-
+      runner: ${{ inputs.coverage_mode == true && 'linux-aarch64-a3-800t-16' || 'linux-aarch64-a3-16-' }}
       test_type: 'accuracy'
       test_suite: base-c-test-acc-16-npu-a3
       image: ${{ needs.set-image-config.outputs.CANN_image_a3 }}
       install_sglang_deps: true
       device_type_for_deps: 'a3'
+      use_coverage_runner: ${{ inputs.coverage_mode == true }}
+      upload_test_logs: ${{ inputs.coverage_mode != true }}
 
   base-c-test-perf-2-npu-a3:
     name: base-c-test-perf-2-npu-a3
     needs: [ check-changes, pr-gate, set-image-config, base-c-test-acc-2-npu-a3, base-c-test-acc-16-npu-a3 ]
-    if: ${{ !failure() && !cancelled() && needs.check-changes.outputs.main_package == 'true' }}
+    if: ${{ !failure() && !cancelled() && (inputs.coverage_mode == true || needs.check-changes.outputs.main_package == 'true') }}
     uses: ./.github/workflows/_npu-single-node-test-stage.yml
     with:
       runner: linux-aarch64-a3-800t-2
@@ -334,11 +361,13 @@ jobs:
       image: ${{ needs.set-image-config.outputs.CANN_image_a3 }}
       install_sglang_deps: true
       device_type_for_deps: 'a3'
+      use_coverage_runner: ${{ inputs.coverage_mode == true }}
+      upload_test_logs: ${{ inputs.coverage_mode != true }}
 
   base-c-test-perf-16-npu-a3:
     name: base-c-test-perf-16-npu-a3
     needs: [ check-changes, pr-gate, set-image-config, base-c-test-acc-2-npu-a3, base-c-test-acc-16-npu-a3 ]
-    if: ${{ !failure() && !cancelled() && needs.check-changes.outputs.main_package == 'true' }}
+    if: ${{ !failure() && !cancelled() && (inputs.coverage_mode == true || needs.check-changes.outputs.main_package == 'true') }}
     uses: ./.github/workflows/_npu-single-node-test-stage.yml
     with:
       runner: linux-aarch64-a3-800t-16
@@ -348,8 +377,262 @@ jobs:
       install_sglang_deps: true
       device_type_for_deps: 'a3'
       test_timeout_minutes: '180'
+      use_coverage_runner: ${{ inputs.coverage_mode == true }}
+      upload_test_logs: ${{ inputs.coverage_mode != true }}
 
 
+  # ==================== Recommend Tests from Coverage ==================== #
+  # Recommends PR-affected test cases from the pre-built coverage baseline.
+  # continue-on-error so it never blocks the PR gate.
+  recommend-tests-from-coverage:
+    name: Recommend tests from coverage
+    runs-on: ubuntu-latest
+    continue-on-error: true
+    if: ${{ !cancelled() && github.event_name == 'pull_request' && inputs.coverage_mode != true }}
+    outputs:
+      coverage_paths: ${{ steps.export-paths.outputs.coverage_paths }}
+    env:
+      GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+    steps:
+      - name: Checkout sglang scripts
+        uses: actions/checkout@v4
+        with:
+          sparse-checkout: |
+            scripts
+          sparse-checkout-cone-mode: false
+
+      - name: Download test case map
+        run: |
+          set -euo pipefail
+          curl -fsSL \
+            --retry 3 \
+            --retry-delay 5 \
+            --retry-all-errors \
+            "https://sglang-npu.obs.cn-southwest-2.myhuaweicloud.com/coverage/test_case_map.json" \
+            -o test_case_map.json
+          test -s test_case_map.json
+          echo "Downloaded test_case_map.json"
+          du -h test_case_map.json
+
+      - name: Download coverage package and extract covstub
+        run: |
+          set -euo pipefail
+          curl -fsSL \
+            --retry 3 \
+            --retry-delay 5 \
+            --retry-all-errors \
+            "https://sglang-npu.obs.cn-southwest-2.myhuaweicloud.com/coverage/outputs.tar.gz" \
+            -o outputs.tar.gz
+          test -s outputs.tar.gz
+          echo "Downloaded outputs.tar.gz"
+          du -h outputs.tar.gz
+          tar xzf outputs.tar.gz outputs/covstub
+          test -d outputs/covstub
+          echo "Extracted outputs/covstub"
+          find outputs/covstub -maxdepth 2 -type d | head -20 || true
+
+      - name: Recommend tests for current PR
+        run: |
+          set -euo pipefail
+          if ! python3 -c "import regex" >/dev/null 2>&1; then
+            echo "regex not found for $(python3 -V); bootstrapping pip and installing regex"
+            curl -fsSL https://bootstrap.pypa.io/get-pip.py -o get-pip.py
+            python3 get-pip.py --break-system-packages
+            python3 -m pip install --break-system-packages regex
+          fi
+          python3 -c "import regex; print('regex ok:', regex.__file__)"
+          PR_SPEC="${{ github.repository }}#${{ github.event.pull_request.number }}"
+          echo "Selecting tests for PR: ${PR_SPEC}"
+          # test_selector.py resolves relative paths against the script's own dir,
+          # so pass absolute paths for files downloaded to the workspace root.
+          python3 scripts/ci/npu/precise-test/test_selector.py \
+            --github-pr "${PR_SPEC}" \
+            --source-dir "$PWD/outputs/covstub" \
+            --map-file "$PWD/test_case_map.json"
+
+      - name: Export coverage paths output
+        id: export-paths
+        if: always()
+        run: |
+          FILE="scripts/ci/npu/precise-test/recommended_pytest_paths.txt"
+          if [ -f "$FILE" ]; then
+            {
+              echo "coverage_paths<<EOF"
+              cat "$FILE"
+              echo "EOF"
+            } >> "$GITHUB_OUTPUT"
+            echo "Exported $(wc -l < "$FILE") recommended paths"
+          else
+            echo "coverage_paths=" >> "$GITHUB_OUTPUT"
+            echo "::notice::recommended_pytest_paths.txt not found"
+          fi
+
+  # ==================== Analyze Failure Report ==================== #
+  # Cross-references test failures with coverage recommendations.
+  # !cancelled() only: skipped jobs in needs poison success().
+  analyze-failure-report:
+    name: Analyze failure report
+    needs:
+      [
+        base-a-test-1-npu-a2,
+        base-b-test-1-npu-a3,
+        base-b-test-2-npu-a3,
+        base-b-test-4-npu-a3,
+        base-b-test-8-npu-a3,
+        base-b-test-16-npu-a3,
+
+        multimodal-gen-test-1-npu-a3,
+        multimodal-gen-test-4-npu-a3,
+
+        base-c-test-acc-2-npu-a3,
+        base-c-test-acc-16-npu-a3,
+        base-c-test-perf-2-npu-a3,
+        base-c-test-perf-16-npu-a3,
+
+        recommend-tests-from-coverage,
+      ]
+    if: ${{ !cancelled() && inputs.coverage_mode != true }}
+    uses: ./.github/workflows/_npu-analyze-failure.yml
+    with:
+      log_artifact_pattern: selected-test-logs-*
+      recommendations_content: ${{ needs.recommend-tests-from-coverage.outputs.coverage_paths || '' }}
+
+  # ==================== Coverage Assembly & Publish ==================== #
+  # Assembles coverage data and publishes the baseline to the shared disk.
+  # !failure() && !cancelled() (not success()): skipped upstream poisons success().
+  setup-covstub:
+    needs:
+      [
+        set-image-config,
+        base-b-test-1-npu-a3,
+        base-b-test-2-npu-a3,
+        base-b-test-4-npu-a3,
+        base-b-test-8-npu-a3,
+        base-b-test-16-npu-a3,
+        base-c-test-acc-2-npu-a3,
+        base-c-test-acc-16-npu-a3,
+        base-c-test-perf-2-npu-a3,
+        base-c-test-perf-16-npu-a3,
+      ]
+    if: ${{ !failure() && !cancelled() && inputs.coverage_mode == true }}
+    runs-on: linux-aarch64-a3-800t-2
+    container:
+      image: ${{ needs.set-image-config.outputs.CANN_image_a3 }}
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+        with:
+          # Pin to the event SHA (not the branch name) so every job in this run
+          # checks out the same commit even if the branch advances mid-run.
+          ref: ${{ inputs.ref || github.sha }}
+
+      - name: Copy sglang source to covstub
+        run: |
+          RUN_DIR="/root/.cache/tests/precise-test/${{ github.run_id }}-attempt-${{ github.run_attempt }}"
+          COVSTUB="${RUN_DIR}/outputs/covstub"
+          mkdir -p "${COVSTUB}"
+          cp -r python/sglang "${COVSTUB}/"
+
+      - name: Build test case map
+        shell: bash
+        run: |
+          set -euo pipefail
+
+          RUN_DIR="/root/.cache/tests/precise-test/${{ github.run_id }}-attempt-${{ github.run_attempt }}"
+          # Date tag derivation must match scripts/ci/npu/precise-test/run_tests_with_coverage.sh.
+          # The custom k8s runner only injects explicitly-set env vars, so
+          # GITHUB_RUN_STARTED_AT may be absent (set -u would abort); fall back
+          # to local date exactly like run_tests_with_coverage.sh does, so both
+          # sides derive the same tag.
+          if [ -n "${GITHUB_RUN_STARTED_AT:-}" ]; then
+            COV_DATE_TAG="${GITHUB_RUN_STARTED_AT:0:10}"
+            COV_DATE_TAG="${COV_DATE_TAG//-/}"
+          else
+            COV_DATE_TAG="$(date +%Y%m%d)"
+          fi
+          COV_ROOT="${RUN_DIR}/outputs/sglang@${COV_DATE_TAG}"
+          COVSTUB="${RUN_DIR}/outputs/covstub"
+          # Placed outside outputs/ so it is NOT included in outputs.tar.gz.
+          MAP_FILE="${RUN_DIR}/test_case_map.json"
+
+          echo "Coverage dir: ${COV_ROOT}"
+          echo "Source dir: ${COVSTUB}"
+
+          # Coverage data may be absent when all test jobs were skipped
+          # (e.g. by the changes filter). Skip map building in that case.
+          if [ ! -d "${COV_ROOT}" ]; then
+            echo "::warning::Coverage dir not found: ${COV_ROOT}, skip building test case map"
+            exit 0
+          fi
+
+          COVERAGE_FILE_COUNT=$(find "${COV_ROOT}" -name 'coverage.*' -type f | wc -l)
+          echo "Found ${COVERAGE_FILE_COUNT} coverage files"
+          if [ "${COVERAGE_FILE_COUNT}" -eq 0 ]; then
+            echo "::warning::No coverage files found under ${COV_ROOT}, skip building test case map"
+            exit 0
+          fi
+
+          if ! python3 -c "import regex" 2>/dev/null; then
+            pip install regex \
+              -i http://cache-service.nginx-pypi-cache.svc.cluster.local/pypi/simple \
+              --trusted-host cache-service.nginx-pypi-cache.svc.cluster.local \
+              --retries 3 --timeout 60 \
+              || pip install regex -i https://mirrors.tuna.tsinghua.edu.cn/pypi/web/simple
+          fi
+
+          python3 scripts/ci/npu/precise-test/test_selector.py \
+            --build-map \
+            --coverage-dir "${COV_ROOT}" \
+            --source-dir "${COVSTUB}" \
+            --map-file "${MAP_FILE}"
+          test -s "${MAP_FILE}"
+          du -h "${MAP_FILE}"
+
+      - name: Archive outputs
+        run: |
+          RUN_DIR="/root/.cache/tests/precise-test/${{ github.run_id }}-attempt-${{ github.run_attempt }}"
+          tar -czf "${RUN_DIR}/outputs.tar.gz" -C "${RUN_DIR}" outputs
+
+      - name: Publish outputs and map to shared disk
+        shell: bash
+        run: |
+          set -euo pipefail
+
+          RUN_DIR="/root/.cache/tests/precise-test/${{ github.run_id }}-attempt-${{ github.run_attempt }}"
+          # Fixed publish dir with no RUN_DIR layer: other pipelines have a
+          # different GITHUB_RUN_ID and must be able to read the baseline.
+          # Must match PUBLISH_DIR in the recommend-tests-from-coverage job.
+          PUBLISH_DIR="/root/.cache/tests/precise-test/coverage"
+
+          # Skip publish when no coverage data was collected this run (map
+          # build was skipped), so we never overwrite the historical
+          # baseline on the shared disk with an incomplete one.
+          if [ ! -s "${RUN_DIR}/test_case_map.json" ]; then
+            echo "::warning::test_case_map.json missing (no coverage data this run), skip publish"
+            exit 0
+          fi
+
+          mkdir -p "${PUBLISH_DIR}"
+
+          # mv (rename) within the same shared filesystem is atomic, so a
+          # concurrent reader sees either the old or the new baseline,
+          # never a partial file.
+          mv "${RUN_DIR}/outputs.tar.gz" "${PUBLISH_DIR}/outputs.tar.gz"
+          mv "${RUN_DIR}/test_case_map.json" "${PUBLISH_DIR}/test_case_map.json"
+
+          echo "Published baseline to ${PUBLISH_DIR}"
+          du -h "${PUBLISH_DIR}/outputs.tar.gz" "${PUBLISH_DIR}/test_case_map.json"
+
+      - name: Clean up per-run directory
+        # Skipped on publish failure, keeping RUN_DIR for manual recovery.
+        shell: bash
+        run: |
+          set -euo pipefail
+          RUN_DIR="/root/.cache/tests/precise-test/${{ github.run_id }}-attempt-${{ github.run_attempt }}"
+          rm -rf "${RUN_DIR}"
+          echo "Removed ${RUN_DIR}"
+
   pr-test-npu-finish:
     needs:
       [
@@ -369,6 +652,8 @@ jobs:
         base-c-test-acc-16-npu-a3,
         base-c-test-perf-2-npu-a3,
         base-c-test-perf-16-npu-a3,
+
+        setup-covstub,
       ]
     if: always()
     runs-on: ubuntu-latest