From 0948e6ebed82c7dd7df8a0c347076e46af84bef6 Mon Sep 17 00:00:00 2001 From: hhhh1252023 <139246842+hhhh1252023@users.noreply.github.com> Date: Sat, 5 Sep 2026 17:16:52 +0800 Subject: [PATCH] [CI] Remove metrics artifact mechanism from nightly NPU workflows (#35489) --- .../workflows/_npu-single-node-test-stage.yml | 27 ++++++------ .../nightly-test-npu-e2e-multi-node.yml | 8 ---- .github/workflows/nightly-test-npu.yml | 43 ------------------- .../test/ascend/e2e/run_npu_e2e_test.py | 11 +++-- 4 files changed, 21 insertions(+), 68 deletions(-) diff --git a/.github/workflows/_npu-single-node-test-stage.yml b/.github/workflows/_npu-single-node-test-stage.yml index c0895c29a..e65a7ee66 100644 --- a/.github/workflows/_npu-single-node-test-stage.yml +++ b/.github/workflows/_npu-single-node-test-stage.yml @@ -411,18 +411,19 @@ jobs: export test_status tc_name echo "## ${tc_name} ${status_icon} ${test_status}" >> $GITHUB_STEP_SUMMARY echo "" >> $GITHUB_STEP_SUMMARY - metric_count=$(grep -c '\[METRIC\]' /tmp/test_output.log 2>/dev/null || echo 0) - if [ "${metric_count}" -gt 0 ]; then - echo "| Metric | Value | Pass |" >> $GITHUB_STEP_SUMMARY - echo "|--------|-------|------|" >> $GITHUB_STEP_SUMMARY - grep '\[METRIC\]' /tmp/test_output.log | while IFS= read -r line; do - metric_name=$(echo "$line" | sed -E 's/.*\[METRIC\] ([^=]+)=.*/\1/') - metric_value=$(echo "$line" | sed -E 's/.*\[METRIC\] [^=]+=([^ ]+).*/\1/') - echo "| ${metric_name} | ${metric_value} | ${status_icon} |" >> $GITHUB_STEP_SUMMARY - done - else - echo "No metrics collected (test may have failed before producing results)." >> $GITHUB_STEP_SUMMARY + if [ "${{ inputs.is_nightly_pipeline_job }}" != "true" ]; then + metric_count=$(grep -c '\[METRIC\]' /tmp/test_output.log 2>/dev/null || echo 0) + if [ "${metric_count}" -gt 0 ]; then + echo "| Metric | Value | Pass |" >> $GITHUB_STEP_SUMMARY + echo "|--------|-------|------|" >> $GITHUB_STEP_SUMMARY + grep '\[METRIC\]' /tmp/test_output.log | while IFS= read -r line; do + metric_name=$(echo "$line" | sed -E 's/.*\[METRIC\] ([^=]+)=.*/\1/') + metric_value=$(echo "$line" | sed -E 's/.*\[METRIC\] [^=]+=([^ ]+).*/\1/') + echo "| ${metric_name} | ${metric_value} | ${status_icon} |" >> $GITHUB_STEP_SUMMARY + done + else + echo "No metrics collected (test may have failed before producing results)." >> $GITHUB_STEP_SUMMARY + fi + echo "" >> $GITHUB_STEP_SUMMARY fi - echo "" >> $GITHUB_STEP_SUMMARY - exit ${test_exit_code} diff --git a/.github/workflows/nightly-test-npu-e2e-multi-node.yml b/.github/workflows/nightly-test-npu-e2e-multi-node.yml index 100e4cf42..7ef09e42f 100644 --- a/.github/workflows/nightly-test-npu-e2e-multi-node.yml +++ b/.github/workflows/nightly-test-npu-e2e-multi-node.yml @@ -225,14 +225,6 @@ jobs: echo "Run command: ${CMD}" eval "${CMD}" - - name: Upload metrics - if: always() - uses: actions/upload-artifact@v4 - with: - name: metrics-${{ inputs.test_config_name }} - path: /tmp/metrics.json - retention-days: 7 - - name: Post process if: always() run: | diff --git a/.github/workflows/nightly-test-npu.yml b/.github/workflows/nightly-test-npu.yml index bff9ac74c..cd29a6c16 100644 --- a/.github/workflows/nightly-test-npu.yml +++ b/.github/workflows/nightly-test-npu.yml @@ -453,13 +453,6 @@ jobs: - nightly-poc-multi-node-mix-tests runs-on: ubuntu-latest steps: - - name: Download all metrics - uses: actions/download-artifact@v4 - with: - pattern: metrics-* - path: /tmp/metrics - merge-multiple: false - - name: Generate results table run: | single_result_a2="${{ needs.nightly-1-npu-a2.result }}" @@ -522,42 +515,6 @@ jobs: echo "| multi-node-mix-poc | $(group_icon ${mix_result}) ${mix_result} |" >> $GITHUB_STEP_SUMMARY echo "" >> $GITHUB_STEP_SUMMARY - echo "## Per-Test Metrics" >> $GITHUB_STEP_SUMMARY - echo "" >> $GITHUB_STEP_SUMMARY - - has_metrics=false - # Each suite job uploads its METRICS_DATA_FILE directory, which - # contains per-case subdirectories with their own metrics.json. - # Recursively find every metrics.json and render one table row per - # test case. - for file in $(find /tmp/metrics -name metrics.json 2>/dev/null); do - if [ -f "$file" ]; then - has_metrics=true - echo "import json" > /tmp/parse_metrics.py - echo "with open('$file') as f:" >> /tmp/parse_metrics.py - echo " d = json.load(f)" >> /tmp/parse_metrics.py - echo "tc = d.get('test_case', '-')" >> /tmp/parse_metrics.py - echo "tp = d.get('test_type', '-')" >> /tmp/parse_metrics.py - echo "st = d.get('status', 'unknown')" >> /tmp/parse_metrics.py - echo "icon = '✅' if st == 'pass' else '❌'" >> /tmp/parse_metrics.py - echo "metrics = d.get('metrics', {})" >> /tmp/parse_metrics.py - echo "baselines = d.get('baselines', {})" >> /tmp/parse_metrics.py - echo "mstr = ', '.join(f'{k}={v}' for k,v in metrics.items()) if metrics else '-'" >> /tmp/parse_metrics.py - echo "bstr = ', '.join(f'{k}={v}' for k,v in baselines.items()) if baselines else '-'" >> /tmp/parse_metrics.py - echo "print(f'| {tc} | {tp} | {icon} {st} | {mstr} | {bstr} |')" >> /tmp/parse_metrics.py - python3 /tmp/parse_metrics.py - fi - done > /tmp/metrics_table.txt - - if [ "${has_metrics}" = "true" ]; then - echo "| Test Case | Type | Status | Metrics | Baseline |" >> $GITHUB_STEP_SUMMARY - echo "|-----------|------|--------|---------|----------|" >> $GITHUB_STEP_SUMMARY - cat /tmp/metrics_table.txt >> $GITHUB_STEP_SUMMARY - else - echo "No per-test metrics available (artifacts not found)." >> $GITHUB_STEP_SUMMARY - fi - echo "" >> $GITHUB_STEP_SUMMARY - FAIL=0 for r in "${single_result_a2}" "${single_result}" "${multi_result}" "${mix_result}"; do if [ "${r}" != "success" ] && [ "${r}" != "skipped" ]; then FAIL=1; fi diff --git a/python/sglang/test/ascend/e2e/run_npu_e2e_test.py b/python/sglang/test/ascend/e2e/run_npu_e2e_test.py index 9e24293df..f0ba2ee50 100644 --- a/python/sglang/test/ascend/e2e/run_npu_e2e_test.py +++ b/python/sglang/test/ascend/e2e/run_npu_e2e_test.py @@ -501,6 +501,13 @@ def monitor_pod_logs( def generate_metrics_json(metrics_data_file, test_case, status): + """Collect [METRIC] lines from the log and write metrics.json. + + This writes ${metrics_data_file}/metrics.json only, which feeds the + persistence-directory output (e.g. /root/.cache/tests/output/...). + It is NOT consumed by CI artifacts (the former /tmp/metrics.json + + 'Upload metrics' path was removed). + """ log_file = os.path.join(metrics_data_file, "test_output.log") metrics = {} @@ -553,10 +560,6 @@ def generate_metrics_json(metrics_data_file, test_case, status): json.dump(output, f, indent=2) logger.info(f"Metrics JSON written to {output_path}") - with open("/tmp/metrics.json", "w") as f: - json.dump(output, f, indent=2) - logger.info("Metrics JSON written to /tmp/metrics.json") - def run_npu_e2e_test_case( docker_image_url: str,