[CI] Remove metrics artifact mechanism from nightly NPU workflows (#35489)
This commit is contained in:
hhhh1252023
@@ -411,18 +411,19 @@ jobs:
|
||||
export test_status tc_name
|
||||
echo "## ${tc_name} ${status_icon} ${test_status}" >> $GITHUB_STEP_SUMMARY
|
||||
echo "" >> $GITHUB_STEP_SUMMARY
|
||||
metric_count=$(grep -c '\[METRIC\]' /tmp/test_output.log 2>/dev/null || echo 0)
|
||||
if [ "${metric_count}" -gt 0 ]; then
|
||||
echo "| Metric | Value | Pass |" >> $GITHUB_STEP_SUMMARY
|
||||
echo "|--------|-------|------|" >> $GITHUB_STEP_SUMMARY
|
||||
grep '\[METRIC\]' /tmp/test_output.log | while IFS= read -r line; do
|
||||
metric_name=$(echo "$line" | sed -E 's/.*\[METRIC\] ([^=]+)=.*/\1/')
|
||||
metric_value=$(echo "$line" | sed -E 's/.*\[METRIC\] [^=]+=([^ ]+).*/\1/')
|
||||
echo "| ${metric_name} | ${metric_value} | ${status_icon} |" >> $GITHUB_STEP_SUMMARY
|
||||
done
|
||||
else
|
||||
echo "No metrics collected (test may have failed before producing results)." >> $GITHUB_STEP_SUMMARY
|
||||
if [ "${{ inputs.is_nightly_pipeline_job }}" != "true" ]; then
|
||||
metric_count=$(grep -c '\[METRIC\]' /tmp/test_output.log 2>/dev/null || echo 0)
|
||||
if [ "${metric_count}" -gt 0 ]; then
|
||||
echo "| Metric | Value | Pass |" >> $GITHUB_STEP_SUMMARY
|
||||
echo "|--------|-------|------|" >> $GITHUB_STEP_SUMMARY
|
||||
grep '\[METRIC\]' /tmp/test_output.log | while IFS= read -r line; do
|
||||
metric_name=$(echo "$line" | sed -E 's/.*\[METRIC\] ([^=]+)=.*/\1/')
|
||||
metric_value=$(echo "$line" | sed -E 's/.*\[METRIC\] [^=]+=([^ ]+).*/\1/')
|
||||
echo "| ${metric_name} | ${metric_value} | ${status_icon} |" >> $GITHUB_STEP_SUMMARY
|
||||
done
|
||||
else
|
||||
echo "No metrics collected (test may have failed before producing results)." >> $GITHUB_STEP_SUMMARY
|
||||
fi
|
||||
echo "" >> $GITHUB_STEP_SUMMARY
|
||||
fi
|
||||
echo "" >> $GITHUB_STEP_SUMMARY
|
||||
|
||||
exit ${test_exit_code}
|
||||
|
||||
@@ -225,14 +225,6 @@ jobs:
|
||||
echo "Run command: ${CMD}"
|
||||
eval "${CMD}"
|
||||
|
||||
- name: Upload metrics
|
||||
if: always()
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: metrics-${{ inputs.test_config_name }}
|
||||
path: /tmp/metrics.json
|
||||
retention-days: 7
|
||||
|
||||
- name: Post process
|
||||
if: always()
|
||||
run: |
|
||||
|
||||
@@ -453,13 +453,6 @@ jobs:
|
||||
- nightly-poc-multi-node-mix-tests
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Download all metrics
|
||||
uses: actions/download-artifact@v4
|
||||
with:
|
||||
pattern: metrics-*
|
||||
path: /tmp/metrics
|
||||
merge-multiple: false
|
||||
|
||||
- name: Generate results table
|
||||
run: |
|
||||
single_result_a2="${{ needs.nightly-1-npu-a2.result }}"
|
||||
@@ -522,42 +515,6 @@ jobs:
|
||||
echo "| multi-node-mix-poc | $(group_icon ${mix_result}) ${mix_result} |" >> $GITHUB_STEP_SUMMARY
|
||||
echo "" >> $GITHUB_STEP_SUMMARY
|
||||
|
||||
echo "## Per-Test Metrics" >> $GITHUB_STEP_SUMMARY
|
||||
echo "" >> $GITHUB_STEP_SUMMARY
|
||||
|
||||
has_metrics=false
|
||||
# Each suite job uploads its METRICS_DATA_FILE directory, which
|
||||
# contains per-case subdirectories with their own metrics.json.
|
||||
# Recursively find every metrics.json and render one table row per
|
||||
# test case.
|
||||
for file in $(find /tmp/metrics -name metrics.json 2>/dev/null); do
|
||||
if [ -f "$file" ]; then
|
||||
has_metrics=true
|
||||
echo "import json" > /tmp/parse_metrics.py
|
||||
echo "with open('$file') as f:" >> /tmp/parse_metrics.py
|
||||
echo " d = json.load(f)" >> /tmp/parse_metrics.py
|
||||
echo "tc = d.get('test_case', '-')" >> /tmp/parse_metrics.py
|
||||
echo "tp = d.get('test_type', '-')" >> /tmp/parse_metrics.py
|
||||
echo "st = d.get('status', 'unknown')" >> /tmp/parse_metrics.py
|
||||
echo "icon = '✅' if st == 'pass' else '❌'" >> /tmp/parse_metrics.py
|
||||
echo "metrics = d.get('metrics', {})" >> /tmp/parse_metrics.py
|
||||
echo "baselines = d.get('baselines', {})" >> /tmp/parse_metrics.py
|
||||
echo "mstr = ', '.join(f'{k}={v}' for k,v in metrics.items()) if metrics else '-'" >> /tmp/parse_metrics.py
|
||||
echo "bstr = ', '.join(f'{k}={v}' for k,v in baselines.items()) if baselines else '-'" >> /tmp/parse_metrics.py
|
||||
echo "print(f'| {tc} | {tp} | {icon} {st} | {mstr} | {bstr} |')" >> /tmp/parse_metrics.py
|
||||
python3 /tmp/parse_metrics.py
|
||||
fi
|
||||
done > /tmp/metrics_table.txt
|
||||
|
||||
if [ "${has_metrics}" = "true" ]; then
|
||||
echo "| Test Case | Type | Status | Metrics | Baseline |" >> $GITHUB_STEP_SUMMARY
|
||||
echo "|-----------|------|--------|---------|----------|" >> $GITHUB_STEP_SUMMARY
|
||||
cat /tmp/metrics_table.txt >> $GITHUB_STEP_SUMMARY
|
||||
else
|
||||
echo "No per-test metrics available (artifacts not found)." >> $GITHUB_STEP_SUMMARY
|
||||
fi
|
||||
echo "" >> $GITHUB_STEP_SUMMARY
|
||||
|
||||
FAIL=0
|
||||
for r in "${single_result_a2}" "${single_result}" "${multi_result}" "${mix_result}"; do
|
||||
if [ "${r}" != "success" ] && [ "${r}" != "skipped" ]; then FAIL=1; fi
|
||||
|
||||
@@ -501,6 +501,13 @@ def monitor_pod_logs(
|
||||
|
||||
|
||||
def generate_metrics_json(metrics_data_file, test_case, status):
|
||||
"""Collect [METRIC] lines from the log and write metrics.json.
|
||||
|
||||
This writes ${metrics_data_file}/metrics.json only, which feeds the
|
||||
persistence-directory output (e.g. /root/.cache/tests/output/...).
|
||||
It is NOT consumed by CI artifacts (the former /tmp/metrics.json +
|
||||
'Upload metrics' path was removed).
|
||||
"""
|
||||
log_file = os.path.join(metrics_data_file, "test_output.log")
|
||||
|
||||
metrics = {}
|
||||
@@ -553,10 +560,6 @@ def generate_metrics_json(metrics_data_file, test_case, status):
|
||||
json.dump(output, f, indent=2)
|
||||
logger.info(f"Metrics JSON written to {output_path}")
|
||||
|
||||
with open("/tmp/metrics.json", "w") as f:
|
||||
json.dump(output, f, indent=2)
|
||||
logger.info("Metrics JSON written to /tmp/metrics.json")
|
||||
|
||||
|
||||
def run_npu_e2e_test_case(
|
||||
docker_image_url: str,
|
||||
|
||||
Reference in New Issue
Block a user