diff --git a/scripts/ci_monitor/ci_failures_analysis.py b/scripts/ci_monitor/ci_failures_analysis.py
index d5a4f6242..c64a2b43b 100644
--- a/scripts/ci_monitor/ci_failures_analysis.py
+++ b/scripts/ci_monitor/ci_failures_analysis.py
@@ -586,6 +586,10 @@ class SGLangFailuresAnalyzer:
runner_instance_first_failure: Dict[str, Optional[Dict]] = {}
runner_instance_last_failure: Dict[str, Optional[Dict]] = {}
runner_instance_recovery: Dict[str, Optional[Dict]] = {}
+ runner_instance_all_failures_in_streak: Dict[str, List[Dict]] = defaultdict(
+ list
+ )
+ runner_instance_all_failures: Dict[str, List[Dict]] = defaultdict(list)
total_runs_processed = len(sorted_runs)
for i, run in enumerate(sorted_runs, 1):
@@ -802,6 +806,12 @@ class SGLangFailuresAnalyzer:
runner_instance_first_failed_job[runner_instance_key]
)
runner_instance_last_failure[runner_instance_key] = failure_info
+ runner_instance_all_failures_in_streak[runner_instance_key].append(
+ failure_info
+ )
+ runner_instance_all_failures[runner_instance_key].append(
+ failure_info
+ )
if (
runner_instance_current_streak[runner_instance_key]
@@ -823,6 +833,7 @@ class SGLangFailuresAnalyzer:
runner_instance_current_streak[runner_instance_key] = 0
runner_instance_first_failure[runner_instance_key] = None
+ runner_instance_all_failures_in_streak[runner_instance_key] = []
runner_instance_last_failure[runner_instance_key] = None
time.sleep(0.05)
@@ -903,6 +914,9 @@ class SGLangFailuresAnalyzer:
"avg_queue_time_seconds": avg_queue_time,
"p90_queue_time_seconds": p90_queue_time,
"queue_time_samples": len(queue_times),
+ "all_failures": list(
+ runner_instance_all_failures.get(instance_key, [])
+ ),
}
# Build runner streak data
@@ -951,6 +965,9 @@ class SGLangFailuresAnalyzer:
"last_failure_in_streak": runner_instance_last_failure.get(
instance_key
),
+ "all_failures_in_streak": list(
+ runner_instance_all_failures_in_streak.get(instance_key, [])
+ ),
"recovery_info": runner_instance_recovery.get(instance_key),
}
@@ -2058,8 +2075,10 @@ class SGLangFailuresAnalyzer:
"total_jobs": stats["total_jobs"],
"unique_jobs": len(stats.get("jobs_failed", {})),
"avg_queue": stats.get("avg_queue_time_seconds", 0),
- "first_failure": streak_data.get("first_failure_in_streak"),
- "last_failure": streak_data.get("last_failure_in_streak"),
+ "all_failures_in_streak": streak_data.get(
+ "all_failures_in_streak", []
+ ),
+ "all_failures": stats.get("all_failures", []),
}
)
@@ -2096,10 +2115,10 @@ class SGLangFailuresAnalyzer:
)
summary_lines.append("")
summary_lines.append(
- "| Machine Name | Current Streak | Max | Fail Rate | Avg Queue | Total Jobs | Unique Jobs | First Failure | Last Failure |"
+ "| Machine Name | Current Streak | Max | Fail Rate | Avg Queue | Total Jobs | Failed Jobs | Unique Jobs | Jobs |"
)
summary_lines.append(
- "|--------------|----------------|-----|-----------|-----------|------------|-------------|---------------|--------------|"
+ "|--------------|----------------|-----|-----------|-----------|------------|-------------|-------------|------|"
)
for runner_data in runners_with_streak[:15]:
@@ -2115,17 +2134,14 @@ class SGLangFailuresAnalyzer:
else "N/A"
)
- first_failure = runner_data.get("first_failure")
- first_str = (
- f"[Run #{first_failure['run_number']}]({first_failure.get('job_url', first_failure['url'])})"
- if first_failure
- else "N/A"
- )
-
- last_failure = runner_data.get("last_failure")
- last_str = (
- f"[Run #{last_failure['run_number']}]({last_failure.get('job_url', last_failure['url'])})"
- if last_failure
+ all_failures = runner_data.get("all_failures_in_streak", [])
+ failed_jobs_count = len(all_failures)
+ jobs_str = (
+ " ".join(
+ f"[#{f.get('run_number', '?')}]({f.get('job_url', f['url'])})"
+ for f in all_failures
+ )
+ if all_failures
else "N/A"
)
@@ -2133,12 +2149,12 @@ class SGLangFailuresAnalyzer:
if runner_data["current_streak"] >= 3:
summary_lines.append(
f"| `{display_name}` | {runner_data['current_streak']} | {runner_data['max_streak']} | "
- f"{runner_data['failure_rate']:.1f}% | {avg_queue_str} | {runner_data['total_jobs']} | {runner_data.get('unique_jobs', 0)} | {first_str} | {last_str} |"
+ f"{runner_data['failure_rate']:.1f}% | {avg_queue_str} | {runner_data['total_jobs']} | {failed_jobs_count} | {runner_data.get('unique_jobs', 0)} | {jobs_str} |"
)
else:
summary_lines.append(
f"| `{display_name}` | {runner_data['current_streak']} | {runner_data['max_streak']} | "
- f"{runner_data['failure_rate']:.1f}% | {avg_queue_str} | {runner_data['total_jobs']} | {runner_data.get('unique_jobs', 0)} | {first_str} | {last_str} |"
+ f"{runner_data['failure_rate']:.1f}% | {avg_queue_str} | {runner_data['total_jobs']} | {failed_jobs_count} | {runner_data.get('unique_jobs', 0)} | {jobs_str} |"
)
summary_lines.append("")
@@ -2150,10 +2166,10 @@ class SGLangFailuresAnalyzer:
)
summary_lines.append("")
summary_lines.append(
- "| Machine Name | Fail Rate | Avg Queue | Total Jobs | Unique Jobs |"
+ "| Machine Name | Fail Rate | Avg Queue | Total Jobs | Failed Jobs | Unique Jobs | Jobs |"
)
summary_lines.append(
- "|--------------|-----------|-----------|------------|-------------|"
+ "|--------------|-----------|-----------|------------|-------------|-------------|------|"
)
for runner_data in runners_high_fail_rate[:15]:
@@ -2169,10 +2185,21 @@ class SGLangFailuresAnalyzer:
else "N/A"
)
+ all_failures = runner_data.get("all_failures", [])
+ failed_jobs_count = len(all_failures)
+ jobs_str = (
+ " ".join(
+ f"[#{f.get('run_number', '?')}]({f.get('job_url', f['url'])})"
+ for f in all_failures
+ )
+ if all_failures
+ else "N/A"
+ )
+
summary_lines.append(
f"| `{display_name}` | {runner_data['failure_rate']:.1f}% | "
f"{avg_queue_str} | {runner_data['total_jobs']} | "
- f"{runner_data.get('unique_jobs', 0)} |"
+ f"{failed_jobs_count} | {runner_data.get('unique_jobs', 0)} | {jobs_str} |"
)
summary_lines.append("")
@@ -2512,7 +2539,9 @@ def main():
)
# Choosing nvidia pr test and nightly for runner health analysis
- runner_runs = pr_test_nvidia_general_runs + nightly_nvidia_general_runs
+ # Use scheduled runs (already limited to 12 PR + 6 nightly) to avoid
+ # pulling months of history from the unfiltered general fetch.
+ runner_runs = pr_test_nvidia_scheduled_runs + nightly_nvidia_scheduled_runs
if not runner_runs and not pr_test_nvidia_scheduled_runs:
print("No workflow runs found")