diff --git a/scripts/ci_monitor/ci_failures_analysis.py b/scripts/ci_monitor/ci_failures_analysis.py index d5a4f6242..c64a2b43b 100644 --- a/scripts/ci_monitor/ci_failures_analysis.py +++ b/scripts/ci_monitor/ci_failures_analysis.py @@ -586,6 +586,10 @@ class SGLangFailuresAnalyzer: runner_instance_first_failure: Dict[str, Optional[Dict]] = {} runner_instance_last_failure: Dict[str, Optional[Dict]] = {} runner_instance_recovery: Dict[str, Optional[Dict]] = {} + runner_instance_all_failures_in_streak: Dict[str, List[Dict]] = defaultdict( + list + ) + runner_instance_all_failures: Dict[str, List[Dict]] = defaultdict(list) total_runs_processed = len(sorted_runs) for i, run in enumerate(sorted_runs, 1): @@ -802,6 +806,12 @@ class SGLangFailuresAnalyzer: runner_instance_first_failed_job[runner_instance_key] ) runner_instance_last_failure[runner_instance_key] = failure_info + runner_instance_all_failures_in_streak[runner_instance_key].append( + failure_info + ) + runner_instance_all_failures[runner_instance_key].append( + failure_info + ) if ( runner_instance_current_streak[runner_instance_key] @@ -823,6 +833,7 @@ class SGLangFailuresAnalyzer: runner_instance_current_streak[runner_instance_key] = 0 runner_instance_first_failure[runner_instance_key] = None + runner_instance_all_failures_in_streak[runner_instance_key] = [] runner_instance_last_failure[runner_instance_key] = None time.sleep(0.05) @@ -903,6 +914,9 @@ class SGLangFailuresAnalyzer: "avg_queue_time_seconds": avg_queue_time, "p90_queue_time_seconds": p90_queue_time, "queue_time_samples": len(queue_times), + "all_failures": list( + runner_instance_all_failures.get(instance_key, []) + ), } # Build runner streak data @@ -951,6 +965,9 @@ class SGLangFailuresAnalyzer: "last_failure_in_streak": runner_instance_last_failure.get( instance_key ), + "all_failures_in_streak": list( + runner_instance_all_failures_in_streak.get(instance_key, []) + ), "recovery_info": runner_instance_recovery.get(instance_key), } @@ -2058,8 +2075,10 @@ class SGLangFailuresAnalyzer: "total_jobs": stats["total_jobs"], "unique_jobs": len(stats.get("jobs_failed", {})), "avg_queue": stats.get("avg_queue_time_seconds", 0), - "first_failure": streak_data.get("first_failure_in_streak"), - "last_failure": streak_data.get("last_failure_in_streak"), + "all_failures_in_streak": streak_data.get( + "all_failures_in_streak", [] + ), + "all_failures": stats.get("all_failures", []), } ) @@ -2096,10 +2115,10 @@ class SGLangFailuresAnalyzer: ) summary_lines.append("") summary_lines.append( - "| Machine Name | Current Streak | Max | Fail Rate | Avg Queue | Total Jobs | Unique Jobs | First Failure | Last Failure |" + "| Machine Name | Current Streak | Max | Fail Rate | Avg Queue | Total Jobs | Failed Jobs | Unique Jobs | Jobs |" ) summary_lines.append( - "|--------------|----------------|-----|-----------|-----------|------------|-------------|---------------|--------------|" + "|--------------|----------------|-----|-----------|-----------|------------|-------------|-------------|------|" ) for runner_data in runners_with_streak[:15]: @@ -2115,17 +2134,14 @@ class SGLangFailuresAnalyzer: else "N/A" ) - first_failure = runner_data.get("first_failure") - first_str = ( - f"[Run #{first_failure['run_number']}]({first_failure.get('job_url', first_failure['url'])})" - if first_failure - else "N/A" - ) - - last_failure = runner_data.get("last_failure") - last_str = ( - f"[Run #{last_failure['run_number']}]({last_failure.get('job_url', last_failure['url'])})" - if last_failure + all_failures = runner_data.get("all_failures_in_streak", []) + failed_jobs_count = len(all_failures) + jobs_str = ( + " ".join( + f"[#{f.get('run_number', '?')}]({f.get('job_url', f['url'])})" + for f in all_failures + ) + if all_failures else "N/A" ) @@ -2133,12 +2149,12 @@ class SGLangFailuresAnalyzer: if runner_data["current_streak"] >= 3: summary_lines.append( f"| `{display_name}` | {runner_data['current_streak']} | {runner_data['max_streak']} | " - f"{runner_data['failure_rate']:.1f}% | {avg_queue_str} | {runner_data['total_jobs']} | {runner_data.get('unique_jobs', 0)} | {first_str} | {last_str} |" + f"{runner_data['failure_rate']:.1f}% | {avg_queue_str} | {runner_data['total_jobs']} | {failed_jobs_count} | {runner_data.get('unique_jobs', 0)} | {jobs_str} |" ) else: summary_lines.append( f"| `{display_name}` | {runner_data['current_streak']} | {runner_data['max_streak']} | " - f"{runner_data['failure_rate']:.1f}% | {avg_queue_str} | {runner_data['total_jobs']} | {runner_data.get('unique_jobs', 0)} | {first_str} | {last_str} |" + f"{runner_data['failure_rate']:.1f}% | {avg_queue_str} | {runner_data['total_jobs']} | {failed_jobs_count} | {runner_data.get('unique_jobs', 0)} | {jobs_str} |" ) summary_lines.append("") @@ -2150,10 +2166,10 @@ class SGLangFailuresAnalyzer: ) summary_lines.append("") summary_lines.append( - "| Machine Name | Fail Rate | Avg Queue | Total Jobs | Unique Jobs |" + "| Machine Name | Fail Rate | Avg Queue | Total Jobs | Failed Jobs | Unique Jobs | Jobs |" ) summary_lines.append( - "|--------------|-----------|-----------|------------|-------------|" + "|--------------|-----------|-----------|------------|-------------|-------------|------|" ) for runner_data in runners_high_fail_rate[:15]: @@ -2169,10 +2185,21 @@ class SGLangFailuresAnalyzer: else "N/A" ) + all_failures = runner_data.get("all_failures", []) + failed_jobs_count = len(all_failures) + jobs_str = ( + " ".join( + f"[#{f.get('run_number', '?')}]({f.get('job_url', f['url'])})" + for f in all_failures + ) + if all_failures + else "N/A" + ) + summary_lines.append( f"| `{display_name}` | {runner_data['failure_rate']:.1f}% | " f"{avg_queue_str} | {runner_data['total_jobs']} | " - f"{runner_data.get('unique_jobs', 0)} |" + f"{failed_jobs_count} | {runner_data.get('unique_jobs', 0)} | {jobs_str} |" ) summary_lines.append("") @@ -2512,7 +2539,9 @@ def main(): ) # Choosing nvidia pr test and nightly for runner health analysis - runner_runs = pr_test_nvidia_general_runs + nightly_nvidia_general_runs + # Use scheduled runs (already limited to 12 PR + 6 nightly) to avoid + # pulling months of history from the unfiltered general fetch. + runner_runs = pr_test_nvidia_scheduled_runs + nightly_nvidia_scheduled_runs if not runner_runs and not pr_test_nvidia_scheduled_runs: print("No workflow runs found")