minke.yu/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 12 Packages Projects Releases Wiki Activity

fix: only showing recent runners from ci failure analysis (#21015)

Browse Source
This commit is contained in:
Douglas Yang
2026-03-31 20:18:17 -07:00
committed by GitHub
parent ea6b22fb85
commit 1b45d81e91
1 changed files with 50 additions and 21 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/ci_monitor/ci_failures_analysis.py
+50 -21
View File
@@ -586,6 +586,10 @@ class SGLangFailuresAnalyzer:
runner_instance_first_failure: Dict[str, Optional[Dict]] = {} runner_instance_first_failure: Dict[str, Optional[Dict]] = {}
runner_instance_last_failure: Dict[str, Optional[Dict]] = {} runner_instance_last_failure: Dict[str, Optional[Dict]] = {}
runner_instance_recovery: Dict[str, Optional[Dict]] = {} runner_instance_recovery: Dict[str, Optional[Dict]] = {}
runner_instance_all_failures_in_streak: Dict[str, List[Dict]] = defaultdict(
list
)
runner_instance_all_failures: Dict[str, List[Dict]] = defaultdict(list)
total_runs_processed = len(sorted_runs) total_runs_processed = len(sorted_runs)
for i, run in enumerate(sorted_runs, 1): for i, run in enumerate(sorted_runs, 1):
@@ -802,6 +806,12 @@ class SGLangFailuresAnalyzer:
runner_instance_first_failed_job[runner_instance_key] runner_instance_first_failed_job[runner_instance_key]
) )
runner_instance_last_failure[runner_instance_key] = failure_info runner_instance_last_failure[runner_instance_key] = failure_info
runner_instance_all_failures_in_streak[runner_instance_key].append(
failure_info
)
runner_instance_all_failures[runner_instance_key].append(
failure_info
)
if ( if (
runner_instance_current_streak[runner_instance_key] runner_instance_current_streak[runner_instance_key]
@@ -823,6 +833,7 @@ class SGLangFailuresAnalyzer:
runner_instance_current_streak[runner_instance_key] = 0 runner_instance_current_streak[runner_instance_key] = 0
runner_instance_first_failure[runner_instance_key] = None runner_instance_first_failure[runner_instance_key] = None
runner_instance_all_failures_in_streak[runner_instance_key] = []
runner_instance_last_failure[runner_instance_key] = None runner_instance_last_failure[runner_instance_key] = None
time.sleep(0.05) time.sleep(0.05)
@@ -903,6 +914,9 @@ class SGLangFailuresAnalyzer:
"avg_queue_time_seconds": avg_queue_time, "avg_queue_time_seconds": avg_queue_time,
"p90_queue_time_seconds": p90_queue_time, "p90_queue_time_seconds": p90_queue_time,
"queue_time_samples": len(queue_times), "queue_time_samples": len(queue_times),
"all_failures": list(
runner_instance_all_failures.get(instance_key, [])
),
} }
# Build runner streak data # Build runner streak data
@@ -951,6 +965,9 @@ class SGLangFailuresAnalyzer:
"last_failure_in_streak": runner_instance_last_failure.get( "last_failure_in_streak": runner_instance_last_failure.get(
instance_key instance_key
), ),
"all_failures_in_streak": list(
runner_instance_all_failures_in_streak.get(instance_key, [])
),
"recovery_info": runner_instance_recovery.get(instance_key), "recovery_info": runner_instance_recovery.get(instance_key),
} }
@@ -2058,8 +2075,10 @@ class SGLangFailuresAnalyzer:
"total_jobs": stats["total_jobs"], "total_jobs": stats["total_jobs"],
"unique_jobs": len(stats.get("jobs_failed", {})), "unique_jobs": len(stats.get("jobs_failed", {})),
"avg_queue": stats.get("avg_queue_time_seconds", 0), "avg_queue": stats.get("avg_queue_time_seconds", 0),
"first_failure": streak_data.get("first_failure_in_streak"), "all_failures_in_streak": streak_data.get(
"last_failure": streak_data.get("last_failure_in_streak"), "all_failures_in_streak", []
),
"all_failures": stats.get("all_failures", []),
} }
) )
@@ -2096,10 +2115,10 @@ class SGLangFailuresAnalyzer:
) )
summary_lines.append("") summary_lines.append("")
summary_lines.append( summary_lines.append(
"| Machine Name | Current Streak | Max | Fail Rate | Avg Queue | Total Jobs | Unique Jobs | First Failure | Last Failure |" "| Machine Name | Current Streak | Max | Fail Rate | Avg Queue | Total Jobs | Failed Jobs | Unique Jobs | Jobs |"
) )
summary_lines.append( summary_lines.append(
"|--------------|----------------|-----|-----------|-----------|------------|-------------|---------------|--------------|" "|--------------|----------------|-----|-----------|-----------|------------|-------------|-------------|------|"
) )
for runner_data in runners_with_streak[:15]: for runner_data in runners_with_streak[:15]:
@@ -2115,17 +2134,14 @@ class SGLangFailuresAnalyzer:
else "N/A" else "N/A"
) )
first_failure = runner_data.get("first_failure") all_failures = runner_data.get("all_failures_in_streak", [])
first_str = ( failed_jobs_count = len(all_failures)
f"[Run #{first_failure['run_number']}]({first_failure.get('job_url', first_failure['url'])})" jobs_str = (
if first_failure " ".join(
else "N/A" f"[#{f.get('run_number', '?')}]({f.get('job_url', f['url'])})"
) for f in all_failures
)
last_failure = runner_data.get("last_failure") if all_failures
last_str = (
f"[Run #{last_failure['run_number']}]({last_failure.get('job_url', last_failure['url'])})"
if last_failure
else "N/A" else "N/A"
) )
@@ -2133,12 +2149,12 @@ class SGLangFailuresAnalyzer:
if runner_data["current_streak"] >= 3: if runner_data["current_streak"] >= 3:
summary_lines.append( summary_lines.append(
f"| <span style='color:red'>`{display_name}`</span> | <span style='color:red'>{runner_data['current_streak']}</span> | <span style='color:red'>{runner_data['max_streak']}</span> | " f"| <span style='color:red'>`{display_name}`</span> | <span style='color:red'>{runner_data['current_streak']}</span> | <span style='color:red'>{runner_data['max_streak']}</span> | "
f"<span style='color:red'>{runner_data['failure_rate']:.1f}%</span> | <span style='color:red'>{avg_queue_str}</span> | <span style='color:red'>{runner_data['total_jobs']}</span> | <span style='color:red'>{runner_data.get('unique_jobs', 0)}</span> | <span style='color:red'>{first_str}</span> | <span style='color:red'>{last_str}</span> |" f"<span style='color:red'>{runner_data['failure_rate']:.1f}%</span> | <span style='color:red'>{avg_queue_str}</span> | <span style='color:red'>{runner_data['total_jobs']}</span> | <span style='color:red'>{failed_jobs_count}</span> | <span style='color:red'>{runner_data.get('unique_jobs', 0)}</span> | <span style='color:red'>{jobs_str}</span> |"
) )
else: else:
summary_lines.append( summary_lines.append(
f"| `{display_name}` | {runner_data['current_streak']} | {runner_data['max_streak']} | " f"| `{display_name}` | {runner_data['current_streak']} | {runner_data['max_streak']} | "
f"{runner_data['failure_rate']:.1f}% | {avg_queue_str} | {runner_data['total_jobs']} | {runner_data.get('unique_jobs', 0)} | {first_str} | {last_str} |" f"{runner_data['failure_rate']:.1f}% | {avg_queue_str} | {runner_data['total_jobs']} | {failed_jobs_count} | {runner_data.get('unique_jobs', 0)} | {jobs_str} |"
) )
summary_lines.append("") summary_lines.append("")
@@ -2150,10 +2166,10 @@ class SGLangFailuresAnalyzer:
) )
summary_lines.append("") summary_lines.append("")
summary_lines.append( summary_lines.append(
"| Machine Name | Fail Rate | Avg Queue | Total Jobs | Unique Jobs |" "| Machine Name | Fail Rate | Avg Queue | Total Jobs | Failed Jobs | Unique Jobs | Jobs |"
) )
summary_lines.append( summary_lines.append(
"|--------------|-----------|-----------|------------|-------------|" "|--------------|-----------|-----------|------------|-------------|-------------|------|"
) )
for runner_data in runners_high_fail_rate[:15]: for runner_data in runners_high_fail_rate[:15]:
@@ -2169,10 +2185,21 @@ class SGLangFailuresAnalyzer:
else "N/A" else "N/A"
) )
all_failures = runner_data.get("all_failures", [])
failed_jobs_count = len(all_failures)
jobs_str = (
" ".join(
f"[#{f.get('run_number', '?')}]({f.get('job_url', f['url'])})"
for f in all_failures
)
if all_failures
else "N/A"
)
summary_lines.append( summary_lines.append(
f"| <span style='color:orange'>`{display_name}`</span> | <span style='color:orange'>{runner_data['failure_rate']:.1f}%</span> | " f"| <span style='color:orange'>`{display_name}`</span> | <span style='color:orange'>{runner_data['failure_rate']:.1f}%</span> | "
f"<span style='color:orange'>{avg_queue_str}</span> | <span style='color:orange'>{runner_data['total_jobs']}</span> | " f"<span style='color:orange'>{avg_queue_str}</span> | <span style='color:orange'>{runner_data['total_jobs']}</span> | "
f"<span style='color:orange'>{runner_data.get('unique_jobs', 0)}</span> |" f"<span style='color:orange'>{failed_jobs_count}</span> | <span style='color:orange'>{runner_data.get('unique_jobs', 0)}</span> | <span style='color:orange'>{jobs_str}</span> |"
) )
summary_lines.append("") summary_lines.append("")
@@ -2512,7 +2539,9 @@ def main():
) )
# Choosing nvidia pr test and nightly for runner health analysis # Choosing nvidia pr test and nightly for runner health analysis
runner_runs = pr_test_nvidia_general_runs + nightly_nvidia_general_runs # Use scheduled runs (already limited to 12 PR + 6 nightly) to avoid
# pulling months of history from the unfiltered general fetch.
runner_runs = pr_test_nvidia_scheduled_runs + nightly_nvidia_scheduled_runs
if not runner_runs and not pr_test_nvidia_scheduled_runs: if not runner_runs and not pr_test_nvidia_scheduled_runs:
print("No workflow runs found") print("No workflow runs found")