fix(ci): rate limit and permission errors in trace publishing (#17238)

scripts/ci/publish_traces.py

@@ -14,8 +14,39 @@ from urllib.request import Request, urlopen
 
 
 def is_rate_limit_error(e):
-    """Check if an exception is a GitHub permission/quota error that should not be retried"""
+    """Check if an exception is a GitHub rate limit error (not permission error)"""
-    return isinstance(e, HTTPError) and e.code in [403, 429]
+    if not isinstance(e, HTTPError):
+        return False
+    if e.code == 429:
+        return True
+    if e.code == 403:
+        # 403 can be rate limit OR permission error - check the message
+        error_body = getattr(e, "error_body", "")
+        if isinstance(error_body, str):
+            # Rate limit errors contain specific phrases
+            rate_limit_phrases = [
+                "rate limit",
+                "abuse detection",
+                "secondary rate limit",
+            ]
+            return any(phrase in error_body.lower() for phrase in rate_limit_phrases)
+    return False
+
+
+def is_permission_error(e):
+    """Check if an exception is a GitHub permission error"""
+    if not isinstance(e, HTTPError) or e.code != 403:
+        return False
+    error_body = getattr(e, "error_body", "")
+    if isinstance(error_body, str):
+        permission_phrases = [
+            "resource not accessible",
+            "must have push access",
+            "permission",
+            "denied",
+        ]
+        return any(phrase in error_body.lower() for phrase in permission_phrases)
+    return False
 
 
 def make_github_request(url, token, method="GET", data=None):
@@ -349,6 +380,14 @@ def publish_traces(traces_dir, run_id, run_number):
                 "GitHub API rate limit exceeded during blob creation. Skipping trace upload."
             )
             return
+        # Check for permission errors - these should fail loudly
+        if is_permission_error(e):
+            print(
+                f"ERROR: Token does not have write permission to {repo_owner}/{repo_name}. "
+                "Please update the GH_PAT_FOR_NIGHTLY_CI_DATA secret with a token that has "
+                "'contents: write' permission for the repository."
+            )
+            sys.exit(1)
         print(f"Failed to create blobs: {e}")
         raise
 
@@ -410,6 +449,15 @@ def publish_traces(traces_dir, run_id, run_number):
                 warnings.warn("GitHub API rate limit exceeded. Skipping trace upload.")
                 return
 
+            # Check for permission errors - these should fail loudly
+            if is_permission_error(e):
+                print(
+                    f"ERROR: Token does not have write permission to {repo_owner}/{repo_name}. "
+                    "Please update the GH_PAT_FOR_NIGHTLY_CI_DATA secret with a token that has "
+                    "'contents: write' permission for the repository."
+                )
+                sys.exit(1)
+
             if is_retryable and attempt < max_retries - 1:
                 print(
                     f"Attempt {attempt + 1}/{max_retries} failed ({error_type}). Retrying in {retry_delay} seconds..."