[CI] Stop the config ratchets re-parsing the package on every scan (#36240)

Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>

test/registered/unit/server_args/test_model_config_reads_resolved_input.py

@@ -13,6 +13,7 @@ one field that is deliberately read before resolution touches it.
 """
 
 import ast
+import functools
 import pathlib
 import unittest
 
@@ -51,6 +52,24 @@ _STALE_FROM_THE_REGISTRIES = frozenset(
 )
 
 
+@functools.lru_cache(maxsize=None)
+def _parsed(path):
+    return ast.parse(path.read_text(encoding="utf-8-sig"))
+
+
+@functools.lru_cache(maxsize=None)
+def _declared_resolution_fields(path):
+    fields = set()
+    for node in ast.walk(_parsed(path)):
+        if (
+            isinstance(node, ast.Call)
+            and isinstance(node.func, ast.Name)
+            and node.func.id == "declare_resolution"
+        ):
+            fields |= {kw.arg for kw in node.keywords if kw.arg}
+    return frozenset(fields)
+
+
 def _registry_declared_fields():
     """What the live registries and passes declare.
 
@@ -79,7 +98,7 @@ def _registry_collection_is_after_the_build():
     collection above this handler's own `get_model_config()` call does not move
     it above the configuration another handler already cached.
     """
-    tree = ast.parse((_SRT / "server_args.py").read_text(encoding="utf-8-sig"))
+    tree = _parsed(_SRT / "server_args.py")
     handler = next(
         node
         for node in ast.walk(tree)
@@ -129,7 +148,7 @@ def _server_args_names(tree, path):
 def _constructor_reads():
     """Fields `ModelConfig.from_server_args` takes off the record."""
     path = _SRT / "configs/model_config.py"
-    tree = ast.parse(path.read_text(encoding="utf-8-sig"))
+    tree = _parsed(path)
     constructor = next(
         node
         for node in ast.walk(tree)
@@ -177,7 +196,7 @@ def _late_resolution_fields():
         path = _SRT / name
         if not path.exists():
             continue
-        tree = ast.parse(path.read_text(encoding="utf-8-sig"))
+        tree = _parsed(path)
         for node in ast.walk(tree):
             if not isinstance(node, ast.Call):
                 continue
@@ -205,7 +224,7 @@ def _hook_declarations(dispatch, source_module):
     `test_every_opaque_callback_is_still_late`.
     """
     imported = {}
-    for node in ast.walk(ast.parse(source_module.read_text(encoding="utf-8-sig"))):
+    for node in ast.walk(_parsed(source_module)):
         if isinstance(node, ast.ImportFrom) and node.module:
             for alias in node.names:
                 imported[alias.asname or alias.name] = node.module
@@ -225,22 +244,14 @@ def _hook_declarations(dispatch, source_module):
         path = _SRT / (module[len("sglang.srt.") :].replace(".", "/") + ".py")
         if not path.exists():
             continue
-        for inner in ast.walk(ast.parse(path.read_text(encoding="utf-8-sig"))):
-            if (
-                isinstance(inner, ast.Call)
-                and isinstance(inner.func, ast.Name)
-                and inner.func.id == "declare_resolution"
-            ):
-                for keyword in inner.keywords:
-                    if keyword.arg:
-                        out[keyword.arg] = max(out.get(keyword.arg, 0), node.lineno)
+        for field in _declared_resolution_fields(path):
+            out[field] = max(out.get(field, 0), node.lineno)
     return out
 
 
 def _pipeline():
     """(ordered steps, {step: methods it reaches}) for the resolution dispatch."""
-    source = (_SRT / "server_args.py").read_text(encoding="utf-8-sig")
-    tree = ast.parse(source)
+    tree = _parsed(_SRT / "server_args.py")
     record = next(
         node
         for node in tree.body
@@ -305,7 +316,7 @@ def _opaque_callback_positions(dispatch, source_module):
     it.
     """
     imported = {}
-    for node in ast.walk(ast.parse(source_module.read_text(encoding="utf-8-sig"))):
+    for node in ast.walk(_parsed(source_module)):
         if isinstance(node, ast.ImportFrom) and node.module:
             for alias in node.names:
                 imported[alias.asname or alias.name] = node.module
@@ -349,7 +360,7 @@ def _opaque_callback_positions(dispatch, source_module):
         path = _SRT / (module[len("sglang.srt.") :].replace(".", "/") + ".py")
         if not path.exists():
             continue
-        for spelling in callbacks_in(ast.parse(path.read_text(encoding="utf-8-sig"))):
+        for spelling in callbacks_in(_parsed(path)):
             positions[spelling] = min(positions.get(spelling, 10**9), node.lineno)
     return positions
 
@@ -392,7 +403,7 @@ def _declaration_positions():
 
     source_module = _SRT / "server_args.py"
     imported = {}
-    for node in ast.walk(ast.parse(source_module.read_text(encoding="utf-8-sig"))):
+    for node in ast.walk(_parsed(source_module)):
         if isinstance(node, ast.ImportFrom) and node.module:
             for alias in node.names:
                 imported[alias.asname or alias.name] = node.module
@@ -405,15 +416,7 @@ def _declaration_positions():
         path = _SRT / (module[len("sglang.srt.") :].replace(".", "/") + ".py")
         if not path.exists():
             return frozenset()
-        fields = set()
-        for node in ast.walk(ast.parse(path.read_text(encoding="utf-8-sig"))):
-            if (
-                isinstance(node, ast.Call)
-                and isinstance(node.func, ast.Name)
-                and node.func.id == "declare_resolution"
-            ):
-                fields |= {kw.arg for kw in node.keywords if kw.arg}
-        return frozenset(fields)
+        return _declared_resolution_fields(path)
 
     declared_at = {}
     for index, step in enumerate(steps):

test/registered/unit/server_args/test_resolution_reads_no_bag.py

@@ -28,6 +28,7 @@ It is a ratchet, not a proof.
 """
 
 import ast
+import functools
 import inspect
 import pathlib
 import unittest
@@ -114,6 +115,7 @@ def _registry_functions():
     return functions
 
 
+@functools.lru_cache(maxsize=None)
 def _registered_entries():
     """Entries the import map cannot reach: passes and override providers.
 
@@ -138,13 +140,17 @@ def _registered_entries():
     # from the source. The entry carries the *defining* file: `_reaches_a_bag`
     # walks functions in the entry's file, so a call-site key walks nothing.
     by_value = set()
-    trees = {}
-    for path in sorted(_SRT.rglob("*.py")):
+    sources = {
+        path: path.read_text(encoding="utf-8-sig")
+        for path in sorted(_SRT.rglob("*.py"))
+    }
+    for path, source in sources.items():
+        if "run_post_process_pass" not in source:
+            continue
         try:
-            trees[path] = ast.parse(path.read_text(encoding="utf-8-sig"))
+            tree = ast.parse(source)
         except SyntaxError:
             continue
-    for path, tree in trees.items():
         for node in ast.walk(tree):
             if (
                 isinstance(node, ast.Call)
@@ -154,6 +160,14 @@ def _registered_entries():
                 and isinstance(node.args[1], ast.Name)
             ):
                 by_value.add(node.args[1].id)
+    trees = {}
+    for path, source in sources.items():
+        if not any(name in source for name in by_value):
+            continue
+        try:
+            trees[path] = ast.parse(source)
+        except SyntaxError:
+            continue
     for name in sorted(by_value):
         defined_in = [
             path
@@ -173,14 +187,19 @@ def _registered_entries():
     return entries
 
 
-def _reaches_a_bag(path, entry):
-    """Does `entry` in `path` reach a bag accessor, following calls in-module?"""
+@functools.lru_cache(maxsize=None)
+def _functions_in(path):
     tree = ast.parse(path.read_text(encoding="utf-8-sig"))
-    functions = {
+    return {
         node.name: node
         for node in ast.walk(tree)
         if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef))
     }
+
+
+def _reaches_a_bag(path, entry):
+    """Does `entry` in `path` reach a bag accessor, following calls in-module?"""
+    functions = _functions_in(path)
     seen = set()
 
     def walk(name):

test/registered/unit/test_chain_read_ratchet.py

@@ -47,8 +47,11 @@ def _declared_by_keyword():
     """Fields named as a keyword at a declaration site."""
     written = set()
     for path in sorted(_SRT.rglob("*.py")):
+        source = path.read_text(encoding="utf-8-sig")
+        if not any(declarer in source for declarer in _DECLARERS):
+            continue
         try:
-            tree = ast.parse(path.read_text(encoding="utf-8-sig"))
+            tree = ast.parse(source)
         except SyntaxError:
             raise AssertionError(f"unparsable module in the census: {path}")
         for node in ast.walk(tree):
@@ -293,8 +296,11 @@ def _subtrees_with_their_own_record():
         rel = path.relative_to(_PACKAGE).as_posix()
         if rel.startswith("srt/") or "/" not in rel:
             continue
+        source = path.read_text(encoding="utf-8-sig")
+        if "ServerArgs" not in source:
+            continue
         try:
-            tree = ast.parse(path.read_text(encoding="utf-8-sig"))
+            tree = ast.parse(source)
         except SyntaxError:
             continue
         if any(
@@ -333,8 +339,11 @@ def _chain_reads(written):
             under_srt = rel[len("srt/") :]
             if path.name in _OWNERS or under_srt.startswith(_OWNERS[-1]):
                 continue
+        source = path.read_text(encoding="utf-8-sig")
+        if "server_args" not in source:
+            continue
         try:
-            tree = ast.parse(path.read_text(encoding="utf-8-sig"))
+            tree = ast.parse(source)
         except SyntaxError:
             raise AssertionError(f"unparsable module in the census: {rel}")
         if (

test/registered/unit/test_global_config_read_ratchet.py

@@ -498,8 +498,11 @@ def _field_reads():
         rel = path.relative_to(_PACKAGE_ROOT).as_posix()
         if rel.startswith(_SLOT_OWNERS):
             continue
+        source = path.read_text()
+        if "get_server_args" not in source:
+            continue
         try:
-            tree = ast.parse(path.read_text())
+            tree = ast.parse(source)
         except SyntaxError:
             continue
         module_direct, module_alias = _collect(rel, tree)
@@ -551,8 +554,11 @@ class TestConfiguredSizeCallSites(CustomTestCase):
             rel = path.relative_to(_PACKAGE_ROOT).as_posix()
             if rel.startswith(_SLOT_OWNERS):
                 continue
+            source = path.read_text()
+            if "configured_" not in source:
+                continue
             try:
-                tree = ast.parse(path.read_text())
+                tree = ast.parse(source)
             except SyntaxError:
                 continue
             for node in ast.walk(tree):
@@ -588,8 +594,11 @@ class TestNoRenamedAccessorImports(CustomTestCase):
         offenders = []
         for path in sorted(_PACKAGE_ROOT.rglob("*.py")):
             rel = path.relative_to(_PACKAGE_ROOT).as_posix()
+            source = path.read_text()
+            if "get_server_args" not in source and "configured_" not in source:
+                continue
             try:
-                tree = ast.parse(path.read_text())
+                tree = ast.parse(source)
             except SyntaxError:
                 continue
             for node in ast.walk(tree):

test/registered/unit/test_launch_path_reads_configured_sizes.py

@@ -8,6 +8,7 @@ reaches, because nothing short of booting a server runs the launcher.
 """
 
 import ast
+import functools
 import pathlib
 import unittest
 
@@ -82,6 +83,7 @@ def _multiprocessing_names(tree):
     return modules, constructors
 
 
+@functools.lru_cache(maxsize=None)
 def _configured_accessors() -> frozenset:
     """The `configured_*_size()` names `runtime_context` exports.
 
@@ -217,11 +219,14 @@ def _launch_paths():
     nothing, which no derivation can reach.
     """
     seen = {}
+    sizes = frozenset(_LIVE_SHADOWED) | _configured_accessors()
     for path in sorted(_PACKAGE_ROOT.rglob("*.py")):
         source = path.read_text()
         # Every spawn shape below names Process, ProcessPoolExecutor or Popen.
         if not any(name in source for name in ("Process", "Popen", "spawn")):
             continue
+        if not any(name in source for name in sizes):
+            continue
         try:
             tree = ast.parse(source)
         except SyntaxError:

test/registered/unit/test_publish_precedes_bag_reads.py

@@ -397,6 +397,9 @@ def _publishing_functions():
         rel = path.relative_to(_PACKAGE_ROOT).as_posix()
         if rel in _PUBLISH_HOMES:
             continue
+        source = path.read_text(encoding="utf-8-sig")
+        if not any(name in source for name in _PUBLISH_NAMES):
+            continue
         mod = _module(rel)
         if mod is None or not mod.publishers:
             continue

test/registered/unit/test_server_args_namespaces.py

@@ -81,8 +81,11 @@ class TestServerArgsNamespaces(CustomTestCase):
         for path in sorted(srt.rglob("*.py")):
             if path.name == "runtime_context.py":
                 continue
+            source = path.read_text(encoding="utf-8-sig")
+            if "runtime_context" not in source:
+                continue
             try:
-                tree = ast.parse(path.read_text(encoding="utf-8-sig"))
+                tree = ast.parse(source)
             except SyntaxError:
                 self.fail(f"unparsable module in the census: {path}")
             bindings = collections.defaultdict(set)
@@ -155,8 +158,11 @@ class TestServerArgsNamespaces(CustomTestCase):
         sites = 0
         disagreements = []
         for path in sorted(srt.rglob("*.py")):
+            source = path.read_text(encoding="utf-8-sig")
+            if not any(name in source for name in accessors):
+                continue
             try:
-                tree = ast.parse(path.read_text(encoding="utf-8-sig"))
+                tree = ast.parse(source)
             except SyntaxError:
                 self.fail(f"unparsable module in the census: {path}")
             for node in ast.walk(tree):

test/registered/unit/test_supplied_instance_exposure_ratchet.py

@@ -682,8 +682,11 @@ class TestSuppliedInstanceExposure(CustomTestCase):
         root = _PACKAGE_ROOT
         for path in sorted(root.rglob("*.py")):
             rel = path.relative_to(root).as_posix()
+            source = path.read_text(encoding="utf-8-sig")
+            if "_late_resolution" not in source:
+                continue
             try:
-                tree = ast.parse(path.read_text(encoding="utf-8-sig"))
+                tree = ast.parse(source)
             except SyntaxError:
                 self.fail(f"unparsable module in the census: {rel}")
             for node in ast.walk(tree):
@@ -734,6 +737,8 @@ class TestSuppliedInstanceExposure(CustomTestCase):
                             written |= _expanded_override_keys(rel, tree, node, kw)
         return written
 
+    _READS_CACHE = None
+
     def _supplied_instance_reads(self) -> set:
         """Three spellings of the same read: ``server_args.field`` off the
         parameter, ``getattr(server_args, "field", default)`` with a literal
@@ -746,13 +751,18 @@ class TestSuppliedInstanceExposure(CustomTestCase):
         census still does not count -- but those reads are gone for every
         resolution-written field and ``test_chain_read_ratchet.py`` holds them
         at zero, so the gap is no longer where the risk is."""
+        if TestSuppliedInstanceExposure._READS_CACHE is not None:
+            return TestSuppliedInstanceExposure._READS_CACHE
         pairs = set()
         for path in sorted(_PACKAGE_ROOT.rglob("*.py")):
             rel = path.relative_to(_PACKAGE_ROOT).as_posix()
             if rel.startswith(_OWNERS):
                 continue
+            source = path.read_text(encoding="utf-8-sig")
+            if "server_args" not in source:
+                continue
             try:
-                tree = ast.parse(path.read_text(encoding="utf-8-sig"))
+                tree = ast.parse(source)
             except SyntaxError:
                 # A silently dropped module shrinks `found` and reads as
                 # intentional surface shrinkage under the bidirectional pin.
@@ -818,6 +828,7 @@ class TestSuppliedInstanceExposure(CustomTestCase):
                         and node.value.value.id == "self"
                     ):
                         pairs.add((rel, node.attr))
+        TestSuppliedInstanceExposure._READS_CACHE = pairs
         return pairs
 
     @staticmethod
@@ -826,8 +837,13 @@ class TestSuppliedInstanceExposure(CustomTestCase):
         written = set()
         for path in sorted(_PACKAGE_ROOT.rglob("*.py")):
             rel = path.relative_to(_PACKAGE_ROOT).as_posix()
+            source = path.read_text(encoding="utf-8-sig")
+            if "record_config_updates" not in source and not (
+                "get_context" in source and "override" in source
+            ):
+                continue
             try:
-                tree = ast.parse(path.read_text(encoding="utf-8-sig"))
+                tree = ast.parse(source)
             except SyntaxError:
                 raise AssertionError(f"unparsable module in the census: {rel}")
             for node in ast.walk(tree):