minke.yu/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 12 Packages Projects Releases Wiki Activity

[CI] Derive registered-test kind from the registry call instead of the path (#40294)

Browse Source
This commit is contained in:
Liangsheng Yin
2026-09-20 02:01:22 -07:00
committed by GitHub
parent 671630abf1
commit 0024efa0de
24 changed files with 41 additions and 744 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/lint/check_registered_tests.py
+29 -127
View File
@@ -2,23 +2,8 @@
"""
Pre-commit hook: validate CI registry calls under test/registered/.
1. Every test file must contain a CI registry call (register_cuda_ci,
register_amd_ci, etc.).
2. A CUDA test must register its suite via the modern
`stage=`/`runner_config=` form. The legacy single-string `suite=` is reserved
for the stress family (and for AMD/CPU/NPU suites); any other CUDA `suite=`
resolves to a name no workflow invokes, so the test silently never runs.
Two shapes are rejected:
a. `{stage}-test-{runner_config}` -- the modern name stuffed back into the
legacy form. Reported with the exact stage/runner split to use.
b. an older `{stage}-{runner_config}` PR-test name (e.g. the pre-migration
`base-b-kernel-unit-1-gpu-large`) -- no longer matches any workflow
suite at all.
The modern form resolves to the identical suite (CIRegistry.effective_suite
is f"{stage}-test-{runner_config}") and is /rerun-test-able.
Reuses ut_parse_one_file() from ci_register.py (AST-based parsing)
to match the same logic used by run_suite.py's collect_tests().
Reuses ut_parse_one_file() from ci_register.py (AST-based parsing) to match
run_suite.py's collect_tests().
"""
import ast
@@ -26,25 +11,17 @@ import glob
import importlib.util
import os
import re
import subprocess
import sys
# Suite names of the form `{stage}-test-{runner_config}` are exactly what the
# modern stage=/runner_config= form produces, so a legacy suite= carrying this
# shape is always expressible (and should be expressed) the modern way.
# Exactly what stage=/runner_config= produces, so a legacy suite= of this shape
# is always expressible the modern way.
_MODERN_SHAPE = re.compile(r"^(.+)-test-(.+)$")
# The only CUDA suite family still allowed on the legacy single-string `suite=`
# form. Anything else needs stage=/runner_config=, or its effective_suite matches
# no suite any workflow invokes and the test silently never runs.
# The only CUDA family still allowed on legacy `suite=`; anything else resolves
# to a suite no workflow invokes and the test silently never runs.
_LEGACY_CUDA_PREFIXES = ("stress",)
_TEST_KINDS = {"unit", "e2e", "accuracy", "perf", "stress"}
_KERNEL_ROOT = "kernels"
# Flat vendor trees. Vendor-only coverage fits no kind above: no XPU/NPU suite
# carries the `-kernel-` infix the kernel tree needs, and these launch device work.
_VENDOR_DIRS = {"amd", "mlx", "musa", "npu", "xpu"}
_KERNEL_LAYOUT = "test/registered/kernels/{ops,benchmark}/<group>/"
def _defines_testcase(tree: ast.AST) -> bool:
@@ -78,43 +55,6 @@ def _main_runs_tests(tree: ast.Module) -> bool:
return False
def _git_lines(*args: str) -> list[str] | None:
result = subprocess.run(["git", *args], capture_output=True, text=True, check=False)
if result.returncode != 0:
return None
return [line for line in result.stdout.splitlines() if line]
def _changed_registered_files() -> set[str]:
"""Return added, copied, or renamed registered-test destinations."""
lines = _git_lines("diff", "--cached", "--name-status", "--diff-filter=ACR")
if not lines:
base_ref = os.environ.get("GITHUB_BASE_REF", "main")
for candidate in (f"origin/{base_ref}", base_ref):
if _git_lines("rev-parse", "--verify", candidate) is None:
continue
merge_base = _git_lines("merge-base", candidate, "HEAD")
if not merge_base:
continue
lines = _git_lines(
"diff",
"--name-status",
"--diff-filter=ACR",
merge_base[0],
"HEAD",
)
break
selected = set()
for line in lines or []:
fields = line.split("\t")
destination = fields[-1]
if destination.startswith("test/registered/") and destination.endswith(".py"):
selected.add(destination)
return selected
def _contains_call(tree: ast.AST, name: str) -> bool:
return any(
isinstance(node, ast.Call)
@@ -126,63 +66,28 @@ def _contains_call(tree: ast.AST, name: str) -> bool:
)
def taxonomy_errors(path: str, registries: list, tree: ast.AST) -> list[str]:
"""Validate the kind/subsystem contract for a newly admitted path."""
def taxonomy_errors(path: str, tree: ast.AST) -> list[str]:
parts = path.split("/")
relative_parts = parts[2:] if parts[:2] == ["test", "registered"] else []
if relative_parts and relative_parts[0] in _VENDOR_DIRS:
if parts[:2] != ["test", "registered"] or len(parts) < 3:
return []
if relative_parts and relative_parts[0] == _KERNEL_ROOT:
errors = []
if len(relative_parts) < 4 or relative_parts[1] not in {"ops", "benchmark"}:
errors.append(
f"{path}: kernel tests must live under "
"test/registered/kernels/{ops,benchmark}/<group>/"
)
if any("-kernel-" not in (r.effective_suite or "") for r in registries):
errors.append(f"{path}: kernel tests must use a *-kernel-* suite")
return errors
if len(relative_parts) < 3 or relative_parts[0] not in _TEST_KINDS:
return [
f"{path}: registered tests must live under "
"test/registered/<kind>/<subsystem>/; kind must be one of "
+ ", ".join(sorted(_TEST_KINDS))
+ "; kernel tests use test/registered/kernels/{ops,benchmark}/<group>/"
]
relative_parts = parts[2:]
root = relative_parts[0]
kind = relative_parts[0]
errors = []
if kind == "unit":
invalid = [
r
for r in registries
if r.backend.name != "CPU" and "-unit-" not in (r.effective_suite or "")
]
if invalid:
errors.append(f"{path}: unit tests must use CPU or dedicated unit suites")
if any(r.est_time > 60 for r in registries):
errors.append(f"{path}: unit test est_time must be <= 60 seconds")
if _contains_call(tree, "popen_launch_server"):
errors.append(f"{path}: unit tests may not launch a server")
elif kind in {"accuracy", "perf"}:
invalid = [
r
for r in registries
if not (r.effective_suite or "").startswith(("nightly-", "weekly-"))
]
if invalid:
errors.append(f"{path}: {kind} tests must use nightly/weekly suites")
elif kind == "stress":
invalid = [
r
for r in registries
if (r.effective_suite or "") != "stress"
and not (r.effective_suite or "").startswith("weekly-")
]
if invalid:
errors.append(f"{path}: stress tests must use stress/weekly suites")
return errors
if root in ("kernel", "kernels"):
canonical = (
root == "kernels"
and len(relative_parts) >= 4
and relative_parts[1] in ("ops", "benchmark")
)
return (
[] if canonical else [f"{path}: kernel tests live under {_KERNEL_LAYOUT}"]
)
if root != "unit":
return []
if _contains_call(tree, "popen_launch_server"):
return [f"{path}: unit tests may not launch a server"]
return []
def main() -> int:
@@ -209,22 +114,19 @@ def main() -> int:
non_dispatchable = [] # (file, suite) -- legacy CUDA suite no workflow invokes
dead_tests = [] # (file) -- TestCase classes that `python3 file.py` never runs
taxonomy_violations = []
changed_files = _changed_registered_files()
for f in files:
try:
registries, _has_main_entry = ci_register.ut_parse_one_file(f)
except Exception:
# Skip files that can't be parsed (syntax errors, etc.)
continue
if len(registries) == 0:
missing.append(f)
continue
# TestCase classes are dead unless __main__ runs them (CI does
# `python3 file.py`); the ERROR text below explains the fix.
# TestCase classes are dead unless __main__ runs them; CI runs the
# registered file as `python3 file.py`.
with open(f, "r", encoding="utf-8") as fh:
tree = ast.parse(fh.read(), filename=f)
if f in changed_files:
taxonomy_violations.extend(taxonomy_errors(f, registries, tree))
taxonomy_violations.extend(taxonomy_errors(f, tree))
if _defines_testcase(tree) and not _main_runs_tests(tree):
dead_tests.append(f)
for r in registries:
scripts/lint/test_check_no_bare_pytest_main.py
-86
View File
@@ -1,86 +0,0 @@
import pathlib
import tempfile
import unittest
from check_no_bare_pytest_main import find_bare_pytest_main
class TestFindBarePytestMain(unittest.TestCase):
def check_source(self, source: str) -> int | None:
with tempfile.TemporaryDirectory() as directory:
path = pathlib.Path(directory) / "example.py"
path.write_text(source, encoding="utf-8")
return find_bare_pytest_main(path)
def test_rejects_discarded_result(self):
source = """
if __name__ == "__main__":
pytest.main([__file__])
"""
self.assertEqual(self.check_source(source), 3)
def test_rejects_discarded_result_with_whitespace(self):
source = """
if __name__ == "__main__":
pytest . main([__file__])
"""
self.assertEqual(self.check_source(source), 3)
def test_accepts_propagated_result(self):
source = """
if __name__ == "__main__":
sys.exit(pytest.main([__file__]))
"""
self.assertIsNone(self.check_source(source))
def test_rejects_assigned_result(self):
source = """
if "__main__" == __name__:
exit_code = pytest.main([__file__])
"""
self.assertEqual(self.check_source(source), 3)
def test_accepts_assigned_result_that_is_later_exited(self):
source = """
if __name__ == "__main__":
exit_code = pytest.main([__file__])
sys.exit(exit_code)
"""
self.assertIsNone(self.check_source(source))
def test_accepts_assigned_result_that_is_raised(self):
source = """
if __name__ == "__main__":
exit_code = pytest.main([__file__])
raise SystemExit(exit_code)
"""
self.assertIsNone(self.check_source(source))
def test_rejects_nested_discarded_result(self):
source = """
if __name__ == "__main__":
if enabled:
pytest.main([__file__])
"""
self.assertEqual(self.check_source(source), 4)
def test_accepts_raised_system_exit(self):
source = """
if __name__ == "__main__":
raise SystemExit(pytest.main([__file__]))
"""
self.assertIsNone(self.check_source(source))
def test_rejects_unraised_system_exit(self):
source = """
if __name__ == "__main__":
error = SystemExit(pytest.main([__file__]))
"""
self.assertEqual(self.check_source(source), 3)
def test_ignores_call_outside_main_guard(self):
self.assertIsNone(self.check_source("pytest.main([__file__])\n"))
if __name__ == "__main__":
unittest.main()
scripts/lint/test_check_registered_tests.py
-51
View File
@@ -1,51 +0,0 @@
import ast
import unittest
from types import SimpleNamespace
from scripts.lint.check_registered_tests import taxonomy_errors
def _registry(suite: str):
return SimpleNamespace(effective_suite=suite, est_time=1)
class TestRegisteredTestTaxonomy(unittest.TestCase):
def setUp(self):
self.tree = ast.parse("")
self.kernel_registry = [_registry("base-b-kernel-unit-test-1-gpu-large")]
def test_plural_kernel_ops_layout_is_accepted(self):
errors = taxonomy_errors(
"test/registered/kernels/ops/attention/test_example.py",
self.kernel_registry,
self.tree,
)
self.assertEqual(errors, [])
def test_plural_kernel_benchmark_layout_is_accepted(self):
errors = taxonomy_errors(
"test/registered/kernels/benchmark/attention/bench_example.py",
[_registry("base-b-kernel-benchmark-test-1-gpu-large")],
self.tree,
)
self.assertEqual(errors, [])
def test_singular_kernel_root_is_rejected(self):
errors = taxonomy_errors(
"test/registered/kernel/attention/test_example.py",
self.kernel_registry,
self.tree,
)
self.assertTrue(errors)
def test_kernel_group_is_required(self):
errors = taxonomy_errors(
"test/registered/kernels/ops/test_example.py",
self.kernel_registry,
self.tree,
)
self.assertTrue(errors)
if __name__ == "__main__":
unittest.main()