[CI] Key scheduled CUDA suites by runner_config instead of hand-written jobs (#34186)
This commit is contained in:
Liangsheng Yin
@@ -4,11 +4,11 @@ Pre-commit hook: validate CI registry calls under test/registered/.
|
||||
|
||||
1. Every test file must contain a CI registry call (register_cuda_ci,
|
||||
register_amd_ci, etc.).
|
||||
2. A CUDA test must register its PR-test suite via the modern
|
||||
2. A CUDA test must register its suite via the modern
|
||||
`stage=`/`runner_config=` form. The legacy single-string `suite=` is reserved
|
||||
for the nightly/stress/weekly families (and for AMD/CPU/NPU suites); any other
|
||||
CUDA `suite=` resolves to a name no PR-test workflow invokes, so the test
|
||||
silently never runs. Two shapes are rejected:
|
||||
for the stress family (and for AMD/CPU/NPU suites); any other CUDA `suite=`
|
||||
resolves to a name no workflow invokes, so the test silently never runs.
|
||||
Two shapes are rejected:
|
||||
a. `{stage}-test-{runner_config}` -- the modern name stuffed back into the
|
||||
legacy form. Reported with the exact stage/runner split to use.
|
||||
b. an older `{stage}-{runner_config}` PR-test name (e.g. the pre-migration
|
||||
@@ -33,11 +33,10 @@ import sys
|
||||
# shape is always expressible (and should be expressed) the modern way.
|
||||
_MODERN_SHAPE = re.compile(r"^(.+)-test-(.+)$")
|
||||
|
||||
# The only suite families a CUDA registry may keep on the legacy single-string
|
||||
# `suite=` form. Everything else is a PR-test/base stage that must use the
|
||||
# modern stage=/runner_config= form (otherwise its effective_suite matches no
|
||||
# suite the PR-test workflows invoke, and the test silently never runs).
|
||||
_LEGACY_CUDA_PREFIXES = ("nightly", "stress", "weekly")
|
||||
# The only CUDA suite family still allowed on the legacy single-string `suite=`
|
||||
# form. Anything else needs stage=/runner_config=, or its effective_suite matches
|
||||
# no suite any workflow invokes and the test silently never runs.
|
||||
_LEGACY_CUDA_PREFIXES = ("stress",)
|
||||
|
||||
|
||||
def _defines_testcase(tree: ast.AST) -> bool:
|
||||
@@ -118,8 +117,6 @@ def main() -> int:
|
||||
and r.runner_config is None
|
||||
):
|
||||
continue
|
||||
# nightly/stress/weekly are the only CUDA suites allowed to stay on
|
||||
# the legacy single-string form.
|
||||
if r.suite.split("-", 1)[0] in _LEGACY_CUDA_PREFIXES:
|
||||
continue
|
||||
m = _MODERN_SHAPE.match(r.suite)
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
{
|
||||
"_comment": "Manual overrides for list_stage_models.py. by_file/by_suite ADD models the static scan cannot see (models built dynamically, read from configs, passed via CLI args). deny REMOVES false-positive ids the heuristic mistakes for models. Keys in by_file are repo-relative test paths (e.g. test/registered/foo/test_bar.py). suite_labels maps legacy suite= registrations (no runner_config) to the GH runner label(s) their dispatching workflow hardcodes in runs-on -- a list, because one suite can run on several labels (nightly-8-gpu-common). $b200_runner is the dynamic-b200 placeholder from runner_configs.yml. Deliberately absent: nightly-4-gpu-gb300-* (run as k8s pods, not GHA runners) and nightly-2-gpu (registered but dispatched by no workflow); both stay visible in unmapped_suites.",
|
||||
"_comment": "Manual overrides for list_stage_models.py. by_file/by_suite ADD models the static scan cannot see (models built dynamically, read from configs, passed via CLI args). deny REMOVES false-positive ids the heuristic mistakes for models. Keys in by_file are repo-relative test paths (e.g. test/registered/foo/test_bar.py). suite_labels maps legacy suite= registrations (no runner_config) to the GH runner label(s) their dispatching workflow hardcodes in runs-on -- a list, because one suite can run on several labels. $b200_runner is the dynamic-b200 placeholder from runner_configs.yml. Suites registered with stage=/runner_config= need no entry: their label resolves through runner_configs.yml. Anything unmappable stays visible in unmapped_suites.",
|
||||
"by_file": {},
|
||||
"by_suite": {},
|
||||
"suite_labels": {
|
||||
@@ -7,21 +7,7 @@
|
||||
"base-b-kernel-unit-1-gpu-b200": ["$b200_runner"],
|
||||
"base-b-kernel-unit-1-gpu-large": ["1-gpu-h100"],
|
||||
"base-b-kernel-unit-8-gpu-h200": ["8-gpu-h200"],
|
||||
"nightly-1-gpu": ["1-gpu-h100"],
|
||||
"nightly-4-gpu": ["4-gpu-h100"],
|
||||
"nightly-4-gpu-b200": ["$b200_runner"],
|
||||
"nightly-8-gpu-b200": ["8-gpu-b200"],
|
||||
"nightly-8-gpu-common": ["8-gpu-h200", "8-gpu-b200"],
|
||||
"nightly-8-gpu-h200": ["8-gpu-h200"],
|
||||
"nightly-eval-text-2-gpu": ["2-gpu-h100"],
|
||||
"nightly-eval-vlm-2-gpu": ["2-gpu-h100"],
|
||||
"nightly-kernel-1-gpu": ["1-gpu-h100"],
|
||||
"nightly-kernel-8-gpu-h200": ["8-gpu-h200"],
|
||||
"nightly-perf-text-2-gpu": ["2-gpu-h100"],
|
||||
"nightly-perf-vlm-2-gpu": ["2-gpu-h100"],
|
||||
"nightly-precision-8-gpu-h200": ["8-gpu-h200"],
|
||||
"stress": ["8-gpu-h200"],
|
||||
"weekly-8-gpu-h200": ["8-gpu-h200"]
|
||||
"stress": ["8-gpu-h200"]
|
||||
},
|
||||
"deny": [
|
||||
"tok/req",
|
||||
|
||||
@@ -128,8 +128,11 @@ def compute_partitions(
|
||||
in-source `est_time` / `(1.0, 0.0)`.
|
||||
`full_parallel=True` lifts the matrix-fanout throttle.
|
||||
"""
|
||||
# Allowlist: stages pr-test.yml dispatches. Stress / weekly /
|
||||
# nightly-* live in test/registered/ but pr-test doesn't run them.
|
||||
# Allowlist of the stages this workflow dispatches -- what keeps stress /
|
||||
# weekly / nightly out, since CUDA scheduled suites no longer carry
|
||||
# `nightly=True`. The nightly filter still matters for CPU: some tests sit on
|
||||
# a dispatched suite with the flag set, so run_suite.py skips them and their
|
||||
# est_time must not inflate the shard count.
|
||||
dispatched_suites = set(run_timeouts) | set(_BASE_A_OVERRIDES)
|
||||
suite_tests = defaultdict(list)
|
||||
for t in tests:
|
||||
|
||||
@@ -702,11 +702,8 @@ def _extract_runner_configs(content):
|
||||
|
||||
|
||||
def _extract_legacy_suites(content):
|
||||
"""Pull every legacy single-string `suite=` from `register_cuda_ci(...)` calls.
|
||||
|
||||
Mirrors _extract_runner_configs for the legacy nightly/weekly shape: a file
|
||||
may register on multiple pools, so collect all of them rather than the first.
|
||||
"""
|
||||
"""Pull every legacy single-string `suite=` from `register_cuda_ci(...)`
|
||||
calls. Used only to report why such a file is not dispatchable."""
|
||||
out = []
|
||||
for args in re.finditer(
|
||||
r"^[^#\n]*register_cuda_ci\s*\(([^)]*)\)", content, re.MULTILINE
|
||||
@@ -717,38 +714,6 @@ def _extract_legacy_suites(content):
|
||||
return out
|
||||
|
||||
|
||||
# Legacy nightly/weekly CUDA suites register with a single-string `suite=`
|
||||
# instead of `runner_config=`, so they carry no runner metadata of their own.
|
||||
# Map each to the runner_config in scripts/ci/runner_configs.yml whose hardware
|
||||
# matches the runner the nightly/weekly pipeline actually uses (see
|
||||
# .github/workflows/{nightly,weekly}-test-nvidia.yml), so /rerun-test can still
|
||||
# dispatch a single nightly/weekly test. The runner label, install script,
|
||||
# timeout and rdma_devices are then resolved from
|
||||
# runner_configs.yml as usual, keeping that file the single source of truth for
|
||||
# runner details.
|
||||
#
|
||||
# Suites on hardware with no matching runner_config (e.g. nightly-4-gpu-gb300)
|
||||
# and non-CUDA suites (npu/amd) are intentionally absent and stay
|
||||
# non-dispatchable until a matching runner_config exists.
|
||||
_LEGACY_SUITE_TO_RUNNER_CONFIG = {
|
||||
"nightly-1-gpu": "1-gpu-large",
|
||||
"nightly-kernel-1-gpu": "1-gpu-large",
|
||||
"nightly-eval-text-2-gpu": "2-gpu-large",
|
||||
"nightly-perf-text-2-gpu": "2-gpu-large",
|
||||
"nightly-eval-vlm-2-gpu": "2-gpu-large",
|
||||
"nightly-perf-vlm-2-gpu": "2-gpu-large",
|
||||
"nightly-4-gpu": "4-gpu-h100",
|
||||
"nightly-4-gpu-b200": "4-gpu-b200",
|
||||
"nightly-8-gpu-common": ["8-gpu-h200", "8-gpu-b200"],
|
||||
"nightly-8-gpu-h200": "8-gpu-h200",
|
||||
"nightly-kernel-8-gpu-h200": "8-gpu-h200",
|
||||
"nightly-precision-8-gpu-h200": "8-gpu-h200",
|
||||
"nightly-8-gpu-h20": "8-gpu-h20",
|
||||
"nightly-8-gpu-b200": "8-gpu-b200",
|
||||
"weekly-8-gpu-h200": "8-gpu-h200",
|
||||
}
|
||||
|
||||
|
||||
def _dispatch_err(suite, msg):
|
||||
"""Build a detect_suite error result for the given suite."""
|
||||
return {
|
||||
@@ -811,11 +776,10 @@ def detect_suite(file_path_from_test):
|
||||
pool it should run on — so this returns a *list* of dispatch dicts, one
|
||||
per registration. Runner label, install script, timeout, and rdma_devices
|
||||
are all resolved from scripts/ci/runner_configs.yml — the
|
||||
same single source of truth that drives the main PR test pipeline.
|
||||
|
||||
Legacy nightly/weekly CUDA suites (single-string `suite=`) are dispatchable
|
||||
too: each suite name is mapped to the matching runner_config via
|
||||
_LEGACY_SUITE_TO_RUNNER_CONFIG, then resolved the same way.
|
||||
same single source of truth that drives the main PR test pipeline. Every
|
||||
dispatchable CUDA suite, per-commit and scheduled alike, goes through that
|
||||
one path; the legacy single-string `suite=` carries no runner_config and is
|
||||
reported as non-dispatchable.
|
||||
|
||||
CPU files yield a single-element list. A file with no recognised (or no
|
||||
dispatchable) registration yields a one-element list whose dict has an
|
||||
@@ -837,19 +801,7 @@ def detect_suite(file_path_from_test):
|
||||
results.append(_resolve_runner_config(rc, full_path, suite))
|
||||
return results
|
||||
|
||||
# Legacy nightly/weekly CUDA suites: single-string `suite=`, no
|
||||
# runner_config. Map each mappable suite to its runner_config and resolve.
|
||||
legacy_suites = _extract_legacy_suites(content)
|
||||
mappable = [s for s in legacy_suites if s in _LEGACY_SUITE_TO_RUNNER_CONFIG]
|
||||
if mappable:
|
||||
results = []
|
||||
for s in mappable:
|
||||
rcs = _LEGACY_SUITE_TO_RUNNER_CONFIG[s]
|
||||
if isinstance(rcs, str):
|
||||
rcs = [rcs]
|
||||
for rc in rcs:
|
||||
results.append(_resolve_runner_config(rc, full_path, s))
|
||||
return results
|
||||
|
||||
if re.search(r"^[^#\n]*register_cpu_ci\s*\(", content, re.MULTILINE):
|
||||
return [
|
||||
@@ -869,11 +821,11 @@ def detect_suite(file_path_from_test):
|
||||
return [
|
||||
_dispatch_err(
|
||||
suite,
|
||||
f"Suite `{suite}` in `{full_path}` is not dispatchable via "
|
||||
f"/rerun-test. It has no entry in _LEGACY_SUITE_TO_RUNNER_CONFIG "
|
||||
f"— either it is a non-CUDA suite (npu/amd) or it runs on "
|
||||
f"hardware with no matching runner_config in "
|
||||
f"scripts/ci/runner_configs.yml.",
|
||||
f"Suite `{suite}` in `{full_path}` is registered with the legacy "
|
||||
f"single-string `suite=`, which carries no runner_config and so "
|
||||
f"is not dispatchable via /rerun-test. Re-register it with "
|
||||
f"`stage=`/`runner_config=` (CUDA), or dispatch its own "
|
||||
f"workflow (npu/amd).",
|
||||
)
|
||||
]
|
||||
|
||||
|
||||
Reference in New Issue
Block a user