[CI] [NPU] precise-test: gate precision on PR judgeability, not bucket count (#46042)

This commit is contained in:
shun8686
2026-10-23 20:41:22 +08:00
committed by GitHub
parent 4b1941e889
commit 0347b2bb4e
2 changed files with 130 additions and 45 deletions
+2 -2
View File
@@ -809,8 +809,8 @@ jobs:
# Bucket the recommended tests per suite. Downstream jobs consume two
# outputs per suite: should_run_<suite> ('false' -> skip the job, e.g.
# empty bucket/recommendation) and only_tests_<suite> (non-empty only in
# precision mode, gated inside the script). On failure outputs stay
# empty bucket in precision mode) and only_tests_<suite> (non-empty only
# in precision mode, gated inside the script). On failure outputs stay
# empty -> callers run full suites.
- name: Bucket recommended tests per suite
id: bucket-tests
@@ -15,15 +15,34 @@ step outputs:
enumerated a non-empty registered pool, and this
suite's bucket is empty -> the caller skips the whole
job; also 'false' for ALL suites when nothing was
recommended and every PR change is under python/sglang/
(--changed-files); 'true' otherwise (incl. full suites).
A suite forced full by the multimodal safety gate stays
'true' regardless of both skip paths.
recommended and the changed-file list is available;
'true' otherwise (incl. full suites). A suite forced
full by the multimodal safety gate stays 'true'
regardless of both skip paths.
Suite keys replace '-' with '_' because GitHub Actions output names must be
valid shell identifiers. Any failure path yields empty/missing outputs, which
callers treat as "run the full suite".
Decision model (precision vs full)
----------------------------------
Precision mode has the highest priority: when --precision-mode is not 'true',
every suite runs in full, no matter what was recommended. Otherwise a full run
is triggered by only two signals:
- a changed path in _FULL_RUN_PATHS (the pipeline configuration files),
- a missing changed-file list (test_selector.py failed),
plus the multimodal baseline safety gate described below.
Everything else is decided by the recommendation: an empty one skips every
suite, a non-empty one runs only the buckets. Precision mode is therefore not
keyed off "is any bucket non-empty" — a recommendation that no scheduled suite
contains (a base-a CPU unit test, a nightly-only test, a deleted test file) is
not runnable in this workflow, and a full run would not run it either. This is
what keeps a PR that only edits a CPU unit test from re-running the whole a3
matrix.
Diffusion (multimodal_gen) suites are enumerated from the NPU diffusion case
config instead of test/registered, and they recommend at test-case
granularity: one file holds several expensive cases, so a file-level list
@@ -119,8 +138,24 @@ def normalize_rel(path: str) -> str:
return p
# The only changes that force a full run. They are exactly the non-test paths
# that trigger this workflow's paths filter and shape its environment or its
# selection logic, so their effect cannot be judged from the coverage baseline
# (and a selector change must not be validated by the selector's own output).
_FULL_RUN_PATHS = {
".github/workflows/pr-test-npu.yml",
"python/pyproject_npu.toml",
"scripts/ci/npu/npu_ci_install_dependency.sh",
}
def needs_full_run(path: str) -> bool:
"""True when this changed path must not be judged by the recommendation."""
return path.strip().replace("\\", "/").removeprefix("./") in _FULL_RUN_PATHS
def enumerate_suite_files(suite: str, work_dir: Path) -> list:
"""List the suite's registered PR-mode test files via list_tests.py."""
"""List the suite's registered NPU PR-mode test files via list_tests.py."""
out_file = work_dir / (suite.replace("-", "_") + ".txt")
subprocess.run(
[
@@ -349,37 +384,48 @@ def write_github_summary(
target_files: list,
raw_entries: list,
recommended: set,
all_pool: set,
buckets: dict,
suite_pool_sizes: dict,
force_full: set,
baseline_gaps: dict,
precision_mode: bool,
skip_all_suites: bool = False,
decision: str = "",
decision_reason: str = "",
full_run_paths: list | None = None,
unknown_recommended: list | None = None,
) -> None:
"""Append a markdown section with the recommendation results to every
target file: $GITHUB_STEP_SUMMARY and/or a plain report file."""
unbucketed = sorted(recommended - all_pool)
full_run_paths = full_run_paths or []
unknown_recommended = unknown_recommended or []
lines = []
lines.append("### Recommended tests from coverage")
lines.append("")
if decision:
lines.append(f"**Decision**: `{decision}` — {decision_reason}")
lines.append("")
if precision_mode:
lines.append(
"Precision mode **on** — a3 suites run only the bucketed subset below."
)
elif skip_all_suites and not force_full:
lines.append("**All suites skipped**: the changes recommend no test.")
else:
lines.append("Precision mode **off** — suites run in full.")
if skip_all_suites and not force_full:
if full_run_paths:
lines.append("")
lines.append(
"**All suites skipped**: nothing was recommended and every PR "
"change is under `python/sglang/`."
"**Full run forced — pipeline configuration changed** "
"(not judgeable from coverage):"
)
lines.append("")
lines.extend(f"- `{p}`" for p in full_run_paths)
lines.append("")
lines.append(
f"Recommended **{len(raw_entries)}** test cases "
f"({len(recommended)} unique files); "
f"{len(unbucketed)} match no suite pool."
f"{len(unknown_recommended)} are not run by this workflow."
)
lines.append("")
lines.append("| Suite | Registered | Bucketed | Action |")
@@ -415,20 +461,21 @@ def write_github_summary(
lines.extend(raw_entries)
lines.append("```")
lines.append("</details>")
if unbucketed:
if unknown_recommended:
lines.append("")
lines.append(
f"<details><summary>Matched no suite pool — not run by a3 suites ({len(unbucketed)})</summary>"
f"<details><summary>Not run by this workflow ({len(unknown_recommended)})</summary>"
)
lines.append("")
lines.append("```")
lines.extend(unbucketed)
lines.extend(unknown_recommended)
lines.append("```")
lines.append("")
lines.append(
f"Not registered in any of the {len(buckets)} bucketed a3 suites — "
"they may still run elsewhere (e.g. base-a runs its full suite; "
"nightly tests run in nightly pipelines)."
"Recommended but not registered in any suite this workflow runs "
"(base-a / CPU / nightly suites belong to other pipelines, or the "
"file was deleted). Running the full suites would not run them "
"either."
)
lines.append("</details>")
if any(buckets.values()):
@@ -480,15 +527,19 @@ def main():
"--precision-mode",
type=str,
default="false",
help="'true' to enable precision mode (run only the recommended tests)",
help="'true' to enable precision mode (run only the recommended tests). "
"Any other value has the highest priority and forces every suite to "
"run in full",
)
parser.add_argument(
"--changed-files",
type=str,
default="",
help="pr_changed_files.txt written by test_selector.py; enables the "
"skip-all rule: nothing recommended + every change under python/sglang/"
" -> should_run 'false' for all suites (empty/missing = rule disabled)",
help="pr_changed_files.txt written by test_selector.py. A path listed "
"in _FULL_RUN_PATHS (pr-test-npu.yml, pyproject_npu.toml, "
"npu_ci_install_dependency.sh) forces a full run; a missing or empty "
"file forces it too, and everything else is decided by the "
"recommendation",
)
parser.add_argument(
"--github-output",
@@ -528,10 +579,8 @@ def main():
print(f"::warning::Recommended file not found: {rec_file}")
print(f"Recommended test entries: {len(raw_entries)} ({len(recommended)} files)")
# Skip-all rule: an empty recommendation is only trustworthy when every
# changed file lives under the covered product tree (python/sglang/);
# test/script/workflow changes are invisible to the coverage baseline, so
# those PRs keep the full-suite fallback.
# Only the pipeline configuration paths force a full run; everything else
# is decided by the coverage recommendation (or the missing-list fallback).
changed_files: list[str] = []
if args.changed_files:
cf = Path(args.changed_files)
@@ -539,11 +588,7 @@ def main():
changed_files = [
ln.strip() for ln in cf.read_text().splitlines() if ln.strip()
]
skip_all_suites = (
not raw_entries
and bool(changed_files)
and all(p.startswith("python/sglang/") for p in changed_files)
)
full_run_paths = sorted(p for p in changed_files if needs_full_run(p))
suites = [s.strip() for s in args.suites.split(",") if s.strip()]
diffusion_suites = {}
@@ -636,14 +681,50 @@ def main():
f"{len(tokens)} recommended"
)
precision_switch = args.precision_mode == "true"
if not precision_switch:
# Highest priority: the switch is the operator's override, so it must
# win over every automatic decision below.
decision = "full"
decision_reason = "precision mode disabled (--precision-mode is not 'true')"
elif not changed_files:
# test_selector.py writes this file on every PR run; missing/empty
# means it failed, so there is no evidence to narrow anything down.
decision = "full"
decision_reason = "changed-file list unavailable (test_selector.py did not run)"
elif full_run_paths:
decision = "full"
decision_reason = "pipeline configuration changed: " + ", ".join(full_run_paths)
elif not raw_entries:
decision = "skip_all"
decision_reason = "no test recommended"
else:
decision = "precision"
decision_reason = f"{len(raw_entries)} recommended test(s)"
skip_all_suites = decision == "skip_all"
print(
f"Decision: {decision} ({decision_reason}); "
f"{len(changed_files)} changed path(s), {len(full_run_paths)} full-run path(s)"
)
total_count = sum(len(b) for b in buckets.values())
if skip_all_suites and not force_full:
# Recommendations no suite here contains: base-a / CPU / nightly tests run
# by other pipelines, or a deleted file. None of them can be run by this
# workflow, and a full run would not run them either, so this is reported
# but never escalates to a full run.
unknown_recommended = sorted(recommended - all_pool)
if unknown_recommended:
print(
"::notice::No tests recommended and every changed file is under "
"python/sglang/ — skipping all suites."
"::notice::Recommended test(s) not run by any suite in this "
f"workflow: {', '.join(unknown_recommended)}"
)
precision_mode = args.precision_mode == "true" and total_count > 0
print(f"total_count={total_count} precision_mode={precision_mode}")
if skip_all_suites and not force_full:
print("::notice::No test recommended — skipping all suites.")
precision_mode = decision == "precision"
print(
f"total_count={total_count} decision={decision} precision_mode={precision_mode}"
)
# Outputs first, summary last: a summary failure must never cost the
# downstream jobs their bucket outputs (they'd fall back to full suites).
@@ -662,13 +743,14 @@ def main():
out.write("EOF\n")
# 'false' when precision mode is on, the suite enumerated a
# non-empty registered pool, and this bucket is empty -> the
# caller skips the whole suite job; or for ALL suites under the
# skip-all rule (nothing recommended, all changes confined to
# python/sglang/). An empty ENUMERATION (0 registered files,
# e.g. a list_tests/run_suite filter drift) falls back to
# 'true' so the job runs the full suite instead of silently
# vanishing. Anything else ('true', or a missing output from a
# failed step) means full-suite run.
# caller skips the whole suite job; or for ALL suites when
# nothing was recommended and the changed-file list was
# available (decision 'skip_all'). An empty ENUMERATION (0
# registered files, e.g. a list_tests/run_suite filter drift)
# falls back to 'true' so the job runs the full suite instead of
# silently vanishing, except under 'skip_all', where no suite is
# expected to run at all. Anything else ('true', or a missing
# output from a failed step) means a full-suite run.
# A force_full suite never takes either skip path: its baseline
# is incomplete, so "nothing recommended" proves nothing.
should_run = (
@@ -690,13 +772,16 @@ def main():
[p for p in (args.github_summary, args.report_file) if p],
raw_entries,
recommended,
all_pool,
buckets,
suite_pool_sizes,
force_full,
baseline_gaps,
precision_mode,
skip_all_suites,
decision=decision,
decision_reason=decision_reason,
full_run_paths=full_run_paths,
unknown_recommended=unknown_recommended,
)