[CI] [NPU] precise-test: gate precision on PR judgeability, not bucket count (#46042)

This commit is contained in:
shun8686
2026-10-23 20:41:22 +08:00
committed by GitHub
parent 4b1941e889
commit 0347b2bb4e
2 changed files with 130 additions and 45 deletions
+2 -2
View File
@@ -809,8 +809,8 @@ jobs:
# Bucket the recommended tests per suite. Downstream jobs consume two # Bucket the recommended tests per suite. Downstream jobs consume two
# outputs per suite: should_run_<suite> ('false' -> skip the job, e.g. # outputs per suite: should_run_<suite> ('false' -> skip the job, e.g.
# empty bucket/recommendation) and only_tests_<suite> (non-empty only in # empty bucket in precision mode) and only_tests_<suite> (non-empty only
# precision mode, gated inside the script). On failure outputs stay # in precision mode, gated inside the script). On failure outputs stay
# empty -> callers run full suites. # empty -> callers run full suites.
- name: Bucket recommended tests per suite - name: Bucket recommended tests per suite
id: bucket-tests id: bucket-tests
@@ -15,15 +15,34 @@ step outputs:
enumerated a non-empty registered pool, and this enumerated a non-empty registered pool, and this
suite's bucket is empty -> the caller skips the whole suite's bucket is empty -> the caller skips the whole
job; also 'false' for ALL suites when nothing was job; also 'false' for ALL suites when nothing was
recommended and every PR change is under python/sglang/ recommended and the changed-file list is available;
(--changed-files); 'true' otherwise (incl. full suites). 'true' otherwise (incl. full suites). A suite forced
A suite forced full by the multimodal safety gate stays full by the multimodal safety gate stays 'true'
'true' regardless of both skip paths. regardless of both skip paths.
Suite keys replace '-' with '_' because GitHub Actions output names must be Suite keys replace '-' with '_' because GitHub Actions output names must be
valid shell identifiers. Any failure path yields empty/missing outputs, which valid shell identifiers. Any failure path yields empty/missing outputs, which
callers treat as "run the full suite". callers treat as "run the full suite".
Decision model (precision vs full)
----------------------------------
Precision mode has the highest priority: when --precision-mode is not 'true',
every suite runs in full, no matter what was recommended. Otherwise a full run
is triggered by only two signals:
- a changed path in _FULL_RUN_PATHS (the pipeline configuration files),
- a missing changed-file list (test_selector.py failed),
plus the multimodal baseline safety gate described below.
Everything else is decided by the recommendation: an empty one skips every
suite, a non-empty one runs only the buckets. Precision mode is therefore not
keyed off "is any bucket non-empty" — a recommendation that no scheduled suite
contains (a base-a CPU unit test, a nightly-only test, a deleted test file) is
not runnable in this workflow, and a full run would not run it either. This is
what keeps a PR that only edits a CPU unit test from re-running the whole a3
matrix.
Diffusion (multimodal_gen) suites are enumerated from the NPU diffusion case Diffusion (multimodal_gen) suites are enumerated from the NPU diffusion case
config instead of test/registered, and they recommend at test-case config instead of test/registered, and they recommend at test-case
granularity: one file holds several expensive cases, so a file-level list granularity: one file holds several expensive cases, so a file-level list
@@ -119,8 +138,24 @@ def normalize_rel(path: str) -> str:
return p return p
# The only changes that force a full run. They are exactly the non-test paths
# that trigger this workflow's paths filter and shape its environment or its
# selection logic, so their effect cannot be judged from the coverage baseline
# (and a selector change must not be validated by the selector's own output).
_FULL_RUN_PATHS = {
".github/workflows/pr-test-npu.yml",
"python/pyproject_npu.toml",
"scripts/ci/npu/npu_ci_install_dependency.sh",
}
def needs_full_run(path: str) -> bool:
"""True when this changed path must not be judged by the recommendation."""
return path.strip().replace("\\", "/").removeprefix("./") in _FULL_RUN_PATHS
def enumerate_suite_files(suite: str, work_dir: Path) -> list: def enumerate_suite_files(suite: str, work_dir: Path) -> list:
"""List the suite's registered PR-mode test files via list_tests.py.""" """List the suite's registered NPU PR-mode test files via list_tests.py."""
out_file = work_dir / (suite.replace("-", "_") + ".txt") out_file = work_dir / (suite.replace("-", "_") + ".txt")
subprocess.run( subprocess.run(
[ [
@@ -349,37 +384,48 @@ def write_github_summary(
target_files: list, target_files: list,
raw_entries: list, raw_entries: list,
recommended: set, recommended: set,
all_pool: set,
buckets: dict, buckets: dict,
suite_pool_sizes: dict, suite_pool_sizes: dict,
force_full: set, force_full: set,
baseline_gaps: dict, baseline_gaps: dict,
precision_mode: bool, precision_mode: bool,
skip_all_suites: bool = False, skip_all_suites: bool = False,
decision: str = "",
decision_reason: str = "",
full_run_paths: list | None = None,
unknown_recommended: list | None = None,
) -> None: ) -> None:
"""Append a markdown section with the recommendation results to every """Append a markdown section with the recommendation results to every
target file: $GITHUB_STEP_SUMMARY and/or a plain report file.""" target file: $GITHUB_STEP_SUMMARY and/or a plain report file."""
unbucketed = sorted(recommended - all_pool) full_run_paths = full_run_paths or []
unknown_recommended = unknown_recommended or []
lines = [] lines = []
lines.append("### Recommended tests from coverage") lines.append("### Recommended tests from coverage")
lines.append("") lines.append("")
if decision:
lines.append(f"**Decision**: `{decision}` — {decision_reason}")
lines.append("")
if precision_mode: if precision_mode:
lines.append( lines.append(
"Precision mode **on** — a3 suites run only the bucketed subset below." "Precision mode **on** — a3 suites run only the bucketed subset below."
) )
elif skip_all_suites and not force_full:
lines.append("**All suites skipped**: the changes recommend no test.")
else: else:
lines.append("Precision mode **off** — suites run in full.") lines.append("Precision mode **off** — suites run in full.")
if skip_all_suites and not force_full: if full_run_paths:
lines.append("") lines.append("")
lines.append( lines.append(
"**All suites skipped**: nothing was recommended and every PR " "**Full run forced — pipeline configuration changed** "
"change is under `python/sglang/`." "(not judgeable from coverage):"
) )
lines.append("")
lines.extend(f"- `{p}`" for p in full_run_paths)
lines.append("") lines.append("")
lines.append( lines.append(
f"Recommended **{len(raw_entries)}** test cases " f"Recommended **{len(raw_entries)}** test cases "
f"({len(recommended)} unique files); " f"({len(recommended)} unique files); "
f"{len(unbucketed)} match no suite pool." f"{len(unknown_recommended)} are not run by this workflow."
) )
lines.append("") lines.append("")
lines.append("| Suite | Registered | Bucketed | Action |") lines.append("| Suite | Registered | Bucketed | Action |")
@@ -415,20 +461,21 @@ def write_github_summary(
lines.extend(raw_entries) lines.extend(raw_entries)
lines.append("```") lines.append("```")
lines.append("</details>") lines.append("</details>")
if unbucketed: if unknown_recommended:
lines.append("") lines.append("")
lines.append( lines.append(
f"<details><summary>Matched no suite pool — not run by a3 suites ({len(unbucketed)})</summary>" f"<details><summary>Not run by this workflow ({len(unknown_recommended)})</summary>"
) )
lines.append("") lines.append("")
lines.append("```") lines.append("```")
lines.extend(unbucketed) lines.extend(unknown_recommended)
lines.append("```") lines.append("```")
lines.append("") lines.append("")
lines.append( lines.append(
f"Not registered in any of the {len(buckets)} bucketed a3 suites — " "Recommended but not registered in any suite this workflow runs "
"they may still run elsewhere (e.g. base-a runs its full suite; " "(base-a / CPU / nightly suites belong to other pipelines, or the "
"nightly tests run in nightly pipelines)." "file was deleted). Running the full suites would not run them "
"either."
) )
lines.append("</details>") lines.append("</details>")
if any(buckets.values()): if any(buckets.values()):
@@ -480,15 +527,19 @@ def main():
"--precision-mode", "--precision-mode",
type=str, type=str,
default="false", default="false",
help="'true' to enable precision mode (run only the recommended tests)", help="'true' to enable precision mode (run only the recommended tests). "
"Any other value has the highest priority and forces every suite to "
"run in full",
) )
parser.add_argument( parser.add_argument(
"--changed-files", "--changed-files",
type=str, type=str,
default="", default="",
help="pr_changed_files.txt written by test_selector.py; enables the " help="pr_changed_files.txt written by test_selector.py. A path listed "
"skip-all rule: nothing recommended + every change under python/sglang/" "in _FULL_RUN_PATHS (pr-test-npu.yml, pyproject_npu.toml, "
" -> should_run 'false' for all suites (empty/missing = rule disabled)", "npu_ci_install_dependency.sh) forces a full run; a missing or empty "
"file forces it too, and everything else is decided by the "
"recommendation",
) )
parser.add_argument( parser.add_argument(
"--github-output", "--github-output",
@@ -528,10 +579,8 @@ def main():
print(f"::warning::Recommended file not found: {rec_file}") print(f"::warning::Recommended file not found: {rec_file}")
print(f"Recommended test entries: {len(raw_entries)} ({len(recommended)} files)") print(f"Recommended test entries: {len(raw_entries)} ({len(recommended)} files)")
# Skip-all rule: an empty recommendation is only trustworthy when every # Only the pipeline configuration paths force a full run; everything else
# changed file lives under the covered product tree (python/sglang/); # is decided by the coverage recommendation (or the missing-list fallback).
# test/script/workflow changes are invisible to the coverage baseline, so
# those PRs keep the full-suite fallback.
changed_files: list[str] = [] changed_files: list[str] = []
if args.changed_files: if args.changed_files:
cf = Path(args.changed_files) cf = Path(args.changed_files)
@@ -539,11 +588,7 @@ def main():
changed_files = [ changed_files = [
ln.strip() for ln in cf.read_text().splitlines() if ln.strip() ln.strip() for ln in cf.read_text().splitlines() if ln.strip()
] ]
skip_all_suites = ( full_run_paths = sorted(p for p in changed_files if needs_full_run(p))
not raw_entries
and bool(changed_files)
and all(p.startswith("python/sglang/") for p in changed_files)
)
suites = [s.strip() for s in args.suites.split(",") if s.strip()] suites = [s.strip() for s in args.suites.split(",") if s.strip()]
diffusion_suites = {} diffusion_suites = {}
@@ -636,14 +681,50 @@ def main():
f"{len(tokens)} recommended" f"{len(tokens)} recommended"
) )
precision_switch = args.precision_mode == "true"
if not precision_switch:
# Highest priority: the switch is the operator's override, so it must
# win over every automatic decision below.
decision = "full"
decision_reason = "precision mode disabled (--precision-mode is not 'true')"
elif not changed_files:
# test_selector.py writes this file on every PR run; missing/empty
# means it failed, so there is no evidence to narrow anything down.
decision = "full"
decision_reason = "changed-file list unavailable (test_selector.py did not run)"
elif full_run_paths:
decision = "full"
decision_reason = "pipeline configuration changed: " + ", ".join(full_run_paths)
elif not raw_entries:
decision = "skip_all"
decision_reason = "no test recommended"
else:
decision = "precision"
decision_reason = f"{len(raw_entries)} recommended test(s)"
skip_all_suites = decision == "skip_all"
print(
f"Decision: {decision} ({decision_reason}); "
f"{len(changed_files)} changed path(s), {len(full_run_paths)} full-run path(s)"
)
total_count = sum(len(b) for b in buckets.values()) total_count = sum(len(b) for b in buckets.values())
if skip_all_suites and not force_full: # Recommendations no suite here contains: base-a / CPU / nightly tests run
# by other pipelines, or a deleted file. None of them can be run by this
# workflow, and a full run would not run them either, so this is reported
# but never escalates to a full run.
unknown_recommended = sorted(recommended - all_pool)
if unknown_recommended:
print( print(
"::notice::No tests recommended and every changed file is under " "::notice::Recommended test(s) not run by any suite in this "
"python/sglang/ — skipping all suites." f"workflow: {', '.join(unknown_recommended)}"
) )
precision_mode = args.precision_mode == "true" and total_count > 0
print(f"total_count={total_count} precision_mode={precision_mode}") if skip_all_suites and not force_full:
print("::notice::No test recommended — skipping all suites.")
precision_mode = decision == "precision"
print(
f"total_count={total_count} decision={decision} precision_mode={precision_mode}"
)
# Outputs first, summary last: a summary failure must never cost the # Outputs first, summary last: a summary failure must never cost the
# downstream jobs their bucket outputs (they'd fall back to full suites). # downstream jobs their bucket outputs (they'd fall back to full suites).
@@ -662,13 +743,14 @@ def main():
out.write("EOF\n") out.write("EOF\n")
# 'false' when precision mode is on, the suite enumerated a # 'false' when precision mode is on, the suite enumerated a
# non-empty registered pool, and this bucket is empty -> the # non-empty registered pool, and this bucket is empty -> the
# caller skips the whole suite job; or for ALL suites under the # caller skips the whole suite job; or for ALL suites when
# skip-all rule (nothing recommended, all changes confined to # nothing was recommended and the changed-file list was
# python/sglang/). An empty ENUMERATION (0 registered files, # available (decision 'skip_all'). An empty ENUMERATION (0
# e.g. a list_tests/run_suite filter drift) falls back to # registered files, e.g. a list_tests/run_suite filter drift)
# 'true' so the job runs the full suite instead of silently # falls back to 'true' so the job runs the full suite instead of
# vanishing. Anything else ('true', or a missing output from a # silently vanishing, except under 'skip_all', where no suite is
# failed step) means full-suite run. # expected to run at all. Anything else ('true', or a missing
# output from a failed step) means a full-suite run.
# A force_full suite never takes either skip path: its baseline # A force_full suite never takes either skip path: its baseline
# is incomplete, so "nothing recommended" proves nothing. # is incomplete, so "nothing recommended" proves nothing.
should_run = ( should_run = (
@@ -690,13 +772,16 @@ def main():
[p for p in (args.github_summary, args.report_file) if p], [p for p in (args.github_summary, args.report_file) if p],
raw_entries, raw_entries,
recommended, recommended,
all_pool,
buckets, buckets,
suite_pool_sizes, suite_pool_sizes,
force_full, force_full,
baseline_gaps, baseline_gaps,
precision_mode, precision_mode,
skip_all_suites, skip_all_suites,
decision=decision,
decision_reason=decision_reason,
full_run_paths=full_run_paths,
unknown_recommended=unknown_recommended,
) )