[CI] [NPU] precise-test: recommend multimodal_gen cases at test-case granularity (#45955)

This commit is contained in:
shun8686
2026-10-23 15:57:31 +08:00
committed by GitHub
parent 6d6c9cd4ae
commit 472208fec0
3 changed files with 400 additions and 19 deletions
+9
View File
@@ -639,6 +639,10 @@ jobs:
only_tests_base_c_test_perf_2_npu_a3: ${{ steps.bucket-tests.outputs.only_tests_base_c_test_perf_2_npu_a3 }}
should_run_base_c_test_perf_16_npu_a3: ${{ steps.bucket-tests.outputs.should_run_base_c_test_perf_16_npu_a3 }}
only_tests_base_c_test_perf_16_npu_a3: ${{ steps.bucket-tests.outputs.only_tests_base_c_test_perf_16_npu_a3 }}
should_run_multimodal_gen_test_1_npu_a3: ${{ steps.bucket-tests.outputs.should_run_multimodal_gen_test_1_npu_a3 }}
only_tests_multimodal_gen_test_1_npu_a3: ${{ steps.bucket-tests.outputs.only_tests_multimodal_gen_test_1_npu_a3 }}
should_run_multimodal_gen_test_4_npu_a3: ${{ steps.bucket-tests.outputs.should_run_multimodal_gen_test_4_npu_a3 }}
only_tests_multimodal_gen_test_4_npu_a3: ${{ steps.bucket-tests.outputs.only_tests_multimodal_gen_test_4_npu_a3 }}
env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
@@ -654,10 +658,13 @@ jobs:
with:
# test/registered + python/sglang/test/ci are needed by
# list_tests.py to enumerate each suite's test files for bucketing.
# The diffusion case config is read (AST only) by
# bucket_recommended_tests.py to enumerate multimodal cases.
sparse-checkout: |
scripts
test/registered
python/sglang/test/ci
python/sglang/multimodal_gen/test/server/ascend/testcase_configs_npu.py
sparse-checkout-cone-mode: false
- name: Download test case map
@@ -731,6 +738,8 @@ jobs:
--recommended scripts/ci/npu/precise-test/recommended_pytest_paths.txt \
--changed-files scripts/ci/npu/precise-test/pr_changed_files.txt \
--suites "base-b-test-1-npu-a3,base-b-test-2-npu-a3,base-b-test-4-npu-a3,base-b-test-8-npu-a3,base-b-test-16-npu-a3,base-c-test-acc-2-npu-a3,base-c-test-acc-16-npu-a3,base-c-test-perf-2-npu-a3,base-c-test-perf-16-npu-a3" \
--diffusion-suites "multimodal-gen-test-1-npu-a3:1-npu,multimodal-gen-test-4-npu-a3:2-npu" \
--case-map "$PWD/test_case_map.json" \
--precision-mode "${PRECISION_MODE}" \
--github-output "${GITHUB_OUTPUT}" \
--github-summary "${GITHUB_STEP_SUMMARY:-}" \
@@ -16,12 +16,36 @@ step outputs:
suite's bucket is empty -> the caller skips the whole
job; also 'false' for ALL suites when nothing was
recommended and every PR change is under python/sglang/
(--changed-files); 'true' otherwise (incl. full suites)
(--changed-files); 'true' otherwise (incl. full suites).
A suite forced full by the multimodal safety gate stays
'true' regardless of both skip paths.
Suite keys replace '-' with '_' because GitHub Actions output names must be
valid shell identifiers. Any failure path yields empty/missing outputs, which
callers treat as "run the full suite".
Diffusion (multimodal_gen) suites are enumerated from the NPU diffusion case
config instead of test/registered, and they recommend at test-case
granularity: one file holds several expensive cases, so a file-level list
would be equivalent to running the whole suite. ``--diffusion-suites`` takes
``job-name:suite`` pairs, e.g.
``multimodal-gen-test-1-npu-a3:1-npu,multimodal-gen-test-4-npu-a3:2-npu``.
Their ``only_tests_<key>`` lines are pytest ``-k`` tokens rather than paths:
[<case_id>] parametrized diffusion case
<standalone file stem> standalone diffusion test file
Safety gate: if the coverage baseline (``--case-map``) has no multimodal_gen
entry, the diffusion buckets are forced to full-suite. Without that gate a
baseline built before multimodal collection existed would make every
multimodal bucket empty, and precision mode would silently skip the diffusion
cases. The same full-suite fallback applies per suite when the baseline is
missing any of that suite's cases or standalone files: a newly added case has
no coverage-map key by definition, so no PR could ever recommend it until the
baseline is rebuilt. A forced suite ignores BOTH the precision-mode skip and
the skip-all rule below — "the baseline cannot judge this suite" is not the
same as "no test is affected".
When --github-summary points at $GITHUB_STEP_SUMMARY, a markdown section is
appended with the recommended test cases, counts, per-suite buckets, and the
files that matched no suite pool (not run by the bucketed a3 suites).
@@ -31,7 +55,10 @@ upload, so the report is downloadable from the run page).
Usage:
python3 bucket_recommended_tests.py \
--recommended recommended_pytest_paths.txt \
--changed-files pr_changed_files.txt \
--suites base-a-test-npu,base-b-test-1-npu-a3 \
--diffusion-suites multimodal-gen-test-1-npu-a3:1-npu,multimodal-gen-test-4-npu-a3:2-npu \
--case-map test_case_map.json \
--precision-mode true \
--github-output "$GITHUB_OUTPUT" \
--github-summary "$GITHUB_STEP_SUMMARY" \
@@ -39,10 +66,15 @@ Usage:
"""
import argparse
import ast
import json
import os
import posixpath
import re
import subprocess
import sys
import tempfile
from dataclasses import dataclass
from pathlib import Path
# Repo layout: this script lives at <repo>/scripts/ci/npu/precise-test/.
@@ -50,6 +82,29 @@ SCRIPT_DIR = Path(__file__).resolve().parent
LIST_TESTS = SCRIPT_DIR / "list_tests.py"
REPO_ROOT = SCRIPT_DIR.parents[3]
DIFFUSION_UTILS_DIR = REPO_ROOT / "scripts" / "ci" / "utils" / "diffusion"
# Any coverage-map key under this prefix proves the baseline includes
# multimodal_gen cases (see the safety gate above).
DIFFUSION_TEST_PREFIX = "python/sglang/multimodal_gen/test/"
# Standalone files in that config are relative to the server/ test root.
DIFFUSION_TEST_ROOT_REL = DIFFUSION_TEST_PREFIX + "server"
DIFFUSION_CASE_CONFIG = (
REPO_ROOT / DIFFUSION_TEST_ROOT_REL / "ascend/testcase_configs_npu.py"
)
# diffusion_case_parser reads this at import time to resolve ONE_NPU_CASES /
# TWO_NPU_CASES instead of the GPU case lists.
os.environ["USE_NPU_CONFIGS"] = "true"
sys.path.insert(0, str(DIFFUSION_UTILS_DIR))
from diffusion_case_parser import ( # noqa: E402
parse_standalone_data,
parse_testcase_configs,
)
# Trailing "[case_id]" of a parametrized pytest node id.
_CASE_ID_RE = re.compile(r"\[([^\[\]]+)\]$")
def normalize_rel(path: str) -> str:
"""Normalize a pytest path entry to a repo-relative posix file path.
@@ -84,6 +139,212 @@ def enumerate_suite_files(suite: str, work_dir: Path) -> list:
return [line.strip() for line in f if line.strip()]
@dataclass
class DiffusionPool:
"""Recommendation pool of one diffusion suite."""
cases_by_file: dict # repo-relative test file -> [case_id, ...]
standalone_files: list # repo-relative standalone test files
@property
def size(self) -> int:
return sum(len(ids) for ids in self.cases_by_file.values()) + len(
self.standalone_files
)
@property
def case_ids(self) -> list:
return [cid for ids in self.cases_by_file.values() for cid in ids]
def files(self) -> set:
return set(self.cases_by_file) | set(self.standalone_files)
def _diffusion_rel(file_rel: str) -> str:
"""Resolve a config-relative test file to a repo-relative posix path."""
return posixpath.normpath(
posixpath.join(DIFFUSION_TEST_ROOT_REL, file_rel.replace("\\", "/"))
)
def _parse_parametrized_groups(case_config: Path) -> dict:
"""Parse PARAMETRIZED_CASE_GROUPS into suite -> [(file, case list var)].
diffusion_case_parser reports case ids and standalone files but not which
file a case lives in, which is needed to expand file-level
recommendations. The mapping dict is parsed here with AST only, so this
stays import-free of torch.
"""
tree = ast.parse(case_config.read_text(encoding="utf-8"), filename=str(case_config))
groups = {}
for node in ast.walk(tree):
if not isinstance(node, ast.Assign):
continue
targets = {t.id for t in node.targets if isinstance(t, ast.Name)}
if "PARAMETRIZED_CASE_GROUPS" not in targets or not isinstance(
node.value, ast.Dict
):
continue
for key, value in zip(node.value.keys, node.value.values):
if not (isinstance(key, ast.Constant) and isinstance(key.value, str)):
continue
if not isinstance(value, ast.List):
continue
entries = []
for elt in value.elts:
if not isinstance(elt, ast.Tuple) or len(elt.elts) != 2:
continue
file_node, cases_node = elt.elts
if isinstance(file_node, ast.Constant) and isinstance(
cases_node, ast.Name
):
entries.append((file_node.value, cases_node.id))
groups[key.value] = entries
return groups
def collect_diffusion_pools() -> dict:
"""Enumerate the NPU diffusion suites and their recommendation pools."""
case_lists = parse_testcase_configs(DIFFUSION_CASE_CONFIG)
standalone_files, _ = parse_standalone_data(DIFFUSION_CASE_CONFIG)
parametrized_groups = _parse_parametrized_groups(DIFFUSION_CASE_CONFIG)
# A suite may have only standalone files, so enumerate the union instead
# of the parametrized groups alone (a missing pool means "run full").
pools = {}
for suite in sorted(set(parametrized_groups) | set(standalone_files)):
cases_by_file = {}
for file_rel, list_name in parametrized_groups.get(suite, []):
rel = _diffusion_rel(file_rel)
case_ids = cases_by_file.setdefault(rel, [])
for case_id in case_lists.get(list_name, []):
if case_id not in case_ids:
case_ids.append(case_id)
pools[suite] = DiffusionPool(
cases_by_file=cases_by_file,
standalone_files=[
_diffusion_rel(f) for f in standalone_files.get(suite, [])
],
)
_warn_standalone_token_collisions(pools)
return pools
def _warn_standalone_token_collisions(pools: dict) -> None:
"""Warn when a standalone -k token is not unique inside the diffusion run.
Standalone files are selected by pytest module-name keyword, so two files
sharing a stem (or a stem that is a substring of a case id) would make one
token select more than intended. Over-running is harmless, hence a warning
instead of an error.
"""
stems = {}
case_ids = set()
for suite, pool in pools.items():
case_ids.update(pool.case_ids)
for rel in pool.standalone_files:
stems.setdefault(Path(rel).stem, []).append(f"{suite}:{rel}")
for stem, owners in sorted(stems.items()):
if len(owners) > 1:
print(
f"::warning::Standalone -k token {stem!r} covers multiple files: "
f"{', '.join(owners)}"
)
hits = sorted(case_id for case_id in case_ids if stem in case_id)
if hits:
print(
f"::warning::Standalone -k token {stem!r} also matches case "
f"id(s): {', '.join(hits)}"
)
def match_diffusion_pool(pool: DiffusionPool, raw_entries: list) -> tuple:
"""Match recommended entries against one diffusion pool.
A file-level entry (a changed test file, or a whole-file coverage target)
selects every case of that file; a case-level entry selects just that
case. Standalone files are selected as whole files.
"""
selected_cases = []
selected_standalones = []
for entry in raw_entries:
rel = normalize_rel(entry)
_, _, node = entry.strip().replace("\\", "/").partition("::")
if rel in pool.standalone_files:
if rel not in selected_standalones:
selected_standalones.append(rel)
continue
case_ids = pool.cases_by_file.get(rel)
if case_ids is None:
continue
match = _CASE_ID_RE.search(node)
if match:
if match.group(1) in case_ids and match.group(1) not in selected_cases:
selected_cases.append(match.group(1))
else:
for case_id in case_ids:
if case_id not in selected_cases:
selected_cases.append(case_id)
return selected_cases, selected_standalones
def diffusion_tokens(selected_cases: list, selected_standalones: list) -> list:
"""Render a diffusion bucket as pytest -k tokens.
Bracket the case id so ``[x]`` cannot match ``[x_v2]``. Standalone files
are matched by module-name keyword (the runner applies -k to pytest, whose
keywords include the module name).
"""
tokens = [f"[{case_id}]" for case_id in selected_cases]
tokens.extend(Path(rel).stem for rel in selected_standalones)
return tokens
def load_multimodal_map_keys(case_map: Path):
"""Return the coverage baseline's multimodal_gen keys, or None.
None means "the baseline cannot be trusted for multimodal" (missing,
unreadable, or built before multimodal collection existed); callers then
run the diffusion suites in full.
"""
if not case_map.is_file():
print(f"::warning::Test case map not found: {case_map}")
return None
try:
with open(case_map, encoding="utf-8") as f:
data = json.load(f)
except (OSError, ValueError) as exc:
print(f"::warning::Cannot read test case map {case_map}: {exc}")
return None
if not isinstance(data, dict):
print(f"::warning::Unexpected test case map format: {case_map}")
return None
keys = {key for key in data if key.startswith(DIFFUSION_TEST_PREFIX)}
if not keys:
print(
"::warning::Test case map has no multimodal_gen entries; the diffusion "
"suites must run in full. Rebuild the coverage baseline."
)
return None
return keys
def missing_from_baseline(pool: DiffusionPool, map_keys: set) -> list:
"""Pool entries that the coverage baseline never collected.
A case or standalone file added after the last coverage run has no map
key, so no PR diff can ever recommend it. Such a suite must run in full
until the baseline is rebuilt.
"""
covered_cases = set()
for key in map_keys:
match = _CASE_ID_RE.search(key)
if match:
covered_cases.add(match.group(1))
missing = [cid for cid in pool.case_ids if cid not in covered_cases]
missing.extend(f for f in pool.standalone_files if f not in map_keys)
return missing
def write_github_summary(
target_files: list,
raw_entries: list,
@@ -91,6 +352,8 @@ def write_github_summary(
all_pool: set,
buckets: dict,
suite_pool_sizes: dict,
force_full: set,
baseline_gaps: dict,
precision_mode: bool,
skip_all_suites: bool = False,
) -> None:
@@ -106,7 +369,7 @@ def write_github_summary(
)
else:
lines.append("Precision mode **off** — suites run in full.")
if skip_all_suites:
if skip_all_suites and not force_full:
lines.append("")
lines.append(
"**All suites skipped**: nothing was recommended and every PR "
@@ -119,10 +382,12 @@ def write_github_summary(
f"{len(unbucketed)} match no suite pool."
)
lines.append("")
lines.append("| Suite | Registered files | Bucketed files | Action |")
lines.append("| Suite | Registered | Bucketed | Action |")
lines.append("|---|---|---|---|")
for suite, files in buckets.items():
if skip_all_suites or (
if suite in force_full:
action = "run full suite (safety gate)"
elif skip_all_suites or (
precision_mode and suite_pool_sizes[suite] and not files
):
action = "skipped (no recommended test)"
@@ -133,6 +398,13 @@ def write_github_summary(
lines.append(
f"| {suite} | {suite_pool_sizes[suite]} | {len(files)} | {action} |"
)
if baseline_gaps:
lines.append("")
lines.append("**Full suite forced — baseline gaps**")
lines.append("")
for suite, missing in baseline_gaps.items():
listed = ", ".join(f"`{item}`" for item in missing)
lines.append(f"- `{suite}`: {listed}")
if raw_entries:
lines.append("")
lines.append(
@@ -154,14 +426,14 @@ def write_github_summary(
lines.append("```")
lines.append("")
lines.append(
"Not registered in any of the 9 bucketed a3 suites — they may still "
"run elsewhere (e.g. base-a runs its full suite; nightly tests run "
"in nightly pipelines)."
f"Not registered in any of the {len(buckets)} bucketed a3 suites — "
"they may still run elsewhere (e.g. base-a runs its full suite; "
"nightly tests run in nightly pipelines)."
)
lines.append("</details>")
if any(buckets.values()):
lines.append("")
lines.append("<details><summary>Bucketed test files per suite</summary>")
lines.append("<details><summary>Bucketed tests per suite</summary>")
lines.append("")
for suite, files in buckets.items():
lines.append(f"**{suite} ({len(files)})**")
@@ -191,6 +463,19 @@ def main():
required=True,
help="Comma-separated suite names to bucket (e.g. base-a-test-npu,base-b-test-1-npu-a3)",
)
parser.add_argument(
"--diffusion-suites",
type=str,
default="",
help="Comma-separated job:suite pairs for multimodal_gen suites "
"(e.g. multimodal-gen-test-1-npu-a3:1-npu,multimodal-gen-test-4-npu-a3:2-npu)",
)
parser.add_argument(
"--case-map",
type=str,
default="",
help="test_case_map.json used as the multimodal safety gate",
)
parser.add_argument(
"--precision-mode",
type=str,
@@ -259,13 +544,17 @@ def main():
and bool(changed_files)
and all(p.startswith("python/sglang/") for p in changed_files)
)
if skip_all_suites:
print(
"::notice::No tests recommended and every changed file is under "
"python/sglang/ — skipping all suites."
)
suites = [s.strip() for s in args.suites.split(",") if s.strip()]
diffusion_suites = {}
for item in args.diffusion_suites.split(","):
item = item.strip()
if not item:
continue
job, sep, suite = item.partition(":")
if not sep or not job or not suite:
parser.error(f"--diffusion-suites entry must be job:suite, got {item!r}")
diffusion_suites[job] = suite
work_dir = Path(tempfile.mkdtemp(prefix="bucket_tests_"))
buckets = {}
@@ -274,6 +563,12 @@ def main():
# the suite from the PR gate — so the output loop falls back to a
# full-suite run in that case.
suite_pool_sizes = {}
# Suites that must run in full even when precision mode is on (safety-gate
# failures, unreadable configs). A forced suite also opts out of the
# skip-all rule: an incomplete baseline cannot prove it is unaffected.
force_full = set()
# job -> tests absent from the coverage baseline (reported in the summary).
baseline_gaps = {}
all_pool = set()
for suite in suites:
suite_files = {normalize_rel(f) for f in enumerate_suite_files(suite, work_dir)}
@@ -287,7 +582,66 @@ def main():
f"Suite {suite}: {len(suite_files)} files, {len(intersection)} recommended"
)
if diffusion_suites:
map_keys = (
load_multimodal_map_keys(Path(args.case_map)) if args.case_map else None
)
if map_keys is None:
force_full |= set(diffusion_suites)
try:
diffusion_pools = collect_diffusion_pools()
except Exception as exc:
print(f"::warning::Cannot enumerate diffusion suites: {exc!r}")
diffusion_pools = {}
force_full |= set(diffusion_suites)
for job, suite in diffusion_suites.items():
pool = diffusion_pools.get(suite)
if pool is None:
print(
f"::warning::Diffusion suite {suite} ({job}) not found in "
f"{DIFFUSION_CASE_CONFIG.name}"
)
buckets[job] = []
suite_pool_sizes[job] = 0
force_full.add(job)
continue
suite_pool_sizes[job] = pool.size
all_pool |= pool.files()
if map_keys is None:
buckets[job] = []
print(
f"Suite {job} (diffusion {suite}): {pool.size} cases, "
"forced full suite by the safety gate"
)
continue
missing = missing_from_baseline(pool, map_keys)
if missing:
buckets[job] = []
force_full.add(job)
baseline_gaps[job] = missing
print(
f"::warning::Diffusion suite {suite} ({job}): {len(missing)} "
f"test(s) missing from the coverage baseline "
f"({', '.join(missing)}); running the full suite. "
"Rebuild the baseline to restore precision."
)
continue
selected_cases, selected_standalones = match_diffusion_pool(
pool, raw_entries
)
tokens = diffusion_tokens(selected_cases, selected_standalones)
buckets[job] = tokens
print(
f"Suite {job} (diffusion {suite}): {pool.size} cases, "
f"{len(tokens)} recommended"
)
total_count = sum(len(b) for b in buckets.values())
if skip_all_suites and not force_full:
print(
"::notice::No tests recommended and every changed file is under "
"python/sglang/ — skipping all suites."
)
precision_mode = args.precision_mode == "true" and total_count > 0
print(f"total_count={total_count} precision_mode={precision_mode}")
@@ -315,11 +669,17 @@ def main():
# 'true' so the job runs the full suite instead of silently
# vanishing. Anything else ('true', or a missing output from a
# failed step) means full-suite run.
# A force_full suite never takes either skip path: its baseline
# is incomplete, so "nothing recommended" proves nothing.
should_run = (
"false"
if (
skip_all_suites
or (precision_mode and suite_pool_sizes[suite] and not files)
not files
and suite not in force_full
and (
skip_all_suites
or (precision_mode and suite_pool_sizes[suite])
)
)
else "true"
)
@@ -333,6 +693,8 @@ def main():
all_pool,
buckets,
suite_pool_sizes,
force_full,
baseline_gaps,
precision_mode,
skip_all_suites,
)
+14 -4
View File
@@ -79,13 +79,18 @@ def _get_test_files_from_pr_diff(diff_file: str) -> list[str]:
print(f" Warning: Failed to read diff file for test file detection: {e}")
return test_files_found
# Pattern to match test file paths: test/registered/ directory
# Pattern to match test file paths: test/registered/ directory and the
# multimodal_gen diffusion tests.
# In diff output:
# - +++ b/test/registered/unit/xxx/test_xxx.py (new/modified test file)
# - rename to test/registered/unit/xxx/test_xxx.py (renamed test file)
# Test files must be in test/ directory and start with test_
test_file_pattern = re.compile(
r"^(?:\+\+\+ [ab]/|rename to )((?:test/registered(?:/.+)?/test_\w+\.py|test/(?:unit|e2e|integration)(?:/.+)?/test_\w+\.py))",
r"^(?:\+\+\+ [ab]/|rename to )("
r"test/registered(?:/.+)?/test_\w+\.py"
r"|test/(?:unit|e2e|integration)(?:/.+)?/test_\w+\.py"
r"|python/sglang/multimodal_gen/test/(?:.*/)?test_\w+\.py"
r")",
re.MULTILINE,
)
@@ -130,10 +135,15 @@ def _get_deleted_test_files_from_pr(diff_file: str, test_case_map: dict) -> list
print(f" Warning: Failed to read diff file for deleted test detection: {e}")
return deleted_test_files
# Pattern to match deleted test files (test/registered/ directory)
# Pattern to match deleted test files (test/registered/ directory and
# multimodal_gen diffusion tests)
# Match --- a/test/... followed by +++ /dev/null (deleted file marker)
deleted_pattern = re.compile(
r"^--- a/(test/registered(?:/.+)?/test_\w+\.py|test/(?:unit|e2e|integration)(?:/.+)?/test_\w+\.py)\s*\n\s*\+\+\+ [ab]?/dev/null",
r"^--- a/("
r"test/registered(?:/.+)?/test_\w+\.py"
r"|test/(?:unit|e2e|integration)(?:/.+)?/test_\w+\.py"
r"|python/sglang/multimodal_gen/test/(?:.*/)?test_\w+\.py"
r")\s*\n\s*\+\+\+ [ab]?/dev/null",
re.MULTILINE,
)