[CI] [NPU] precise-test: recommend multimodal_gen cases at test-case granularity (#45955)
This commit is contained in:
@@ -639,6 +639,10 @@ jobs:
|
||||
only_tests_base_c_test_perf_2_npu_a3: ${{ steps.bucket-tests.outputs.only_tests_base_c_test_perf_2_npu_a3 }}
|
||||
should_run_base_c_test_perf_16_npu_a3: ${{ steps.bucket-tests.outputs.should_run_base_c_test_perf_16_npu_a3 }}
|
||||
only_tests_base_c_test_perf_16_npu_a3: ${{ steps.bucket-tests.outputs.only_tests_base_c_test_perf_16_npu_a3 }}
|
||||
should_run_multimodal_gen_test_1_npu_a3: ${{ steps.bucket-tests.outputs.should_run_multimodal_gen_test_1_npu_a3 }}
|
||||
only_tests_multimodal_gen_test_1_npu_a3: ${{ steps.bucket-tests.outputs.only_tests_multimodal_gen_test_1_npu_a3 }}
|
||||
should_run_multimodal_gen_test_4_npu_a3: ${{ steps.bucket-tests.outputs.should_run_multimodal_gen_test_4_npu_a3 }}
|
||||
only_tests_multimodal_gen_test_4_npu_a3: ${{ steps.bucket-tests.outputs.only_tests_multimodal_gen_test_4_npu_a3 }}
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
@@ -654,10 +658,13 @@ jobs:
|
||||
with:
|
||||
# test/registered + python/sglang/test/ci are needed by
|
||||
# list_tests.py to enumerate each suite's test files for bucketing.
|
||||
# The diffusion case config is read (AST only) by
|
||||
# bucket_recommended_tests.py to enumerate multimodal cases.
|
||||
sparse-checkout: |
|
||||
scripts
|
||||
test/registered
|
||||
python/sglang/test/ci
|
||||
python/sglang/multimodal_gen/test/server/ascend/testcase_configs_npu.py
|
||||
sparse-checkout-cone-mode: false
|
||||
|
||||
- name: Download test case map
|
||||
@@ -731,6 +738,8 @@ jobs:
|
||||
--recommended scripts/ci/npu/precise-test/recommended_pytest_paths.txt \
|
||||
--changed-files scripts/ci/npu/precise-test/pr_changed_files.txt \
|
||||
--suites "base-b-test-1-npu-a3,base-b-test-2-npu-a3,base-b-test-4-npu-a3,base-b-test-8-npu-a3,base-b-test-16-npu-a3,base-c-test-acc-2-npu-a3,base-c-test-acc-16-npu-a3,base-c-test-perf-2-npu-a3,base-c-test-perf-16-npu-a3" \
|
||||
--diffusion-suites "multimodal-gen-test-1-npu-a3:1-npu,multimodal-gen-test-4-npu-a3:2-npu" \
|
||||
--case-map "$PWD/test_case_map.json" \
|
||||
--precision-mode "${PRECISION_MODE}" \
|
||||
--github-output "${GITHUB_OUTPUT}" \
|
||||
--github-summary "${GITHUB_STEP_SUMMARY:-}" \
|
||||
|
||||
@@ -16,12 +16,36 @@ step outputs:
|
||||
suite's bucket is empty -> the caller skips the whole
|
||||
job; also 'false' for ALL suites when nothing was
|
||||
recommended and every PR change is under python/sglang/
|
||||
(--changed-files); 'true' otherwise (incl. full suites)
|
||||
(--changed-files); 'true' otherwise (incl. full suites).
|
||||
A suite forced full by the multimodal safety gate stays
|
||||
'true' regardless of both skip paths.
|
||||
|
||||
Suite keys replace '-' with '_' because GitHub Actions output names must be
|
||||
valid shell identifiers. Any failure path yields empty/missing outputs, which
|
||||
callers treat as "run the full suite".
|
||||
|
||||
Diffusion (multimodal_gen) suites are enumerated from the NPU diffusion case
|
||||
config instead of test/registered, and they recommend at test-case
|
||||
granularity: one file holds several expensive cases, so a file-level list
|
||||
would be equivalent to running the whole suite. ``--diffusion-suites`` takes
|
||||
``job-name:suite`` pairs, e.g.
|
||||
``multimodal-gen-test-1-npu-a3:1-npu,multimodal-gen-test-4-npu-a3:2-npu``.
|
||||
Their ``only_tests_<key>`` lines are pytest ``-k`` tokens rather than paths:
|
||||
|
||||
[<case_id>] parametrized diffusion case
|
||||
<standalone file stem> standalone diffusion test file
|
||||
|
||||
Safety gate: if the coverage baseline (``--case-map``) has no multimodal_gen
|
||||
entry, the diffusion buckets are forced to full-suite. Without that gate a
|
||||
baseline built before multimodal collection existed would make every
|
||||
multimodal bucket empty, and precision mode would silently skip the diffusion
|
||||
cases. The same full-suite fallback applies per suite when the baseline is
|
||||
missing any of that suite's cases or standalone files: a newly added case has
|
||||
no coverage-map key by definition, so no PR could ever recommend it until the
|
||||
baseline is rebuilt. A forced suite ignores BOTH the precision-mode skip and
|
||||
the skip-all rule below — "the baseline cannot judge this suite" is not the
|
||||
same as "no test is affected".
|
||||
|
||||
When --github-summary points at $GITHUB_STEP_SUMMARY, a markdown section is
|
||||
appended with the recommended test cases, counts, per-suite buckets, and the
|
||||
files that matched no suite pool (not run by the bucketed a3 suites).
|
||||
@@ -31,7 +55,10 @@ upload, so the report is downloadable from the run page).
|
||||
Usage:
|
||||
python3 bucket_recommended_tests.py \
|
||||
--recommended recommended_pytest_paths.txt \
|
||||
--changed-files pr_changed_files.txt \
|
||||
--suites base-a-test-npu,base-b-test-1-npu-a3 \
|
||||
--diffusion-suites multimodal-gen-test-1-npu-a3:1-npu,multimodal-gen-test-4-npu-a3:2-npu \
|
||||
--case-map test_case_map.json \
|
||||
--precision-mode true \
|
||||
--github-output "$GITHUB_OUTPUT" \
|
||||
--github-summary "$GITHUB_STEP_SUMMARY" \
|
||||
@@ -39,10 +66,15 @@ Usage:
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import ast
|
||||
import json
|
||||
import os
|
||||
import posixpath
|
||||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
|
||||
# Repo layout: this script lives at <repo>/scripts/ci/npu/precise-test/.
|
||||
@@ -50,6 +82,29 @@ SCRIPT_DIR = Path(__file__).resolve().parent
|
||||
LIST_TESTS = SCRIPT_DIR / "list_tests.py"
|
||||
REPO_ROOT = SCRIPT_DIR.parents[3]
|
||||
|
||||
DIFFUSION_UTILS_DIR = REPO_ROOT / "scripts" / "ci" / "utils" / "diffusion"
|
||||
# Any coverage-map key under this prefix proves the baseline includes
|
||||
# multimodal_gen cases (see the safety gate above).
|
||||
DIFFUSION_TEST_PREFIX = "python/sglang/multimodal_gen/test/"
|
||||
# Standalone files in that config are relative to the server/ test root.
|
||||
DIFFUSION_TEST_ROOT_REL = DIFFUSION_TEST_PREFIX + "server"
|
||||
DIFFUSION_CASE_CONFIG = (
|
||||
REPO_ROOT / DIFFUSION_TEST_ROOT_REL / "ascend/testcase_configs_npu.py"
|
||||
)
|
||||
|
||||
# diffusion_case_parser reads this at import time to resolve ONE_NPU_CASES /
|
||||
# TWO_NPU_CASES instead of the GPU case lists.
|
||||
os.environ["USE_NPU_CONFIGS"] = "true"
|
||||
sys.path.insert(0, str(DIFFUSION_UTILS_DIR))
|
||||
|
||||
from diffusion_case_parser import ( # noqa: E402
|
||||
parse_standalone_data,
|
||||
parse_testcase_configs,
|
||||
)
|
||||
|
||||
# Trailing "[case_id]" of a parametrized pytest node id.
|
||||
_CASE_ID_RE = re.compile(r"\[([^\[\]]+)\]$")
|
||||
|
||||
|
||||
def normalize_rel(path: str) -> str:
|
||||
"""Normalize a pytest path entry to a repo-relative posix file path.
|
||||
@@ -84,6 +139,212 @@ def enumerate_suite_files(suite: str, work_dir: Path) -> list:
|
||||
return [line.strip() for line in f if line.strip()]
|
||||
|
||||
|
||||
@dataclass
|
||||
class DiffusionPool:
|
||||
"""Recommendation pool of one diffusion suite."""
|
||||
|
||||
cases_by_file: dict # repo-relative test file -> [case_id, ...]
|
||||
standalone_files: list # repo-relative standalone test files
|
||||
|
||||
@property
|
||||
def size(self) -> int:
|
||||
return sum(len(ids) for ids in self.cases_by_file.values()) + len(
|
||||
self.standalone_files
|
||||
)
|
||||
|
||||
@property
|
||||
def case_ids(self) -> list:
|
||||
return [cid for ids in self.cases_by_file.values() for cid in ids]
|
||||
|
||||
def files(self) -> set:
|
||||
return set(self.cases_by_file) | set(self.standalone_files)
|
||||
|
||||
|
||||
def _diffusion_rel(file_rel: str) -> str:
|
||||
"""Resolve a config-relative test file to a repo-relative posix path."""
|
||||
return posixpath.normpath(
|
||||
posixpath.join(DIFFUSION_TEST_ROOT_REL, file_rel.replace("\\", "/"))
|
||||
)
|
||||
|
||||
|
||||
def _parse_parametrized_groups(case_config: Path) -> dict:
|
||||
"""Parse PARAMETRIZED_CASE_GROUPS into suite -> [(file, case list var)].
|
||||
|
||||
diffusion_case_parser reports case ids and standalone files but not which
|
||||
file a case lives in, which is needed to expand file-level
|
||||
recommendations. The mapping dict is parsed here with AST only, so this
|
||||
stays import-free of torch.
|
||||
"""
|
||||
tree = ast.parse(case_config.read_text(encoding="utf-8"), filename=str(case_config))
|
||||
groups = {}
|
||||
for node in ast.walk(tree):
|
||||
if not isinstance(node, ast.Assign):
|
||||
continue
|
||||
targets = {t.id for t in node.targets if isinstance(t, ast.Name)}
|
||||
if "PARAMETRIZED_CASE_GROUPS" not in targets or not isinstance(
|
||||
node.value, ast.Dict
|
||||
):
|
||||
continue
|
||||
for key, value in zip(node.value.keys, node.value.values):
|
||||
if not (isinstance(key, ast.Constant) and isinstance(key.value, str)):
|
||||
continue
|
||||
if not isinstance(value, ast.List):
|
||||
continue
|
||||
entries = []
|
||||
for elt in value.elts:
|
||||
if not isinstance(elt, ast.Tuple) or len(elt.elts) != 2:
|
||||
continue
|
||||
file_node, cases_node = elt.elts
|
||||
if isinstance(file_node, ast.Constant) and isinstance(
|
||||
cases_node, ast.Name
|
||||
):
|
||||
entries.append((file_node.value, cases_node.id))
|
||||
groups[key.value] = entries
|
||||
return groups
|
||||
|
||||
|
||||
def collect_diffusion_pools() -> dict:
|
||||
"""Enumerate the NPU diffusion suites and their recommendation pools."""
|
||||
case_lists = parse_testcase_configs(DIFFUSION_CASE_CONFIG)
|
||||
standalone_files, _ = parse_standalone_data(DIFFUSION_CASE_CONFIG)
|
||||
parametrized_groups = _parse_parametrized_groups(DIFFUSION_CASE_CONFIG)
|
||||
# A suite may have only standalone files, so enumerate the union instead
|
||||
# of the parametrized groups alone (a missing pool means "run full").
|
||||
pools = {}
|
||||
for suite in sorted(set(parametrized_groups) | set(standalone_files)):
|
||||
cases_by_file = {}
|
||||
for file_rel, list_name in parametrized_groups.get(suite, []):
|
||||
rel = _diffusion_rel(file_rel)
|
||||
case_ids = cases_by_file.setdefault(rel, [])
|
||||
for case_id in case_lists.get(list_name, []):
|
||||
if case_id not in case_ids:
|
||||
case_ids.append(case_id)
|
||||
pools[suite] = DiffusionPool(
|
||||
cases_by_file=cases_by_file,
|
||||
standalone_files=[
|
||||
_diffusion_rel(f) for f in standalone_files.get(suite, [])
|
||||
],
|
||||
)
|
||||
_warn_standalone_token_collisions(pools)
|
||||
return pools
|
||||
|
||||
|
||||
def _warn_standalone_token_collisions(pools: dict) -> None:
|
||||
"""Warn when a standalone -k token is not unique inside the diffusion run.
|
||||
|
||||
Standalone files are selected by pytest module-name keyword, so two files
|
||||
sharing a stem (or a stem that is a substring of a case id) would make one
|
||||
token select more than intended. Over-running is harmless, hence a warning
|
||||
instead of an error.
|
||||
"""
|
||||
stems = {}
|
||||
case_ids = set()
|
||||
for suite, pool in pools.items():
|
||||
case_ids.update(pool.case_ids)
|
||||
for rel in pool.standalone_files:
|
||||
stems.setdefault(Path(rel).stem, []).append(f"{suite}:{rel}")
|
||||
for stem, owners in sorted(stems.items()):
|
||||
if len(owners) > 1:
|
||||
print(
|
||||
f"::warning::Standalone -k token {stem!r} covers multiple files: "
|
||||
f"{', '.join(owners)}"
|
||||
)
|
||||
hits = sorted(case_id for case_id in case_ids if stem in case_id)
|
||||
if hits:
|
||||
print(
|
||||
f"::warning::Standalone -k token {stem!r} also matches case "
|
||||
f"id(s): {', '.join(hits)}"
|
||||
)
|
||||
|
||||
|
||||
def match_diffusion_pool(pool: DiffusionPool, raw_entries: list) -> tuple:
|
||||
"""Match recommended entries against one diffusion pool.
|
||||
|
||||
A file-level entry (a changed test file, or a whole-file coverage target)
|
||||
selects every case of that file; a case-level entry selects just that
|
||||
case. Standalone files are selected as whole files.
|
||||
"""
|
||||
selected_cases = []
|
||||
selected_standalones = []
|
||||
for entry in raw_entries:
|
||||
rel = normalize_rel(entry)
|
||||
_, _, node = entry.strip().replace("\\", "/").partition("::")
|
||||
if rel in pool.standalone_files:
|
||||
if rel not in selected_standalones:
|
||||
selected_standalones.append(rel)
|
||||
continue
|
||||
case_ids = pool.cases_by_file.get(rel)
|
||||
if case_ids is None:
|
||||
continue
|
||||
match = _CASE_ID_RE.search(node)
|
||||
if match:
|
||||
if match.group(1) in case_ids and match.group(1) not in selected_cases:
|
||||
selected_cases.append(match.group(1))
|
||||
else:
|
||||
for case_id in case_ids:
|
||||
if case_id not in selected_cases:
|
||||
selected_cases.append(case_id)
|
||||
return selected_cases, selected_standalones
|
||||
|
||||
|
||||
def diffusion_tokens(selected_cases: list, selected_standalones: list) -> list:
|
||||
"""Render a diffusion bucket as pytest -k tokens.
|
||||
|
||||
Bracket the case id so ``[x]`` cannot match ``[x_v2]``. Standalone files
|
||||
are matched by module-name keyword (the runner applies -k to pytest, whose
|
||||
keywords include the module name).
|
||||
"""
|
||||
tokens = [f"[{case_id}]" for case_id in selected_cases]
|
||||
tokens.extend(Path(rel).stem for rel in selected_standalones)
|
||||
return tokens
|
||||
|
||||
|
||||
def load_multimodal_map_keys(case_map: Path):
|
||||
"""Return the coverage baseline's multimodal_gen keys, or None.
|
||||
|
||||
None means "the baseline cannot be trusted for multimodal" (missing,
|
||||
unreadable, or built before multimodal collection existed); callers then
|
||||
run the diffusion suites in full.
|
||||
"""
|
||||
if not case_map.is_file():
|
||||
print(f"::warning::Test case map not found: {case_map}")
|
||||
return None
|
||||
try:
|
||||
with open(case_map, encoding="utf-8") as f:
|
||||
data = json.load(f)
|
||||
except (OSError, ValueError) as exc:
|
||||
print(f"::warning::Cannot read test case map {case_map}: {exc}")
|
||||
return None
|
||||
if not isinstance(data, dict):
|
||||
print(f"::warning::Unexpected test case map format: {case_map}")
|
||||
return None
|
||||
keys = {key for key in data if key.startswith(DIFFUSION_TEST_PREFIX)}
|
||||
if not keys:
|
||||
print(
|
||||
"::warning::Test case map has no multimodal_gen entries; the diffusion "
|
||||
"suites must run in full. Rebuild the coverage baseline."
|
||||
)
|
||||
return None
|
||||
return keys
|
||||
|
||||
|
||||
def missing_from_baseline(pool: DiffusionPool, map_keys: set) -> list:
|
||||
"""Pool entries that the coverage baseline never collected.
|
||||
|
||||
A case or standalone file added after the last coverage run has no map
|
||||
key, so no PR diff can ever recommend it. Such a suite must run in full
|
||||
until the baseline is rebuilt.
|
||||
"""
|
||||
covered_cases = set()
|
||||
for key in map_keys:
|
||||
match = _CASE_ID_RE.search(key)
|
||||
if match:
|
||||
covered_cases.add(match.group(1))
|
||||
missing = [cid for cid in pool.case_ids if cid not in covered_cases]
|
||||
missing.extend(f for f in pool.standalone_files if f not in map_keys)
|
||||
return missing
|
||||
|
||||
|
||||
def write_github_summary(
|
||||
target_files: list,
|
||||
raw_entries: list,
|
||||
@@ -91,6 +352,8 @@ def write_github_summary(
|
||||
all_pool: set,
|
||||
buckets: dict,
|
||||
suite_pool_sizes: dict,
|
||||
force_full: set,
|
||||
baseline_gaps: dict,
|
||||
precision_mode: bool,
|
||||
skip_all_suites: bool = False,
|
||||
) -> None:
|
||||
@@ -106,7 +369,7 @@ def write_github_summary(
|
||||
)
|
||||
else:
|
||||
lines.append("Precision mode **off** — suites run in full.")
|
||||
if skip_all_suites:
|
||||
if skip_all_suites and not force_full:
|
||||
lines.append("")
|
||||
lines.append(
|
||||
"**All suites skipped**: nothing was recommended and every PR "
|
||||
@@ -119,10 +382,12 @@ def write_github_summary(
|
||||
f"{len(unbucketed)} match no suite pool."
|
||||
)
|
||||
lines.append("")
|
||||
lines.append("| Suite | Registered files | Bucketed files | Action |")
|
||||
lines.append("| Suite | Registered | Bucketed | Action |")
|
||||
lines.append("|---|---|---|---|")
|
||||
for suite, files in buckets.items():
|
||||
if skip_all_suites or (
|
||||
if suite in force_full:
|
||||
action = "run full suite (safety gate)"
|
||||
elif skip_all_suites or (
|
||||
precision_mode and suite_pool_sizes[suite] and not files
|
||||
):
|
||||
action = "skipped (no recommended test)"
|
||||
@@ -133,6 +398,13 @@ def write_github_summary(
|
||||
lines.append(
|
||||
f"| {suite} | {suite_pool_sizes[suite]} | {len(files)} | {action} |"
|
||||
)
|
||||
if baseline_gaps:
|
||||
lines.append("")
|
||||
lines.append("**Full suite forced — baseline gaps**")
|
||||
lines.append("")
|
||||
for suite, missing in baseline_gaps.items():
|
||||
listed = ", ".join(f"`{item}`" for item in missing)
|
||||
lines.append(f"- `{suite}`: {listed}")
|
||||
if raw_entries:
|
||||
lines.append("")
|
||||
lines.append(
|
||||
@@ -154,14 +426,14 @@ def write_github_summary(
|
||||
lines.append("```")
|
||||
lines.append("")
|
||||
lines.append(
|
||||
"Not registered in any of the 9 bucketed a3 suites — they may still "
|
||||
"run elsewhere (e.g. base-a runs its full suite; nightly tests run "
|
||||
"in nightly pipelines)."
|
||||
f"Not registered in any of the {len(buckets)} bucketed a3 suites — "
|
||||
"they may still run elsewhere (e.g. base-a runs its full suite; "
|
||||
"nightly tests run in nightly pipelines)."
|
||||
)
|
||||
lines.append("</details>")
|
||||
if any(buckets.values()):
|
||||
lines.append("")
|
||||
lines.append("<details><summary>Bucketed test files per suite</summary>")
|
||||
lines.append("<details><summary>Bucketed tests per suite</summary>")
|
||||
lines.append("")
|
||||
for suite, files in buckets.items():
|
||||
lines.append(f"**{suite} ({len(files)})**")
|
||||
@@ -191,6 +463,19 @@ def main():
|
||||
required=True,
|
||||
help="Comma-separated suite names to bucket (e.g. base-a-test-npu,base-b-test-1-npu-a3)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--diffusion-suites",
|
||||
type=str,
|
||||
default="",
|
||||
help="Comma-separated job:suite pairs for multimodal_gen suites "
|
||||
"(e.g. multimodal-gen-test-1-npu-a3:1-npu,multimodal-gen-test-4-npu-a3:2-npu)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--case-map",
|
||||
type=str,
|
||||
default="",
|
||||
help="test_case_map.json used as the multimodal safety gate",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--precision-mode",
|
||||
type=str,
|
||||
@@ -259,13 +544,17 @@ def main():
|
||||
and bool(changed_files)
|
||||
and all(p.startswith("python/sglang/") for p in changed_files)
|
||||
)
|
||||
if skip_all_suites:
|
||||
print(
|
||||
"::notice::No tests recommended and every changed file is under "
|
||||
"python/sglang/ — skipping all suites."
|
||||
)
|
||||
|
||||
suites = [s.strip() for s in args.suites.split(",") if s.strip()]
|
||||
diffusion_suites = {}
|
||||
for item in args.diffusion_suites.split(","):
|
||||
item = item.strip()
|
||||
if not item:
|
||||
continue
|
||||
job, sep, suite = item.partition(":")
|
||||
if not sep or not job or not suite:
|
||||
parser.error(f"--diffusion-suites entry must be job:suite, got {item!r}")
|
||||
diffusion_suites[job] = suite
|
||||
work_dir = Path(tempfile.mkdtemp(prefix="bucket_tests_"))
|
||||
|
||||
buckets = {}
|
||||
@@ -274,6 +563,12 @@ def main():
|
||||
# the suite from the PR gate — so the output loop falls back to a
|
||||
# full-suite run in that case.
|
||||
suite_pool_sizes = {}
|
||||
# Suites that must run in full even when precision mode is on (safety-gate
|
||||
# failures, unreadable configs). A forced suite also opts out of the
|
||||
# skip-all rule: an incomplete baseline cannot prove it is unaffected.
|
||||
force_full = set()
|
||||
# job -> tests absent from the coverage baseline (reported in the summary).
|
||||
baseline_gaps = {}
|
||||
all_pool = set()
|
||||
for suite in suites:
|
||||
suite_files = {normalize_rel(f) for f in enumerate_suite_files(suite, work_dir)}
|
||||
@@ -287,7 +582,66 @@ def main():
|
||||
f"Suite {suite}: {len(suite_files)} files, {len(intersection)} recommended"
|
||||
)
|
||||
|
||||
if diffusion_suites:
|
||||
map_keys = (
|
||||
load_multimodal_map_keys(Path(args.case_map)) if args.case_map else None
|
||||
)
|
||||
if map_keys is None:
|
||||
force_full |= set(diffusion_suites)
|
||||
try:
|
||||
diffusion_pools = collect_diffusion_pools()
|
||||
except Exception as exc:
|
||||
print(f"::warning::Cannot enumerate diffusion suites: {exc!r}")
|
||||
diffusion_pools = {}
|
||||
force_full |= set(diffusion_suites)
|
||||
for job, suite in diffusion_suites.items():
|
||||
pool = diffusion_pools.get(suite)
|
||||
if pool is None:
|
||||
print(
|
||||
f"::warning::Diffusion suite {suite} ({job}) not found in "
|
||||
f"{DIFFUSION_CASE_CONFIG.name}"
|
||||
)
|
||||
buckets[job] = []
|
||||
suite_pool_sizes[job] = 0
|
||||
force_full.add(job)
|
||||
continue
|
||||
suite_pool_sizes[job] = pool.size
|
||||
all_pool |= pool.files()
|
||||
if map_keys is None:
|
||||
buckets[job] = []
|
||||
print(
|
||||
f"Suite {job} (diffusion {suite}): {pool.size} cases, "
|
||||
"forced full suite by the safety gate"
|
||||
)
|
||||
continue
|
||||
missing = missing_from_baseline(pool, map_keys)
|
||||
if missing:
|
||||
buckets[job] = []
|
||||
force_full.add(job)
|
||||
baseline_gaps[job] = missing
|
||||
print(
|
||||
f"::warning::Diffusion suite {suite} ({job}): {len(missing)} "
|
||||
f"test(s) missing from the coverage baseline "
|
||||
f"({', '.join(missing)}); running the full suite. "
|
||||
"Rebuild the baseline to restore precision."
|
||||
)
|
||||
continue
|
||||
selected_cases, selected_standalones = match_diffusion_pool(
|
||||
pool, raw_entries
|
||||
)
|
||||
tokens = diffusion_tokens(selected_cases, selected_standalones)
|
||||
buckets[job] = tokens
|
||||
print(
|
||||
f"Suite {job} (diffusion {suite}): {pool.size} cases, "
|
||||
f"{len(tokens)} recommended"
|
||||
)
|
||||
|
||||
total_count = sum(len(b) for b in buckets.values())
|
||||
if skip_all_suites and not force_full:
|
||||
print(
|
||||
"::notice::No tests recommended and every changed file is under "
|
||||
"python/sglang/ — skipping all suites."
|
||||
)
|
||||
precision_mode = args.precision_mode == "true" and total_count > 0
|
||||
print(f"total_count={total_count} precision_mode={precision_mode}")
|
||||
|
||||
@@ -315,11 +669,17 @@ def main():
|
||||
# 'true' so the job runs the full suite instead of silently
|
||||
# vanishing. Anything else ('true', or a missing output from a
|
||||
# failed step) means full-suite run.
|
||||
# A force_full suite never takes either skip path: its baseline
|
||||
# is incomplete, so "nothing recommended" proves nothing.
|
||||
should_run = (
|
||||
"false"
|
||||
if (
|
||||
skip_all_suites
|
||||
or (precision_mode and suite_pool_sizes[suite] and not files)
|
||||
not files
|
||||
and suite not in force_full
|
||||
and (
|
||||
skip_all_suites
|
||||
or (precision_mode and suite_pool_sizes[suite])
|
||||
)
|
||||
)
|
||||
else "true"
|
||||
)
|
||||
@@ -333,6 +693,8 @@ def main():
|
||||
all_pool,
|
||||
buckets,
|
||||
suite_pool_sizes,
|
||||
force_full,
|
||||
baseline_gaps,
|
||||
precision_mode,
|
||||
skip_all_suites,
|
||||
)
|
||||
|
||||
@@ -79,13 +79,18 @@ def _get_test_files_from_pr_diff(diff_file: str) -> list[str]:
|
||||
print(f" Warning: Failed to read diff file for test file detection: {e}")
|
||||
return test_files_found
|
||||
|
||||
# Pattern to match test file paths: test/registered/ directory
|
||||
# Pattern to match test file paths: test/registered/ directory and the
|
||||
# multimodal_gen diffusion tests.
|
||||
# In diff output:
|
||||
# - +++ b/test/registered/unit/xxx/test_xxx.py (new/modified test file)
|
||||
# - rename to test/registered/unit/xxx/test_xxx.py (renamed test file)
|
||||
# Test files must be in test/ directory and start with test_
|
||||
test_file_pattern = re.compile(
|
||||
r"^(?:\+\+\+ [ab]/|rename to )((?:test/registered(?:/.+)?/test_\w+\.py|test/(?:unit|e2e|integration)(?:/.+)?/test_\w+\.py))",
|
||||
r"^(?:\+\+\+ [ab]/|rename to )("
|
||||
r"test/registered(?:/.+)?/test_\w+\.py"
|
||||
r"|test/(?:unit|e2e|integration)(?:/.+)?/test_\w+\.py"
|
||||
r"|python/sglang/multimodal_gen/test/(?:.*/)?test_\w+\.py"
|
||||
r")",
|
||||
re.MULTILINE,
|
||||
)
|
||||
|
||||
@@ -130,10 +135,15 @@ def _get_deleted_test_files_from_pr(diff_file: str, test_case_map: dict) -> list
|
||||
print(f" Warning: Failed to read diff file for deleted test detection: {e}")
|
||||
return deleted_test_files
|
||||
|
||||
# Pattern to match deleted test files (test/registered/ directory)
|
||||
# Pattern to match deleted test files (test/registered/ directory and
|
||||
# multimodal_gen diffusion tests)
|
||||
# Match --- a/test/... followed by +++ /dev/null (deleted file marker)
|
||||
deleted_pattern = re.compile(
|
||||
r"^--- a/(test/registered(?:/.+)?/test_\w+\.py|test/(?:unit|e2e|integration)(?:/.+)?/test_\w+\.py)\s*\n\s*\+\+\+ [ab]?/dev/null",
|
||||
r"^--- a/("
|
||||
r"test/registered(?:/.+)?/test_\w+\.py"
|
||||
r"|test/(?:unit|e2e|integration)(?:/.+)?/test_\w+\.py"
|
||||
r"|python/sglang/multimodal_gen/test/(?:.*/)?test_\w+\.py"
|
||||
r")\s*\n\s*\+\+\+ [ab]?/dev/null",
|
||||
re.MULTILINE,
|
||||
)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user