From 472208fec00d7023db7f2039c64056d746de96a8 Mon Sep 17 00:00:00 2001 From: shun8686 Date: Fri, 23 Oct 2026 15:57:31 +0800 Subject: [PATCH] [CI] [NPU] precise-test: recommend multimodal_gen cases at test-case granularity (#45955) --- .github/workflows/pr-test-npu.yml | 9 + .../precise-test/bucket_recommended_tests.py | 392 +++++++++++++++++- scripts/ci/npu/precise-test/test_selector.py | 18 +- 3 files changed, 400 insertions(+), 19 deletions(-) diff --git a/.github/workflows/pr-test-npu.yml b/.github/workflows/pr-test-npu.yml index 05010784c0..d5f19f197b 100644 --- a/.github/workflows/pr-test-npu.yml +++ b/.github/workflows/pr-test-npu.yml @@ -639,6 +639,10 @@ jobs: only_tests_base_c_test_perf_2_npu_a3: ${{ steps.bucket-tests.outputs.only_tests_base_c_test_perf_2_npu_a3 }} should_run_base_c_test_perf_16_npu_a3: ${{ steps.bucket-tests.outputs.should_run_base_c_test_perf_16_npu_a3 }} only_tests_base_c_test_perf_16_npu_a3: ${{ steps.bucket-tests.outputs.only_tests_base_c_test_perf_16_npu_a3 }} + should_run_multimodal_gen_test_1_npu_a3: ${{ steps.bucket-tests.outputs.should_run_multimodal_gen_test_1_npu_a3 }} + only_tests_multimodal_gen_test_1_npu_a3: ${{ steps.bucket-tests.outputs.only_tests_multimodal_gen_test_1_npu_a3 }} + should_run_multimodal_gen_test_4_npu_a3: ${{ steps.bucket-tests.outputs.should_run_multimodal_gen_test_4_npu_a3 }} + only_tests_multimodal_gen_test_4_npu_a3: ${{ steps.bucket-tests.outputs.only_tests_multimodal_gen_test_4_npu_a3 }} env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} @@ -654,10 +658,13 @@ jobs: with: # test/registered + python/sglang/test/ci are needed by # list_tests.py to enumerate each suite's test files for bucketing. + # The diffusion case config is read (AST only) by + # bucket_recommended_tests.py to enumerate multimodal cases. sparse-checkout: | scripts test/registered python/sglang/test/ci + python/sglang/multimodal_gen/test/server/ascend/testcase_configs_npu.py sparse-checkout-cone-mode: false - name: Download test case map @@ -731,6 +738,8 @@ jobs: --recommended scripts/ci/npu/precise-test/recommended_pytest_paths.txt \ --changed-files scripts/ci/npu/precise-test/pr_changed_files.txt \ --suites "base-b-test-1-npu-a3,base-b-test-2-npu-a3,base-b-test-4-npu-a3,base-b-test-8-npu-a3,base-b-test-16-npu-a3,base-c-test-acc-2-npu-a3,base-c-test-acc-16-npu-a3,base-c-test-perf-2-npu-a3,base-c-test-perf-16-npu-a3" \ + --diffusion-suites "multimodal-gen-test-1-npu-a3:1-npu,multimodal-gen-test-4-npu-a3:2-npu" \ + --case-map "$PWD/test_case_map.json" \ --precision-mode "${PRECISION_MODE}" \ --github-output "${GITHUB_OUTPUT}" \ --github-summary "${GITHUB_STEP_SUMMARY:-}" \ diff --git a/scripts/ci/npu/precise-test/bucket_recommended_tests.py b/scripts/ci/npu/precise-test/bucket_recommended_tests.py index a16e2e9fbe..e7de25f54a 100755 --- a/scripts/ci/npu/precise-test/bucket_recommended_tests.py +++ b/scripts/ci/npu/precise-test/bucket_recommended_tests.py @@ -16,12 +16,36 @@ step outputs: suite's bucket is empty -> the caller skips the whole job; also 'false' for ALL suites when nothing was recommended and every PR change is under python/sglang/ - (--changed-files); 'true' otherwise (incl. full suites) + (--changed-files); 'true' otherwise (incl. full suites). + A suite forced full by the multimodal safety gate stays + 'true' regardless of both skip paths. Suite keys replace '-' with '_' because GitHub Actions output names must be valid shell identifiers. Any failure path yields empty/missing outputs, which callers treat as "run the full suite". +Diffusion (multimodal_gen) suites are enumerated from the NPU diffusion case +config instead of test/registered, and they recommend at test-case +granularity: one file holds several expensive cases, so a file-level list +would be equivalent to running the whole suite. ``--diffusion-suites`` takes +``job-name:suite`` pairs, e.g. +``multimodal-gen-test-1-npu-a3:1-npu,multimodal-gen-test-4-npu-a3:2-npu``. +Their ``only_tests_`` lines are pytest ``-k`` tokens rather than paths: + + [] parametrized diffusion case + standalone diffusion test file + +Safety gate: if the coverage baseline (``--case-map``) has no multimodal_gen +entry, the diffusion buckets are forced to full-suite. Without that gate a +baseline built before multimodal collection existed would make every +multimodal bucket empty, and precision mode would silently skip the diffusion +cases. The same full-suite fallback applies per suite when the baseline is +missing any of that suite's cases or standalone files: a newly added case has +no coverage-map key by definition, so no PR could ever recommend it until the +baseline is rebuilt. A forced suite ignores BOTH the precision-mode skip and +the skip-all rule below — "the baseline cannot judge this suite" is not the +same as "no test is affected". + When --github-summary points at $GITHUB_STEP_SUMMARY, a markdown section is appended with the recommended test cases, counts, per-suite buckets, and the files that matched no suite pool (not run by the bucketed a3 suites). @@ -31,7 +55,10 @@ upload, so the report is downloadable from the run page). Usage: python3 bucket_recommended_tests.py \ --recommended recommended_pytest_paths.txt \ + --changed-files pr_changed_files.txt \ --suites base-a-test-npu,base-b-test-1-npu-a3 \ + --diffusion-suites multimodal-gen-test-1-npu-a3:1-npu,multimodal-gen-test-4-npu-a3:2-npu \ + --case-map test_case_map.json \ --precision-mode true \ --github-output "$GITHUB_OUTPUT" \ --github-summary "$GITHUB_STEP_SUMMARY" \ @@ -39,10 +66,15 @@ Usage: """ import argparse +import ast +import json import os +import posixpath +import re import subprocess import sys import tempfile +from dataclasses import dataclass from pathlib import Path # Repo layout: this script lives at /scripts/ci/npu/precise-test/. @@ -50,6 +82,29 @@ SCRIPT_DIR = Path(__file__).resolve().parent LIST_TESTS = SCRIPT_DIR / "list_tests.py" REPO_ROOT = SCRIPT_DIR.parents[3] +DIFFUSION_UTILS_DIR = REPO_ROOT / "scripts" / "ci" / "utils" / "diffusion" +# Any coverage-map key under this prefix proves the baseline includes +# multimodal_gen cases (see the safety gate above). +DIFFUSION_TEST_PREFIX = "python/sglang/multimodal_gen/test/" +# Standalone files in that config are relative to the server/ test root. +DIFFUSION_TEST_ROOT_REL = DIFFUSION_TEST_PREFIX + "server" +DIFFUSION_CASE_CONFIG = ( + REPO_ROOT / DIFFUSION_TEST_ROOT_REL / "ascend/testcase_configs_npu.py" +) + +# diffusion_case_parser reads this at import time to resolve ONE_NPU_CASES / +# TWO_NPU_CASES instead of the GPU case lists. +os.environ["USE_NPU_CONFIGS"] = "true" +sys.path.insert(0, str(DIFFUSION_UTILS_DIR)) + +from diffusion_case_parser import ( # noqa: E402 + parse_standalone_data, + parse_testcase_configs, +) + +# Trailing "[case_id]" of a parametrized pytest node id. +_CASE_ID_RE = re.compile(r"\[([^\[\]]+)\]$") + def normalize_rel(path: str) -> str: """Normalize a pytest path entry to a repo-relative posix file path. @@ -84,6 +139,212 @@ def enumerate_suite_files(suite: str, work_dir: Path) -> list: return [line.strip() for line in f if line.strip()] +@dataclass +class DiffusionPool: + """Recommendation pool of one diffusion suite.""" + + cases_by_file: dict # repo-relative test file -> [case_id, ...] + standalone_files: list # repo-relative standalone test files + + @property + def size(self) -> int: + return sum(len(ids) for ids in self.cases_by_file.values()) + len( + self.standalone_files + ) + + @property + def case_ids(self) -> list: + return [cid for ids in self.cases_by_file.values() for cid in ids] + + def files(self) -> set: + return set(self.cases_by_file) | set(self.standalone_files) + + +def _diffusion_rel(file_rel: str) -> str: + """Resolve a config-relative test file to a repo-relative posix path.""" + return posixpath.normpath( + posixpath.join(DIFFUSION_TEST_ROOT_REL, file_rel.replace("\\", "/")) + ) + + +def _parse_parametrized_groups(case_config: Path) -> dict: + """Parse PARAMETRIZED_CASE_GROUPS into suite -> [(file, case list var)]. + + diffusion_case_parser reports case ids and standalone files but not which + file a case lives in, which is needed to expand file-level + recommendations. The mapping dict is parsed here with AST only, so this + stays import-free of torch. + """ + tree = ast.parse(case_config.read_text(encoding="utf-8"), filename=str(case_config)) + groups = {} + for node in ast.walk(tree): + if not isinstance(node, ast.Assign): + continue + targets = {t.id for t in node.targets if isinstance(t, ast.Name)} + if "PARAMETRIZED_CASE_GROUPS" not in targets or not isinstance( + node.value, ast.Dict + ): + continue + for key, value in zip(node.value.keys, node.value.values): + if not (isinstance(key, ast.Constant) and isinstance(key.value, str)): + continue + if not isinstance(value, ast.List): + continue + entries = [] + for elt in value.elts: + if not isinstance(elt, ast.Tuple) or len(elt.elts) != 2: + continue + file_node, cases_node = elt.elts + if isinstance(file_node, ast.Constant) and isinstance( + cases_node, ast.Name + ): + entries.append((file_node.value, cases_node.id)) + groups[key.value] = entries + return groups + + +def collect_diffusion_pools() -> dict: + """Enumerate the NPU diffusion suites and their recommendation pools.""" + case_lists = parse_testcase_configs(DIFFUSION_CASE_CONFIG) + standalone_files, _ = parse_standalone_data(DIFFUSION_CASE_CONFIG) + parametrized_groups = _parse_parametrized_groups(DIFFUSION_CASE_CONFIG) + # A suite may have only standalone files, so enumerate the union instead + # of the parametrized groups alone (a missing pool means "run full"). + pools = {} + for suite in sorted(set(parametrized_groups) | set(standalone_files)): + cases_by_file = {} + for file_rel, list_name in parametrized_groups.get(suite, []): + rel = _diffusion_rel(file_rel) + case_ids = cases_by_file.setdefault(rel, []) + for case_id in case_lists.get(list_name, []): + if case_id not in case_ids: + case_ids.append(case_id) + pools[suite] = DiffusionPool( + cases_by_file=cases_by_file, + standalone_files=[ + _diffusion_rel(f) for f in standalone_files.get(suite, []) + ], + ) + _warn_standalone_token_collisions(pools) + return pools + + +def _warn_standalone_token_collisions(pools: dict) -> None: + """Warn when a standalone -k token is not unique inside the diffusion run. + + Standalone files are selected by pytest module-name keyword, so two files + sharing a stem (or a stem that is a substring of a case id) would make one + token select more than intended. Over-running is harmless, hence a warning + instead of an error. + """ + stems = {} + case_ids = set() + for suite, pool in pools.items(): + case_ids.update(pool.case_ids) + for rel in pool.standalone_files: + stems.setdefault(Path(rel).stem, []).append(f"{suite}:{rel}") + for stem, owners in sorted(stems.items()): + if len(owners) > 1: + print( + f"::warning::Standalone -k token {stem!r} covers multiple files: " + f"{', '.join(owners)}" + ) + hits = sorted(case_id for case_id in case_ids if stem in case_id) + if hits: + print( + f"::warning::Standalone -k token {stem!r} also matches case " + f"id(s): {', '.join(hits)}" + ) + + +def match_diffusion_pool(pool: DiffusionPool, raw_entries: list) -> tuple: + """Match recommended entries against one diffusion pool. + + A file-level entry (a changed test file, or a whole-file coverage target) + selects every case of that file; a case-level entry selects just that + case. Standalone files are selected as whole files. + """ + selected_cases = [] + selected_standalones = [] + for entry in raw_entries: + rel = normalize_rel(entry) + _, _, node = entry.strip().replace("\\", "/").partition("::") + if rel in pool.standalone_files: + if rel not in selected_standalones: + selected_standalones.append(rel) + continue + case_ids = pool.cases_by_file.get(rel) + if case_ids is None: + continue + match = _CASE_ID_RE.search(node) + if match: + if match.group(1) in case_ids and match.group(1) not in selected_cases: + selected_cases.append(match.group(1)) + else: + for case_id in case_ids: + if case_id not in selected_cases: + selected_cases.append(case_id) + return selected_cases, selected_standalones + + +def diffusion_tokens(selected_cases: list, selected_standalones: list) -> list: + """Render a diffusion bucket as pytest -k tokens. + + Bracket the case id so ``[x]`` cannot match ``[x_v2]``. Standalone files + are matched by module-name keyword (the runner applies -k to pytest, whose + keywords include the module name). + """ + tokens = [f"[{case_id}]" for case_id in selected_cases] + tokens.extend(Path(rel).stem for rel in selected_standalones) + return tokens + + +def load_multimodal_map_keys(case_map: Path): + """Return the coverage baseline's multimodal_gen keys, or None. + + None means "the baseline cannot be trusted for multimodal" (missing, + unreadable, or built before multimodal collection existed); callers then + run the diffusion suites in full. + """ + if not case_map.is_file(): + print(f"::warning::Test case map not found: {case_map}") + return None + try: + with open(case_map, encoding="utf-8") as f: + data = json.load(f) + except (OSError, ValueError) as exc: + print(f"::warning::Cannot read test case map {case_map}: {exc}") + return None + if not isinstance(data, dict): + print(f"::warning::Unexpected test case map format: {case_map}") + return None + keys = {key for key in data if key.startswith(DIFFUSION_TEST_PREFIX)} + if not keys: + print( + "::warning::Test case map has no multimodal_gen entries; the diffusion " + "suites must run in full. Rebuild the coverage baseline." + ) + return None + return keys + + +def missing_from_baseline(pool: DiffusionPool, map_keys: set) -> list: + """Pool entries that the coverage baseline never collected. + + A case or standalone file added after the last coverage run has no map + key, so no PR diff can ever recommend it. Such a suite must run in full + until the baseline is rebuilt. + """ + covered_cases = set() + for key in map_keys: + match = _CASE_ID_RE.search(key) + if match: + covered_cases.add(match.group(1)) + missing = [cid for cid in pool.case_ids if cid not in covered_cases] + missing.extend(f for f in pool.standalone_files if f not in map_keys) + return missing + + def write_github_summary( target_files: list, raw_entries: list, @@ -91,6 +352,8 @@ def write_github_summary( all_pool: set, buckets: dict, suite_pool_sizes: dict, + force_full: set, + baseline_gaps: dict, precision_mode: bool, skip_all_suites: bool = False, ) -> None: @@ -106,7 +369,7 @@ def write_github_summary( ) else: lines.append("Precision mode **off** — suites run in full.") - if skip_all_suites: + if skip_all_suites and not force_full: lines.append("") lines.append( "**All suites skipped**: nothing was recommended and every PR " @@ -119,10 +382,12 @@ def write_github_summary( f"{len(unbucketed)} match no suite pool." ) lines.append("") - lines.append("| Suite | Registered files | Bucketed files | Action |") + lines.append("| Suite | Registered | Bucketed | Action |") lines.append("|---|---|---|---|") for suite, files in buckets.items(): - if skip_all_suites or ( + if suite in force_full: + action = "run full suite (safety gate)" + elif skip_all_suites or ( precision_mode and suite_pool_sizes[suite] and not files ): action = "skipped (no recommended test)" @@ -133,6 +398,13 @@ def write_github_summary( lines.append( f"| {suite} | {suite_pool_sizes[suite]} | {len(files)} | {action} |" ) + if baseline_gaps: + lines.append("") + lines.append("**Full suite forced — baseline gaps**") + lines.append("") + for suite, missing in baseline_gaps.items(): + listed = ", ".join(f"`{item}`" for item in missing) + lines.append(f"- `{suite}`: {listed}") if raw_entries: lines.append("") lines.append( @@ -154,14 +426,14 @@ def write_github_summary( lines.append("```") lines.append("") lines.append( - "Not registered in any of the 9 bucketed a3 suites — they may still " - "run elsewhere (e.g. base-a runs its full suite; nightly tests run " - "in nightly pipelines)." + f"Not registered in any of the {len(buckets)} bucketed a3 suites — " + "they may still run elsewhere (e.g. base-a runs its full suite; " + "nightly tests run in nightly pipelines)." ) lines.append("") if any(buckets.values()): lines.append("") - lines.append("
Bucketed test files per suite") + lines.append("
Bucketed tests per suite") lines.append("") for suite, files in buckets.items(): lines.append(f"**{suite} ({len(files)})**") @@ -191,6 +463,19 @@ def main(): required=True, help="Comma-separated suite names to bucket (e.g. base-a-test-npu,base-b-test-1-npu-a3)", ) + parser.add_argument( + "--diffusion-suites", + type=str, + default="", + help="Comma-separated job:suite pairs for multimodal_gen suites " + "(e.g. multimodal-gen-test-1-npu-a3:1-npu,multimodal-gen-test-4-npu-a3:2-npu)", + ) + parser.add_argument( + "--case-map", + type=str, + default="", + help="test_case_map.json used as the multimodal safety gate", + ) parser.add_argument( "--precision-mode", type=str, @@ -259,13 +544,17 @@ def main(): and bool(changed_files) and all(p.startswith("python/sglang/") for p in changed_files) ) - if skip_all_suites: - print( - "::notice::No tests recommended and every changed file is under " - "python/sglang/ — skipping all suites." - ) suites = [s.strip() for s in args.suites.split(",") if s.strip()] + diffusion_suites = {} + for item in args.diffusion_suites.split(","): + item = item.strip() + if not item: + continue + job, sep, suite = item.partition(":") + if not sep or not job or not suite: + parser.error(f"--diffusion-suites entry must be job:suite, got {item!r}") + diffusion_suites[job] = suite work_dir = Path(tempfile.mkdtemp(prefix="bucket_tests_")) buckets = {} @@ -274,6 +563,12 @@ def main(): # the suite from the PR gate — so the output loop falls back to a # full-suite run in that case. suite_pool_sizes = {} + # Suites that must run in full even when precision mode is on (safety-gate + # failures, unreadable configs). A forced suite also opts out of the + # skip-all rule: an incomplete baseline cannot prove it is unaffected. + force_full = set() + # job -> tests absent from the coverage baseline (reported in the summary). + baseline_gaps = {} all_pool = set() for suite in suites: suite_files = {normalize_rel(f) for f in enumerate_suite_files(suite, work_dir)} @@ -287,7 +582,66 @@ def main(): f"Suite {suite}: {len(suite_files)} files, {len(intersection)} recommended" ) + if diffusion_suites: + map_keys = ( + load_multimodal_map_keys(Path(args.case_map)) if args.case_map else None + ) + if map_keys is None: + force_full |= set(diffusion_suites) + try: + diffusion_pools = collect_diffusion_pools() + except Exception as exc: + print(f"::warning::Cannot enumerate diffusion suites: {exc!r}") + diffusion_pools = {} + force_full |= set(diffusion_suites) + for job, suite in diffusion_suites.items(): + pool = diffusion_pools.get(suite) + if pool is None: + print( + f"::warning::Diffusion suite {suite} ({job}) not found in " + f"{DIFFUSION_CASE_CONFIG.name}" + ) + buckets[job] = [] + suite_pool_sizes[job] = 0 + force_full.add(job) + continue + suite_pool_sizes[job] = pool.size + all_pool |= pool.files() + if map_keys is None: + buckets[job] = [] + print( + f"Suite {job} (diffusion {suite}): {pool.size} cases, " + "forced full suite by the safety gate" + ) + continue + missing = missing_from_baseline(pool, map_keys) + if missing: + buckets[job] = [] + force_full.add(job) + baseline_gaps[job] = missing + print( + f"::warning::Diffusion suite {suite} ({job}): {len(missing)} " + f"test(s) missing from the coverage baseline " + f"({', '.join(missing)}); running the full suite. " + "Rebuild the baseline to restore precision." + ) + continue + selected_cases, selected_standalones = match_diffusion_pool( + pool, raw_entries + ) + tokens = diffusion_tokens(selected_cases, selected_standalones) + buckets[job] = tokens + print( + f"Suite {job} (diffusion {suite}): {pool.size} cases, " + f"{len(tokens)} recommended" + ) + total_count = sum(len(b) for b in buckets.values()) + if skip_all_suites and not force_full: + print( + "::notice::No tests recommended and every changed file is under " + "python/sglang/ — skipping all suites." + ) precision_mode = args.precision_mode == "true" and total_count > 0 print(f"total_count={total_count} precision_mode={precision_mode}") @@ -315,11 +669,17 @@ def main(): # 'true' so the job runs the full suite instead of silently # vanishing. Anything else ('true', or a missing output from a # failed step) means full-suite run. + # A force_full suite never takes either skip path: its baseline + # is incomplete, so "nothing recommended" proves nothing. should_run = ( "false" if ( - skip_all_suites - or (precision_mode and suite_pool_sizes[suite] and not files) + not files + and suite not in force_full + and ( + skip_all_suites + or (precision_mode and suite_pool_sizes[suite]) + ) ) else "true" ) @@ -333,6 +693,8 @@ def main(): all_pool, buckets, suite_pool_sizes, + force_full, + baseline_gaps, precision_mode, skip_all_suites, ) diff --git a/scripts/ci/npu/precise-test/test_selector.py b/scripts/ci/npu/precise-test/test_selector.py index 0b49a08f25..ac61327d2b 100644 --- a/scripts/ci/npu/precise-test/test_selector.py +++ b/scripts/ci/npu/precise-test/test_selector.py @@ -79,13 +79,18 @@ def _get_test_files_from_pr_diff(diff_file: str) -> list[str]: print(f" Warning: Failed to read diff file for test file detection: {e}") return test_files_found - # Pattern to match test file paths: test/registered/ directory + # Pattern to match test file paths: test/registered/ directory and the + # multimodal_gen diffusion tests. # In diff output: # - +++ b/test/registered/unit/xxx/test_xxx.py (new/modified test file) # - rename to test/registered/unit/xxx/test_xxx.py (renamed test file) # Test files must be in test/ directory and start with test_ test_file_pattern = re.compile( - r"^(?:\+\+\+ [ab]/|rename to )((?:test/registered(?:/.+)?/test_\w+\.py|test/(?:unit|e2e|integration)(?:/.+)?/test_\w+\.py))", + r"^(?:\+\+\+ [ab]/|rename to )(" + r"test/registered(?:/.+)?/test_\w+\.py" + r"|test/(?:unit|e2e|integration)(?:/.+)?/test_\w+\.py" + r"|python/sglang/multimodal_gen/test/(?:.*/)?test_\w+\.py" + r")", re.MULTILINE, ) @@ -130,10 +135,15 @@ def _get_deleted_test_files_from_pr(diff_file: str, test_case_map: dict) -> list print(f" Warning: Failed to read diff file for deleted test detection: {e}") return deleted_test_files - # Pattern to match deleted test files (test/registered/ directory) + # Pattern to match deleted test files (test/registered/ directory and + # multimodal_gen diffusion tests) # Match --- a/test/... followed by +++ /dev/null (deleted file marker) deleted_pattern = re.compile( - r"^--- a/(test/registered(?:/.+)?/test_\w+\.py|test/(?:unit|e2e|integration)(?:/.+)?/test_\w+\.py)\s*\n\s*\+\+\+ [ab]?/dev/null", + r"^--- a/(" + r"test/registered(?:/.+)?/test_\w+\.py" + r"|test/(?:unit|e2e|integration)(?:/.+)?/test_\w+\.py" + r"|python/sglang/multimodal_gen/test/(?:.*/)?test_\w+\.py" + r")\s*\n\s*\+\+\+ [ab]?/dev/null", re.MULTILINE, )