From 33a0734ca16b3f102f548ce66ae20265f75cf4af Mon Sep 17 00:00:00 2001 From: shun8686 Date: Thu, 22 Oct 2026 15:11:47 +0800 Subject: [PATCH] [CI] [NPU] precise-test: collect multimodal_gen coverage per test case (#45553) --- .github/workflows/coverage-collection-npu.yml | 6 +- .github/workflows/pr-test-npu.yml | 72 +++++++++- .../ascend/test_glm_image_distributed.py | 12 +- .../npu/precise-test/list_multimodal_tests.py | 127 ++++++++++++++++++ .../precise_test_coverage_plugin.py | 63 +++++++-- 5 files changed, 266 insertions(+), 14 deletions(-) create mode 100755 scripts/ci/npu/precise-test/list_multimodal_tests.py diff --git a/.github/workflows/coverage-collection-npu.yml b/.github/workflows/coverage-collection-npu.yml index a008a09703..40488e8992 100644 --- a/.github/workflows/coverage-collection-npu.yml +++ b/.github/workflows/coverage-collection-npu.yml @@ -3,7 +3,11 @@ # - Runners to the coverage pool (shared /root/.cache with setup-covstub) # - use_coverage_runner=true on all test jobs # - Runs the base-a unit suite on the coverage pool (base-a-test-npu-cov) -# - Skips pr-gate, multimodal-gen, recommend, analyze +# - Runs the multimodal-gen suites too. Their runner is asked for this +# shard's targets (case-level pytest node ids) and each target is run in +# its own coverage invocation, so diffusion coverage is attributed per +# test case instead of per shard. +# - Skips pr-gate, recommend, analyze # - Runs setup-covstub to assemble coverage data and publish it to the # shared disk at a fixed path (/root/.cache/tests/precise-test/coverage) # pr-test-npu-finish is the final gate in the called workflow. diff --git a/.github/workflows/pr-test-npu.yml b/.github/workflows/pr-test-npu.yml index b3209293c1..8fd77e432f 100644 --- a/.github/workflows/pr-test-npu.yml +++ b/.github/workflows/pr-test-npu.yml @@ -337,7 +337,7 @@ jobs: multimodal-gen-test-1-npu-a3: needs: [check-changes, pr-gate, set-image-config, base-a-test-npu] - if: ${{ !failure() && !cancelled() && inputs.coverage_mode != true && needs.check-changes.outputs.multimodal_gen == 'true' }} + if: ${{ !failure() && !cancelled() && (inputs.coverage_mode == true || needs.check-changes.outputs.multimodal_gen == 'true') }} runs-on: linux-aarch64-a3-800t-2 strategy: fail-fast: false @@ -385,7 +385,8 @@ jobs: run: python3 scripts/ci/utils/diffusion/prefetch_input_image.py - name: Run test - timeout-minutes: 60 + timeout-minutes: ${{ inputs.coverage_mode == true && 150 || 60 }} + shell: bash env: SGLANG_USE_MODELSCOPE: true SGLANG_IS_IN_CI: true @@ -395,6 +396,36 @@ jobs: STREAMS_PER_DEVICE: 32 SGLANG_DIFFUSION_ARTIFACT_DIR: ${{ github.workspace }}/diffusion-failures run: | + # Coverage mode: enumerate this shard's targets with an NPU-only CI + # helper (it reuses the shared diffusion runner read-only) and run + # each target in its own `coverage run -m pytest` invocation. The + # runner normally batches a whole shard into one pytest call, which + # cannot be attributed per test case by the precise-test selector. + if [[ "${{ inputs.coverage_mode }}" == "true" ]]; then + set -euo pipefail + pip install coverage==7.8.* + # Coverage tracing slows down the Python glue, which shows up in + # wall-clock assertions. Relax only the timing tolerances for the + # collection run; PR runs (no coverage) keep the strict profile and + # the memory/VRAM checks are untouched. + export SGLANG_E2E_TOLERANCE=1.0 + export SGLANG_STAGE_TIME_TOLERANCE=1.0 + export SGLANG_NON_DENOISE_STAGE_TIME_TOLERANCE=2.0 + export SGLANG_DENOISE_STEP_TOLERANCE=1.0 + export SGLANG_DENOISE_AGG_TOLERANCE=1.0 + python3 scripts/ci/npu/precise-test/list_multimodal_tests.py \ + --suite 1-npu \ + --partition-id ${{ matrix.part }} \ + --total-partitions 2 \ + -o /tmp/selected_tests.txt + echo "Selected test targets:" + cat /tmp/selected_tests.txt + if [ -s /tmp/selected_tests.txt ]; then + bash scripts/ci/npu/precise-test/run_tests_with_coverage.sh $(cat /tmp/selected_tests.txt) + fi + exit 0 + fi + cd python python3 sglang/multimodal_gen/test/run_suite.py \ --suite 1-npu \ @@ -412,7 +443,7 @@ jobs: multimodal-gen-test-4-npu-a3: needs: [check-changes, pr-gate, set-image-config, base-a-test-npu] - if: ${{ !failure() && !cancelled() && inputs.coverage_mode != true && needs.check-changes.outputs.multimodal_gen == 'true' }} + if: ${{ !failure() && !cancelled() && (inputs.coverage_mode == true || needs.check-changes.outputs.multimodal_gen == 'true') }} runs-on: linux-aarch64-a3-800t-4 strategy: fail-fast: false @@ -455,7 +486,8 @@ jobs: python3 scripts/ci/utils/diffusion/prefetch_consistency_gt.py - name: Run test - timeout-minutes: 60 + timeout-minutes: ${{ inputs.coverage_mode == true && 150 || 60 }} + shell: bash env: SGLANG_USE_MODELSCOPE: true SGLANG_IS_IN_CI: true @@ -465,6 +497,36 @@ jobs: STREAMS_PER_DEVICE: 32 SGLANG_DIFFUSION_ARTIFACT_DIR: ${{ github.workspace }}/diffusion-failures run: | + # Coverage mode: enumerate this shard's targets with an NPU-only CI + # helper (it reuses the shared diffusion runner read-only) and run + # each target in its own `coverage run -m pytest` invocation. The + # runner normally batches a whole shard into one pytest call, which + # cannot be attributed per test case by the precise-test selector. + if [[ "${{ inputs.coverage_mode }}" == "true" ]]; then + set -euo pipefail + pip install coverage==7.8.* + # Coverage tracing slows down the Python glue, which shows up in + # wall-clock assertions. Relax only the timing tolerances for the + # collection run; PR runs (no coverage) keep the strict profile and + # the memory/VRAM checks are untouched. + export SGLANG_E2E_TOLERANCE=1.0 + export SGLANG_STAGE_TIME_TOLERANCE=1.0 + export SGLANG_NON_DENOISE_STAGE_TIME_TOLERANCE=2.0 + export SGLANG_DENOISE_STEP_TOLERANCE=1.0 + export SGLANG_DENOISE_AGG_TOLERANCE=1.0 + python3 scripts/ci/npu/precise-test/list_multimodal_tests.py \ + --suite 2-npu \ + --partition-id ${{ matrix.part }} \ + --total-partitions 2 \ + -o /tmp/selected_tests.txt + echo "Selected test targets:" + cat /tmp/selected_tests.txt + if [ -s /tmp/selected_tests.txt ]; then + bash scripts/ci/npu/precise-test/run_tests_with_coverage.sh $(cat /tmp/selected_tests.txt) + fi + exit 0 + fi + cd python python3 sglang/multimodal_gen/test/run_suite.py \ --suite 2-npu \ @@ -696,6 +758,8 @@ jobs: base-c-test-acc-16-npu-a3, base-c-test-perf-2-npu-a3, base-c-test-perf-16-npu-a3, + multimodal-gen-test-1-npu-a3, + multimodal-gen-test-4-npu-a3, ] if: ${{ !failure() && !cancelled() && inputs.coverage_mode == true }} runs-on: linux-aarch64-a3-800t-2 diff --git a/python/sglang/multimodal_gen/test/server/ascend/test_glm_image_distributed.py b/python/sglang/multimodal_gen/test/server/ascend/test_glm_image_distributed.py index 65aa25d9c1..9e1421eb93 100644 --- a/python/sglang/multimodal_gen/test/server/ascend/test_glm_image_distributed.py +++ b/python/sglang/multimodal_gen/test/server/ascend/test_glm_image_distributed.py @@ -30,7 +30,17 @@ _PERFORMANCE_TOLERANCE = 0.25 def _kill_process_tree(proc: subprocess.Popen) -> None: try: - os.killpg(os.getpgid(proc.pid), signal.SIGKILL) + pgid = os.getpgid(proc.pid) + # Coverage collection can only flush a traced subprocess on a graceful + # stop (SIGKILL skips atexit). Give the cluster a SIGTERM head start + # when coverage is active, then always escalate to SIGKILL. + if os.environ.get("COVERAGE_PROCESS_START"): + os.killpg(pgid, signal.SIGTERM) + try: + proc.wait(timeout=30) + except subprocess.TimeoutExpired: + pass + os.killpg(pgid, signal.SIGKILL) except (ProcessLookupError, PermissionError): pass diff --git a/scripts/ci/npu/precise-test/list_multimodal_tests.py b/scripts/ci/npu/precise-test/list_multimodal_tests.py new file mode 100755 index 0000000000..c921df0774 --- /dev/null +++ b/scripts/ci/npu/precise-test/list_multimodal_tests.py @@ -0,0 +1,127 @@ +#!/usr/bin/env python3 +"""List the diffusion pytest targets of one NPU shard (coverage collection only). + +This is an NPU CI helper: it imports the shared multimodal_gen runner +READ-ONLY. The shard composition must match what +``python/sglang/multimodal_gen/test/run_suite.py`` would execute, so the LPT +assignment is reused instead of reimplemented, but no platform-shared test +code is modified. + +Output: one pytest target per line, with absolute paths, e.g. + /python/sglang/multimodal_gen/test/server/ascend/test_server_1_npu.py::TestDiffusionServerOneNpu::test_diffusion_generation[ernie_image_t2i_1npu] + /python/sglang/multimodal_gen/test/server/ascend/test_glm_image_distributed.py + +Absolute paths matter: run_tests_with_coverage.sh encodes them into the +coverage directory name and test_selector.py recovers the repo-relative test +path from the ``__sglang__sglang__`` workspace marker. + +Usage: + python3 list_multimodal_tests.py --suite 1-npu \ + --partition-id 0 --total-partitions 2 -o /tmp/selected_tests.txt +""" + +import argparse +import sys +from pathlib import Path + +# Repo layout: this script lives at /scripts/ci/npu/precise-test/. +SCRIPT_DIR = Path(__file__).resolve().parent +REPO_ROOT = SCRIPT_DIR.parents[3] + +# The runner imports sglang (torch / torch_npu), so make the checkout importable +# even when the package is not installed in editable mode. +sys.path.insert(0, str(REPO_ROOT / "python")) + +from sglang.multimodal_gen.test.runner import ( # noqa: E402 + diffusion_suite_runner as runner, +) +from sglang.multimodal_gen.test.runner.pytest_runner import ( # noqa: E402 + collect_test_items, + partition_items_by_index, +) + + +def _case_files(suite: str, server_dir: Path) -> dict: + """Map case id -> absolute test file for a parametrized diffusion suite.""" + mapping = {} + for filename, case_group in runner.PARAMETRIZED_CASE_GROUPS.get(suite, []): + file_path = server_dir / filename + for case in case_group: + mapping[case.id] = file_path + return mapping + + +def _node_targets(file_path: Path, case_ids: list) -> list: + """Expand case ids into absolute pytest node-id targets for one file.""" + filter_expr = " or ".join(f"[{case_id}]" for case_id in case_ids) + targets = [] + for item in collect_test_items([str(file_path)], filter_expr): + # collect_test_items reports node ids relative to pytest's rootdir, + # which is not the checkout root. Rebuild them from the absolute file + # so the coverage collector can encode the workspace layout. + node = item.partition("::")[2] + targets.append(f"{file_path}::{node}" if node else str(file_path)) + return targets + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--suite", required=True, help="Diffusion suite, e.g. 1-npu") + parser.add_argument("--partition-id", type=int, default=0) + parser.add_argument("--total-partitions", type=int, default=1) + parser.add_argument( + "-o", + "--output", + required=True, + help="Write the pytest targets (one per line) to this file.", + ) + args = parser.parse_args() + + test_root = Path(runner.__file__).resolve().parents[1] + server_dir = test_root / "server" + + if args.suite in runner.PARAMETRIZED_CASE_GROUPS: + assignment = runner.build_local_partition_assignment( + args.suite, args.partition_id, args.total_partitions + ) + case_files = _case_files(args.suite, server_dir) + missing = [cid for cid in assignment.case_ids if cid not in case_files] + if missing: + print(f"::warning::Cases absent from the {args.suite} config: {missing}") + + targets = [] + for file_path in sorted( + {case_files[cid] for cid in assignment.case_ids if cid in case_files} + ): + case_ids = [ + cid for cid in assignment.case_ids if case_files.get(cid) == file_path + ] + targets.extend(_node_targets(file_path, case_ids)) + for standalone_rel in assignment.standalone_files: + targets.append(str(server_dir / standalone_rel)) + elif args.suite in runner.FILE_SUITES: + files = [str(server_dir / rel) for rel in runner.FILE_SUITES[args.suite]] + targets = partition_items_by_index( + files, args.partition_id, args.total_partitions + ) + else: + parser.error(f"Unknown diffusion suite: {args.suite!r}") + + unique_targets = [] + seen = set() + for target in targets: + if target not in seen: + seen.add(target) + unique_targets.append(target) + + with open(args.output, "w", encoding="utf-8") as f: + for target in unique_targets: + f.write(target + "\n") + + print(f"Listed {len(unique_targets)} test target(s) to {args.output}:") + for target in unique_targets: + print(f" - {target}") + + +if __name__ == "__main__": + main() diff --git a/scripts/ci/npu/precise-test/precise_test_coverage_plugin.py b/scripts/ci/npu/precise-test/precise_test_coverage_plugin.py index 2b4159ab5a..4ad0a58da4 100644 --- a/scripts/ci/npu/precise-test/precise_test_coverage_plugin.py +++ b/scripts/ci/npu/precise-test/precise_test_coverage_plugin.py @@ -17,7 +17,10 @@ defers to the original SIGKILL implementation for whatever is left. All server teardowns in the coverage suites converge on the single sglang.srt.utils.common.kill_process_tree (re-exported by sglang.srt.utils, bound into sglang.test.test_utils and the ascend e2e -utils), so patching that one attribute covers base-b and base-c. +utils), so patching that one attribute covers base-b and base-c. The +diffusion suites use their own +sglang.multimodal_gen.runtime.utils.process.kill_process_tree, patched +lazily after collection (see _patch_multimodal_kill_process_tree). """ import os @@ -27,11 +30,11 @@ import sys import psutil -def _patch_kill_process_tree(): - import sglang.srt.utils.common as common +def _enabled() -> bool: + return os.environ.get("SGLANG_KILL_PROCESS_TREE_GRACEFUL") == "1" - original = common.kill_process_tree +def _make_graceful_kill_process_tree(original): graceful_timeout = float( os.environ.get("SGLANG_KILL_PROCESS_TREE_GRACEFUL_TIMEOUT", "5") ) @@ -64,18 +67,51 @@ def _patch_kill_process_tree(): wait_timeout=wait_timeout, ) - common.kill_process_tree = graceful = kill_process_tree_graceful + kill_process_tree_graceful._sglang_ci_graceful = True + return kill_process_tree_graceful + + +def _patch_module_kill_process_tree(module): + original = module.kill_process_tree + graceful = _make_graceful_kill_process_tree(original) + module.kill_process_tree = graceful # Refresh modules that already bound the original (the sglang.srt.utils - # re-export, anything an early conftest imported). Modules imported - # later pick up the patched attribute on their own. + # re-export, anything an early conftest imported, the diffusion test + # helpers). Modules imported later pick up the patched attribute on + # their own. for mod in list(sys.modules.values()): if getattr(mod, "kill_process_tree", None) is original: mod.kill_process_tree = graceful +def _patch_kill_process_tree(): + import sglang.srt.utils.common as common + + _patch_module_kill_process_tree(common) + + +def _patch_multimodal_kill_process_tree(): + """Patch the diffusion runtime's SIGKILL-only teardown, if it is loaded. + + The diffusion suites tear their servers down via + sglang.multimodal_gen.runtime.utils.process.kill_process_tree instead of + the srt helper patched above, so subprocess coverage would be lost on the + SIGKILL. Unlike srt this module is imported by the diffusion test modules + during collection, so patch lazily AFTER collection: importing it here + would pull the whole diffusion runtime into every other suite's pytest + process for no benefit. + """ + module = sys.modules.get("sglang.multimodal_gen.runtime.utils.process") + if module is None: + return + if getattr(module.kill_process_tree, "_sglang_ci_graceful", False): + return + _patch_module_kill_process_tree(module) + + def pytest_configure(config): - if os.environ.get("SGLANG_KILL_PROCESS_TREE_GRACEFUL") != "1": + if not _enabled(): return try: _patch_kill_process_tree() @@ -84,3 +120,14 @@ def pytest_configure(config): # outside the tests' own stubs. No servers are launched there, so # skipping the patch is fine — just make it visible in the log. print(f"precise_test_coverage_plugin: patch skipped: {exc!r}") + + +def pytest_collection_modifyitems(config, items): + if not _enabled(): + return + try: + _patch_multimodal_kill_process_tree() + except Exception as exc: + print( + f"precise_test_coverage_plugin: multimodal teardown patch skipped: {exc!r}" + )