[CI] [NPU] precise-test: collect multimodal_gen coverage per test case (#45553)

This commit is contained in:
shun8686
2026-10-22 15:11:47 +08:00
committed by GitHub
parent 569dda1e61
commit 33a0734ca1
5 changed files with 266 additions and 14 deletions
@@ -3,7 +3,11 @@
# - Runners to the coverage pool (shared /root/.cache with setup-covstub)
# - use_coverage_runner=true on all test jobs
# - Runs the base-a unit suite on the coverage pool (base-a-test-npu-cov)
# - Skips pr-gate, multimodal-gen, recommend, analyze
# - Runs the multimodal-gen suites too. Their runner is asked for this
# shard's targets (case-level pytest node ids) and each target is run in
# its own coverage invocation, so diffusion coverage is attributed per
# test case instead of per shard.
# - Skips pr-gate, recommend, analyze
# - Runs setup-covstub to assemble coverage data and publish it to the
# shared disk at a fixed path (/root/.cache/tests/precise-test/coverage)
# pr-test-npu-finish is the final gate in the called workflow.
+68 -4
View File
@@ -337,7 +337,7 @@ jobs:
multimodal-gen-test-1-npu-a3:
needs: [check-changes, pr-gate, set-image-config, base-a-test-npu]
if: ${{ !failure() && !cancelled() && inputs.coverage_mode != true && needs.check-changes.outputs.multimodal_gen == 'true' }}
if: ${{ !failure() && !cancelled() && (inputs.coverage_mode == true || needs.check-changes.outputs.multimodal_gen == 'true') }}
runs-on: linux-aarch64-a3-800t-2
strategy:
fail-fast: false
@@ -385,7 +385,8 @@ jobs:
run: python3 scripts/ci/utils/diffusion/prefetch_input_image.py
- name: Run test
timeout-minutes: 60
timeout-minutes: ${{ inputs.coverage_mode == true && 150 || 60 }}
shell: bash
env:
SGLANG_USE_MODELSCOPE: true
SGLANG_IS_IN_CI: true
@@ -395,6 +396,36 @@ jobs:
STREAMS_PER_DEVICE: 32
SGLANG_DIFFUSION_ARTIFACT_DIR: ${{ github.workspace }}/diffusion-failures
run: |
# Coverage mode: enumerate this shard's targets with an NPU-only CI
# helper (it reuses the shared diffusion runner read-only) and run
# each target in its own `coverage run -m pytest` invocation. The
# runner normally batches a whole shard into one pytest call, which
# cannot be attributed per test case by the precise-test selector.
if [[ "${{ inputs.coverage_mode }}" == "true" ]]; then
set -euo pipefail
pip install coverage==7.8.*
# Coverage tracing slows down the Python glue, which shows up in
# wall-clock assertions. Relax only the timing tolerances for the
# collection run; PR runs (no coverage) keep the strict profile and
# the memory/VRAM checks are untouched.
export SGLANG_E2E_TOLERANCE=1.0
export SGLANG_STAGE_TIME_TOLERANCE=1.0
export SGLANG_NON_DENOISE_STAGE_TIME_TOLERANCE=2.0
export SGLANG_DENOISE_STEP_TOLERANCE=1.0
export SGLANG_DENOISE_AGG_TOLERANCE=1.0
python3 scripts/ci/npu/precise-test/list_multimodal_tests.py \
--suite 1-npu \
--partition-id ${{ matrix.part }} \
--total-partitions 2 \
-o /tmp/selected_tests.txt
echo "Selected test targets:"
cat /tmp/selected_tests.txt
if [ -s /tmp/selected_tests.txt ]; then
bash scripts/ci/npu/precise-test/run_tests_with_coverage.sh $(cat /tmp/selected_tests.txt)
fi
exit 0
fi
cd python
python3 sglang/multimodal_gen/test/run_suite.py \
--suite 1-npu \
@@ -412,7 +443,7 @@ jobs:
multimodal-gen-test-4-npu-a3:
needs: [check-changes, pr-gate, set-image-config, base-a-test-npu]
if: ${{ !failure() && !cancelled() && inputs.coverage_mode != true && needs.check-changes.outputs.multimodal_gen == 'true' }}
if: ${{ !failure() && !cancelled() && (inputs.coverage_mode == true || needs.check-changes.outputs.multimodal_gen == 'true') }}
runs-on: linux-aarch64-a3-800t-4
strategy:
fail-fast: false
@@ -455,7 +486,8 @@ jobs:
python3 scripts/ci/utils/diffusion/prefetch_consistency_gt.py
- name: Run test
timeout-minutes: 60
timeout-minutes: ${{ inputs.coverage_mode == true && 150 || 60 }}
shell: bash
env:
SGLANG_USE_MODELSCOPE: true
SGLANG_IS_IN_CI: true
@@ -465,6 +497,36 @@ jobs:
STREAMS_PER_DEVICE: 32
SGLANG_DIFFUSION_ARTIFACT_DIR: ${{ github.workspace }}/diffusion-failures
run: |
# Coverage mode: enumerate this shard's targets with an NPU-only CI
# helper (it reuses the shared diffusion runner read-only) and run
# each target in its own `coverage run -m pytest` invocation. The
# runner normally batches a whole shard into one pytest call, which
# cannot be attributed per test case by the precise-test selector.
if [[ "${{ inputs.coverage_mode }}" == "true" ]]; then
set -euo pipefail
pip install coverage==7.8.*
# Coverage tracing slows down the Python glue, which shows up in
# wall-clock assertions. Relax only the timing tolerances for the
# collection run; PR runs (no coverage) keep the strict profile and
# the memory/VRAM checks are untouched.
export SGLANG_E2E_TOLERANCE=1.0
export SGLANG_STAGE_TIME_TOLERANCE=1.0
export SGLANG_NON_DENOISE_STAGE_TIME_TOLERANCE=2.0
export SGLANG_DENOISE_STEP_TOLERANCE=1.0
export SGLANG_DENOISE_AGG_TOLERANCE=1.0
python3 scripts/ci/npu/precise-test/list_multimodal_tests.py \
--suite 2-npu \
--partition-id ${{ matrix.part }} \
--total-partitions 2 \
-o /tmp/selected_tests.txt
echo "Selected test targets:"
cat /tmp/selected_tests.txt
if [ -s /tmp/selected_tests.txt ]; then
bash scripts/ci/npu/precise-test/run_tests_with_coverage.sh $(cat /tmp/selected_tests.txt)
fi
exit 0
fi
cd python
python3 sglang/multimodal_gen/test/run_suite.py \
--suite 2-npu \
@@ -696,6 +758,8 @@ jobs:
base-c-test-acc-16-npu-a3,
base-c-test-perf-2-npu-a3,
base-c-test-perf-16-npu-a3,
multimodal-gen-test-1-npu-a3,
multimodal-gen-test-4-npu-a3,
]
if: ${{ !failure() && !cancelled() && inputs.coverage_mode == true }}
runs-on: linux-aarch64-a3-800t-2
@@ -30,7 +30,17 @@ _PERFORMANCE_TOLERANCE = 0.25
def _kill_process_tree(proc: subprocess.Popen) -> None:
try:
os.killpg(os.getpgid(proc.pid), signal.SIGKILL)
pgid = os.getpgid(proc.pid)
# Coverage collection can only flush a traced subprocess on a graceful
# stop (SIGKILL skips atexit). Give the cluster a SIGTERM head start
# when coverage is active, then always escalate to SIGKILL.
if os.environ.get("COVERAGE_PROCESS_START"):
os.killpg(pgid, signal.SIGTERM)
try:
proc.wait(timeout=30)
except subprocess.TimeoutExpired:
pass
os.killpg(pgid, signal.SIGKILL)
except (ProcessLookupError, PermissionError):
pass
+127
View File
@@ -0,0 +1,127 @@
#!/usr/bin/env python3
"""List the diffusion pytest targets of one NPU shard (coverage collection only).
This is an NPU CI helper: it imports the shared multimodal_gen runner
READ-ONLY. The shard composition must match what
``python/sglang/multimodal_gen/test/run_suite.py`` would execute, so the LPT
assignment is reused instead of reimplemented, but no platform-shared test
code is modified.
Output: one pytest target per line, with absolute paths, e.g.
<workspace>/python/sglang/multimodal_gen/test/server/ascend/test_server_1_npu.py::TestDiffusionServerOneNpu::test_diffusion_generation[ernie_image_t2i_1npu]
<workspace>/python/sglang/multimodal_gen/test/server/ascend/test_glm_image_distributed.py
Absolute paths matter: run_tests_with_coverage.sh encodes them into the
coverage directory name and test_selector.py recovers the repo-relative test
path from the ``__sglang__sglang__`` workspace marker.
Usage:
python3 list_multimodal_tests.py --suite 1-npu \
--partition-id 0 --total-partitions 2 -o /tmp/selected_tests.txt
"""
import argparse
import sys
from pathlib import Path
# Repo layout: this script lives at <repo>/scripts/ci/npu/precise-test/.
SCRIPT_DIR = Path(__file__).resolve().parent
REPO_ROOT = SCRIPT_DIR.parents[3]
# The runner imports sglang (torch / torch_npu), so make the checkout importable
# even when the package is not installed in editable mode.
sys.path.insert(0, str(REPO_ROOT / "python"))
from sglang.multimodal_gen.test.runner import ( # noqa: E402
diffusion_suite_runner as runner,
)
from sglang.multimodal_gen.test.runner.pytest_runner import ( # noqa: E402
collect_test_items,
partition_items_by_index,
)
def _case_files(suite: str, server_dir: Path) -> dict:
"""Map case id -> absolute test file for a parametrized diffusion suite."""
mapping = {}
for filename, case_group in runner.PARAMETRIZED_CASE_GROUPS.get(suite, []):
file_path = server_dir / filename
for case in case_group:
mapping[case.id] = file_path
return mapping
def _node_targets(file_path: Path, case_ids: list) -> list:
"""Expand case ids into absolute pytest node-id targets for one file."""
filter_expr = " or ".join(f"[{case_id}]" for case_id in case_ids)
targets = []
for item in collect_test_items([str(file_path)], filter_expr):
# collect_test_items reports node ids relative to pytest's rootdir,
# which is not the checkout root. Rebuild them from the absolute file
# so the coverage collector can encode the workspace layout.
node = item.partition("::")[2]
targets.append(f"{file_path}::{node}" if node else str(file_path))
return targets
def main() -> None:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--suite", required=True, help="Diffusion suite, e.g. 1-npu")
parser.add_argument("--partition-id", type=int, default=0)
parser.add_argument("--total-partitions", type=int, default=1)
parser.add_argument(
"-o",
"--output",
required=True,
help="Write the pytest targets (one per line) to this file.",
)
args = parser.parse_args()
test_root = Path(runner.__file__).resolve().parents[1]
server_dir = test_root / "server"
if args.suite in runner.PARAMETRIZED_CASE_GROUPS:
assignment = runner.build_local_partition_assignment(
args.suite, args.partition_id, args.total_partitions
)
case_files = _case_files(args.suite, server_dir)
missing = [cid for cid in assignment.case_ids if cid not in case_files]
if missing:
print(f"::warning::Cases absent from the {args.suite} config: {missing}")
targets = []
for file_path in sorted(
{case_files[cid] for cid in assignment.case_ids if cid in case_files}
):
case_ids = [
cid for cid in assignment.case_ids if case_files.get(cid) == file_path
]
targets.extend(_node_targets(file_path, case_ids))
for standalone_rel in assignment.standalone_files:
targets.append(str(server_dir / standalone_rel))
elif args.suite in runner.FILE_SUITES:
files = [str(server_dir / rel) for rel in runner.FILE_SUITES[args.suite]]
targets = partition_items_by_index(
files, args.partition_id, args.total_partitions
)
else:
parser.error(f"Unknown diffusion suite: {args.suite!r}")
unique_targets = []
seen = set()
for target in targets:
if target not in seen:
seen.add(target)
unique_targets.append(target)
with open(args.output, "w", encoding="utf-8") as f:
for target in unique_targets:
f.write(target + "\n")
print(f"Listed {len(unique_targets)} test target(s) to {args.output}:")
for target in unique_targets:
print(f" - {target}")
if __name__ == "__main__":
main()
@@ -17,7 +17,10 @@ defers to the original SIGKILL implementation for whatever is left.
All server teardowns in the coverage suites converge on the single
sglang.srt.utils.common.kill_process_tree (re-exported by
sglang.srt.utils, bound into sglang.test.test_utils and the ascend e2e
utils), so patching that one attribute covers base-b and base-c.
utils), so patching that one attribute covers base-b and base-c. The
diffusion suites use their own
sglang.multimodal_gen.runtime.utils.process.kill_process_tree, patched
lazily after collection (see _patch_multimodal_kill_process_tree).
"""
import os
@@ -27,11 +30,11 @@ import sys
import psutil
def _patch_kill_process_tree():
import sglang.srt.utils.common as common
def _enabled() -> bool:
return os.environ.get("SGLANG_KILL_PROCESS_TREE_GRACEFUL") == "1"
original = common.kill_process_tree
def _make_graceful_kill_process_tree(original):
graceful_timeout = float(
os.environ.get("SGLANG_KILL_PROCESS_TREE_GRACEFUL_TIMEOUT", "5")
)
@@ -64,18 +67,51 @@ def _patch_kill_process_tree():
wait_timeout=wait_timeout,
)
common.kill_process_tree = graceful = kill_process_tree_graceful
kill_process_tree_graceful._sglang_ci_graceful = True
return kill_process_tree_graceful
def _patch_module_kill_process_tree(module):
original = module.kill_process_tree
graceful = _make_graceful_kill_process_tree(original)
module.kill_process_tree = graceful
# Refresh modules that already bound the original (the sglang.srt.utils
# re-export, anything an early conftest imported). Modules imported
# later pick up the patched attribute on their own.
# re-export, anything an early conftest imported, the diffusion test
# helpers). Modules imported later pick up the patched attribute on
# their own.
for mod in list(sys.modules.values()):
if getattr(mod, "kill_process_tree", None) is original:
mod.kill_process_tree = graceful
def _patch_kill_process_tree():
import sglang.srt.utils.common as common
_patch_module_kill_process_tree(common)
def _patch_multimodal_kill_process_tree():
"""Patch the diffusion runtime's SIGKILL-only teardown, if it is loaded.
The diffusion suites tear their servers down via
sglang.multimodal_gen.runtime.utils.process.kill_process_tree instead of
the srt helper patched above, so subprocess coverage would be lost on the
SIGKILL. Unlike srt this module is imported by the diffusion test modules
during collection, so patch lazily AFTER collection: importing it here
would pull the whole diffusion runtime into every other suite's pytest
process for no benefit.
"""
module = sys.modules.get("sglang.multimodal_gen.runtime.utils.process")
if module is None:
return
if getattr(module.kill_process_tree, "_sglang_ci_graceful", False):
return
_patch_module_kill_process_tree(module)
def pytest_configure(config):
if os.environ.get("SGLANG_KILL_PROCESS_TREE_GRACEFUL") != "1":
if not _enabled():
return
try:
_patch_kill_process_tree()
@@ -84,3 +120,14 @@ def pytest_configure(config):
# outside the tests' own stubs. No servers are launched there, so
# skipping the patch is fine — just make it visible in the log.
print(f"precise_test_coverage_plugin: patch skipped: {exc!r}")
def pytest_collection_modifyitems(config, items):
if not _enabled():
return
try:
_patch_multimodal_kill_process_tree()
except Exception as exc:
print(
f"precise_test_coverage_plugin: multimodal teardown patch skipped: {exc!r}"
)