[CI] [NPU] precise-test: collect multimodal_gen coverage per test case (#45553)
This commit is contained in:
@@ -3,7 +3,11 @@
|
||||
# - Runners to the coverage pool (shared /root/.cache with setup-covstub)
|
||||
# - use_coverage_runner=true on all test jobs
|
||||
# - Runs the base-a unit suite on the coverage pool (base-a-test-npu-cov)
|
||||
# - Skips pr-gate, multimodal-gen, recommend, analyze
|
||||
# - Runs the multimodal-gen suites too. Their runner is asked for this
|
||||
# shard's targets (case-level pytest node ids) and each target is run in
|
||||
# its own coverage invocation, so diffusion coverage is attributed per
|
||||
# test case instead of per shard.
|
||||
# - Skips pr-gate, recommend, analyze
|
||||
# - Runs setup-covstub to assemble coverage data and publish it to the
|
||||
# shared disk at a fixed path (/root/.cache/tests/precise-test/coverage)
|
||||
# pr-test-npu-finish is the final gate in the called workflow.
|
||||
|
||||
@@ -337,7 +337,7 @@ jobs:
|
||||
|
||||
multimodal-gen-test-1-npu-a3:
|
||||
needs: [check-changes, pr-gate, set-image-config, base-a-test-npu]
|
||||
if: ${{ !failure() && !cancelled() && inputs.coverage_mode != true && needs.check-changes.outputs.multimodal_gen == 'true' }}
|
||||
if: ${{ !failure() && !cancelled() && (inputs.coverage_mode == true || needs.check-changes.outputs.multimodal_gen == 'true') }}
|
||||
runs-on: linux-aarch64-a3-800t-2
|
||||
strategy:
|
||||
fail-fast: false
|
||||
@@ -385,7 +385,8 @@ jobs:
|
||||
run: python3 scripts/ci/utils/diffusion/prefetch_input_image.py
|
||||
|
||||
- name: Run test
|
||||
timeout-minutes: 60
|
||||
timeout-minutes: ${{ inputs.coverage_mode == true && 150 || 60 }}
|
||||
shell: bash
|
||||
env:
|
||||
SGLANG_USE_MODELSCOPE: true
|
||||
SGLANG_IS_IN_CI: true
|
||||
@@ -395,6 +396,36 @@ jobs:
|
||||
STREAMS_PER_DEVICE: 32
|
||||
SGLANG_DIFFUSION_ARTIFACT_DIR: ${{ github.workspace }}/diffusion-failures
|
||||
run: |
|
||||
# Coverage mode: enumerate this shard's targets with an NPU-only CI
|
||||
# helper (it reuses the shared diffusion runner read-only) and run
|
||||
# each target in its own `coverage run -m pytest` invocation. The
|
||||
# runner normally batches a whole shard into one pytest call, which
|
||||
# cannot be attributed per test case by the precise-test selector.
|
||||
if [[ "${{ inputs.coverage_mode }}" == "true" ]]; then
|
||||
set -euo pipefail
|
||||
pip install coverage==7.8.*
|
||||
# Coverage tracing slows down the Python glue, which shows up in
|
||||
# wall-clock assertions. Relax only the timing tolerances for the
|
||||
# collection run; PR runs (no coverage) keep the strict profile and
|
||||
# the memory/VRAM checks are untouched.
|
||||
export SGLANG_E2E_TOLERANCE=1.0
|
||||
export SGLANG_STAGE_TIME_TOLERANCE=1.0
|
||||
export SGLANG_NON_DENOISE_STAGE_TIME_TOLERANCE=2.0
|
||||
export SGLANG_DENOISE_STEP_TOLERANCE=1.0
|
||||
export SGLANG_DENOISE_AGG_TOLERANCE=1.0
|
||||
python3 scripts/ci/npu/precise-test/list_multimodal_tests.py \
|
||||
--suite 1-npu \
|
||||
--partition-id ${{ matrix.part }} \
|
||||
--total-partitions 2 \
|
||||
-o /tmp/selected_tests.txt
|
||||
echo "Selected test targets:"
|
||||
cat /tmp/selected_tests.txt
|
||||
if [ -s /tmp/selected_tests.txt ]; then
|
||||
bash scripts/ci/npu/precise-test/run_tests_with_coverage.sh $(cat /tmp/selected_tests.txt)
|
||||
fi
|
||||
exit 0
|
||||
fi
|
||||
|
||||
cd python
|
||||
python3 sglang/multimodal_gen/test/run_suite.py \
|
||||
--suite 1-npu \
|
||||
@@ -412,7 +443,7 @@ jobs:
|
||||
|
||||
multimodal-gen-test-4-npu-a3:
|
||||
needs: [check-changes, pr-gate, set-image-config, base-a-test-npu]
|
||||
if: ${{ !failure() && !cancelled() && inputs.coverage_mode != true && needs.check-changes.outputs.multimodal_gen == 'true' }}
|
||||
if: ${{ !failure() && !cancelled() && (inputs.coverage_mode == true || needs.check-changes.outputs.multimodal_gen == 'true') }}
|
||||
runs-on: linux-aarch64-a3-800t-4
|
||||
strategy:
|
||||
fail-fast: false
|
||||
@@ -455,7 +486,8 @@ jobs:
|
||||
python3 scripts/ci/utils/diffusion/prefetch_consistency_gt.py
|
||||
|
||||
- name: Run test
|
||||
timeout-minutes: 60
|
||||
timeout-minutes: ${{ inputs.coverage_mode == true && 150 || 60 }}
|
||||
shell: bash
|
||||
env:
|
||||
SGLANG_USE_MODELSCOPE: true
|
||||
SGLANG_IS_IN_CI: true
|
||||
@@ -465,6 +497,36 @@ jobs:
|
||||
STREAMS_PER_DEVICE: 32
|
||||
SGLANG_DIFFUSION_ARTIFACT_DIR: ${{ github.workspace }}/diffusion-failures
|
||||
run: |
|
||||
# Coverage mode: enumerate this shard's targets with an NPU-only CI
|
||||
# helper (it reuses the shared diffusion runner read-only) and run
|
||||
# each target in its own `coverage run -m pytest` invocation. The
|
||||
# runner normally batches a whole shard into one pytest call, which
|
||||
# cannot be attributed per test case by the precise-test selector.
|
||||
if [[ "${{ inputs.coverage_mode }}" == "true" ]]; then
|
||||
set -euo pipefail
|
||||
pip install coverage==7.8.*
|
||||
# Coverage tracing slows down the Python glue, which shows up in
|
||||
# wall-clock assertions. Relax only the timing tolerances for the
|
||||
# collection run; PR runs (no coverage) keep the strict profile and
|
||||
# the memory/VRAM checks are untouched.
|
||||
export SGLANG_E2E_TOLERANCE=1.0
|
||||
export SGLANG_STAGE_TIME_TOLERANCE=1.0
|
||||
export SGLANG_NON_DENOISE_STAGE_TIME_TOLERANCE=2.0
|
||||
export SGLANG_DENOISE_STEP_TOLERANCE=1.0
|
||||
export SGLANG_DENOISE_AGG_TOLERANCE=1.0
|
||||
python3 scripts/ci/npu/precise-test/list_multimodal_tests.py \
|
||||
--suite 2-npu \
|
||||
--partition-id ${{ matrix.part }} \
|
||||
--total-partitions 2 \
|
||||
-o /tmp/selected_tests.txt
|
||||
echo "Selected test targets:"
|
||||
cat /tmp/selected_tests.txt
|
||||
if [ -s /tmp/selected_tests.txt ]; then
|
||||
bash scripts/ci/npu/precise-test/run_tests_with_coverage.sh $(cat /tmp/selected_tests.txt)
|
||||
fi
|
||||
exit 0
|
||||
fi
|
||||
|
||||
cd python
|
||||
python3 sglang/multimodal_gen/test/run_suite.py \
|
||||
--suite 2-npu \
|
||||
@@ -696,6 +758,8 @@ jobs:
|
||||
base-c-test-acc-16-npu-a3,
|
||||
base-c-test-perf-2-npu-a3,
|
||||
base-c-test-perf-16-npu-a3,
|
||||
multimodal-gen-test-1-npu-a3,
|
||||
multimodal-gen-test-4-npu-a3,
|
||||
]
|
||||
if: ${{ !failure() && !cancelled() && inputs.coverage_mode == true }}
|
||||
runs-on: linux-aarch64-a3-800t-2
|
||||
|
||||
@@ -30,7 +30,17 @@ _PERFORMANCE_TOLERANCE = 0.25
|
||||
|
||||
def _kill_process_tree(proc: subprocess.Popen) -> None:
|
||||
try:
|
||||
os.killpg(os.getpgid(proc.pid), signal.SIGKILL)
|
||||
pgid = os.getpgid(proc.pid)
|
||||
# Coverage collection can only flush a traced subprocess on a graceful
|
||||
# stop (SIGKILL skips atexit). Give the cluster a SIGTERM head start
|
||||
# when coverage is active, then always escalate to SIGKILL.
|
||||
if os.environ.get("COVERAGE_PROCESS_START"):
|
||||
os.killpg(pgid, signal.SIGTERM)
|
||||
try:
|
||||
proc.wait(timeout=30)
|
||||
except subprocess.TimeoutExpired:
|
||||
pass
|
||||
os.killpg(pgid, signal.SIGKILL)
|
||||
except (ProcessLookupError, PermissionError):
|
||||
pass
|
||||
|
||||
|
||||
+127
@@ -0,0 +1,127 @@
|
||||
#!/usr/bin/env python3
|
||||
"""List the diffusion pytest targets of one NPU shard (coverage collection only).
|
||||
|
||||
This is an NPU CI helper: it imports the shared multimodal_gen runner
|
||||
READ-ONLY. The shard composition must match what
|
||||
``python/sglang/multimodal_gen/test/run_suite.py`` would execute, so the LPT
|
||||
assignment is reused instead of reimplemented, but no platform-shared test
|
||||
code is modified.
|
||||
|
||||
Output: one pytest target per line, with absolute paths, e.g.
|
||||
<workspace>/python/sglang/multimodal_gen/test/server/ascend/test_server_1_npu.py::TestDiffusionServerOneNpu::test_diffusion_generation[ernie_image_t2i_1npu]
|
||||
<workspace>/python/sglang/multimodal_gen/test/server/ascend/test_glm_image_distributed.py
|
||||
|
||||
Absolute paths matter: run_tests_with_coverage.sh encodes them into the
|
||||
coverage directory name and test_selector.py recovers the repo-relative test
|
||||
path from the ``__sglang__sglang__`` workspace marker.
|
||||
|
||||
Usage:
|
||||
python3 list_multimodal_tests.py --suite 1-npu \
|
||||
--partition-id 0 --total-partitions 2 -o /tmp/selected_tests.txt
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
# Repo layout: this script lives at <repo>/scripts/ci/npu/precise-test/.
|
||||
SCRIPT_DIR = Path(__file__).resolve().parent
|
||||
REPO_ROOT = SCRIPT_DIR.parents[3]
|
||||
|
||||
# The runner imports sglang (torch / torch_npu), so make the checkout importable
|
||||
# even when the package is not installed in editable mode.
|
||||
sys.path.insert(0, str(REPO_ROOT / "python"))
|
||||
|
||||
from sglang.multimodal_gen.test.runner import ( # noqa: E402
|
||||
diffusion_suite_runner as runner,
|
||||
)
|
||||
from sglang.multimodal_gen.test.runner.pytest_runner import ( # noqa: E402
|
||||
collect_test_items,
|
||||
partition_items_by_index,
|
||||
)
|
||||
|
||||
|
||||
def _case_files(suite: str, server_dir: Path) -> dict:
|
||||
"""Map case id -> absolute test file for a parametrized diffusion suite."""
|
||||
mapping = {}
|
||||
for filename, case_group in runner.PARAMETRIZED_CASE_GROUPS.get(suite, []):
|
||||
file_path = server_dir / filename
|
||||
for case in case_group:
|
||||
mapping[case.id] = file_path
|
||||
return mapping
|
||||
|
||||
|
||||
def _node_targets(file_path: Path, case_ids: list) -> list:
|
||||
"""Expand case ids into absolute pytest node-id targets for one file."""
|
||||
filter_expr = " or ".join(f"[{case_id}]" for case_id in case_ids)
|
||||
targets = []
|
||||
for item in collect_test_items([str(file_path)], filter_expr):
|
||||
# collect_test_items reports node ids relative to pytest's rootdir,
|
||||
# which is not the checkout root. Rebuild them from the absolute file
|
||||
# so the coverage collector can encode the workspace layout.
|
||||
node = item.partition("::")[2]
|
||||
targets.append(f"{file_path}::{node}" if node else str(file_path))
|
||||
return targets
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--suite", required=True, help="Diffusion suite, e.g. 1-npu")
|
||||
parser.add_argument("--partition-id", type=int, default=0)
|
||||
parser.add_argument("--total-partitions", type=int, default=1)
|
||||
parser.add_argument(
|
||||
"-o",
|
||||
"--output",
|
||||
required=True,
|
||||
help="Write the pytest targets (one per line) to this file.",
|
||||
)
|
||||
args = parser.parse_args()
|
||||
|
||||
test_root = Path(runner.__file__).resolve().parents[1]
|
||||
server_dir = test_root / "server"
|
||||
|
||||
if args.suite in runner.PARAMETRIZED_CASE_GROUPS:
|
||||
assignment = runner.build_local_partition_assignment(
|
||||
args.suite, args.partition_id, args.total_partitions
|
||||
)
|
||||
case_files = _case_files(args.suite, server_dir)
|
||||
missing = [cid for cid in assignment.case_ids if cid not in case_files]
|
||||
if missing:
|
||||
print(f"::warning::Cases absent from the {args.suite} config: {missing}")
|
||||
|
||||
targets = []
|
||||
for file_path in sorted(
|
||||
{case_files[cid] for cid in assignment.case_ids if cid in case_files}
|
||||
):
|
||||
case_ids = [
|
||||
cid for cid in assignment.case_ids if case_files.get(cid) == file_path
|
||||
]
|
||||
targets.extend(_node_targets(file_path, case_ids))
|
||||
for standalone_rel in assignment.standalone_files:
|
||||
targets.append(str(server_dir / standalone_rel))
|
||||
elif args.suite in runner.FILE_SUITES:
|
||||
files = [str(server_dir / rel) for rel in runner.FILE_SUITES[args.suite]]
|
||||
targets = partition_items_by_index(
|
||||
files, args.partition_id, args.total_partitions
|
||||
)
|
||||
else:
|
||||
parser.error(f"Unknown diffusion suite: {args.suite!r}")
|
||||
|
||||
unique_targets = []
|
||||
seen = set()
|
||||
for target in targets:
|
||||
if target not in seen:
|
||||
seen.add(target)
|
||||
unique_targets.append(target)
|
||||
|
||||
with open(args.output, "w", encoding="utf-8") as f:
|
||||
for target in unique_targets:
|
||||
f.write(target + "\n")
|
||||
|
||||
print(f"Listed {len(unique_targets)} test target(s) to {args.output}:")
|
||||
for target in unique_targets:
|
||||
print(f" - {target}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -17,7 +17,10 @@ defers to the original SIGKILL implementation for whatever is left.
|
||||
All server teardowns in the coverage suites converge on the single
|
||||
sglang.srt.utils.common.kill_process_tree (re-exported by
|
||||
sglang.srt.utils, bound into sglang.test.test_utils and the ascend e2e
|
||||
utils), so patching that one attribute covers base-b and base-c.
|
||||
utils), so patching that one attribute covers base-b and base-c. The
|
||||
diffusion suites use their own
|
||||
sglang.multimodal_gen.runtime.utils.process.kill_process_tree, patched
|
||||
lazily after collection (see _patch_multimodal_kill_process_tree).
|
||||
"""
|
||||
|
||||
import os
|
||||
@@ -27,11 +30,11 @@ import sys
|
||||
import psutil
|
||||
|
||||
|
||||
def _patch_kill_process_tree():
|
||||
import sglang.srt.utils.common as common
|
||||
def _enabled() -> bool:
|
||||
return os.environ.get("SGLANG_KILL_PROCESS_TREE_GRACEFUL") == "1"
|
||||
|
||||
original = common.kill_process_tree
|
||||
|
||||
def _make_graceful_kill_process_tree(original):
|
||||
graceful_timeout = float(
|
||||
os.environ.get("SGLANG_KILL_PROCESS_TREE_GRACEFUL_TIMEOUT", "5")
|
||||
)
|
||||
@@ -64,18 +67,51 @@ def _patch_kill_process_tree():
|
||||
wait_timeout=wait_timeout,
|
||||
)
|
||||
|
||||
common.kill_process_tree = graceful = kill_process_tree_graceful
|
||||
kill_process_tree_graceful._sglang_ci_graceful = True
|
||||
return kill_process_tree_graceful
|
||||
|
||||
|
||||
def _patch_module_kill_process_tree(module):
|
||||
original = module.kill_process_tree
|
||||
graceful = _make_graceful_kill_process_tree(original)
|
||||
module.kill_process_tree = graceful
|
||||
|
||||
# Refresh modules that already bound the original (the sglang.srt.utils
|
||||
# re-export, anything an early conftest imported). Modules imported
|
||||
# later pick up the patched attribute on their own.
|
||||
# re-export, anything an early conftest imported, the diffusion test
|
||||
# helpers). Modules imported later pick up the patched attribute on
|
||||
# their own.
|
||||
for mod in list(sys.modules.values()):
|
||||
if getattr(mod, "kill_process_tree", None) is original:
|
||||
mod.kill_process_tree = graceful
|
||||
|
||||
|
||||
def _patch_kill_process_tree():
|
||||
import sglang.srt.utils.common as common
|
||||
|
||||
_patch_module_kill_process_tree(common)
|
||||
|
||||
|
||||
def _patch_multimodal_kill_process_tree():
|
||||
"""Patch the diffusion runtime's SIGKILL-only teardown, if it is loaded.
|
||||
|
||||
The diffusion suites tear their servers down via
|
||||
sglang.multimodal_gen.runtime.utils.process.kill_process_tree instead of
|
||||
the srt helper patched above, so subprocess coverage would be lost on the
|
||||
SIGKILL. Unlike srt this module is imported by the diffusion test modules
|
||||
during collection, so patch lazily AFTER collection: importing it here
|
||||
would pull the whole diffusion runtime into every other suite's pytest
|
||||
process for no benefit.
|
||||
"""
|
||||
module = sys.modules.get("sglang.multimodal_gen.runtime.utils.process")
|
||||
if module is None:
|
||||
return
|
||||
if getattr(module.kill_process_tree, "_sglang_ci_graceful", False):
|
||||
return
|
||||
_patch_module_kill_process_tree(module)
|
||||
|
||||
|
||||
def pytest_configure(config):
|
||||
if os.environ.get("SGLANG_KILL_PROCESS_TREE_GRACEFUL") != "1":
|
||||
if not _enabled():
|
||||
return
|
||||
try:
|
||||
_patch_kill_process_tree()
|
||||
@@ -84,3 +120,14 @@ def pytest_configure(config):
|
||||
# outside the tests' own stubs. No servers are launched there, so
|
||||
# skipping the patch is fine — just make it visible in the log.
|
||||
print(f"precise_test_coverage_plugin: patch skipped: {exc!r}")
|
||||
|
||||
|
||||
def pytest_collection_modifyitems(config, items):
|
||||
if not _enabled():
|
||||
return
|
||||
try:
|
||||
_patch_multimodal_kill_process_tree()
|
||||
except Exception as exc:
|
||||
print(
|
||||
f"precise_test_coverage_plugin: multimodal teardown patch skipped: {exc!r}"
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user