[npu] Kill evalscope session by process group and fix report score parsing (#35988)

Co-authored-by: sglang-npu-bot <sglangnpu@163.com>
This commit is contained in:
pllimax
2026-08-23 21:31:42 +08:00
committed by GitHub
co-authored by sglang-npu-bot
parent 939c00a7e3
commit 8014d9d062
@@ -5,6 +5,7 @@ import logging
import os
import re
import shutil
import signal
import subprocess
import sys
import threading
@@ -104,6 +105,21 @@ def get_max_retries(datasets):
return 1
def _kill_evalscope_session(process):
"""Kill the whole evalscope session (process.pid is the leader == pgid)."""
if process is None:
return
try:
os.killpg(process.pid, signal.SIGKILL)
logger.info(f"run_evalscope killed session pgid={process.pid}")
except ProcessLookupError:
logger.info(f"run_evalscope session pgid={process.pid} already gone")
except PermissionError:
logger.warning(
f"run_evalscope no permission to kill session pgid={process.pid}"
)
def run_evalscope(
host,
port,
@@ -177,6 +193,12 @@ def run_evalscope(
text=True,
bufsize=1,
shell=True,
start_new_session=True,
)
logger.info(
f"run_evalscope spawned: pid={process.pid} "
f"pgid={os.getpgid(process.pid)} cmd={cmd}"
)
output_lines = []
@@ -188,6 +210,13 @@ def run_evalscope(
process.wait()
logger.info(
f"run_evalscope finished: pid={process.pid} "
f"returncode={process.returncode}"
)
_kill_evalscope_session(process)
if process.returncode != 0:
logger.error(f"Command failed with return code: {process.returncode}")
raise subprocess.CalledProcessError(process.returncode, cmd)
@@ -204,12 +233,10 @@ def run_evalscope(
try:
with open(report_path, "r") as rf:
report_data = json.load(rf)
for item in report_data:
score = item.get("score")
if score is not None:
metrics["accuracy"] = float(score)
logger.info(f"The Final Accuracy from report: {score}")
break
score = report_data.get("score")
if score is not None:
metrics["accuracy"] = float(score)
logger.info(f"The Final Accuracy from report: {score}")
except Exception as e:
logger.warning(f"Failed to read report file {report_path}: {e}")
@@ -245,11 +272,14 @@ def run_evalscope(
logger.warning("Process did not terminate gracefully, killing it...")
process.kill()
logger.info("Process killed")
_kill_evalscope_session(process)
raise
except Exception as e:
logger.error(f"Error executing command: {e}")
process.terminate()
process.wait(timeout=5)
_kill_evalscope_session(process)
raise