[npu] Kill evalscope session by process group and fix report score parsing (#35988)
Co-authored-by: sglang-npu-bot <sglangnpu@163.com>
This commit is contained in:
co-authored by
sglang-npu-bot
parent
939c00a7e3
commit
8014d9d062
@@ -5,6 +5,7 @@ import logging
|
|||||||
import os
|
import os
|
||||||
import re
|
import re
|
||||||
import shutil
|
import shutil
|
||||||
|
import signal
|
||||||
import subprocess
|
import subprocess
|
||||||
import sys
|
import sys
|
||||||
import threading
|
import threading
|
||||||
@@ -104,6 +105,21 @@ def get_max_retries(datasets):
|
|||||||
return 1
|
return 1
|
||||||
|
|
||||||
|
|
||||||
|
def _kill_evalscope_session(process):
|
||||||
|
"""Kill the whole evalscope session (process.pid is the leader == pgid)."""
|
||||||
|
if process is None:
|
||||||
|
return
|
||||||
|
try:
|
||||||
|
os.killpg(process.pid, signal.SIGKILL)
|
||||||
|
logger.info(f"run_evalscope killed session pgid={process.pid}")
|
||||||
|
except ProcessLookupError:
|
||||||
|
logger.info(f"run_evalscope session pgid={process.pid} already gone")
|
||||||
|
except PermissionError:
|
||||||
|
logger.warning(
|
||||||
|
f"run_evalscope no permission to kill session pgid={process.pid}"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def run_evalscope(
|
def run_evalscope(
|
||||||
host,
|
host,
|
||||||
port,
|
port,
|
||||||
@@ -177,6 +193,12 @@ def run_evalscope(
|
|||||||
text=True,
|
text=True,
|
||||||
bufsize=1,
|
bufsize=1,
|
||||||
shell=True,
|
shell=True,
|
||||||
|
start_new_session=True,
|
||||||
|
)
|
||||||
|
|
||||||
|
logger.info(
|
||||||
|
f"run_evalscope spawned: pid={process.pid} "
|
||||||
|
f"pgid={os.getpgid(process.pid)} cmd={cmd}"
|
||||||
)
|
)
|
||||||
|
|
||||||
output_lines = []
|
output_lines = []
|
||||||
@@ -188,6 +210,13 @@ def run_evalscope(
|
|||||||
|
|
||||||
process.wait()
|
process.wait()
|
||||||
|
|
||||||
|
logger.info(
|
||||||
|
f"run_evalscope finished: pid={process.pid} "
|
||||||
|
f"returncode={process.returncode}"
|
||||||
|
)
|
||||||
|
|
||||||
|
_kill_evalscope_session(process)
|
||||||
|
|
||||||
if process.returncode != 0:
|
if process.returncode != 0:
|
||||||
logger.error(f"Command failed with return code: {process.returncode}")
|
logger.error(f"Command failed with return code: {process.returncode}")
|
||||||
raise subprocess.CalledProcessError(process.returncode, cmd)
|
raise subprocess.CalledProcessError(process.returncode, cmd)
|
||||||
@@ -204,12 +233,10 @@ def run_evalscope(
|
|||||||
try:
|
try:
|
||||||
with open(report_path, "r") as rf:
|
with open(report_path, "r") as rf:
|
||||||
report_data = json.load(rf)
|
report_data = json.load(rf)
|
||||||
for item in report_data:
|
score = report_data.get("score")
|
||||||
score = item.get("score")
|
|
||||||
if score is not None:
|
if score is not None:
|
||||||
metrics["accuracy"] = float(score)
|
metrics["accuracy"] = float(score)
|
||||||
logger.info(f"The Final Accuracy from report: {score}")
|
logger.info(f"The Final Accuracy from report: {score}")
|
||||||
break
|
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
logger.warning(f"Failed to read report file {report_path}: {e}")
|
logger.warning(f"Failed to read report file {report_path}: {e}")
|
||||||
|
|
||||||
@@ -245,11 +272,14 @@ def run_evalscope(
|
|||||||
logger.warning("Process did not terminate gracefully, killing it...")
|
logger.warning("Process did not terminate gracefully, killing it...")
|
||||||
process.kill()
|
process.kill()
|
||||||
logger.info("Process killed")
|
logger.info("Process killed")
|
||||||
|
_kill_evalscope_session(process)
|
||||||
raise
|
raise
|
||||||
|
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
logger.error(f"Error executing command: {e}")
|
logger.error(f"Error executing command: {e}")
|
||||||
process.terminate()
|
process.terminate()
|
||||||
process.wait(timeout=5)
|
process.wait(timeout=5)
|
||||||
|
_kill_evalscope_session(process)
|
||||||
raise
|
raise
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user