refactor(e2e_test): fix smg ci e2e test code quality (#16664)
This commit is contained in:
@@ -152,13 +152,20 @@ def genai_bench_runner():
|
||||
)
|
||||
timeout = timeout_sec or int(os.environ.get("GENAI_BENCH_TEST_TIMEOUT", "120"))
|
||||
|
||||
proc = subprocess.Popen(
|
||||
cmd,
|
||||
env=os.environ.copy(),
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE,
|
||||
text=True,
|
||||
)
|
||||
try:
|
||||
proc = subprocess.Popen(
|
||||
cmd,
|
||||
env=os.environ.copy(),
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE,
|
||||
text=True,
|
||||
)
|
||||
except FileNotFoundError:
|
||||
pytest.fail(f"genai-bench executable not found at {cli}")
|
||||
except PermissionError:
|
||||
pytest.fail(f"Permission denied executing {cli}")
|
||||
except OSError as e:
|
||||
pytest.fail(f"Failed to start genai-bench: {e}")
|
||||
|
||||
# Start GPU monitor if needed
|
||||
gpu_monitor: GPUMonitor | None = None
|
||||
@@ -172,6 +179,16 @@ def genai_bench_runner():
|
||||
except subprocess.TimeoutExpired:
|
||||
proc.kill()
|
||||
stdout, stderr = proc.communicate()
|
||||
logger.error("genai-bench timed out after %ds", timeout)
|
||||
|
||||
# Log output if process failed or for debugging
|
||||
if proc.returncode != 0:
|
||||
logger.error(
|
||||
"genai-bench exited with code %d\nstdout:\n%s\nstderr:\n%s",
|
||||
proc.returncode,
|
||||
stdout or "(empty)",
|
||||
stderr or "(empty)",
|
||||
)
|
||||
|
||||
try:
|
||||
# Parse and validate results
|
||||
@@ -187,6 +204,16 @@ def genai_bench_runner():
|
||||
gpu_monitor.log_summary()
|
||||
gpu_monitor.assert_thresholds(thresholds)
|
||||
|
||||
except AssertionError:
|
||||
# Log genai-bench output when results not found
|
||||
logger.error(
|
||||
"genai-bench output (returncode=%d):\nstdout:\n%s\nstderr:\n%s",
|
||||
proc.returncode,
|
||||
stdout or "(empty)",
|
||||
stderr or "(empty)",
|
||||
)
|
||||
raise
|
||||
|
||||
finally:
|
||||
_cleanup_procs(kill_procs, drain_delay_sec)
|
||||
if gpu_monitor:
|
||||
|
||||
Reference in New Issue
Block a user