Pull test_utils server-launch boilerplate into reusable helpers (#26804)
This commit is contained in:
@@ -755,6 +755,43 @@ def _create_clean_subprocess_env(env: dict) -> dict:
|
|||||||
return child_env
|
return child_env
|
||||||
|
|
||||||
|
|
||||||
|
def _subprocess_popen_with_outputs(
|
||||||
|
command: list,
|
||||||
|
env: Optional[dict],
|
||||||
|
return_stdout_stderr: Optional[tuple],
|
||||||
|
) -> subprocess.Popen:
|
||||||
|
if not return_stdout_stderr:
|
||||||
|
return subprocess.Popen(command, stdout=None, stderr=None, env=env)
|
||||||
|
|
||||||
|
process = subprocess.Popen(
|
||||||
|
command,
|
||||||
|
stdout=subprocess.PIPE,
|
||||||
|
stderr=subprocess.PIPE,
|
||||||
|
env=env,
|
||||||
|
text=True,
|
||||||
|
bufsize=1,
|
||||||
|
)
|
||||||
|
|
||||||
|
def _dump(src, sinks):
|
||||||
|
for line in iter(src.readline, ""):
|
||||||
|
for sink in sinks:
|
||||||
|
sink.write(line)
|
||||||
|
sink.flush()
|
||||||
|
src.close()
|
||||||
|
|
||||||
|
threading.Thread(
|
||||||
|
target=_dump,
|
||||||
|
args=(process.stdout, [return_stdout_stderr[0], sys.stdout]),
|
||||||
|
daemon=True,
|
||||||
|
).start()
|
||||||
|
threading.Thread(
|
||||||
|
target=_dump,
|
||||||
|
args=(process.stderr, [return_stdout_stderr[1], sys.stderr]),
|
||||||
|
daemon=True,
|
||||||
|
).start()
|
||||||
|
return process
|
||||||
|
|
||||||
|
|
||||||
def _launch_server_process(
|
def _launch_server_process(
|
||||||
command: List[str],
|
command: List[str],
|
||||||
env: dict,
|
env: dict,
|
||||||
@@ -777,37 +814,11 @@ def _launch_server_process(
|
|||||||
hf_hub_offline = child_env.get("HF_HUB_OFFLINE", "0")
|
hf_hub_offline = child_env.get("HF_HUB_OFFLINE", "0")
|
||||||
print(f"CI_OFFLINE: Launching server HF_HUB_OFFLINE={hf_hub_offline} model={model}")
|
print(f"CI_OFFLINE: Launching server HF_HUB_OFFLINE={hf_hub_offline} model={model}")
|
||||||
|
|
||||||
if return_stdout_stderr:
|
return _subprocess_popen_with_outputs(
|
||||||
proc = subprocess.Popen(
|
command=command,
|
||||||
command,
|
env=child_env,
|
||||||
stdout=subprocess.PIPE,
|
return_stdout_stderr=return_stdout_stderr,
|
||||||
stderr=subprocess.PIPE,
|
)
|
||||||
env=child_env,
|
|
||||||
text=True,
|
|
||||||
bufsize=1,
|
|
||||||
)
|
|
||||||
|
|
||||||
def _dump(src, sinks):
|
|
||||||
for line in iter(src.readline, ""):
|
|
||||||
for sink in sinks:
|
|
||||||
sink.write(line)
|
|
||||||
sink.flush()
|
|
||||||
src.close()
|
|
||||||
|
|
||||||
threading.Thread(
|
|
||||||
target=_dump,
|
|
||||||
args=(proc.stdout, [return_stdout_stderr[0], sys.stdout]),
|
|
||||||
daemon=True,
|
|
||||||
).start()
|
|
||||||
threading.Thread(
|
|
||||||
target=_dump,
|
|
||||||
args=(proc.stderr, [return_stdout_stderr[1], sys.stderr]),
|
|
||||||
daemon=True,
|
|
||||||
).start()
|
|
||||||
else:
|
|
||||||
proc = subprocess.Popen(command, stdout=None, stderr=None, env=child_env)
|
|
||||||
|
|
||||||
return proc
|
|
||||||
|
|
||||||
|
|
||||||
def _wait_for_server_health(
|
def _wait_for_server_health(
|
||||||
@@ -1024,6 +1035,7 @@ def popen_launch_pd_server(
|
|||||||
api_key: Optional[str] = None,
|
api_key: Optional[str] = None,
|
||||||
other_args: list[str] = (),
|
other_args: list[str] = (),
|
||||||
env: Optional[dict] = None,
|
env: Optional[dict] = None,
|
||||||
|
return_stdout_stderr: Optional[tuple] = None,
|
||||||
):
|
):
|
||||||
_, host, port = base_url.split(":")
|
_, host, port = base_url.split(":")
|
||||||
host = host[2:]
|
host = host[2:]
|
||||||
@@ -1059,9 +1071,11 @@ def popen_launch_pd_server(
|
|||||||
if env is not None:
|
if env is not None:
|
||||||
env = {**os.environ, **env}
|
env = {**os.environ, **env}
|
||||||
|
|
||||||
process = subprocess.Popen(command, stdout=None, stderr=None, env=env)
|
return _subprocess_popen_with_outputs(
|
||||||
|
command=command,
|
||||||
return process
|
env=env,
|
||||||
|
return_stdout_stderr=return_stdout_stderr,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def get_similarities(vec1, vec2):
|
def get_similarities(vec1, vec2):
|
||||||
|
|||||||
Reference in New Issue
Block a user