[weight_cache] Coordinate daemon readiness across ranks (#38828)
This commit is contained in:
@@ -36,6 +36,7 @@ Usage:
|
||||
|
||||
import argparse
|
||||
import dataclasses
|
||||
import datetime
|
||||
import logging
|
||||
import multiprocessing
|
||||
import os
|
||||
@@ -396,8 +397,7 @@ class WeightCacheDaemon:
|
||||
|
||||
logger.info(
|
||||
f"[WeightCacheDaemon gpu={self.gpu_id} tp_rank={self.tp_rank}] "
|
||||
f"Exported {len(self.state_entries)} tensors as IPC handles. "
|
||||
f"Ready to serve."
|
||||
f"Exported {len(self.state_entries)} tensors as IPC handles."
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
@@ -652,9 +652,32 @@ def run_weight_cache_daemon(
|
||||
)
|
||||
|
||||
daemon.load()
|
||||
_await_cluster_ready_to_serve(
|
||||
timeout=resolving_view(server_args).weight_cache_timeout
|
||||
)
|
||||
daemon.serve()
|
||||
|
||||
|
||||
def _await_cluster_ready_to_serve(timeout: int) -> None:
|
||||
if timeout <= 0:
|
||||
# Gloo reads timedelta(0) as its wait-forever sentinel.
|
||||
raise ValueError(f"--weight-cache-timeout must be positive, got {timeout}")
|
||||
|
||||
try:
|
||||
# Gate publication on the full PP×TP daemon world.
|
||||
dist.monitored_barrier(
|
||||
group=get_parallel().world_group.cpu_group,
|
||||
timeout=datetime.timedelta(seconds=timeout),
|
||||
wait_all_ranks=True,
|
||||
)
|
||||
except RuntimeError:
|
||||
logger.exception(
|
||||
"[weight_cache] cluster readiness barrier failed; exiting without serving."
|
||||
)
|
||||
# Process-group teardown can block after a failed collective.
|
||||
os._exit(1)
|
||||
|
||||
|
||||
def spawn_weight_cache_daemon(
|
||||
server_args: "ServerArgs",
|
||||
*,
|
||||
|
||||
Reference in New Issue
Block a user