diff --git a/python/sglang/srt/weight_cache/daemon.py b/python/sglang/srt/weight_cache/daemon.py index a40af383b8..864e7b104e 100644 --- a/python/sglang/srt/weight_cache/daemon.py +++ b/python/sglang/srt/weight_cache/daemon.py @@ -36,6 +36,7 @@ Usage: import argparse import dataclasses +import datetime import logging import multiprocessing import os @@ -396,8 +397,7 @@ class WeightCacheDaemon: logger.info( f"[WeightCacheDaemon gpu={self.gpu_id} tp_rank={self.tp_rank}] " - f"Exported {len(self.state_entries)} tensors as IPC handles. " - f"Ready to serve." + f"Exported {len(self.state_entries)} tensors as IPC handles." ) @staticmethod @@ -652,9 +652,32 @@ def run_weight_cache_daemon( ) daemon.load() + _await_cluster_ready_to_serve( + timeout=resolving_view(server_args).weight_cache_timeout + ) daemon.serve() +def _await_cluster_ready_to_serve(timeout: int) -> None: + if timeout <= 0: + # Gloo reads timedelta(0) as its wait-forever sentinel. + raise ValueError(f"--weight-cache-timeout must be positive, got {timeout}") + + try: + # Gate publication on the full PPĂ—TP daemon world. + dist.monitored_barrier( + group=get_parallel().world_group.cpu_group, + timeout=datetime.timedelta(seconds=timeout), + wait_all_ranks=True, + ) + except RuntimeError: + logger.exception( + "[weight_cache] cluster readiness barrier failed; exiting without serving." + ) + # Process-group teardown can block after a failed collective. + os._exit(1) + + def spawn_weight_cache_daemon( server_args: "ServerArgs", *,