Fix PD bootstrap failure handling (#24772)
Co-authored-by: Cheng Wan <chwan@rice.edu>
This commit is contained in:
@@ -608,6 +608,8 @@ class CommonKVReceiver(BaseKVReceiver):
|
|||||||
|
|
||||||
self.prefill_dp_rank = prefill_dp_rank
|
self.prefill_dp_rank = prefill_dp_rank
|
||||||
self._setup_bootstrap_infos()
|
self._setup_bootstrap_infos()
|
||||||
|
if self.conclude_state == KVPoll.Failed:
|
||||||
|
return
|
||||||
self.kv_mgr.update_status(self.bootstrap_room, KVPoll.WaitingForInput)
|
self.kv_mgr.update_status(self.bootstrap_room, KVPoll.WaitingForInput)
|
||||||
|
|
||||||
def _setup_bootstrap_infos(self):
|
def _setup_bootstrap_infos(self):
|
||||||
@@ -650,6 +652,7 @@ class CommonKVReceiver(BaseKVReceiver):
|
|||||||
self.kv_mgr.update_status(
|
self.kv_mgr.update_status(
|
||||||
self.bootstrap_room, KVPoll.Failed
|
self.bootstrap_room, KVPoll.Failed
|
||||||
)
|
)
|
||||||
|
self.bootstrap_infos = None
|
||||||
return
|
return
|
||||||
|
|
||||||
self.bootstrap_infos = bootstrap_infos
|
self.bootstrap_infos = bootstrap_infos
|
||||||
|
|||||||
Reference in New Issue
Block a user