[PD] Reload NIXL peer metadata after the peer is invalidated (#41691)

Co-authored-by: agent <agent@local>
Co-authored-by: Jimmy Shong <69131491+Jiminator@users.noreply.github.com>
This commit is contained in:
Kevin Flansburg
2026-10-05 00:49:26 +00:00
committed by GitHub
co-authored by agent Jimmy Shong
parent f70e8c68fc
commit 734cf3cf3b
2 changed files with 249 additions and 0 deletions
@@ -72,6 +72,9 @@ except ImportError:
logger = logging.getLogger(__name__)
# Arbitrary; bounds how often an unreachable peer's dlists are rebuilt.
_PEER_RELOAD_MIN_INTERVAL_S = 1.0
GUARD = "NixlMsgGuard".encode("ascii")
KV_MEM_KINDS = {"VRAM", "DRAM"}
@@ -527,6 +530,8 @@ class NixlKVManager(StagingManagerMixin, CommonKVManager):
# peer_name -> (handle, num_slots, head_group_idx)
self.prep_handles_segment_src: Dict[Tuple[int, int, str], Any] = {}
self._num_slots_src: int = 0
self._peer_reload_lock = threading.Lock()
self._peer_reload_times: Dict[str, float] = {}
if self.disaggregation_mode == DisaggregationMode.PREFILL:
if self.kv_args.kv_item_lens:
@@ -1225,6 +1230,7 @@ class NixlKVManager(StagingManagerMixin, CommonKVManager):
room = kv_chunk.room
handles: List[Any] = []
settle_timed_out = False
room_transfer_infos = None
try:
if room not in self.request_status:
logger.debug(
@@ -1575,6 +1581,9 @@ class NixlKVManager(StagingManagerMixin, CommonKVManager):
# drain ACK can follow; discard the target and fall back to
# the timeout.
self.poison_deferred_ack_room(room)
# Settle first; NIXL 1.3.0 undoes a reload when an old handle fails.
# room_transfer_infos survives the sender's clear() of this room.
self._reload_invalidated_peers(room_transfer_infos or {})
def register_buffer_to_engine(self):
self.kv_descs = []
@@ -1649,6 +1658,45 @@ class NixlKVManager(StagingManagerMixin, CommonKVManager):
if self.disaggregation_mode == DisaggregationMode.PREFILL:
self._prepare_payload_xfer(decode_kv_args)
def _reload_invalidated_peers(self, room_transfer_infos: Dict[str, Any]) -> None:
# NIXL drops a peer's metadata after a remote-disconnect error and never
# restores it; reload it from the registration we already hold.
for agent_name in list(room_transfer_infos):
peer_info = self.decode_kv_args_table.get(agent_name)
if peer_info is None:
continue
try:
if self.agent.check_remote_metadata(agent_name):
continue
with self._peer_reload_lock:
if self.agent.check_remote_metadata(agent_name):
continue
now = time.monotonic()
last = self._peer_reload_times.get(agent_name)
if last is not None and now - last < _PEER_RELOAD_MIN_INTERVAL_S:
continue
self._peer_reload_times[agent_name] = now
logger.warning(
"NIXL invalidated remote agent %s; reloading its metadata",
agent_name,
)
# No other thread holds these dlists: every room with this
# peer has the same peer set, so all shard to this worker.
self.prep_handles.pop(agent_name, None)
self.prep_handles_slice_dst.pop(agent_name, None)
peer_info.kv_xfer_segments = None
self.agent.add_remote_agent(peer_info.agent_metadata)
try:
self._prepare_payload_xfer(peer_info)
except Exception:
# Leave the peer invalid so a later failure retries.
self.agent.remove_remote_agent(agent_name)
raise
except Exception:
logger.exception(
"Failed to reload NIXL metadata for remote agent %s", agent_name
)
def _send_kvcache_generic(
self,
peer_name: str,