diff --git a/python/sglang/srt/disaggregation/common/conn.py b/python/sglang/srt/disaggregation/common/conn.py index f619534903..c494553ed8 100644 --- a/python/sglang/srt/disaggregation/common/conn.py +++ b/python/sglang/srt/disaggregation/common/conn.py @@ -8,7 +8,16 @@ import threading import time from collections import defaultdict from contextlib import nullcontext -from typing import Dict, List, NamedTuple, Optional, Set, Tuple, Union +from typing import ( + Dict, + List, + Literal, + NamedTuple, + Optional, + Set, + Tuple, + Union, +) import msgspec import numpy as np @@ -42,9 +51,11 @@ from sglang.srt.disaggregation.utils import ( ) from sglang.srt.environ import envs from sglang.srt.mem_cache.page_interleave import get_kv_shard_group +from sglang.srt.observability.trace import TraceNullContext, TraceReqContext from sglang.srt.runtime_context import ( get_device, get_disagg, + get_observability, get_parallel, get_schedule, get_serving, @@ -275,6 +286,7 @@ class DeferredAbortAckState(msgspec.Struct): class CommonKVManager(BaseKVManager): defer_decode_allocation: bool = False deferred_bootstrap: Optional[DeferredBootstrap] = None + _staging_ctx = None # Wire layout of the prefill->decode terminal status message. The legacy # layout (mooncake, and ascend which inherits it) is three untagged frames @@ -283,6 +295,10 @@ class CommonKVManager(BaseKVManager): # ``[tag, room, status, prefill_rank, reason]``. kv_status_msg_tag: Optional[bytes] = None kv_status_msg_carries_reason: bool = False + sender_trace_module: Optional[str] = None + # Sender reports transfer_latency_s (first enqueue -> Success). Otherwise + # the metric keeps its last-chunk fallback, so dashboards do not move. + reports_transfer_latency: bool = False kv_layout: Optional[str] = None dsv41_spec_layout: Optional[dict] = None @@ -453,7 +469,7 @@ class CommonKVManager(BaseKVManager): self.session_pool: Dict = defaultdict(requests.Session) self.session_pool_lock = threading.Lock() self.addr_to_rooms_tracker: Dict[str, Set[int]] = defaultdict(set) - self.prefill_response_tracker: Dict[int, Set[int]] = defaultdict(set) + self.prefill_response_tracker: Dict[int, Set[int]] = {} # Deferred KV release: room -> current generation and ranks that # ACKed. The generation rejects an old request's late ACK after room # reuse. @@ -489,6 +505,11 @@ class CommonKVManager(BaseKVManager): f"Unsupported DisaggregationMode: {self.disaggregation_mode}" ) + def prepare_send_state_indices( + self, state_indices: Optional[List] + ) -> Optional[List]: + return state_indices + def _should_skip_cp_replicated_state_transfer(self) -> bool: """Whether this prefill rank should omit CP-replicated state. @@ -590,6 +611,23 @@ class CommonKVManager(BaseKVManager): return KVPoll.Bootstrapping return min(statuses) + def build_receiver_registration_message(self) -> List[bytes]: + raise NotImplementedError + + def build_receiver_metadata_message( + self, + *, + bootstrap_room: int, + required_dst_info_num: int, + kv_indices: npt.NDArray[np.int32], + aux_index: Optional[int], + state_indices: Optional[List], + decode_prefix_len: Optional[int], + is_dummy: bool, + device_kv_indices: Optional[npt.NDArray[np.int32]] = None, + ) -> List[bytes]: + raise NotImplementedError + def update_status(self, bootstrap_room: int, status: KVPoll): current = self.request_status.get(bootstrap_room) if current is None: @@ -747,6 +785,51 @@ class CommonKVManager(BaseKVManager): f"{na.to_host_port_str()}: {e}" ) + def send_chunk_ready( + self, + *, + room: int, + chunk_idx: int, + page_start: int, + num_pages: int, + writer_id: str, + targets: List[Tuple[str, int]], + ) -> None: + """Notify decode that a staging chunk RDMA is complete (every chunk; + scatter is arrival-driven).""" + parts = [ + b"CHUNK_READY", + str(room).encode("ascii"), + str(chunk_idx).encode("ascii"), + str(page_start).encode("ascii"), + str(num_pages).encode("ascii"), + writer_id.encode("ascii"), + str(self._prefill_unique_rank()).encode("ascii"), + ] + for endpoint, port in targets: + na = NetworkAddress(endpoint, port) + self._send_multipart_locked(na.to_tcp(), parts, is_ipv6=na.is_ipv6) + + def handle_chunk_ready(self, msg: List[bytes]) -> bool: + if not msg or msg[0] != b"CHUNK_READY": + return False + try: + room, chunk_idx, page_start, num_pages = ( + int(part.decode("ascii")) for part in msg[1:5] + ) + writer_id = msg[5].decode("ascii") + except (IndexError, ValueError, UnicodeDecodeError): + logger.warning("Dropping malformed CHUNK_READY message") + return True + if room not in self.request_status: + return True + handler = self._staging_handler + if handler is None: + logger.warning("CHUNK_READY for room %s without a staging handler", room) + return True + handler.handle_chunk_arrived(room, chunk_idx, page_start, num_pages, writer_id) + return True + def conclude_transfer( self, *, @@ -834,10 +917,13 @@ class CommonKVManager(BaseKVManager): self.update_status(bootstrap_room, KVPoll.WaitingForInput) return if status == KVPoll.Success: - self.prefill_response_tracker[bootstrap_room].add(prefill_rank) + completed_ranks = self.prefill_response_tracker.get(bootstrap_room) + if completed_ranks is None: + return expected_response_num = self.required_prefill_response_num_table.get( bootstrap_room ) + completed_ranks.add(prefill_rank) if expected_response_num is None: logger.warning( "No expected prefill response count for room %s, prefill rank %s", @@ -845,14 +931,10 @@ class CommonKVManager(BaseKVManager): prefill_rank, ) return - if ( - len(self.prefill_response_tracker[bootstrap_room]) - < expected_response_num - ): + if len(completed_ranks) < expected_response_num: return # Tell the staging handler no more chunks are coming, before any - # poller can see Success. Only mooncake gets here: NIXL arms the - # handler from its own notifications, mori has no staging. + # poller can see Success. if self.enable_staging and self._staging_handler is not None: handler = self._staging_handler if handler.is_staging_room(bootstrap_room): @@ -2007,6 +2089,19 @@ class CommonKVSender(BaseKVSender): self._transfer_metric = KVTransferMetric() self._transfer_num_kv_indices = 0 self._transfer_num_state_indices = 0 + self._transfer_start_time: Optional[float] = None + self._early_send_wait_event: Optional[object] = None + self.trace_ctx = TraceNullContext() + if self.kv_mgr.sender_trace_module and get_observability().enable_trace: + self.trace_ctx = TraceReqContext( + rid=str(hex(self.bootstrap_room)), + bootstrap_room=self.bootstrap_room, + role="Sender", + module_name=self.kv_mgr.sender_trace_module, + ) + if not self.trace_ctx.tracing_enable: + self.trace_ctx = TraceNullContext() + self.trace_ctx.trace_req_start() # inner state self.curr_idx = 0 self.init_time: Optional[float] = None @@ -2051,6 +2146,8 @@ class CommonKVSender(BaseKVSender): ) self.kv_mgr.update_status(self.bootstrap_room, KVPoll.Failed) return + # Bootstrap timeout clock starts after the registration above. + self.init_time = time.time() def _register_prefill_dp_rank(self): """Register this request's prefill dp_rank to the bootstrap server.""" @@ -2175,8 +2272,82 @@ class CommonKVSender(BaseKVSender): kv_indices: npt.NDArray[np.int32], state_indices: Optional[List] = None, num_kv_tokens: Optional[int] = None, + indices_event: Optional[object] = None, ): - pass + if self.conclude_state in (KVPoll.Success, KVPoll.Failed): + return + + kv_indices, index_slice, is_last_chunk, should_skip = ( + self._prepare_send_indices(kv_indices, state_indices) + ) + if should_skip: + return + + state_indices = self.kv_mgr.prepare_send_state_indices(state_indices) + if self._transfer_start_time is None and ( + len(kv_indices) > 0 or state_indices is not None + ): + self._transfer_start_time = time.perf_counter() + + # The early-send event belongs to exactly one chunk; the Mooncake and + # Mori workers synchronize on it before reading source KV. + wait_event = self._early_send_wait_event + self._early_send_wait_event = None + self.kv_mgr.add_transfer_request( + self.bootstrap_room, + kv_indices, + index_slice, + is_last_chunk, + aux_index=self.aux_index if is_last_chunk else None, + state_indices=state_indices if is_last_chunk else None, + num_kv_tokens=num_kv_tokens, + wait_event=wait_event, + indices_event=indices_event, + trace_ctx=self.trace_ctx.copy_for_thread(), + ) + self._record_transfer_indices(kv_indices, state_indices) + + def poll(self) -> KVPoll: + return self._poll_with_status(self.kv_mgr.check_status) + + def poll_pp_consensus(self) -> KVPoll: + return self._poll_with_status(self.kv_mgr.check_status_pp_consensus) + + def _poll_with_status(self, check_status) -> KVPoll: + if self.conclude_state is not None: + return self.conclude_state + + if self.bootstrap_room in self.kv_mgr.request_status: + status = check_status(self.bootstrap_room) + else: + status = KVPoll.Failed + # Hold Success until all staging chunks transferred. With deferred + # allocation also wait for worker cleanup (the zero entry is only + # removed after cleanup), before permitting the room to be reused. + if status == KVPoll.Success and ( + self.kv_mgr._staging_outstanding.get(self.bootstrap_room, 0) > 0 + or ( + self.kv_mgr.defer_decode_allocation + and self.bootstrap_room in self.kv_mgr._staging_outstanding + ) + ): + return KVPoll.Transferring + if status == KVPoll.Bootstrapping: + timeout_status = self._check_bootstrap_timeout() + if timeout_status is not None: + status = timeout_status + if status in (KVPoll.Success, KVPoll.Failed): + self.conclude_state = status + if ( + status == KVPoll.Success + and self.kv_mgr.reports_transfer_latency + and self._transfer_start_time is not None + ): + self._transfer_metric.transfer_latency_s = ( + time.perf_counter() - self._transfer_start_time + ) + self.trace_ctx.trace_req_finish() + return status def _check_bootstrap_timeout(self) -> Optional[KVPoll]: start = ( @@ -2214,12 +2385,22 @@ class CommonKVSender(BaseKVSender): == KVPoll.Success, ) self._owns_bootstrap_room = False + if self.conclude_state is None: + status = self.kv_mgr.request_status.get(self.bootstrap_room) + if status in (KVPoll.Success, KVPoll.Failed): + self.conclude_state = status self.kv_mgr.request_status.pop(self.bootstrap_room, None) self.kv_mgr.req_to_decode_prefix_len.pop(self.bootstrap_room, None) self.kv_mgr.transfer_infos.pop(self.bootstrap_room, None) # A sender can be cleared while a counted transfer is still writing: # keep the target for the worker, or ACK now if nothing is in flight. self.kv_mgr._maybe_ack_drained_abort(self.bootstrap_room) + staging_ctx = self.kv_mgr._staging_ctx + if staging_ctx is not None: + staging_ctx.prefetched_rooms.discard(self.bootstrap_room) + for key in list(staging_ctx.prefetch_requested): + if key[0] == self.bootstrap_room: + staging_ctx.prefetch_requested.discard(key) if notification is not None: endpoint, failed = notification self.kv_mgr.send_kv_status_message( @@ -2229,6 +2410,18 @@ class CommonKVSender(BaseKVSender): failure_reason="Prefill bootstrap failed" if failed else None, ) + def failure_exception(self): + self.conclude_state = KVPoll.Failed + self.clear() + with self.kv_mgr.failure_lock: + failure_reason = self.kv_mgr.failure_records.pop(self.bootstrap_room, None) + self.trace_ctx.trace_req_finish() + raise KVTransferError( + self.bootstrap_room, + failure_reason or "Failed due to an unknown reason from another rank", + is_from_another_rank=failure_reason is None, + ) + def abort(self): if self.kv_mgr.deferred_bootstrap is not None and not self._owns_bootstrap_room: return @@ -2238,6 +2431,8 @@ class CommonKVSender(BaseKVSender): ) self.kv_mgr.update_status(self.bootstrap_room, KVPoll.Failed) self.conclude_state = KVPoll.Failed + self.trace_ctx.abort(abort_info={"reason": "Aborted"}) + self.trace_ctx.trace_req_finish() class CommonKVReceiver(BaseKVReceiver): @@ -2278,6 +2473,7 @@ class CommonKVReceiver(BaseKVReceiver): self.conclude_state = KVPoll.Failed return self.kv_mgr.addr_to_rooms_tracker[self.bootstrap_addr].add(self.bootstrap_room) + self.kv_mgr.prefill_response_tracker.setdefault(self.bootstrap_room, set()) self.kv_mgr.update_status(self.bootstrap_room, KVPoll.Bootstrapping) def init(self, prefill_dp_rank: int): @@ -2571,17 +2767,90 @@ class CommonKVReceiver(BaseKVReceiver): return sock, lock def _register_kv_args(self) -> bool: + if self.bootstrap_infos is None: + return False + if not self.bootstrap_infos: + return True + + message = self.kv_mgr.build_receiver_registration_message() + for bootstrap_info in self.bootstrap_infos: + if not self._send_bootstrap_message( + bootstrap_info=bootstrap_info, + message=message, + operation="_register_kv_args", + ): + return False + return True + + def _send_bootstrap_message( + self, + *, + bootstrap_info: dict, + message: List[bytes], + operation: Literal["_register_kv_args", "send_metadata"], + ) -> bool: + try: + sock, lock = self._connect_to_bootstrap_server(bootstrap_info) + with lock: + sock.send_multipart(message) + except zmq.ZMQError: + self.kv_mgr.record_failure( + self.bootstrap_room, + f"{operation} to prefill {bootstrap_info.get('rank_ip')}:{bootstrap_info.get('rank_port')} failed", + ) + self.conclude_state = KVPoll.Failed + self.kv_mgr.update_status(self.bootstrap_room, KVPoll.Failed) + return False return True def send_metadata( self, kv_indices: npt.NDArray[np.int32], aux_index: Optional[int] = None, - state_indices: Optional[List[int]] = None, + state_indices: Optional[List] = None, decode_prefix_len: Optional[int] = None, destination: KVTransferDestination = KVTransferDestination.DEVICE, + device_kv_indices: Optional[npt.NDArray[np.int32]] = None, ): - raise NotImplementedError + if self.bootstrap_infos is None: + self.kv_mgr.record_failure( + self.bootstrap_room, + f"Could not fetch prefill parallel info from bootstrap_addr: {self.bootstrap_addr}", + ) + self.kv_mgr.update_status(self.bootstrap_room, KVPoll.Failed) + return + if destination != KVTransferDestination.DEVICE: + raise NotImplementedError("Host KV destinations are not supported") + + self.chunk_staging_infos = [] + if ( + self.kv_mgr.enable_staging + and self.kv_mgr._staging_ctx.allocator is not None + ): + self.kv_mgr.register_staging_room_bootstrap( + self.bootstrap_room, self.bootstrap_infos, self + ) + + for bootstrap_info in self.bootstrap_infos: + message = self.kv_mgr.build_receiver_metadata_message( + bootstrap_room=self.bootstrap_room, + required_dst_info_num=self.required_dst_info_num, + kv_indices=kv_indices, + aux_index=aux_index, + state_indices=state_indices, + decode_prefix_len=decode_prefix_len, + is_dummy=bootstrap_info["is_dummy"], + device_kv_indices=device_kv_indices, + ) + if not self._send_bootstrap_message( + bootstrap_info=bootstrap_info, + message=message, + operation="send_metadata", + ): + self.invalidate_cached_bootstrap_infos() + return + + self.init_time = time.time() def _check_waiting_timeout(self) -> Optional[KVPoll]: if self.init_time is None: @@ -2625,6 +2894,11 @@ class CommonKVReceiver(BaseKVReceiver): # Preserve the recorded root cause; this is cleanup, not a new abort. self._send_abort_notification() self.abort_notified = True + if self.conclude_state is None: + status = self.kv_mgr.request_status.get(self.bootstrap_room) + self.conclude_state = ( + status if status in (KVPoll.Success, KVPoll.Failed) else KVPoll.Failed + ) self.kv_mgr.request_status.pop(self.bootstrap_room, None) self.kv_mgr.required_prefill_response_num_table.pop(self.bootstrap_room, None) self.kv_mgr.prefill_response_tracker.pop(self.bootstrap_room, None) @@ -2634,16 +2908,25 @@ class CommonKVReceiver(BaseKVReceiver): if self.kv_mgr.defer_decode_allocation: self._owns_bootstrap_room = False + def failure_exception(self): + self.conclude_state = KVPoll.Failed + self.clear() + with self.kv_mgr.failure_lock: + failure_reason = self.kv_mgr.failure_records.pop(self.bootstrap_room, None) + raise KVTransferError( + self.bootstrap_room, + failure_reason or "Failed due to an unknown reason from another rank", + is_from_another_rank=failure_reason is None, + ) + def abort(self): if not self._owns_bootstrap_room: self.conclude_state = KVPoll.Failed return - self.kv_mgr.record_failure( - self.bootstrap_room, - "Aborted by AbortReq.", - ) - self.kv_mgr.update_status(self.bootstrap_room, KVPoll.Failed) - self.conclude_state = KVPoll.Failed + if self.conclude_state != KVPoll.Failed: + self.kv_mgr.record_failure(self.bootstrap_room, "Aborted by AbortReq.") + self.kv_mgr.update_status(self.bootstrap_room, KVPoll.Failed) + self.conclude_state = KVPoll.Failed self.ensure_abort_notified() def ensure_abort_notified(self, *, force_arm: bool = False) -> None: diff --git a/python/sglang/srt/disaggregation/common/staging_handler.py b/python/sglang/srt/disaggregation/common/staging_handler.py index 87bead96a5..5648fedf9b 100644 --- a/python/sglang/srt/disaggregation/common/staging_handler.py +++ b/python/sglang/srt/disaggregation/common/staging_handler.py @@ -312,7 +312,7 @@ class DecodeStagingHandler: num_pages: int, writer_id: str, ) -> bool: - """Process a staging chunk arrival from any transport (NIXL RDMA notif or ZMQ CHUNK_READY). + """Process a staging chunk arrival reported by a ZMQ CHUNK_READY message. Accumulates writer arrivals and submits scatter once all writers for this chunk have reported in. Returns True if scatter was submitted. diff --git a/python/sglang/srt/disaggregation/common/utils.py b/python/sglang/srt/disaggregation/common/utils.py index 9a21ff624f..7946c4aec1 100644 --- a/python/sglang/srt/disaggregation/common/utils.py +++ b/python/sglang/srt/disaggregation/common/utils.py @@ -24,7 +24,6 @@ class TransferKVChunk: is_last_chunk: bool prefill_aux_index: Optional[int] state_indices: Optional[List] - chunk_id: Optional[int] = None num_kv_tokens: Optional[int] = None trace_ctx: Union[TraceReqContext, TraceNullContext] = dataclasses.field( default_factory=TraceNullContext diff --git a/python/sglang/srt/disaggregation/mooncake/conn.py b/python/sglang/srt/disaggregation/mooncake/conn.py index 29e06e6763..f6cd31976c 100644 --- a/python/sglang/srt/disaggregation/mooncake/conn.py +++ b/python/sglang/srt/disaggregation/mooncake/conn.py @@ -13,7 +13,6 @@ from typing import Dict, List, Optional, Set, Tuple, Union import numpy as np import numpy.typing as npt import torch -import zmq from prometheus_client import Counter from sglang.srt.disaggregation.base.conn import KVArgs, KVPoll, StateType @@ -25,7 +24,6 @@ from sglang.srt.disaggregation.common.conn import ( CommonKVManager, CommonKVReceiver, CommonKVSender, - KVTransferError, ) from sglang.srt.disaggregation.common.staging_handler import ( STAGING_WATERMARK_WAIT_S, @@ -251,6 +249,8 @@ def _uses_flat_entry_layout( class MooncakeKVManager(StagingManagerMixin, CommonKVManager): + sender_trace_module = "mooncake" + AUX_DATA_HEADER = b"AUX_DATA" # Implements teardown() below, so runtime PD role switching is supported. supports_role_switch = True @@ -585,24 +585,6 @@ class MooncakeKVManager(StagingManagerMixin, CommonKVManager): return PrefillStagingStrategy(self, staging_buffer) - def _send_chunk_ready(self, req, chunk_idx, kv_chunk, prefill_unique_rank): - """Notify decode that a staging chunk RDMA is complete (every chunk; - scatter is arrival-driven).""" - na = NetworkAddress(req.endpoint, req.dst_port) - self._send_multipart_locked( - na.to_tcp(), - [ - b"CHUNK_READY", - str(req.room).encode("ascii"), - str(chunk_idx).encode("ascii"), - str(kv_chunk.index_slice.start).encode("ascii"), - str(len(kv_chunk.prefill_kv_indices)).encode("ascii"), - req.mooncake_session_id.encode("ascii"), - str(prefill_unique_rank).encode("ascii"), - ], - is_ipv6=na.is_ipv6, - ) - def _do_staging_transfer( self, staging_strategy, @@ -612,7 +594,6 @@ class MooncakeKVManager(StagingManagerMixin, CommonKVManager): chunked_dst_kv_indice, executor, queue, - prefill_unique_rank, ): """Execute staging transfer for one chunk. Returns (ret, deferred). @@ -663,7 +644,14 @@ class MooncakeKVManager(StagingManagerMixin, CommonKVManager): ) return (-1, False) if ret == 0: - self._send_chunk_ready(req, chunk_idx, kv_chunk, prefill_unique_rank) + self.send_chunk_ready( + room=req.room, + chunk_idx=chunk_idx, + page_start=kv_chunk.index_slice.start, + num_pages=len(kv_chunk.prefill_kv_indices), + writer_id=req.mooncake_session_id, + targets=[(req.endpoint, req.dst_port)], + ) return (ret, False) def _prefetch_staging_reqs(self, room: int): @@ -2559,8 +2547,6 @@ class MooncakeKVManager(StagingManagerMixin, CommonKVManager): ) polls = [] dst_ranks_infos = [] - # Unique id per prefill sender so decode's response set size matches expected_response_num. - prefill_unique_rank = self._prefill_unique_rank() # When staging transfer is not yet ready (watermark/allocation pending), # the chunk is re-enqueued and we break out of the req loop to retry later. staging_deferred = False @@ -2707,7 +2693,6 @@ class MooncakeKVManager(StagingManagerMixin, CommonKVManager): chunked_dst_kv_indice, executor, queue, - prefill_unique_rank, ) if deferred: staging_deferred = True @@ -3038,23 +3023,7 @@ class MooncakeKVManager(StagingManagerMixin, CommonKVManager): continue # Staging: prefill notifies a chunk written to staging buffer - if msg[0] == b"CHUNK_READY": - room = int(msg[1].decode("ascii")) - chunk_idx = int(msg[2].decode("ascii")) - page_start = int(msg[3].decode("ascii")) - num_pages = int(msg[4].decode("ascii")) - session_id = msg[5].decode("ascii") - handler = self._staging_handler - assert handler is not None, ( - "CHUNK_READY received before staging handler initialized" - ) - handler.handle_chunk_arrived( - room, - chunk_idx, - page_start, - num_pages, - session_id, - ) + if self.handle_chunk_ready(msg): continue # Staging: prefill pre-requests staging allocation before forward @@ -3091,9 +3060,10 @@ class MooncakeKVManager(StagingManagerMixin, CommonKVManager): aux_index: Optional[int] = None, state_indices: Optional[List] = None, num_kv_tokens: Optional[int] = None, - trace_ctx: Optional[Union[TraceReqContext, TraceNullContext]] = None, + *, wait_event: Optional[object] = None, indices_event: Optional[object] = None, + trace_ctx: Optional[Union[TraceReqContext, TraceNullContext]] = None, ): assert self.disaggregation_mode == DisaggregationMode.PREFILL assert not is_last_chunk or (is_last_chunk and aux_index is not None) @@ -3215,51 +3185,103 @@ class MooncakeKVManager(StagingManagerMixin, CommonKVManager): ): self._run_one_probe_pass() - -class MooncakeFailureExceptionMixin: - """Shared `failure_exception` for the Mooncake sender and receiver. - - Both sides conclude a failed room identically: latch Failed, clear local - state, then raise with the recorded reason -- or, when no reason was - recorded locally, report it as propagated from another rank. Expects the - concrete class to provide ``conclude_state``, ``clear()``, - ``bootstrap_room`` and ``kv_mgr``. - """ - - def failure_exception(self): - # A room with no locally recorded reason failed on another rank. - if self.conclude_state is None: - self.conclude_state = KVPoll.Failed - - self.clear() - - with self.kv_mgr.failure_lock: - failure_reason = self.kv_mgr.failure_records.pop(self.bootstrap_room, None) - is_propagated = failure_reason is None - if is_propagated: - failure_reason = "Failed due to an unknown reason from another rank" - raise KVTransferError( - self.bootstrap_room, failure_reason, is_from_another_rank=is_propagated + def build_receiver_registration_message(self) -> List[bytes]: + session_id = self.get_session_id() + packed_kv_data_ptrs = b"".join( + struct.pack("Q", ptr) for ptr in self.kv_args.kv_data_ptrs + ) + packed_aux_data_ptrs = b"".join( + struct.pack("Q", ptr) for ptr in self.kv_args.aux_data_ptrs + ) + packed_state_data_ptrs = pack_int_lists(self.kv_args.state_data_ptrs, "Q") + packed_state_item_lens = pack_int_lists(self.kv_args.state_item_lens, "I") + packed_state_dim_per_tensor = pack_int_lists( + getattr(self.kv_args, "state_dim_per_tensor", []) or [], "I" + ) + packed_state_layer_ids = pack_int_lists(self.kv_args.state_layer_ids, "I") + packed_kv_layer_ids = b"".join( + struct.pack("I", layer_id) for layer_id in self.kv_args.kv_layer_ids + ) + # Note(shangming): No need to add pp rank here since decode pp size should be equal to prefill pp size or 1 + tp_rank = self.kv_args.engine_rank + # Some pools have no full-token contiguous KV (kv_item_lens empty) + # and ship per-pool instead, so report 0. + kv_item_len = self.kv_args.kv_item_lens[0] if self.kv_args.kv_item_lens else 0 + dst_tp_rank = str(tp_rank).encode("ascii") + dst_attn_tp_size = str(self.attn_tp_size).encode("ascii") + dst_kv_item_len = str(kv_item_len).encode("ascii") + dst_dcp_size = str(self.dcp_size).encode("ascii") + dst_dcp_rank = str(self.dcp_rank).encode("ascii") + if self.enable_staging and self._staging_ctx.allocator is not None: + _alloc = self._staging_ctx.allocator + packed_staging_base_ptr = struct.pack("Q", _alloc.get_base_ptr()) + staging_total_size_str = str(_alloc.get_total_size()).encode("ascii") + else: + packed_staging_base_ptr = b"" + staging_total_size_str = b"" + staging_slots = getattr(self, "kv_buffer_tensors", None) or {} + packed_staging_slot_layer_ids = b"".join( + struct.pack("Q", layer_id) + for layer_id in (staging_slots.get("slot_layer_ids") or []) ) + return [ + "None".encode("ascii"), + self.local_ip.encode("ascii"), + str(self.rank_port).encode("ascii"), + session_id.encode("ascii"), + packed_kv_data_ptrs, + packed_aux_data_ptrs, + packed_state_data_ptrs, + dst_tp_rank, + dst_attn_tp_size, + dst_kv_item_len, + packed_state_item_lens, + packed_state_dim_per_tensor, + packed_kv_layer_ids, + packed_state_layer_ids, + packed_staging_base_ptr, + staging_total_size_str, + dst_dcp_size, + dst_dcp_rank, + packed_staging_slot_layer_ids, + struct.pack( + f"{len(self.kv_args.kv_item_lens)}Q", + *self.kv_args.kv_item_lens, + ), + ] -class MooncakeKVSender(MooncakeFailureExceptionMixin, CommonKVSender): - def __init__( + def build_receiver_metadata_message( self, - mgr: MooncakeKVManager, - bootstrap_addr: str, + *, bootstrap_room: int, - req_has_disagg_prefill_dp_rank: bool = False, - ): - super().__init__( - mgr, - bootstrap_addr, - bootstrap_room, - req_has_disagg_prefill_dp_rank, - ) - self.init_time = time.time() - self._init_trace_ctx() + required_dst_info_num: int, + kv_indices: npt.NDArray[np.int32], + aux_index: Optional[int], + state_indices: Optional[List], + decode_prefix_len: Optional[int], + is_dummy: bool, + device_kv_indices: Optional[npt.NDArray[np.int32]] = None, + ) -> List[bytes]: + return [ + str(bootstrap_room).encode("ascii"), + self.local_ip.encode("ascii"), + str(self.rank_port).encode("ascii"), + self.get_session_id().encode("ascii"), + kv_indices.tobytes() if not is_dummy else b"", + str(aux_index).encode("ascii") if not is_dummy else b"", + pack_int_lists(state_indices, "i") + if not is_dummy and state_indices + else b"", + str(required_dst_info_num).encode("ascii"), + str(decode_prefix_len or 0).encode("ascii"), + np.asarray(device_kv_indices, dtype=np.int32).tobytes() + if not is_dummy and device_kv_indices is not None + else b"", + ] + +class MooncakeKVSender(CommonKVSender): supports_host_tensor_indices = True @mooncake_trace_func(MooncakeRequestStage.MOONCAKE_SEND) @@ -3270,260 +3292,16 @@ class MooncakeKVSender(MooncakeFailureExceptionMixin, CommonKVSender): num_kv_tokens: Optional[int] = None, indices_event: Optional[object] = None, ): - kv_indices, index_slice, is_last_chunk, should_skip = ( - self._prepare_send_indices(kv_indices, state_indices) + super().send( + kv_indices=kv_indices, + state_indices=state_indices, + num_kv_tokens=num_kv_tokens, + indices_event=indices_event, ) - if should_skip: - return - - wait_event = getattr(self, "_early_send_wait_event", None) - self._early_send_wait_event = None - if not is_last_chunk: - self.kv_mgr.add_transfer_request( - self.bootstrap_room, - kv_indices, - index_slice, - False, - num_kv_tokens=num_kv_tokens, - trace_ctx=self.trace_ctx.copy_for_thread(), - wait_event=wait_event, - indices_event=indices_event, - ) - else: - self.kv_mgr.add_transfer_request( - self.bootstrap_room, - kv_indices, - index_slice, - True, - aux_index=self.aux_index, - state_indices=state_indices, - num_kv_tokens=num_kv_tokens, - trace_ctx=self.trace_ctx.copy_for_thread(), - wait_event=wait_event, - indices_event=indices_event, - ) - self._record_transfer_indices(kv_indices, state_indices) - - def poll(self) -> KVPoll: - return self._poll_with_status(self.kv_mgr.check_status) - - def poll_pp_consensus(self) -> KVPoll: - return self._poll_with_status(self.kv_mgr.check_status_pp_consensus) - - def _poll_with_status(self, check_status) -> KVPoll: - if self.conclude_state is not None: - return self.conclude_state - - status = check_status(self.bootstrap_room) - # Hold Success until all staging chunks transferred. With deferred - # allocation also wait for worker cleanup (the zero entry is only - # removed after cleanup), before permitting the room to be reused. - if status == KVPoll.Success and ( - self.kv_mgr._staging_outstanding.get(self.bootstrap_room, 0) > 0 - or ( - self.kv_mgr.defer_decode_allocation - and self.bootstrap_room in self.kv_mgr._staging_outstanding - ) - ): - return KVPoll.Transferring - if status in (KVPoll.Success, KVPoll.Failed): - self.conclude_state = status - self.trace_ctx.trace_req_finish() - elif status == KVPoll.Bootstrapping: - timeout_result = self._check_bootstrap_timeout() - if timeout_result is not None: - return timeout_result - - return status - - def _init_trace_ctx(self): - if self.kv_mgr.enable_trace: - self.trace_ctx = TraceReqContext( - rid=str(hex(self.bootstrap_room)), - bootstrap_room=self.bootstrap_room, - role="Sender", - module_name="mooncake", - ) - if not self.trace_ctx.tracing_enable: - self.trace_ctx = TraceNullContext() - else: - self.trace_ctx = TraceNullContext() - - self.trace_ctx.trace_req_start() - - def abort(self): - super().abort() - self.trace_ctx.abort(abort_info={"reason": "Aborted"}) - self.trace_ctx.trace_req_finish() -class MooncakeKVReceiver(MooncakeFailureExceptionMixin, CommonKVReceiver): - def __init__( - self, - mgr: MooncakeKVManager, - bootstrap_addr: str, - bootstrap_room: Optional[int] = None, - ): - self.session_id = mgr.get_session_id() - self.init_time = None - super().__init__(mgr, bootstrap_addr, bootstrap_room) - - def _register_kv_args(self) -> bool: - for bootstrap_info in self.bootstrap_infos: - packed_kv_data_ptrs = b"".join( - struct.pack("Q", ptr) for ptr in self.kv_mgr.kv_args.kv_data_ptrs - ) - packed_aux_data_ptrs = b"".join( - struct.pack("Q", ptr) for ptr in self.kv_mgr.kv_args.aux_data_ptrs - ) - packed_state_data_ptrs = pack_int_lists( - self.kv_mgr.kv_args.state_data_ptrs, "Q" - ) - packed_state_item_lens = pack_int_lists( - self.kv_mgr.kv_args.state_item_lens, "I" - ) - packed_state_dim_per_tensor = pack_int_lists( - getattr(self.kv_mgr.kv_args, "state_dim_per_tensor", []) or [], "I" - ) - packed_state_layer_ids = pack_int_lists( - self.kv_mgr.kv_args.state_layer_ids, "I" - ) - packed_kv_layer_ids = b"".join( - struct.pack("I", layer_id) - for layer_id in self.kv_mgr.kv_args.kv_layer_ids - ) - # Note(shangming): No need to add pp rank here since decode pp size should be equal to prefill pp size or 1 - tp_rank = self.kv_mgr.kv_args.engine_rank - # Some pools have no full-token contiguous KV (kv_item_lens empty) - # and ship per-pool instead, so report 0. - kv_item_len = ( - self.kv_mgr.kv_args.kv_item_lens[0] - if self.kv_mgr.kv_args.kv_item_lens - else 0 - ) - dst_tp_rank = str(tp_rank).encode("ascii") - dst_attn_tp_size = str(self.kv_mgr.attn_tp_size).encode("ascii") - dst_kv_item_len = str(kv_item_len).encode("ascii") - dst_dcp_size = str(self.kv_mgr.dcp_size).encode("ascii") - dst_dcp_rank = str(self.kv_mgr.dcp_rank).encode("ascii") - if ( - self.kv_mgr.enable_staging - and self.kv_mgr._staging_ctx.allocator is not None - ): - _alloc = self.kv_mgr._staging_ctx.allocator - packed_staging_base_ptr = struct.pack("Q", _alloc.get_base_ptr()) - staging_total_size_str = str(_alloc.get_total_size()).encode("ascii") - else: - packed_staging_base_ptr = b"" - staging_total_size_str = b"" - staging_slots = getattr(self.kv_mgr, "kv_buffer_tensors", None) or {} - packed_staging_slot_layer_ids = b"".join( - struct.pack("Q", layer_id) - for layer_id in (staging_slots.get("slot_layer_ids") or []) - ) - - try: - sock, lock = self._connect_to_bootstrap_server(bootstrap_info) - with lock: - sock.send_multipart( - [ - "None".encode("ascii"), - self.kv_mgr.local_ip.encode("ascii"), - str(self.kv_mgr.rank_port).encode("ascii"), - self.session_id.encode("ascii"), - packed_kv_data_ptrs, - packed_aux_data_ptrs, - packed_state_data_ptrs, - dst_tp_rank, - dst_attn_tp_size, - dst_kv_item_len, - packed_state_item_lens, - packed_state_dim_per_tensor, - packed_kv_layer_ids, - packed_state_layer_ids, - packed_staging_base_ptr, - staging_total_size_str, - dst_dcp_size, - dst_dcp_rank, - packed_staging_slot_layer_ids, - struct.pack( - f"{len(self.kv_mgr.kv_args.kv_item_lens)}Q", - *self.kv_mgr.kv_args.kv_item_lens, - ), - ] - ) - except zmq.ZMQError: - self.kv_mgr.record_failure( - self.bootstrap_room, - f"_register_kv_args to prefill {bootstrap_info.get('rank_ip')}:{bootstrap_info.get('rank_port')} failed", - ) - self.conclude_state = KVPoll.Failed - self.kv_mgr.update_status(self.bootstrap_room, KVPoll.Failed) - return False - return True - - def send_metadata( - self, - kv_indices: npt.NDArray[np.int32], - aux_index: Optional[int] = None, - state_indices: Optional[List] = None, - decode_prefix_len: Optional[int] = None, - device_kv_indices: Optional[npt.NDArray[np.int32]] = None, - ): - if self.bootstrap_infos is None: - self.kv_mgr.record_failure( - self.bootstrap_room, - f"Could not fetch prefill parallel info from bootstrap_addr: {self.bootstrap_addr}", - ) - self.kv_mgr.update_status(self.bootstrap_room, KVPoll.Failed) - return - - self.chunk_staging_infos = [] - if ( - self.kv_mgr.enable_staging - and self.kv_mgr._staging_ctx.allocator is not None - ): - self.kv_mgr.register_staging_room_bootstrap( - self.bootstrap_room, self.bootstrap_infos, self - ) - - for bootstrap_info in self.bootstrap_infos: - is_dummy = bootstrap_info["is_dummy"] - try: - sock, lock = self._connect_to_bootstrap_server(bootstrap_info) - with lock: - sock.send_multipart( - [ - str(self.bootstrap_room).encode("ascii"), - self.kv_mgr.local_ip.encode("ascii"), - str(self.kv_mgr.rank_port).encode("ascii"), - self.session_id.encode("ascii"), - kv_indices.tobytes() if not is_dummy else b"", - str(aux_index).encode("ascii") if not is_dummy else b"", - ( - pack_int_lists(state_indices, "i") - if not is_dummy and state_indices - else b"" - ), - str(self.required_dst_info_num).encode("ascii"), - str(decode_prefix_len or 0).encode("ascii"), - ( - np.asarray(device_kv_indices, dtype=np.int32).tobytes() - if not is_dummy and device_kv_indices is not None - else b"" - ), - ] - ) - except zmq.ZMQError: - self.invalidate_cached_bootstrap_infos() - self.kv_mgr.record_failure( - self.bootstrap_room, - f"send_metadata to prefill {bootstrap_info.get('rank_ip')}:{bootstrap_info.get('rank_port')} failed", - ) - self.conclude_state = KVPoll.Failed - self.kv_mgr.update_status(self.bootstrap_room, KVPoll.Failed) - return - self.init_time = time.time() +class MooncakeKVReceiver(CommonKVReceiver): + pass class MooncakeKVBootstrapServer(CommonKVBootstrapServer): diff --git a/python/sglang/srt/disaggregation/mori/conn.py b/python/sglang/srt/disaggregation/mori/conn.py index c52f5839a6..6700bf59b9 100644 --- a/python/sglang/srt/disaggregation/mori/conn.py +++ b/python/sglang/srt/disaggregation/mori/conn.py @@ -36,7 +36,6 @@ from sglang.srt.disaggregation.common.conn import ( CommonKVManager, CommonKVReceiver, CommonKVSender, - KVTransferError, ) from sglang.srt.disaggregation.common.utils import ( AuxDataCodec, @@ -785,6 +784,13 @@ class MoriKVManager(CommonKVManager): return f"KV transfer failed: {status.Message()}" return "KV transfer failed due to unknown reason" + def prepare_send_state_indices( + self, state_indices: Optional[List] + ) -> Optional[List]: + if self._should_skip_cp_replicated_state_transfer(): + return None + return _normalize_state_indices_per_component(state_indices) + def add_transfer_request( self, bootstrap_room: int, @@ -794,7 +800,10 @@ class MoriKVManager(CommonKVManager): aux_index: Optional[int] = None, state_indices: Optional[List] = None, num_kv_tokens: Optional[int] = None, + *, wait_event: Optional[object] = None, + indices_event: Optional[object] = None, + trace_ctx=None, ) -> None: assert self.disaggregation_mode == DisaggregationMode.PREFILL assert not is_last_chunk or (is_last_chunk and aux_index is not None) @@ -2056,95 +2065,95 @@ class MoriKVManager(CommonKVManager): return result_statuses + def build_receiver_registration_message(self) -> List[bytes]: + engine_desc_blob = self.engine_desc.pack() + packed_kv_descs = _pack_mem_desc_list(self.kv_mem_descs) + packed_aux_descs = _pack_mem_desc_list(self.aux_mem_descs) + packed_state_descs = _pack_mem_desc_lists(self.state_mem_descs) + gpu_id = str(self.kv_args.gpu_id).encode("ascii") + decode_tp_size = str(self.attn_tp_size).encode("ascii") + decode_tp_rank = str(self.kv_args.engine_rank).encode("ascii") + kv_item_len = str(self.kv_args.kv_item_lens[0]).encode("ascii") + packed_state_item_lens = pack_int_lists(self.kv_args.state_item_lens, "I") + packed_state_dim_per_tensor = pack_int_lists( + self.kv_args.state_dim_per_tensor, "I" + ) + packed_state_layer_ids = pack_int_lists(self.kv_args.state_layer_ids, "I") + packed_kv_layer_ids = struct.pack( + f"<{len(self.kv_args.kv_layer_ids)}I", + *self.kv_args.kv_layer_ids, + ) + dcp_size = str(self.dcp_size).encode("ascii") + dcp_rank = str(self.dcp_rank).encode("ascii") + + return [ + MORI_GUARD, + "None".encode("ascii"), + self.local_ip.encode("ascii"), + str(self.rank_port).encode("ascii"), + engine_desc_blob, + packed_kv_descs, + packed_aux_descs, + packed_state_descs, + gpu_id, + decode_tp_size, + decode_tp_rank, + kv_item_len, + packed_state_item_lens, + packed_state_dim_per_tensor, + dcp_size, + dcp_rank, + packed_state_layer_ids, + packed_kv_layer_ids, + ] + + def build_receiver_metadata_message( + self, + *, + bootstrap_room: int, + required_dst_info_num: int, + kv_indices: npt.NDArray[np.int32], + aux_index: Optional[int], + state_indices: Optional[List], + decode_prefix_len: Optional[int], + is_dummy: bool, + device_kv_indices: Optional[npt.NDArray[np.int32]] = None, + ) -> List[bytes]: + if is_dummy: + kv_indices_bytes = aux_bytes = state_bytes = b"" + else: + kv_indices_bytes = ( + np.asarray(kv_indices, dtype=np.int32).tobytes() + if kv_indices.size + else b"" + ) + aux_bytes = str(aux_index).encode("ascii") if aux_index is not None else b"" + normalized_state = _normalize_state_indices_per_component(state_indices) + state_bytes = ( + _pack_state_indices(normalized_state) + if normalized_state is not None + else b"" + ) + decode_prefix_bytes = ( + str(int(decode_prefix_len)).encode("ascii") + if decode_prefix_len is not None and decode_prefix_len > 0 + else b"" + ) + return [ + MORI_GUARD, + str(bootstrap_room).encode("ascii"), + self.local_ip.encode("ascii"), + str(self.rank_port).encode("ascii"), + self.engine_desc.key.encode("ascii"), + kv_indices_bytes, + aux_bytes, + state_bytes, + str(required_dst_info_num).encode("ascii"), + decode_prefix_bytes, + ] + class MoriKVSender(CommonKVSender): - def __init__( - self, - mgr: MoriKVManager, - bootstrap_addr: str, - bootstrap_room: int, - req_has_disagg_prefill_dp_rank: bool = False, - ): - super().__init__( - mgr, - bootstrap_addr, - bootstrap_room, - req_has_disagg_prefill_dp_rank, - ) - self.conclude_state: Optional[KVPoll] = None - self.init_time = time.time() - - def send( - self, - kv_indices: npt.NDArray[np.int32], - state_indices: Optional[List] = None, - num_kv_tokens: Optional[int] = None, - ): - kv_indices, index_slice, is_last_chunk, should_skip = ( - self._prepare_send_indices(kv_indices, state_indices) - ) - if should_skip: - return - - transfer_state_indices = ( - None - if self.kv_mgr._should_skip_cp_replicated_state_transfer() - else state_indices - ) - normalized_state = ( - _normalize_state_indices_per_component(transfer_state_indices) - if is_last_chunk - else None - ) - self._record_transfer_indices(kv_indices, transfer_state_indices) - wait_event = getattr(self, "_early_send_wait_event", None) - self._early_send_wait_event = None - - if not is_last_chunk: - self.kv_mgr.add_transfer_request( - self.bootstrap_room, - kv_indices, - index_slice, - False, - num_kv_tokens=num_kv_tokens, - wait_event=wait_event, - ) - else: - self.kv_mgr.add_transfer_request( - self.bootstrap_room, - kv_indices, - index_slice, - True, - aux_index=self.aux_index, - state_indices=normalized_state, - num_kv_tokens=num_kv_tokens, - wait_event=wait_event, - ) - - def poll(self) -> KVPoll: - return self._poll_with_status(self.kv_mgr.check_status) - - def poll_pp_consensus(self) -> KVPoll: - return self._poll_with_status(self.kv_mgr.check_status_pp_consensus) - - def _poll_with_status(self, check_status) -> KVPoll: - if self.conclude_state is not None: - return self.conclude_state - - if self.bootstrap_room not in self.kv_mgr.request_status: - self.conclude_state = KVPoll.Failed - return self.conclude_state - - status = check_status(self.bootstrap_room) - if status == KVPoll.Bootstrapping: - timeout_result = self._check_bootstrap_timeout() - if timeout_result is not None: - self.conclude_state = timeout_result - return timeout_result - if status in (KVPoll.Success, KVPoll.Failed): - self.conclude_state = status - return status - def clear(self) -> None: if self.kv_mgr.enable_deferred_decode_kv_release: with self.kv_mgr._abort_ack_lock: @@ -2152,200 +2161,9 @@ class MoriKVSender(CommonKVSender): else: super().clear() - def failure_exception(self): - if self.conclude_state is None: - self.conclude_state = KVPoll.Failed - - self.clear() - - with self.kv_mgr.failure_lock: - failure_reason = self.kv_mgr.failure_records.pop(self.bootstrap_room, None) - is_propagated = failure_reason is None - if is_propagated: - failure_reason = "KV transfer failed" - raise KVTransferError( - self.bootstrap_room, failure_reason, is_from_another_rank=is_propagated - ) - class MoriKVReceiver(CommonKVReceiver): - def __init__( - self, - mgr: MoriKVManager, - bootstrap_addr: str, - bootstrap_room: Optional[int] = None, - ): - super().__init__(mgr, bootstrap_addr, bootstrap_room) - self.init_time: Optional[float] = None - - def init( - self, - prefill_dp_rank: int, - ): - super().init(prefill_dp_rank) - - def _register_kv_args(self) -> bool: - if self.bootstrap_infos is None: - return False - engine_desc_blob = self.kv_mgr.engine_desc.pack() - packed_kv_descs = _pack_mem_desc_list(self.kv_mgr.kv_mem_descs) - packed_aux_descs = _pack_mem_desc_list(self.kv_mgr.aux_mem_descs) - packed_state_descs = _pack_mem_desc_lists(self.kv_mgr.state_mem_descs) - gpu_id = str(self.kv_mgr.kv_args.gpu_id).encode("ascii") - decode_tp_size = str(self.kv_mgr.attn_tp_size).encode("ascii") - decode_tp_rank = str(self.kv_mgr.kv_args.engine_rank).encode("ascii") - kv_item_len = str(self.kv_mgr.kv_args.kv_item_lens[0]).encode("ascii") - packed_state_item_lens = pack_int_lists( - self.kv_mgr.kv_args.state_item_lens, "I" - ) - packed_state_dim_per_tensor = pack_int_lists( - self.kv_mgr.kv_args.state_dim_per_tensor, "I" - ) - packed_state_layer_ids = pack_int_lists( - self.kv_mgr.kv_args.state_layer_ids, "I" - ) - packed_kv_layer_ids = struct.pack( - f"<{len(self.kv_mgr.kv_args.kv_layer_ids)}I", - *self.kv_mgr.kv_args.kv_layer_ids, - ) - dcp_size = str(self.kv_mgr.dcp_size).encode("ascii") - dcp_rank = str(self.kv_mgr.dcp_rank).encode("ascii") - - for bootstrap_info in self.bootstrap_infos: - try: - sock, lock = self._connect_to_bootstrap_server(bootstrap_info) - with lock: - sock.send_multipart( - [ - MORI_GUARD, - "None".encode("ascii"), - self.kv_mgr.local_ip.encode("ascii"), - str(self.kv_mgr.rank_port).encode("ascii"), - engine_desc_blob, - packed_kv_descs, - packed_aux_descs, - packed_state_descs, - gpu_id, - decode_tp_size, - decode_tp_rank, - kv_item_len, - packed_state_item_lens, - packed_state_dim_per_tensor, - dcp_size, - dcp_rank, - packed_state_layer_ids, - packed_kv_layer_ids, - ] - ) - except zmq.ZMQError: - self.kv_mgr.record_failure( - self.bootstrap_room, - f"_register_kv_args to prefill {bootstrap_info.get('rank_ip')}:{bootstrap_info.get('rank_port')} failed", - ) - self.conclude_state = KVPoll.Failed - self.kv_mgr.update_status(self.bootstrap_room, KVPoll.Failed) - return False - return True - - def send_metadata( - self, - kv_indices: npt.NDArray[np.int32], - aux_index: Optional[int] = None, - state_indices: Optional[List] = None, - decode_prefix_len: Optional[int] = None, - ): - if self.bootstrap_infos is None or self.bootstrap_room is None: - return - - kv_indices_bytes = ( - np.asarray(kv_indices, dtype=np.int32).tobytes() if kv_indices.size else b"" - ) - aux_bytes = str(aux_index).encode("ascii") if aux_index is not None else b"" - normalized_state = _normalize_state_indices_per_component(state_indices) - - decode_prefix_bytes = ( - str(int(decode_prefix_len)).encode("ascii") - if decode_prefix_len is not None and decode_prefix_len > 0 - else b"" - ) - - for bootstrap_info in self.bootstrap_infos: - is_dummy = bootstrap_info.get("is_dummy", False) - if not is_dummy and normalized_state is not None: - state_bytes = _pack_state_indices(normalized_state) - else: - state_bytes = b"" - try: - sock, lock = self._connect_to_bootstrap_server(bootstrap_info) - with lock: - sock.send_multipart( - [ - MORI_GUARD, - str(self.bootstrap_room).encode("ascii"), - self.kv_mgr.local_ip.encode("ascii"), - str(self.kv_mgr.rank_port).encode("ascii"), - self.kv_mgr.engine_desc.key.encode("ascii"), - kv_indices_bytes if not is_dummy else b"", - aux_bytes if not is_dummy else b"", - state_bytes, - str(self.required_dst_info_num).encode("ascii"), - decode_prefix_bytes, - ] - ) - except zmq.ZMQError: - self.invalidate_cached_bootstrap_infos() - self.kv_mgr.record_failure( - self.bootstrap_room, - f"send_metadata to prefill {bootstrap_info.get('rank_ip')}:{bootstrap_info.get('rank_port')} failed", - ) - self.conclude_state = KVPoll.Failed - self.kv_mgr.update_status(self.bootstrap_room, KVPoll.Failed) - return - self.init_time = time.time() - - def poll(self) -> KVPoll: - if self.conclude_state is not None: - return self.conclude_state - - status = self.kv_mgr.check_status(self.bootstrap_room) - if status in (KVPoll.Success, KVPoll.Failed): - self.conclude_state = status - return status - - if status == KVPoll.WaitingForInput: - timeout_result = self._check_waiting_timeout() - if timeout_result is not None: - return timeout_result - - return status - - def clear(self) -> None: - if self.bootstrap_room is None: - return - super().clear() - - def failure_exception(self): - if self.conclude_state is None: - self.conclude_state = KVPoll.Failed - - self.clear() - with self.kv_mgr.failure_lock: - failure_reason = self.kv_mgr.failure_records.pop(self.bootstrap_room, None) - is_propagated = failure_reason is None - if is_propagated: - failure_reason = "KV transfer failed" - raise KVTransferError( - self.bootstrap_room, failure_reason, is_from_another_rank=is_propagated - ) - - def abort(self): - if self.bootstrap_room is None: - return - bootstrap_room = self.bootstrap_room - super().abort() - self.clear() - with self.kv_mgr.failure_lock: - self.kv_mgr.failure_records.pop(bootstrap_room, None) + pass class MoriKVBootstrapServer(CommonKVBootstrapServer): diff --git a/python/sglang/srt/disaggregation/nixl/conn.py b/python/sglang/srt/disaggregation/nixl/conn.py index 1d1cf55396..35c8c62d19 100644 --- a/python/sglang/srt/disaggregation/nixl/conn.py +++ b/python/sglang/srt/disaggregation/nixl/conn.py @@ -8,14 +8,12 @@ import struct import threading import time import uuid -from collections import defaultdict -from typing import TYPE_CHECKING, Any, Dict, List, Optional, Set, Tuple +from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple import msgspec import numpy as np import numpy.typing as npt import torch -import zmq if TYPE_CHECKING: from sglang.srt.disaggregation.common.staging_handler import StagingTransferInfo @@ -28,7 +26,6 @@ from sglang.srt.disaggregation.common.conn import ( CommonKVManager, CommonKVReceiver, CommonKVSender, - KVTransferError, ) from sglang.srt.disaggregation.common.staging_handler import ( STAGING_WATERMARK_WAIT_S, @@ -54,9 +51,9 @@ from sglang.srt.disaggregation.utils import ( slice_dsa_tail_dst_ptrs_for_pp, ) from sglang.srt.environ import envs -from sglang.srt.runtime_context import get_device, get_parallel, get_schedule +from sglang.srt.runtime_context import get_device, get_schedule from sglang.srt.server_args import ServerArgs -from sglang.srt.utils.common import is_xpu, run_with_deadline +from sglang.srt.utils.common import run_with_deadline try: from nixl._bindings import ( @@ -73,8 +70,6 @@ try: except ImportError: _NIXL_TRANSPORT_ERRORS = (RuntimeError,) -_is_xpu = is_xpu() - logger = logging.getLogger(__name__) # Arbitrary; bounds how often an unreachable peer's dlists are rebuilt. @@ -514,53 +509,12 @@ def repeat_indices_over_layers( return (offsets[:, None] + indices[None, :]).ravel().astype(np.int32) -@dataclasses.dataclass -class TransferStatus: - """Used by KV Receiver to know when a transfer is done.""" - - # KV chunks received per pp_rank: {pp_rank: set of chunk_ids} - received_kvs_per_pp: Dict[int, Set[int]] = dataclasses.field( - default_factory=lambda: defaultdict(set) - ) - # Expected chunk count per pp_rank (set when is_last_chunk=True): {pp_rank: expected_count} - expected_kvs_per_pp: Dict[int, int] = dataclasses.field(default_factory=dict) - # Number of PP ranks expected to send data. - num_pp_ranks_expected: Optional[int] = None - # Whether aux data has been received. - received_aux: bool = False - # PP ranks that have sent state data (state is layer-specific, each PP rank sends its portion). - received_state_per_pp: Set[int] = dataclasses.field(default_factory=set) - # Whether state data is expected (set based on state_type). - expects_state: bool = False - # KV part notifications for mixed-memory transfers. Keyed by - # (pp_rank, chunk_id); normal homogeneous transfers bypass this. - received_kv_parts_per_pp: Optional[Dict[Tuple[int, int], Set[int]]] = None - expected_kv_parts_per_pp: Optional[Dict[Tuple[int, int], int]] = None - - def is_done(self): - if self.num_pp_ranks_expected is None or not self.received_aux: - return False - # If state data is expected, check all PP ranks have sent it - if ( - self.expects_state - and len(self.received_state_per_pp) < self.num_pp_ranks_expected - ): - return False - # All PP ranks must have reported their expected count - if len(self.expected_kvs_per_pp) < self.num_pp_ranks_expected: - return False - # Each PP rank must have received all expected chunks - for pp_rank, expected in self.expected_kvs_per_pp.items(): - if len(self.received_kvs_per_pp[pp_rank]) != expected: - return False - return True - - class NixlKVManager(StagingManagerMixin, CommonKVManager): # The decode control socket multiplexes tagged messages, so the status # message is tagged too. It is new to NIXL, hence free to carry the reason. kv_status_msg_tag = b"KV_STATUS" kv_status_msg_carries_reason = True + reports_transfer_latency = True # ABORT handler defers the ack until the transfer worker drains. supports_deferred_decode_kv_release = True # Newer NIXL prepares compressed (addr, len, dev_id, stride, count) runs @@ -575,9 +529,6 @@ class NixlKVManager(StagingManagerMixin, CommonKVManager): is_mla_backend: Optional[bool] = False, ): super().__init__(args, disaggregation_mode, server_args, is_mla_backend) - self.transfer_source_rank = ( - self.kv_args.pp_rank * get_parallel().tp_size + self.kv_args.engine_rank - ) self.kv_args.kv_data_mem_kinds = _normalize_kv_mem_kinds( getattr(self.kv_args, "kv_data_mem_kinds", None), len(self.kv_args.kv_data_ptrs), @@ -710,7 +661,6 @@ class NixlKVManager(StagingManagerMixin, CommonKVManager): self.transfer_queues: List[FastQueue] = [ FastQueue() for _ in range(transfer_queue_size) ] - self.exceptions: Dict[int, Exception] = {} # Mirror mooncake: one staging buffer per worker queue, all # built before workers spawn so each worker owns a private # buffer (no cross-worker contention on the staging ring). @@ -730,9 +680,6 @@ class NixlKVManager(StagingManagerMixin, CommonKVManager): ).start() self._start_bootstrap_thread() elif self.disaggregation_mode == DisaggregationMode.DECODE: - self.transfer_statuses: Dict[int, TransferStatus] = defaultdict( - TransferStatus - ) if self.enable_staging: self._init_staging_decode_ctx() self._start_decode_listener_thread() @@ -979,7 +926,7 @@ class NixlKVManager(StagingManagerMixin, CommonKVManager): dst_parts[0] if len(dst_parts) == 1 else np.concatenate(dst_parts) ) xfer_handle = self.agent.make_prepped_xfer( - "WRITE", plan.src_handle, src_selectors, plan.dst_handle, dst_selectors, b"" + "WRITE", plan.src_handle, src_selectors, plan.dst_handle, dst_selectors ) if not xfer_handle: raise RuntimeError("Failed to create prepared state transfer") @@ -1061,16 +1008,17 @@ class NixlKVManager(StagingManagerMixin, CommonKVManager): self._staging_ctx.room_receivers[room] = receiver def _start_decode_listener_thread(self): - """Decode-side ZMQ listener for KV_STATUS, STAGING_REQ and ABORT_ACK. A - thread, not NIXL notifs: the decode agent has no progress thread, so notifs - only drain inside a live receiver's poll() and would be missed while idle. + """Decode-side ZMQ listener for KV_STATUS, CHUNK_READY, STAGING_REQ and + ABORT_ACK. - Started unconditionally: KV_STATUS carries prefill-side transfer failures, + Started unconditionally: KV_STATUS carries prefill-side transfer results, which are independent of staging and deferred KV release.""" def decode_listener_thread(): while True: msg = self.server_socket.recv_multipart() + if self.handle_chunk_ready(msg): + continue if msg[0] == b"STAGING_REQ": if self.enable_staging: self._handle_staging_req(msg) @@ -1099,8 +1047,7 @@ class NixlKVManager(StagingManagerMixin, CommonKVManager): Idempotent per room: the first call for a given room does the full fan-out (one STAGING_REQ per chunk per peer); subsequent calls return - immediately. This lets the caller invoke this on every chunk without - depending on a chunk_id == 0 sentinel. + immediately, so the caller can invoke this on every chunk. """ if not self.enable_staging or self.kv_buffer_tensors is None: return @@ -1662,7 +1609,6 @@ class NixlKVManager(StagingManagerMixin, CommonKVManager): kv_chunk: TransferKVChunk = queue.get() room = kv_chunk.room handles: List[Any] = [] - state_notifs: List[Tuple[str, str]] = [] settle_timed_out = False room_transfer_infos = None try: @@ -1767,11 +1713,6 @@ class NixlKVManager(StagingManagerMixin, CommonKVManager): src_prefill_kv_indices = kv_chunk.prefill_kv_indices - notif = ( - f"{req.room}_kv_{kv_chunk.chunk_id}" - f"_{int(kv_chunk.is_last_chunk)}_{self.transfer_source_rank}" - ) - # Decide which kv send path to use: # 1. Staging (heterogeneous TP, both sides have # registered staging, watermark/alloc ready) @@ -1792,13 +1733,15 @@ class NixlKVManager(StagingManagerMixin, CommonKVManager): kv_xfer_handle = None if use_staging: - kv_xfer_handle, deferred = self._do_staging_transfer( - staging_strategy, - kv_chunk, - src_prefill_kv_indices, - req, - dst_info, - queue, + kv_xfer_handle, deferred, staging_chunk_idx = ( + self._do_staging_transfer( + staging_strategy, + kv_chunk, + src_prefill_kv_indices, + req, + dst_info, + queue, + ) ) if deferred: # Chunk re-enqueued; stop processing remaining @@ -1806,13 +1749,6 @@ class NixlKVManager(StagingManagerMixin, CommonKVManager): # pick it up again on the next pop. staging_deferred = True break - # NIXL reads the staging buffer asynchronously, and the next - # decode rank's gather overwrites it from offset 0. Gated to - # XPU, where this corrupted KV under concurrent requests. - if _is_xpu and kv_xfer_handle is not None: - self._await_handles( - [kv_xfer_handle], failure_seen=False - ) if kv_xfer_handle is None: if is_dcp_transfer: @@ -1846,7 +1782,6 @@ class NixlKVManager(StagingManagerMixin, CommonKVManager): req.agent_name, dst_info, plan, - notif, packed_src, ) ) @@ -1866,7 +1801,6 @@ class NixlKVManager(StagingManagerMixin, CommonKVManager): dst_info.dst_kv_ptrs, chunked_dst_kv_indice, dst_info.gpu_id, - notif, dst_mem_kind=( dst_info.dst_homogeneous_mem_kind ), @@ -1877,7 +1811,6 @@ class NixlKVManager(StagingManagerMixin, CommonKVManager): req.agent_name, src_prefill_kv_indices, chunked_dst_kv_indice, - notif, ) ) else: @@ -1885,11 +1818,31 @@ class NixlKVManager(StagingManagerMixin, CommonKVManager): req.agent_name, src_prefill_kv_indices, chunked_dst_kv_indice, - notif, ) if kv_xfer_handle is not None: handles.append(kv_xfer_handle) + if use_staging: + # The next peer may gather into the same source + # staging buffer. Wait before reusing it and + # before allowing decode to scatter this write. + settled, failed = self._await_handles( + [kv_xfer_handle], failure_seen=False + ) + if not settled: + settle_timed_out = True + if not settled or failed: + raise RuntimeError( + f"NIXL staging transfer failed room={room}" + ) + self.send_chunk_ready( + room=room, + chunk_idx=staging_chunk_idx, + page_start=kv_chunk.index_slice.start, + num_pages=len(kv_chunk.prefill_kv_indices), + writer_id=req.agent_name, + targets=[(req.endpoint, req.dst_port)], + ) if kv_chunk.is_last_chunk: dst_info = self.decode_kv_args_table[req.agent_name] @@ -1900,7 +1853,6 @@ class NixlKVManager(StagingManagerMixin, CommonKVManager): dst_info.dst_state_data_ptrs, req.dst_state_indices, dst_info.gpu_id, - f"{req.room}_state_{self.transfer_source_rank}", decode_tp_size, decode_tp_rank=dst_info.decode_tp_rank, dst_state_item_lens=dst_info.dst_state_item_lens, @@ -1910,45 +1862,16 @@ class NixlKVManager(StagingManagerMixin, CommonKVManager): for state_xfer_handle in state_xfer_handles: handles.append(state_xfer_handle) - if req.dst_state_indices: - if not kv_chunk.state_indices: - logger.warning_once( - "Decode expects state but prefill has none to " - "send; announcing state completion anyway" - ) - state_notifs.append( - ( - req.agent_name, - f"{req.room}_state_{self.transfer_source_rank}", - ) - ) - if kv_chunk.prefill_aux_index is None: raise RuntimeError("Missing aux index for last chunk") - # A no-KV notification still identifies its PP source. - # Empty non-final chunks do not consume chunk IDs, so a - # final no-KV chunk_id equals the prior KV chunk count. - aux_notif = f"{req.room}_aux" - if ( - len(kv_chunk.prefill_kv_indices) == 0 - or not self.kv_args.kv_data_ptrs - ): - aux_notif += ( - f"_nokv_{self.transfer_source_rank}_{kv_chunk.chunk_id}" - ) aux_xfer_handle = self.send_aux( req.agent_name, kv_chunk.prefill_aux_index, dst_info.dst_aux_ptrs, req.dst_aux_index, - aux_notif, ) handles.append(aux_xfer_handle) - if staging_deferred: - # Chunk has been re-enqueued; do not advance status. - continue - # Raise only once every handle of this batch settled, not on the # first ERR: a sibling still in PROC keeps writing into the # decode's KV pages, and the failure path below tells the decode @@ -1963,10 +1886,10 @@ class NixlKVManager(StagingManagerMixin, CommonKVManager): if any_failed: raise RuntimeError(f"NIXL transfer encountered ERR room={room}") - if self.request_status.get(room) not in (None, KVPoll.Failed): - for peer_name, notif in state_notifs: - self.agent.send_notif(peer_name, notif.encode("ascii")) - + if staging_deferred: + # Drain posted writes before the retry reuses this worker's + # packing buffers; the re-enqueued chunk stays counted. + continue # Clear with the decrement: the failure path below reads this # flag, so a raise after this point must not uncount twice. kv_chunk.staging_counted = False @@ -1976,11 +1899,7 @@ class NixlKVManager(StagingManagerMixin, CommonKVManager): # was aborted and nothing else is outstanding. self._maybe_ack_drained_abort(room) if kv_chunk.is_last_chunk: - self.update_status(room, KVPoll.Success) - elif self.check_status(room) != KVPoll.Success: - # A deferred earlier chunk can complete after the last chunk - # already concluded Success; don't regress the status. - self.update_status(room, KVPoll.Transferring) + self.conclude_transfer(bootstrap_room=room, status=KVPoll.Success) # Drop per-room state only when no chunk is still outstanding and # the room has concluded: Success, or a Failed *last* chunk. A @@ -2013,7 +1932,6 @@ class NixlKVManager(StagingManagerMixin, CommonKVManager): logger.exception( f"Unexpected transfer worker error for room {room}" ) - self.exceptions[room] = e # An exception raised while the batch was still being built # leaves the handles posted so far running, so settle here too # rather than only after the barrier. @@ -2175,7 +2093,6 @@ class NixlKVManager(StagingManagerMixin, CommonKVManager): prefill_data_indices: npt.NDArray[np.int32], dst_data_indices: npt.NDArray[np.int32], dst_gpu_id: int, - notif: str, state_type: Optional[StateType] = None, src_mem_kind: str = "VRAM", dst_mem_kind: str = "VRAM", @@ -2219,7 +2136,6 @@ class NixlKVManager(StagingManagerMixin, CommonKVManager): src_indices, dst_prep, dst_indices, - notif.encode("ascii"), ) if not xfer_handle: raise Exception("KVSender failed to create prepped transfer") @@ -2240,7 +2156,7 @@ class NixlKVManager(StagingManagerMixin, CommonKVManager): prefill_data_indices, dst_data_indices ) - logger.debug(f"sending kvcache to {peer_name} with notif {notif}") + logger.debug(f"sending kvcache to {peer_name}") # Make descs if self.is_mla_backend or force_flat: if src_layer_ids or dst_layer_ids: @@ -2373,7 +2289,6 @@ class NixlKVManager(StagingManagerMixin, CommonKVManager): src_descs, dst_descs, peer_name, - notif.encode("ascii"), # type: ignore ) if not xfer_handle: raise Exception("KVSender failed to create transfer") @@ -2387,7 +2302,6 @@ class NixlKVManager(StagingManagerMixin, CommonKVManager): dst_kv_ptrs: list[int], dst_kv_indices: npt.NDArray[np.int32], dst_gpu_id: int, - notif: str, dst_mem_kind: str = "VRAM", ): assert self.src_mem_kind is not None @@ -2399,7 +2313,6 @@ class NixlKVManager(StagingManagerMixin, CommonKVManager): prefill_data_indices=prefill_kv_indices, dst_data_indices=dst_kv_indices, dst_gpu_id=dst_gpu_id, - notif=notif, src_mem_kind=self.src_mem_kind, dst_mem_kind=dst_mem_kind, ) @@ -2444,7 +2357,6 @@ class NixlKVManager(StagingManagerMixin, CommonKVManager): peer_name: str, dst_info: KVArgsRegisterInfo, plan, - notif: str, packed_src, ): if self.src_mem_kind is None: @@ -2486,21 +2398,14 @@ class NixlKVManager(StagingManagerMixin, CommonKVManager): ) ) - if not parts: - self.agent.send_notif(peer_name, notif.encode("ascii")) - return [] - handles = [] - for part_idx, ( + for ( src_ptrs, part_dst_ptrs, part_item_lens, src_indices, dst_indices, - ) in enumerate(parts): - part_notif = ( - notif if len(parts) == 1 else f"{notif}_part_{part_idx}_{len(parts)}" - ) + ) in parts: handles.append( self._send_kvcache_generic( peer_name=peer_name, @@ -2510,7 +2415,6 @@ class NixlKVManager(StagingManagerMixin, CommonKVManager): prefill_data_indices=src_indices, dst_data_indices=dst_indices, dst_gpu_id=dst_info.gpu_id, - notif=part_notif, src_mem_kind=self.src_mem_kind, dst_mem_kind=dst_info.dst_homogeneous_mem_kind, force_flat=True, @@ -2524,7 +2428,6 @@ class NixlKVManager(StagingManagerMixin, CommonKVManager): peer_name: str, prefill_kv_indices: npt.NDArray[np.int32], dst_kv_indices: npt.NDArray[np.int32], - notif: str, ): info = self.decode_kv_args_table[peer_name] segments = info.kv_xfer_segments @@ -2532,9 +2435,8 @@ class NixlKVManager(StagingManagerMixin, CommonKVManager): if not segments: raise RuntimeError(f"Missing NIXL mixed KV transfer plan for {peer_name}") - num_parts = len(segments) handles = [] - for part_idx, seg in enumerate(segments): + for seg in segments: num_layers = seg.end - seg.start src_indices = repeat_indices_over_layers( prefill_kv_indices, num_layers, self._num_slots_src @@ -2542,14 +2444,12 @@ class NixlKVManager(StagingManagerMixin, CommonKVManager): dst_indices = repeat_indices_over_layers( dst_kv_indices, num_layers, seg.dst_num_slots ) - part_notif = f"{notif}_part_{part_idx}_{num_parts}" xfer_handle = self.agent.make_prepped_xfer( "WRITE", seg.src_handle, src_indices, seg.dst_handle, dst_indices, - part_notif.encode("ascii"), ) if not xfer_handle: raise Exception("KVSender failed to create mixed prepped transfer") @@ -2564,7 +2464,6 @@ class NixlKVManager(StagingManagerMixin, CommonKVManager): peer_name: str, prefill_kv_indices: npt.NDArray[np.int32], dst_kv_indices: npt.NDArray[np.int32], - notif: str, ): # Prepped path: src dlist is shared per decode_tp_size; dst is per peer. assert self.prep_handle_slice_src is not None @@ -2596,7 +2495,6 @@ class NixlKVManager(StagingManagerMixin, CommonKVManager): src_indices, dst_handle, dst_indices, - notif.encode("ascii"), ) if not xfer_handle: raise Exception("KVSender failed to create prepped slice transfer") @@ -2613,7 +2511,6 @@ class NixlKVManager(StagingManagerMixin, CommonKVManager): dst_tp_rank: int, dst_attn_tp_size: int, dst_kv_item_len: int, - notif: str, staging_buffer=None, ): """Transfer KV cache via staging buffers (gather -> bulk RDMA -> scatter on decode).""" @@ -2701,7 +2598,7 @@ class NixlKVManager(StagingManagerMixin, CommonKVManager): dst_descs = self.agent.get_xfer_descs(dst_reqs, "VRAM") xfer_handle = self.agent.initialize_xfer( - "WRITE", src_descs, dst_descs, peer_name, notif.encode("ascii") + "WRITE", src_descs, dst_descs, peer_name ) if not xfer_handle: raise RuntimeError( @@ -2737,18 +2634,18 @@ class NixlKVManager(StagingManagerMixin, CommonKVManager): dst_info: KVArgsRegisterInfo, queue: FastQueue, ): - """Attempt staging transfer for one chunk. Returns (xfer_handle, deferred). + """Attempt staging transfer. Returns (xfer_handle, deferred, chunk_idx). Mirrors mooncake._do_staging_transfer semantics: - staging not ready (watermark/alloc pending) -> ``queue.put(kv_chunk)`` - re-enqueue the chunk and return ``(None, True)``. Caller should + re-enqueue the chunk and return ``(None, True, chunk_idx)``. Caller should ``break`` out of the per-req loop and ``continue`` the worker main loop without updating room status -- the chunk will be retried on the next pop. - oversized chunk (will never fit) -> raise RuntimeError. - - staging successfully posted -> return ``(handle, False)``. The - caller appends the handle to the per-chunk handle list and - busy-polls it to DONE alongside other handles. + - staging successfully posted -> return ``(handle, False, chunk_idx)``. + The caller waits for DONE before reusing its staging buffer and + sending ZMQ CHUNK_READY. - send_kvcache_staged returned None (chunk cannot fit; decode buffer too small, kv_buffer_tensors missing, etc.) -> raise RuntimeError instead of falling back to the slice path. @@ -2777,13 +2674,8 @@ class NixlKVManager(StagingManagerMixin, CommonKVManager): with self._staging_ctx.watermark_cv: self._staging_ctx.watermark_cv.wait(STAGING_WATERMARK_WAIT_S) queue.put(kv_chunk) - return (None, True) + return (None, True, chunk_idx) - notif_tag = ( - f"{req.room}_stg_{kv_chunk.chunk_id}_{int(kv_chunk.is_last_chunk)}" - f"_{self.transfer_source_rank}_{chunk_idx}" - f"_{page_start}_{num_pages}_{req.agent_name}" - ) handle = self.send_kvcache_staged( req.agent_name, src_prefill_kv_indices, @@ -2793,7 +2685,6 @@ class NixlKVManager(StagingManagerMixin, CommonKVManager): dst_info.decode_tp_rank, dst_info.decode_tp_size, dst_info.dst_kv_item_len, - notif_tag, staging_buffer=staging_strategy.staging_buffer, ) if handle is None: @@ -2807,7 +2698,7 @@ class NixlKVManager(StagingManagerMixin, CommonKVManager): f"SGLANG_DISAGG_STAGING_POOL_SIZE_MB or reduce " f"chunked_prefill_size." ) - return (handle, False) + return (handle, False, chunk_idx) def send_aux( self, @@ -2815,7 +2706,6 @@ class NixlKVManager(StagingManagerMixin, CommonKVManager): prefill_aux_index: int, dst_aux_ptrs: list[int], dst_aux_index: int, - notif: str, ): src_addrs = [] dst_addrs = [] @@ -2838,7 +2728,6 @@ class NixlKVManager(StagingManagerMixin, CommonKVManager): src_descs, dst_descs, peer_name, - notif.encode("ascii"), # type: ignore ) if not xfer_handle: raise Exception("KVSender failed to create transfer") @@ -2855,7 +2744,6 @@ class NixlKVManager(StagingManagerMixin, CommonKVManager): src_indices: list[int], dst_indices: list[int], dst_gpu_id: int, - notif: str, ): dst_data_ptrs = slice_dsa_tail_dst_ptrs_for_pp( src_data_ptrs, @@ -2890,7 +2778,7 @@ class NixlKVManager(StagingManagerMixin, CommonKVManager): src_descs = self.agent.get_xfer_descs(src_addrs, "VRAM") dst_descs = self.agent.get_xfer_descs(dst_addrs, "VRAM") xfer_handle = self.agent.initialize_xfer( - "WRITE", src_descs, dst_descs, peer_name, notif.encode("ascii") + "WRITE", src_descs, dst_descs, peer_name ) if not xfer_handle: raise Exception("KVSender failed to create dsa_tail transfer") @@ -2906,7 +2794,6 @@ class NixlKVManager(StagingManagerMixin, CommonKVManager): dst_state_data_ptrs: list[int], dst_state_indices: List[int], dst_gpu_id: int, - notif: str, src_layer_ids: list[int] = None, dst_layer_ids: list[int] = None, ): @@ -2944,7 +2831,6 @@ class NixlKVManager(StagingManagerMixin, CommonKVManager): src_descs, dst_descs, peer_name, - notif.encode("ascii"), ) if not xfer_handle: raise Exception("Failed to create Mamba state transfer") @@ -2963,7 +2849,6 @@ class NixlKVManager(StagingManagerMixin, CommonKVManager): dst_state_item_lens: list[int], dst_state_dim_per_tensor: list[int], dst_gpu_id: int, - notif: str, decode_tp_size: int, decode_tp_rank: int, src_state_conv_shard_groups: list = None, @@ -2997,7 +2882,6 @@ class NixlKVManager(StagingManagerMixin, CommonKVManager): dst_state_data_ptrs, dst_state_indices, dst_gpu_id, - notif, src_layer_ids=src_layer_ids, dst_layer_ids=dst_layer_ids, ) @@ -3072,7 +2956,6 @@ class NixlKVManager(StagingManagerMixin, CommonKVManager): src_descs, dst_descs, peer_name, - notif.encode("ascii"), ) if not xfer_handle: raise Exception("Failed to create Mamba state slice transfer") @@ -3089,7 +2972,6 @@ class NixlKVManager(StagingManagerMixin, CommonKVManager): dst_swa_page_indices: List[int], dst_state_item_lens: List[int], dst_gpu_id: int, - notif: str, decode_tp_size: int, decode_tp_rank: int, ): @@ -3127,7 +3009,6 @@ class NixlKVManager(StagingManagerMixin, CommonKVManager): src_descs, dst_descs, peer_name, - notif.encode("ascii"), ) if not xfer_handle: raise Exception("Failed to create SWA state slice transfer") @@ -3141,7 +3022,6 @@ class NixlKVManager(StagingManagerMixin, CommonKVManager): dst_state_data_ptrs: List[List[int]], dst_state_indices: List[List[int]], dst_gpu_id: int, - notif: str, decode_tp_size: int, decode_tp_rank: int = 0, dst_state_item_lens: List[List[int]] | None = None, @@ -3232,7 +3112,6 @@ class NixlKVManager(StagingManagerMixin, CommonKVManager): dst_lens, dst_dims, dst_gpu_id, - "", decode_tp_size, decode_tp_rank, src_conv, @@ -3249,7 +3128,6 @@ class NixlKVManager(StagingManagerMixin, CommonKVManager): dst_ptrs, dst_indices, dst_gpu_id, - "", src_layer_ids=src_lids, dst_layer_ids=dst_lids, ) @@ -3263,7 +3141,6 @@ class NixlKVManager(StagingManagerMixin, CommonKVManager): list(src_indices), list(dst_indices), dst_gpu_id, - "", ) elif st == StateType.DSA: if len(src_indices) != len(dst_indices): @@ -3279,7 +3156,6 @@ class NixlKVManager(StagingManagerMixin, CommonKVManager): prefill_data_indices=np.array(src_indices, dtype=np.int32), dst_data_indices=np.array(dst_indices, dtype=np.int32), dst_gpu_id=dst_gpu_id, - notif="", state_type=st, force_flat=True, ) @@ -3297,7 +3173,6 @@ class NixlKVManager(StagingManagerMixin, CommonKVManager): dst_swa_page_indices=dst_indices, dst_state_item_lens=dst_lens, dst_gpu_id=dst_gpu_id, - notif="", decode_tp_size=decode_tp_size, decode_tp_rank=decode_tp_rank, ) @@ -3340,7 +3215,6 @@ class NixlKVManager(StagingManagerMixin, CommonKVManager): prefill_data_indices=np.array(src_indices, dtype=np.int32), dst_data_indices=np.array(dst_indices, dtype=np.int32), dst_gpu_id=dst_gpu_id, - notif="", state_type=st, force_flat=st in (StateType.QSA_PENDING, StateType.QSA_COMPRESSED), src_layer_ids=src_lids, @@ -3371,7 +3245,6 @@ class NixlKVManager(StagingManagerMixin, CommonKVManager): prefill_data_indices=np.array(src_indices, dtype=np.int32), dst_data_indices=np.array(dst_indices, dtype=np.int32), dst_gpu_id=dst_gpu_id, - notif="", force_flat=True, ) else: @@ -3387,10 +3260,13 @@ class NixlKVManager(StagingManagerMixin, CommonKVManager): kv_indices: npt.NDArray[np.int32], index_slice: slice, is_last_chunk: bool, - chunk_id: int, aux_index: Optional[int] = None, state_indices: Optional[List] = None, num_kv_tokens: Optional[int] = None, + *, + wait_event=None, + indices_event=None, + trace_ctx=None, ): assert self.disaggregation_mode == DisaggregationMode.PREFILL assert not is_last_chunk or (is_last_chunk and aux_index is not None) @@ -3418,7 +3294,6 @@ class NixlKVManager(StagingManagerMixin, CommonKVManager): prefill_kv_indices=kv_indices, index_slice=index_slice, is_last_chunk=is_last_chunk, - chunk_id=chunk_id, prefill_aux_index=aux_index, state_indices=state_indices, num_kv_tokens=num_kv_tokens, @@ -3426,198 +3301,6 @@ class NixlKVManager(StagingManagerMixin, CommonKVManager): ) return None - def update_transfer_status(self): - # Process notifications from received transfers. - notif_map = self.agent.get_new_notifs() - for peer_name, messages in notif_map.items(): - for msg in messages: - # Notification tag layouts (underscore-separated): - # kv: {room}_kv_{chunk_id}_{is_last}_{pp_rank} -> 5 fields - # kvpart:{room}_kv_{chunk_id}_{is_last}_{pp_rank}_part_{i}_{n}-> 8 fields - # stg: {room}_stg_{chunk_id}_{is_last}_{pp_rank}_{chunk_idx} - # _{page_start}_{num_pages}_{agent_name} -> 9 fields - # aux: {room}_aux -> 2 fields - # state: {room}_state_{pp_rank} -> 3 fields - # maxsplit=8 keeps everything past the 8th underscore in the - # last component, so agent_name (which may itself contain - # underscores) lands intact in components[8] for the stg path. - components = msg.decode("ascii").split("_", 8) - room = int(components[0]) - tag = components[1] - if tag == "kv": - chunk_id = int(components[2]) - is_last_chunk = bool(int(components[3])) - pp_rank = int(components[4]) if len(components) > 4 else 0 - if len(components) > 7 and components[5] == "part": - self._track_kv_part_arrival( - room, - chunk_id, - is_last_chunk, - pp_rank, - int(components[6]), - int(components[7]), - ) - else: - self._track_kv_arrival(room, chunk_id, is_last_chunk, pp_rank) - elif tag == "stg": - self._handle_stg_notification(components, room) - elif tag == "aux": - # Main's "nokv" marker carries the number of earlier KV - # chunks expected from this PP rank. - self._handle_aux_notification(room, components) - elif tag == "state": - pp_rank = int(components[2]) if len(components) > 2 else 0 - self.transfer_statuses[room].received_state_per_pp.add(pp_rank) - - def _handle_stg_notification(self, components, room: int): - """Handle a staging RDMA notification tag. - - Format: {room}_stg_{chunk_id}_{is_last}_{pp_rank}_{chunk_idx}_{page_start}_{num_pages}_{agent_name} - """ - chunk_id = int(components[2]) - is_last_chunk = bool(int(components[3])) - pp_rank = int(components[4]) - chunk_idx = int(components[5]) - page_start = int(components[6]) - num_pages = int(components[7]) - agent_name = components[8] if len(components) > 8 else "" - # Count this notif's own arrival BEFORE _track_kv_arrival, which can - # conclude the transfer and record all-ranks Success. - self._handle_staging_chunk_arrived( - room, chunk_idx, page_start, num_pages, agent_name - ) - self._track_kv_arrival(room, chunk_id, is_last_chunk, pp_rank) - - def _handle_aux_notification(self, room: int, components: List[str]): - """Handle an aux notification and trigger last scatter if staging is complete. - - Notification tag layouts: - aux: {room}_aux -> 2 fields - aux (nokv): {room}_aux_nokv_{pp_rank}_{expected} -> 5 fields - (the last chunk had no KV pages for this rank; - `expected` is the number of prior KV chunks) - """ - self.transfer_statuses[room].received_aux = True - # main's "nokv" marker (decode-side radix cache hit, see #19746). - if len(components) > 3 and components[2] == "nokv": - pp_rank = int(components[3]) - expected = int(components[4]) if len(components) > 4 else 0 - self.transfer_statuses[room].expected_kvs_per_pp[pp_rank] = expected - if self.transfer_statuses[room].num_pp_ranks_expected is None: - self.transfer_statuses[ - room - ].num_pp_ranks_expected = self.required_prefill_response_num_table.get( - room, 1 - ) - if ( - self.enable_staging - and self._staging_handler is not None - and self._staging_handler.is_staging_room(room) - ): - self._maybe_submit_last_scatter(room) - - def _track_kv_arrival( - self, room: int, chunk_id: int, is_last_chunk: bool, pp_rank: int - ): - """Update transfer status tracking for a kv chunk arrival.""" - self.transfer_statuses[room].received_kvs_per_pp[pp_rank].add(chunk_id) - if is_last_chunk: - self.transfer_statuses[room].expected_kvs_per_pp[pp_rank] = chunk_id + 1 - if self.transfer_statuses[room].num_pp_ranks_expected is None: - self.transfer_statuses[ - room - ].num_pp_ranks_expected = self.required_prefill_response_num_table.get( - room, 1 - ) - if ( - self.enable_staging - and self._staging_handler is not None - and self._staging_handler.is_staging_room(room) - ): - self._maybe_submit_last_scatter(room) - - def _track_kv_part_arrival( - self, - room: int, - chunk_id: int, - is_last_chunk: bool, - pp_rank: int, - part_idx: int, - num_parts: int, - ): - """Track one segment of a mixed-memory KV transfer.""" - if num_parts <= 1: - self._track_kv_arrival(room, chunk_id, is_last_chunk, pp_rank) - return - if part_idx < 0 or part_idx >= num_parts: - raise RuntimeError( - f"NIXL KV part index out of range for room={room}, " - f"chunk={chunk_id}, pp_rank={pp_rank}: part={part_idx}, " - f"num_parts={num_parts}" - ) - - key = (pp_rank, chunk_id) - status = self.transfer_statuses[room] - if status.received_kv_parts_per_pp is None: - status.received_kv_parts_per_pp = defaultdict(set) - if status.expected_kv_parts_per_pp is None: - status.expected_kv_parts_per_pp = {} - expected = status.expected_kv_parts_per_pp.setdefault(key, num_parts) - if expected != num_parts: - raise RuntimeError( - f"NIXL KV part count mismatch for room={room}, chunk={chunk_id}, " - f"pp_rank={pp_rank}: got {num_parts}, expected {expected}" - ) - parts = status.received_kv_parts_per_pp[key] - parts.add(part_idx) - if len(parts) == num_parts: - status.received_kv_parts_per_pp.pop(key, None) - status.expected_kv_parts_per_pp.pop(key, None) - self._track_kv_arrival(room, chunk_id, is_last_chunk, pp_rank) - - def _handle_staging_chunk_arrived( - self, - room: int, - chunk_idx: int, - page_start: int, - num_pages: int, - agent_name: str, - ): - """Process a staging chunk arrival via RDMA notification.""" - handler = self._staging_handler - if handler is None: - return - handler.handle_chunk_arrived( - room, - chunk_idx, - page_start, - num_pages, - agent_name, - ) - - def _maybe_submit_last_scatter(self, room: int): - """Check if all kv+aux transfers are done and submit last scatter if so.""" - status = self.transfer_statuses.get(room) - if status is None: - return - if not status.received_aux: - return - if status.num_pp_ranks_expected is None: - return - if len(status.expected_kvs_per_pp) < status.num_pp_ranks_expected: - return - for pp_rank, expected in status.expected_kvs_per_pp.items(): - if len(status.received_kvs_per_pp[pp_rank]) != expected: - return - handler = self._staging_handler - if handler is not None and handler.is_staging_room(room): - handler.submit_last_scatter_async(room) - - def check_transfer_done(self, room: int): - if room not in self.transfer_statuses: - return False - return self.transfer_statuses[room].is_done() - def _handle_abort_notification(self, msg: List[bytes]) -> bool: if not msg or msg[0] != ABORT_TAG: return False @@ -3738,358 +3421,113 @@ class NixlKVManager(StagingManagerMixin, CommonKVManager): threading.Thread(target=bootstrap_thread_guarded).start() + def build_receiver_registration_message(self) -> List[bytes]: + packed_kv_data_ptrs = b"".join( + struct.pack("Q", ptr) for ptr in self.kv_args.kv_data_ptrs + ) + packed_kv_data_mem_kinds = _pack_kv_mem_kinds(self.kv_args.kv_data_mem_kinds) + packed_kv_item_lens = b"".join( + struct.pack("Q", item_len) for item_len in self.kv_args.kv_item_lens + ) + packed_kv_layer_ids = b"".join( + struct.pack("I", layer_id) for layer_id in self.kv_args.kv_layer_ids + ) + packed_aux_data_ptrs = b"".join( + struct.pack("Q", ptr) for ptr in self.kv_args.aux_data_ptrs + ) + packed_state_data_ptrs = pack_int_lists(self.kv_args.state_data_ptrs or [], "Q") + packed_state_item_lens = pack_int_lists(self.kv_args.state_item_lens or [], "I") + packed_state_data_lens = pack_int_lists(self.kv_args.state_data_lens or [], "Q") + packed_state_dim_per_tensor = pack_int_lists( + getattr(self.kv_args, "state_dim_per_tensor", []) or [], "I" + ) + packed_state_layer_ids = pack_int_lists(self.kv_args.state_layer_ids, "I") + + # Include staging allocator metadata if available + if self.enable_staging and self._staging_ctx.allocator is not None: + _alloc = self._staging_ctx.allocator + packed_staging_base_ptr = struct.pack("Q", _alloc.get_base_ptr()) + staging_total_size_str = str(_alloc.get_total_size()).encode("ascii") + else: + packed_staging_base_ptr = b"" + staging_total_size_str = b"" + if self.kv_args.kv_item_lens: + dst_kv_item_len = self.kv_args.kv_item_lens[0] + dst_num_slots = num_kv_slots( + self.kv_args.kv_data_lens, self.kv_args.kv_item_lens + ) + else: + dst_kv_item_len = 0 + dst_num_slots = 0 + + return [ + GUARD, + "None".encode("ascii"), + self.local_ip.encode("ascii"), + str(self.rank_port).encode("ascii"), + self.agent.name.encode("ascii"), + self.agent.get_agent_metadata(), + packed_kv_data_ptrs, + packed_aux_data_ptrs, + packed_state_data_ptrs, + str(self.kv_args.gpu_id).encode("ascii"), + str(self.attn_tp_size).encode("ascii"), + str(self.kv_args.engine_rank).encode("ascii"), + str(dst_kv_item_len).encode("ascii"), + packed_state_item_lens, + packed_state_dim_per_tensor, + packed_staging_base_ptr, + staging_total_size_str, + str(dst_num_slots).encode("ascii"), + packed_kv_data_mem_kinds, + packed_kv_item_lens, + packed_state_layer_ids, + packed_kv_layer_ids, + str(self.dcp_size).encode("ascii"), + str(self.dcp_rank).encode("ascii"), + packed_state_data_lens, + ] + + def build_receiver_metadata_message( + self, + *, + bootstrap_room: int, + required_dst_info_num: int, + kv_indices: npt.NDArray[np.int32], + aux_index: Optional[int], + state_indices: Optional[List], + decode_prefix_len: Optional[int], + is_dummy: bool, + device_kv_indices: Optional[npt.NDArray[np.int32]] = None, + ) -> List[bytes]: + packed_state_indices = ( + pack_int_lists( + [(idx if idx is not None else []) for idx in state_indices], "i" + ) + if not is_dummy and state_indices is not None + else b"" + ) + return [ + GUARD, + str(bootstrap_room).encode("ascii"), + self.local_ip.encode("ascii"), + str(self.rank_port).encode("ascii"), + self.agent.name.encode("ascii"), + kv_indices.tobytes() if not is_dummy else b"", + str(aux_index).encode("ascii"), + str(required_dst_info_num).encode("ascii"), + packed_state_indices, + str(decode_prefix_len or 0).encode("ascii"), + str(int(is_dummy)).encode("ascii"), + ] + class NixlKVSender(CommonKVSender): - def __init__( - self, - mgr: NixlKVManager, - bootstrap_addr: str, - bootstrap_room: int, - req_has_disagg_prefill_dp_rank: bool = False, - ): - super().__init__( - mgr, - bootstrap_addr, - bootstrap_room, - req_has_disagg_prefill_dp_rank, - ) - self.init_time = time.time() - self.has_sent = False - self.chunk_id = 0 - self._send_failed = False - self._send_error: Optional[Exception] = None - self._transfer_start_time: Optional[float] = None - - def send( - self, - kv_indices: npt.NDArray[np.int32], - state_indices: Optional[List] = None, - num_kv_tokens: Optional[int] = None, - ): - if self._send_failed: - return - - kv_indices, index_slice, is_last_chunk, should_skip = ( - self._prepare_send_indices(kv_indices, state_indices) - ) - if should_skip: - return - - if self._transfer_start_time is None and ( - len(kv_indices) > 0 or state_indices is not None - ): - self._transfer_start_time = time.perf_counter() - - self.kv_mgr.add_transfer_request( - self.bootstrap_room, - kv_indices, - index_slice, - is_last_chunk, - self.chunk_id, - self.aux_index, - state_indices, - num_kv_tokens, - ) - self._record_transfer_indices(kv_indices, state_indices) - self.chunk_id += 1 - if is_last_chunk: - self.has_sent = True - - def poll(self) -> KVPoll: - return self._poll_with_status(self.kv_mgr.check_status) - - def poll_pp_consensus(self) -> KVPoll: - return self._poll_with_status(self.kv_mgr.check_status_pp_consensus) - - def _poll_with_status(self, check_status) -> KVPoll: - if self._send_failed: - return KVPoll.Failed # type: ignore - status = check_status(self.bootstrap_room) - if status == KVPoll.Bootstrapping: - timeout_result = self._check_bootstrap_timeout() - if timeout_result is not None: - return timeout_result - # Hold Success until all staging chunks transferred: a deferred chunk - # can still be pending, and concluding now would drop it. - if ( - status == KVPoll.Success - and self.kv_mgr._staging_outstanding.get(self.bootstrap_room, 0) > 0 - ): - return KVPoll.Transferring # type: ignore - if ( - status == KVPoll.Success - and self._transfer_start_time is not None - and self._transfer_metric.transfer_latency_s is None - ): - self._transfer_metric.transfer_latency_s = ( - time.perf_counter() - self._transfer_start_time - ) - return status - - def clear(self) -> None: - super().clear() - if self.kv_mgr.enable_staging and self.kv_mgr._staging_ctx is not None: - self.kv_mgr._staging_ctx.prefetched_rooms.discard(self.bootstrap_room) - self.kv_mgr._staging_ctx.prefetch_requested = { - key - for key in self.kv_mgr._staging_ctx.prefetch_requested - if key[0] != self.bootstrap_room - } - - def failure_exception(self): - exc = self.kv_mgr.exceptions.pop(self.bootstrap_room, None) - with self.kv_mgr.failure_lock: - failure_reason = self.kv_mgr.failure_records.pop(self.bootstrap_room, None) - - if self.conclude_state is None: - self.conclude_state = KVPoll.Failed - self._send_failed = True - - self.clear() - - if self._send_error is not None: - raise self._send_error - if exc is not None: - raise exc - if failure_reason is not None: - raise KVTransferError(self.bootstrap_room, failure_reason) - raise KVTransferError( - self.bootstrap_room, "NIXL KVSender Exception", is_from_another_rank=True - ) + pass class NixlKVReceiver(CommonKVReceiver): - def __init__( - self, - mgr: NixlKVManager, - bootstrap_addr: str, - bootstrap_room: Optional[int] = None, - ): - self.started_transfer = False - super().__init__(mgr, bootstrap_addr, bootstrap_room) - self.init_time = None - - def clear(self) -> None: - super().clear() - # transfer_statuses is NIXL's own per-room bookkeeping -- the other - # backends track completion through prefill_response_tracker, which - # CommonKVReceiver.clear() already drops -- so it needs its own pop. - self.kv_mgr.transfer_statuses.pop(self.bootstrap_room, None) - - def send_metadata( - self, - kv_indices: npt.NDArray[np.int32], - aux_index: Optional[int] = None, - state_indices: Optional[List] = None, - decode_prefix_len: Optional[int] = None, - ): - if self.bootstrap_infos is None: - logger.error( - f"Could not fetch prefill parallel info from bootstrap_addr: {self.bootstrap_addr}", - ) - self.kv_mgr.update_status(self.bootstrap_room, KVPoll.Failed) - return - - # Register staging room bootstrap info for staging handler - self.chunk_staging_infos = [] - if ( - self.kv_mgr.enable_staging - and self.kv_mgr._staging_ctx.allocator is not None - ): - self.kv_mgr.register_staging_room_bootstrap( - self.bootstrap_room, self.bootstrap_infos, self - ) - - for bootstrap_info in self.bootstrap_infos: - logger.debug( - f"Fetched bootstrap info: {bootstrap_info} for engine rank: {self.kv_mgr.kv_args.engine_rank}" - ) - is_dummy = bootstrap_info["is_dummy"] - logger.debug( - f"Sending to prefill server with bootstrap room {self.bootstrap_room} {is_dummy=}" - ) - packed_state_indices = ( - pack_int_lists( - [(idx if idx is not None else []) for idx in state_indices], "i" - ) - if not is_dummy and state_indices is not None - else b"" - ) - try: - sock, lock = self._connect_to_bootstrap_server(bootstrap_info) - with lock: - sock.send_multipart( - [ - GUARD, - str(self.bootstrap_room).encode("ascii"), - self.kv_mgr.local_ip.encode("ascii"), - str(self.kv_mgr.rank_port).encode("ascii"), - self.kv_mgr.agent.name.encode("ascii"), - kv_indices.tobytes() if not is_dummy else b"", - str(aux_index).encode("ascii"), - str(self.required_dst_info_num).encode("ascii"), - packed_state_indices, - str(decode_prefix_len or 0).encode("ascii"), - str(int(is_dummy)).encode("ascii"), - ] - ) - except zmq.ZMQError: - self.invalidate_cached_bootstrap_infos() - self.kv_mgr.record_failure( - self.bootstrap_room, - f"send_metadata to prefill {bootstrap_info.get('rank_ip')}:{bootstrap_info.get('rank_port')} failed", - ) - self.conclude_state = KVPoll.Failed - self.kv_mgr.update_status(self.bootstrap_room, KVPoll.Failed) - return - - # Mark that we expect state data if state_indices was provided. - # Match the prefill-side truthy check: an empty list means the - # model has no state types (e.g. dense LLaMA/Qwen), and prefill - # won't send state notifs, so we must not expect them. - if state_indices: - self.kv_mgr.transfer_statuses[self.bootstrap_room].expects_state = True - - self.started_transfer = True - self.init_time = time.time() - - def poll(self) -> KVPoll: - if self.conclude_state is not None: - return self.conclude_state - status = self.kv_mgr.check_status(self.bootstrap_room) - if status in (KVPoll.Success, KVPoll.Failed): - self.conclude_state = status - return status - if not self.started_transfer: - return status - - # Drain notifications before enforcing the waiting deadline. The decode - # agent has no NIXL progress thread (num_threads=0), so incoming - # completion notifications are only ingested here via - # update_transfer_status(); a completion queued by NIXL at/after the - # deadline would otherwise lose to the timeout purely by poll ordering. - self.kv_mgr.update_transfer_status() - if self.kv_mgr.check_transfer_done(self.bootstrap_room): # type: ignore - self.conclude_state = KVPoll.Success - return self.conclude_state # type: ignore - - timeout_result = self._check_waiting_timeout() - if timeout_result is not None: - return timeout_result - - return KVPoll.WaitingForInput # type: ignore - - def _register_kv_args(self) -> bool: - for bootstrap_info in self.bootstrap_infos: - packed_kv_data_ptrs = b"".join( - struct.pack("Q", ptr) for ptr in self.kv_mgr.kv_args.kv_data_ptrs - ) - packed_kv_data_mem_kinds = _pack_kv_mem_kinds( - self.kv_mgr.kv_args.kv_data_mem_kinds - ) - packed_kv_item_lens = b"".join( - struct.pack("Q", item_len) - for item_len in self.kv_mgr.kv_args.kv_item_lens - ) - packed_kv_layer_ids = b"".join( - struct.pack("I", layer_id) - for layer_id in self.kv_mgr.kv_args.kv_layer_ids - ) - packed_aux_data_ptrs = b"".join( - struct.pack("Q", ptr) for ptr in self.kv_mgr.kv_args.aux_data_ptrs - ) - packed_state_data_ptrs = pack_int_lists( - self.kv_mgr.kv_args.state_data_ptrs or [], "Q" - ) - packed_state_item_lens = pack_int_lists( - self.kv_mgr.kv_args.state_item_lens or [], "I" - ) - packed_state_data_lens = pack_int_lists( - self.kv_mgr.kv_args.state_data_lens or [], "Q" - ) - packed_state_dim_per_tensor = pack_int_lists( - getattr(self.kv_mgr.kv_args, "state_dim_per_tensor", []) or [], "I" - ) - packed_state_layer_ids = pack_int_lists( - self.kv_mgr.kv_args.state_layer_ids, "I" - ) - - # Include staging allocator metadata if available - if ( - self.kv_mgr.enable_staging - and self.kv_mgr._staging_ctx.allocator is not None - ): - _alloc = self.kv_mgr._staging_ctx.allocator - packed_staging_base_ptr = struct.pack("Q", _alloc.get_base_ptr()) - staging_total_size_str = str(_alloc.get_total_size()).encode("ascii") - else: - packed_staging_base_ptr = b"" - staging_total_size_str = b"" - if self.kv_mgr.kv_args.kv_item_lens: - dst_kv_item_len = self.kv_mgr.kv_args.kv_item_lens[0] - dst_num_slots = num_kv_slots( - self.kv_mgr.kv_args.kv_data_lens, self.kv_mgr.kv_args.kv_item_lens - ) - else: - dst_kv_item_len = 0 - dst_num_slots = 0 - - try: - sock, lock = self._connect_to_bootstrap_server(bootstrap_info) - with lock: - sock.send_multipart( - [ - GUARD, - "None".encode("ascii"), - self.kv_mgr.local_ip.encode("ascii"), - str(self.kv_mgr.rank_port).encode("ascii"), - self.kv_mgr.agent.name.encode("ascii"), - self.kv_mgr.agent.get_agent_metadata(), - packed_kv_data_ptrs, - packed_aux_data_ptrs, - packed_state_data_ptrs, - str(self.kv_mgr.kv_args.gpu_id).encode("ascii"), - str(self.kv_mgr.attn_tp_size).encode("ascii"), - str(self.kv_mgr.kv_args.engine_rank).encode("ascii"), - str(dst_kv_item_len).encode("ascii"), - packed_state_item_lens, - packed_state_dim_per_tensor, - packed_staging_base_ptr, - staging_total_size_str, - str(dst_num_slots).encode("ascii"), - packed_kv_data_mem_kinds, - packed_kv_item_lens, - packed_state_layer_ids, - packed_kv_layer_ids, - str(self.kv_mgr.dcp_size).encode("ascii"), - str(self.kv_mgr.dcp_rank).encode("ascii"), - packed_state_data_lens, - ] - ) - except zmq.ZMQError: - self.kv_mgr.record_failure( - self.bootstrap_room, - f"_register_kv_args to prefill {bootstrap_info.get('rank_ip')}:{bootstrap_info.get('rank_port')} failed", - ) - self.conclude_state = KVPoll.Failed - self.kv_mgr.update_status(self.bootstrap_room, KVPoll.Failed) - return False - return True - - def failure_exception(self): - if self.conclude_state is None: - self.conclude_state = KVPoll.Failed - - self.clear() - - with self.kv_mgr.failure_lock: - failure_reason = self.kv_mgr.failure_records.pop(self.bootstrap_room, None) - is_propagated = failure_reason is None - if is_propagated: - failure_reason = "NIXL KVReceiver Exception" - raise KVTransferError( - self.bootstrap_room, failure_reason, is_from_another_rank=is_propagated - ) + pass class NixlKVBootstrapServer(CommonKVBootstrapServer): diff --git a/test/registered/unit/disaggregation/test_common_sender_lifecycle.py b/test/registered/unit/disaggregation/test_common_sender_lifecycle.py new file mode 100644 index 0000000000..5b5b889dc2 --- /dev/null +++ b/test/registered/unit/disaggregation/test_common_sender_lifecycle.py @@ -0,0 +1,181 @@ +import importlib.util +import queue +import sys +import unittest +from types import ModuleType, SimpleNamespace +from unittest.mock import ANY, Mock, patch + +import numpy as np + +from sglang.srt.disaggregation.base.conn import KVPoll +from sglang.srt.disaggregation.common.conn import CommonKVSender +from sglang.srt.disaggregation.mooncake.conn import MooncakeKVManager, MooncakeKVSender +from sglang.srt.disaggregation.nixl.conn import NixlKVManager +from sglang.srt.disaggregation.utils import DisaggregationMode +from sglang.srt.runtime_context import get_context +from sglang.test.ci.ci_register import register_cpu_ci + +register_cpu_ci(est_time=10, suite="base-a-test-cpu") + + +def make_sender(manager_type, room=42, *, sender_type=CommonKVSender): + manager = object.__new__(manager_type) + manager.disaggregation_mode = DisaggregationMode.PREFILL + manager.request_status = {} + manager.transfer_infos = {room: {"127.0.0.1:1234": SimpleNamespace(dst_port=1234)}} + manager.is_dummy_cp_rank = False + manager.enable_all_cp_ranks_for_transfer = False + manager.attn_cp_size = 1 + manager.attn_cp_rank = 0 + manager.enable_staging = False + manager._staging_outstanding = {} + manager._deferred_ack_targets = {} + manager._deferred_ack_poisoned_rooms = set() + manager._num_shards = 1 + manager.transfer_queues = [queue.Queue()] + manager._transfer_queues = manager.transfer_queues + with get_context().override_server_args(dp_size=1, enable_trace=False): + sender = sender_type(manager, "unused", room) + return sender, manager, manager.transfer_queues[0] + + +class FakePPStatus(dict): + """PP rank 0's request_status: local statuses plus one peer stage.""" + + def __init__(self, local, peer): + super().__init__(local) + self.peer = peer + + def collect(self, room): + return [self.get(room), self.peer.get(room)] + + +class TestCommonSenderLifecycle(unittest.TestCase): + @classmethod + def setUpClass(cls): + # These tests exercise Mori's Python queue adapter, not its RDMA engine. + if "mori" not in sys.modules and importlib.util.find_spec("mori") is None: + package, cpp, io = ( + ModuleType("mori"), + ModuleType("mori.cpp"), + ModuleType("mori.io"), + ) + cpp.TransferStatus = type("TransferStatus", (), {}) + for name in ( + "BackendType", + "EngineDesc", + "IOEngine", + "IOEngineConfig", + "MemoryDesc", + "MemoryLocationType", + "PollCqMode", + "RdmaBackendConfig", + ): + setattr(io, name, type(name, (), {})) + io.StatusCode = SimpleNamespace(SUCCESS=0, IN_PROGRESS=1) + package.cpp, package.io = cpp, io + stub = patch.dict( + sys.modules, {"mori": package, "mori.cpp": cpp, "mori.io": io} + ) + stub.start() + cls.addClassCleanup(stub.stop) + from sglang.srt.disaggregation.mori.conn import MoriKVManager + + cls.mori_manager = MoriKVManager + + def test_two_chunks_keep_backend_queue_contracts(self): + for manager_type in (MooncakeKVManager, self.mori_manager, NixlKVManager): + with self.subTest(backend=manager_type.__name__): + sender, manager, chunks = make_sender(manager_type) + sender.init(4, aux_index=9) + sender.send(np.array([10, 11], dtype=np.int32), num_kv_tokens=128) + sender.send( + np.array([12, 13], dtype=np.int32), + state_indices=[[3, 4], None], + num_kv_tokens=100, + ) + first, last = chunks.get_nowait(), chunks.get_nowait() + np.testing.assert_array_equal(first.prefill_kv_indices, [10, 11]) + self.assertEqual(first.index_slice, slice(0, 2)) + self.assertFalse(first.is_last_chunk) + self.assertIsNone(first.prefill_aux_index) + self.assertIsNone(first.state_indices) + np.testing.assert_array_equal(last.prefill_kv_indices, [12, 13]) + self.assertEqual(last.index_slice, slice(2, 4)) + self.assertTrue(last.is_last_chunk) + self.assertEqual(last.prefill_aux_index, 9) + np.testing.assert_array_equal(last.state_indices[0], [3, 4]) + self.assertIsNone(last.state_indices[1]) + self.assertEqual(last.num_kv_tokens, 100) + self.assertEqual(sender._transfer_num_kv_indices, 4) + self.assertEqual(sender._transfer_num_state_indices, 2) + + def test_mori_consumes_early_event_once_and_skips_cp_replicated_state(self): + sender, manager, chunks = make_sender(self.mori_manager) + manager.attn_cp_size = 2 + manager.attn_cp_rank = 1 + sender.init(2, aux_index=9) + event = object() + sender._early_send_wait_event = event + with get_context().override_server_args(enable_dsa_cache_layer_split=False): + sender.send(np.array([10], dtype=np.int32)) + sender.send(np.array([11], dtype=np.int32), state_indices=[[3]]) + first, last = chunks.get_nowait(), chunks.get_nowait() + self.assertIs(first.wait_event, event) + self.assertIsNone(last.wait_event) + self.assertIsNone(last.state_indices) + self.assertEqual(sender._transfer_num_state_indices, 0) + + def test_success_waits_for_deferred_staging_chunk(self): + sender, manager, _ = make_sender(MooncakeKVManager) + manager.request_status[42] = KVPoll.Success + manager._staging_outstanding[42] = 1 + self.assertEqual(sender.poll(), KVPoll.Transferring) + manager._staging_outstanding[42] = 0 + self.assertEqual(sender.poll(), KVPoll.Success) + + def test_pp_consensus_poll_concludes_on_the_slowest_stage(self): + sender, manager, _ = make_sender(MooncakeKVManager) + manager.bootstrap_timeout = 300 + peer = {} + manager.request_status = FakePPStatus(manager.request_status, peer) + manager.request_status[42] = KVPoll.Success + self.assertEqual(sender.poll_pp_consensus(), KVPoll.Bootstrapping) + peer[42] = KVPoll.Transferring + self.assertEqual(sender.poll_pp_consensus(), KVPoll.Transferring) + peer[42] = KVPoll.Success + manager._staging_outstanding[42] = 1 + self.assertEqual(sender.poll_pp_consensus(), KVPoll.Transferring) + manager._staging_outstanding[42] = 0 + self.assertEqual(sender.poll_pp_consensus(), KVPoll.Success) + self.assertEqual(sender.conclude_state, KVPoll.Success) + + def test_mooncake_trace_context_reaches_worker_and_finishes(self): + trace = Mock(tracing_enable=True) + with ( + patch( + "sglang.srt.disaggregation.common.conn.get_observability", + return_value=SimpleNamespace(enable_trace=True), + ), + patch( + "sglang.srt.disaggregation.common.conn.TraceReqContext", + return_value=trace, + ), + ): + sender, manager, chunks = make_sender( + MooncakeKVManager, sender_type=MooncakeKVSender + ) + sender.init(1, aux_index=9) + sender.send(np.array([1], dtype=np.int32)) + self.assertIs(chunks.get_nowait().trace_ctx, trace.copy_for_thread.return_value) + trace.trace_req_start.assert_called_once() + trace.trace_slice_start.assert_called_once_with("mooncake_send", 1, ANY) + trace.trace_slice_end.assert_called_once_with("mooncake_send", 1) + manager.request_status[42] = KVPoll.Success + self.assertEqual(sender.poll(), KVPoll.Success) + self.assertEqual(sender.poll(), KVPoll.Success) + trace.trace_req_finish.assert_called_once() + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/unit/disaggregation/test_dcp_pack.py b/test/registered/unit/disaggregation/test_dcp_pack.py index ec5b330f62..18d2b2d3db 100644 --- a/test/registered/unit/disaggregation/test_dcp_pack.py +++ b/test/registered/unit/disaggregation/test_dcp_pack.py @@ -104,7 +104,7 @@ class TestDcpCachedPrefixSend(CustomTestCase): scheduler, req, last_chunk=True ) calls = mgr.add_transfer_request.call_args_list - token_counts = [c.args[7] for c in calls] + token_counts = [c.kwargs["num_kv_tokens"] for c in calls] if peer == "dcp": self.assertLessEqual(max(token_counts), limit) else: @@ -124,8 +124,10 @@ class TestDcpCachedPrefixSend(CustomTestCase): self.assertEqual( [c.args[3] for c in calls], [False] * (len(calls) - 1) + [True] ) - self.assertTrue(all(c.args[6] is None for c in calls[:-1])) - self.assertEqual(int(calls[-1].args[6][0][0]), 17) + self.assertTrue( + all(c.kwargs["state_indices"] is None for c in calls[:-1]) + ) + self.assertEqual(int(calls[-1].kwargs["state_indices"][0][0]), 17) class TestDcpPackBufferBytes(CustomTestCase): diff --git a/test/registered/unit/disaggregation/test_deferred_decode_kv_release.py b/test/registered/unit/disaggregation/test_deferred_decode_kv_release.py index 5e914e68ff..5826b4cc46 100644 --- a/test/registered/unit/disaggregation/test_deferred_decode_kv_release.py +++ b/test/registered/unit/disaggregation/test_deferred_decode_kv_release.py @@ -365,6 +365,7 @@ class TestDeferredAckTargets(CustomTestCase): sender = _TestSender.__new__(_TestSender) sender.kv_mgr = mgr sender.bootstrap_room = 7 + sender.conclude_state = None sender.clear() @@ -382,6 +383,7 @@ class TestDeferredAckTargets(CustomTestCase): sender = _TestSender.__new__(_TestSender) sender.kv_mgr = mgr sender.bootstrap_room = 7 + sender.conclude_state = None sender.clear() diff --git a/test/registered/unit/disaggregation/test_disaggregation_wire.py b/test/registered/unit/disaggregation/test_disaggregation_wire.py index 1e7ce1fb05..4404cefc5b 100644 --- a/test/registered/unit/disaggregation/test_disaggregation_wire.py +++ b/test/registered/unit/disaggregation/test_disaggregation_wire.py @@ -597,6 +597,7 @@ class TestDisaggregationWire(unittest.TestCase): manager = object.__new__(MooncakeKVManager) sender = object.__new__(MooncakeKVSender) sender.kv_mgr, sender.bootstrap_room = manager, 42 + sender.conclude_state = None target = AckTarget("127.0.0.1", 1234, 7) for outstanding in (0, 1): with self.subTest(outstanding=outstanding): @@ -1137,7 +1138,7 @@ class _MemcpyNixlAgent: def get_xfer_descs(self, reqs, mem_type): return reqs - def initialize_xfer(self, op, src_descs, dst_descs, peer_name, notif): + def initialize_xfer(self, op, src_descs, dst_descs, peer_name, notif=b""): return src_descs, dst_descs def transfer(self, handle): @@ -1217,7 +1218,6 @@ class TestHeteroTpSwaStateTransfer(CustomTestCase): [dst_ptrs], [self.DST_PAGES], 0, - "7_state_0", decode_tp, decode_tp_rank=dst_rank, dst_state_item_lens=[[item_len] * len(dst_ptrs)], diff --git a/test/registered/unit/disaggregation/test_mooncake_deferred_kv_release.py b/test/registered/unit/disaggregation/test_mooncake_deferred_kv_release.py index bc5e575751..391a3d0d90 100644 --- a/test/registered/unit/disaggregation/test_mooncake_deferred_kv_release.py +++ b/test/registered/unit/disaggregation/test_mooncake_deferred_kv_release.py @@ -56,7 +56,6 @@ def _chunk() -> TransferKVChunk: prefill_kv_indices=np.array([1], dtype=np.int32), index_slice=slice(0, 1), is_last_chunk=False, - chunk_id=0, prefill_aux_index=None, state_indices=None, ) @@ -68,7 +67,6 @@ def _last_chunk() -> TransferKVChunk: prefill_kv_indices=np.array([1], dtype=np.int32), index_slice=slice(0, 1), is_last_chunk=True, - chunk_id=0, prefill_aux_index=0, state_indices=[[1]], ) diff --git a/test/registered/unit/disaggregation/test_mooncake_transfer_batching.py b/test/registered/unit/disaggregation/test_mooncake_transfer_batching.py index 49479f6581..16508287c2 100644 --- a/test/registered/unit/disaggregation/test_mooncake_transfer_batching.py +++ b/test/registered/unit/disaggregation/test_mooncake_transfer_batching.py @@ -418,11 +418,14 @@ class TestMooncakeEarlySend(CustomTestCase): with self.subTest(last=last): ready = MagicMock() manager = MagicMock() - sender = SimpleNamespace( + sender = object.__new__(MooncakeKVSender) + vars(sender).update( kv_mgr=manager, bootstrap_room=1, aux_index=0, trace_ctx=MagicMock(), + conclude_state=None, + _transfer_start_time=None, _early_send_wait_event=ready, _prepare_send_indices=lambda ids, state: ( ids, @@ -432,15 +435,17 @@ class TestMooncakeEarlySend(CustomTestCase): ), _record_transfer_indices=MagicMock(), ) - MooncakeKVSender.send(sender, [0]) - self.assertIs( - manager.add_transfer_request.call_args.kwargs["wait_event"], ready - ) + indices_ready = MagicMock() + sender.send([0], indices_event=indices_ready) + kwargs = manager.add_transfer_request.call_args.kwargs + self.assertIs(kwargs["wait_event"], ready) + self.assertIs(kwargs["indices_event"], indices_ready) ready.synchronize.assert_not_called() - MooncakeKVSender.send(sender, [1]) - self.assertIsNone( - manager.add_transfer_request.call_args.kwargs["wait_event"] - ) + indices_ready.synchronize.assert_not_called() + sender.send([1]) + kwargs = manager.add_transfer_request.call_args.kwargs + self.assertIsNone(kwargs["wait_event"]) + self.assertIsNone(kwargs["indices_event"]) def test_manager_preserves_event_in_queued_chunk(self): queue = SimpleQueue() @@ -452,11 +457,21 @@ class TestMooncakeEarlySend(CustomTestCase): transfer_infos={1: {"127.0.0.1:80": object()}}, transfer_queues=[queue], ) + indices_ready = MagicMock() MooncakeKVManager.add_transfer_request( - manager, 1, [0], slice(0, 1), False, wait_event=ready + manager, + 1, + [0], + slice(0, 1), + False, + wait_event=ready, + indices_event=indices_ready, ) - self.assertIs(queue.get_nowait().wait_event, ready) + chunk = queue.get_nowait() + self.assertIs(chunk.wait_event, ready) + self.assertIs(chunk.indices_event, indices_ready) ready.synchronize.assert_not_called() + indices_ready.synchronize.assert_not_called() def _worker(self, event): chunk = TransferKVChunk( diff --git a/test/registered/unit/disaggregation/test_nixl_backend_basic.py b/test/registered/unit/disaggregation/test_nixl_backend_basic.py index 9a3e7552f2..1a6f9a973d 100644 --- a/test/registered/unit/disaggregation/test_nixl_backend_basic.py +++ b/test/registered/unit/disaggregation/test_nixl_backend_basic.py @@ -19,6 +19,7 @@ from sglang.srt.disaggregation.common.staging_handler import PrefillStagingConte from sglang.srt.disaggregation.nixl.conn import ( KVArgsRegisterInfo, NixlKVManager, + NixlKVReceiver, TransferInfo, TransferKVChunk, ) @@ -132,10 +133,6 @@ class TestNixlBackendInitialization(CustomTestCase): ), get_context().override_server_args(device="cuda") as server_args, patch.object(CommonKVManager, "__init__", return_value=None), - patch( - "sglang.srt.disaggregation.nixl.conn.get_parallel", - return_value=SimpleNamespace(tp_size=4), - ), patch( "sglang.srt.disaggregation.nixl.conn.torch.get_device_module", return_value=device_module, @@ -298,7 +295,6 @@ class TestNixlEmptyStateTransfer(CustomTestCase): peer.dst_state_data_ptrs, [[2]], 0, - "7_state_0", 1, ) ) @@ -320,7 +316,6 @@ class TestNixlEmptyStateTransfer(CustomTestCase): prefill_data_indices=np.array([3], dtype=np.int32), dst_data_indices=np.array([5], dtype=np.int32), dst_gpu_id=0, - notif="qsa-mismatch", state_type=StateType.QSA_PENDING, force_flat=True, src_layer_ids=[24], @@ -374,13 +369,12 @@ class TestNixlTransferWorker(CustomTestCase): mgr.is_mla_backend = False mgr.is_hybrid_mla_backend = False mgr.attn_tp_size = 1 - mgr.transfer_source_rank = 0 mgr.kv_args = SimpleNamespace( engine_rank=0, kv_data_ptrs=[0], num_draft_entries=0 ) - mgr.exceptions = {} mgr.failure_lock = threading.Lock() mgr.failure_records = {} + mgr.send_kv_status_message = MagicMock() def check_xfer_state(_handle): mgr.update_status(room, KVPoll.Failed) @@ -395,7 +389,6 @@ class TestNixlTransferWorker(CustomTestCase): prefill_kv_indices=np.array(prefill_kv_indices, dtype=np.int32), index_slice=slice(0, len(prefill_kv_indices)), is_last_chunk=is_last_chunk, - chunk_id=0, prefill_aux_index=0 if is_last_chunk else None, state_indices=None, ) @@ -419,7 +412,6 @@ class TestNixlTransferWorker(CustomTestCase): self.assertNotIn(room, mgr.transfer_infos) self.assertNotIn(room, mgr.req_to_decode_prefix_len) mgr.send_aux.assert_called_once() - self.assertEqual(mgr.send_aux.call_args.args[-1], "21_aux_nokv_0_0") def test_given_non_last_chunk_aborts_mid_transfer_when_worker_finishes_then_failed_status_is_preserved( self, @@ -524,6 +516,141 @@ class TestNixlTransferWorker(CustomTestCase): ) self.assertEqual(submitted_counts_at_poll, [3, 3, 3]) + def test_success_is_sent_after_kv_aux_and_state_handles_complete(self): + room = 24 + mgr = self._make_manager(room) + peer = mgr.decode_kv_args_table["agent"] + peer.dst_state_data_ptrs = [[0]] + peer.decode_tp_rank = 0 + peer.dst_state_item_lens = [[4]] + peer.dst_state_dim_per_tensor = [[1]] + peer.dst_state_layer_ids = [[0]] + events = [] + mgr.send_kvcache = MagicMock(return_value="kv") + mgr.maybe_send_extra = MagicMock(return_value=["state0", "state1"]) + mgr.send_aux = MagicMock(return_value="aux") + mgr.agent.check_xfer_state = lambda handle: events.append(handle) or "DONE" + mgr.send_kv_status_message.side_effect = lambda **kwargs: events.append( + kwargs["status"] + ) + chunk = self._make_chunk(room, [1], is_last_chunk=True) + chunk.state_indices = [[0]] + + self._run_worker_once(mgr, chunk) + + self.assertEqual(events, ["kv", "state0", "state1", "aux", KVPoll.Success]) + self.assertEqual( + mgr.send_kv_status_message.call_args.kwargs["targets"], + [("127.0.0.1", 5555)], + ) + self.assertNotIn(room, mgr.transfer_infos) + + def test_failed_handle_never_emits_success(self): + room = 25 + mgr = self._make_manager(room) + mgr.send_kvcache = MagicMock(return_value="kv") + mgr.send_aux = MagicMock(return_value="aux") + mgr.agent.check_xfer_state = lambda handle: "ERR" if handle == "kv" else "DONE" + self._run_worker_once(mgr, self._make_chunk(room, [1], is_last_chunk=True)) + self.assertEqual(mgr.request_status[room], KVPoll.Failed) + mgr.send_kv_status_message.assert_called_once() + self.assertEqual( + mgr.send_kv_status_message.call_args.kwargs["status"], KVPoll.Failed + ) + + def _make_staging_manager(self, room): + mgr = self._make_manager(room) + req = TransferInfo(**vars(mgr.transfer_infos[room]["agent"])) + req.dst_kv_indices = np.array([100, 101], dtype=np.int32) + mgr.transfer_infos[room] = {"agent": req} + peer = SimpleNamespace(**vars(mgr.decode_kv_args_table["agent"])) + peer.decode_tp_size = 2 + peer.staging_base_ptr = 0x8000 + mgr.decode_kv_args_table = {"agent": peer} + mgr.enable_staging = True + mgr._staging_ctx = PrefillStagingContext() + mgr._try_create_staging_strategy = MagicMock(return_value=object()) + mgr.agent.check_xfer_state = MagicMock(return_value="DONE") + mgr.send_aux = MagicMock(side_effect=lambda peer, *args: f"aux-{peer}") + mgr.send_chunk_ready = MagicMock() + return mgr + + def _run_staging_worker(self, mgr, chunks): + pending = list(chunks) + + def get(): + if not pending: + raise SystemExit() + return pending.pop(0) + + queue = SimpleNamespace(get=get, put=pending.append) + with self.assertRaises(SystemExit): + mgr.transfer_worker(queue, staging_buffer=object()) + + def test_deferred_staging_chunk_still_transfers_after_last_chunk_success(self): + room = 27 + mgr = self._make_staging_manager(room) + first = self._make_chunk(room, [1], is_last_chunk=False) + last = self._make_chunk(room, [2], is_last_chunk=True) + last.index_slice = slice(1, 2) + events = [] + attempts = [] + + def stage(_strategy, chunk, _indices, _req, _info, queue): + page_start = chunk.index_slice.start + attempts.append(page_start) + if len(attempts) == 1: + queue.put(chunk) + return None, True, 0 + return f"kv-{page_start}", False, page_start + + mgr._do_staging_transfer = stage + mgr.send_chunk_ready.side_effect = lambda **kw: events.append( + ("chunk", kw["chunk_idx"]) + ) + mgr.send_kv_status_message.side_effect = lambda **kw: events.append( + ("status", kw["status"]) + ) + self._run_staging_worker(mgr, [first, last]) + self.assertEqual(attempts, [0, 1, 0]) + self.assertEqual( + events, [("chunk", 1), ("status", KVPoll.Success), ("chunk", 0)] + ) + self.assertNotIn(room, mgr.transfer_infos) + self.assertNotIn(room, mgr._staging_outstanding) + + +class TestNixlDecodeListener(CustomTestCase): + def test_decode_listener_receives_zmq_chunk_and_success(self): + mgr = object.__new__(NixlKVManager) + mgr.request_status = {} + mgr.failure_records = {} + mgr.failure_lock = threading.Lock() + mgr.prefill_response_tracker = {} + mgr.required_prefill_response_num_table = {11: 1} + mgr.addr_to_rooms_tracker = defaultdict(set) + mgr._deferred_ack_targets = {} + mgr._deferred_ack_poisoned_rooms = set() + mgr._staging_outstanding = {} + mgr.enable_staging = False + mgr._staging_handler = MagicMock() + receiver = NixlKVReceiver(mgr, "prefill:8998", 11) + mgr.update_status(11, KVPoll.WaitingForInput) + mgr.server_socket = MagicMock() + mgr.server_socket.recv_multipart.side_effect = [ + [b"CHUNK_READY", b"11", b"0", b"0", b"2", b"decode_agent", b"0"], + [b"KV_STATUS", b"11", str(int(KVPoll.Success)).encode(), b"0", b""], + SystemExit(), + ] + with patch("sglang.srt.disaggregation.nixl.conn.threading.Thread") as thread: + mgr._start_decode_listener_thread() + with self.assertRaises(SystemExit): + thread.call_args.kwargs["target"]() + mgr._staging_handler.handle_chunk_arrived.assert_called_once_with( + 11, 0, 0, 2, "decode_agent" + ) + self.assertEqual(receiver.poll(), KVPoll.Success) + class TestNixlNodeFailure(CustomTestCase): def _make_manager(self): @@ -619,7 +746,6 @@ class TestNixlStaging(CustomTestCase): mgr.agent = agent or StagingFakeAgent() mgr.attn_tp_size = 2 mgr.is_mla_backend = False - mgr.transfer_source_rank = 1 mgr.kv_args = SimpleNamespace( gpu_id=1, engine_rank=1, @@ -640,7 +766,6 @@ class TestNixlStaging(CustomTestCase): prefill_kv_indices=np.array([10, 11], dtype=np.int32), index_slice=slice(0, 2), is_last_chunk=False, - chunk_id=0, prefill_aux_index=None, state_indices=None, ) @@ -655,7 +780,7 @@ class TestNixlStaging(CustomTestCase): ) }, ): - handle, deferred = mgr._do_staging_transfer( + handle, deferred, chunk_idx = mgr._do_staging_transfer( strategy, kv_chunk, kv_chunk.prefill_kv_indices, @@ -683,7 +808,6 @@ class TestNixlStaging(CustomTestCase): prefill_kv_indices=np.array([10], dtype=np.int32), index_slice=slice(0, 1), is_last_chunk=False, - chunk_id=0, prefill_aux_index=None, state_indices=None, ) @@ -843,7 +967,7 @@ class ReloadFakeAgent: raise RuntimeError("NIXL_ERR_NOT_FOUND") return name, self.sections.get(name) - def make_prepped_xfer(self, op, src, src_indices, dst, dst_indices, notif): + def make_prepped_xfer(self, op, src, src_indices, dst, dst_indices, notif=b""): name, generation = dst if generation != self.sections.get(name): raise RuntimeError("NIXL_ERR_NOT_FOUND") @@ -944,8 +1068,6 @@ class TestNixlInvalidatedPeerReload(CustomTestCase): mgr._staging_outstanding = defaultdict(int) mgr._deferred_ack_targets = {} mgr._deferred_ack_poisoned_rooms = set() - mgr.transfer_source_rank = 0 - mgr.exceptions = {} mgr.failure_lock = threading.Lock() mgr.failure_records = {} mgr.send_kv_status_message = MagicMock() @@ -976,7 +1098,6 @@ class TestNixlInvalidatedPeerReload(CustomTestCase): prefill_kv_indices=np.array([1], dtype=np.int32), index_slice=slice(0, 1), is_last_chunk=True, - chunk_id=0, prefill_aux_index=0, state_indices=None, ) @@ -1016,7 +1137,6 @@ class TestNixlInvalidatedPeerReload(CustomTestCase): mgr.decode_kv_args_table["dropped"].dst_kv_ptrs, np.array([2], dtype=np.int32), 0, - "notif", ) self.assertEqual(handle, ("dropped", mgr.agent.sections["dropped"])) diff --git a/test/registered/unit/disaggregation/test_nixl_deferred_kv_release.py b/test/registered/unit/disaggregation/test_nixl_deferred_kv_release.py index 868883685b..3dfe72dced 100644 --- a/test/registered/unit/disaggregation/test_nixl_deferred_kv_release.py +++ b/test/registered/unit/disaggregation/test_nixl_deferred_kv_release.py @@ -58,7 +58,6 @@ def _chunk() -> TransferKVChunk: prefill_kv_indices=np.array([1], dtype=np.int32), index_slice=slice(0, 1), is_last_chunk=False, - chunk_id=0, prefill_aux_index=None, state_indices=None, ) diff --git a/test/registered/unit/disaggregation/test_receiver_connection_pool.py b/test/registered/unit/disaggregation/test_receiver_connection_pool.py index 0ed94255ae..c02ab9868f 100644 --- a/test/registered/unit/disaggregation/test_receiver_connection_pool.py +++ b/test/registered/unit/disaggregation/test_receiver_connection_pool.py @@ -5,13 +5,23 @@ from sglang.test.ci.ci_register import register_cpu_ci register_cpu_ci(est_time=8, suite="base-a-test-cpu") +import importlib.util +import struct +import sys import threading import unittest -from types import SimpleNamespace +from collections import defaultdict +from types import ModuleType, SimpleNamespace from unittest.mock import Mock, patch +import msgspec +import numpy as np +import zmq + from sglang.srt.disaggregation.base.conn import KVPoll -from sglang.srt.disaggregation.common.conn import CommonKVReceiver +from sglang.srt.disaggregation.common.conn import CommonKVManager, CommonKVReceiver +from sglang.srt.disaggregation.mooncake.conn import MooncakeKVManager +from sglang.srt.disaggregation.nixl.conn import NixlKVManager from sglang.test.test_utils import CustomTestCase @@ -23,6 +33,20 @@ class _ConcreteReceiver(CommonKVReceiver): raise NotImplementedError +def _manager(manager_class=CommonKVManager): + """Construct control-plane state without creating an engine or worker threads.""" + manager = manager_class.__new__(manager_class) + manager.request_status = {} + manager.prefill_response_tracker = {} + manager.required_prefill_response_num_table = {} + manager.addr_to_rooms_tracker = defaultdict(set) + manager._staging_outstanding = {} + manager._deferred_ack_targets = {} + manager._deferred_ack_poisoned_rooms = set() + manager.enable_staging = False + return manager + + def _receiver(connection_pool, entries): receiver = object.__new__(_ConcreteReceiver) receiver.kv_mgr = SimpleNamespace( @@ -99,5 +123,350 @@ class TestReceiverConnectionPool(CustomTestCase): self.assertEqual(receiver.kv_mgr.connection_pool, {}) +class TestReceiverControlMessages(CustomTestCase): + @classmethod + def setUpClass(cls): + super().setUpClass() + if "mori" not in sys.modules and importlib.util.find_spec("mori") is None: + package = ModuleType("mori") + cpp = ModuleType("mori.cpp") + cpp.TransferStatus = type("TransferStatus", (), {}) + io = ModuleType("mori.io") + for name in ( + "BackendType", + "EngineDesc", + "IOEngine", + "IOEngineConfig", + "MemoryDesc", + "MemoryLocationType", + "PollCqMode", + "RdmaBackendConfig", + ): + setattr(io, name, type(name, (), {})) + io.StatusCode = SimpleNamespace(SUCCESS=0, IN_PROGRESS=1) + package.cpp, package.io = cpp, io + stub = patch.dict( + sys.modules, {"mori": package, "mori.cpp": cpp, "mori.io": io} + ) + stub.start() + cls.addClassCleanup(stub.stop) + from sglang.srt.disaggregation.mori.conn import MoriKVManager + + cls.backends = { + "mooncake": MooncakeKVManager, + "nixl": NixlKVManager, + "mori": MoriKVManager, + } + + def _make_receiver(self, backend): + manager = _manager(self.backends[backend]) + manager.local_ip = "127.0.0.1" + manager.rank_port = 5000 + manager.attn_tp_size = 2 + manager.dcp_size, manager.dcp_rank = 1, 0 + manager.kv_args = SimpleNamespace( + kv_data_ptrs=[0x1000], + kv_data_lens=[256], + kv_item_lens=[16], + kv_data_mem_kinds=["VRAM"], + kv_layer_ids=[7], + aux_data_ptrs=[0x2000], + state_data_ptrs=[[0x3000]], + state_data_lens=[[512]], + state_item_lens=[[32]], + state_dim_per_tensor=[[4]], + state_layer_ids=[[7]], + engine_rank=0, + gpu_id=0, + ) + manager.agent = SimpleNamespace( + name="nixl-peer", get_agent_metadata=lambda: b"agent-desc" + ) + manager.engine_desc = SimpleNamespace( + key="mori-peer", pack=lambda: b"engine-desc" + ) + manager.kv_mem_descs = [SimpleNamespace(pack=lambda: b"kv-desc")] + manager.aux_mem_descs = [SimpleNamespace(pack=lambda: b"aux-desc")] + manager.state_mem_descs = [[SimpleNamespace(pack=lambda: b"state-desc")]] + manager.engine = SimpleNamespace(get_session_id=lambda: "mooncake-peer") + receiver = CommonKVReceiver(manager, "prefill:8998", 17) + receiver.required_dst_info_num = 3 + receiver.bootstrap_infos = [ + {"rank_ip": "::1", "rank_port": port, "is_dummy": port == 6001} + for port in (6000, 6001, 6002) + ] + sent = [] + + def connect(info): + def send(frames): + sent.append((info["rank_port"], frames)) + + return SimpleNamespace(send_multipart=send), threading.Lock() + + receiver._connect_to_bootstrap_server = connect + return receiver, sent + + def test_registration_wire_is_preserved_for_all_peers(self): + """Registration refactors must preserve each backend's legacy frame layout.""" + ptr = struct.pack("Q", 0x1000) + aux = struct.pack("Q", 0x2000) + state = struct.pack("