[Refactor] pd: Consolidate sender and receiver implementations (#41140)

Co-authored-by: Shangming Cai <csmthu@gmail.com>
This commit is contained in:
jambow0320
2026-10-24 10:50:10 +08:00
committed by GitHub
co-authored by Shangming Cai
parent 88ac7c02a6
commit de7055f596
15 changed files with 1402 additions and 1400 deletions
+301 -18
View File
@@ -8,7 +8,16 @@ import threading
import time
from collections import defaultdict
from contextlib import nullcontext
from typing import Dict, List, NamedTuple, Optional, Set, Tuple, Union
from typing import (
Dict,
List,
Literal,
NamedTuple,
Optional,
Set,
Tuple,
Union,
)
import msgspec
import numpy as np
@@ -42,9 +51,11 @@ from sglang.srt.disaggregation.utils import (
)
from sglang.srt.environ import envs
from sglang.srt.mem_cache.page_interleave import get_kv_shard_group
from sglang.srt.observability.trace import TraceNullContext, TraceReqContext
from sglang.srt.runtime_context import (
get_device,
get_disagg,
get_observability,
get_parallel,
get_schedule,
get_serving,
@@ -275,6 +286,7 @@ class DeferredAbortAckState(msgspec.Struct):
class CommonKVManager(BaseKVManager):
defer_decode_allocation: bool = False
deferred_bootstrap: Optional[DeferredBootstrap] = None
_staging_ctx = None
# Wire layout of the prefill->decode terminal status message. The legacy
# layout (mooncake, and ascend which inherits it) is three untagged frames
@@ -283,6 +295,10 @@ class CommonKVManager(BaseKVManager):
# ``[tag, room, status, prefill_rank, reason]``.
kv_status_msg_tag: Optional[bytes] = None
kv_status_msg_carries_reason: bool = False
sender_trace_module: Optional[str] = None
# Sender reports transfer_latency_s (first enqueue -> Success). Otherwise
# the metric keeps its last-chunk fallback, so dashboards do not move.
reports_transfer_latency: bool = False
kv_layout: Optional[str] = None
dsv41_spec_layout: Optional[dict] = None
@@ -453,7 +469,7 @@ class CommonKVManager(BaseKVManager):
self.session_pool: Dict = defaultdict(requests.Session)
self.session_pool_lock = threading.Lock()
self.addr_to_rooms_tracker: Dict[str, Set[int]] = defaultdict(set)
self.prefill_response_tracker: Dict[int, Set[int]] = defaultdict(set)
self.prefill_response_tracker: Dict[int, Set[int]] = {}
# Deferred KV release: room -> current generation and ranks that
# ACKed. The generation rejects an old request's late ACK after room
# reuse.
@@ -489,6 +505,11 @@ class CommonKVManager(BaseKVManager):
f"Unsupported DisaggregationMode: {self.disaggregation_mode}"
)
def prepare_send_state_indices(
self, state_indices: Optional[List]
) -> Optional[List]:
return state_indices
def _should_skip_cp_replicated_state_transfer(self) -> bool:
"""Whether this prefill rank should omit CP-replicated state.
@@ -590,6 +611,23 @@ class CommonKVManager(BaseKVManager):
return KVPoll.Bootstrapping
return min(statuses)
def build_receiver_registration_message(self) -> List[bytes]:
raise NotImplementedError
def build_receiver_metadata_message(
self,
*,
bootstrap_room: int,
required_dst_info_num: int,
kv_indices: npt.NDArray[np.int32],
aux_index: Optional[int],
state_indices: Optional[List],
decode_prefix_len: Optional[int],
is_dummy: bool,
device_kv_indices: Optional[npt.NDArray[np.int32]] = None,
) -> List[bytes]:
raise NotImplementedError
def update_status(self, bootstrap_room: int, status: KVPoll):
current = self.request_status.get(bootstrap_room)
if current is None:
@@ -747,6 +785,51 @@ class CommonKVManager(BaseKVManager):
f"{na.to_host_port_str()}: {e}"
)
def send_chunk_ready(
self,
*,
room: int,
chunk_idx: int,
page_start: int,
num_pages: int,
writer_id: str,
targets: List[Tuple[str, int]],
) -> None:
"""Notify decode that a staging chunk RDMA is complete (every chunk;
scatter is arrival-driven)."""
parts = [
b"CHUNK_READY",
str(room).encode("ascii"),
str(chunk_idx).encode("ascii"),
str(page_start).encode("ascii"),
str(num_pages).encode("ascii"),
writer_id.encode("ascii"),
str(self._prefill_unique_rank()).encode("ascii"),
]
for endpoint, port in targets:
na = NetworkAddress(endpoint, port)
self._send_multipart_locked(na.to_tcp(), parts, is_ipv6=na.is_ipv6)
def handle_chunk_ready(self, msg: List[bytes]) -> bool:
if not msg or msg[0] != b"CHUNK_READY":
return False
try:
room, chunk_idx, page_start, num_pages = (
int(part.decode("ascii")) for part in msg[1:5]
)
writer_id = msg[5].decode("ascii")
except (IndexError, ValueError, UnicodeDecodeError):
logger.warning("Dropping malformed CHUNK_READY message")
return True
if room not in self.request_status:
return True
handler = self._staging_handler
if handler is None:
logger.warning("CHUNK_READY for room %s without a staging handler", room)
return True
handler.handle_chunk_arrived(room, chunk_idx, page_start, num_pages, writer_id)
return True
def conclude_transfer(
self,
*,
@@ -834,10 +917,13 @@ class CommonKVManager(BaseKVManager):
self.update_status(bootstrap_room, KVPoll.WaitingForInput)
return
if status == KVPoll.Success:
self.prefill_response_tracker[bootstrap_room].add(prefill_rank)
completed_ranks = self.prefill_response_tracker.get(bootstrap_room)
if completed_ranks is None:
return
expected_response_num = self.required_prefill_response_num_table.get(
bootstrap_room
)
completed_ranks.add(prefill_rank)
if expected_response_num is None:
logger.warning(
"No expected prefill response count for room %s, prefill rank %s",
@@ -845,14 +931,10 @@ class CommonKVManager(BaseKVManager):
prefill_rank,
)
return
if (
len(self.prefill_response_tracker[bootstrap_room])
< expected_response_num
):
if len(completed_ranks) < expected_response_num:
return
# Tell the staging handler no more chunks are coming, before any
# poller can see Success. Only mooncake gets here: NIXL arms the
# handler from its own notifications, mori has no staging.
# poller can see Success.
if self.enable_staging and self._staging_handler is not None:
handler = self._staging_handler
if handler.is_staging_room(bootstrap_room):
@@ -2007,6 +2089,19 @@ class CommonKVSender(BaseKVSender):
self._transfer_metric = KVTransferMetric()
self._transfer_num_kv_indices = 0
self._transfer_num_state_indices = 0
self._transfer_start_time: Optional[float] = None
self._early_send_wait_event: Optional[object] = None
self.trace_ctx = TraceNullContext()
if self.kv_mgr.sender_trace_module and get_observability().enable_trace:
self.trace_ctx = TraceReqContext(
rid=str(hex(self.bootstrap_room)),
bootstrap_room=self.bootstrap_room,
role="Sender",
module_name=self.kv_mgr.sender_trace_module,
)
if not self.trace_ctx.tracing_enable:
self.trace_ctx = TraceNullContext()
self.trace_ctx.trace_req_start()
# inner state
self.curr_idx = 0
self.init_time: Optional[float] = None
@@ -2051,6 +2146,8 @@ class CommonKVSender(BaseKVSender):
)
self.kv_mgr.update_status(self.bootstrap_room, KVPoll.Failed)
return
# Bootstrap timeout clock starts after the registration above.
self.init_time = time.time()
def _register_prefill_dp_rank(self):
"""Register this request's prefill dp_rank to the bootstrap server."""
@@ -2175,8 +2272,82 @@ class CommonKVSender(BaseKVSender):
kv_indices: npt.NDArray[np.int32],
state_indices: Optional[List] = None,
num_kv_tokens: Optional[int] = None,
indices_event: Optional[object] = None,
):
pass
if self.conclude_state in (KVPoll.Success, KVPoll.Failed):
return
kv_indices, index_slice, is_last_chunk, should_skip = (
self._prepare_send_indices(kv_indices, state_indices)
)
if should_skip:
return
state_indices = self.kv_mgr.prepare_send_state_indices(state_indices)
if self._transfer_start_time is None and (
len(kv_indices) > 0 or state_indices is not None
):
self._transfer_start_time = time.perf_counter()
# The early-send event belongs to exactly one chunk; the Mooncake and
# Mori workers synchronize on it before reading source KV.
wait_event = self._early_send_wait_event
self._early_send_wait_event = None
self.kv_mgr.add_transfer_request(
self.bootstrap_room,
kv_indices,
index_slice,
is_last_chunk,
aux_index=self.aux_index if is_last_chunk else None,
state_indices=state_indices if is_last_chunk else None,
num_kv_tokens=num_kv_tokens,
wait_event=wait_event,
indices_event=indices_event,
trace_ctx=self.trace_ctx.copy_for_thread(),
)
self._record_transfer_indices(kv_indices, state_indices)
def poll(self) -> KVPoll:
return self._poll_with_status(self.kv_mgr.check_status)
def poll_pp_consensus(self) -> KVPoll:
return self._poll_with_status(self.kv_mgr.check_status_pp_consensus)
def _poll_with_status(self, check_status) -> KVPoll:
if self.conclude_state is not None:
return self.conclude_state
if self.bootstrap_room in self.kv_mgr.request_status:
status = check_status(self.bootstrap_room)
else:
status = KVPoll.Failed
# Hold Success until all staging chunks transferred. With deferred
# allocation also wait for worker cleanup (the zero entry is only
# removed after cleanup), before permitting the room to be reused.
if status == KVPoll.Success and (
self.kv_mgr._staging_outstanding.get(self.bootstrap_room, 0) > 0
or (
self.kv_mgr.defer_decode_allocation
and self.bootstrap_room in self.kv_mgr._staging_outstanding
)
):
return KVPoll.Transferring
if status == KVPoll.Bootstrapping:
timeout_status = self._check_bootstrap_timeout()
if timeout_status is not None:
status = timeout_status
if status in (KVPoll.Success, KVPoll.Failed):
self.conclude_state = status
if (
status == KVPoll.Success
and self.kv_mgr.reports_transfer_latency
and self._transfer_start_time is not None
):
self._transfer_metric.transfer_latency_s = (
time.perf_counter() - self._transfer_start_time
)
self.trace_ctx.trace_req_finish()
return status
def _check_bootstrap_timeout(self) -> Optional[KVPoll]:
start = (
@@ -2214,12 +2385,22 @@ class CommonKVSender(BaseKVSender):
== KVPoll.Success,
)
self._owns_bootstrap_room = False
if self.conclude_state is None:
status = self.kv_mgr.request_status.get(self.bootstrap_room)
if status in (KVPoll.Success, KVPoll.Failed):
self.conclude_state = status
self.kv_mgr.request_status.pop(self.bootstrap_room, None)
self.kv_mgr.req_to_decode_prefix_len.pop(self.bootstrap_room, None)
self.kv_mgr.transfer_infos.pop(self.bootstrap_room, None)
# A sender can be cleared while a counted transfer is still writing:
# keep the target for the worker, or ACK now if nothing is in flight.
self.kv_mgr._maybe_ack_drained_abort(self.bootstrap_room)
staging_ctx = self.kv_mgr._staging_ctx
if staging_ctx is not None:
staging_ctx.prefetched_rooms.discard(self.bootstrap_room)
for key in list(staging_ctx.prefetch_requested):
if key[0] == self.bootstrap_room:
staging_ctx.prefetch_requested.discard(key)
if notification is not None:
endpoint, failed = notification
self.kv_mgr.send_kv_status_message(
@@ -2229,6 +2410,18 @@ class CommonKVSender(BaseKVSender):
failure_reason="Prefill bootstrap failed" if failed else None,
)
def failure_exception(self):
self.conclude_state = KVPoll.Failed
self.clear()
with self.kv_mgr.failure_lock:
failure_reason = self.kv_mgr.failure_records.pop(self.bootstrap_room, None)
self.trace_ctx.trace_req_finish()
raise KVTransferError(
self.bootstrap_room,
failure_reason or "Failed due to an unknown reason from another rank",
is_from_another_rank=failure_reason is None,
)
def abort(self):
if self.kv_mgr.deferred_bootstrap is not None and not self._owns_bootstrap_room:
return
@@ -2238,6 +2431,8 @@ class CommonKVSender(BaseKVSender):
)
self.kv_mgr.update_status(self.bootstrap_room, KVPoll.Failed)
self.conclude_state = KVPoll.Failed
self.trace_ctx.abort(abort_info={"reason": "Aborted"})
self.trace_ctx.trace_req_finish()
class CommonKVReceiver(BaseKVReceiver):
@@ -2278,6 +2473,7 @@ class CommonKVReceiver(BaseKVReceiver):
self.conclude_state = KVPoll.Failed
return
self.kv_mgr.addr_to_rooms_tracker[self.bootstrap_addr].add(self.bootstrap_room)
self.kv_mgr.prefill_response_tracker.setdefault(self.bootstrap_room, set())
self.kv_mgr.update_status(self.bootstrap_room, KVPoll.Bootstrapping)
def init(self, prefill_dp_rank: int):
@@ -2571,17 +2767,90 @@ class CommonKVReceiver(BaseKVReceiver):
return sock, lock
def _register_kv_args(self) -> bool:
if self.bootstrap_infos is None:
return False
if not self.bootstrap_infos:
return True
message = self.kv_mgr.build_receiver_registration_message()
for bootstrap_info in self.bootstrap_infos:
if not self._send_bootstrap_message(
bootstrap_info=bootstrap_info,
message=message,
operation="_register_kv_args",
):
return False
return True
def _send_bootstrap_message(
self,
*,
bootstrap_info: dict,
message: List[bytes],
operation: Literal["_register_kv_args", "send_metadata"],
) -> bool:
try:
sock, lock = self._connect_to_bootstrap_server(bootstrap_info)
with lock:
sock.send_multipart(message)
except zmq.ZMQError:
self.kv_mgr.record_failure(
self.bootstrap_room,
f"{operation} to prefill {bootstrap_info.get('rank_ip')}:{bootstrap_info.get('rank_port')} failed",
)
self.conclude_state = KVPoll.Failed
self.kv_mgr.update_status(self.bootstrap_room, KVPoll.Failed)
return False
return True
def send_metadata(
self,
kv_indices: npt.NDArray[np.int32],
aux_index: Optional[int] = None,
state_indices: Optional[List[int]] = None,
state_indices: Optional[List] = None,
decode_prefix_len: Optional[int] = None,
destination: KVTransferDestination = KVTransferDestination.DEVICE,
device_kv_indices: Optional[npt.NDArray[np.int32]] = None,
):
raise NotImplementedError
if self.bootstrap_infos is None:
self.kv_mgr.record_failure(
self.bootstrap_room,
f"Could not fetch prefill parallel info from bootstrap_addr: {self.bootstrap_addr}",
)
self.kv_mgr.update_status(self.bootstrap_room, KVPoll.Failed)
return
if destination != KVTransferDestination.DEVICE:
raise NotImplementedError("Host KV destinations are not supported")
self.chunk_staging_infos = []
if (
self.kv_mgr.enable_staging
and self.kv_mgr._staging_ctx.allocator is not None
):
self.kv_mgr.register_staging_room_bootstrap(
self.bootstrap_room, self.bootstrap_infos, self
)
for bootstrap_info in self.bootstrap_infos:
message = self.kv_mgr.build_receiver_metadata_message(
bootstrap_room=self.bootstrap_room,
required_dst_info_num=self.required_dst_info_num,
kv_indices=kv_indices,
aux_index=aux_index,
state_indices=state_indices,
decode_prefix_len=decode_prefix_len,
is_dummy=bootstrap_info["is_dummy"],
device_kv_indices=device_kv_indices,
)
if not self._send_bootstrap_message(
bootstrap_info=bootstrap_info,
message=message,
operation="send_metadata",
):
self.invalidate_cached_bootstrap_infos()
return
self.init_time = time.time()
def _check_waiting_timeout(self) -> Optional[KVPoll]:
if self.init_time is None:
@@ -2625,6 +2894,11 @@ class CommonKVReceiver(BaseKVReceiver):
# Preserve the recorded root cause; this is cleanup, not a new abort.
self._send_abort_notification()
self.abort_notified = True
if self.conclude_state is None:
status = self.kv_mgr.request_status.get(self.bootstrap_room)
self.conclude_state = (
status if status in (KVPoll.Success, KVPoll.Failed) else KVPoll.Failed
)
self.kv_mgr.request_status.pop(self.bootstrap_room, None)
self.kv_mgr.required_prefill_response_num_table.pop(self.bootstrap_room, None)
self.kv_mgr.prefill_response_tracker.pop(self.bootstrap_room, None)
@@ -2634,16 +2908,25 @@ class CommonKVReceiver(BaseKVReceiver):
if self.kv_mgr.defer_decode_allocation:
self._owns_bootstrap_room = False
def failure_exception(self):
self.conclude_state = KVPoll.Failed
self.clear()
with self.kv_mgr.failure_lock:
failure_reason = self.kv_mgr.failure_records.pop(self.bootstrap_room, None)
raise KVTransferError(
self.bootstrap_room,
failure_reason or "Failed due to an unknown reason from another rank",
is_from_another_rank=failure_reason is None,
)
def abort(self):
if not self._owns_bootstrap_room:
self.conclude_state = KVPoll.Failed
return
self.kv_mgr.record_failure(
self.bootstrap_room,
"Aborted by AbortReq.",
)
self.kv_mgr.update_status(self.bootstrap_room, KVPoll.Failed)
self.conclude_state = KVPoll.Failed
if self.conclude_state != KVPoll.Failed:
self.kv_mgr.record_failure(self.bootstrap_room, "Aborted by AbortReq.")
self.kv_mgr.update_status(self.bootstrap_room, KVPoll.Failed)
self.conclude_state = KVPoll.Failed
self.ensure_abort_notified()
def ensure_abort_notified(self, *, force_arm: bool = False) -> None:
@@ -312,7 +312,7 @@ class DecodeStagingHandler:
num_pages: int,
writer_id: str,
) -> bool:
"""Process a staging chunk arrival from any transport (NIXL RDMA notif or ZMQ CHUNK_READY).
"""Process a staging chunk arrival reported by a ZMQ CHUNK_READY message.
Accumulates writer arrivals and submits scatter once all writers for
this chunk have reported in. Returns True if scatter was submitted.
@@ -24,7 +24,6 @@ class TransferKVChunk:
is_last_chunk: bool
prefill_aux_index: Optional[int]
state_indices: Optional[List]
chunk_id: Optional[int] = None
num_kv_tokens: Optional[int] = None
trace_ctx: Union[TraceReqContext, TraceNullContext] = dataclasses.field(
default_factory=TraceNullContext
+111 -333
View File
@@ -13,7 +13,6 @@ from typing import Dict, List, Optional, Set, Tuple, Union
import numpy as np
import numpy.typing as npt
import torch
import zmq
from prometheus_client import Counter
from sglang.srt.disaggregation.base.conn import KVArgs, KVPoll, StateType
@@ -25,7 +24,6 @@ from sglang.srt.disaggregation.common.conn import (
CommonKVManager,
CommonKVReceiver,
CommonKVSender,
KVTransferError,
)
from sglang.srt.disaggregation.common.staging_handler import (
STAGING_WATERMARK_WAIT_S,
@@ -251,6 +249,8 @@ def _uses_flat_entry_layout(
class MooncakeKVManager(StagingManagerMixin, CommonKVManager):
sender_trace_module = "mooncake"
AUX_DATA_HEADER = b"AUX_DATA"
# Implements teardown() below, so runtime PD role switching is supported.
supports_role_switch = True
@@ -585,24 +585,6 @@ class MooncakeKVManager(StagingManagerMixin, CommonKVManager):
return PrefillStagingStrategy(self, staging_buffer)
def _send_chunk_ready(self, req, chunk_idx, kv_chunk, prefill_unique_rank):
"""Notify decode that a staging chunk RDMA is complete (every chunk;
scatter is arrival-driven)."""
na = NetworkAddress(req.endpoint, req.dst_port)
self._send_multipart_locked(
na.to_tcp(),
[
b"CHUNK_READY",
str(req.room).encode("ascii"),
str(chunk_idx).encode("ascii"),
str(kv_chunk.index_slice.start).encode("ascii"),
str(len(kv_chunk.prefill_kv_indices)).encode("ascii"),
req.mooncake_session_id.encode("ascii"),
str(prefill_unique_rank).encode("ascii"),
],
is_ipv6=na.is_ipv6,
)
def _do_staging_transfer(
self,
staging_strategy,
@@ -612,7 +594,6 @@ class MooncakeKVManager(StagingManagerMixin, CommonKVManager):
chunked_dst_kv_indice,
executor,
queue,
prefill_unique_rank,
):
"""Execute staging transfer for one chunk. Returns (ret, deferred).
@@ -663,7 +644,14 @@ class MooncakeKVManager(StagingManagerMixin, CommonKVManager):
)
return (-1, False)
if ret == 0:
self._send_chunk_ready(req, chunk_idx, kv_chunk, prefill_unique_rank)
self.send_chunk_ready(
room=req.room,
chunk_idx=chunk_idx,
page_start=kv_chunk.index_slice.start,
num_pages=len(kv_chunk.prefill_kv_indices),
writer_id=req.mooncake_session_id,
targets=[(req.endpoint, req.dst_port)],
)
return (ret, False)
def _prefetch_staging_reqs(self, room: int):
@@ -2559,8 +2547,6 @@ class MooncakeKVManager(StagingManagerMixin, CommonKVManager):
)
polls = []
dst_ranks_infos = []
# Unique id per prefill sender so decode's response set size matches expected_response_num.
prefill_unique_rank = self._prefill_unique_rank()
# When staging transfer is not yet ready (watermark/allocation pending),
# the chunk is re-enqueued and we break out of the req loop to retry later.
staging_deferred = False
@@ -2707,7 +2693,6 @@ class MooncakeKVManager(StagingManagerMixin, CommonKVManager):
chunked_dst_kv_indice,
executor,
queue,
prefill_unique_rank,
)
if deferred:
staging_deferred = True
@@ -3038,23 +3023,7 @@ class MooncakeKVManager(StagingManagerMixin, CommonKVManager):
continue
# Staging: prefill notifies a chunk written to staging buffer
if msg[0] == b"CHUNK_READY":
room = int(msg[1].decode("ascii"))
chunk_idx = int(msg[2].decode("ascii"))
page_start = int(msg[3].decode("ascii"))
num_pages = int(msg[4].decode("ascii"))
session_id = msg[5].decode("ascii")
handler = self._staging_handler
assert handler is not None, (
"CHUNK_READY received before staging handler initialized"
)
handler.handle_chunk_arrived(
room,
chunk_idx,
page_start,
num_pages,
session_id,
)
if self.handle_chunk_ready(msg):
continue
# Staging: prefill pre-requests staging allocation before forward
@@ -3091,9 +3060,10 @@ class MooncakeKVManager(StagingManagerMixin, CommonKVManager):
aux_index: Optional[int] = None,
state_indices: Optional[List] = None,
num_kv_tokens: Optional[int] = None,
trace_ctx: Optional[Union[TraceReqContext, TraceNullContext]] = None,
*,
wait_event: Optional[object] = None,
indices_event: Optional[object] = None,
trace_ctx: Optional[Union[TraceReqContext, TraceNullContext]] = None,
):
assert self.disaggregation_mode == DisaggregationMode.PREFILL
assert not is_last_chunk or (is_last_chunk and aux_index is not None)
@@ -3215,51 +3185,103 @@ class MooncakeKVManager(StagingManagerMixin, CommonKVManager):
):
self._run_one_probe_pass()
class MooncakeFailureExceptionMixin:
"""Shared `failure_exception` for the Mooncake sender and receiver.
Both sides conclude a failed room identically: latch Failed, clear local
state, then raise with the recorded reason -- or, when no reason was
recorded locally, report it as propagated from another rank. Expects the
concrete class to provide ``conclude_state``, ``clear()``,
``bootstrap_room`` and ``kv_mgr``.
"""
def failure_exception(self):
# A room with no locally recorded reason failed on another rank.
if self.conclude_state is None:
self.conclude_state = KVPoll.Failed
self.clear()
with self.kv_mgr.failure_lock:
failure_reason = self.kv_mgr.failure_records.pop(self.bootstrap_room, None)
is_propagated = failure_reason is None
if is_propagated:
failure_reason = "Failed due to an unknown reason from another rank"
raise KVTransferError(
self.bootstrap_room, failure_reason, is_from_another_rank=is_propagated
def build_receiver_registration_message(self) -> List[bytes]:
session_id = self.get_session_id()
packed_kv_data_ptrs = b"".join(
struct.pack("Q", ptr) for ptr in self.kv_args.kv_data_ptrs
)
packed_aux_data_ptrs = b"".join(
struct.pack("Q", ptr) for ptr in self.kv_args.aux_data_ptrs
)
packed_state_data_ptrs = pack_int_lists(self.kv_args.state_data_ptrs, "Q")
packed_state_item_lens = pack_int_lists(self.kv_args.state_item_lens, "I")
packed_state_dim_per_tensor = pack_int_lists(
getattr(self.kv_args, "state_dim_per_tensor", []) or [], "I"
)
packed_state_layer_ids = pack_int_lists(self.kv_args.state_layer_ids, "I")
packed_kv_layer_ids = b"".join(
struct.pack("I", layer_id) for layer_id in self.kv_args.kv_layer_ids
)
# Note(shangming): No need to add pp rank here since decode pp size should be equal to prefill pp size or 1
tp_rank = self.kv_args.engine_rank
# Some pools have no full-token contiguous KV (kv_item_lens empty)
# and ship per-pool instead, so report 0.
kv_item_len = self.kv_args.kv_item_lens[0] if self.kv_args.kv_item_lens else 0
dst_tp_rank = str(tp_rank).encode("ascii")
dst_attn_tp_size = str(self.attn_tp_size).encode("ascii")
dst_kv_item_len = str(kv_item_len).encode("ascii")
dst_dcp_size = str(self.dcp_size).encode("ascii")
dst_dcp_rank = str(self.dcp_rank).encode("ascii")
if self.enable_staging and self._staging_ctx.allocator is not None:
_alloc = self._staging_ctx.allocator
packed_staging_base_ptr = struct.pack("Q", _alloc.get_base_ptr())
staging_total_size_str = str(_alloc.get_total_size()).encode("ascii")
else:
packed_staging_base_ptr = b""
staging_total_size_str = b""
staging_slots = getattr(self, "kv_buffer_tensors", None) or {}
packed_staging_slot_layer_ids = b"".join(
struct.pack("Q", layer_id)
for layer_id in (staging_slots.get("slot_layer_ids") or [])
)
return [
"None".encode("ascii"),
self.local_ip.encode("ascii"),
str(self.rank_port).encode("ascii"),
session_id.encode("ascii"),
packed_kv_data_ptrs,
packed_aux_data_ptrs,
packed_state_data_ptrs,
dst_tp_rank,
dst_attn_tp_size,
dst_kv_item_len,
packed_state_item_lens,
packed_state_dim_per_tensor,
packed_kv_layer_ids,
packed_state_layer_ids,
packed_staging_base_ptr,
staging_total_size_str,
dst_dcp_size,
dst_dcp_rank,
packed_staging_slot_layer_ids,
struct.pack(
f"{len(self.kv_args.kv_item_lens)}Q",
*self.kv_args.kv_item_lens,
),
]
class MooncakeKVSender(MooncakeFailureExceptionMixin, CommonKVSender):
def __init__(
def build_receiver_metadata_message(
self,
mgr: MooncakeKVManager,
bootstrap_addr: str,
*,
bootstrap_room: int,
req_has_disagg_prefill_dp_rank: bool = False,
):
super().__init__(
mgr,
bootstrap_addr,
bootstrap_room,
req_has_disagg_prefill_dp_rank,
)
self.init_time = time.time()
self._init_trace_ctx()
required_dst_info_num: int,
kv_indices: npt.NDArray[np.int32],
aux_index: Optional[int],
state_indices: Optional[List],
decode_prefix_len: Optional[int],
is_dummy: bool,
device_kv_indices: Optional[npt.NDArray[np.int32]] = None,
) -> List[bytes]:
return [
str(bootstrap_room).encode("ascii"),
self.local_ip.encode("ascii"),
str(self.rank_port).encode("ascii"),
self.get_session_id().encode("ascii"),
kv_indices.tobytes() if not is_dummy else b"",
str(aux_index).encode("ascii") if not is_dummy else b"",
pack_int_lists(state_indices, "i")
if not is_dummy and state_indices
else b"",
str(required_dst_info_num).encode("ascii"),
str(decode_prefix_len or 0).encode("ascii"),
np.asarray(device_kv_indices, dtype=np.int32).tobytes()
if not is_dummy and device_kv_indices is not None
else b"",
]
class MooncakeKVSender(CommonKVSender):
supports_host_tensor_indices = True
@mooncake_trace_func(MooncakeRequestStage.MOONCAKE_SEND)
@@ -3270,260 +3292,16 @@ class MooncakeKVSender(MooncakeFailureExceptionMixin, CommonKVSender):
num_kv_tokens: Optional[int] = None,
indices_event: Optional[object] = None,
):
kv_indices, index_slice, is_last_chunk, should_skip = (
self._prepare_send_indices(kv_indices, state_indices)
super().send(
kv_indices=kv_indices,
state_indices=state_indices,
num_kv_tokens=num_kv_tokens,
indices_event=indices_event,
)
if should_skip:
return
wait_event = getattr(self, "_early_send_wait_event", None)
self._early_send_wait_event = None
if not is_last_chunk:
self.kv_mgr.add_transfer_request(
self.bootstrap_room,
kv_indices,
index_slice,
False,
num_kv_tokens=num_kv_tokens,
trace_ctx=self.trace_ctx.copy_for_thread(),
wait_event=wait_event,
indices_event=indices_event,
)
else:
self.kv_mgr.add_transfer_request(
self.bootstrap_room,
kv_indices,
index_slice,
True,
aux_index=self.aux_index,
state_indices=state_indices,
num_kv_tokens=num_kv_tokens,
trace_ctx=self.trace_ctx.copy_for_thread(),
wait_event=wait_event,
indices_event=indices_event,
)
self._record_transfer_indices(kv_indices, state_indices)
def poll(self) -> KVPoll:
return self._poll_with_status(self.kv_mgr.check_status)
def poll_pp_consensus(self) -> KVPoll:
return self._poll_with_status(self.kv_mgr.check_status_pp_consensus)
def _poll_with_status(self, check_status) -> KVPoll:
if self.conclude_state is not None:
return self.conclude_state
status = check_status(self.bootstrap_room)
# Hold Success until all staging chunks transferred. With deferred
# allocation also wait for worker cleanup (the zero entry is only
# removed after cleanup), before permitting the room to be reused.
if status == KVPoll.Success and (
self.kv_mgr._staging_outstanding.get(self.bootstrap_room, 0) > 0
or (
self.kv_mgr.defer_decode_allocation
and self.bootstrap_room in self.kv_mgr._staging_outstanding
)
):
return KVPoll.Transferring
if status in (KVPoll.Success, KVPoll.Failed):
self.conclude_state = status
self.trace_ctx.trace_req_finish()
elif status == KVPoll.Bootstrapping:
timeout_result = self._check_bootstrap_timeout()
if timeout_result is not None:
return timeout_result
return status
def _init_trace_ctx(self):
if self.kv_mgr.enable_trace:
self.trace_ctx = TraceReqContext(
rid=str(hex(self.bootstrap_room)),
bootstrap_room=self.bootstrap_room,
role="Sender",
module_name="mooncake",
)
if not self.trace_ctx.tracing_enable:
self.trace_ctx = TraceNullContext()
else:
self.trace_ctx = TraceNullContext()
self.trace_ctx.trace_req_start()
def abort(self):
super().abort()
self.trace_ctx.abort(abort_info={"reason": "Aborted"})
self.trace_ctx.trace_req_finish()
class MooncakeKVReceiver(MooncakeFailureExceptionMixin, CommonKVReceiver):
def __init__(
self,
mgr: MooncakeKVManager,
bootstrap_addr: str,
bootstrap_room: Optional[int] = None,
):
self.session_id = mgr.get_session_id()
self.init_time = None
super().__init__(mgr, bootstrap_addr, bootstrap_room)
def _register_kv_args(self) -> bool:
for bootstrap_info in self.bootstrap_infos:
packed_kv_data_ptrs = b"".join(
struct.pack("Q", ptr) for ptr in self.kv_mgr.kv_args.kv_data_ptrs
)
packed_aux_data_ptrs = b"".join(
struct.pack("Q", ptr) for ptr in self.kv_mgr.kv_args.aux_data_ptrs
)
packed_state_data_ptrs = pack_int_lists(
self.kv_mgr.kv_args.state_data_ptrs, "Q"
)
packed_state_item_lens = pack_int_lists(
self.kv_mgr.kv_args.state_item_lens, "I"
)
packed_state_dim_per_tensor = pack_int_lists(
getattr(self.kv_mgr.kv_args, "state_dim_per_tensor", []) or [], "I"
)
packed_state_layer_ids = pack_int_lists(
self.kv_mgr.kv_args.state_layer_ids, "I"
)
packed_kv_layer_ids = b"".join(
struct.pack("I", layer_id)
for layer_id in self.kv_mgr.kv_args.kv_layer_ids
)
# Note(shangming): No need to add pp rank here since decode pp size should be equal to prefill pp size or 1
tp_rank = self.kv_mgr.kv_args.engine_rank
# Some pools have no full-token contiguous KV (kv_item_lens empty)
# and ship per-pool instead, so report 0.
kv_item_len = (
self.kv_mgr.kv_args.kv_item_lens[0]
if self.kv_mgr.kv_args.kv_item_lens
else 0
)
dst_tp_rank = str(tp_rank).encode("ascii")
dst_attn_tp_size = str(self.kv_mgr.attn_tp_size).encode("ascii")
dst_kv_item_len = str(kv_item_len).encode("ascii")
dst_dcp_size = str(self.kv_mgr.dcp_size).encode("ascii")
dst_dcp_rank = str(self.kv_mgr.dcp_rank).encode("ascii")
if (
self.kv_mgr.enable_staging
and self.kv_mgr._staging_ctx.allocator is not None
):
_alloc = self.kv_mgr._staging_ctx.allocator
packed_staging_base_ptr = struct.pack("Q", _alloc.get_base_ptr())
staging_total_size_str = str(_alloc.get_total_size()).encode("ascii")
else:
packed_staging_base_ptr = b""
staging_total_size_str = b""
staging_slots = getattr(self.kv_mgr, "kv_buffer_tensors", None) or {}
packed_staging_slot_layer_ids = b"".join(
struct.pack("Q", layer_id)
for layer_id in (staging_slots.get("slot_layer_ids") or [])
)
try:
sock, lock = self._connect_to_bootstrap_server(bootstrap_info)
with lock:
sock.send_multipart(
[
"None".encode("ascii"),
self.kv_mgr.local_ip.encode("ascii"),
str(self.kv_mgr.rank_port).encode("ascii"),
self.session_id.encode("ascii"),
packed_kv_data_ptrs,
packed_aux_data_ptrs,
packed_state_data_ptrs,
dst_tp_rank,
dst_attn_tp_size,
dst_kv_item_len,
packed_state_item_lens,
packed_state_dim_per_tensor,
packed_kv_layer_ids,
packed_state_layer_ids,
packed_staging_base_ptr,
staging_total_size_str,
dst_dcp_size,
dst_dcp_rank,
packed_staging_slot_layer_ids,
struct.pack(
f"{len(self.kv_mgr.kv_args.kv_item_lens)}Q",
*self.kv_mgr.kv_args.kv_item_lens,
),
]
)
except zmq.ZMQError:
self.kv_mgr.record_failure(
self.bootstrap_room,
f"_register_kv_args to prefill {bootstrap_info.get('rank_ip')}:{bootstrap_info.get('rank_port')} failed",
)
self.conclude_state = KVPoll.Failed
self.kv_mgr.update_status(self.bootstrap_room, KVPoll.Failed)
return False
return True
def send_metadata(
self,
kv_indices: npt.NDArray[np.int32],
aux_index: Optional[int] = None,
state_indices: Optional[List] = None,
decode_prefix_len: Optional[int] = None,
device_kv_indices: Optional[npt.NDArray[np.int32]] = None,
):
if self.bootstrap_infos is None:
self.kv_mgr.record_failure(
self.bootstrap_room,
f"Could not fetch prefill parallel info from bootstrap_addr: {self.bootstrap_addr}",
)
self.kv_mgr.update_status(self.bootstrap_room, KVPoll.Failed)
return
self.chunk_staging_infos = []
if (
self.kv_mgr.enable_staging
and self.kv_mgr._staging_ctx.allocator is not None
):
self.kv_mgr.register_staging_room_bootstrap(
self.bootstrap_room, self.bootstrap_infos, self
)
for bootstrap_info in self.bootstrap_infos:
is_dummy = bootstrap_info["is_dummy"]
try:
sock, lock = self._connect_to_bootstrap_server(bootstrap_info)
with lock:
sock.send_multipart(
[
str(self.bootstrap_room).encode("ascii"),
self.kv_mgr.local_ip.encode("ascii"),
str(self.kv_mgr.rank_port).encode("ascii"),
self.session_id.encode("ascii"),
kv_indices.tobytes() if not is_dummy else b"",
str(aux_index).encode("ascii") if not is_dummy else b"",
(
pack_int_lists(state_indices, "i")
if not is_dummy and state_indices
else b""
),
str(self.required_dst_info_num).encode("ascii"),
str(decode_prefix_len or 0).encode("ascii"),
(
np.asarray(device_kv_indices, dtype=np.int32).tobytes()
if not is_dummy and device_kv_indices is not None
else b""
),
]
)
except zmq.ZMQError:
self.invalidate_cached_bootstrap_infos()
self.kv_mgr.record_failure(
self.bootstrap_room,
f"send_metadata to prefill {bootstrap_info.get('rank_ip')}:{bootstrap_info.get('rank_port')} failed",
)
self.conclude_state = KVPoll.Failed
self.kv_mgr.update_status(self.bootstrap_room, KVPoll.Failed)
return
self.init_time = time.time()
class MooncakeKVReceiver(CommonKVReceiver):
pass
class MooncakeKVBootstrapServer(CommonKVBootstrapServer):
+98 -280
View File
@@ -36,7 +36,6 @@ from sglang.srt.disaggregation.common.conn import (
CommonKVManager,
CommonKVReceiver,
CommonKVSender,
KVTransferError,
)
from sglang.srt.disaggregation.common.utils import (
AuxDataCodec,
@@ -785,6 +784,13 @@ class MoriKVManager(CommonKVManager):
return f"KV transfer failed: {status.Message()}"
return "KV transfer failed due to unknown reason"
def prepare_send_state_indices(
self, state_indices: Optional[List]
) -> Optional[List]:
if self._should_skip_cp_replicated_state_transfer():
return None
return _normalize_state_indices_per_component(state_indices)
def add_transfer_request(
self,
bootstrap_room: int,
@@ -794,7 +800,10 @@ class MoriKVManager(CommonKVManager):
aux_index: Optional[int] = None,
state_indices: Optional[List] = None,
num_kv_tokens: Optional[int] = None,
*,
wait_event: Optional[object] = None,
indices_event: Optional[object] = None,
trace_ctx=None,
) -> None:
assert self.disaggregation_mode == DisaggregationMode.PREFILL
assert not is_last_chunk or (is_last_chunk and aux_index is not None)
@@ -2056,95 +2065,95 @@ class MoriKVManager(CommonKVManager):
return result_statuses
def build_receiver_registration_message(self) -> List[bytes]:
engine_desc_blob = self.engine_desc.pack()
packed_kv_descs = _pack_mem_desc_list(self.kv_mem_descs)
packed_aux_descs = _pack_mem_desc_list(self.aux_mem_descs)
packed_state_descs = _pack_mem_desc_lists(self.state_mem_descs)
gpu_id = str(self.kv_args.gpu_id).encode("ascii")
decode_tp_size = str(self.attn_tp_size).encode("ascii")
decode_tp_rank = str(self.kv_args.engine_rank).encode("ascii")
kv_item_len = str(self.kv_args.kv_item_lens[0]).encode("ascii")
packed_state_item_lens = pack_int_lists(self.kv_args.state_item_lens, "I")
packed_state_dim_per_tensor = pack_int_lists(
self.kv_args.state_dim_per_tensor, "I"
)
packed_state_layer_ids = pack_int_lists(self.kv_args.state_layer_ids, "I")
packed_kv_layer_ids = struct.pack(
f"<{len(self.kv_args.kv_layer_ids)}I",
*self.kv_args.kv_layer_ids,
)
dcp_size = str(self.dcp_size).encode("ascii")
dcp_rank = str(self.dcp_rank).encode("ascii")
return [
MORI_GUARD,
"None".encode("ascii"),
self.local_ip.encode("ascii"),
str(self.rank_port).encode("ascii"),
engine_desc_blob,
packed_kv_descs,
packed_aux_descs,
packed_state_descs,
gpu_id,
decode_tp_size,
decode_tp_rank,
kv_item_len,
packed_state_item_lens,
packed_state_dim_per_tensor,
dcp_size,
dcp_rank,
packed_state_layer_ids,
packed_kv_layer_ids,
]
def build_receiver_metadata_message(
self,
*,
bootstrap_room: int,
required_dst_info_num: int,
kv_indices: npt.NDArray[np.int32],
aux_index: Optional[int],
state_indices: Optional[List],
decode_prefix_len: Optional[int],
is_dummy: bool,
device_kv_indices: Optional[npt.NDArray[np.int32]] = None,
) -> List[bytes]:
if is_dummy:
kv_indices_bytes = aux_bytes = state_bytes = b""
else:
kv_indices_bytes = (
np.asarray(kv_indices, dtype=np.int32).tobytes()
if kv_indices.size
else b""
)
aux_bytes = str(aux_index).encode("ascii") if aux_index is not None else b""
normalized_state = _normalize_state_indices_per_component(state_indices)
state_bytes = (
_pack_state_indices(normalized_state)
if normalized_state is not None
else b""
)
decode_prefix_bytes = (
str(int(decode_prefix_len)).encode("ascii")
if decode_prefix_len is not None and decode_prefix_len > 0
else b""
)
return [
MORI_GUARD,
str(bootstrap_room).encode("ascii"),
self.local_ip.encode("ascii"),
str(self.rank_port).encode("ascii"),
self.engine_desc.key.encode("ascii"),
kv_indices_bytes,
aux_bytes,
state_bytes,
str(required_dst_info_num).encode("ascii"),
decode_prefix_bytes,
]
class MoriKVSender(CommonKVSender):
def __init__(
self,
mgr: MoriKVManager,
bootstrap_addr: str,
bootstrap_room: int,
req_has_disagg_prefill_dp_rank: bool = False,
):
super().__init__(
mgr,
bootstrap_addr,
bootstrap_room,
req_has_disagg_prefill_dp_rank,
)
self.conclude_state: Optional[KVPoll] = None
self.init_time = time.time()
def send(
self,
kv_indices: npt.NDArray[np.int32],
state_indices: Optional[List] = None,
num_kv_tokens: Optional[int] = None,
):
kv_indices, index_slice, is_last_chunk, should_skip = (
self._prepare_send_indices(kv_indices, state_indices)
)
if should_skip:
return
transfer_state_indices = (
None
if self.kv_mgr._should_skip_cp_replicated_state_transfer()
else state_indices
)
normalized_state = (
_normalize_state_indices_per_component(transfer_state_indices)
if is_last_chunk
else None
)
self._record_transfer_indices(kv_indices, transfer_state_indices)
wait_event = getattr(self, "_early_send_wait_event", None)
self._early_send_wait_event = None
if not is_last_chunk:
self.kv_mgr.add_transfer_request(
self.bootstrap_room,
kv_indices,
index_slice,
False,
num_kv_tokens=num_kv_tokens,
wait_event=wait_event,
)
else:
self.kv_mgr.add_transfer_request(
self.bootstrap_room,
kv_indices,
index_slice,
True,
aux_index=self.aux_index,
state_indices=normalized_state,
num_kv_tokens=num_kv_tokens,
wait_event=wait_event,
)
def poll(self) -> KVPoll:
return self._poll_with_status(self.kv_mgr.check_status)
def poll_pp_consensus(self) -> KVPoll:
return self._poll_with_status(self.kv_mgr.check_status_pp_consensus)
def _poll_with_status(self, check_status) -> KVPoll:
if self.conclude_state is not None:
return self.conclude_state
if self.bootstrap_room not in self.kv_mgr.request_status:
self.conclude_state = KVPoll.Failed
return self.conclude_state
status = check_status(self.bootstrap_room)
if status == KVPoll.Bootstrapping:
timeout_result = self._check_bootstrap_timeout()
if timeout_result is not None:
self.conclude_state = timeout_result
return timeout_result
if status in (KVPoll.Success, KVPoll.Failed):
self.conclude_state = status
return status
def clear(self) -> None:
if self.kv_mgr.enable_deferred_decode_kv_release:
with self.kv_mgr._abort_ack_lock:
@@ -2152,200 +2161,9 @@ class MoriKVSender(CommonKVSender):
else:
super().clear()
def failure_exception(self):
if self.conclude_state is None:
self.conclude_state = KVPoll.Failed
self.clear()
with self.kv_mgr.failure_lock:
failure_reason = self.kv_mgr.failure_records.pop(self.bootstrap_room, None)
is_propagated = failure_reason is None
if is_propagated:
failure_reason = "KV transfer failed"
raise KVTransferError(
self.bootstrap_room, failure_reason, is_from_another_rank=is_propagated
)
class MoriKVReceiver(CommonKVReceiver):
def __init__(
self,
mgr: MoriKVManager,
bootstrap_addr: str,
bootstrap_room: Optional[int] = None,
):
super().__init__(mgr, bootstrap_addr, bootstrap_room)
self.init_time: Optional[float] = None
def init(
self,
prefill_dp_rank: int,
):
super().init(prefill_dp_rank)
def _register_kv_args(self) -> bool:
if self.bootstrap_infos is None:
return False
engine_desc_blob = self.kv_mgr.engine_desc.pack()
packed_kv_descs = _pack_mem_desc_list(self.kv_mgr.kv_mem_descs)
packed_aux_descs = _pack_mem_desc_list(self.kv_mgr.aux_mem_descs)
packed_state_descs = _pack_mem_desc_lists(self.kv_mgr.state_mem_descs)
gpu_id = str(self.kv_mgr.kv_args.gpu_id).encode("ascii")
decode_tp_size = str(self.kv_mgr.attn_tp_size).encode("ascii")
decode_tp_rank = str(self.kv_mgr.kv_args.engine_rank).encode("ascii")
kv_item_len = str(self.kv_mgr.kv_args.kv_item_lens[0]).encode("ascii")
packed_state_item_lens = pack_int_lists(
self.kv_mgr.kv_args.state_item_lens, "I"
)
packed_state_dim_per_tensor = pack_int_lists(
self.kv_mgr.kv_args.state_dim_per_tensor, "I"
)
packed_state_layer_ids = pack_int_lists(
self.kv_mgr.kv_args.state_layer_ids, "I"
)
packed_kv_layer_ids = struct.pack(
f"<{len(self.kv_mgr.kv_args.kv_layer_ids)}I",
*self.kv_mgr.kv_args.kv_layer_ids,
)
dcp_size = str(self.kv_mgr.dcp_size).encode("ascii")
dcp_rank = str(self.kv_mgr.dcp_rank).encode("ascii")
for bootstrap_info in self.bootstrap_infos:
try:
sock, lock = self._connect_to_bootstrap_server(bootstrap_info)
with lock:
sock.send_multipart(
[
MORI_GUARD,
"None".encode("ascii"),
self.kv_mgr.local_ip.encode("ascii"),
str(self.kv_mgr.rank_port).encode("ascii"),
engine_desc_blob,
packed_kv_descs,
packed_aux_descs,
packed_state_descs,
gpu_id,
decode_tp_size,
decode_tp_rank,
kv_item_len,
packed_state_item_lens,
packed_state_dim_per_tensor,
dcp_size,
dcp_rank,
packed_state_layer_ids,
packed_kv_layer_ids,
]
)
except zmq.ZMQError:
self.kv_mgr.record_failure(
self.bootstrap_room,
f"_register_kv_args to prefill {bootstrap_info.get('rank_ip')}:{bootstrap_info.get('rank_port')} failed",
)
self.conclude_state = KVPoll.Failed
self.kv_mgr.update_status(self.bootstrap_room, KVPoll.Failed)
return False
return True
def send_metadata(
self,
kv_indices: npt.NDArray[np.int32],
aux_index: Optional[int] = None,
state_indices: Optional[List] = None,
decode_prefix_len: Optional[int] = None,
):
if self.bootstrap_infos is None or self.bootstrap_room is None:
return
kv_indices_bytes = (
np.asarray(kv_indices, dtype=np.int32).tobytes() if kv_indices.size else b""
)
aux_bytes = str(aux_index).encode("ascii") if aux_index is not None else b""
normalized_state = _normalize_state_indices_per_component(state_indices)
decode_prefix_bytes = (
str(int(decode_prefix_len)).encode("ascii")
if decode_prefix_len is not None and decode_prefix_len > 0
else b""
)
for bootstrap_info in self.bootstrap_infos:
is_dummy = bootstrap_info.get("is_dummy", False)
if not is_dummy and normalized_state is not None:
state_bytes = _pack_state_indices(normalized_state)
else:
state_bytes = b""
try:
sock, lock = self._connect_to_bootstrap_server(bootstrap_info)
with lock:
sock.send_multipart(
[
MORI_GUARD,
str(self.bootstrap_room).encode("ascii"),
self.kv_mgr.local_ip.encode("ascii"),
str(self.kv_mgr.rank_port).encode("ascii"),
self.kv_mgr.engine_desc.key.encode("ascii"),
kv_indices_bytes if not is_dummy else b"",
aux_bytes if not is_dummy else b"",
state_bytes,
str(self.required_dst_info_num).encode("ascii"),
decode_prefix_bytes,
]
)
except zmq.ZMQError:
self.invalidate_cached_bootstrap_infos()
self.kv_mgr.record_failure(
self.bootstrap_room,
f"send_metadata to prefill {bootstrap_info.get('rank_ip')}:{bootstrap_info.get('rank_port')} failed",
)
self.conclude_state = KVPoll.Failed
self.kv_mgr.update_status(self.bootstrap_room, KVPoll.Failed)
return
self.init_time = time.time()
def poll(self) -> KVPoll:
if self.conclude_state is not None:
return self.conclude_state
status = self.kv_mgr.check_status(self.bootstrap_room)
if status in (KVPoll.Success, KVPoll.Failed):
self.conclude_state = status
return status
if status == KVPoll.WaitingForInput:
timeout_result = self._check_waiting_timeout()
if timeout_result is not None:
return timeout_result
return status
def clear(self) -> None:
if self.bootstrap_room is None:
return
super().clear()
def failure_exception(self):
if self.conclude_state is None:
self.conclude_state = KVPoll.Failed
self.clear()
with self.kv_mgr.failure_lock:
failure_reason = self.kv_mgr.failure_records.pop(self.bootstrap_room, None)
is_propagated = failure_reason is None
if is_propagated:
failure_reason = "KV transfer failed"
raise KVTransferError(
self.bootstrap_room, failure_reason, is_from_another_rank=is_propagated
)
def abort(self):
if self.bootstrap_room is None:
return
bootstrap_room = self.bootstrap_room
super().abort()
self.clear()
with self.kv_mgr.failure_lock:
self.kv_mgr.failure_records.pop(bootstrap_room, None)
pass
class MoriKVBootstrapServer(CommonKVBootstrapServer):
File diff suppressed because it is too large Load Diff