[Refactor] pd: Consolidate sender and receiver implementations (#41140)
Co-authored-by: Shangming Cai <csmthu@gmail.com>
This commit is contained in:
co-authored by
Shangming Cai
parent
88ac7c02a6
commit
de7055f596
@@ -8,7 +8,16 @@ import threading
|
||||
import time
|
||||
from collections import defaultdict
|
||||
from contextlib import nullcontext
|
||||
from typing import Dict, List, NamedTuple, Optional, Set, Tuple, Union
|
||||
from typing import (
|
||||
Dict,
|
||||
List,
|
||||
Literal,
|
||||
NamedTuple,
|
||||
Optional,
|
||||
Set,
|
||||
Tuple,
|
||||
Union,
|
||||
)
|
||||
|
||||
import msgspec
|
||||
import numpy as np
|
||||
@@ -42,9 +51,11 @@ from sglang.srt.disaggregation.utils import (
|
||||
)
|
||||
from sglang.srt.environ import envs
|
||||
from sglang.srt.mem_cache.page_interleave import get_kv_shard_group
|
||||
from sglang.srt.observability.trace import TraceNullContext, TraceReqContext
|
||||
from sglang.srt.runtime_context import (
|
||||
get_device,
|
||||
get_disagg,
|
||||
get_observability,
|
||||
get_parallel,
|
||||
get_schedule,
|
||||
get_serving,
|
||||
@@ -275,6 +286,7 @@ class DeferredAbortAckState(msgspec.Struct):
|
||||
class CommonKVManager(BaseKVManager):
|
||||
defer_decode_allocation: bool = False
|
||||
deferred_bootstrap: Optional[DeferredBootstrap] = None
|
||||
_staging_ctx = None
|
||||
|
||||
# Wire layout of the prefill->decode terminal status message. The legacy
|
||||
# layout (mooncake, and ascend which inherits it) is three untagged frames
|
||||
@@ -283,6 +295,10 @@ class CommonKVManager(BaseKVManager):
|
||||
# ``[tag, room, status, prefill_rank, reason]``.
|
||||
kv_status_msg_tag: Optional[bytes] = None
|
||||
kv_status_msg_carries_reason: bool = False
|
||||
sender_trace_module: Optional[str] = None
|
||||
# Sender reports transfer_latency_s (first enqueue -> Success). Otherwise
|
||||
# the metric keeps its last-chunk fallback, so dashboards do not move.
|
||||
reports_transfer_latency: bool = False
|
||||
|
||||
kv_layout: Optional[str] = None
|
||||
dsv41_spec_layout: Optional[dict] = None
|
||||
@@ -453,7 +469,7 @@ class CommonKVManager(BaseKVManager):
|
||||
self.session_pool: Dict = defaultdict(requests.Session)
|
||||
self.session_pool_lock = threading.Lock()
|
||||
self.addr_to_rooms_tracker: Dict[str, Set[int]] = defaultdict(set)
|
||||
self.prefill_response_tracker: Dict[int, Set[int]] = defaultdict(set)
|
||||
self.prefill_response_tracker: Dict[int, Set[int]] = {}
|
||||
# Deferred KV release: room -> current generation and ranks that
|
||||
# ACKed. The generation rejects an old request's late ACK after room
|
||||
# reuse.
|
||||
@@ -489,6 +505,11 @@ class CommonKVManager(BaseKVManager):
|
||||
f"Unsupported DisaggregationMode: {self.disaggregation_mode}"
|
||||
)
|
||||
|
||||
def prepare_send_state_indices(
|
||||
self, state_indices: Optional[List]
|
||||
) -> Optional[List]:
|
||||
return state_indices
|
||||
|
||||
def _should_skip_cp_replicated_state_transfer(self) -> bool:
|
||||
"""Whether this prefill rank should omit CP-replicated state.
|
||||
|
||||
@@ -590,6 +611,23 @@ class CommonKVManager(BaseKVManager):
|
||||
return KVPoll.Bootstrapping
|
||||
return min(statuses)
|
||||
|
||||
def build_receiver_registration_message(self) -> List[bytes]:
|
||||
raise NotImplementedError
|
||||
|
||||
def build_receiver_metadata_message(
|
||||
self,
|
||||
*,
|
||||
bootstrap_room: int,
|
||||
required_dst_info_num: int,
|
||||
kv_indices: npt.NDArray[np.int32],
|
||||
aux_index: Optional[int],
|
||||
state_indices: Optional[List],
|
||||
decode_prefix_len: Optional[int],
|
||||
is_dummy: bool,
|
||||
device_kv_indices: Optional[npt.NDArray[np.int32]] = None,
|
||||
) -> List[bytes]:
|
||||
raise NotImplementedError
|
||||
|
||||
def update_status(self, bootstrap_room: int, status: KVPoll):
|
||||
current = self.request_status.get(bootstrap_room)
|
||||
if current is None:
|
||||
@@ -747,6 +785,51 @@ class CommonKVManager(BaseKVManager):
|
||||
f"{na.to_host_port_str()}: {e}"
|
||||
)
|
||||
|
||||
def send_chunk_ready(
|
||||
self,
|
||||
*,
|
||||
room: int,
|
||||
chunk_idx: int,
|
||||
page_start: int,
|
||||
num_pages: int,
|
||||
writer_id: str,
|
||||
targets: List[Tuple[str, int]],
|
||||
) -> None:
|
||||
"""Notify decode that a staging chunk RDMA is complete (every chunk;
|
||||
scatter is arrival-driven)."""
|
||||
parts = [
|
||||
b"CHUNK_READY",
|
||||
str(room).encode("ascii"),
|
||||
str(chunk_idx).encode("ascii"),
|
||||
str(page_start).encode("ascii"),
|
||||
str(num_pages).encode("ascii"),
|
||||
writer_id.encode("ascii"),
|
||||
str(self._prefill_unique_rank()).encode("ascii"),
|
||||
]
|
||||
for endpoint, port in targets:
|
||||
na = NetworkAddress(endpoint, port)
|
||||
self._send_multipart_locked(na.to_tcp(), parts, is_ipv6=na.is_ipv6)
|
||||
|
||||
def handle_chunk_ready(self, msg: List[bytes]) -> bool:
|
||||
if not msg or msg[0] != b"CHUNK_READY":
|
||||
return False
|
||||
try:
|
||||
room, chunk_idx, page_start, num_pages = (
|
||||
int(part.decode("ascii")) for part in msg[1:5]
|
||||
)
|
||||
writer_id = msg[5].decode("ascii")
|
||||
except (IndexError, ValueError, UnicodeDecodeError):
|
||||
logger.warning("Dropping malformed CHUNK_READY message")
|
||||
return True
|
||||
if room not in self.request_status:
|
||||
return True
|
||||
handler = self._staging_handler
|
||||
if handler is None:
|
||||
logger.warning("CHUNK_READY for room %s without a staging handler", room)
|
||||
return True
|
||||
handler.handle_chunk_arrived(room, chunk_idx, page_start, num_pages, writer_id)
|
||||
return True
|
||||
|
||||
def conclude_transfer(
|
||||
self,
|
||||
*,
|
||||
@@ -834,10 +917,13 @@ class CommonKVManager(BaseKVManager):
|
||||
self.update_status(bootstrap_room, KVPoll.WaitingForInput)
|
||||
return
|
||||
if status == KVPoll.Success:
|
||||
self.prefill_response_tracker[bootstrap_room].add(prefill_rank)
|
||||
completed_ranks = self.prefill_response_tracker.get(bootstrap_room)
|
||||
if completed_ranks is None:
|
||||
return
|
||||
expected_response_num = self.required_prefill_response_num_table.get(
|
||||
bootstrap_room
|
||||
)
|
||||
completed_ranks.add(prefill_rank)
|
||||
if expected_response_num is None:
|
||||
logger.warning(
|
||||
"No expected prefill response count for room %s, prefill rank %s",
|
||||
@@ -845,14 +931,10 @@ class CommonKVManager(BaseKVManager):
|
||||
prefill_rank,
|
||||
)
|
||||
return
|
||||
if (
|
||||
len(self.prefill_response_tracker[bootstrap_room])
|
||||
< expected_response_num
|
||||
):
|
||||
if len(completed_ranks) < expected_response_num:
|
||||
return
|
||||
# Tell the staging handler no more chunks are coming, before any
|
||||
# poller can see Success. Only mooncake gets here: NIXL arms the
|
||||
# handler from its own notifications, mori has no staging.
|
||||
# poller can see Success.
|
||||
if self.enable_staging and self._staging_handler is not None:
|
||||
handler = self._staging_handler
|
||||
if handler.is_staging_room(bootstrap_room):
|
||||
@@ -2007,6 +2089,19 @@ class CommonKVSender(BaseKVSender):
|
||||
self._transfer_metric = KVTransferMetric()
|
||||
self._transfer_num_kv_indices = 0
|
||||
self._transfer_num_state_indices = 0
|
||||
self._transfer_start_time: Optional[float] = None
|
||||
self._early_send_wait_event: Optional[object] = None
|
||||
self.trace_ctx = TraceNullContext()
|
||||
if self.kv_mgr.sender_trace_module and get_observability().enable_trace:
|
||||
self.trace_ctx = TraceReqContext(
|
||||
rid=str(hex(self.bootstrap_room)),
|
||||
bootstrap_room=self.bootstrap_room,
|
||||
role="Sender",
|
||||
module_name=self.kv_mgr.sender_trace_module,
|
||||
)
|
||||
if not self.trace_ctx.tracing_enable:
|
||||
self.trace_ctx = TraceNullContext()
|
||||
self.trace_ctx.trace_req_start()
|
||||
# inner state
|
||||
self.curr_idx = 0
|
||||
self.init_time: Optional[float] = None
|
||||
@@ -2051,6 +2146,8 @@ class CommonKVSender(BaseKVSender):
|
||||
)
|
||||
self.kv_mgr.update_status(self.bootstrap_room, KVPoll.Failed)
|
||||
return
|
||||
# Bootstrap timeout clock starts after the registration above.
|
||||
self.init_time = time.time()
|
||||
|
||||
def _register_prefill_dp_rank(self):
|
||||
"""Register this request's prefill dp_rank to the bootstrap server."""
|
||||
@@ -2175,8 +2272,82 @@ class CommonKVSender(BaseKVSender):
|
||||
kv_indices: npt.NDArray[np.int32],
|
||||
state_indices: Optional[List] = None,
|
||||
num_kv_tokens: Optional[int] = None,
|
||||
indices_event: Optional[object] = None,
|
||||
):
|
||||
pass
|
||||
if self.conclude_state in (KVPoll.Success, KVPoll.Failed):
|
||||
return
|
||||
|
||||
kv_indices, index_slice, is_last_chunk, should_skip = (
|
||||
self._prepare_send_indices(kv_indices, state_indices)
|
||||
)
|
||||
if should_skip:
|
||||
return
|
||||
|
||||
state_indices = self.kv_mgr.prepare_send_state_indices(state_indices)
|
||||
if self._transfer_start_time is None and (
|
||||
len(kv_indices) > 0 or state_indices is not None
|
||||
):
|
||||
self._transfer_start_time = time.perf_counter()
|
||||
|
||||
# The early-send event belongs to exactly one chunk; the Mooncake and
|
||||
# Mori workers synchronize on it before reading source KV.
|
||||
wait_event = self._early_send_wait_event
|
||||
self._early_send_wait_event = None
|
||||
self.kv_mgr.add_transfer_request(
|
||||
self.bootstrap_room,
|
||||
kv_indices,
|
||||
index_slice,
|
||||
is_last_chunk,
|
||||
aux_index=self.aux_index if is_last_chunk else None,
|
||||
state_indices=state_indices if is_last_chunk else None,
|
||||
num_kv_tokens=num_kv_tokens,
|
||||
wait_event=wait_event,
|
||||
indices_event=indices_event,
|
||||
trace_ctx=self.trace_ctx.copy_for_thread(),
|
||||
)
|
||||
self._record_transfer_indices(kv_indices, state_indices)
|
||||
|
||||
def poll(self) -> KVPoll:
|
||||
return self._poll_with_status(self.kv_mgr.check_status)
|
||||
|
||||
def poll_pp_consensus(self) -> KVPoll:
|
||||
return self._poll_with_status(self.kv_mgr.check_status_pp_consensus)
|
||||
|
||||
def _poll_with_status(self, check_status) -> KVPoll:
|
||||
if self.conclude_state is not None:
|
||||
return self.conclude_state
|
||||
|
||||
if self.bootstrap_room in self.kv_mgr.request_status:
|
||||
status = check_status(self.bootstrap_room)
|
||||
else:
|
||||
status = KVPoll.Failed
|
||||
# Hold Success until all staging chunks transferred. With deferred
|
||||
# allocation also wait for worker cleanup (the zero entry is only
|
||||
# removed after cleanup), before permitting the room to be reused.
|
||||
if status == KVPoll.Success and (
|
||||
self.kv_mgr._staging_outstanding.get(self.bootstrap_room, 0) > 0
|
||||
or (
|
||||
self.kv_mgr.defer_decode_allocation
|
||||
and self.bootstrap_room in self.kv_mgr._staging_outstanding
|
||||
)
|
||||
):
|
||||
return KVPoll.Transferring
|
||||
if status == KVPoll.Bootstrapping:
|
||||
timeout_status = self._check_bootstrap_timeout()
|
||||
if timeout_status is not None:
|
||||
status = timeout_status
|
||||
if status in (KVPoll.Success, KVPoll.Failed):
|
||||
self.conclude_state = status
|
||||
if (
|
||||
status == KVPoll.Success
|
||||
and self.kv_mgr.reports_transfer_latency
|
||||
and self._transfer_start_time is not None
|
||||
):
|
||||
self._transfer_metric.transfer_latency_s = (
|
||||
time.perf_counter() - self._transfer_start_time
|
||||
)
|
||||
self.trace_ctx.trace_req_finish()
|
||||
return status
|
||||
|
||||
def _check_bootstrap_timeout(self) -> Optional[KVPoll]:
|
||||
start = (
|
||||
@@ -2214,12 +2385,22 @@ class CommonKVSender(BaseKVSender):
|
||||
== KVPoll.Success,
|
||||
)
|
||||
self._owns_bootstrap_room = False
|
||||
if self.conclude_state is None:
|
||||
status = self.kv_mgr.request_status.get(self.bootstrap_room)
|
||||
if status in (KVPoll.Success, KVPoll.Failed):
|
||||
self.conclude_state = status
|
||||
self.kv_mgr.request_status.pop(self.bootstrap_room, None)
|
||||
self.kv_mgr.req_to_decode_prefix_len.pop(self.bootstrap_room, None)
|
||||
self.kv_mgr.transfer_infos.pop(self.bootstrap_room, None)
|
||||
# A sender can be cleared while a counted transfer is still writing:
|
||||
# keep the target for the worker, or ACK now if nothing is in flight.
|
||||
self.kv_mgr._maybe_ack_drained_abort(self.bootstrap_room)
|
||||
staging_ctx = self.kv_mgr._staging_ctx
|
||||
if staging_ctx is not None:
|
||||
staging_ctx.prefetched_rooms.discard(self.bootstrap_room)
|
||||
for key in list(staging_ctx.prefetch_requested):
|
||||
if key[0] == self.bootstrap_room:
|
||||
staging_ctx.prefetch_requested.discard(key)
|
||||
if notification is not None:
|
||||
endpoint, failed = notification
|
||||
self.kv_mgr.send_kv_status_message(
|
||||
@@ -2229,6 +2410,18 @@ class CommonKVSender(BaseKVSender):
|
||||
failure_reason="Prefill bootstrap failed" if failed else None,
|
||||
)
|
||||
|
||||
def failure_exception(self):
|
||||
self.conclude_state = KVPoll.Failed
|
||||
self.clear()
|
||||
with self.kv_mgr.failure_lock:
|
||||
failure_reason = self.kv_mgr.failure_records.pop(self.bootstrap_room, None)
|
||||
self.trace_ctx.trace_req_finish()
|
||||
raise KVTransferError(
|
||||
self.bootstrap_room,
|
||||
failure_reason or "Failed due to an unknown reason from another rank",
|
||||
is_from_another_rank=failure_reason is None,
|
||||
)
|
||||
|
||||
def abort(self):
|
||||
if self.kv_mgr.deferred_bootstrap is not None and not self._owns_bootstrap_room:
|
||||
return
|
||||
@@ -2238,6 +2431,8 @@ class CommonKVSender(BaseKVSender):
|
||||
)
|
||||
self.kv_mgr.update_status(self.bootstrap_room, KVPoll.Failed)
|
||||
self.conclude_state = KVPoll.Failed
|
||||
self.trace_ctx.abort(abort_info={"reason": "Aborted"})
|
||||
self.trace_ctx.trace_req_finish()
|
||||
|
||||
|
||||
class CommonKVReceiver(BaseKVReceiver):
|
||||
@@ -2278,6 +2473,7 @@ class CommonKVReceiver(BaseKVReceiver):
|
||||
self.conclude_state = KVPoll.Failed
|
||||
return
|
||||
self.kv_mgr.addr_to_rooms_tracker[self.bootstrap_addr].add(self.bootstrap_room)
|
||||
self.kv_mgr.prefill_response_tracker.setdefault(self.bootstrap_room, set())
|
||||
self.kv_mgr.update_status(self.bootstrap_room, KVPoll.Bootstrapping)
|
||||
|
||||
def init(self, prefill_dp_rank: int):
|
||||
@@ -2571,17 +2767,90 @@ class CommonKVReceiver(BaseKVReceiver):
|
||||
return sock, lock
|
||||
|
||||
def _register_kv_args(self) -> bool:
|
||||
if self.bootstrap_infos is None:
|
||||
return False
|
||||
if not self.bootstrap_infos:
|
||||
return True
|
||||
|
||||
message = self.kv_mgr.build_receiver_registration_message()
|
||||
for bootstrap_info in self.bootstrap_infos:
|
||||
if not self._send_bootstrap_message(
|
||||
bootstrap_info=bootstrap_info,
|
||||
message=message,
|
||||
operation="_register_kv_args",
|
||||
):
|
||||
return False
|
||||
return True
|
||||
|
||||
def _send_bootstrap_message(
|
||||
self,
|
||||
*,
|
||||
bootstrap_info: dict,
|
||||
message: List[bytes],
|
||||
operation: Literal["_register_kv_args", "send_metadata"],
|
||||
) -> bool:
|
||||
try:
|
||||
sock, lock = self._connect_to_bootstrap_server(bootstrap_info)
|
||||
with lock:
|
||||
sock.send_multipart(message)
|
||||
except zmq.ZMQError:
|
||||
self.kv_mgr.record_failure(
|
||||
self.bootstrap_room,
|
||||
f"{operation} to prefill {bootstrap_info.get('rank_ip')}:{bootstrap_info.get('rank_port')} failed",
|
||||
)
|
||||
self.conclude_state = KVPoll.Failed
|
||||
self.kv_mgr.update_status(self.bootstrap_room, KVPoll.Failed)
|
||||
return False
|
||||
return True
|
||||
|
||||
def send_metadata(
|
||||
self,
|
||||
kv_indices: npt.NDArray[np.int32],
|
||||
aux_index: Optional[int] = None,
|
||||
state_indices: Optional[List[int]] = None,
|
||||
state_indices: Optional[List] = None,
|
||||
decode_prefix_len: Optional[int] = None,
|
||||
destination: KVTransferDestination = KVTransferDestination.DEVICE,
|
||||
device_kv_indices: Optional[npt.NDArray[np.int32]] = None,
|
||||
):
|
||||
raise NotImplementedError
|
||||
if self.bootstrap_infos is None:
|
||||
self.kv_mgr.record_failure(
|
||||
self.bootstrap_room,
|
||||
f"Could not fetch prefill parallel info from bootstrap_addr: {self.bootstrap_addr}",
|
||||
)
|
||||
self.kv_mgr.update_status(self.bootstrap_room, KVPoll.Failed)
|
||||
return
|
||||
if destination != KVTransferDestination.DEVICE:
|
||||
raise NotImplementedError("Host KV destinations are not supported")
|
||||
|
||||
self.chunk_staging_infos = []
|
||||
if (
|
||||
self.kv_mgr.enable_staging
|
||||
and self.kv_mgr._staging_ctx.allocator is not None
|
||||
):
|
||||
self.kv_mgr.register_staging_room_bootstrap(
|
||||
self.bootstrap_room, self.bootstrap_infos, self
|
||||
)
|
||||
|
||||
for bootstrap_info in self.bootstrap_infos:
|
||||
message = self.kv_mgr.build_receiver_metadata_message(
|
||||
bootstrap_room=self.bootstrap_room,
|
||||
required_dst_info_num=self.required_dst_info_num,
|
||||
kv_indices=kv_indices,
|
||||
aux_index=aux_index,
|
||||
state_indices=state_indices,
|
||||
decode_prefix_len=decode_prefix_len,
|
||||
is_dummy=bootstrap_info["is_dummy"],
|
||||
device_kv_indices=device_kv_indices,
|
||||
)
|
||||
if not self._send_bootstrap_message(
|
||||
bootstrap_info=bootstrap_info,
|
||||
message=message,
|
||||
operation="send_metadata",
|
||||
):
|
||||
self.invalidate_cached_bootstrap_infos()
|
||||
return
|
||||
|
||||
self.init_time = time.time()
|
||||
|
||||
def _check_waiting_timeout(self) -> Optional[KVPoll]:
|
||||
if self.init_time is None:
|
||||
@@ -2625,6 +2894,11 @@ class CommonKVReceiver(BaseKVReceiver):
|
||||
# Preserve the recorded root cause; this is cleanup, not a new abort.
|
||||
self._send_abort_notification()
|
||||
self.abort_notified = True
|
||||
if self.conclude_state is None:
|
||||
status = self.kv_mgr.request_status.get(self.bootstrap_room)
|
||||
self.conclude_state = (
|
||||
status if status in (KVPoll.Success, KVPoll.Failed) else KVPoll.Failed
|
||||
)
|
||||
self.kv_mgr.request_status.pop(self.bootstrap_room, None)
|
||||
self.kv_mgr.required_prefill_response_num_table.pop(self.bootstrap_room, None)
|
||||
self.kv_mgr.prefill_response_tracker.pop(self.bootstrap_room, None)
|
||||
@@ -2634,16 +2908,25 @@ class CommonKVReceiver(BaseKVReceiver):
|
||||
if self.kv_mgr.defer_decode_allocation:
|
||||
self._owns_bootstrap_room = False
|
||||
|
||||
def failure_exception(self):
|
||||
self.conclude_state = KVPoll.Failed
|
||||
self.clear()
|
||||
with self.kv_mgr.failure_lock:
|
||||
failure_reason = self.kv_mgr.failure_records.pop(self.bootstrap_room, None)
|
||||
raise KVTransferError(
|
||||
self.bootstrap_room,
|
||||
failure_reason or "Failed due to an unknown reason from another rank",
|
||||
is_from_another_rank=failure_reason is None,
|
||||
)
|
||||
|
||||
def abort(self):
|
||||
if not self._owns_bootstrap_room:
|
||||
self.conclude_state = KVPoll.Failed
|
||||
return
|
||||
self.kv_mgr.record_failure(
|
||||
self.bootstrap_room,
|
||||
"Aborted by AbortReq.",
|
||||
)
|
||||
self.kv_mgr.update_status(self.bootstrap_room, KVPoll.Failed)
|
||||
self.conclude_state = KVPoll.Failed
|
||||
if self.conclude_state != KVPoll.Failed:
|
||||
self.kv_mgr.record_failure(self.bootstrap_room, "Aborted by AbortReq.")
|
||||
self.kv_mgr.update_status(self.bootstrap_room, KVPoll.Failed)
|
||||
self.conclude_state = KVPoll.Failed
|
||||
self.ensure_abort_notified()
|
||||
|
||||
def ensure_abort_notified(self, *, force_arm: bool = False) -> None:
|
||||
|
||||
@@ -312,7 +312,7 @@ class DecodeStagingHandler:
|
||||
num_pages: int,
|
||||
writer_id: str,
|
||||
) -> bool:
|
||||
"""Process a staging chunk arrival from any transport (NIXL RDMA notif or ZMQ CHUNK_READY).
|
||||
"""Process a staging chunk arrival reported by a ZMQ CHUNK_READY message.
|
||||
|
||||
Accumulates writer arrivals and submits scatter once all writers for
|
||||
this chunk have reported in. Returns True if scatter was submitted.
|
||||
|
||||
@@ -24,7 +24,6 @@ class TransferKVChunk:
|
||||
is_last_chunk: bool
|
||||
prefill_aux_index: Optional[int]
|
||||
state_indices: Optional[List]
|
||||
chunk_id: Optional[int] = None
|
||||
num_kv_tokens: Optional[int] = None
|
||||
trace_ctx: Union[TraceReqContext, TraceNullContext] = dataclasses.field(
|
||||
default_factory=TraceNullContext
|
||||
|
||||
@@ -13,7 +13,6 @@ from typing import Dict, List, Optional, Set, Tuple, Union
|
||||
import numpy as np
|
||||
import numpy.typing as npt
|
||||
import torch
|
||||
import zmq
|
||||
from prometheus_client import Counter
|
||||
|
||||
from sglang.srt.disaggregation.base.conn import KVArgs, KVPoll, StateType
|
||||
@@ -25,7 +24,6 @@ from sglang.srt.disaggregation.common.conn import (
|
||||
CommonKVManager,
|
||||
CommonKVReceiver,
|
||||
CommonKVSender,
|
||||
KVTransferError,
|
||||
)
|
||||
from sglang.srt.disaggregation.common.staging_handler import (
|
||||
STAGING_WATERMARK_WAIT_S,
|
||||
@@ -251,6 +249,8 @@ def _uses_flat_entry_layout(
|
||||
|
||||
|
||||
class MooncakeKVManager(StagingManagerMixin, CommonKVManager):
|
||||
sender_trace_module = "mooncake"
|
||||
|
||||
AUX_DATA_HEADER = b"AUX_DATA"
|
||||
# Implements teardown() below, so runtime PD role switching is supported.
|
||||
supports_role_switch = True
|
||||
@@ -585,24 +585,6 @@ class MooncakeKVManager(StagingManagerMixin, CommonKVManager):
|
||||
|
||||
return PrefillStagingStrategy(self, staging_buffer)
|
||||
|
||||
def _send_chunk_ready(self, req, chunk_idx, kv_chunk, prefill_unique_rank):
|
||||
"""Notify decode that a staging chunk RDMA is complete (every chunk;
|
||||
scatter is arrival-driven)."""
|
||||
na = NetworkAddress(req.endpoint, req.dst_port)
|
||||
self._send_multipart_locked(
|
||||
na.to_tcp(),
|
||||
[
|
||||
b"CHUNK_READY",
|
||||
str(req.room).encode("ascii"),
|
||||
str(chunk_idx).encode("ascii"),
|
||||
str(kv_chunk.index_slice.start).encode("ascii"),
|
||||
str(len(kv_chunk.prefill_kv_indices)).encode("ascii"),
|
||||
req.mooncake_session_id.encode("ascii"),
|
||||
str(prefill_unique_rank).encode("ascii"),
|
||||
],
|
||||
is_ipv6=na.is_ipv6,
|
||||
)
|
||||
|
||||
def _do_staging_transfer(
|
||||
self,
|
||||
staging_strategy,
|
||||
@@ -612,7 +594,6 @@ class MooncakeKVManager(StagingManagerMixin, CommonKVManager):
|
||||
chunked_dst_kv_indice,
|
||||
executor,
|
||||
queue,
|
||||
prefill_unique_rank,
|
||||
):
|
||||
"""Execute staging transfer for one chunk. Returns (ret, deferred).
|
||||
|
||||
@@ -663,7 +644,14 @@ class MooncakeKVManager(StagingManagerMixin, CommonKVManager):
|
||||
)
|
||||
return (-1, False)
|
||||
if ret == 0:
|
||||
self._send_chunk_ready(req, chunk_idx, kv_chunk, prefill_unique_rank)
|
||||
self.send_chunk_ready(
|
||||
room=req.room,
|
||||
chunk_idx=chunk_idx,
|
||||
page_start=kv_chunk.index_slice.start,
|
||||
num_pages=len(kv_chunk.prefill_kv_indices),
|
||||
writer_id=req.mooncake_session_id,
|
||||
targets=[(req.endpoint, req.dst_port)],
|
||||
)
|
||||
return (ret, False)
|
||||
|
||||
def _prefetch_staging_reqs(self, room: int):
|
||||
@@ -2559,8 +2547,6 @@ class MooncakeKVManager(StagingManagerMixin, CommonKVManager):
|
||||
)
|
||||
polls = []
|
||||
dst_ranks_infos = []
|
||||
# Unique id per prefill sender so decode's response set size matches expected_response_num.
|
||||
prefill_unique_rank = self._prefill_unique_rank()
|
||||
# When staging transfer is not yet ready (watermark/allocation pending),
|
||||
# the chunk is re-enqueued and we break out of the req loop to retry later.
|
||||
staging_deferred = False
|
||||
@@ -2707,7 +2693,6 @@ class MooncakeKVManager(StagingManagerMixin, CommonKVManager):
|
||||
chunked_dst_kv_indice,
|
||||
executor,
|
||||
queue,
|
||||
prefill_unique_rank,
|
||||
)
|
||||
if deferred:
|
||||
staging_deferred = True
|
||||
@@ -3038,23 +3023,7 @@ class MooncakeKVManager(StagingManagerMixin, CommonKVManager):
|
||||
continue
|
||||
|
||||
# Staging: prefill notifies a chunk written to staging buffer
|
||||
if msg[0] == b"CHUNK_READY":
|
||||
room = int(msg[1].decode("ascii"))
|
||||
chunk_idx = int(msg[2].decode("ascii"))
|
||||
page_start = int(msg[3].decode("ascii"))
|
||||
num_pages = int(msg[4].decode("ascii"))
|
||||
session_id = msg[5].decode("ascii")
|
||||
handler = self._staging_handler
|
||||
assert handler is not None, (
|
||||
"CHUNK_READY received before staging handler initialized"
|
||||
)
|
||||
handler.handle_chunk_arrived(
|
||||
room,
|
||||
chunk_idx,
|
||||
page_start,
|
||||
num_pages,
|
||||
session_id,
|
||||
)
|
||||
if self.handle_chunk_ready(msg):
|
||||
continue
|
||||
|
||||
# Staging: prefill pre-requests staging allocation before forward
|
||||
@@ -3091,9 +3060,10 @@ class MooncakeKVManager(StagingManagerMixin, CommonKVManager):
|
||||
aux_index: Optional[int] = None,
|
||||
state_indices: Optional[List] = None,
|
||||
num_kv_tokens: Optional[int] = None,
|
||||
trace_ctx: Optional[Union[TraceReqContext, TraceNullContext]] = None,
|
||||
*,
|
||||
wait_event: Optional[object] = None,
|
||||
indices_event: Optional[object] = None,
|
||||
trace_ctx: Optional[Union[TraceReqContext, TraceNullContext]] = None,
|
||||
):
|
||||
assert self.disaggregation_mode == DisaggregationMode.PREFILL
|
||||
assert not is_last_chunk or (is_last_chunk and aux_index is not None)
|
||||
@@ -3215,51 +3185,103 @@ class MooncakeKVManager(StagingManagerMixin, CommonKVManager):
|
||||
):
|
||||
self._run_one_probe_pass()
|
||||
|
||||
|
||||
class MooncakeFailureExceptionMixin:
|
||||
"""Shared `failure_exception` for the Mooncake sender and receiver.
|
||||
|
||||
Both sides conclude a failed room identically: latch Failed, clear local
|
||||
state, then raise with the recorded reason -- or, when no reason was
|
||||
recorded locally, report it as propagated from another rank. Expects the
|
||||
concrete class to provide ``conclude_state``, ``clear()``,
|
||||
``bootstrap_room`` and ``kv_mgr``.
|
||||
"""
|
||||
|
||||
def failure_exception(self):
|
||||
# A room with no locally recorded reason failed on another rank.
|
||||
if self.conclude_state is None:
|
||||
self.conclude_state = KVPoll.Failed
|
||||
|
||||
self.clear()
|
||||
|
||||
with self.kv_mgr.failure_lock:
|
||||
failure_reason = self.kv_mgr.failure_records.pop(self.bootstrap_room, None)
|
||||
is_propagated = failure_reason is None
|
||||
if is_propagated:
|
||||
failure_reason = "Failed due to an unknown reason from another rank"
|
||||
raise KVTransferError(
|
||||
self.bootstrap_room, failure_reason, is_from_another_rank=is_propagated
|
||||
def build_receiver_registration_message(self) -> List[bytes]:
|
||||
session_id = self.get_session_id()
|
||||
packed_kv_data_ptrs = b"".join(
|
||||
struct.pack("Q", ptr) for ptr in self.kv_args.kv_data_ptrs
|
||||
)
|
||||
packed_aux_data_ptrs = b"".join(
|
||||
struct.pack("Q", ptr) for ptr in self.kv_args.aux_data_ptrs
|
||||
)
|
||||
packed_state_data_ptrs = pack_int_lists(self.kv_args.state_data_ptrs, "Q")
|
||||
packed_state_item_lens = pack_int_lists(self.kv_args.state_item_lens, "I")
|
||||
packed_state_dim_per_tensor = pack_int_lists(
|
||||
getattr(self.kv_args, "state_dim_per_tensor", []) or [], "I"
|
||||
)
|
||||
packed_state_layer_ids = pack_int_lists(self.kv_args.state_layer_ids, "I")
|
||||
packed_kv_layer_ids = b"".join(
|
||||
struct.pack("I", layer_id) for layer_id in self.kv_args.kv_layer_ids
|
||||
)
|
||||
# Note(shangming): No need to add pp rank here since decode pp size should be equal to prefill pp size or 1
|
||||
tp_rank = self.kv_args.engine_rank
|
||||
# Some pools have no full-token contiguous KV (kv_item_lens empty)
|
||||
# and ship per-pool instead, so report 0.
|
||||
kv_item_len = self.kv_args.kv_item_lens[0] if self.kv_args.kv_item_lens else 0
|
||||
dst_tp_rank = str(tp_rank).encode("ascii")
|
||||
dst_attn_tp_size = str(self.attn_tp_size).encode("ascii")
|
||||
dst_kv_item_len = str(kv_item_len).encode("ascii")
|
||||
dst_dcp_size = str(self.dcp_size).encode("ascii")
|
||||
dst_dcp_rank = str(self.dcp_rank).encode("ascii")
|
||||
if self.enable_staging and self._staging_ctx.allocator is not None:
|
||||
_alloc = self._staging_ctx.allocator
|
||||
packed_staging_base_ptr = struct.pack("Q", _alloc.get_base_ptr())
|
||||
staging_total_size_str = str(_alloc.get_total_size()).encode("ascii")
|
||||
else:
|
||||
packed_staging_base_ptr = b""
|
||||
staging_total_size_str = b""
|
||||
staging_slots = getattr(self, "kv_buffer_tensors", None) or {}
|
||||
packed_staging_slot_layer_ids = b"".join(
|
||||
struct.pack("Q", layer_id)
|
||||
for layer_id in (staging_slots.get("slot_layer_ids") or [])
|
||||
)
|
||||
|
||||
return [
|
||||
"None".encode("ascii"),
|
||||
self.local_ip.encode("ascii"),
|
||||
str(self.rank_port).encode("ascii"),
|
||||
session_id.encode("ascii"),
|
||||
packed_kv_data_ptrs,
|
||||
packed_aux_data_ptrs,
|
||||
packed_state_data_ptrs,
|
||||
dst_tp_rank,
|
||||
dst_attn_tp_size,
|
||||
dst_kv_item_len,
|
||||
packed_state_item_lens,
|
||||
packed_state_dim_per_tensor,
|
||||
packed_kv_layer_ids,
|
||||
packed_state_layer_ids,
|
||||
packed_staging_base_ptr,
|
||||
staging_total_size_str,
|
||||
dst_dcp_size,
|
||||
dst_dcp_rank,
|
||||
packed_staging_slot_layer_ids,
|
||||
struct.pack(
|
||||
f"{len(self.kv_args.kv_item_lens)}Q",
|
||||
*self.kv_args.kv_item_lens,
|
||||
),
|
||||
]
|
||||
|
||||
class MooncakeKVSender(MooncakeFailureExceptionMixin, CommonKVSender):
|
||||
def __init__(
|
||||
def build_receiver_metadata_message(
|
||||
self,
|
||||
mgr: MooncakeKVManager,
|
||||
bootstrap_addr: str,
|
||||
*,
|
||||
bootstrap_room: int,
|
||||
req_has_disagg_prefill_dp_rank: bool = False,
|
||||
):
|
||||
super().__init__(
|
||||
mgr,
|
||||
bootstrap_addr,
|
||||
bootstrap_room,
|
||||
req_has_disagg_prefill_dp_rank,
|
||||
)
|
||||
self.init_time = time.time()
|
||||
self._init_trace_ctx()
|
||||
required_dst_info_num: int,
|
||||
kv_indices: npt.NDArray[np.int32],
|
||||
aux_index: Optional[int],
|
||||
state_indices: Optional[List],
|
||||
decode_prefix_len: Optional[int],
|
||||
is_dummy: bool,
|
||||
device_kv_indices: Optional[npt.NDArray[np.int32]] = None,
|
||||
) -> List[bytes]:
|
||||
return [
|
||||
str(bootstrap_room).encode("ascii"),
|
||||
self.local_ip.encode("ascii"),
|
||||
str(self.rank_port).encode("ascii"),
|
||||
self.get_session_id().encode("ascii"),
|
||||
kv_indices.tobytes() if not is_dummy else b"",
|
||||
str(aux_index).encode("ascii") if not is_dummy else b"",
|
||||
pack_int_lists(state_indices, "i")
|
||||
if not is_dummy and state_indices
|
||||
else b"",
|
||||
str(required_dst_info_num).encode("ascii"),
|
||||
str(decode_prefix_len or 0).encode("ascii"),
|
||||
np.asarray(device_kv_indices, dtype=np.int32).tobytes()
|
||||
if not is_dummy and device_kv_indices is not None
|
||||
else b"",
|
||||
]
|
||||
|
||||
|
||||
class MooncakeKVSender(CommonKVSender):
|
||||
supports_host_tensor_indices = True
|
||||
|
||||
@mooncake_trace_func(MooncakeRequestStage.MOONCAKE_SEND)
|
||||
@@ -3270,260 +3292,16 @@ class MooncakeKVSender(MooncakeFailureExceptionMixin, CommonKVSender):
|
||||
num_kv_tokens: Optional[int] = None,
|
||||
indices_event: Optional[object] = None,
|
||||
):
|
||||
kv_indices, index_slice, is_last_chunk, should_skip = (
|
||||
self._prepare_send_indices(kv_indices, state_indices)
|
||||
super().send(
|
||||
kv_indices=kv_indices,
|
||||
state_indices=state_indices,
|
||||
num_kv_tokens=num_kv_tokens,
|
||||
indices_event=indices_event,
|
||||
)
|
||||
if should_skip:
|
||||
return
|
||||
|
||||
wait_event = getattr(self, "_early_send_wait_event", None)
|
||||
self._early_send_wait_event = None
|
||||
if not is_last_chunk:
|
||||
self.kv_mgr.add_transfer_request(
|
||||
self.bootstrap_room,
|
||||
kv_indices,
|
||||
index_slice,
|
||||
False,
|
||||
num_kv_tokens=num_kv_tokens,
|
||||
trace_ctx=self.trace_ctx.copy_for_thread(),
|
||||
wait_event=wait_event,
|
||||
indices_event=indices_event,
|
||||
)
|
||||
else:
|
||||
self.kv_mgr.add_transfer_request(
|
||||
self.bootstrap_room,
|
||||
kv_indices,
|
||||
index_slice,
|
||||
True,
|
||||
aux_index=self.aux_index,
|
||||
state_indices=state_indices,
|
||||
num_kv_tokens=num_kv_tokens,
|
||||
trace_ctx=self.trace_ctx.copy_for_thread(),
|
||||
wait_event=wait_event,
|
||||
indices_event=indices_event,
|
||||
)
|
||||
self._record_transfer_indices(kv_indices, state_indices)
|
||||
|
||||
def poll(self) -> KVPoll:
|
||||
return self._poll_with_status(self.kv_mgr.check_status)
|
||||
|
||||
def poll_pp_consensus(self) -> KVPoll:
|
||||
return self._poll_with_status(self.kv_mgr.check_status_pp_consensus)
|
||||
|
||||
def _poll_with_status(self, check_status) -> KVPoll:
|
||||
if self.conclude_state is not None:
|
||||
return self.conclude_state
|
||||
|
||||
status = check_status(self.bootstrap_room)
|
||||
# Hold Success until all staging chunks transferred. With deferred
|
||||
# allocation also wait for worker cleanup (the zero entry is only
|
||||
# removed after cleanup), before permitting the room to be reused.
|
||||
if status == KVPoll.Success and (
|
||||
self.kv_mgr._staging_outstanding.get(self.bootstrap_room, 0) > 0
|
||||
or (
|
||||
self.kv_mgr.defer_decode_allocation
|
||||
and self.bootstrap_room in self.kv_mgr._staging_outstanding
|
||||
)
|
||||
):
|
||||
return KVPoll.Transferring
|
||||
if status in (KVPoll.Success, KVPoll.Failed):
|
||||
self.conclude_state = status
|
||||
self.trace_ctx.trace_req_finish()
|
||||
elif status == KVPoll.Bootstrapping:
|
||||
timeout_result = self._check_bootstrap_timeout()
|
||||
if timeout_result is not None:
|
||||
return timeout_result
|
||||
|
||||
return status
|
||||
|
||||
def _init_trace_ctx(self):
|
||||
if self.kv_mgr.enable_trace:
|
||||
self.trace_ctx = TraceReqContext(
|
||||
rid=str(hex(self.bootstrap_room)),
|
||||
bootstrap_room=self.bootstrap_room,
|
||||
role="Sender",
|
||||
module_name="mooncake",
|
||||
)
|
||||
if not self.trace_ctx.tracing_enable:
|
||||
self.trace_ctx = TraceNullContext()
|
||||
else:
|
||||
self.trace_ctx = TraceNullContext()
|
||||
|
||||
self.trace_ctx.trace_req_start()
|
||||
|
||||
def abort(self):
|
||||
super().abort()
|
||||
self.trace_ctx.abort(abort_info={"reason": "Aborted"})
|
||||
self.trace_ctx.trace_req_finish()
|
||||
|
||||
|
||||
class MooncakeKVReceiver(MooncakeFailureExceptionMixin, CommonKVReceiver):
|
||||
def __init__(
|
||||
self,
|
||||
mgr: MooncakeKVManager,
|
||||
bootstrap_addr: str,
|
||||
bootstrap_room: Optional[int] = None,
|
||||
):
|
||||
self.session_id = mgr.get_session_id()
|
||||
self.init_time = None
|
||||
super().__init__(mgr, bootstrap_addr, bootstrap_room)
|
||||
|
||||
def _register_kv_args(self) -> bool:
|
||||
for bootstrap_info in self.bootstrap_infos:
|
||||
packed_kv_data_ptrs = b"".join(
|
||||
struct.pack("Q", ptr) for ptr in self.kv_mgr.kv_args.kv_data_ptrs
|
||||
)
|
||||
packed_aux_data_ptrs = b"".join(
|
||||
struct.pack("Q", ptr) for ptr in self.kv_mgr.kv_args.aux_data_ptrs
|
||||
)
|
||||
packed_state_data_ptrs = pack_int_lists(
|
||||
self.kv_mgr.kv_args.state_data_ptrs, "Q"
|
||||
)
|
||||
packed_state_item_lens = pack_int_lists(
|
||||
self.kv_mgr.kv_args.state_item_lens, "I"
|
||||
)
|
||||
packed_state_dim_per_tensor = pack_int_lists(
|
||||
getattr(self.kv_mgr.kv_args, "state_dim_per_tensor", []) or [], "I"
|
||||
)
|
||||
packed_state_layer_ids = pack_int_lists(
|
||||
self.kv_mgr.kv_args.state_layer_ids, "I"
|
||||
)
|
||||
packed_kv_layer_ids = b"".join(
|
||||
struct.pack("I", layer_id)
|
||||
for layer_id in self.kv_mgr.kv_args.kv_layer_ids
|
||||
)
|
||||
# Note(shangming): No need to add pp rank here since decode pp size should be equal to prefill pp size or 1
|
||||
tp_rank = self.kv_mgr.kv_args.engine_rank
|
||||
# Some pools have no full-token contiguous KV (kv_item_lens empty)
|
||||
# and ship per-pool instead, so report 0.
|
||||
kv_item_len = (
|
||||
self.kv_mgr.kv_args.kv_item_lens[0]
|
||||
if self.kv_mgr.kv_args.kv_item_lens
|
||||
else 0
|
||||
)
|
||||
dst_tp_rank = str(tp_rank).encode("ascii")
|
||||
dst_attn_tp_size = str(self.kv_mgr.attn_tp_size).encode("ascii")
|
||||
dst_kv_item_len = str(kv_item_len).encode("ascii")
|
||||
dst_dcp_size = str(self.kv_mgr.dcp_size).encode("ascii")
|
||||
dst_dcp_rank = str(self.kv_mgr.dcp_rank).encode("ascii")
|
||||
if (
|
||||
self.kv_mgr.enable_staging
|
||||
and self.kv_mgr._staging_ctx.allocator is not None
|
||||
):
|
||||
_alloc = self.kv_mgr._staging_ctx.allocator
|
||||
packed_staging_base_ptr = struct.pack("Q", _alloc.get_base_ptr())
|
||||
staging_total_size_str = str(_alloc.get_total_size()).encode("ascii")
|
||||
else:
|
||||
packed_staging_base_ptr = b""
|
||||
staging_total_size_str = b""
|
||||
staging_slots = getattr(self.kv_mgr, "kv_buffer_tensors", None) or {}
|
||||
packed_staging_slot_layer_ids = b"".join(
|
||||
struct.pack("Q", layer_id)
|
||||
for layer_id in (staging_slots.get("slot_layer_ids") or [])
|
||||
)
|
||||
|
||||
try:
|
||||
sock, lock = self._connect_to_bootstrap_server(bootstrap_info)
|
||||
with lock:
|
||||
sock.send_multipart(
|
||||
[
|
||||
"None".encode("ascii"),
|
||||
self.kv_mgr.local_ip.encode("ascii"),
|
||||
str(self.kv_mgr.rank_port).encode("ascii"),
|
||||
self.session_id.encode("ascii"),
|
||||
packed_kv_data_ptrs,
|
||||
packed_aux_data_ptrs,
|
||||
packed_state_data_ptrs,
|
||||
dst_tp_rank,
|
||||
dst_attn_tp_size,
|
||||
dst_kv_item_len,
|
||||
packed_state_item_lens,
|
||||
packed_state_dim_per_tensor,
|
||||
packed_kv_layer_ids,
|
||||
packed_state_layer_ids,
|
||||
packed_staging_base_ptr,
|
||||
staging_total_size_str,
|
||||
dst_dcp_size,
|
||||
dst_dcp_rank,
|
||||
packed_staging_slot_layer_ids,
|
||||
struct.pack(
|
||||
f"{len(self.kv_mgr.kv_args.kv_item_lens)}Q",
|
||||
*self.kv_mgr.kv_args.kv_item_lens,
|
||||
),
|
||||
]
|
||||
)
|
||||
except zmq.ZMQError:
|
||||
self.kv_mgr.record_failure(
|
||||
self.bootstrap_room,
|
||||
f"_register_kv_args to prefill {bootstrap_info.get('rank_ip')}:{bootstrap_info.get('rank_port')} failed",
|
||||
)
|
||||
self.conclude_state = KVPoll.Failed
|
||||
self.kv_mgr.update_status(self.bootstrap_room, KVPoll.Failed)
|
||||
return False
|
||||
return True
|
||||
|
||||
def send_metadata(
|
||||
self,
|
||||
kv_indices: npt.NDArray[np.int32],
|
||||
aux_index: Optional[int] = None,
|
||||
state_indices: Optional[List] = None,
|
||||
decode_prefix_len: Optional[int] = None,
|
||||
device_kv_indices: Optional[npt.NDArray[np.int32]] = None,
|
||||
):
|
||||
if self.bootstrap_infos is None:
|
||||
self.kv_mgr.record_failure(
|
||||
self.bootstrap_room,
|
||||
f"Could not fetch prefill parallel info from bootstrap_addr: {self.bootstrap_addr}",
|
||||
)
|
||||
self.kv_mgr.update_status(self.bootstrap_room, KVPoll.Failed)
|
||||
return
|
||||
|
||||
self.chunk_staging_infos = []
|
||||
if (
|
||||
self.kv_mgr.enable_staging
|
||||
and self.kv_mgr._staging_ctx.allocator is not None
|
||||
):
|
||||
self.kv_mgr.register_staging_room_bootstrap(
|
||||
self.bootstrap_room, self.bootstrap_infos, self
|
||||
)
|
||||
|
||||
for bootstrap_info in self.bootstrap_infos:
|
||||
is_dummy = bootstrap_info["is_dummy"]
|
||||
try:
|
||||
sock, lock = self._connect_to_bootstrap_server(bootstrap_info)
|
||||
with lock:
|
||||
sock.send_multipart(
|
||||
[
|
||||
str(self.bootstrap_room).encode("ascii"),
|
||||
self.kv_mgr.local_ip.encode("ascii"),
|
||||
str(self.kv_mgr.rank_port).encode("ascii"),
|
||||
self.session_id.encode("ascii"),
|
||||
kv_indices.tobytes() if not is_dummy else b"",
|
||||
str(aux_index).encode("ascii") if not is_dummy else b"",
|
||||
(
|
||||
pack_int_lists(state_indices, "i")
|
||||
if not is_dummy and state_indices
|
||||
else b""
|
||||
),
|
||||
str(self.required_dst_info_num).encode("ascii"),
|
||||
str(decode_prefix_len or 0).encode("ascii"),
|
||||
(
|
||||
np.asarray(device_kv_indices, dtype=np.int32).tobytes()
|
||||
if not is_dummy and device_kv_indices is not None
|
||||
else b""
|
||||
),
|
||||
]
|
||||
)
|
||||
except zmq.ZMQError:
|
||||
self.invalidate_cached_bootstrap_infos()
|
||||
self.kv_mgr.record_failure(
|
||||
self.bootstrap_room,
|
||||
f"send_metadata to prefill {bootstrap_info.get('rank_ip')}:{bootstrap_info.get('rank_port')} failed",
|
||||
)
|
||||
self.conclude_state = KVPoll.Failed
|
||||
self.kv_mgr.update_status(self.bootstrap_room, KVPoll.Failed)
|
||||
return
|
||||
self.init_time = time.time()
|
||||
class MooncakeKVReceiver(CommonKVReceiver):
|
||||
pass
|
||||
|
||||
|
||||
class MooncakeKVBootstrapServer(CommonKVBootstrapServer):
|
||||
|
||||
@@ -36,7 +36,6 @@ from sglang.srt.disaggregation.common.conn import (
|
||||
CommonKVManager,
|
||||
CommonKVReceiver,
|
||||
CommonKVSender,
|
||||
KVTransferError,
|
||||
)
|
||||
from sglang.srt.disaggregation.common.utils import (
|
||||
AuxDataCodec,
|
||||
@@ -785,6 +784,13 @@ class MoriKVManager(CommonKVManager):
|
||||
return f"KV transfer failed: {status.Message()}"
|
||||
return "KV transfer failed due to unknown reason"
|
||||
|
||||
def prepare_send_state_indices(
|
||||
self, state_indices: Optional[List]
|
||||
) -> Optional[List]:
|
||||
if self._should_skip_cp_replicated_state_transfer():
|
||||
return None
|
||||
return _normalize_state_indices_per_component(state_indices)
|
||||
|
||||
def add_transfer_request(
|
||||
self,
|
||||
bootstrap_room: int,
|
||||
@@ -794,7 +800,10 @@ class MoriKVManager(CommonKVManager):
|
||||
aux_index: Optional[int] = None,
|
||||
state_indices: Optional[List] = None,
|
||||
num_kv_tokens: Optional[int] = None,
|
||||
*,
|
||||
wait_event: Optional[object] = None,
|
||||
indices_event: Optional[object] = None,
|
||||
trace_ctx=None,
|
||||
) -> None:
|
||||
assert self.disaggregation_mode == DisaggregationMode.PREFILL
|
||||
assert not is_last_chunk or (is_last_chunk and aux_index is not None)
|
||||
@@ -2056,95 +2065,95 @@ class MoriKVManager(CommonKVManager):
|
||||
|
||||
return result_statuses
|
||||
|
||||
def build_receiver_registration_message(self) -> List[bytes]:
|
||||
engine_desc_blob = self.engine_desc.pack()
|
||||
packed_kv_descs = _pack_mem_desc_list(self.kv_mem_descs)
|
||||
packed_aux_descs = _pack_mem_desc_list(self.aux_mem_descs)
|
||||
packed_state_descs = _pack_mem_desc_lists(self.state_mem_descs)
|
||||
gpu_id = str(self.kv_args.gpu_id).encode("ascii")
|
||||
decode_tp_size = str(self.attn_tp_size).encode("ascii")
|
||||
decode_tp_rank = str(self.kv_args.engine_rank).encode("ascii")
|
||||
kv_item_len = str(self.kv_args.kv_item_lens[0]).encode("ascii")
|
||||
packed_state_item_lens = pack_int_lists(self.kv_args.state_item_lens, "I")
|
||||
packed_state_dim_per_tensor = pack_int_lists(
|
||||
self.kv_args.state_dim_per_tensor, "I"
|
||||
)
|
||||
packed_state_layer_ids = pack_int_lists(self.kv_args.state_layer_ids, "I")
|
||||
packed_kv_layer_ids = struct.pack(
|
||||
f"<{len(self.kv_args.kv_layer_ids)}I",
|
||||
*self.kv_args.kv_layer_ids,
|
||||
)
|
||||
dcp_size = str(self.dcp_size).encode("ascii")
|
||||
dcp_rank = str(self.dcp_rank).encode("ascii")
|
||||
|
||||
return [
|
||||
MORI_GUARD,
|
||||
"None".encode("ascii"),
|
||||
self.local_ip.encode("ascii"),
|
||||
str(self.rank_port).encode("ascii"),
|
||||
engine_desc_blob,
|
||||
packed_kv_descs,
|
||||
packed_aux_descs,
|
||||
packed_state_descs,
|
||||
gpu_id,
|
||||
decode_tp_size,
|
||||
decode_tp_rank,
|
||||
kv_item_len,
|
||||
packed_state_item_lens,
|
||||
packed_state_dim_per_tensor,
|
||||
dcp_size,
|
||||
dcp_rank,
|
||||
packed_state_layer_ids,
|
||||
packed_kv_layer_ids,
|
||||
]
|
||||
|
||||
def build_receiver_metadata_message(
|
||||
self,
|
||||
*,
|
||||
bootstrap_room: int,
|
||||
required_dst_info_num: int,
|
||||
kv_indices: npt.NDArray[np.int32],
|
||||
aux_index: Optional[int],
|
||||
state_indices: Optional[List],
|
||||
decode_prefix_len: Optional[int],
|
||||
is_dummy: bool,
|
||||
device_kv_indices: Optional[npt.NDArray[np.int32]] = None,
|
||||
) -> List[bytes]:
|
||||
if is_dummy:
|
||||
kv_indices_bytes = aux_bytes = state_bytes = b""
|
||||
else:
|
||||
kv_indices_bytes = (
|
||||
np.asarray(kv_indices, dtype=np.int32).tobytes()
|
||||
if kv_indices.size
|
||||
else b""
|
||||
)
|
||||
aux_bytes = str(aux_index).encode("ascii") if aux_index is not None else b""
|
||||
normalized_state = _normalize_state_indices_per_component(state_indices)
|
||||
state_bytes = (
|
||||
_pack_state_indices(normalized_state)
|
||||
if normalized_state is not None
|
||||
else b""
|
||||
)
|
||||
decode_prefix_bytes = (
|
||||
str(int(decode_prefix_len)).encode("ascii")
|
||||
if decode_prefix_len is not None and decode_prefix_len > 0
|
||||
else b""
|
||||
)
|
||||
return [
|
||||
MORI_GUARD,
|
||||
str(bootstrap_room).encode("ascii"),
|
||||
self.local_ip.encode("ascii"),
|
||||
str(self.rank_port).encode("ascii"),
|
||||
self.engine_desc.key.encode("ascii"),
|
||||
kv_indices_bytes,
|
||||
aux_bytes,
|
||||
state_bytes,
|
||||
str(required_dst_info_num).encode("ascii"),
|
||||
decode_prefix_bytes,
|
||||
]
|
||||
|
||||
|
||||
class MoriKVSender(CommonKVSender):
|
||||
def __init__(
|
||||
self,
|
||||
mgr: MoriKVManager,
|
||||
bootstrap_addr: str,
|
||||
bootstrap_room: int,
|
||||
req_has_disagg_prefill_dp_rank: bool = False,
|
||||
):
|
||||
super().__init__(
|
||||
mgr,
|
||||
bootstrap_addr,
|
||||
bootstrap_room,
|
||||
req_has_disagg_prefill_dp_rank,
|
||||
)
|
||||
self.conclude_state: Optional[KVPoll] = None
|
||||
self.init_time = time.time()
|
||||
|
||||
def send(
|
||||
self,
|
||||
kv_indices: npt.NDArray[np.int32],
|
||||
state_indices: Optional[List] = None,
|
||||
num_kv_tokens: Optional[int] = None,
|
||||
):
|
||||
kv_indices, index_slice, is_last_chunk, should_skip = (
|
||||
self._prepare_send_indices(kv_indices, state_indices)
|
||||
)
|
||||
if should_skip:
|
||||
return
|
||||
|
||||
transfer_state_indices = (
|
||||
None
|
||||
if self.kv_mgr._should_skip_cp_replicated_state_transfer()
|
||||
else state_indices
|
||||
)
|
||||
normalized_state = (
|
||||
_normalize_state_indices_per_component(transfer_state_indices)
|
||||
if is_last_chunk
|
||||
else None
|
||||
)
|
||||
self._record_transfer_indices(kv_indices, transfer_state_indices)
|
||||
wait_event = getattr(self, "_early_send_wait_event", None)
|
||||
self._early_send_wait_event = None
|
||||
|
||||
if not is_last_chunk:
|
||||
self.kv_mgr.add_transfer_request(
|
||||
self.bootstrap_room,
|
||||
kv_indices,
|
||||
index_slice,
|
||||
False,
|
||||
num_kv_tokens=num_kv_tokens,
|
||||
wait_event=wait_event,
|
||||
)
|
||||
else:
|
||||
self.kv_mgr.add_transfer_request(
|
||||
self.bootstrap_room,
|
||||
kv_indices,
|
||||
index_slice,
|
||||
True,
|
||||
aux_index=self.aux_index,
|
||||
state_indices=normalized_state,
|
||||
num_kv_tokens=num_kv_tokens,
|
||||
wait_event=wait_event,
|
||||
)
|
||||
|
||||
def poll(self) -> KVPoll:
|
||||
return self._poll_with_status(self.kv_mgr.check_status)
|
||||
|
||||
def poll_pp_consensus(self) -> KVPoll:
|
||||
return self._poll_with_status(self.kv_mgr.check_status_pp_consensus)
|
||||
|
||||
def _poll_with_status(self, check_status) -> KVPoll:
|
||||
if self.conclude_state is not None:
|
||||
return self.conclude_state
|
||||
|
||||
if self.bootstrap_room not in self.kv_mgr.request_status:
|
||||
self.conclude_state = KVPoll.Failed
|
||||
return self.conclude_state
|
||||
|
||||
status = check_status(self.bootstrap_room)
|
||||
if status == KVPoll.Bootstrapping:
|
||||
timeout_result = self._check_bootstrap_timeout()
|
||||
if timeout_result is not None:
|
||||
self.conclude_state = timeout_result
|
||||
return timeout_result
|
||||
if status in (KVPoll.Success, KVPoll.Failed):
|
||||
self.conclude_state = status
|
||||
return status
|
||||
|
||||
def clear(self) -> None:
|
||||
if self.kv_mgr.enable_deferred_decode_kv_release:
|
||||
with self.kv_mgr._abort_ack_lock:
|
||||
@@ -2152,200 +2161,9 @@ class MoriKVSender(CommonKVSender):
|
||||
else:
|
||||
super().clear()
|
||||
|
||||
def failure_exception(self):
|
||||
if self.conclude_state is None:
|
||||
self.conclude_state = KVPoll.Failed
|
||||
|
||||
self.clear()
|
||||
|
||||
with self.kv_mgr.failure_lock:
|
||||
failure_reason = self.kv_mgr.failure_records.pop(self.bootstrap_room, None)
|
||||
is_propagated = failure_reason is None
|
||||
if is_propagated:
|
||||
failure_reason = "KV transfer failed"
|
||||
raise KVTransferError(
|
||||
self.bootstrap_room, failure_reason, is_from_another_rank=is_propagated
|
||||
)
|
||||
|
||||
|
||||
class MoriKVReceiver(CommonKVReceiver):
|
||||
def __init__(
|
||||
self,
|
||||
mgr: MoriKVManager,
|
||||
bootstrap_addr: str,
|
||||
bootstrap_room: Optional[int] = None,
|
||||
):
|
||||
super().__init__(mgr, bootstrap_addr, bootstrap_room)
|
||||
self.init_time: Optional[float] = None
|
||||
|
||||
def init(
|
||||
self,
|
||||
prefill_dp_rank: int,
|
||||
):
|
||||
super().init(prefill_dp_rank)
|
||||
|
||||
def _register_kv_args(self) -> bool:
|
||||
if self.bootstrap_infos is None:
|
||||
return False
|
||||
engine_desc_blob = self.kv_mgr.engine_desc.pack()
|
||||
packed_kv_descs = _pack_mem_desc_list(self.kv_mgr.kv_mem_descs)
|
||||
packed_aux_descs = _pack_mem_desc_list(self.kv_mgr.aux_mem_descs)
|
||||
packed_state_descs = _pack_mem_desc_lists(self.kv_mgr.state_mem_descs)
|
||||
gpu_id = str(self.kv_mgr.kv_args.gpu_id).encode("ascii")
|
||||
decode_tp_size = str(self.kv_mgr.attn_tp_size).encode("ascii")
|
||||
decode_tp_rank = str(self.kv_mgr.kv_args.engine_rank).encode("ascii")
|
||||
kv_item_len = str(self.kv_mgr.kv_args.kv_item_lens[0]).encode("ascii")
|
||||
packed_state_item_lens = pack_int_lists(
|
||||
self.kv_mgr.kv_args.state_item_lens, "I"
|
||||
)
|
||||
packed_state_dim_per_tensor = pack_int_lists(
|
||||
self.kv_mgr.kv_args.state_dim_per_tensor, "I"
|
||||
)
|
||||
packed_state_layer_ids = pack_int_lists(
|
||||
self.kv_mgr.kv_args.state_layer_ids, "I"
|
||||
)
|
||||
packed_kv_layer_ids = struct.pack(
|
||||
f"<{len(self.kv_mgr.kv_args.kv_layer_ids)}I",
|
||||
*self.kv_mgr.kv_args.kv_layer_ids,
|
||||
)
|
||||
dcp_size = str(self.kv_mgr.dcp_size).encode("ascii")
|
||||
dcp_rank = str(self.kv_mgr.dcp_rank).encode("ascii")
|
||||
|
||||
for bootstrap_info in self.bootstrap_infos:
|
||||
try:
|
||||
sock, lock = self._connect_to_bootstrap_server(bootstrap_info)
|
||||
with lock:
|
||||
sock.send_multipart(
|
||||
[
|
||||
MORI_GUARD,
|
||||
"None".encode("ascii"),
|
||||
self.kv_mgr.local_ip.encode("ascii"),
|
||||
str(self.kv_mgr.rank_port).encode("ascii"),
|
||||
engine_desc_blob,
|
||||
packed_kv_descs,
|
||||
packed_aux_descs,
|
||||
packed_state_descs,
|
||||
gpu_id,
|
||||
decode_tp_size,
|
||||
decode_tp_rank,
|
||||
kv_item_len,
|
||||
packed_state_item_lens,
|
||||
packed_state_dim_per_tensor,
|
||||
dcp_size,
|
||||
dcp_rank,
|
||||
packed_state_layer_ids,
|
||||
packed_kv_layer_ids,
|
||||
]
|
||||
)
|
||||
except zmq.ZMQError:
|
||||
self.kv_mgr.record_failure(
|
||||
self.bootstrap_room,
|
||||
f"_register_kv_args to prefill {bootstrap_info.get('rank_ip')}:{bootstrap_info.get('rank_port')} failed",
|
||||
)
|
||||
self.conclude_state = KVPoll.Failed
|
||||
self.kv_mgr.update_status(self.bootstrap_room, KVPoll.Failed)
|
||||
return False
|
||||
return True
|
||||
|
||||
def send_metadata(
|
||||
self,
|
||||
kv_indices: npt.NDArray[np.int32],
|
||||
aux_index: Optional[int] = None,
|
||||
state_indices: Optional[List] = None,
|
||||
decode_prefix_len: Optional[int] = None,
|
||||
):
|
||||
if self.bootstrap_infos is None or self.bootstrap_room is None:
|
||||
return
|
||||
|
||||
kv_indices_bytes = (
|
||||
np.asarray(kv_indices, dtype=np.int32).tobytes() if kv_indices.size else b""
|
||||
)
|
||||
aux_bytes = str(aux_index).encode("ascii") if aux_index is not None else b""
|
||||
normalized_state = _normalize_state_indices_per_component(state_indices)
|
||||
|
||||
decode_prefix_bytes = (
|
||||
str(int(decode_prefix_len)).encode("ascii")
|
||||
if decode_prefix_len is not None and decode_prefix_len > 0
|
||||
else b""
|
||||
)
|
||||
|
||||
for bootstrap_info in self.bootstrap_infos:
|
||||
is_dummy = bootstrap_info.get("is_dummy", False)
|
||||
if not is_dummy and normalized_state is not None:
|
||||
state_bytes = _pack_state_indices(normalized_state)
|
||||
else:
|
||||
state_bytes = b""
|
||||
try:
|
||||
sock, lock = self._connect_to_bootstrap_server(bootstrap_info)
|
||||
with lock:
|
||||
sock.send_multipart(
|
||||
[
|
||||
MORI_GUARD,
|
||||
str(self.bootstrap_room).encode("ascii"),
|
||||
self.kv_mgr.local_ip.encode("ascii"),
|
||||
str(self.kv_mgr.rank_port).encode("ascii"),
|
||||
self.kv_mgr.engine_desc.key.encode("ascii"),
|
||||
kv_indices_bytes if not is_dummy else b"",
|
||||
aux_bytes if not is_dummy else b"",
|
||||
state_bytes,
|
||||
str(self.required_dst_info_num).encode("ascii"),
|
||||
decode_prefix_bytes,
|
||||
]
|
||||
)
|
||||
except zmq.ZMQError:
|
||||
self.invalidate_cached_bootstrap_infos()
|
||||
self.kv_mgr.record_failure(
|
||||
self.bootstrap_room,
|
||||
f"send_metadata to prefill {bootstrap_info.get('rank_ip')}:{bootstrap_info.get('rank_port')} failed",
|
||||
)
|
||||
self.conclude_state = KVPoll.Failed
|
||||
self.kv_mgr.update_status(self.bootstrap_room, KVPoll.Failed)
|
||||
return
|
||||
self.init_time = time.time()
|
||||
|
||||
def poll(self) -> KVPoll:
|
||||
if self.conclude_state is not None:
|
||||
return self.conclude_state
|
||||
|
||||
status = self.kv_mgr.check_status(self.bootstrap_room)
|
||||
if status in (KVPoll.Success, KVPoll.Failed):
|
||||
self.conclude_state = status
|
||||
return status
|
||||
|
||||
if status == KVPoll.WaitingForInput:
|
||||
timeout_result = self._check_waiting_timeout()
|
||||
if timeout_result is not None:
|
||||
return timeout_result
|
||||
|
||||
return status
|
||||
|
||||
def clear(self) -> None:
|
||||
if self.bootstrap_room is None:
|
||||
return
|
||||
super().clear()
|
||||
|
||||
def failure_exception(self):
|
||||
if self.conclude_state is None:
|
||||
self.conclude_state = KVPoll.Failed
|
||||
|
||||
self.clear()
|
||||
with self.kv_mgr.failure_lock:
|
||||
failure_reason = self.kv_mgr.failure_records.pop(self.bootstrap_room, None)
|
||||
is_propagated = failure_reason is None
|
||||
if is_propagated:
|
||||
failure_reason = "KV transfer failed"
|
||||
raise KVTransferError(
|
||||
self.bootstrap_room, failure_reason, is_from_another_rank=is_propagated
|
||||
)
|
||||
|
||||
def abort(self):
|
||||
if self.bootstrap_room is None:
|
||||
return
|
||||
bootstrap_room = self.bootstrap_room
|
||||
super().abort()
|
||||
self.clear()
|
||||
with self.kv_mgr.failure_lock:
|
||||
self.kv_mgr.failure_records.pop(bootstrap_room, None)
|
||||
pass
|
||||
|
||||
|
||||
class MoriKVBootstrapServer(CommonKVBootstrapServer):
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
Reference in New Issue
Block a user