[PD] Add opt-in prefill-complete decode KV allocation (#40703)

Co-authored-by: Po-Han Huang (NVIDIA) <53919306+nvpohanh@users.noreply.github.com>
Co-authored-by: Po-Han Huang <pohanh@nvidia.com>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
Co-authored-by: Shangming Cai <csmthu@gmail.com>
This commit is contained in:
Evan Kriminger
2026-10-03 21:03:09 +08:00
committed by GitHub
co-authored by Po-Han Huang Po-Han Huang Claude Opus 5.5 Shangming Cai
parent 5916999afb
commit 5b5d721239
19 changed files with 2125 additions and 40 deletions
@@ -56,6 +56,12 @@ class Disagg(msgspec.Struct):
Literal["null", "prefill", "decode"],
'Only used for PD disaggregation. "prefill" for prefill-only server, and "decode" for decode-only server. If not specified, it is not PD disaggregated',
] = "null"
disaggregation_decode_allocation_policy: A[
Literal["early", "prefill_complete"],
"When to reserve per-request decode KV in disaggregated serving. "
"prefill_complete retains KV on prefill until prefill finishes, then "
"admits it on decode. Set the same policy on both roles. Default: early.",
] = "early"
disaggregation_transfer_backend: A[
str,
Arg(
@@ -44,6 +44,16 @@ def handle_pd_disaggregation(server_args: ServerArgs) -> None:
"with MC_FORCE_TCP=1 (TCP transport, no RDMA)"
)
if cfg.disaggregation_decode_allocation_policy == "prefill_complete":
_validate_prefill_complete(server_args)
if (
cfg.disaggregation_mode == "prefill"
and cfg.optimistic_prefill_attempts == 0
):
declare_resolution(
server_args, "handle_pd_disaggregation", optimistic_prefill_attempts=1
)
if cfg.disaggregation_mode == "prefill" and cfg.dcp_size > 1:
logger.warning(
"DCP on a PD prefill server is supported when prefill and decode "
@@ -191,6 +201,52 @@ def handle_pd_disaggregation(server_args: ServerArgs) -> None:
)
def _validate_prefill_complete(server_args: ServerArgs) -> None:
cfg = resolving_view(server_args)
unsupported = []
if cfg.disaggregation_mode not in ("prefill", "decode"):
unsupported.append("aggregated serving")
if cfg.disaggregation_transfer_backend != "mooncake":
unsupported.append("transfer backends other than Mooncake")
if cfg.pp_size != 1 or cfg.attn_cp_size != 1 or cfg.dcp_size != 1:
unsupported.append("pipeline or attention context parallelism")
attn_dp_size = cfg.dp_size if cfg.enable_dp_attention else 1
if cfg.tp_size != attn_dp_size:
unsupported.append("attention tensor parallelism greater than one")
if cfg.enable_pdmux or cfg.decoupled_spec_role != "null":
unsupported.append("PD multiplexing or decoupled speculation")
if cfg.enable_pd_role_switch:
unsupported.append("runtime PD role switching")
if cfg.encoder_only or cfg.language_only:
unsupported.append("encoder disaggregation")
if cfg.enable_hisparse:
unsupported.append("HiSparse")
if cfg.disaggregation_decode_host_receive_threshold > 0:
unsupported.append("decode host KV buffering")
if cfg.disaggregation_mode == "decode" and (
cfg.disaggregation_decode_enable_radix_cache or cfg.enable_hierarchical_cache
):
unsupported.append("decode radix cache or decode HiCache")
if (
cfg.disaggregation_mode == "prefill"
and cfg.enable_hierarchical_cache
and (
cfg.hicache_storage_backend is not None
or cfg.hicache_write_policy != "write_back"
)
):
unsupported.append("prefill HiCache other than L2 write_back")
if envs.SGLANG_DISAGG_STAGING_BUFFER.get():
unsupported.append("staging buffers")
if envs.SGLANG_RUST_SERVER.get():
unsupported.append("the Rust bootstrap registry")
if unsupported:
raise ValueError(
"--disaggregation-decode-allocation-policy prefill_complete does not "
"yet support " + ", ".join(unsupported)
)
def _alias_bootstrap_port_to_api_port(server_args: ServerArgs) -> None:
"""Rust-server prefill serves the KV bootstrap registry on the api listener
itself, so the resolved bootstrap port must BE the api port — every internal
@@ -494,6 +494,16 @@ def handle_other_validations(server_args: Any):
optimistic_prefill_attempts=0,
)
if (
cfg.disaggregation_decode_allocation_policy == "prefill_complete"
and cfg.disaggregation_mode == "prefill"
and resolving_view(server_args).optimistic_prefill_attempts <= 0
):
raise ValueError(
"prefill_complete requires optimistic prefill, but the resolved "
"model/cache configuration disabled it."
)
# Handle model inference tensor dump.
if cfg.debug_tensor_dump_output_folder is not None:
logger.warning(
@@ -182,6 +182,10 @@ class BaseKVSender(ABC):
"""
...
def mark_prefill_complete(self) -> None:
"""Publish retained final-prefill KV for an enabled allocation policy."""
raise NotImplementedError
def pop_decode_prefix_len(self) -> int:
return 0
@@ -0,0 +1,112 @@
"""Bookkeeping for bootstrap rendezvous before decode has reserved KV.
The transfer backend still owns the destination-pointer handshake and abort
draining. This table only joins a completed optimistic prefill with decode's
control endpoint. It performs no I/O, including while holding its lock.
"""
from __future__ import annotations
import threading
import time
from dataclasses import dataclass
BootstrapNotification = tuple[tuple[str, int], bool] | None
@dataclass
class BootstrapRoom:
owner: object | None = None
endpoint: tuple[str, int] | None = None
ready: bool = False
ready_notified: bool = False
failed: bool = False
expires_at: float = 0.0
class DeferredBootstrap:
def __init__(self, timeout: float):
self.timeout = timeout
self.rooms: dict[int, BootstrapRoom] = {}
self.lock = threading.Lock()
self._next_cleanup = 0.0
def _room(self, room: int) -> BootstrapRoom:
now = time.monotonic()
if now >= self._next_cleanup:
# Active senders own their lifetime. Unmatched endpoints and closed
# rooms expire, including cancellation before the sender exists.
self.rooms = {
key: state
for key, state in self.rooms.items()
if state.owner is not None or state.expires_at > now
}
self._next_cleanup = now + min(1.0, self.timeout)
return self.rooms.setdefault(room, BootstrapRoom(expires_at=now + self.timeout))
@staticmethod
def _notification(state: BootstrapRoom) -> BootstrapNotification:
if state.endpoint is not None:
if state.failed:
return state.endpoint, True
if state.ready and not state.ready_notified:
# One readiness message per transfer. Decode must consume it
# before publishing destinations, so none can trail Success
# and accidentally admit the next transfer using this room.
state.ready_notified = True
return state.endpoint, False
return None
def open(self, room: int, owner: object) -> BootstrapRoom | None:
with self.lock:
state = self._room(room)
if state.owner is not None:
return None
state.owner = owner
return state
def register(self, room: int, endpoint: tuple[str, int]) -> BootstrapNotification:
with self.lock:
state = self._room(room)
if state.endpoint is not None and state.endpoint != endpoint:
# A room identifies one request, not a reusable allocation slot.
# Reject another receiver without replacing the first endpoint.
return endpoint, True
state.endpoint = endpoint
return self._notification(state)
def complete(self, room: int, owner: object) -> BootstrapNotification:
with self.lock:
state = self.rooms.get(room)
if state is None or state.owner is not owner or state.failed:
return None
state.ready = True
return self._notification(state)
def fail(self, room: int) -> BootstrapNotification:
with self.lock:
state = self._room(room)
if state.failed:
return None
state.failed = True
return self._notification(state)
def close(
self, room: int, owner: object, *, success: bool = False
) -> BootstrapNotification:
with self.lock:
state = self.rooms.get(room)
if state is None or state.owner is not owner:
return None
if success and not state.failed:
# The caller clears transport state before sending Success.
# Only then can decode reuse this room for recomputation.
del self.rooms[room]
return (state.endpoint, False) if state.endpoint is not None else None
state.owner = None
already_failed = state.failed
state.failed = True
state.expires_at = time.monotonic() + self.timeout
# Retain a bounded tombstone: a late endpoint must not wait forever
# or attach to a second request using the same bootstrap room.
return None if already_failed else self._notification(state)
+266 -8
View File
@@ -7,6 +7,7 @@ import logging
import threading
import time
from collections import defaultdict
from contextlib import nullcontext
from typing import Dict, List, NamedTuple, Optional, Set, Tuple, Union
import msgspec
@@ -28,6 +29,10 @@ from sglang.srt.disaggregation.base.conn import (
KVTransferMetric,
StateType,
)
from sglang.srt.disaggregation.common.bootstrap import (
BootstrapNotification,
DeferredBootstrap,
)
from sglang.srt.disaggregation.utils import (
DisaggregationMode,
filter_kv_indices_for_cp_rank,
@@ -107,6 +112,7 @@ class PrefillServerInfo:
follow_bootstrap_room: bool
enable_dsa_cache_layer_split: bool = False
dsv41_spec_layout: Optional[dict] = None
decode_allocation_policy: str = "early"
# PD true-retraction rebootstrap: the prefill's HTTP API port. The decode
# already knows the prefill host (the bootstrap_addr host), so it can POST
@@ -259,6 +265,9 @@ class DeferredAbortAckState(msgspec.Struct):
class CommonKVManager(BaseKVManager):
defer_decode_allocation: bool = False
deferred_bootstrap: Optional[DeferredBootstrap] = None
# Wire layout of the prefill->decode terminal status message. The legacy
# layout (mooncake, and ascend which inherits it) is three untagged frames
# ``[room, status, prefill_rank]``; backends whose control socket also
@@ -359,6 +368,20 @@ class CommonKVManager(BaseKVManager):
# A poisoned room has a permanently outstanding transfer whose backend
# can no longer prove quiescent. A new room lifecycle clears this state.
self._deferred_ack_poisoned_rooms: Set[int] = set()
self.defer_decode_allocation = (
get_disagg().disaggregation_decode_allocation_policy == "prefill_complete"
)
self.deferred_bootstrap = None
if self.defer_decode_allocation:
if any(state != StateType.DSA for state in args.state_types):
raise ValueError(
"prefill_complete currently supports full-attention and DSA KV pools only"
)
if disaggregation_mode == DisaggregationMode.PREFILL:
self.deferred_bootstrap = DeferredBootstrap(
envs.SGLANG_DISAGGREGATION_WAITING_TIMEOUT.get()
+ envs.SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT.get()
)
if self.disaggregation_mode == DisaggregationMode.PREFILL:
# When SGLANG_DISAGGREGATION_ALL_CP_RANKS_TRANSFER is True, all CP ranks
@@ -542,6 +565,10 @@ class CommonKVManager(BaseKVManager):
return
if status == KVPoll.Failed:
self.request_status[bootstrap_room] = KVPoll.Failed
if self.deferred_bootstrap is not None:
self._notify_bootstrap(
bootstrap_room, self.deferred_bootstrap.fail(bootstrap_room)
)
return
if current == KVPoll.Failed:
# Failed is terminal. It also sorts lowest, so the max() below would
@@ -549,6 +576,42 @@ class CommonKVManager(BaseKVManager):
return
self.request_status[bootstrap_room] = max(current, status)
def _notify_bootstrap(self, room: int, notification: BootstrapNotification) -> None:
if notification is None:
return
endpoint, failed = notification
# Use the normal status channel; readiness does not authorize writes.
# No room-table lock is held while sending to a potentially dead peer.
self.send_kv_status_message(
targets=[endpoint],
bootstrap_room=room,
status=KVPoll.Failed if failed else KVPoll.WaitingForInput,
failure_reason="Prefill bootstrap failed" if failed else None,
)
def _handle_bootstrap_message(self, msg: List[bytes]) -> bool:
if self.deferred_bootstrap is None or not msg:
return False
if msg[0] not in (b"BOOTSTRAP", b"ABORT"):
return False
try:
room = int(msg[1])
endpoint = (msg[2].decode("ascii"), int(msg[3]))
except (IndexError, ValueError, UnicodeDecodeError):
logger.warning("Dropping malformed bootstrap control message")
return True
if msg[0] == b"BOOTSTRAP":
self._notify_bootstrap(
room, self.deferred_bootstrap.register(room, endpoint)
)
return True
# Retain cancellation even if it beats source creation. Let the backend
# process ABORT normally as well, including ACK/draining after allocation.
self.deferred_bootstrap.fail(room)
if room in self.request_status and self.request_status[room] != KVPoll.Success:
self.update_status(room, KVPoll.Failed)
return False
def record_failure(self, bootstrap_room: int, failure_reason: str):
with self.failure_lock:
self.failure_records[bootstrap_room] = failure_reason
@@ -623,7 +686,7 @@ class CommonKVManager(BaseKVManager):
status: KVPoll,
failure_reason: Optional[str] = None,
) -> None:
"""Push of a terminal transfer status to decode endpoints."""
"""Push bootstrap readiness or transfer status to decode endpoints."""
if not targets:
return
parts = self._encode_kv_status_message(
@@ -680,6 +743,10 @@ class CommonKVManager(BaseKVManager):
if targets is None:
targets = self._room_notify_targets(bootstrap_room)
self.update_status(bootstrap_room, status)
if status == KVPoll.Success and self.deferred_bootstrap is not None:
# Success permits decode to reuse the room after true retraction.
# Publish it from sender.clear(), after worker and scheduler cleanup.
return status
self.send_kv_status_message(
targets=targets,
bootstrap_room=bootstrap_room,
@@ -711,13 +778,18 @@ class CommonKVManager(BaseKVManager):
prefill_rank: int,
failure_reason: Optional[str] = None,
) -> None:
"""Decode-side handling of one prefill rank's terminal status."""
"""Decode-side handling of one prefill rank's bootstrap/transfer status."""
if bootstrap_room not in self.request_status:
# The room concluded and was cleared. Recording a failure now would
# leave an entry that a later request reusing this bootstrap_room
# would pick up as its own root cause.
logger.debug("Dropping late status for cleared room %s", bootstrap_room)
return
if status == KVPoll.WaitingForInput and self.defer_decode_allocation:
# The policy currently requires one authoritative prefill rank.
# Do not mix this notification into the transfer-success counter.
self.update_status(bootstrap_room, KVPoll.WaitingForInput)
return
if status == KVPoll.Success:
self.prefill_response_tracker[bootstrap_room].add(prefill_rank)
expected_response_num = self.required_prefill_response_num_table.get(
@@ -891,7 +963,16 @@ class CommonKVManager(BaseKVManager):
stop flag that a blocked recv can never observe, so in that mode poll
with a timeout and return None when it expires. Deployments without
--enable-pd-role-switch keep the original blocking recv and pay nothing.
Deferred allocation consumes endpoint registration here, before backend
metadata handling; ABORT still reaches the backend's drain/ACK handler.
"""
if self.deferred_bootstrap is not None:
def recv_bootstrap():
msg = socket.recv_multipart()
return None if self._handle_bootstrap_message(msg) else msg
return recv_bootstrap
if not self.server_args.enable_pd_role_switch:
return socket.recv_multipart
@@ -1016,8 +1097,18 @@ class CommonKVManager(BaseKVManager):
the rebootstrap ``/generate`` failed rather than reporting a spurious
``AbortReq``.
"""
kv_receiver.abort()
self.record_failure(kv_receiver.bootstrap_room, reason)
with kv_receiver._lifecycle_lock:
if self.defer_decode_allocation and (
not kv_receiver._owns_bootstrap_room
or kv_receiver.conclude_state == KVPoll.Success
or self.request_status.get(kv_receiver.bootstrap_room) == KVPoll.Success
):
# The HTTP response can arrive after KV transfer has succeeded.
# Serialize errors with polling/retirement so an old callback
# cannot leave a failure record for a reused room.
return
kv_receiver.abort()
self.record_failure(kv_receiver.bootstrap_room, reason)
def _run_prefill_recompute(
self, kv_receiver: CommonKVReceiver, prefill_url: str, payload: dict
@@ -1283,6 +1374,10 @@ class CommonKVManager(BaseKVManager):
"prefill_http_port": get_serving().port,
}
payload["decode_allocation_policy"] = (
get_disagg().disaggregation_decode_allocation_policy
)
if envs.SGLANG_RUST_SERVER.get() and self.attn_dp_size > 1:
topology_rows = get_parallel().world_group.all_gather_object(payload)
# Every scheduler contributes a topology row. Only the scheduler
@@ -1359,7 +1454,12 @@ class CommonKVManager(BaseKVManager):
if is_ipv6:
sock.setsockopt(zmq.IPV6, 1)
sock.setsockopt(zmq.RECONNECT_IVL, -1)
sock.setsockopt(zmq.SNDTIMEO, 30000)
sock.setsockopt(
zmq.SNDTIMEO,
envs.SGLANG_DISAGGREGATION_ZMQ_SEND_TIMEOUT.get() * 1000
if self.defer_decode_allocation
else 30000,
)
sock.setsockopt(zmq.LINGER, 0)
sock.setsockopt(zmq.TCP_KEEPALIVE, 1)
sock.setsockopt(zmq.TCP_KEEPALIVE_IDLE, 30)
@@ -1681,6 +1781,19 @@ class CommonKVSender(BaseKVSender):
# inner state
self.curr_idx = 0
self.init_time: Optional[float] = None
self._prefill_complete_time: Optional[float] = None
self._owns_bootstrap_room = True
if mgr.deferred_bootstrap is not None:
room_state = mgr.deferred_bootstrap.open(bootstrap_room, self)
self._owns_bootstrap_room = room_state is not None
mgr.update_status(bootstrap_room, KVPoll.Bootstrapping)
if room_state is None or room_state.failed:
mgr.record_failure(
bootstrap_room, "Duplicate or cancelled bootstrap room"
)
mgr.update_status(bootstrap_room, KVPoll.Failed)
self.conclude_state = KVPoll.Failed
return
if self.kv_mgr.is_dummy_cp_rank:
# Non-authoritative CP ranks are dummy participants.
self.kv_mgr.update_status(self.bootstrap_room, KVPoll.WaitingForInput)
@@ -1733,6 +1846,17 @@ class CommonKVSender(BaseKVSender):
f"CommonKVSender init with num_kv_indices: {num_kv_indices} and aux_index: {aux_index}"
)
def mark_prefill_complete(self) -> None:
if self.kv_mgr.deferred_bootstrap is None:
return
# Give decode admission a fresh budget, excluding prefill queue/compute.
if self._prefill_complete_time is None:
self._prefill_complete_time = time.time()
self.kv_mgr._notify_bootstrap(
self.bootstrap_room,
self.kv_mgr.deferred_bootstrap.complete(self.bootstrap_room, self),
)
def pop_decode_prefix_len(self) -> int:
return self.kv_mgr.req_to_decode_prefix_len.pop(self.bootstrap_room, 0)
@@ -1816,9 +1940,14 @@ class CommonKVSender(BaseKVSender):
pass
def _check_bootstrap_timeout(self) -> Optional[KVPoll]:
if self.init_time is None:
start = (
self._prefill_complete_time
if self.kv_mgr.defer_decode_allocation
else self.init_time
)
if start is None:
return None
elapsed = time.time() - self.init_time
elapsed = time.time() - start
if elapsed < self.kv_mgr.bootstrap_timeout:
return None
logger.warning_once(
@@ -1835,14 +1964,35 @@ class CommonKVSender(BaseKVSender):
return KVPoll.Failed
def clear(self) -> None:
notification = None
if self.kv_mgr.deferred_bootstrap is not None:
if not self._owns_bootstrap_room:
return
notification = self.kv_mgr.deferred_bootstrap.close(
self.bootstrap_room,
self,
success=self.kv_mgr.request_status.get(self.bootstrap_room)
== KVPoll.Success,
)
self._owns_bootstrap_room = False
self.kv_mgr.request_status.pop(self.bootstrap_room, None)
self.kv_mgr.req_to_decode_prefix_len.pop(self.bootstrap_room, None)
self.kv_mgr.transfer_infos.pop(self.bootstrap_room, None)
# A sender can be cleared while a counted transfer is still writing:
# keep the target for the worker, or ACK now if nothing is in flight.
self.kv_mgr._maybe_ack_drained_abort(self.bootstrap_room)
if notification is not None:
endpoint, failed = notification
self.kv_mgr.send_kv_status_message(
targets=[endpoint],
bootstrap_room=self.bootstrap_room,
status=KVPoll.Failed if failed else KVPoll.Success,
failure_reason="Prefill bootstrap failed" if failed else None,
)
def abort(self):
if self.kv_mgr.deferred_bootstrap is not None and not self._owns_bootstrap_room:
return
self.kv_mgr.record_failure(
self.bootstrap_room,
"Aborted by AbortReq.",
@@ -1852,6 +2002,10 @@ class CommonKVSender(BaseKVSender):
class CommonKVReceiver(BaseKVReceiver):
_prefill_wait_start: Optional[float] = None
_owns_bootstrap_room: bool = True
_lifecycle_lock = nullcontext()
_ctx = zmq.Context()
_ctx.set(zmq.MAX_SOCKETS, envs.SGLANG_DISAGGREGATION_ZMQ_MAX_SOCKETS.get())
_socket_cache = {}
@@ -1867,16 +2021,29 @@ class CommonKVReceiver(BaseKVReceiver):
self.bootstrap_room = bootstrap_room
self.bootstrap_addr = bootstrap_addr
self.kv_mgr = mgr
if mgr.defer_decode_allocation:
self._lifecycle_lock = threading.RLock()
self.conclude_state: Optional[KVPoll] = None
self.require_staging: bool = False
self.init_time: Optional[float] = None
self._prefill_wait_start: Optional[float] = None
self.abort_notified: bool = False
self._abort_generation: Optional[int] = None
self._connection_pool_entries: Dict[str, List[Dict]] = {}
self._owns_bootstrap_room = not (
mgr.defer_decode_allocation and bootstrap_room in mgr.request_status
)
if not self._owns_bootstrap_room:
mgr.record_failure(bootstrap_room, "Duplicate decode bootstrap room")
mgr.update_status(bootstrap_room, KVPoll.Failed)
self.conclude_state = KVPoll.Failed
return
self.kv_mgr.addr_to_rooms_tracker[self.bootstrap_addr].add(self.bootstrap_room)
self.kv_mgr.update_status(self.bootstrap_room, KVPoll.Bootstrapping)
def init(self, prefill_dp_rank: int):
if self.conclude_state == KVPoll.Failed:
return
if self.bootstrap_addr not in self.kv_mgr.prefill_info_table:
self.kv_mgr.record_failure(
self.bootstrap_room,
@@ -1888,6 +2055,24 @@ class CommonKVReceiver(BaseKVReceiver):
# Read pre-computed rank mapping from prefill_info (computed in try_ensure_parallel_info)
self.prefill_info = self.kv_mgr.prefill_info_table[self.bootstrap_addr]
info = self.prefill_info
policy = get_disagg().disaggregation_decode_allocation_policy
reason = None
if info.decode_allocation_policy != policy:
reason = (
"Disaggregation decode allocation policy mismatch: "
f"prefill={info.decode_allocation_policy}, decode={policy}. "
"Set --disaggregation-decode-allocation-policy identically on both roles."
)
elif policy == "prefill_complete" and (
info.attn_tp_size != 1 or info.attn_cp_size != 1 or info.pp_size != 1
):
reason = "prefill_complete requires peer attention TP1/CP1/PP1"
if reason is not None:
self.kv_mgr.record_failure(self.bootstrap_room, reason)
self.kv_mgr.update_status(self.bootstrap_room, KVPoll.Failed)
self.conclude_state = KVPoll.Failed
return
self.target_tp_rank = self.prefill_info.target_tp_rank
self.target_tp_ranks = self.prefill_info.target_tp_ranks
self.target_cp_ranks = self.prefill_info.target_cp_ranks
@@ -1911,7 +2096,52 @@ class CommonKVReceiver(BaseKVReceiver):
self._setup_bootstrap_infos()
if self.conclude_state == KVPoll.Failed:
return
self.kv_mgr.update_status(self.bootstrap_room, KVPoll.WaitingForInput)
if policy == "early":
self.kv_mgr.update_status(self.bootstrap_room, KVPoll.WaitingForInput)
return
self._prefill_wait_start = time.time()
try:
for info in self.bootstrap_infos:
sock, lock = self._connect_to_bootstrap_server(info)
with lock:
sock.send_multipart(
[
b"BOOTSTRAP",
str(self.bootstrap_room).encode("ascii"),
self.kv_mgr.local_ip.encode("ascii"),
str(self.kv_mgr.rank_port).encode("ascii"),
]
)
except zmq.ZMQError:
self.abort()
def poll(self) -> KVPoll:
with self._lifecycle_lock:
return self._poll()
def _poll(self) -> KVPoll:
if self.conclude_state is not None:
return self.conclude_state
status = self.kv_mgr.check_status(self.bootstrap_room)
if status in (KVPoll.Success, KVPoll.Failed):
self.conclude_state = status
elif status == KVPoll.Bootstrapping and self._prefill_wait_start is not None:
if time.time() - self._prefill_wait_start >= self.kv_mgr.waiting_timeout:
self.abort()
self.kv_mgr.record_failure(
self.bootstrap_room, "Timed out waiting for prefill completion"
)
return KVPoll.Failed
elif status == KVPoll.WaitingForInput:
# Observe readiness before evaluating the old compute deadline.
# send_metadata starts the transfer timeout using this same clock.
if self._prefill_wait_start is not None:
self._prefill_wait_start = None
self.init_time = time.time()
timeout = self._check_waiting_timeout()
if timeout is not None:
return timeout
return status
def _setup_bootstrap_infos(self):
all_bootstrap_infos = []
@@ -2141,14 +2371,34 @@ class CommonKVReceiver(BaseKVReceiver):
return KVPoll.Failed
def clear(self) -> None:
with self._lifecycle_lock:
self._clear()
def _clear(self) -> None:
if not self._owns_bootstrap_room:
return
if (
self.kv_mgr.defer_decode_allocation
and self.kv_mgr.request_status.get(self.bootstrap_room) != KVPoll.Success
and not self.abort_notified
and getattr(self, "bootstrap_infos", None) is not None
):
# Preserve the recorded root cause; this is cleanup, not a new abort.
self._send_abort_notification()
self.abort_notified = True
self.kv_mgr.request_status.pop(self.bootstrap_room, None)
self.kv_mgr.required_prefill_response_num_table.pop(self.bootstrap_room, None)
self.kv_mgr.prefill_response_tracker.pop(self.bootstrap_room, None)
self.kv_mgr.addr_to_rooms_tracker[self.bootstrap_addr].discard(
self.bootstrap_room
)
if self.kv_mgr.defer_decode_allocation:
self._owns_bootstrap_room = False
def abort(self):
if not self._owns_bootstrap_room:
self.conclude_state = KVPoll.Failed
return
self.kv_mgr.record_failure(
self.bootstrap_room,
"Aborted by AbortReq.",
@@ -2228,6 +2478,7 @@ class CommonKVBootstrapServer(BaseKVBootstrapServer):
self.follow_bootstrap_room: Optional[bool] = None
self.enable_dsa_cache_layer_split: Optional[bool] = None
self.prefill_http_port: Optional[int] = None
self.decode_allocation_policy = "early"
self.prefill_port_table: Dict[
int, Dict[int, Dict[int, Dict[int, PrefillRankInfo]]]
] = {}
@@ -2302,7 +2553,13 @@ class CommonKVBootstrapServer(BaseKVBootstrapServer):
text="DeepSeek-V4.1 DSpark PD layout differs across prefill ranks",
status=400,
)
policy = data.get("decode_allocation_policy", "early")
if self._registered_count and self.decode_allocation_policy != policy:
return web.Response(
text="Inconsistent prefill allocation policies", status=400
)
self.dsv41_spec_layout = dsv41_spec_layout
self.decode_allocation_policy = policy
if self.attn_tp_size is None:
self.attn_tp_size = attn_tp_size
@@ -2402,6 +2659,7 @@ class CommonKVBootstrapServer(BaseKVBootstrapServer):
),
enable_dsa_cache_layer_split=bool(self.enable_dsa_cache_layer_split),
prefill_http_port=self.prefill_http_port,
decode_allocation_policy=self.decode_allocation_policy,
)
payload = dataclasses.asdict(info)
if info.dsv41_spec_layout is None:
+23 -6
View File
@@ -738,9 +738,9 @@ class DecodePreallocQueue(DecodeHiCachePreallocMixin):
``is_rebootstrap`` marks a PD true-retraction request whose prefix KV
must be recomputed by the original prefill worker under the current
weights (rather than resumed from stale CPU KV). It otherwise follows the
same bootstrap-handshake path as a fresh request; the ``/generate``
dispatch happens later, after preallocation and ``send_metadata`` (see
``pop_preallocated``).
same bootstrap-handshake path as a fresh request. Early allocation
dispatches ``/generate`` after ``send_metadata``; deferred allocation
dispatches once the peer is resolved, before waiting for completion.
"""
# See `PrefillBootstrapQueue.add`. A retracted or rebootstrapping
# request owns a host KV backup that `retracted_queue` releases, and by
@@ -779,7 +779,7 @@ class DecodePreallocQueue(DecodeHiCachePreallocMixin):
prefill_dp_rank = self._resolve_prefill_dp_rank(req)
logger.debug(f"prefill_dp_rank: {prefill_dp_rank}")
if prefill_dp_rank is not None:
decode_req.kv_receiver.init(prefill_dp_rank)
self._init_receiver(decode_req, prefill_dp_rank)
return
self.pending_reqs.append(decode_req)
@@ -1018,6 +1018,20 @@ class DecodePreallocQueue(DecodeHiCachePreallocMixin):
return resumed_reqs
def _init_receiver(self, decode_req: DecodeRequest, prefill_dp_rank: int) -> None:
decode_req.kv_receiver.init(prefill_dp_rank)
if (
get_disagg().disaggregation_decode_allocation_policy == "prefill_complete"
and decode_req.kv_receiver.conclude_state != KVPoll.Failed
):
decode_req.req.disagg_prefill_dp_rank = prefill_dp_rank
# Deferred allocation cannot precede the recompute that makes its
# source ready. Keep the existing leader election and error path.
if decode_req.is_rebootstrap:
self.kv_manager.submit_prefill_recompute(
decode_req.kv_receiver, decode_req.req.build_rebootstrap_payload()
)
def _update_handshake_waiters(
self,
rids_to_check: Optional[List[str]] = None,
@@ -1225,7 +1239,7 @@ class DecodePreallocQueue(DecodeHiCachePreallocMixin):
self.pending_reqs = remaining
for decode_req, prefill_dp_rank in resolved:
decode_req.kv_receiver.init(prefill_dp_rank)
self._init_receiver(decode_req, prefill_dp_rank)
def pop_preallocated(
self,
@@ -1747,7 +1761,10 @@ class DecodePreallocQueue(DecodeHiCachePreallocMixin):
state_indices,
**metadata_kwargs,
)
if decode_req.is_rebootstrap:
if (
decode_req.is_rebootstrap
and get_disagg().disaggregation_decode_allocation_policy == "early"
):
self.kv_manager.submit_prefill_recompute(
decode_req.kv_receiver,
decode_req.req.build_rebootstrap_payload(),
@@ -104,6 +104,11 @@ class FakeKVSender(BaseKVSender):
def get_transfer_metric(self) -> KVTransferMetric:
return KVTransferMetric()
def mark_prefill_complete(self) -> None:
# Warmup and health checks use a fake sender even with a real PD manager.
# They have no decode peer to admit or notify.
pass
def init(
self,
num_kv_indices: int,
@@ -2457,7 +2457,6 @@ class MooncakeKVManager(StagingManagerMixin, CommonKVManager):
and self.check_status(kv_chunk.room) == KVPoll.Failed
)
):
self._staging_outstanding.pop(kv_chunk.room, None)
if kv_chunk.room in self.transfer_infos:
self.transfer_infos.pop(kv_chunk.room)
self.req_to_decode_prefix_len.pop(kv_chunk.room, None)
@@ -2468,6 +2467,10 @@ class MooncakeKVManager(StagingManagerMixin, CommonKVManager):
if key[0] == kv_chunk.room:
self._staging_ctx.prefetch_requested.discard(key)
self._staging_ctx.prefetched_rooms.discard(kv_chunk.room)
# Publish worker cleanup last: deferred allocation must not
# acknowledge Success while this worker can still erase
# bookkeeping for a subsequent transfer using the same room.
self._staging_outstanding.pop(kv_chunk.room, None)
except Exception as e:
# NOTE(shangming): Remove this when we make sure the transfer thread is bug-free
@@ -2822,7 +2825,6 @@ class MooncakeKVSender(MooncakeFailureExceptionMixin, CommonKVSender):
pp_rank,
req_has_disagg_prefill_dp_rank,
)
self.conclude_state = None
self.init_time = time.time()
self._init_trace_ctx()
@@ -2864,11 +2866,15 @@ class MooncakeKVSender(MooncakeFailureExceptionMixin, CommonKVSender):
def poll(self) -> KVPoll:
if self.conclude_state is None:
status = self.kv_mgr.check_status(self.bootstrap_room)
# Hold Success until all staging chunks transferred: a deferred
# chunk can still be pending, and concluding now would drop it.
if (
status == KVPoll.Success
and self.kv_mgr._staging_outstanding.get(self.bootstrap_room, 0) > 0
# Hold Success until all staging chunks transferred. With deferred
# allocation also wait for worker cleanup (the zero entry is only
# removed after cleanup), before permitting the room to be reused.
if status == KVPoll.Success and (
self.kv_mgr._staging_outstanding.get(self.bootstrap_room, 0) > 0
or (
self.kv_mgr.defer_decode_allocation
and self.bootstrap_room in self.kv_mgr._staging_outstanding
)
):
return KVPoll.Transferring
if status in (KVPoll.Success, KVPoll.Failed):
@@ -3072,20 +3078,6 @@ class MooncakeKVReceiver(MooncakeFailureExceptionMixin, CommonKVReceiver):
return
self.init_time = time.time()
def poll(self) -> KVPoll:
if self.conclude_state is not None:
return self.conclude_state
status = self.kv_mgr.check_status(self.bootstrap_room)
if status in (KVPoll.Success, KVPoll.Failed):
self.conclude_state = status
elif status == KVPoll.WaitingForInput:
timeout_result = self._check_waiting_timeout()
if timeout_result is not None:
return timeout_result
return status
class MooncakeKVBootstrapServer(CommonKVBootstrapServer):
pass
+21 -3
View File
@@ -513,7 +513,12 @@ class PrefillBootstrapQueue:
failed_reqs.append(req)
elif poll == KVPoll.Bootstrapping:
if (
req.prefill_attempt_count < get_disagg().optimistic_prefill_attempts
(
req.prefill_attempt_count
< get_disagg().optimistic_prefill_attempts
or get_disagg().disaggregation_decode_allocation_policy
== "prefill_complete"
)
and not req.is_retracted # engine paused
and not (
_uses_write_through_cache(self.scheduler.tree_cache)
@@ -929,6 +934,11 @@ class SchedulerDisaggregationPrefillMixin:
self.batch_result_processor.add_sampling_mask_return_values(
i, req, logits_output
)
if (
get_disagg().disaggregation_decode_allocation_policy
== "prefill_complete"
):
req.disagg_kv_sender.mark_prefill_complete()
if not req.pending_bootstrap:
self.send_kv_chunk(req, last_chunk=True)
req.time_stats.set_prefill_transfer_queue_entry_time()
@@ -1261,7 +1271,11 @@ class SchedulerDisaggregationPrefillMixin:
if is_aborted(req):
# bootstrap failed
self.chunked_req = None
elif self.has_bootstrapped_waiting_req():
elif (
get_disagg().disaggregation_decode_allocation_policy
!= "prefill_complete"
and self.has_bootstrapped_waiting_req()
):
# optimistic request yields to waiting requests
self.chunked_req = None
if not self.enable_overlap:
@@ -1586,7 +1600,11 @@ class SchedulerDisaggregationPrefillMixin:
# A fresh lookup budget for the new attempt, as after a retraction.
req.storage_prefetch_retry_attempts = 0
req.storage_prefetch_last_match_len = yielded_prefix_len or None
if req.prefill_attempt_count >= max_attempts:
if (
req.prefill_attempt_count >= max_attempts
and get_disagg().disaggregation_decode_allocation_policy
!= "prefill_complete"
):
logger.info(
f"Req {req.rid} exhausted optimistic prefill attempts "
"falling back to bootstrap queue"