diff --git a/docs/docs/advanced_features/pd_disaggregation.mdx b/docs/docs/advanced_features/pd_disaggregation.mdx index 5ce40a82e4..17fa62441d 100644 --- a/docs/docs/advanced_features/pd_disaggregation.mdx +++ b/docs/docs/advanced_features/pd_disaggregation.mdx @@ -20,6 +20,60 @@ For the design details, please refer to [link](https://docs.google.com/document/ Currently, we support Mooncake and NIXL as the transfer engine. +## Decode allocation policy + +By default, decode reserves each request's destination KV slots before prefill +finishes. Long prefills can hold these slots while decode waits for the KV data. +To delay this reservation, set the following flag on **both prefill and decode**: + +```bash +--disaggregation-decode-allocation-policy prefill_complete +``` + +With `prefill_complete`, prefill computes without waiting for destination slots, +then signals readiness after accepting its final prefill result. Decode reserves +slots and starts the existing KV transfer handshake only after that signal. +This changes per-request KV reservation, not the size or creation of the global +KV pool. The default `early` policy preserves the existing behavior. + +Deferring allocation can help when long prefills compete with active decode +requests for destination KV capacity. It also retains KV on prefill longer and +removes overlap between chunked prefill and KV transfer. Compare both policies +for your workload; this option is not a general throughput improvement. + +This initial implementation requires Mooncake (RDMA or `mooncake_tcp`), the Python +bootstrap registry, and attention TP/CP and pipeline sizes of one. Attention DP +is allowed when each DP rank has attention TP size one. Prefill automatically +resolves `--optimistic-prefill-attempts` from zero to one; configurations that +disable optimistic prefill after model validation are rejected. +Chunked prefill does not yield just because decode has not admitted the request. +Completed requests retain their KV in the existing optimistic-prefill inflight +queue until decode supplies destination indices; readiness alone never starts +a KV write. +After a successful transfer, prefill clears the room's readiness and transfer +state before notifying decode of success. Retraction recovery can then reuse +the same bootstrap room. This notification waits for worker and scheduler +cleanup; failure and cancellation records remain until their timeout expires. +Retries remain eligible for prefill after the optimistic attempt budget is +exhausted and remain subject to the existing cancellation and timeout handling. + +On prefill, `SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT` starts after the final +prefill result, giving decode admission its own budget. On decode, +`SGLANG_DISAGGREGATION_WAITING_TIMEOUT` bounds the wait for prefill completion, +then starts afresh for admission and again when destination indices are sent +for transfer. + +The option does not support staging buffers, runtime PD role switching, PD +multiplexing, encoder disaggregation, decoupled speculation, HiSparse, decode radix cache, decode +HiCache, or non-DSA auxiliary state pools. Prefill HiCache is limited to L2 with +`write_back` and no storage backend. Unsupported configurations fail validation. + +All communicating workers must use the same revision and enable the same policy. This +version of the decode worker rejects policy or peer-layout mismatches before +destination registration. Older decode workers do not enforce this check. Keep +`early` on both sides until every worker has been upgraded, then enable the new +policy on both sides. + ## Profiling in PD Disaggregation Mode When you need to profile prefill or decode workers in PD disaggregation mode, please refer to the [Profile In PD Disaggregation Mode](../developer_guide/benchmark_and_profiling#profile-in-pd-disaggregation-mode) section in the Benchmark and Profiling guide. Due to torch profiler limitations, prefill and decode workers must be profiled separately using dedicated command-line options. diff --git a/docs/docs/advanced_features/server_arguments.mdx b/docs/docs/advanced_features/server_arguments.mdx index d8453e95a9..800f047826 100644 --- a/docs/docs/advanced_features/server_arguments.mdx +++ b/docs/docs/advanced_features/server_arguments.mdx @@ -3007,6 +3007,12 @@ This option does not disable required attention or MoE collectives, including ex None Type: str + + `--disaggregation-decode-allocation-policy` + When to reserve per-request decode KV slots. Set the same policy on prefill and decode. See requirements and tradeoffs. + `early` + early, prefill_complete + `--disaggregation-decode-enable-offload-kvcache` Enable async KV cache offloading on decode server (PD mode). diff --git a/python/sglang/srt/arg_groups/fields/disagg.py b/python/sglang/srt/arg_groups/fields/disagg.py index 89985577dd..aab97a017a 100644 --- a/python/sglang/srt/arg_groups/fields/disagg.py +++ b/python/sglang/srt/arg_groups/fields/disagg.py @@ -56,6 +56,12 @@ class Disagg(msgspec.Struct): Literal["null", "prefill", "decode"], 'Only used for PD disaggregation. "prefill" for prefill-only server, and "decode" for decode-only server. If not specified, it is not PD disaggregated', ] = "null" + disaggregation_decode_allocation_policy: A[ + Literal["early", "prefill_complete"], + "When to reserve per-request decode KV in disaggregated serving. " + "prefill_complete retains KV on prefill until prefill finishes, then " + "admits it on decode. Set the same policy on both roles. Default: early.", + ] = "early" disaggregation_transfer_backend: A[ str, Arg( diff --git a/python/sglang/srt/arg_groups/pd_disaggregation_hook.py b/python/sglang/srt/arg_groups/pd_disaggregation_hook.py index f7d619d2f3..da8125869f 100644 --- a/python/sglang/srt/arg_groups/pd_disaggregation_hook.py +++ b/python/sglang/srt/arg_groups/pd_disaggregation_hook.py @@ -44,6 +44,16 @@ def handle_pd_disaggregation(server_args: ServerArgs) -> None: "with MC_FORCE_TCP=1 (TCP transport, no RDMA)" ) + if cfg.disaggregation_decode_allocation_policy == "prefill_complete": + _validate_prefill_complete(server_args) + if ( + cfg.disaggregation_mode == "prefill" + and cfg.optimistic_prefill_attempts == 0 + ): + declare_resolution( + server_args, "handle_pd_disaggregation", optimistic_prefill_attempts=1 + ) + if cfg.disaggregation_mode == "prefill" and cfg.dcp_size > 1: logger.warning( "DCP on a PD prefill server is supported when prefill and decode " @@ -191,6 +201,52 @@ def handle_pd_disaggregation(server_args: ServerArgs) -> None: ) +def _validate_prefill_complete(server_args: ServerArgs) -> None: + cfg = resolving_view(server_args) + unsupported = [] + if cfg.disaggregation_mode not in ("prefill", "decode"): + unsupported.append("aggregated serving") + if cfg.disaggregation_transfer_backend != "mooncake": + unsupported.append("transfer backends other than Mooncake") + if cfg.pp_size != 1 or cfg.attn_cp_size != 1 or cfg.dcp_size != 1: + unsupported.append("pipeline or attention context parallelism") + attn_dp_size = cfg.dp_size if cfg.enable_dp_attention else 1 + if cfg.tp_size != attn_dp_size: + unsupported.append("attention tensor parallelism greater than one") + if cfg.enable_pdmux or cfg.decoupled_spec_role != "null": + unsupported.append("PD multiplexing or decoupled speculation") + if cfg.enable_pd_role_switch: + unsupported.append("runtime PD role switching") + if cfg.encoder_only or cfg.language_only: + unsupported.append("encoder disaggregation") + if cfg.enable_hisparse: + unsupported.append("HiSparse") + if cfg.disaggregation_decode_host_receive_threshold > 0: + unsupported.append("decode host KV buffering") + if cfg.disaggregation_mode == "decode" and ( + cfg.disaggregation_decode_enable_radix_cache or cfg.enable_hierarchical_cache + ): + unsupported.append("decode radix cache or decode HiCache") + if ( + cfg.disaggregation_mode == "prefill" + and cfg.enable_hierarchical_cache + and ( + cfg.hicache_storage_backend is not None + or cfg.hicache_write_policy != "write_back" + ) + ): + unsupported.append("prefill HiCache other than L2 write_back") + if envs.SGLANG_DISAGG_STAGING_BUFFER.get(): + unsupported.append("staging buffers") + if envs.SGLANG_RUST_SERVER.get(): + unsupported.append("the Rust bootstrap registry") + if unsupported: + raise ValueError( + "--disaggregation-decode-allocation-policy prefill_complete does not " + "yet support " + ", ".join(unsupported) + ) + + def _alias_bootstrap_port_to_api_port(server_args: ServerArgs) -> None: """Rust-server prefill serves the KV bootstrap registry on the api listener itself, so the resolved bootstrap port must BE the api port — every internal diff --git a/python/sglang/srt/arg_groups/serving_hook.py b/python/sglang/srt/arg_groups/serving_hook.py index 452637ee11..5e390960cb 100644 --- a/python/sglang/srt/arg_groups/serving_hook.py +++ b/python/sglang/srt/arg_groups/serving_hook.py @@ -494,6 +494,16 @@ def handle_other_validations(server_args: Any): optimistic_prefill_attempts=0, ) + if ( + cfg.disaggregation_decode_allocation_policy == "prefill_complete" + and cfg.disaggregation_mode == "prefill" + and resolving_view(server_args).optimistic_prefill_attempts <= 0 + ): + raise ValueError( + "prefill_complete requires optimistic prefill, but the resolved " + "model/cache configuration disabled it." + ) + # Handle model inference tensor dump. if cfg.debug_tensor_dump_output_folder is not None: logger.warning( diff --git a/python/sglang/srt/disaggregation/base/conn.py b/python/sglang/srt/disaggregation/base/conn.py index 5e31360193..88b263d208 100644 --- a/python/sglang/srt/disaggregation/base/conn.py +++ b/python/sglang/srt/disaggregation/base/conn.py @@ -182,6 +182,10 @@ class BaseKVSender(ABC): """ ... + def mark_prefill_complete(self) -> None: + """Publish retained final-prefill KV for an enabled allocation policy.""" + raise NotImplementedError + def pop_decode_prefix_len(self) -> int: return 0 diff --git a/python/sglang/srt/disaggregation/common/bootstrap.py b/python/sglang/srt/disaggregation/common/bootstrap.py new file mode 100644 index 0000000000..47ddbf6b5a --- /dev/null +++ b/python/sglang/srt/disaggregation/common/bootstrap.py @@ -0,0 +1,112 @@ +"""Bookkeeping for bootstrap rendezvous before decode has reserved KV. + +The transfer backend still owns the destination-pointer handshake and abort +draining. This table only joins a completed optimistic prefill with decode's +control endpoint. It performs no I/O, including while holding its lock. +""" + +from __future__ import annotations + +import threading +import time +from dataclasses import dataclass + +BootstrapNotification = tuple[tuple[str, int], bool] | None + + +@dataclass +class BootstrapRoom: + owner: object | None = None + endpoint: tuple[str, int] | None = None + ready: bool = False + ready_notified: bool = False + failed: bool = False + expires_at: float = 0.0 + + +class DeferredBootstrap: + def __init__(self, timeout: float): + self.timeout = timeout + self.rooms: dict[int, BootstrapRoom] = {} + self.lock = threading.Lock() + self._next_cleanup = 0.0 + + def _room(self, room: int) -> BootstrapRoom: + now = time.monotonic() + if now >= self._next_cleanup: + # Active senders own their lifetime. Unmatched endpoints and closed + # rooms expire, including cancellation before the sender exists. + self.rooms = { + key: state + for key, state in self.rooms.items() + if state.owner is not None or state.expires_at > now + } + self._next_cleanup = now + min(1.0, self.timeout) + return self.rooms.setdefault(room, BootstrapRoom(expires_at=now + self.timeout)) + + @staticmethod + def _notification(state: BootstrapRoom) -> BootstrapNotification: + if state.endpoint is not None: + if state.failed: + return state.endpoint, True + if state.ready and not state.ready_notified: + # One readiness message per transfer. Decode must consume it + # before publishing destinations, so none can trail Success + # and accidentally admit the next transfer using this room. + state.ready_notified = True + return state.endpoint, False + return None + + def open(self, room: int, owner: object) -> BootstrapRoom | None: + with self.lock: + state = self._room(room) + if state.owner is not None: + return None + state.owner = owner + return state + + def register(self, room: int, endpoint: tuple[str, int]) -> BootstrapNotification: + with self.lock: + state = self._room(room) + if state.endpoint is not None and state.endpoint != endpoint: + # A room identifies one request, not a reusable allocation slot. + # Reject another receiver without replacing the first endpoint. + return endpoint, True + state.endpoint = endpoint + return self._notification(state) + + def complete(self, room: int, owner: object) -> BootstrapNotification: + with self.lock: + state = self.rooms.get(room) + if state is None or state.owner is not owner or state.failed: + return None + state.ready = True + return self._notification(state) + + def fail(self, room: int) -> BootstrapNotification: + with self.lock: + state = self._room(room) + if state.failed: + return None + state.failed = True + return self._notification(state) + + def close( + self, room: int, owner: object, *, success: bool = False + ) -> BootstrapNotification: + with self.lock: + state = self.rooms.get(room) + if state is None or state.owner is not owner: + return None + if success and not state.failed: + # The caller clears transport state before sending Success. + # Only then can decode reuse this room for recomputation. + del self.rooms[room] + return (state.endpoint, False) if state.endpoint is not None else None + state.owner = None + already_failed = state.failed + state.failed = True + state.expires_at = time.monotonic() + self.timeout + # Retain a bounded tombstone: a late endpoint must not wait forever + # or attach to a second request using the same bootstrap room. + return None if already_failed else self._notification(state) diff --git a/python/sglang/srt/disaggregation/common/conn.py b/python/sglang/srt/disaggregation/common/conn.py index 271f9a370d..e4ba54615b 100644 --- a/python/sglang/srt/disaggregation/common/conn.py +++ b/python/sglang/srt/disaggregation/common/conn.py @@ -7,6 +7,7 @@ import logging import threading import time from collections import defaultdict +from contextlib import nullcontext from typing import Dict, List, NamedTuple, Optional, Set, Tuple, Union import msgspec @@ -28,6 +29,10 @@ from sglang.srt.disaggregation.base.conn import ( KVTransferMetric, StateType, ) +from sglang.srt.disaggregation.common.bootstrap import ( + BootstrapNotification, + DeferredBootstrap, +) from sglang.srt.disaggregation.utils import ( DisaggregationMode, filter_kv_indices_for_cp_rank, @@ -107,6 +112,7 @@ class PrefillServerInfo: follow_bootstrap_room: bool enable_dsa_cache_layer_split: bool = False dsv41_spec_layout: Optional[dict] = None + decode_allocation_policy: str = "early" # PD true-retraction rebootstrap: the prefill's HTTP API port. The decode # already knows the prefill host (the bootstrap_addr host), so it can POST @@ -259,6 +265,9 @@ class DeferredAbortAckState(msgspec.Struct): class CommonKVManager(BaseKVManager): + defer_decode_allocation: bool = False + deferred_bootstrap: Optional[DeferredBootstrap] = None + # Wire layout of the prefill->decode terminal status message. The legacy # layout (mooncake, and ascend which inherits it) is three untagged frames # ``[room, status, prefill_rank]``; backends whose control socket also @@ -359,6 +368,20 @@ class CommonKVManager(BaseKVManager): # A poisoned room has a permanently outstanding transfer whose backend # can no longer prove quiescent. A new room lifecycle clears this state. self._deferred_ack_poisoned_rooms: Set[int] = set() + self.defer_decode_allocation = ( + get_disagg().disaggregation_decode_allocation_policy == "prefill_complete" + ) + self.deferred_bootstrap = None + if self.defer_decode_allocation: + if any(state != StateType.DSA for state in args.state_types): + raise ValueError( + "prefill_complete currently supports full-attention and DSA KV pools only" + ) + if disaggregation_mode == DisaggregationMode.PREFILL: + self.deferred_bootstrap = DeferredBootstrap( + envs.SGLANG_DISAGGREGATION_WAITING_TIMEOUT.get() + + envs.SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT.get() + ) if self.disaggregation_mode == DisaggregationMode.PREFILL: # When SGLANG_DISAGGREGATION_ALL_CP_RANKS_TRANSFER is True, all CP ranks @@ -542,6 +565,10 @@ class CommonKVManager(BaseKVManager): return if status == KVPoll.Failed: self.request_status[bootstrap_room] = KVPoll.Failed + if self.deferred_bootstrap is not None: + self._notify_bootstrap( + bootstrap_room, self.deferred_bootstrap.fail(bootstrap_room) + ) return if current == KVPoll.Failed: # Failed is terminal. It also sorts lowest, so the max() below would @@ -549,6 +576,42 @@ class CommonKVManager(BaseKVManager): return self.request_status[bootstrap_room] = max(current, status) + def _notify_bootstrap(self, room: int, notification: BootstrapNotification) -> None: + if notification is None: + return + endpoint, failed = notification + # Use the normal status channel; readiness does not authorize writes. + # No room-table lock is held while sending to a potentially dead peer. + self.send_kv_status_message( + targets=[endpoint], + bootstrap_room=room, + status=KVPoll.Failed if failed else KVPoll.WaitingForInput, + failure_reason="Prefill bootstrap failed" if failed else None, + ) + + def _handle_bootstrap_message(self, msg: List[bytes]) -> bool: + if self.deferred_bootstrap is None or not msg: + return False + if msg[0] not in (b"BOOTSTRAP", b"ABORT"): + return False + try: + room = int(msg[1]) + endpoint = (msg[2].decode("ascii"), int(msg[3])) + except (IndexError, ValueError, UnicodeDecodeError): + logger.warning("Dropping malformed bootstrap control message") + return True + if msg[0] == b"BOOTSTRAP": + self._notify_bootstrap( + room, self.deferred_bootstrap.register(room, endpoint) + ) + return True + # Retain cancellation even if it beats source creation. Let the backend + # process ABORT normally as well, including ACK/draining after allocation. + self.deferred_bootstrap.fail(room) + if room in self.request_status and self.request_status[room] != KVPoll.Success: + self.update_status(room, KVPoll.Failed) + return False + def record_failure(self, bootstrap_room: int, failure_reason: str): with self.failure_lock: self.failure_records[bootstrap_room] = failure_reason @@ -623,7 +686,7 @@ class CommonKVManager(BaseKVManager): status: KVPoll, failure_reason: Optional[str] = None, ) -> None: - """Push of a terminal transfer status to decode endpoints.""" + """Push bootstrap readiness or transfer status to decode endpoints.""" if not targets: return parts = self._encode_kv_status_message( @@ -680,6 +743,10 @@ class CommonKVManager(BaseKVManager): if targets is None: targets = self._room_notify_targets(bootstrap_room) self.update_status(bootstrap_room, status) + if status == KVPoll.Success and self.deferred_bootstrap is not None: + # Success permits decode to reuse the room after true retraction. + # Publish it from sender.clear(), after worker and scheduler cleanup. + return status self.send_kv_status_message( targets=targets, bootstrap_room=bootstrap_room, @@ -711,13 +778,18 @@ class CommonKVManager(BaseKVManager): prefill_rank: int, failure_reason: Optional[str] = None, ) -> None: - """Decode-side handling of one prefill rank's terminal status.""" + """Decode-side handling of one prefill rank's bootstrap/transfer status.""" if bootstrap_room not in self.request_status: # The room concluded and was cleared. Recording a failure now would # leave an entry that a later request reusing this bootstrap_room # would pick up as its own root cause. logger.debug("Dropping late status for cleared room %s", bootstrap_room) return + if status == KVPoll.WaitingForInput and self.defer_decode_allocation: + # The policy currently requires one authoritative prefill rank. + # Do not mix this notification into the transfer-success counter. + self.update_status(bootstrap_room, KVPoll.WaitingForInput) + return if status == KVPoll.Success: self.prefill_response_tracker[bootstrap_room].add(prefill_rank) expected_response_num = self.required_prefill_response_num_table.get( @@ -891,7 +963,16 @@ class CommonKVManager(BaseKVManager): stop flag that a blocked recv can never observe, so in that mode poll with a timeout and return None when it expires. Deployments without --enable-pd-role-switch keep the original blocking recv and pay nothing. + Deferred allocation consumes endpoint registration here, before backend + metadata handling; ABORT still reaches the backend's drain/ACK handler. """ + if self.deferred_bootstrap is not None: + + def recv_bootstrap(): + msg = socket.recv_multipart() + return None if self._handle_bootstrap_message(msg) else msg + + return recv_bootstrap if not self.server_args.enable_pd_role_switch: return socket.recv_multipart @@ -1016,8 +1097,18 @@ class CommonKVManager(BaseKVManager): the rebootstrap ``/generate`` failed rather than reporting a spurious ``AbortReq``. """ - kv_receiver.abort() - self.record_failure(kv_receiver.bootstrap_room, reason) + with kv_receiver._lifecycle_lock: + if self.defer_decode_allocation and ( + not kv_receiver._owns_bootstrap_room + or kv_receiver.conclude_state == KVPoll.Success + or self.request_status.get(kv_receiver.bootstrap_room) == KVPoll.Success + ): + # The HTTP response can arrive after KV transfer has succeeded. + # Serialize errors with polling/retirement so an old callback + # cannot leave a failure record for a reused room. + return + kv_receiver.abort() + self.record_failure(kv_receiver.bootstrap_room, reason) def _run_prefill_recompute( self, kv_receiver: CommonKVReceiver, prefill_url: str, payload: dict @@ -1283,6 +1374,10 @@ class CommonKVManager(BaseKVManager): "prefill_http_port": get_serving().port, } + payload["decode_allocation_policy"] = ( + get_disagg().disaggregation_decode_allocation_policy + ) + if envs.SGLANG_RUST_SERVER.get() and self.attn_dp_size > 1: topology_rows = get_parallel().world_group.all_gather_object(payload) # Every scheduler contributes a topology row. Only the scheduler @@ -1359,7 +1454,12 @@ class CommonKVManager(BaseKVManager): if is_ipv6: sock.setsockopt(zmq.IPV6, 1) sock.setsockopt(zmq.RECONNECT_IVL, -1) - sock.setsockopt(zmq.SNDTIMEO, 30000) + sock.setsockopt( + zmq.SNDTIMEO, + envs.SGLANG_DISAGGREGATION_ZMQ_SEND_TIMEOUT.get() * 1000 + if self.defer_decode_allocation + else 30000, + ) sock.setsockopt(zmq.LINGER, 0) sock.setsockopt(zmq.TCP_KEEPALIVE, 1) sock.setsockopt(zmq.TCP_KEEPALIVE_IDLE, 30) @@ -1681,6 +1781,19 @@ class CommonKVSender(BaseKVSender): # inner state self.curr_idx = 0 self.init_time: Optional[float] = None + self._prefill_complete_time: Optional[float] = None + self._owns_bootstrap_room = True + if mgr.deferred_bootstrap is not None: + room_state = mgr.deferred_bootstrap.open(bootstrap_room, self) + self._owns_bootstrap_room = room_state is not None + mgr.update_status(bootstrap_room, KVPoll.Bootstrapping) + if room_state is None or room_state.failed: + mgr.record_failure( + bootstrap_room, "Duplicate or cancelled bootstrap room" + ) + mgr.update_status(bootstrap_room, KVPoll.Failed) + self.conclude_state = KVPoll.Failed + return if self.kv_mgr.is_dummy_cp_rank: # Non-authoritative CP ranks are dummy participants. self.kv_mgr.update_status(self.bootstrap_room, KVPoll.WaitingForInput) @@ -1733,6 +1846,17 @@ class CommonKVSender(BaseKVSender): f"CommonKVSender init with num_kv_indices: {num_kv_indices} and aux_index: {aux_index}" ) + def mark_prefill_complete(self) -> None: + if self.kv_mgr.deferred_bootstrap is None: + return + # Give decode admission a fresh budget, excluding prefill queue/compute. + if self._prefill_complete_time is None: + self._prefill_complete_time = time.time() + self.kv_mgr._notify_bootstrap( + self.bootstrap_room, + self.kv_mgr.deferred_bootstrap.complete(self.bootstrap_room, self), + ) + def pop_decode_prefix_len(self) -> int: return self.kv_mgr.req_to_decode_prefix_len.pop(self.bootstrap_room, 0) @@ -1816,9 +1940,14 @@ class CommonKVSender(BaseKVSender): pass def _check_bootstrap_timeout(self) -> Optional[KVPoll]: - if self.init_time is None: + start = ( + self._prefill_complete_time + if self.kv_mgr.defer_decode_allocation + else self.init_time + ) + if start is None: return None - elapsed = time.time() - self.init_time + elapsed = time.time() - start if elapsed < self.kv_mgr.bootstrap_timeout: return None logger.warning_once( @@ -1835,14 +1964,35 @@ class CommonKVSender(BaseKVSender): return KVPoll.Failed def clear(self) -> None: + notification = None + if self.kv_mgr.deferred_bootstrap is not None: + if not self._owns_bootstrap_room: + return + notification = self.kv_mgr.deferred_bootstrap.close( + self.bootstrap_room, + self, + success=self.kv_mgr.request_status.get(self.bootstrap_room) + == KVPoll.Success, + ) + self._owns_bootstrap_room = False self.kv_mgr.request_status.pop(self.bootstrap_room, None) self.kv_mgr.req_to_decode_prefix_len.pop(self.bootstrap_room, None) self.kv_mgr.transfer_infos.pop(self.bootstrap_room, None) # A sender can be cleared while a counted transfer is still writing: # keep the target for the worker, or ACK now if nothing is in flight. self.kv_mgr._maybe_ack_drained_abort(self.bootstrap_room) + if notification is not None: + endpoint, failed = notification + self.kv_mgr.send_kv_status_message( + targets=[endpoint], + bootstrap_room=self.bootstrap_room, + status=KVPoll.Failed if failed else KVPoll.Success, + failure_reason="Prefill bootstrap failed" if failed else None, + ) def abort(self): + if self.kv_mgr.deferred_bootstrap is not None and not self._owns_bootstrap_room: + return self.kv_mgr.record_failure( self.bootstrap_room, "Aborted by AbortReq.", @@ -1852,6 +2002,10 @@ class CommonKVSender(BaseKVSender): class CommonKVReceiver(BaseKVReceiver): + _prefill_wait_start: Optional[float] = None + _owns_bootstrap_room: bool = True + _lifecycle_lock = nullcontext() + _ctx = zmq.Context() _ctx.set(zmq.MAX_SOCKETS, envs.SGLANG_DISAGGREGATION_ZMQ_MAX_SOCKETS.get()) _socket_cache = {} @@ -1867,16 +2021,29 @@ class CommonKVReceiver(BaseKVReceiver): self.bootstrap_room = bootstrap_room self.bootstrap_addr = bootstrap_addr self.kv_mgr = mgr + if mgr.defer_decode_allocation: + self._lifecycle_lock = threading.RLock() self.conclude_state: Optional[KVPoll] = None self.require_staging: bool = False self.init_time: Optional[float] = None + self._prefill_wait_start: Optional[float] = None self.abort_notified: bool = False self._abort_generation: Optional[int] = None self._connection_pool_entries: Dict[str, List[Dict]] = {} + self._owns_bootstrap_room = not ( + mgr.defer_decode_allocation and bootstrap_room in mgr.request_status + ) + if not self._owns_bootstrap_room: + mgr.record_failure(bootstrap_room, "Duplicate decode bootstrap room") + mgr.update_status(bootstrap_room, KVPoll.Failed) + self.conclude_state = KVPoll.Failed + return self.kv_mgr.addr_to_rooms_tracker[self.bootstrap_addr].add(self.bootstrap_room) self.kv_mgr.update_status(self.bootstrap_room, KVPoll.Bootstrapping) def init(self, prefill_dp_rank: int): + if self.conclude_state == KVPoll.Failed: + return if self.bootstrap_addr not in self.kv_mgr.prefill_info_table: self.kv_mgr.record_failure( self.bootstrap_room, @@ -1888,6 +2055,24 @@ class CommonKVReceiver(BaseKVReceiver): # Read pre-computed rank mapping from prefill_info (computed in try_ensure_parallel_info) self.prefill_info = self.kv_mgr.prefill_info_table[self.bootstrap_addr] + info = self.prefill_info + policy = get_disagg().disaggregation_decode_allocation_policy + reason = None + if info.decode_allocation_policy != policy: + reason = ( + "Disaggregation decode allocation policy mismatch: " + f"prefill={info.decode_allocation_policy}, decode={policy}. " + "Set --disaggregation-decode-allocation-policy identically on both roles." + ) + elif policy == "prefill_complete" and ( + info.attn_tp_size != 1 or info.attn_cp_size != 1 or info.pp_size != 1 + ): + reason = "prefill_complete requires peer attention TP1/CP1/PP1" + if reason is not None: + self.kv_mgr.record_failure(self.bootstrap_room, reason) + self.kv_mgr.update_status(self.bootstrap_room, KVPoll.Failed) + self.conclude_state = KVPoll.Failed + return self.target_tp_rank = self.prefill_info.target_tp_rank self.target_tp_ranks = self.prefill_info.target_tp_ranks self.target_cp_ranks = self.prefill_info.target_cp_ranks @@ -1911,7 +2096,52 @@ class CommonKVReceiver(BaseKVReceiver): self._setup_bootstrap_infos() if self.conclude_state == KVPoll.Failed: return - self.kv_mgr.update_status(self.bootstrap_room, KVPoll.WaitingForInput) + if policy == "early": + self.kv_mgr.update_status(self.bootstrap_room, KVPoll.WaitingForInput) + return + self._prefill_wait_start = time.time() + try: + for info in self.bootstrap_infos: + sock, lock = self._connect_to_bootstrap_server(info) + with lock: + sock.send_multipart( + [ + b"BOOTSTRAP", + str(self.bootstrap_room).encode("ascii"), + self.kv_mgr.local_ip.encode("ascii"), + str(self.kv_mgr.rank_port).encode("ascii"), + ] + ) + except zmq.ZMQError: + self.abort() + + def poll(self) -> KVPoll: + with self._lifecycle_lock: + return self._poll() + + def _poll(self) -> KVPoll: + if self.conclude_state is not None: + return self.conclude_state + status = self.kv_mgr.check_status(self.bootstrap_room) + if status in (KVPoll.Success, KVPoll.Failed): + self.conclude_state = status + elif status == KVPoll.Bootstrapping and self._prefill_wait_start is not None: + if time.time() - self._prefill_wait_start >= self.kv_mgr.waiting_timeout: + self.abort() + self.kv_mgr.record_failure( + self.bootstrap_room, "Timed out waiting for prefill completion" + ) + return KVPoll.Failed + elif status == KVPoll.WaitingForInput: + # Observe readiness before evaluating the old compute deadline. + # send_metadata starts the transfer timeout using this same clock. + if self._prefill_wait_start is not None: + self._prefill_wait_start = None + self.init_time = time.time() + timeout = self._check_waiting_timeout() + if timeout is not None: + return timeout + return status def _setup_bootstrap_infos(self): all_bootstrap_infos = [] @@ -2141,14 +2371,34 @@ class CommonKVReceiver(BaseKVReceiver): return KVPoll.Failed def clear(self) -> None: + with self._lifecycle_lock: + self._clear() + + def _clear(self) -> None: + if not self._owns_bootstrap_room: + return + if ( + self.kv_mgr.defer_decode_allocation + and self.kv_mgr.request_status.get(self.bootstrap_room) != KVPoll.Success + and not self.abort_notified + and getattr(self, "bootstrap_infos", None) is not None + ): + # Preserve the recorded root cause; this is cleanup, not a new abort. + self._send_abort_notification() + self.abort_notified = True self.kv_mgr.request_status.pop(self.bootstrap_room, None) self.kv_mgr.required_prefill_response_num_table.pop(self.bootstrap_room, None) self.kv_mgr.prefill_response_tracker.pop(self.bootstrap_room, None) self.kv_mgr.addr_to_rooms_tracker[self.bootstrap_addr].discard( self.bootstrap_room ) + if self.kv_mgr.defer_decode_allocation: + self._owns_bootstrap_room = False def abort(self): + if not self._owns_bootstrap_room: + self.conclude_state = KVPoll.Failed + return self.kv_mgr.record_failure( self.bootstrap_room, "Aborted by AbortReq.", @@ -2228,6 +2478,7 @@ class CommonKVBootstrapServer(BaseKVBootstrapServer): self.follow_bootstrap_room: Optional[bool] = None self.enable_dsa_cache_layer_split: Optional[bool] = None self.prefill_http_port: Optional[int] = None + self.decode_allocation_policy = "early" self.prefill_port_table: Dict[ int, Dict[int, Dict[int, Dict[int, PrefillRankInfo]]] ] = {} @@ -2302,7 +2553,13 @@ class CommonKVBootstrapServer(BaseKVBootstrapServer): text="DeepSeek-V4.1 DSpark PD layout differs across prefill ranks", status=400, ) + policy = data.get("decode_allocation_policy", "early") + if self._registered_count and self.decode_allocation_policy != policy: + return web.Response( + text="Inconsistent prefill allocation policies", status=400 + ) self.dsv41_spec_layout = dsv41_spec_layout + self.decode_allocation_policy = policy if self.attn_tp_size is None: self.attn_tp_size = attn_tp_size @@ -2402,6 +2659,7 @@ class CommonKVBootstrapServer(BaseKVBootstrapServer): ), enable_dsa_cache_layer_split=bool(self.enable_dsa_cache_layer_split), prefill_http_port=self.prefill_http_port, + decode_allocation_policy=self.decode_allocation_policy, ) payload = dataclasses.asdict(info) if info.dsv41_spec_layout is None: diff --git a/python/sglang/srt/disaggregation/decode.py b/python/sglang/srt/disaggregation/decode.py index 69c189b1bc..536d4d9b57 100644 --- a/python/sglang/srt/disaggregation/decode.py +++ b/python/sglang/srt/disaggregation/decode.py @@ -738,9 +738,9 @@ class DecodePreallocQueue(DecodeHiCachePreallocMixin): ``is_rebootstrap`` marks a PD true-retraction request whose prefix KV must be recomputed by the original prefill worker under the current weights (rather than resumed from stale CPU KV). It otherwise follows the - same bootstrap-handshake path as a fresh request; the ``/generate`` - dispatch happens later, after preallocation and ``send_metadata`` (see - ``pop_preallocated``). + same bootstrap-handshake path as a fresh request. Early allocation + dispatches ``/generate`` after ``send_metadata``; deferred allocation + dispatches once the peer is resolved, before waiting for completion. """ # See `PrefillBootstrapQueue.add`. A retracted or rebootstrapping # request owns a host KV backup that `retracted_queue` releases, and by @@ -779,7 +779,7 @@ class DecodePreallocQueue(DecodeHiCachePreallocMixin): prefill_dp_rank = self._resolve_prefill_dp_rank(req) logger.debug(f"prefill_dp_rank: {prefill_dp_rank}") if prefill_dp_rank is not None: - decode_req.kv_receiver.init(prefill_dp_rank) + self._init_receiver(decode_req, prefill_dp_rank) return self.pending_reqs.append(decode_req) @@ -1018,6 +1018,20 @@ class DecodePreallocQueue(DecodeHiCachePreallocMixin): return resumed_reqs + def _init_receiver(self, decode_req: DecodeRequest, prefill_dp_rank: int) -> None: + decode_req.kv_receiver.init(prefill_dp_rank) + if ( + get_disagg().disaggregation_decode_allocation_policy == "prefill_complete" + and decode_req.kv_receiver.conclude_state != KVPoll.Failed + ): + decode_req.req.disagg_prefill_dp_rank = prefill_dp_rank + # Deferred allocation cannot precede the recompute that makes its + # source ready. Keep the existing leader election and error path. + if decode_req.is_rebootstrap: + self.kv_manager.submit_prefill_recompute( + decode_req.kv_receiver, decode_req.req.build_rebootstrap_payload() + ) + def _update_handshake_waiters( self, rids_to_check: Optional[List[str]] = None, @@ -1225,7 +1239,7 @@ class DecodePreallocQueue(DecodeHiCachePreallocMixin): self.pending_reqs = remaining for decode_req, prefill_dp_rank in resolved: - decode_req.kv_receiver.init(prefill_dp_rank) + self._init_receiver(decode_req, prefill_dp_rank) def pop_preallocated( self, @@ -1747,7 +1761,10 @@ class DecodePreallocQueue(DecodeHiCachePreallocMixin): state_indices, **metadata_kwargs, ) - if decode_req.is_rebootstrap: + if ( + decode_req.is_rebootstrap + and get_disagg().disaggregation_decode_allocation_policy == "early" + ): self.kv_manager.submit_prefill_recompute( decode_req.kv_receiver, decode_req.req.build_rebootstrap_payload(), diff --git a/python/sglang/srt/disaggregation/fake/conn.py b/python/sglang/srt/disaggregation/fake/conn.py index e25080ac58..a701cb062f 100644 --- a/python/sglang/srt/disaggregation/fake/conn.py +++ b/python/sglang/srt/disaggregation/fake/conn.py @@ -104,6 +104,11 @@ class FakeKVSender(BaseKVSender): def get_transfer_metric(self) -> KVTransferMetric: return KVTransferMetric() + def mark_prefill_complete(self) -> None: + # Warmup and health checks use a fake sender even with a real PD manager. + # They have no decode peer to admit or notify. + pass + def init( self, num_kv_indices: int, diff --git a/python/sglang/srt/disaggregation/mooncake/conn.py b/python/sglang/srt/disaggregation/mooncake/conn.py index 9f1ade96d8..394cc49c3e 100644 --- a/python/sglang/srt/disaggregation/mooncake/conn.py +++ b/python/sglang/srt/disaggregation/mooncake/conn.py @@ -2457,7 +2457,6 @@ class MooncakeKVManager(StagingManagerMixin, CommonKVManager): and self.check_status(kv_chunk.room) == KVPoll.Failed ) ): - self._staging_outstanding.pop(kv_chunk.room, None) if kv_chunk.room in self.transfer_infos: self.transfer_infos.pop(kv_chunk.room) self.req_to_decode_prefix_len.pop(kv_chunk.room, None) @@ -2468,6 +2467,10 @@ class MooncakeKVManager(StagingManagerMixin, CommonKVManager): if key[0] == kv_chunk.room: self._staging_ctx.prefetch_requested.discard(key) self._staging_ctx.prefetched_rooms.discard(kv_chunk.room) + # Publish worker cleanup last: deferred allocation must not + # acknowledge Success while this worker can still erase + # bookkeeping for a subsequent transfer using the same room. + self._staging_outstanding.pop(kv_chunk.room, None) except Exception as e: # NOTE(shangming): Remove this when we make sure the transfer thread is bug-free @@ -2822,7 +2825,6 @@ class MooncakeKVSender(MooncakeFailureExceptionMixin, CommonKVSender): pp_rank, req_has_disagg_prefill_dp_rank, ) - self.conclude_state = None self.init_time = time.time() self._init_trace_ctx() @@ -2864,11 +2866,15 @@ class MooncakeKVSender(MooncakeFailureExceptionMixin, CommonKVSender): def poll(self) -> KVPoll: if self.conclude_state is None: status = self.kv_mgr.check_status(self.bootstrap_room) - # Hold Success until all staging chunks transferred: a deferred - # chunk can still be pending, and concluding now would drop it. - if ( - status == KVPoll.Success - and self.kv_mgr._staging_outstanding.get(self.bootstrap_room, 0) > 0 + # Hold Success until all staging chunks transferred. With deferred + # allocation also wait for worker cleanup (the zero entry is only + # removed after cleanup), before permitting the room to be reused. + if status == KVPoll.Success and ( + self.kv_mgr._staging_outstanding.get(self.bootstrap_room, 0) > 0 + or ( + self.kv_mgr.defer_decode_allocation + and self.bootstrap_room in self.kv_mgr._staging_outstanding + ) ): return KVPoll.Transferring if status in (KVPoll.Success, KVPoll.Failed): @@ -3072,20 +3078,6 @@ class MooncakeKVReceiver(MooncakeFailureExceptionMixin, CommonKVReceiver): return self.init_time = time.time() - def poll(self) -> KVPoll: - if self.conclude_state is not None: - return self.conclude_state - - status = self.kv_mgr.check_status(self.bootstrap_room) - if status in (KVPoll.Success, KVPoll.Failed): - self.conclude_state = status - elif status == KVPoll.WaitingForInput: - timeout_result = self._check_waiting_timeout() - if timeout_result is not None: - return timeout_result - - return status - class MooncakeKVBootstrapServer(CommonKVBootstrapServer): pass diff --git a/python/sglang/srt/disaggregation/prefill.py b/python/sglang/srt/disaggregation/prefill.py index 3f16ffcb52..ef39838cda 100644 --- a/python/sglang/srt/disaggregation/prefill.py +++ b/python/sglang/srt/disaggregation/prefill.py @@ -513,7 +513,12 @@ class PrefillBootstrapQueue: failed_reqs.append(req) elif poll == KVPoll.Bootstrapping: if ( - req.prefill_attempt_count < get_disagg().optimistic_prefill_attempts + ( + req.prefill_attempt_count + < get_disagg().optimistic_prefill_attempts + or get_disagg().disaggregation_decode_allocation_policy + == "prefill_complete" + ) and not req.is_retracted # engine paused and not ( _uses_write_through_cache(self.scheduler.tree_cache) @@ -929,6 +934,11 @@ class SchedulerDisaggregationPrefillMixin: self.batch_result_processor.add_sampling_mask_return_values( i, req, logits_output ) + if ( + get_disagg().disaggregation_decode_allocation_policy + == "prefill_complete" + ): + req.disagg_kv_sender.mark_prefill_complete() if not req.pending_bootstrap: self.send_kv_chunk(req, last_chunk=True) req.time_stats.set_prefill_transfer_queue_entry_time() @@ -1261,7 +1271,11 @@ class SchedulerDisaggregationPrefillMixin: if is_aborted(req): # bootstrap failed self.chunked_req = None - elif self.has_bootstrapped_waiting_req(): + elif ( + get_disagg().disaggregation_decode_allocation_policy + != "prefill_complete" + and self.has_bootstrapped_waiting_req() + ): # optimistic request yields to waiting requests self.chunked_req = None if not self.enable_overlap: @@ -1586,7 +1600,11 @@ class SchedulerDisaggregationPrefillMixin: # A fresh lookup budget for the new attempt, as after a retraction. req.storage_prefetch_retry_attempts = 0 req.storage_prefetch_last_match_len = yielded_prefix_len or None - if req.prefill_attempt_count >= max_attempts: + if ( + req.prefill_attempt_count >= max_attempts + and get_disagg().disaggregation_decode_allocation_policy + != "prefill_complete" + ): logger.info( f"Req {req.rid} exhausted optimistic prefill attempts " "falling back to bootstrap queue" diff --git a/test/registered/e2e/disaggregation/test_prefill_complete_allocation.py b/test/registered/e2e/disaggregation/test_prefill_complete_allocation.py new file mode 100644 index 0000000000..7cd7a0c1b5 --- /dev/null +++ b/test/registered/e2e/disaggregation/test_prefill_complete_allocation.py @@ -0,0 +1,390 @@ +"""Real, multi-chunk PD inference with deferred destination KV allocation.""" + +import json +import math +import time +import unittest +import uuid +from concurrent.futures import ThreadPoolExecutor + +import requests + +from sglang.test.ci.ci_register import register_cuda_ci +from sglang.test.server_fixtures.disaggregation_fixture import ( + PDDisaggregationServerBase, +) + +register_cuda_ci(est_time=480, stage="extra-a", runner_config="2-gpu-large") + + +class _PrefillCompleteServer(PDDisaggregationServerBase): + model = "Qwen/Qwen3-0.6B" + + model_revision = "c1899de289a04d12100db370d81485cdf75e47ca" + + policy = "prefill_complete" + + @classmethod + def setUpClass(cls): + super().setUpClass() + common = [ + "--revision", + cls.model_revision, + "--disaggregation-decode-allocation-policy", + cls.policy, + "--attention-backend", + "flashinfer", + "--mem-fraction-static", + "0.65", + "--max-total-tokens", + "32768", + "--context-length", + "8192", + "--max-running-requests", + "16", + "--cuda-graph-max-bs-decode", + "16", + "--enable-metrics", + ] + cls.extra_prefill_args = common + ["--chunked-prefill-size", "512"] + cls.extra_decode_args = list(common) + cls.launch_all() + + def generate(self, text, *, input_logprob=False): + response = requests.post( + self.lb_url + "/generate", + json={ + "text": text, + "sampling_params": { + "temperature": 0, + "max_new_tokens": 32, + "ignore_eos": True, + }, + "return_logprob": True, + "return_input_logprob": input_logprob, + "logprob_start_len": 0 if input_logprob else -1, + }, + timeout=120, + ) + self.assertEqual(response.status_code, 200, response.text) + result = response.json() + meta = result["meta_info"] + self.assertEqual(meta["completion_tokens"], 32, result) + self.assertEqual(len(meta["output_token_logprobs"]), 32) + self.assertTrue( + all(math.isfinite(item[0]) for item in meta["output_token_logprobs"]) + ) + return result + + def _run_concurrent_requests(self): + prompts = [ + f"Example {i}. " + "Explain how a compiler works. " * 120 for i in range(8) + ] + with ThreadPoolExecutor(max_workers=8) as executor: + results = list(executor.map(self.generate, prompts)) + self.assertEqual(len(results), len(prompts)) + for process in (self.process_prefill, self.process_decode, self.process_lb): + self.assertIsNone(process.poll()) + + def _decode_load(self): + response = requests.get( + self.decode_url + "/v1/loads?include=core,disagg,queues", timeout=10 + ) + response.raise_for_status() + loads = response.json()["loads"] + self.assertEqual(len(loads), 1) + return loads[0] + + def _flush_when_idle(self): + for url in (self.prefill_url, self.decode_url): + deadline = time.monotonic() + 30 + while time.monotonic() < deadline: + response = requests.post(url + "/flush_cache", timeout=10) + if response.status_code == 200: + break + time.sleep(0.1) + else: + self.fail(f"Worker did not release the aborted request: {url}") + + +class TestPrefillCompleteAllocation(_PrefillCompleteServer): + def test_multi_chunk_input_logprobs(self): + prompt = "The capital of France is Paris. " * 300 + first = self.generate(prompt, input_logprob=True) + second = self.generate(prompt, input_logprob=True) + self.assertGreater(first["meta_info"]["prompt_tokens"], 512) + for result in (first, second): + self.assertGreaterEqual( + len(result["meta_info"]["input_token_logprobs"]), + result["meta_info"]["prompt_tokens"] - 1, + ) + self.assertEqual(first["text"], second["text"]) + self.assertEqual( + [x[1] for x in first["meta_info"]["output_token_logprobs"]], + [x[1] for x in second["meta_info"]["output_token_logprobs"]], + ) + + def test_full_prefix_reuse(self): + prompt = ( + "A compiler translates source code into executable instructions. " * 250 + ) + first = self.generate(prompt) + second = self.generate(prompt) + self.assertGreater(second["meta_info"]["prompt_tokens"], 512) + self.assertGreaterEqual( + second["meta_info"]["cached_tokens"], + second["meta_info"]["prompt_tokens"] - 1, + ) + self.assertEqual(first["text"], second["text"]) + self.assertEqual( + [item[1] for item in first["meta_info"]["output_token_logprobs"]], + [item[1] for item in second["meta_info"]["output_token_logprobs"]], + ) + + def test_concurrent_requests_complete(self): + self._run_concurrent_requests() + + def test_decode_abort_before_allocation_releases_prefill(self): + self._flush_when_idle() + rid = "prefill-complete-abort-" + uuid.uuid4().hex + response = requests.post( + self.prefill_url + "/slow_down", + json={"forward_sleep_time": 1.0}, + timeout=10, + ) + response.raise_for_status() + try: + with ThreadPoolExecutor(max_workers=1) as executor: + future = executor.submit( + requests.post, + self.lb_url + "/generate", + json={ + "rid": rid, + "text": rid + " Explain how a compiler works." * 700, + "sampling_params": { + "temperature": 0, + "max_new_tokens": 32, + "ignore_eos": True, + }, + }, + timeout=60, + ) + try: + deadline = time.monotonic() + 15 + while time.monotonic() < deadline: + load = self._decode_load() + if load["disaggregation"]["decode_prealloc_queue_reqs"] > 0: + break + self.assertFalse( + future.done(), + "Request finished before the gate was observed", + ) + time.sleep(0.1) + else: + self.fail("Did not observe the decode readiness wait") + self.assertEqual(load["num_used_tokens"], 0, load) + self.assertEqual(load["queues"]["prealloc_ready"], 0, load) + # Only abort decode: the readiness CANCEL must release the + # source request even though no KV metadata was published. + abort = requests.post( + self.decode_url + "/abort_request", + json={"rid": rid}, + timeout=10, + ) + abort.raise_for_status() + result = future.result(timeout=30) + self.assertIn("abort", result.text.lower(), result.text) + finally: + if not future.done(): + for url in (self.prefill_url, self.decode_url): + requests.post( + url + "/abort_request", json={"rid": rid}, timeout=10 + ) + finally: + requests.post( + self.prefill_url + "/slow_down", + json={"forward_sleep_time": 0.0}, + timeout=10, + ).raise_for_status() + self._flush_when_idle() + self.generate("The capital of France is") + + def test_decode_abort_after_transfer_remains_usable(self): + rid = "prefill-complete-stream-abort-" + uuid.uuid4().hex + with requests.post( + self.lb_url + "/generate", + json={ + "rid": rid, + "text": "Explain how a compiler works in detail.", + "stream": True, + "sampling_params": { + "temperature": 0, + "max_new_tokens": 4096, + "ignore_eos": True, + }, + }, + stream=True, + timeout=60, + ) as response: + response.raise_for_status() + for line in response.iter_lines(): + if line.startswith(b"data: ") and line != b"data: [DONE]": + event = json.loads(line[6:]) + if event["meta_info"]["completion_tokens"] > 0: + break + else: + self.fail("No token arrived before streaming abort") + requests.post( + self.decode_url + "/abort_request", json={"rid": rid}, timeout=10 + ).raise_for_status() + self._flush_when_idle() + self.generate("The capital of France is") + + def test_retract_resume_recomputes_before_reallocation(self): + payload = { + "text": "Explain how a compiler works in detail.", + "return_logprob": True, + "sampling_params": { + "temperature": 0, + "max_new_tokens": 256, + "ignore_eos": True, + }, + } + rid = "prefill-complete-rebootstrap-" + uuid.uuid4().hex + requests.post( + self.decode_url + "/slow_down", + json={"forward_sleep_time": 0.01}, + timeout=10, + ).raise_for_status() + paused = False + try: + with requests.post( + self.lb_url + "/generate", + json={**payload, "rid": rid, "stream": True}, + stream=True, + timeout=90, + ) as response: + response.raise_for_status() + retractions = 0 + final = None + emitted = [] + for line in response.iter_lines(): + if not line.startswith(b"data: ") or line == b"data: [DONE]": + continue + final = json.loads(line[6:]) + self.assertNotIn("error", final) + meta = final["meta_info"] + output = [ + (logprob, token_id) + for logprob, token_id, *_ in meta["output_token_logprobs"] + ] + self.assertEqual(len(output), meta["completion_tokens"]) + # Recompute may change future token choices, but must not + # rewrite tokens or logprobs already emitted by this request. + self.assertEqual(output[: len(emitted)], emitted) + emitted = output + if ( + retractions < 3 + and final["meta_info"]["completion_tokens"] > retractions * 32 + ): + requests.post( + self.decode_url + "/pause_generation", + json={"mode": "retract"}, + timeout=15, + ).raise_for_status() + paused = True + # Retract pause frees destination KV and stages a true + # rebootstrap; flush must succeed before resuming it. + requests.post( + self.decode_url + "/flush_cache", timeout=10 + ).raise_for_status() + requests.post( + self.decode_url + "/continue_generation", + json={"torch_empty_cache": False}, + timeout=15, + ).raise_for_status() + paused = False + retractions += 1 + self.assertIsNotNone(final) + self.assertEqual(retractions, 3) + self.assertGreaterEqual(final["meta_info"]["num_retractions"], 3, final) + self.assertEqual(final["meta_info"]["completion_tokens"], 256, final) + self.assertTrue(all(math.isfinite(logprob) for logprob, _ in emitted)) + finally: + if paused: + requests.post( + self.decode_url + "/continue_generation", json={}, timeout=15 + ) + for url in (self.prefill_url, self.decode_url): + requests.post(url + "/abort_request", json={"rid": rid}, timeout=10) + requests.post( + self.decode_url + "/slow_down", + json={"forward_sleep_time": 0.0}, + timeout=10, + ).raise_for_status() + self._flush_when_idle() + self.generate("The capital of France is") + + +class TestPrefillCompleteRetry(_PrefillCompleteServer): + extra_prefill_env = {"SGLANG_TEST_FORCE_OPTIMISTIC_PREFILL_RETRY_PROB": "1"} + + def _retry_count(self): + response = requests.get(self.prefill_url + "/metrics", timeout=10) + response.raise_for_status() + return sum( + float(line.split()[-1]) + for line in response.text.splitlines() + if line.startswith("sglang:num_prefill_retries_total{") + ) + + def test_retry_after_attempt_budget_still_completes(self): + # The policy resolves to one optimistic attempt. Force that first + # attempt to yield, then verify real retries and eventual completion. + before = self._retry_count() + self._run_concurrent_requests() + self.assertGreaterEqual(self._retry_count() - before, 8) + self._flush_when_idle() + + +class TestPrefillCompleteTimeout(_PrefillCompleteServer): + extra_decode_env = {"SGLANG_DISAGGREGATION_WAITING_TIMEOUT": "3"} + + def test_readiness_timeout_releases_both_sides(self): + self._flush_when_idle() + requests.post( + self.prefill_url + "/slow_down", + json={"forward_sleep_time": 1.0}, + timeout=10, + ).raise_for_status() + start = time.monotonic() + try: + response = requests.post( + self.lb_url + "/generate", + json={ + "rid": "prefill-complete-timeout-" + uuid.uuid4().hex, + "text": "Explain how a compiler works. " * 700, + "sampling_params": { + "temperature": 0, + "max_new_tokens": 32, + "ignore_eos": True, + }, + }, + timeout=30, + ) + self.assertIn("Timed out waiting for prefill completion", response.text) + self.assertLess(time.monotonic() - start, 15) + finally: + requests.post( + self.prefill_url + "/slow_down", + json={"forward_sleep_time": 0.0}, + timeout=10, + ).raise_for_status() + # Do not explicitly abort either side: timeout handling must release + # the source as well as the destination before cache flush can succeed. + self._flush_when_idle() + self.generate("The capital of France is") + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/unit/disaggregation/test_admission_abort_not_enqueued.py b/test/registered/unit/disaggregation/test_admission_abort_not_enqueued.py index b8b7bdd981..d1d74837d7 100644 --- a/test/registered/unit/disaggregation/test_admission_abort_not_enqueued.py +++ b/test/registered/unit/disaggregation/test_admission_abort_not_enqueued.py @@ -22,7 +22,7 @@ them by then -- so it gates on its own `is_retracted` / `is_rebootstrap` flags. import unittest from array import array from http import HTTPStatus -from types import SimpleNamespace +from types import MethodType, SimpleNamespace from unittest.mock import MagicMock, patch from sglang.srt.disaggregation.decode import DecodePreallocQueue @@ -95,6 +95,8 @@ def _decode_queue(sched): ), _resolve_prefill_dp_rank=MagicMock(return_value=0), ) + # Exercise the production initializer reached by the cache-hit fast path. + q._init_receiver = MethodType(DecodePreallocQueue._init_receiver, q) return q @@ -212,6 +214,10 @@ class TestAdmissionAbortNotEnqueued(CustomTestCase): self.assertEqual(q.retracted_queue, [req]) else: q._create_receiver_and_enqueue.assert_called_once() + receiver = ( + q._create_receiver_and_enqueue.return_value.kv_receiver + ) + receiver.init.assert_called_once_with(0) def test_preempted_reentry_is_not_retired(self): """Preemption requeues through a bare `_add_request_to_queue`. @@ -243,6 +249,8 @@ class TestAdmissionAbortNotEnqueued(CustomTestCase): q = _decode_queue(sched) _admit_decode(q, req) q._create_receiver_and_enqueue.assert_called_once() + receiver = q._create_receiver_and_enqueue.return_value.kv_receiver + receiver.init.assert_called_once_with(0) sched.retire_unadmitted_request.assert_not_called() def test_valid_request_still_admitted(self): @@ -258,6 +266,8 @@ class TestAdmissionAbortNotEnqueued(CustomTestCase): q = _decode_queue(sched) _admit_decode(q, req) q._create_receiver_and_enqueue.assert_called_once() + receiver = q._create_receiver_and_enqueue.return_value.kv_receiver + receiver.init.assert_called_once_with(0) sched.retire_unadmitted_request.assert_not_called() def test_null_mode_is_deliberately_untouched(self): diff --git a/test/registered/unit/disaggregation/test_deferred_bootstrap.py b/test/registered/unit/disaggregation/test_deferred_bootstrap.py new file mode 100644 index 0000000000..52282edebf --- /dev/null +++ b/test/registered/unit/disaggregation/test_deferred_bootstrap.py @@ -0,0 +1,158 @@ +"""Lifecycle tests for the shared pre-allocation bootstrap rendezvous.""" + +import itertools +import unittest +from unittest.mock import patch + +from sglang.srt.disaggregation.common.bootstrap import DeferredBootstrap +from sglang.test.ci.ci_register import register_cpu_ci + +register_cpu_ci(est_time=1, suite="base-a-test-cpu") + + +class TestDeferredBootstrap(unittest.TestCase): + def setUp(self): + self.table = DeferredBootstrap(timeout=10) + self.owner = object() + self.endpoint = ("decode", 1234) + + def test_either_arrival_order_and_no_completion_before_source(self): + for order in ( + ("open", "register", "complete"), + ("register", "open", "complete"), + ("open", "complete", "register"), + ): + with self.subTest(order=order): + table = DeferredBootstrap(10) + result = None + for action in order: + if action == "open": + state = table.open(1, self.owner) + self.assertFalse(state.failed) + elif action == "register": + result = table.register(1, self.endpoint) + else: + result = table.complete(1, self.owner) + self.assertEqual(result, (self.endpoint, False)) + + def test_cancel_at_every_point_prevents_later_ready(self): + for order in itertools.permutations(("open", "register", "cancel")): + with self.subTest(order=order): + table = DeferredBootstrap(10) + notifications = [] + for action in order: + if action == "open": + table.open(1, self.owner) + elif action == "register": + notifications.append(table.register(1, self.endpoint)) + else: + notifications.append(table.fail(1)) + self.assertTrue(table.rooms[1].failed) + self.assertIsNone(table.complete(1, self.owner)) + self.assertIn((self.endpoint, True), notifications) + + def test_duplicate_source_cannot_replace_owner(self): + self.table.open(1, self.owner) + other = object() + self.assertIsNone(self.table.open(1, other)) + self.assertIsNone(self.table.complete(1, other)) + self.assertIsNone(self.table.close(1, other)) + self.assertIs(self.table.rooms[1].owner, self.owner) + + def test_endpoint_registration_is_idempotent_not_replaceable(self): + self.table.open(1, self.owner) + self.table.complete(1, self.owner) + self.assertEqual(self.table.register(1, self.endpoint), (self.endpoint, False)) + self.assertIsNone(self.table.register(1, self.endpoint)) + self.assertIsNone(self.table.complete(1, self.owner)) + other = ("other", 9999) + self.assertEqual(self.table.register(1, other), (other, True)) + self.assertEqual(self.table.rooms[1].endpoint, self.endpoint) + + def test_clear_keeps_failure_for_late_receiver(self): + self.table.open(1, self.owner) + self.table.close(1, self.owner) + self.assertEqual(self.table.register(1, self.endpoint), (self.endpoint, True)) + self.assertIsNone(self.table.complete(1, self.owner)) + + def test_success_resets_room_for_either_arrival_order(self): + for receiver_first in (True, False): + with self.subTest(receiver_first=receiver_first): + self.table.open(1, self.owner) + self.table.register(1, self.endpoint) + self.table.complete(1, self.owner) + self.assertEqual( + self.table.close(1, self.owner, success=True), + (self.endpoint, False), + ) + self.assertNotIn(1, self.table.rooms) + new_owner = object() + if receiver_first: + self.assertIsNone(self.table.register(1, self.endpoint)) + state = self.table.open(1, new_owner) + self.assertFalse(state.ready) + self.assertFalse(state.failed) + if not receiver_first: + self.assertIsNone(self.table.register(1, self.endpoint)) + # A repeated cleanup from the retired sender cannot close this one. + self.assertIsNone(self.table.close(1, self.owner, success=True)) + self.assertEqual( + self.table.complete(1, new_owner), (self.endpoint, False) + ) + self.table.close(1, new_owner, success=True) + + def test_success_cannot_reset_cancelled_room(self): + self.table.open(1, self.owner) + self.table.fail(1) + self.table.close(1, self.owner, success=True) + self.assertEqual(self.table.register(1, self.endpoint), (self.endpoint, True)) + + def test_expire_unmatched_and_terminal_rooms_not_active_sources(self): + with patch( + "sglang.srt.disaggregation.common.bootstrap.time.monotonic", return_value=0 + ): + self.table.open(1, self.owner) + self.table.register(2, self.endpoint) + self.table.fail(3) + self.table.open(4, object()) + owner = self.table.rooms[4].owner + self.table.close(4, owner) + with patch( + "sglang.srt.disaggregation.common.bootstrap.time.monotonic", return_value=11 + ): + self.table.register(5, self.endpoint) + self.assertEqual(set(self.table.rooms), {1, 5}) + self.assertIs(self.table.rooms[1].owner, self.owner) + + def test_cancelled_source_owns_cleanup(self): + self.table.fail(1) + state = self.table.open(1, self.owner) + self.assertTrue(state.failed) + self.assertIs(state.owner, self.owner) + self.table.close(1, self.owner) + self.assertIsNone(state.owner) + + def test_abort_then_clear_does_not_send_duplicate_failures(self): + self.table.open(1, self.owner) + self.table.register(1, self.endpoint) + self.assertEqual(self.table.fail(1), (self.endpoint, True)) + self.assertIsNone(self.table.fail(1)) + self.assertIsNone(self.table.close(1, self.owner)) + # A later endpoint must still learn that the source is gone. + self.assertEqual(self.table.register(1, self.endpoint), (self.endpoint, True)) + + def test_no_lock_escapes_any_operation(self): + for operation in ( + lambda: self.table.open(1, self.owner), + lambda: self.table.register(1, self.endpoint), + lambda: self.table.complete(1, self.owner), + lambda: self.table.fail(1), + lambda: self.table.close(1, self.owner), + ): + operation() + self.assertTrue(self.table.lock.acquire(blocking=False)) + self.table.lock.release() + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/unit/disaggregation/test_nixl_sender_failure_cleanup.py b/test/registered/unit/disaggregation/test_nixl_sender_failure_cleanup.py index 91d6a33b5b..c0bbf20642 100644 --- a/test/registered/unit/disaggregation/test_nixl_sender_failure_cleanup.py +++ b/test/registered/unit/disaggregation/test_nixl_sender_failure_cleanup.py @@ -28,6 +28,7 @@ class TestNixlSenderFailureCleanup(unittest.TestCase): # happening, spy on clear() here and test its contents where clear() # is tested, or build a real manager instance. sender.kv_mgr = SimpleNamespace( + deferred_bootstrap=None, enable_staging=True, _staging_ctx=staging_ctx, request_status={room: object()}, diff --git a/test/registered/unit/disaggregation/test_pd_role_switch.py b/test/registered/unit/disaggregation/test_pd_role_switch.py index adea135539..7114d5facc 100644 --- a/test/registered/unit/disaggregation/test_pd_role_switch.py +++ b/test/registered/unit/disaggregation/test_pd_role_switch.py @@ -317,7 +317,8 @@ class TestPdRoleSwitchStartupValidation(unittest.TestCase): speculative_algorithm=None, ) base.update(kw) - return SimpleNamespace(**base) + # Keep the production defaults for fields read by the shared PD hook. + return ServerArgs(model_path="dummy", **base) def _run(self, sa): from sglang.srt.arg_groups.pd_disaggregation_hook import ( diff --git a/test/registered/unit/disaggregation/test_prefill_abort_result_cleanup.py b/test/registered/unit/disaggregation/test_prefill_abort_result_cleanup.py index 8152440032..5ccfef15fa 100644 --- a/test/registered/unit/disaggregation/test_prefill_abort_result_cleanup.py +++ b/test/registered/unit/disaggregation/test_prefill_abort_result_cleanup.py @@ -17,6 +17,7 @@ from sglang.srt.mem_cache.base_prefix_cache import ( ) from sglang.srt.runtime_context import get_context from sglang.test.ci.ci_register import register_cpu_ci +from sglang.test.test_utils import CustomTestCase register_cpu_ci(est_time=1, suite="base-a-test-cpu") @@ -268,5 +269,74 @@ def test_sampling_mask_abort_preserves_error_and_releases_once( scheduler.send_kv_chunk.assert_not_called() +class TestPrefillCompleteResult(CustomTestCase): + def setUp(self): + super().setUp() + override = get_context().override_server_args( + disaggregation_decode_allocation_policy="prefill_complete" + ) + override.install() + self.addCleanup(override.restore) + + @patch("sglang.srt.disaggregation.prefill.checkpoint_kv_cache") + def test_ready_observes_accepted_token_and_speculative_sidecars(self, cache): + scheduler = _Scheduler() + scheduler.spec_algorithm = SimpleNamespace(is_eagle=lambda: True) + req = _Req(inflight_middle_chunks=0) + req.to_finish = None + req.time_stats.set_prefill_transfer_queue_entry_time = Mock() + draft = SimpleNamespace( + hidden_states=torch.tensor([[1.0, 2.0]]), + topk_p=torch.tensor([[0.75, 0.25]]), + topk_index=torch.tensor([[3, 4]]), + dsa_topk_indices=torch.tensor([[5, 6]]), + ) + batch = _batch(req) + batch.spec_info = draft + result = GenerationBatchResult( + next_token_ids=torch.tensor([11]), next_draft_input=draft + ) + observed = [] + + def ready(): + observed.append( + ( + list(req.output_ids), + req.hidden_states_tensor.tolist(), + req.output_dsa_topk_indices.tolist(), + req in scheduler.disagg_prefill_inflight_queue, + ) + ) + + req.disagg_kv_sender.mark_prefill_complete.side_effect = ready + scheduler.process_batch_result_disagg_prefill(batch, result) + self.assertEqual(observed, [([11], [1.0, 2.0], [5, 6], True)]) + self.assertTrue(req.pending_bootstrap) + scheduler.send_kv_chunk.assert_not_called() + + @patch("sglang.srt.disaggregation.prefill.release_kv_cache", side_effect=_free_req) + def test_aborted_final_result_never_publishes_ready(self, release): + scheduler = _Scheduler() + req = _Req(inflight_middle_chunks=0) + scheduler.process_batch_result_disagg_prefill(_batch(req), _result()) + self.assertTrue(req.finished()) + self.assertFalse(req.kv.holds_kv) + self.assertEqual(scheduler.disagg_prefill_inflight_queue, []) + req.disagg_kv_sender.mark_prefill_complete.assert_not_called() + + @patch("sglang.srt.disaggregation.prefill.should_force_retry", return_value=True) + def test_discarded_attempt_never_publishes_ready(self, force_retry): + scheduler = _Scheduler() + req = _Req(inflight_middle_chunks=0) + req.to_finish = None + retried = [] + scheduler.optimistic_release_and_requeue = lambda r: retried.append(r) + scheduler.process_batch_result_disagg_prefill(_batch(req), _result()) + self.assertEqual(retried, [req]) + self.assertEqual(req.output_ids, []) + self.assertEqual(scheduler.disagg_prefill_inflight_queue, []) + req.disagg_kv_sender.mark_prefill_complete.assert_not_called() + + if __name__ == "__main__": raise SystemExit(pytest.main([__file__, "-v"])) diff --git a/test/registered/unit/disaggregation/test_prefill_complete_integration.py b/test/registered/unit/disaggregation/test_prefill_complete_integration.py new file mode 100644 index 0000000000..284fc93dcd --- /dev/null +++ b/test/registered/unit/disaggregation/test_prefill_complete_integration.py @@ -0,0 +1,917 @@ +"""Exercise the allocation policy through production transport and queue methods.""" + +import threading +import time +import unittest +from collections import defaultdict, deque +from concurrent.futures import ThreadPoolExecutor, TimeoutError +from types import MethodType, SimpleNamespace +from unittest.mock import MagicMock, Mock, patch + +from sglang.srt.arg_groups.overrides import resolving_view +from sglang.srt.arg_groups.pd_disaggregation_hook import handle_pd_disaggregation +from sglang.srt.disaggregation.base import KVPoll +from sglang.srt.disaggregation.common.bootstrap import DeferredBootstrap +from sglang.srt.disaggregation.common.conn import CommonKVReceiver, PrefillServerInfo +from sglang.srt.disaggregation.decode import DecodePreallocQueue, DecodeRequest +from sglang.srt.disaggregation.fake.conn import FakeKVSender +from sglang.srt.disaggregation.mooncake.conn import ( + MooncakeKVManager, + MooncakeKVReceiver, + MooncakeKVSender, +) +from sglang.srt.disaggregation.prefill import SchedulerDisaggregationPrefillMixin +from sglang.srt.runtime_context import get_context, publish, reset_context +from sglang.srt.server_args import ServerArgs +from sglang.test.ci.ci_register import register_cpu_ci +from sglang.test.test_utils import CustomTestCase + +register_cpu_ci(est_time=8, suite="base-a-test-cpu") + + +class TestAllocationConfiguration(CustomTestCase): + def test_policy_owns_optimistic_prefill_dependency(self): + args = ServerArgs( + model_path="dummy", + disaggregation_mode="prefill", + disaggregation_decode_allocation_policy="prefill_complete", + ) + handle_pd_disaggregation(args) + self.assertEqual(resolving_view(args).optimistic_prefill_attempts, 1) + # Resolution must not rewrite the raw user input. + self.assertEqual(args.optimistic_prefill_attempts, 0) + + def test_early_does_not_enable_optimistic_prefill(self): + args = ServerArgs(model_path="dummy", disaggregation_mode="prefill") + handle_pd_disaggregation(args) + self.assertEqual(resolving_view(args).optimistic_prefill_attempts, 0) + + def test_dp_attention_topology_is_supported(self): + for size in (1, 4, 8): + with self.subTest(size=size): + args = ServerArgs( + model_path="dummy", + disaggregation_mode="prefill", + disaggregation_decode_allocation_policy="prefill_complete", + tp_size=size, + dp_size=size, + enable_dp_attention=size > 1, + ) + handle_pd_disaggregation(args) + self.assertEqual(resolving_view(args).optimistic_prefill_attempts, 1) + + def test_unsupported_modes_fail_explicitly(self): + combinations = [ + {"disaggregation_mode": "null"}, + {"disaggregation_transfer_backend": "nixl"}, + {"tp_size": 2}, + {"pp_size": 2}, + {"attn_cp_size": 2}, + {"dcp_size": 2}, + {"enable_hisparse": True}, + { + "disaggregation_mode": "decode", + "disaggregation_decode_host_receive_threshold": 0.5, + }, + {"enable_pdmux": True}, + {"enable_pd_role_switch": True}, + {"language_only": True}, + { + "enable_hierarchical_cache": True, + "hicache_write_policy": "write_through", + }, + { + "disaggregation_mode": "decode", + "disaggregation_decode_enable_radix_cache": True, + }, + ] + for changed in combinations: + with self.subTest(changed=changed): + values = dict( + model_path="dummy", + disaggregation_mode="prefill", + disaggregation_decode_allocation_policy="prefill_complete", + ) + values.update(changed) + with self.assertRaisesRegex(ValueError, "prefill_complete"): + handle_pd_disaggregation(ServerArgs(**values)) + + +class TestAllocationIntegration(CustomTestCase): + def setUp(self): + super().setUp() + reset_context() + self.addCleanup(reset_context) + publish(ServerArgs(model_path="dummy"), role="tokenizer") + override = get_context().override_server_args( + disaggregation_decode_allocation_policy="prefill_complete" + ) + override.install() + self.addCleanup(override.restore) + self.wire = deque() + self.prefill = self.manager("prefill", 1) + self.decode = self.manager("decode", 2) + self.endpoints = {("prefill", 1): self.prefill, ("decode", 2): self.decode} + + def manager(self, ip, port): + mgr = MooncakeKVManager.__new__(MooncakeKVManager) + mgr.local_ip, mgr.rank_port = ip, port + mgr.request_status = {} + mgr.failure_lock = threading.Lock() + mgr.failure_records = {} + mgr.waiting_timeout = 5 + mgr.bootstrap_timeout = 3 + mgr.req_to_decode_prefix_len = {} + mgr.transfer_infos = {} + mgr._deferred_ack_targets = {} + mgr._deferred_ack_poisoned_rooms = set() + mgr.required_prefill_response_num_table = {} + mgr.prefill_response_tracker = defaultdict(set) + mgr.addr_to_rooms_tracker = defaultdict(set) + mgr.defer_decode_allocation = True + mgr.deferred_bootstrap = DeferredBootstrap(8) if ip == "prefill" else None + mgr.enable_trace = False + mgr.enable_staging = False + mgr.is_dummy_cp_rank = False + mgr._staging_outstanding = {} + mgr._prefill_unique_rank = Mock(return_value=0) + mgr.get_session_id = Mock(return_value="decode-session") + mgr._send_multipart_locked = lambda endpoint, msg, **kw: self.wire.append( + (endpoint, msg) + ) + return mgr + + def flush(self): + while self.wire: + endpoint, msg = self.wire.popleft() + if endpoint == "tcp://prefill:1": + self.prefill._handle_bootstrap_message(msg) + else: + self.assertEqual(endpoint, "tcp://decode:2") + room, status, rank, reason = self.decode.parse_kv_status_message(msg) + self.decode.apply_prefill_status( + bootstrap_room=room, + status=status, + prefill_rank=rank, + failure_reason=reason, + ) + + def sender(self, room=1): + return MooncakeKVSender(self.prefill, "prefill:8998", room, [0], 0) + + def receiver(self, room=1, policy="prefill_complete"): + receiver = MooncakeKVReceiver(self.decode, "prefill:8998", room) + receiver.bootstrap_infos = [{"rank_ip": "prefill", "rank_port": 1}] + self.decode.prefill_info_table = { + receiver.bootstrap_addr: PrefillServerInfo( + attn_tp_size=1, + attn_cp_size=1, + dp_size=1, + pp_size=1, + page_size=1, + kv_cache_dtype=None, + follow_bootstrap_room=True, + decode_allocation_policy=policy, + target_tp_rank=0, + target_tp_ranks=[0], + target_cp_ranks=[0], + target_pp_ranks=[0], + required_dst_info_num=1, + required_prefill_response_num=1, + ) + } + sock = SimpleNamespace( + send_multipart=lambda msg: self.wire.append(("tcp://prefill:1", msg)) + ) + receiver._connect_to_bootstrap_server = Mock( + return_value=(sock, threading.Lock()) + ) + with patch.object(receiver, "_setup_bootstrap_infos"): + receiver.init(0) + return receiver + + def test_receiver_hides_allocatable_status_until_source_finishes(self): + sender, receiver = self.sender(), self.receiver() + self.assertEqual(receiver.poll(), KVPoll.Bootstrapping) + self.flush() + self.assertEqual(receiver.poll(), KVPoll.Bootstrapping) + sender.mark_prefill_complete() + self.flush() + self.assertEqual(receiver.poll(), KVPoll.WaitingForInput) + + def test_early_receiver_stays_allocatable_without_control_registration(self): + override = get_context().override_server_args( + disaggregation_decode_allocation_policy="early" + ) + override.install() + self.addCleanup(override.restore) + self.decode.defer_decode_allocation = False + receiver = self.receiver(policy="early") + self.assertEqual(receiver.poll(), KVPoll.WaitingForInput) + self.assertIsNone(receiver.init_time) + self.assertEqual(list(self.wire), []) + + def test_shared_receive_wrapper_preserves_native_abort_and_metadata(self): + sender = self.sender() + bootstrap = [b"BOOTSTRAP", b"1", b"decode", b"2"] + abort = [b"ABORT", b"1", b"decode", b"2"] + metadata = [b"1", b"decode", b"2", b"session", b"indices"] + sock = Mock(recv_multipart=Mock(side_effect=[bootstrap, abort, metadata])) + recv = self.prefill._make_worker_recv(sock) + self.assertIsNone(recv()) + self.assertEqual(recv(), abort) + self.assertEqual(sender.poll(), KVPoll.Failed) + self.assertEqual(recv(), metadata) + + def test_repeated_transfers_reuse_room_after_success_cleanup(self): + retired = [] + for receiver_first in (True, False, True): + with self.subTest(receiver_first=receiver_first, iteration=len(retired)): + if receiver_first: + receiver = self.receiver() + self.flush() + sender = self.sender() + else: + sender, receiver = self.sender(), self.receiver() + self.flush() + self.assertEqual(receiver.poll(), KVPoll.Bootstrapping) + for old_sender, old_receiver in retired: + old_sender.clear() + old_sender.abort() + old_receiver.clear() + self.decode._fail_prefill_recompute(old_receiver, "late HTTP error") + self.assertEqual(sender.poll(), KVPoll.Bootstrapping) + self.assertNotIn(1, self.decode.failure_records) + sender.mark_prefill_complete() + self.flush() + self.assertEqual(receiver.poll(), KVPoll.WaitingForInput) + # Duplicate control registration must not enqueue another ready + # message that could outlive the transfer's completion. + self.prefill._handle_bootstrap_message( + [b"BOOTSTRAP", b"1", b"decode", b"2"] + ) + self.assertEqual(list(self.wire), []) + self.prefill.conclude_transfer(bootstrap_room=1, status=KVPoll.Success) + self.flush() + self.assertEqual(receiver.poll(), KVPoll.WaitingForInput) + self.assertEqual(sender.poll(), KVPoll.Success) + sender.clear() + self.flush() + self.assertEqual(receiver.poll(), KVPoll.Success) + receiver.clear() + self.assertNotIn(1, self.prefill.deferred_bootstrap.rooms) + self.assertNotIn(1, self.prefill.request_status) + self.assertEqual(list(self.wire), []) + retired.append((sender, receiver)) + + def test_completion_before_receiver_uses_same_bootstrap(self): + sender = self.sender() + sender.mark_prefill_complete() + receiver = self.receiver() + self.flush() + self.assertEqual(receiver.poll(), KVPoll.WaitingForInput) + + def test_receiver_before_sender_uses_no_subscription_retries(self): + receiver = self.receiver() + self.flush() + self.assertEqual(receiver.poll(), KVPoll.Bootstrapping) + sender = self.sender() + sender.mark_prefill_complete() + self.flush() + self.assertEqual(receiver.poll(), KVPoll.WaitingForInput) + self.assertEqual(list(self.wire), []) + + def test_sender_abort_fails_decoder_before_metadata(self): + sender, receiver = self.sender(), self.receiver() + self.flush() + sender.abort() + self.flush() + self.assertEqual(receiver.poll(), KVPoll.Failed) + sender.clear() + self.assertIsNone(self.prefill.deferred_bootstrap.rooms[1].owner) + + def test_sender_failure_before_receiver_is_not_lost(self): + sender = self.sender() + sender.abort() + sender.clear() + receiver = self.receiver() + self.flush() + self.assertEqual(receiver.poll(), KVPoll.Failed) + + def test_receiver_abort_notifies_source_using_normal_abort(self): + sender, receiver = self.sender(), self.receiver() + self.flush() + receiver.abort() + self.assertEqual(self.wire[0][1][0], b"ABORT") + self.flush() + self.assertEqual(sender.poll(), KVPoll.Failed) + sender.clear() + receiver.clear() + self.assertNotIn(1, self.decode.request_status) + + def test_cancel_before_sender_creation_is_not_lost(self): + receiver = self.receiver() + receiver.abort() + self.flush() + sender = self.sender() + self.assertEqual(sender.poll(), KVPoll.Failed) + sender.clear() + self.assertNotIn(1, self.prefill.request_status) + + def test_readiness_timeout_aborts_before_allocation(self): + sender, receiver = self.sender(), self.receiver() + receiver._prefill_wait_start = 0 + self.assertEqual(receiver.poll(), KVPoll.Failed) + self.flush() + self.assertEqual(sender.poll(), KVPoll.Failed) + + def test_ready_observed_after_compute_deadline_wins(self): + sender, receiver = self.sender(), self.receiver() + self.flush() + receiver._prefill_wait_start = 0 + sender.mark_prefill_complete() + self.flush() + self.assertEqual(receiver.poll(), KVPoll.WaitingForInput) + self.assertIsNone(receiver._prefill_wait_start) + + def test_admission_timeout_excludes_prefill_compute(self): + sender, receiver = self.sender(), self.receiver() + sender.init_time = 0 + self.assertEqual(sender.poll(), KVPoll.Bootstrapping) + self.flush() + sender.mark_prefill_complete() + self.assertEqual(sender.poll(), KVPoll.Bootstrapping) + sender._prefill_complete_time = 0 + self.assertEqual(sender.poll(), KVPoll.Failed) + self.flush() + self.assertEqual(receiver.poll(), KVPoll.Failed) + + def test_failure_wins_over_late_ready(self): + sender, receiver = self.sender(), self.receiver() + self.flush() + sender.mark_prefill_complete() + sender.abort() + # Deliver failure before the already-emitted ready status. + self.wire.reverse() + self.flush() + self.assertEqual(receiver.poll(), KVPoll.Failed) + + def test_duplicate_sender_does_not_crash_or_clear_owner(self): + sender, receiver = self.sender(), self.receiver() + duplicate = self.sender() + self.assertEqual(duplicate.poll(), KVPoll.Failed) + duplicate.clear() + self.assertIs(self.prefill.deferred_bootstrap.rooms[1].owner, sender) + sender.clear() + self.flush() + self.assertEqual(receiver.poll(), KVPoll.Failed) + + def test_duplicate_receiver_does_not_crash_or_clear_owner(self): + sender, receiver = self.sender(), self.receiver() + self.flush() + duplicate = self.receiver() + self.assertEqual(duplicate.poll(), KVPoll.Failed) + duplicate.clear() + self.assertIn(1, self.decode.request_status) + self.assertIn(1, self.decode.addr_to_rooms_tracker[receiver.bootstrap_addr]) + receiver.clear() + self.flush() + self.assertEqual(sender.poll(), KVPoll.Failed) + + def test_rejected_sender_stays_failed_after_original_clears(self): + original = self.sender() + duplicate = self.sender() + original.clear() + self.assertEqual(duplicate.poll(), KVPoll.Failed) + duplicate.clear() + + def test_fake_warmup_sender_never_registers_deferred_bootstrap(self): + sender = FakeKVSender(self.prefill, "fake:0", 0, [0], 0) + sender.init(0) + sender.mark_prefill_complete() + self.assertEqual(sender.poll(), KVPoll.WaitingForInput) + sender.send([]) + self.assertEqual(sender.poll(), KVPoll.Success) + self.assertEqual(self.prefill.deferred_bootstrap.rooms, {}) + self.assertEqual(list(self.wire), []) + + def test_abort_after_metadata_uses_existing_drain_protocol(self): + receiver = self.receiver() + receiver._prefill_wait_start = None + receiver.init_time = time.time() + self.wire.clear() + with patch.object(CommonKVReceiver, "_send_abort_notification") as notify: + receiver.abort() + notify.assert_called_once() + self.assertEqual(receiver.conclude_state, KVPoll.Failed) + + def test_mismatched_peer_fails_request_before_destination_registration(self): + receiver = self.receiver() + self.decode.prefill_info_table = { + receiver.bootstrap_addr: PrefillServerInfo( + attn_tp_size=1, + attn_cp_size=1, + dp_size=1, + pp_size=1, + page_size=1, + kv_cache_dtype=None, + follow_bootstrap_room=True, + decode_allocation_policy="early", + ) + } + with patch.object(receiver, "_setup_bootstrap_infos") as setup: + CommonKVReceiver.init(receiver, 0) + self.assertEqual(receiver.conclude_state, KVPoll.Failed) + self.assertIn("policy mismatch", self.decode.failure_records[1]) + setup.assert_not_called() + + def test_scheduler_does_not_reserve_request_rows_for_unready_prefill(self): + sender, receiver = self.sender(), self.receiver() + req = SimpleNamespace( + rid="req", + bootstrap_room=1, + finished_reason=None, + return_logprob=False, + time_stats=MagicMock(), + ) + entry = DecodeRequest(req=req, kv_receiver=receiver) + queue = DecodePreallocQueue.__new__(DecodePreallocQueue) + queue.queue, queue.pending_reqs = [entry], [] + queue.pp_size, queue.tp_rank = 1, 0 + queue.gloo_group = object() + queue._resolve_pending_reqs = Mock() + queue._uses_swa_tail_prealloc = Mock(return_value=False) + queue.token_to_kv_pool_allocator = SimpleNamespace( + prealloc_fits_assumes_reclaim=Mock(return_value=False) + ) + queue._allocatable_token_budgets = Mock(return_value=100000) + queue._hicache_pending_restore_tokens = Mock(return_value=0) + queue.scheduler = SimpleNamespace( + running_batch=SimpleNamespace(reqs=[]), + enable_priority_scheduling=False, + enable_hisparse=False, + enable_lora=False, + metrics_reporter=SimpleNamespace(enable_metrics=False), + ) + queue.req_to_metadata_buffer_idx_allocator = SimpleNamespace( + available_size=Mock(return_value=1) + ) + queue.req_to_token_pool = SimpleNamespace(available_size=Mock(return_value=0)) + with patch( + "sglang.srt.disaggregation.decode.poll_and_all_reduce", + side_effect=lambda receivers, group: [r.poll() for r in receivers], + ): + self.assertEqual(queue.pop_preallocated(), ([], [])) + self.flush() + self.assertFalse(entry.waiting_for_input) + queue.req_to_metadata_buffer_idx_allocator.available_size.assert_not_called() + queue.req_to_token_pool.available_size.assert_not_called() + sender.mark_prefill_complete() + self.flush() + self.assertEqual(queue.pop_preallocated(), ([], [])) + self.assertTrue(entry.waiting_for_input) + queue.req_to_metadata_buffer_idx_allocator.available_size.assert_called_once() + queue.req_to_token_pool.available_size.assert_called_once() + self.assertEqual(queue.queue, [entry]) + + def test_exhausted_optimistic_attempt_remains_compute_eligible(self): + override = get_context().override_server_args( + optimistic_prefill_attempts=1, + disaggregation_decode_allocation_policy="prefill_complete", + ) + override.install() + self.addCleanup(override.restore) + req = SimpleNamespace( + rid="retry", + prefill_attempt_count=1, + output_ids=[42], + skip_radix_cache_insert=False, + kv=SimpleNamespace(cache_protected_len=16), + full_untruncated_fill_ids=list(range(32)), + _compute_max_prefix_len=Mock(return_value=31), + storage_prefetch_retry_attempts=3, + storage_prefetch_last_match_len=8, + reset_for_retract=Mock(), + advance_cache_request_handle=Mock(), + time_stats=MagicMock(), + ) + scheduler = SimpleNamespace( + tree_cache=Mock(supports_mamba=Mock(return_value=False)), + _release_aborted_request=Mock(), + clear_pending_chunk_send=Mock(), + waiting_queue=[], + disagg_prefill_bootstrap_queue=SimpleNamespace(queue=[]), + metrics_reporter=SimpleNamespace(enable_metrics=False), + processed_tokens_counter=0, + ) + with ( + patch("sglang.srt.disaggregation.prefill.checkpoint_kv_cache"), + patch("sglang.srt.disaggregation.prefill.release_kv_cache"), + ): + SchedulerDisaggregationPrefillMixin.optimistic_release_and_requeue( + scheduler, req + ) + self.assertEqual(scheduler.waiting_queue, [req]) + self.assertEqual(scheduler.disagg_prefill_bootstrap_queue.queue, []) + self.assertEqual(req.prefill_attempt_count, 2) + self.assertEqual(list(req.output_ids), []) + self.assertTrue(req.pending_bootstrap) + self.assertEqual(req.storage_prefetch_retry_attempts, 0) + self.assertEqual(req.storage_prefetch_last_match_len, 16) + + def test_chunked_prefill_does_not_yield_for_decode_admission(self): + for policy in ("early", "prefill_complete"): + for overlap in (False, True): + with self.subTest(policy=policy, overlap=overlap): + override = get_context().override_server_args( + disaggregation_decode_allocation_policy=policy + ) + override.install() + try: + req = Mock(finished_reason=None, to_finish=None) + scheduler = SimpleNamespace( + chunked_req=req, + tree_cache=Mock(), + enable_overlap=overlap, + check_bootstrap=Mock(return_value=False), + has_bootstrapped_waiting_req=Mock(return_value=True), + optimistic_release_and_requeue=Mock(), + ) + scheduler.checkpoint_disagg_prefill = MethodType( + SchedulerDisaggregationPrefillMixin.checkpoint_disagg_prefill, + scheduler, + ) + running = SimpleNamespace(batch_is_full=True) + with patch( + "sglang.srt.disaggregation.prefill.checkpoint_kv_cache" + ): + SchedulerDisaggregationPrefillMixin.process_prefill_chunk( + scheduler, None, running + ) + if policy == "prefill_complete": + self.assertIs(scheduler.chunked_req, req) + self.assertFalse(running.batch_is_full) + scheduler.optimistic_release_and_requeue.assert_not_called() + else: + self.assertIsNone(scheduler.chunked_req) + self.assertEqual( + scheduler.optimistic_release_and_requeue.call_count, + 0 if overlap else 1, + ) + finally: + override.restore() + + def test_completed_prefill_parks_until_normal_pointer_handshake(self): + sender, receiver = self.sender(), self.receiver() + self.flush() + sender.mark_prefill_complete() + self.flush() + self.assertEqual(receiver.poll(), KVPoll.WaitingForInput) + # Decode may reserve KV now, but prefill still has no destination indices. + self.assertEqual(sender.poll(), KVPoll.Bootstrapping) + req = SimpleNamespace( + rid="parked", + disagg_kv_sender=sender, + pending_bootstrap=True, + prefill_attempt_count=1, + skip_radix_cache_insert=False, + ) + scheduler = SchedulerDisaggregationPrefillMixin() + scheduler.tree_cache = Mock() + scheduler.scheduler_stage_metrics = None + scheduler.disagg_prefill_inflight_queue = [req] + scheduler.attn_cp_cpu_group = scheduler.attn_tp_cpu_group = object() + scheduler.output_streamer = Mock() + scheduler.send_kv_chunk = Mock() + + def finalize(r): + r.pending_bootstrap = False + return True + + scheduler.disagg_prefill_bootstrap_queue = SimpleNamespace( + finalize_bootstrap=Mock(side_effect=finalize) + ) + with ( + patch( + "sglang.srt.disaggregation.prefill.poll_and_all_reduce_attn_cp_tp_group", + side_effect=lambda senders, *groups: [s.poll() for s in senders], + ), + patch("sglang.srt.disaggregation.prefill.checkpoint_kv_cache"), + ): + self.assertEqual(scheduler.process_disagg_prefill_inflight_queue(), []) + self.assertEqual(scheduler.disagg_prefill_inflight_queue, [req]) + scheduler.send_kv_chunk.assert_not_called() + # This is the existing metadata-received transition, not readiness. + self.prefill.update_status(1, KVPoll.WaitingForInput) + self.assertEqual(scheduler.process_disagg_prefill_inflight_queue(), []) + scheduler.send_kv_chunk.assert_called_once_with(req, last_chunk=True) + self.assertFalse(req.pending_bootstrap) + + def test_clear_before_allocation_notifies_prefill(self): + sender, receiver = self.sender(), self.receiver() + self.flush() + receiver.clear() + self.flush() + self.assertEqual(sender.poll(), KVPoll.Failed) + self.assertNotIn(1, self.decode.request_status) + + def test_success_is_sent_once_after_all_sender_state_is_cleared(self): + sender, receiver = self.sender(), self.receiver() + self.flush() + sender.mark_prefill_complete() + self.flush() + self.prefill.conclude_transfer(bootstrap_room=1, status=KVPoll.Success) + self.assertEqual(sender.poll(), KVPoll.Success) + self.prefill.req_to_decode_prefix_len[1] = 0 + self.prefill.transfer_infos[1] = {} + send = self.prefill.send_kv_status_message + next_transfer = [] + + def success(**kwargs): + self.assertEqual(kwargs["status"], KVPoll.Success) + for table in ( + self.prefill.deferred_bootstrap.rooms, + self.prefill.request_status, + self.prefill.req_to_decode_prefix_len, + self.prefill.transfer_infos, + ): + self.assertNotIn(1, table) + send(**kwargs) + self.flush() + self.assertEqual(receiver.poll(), KVPoll.Success) + receiver.clear() + # Resume immediately, before the old clear() call even returns. + new_sender, new_receiver = self.sender(), self.receiver() + self.flush() + next_transfer.extend((new_sender, new_receiver)) + + self.prefill.send_kv_status_message = Mock(side_effect=success) + sender.clear() + sender.clear() + self.prefill.send_kv_status_message.assert_called_once() + new_sender, new_receiver = next_transfer + self.assertIs(self.prefill.deferred_bootstrap.rooms[1].owner, new_sender) + self.assertEqual(new_receiver.poll(), KVPoll.Bootstrapping) + self.assertEqual(list(self.wire), []) + + def test_worker_cleanup_finishes_before_sender_can_report_success(self): + sender, receiver = self.sender(), self.receiver() + self.flush() + sender.mark_prefill_complete() + self.flush() + mgr = self.prefill + mgr._staging_outstanding = defaultdict(int) + mgr.session_lock = threading.Lock() + mgr.failed_sessions = set() + mgr.enable_deferred_decode_kv_release = True + mgr.decode_kv_args_table = { + "session": SimpleNamespace(requires_dcp_relayout=False, dst_aux_ptrs=[]) + } + mgr._get_dsa_cache_transfer_skip_flags = Mock(return_value=(False, False)) + mgr.send_aux = Mock(return_value=0) + mgr.transfer_infos[1] = { + "session": SimpleNamespace( + is_dummy=False, + mooncake_session_id="session", + dst_kv_indices=[], + dst_device_kv_indices=None, + endpoint="decode", + dst_port=2, + room=1, + required_dst_info_num=1, + ) + } + cleanup_observed = [] + + def during_cleanup(room, default=None): + # The worker has finished writes and decremented its counter, but + # is still deleting room-indexed state. Force a scheduler poll here. + self.assertEqual(mgr._staging_outstanding[room], 0) + self.assertEqual(sender.poll(), KVPoll.Transferring) + self.flush() + self.assertEqual(receiver.poll(), KVPoll.WaitingForInput) + cleanup_observed.append(room) + + mgr.req_to_decode_prefix_len = SimpleNamespace(pop=during_cleanup) + chunk = SimpleNamespace( + room=1, + staging_counted=False, + index_slice=slice(0, 0), + prefill_kv_indices=[], + is_last_chunk=True, + state_indices=None, + prefill_aux_index=0, + ) + mgr.transfer_worker(Mock(get=Mock(side_effect=[chunk, None])), None) + self.assertEqual(cleanup_observed, [1]) + self.assertNotIn(1, mgr._staging_outstanding) + self.assertEqual(sender.poll(), KVPoll.Success) + self.assertEqual(receiver.poll(), KVPoll.WaitingForInput) + mgr.req_to_decode_prefix_len = {} + sender.clear() + self.flush() + self.assertEqual(receiver.poll(), KVPoll.Success) + + def test_early_transfer_success_does_not_wait_for_sender_cleanup(self): + override = get_context().override_server_args( + disaggregation_decode_allocation_policy="early" + ) + override.install() + self.addCleanup(override.restore) + self.prefill.defer_decode_allocation = self.decode.defer_decode_allocation = ( + False + ) + self.prefill.deferred_bootstrap = None + sender, receiver = self.sender(), self.receiver(policy="early") + self.prefill.conclude_transfer( + bootstrap_room=1, status=KVPoll.Success, targets=[("decode", 2)] + ) + self.flush() + self.assertEqual(receiver.poll(), KVPoll.Success) + self.assertIn(1, self.prefill.request_status) + self.assertEqual(sender.poll(), KVPoll.Success) + + def test_recompute_http_error_after_success_before_retirement_is_ignored(self): + sender, receiver = self.sender(), self.receiver() + self.flush() + sender.mark_prefill_complete() + self.flush() + self.prefill.conclude_transfer(bootstrap_room=1, status=KVPoll.Success) + self.assertEqual(sender.poll(), KVPoll.Success) + sender.clear() + self.flush() + # Success is in the status table, but the scheduler has not polled it. + self.assertIsNone(receiver.conclude_state) + self.decode._fail_prefill_recompute(receiver, "late HTTP timeout") + self.assertEqual(receiver.poll(), KVPoll.Success) + self.assertNotIn(1, self.decode.failure_records) + + def test_receiver_retirement_waits_for_recompute_error_callback(self): + receiver = self.receiver() + error_started = threading.Event() + finish_error = threading.Event() + retirement_started = threading.Event() + record_failure = self.decode.record_failure + + def delayed_failure(room, reason): + error_started.set() + if not finish_error.wait(timeout=5): + raise TimeoutError("test did not release the error callback") + record_failure(room, reason) + + def retire(): + retirement_started.set() + receiver.clear() + + self.decode.record_failure = delayed_failure + with ThreadPoolExecutor(max_workers=2) as executor: + error = executor.submit( + self.decode._fail_prefill_recompute, receiver, "HTTP error" + ) + try: + self.assertTrue(error_started.wait(timeout=5)) + retirement = executor.submit(retire) + self.assertTrue(retirement_started.wait(timeout=5)) + with self.assertRaises(TimeoutError): + retirement.result(timeout=0.05) + finally: + finish_error.set() + error.result(timeout=5) + retirement.result(timeout=5) + self.assertFalse(receiver._owns_bootstrap_room) + self.assertEqual(self.decode.failure_records[1], "HTTP error") + + def test_slow_status_delivery_does_not_hold_bootstrap_lock(self): + sender, receiver = self.sender(), self.receiver() + self.flush() + + def send(*args, **kwargs): + acquired = self.prefill.deferred_bootstrap.lock.acquire(blocking=False) + self.assertTrue(acquired) + if acquired: + self.prefill.deferred_bootstrap.lock.release() + + self.prefill.send_kv_status_message = send + sender.mark_prefill_complete() + sender.abort() + sender.clear() + + def test_rebootstrap_dispatch_precedes_ready_without_allocating(self): + receiver = self.receiver() + receiver.init = Mock() + req = SimpleNamespace(build_rebootstrap_payload=lambda: {"rid": "resume"}) + entry = DecodeRequest(req=req, kv_receiver=receiver, is_rebootstrap=True) + queue = DecodePreallocQueue.__new__(DecodePreallocQueue) + dispatched = [] + queue.kv_manager = SimpleNamespace( + submit_prefill_recompute=lambda r, payload: dispatched.append((r, payload)) + ) + queue._init_receiver(entry, 0) + self.assertEqual(dispatched, [(receiver, {"rid": "resume"})]) + self.assertFalse(entry.waiting_for_input) + self.assertEqual(req.disagg_prefill_dp_rank, 0) + + def test_rebootstrap_dispatches_once_across_init_and_metadata(self): + for policy in ("early", "prefill_complete"): + with self.subTest(policy=policy): + override = get_context().override_server_args( + disaggregation_decode_allocation_policy=policy + ) + override.install() + try: + receiver = Mock(conclude_state=None) + req = SimpleNamespace( + build_rebootstrap_payload=lambda: {"rid": "resume"}, + time_stats=Mock(), + ) + entry = DecodeRequest( + req=req, kv_receiver=receiver, is_rebootstrap=True + ) + queue = DecodePreallocQueue.__new__(DecodePreallocQueue) + queue.kv_manager = Mock() + queue.req_to_metadata_buffer_idx_allocator = Mock() + queue._transfer_page_indices = Mock(return_value=[0]) + queue.transfer_queue = SimpleNamespace(enable_staging=False) + queue._num_published_destinations = 0 + events = Mock() + events.attach_mock(receiver.send_metadata, "metadata") + events.attach_mock( + queue.kv_manager.submit_prefill_recompute, "recompute" + ) + queue._init_receiver(entry, 0) + queue._send_kv_metadata(entry, [0], 1) + queue.kv_manager.submit_prefill_recompute.assert_called_once_with( + receiver, {"rid": "resume"} + ) + self.assertEqual( + [event[0] for event in events.mock_calls], + ["metadata", "recompute"] + if policy == "early" + else ["recompute", "metadata"], + ) + self.assertEqual(queue._num_published_destinations, 1) + finally: + override.restore() + + def test_normal_bootstrap_keeps_room_without_dispatching_recompute(self): + queue = DecodePreallocQueue.__new__(DecodePreallocQueue) + queue._check_if_req_exceed_kv_capacity = Mock(return_value=False) + queue.pending_reqs = [] + req = SimpleNamespace( + bootstrap_room=7, + bootstrap_host="prefill", + bootstrap_port=8998, + disagg_prefill_dp_rank=None, + to_finish=None, + finished_reason=None, + ) + receiver = Mock(conclude_state=None) + entry = DecodeRequest(req=req, kv_receiver=receiver) + queue._create_receiver_and_enqueue = Mock(return_value=entry) + queue.kv_manager = SimpleNamespace( + prefill_info_table={ + "prefill:8998": SimpleNamespace(dp_size=1), + }, + submit_prefill_recompute=Mock(), + ) + queue.add(req) + self.assertEqual(req.bootstrap_room, 7) + receiver.init.assert_called_once_with(0) + queue.kv_manager.submit_prefill_recompute.assert_not_called() + self.assertFalse(entry.waiting_for_input) + + def test_rebootstrap_keeps_room_and_prefill_rank(self): + queue = DecodePreallocQueue.__new__(DecodePreallocQueue) + queue._check_if_req_exceed_kv_capacity = Mock(return_value=False) + queue.queue = [] + queue.pending_reqs = [] + queue._create_receiver_and_enqueue = Mock() + queue._init_receiver = Mock() + req = SimpleNamespace( + bootstrap_room=1, + bootstrap_host="prefill", + bootstrap_port=8998, + disagg_prefill_dp_rank=3, + ) + queue.kv_manager = SimpleNamespace( + prefill_info_table={"prefill:8998": SimpleNamespace(dp_size=4)} + ) + queue.add(req, is_rebootstrap=True) + self.assertEqual(req.bootstrap_room, 1) + queue._init_receiver.assert_called_once_with( + queue._create_receiver_and_enqueue.return_value, 3 + ) + queue.add(req, is_rebootstrap=True) + self.assertEqual(req.bootstrap_room, 1) + self.assertEqual(req.disagg_prefill_dp_rank, 3) + + def test_receiver_clear_preserves_failure_reason(self): + receiver = self.receiver() + self.decode.record_failure(1, "original failure") + self.decode.update_status(1, KVPoll.Failed) + receiver.clear() + self.assertEqual(self.decode.failure_records[1], "original failure") + + +if __name__ == "__main__": + unittest.main()