diff --git a/docs/docs/advanced_features/pd_disaggregation.mdx b/docs/docs/advanced_features/pd_disaggregation.mdx
index 5ce40a82e4..17fa62441d 100644
--- a/docs/docs/advanced_features/pd_disaggregation.mdx
+++ b/docs/docs/advanced_features/pd_disaggregation.mdx
@@ -20,6 +20,60 @@ For the design details, please refer to [link](https://docs.google.com/document/
Currently, we support Mooncake and NIXL as the transfer engine.
+## Decode allocation policy
+
+By default, decode reserves each request's destination KV slots before prefill
+finishes. Long prefills can hold these slots while decode waits for the KV data.
+To delay this reservation, set the following flag on **both prefill and decode**:
+
+```bash
+--disaggregation-decode-allocation-policy prefill_complete
+```
+
+With `prefill_complete`, prefill computes without waiting for destination slots,
+then signals readiness after accepting its final prefill result. Decode reserves
+slots and starts the existing KV transfer handshake only after that signal.
+This changes per-request KV reservation, not the size or creation of the global
+KV pool. The default `early` policy preserves the existing behavior.
+
+Deferring allocation can help when long prefills compete with active decode
+requests for destination KV capacity. It also retains KV on prefill longer and
+removes overlap between chunked prefill and KV transfer. Compare both policies
+for your workload; this option is not a general throughput improvement.
+
+This initial implementation requires Mooncake (RDMA or `mooncake_tcp`), the Python
+bootstrap registry, and attention TP/CP and pipeline sizes of one. Attention DP
+is allowed when each DP rank has attention TP size one. Prefill automatically
+resolves `--optimistic-prefill-attempts` from zero to one; configurations that
+disable optimistic prefill after model validation are rejected.
+Chunked prefill does not yield just because decode has not admitted the request.
+Completed requests retain their KV in the existing optimistic-prefill inflight
+queue until decode supplies destination indices; readiness alone never starts
+a KV write.
+After a successful transfer, prefill clears the room's readiness and transfer
+state before notifying decode of success. Retraction recovery can then reuse
+the same bootstrap room. This notification waits for worker and scheduler
+cleanup; failure and cancellation records remain until their timeout expires.
+Retries remain eligible for prefill after the optimistic attempt budget is
+exhausted and remain subject to the existing cancellation and timeout handling.
+
+On prefill, `SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT` starts after the final
+prefill result, giving decode admission its own budget. On decode,
+`SGLANG_DISAGGREGATION_WAITING_TIMEOUT` bounds the wait for prefill completion,
+then starts afresh for admission and again when destination indices are sent
+for transfer.
+
+The option does not support staging buffers, runtime PD role switching, PD
+multiplexing, encoder disaggregation, decoupled speculation, HiSparse, decode radix cache, decode
+HiCache, or non-DSA auxiliary state pools. Prefill HiCache is limited to L2 with
+`write_back` and no storage backend. Unsupported configurations fail validation.
+
+All communicating workers must use the same revision and enable the same policy. This
+version of the decode worker rejects policy or peer-layout mismatches before
+destination registration. Older decode workers do not enforce this check. Keep
+`early` on both sides until every worker has been upgraded, then enable the new
+policy on both sides.
+
## Profiling in PD Disaggregation Mode
When you need to profile prefill or decode workers in PD disaggregation mode, please refer to the [Profile In PD Disaggregation Mode](../developer_guide/benchmark_and_profiling#profile-in-pd-disaggregation-mode) section in the Benchmark and Profiling guide. Due to torch profiler limitations, prefill and decode workers must be profiled separately using dedicated command-line options.
diff --git a/docs/docs/advanced_features/server_arguments.mdx b/docs/docs/advanced_features/server_arguments.mdx
index d8453e95a9..800f047826 100644
--- a/docs/docs/advanced_features/server_arguments.mdx
+++ b/docs/docs/advanced_features/server_arguments.mdx
@@ -3007,6 +3007,12 @@ This option does not disable required attention or MoE collectives, including ex
None |
Type: str |
+
+ | `--disaggregation-decode-allocation-policy` |
+ When to reserve per-request decode KV slots. Set the same policy on prefill and decode. See requirements and tradeoffs. |
+ `early` |
+ early, prefill_complete |
+
| `--disaggregation-decode-enable-offload-kvcache` |
Enable async KV cache offloading on decode server (PD mode). |
diff --git a/python/sglang/srt/arg_groups/fields/disagg.py b/python/sglang/srt/arg_groups/fields/disagg.py
index 89985577dd..aab97a017a 100644
--- a/python/sglang/srt/arg_groups/fields/disagg.py
+++ b/python/sglang/srt/arg_groups/fields/disagg.py
@@ -56,6 +56,12 @@ class Disagg(msgspec.Struct):
Literal["null", "prefill", "decode"],
'Only used for PD disaggregation. "prefill" for prefill-only server, and "decode" for decode-only server. If not specified, it is not PD disaggregated',
] = "null"
+ disaggregation_decode_allocation_policy: A[
+ Literal["early", "prefill_complete"],
+ "When to reserve per-request decode KV in disaggregated serving. "
+ "prefill_complete retains KV on prefill until prefill finishes, then "
+ "admits it on decode. Set the same policy on both roles. Default: early.",
+ ] = "early"
disaggregation_transfer_backend: A[
str,
Arg(
diff --git a/python/sglang/srt/arg_groups/pd_disaggregation_hook.py b/python/sglang/srt/arg_groups/pd_disaggregation_hook.py
index f7d619d2f3..da8125869f 100644
--- a/python/sglang/srt/arg_groups/pd_disaggregation_hook.py
+++ b/python/sglang/srt/arg_groups/pd_disaggregation_hook.py
@@ -44,6 +44,16 @@ def handle_pd_disaggregation(server_args: ServerArgs) -> None:
"with MC_FORCE_TCP=1 (TCP transport, no RDMA)"
)
+ if cfg.disaggregation_decode_allocation_policy == "prefill_complete":
+ _validate_prefill_complete(server_args)
+ if (
+ cfg.disaggregation_mode == "prefill"
+ and cfg.optimistic_prefill_attempts == 0
+ ):
+ declare_resolution(
+ server_args, "handle_pd_disaggregation", optimistic_prefill_attempts=1
+ )
+
if cfg.disaggregation_mode == "prefill" and cfg.dcp_size > 1:
logger.warning(
"DCP on a PD prefill server is supported when prefill and decode "
@@ -191,6 +201,52 @@ def handle_pd_disaggregation(server_args: ServerArgs) -> None:
)
+def _validate_prefill_complete(server_args: ServerArgs) -> None:
+ cfg = resolving_view(server_args)
+ unsupported = []
+ if cfg.disaggregation_mode not in ("prefill", "decode"):
+ unsupported.append("aggregated serving")
+ if cfg.disaggregation_transfer_backend != "mooncake":
+ unsupported.append("transfer backends other than Mooncake")
+ if cfg.pp_size != 1 or cfg.attn_cp_size != 1 or cfg.dcp_size != 1:
+ unsupported.append("pipeline or attention context parallelism")
+ attn_dp_size = cfg.dp_size if cfg.enable_dp_attention else 1
+ if cfg.tp_size != attn_dp_size:
+ unsupported.append("attention tensor parallelism greater than one")
+ if cfg.enable_pdmux or cfg.decoupled_spec_role != "null":
+ unsupported.append("PD multiplexing or decoupled speculation")
+ if cfg.enable_pd_role_switch:
+ unsupported.append("runtime PD role switching")
+ if cfg.encoder_only or cfg.language_only:
+ unsupported.append("encoder disaggregation")
+ if cfg.enable_hisparse:
+ unsupported.append("HiSparse")
+ if cfg.disaggregation_decode_host_receive_threshold > 0:
+ unsupported.append("decode host KV buffering")
+ if cfg.disaggregation_mode == "decode" and (
+ cfg.disaggregation_decode_enable_radix_cache or cfg.enable_hierarchical_cache
+ ):
+ unsupported.append("decode radix cache or decode HiCache")
+ if (
+ cfg.disaggregation_mode == "prefill"
+ and cfg.enable_hierarchical_cache
+ and (
+ cfg.hicache_storage_backend is not None
+ or cfg.hicache_write_policy != "write_back"
+ )
+ ):
+ unsupported.append("prefill HiCache other than L2 write_back")
+ if envs.SGLANG_DISAGG_STAGING_BUFFER.get():
+ unsupported.append("staging buffers")
+ if envs.SGLANG_RUST_SERVER.get():
+ unsupported.append("the Rust bootstrap registry")
+ if unsupported:
+ raise ValueError(
+ "--disaggregation-decode-allocation-policy prefill_complete does not "
+ "yet support " + ", ".join(unsupported)
+ )
+
+
def _alias_bootstrap_port_to_api_port(server_args: ServerArgs) -> None:
"""Rust-server prefill serves the KV bootstrap registry on the api listener
itself, so the resolved bootstrap port must BE the api port — every internal
diff --git a/python/sglang/srt/arg_groups/serving_hook.py b/python/sglang/srt/arg_groups/serving_hook.py
index 452637ee11..5e390960cb 100644
--- a/python/sglang/srt/arg_groups/serving_hook.py
+++ b/python/sglang/srt/arg_groups/serving_hook.py
@@ -494,6 +494,16 @@ def handle_other_validations(server_args: Any):
optimistic_prefill_attempts=0,
)
+ if (
+ cfg.disaggregation_decode_allocation_policy == "prefill_complete"
+ and cfg.disaggregation_mode == "prefill"
+ and resolving_view(server_args).optimistic_prefill_attempts <= 0
+ ):
+ raise ValueError(
+ "prefill_complete requires optimistic prefill, but the resolved "
+ "model/cache configuration disabled it."
+ )
+
# Handle model inference tensor dump.
if cfg.debug_tensor_dump_output_folder is not None:
logger.warning(
diff --git a/python/sglang/srt/disaggregation/base/conn.py b/python/sglang/srt/disaggregation/base/conn.py
index 5e31360193..88b263d208 100644
--- a/python/sglang/srt/disaggregation/base/conn.py
+++ b/python/sglang/srt/disaggregation/base/conn.py
@@ -182,6 +182,10 @@ class BaseKVSender(ABC):
"""
...
+ def mark_prefill_complete(self) -> None:
+ """Publish retained final-prefill KV for an enabled allocation policy."""
+ raise NotImplementedError
+
def pop_decode_prefix_len(self) -> int:
return 0
diff --git a/python/sglang/srt/disaggregation/common/bootstrap.py b/python/sglang/srt/disaggregation/common/bootstrap.py
new file mode 100644
index 0000000000..47ddbf6b5a
--- /dev/null
+++ b/python/sglang/srt/disaggregation/common/bootstrap.py
@@ -0,0 +1,112 @@
+"""Bookkeeping for bootstrap rendezvous before decode has reserved KV.
+
+The transfer backend still owns the destination-pointer handshake and abort
+draining. This table only joins a completed optimistic prefill with decode's
+control endpoint. It performs no I/O, including while holding its lock.
+"""
+
+from __future__ import annotations
+
+import threading
+import time
+from dataclasses import dataclass
+
+BootstrapNotification = tuple[tuple[str, int], bool] | None
+
+
+@dataclass
+class BootstrapRoom:
+ owner: object | None = None
+ endpoint: tuple[str, int] | None = None
+ ready: bool = False
+ ready_notified: bool = False
+ failed: bool = False
+ expires_at: float = 0.0
+
+
+class DeferredBootstrap:
+ def __init__(self, timeout: float):
+ self.timeout = timeout
+ self.rooms: dict[int, BootstrapRoom] = {}
+ self.lock = threading.Lock()
+ self._next_cleanup = 0.0
+
+ def _room(self, room: int) -> BootstrapRoom:
+ now = time.monotonic()
+ if now >= self._next_cleanup:
+ # Active senders own their lifetime. Unmatched endpoints and closed
+ # rooms expire, including cancellation before the sender exists.
+ self.rooms = {
+ key: state
+ for key, state in self.rooms.items()
+ if state.owner is not None or state.expires_at > now
+ }
+ self._next_cleanup = now + min(1.0, self.timeout)
+ return self.rooms.setdefault(room, BootstrapRoom(expires_at=now + self.timeout))
+
+ @staticmethod
+ def _notification(state: BootstrapRoom) -> BootstrapNotification:
+ if state.endpoint is not None:
+ if state.failed:
+ return state.endpoint, True
+ if state.ready and not state.ready_notified:
+ # One readiness message per transfer. Decode must consume it
+ # before publishing destinations, so none can trail Success
+ # and accidentally admit the next transfer using this room.
+ state.ready_notified = True
+ return state.endpoint, False
+ return None
+
+ def open(self, room: int, owner: object) -> BootstrapRoom | None:
+ with self.lock:
+ state = self._room(room)
+ if state.owner is not None:
+ return None
+ state.owner = owner
+ return state
+
+ def register(self, room: int, endpoint: tuple[str, int]) -> BootstrapNotification:
+ with self.lock:
+ state = self._room(room)
+ if state.endpoint is not None and state.endpoint != endpoint:
+ # A room identifies one request, not a reusable allocation slot.
+ # Reject another receiver without replacing the first endpoint.
+ return endpoint, True
+ state.endpoint = endpoint
+ return self._notification(state)
+
+ def complete(self, room: int, owner: object) -> BootstrapNotification:
+ with self.lock:
+ state = self.rooms.get(room)
+ if state is None or state.owner is not owner or state.failed:
+ return None
+ state.ready = True
+ return self._notification(state)
+
+ def fail(self, room: int) -> BootstrapNotification:
+ with self.lock:
+ state = self._room(room)
+ if state.failed:
+ return None
+ state.failed = True
+ return self._notification(state)
+
+ def close(
+ self, room: int, owner: object, *, success: bool = False
+ ) -> BootstrapNotification:
+ with self.lock:
+ state = self.rooms.get(room)
+ if state is None or state.owner is not owner:
+ return None
+ if success and not state.failed:
+ # The caller clears transport state before sending Success.
+ # Only then can decode reuse this room for recomputation.
+ del self.rooms[room]
+ return (state.endpoint, False) if state.endpoint is not None else None
+ state.owner = None
+ already_failed = state.failed
+ state.failed = True
+ state.expires_at = time.monotonic() + self.timeout
+ # Retain a bounded tombstone: a late endpoint must not wait forever
+ # or attach to a second request using the same bootstrap room.
+ return None if already_failed else self._notification(state)
diff --git a/python/sglang/srt/disaggregation/common/conn.py b/python/sglang/srt/disaggregation/common/conn.py
index 271f9a370d..e4ba54615b 100644
--- a/python/sglang/srt/disaggregation/common/conn.py
+++ b/python/sglang/srt/disaggregation/common/conn.py
@@ -7,6 +7,7 @@ import logging
import threading
import time
from collections import defaultdict
+from contextlib import nullcontext
from typing import Dict, List, NamedTuple, Optional, Set, Tuple, Union
import msgspec
@@ -28,6 +29,10 @@ from sglang.srt.disaggregation.base.conn import (
KVTransferMetric,
StateType,
)
+from sglang.srt.disaggregation.common.bootstrap import (
+ BootstrapNotification,
+ DeferredBootstrap,
+)
from sglang.srt.disaggregation.utils import (
DisaggregationMode,
filter_kv_indices_for_cp_rank,
@@ -107,6 +112,7 @@ class PrefillServerInfo:
follow_bootstrap_room: bool
enable_dsa_cache_layer_split: bool = False
dsv41_spec_layout: Optional[dict] = None
+ decode_allocation_policy: str = "early"
# PD true-retraction rebootstrap: the prefill's HTTP API port. The decode
# already knows the prefill host (the bootstrap_addr host), so it can POST
@@ -259,6 +265,9 @@ class DeferredAbortAckState(msgspec.Struct):
class CommonKVManager(BaseKVManager):
+ defer_decode_allocation: bool = False
+ deferred_bootstrap: Optional[DeferredBootstrap] = None
+
# Wire layout of the prefill->decode terminal status message. The legacy
# layout (mooncake, and ascend which inherits it) is three untagged frames
# ``[room, status, prefill_rank]``; backends whose control socket also
@@ -359,6 +368,20 @@ class CommonKVManager(BaseKVManager):
# A poisoned room has a permanently outstanding transfer whose backend
# can no longer prove quiescent. A new room lifecycle clears this state.
self._deferred_ack_poisoned_rooms: Set[int] = set()
+ self.defer_decode_allocation = (
+ get_disagg().disaggregation_decode_allocation_policy == "prefill_complete"
+ )
+ self.deferred_bootstrap = None
+ if self.defer_decode_allocation:
+ if any(state != StateType.DSA for state in args.state_types):
+ raise ValueError(
+ "prefill_complete currently supports full-attention and DSA KV pools only"
+ )
+ if disaggregation_mode == DisaggregationMode.PREFILL:
+ self.deferred_bootstrap = DeferredBootstrap(
+ envs.SGLANG_DISAGGREGATION_WAITING_TIMEOUT.get()
+ + envs.SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT.get()
+ )
if self.disaggregation_mode == DisaggregationMode.PREFILL:
# When SGLANG_DISAGGREGATION_ALL_CP_RANKS_TRANSFER is True, all CP ranks
@@ -542,6 +565,10 @@ class CommonKVManager(BaseKVManager):
return
if status == KVPoll.Failed:
self.request_status[bootstrap_room] = KVPoll.Failed
+ if self.deferred_bootstrap is not None:
+ self._notify_bootstrap(
+ bootstrap_room, self.deferred_bootstrap.fail(bootstrap_room)
+ )
return
if current == KVPoll.Failed:
# Failed is terminal. It also sorts lowest, so the max() below would
@@ -549,6 +576,42 @@ class CommonKVManager(BaseKVManager):
return
self.request_status[bootstrap_room] = max(current, status)
+ def _notify_bootstrap(self, room: int, notification: BootstrapNotification) -> None:
+ if notification is None:
+ return
+ endpoint, failed = notification
+ # Use the normal status channel; readiness does not authorize writes.
+ # No room-table lock is held while sending to a potentially dead peer.
+ self.send_kv_status_message(
+ targets=[endpoint],
+ bootstrap_room=room,
+ status=KVPoll.Failed if failed else KVPoll.WaitingForInput,
+ failure_reason="Prefill bootstrap failed" if failed else None,
+ )
+
+ def _handle_bootstrap_message(self, msg: List[bytes]) -> bool:
+ if self.deferred_bootstrap is None or not msg:
+ return False
+ if msg[0] not in (b"BOOTSTRAP", b"ABORT"):
+ return False
+ try:
+ room = int(msg[1])
+ endpoint = (msg[2].decode("ascii"), int(msg[3]))
+ except (IndexError, ValueError, UnicodeDecodeError):
+ logger.warning("Dropping malformed bootstrap control message")
+ return True
+ if msg[0] == b"BOOTSTRAP":
+ self._notify_bootstrap(
+ room, self.deferred_bootstrap.register(room, endpoint)
+ )
+ return True
+ # Retain cancellation even if it beats source creation. Let the backend
+ # process ABORT normally as well, including ACK/draining after allocation.
+ self.deferred_bootstrap.fail(room)
+ if room in self.request_status and self.request_status[room] != KVPoll.Success:
+ self.update_status(room, KVPoll.Failed)
+ return False
+
def record_failure(self, bootstrap_room: int, failure_reason: str):
with self.failure_lock:
self.failure_records[bootstrap_room] = failure_reason
@@ -623,7 +686,7 @@ class CommonKVManager(BaseKVManager):
status: KVPoll,
failure_reason: Optional[str] = None,
) -> None:
- """Push of a terminal transfer status to decode endpoints."""
+ """Push bootstrap readiness or transfer status to decode endpoints."""
if not targets:
return
parts = self._encode_kv_status_message(
@@ -680,6 +743,10 @@ class CommonKVManager(BaseKVManager):
if targets is None:
targets = self._room_notify_targets(bootstrap_room)
self.update_status(bootstrap_room, status)
+ if status == KVPoll.Success and self.deferred_bootstrap is not None:
+ # Success permits decode to reuse the room after true retraction.
+ # Publish it from sender.clear(), after worker and scheduler cleanup.
+ return status
self.send_kv_status_message(
targets=targets,
bootstrap_room=bootstrap_room,
@@ -711,13 +778,18 @@ class CommonKVManager(BaseKVManager):
prefill_rank: int,
failure_reason: Optional[str] = None,
) -> None:
- """Decode-side handling of one prefill rank's terminal status."""
+ """Decode-side handling of one prefill rank's bootstrap/transfer status."""
if bootstrap_room not in self.request_status:
# The room concluded and was cleared. Recording a failure now would
# leave an entry that a later request reusing this bootstrap_room
# would pick up as its own root cause.
logger.debug("Dropping late status for cleared room %s", bootstrap_room)
return
+ if status == KVPoll.WaitingForInput and self.defer_decode_allocation:
+ # The policy currently requires one authoritative prefill rank.
+ # Do not mix this notification into the transfer-success counter.
+ self.update_status(bootstrap_room, KVPoll.WaitingForInput)
+ return
if status == KVPoll.Success:
self.prefill_response_tracker[bootstrap_room].add(prefill_rank)
expected_response_num = self.required_prefill_response_num_table.get(
@@ -891,7 +963,16 @@ class CommonKVManager(BaseKVManager):
stop flag that a blocked recv can never observe, so in that mode poll
with a timeout and return None when it expires. Deployments without
--enable-pd-role-switch keep the original blocking recv and pay nothing.
+ Deferred allocation consumes endpoint registration here, before backend
+ metadata handling; ABORT still reaches the backend's drain/ACK handler.
"""
+ if self.deferred_bootstrap is not None:
+
+ def recv_bootstrap():
+ msg = socket.recv_multipart()
+ return None if self._handle_bootstrap_message(msg) else msg
+
+ return recv_bootstrap
if not self.server_args.enable_pd_role_switch:
return socket.recv_multipart
@@ -1016,8 +1097,18 @@ class CommonKVManager(BaseKVManager):
the rebootstrap ``/generate`` failed rather than reporting a spurious
``AbortReq``.
"""
- kv_receiver.abort()
- self.record_failure(kv_receiver.bootstrap_room, reason)
+ with kv_receiver._lifecycle_lock:
+ if self.defer_decode_allocation and (
+ not kv_receiver._owns_bootstrap_room
+ or kv_receiver.conclude_state == KVPoll.Success
+ or self.request_status.get(kv_receiver.bootstrap_room) == KVPoll.Success
+ ):
+ # The HTTP response can arrive after KV transfer has succeeded.
+ # Serialize errors with polling/retirement so an old callback
+ # cannot leave a failure record for a reused room.
+ return
+ kv_receiver.abort()
+ self.record_failure(kv_receiver.bootstrap_room, reason)
def _run_prefill_recompute(
self, kv_receiver: CommonKVReceiver, prefill_url: str, payload: dict
@@ -1283,6 +1374,10 @@ class CommonKVManager(BaseKVManager):
"prefill_http_port": get_serving().port,
}
+ payload["decode_allocation_policy"] = (
+ get_disagg().disaggregation_decode_allocation_policy
+ )
+
if envs.SGLANG_RUST_SERVER.get() and self.attn_dp_size > 1:
topology_rows = get_parallel().world_group.all_gather_object(payload)
# Every scheduler contributes a topology row. Only the scheduler
@@ -1359,7 +1454,12 @@ class CommonKVManager(BaseKVManager):
if is_ipv6:
sock.setsockopt(zmq.IPV6, 1)
sock.setsockopt(zmq.RECONNECT_IVL, -1)
- sock.setsockopt(zmq.SNDTIMEO, 30000)
+ sock.setsockopt(
+ zmq.SNDTIMEO,
+ envs.SGLANG_DISAGGREGATION_ZMQ_SEND_TIMEOUT.get() * 1000
+ if self.defer_decode_allocation
+ else 30000,
+ )
sock.setsockopt(zmq.LINGER, 0)
sock.setsockopt(zmq.TCP_KEEPALIVE, 1)
sock.setsockopt(zmq.TCP_KEEPALIVE_IDLE, 30)
@@ -1681,6 +1781,19 @@ class CommonKVSender(BaseKVSender):
# inner state
self.curr_idx = 0
self.init_time: Optional[float] = None
+ self._prefill_complete_time: Optional[float] = None
+ self._owns_bootstrap_room = True
+ if mgr.deferred_bootstrap is not None:
+ room_state = mgr.deferred_bootstrap.open(bootstrap_room, self)
+ self._owns_bootstrap_room = room_state is not None
+ mgr.update_status(bootstrap_room, KVPoll.Bootstrapping)
+ if room_state is None or room_state.failed:
+ mgr.record_failure(
+ bootstrap_room, "Duplicate or cancelled bootstrap room"
+ )
+ mgr.update_status(bootstrap_room, KVPoll.Failed)
+ self.conclude_state = KVPoll.Failed
+ return
if self.kv_mgr.is_dummy_cp_rank:
# Non-authoritative CP ranks are dummy participants.
self.kv_mgr.update_status(self.bootstrap_room, KVPoll.WaitingForInput)
@@ -1733,6 +1846,17 @@ class CommonKVSender(BaseKVSender):
f"CommonKVSender init with num_kv_indices: {num_kv_indices} and aux_index: {aux_index}"
)
+ def mark_prefill_complete(self) -> None:
+ if self.kv_mgr.deferred_bootstrap is None:
+ return
+ # Give decode admission a fresh budget, excluding prefill queue/compute.
+ if self._prefill_complete_time is None:
+ self._prefill_complete_time = time.time()
+ self.kv_mgr._notify_bootstrap(
+ self.bootstrap_room,
+ self.kv_mgr.deferred_bootstrap.complete(self.bootstrap_room, self),
+ )
+
def pop_decode_prefix_len(self) -> int:
return self.kv_mgr.req_to_decode_prefix_len.pop(self.bootstrap_room, 0)
@@ -1816,9 +1940,14 @@ class CommonKVSender(BaseKVSender):
pass
def _check_bootstrap_timeout(self) -> Optional[KVPoll]:
- if self.init_time is None:
+ start = (
+ self._prefill_complete_time
+ if self.kv_mgr.defer_decode_allocation
+ else self.init_time
+ )
+ if start is None:
return None
- elapsed = time.time() - self.init_time
+ elapsed = time.time() - start
if elapsed < self.kv_mgr.bootstrap_timeout:
return None
logger.warning_once(
@@ -1835,14 +1964,35 @@ class CommonKVSender(BaseKVSender):
return KVPoll.Failed
def clear(self) -> None:
+ notification = None
+ if self.kv_mgr.deferred_bootstrap is not None:
+ if not self._owns_bootstrap_room:
+ return
+ notification = self.kv_mgr.deferred_bootstrap.close(
+ self.bootstrap_room,
+ self,
+ success=self.kv_mgr.request_status.get(self.bootstrap_room)
+ == KVPoll.Success,
+ )
+ self._owns_bootstrap_room = False
self.kv_mgr.request_status.pop(self.bootstrap_room, None)
self.kv_mgr.req_to_decode_prefix_len.pop(self.bootstrap_room, None)
self.kv_mgr.transfer_infos.pop(self.bootstrap_room, None)
# A sender can be cleared while a counted transfer is still writing:
# keep the target for the worker, or ACK now if nothing is in flight.
self.kv_mgr._maybe_ack_drained_abort(self.bootstrap_room)
+ if notification is not None:
+ endpoint, failed = notification
+ self.kv_mgr.send_kv_status_message(
+ targets=[endpoint],
+ bootstrap_room=self.bootstrap_room,
+ status=KVPoll.Failed if failed else KVPoll.Success,
+ failure_reason="Prefill bootstrap failed" if failed else None,
+ )
def abort(self):
+ if self.kv_mgr.deferred_bootstrap is not None and not self._owns_bootstrap_room:
+ return
self.kv_mgr.record_failure(
self.bootstrap_room,
"Aborted by AbortReq.",
@@ -1852,6 +2002,10 @@ class CommonKVSender(BaseKVSender):
class CommonKVReceiver(BaseKVReceiver):
+ _prefill_wait_start: Optional[float] = None
+ _owns_bootstrap_room: bool = True
+ _lifecycle_lock = nullcontext()
+
_ctx = zmq.Context()
_ctx.set(zmq.MAX_SOCKETS, envs.SGLANG_DISAGGREGATION_ZMQ_MAX_SOCKETS.get())
_socket_cache = {}
@@ -1867,16 +2021,29 @@ class CommonKVReceiver(BaseKVReceiver):
self.bootstrap_room = bootstrap_room
self.bootstrap_addr = bootstrap_addr
self.kv_mgr = mgr
+ if mgr.defer_decode_allocation:
+ self._lifecycle_lock = threading.RLock()
self.conclude_state: Optional[KVPoll] = None
self.require_staging: bool = False
self.init_time: Optional[float] = None
+ self._prefill_wait_start: Optional[float] = None
self.abort_notified: bool = False
self._abort_generation: Optional[int] = None
self._connection_pool_entries: Dict[str, List[Dict]] = {}
+ self._owns_bootstrap_room = not (
+ mgr.defer_decode_allocation and bootstrap_room in mgr.request_status
+ )
+ if not self._owns_bootstrap_room:
+ mgr.record_failure(bootstrap_room, "Duplicate decode bootstrap room")
+ mgr.update_status(bootstrap_room, KVPoll.Failed)
+ self.conclude_state = KVPoll.Failed
+ return
self.kv_mgr.addr_to_rooms_tracker[self.bootstrap_addr].add(self.bootstrap_room)
self.kv_mgr.update_status(self.bootstrap_room, KVPoll.Bootstrapping)
def init(self, prefill_dp_rank: int):
+ if self.conclude_state == KVPoll.Failed:
+ return
if self.bootstrap_addr not in self.kv_mgr.prefill_info_table:
self.kv_mgr.record_failure(
self.bootstrap_room,
@@ -1888,6 +2055,24 @@ class CommonKVReceiver(BaseKVReceiver):
# Read pre-computed rank mapping from prefill_info (computed in try_ensure_parallel_info)
self.prefill_info = self.kv_mgr.prefill_info_table[self.bootstrap_addr]
+ info = self.prefill_info
+ policy = get_disagg().disaggregation_decode_allocation_policy
+ reason = None
+ if info.decode_allocation_policy != policy:
+ reason = (
+ "Disaggregation decode allocation policy mismatch: "
+ f"prefill={info.decode_allocation_policy}, decode={policy}. "
+ "Set --disaggregation-decode-allocation-policy identically on both roles."
+ )
+ elif policy == "prefill_complete" and (
+ info.attn_tp_size != 1 or info.attn_cp_size != 1 or info.pp_size != 1
+ ):
+ reason = "prefill_complete requires peer attention TP1/CP1/PP1"
+ if reason is not None:
+ self.kv_mgr.record_failure(self.bootstrap_room, reason)
+ self.kv_mgr.update_status(self.bootstrap_room, KVPoll.Failed)
+ self.conclude_state = KVPoll.Failed
+ return
self.target_tp_rank = self.prefill_info.target_tp_rank
self.target_tp_ranks = self.prefill_info.target_tp_ranks
self.target_cp_ranks = self.prefill_info.target_cp_ranks
@@ -1911,7 +2096,52 @@ class CommonKVReceiver(BaseKVReceiver):
self._setup_bootstrap_infos()
if self.conclude_state == KVPoll.Failed:
return
- self.kv_mgr.update_status(self.bootstrap_room, KVPoll.WaitingForInput)
+ if policy == "early":
+ self.kv_mgr.update_status(self.bootstrap_room, KVPoll.WaitingForInput)
+ return
+ self._prefill_wait_start = time.time()
+ try:
+ for info in self.bootstrap_infos:
+ sock, lock = self._connect_to_bootstrap_server(info)
+ with lock:
+ sock.send_multipart(
+ [
+ b"BOOTSTRAP",
+ str(self.bootstrap_room).encode("ascii"),
+ self.kv_mgr.local_ip.encode("ascii"),
+ str(self.kv_mgr.rank_port).encode("ascii"),
+ ]
+ )
+ except zmq.ZMQError:
+ self.abort()
+
+ def poll(self) -> KVPoll:
+ with self._lifecycle_lock:
+ return self._poll()
+
+ def _poll(self) -> KVPoll:
+ if self.conclude_state is not None:
+ return self.conclude_state
+ status = self.kv_mgr.check_status(self.bootstrap_room)
+ if status in (KVPoll.Success, KVPoll.Failed):
+ self.conclude_state = status
+ elif status == KVPoll.Bootstrapping and self._prefill_wait_start is not None:
+ if time.time() - self._prefill_wait_start >= self.kv_mgr.waiting_timeout:
+ self.abort()
+ self.kv_mgr.record_failure(
+ self.bootstrap_room, "Timed out waiting for prefill completion"
+ )
+ return KVPoll.Failed
+ elif status == KVPoll.WaitingForInput:
+ # Observe readiness before evaluating the old compute deadline.
+ # send_metadata starts the transfer timeout using this same clock.
+ if self._prefill_wait_start is not None:
+ self._prefill_wait_start = None
+ self.init_time = time.time()
+ timeout = self._check_waiting_timeout()
+ if timeout is not None:
+ return timeout
+ return status
def _setup_bootstrap_infos(self):
all_bootstrap_infos = []
@@ -2141,14 +2371,34 @@ class CommonKVReceiver(BaseKVReceiver):
return KVPoll.Failed
def clear(self) -> None:
+ with self._lifecycle_lock:
+ self._clear()
+
+ def _clear(self) -> None:
+ if not self._owns_bootstrap_room:
+ return
+ if (
+ self.kv_mgr.defer_decode_allocation
+ and self.kv_mgr.request_status.get(self.bootstrap_room) != KVPoll.Success
+ and not self.abort_notified
+ and getattr(self, "bootstrap_infos", None) is not None
+ ):
+ # Preserve the recorded root cause; this is cleanup, not a new abort.
+ self._send_abort_notification()
+ self.abort_notified = True
self.kv_mgr.request_status.pop(self.bootstrap_room, None)
self.kv_mgr.required_prefill_response_num_table.pop(self.bootstrap_room, None)
self.kv_mgr.prefill_response_tracker.pop(self.bootstrap_room, None)
self.kv_mgr.addr_to_rooms_tracker[self.bootstrap_addr].discard(
self.bootstrap_room
)
+ if self.kv_mgr.defer_decode_allocation:
+ self._owns_bootstrap_room = False
def abort(self):
+ if not self._owns_bootstrap_room:
+ self.conclude_state = KVPoll.Failed
+ return
self.kv_mgr.record_failure(
self.bootstrap_room,
"Aborted by AbortReq.",
@@ -2228,6 +2478,7 @@ class CommonKVBootstrapServer(BaseKVBootstrapServer):
self.follow_bootstrap_room: Optional[bool] = None
self.enable_dsa_cache_layer_split: Optional[bool] = None
self.prefill_http_port: Optional[int] = None
+ self.decode_allocation_policy = "early"
self.prefill_port_table: Dict[
int, Dict[int, Dict[int, Dict[int, PrefillRankInfo]]]
] = {}
@@ -2302,7 +2553,13 @@ class CommonKVBootstrapServer(BaseKVBootstrapServer):
text="DeepSeek-V4.1 DSpark PD layout differs across prefill ranks",
status=400,
)
+ policy = data.get("decode_allocation_policy", "early")
+ if self._registered_count and self.decode_allocation_policy != policy:
+ return web.Response(
+ text="Inconsistent prefill allocation policies", status=400
+ )
self.dsv41_spec_layout = dsv41_spec_layout
+ self.decode_allocation_policy = policy
if self.attn_tp_size is None:
self.attn_tp_size = attn_tp_size
@@ -2402,6 +2659,7 @@ class CommonKVBootstrapServer(BaseKVBootstrapServer):
),
enable_dsa_cache_layer_split=bool(self.enable_dsa_cache_layer_split),
prefill_http_port=self.prefill_http_port,
+ decode_allocation_policy=self.decode_allocation_policy,
)
payload = dataclasses.asdict(info)
if info.dsv41_spec_layout is None:
diff --git a/python/sglang/srt/disaggregation/decode.py b/python/sglang/srt/disaggregation/decode.py
index 69c189b1bc..536d4d9b57 100644
--- a/python/sglang/srt/disaggregation/decode.py
+++ b/python/sglang/srt/disaggregation/decode.py
@@ -738,9 +738,9 @@ class DecodePreallocQueue(DecodeHiCachePreallocMixin):
``is_rebootstrap`` marks a PD true-retraction request whose prefix KV
must be recomputed by the original prefill worker under the current
weights (rather than resumed from stale CPU KV). It otherwise follows the
- same bootstrap-handshake path as a fresh request; the ``/generate``
- dispatch happens later, after preallocation and ``send_metadata`` (see
- ``pop_preallocated``).
+ same bootstrap-handshake path as a fresh request. Early allocation
+ dispatches ``/generate`` after ``send_metadata``; deferred allocation
+ dispatches once the peer is resolved, before waiting for completion.
"""
# See `PrefillBootstrapQueue.add`. A retracted or rebootstrapping
# request owns a host KV backup that `retracted_queue` releases, and by
@@ -779,7 +779,7 @@ class DecodePreallocQueue(DecodeHiCachePreallocMixin):
prefill_dp_rank = self._resolve_prefill_dp_rank(req)
logger.debug(f"prefill_dp_rank: {prefill_dp_rank}")
if prefill_dp_rank is not None:
- decode_req.kv_receiver.init(prefill_dp_rank)
+ self._init_receiver(decode_req, prefill_dp_rank)
return
self.pending_reqs.append(decode_req)
@@ -1018,6 +1018,20 @@ class DecodePreallocQueue(DecodeHiCachePreallocMixin):
return resumed_reqs
+ def _init_receiver(self, decode_req: DecodeRequest, prefill_dp_rank: int) -> None:
+ decode_req.kv_receiver.init(prefill_dp_rank)
+ if (
+ get_disagg().disaggregation_decode_allocation_policy == "prefill_complete"
+ and decode_req.kv_receiver.conclude_state != KVPoll.Failed
+ ):
+ decode_req.req.disagg_prefill_dp_rank = prefill_dp_rank
+ # Deferred allocation cannot precede the recompute that makes its
+ # source ready. Keep the existing leader election and error path.
+ if decode_req.is_rebootstrap:
+ self.kv_manager.submit_prefill_recompute(
+ decode_req.kv_receiver, decode_req.req.build_rebootstrap_payload()
+ )
+
def _update_handshake_waiters(
self,
rids_to_check: Optional[List[str]] = None,
@@ -1225,7 +1239,7 @@ class DecodePreallocQueue(DecodeHiCachePreallocMixin):
self.pending_reqs = remaining
for decode_req, prefill_dp_rank in resolved:
- decode_req.kv_receiver.init(prefill_dp_rank)
+ self._init_receiver(decode_req, prefill_dp_rank)
def pop_preallocated(
self,
@@ -1747,7 +1761,10 @@ class DecodePreallocQueue(DecodeHiCachePreallocMixin):
state_indices,
**metadata_kwargs,
)
- if decode_req.is_rebootstrap:
+ if (
+ decode_req.is_rebootstrap
+ and get_disagg().disaggregation_decode_allocation_policy == "early"
+ ):
self.kv_manager.submit_prefill_recompute(
decode_req.kv_receiver,
decode_req.req.build_rebootstrap_payload(),
diff --git a/python/sglang/srt/disaggregation/fake/conn.py b/python/sglang/srt/disaggregation/fake/conn.py
index e25080ac58..a701cb062f 100644
--- a/python/sglang/srt/disaggregation/fake/conn.py
+++ b/python/sglang/srt/disaggregation/fake/conn.py
@@ -104,6 +104,11 @@ class FakeKVSender(BaseKVSender):
def get_transfer_metric(self) -> KVTransferMetric:
return KVTransferMetric()
+ def mark_prefill_complete(self) -> None:
+ # Warmup and health checks use a fake sender even with a real PD manager.
+ # They have no decode peer to admit or notify.
+ pass
+
def init(
self,
num_kv_indices: int,
diff --git a/python/sglang/srt/disaggregation/mooncake/conn.py b/python/sglang/srt/disaggregation/mooncake/conn.py
index 9f1ade96d8..394cc49c3e 100644
--- a/python/sglang/srt/disaggregation/mooncake/conn.py
+++ b/python/sglang/srt/disaggregation/mooncake/conn.py
@@ -2457,7 +2457,6 @@ class MooncakeKVManager(StagingManagerMixin, CommonKVManager):
and self.check_status(kv_chunk.room) == KVPoll.Failed
)
):
- self._staging_outstanding.pop(kv_chunk.room, None)
if kv_chunk.room in self.transfer_infos:
self.transfer_infos.pop(kv_chunk.room)
self.req_to_decode_prefix_len.pop(kv_chunk.room, None)
@@ -2468,6 +2467,10 @@ class MooncakeKVManager(StagingManagerMixin, CommonKVManager):
if key[0] == kv_chunk.room:
self._staging_ctx.prefetch_requested.discard(key)
self._staging_ctx.prefetched_rooms.discard(kv_chunk.room)
+ # Publish worker cleanup last: deferred allocation must not
+ # acknowledge Success while this worker can still erase
+ # bookkeeping for a subsequent transfer using the same room.
+ self._staging_outstanding.pop(kv_chunk.room, None)
except Exception as e:
# NOTE(shangming): Remove this when we make sure the transfer thread is bug-free
@@ -2822,7 +2825,6 @@ class MooncakeKVSender(MooncakeFailureExceptionMixin, CommonKVSender):
pp_rank,
req_has_disagg_prefill_dp_rank,
)
- self.conclude_state = None
self.init_time = time.time()
self._init_trace_ctx()
@@ -2864,11 +2866,15 @@ class MooncakeKVSender(MooncakeFailureExceptionMixin, CommonKVSender):
def poll(self) -> KVPoll:
if self.conclude_state is None:
status = self.kv_mgr.check_status(self.bootstrap_room)
- # Hold Success until all staging chunks transferred: a deferred
- # chunk can still be pending, and concluding now would drop it.
- if (
- status == KVPoll.Success
- and self.kv_mgr._staging_outstanding.get(self.bootstrap_room, 0) > 0
+ # Hold Success until all staging chunks transferred. With deferred
+ # allocation also wait for worker cleanup (the zero entry is only
+ # removed after cleanup), before permitting the room to be reused.
+ if status == KVPoll.Success and (
+ self.kv_mgr._staging_outstanding.get(self.bootstrap_room, 0) > 0
+ or (
+ self.kv_mgr.defer_decode_allocation
+ and self.bootstrap_room in self.kv_mgr._staging_outstanding
+ )
):
return KVPoll.Transferring
if status in (KVPoll.Success, KVPoll.Failed):
@@ -3072,20 +3078,6 @@ class MooncakeKVReceiver(MooncakeFailureExceptionMixin, CommonKVReceiver):
return
self.init_time = time.time()
- def poll(self) -> KVPoll:
- if self.conclude_state is not None:
- return self.conclude_state
-
- status = self.kv_mgr.check_status(self.bootstrap_room)
- if status in (KVPoll.Success, KVPoll.Failed):
- self.conclude_state = status
- elif status == KVPoll.WaitingForInput:
- timeout_result = self._check_waiting_timeout()
- if timeout_result is not None:
- return timeout_result
-
- return status
-
class MooncakeKVBootstrapServer(CommonKVBootstrapServer):
pass
diff --git a/python/sglang/srt/disaggregation/prefill.py b/python/sglang/srt/disaggregation/prefill.py
index 3f16ffcb52..ef39838cda 100644
--- a/python/sglang/srt/disaggregation/prefill.py
+++ b/python/sglang/srt/disaggregation/prefill.py
@@ -513,7 +513,12 @@ class PrefillBootstrapQueue:
failed_reqs.append(req)
elif poll == KVPoll.Bootstrapping:
if (
- req.prefill_attempt_count < get_disagg().optimistic_prefill_attempts
+ (
+ req.prefill_attempt_count
+ < get_disagg().optimistic_prefill_attempts
+ or get_disagg().disaggregation_decode_allocation_policy
+ == "prefill_complete"
+ )
and not req.is_retracted # engine paused
and not (
_uses_write_through_cache(self.scheduler.tree_cache)
@@ -929,6 +934,11 @@ class SchedulerDisaggregationPrefillMixin:
self.batch_result_processor.add_sampling_mask_return_values(
i, req, logits_output
)
+ if (
+ get_disagg().disaggregation_decode_allocation_policy
+ == "prefill_complete"
+ ):
+ req.disagg_kv_sender.mark_prefill_complete()
if not req.pending_bootstrap:
self.send_kv_chunk(req, last_chunk=True)
req.time_stats.set_prefill_transfer_queue_entry_time()
@@ -1261,7 +1271,11 @@ class SchedulerDisaggregationPrefillMixin:
if is_aborted(req):
# bootstrap failed
self.chunked_req = None
- elif self.has_bootstrapped_waiting_req():
+ elif (
+ get_disagg().disaggregation_decode_allocation_policy
+ != "prefill_complete"
+ and self.has_bootstrapped_waiting_req()
+ ):
# optimistic request yields to waiting requests
self.chunked_req = None
if not self.enable_overlap:
@@ -1586,7 +1600,11 @@ class SchedulerDisaggregationPrefillMixin:
# A fresh lookup budget for the new attempt, as after a retraction.
req.storage_prefetch_retry_attempts = 0
req.storage_prefetch_last_match_len = yielded_prefix_len or None
- if req.prefill_attempt_count >= max_attempts:
+ if (
+ req.prefill_attempt_count >= max_attempts
+ and get_disagg().disaggregation_decode_allocation_policy
+ != "prefill_complete"
+ ):
logger.info(
f"Req {req.rid} exhausted optimistic prefill attempts "
"falling back to bootstrap queue"
diff --git a/test/registered/e2e/disaggregation/test_prefill_complete_allocation.py b/test/registered/e2e/disaggregation/test_prefill_complete_allocation.py
new file mode 100644
index 0000000000..7cd7a0c1b5
--- /dev/null
+++ b/test/registered/e2e/disaggregation/test_prefill_complete_allocation.py
@@ -0,0 +1,390 @@
+"""Real, multi-chunk PD inference with deferred destination KV allocation."""
+
+import json
+import math
+import time
+import unittest
+import uuid
+from concurrent.futures import ThreadPoolExecutor
+
+import requests
+
+from sglang.test.ci.ci_register import register_cuda_ci
+from sglang.test.server_fixtures.disaggregation_fixture import (
+ PDDisaggregationServerBase,
+)
+
+register_cuda_ci(est_time=480, stage="extra-a", runner_config="2-gpu-large")
+
+
+class _PrefillCompleteServer(PDDisaggregationServerBase):
+ model = "Qwen/Qwen3-0.6B"
+
+ model_revision = "c1899de289a04d12100db370d81485cdf75e47ca"
+
+ policy = "prefill_complete"
+
+ @classmethod
+ def setUpClass(cls):
+ super().setUpClass()
+ common = [
+ "--revision",
+ cls.model_revision,
+ "--disaggregation-decode-allocation-policy",
+ cls.policy,
+ "--attention-backend",
+ "flashinfer",
+ "--mem-fraction-static",
+ "0.65",
+ "--max-total-tokens",
+ "32768",
+ "--context-length",
+ "8192",
+ "--max-running-requests",
+ "16",
+ "--cuda-graph-max-bs-decode",
+ "16",
+ "--enable-metrics",
+ ]
+ cls.extra_prefill_args = common + ["--chunked-prefill-size", "512"]
+ cls.extra_decode_args = list(common)
+ cls.launch_all()
+
+ def generate(self, text, *, input_logprob=False):
+ response = requests.post(
+ self.lb_url + "/generate",
+ json={
+ "text": text,
+ "sampling_params": {
+ "temperature": 0,
+ "max_new_tokens": 32,
+ "ignore_eos": True,
+ },
+ "return_logprob": True,
+ "return_input_logprob": input_logprob,
+ "logprob_start_len": 0 if input_logprob else -1,
+ },
+ timeout=120,
+ )
+ self.assertEqual(response.status_code, 200, response.text)
+ result = response.json()
+ meta = result["meta_info"]
+ self.assertEqual(meta["completion_tokens"], 32, result)
+ self.assertEqual(len(meta["output_token_logprobs"]), 32)
+ self.assertTrue(
+ all(math.isfinite(item[0]) for item in meta["output_token_logprobs"])
+ )
+ return result
+
+ def _run_concurrent_requests(self):
+ prompts = [
+ f"Example {i}. " + "Explain how a compiler works. " * 120 for i in range(8)
+ ]
+ with ThreadPoolExecutor(max_workers=8) as executor:
+ results = list(executor.map(self.generate, prompts))
+ self.assertEqual(len(results), len(prompts))
+ for process in (self.process_prefill, self.process_decode, self.process_lb):
+ self.assertIsNone(process.poll())
+
+ def _decode_load(self):
+ response = requests.get(
+ self.decode_url + "/v1/loads?include=core,disagg,queues", timeout=10
+ )
+ response.raise_for_status()
+ loads = response.json()["loads"]
+ self.assertEqual(len(loads), 1)
+ return loads[0]
+
+ def _flush_when_idle(self):
+ for url in (self.prefill_url, self.decode_url):
+ deadline = time.monotonic() + 30
+ while time.monotonic() < deadline:
+ response = requests.post(url + "/flush_cache", timeout=10)
+ if response.status_code == 200:
+ break
+ time.sleep(0.1)
+ else:
+ self.fail(f"Worker did not release the aborted request: {url}")
+
+
+class TestPrefillCompleteAllocation(_PrefillCompleteServer):
+ def test_multi_chunk_input_logprobs(self):
+ prompt = "The capital of France is Paris. " * 300
+ first = self.generate(prompt, input_logprob=True)
+ second = self.generate(prompt, input_logprob=True)
+ self.assertGreater(first["meta_info"]["prompt_tokens"], 512)
+ for result in (first, second):
+ self.assertGreaterEqual(
+ len(result["meta_info"]["input_token_logprobs"]),
+ result["meta_info"]["prompt_tokens"] - 1,
+ )
+ self.assertEqual(first["text"], second["text"])
+ self.assertEqual(
+ [x[1] for x in first["meta_info"]["output_token_logprobs"]],
+ [x[1] for x in second["meta_info"]["output_token_logprobs"]],
+ )
+
+ def test_full_prefix_reuse(self):
+ prompt = (
+ "A compiler translates source code into executable instructions. " * 250
+ )
+ first = self.generate(prompt)
+ second = self.generate(prompt)
+ self.assertGreater(second["meta_info"]["prompt_tokens"], 512)
+ self.assertGreaterEqual(
+ second["meta_info"]["cached_tokens"],
+ second["meta_info"]["prompt_tokens"] - 1,
+ )
+ self.assertEqual(first["text"], second["text"])
+ self.assertEqual(
+ [item[1] for item in first["meta_info"]["output_token_logprobs"]],
+ [item[1] for item in second["meta_info"]["output_token_logprobs"]],
+ )
+
+ def test_concurrent_requests_complete(self):
+ self._run_concurrent_requests()
+
+ def test_decode_abort_before_allocation_releases_prefill(self):
+ self._flush_when_idle()
+ rid = "prefill-complete-abort-" + uuid.uuid4().hex
+ response = requests.post(
+ self.prefill_url + "/slow_down",
+ json={"forward_sleep_time": 1.0},
+ timeout=10,
+ )
+ response.raise_for_status()
+ try:
+ with ThreadPoolExecutor(max_workers=1) as executor:
+ future = executor.submit(
+ requests.post,
+ self.lb_url + "/generate",
+ json={
+ "rid": rid,
+ "text": rid + " Explain how a compiler works." * 700,
+ "sampling_params": {
+ "temperature": 0,
+ "max_new_tokens": 32,
+ "ignore_eos": True,
+ },
+ },
+ timeout=60,
+ )
+ try:
+ deadline = time.monotonic() + 15
+ while time.monotonic() < deadline:
+ load = self._decode_load()
+ if load["disaggregation"]["decode_prealloc_queue_reqs"] > 0:
+ break
+ self.assertFalse(
+ future.done(),
+ "Request finished before the gate was observed",
+ )
+ time.sleep(0.1)
+ else:
+ self.fail("Did not observe the decode readiness wait")
+ self.assertEqual(load["num_used_tokens"], 0, load)
+ self.assertEqual(load["queues"]["prealloc_ready"], 0, load)
+ # Only abort decode: the readiness CANCEL must release the
+ # source request even though no KV metadata was published.
+ abort = requests.post(
+ self.decode_url + "/abort_request",
+ json={"rid": rid},
+ timeout=10,
+ )
+ abort.raise_for_status()
+ result = future.result(timeout=30)
+ self.assertIn("abort", result.text.lower(), result.text)
+ finally:
+ if not future.done():
+ for url in (self.prefill_url, self.decode_url):
+ requests.post(
+ url + "/abort_request", json={"rid": rid}, timeout=10
+ )
+ finally:
+ requests.post(
+ self.prefill_url + "/slow_down",
+ json={"forward_sleep_time": 0.0},
+ timeout=10,
+ ).raise_for_status()
+ self._flush_when_idle()
+ self.generate("The capital of France is")
+
+ def test_decode_abort_after_transfer_remains_usable(self):
+ rid = "prefill-complete-stream-abort-" + uuid.uuid4().hex
+ with requests.post(
+ self.lb_url + "/generate",
+ json={
+ "rid": rid,
+ "text": "Explain how a compiler works in detail.",
+ "stream": True,
+ "sampling_params": {
+ "temperature": 0,
+ "max_new_tokens": 4096,
+ "ignore_eos": True,
+ },
+ },
+ stream=True,
+ timeout=60,
+ ) as response:
+ response.raise_for_status()
+ for line in response.iter_lines():
+ if line.startswith(b"data: ") and line != b"data: [DONE]":
+ event = json.loads(line[6:])
+ if event["meta_info"]["completion_tokens"] > 0:
+ break
+ else:
+ self.fail("No token arrived before streaming abort")
+ requests.post(
+ self.decode_url + "/abort_request", json={"rid": rid}, timeout=10
+ ).raise_for_status()
+ self._flush_when_idle()
+ self.generate("The capital of France is")
+
+ def test_retract_resume_recomputes_before_reallocation(self):
+ payload = {
+ "text": "Explain how a compiler works in detail.",
+ "return_logprob": True,
+ "sampling_params": {
+ "temperature": 0,
+ "max_new_tokens": 256,
+ "ignore_eos": True,
+ },
+ }
+ rid = "prefill-complete-rebootstrap-" + uuid.uuid4().hex
+ requests.post(
+ self.decode_url + "/slow_down",
+ json={"forward_sleep_time": 0.01},
+ timeout=10,
+ ).raise_for_status()
+ paused = False
+ try:
+ with requests.post(
+ self.lb_url + "/generate",
+ json={**payload, "rid": rid, "stream": True},
+ stream=True,
+ timeout=90,
+ ) as response:
+ response.raise_for_status()
+ retractions = 0
+ final = None
+ emitted = []
+ for line in response.iter_lines():
+ if not line.startswith(b"data: ") or line == b"data: [DONE]":
+ continue
+ final = json.loads(line[6:])
+ self.assertNotIn("error", final)
+ meta = final["meta_info"]
+ output = [
+ (logprob, token_id)
+ for logprob, token_id, *_ in meta["output_token_logprobs"]
+ ]
+ self.assertEqual(len(output), meta["completion_tokens"])
+ # Recompute may change future token choices, but must not
+ # rewrite tokens or logprobs already emitted by this request.
+ self.assertEqual(output[: len(emitted)], emitted)
+ emitted = output
+ if (
+ retractions < 3
+ and final["meta_info"]["completion_tokens"] > retractions * 32
+ ):
+ requests.post(
+ self.decode_url + "/pause_generation",
+ json={"mode": "retract"},
+ timeout=15,
+ ).raise_for_status()
+ paused = True
+ # Retract pause frees destination KV and stages a true
+ # rebootstrap; flush must succeed before resuming it.
+ requests.post(
+ self.decode_url + "/flush_cache", timeout=10
+ ).raise_for_status()
+ requests.post(
+ self.decode_url + "/continue_generation",
+ json={"torch_empty_cache": False},
+ timeout=15,
+ ).raise_for_status()
+ paused = False
+ retractions += 1
+ self.assertIsNotNone(final)
+ self.assertEqual(retractions, 3)
+ self.assertGreaterEqual(final["meta_info"]["num_retractions"], 3, final)
+ self.assertEqual(final["meta_info"]["completion_tokens"], 256, final)
+ self.assertTrue(all(math.isfinite(logprob) for logprob, _ in emitted))
+ finally:
+ if paused:
+ requests.post(
+ self.decode_url + "/continue_generation", json={}, timeout=15
+ )
+ for url in (self.prefill_url, self.decode_url):
+ requests.post(url + "/abort_request", json={"rid": rid}, timeout=10)
+ requests.post(
+ self.decode_url + "/slow_down",
+ json={"forward_sleep_time": 0.0},
+ timeout=10,
+ ).raise_for_status()
+ self._flush_when_idle()
+ self.generate("The capital of France is")
+
+
+class TestPrefillCompleteRetry(_PrefillCompleteServer):
+ extra_prefill_env = {"SGLANG_TEST_FORCE_OPTIMISTIC_PREFILL_RETRY_PROB": "1"}
+
+ def _retry_count(self):
+ response = requests.get(self.prefill_url + "/metrics", timeout=10)
+ response.raise_for_status()
+ return sum(
+ float(line.split()[-1])
+ for line in response.text.splitlines()
+ if line.startswith("sglang:num_prefill_retries_total{")
+ )
+
+ def test_retry_after_attempt_budget_still_completes(self):
+ # The policy resolves to one optimistic attempt. Force that first
+ # attempt to yield, then verify real retries and eventual completion.
+ before = self._retry_count()
+ self._run_concurrent_requests()
+ self.assertGreaterEqual(self._retry_count() - before, 8)
+ self._flush_when_idle()
+
+
+class TestPrefillCompleteTimeout(_PrefillCompleteServer):
+ extra_decode_env = {"SGLANG_DISAGGREGATION_WAITING_TIMEOUT": "3"}
+
+ def test_readiness_timeout_releases_both_sides(self):
+ self._flush_when_idle()
+ requests.post(
+ self.prefill_url + "/slow_down",
+ json={"forward_sleep_time": 1.0},
+ timeout=10,
+ ).raise_for_status()
+ start = time.monotonic()
+ try:
+ response = requests.post(
+ self.lb_url + "/generate",
+ json={
+ "rid": "prefill-complete-timeout-" + uuid.uuid4().hex,
+ "text": "Explain how a compiler works. " * 700,
+ "sampling_params": {
+ "temperature": 0,
+ "max_new_tokens": 32,
+ "ignore_eos": True,
+ },
+ },
+ timeout=30,
+ )
+ self.assertIn("Timed out waiting for prefill completion", response.text)
+ self.assertLess(time.monotonic() - start, 15)
+ finally:
+ requests.post(
+ self.prefill_url + "/slow_down",
+ json={"forward_sleep_time": 0.0},
+ timeout=10,
+ ).raise_for_status()
+ # Do not explicitly abort either side: timeout handling must release
+ # the source as well as the destination before cache flush can succeed.
+ self._flush_when_idle()
+ self.generate("The capital of France is")
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/test/registered/unit/disaggregation/test_admission_abort_not_enqueued.py b/test/registered/unit/disaggregation/test_admission_abort_not_enqueued.py
index b8b7bdd981..d1d74837d7 100644
--- a/test/registered/unit/disaggregation/test_admission_abort_not_enqueued.py
+++ b/test/registered/unit/disaggregation/test_admission_abort_not_enqueued.py
@@ -22,7 +22,7 @@ them by then -- so it gates on its own `is_retracted` / `is_rebootstrap` flags.
import unittest
from array import array
from http import HTTPStatus
-from types import SimpleNamespace
+from types import MethodType, SimpleNamespace
from unittest.mock import MagicMock, patch
from sglang.srt.disaggregation.decode import DecodePreallocQueue
@@ -95,6 +95,8 @@ def _decode_queue(sched):
),
_resolve_prefill_dp_rank=MagicMock(return_value=0),
)
+ # Exercise the production initializer reached by the cache-hit fast path.
+ q._init_receiver = MethodType(DecodePreallocQueue._init_receiver, q)
return q
@@ -212,6 +214,10 @@ class TestAdmissionAbortNotEnqueued(CustomTestCase):
self.assertEqual(q.retracted_queue, [req])
else:
q._create_receiver_and_enqueue.assert_called_once()
+ receiver = (
+ q._create_receiver_and_enqueue.return_value.kv_receiver
+ )
+ receiver.init.assert_called_once_with(0)
def test_preempted_reentry_is_not_retired(self):
"""Preemption requeues through a bare `_add_request_to_queue`.
@@ -243,6 +249,8 @@ class TestAdmissionAbortNotEnqueued(CustomTestCase):
q = _decode_queue(sched)
_admit_decode(q, req)
q._create_receiver_and_enqueue.assert_called_once()
+ receiver = q._create_receiver_and_enqueue.return_value.kv_receiver
+ receiver.init.assert_called_once_with(0)
sched.retire_unadmitted_request.assert_not_called()
def test_valid_request_still_admitted(self):
@@ -258,6 +266,8 @@ class TestAdmissionAbortNotEnqueued(CustomTestCase):
q = _decode_queue(sched)
_admit_decode(q, req)
q._create_receiver_and_enqueue.assert_called_once()
+ receiver = q._create_receiver_and_enqueue.return_value.kv_receiver
+ receiver.init.assert_called_once_with(0)
sched.retire_unadmitted_request.assert_not_called()
def test_null_mode_is_deliberately_untouched(self):
diff --git a/test/registered/unit/disaggregation/test_deferred_bootstrap.py b/test/registered/unit/disaggregation/test_deferred_bootstrap.py
new file mode 100644
index 0000000000..52282edebf
--- /dev/null
+++ b/test/registered/unit/disaggregation/test_deferred_bootstrap.py
@@ -0,0 +1,158 @@
+"""Lifecycle tests for the shared pre-allocation bootstrap rendezvous."""
+
+import itertools
+import unittest
+from unittest.mock import patch
+
+from sglang.srt.disaggregation.common.bootstrap import DeferredBootstrap
+from sglang.test.ci.ci_register import register_cpu_ci
+
+register_cpu_ci(est_time=1, suite="base-a-test-cpu")
+
+
+class TestDeferredBootstrap(unittest.TestCase):
+ def setUp(self):
+ self.table = DeferredBootstrap(timeout=10)
+ self.owner = object()
+ self.endpoint = ("decode", 1234)
+
+ def test_either_arrival_order_and_no_completion_before_source(self):
+ for order in (
+ ("open", "register", "complete"),
+ ("register", "open", "complete"),
+ ("open", "complete", "register"),
+ ):
+ with self.subTest(order=order):
+ table = DeferredBootstrap(10)
+ result = None
+ for action in order:
+ if action == "open":
+ state = table.open(1, self.owner)
+ self.assertFalse(state.failed)
+ elif action == "register":
+ result = table.register(1, self.endpoint)
+ else:
+ result = table.complete(1, self.owner)
+ self.assertEqual(result, (self.endpoint, False))
+
+ def test_cancel_at_every_point_prevents_later_ready(self):
+ for order in itertools.permutations(("open", "register", "cancel")):
+ with self.subTest(order=order):
+ table = DeferredBootstrap(10)
+ notifications = []
+ for action in order:
+ if action == "open":
+ table.open(1, self.owner)
+ elif action == "register":
+ notifications.append(table.register(1, self.endpoint))
+ else:
+ notifications.append(table.fail(1))
+ self.assertTrue(table.rooms[1].failed)
+ self.assertIsNone(table.complete(1, self.owner))
+ self.assertIn((self.endpoint, True), notifications)
+
+ def test_duplicate_source_cannot_replace_owner(self):
+ self.table.open(1, self.owner)
+ other = object()
+ self.assertIsNone(self.table.open(1, other))
+ self.assertIsNone(self.table.complete(1, other))
+ self.assertIsNone(self.table.close(1, other))
+ self.assertIs(self.table.rooms[1].owner, self.owner)
+
+ def test_endpoint_registration_is_idempotent_not_replaceable(self):
+ self.table.open(1, self.owner)
+ self.table.complete(1, self.owner)
+ self.assertEqual(self.table.register(1, self.endpoint), (self.endpoint, False))
+ self.assertIsNone(self.table.register(1, self.endpoint))
+ self.assertIsNone(self.table.complete(1, self.owner))
+ other = ("other", 9999)
+ self.assertEqual(self.table.register(1, other), (other, True))
+ self.assertEqual(self.table.rooms[1].endpoint, self.endpoint)
+
+ def test_clear_keeps_failure_for_late_receiver(self):
+ self.table.open(1, self.owner)
+ self.table.close(1, self.owner)
+ self.assertEqual(self.table.register(1, self.endpoint), (self.endpoint, True))
+ self.assertIsNone(self.table.complete(1, self.owner))
+
+ def test_success_resets_room_for_either_arrival_order(self):
+ for receiver_first in (True, False):
+ with self.subTest(receiver_first=receiver_first):
+ self.table.open(1, self.owner)
+ self.table.register(1, self.endpoint)
+ self.table.complete(1, self.owner)
+ self.assertEqual(
+ self.table.close(1, self.owner, success=True),
+ (self.endpoint, False),
+ )
+ self.assertNotIn(1, self.table.rooms)
+ new_owner = object()
+ if receiver_first:
+ self.assertIsNone(self.table.register(1, self.endpoint))
+ state = self.table.open(1, new_owner)
+ self.assertFalse(state.ready)
+ self.assertFalse(state.failed)
+ if not receiver_first:
+ self.assertIsNone(self.table.register(1, self.endpoint))
+ # A repeated cleanup from the retired sender cannot close this one.
+ self.assertIsNone(self.table.close(1, self.owner, success=True))
+ self.assertEqual(
+ self.table.complete(1, new_owner), (self.endpoint, False)
+ )
+ self.table.close(1, new_owner, success=True)
+
+ def test_success_cannot_reset_cancelled_room(self):
+ self.table.open(1, self.owner)
+ self.table.fail(1)
+ self.table.close(1, self.owner, success=True)
+ self.assertEqual(self.table.register(1, self.endpoint), (self.endpoint, True))
+
+ def test_expire_unmatched_and_terminal_rooms_not_active_sources(self):
+ with patch(
+ "sglang.srt.disaggregation.common.bootstrap.time.monotonic", return_value=0
+ ):
+ self.table.open(1, self.owner)
+ self.table.register(2, self.endpoint)
+ self.table.fail(3)
+ self.table.open(4, object())
+ owner = self.table.rooms[4].owner
+ self.table.close(4, owner)
+ with patch(
+ "sglang.srt.disaggregation.common.bootstrap.time.monotonic", return_value=11
+ ):
+ self.table.register(5, self.endpoint)
+ self.assertEqual(set(self.table.rooms), {1, 5})
+ self.assertIs(self.table.rooms[1].owner, self.owner)
+
+ def test_cancelled_source_owns_cleanup(self):
+ self.table.fail(1)
+ state = self.table.open(1, self.owner)
+ self.assertTrue(state.failed)
+ self.assertIs(state.owner, self.owner)
+ self.table.close(1, self.owner)
+ self.assertIsNone(state.owner)
+
+ def test_abort_then_clear_does_not_send_duplicate_failures(self):
+ self.table.open(1, self.owner)
+ self.table.register(1, self.endpoint)
+ self.assertEqual(self.table.fail(1), (self.endpoint, True))
+ self.assertIsNone(self.table.fail(1))
+ self.assertIsNone(self.table.close(1, self.owner))
+ # A later endpoint must still learn that the source is gone.
+ self.assertEqual(self.table.register(1, self.endpoint), (self.endpoint, True))
+
+ def test_no_lock_escapes_any_operation(self):
+ for operation in (
+ lambda: self.table.open(1, self.owner),
+ lambda: self.table.register(1, self.endpoint),
+ lambda: self.table.complete(1, self.owner),
+ lambda: self.table.fail(1),
+ lambda: self.table.close(1, self.owner),
+ ):
+ operation()
+ self.assertTrue(self.table.lock.acquire(blocking=False))
+ self.table.lock.release()
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/test/registered/unit/disaggregation/test_nixl_sender_failure_cleanup.py b/test/registered/unit/disaggregation/test_nixl_sender_failure_cleanup.py
index 91d6a33b5b..c0bbf20642 100644
--- a/test/registered/unit/disaggregation/test_nixl_sender_failure_cleanup.py
+++ b/test/registered/unit/disaggregation/test_nixl_sender_failure_cleanup.py
@@ -28,6 +28,7 @@ class TestNixlSenderFailureCleanup(unittest.TestCase):
# happening, spy on clear() here and test its contents where clear()
# is tested, or build a real manager instance.
sender.kv_mgr = SimpleNamespace(
+ deferred_bootstrap=None,
enable_staging=True,
_staging_ctx=staging_ctx,
request_status={room: object()},
diff --git a/test/registered/unit/disaggregation/test_pd_role_switch.py b/test/registered/unit/disaggregation/test_pd_role_switch.py
index adea135539..7114d5facc 100644
--- a/test/registered/unit/disaggregation/test_pd_role_switch.py
+++ b/test/registered/unit/disaggregation/test_pd_role_switch.py
@@ -317,7 +317,8 @@ class TestPdRoleSwitchStartupValidation(unittest.TestCase):
speculative_algorithm=None,
)
base.update(kw)
- return SimpleNamespace(**base)
+ # Keep the production defaults for fields read by the shared PD hook.
+ return ServerArgs(model_path="dummy", **base)
def _run(self, sa):
from sglang.srt.arg_groups.pd_disaggregation_hook import (
diff --git a/test/registered/unit/disaggregation/test_prefill_abort_result_cleanup.py b/test/registered/unit/disaggregation/test_prefill_abort_result_cleanup.py
index 8152440032..5ccfef15fa 100644
--- a/test/registered/unit/disaggregation/test_prefill_abort_result_cleanup.py
+++ b/test/registered/unit/disaggregation/test_prefill_abort_result_cleanup.py
@@ -17,6 +17,7 @@ from sglang.srt.mem_cache.base_prefix_cache import (
)
from sglang.srt.runtime_context import get_context
from sglang.test.ci.ci_register import register_cpu_ci
+from sglang.test.test_utils import CustomTestCase
register_cpu_ci(est_time=1, suite="base-a-test-cpu")
@@ -268,5 +269,74 @@ def test_sampling_mask_abort_preserves_error_and_releases_once(
scheduler.send_kv_chunk.assert_not_called()
+class TestPrefillCompleteResult(CustomTestCase):
+ def setUp(self):
+ super().setUp()
+ override = get_context().override_server_args(
+ disaggregation_decode_allocation_policy="prefill_complete"
+ )
+ override.install()
+ self.addCleanup(override.restore)
+
+ @patch("sglang.srt.disaggregation.prefill.checkpoint_kv_cache")
+ def test_ready_observes_accepted_token_and_speculative_sidecars(self, cache):
+ scheduler = _Scheduler()
+ scheduler.spec_algorithm = SimpleNamespace(is_eagle=lambda: True)
+ req = _Req(inflight_middle_chunks=0)
+ req.to_finish = None
+ req.time_stats.set_prefill_transfer_queue_entry_time = Mock()
+ draft = SimpleNamespace(
+ hidden_states=torch.tensor([[1.0, 2.0]]),
+ topk_p=torch.tensor([[0.75, 0.25]]),
+ topk_index=torch.tensor([[3, 4]]),
+ dsa_topk_indices=torch.tensor([[5, 6]]),
+ )
+ batch = _batch(req)
+ batch.spec_info = draft
+ result = GenerationBatchResult(
+ next_token_ids=torch.tensor([11]), next_draft_input=draft
+ )
+ observed = []
+
+ def ready():
+ observed.append(
+ (
+ list(req.output_ids),
+ req.hidden_states_tensor.tolist(),
+ req.output_dsa_topk_indices.tolist(),
+ req in scheduler.disagg_prefill_inflight_queue,
+ )
+ )
+
+ req.disagg_kv_sender.mark_prefill_complete.side_effect = ready
+ scheduler.process_batch_result_disagg_prefill(batch, result)
+ self.assertEqual(observed, [([11], [1.0, 2.0], [5, 6], True)])
+ self.assertTrue(req.pending_bootstrap)
+ scheduler.send_kv_chunk.assert_not_called()
+
+ @patch("sglang.srt.disaggregation.prefill.release_kv_cache", side_effect=_free_req)
+ def test_aborted_final_result_never_publishes_ready(self, release):
+ scheduler = _Scheduler()
+ req = _Req(inflight_middle_chunks=0)
+ scheduler.process_batch_result_disagg_prefill(_batch(req), _result())
+ self.assertTrue(req.finished())
+ self.assertFalse(req.kv.holds_kv)
+ self.assertEqual(scheduler.disagg_prefill_inflight_queue, [])
+ req.disagg_kv_sender.mark_prefill_complete.assert_not_called()
+
+ @patch("sglang.srt.disaggregation.prefill.should_force_retry", return_value=True)
+ def test_discarded_attempt_never_publishes_ready(self, force_retry):
+ scheduler = _Scheduler()
+ req = _Req(inflight_middle_chunks=0)
+ req.to_finish = None
+ retried = []
+ scheduler.optimistic_release_and_requeue = lambda r: retried.append(r)
+ scheduler.process_batch_result_disagg_prefill(_batch(req), _result())
+ self.assertEqual(retried, [req])
+ self.assertEqual(req.output_ids, [])
+ self.assertEqual(scheduler.disagg_prefill_inflight_queue, [])
+ req.disagg_kv_sender.mark_prefill_complete.assert_not_called()
+
+
if __name__ == "__main__":
raise SystemExit(pytest.main([__file__, "-v"]))
diff --git a/test/registered/unit/disaggregation/test_prefill_complete_integration.py b/test/registered/unit/disaggregation/test_prefill_complete_integration.py
new file mode 100644
index 0000000000..284fc93dcd
--- /dev/null
+++ b/test/registered/unit/disaggregation/test_prefill_complete_integration.py
@@ -0,0 +1,917 @@
+"""Exercise the allocation policy through production transport and queue methods."""
+
+import threading
+import time
+import unittest
+from collections import defaultdict, deque
+from concurrent.futures import ThreadPoolExecutor, TimeoutError
+from types import MethodType, SimpleNamespace
+from unittest.mock import MagicMock, Mock, patch
+
+from sglang.srt.arg_groups.overrides import resolving_view
+from sglang.srt.arg_groups.pd_disaggregation_hook import handle_pd_disaggregation
+from sglang.srt.disaggregation.base import KVPoll
+from sglang.srt.disaggregation.common.bootstrap import DeferredBootstrap
+from sglang.srt.disaggregation.common.conn import CommonKVReceiver, PrefillServerInfo
+from sglang.srt.disaggregation.decode import DecodePreallocQueue, DecodeRequest
+from sglang.srt.disaggregation.fake.conn import FakeKVSender
+from sglang.srt.disaggregation.mooncake.conn import (
+ MooncakeKVManager,
+ MooncakeKVReceiver,
+ MooncakeKVSender,
+)
+from sglang.srt.disaggregation.prefill import SchedulerDisaggregationPrefillMixin
+from sglang.srt.runtime_context import get_context, publish, reset_context
+from sglang.srt.server_args import ServerArgs
+from sglang.test.ci.ci_register import register_cpu_ci
+from sglang.test.test_utils import CustomTestCase
+
+register_cpu_ci(est_time=8, suite="base-a-test-cpu")
+
+
+class TestAllocationConfiguration(CustomTestCase):
+ def test_policy_owns_optimistic_prefill_dependency(self):
+ args = ServerArgs(
+ model_path="dummy",
+ disaggregation_mode="prefill",
+ disaggregation_decode_allocation_policy="prefill_complete",
+ )
+ handle_pd_disaggregation(args)
+ self.assertEqual(resolving_view(args).optimistic_prefill_attempts, 1)
+ # Resolution must not rewrite the raw user input.
+ self.assertEqual(args.optimistic_prefill_attempts, 0)
+
+ def test_early_does_not_enable_optimistic_prefill(self):
+ args = ServerArgs(model_path="dummy", disaggregation_mode="prefill")
+ handle_pd_disaggregation(args)
+ self.assertEqual(resolving_view(args).optimistic_prefill_attempts, 0)
+
+ def test_dp_attention_topology_is_supported(self):
+ for size in (1, 4, 8):
+ with self.subTest(size=size):
+ args = ServerArgs(
+ model_path="dummy",
+ disaggregation_mode="prefill",
+ disaggregation_decode_allocation_policy="prefill_complete",
+ tp_size=size,
+ dp_size=size,
+ enable_dp_attention=size > 1,
+ )
+ handle_pd_disaggregation(args)
+ self.assertEqual(resolving_view(args).optimistic_prefill_attempts, 1)
+
+ def test_unsupported_modes_fail_explicitly(self):
+ combinations = [
+ {"disaggregation_mode": "null"},
+ {"disaggregation_transfer_backend": "nixl"},
+ {"tp_size": 2},
+ {"pp_size": 2},
+ {"attn_cp_size": 2},
+ {"dcp_size": 2},
+ {"enable_hisparse": True},
+ {
+ "disaggregation_mode": "decode",
+ "disaggregation_decode_host_receive_threshold": 0.5,
+ },
+ {"enable_pdmux": True},
+ {"enable_pd_role_switch": True},
+ {"language_only": True},
+ {
+ "enable_hierarchical_cache": True,
+ "hicache_write_policy": "write_through",
+ },
+ {
+ "disaggregation_mode": "decode",
+ "disaggregation_decode_enable_radix_cache": True,
+ },
+ ]
+ for changed in combinations:
+ with self.subTest(changed=changed):
+ values = dict(
+ model_path="dummy",
+ disaggregation_mode="prefill",
+ disaggregation_decode_allocation_policy="prefill_complete",
+ )
+ values.update(changed)
+ with self.assertRaisesRegex(ValueError, "prefill_complete"):
+ handle_pd_disaggregation(ServerArgs(**values))
+
+
+class TestAllocationIntegration(CustomTestCase):
+ def setUp(self):
+ super().setUp()
+ reset_context()
+ self.addCleanup(reset_context)
+ publish(ServerArgs(model_path="dummy"), role="tokenizer")
+ override = get_context().override_server_args(
+ disaggregation_decode_allocation_policy="prefill_complete"
+ )
+ override.install()
+ self.addCleanup(override.restore)
+ self.wire = deque()
+ self.prefill = self.manager("prefill", 1)
+ self.decode = self.manager("decode", 2)
+ self.endpoints = {("prefill", 1): self.prefill, ("decode", 2): self.decode}
+
+ def manager(self, ip, port):
+ mgr = MooncakeKVManager.__new__(MooncakeKVManager)
+ mgr.local_ip, mgr.rank_port = ip, port
+ mgr.request_status = {}
+ mgr.failure_lock = threading.Lock()
+ mgr.failure_records = {}
+ mgr.waiting_timeout = 5
+ mgr.bootstrap_timeout = 3
+ mgr.req_to_decode_prefix_len = {}
+ mgr.transfer_infos = {}
+ mgr._deferred_ack_targets = {}
+ mgr._deferred_ack_poisoned_rooms = set()
+ mgr.required_prefill_response_num_table = {}
+ mgr.prefill_response_tracker = defaultdict(set)
+ mgr.addr_to_rooms_tracker = defaultdict(set)
+ mgr.defer_decode_allocation = True
+ mgr.deferred_bootstrap = DeferredBootstrap(8) if ip == "prefill" else None
+ mgr.enable_trace = False
+ mgr.enable_staging = False
+ mgr.is_dummy_cp_rank = False
+ mgr._staging_outstanding = {}
+ mgr._prefill_unique_rank = Mock(return_value=0)
+ mgr.get_session_id = Mock(return_value="decode-session")
+ mgr._send_multipart_locked = lambda endpoint, msg, **kw: self.wire.append(
+ (endpoint, msg)
+ )
+ return mgr
+
+ def flush(self):
+ while self.wire:
+ endpoint, msg = self.wire.popleft()
+ if endpoint == "tcp://prefill:1":
+ self.prefill._handle_bootstrap_message(msg)
+ else:
+ self.assertEqual(endpoint, "tcp://decode:2")
+ room, status, rank, reason = self.decode.parse_kv_status_message(msg)
+ self.decode.apply_prefill_status(
+ bootstrap_room=room,
+ status=status,
+ prefill_rank=rank,
+ failure_reason=reason,
+ )
+
+ def sender(self, room=1):
+ return MooncakeKVSender(self.prefill, "prefill:8998", room, [0], 0)
+
+ def receiver(self, room=1, policy="prefill_complete"):
+ receiver = MooncakeKVReceiver(self.decode, "prefill:8998", room)
+ receiver.bootstrap_infos = [{"rank_ip": "prefill", "rank_port": 1}]
+ self.decode.prefill_info_table = {
+ receiver.bootstrap_addr: PrefillServerInfo(
+ attn_tp_size=1,
+ attn_cp_size=1,
+ dp_size=1,
+ pp_size=1,
+ page_size=1,
+ kv_cache_dtype=None,
+ follow_bootstrap_room=True,
+ decode_allocation_policy=policy,
+ target_tp_rank=0,
+ target_tp_ranks=[0],
+ target_cp_ranks=[0],
+ target_pp_ranks=[0],
+ required_dst_info_num=1,
+ required_prefill_response_num=1,
+ )
+ }
+ sock = SimpleNamespace(
+ send_multipart=lambda msg: self.wire.append(("tcp://prefill:1", msg))
+ )
+ receiver._connect_to_bootstrap_server = Mock(
+ return_value=(sock, threading.Lock())
+ )
+ with patch.object(receiver, "_setup_bootstrap_infos"):
+ receiver.init(0)
+ return receiver
+
+ def test_receiver_hides_allocatable_status_until_source_finishes(self):
+ sender, receiver = self.sender(), self.receiver()
+ self.assertEqual(receiver.poll(), KVPoll.Bootstrapping)
+ self.flush()
+ self.assertEqual(receiver.poll(), KVPoll.Bootstrapping)
+ sender.mark_prefill_complete()
+ self.flush()
+ self.assertEqual(receiver.poll(), KVPoll.WaitingForInput)
+
+ def test_early_receiver_stays_allocatable_without_control_registration(self):
+ override = get_context().override_server_args(
+ disaggregation_decode_allocation_policy="early"
+ )
+ override.install()
+ self.addCleanup(override.restore)
+ self.decode.defer_decode_allocation = False
+ receiver = self.receiver(policy="early")
+ self.assertEqual(receiver.poll(), KVPoll.WaitingForInput)
+ self.assertIsNone(receiver.init_time)
+ self.assertEqual(list(self.wire), [])
+
+ def test_shared_receive_wrapper_preserves_native_abort_and_metadata(self):
+ sender = self.sender()
+ bootstrap = [b"BOOTSTRAP", b"1", b"decode", b"2"]
+ abort = [b"ABORT", b"1", b"decode", b"2"]
+ metadata = [b"1", b"decode", b"2", b"session", b"indices"]
+ sock = Mock(recv_multipart=Mock(side_effect=[bootstrap, abort, metadata]))
+ recv = self.prefill._make_worker_recv(sock)
+ self.assertIsNone(recv())
+ self.assertEqual(recv(), abort)
+ self.assertEqual(sender.poll(), KVPoll.Failed)
+ self.assertEqual(recv(), metadata)
+
+ def test_repeated_transfers_reuse_room_after_success_cleanup(self):
+ retired = []
+ for receiver_first in (True, False, True):
+ with self.subTest(receiver_first=receiver_first, iteration=len(retired)):
+ if receiver_first:
+ receiver = self.receiver()
+ self.flush()
+ sender = self.sender()
+ else:
+ sender, receiver = self.sender(), self.receiver()
+ self.flush()
+ self.assertEqual(receiver.poll(), KVPoll.Bootstrapping)
+ for old_sender, old_receiver in retired:
+ old_sender.clear()
+ old_sender.abort()
+ old_receiver.clear()
+ self.decode._fail_prefill_recompute(old_receiver, "late HTTP error")
+ self.assertEqual(sender.poll(), KVPoll.Bootstrapping)
+ self.assertNotIn(1, self.decode.failure_records)
+ sender.mark_prefill_complete()
+ self.flush()
+ self.assertEqual(receiver.poll(), KVPoll.WaitingForInput)
+ # Duplicate control registration must not enqueue another ready
+ # message that could outlive the transfer's completion.
+ self.prefill._handle_bootstrap_message(
+ [b"BOOTSTRAP", b"1", b"decode", b"2"]
+ )
+ self.assertEqual(list(self.wire), [])
+ self.prefill.conclude_transfer(bootstrap_room=1, status=KVPoll.Success)
+ self.flush()
+ self.assertEqual(receiver.poll(), KVPoll.WaitingForInput)
+ self.assertEqual(sender.poll(), KVPoll.Success)
+ sender.clear()
+ self.flush()
+ self.assertEqual(receiver.poll(), KVPoll.Success)
+ receiver.clear()
+ self.assertNotIn(1, self.prefill.deferred_bootstrap.rooms)
+ self.assertNotIn(1, self.prefill.request_status)
+ self.assertEqual(list(self.wire), [])
+ retired.append((sender, receiver))
+
+ def test_completion_before_receiver_uses_same_bootstrap(self):
+ sender = self.sender()
+ sender.mark_prefill_complete()
+ receiver = self.receiver()
+ self.flush()
+ self.assertEqual(receiver.poll(), KVPoll.WaitingForInput)
+
+ def test_receiver_before_sender_uses_no_subscription_retries(self):
+ receiver = self.receiver()
+ self.flush()
+ self.assertEqual(receiver.poll(), KVPoll.Bootstrapping)
+ sender = self.sender()
+ sender.mark_prefill_complete()
+ self.flush()
+ self.assertEqual(receiver.poll(), KVPoll.WaitingForInput)
+ self.assertEqual(list(self.wire), [])
+
+ def test_sender_abort_fails_decoder_before_metadata(self):
+ sender, receiver = self.sender(), self.receiver()
+ self.flush()
+ sender.abort()
+ self.flush()
+ self.assertEqual(receiver.poll(), KVPoll.Failed)
+ sender.clear()
+ self.assertIsNone(self.prefill.deferred_bootstrap.rooms[1].owner)
+
+ def test_sender_failure_before_receiver_is_not_lost(self):
+ sender = self.sender()
+ sender.abort()
+ sender.clear()
+ receiver = self.receiver()
+ self.flush()
+ self.assertEqual(receiver.poll(), KVPoll.Failed)
+
+ def test_receiver_abort_notifies_source_using_normal_abort(self):
+ sender, receiver = self.sender(), self.receiver()
+ self.flush()
+ receiver.abort()
+ self.assertEqual(self.wire[0][1][0], b"ABORT")
+ self.flush()
+ self.assertEqual(sender.poll(), KVPoll.Failed)
+ sender.clear()
+ receiver.clear()
+ self.assertNotIn(1, self.decode.request_status)
+
+ def test_cancel_before_sender_creation_is_not_lost(self):
+ receiver = self.receiver()
+ receiver.abort()
+ self.flush()
+ sender = self.sender()
+ self.assertEqual(sender.poll(), KVPoll.Failed)
+ sender.clear()
+ self.assertNotIn(1, self.prefill.request_status)
+
+ def test_readiness_timeout_aborts_before_allocation(self):
+ sender, receiver = self.sender(), self.receiver()
+ receiver._prefill_wait_start = 0
+ self.assertEqual(receiver.poll(), KVPoll.Failed)
+ self.flush()
+ self.assertEqual(sender.poll(), KVPoll.Failed)
+
+ def test_ready_observed_after_compute_deadline_wins(self):
+ sender, receiver = self.sender(), self.receiver()
+ self.flush()
+ receiver._prefill_wait_start = 0
+ sender.mark_prefill_complete()
+ self.flush()
+ self.assertEqual(receiver.poll(), KVPoll.WaitingForInput)
+ self.assertIsNone(receiver._prefill_wait_start)
+
+ def test_admission_timeout_excludes_prefill_compute(self):
+ sender, receiver = self.sender(), self.receiver()
+ sender.init_time = 0
+ self.assertEqual(sender.poll(), KVPoll.Bootstrapping)
+ self.flush()
+ sender.mark_prefill_complete()
+ self.assertEqual(sender.poll(), KVPoll.Bootstrapping)
+ sender._prefill_complete_time = 0
+ self.assertEqual(sender.poll(), KVPoll.Failed)
+ self.flush()
+ self.assertEqual(receiver.poll(), KVPoll.Failed)
+
+ def test_failure_wins_over_late_ready(self):
+ sender, receiver = self.sender(), self.receiver()
+ self.flush()
+ sender.mark_prefill_complete()
+ sender.abort()
+ # Deliver failure before the already-emitted ready status.
+ self.wire.reverse()
+ self.flush()
+ self.assertEqual(receiver.poll(), KVPoll.Failed)
+
+ def test_duplicate_sender_does_not_crash_or_clear_owner(self):
+ sender, receiver = self.sender(), self.receiver()
+ duplicate = self.sender()
+ self.assertEqual(duplicate.poll(), KVPoll.Failed)
+ duplicate.clear()
+ self.assertIs(self.prefill.deferred_bootstrap.rooms[1].owner, sender)
+ sender.clear()
+ self.flush()
+ self.assertEqual(receiver.poll(), KVPoll.Failed)
+
+ def test_duplicate_receiver_does_not_crash_or_clear_owner(self):
+ sender, receiver = self.sender(), self.receiver()
+ self.flush()
+ duplicate = self.receiver()
+ self.assertEqual(duplicate.poll(), KVPoll.Failed)
+ duplicate.clear()
+ self.assertIn(1, self.decode.request_status)
+ self.assertIn(1, self.decode.addr_to_rooms_tracker[receiver.bootstrap_addr])
+ receiver.clear()
+ self.flush()
+ self.assertEqual(sender.poll(), KVPoll.Failed)
+
+ def test_rejected_sender_stays_failed_after_original_clears(self):
+ original = self.sender()
+ duplicate = self.sender()
+ original.clear()
+ self.assertEqual(duplicate.poll(), KVPoll.Failed)
+ duplicate.clear()
+
+ def test_fake_warmup_sender_never_registers_deferred_bootstrap(self):
+ sender = FakeKVSender(self.prefill, "fake:0", 0, [0], 0)
+ sender.init(0)
+ sender.mark_prefill_complete()
+ self.assertEqual(sender.poll(), KVPoll.WaitingForInput)
+ sender.send([])
+ self.assertEqual(sender.poll(), KVPoll.Success)
+ self.assertEqual(self.prefill.deferred_bootstrap.rooms, {})
+ self.assertEqual(list(self.wire), [])
+
+ def test_abort_after_metadata_uses_existing_drain_protocol(self):
+ receiver = self.receiver()
+ receiver._prefill_wait_start = None
+ receiver.init_time = time.time()
+ self.wire.clear()
+ with patch.object(CommonKVReceiver, "_send_abort_notification") as notify:
+ receiver.abort()
+ notify.assert_called_once()
+ self.assertEqual(receiver.conclude_state, KVPoll.Failed)
+
+ def test_mismatched_peer_fails_request_before_destination_registration(self):
+ receiver = self.receiver()
+ self.decode.prefill_info_table = {
+ receiver.bootstrap_addr: PrefillServerInfo(
+ attn_tp_size=1,
+ attn_cp_size=1,
+ dp_size=1,
+ pp_size=1,
+ page_size=1,
+ kv_cache_dtype=None,
+ follow_bootstrap_room=True,
+ decode_allocation_policy="early",
+ )
+ }
+ with patch.object(receiver, "_setup_bootstrap_infos") as setup:
+ CommonKVReceiver.init(receiver, 0)
+ self.assertEqual(receiver.conclude_state, KVPoll.Failed)
+ self.assertIn("policy mismatch", self.decode.failure_records[1])
+ setup.assert_not_called()
+
+ def test_scheduler_does_not_reserve_request_rows_for_unready_prefill(self):
+ sender, receiver = self.sender(), self.receiver()
+ req = SimpleNamespace(
+ rid="req",
+ bootstrap_room=1,
+ finished_reason=None,
+ return_logprob=False,
+ time_stats=MagicMock(),
+ )
+ entry = DecodeRequest(req=req, kv_receiver=receiver)
+ queue = DecodePreallocQueue.__new__(DecodePreallocQueue)
+ queue.queue, queue.pending_reqs = [entry], []
+ queue.pp_size, queue.tp_rank = 1, 0
+ queue.gloo_group = object()
+ queue._resolve_pending_reqs = Mock()
+ queue._uses_swa_tail_prealloc = Mock(return_value=False)
+ queue.token_to_kv_pool_allocator = SimpleNamespace(
+ prealloc_fits_assumes_reclaim=Mock(return_value=False)
+ )
+ queue._allocatable_token_budgets = Mock(return_value=100000)
+ queue._hicache_pending_restore_tokens = Mock(return_value=0)
+ queue.scheduler = SimpleNamespace(
+ running_batch=SimpleNamespace(reqs=[]),
+ enable_priority_scheduling=False,
+ enable_hisparse=False,
+ enable_lora=False,
+ metrics_reporter=SimpleNamespace(enable_metrics=False),
+ )
+ queue.req_to_metadata_buffer_idx_allocator = SimpleNamespace(
+ available_size=Mock(return_value=1)
+ )
+ queue.req_to_token_pool = SimpleNamespace(available_size=Mock(return_value=0))
+ with patch(
+ "sglang.srt.disaggregation.decode.poll_and_all_reduce",
+ side_effect=lambda receivers, group: [r.poll() for r in receivers],
+ ):
+ self.assertEqual(queue.pop_preallocated(), ([], []))
+ self.flush()
+ self.assertFalse(entry.waiting_for_input)
+ queue.req_to_metadata_buffer_idx_allocator.available_size.assert_not_called()
+ queue.req_to_token_pool.available_size.assert_not_called()
+ sender.mark_prefill_complete()
+ self.flush()
+ self.assertEqual(queue.pop_preallocated(), ([], []))
+ self.assertTrue(entry.waiting_for_input)
+ queue.req_to_metadata_buffer_idx_allocator.available_size.assert_called_once()
+ queue.req_to_token_pool.available_size.assert_called_once()
+ self.assertEqual(queue.queue, [entry])
+
+ def test_exhausted_optimistic_attempt_remains_compute_eligible(self):
+ override = get_context().override_server_args(
+ optimistic_prefill_attempts=1,
+ disaggregation_decode_allocation_policy="prefill_complete",
+ )
+ override.install()
+ self.addCleanup(override.restore)
+ req = SimpleNamespace(
+ rid="retry",
+ prefill_attempt_count=1,
+ output_ids=[42],
+ skip_radix_cache_insert=False,
+ kv=SimpleNamespace(cache_protected_len=16),
+ full_untruncated_fill_ids=list(range(32)),
+ _compute_max_prefix_len=Mock(return_value=31),
+ storage_prefetch_retry_attempts=3,
+ storage_prefetch_last_match_len=8,
+ reset_for_retract=Mock(),
+ advance_cache_request_handle=Mock(),
+ time_stats=MagicMock(),
+ )
+ scheduler = SimpleNamespace(
+ tree_cache=Mock(supports_mamba=Mock(return_value=False)),
+ _release_aborted_request=Mock(),
+ clear_pending_chunk_send=Mock(),
+ waiting_queue=[],
+ disagg_prefill_bootstrap_queue=SimpleNamespace(queue=[]),
+ metrics_reporter=SimpleNamespace(enable_metrics=False),
+ processed_tokens_counter=0,
+ )
+ with (
+ patch("sglang.srt.disaggregation.prefill.checkpoint_kv_cache"),
+ patch("sglang.srt.disaggregation.prefill.release_kv_cache"),
+ ):
+ SchedulerDisaggregationPrefillMixin.optimistic_release_and_requeue(
+ scheduler, req
+ )
+ self.assertEqual(scheduler.waiting_queue, [req])
+ self.assertEqual(scheduler.disagg_prefill_bootstrap_queue.queue, [])
+ self.assertEqual(req.prefill_attempt_count, 2)
+ self.assertEqual(list(req.output_ids), [])
+ self.assertTrue(req.pending_bootstrap)
+ self.assertEqual(req.storage_prefetch_retry_attempts, 0)
+ self.assertEqual(req.storage_prefetch_last_match_len, 16)
+
+ def test_chunked_prefill_does_not_yield_for_decode_admission(self):
+ for policy in ("early", "prefill_complete"):
+ for overlap in (False, True):
+ with self.subTest(policy=policy, overlap=overlap):
+ override = get_context().override_server_args(
+ disaggregation_decode_allocation_policy=policy
+ )
+ override.install()
+ try:
+ req = Mock(finished_reason=None, to_finish=None)
+ scheduler = SimpleNamespace(
+ chunked_req=req,
+ tree_cache=Mock(),
+ enable_overlap=overlap,
+ check_bootstrap=Mock(return_value=False),
+ has_bootstrapped_waiting_req=Mock(return_value=True),
+ optimistic_release_and_requeue=Mock(),
+ )
+ scheduler.checkpoint_disagg_prefill = MethodType(
+ SchedulerDisaggregationPrefillMixin.checkpoint_disagg_prefill,
+ scheduler,
+ )
+ running = SimpleNamespace(batch_is_full=True)
+ with patch(
+ "sglang.srt.disaggregation.prefill.checkpoint_kv_cache"
+ ):
+ SchedulerDisaggregationPrefillMixin.process_prefill_chunk(
+ scheduler, None, running
+ )
+ if policy == "prefill_complete":
+ self.assertIs(scheduler.chunked_req, req)
+ self.assertFalse(running.batch_is_full)
+ scheduler.optimistic_release_and_requeue.assert_not_called()
+ else:
+ self.assertIsNone(scheduler.chunked_req)
+ self.assertEqual(
+ scheduler.optimistic_release_and_requeue.call_count,
+ 0 if overlap else 1,
+ )
+ finally:
+ override.restore()
+
+ def test_completed_prefill_parks_until_normal_pointer_handshake(self):
+ sender, receiver = self.sender(), self.receiver()
+ self.flush()
+ sender.mark_prefill_complete()
+ self.flush()
+ self.assertEqual(receiver.poll(), KVPoll.WaitingForInput)
+ # Decode may reserve KV now, but prefill still has no destination indices.
+ self.assertEqual(sender.poll(), KVPoll.Bootstrapping)
+ req = SimpleNamespace(
+ rid="parked",
+ disagg_kv_sender=sender,
+ pending_bootstrap=True,
+ prefill_attempt_count=1,
+ skip_radix_cache_insert=False,
+ )
+ scheduler = SchedulerDisaggregationPrefillMixin()
+ scheduler.tree_cache = Mock()
+ scheduler.scheduler_stage_metrics = None
+ scheduler.disagg_prefill_inflight_queue = [req]
+ scheduler.attn_cp_cpu_group = scheduler.attn_tp_cpu_group = object()
+ scheduler.output_streamer = Mock()
+ scheduler.send_kv_chunk = Mock()
+
+ def finalize(r):
+ r.pending_bootstrap = False
+ return True
+
+ scheduler.disagg_prefill_bootstrap_queue = SimpleNamespace(
+ finalize_bootstrap=Mock(side_effect=finalize)
+ )
+ with (
+ patch(
+ "sglang.srt.disaggregation.prefill.poll_and_all_reduce_attn_cp_tp_group",
+ side_effect=lambda senders, *groups: [s.poll() for s in senders],
+ ),
+ patch("sglang.srt.disaggregation.prefill.checkpoint_kv_cache"),
+ ):
+ self.assertEqual(scheduler.process_disagg_prefill_inflight_queue(), [])
+ self.assertEqual(scheduler.disagg_prefill_inflight_queue, [req])
+ scheduler.send_kv_chunk.assert_not_called()
+ # This is the existing metadata-received transition, not readiness.
+ self.prefill.update_status(1, KVPoll.WaitingForInput)
+ self.assertEqual(scheduler.process_disagg_prefill_inflight_queue(), [])
+ scheduler.send_kv_chunk.assert_called_once_with(req, last_chunk=True)
+ self.assertFalse(req.pending_bootstrap)
+
+ def test_clear_before_allocation_notifies_prefill(self):
+ sender, receiver = self.sender(), self.receiver()
+ self.flush()
+ receiver.clear()
+ self.flush()
+ self.assertEqual(sender.poll(), KVPoll.Failed)
+ self.assertNotIn(1, self.decode.request_status)
+
+ def test_success_is_sent_once_after_all_sender_state_is_cleared(self):
+ sender, receiver = self.sender(), self.receiver()
+ self.flush()
+ sender.mark_prefill_complete()
+ self.flush()
+ self.prefill.conclude_transfer(bootstrap_room=1, status=KVPoll.Success)
+ self.assertEqual(sender.poll(), KVPoll.Success)
+ self.prefill.req_to_decode_prefix_len[1] = 0
+ self.prefill.transfer_infos[1] = {}
+ send = self.prefill.send_kv_status_message
+ next_transfer = []
+
+ def success(**kwargs):
+ self.assertEqual(kwargs["status"], KVPoll.Success)
+ for table in (
+ self.prefill.deferred_bootstrap.rooms,
+ self.prefill.request_status,
+ self.prefill.req_to_decode_prefix_len,
+ self.prefill.transfer_infos,
+ ):
+ self.assertNotIn(1, table)
+ send(**kwargs)
+ self.flush()
+ self.assertEqual(receiver.poll(), KVPoll.Success)
+ receiver.clear()
+ # Resume immediately, before the old clear() call even returns.
+ new_sender, new_receiver = self.sender(), self.receiver()
+ self.flush()
+ next_transfer.extend((new_sender, new_receiver))
+
+ self.prefill.send_kv_status_message = Mock(side_effect=success)
+ sender.clear()
+ sender.clear()
+ self.prefill.send_kv_status_message.assert_called_once()
+ new_sender, new_receiver = next_transfer
+ self.assertIs(self.prefill.deferred_bootstrap.rooms[1].owner, new_sender)
+ self.assertEqual(new_receiver.poll(), KVPoll.Bootstrapping)
+ self.assertEqual(list(self.wire), [])
+
+ def test_worker_cleanup_finishes_before_sender_can_report_success(self):
+ sender, receiver = self.sender(), self.receiver()
+ self.flush()
+ sender.mark_prefill_complete()
+ self.flush()
+ mgr = self.prefill
+ mgr._staging_outstanding = defaultdict(int)
+ mgr.session_lock = threading.Lock()
+ mgr.failed_sessions = set()
+ mgr.enable_deferred_decode_kv_release = True
+ mgr.decode_kv_args_table = {
+ "session": SimpleNamespace(requires_dcp_relayout=False, dst_aux_ptrs=[])
+ }
+ mgr._get_dsa_cache_transfer_skip_flags = Mock(return_value=(False, False))
+ mgr.send_aux = Mock(return_value=0)
+ mgr.transfer_infos[1] = {
+ "session": SimpleNamespace(
+ is_dummy=False,
+ mooncake_session_id="session",
+ dst_kv_indices=[],
+ dst_device_kv_indices=None,
+ endpoint="decode",
+ dst_port=2,
+ room=1,
+ required_dst_info_num=1,
+ )
+ }
+ cleanup_observed = []
+
+ def during_cleanup(room, default=None):
+ # The worker has finished writes and decremented its counter, but
+ # is still deleting room-indexed state. Force a scheduler poll here.
+ self.assertEqual(mgr._staging_outstanding[room], 0)
+ self.assertEqual(sender.poll(), KVPoll.Transferring)
+ self.flush()
+ self.assertEqual(receiver.poll(), KVPoll.WaitingForInput)
+ cleanup_observed.append(room)
+
+ mgr.req_to_decode_prefix_len = SimpleNamespace(pop=during_cleanup)
+ chunk = SimpleNamespace(
+ room=1,
+ staging_counted=False,
+ index_slice=slice(0, 0),
+ prefill_kv_indices=[],
+ is_last_chunk=True,
+ state_indices=None,
+ prefill_aux_index=0,
+ )
+ mgr.transfer_worker(Mock(get=Mock(side_effect=[chunk, None])), None)
+ self.assertEqual(cleanup_observed, [1])
+ self.assertNotIn(1, mgr._staging_outstanding)
+ self.assertEqual(sender.poll(), KVPoll.Success)
+ self.assertEqual(receiver.poll(), KVPoll.WaitingForInput)
+ mgr.req_to_decode_prefix_len = {}
+ sender.clear()
+ self.flush()
+ self.assertEqual(receiver.poll(), KVPoll.Success)
+
+ def test_early_transfer_success_does_not_wait_for_sender_cleanup(self):
+ override = get_context().override_server_args(
+ disaggregation_decode_allocation_policy="early"
+ )
+ override.install()
+ self.addCleanup(override.restore)
+ self.prefill.defer_decode_allocation = self.decode.defer_decode_allocation = (
+ False
+ )
+ self.prefill.deferred_bootstrap = None
+ sender, receiver = self.sender(), self.receiver(policy="early")
+ self.prefill.conclude_transfer(
+ bootstrap_room=1, status=KVPoll.Success, targets=[("decode", 2)]
+ )
+ self.flush()
+ self.assertEqual(receiver.poll(), KVPoll.Success)
+ self.assertIn(1, self.prefill.request_status)
+ self.assertEqual(sender.poll(), KVPoll.Success)
+
+ def test_recompute_http_error_after_success_before_retirement_is_ignored(self):
+ sender, receiver = self.sender(), self.receiver()
+ self.flush()
+ sender.mark_prefill_complete()
+ self.flush()
+ self.prefill.conclude_transfer(bootstrap_room=1, status=KVPoll.Success)
+ self.assertEqual(sender.poll(), KVPoll.Success)
+ sender.clear()
+ self.flush()
+ # Success is in the status table, but the scheduler has not polled it.
+ self.assertIsNone(receiver.conclude_state)
+ self.decode._fail_prefill_recompute(receiver, "late HTTP timeout")
+ self.assertEqual(receiver.poll(), KVPoll.Success)
+ self.assertNotIn(1, self.decode.failure_records)
+
+ def test_receiver_retirement_waits_for_recompute_error_callback(self):
+ receiver = self.receiver()
+ error_started = threading.Event()
+ finish_error = threading.Event()
+ retirement_started = threading.Event()
+ record_failure = self.decode.record_failure
+
+ def delayed_failure(room, reason):
+ error_started.set()
+ if not finish_error.wait(timeout=5):
+ raise TimeoutError("test did not release the error callback")
+ record_failure(room, reason)
+
+ def retire():
+ retirement_started.set()
+ receiver.clear()
+
+ self.decode.record_failure = delayed_failure
+ with ThreadPoolExecutor(max_workers=2) as executor:
+ error = executor.submit(
+ self.decode._fail_prefill_recompute, receiver, "HTTP error"
+ )
+ try:
+ self.assertTrue(error_started.wait(timeout=5))
+ retirement = executor.submit(retire)
+ self.assertTrue(retirement_started.wait(timeout=5))
+ with self.assertRaises(TimeoutError):
+ retirement.result(timeout=0.05)
+ finally:
+ finish_error.set()
+ error.result(timeout=5)
+ retirement.result(timeout=5)
+ self.assertFalse(receiver._owns_bootstrap_room)
+ self.assertEqual(self.decode.failure_records[1], "HTTP error")
+
+ def test_slow_status_delivery_does_not_hold_bootstrap_lock(self):
+ sender, receiver = self.sender(), self.receiver()
+ self.flush()
+
+ def send(*args, **kwargs):
+ acquired = self.prefill.deferred_bootstrap.lock.acquire(blocking=False)
+ self.assertTrue(acquired)
+ if acquired:
+ self.prefill.deferred_bootstrap.lock.release()
+
+ self.prefill.send_kv_status_message = send
+ sender.mark_prefill_complete()
+ sender.abort()
+ sender.clear()
+
+ def test_rebootstrap_dispatch_precedes_ready_without_allocating(self):
+ receiver = self.receiver()
+ receiver.init = Mock()
+ req = SimpleNamespace(build_rebootstrap_payload=lambda: {"rid": "resume"})
+ entry = DecodeRequest(req=req, kv_receiver=receiver, is_rebootstrap=True)
+ queue = DecodePreallocQueue.__new__(DecodePreallocQueue)
+ dispatched = []
+ queue.kv_manager = SimpleNamespace(
+ submit_prefill_recompute=lambda r, payload: dispatched.append((r, payload))
+ )
+ queue._init_receiver(entry, 0)
+ self.assertEqual(dispatched, [(receiver, {"rid": "resume"})])
+ self.assertFalse(entry.waiting_for_input)
+ self.assertEqual(req.disagg_prefill_dp_rank, 0)
+
+ def test_rebootstrap_dispatches_once_across_init_and_metadata(self):
+ for policy in ("early", "prefill_complete"):
+ with self.subTest(policy=policy):
+ override = get_context().override_server_args(
+ disaggregation_decode_allocation_policy=policy
+ )
+ override.install()
+ try:
+ receiver = Mock(conclude_state=None)
+ req = SimpleNamespace(
+ build_rebootstrap_payload=lambda: {"rid": "resume"},
+ time_stats=Mock(),
+ )
+ entry = DecodeRequest(
+ req=req, kv_receiver=receiver, is_rebootstrap=True
+ )
+ queue = DecodePreallocQueue.__new__(DecodePreallocQueue)
+ queue.kv_manager = Mock()
+ queue.req_to_metadata_buffer_idx_allocator = Mock()
+ queue._transfer_page_indices = Mock(return_value=[0])
+ queue.transfer_queue = SimpleNamespace(enable_staging=False)
+ queue._num_published_destinations = 0
+ events = Mock()
+ events.attach_mock(receiver.send_metadata, "metadata")
+ events.attach_mock(
+ queue.kv_manager.submit_prefill_recompute, "recompute"
+ )
+ queue._init_receiver(entry, 0)
+ queue._send_kv_metadata(entry, [0], 1)
+ queue.kv_manager.submit_prefill_recompute.assert_called_once_with(
+ receiver, {"rid": "resume"}
+ )
+ self.assertEqual(
+ [event[0] for event in events.mock_calls],
+ ["metadata", "recompute"]
+ if policy == "early"
+ else ["recompute", "metadata"],
+ )
+ self.assertEqual(queue._num_published_destinations, 1)
+ finally:
+ override.restore()
+
+ def test_normal_bootstrap_keeps_room_without_dispatching_recompute(self):
+ queue = DecodePreallocQueue.__new__(DecodePreallocQueue)
+ queue._check_if_req_exceed_kv_capacity = Mock(return_value=False)
+ queue.pending_reqs = []
+ req = SimpleNamespace(
+ bootstrap_room=7,
+ bootstrap_host="prefill",
+ bootstrap_port=8998,
+ disagg_prefill_dp_rank=None,
+ to_finish=None,
+ finished_reason=None,
+ )
+ receiver = Mock(conclude_state=None)
+ entry = DecodeRequest(req=req, kv_receiver=receiver)
+ queue._create_receiver_and_enqueue = Mock(return_value=entry)
+ queue.kv_manager = SimpleNamespace(
+ prefill_info_table={
+ "prefill:8998": SimpleNamespace(dp_size=1),
+ },
+ submit_prefill_recompute=Mock(),
+ )
+ queue.add(req)
+ self.assertEqual(req.bootstrap_room, 7)
+ receiver.init.assert_called_once_with(0)
+ queue.kv_manager.submit_prefill_recompute.assert_not_called()
+ self.assertFalse(entry.waiting_for_input)
+
+ def test_rebootstrap_keeps_room_and_prefill_rank(self):
+ queue = DecodePreallocQueue.__new__(DecodePreallocQueue)
+ queue._check_if_req_exceed_kv_capacity = Mock(return_value=False)
+ queue.queue = []
+ queue.pending_reqs = []
+ queue._create_receiver_and_enqueue = Mock()
+ queue._init_receiver = Mock()
+ req = SimpleNamespace(
+ bootstrap_room=1,
+ bootstrap_host="prefill",
+ bootstrap_port=8998,
+ disagg_prefill_dp_rank=3,
+ )
+ queue.kv_manager = SimpleNamespace(
+ prefill_info_table={"prefill:8998": SimpleNamespace(dp_size=4)}
+ )
+ queue.add(req, is_rebootstrap=True)
+ self.assertEqual(req.bootstrap_room, 1)
+ queue._init_receiver.assert_called_once_with(
+ queue._create_receiver_and_enqueue.return_value, 3
+ )
+ queue.add(req, is_rebootstrap=True)
+ self.assertEqual(req.bootstrap_room, 1)
+ self.assertEqual(req.disagg_prefill_dp_rank, 3)
+
+ def test_receiver_clear_preserves_failure_reason(self):
+ receiver = self.receiver()
+ self.decode.record_failure(1, "original failure")
+ self.decode.update_status(1, KVPoll.Failed)
+ receiver.clear()
+ self.assertEqual(self.decode.failure_records[1], "original failure")
+
+
+if __name__ == "__main__":
+ unittest.main()