[PP+PD] Fix #38206 Reduce bootstrap consensus latency (#38959)

Co-authored-by: inkcherry <mingzhi.liu@amd.com>
Co-authored-by: Shangming Cai <csmthu@gmail.com>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Chao Shi
2026-10-09 20:53:48 +08:00
committed by GitHub
co-authored by inkcherry Shangming Cai Claude Opus 5.5
parent f1e0472fcc
commit 348689da40
13 changed files with 515 additions and 166 deletions
@@ -33,6 +33,7 @@ from sglang.srt.disaggregation.common.bootstrap import (
BootstrapNotification,
DeferredBootstrap,
)
from sglang.srt.disaggregation.pp_consensus_store import PPConsensusStore
from sglang.srt.disaggregation.utils import (
DisaggregationMode,
filter_kv_indices_for_cp_rank,
@@ -356,7 +357,7 @@ class CommonKVManager(BaseKVManager):
)
logger.debug(f"kv manager bind to {self.local_ip}:{self.rank_port}")
self.request_status: Dict[int, KVPoll] = {}
self.request_status: Union[Dict[int, KVPoll], PPConsensusStore] = {}
self._socket_cache: Dict[str, zmq.Socket] = {}
self._monitor_cache: Dict[str, zmq.Socket] = {}
self._socket_send_locks: Dict[str, threading.Lock] = {}
@@ -403,6 +404,10 @@ class CommonKVManager(BaseKVManager):
self.req_to_decode_prefix_len: Dict[int, int] = {}
self.decode_kv_args_table = {}
self.pp_group = get_parallel().pp_group
if self.pp_size > 1:
self.request_status = PPConsensusStore(
self.pp_size, self.pp_rank, self.pp_group
)
# If a timeout happens on the prefill side, it means prefill instances
# fail to receive the KV indices from the decode instance of this request.
# These timeout requests should be aborted to release the tree cache.
@@ -547,6 +552,12 @@ class CommonKVManager(BaseKVManager):
def check_status(self, bootstrap_room: int) -> KVPoll:
return self.request_status[bootstrap_room]
def check_status_pp_consensus(self, bootstrap_room: int) -> KVPoll:
statuses = self.request_status.collect(bootstrap_room)
if any(status is None for status in statuses):
return KVPoll.Bootstrapping
return min(statuses)
def update_status(self, bootstrap_room: int, status: KVPoll):
current = self.request_status.get(bootstrap_room)
if current is None:
@@ -74,6 +74,9 @@ class FakeKVSender(BaseKVSender):
self.conclude_state = KVPoll.Success
return KVPoll.Success
def poll_pp_consensus(self) -> KVPoll:
return self.poll()
def _check_waiting_timeout(self) -> Optional[KVPoll]:
# A send() that never comes must not pin the prefill inflight queue forever.
# No deadline before init(): the request is still in the bootstrap queue.
@@ -3064,31 +3064,37 @@ class MooncakeKVSender(MooncakeFailureExceptionMixin, CommonKVSender):
self._record_transfer_indices(kv_indices, state_indices)
def poll(self) -> KVPoll:
if self.conclude_state is None:
status = self.kv_mgr.check_status(self.bootstrap_room)
# Hold Success until all staging chunks transferred. With deferred
# allocation also wait for worker cleanup (the zero entry is only
# removed after cleanup), before permitting the room to be reused.
if status == KVPoll.Success and (
self.kv_mgr._staging_outstanding.get(self.bootstrap_room, 0) > 0
or (
self.kv_mgr.defer_decode_allocation
and self.bootstrap_room in self.kv_mgr._staging_outstanding
)
):
return KVPoll.Transferring
if status in (KVPoll.Success, KVPoll.Failed):
self.conclude_state = status
self.trace_ctx.trace_req_finish()
elif status == KVPoll.Bootstrapping:
timeout_result = self._check_bootstrap_timeout()
if timeout_result is not None:
return timeout_result
return self._poll_with_status(self.kv_mgr.check_status)
return status
else:
def poll_pp_consensus(self) -> KVPoll:
return self._poll_with_status(self.kv_mgr.check_status_pp_consensus)
def _poll_with_status(self, check_status) -> KVPoll:
if self.conclude_state is not None:
return self.conclude_state
status = check_status(self.bootstrap_room)
# Hold Success until all staging chunks transferred. With deferred
# allocation also wait for worker cleanup (the zero entry is only
# removed after cleanup), before permitting the room to be reused.
if status == KVPoll.Success and (
self.kv_mgr._staging_outstanding.get(self.bootstrap_room, 0) > 0
or (
self.kv_mgr.defer_decode_allocation
and self.bootstrap_room in self.kv_mgr._staging_outstanding
)
):
return KVPoll.Transferring
if status in (KVPoll.Success, KVPoll.Failed):
self.conclude_state = status
self.trace_ctx.trace_req_finish()
elif status == KVPoll.Bootstrapping:
timeout_result = self._check_bootstrap_timeout()
if timeout_result is not None:
return timeout_result
return status
def _init_trace_ctx(self):
if self.kv_mgr.enable_trace:
self.trace_ctx = TraceReqContext(
@@ -1844,6 +1844,12 @@ class MoriKVSender(CommonKVSender):
)
def poll(self) -> KVPoll:
return self._poll_with_status(self.kv_mgr.check_status)
def poll_pp_consensus(self) -> KVPoll:
return self._poll_with_status(self.kv_mgr.check_status_pp_consensus)
def _poll_with_status(self, check_status) -> KVPoll:
if self.conclude_state is not None:
return self.conclude_state
@@ -1851,7 +1857,7 @@ class MoriKVSender(CommonKVSender):
self.conclude_state = KVPoll.Failed
return self.conclude_state
status = self.kv_mgr.check_status(self.bootstrap_room)
status = check_status(self.bootstrap_room)
if status == KVPoll.Bootstrapping:
timeout_result = self._check_bootstrap_timeout()
if timeout_result is not None:
@@ -3232,9 +3232,15 @@ class NixlKVSender(CommonKVSender):
self.has_sent = True
def poll(self) -> KVPoll:
return self._poll_with_status(self.kv_mgr.check_status)
def poll_pp_consensus(self) -> KVPoll:
return self._poll_with_status(self.kv_mgr.check_status_pp_consensus)
def _poll_with_status(self, check_status) -> KVPoll:
if self._send_failed:
return KVPoll.Failed # type: ignore
status = self.kv_mgr.check_status(self.bootstrap_room)
status = check_status(self.bootstrap_room)
if status == KVPoll.Bootstrapping:
timeout_result = self._check_bootstrap_timeout()
if timeout_result is not None:
@@ -0,0 +1,172 @@
from __future__ import annotations
import logging
import os
import pickle
import queue
import threading
from enum import Enum, auto
from typing import Any, Dict, List, Optional
import zmq
from sglang.srt.utils.network import (
NetworkAddress,
get_local_ip_auto,
get_zmq_socket,
get_zmq_socket_on_host,
)
logger = logging.getLogger(__name__)
class _OpKind(Enum):
PUT = auto()
DELETE = auto()
class PPConsensusStore:
"""A map that replicates metadata to PP rank 0.
All ranks may use this as a normal Dict, e.g. ``store["key"] = value`` or ``del store["key"]``.
All modifications done by all ranks are replicated to rank 0 in background. Rank 0 calls
``collect`` to retrieve values of all ranks for the specified key.
"""
def __init__(self, pp_size: int, pp_rank: int, pp_group) -> None:
self._pp_size = pp_size
self._pp_rank = pp_rank
self._pp_group = pp_group
self._local_map: Dict[Any, Any] = {}
self._peer_map: Dict[int, Dict[Any, Any]] = {}
self._cache_lock = threading.Lock()
self._replication_queue: Optional[queue.Queue] = None
self._zmq_ctx: Optional[zmq.Context] = None
self._socket: Optional[zmq.Socket] = None
self._init_zmq()
if self._pp_rank > 0:
self._replication_queue = queue.Queue()
self._worker_thread = threading.Thread(target=self._worker_loop, daemon=True)
self._worker_thread.start()
def _init_zmq(self) -> None:
self._zmq_ctx = zmq.Context()
# PP0 listens.
if self._pp_rank == 0:
host = get_local_ip_auto()
port, self._socket = get_zmq_socket_on_host(
self._zmq_ctx, zmq.PULL, host=host
)
store_info = (host, port)
logger.info("PPConsensusStore is listening at %s:%d", host, port)
else:
store_info = None
# Broadcast our zmq host and port to other ranks.
host, port = self._pp_group.broadcast_object(store_info, src=0)
# PP>0 connects to PP0.
if self._pp_rank > 0:
self._socket = get_zmq_socket(
self._zmq_ctx,
zmq.PUSH,
NetworkAddress(host, port).to_tcp(),
bind=False,
)
logger.info("PPConsensusStore connected to %s:%d", host, port)
def _maybe_replicate_to_rank0(self, op: tuple) -> None:
if self._pp_rank > 0:
self._replication_queue.put(op)
def pop(self, key: Any, default: Any = None) -> Any:
with self._cache_lock:
if key not in self._local_map:
return default
value = self._local_map.pop(key)
self._maybe_replicate_to_rank0((_OpKind.DELETE, self._pp_rank, key))
return value
def get(self, key: Any, default: Any = None) -> Any:
with self._cache_lock:
return self._local_map.get(key, default)
def __contains__(self, key: Any) -> bool:
with self._cache_lock:
return key in self._local_map
def __getitem__(self, key: Any) -> Any:
with self._cache_lock:
return self._local_map[key]
def __setitem__(self, key: Any, value: Any) -> None:
with self._cache_lock:
self._local_map[key] = value
self._maybe_replicate_to_rank0((_OpKind.PUT, self._pp_rank, key, value))
def __delitem__(self, key: Any) -> None:
with self._cache_lock:
if key not in self._local_map:
raise KeyError(key)
self._local_map.pop(key)
self._maybe_replicate_to_rank0((_OpKind.DELETE, self._pp_rank, key))
def close(self) -> None:
if self._pp_rank > 0:
self._replication_queue.put(None)
self._zmq_ctx.term() # This blocks until all zmq socket closed.
self._worker_thread.join()
def _worker_loop(self) -> None:
try:
if self._pp_rank == 0:
self._recv_loop()
else:
self._send_loop()
except zmq.error.ContextTerminated:
pass # Raised by close().
except Exception as e:
logger.critical(
"PPConsensusStore background thread crashed: %s",
e,
)
for handler in logger.handlers:
handler.flush()
os._exit(1)
finally:
self._socket.close(linger=0)
def _recv_once(self) -> None:
op = pickle.loads(self._socket.recv())
if op[0] == _OpKind.PUT:
_, rank, key, value = op
logger.debug("recv put rank=%s %s = %s", rank, key, value)
with self._cache_lock:
self._peer_map.setdefault(rank, {})[key] = value
elif op[0] == _OpKind.DELETE:
_, rank, key = op
logger.debug("recv delete rank=%s %s", rank, key)
with self._cache_lock:
self._peer_map.get(rank, {}).pop(key, None)
def _recv_loop(self) -> None:
assert self._pp_rank == 0
while True:
self._recv_once()
def _send_loop(self) -> None:
assert self._pp_rank > 0
while True:
op = self._replication_queue.get()
if op is None: # A signal for shutdown.
return
logger.debug("send %s", op)
self._socket.send(pickle.dumps(op))
def collect(self, key: Any) -> List[Any]:
assert self._pp_rank == 0, "collect can only be used on PP rank 0"
with self._cache_lock:
values: List[Any] = [self._local_map.get(key)]
for rank in range(1, self._pp_size):
values.append(self._peer_map.get(rank, {}).get(key))
return values
+14 -42
View File
@@ -60,7 +60,7 @@ from sglang.srt.disaggregation.utils import (
is_mla_backend,
is_unadmitted_reject,
poll_and_all_reduce_attn_cp_tp_group,
poll_and_all_reduce_pp,
poll_and_all_reduce_prefill_pp,
prepare_abort,
setup_state_kv_args,
)
@@ -194,6 +194,7 @@ class PrefillBootstrapQueue:
self.scheduler.tp_worker.model_runner.effective_logical_max_total_num_tokens
)
self.transfer_backend = transfer_backend
self.pp_poll_sync_work_list: List = []
if envs.SGLANG_DISAGG_STAGING_BUFFER.get():
if self.is_mla_backend:
raise RuntimeError(
@@ -458,47 +459,25 @@ class PrefillBootstrapQueue:
req.sampling_params.max_new_tokens = 1
@scheduler_stage_method(SCHEDULER_STAGE_PROCESS_QUEUE)
def pop_bootstrapped(
self,
return_failed_reqs: bool = False,
pp_good_rids: Optional[List[str]] = None,
pp_bad_rids: Optional[List[str]] = None,
) -> List[Req] | tuple[List[Req], List[Req]]:
"""
pop the reqs which has finished bootstrapping
return_failed_reqs: For PP, on rank 0, also return the failed reqs to notify the next rank
pp_good_rids: RIDs that PP consensus determined as WaitingForInput.
pp_bad_rids: RIDs that PP consensus determined as Failed.
"""
def pop_bootstrapped(self) -> List[Req]:
"""Pop the reqs which have finished bootstrapping."""
bootstrapped_reqs = []
failed_reqs = []
indices_to_remove = set()
if len(self.queue) == 0:
if return_failed_reqs is False:
return []
else:
return [], []
return []
if self.pp_size > 1:
polls = poll_and_all_reduce_pp(
(req.rid for req in self.queue),
KVPoll.WaitingForInput,
pp_good_rids,
pp_bad_rids,
polls = poll_and_all_reduce_prefill_pp(
[req.disagg_kv_sender for req in self.queue],
self.scheduler.attn_cp_cpu_group,
self.scheduler.attn_tp_cpu_group,
self.scheduler.pp_group,
self.pp_rank,
self.pp_size,
self.pp_poll_sync_work_list,
)
uncovered = [i for i, poll in enumerate(polls) if poll is None]
if uncovered:
local_polls = poll_and_all_reduce_attn_cp_tp_group(
[self.queue[i].disagg_kv_sender for i in uncovered],
self.scheduler.attn_cp_cpu_group,
self.scheduler.attn_tp_cpu_group,
)
for i, local_poll in zip(uncovered, local_polls):
if local_poll == KVPoll.Failed:
polls[i] = KVPoll.Failed
else:
polls = poll_and_all_reduce_attn_cp_tp_group(
[req.disagg_kv_sender for req in self.queue],
@@ -507,13 +486,9 @@ class PrefillBootstrapQueue:
)
for i, (req, poll) in enumerate(zip(self.queue, polls)):
if poll is None:
continue
if poll == KVPoll.Failed:
self.scheduler.handle_bootstrap_failure(req)
indices_to_remove.add(i)
failed_reqs.append(req)
elif poll == KVPoll.Bootstrapping:
if (
(
@@ -558,10 +533,7 @@ class PrefillBootstrapQueue:
entry for i, entry in enumerate(self.queue) if i not in indices_to_remove
]
if return_failed_reqs is False:
return bootstrapped_reqs
else:
return bootstrapped_reqs, failed_reqs
return bootstrapped_reqs
def release_memory_occupation(self):
self.queue.clear()
+70
View File
@@ -21,6 +21,8 @@ import torch.distributed as dist
from sglang.srt.configs.model_config import get_dsa_mtp_topk_width, is_deepseek_dsa
from sglang.srt.disaggregation.base import KVPoll
from sglang.srt.distributed.communication_tags import P2PTag
from sglang.srt.distributed.parallel_state import GroupCoordinator
from sglang.srt.environ import envs
from sglang.srt.runtime_context import (
get_disagg,
@@ -49,6 +51,44 @@ if is_npu():
FAKE_BOOTSTRAP_HOST = "2.2.2.2"
def pp_sync_polls(
polls: Optional[List[int]],
pp_group: GroupCoordinator,
pp_rank: int,
pp_size: int,
sync_work_list: List,
) -> List[int]:
"""Synchronize poll states from PP0 through the PP pipeline.
PP0 provides the data. Each later PP rank receives it from the previous
rank and asynchronously forwards it to the next rank.
"""
if pp_size <= 1:
assert polls is not None
return polls
for p2p_work in sync_work_list:
p2p_work.work.wait()
sync_work_list.clear()
data = polls
if pp_rank > 0:
data = pp_group.recv_object(
src=pp_rank - 1,
tag=P2PTag.DISAGG_BOOTSTRAP_PP_SYNC,
)
if pp_rank + 1 < pp_size:
sync_work_list.extend(
pp_group.send_object(
data,
dst=pp_rank + 1,
async_send=True,
tag=P2PTag.DISAGG_BOOTSTRAP_PP_SYNC,
)
)
return data
def poll_and_all_reduce_pp(
rids: Iterable[str],
ready_poll: int,
@@ -67,6 +107,25 @@ def poll_and_all_reduce_pp(
]
def poll_and_all_reduce_prefill_pp(
pollers: List[CommonKVSender],
attn_cp_cpu_group: dist.ProcessGroup,
attn_tp_cpu_group: dist.ProcessGroup,
pp_group: GroupCoordinator,
pp_rank: int,
pp_size: int,
pp_poll_sync_work_list: List[torch.distributed.Work],
) -> List[int]:
polls: List[int] = []
if pp_rank == 0:
polls = _poll_with_failure_injection_pp(pollers)
polls = _all_reduce_polls(polls, attn_tp_cpu_group)
polls = _all_reduce_polls(polls, attn_cp_cpu_group)
# Propagate PP0's data to other ranks
polls = pp_sync_polls(polls, pp_group, pp_rank, pp_size, pp_poll_sync_work_list)
return polls
def get_dsa_seed_metadata_dim(hf_config) -> int:
"""Return the model-defined PD seed width, independent of local spec mode."""
if not getattr(hf_config, "index_share_for_mtp_iteration", False):
@@ -163,6 +222,17 @@ def _poll_with_failure_injection(pollers) -> List[int]:
return [int(poller.poll()) for poller in pollers]
def _poll_with_failure_injection_pp(pollers) -> List[int]:
if (failure_prob := envs.SGLANG_TEST_DISAGG_FAILURE_PROB.get()) > 0:
return [
int(KVPoll.Failed)
if random.random() < failure_prob
else int(poller.poll_pp_consensus())
for poller in pollers
]
return [int(poller.poll_pp_consensus()) for poller in pollers]
def _is_fake_transfer(req: Req) -> bool:
return req.bootstrap_host == FAKE_BOOTSTRAP_HOST or (
req.bootstrap_host is None
@@ -13,3 +13,4 @@ class P2PTag(IntEnum):
DEFAULT = 0
HICACHE_PP_SYNC = int.from_bytes(b"PpHi", byteorder="big")
GRAMMAR_PP_SYNC = int.from_bytes(b"PpGr", byteorder="big")
DISAGG_BOOTSTRAP_PP_SYNC = int.from_bytes(b"PpBs", byteorder="big")
@@ -223,18 +223,15 @@ class SchedulerPPMixin:
====================================================================
Stage P
recv ith req from previous stage
recv ith bootstrap req from previous stage
pop bootstrapped reqs (PPConsensusStore + pp_sync polls)
recv ith transferred req from previous stage
recv ith proxy from previous stage
run ith batch
recv prev (i+1) % mb_size th consensus bootstrapped req from previous stage
local consensus on bootstrapped req
recv prev (i+1) % mb_size th release req from previous stage
local consensus on release req
recv prev (i+1) % mb_size th outputs
process batch result of prev (i+1)% mb_size th batch (can be run in parallel with the curr batch GPU computation)
send ith req to next stage
send ith bootstrap req to next stage
send ith transferred req to next stage
send ith proxy to next stage
send current stage's outputs to next stage (can be stashed and delayed to send later)
@@ -242,23 +239,17 @@ class SchedulerPPMixin:
the above order can be optimized and reordered to minimize communication-related CPU stall and overhead bubbles.
====================================================================
There are two additional elements compared to the regular schedule:
Bootstrap Requests + Release Requests:
- Both can have local failure and need to be consensus on. PP needs to guarantee eventual consistency of local failure and flush malfunc requests out as soft error.
Release Requests still use the two-pass PP consensus. Bootstrap uses
PPConsensusStore instead of the legacy two-round RID sync.
"""
self.init_pp_loop_state()
# PD additional state initialization
bmbs = [None] * self.pp_loop_size
tmbs = [None] * self.pp_loop_size
consensus_bootstrapped_rids: Optional[List[str]] = None
transferred_rids: List[str] = []
release_rids: Optional[List[str]] = None
send_bootstrapped_work = []
send_transfer_work = []
send_consensus_bootstrapped_work = []
send_release_work = []
while True:
@@ -273,7 +264,6 @@ class SchedulerPPMixin:
next_pp_outputs = None
next_release_rids = None
next_consensus_bootstrapped_rids = None
d2h_event = None
next_batch_result = None
@@ -282,9 +272,7 @@ class SchedulerPPMixin:
if not self.pp_group.is_last_rank:
self._pp_commit_comm_work(self.send_req_work)
bootstrapped_rids = self._pp_pd_get_bootstrapped_ids()
bmbs[mb_id] = bootstrapped_rids
self._pp_commit_comm_work(send_bootstrapped_work)
self.process_bootstrapped_queue()
transferred_rids = self._pp_pd_get_prefill_transferred_ids()
self._pp_commit_comm_work(send_transfer_work)
@@ -332,28 +320,12 @@ class SchedulerPPMixin:
next_mb_id,
)
)
send_consensus_bootstrapped_work, consensus_bootstrapped_rids = (
self._pp_pd_send_consensus_bootstrapped_ids(
bmbs,
next_first_rank_mb_id,
consensus_bootstrapped_rids,
bootstrapped_rids,
)
)
send_release_work, release_rids = (
self._pp_pd_send_consensus_release_ids(
tmbs, next_first_rank_mb_id, release_rids, transferred_rids
)
)
if bmbs[next_mb_id] is not None:
next_consensus_bootstrapped_rids = (
self._pp_recv_pyobj_from_prev_stage()
)
next_consensus_bootstrapped_rids = self.process_bootstrapped_queue(
next_consensus_bootstrapped_rids
)
self._pp_commit_comm_work(send_consensus_bootstrapped_work)
if tmbs[next_mb_id] is not None:
next_release_rids = self._pp_recv_pyobj_from_prev_stage()
self._pp_commit_comm_work(send_release_work)
@@ -372,9 +344,6 @@ class SchedulerPPMixin:
self.send_req_work = self._pp_send_pyobj_to_next_stage(
recv_reqs, async_send=True
)
send_bootstrapped_work = self._pp_send_pyobj_to_next_stage(
bootstrapped_rids, async_send=True
)
send_transfer_work = self._pp_send_pyobj_to_next_stage(
transferred_rids, async_send=True
)
@@ -383,7 +352,6 @@ class SchedulerPPMixin:
self.pp_outputs = next_pp_outputs
release_rids = next_release_rids
consensus_bootstrapped_rids = next_consensus_bootstrapped_rids
self.running_batch.batch_is_full = False
@@ -636,67 +604,9 @@ class SchedulerPPMixin:
str, deque[Tuple[Dict[str, torch.Tensor], Optional[torch.Event]]]
] = defaultdict(deque)
def process_bootstrapped_queue(
self: Scheduler, bootstrapped_rids: Optional[List[str]]
):
# finished consensus bootstrapped reqs and prepare the waiting queue
if bootstrapped_rids is not None:
(
good_consensus_bootstrapped_rids,
bad_consensus_bootstrapped_rids,
) = bootstrapped_rids
good_reqs, failed_reqs = (
self.disagg_prefill_bootstrap_queue.pop_bootstrapped(
return_failed_reqs=True,
pp_good_rids=good_consensus_bootstrapped_rids,
pp_bad_rids=bad_consensus_bootstrapped_rids,
)
)
self.waiting_queue.extend(good_reqs)
return [[req.rid for req in good_reqs], [req.rid for req in failed_reqs]]
return None
def _pp_pd_get_bootstrapped_ids(self: Scheduler):
# communicate pre-consensus bootstrapp reqs
if self.pp_group.is_first_rank:
# First rank, pop the bootstrap reqs from the bootstrap queue
good_bootstrapped_rids, bad_bootstrapped_rids = self.get_rids(
self.disagg_prefill_bootstrap_queue.queue,
True,
[KVPoll.WaitingForInput],
[KVPoll.Failed],
)
else:
# Other ranks, receive the bootstrap reqs info from the previous rank and ensure the consensus
prev_bootstrapped_rids = self._pp_recv_pyobj_from_prev_stage()
prev_good_bootstrapped_rids, prev_bad_bootstrapped_rids = (
prev_bootstrapped_rids
)
curr_good_bootstrapped_rids, curr_bad_bootstrapped_rids = self.get_rids(
self.disagg_prefill_bootstrap_queue.queue,
True,
[KVPoll.WaitingForInput],
[KVPoll.Failed],
)
good_bootstrapped_rids = list(
set(prev_good_bootstrapped_rids) & set(curr_good_bootstrapped_rids)
)
bad_bootstrapped_rids = list(
set(prev_bad_bootstrapped_rids) | set(curr_bad_bootstrapped_rids)
)
# Route locally-aborted reqs through the bad-union consensus so every PP
# rank flushes them in the same consensus round, regardless of when the
# AbortReq reaches each rank and regardless of whether
# disagg_kv_sender.abort() drives the poll to Failed (it is optional).
aborted_rids = {
req.rid
for req in self.disagg_prefill_bootstrap_queue.queue
if isinstance(req.finished_reason, FINISH_ABORT)
}
good_bootstrapped_rids, bad_bootstrapped_rids = self._route_aborts_to_bad(
good_bootstrapped_rids, bad_bootstrapped_rids, aborted_rids
)
return [good_bootstrapped_rids, bad_bootstrapped_rids]
def process_bootstrapped_queue(self: Scheduler):
reqs = self.disagg_prefill_bootstrap_queue.pop_bootstrapped()
self.waiting_queue.extend(reqs)
def _pp_pd_get_prefill_transferred_ids(self: Scheduler):
# get the current stage transfer success
@@ -0,0 +1,192 @@
"""Unit tests for srt/disaggregation/pp_consensus_store."""
import time
import unittest
from collections.abc import Callable
from sglang.srt.disaggregation.pp_consensus_store import PPConsensusStore
from sglang.test.ci.ci_register import register_cpu_ci
from sglang.test.test_utils import CustomTestCase
register_cpu_ci(est_time=5, suite="base-a-test-cpu")
class _MockPPGroup:
def __init__(self):
self._obj = None
def broadcast_object(self, obj, src=0):
if obj is not None:
self._obj = obj
return self._obj
class TestPPConsensusStore(CustomTestCase):
def _wait_until(self, condition: Callable[[], bool]) -> None:
now = time.time()
start = now
deadline = now + 1
delay_time = 0.01
while now < deadline and not condition():
time.sleep(delay_time)
now = time.time()
delay_time *= 2
self.assertTrue(
now < deadline, f"Condition not met after {now - start} seconds"
)
def test_setitem(self):
group = _MockPPGroup()
store0 = PPConsensusStore(1, 0, group)
store0["key"] = 10
self.assertEqual(store0["key"], 10)
store0.close()
def test_setitem_replicate_to_rank0(self):
group = _MockPPGroup()
store0 = PPConsensusStore(2, 0, group)
store1 = PPConsensusStore(2, 1, group)
store0["key"] = 10
store1["key"] = 11
self._wait_until(lambda: store0.collect("key") == [10, 11])
store0.close()
store1.close()
def test_contains(self):
group = _MockPPGroup()
store0 = PPConsensusStore(1, 0, group)
store0["key"] = 10
self.assertTrue("key" in store0)
self.assertFalse("no_such_key" in store0)
store0.close()
def test_get(self):
group = _MockPPGroup()
store0 = PPConsensusStore(1, 0, group)
store0["key"] = 10
self.assertEqual(store0.get("key"), 10)
store0.close()
def test_get_not_exist(self):
group = _MockPPGroup()
store0 = PPConsensusStore(1, 0, group)
self.assertIsNone(store0.get("no_such_key"))
self.assertEqual(store0.get("no_such_key", "default"), "default")
store0.close()
def test_get_default(self):
group = _MockPPGroup()
store0 = PPConsensusStore(1, 0, group)
self.assertEqual(store0.get("no_such_key", "default"), "default")
store0.close()
def test_getitem_not_exist(self):
group = _MockPPGroup()
store0 = PPConsensusStore(1, 0, group)
with self.assertRaises(KeyError):
store0["no_such_key"]
store0.close()
def test_delitem(self):
group = _MockPPGroup()
store0 = PPConsensusStore(1, 0, group)
store0["key"] = 10
del store0["key"]
self.assertFalse("key" in store0)
store0.close()
def test_delitem_not_exist(self):
group = _MockPPGroup()
store0 = PPConsensusStore(1, 0, group)
with self.assertRaises(KeyError):
del store0["no_such_key"]
store0.close()
def test_delitem_replicate_to_rank0(self):
group = _MockPPGroup()
store0 = PPConsensusStore(2, 0, group)
store1 = PPConsensusStore(2, 1, group)
store0["key"] = 10
store1["key"] = 11
self._wait_until(lambda: store0.collect("key") == [10, 11])
del store1["key"]
self._wait_until(lambda: store0.collect("key") == [10, None])
store0.close()
store1.close()
def test_pop(self):
group = _MockPPGroup()
store0 = PPConsensusStore(1, 0, group)
store0["key"] = 10
self.assertEqual(store0.pop("key"), 10)
self.assertFalse("key" in store0)
store0.close()
def test_pop_not_exist(self):
group = _MockPPGroup()
store0 = PPConsensusStore(1, 0, group)
self.assertIsNone(store0.pop("no_such_key"))
store0.close()
def test_pop_default(self):
group = _MockPPGroup()
store0 = PPConsensusStore(1, 0, group)
self.assertEqual(store0.pop("no_such_key", "default"), "default")
store0.close()
def test_pop_replicate_to_rank0(self):
group = _MockPPGroup()
store0 = PPConsensusStore(2, 0, group)
store1 = PPConsensusStore(2, 1, group)
store0["key"] = 10
store1["key"] = 11
self._wait_until(lambda: store0.collect("key") == [10, 11])
self.assertEqual(store1.pop("key"), 11)
self.assertFalse("key" in store1)
self._wait_until(lambda: store0.collect("key") == [10, None])
store0.close()
store1.close()
def test_collect(self):
group = _MockPPGroup()
store0 = PPConsensusStore(3, 0, group)
store1 = PPConsensusStore(3, 1, group)
store2 = PPConsensusStore(3, 2, group)
store0["key"] = 10
store1["key"] = 11
store2["key"] = 12
self._wait_until(lambda: store0.collect("key") == [10, 11, 12])
store0.close()
store1.close()
store2.close()
def test_collect_missing(self):
group = _MockPPGroup()
store0 = PPConsensusStore(3, 0, group)
store1 = PPConsensusStore(3, 1, group)
store2 = PPConsensusStore(3, 2, group)
store0["key"] = 10
store2["key"] = 12
self._wait_until(lambda: store0.collect("key") == [10, None, 12])
store0.close()
store1.close()
store2.close()
def test_collect_not_exist(self):
group = _MockPPGroup()
store0 = PPConsensusStore(1, 0, group)
self.assertEqual(store0.collect("no_such_key"), [None])
store0.close()
def test_collect_requires_pp0(self):
group = _MockPPGroup()
store0 = PPConsensusStore(2, 0, group)
store1 = PPConsensusStore(2, 1, group)
with self.assertRaises(AssertionError):
store1.collect("key")
store0.close()
store1.close()
if __name__ == "__main__":
unittest.main()
@@ -552,7 +552,6 @@ class TestSchedulerIdleStepCounters(CustomTestCase):
scheduler._pp_commit_send_output_work_and_preprocess_output_tensors = Mock(
return_value=(None, GenerationBatchResult(), Mock())
)
scheduler._pp_pd_get_bootstrapped_ids = Mock(return_value=None)
scheduler._pp_pd_get_prefill_transferred_ids = Mock(return_value=None)
scheduler._pp_pd_get_retract_ids = Mock(return_value=None)
scheduler._pp_pd_get_prealloc_ids = Mock(return_value=None)
@@ -124,7 +124,8 @@ class TestSchedulerHiCacheEvents(unittest.TestCase):
s.running_mbs = [self.running_batch]
s.last_mbs = [None]
s.ingest_requests = Mock(return_value=[])
s._pp_pd_get_bootstrapped_ids = Mock(return_value=[])
s.waiting_queue = []
s.disagg_prefill_bootstrap_queue = SimpleNamespace(pop_bootstrapped=lambda: [])
s._pp_pd_get_prefill_transferred_ids = Mock(return_value=[])
s._pp_commit_comm_work = Mock()
s.get_new_batch_prefill = self.calls.admit