From 348689da40fac15a48620dc2437a44453b797375 Mon Sep 17 00:00:00 2001 From: Chao Shi Date: Fri, 9 Oct 2026 20:53:48 +0800 Subject: [PATCH] [PP+PD] Fix #38206 Reduce bootstrap consensus latency (#38959) Co-authored-by: inkcherry Co-authored-by: Shangming Cai Co-authored-by: Claude Opus 5.5 --- .../sglang/srt/disaggregation/common/conn.py | 13 +- python/sglang/srt/disaggregation/fake/conn.py | 3 + .../srt/disaggregation/mooncake/conn.py | 50 +++-- python/sglang/srt/disaggregation/mori/conn.py | 8 +- python/sglang/srt/disaggregation/nixl/conn.py | 8 +- .../srt/disaggregation/pp_consensus_store.py | 172 ++++++++++++++++ python/sglang/srt/disaggregation/prefill.py | 56 ++--- python/sglang/srt/disaggregation/utils.py | 70 +++++++ .../srt/distributed/communication_tags.py | 1 + .../sglang/srt/managers/scheduler_pp_mixin.py | 104 +--------- .../disaggregation/test_pp_consensus_store.py | 192 ++++++++++++++++++ .../test_disagg_idle_step_counters.py | 1 - .../managers/test_scheduler_hicache_events.py | 3 +- 13 files changed, 515 insertions(+), 166 deletions(-) create mode 100644 python/sglang/srt/disaggregation/pp_consensus_store.py create mode 100644 test/registered/unit/disaggregation/test_pp_consensus_store.py diff --git a/python/sglang/srt/disaggregation/common/conn.py b/python/sglang/srt/disaggregation/common/conn.py index e87bc6d963..45cb4c20bd 100644 --- a/python/sglang/srt/disaggregation/common/conn.py +++ b/python/sglang/srt/disaggregation/common/conn.py @@ -33,6 +33,7 @@ from sglang.srt.disaggregation.common.bootstrap import ( BootstrapNotification, DeferredBootstrap, ) +from sglang.srt.disaggregation.pp_consensus_store import PPConsensusStore from sglang.srt.disaggregation.utils import ( DisaggregationMode, filter_kv_indices_for_cp_rank, @@ -356,7 +357,7 @@ class CommonKVManager(BaseKVManager): ) logger.debug(f"kv manager bind to {self.local_ip}:{self.rank_port}") - self.request_status: Dict[int, KVPoll] = {} + self.request_status: Union[Dict[int, KVPoll], PPConsensusStore] = {} self._socket_cache: Dict[str, zmq.Socket] = {} self._monitor_cache: Dict[str, zmq.Socket] = {} self._socket_send_locks: Dict[str, threading.Lock] = {} @@ -403,6 +404,10 @@ class CommonKVManager(BaseKVManager): self.req_to_decode_prefix_len: Dict[int, int] = {} self.decode_kv_args_table = {} self.pp_group = get_parallel().pp_group + if self.pp_size > 1: + self.request_status = PPConsensusStore( + self.pp_size, self.pp_rank, self.pp_group + ) # If a timeout happens on the prefill side, it means prefill instances # fail to receive the KV indices from the decode instance of this request. # These timeout requests should be aborted to release the tree cache. @@ -547,6 +552,12 @@ class CommonKVManager(BaseKVManager): def check_status(self, bootstrap_room: int) -> KVPoll: return self.request_status[bootstrap_room] + def check_status_pp_consensus(self, bootstrap_room: int) -> KVPoll: + statuses = self.request_status.collect(bootstrap_room) + if any(status is None for status in statuses): + return KVPoll.Bootstrapping + return min(statuses) + def update_status(self, bootstrap_room: int, status: KVPoll): current = self.request_status.get(bootstrap_room) if current is None: diff --git a/python/sglang/srt/disaggregation/fake/conn.py b/python/sglang/srt/disaggregation/fake/conn.py index 63d5e1c107..6132350317 100644 --- a/python/sglang/srt/disaggregation/fake/conn.py +++ b/python/sglang/srt/disaggregation/fake/conn.py @@ -74,6 +74,9 @@ class FakeKVSender(BaseKVSender): self.conclude_state = KVPoll.Success return KVPoll.Success + def poll_pp_consensus(self) -> KVPoll: + return self.poll() + def _check_waiting_timeout(self) -> Optional[KVPoll]: # A send() that never comes must not pin the prefill inflight queue forever. # No deadline before init(): the request is still in the bootstrap queue. diff --git a/python/sglang/srt/disaggregation/mooncake/conn.py b/python/sglang/srt/disaggregation/mooncake/conn.py index 8924a03569..e8a268aa55 100644 --- a/python/sglang/srt/disaggregation/mooncake/conn.py +++ b/python/sglang/srt/disaggregation/mooncake/conn.py @@ -3064,31 +3064,37 @@ class MooncakeKVSender(MooncakeFailureExceptionMixin, CommonKVSender): self._record_transfer_indices(kv_indices, state_indices) def poll(self) -> KVPoll: - if self.conclude_state is None: - status = self.kv_mgr.check_status(self.bootstrap_room) - # Hold Success until all staging chunks transferred. With deferred - # allocation also wait for worker cleanup (the zero entry is only - # removed after cleanup), before permitting the room to be reused. - if status == KVPoll.Success and ( - self.kv_mgr._staging_outstanding.get(self.bootstrap_room, 0) > 0 - or ( - self.kv_mgr.defer_decode_allocation - and self.bootstrap_room in self.kv_mgr._staging_outstanding - ) - ): - return KVPoll.Transferring - if status in (KVPoll.Success, KVPoll.Failed): - self.conclude_state = status - self.trace_ctx.trace_req_finish() - elif status == KVPoll.Bootstrapping: - timeout_result = self._check_bootstrap_timeout() - if timeout_result is not None: - return timeout_result + return self._poll_with_status(self.kv_mgr.check_status) - return status - else: + def poll_pp_consensus(self) -> KVPoll: + return self._poll_with_status(self.kv_mgr.check_status_pp_consensus) + + def _poll_with_status(self, check_status) -> KVPoll: + if self.conclude_state is not None: return self.conclude_state + status = check_status(self.bootstrap_room) + # Hold Success until all staging chunks transferred. With deferred + # allocation also wait for worker cleanup (the zero entry is only + # removed after cleanup), before permitting the room to be reused. + if status == KVPoll.Success and ( + self.kv_mgr._staging_outstanding.get(self.bootstrap_room, 0) > 0 + or ( + self.kv_mgr.defer_decode_allocation + and self.bootstrap_room in self.kv_mgr._staging_outstanding + ) + ): + return KVPoll.Transferring + if status in (KVPoll.Success, KVPoll.Failed): + self.conclude_state = status + self.trace_ctx.trace_req_finish() + elif status == KVPoll.Bootstrapping: + timeout_result = self._check_bootstrap_timeout() + if timeout_result is not None: + return timeout_result + + return status + def _init_trace_ctx(self): if self.kv_mgr.enable_trace: self.trace_ctx = TraceReqContext( diff --git a/python/sglang/srt/disaggregation/mori/conn.py b/python/sglang/srt/disaggregation/mori/conn.py index 95b891069a..4da48d7291 100644 --- a/python/sglang/srt/disaggregation/mori/conn.py +++ b/python/sglang/srt/disaggregation/mori/conn.py @@ -1844,6 +1844,12 @@ class MoriKVSender(CommonKVSender): ) def poll(self) -> KVPoll: + return self._poll_with_status(self.kv_mgr.check_status) + + def poll_pp_consensus(self) -> KVPoll: + return self._poll_with_status(self.kv_mgr.check_status_pp_consensus) + + def _poll_with_status(self, check_status) -> KVPoll: if self.conclude_state is not None: return self.conclude_state @@ -1851,7 +1857,7 @@ class MoriKVSender(CommonKVSender): self.conclude_state = KVPoll.Failed return self.conclude_state - status = self.kv_mgr.check_status(self.bootstrap_room) + status = check_status(self.bootstrap_room) if status == KVPoll.Bootstrapping: timeout_result = self._check_bootstrap_timeout() if timeout_result is not None: diff --git a/python/sglang/srt/disaggregation/nixl/conn.py b/python/sglang/srt/disaggregation/nixl/conn.py index 0612bed882..5959701ec1 100644 --- a/python/sglang/srt/disaggregation/nixl/conn.py +++ b/python/sglang/srt/disaggregation/nixl/conn.py @@ -3232,9 +3232,15 @@ class NixlKVSender(CommonKVSender): self.has_sent = True def poll(self) -> KVPoll: + return self._poll_with_status(self.kv_mgr.check_status) + + def poll_pp_consensus(self) -> KVPoll: + return self._poll_with_status(self.kv_mgr.check_status_pp_consensus) + + def _poll_with_status(self, check_status) -> KVPoll: if self._send_failed: return KVPoll.Failed # type: ignore - status = self.kv_mgr.check_status(self.bootstrap_room) + status = check_status(self.bootstrap_room) if status == KVPoll.Bootstrapping: timeout_result = self._check_bootstrap_timeout() if timeout_result is not None: diff --git a/python/sglang/srt/disaggregation/pp_consensus_store.py b/python/sglang/srt/disaggregation/pp_consensus_store.py new file mode 100644 index 0000000000..afa20a3bab --- /dev/null +++ b/python/sglang/srt/disaggregation/pp_consensus_store.py @@ -0,0 +1,172 @@ +from __future__ import annotations + +import logging +import os +import pickle +import queue +import threading +from enum import Enum, auto +from typing import Any, Dict, List, Optional + +import zmq + +from sglang.srt.utils.network import ( + NetworkAddress, + get_local_ip_auto, + get_zmq_socket, + get_zmq_socket_on_host, +) + +logger = logging.getLogger(__name__) + + +class _OpKind(Enum): + PUT = auto() + DELETE = auto() + + +class PPConsensusStore: + """A map that replicates metadata to PP rank 0. + + All ranks may use this as a normal Dict, e.g. ``store["key"] = value`` or ``del store["key"]``. + All modifications done by all ranks are replicated to rank 0 in background. Rank 0 calls + ``collect`` to retrieve values of all ranks for the specified key. + """ + + def __init__(self, pp_size: int, pp_rank: int, pp_group) -> None: + self._pp_size = pp_size + self._pp_rank = pp_rank + self._pp_group = pp_group + self._local_map: Dict[Any, Any] = {} + self._peer_map: Dict[int, Dict[Any, Any]] = {} + self._cache_lock = threading.Lock() + self._replication_queue: Optional[queue.Queue] = None + self._zmq_ctx: Optional[zmq.Context] = None + self._socket: Optional[zmq.Socket] = None + self._init_zmq() + if self._pp_rank > 0: + self._replication_queue = queue.Queue() + self._worker_thread = threading.Thread(target=self._worker_loop, daemon=True) + self._worker_thread.start() + + def _init_zmq(self) -> None: + self._zmq_ctx = zmq.Context() + # PP0 listens. + if self._pp_rank == 0: + host = get_local_ip_auto() + port, self._socket = get_zmq_socket_on_host( + self._zmq_ctx, zmq.PULL, host=host + ) + store_info = (host, port) + logger.info("PPConsensusStore is listening at %s:%d", host, port) + else: + store_info = None + + # Broadcast our zmq host and port to other ranks. + host, port = self._pp_group.broadcast_object(store_info, src=0) + + # PP>0 connects to PP0. + if self._pp_rank > 0: + self._socket = get_zmq_socket( + self._zmq_ctx, + zmq.PUSH, + NetworkAddress(host, port).to_tcp(), + bind=False, + ) + logger.info("PPConsensusStore connected to %s:%d", host, port) + + def _maybe_replicate_to_rank0(self, op: tuple) -> None: + if self._pp_rank > 0: + self._replication_queue.put(op) + + def pop(self, key: Any, default: Any = None) -> Any: + with self._cache_lock: + if key not in self._local_map: + return default + value = self._local_map.pop(key) + self._maybe_replicate_to_rank0((_OpKind.DELETE, self._pp_rank, key)) + return value + + def get(self, key: Any, default: Any = None) -> Any: + with self._cache_lock: + return self._local_map.get(key, default) + + def __contains__(self, key: Any) -> bool: + with self._cache_lock: + return key in self._local_map + + def __getitem__(self, key: Any) -> Any: + with self._cache_lock: + return self._local_map[key] + + def __setitem__(self, key: Any, value: Any) -> None: + with self._cache_lock: + self._local_map[key] = value + self._maybe_replicate_to_rank0((_OpKind.PUT, self._pp_rank, key, value)) + + def __delitem__(self, key: Any) -> None: + with self._cache_lock: + if key not in self._local_map: + raise KeyError(key) + self._local_map.pop(key) + self._maybe_replicate_to_rank0((_OpKind.DELETE, self._pp_rank, key)) + + def close(self) -> None: + if self._pp_rank > 0: + self._replication_queue.put(None) + self._zmq_ctx.term() # This blocks until all zmq socket closed. + self._worker_thread.join() + + def _worker_loop(self) -> None: + try: + if self._pp_rank == 0: + self._recv_loop() + else: + self._send_loop() + except zmq.error.ContextTerminated: + pass # Raised by close(). + except Exception as e: + logger.critical( + "PPConsensusStore background thread crashed: %s", + e, + ) + for handler in logger.handlers: + handler.flush() + os._exit(1) + finally: + self._socket.close(linger=0) + + def _recv_once(self) -> None: + op = pickle.loads(self._socket.recv()) + if op[0] == _OpKind.PUT: + _, rank, key, value = op + logger.debug("recv put rank=%s %s = %s", rank, key, value) + with self._cache_lock: + self._peer_map.setdefault(rank, {})[key] = value + elif op[0] == _OpKind.DELETE: + _, rank, key = op + logger.debug("recv delete rank=%s %s", rank, key) + with self._cache_lock: + self._peer_map.get(rank, {}).pop(key, None) + + def _recv_loop(self) -> None: + assert self._pp_rank == 0 + while True: + self._recv_once() + + def _send_loop(self) -> None: + assert self._pp_rank > 0 + while True: + op = self._replication_queue.get() + if op is None: # A signal for shutdown. + return + logger.debug("send %s", op) + self._socket.send(pickle.dumps(op)) + + def collect(self, key: Any) -> List[Any]: + assert self._pp_rank == 0, "collect can only be used on PP rank 0" + with self._cache_lock: + values: List[Any] = [self._local_map.get(key)] + for rank in range(1, self._pp_size): + values.append(self._peer_map.get(rank, {}).get(key)) + return values diff --git a/python/sglang/srt/disaggregation/prefill.py b/python/sglang/srt/disaggregation/prefill.py index 6c9ad51ae3..556dcc5798 100644 --- a/python/sglang/srt/disaggregation/prefill.py +++ b/python/sglang/srt/disaggregation/prefill.py @@ -60,7 +60,7 @@ from sglang.srt.disaggregation.utils import ( is_mla_backend, is_unadmitted_reject, poll_and_all_reduce_attn_cp_tp_group, - poll_and_all_reduce_pp, + poll_and_all_reduce_prefill_pp, prepare_abort, setup_state_kv_args, ) @@ -194,6 +194,7 @@ class PrefillBootstrapQueue: self.scheduler.tp_worker.model_runner.effective_logical_max_total_num_tokens ) self.transfer_backend = transfer_backend + self.pp_poll_sync_work_list: List = [] if envs.SGLANG_DISAGG_STAGING_BUFFER.get(): if self.is_mla_backend: raise RuntimeError( @@ -458,47 +459,25 @@ class PrefillBootstrapQueue: req.sampling_params.max_new_tokens = 1 @scheduler_stage_method(SCHEDULER_STAGE_PROCESS_QUEUE) - def pop_bootstrapped( - self, - return_failed_reqs: bool = False, - pp_good_rids: Optional[List[str]] = None, - pp_bad_rids: Optional[List[str]] = None, - ) -> List[Req] | tuple[List[Req], List[Req]]: - """ - pop the reqs which has finished bootstrapping - - return_failed_reqs: For PP, on rank 0, also return the failed reqs to notify the next rank - pp_good_rids: RIDs that PP consensus determined as WaitingForInput. - pp_bad_rids: RIDs that PP consensus determined as Failed. - """ + def pop_bootstrapped(self) -> List[Req]: + """Pop the reqs which have finished bootstrapping.""" bootstrapped_reqs = [] - failed_reqs = [] indices_to_remove = set() if len(self.queue) == 0: - if return_failed_reqs is False: - return [] - else: - return [], [] + return [] if self.pp_size > 1: - polls = poll_and_all_reduce_pp( - (req.rid for req in self.queue), - KVPoll.WaitingForInput, - pp_good_rids, - pp_bad_rids, + polls = poll_and_all_reduce_prefill_pp( + [req.disagg_kv_sender for req in self.queue], + self.scheduler.attn_cp_cpu_group, + self.scheduler.attn_tp_cpu_group, + self.scheduler.pp_group, + self.pp_rank, + self.pp_size, + self.pp_poll_sync_work_list, ) - uncovered = [i for i, poll in enumerate(polls) if poll is None] - if uncovered: - local_polls = poll_and_all_reduce_attn_cp_tp_group( - [self.queue[i].disagg_kv_sender for i in uncovered], - self.scheduler.attn_cp_cpu_group, - self.scheduler.attn_tp_cpu_group, - ) - for i, local_poll in zip(uncovered, local_polls): - if local_poll == KVPoll.Failed: - polls[i] = KVPoll.Failed else: polls = poll_and_all_reduce_attn_cp_tp_group( [req.disagg_kv_sender for req in self.queue], @@ -507,13 +486,9 @@ class PrefillBootstrapQueue: ) for i, (req, poll) in enumerate(zip(self.queue, polls)): - if poll is None: - continue - if poll == KVPoll.Failed: self.scheduler.handle_bootstrap_failure(req) indices_to_remove.add(i) - failed_reqs.append(req) elif poll == KVPoll.Bootstrapping: if ( ( @@ -558,10 +533,7 @@ class PrefillBootstrapQueue: entry for i, entry in enumerate(self.queue) if i not in indices_to_remove ] - if return_failed_reqs is False: - return bootstrapped_reqs - else: - return bootstrapped_reqs, failed_reqs + return bootstrapped_reqs def release_memory_occupation(self): self.queue.clear() diff --git a/python/sglang/srt/disaggregation/utils.py b/python/sglang/srt/disaggregation/utils.py index db8cb7a27f..0eb819e23d 100644 --- a/python/sglang/srt/disaggregation/utils.py +++ b/python/sglang/srt/disaggregation/utils.py @@ -21,6 +21,8 @@ import torch.distributed as dist from sglang.srt.configs.model_config import get_dsa_mtp_topk_width, is_deepseek_dsa from sglang.srt.disaggregation.base import KVPoll +from sglang.srt.distributed.communication_tags import P2PTag +from sglang.srt.distributed.parallel_state import GroupCoordinator from sglang.srt.environ import envs from sglang.srt.runtime_context import ( get_disagg, @@ -49,6 +51,44 @@ if is_npu(): FAKE_BOOTSTRAP_HOST = "2.2.2.2" +def pp_sync_polls( + polls: Optional[List[int]], + pp_group: GroupCoordinator, + pp_rank: int, + pp_size: int, + sync_work_list: List, +) -> List[int]: + """Synchronize poll states from PP0 through the PP pipeline. + + PP0 provides the data. Each later PP rank receives it from the previous + rank and asynchronously forwards it to the next rank. + """ + if pp_size <= 1: + assert polls is not None + return polls + + for p2p_work in sync_work_list: + p2p_work.work.wait() + sync_work_list.clear() + + data = polls + if pp_rank > 0: + data = pp_group.recv_object( + src=pp_rank - 1, + tag=P2PTag.DISAGG_BOOTSTRAP_PP_SYNC, + ) + if pp_rank + 1 < pp_size: + sync_work_list.extend( + pp_group.send_object( + data, + dst=pp_rank + 1, + async_send=True, + tag=P2PTag.DISAGG_BOOTSTRAP_PP_SYNC, + ) + ) + return data + + def poll_and_all_reduce_pp( rids: Iterable[str], ready_poll: int, @@ -67,6 +107,25 @@ def poll_and_all_reduce_pp( ] +def poll_and_all_reduce_prefill_pp( + pollers: List[CommonKVSender], + attn_cp_cpu_group: dist.ProcessGroup, + attn_tp_cpu_group: dist.ProcessGroup, + pp_group: GroupCoordinator, + pp_rank: int, + pp_size: int, + pp_poll_sync_work_list: List[torch.distributed.Work], +) -> List[int]: + polls: List[int] = [] + if pp_rank == 0: + polls = _poll_with_failure_injection_pp(pollers) + polls = _all_reduce_polls(polls, attn_tp_cpu_group) + polls = _all_reduce_polls(polls, attn_cp_cpu_group) + # Propagate PP0's data to other ranks + polls = pp_sync_polls(polls, pp_group, pp_rank, pp_size, pp_poll_sync_work_list) + return polls + + def get_dsa_seed_metadata_dim(hf_config) -> int: """Return the model-defined PD seed width, independent of local spec mode.""" if not getattr(hf_config, "index_share_for_mtp_iteration", False): @@ -163,6 +222,17 @@ def _poll_with_failure_injection(pollers) -> List[int]: return [int(poller.poll()) for poller in pollers] +def _poll_with_failure_injection_pp(pollers) -> List[int]: + if (failure_prob := envs.SGLANG_TEST_DISAGG_FAILURE_PROB.get()) > 0: + return [ + int(KVPoll.Failed) + if random.random() < failure_prob + else int(poller.poll_pp_consensus()) + for poller in pollers + ] + return [int(poller.poll_pp_consensus()) for poller in pollers] + + def _is_fake_transfer(req: Req) -> bool: return req.bootstrap_host == FAKE_BOOTSTRAP_HOST or ( req.bootstrap_host is None diff --git a/python/sglang/srt/distributed/communication_tags.py b/python/sglang/srt/distributed/communication_tags.py index 55962e7d69..50e9038ffa 100644 --- a/python/sglang/srt/distributed/communication_tags.py +++ b/python/sglang/srt/distributed/communication_tags.py @@ -13,3 +13,4 @@ class P2PTag(IntEnum): DEFAULT = 0 HICACHE_PP_SYNC = int.from_bytes(b"PpHi", byteorder="big") GRAMMAR_PP_SYNC = int.from_bytes(b"PpGr", byteorder="big") + DISAGG_BOOTSTRAP_PP_SYNC = int.from_bytes(b"PpBs", byteorder="big") diff --git a/python/sglang/srt/managers/scheduler_pp_mixin.py b/python/sglang/srt/managers/scheduler_pp_mixin.py index 0013308fa9..2c5a3e2830 100644 --- a/python/sglang/srt/managers/scheduler_pp_mixin.py +++ b/python/sglang/srt/managers/scheduler_pp_mixin.py @@ -223,18 +223,15 @@ class SchedulerPPMixin: ==================================================================== Stage P recv ith req from previous stage - recv ith bootstrap req from previous stage + pop bootstrapped reqs (PPConsensusStore + pp_sync polls) recv ith transferred req from previous stage recv ith proxy from previous stage run ith batch - recv prev (i+1) % mb_size th consensus bootstrapped req from previous stage - local consensus on bootstrapped req recv prev (i+1) % mb_size th release req from previous stage local consensus on release req recv prev (i+1) % mb_size th outputs process batch result of prev (i+1)% mb_size th batch (can be run in parallel with the curr batch GPU computation) send ith req to next stage - send ith bootstrap req to next stage send ith transferred req to next stage send ith proxy to next stage send current stage's outputs to next stage (can be stashed and delayed to send later) @@ -242,23 +239,17 @@ class SchedulerPPMixin: the above order can be optimized and reordered to minimize communication-related CPU stall and overhead bubbles. ==================================================================== - There are two additional elements compared to the regular schedule: - - Bootstrap Requests + Release Requests: - - Both can have local failure and need to be consensus on. PP needs to guarantee eventual consistency of local failure and flush malfunc requests out as soft error. + Release Requests still use the two-pass PP consensus. Bootstrap uses + PPConsensusStore instead of the legacy two-round RID sync. """ self.init_pp_loop_state() # PD additional state initialization - bmbs = [None] * self.pp_loop_size tmbs = [None] * self.pp_loop_size - consensus_bootstrapped_rids: Optional[List[str]] = None transferred_rids: List[str] = [] release_rids: Optional[List[str]] = None - send_bootstrapped_work = [] send_transfer_work = [] - send_consensus_bootstrapped_work = [] send_release_work = [] while True: @@ -273,7 +264,6 @@ class SchedulerPPMixin: next_pp_outputs = None next_release_rids = None - next_consensus_bootstrapped_rids = None d2h_event = None next_batch_result = None @@ -282,9 +272,7 @@ class SchedulerPPMixin: if not self.pp_group.is_last_rank: self._pp_commit_comm_work(self.send_req_work) - bootstrapped_rids = self._pp_pd_get_bootstrapped_ids() - bmbs[mb_id] = bootstrapped_rids - self._pp_commit_comm_work(send_bootstrapped_work) + self.process_bootstrapped_queue() transferred_rids = self._pp_pd_get_prefill_transferred_ids() self._pp_commit_comm_work(send_transfer_work) @@ -332,28 +320,12 @@ class SchedulerPPMixin: next_mb_id, ) ) - send_consensus_bootstrapped_work, consensus_bootstrapped_rids = ( - self._pp_pd_send_consensus_bootstrapped_ids( - bmbs, - next_first_rank_mb_id, - consensus_bootstrapped_rids, - bootstrapped_rids, - ) - ) send_release_work, release_rids = ( self._pp_pd_send_consensus_release_ids( tmbs, next_first_rank_mb_id, release_rids, transferred_rids ) ) - if bmbs[next_mb_id] is not None: - next_consensus_bootstrapped_rids = ( - self._pp_recv_pyobj_from_prev_stage() - ) - next_consensus_bootstrapped_rids = self.process_bootstrapped_queue( - next_consensus_bootstrapped_rids - ) - self._pp_commit_comm_work(send_consensus_bootstrapped_work) if tmbs[next_mb_id] is not None: next_release_rids = self._pp_recv_pyobj_from_prev_stage() self._pp_commit_comm_work(send_release_work) @@ -372,9 +344,6 @@ class SchedulerPPMixin: self.send_req_work = self._pp_send_pyobj_to_next_stage( recv_reqs, async_send=True ) - send_bootstrapped_work = self._pp_send_pyobj_to_next_stage( - bootstrapped_rids, async_send=True - ) send_transfer_work = self._pp_send_pyobj_to_next_stage( transferred_rids, async_send=True ) @@ -383,7 +352,6 @@ class SchedulerPPMixin: self.pp_outputs = next_pp_outputs release_rids = next_release_rids - consensus_bootstrapped_rids = next_consensus_bootstrapped_rids self.running_batch.batch_is_full = False @@ -636,67 +604,9 @@ class SchedulerPPMixin: str, deque[Tuple[Dict[str, torch.Tensor], Optional[torch.Event]]] ] = defaultdict(deque) - def process_bootstrapped_queue( - self: Scheduler, bootstrapped_rids: Optional[List[str]] - ): - # finished consensus bootstrapped reqs and prepare the waiting queue - if bootstrapped_rids is not None: - ( - good_consensus_bootstrapped_rids, - bad_consensus_bootstrapped_rids, - ) = bootstrapped_rids - good_reqs, failed_reqs = ( - self.disagg_prefill_bootstrap_queue.pop_bootstrapped( - return_failed_reqs=True, - pp_good_rids=good_consensus_bootstrapped_rids, - pp_bad_rids=bad_consensus_bootstrapped_rids, - ) - ) - self.waiting_queue.extend(good_reqs) - return [[req.rid for req in good_reqs], [req.rid for req in failed_reqs]] - return None - - def _pp_pd_get_bootstrapped_ids(self: Scheduler): - # communicate pre-consensus bootstrapp reqs - if self.pp_group.is_first_rank: - # First rank, pop the bootstrap reqs from the bootstrap queue - good_bootstrapped_rids, bad_bootstrapped_rids = self.get_rids( - self.disagg_prefill_bootstrap_queue.queue, - True, - [KVPoll.WaitingForInput], - [KVPoll.Failed], - ) - else: - # Other ranks, receive the bootstrap reqs info from the previous rank and ensure the consensus - prev_bootstrapped_rids = self._pp_recv_pyobj_from_prev_stage() - prev_good_bootstrapped_rids, prev_bad_bootstrapped_rids = ( - prev_bootstrapped_rids - ) - curr_good_bootstrapped_rids, curr_bad_bootstrapped_rids = self.get_rids( - self.disagg_prefill_bootstrap_queue.queue, - True, - [KVPoll.WaitingForInput], - [KVPoll.Failed], - ) - good_bootstrapped_rids = list( - set(prev_good_bootstrapped_rids) & set(curr_good_bootstrapped_rids) - ) - bad_bootstrapped_rids = list( - set(prev_bad_bootstrapped_rids) | set(curr_bad_bootstrapped_rids) - ) - # Route locally-aborted reqs through the bad-union consensus so every PP - # rank flushes them in the same consensus round, regardless of when the - # AbortReq reaches each rank and regardless of whether - # disagg_kv_sender.abort() drives the poll to Failed (it is optional). - aborted_rids = { - req.rid - for req in self.disagg_prefill_bootstrap_queue.queue - if isinstance(req.finished_reason, FINISH_ABORT) - } - good_bootstrapped_rids, bad_bootstrapped_rids = self._route_aborts_to_bad( - good_bootstrapped_rids, bad_bootstrapped_rids, aborted_rids - ) - return [good_bootstrapped_rids, bad_bootstrapped_rids] + def process_bootstrapped_queue(self: Scheduler): + reqs = self.disagg_prefill_bootstrap_queue.pop_bootstrapped() + self.waiting_queue.extend(reqs) def _pp_pd_get_prefill_transferred_ids(self: Scheduler): # get the current stage transfer success diff --git a/test/registered/unit/disaggregation/test_pp_consensus_store.py b/test/registered/unit/disaggregation/test_pp_consensus_store.py new file mode 100644 index 0000000000..5f0dfa2b6a --- /dev/null +++ b/test/registered/unit/disaggregation/test_pp_consensus_store.py @@ -0,0 +1,192 @@ +"""Unit tests for srt/disaggregation/pp_consensus_store.""" + +import time +import unittest +from collections.abc import Callable + +from sglang.srt.disaggregation.pp_consensus_store import PPConsensusStore +from sglang.test.ci.ci_register import register_cpu_ci +from sglang.test.test_utils import CustomTestCase + +register_cpu_ci(est_time=5, suite="base-a-test-cpu") + + +class _MockPPGroup: + def __init__(self): + self._obj = None + + def broadcast_object(self, obj, src=0): + if obj is not None: + self._obj = obj + return self._obj + + +class TestPPConsensusStore(CustomTestCase): + def _wait_until(self, condition: Callable[[], bool]) -> None: + now = time.time() + start = now + deadline = now + 1 + delay_time = 0.01 + while now < deadline and not condition(): + time.sleep(delay_time) + now = time.time() + delay_time *= 2 + self.assertTrue( + now < deadline, f"Condition not met after {now - start} seconds" + ) + + def test_setitem(self): + group = _MockPPGroup() + store0 = PPConsensusStore(1, 0, group) + store0["key"] = 10 + self.assertEqual(store0["key"], 10) + store0.close() + + def test_setitem_replicate_to_rank0(self): + group = _MockPPGroup() + store0 = PPConsensusStore(2, 0, group) + store1 = PPConsensusStore(2, 1, group) + store0["key"] = 10 + store1["key"] = 11 + self._wait_until(lambda: store0.collect("key") == [10, 11]) + store0.close() + store1.close() + + def test_contains(self): + group = _MockPPGroup() + store0 = PPConsensusStore(1, 0, group) + store0["key"] = 10 + self.assertTrue("key" in store0) + self.assertFalse("no_such_key" in store0) + store0.close() + + def test_get(self): + group = _MockPPGroup() + store0 = PPConsensusStore(1, 0, group) + store0["key"] = 10 + self.assertEqual(store0.get("key"), 10) + store0.close() + + def test_get_not_exist(self): + group = _MockPPGroup() + store0 = PPConsensusStore(1, 0, group) + self.assertIsNone(store0.get("no_such_key")) + self.assertEqual(store0.get("no_such_key", "default"), "default") + store0.close() + + def test_get_default(self): + group = _MockPPGroup() + store0 = PPConsensusStore(1, 0, group) + self.assertEqual(store0.get("no_such_key", "default"), "default") + store0.close() + + def test_getitem_not_exist(self): + group = _MockPPGroup() + store0 = PPConsensusStore(1, 0, group) + with self.assertRaises(KeyError): + store0["no_such_key"] + store0.close() + + def test_delitem(self): + group = _MockPPGroup() + store0 = PPConsensusStore(1, 0, group) + store0["key"] = 10 + del store0["key"] + self.assertFalse("key" in store0) + store0.close() + + def test_delitem_not_exist(self): + group = _MockPPGroup() + store0 = PPConsensusStore(1, 0, group) + with self.assertRaises(KeyError): + del store0["no_such_key"] + store0.close() + + def test_delitem_replicate_to_rank0(self): + group = _MockPPGroup() + store0 = PPConsensusStore(2, 0, group) + store1 = PPConsensusStore(2, 1, group) + store0["key"] = 10 + store1["key"] = 11 + self._wait_until(lambda: store0.collect("key") == [10, 11]) + del store1["key"] + self._wait_until(lambda: store0.collect("key") == [10, None]) + store0.close() + store1.close() + + def test_pop(self): + group = _MockPPGroup() + store0 = PPConsensusStore(1, 0, group) + store0["key"] = 10 + self.assertEqual(store0.pop("key"), 10) + self.assertFalse("key" in store0) + store0.close() + + def test_pop_not_exist(self): + group = _MockPPGroup() + store0 = PPConsensusStore(1, 0, group) + self.assertIsNone(store0.pop("no_such_key")) + store0.close() + + def test_pop_default(self): + group = _MockPPGroup() + store0 = PPConsensusStore(1, 0, group) + self.assertEqual(store0.pop("no_such_key", "default"), "default") + store0.close() + + def test_pop_replicate_to_rank0(self): + group = _MockPPGroup() + store0 = PPConsensusStore(2, 0, group) + store1 = PPConsensusStore(2, 1, group) + store0["key"] = 10 + store1["key"] = 11 + self._wait_until(lambda: store0.collect("key") == [10, 11]) + self.assertEqual(store1.pop("key"), 11) + self.assertFalse("key" in store1) + self._wait_until(lambda: store0.collect("key") == [10, None]) + store0.close() + store1.close() + + def test_collect(self): + group = _MockPPGroup() + store0 = PPConsensusStore(3, 0, group) + store1 = PPConsensusStore(3, 1, group) + store2 = PPConsensusStore(3, 2, group) + store0["key"] = 10 + store1["key"] = 11 + store2["key"] = 12 + self._wait_until(lambda: store0.collect("key") == [10, 11, 12]) + store0.close() + store1.close() + store2.close() + + def test_collect_missing(self): + group = _MockPPGroup() + store0 = PPConsensusStore(3, 0, group) + store1 = PPConsensusStore(3, 1, group) + store2 = PPConsensusStore(3, 2, group) + store0["key"] = 10 + store2["key"] = 12 + self._wait_until(lambda: store0.collect("key") == [10, None, 12]) + store0.close() + store1.close() + store2.close() + + def test_collect_not_exist(self): + group = _MockPPGroup() + store0 = PPConsensusStore(1, 0, group) + self.assertEqual(store0.collect("no_such_key"), [None]) + store0.close() + + def test_collect_requires_pp0(self): + group = _MockPPGroup() + store0 = PPConsensusStore(2, 0, group) + store1 = PPConsensusStore(2, 1, group) + with self.assertRaises(AssertionError): + store1.collect("key") + store0.close() + store1.close() + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/unit/managers/test_disagg_idle_step_counters.py b/test/registered/unit/managers/test_disagg_idle_step_counters.py index 2aba2b0582..2719c6dd8e 100644 --- a/test/registered/unit/managers/test_disagg_idle_step_counters.py +++ b/test/registered/unit/managers/test_disagg_idle_step_counters.py @@ -552,7 +552,6 @@ class TestSchedulerIdleStepCounters(CustomTestCase): scheduler._pp_commit_send_output_work_and_preprocess_output_tensors = Mock( return_value=(None, GenerationBatchResult(), Mock()) ) - scheduler._pp_pd_get_bootstrapped_ids = Mock(return_value=None) scheduler._pp_pd_get_prefill_transferred_ids = Mock(return_value=None) scheduler._pp_pd_get_retract_ids = Mock(return_value=None) scheduler._pp_pd_get_prealloc_ids = Mock(return_value=None) diff --git a/test/registered/unit/managers/test_scheduler_hicache_events.py b/test/registered/unit/managers/test_scheduler_hicache_events.py index 9054de5ded..2df3d91bb8 100644 --- a/test/registered/unit/managers/test_scheduler_hicache_events.py +++ b/test/registered/unit/managers/test_scheduler_hicache_events.py @@ -124,7 +124,8 @@ class TestSchedulerHiCacheEvents(unittest.TestCase): s.running_mbs = [self.running_batch] s.last_mbs = [None] s.ingest_requests = Mock(return_value=[]) - s._pp_pd_get_bootstrapped_ids = Mock(return_value=[]) + s.waiting_queue = [] + s.disagg_prefill_bootstrap_queue = SimpleNamespace(pop_bootstrapped=lambda: []) s._pp_pd_get_prefill_transferred_ids = Mock(return_value=[]) s._pp_commit_comm_work = Mock() s.get_new_batch_prefill = self.calls.admit