Co-authored-by: inkcherry <mingzhi.liu@amd.com> Co-authored-by: Shangming Cai <csmthu@gmail.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
inkcherry
Shangming Cai
Claude Opus 5.5
parent
f1e0472fcc
commit
348689da40
@@ -33,6 +33,7 @@ from sglang.srt.disaggregation.common.bootstrap import (
|
||||
BootstrapNotification,
|
||||
DeferredBootstrap,
|
||||
)
|
||||
from sglang.srt.disaggregation.pp_consensus_store import PPConsensusStore
|
||||
from sglang.srt.disaggregation.utils import (
|
||||
DisaggregationMode,
|
||||
filter_kv_indices_for_cp_rank,
|
||||
@@ -356,7 +357,7 @@ class CommonKVManager(BaseKVManager):
|
||||
)
|
||||
logger.debug(f"kv manager bind to {self.local_ip}:{self.rank_port}")
|
||||
|
||||
self.request_status: Dict[int, KVPoll] = {}
|
||||
self.request_status: Union[Dict[int, KVPoll], PPConsensusStore] = {}
|
||||
self._socket_cache: Dict[str, zmq.Socket] = {}
|
||||
self._monitor_cache: Dict[str, zmq.Socket] = {}
|
||||
self._socket_send_locks: Dict[str, threading.Lock] = {}
|
||||
@@ -403,6 +404,10 @@ class CommonKVManager(BaseKVManager):
|
||||
self.req_to_decode_prefix_len: Dict[int, int] = {}
|
||||
self.decode_kv_args_table = {}
|
||||
self.pp_group = get_parallel().pp_group
|
||||
if self.pp_size > 1:
|
||||
self.request_status = PPConsensusStore(
|
||||
self.pp_size, self.pp_rank, self.pp_group
|
||||
)
|
||||
# If a timeout happens on the prefill side, it means prefill instances
|
||||
# fail to receive the KV indices from the decode instance of this request.
|
||||
# These timeout requests should be aborted to release the tree cache.
|
||||
@@ -547,6 +552,12 @@ class CommonKVManager(BaseKVManager):
|
||||
def check_status(self, bootstrap_room: int) -> KVPoll:
|
||||
return self.request_status[bootstrap_room]
|
||||
|
||||
def check_status_pp_consensus(self, bootstrap_room: int) -> KVPoll:
|
||||
statuses = self.request_status.collect(bootstrap_room)
|
||||
if any(status is None for status in statuses):
|
||||
return KVPoll.Bootstrapping
|
||||
return min(statuses)
|
||||
|
||||
def update_status(self, bootstrap_room: int, status: KVPoll):
|
||||
current = self.request_status.get(bootstrap_room)
|
||||
if current is None:
|
||||
|
||||
@@ -74,6 +74,9 @@ class FakeKVSender(BaseKVSender):
|
||||
self.conclude_state = KVPoll.Success
|
||||
return KVPoll.Success
|
||||
|
||||
def poll_pp_consensus(self) -> KVPoll:
|
||||
return self.poll()
|
||||
|
||||
def _check_waiting_timeout(self) -> Optional[KVPoll]:
|
||||
# A send() that never comes must not pin the prefill inflight queue forever.
|
||||
# No deadline before init(): the request is still in the bootstrap queue.
|
||||
|
||||
@@ -3064,31 +3064,37 @@ class MooncakeKVSender(MooncakeFailureExceptionMixin, CommonKVSender):
|
||||
self._record_transfer_indices(kv_indices, state_indices)
|
||||
|
||||
def poll(self) -> KVPoll:
|
||||
if self.conclude_state is None:
|
||||
status = self.kv_mgr.check_status(self.bootstrap_room)
|
||||
# Hold Success until all staging chunks transferred. With deferred
|
||||
# allocation also wait for worker cleanup (the zero entry is only
|
||||
# removed after cleanup), before permitting the room to be reused.
|
||||
if status == KVPoll.Success and (
|
||||
self.kv_mgr._staging_outstanding.get(self.bootstrap_room, 0) > 0
|
||||
or (
|
||||
self.kv_mgr.defer_decode_allocation
|
||||
and self.bootstrap_room in self.kv_mgr._staging_outstanding
|
||||
)
|
||||
):
|
||||
return KVPoll.Transferring
|
||||
if status in (KVPoll.Success, KVPoll.Failed):
|
||||
self.conclude_state = status
|
||||
self.trace_ctx.trace_req_finish()
|
||||
elif status == KVPoll.Bootstrapping:
|
||||
timeout_result = self._check_bootstrap_timeout()
|
||||
if timeout_result is not None:
|
||||
return timeout_result
|
||||
return self._poll_with_status(self.kv_mgr.check_status)
|
||||
|
||||
return status
|
||||
else:
|
||||
def poll_pp_consensus(self) -> KVPoll:
|
||||
return self._poll_with_status(self.kv_mgr.check_status_pp_consensus)
|
||||
|
||||
def _poll_with_status(self, check_status) -> KVPoll:
|
||||
if self.conclude_state is not None:
|
||||
return self.conclude_state
|
||||
|
||||
status = check_status(self.bootstrap_room)
|
||||
# Hold Success until all staging chunks transferred. With deferred
|
||||
# allocation also wait for worker cleanup (the zero entry is only
|
||||
# removed after cleanup), before permitting the room to be reused.
|
||||
if status == KVPoll.Success and (
|
||||
self.kv_mgr._staging_outstanding.get(self.bootstrap_room, 0) > 0
|
||||
or (
|
||||
self.kv_mgr.defer_decode_allocation
|
||||
and self.bootstrap_room in self.kv_mgr._staging_outstanding
|
||||
)
|
||||
):
|
||||
return KVPoll.Transferring
|
||||
if status in (KVPoll.Success, KVPoll.Failed):
|
||||
self.conclude_state = status
|
||||
self.trace_ctx.trace_req_finish()
|
||||
elif status == KVPoll.Bootstrapping:
|
||||
timeout_result = self._check_bootstrap_timeout()
|
||||
if timeout_result is not None:
|
||||
return timeout_result
|
||||
|
||||
return status
|
||||
|
||||
def _init_trace_ctx(self):
|
||||
if self.kv_mgr.enable_trace:
|
||||
self.trace_ctx = TraceReqContext(
|
||||
|
||||
@@ -1844,6 +1844,12 @@ class MoriKVSender(CommonKVSender):
|
||||
)
|
||||
|
||||
def poll(self) -> KVPoll:
|
||||
return self._poll_with_status(self.kv_mgr.check_status)
|
||||
|
||||
def poll_pp_consensus(self) -> KVPoll:
|
||||
return self._poll_with_status(self.kv_mgr.check_status_pp_consensus)
|
||||
|
||||
def _poll_with_status(self, check_status) -> KVPoll:
|
||||
if self.conclude_state is not None:
|
||||
return self.conclude_state
|
||||
|
||||
@@ -1851,7 +1857,7 @@ class MoriKVSender(CommonKVSender):
|
||||
self.conclude_state = KVPoll.Failed
|
||||
return self.conclude_state
|
||||
|
||||
status = self.kv_mgr.check_status(self.bootstrap_room)
|
||||
status = check_status(self.bootstrap_room)
|
||||
if status == KVPoll.Bootstrapping:
|
||||
timeout_result = self._check_bootstrap_timeout()
|
||||
if timeout_result is not None:
|
||||
|
||||
@@ -3232,9 +3232,15 @@ class NixlKVSender(CommonKVSender):
|
||||
self.has_sent = True
|
||||
|
||||
def poll(self) -> KVPoll:
|
||||
return self._poll_with_status(self.kv_mgr.check_status)
|
||||
|
||||
def poll_pp_consensus(self) -> KVPoll:
|
||||
return self._poll_with_status(self.kv_mgr.check_status_pp_consensus)
|
||||
|
||||
def _poll_with_status(self, check_status) -> KVPoll:
|
||||
if self._send_failed:
|
||||
return KVPoll.Failed # type: ignore
|
||||
status = self.kv_mgr.check_status(self.bootstrap_room)
|
||||
status = check_status(self.bootstrap_room)
|
||||
if status == KVPoll.Bootstrapping:
|
||||
timeout_result = self._check_bootstrap_timeout()
|
||||
if timeout_result is not None:
|
||||
|
||||
@@ -0,0 +1,172 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import os
|
||||
import pickle
|
||||
import queue
|
||||
import threading
|
||||
from enum import Enum, auto
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
import zmq
|
||||
|
||||
from sglang.srt.utils.network import (
|
||||
NetworkAddress,
|
||||
get_local_ip_auto,
|
||||
get_zmq_socket,
|
||||
get_zmq_socket_on_host,
|
||||
)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class _OpKind(Enum):
|
||||
PUT = auto()
|
||||
DELETE = auto()
|
||||
|
||||
|
||||
class PPConsensusStore:
|
||||
"""A map that replicates metadata to PP rank 0.
|
||||
|
||||
All ranks may use this as a normal Dict, e.g. ``store["key"] = value`` or ``del store["key"]``.
|
||||
All modifications done by all ranks are replicated to rank 0 in background. Rank 0 calls
|
||||
``collect`` to retrieve values of all ranks for the specified key.
|
||||
"""
|
||||
|
||||
def __init__(self, pp_size: int, pp_rank: int, pp_group) -> None:
|
||||
self._pp_size = pp_size
|
||||
self._pp_rank = pp_rank
|
||||
self._pp_group = pp_group
|
||||
self._local_map: Dict[Any, Any] = {}
|
||||
self._peer_map: Dict[int, Dict[Any, Any]] = {}
|
||||
self._cache_lock = threading.Lock()
|
||||
self._replication_queue: Optional[queue.Queue] = None
|
||||
self._zmq_ctx: Optional[zmq.Context] = None
|
||||
self._socket: Optional[zmq.Socket] = None
|
||||
self._init_zmq()
|
||||
if self._pp_rank > 0:
|
||||
self._replication_queue = queue.Queue()
|
||||
self._worker_thread = threading.Thread(target=self._worker_loop, daemon=True)
|
||||
self._worker_thread.start()
|
||||
|
||||
def _init_zmq(self) -> None:
|
||||
self._zmq_ctx = zmq.Context()
|
||||
# PP0 listens.
|
||||
if self._pp_rank == 0:
|
||||
host = get_local_ip_auto()
|
||||
port, self._socket = get_zmq_socket_on_host(
|
||||
self._zmq_ctx, zmq.PULL, host=host
|
||||
)
|
||||
store_info = (host, port)
|
||||
logger.info("PPConsensusStore is listening at %s:%d", host, port)
|
||||
else:
|
||||
store_info = None
|
||||
|
||||
# Broadcast our zmq host and port to other ranks.
|
||||
host, port = self._pp_group.broadcast_object(store_info, src=0)
|
||||
|
||||
# PP>0 connects to PP0.
|
||||
if self._pp_rank > 0:
|
||||
self._socket = get_zmq_socket(
|
||||
self._zmq_ctx,
|
||||
zmq.PUSH,
|
||||
NetworkAddress(host, port).to_tcp(),
|
||||
bind=False,
|
||||
)
|
||||
logger.info("PPConsensusStore connected to %s:%d", host, port)
|
||||
|
||||
def _maybe_replicate_to_rank0(self, op: tuple) -> None:
|
||||
if self._pp_rank > 0:
|
||||
self._replication_queue.put(op)
|
||||
|
||||
def pop(self, key: Any, default: Any = None) -> Any:
|
||||
with self._cache_lock:
|
||||
if key not in self._local_map:
|
||||
return default
|
||||
value = self._local_map.pop(key)
|
||||
self._maybe_replicate_to_rank0((_OpKind.DELETE, self._pp_rank, key))
|
||||
return value
|
||||
|
||||
def get(self, key: Any, default: Any = None) -> Any:
|
||||
with self._cache_lock:
|
||||
return self._local_map.get(key, default)
|
||||
|
||||
def __contains__(self, key: Any) -> bool:
|
||||
with self._cache_lock:
|
||||
return key in self._local_map
|
||||
|
||||
def __getitem__(self, key: Any) -> Any:
|
||||
with self._cache_lock:
|
||||
return self._local_map[key]
|
||||
|
||||
def __setitem__(self, key: Any, value: Any) -> None:
|
||||
with self._cache_lock:
|
||||
self._local_map[key] = value
|
||||
self._maybe_replicate_to_rank0((_OpKind.PUT, self._pp_rank, key, value))
|
||||
|
||||
def __delitem__(self, key: Any) -> None:
|
||||
with self._cache_lock:
|
||||
if key not in self._local_map:
|
||||
raise KeyError(key)
|
||||
self._local_map.pop(key)
|
||||
self._maybe_replicate_to_rank0((_OpKind.DELETE, self._pp_rank, key))
|
||||
|
||||
def close(self) -> None:
|
||||
if self._pp_rank > 0:
|
||||
self._replication_queue.put(None)
|
||||
self._zmq_ctx.term() # This blocks until all zmq socket closed.
|
||||
self._worker_thread.join()
|
||||
|
||||
def _worker_loop(self) -> None:
|
||||
try:
|
||||
if self._pp_rank == 0:
|
||||
self._recv_loop()
|
||||
else:
|
||||
self._send_loop()
|
||||
except zmq.error.ContextTerminated:
|
||||
pass # Raised by close().
|
||||
except Exception as e:
|
||||
logger.critical(
|
||||
"PPConsensusStore background thread crashed: %s",
|
||||
e,
|
||||
)
|
||||
for handler in logger.handlers:
|
||||
handler.flush()
|
||||
os._exit(1)
|
||||
finally:
|
||||
self._socket.close(linger=0)
|
||||
|
||||
def _recv_once(self) -> None:
|
||||
op = pickle.loads(self._socket.recv())
|
||||
if op[0] == _OpKind.PUT:
|
||||
_, rank, key, value = op
|
||||
logger.debug("recv put rank=%s %s = %s", rank, key, value)
|
||||
with self._cache_lock:
|
||||
self._peer_map.setdefault(rank, {})[key] = value
|
||||
elif op[0] == _OpKind.DELETE:
|
||||
_, rank, key = op
|
||||
logger.debug("recv delete rank=%s %s", rank, key)
|
||||
with self._cache_lock:
|
||||
self._peer_map.get(rank, {}).pop(key, None)
|
||||
|
||||
def _recv_loop(self) -> None:
|
||||
assert self._pp_rank == 0
|
||||
while True:
|
||||
self._recv_once()
|
||||
|
||||
def _send_loop(self) -> None:
|
||||
assert self._pp_rank > 0
|
||||
while True:
|
||||
op = self._replication_queue.get()
|
||||
if op is None: # A signal for shutdown.
|
||||
return
|
||||
logger.debug("send %s", op)
|
||||
self._socket.send(pickle.dumps(op))
|
||||
|
||||
def collect(self, key: Any) -> List[Any]:
|
||||
assert self._pp_rank == 0, "collect can only be used on PP rank 0"
|
||||
with self._cache_lock:
|
||||
values: List[Any] = [self._local_map.get(key)]
|
||||
for rank in range(1, self._pp_size):
|
||||
values.append(self._peer_map.get(rank, {}).get(key))
|
||||
return values
|
||||
@@ -60,7 +60,7 @@ from sglang.srt.disaggregation.utils import (
|
||||
is_mla_backend,
|
||||
is_unadmitted_reject,
|
||||
poll_and_all_reduce_attn_cp_tp_group,
|
||||
poll_and_all_reduce_pp,
|
||||
poll_and_all_reduce_prefill_pp,
|
||||
prepare_abort,
|
||||
setup_state_kv_args,
|
||||
)
|
||||
@@ -194,6 +194,7 @@ class PrefillBootstrapQueue:
|
||||
self.scheduler.tp_worker.model_runner.effective_logical_max_total_num_tokens
|
||||
)
|
||||
self.transfer_backend = transfer_backend
|
||||
self.pp_poll_sync_work_list: List = []
|
||||
if envs.SGLANG_DISAGG_STAGING_BUFFER.get():
|
||||
if self.is_mla_backend:
|
||||
raise RuntimeError(
|
||||
@@ -458,47 +459,25 @@ class PrefillBootstrapQueue:
|
||||
req.sampling_params.max_new_tokens = 1
|
||||
|
||||
@scheduler_stage_method(SCHEDULER_STAGE_PROCESS_QUEUE)
|
||||
def pop_bootstrapped(
|
||||
self,
|
||||
return_failed_reqs: bool = False,
|
||||
pp_good_rids: Optional[List[str]] = None,
|
||||
pp_bad_rids: Optional[List[str]] = None,
|
||||
) -> List[Req] | tuple[List[Req], List[Req]]:
|
||||
"""
|
||||
pop the reqs which has finished bootstrapping
|
||||
|
||||
return_failed_reqs: For PP, on rank 0, also return the failed reqs to notify the next rank
|
||||
pp_good_rids: RIDs that PP consensus determined as WaitingForInput.
|
||||
pp_bad_rids: RIDs that PP consensus determined as Failed.
|
||||
"""
|
||||
def pop_bootstrapped(self) -> List[Req]:
|
||||
"""Pop the reqs which have finished bootstrapping."""
|
||||
|
||||
bootstrapped_reqs = []
|
||||
failed_reqs = []
|
||||
indices_to_remove = set()
|
||||
|
||||
if len(self.queue) == 0:
|
||||
if return_failed_reqs is False:
|
||||
return []
|
||||
else:
|
||||
return [], []
|
||||
return []
|
||||
|
||||
if self.pp_size > 1:
|
||||
polls = poll_and_all_reduce_pp(
|
||||
(req.rid for req in self.queue),
|
||||
KVPoll.WaitingForInput,
|
||||
pp_good_rids,
|
||||
pp_bad_rids,
|
||||
polls = poll_and_all_reduce_prefill_pp(
|
||||
[req.disagg_kv_sender for req in self.queue],
|
||||
self.scheduler.attn_cp_cpu_group,
|
||||
self.scheduler.attn_tp_cpu_group,
|
||||
self.scheduler.pp_group,
|
||||
self.pp_rank,
|
||||
self.pp_size,
|
||||
self.pp_poll_sync_work_list,
|
||||
)
|
||||
uncovered = [i for i, poll in enumerate(polls) if poll is None]
|
||||
if uncovered:
|
||||
local_polls = poll_and_all_reduce_attn_cp_tp_group(
|
||||
[self.queue[i].disagg_kv_sender for i in uncovered],
|
||||
self.scheduler.attn_cp_cpu_group,
|
||||
self.scheduler.attn_tp_cpu_group,
|
||||
)
|
||||
for i, local_poll in zip(uncovered, local_polls):
|
||||
if local_poll == KVPoll.Failed:
|
||||
polls[i] = KVPoll.Failed
|
||||
else:
|
||||
polls = poll_and_all_reduce_attn_cp_tp_group(
|
||||
[req.disagg_kv_sender for req in self.queue],
|
||||
@@ -507,13 +486,9 @@ class PrefillBootstrapQueue:
|
||||
)
|
||||
|
||||
for i, (req, poll) in enumerate(zip(self.queue, polls)):
|
||||
if poll is None:
|
||||
continue
|
||||
|
||||
if poll == KVPoll.Failed:
|
||||
self.scheduler.handle_bootstrap_failure(req)
|
||||
indices_to_remove.add(i)
|
||||
failed_reqs.append(req)
|
||||
elif poll == KVPoll.Bootstrapping:
|
||||
if (
|
||||
(
|
||||
@@ -558,10 +533,7 @@ class PrefillBootstrapQueue:
|
||||
entry for i, entry in enumerate(self.queue) if i not in indices_to_remove
|
||||
]
|
||||
|
||||
if return_failed_reqs is False:
|
||||
return bootstrapped_reqs
|
||||
else:
|
||||
return bootstrapped_reqs, failed_reqs
|
||||
return bootstrapped_reqs
|
||||
|
||||
def release_memory_occupation(self):
|
||||
self.queue.clear()
|
||||
|
||||
@@ -21,6 +21,8 @@ import torch.distributed as dist
|
||||
|
||||
from sglang.srt.configs.model_config import get_dsa_mtp_topk_width, is_deepseek_dsa
|
||||
from sglang.srt.disaggregation.base import KVPoll
|
||||
from sglang.srt.distributed.communication_tags import P2PTag
|
||||
from sglang.srt.distributed.parallel_state import GroupCoordinator
|
||||
from sglang.srt.environ import envs
|
||||
from sglang.srt.runtime_context import (
|
||||
get_disagg,
|
||||
@@ -49,6 +51,44 @@ if is_npu():
|
||||
FAKE_BOOTSTRAP_HOST = "2.2.2.2"
|
||||
|
||||
|
||||
def pp_sync_polls(
|
||||
polls: Optional[List[int]],
|
||||
pp_group: GroupCoordinator,
|
||||
pp_rank: int,
|
||||
pp_size: int,
|
||||
sync_work_list: List,
|
||||
) -> List[int]:
|
||||
"""Synchronize poll states from PP0 through the PP pipeline.
|
||||
|
||||
PP0 provides the data. Each later PP rank receives it from the previous
|
||||
rank and asynchronously forwards it to the next rank.
|
||||
"""
|
||||
if pp_size <= 1:
|
||||
assert polls is not None
|
||||
return polls
|
||||
|
||||
for p2p_work in sync_work_list:
|
||||
p2p_work.work.wait()
|
||||
sync_work_list.clear()
|
||||
|
||||
data = polls
|
||||
if pp_rank > 0:
|
||||
data = pp_group.recv_object(
|
||||
src=pp_rank - 1,
|
||||
tag=P2PTag.DISAGG_BOOTSTRAP_PP_SYNC,
|
||||
)
|
||||
if pp_rank + 1 < pp_size:
|
||||
sync_work_list.extend(
|
||||
pp_group.send_object(
|
||||
data,
|
||||
dst=pp_rank + 1,
|
||||
async_send=True,
|
||||
tag=P2PTag.DISAGG_BOOTSTRAP_PP_SYNC,
|
||||
)
|
||||
)
|
||||
return data
|
||||
|
||||
|
||||
def poll_and_all_reduce_pp(
|
||||
rids: Iterable[str],
|
||||
ready_poll: int,
|
||||
@@ -67,6 +107,25 @@ def poll_and_all_reduce_pp(
|
||||
]
|
||||
|
||||
|
||||
def poll_and_all_reduce_prefill_pp(
|
||||
pollers: List[CommonKVSender],
|
||||
attn_cp_cpu_group: dist.ProcessGroup,
|
||||
attn_tp_cpu_group: dist.ProcessGroup,
|
||||
pp_group: GroupCoordinator,
|
||||
pp_rank: int,
|
||||
pp_size: int,
|
||||
pp_poll_sync_work_list: List[torch.distributed.Work],
|
||||
) -> List[int]:
|
||||
polls: List[int] = []
|
||||
if pp_rank == 0:
|
||||
polls = _poll_with_failure_injection_pp(pollers)
|
||||
polls = _all_reduce_polls(polls, attn_tp_cpu_group)
|
||||
polls = _all_reduce_polls(polls, attn_cp_cpu_group)
|
||||
# Propagate PP0's data to other ranks
|
||||
polls = pp_sync_polls(polls, pp_group, pp_rank, pp_size, pp_poll_sync_work_list)
|
||||
return polls
|
||||
|
||||
|
||||
def get_dsa_seed_metadata_dim(hf_config) -> int:
|
||||
"""Return the model-defined PD seed width, independent of local spec mode."""
|
||||
if not getattr(hf_config, "index_share_for_mtp_iteration", False):
|
||||
@@ -163,6 +222,17 @@ def _poll_with_failure_injection(pollers) -> List[int]:
|
||||
return [int(poller.poll()) for poller in pollers]
|
||||
|
||||
|
||||
def _poll_with_failure_injection_pp(pollers) -> List[int]:
|
||||
if (failure_prob := envs.SGLANG_TEST_DISAGG_FAILURE_PROB.get()) > 0:
|
||||
return [
|
||||
int(KVPoll.Failed)
|
||||
if random.random() < failure_prob
|
||||
else int(poller.poll_pp_consensus())
|
||||
for poller in pollers
|
||||
]
|
||||
return [int(poller.poll_pp_consensus()) for poller in pollers]
|
||||
|
||||
|
||||
def _is_fake_transfer(req: Req) -> bool:
|
||||
return req.bootstrap_host == FAKE_BOOTSTRAP_HOST or (
|
||||
req.bootstrap_host is None
|
||||
|
||||
@@ -13,3 +13,4 @@ class P2PTag(IntEnum):
|
||||
DEFAULT = 0
|
||||
HICACHE_PP_SYNC = int.from_bytes(b"PpHi", byteorder="big")
|
||||
GRAMMAR_PP_SYNC = int.from_bytes(b"PpGr", byteorder="big")
|
||||
DISAGG_BOOTSTRAP_PP_SYNC = int.from_bytes(b"PpBs", byteorder="big")
|
||||
|
||||
@@ -223,18 +223,15 @@ class SchedulerPPMixin:
|
||||
====================================================================
|
||||
Stage P
|
||||
recv ith req from previous stage
|
||||
recv ith bootstrap req from previous stage
|
||||
pop bootstrapped reqs (PPConsensusStore + pp_sync polls)
|
||||
recv ith transferred req from previous stage
|
||||
recv ith proxy from previous stage
|
||||
run ith batch
|
||||
recv prev (i+1) % mb_size th consensus bootstrapped req from previous stage
|
||||
local consensus on bootstrapped req
|
||||
recv prev (i+1) % mb_size th release req from previous stage
|
||||
local consensus on release req
|
||||
recv prev (i+1) % mb_size th outputs
|
||||
process batch result of prev (i+1)% mb_size th batch (can be run in parallel with the curr batch GPU computation)
|
||||
send ith req to next stage
|
||||
send ith bootstrap req to next stage
|
||||
send ith transferred req to next stage
|
||||
send ith proxy to next stage
|
||||
send current stage's outputs to next stage (can be stashed and delayed to send later)
|
||||
@@ -242,23 +239,17 @@ class SchedulerPPMixin:
|
||||
the above order can be optimized and reordered to minimize communication-related CPU stall and overhead bubbles.
|
||||
====================================================================
|
||||
|
||||
There are two additional elements compared to the regular schedule:
|
||||
|
||||
Bootstrap Requests + Release Requests:
|
||||
- Both can have local failure and need to be consensus on. PP needs to guarantee eventual consistency of local failure and flush malfunc requests out as soft error.
|
||||
Release Requests still use the two-pass PP consensus. Bootstrap uses
|
||||
PPConsensusStore instead of the legacy two-round RID sync.
|
||||
|
||||
"""
|
||||
self.init_pp_loop_state()
|
||||
|
||||
# PD additional state initialization
|
||||
bmbs = [None] * self.pp_loop_size
|
||||
tmbs = [None] * self.pp_loop_size
|
||||
consensus_bootstrapped_rids: Optional[List[str]] = None
|
||||
transferred_rids: List[str] = []
|
||||
release_rids: Optional[List[str]] = None
|
||||
send_bootstrapped_work = []
|
||||
send_transfer_work = []
|
||||
send_consensus_bootstrapped_work = []
|
||||
send_release_work = []
|
||||
|
||||
while True:
|
||||
@@ -273,7 +264,6 @@ class SchedulerPPMixin:
|
||||
|
||||
next_pp_outputs = None
|
||||
next_release_rids = None
|
||||
next_consensus_bootstrapped_rids = None
|
||||
d2h_event = None
|
||||
next_batch_result = None
|
||||
|
||||
@@ -282,9 +272,7 @@ class SchedulerPPMixin:
|
||||
if not self.pp_group.is_last_rank:
|
||||
self._pp_commit_comm_work(self.send_req_work)
|
||||
|
||||
bootstrapped_rids = self._pp_pd_get_bootstrapped_ids()
|
||||
bmbs[mb_id] = bootstrapped_rids
|
||||
self._pp_commit_comm_work(send_bootstrapped_work)
|
||||
self.process_bootstrapped_queue()
|
||||
|
||||
transferred_rids = self._pp_pd_get_prefill_transferred_ids()
|
||||
self._pp_commit_comm_work(send_transfer_work)
|
||||
@@ -332,28 +320,12 @@ class SchedulerPPMixin:
|
||||
next_mb_id,
|
||||
)
|
||||
)
|
||||
send_consensus_bootstrapped_work, consensus_bootstrapped_rids = (
|
||||
self._pp_pd_send_consensus_bootstrapped_ids(
|
||||
bmbs,
|
||||
next_first_rank_mb_id,
|
||||
consensus_bootstrapped_rids,
|
||||
bootstrapped_rids,
|
||||
)
|
||||
)
|
||||
send_release_work, release_rids = (
|
||||
self._pp_pd_send_consensus_release_ids(
|
||||
tmbs, next_first_rank_mb_id, release_rids, transferred_rids
|
||||
)
|
||||
)
|
||||
|
||||
if bmbs[next_mb_id] is not None:
|
||||
next_consensus_bootstrapped_rids = (
|
||||
self._pp_recv_pyobj_from_prev_stage()
|
||||
)
|
||||
next_consensus_bootstrapped_rids = self.process_bootstrapped_queue(
|
||||
next_consensus_bootstrapped_rids
|
||||
)
|
||||
self._pp_commit_comm_work(send_consensus_bootstrapped_work)
|
||||
if tmbs[next_mb_id] is not None:
|
||||
next_release_rids = self._pp_recv_pyobj_from_prev_stage()
|
||||
self._pp_commit_comm_work(send_release_work)
|
||||
@@ -372,9 +344,6 @@ class SchedulerPPMixin:
|
||||
self.send_req_work = self._pp_send_pyobj_to_next_stage(
|
||||
recv_reqs, async_send=True
|
||||
)
|
||||
send_bootstrapped_work = self._pp_send_pyobj_to_next_stage(
|
||||
bootstrapped_rids, async_send=True
|
||||
)
|
||||
send_transfer_work = self._pp_send_pyobj_to_next_stage(
|
||||
transferred_rids, async_send=True
|
||||
)
|
||||
@@ -383,7 +352,6 @@ class SchedulerPPMixin:
|
||||
|
||||
self.pp_outputs = next_pp_outputs
|
||||
release_rids = next_release_rids
|
||||
consensus_bootstrapped_rids = next_consensus_bootstrapped_rids
|
||||
|
||||
self.running_batch.batch_is_full = False
|
||||
|
||||
@@ -636,67 +604,9 @@ class SchedulerPPMixin:
|
||||
str, deque[Tuple[Dict[str, torch.Tensor], Optional[torch.Event]]]
|
||||
] = defaultdict(deque)
|
||||
|
||||
def process_bootstrapped_queue(
|
||||
self: Scheduler, bootstrapped_rids: Optional[List[str]]
|
||||
):
|
||||
# finished consensus bootstrapped reqs and prepare the waiting queue
|
||||
if bootstrapped_rids is not None:
|
||||
(
|
||||
good_consensus_bootstrapped_rids,
|
||||
bad_consensus_bootstrapped_rids,
|
||||
) = bootstrapped_rids
|
||||
good_reqs, failed_reqs = (
|
||||
self.disagg_prefill_bootstrap_queue.pop_bootstrapped(
|
||||
return_failed_reqs=True,
|
||||
pp_good_rids=good_consensus_bootstrapped_rids,
|
||||
pp_bad_rids=bad_consensus_bootstrapped_rids,
|
||||
)
|
||||
)
|
||||
self.waiting_queue.extend(good_reqs)
|
||||
return [[req.rid for req in good_reqs], [req.rid for req in failed_reqs]]
|
||||
return None
|
||||
|
||||
def _pp_pd_get_bootstrapped_ids(self: Scheduler):
|
||||
# communicate pre-consensus bootstrapp reqs
|
||||
if self.pp_group.is_first_rank:
|
||||
# First rank, pop the bootstrap reqs from the bootstrap queue
|
||||
good_bootstrapped_rids, bad_bootstrapped_rids = self.get_rids(
|
||||
self.disagg_prefill_bootstrap_queue.queue,
|
||||
True,
|
||||
[KVPoll.WaitingForInput],
|
||||
[KVPoll.Failed],
|
||||
)
|
||||
else:
|
||||
# Other ranks, receive the bootstrap reqs info from the previous rank and ensure the consensus
|
||||
prev_bootstrapped_rids = self._pp_recv_pyobj_from_prev_stage()
|
||||
prev_good_bootstrapped_rids, prev_bad_bootstrapped_rids = (
|
||||
prev_bootstrapped_rids
|
||||
)
|
||||
curr_good_bootstrapped_rids, curr_bad_bootstrapped_rids = self.get_rids(
|
||||
self.disagg_prefill_bootstrap_queue.queue,
|
||||
True,
|
||||
[KVPoll.WaitingForInput],
|
||||
[KVPoll.Failed],
|
||||
)
|
||||
good_bootstrapped_rids = list(
|
||||
set(prev_good_bootstrapped_rids) & set(curr_good_bootstrapped_rids)
|
||||
)
|
||||
bad_bootstrapped_rids = list(
|
||||
set(prev_bad_bootstrapped_rids) | set(curr_bad_bootstrapped_rids)
|
||||
)
|
||||
# Route locally-aborted reqs through the bad-union consensus so every PP
|
||||
# rank flushes them in the same consensus round, regardless of when the
|
||||
# AbortReq reaches each rank and regardless of whether
|
||||
# disagg_kv_sender.abort() drives the poll to Failed (it is optional).
|
||||
aborted_rids = {
|
||||
req.rid
|
||||
for req in self.disagg_prefill_bootstrap_queue.queue
|
||||
if isinstance(req.finished_reason, FINISH_ABORT)
|
||||
}
|
||||
good_bootstrapped_rids, bad_bootstrapped_rids = self._route_aborts_to_bad(
|
||||
good_bootstrapped_rids, bad_bootstrapped_rids, aborted_rids
|
||||
)
|
||||
return [good_bootstrapped_rids, bad_bootstrapped_rids]
|
||||
def process_bootstrapped_queue(self: Scheduler):
|
||||
reqs = self.disagg_prefill_bootstrap_queue.pop_bootstrapped()
|
||||
self.waiting_queue.extend(reqs)
|
||||
|
||||
def _pp_pd_get_prefill_transferred_ids(self: Scheduler):
|
||||
# get the current stage transfer success
|
||||
|
||||
@@ -0,0 +1,192 @@
|
||||
"""Unit tests for srt/disaggregation/pp_consensus_store."""
|
||||
|
||||
import time
|
||||
import unittest
|
||||
from collections.abc import Callable
|
||||
|
||||
from sglang.srt.disaggregation.pp_consensus_store import PPConsensusStore
|
||||
from sglang.test.ci.ci_register import register_cpu_ci
|
||||
from sglang.test.test_utils import CustomTestCase
|
||||
|
||||
register_cpu_ci(est_time=5, suite="base-a-test-cpu")
|
||||
|
||||
|
||||
class _MockPPGroup:
|
||||
def __init__(self):
|
||||
self._obj = None
|
||||
|
||||
def broadcast_object(self, obj, src=0):
|
||||
if obj is not None:
|
||||
self._obj = obj
|
||||
return self._obj
|
||||
|
||||
|
||||
class TestPPConsensusStore(CustomTestCase):
|
||||
def _wait_until(self, condition: Callable[[], bool]) -> None:
|
||||
now = time.time()
|
||||
start = now
|
||||
deadline = now + 1
|
||||
delay_time = 0.01
|
||||
while now < deadline and not condition():
|
||||
time.sleep(delay_time)
|
||||
now = time.time()
|
||||
delay_time *= 2
|
||||
self.assertTrue(
|
||||
now < deadline, f"Condition not met after {now - start} seconds"
|
||||
)
|
||||
|
||||
def test_setitem(self):
|
||||
group = _MockPPGroup()
|
||||
store0 = PPConsensusStore(1, 0, group)
|
||||
store0["key"] = 10
|
||||
self.assertEqual(store0["key"], 10)
|
||||
store0.close()
|
||||
|
||||
def test_setitem_replicate_to_rank0(self):
|
||||
group = _MockPPGroup()
|
||||
store0 = PPConsensusStore(2, 0, group)
|
||||
store1 = PPConsensusStore(2, 1, group)
|
||||
store0["key"] = 10
|
||||
store1["key"] = 11
|
||||
self._wait_until(lambda: store0.collect("key") == [10, 11])
|
||||
store0.close()
|
||||
store1.close()
|
||||
|
||||
def test_contains(self):
|
||||
group = _MockPPGroup()
|
||||
store0 = PPConsensusStore(1, 0, group)
|
||||
store0["key"] = 10
|
||||
self.assertTrue("key" in store0)
|
||||
self.assertFalse("no_such_key" in store0)
|
||||
store0.close()
|
||||
|
||||
def test_get(self):
|
||||
group = _MockPPGroup()
|
||||
store0 = PPConsensusStore(1, 0, group)
|
||||
store0["key"] = 10
|
||||
self.assertEqual(store0.get("key"), 10)
|
||||
store0.close()
|
||||
|
||||
def test_get_not_exist(self):
|
||||
group = _MockPPGroup()
|
||||
store0 = PPConsensusStore(1, 0, group)
|
||||
self.assertIsNone(store0.get("no_such_key"))
|
||||
self.assertEqual(store0.get("no_such_key", "default"), "default")
|
||||
store0.close()
|
||||
|
||||
def test_get_default(self):
|
||||
group = _MockPPGroup()
|
||||
store0 = PPConsensusStore(1, 0, group)
|
||||
self.assertEqual(store0.get("no_such_key", "default"), "default")
|
||||
store0.close()
|
||||
|
||||
def test_getitem_not_exist(self):
|
||||
group = _MockPPGroup()
|
||||
store0 = PPConsensusStore(1, 0, group)
|
||||
with self.assertRaises(KeyError):
|
||||
store0["no_such_key"]
|
||||
store0.close()
|
||||
|
||||
def test_delitem(self):
|
||||
group = _MockPPGroup()
|
||||
store0 = PPConsensusStore(1, 0, group)
|
||||
store0["key"] = 10
|
||||
del store0["key"]
|
||||
self.assertFalse("key" in store0)
|
||||
store0.close()
|
||||
|
||||
def test_delitem_not_exist(self):
|
||||
group = _MockPPGroup()
|
||||
store0 = PPConsensusStore(1, 0, group)
|
||||
with self.assertRaises(KeyError):
|
||||
del store0["no_such_key"]
|
||||
store0.close()
|
||||
|
||||
def test_delitem_replicate_to_rank0(self):
|
||||
group = _MockPPGroup()
|
||||
store0 = PPConsensusStore(2, 0, group)
|
||||
store1 = PPConsensusStore(2, 1, group)
|
||||
store0["key"] = 10
|
||||
store1["key"] = 11
|
||||
self._wait_until(lambda: store0.collect("key") == [10, 11])
|
||||
del store1["key"]
|
||||
self._wait_until(lambda: store0.collect("key") == [10, None])
|
||||
store0.close()
|
||||
store1.close()
|
||||
|
||||
def test_pop(self):
|
||||
group = _MockPPGroup()
|
||||
store0 = PPConsensusStore(1, 0, group)
|
||||
store0["key"] = 10
|
||||
self.assertEqual(store0.pop("key"), 10)
|
||||
self.assertFalse("key" in store0)
|
||||
store0.close()
|
||||
|
||||
def test_pop_not_exist(self):
|
||||
group = _MockPPGroup()
|
||||
store0 = PPConsensusStore(1, 0, group)
|
||||
self.assertIsNone(store0.pop("no_such_key"))
|
||||
store0.close()
|
||||
|
||||
def test_pop_default(self):
|
||||
group = _MockPPGroup()
|
||||
store0 = PPConsensusStore(1, 0, group)
|
||||
self.assertEqual(store0.pop("no_such_key", "default"), "default")
|
||||
store0.close()
|
||||
|
||||
def test_pop_replicate_to_rank0(self):
|
||||
group = _MockPPGroup()
|
||||
store0 = PPConsensusStore(2, 0, group)
|
||||
store1 = PPConsensusStore(2, 1, group)
|
||||
store0["key"] = 10
|
||||
store1["key"] = 11
|
||||
self._wait_until(lambda: store0.collect("key") == [10, 11])
|
||||
self.assertEqual(store1.pop("key"), 11)
|
||||
self.assertFalse("key" in store1)
|
||||
self._wait_until(lambda: store0.collect("key") == [10, None])
|
||||
store0.close()
|
||||
store1.close()
|
||||
|
||||
def test_collect(self):
|
||||
group = _MockPPGroup()
|
||||
store0 = PPConsensusStore(3, 0, group)
|
||||
store1 = PPConsensusStore(3, 1, group)
|
||||
store2 = PPConsensusStore(3, 2, group)
|
||||
store0["key"] = 10
|
||||
store1["key"] = 11
|
||||
store2["key"] = 12
|
||||
self._wait_until(lambda: store0.collect("key") == [10, 11, 12])
|
||||
store0.close()
|
||||
store1.close()
|
||||
store2.close()
|
||||
|
||||
def test_collect_missing(self):
|
||||
group = _MockPPGroup()
|
||||
store0 = PPConsensusStore(3, 0, group)
|
||||
store1 = PPConsensusStore(3, 1, group)
|
||||
store2 = PPConsensusStore(3, 2, group)
|
||||
store0["key"] = 10
|
||||
store2["key"] = 12
|
||||
self._wait_until(lambda: store0.collect("key") == [10, None, 12])
|
||||
store0.close()
|
||||
store1.close()
|
||||
store2.close()
|
||||
|
||||
def test_collect_not_exist(self):
|
||||
group = _MockPPGroup()
|
||||
store0 = PPConsensusStore(1, 0, group)
|
||||
self.assertEqual(store0.collect("no_such_key"), [None])
|
||||
store0.close()
|
||||
|
||||
def test_collect_requires_pp0(self):
|
||||
group = _MockPPGroup()
|
||||
store0 = PPConsensusStore(2, 0, group)
|
||||
store1 = PPConsensusStore(2, 1, group)
|
||||
with self.assertRaises(AssertionError):
|
||||
store1.collect("key")
|
||||
store0.close()
|
||||
store1.close()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -552,7 +552,6 @@ class TestSchedulerIdleStepCounters(CustomTestCase):
|
||||
scheduler._pp_commit_send_output_work_and_preprocess_output_tensors = Mock(
|
||||
return_value=(None, GenerationBatchResult(), Mock())
|
||||
)
|
||||
scheduler._pp_pd_get_bootstrapped_ids = Mock(return_value=None)
|
||||
scheduler._pp_pd_get_prefill_transferred_ids = Mock(return_value=None)
|
||||
scheduler._pp_pd_get_retract_ids = Mock(return_value=None)
|
||||
scheduler._pp_pd_get_prealloc_ids = Mock(return_value=None)
|
||||
|
||||
@@ -124,7 +124,8 @@ class TestSchedulerHiCacheEvents(unittest.TestCase):
|
||||
s.running_mbs = [self.running_batch]
|
||||
s.last_mbs = [None]
|
||||
s.ingest_requests = Mock(return_value=[])
|
||||
s._pp_pd_get_bootstrapped_ids = Mock(return_value=[])
|
||||
s.waiting_queue = []
|
||||
s.disagg_prefill_bootstrap_queue = SimpleNamespace(pop_bootstrapped=lambda: [])
|
||||
s._pp_pd_get_prefill_transferred_ids = Mock(return_value=[])
|
||||
s._pp_commit_comm_work = Mock()
|
||||
s.get_new_batch_prefill = self.calls.admit
|
||||
|
||||
Reference in New Issue
Block a user