[Refactor] disaggregation: Share the flag all-reduce and host-destination predicate in the decode transfer queue (#45641)
This commit is contained in:
@@ -57,6 +57,7 @@ from sglang.srt.disaggregation.utils import (
|
||||
ReqToMetadataIdxAllocator,
|
||||
TransferBackend,
|
||||
_is_fake_transfer,
|
||||
all_reduce_flags,
|
||||
build_kv_layer_ids,
|
||||
build_staging_slot_metadata,
|
||||
get_dsa_tail_state_indices,
|
||||
@@ -2747,12 +2748,15 @@ class DecodeTransferQueue(DecodeHiCacheTransferMixin):
|
||||
return
|
||||
release_kv_cache(decode_req.req, self.tree_cache, checkpoint=False)
|
||||
|
||||
def _is_host_destination(self, decode_req: DecodeRequest) -> bool:
|
||||
return self.enable_host_receive and decode_req.host_staged
|
||||
|
||||
def _record_transfer_failure(self, failure_reason: str) -> None:
|
||||
if self.scheduler.metrics_reporter.enable_metrics:
|
||||
self.scheduler.metrics_collector.increment_transfer_failed_reqs()
|
||||
|
||||
def _abort_failed_transfer(self, decode_req: DecodeRequest) -> None:
|
||||
if self.enable_host_receive and decode_req.host_staged:
|
||||
if self._is_host_destination(decode_req):
|
||||
decode_req.kv_receiver.abort()
|
||||
|
||||
def pop_transferred(self, rids_to_check: Optional[List[str]] = None) -> List[Req]:
|
||||
@@ -2816,9 +2820,7 @@ class DecodeTransferQueue(DecodeHiCacheTransferMixin):
|
||||
[decode_req.req],
|
||||
decode_req.req.return_logprob,
|
||||
)
|
||||
requires_host_drain = (
|
||||
self.enable_host_receive and decode_req.host_staged
|
||||
)
|
||||
requires_host_drain = self._is_host_destination(decode_req)
|
||||
deferrable = (
|
||||
self.enable_deferred_kv_release
|
||||
and decode_req.kv_receiver.kv_mgr.enable_deferred_decode_kv_release
|
||||
@@ -2843,7 +2845,7 @@ class DecodeTransferQueue(DecodeHiCacheTransferMixin):
|
||||
self._record_transfer_failure(failure_reason)
|
||||
continue
|
||||
elif poll == KVPoll.Success:
|
||||
if self.enable_host_receive and decode_req.host_staged:
|
||||
if self._is_host_destination(decode_req):
|
||||
if isinstance(decode_req.req.finished_reason, FINISH_ABORT):
|
||||
self.scheduler.output_streamer.stream_output(
|
||||
[decode_req.req], decode_req.req.return_logprob
|
||||
@@ -2883,8 +2885,10 @@ class DecodeTransferQueue(DecodeHiCacheTransferMixin):
|
||||
|
||||
# Notification state is rank-local. If any rank must hold a failed
|
||||
# transfer, all must retain its capacity so admission stays identical.
|
||||
held = self._all_reduce_release_flags(
|
||||
[hold for _, _, hold in failed], torch.distributed.ReduceOp.MAX
|
||||
held = all_reduce_flags(
|
||||
flags=[hold for _, _, hold in failed],
|
||||
group=self.gloo_group,
|
||||
op=torch.distributed.ReduceOp.MAX,
|
||||
)
|
||||
for (i, decode_req, _), hold in zip(failed, held):
|
||||
if hold:
|
||||
@@ -2926,7 +2930,7 @@ class DecodeTransferQueue(DecodeHiCacheTransferMixin):
|
||||
# A rank that only holds for a peer has no local acknowledgements to await.
|
||||
required_acks = (
|
||||
0
|
||||
if (self.enable_host_receive and decode_req.host_staged)
|
||||
if self._is_host_destination(decode_req)
|
||||
or not decode_req.kv_receiver.abort_notified
|
||||
else len(decode_req.kv_receiver.bootstrap_infos)
|
||||
)
|
||||
@@ -2948,7 +2952,7 @@ class DecodeTransferQueue(DecodeHiCacheTransferMixin):
|
||||
self._release_request(decode_req)
|
||||
self.metadata_buffers.bootstrap_room[idx] = 0
|
||||
self.req_to_metadata_buffer_idx_allocator.free(idx)
|
||||
if not (self.enable_host_receive and decode_req.host_staged):
|
||||
if not self._is_host_destination(decode_req):
|
||||
decode_req.kv_receiver.kv_mgr.clear_deferred_abort_state(room)
|
||||
decode_req.kv_receiver.clear()
|
||||
decode_req.kv_receiver = None
|
||||
@@ -2968,45 +2972,31 @@ class DecodeTransferQueue(DecodeHiCacheTransferMixin):
|
||||
outcome=outcome,
|
||||
)
|
||||
|
||||
def _all_reduce_release_flags(
|
||||
self, flags: List[bool], op: torch.distributed.ReduceOp
|
||||
) -> List[bool]:
|
||||
# Failures come from reduced polls and deferred entries from agreed holds,
|
||||
# so every rank supplies the same entries in the same order, even if empty.
|
||||
if not flags or torch.distributed.get_world_size(self.gloo_group) == 1:
|
||||
return flags
|
||||
reduced = torch.tensor(flags, dtype=torch.int32, device="cpu")
|
||||
torch.distributed.all_reduce(reduced, op=op, group=self.gloo_group)
|
||||
return [bool(flag) for flag in reduced.tolist()]
|
||||
|
||||
def resolve_deferred_releases(self) -> None:
|
||||
"""Release drained transfers; device destinations may also time out."""
|
||||
if not self._deferred_releases:
|
||||
return
|
||||
now = time.monotonic()
|
||||
drained = [
|
||||
decode_req.kv_receiver.is_abort_release_safe()
|
||||
if self.enable_host_receive and decode_req.host_staged
|
||||
else decode_req.kv_receiver.kv_mgr.is_abort_release_safe(
|
||||
decode_req.req.bootstrap_room, required_acks
|
||||
)
|
||||
for decode_req, _, _, _, required_acks in self._deferred_releases
|
||||
]
|
||||
drained = []
|
||||
ready = []
|
||||
for decode_req, _, deadline, _, required_acks in self._deferred_releases:
|
||||
if self._is_host_destination(decode_req):
|
||||
# A timeout cannot prove that a remote write has stopped.
|
||||
is_drained = decode_req.kv_receiver.is_abort_release_safe()
|
||||
is_ready = is_drained
|
||||
else:
|
||||
is_drained = decode_req.kv_receiver.kv_mgr.is_abort_release_safe(
|
||||
decode_req.req.bootstrap_room, required_acks
|
||||
)
|
||||
is_ready = is_drained or now >= deadline
|
||||
drained.append(is_drained)
|
||||
ready.append(is_ready)
|
||||
# Acks and deadlines are rank-local. Releasing only on some ranks would
|
||||
# diverge admission and the sizes of subsequent polling collectives.
|
||||
# Host pages still require a drain: a timeout cannot rule out remote writes.
|
||||
ready = self._all_reduce_release_flags(
|
||||
[
|
||||
is_drained
|
||||
or (
|
||||
not (self.enable_host_receive and decode_req.host_staged)
|
||||
and now >= deadline
|
||||
)
|
||||
for (decode_req, _, deadline, _, _), is_drained in zip(
|
||||
self._deferred_releases, drained
|
||||
)
|
||||
],
|
||||
torch.distributed.ReduceOp.MIN,
|
||||
# Entries are appended from agreed holds, so every rank reduces the same
|
||||
# list in the same order.
|
||||
ready = all_reduce_flags(
|
||||
flags=ready, group=self.gloo_group, op=torch.distributed.ReduceOp.MIN
|
||||
)
|
||||
still_held = []
|
||||
to_release = []
|
||||
|
||||
@@ -259,14 +259,22 @@ def _apply_metadata_gate(polls, decode_reqs, metadata_buffers) -> None:
|
||||
polls[i] = int(KVPoll.Transferring)
|
||||
|
||||
|
||||
def all_reduce_flags(
|
||||
flags: List[int], group: dist.ProcessGroup, op: dist.ReduceOp
|
||||
) -> List[int]:
|
||||
"""Reduce one small int per entry across `group`; every rank must pass the
|
||||
same number of entries in the same order. Empty or single-rank: unchanged."""
|
||||
if not flags or dist.get_world_size(group) == 1:
|
||||
return flags
|
||||
|
||||
tensor_to_reduce = torch.tensor(flags, dtype=torch.int32, device="cpu")
|
||||
dist.all_reduce(tensor_to_reduce, op=op, group=group)
|
||||
return tensor_to_reduce.tolist()
|
||||
|
||||
|
||||
def _all_reduce_polls(polls: List[int], group: dist.ProcessGroup) -> List[int]:
|
||||
"""MIN-reduce poll states so no rank commits ahead of its peers."""
|
||||
if dist.get_world_size(group) == 1:
|
||||
return polls
|
||||
|
||||
tensor_to_reduce = torch.tensor(polls, dtype=torch.uint8, device="cpu")
|
||||
dist.all_reduce(tensor_to_reduce, op=dist.ReduceOp.MIN, group=group)
|
||||
return tensor_to_reduce.tolist()
|
||||
return all_reduce_flags(flags=polls, group=group, op=dist.ReduceOp.MIN)
|
||||
|
||||
|
||||
def poll_and_all_reduce(
|
||||
|
||||
Reference in New Issue
Block a user