[Refactor] disaggregation: Share the flag all-reduce and host-destination predicate in the decode transfer queue (#45641)

This commit is contained in:
Shangming Cai
2026-10-22 13:42:55 +08:00
committed by GitHub
parent d9ce511646
commit a6a6beffac
4 changed files with 65 additions and 59 deletions
+31 -41
View File
@@ -57,6 +57,7 @@ from sglang.srt.disaggregation.utils import (
ReqToMetadataIdxAllocator,
TransferBackend,
_is_fake_transfer,
all_reduce_flags,
build_kv_layer_ids,
build_staging_slot_metadata,
get_dsa_tail_state_indices,
@@ -2747,12 +2748,15 @@ class DecodeTransferQueue(DecodeHiCacheTransferMixin):
return
release_kv_cache(decode_req.req, self.tree_cache, checkpoint=False)
def _is_host_destination(self, decode_req: DecodeRequest) -> bool:
return self.enable_host_receive and decode_req.host_staged
def _record_transfer_failure(self, failure_reason: str) -> None:
if self.scheduler.metrics_reporter.enable_metrics:
self.scheduler.metrics_collector.increment_transfer_failed_reqs()
def _abort_failed_transfer(self, decode_req: DecodeRequest) -> None:
if self.enable_host_receive and decode_req.host_staged:
if self._is_host_destination(decode_req):
decode_req.kv_receiver.abort()
def pop_transferred(self, rids_to_check: Optional[List[str]] = None) -> List[Req]:
@@ -2816,9 +2820,7 @@ class DecodeTransferQueue(DecodeHiCacheTransferMixin):
[decode_req.req],
decode_req.req.return_logprob,
)
requires_host_drain = (
self.enable_host_receive and decode_req.host_staged
)
requires_host_drain = self._is_host_destination(decode_req)
deferrable = (
self.enable_deferred_kv_release
and decode_req.kv_receiver.kv_mgr.enable_deferred_decode_kv_release
@@ -2843,7 +2845,7 @@ class DecodeTransferQueue(DecodeHiCacheTransferMixin):
self._record_transfer_failure(failure_reason)
continue
elif poll == KVPoll.Success:
if self.enable_host_receive and decode_req.host_staged:
if self._is_host_destination(decode_req):
if isinstance(decode_req.req.finished_reason, FINISH_ABORT):
self.scheduler.output_streamer.stream_output(
[decode_req.req], decode_req.req.return_logprob
@@ -2883,8 +2885,10 @@ class DecodeTransferQueue(DecodeHiCacheTransferMixin):
# Notification state is rank-local. If any rank must hold a failed
# transfer, all must retain its capacity so admission stays identical.
held = self._all_reduce_release_flags(
[hold for _, _, hold in failed], torch.distributed.ReduceOp.MAX
held = all_reduce_flags(
flags=[hold for _, _, hold in failed],
group=self.gloo_group,
op=torch.distributed.ReduceOp.MAX,
)
for (i, decode_req, _), hold in zip(failed, held):
if hold:
@@ -2926,7 +2930,7 @@ class DecodeTransferQueue(DecodeHiCacheTransferMixin):
# A rank that only holds for a peer has no local acknowledgements to await.
required_acks = (
0
if (self.enable_host_receive and decode_req.host_staged)
if self._is_host_destination(decode_req)
or not decode_req.kv_receiver.abort_notified
else len(decode_req.kv_receiver.bootstrap_infos)
)
@@ -2948,7 +2952,7 @@ class DecodeTransferQueue(DecodeHiCacheTransferMixin):
self._release_request(decode_req)
self.metadata_buffers.bootstrap_room[idx] = 0
self.req_to_metadata_buffer_idx_allocator.free(idx)
if not (self.enable_host_receive and decode_req.host_staged):
if not self._is_host_destination(decode_req):
decode_req.kv_receiver.kv_mgr.clear_deferred_abort_state(room)
decode_req.kv_receiver.clear()
decode_req.kv_receiver = None
@@ -2968,45 +2972,31 @@ class DecodeTransferQueue(DecodeHiCacheTransferMixin):
outcome=outcome,
)
def _all_reduce_release_flags(
self, flags: List[bool], op: torch.distributed.ReduceOp
) -> List[bool]:
# Failures come from reduced polls and deferred entries from agreed holds,
# so every rank supplies the same entries in the same order, even if empty.
if not flags or torch.distributed.get_world_size(self.gloo_group) == 1:
return flags
reduced = torch.tensor(flags, dtype=torch.int32, device="cpu")
torch.distributed.all_reduce(reduced, op=op, group=self.gloo_group)
return [bool(flag) for flag in reduced.tolist()]
def resolve_deferred_releases(self) -> None:
"""Release drained transfers; device destinations may also time out."""
if not self._deferred_releases:
return
now = time.monotonic()
drained = [
decode_req.kv_receiver.is_abort_release_safe()
if self.enable_host_receive and decode_req.host_staged
else decode_req.kv_receiver.kv_mgr.is_abort_release_safe(
decode_req.req.bootstrap_room, required_acks
)
for decode_req, _, _, _, required_acks in self._deferred_releases
]
drained = []
ready = []
for decode_req, _, deadline, _, required_acks in self._deferred_releases:
if self._is_host_destination(decode_req):
# A timeout cannot prove that a remote write has stopped.
is_drained = decode_req.kv_receiver.is_abort_release_safe()
is_ready = is_drained
else:
is_drained = decode_req.kv_receiver.kv_mgr.is_abort_release_safe(
decode_req.req.bootstrap_room, required_acks
)
is_ready = is_drained or now >= deadline
drained.append(is_drained)
ready.append(is_ready)
# Acks and deadlines are rank-local. Releasing only on some ranks would
# diverge admission and the sizes of subsequent polling collectives.
# Host pages still require a drain: a timeout cannot rule out remote writes.
ready = self._all_reduce_release_flags(
[
is_drained
or (
not (self.enable_host_receive and decode_req.host_staged)
and now >= deadline
)
for (decode_req, _, deadline, _, _), is_drained in zip(
self._deferred_releases, drained
)
],
torch.distributed.ReduceOp.MIN,
# Entries are appended from agreed holds, so every rank reduces the same
# list in the same order.
ready = all_reduce_flags(
flags=ready, group=self.gloo_group, op=torch.distributed.ReduceOp.MIN
)
still_held = []
to_release = []
+14 -6
View File
@@ -259,14 +259,22 @@ def _apply_metadata_gate(polls, decode_reqs, metadata_buffers) -> None:
polls[i] = int(KVPoll.Transferring)
def all_reduce_flags(
flags: List[int], group: dist.ProcessGroup, op: dist.ReduceOp
) -> List[int]:
"""Reduce one small int per entry across `group`; every rank must pass the
same number of entries in the same order. Empty or single-rank: unchanged."""
if not flags or dist.get_world_size(group) == 1:
return flags
tensor_to_reduce = torch.tensor(flags, dtype=torch.int32, device="cpu")
dist.all_reduce(tensor_to_reduce, op=op, group=group)
return tensor_to_reduce.tolist()
def _all_reduce_polls(polls: List[int], group: dist.ProcessGroup) -> List[int]:
"""MIN-reduce poll states so no rank commits ahead of its peers."""
if dist.get_world_size(group) == 1:
return polls
tensor_to_reduce = torch.tensor(polls, dtype=torch.uint8, device="cpu")
dist.all_reduce(tensor_to_reduce, op=dist.ReduceOp.MIN, group=group)
return tensor_to_reduce.tolist()
return all_reduce_flags(flags=polls, group=group, op=dist.ReduceOp.MIN)
def poll_and_all_reduce(