[Fix] disaggregation: Synchronize deferred decode KV release across TP ranks (#45453)
This commit is contained in:
@@ -2775,6 +2775,7 @@ class DecodeTransferQueue(DecodeHiCacheTransferMixin):
|
||||
# Queue-removed but held for deferred release; excluded from the metadata
|
||||
# teardown below.
|
||||
deferred_indices = set()
|
||||
failed: List[Tuple[int, DecodeRequest, bool]] = []
|
||||
for i, (decode_req, poll) in enumerate(zip(self.queue, polls)):
|
||||
if rids_to_check is not None and decode_req.req.rid not in rids_to_check:
|
||||
continue
|
||||
@@ -2829,22 +2830,15 @@ class DecodeTransferQueue(DecodeHiCacheTransferMixin):
|
||||
# already hold destinations; this path always reaches
|
||||
# _do_release, which clears the tracker.
|
||||
decode_req.kv_receiver.ensure_abort_notified(force_arm=True)
|
||||
if requires_host_drain or (
|
||||
deferrable and decode_req.kv_receiver.abort_notified
|
||||
):
|
||||
# Host pages always await a drain ack. A receiver that could
|
||||
# not notify (metadata never published, so no prefill holds
|
||||
# its destination info) has nothing in flight and releases
|
||||
# immediately below.
|
||||
self._defer_release(decode_req)
|
||||
deferred_indices.add(i)
|
||||
indices_to_remove.add(i)
|
||||
else:
|
||||
# release pre-allocated kv cache, but don't insert into the tree since it's failed
|
||||
self._release_request(decode_req)
|
||||
decode_req.kv_receiver.clear()
|
||||
decode_req.kv_receiver = None
|
||||
indices_to_remove.add(i)
|
||||
failed.append(
|
||||
(
|
||||
i,
|
||||
decode_req,
|
||||
requires_host_drain
|
||||
or (deferrable and decode_req.kv_receiver.abort_notified),
|
||||
)
|
||||
)
|
||||
indices_to_remove.add(i)
|
||||
self._record_transfer_failure(failure_reason)
|
||||
continue
|
||||
elif poll == KVPoll.Success:
|
||||
@@ -2890,6 +2884,20 @@ class DecodeTransferQueue(DecodeHiCacheTransferMixin):
|
||||
else:
|
||||
raise ValueError(f"Unexpected poll case: {poll}")
|
||||
|
||||
# Notification state is rank-local. If any rank must hold a failed
|
||||
# transfer, all must retain its capacity so admission stays identical.
|
||||
held = self._all_reduce_release_flags(
|
||||
[hold for _, _, hold in failed], torch.distributed.ReduceOp.MAX
|
||||
)
|
||||
for (i, decode_req, _), hold in zip(failed, held):
|
||||
if hold:
|
||||
self._defer_release(decode_req)
|
||||
deferred_indices.add(i)
|
||||
else:
|
||||
self._release_request(decode_req)
|
||||
decode_req.kv_receiver.clear()
|
||||
decode_req.kv_receiver = None
|
||||
|
||||
for i in indices_to_remove:
|
||||
if i in deferred_indices:
|
||||
# Held for deferred release; metadata buffer freed at resolve time.
|
||||
@@ -2918,9 +2926,11 @@ class DecodeTransferQueue(DecodeHiCacheTransferMixin):
|
||||
deadline = start_time + self.deferred_kv_release_timeout
|
||||
# Host-capable receivers own their drain tracking. Device destinations
|
||||
# snapshot the number of prefill acknowledgements required by the manager.
|
||||
# A rank that only holds for a peer has no local acknowledgements to await.
|
||||
required_acks = (
|
||||
0
|
||||
if self.enable_host_receive and decode_req.host_staged
|
||||
if (self.enable_host_receive and decode_req.host_staged)
|
||||
or not decode_req.kv_receiver.abort_notified
|
||||
else len(decode_req.kv_receiver.bootstrap_infos)
|
||||
)
|
||||
self._deferred_releases.append(
|
||||
@@ -2961,52 +2971,62 @@ class DecodeTransferQueue(DecodeHiCacheTransferMixin):
|
||||
outcome=outcome,
|
||||
)
|
||||
|
||||
def _all_reduce_release_flags(
|
||||
self, flags: List[bool], op: torch.distributed.ReduceOp
|
||||
) -> List[bool]:
|
||||
# Failures come from reduced polls and deferred entries from agreed holds,
|
||||
# so every rank supplies the same entries in the same order, even if empty.
|
||||
if not flags or torch.distributed.get_world_size(self.gloo_group) == 1:
|
||||
return flags
|
||||
reduced = torch.tensor(flags, dtype=torch.int32, device="cpu")
|
||||
torch.distributed.all_reduce(reduced, op=op, group=self.gloo_group)
|
||||
return [bool(flag) for flag in reduced.tolist()]
|
||||
|
||||
def resolve_deferred_releases(self) -> None:
|
||||
"""Release drained transfers; device destinations may also time out."""
|
||||
if not self._deferred_releases:
|
||||
return
|
||||
host_ready = torch.tensor(
|
||||
[
|
||||
req.kv_receiver.is_abort_release_safe()
|
||||
for req, _, _, _, _ in self._deferred_releases
|
||||
if self.enable_host_receive and req.host_staged
|
||||
],
|
||||
dtype=torch.int32,
|
||||
)
|
||||
if host_ready.numel() and torch.distributed.get_world_size(self.gloo_group) > 1:
|
||||
# Admission must see identical host capacity and page order on every rank.
|
||||
torch.distributed.all_reduce(
|
||||
host_ready, op=torch.distributed.ReduceOp.MIN, group=self.gloo_group
|
||||
)
|
||||
host_ready = iter(host_ready.tolist())
|
||||
now = time.monotonic()
|
||||
drained = [
|
||||
decode_req.kv_receiver.is_abort_release_safe()
|
||||
if self.enable_host_receive and decode_req.host_staged
|
||||
else decode_req.kv_receiver.kv_mgr.is_abort_release_safe(
|
||||
decode_req.req.bootstrap_room, required_acks
|
||||
)
|
||||
for decode_req, _, _, _, required_acks in self._deferred_releases
|
||||
]
|
||||
# Acks and deadlines are rank-local. Releasing only on some ranks would
|
||||
# diverge admission and the sizes of subsequent polling collectives.
|
||||
# Host pages still require a drain: a timeout cannot rule out remote writes.
|
||||
ready = self._all_reduce_release_flags(
|
||||
[
|
||||
is_drained
|
||||
or (
|
||||
not (self.enable_host_receive and decode_req.host_staged)
|
||||
and now >= deadline
|
||||
)
|
||||
for (decode_req, _, deadline, _, _), is_drained in zip(
|
||||
self._deferred_releases, drained
|
||||
)
|
||||
],
|
||||
torch.distributed.ReduceOp.MIN,
|
||||
)
|
||||
still_held = []
|
||||
to_release = []
|
||||
for (
|
||||
decode_req,
|
||||
start_time,
|
||||
deadline,
|
||||
idx,
|
||||
required_acks,
|
||||
) in self._deferred_releases:
|
||||
room = decode_req.req.bootstrap_room
|
||||
if self.enable_host_receive and decode_req.host_staged:
|
||||
if next(host_ready):
|
||||
to_release.append((decode_req, start_time, idx, room, True))
|
||||
else:
|
||||
# A timeout cannot prove that a remote write has stopped.
|
||||
still_held.append(
|
||||
(decode_req, start_time, deadline, idx, required_acks)
|
||||
for entry, is_ready, is_drained in zip(self._deferred_releases, ready, drained):
|
||||
decode_req, start_time, _, idx, _ = entry
|
||||
if is_ready:
|
||||
to_release.append(
|
||||
(
|
||||
decode_req,
|
||||
start_time,
|
||||
idx,
|
||||
decode_req.req.bootstrap_room,
|
||||
is_drained,
|
||||
)
|
||||
continue
|
||||
kv_mgr = decode_req.kv_receiver.kv_mgr
|
||||
drained = kv_mgr.is_abort_release_safe(room, required_acks)
|
||||
if not drained and now < deadline:
|
||||
still_held.append(
|
||||
(decode_req, start_time, deadline, idx, required_acks)
|
||||
)
|
||||
else:
|
||||
to_release.append((decode_req, start_time, idx, room, drained))
|
||||
still_held.append(entry)
|
||||
# Commit the survivors before releasing so a _do_release exception can't
|
||||
# leave a released entry in the list (double-free / None receiver on retry).
|
||||
self._deferred_releases = still_held
|
||||
|
||||
@@ -45,6 +45,9 @@ class FakeReceiver:
|
||||
|
||||
class TestDecodeQueueCleanup(CustomTestCase):
|
||||
def setUp(self):
|
||||
world_size = patch("torch.distributed.get_world_size", return_value=1)
|
||||
world_size.start()
|
||||
self.addCleanup(world_size.stop)
|
||||
# The code under test reads its config from the bags.
|
||||
reset_context()
|
||||
self.addCleanup(reset_context)
|
||||
@@ -572,6 +575,12 @@ def _gated_decode_req(poll, restore_status):
|
||||
|
||||
|
||||
class TestDecodeHiCacheRestoreGate(CustomTestCase):
|
||||
def setUp(self):
|
||||
super().setUp()
|
||||
world_size = patch("torch.distributed.get_world_size", return_value=1)
|
||||
world_size.start()
|
||||
self.addCleanup(world_size.stop)
|
||||
|
||||
def test_gate_folds_restore_outcome_into_terminal_poll_only(self):
|
||||
cases = [
|
||||
(KVPoll.Success, HiCacheRestoreResult.READY, KVPoll.Success),
|
||||
|
||||
@@ -6,14 +6,21 @@ them immediately (which could let the still-in-flight write land on pages alread
|
||||
reused by another request). The pages are released once every prefill rank acks
|
||||
that its transfer drained (CommonKVManager.is_abort_release_safe). Device
|
||||
destinations may also release on timeout; host destinations require the ack.
|
||||
All decode attention-TP ranks must agree on holding and releasing each request.
|
||||
"""
|
||||
|
||||
import datetime
|
||||
import tempfile
|
||||
import threading
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
from typing import NamedTuple
|
||||
from unittest.mock import MagicMock, call, patch
|
||||
|
||||
import torch.distributed as dist
|
||||
import torch.multiprocessing as mp
|
||||
|
||||
from sglang.srt.disaggregation import decode as decode_mod
|
||||
from sglang.srt.disaggregation.base.conn import BaseKVManager, KVPoll
|
||||
from sglang.srt.disaggregation.common.conn import (
|
||||
@@ -27,10 +34,11 @@ from sglang.srt.disaggregation.common.conn import (
|
||||
CommonKVSender,
|
||||
)
|
||||
from sglang.srt.disaggregation.decode import DecodeTransferQueue
|
||||
from sglang.srt.disaggregation.decode_hicache_mixin import HiCacheRestoreResult
|
||||
from sglang.test.ci.ci_register import register_cpu_ci
|
||||
from sglang.test.test_utils import CustomTestCase
|
||||
|
||||
register_cpu_ci(est_time=10, suite="base-a-test-cpu")
|
||||
register_cpu_ci(est_time=45, suite="base-a-test-cpu")
|
||||
|
||||
ABORT_GENERATION = 7
|
||||
|
||||
@@ -661,14 +669,21 @@ class _FakeIdxAllocator:
|
||||
def _make_queue(timeout=30.0, enable_metrics=False):
|
||||
q = DecodeTransferQueue.__new__(DecodeTransferQueue)
|
||||
q._deferred_releases = []
|
||||
q.queue = []
|
||||
q.gloo_group = dist.group.WORLD
|
||||
q.tp_rank = 0
|
||||
q.enable_deferred_kv_release = True
|
||||
q.deferred_kv_release_timeout = timeout
|
||||
q.enable_host_receive = False
|
||||
q.enable_staging = False
|
||||
q.staging_handler = None
|
||||
q.tree_cache = object()
|
||||
q.tree_cache = MagicMock()
|
||||
q.metadata_buffers = SimpleNamespace(bootstrap_room={})
|
||||
q.req_to_metadata_buffer_idx_allocator = _FakeIdxAllocator()
|
||||
q.scheduler = SimpleNamespace(
|
||||
enable_decode_hicache=False,
|
||||
enable_hisparse=False,
|
||||
output_streamer=MagicMock(),
|
||||
metrics_reporter=SimpleNamespace(enable_metrics=enable_metrics),
|
||||
metrics_collector=SimpleNamespace(
|
||||
observe_decode_deferred_kv_release=MagicMock()
|
||||
@@ -680,6 +695,10 @@ def _make_queue(timeout=30.0, enable_metrics=False):
|
||||
def _make_decode_req(room, idx, mgr, n_prefill_ranks=1):
|
||||
receiver = SimpleNamespace(
|
||||
kv_mgr=mgr,
|
||||
abort_notified=True,
|
||||
ensure_abort_notified=MagicMock(),
|
||||
failure_exception=lambda: None,
|
||||
poll=lambda: KVPoll.Failed,
|
||||
# One entry per prefill rank the decode notified of the abort; its length
|
||||
# is the required drain-ack count (see DecodeTransferQueue._defer_release).
|
||||
bootstrap_infos=[{"rank": r} for r in range(n_prefill_ranks)],
|
||||
@@ -687,13 +706,23 @@ def _make_decode_req(room, idx, mgr, n_prefill_ranks=1):
|
||||
clear=lambda: None,
|
||||
)
|
||||
return SimpleNamespace(
|
||||
req=SimpleNamespace(bootstrap_room=room),
|
||||
req=SimpleNamespace(bootstrap_room=room, rid=str(room), return_logprob=False),
|
||||
host_staged=False,
|
||||
hicache_restore_status=HiCacheRestoreResult.READY,
|
||||
prefix_match=None,
|
||||
hicache_restore_lock=None,
|
||||
kv_receiver=receiver,
|
||||
metadata_buffer_index=idx,
|
||||
)
|
||||
|
||||
|
||||
class TestResolveDeferredReleases(CustomTestCase):
|
||||
def setUp(self):
|
||||
super().setUp()
|
||||
world_size = patch.object(dist, "get_world_size", return_value=1)
|
||||
world_size.start()
|
||||
self.addCleanup(world_size.stop)
|
||||
|
||||
def test_host_release_requires_drain_on_every_rank_even_after_timeout(self):
|
||||
mgr = _make_manager()
|
||||
q = _make_queue(timeout=-1)
|
||||
@@ -866,6 +895,123 @@ class TestResolveDeferredReleases(CustomTestCase):
|
||||
self.assertEqual(held_idx, 9)
|
||||
self.assertEqual(required, 1)
|
||||
|
||||
def test_two_rank_release_consensus(self):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
mp.spawn(
|
||||
_run_release_consensus,
|
||||
args=(str(Path(tmp) / "rendezvous"),),
|
||||
nprocs=2,
|
||||
join=True,
|
||||
)
|
||||
|
||||
|
||||
def _run_release_consensus(rank: int, rendezvous: str) -> None:
|
||||
dist.init_process_group(
|
||||
"gloo",
|
||||
init_method=f"file://{rendezvous}",
|
||||
rank=rank,
|
||||
world_size=2,
|
||||
timeout=datetime.timedelta(seconds=30),
|
||||
)
|
||||
try:
|
||||
observations = []
|
||||
expected = []
|
||||
for scenario, host, drained, now, first_freed in (
|
||||
("ack", False, rank == 0, 10.0, [1]),
|
||||
("deadline", False, False, 31.0 if rank == 0 else 29.0, [1]),
|
||||
("mixed", False, rank == 0, 31.0, [0, 1]),
|
||||
("host_ack", True, rank == 0, 10.0, [1]),
|
||||
("host_timeout", True, False, 31.0, [1]),
|
||||
):
|
||||
mgr = _make_manager()
|
||||
q = _make_queue()
|
||||
q.enable_host_receive = host
|
||||
dreq = _make_decode_req(42, 0, mgr)
|
||||
dreq.host_staged = host
|
||||
dreq.kv_receiver.is_abort_release_safe = lambda: drained
|
||||
generation = mgr.register_deferred_abort_room(42)
|
||||
if drained:
|
||||
mgr.note_abort_ack(42, 0, generation)
|
||||
# An independently ready entry must be released even while the
|
||||
# first entry waits for a peer; agreement is per request.
|
||||
ready_req = _make_decode_req(44, 1, mgr)
|
||||
ready_req.host_staged = host
|
||||
ready_req.kv_receiver.is_abort_release_safe = lambda: True
|
||||
ready_generation = mgr.register_deferred_abort_room(44)
|
||||
mgr.note_abort_ack(44, 0, ready_generation)
|
||||
with patch.object(decode_mod.time, "monotonic", return_value=0.0):
|
||||
q._defer_release(dreq)
|
||||
q._defer_release(ready_req)
|
||||
with (
|
||||
patch.object(decode_mod.time, "monotonic", return_value=now) as clock,
|
||||
patch.object(decode_mod, "release_kv_cache"),
|
||||
patch.object(decode_mod, "discard_kv_cache_backup"),
|
||||
):
|
||||
q.resolve_deferred_releases()
|
||||
observations.append(
|
||||
(scenario, list(q.req_to_metadata_buffer_idx_allocator.freed))
|
||||
)
|
||||
expected.append((scenario, first_freed))
|
||||
|
||||
drained = True
|
||||
if scenario != "deadline":
|
||||
mgr.note_abort_ack(42, 0, generation)
|
||||
clock.return_value = 31.0
|
||||
q.resolve_deferred_releases()
|
||||
q.resolve_deferred_releases()
|
||||
observations.append(
|
||||
(
|
||||
scenario + "_drained",
|
||||
list(q.req_to_metadata_buffer_idx_allocator.freed),
|
||||
)
|
||||
)
|
||||
expected.append(
|
||||
(scenario + "_drained", [0, 1] if scenario == "mixed" else [1, 0])
|
||||
)
|
||||
|
||||
for notification in ("some", "all", "none", "disabled"):
|
||||
mgr = _make_manager()
|
||||
q = _make_queue()
|
||||
q.enable_deferred_kv_release = notification != "disabled"
|
||||
dreq = _make_decode_req(43, 1, mgr)
|
||||
receiver = dreq.kv_receiver
|
||||
receiver.abort_notified = notification == "all" or (
|
||||
notification == "some" and rank == 0
|
||||
)
|
||||
if not receiver.abort_notified:
|
||||
receiver.bootstrap_infos = None
|
||||
generation = mgr.register_deferred_abort_room(43)
|
||||
q.queue = [dreq]
|
||||
with (
|
||||
patch.object(decode_mod, "prepare_abort"),
|
||||
patch.object(decode_mod, "release_kv_cache"),
|
||||
):
|
||||
q.pop_transferred()
|
||||
held = len(q._deferred_releases)
|
||||
observations.append((notification, held))
|
||||
expected.append((notification, int(notification in ("some", "all"))))
|
||||
held_by_rank = [None, None]
|
||||
dist.all_gather_object(held_by_rank, held)
|
||||
# Keep a broken implementation from deadlocking the test when
|
||||
# just one rank enters the release collective.
|
||||
if held_by_rank == [1, 1]:
|
||||
mgr.note_abort_ack(43, 0, generation)
|
||||
q.resolve_deferred_releases()
|
||||
observations.append(
|
||||
(
|
||||
notification + "_freed",
|
||||
list(q.req_to_metadata_buffer_idx_allocator.freed),
|
||||
)
|
||||
)
|
||||
expected.append((notification + "_freed", [1]))
|
||||
|
||||
# Both ranks reach the same assertion even if the old code diverges.
|
||||
results = [None, None]
|
||||
dist.all_gather_object(results, observations)
|
||||
assert results == [expected, expected], results
|
||||
finally:
|
||||
dist.destroy_process_group()
|
||||
|
||||
|
||||
def _make_pop_queue(mgr):
|
||||
"""DecodeTransferQueue shell wired for pop_transferred's Failed branch."""
|
||||
@@ -918,6 +1064,12 @@ class TestFailedTransfersDeferOnEveryFailure(CustomTestCase):
|
||||
to prevent. pop_transferred must notify the prefill ranks and defer on
|
||||
every failure kind, not only when abort_notified is already set."""
|
||||
|
||||
def setUp(self):
|
||||
super().setUp()
|
||||
world_size = patch.object(dist, "get_world_size", return_value=1)
|
||||
world_size.start()
|
||||
self.addCleanup(world_size.stop)
|
||||
|
||||
def _pop(self, q):
|
||||
released = []
|
||||
q._release_request = released.append
|
||||
|
||||
Reference in New Issue
Block a user