Files
sglang/test/registered/unit/disaggregation/test_decode_queue_cleanup.py
T

222 lines
8.4 KiB
Python

import unittest
from concurrent.futures import Future
from types import SimpleNamespace
from unittest.mock import MagicMock, patch
from sglang.srt.disaggregation.base import KVPoll
from sglang.srt.disaggregation.common.conn import KVTransferError
from sglang.srt.disaggregation.decode import (
DecodePreallocQueue,
DecodeTransferQueue,
HiCacheRestoreResult,
)
from sglang.srt.disaggregation.decode_hicache_mixin import (
HiCacheRestoreGatedKVReceiver,
)
from sglang.srt.runtime_context import publish, reset_context
from sglang.srt.server_args import ServerArgs
from sglang.test.ci.ci_register import register_cpu_ci
from sglang.test.test_utils import CustomTestCase
register_cpu_ci(est_time=4, suite="base-a-test-cpu")
def _run_as_single_rank(test: unittest.TestCase) -> None:
# The transfer queue sizes its release collectives from the gloo group;
# these tests have no process group, so answer as one rank.
world_size = patch("torch.distributed.get_world_size", return_value=1)
world_size.start()
test.addCleanup(world_size.stop)
class TestDecodeQueueCleanup(CustomTestCase):
def setUp(self):
_run_as_single_rank(self)
# The code under test reads its config from the bags.
reset_context()
self.addCleanup(reset_context)
publish(ServerArgs(model_path="dummy"), role="tokenizer")
def test_ensure_prefill_info_tolerates_cleared_receiver(self):
# A req whose kv_receiver was already cleared must not crash on .abort().
queue = DecodePreallocQueue.__new__(DecodePreallocQueue)
queue._max_ensure_retries = 1
queue._ensure_retry_interval = 0
queue._ensure_retry_count = {"127.0.0.1:11500": 0}
queue._ensure_last_attempt_time = {}
queue.kv_manager = MagicMock()
queue.kv_manager.try_ensure_parallel_info.return_value = False
cleared_req = SimpleNamespace(
req=SimpleNamespace(rid="cleared"), kv_receiver=None
)
addr_to_reqs = {"127.0.0.1:11500": [cleared_req]}
ready, remaining = queue._ensure_prefill_info(addr_to_reqs)
self.assertEqual(ready, {})
self.assertEqual(remaining, [])
def test_prefetches_prefill_dp_rank_query(self):
addr = "127.0.0.1:11500"
executor = MagicMock()
future = Future()
future.set_result({"7": 1})
executor.submit.return_value = future
def decode_req(room):
return SimpleNamespace(
req=SimpleNamespace(
bootstrap_host="127.0.0.1",
bootstrap_port=11500,
bootstrap_room=room,
),
kv_receiver=MagicMock(),
)
first = decode_req(7)
queue = DecodePreallocQueue.__new__(DecodePreallocQueue)
queue.pending_reqs = [first]
queue._prefill_dp_rank_queries = {}
queue.kv_manager = SimpleNamespace(
prefill_info_table={addr: object()},
_ensure_prefill_recompute_executor=lambda: executor,
)
queue._resolve_prefill_dp_rank = MagicMock(return_value=None)
queue._ensure_prefill_info = lambda groups: (groups, [])
queue.prefetch_prefill_dp_rank_queries()
tail = decode_req(8)
queue.pending_reqs.append(tail)
with patch(
"sglang.srt.disaggregation.decode.CommonKVReceiver.query_prefill_dp_ranks",
return_value={"8": 2},
) as query:
queue._resolve_pending_reqs()
_, called_addr, called_rooms = executor.submit.call_args.args
self.assertEqual((called_addr, called_rooms), (addr, [7]))
query.assert_called_once_with(addr, [8])
first.kv_receiver.init.assert_called_once_with(1)
tail.kv_receiver.init.assert_called_once_with(2)
self.assertEqual(queue.pending_reqs, [])
class _GateReceiver:
def __init__(self, poll):
self._poll = poll
self.require_staging = False
def poll(self):
return self._poll
def clear(self):
pass
def failure_exception(self):
raise KVTransferError(
7, "Failed due to an unknown reason from another rank", True
)
def _gated_decode_req(poll, restore_status):
return SimpleNamespace(
req=SimpleNamespace(rid="r", bootstrap_room=7, return_logprob=False),
kv_receiver=_GateReceiver(poll),
metadata_buffer_index=3,
hicache_restore_status=restore_status,
)
class TestDecodeHiCacheRestoreGate(CustomTestCase):
def setUp(self):
super().setUp()
_run_as_single_rank(self)
def test_gate_folds_restore_outcome_into_terminal_poll_only(self):
cases = [
(KVPoll.Success, HiCacheRestoreResult.READY, KVPoll.Success),
(KVPoll.Success, HiCacheRestoreResult.PENDING, KVPoll.Transferring),
(KVPoll.Success, HiCacheRestoreResult.FAILED, KVPoll.Failed),
(KVPoll.Transferring, HiCacheRestoreResult.FAILED, KVPoll.Transferring),
(
KVPoll.WaitingForInput,
HiCacheRestoreResult.FAILED,
KVPoll.WaitingForInput,
),
(KVPoll.Failed, HiCacheRestoreResult.PENDING, KVPoll.Failed),
]
for poll, status, expected in cases:
with self.subTest(poll=poll, status=status):
gated = HiCacheRestoreGatedKVReceiver(_gated_decode_req(poll, status))
self.assertEqual(gated.poll(), expected)
@patch("sglang.srt.disaggregation.decode.release_kv_cache")
@patch("sglang.srt.disaggregation.decode.prepare_abort")
def test_pop_transferred_branches_on_reduced_poll_only(
self, mock_prepare_abort, mock_release_kv_cache
):
def make_queue(decode_req, peer_poll):
queue = DecodeTransferQueue.__new__(DecodeTransferQueue)
queue.queue = [decode_req]
queue.enable_staging = False
queue.enable_host_receive = False
queue.enable_deferred_kv_release = False
queue.gloo_group = MagicMock()
queue.req_to_metadata_buffer_idx_allocator = MagicMock()
queue.tp_rank = 0
queue.tree_cache = MagicMock()
queue.metadata_buffers = SimpleNamespace(bootstrap_room=[None] * 4)
queue._clean_hicache_prefetch_resources = MagicMock()
queue._commit_transfer_to_req = MagicMock()
queue._record_transfer_failure = MagicMock()
scheduler = MagicMock()
scheduler.enable_decode_hicache = True
scheduler.enable_hisparse = False
scheduler.metrics_reporter.enable_metrics = False
queue.scheduler = scheduler
def reduce_with_peer(
pollers, group, decode_reqs=None, metadata_buffers=None
):
return [min(int(p.poll()), int(peer_poll)) for p in pollers]
patcher = patch(
"sglang.srt.disaggregation.decode.poll_and_all_reduce",
side_effect=reduce_with_peer,
)
patcher.start()
self.addCleanup(patcher.stop)
return queue
decode_req = _gated_decode_req(KVPoll.Success, HiCacheRestoreResult.FAILED)
queue = make_queue(decode_req, peer_poll=KVPoll.Failed)
self.assertEqual(queue.pop_transferred(), [])
self.assertEqual(queue.queue, [])
queue._commit_transfer_to_req.assert_not_called()
queue._record_transfer_failure.assert_called_once_with("HiCache restore failed")
mock_prepare_abort.assert_called_once()
mock_prepare_abort.reset_mock()
decode_req = _gated_decode_req(KVPoll.Success, HiCacheRestoreResult.READY)
queue = make_queue(decode_req, peer_poll=KVPoll.Failed)
self.assertEqual(queue.pop_transferred(), [])
queue._commit_transfer_to_req.assert_not_called()
queue._record_transfer_failure.assert_called_once_with(
"Failed due to an unknown reason from another rank"
)
mock_prepare_abort.assert_called_once()
mock_prepare_abort.reset_mock()
mock_release_kv_cache.reset_mock()
decode_req = _gated_decode_req(KVPoll.Transferring, HiCacheRestoreResult.FAILED)
queue = make_queue(decode_req, peer_poll=KVPoll.Success)
self.assertEqual(queue.pop_transferred(), [])
self.assertEqual(queue.queue, [decode_req])
mock_prepare_abort.assert_not_called()
mock_release_kv_cache.assert_not_called()
if __name__ == "__main__":
unittest.main()