222 lines
8.4 KiB
Python
222 lines
8.4 KiB
Python
import unittest
|
|
from concurrent.futures import Future
|
|
from types import SimpleNamespace
|
|
from unittest.mock import MagicMock, patch
|
|
|
|
from sglang.srt.disaggregation.base import KVPoll
|
|
from sglang.srt.disaggregation.common.conn import KVTransferError
|
|
from sglang.srt.disaggregation.decode import (
|
|
DecodePreallocQueue,
|
|
DecodeTransferQueue,
|
|
HiCacheRestoreResult,
|
|
)
|
|
from sglang.srt.disaggregation.decode_hicache_mixin import (
|
|
HiCacheRestoreGatedKVReceiver,
|
|
)
|
|
from sglang.srt.runtime_context import publish, reset_context
|
|
from sglang.srt.server_args import ServerArgs
|
|
from sglang.test.ci.ci_register import register_cpu_ci
|
|
from sglang.test.test_utils import CustomTestCase
|
|
|
|
register_cpu_ci(est_time=4, suite="base-a-test-cpu")
|
|
|
|
|
|
def _run_as_single_rank(test: unittest.TestCase) -> None:
|
|
# The transfer queue sizes its release collectives from the gloo group;
|
|
# these tests have no process group, so answer as one rank.
|
|
world_size = patch("torch.distributed.get_world_size", return_value=1)
|
|
world_size.start()
|
|
test.addCleanup(world_size.stop)
|
|
|
|
|
|
class TestDecodeQueueCleanup(CustomTestCase):
|
|
def setUp(self):
|
|
_run_as_single_rank(self)
|
|
# The code under test reads its config from the bags.
|
|
reset_context()
|
|
self.addCleanup(reset_context)
|
|
publish(ServerArgs(model_path="dummy"), role="tokenizer")
|
|
|
|
def test_ensure_prefill_info_tolerates_cleared_receiver(self):
|
|
# A req whose kv_receiver was already cleared must not crash on .abort().
|
|
queue = DecodePreallocQueue.__new__(DecodePreallocQueue)
|
|
queue._max_ensure_retries = 1
|
|
queue._ensure_retry_interval = 0
|
|
queue._ensure_retry_count = {"127.0.0.1:11500": 0}
|
|
queue._ensure_last_attempt_time = {}
|
|
queue.kv_manager = MagicMock()
|
|
queue.kv_manager.try_ensure_parallel_info.return_value = False
|
|
|
|
cleared_req = SimpleNamespace(
|
|
req=SimpleNamespace(rid="cleared"), kv_receiver=None
|
|
)
|
|
addr_to_reqs = {"127.0.0.1:11500": [cleared_req]}
|
|
|
|
ready, remaining = queue._ensure_prefill_info(addr_to_reqs)
|
|
|
|
self.assertEqual(ready, {})
|
|
self.assertEqual(remaining, [])
|
|
|
|
def test_prefetches_prefill_dp_rank_query(self):
|
|
addr = "127.0.0.1:11500"
|
|
executor = MagicMock()
|
|
future = Future()
|
|
future.set_result({"7": 1})
|
|
executor.submit.return_value = future
|
|
|
|
def decode_req(room):
|
|
return SimpleNamespace(
|
|
req=SimpleNamespace(
|
|
bootstrap_host="127.0.0.1",
|
|
bootstrap_port=11500,
|
|
bootstrap_room=room,
|
|
),
|
|
kv_receiver=MagicMock(),
|
|
)
|
|
|
|
first = decode_req(7)
|
|
queue = DecodePreallocQueue.__new__(DecodePreallocQueue)
|
|
queue.pending_reqs = [first]
|
|
queue._prefill_dp_rank_queries = {}
|
|
queue.kv_manager = SimpleNamespace(
|
|
prefill_info_table={addr: object()},
|
|
_ensure_prefill_recompute_executor=lambda: executor,
|
|
)
|
|
queue._resolve_prefill_dp_rank = MagicMock(return_value=None)
|
|
queue._ensure_prefill_info = lambda groups: (groups, [])
|
|
|
|
queue.prefetch_prefill_dp_rank_queries()
|
|
tail = decode_req(8)
|
|
queue.pending_reqs.append(tail)
|
|
with patch(
|
|
"sglang.srt.disaggregation.decode.CommonKVReceiver.query_prefill_dp_ranks",
|
|
return_value={"8": 2},
|
|
) as query:
|
|
queue._resolve_pending_reqs()
|
|
|
|
_, called_addr, called_rooms = executor.submit.call_args.args
|
|
self.assertEqual((called_addr, called_rooms), (addr, [7]))
|
|
query.assert_called_once_with(addr, [8])
|
|
first.kv_receiver.init.assert_called_once_with(1)
|
|
tail.kv_receiver.init.assert_called_once_with(2)
|
|
self.assertEqual(queue.pending_reqs, [])
|
|
|
|
|
|
class _GateReceiver:
|
|
def __init__(self, poll):
|
|
self._poll = poll
|
|
self.require_staging = False
|
|
|
|
def poll(self):
|
|
return self._poll
|
|
|
|
def clear(self):
|
|
pass
|
|
|
|
def failure_exception(self):
|
|
raise KVTransferError(
|
|
7, "Failed due to an unknown reason from another rank", True
|
|
)
|
|
|
|
|
|
def _gated_decode_req(poll, restore_status):
|
|
return SimpleNamespace(
|
|
req=SimpleNamespace(rid="r", bootstrap_room=7, return_logprob=False),
|
|
kv_receiver=_GateReceiver(poll),
|
|
metadata_buffer_index=3,
|
|
hicache_restore_status=restore_status,
|
|
)
|
|
|
|
|
|
class TestDecodeHiCacheRestoreGate(CustomTestCase):
|
|
def setUp(self):
|
|
super().setUp()
|
|
_run_as_single_rank(self)
|
|
|
|
def test_gate_folds_restore_outcome_into_terminal_poll_only(self):
|
|
cases = [
|
|
(KVPoll.Success, HiCacheRestoreResult.READY, KVPoll.Success),
|
|
(KVPoll.Success, HiCacheRestoreResult.PENDING, KVPoll.Transferring),
|
|
(KVPoll.Success, HiCacheRestoreResult.FAILED, KVPoll.Failed),
|
|
(KVPoll.Transferring, HiCacheRestoreResult.FAILED, KVPoll.Transferring),
|
|
(
|
|
KVPoll.WaitingForInput,
|
|
HiCacheRestoreResult.FAILED,
|
|
KVPoll.WaitingForInput,
|
|
),
|
|
(KVPoll.Failed, HiCacheRestoreResult.PENDING, KVPoll.Failed),
|
|
]
|
|
for poll, status, expected in cases:
|
|
with self.subTest(poll=poll, status=status):
|
|
gated = HiCacheRestoreGatedKVReceiver(_gated_decode_req(poll, status))
|
|
self.assertEqual(gated.poll(), expected)
|
|
|
|
@patch("sglang.srt.disaggregation.decode.release_kv_cache")
|
|
@patch("sglang.srt.disaggregation.decode.prepare_abort")
|
|
def test_pop_transferred_branches_on_reduced_poll_only(
|
|
self, mock_prepare_abort, mock_release_kv_cache
|
|
):
|
|
def make_queue(decode_req, peer_poll):
|
|
queue = DecodeTransferQueue.__new__(DecodeTransferQueue)
|
|
queue.queue = [decode_req]
|
|
queue.enable_staging = False
|
|
queue.enable_host_receive = False
|
|
queue.enable_deferred_kv_release = False
|
|
queue.gloo_group = MagicMock()
|
|
queue.req_to_metadata_buffer_idx_allocator = MagicMock()
|
|
queue.tp_rank = 0
|
|
queue.tree_cache = MagicMock()
|
|
queue.metadata_buffers = SimpleNamespace(bootstrap_room=[None] * 4)
|
|
queue._clean_hicache_prefetch_resources = MagicMock()
|
|
queue._commit_transfer_to_req = MagicMock()
|
|
queue._record_transfer_failure = MagicMock()
|
|
scheduler = MagicMock()
|
|
scheduler.enable_decode_hicache = True
|
|
scheduler.enable_hisparse = False
|
|
scheduler.metrics_reporter.enable_metrics = False
|
|
queue.scheduler = scheduler
|
|
|
|
def reduce_with_peer(
|
|
pollers, group, decode_reqs=None, metadata_buffers=None
|
|
):
|
|
return [min(int(p.poll()), int(peer_poll)) for p in pollers]
|
|
|
|
patcher = patch(
|
|
"sglang.srt.disaggregation.decode.poll_and_all_reduce",
|
|
side_effect=reduce_with_peer,
|
|
)
|
|
patcher.start()
|
|
self.addCleanup(patcher.stop)
|
|
return queue
|
|
|
|
decode_req = _gated_decode_req(KVPoll.Success, HiCacheRestoreResult.FAILED)
|
|
queue = make_queue(decode_req, peer_poll=KVPoll.Failed)
|
|
self.assertEqual(queue.pop_transferred(), [])
|
|
self.assertEqual(queue.queue, [])
|
|
queue._commit_transfer_to_req.assert_not_called()
|
|
queue._record_transfer_failure.assert_called_once_with("HiCache restore failed")
|
|
mock_prepare_abort.assert_called_once()
|
|
|
|
mock_prepare_abort.reset_mock()
|
|
decode_req = _gated_decode_req(KVPoll.Success, HiCacheRestoreResult.READY)
|
|
queue = make_queue(decode_req, peer_poll=KVPoll.Failed)
|
|
self.assertEqual(queue.pop_transferred(), [])
|
|
queue._commit_transfer_to_req.assert_not_called()
|
|
queue._record_transfer_failure.assert_called_once_with(
|
|
"Failed due to an unknown reason from another rank"
|
|
)
|
|
mock_prepare_abort.assert_called_once()
|
|
|
|
mock_prepare_abort.reset_mock()
|
|
mock_release_kv_cache.reset_mock()
|
|
decode_req = _gated_decode_req(KVPoll.Transferring, HiCacheRestoreResult.FAILED)
|
|
queue = make_queue(decode_req, peer_poll=KVPoll.Success)
|
|
self.assertEqual(queue.pop_transferred(), [])
|
|
self.assertEqual(queue.queue, [decode_req])
|
|
mock_prepare_abort.assert_not_called()
|
|
mock_release_kv_cache.assert_not_called()
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|