import asyncio import pickle import unittest from types import SimpleNamespace from unittest.mock import AsyncMock, patch import numpy as np import torch import sglang.srt.disaggregation.encoder.server as encoder_server from sglang.srt.disaggregation.encoder import http_server from sglang.srt.disaggregation.encoder import runtime as encoder_runtime from sglang.srt.disaggregation.encoder.preprocessor import EncoderPreprocessor from sglang.srt.disaggregation.encoder.receiver import EmbeddingData from sglang.srt.disaggregation.encoder.runtime import ( _DP_RELEASE_AFTER_ENCODE, DPDispatcher, _retire_abandoned_encode, ) from sglang.srt.disaggregation.encoder.server import ( EncoderMetaRegistry, MMEncoder, MMError, meta_registry, ) from sglang.srt.distributed.device_communicators.mooncake_transfer_engine import ( MooncakeTransferEngine, ) from sglang.srt.managers.io_struct import unwrap_from_pickle from sglang.srt.managers.schedule_batch import Modality from sglang.srt.utils.common import safe_pickle_loads from sglang.test.ci.ci_register import register_cpu_ci from sglang.test.test_utils import CustomTestCase register_cpu_ci(est_time=3, suite="base-a-test-cpu") class TestEncoderDPErrorHandling(CustomTestCase): @staticmethod async def _run_registration_error(error): encoder = SimpleNamespace( register_embedding_destinations=AsyncMock(side_effect=error) ) send = AsyncMock() request = { "req_id": "req", "receive_count": 1, "receive_url": "tcp://127.0.0.1:1", } with patch.object(encoder_runtime, "async_sock_send", send): await encoder_runtime._dp_worker_handle_request( encoder, None, object(), asyncio.Lock(), 0, request, "register_destinations", ) return unwrap_from_pickle(send.await_args.args[1]) def test_worker_reports_third_party_exception_with_callable_code(self): class RpcLikeError(Exception): def code(self): return "INTERNAL" envelope = asyncio.run( self._run_registration_error(RpcLikeError("registration failed")) ) self.assertEqual(envelope["_error"], "registration failed") self.assertEqual(envelope["_error_code"], 500) def test_worker_preserves_mm_error_status(self): envelope = asyncio.run( self._run_registration_error(MMError("bad destination", code=400)) ) self.assertEqual(envelope["_error_code"], 400) def test_dispatcher_drops_malformed_result_without_stopping_listener(self): async def run(): dispatcher = encoder_runtime.DPDispatcher( dp_size=1, dispatch_sockets=[object()], release_sockets=[object()], result_socket=object(), worker_processes=[], ) future = asyncio.get_running_loop().create_future() dispatcher.pending_futures[0]["req"] = future dispatcher.req_id_to_rank["req"] = 0 valid = {"req_id": "req", "_dp_type": "encode", "content": None} recv = AsyncMock( side_effect=[ ["not", "an", "envelope"], valid, asyncio.CancelledError(), ] ) with patch.object(encoder_runtime, "async_sock_recv", recv): listener = asyncio.create_task(dispatcher._result_listener()) await asyncio.wait_for(future, timeout=1) listener.cancel() with self.assertRaises(asyncio.CancelledError): await listener self.assertEqual(future.result(), valid) asyncio.run(run()) class TestEncoderMetaRegistry(CustomTestCase): def test_stale_releases_do_not_block_each_other(self): async def run(): registry = EncoderMetaRegistry(wait_timeout=1, sweep_timeout=1) blocked_started = asyncio.Event() unblock = asyncio.Event() fast_released = asyncio.Event() async def release(req_id): if req_id == "blocked": blocked_started.set() await unblock.wait() else: fast_released.set() registry.on_release = release registry._pending_at.update(blocked=0, fast=0) blocked_task = registry._schedule_stale_release("blocked") await asyncio.wait_for(blocked_started.wait(), timeout=1) fast_task = registry._schedule_stale_release("fast") await asyncio.wait_for(fast_released.wait(), timeout=1) await fast_task self.assertIn("blocked", registry._pending_at) self.assertNotIn("fast", registry._pending_at) unblock.set() await blocked_task self.assertNotIn("blocked", registry._pending_at) asyncio.run(run()) def test_failed_stale_release_is_retried(self): async def run(): registry = EncoderMetaRegistry(wait_timeout=1, sweep_timeout=1) attempts = 0 async def release(_req_id): nonlocal attempts attempts += 1 if attempts == 1: raise RuntimeError("transient cleanup failure") registry.on_release = release registry._pending_at["req"] = 0 await registry._release_stale("req") self.assertIn("req", registry._pending_at) retry_at = registry._pending_at["req"] self.assertGreater(retry_at, 0) await registry._release_stale("req") self.assertNotIn("req", registry._pending_at) self.assertEqual(attempts, 2) asyncio.run(run()) def test_send_retries_do_not_release_before_all_destinations_finish(self): async def run(): registry = EncoderMetaRegistry(wait_timeout=1, sweep_timeout=1) released = AsyncMock() registry.on_release = released await registry.note_send_done("req", 2, "10.0.0.1:5000") await registry.note_send_done("req", 2, "10.0.0.1:5000") released.assert_not_awaited() await registry.note_send_done("req", 2, "10.0.0.2:5000") released.assert_awaited_once_with("req") asyncio.run(run()) def test_http_send_counts_the_normalized_destination(self): async def run(): send = AsyncMock(return_value=True) note_send_done = AsyncMock() request = { "req_id": "req", "prefill_host": "127.0.0.1", "embedding_port": 5000, "session_id": "session", "buffer_address": 1234, "receive_count": 2, } with ( patch.object(http_server, "dp_dispatcher", None), patch.object(http_server, "encoder", SimpleNamespace(send=send)), patch.object( encoder_server.meta_registry, "note_send_done", note_send_done, ), ): response = await http_server.handle_send_request(request) self.assertEqual(response.status_code, 200) note_send_done.assert_awaited_once_with("req", 2, "127.0.0.1:5000") asyncio.run(run()) def test_dp_send_counts_the_normalized_destination(self): async def run(): encoder = SimpleNamespace(send=AsyncMock(return_value=True)) note_send_done = AsyncMock() request = { "req_id": "req", "prefill_host": "127.0.0.1", "embedding_port": 5000, "session_id": "session", "buffer_address": 1234, "receive_count": 2, } with ( patch.object(encoder_runtime, "async_sock_send", AsyncMock()), patch.object( encoder_server.meta_registry, "note_send_done", note_send_done, ), ): await encoder_runtime._dp_worker_handle_request( encoder, None, object(), asyncio.Lock(), 0, request, "send", ) note_send_done.assert_awaited_once_with("req", 2, "127.0.0.1:5000") asyncio.run(run()) class TestEncoderPreprocessorKimiGrid(CustomTestCase): @staticmethod def _make_preprocessor(model_type="kimi_vl"): preprocessor = EncoderPreprocessor.__new__(EncoderPreprocessor) preprocessor.model_type = model_type preprocessor.model_config = SimpleNamespace( hf_config=SimpleNamespace( vision_config=SimpleNamespace(merge_kernel_size=(2, 2)) ) ) preprocessor.image_processor = SimpleNamespace(merge_size=2) preprocessor._model_preprocessor = None return preprocessor @staticmethod def _make_encoder(): return MMEncoder.__new__(MMEncoder) def test_kimi_vl_prefers_and_normalizes_hw_grid(self): mm_inputs = { "image_grid_hws": np.array([[40, 60]], dtype=np.int64), "image_grid_thw": torch.tensor([[1, 20, 30]]), "grid_thws": torch.tensor([[1, 10, 15]]), } grid = self._make_preprocessor()._get_mm_grid_dim(mm_inputs, Modality.IMAGE) self.assertIsInstance(grid, torch.Tensor) torch.testing.assert_close(grid, torch.tensor([[40, 60]])) def test_kimi_k25_keeps_thw_grid_preference(self): mm_inputs = { "image_grid_hws": np.array([[40, 60]], dtype=np.int64), "grid_thws": np.array([[1, 10, 15]], dtype=np.int64), } grid = self._make_preprocessor("kimi_k25")._get_mm_grid_dim( mm_inputs, Modality.IMAGE ) torch.testing.assert_close(grid, torch.tensor([[1, 10, 15]])) def test_kimi_vl_2d_grid_counting_and_slicing(self): preprocessor = self._make_preprocessor() encoder = self._make_encoder() grids = torch.tensor([[40, 60], [20, 40]]) embedding = torch.arange(800 * 2).reshape(800, 2) self.assertEqual( preprocessor.get_num_patches(grids[0], Modality.IMAGE), 2400, ) self.assertEqual( preprocessor.get_num_tokens(grids[0], Modality.IMAGE), 600, ) slices = encoder.slice_embedding(embedding, [600, 200]) self.assertEqual([item.shape for item in slices], [(600, 2), (200, 2)]) torch.testing.assert_close(slices[0], embedding[:600]) torch.testing.assert_close(slices[1], embedding[600:]) def test_kimi_3d_grid_remains_supported(self): preprocessor = self._make_preprocessor() grid = torch.tensor([1, 40, 60]) self.assertEqual(preprocessor.get_num_patches(grid, Modality.IMAGE), 2400) self.assertEqual(preprocessor.get_num_tokens(grid, Modality.IMAGE), 600) def test_kimi_k25_3d_patch_counting_is_unchanged(self): preprocessor = self._make_preprocessor("kimi_k25") grid = torch.tensor([2, 12, 16]) self.assertEqual(preprocessor.get_num_patches(grid, Modality.IMAGE), 384) self.assertEqual(preprocessor.get_num_tokens(grid, Modality.IMAGE), 48) def test_grid_metadata_is_safe_to_deserialize(self): grid = self._make_preprocessor()._get_mm_grid_dim( {"image_grid_hws": np.array([[40, 60]], dtype=np.int64)}, Modality.IMAGE, ) embedding_data = EmbeddingData( req_id="test-request", num_parts=1, part_idx=0, grid_dim=grid, modality=Modality.IMAGE, embedding=torch.zeros((600, 4)), ) restored = safe_pickle_loads( pickle.dumps(embedding_data.copy_without_embedding()) ) torch.testing.assert_close(restored.grid_dim, torch.tensor([[40, 60]])) class TestEncoderDPAbandonedRequest(CustomTestCase): @staticmethod def _make_dispatcher(): return DPDispatcher( dp_size=1, dispatch_sockets=[object()], release_sockets=[object()], result_socket=object(), worker_processes=[], ) def test_dispatch_timeout_notifies_worker_to_release(self): async def run(): dispatcher = self._make_dispatcher() sent = [] release_sent = asyncio.Event() async def send(socket, payload): message = unwrap_from_pickle(payload) sent.append((socket, message)) if message.get("_dp_type") == _DP_RELEASE_AFTER_ENCODE: release_sent.set() request = {"req_id": "timed-out", "modality": "image"} with ( patch( "sglang.srt.disaggregation.encoder.runtime.async_sock_send", side_effect=send, ), patch( "sglang.srt.disaggregation.encoder.runtime.server_module.ENCODER_REQ_TIMEOUT", 0.01, ), ): result = await dispatcher.dispatch(request) await asyncio.wait_for(release_sent.wait(), timeout=1) self.assertEqual(result["_error_type"], "TimeoutError") self.assertIs(sent[0][0], dispatcher.dispatch_sockets[0]) self.assertEqual(sent[0][1], request) self.assertIs(sent[1][0], dispatcher.release_sockets[0]) self.assertEqual( sent[1][1], { "_dp_type": _DP_RELEASE_AFTER_ENCODE, "req_id": "timed-out", }, ) self.assertEqual(dispatcher.pending_counts, [0]) self.assertNotIn("timed-out", dispatcher.req_id_to_rank) asyncio.run(run()) def test_dispatch_cancellation_notifies_worker_to_release(self): async def run(): dispatcher = self._make_dispatcher() encode_sent = asyncio.Event() release_sent = asyncio.Event() async def send(_socket, payload): message = unwrap_from_pickle(payload) if message.get("_dp_type") == _DP_RELEASE_AFTER_ENCODE: release_sent.set() else: encode_sent.set() with patch( "sglang.srt.disaggregation.encoder.runtime.async_sock_send", side_effect=send, ): task = asyncio.create_task( dispatcher.dispatch({"req_id": "cancelled", "modality": "image"}) ) await encode_sent.wait() task.cancel() with self.assertRaises(asyncio.CancelledError): await task await asyncio.wait_for(release_sent.wait(), timeout=1) self.assertEqual(dispatcher.pending_counts, [0]) self.assertNotIn("cancelled", dispatcher.req_id_to_rank) asyncio.run(run()) def test_worker_marks_running_encode_abandoned(self): async def run(): async def encode(): await asyncio.Event().wait() encode_task = asyncio.create_task(encode()) encoder = SimpleNamespace( abandon_request=AsyncMock(), release_request=AsyncMock(), ) await _retire_abandoned_encode(encoder, encode_task, "abandoned") encoder.abandon_request.assert_awaited_once_with("abandoned") encoder.release_request.assert_not_awaited() encode_task.cancel() await asyncio.gather(encode_task, return_exceptions=True) asyncio.run(run()) def test_worker_preserves_release_before_encode_task_exists(self): async def run(): encoder = MMEncoder.__new__(MMEncoder) encoder.rank = 0 encoder.req_states = {} encoder.abandoned_req_ids = set() encoder.delivery = SimpleNamespace(release=AsyncMock()) await _retire_abandoned_encode(encoder, None, "abandoned") self.assertIn("abandoned", encoder.abandoned_req_ids) state = encoder._acquire_encode_ref("abandoned") self.assertTrue(state.release_requested) with patch.object(meta_registry, "discard", AsyncMock()): await encoder._release_encode_ref(state) encoder.delivery.release.assert_awaited_once_with(state) self.assertNotIn("abandoned", encoder.req_states) asyncio.run(run()) def test_worker_release_survives_encode_failure(self): async def run(): async def encode(): raise RuntimeError("bad image") encode_task = asyncio.create_task(encode()) await asyncio.sleep(0) encoder = SimpleNamespace( abandon_request=AsyncMock(), release_request=AsyncMock(), ) await _retire_abandoned_encode( encoder, encode_task, "failed", ) encoder.release_request.assert_awaited_once_with("failed") encoder.abandon_request.assert_not_awaited() await asyncio.gather(encode_task, return_exceptions=True) asyncio.run(run()) class TestMooncakeRegistration(CustomTestCase): def setUp(self): self.engine = MooncakeTransferEngine.__new__(MooncakeTransferEngine) self.engine.engine = unittest.mock.Mock() def test_register_raises_on_nonzero_status(self): self.engine.engine.register_memory.return_value = -1 with self.assertRaisesRegex(RuntimeError, "registration failed.*ret=-1"): self.engine.register(1234, 4096) def test_deregister_preserves_backend_failure(self): backend_error = OSError("backend failed") self.engine.engine.unregister_memory.side_effect = backend_error with self.assertRaisesRegex(RuntimeError, "deregistration failed") as ctx: self.engine.deregister(1234) self.assertIs(ctx.exception.__cause__, backend_error) if __name__ == "__main__": unittest.main()