Files
sglang/test/registered/unit/disaggregation/test_encode_server.py
T

517 lines
18 KiB
Python

import asyncio
import pickle
import unittest
from types import SimpleNamespace
from unittest.mock import AsyncMock, patch
import numpy as np
import torch
import sglang.srt.disaggregation.encoder.server as encoder_server
from sglang.srt.disaggregation.encoder import http_server
from sglang.srt.disaggregation.encoder import runtime as encoder_runtime
from sglang.srt.disaggregation.encoder.preprocessor import EncoderPreprocessor
from sglang.srt.disaggregation.encoder.receiver import EmbeddingData
from sglang.srt.disaggregation.encoder.runtime import (
_DP_RELEASE_AFTER_ENCODE,
DPDispatcher,
_retire_abandoned_encode,
)
from sglang.srt.disaggregation.encoder.server import (
EncoderMetaRegistry,
MMEncoder,
MMError,
meta_registry,
)
from sglang.srt.distributed.device_communicators.mooncake_transfer_engine import (
MooncakeTransferEngine,
)
from sglang.srt.managers.io_struct import unwrap_from_pickle
from sglang.srt.managers.schedule_batch import Modality
from sglang.srt.utils.common import safe_pickle_loads
from sglang.test.ci.ci_register import register_cpu_ci
from sglang.test.test_utils import CustomTestCase
register_cpu_ci(est_time=3, suite="base-a-test-cpu")
class TestEncoderDPErrorHandling(CustomTestCase):
@staticmethod
async def _run_registration_error(error):
encoder = SimpleNamespace(
register_embedding_destinations=AsyncMock(side_effect=error)
)
send = AsyncMock()
request = {
"req_id": "req",
"receive_count": 1,
"receive_url": "tcp://127.0.0.1:1",
}
with patch.object(encoder_runtime, "async_sock_send", send):
await encoder_runtime._dp_worker_handle_request(
encoder,
None,
object(),
asyncio.Lock(),
0,
request,
"register_destinations",
)
return unwrap_from_pickle(send.await_args.args[1])
def test_worker_reports_third_party_exception_with_callable_code(self):
class RpcLikeError(Exception):
def code(self):
return "INTERNAL"
envelope = asyncio.run(
self._run_registration_error(RpcLikeError("registration failed"))
)
self.assertEqual(envelope["_error"], "registration failed")
self.assertEqual(envelope["_error_code"], 500)
def test_worker_preserves_mm_error_status(self):
envelope = asyncio.run(
self._run_registration_error(MMError("bad destination", code=400))
)
self.assertEqual(envelope["_error_code"], 400)
def test_dispatcher_drops_malformed_result_without_stopping_listener(self):
async def run():
dispatcher = encoder_runtime.DPDispatcher(
dp_size=1,
dispatch_sockets=[object()],
release_sockets=[object()],
result_socket=object(),
worker_processes=[],
)
future = asyncio.get_running_loop().create_future()
dispatcher.pending_futures[0]["req"] = future
dispatcher.req_id_to_rank["req"] = 0
valid = {"req_id": "req", "_dp_type": "encode", "content": None}
recv = AsyncMock(
side_effect=[
["not", "an", "envelope"],
valid,
asyncio.CancelledError(),
]
)
with patch.object(encoder_runtime, "async_sock_recv", recv):
listener = asyncio.create_task(dispatcher._result_listener())
await asyncio.wait_for(future, timeout=1)
listener.cancel()
with self.assertRaises(asyncio.CancelledError):
await listener
self.assertEqual(future.result(), valid)
asyncio.run(run())
class TestEncoderMetaRegistry(CustomTestCase):
def test_stale_releases_do_not_block_each_other(self):
async def run():
registry = EncoderMetaRegistry(wait_timeout=1, sweep_timeout=1)
blocked_started = asyncio.Event()
unblock = asyncio.Event()
fast_released = asyncio.Event()
async def release(req_id):
if req_id == "blocked":
blocked_started.set()
await unblock.wait()
else:
fast_released.set()
registry.on_release = release
registry._pending_at.update(blocked=0, fast=0)
blocked_task = registry._schedule_stale_release("blocked")
await asyncio.wait_for(blocked_started.wait(), timeout=1)
fast_task = registry._schedule_stale_release("fast")
await asyncio.wait_for(fast_released.wait(), timeout=1)
await fast_task
self.assertIn("blocked", registry._pending_at)
self.assertNotIn("fast", registry._pending_at)
unblock.set()
await blocked_task
self.assertNotIn("blocked", registry._pending_at)
asyncio.run(run())
def test_failed_stale_release_is_retried(self):
async def run():
registry = EncoderMetaRegistry(wait_timeout=1, sweep_timeout=1)
attempts = 0
async def release(_req_id):
nonlocal attempts
attempts += 1
if attempts == 1:
raise RuntimeError("transient cleanup failure")
registry.on_release = release
registry._pending_at["req"] = 0
await registry._release_stale("req")
self.assertIn("req", registry._pending_at)
retry_at = registry._pending_at["req"]
self.assertGreater(retry_at, 0)
await registry._release_stale("req")
self.assertNotIn("req", registry._pending_at)
self.assertEqual(attempts, 2)
asyncio.run(run())
def test_send_retries_do_not_release_before_all_destinations_finish(self):
async def run():
registry = EncoderMetaRegistry(wait_timeout=1, sweep_timeout=1)
released = AsyncMock()
registry.on_release = released
await registry.note_send_done("req", 2, "10.0.0.1:5000")
await registry.note_send_done("req", 2, "10.0.0.1:5000")
released.assert_not_awaited()
await registry.note_send_done("req", 2, "10.0.0.2:5000")
released.assert_awaited_once_with("req")
asyncio.run(run())
def test_http_send_counts_the_normalized_destination(self):
async def run():
send = AsyncMock(return_value=True)
note_send_done = AsyncMock()
request = {
"req_id": "req",
"prefill_host": "127.0.0.1",
"embedding_port": 5000,
"session_id": "session",
"buffer_address": 1234,
"receive_count": 2,
}
with (
patch.object(http_server, "dp_dispatcher", None),
patch.object(http_server, "encoder", SimpleNamespace(send=send)),
patch.object(
encoder_server.meta_registry,
"note_send_done",
note_send_done,
),
):
response = await http_server.handle_send_request(request)
self.assertEqual(response.status_code, 200)
note_send_done.assert_awaited_once_with("req", 2, "127.0.0.1:5000")
asyncio.run(run())
def test_dp_send_counts_the_normalized_destination(self):
async def run():
encoder = SimpleNamespace(send=AsyncMock(return_value=True))
note_send_done = AsyncMock()
request = {
"req_id": "req",
"prefill_host": "127.0.0.1",
"embedding_port": 5000,
"session_id": "session",
"buffer_address": 1234,
"receive_count": 2,
}
with (
patch.object(encoder_runtime, "async_sock_send", AsyncMock()),
patch.object(
encoder_server.meta_registry,
"note_send_done",
note_send_done,
),
):
await encoder_runtime._dp_worker_handle_request(
encoder,
None,
object(),
asyncio.Lock(),
0,
request,
"send",
)
note_send_done.assert_awaited_once_with("req", 2, "127.0.0.1:5000")
asyncio.run(run())
class TestEncoderPreprocessorKimiGrid(CustomTestCase):
@staticmethod
def _make_preprocessor(model_type="kimi_vl"):
preprocessor = EncoderPreprocessor.__new__(EncoderPreprocessor)
preprocessor.model_type = model_type
preprocessor.model_config = SimpleNamespace(
hf_config=SimpleNamespace(
vision_config=SimpleNamespace(merge_kernel_size=(2, 2))
)
)
preprocessor.image_processor = SimpleNamespace(merge_size=2)
preprocessor._model_preprocessor = None
return preprocessor
@staticmethod
def _make_encoder():
return MMEncoder.__new__(MMEncoder)
def test_kimi_vl_prefers_and_normalizes_hw_grid(self):
mm_inputs = {
"image_grid_hws": np.array([[40, 60]], dtype=np.int64),
"image_grid_thw": torch.tensor([[1, 20, 30]]),
"grid_thws": torch.tensor([[1, 10, 15]]),
}
grid = self._make_preprocessor()._get_mm_grid_dim(mm_inputs, Modality.IMAGE)
self.assertIsInstance(grid, torch.Tensor)
torch.testing.assert_close(grid, torch.tensor([[40, 60]]))
def test_kimi_k25_keeps_thw_grid_preference(self):
mm_inputs = {
"image_grid_hws": np.array([[40, 60]], dtype=np.int64),
"grid_thws": np.array([[1, 10, 15]], dtype=np.int64),
}
grid = self._make_preprocessor("kimi_k25")._get_mm_grid_dim(
mm_inputs, Modality.IMAGE
)
torch.testing.assert_close(grid, torch.tensor([[1, 10, 15]]))
def test_kimi_vl_2d_grid_counting_and_slicing(self):
preprocessor = self._make_preprocessor()
encoder = self._make_encoder()
grids = torch.tensor([[40, 60], [20, 40]])
embedding = torch.arange(800 * 2).reshape(800, 2)
self.assertEqual(
preprocessor.get_num_patches(grids[0], Modality.IMAGE),
2400,
)
self.assertEqual(
preprocessor.get_num_tokens(grids[0], Modality.IMAGE),
600,
)
slices = encoder.slice_embedding(embedding, [600, 200])
self.assertEqual([item.shape for item in slices], [(600, 2), (200, 2)])
torch.testing.assert_close(slices[0], embedding[:600])
torch.testing.assert_close(slices[1], embedding[600:])
def test_kimi_3d_grid_remains_supported(self):
preprocessor = self._make_preprocessor()
grid = torch.tensor([1, 40, 60])
self.assertEqual(preprocessor.get_num_patches(grid, Modality.IMAGE), 2400)
self.assertEqual(preprocessor.get_num_tokens(grid, Modality.IMAGE), 600)
def test_kimi_k25_3d_patch_counting_is_unchanged(self):
preprocessor = self._make_preprocessor("kimi_k25")
grid = torch.tensor([2, 12, 16])
self.assertEqual(preprocessor.get_num_patches(grid, Modality.IMAGE), 384)
self.assertEqual(preprocessor.get_num_tokens(grid, Modality.IMAGE), 48)
def test_grid_metadata_is_safe_to_deserialize(self):
grid = self._make_preprocessor()._get_mm_grid_dim(
{"image_grid_hws": np.array([[40, 60]], dtype=np.int64)},
Modality.IMAGE,
)
embedding_data = EmbeddingData(
req_id="test-request",
num_parts=1,
part_idx=0,
grid_dim=grid,
modality=Modality.IMAGE,
embedding=torch.zeros((600, 4)),
)
restored = safe_pickle_loads(
pickle.dumps(embedding_data.copy_without_embedding())
)
torch.testing.assert_close(restored.grid_dim, torch.tensor([[40, 60]]))
class TestEncoderDPAbandonedRequest(CustomTestCase):
@staticmethod
def _make_dispatcher():
return DPDispatcher(
dp_size=1,
dispatch_sockets=[object()],
release_sockets=[object()],
result_socket=object(),
worker_processes=[],
)
def test_dispatch_timeout_notifies_worker_to_release(self):
async def run():
dispatcher = self._make_dispatcher()
sent = []
release_sent = asyncio.Event()
async def send(socket, payload):
message = unwrap_from_pickle(payload)
sent.append((socket, message))
if message.get("_dp_type") == _DP_RELEASE_AFTER_ENCODE:
release_sent.set()
request = {"req_id": "timed-out", "modality": "image"}
with (
patch(
"sglang.srt.disaggregation.encoder.runtime.async_sock_send",
side_effect=send,
),
patch(
"sglang.srt.disaggregation.encoder.runtime.server_module.ENCODER_REQ_TIMEOUT",
0.01,
),
):
result = await dispatcher.dispatch(request)
await asyncio.wait_for(release_sent.wait(), timeout=1)
self.assertEqual(result["_error_type"], "TimeoutError")
self.assertIs(sent[0][0], dispatcher.dispatch_sockets[0])
self.assertEqual(sent[0][1], request)
self.assertIs(sent[1][0], dispatcher.release_sockets[0])
self.assertEqual(
sent[1][1],
{
"_dp_type": _DP_RELEASE_AFTER_ENCODE,
"req_id": "timed-out",
},
)
self.assertEqual(dispatcher.pending_counts, [0])
self.assertNotIn("timed-out", dispatcher.req_id_to_rank)
asyncio.run(run())
def test_dispatch_cancellation_notifies_worker_to_release(self):
async def run():
dispatcher = self._make_dispatcher()
encode_sent = asyncio.Event()
release_sent = asyncio.Event()
async def send(_socket, payload):
message = unwrap_from_pickle(payload)
if message.get("_dp_type") == _DP_RELEASE_AFTER_ENCODE:
release_sent.set()
else:
encode_sent.set()
with patch(
"sglang.srt.disaggregation.encoder.runtime.async_sock_send",
side_effect=send,
):
task = asyncio.create_task(
dispatcher.dispatch({"req_id": "cancelled", "modality": "image"})
)
await encode_sent.wait()
task.cancel()
with self.assertRaises(asyncio.CancelledError):
await task
await asyncio.wait_for(release_sent.wait(), timeout=1)
self.assertEqual(dispatcher.pending_counts, [0])
self.assertNotIn("cancelled", dispatcher.req_id_to_rank)
asyncio.run(run())
def test_worker_marks_running_encode_abandoned(self):
async def run():
async def encode():
await asyncio.Event().wait()
encode_task = asyncio.create_task(encode())
encoder = SimpleNamespace(
abandon_request=AsyncMock(),
release_request=AsyncMock(),
)
await _retire_abandoned_encode(encoder, encode_task, "abandoned")
encoder.abandon_request.assert_awaited_once_with("abandoned")
encoder.release_request.assert_not_awaited()
encode_task.cancel()
await asyncio.gather(encode_task, return_exceptions=True)
asyncio.run(run())
def test_worker_preserves_release_before_encode_task_exists(self):
async def run():
encoder = MMEncoder.__new__(MMEncoder)
encoder.rank = 0
encoder.req_states = {}
encoder.abandoned_req_ids = set()
encoder.delivery = SimpleNamespace(release=AsyncMock())
await _retire_abandoned_encode(encoder, None, "abandoned")
self.assertIn("abandoned", encoder.abandoned_req_ids)
state = encoder._acquire_encode_ref("abandoned")
self.assertTrue(state.release_requested)
with patch.object(meta_registry, "discard", AsyncMock()):
await encoder._release_encode_ref(state)
encoder.delivery.release.assert_awaited_once_with(state)
self.assertNotIn("abandoned", encoder.req_states)
asyncio.run(run())
def test_worker_release_survives_encode_failure(self):
async def run():
async def encode():
raise RuntimeError("bad image")
encode_task = asyncio.create_task(encode())
await asyncio.sleep(0)
encoder = SimpleNamespace(
abandon_request=AsyncMock(),
release_request=AsyncMock(),
)
await _retire_abandoned_encode(
encoder,
encode_task,
"failed",
)
encoder.release_request.assert_awaited_once_with("failed")
encoder.abandon_request.assert_not_awaited()
await asyncio.gather(encode_task, return_exceptions=True)
asyncio.run(run())
class TestMooncakeRegistration(CustomTestCase):
def setUp(self):
self.engine = MooncakeTransferEngine.__new__(MooncakeTransferEngine)
self.engine.engine = unittest.mock.Mock()
def test_register_raises_on_nonzero_status(self):
self.engine.engine.register_memory.return_value = -1
with self.assertRaisesRegex(RuntimeError, "registration failed.*ret=-1"):
self.engine.register(1234, 4096)
def test_deregister_preserves_backend_failure(self):
backend_error = OSError("backend failed")
self.engine.engine.unregister_memory.side_effect = backend_error
with self.assertRaisesRegex(RuntimeError, "deregistration failed") as ctx:
self.engine.deregister(1234)
self.assertIs(ctx.exception.__cause__, backend_error)
if __name__ == "__main__":
unittest.main()