diff --git a/python/sglang/srt/managers/tokenizer_manager.py b/python/sglang/srt/managers/tokenizer_manager.py index 21f6b8bf83..eb74ca0223 100644 --- a/python/sglang/srt/managers/tokenizer_manager.py +++ b/python/sglang/srt/managers/tokenizer_manager.py @@ -1216,9 +1216,7 @@ class TokenizerManager(TokenizerControlMixin, TokenizerManagerScoreMixin): and mm_inputs and mm_inputs.mm_items ): - for item in mm_inputs.mm_items: - if isinstance(item, MultimodalDataItem): - item.set_pad_value() + await self.mm_processor.hash_executor.set_pad_values(mm_inputs.mm_items) else: mm_inputs = None diff --git a/python/sglang/srt/multimodal/processors/base_processor.py b/python/sglang/srt/multimodal/processors/base_processor.py index 2e7d9de1f9..7b9e979f5f 100644 --- a/python/sglang/srt/multimodal/processors/base_processor.py +++ b/python/sglang/srt/multimodal/processors/base_processor.py @@ -36,6 +36,7 @@ from sglang.srt.multimodal.cache import ( build_processor_fingerprint, ) from sglang.srt.multimodal.processors.executor import MultimodalProcessorExecutor +from sglang.srt.multimodal.processors.hash_executor import MultimodalHashExecutor from sglang.srt.multimodal.transport.cuda_ipc import ( MM_FEATURE_CACHE_SIZE, MM_ITEM_MEMORY_POOL_RECYCLE_INTERVAL, @@ -331,6 +332,7 @@ class BaseMultimodalProcessor(ABC): max_workers=self.mm_io_worker_num, thread_name_prefix="sglang-mm-io", ) + self.hash_executor = MultimodalHashExecutor() if self.mm_io_worker_num > 4: logger.info( "Multimodal data loading enabled with %d worker threads (%s).", @@ -494,6 +496,7 @@ class BaseMultimodalProcessor(ABC): def shutdown(self) -> None: """Drop cached artifacts and stop every processor-side executor.""" self.clear_preprocess_cache() + self.hash_executor.shutdown() self.io_executor.shutdown(wait=False, cancel_futures=True) self.cpu_executor.shutdown(wait=False, cancel_futures=True) if self.mm_processor_executor is not None: diff --git a/python/sglang/srt/multimodal/processors/hash_executor.py b/python/sglang/srt/multimodal/processors/hash_executor.py new file mode 100644 index 0000000000..b3f8eb237a --- /dev/null +++ b/python/sglang/srt/multimodal/processors/hash_executor.py @@ -0,0 +1,86 @@ +"""Bounded CPU feature hashing for the tokenizer's event loop.""" + +import asyncio +from concurrent.futures import ThreadPoolExecutor + +import numpy as np +import torch + +from sglang.srt.environ import envs +from sglang.srt.managers.schedule_batch import MultimodalDataItem + + +def _is_cpu_feature(value: object) -> bool: + # Unknown transports stay on the caller thread, where their device context + # and lifetime are established. In particular, do not reconstruct CUDA IPC. + if isinstance(value, torch.Tensor): + return value.device.type == "cpu" + if isinstance(value, np.ndarray): + return True + if isinstance(value, list): + return bool(value) and all(_is_cpu_feature(part) for part in value) + return False + + +def _set_pad_values(items: list[MultimodalDataItem]) -> None: + for item in items: + item.set_pad_value() + + +class MultimodalHashExecutor: + """Keep at most ``max_workers`` native hash jobs in flight per processor. + + Admission happens before submission, so the thread pool has no unbounded + work queue. A cancelled caller retains its inputs and admission slot until + the native reader stops, including when cancellation is repeated. + """ + + def __init__(self, *, max_workers: int = 2): + self._executor = ThreadPoolExecutor( + max_workers=max_workers, thread_name_prefix="sglang-mm-hash" + ) + self._slots = asyncio.Semaphore(max_workers) + + async def set_pad_values(self, items: list[MultimodalDataItem]) -> None: + cpu_items = [] + for item in items: + if not isinstance(item, MultimodalDataItem): + continue + feature = ( + item.feature + if item.feature is not None + else item.precomputed_embeddings + ) + if ( + item.pad_value is None + and item.hash is None + and not envs.SGLANG_MM_SKIP_COMPUTE_HASH.get() + and _is_cpu_feature(feature) + ): + cpu_items.append(item) + else: + item.set_pad_value() + + if not cpu_items: + return + async with self._slots: + future = asyncio.wrap_future( + self._executor.submit(_set_pad_values, cpu_items) + ) + cancelled = False + while not future.done(): + try: + await asyncio.shield(future) + except asyncio.CancelledError: + cancelled = True + except Exception: + break + if cancelled: + # Retrieve a worker exception as well; cancellation still wins. + if not future.cancelled(): + future.exception() + raise asyncio.CancelledError + future.result() + + def shutdown(self) -> None: + self._executor.shutdown(wait=True, cancel_futures=True) diff --git a/test/registered/unit/managers/test_mm_hashes.py b/test/registered/unit/managers/test_mm_hashes.py index 40423bb5f8..5247fa4ac9 100644 --- a/test/registered/unit/managers/test_mm_hashes.py +++ b/test/registered/unit/managers/test_mm_hashes.py @@ -13,15 +13,24 @@ the e2e serve tests; this file pins the unit-level invariants the wiring relies on. """ +import asyncio +import pickle +import threading import unittest from unittest.mock import patch +import numpy as np +import torch + +from sglang.srt.environ import envs from sglang.srt.managers.io_struct import GenerateReqInput +from sglang.srt.managers.mm_utils import hash_feature from sglang.srt.managers.schedule_batch import ( Modality, MultimodalDataItem, _compute_pad_value, ) +from sglang.srt.multimodal.processors.hash_executor import MultimodalHashExecutor from sglang.test.ci.ci_register import register_cpu_ci from sglang.test.test_utils import CustomTestCase @@ -87,5 +96,127 @@ class TestMmHashesContract(CustomTestCase): self.assertEqual(item.pad_value, _compute_pad_value(0xBBBB)) +class TestMultimodalHashExecutor(unittest.IsolatedAsyncioTestCase, CustomTestCase): + def setUp(self): + self.executor = MultimodalHashExecutor(max_workers=1) + self.addCleanup(self.executor.shutdown) + + async def test_cpu_hashes_match_scheduler_and_survive_pickle(self): + """Offloading must preserve the prefix-cache identity for every CPU layout.""" + features = [ + torch.arange(24).reshape(4, 6).t(), + torch.arange(8, dtype=torch.bfloat16), + np.arange(24).reshape(4, 6)[:, ::2], + [torch.arange(3), [torch.arange(7), torch.arange(2)]], + ] + for feature in features: + for field in ("feature", "precomputed_embeddings"): + with self.subTest(type=type(feature), field=field): + expected = MultimodalDataItem( + modality=Modality.IMAGE, **{field: feature} + ) + expected.set_pad_value() + item = MultimodalDataItem( + modality=Modality.IMAGE, **{field: feature} + ) + await self.executor.set_pad_values([item]) + received = pickle.loads(pickle.dumps(item)) + with patch( + "sglang.srt.managers.mm_utils.hash_feature", + side_effect=AssertionError("scheduler rehashed a feature"), + ): + received.set_pad_value() + self.assertEqual( + (received.hash, received.pad_value), + (expected.hash, expected.pad_value), + ) + + async def test_overrides_and_device_features_stay_on_caller_thread(self): + caller_thread = threading.get_ident() + overridden = MultimodalDataItem(modality=Modality.IMAGE, hash=1234) + padded = MultimodalDataItem(modality=Modality.IMAGE, hash=99, pad_value=1) + device_item = MultimodalDataItem( + modality=Modality.IMAGE, feature=torch.empty(1, device="meta") + ) + + def device_hash(feature): + self.assertEqual(threading.get_ident(), caller_thread) + self.assertIs(feature, device_item.feature) + return 42 + + with patch("sglang.srt.managers.mm_utils.hash_feature", device_hash): + await self.executor.set_pad_values([overridden, padded, device_item]) + self.assertEqual(overridden.hash, 1234) + self.assertEqual((padded.hash, padded.pad_value), (99, 1)) + self.assertEqual(device_item.hash, 42) + + async def test_skip_hash_preserves_existing_semantics(self): + item = MultimodalDataItem(modality=Modality.IMAGE, feature=torch.ones(4)) + with ( + envs.SGLANG_MM_SKIP_COMPUTE_HASH.override(True), + patch( + "sglang.srt.managers.mm_utils.hash_feature", + side_effect=AssertionError("skip mode read a feature"), + ), + ): + await self.executor.set_pad_values([item]) + self.assertIsNotNone(item.pad_value) + + async def test_cancelled_native_reader_keeps_capacity_until_finished(self): + """Repeated cancellation cannot admit another reader or free its input.""" + started = asyncio.Event() + release = threading.Event() + self.addCleanup(release.set) + loop = asyncio.get_running_loop() + first = MultimodalDataItem(modality=Modality.IMAGE, feature=torch.ones(4)) + second = MultimodalDataItem(modality=Modality.IMAGE, feature=torch.zeros(4)) + reads = [] + + def blocking_hash(feature): + reads.append(feature) + if feature is first.feature: + loop.call_soon_threadsafe(started.set) + if not release.wait(5): + raise TimeoutError("test did not release hash worker") + return hash_feature(feature) + + with patch("sglang.srt.managers.mm_utils.hash_feature", blocking_hash): + task = asyncio.create_task(self.executor.set_pad_values([first])) + await asyncio.wait_for(started.wait(), 3) + # Reaching here while the hash blocks also proves loop responsiveness. + task.cancel() + await asyncio.sleep(0) + task.cancel() + queued = asyncio.create_task(self.executor.set_pad_values([second])) + await asyncio.sleep(0) + self.assertFalse(task.done()) + self.assertEqual(len(reads), 1) + queued.cancel() + with self.assertRaises(asyncio.CancelledError): + await queued + release.set() + with self.assertRaises(asyncio.CancelledError): + await task + self.assertIsNone(second.hash) + await self.executor.set_pad_values([second]) + self.assertEqual(len(reads), 2) + self.assertIsNotNone(second.hash) + + async def test_error_releases_capacity_and_shutdown_rejects_new_work(self): + item = MultimodalDataItem(modality=Modality.IMAGE, feature=torch.ones(4)) + with patch( + "sglang.srt.managers.mm_utils.hash_feature", + side_effect=ValueError("invalid feature"), + ): + with self.assertRaisesRegex(ValueError, "invalid feature"): + await self.executor.set_pad_values([item]) + await asyncio.wait_for(self.executor.set_pad_values([item]), 3) + self.executor.shutdown() + self.executor.shutdown() + item.hash = item.pad_value = None + with self.assertRaisesRegex(RuntimeError, "shutdown"): + await self.executor.set_pad_values([item]) + + if __name__ == "__main__": unittest.main()