perf(multimodal): offload CPU feature hashing with bounded admission (#39539)
This commit is contained in:
@@ -1216,9 +1216,7 @@ class TokenizerManager(TokenizerControlMixin, TokenizerManagerScoreMixin):
|
||||
and mm_inputs
|
||||
and mm_inputs.mm_items
|
||||
):
|
||||
for item in mm_inputs.mm_items:
|
||||
if isinstance(item, MultimodalDataItem):
|
||||
item.set_pad_value()
|
||||
await self.mm_processor.hash_executor.set_pad_values(mm_inputs.mm_items)
|
||||
else:
|
||||
mm_inputs = None
|
||||
|
||||
|
||||
@@ -36,6 +36,7 @@ from sglang.srt.multimodal.cache import (
|
||||
build_processor_fingerprint,
|
||||
)
|
||||
from sglang.srt.multimodal.processors.executor import MultimodalProcessorExecutor
|
||||
from sglang.srt.multimodal.processors.hash_executor import MultimodalHashExecutor
|
||||
from sglang.srt.multimodal.transport.cuda_ipc import (
|
||||
MM_FEATURE_CACHE_SIZE,
|
||||
MM_ITEM_MEMORY_POOL_RECYCLE_INTERVAL,
|
||||
@@ -331,6 +332,7 @@ class BaseMultimodalProcessor(ABC):
|
||||
max_workers=self.mm_io_worker_num,
|
||||
thread_name_prefix="sglang-mm-io",
|
||||
)
|
||||
self.hash_executor = MultimodalHashExecutor()
|
||||
if self.mm_io_worker_num > 4:
|
||||
logger.info(
|
||||
"Multimodal data loading enabled with %d worker threads (%s).",
|
||||
@@ -494,6 +496,7 @@ class BaseMultimodalProcessor(ABC):
|
||||
def shutdown(self) -> None:
|
||||
"""Drop cached artifacts and stop every processor-side executor."""
|
||||
self.clear_preprocess_cache()
|
||||
self.hash_executor.shutdown()
|
||||
self.io_executor.shutdown(wait=False, cancel_futures=True)
|
||||
self.cpu_executor.shutdown(wait=False, cancel_futures=True)
|
||||
if self.mm_processor_executor is not None:
|
||||
|
||||
@@ -0,0 +1,86 @@
|
||||
"""Bounded CPU feature hashing for the tokenizer's event loop."""
|
||||
|
||||
import asyncio
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
|
||||
from sglang.srt.environ import envs
|
||||
from sglang.srt.managers.schedule_batch import MultimodalDataItem
|
||||
|
||||
|
||||
def _is_cpu_feature(value: object) -> bool:
|
||||
# Unknown transports stay on the caller thread, where their device context
|
||||
# and lifetime are established. In particular, do not reconstruct CUDA IPC.
|
||||
if isinstance(value, torch.Tensor):
|
||||
return value.device.type == "cpu"
|
||||
if isinstance(value, np.ndarray):
|
||||
return True
|
||||
if isinstance(value, list):
|
||||
return bool(value) and all(_is_cpu_feature(part) for part in value)
|
||||
return False
|
||||
|
||||
|
||||
def _set_pad_values(items: list[MultimodalDataItem]) -> None:
|
||||
for item in items:
|
||||
item.set_pad_value()
|
||||
|
||||
|
||||
class MultimodalHashExecutor:
|
||||
"""Keep at most ``max_workers`` native hash jobs in flight per processor.
|
||||
|
||||
Admission happens before submission, so the thread pool has no unbounded
|
||||
work queue. A cancelled caller retains its inputs and admission slot until
|
||||
the native reader stops, including when cancellation is repeated.
|
||||
"""
|
||||
|
||||
def __init__(self, *, max_workers: int = 2):
|
||||
self._executor = ThreadPoolExecutor(
|
||||
max_workers=max_workers, thread_name_prefix="sglang-mm-hash"
|
||||
)
|
||||
self._slots = asyncio.Semaphore(max_workers)
|
||||
|
||||
async def set_pad_values(self, items: list[MultimodalDataItem]) -> None:
|
||||
cpu_items = []
|
||||
for item in items:
|
||||
if not isinstance(item, MultimodalDataItem):
|
||||
continue
|
||||
feature = (
|
||||
item.feature
|
||||
if item.feature is not None
|
||||
else item.precomputed_embeddings
|
||||
)
|
||||
if (
|
||||
item.pad_value is None
|
||||
and item.hash is None
|
||||
and not envs.SGLANG_MM_SKIP_COMPUTE_HASH.get()
|
||||
and _is_cpu_feature(feature)
|
||||
):
|
||||
cpu_items.append(item)
|
||||
else:
|
||||
item.set_pad_value()
|
||||
|
||||
if not cpu_items:
|
||||
return
|
||||
async with self._slots:
|
||||
future = asyncio.wrap_future(
|
||||
self._executor.submit(_set_pad_values, cpu_items)
|
||||
)
|
||||
cancelled = False
|
||||
while not future.done():
|
||||
try:
|
||||
await asyncio.shield(future)
|
||||
except asyncio.CancelledError:
|
||||
cancelled = True
|
||||
except Exception:
|
||||
break
|
||||
if cancelled:
|
||||
# Retrieve a worker exception as well; cancellation still wins.
|
||||
if not future.cancelled():
|
||||
future.exception()
|
||||
raise asyncio.CancelledError
|
||||
future.result()
|
||||
|
||||
def shutdown(self) -> None:
|
||||
self._executor.shutdown(wait=True, cancel_futures=True)
|
||||
Reference in New Issue
Block a user