minke.yu/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 12 Packages Projects Releases Wiki Activity

Gate multimodal feature transport by model capability (#33653)

Browse Source
This commit is contained in:
Mohammad Miadh Angkad
2026-08-06 06:48:32 -07:00
committed by GitHub
parent e8d0fe92e9
commit 1a15cf1536
2 changed files with 59 additions and 10 deletions
Download Patch File Download Diff File Expand all files Collapse all files
python/sglang/srt/server_args.py
+15 -10
View File
@@ -2742,10 +2742,11 @@ class ServerArgs:
mm_feature_transport: A[
Optional[Literal["cpu", "cuda_ipc"]],
"Transport multimodal features through CPU memory or a bounded CUDA IPC pool. "
"Unset resolves automatically: single-node CUDA deployments (without "
"disaggregation) use cuda_ipc, everything else uses cpu. CUDA IPC reserves "
"SGLANG_MM_FEATURE_CACHE_MB (default 1024 MiB) on the base GPU and falls "
"back to CPU transport per tensor when the pool is full.",
"Unset resolves automatically: multimodal models on single-node CUDA "
"deployments (without disaggregation) use cuda_ipc, everything else uses "
"cpu. CUDA IPC reserves SGLANG_MM_FEATURE_CACHE_MB (default 1024 MiB) on "
"the base GPU and falls back to CPU transport per tensor when the pool is "
"full.",
NS("mm"),
] = None
keep_mm_feature_on_device: A[
@@ -7598,13 +7599,17 @@ class ServerArgs:
"encoder-only serving; encoder outputs use "
"--encoder-transfer-backend instead."
)
elif is_cuda() and self.nnodes == 1 and self.disaggregation_mode == "null":
elif (
self.get_model_config().is_multimodal
and is_cuda()
and self.nnodes == 1
and self.disaggregation_mode == "null"
):
# Auto policy: single-node CUDA serving defaults to the bounded
# CUDA-IPC pool. The pool is only allocated when a multimodal
# processor exists, so text-only deployments are unaffected; a
# full pool degrades to CPU transport per tensor. Multi-node
# (IPC handles are intra-node) and PD-disaggregated deployments
# keep CPU transport.
# CUDA-IPC pool for multimodal models. Text-only deployments do
# not need feature transport. Multi-node (IPC handles are
# intra-node) and PD-disaggregated deployments keep CPU transport.
# A full pool degrades to CPU transport per tensor.
requested_transport = "cuda_ipc"
logger.info(
"Multimodal feature transport auto-resolved to cuda_ipc "
test/registered/unit/server_args/test_server_args.py
+44
View File
@@ -131,6 +131,10 @@ class TestMmEncoderDataParallelLogging(CustomTestCase):
class TestMultimodalFeatureTransport(CustomTestCase):
@staticmethod
def _set_model_type(server_args, *, is_multimodal):
server_args.model_config = SimpleNamespace(is_multimodal=is_multimodal)
@patch("sglang.srt.server_args.is_cuda", return_value=True)
def test_cuda_ipc_is_explicit_and_bounded(self, _mock_is_cuda):
server_args = ServerArgs(
@@ -187,6 +191,46 @@ class TestMultimodalFeatureTransport(CustomTestCase):
self.assertEqual(server_args.mm_feature_transport, "cpu")
self.assertFalse(envs.SGLANG_USE_CUDA_IPC_TRANSPORT.get())
@patch("sglang.srt.server_args.is_cuda", return_value=True)
def test_default_transport_is_cpu_for_text_only_model(self, _mock_is_cuda):
server_args = ServerArgs(model_path="dummy")
self._set_model_type(server_args, is_multimodal=False)
with patch.dict(os.environ, {}, clear=False):
envs.SGLANG_USE_CUDA_IPC_TRANSPORT.clear()
with self.assertNoLogs(server_args_module.logger, level="INFO"):
server_args._handle_multimodal_feature_transport()
self.assertEqual(server_args.mm_feature_transport, "cpu")
self.assertFalse(envs.SGLANG_USE_CUDA_IPC_TRANSPORT.get())
@patch("sglang.srt.server_args.is_cuda", return_value=True)
def test_default_transport_is_cuda_ipc_for_multimodal_model(self, _mock_is_cuda):
server_args = ServerArgs(model_path="dummy")
self._set_model_type(server_args, is_multimodal=True)
with patch.dict(os.environ, {}, clear=False):
envs.SGLANG_USE_CUDA_IPC_TRANSPORT.clear()
with self.assertLogs(server_args_module.logger, level="INFO") as logs:
server_args._handle_multimodal_feature_transport()
self.assertEqual(server_args.mm_feature_transport, "cuda_ipc")
self.assertTrue(envs.SGLANG_USE_CUDA_IPC_TRANSPORT.get())
self.assertIn("auto-resolved to cuda_ipc", "\n".join(logs.output))
@patch("sglang.srt.server_args.is_cuda", return_value=True)
def test_default_transport_is_cuda_ipc_for_language_only_model(self, _mock_is_cuda):
server_args = ServerArgs(model_path="dummy", language_only=True)
self._set_model_type(server_args, is_multimodal=True)
with patch.dict(os.environ, {}, clear=False):
envs.SGLANG_USE_CUDA_IPC_TRANSPORT.clear()
server_args._handle_multimodal_feature_transport()
self.assertEqual(server_args.mm_feature_transport, "cuda_ipc")
self.assertTrue(envs.SGLANG_USE_CUDA_IPC_TRANSPORT.get())
@patch("sglang.srt.server_args.is_cuda", return_value=False)
def test_cuda_ipc_rejects_non_nvidia_platforms(self, _mock_is_cuda):
server_args = ServerArgs(model_path="dummy", mm_feature_transport="cuda_ipc")