Gate multimodal feature transport by model capability (#33653)

python/sglang/srt/server_args.py

@@ -2742,10 +2742,11 @@ class ServerArgs:
     mm_feature_transport: A[
         Optional[Literal["cpu", "cuda_ipc"]],
         "Transport multimodal features through CPU memory or a bounded CUDA IPC pool. "
-        "Unset resolves automatically: single-node CUDA deployments (without "
+        "Unset resolves automatically: multimodal models on single-node CUDA "
-        "disaggregation) use cuda_ipc, everything else uses cpu. CUDA IPC reserves "
+        "deployments (without disaggregation) use cuda_ipc, everything else uses "
-        "SGLANG_MM_FEATURE_CACHE_MB (default 1024 MiB) on the base GPU and falls "
+        "cpu. CUDA IPC reserves SGLANG_MM_FEATURE_CACHE_MB (default 1024 MiB) on "
-        "back to CPU transport per tensor when the pool is full.",
+        "the base GPU and falls back to CPU transport per tensor when the pool is "
+        "full.",
         NS("mm"),
     ] = None
     keep_mm_feature_on_device: A[
@@ -7598,13 +7599,17 @@ class ServerArgs:
                     "encoder-only serving; encoder outputs use "
                     "--encoder-transfer-backend instead."
                 )
-            elif is_cuda() and self.nnodes == 1 and self.disaggregation_mode == "null":
+            elif (
+                self.get_model_config().is_multimodal
+                and is_cuda()
+                and self.nnodes == 1
+                and self.disaggregation_mode == "null"
+            ):
                 # Auto policy: single-node CUDA serving defaults to the bounded
-                # CUDA-IPC pool. The pool is only allocated when a multimodal
+                # CUDA-IPC pool for multimodal models. Text-only deployments do
-                # processor exists, so text-only deployments are unaffected; a
+                # not need feature transport. Multi-node (IPC handles are
-                # full pool degrades to CPU transport per tensor. Multi-node
+                # intra-node) and PD-disaggregated deployments keep CPU transport.
-                # (IPC handles are intra-node) and PD-disaggregated deployments
+                # A full pool degrades to CPU transport per tensor.
-                # keep CPU transport.
                 requested_transport = "cuda_ipc"
                 logger.info(
                     "Multimodal feature transport auto-resolved to cuda_ipc "

test/registered/unit/server_args/test_server_args.py

@@ -131,6 +131,10 @@ class TestMmEncoderDataParallelLogging(CustomTestCase):
 
 
 class TestMultimodalFeatureTransport(CustomTestCase):
+    @staticmethod
+    def _set_model_type(server_args, *, is_multimodal):
+        server_args.model_config = SimpleNamespace(is_multimodal=is_multimodal)
+
     @patch("sglang.srt.server_args.is_cuda", return_value=True)
     def test_cuda_ipc_is_explicit_and_bounded(self, _mock_is_cuda):
         server_args = ServerArgs(
@@ -187,6 +191,46 @@ class TestMultimodalFeatureTransport(CustomTestCase):
             self.assertEqual(server_args.mm_feature_transport, "cpu")
             self.assertFalse(envs.SGLANG_USE_CUDA_IPC_TRANSPORT.get())
 
+    @patch("sglang.srt.server_args.is_cuda", return_value=True)
+    def test_default_transport_is_cpu_for_text_only_model(self, _mock_is_cuda):
+        server_args = ServerArgs(model_path="dummy")
+        self._set_model_type(server_args, is_multimodal=False)
+
+        with patch.dict(os.environ, {}, clear=False):
+            envs.SGLANG_USE_CUDA_IPC_TRANSPORT.clear()
+            with self.assertNoLogs(server_args_module.logger, level="INFO"):
+                server_args._handle_multimodal_feature_transport()
+
+            self.assertEqual(server_args.mm_feature_transport, "cpu")
+            self.assertFalse(envs.SGLANG_USE_CUDA_IPC_TRANSPORT.get())
+
+    @patch("sglang.srt.server_args.is_cuda", return_value=True)
+    def test_default_transport_is_cuda_ipc_for_multimodal_model(self, _mock_is_cuda):
+        server_args = ServerArgs(model_path="dummy")
+        self._set_model_type(server_args, is_multimodal=True)
+
+        with patch.dict(os.environ, {}, clear=False):
+            envs.SGLANG_USE_CUDA_IPC_TRANSPORT.clear()
+            with self.assertLogs(server_args_module.logger, level="INFO") as logs:
+                server_args._handle_multimodal_feature_transport()
+
+            self.assertEqual(server_args.mm_feature_transport, "cuda_ipc")
+            self.assertTrue(envs.SGLANG_USE_CUDA_IPC_TRANSPORT.get())
+
+        self.assertIn("auto-resolved to cuda_ipc", "\n".join(logs.output))
+
+    @patch("sglang.srt.server_args.is_cuda", return_value=True)
+    def test_default_transport_is_cuda_ipc_for_language_only_model(self, _mock_is_cuda):
+        server_args = ServerArgs(model_path="dummy", language_only=True)
+        self._set_model_type(server_args, is_multimodal=True)
+
+        with patch.dict(os.environ, {}, clear=False):
+            envs.SGLANG_USE_CUDA_IPC_TRANSPORT.clear()
+            server_args._handle_multimodal_feature_transport()
+
+            self.assertEqual(server_args.mm_feature_transport, "cuda_ipc")
+            self.assertTrue(envs.SGLANG_USE_CUDA_IPC_TRANSPORT.get())
+
     @patch("sglang.srt.server_args.is_cuda", return_value=False)
     def test_cuda_ipc_rejects_non_nvidia_platforms(self, _mock_is_cuda):
         server_args = ServerArgs(model_path="dummy", mm_feature_transport="cuda_ipc")