diff --git a/python/sglang/srt/server_args.py b/python/sglang/srt/server_args.py index 7deebe8b3..f5d78434f 100644 --- a/python/sglang/srt/server_args.py +++ b/python/sglang/srt/server_args.py @@ -2806,10 +2806,9 @@ class ServerArgs: Optional[Literal["cpu", "cuda_ipc", "cuda_vmm"]], "Transport multimodal features through CPU memory, a bounded CUDA IPC " "pool, or a bounded CUDA VMM pool. " - "Unset resolves automatically: multimodal models on single-node CUDA " - "deployments (without disaggregation) use cuda_ipc; validated multi-node " - "GB200/GB300 MNNVL models use cuda_vmm when an IMEX channel is available; " - "all other deployments use cpu. GPU transports reserve " + "Unset uses cpu except for validated multi-node GB200/GB300 MNNVL models, " + "which use cuda_vmm when an IMEX channel is available. Select cuda_ipc " + "explicitly for single-node GPU transport. GPU transports reserve " "SGLANG_MM_FEATURE_CACHE_MB (default 1024 MiB) on the base GPU and fall " "back to CPU transport when the pool is full.", NS("mm"), @@ -7749,10 +7748,10 @@ class ServerArgs: def _handle_multimodal_feature_transport(self): """Resolve multimodal feature transport before tokenizer workers start. - GPU transports use a fixed pool on ``base_gpu_id`` and therefore reduce - the memory left for model/KV-cache allocations. The legacy CUDA IPC flag - and environment variable remain supported so existing deployments map - to this single policy. + CUDA IPC is opt-in because its fixed pool on ``base_gpu_id`` reduces the + memory left for model/KV-cache allocations. Multi-node MNNVL deployments + may still auto-select CUDA VMM. The legacy CUDA IPC flag and environment + variable remain supported so existing deployments map to this policy. """ requested_transport = self.mm_feature_transport legacy_ipc_is_set = envs.SGLANG_USE_CUDA_IPC_TRANSPORT.is_set() @@ -7792,16 +7791,12 @@ class ServerArgs: and self.disaggregation_mode == "null" ): # A full GPU pool always degrades to CPU transport per tensor. - # CUDA IPC is intra-node; multi-node auto-selection is limited - # to GB200/GB300 systems where the runtime already enables the - # MNNVL/IMEX communication stack. + # Keep CUDA IPC opt-in because even an idle pool consumes HBM + # that would otherwise back the KV cache. Multi-node + # auto-selection is limited to GB200/GB300 systems where the + # runtime already enables the MNNVL/IMEX communication stack. if self.nnodes == 1: - requested_transport = "cuda_ipc" - logger.info( - "Multimodal feature transport auto-resolved to cuda_ipc " - "(single-node CUDA). Pass --mm-feature-transport=cpu to " - "opt out." - ) + requested_transport = "cpu" elif is_mnnvl_fabric_device() and os.path.exists( "/dev/nvidia-caps-imex-channels/channel0" ): diff --git a/python/sglang/test/test_utils.py b/python/sglang/test/test_utils.py index 5ebbca864..1c65776bc 100644 --- a/python/sglang/test/test_utils.py +++ b/python/sglang/test/test_utils.py @@ -2223,12 +2223,6 @@ class ModelLaunchSettings: self.extra_args.append(fixed_arg) -class ModelEvalMetrics: - def __init__(self, accuracy: float, eval_time: float): - self.accuracy = accuracy - self.eval_time = eval_time - - def extract_trace_link_from_bench_one_batch_server_output(output: str) -> str: match = re.search(r"\[Profile\]\((.*?)\)", output) if match: diff --git a/test/registered/eval/test_vlms_mmmu_eval.py b/test/registered/eval/test_vlms_mmmu_eval.py index 939d3ed20..3b53710bd 100644 --- a/test/registered/eval/test_vlms_mmmu_eval.py +++ b/test/registered/eval/test_vlms_mmmu_eval.py @@ -8,7 +8,6 @@ from sglang.test.ci.ci_register import register_cuda_ci from sglang.test.run_eval import run_eval from sglang.test.test_utils import ( DEFAULT_URL_FOR_TEST, - ModelEvalMetrics, ModelLaunchSettings, check_evaluation_test_results, popen_launch_server, @@ -22,45 +21,34 @@ NIGHTLY_EVAL_SERVER_TIMEOUT = 1800 register_cuda_ci(est_time=7200, stage="nightly", runner_config="2-gpu-large") MODEL_THRESHOLDS = { - # Conservative thresholds on 100 MMMU samples, especially for latency thresholds - ModelLaunchSettings("deepseek-ai/deepseek-vl2-small"): ModelEvalMetrics( - 0.320, 56.1 - ), - ModelLaunchSettings("deepseek-ai/Janus-Pro-7B"): ModelEvalMetrics(0.285, 40.3), - ModelLaunchSettings("Efficient-Large-Model/NVILA-8B-hf"): ModelEvalMetrics( - 0.270, 56.7 - ), - ModelLaunchSettings("Efficient-Large-Model/NVILA-Lite-2B-hf"): ModelEvalMetrics( - 0.270, 23.8 - ), - ModelLaunchSettings("google/gemma-4-E4B-it"): ModelEvalMetrics(0.26, 15.0), - ModelLaunchSettings( - "google/gemma-4-26B-A4B-it", extra_args=["--tp=2"] - ): ModelEvalMetrics(0.27, 22.3), - ModelLaunchSettings( - "google/gemma-4-31B-it", extra_args=["--tp=2"] - ): ModelEvalMetrics(0.28, 25.5), - ModelLaunchSettings("mistral-community/pixtral-12b"): ModelEvalMetrics(0.360, 16.6), - ModelLaunchSettings("moonshotai/Kimi-VL-A3B-Instruct"): ModelEvalMetrics( - 0.330, 23.5 + # Conservative thresholds on 100 MMMU samples. Latency baselines account for + # the 1024-token CoT budget introduced in #27327; older values measured only + # 30 output tokens and are not comparable. + ModelLaunchSettings("deepseek-ai/deepseek-vl2-small"): (0.320, 56.1), + ModelLaunchSettings("deepseek-ai/Janus-Pro-7B"): (0.285, 40.3), + ModelLaunchSettings("google/gemma-4-E4B-it"): (0.26, 24.0), + ModelLaunchSettings("google/gemma-4-26B-A4B-it", extra_args=["--tp=2"]): ( + 0.27, + 32.0, ), + ModelLaunchSettings("google/gemma-4-31B-it", extra_args=["--tp=2"]): (0.28, 42.0), + # This 100-sample score has ranged from 0.33 to 0.37 since #27327. + ModelLaunchSettings("mistral-community/pixtral-12b"): (0.320, 28.0), + ModelLaunchSettings("moonshotai/Kimi-VL-A3B-Instruct"): (0.330, 23.5), # temporarily disabled: NaN in next_token_logits - # ModelLaunchSettings("openbmb/MiniCPM-o-2_6"): ModelEvalMetrics(0.330, 29.5), - # ModelLaunchSettings("openbmb/MiniCPM-v-2_6"): ModelEvalMetrics(0.259, 36.3), - ModelLaunchSettings("OpenGVLab/InternVL2_5-2B"): ModelEvalMetrics(0.300, 18.0), - ModelLaunchSettings("Qwen/Qwen2-VL-7B-Instruct"): ModelEvalMetrics(0.310, 83.3), - ModelLaunchSettings("Qwen/Qwen2.5-VL-7B-Instruct"): ModelEvalMetrics(0.330, 31.9), - ModelLaunchSettings( - "Qwen/Qwen3-VL-30B-A3B-Instruct", extra_args=["--tp=2"] - ): ModelEvalMetrics(0.29, 37.0), - ModelLaunchSettings( - "unsloth/Mistral-Small-3.1-24B-Instruct-2503" - ): ModelEvalMetrics(0.30, 16.7), - ModelLaunchSettings("XiaomiMiMo/MiMo-VL-7B-RL"): ModelEvalMetrics(0.28, 40.0), - ModelLaunchSettings("zai-org/GLM-4.1V-9B-Thinking"): ModelEvalMetrics(0.280, 30.4), - ModelLaunchSettings( - "zai-org/GLM-4.5V-FP8", extra_args=["--tp=2"] - ): ModelEvalMetrics(0.26, 34.0), + # ModelLaunchSettings("openbmb/MiniCPM-o-2_6"): (0.330, 29.5), + # ModelLaunchSettings("openbmb/MiniCPM-v-2_6"): (0.259, 36.3), + ModelLaunchSettings("OpenGVLab/InternVL2_5-2B"): (0.300, 18.0), + ModelLaunchSettings("Qwen/Qwen2-VL-7B-Instruct"): (0.310, 83.3), + ModelLaunchSettings("Qwen/Qwen2.5-VL-7B-Instruct"): (0.330, 31.9), + ModelLaunchSettings("Qwen/Qwen3-VL-30B-A3B-Instruct", extra_args=["--tp=2"]): ( + 0.29, + 37.0, + ), + ModelLaunchSettings("unsloth/Mistral-Small-3.1-24B-Instruct-2503"): (0.30, 43.0), + ModelLaunchSettings("XiaomiMiMo/MiMo-VL-7B-RL"): (0.28, 40.0), + ModelLaunchSettings("zai-org/GLM-4.1V-9B-Thinking"): (0.280, 30.4), + ModelLaunchSettings("zai-org/GLM-4.5V-FP8", extra_args=["--tp=2"]): (0.26, 140.0), } @@ -135,12 +123,12 @@ class TestNightlyVLMMmmuEval(unittest.TestCase): print(f"Error reading results: {e}") model_accuracy_thresholds = { - model.model_path: threshold.accuracy - for model, threshold in MODEL_THRESHOLDS.items() + model.model_path: accuracy + for model, (accuracy, _) in MODEL_THRESHOLDS.items() } model_latency_thresholds = { - model.model_path: threshold.eval_time - for model, threshold in MODEL_THRESHOLDS.items() + model.model_path: latency + for model, (_, latency) in MODEL_THRESHOLDS.items() } check_evaluation_test_results( all_results, diff --git a/test/registered/unit/server_args/test_server_args.py b/test/registered/unit/server_args/test_server_args.py index d902d6662..2acef3594 100644 --- a/test/registered/unit/server_args/test_server_args.py +++ b/test/registered/unit/server_args/test_server_args.py @@ -262,19 +262,17 @@ class TestMultimodalFeatureTransport(CustomTestCase): self.assertFalse(envs.SGLANG_USE_CUDA_IPC_TRANSPORT.get()) @patch("sglang.srt.server_args.is_cuda", return_value=True) - def test_default_transport_is_cuda_ipc_for_multimodal_model(self, _mock_is_cuda): + def test_default_transport_is_cpu_for_multimodal_model(self, _mock_is_cuda): server_args = ServerArgs(model_path="dummy") self._set_model_type(server_args, is_multimodal=True) with patch.dict(os.environ, {}, clear=False): envs.SGLANG_USE_CUDA_IPC_TRANSPORT.clear() - with self.assertLogs(server_args_module.logger, level="INFO") as logs: + with self.assertNoLogs(server_args_module.logger, level="INFO"): server_args._handle_multimodal_feature_transport() - self.assertEqual(server_args.mm_feature_transport, "cuda_ipc") - self.assertTrue(envs.SGLANG_USE_CUDA_IPC_TRANSPORT.get()) - - self.assertIn("auto-resolved to cuda_ipc", "\n".join(logs.output)) + self.assertEqual(server_args.mm_feature_transport, "cpu") + self.assertFalse(envs.SGLANG_USE_CUDA_IPC_TRANSPORT.get()) @patch("sglang.srt.server_args.os.path.exists", return_value=True) @patch("sglang.srt.server_args.is_mnnvl_fabric_device", return_value=True) @@ -362,7 +360,7 @@ class TestMultimodalFeatureTransport(CustomTestCase): self.assertFalse(envs.SGLANG_USE_CUDA_IPC_TRANSPORT.get()) @patch("sglang.srt.server_args.is_cuda", return_value=True) - def test_default_transport_is_cuda_ipc_for_language_only_model(self, _mock_is_cuda): + def test_default_transport_is_cpu_for_language_only_model(self, _mock_is_cuda): server_args = ServerArgs(model_path="dummy", language_only=True) self._set_model_type(server_args, is_multimodal=True) @@ -370,8 +368,8 @@ class TestMultimodalFeatureTransport(CustomTestCase): envs.SGLANG_USE_CUDA_IPC_TRANSPORT.clear() server_args._handle_multimodal_feature_transport() - self.assertEqual(server_args.mm_feature_transport, "cuda_ipc") - self.assertTrue(envs.SGLANG_USE_CUDA_IPC_TRANSPORT.get()) + self.assertEqual(server_args.mm_feature_transport, "cpu") + self.assertFalse(envs.SGLANG_USE_CUDA_IPC_TRANSPORT.get()) @patch("sglang.srt.server_args.is_cuda", return_value=False) def test_cuda_ipc_rejects_non_nvidia_platforms(self, _mock_is_cuda):