feat: log multimodal encoder DP tradeoffs (#30903)
This commit is contained in:
@@ -6351,6 +6351,24 @@ class ServerArgs:
|
|||||||
|
|
||||||
run_post_process_pass(self, _data_parallelism_defaults)
|
run_post_process_pass(self, _data_parallelism_defaults)
|
||||||
|
|
||||||
|
if self.mm_enable_dp_encoder:
|
||||||
|
if self.tp_size == 1:
|
||||||
|
logger.warning(
|
||||||
|
"--mm-enable-dp-encoder is enabled with TP=1, so the encoder "
|
||||||
|
"has no data-parallel work to distribute. Disable it unless "
|
||||||
|
"you need to validate this configuration."
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
logger.info(
|
||||||
|
"--mm-enable-dp-encoder is enabled across TP=%d. It replicates "
|
||||||
|
"the vision encoder and distributes image work across ranks; "
|
||||||
|
"this is most useful when high-resolution or multi-image ViT "
|
||||||
|
"prefill is a material part of TTFT. Measure against the default "
|
||||||
|
"for small-image workloads because replication and aggregation "
|
||||||
|
"can increase memory use and overhead.",
|
||||||
|
self.tp_size,
|
||||||
|
)
|
||||||
|
|
||||||
if self._resolved().enable_dp_attention:
|
if self._resolved().enable_dp_attention:
|
||||||
self.schedule_conservativeness = self.schedule_conservativeness * 0.3
|
self.schedule_conservativeness = self.schedule_conservativeness * 0.3
|
||||||
assert self.tp_size % self.dp_size == 0
|
assert self.tp_size % self.dp_size == 0
|
||||||
|
|||||||
@@ -66,6 +66,30 @@ class TestPrepareServerArgs(CustomTestCase):
|
|||||||
os.unlink(config_file)
|
os.unlink(config_file)
|
||||||
|
|
||||||
|
|
||||||
|
class TestMmEncoderDataParallelLogging(CustomTestCase):
|
||||||
|
def test_logs_when_encoder_dp_has_no_parallelism(self):
|
||||||
|
server_args = ServerArgs(
|
||||||
|
model_path="dummy", mm_enable_dp_encoder=True, tp_size=1
|
||||||
|
)
|
||||||
|
|
||||||
|
with self.assertLogs(server_args_module.logger, level="WARNING") as logs:
|
||||||
|
server_args._handle_data_parallelism()
|
||||||
|
|
||||||
|
self.assertIn("TP=1", logs.output[0])
|
||||||
|
self.assertIn("no data-parallel work", logs.output[0])
|
||||||
|
|
||||||
|
def test_logs_encoder_dp_tradeoff_for_tp(self):
|
||||||
|
server_args = ServerArgs(
|
||||||
|
model_path="dummy", mm_enable_dp_encoder=True, tp_size=4
|
||||||
|
)
|
||||||
|
|
||||||
|
with self.assertLogs(server_args_module.logger, level="INFO") as logs:
|
||||||
|
server_args._handle_data_parallelism()
|
||||||
|
|
||||||
|
self.assertIn("TP=4", logs.output[0])
|
||||||
|
self.assertIn("high-resolution or multi-image", logs.output[0])
|
||||||
|
|
||||||
|
|
||||||
class TestMultimodalFeatureTransport(CustomTestCase):
|
class TestMultimodalFeatureTransport(CustomTestCase):
|
||||||
@patch("sglang.srt.server_args.is_cuda", return_value=True)
|
@patch("sglang.srt.server_args.is_cuda", return_value=True)
|
||||||
def test_cuda_ipc_is_explicit_and_bounded(self, _mock_is_cuda):
|
def test_cuda_ipc_is_explicit_and_bounded(self, _mock_is_cuda):
|
||||||
|
|||||||
Reference in New Issue
Block a user