diff --git a/python/sglang/srt/multimodal/mm_utils.py b/python/sglang/srt/multimodal/mm_utils.py
index f57a97b41..4abf3dfb2 100644
--- a/python/sglang/srt/multimodal/mm_utils.py
+++ b/python/sglang/srt/multimodal/mm_utils.py
@@ -707,7 +707,7 @@ def run_dp_sharded_mrope_vision_model(
             out_dim = getattr(vision_model.config, "hidden_size", None)
             image_embeds_local = torch.empty(
                 (0, embed_dim_reduction_factor, out_dim),
-                device=input_device,
+                device=vision_model.device,
                 dtype=input_dtype,
             )
     else:
@@ -725,7 +725,7 @@ def run_dp_sharded_mrope_vision_model(
                 out_dim = vision_model.config.hidden_size
             image_embeds_local = torch.empty(
                 (0, out_dim),
-                device=input_device,
+                device=vision_model.device,
                 dtype=input_dtype,
             )