minke.yu/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 13 Packages Projects Releases Wiki Activity

transfer mrope_position_delta to device when first running (#11047)

Browse Source
This commit is contained in:
ash-sigh
2025-10-26 13:06:09 -07:00
committed by GitHub
parent a1d5bc4cce
commit 0b3b3e9a69
1 changed files with 9 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files
python/sglang/srt/model_executor/forward_batch_info.py
+7 -1
View File
@@ -575,9 +575,15 @@ class ForwardBatch:
device=model_runner.device, device=model_runner.device,
) )
else: else:
mrope_position_deltas = mm_input.mrope_position_delta.flatten().to( if mm_input.mrope_position_delta.device.type != model_runner.device:
# transfer mrope_position_delta to device when the first running,
# avoiding successvie host-to-device data transfer
mm_input.mrope_position_delta = (
mm_input.mrope_position_delta.to(
model_runner.device, non_blocking=True model_runner.device, non_blocking=True
) )
)
mrope_position_deltas = mm_input.mrope_position_delta.flatten()
mrope_positions_list[batch_idx] = ( mrope_positions_list[batch_idx] = (
(mrope_position_deltas + self.seq_lens[batch_idx] - 1) (mrope_position_deltas + self.seq_lens[batch_idx] - 1)
.unsqueeze(0) .unsqueeze(0)