Use pinned memory for asynchronous sampling metadata transfers (#39777)

Co-authored-by: Xinyuan Tong <xinyuantong.cs@gmail.com>
Co-authored-by: hnyls2002 <lsyincs@gmail.com>
This commit is contained in:
Yuxuan Zhang
2026-09-20 15:31:31 -07:00
committed by GitHub
co-authored by Xinyuan Tong hnyls2002
parent 95521da18d
commit f31a7bd45c
7 changed files with 245 additions and 24 deletions
@@ -1,6 +1,7 @@
import torch
from sglang.srt.sampling.penaltylib.orchestrator import _BatchedPenalizer
from sglang.srt.utils.common import is_pin_memory_available
class BatchedFrequencyPenalizer(_BatchedPenalizer):
@@ -15,6 +16,7 @@ class BatchedFrequencyPenalizer(_BatchedPenalizer):
)
def _prepare(self):
pin_memory = is_pin_memory_available(self.orchestrator.device)
self.cumulated_frequency_penalties = torch.zeros(
(len(self.orchestrator.reqs()), self.orchestrator.vocab_size),
dtype=torch.float32,
@@ -28,9 +30,11 @@ class BatchedFrequencyPenalizer(_BatchedPenalizer):
for req in self.orchestrator.reqs()
],
dtype=torch.float32,
device=self.orchestrator.device,
pin_memory=pin_memory,
)
).unsqueeze_(1)
.to(self.orchestrator.device, non_blocking=True)
.unsqueeze_(1)
)
def _cumulate_output_tokens(self, output_ids: torch.Tensor):
self.cumulated_frequency_penalties.scatter_add_(
@@ -1,6 +1,7 @@
import torch
from sglang.srt.sampling.penaltylib.orchestrator import _BatchedPenalizer
from sglang.srt.utils.common import is_pin_memory_available
class BatchedMinNewTokensPenalizer(_BatchedPenalizer):
@@ -14,15 +15,21 @@ class BatchedMinNewTokensPenalizer(_BatchedPenalizer):
)
def _prepare(self):
self.min_new_tokens = torch.tensor(
data=[
req.sampling_params.min_new_tokens for req in self.orchestrator.reqs()
],
dtype=torch.int32,
device=self.orchestrator.device,
).unsqueeze_(1)
pin_memory = is_pin_memory_available(self.orchestrator.device)
self.min_new_tokens = (
torch.tensor(
data=[
req.sampling_params.min_new_tokens
for req in self.orchestrator.reqs()
],
dtype=torch.int32,
pin_memory=pin_memory,
)
.to(self.orchestrator.device, non_blocking=True)
.unsqueeze_(1)
)
padded_stop_token_ids = torch.nn.utils.rnn.pad_sequence(
padded_stop_token_ids_cpu = torch.nn.utils.rnn.pad_sequence(
sequences=[
torch.tensor(
data=[
@@ -40,13 +47,17 @@ class BatchedMinNewTokensPenalizer(_BatchedPenalizer):
if token_id is not None
],
dtype=torch.int64,
device=self.orchestrator.device,
)
for req in self.orchestrator.reqs()
],
batch_first=True,
padding_value=self.orchestrator.vocab_size,
)
if pin_memory:
padded_stop_token_ids_cpu = padded_stop_token_ids_cpu.pin_memory()
padded_stop_token_ids = padded_stop_token_ids_cpu.to(
self.orchestrator.device, non_blocking=True
)
self.stop_token_penalties = torch.zeros(
size=(len(self.orchestrator.reqs()), self.orchestrator.vocab_size + 1),
dtype=torch.float32,
@@ -1,6 +1,7 @@
import torch
from sglang.srt.sampling.penaltylib.orchestrator import _BatchedPenalizer
from sglang.srt.utils.common import is_pin_memory_available
class BatchedPresencePenalizer(_BatchedPenalizer):
@@ -15,6 +16,7 @@ class BatchedPresencePenalizer(_BatchedPenalizer):
)
def _prepare(self):
pin_memory = is_pin_memory_available(self.orchestrator.device)
self.cumulated_presence_penalties = torch.zeros(
(len(self.orchestrator.reqs()), self.orchestrator.vocab_size),
dtype=torch.float32,
@@ -28,9 +30,11 @@ class BatchedPresencePenalizer(_BatchedPenalizer):
for req in self.orchestrator.reqs()
],
dtype=torch.float32,
device=self.orchestrator.device,
pin_memory=pin_memory,
)
).unsqueeze_(1)
.to(self.orchestrator.device, non_blocking=True)
.unsqueeze_(1)
)
def _cumulate_output_tokens(self, output_ids: torch.Tensor):
self.cumulated_presence_penalties.scatter_(
@@ -2,6 +2,7 @@ import torch
from sglang.srt.sampling.penaltylib.orchestrator import _BatchedPenalizer
from sglang.srt.utils import get_compiler_backend, is_npu
from sglang.srt.utils.common import is_pin_memory_available
_is_npu = is_npu()
@@ -29,6 +30,7 @@ class BatchedRepetitionPenalizer(_BatchedPenalizer):
)
def _prepare(self):
pin_memory = is_pin_memory_available(self.orchestrator.device)
self.cumulated_repetition_penalties = torch.ones(
(len(self.orchestrator.reqs()), self.orchestrator.vocab_size),
dtype=torch.float32,
@@ -41,9 +43,11 @@ class BatchedRepetitionPenalizer(_BatchedPenalizer):
for req in self.orchestrator.reqs()
],
dtype=torch.float32,
device=self.orchestrator.device,
pin_memory=pin_memory,
)
).unsqueeze_(1)
.to(self.orchestrator.device, non_blocking=True)
.unsqueeze_(1)
)
def _cumulate_output_tokens(self, output_ids: torch.Tensor):
self.cumulated_repetition_penalties.scatter_(
@@ -158,10 +158,26 @@ class SamplingBatchInfo:
logit_bias = None
if any(r.sampling_params.logit_bias is not None for r in reqs):
logit_bias = torch.zeros(len(reqs), vocab_size, device=device)
rows, cols, vals = [], [], []
for i, r in enumerate(reqs):
if r.sampling_params.logit_bias is not None:
for key, value in r.sampling_params.logit_bias.items():
logit_bias[i, int(key)] = value
# Dedup on int(key) first ("1" and "01" collide): duplicate
# indices make the index_put below nondeterministic on CUDA.
row_bias = {
int(key): value
for key, value in r.sampling_params.logit_bias.items()
}
for token_id, value in row_bias.items():
rows.append(i)
cols.append(token_id)
vals.append(value)
if rows:
logit_bias[
_rows_to_device_indices(rows, device, _pin),
_rows_to_device_indices(cols, device, _pin),
] = torch.tensor(vals, dtype=logit_bias.dtype, pin_memory=_pin).to(
device, non_blocking=True
)
# Check if any request has custom logit processor
has_custom_logit_processor = (