[Sampling] Add selected/support sampling logprob modes (#40932)

This commit is contained in:
Nan Jiang
2026-09-23 21:38:19 -07:00
committed by GitHub
parent 8eedf616f5
commit c3685dff42
26 changed files with 572 additions and 84 deletions
+10 -3
View File
@@ -2396,9 +2396,16 @@ class DecodeTransferQueue(DecodeHiCacheTransferMixin):
decode_req.req.output_token_sampling_mask.append(
output_token_sampling_mask_idx[:sampling_mask_len].cpu().tolist()
)
decode_req.req.output_token_sampling_logprobs.append(
float(output_token_sampling_logprobs[0].item())
)
if decode_req.req.sampling_logprobs_mode == "support":
decode_req.req.output_token_sampling_logprobs.append(
output_token_sampling_logprobs[:sampling_mask_len]
.cpu()
.tolist()
)
else:
decode_req.req.output_token_sampling_logprobs.append(
float(output_token_sampling_logprobs[0].item())
)
decode_req.kv_receiver.clear()
decode_req.kv_receiver = None
+21 -6
View File
@@ -349,7 +349,9 @@ class MetadataBuffers:
(size, max_sampling_mask_tokens), dtype=torch.int32, device=device
)
self.output_token_sampling_logprobs = torch.zeros(
(size, 16), dtype=torch.float32, device=device
(size, max_sampling_mask_tokens),
dtype=torch.float32,
device=device,
)
# For PD + spec decode
self.output_topk_p = torch.zeros(
@@ -517,8 +519,10 @@ class MetadataBuffers:
sampling_logprobs = req.output_token_sampling_logprobs
if sampling_masks:
sampling_mask = sampling_masks[0]
sampling_logprob = sampling_logprobs[0] if sampling_logprobs else None
if sampling_mask is not None and sampling_logprob is not None:
sampling_logprobs_row = (
sampling_logprobs[0] if sampling_logprobs else None
)
if sampling_mask is not None and sampling_logprobs_row is not None:
mask_len = len(sampling_mask)
max_mask_len = self.output_token_sampling_mask_idx.shape[1]
if mask_len > max_mask_len:
@@ -540,9 +544,20 @@ class MetadataBuffers:
device=self.output_token_sampling_mask_idx.device,
)
)
self.output_token_sampling_logprobs[req.metadata_buffer_index][
0
] = float(sampling_logprob)
if req.sampling_logprobs_mode == "support" and mask_len:
self.output_token_sampling_logprobs[
req.metadata_buffer_index, :mask_len
].copy_(
torch.tensor(
sampling_logprobs_row,
dtype=torch.float32,
device=self.output_token_sampling_logprobs.device,
)
)
elif req.sampling_logprobs_mode == "selected":
self.output_token_sampling_logprobs[
req.metadata_buffer_index, 0
] = float(sampling_logprobs_row)
# For PD + spec decode
if req.hidden_states_tensor is not None:
# speculative_eagle_topk should not be greater than 16 currently
@@ -894,6 +894,7 @@ class ChatCompletionRequest(BaseModel):
return_input_ids_in_sglext: bool = False
return_output_ids_in_sglext: bool = False
return_sampling_mask: bool = False
sampling_logprobs_mode: Optional[Literal["selected", "support"]] = None
reasoning_effort: ReasoningEffortType = Field(
default=None,
description="Constrains effort on reasoning for reasoning models. "
@@ -1239,6 +1239,7 @@ class OpenAIServingChat(OpenAIServingBase):
logprob_start_len=-1,
top_logprobs_num=request.top_logprobs or 0,
return_sampling_mask=request.return_sampling_mask,
sampling_logprobs_mode=request.sampling_logprobs_mode,
stream=request.stream,
return_text_in_logprobs=True,
modalities=processed_messages.modalities,
+9 -4
View File
@@ -93,17 +93,20 @@ class SamplingMaskStatus(IntEnum):
@dataclasses.dataclass
class SamplingMaskOutput:
"""Tensor result for opted-in rows in batch order."""
"""Sampling-support IDs and optional full-support behavior logprobs."""
token_ids: torch.Tensor
lengths: torch.Tensor
selected_logprobs: torch.Tensor
support_logprobs: Optional[torch.Tensor]
statuses: torch.Tensor
def map_device_tensors(self, fn) -> None:
self.token_ids = fn(self.token_ids)
self.lengths = fn(self.lengths)
self.selected_logprobs = fn(self.selected_logprobs)
if self.support_logprobs is not None:
self.support_logprobs = fn(self.support_logprobs)
self.statuses = fn(self.statuses)
@@ -223,11 +226,13 @@ class LogitsProcessorOutput:
List[Union[List[float], torch.Tensor]]
] = None
next_token_token_ids_logprobs_idx: Optional[List] = None
# Post-filter support IDs, bounded by server capacity, and selected-token
# logprob over the full realized support.
# Post-filter support IDs and requested behavior logprobs, bounded by server
# capacity. Logprobs are normalized over the full realized support.
sampling_mask_output: Optional[SamplingMaskOutput] = None
next_token_sampling_mask_idx: Optional[List[Optional[List[int]]]] = None
next_token_sampling_logprobs: Optional[List[Optional[float]]] = None
next_token_sampling_logprobs: Optional[
List[Optional[Union[float, List[float]]]]
] = None
next_token_sampling_mask_status: Optional[List[Optional[int]]] = None
## Part 3: Prefill-only. This part will be assigned in python/sglang/srt/layers/logits_processor.py::LogitsProcessor
+28 -2
View File
@@ -176,6 +176,9 @@ class Sampler(nn.Module):
_trace_e2e_sampler("preprocess_returned")
sampling_mask_batch_indices = sampling_info.sampling_mask_batch_indices
return_sampling_mask = sampling_mask_batch_indices is not None
sampling_support_logprobs_capture_indices = (
sampling_info.sampling_support_logprobs_capture_indices
)
sampling_mask_capture = None
if sampling_info.is_all_greedy:
@@ -296,7 +299,9 @@ class Sampler(nn.Module):
if sampling_info.is_all_greedy:
logits_output.sampling_mask_output = (
self._build_greedy_sampling_mask_output(
sampling_mask_batch_indices, batch_next_token_ids
sampling_mask_batch_indices,
batch_next_token_ids,
sampling_support_logprobs_capture_indices,
)
)
else:
@@ -309,6 +314,7 @@ class Sampler(nn.Module):
logits_output.sampling_mask_output = self._build_sampling_mask_output(
batch_next_token_ids,
sampling_mask_capture,
sampling_support_logprobs_capture_indices,
)
_trace_e2e_sampler("forward_returned")
@@ -461,6 +467,7 @@ class Sampler(nn.Module):
self,
batch_indices: torch.Tensor,
batch_next_token_ids: torch.Tensor,
support_capture_indices: Optional[torch.Tensor],
) -> SamplingMaskOutput:
token_ids = batch_next_token_ids.index_select(0, batch_indices).to(torch.int32)
num_requests = batch_indices.numel()
@@ -472,6 +479,15 @@ class Sampler(nn.Module):
selected_logprobs=torch.zeros(
num_requests, dtype=torch.float32, device=token_ids.device
),
support_logprobs=(
torch.zeros(
(support_capture_indices.numel(), 1),
dtype=torch.float32,
device=token_ids.device,
)
if support_capture_indices is not None
else None
),
statuses=torch.full(
(num_requests,),
SamplingMaskStatus.OK,
@@ -484,6 +500,7 @@ class Sampler(nn.Module):
self,
batch_next_token_ids: torch.Tensor,
sampling_mask_capture: _SamplingMaskCapture,
support_capture_indices: Optional[torch.Tensor],
) -> SamplingMaskOutput:
"""Pack captured positive support into the fixed-cap device result."""
batch_indices = sampling_mask_capture.batch_rows
@@ -549,18 +566,27 @@ class Sampler(nn.Module):
packed_size = min(self.sampling_mask_max_tokens, weights.shape[-1])
if token_ids is None:
_, packed_positions = torch.topk(
packed_weights, packed_positions = torch.topk(
weights, k=packed_size, dim=-1, largest=True, sorted=True
)
packed_token_ids = packed_positions.to(torch.int32)
else:
# The PyTorch producer already sorts weights and IDs together.
packed_token_ids = token_ids[:, :packed_size].contiguous()
packed_weights = weights[:, :packed_size]
support_logprobs = None
if support_capture_indices is not None:
support_logprobs = torch.log(
packed_weights.index_select(0, support_capture_indices).float()
/ support_mass.index_select(0, support_capture_indices).unsqueeze(-1)
)
return SamplingMaskOutput(
token_ids=packed_token_ids,
lengths=realized_lengths.clamp(max=packed_size),
selected_logprobs=selected_logprobs,
support_logprobs=support_logprobs,
statuses=statuses,
)
+24 -10
View File
@@ -59,6 +59,7 @@ from sglang.srt.managers.schedule_batch import (
Modality,
MultimodalProcessorOutput,
ReturnHiddenStatesMode,
SamplingLogprobsMode,
get_return_hidden_states_mode,
)
from sglang.srt.multimodal.mm_utils import has_valid_data
@@ -232,8 +233,14 @@ class GenerateReqInput:
top_logprobs_num: Optional[Union[List[int], int]] = None
# If return logprobs, the token ids to return logprob for.
token_ids_logprob: Optional[Union[List[List[int]], List[int]]] = None
# Whether to return output-token sampling support and renormalized logprobs.
# Whether to return each output token's sampling support and behavior logprob.
return_sampling_mask: Optional[Union[List[bool], bool]] = None
# Return either the selected token's behavior logprob or behavior logprobs for
# the full sampling support. When omitted, selected mode is used if
# return_sampling_mask is enabled.
sampling_logprobs_mode: Optional[
Union[List[Optional[SamplingLogprobsMode]], SamplingLogprobsMode]
] = None
# Whether to detokenize tokens in text in the returned logprobs.
return_text_in_logprobs: bool = False
# Return prompt top logprobs as flat arrays plus shape metadata instead of
@@ -768,6 +775,9 @@ class GenerateReqInput:
self.return_sampling_mask = normalize_param(
self.return_sampling_mask, False, "return_sampling_mask"
)
self.sampling_logprobs_mode = normalize_param(
self.sampling_logprobs_mode, None, "sampling_logprobs_mode"
)
# Handle token_ids_logprob specially due to its nested structure
if not self.token_ids_logprob: # covers both None and []
@@ -948,6 +958,7 @@ class GenerateReqInput:
top_logprobs_num=self.top_logprobs_num[i],
token_ids_logprob=self.token_ids_logprob[i],
return_sampling_mask=self.return_sampling_mask[i],
sampling_logprobs_mode=self.sampling_logprobs_mode[i],
return_text_in_logprobs=self.return_text_in_logprobs,
return_flat_raw_top_logprobs=self.return_flat_raw_top_logprobs,
return_flat_raw_top_logprobs_b64=self.return_flat_raw_top_logprobs_b64,
@@ -1121,8 +1132,11 @@ class TokenizedGenerateReqInput(BaseReq, kw_only=True):
kv_hints: Optional[KvHintsEnvelope] = None
# Internal PP control bit, set by PP0 before forwarding the request.
# Keep at the end to preserve the positional Rust wire schema.
# Keep tail fields append-only to preserve the positional Rust wire schema.
pp_prefetch_ticketed: bool = False
# Shape of output_token_sampling_logprobs for each output token. This is a
# defaulted tail field so older IPC senders decode as selected mode.
sampling_logprobs_mode: SamplingLogprobsMode = "selected"
def wrap_pickle_fields(self):
self.time_stats = wrap_as_pickle(self.time_stats)
@@ -1514,10 +1528,10 @@ class BatchTokenIDOutput(BaseBatchReq, kw_only=True):
output_token_entropy_val: Optional[List[Optional[float]]]
# Per-request chunks of output-token sampling supports. None when no request
# in the batch asks for return_sampling_mask.
output_token_sampling_mask: Optional[List[List]]
# Per-request chunks of selected-token logprobs renormalized over the
# corresponding sampling supports. None when sampling masks are not returned.
output_token_sampling_logprobs: Optional[List[List]]
output_token_sampling_mask: Optional[List[List[List[int]]]]
# Per-request chunks. Each output-token entry is a selected-token scalar or
# a list aligned with output_token_sampling_mask, according to the request.
output_token_sampling_logprobs: Optional[List[List[Union[float, List[float]]]]]
# Hidden states
output_hidden_states: OutputHiddenStates
@@ -1612,10 +1626,10 @@ class BatchStrOutput(BaseBatchReq, kw_only=True):
output_token_ids_logprobs_val: TokenIdsLogprobValues
output_token_ids_logprobs_idx: TokenIdsLogprobIndices
output_token_entropy_val: Optional[List[Optional[float]]]
# Detokenizer pass-through for BatchTokenIDOutput.output_token_sampling_*.
# None when sampling masks are not returned.
output_token_sampling_mask: Optional[List[List]]
output_token_sampling_logprobs: Optional[List[List]]
# Detokenizer pass-through for BatchTokenIDOutput.output_token_sampling_*;
# support-mode logprobs are aligned elementwise with the token IDs.
output_token_sampling_mask: Optional[List[List[List[int]]]]
output_token_sampling_logprobs: Optional[List[List[Union[float, List[float]]]]]
# Hidden states
output_hidden_states: OutputHiddenStates
@@ -165,6 +165,7 @@ logger = logging.getLogger(__name__)
ReturnHiddenStatesMode = Union[bool, Literal["last"]]
SamplingLogprobsMode = Literal["selected", "support"]
def get_return_hidden_states_mode(
@@ -989,6 +990,7 @@ class Req(ReqDllmMixin):
dllm_config: Optional[DllmConfig] = None,
token_ids_logprob: List[int] = None,
return_sampling_mask: bool = False,
sampling_logprobs_mode: SamplingLogprobsMode = "selected",
return_flat_raw_top_logprobs: bool = False,
stream: bool = False,
origin_input_ids_unpadded: Optional[array[int]] = None,
@@ -1226,6 +1228,7 @@ class Req(ReqDllmMixin):
self.temp_scaled_logprobs = False
self.top_p_normalized_logprobs = False
self.return_sampling_mask = return_sampling_mask
self.sampling_logprobs_mode = sampling_logprobs_mode
self.return_flat_raw_top_logprobs = return_flat_raw_top_logprobs
# Logprobs (return values)
+1
View File
@@ -2762,6 +2762,7 @@ class Scheduler(
top_logprobs_num=recv_req.top_logprobs_num,
token_ids_logprob=recv_req.token_ids_logprob,
return_sampling_mask=recv_req.return_sampling_mask,
sampling_logprobs_mode=recv_req.sampling_logprobs_mode,
return_flat_raw_top_logprobs=recv_req.return_flat_raw_top_logprobs,
stream=recv_req.stream,
lora_id=recv_req.lora_id,
@@ -1191,8 +1191,17 @@ class SchedulerBatchResultProcessor:
logprobs = [None] * batch_size
status_by_batch = [None] * batch_size
token_ids = sampling_output.token_ids.cpu()
support_logprobs = (
None
if sampling_output.support_logprobs is None
else sampling_output.support_logprobs.cpu()
)
packed_width = token_ids.shape[1]
support_row = 0
for row, batch_index in enumerate(batch_indices):
returns_support_logprobs = (
reqs[batch_index].sampling_logprobs_mode == "support"
)
status = int(statuses[row])
length = int(lengths[row])
if status == SamplingMaskStatus.OK and not (0 <= length <= packed_width):
@@ -1200,7 +1209,14 @@ class SchedulerBatchResultProcessor:
status_by_batch[batch_index] = status
if status == SamplingMaskStatus.OK:
masks[batch_index] = token_ids[row, :length].tolist()
logprobs[batch_index] = float(selected_logprobs[row])
if returns_support_logprobs:
logprobs[batch_index] = support_logprobs[
support_row, :length
].tolist()
else:
logprobs[batch_index] = float(selected_logprobs[row])
if returns_support_logprobs:
support_row += 1
output.next_token_sampling_mask_idx = masks
output.next_token_sampling_logprobs = logprobs
@@ -374,8 +374,8 @@ class _GenerationStreamAccumulator:
input_token_ids_logprobs_idx: Optional[list] = None
output_token_ids_logprobs_val: Optional[list] = None
output_token_ids_logprobs_idx: Optional[list] = None
output_token_sampling_mask: Optional[list] = None
output_token_sampling_logprobs: Optional[list] = None
output_token_sampling_mask: Optional[list[list[list[int]]]] = None
output_token_sampling_logprobs: Optional[list[list[float | list[float]]]] = None
# Rust server mode: the Rust detokenizer reconstructs text/ids from the raw
# output tokens itself and never consumes the scheduler's incremental-detok
# offsets (decode_ids / read_offset), so that per-step bookkeeping is skipped.
@@ -289,8 +289,12 @@ class ReqState:
input_token_ids_logprobs_idx: List = dataclasses.field(default_factory=list)
output_token_ids_logprobs_val: List = dataclasses.field(default_factory=list)
output_token_ids_logprobs_idx: List = dataclasses.field(default_factory=list)
output_token_sampling_mask: List = dataclasses.field(default_factory=list)
output_token_sampling_logprobs: List = dataclasses.field(default_factory=list)
output_token_sampling_mask: List[List[int]] = dataclasses.field(
default_factory=list
)
output_token_sampling_logprobs: List[Union[float, List[float]]] = dataclasses.field(
default_factory=list
)
# Cached flat-format prompt top logprob fields; rebuilt only when more
# prefill chunks arrive, so streaming decode chunks reuse the payload.
@@ -1377,6 +1381,11 @@ class TokenizerManager(TokenizerControlMixin, TokenizerManagerScoreMixin):
"return_sampling_mask only supports DisallowedTokensLogitsProcessor "
"among custom logit processors."
)
if obj.sampling_logprobs_mode is not None and not obj.return_sampling_mask:
raise ValueError(
"sampling_logprobs_mode can only be set when "
"return_sampling_mask=true."
)
def _validate_mm_limits(
self, obj: Union[GenerateReqInput, EmbeddingReqInput]
@@ -1494,6 +1503,7 @@ class TokenizerManager(TokenizerControlMixin, TokenizerManagerScoreMixin):
top_logprobs_num=obj.top_logprobs_num,
token_ids_logprob=obj.token_ids_logprob,
return_sampling_mask=obj.return_sampling_mask,
sampling_logprobs_mode=obj.sampling_logprobs_mode or "selected",
return_flat_raw_top_logprobs=obj.return_flat_raw_top_logprobs,
stream=obj.stream,
rid=obj.rid,
+6
View File
@@ -304,6 +304,11 @@ def get_logprob_dict_from_result(result: GenerationBatchResult) -> dict:
if sampling_mask_output is None
else sampling_mask_output.selected_logprobs
),
"sampling_mask_support_logprobs": (
None
if sampling_mask_output is None
else sampling_mask_output.support_logprobs
),
"sampling_mask_statuses": (
None if sampling_mask_output is None else sampling_mask_output.statuses
),
@@ -325,6 +330,7 @@ def get_logprob_from_pp_outputs(
token_ids=next_pp_outputs["sampling_mask_token_ids"],
lengths=next_pp_outputs["sampling_mask_lengths"],
selected_logprobs=next_pp_outputs["sampling_mask_selected_logprobs"],
support_logprobs=next_pp_outputs["sampling_mask_support_logprobs"],
statuses=next_pp_outputs["sampling_mask_statuses"],
)
logits_output = LogitsProcessorOutput(
@@ -95,7 +95,9 @@ class SamplingBatchInfo:
# CPU request flags and their derived device indices. The flags make batch
# filtering cheap; the indices keep sampler work limited to opted-in rows.
return_sampling_masks: Optional[List[bool]] = None
return_sampling_support_logprobs: Optional[List[bool]] = None
sampling_mask_batch_indices: Optional[torch.Tensor] = None
sampling_support_logprobs_capture_indices: Optional[torch.Tensor] = None
# Device
device: str = "cuda"
@@ -185,9 +187,20 @@ class SamplingBatchInfo:
and any(r.custom_logit_processor for r in reqs) # check the flag first.
) # then check the requests.
return_sampling_masks = [r.return_sampling_mask for r in reqs]
return_sampling_support_logprobs = [
r.return_sampling_mask and r.sampling_logprobs_mode == "support"
for r in reqs
]
sampling_mask_batch_indices = cls._make_sampling_mask_batch_indices(
return_sampling_masks, device
)
sampling_support_logprobs_capture_indices = (
cls._make_sampling_support_logprobs_capture_indices(
return_sampling_masks,
return_sampling_support_logprobs,
device,
)
)
if has_custom_logit_processor:
# Merge the same type of custom logit processors together
@@ -253,7 +266,11 @@ class SamplingBatchInfo:
device=device,
logit_bias=logit_bias,
return_sampling_masks=return_sampling_masks,
return_sampling_support_logprobs=return_sampling_support_logprobs,
sampling_mask_batch_indices=sampling_mask_batch_indices,
sampling_support_logprobs_capture_indices=(
sampling_support_logprobs_capture_indices
),
)
ret.adjusted_from_schedule_batch(batch, vocab_size)
return ret
@@ -284,6 +301,30 @@ class SamplingBatchInfo:
return None
return _rows_to_device_indices(indices, device)
@staticmethod
def _make_sampling_support_logprobs_capture_indices(
return_sampling_masks: List[bool],
return_sampling_support_logprobs: Optional[List[bool]],
device: str,
) -> Optional[torch.Tensor]:
"""Map support-mode requests to rows in the compact mask capture."""
if return_sampling_support_logprobs is None:
return None
capture_index = -1
indices = []
for return_mask, return_support in zip(
return_sampling_masks,
return_sampling_support_logprobs,
strict=True,
):
if return_mask:
capture_index += 1
if return_support:
indices.append(capture_index)
if not indices:
return None
return _rows_to_device_indices(indices, device)
def __len__(self):
return len(self.temperatures)
@@ -394,12 +435,23 @@ class SamplingBatchInfo:
self.return_sampling_masks = [
self.return_sampling_masks[i] for i in keep_indices
]
if self.return_sampling_support_logprobs is not None:
self.return_sampling_support_logprobs = [
self.return_sampling_support_logprobs[i] for i in keep_indices
]
if self.sampling_mask_batch_indices is not None:
self.sampling_mask_batch_indices = (
self._make_sampling_mask_batch_indices(
self.return_sampling_masks, self.device
)
)
self.sampling_support_logprobs_capture_indices = (
self._make_sampling_support_logprobs_capture_indices(
self.return_sampling_masks,
self.return_sampling_support_logprobs,
self.device,
)
)
self.adjusted_filter_batch(keep_indices, keep_indices_device)
@@ -475,6 +527,16 @@ class SamplingBatchInfo:
self.return_sampling_masks = (
self.return_sampling_masks or [False] * self_len
) + (other.return_sampling_masks or [False] * other_len)
self.return_sampling_support_logprobs = (
self.return_sampling_support_logprobs or [False] * self_len
) + (other.return_sampling_support_logprobs or [False] * other_len)
self.sampling_support_logprobs_capture_indices = (
self._make_sampling_support_logprobs_capture_indices(
self.return_sampling_masks,
self.return_sampling_support_logprobs,
self.device,
)
)
# Note: because the __len()__ operator is defined on the temperatures tensor,
# please make sure any merge operation with len(self) or len(other) is done before
@@ -310,6 +310,7 @@ class Session:
top_logprobs_num=req.top_logprobs_num,
token_ids_logprob=req.token_ids_logprob,
return_sampling_mask=req.return_sampling_mask,
sampling_logprobs_mode=req.sampling_logprobs_mode,
vocab_size=vocab_size,
eos_token_ids=eos_token_ids,
require_reasoning=req.require_reasoning,