[Feature] Return output top logprobs as flat base64 arrays on request (#42890)

Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
HEJIAN SANG
2026-10-13 15:58:31 -07:00
committed by GitHub
co-authored by Claude Opus 5.5
parent baa632b049
commit a832609b92
11 changed files with 372 additions and 93 deletions
@@ -911,6 +911,11 @@ class ChatCompletionRequest(PDRoutingFields):
return_output_ids_in_sglext: bool = False
return_sampling_mask: bool = False
sampling_logprobs_mode: Optional[Literal["selected", "support"]] = None
# Flat output top logprobs in choices[].meta_info (requires return_meta_info);
# choices[].logprobs top_logprobs are then empty.
return_flat_raw_output_top_logprobs: bool = False
# Base64 by default on chat, unlike /generate; false returns plain JSON lists.
return_flat_raw_top_logprobs_b64: bool = True
reasoning_effort: ReasoningEffortType = Field(
default=None,
description="Constrains effort on reasoning for reasoning models. "
@@ -1047,6 +1047,9 @@ class OpenAIServingChat(OpenAIServingBase):
if request.return_sampling_mask and not request.return_meta_info:
return "return_sampling_mask requires return_meta_info=true."
if request.return_flat_raw_output_top_logprobs and not request.return_meta_info:
return "return_flat_raw_output_top_logprobs requires return_meta_info=true."
media_error = self._validate_media_content(request)
if media_error:
return media_error
@@ -1298,6 +1301,12 @@ class OpenAIServingChat(OpenAIServingBase):
sampling_logprobs_mode=request.sampling_logprobs_mode,
stream=request.stream,
return_text_in_logprobs=True,
return_flat_raw_output_top_logprobs=request.return_flat_raw_output_top_logprobs,
# GenerateReqInput rejects b64 without a flat flag; chat has only the output one
return_flat_raw_top_logprobs_b64=(
request.return_flat_raw_output_top_logprobs
and request.return_flat_raw_top_logprobs_b64
),
modalities=processed_messages.modalities,
lora_path=lora_path,
bootstrap_host=request.bootstrap_host,
+45 -24
View File
@@ -248,7 +248,9 @@ class GenerateReqInput:
# Return prompt top logprobs as flat arrays plus shape metadata instead of
# the nested per-position [logprob, token_id, text] lists.
return_flat_raw_top_logprobs: bool = False
# Base64-encode the flat arrays. Requires return_flat_raw_top_logprobs.
# Return output top logprobs the same way, on the finished response only.
return_flat_raw_output_top_logprobs: bool = False
# Base64-encode the flat arrays. Requires one of the two flags above.
return_flat_raw_top_logprobs_b64: bool = False
# Whether to stream output.
stream: bool = False
@@ -459,12 +461,18 @@ class GenerateReqInput:
"scoring: delimiter-sparse top logprob rows have no contiguous "
"position mapping."
)
if (
self.return_flat_raw_top_logprobs_b64
and not self.return_flat_raw_top_logprobs
if self.return_flat_raw_output_top_logprobs and self.stream:
raise ValueError(
"return_flat_raw_output_top_logprobs is only set on the finished "
"response and does not support stream=True."
)
if self.return_flat_raw_top_logprobs_b64 and not (
self.return_flat_raw_top_logprobs
or self.return_flat_raw_output_top_logprobs
):
raise ValueError(
"return_flat_raw_top_logprobs_b64 requires return_flat_raw_top_logprobs."
"return_flat_raw_top_logprobs_b64 requires return_flat_raw_top_logprobs "
"or return_flat_raw_output_top_logprobs."
)
def _determine_batch_size(self):
@@ -980,6 +988,7 @@ class GenerateReqInput:
sampling_logprobs_mode=self.sampling_logprobs_mode[i],
return_text_in_logprobs=self.return_text_in_logprobs,
return_flat_raw_top_logprobs=self.return_flat_raw_top_logprobs,
return_flat_raw_output_top_logprobs=self.return_flat_raw_output_top_logprobs,
return_flat_raw_top_logprobs_b64=self.return_flat_raw_top_logprobs_b64,
stream=self.stream,
log_metrics=self.log_metrics,
@@ -1564,34 +1573,46 @@ CachedTokensDetails = Dict[str, Union[int, str]]
FinishReasonDict = Dict[str, Optional[Union[str, int, List[int]]]]
def build_flat_input_top_logprobs_arrays(
input_top_logprobs_val: List[Optional[List[float]]],
input_top_logprobs_idx: List[Optional[List[int]]],
top_logprobs_num: int,
) -> Tuple[np.ndarray, np.ndarray, int]:
"""Convert nested per-position prompt top logprob rows into the flat
arrays of the `return_flat_raw_top_logprobs` response format.
def flat_top_logprobs_layout(
top_logprobs_val: List[Optional[List[float]]], top_logprobs_num: int
) -> Tuple[int, int]:
"""(null_prefix, k) of nested per-position top logprob rows in the flat format.
Returns (float32 values [rows, k], int32 token ids [rows, k],
null_prefix). The leading null rows are counted into null_prefix and
excluded from the arrays. Raises ValueError when the rows are not
representable by (shape, null_prefix): interior nulls or ragged k,
e.g. multi-item scoring.
Raises ValueError when the rows are not representable by (shape,
null_prefix): interior nulls or ragged k, e.g. multi-item scoring.
"""
num_rows = len(input_top_logprobs_val)
num_rows = len(top_logprobs_val)
null_prefix = 0
while null_prefix < num_rows and not input_top_logprobs_val[null_prefix]:
while null_prefix < num_rows and not top_logprobs_val[null_prefix]:
null_prefix += 1
val_rows = input_top_logprobs_val[null_prefix:]
idx_rows = input_top_logprobs_idx[null_prefix:]
val_rows = top_logprobs_val[null_prefix:]
k = len(val_rows[0]) if val_rows else top_logprobs_num
for offset, row in enumerate(val_rows):
if row is None or len(row) != k:
raise ValueError(
"return_flat_raw_top_logprobs requires rectangular top logprob "
f"rows with nulls only in the leading prefix; row {null_prefix + offset} "
"Flat top logprobs require rectangular rows with nulls only in "
f"the leading prefix; row {null_prefix + offset} "
f"has {None if row is None else len(row)} entries (expected {k})."
)
return null_prefix, k
def build_flat_top_logprobs_arrays(
top_logprobs_val: List[Optional[List[float]]],
top_logprobs_idx: List[Optional[List[int]]],
top_logprobs_num: int,
) -> Tuple[np.ndarray, np.ndarray, int]:
"""Convert nested per-position top logprob rows (prompt or output) into
the flat arrays of the flat raw top logprob response format.
Returns (float32 values [rows, k], int32 token ids [rows, k],
null_prefix). The leading null rows are counted into null_prefix and
excluded from the arrays; see `flat_top_logprobs_layout` for the rows
that raise ValueError.
"""
null_prefix, k = flat_top_logprobs_layout(top_logprobs_val, top_logprobs_num)
val_rows = top_logprobs_val[null_prefix:]
idx_rows = top_logprobs_idx[null_prefix:]
val_arr = np.asarray(val_rows, dtype=np.float32).reshape(len(val_rows), k)
idx_arr = np.asarray(idx_rows, dtype=np.int32).reshape(len(idx_rows), k)
return val_arr, idx_arr, null_prefix
@@ -1690,7 +1711,7 @@ class BatchTokenIDOutput(BaseBatchReq, kw_only=True):
# Scheduler-side flat assembly of prompt top logprobs for requests with
# return_flat_raw_top_logprobs: float32 / int32 [rows, k] arrays plus the
# leading-null count (see build_flat_input_top_logprobs_arrays). For such
# leading-null count (see build_flat_top_logprobs_arrays). For such
# requests the nested input_top_logprobs_val/idx entry is empty. None when
# no request in the batch uses the flat format.
input_top_logprobs_val_flat: Optional[List[Optional[np.ndarray]]] = None
+1 -1
View File
@@ -904,7 +904,7 @@ class ReqLogprob:
input_top_logprobs_val: Optional[List[List[float]]] = None
input_top_logprobs_idx: Optional[List[List[int]]] = None
# Flat replacements for the rows above (see
# build_flat_input_top_logprobs_arrays); when set, the nested rows are
# build_flat_top_logprobs_arrays); when set, the nested rows are
# emptied and the arrays ship instead.
input_top_logprobs_val_flat: Optional[np.ndarray] = None
input_top_logprobs_idx_flat: Optional[np.ndarray] = None
@@ -12,7 +12,7 @@ import torch
from sglang.srt.configs.model_config import ModelConfig
from sglang.srt.constants import MIS_DELIMITER_TOKEN_ID
from sglang.srt.layers.logits_processor import LogitsProcessorOutput
from sglang.srt.managers.io_struct import build_flat_input_top_logprobs_arrays
from sglang.srt.managers.io_struct import build_flat_top_logprobs_arrays
from sglang.srt.managers.schedule_batch import Req
from sglang.srt.runtime_context import get_exec
@@ -96,7 +96,7 @@ class SchedulerLogprobResultProcessor:
req.logprob.input_top_logprobs_val_flat,
req.logprob.input_top_logprobs_idx_flat,
req.logprob.input_top_logprobs_flat_null_prefix,
) = build_flat_input_top_logprobs_arrays(
) = build_flat_top_logprobs_arrays(
req.logprob.input_top_logprobs_val,
req.logprob.input_top_logprobs_idx,
req.logprob.top_logprobs_num,
+78 -41
View File
@@ -103,7 +103,8 @@ from sglang.srt.managers.io_struct import (
UpdateWeightFromDiskReqOutput,
async_sock_recv,
async_sock_send,
build_flat_input_top_logprobs_arrays,
build_flat_top_logprobs_arrays,
flat_top_logprobs_layout,
sock_send,
unwrap_from_pickle,
)
@@ -363,67 +364,70 @@ def _slice_streaming_output_meta_info(
meta_info[key] = meta_info[key][last_output_offset:]
def _build_flat_input_top_logprobs_fields(
input_top_logprobs_val: List[Optional[List[float]]],
input_top_logprobs_idx: List[Optional[List[int]]],
def _build_flat_top_logprobs_fields(
side: str,
top_logprobs_val: List[Optional[List[float]]],
top_logprobs_idx: List[Optional[List[int]]],
top_logprobs_num: int,
return_b64: bool = False,
) -> Dict[str, Any]:
"""Build the flat raw prompt top logprob response fields.
"""Build the flat raw top logprob response fields under `{side}_top_logprobs_*`
keys, where side is "input" (prompt) or "output".
`input_top_logprobs_shape` is the literal [rows, k] shape of the flat
`{side}_top_logprobs_shape` is the literal [rows, k] shape of the flat
arrays. The leading null positions (counted by
`input_top_logprobs_null_prefix`) precede the arrays, so covered position
`{side}_top_logprobs_null_prefix`) precede the arrays, so covered position
i, entry j lives at flat[(i - null_prefix) * k + j] and the covered range
spans null_prefix + rows positions. With ``return_b64``, the arrays are
base64 contiguous little-endian binary; the dtype marker fields let the
widths change later without a wire break.
"""
val_arr, idx_arr, null_prefix = build_flat_input_top_logprobs_arrays(
input_top_logprobs_val, input_top_logprobs_idx, top_logprobs_num
)
if return_b64:
return _build_flat_input_top_logprobs_fields_from_arrays(
val_arr, idx_arr, null_prefix, return_b64=True
val_arr, idx_arr, null_prefix = build_flat_top_logprobs_arrays(
top_logprobs_val, top_logprobs_idx, top_logprobs_num
)
return _build_flat_top_logprobs_fields_from_arrays(
side, val_arr, idx_arr, null_prefix, return_b64=True
)
null_prefix, k = flat_top_logprobs_layout(top_logprobs_val, top_logprobs_num)
# Flatten the original python rows so the JSON numbers keep their full
# (float64) precision, matching the pre-scheduler-flat output.
# (float64) precision.
return {
"input_top_logprobs_val_flat": [
v for row in input_top_logprobs_val[null_prefix:] for v in row
f"{side}_top_logprobs_val_flat": [
v for row in top_logprobs_val[null_prefix:] for v in row
],
"input_top_logprobs_idx_flat": [
i for row in input_top_logprobs_idx[null_prefix:] for i in row
f"{side}_top_logprobs_idx_flat": [
i for row in top_logprobs_idx[null_prefix:] for i in row
],
"input_top_logprobs_shape": [val_arr.shape[0], val_arr.shape[1]],
"input_top_logprobs_null_prefix": null_prefix,
f"{side}_top_logprobs_shape": [len(top_logprobs_val) - null_prefix, k],
f"{side}_top_logprobs_null_prefix": null_prefix,
}
def _build_flat_input_top_logprobs_fields_from_arrays(
def _build_flat_top_logprobs_fields_from_arrays(
side: str,
val_arr: np.ndarray,
idx_arr: np.ndarray,
null_prefix: int,
return_b64: bool = False,
) -> Dict[str, Any]:
"""Build the flat response fields from scheduler-assembled [rows, k]
arrays (see `_build_flat_input_top_logprobs_fields` for the field
semantics)."""
"""Build the flat response fields from [rows, k] arrays (see
`_build_flat_top_logprobs_fields` for the field semantics)."""
fields: Dict[str, Any] = {}
if return_b64:
fields["input_top_logprobs_val_flat_b64"] = pybase64.b64encode(
fields[f"{side}_top_logprobs_val_flat_b64"] = pybase64.b64encode(
val_arr.tobytes()
).decode("utf-8")
fields["input_top_logprobs_idx_flat_b64"] = pybase64.b64encode(
fields[f"{side}_top_logprobs_idx_flat_b64"] = pybase64.b64encode(
idx_arr.tobytes()
).decode("utf-8")
fields["input_top_logprobs_val_flat_b64_dtype"] = "float32"
fields["input_top_logprobs_idx_flat_b64_dtype"] = "int32"
fields[f"{side}_top_logprobs_val_flat_b64_dtype"] = "float32"
fields[f"{side}_top_logprobs_idx_flat_b64_dtype"] = "int32"
else:
fields["input_top_logprobs_val_flat"] = val_arr.reshape(-1).tolist()
fields["input_top_logprobs_idx_flat"] = idx_arr.reshape(-1).tolist()
fields["input_top_logprobs_shape"] = [val_arr.shape[0], val_arr.shape[1]]
fields["input_top_logprobs_null_prefix"] = null_prefix
fields[f"{side}_top_logprobs_val_flat"] = val_arr.reshape(-1).tolist()
fields[f"{side}_top_logprobs_idx_flat"] = idx_arr.reshape(-1).tolist()
fields[f"{side}_top_logprobs_shape"] = [val_arr.shape[0], val_arr.shape[1]]
fields[f"{side}_top_logprobs_null_prefix"] = null_prefix
return fields
@@ -2520,6 +2524,7 @@ class TokenizerManager(TokenizerControlMixin, TokenizerManagerScoreMixin):
state.obj.return_text_in_logprobs and not self.skip_tokenizer_init,
recv_obj,
i,
finished=recv_obj.finished_reasons[i] is not None,
)
if (
isinstance(state.obj, GenerateReqInput)
@@ -2836,6 +2841,7 @@ class TokenizerManager(TokenizerControlMixin, TokenizerManagerScoreMixin):
top_logprobs_num: int,
token_ids_logprob: List[int],
return_text_in_logprobs: bool,
finished: bool,
):
# 1. Handle regular logprobs
if len(state.input_token_logprobs_val) > len(state.input_token_logprobs):
@@ -2873,7 +2879,8 @@ class TokenizerManager(TokenizerControlMixin, TokenizerManagerScoreMixin):
state.input_top_logprobs_scheduler_flat
)
state.input_top_logprobs_flat_fields = (
_build_flat_input_top_logprobs_fields_from_arrays(
_build_flat_top_logprobs_fields_from_arrays(
"input",
val_arr,
idx_arr,
null_prefix,
@@ -2888,7 +2895,8 @@ class TokenizerManager(TokenizerControlMixin, TokenizerManagerScoreMixin):
):
try:
state.input_top_logprobs_flat_fields = (
_build_flat_input_top_logprobs_fields(
_build_flat_top_logprobs_fields(
"input",
state.input_top_logprobs_val,
state.input_top_logprobs_idx,
top_logprobs_num,
@@ -2925,15 +2933,41 @@ class TokenizerManager(TokenizerControlMixin, TokenizerManagerScoreMixin):
)
)
meta_info["input_top_logprobs"] = state.input_top_logprobs
if len(state.output_top_logprobs_val) > len(state.output_top_logprobs):
state.output_top_logprobs.extend(
self.detokenize_top_logprobs_tokens(
state.output_top_logprobs_val[len(state.output_top_logprobs) :],
state.output_top_logprobs_idx[len(state.output_top_logprobs) :],
return_text_in_logprobs,
use_flat_output = state.obj.return_flat_raw_output_top_logprobs
# Built once on the finished response; streaming is rejected and
# only that response reaches a non-streaming client.
if use_flat_output and finished:
try:
meta_info.update(
_build_flat_top_logprobs_fields(
"output",
state.output_top_logprobs_val,
state.output_top_logprobs_idx,
top_logprobs_num,
return_b64=state.obj.return_flat_raw_top_logprobs_b64,
)
)
)
meta_info["output_top_logprobs"] = state.output_top_logprobs
except ValueError as e:
use_flat_output = False
logger.error(
"Falling back to nested output top logprobs for rid=%s: %s",
meta_info.get("id"),
e,
)
if not use_flat_output:
if len(state.output_top_logprobs_val) > len(state.output_top_logprobs):
state.output_top_logprobs.extend(
self.detokenize_top_logprobs_tokens(
state.output_top_logprobs_val[
len(state.output_top_logprobs) :
],
state.output_top_logprobs_idx[
len(state.output_top_logprobs) :
],
return_text_in_logprobs,
)
)
meta_info["output_top_logprobs"] = state.output_top_logprobs
# 3. Handle token_ids_logprob
if token_ids_logprob is not None:
@@ -2978,6 +3012,7 @@ class TokenizerManager(TokenizerControlMixin, TokenizerManagerScoreMixin):
return_text_in_logprobs: bool,
recv_obj: BatchStrOutput,
recv_obj_index: int,
finished: bool,
):
if (
recv_obj.input_token_logprobs_val is not None
@@ -3054,6 +3089,7 @@ class TokenizerManager(TokenizerControlMixin, TokenizerManagerScoreMixin):
state.obj.top_logprobs_num,
state.obj.token_ids_logprob,
return_text_in_logprobs,
finished=finished,
)
def detokenize_logprob_tokens(
@@ -3582,6 +3618,7 @@ class TokenizerManager(TokenizerControlMixin, TokenizerManagerScoreMixin):
state.obj.token_ids_logprob,
state.obj.return_text_in_logprobs
and not get_serving().skip_tokenizer_init,
finished=True,
)
output_ids = state.output_ids