[Feature] Return output top logprobs as flat base64 arrays on request (#42890)
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
parent
baa632b049
commit
a832609b92
@@ -911,6 +911,11 @@ class ChatCompletionRequest(PDRoutingFields):
|
||||
return_output_ids_in_sglext: bool = False
|
||||
return_sampling_mask: bool = False
|
||||
sampling_logprobs_mode: Optional[Literal["selected", "support"]] = None
|
||||
# Flat output top logprobs in choices[].meta_info (requires return_meta_info);
|
||||
# choices[].logprobs top_logprobs are then empty.
|
||||
return_flat_raw_output_top_logprobs: bool = False
|
||||
# Base64 by default on chat, unlike /generate; false returns plain JSON lists.
|
||||
return_flat_raw_top_logprobs_b64: bool = True
|
||||
reasoning_effort: ReasoningEffortType = Field(
|
||||
default=None,
|
||||
description="Constrains effort on reasoning for reasoning models. "
|
||||
|
||||
@@ -1047,6 +1047,9 @@ class OpenAIServingChat(OpenAIServingBase):
|
||||
if request.return_sampling_mask and not request.return_meta_info:
|
||||
return "return_sampling_mask requires return_meta_info=true."
|
||||
|
||||
if request.return_flat_raw_output_top_logprobs and not request.return_meta_info:
|
||||
return "return_flat_raw_output_top_logprobs requires return_meta_info=true."
|
||||
|
||||
media_error = self._validate_media_content(request)
|
||||
if media_error:
|
||||
return media_error
|
||||
@@ -1298,6 +1301,12 @@ class OpenAIServingChat(OpenAIServingBase):
|
||||
sampling_logprobs_mode=request.sampling_logprobs_mode,
|
||||
stream=request.stream,
|
||||
return_text_in_logprobs=True,
|
||||
return_flat_raw_output_top_logprobs=request.return_flat_raw_output_top_logprobs,
|
||||
# GenerateReqInput rejects b64 without a flat flag; chat has only the output one
|
||||
return_flat_raw_top_logprobs_b64=(
|
||||
request.return_flat_raw_output_top_logprobs
|
||||
and request.return_flat_raw_top_logprobs_b64
|
||||
),
|
||||
modalities=processed_messages.modalities,
|
||||
lora_path=lora_path,
|
||||
bootstrap_host=request.bootstrap_host,
|
||||
|
||||
@@ -248,7 +248,9 @@ class GenerateReqInput:
|
||||
# Return prompt top logprobs as flat arrays plus shape metadata instead of
|
||||
# the nested per-position [logprob, token_id, text] lists.
|
||||
return_flat_raw_top_logprobs: bool = False
|
||||
# Base64-encode the flat arrays. Requires return_flat_raw_top_logprobs.
|
||||
# Return output top logprobs the same way, on the finished response only.
|
||||
return_flat_raw_output_top_logprobs: bool = False
|
||||
# Base64-encode the flat arrays. Requires one of the two flags above.
|
||||
return_flat_raw_top_logprobs_b64: bool = False
|
||||
# Whether to stream output.
|
||||
stream: bool = False
|
||||
@@ -459,12 +461,18 @@ class GenerateReqInput:
|
||||
"scoring: delimiter-sparse top logprob rows have no contiguous "
|
||||
"position mapping."
|
||||
)
|
||||
if (
|
||||
self.return_flat_raw_top_logprobs_b64
|
||||
and not self.return_flat_raw_top_logprobs
|
||||
if self.return_flat_raw_output_top_logprobs and self.stream:
|
||||
raise ValueError(
|
||||
"return_flat_raw_output_top_logprobs is only set on the finished "
|
||||
"response and does not support stream=True."
|
||||
)
|
||||
if self.return_flat_raw_top_logprobs_b64 and not (
|
||||
self.return_flat_raw_top_logprobs
|
||||
or self.return_flat_raw_output_top_logprobs
|
||||
):
|
||||
raise ValueError(
|
||||
"return_flat_raw_top_logprobs_b64 requires return_flat_raw_top_logprobs."
|
||||
"return_flat_raw_top_logprobs_b64 requires return_flat_raw_top_logprobs "
|
||||
"or return_flat_raw_output_top_logprobs."
|
||||
)
|
||||
|
||||
def _determine_batch_size(self):
|
||||
@@ -980,6 +988,7 @@ class GenerateReqInput:
|
||||
sampling_logprobs_mode=self.sampling_logprobs_mode[i],
|
||||
return_text_in_logprobs=self.return_text_in_logprobs,
|
||||
return_flat_raw_top_logprobs=self.return_flat_raw_top_logprobs,
|
||||
return_flat_raw_output_top_logprobs=self.return_flat_raw_output_top_logprobs,
|
||||
return_flat_raw_top_logprobs_b64=self.return_flat_raw_top_logprobs_b64,
|
||||
stream=self.stream,
|
||||
log_metrics=self.log_metrics,
|
||||
@@ -1564,34 +1573,46 @@ CachedTokensDetails = Dict[str, Union[int, str]]
|
||||
FinishReasonDict = Dict[str, Optional[Union[str, int, List[int]]]]
|
||||
|
||||
|
||||
def build_flat_input_top_logprobs_arrays(
|
||||
input_top_logprobs_val: List[Optional[List[float]]],
|
||||
input_top_logprobs_idx: List[Optional[List[int]]],
|
||||
top_logprobs_num: int,
|
||||
) -> Tuple[np.ndarray, np.ndarray, int]:
|
||||
"""Convert nested per-position prompt top logprob rows into the flat
|
||||
arrays of the `return_flat_raw_top_logprobs` response format.
|
||||
def flat_top_logprobs_layout(
|
||||
top_logprobs_val: List[Optional[List[float]]], top_logprobs_num: int
|
||||
) -> Tuple[int, int]:
|
||||
"""(null_prefix, k) of nested per-position top logprob rows in the flat format.
|
||||
|
||||
Returns (float32 values [rows, k], int32 token ids [rows, k],
|
||||
null_prefix). The leading null rows are counted into null_prefix and
|
||||
excluded from the arrays. Raises ValueError when the rows are not
|
||||
representable by (shape, null_prefix): interior nulls or ragged k,
|
||||
e.g. multi-item scoring.
|
||||
Raises ValueError when the rows are not representable by (shape,
|
||||
null_prefix): interior nulls or ragged k, e.g. multi-item scoring.
|
||||
"""
|
||||
num_rows = len(input_top_logprobs_val)
|
||||
num_rows = len(top_logprobs_val)
|
||||
null_prefix = 0
|
||||
while null_prefix < num_rows and not input_top_logprobs_val[null_prefix]:
|
||||
while null_prefix < num_rows and not top_logprobs_val[null_prefix]:
|
||||
null_prefix += 1
|
||||
val_rows = input_top_logprobs_val[null_prefix:]
|
||||
idx_rows = input_top_logprobs_idx[null_prefix:]
|
||||
val_rows = top_logprobs_val[null_prefix:]
|
||||
k = len(val_rows[0]) if val_rows else top_logprobs_num
|
||||
for offset, row in enumerate(val_rows):
|
||||
if row is None or len(row) != k:
|
||||
raise ValueError(
|
||||
"return_flat_raw_top_logprobs requires rectangular top logprob "
|
||||
f"rows with nulls only in the leading prefix; row {null_prefix + offset} "
|
||||
"Flat top logprobs require rectangular rows with nulls only in "
|
||||
f"the leading prefix; row {null_prefix + offset} "
|
||||
f"has {None if row is None else len(row)} entries (expected {k})."
|
||||
)
|
||||
return null_prefix, k
|
||||
|
||||
|
||||
def build_flat_top_logprobs_arrays(
|
||||
top_logprobs_val: List[Optional[List[float]]],
|
||||
top_logprobs_idx: List[Optional[List[int]]],
|
||||
top_logprobs_num: int,
|
||||
) -> Tuple[np.ndarray, np.ndarray, int]:
|
||||
"""Convert nested per-position top logprob rows (prompt or output) into
|
||||
the flat arrays of the flat raw top logprob response format.
|
||||
|
||||
Returns (float32 values [rows, k], int32 token ids [rows, k],
|
||||
null_prefix). The leading null rows are counted into null_prefix and
|
||||
excluded from the arrays; see `flat_top_logprobs_layout` for the rows
|
||||
that raise ValueError.
|
||||
"""
|
||||
null_prefix, k = flat_top_logprobs_layout(top_logprobs_val, top_logprobs_num)
|
||||
val_rows = top_logprobs_val[null_prefix:]
|
||||
idx_rows = top_logprobs_idx[null_prefix:]
|
||||
val_arr = np.asarray(val_rows, dtype=np.float32).reshape(len(val_rows), k)
|
||||
idx_arr = np.asarray(idx_rows, dtype=np.int32).reshape(len(idx_rows), k)
|
||||
return val_arr, idx_arr, null_prefix
|
||||
@@ -1690,7 +1711,7 @@ class BatchTokenIDOutput(BaseBatchReq, kw_only=True):
|
||||
|
||||
# Scheduler-side flat assembly of prompt top logprobs for requests with
|
||||
# return_flat_raw_top_logprobs: float32 / int32 [rows, k] arrays plus the
|
||||
# leading-null count (see build_flat_input_top_logprobs_arrays). For such
|
||||
# leading-null count (see build_flat_top_logprobs_arrays). For such
|
||||
# requests the nested input_top_logprobs_val/idx entry is empty. None when
|
||||
# no request in the batch uses the flat format.
|
||||
input_top_logprobs_val_flat: Optional[List[Optional[np.ndarray]]] = None
|
||||
|
||||
@@ -904,7 +904,7 @@ class ReqLogprob:
|
||||
input_top_logprobs_val: Optional[List[List[float]]] = None
|
||||
input_top_logprobs_idx: Optional[List[List[int]]] = None
|
||||
# Flat replacements for the rows above (see
|
||||
# build_flat_input_top_logprobs_arrays); when set, the nested rows are
|
||||
# build_flat_top_logprobs_arrays); when set, the nested rows are
|
||||
# emptied and the arrays ship instead.
|
||||
input_top_logprobs_val_flat: Optional[np.ndarray] = None
|
||||
input_top_logprobs_idx_flat: Optional[np.ndarray] = None
|
||||
|
||||
@@ -12,7 +12,7 @@ import torch
|
||||
from sglang.srt.configs.model_config import ModelConfig
|
||||
from sglang.srt.constants import MIS_DELIMITER_TOKEN_ID
|
||||
from sglang.srt.layers.logits_processor import LogitsProcessorOutput
|
||||
from sglang.srt.managers.io_struct import build_flat_input_top_logprobs_arrays
|
||||
from sglang.srt.managers.io_struct import build_flat_top_logprobs_arrays
|
||||
from sglang.srt.managers.schedule_batch import Req
|
||||
from sglang.srt.runtime_context import get_exec
|
||||
|
||||
@@ -96,7 +96,7 @@ class SchedulerLogprobResultProcessor:
|
||||
req.logprob.input_top_logprobs_val_flat,
|
||||
req.logprob.input_top_logprobs_idx_flat,
|
||||
req.logprob.input_top_logprobs_flat_null_prefix,
|
||||
) = build_flat_input_top_logprobs_arrays(
|
||||
) = build_flat_top_logprobs_arrays(
|
||||
req.logprob.input_top_logprobs_val,
|
||||
req.logprob.input_top_logprobs_idx,
|
||||
req.logprob.top_logprobs_num,
|
||||
|
||||
@@ -103,7 +103,8 @@ from sglang.srt.managers.io_struct import (
|
||||
UpdateWeightFromDiskReqOutput,
|
||||
async_sock_recv,
|
||||
async_sock_send,
|
||||
build_flat_input_top_logprobs_arrays,
|
||||
build_flat_top_logprobs_arrays,
|
||||
flat_top_logprobs_layout,
|
||||
sock_send,
|
||||
unwrap_from_pickle,
|
||||
)
|
||||
@@ -363,67 +364,70 @@ def _slice_streaming_output_meta_info(
|
||||
meta_info[key] = meta_info[key][last_output_offset:]
|
||||
|
||||
|
||||
def _build_flat_input_top_logprobs_fields(
|
||||
input_top_logprobs_val: List[Optional[List[float]]],
|
||||
input_top_logprobs_idx: List[Optional[List[int]]],
|
||||
def _build_flat_top_logprobs_fields(
|
||||
side: str,
|
||||
top_logprobs_val: List[Optional[List[float]]],
|
||||
top_logprobs_idx: List[Optional[List[int]]],
|
||||
top_logprobs_num: int,
|
||||
return_b64: bool = False,
|
||||
) -> Dict[str, Any]:
|
||||
"""Build the flat raw prompt top logprob response fields.
|
||||
"""Build the flat raw top logprob response fields under `{side}_top_logprobs_*`
|
||||
keys, where side is "input" (prompt) or "output".
|
||||
|
||||
`input_top_logprobs_shape` is the literal [rows, k] shape of the flat
|
||||
`{side}_top_logprobs_shape` is the literal [rows, k] shape of the flat
|
||||
arrays. The leading null positions (counted by
|
||||
`input_top_logprobs_null_prefix`) precede the arrays, so covered position
|
||||
`{side}_top_logprobs_null_prefix`) precede the arrays, so covered position
|
||||
i, entry j lives at flat[(i - null_prefix) * k + j] and the covered range
|
||||
spans null_prefix + rows positions. With ``return_b64``, the arrays are
|
||||
base64 contiguous little-endian binary; the dtype marker fields let the
|
||||
widths change later without a wire break.
|
||||
"""
|
||||
val_arr, idx_arr, null_prefix = build_flat_input_top_logprobs_arrays(
|
||||
input_top_logprobs_val, input_top_logprobs_idx, top_logprobs_num
|
||||
)
|
||||
if return_b64:
|
||||
return _build_flat_input_top_logprobs_fields_from_arrays(
|
||||
val_arr, idx_arr, null_prefix, return_b64=True
|
||||
val_arr, idx_arr, null_prefix = build_flat_top_logprobs_arrays(
|
||||
top_logprobs_val, top_logprobs_idx, top_logprobs_num
|
||||
)
|
||||
return _build_flat_top_logprobs_fields_from_arrays(
|
||||
side, val_arr, idx_arr, null_prefix, return_b64=True
|
||||
)
|
||||
null_prefix, k = flat_top_logprobs_layout(top_logprobs_val, top_logprobs_num)
|
||||
# Flatten the original python rows so the JSON numbers keep their full
|
||||
# (float64) precision, matching the pre-scheduler-flat output.
|
||||
# (float64) precision.
|
||||
return {
|
||||
"input_top_logprobs_val_flat": [
|
||||
v for row in input_top_logprobs_val[null_prefix:] for v in row
|
||||
f"{side}_top_logprobs_val_flat": [
|
||||
v for row in top_logprobs_val[null_prefix:] for v in row
|
||||
],
|
||||
"input_top_logprobs_idx_flat": [
|
||||
i for row in input_top_logprobs_idx[null_prefix:] for i in row
|
||||
f"{side}_top_logprobs_idx_flat": [
|
||||
i for row in top_logprobs_idx[null_prefix:] for i in row
|
||||
],
|
||||
"input_top_logprobs_shape": [val_arr.shape[0], val_arr.shape[1]],
|
||||
"input_top_logprobs_null_prefix": null_prefix,
|
||||
f"{side}_top_logprobs_shape": [len(top_logprobs_val) - null_prefix, k],
|
||||
f"{side}_top_logprobs_null_prefix": null_prefix,
|
||||
}
|
||||
|
||||
|
||||
def _build_flat_input_top_logprobs_fields_from_arrays(
|
||||
def _build_flat_top_logprobs_fields_from_arrays(
|
||||
side: str,
|
||||
val_arr: np.ndarray,
|
||||
idx_arr: np.ndarray,
|
||||
null_prefix: int,
|
||||
return_b64: bool = False,
|
||||
) -> Dict[str, Any]:
|
||||
"""Build the flat response fields from scheduler-assembled [rows, k]
|
||||
arrays (see `_build_flat_input_top_logprobs_fields` for the field
|
||||
semantics)."""
|
||||
"""Build the flat response fields from [rows, k] arrays (see
|
||||
`_build_flat_top_logprobs_fields` for the field semantics)."""
|
||||
fields: Dict[str, Any] = {}
|
||||
if return_b64:
|
||||
fields["input_top_logprobs_val_flat_b64"] = pybase64.b64encode(
|
||||
fields[f"{side}_top_logprobs_val_flat_b64"] = pybase64.b64encode(
|
||||
val_arr.tobytes()
|
||||
).decode("utf-8")
|
||||
fields["input_top_logprobs_idx_flat_b64"] = pybase64.b64encode(
|
||||
fields[f"{side}_top_logprobs_idx_flat_b64"] = pybase64.b64encode(
|
||||
idx_arr.tobytes()
|
||||
).decode("utf-8")
|
||||
fields["input_top_logprobs_val_flat_b64_dtype"] = "float32"
|
||||
fields["input_top_logprobs_idx_flat_b64_dtype"] = "int32"
|
||||
fields[f"{side}_top_logprobs_val_flat_b64_dtype"] = "float32"
|
||||
fields[f"{side}_top_logprobs_idx_flat_b64_dtype"] = "int32"
|
||||
else:
|
||||
fields["input_top_logprobs_val_flat"] = val_arr.reshape(-1).tolist()
|
||||
fields["input_top_logprobs_idx_flat"] = idx_arr.reshape(-1).tolist()
|
||||
fields["input_top_logprobs_shape"] = [val_arr.shape[0], val_arr.shape[1]]
|
||||
fields["input_top_logprobs_null_prefix"] = null_prefix
|
||||
fields[f"{side}_top_logprobs_val_flat"] = val_arr.reshape(-1).tolist()
|
||||
fields[f"{side}_top_logprobs_idx_flat"] = idx_arr.reshape(-1).tolist()
|
||||
fields[f"{side}_top_logprobs_shape"] = [val_arr.shape[0], val_arr.shape[1]]
|
||||
fields[f"{side}_top_logprobs_null_prefix"] = null_prefix
|
||||
return fields
|
||||
|
||||
|
||||
@@ -2520,6 +2524,7 @@ class TokenizerManager(TokenizerControlMixin, TokenizerManagerScoreMixin):
|
||||
state.obj.return_text_in_logprobs and not self.skip_tokenizer_init,
|
||||
recv_obj,
|
||||
i,
|
||||
finished=recv_obj.finished_reasons[i] is not None,
|
||||
)
|
||||
if (
|
||||
isinstance(state.obj, GenerateReqInput)
|
||||
@@ -2836,6 +2841,7 @@ class TokenizerManager(TokenizerControlMixin, TokenizerManagerScoreMixin):
|
||||
top_logprobs_num: int,
|
||||
token_ids_logprob: List[int],
|
||||
return_text_in_logprobs: bool,
|
||||
finished: bool,
|
||||
):
|
||||
# 1. Handle regular logprobs
|
||||
if len(state.input_token_logprobs_val) > len(state.input_token_logprobs):
|
||||
@@ -2873,7 +2879,8 @@ class TokenizerManager(TokenizerControlMixin, TokenizerManagerScoreMixin):
|
||||
state.input_top_logprobs_scheduler_flat
|
||||
)
|
||||
state.input_top_logprobs_flat_fields = (
|
||||
_build_flat_input_top_logprobs_fields_from_arrays(
|
||||
_build_flat_top_logprobs_fields_from_arrays(
|
||||
"input",
|
||||
val_arr,
|
||||
idx_arr,
|
||||
null_prefix,
|
||||
@@ -2888,7 +2895,8 @@ class TokenizerManager(TokenizerControlMixin, TokenizerManagerScoreMixin):
|
||||
):
|
||||
try:
|
||||
state.input_top_logprobs_flat_fields = (
|
||||
_build_flat_input_top_logprobs_fields(
|
||||
_build_flat_top_logprobs_fields(
|
||||
"input",
|
||||
state.input_top_logprobs_val,
|
||||
state.input_top_logprobs_idx,
|
||||
top_logprobs_num,
|
||||
@@ -2925,15 +2933,41 @@ class TokenizerManager(TokenizerControlMixin, TokenizerManagerScoreMixin):
|
||||
)
|
||||
)
|
||||
meta_info["input_top_logprobs"] = state.input_top_logprobs
|
||||
if len(state.output_top_logprobs_val) > len(state.output_top_logprobs):
|
||||
state.output_top_logprobs.extend(
|
||||
self.detokenize_top_logprobs_tokens(
|
||||
state.output_top_logprobs_val[len(state.output_top_logprobs) :],
|
||||
state.output_top_logprobs_idx[len(state.output_top_logprobs) :],
|
||||
return_text_in_logprobs,
|
||||
use_flat_output = state.obj.return_flat_raw_output_top_logprobs
|
||||
# Built once on the finished response; streaming is rejected and
|
||||
# only that response reaches a non-streaming client.
|
||||
if use_flat_output and finished:
|
||||
try:
|
||||
meta_info.update(
|
||||
_build_flat_top_logprobs_fields(
|
||||
"output",
|
||||
state.output_top_logprobs_val,
|
||||
state.output_top_logprobs_idx,
|
||||
top_logprobs_num,
|
||||
return_b64=state.obj.return_flat_raw_top_logprobs_b64,
|
||||
)
|
||||
)
|
||||
)
|
||||
meta_info["output_top_logprobs"] = state.output_top_logprobs
|
||||
except ValueError as e:
|
||||
use_flat_output = False
|
||||
logger.error(
|
||||
"Falling back to nested output top logprobs for rid=%s: %s",
|
||||
meta_info.get("id"),
|
||||
e,
|
||||
)
|
||||
if not use_flat_output:
|
||||
if len(state.output_top_logprobs_val) > len(state.output_top_logprobs):
|
||||
state.output_top_logprobs.extend(
|
||||
self.detokenize_top_logprobs_tokens(
|
||||
state.output_top_logprobs_val[
|
||||
len(state.output_top_logprobs) :
|
||||
],
|
||||
state.output_top_logprobs_idx[
|
||||
len(state.output_top_logprobs) :
|
||||
],
|
||||
return_text_in_logprobs,
|
||||
)
|
||||
)
|
||||
meta_info["output_top_logprobs"] = state.output_top_logprobs
|
||||
|
||||
# 3. Handle token_ids_logprob
|
||||
if token_ids_logprob is not None:
|
||||
@@ -2978,6 +3012,7 @@ class TokenizerManager(TokenizerControlMixin, TokenizerManagerScoreMixin):
|
||||
return_text_in_logprobs: bool,
|
||||
recv_obj: BatchStrOutput,
|
||||
recv_obj_index: int,
|
||||
finished: bool,
|
||||
):
|
||||
if (
|
||||
recv_obj.input_token_logprobs_val is not None
|
||||
@@ -3054,6 +3089,7 @@ class TokenizerManager(TokenizerControlMixin, TokenizerManagerScoreMixin):
|
||||
state.obj.top_logprobs_num,
|
||||
state.obj.token_ids_logprob,
|
||||
return_text_in_logprobs,
|
||||
finished=finished,
|
||||
)
|
||||
|
||||
def detokenize_logprob_tokens(
|
||||
@@ -3582,6 +3618,7 @@ class TokenizerManager(TokenizerControlMixin, TokenizerManagerScoreMixin):
|
||||
state.obj.token_ids_logprob,
|
||||
state.obj.return_text_in_logprobs
|
||||
and not get_serving().skip_tokenizer_init,
|
||||
finished=True,
|
||||
)
|
||||
|
||||
output_ids = state.output_ids
|
||||
|
||||
Reference in New Issue
Block a user