diff --git a/docs/docs/basic_usage/openai_api_completions.mdx b/docs/docs/basic_usage/openai_api_completions.mdx index be4e68c3d1..1da3493f81 100644 --- a/docs/docs/basic_usage/openai_api_completions.mdx +++ b/docs/docs/basic_usage/openai_api_completions.mdx @@ -365,6 +365,37 @@ for chunk in stream: For MoE models, set `return_routed_experts: true` in `extra_body` to return expert routing data. Requires `--enable-return-routed-experts` server flag. By default, `routed_experts` is returned in the response-level `sglext` object as base64-encoded int32 expert IDs in a flattened array with logical shape `[num_tokens, num_layers, top_k]`. For non-streaming chat responses with `return_meta_info: true`, `routed_experts` is returned only in each choice's `meta_info` object and omitted from response-level `sglext` to avoid duplicating the routing data. Other requested extension fields, such as `cached_tokens_details` and `spec_tokens_details`, remain in response-level `sglext`. By default, routed experts cover `[0, seqlen - 1)`, the full available sequence, because RL workflows need them for the full sequence. Set `routed_experts_start_len` in `extra_body` to an absolute prefix length to return only `[routed_experts_start_len, seqlen - 1)`. For example, in multi-turn RL rollouts, routed experts for tokens from previous turns have already been collected, so setting this value avoids unnecessary transfer that causes bottlenecks. +#### Returning Flat Output Top Logprobs + +With `logprobs: true` and `top_logprobs: k`, set `return_flat_raw_output_top_logprobs: true` and `return_meta_info: true` in `extra_body` to receive the output top logprobs as flat arrays in each choice's `meta_info` instead of nested objects. This skips detokenizing every candidate and shrinks the response, which matters for large `k`. The arrays arrive with the finished response only, so `stream: true` is not supported. `choices[].logprobs` still carries the selected-token logprobs, but its `top_logprobs` lists are empty. + +`meta_info` then carries `output_top_logprobs_shape` (`[rows, k]`) and `output_top_logprobs_null_prefix`; covered position `i`, rank `j` sits at flat index `(i - null_prefix) * k + j`. On chat completions the arrays are base64 by default: `output_top_logprobs_{val,idx}_flat_b64`, little-endian bytes whose dtypes are given by the `_dtype` fields (float32 and int32), so non-finite logprobs such as `-inf` survive. Set `return_flat_raw_top_logprobs_b64: false` for plain JSON lists in `output_top_logprobs_{val,idx}_flat`. The native `/generate` API takes the same fields, with base64 off by default. + +```python +import base64 + +import numpy as np + +response = client.chat.completions.create( + model="qwen/qwen2.5-0.5b-instruct", + messages=[{"role": "user", "content": "Say this is a test"}], + max_tokens=16, + logprobs=True, + top_logprobs=5, + extra_body={"return_meta_info": True, "return_flat_raw_output_top_logprobs": True}, +) +meta = response.choices[0].meta_info +shape = meta["output_top_logprobs_shape"] +val = np.frombuffer( + base64.b64decode(meta["output_top_logprobs_val_flat_b64"]), + dtype=meta["output_top_logprobs_val_flat_b64_dtype"], +).reshape(shape) +idx = np.frombuffer( + base64.b64decode(meta["output_top_logprobs_idx_flat_b64"]), + dtype=meta["output_top_logprobs_idx_flat_b64_dtype"], +).reshape(shape) +``` + ```python Example # Example with logit_bias parameter for completions API # Note: You need to get the actual token IDs from your tokenizer diff --git a/python/sglang/srt/entrypoints/openai/protocol.py b/python/sglang/srt/entrypoints/openai/protocol.py index 136abac376..3bb5af6a06 100644 --- a/python/sglang/srt/entrypoints/openai/protocol.py +++ b/python/sglang/srt/entrypoints/openai/protocol.py @@ -911,6 +911,11 @@ class ChatCompletionRequest(PDRoutingFields): return_output_ids_in_sglext: bool = False return_sampling_mask: bool = False sampling_logprobs_mode: Optional[Literal["selected", "support"]] = None + # Flat output top logprobs in choices[].meta_info (requires return_meta_info); + # choices[].logprobs top_logprobs are then empty. + return_flat_raw_output_top_logprobs: bool = False + # Base64 by default on chat, unlike /generate; false returns plain JSON lists. + return_flat_raw_top_logprobs_b64: bool = True reasoning_effort: ReasoningEffortType = Field( default=None, description="Constrains effort on reasoning for reasoning models. " diff --git a/python/sglang/srt/entrypoints/openai/serving_chat.py b/python/sglang/srt/entrypoints/openai/serving_chat.py index 9034dfa192..4b361f28f7 100644 --- a/python/sglang/srt/entrypoints/openai/serving_chat.py +++ b/python/sglang/srt/entrypoints/openai/serving_chat.py @@ -1047,6 +1047,9 @@ class OpenAIServingChat(OpenAIServingBase): if request.return_sampling_mask and not request.return_meta_info: return "return_sampling_mask requires return_meta_info=true." + if request.return_flat_raw_output_top_logprobs and not request.return_meta_info: + return "return_flat_raw_output_top_logprobs requires return_meta_info=true." + media_error = self._validate_media_content(request) if media_error: return media_error @@ -1298,6 +1301,12 @@ class OpenAIServingChat(OpenAIServingBase): sampling_logprobs_mode=request.sampling_logprobs_mode, stream=request.stream, return_text_in_logprobs=True, + return_flat_raw_output_top_logprobs=request.return_flat_raw_output_top_logprobs, + # GenerateReqInput rejects b64 without a flat flag; chat has only the output one + return_flat_raw_top_logprobs_b64=( + request.return_flat_raw_output_top_logprobs + and request.return_flat_raw_top_logprobs_b64 + ), modalities=processed_messages.modalities, lora_path=lora_path, bootstrap_host=request.bootstrap_host, diff --git a/python/sglang/srt/managers/io_struct.py b/python/sglang/srt/managers/io_struct.py index 6b6d37116f..e32fc01140 100644 --- a/python/sglang/srt/managers/io_struct.py +++ b/python/sglang/srt/managers/io_struct.py @@ -248,7 +248,9 @@ class GenerateReqInput: # Return prompt top logprobs as flat arrays plus shape metadata instead of # the nested per-position [logprob, token_id, text] lists. return_flat_raw_top_logprobs: bool = False - # Base64-encode the flat arrays. Requires return_flat_raw_top_logprobs. + # Return output top logprobs the same way, on the finished response only. + return_flat_raw_output_top_logprobs: bool = False + # Base64-encode the flat arrays. Requires one of the two flags above. return_flat_raw_top_logprobs_b64: bool = False # Whether to stream output. stream: bool = False @@ -459,12 +461,18 @@ class GenerateReqInput: "scoring: delimiter-sparse top logprob rows have no contiguous " "position mapping." ) - if ( - self.return_flat_raw_top_logprobs_b64 - and not self.return_flat_raw_top_logprobs + if self.return_flat_raw_output_top_logprobs and self.stream: + raise ValueError( + "return_flat_raw_output_top_logprobs is only set on the finished " + "response and does not support stream=True." + ) + if self.return_flat_raw_top_logprobs_b64 and not ( + self.return_flat_raw_top_logprobs + or self.return_flat_raw_output_top_logprobs ): raise ValueError( - "return_flat_raw_top_logprobs_b64 requires return_flat_raw_top_logprobs." + "return_flat_raw_top_logprobs_b64 requires return_flat_raw_top_logprobs " + "or return_flat_raw_output_top_logprobs." ) def _determine_batch_size(self): @@ -980,6 +988,7 @@ class GenerateReqInput: sampling_logprobs_mode=self.sampling_logprobs_mode[i], return_text_in_logprobs=self.return_text_in_logprobs, return_flat_raw_top_logprobs=self.return_flat_raw_top_logprobs, + return_flat_raw_output_top_logprobs=self.return_flat_raw_output_top_logprobs, return_flat_raw_top_logprobs_b64=self.return_flat_raw_top_logprobs_b64, stream=self.stream, log_metrics=self.log_metrics, @@ -1564,34 +1573,46 @@ CachedTokensDetails = Dict[str, Union[int, str]] FinishReasonDict = Dict[str, Optional[Union[str, int, List[int]]]] -def build_flat_input_top_logprobs_arrays( - input_top_logprobs_val: List[Optional[List[float]]], - input_top_logprobs_idx: List[Optional[List[int]]], - top_logprobs_num: int, -) -> Tuple[np.ndarray, np.ndarray, int]: - """Convert nested per-position prompt top logprob rows into the flat - arrays of the `return_flat_raw_top_logprobs` response format. +def flat_top_logprobs_layout( + top_logprobs_val: List[Optional[List[float]]], top_logprobs_num: int +) -> Tuple[int, int]: + """(null_prefix, k) of nested per-position top logprob rows in the flat format. - Returns (float32 values [rows, k], int32 token ids [rows, k], - null_prefix). The leading null rows are counted into null_prefix and - excluded from the arrays. Raises ValueError when the rows are not - representable by (shape, null_prefix): interior nulls or ragged k, - e.g. multi-item scoring. + Raises ValueError when the rows are not representable by (shape, + null_prefix): interior nulls or ragged k, e.g. multi-item scoring. """ - num_rows = len(input_top_logprobs_val) + num_rows = len(top_logprobs_val) null_prefix = 0 - while null_prefix < num_rows and not input_top_logprobs_val[null_prefix]: + while null_prefix < num_rows and not top_logprobs_val[null_prefix]: null_prefix += 1 - val_rows = input_top_logprobs_val[null_prefix:] - idx_rows = input_top_logprobs_idx[null_prefix:] + val_rows = top_logprobs_val[null_prefix:] k = len(val_rows[0]) if val_rows else top_logprobs_num for offset, row in enumerate(val_rows): if row is None or len(row) != k: raise ValueError( - "return_flat_raw_top_logprobs requires rectangular top logprob " - f"rows with nulls only in the leading prefix; row {null_prefix + offset} " + "Flat top logprobs require rectangular rows with nulls only in " + f"the leading prefix; row {null_prefix + offset} " f"has {None if row is None else len(row)} entries (expected {k})." ) + return null_prefix, k + + +def build_flat_top_logprobs_arrays( + top_logprobs_val: List[Optional[List[float]]], + top_logprobs_idx: List[Optional[List[int]]], + top_logprobs_num: int, +) -> Tuple[np.ndarray, np.ndarray, int]: + """Convert nested per-position top logprob rows (prompt or output) into + the flat arrays of the flat raw top logprob response format. + + Returns (float32 values [rows, k], int32 token ids [rows, k], + null_prefix). The leading null rows are counted into null_prefix and + excluded from the arrays; see `flat_top_logprobs_layout` for the rows + that raise ValueError. + """ + null_prefix, k = flat_top_logprobs_layout(top_logprobs_val, top_logprobs_num) + val_rows = top_logprobs_val[null_prefix:] + idx_rows = top_logprobs_idx[null_prefix:] val_arr = np.asarray(val_rows, dtype=np.float32).reshape(len(val_rows), k) idx_arr = np.asarray(idx_rows, dtype=np.int32).reshape(len(idx_rows), k) return val_arr, idx_arr, null_prefix @@ -1690,7 +1711,7 @@ class BatchTokenIDOutput(BaseBatchReq, kw_only=True): # Scheduler-side flat assembly of prompt top logprobs for requests with # return_flat_raw_top_logprobs: float32 / int32 [rows, k] arrays plus the - # leading-null count (see build_flat_input_top_logprobs_arrays). For such + # leading-null count (see build_flat_top_logprobs_arrays). For such # requests the nested input_top_logprobs_val/idx entry is empty. None when # no request in the batch uses the flat format. input_top_logprobs_val_flat: Optional[List[Optional[np.ndarray]]] = None diff --git a/python/sglang/srt/managers/schedule_batch.py b/python/sglang/srt/managers/schedule_batch.py index fe7ac1bfd1..6bc0dbf6a4 100755 --- a/python/sglang/srt/managers/schedule_batch.py +++ b/python/sglang/srt/managers/schedule_batch.py @@ -904,7 +904,7 @@ class ReqLogprob: input_top_logprobs_val: Optional[List[List[float]]] = None input_top_logprobs_idx: Optional[List[List[int]]] = None # Flat replacements for the rows above (see - # build_flat_input_top_logprobs_arrays); when set, the nested rows are + # build_flat_top_logprobs_arrays); when set, the nested rows are # emptied and the arrays ship instead. input_top_logprobs_val_flat: Optional[np.ndarray] = None input_top_logprobs_idx_flat: Optional[np.ndarray] = None diff --git a/python/sglang/srt/managers/scheduler_components/logprob_result_processor.py b/python/sglang/srt/managers/scheduler_components/logprob_result_processor.py index e55906c49f..86e1a7bfe7 100644 --- a/python/sglang/srt/managers/scheduler_components/logprob_result_processor.py +++ b/python/sglang/srt/managers/scheduler_components/logprob_result_processor.py @@ -12,7 +12,7 @@ import torch from sglang.srt.configs.model_config import ModelConfig from sglang.srt.constants import MIS_DELIMITER_TOKEN_ID from sglang.srt.layers.logits_processor import LogitsProcessorOutput -from sglang.srt.managers.io_struct import build_flat_input_top_logprobs_arrays +from sglang.srt.managers.io_struct import build_flat_top_logprobs_arrays from sglang.srt.managers.schedule_batch import Req from sglang.srt.runtime_context import get_exec @@ -96,7 +96,7 @@ class SchedulerLogprobResultProcessor: req.logprob.input_top_logprobs_val_flat, req.logprob.input_top_logprobs_idx_flat, req.logprob.input_top_logprobs_flat_null_prefix, - ) = build_flat_input_top_logprobs_arrays( + ) = build_flat_top_logprobs_arrays( req.logprob.input_top_logprobs_val, req.logprob.input_top_logprobs_idx, req.logprob.top_logprobs_num, diff --git a/python/sglang/srt/managers/tokenizer_manager.py b/python/sglang/srt/managers/tokenizer_manager.py index 2b2c405fd4..3dd350f411 100644 --- a/python/sglang/srt/managers/tokenizer_manager.py +++ b/python/sglang/srt/managers/tokenizer_manager.py @@ -103,7 +103,8 @@ from sglang.srt.managers.io_struct import ( UpdateWeightFromDiskReqOutput, async_sock_recv, async_sock_send, - build_flat_input_top_logprobs_arrays, + build_flat_top_logprobs_arrays, + flat_top_logprobs_layout, sock_send, unwrap_from_pickle, ) @@ -363,67 +364,70 @@ def _slice_streaming_output_meta_info( meta_info[key] = meta_info[key][last_output_offset:] -def _build_flat_input_top_logprobs_fields( - input_top_logprobs_val: List[Optional[List[float]]], - input_top_logprobs_idx: List[Optional[List[int]]], +def _build_flat_top_logprobs_fields( + side: str, + top_logprobs_val: List[Optional[List[float]]], + top_logprobs_idx: List[Optional[List[int]]], top_logprobs_num: int, return_b64: bool = False, ) -> Dict[str, Any]: - """Build the flat raw prompt top logprob response fields. + """Build the flat raw top logprob response fields under `{side}_top_logprobs_*` + keys, where side is "input" (prompt) or "output". - `input_top_logprobs_shape` is the literal [rows, k] shape of the flat + `{side}_top_logprobs_shape` is the literal [rows, k] shape of the flat arrays. The leading null positions (counted by - `input_top_logprobs_null_prefix`) precede the arrays, so covered position + `{side}_top_logprobs_null_prefix`) precede the arrays, so covered position i, entry j lives at flat[(i - null_prefix) * k + j] and the covered range spans null_prefix + rows positions. With ``return_b64``, the arrays are base64 contiguous little-endian binary; the dtype marker fields let the widths change later without a wire break. """ - val_arr, idx_arr, null_prefix = build_flat_input_top_logprobs_arrays( - input_top_logprobs_val, input_top_logprobs_idx, top_logprobs_num - ) if return_b64: - return _build_flat_input_top_logprobs_fields_from_arrays( - val_arr, idx_arr, null_prefix, return_b64=True + val_arr, idx_arr, null_prefix = build_flat_top_logprobs_arrays( + top_logprobs_val, top_logprobs_idx, top_logprobs_num ) + return _build_flat_top_logprobs_fields_from_arrays( + side, val_arr, idx_arr, null_prefix, return_b64=True + ) + null_prefix, k = flat_top_logprobs_layout(top_logprobs_val, top_logprobs_num) # Flatten the original python rows so the JSON numbers keep their full - # (float64) precision, matching the pre-scheduler-flat output. + # (float64) precision. return { - "input_top_logprobs_val_flat": [ - v for row in input_top_logprobs_val[null_prefix:] for v in row + f"{side}_top_logprobs_val_flat": [ + v for row in top_logprobs_val[null_prefix:] for v in row ], - "input_top_logprobs_idx_flat": [ - i for row in input_top_logprobs_idx[null_prefix:] for i in row + f"{side}_top_logprobs_idx_flat": [ + i for row in top_logprobs_idx[null_prefix:] for i in row ], - "input_top_logprobs_shape": [val_arr.shape[0], val_arr.shape[1]], - "input_top_logprobs_null_prefix": null_prefix, + f"{side}_top_logprobs_shape": [len(top_logprobs_val) - null_prefix, k], + f"{side}_top_logprobs_null_prefix": null_prefix, } -def _build_flat_input_top_logprobs_fields_from_arrays( +def _build_flat_top_logprobs_fields_from_arrays( + side: str, val_arr: np.ndarray, idx_arr: np.ndarray, null_prefix: int, return_b64: bool = False, ) -> Dict[str, Any]: - """Build the flat response fields from scheduler-assembled [rows, k] - arrays (see `_build_flat_input_top_logprobs_fields` for the field - semantics).""" + """Build the flat response fields from [rows, k] arrays (see + `_build_flat_top_logprobs_fields` for the field semantics).""" fields: Dict[str, Any] = {} if return_b64: - fields["input_top_logprobs_val_flat_b64"] = pybase64.b64encode( + fields[f"{side}_top_logprobs_val_flat_b64"] = pybase64.b64encode( val_arr.tobytes() ).decode("utf-8") - fields["input_top_logprobs_idx_flat_b64"] = pybase64.b64encode( + fields[f"{side}_top_logprobs_idx_flat_b64"] = pybase64.b64encode( idx_arr.tobytes() ).decode("utf-8") - fields["input_top_logprobs_val_flat_b64_dtype"] = "float32" - fields["input_top_logprobs_idx_flat_b64_dtype"] = "int32" + fields[f"{side}_top_logprobs_val_flat_b64_dtype"] = "float32" + fields[f"{side}_top_logprobs_idx_flat_b64_dtype"] = "int32" else: - fields["input_top_logprobs_val_flat"] = val_arr.reshape(-1).tolist() - fields["input_top_logprobs_idx_flat"] = idx_arr.reshape(-1).tolist() - fields["input_top_logprobs_shape"] = [val_arr.shape[0], val_arr.shape[1]] - fields["input_top_logprobs_null_prefix"] = null_prefix + fields[f"{side}_top_logprobs_val_flat"] = val_arr.reshape(-1).tolist() + fields[f"{side}_top_logprobs_idx_flat"] = idx_arr.reshape(-1).tolist() + fields[f"{side}_top_logprobs_shape"] = [val_arr.shape[0], val_arr.shape[1]] + fields[f"{side}_top_logprobs_null_prefix"] = null_prefix return fields @@ -2520,6 +2524,7 @@ class TokenizerManager(TokenizerControlMixin, TokenizerManagerScoreMixin): state.obj.return_text_in_logprobs and not self.skip_tokenizer_init, recv_obj, i, + finished=recv_obj.finished_reasons[i] is not None, ) if ( isinstance(state.obj, GenerateReqInput) @@ -2836,6 +2841,7 @@ class TokenizerManager(TokenizerControlMixin, TokenizerManagerScoreMixin): top_logprobs_num: int, token_ids_logprob: List[int], return_text_in_logprobs: bool, + finished: bool, ): # 1. Handle regular logprobs if len(state.input_token_logprobs_val) > len(state.input_token_logprobs): @@ -2873,7 +2879,8 @@ class TokenizerManager(TokenizerControlMixin, TokenizerManagerScoreMixin): state.input_top_logprobs_scheduler_flat ) state.input_top_logprobs_flat_fields = ( - _build_flat_input_top_logprobs_fields_from_arrays( + _build_flat_top_logprobs_fields_from_arrays( + "input", val_arr, idx_arr, null_prefix, @@ -2888,7 +2895,8 @@ class TokenizerManager(TokenizerControlMixin, TokenizerManagerScoreMixin): ): try: state.input_top_logprobs_flat_fields = ( - _build_flat_input_top_logprobs_fields( + _build_flat_top_logprobs_fields( + "input", state.input_top_logprobs_val, state.input_top_logprobs_idx, top_logprobs_num, @@ -2925,15 +2933,41 @@ class TokenizerManager(TokenizerControlMixin, TokenizerManagerScoreMixin): ) ) meta_info["input_top_logprobs"] = state.input_top_logprobs - if len(state.output_top_logprobs_val) > len(state.output_top_logprobs): - state.output_top_logprobs.extend( - self.detokenize_top_logprobs_tokens( - state.output_top_logprobs_val[len(state.output_top_logprobs) :], - state.output_top_logprobs_idx[len(state.output_top_logprobs) :], - return_text_in_logprobs, + use_flat_output = state.obj.return_flat_raw_output_top_logprobs + # Built once on the finished response; streaming is rejected and + # only that response reaches a non-streaming client. + if use_flat_output and finished: + try: + meta_info.update( + _build_flat_top_logprobs_fields( + "output", + state.output_top_logprobs_val, + state.output_top_logprobs_idx, + top_logprobs_num, + return_b64=state.obj.return_flat_raw_top_logprobs_b64, + ) ) - ) - meta_info["output_top_logprobs"] = state.output_top_logprobs + except ValueError as e: + use_flat_output = False + logger.error( + "Falling back to nested output top logprobs for rid=%s: %s", + meta_info.get("id"), + e, + ) + if not use_flat_output: + if len(state.output_top_logprobs_val) > len(state.output_top_logprobs): + state.output_top_logprobs.extend( + self.detokenize_top_logprobs_tokens( + state.output_top_logprobs_val[ + len(state.output_top_logprobs) : + ], + state.output_top_logprobs_idx[ + len(state.output_top_logprobs) : + ], + return_text_in_logprobs, + ) + ) + meta_info["output_top_logprobs"] = state.output_top_logprobs # 3. Handle token_ids_logprob if token_ids_logprob is not None: @@ -2978,6 +3012,7 @@ class TokenizerManager(TokenizerControlMixin, TokenizerManagerScoreMixin): return_text_in_logprobs: bool, recv_obj: BatchStrOutput, recv_obj_index: int, + finished: bool, ): if ( recv_obj.input_token_logprobs_val is not None @@ -3054,6 +3089,7 @@ class TokenizerManager(TokenizerControlMixin, TokenizerManagerScoreMixin): state.obj.top_logprobs_num, state.obj.token_ids_logprob, return_text_in_logprobs, + finished=finished, ) def detokenize_logprob_tokens( @@ -3582,6 +3618,7 @@ class TokenizerManager(TokenizerControlMixin, TokenizerManagerScoreMixin): state.obj.token_ids_logprob, state.obj.return_text_in_logprobs and not get_serving().skip_tokenizer_init, + finished=True, ) output_ids = state.output_ids diff --git a/rust/sglang-processor/src/openai/chat.rs b/rust/sglang-processor/src/openai/chat.rs index d28f278be7..6bedac9744 100644 --- a/rust/sglang-processor/src/openai/chat.rs +++ b/rust/sglang-processor/src/openai/chat.rs @@ -109,6 +109,7 @@ const UNSUPPORTED_FIELDS: &[&str] = &[ "return_input_ids_in_sglext", "return_output_ids_in_sglext", "return_sampling_mask", + "return_flat_raw_output_top_logprobs", "sampling_logprobs_mode", "video_config", "max_dynamic_patch", diff --git a/test/registered/unit/api/v1/test_parity.py b/test/registered/unit/api/v1/test_parity.py index e48e3fc258..1b2b7a8ad2 100644 --- a/test/registered/unit/api/v1/test_parity.py +++ b/test/registered/unit/api/v1/test_parity.py @@ -74,6 +74,7 @@ PYTHON_ONLY_REQUEST_FIELDS = frozenset( "require_reasoning", "return_bytes", "return_entropy", + "return_flat_raw_output_top_logprobs", "return_flat_raw_top_logprobs", "return_flat_raw_top_logprobs_b64", "return_indexer_topk", diff --git a/test/registered/unit/entrypoints/openai/test_serving_chat.py b/test/registered/unit/entrypoints/openai/test_serving_chat.py index e471a93bf5..81b931f5f4 100644 --- a/test/registered/unit/entrypoints/openai/test_serving_chat.py +++ b/test/registered/unit/entrypoints/openai/test_serving_chat.py @@ -759,6 +759,58 @@ class ServingChatTestCase(CustomTestCase): "return_sampling_mask requires return_meta_info=true.", ) + def test_validate_request_rejects_flat_output_top_logprobs_without_meta_info( + self, + ): + req = ChatCompletionRequest( + model="x", + messages=[{"role": "user", "content": "Hi?"}], + logprobs=True, + top_logprobs=2, + return_flat_raw_output_top_logprobs=True, + ) + + self.assertEqual( + self.chat._validate_request(req), + "return_flat_raw_output_top_logprobs requires return_meta_info=true.", + ) + + def test_convert_to_internal_request_flat_output_top_logprobs_default_to_b64(self): + self.tm.tokenizer = None + # (request flags, expected flat output, expected b64) + cases = ( + ({"return_flat_raw_output_top_logprobs": True}, True, True), + ( + { + "return_flat_raw_output_top_logprobs": True, + "return_flat_raw_top_logprobs_b64": False, + }, + True, + False, + ), + ({}, False, False), + ) + for flags, flat, b64 in cases: + req = ChatCompletionRequest( + model="x", + messages=[{"role": "user", "content": "Hi?"}], + input_ids=[101, 102, 103], + logprobs=True, + top_logprobs=2, + return_meta_info=True, + **flags, + ) + with self.subTest(flags=flags): + adapted, _ = self.chat._convert_to_internal_request( + req, self.fastapi_request + ) + # The b64 default must not trip GenerateReqInput validation. + adapted.normalize_batch_and_arguments() + + self.assertEqual(adapted.top_logprobs_num, 2) + self.assertEqual(adapted.return_flat_raw_output_top_logprobs, flat) + self.assertEqual(adapted.return_flat_raw_top_logprobs_b64, b64) + def test_convert_to_internal_request_rejects_stream_return_meta_info(self): req = ChatCompletionRequest( model="x", diff --git a/test/registered/unit/managers/test_flat_raw_top_logprobs.py b/test/registered/unit/managers/test_flat_raw_top_logprobs.py index e12e5e1fc6..e739a46aab 100644 --- a/test/registered/unit/managers/test_flat_raw_top_logprobs.py +++ b/test/registered/unit/managers/test_flat_raw_top_logprobs.py @@ -1,6 +1,4 @@ -"""Unit tests for the flat raw prompt top logprob response format -(`return_flat_raw_top_logprobs` / `return_flat_raw_top_logprobs_b64`). -""" +"""Unit tests for the flat raw top logprob response format (prompt and output).""" import asyncio import base64 @@ -20,7 +18,7 @@ from sglang.srt import runtime_context as rc from sglang.srt.managers.io_struct import ( BatchTokenIDOutput, GenerateReqInput, - build_flat_input_top_logprobs_arrays, + build_flat_top_logprobs_arrays, msgpack_decode, msgpack_encode, ) @@ -31,8 +29,8 @@ from sglang.srt.managers.scheduler_components.logprob_result_processor import ( from sglang.srt.managers.tokenizer_manager import ( ReqState, TokenizerManager, - _build_flat_input_top_logprobs_fields, - _build_flat_input_top_logprobs_fields_from_arrays, + _build_flat_top_logprobs_fields, + _build_flat_top_logprobs_fields_from_arrays, ) from sglang.srt.observability.req_time_stats import APIServerReqTimeStats from sglang.srt.sampling.sampling_params import SamplingParams @@ -80,6 +78,7 @@ def _add_logprob_meta_info(state: ReqState, top_logprobs_num: int = 2) -> dict: top_logprobs_num=top_logprobs_num, token_ids_logprob=None, return_text_in_logprobs=False, + finished=False, ) return meta_info @@ -126,11 +125,29 @@ class TestFlatRawTopLogprobsValidation(CustomTestCase): for i in range(2): self.assertTrue(req[i].return_flat_raw_top_logprobs_b64) + def test_output_flag_enables_b64_and_propagates_to_batch_items(self): + req = GenerateReqInput( + text=["a", "b"], + return_flat_raw_output_top_logprobs=True, + return_flat_raw_top_logprobs_b64=True, + ) + req.normalize_batch_and_arguments() + for i in range(2): + self.assertTrue(req[i].return_flat_raw_output_top_logprobs) + self.assertTrue(req[i].return_flat_raw_top_logprobs_b64) + + def test_output_flag_rejects_stream(self): + req = GenerateReqInput( + text="hello", stream=True, return_flat_raw_output_top_logprobs=True + ) + with self.assertRaisesRegex(ValueError, "stream"): + req.normalize_batch_and_arguments() + class TestFlatAssembly(CustomTestCase): def test_flat_matches_nested_rows(self): - fields = _build_flat_input_top_logprobs_fields( - _VAL_ROWS, _IDX_ROWS, top_logprobs_num=2 + fields = _build_flat_top_logprobs_fields( + "input", _VAL_ROWS, _IDX_ROWS, top_logprobs_num=2 ) self.assertEqual(fields["input_top_logprobs_shape"], [3, 2]) self.assertEqual(fields["input_top_logprobs_null_prefix"], 1) @@ -153,8 +170,8 @@ class TestFlatAssembly(CustomTestCase): self.assertEqual(flat_val[start : start + k], _VAL_ROWS[i]) def test_b64_roundtrip(self): - fields = _build_flat_input_top_logprobs_fields( - _VAL_ROWS, _IDX_ROWS, top_logprobs_num=2, return_b64=True + fields = _build_flat_top_logprobs_fields( + "input", _VAL_ROWS, _IDX_ROWS, top_logprobs_num=2, return_b64=True ) self.assertEqual(fields["input_top_logprobs_shape"], [3, 2]) self.assertEqual(fields["input_top_logprobs_null_prefix"], 1) @@ -173,8 +190,8 @@ class TestFlatAssembly(CustomTestCase): np.testing.assert_array_equal(idx, np.asarray(_IDX_ROWS[1:], dtype=np.int32)) def test_all_null_rows(self): - fields = _build_flat_input_top_logprobs_fields( - [None], [None], top_logprobs_num=2 + fields = _build_flat_top_logprobs_fields( + "input", [None], [None], top_logprobs_num=2 ) self.assertEqual(fields["input_top_logprobs_shape"], [0, 2]) self.assertEqual(fields["input_top_logprobs_null_prefix"], 1) @@ -183,7 +200,8 @@ class TestFlatAssembly(CustomTestCase): def test_rejects_null_row_after_prefix(self): with self.assertRaisesRegex(ValueError, "leading prefix"): - _build_flat_input_top_logprobs_fields( + _build_flat_top_logprobs_fields( + "input", [None, [-0.1, -2.5], None, [-0.3, -1.5]], [None, [11, 22], None, [33, 44]], top_logprobs_num=2, @@ -191,7 +209,8 @@ class TestFlatAssembly(CustomTestCase): def test_rejects_ragged_rows(self): with self.assertRaisesRegex(ValueError, "rectangular"): - _build_flat_input_top_logprobs_fields( + _build_flat_top_logprobs_fields( + "input", [None, [-0.1, -2.5], [-0.3]], [None, [11, 22], [33]], top_logprobs_num=2, @@ -316,6 +335,101 @@ class TestB64MetaInfo(CustomTestCase): ) +class TestFlatOutputTopLogprobs(CustomTestCase): + def _state(self, val_rows=None, **obj_kwargs) -> ReqState: + state = _make_state( + return_logprob=True, + top_logprobs_num=2, + return_flat_raw_output_top_logprobs=True, + **obj_kwargs, + ) + state.output_top_logprobs_val.extend(val_rows or _EXACT_VAL_ROWS[1:]) + state.output_top_logprobs_idx.extend(_IDX_ROWS[1:]) + return state + + def _meta_info(self, state: ReqState, finished: bool) -> dict: + meta_info = {} + _TokenizerManagerStub().add_logprob_to_meta_info( + meta_info, + state, + top_logprobs_num=2, + token_ids_logprob=None, + return_text_in_logprobs=False, + finished=finished, + ) + return meta_info + + def test_unfinished_output_carries_no_top_logprobs(self): + state = self._state() + + meta_info = self._meta_info(state, finished=False) + + self.assertFalse(any(k.startswith("output_top_logprobs") for k in meta_info)) + self.assertEqual(state.output_top_logprobs, []) + + def test_finished_output_replaces_nested(self): + meta_info = self._meta_info(self._state(), finished=True) + + self.assertNotIn("output_top_logprobs", meta_info) + self.assertEqual(meta_info["output_top_logprobs_shape"], [3, 2]) + self.assertEqual(meta_info["output_top_logprobs_null_prefix"], 0) + self.assertEqual( + meta_info["output_top_logprobs_val_flat"], + [v for row in _EXACT_VAL_ROWS[1:] for v in row], + ) + self.assertEqual( + meta_info["output_top_logprobs_idx_flat"], + [i for row in _IDX_ROWS[1:] for i in row], + ) + # The prompt side keeps its own format. + self.assertIn("input_top_logprobs", meta_info) + + def test_plain_json_flattens_the_original_rows(self): + val_rows = [None, [-0.5, -2.5], [-0.25, -1.5]] + state = _make_state( + return_logprob=True, + top_logprobs_num=2, + return_flat_raw_output_top_logprobs=True, + ) + state.output_top_logprobs_val.extend(val_rows) + state.output_top_logprobs_idx.extend([None, [11, 22], [33, 44]]) + + meta_info = self._meta_info(state, finished=True) + + self.assertEqual(meta_info["output_top_logprobs_shape"], [2, 2]) + self.assertEqual(meta_info["output_top_logprobs_null_prefix"], 1) + self.assertEqual(meta_info["output_top_logprobs_idx_flat"], [11, 22, 33, 44]) + # Plain JSON reuses the engine's floats; no NumPy round trip. + self.assertIs(meta_info["output_top_logprobs_val_flat"][0], val_rows[1][0]) + + def test_b64_roundtrip_keeps_non_finite_logprobs(self): + val_rows = [[-0.5, float("-inf")], [-0.25, -1.5], [-0.125, -4.0]] + state = self._state(val_rows, return_flat_raw_top_logprobs_b64=True) + + meta_info = self._meta_info(state, finished=True) + + self.assertNotIn("output_top_logprobs_val_flat", meta_info) + shape = meta_info["output_top_logprobs_shape"] + val = np.frombuffer( + base64.b64decode(meta_info["output_top_logprobs_val_flat_b64"]), + dtype=meta_info["output_top_logprobs_val_flat_b64_dtype"], + ).reshape(shape) + idx = np.frombuffer( + base64.b64decode(meta_info["output_top_logprobs_idx_flat_b64"]), + dtype=meta_info["output_top_logprobs_idx_flat_b64_dtype"], + ).reshape(shape) + np.testing.assert_array_equal(val, np.asarray(val_rows, dtype=np.float32)) + np.testing.assert_array_equal(idx, np.asarray(_IDX_ROWS[1:], dtype=np.int32)) + + def test_unrepresentable_rows_fall_back_to_nested(self): + state = self._state([[-0.5, -2.5], [-0.25], [-0.125, -4.0]]) + + meta_info = self._meta_info(state, finished=True) + + self.assertNotIn("output_top_logprobs_shape", meta_info) + self.assertEqual(len(meta_info["output_top_logprobs"]), 3) + + def _make_logprob_processor() -> SchedulerLogprobResultProcessor: # enable_mis comes from the published exec bag, not from here; see setUp. return SchedulerLogprobResultProcessor( @@ -461,14 +575,18 @@ class TestFromArraysMatchesFromRows(CustomTestCase): response fields as the rows-based builder.""" def _both(self, return_b64: bool): - from_rows = _build_flat_input_top_logprobs_fields( - _EXACT_VAL_ROWS, _IDX_ROWS, top_logprobs_num=2, return_b64=return_b64 + from_rows = _build_flat_top_logprobs_fields( + "input", + _EXACT_VAL_ROWS, + _IDX_ROWS, + top_logprobs_num=2, + return_b64=return_b64, ) - val_arr, idx_arr, null_prefix = build_flat_input_top_logprobs_arrays( + val_arr, idx_arr, null_prefix = build_flat_top_logprobs_arrays( _EXACT_VAL_ROWS, _IDX_ROWS, top_logprobs_num=2 ) - from_arrays = _build_flat_input_top_logprobs_fields_from_arrays( - val_arr, idx_arr, null_prefix, return_b64=return_b64 + from_arrays = _build_flat_top_logprobs_fields_from_arrays( + "input", val_arr, idx_arr, null_prefix, return_b64=return_b64 ) return from_rows, from_arrays @@ -481,17 +599,19 @@ class TestFromArraysMatchesFromRows(CustomTestCase): self.assertEqual(from_rows, from_arrays) def test_all_null_rows(self): - val_arr, idx_arr, null_prefix = build_flat_input_top_logprobs_arrays( + val_arr, idx_arr, null_prefix = build_flat_top_logprobs_arrays( [None], [None], top_logprobs_num=2 ) self.assertEqual(val_arr.shape, (0, 2)) self.assertEqual(null_prefix, 1) - fields = _build_flat_input_top_logprobs_fields_from_arrays( - val_arr, idx_arr, null_prefix + fields = _build_flat_top_logprobs_fields_from_arrays( + "input", val_arr, idx_arr, null_prefix ) self.assertEqual( fields, - _build_flat_input_top_logprobs_fields([None], [None], top_logprobs_num=2), + _build_flat_top_logprobs_fields( + "input", [None], [None], top_logprobs_num=2 + ), ) @@ -517,7 +637,7 @@ class TestMetaInfoFromSchedulerArrays(CustomTestCase): **state_kwargs, ) # Scheduler-flat requests arrive with empty nested rows and the arrays. - state.input_top_logprobs_scheduler_flat = build_flat_input_top_logprobs_arrays( + state.input_top_logprobs_scheduler_flat = build_flat_top_logprobs_arrays( _EXACT_VAL_ROWS, _IDX_ROWS, top_logprobs_num=2 ) return state @@ -560,6 +680,7 @@ class TestTokenizerManagerLogprobs(CustomTestCase): return_text_in_logprobs=False, recv_obj=recv_obj, recv_obj_index=0, + finished=False, ) self.assertEqual(meta_info["input_token_logprobs"], []) @@ -591,6 +712,7 @@ class TestTokenizerManagerLogprobs(CustomTestCase): return_text_in_logprobs=False, recv_obj=recv_obj, recv_obj_index=0, + finished=True, ) for key in ( @@ -648,7 +770,7 @@ class TestBatchOutputTransport(CustomTestCase): (SGLANG_USE_PICKLE_IPC, the default) and msgpack (enc/dec hooks).""" def _flat_output(self) -> BatchTokenIDOutput: - val_arr, idx_arr, null_prefix = build_flat_input_top_logprobs_arrays( + val_arr, idx_arr, null_prefix = build_flat_top_logprobs_arrays( _EXACT_VAL_ROWS, _IDX_ROWS, top_logprobs_num=2 ) return _make_batch_token_id_output(