[feat] add a system one compatible /v1/systemone route (#41208)
This commit is contained in:
@@ -58,6 +58,7 @@ from fastapi import (
|
||||
UploadFile,
|
||||
WebSocket,
|
||||
)
|
||||
from fastapi.encoders import jsonable_encoder
|
||||
from fastapi.exceptions import RequestValidationError
|
||||
from fastapi.middleware.cors import CORSMiddleware
|
||||
from fastapi.responses import ORJSONResponse, Response, StreamingResponse
|
||||
@@ -88,6 +89,7 @@ from sglang.srt.entrypoints.openai.protocol import (
|
||||
ChatCompletionRequest,
|
||||
ClassifyRequest,
|
||||
CompletionRequest,
|
||||
DecisionRequest,
|
||||
DetokenizeRequest,
|
||||
EmbeddingRequest,
|
||||
ErrorResponse,
|
||||
@@ -100,6 +102,7 @@ from sglang.srt.entrypoints.openai.protocol import (
|
||||
)
|
||||
from sglang.srt.entrypoints.openai.serving_classify import OpenAIServingClassify
|
||||
from sglang.srt.entrypoints.openai.serving_completions import OpenAIServingCompletion
|
||||
from sglang.srt.entrypoints.openai.serving_decisions import OpenAIServingDecisions
|
||||
from sglang.srt.entrypoints.openai.serving_embedding import OpenAIServingEmbedding
|
||||
from sglang.srt.entrypoints.openai.serving_rerank import OpenAIServingRerank
|
||||
from sglang.srt.entrypoints.openai.serving_score import OpenAIServingScore
|
||||
@@ -111,6 +114,8 @@ from sglang.srt.entrypoints.openai.serving_transcription import (
|
||||
OpenAIServingTranscription,
|
||||
)
|
||||
from sglang.srt.entrypoints.request_headers import apply_header_overrides
|
||||
from sglang.srt.entrypoints.systemone.protocol import SystemOneRequest
|
||||
from sglang.srt.entrypoints.systemone.serving import SystemOneServing
|
||||
from sglang.srt.entrypoints.warmup import execute_warmups
|
||||
from sglang.srt.environ import envs
|
||||
from sglang.srt.function_call.function_call_parser import FunctionCallParser
|
||||
@@ -323,6 +328,9 @@ async def lifespan(fast_api_app: FastAPI):
|
||||
fast_api_app.state.openai_serving_score = OpenAIServingScore(
|
||||
_global_state.tokenizer_manager
|
||||
)
|
||||
fast_api_app.state.openai_serving_decisions = OpenAIServingDecisions(
|
||||
fast_api_app.state.openai_serving_chat
|
||||
)
|
||||
fast_api_app.state.openai_serving_rerank = OpenAIServingRerank(
|
||||
_global_state.tokenizer_manager, _global_state.template_manager
|
||||
)
|
||||
@@ -344,6 +352,11 @@ async def lifespan(fast_api_app: FastAPI):
|
||||
fast_api_app.state.openai_serving_chat
|
||||
)
|
||||
|
||||
# Initialize System One compatible decision handler
|
||||
fast_api_app.state.systemone_serving = SystemOneServing(
|
||||
fast_api_app.state.openai_serving_chat
|
||||
)
|
||||
|
||||
# Launch tool server
|
||||
tool_server = None
|
||||
if get_serving().tool_server == "demo":
|
||||
@@ -603,8 +616,9 @@ async def validation_exception_handler(request: Request, exc: RequestValidationE
|
||||
|
||||
For /v1/messages, emit Anthropic-style envelope and scrub the message so
|
||||
file paths or Python internals from the default ``str(exc)`` representation
|
||||
never reach the client. For /v1/responses, keep OpenAI-style. Otherwise
|
||||
use the legacy ErrorResponse shape.
|
||||
never reach the client. For /v1/responses, keep OpenAI-style. For
|
||||
/v1/systemone, keep the 422 detail list that the System One API documents.
|
||||
Otherwise use the legacy ErrorResponse shape.
|
||||
"""
|
||||
if request.url.path.startswith("/v1/messages"):
|
||||
return _anthropic_error_response(
|
||||
@@ -613,6 +627,19 @@ async def validation_exception_handler(request: Request, exc: RequestValidationE
|
||||
message=_anthropic_validation_message(exc.errors()),
|
||||
)
|
||||
|
||||
route_path = request.url.path.removeprefix(request.scope.get("root_path", ""))
|
||||
if route_path == "/v1/systemone":
|
||||
# The System One API documents FastAPI's default 422 detail list. The
|
||||
# optional input echo is left out, since it can be any client value.
|
||||
detail = [
|
||||
{key: value for key, value in error.items() if key != "input"}
|
||||
for error in exc.errors()
|
||||
]
|
||||
return ORJSONResponse(
|
||||
status_code=HTTPStatus.UNPROCESSABLE_ENTITY.value,
|
||||
content={"detail": jsonable_encoder(detail)},
|
||||
)
|
||||
|
||||
exc_str = str(exc)
|
||||
errors_str = str(exc.errors())
|
||||
|
||||
@@ -1933,6 +1960,14 @@ async def v1_score_request(request: ScoringRequest, raw_request: Request):
|
||||
)
|
||||
|
||||
|
||||
@app.post("/v1/decisions", dependencies=[Depends(validate_json_request)])
|
||||
async def v1_decisions_request(request: DecisionRequest, raw_request: Request):
|
||||
"""Answer typed choice, score, and yes or no questions about an input by scoring single-token answer labels through the scoring API, without generation."""
|
||||
return await raw_request.app.state.openai_serving_decisions.handle_request(
|
||||
request, raw_request
|
||||
)
|
||||
|
||||
|
||||
@app.post("/v1/responses", dependencies=[Depends(validate_json_request)])
|
||||
async def v1_responses_request(request: ResponsesRequest, raw_request: Request):
|
||||
"""Endpoint for the responses API with reasoning support."""
|
||||
@@ -2047,6 +2082,15 @@ async def anthropic_v1_count_tokens(
|
||||
)
|
||||
|
||||
|
||||
## System One compatible decision API
|
||||
@app.post("/v1/systemone", dependencies=[Depends(validate_json_request)])
|
||||
async def systemone_decisions(request: SystemOneRequest, raw_request: Request):
|
||||
"""System One compatible decisions, answered by candidate scoring without generation."""
|
||||
return await raw_request.app.state.systemone_serving.handle_request(
|
||||
request, raw_request
|
||||
)
|
||||
|
||||
|
||||
## SageMaker API
|
||||
@app.get("/ping")
|
||||
async def sagemaker_health() -> Response:
|
||||
|
||||
@@ -17,6 +17,7 @@ from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import time
|
||||
import unicodedata
|
||||
import uuid
|
||||
from dataclasses import dataclass
|
||||
from typing import (
|
||||
@@ -1438,6 +1439,137 @@ class ScoringResponse(BaseModel):
|
||||
object: str = "scoring"
|
||||
|
||||
|
||||
def is_blank_decision_text(value) -> bool:
|
||||
return not (value.strip() if isinstance(value, str) else value)
|
||||
|
||||
|
||||
def _nonblank_decision_text(value):
|
||||
if is_blank_decision_text(value):
|
||||
raise ValueError("must not be blank")
|
||||
return value
|
||||
|
||||
|
||||
def check_option_names(names) -> None:
|
||||
"""Refuse option names that would make the rendered option lines ambiguous."""
|
||||
seen = set()
|
||||
for name in names:
|
||||
key = name.strip().casefold()
|
||||
if not key:
|
||||
raise ValueError("option names must be nonempty")
|
||||
# Each option is rendered as one prompt line.
|
||||
if any(unicodedata.category(c) in ("Cc", "Zl", "Zp") for c in name):
|
||||
raise ValueError(
|
||||
f"option name {name!r} must not contain control or line break "
|
||||
"characters"
|
||||
)
|
||||
if key in seen:
|
||||
raise ValueError(f"option name {name!r} repeats another option")
|
||||
seen.add(key)
|
||||
|
||||
|
||||
# Objects and arrays are rendered into the prompt as compact JSON.
|
||||
DecisionText = Union[str, Dict[str, Any], List[Any]]
|
||||
RequiredDecisionText = Annotated[DecisionText, AfterValidator(_nonblank_decision_text)]
|
||||
|
||||
|
||||
class DecisionOption(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
name: str
|
||||
description: Optional[DecisionText] = None
|
||||
|
||||
|
||||
class DecisionChoiceQuestion(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
id: Annotated[str, AfterValidator(_nonblank_decision_text)]
|
||||
type: Literal["choice"]
|
||||
question: RequiredDecisionText
|
||||
# Options are labeled A to Z in order, so at most 26.
|
||||
options: List[DecisionOption] = Field(min_length=2, max_length=26)
|
||||
|
||||
@model_validator(mode="after")
|
||||
def _option_names_distinct(self):
|
||||
try:
|
||||
check_option_names(option.name for option in self.options)
|
||||
except ValueError as e:
|
||||
raise ValueError(f"question {self.id!r}: {e}") from None
|
||||
return self
|
||||
|
||||
|
||||
class DecisionScoreQuestion(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
id: Annotated[str, AfterValidator(_nonblank_decision_text)]
|
||||
type: Literal["score"]
|
||||
question: RequiredDecisionText
|
||||
# Levels are labeled 0 to 9 in order, so at most 10.
|
||||
levels: List[RequiredDecisionText] = Field(min_length=2, max_length=10)
|
||||
|
||||
|
||||
class DecisionYesNoQuestion(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
id: Annotated[str, AfterValidator(_nonblank_decision_text)]
|
||||
type: Literal["yes_no"]
|
||||
question: RequiredDecisionText
|
||||
yes: Optional[DecisionText] = None
|
||||
no: Optional[DecisionText] = None
|
||||
|
||||
|
||||
DecisionQuestion = Annotated[
|
||||
Union[DecisionChoiceQuestion, DecisionScoreQuestion, DecisionYesNoQuestion],
|
||||
Field(discriminator="type"),
|
||||
]
|
||||
|
||||
|
||||
class DecisionRequest(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
input: RequiredDecisionText
|
||||
questions: List[DecisionQuestion] = Field(min_length=1)
|
||||
# Scales option probabilities only, not label_mass.
|
||||
temperature: float = Field(default=1.0, gt=0, allow_inf_nan=False)
|
||||
# Applied over the server defaults, with the template reasoning toggle off.
|
||||
chat_template_kwargs: Dict[str, Any] = Field(default_factory=dict)
|
||||
# Pins the server-owned prompt wording. A different served version is refused.
|
||||
prompt_format_version: Optional[int] = None
|
||||
return_prompt_token_ids: bool = False
|
||||
model: str = DEFAULT_MODEL_NAME
|
||||
|
||||
@field_validator("questions")
|
||||
@classmethod
|
||||
def _question_ids_distinct(cls, questions):
|
||||
seen = set()
|
||||
for question in questions:
|
||||
if question.id in seen:
|
||||
raise ValueError(
|
||||
f"question id {question.id!r} repeats another question"
|
||||
)
|
||||
seen.add(question.id)
|
||||
return questions
|
||||
|
||||
|
||||
class DecisionAnswer(BaseModel):
|
||||
type: Literal["choice", "score", "yes_no"]
|
||||
probabilities: Dict[str, float]
|
||||
# Full-vocabulary probability of all answer labels at the answer position.
|
||||
label_mass: float
|
||||
choice: Optional[str] = None
|
||||
score: Optional[float] = None
|
||||
# The exact /v1/score inputs, with return_prompt_token_ids.
|
||||
prompt_token_ids: Optional[List[int]] = None
|
||||
label_token_ids: Optional[List[int]] = None
|
||||
|
||||
|
||||
class DecisionResponse(BaseModel):
|
||||
object: str = "decisions"
|
||||
model: str
|
||||
prompt_format_version: int
|
||||
answers: Dict[str, DecisionAnswer]
|
||||
usage: UsageInfo
|
||||
|
||||
|
||||
class V1RerankReqInput(BaseModel):
|
||||
query: RerankContent = Field(
|
||||
...,
|
||||
@@ -1563,6 +1695,7 @@ OpenAIServingRequest = Union[
|
||||
EmbeddingRequest,
|
||||
ClassifyRequest,
|
||||
ScoringRequest,
|
||||
DecisionRequest,
|
||||
V1RerankReqInput,
|
||||
TokenizeRequest,
|
||||
DetokenizeRequest,
|
||||
|
||||
@@ -2445,7 +2445,7 @@ class OpenAIServingChat(OpenAIServingBase):
|
||||
output_token_logprobs = ret_item["meta_info"]["output_token_logprobs"]
|
||||
output_top_logprobs = ret_item["meta_info"].get("output_top_logprobs", None)
|
||||
token_logprobs = self._build_token_logprobs_from_raw(
|
||||
output_token_logprobs, output_top_logprobs, use_token_index=True
|
||||
output_token_logprobs, output_top_logprobs
|
||||
)
|
||||
return ChoiceLogprobs(content=token_logprobs)
|
||||
|
||||
@@ -2453,7 +2453,6 @@ class OpenAIServingChat(OpenAIServingBase):
|
||||
self,
|
||||
output_token_logprobs: list[Any],
|
||||
output_top_logprobs: list[Any] | None,
|
||||
use_token_index: bool = False,
|
||||
) -> list[ChatCompletionTokenLogprob]:
|
||||
"""Build OpenAI ChatCompletionTokenLogprob from the engine's raw
|
||||
``(logprob, token_id, token_text)`` triples.
|
||||
@@ -2487,13 +2486,9 @@ class OpenAIServingChat(OpenAIServingBase):
|
||||
|
||||
top_logprobs: list[TopLogprob] = []
|
||||
if output_top_logprobs:
|
||||
# - Non-streaming (use_token_index=True): output_top_logprobs is
|
||||
# the full per-position list; take the row for this token.
|
||||
# - Streaming (use_token_index=False): rows are pre-sliced so the
|
||||
# current chunk holds exactly one row at index 0.
|
||||
top_row_idx = token_idx if use_token_index else 0
|
||||
if top_row_idx < len(output_top_logprobs):
|
||||
top_row = output_top_logprobs[top_row_idx]
|
||||
# Both callers pass rows aligned with output_token_logprobs.
|
||||
if token_idx < len(output_top_logprobs):
|
||||
top_row = output_top_logprobs[token_idx]
|
||||
if top_row is not None:
|
||||
for top_logprob, top_id, top_text in top_row:
|
||||
if is_byte_level:
|
||||
@@ -2686,7 +2681,7 @@ class OpenAIServingChat(OpenAIServingBase):
|
||||
n_prev_token:total_output_logprobs
|
||||
]
|
||||
token_logprobs = self._build_token_logprobs_from_raw(
|
||||
output_token_logprobs, output_top_logprobs, use_token_index=False
|
||||
output_token_logprobs, output_top_logprobs
|
||||
)
|
||||
return ChoiceLogprobs(content=token_logprobs)
|
||||
|
||||
|
||||
@@ -0,0 +1,535 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import logging
|
||||
import math
|
||||
import string
|
||||
from typing import TYPE_CHECKING, Any, Dict, Iterator, List, Optional, Tuple
|
||||
|
||||
import msgspec
|
||||
from fastapi import Request
|
||||
from fastapi.responses import ORJSONResponse
|
||||
from transformers import PreTrainedTokenizerBase
|
||||
|
||||
from sglang.srt.entrypoints.openai.protocol import (
|
||||
DecisionAnswer,
|
||||
DecisionChoiceQuestion,
|
||||
DecisionQuestion,
|
||||
DecisionRequest,
|
||||
DecisionResponse,
|
||||
DecisionScoreQuestion,
|
||||
DecisionText,
|
||||
UsageInfo,
|
||||
is_blank_decision_text,
|
||||
)
|
||||
from sglang.srt.entrypoints.openai.serving_base import OpenAIServingBase
|
||||
from sglang.srt.entrypoints.openai.serving_chat import _CHAT_TEMPLATE_CLIENT_ERRORS
|
||||
from sglang.srt.parser.reasoning_parser import ReasoningParser
|
||||
from sglang.srt.runtime_context import get_exec
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from sglang.srt.entrypoints.openai.serving_chat import OpenAIServingChat
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Version of the server-owned prompt wording and answer labels.
|
||||
# Any change to a rendering /v1/decisions can produce needs a new version.
|
||||
PROMPT_FORMAT_VERSION = 1
|
||||
|
||||
# Parser defaults that name the chat template kwarg toggling reasoning.
|
||||
_PARSER_TOGGLE_MODES = (
|
||||
"thinking",
|
||||
"enable_thinking",
|
||||
"explicit_thinking",
|
||||
"explicit_enable_thinking",
|
||||
)
|
||||
# Answer text of a finished reply, rendered only to see what precedes an answer.
|
||||
_REPLY_SENTINEL = "DECISION_ANSWER"
|
||||
|
||||
|
||||
class QuestionView(msgspec.Struct, frozen=True):
|
||||
"""A question as the renderer and scorer see it, shared by the decision routes."""
|
||||
|
||||
# choice, score, or yes_no
|
||||
kind: str
|
||||
# None or blank when the question has no text of its own
|
||||
question: Any
|
||||
# Option names, level indices, or yes and no, in candidate order
|
||||
names: List[str]
|
||||
# Option descriptions, levels, or the yes and no descriptions
|
||||
details: List[Any]
|
||||
|
||||
|
||||
class OpenAIServingDecisions(OpenAIServingBase):
|
||||
"""Handler for /v1/decisions requests, answered by candidate scoring without generation"""
|
||||
|
||||
# Named in setup refusals, so each decision route reports itself.
|
||||
route = "/v1/decisions"
|
||||
|
||||
def __init__(self, chat_serving: OpenAIServingChat):
|
||||
super().__init__(chat_serving.tokenizer_manager)
|
||||
# Render the way the chat route does, and refuse where it renders differently.
|
||||
self.template_manager = chat_serving.template_manager
|
||||
self.default_chat_template_kwargs = chat_serving.default_chat_template_kwargs
|
||||
self.chat_encoding_spec = chat_serving.chat_encoding_spec
|
||||
self.prompt_text_is_lossy = chat_serving._prompt_text_round_trip_is_lossy
|
||||
tokenizer = self.tokenizer_manager.tokenizer
|
||||
# Other tokenizers skip the shortcut in _encode_labels and check the full prompt.
|
||||
self.added_tokens = (
|
||||
{i: token for token, i in tokenizer.get_added_vocab().items()}
|
||||
if isinstance(tokenizer, PreTrainedTokenizerBase)
|
||||
else {}
|
||||
)
|
||||
# The configured reasoning parser, or the one the chat template suggests,
|
||||
# tells where reasoning blocks start and end and whether answers open one.
|
||||
parser = (
|
||||
chat_serving.reasoning_parser
|
||||
or self.template_manager.suggested_reasoning_parser
|
||||
)
|
||||
self.reasoning_markers = None
|
||||
self.answers_open_reasoning = False
|
||||
# The kwarg that turns reasoning on and off: the template's, else, when
|
||||
# detection finds no config, the one the configured or suggested parser names.
|
||||
config = self.template_manager.reasoning_config
|
||||
self.reasoning_toggle = config.toggle_param if config is not None else None
|
||||
if parser is not None:
|
||||
try:
|
||||
detector = ReasoningParser(
|
||||
model_type=parser, tokenizer=tokenizer
|
||||
).detector
|
||||
except ValueError as e:
|
||||
logger.warning(
|
||||
"No reasoning block check for %s with parser '%s': %s",
|
||||
self.route,
|
||||
parser,
|
||||
e,
|
||||
)
|
||||
else:
|
||||
mode = detector.reasoning_default
|
||||
if config is None and mode in _PARSER_TOGGLE_MODES:
|
||||
self.reasoning_toggle = mode.removeprefix("explicit_")
|
||||
if detector.think_start_token and detector.think_end_token:
|
||||
self.reasoning_markers = (
|
||||
detector.think_start_token,
|
||||
detector.think_end_token,
|
||||
)
|
||||
self.answers_open_reasoning = detector.reasoning_default == "always"
|
||||
|
||||
def _request_id_prefix(self) -> str:
|
||||
return "decision-"
|
||||
|
||||
def _validate_request(self, request: DecisionRequest) -> Optional[str]:
|
||||
error = self._validate_server(request.model)
|
||||
if error is not None:
|
||||
return error
|
||||
version = request.prompt_format_version
|
||||
if version is not None and version != PROMPT_FORMAT_VERSION:
|
||||
return (
|
||||
f"prompt_format_version {version} is not served, this server "
|
||||
f"uses version {PROMPT_FORMAT_VERSION}"
|
||||
)
|
||||
return self._validate_reasoning(request.chat_template_kwargs)
|
||||
|
||||
def _validate_server(self, model: str) -> Optional[str]:
|
||||
"""Refuse servers this route cannot render faithfully."""
|
||||
route = self.route
|
||||
if not self.tokenizer_manager.is_generation:
|
||||
return f"{route} requires a generation model"
|
||||
if self.tokenizer_manager.tokenizer is None:
|
||||
return f"{route} requires the server tokenizer"
|
||||
if self.chat_encoding_spec is not None:
|
||||
return (
|
||||
f"{route} requires a chat template, but this model's chat "
|
||||
f"route uses the {self.chat_encoding_spec!r} encoder"
|
||||
)
|
||||
if self.prompt_text_is_lossy:
|
||||
return (
|
||||
f"{route} places answer labels on the rendered chat text, "
|
||||
"which this tokenizer does not encode back to the same ids"
|
||||
)
|
||||
if self.template_manager.chat_template_name is not None:
|
||||
return (
|
||||
f"{route} renders the tokenizer's Jinja chat template, but "
|
||||
"this server uses the built-in chat template "
|
||||
f"{self.template_manager.chat_template_name!r}"
|
||||
)
|
||||
if get_exec().features.enable_mis:
|
||||
return f"{route} does not support --enable-mis"
|
||||
if get_exec().dllm.dllm_algorithm is not None:
|
||||
return (
|
||||
f"{route} does not support diffusion language models "
|
||||
"served with --dllm-algorithm"
|
||||
)
|
||||
_, adapter = self._parse_model_parameter(model)
|
||||
if adapter is not None:
|
||||
return (
|
||||
f"model names the LoRA adapter {adapter!r}, which {route} "
|
||||
"does not support"
|
||||
)
|
||||
return None
|
||||
|
||||
def _validate_reasoning(
|
||||
self, chat_template_kwargs: Dict[str, Any]
|
||||
) -> Optional[str]:
|
||||
"""The answer position must follow the reasoning block, not sit inside it."""
|
||||
config = self.template_manager.reasoning_config
|
||||
if config is not None and config.always_on:
|
||||
return (
|
||||
f"{self.route} does not support chat templates that always "
|
||||
"reason before answering"
|
||||
)
|
||||
toggle = self.reasoning_toggle
|
||||
if toggle in chat_template_kwargs and chat_template_kwargs[toggle] is not False:
|
||||
return (
|
||||
f"chat_template_kwargs sets {toggle!r} to "
|
||||
f"{chat_template_kwargs[toggle]!r}, but decisions need it false or unset"
|
||||
)
|
||||
return None
|
||||
|
||||
def _chat_template_kwargs(self, request_kwargs: Dict[str, Any]) -> Dict[str, Any]:
|
||||
"""Reasoning off, then the server defaults, then the request kwargs."""
|
||||
kwargs = {}
|
||||
if self.reasoning_toggle is not None:
|
||||
kwargs[self.reasoning_toggle] = False
|
||||
for key, value in self.default_chat_template_kwargs.items():
|
||||
kwargs.setdefault(key, value)
|
||||
kwargs.update(request_kwargs)
|
||||
return kwargs
|
||||
|
||||
def _convert_to_internal_request(
|
||||
self,
|
||||
request: DecisionRequest,
|
||||
raw_request: Request = None,
|
||||
) -> Tuple[Iterator[Tuple[List[int], List[int]]], DecisionRequest]:
|
||||
# Lazy, so the handler can let other requests run between questions.
|
||||
return self._encoded_questions(request), request
|
||||
|
||||
def _encoded_questions(
|
||||
self, request: DecisionRequest
|
||||
) -> Iterator[Tuple[List[int], List[int]]]:
|
||||
"""Prompt and label ids for each question, in request order."""
|
||||
text = render_text(request.input)
|
||||
chat_template_kwargs = self._chat_template_kwargs(request.chat_template_kwargs)
|
||||
for question in request.questions:
|
||||
view = _decision_view(question)
|
||||
try:
|
||||
encoded = self._encode_question(
|
||||
text=text,
|
||||
view=view,
|
||||
labels=default_labels(view),
|
||||
chat_template_kwargs=chat_template_kwargs,
|
||||
)
|
||||
except ValueError as e:
|
||||
raise ValueError(f"question {question.id!r}: {e}") from e
|
||||
yield encoded
|
||||
|
||||
def _encode_question(
|
||||
self,
|
||||
text: str,
|
||||
view: QuestionView,
|
||||
labels: List[str],
|
||||
chat_template_kwargs: Dict[str, Any],
|
||||
) -> Tuple[List[int], List[int]]:
|
||||
tokenizer = self.tokenizer_manager.tokenizer
|
||||
content = _render_question(text=text, view=view, labels=labels)
|
||||
prompt = self._apply_chat_template(content, chat_template_kwargs)
|
||||
if self.reasoning_markers is not None:
|
||||
# Look only after the message, whose last line is fixed text.
|
||||
closing = content.rsplit("\n", 1)[-1]
|
||||
cut = prompt.rfind(closing)
|
||||
generation_prompt = prompt if cut < 0 else prompt[cut + len(closing) :]
|
||||
start, end = self.reasoning_markers
|
||||
opened = generation_prompt.rfind(start)
|
||||
closed = generation_prompt.rfind(end)
|
||||
if opened > closed:
|
||||
raise ValueError(
|
||||
"the chat template leaves a reasoning block open at the "
|
||||
"answer position, so this model is not supported with these "
|
||||
"chat_template_kwargs"
|
||||
)
|
||||
if self.answers_open_reasoning and closed < 0:
|
||||
raise ValueError(
|
||||
"the reasoning parser for this model expects answers to start "
|
||||
"with a reasoning block, and the chat template does not close "
|
||||
"one. Send chat_template_kwargs that turn thinking off, if the "
|
||||
"template supports it"
|
||||
)
|
||||
# The template's own finished reply shows whether answers start with
|
||||
# a reasoning block that the generation prompt leaves out.
|
||||
reply = self._render_reply(closing, chat_template_kwargs)
|
||||
begin = reply.rfind(closing) if reply is not None else -1
|
||||
answer = reply.find(_REPLY_SENTINEL, begin) if begin >= 0 else -1
|
||||
if answer >= 0 and reply[begin:answer].count(start) > (
|
||||
generation_prompt.count(start)
|
||||
):
|
||||
raise ValueError(
|
||||
"the chat template starts every answer with a reasoning "
|
||||
"block, so this model is not supported"
|
||||
)
|
||||
prompt_ids = tokenizer.encode(prompt, add_special_tokens=False)
|
||||
# Refuse here because --allow-auto-truncate would cut off the answer position.
|
||||
context_len = self.tokenizer_manager.context_len
|
||||
if len(prompt_ids) + self.tokenizer_manager.num_reserved_tokens >= context_len:
|
||||
raise ValueError(
|
||||
f"the prompt has {len(prompt_ids)} tokens, which does not fit "
|
||||
f"the context length of {context_len} tokens"
|
||||
)
|
||||
label_ids = _encode_labels(
|
||||
tokenizer=tokenizer,
|
||||
prompt=prompt,
|
||||
prompt_ids=prompt_ids,
|
||||
labels=labels,
|
||||
added_tokens=self.added_tokens,
|
||||
)
|
||||
return prompt_ids, label_ids
|
||||
|
||||
def _apply_chat_template(
|
||||
self, content: str, chat_template_kwargs: Dict[str, Any]
|
||||
) -> str:
|
||||
try:
|
||||
return self.tokenizer_manager.tokenizer.apply_chat_template(
|
||||
[{"role": "user", "content": content}],
|
||||
tokenize=False,
|
||||
add_generation_prompt=True,
|
||||
**chat_template_kwargs,
|
||||
)
|
||||
except _CHAT_TEMPLATE_CLIENT_ERRORS as e:
|
||||
raise ValueError(f"the chat template failed: {e}") from e
|
||||
|
||||
def _render_reply(
|
||||
self, message: str, chat_template_kwargs: Dict[str, Any]
|
||||
) -> Optional[str]:
|
||||
try:
|
||||
return self.tokenizer_manager.tokenizer.apply_chat_template(
|
||||
[
|
||||
{"role": "user", "content": message},
|
||||
{"role": "assistant", "content": _REPLY_SENTINEL},
|
||||
],
|
||||
tokenize=False,
|
||||
**chat_template_kwargs,
|
||||
)
|
||||
except _CHAT_TEMPLATE_CLIENT_ERRORS:
|
||||
# The generation prompt checks above still apply.
|
||||
return None
|
||||
|
||||
async def _score(
|
||||
self,
|
||||
adapted_request: Iterator[Tuple[List[int], List[int]]],
|
||||
raw_request: Request,
|
||||
temperature: float = 1.0,
|
||||
):
|
||||
"""Encode every question, then score them all in one call."""
|
||||
prompts, label_token_ids = await _encode_all(adapted_request)
|
||||
result = await self.tokenizer_manager.score_prompts(
|
||||
prompts=prompts,
|
||||
label_token_ids=label_token_ids,
|
||||
apply_softmax=True,
|
||||
request=raw_request,
|
||||
temperature=temperature,
|
||||
return_token_logprobs=True,
|
||||
)
|
||||
return prompts, label_token_ids, result
|
||||
|
||||
async def _handle_non_streaming_request(
|
||||
self,
|
||||
adapted_request: Iterator[Tuple[List[int], List[int]]],
|
||||
request: DecisionRequest,
|
||||
raw_request: Request,
|
||||
) -> ORJSONResponse:
|
||||
prompts, label_token_ids, result = await self._score(
|
||||
adapted_request=adapted_request,
|
||||
raw_request=raw_request,
|
||||
temperature=request.temperature,
|
||||
)
|
||||
answers = {}
|
||||
for i, question in enumerate(request.questions):
|
||||
answer = _build_answer(
|
||||
question=question,
|
||||
view=_decision_view(question),
|
||||
probabilities=result.scores[i],
|
||||
token_logprobs=result.token_logprobs[i],
|
||||
)
|
||||
if request.return_prompt_token_ids:
|
||||
answer.prompt_token_ids = prompts[i]
|
||||
answer.label_token_ids = label_token_ids[i]
|
||||
answers[question.id] = answer
|
||||
response = DecisionResponse(
|
||||
model=request.model,
|
||||
prompt_format_version=PROMPT_FORMAT_VERSION,
|
||||
answers=answers,
|
||||
usage=UsageInfo(
|
||||
prompt_tokens=result.prompt_tokens,
|
||||
total_tokens=result.prompt_tokens,
|
||||
),
|
||||
)
|
||||
return ORJSONResponse(content=response.model_dump(exclude_none=True))
|
||||
|
||||
|
||||
def render_text(value: Optional[DecisionText]) -> str:
|
||||
if value is None:
|
||||
return ""
|
||||
if isinstance(value, str):
|
||||
return value
|
||||
return json.dumps(value, ensure_ascii=False, separators=(",", ":"))
|
||||
|
||||
|
||||
async def _encode_all(
|
||||
encoded: Iterator[Tuple[List[int], List[int]]],
|
||||
) -> Tuple[List[List[int]], List[List[int]]]:
|
||||
"""Collect prompt and label ids, letting other requests run between questions."""
|
||||
prompts, label_token_ids = [], []
|
||||
for prompt_ids, label_ids in encoded:
|
||||
prompts.append(prompt_ids)
|
||||
label_token_ids.append(label_ids)
|
||||
# Each question renders and tokenizes the whole input on the event loop.
|
||||
await asyncio.sleep(0)
|
||||
return prompts, label_token_ids
|
||||
|
||||
|
||||
def _decision_view(question: DecisionQuestion) -> QuestionView:
|
||||
if isinstance(question, DecisionChoiceQuestion):
|
||||
return QuestionView(
|
||||
kind="choice",
|
||||
question=question.question,
|
||||
names=[option.name for option in question.options],
|
||||
details=[option.description for option in question.options],
|
||||
)
|
||||
if isinstance(question, DecisionScoreQuestion):
|
||||
return QuestionView(
|
||||
kind="score",
|
||||
question=question.question,
|
||||
names=[str(level) for level in range(len(question.levels))],
|
||||
details=list(question.levels),
|
||||
)
|
||||
return QuestionView(
|
||||
kind="yes_no",
|
||||
question=question.question,
|
||||
names=["yes", "no"],
|
||||
details=[question.yes, question.no],
|
||||
)
|
||||
|
||||
|
||||
def default_labels(view: QuestionView) -> List[str]:
|
||||
"""Single-token labels in candidate order: A to Z, level indices, or yes and no."""
|
||||
if view.kind == "choice":
|
||||
return list(string.ascii_uppercase[: len(view.names)])
|
||||
return list(view.names)
|
||||
|
||||
|
||||
def _render_question(text: str, view: QuestionView, labels: List[str]) -> str:
|
||||
"""Prompt wording of PROMPT_FORMAT_VERSION."""
|
||||
# Every /v1/decisions question has text. A question without its own text
|
||||
# drops the question line, and a yes or no question keeps its lead in.
|
||||
question_text = (
|
||||
"" if is_blank_decision_text(view.question) else render_text(view.question)
|
||||
)
|
||||
if view.kind == "choice":
|
||||
lines = [f"Question: {question_text}"] if question_text else []
|
||||
for label, name, description in zip(labels, view.names, view.details):
|
||||
detail = render_text(description)
|
||||
lines.append(
|
||||
f"{label}: {name} - {detail}" if detail else f"{label}: {name}"
|
||||
)
|
||||
lines.append("Answer with the letter of one option only.")
|
||||
elif view.kind == "score":
|
||||
lines = [f"Question: {question_text}"] if question_text else []
|
||||
lines += [
|
||||
f"{label}: {render_text(level)}"
|
||||
for label, level in zip(labels, view.details)
|
||||
]
|
||||
lines.append("Answer with the number of one level only.")
|
||||
else:
|
||||
lines = [
|
||||
f"Is the following true? {question_text}"
|
||||
if question_text
|
||||
else "Is the following true?"
|
||||
]
|
||||
for label, description in zip(labels, view.details):
|
||||
detail = render_text(description)
|
||||
if detail:
|
||||
lines.append(f"{label}: {detail}")
|
||||
lines.append("Answer with yes or no only.")
|
||||
return "\n".join([text, "", *lines])
|
||||
|
||||
|
||||
def label_context(
|
||||
tokenizer: Any,
|
||||
prompt: str,
|
||||
prompt_ids: List[int],
|
||||
added_tokens: Dict[int, str],
|
||||
) -> Tuple[str, List[int], bool]:
|
||||
"""Text and ids after which labels are checked, and whether the shortcut applies."""
|
||||
# Added tokens are split off before tokenization.
|
||||
# The text after the last one tokenizes on its own,
|
||||
# so the check does not grow with the input.
|
||||
# When the prompt ends with an added token, each label starts a new segment,
|
||||
# which is how the model continues after that token.
|
||||
last = next(
|
||||
(i for i in reversed(range(len(prompt_ids))) if prompt_ids[i] in added_tokens),
|
||||
None,
|
||||
)
|
||||
if last is not None:
|
||||
token = added_tokens[prompt_ids[last]]
|
||||
start = prompt.rfind(token)
|
||||
suffix = prompt[start + len(token) :]
|
||||
if start >= 0 and (
|
||||
tokenizer.encode(suffix, add_special_tokens=False) == prompt_ids[last + 1 :]
|
||||
):
|
||||
return suffix, prompt_ids[last + 1 :], True
|
||||
return prompt, prompt_ids, False
|
||||
|
||||
|
||||
def label_token_id(
|
||||
tokenizer: Any, text: str, text_ids: List[int], label: str
|
||||
) -> Optional[int]:
|
||||
"""The token a label adds after the text, or None when it is not exactly one."""
|
||||
ids = tokenizer.encode(text + label, add_special_tokens=False)
|
||||
if len(ids) != len(text_ids) + 1 or ids[:-1] != text_ids:
|
||||
return None
|
||||
return ids[-1]
|
||||
|
||||
|
||||
def _encode_labels(
|
||||
tokenizer: Any,
|
||||
prompt: str,
|
||||
prompt_ids: List[int],
|
||||
labels: List[str],
|
||||
added_tokens: Dict[int, str],
|
||||
) -> List[int]:
|
||||
"""Check that each label adds exactly one distinct token after the prompt."""
|
||||
text, text_ids, _ = label_context(tokenizer, prompt, prompt_ids, added_tokens)
|
||||
label_ids = []
|
||||
for label in labels:
|
||||
token_id = label_token_id(tokenizer, text, text_ids, label)
|
||||
if token_id is None or token_id in label_ids:
|
||||
raise ValueError(
|
||||
f"the answer label {label!r} is not one distinct token after the "
|
||||
"chat prompt for this tokenizer, so this model is not supported"
|
||||
)
|
||||
label_ids.append(token_id)
|
||||
return label_ids
|
||||
|
||||
|
||||
def label_mass(token_logprobs: List[float]) -> float:
|
||||
return math.fsum(math.exp(logprob) for logprob in token_logprobs)
|
||||
|
||||
|
||||
def _build_answer(
|
||||
question: DecisionQuestion,
|
||||
view: QuestionView,
|
||||
probabilities: List[float],
|
||||
token_logprobs: List[float],
|
||||
) -> DecisionAnswer:
|
||||
names = view.names
|
||||
value = {}
|
||||
if view.kind == "choice":
|
||||
value["choice"] = names[probabilities.index(max(probabilities))]
|
||||
elif view.kind == "score":
|
||||
value["score"] = math.fsum(i * p for i, p in enumerate(probabilities))
|
||||
return DecisionAnswer(
|
||||
type=question.type,
|
||||
probabilities=dict(zip(names, probabilities)),
|
||||
label_mass=label_mass(token_logprobs),
|
||||
**value,
|
||||
)
|
||||
@@ -0,0 +1,150 @@
|
||||
"""Pydantic models for the System One decision API.
|
||||
|
||||
Follows the published System One OpenAPI 0.2.0 request and response shapes:
|
||||
a state, a map of noul, choice, and score questions keyed by caller ids, and
|
||||
one answer per question id.
|
||||
"""
|
||||
|
||||
from typing import Annotated, Any, Dict, List, Literal, Optional, Union
|
||||
|
||||
from pydantic import (
|
||||
BaseModel,
|
||||
ConfigDict,
|
||||
Field,
|
||||
NonNegativeInt,
|
||||
ValidationInfo,
|
||||
field_validator,
|
||||
model_validator,
|
||||
)
|
||||
from pydantic.json_schema import SkipJsonSchema
|
||||
|
||||
from sglang.srt.entrypoints.openai.protocol import (
|
||||
DecisionText,
|
||||
RequiredDecisionText,
|
||||
check_option_names,
|
||||
is_blank_decision_text,
|
||||
)
|
||||
|
||||
# The documented maximum. Options beyond 26 get two-letter labels, checked per request.
|
||||
MAX_CHOICE_OPTIONS = 255
|
||||
# Levels are labeled 0 to 9, so at most 10.
|
||||
MAX_SCORE_LEVELS = 10
|
||||
|
||||
|
||||
class _Question(BaseModel):
|
||||
# Misspelled keys inside a question would otherwise answer a different question.
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
instructions: Optional[DecisionText] = None
|
||||
|
||||
|
||||
class SystemOneNoulCriteria(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
true: Optional[DecisionText] = None
|
||||
false: Optional[DecisionText] = None
|
||||
|
||||
|
||||
class SystemOneNoulQuestion(_Question):
|
||||
type: Literal["noul"]
|
||||
criteria: Optional[SystemOneNoulCriteria] = None
|
||||
|
||||
@model_validator(mode="after")
|
||||
def _asks_something(self):
|
||||
criteria = self.criteria or SystemOneNoulCriteria()
|
||||
if all(
|
||||
is_blank_decision_text(value)
|
||||
for value in (self.instructions, criteria.true, criteria.false)
|
||||
):
|
||||
raise ValueError(
|
||||
"a noul question needs instructions or a true or false "
|
||||
"description to decide on"
|
||||
)
|
||||
return self
|
||||
|
||||
|
||||
class SystemOneChoiceQuestion(_Question):
|
||||
type: Literal["choice"]
|
||||
criteria: Dict[str, Optional[DecisionText]] = Field(
|
||||
min_length=1, max_length=MAX_CHOICE_OPTIONS
|
||||
)
|
||||
|
||||
@field_validator("criteria")
|
||||
@classmethod
|
||||
def _option_names_distinct(cls, criteria):
|
||||
check_option_names(criteria)
|
||||
return criteria
|
||||
|
||||
|
||||
class SystemOneScoreQuestion(_Question):
|
||||
type: Literal["score"]
|
||||
criteria: List[RequiredDecisionText] = Field(
|
||||
min_length=1, max_length=MAX_SCORE_LEVELS
|
||||
)
|
||||
|
||||
|
||||
SystemOneQuestion = Annotated[
|
||||
Union[SystemOneNoulQuestion, SystemOneChoiceQuestion, SystemOneScoreQuestion],
|
||||
Field(discriminator="type"),
|
||||
]
|
||||
|
||||
|
||||
class SystemOneRequest(BaseModel):
|
||||
# Unknown top-level fields are ignored, as the published schema allows.
|
||||
state: DecisionText
|
||||
model: str
|
||||
questions: Dict[str, SystemOneQuestion] = Field(min_length=1)
|
||||
# SGLang extension, for chat templates whose reasoning toggle needs a kwarg.
|
||||
chat_template_kwargs: Dict[str, Any] = Field(default_factory=dict)
|
||||
|
||||
# /v1/decisions fields, which would change the answers if honored or ignored.
|
||||
# Declared only to refuse them by name, and hidden from the schema.
|
||||
temperature: SkipJsonSchema[Any] = None
|
||||
prompt_format_version: SkipJsonSchema[Any] = None
|
||||
return_prompt_token_ids: SkipJsonSchema[Any] = None
|
||||
|
||||
@field_validator("temperature", "prompt_format_version", "return_prompt_token_ids")
|
||||
@classmethod
|
||||
def _refuse_decisions_fields(cls, value, info: ValidationInfo):
|
||||
if value is not None:
|
||||
raise ValueError(
|
||||
f"{info.field_name} is not part of this API, use /v1/decisions for it"
|
||||
)
|
||||
return value
|
||||
|
||||
|
||||
class SystemOneNoulAnswer(BaseModel):
|
||||
type: Literal["noul"] = "noul"
|
||||
noul: float
|
||||
# Full-vocabulary probability of the answer labels, an SGLang extension.
|
||||
x_label_mass: float
|
||||
|
||||
|
||||
class SystemOneChoiceAnswer(BaseModel):
|
||||
type: Literal["choice"] = "choice"
|
||||
choice: str
|
||||
confidence: float
|
||||
probabilities: Dict[str, float]
|
||||
x_label_mass: float
|
||||
|
||||
|
||||
class SystemOneScoreAnswer(BaseModel):
|
||||
type: Literal["score"] = "score"
|
||||
score: float
|
||||
confidence: float
|
||||
legend: Dict[str, Any]
|
||||
probabilities: Dict[str, float]
|
||||
x_label_mass: float
|
||||
|
||||
|
||||
class SystemOneUsage(BaseModel):
|
||||
input_tokens: NonNegativeInt
|
||||
output_tokens: NonNegativeInt = 0
|
||||
|
||||
|
||||
class SystemOneResponse(BaseModel):
|
||||
model: str
|
||||
answers: Dict[
|
||||
str, Union[SystemOneNoulAnswer, SystemOneChoiceAnswer, SystemOneScoreAnswer]
|
||||
]
|
||||
usage: SystemOneUsage
|
||||
@@ -0,0 +1,258 @@
|
||||
"""Handler for the System One compatible decision API, on the /v1/decisions scoring path."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import math
|
||||
import string
|
||||
from typing import Any, Dict, Iterator, List, Optional, Tuple
|
||||
|
||||
import orjson
|
||||
from fastapi import Request
|
||||
from fastapi.responses import ORJSONResponse
|
||||
|
||||
from sglang.srt.entrypoints.openai.serving_decisions import (
|
||||
OpenAIServingDecisions,
|
||||
QuestionView,
|
||||
default_labels,
|
||||
label_context,
|
||||
label_mass,
|
||||
label_token_id,
|
||||
render_text,
|
||||
)
|
||||
from sglang.srt.entrypoints.systemone.protocol import (
|
||||
SystemOneChoiceAnswer,
|
||||
SystemOneChoiceQuestion,
|
||||
SystemOneNoulAnswer,
|
||||
SystemOneNoulQuestion,
|
||||
SystemOneQuestion,
|
||||
SystemOneRequest,
|
||||
SystemOneResponse,
|
||||
SystemOneScoreAnswer,
|
||||
SystemOneUsage,
|
||||
)
|
||||
|
||||
# Beyond A to Z, every option gets a two-letter label, in this fixed order.
|
||||
_PAIR_LABELS = [a + b for a in string.ascii_uppercase for b in string.ascii_uppercase]
|
||||
|
||||
|
||||
class SystemOneServing(OpenAIServingDecisions):
|
||||
"""Answers System One questions with the rendering, label checks, and scoring of /v1/decisions."""
|
||||
|
||||
route = "/v1/systemone"
|
||||
|
||||
def _request_id_prefix(self) -> str:
|
||||
return "systemone-"
|
||||
|
||||
def _validate_request(self, request: SystemOneRequest) -> Optional[str]:
|
||||
return self._validate_server(request.model) or self._validate_reasoning(
|
||||
request.chat_template_kwargs
|
||||
)
|
||||
|
||||
def _convert_to_internal_request(
|
||||
self,
|
||||
request: SystemOneRequest,
|
||||
raw_request: Request = None,
|
||||
) -> Tuple[
|
||||
Iterator[Tuple[List[int], List[int]]],
|
||||
Tuple[SystemOneRequest, List[QuestionView]],
|
||||
]:
|
||||
views = [_view(question) for question in request.questions.values()]
|
||||
# Lazy, so the async handler can yield to other requests between questions.
|
||||
return self._encoded_systemone_questions(request, views), (request, views)
|
||||
|
||||
def _encoded_systemone_questions(
|
||||
self, request: SystemOneRequest, views: List[QuestionView]
|
||||
) -> Iterator[Tuple[List[int], List[int]]]:
|
||||
"""Prompt and label ids for each question, in request order."""
|
||||
text = render_text(request.state)
|
||||
chat_template_kwargs = self._chat_template_kwargs(request.chat_template_kwargs)
|
||||
pair_labels = None
|
||||
for question_id, view in zip(request.questions, views):
|
||||
try:
|
||||
labels = default_labels(view)
|
||||
if view.kind == "choice" and len(view.names) > len(labels):
|
||||
if pair_labels is None:
|
||||
pair_labels = self._pair_labels(chat_template_kwargs)
|
||||
if len(view.names) > len(pair_labels):
|
||||
raise ValueError(
|
||||
f"it has {len(view.names)} options, but the served "
|
||||
"tokenizer and chat template can label at most "
|
||||
f"{max(len(pair_labels), len(string.ascii_uppercase))}"
|
||||
)
|
||||
labels = pair_labels[: len(view.names)]
|
||||
if view.kind == "score":
|
||||
_check_legend(question_id, view)
|
||||
encoded = self._encode_question(
|
||||
text=text,
|
||||
view=view,
|
||||
labels=labels,
|
||||
chat_template_kwargs=chat_template_kwargs,
|
||||
)
|
||||
except ValueError as e:
|
||||
raise ValueError(f"question {question_id!r}: {e}") from e
|
||||
yield encoded
|
||||
|
||||
def _pair_labels(self, chat_template_kwargs: Dict[str, Any]) -> List[str]:
|
||||
"""Two-letter labels that are distinct single tokens at the answer position."""
|
||||
if not self.added_tokens:
|
||||
raise ValueError(
|
||||
"more than 26 options needs added tokens the server can read "
|
||||
"from the tokenizer, and the served tokenizer reports none"
|
||||
)
|
||||
tokenizer = self.tokenizer_manager.tokenizer
|
||||
contexts = []
|
||||
for message in ("x", "y"):
|
||||
prompt = self._apply_chat_template(message, chat_template_kwargs)
|
||||
prompt_ids = tokenizer.encode(prompt, add_special_tokens=False)
|
||||
contexts.append(
|
||||
label_context(tokenizer, prompt, prompt_ids, self.added_tokens)
|
||||
)
|
||||
(text, text_ids, shortcut), other = contexts
|
||||
# Labels must be checked on text that excludes the message. Otherwise
|
||||
# every label re-encodes the whole input on the event loop, which is
|
||||
# unbounded work for hundreds of options.
|
||||
if not (shortcut and other[2] and other[0] == text):
|
||||
raise ValueError(
|
||||
"more than 26 options needs an added token between the message "
|
||||
"and the answer position, after which the text tokenizes the same on "
|
||||
"its own, which this tokenizer and chat template do not provide"
|
||||
)
|
||||
labels, label_ids = [], set()
|
||||
for label in _PAIR_LABELS:
|
||||
token_id = label_token_id(tokenizer, text, text_ids, label)
|
||||
if token_id is not None and token_id not in label_ids:
|
||||
labels.append(label)
|
||||
label_ids.add(token_id)
|
||||
# Each final prompt is still checked as a whole in _encode_question.
|
||||
return labels
|
||||
|
||||
async def _handle_non_streaming_request(
|
||||
self,
|
||||
adapted_request: Iterator[Tuple[List[int], List[int]]],
|
||||
processed: Tuple[SystemOneRequest, List[QuestionView]],
|
||||
raw_request: Request,
|
||||
) -> ORJSONResponse:
|
||||
request, views = processed
|
||||
_, _, result = await self._score(
|
||||
adapted_request=adapted_request, raw_request=raw_request
|
||||
)
|
||||
answers = {}
|
||||
for i, question_id in enumerate(request.questions):
|
||||
answers[question_id] = _answer(
|
||||
view=views[i],
|
||||
probabilities=result.scores[i],
|
||||
mass=label_mass(result.token_logprobs[i]),
|
||||
question_id=question_id,
|
||||
)
|
||||
response = SystemOneResponse(
|
||||
# The served model answered, whatever name the request used.
|
||||
model=self.tokenizer_manager.served_model_name,
|
||||
answers=answers,
|
||||
usage=SystemOneUsage(input_tokens=result.prompt_tokens),
|
||||
)
|
||||
return ORJSONResponse(content=response.model_dump())
|
||||
|
||||
|
||||
def _view(question: SystemOneQuestion) -> QuestionView:
|
||||
if isinstance(question, SystemOneChoiceQuestion):
|
||||
return QuestionView(
|
||||
kind="choice",
|
||||
question=question.instructions,
|
||||
names=list(question.criteria),
|
||||
details=list(question.criteria.values()),
|
||||
)
|
||||
if isinstance(question, SystemOneNoulQuestion):
|
||||
criteria = question.criteria
|
||||
return QuestionView(
|
||||
kind="yes_no",
|
||||
question=question.instructions,
|
||||
names=["yes", "no"],
|
||||
details=[
|
||||
criteria.true if criteria else None,
|
||||
criteria.false if criteria else None,
|
||||
],
|
||||
)
|
||||
return QuestionView(
|
||||
kind="score",
|
||||
question=question.instructions,
|
||||
names=[str(level) for level in range(len(question.criteria))],
|
||||
details=list(question.criteria),
|
||||
)
|
||||
|
||||
|
||||
def _answer(
|
||||
view: QuestionView,
|
||||
probabilities: List[float],
|
||||
mass: float,
|
||||
question_id: str,
|
||||
):
|
||||
if not all(math.isfinite(value) for value in [*probabilities, mass]):
|
||||
# A server fault, reported as 500 rather than as a client error.
|
||||
raise RuntimeError(f"question {question_id!r} scored non-finite values")
|
||||
# Reported as scored, like /v1/decisions, and normalized only for confidence.
|
||||
if view.kind == "yes_no":
|
||||
return SystemOneNoulAnswer(noul=probabilities[0], x_label_mass=mass)
|
||||
probabilities_by_name = dict(zip(view.names, probabilities))
|
||||
if view.kind == "choice":
|
||||
return SystemOneChoiceAnswer(
|
||||
choice=view.names[probabilities.index(max(probabilities))],
|
||||
confidence=_choice_confidence(_normalized(probabilities)),
|
||||
probabilities=probabilities_by_name,
|
||||
x_label_mass=mass,
|
||||
)
|
||||
return SystemOneScoreAnswer(
|
||||
score=math.fsum(i * p for i, p in enumerate(probabilities)),
|
||||
confidence=_score_confidence(_normalized(probabilities)),
|
||||
legend=_legend(view),
|
||||
probabilities=probabilities_by_name,
|
||||
x_label_mass=mass,
|
||||
)
|
||||
|
||||
|
||||
def _legend(view: QuestionView) -> Dict[str, Any]:
|
||||
return dict(zip(view.names, view.details))
|
||||
|
||||
|
||||
def _check_legend(question_id: str, view: QuestionView) -> None:
|
||||
"""Refuse, before scoring, levels that the response cannot echo in its legend."""
|
||||
answer = SystemOneScoreAnswer(
|
||||
score=0.0,
|
||||
confidence=0.0,
|
||||
legend=_legend(view),
|
||||
probabilities={},
|
||||
x_label_mass=0.0,
|
||||
)
|
||||
response = SystemOneResponse(
|
||||
model="", answers={question_id: answer}, usage=SystemOneUsage(input_tokens=0)
|
||||
)
|
||||
try:
|
||||
# The same encoder and options as the real response.
|
||||
ORJSONResponse(content=response.model_dump())
|
||||
except orjson.JSONEncodeError as e:
|
||||
raise ValueError(f"a level cannot be returned in the legend: {e}") from e
|
||||
|
||||
|
||||
def _normalized(probabilities: List[float]) -> List[float]:
|
||||
total = math.fsum(probabilities)
|
||||
if total <= 0:
|
||||
return [1.0 / len(probabilities)] * len(probabilities)
|
||||
return [p / total for p in probabilities]
|
||||
|
||||
|
||||
def _choice_confidence(q: List[float]) -> float:
|
||||
"""How far the top option stands above a uniform guess, from 0 to 1."""
|
||||
n = len(q)
|
||||
if n == 1:
|
||||
return 1.0
|
||||
return min(1.0, max(0.0, (n * max(q) - 1) / (n - 1)))
|
||||
|
||||
|
||||
def _score_confidence(q: List[float]) -> float:
|
||||
"""One minus the spread around the top level relative to a uniform spread, floored at 0."""
|
||||
n = len(q)
|
||||
if n == 1:
|
||||
return 1.0
|
||||
top = q.index(max(q))
|
||||
spread = math.fsum(p * abs(i - top) for i, p in enumerate(q))
|
||||
uniform_spread = math.fsum(abs(i - (n - 1) / 2) for i in range(n)) / n
|
||||
return max(0.0, 1 - spread / uniform_spread)
|
||||
@@ -155,15 +155,17 @@ def get_token_ids_logprobs_raw(
|
||||
if stage == LogprobStage.DECODE:
|
||||
for i, token_ids in enumerate(token_ids_logprobs_list):
|
||||
if token_ids is None:
|
||||
vals.append([])
|
||||
idxs.append([])
|
||||
# The CPU copy expects a tensor in every entry under no_copy_to_cpu.
|
||||
# A new empty one does not keep the full logprobs alive like a view.
|
||||
row = logprobs.new_empty((0,))
|
||||
token_ids = []
|
||||
else:
|
||||
token_ids_tensor = torch.tensor(
|
||||
token_ids, dtype=torch.long, pin_memory=pin_memory
|
||||
).to(logprobs.device, non_blocking=True)
|
||||
row = logprobs[i, token_ids_tensor]
|
||||
vals.append(row if no_copy_to_cpu else row.tolist())
|
||||
idxs.append(token_ids)
|
||||
vals.append(row if no_copy_to_cpu else row.tolist())
|
||||
idxs.append(token_ids)
|
||||
else: # prefill
|
||||
pt = 0
|
||||
for i, (token_ids, pruned_len) in enumerate(
|
||||
|
||||
@@ -2853,7 +2853,10 @@ class TokenizerManager(TokenizerControlMixin, TokenizerManagerScoreMixin):
|
||||
)
|
||||
|
||||
if top_logprobs_num > 0:
|
||||
if len(recv_obj.input_top_logprobs_val) > 0:
|
||||
if (
|
||||
recv_obj.input_top_logprobs_val is not None
|
||||
and len(recv_obj.input_top_logprobs_val) > 0
|
||||
):
|
||||
state.input_top_logprobs_val.extend(
|
||||
recv_obj.input_top_logprobs_val[recv_obj_index]
|
||||
)
|
||||
@@ -2869,27 +2872,32 @@ class TokenizerManager(TokenizerControlMixin, TokenizerManagerScoreMixin):
|
||||
recv_obj.input_top_logprobs_idx_flat[recv_obj_index],
|
||||
recv_obj.input_top_logprobs_flat_null_prefix[recv_obj_index],
|
||||
)
|
||||
state.output_top_logprobs_val.extend(
|
||||
recv_obj.output_top_logprobs_val[recv_obj_index]
|
||||
)
|
||||
state.output_top_logprobs_idx.extend(
|
||||
recv_obj.output_top_logprobs_idx[recv_obj_index]
|
||||
)
|
||||
if recv_obj.output_top_logprobs_val is not None:
|
||||
state.output_top_logprobs_val.extend(
|
||||
recv_obj.output_top_logprobs_val[recv_obj_index]
|
||||
)
|
||||
state.output_top_logprobs_idx.extend(
|
||||
recv_obj.output_top_logprobs_idx[recv_obj_index]
|
||||
)
|
||||
|
||||
if token_ids_logprob is not None:
|
||||
if len(recv_obj.input_token_ids_logprobs_val) > 0:
|
||||
if (
|
||||
recv_obj.input_token_ids_logprobs_val is not None
|
||||
and len(recv_obj.input_token_ids_logprobs_val) > 0
|
||||
):
|
||||
state.input_token_ids_logprobs_val.extend(
|
||||
recv_obj.input_token_ids_logprobs_val[recv_obj_index]
|
||||
)
|
||||
state.input_token_ids_logprobs_idx.extend(
|
||||
recv_obj.input_token_ids_logprobs_idx[recv_obj_index]
|
||||
)
|
||||
state.output_token_ids_logprobs_val.extend(
|
||||
recv_obj.output_token_ids_logprobs_val[recv_obj_index]
|
||||
)
|
||||
state.output_token_ids_logprobs_idx.extend(
|
||||
recv_obj.output_token_ids_logprobs_idx[recv_obj_index]
|
||||
)
|
||||
if recv_obj.output_token_ids_logprobs_val is not None:
|
||||
state.output_token_ids_logprobs_val.extend(
|
||||
recv_obj.output_token_ids_logprobs_val[recv_obj_index]
|
||||
)
|
||||
state.output_token_ids_logprobs_idx.extend(
|
||||
recv_obj.output_token_ids_logprobs_idx[recv_obj_index]
|
||||
)
|
||||
|
||||
self.add_logprob_to_meta_info(
|
||||
meta_info,
|
||||
|
||||
Reference in New Issue
Block a user