[feat] add a system one compatible /v1/systemone route (#41208)

This commit is contained in:
Richard Wang
2026-09-25 12:59:11 +08:00
committed by GitHub
parent cec70a4708
commit 16d1c93b3d
19 changed files with 2803 additions and 33 deletions
+46 -2
View File
@@ -58,6 +58,7 @@ from fastapi import (
UploadFile,
WebSocket,
)
from fastapi.encoders import jsonable_encoder
from fastapi.exceptions import RequestValidationError
from fastapi.middleware.cors import CORSMiddleware
from fastapi.responses import ORJSONResponse, Response, StreamingResponse
@@ -88,6 +89,7 @@ from sglang.srt.entrypoints.openai.protocol import (
ChatCompletionRequest,
ClassifyRequest,
CompletionRequest,
DecisionRequest,
DetokenizeRequest,
EmbeddingRequest,
ErrorResponse,
@@ -100,6 +102,7 @@ from sglang.srt.entrypoints.openai.protocol import (
)
from sglang.srt.entrypoints.openai.serving_classify import OpenAIServingClassify
from sglang.srt.entrypoints.openai.serving_completions import OpenAIServingCompletion
from sglang.srt.entrypoints.openai.serving_decisions import OpenAIServingDecisions
from sglang.srt.entrypoints.openai.serving_embedding import OpenAIServingEmbedding
from sglang.srt.entrypoints.openai.serving_rerank import OpenAIServingRerank
from sglang.srt.entrypoints.openai.serving_score import OpenAIServingScore
@@ -111,6 +114,8 @@ from sglang.srt.entrypoints.openai.serving_transcription import (
OpenAIServingTranscription,
)
from sglang.srt.entrypoints.request_headers import apply_header_overrides
from sglang.srt.entrypoints.systemone.protocol import SystemOneRequest
from sglang.srt.entrypoints.systemone.serving import SystemOneServing
from sglang.srt.entrypoints.warmup import execute_warmups
from sglang.srt.environ import envs
from sglang.srt.function_call.function_call_parser import FunctionCallParser
@@ -323,6 +328,9 @@ async def lifespan(fast_api_app: FastAPI):
fast_api_app.state.openai_serving_score = OpenAIServingScore(
_global_state.tokenizer_manager
)
fast_api_app.state.openai_serving_decisions = OpenAIServingDecisions(
fast_api_app.state.openai_serving_chat
)
fast_api_app.state.openai_serving_rerank = OpenAIServingRerank(
_global_state.tokenizer_manager, _global_state.template_manager
)
@@ -344,6 +352,11 @@ async def lifespan(fast_api_app: FastAPI):
fast_api_app.state.openai_serving_chat
)
# Initialize System One compatible decision handler
fast_api_app.state.systemone_serving = SystemOneServing(
fast_api_app.state.openai_serving_chat
)
# Launch tool server
tool_server = None
if get_serving().tool_server == "demo":
@@ -603,8 +616,9 @@ async def validation_exception_handler(request: Request, exc: RequestValidationE
For /v1/messages, emit Anthropic-style envelope and scrub the message so
file paths or Python internals from the default ``str(exc)`` representation
never reach the client. For /v1/responses, keep OpenAI-style. Otherwise
use the legacy ErrorResponse shape.
never reach the client. For /v1/responses, keep OpenAI-style. For
/v1/systemone, keep the 422 detail list that the System One API documents.
Otherwise use the legacy ErrorResponse shape.
"""
if request.url.path.startswith("/v1/messages"):
return _anthropic_error_response(
@@ -613,6 +627,19 @@ async def validation_exception_handler(request: Request, exc: RequestValidationE
message=_anthropic_validation_message(exc.errors()),
)
route_path = request.url.path.removeprefix(request.scope.get("root_path", ""))
if route_path == "/v1/systemone":
# The System One API documents FastAPI's default 422 detail list. The
# optional input echo is left out, since it can be any client value.
detail = [
{key: value for key, value in error.items() if key != "input"}
for error in exc.errors()
]
return ORJSONResponse(
status_code=HTTPStatus.UNPROCESSABLE_ENTITY.value,
content={"detail": jsonable_encoder(detail)},
)
exc_str = str(exc)
errors_str = str(exc.errors())
@@ -1933,6 +1960,14 @@ async def v1_score_request(request: ScoringRequest, raw_request: Request):
)
@app.post("/v1/decisions", dependencies=[Depends(validate_json_request)])
async def v1_decisions_request(request: DecisionRequest, raw_request: Request):
"""Answer typed choice, score, and yes or no questions about an input by scoring single-token answer labels through the scoring API, without generation."""
return await raw_request.app.state.openai_serving_decisions.handle_request(
request, raw_request
)
@app.post("/v1/responses", dependencies=[Depends(validate_json_request)])
async def v1_responses_request(request: ResponsesRequest, raw_request: Request):
"""Endpoint for the responses API with reasoning support."""
@@ -2047,6 +2082,15 @@ async def anthropic_v1_count_tokens(
)
## System One compatible decision API
@app.post("/v1/systemone", dependencies=[Depends(validate_json_request)])
async def systemone_decisions(request: SystemOneRequest, raw_request: Request):
"""System One compatible decisions, answered by candidate scoring without generation."""
return await raw_request.app.state.systemone_serving.handle_request(
request, raw_request
)
## SageMaker API
@app.get("/ping")
async def sagemaker_health() -> Response:
@@ -17,6 +17,7 @@ from __future__ import annotations
import logging
import time
import unicodedata
import uuid
from dataclasses import dataclass
from typing import (
@@ -1438,6 +1439,137 @@ class ScoringResponse(BaseModel):
object: str = "scoring"
def is_blank_decision_text(value) -> bool:
return not (value.strip() if isinstance(value, str) else value)
def _nonblank_decision_text(value):
if is_blank_decision_text(value):
raise ValueError("must not be blank")
return value
def check_option_names(names) -> None:
"""Refuse option names that would make the rendered option lines ambiguous."""
seen = set()
for name in names:
key = name.strip().casefold()
if not key:
raise ValueError("option names must be nonempty")
# Each option is rendered as one prompt line.
if any(unicodedata.category(c) in ("Cc", "Zl", "Zp") for c in name):
raise ValueError(
f"option name {name!r} must not contain control or line break "
"characters"
)
if key in seen:
raise ValueError(f"option name {name!r} repeats another option")
seen.add(key)
# Objects and arrays are rendered into the prompt as compact JSON.
DecisionText = Union[str, Dict[str, Any], List[Any]]
RequiredDecisionText = Annotated[DecisionText, AfterValidator(_nonblank_decision_text)]
class DecisionOption(BaseModel):
model_config = ConfigDict(extra="forbid")
name: str
description: Optional[DecisionText] = None
class DecisionChoiceQuestion(BaseModel):
model_config = ConfigDict(extra="forbid")
id: Annotated[str, AfterValidator(_nonblank_decision_text)]
type: Literal["choice"]
question: RequiredDecisionText
# Options are labeled A to Z in order, so at most 26.
options: List[DecisionOption] = Field(min_length=2, max_length=26)
@model_validator(mode="after")
def _option_names_distinct(self):
try:
check_option_names(option.name for option in self.options)
except ValueError as e:
raise ValueError(f"question {self.id!r}: {e}") from None
return self
class DecisionScoreQuestion(BaseModel):
model_config = ConfigDict(extra="forbid")
id: Annotated[str, AfterValidator(_nonblank_decision_text)]
type: Literal["score"]
question: RequiredDecisionText
# Levels are labeled 0 to 9 in order, so at most 10.
levels: List[RequiredDecisionText] = Field(min_length=2, max_length=10)
class DecisionYesNoQuestion(BaseModel):
model_config = ConfigDict(extra="forbid")
id: Annotated[str, AfterValidator(_nonblank_decision_text)]
type: Literal["yes_no"]
question: RequiredDecisionText
yes: Optional[DecisionText] = None
no: Optional[DecisionText] = None
DecisionQuestion = Annotated[
Union[DecisionChoiceQuestion, DecisionScoreQuestion, DecisionYesNoQuestion],
Field(discriminator="type"),
]
class DecisionRequest(BaseModel):
model_config = ConfigDict(extra="forbid")
input: RequiredDecisionText
questions: List[DecisionQuestion] = Field(min_length=1)
# Scales option probabilities only, not label_mass.
temperature: float = Field(default=1.0, gt=0, allow_inf_nan=False)
# Applied over the server defaults, with the template reasoning toggle off.
chat_template_kwargs: Dict[str, Any] = Field(default_factory=dict)
# Pins the server-owned prompt wording. A different served version is refused.
prompt_format_version: Optional[int] = None
return_prompt_token_ids: bool = False
model: str = DEFAULT_MODEL_NAME
@field_validator("questions")
@classmethod
def _question_ids_distinct(cls, questions):
seen = set()
for question in questions:
if question.id in seen:
raise ValueError(
f"question id {question.id!r} repeats another question"
)
seen.add(question.id)
return questions
class DecisionAnswer(BaseModel):
type: Literal["choice", "score", "yes_no"]
probabilities: Dict[str, float]
# Full-vocabulary probability of all answer labels at the answer position.
label_mass: float
choice: Optional[str] = None
score: Optional[float] = None
# The exact /v1/score inputs, with return_prompt_token_ids.
prompt_token_ids: Optional[List[int]] = None
label_token_ids: Optional[List[int]] = None
class DecisionResponse(BaseModel):
object: str = "decisions"
model: str
prompt_format_version: int
answers: Dict[str, DecisionAnswer]
usage: UsageInfo
class V1RerankReqInput(BaseModel):
query: RerankContent = Field(
...,
@@ -1563,6 +1695,7 @@ OpenAIServingRequest = Union[
EmbeddingRequest,
ClassifyRequest,
ScoringRequest,
DecisionRequest,
V1RerankReqInput,
TokenizeRequest,
DetokenizeRequest,
@@ -2445,7 +2445,7 @@ class OpenAIServingChat(OpenAIServingBase):
output_token_logprobs = ret_item["meta_info"]["output_token_logprobs"]
output_top_logprobs = ret_item["meta_info"].get("output_top_logprobs", None)
token_logprobs = self._build_token_logprobs_from_raw(
output_token_logprobs, output_top_logprobs, use_token_index=True
output_token_logprobs, output_top_logprobs
)
return ChoiceLogprobs(content=token_logprobs)
@@ -2453,7 +2453,6 @@ class OpenAIServingChat(OpenAIServingBase):
self,
output_token_logprobs: list[Any],
output_top_logprobs: list[Any] | None,
use_token_index: bool = False,
) -> list[ChatCompletionTokenLogprob]:
"""Build OpenAI ChatCompletionTokenLogprob from the engine's raw
``(logprob, token_id, token_text)`` triples.
@@ -2487,13 +2486,9 @@ class OpenAIServingChat(OpenAIServingBase):
top_logprobs: list[TopLogprob] = []
if output_top_logprobs:
# - Non-streaming (use_token_index=True): output_top_logprobs is
# the full per-position list; take the row for this token.
# - Streaming (use_token_index=False): rows are pre-sliced so the
# current chunk holds exactly one row at index 0.
top_row_idx = token_idx if use_token_index else 0
if top_row_idx < len(output_top_logprobs):
top_row = output_top_logprobs[top_row_idx]
# Both callers pass rows aligned with output_token_logprobs.
if token_idx < len(output_top_logprobs):
top_row = output_top_logprobs[token_idx]
if top_row is not None:
for top_logprob, top_id, top_text in top_row:
if is_byte_level:
@@ -2686,7 +2681,7 @@ class OpenAIServingChat(OpenAIServingBase):
n_prev_token:total_output_logprobs
]
token_logprobs = self._build_token_logprobs_from_raw(
output_token_logprobs, output_top_logprobs, use_token_index=False
output_token_logprobs, output_top_logprobs
)
return ChoiceLogprobs(content=token_logprobs)
@@ -0,0 +1,535 @@
from __future__ import annotations
import asyncio
import json
import logging
import math
import string
from typing import TYPE_CHECKING, Any, Dict, Iterator, List, Optional, Tuple
import msgspec
from fastapi import Request
from fastapi.responses import ORJSONResponse
from transformers import PreTrainedTokenizerBase
from sglang.srt.entrypoints.openai.protocol import (
DecisionAnswer,
DecisionChoiceQuestion,
DecisionQuestion,
DecisionRequest,
DecisionResponse,
DecisionScoreQuestion,
DecisionText,
UsageInfo,
is_blank_decision_text,
)
from sglang.srt.entrypoints.openai.serving_base import OpenAIServingBase
from sglang.srt.entrypoints.openai.serving_chat import _CHAT_TEMPLATE_CLIENT_ERRORS
from sglang.srt.parser.reasoning_parser import ReasoningParser
from sglang.srt.runtime_context import get_exec
if TYPE_CHECKING:
from sglang.srt.entrypoints.openai.serving_chat import OpenAIServingChat
logger = logging.getLogger(__name__)
# Version of the server-owned prompt wording and answer labels.
# Any change to a rendering /v1/decisions can produce needs a new version.
PROMPT_FORMAT_VERSION = 1
# Parser defaults that name the chat template kwarg toggling reasoning.
_PARSER_TOGGLE_MODES = (
"thinking",
"enable_thinking",
"explicit_thinking",
"explicit_enable_thinking",
)
# Answer text of a finished reply, rendered only to see what precedes an answer.
_REPLY_SENTINEL = "DECISION_ANSWER"
class QuestionView(msgspec.Struct, frozen=True):
"""A question as the renderer and scorer see it, shared by the decision routes."""
# choice, score, or yes_no
kind: str
# None or blank when the question has no text of its own
question: Any
# Option names, level indices, or yes and no, in candidate order
names: List[str]
# Option descriptions, levels, or the yes and no descriptions
details: List[Any]
class OpenAIServingDecisions(OpenAIServingBase):
"""Handler for /v1/decisions requests, answered by candidate scoring without generation"""
# Named in setup refusals, so each decision route reports itself.
route = "/v1/decisions"
def __init__(self, chat_serving: OpenAIServingChat):
super().__init__(chat_serving.tokenizer_manager)
# Render the way the chat route does, and refuse where it renders differently.
self.template_manager = chat_serving.template_manager
self.default_chat_template_kwargs = chat_serving.default_chat_template_kwargs
self.chat_encoding_spec = chat_serving.chat_encoding_spec
self.prompt_text_is_lossy = chat_serving._prompt_text_round_trip_is_lossy
tokenizer = self.tokenizer_manager.tokenizer
# Other tokenizers skip the shortcut in _encode_labels and check the full prompt.
self.added_tokens = (
{i: token for token, i in tokenizer.get_added_vocab().items()}
if isinstance(tokenizer, PreTrainedTokenizerBase)
else {}
)
# The configured reasoning parser, or the one the chat template suggests,
# tells where reasoning blocks start and end and whether answers open one.
parser = (
chat_serving.reasoning_parser
or self.template_manager.suggested_reasoning_parser
)
self.reasoning_markers = None
self.answers_open_reasoning = False
# The kwarg that turns reasoning on and off: the template's, else, when
# detection finds no config, the one the configured or suggested parser names.
config = self.template_manager.reasoning_config
self.reasoning_toggle = config.toggle_param if config is not None else None
if parser is not None:
try:
detector = ReasoningParser(
model_type=parser, tokenizer=tokenizer
).detector
except ValueError as e:
logger.warning(
"No reasoning block check for %s with parser '%s': %s",
self.route,
parser,
e,
)
else:
mode = detector.reasoning_default
if config is None and mode in _PARSER_TOGGLE_MODES:
self.reasoning_toggle = mode.removeprefix("explicit_")
if detector.think_start_token and detector.think_end_token:
self.reasoning_markers = (
detector.think_start_token,
detector.think_end_token,
)
self.answers_open_reasoning = detector.reasoning_default == "always"
def _request_id_prefix(self) -> str:
return "decision-"
def _validate_request(self, request: DecisionRequest) -> Optional[str]:
error = self._validate_server(request.model)
if error is not None:
return error
version = request.prompt_format_version
if version is not None and version != PROMPT_FORMAT_VERSION:
return (
f"prompt_format_version {version} is not served, this server "
f"uses version {PROMPT_FORMAT_VERSION}"
)
return self._validate_reasoning(request.chat_template_kwargs)
def _validate_server(self, model: str) -> Optional[str]:
"""Refuse servers this route cannot render faithfully."""
route = self.route
if not self.tokenizer_manager.is_generation:
return f"{route} requires a generation model"
if self.tokenizer_manager.tokenizer is None:
return f"{route} requires the server tokenizer"
if self.chat_encoding_spec is not None:
return (
f"{route} requires a chat template, but this model's chat "
f"route uses the {self.chat_encoding_spec!r} encoder"
)
if self.prompt_text_is_lossy:
return (
f"{route} places answer labels on the rendered chat text, "
"which this tokenizer does not encode back to the same ids"
)
if self.template_manager.chat_template_name is not None:
return (
f"{route} renders the tokenizer's Jinja chat template, but "
"this server uses the built-in chat template "
f"{self.template_manager.chat_template_name!r}"
)
if get_exec().features.enable_mis:
return f"{route} does not support --enable-mis"
if get_exec().dllm.dllm_algorithm is not None:
return (
f"{route} does not support diffusion language models "
"served with --dllm-algorithm"
)
_, adapter = self._parse_model_parameter(model)
if adapter is not None:
return (
f"model names the LoRA adapter {adapter!r}, which {route} "
"does not support"
)
return None
def _validate_reasoning(
self, chat_template_kwargs: Dict[str, Any]
) -> Optional[str]:
"""The answer position must follow the reasoning block, not sit inside it."""
config = self.template_manager.reasoning_config
if config is not None and config.always_on:
return (
f"{self.route} does not support chat templates that always "
"reason before answering"
)
toggle = self.reasoning_toggle
if toggle in chat_template_kwargs and chat_template_kwargs[toggle] is not False:
return (
f"chat_template_kwargs sets {toggle!r} to "
f"{chat_template_kwargs[toggle]!r}, but decisions need it false or unset"
)
return None
def _chat_template_kwargs(self, request_kwargs: Dict[str, Any]) -> Dict[str, Any]:
"""Reasoning off, then the server defaults, then the request kwargs."""
kwargs = {}
if self.reasoning_toggle is not None:
kwargs[self.reasoning_toggle] = False
for key, value in self.default_chat_template_kwargs.items():
kwargs.setdefault(key, value)
kwargs.update(request_kwargs)
return kwargs
def _convert_to_internal_request(
self,
request: DecisionRequest,
raw_request: Request = None,
) -> Tuple[Iterator[Tuple[List[int], List[int]]], DecisionRequest]:
# Lazy, so the handler can let other requests run between questions.
return self._encoded_questions(request), request
def _encoded_questions(
self, request: DecisionRequest
) -> Iterator[Tuple[List[int], List[int]]]:
"""Prompt and label ids for each question, in request order."""
text = render_text(request.input)
chat_template_kwargs = self._chat_template_kwargs(request.chat_template_kwargs)
for question in request.questions:
view = _decision_view(question)
try:
encoded = self._encode_question(
text=text,
view=view,
labels=default_labels(view),
chat_template_kwargs=chat_template_kwargs,
)
except ValueError as e:
raise ValueError(f"question {question.id!r}: {e}") from e
yield encoded
def _encode_question(
self,
text: str,
view: QuestionView,
labels: List[str],
chat_template_kwargs: Dict[str, Any],
) -> Tuple[List[int], List[int]]:
tokenizer = self.tokenizer_manager.tokenizer
content = _render_question(text=text, view=view, labels=labels)
prompt = self._apply_chat_template(content, chat_template_kwargs)
if self.reasoning_markers is not None:
# Look only after the message, whose last line is fixed text.
closing = content.rsplit("\n", 1)[-1]
cut = prompt.rfind(closing)
generation_prompt = prompt if cut < 0 else prompt[cut + len(closing) :]
start, end = self.reasoning_markers
opened = generation_prompt.rfind(start)
closed = generation_prompt.rfind(end)
if opened > closed:
raise ValueError(
"the chat template leaves a reasoning block open at the "
"answer position, so this model is not supported with these "
"chat_template_kwargs"
)
if self.answers_open_reasoning and closed < 0:
raise ValueError(
"the reasoning parser for this model expects answers to start "
"with a reasoning block, and the chat template does not close "
"one. Send chat_template_kwargs that turn thinking off, if the "
"template supports it"
)
# The template's own finished reply shows whether answers start with
# a reasoning block that the generation prompt leaves out.
reply = self._render_reply(closing, chat_template_kwargs)
begin = reply.rfind(closing) if reply is not None else -1
answer = reply.find(_REPLY_SENTINEL, begin) if begin >= 0 else -1
if answer >= 0 and reply[begin:answer].count(start) > (
generation_prompt.count(start)
):
raise ValueError(
"the chat template starts every answer with a reasoning "
"block, so this model is not supported"
)
prompt_ids = tokenizer.encode(prompt, add_special_tokens=False)
# Refuse here because --allow-auto-truncate would cut off the answer position.
context_len = self.tokenizer_manager.context_len
if len(prompt_ids) + self.tokenizer_manager.num_reserved_tokens >= context_len:
raise ValueError(
f"the prompt has {len(prompt_ids)} tokens, which does not fit "
f"the context length of {context_len} tokens"
)
label_ids = _encode_labels(
tokenizer=tokenizer,
prompt=prompt,
prompt_ids=prompt_ids,
labels=labels,
added_tokens=self.added_tokens,
)
return prompt_ids, label_ids
def _apply_chat_template(
self, content: str, chat_template_kwargs: Dict[str, Any]
) -> str:
try:
return self.tokenizer_manager.tokenizer.apply_chat_template(
[{"role": "user", "content": content}],
tokenize=False,
add_generation_prompt=True,
**chat_template_kwargs,
)
except _CHAT_TEMPLATE_CLIENT_ERRORS as e:
raise ValueError(f"the chat template failed: {e}") from e
def _render_reply(
self, message: str, chat_template_kwargs: Dict[str, Any]
) -> Optional[str]:
try:
return self.tokenizer_manager.tokenizer.apply_chat_template(
[
{"role": "user", "content": message},
{"role": "assistant", "content": _REPLY_SENTINEL},
],
tokenize=False,
**chat_template_kwargs,
)
except _CHAT_TEMPLATE_CLIENT_ERRORS:
# The generation prompt checks above still apply.
return None
async def _score(
self,
adapted_request: Iterator[Tuple[List[int], List[int]]],
raw_request: Request,
temperature: float = 1.0,
):
"""Encode every question, then score them all in one call."""
prompts, label_token_ids = await _encode_all(adapted_request)
result = await self.tokenizer_manager.score_prompts(
prompts=prompts,
label_token_ids=label_token_ids,
apply_softmax=True,
request=raw_request,
temperature=temperature,
return_token_logprobs=True,
)
return prompts, label_token_ids, result
async def _handle_non_streaming_request(
self,
adapted_request: Iterator[Tuple[List[int], List[int]]],
request: DecisionRequest,
raw_request: Request,
) -> ORJSONResponse:
prompts, label_token_ids, result = await self._score(
adapted_request=adapted_request,
raw_request=raw_request,
temperature=request.temperature,
)
answers = {}
for i, question in enumerate(request.questions):
answer = _build_answer(
question=question,
view=_decision_view(question),
probabilities=result.scores[i],
token_logprobs=result.token_logprobs[i],
)
if request.return_prompt_token_ids:
answer.prompt_token_ids = prompts[i]
answer.label_token_ids = label_token_ids[i]
answers[question.id] = answer
response = DecisionResponse(
model=request.model,
prompt_format_version=PROMPT_FORMAT_VERSION,
answers=answers,
usage=UsageInfo(
prompt_tokens=result.prompt_tokens,
total_tokens=result.prompt_tokens,
),
)
return ORJSONResponse(content=response.model_dump(exclude_none=True))
def render_text(value: Optional[DecisionText]) -> str:
if value is None:
return ""
if isinstance(value, str):
return value
return json.dumps(value, ensure_ascii=False, separators=(",", ":"))
async def _encode_all(
encoded: Iterator[Tuple[List[int], List[int]]],
) -> Tuple[List[List[int]], List[List[int]]]:
"""Collect prompt and label ids, letting other requests run between questions."""
prompts, label_token_ids = [], []
for prompt_ids, label_ids in encoded:
prompts.append(prompt_ids)
label_token_ids.append(label_ids)
# Each question renders and tokenizes the whole input on the event loop.
await asyncio.sleep(0)
return prompts, label_token_ids
def _decision_view(question: DecisionQuestion) -> QuestionView:
if isinstance(question, DecisionChoiceQuestion):
return QuestionView(
kind="choice",
question=question.question,
names=[option.name for option in question.options],
details=[option.description for option in question.options],
)
if isinstance(question, DecisionScoreQuestion):
return QuestionView(
kind="score",
question=question.question,
names=[str(level) for level in range(len(question.levels))],
details=list(question.levels),
)
return QuestionView(
kind="yes_no",
question=question.question,
names=["yes", "no"],
details=[question.yes, question.no],
)
def default_labels(view: QuestionView) -> List[str]:
"""Single-token labels in candidate order: A to Z, level indices, or yes and no."""
if view.kind == "choice":
return list(string.ascii_uppercase[: len(view.names)])
return list(view.names)
def _render_question(text: str, view: QuestionView, labels: List[str]) -> str:
"""Prompt wording of PROMPT_FORMAT_VERSION."""
# Every /v1/decisions question has text. A question without its own text
# drops the question line, and a yes or no question keeps its lead in.
question_text = (
"" if is_blank_decision_text(view.question) else render_text(view.question)
)
if view.kind == "choice":
lines = [f"Question: {question_text}"] if question_text else []
for label, name, description in zip(labels, view.names, view.details):
detail = render_text(description)
lines.append(
f"{label}: {name} - {detail}" if detail else f"{label}: {name}"
)
lines.append("Answer with the letter of one option only.")
elif view.kind == "score":
lines = [f"Question: {question_text}"] if question_text else []
lines += [
f"{label}: {render_text(level)}"
for label, level in zip(labels, view.details)
]
lines.append("Answer with the number of one level only.")
else:
lines = [
f"Is the following true? {question_text}"
if question_text
else "Is the following true?"
]
for label, description in zip(labels, view.details):
detail = render_text(description)
if detail:
lines.append(f"{label}: {detail}")
lines.append("Answer with yes or no only.")
return "\n".join([text, "", *lines])
def label_context(
tokenizer: Any,
prompt: str,
prompt_ids: List[int],
added_tokens: Dict[int, str],
) -> Tuple[str, List[int], bool]:
"""Text and ids after which labels are checked, and whether the shortcut applies."""
# Added tokens are split off before tokenization.
# The text after the last one tokenizes on its own,
# so the check does not grow with the input.
# When the prompt ends with an added token, each label starts a new segment,
# which is how the model continues after that token.
last = next(
(i for i in reversed(range(len(prompt_ids))) if prompt_ids[i] in added_tokens),
None,
)
if last is not None:
token = added_tokens[prompt_ids[last]]
start = prompt.rfind(token)
suffix = prompt[start + len(token) :]
if start >= 0 and (
tokenizer.encode(suffix, add_special_tokens=False) == prompt_ids[last + 1 :]
):
return suffix, prompt_ids[last + 1 :], True
return prompt, prompt_ids, False
def label_token_id(
tokenizer: Any, text: str, text_ids: List[int], label: str
) -> Optional[int]:
"""The token a label adds after the text, or None when it is not exactly one."""
ids = tokenizer.encode(text + label, add_special_tokens=False)
if len(ids) != len(text_ids) + 1 or ids[:-1] != text_ids:
return None
return ids[-1]
def _encode_labels(
tokenizer: Any,
prompt: str,
prompt_ids: List[int],
labels: List[str],
added_tokens: Dict[int, str],
) -> List[int]:
"""Check that each label adds exactly one distinct token after the prompt."""
text, text_ids, _ = label_context(tokenizer, prompt, prompt_ids, added_tokens)
label_ids = []
for label in labels:
token_id = label_token_id(tokenizer, text, text_ids, label)
if token_id is None or token_id in label_ids:
raise ValueError(
f"the answer label {label!r} is not one distinct token after the "
"chat prompt for this tokenizer, so this model is not supported"
)
label_ids.append(token_id)
return label_ids
def label_mass(token_logprobs: List[float]) -> float:
return math.fsum(math.exp(logprob) for logprob in token_logprobs)
def _build_answer(
question: DecisionQuestion,
view: QuestionView,
probabilities: List[float],
token_logprobs: List[float],
) -> DecisionAnswer:
names = view.names
value = {}
if view.kind == "choice":
value["choice"] = names[probabilities.index(max(probabilities))]
elif view.kind == "score":
value["score"] = math.fsum(i * p for i, p in enumerate(probabilities))
return DecisionAnswer(
type=question.type,
probabilities=dict(zip(names, probabilities)),
label_mass=label_mass(token_logprobs),
**value,
)
@@ -0,0 +1,150 @@
"""Pydantic models for the System One decision API.
Follows the published System One OpenAPI 0.2.0 request and response shapes:
a state, a map of noul, choice, and score questions keyed by caller ids, and
one answer per question id.
"""
from typing import Annotated, Any, Dict, List, Literal, Optional, Union
from pydantic import (
BaseModel,
ConfigDict,
Field,
NonNegativeInt,
ValidationInfo,
field_validator,
model_validator,
)
from pydantic.json_schema import SkipJsonSchema
from sglang.srt.entrypoints.openai.protocol import (
DecisionText,
RequiredDecisionText,
check_option_names,
is_blank_decision_text,
)
# The documented maximum. Options beyond 26 get two-letter labels, checked per request.
MAX_CHOICE_OPTIONS = 255
# Levels are labeled 0 to 9, so at most 10.
MAX_SCORE_LEVELS = 10
class _Question(BaseModel):
# Misspelled keys inside a question would otherwise answer a different question.
model_config = ConfigDict(extra="forbid")
instructions: Optional[DecisionText] = None
class SystemOneNoulCriteria(BaseModel):
model_config = ConfigDict(extra="forbid")
true: Optional[DecisionText] = None
false: Optional[DecisionText] = None
class SystemOneNoulQuestion(_Question):
type: Literal["noul"]
criteria: Optional[SystemOneNoulCriteria] = None
@model_validator(mode="after")
def _asks_something(self):
criteria = self.criteria or SystemOneNoulCriteria()
if all(
is_blank_decision_text(value)
for value in (self.instructions, criteria.true, criteria.false)
):
raise ValueError(
"a noul question needs instructions or a true or false "
"description to decide on"
)
return self
class SystemOneChoiceQuestion(_Question):
type: Literal["choice"]
criteria: Dict[str, Optional[DecisionText]] = Field(
min_length=1, max_length=MAX_CHOICE_OPTIONS
)
@field_validator("criteria")
@classmethod
def _option_names_distinct(cls, criteria):
check_option_names(criteria)
return criteria
class SystemOneScoreQuestion(_Question):
type: Literal["score"]
criteria: List[RequiredDecisionText] = Field(
min_length=1, max_length=MAX_SCORE_LEVELS
)
SystemOneQuestion = Annotated[
Union[SystemOneNoulQuestion, SystemOneChoiceQuestion, SystemOneScoreQuestion],
Field(discriminator="type"),
]
class SystemOneRequest(BaseModel):
# Unknown top-level fields are ignored, as the published schema allows.
state: DecisionText
model: str
questions: Dict[str, SystemOneQuestion] = Field(min_length=1)
# SGLang extension, for chat templates whose reasoning toggle needs a kwarg.
chat_template_kwargs: Dict[str, Any] = Field(default_factory=dict)
# /v1/decisions fields, which would change the answers if honored or ignored.
# Declared only to refuse them by name, and hidden from the schema.
temperature: SkipJsonSchema[Any] = None
prompt_format_version: SkipJsonSchema[Any] = None
return_prompt_token_ids: SkipJsonSchema[Any] = None
@field_validator("temperature", "prompt_format_version", "return_prompt_token_ids")
@classmethod
def _refuse_decisions_fields(cls, value, info: ValidationInfo):
if value is not None:
raise ValueError(
f"{info.field_name} is not part of this API, use /v1/decisions for it"
)
return value
class SystemOneNoulAnswer(BaseModel):
type: Literal["noul"] = "noul"
noul: float
# Full-vocabulary probability of the answer labels, an SGLang extension.
x_label_mass: float
class SystemOneChoiceAnswer(BaseModel):
type: Literal["choice"] = "choice"
choice: str
confidence: float
probabilities: Dict[str, float]
x_label_mass: float
class SystemOneScoreAnswer(BaseModel):
type: Literal["score"] = "score"
score: float
confidence: float
legend: Dict[str, Any]
probabilities: Dict[str, float]
x_label_mass: float
class SystemOneUsage(BaseModel):
input_tokens: NonNegativeInt
output_tokens: NonNegativeInt = 0
class SystemOneResponse(BaseModel):
model: str
answers: Dict[
str, Union[SystemOneNoulAnswer, SystemOneChoiceAnswer, SystemOneScoreAnswer]
]
usage: SystemOneUsage
@@ -0,0 +1,258 @@
"""Handler for the System One compatible decision API, on the /v1/decisions scoring path."""
from __future__ import annotations
import math
import string
from typing import Any, Dict, Iterator, List, Optional, Tuple
import orjson
from fastapi import Request
from fastapi.responses import ORJSONResponse
from sglang.srt.entrypoints.openai.serving_decisions import (
OpenAIServingDecisions,
QuestionView,
default_labels,
label_context,
label_mass,
label_token_id,
render_text,
)
from sglang.srt.entrypoints.systemone.protocol import (
SystemOneChoiceAnswer,
SystemOneChoiceQuestion,
SystemOneNoulAnswer,
SystemOneNoulQuestion,
SystemOneQuestion,
SystemOneRequest,
SystemOneResponse,
SystemOneScoreAnswer,
SystemOneUsage,
)
# Beyond A to Z, every option gets a two-letter label, in this fixed order.
_PAIR_LABELS = [a + b for a in string.ascii_uppercase for b in string.ascii_uppercase]
class SystemOneServing(OpenAIServingDecisions):
"""Answers System One questions with the rendering, label checks, and scoring of /v1/decisions."""
route = "/v1/systemone"
def _request_id_prefix(self) -> str:
return "systemone-"
def _validate_request(self, request: SystemOneRequest) -> Optional[str]:
return self._validate_server(request.model) or self._validate_reasoning(
request.chat_template_kwargs
)
def _convert_to_internal_request(
self,
request: SystemOneRequest,
raw_request: Request = None,
) -> Tuple[
Iterator[Tuple[List[int], List[int]]],
Tuple[SystemOneRequest, List[QuestionView]],
]:
views = [_view(question) for question in request.questions.values()]
# Lazy, so the async handler can yield to other requests between questions.
return self._encoded_systemone_questions(request, views), (request, views)
def _encoded_systemone_questions(
self, request: SystemOneRequest, views: List[QuestionView]
) -> Iterator[Tuple[List[int], List[int]]]:
"""Prompt and label ids for each question, in request order."""
text = render_text(request.state)
chat_template_kwargs = self._chat_template_kwargs(request.chat_template_kwargs)
pair_labels = None
for question_id, view in zip(request.questions, views):
try:
labels = default_labels(view)
if view.kind == "choice" and len(view.names) > len(labels):
if pair_labels is None:
pair_labels = self._pair_labels(chat_template_kwargs)
if len(view.names) > len(pair_labels):
raise ValueError(
f"it has {len(view.names)} options, but the served "
"tokenizer and chat template can label at most "
f"{max(len(pair_labels), len(string.ascii_uppercase))}"
)
labels = pair_labels[: len(view.names)]
if view.kind == "score":
_check_legend(question_id, view)
encoded = self._encode_question(
text=text,
view=view,
labels=labels,
chat_template_kwargs=chat_template_kwargs,
)
except ValueError as e:
raise ValueError(f"question {question_id!r}: {e}") from e
yield encoded
def _pair_labels(self, chat_template_kwargs: Dict[str, Any]) -> List[str]:
"""Two-letter labels that are distinct single tokens at the answer position."""
if not self.added_tokens:
raise ValueError(
"more than 26 options needs added tokens the server can read "
"from the tokenizer, and the served tokenizer reports none"
)
tokenizer = self.tokenizer_manager.tokenizer
contexts = []
for message in ("x", "y"):
prompt = self._apply_chat_template(message, chat_template_kwargs)
prompt_ids = tokenizer.encode(prompt, add_special_tokens=False)
contexts.append(
label_context(tokenizer, prompt, prompt_ids, self.added_tokens)
)
(text, text_ids, shortcut), other = contexts
# Labels must be checked on text that excludes the message. Otherwise
# every label re-encodes the whole input on the event loop, which is
# unbounded work for hundreds of options.
if not (shortcut and other[2] and other[0] == text):
raise ValueError(
"more than 26 options needs an added token between the message "
"and the answer position, after which the text tokenizes the same on "
"its own, which this tokenizer and chat template do not provide"
)
labels, label_ids = [], set()
for label in _PAIR_LABELS:
token_id = label_token_id(tokenizer, text, text_ids, label)
if token_id is not None and token_id not in label_ids:
labels.append(label)
label_ids.add(token_id)
# Each final prompt is still checked as a whole in _encode_question.
return labels
async def _handle_non_streaming_request(
self,
adapted_request: Iterator[Tuple[List[int], List[int]]],
processed: Tuple[SystemOneRequest, List[QuestionView]],
raw_request: Request,
) -> ORJSONResponse:
request, views = processed
_, _, result = await self._score(
adapted_request=adapted_request, raw_request=raw_request
)
answers = {}
for i, question_id in enumerate(request.questions):
answers[question_id] = _answer(
view=views[i],
probabilities=result.scores[i],
mass=label_mass(result.token_logprobs[i]),
question_id=question_id,
)
response = SystemOneResponse(
# The served model answered, whatever name the request used.
model=self.tokenizer_manager.served_model_name,
answers=answers,
usage=SystemOneUsage(input_tokens=result.prompt_tokens),
)
return ORJSONResponse(content=response.model_dump())
def _view(question: SystemOneQuestion) -> QuestionView:
if isinstance(question, SystemOneChoiceQuestion):
return QuestionView(
kind="choice",
question=question.instructions,
names=list(question.criteria),
details=list(question.criteria.values()),
)
if isinstance(question, SystemOneNoulQuestion):
criteria = question.criteria
return QuestionView(
kind="yes_no",
question=question.instructions,
names=["yes", "no"],
details=[
criteria.true if criteria else None,
criteria.false if criteria else None,
],
)
return QuestionView(
kind="score",
question=question.instructions,
names=[str(level) for level in range(len(question.criteria))],
details=list(question.criteria),
)
def _answer(
view: QuestionView,
probabilities: List[float],
mass: float,
question_id: str,
):
if not all(math.isfinite(value) for value in [*probabilities, mass]):
# A server fault, reported as 500 rather than as a client error.
raise RuntimeError(f"question {question_id!r} scored non-finite values")
# Reported as scored, like /v1/decisions, and normalized only for confidence.
if view.kind == "yes_no":
return SystemOneNoulAnswer(noul=probabilities[0], x_label_mass=mass)
probabilities_by_name = dict(zip(view.names, probabilities))
if view.kind == "choice":
return SystemOneChoiceAnswer(
choice=view.names[probabilities.index(max(probabilities))],
confidence=_choice_confidence(_normalized(probabilities)),
probabilities=probabilities_by_name,
x_label_mass=mass,
)
return SystemOneScoreAnswer(
score=math.fsum(i * p for i, p in enumerate(probabilities)),
confidence=_score_confidence(_normalized(probabilities)),
legend=_legend(view),
probabilities=probabilities_by_name,
x_label_mass=mass,
)
def _legend(view: QuestionView) -> Dict[str, Any]:
return dict(zip(view.names, view.details))
def _check_legend(question_id: str, view: QuestionView) -> None:
"""Refuse, before scoring, levels that the response cannot echo in its legend."""
answer = SystemOneScoreAnswer(
score=0.0,
confidence=0.0,
legend=_legend(view),
probabilities={},
x_label_mass=0.0,
)
response = SystemOneResponse(
model="", answers={question_id: answer}, usage=SystemOneUsage(input_tokens=0)
)
try:
# The same encoder and options as the real response.
ORJSONResponse(content=response.model_dump())
except orjson.JSONEncodeError as e:
raise ValueError(f"a level cannot be returned in the legend: {e}") from e
def _normalized(probabilities: List[float]) -> List[float]:
total = math.fsum(probabilities)
if total <= 0:
return [1.0 / len(probabilities)] * len(probabilities)
return [p / total for p in probabilities]
def _choice_confidence(q: List[float]) -> float:
"""How far the top option stands above a uniform guess, from 0 to 1."""
n = len(q)
if n == 1:
return 1.0
return min(1.0, max(0.0, (n * max(q) - 1) / (n - 1)))
def _score_confidence(q: List[float]) -> float:
"""One minus the spread around the top level relative to a uniform spread, floored at 0."""
n = len(q)
if n == 1:
return 1.0
top = q.index(max(q))
spread = math.fsum(p * abs(i - top) for i, p in enumerate(q))
uniform_spread = math.fsum(abs(i - (n - 1) / 2) for i in range(n)) / n
return max(0.0, 1 - spread / uniform_spread)
@@ -155,15 +155,17 @@ def get_token_ids_logprobs_raw(
if stage == LogprobStage.DECODE:
for i, token_ids in enumerate(token_ids_logprobs_list):
if token_ids is None:
vals.append([])
idxs.append([])
# The CPU copy expects a tensor in every entry under no_copy_to_cpu.
# A new empty one does not keep the full logprobs alive like a view.
row = logprobs.new_empty((0,))
token_ids = []
else:
token_ids_tensor = torch.tensor(
token_ids, dtype=torch.long, pin_memory=pin_memory
).to(logprobs.device, non_blocking=True)
row = logprobs[i, token_ids_tensor]
vals.append(row if no_copy_to_cpu else row.tolist())
idxs.append(token_ids)
vals.append(row if no_copy_to_cpu else row.tolist())
idxs.append(token_ids)
else: # prefill
pt = 0
for i, (token_ids, pruned_len) in enumerate(
+22 -14
View File
@@ -2853,7 +2853,10 @@ class TokenizerManager(TokenizerControlMixin, TokenizerManagerScoreMixin):
)
if top_logprobs_num > 0:
if len(recv_obj.input_top_logprobs_val) > 0:
if (
recv_obj.input_top_logprobs_val is not None
and len(recv_obj.input_top_logprobs_val) > 0
):
state.input_top_logprobs_val.extend(
recv_obj.input_top_logprobs_val[recv_obj_index]
)
@@ -2869,27 +2872,32 @@ class TokenizerManager(TokenizerControlMixin, TokenizerManagerScoreMixin):
recv_obj.input_top_logprobs_idx_flat[recv_obj_index],
recv_obj.input_top_logprobs_flat_null_prefix[recv_obj_index],
)
state.output_top_logprobs_val.extend(
recv_obj.output_top_logprobs_val[recv_obj_index]
)
state.output_top_logprobs_idx.extend(
recv_obj.output_top_logprobs_idx[recv_obj_index]
)
if recv_obj.output_top_logprobs_val is not None:
state.output_top_logprobs_val.extend(
recv_obj.output_top_logprobs_val[recv_obj_index]
)
state.output_top_logprobs_idx.extend(
recv_obj.output_top_logprobs_idx[recv_obj_index]
)
if token_ids_logprob is not None:
if len(recv_obj.input_token_ids_logprobs_val) > 0:
if (
recv_obj.input_token_ids_logprobs_val is not None
and len(recv_obj.input_token_ids_logprobs_val) > 0
):
state.input_token_ids_logprobs_val.extend(
recv_obj.input_token_ids_logprobs_val[recv_obj_index]
)
state.input_token_ids_logprobs_idx.extend(
recv_obj.input_token_ids_logprobs_idx[recv_obj_index]
)
state.output_token_ids_logprobs_val.extend(
recv_obj.output_token_ids_logprobs_val[recv_obj_index]
)
state.output_token_ids_logprobs_idx.extend(
recv_obj.output_token_ids_logprobs_idx[recv_obj_index]
)
if recv_obj.output_token_ids_logprobs_val is not None:
state.output_token_ids_logprobs_val.extend(
recv_obj.output_token_ids_logprobs_val[recv_obj_index]
)
state.output_token_ids_logprobs_idx.extend(
recv_obj.output_token_ids_logprobs_idx[recv_obj_index]
)
self.add_logprob_to_meta_info(
meta_info,